feat(ingest): 진행 로그 개선 — 파일명/phase/heartbeat/slowest 요약
OCR/caption 켜진 볼트 ingest 가 중간부터 느릴 때 TTY 진행바가 파일명·phase·
모델·경과시간을 안 보여 "멈춤"처럼 보이던 문제 해결.
- 신규 wire AssetPhase{idx,total,phase,model} + AssetTimings.ocr_ms/caption_ms
(additive, ingest_progress.v1 유지)
- app: apply_ocr/apply_caption/embed 진입 시 AssetPhase emit + ocr/caption 시간 측정
- cli: TTY 진행바에 현재 파일명 + phase(model) + asset 경과초(heartbeat),
종료 시 최장 소요 파일 top-5 요약(quiet 여도 출력, --json 미출력)
- wire schema / README / HANDOFF / HOTFIXES 동기화, version 0.26.0 → 0.27.0
검증(리더): clippy 0, kebab-app/cli 61그룹·parse-image/tui 14그룹 0실패(-j8).
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -96,14 +96,33 @@ pub enum IngestEvent {
|
||||
/// `idx/total` while its per-chunk phases churn. `chunks` is the chunk
|
||||
/// count for asset `idx`.
|
||||
AssetChunked { idx: u32, total: u32, chunks: u32 },
|
||||
/// v0.26.0 (additive): emitted when an asset enters a *slow* internal
|
||||
/// phase, so the interactive progress bar can show **which** phase
|
||||
/// (and which model) is currently running instead of looking frozen.
|
||||
/// `phase` ∈ {`"ocr"`, `"caption"`, `"embed"`}; short phases
|
||||
/// (parse / chunk / store) are intentionally *not* emitted to avoid
|
||||
/// noise. `model` is the model performing the phase — the vision LLM
|
||||
/// id for `ocr` / `caption`, the embedder `model_id` for `embed`
|
||||
/// (`None` when the phase runs without a configured model, e.g. embed
|
||||
/// with no embedder wired). Emitted once per (asset, phase); no
|
||||
/// throttle needed (low frequency). Wire v1 consumers that predate
|
||||
/// this variant simply ignore the unknown `asset_phase` kind.
|
||||
AssetPhase {
|
||||
idx: u32,
|
||||
total: u32,
|
||||
phase: String,
|
||||
model: Option<String>,
|
||||
},
|
||||
/// v0.24.0 (additive): per-phase wall-clock (milliseconds) for asset
|
||||
/// `idx`, emitted once the asset's markdown pipeline finishes. Lets a
|
||||
/// user see *where* the time went (parse / chunk / embed / store)
|
||||
/// without parsing logs. Only the markdown path emits this; the
|
||||
/// image / PDF paths surface `AssetChunked` but skip phase timing (their
|
||||
/// phase shapes differ — OCR / caption). `expansion_ms` is retained for
|
||||
/// wire compatibility but is always 0 since doc-side expansion was
|
||||
/// removed (HOTFIXES 2026-06-03).
|
||||
/// `idx`, emitted once the asset's pipeline finishes. Lets a user see
|
||||
/// *where* the time went (parse / chunk / ocr / caption / embed /
|
||||
/// store) without parsing logs. The markdown path leaves `ocr_ms` /
|
||||
/// `caption_ms` at 0 (no image analysis); the image / PDF paths fill
|
||||
/// them so the slowest-asset summary attributes vision-model time
|
||||
/// correctly. `expansion_ms` is retained for wire compatibility but is
|
||||
/// always 0 since doc-side expansion was removed (HOTFIXES 2026-06-03).
|
||||
/// `ocr_ms` / `caption_ms` (v0.26.0) are additive with serde default 0
|
||||
/// so pre-v0.26.0 consumers deserialize cleanly.
|
||||
AssetTimings {
|
||||
idx: u32,
|
||||
total: u32,
|
||||
@@ -112,6 +131,10 @@ pub enum IngestEvent {
|
||||
expansion_ms: u64,
|
||||
embed_ms: u64,
|
||||
store_ms: u64,
|
||||
#[serde(default)]
|
||||
ocr_ms: u64,
|
||||
#[serde(default)]
|
||||
caption_ms: u64,
|
||||
},
|
||||
/// Run finished normally. `counts` is the final aggregate.
|
||||
Completed { counts: AggregateCounts },
|
||||
@@ -261,19 +284,23 @@ mod tests {
|
||||
expansion_ms: 45_000,
|
||||
embed_ms: 800,
|
||||
store_ms: 20,
|
||||
ocr_ms: 1_200,
|
||||
caption_ms: 3_400,
|
||||
};
|
||||
let v = serde_json::to_value(&ev).unwrap();
|
||||
assert_eq!(
|
||||
v.get("kind").and_then(|s| s.as_str()),
|
||||
Some("asset_timings")
|
||||
);
|
||||
// All five phase fields are present (plain u64, always serialized).
|
||||
// All phase fields are present (plain u64, always serialized).
|
||||
for (field, want) in [
|
||||
("parse_ms", 12u64),
|
||||
("chunk_ms", 3),
|
||||
("expansion_ms", 45_000),
|
||||
("embed_ms", 800),
|
||||
("store_ms", 20),
|
||||
("ocr_ms", 1_200),
|
||||
("caption_ms", 3_400),
|
||||
] {
|
||||
assert_eq!(
|
||||
v.get(field).and_then(serde_json::Value::as_u64),
|
||||
@@ -283,6 +310,64 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn asset_timings_ocr_caption_default_to_zero_for_legacy_wire() {
|
||||
// v0.26.0 additive: a pre-v0.26.0 wire payload omits ocr_ms /
|
||||
// caption_ms; serde `default` must fill 0 so old producers stay
|
||||
// compatible.
|
||||
let legacy = serde_json::json!({
|
||||
"kind": "asset_timings",
|
||||
"idx": 1, "total": 1,
|
||||
"parse_ms": 5, "chunk_ms": 2, "expansion_ms": 0,
|
||||
"embed_ms": 10, "store_ms": 3
|
||||
});
|
||||
let ev: IngestEvent = serde_json::from_value(legacy).unwrap();
|
||||
match ev {
|
||||
IngestEvent::AssetTimings {
|
||||
ocr_ms,
|
||||
caption_ms,
|
||||
embed_ms,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(ocr_ms, 0);
|
||||
assert_eq!(caption_ms, 0);
|
||||
assert_eq!(embed_ms, 10);
|
||||
}
|
||||
other => panic!("unexpected event: {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn asset_phase_serializes_with_discriminator() {
|
||||
// v0.26.0 additive variant — `kind` must be snake_case
|
||||
// `asset_phase`, `phase` is the slow-phase label, `model` the
|
||||
// model id (nullable).
|
||||
let ev = IngestEvent::AssetPhase {
|
||||
idx: 4,
|
||||
total: 12,
|
||||
phase: "ocr".into(),
|
||||
model: Some("gemma4:e4b".into()),
|
||||
};
|
||||
let v = serde_json::to_value(&ev).unwrap();
|
||||
assert_eq!(v.get("kind").and_then(|s| s.as_str()), Some("asset_phase"));
|
||||
assert_eq!(v.get("idx").and_then(serde_json::Value::as_u64), Some(4));
|
||||
assert_eq!(v.get("phase").and_then(|s| s.as_str()), Some("ocr"));
|
||||
assert_eq!(v.get("model").and_then(|s| s.as_str()), Some("gemma4:e4b"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn asset_phase_model_none_serializes_as_null() {
|
||||
let ev = IngestEvent::AssetPhase {
|
||||
idx: 1,
|
||||
total: 1,
|
||||
phase: "embed".into(),
|
||||
model: None,
|
||||
};
|
||||
let v = serde_json::to_value(&ev).unwrap();
|
||||
assert_eq!(v.get("phase").and_then(|s| s.as_str()), Some("embed"));
|
||||
assert!(v.get("model").is_some_and(serde_json::Value::is_null));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ingest_event_completed_has_counts() {
|
||||
let ev = IngestEvent::Completed {
|
||||
|
||||
@@ -1350,6 +1350,17 @@ fn ingest_one_asset(
|
||||
let store_ms = u64::try_from(t_store.elapsed().as_millis()).unwrap_or(u64::MAX);
|
||||
|
||||
// Embed + vector upsert (only when both sides are configured).
|
||||
// v0.26.0: surface the embed phase + model so a long embed run reads as
|
||||
// "embedding(<model>)…" rather than a frozen bar (markdown path too).
|
||||
crate::ingest_progress::emit(
|
||||
progress,
|
||||
crate::ingest_progress::IngestEvent::AssetPhase {
|
||||
idx,
|
||||
total,
|
||||
phase: "embed".to_string(),
|
||||
model: embedder.map(|e| e.model_id().0),
|
||||
},
|
||||
);
|
||||
let t_embed = std::time::Instant::now();
|
||||
// Stale-vector purge is LanceDB I/O, so it belongs to the embed/vector
|
||||
// phase — not the SQLite `store` phase. Keeping it here makes `store_ms`
|
||||
@@ -1414,7 +1425,8 @@ fn ingest_one_asset(
|
||||
|
||||
let embed_ms = u64::try_from(t_embed.elapsed().as_millis()).unwrap_or(u64::MAX);
|
||||
|
||||
// v0.24.0: phase-timing breakdown for this asset (markdown path only).
|
||||
// v0.24.0: phase-timing breakdown for this asset (markdown path).
|
||||
// ocr_ms / caption_ms are 0 — markdown has no image-analysis phases.
|
||||
crate::ingest_progress::emit(
|
||||
progress,
|
||||
crate::ingest_progress::IngestEvent::AssetTimings {
|
||||
@@ -1425,6 +1437,8 @@ fn ingest_one_asset(
|
||||
expansion_ms,
|
||||
embed_ms,
|
||||
store_ms,
|
||||
ocr_ms: 0,
|
||||
caption_ms: 0,
|
||||
},
|
||||
);
|
||||
|
||||
@@ -1545,9 +1559,11 @@ fn ingest_one_image_asset(
|
||||
workspace_root: &workspace_root,
|
||||
config: &extract_config,
|
||||
};
|
||||
let t_parse = std::time::Instant::now();
|
||||
let mut canonical = app
|
||||
.extract_for(&asset.media_type, &ctx, &bytes)
|
||||
.context("kb-app::extract_for (image)")?;
|
||||
let parse_ms = u64::try_from(t_parse.elapsed().as_millis()).unwrap_or(u64::MAX);
|
||||
|
||||
// 2 + 3. Apply OCR / caption when their adapters exist. Both are
|
||||
// Lenient — failure is captured into Provenance Warning,
|
||||
@@ -1562,44 +1578,74 @@ fn ingest_one_image_asset(
|
||||
let lang_hint = lang_hint_from_doc(&canonical);
|
||||
let now = time::OffsetDateTime::now_utc();
|
||||
let mut warning_notes: Vec<String> = Vec::new();
|
||||
// v0.26.0: vision phases (OCR / caption) are the usual bottleneck on an
|
||||
// image-heavy vault and emitted no progress before — so the bar looked
|
||||
// frozen. Surface each as an `AssetPhase` and measure its wall-clock for
|
||||
// the slowest-asset summary.
|
||||
let mut ocr_ms = 0_u64;
|
||||
let mut caption_ms = 0_u64;
|
||||
match canonical.blocks.first_mut() {
|
||||
Some(Block::ImageRef(block)) => {
|
||||
if let Some(engine) = ocr_engine
|
||||
&& let Err(e) = apply_ocr(
|
||||
if let Some(engine) = ocr_engine {
|
||||
crate::ingest_progress::emit(
|
||||
progress,
|
||||
crate::ingest_progress::IngestEvent::AssetPhase {
|
||||
idx,
|
||||
total,
|
||||
phase: "ocr".to_string(),
|
||||
model: Some(engine.model().to_string()),
|
||||
},
|
||||
);
|
||||
let t_ocr = std::time::Instant::now();
|
||||
let res = apply_ocr(
|
||||
engine,
|
||||
&bytes,
|
||||
block,
|
||||
lang_hint.as_ref(),
|
||||
&mut canonical.provenance.events,
|
||||
)
|
||||
{
|
||||
record_image_analysis_failure(
|
||||
asset,
|
||||
&mut canonical.provenance.events,
|
||||
&mut warning_notes,
|
||||
"OcrFailed",
|
||||
e,
|
||||
now,
|
||||
);
|
||||
ocr_ms = u64::try_from(t_ocr.elapsed().as_millis()).unwrap_or(u64::MAX);
|
||||
if let Err(e) = res {
|
||||
record_image_analysis_failure(
|
||||
asset,
|
||||
&mut canonical.provenance.events,
|
||||
&mut warning_notes,
|
||||
"OcrFailed",
|
||||
e,
|
||||
now,
|
||||
);
|
||||
}
|
||||
}
|
||||
if let Some(llm) = caption_llm
|
||||
&& let Err(e) = apply_caption(
|
||||
if let Some(llm) = caption_llm {
|
||||
crate::ingest_progress::emit(
|
||||
progress,
|
||||
crate::ingest_progress::IngestEvent::AssetPhase {
|
||||
idx,
|
||||
total,
|
||||
phase: "caption".to_string(),
|
||||
model: Some(llm.model_ref().id),
|
||||
},
|
||||
);
|
||||
let t_caption = std::time::Instant::now();
|
||||
let res = apply_caption(
|
||||
llm,
|
||||
&bytes,
|
||||
block,
|
||||
lang_hint.as_ref(),
|
||||
&app.config,
|
||||
&mut canonical.provenance.events,
|
||||
)
|
||||
{
|
||||
record_image_analysis_failure(
|
||||
asset,
|
||||
&mut canonical.provenance.events,
|
||||
&mut warning_notes,
|
||||
"CaptionFailed",
|
||||
e,
|
||||
now,
|
||||
);
|
||||
caption_ms = u64::try_from(t_caption.elapsed().as_millis()).unwrap_or(u64::MAX);
|
||||
if let Err(e) = res {
|
||||
record_image_analysis_failure(
|
||||
asset,
|
||||
&mut canonical.provenance.events,
|
||||
&mut warning_notes,
|
||||
"CaptionFailed",
|
||||
e,
|
||||
now,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
// P6-1 contract: image documents always have exactly one
|
||||
@@ -1634,12 +1680,13 @@ fn ingest_one_image_asset(
|
||||
// `Block::ImageRef` arm already produces a single chunk per
|
||||
// image (P1-5). The chunk text now follows the (β) plain-concat
|
||||
// contract per the kebab-chunk render_block_text update.
|
||||
let t_chunk = std::time::Instant::now();
|
||||
let chunks = MdHeadingV1Chunker
|
||||
.chunk(&canonical, chunk_policy)
|
||||
.context("kb-chunk::MdHeadingV1Chunker::chunk (image)")?;
|
||||
let chunk_ms = u64::try_from(t_chunk.elapsed().as_millis()).unwrap_or(u64::MAX);
|
||||
|
||||
// v0.24.0: surface chunk count for the image path too (phase timing is
|
||||
// markdown-only, but AssetChunked is consistent across media).
|
||||
// v0.24.0: surface chunk count for the image path too.
|
||||
crate::ingest_progress::emit(
|
||||
progress,
|
||||
crate::ingest_progress::IngestEvent::AssetChunked {
|
||||
@@ -1656,6 +1703,7 @@ fn ingest_one_image_asset(
|
||||
if let Some(emb) = embedder {
|
||||
canonical.last_embedding_version = Some(emb.model_version());
|
||||
}
|
||||
let t_store = std::time::Instant::now();
|
||||
purge_vector_orphans_for_workspace_path(app, asset, vector_store)?;
|
||||
app.sqlite
|
||||
.put_asset_with_bytes(asset, &bytes)
|
||||
@@ -1669,7 +1717,18 @@ fn ingest_one_image_asset(
|
||||
app.sqlite
|
||||
.put_chunks(&canonical.doc_id, &chunks)
|
||||
.context("DocumentStore::put_chunks (image)")?;
|
||||
let store_ms = u64::try_from(t_store.elapsed().as_millis()).unwrap_or(u64::MAX);
|
||||
|
||||
crate::ingest_progress::emit(
|
||||
progress,
|
||||
crate::ingest_progress::IngestEvent::AssetPhase {
|
||||
idx,
|
||||
total,
|
||||
phase: "embed".to_string(),
|
||||
model: embedder.map(|e| e.model_id().0),
|
||||
},
|
||||
);
|
||||
let t_embed = std::time::Instant::now();
|
||||
if let (Some(emb), Some(vec_store)) = (embedder, vector_store)
|
||||
&& !chunks.is_empty()
|
||||
{
|
||||
@@ -1710,6 +1769,25 @@ fn ingest_one_image_asset(
|
||||
.upsert(&records)
|
||||
.context("VectorStore::upsert (image)")?;
|
||||
}
|
||||
let embed_ms = u64::try_from(t_embed.elapsed().as_millis()).unwrap_or(u64::MAX);
|
||||
|
||||
// v0.26.0: per-phase timing for the image path — ocr_ms / caption_ms
|
||||
// carry the vision-model cost so the slowest-asset summary attributes
|
||||
// an image-heavy run's bottleneck correctly.
|
||||
crate::ingest_progress::emit(
|
||||
progress,
|
||||
crate::ingest_progress::IngestEvent::AssetTimings {
|
||||
idx,
|
||||
total,
|
||||
parse_ms,
|
||||
chunk_ms,
|
||||
expansion_ms: 0,
|
||||
embed_ms,
|
||||
store_ms,
|
||||
ocr_ms,
|
||||
caption_ms,
|
||||
},
|
||||
);
|
||||
|
||||
let kind = if existing_doc_ids.contains(&canonical.doc_id.0) {
|
||||
kebab_core::IngestItemKind::Updated
|
||||
@@ -2053,9 +2131,11 @@ fn ingest_one_pdf_asset(
|
||||
workspace_root: &workspace_root,
|
||||
config: &extract_config,
|
||||
};
|
||||
let t_parse = std::time::Instant::now();
|
||||
let mut canonical = app
|
||||
.extract_for(&asset.media_type, &ctx, &bytes)
|
||||
.context("kb-app::extract_for (pdf)")?;
|
||||
let parse_ms = u64::try_from(t_parse.elapsed().as_millis()).unwrap_or(u64::MAX);
|
||||
|
||||
// v0.20 sub-item 1: post-extract OCR enrichment (PR #187 registry
|
||||
// dispatch invariant 보존 — extract_for 가 normal entry).
|
||||
@@ -2191,9 +2271,11 @@ fn ingest_one_pdf_asset(
|
||||
// validates every block carries `SourceSpan::Page`; failure here
|
||||
// means the parser drifted from its contract.
|
||||
let chunker = PdfPageV1Chunker;
|
||||
let t_chunk = std::time::Instant::now();
|
||||
let chunks = chunker
|
||||
.chunk(&canonical, chunk_policy)
|
||||
.context("kb-chunk::PdfPageV1Chunker::chunk")?;
|
||||
let chunk_ms = u64::try_from(t_chunk.elapsed().as_millis()).unwrap_or(u64::MAX);
|
||||
|
||||
// v0.24.0: surface chunk count for the PDF path too.
|
||||
crate::ingest_progress::emit(
|
||||
@@ -2212,6 +2294,7 @@ fn ingest_one_pdf_asset(
|
||||
canonical.last_embedding_version = Some(emb.model_version());
|
||||
}
|
||||
|
||||
let t_store = std::time::Instant::now();
|
||||
purge_vector_orphans_for_workspace_path(app, asset, vector_store)?;
|
||||
app.sqlite
|
||||
.put_asset_with_bytes(asset, &bytes)
|
||||
@@ -2225,7 +2308,18 @@ fn ingest_one_pdf_asset(
|
||||
app.sqlite
|
||||
.put_chunks(&canonical.doc_id, &chunks)
|
||||
.context("DocumentStore::put_chunks (pdf)")?;
|
||||
let store_ms = u64::try_from(t_store.elapsed().as_millis()).unwrap_or(u64::MAX);
|
||||
|
||||
crate::ingest_progress::emit(
|
||||
progress,
|
||||
crate::ingest_progress::IngestEvent::AssetPhase {
|
||||
idx,
|
||||
total,
|
||||
phase: "embed".to_string(),
|
||||
model: embedder.map(|e| e.model_id().0),
|
||||
},
|
||||
);
|
||||
let t_embed = std::time::Instant::now();
|
||||
if let (Some(emb), Some(vec_store)) = (embedder, vector_store)
|
||||
&& !chunks.is_empty()
|
||||
{
|
||||
@@ -2264,6 +2358,25 @@ fn ingest_one_pdf_asset(
|
||||
.upsert(&records)
|
||||
.context("VectorStore::upsert (pdf)")?;
|
||||
}
|
||||
let embed_ms = u64::try_from(t_embed.elapsed().as_millis()).unwrap_or(u64::MAX);
|
||||
|
||||
// v0.26.0: per-phase timing for the PDF path. `ocr_ms` reuses the
|
||||
// page-OCR total already computed above so a scanned-PDF run's OCR cost
|
||||
// shows up in the slowest-asset summary; caption is markdown/image-only.
|
||||
crate::ingest_progress::emit(
|
||||
progress,
|
||||
crate::ingest_progress::IngestEvent::AssetTimings {
|
||||
idx,
|
||||
total,
|
||||
parse_ms,
|
||||
chunk_ms,
|
||||
expansion_ms: 0,
|
||||
embed_ms,
|
||||
store_ms,
|
||||
ocr_ms: pdf_ocr_ms_total.unwrap_or(0),
|
||||
caption_ms: 0,
|
||||
},
|
||||
);
|
||||
|
||||
let kind = if existing_doc_ids.contains(&canonical.doc_id.0) {
|
||||
kebab_core::IngestItemKind::Updated
|
||||
|
||||
@@ -19,16 +19,23 @@
|
||||
//! `Sender` end is dropped (i.e. when `ingest_with_config_progress`
|
||||
//! returns).
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::io::{IsTerminal, Write};
|
||||
use std::sync::mpsc::Receiver;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Instant;
|
||||
|
||||
use indicatif::{ProgressBar, ProgressDrawTarget, ProgressStyle};
|
||||
use indicatif::{ProgressBar, ProgressDrawTarget, ProgressState, ProgressStyle};
|
||||
use kebab_app::IngestEvent;
|
||||
use time::OffsetDateTime;
|
||||
use time::format_description::well_known::Rfc3339;
|
||||
|
||||
use crate::wire;
|
||||
|
||||
/// v0.26.0: number of slowest assets surfaced in the end-of-run summary.
|
||||
/// Constant for now (spec defers the config knob).
|
||||
const SLOWEST_TOP_N: usize = 5;
|
||||
|
||||
/// Rendering mode for `ProgressDisplay`. The mode is fixed at
|
||||
/// construction — each `kebab ingest` invocation is a single mode
|
||||
/// (chosen from `--json` plus `IsTerminal` detection).
|
||||
@@ -65,11 +72,33 @@ impl ProgressMode {
|
||||
pub struct ProgressDisplay {
|
||||
mode: ProgressMode,
|
||||
bar: Option<ProgressBar>,
|
||||
/// v0.26.0 heartbeat: start `Instant` of the asset currently in
|
||||
/// flight, shared with the bar's steady-tick custom template key so
|
||||
/// the `(Ns)` elapsed counter advances *between* events (the drain
|
||||
/// loop blocks on `recv()`, so without the ticker the counter would
|
||||
/// freeze). `None` while scanning / between assets / after completion.
|
||||
asset_start: Arc<Mutex<Option<Instant>>>,
|
||||
/// v0.26.0: workspace path of the asset currently in flight — set on
|
||||
/// `AssetStarted`, reused by `AssetPhase` to render `{path} · {phase}…`.
|
||||
current_path: Option<String>,
|
||||
/// v0.26.0 slowest summary: idx → path, captured from `AssetStarted`
|
||||
/// so `AssetTimings` (which only carries `idx`) can name the asset.
|
||||
asset_paths: HashMap<u32, String>,
|
||||
/// v0.26.0 slowest summary: (path, total_ms) per asset that reported
|
||||
/// `AssetTimings`. Sorted + truncated to top-N on `Completed`.
|
||||
timings: Vec<(String, u64)>,
|
||||
}
|
||||
|
||||
impl ProgressDisplay {
|
||||
pub fn new(mode: ProgressMode) -> Self {
|
||||
Self { mode, bar: None }
|
||||
Self {
|
||||
mode,
|
||||
bar: None,
|
||||
asset_start: Arc::new(Mutex::new(None)),
|
||||
current_path: None,
|
||||
asset_paths: HashMap::new(),
|
||||
timings: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Block until `rx` returns `Err` (sender dropped). Renders one
|
||||
@@ -120,15 +149,43 @@ impl ProgressDisplay {
|
||||
}
|
||||
IngestEvent::ScanCompleted { total } => {
|
||||
if let Some(bar) = self.bar.as_mut() {
|
||||
bar.disable_steady_tick();
|
||||
bar.set_length(u64::from(*total));
|
||||
bar.set_position(0);
|
||||
// v0.26.0: a custom `{asset_elapsed}` key reads the shared
|
||||
// per-asset start `Instant` and appends ` (Ns)`. Combined
|
||||
// with the steady tick below, the elapsed counter advances
|
||||
// even while the drain loop is blocked on `recv()` waiting
|
||||
// for the next (possibly very slow) phase event.
|
||||
let asset_start = Arc::clone(&self.asset_start);
|
||||
bar.set_style(
|
||||
ProgressStyle::with_template("ingest [{bar:30}] {pos}/{len} {wide_msg}")
|
||||
.unwrap()
|
||||
.progress_chars("=> "),
|
||||
ProgressStyle::with_template(
|
||||
"ingest [{bar:30}] {pos}/{len} {wide_msg}{asset_elapsed}",
|
||||
)
|
||||
.unwrap()
|
||||
.with_key(
|
||||
"asset_elapsed",
|
||||
move |_: &ProgressState, w: &mut dyn std::fmt::Write| {
|
||||
if let Ok(guard) = asset_start.lock()
|
||||
&& let Some(started) = *guard
|
||||
{
|
||||
let secs = started.elapsed().as_secs();
|
||||
// Only show once the asset has been running
|
||||
// a moment — avoids `(0s)` flicker on fast
|
||||
// assets.
|
||||
if secs >= 1 {
|
||||
let _ = write!(w, " ({secs}s)");
|
||||
}
|
||||
}
|
||||
},
|
||||
)
|
||||
.progress_chars("=> "),
|
||||
);
|
||||
bar.set_message("");
|
||||
if tty && !quiet {
|
||||
bar.enable_steady_tick(std::time::Duration::from_secs(1));
|
||||
} else {
|
||||
bar.disable_steady_tick();
|
||||
}
|
||||
}
|
||||
if !tty && !quiet {
|
||||
let mut err = std::io::stderr().lock();
|
||||
@@ -141,11 +198,22 @@ impl ProgressDisplay {
|
||||
path,
|
||||
media,
|
||||
} => {
|
||||
// v0.26.0: remember the path so AssetPhase can render it and
|
||||
// the slowest summary (keyed by idx in AssetTimings) can name
|
||||
// the asset.
|
||||
self.current_path = Some(path.clone());
|
||||
self.asset_paths.insert(*idx, path.clone());
|
||||
// v0.26.0: (re)start the per-asset heartbeat clock.
|
||||
if let Ok(mut guard) = self.asset_start.lock() {
|
||||
*guard = Some(Instant::now());
|
||||
}
|
||||
if let Some(bar) = self.bar.as_ref() {
|
||||
// One draw per file: position only. set_message() would
|
||||
// trigger a second independent draw and pollute TTY scrollback.
|
||||
// Filename is visible in the non-TTY plain-line path below.
|
||||
bar.set_position(u64::from(idx.saturating_sub(1)));
|
||||
// v0.26.0: show the current filename on the bar (TTY).
|
||||
// Previously position-only — the interactive user couldn't
|
||||
// tell which file was in flight. The steady tick redraws
|
||||
// in place, so this no longer pollutes scrollback.
|
||||
bar.set_message(abbreviate_path(path));
|
||||
}
|
||||
if !tty && !quiet {
|
||||
let mut err = std::io::stderr().lock();
|
||||
@@ -154,8 +222,35 @@ impl ProgressDisplay {
|
||||
}
|
||||
IngestEvent::AssetFinished { .. } => {
|
||||
// Position is advanced in AssetStarted; bar.finish_and_clear()
|
||||
// in Completed handles the final state. No per-asset bar update
|
||||
// here avoids the duplicate-frame artifact in TTY scrollback.
|
||||
// in Completed handles the final state. v0.26.0: stop the
|
||||
// heartbeat clock so the bar doesn't show a stale `(Ns)` in the
|
||||
// gap before the next AssetStarted.
|
||||
if let Ok(mut guard) = self.asset_start.lock() {
|
||||
*guard = None;
|
||||
}
|
||||
self.current_path = None;
|
||||
}
|
||||
// v0.26.0: an asset entered a slow internal phase (ocr / caption /
|
||||
// embed). Surface which phase + model is running so a multi-second
|
||||
// vision-model call no longer looks frozen.
|
||||
IngestEvent::AssetPhase {
|
||||
idx,
|
||||
total,
|
||||
phase,
|
||||
model,
|
||||
} => {
|
||||
let label = match model {
|
||||
Some(m) => format!("{phase}({m})"),
|
||||
None => phase.clone(),
|
||||
};
|
||||
if let Some(bar) = self.bar.as_ref() {
|
||||
let path = self.current_path.as_deref().unwrap_or("");
|
||||
bar.set_message(format!("{} · {label}…", abbreviate_path(path)));
|
||||
}
|
||||
if !tty && !quiet {
|
||||
let mut err = std::io::stderr().lock();
|
||||
let _ = writeln!(err, "ingest: {idx}/{total} · {label}…");
|
||||
}
|
||||
}
|
||||
// v0.24.0: asset-internal phase visibility. AssetChunked uses the
|
||||
// bar *message* (live sub-progress for the current asset) —
|
||||
@@ -172,31 +267,50 @@ impl ProgressDisplay {
|
||||
}
|
||||
}
|
||||
IngestEvent::AssetTimings {
|
||||
idx,
|
||||
parse_ms,
|
||||
chunk_ms,
|
||||
embed_ms,
|
||||
store_ms,
|
||||
ocr_ms,
|
||||
caption_ms,
|
||||
..
|
||||
} => {
|
||||
// v0.26.0: accumulate (path, total_ms) for the slowest summary.
|
||||
// total = every measured phase (expansion_ms is always 0).
|
||||
let total_ms = parse_ms + chunk_ms + embed_ms + store_ms + ocr_ms + caption_ms;
|
||||
if let Some(path) = self.asset_paths.get(idx) {
|
||||
self.timings.push((path.clone(), total_ms));
|
||||
}
|
||||
if let Some(bar) = self.bar.as_ref() {
|
||||
bar.set_message("");
|
||||
}
|
||||
if !quiet {
|
||||
let mut err = std::io::stderr().lock();
|
||||
let _ = writeln!(
|
||||
err,
|
||||
" ⏱ parse {} · chunk {} · embed {} · store {}",
|
||||
fmt_ms(*parse_ms),
|
||||
fmt_ms(*chunk_ms),
|
||||
fmt_ms(*embed_ms),
|
||||
fmt_ms(*store_ms),
|
||||
);
|
||||
// v0.26.0: only print ocr / caption when they actually ran
|
||||
// (markdown leaves them 0) so the text path stays uncluttered.
|
||||
let mut parts = vec![
|
||||
format!("parse {}", fmt_ms(*parse_ms)),
|
||||
format!("chunk {}", fmt_ms(*chunk_ms)),
|
||||
];
|
||||
if *ocr_ms > 0 {
|
||||
parts.push(format!("ocr {}", fmt_ms(*ocr_ms)));
|
||||
}
|
||||
if *caption_ms > 0 {
|
||||
parts.push(format!("caption {}", fmt_ms(*caption_ms)));
|
||||
}
|
||||
parts.push(format!("embed {}", fmt_ms(*embed_ms)));
|
||||
parts.push(format!("store {}", fmt_ms(*store_ms)));
|
||||
let _ = writeln!(err, " ⏱ {}", parts.join(" · "));
|
||||
}
|
||||
}
|
||||
IngestEvent::Completed { counts } => {
|
||||
if let Some(bar) = self.bar.take() {
|
||||
bar.finish_and_clear();
|
||||
}
|
||||
if let Ok(mut guard) = self.asset_start.lock() {
|
||||
*guard = None;
|
||||
}
|
||||
// Always emit summary in both TTY and non-TTY (unless quiet).
|
||||
// Bug fix: previously TTY had no summary line after bar.finish_and_clear().
|
||||
if !quiet {
|
||||
@@ -206,6 +320,10 @@ impl ProgressDisplay {
|
||||
"ingest: complete (scanned={} new={} updated={} skipped={} errors={})",
|
||||
counts.scanned, counts.new, counts.updated, counts.skipped, counts.errors,
|
||||
);
|
||||
// v0.26.0: slowest-asset summary. Useful in both TTY and
|
||||
// non-TTY (it pinpoints the bottleneck file), so it prints
|
||||
// unless --quiet. --json mode never reaches here (emit_json).
|
||||
let _ = write_slowest_summary(&mut err, &self.timings, SLOWEST_TOP_N);
|
||||
}
|
||||
}
|
||||
IngestEvent::Aborted { counts } => {
|
||||
@@ -286,6 +404,48 @@ fn fmt_ms(ms: u64) -> String {
|
||||
}
|
||||
}
|
||||
|
||||
/// v0.26.0: shorten an over-long workspace path for the progress-bar
|
||||
/// message so the live `(Ns)` heartbeat suffix stays visible on a narrow
|
||||
/// terminal. Keeps the tail (filename + a couple of parents) — that's the
|
||||
/// distinguishing part — and prefixes `…` when truncated. Paths up to the
|
||||
/// budget pass through verbatim.
|
||||
fn abbreviate_path(path: &str) -> String {
|
||||
const MAX: usize = 48;
|
||||
let char_count = path.chars().count();
|
||||
if char_count <= MAX {
|
||||
return path.to_string();
|
||||
}
|
||||
// Keep the last MAX-1 chars (1 reserved for the leading ellipsis).
|
||||
let tail: String = path
|
||||
.chars()
|
||||
.skip(char_count - (MAX - 1))
|
||||
.collect::<String>();
|
||||
format!("…{tail}")
|
||||
}
|
||||
|
||||
/// v0.26.0: render the end-of-run "slowest assets" summary. Sorts
|
||||
/// `(path, total_ms)` descending by time, takes the top `n`, and writes a
|
||||
/// compact table to `w`. No-op (writes nothing) when `timings` is empty so
|
||||
/// a run with no per-asset timing (e.g. all-skipped) prints no stray header.
|
||||
fn write_slowest_summary(
|
||||
w: &mut impl Write,
|
||||
timings: &[(String, u64)],
|
||||
n: usize,
|
||||
) -> std::io::Result<()> {
|
||||
if timings.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let mut sorted: Vec<&(String, u64)> = timings.iter().collect();
|
||||
// desc by ms; ties broken by path for deterministic output.
|
||||
sorted.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
|
||||
let top = &sorted[..sorted.len().min(n)];
|
||||
writeln!(w, "⏱ 최장 소요 top-{}:", top.len())?;
|
||||
for (rank, (path, ms)) in top.iter().enumerate() {
|
||||
writeln!(w, " {}. {} — {}", rank + 1, path, fmt_ms(*ms))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Format the current wall-clock as RFC 3339 — used by `wire_ingest_progress`
|
||||
/// so every emitted event carries an `ts` field per §2.4a / the wire schema.
|
||||
pub(crate) fn now_rfc3339() -> anyhow::Result<String> {
|
||||
@@ -348,4 +508,61 @@ mod tests {
|
||||
// well-formed RFC 3339 string.
|
||||
OffsetDateTime::parse(&s, &Rfc3339).expect("RFC 3339 round-trip");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn abbreviate_path_passes_short_paths_through() {
|
||||
assert_eq!(abbreviate_path("notes/foo.md"), "notes/foo.md");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn abbreviate_path_keeps_tail_with_ellipsis() {
|
||||
let long = "a/very/deeply/nested/directory/structure/that/exceeds/the/budget/file.md";
|
||||
let out = abbreviate_path(long);
|
||||
assert!(out.starts_with('…'), "should be prefixed with ellipsis: {out}");
|
||||
assert!(out.ends_with("file.md"), "should keep the filename tail: {out}");
|
||||
// 48-char budget: 1 ellipsis + 47 tail chars.
|
||||
assert_eq!(out.chars().count(), 48);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn write_slowest_summary_empty_writes_nothing() {
|
||||
let mut buf = Vec::new();
|
||||
write_slowest_summary(&mut buf, &[], 5).unwrap();
|
||||
assert!(buf.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn write_slowest_summary_sorts_desc_and_truncates() {
|
||||
let timings = vec![
|
||||
("a.md".to_string(), 100),
|
||||
("b.png".to_string(), 5_000),
|
||||
("c.pdf".to_string(), 2_000),
|
||||
("d.md".to_string(), 50),
|
||||
];
|
||||
let mut buf = Vec::new();
|
||||
write_slowest_summary(&mut buf, &timings, 2).unwrap();
|
||||
let out = String::from_utf8(buf).unwrap();
|
||||
assert!(out.contains("top-2:"), "{out}");
|
||||
// b (5s) ranks first, c (2s) second; a/d excluded.
|
||||
let b_pos = out.find("b.png").expect("b.png present");
|
||||
let c_pos = out.find("c.pdf").expect("c.pdf present");
|
||||
assert!(b_pos < c_pos, "b before c: {out}");
|
||||
assert!(!out.contains("a.md"), "a.md excluded by top-2: {out}");
|
||||
assert!(out.contains("5.0s"), "b renders as 5.0s: {out}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn write_slowest_summary_tie_breaks_by_path() {
|
||||
let timings = vec![
|
||||
("z.md".to_string(), 1_000),
|
||||
("a.md".to_string(), 1_000),
|
||||
];
|
||||
let mut buf = Vec::new();
|
||||
write_slowest_summary(&mut buf, &timings, 5).unwrap();
|
||||
let out = String::from_utf8(buf).unwrap();
|
||||
assert!(
|
||||
out.find("a.md").unwrap() < out.find("z.md").unwrap(),
|
||||
"equal ms ties break alphabetically: {out}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -209,6 +209,13 @@ impl OllamaVisionOcr {
|
||||
self.max_pixels
|
||||
}
|
||||
|
||||
/// The Ollama model id this engine drives (e.g. `gemma4:e4b`).
|
||||
/// Surfaced so the ingest progress display can name the model
|
||||
/// running a slow OCR phase (`AssetPhase{phase:"ocr", model}`).
|
||||
pub fn model(&self) -> &str {
|
||||
&self.model
|
||||
}
|
||||
|
||||
fn build_prompt(&self, lang_hint: Option<&Lang>) -> String {
|
||||
let langs = if self.languages.is_empty() {
|
||||
"any".to_string()
|
||||
|
||||
@@ -160,7 +160,11 @@ fn apply_event(state: &mut IngestState, event: IngestEvent) {
|
||||
// per-asset counters, not sub-asset phase progress, so these are
|
||||
// no-ops here (the CLI / --json surfaces render them).
|
||||
| IngestEvent::AssetChunked { .. }
|
||||
| IngestEvent::AssetTimings { .. } => {}
|
||||
| IngestEvent::AssetTimings { .. }
|
||||
// v0.26.0 slow-phase hint (ocr / caption / embed): the CLI bar uses
|
||||
// it for a live phase message; the TUI status-bar reducer tracks only
|
||||
// per-asset counters, so it's a no-op here.
|
||||
| IngestEvent::AssetPhase { .. } => {}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user