refactor(app): fingerprint_and_skip — effective version+skip 결정 중앙화 (4× 복제 제거)

This commit is contained in:
2026-06-24 13:06:42 +00:00
parent 0f9a76997d
commit 9f40c88722

View File

@@ -906,6 +906,52 @@ struct ImagePipeline<'a> {
caption_llm: Option<&'a dyn LanguageModel>,
}
/// Result of [`fingerprint_and_skip`]: the composite effective
/// `parser_version` for this asset (used downstream when stamping the
/// persisted document) plus the early-skip decision.
struct FingerprintOutcome {
/// Composite version = base extractor `parser_version` folded with the
/// ingest-config signature (see [`effective_parser_version`]). Each
/// handler assigns this to `canonical.parser_version` after a non-skip.
effective_parser_version: ParserVersion,
/// `Some(..)` when the asset is `Unchanged` and the full re-process can
/// be skipped; `None` when the caller must re-parse / re-chunk / re-embed.
skip: Option<kebab_core::IngestItem>,
}
/// Central per-asset "effective version + skip-unchanged" decision shared
/// by every media handler (markdown / image / PDF / code). Composes the
/// composite `parser_version` ([`effective_parser_version`]) and the
/// incremental-ingest early-skip predicate ([`try_skip_unchanged`]) into a
/// single call so each handler runs the identical sequence instead of
/// replicating the two calls. The per-media inputs (`base_parser_version`,
/// `chunker_version`, `fallback_chunker_version`) are threaded through
/// unchanged — this is a de-dup, not a behavior change.
fn fingerprint_and_skip(
app: &App,
asset: &RawAsset,
base_parser_version: &ParserVersion,
chunker_version: &ChunkerVersion,
embedder: Option<&Arc<dyn Embedder + Send + Sync>>,
force_reingest: bool,
fallback_chunker_version: Option<&ChunkerVersion>,
) -> anyhow::Result<FingerprintOutcome> {
let effective_parser_version = effective_parser_version(&app.config, asset, base_parser_version);
let skip = try_skip_unchanged(
app,
asset,
&effective_parser_version,
chunker_version,
embedder.map(|e| e.model_version()).as_ref(),
force_reingest,
fallback_chunker_version,
)?;
Ok(FingerprintOutcome {
effective_parser_version,
skip,
})
}
/// p9-fb-23 task 7: incremental-ingest early-skip predicate. Shared
/// across the markdown / image / PDF per-asset flows. Returns
/// `Some(IngestItem { kind: Unchanged, .. })` when ALL FOUR conditions
@@ -1320,23 +1366,24 @@ fn ingest_one_asset(
// parser_version for the skip compare + the stored doc field, so a
// change to any markdown-affecting setting (chunking params) re-indexes.
// `doc_id` keeps deriving from the base version below (stability).
let eff_parser_version = effective_parser_version(&app.config, asset, parser_version);
//
// p9-fb-23 task 7: incremental-ingest early-skip. When force_reingest
// is false AND the on-disk asset's checksum + parser_version +
// last_chunker_version + last_embedding_version all match the existing
// DB record, this asset doesn't need to be re-parsed / re-chunked /
// re-embedded. Return Unchanged so the caller bumps `aggregate.unchanged`
// and the AssetFinished progress event reflects the skip.
if let Some(item) = try_skip_unchanged(
let fp = fingerprint_and_skip(
app,
asset,
&eff_parser_version,
parser_version,
&md_chunker_from_config(&app.config).chunker_version(),
embedder.map(|e| e.model_version()).as_ref(),
embedder,
force_reingest,
None,
)? {
)?;
let eff_parser_version = fp.effective_parser_version;
if let Some(item) = fp.skip {
return Ok(item);
}
@@ -1607,16 +1654,17 @@ fn ingest_one_image_asset(
// settings, so toggling `[image.ocr]` / `[image.caption]` (or changing
// their model / prompt version) auto-re-indexes the affected images.
let image_parser_version = ParserVersion(kebab_parse_image::PARSER_VERSION.to_string());
let eff_parser_version = effective_parser_version(&app.config, asset, &image_parser_version);
if let Some(item) = try_skip_unchanged(
let fp = fingerprint_and_skip(
app,
asset,
&eff_parser_version,
&image_parser_version,
&md_chunker_from_config(&app.config).chunker_version(),
embedder.map(|e| e.model_version()).as_ref(),
embedder,
force_reingest,
None,
)? {
)?;
let eff_parser_version = fp.effective_parser_version;
if let Some(item) = fp.skip {
return Ok(item);
}
let bytes = std::fs::read(&path)
@@ -2223,16 +2271,17 @@ fn ingest_one_pdf_asset(
// v0.26.2: composite parser_version folds pdf.ocr (enabled/always_on/
// model) + chunking, so enabling scanned-PDF OCR auto-re-indexes PDFs.
let pdf_parser_version = ParserVersion(kebab_parse_pdf::PARSER_VERSION.to_string());
let eff_parser_version = effective_parser_version(&app.config, asset, &pdf_parser_version);
if let Some(item) = try_skip_unchanged(
let fp = fingerprint_and_skip(
app,
asset,
&eff_parser_version,
&pdf_parser_version,
&pdf_chunker_from_config(&app.config).chunker_version(),
embedder.map(|e| e.model_version()).as_ref(),
embedder,
force_reingest,
None,
)? {
)?;
let eff_parser_version = fp.effective_parser_version;
if let Some(item) = fp.skip {
return Ok(item);
}
let bytes = std::fs::read(&path)
@@ -2635,16 +2684,17 @@ fn ingest_one_code_asset(
// `stored_is_tier3_fallback` bypass in try_skip_unchanged depends on the
// exact "none-v1" sentinel), so the composite is only stamped on the
// normal (non-fallback) outcome below.
let eff_parser_version = effective_parser_version(&app.config, asset, &parser_version);
if let Some(item) = try_skip_unchanged(
let fp = fingerprint_and_skip(
app,
asset,
&eff_parser_version,
&parser_version,
&chunker_version,
embedder.map(|e| e.model_version()).as_ref(),
embedder,
force_reingest,
tier3_fallback_cv.as_ref(),
)? {
)?;
let eff_parser_version = fp.effective_parser_version;
if let Some(item) = fp.skip {
return Ok(item);
}
let bytes = std::fs::read(&path)