feat(app): first-boot eager backfill for tokenized_korean_text

V007 → V009 업그레이드 시 기존 chunks 의 tokenized_korean_text 가
NULL — 첫 App::open_with_config 호출 시 자동으로 lindera ko-dic
으로 분해 후 UPDATE. chunks_au trigger 가 chunks_fts 를 자동 재-index.
사용자 재-ingest 불필요.

- crates/kebab-store-sqlite/src/store.rs:
  backfill_tokenized_korean_text(progress_cb, tokenize) API. 1000 row 마다
  commit + progress 콜백. idempotent (IS NULL 필터로 partial
  completion 재실행 안전). tokenizer 를 파라미터로 받아 §8 dep 경계 유지.
- crates/kebab-app/src/app.rs::open_with_config: run_migrations 직후
  backfill 호출. 실패 시 warn log 만 (App open 은 성공 — vector/hybrid
  mode 계속 가능). 500 row 마다 info log progress.
- crates/kebab-store-sqlite/tests/fts.rs:
  backfill_tokenized_korean_text_populates_nullable_rows 단위 test
  (idempotency 포함).
- clippy pre-existing 오류 수정 (redundant_closure, map_unwrap_or,
  cast_lossless, uninlined_format_args — kebab-app/ingest_log.rs,
  pdf_ocr_apply.rs, app.rs, tests/ocr_inspect_smoke.rs).

Spec: docs/superpowers/specs/2026-05-28-v0.20.x-korean-morphological-tokenizer-spec.md §8.1, §8.2
Plan: docs/superpowers/plans/2026-05-28-v0.20.x-korean-morphological-tokenizer-plan.md (S4)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-05-28 11:01:00 +00:00
parent e8f44a57e3
commit b63af20b72
6 changed files with 148 additions and 17 deletions

View File

@@ -212,6 +212,34 @@ impl App {
sqlite
.run_migrations()
.context("kb-app: run SqliteStore migrations")?;
// V009 의 tokenized_korean_text column 의 first-boot eager backfill.
// 신규 ingest 의 chunks_ai trigger 가 이미 채우므로 NULL row 가 없으면 즉시 0 반환 (idempotent).
// V007 → V009 업그레이드 시 KB 크기 비례 (~10000 chunk 당 ~30-60s).
let backfill_count = sqlite
.backfill_tokenized_korean_text(
|done, total| {
if total > 0 && done % 500 == 0 {
tracing::info!(
target: "kebab-app",
"korean tokenizer backfill: {done}/{total}"
);
}
},
kebab_chunk::tokenize_korean_morphological,
)
.unwrap_or_else(|e| {
tracing::warn!(
target: "kebab-app",
"korean tokenizer backfill failed: {e}"
);
0
});
if backfill_count > 0 {
tracing::info!(
target: "kebab-app",
"korean tokenizer backfill complete: {backfill_count} chunks updated"
);
}
// p9-fb-19: build the LRU cache from config. Capacity 0 →
// `None` (cache disabled — every search hits the retrievers).
let search_cache = NonZeroUsize::new(config.search.cache_capacity)
@@ -1177,7 +1205,7 @@ impl App {
.context("prepare ms query")?;
stmt.query_map([], |r| r.get::<_, u64>(0))
.context("query ms")?
.filter_map(|r| r.ok())
.filter_map(Result::ok)
.collect()
};
let (p50_ms, p90_ms, p99_ms, max_ms) = percentiles(&samples);
@@ -1191,7 +1219,7 @@ impl App {
let rows = stmt
.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, u64>(1)?)))
.context("query engine")?;
for row in rows.filter_map(|r| r.ok()) {
for row in rows.filter_map(Result::ok) {
by_engine.insert(row.0, row.1);
}
}
@@ -1219,7 +1247,7 @@ impl App {
})
})
.context("query by_doc")?
.filter_map(|r| r.ok())
.filter_map(Result::ok)
.collect()
};
@@ -1276,7 +1304,7 @@ impl App {
})
})
.context("query failures by doc_id")?
.filter_map(|r| r.ok())
.filter_map(Result::ok)
.collect()
} else {
let mut stmt = conn
@@ -1298,7 +1326,7 @@ impl App {
})
})
.context("query failures corpus-wide")?
.filter_map(|r| r.ok())
.filter_map(Result::ok)
.collect()
};
Ok(OcrFailuresV1 {