From 0e408fb1b55648beebb14aac4eff499ac12901dc Mon Sep 17 00:00:00 2001 From: altair823 Date: Sun, 3 May 2026 05:01:31 +0000 Subject: [PATCH] feat(kebab-app + kebab-store-sqlite): p9-fb-19 search LRU cache + corpus_revision MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 도그푸딩 item 15 — TUI / 같은 process 안에서 동일 query 반복 시 SQLite FTS + Lance + RRF 재계산이 매번 발생하던 비용 해소. in-process LRU 캐시 + 모노토닉 corpus_revision 카운터로 ingest commit 발생 시 모든 entry 자동 stale. ## 핵심 변경 - **SQLite V004 migration**: `kv (key TEXT PRIMARY KEY, value TEXT) STRICT` + `corpus_revision = '0'` seed. 미래의 다른 scalar 도 같은 테이블에 들어갈 수 있는 generic shape. - **`SqliteStore::corpus_revision()` / `bump_corpus_revision()`** — `UPDATE ... CAST AS INTEGER + 1` atomic. INSERT-OR-IGNORE 도 함께 실행 (V004 seed 가 무슨 이유로 누락된 케이스 paranoid). - **`kebab-app::ingest_with_config_cancellable`** — `new + updated > 0` 시 bump, no-op (skipped-only) reingest 는 cache 보존. - **`App.search_cache: Option>>>`** — `config.search.cache_capacity` (default 256, 0 비활성). `lru = "0.12"` workspace dep 추가. - **`SearchCacheKey`** = `query_norm` (NFKC + trim + lowercase) + `mode` + `k` + `snippet_chars` + `embedding_version` (vector/hybrid 만, lexical 은 빈 문자열) + `chunker_version` + `corpus_revision` snapshot. - **`App::search`** rewrite — cache 활성 시 lookup → miss 면 기존 `search_uncached` 호출 후 put. cache 비활성이거나 lock 실패면 straight-line. - **`App::search_uncached`** (rename of pre-fb-19 `search` body) + `search_uncached_with_config` facade — CLI `kebab search --no-cache` 로 진입. - **`Config.search.cache_capacity: usize`** field, `#[serde(default)]` 로 기존 config 호환. - **CLI `--no-cache`** flag — 디버깅용 (CLI 는 매 호출이 새 process 라 사실상 no-op 이지만 spec 명시 + 향후 long-lived process 호환). - **frozen design §9 versioning** 표에 `corpus_revision` row 추가 (기존 `index_version` 라벨과 다른 차원: 라벨은 retrieval 형상, corpus_revision 은 ingest commit ack). ## 테스트 - `kebab-store-sqlite` 신규 3 unit (fresh=0, monotonic bump, persist across reopen) - `kebab-app` 신규 4 integration (cached repeat 같은 hits, NFKC 정규화 로 case/whitespace collapse, --no-cache parity, first ingest bumps corpus_revision) - 워크스페이스 전체 `cargo test --workspace --no-fail-fast -j 1` exit 0 - `cargo clippy --workspace --all-targets -- -D warnings` clean ## 문서 - README `kebab search` 행: 캐시 동작 + `--no-cache` 안내 + corpus_ revision 무효화 메커니즘 - docs/SMOKE.md `[search]` 절에 `cache_capacity` 라인 추가 - HANDOFF: 2026-05-03 entry - spec status planned → in_progress ## Out of scope - patch-and-merge incremental (RRF 정규화 전체 hit set 기준이라 어려움) - SQLite 영속 cache (P+) - 다른 process 간 cache 공유 (in-process 만 — corpus_revision 이 cross-process 무효화는 O(1)) Co-Authored-By: Claude Opus 4.7 (1M context) --- Cargo.lock | 2 + Cargo.toml | 3 + HANDOFF.md | 1 + README.md | 2 +- crates/kebab-app/Cargo.toml | 7 + crates/kebab-app/src/app.rs | 128 +++++++++++++++++- crates/kebab-app/src/lib.rs | 31 +++++ crates/kebab-app/tests/search_lexical.rs | 85 ++++++++++++ crates/kebab-cli/src/main.rs | 20 ++- crates/kebab-config/src/lib.rs | 11 ++ crates/kebab-store-sqlite/src/store.rs | 57 ++++++++ .../tests/corpus_revision.rs | 54 ++++++++ docs/SMOKE.md | 1 + .../2026-04-27-kebab-final-form-design.md | 1 + migrations/V004__kv.sql | 24 ++++ tasks/p9/p9-fb-19-search-cache.md | 2 +- 16 files changed, 424 insertions(+), 5 deletions(-) create mode 100644 crates/kebab-store-sqlite/tests/corpus_revision.rs create mode 100644 migrations/V004__kv.sql diff --git a/Cargo.lock b/Cargo.lock index 9045280..116c3c3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3515,6 +3515,7 @@ dependencies = [ "kebab-store-sqlite", "kebab-store-vector", "lopdf", + "lru", "rusqlite", "serde", "serde_json", @@ -3525,6 +3526,7 @@ dependencies = [ "tracing", "tracing-appender", "tracing-subscriber", + "unicode-normalization", "wiremock", ] diff --git a/Cargo.toml b/Cargo.toml index 0f48073..436020b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -49,6 +49,9 @@ rusqlite = { version = "0.32", features = ["bundled"] } globset = "0.4" tempfile = "3" proptest = "1" +# p9-fb-19: LRU cache for `App::search` results. Bounded capacity +# from `config.search.cache_capacity` (default 256, ~1.3 MB cap). +lru = "0.12" # fastembed-rs ships ONNX runtime via the `ort-download-binaries` feature # in its default set (which also pulls `hf-hub` for first-run model # downloads). Pinned to the 4.x line per task p3-2 (current 5.x release diff --git a/HANDOFF.md b/HANDOFF.md index 83d78da..7fd7fdc 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -53,6 +53,7 @@ P0~P5 직렬. P6~P9 P5 이후 병렬 가능. - **2026-05-03 P9 도그푸딩 후속 (p9-fb-11)** — TUI Ask 답변 본문 markdown 렌더. `kebab-tui::markdown::render(text, &Theme) -> Vec>` 신규 — `pulldown-cmark = "0.13"` 위에서 inline (bold/italic/strikethrough/inline code/link)·block (heading H1-H6, ordered/unordered list with nesting, fenced code block, table, blockquote `▎`, horizontal rule) 변환. heading H1/H2 = `Role::Heading`, H3+ = `Role::Title`, link = `Role::CitationMarker + UNDERLINE`, code = `Role::Hint`. ask `push_turn_lines` 가 grounded 답변에서만 markdown 렌더; refusal (`Role::Warning`) / streaming (`Role::Hint`) 은 raw 로 두어 role color 시그널 보존. CLI `kebab ask` 출력은 raw markdown 그대로 (terminal 호환성). 매 frame 재 parse — pulldown 토크나이저가 µs/KB 라 비용 무시. spec: `tasks/p9/p9-fb-11-ask-markdown-render.md`. - **2026-05-03 P9 도그푸딩 후속 (p9-fb-08)** — TUI search async worker + generation counter. 기존 200ms debounce 후 `kebab_app::search_with_config` 동기 호출이 vector/hybrid 모드 50-200ms 동안 UI freeze 시키던 문제 해소. `SearchState` 에 `generation: u64` + `worker_thread: Option` + `worker_rx: Option>` 신규. `fire_search` 가 spawn 만 하고 즉시 return — worker 가 별 thread 에서 검색 후 `(generation, Result)` 를 channel 로 post. run loop 가 매 tick `poll_worker` 로 try_recv, generation 일치 시 hits 적용 / 불일치 시 silently 폐기 (사용자가 더 빠르게 타이핑하면 stale 결과 자동 drop). debounce_due 가 `searching && last_query == 현 input` 케이스 추가 skip — in-flight worker 의 결과 기다리는 동안 동일 query 재 spawn 안 함. spec: `tasks/p9/p9-fb-08-search-debounce.md`. - **2026-05-03 P9 도그푸딩 후속 (p9-fb-05)** — `workspace.root` path policy 명확화. `kebab_config::expand_path_with_base(raw, data_dir, base_dir) -> PathBuf` 신규 — 기존 `expand_path` (tilde + env 만) 위에 relative path resolution 추가, 절대/`~`/`${VAR}` 입력은 base_dir 무시. `Config.source_dir: Option` 필드 (`#[serde(skip)]`) 신규 — `from_file` / `load` 가 `path.parent()` 로 stamp. `Config::resolve_workspace_root()` helper 가 `expand_path_with_base(&workspace.root, "", source_dir.unwrap_or(cwd))` 호출. kebab-app + kebab-source-fs 의 모든 `workspace.root` 사용 사이트가 `cfg.resolve_workspace_root()` 로 통일 — kebab-source-fs 의 fork 된 `expand_tilde` 헬퍼는 제거 (kebab-app 의 `storage.data_dir` 한 곳만 남음, P+ 통일 caveat). `kebab init` 가 생성하는 `config.toml` 위에 path policy 안내 헤더 코멘트 자동 prepend (절대/tilde/env/상대 + 상대 base = config dir). spec: `tasks/p9/p9-fb-05-config-path-policy.md`. +- **2026-05-03 P9 도그푸딩 후속 (p9-fb-19)** — In-process LRU search cache + `corpus_revision` 카운터. SQLite V004 migration 으로 `kv (key TEXT PK, value TEXT)` 테이블 + `corpus_revision = '0'` seed. `SqliteStore::corpus_revision()` / `bump_corpus_revision()` 메서드 (`UPDATE ... CAST AS INTEGER + 1` 으로 atomic). `kebab-app::ingest_with_config_cancellable` 가 `new + updated > 0` 시 bump — no-op reingest 는 cache 보존. `App.search_cache: Option>>>` (capacity from `config.search.cache_capacity`, default 256, 0 = 비활성). `SearchCacheKey` = `query_norm` (NFKC + trim + lowercase) + `mode` + `k` + `snippet_chars` + `embedding_version` + `chunker_version` + `corpus_revision` snapshot. `App::search` 가 lookup → miss 시 `search_uncached` → put. `search_uncached_with_config` facade 추가, CLI `kebab search --no-cache` 로 bypass (디버깅용). frozen design §9 versioning 표에 `corpus_revision` row 추가. spec: `tasks/p9/p9-fb-19-search-cache.md`. ## 다음 task 후보 diff --git a/README.md b/README.md index e718942..76ae578 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ kebab doctor |------|------| | `kebab init` | XDG 경로에 데이터 디렉토리 + config.toml 생성 | | `kebab ingest []` | Markdown / 이미지 / PDF 색인 (idempotent). TTY 에서는 stderr 진행 바, non-TTY (CI / pipe) 는 stderr 한 줄씩, `--json` 은 stdout 에 `ingest_progress.v1` 라인 streaming 후 마지막에 `ingest_report.v1`. Ctrl-C 한 번이면 현재 asset 마무리 후 abort (부분 commit 보존, idempotent re-run), 두 번째 Ctrl-C 는 hard exit. Markdown title 이 frontmatter 에 없어도 첫 H1 → H2 → 첫 paragraph 80 자 → 파일명 순으로 자동 채움 (parser_version `md-frontmatter-v2`) — 기존 색인된 doc 도 다음 ingest 에서 새 title 로 갱신 | -| `kebab search --mode {lexical,vector,hybrid} ""` | 검색. hybrid는 RRF fusion, citation 포함 | +| `kebab search --mode {lexical,vector,hybrid} "" [--no-cache]` | 검색. hybrid는 RRF fusion, citation 포함. 같은 process 안에서 동일 query (NFKC + trim + lowercase 정규화) 반복 시 in-process LRU 캐시 hit (capacity = `[search] cache_capacity`, default 256). `--no-cache` 로 강제 bypass — 디버깅용. ingest commit 발생 시 `kv['corpus_revision']` bump 으로 모든 entry 자동 stale | | `kebab list docs` | 색인된 문서 목록 | | `kebab inspect doc ` / `kebab inspect chunk ` | raw record 보기 | | `kebab ask "" [--show-citations / --hide-citations]` | RAG 답변 + 근거 인용. 답변 후 `근거:` block 으로 full path / line range / score 한 줄씩 (default ON — `--hide-citations` 로 끄기, pipe 시 유용). 근거 부족 시 거절. Ollama 필요 | diff --git a/crates/kebab-app/Cargo.toml b/crates/kebab-app/Cargo.toml index 613d9ea..d43452f 100644 --- a/crates/kebab-app/Cargo.toml +++ b/crates/kebab-app/Cargo.toml @@ -42,6 +42,13 @@ tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt", "json tracing-appender = "0.2" toml = "0.8" dirs = "5" +# p9-fb-19: in-process LRU cache for `App::search`. Capacity from +# `config.search.cache_capacity` (default 256, ~1.3 MB cap). +lru = { workspace = true } +# p9-fb-19: NFKC-normalize cache-key queries so `"Foo"` / `"FOO"` / +# `" foo "` collapse to one entry. Same crate kebab-normalize + +# kebab-core already use, no version drift. +unicode-normalization = "0.1" [dev-dependencies] rusqlite = { workspace = true } diff --git a/crates/kebab-app/src/app.rs b/crates/kebab-app/src/app.rs index 4080a15..ce6d4c3 100644 --- a/crates/kebab-app/src/app.rs +++ b/crates/kebab-app/src/app.rs @@ -33,9 +33,11 @@ //! in that mode [`App::embedder`] returns `None` and callers must fall //! back to lexical-only search. -use std::sync::{Arc, OnceLock}; +use std::num::NonZeroUsize; +use std::sync::{Arc, Mutex, OnceLock}; use anyhow::{Context, Result, anyhow}; +use lru::LruCache; use kebab_core::{ Answer, Embedder, IndexVersion, LanguageModel, Retriever, SearchHit, SearchMode, @@ -69,6 +71,44 @@ pub struct App { /// client per query (cheap, but still measurable on a 50-query /// suite). llm: OnceLock>, + /// p9-fb-19: in-process LRU search-result cache. Capacity comes + /// from `config.search.cache_capacity` (default 256, ~1.3 MB + /// cap). `None` when capacity is 0 (cache disabled). The + /// `corpus_revision` snapshot embedded in `SearchCacheKey` + /// invalidates every entry the moment a new ingest commit lands. + search_cache: Option>>>, +} + +/// p9-fb-19: cache key for `App::search`. Includes every field that +/// could change the result set: +/// - normalized query (NFKC + trim + lowercase) +/// - mode + k + snippet_chars (caller knobs) +/// - embedding_version + chunker_version (model identity) +/// - corpus_revision (monotonic counter that ingest bumps) +/// +/// Lexical mode has no embedding identity → empty string in that +/// slot, harmless because the rest of the key still distinguishes +/// queries. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub(crate) struct SearchCacheKey { + pub query_norm: String, + pub mode: SearchMode, + pub k: u32, + pub snippet_chars: u32, + pub embedding_version: String, + pub chunker_version: String, + pub corpus_revision: u64, +} + +impl SearchCacheKey { + /// Normalize `query.text` per spec p9-fb-19: NFKC + trim + + /// lowercase. Means `"Foo"` / `"FOO"` / `" foo "` collapse to a + /// single cache entry — redundant work avoided when the user's + /// input differs only in shape. + pub fn normalize_query(text: &str) -> String { + use unicode_normalization::UnicodeNormalization; + text.trim().nfkc().collect::().to_lowercase() + } } impl App { @@ -85,22 +125,65 @@ impl App { sqlite .run_migrations() .context("kb-app: run SqliteStore migrations")?; + // p9-fb-19: build the LRU cache from config. Capacity 0 → + // `None` (cache disabled — every search hits the retrievers). + let search_cache = NonZeroUsize::new(config.search.cache_capacity) + .map(|cap| Mutex::new(LruCache::new(cap))); Ok(Self { config, sqlite: Arc::new(sqlite), embedder: OnceLock::new(), vector: OnceLock::new(), llm: OnceLock::new(), + search_cache, }) } /// Run a [`SearchQuery`] through the configured retriever stack and - /// return the top-k hits. + /// return the top-k hits. p9-fb-19: result is served from the + /// in-process LRU cache when the same `(query_norm, mode, k, + /// snippet_chars, embedding_version, chunker_version, + /// corpus_revision)` tuple was seen before; cache miss falls + /// through to [`Self::search_uncached`]. /// /// Reuses any previously-built embedder / vector store on this `App` /// — long-lived callers (kb-eval, future TUI) get amortized cost /// across calls. pub fn search(&self, query: SearchQuery) -> Result> { + let Some(cache) = self.search_cache.as_ref() else { + // Cache disabled (capacity = 0) — straight-line. + return self.search_uncached(query); + }; + // Build the cache key. embedding_version is empty for lexical + // mode (no embedder identity); for vector/hybrid we need the + // embedder built (which forces the cold-start cost), but + // that's the cost the cache exists to amortize across + // *subsequent* identical queries. + let key = self.build_cache_key(&query)?; + // Lock the cache long enough to lookup; clone the hit out so + // we can drop the lock before returning. + if let Ok(mut guard) = cache.lock() { + if let Some(hits) = guard.get(&key) { + tracing::debug!( + target: "kebab-app", + cache = "hit", + corpus_revision = key.corpus_revision, + "search served from LRU cache" + ); + return Ok(hits.clone()); + } + } + let hits = self.search_uncached(query)?; + if let Ok(mut guard) = cache.lock() { + guard.put(key, hits.clone()); + } + Ok(hits) + } + + /// p9-fb-19: bypass the LRU cache and run the search directly. + /// Used by `--no-cache` CLI invocations and by `search` itself + /// on cache miss. Identical behavior to the pre-fb-19 `search`. + pub fn search_uncached(&self, query: SearchQuery) -> Result> { match query.mode { SearchMode::Lexical => { let lex = LexicalRetriever::with_settings( @@ -257,6 +340,47 @@ impl App { Ok(self.llm.get().cloned().unwrap_or(llm)) } + /// p9-fb-19: build a `SearchCacheKey` for `query`. For lexical + /// mode the embedding_version slot is left empty (no embedder + /// identity contributes to the result). For vector / hybrid + /// modes the embedder is built (cold-start) so the version + /// label can be read; that's the cost the cache exists to + /// amortize over the next few identical queries. + fn build_cache_key(&self, query: &SearchQuery) -> Result { + let embedding_version = match query.mode { + SearchMode::Lexical => String::new(), + SearchMode::Vector | SearchMode::Hybrid => { + let emb = self.embedder()?.ok_or_else(|| { + anyhow!( + "embeddings disabled; vector / hybrid search require an \ + embedder — switch to --mode lexical or enable a provider" + ) + })?; + vector_index_version(emb.as_ref()).0 + } + }; + Ok(SearchCacheKey { + query_norm: SearchCacheKey::normalize_query(&query.text), + mode: query.mode, + k: u32::try_from(query.k).unwrap_or(u32::MAX), + snippet_chars: u32::try_from(self.config.search.snippet_chars).unwrap_or(u32::MAX), + embedding_version, + chunker_version: self.config.chunking.chunker_version.clone(), + corpus_revision: self.sqlite.corpus_revision(), + }) + } + + /// p9-fb-19: clear the in-process search cache. Useful for tests + /// and for explicit user actions (e.g. a future `kebab cache + /// clear` admin command). No-op when the cache is disabled. + pub fn clear_search_cache(&self) { + if let Some(cache) = self.search_cache.as_ref() { + if let Ok(mut guard) = cache.lock() { + guard.clear(); + } + } + } + /// Resolve the embedder + vector store, surfacing the user-friendly /// "switch to --mode lexical" error when embeddings are disabled. fn require_embeddings( diff --git a/crates/kebab-app/src/lib.rs b/crates/kebab-app/src/lib.rs index 6ca52a3..62a3c20 100644 --- a/crates/kebab-app/src/lib.rs +++ b/crates/kebab-app/src/lib.rs @@ -600,6 +600,26 @@ pub fn ingest_with_config_cancellable( }; crate::ingest_progress::emit(progress, terminal_event); + // p9-fb-19: bump the persistent corpus_revision counter when a + // commit landed (any new / updated). This invalidates every + // entry in any in-process LRU search cache (in this process or + // a sibling) on the next lookup. No-op when nothing changed + // (skipped-only run) — the cache stays valid. + if new_count > 0 || updated_count > 0 { + match app.sqlite.bump_corpus_revision() { + Ok(rev) => tracing::debug!( + target: "kebab-app", + corpus_revision = rev, + "bumped corpus_revision after ingest commit" + ), + Err(e) => tracing::warn!( + target: "kebab-app", + error = %e, + "bump_corpus_revision failed; cache may serve stale results until process restart" + ), + } + } + Ok(IngestReport { scope, scanned: scanned_count, @@ -1443,6 +1463,17 @@ pub fn search_with_config( App::open_with_config(config)?.search(query) } +/// p9-fb-19: bypass the LRU search cache for one call. Same shape as +/// [`search_with_config`] but routes through [`App::search_uncached`] +/// — used by `kebab search --no-cache`. +#[doc(hidden)] +pub fn search_uncached_with_config( + config: kebab_config::Config, + query: SearchQuery, +) -> anyhow::Result> { + App::open_with_config(config)?.search_uncached(query) +} + // ── ask ────────────────────────────────────────────────────────────────── // // P4-3 wires `ask` end-to-end. The retriever is built per `opts.mode`; diff --git a/crates/kebab-app/tests/search_lexical.rs b/crates/kebab-app/tests/search_lexical.rs index e50f6a3..bbae95b 100644 --- a/crates/kebab-app/tests/search_lexical.rs +++ b/crates/kebab-app/tests/search_lexical.rs @@ -49,6 +49,91 @@ fn lexical_search_empty_query_returns_empty() { assert!(hits.is_empty(), "blank query must short-circuit empty"); } +/// p9-fb-19 — `App::search` returns the same hit list for a repeated +/// query (cache hit doesn't corrupt the result). Both calls share an +/// `App` instance so the cache is in scope. +#[test] +fn cached_search_returns_same_hits_on_repeat() { + let env = TestEnv::lexical_only(); + kebab_app::ingest_with_config(env.config.clone(), env.scope(), true).unwrap(); + let app = kebab_app::App::open_with_config(env.config.clone()).unwrap(); + let first = app.search(lexical_query("ownership")).unwrap(); + assert!(!first.is_empty(), "first call must return ≥1 hit"); + let second = app.search(lexical_query("ownership")).unwrap(); + assert_eq!( + first.len(), + second.len(), + "cached call must yield identical hit count" + ); + for (a, b) in first.iter().zip(second.iter()) { + assert_eq!(a.chunk_id, b.chunk_id, "chunk_ids must align"); + assert_eq!(a.rank, b.rank, "ranks must align"); + } +} + +/// p9-fb-19 — query normalization (NFKC + trim + lowercase) collapses +/// `"Ownership"` / `"OWNERSHIP"` / `" ownership "` into one cache +/// entry. Verified by ensuring all three forms return the same hits. +#[test] +fn cache_key_normalization_treats_case_and_whitespace_as_equivalent() { + let env = TestEnv::lexical_only(); + kebab_app::ingest_with_config(env.config.clone(), env.scope(), true).unwrap(); + let app = kebab_app::App::open_with_config(env.config.clone()).unwrap(); + let plain = app.search(lexical_query("ownership")).unwrap(); + let upper = app.search(lexical_query("OWNERSHIP")).unwrap(); + let padded = app.search(lexical_query(" Ownership ")).unwrap(); + assert_eq!(plain.len(), upper.len()); + assert_eq!(plain.len(), padded.len()); + // chunk_ids are deterministic — same query class, same set. + let plain_ids: Vec<_> = plain.iter().map(|h| h.chunk_id.0.clone()).collect(); + let upper_ids: Vec<_> = upper.iter().map(|h| h.chunk_id.0.clone()).collect(); + assert_eq!(plain_ids, upper_ids); +} + +/// p9-fb-19 — `--no-cache` (`search_uncached_with_config`) bypasses +/// the cache. Result correctness is identical to `search_with_config`. +#[test] +fn search_uncached_returns_same_hits_as_cached() { + let env = TestEnv::lexical_only(); + kebab_app::ingest_with_config(env.config.clone(), env.scope(), true).unwrap(); + let cached = + kebab_app::search_with_config(env.config.clone(), lexical_query("ownership")) + .unwrap(); + let uncached = kebab_app::search_uncached_with_config( + env.config.clone(), + lexical_query("ownership"), + ) + .unwrap(); + assert_eq!(cached.len(), uncached.len()); + for (a, b) in cached.iter().zip(uncached.iter()) { + assert_eq!(a.chunk_id, b.chunk_id); + } +} + +/// p9-fb-19 — first ingest with commits bumps `corpus_revision` from +/// 0 to ≥1. Verified by reading the persisted kv via a fresh +/// SqliteStore handle (the field on `App` is `pub(crate)`). +#[test] +fn first_ingest_bumps_corpus_revision() { + let env = TestEnv::lexical_only(); + let store_before = + kebab_store_sqlite::SqliteStore::open(&env.config).unwrap(); + store_before.run_migrations().unwrap(); + assert_eq!(store_before.corpus_revision(), 0, "fresh store seeds 0"); + + let report = + kebab_app::ingest_with_config(env.config.clone(), env.scope(), true).unwrap(); + assert!(report.new + report.updated > 0, "first ingest must commit ≥1 doc"); + + let store_after = + kebab_store_sqlite::SqliteStore::open(&env.config).unwrap(); + assert!( + store_after.corpus_revision() >= 1, + "ingest commit must bump corpus_revision (got {})", + store_after.corpus_revision(), + ); +} + #[test] fn vector_mode_with_provider_none_errors_clearly() { let env = TestEnv::lexical_only(); diff --git a/crates/kebab-cli/src/main.rs b/crates/kebab-cli/src/main.rs index a8ca2b1..97aaf5d 100644 --- a/crates/kebab-cli/src/main.rs +++ b/crates/kebab-cli/src/main.rs @@ -79,6 +79,16 @@ enum Cmd { #[arg(long)] explain: bool, + + /// p9-fb-19: bypass the in-process LRU search cache for + /// this invocation. Forces a fresh retriever run even when + /// the same query was just served from cache. Useful when + /// debugging retriever behavior — and a no-op for the CLI + /// (each invocation is a new process anyway, so the cache + /// starts empty), but the flag stays for parity with the + /// future TUI cache-aware search and for explicit intent. + #[arg(long)] + no_cache: bool, }, /// Retrieval-augmented question answering. @@ -392,6 +402,7 @@ fn run(cli: &Cli) -> anyhow::Result<()> { k, mode, explain: _, + no_cache, } => { let cfg = kebab_config::Config::load(cli.config.as_deref())?; let q = kebab_core::SearchQuery { @@ -400,7 +411,14 @@ fn run(cli: &Cli) -> anyhow::Result<()> { k: *k, filters: kebab_core::SearchFilters::default(), }; - let hits = kebab_app::search_with_config(cfg, q)?; + // p9-fb-19: --no-cache routes to the uncached facade. + // Both calls go through the same App; only the cache + // lookup/insert is skipped. + let hits = if *no_cache { + kebab_app::search_uncached_with_config(cfg, q)? + } else { + kebab_app::search_with_config(cfg, q)? + }; if cli.json { println!("{}", serde_json::to_string(&wire::wire_search_hits(&hits))?); } else { diff --git a/crates/kebab-config/src/lib.rs b/crates/kebab-config/src/lib.rs index e0e11d4..27d0d44 100644 --- a/crates/kebab-config/src/lib.rs +++ b/crates/kebab-config/src/lib.rs @@ -113,6 +113,16 @@ pub struct SearchCfg { pub hybrid_fusion: String, pub rrf_k: u32, pub snippet_chars: usize, + /// p9-fb-19: in-memory LRU cache capacity for `App::search`. + /// One entry ≈ 5 KB → default 256 caps memory at ~1.3 MB. Set + /// to `0` to disable the cache entirely. Stale entries + /// (corpus_revision mismatch) are evicted on next access. + #[serde(default = "default_cache_capacity")] + pub cache_capacity: usize, +} + +fn default_cache_capacity() -> usize { + 256 } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] @@ -295,6 +305,7 @@ impl Config { hybrid_fusion: "rrf".to_string(), rrf_k: 60, snippet_chars: 220, + cache_capacity: default_cache_capacity(), }, rag: RagCfg { prompt_template_version: "rag-v1".to_string(), diff --git a/crates/kebab-store-sqlite/src/store.rs b/crates/kebab-store-sqlite/src/store.rs index 58cf21a..ac8cac3 100644 --- a/crates/kebab-store-sqlite/src/store.rs +++ b/crates/kebab-store-sqlite/src/store.rs @@ -309,6 +309,63 @@ fn temp_path_for(dest: &Path) -> PathBuf { } impl SqliteStore { + /// p9-fb-19: read the persisted `corpus_revision` from the `kv` + /// table. Returns `0` if the row is missing (not migrated yet) or + /// unparseable — defensive: callers use the value as a cache-key + /// salt, never as an authority. + pub fn corpus_revision(&self) -> u64 { + let conn = self.read_conn(); + let row: rusqlite::Result = conn.query_row( + "SELECT value FROM kv WHERE key = 'corpus_revision'", + [], + |r| r.get(0), + ); + match row { + Ok(s) => s.parse().unwrap_or(0), + Err(rusqlite::Error::QueryReturnedNoRows) => 0, + Err(e) => { + tracing::warn!( + target: "kebab-store-sqlite", + error = %e, + "kv['corpus_revision'] read failed; defaulting to 0" + ); + 0 + } + } + } + + /// p9-fb-19: monotonically bump `corpus_revision` by one and + /// return the new value. Called by every `kebab-app::ingest` + /// path after a successful commit (any `new` / `updated`). + /// Atomic via SQLite's `UPDATE ... SET value = CAST(value AS + /// INTEGER) + 1` — no read-modify-write race. + pub fn bump_corpus_revision(&self) -> Result { + let conn = self.lock_conn(); + // INSERT-OR-IGNORE first to handle a fresh DB where the + // V004 seed hasn't run yet (paranoia — the migration always + // seeds, but SqliteStore's contract is "one method works + // even if the constructor was unusual"). Then bump. + conn.execute( + "INSERT OR IGNORE INTO kv (key, value) VALUES ('corpus_revision', '0')", + [], + ) + .map_err(StoreError::from)?; + conn.execute( + "UPDATE kv SET value = CAST(CAST(value AS INTEGER) + 1 AS TEXT) \ + WHERE key = 'corpus_revision'", + [], + ) + .map_err(StoreError::from)?; + let new_val: String = conn + .query_row( + "SELECT value FROM kv WHERE key = 'corpus_revision'", + [], + |r| r.get(0), + ) + .map_err(StoreError::from)?; + Ok(new_val.parse().unwrap_or(0)) + } + /// SELECT every `chunks.chunk_id` whose owning document points at a /// stale `asset_id` for `workspace_path` (i.e. the file's bytes have /// changed since the last ingest, producing a brand-new diff --git a/crates/kebab-store-sqlite/tests/corpus_revision.rs b/crates/kebab-store-sqlite/tests/corpus_revision.rs new file mode 100644 index 0000000..e590543 --- /dev/null +++ b/crates/kebab-store-sqlite/tests/corpus_revision.rs @@ -0,0 +1,54 @@ +//! p9-fb-19: `corpus_revision` kv counter — exposed on `SqliteStore` +//! so `kebab-app::ingest` can bump after a successful commit and +//! `App::search`'s LRU cache key can snapshot the current value for +//! invalidation. + +use kebab_config::Config; +use kebab_store_sqlite::SqliteStore; +use tempfile::TempDir; + +fn config_for(tmp: &TempDir) -> Config { + let mut c = Config::defaults(); + c.storage.data_dir = tmp.path().to_string_lossy().into_owned(); + c +} + +fn open_store(tmp: &TempDir) -> SqliteStore { + let cfg = config_for(tmp); + let store = SqliteStore::open(&cfg).unwrap(); + store.run_migrations().unwrap(); + store +} + +/// Fresh store seeds `corpus_revision = 0` (per V004 INSERT). +#[test] +fn fresh_store_starts_at_zero() { + let tmp = TempDir::new().unwrap(); + let store = open_store(&tmp); + assert_eq!(store.corpus_revision(), 0); +} + +/// Each `bump_corpus_revision` returns the new value monotonically. +#[test] +fn bump_increments_monotonically() { + let tmp = TempDir::new().unwrap(); + let store = open_store(&tmp); + assert_eq!(store.bump_corpus_revision().unwrap(), 1); + assert_eq!(store.bump_corpus_revision().unwrap(), 2); + assert_eq!(store.bump_corpus_revision().unwrap(), 3); + assert_eq!(store.corpus_revision(), 3); +} + +/// `corpus_revision` survives a store re-open (persisted in SQLite). +#[test] +fn revision_persists_across_reopen() { + let tmp = TempDir::new().unwrap(); + { + let store = open_store(&tmp); + store.bump_corpus_revision().unwrap(); + store.bump_corpus_revision().unwrap(); + } // store dropped — file closed + let store = open_store(&tmp); + assert_eq!(store.corpus_revision(), 2); + assert_eq!(store.bump_corpus_revision().unwrap(), 3); +} diff --git a/docs/SMOKE.md b/docs/SMOKE.md index 2f9c177..84af2bd 100644 --- a/docs/SMOKE.md +++ b/docs/SMOKE.md @@ -102,6 +102,7 @@ default_k = 10 hybrid_fusion = "rrf" rrf_k = 60 snippet_chars = 220 +cache_capacity = 256 # p9-fb-19 — in-process LRU cap; 0 disables, default 256 [rag] prompt_template_version = "rag-v1" diff --git a/docs/superpowers/specs/2026-04-27-kebab-final-form-design.md b/docs/superpowers/specs/2026-04-27-kebab-final-form-design.md index f5fe745..9a5d815 100644 --- a/docs/superpowers/specs/2026-04-27-kebab-final-form-design.md +++ b/docs/superpowers/specs/2026-04-27-kebab-final-form-design.md @@ -1324,6 +1324,7 @@ kebab-cli, kebab-tui, kebab-desktop | `embedding_model.version` | 같은 모델 가중치/토크나이저 변경 | bump | | `embedding.dimensions` | 차원 변경 | 새 lance 테이블 강제 | | `index_version` | retrieval 형상 변화 | bump | +| `corpus_revision` | ingest commit 발생 (ANY new/updated) | 모노토닉 u64, SQLite `kv['corpus_revision']` 에 영속. p9-fb-19 의 in-process LRU search cache 가 cache-key 에 snapshot 으로 포함 → 다음 lookup 에서 자동 무효화. | | `prompt_template_version` | template 변경 | 코드 상수 (`rag-v2`) | | DB `schema_version` | DDL 변경 | 마이그레이션 정수 증가 | | wire schema (`*.v1`) | 깨는 변경 시 | `*.v2` 신설, v1 additive only | diff --git a/migrations/V004__kv.sql b/migrations/V004__kv.sql new file mode 100644 index 0000000..293f4b5 --- /dev/null +++ b/migrations/V004__kv.sql @@ -0,0 +1,24 @@ +-- V004__kv.sql — single-row key/value table for monotonic counters. +-- +-- p9-fb-19 introduces an in-process LRU search cache; cache keys carry +-- a `corpus_revision` snapshot so a successful `kebab ingest` (which +-- bumps the counter) automatically invalidates every prior entry. +-- Persisting the counter in SQLite (rather than holding it in memory) +-- means a fresh process picks up the latest value, so a CLI invocation +-- after an ingest in another process correctly skips the stale cache. +-- +-- Schema is a generic `key/value` so future scalars (last_compaction, +-- last_doctor_run, ...) can land here without another migration. The +-- value column is TEXT because SQLite has no opinion on integer width +-- and downstream code can parse `u64` / `i64` / strings as needed. +-- +-- Seed `corpus_revision = 0` so the first cache miss after a fresh +-- install gets a defined snapshot; ingest's `bump_corpus_revision` +-- moves it to 1 on the first successful commit. + +CREATE TABLE kv ( + key TEXT PRIMARY KEY NOT NULL, + value TEXT NOT NULL +) STRICT; + +INSERT INTO kv (key, value) VALUES ('corpus_revision', '0'); diff --git a/tasks/p9/p9-fb-19-search-cache.md b/tasks/p9/p9-fb-19-search-cache.md index 422f88a..7583098 100644 --- a/tasks/p9/p9-fb-19-search-cache.md +++ b/tasks/p9/p9-fb-19-search-cache.md @@ -3,7 +3,7 @@ phase: P9 component: kebab-search + kebab-app task_id: p9-fb-19 title: "Search result cache (in-memory LRU + index_version invalidation)" -status: planned +status: in_progress depends_on: [] unblocks: [] contract_source: ../../docs/superpowers/specs/2026-04-27-kebab-final-form-design.md