altair823
0e408fb1b5
도그푸딩 item 15 — TUI / 같은 process 안에서 동일 query 반복 시 SQLite FTS + Lance + RRF 재계산이 매번 발생하던 비용 해소. in-process LRU 캐시 + 모노토닉 corpus_revision 카운터로 ingest commit 발생 시 모든 entry 자동 stale. ## 핵심 변경 - **SQLite V004 migration**: `kv (key TEXT PRIMARY KEY, value TEXT) STRICT` + `corpus_revision = '0'` seed. 미래의 다른 scalar 도 같은 테이블에 들어갈 수 있는 generic shape. - **`SqliteStore::corpus_revision()` / `bump_corpus_revision()`** — `UPDATE ... CAST AS INTEGER + 1` atomic. INSERT-OR-IGNORE 도 함께 실행 (V004 seed 가 무슨 이유로 누락된 케이스 paranoid). - **`kebab-app::ingest_with_config_cancellable`** — `new + updated > 0` 시 bump, no-op (skipped-only) reingest 는 cache 보존. - **`App.search_cache: Option<Mutex<LruCache<SearchCacheKey, Vec< SearchHit>>>>`** — `config.search.cache_capacity` (default 256, 0 비활성). `lru = "0.12"` workspace dep 추가. - **`SearchCacheKey`** = `query_norm` (NFKC + trim + lowercase) + `mode` + `k` + `snippet_chars` + `embedding_version` (vector/hybrid 만, lexical 은 빈 문자열) + `chunker_version` + `corpus_revision` snapshot. - **`App::search`** rewrite — cache 활성 시 lookup → miss 면 기존 `search_uncached` 호출 후 put. cache 비활성이거나 lock 실패면 straight-line. - **`App::search_uncached`** (rename of pre-fb-19 `search` body) + `search_uncached_with_config` facade — CLI `kebab search --no-cache` 로 진입. - **`Config.search.cache_capacity: usize`** field, `#[serde(default)]` 로 기존 config 호환. - **CLI `--no-cache`** flag — 디버깅용 (CLI 는 매 호출이 새 process 라 사실상 no-op 이지만 spec 명시 + 향후 long-lived process 호환). - **frozen design §9 versioning** 표에 `corpus_revision` row 추가 (기존 `index_version` 라벨과 다른 차원: 라벨은 retrieval 형상, corpus_revision 은 ingest commit ack). ## 테스트 - `kebab-store-sqlite` 신규 3 unit (fresh=0, monotonic bump, persist across reopen) - `kebab-app` 신규 4 integration (cached repeat 같은 hits, NFKC 정규화 로 case/whitespace collapse, --no-cache parity, first ingest bumps corpus_revision) - 워크스페이스 전체 `cargo test --workspace --no-fail-fast -j 1` exit 0 - `cargo clippy --workspace --all-targets -- -D warnings` clean ## 문서 - README `kebab search` 행: 캐시 동작 + `--no-cache` 안내 + corpus_ revision 무효화 메커니즘 - docs/SMOKE.md `[search]` 절에 `cache_capacity` 라인 추가 - HANDOFF: 2026-05-03 entry - spec status planned → in_progress ## Out of scope - patch-and-merge incremental (RRF 정규화 전체 hit set 기준이라 어려움) - SQLite 영속 cache (P+) - 다른 process 간 cache 공유 (in-process 만 — corpus_revision 이 cross-process 무효화는 O(1)) Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
92 lines
3.3 KiB
TOML
92 lines
3.3 KiB
TOML
[workspace]
|
|
resolver = "3"
|
|
members = [
|
|
"crates/kebab-core",
|
|
"crates/kebab-parse-types",
|
|
"crates/kebab-config",
|
|
"crates/kebab-source-fs",
|
|
"crates/kebab-parse-md",
|
|
"crates/kebab-normalize",
|
|
"crates/kebab-chunk",
|
|
"crates/kebab-store-sqlite",
|
|
"crates/kebab-store-vector",
|
|
"crates/kebab-search",
|
|
"crates/kebab-embed",
|
|
"crates/kebab-embed-local",
|
|
"crates/kebab-llm",
|
|
"crates/kebab-llm-local",
|
|
"crates/kebab-rag",
|
|
"crates/kebab-app",
|
|
"crates/kebab-cli",
|
|
"crates/kebab-eval",
|
|
"crates/kebab-parse-image",
|
|
"crates/kebab-parse-pdf",
|
|
"crates/kebab-tui",
|
|
]
|
|
|
|
[workspace.package]
|
|
edition = "2024"
|
|
rust-version = "1.85"
|
|
license = "MIT OR Apache-2.0"
|
|
repository = "https://github.com/altair823/kebab"
|
|
version = "0.1.0"
|
|
|
|
[workspace.dependencies]
|
|
anyhow = "1"
|
|
thiserror = "2"
|
|
serde = { version = "1", features = ["derive"] }
|
|
serde_json = "1"
|
|
# Golden-fixture loader (P5-1, kebab-eval) parses YAML; pinned in the
|
|
# workspace so future eval-adjacent crates share the same major.
|
|
serde_yaml = "0.9"
|
|
time = { version = "0.3", features = ["serde", "macros", "formatting", "parsing"] }
|
|
uuid = { version = "1", features = ["v7", "serde"] }
|
|
blake3 = "1"
|
|
tracing = "0.1"
|
|
# `bundled` ships SQLite source so the workspace doesn't depend on a
|
|
# system libsqlite3 (matches the kebab-store-sqlite feature set).
|
|
rusqlite = { version = "0.32", features = ["bundled"] }
|
|
globset = "0.4"
|
|
tempfile = "3"
|
|
proptest = "1"
|
|
# p9-fb-19: LRU cache for `App::search` results. Bounded capacity
|
|
# from `config.search.cache_capacity` (default 256, ~1.3 MB cap).
|
|
lru = "0.12"
|
|
# fastembed-rs ships ONNX runtime via the `ort-download-binaries` feature
|
|
# in its default set (which also pulls `hf-hub` for first-run model
|
|
# downloads). Pinned to the 4.x line per task p3-2 (current 5.x release
|
|
# remains untested for this workspace).
|
|
fastembed = "4.9"
|
|
# LanceDB embedded vector store (P3-3). 0.23.x pulls arrow / arrow-array /
|
|
# arrow-schema 56.x transitively (via lance 1.0); the kebab-store-vector
|
|
# crate matches that major to share the same Arrow types without a
|
|
# re-export adapter.
|
|
lancedb = { version = "0.23", default-features = false }
|
|
arrow = "56"
|
|
arrow-array = "56"
|
|
arrow-schema = "56"
|
|
tokio = { version = "1", features = ["rt", "macros"] }
|
|
futures = "0.3"
|
|
# Strict citation-marker extraction in kebab-rag (P4-3) needs a single regex
|
|
# pass; pulled into the workspace deps so future crates can share the
|
|
# same major.
|
|
regex = "1"
|
|
# Dev-only HTTP mock server for kebab-llm-local Ollama adapter tests. Requires
|
|
# a tokio runtime to host its mock server (the runtime adapter crate stays
|
|
# sync via reqwest::blocking — wiremock is dev-only there).
|
|
wiremock = "0.6"
|
|
|
|
# Disk-footprint trim for dev / test builds. Codegen, opt-level, and
|
|
# behavior are unchanged — only DWARF debug info is reduced (line
|
|
# numbers kept, column numbers dropped) and split into separate
|
|
# `.dwo` files. backtrace stays useful (function + line). release
|
|
# profile is untouched, so CI / `--release` runs are byte-identical
|
|
# to upstream defaults.
|
|
[profile.dev]
|
|
debug = "line-tables-only"
|
|
split-debuginfo = "unpacked"
|
|
|
|
[profile.test]
|
|
debug = "line-tables-only"
|
|
split-debuginfo = "unpacked"
|