diff --git a/crates/kebab-search/src/lexical.rs b/crates/kebab-search/src/lexical.rs index f5c9080..09351e8 100644 --- a/crates/kebab-search/src/lexical.rs +++ b/crates/kebab-search/src/lexical.rs @@ -169,12 +169,26 @@ impl Retriever for LexicalRetriever { /// branch. Korean compounds typically split into 2-char eojeols (e.g. /// `해시 충돌`), so a naive token AND drops the dominant usage pattern. /// +/// post-v0.17.1 dogfood — `text` column filter (closure of HOTFIXES +/// 2026-05-24 `heading_path_json` 노이즈). The `chunks_fts` virtual +/// table indexes both `heading_path` (the JSON-serialized +/// `chunks.heading_path_json` per V002/V007 triggers) and `text`. Under +/// the trigram tokenizer the JSON punctuation (`[`, `"`, `,`) plus the +/// path segments (`app`, `src`, …) become indexable 3-grams, so a +/// query can hit a chunk purely because its file's heading JSON shares +/// a path segment with the query — false positives that have no body +/// relevance. The default match expression therefore scopes to the +/// `text` column. The `heading_path` column stays indexed (V007 / §5.5 +/// verbatim block is preserved) so a user who *wants* heading matching +/// can opt in via raw mode (`'heading_path : foo'`). +/// /// Rules: /// /// - Raw mode (unchanged): the query is wrapped in a single pair of /// `'...'` → strip the quotes and pass the inner text through verbatim. /// The user has explicitly opted into FTS5 syntax (e.g. -/// `'rust AND cargo'`, `'foo*'`). +/// `'rust AND cargo'`, `'foo*'`, `'heading_path : agent'`). No column +/// scoping is applied — the raw expression is honored as-is. /// /// - Otherwise build up to two MATCH candidates: /// 1. **whole-phrase**: the entire trimmed input wrapped as one FTS5 @@ -191,6 +205,10 @@ impl Retriever for LexicalRetriever { /// /// - A single-token long query (`러스트`, `foo`) yields `whole == token_and` /// → return the bare quoted form so the OR doesn't duplicate. +/// +/// - Finally wrap the combined expression in `text : ()` so the +/// match is scoped to the body column. FTS5's column-filter syntax +/// accepts an arbitrary OR/AND sub-expression inside the parens. fn build_match_string(text: &str) -> Option { let trimmed = text.trim(); if trimmed.is_empty() { @@ -218,13 +236,14 @@ fn build_match_string(text: &str) -> Option { (!toks.is_empty()).then(|| toks.join(" ")) }; - match (whole_candidate, token_and_candidate) { - (None, None) => None, - (Some(w), None) => Some(w), - (None, Some(a)) => Some(a), - (Some(w), Some(a)) if w == a => Some(w), - (Some(w), Some(a)) => Some(format!("({w}) OR ({a})")), - } + let expression = match (whole_candidate, token_and_candidate) { + (None, None) => return None, + (Some(w), None) => w, + (None, Some(a)) => a, + (Some(w), Some(a)) if w == a => w, + (Some(w), Some(a)) => format!("({w}) OR ({a})"), + }; + Some(format!("text : ({expression})")) } /// Return `Some(inner)` if `s` is wrapped in a matching pair of single @@ -587,9 +606,11 @@ mod tests { #[test] fn build_match_string_default_emits_or_of_phrase_and_and() { // Two long tokens: both whole-phrase and token-AND candidates - // exist and differ, so the builder combines them with OR. + // exist and differ, so the builder combines them with OR + // inside a `text : (...)` column filter (post-v0.17.1 dogfood: + // text-only scoping to avoid heading_path_json false positives). let s = build_match_string("rust cargo").unwrap(); - assert_eq!(s, r#"("rust cargo") OR ("rust" "cargo")"#); + assert_eq!(s, r#"text : (("rust cargo") OR ("rust" "cargo"))"#); } #[test] @@ -597,28 +618,41 @@ mod tests { // `*`, `(`, `)`, `:`, `^`, `"` should all be wrapped inside // FTS5 string-literal quotes so they're treated as literal // text rather than FTS5 operators. Every token is ≥3 chars, - // so both the whole-phrase and token-AND candidates exist. + // so both the whole-phrase and token-AND candidates exist, + // wrapped in the `text : (...)` column filter. let s = build_match_string(r#"foo* (bar) baz:qux ^head he"llo"#).unwrap(); assert_eq!( s, - r#"("foo* (bar) baz:qux ^head he""llo") OR ("foo*" "(bar)" "baz:qux" "^head" "he""llo")"# + r#"text : (("foo* (bar) baz:qux ^head he""llo") OR ("foo*" "(bar)" "baz:qux" "^head" "he""llo"))"# ); // The doubled `""` is FTS5's way of embedding a literal quote // inside a string literal. Appears in both whole-phrase and // token-AND halves. assert!(s.contains(r#"he""llo"#)); - // Sanity: the combined expression is `(...) OR (...)` so it - // starts with `(` and ends with `)`. - assert!(s.starts_with('(') && s.ends_with(')')); + // Sanity: outermost wrapper is the column filter. + assert!(s.starts_with("text : (")); + assert!(s.ends_with(')')); } #[test] fn build_match_string_passthrough_when_single_quoted() { - // The FTS5 expression is preserved verbatim. + // Raw mode bypasses column scoping — the FTS5 expression is + // preserved verbatim, including any explicit column filter + // (e.g. `'heading_path : foo'`) the user opts into. let s = build_match_string("'foo OR bar*'").unwrap(); assert_eq!(s, "foo OR bar*"); } + /// Raw mode preserves an explicit `heading_path :` column filter + /// — opt-in path for users who deliberately want heading matching + /// (post-v0.17.1 dogfood default scopes to `text` only). + #[test] + fn build_match_string_raw_mode_preserves_heading_filter() { + let s = build_match_string("'heading_path : agent'").unwrap(); + assert_eq!(s, "heading_path : agent"); + assert!(!s.starts_with("text : ")); + } + // ── v0.17.0 trigram-aware redesign coverage ────────────────────────── /// 2-char Korean query (`충돌`) yields neither a whole-phrase nor a @@ -634,38 +668,43 @@ mod tests { /// `해시 충돌` — both tokens are 2 chars (dropped from the AND), but /// the whole-phrase candidate (`"해시 충돌"`, 5 chars total) survives. /// This is the dominant Korean usage pattern targeted by A5. + /// The whole-phrase candidate is then wrapped in the `text : (...)` + /// column filter. #[test] fn build_match_string_whole_phrase_only_when_all_tokens_short() { let s = build_match_string("해시 충돌").unwrap(); - assert_eq!(s, r#""해시 충돌""#); + assert_eq!(s, r#"text : ("해시 충돌")"#); } /// Single long token: whole-phrase and token-AND candidates collapse /// to the same string. The builder returns the bare quoted form so - /// the MATCH expression doesn't carry a redundant `(x) OR (x)`. + /// the MATCH expression doesn't carry a redundant `(x) OR (x)`, + /// wrapped in `text : (...)`. #[test] fn build_match_string_single_long_token_no_duplicate_or() { - assert_eq!(build_match_string("러스트").unwrap(), r#""러스트""#); - assert_eq!(build_match_string("rust").unwrap(), r#""rust""#); + assert_eq!(build_match_string("러스트").unwrap(), r#"text : ("러스트")"#); + assert_eq!(build_match_string("rust").unwrap(), r#"text : ("rust")"#); } /// Mixed Korean+English multi-token query where every token is ≥3 - /// chars: both candidates exist and differ, OR-combined. + /// chars: both candidates exist and differ, OR-combined inside + /// `text : (...)`. #[test] fn build_match_string_mixed_lang_emits_or_of_phrase_and_and() { let s = build_match_string("Rust 충돌은").unwrap(); - assert_eq!(s, r#"("Rust 충돌은") OR ("Rust" "충돌은")"#); + assert_eq!(s, r#"text : (("Rust 충돌은") OR ("Rust" "충돌은"))"#); } /// One ≥3 token + one <3 token: short token is dropped from the /// AND, leaving a single long token there; whole-phrase exists - /// independently. Both candidates differ → OR-combined. + /// independently. Both candidates differ → OR-combined inside + /// `text : (...)`. #[test] fn build_match_string_drops_short_token_in_and_keeps_whole() { // "키" (1 char) dropped from AND; "해시테이블" (5 chars) kept. // Whole phrase "키 해시테이블" (7 chars) keeps the short token. let s = build_match_string("키 해시테이블").unwrap(); - assert_eq!(s, r#"("키 해시테이블") OR ("해시테이블")"#); + assert_eq!(s, r#"text : (("키 해시테이블") OR ("해시테이블"))"#); } #[test] diff --git a/crates/kebab-search/tests/lexical.rs b/crates/kebab-search/tests/lexical.rs index 7fcf5a5..8206265 100644 --- a/crates/kebab-search/tests/lexical.rs +++ b/crates/kebab-search/tests/lexical.rs @@ -1060,3 +1060,99 @@ fn lexical_snapshot_run_1() { let expected: serde_json::Value = serde_json::from_str(&baseline_text).unwrap(); assert_eq!(actual, expected, "lexical run-1 snapshot drift"); } + +// ── post-v0.17.1 dogfood — `text` column filter ────────────────────────── + +/// Heading-only token (unique to `chunks.heading_path_json`, absent +/// from `chunks.text`) must NOT hit in default mode after the column +/// filter clamp. Pins HOTFIXES 2026-05-24 closure — the JSON +/// punctuation + path segments in `heading_path_json` are no longer +/// matchable from a plain query. +#[test] +fn lexical_heading_only_token_does_not_hit_default_mode() { + let env = Env::new(); + let conn = env.raw_conn(); + insert_document( + &conn, + &id32("d"), + "notes/heading-only.md", + "Heading-only fixture", + "en", + "primary", + &[], + ); + insert_chunk( + &conn, + &id32("c1"), + &id32("d"), + "bravo charlie delta echo", + &["kubernetes-agent-controller"], + Some("Heading"), + r#"[{"kind":"line","start":1,"end":2}]"#, + "v1", + ); + drop(conn); + + let r = env.retriever(); + let hits = r + .search(&SearchQuery { + // "kubernetes-agent-controller" is in heading_path only. + text: "kubernetes-agent-controller".to_string(), + mode: SearchMode::Lexical, + k: 10, + filters: SearchFilters::default(), + }) + .unwrap(); + assert!( + hits.is_empty(), + "heading-only token must not hit text column; got {} hits", + hits.len() + ); +} + +/// Raw mode (`'heading_path : '`) is the opt-in escape hatch +/// for users who deliberately want heading-column matching after the +/// default text-only clamp. The same fixture that 0-hits in default +/// mode must hit when the user explicitly scopes to `heading_path`. +#[test] +fn lexical_raw_mode_can_opt_into_heading_path_filter() { + let env = Env::new(); + let conn = env.raw_conn(); + insert_document( + &conn, + &id32("d"), + "notes/heading-only.md", + "Heading-only fixture", + "en", + "primary", + &[], + ); + insert_chunk( + &conn, + &id32("c1"), + &id32("d"), + "bravo charlie delta echo", + &["kubernetes-agent-controller"], + Some("Heading"), + r#"[{"kind":"line","start":1,"end":2}]"#, + "v1", + ); + drop(conn); + + let r = env.retriever(); + let hits = r + .search(&SearchQuery { + // Raw mode: outer single quotes opt out of column-filter + // wrapping and pass the FTS5 expression through verbatim. + text: "'heading_path : \"kubernetes-agent-controller\"'".to_string(), + mode: SearchMode::Lexical, + k: 10, + filters: SearchFilters::default(), + }) + .unwrap(); + assert_eq!( + hits.len(), + 1, + "raw-mode heading_path filter must hit the seeded chunk" + ); +} diff --git a/tasks/HOTFIXES.md b/tasks/HOTFIXES.md index 0f4b07f..63498bd 100644 --- a/tasks/HOTFIXES.md +++ b/tasks/HOTFIXES.md @@ -49,6 +49,29 @@ v0.17.1 entry 의 첫 번째 미진행 항목 closure. LLM 쪽이 v0.17.1 에서 Cross-link: `crates/kebab-config/src/lib.rs::OcrCfg::request_timeout_secs`, `crates/kebab-parse-image/src/ocr.rs::OllamaVisionOcr::build`. +## 2026-05-25 — post-v0.17.1 dogfood: `heading_path` FTS5 column filter (text-only matching, closure of 2026-05-24 `heading_path_json` 노이즈) + +v0.17.0 의 한국어 trigram tokenizer 채택 entry (2026-05-24 위) 가 미수정으로 남겨둔 `heading_path_json` JSON 노이즈 closure. trigram 이 `chunks_fts.heading_path` 컬럼 (V002/V007 트리거가 `chunks.heading_path_json` 을 그대로 INSERT) 의 JSON 표기 (`[`, `"`, `,`) + 안의 path 세그먼트 (`app`, `src`) 까지 3-gram 색인해서 query 가 우연히 false positive hit 하는 문제. 사용자 결정 (column filter vs 평문 heading 변환): **column filter** — `heading_path` 색인은 V007 verbatim 그대로 유지, 매칭 대상만 `text` 컬럼으로 한정. V008 migration / design §5.5 verbatim 블록 변경 불필요. + +**변경**: +- `crates/kebab-search/src/lexical.rs::build_match_string` 가 non-raw 분기에서 combined expression 을 `text : ()` 로 wrap. FTS5 column filter syntax (`column:expr`) 가 OR/AND sub-expression 허용 — 한국어 trigram 빌더의 `(whole) OR (token_and)` 형태가 그대로 들어감. +- Raw mode (`'...'`) 는 변경 없음 — 사용자가 명시 의도로 `'heading_path : agent'` 같은 explicit column filter opt-in 가능 (escape hatch). +- 9 신규 / 갱신 unit test: + - `build_match_string_*` 8 expected string 갱신 (column filter prefix 추가) + - `build_match_string_raw_mode_preserves_heading_filter` 신규 — raw mode 가 `heading_path : ...` 보존 + - `lexical_heading_only_token_does_not_hit_default_mode` 신규 (`crates/kebab-search/tests/lexical.rs`) — heading-only unique token 이 default mode 에서 0 hit + - `lexical_raw_mode_can_opt_into_heading_path_filter` 신규 — 같은 fixture 가 raw mode 로 hit 확인 + +**사용자 영향**: +- 기본 lexical / hybrid 검색에서 heading 만 매칭되던 false positive 차단. 한국어 / 영어 substring 매칭의 recall 은 그대로 (text 본문에 있는 token 은 변함없이 hit). 본문 검색의 precision 가 올라감. +- heading 으로 일부러 검색하던 사용자는 `'heading_path : '` 형태로 raw mode 진입. CLI / TUI / MCP 모든 surface 동일. +- `kebab.sqlite` 크기 변화 없음 (색인 column 그대로 유지). re-ingest 불필요 (FTS query 시점의 매칭 범위만 변경). +- BM25 score 영향: `lexical_snapshot_run_1` + `hybrid_snapshot_run_1` 둘 다 column filter 적용 후에도 점수 동일 (text 본문에만 매칭되던 query 라 column filter 가 점수 분포에 영향 안 줌). fixture regenerate 불필요. + +**MCP / agent 가시성**: `search_response.v1` 의 wire shape 변경 없음. 사용자가 heading 검색을 명시 의도하던 케이스는 raw mode 안내 — `integrations/claude-code/kebab/SKILL.md` 의 search 절은 v0.17.0 의 raw mode 안내 (`'foo OR bar*'`) 가 그대로 적용. 별도 SKILL.md 갱신 불필요 (raw mode 가 이미 documented escape hatch). + +Cross-link: `crates/kebab-search/src/lexical.rs::build_match_string`, `migrations/V007__fts_trigram.sql` (verbatim 유지), design §5.5 (verbatim 유지, query-time 동작만 변경). + ## 2026-05-24 — v0.17.0: 한국어 trigram FTS5 tokenizer 채택 (closure of 2026-05-22 한국어 lexical) V007 migration 으로 `chunks_fts` 의 tokenizer 를 `unicode61` → `trigram` 으로 교체. `chunks` 원본 + embedding + vector index 는 그대로, FTS shadow 만 재구축 + 자동 backfill — 사용자는 `kebab ingest` 재실행 불필요 (binary 만 교체하면 다음 open 시 V007 가 즉시 적용). 같은 라운드의 다른 두 follow-up (`code_lang_chunk_breakdown`, C typedef) 은 별 PR (PR-C / PR-B). @@ -61,7 +84,7 @@ V007 migration 으로 `chunks_fts` 의 tokenizer 를 `unicode61` → `trigram` **디스크 용량**: trigram 인덱스는 unicode61 대비 통상 2-10배. V007 자동 backfill 후 `kebab.sqlite` 파일 크기 증가 (도그푸딩 KB 기준 ~2-5배 또는 수백 MB). release notes 명시. -**`heading_path_json` JSON 노이즈 (관찰, 미수정)**: trigram 이 JSON 표기 (`[`, `"`, `,`) 와 그 안의 단어 (`app`, `src`) 까지 3-gram 색인 → query 가 우연히 JSON 구문 / 흔한 경로 단어와 겹쳐 false positive 가능. v0.17.0 에서는 컬럼 구성 유지, 도그푸딩 후 column filter (`{text} : ` 한정) 또는 평문 heading 변환 결정. 후속 도그푸딩 entry 로 등재 예정. +**`heading_path_json` JSON 노이즈 (관찰, 미수정)**: trigram 이 JSON 표기 (`[`, `"`, `,`) 와 그 안의 단어 (`app`, `src`) 까지 3-gram 색인 → query 가 우연히 JSON 구문 / 흔한 경로 단어와 겹쳐 false positive 가능. v0.17.0 에서는 컬럼 구성 유지, 도그푸딩 후 column filter (`{text} : ` 한정) 또는 평문 heading 변환 결정. 후속 도그푸딩 entry 로 등재 예정. → **closure**: 아래 2026-05-25 v0.17.1 post-dogfood heading text column filter entry 참조 (column filter 방식 채택, V008 migration 불필요). **MCP / agent 가시성**: `search_response.v1` 에 `hint: Option` additive 필드. 결과가 비어 있고 query trimmed.chars().count() < 3 + raw mode 아닐 때만 set (helper `kebab_app::short_query_hint`). `integrations/claude-code/kebab/SKILL.md` 의 search 절에 "한국어 lexical 은 3자 이상 권장, `hint` 필드 확인" 안내 추가.