test(fts,app): V009 morphological tokenizer integration tests

신규 4 test 추가:

- crates/kebab-store-sqlite/tests/fts.rs:
  - fts_v009_korean_morphological_2char_query_hits: tokenized_korean_text
    column 이 채워진 chunk 의 '한국' 2-char query hit.
  - fts_v009_english_whole_token_only: V007 trigram substring 매칭
    회귀 (Path A) — 'token' query 가 'tokenizer' chunk 에서 0-hit.
- crates/kebab-app/tests/search_korean.rs:
  - korean_morphological_2char_query_lexical_mode: end-to-end
    한국어 wiki fixture ingest → '한국' / '서울' query hit.
  - korean_morphological_mixed_english_korean_query: 'Rust' English
    whole-token + '최적화' Korean morpheme hit.

crates/kebab-search/src/lexical.rs:
  - build_match_string() 의 MIN_TRIGRAM_CHARS(3) → MIN_QUERY_CHARS(2).
    V009 unicode61 은 최소 token 길이 제한 없어 2자 한국어 morpheme
    query 가 통과되어야 함. 1자 단독은 여전히 필터.
  - 관련 unit test 2개 V009 동작으로 갱신.

fixture text 는 lindera ko-dic 의 실제 segmentation 동작에 의존
(spec Appendix B prior-knowledge 예측). 실측 시 fixture 조정 가능.

Spec: docs/superpowers/specs/2026-05-28-v0.20.x-korean-morphological-tokenizer-spec.md §9.1, §9.2
Plan: docs/superpowers/plans/2026-05-28-v0.20.x-korean-morphological-tokenizer-plan.md (S7)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-05-28 11:38:52 +00:00
parent f94e0c4a9b
commit c5de5f812b
3 changed files with 177 additions and 34 deletions

View File

@@ -581,3 +581,67 @@ fn fts_v009_unicode61_space_separated_korean_token_hits() {
// substring (token 의 부분 문자열) 은 V009 unicode61 에서 0-hit.
assert_eq!(count_match(&conn, "발생한"), 0, "substring '발생한' of '발생한다' 0-hit");
}
// ── 8. V009 morphological tokenizer behavior ──────────────────────────
/// V009 의 핵심 가치: 한국어 2자 query 가 hit. 형태소 분해된
/// tokenized_korean_text column 이 chunks_fts 에 indexed.
#[test]
fn fts_v009_korean_morphological_2char_query_hits() {
let env = common::TestEnv::new();
let store = SqliteStore::open(&env.config()).unwrap();
store.run_migrations().unwrap();
let conn = raw_conn_no_fk(&env);
let text = "한국 문화는 오래되었다";
let tokenized = tokenize_korean_morphological(text);
conn.execute(
"INSERT INTO chunks (
chunk_id, doc_id, text, heading_path_json, section_label,
source_spans_json, token_estimate, chunker_version,
policy_hash, block_ids_json, created_at,
tokenized_korean_text
) VALUES (?, ?, ?, '[]', NULL, '[]', 0, 'v1', 'h', '[]', '2024-01-01T00:00:00Z', ?)",
rusqlite::params![
&"k".repeat(32),
&"d".repeat(32),
text,
tokenized,
],
)
.expect("insert chunk with tokenized_korean_text");
assert!(
count_match(&conn, "한국") >= 1,
"2-char Korean morpheme '한국' must hit when tokenized column is populated"
);
}
/// V009 의 Path A 회귀 확인: 영어 substring 매칭이 사라짐
/// (unicode61 의 whole-token only 동작).
#[test]
fn fts_v009_english_whole_token_only() {
let env = common::TestEnv::new();
let store = SqliteStore::open(&env.config()).unwrap();
store.run_migrations().unwrap();
let conn = raw_conn_no_fk(&env);
insert_chunk(
&conn,
&"e".repeat(32),
&"d".repeat(32),
"[]",
"the tokenizer normalizes whitespace before matching",
);
assert_eq!(
count_match(&conn, "token"),
0,
"V009 unicode61: 'token' is substring of 'tokenizer', should NOT hit"
);
assert_eq!(
count_match(&conn, "tokenizer"),
1,
"V009 unicode61: whole-token 'tokenizer' must hit"
);
}