refactor: audit-tail 정리 — FusionPolicy inline + dual-YAML 통일 + NLI required
ponytail-audit 의 작은 over-engineering 잔여 3건 (output/동작 불변):
1. FusionPolicy 1-arm enum 제거 — kebab-search 내부 전용(외부 importer 0)이던
`enum FusionPolicy { Rrf { k_rrf } }` 를 `HybridRetriever.k_rrf: u32` 로 inline.
parse_fusion 은 u32 반환. RRF 수식 `Σ 1/(k_rrf+rank)` byte-identical.
2. dual YAML 통일 — serde_yaml(0.9) 를 maintained fork serde_yaml_ng(0.10) 로 일원화.
k8s manifest chunker + eval golden loader 2 site 마이그레이션(API drop-in),
workspace dep 에서 serde_yaml 제거. parse 결과 불변.
3. NliVerifier::hypothesis_token_count 의 `Ok(0)` default 제거 → required.
prod OnnxNliVerifier 가 이미 override 라 동작 불변, MockNliVerifier 에 explicit 추가.
문서 동기화: search/eval/parse component README + ARCHITECTURE 의 FusionPolicy/
serde_yaml 참조를 현 상태로 갱신. net −37줄. 적대적 검증 2렌즈(output-identity +
completeness-build) 통과, 영향 crate 테스트 + clippy -D warnings 클린.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012Mc6W1fgsrbFKTsqA6P8La
This commit is contained in:
17
Cargo.lock
generated
17
Cargo.lock
generated
@@ -4291,7 +4291,7 @@ dependencies = [
|
||||
"lindera-ko-dic",
|
||||
"serde_json",
|
||||
"serde_json_canonicalizer",
|
||||
"serde_yaml",
|
||||
"serde_yaml_ng",
|
||||
"time",
|
||||
"tracing",
|
||||
]
|
||||
@@ -4387,7 +4387,7 @@ dependencies = [
|
||||
"rusqlite",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_yaml",
|
||||
"serde_yaml_ng",
|
||||
"tempfile",
|
||||
"time",
|
||||
"tracing",
|
||||
@@ -7794,19 +7794,6 @@ dependencies = [
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_yaml"
|
||||
version = "0.9.34+deprecated"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47"
|
||||
dependencies = [
|
||||
"indexmap 2.14.0",
|
||||
"itoa",
|
||||
"ryu",
|
||||
"serde",
|
||||
"unsafe-libyaml",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_yaml_ng"
|
||||
version = "0.10.0"
|
||||
|
||||
@@ -123,9 +123,6 @@ anyhow = "1"
|
||||
thiserror = "2"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
# Golden-fixture loader (P5-1, kebab-eval) parses YAML; pinned in the
|
||||
# workspace so future eval-adjacent crates share the same major.
|
||||
serde_yaml = "0.9"
|
||||
time = { version = "0.3", features = ["serde", "macros", "formatting", "parsing"] }
|
||||
uuid = { version = "1", features = ["v7", "serde"] }
|
||||
blake3 = "1"
|
||||
|
||||
@@ -13,7 +13,7 @@ serde_json_canonicalizer = "0.3"
|
||||
blake3 = { workspace = true }
|
||||
anyhow = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
serde_yaml = { workspace = true }
|
||||
serde_yaml_ng = "0.10"
|
||||
lindera = { workspace = true, features = ["embed-ko-dic"] }
|
||||
lindera-ko-dic = { workspace = true, features = ["embed-ko-dic"] }
|
||||
|
||||
|
||||
@@ -36,7 +36,7 @@ impl Chunker for K8sManifestResourceV1Chunker {
|
||||
|
||||
for slice in slices {
|
||||
// Invalid YAML in any document → return 0 chunks for the file.
|
||||
let value: serde_yaml::Value = match serde_yaml::from_str(slice.text) {
|
||||
let value: serde_yaml_ng::Value = match serde_yaml_ng::from_str(slice.text) {
|
||||
Ok(v) => v,
|
||||
Err(_) => return Ok(vec![]),
|
||||
};
|
||||
|
||||
@@ -156,7 +156,7 @@ fn k8s_multi_doc_emits_one_chunk_per_resource() {
|
||||
/// must cause the chunker to return 0 chunks for the entire file.
|
||||
#[test]
|
||||
fn k8s_invalid_yaml_emits_zero_chunks() {
|
||||
// serde_yaml 0.9 is lenient about duplicate keys (last wins), so use a
|
||||
// serde_yaml_ng is lenient about duplicate keys (last wins), so use a
|
||||
// genuine YAML structural error (unclosed flow sequence) to force a parse
|
||||
// failure.
|
||||
let actually_bad = "apiVersion: v1\nkind: Service\nfoo: [\nbar\n";
|
||||
|
||||
@@ -15,7 +15,7 @@ kebab-app = { path = "../kebab-app" }
|
||||
kebab-store-sqlite = { path = "../kebab-store-sqlite" }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
serde_yaml = { workspace = true }
|
||||
serde_yaml_ng = "0.10"
|
||||
time = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
anyhow = { workspace = true }
|
||||
|
||||
@@ -11,7 +11,7 @@
|
||||
//! ## Allowed deps (per task spec)
|
||||
//!
|
||||
//! `kb-core`, `kb-config`, `kb-app`, `kb-store-sqlite`, plus `serde`,
|
||||
//! `serde_yaml`, `serde_json`, `time`, `tracing`,
|
||||
//! `serde_yaml_ng`, `serde_json`, `time`, `tracing`,
|
||||
//! `anyhow`, `uuid`. Retrieval / embedding / LLM crates are NOT
|
||||
//! reachable here — every retrieval and `ask` call must go through
|
||||
//! `kb-app`.
|
||||
|
||||
@@ -27,7 +27,7 @@ use crate::types::GoldenQuery;
|
||||
pub fn load_golden_set(path: &Path) -> Result<Vec<GoldenQuery>> {
|
||||
let bytes =
|
||||
std::fs::read(path).with_context(|| format!("read golden YAML from {}", path.display()))?;
|
||||
let queries: Vec<GoldenQuery> = serde_yaml::from_slice(&bytes)
|
||||
let queries: Vec<GoldenQuery> = serde_yaml_ng::from_slice(&bytes)
|
||||
.with_context(|| format!("parse golden YAML at {}", path.display()))?;
|
||||
check_unique_ids(&queries)?;
|
||||
check_group_integrity(&queries)?;
|
||||
|
||||
@@ -470,7 +470,7 @@ fn lang_default_is_used_when_omitted_in_yaml() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let golden = tmp.path().join("g.yaml");
|
||||
fs::write(&golden, yaml).unwrap();
|
||||
let qs: Vec<GoldenQuery> = serde_yaml::from_str(yaml).unwrap();
|
||||
let qs: Vec<GoldenQuery> = serde_yaml_ng::from_str(yaml).unwrap();
|
||||
assert_eq!(qs.len(), 1);
|
||||
assert_eq!(qs[0].lang, Lang(String::new()));
|
||||
}
|
||||
|
||||
@@ -53,15 +53,10 @@ pub trait NliVerifier: Send + Sync {
|
||||
/// 이 API 로 mDeBERTa-v3 의 `OnlyFirst` dead-end (hypothesis 단독이
|
||||
/// 512-token cap 초과 시 truncate 불가) 를 회피.
|
||||
///
|
||||
/// **Default impl 반환 `Ok(0)`** — 기존 mock implementations
|
||||
/// (`MockNliVerifier` 등) 가 trait 확장 후에도 backward-compat
|
||||
/// (compile fail 회피, retry loop immediate 통과). `OnnxNliVerifier`
|
||||
/// 는 real tokenizer 로 *trait impl 블록 안에서* override 해야 함
|
||||
/// — inherent method 는 vtable 미등록 → trait dispatch 시 default
|
||||
/// 호출 → production silent NO-OP.
|
||||
fn hypothesis_token_count(&self, _hypothesis: &str) -> anyhow::Result<usize> {
|
||||
Ok(0)
|
||||
}
|
||||
/// Required: `OnnxNliVerifier` 는 real tokenizer 로 *trait impl 블록
|
||||
/// 안에서* 구현해야 함 — inherent method 는 vtable 미등록 → trait
|
||||
/// dispatch 시 호출 안 됨 → production silent NO-OP.
|
||||
fn hypothesis_token_count(&self, hypothesis: &str) -> anyhow::Result<usize>;
|
||||
}
|
||||
|
||||
/// Numerically stable 3-way softmax (subtract max for log-sum-exp safety).
|
||||
|
||||
@@ -454,6 +454,12 @@ impl NliVerifier for MockNliVerifier {
|
||||
MockMode::Err(e) => anyhow::bail!("{e}"),
|
||||
}
|
||||
}
|
||||
|
||||
// Mock doesn't model tokenization; preserves the old trait default
|
||||
// (Ok(0)) so the char-budget retry loop passes through immediately.
|
||||
fn hypothesis_token_count(&self, _hypothesis: &str) -> anyhow::Result<usize> {
|
||||
Ok(0)
|
||||
}
|
||||
}
|
||||
|
||||
/// S3 follow-up (2026-05-26): closure type aliases for `SpyNliVerifier`
|
||||
|
||||
@@ -43,17 +43,6 @@ const HYBRID_FANOUT_MULTIPLIER: usize = 2;
|
||||
/// Default `k` when `SearchQuery::k == 0`. Mirrors §6.4 default_k=10.
|
||||
const DEFAULT_K: usize = 10;
|
||||
|
||||
/// Fusion algorithm. Today only Reciprocal Rank Fusion is supported;
|
||||
/// listing as an enum so future score-calibration policies (P+) can
|
||||
/// land without an API break.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub enum FusionPolicy {
|
||||
/// Reciprocal Rank Fusion. `k_rrf` is the standard rank-bias
|
||||
/// hyperparameter (§6.4); larger values flatten the rank-bias
|
||||
/// curve, smaller values privilege top-of-list hits.
|
||||
Rrf { k_rrf: u32 },
|
||||
}
|
||||
|
||||
/// Hybrid retriever composing a lexical and a vector retriever.
|
||||
///
|
||||
/// For chunks that appear in both retrievers, the lexical-side hit
|
||||
@@ -66,7 +55,7 @@ pub enum FusionPolicy {
|
||||
pub struct HybridRetriever {
|
||||
lexical: Arc<dyn Retriever>,
|
||||
vector: Arc<dyn Retriever>,
|
||||
fusion: FusionPolicy,
|
||||
k_rrf: u32,
|
||||
/// Default `k` for queries that arrive with `k == 0`. Pulled from
|
||||
/// `config.search.default_k` at construction.
|
||||
default_k: usize,
|
||||
@@ -81,7 +70,7 @@ impl HybridRetriever {
|
||||
lexical: Arc<dyn Retriever>,
|
||||
vector: Arc<dyn Retriever>,
|
||||
) -> Self {
|
||||
let fusion = parse_fusion(&search.hybrid_fusion, search.rrf_k);
|
||||
let k_rrf = parse_fusion(&search.hybrid_fusion, search.rrf_k);
|
||||
let default_k = if search.default_k == 0 {
|
||||
DEFAULT_K
|
||||
} else {
|
||||
@@ -104,7 +93,7 @@ impl HybridRetriever {
|
||||
Self {
|
||||
lexical,
|
||||
vector,
|
||||
fusion,
|
||||
k_rrf,
|
||||
default_k,
|
||||
}
|
||||
}
|
||||
@@ -114,13 +103,13 @@ impl HybridRetriever {
|
||||
pub fn with_policy(
|
||||
lexical: Arc<dyn Retriever>,
|
||||
vector: Arc<dyn Retriever>,
|
||||
fusion: FusionPolicy,
|
||||
k_rrf: u32,
|
||||
default_k: usize,
|
||||
) -> Self {
|
||||
Self {
|
||||
lexical,
|
||||
vector,
|
||||
fusion,
|
||||
k_rrf,
|
||||
default_k: if default_k == 0 { DEFAULT_K } else { default_k },
|
||||
}
|
||||
}
|
||||
@@ -219,7 +208,7 @@ impl HybridRetriever {
|
||||
// the same positive constant), so the sort + tiebreak path is
|
||||
// unchanged. Wire schema label `fusion_score` keeps its slot in
|
||||
// `RetrievalDetail`; only the magnitude shifts.
|
||||
let FusionPolicy::Rrf { k_rrf } = self.fusion;
|
||||
let k_rrf = self.k_rrf;
|
||||
let k_rrf_f = f64::from(k_rrf);
|
||||
// Both retrievers can contribute, so the per-mode RRF max is
|
||||
// 2 / (k_rrf + 1). Even when a chunk lands in only one mode, we
|
||||
@@ -412,20 +401,20 @@ impl HybridRetriever {
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse the `hybrid_fusion` config string into a [`FusionPolicy`].
|
||||
/// Parse the `hybrid_fusion` config string into an RRF `k_rrf`.
|
||||
/// Today only `"rrf"` is recognised; anything else falls back to RRF
|
||||
/// with a warn log so misconfiguration is visible but not fatal.
|
||||
fn parse_fusion(name: &str, k_rrf: u32) -> FusionPolicy {
|
||||
fn parse_fusion(name: &str, k_rrf: u32) -> u32 {
|
||||
let k = if k_rrf == 0 { DEFAULT_K_RRF } else { k_rrf };
|
||||
match name {
|
||||
"rrf" => FusionPolicy::Rrf { k_rrf: k },
|
||||
"rrf" => k,
|
||||
other => {
|
||||
tracing::warn!(
|
||||
target: "kebab-search",
|
||||
policy = other,
|
||||
"kb-search hybrid: unknown fusion policy; falling back to RRF"
|
||||
);
|
||||
FusionPolicy::Rrf { k_rrf: k }
|
||||
k
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -520,8 +509,8 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn rrf_policy(k_rrf: u32) -> FusionPolicy {
|
||||
FusionPolicy::Rrf { k_rrf }
|
||||
fn rrf_policy(k_rrf: u32) -> u32 {
|
||||
k_rrf
|
||||
}
|
||||
|
||||
fn make_query(mode: SearchMode, k: usize) -> SearchQuery {
|
||||
@@ -743,14 +732,13 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn parse_fusion_falls_back_to_rrf_on_unknown() {
|
||||
let p = parse_fusion("nonsense", 60);
|
||||
let FusionPolicy::Rrf { k_rrf } = p;
|
||||
let k_rrf = parse_fusion("nonsense", 60);
|
||||
assert_eq!(k_rrf, 60);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_fusion_zero_k_falls_back_to_default() {
|
||||
let FusionPolicy::Rrf { k_rrf } = parse_fusion("rrf", 0);
|
||||
let k_rrf = parse_fusion("rrf", 0);
|
||||
assert_eq!(k_rrf, DEFAULT_K_RRF);
|
||||
}
|
||||
|
||||
@@ -838,12 +826,7 @@ mod tests {
|
||||
mk_hit(2, "c3", 0.6, SearchMode::Vector),
|
||||
],
|
||||
});
|
||||
let hybrid = HybridRetriever::with_policy(
|
||||
lex.clone(),
|
||||
vec_r.clone(),
|
||||
FusionPolicy::Rrf { k_rrf: 60 },
|
||||
2,
|
||||
);
|
||||
let hybrid = HybridRetriever::with_policy(lex.clone(), vec_r.clone(), 60, 2);
|
||||
let q = SearchQuery {
|
||||
text: "x".into(),
|
||||
mode: SearchMode::Hybrid,
|
||||
@@ -873,7 +856,7 @@ mod tests {
|
||||
}
|
||||
let lex = Arc::new(EmptyR);
|
||||
let vec_r = Arc::new(EmptyR);
|
||||
let hybrid = HybridRetriever::with_policy(lex, vec_r, FusionPolicy::Rrf { k_rrf: 60 }, 2);
|
||||
let hybrid = HybridRetriever::with_policy(lex, vec_r, 60, 2);
|
||||
let q = SearchQuery {
|
||||
text: "x".into(),
|
||||
mode: SearchMode::Lexical,
|
||||
@@ -908,7 +891,7 @@ mod tests {
|
||||
let vec_r = Arc::new(Stub {
|
||||
hits: vec![mk_hit("c1", 1, SearchMode::Vector, 0.8)],
|
||||
});
|
||||
let hybrid = HybridRetriever::with_policy(lex, vec_r, FusionPolicy::Rrf { k_rrf: 60 }, 2);
|
||||
let hybrid = HybridRetriever::with_policy(lex, vec_r, 60, 2);
|
||||
let q = SearchQuery {
|
||||
text: "x".into(),
|
||||
mode: SearchMode::Hybrid,
|
||||
@@ -944,7 +927,7 @@ mod tests {
|
||||
hits: vec![lex_hit],
|
||||
});
|
||||
let vec_r = Arc::new(Stub { hits: vec![] });
|
||||
let hybrid = HybridRetriever::with_policy(lex, vec_r, FusionPolicy::Rrf { k_rrf: 60 }, 2);
|
||||
let hybrid = HybridRetriever::with_policy(lex, vec_r, 60, 2);
|
||||
let q = SearchQuery {
|
||||
text: "x".into(),
|
||||
mode: SearchMode::Lexical,
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
//! `kb-store-vector::LanceVectorStore`) and a `dyn Embedder`,
|
||||
//! hydrating SQLite metadata for full `SearchHit`s.
|
||||
//! - [`HybridRetriever`] (P3-4): composes lexical + vector retrievers,
|
||||
//! dispatches by `SearchMode`, fuses Hybrid via [`FusionPolicy::Rrf`].
|
||||
//! dispatches by `SearchMode`, fuses Hybrid via Reciprocal Rank Fusion.
|
||||
//!
|
||||
//! Allowed deps per the P2-2 + P3-4 task specs: `kb-core`, `kb-config`,
|
||||
//! `kb-store-sqlite`, `kb-store-vector`, `kb-embed` (trait re-export
|
||||
@@ -22,6 +22,6 @@ mod lexical;
|
||||
mod trace;
|
||||
mod vector;
|
||||
|
||||
pub use hybrid::{FusionPolicy, HybridRetriever};
|
||||
pub use hybrid::HybridRetriever;
|
||||
pub use lexical::LexicalRetriever;
|
||||
pub use vector::VectorRetriever;
|
||||
|
||||
@@ -15,14 +15,14 @@ use common::{
|
||||
HybridEnv, TEST_LEX_INDEX_VERSION, TEST_VEC_INDEX_VERSION, id32, require_avx_or_panic,
|
||||
};
|
||||
use kebab_core::{MediaType, Retriever, SearchFilters, SearchHit, SearchMode, SearchQuery};
|
||||
use kebab_search::{FusionPolicy, HybridRetriever};
|
||||
use kebab_search::HybridRetriever;
|
||||
use rusqlite::params;
|
||||
use serde_json::json;
|
||||
|
||||
fn build_hybrid(env: &HybridEnv) -> HybridRetriever {
|
||||
let lex: Arc<dyn Retriever> = Arc::new(env.lexical_retriever());
|
||||
let vec: Arc<dyn Retriever> = Arc::new(env.vector_retriever());
|
||||
HybridRetriever::with_policy(lex, vec, FusionPolicy::Rrf { k_rrf: 60 }, 5)
|
||||
HybridRetriever::with_policy(lex, vec, 60, 5)
|
||||
}
|
||||
|
||||
/// Seed a tiny corpus that lets us prove hybrid recall ≥ each side
|
||||
|
||||
@@ -25,7 +25,7 @@ Cargo workspace, 함수 호출 기반 모듈러 모놀리스. UI binary (`kebab-
|
||||
| Image caption | Ollama vision LM, runtime gate `image.caption.enabled` (default OFF) |
|
||||
| RAG groundedness 검증 | `kebab-nli` 의 mDeBERTa-v3 XNLI 가 `(packed_chunks, generated_answer)` entailment 검사 (fb-41). `[rag] nli_threshold > 0` (default 0 = disabled, production 권장 0.5) 일 때 활성 — 미달 시 `refusal_reason = nli_verification_failed` (LLM self-judge ceiling 보완). 첫 호출 시 ~280 MB ONNX 자동 다운로드 |
|
||||
| PDF parser | `lopdf` per-page 텍스트 + scanned-page image extract (`page_image::extract_dctdecode_page_image`, v0.20.0). `chunker_version = "pdf-page-v1"` 하드코딩 (HOTFIXES P7-3). `parser_version = "pdf-text-v1"` 보존 (v0.20 OCR 후에도) — provenance event 로 OCR 사용 차별화. force-reingest 가 v0.19 indexed scanned PDF 의 재처리에 필요. |
|
||||
| code parser | `tree-sitter` + `tree-sitter-rust` / `tree-sitter-python` / `tree-sitter-typescript` / `tree-sitter-javascript` / `tree-sitter-go` / `tree-sitter-java` / `tree-sitter-kotlin-ng` — **parser-side** (`kebab-parse-code`), chunker-side 아님 (design §6.3). chunker versions: Rust = `code-rust-ast-v1`, Python = `code-python-ast-v1`, TypeScript = `code-ts-ast-v1`, JavaScript = `code-js-ast-v1`, Go = `code-go-ast-v1`, Java = `code-java-ast-v1`, Kotlin = `code-kotlin-ast-v1`. `ast_chunk_max_lines = 200` 상수 고정 (HOTFIXES 2026-05-19 — Chunker trait 이 per-medium config 미노출). Kotlin grammar 은 `tree-sitter-kotlin-ng` 사용 — bare `tree-sitter-kotlin` 은 tree-sitter 0.21–0.23 에 고착되어 있어 사용 불가. **Tier 2 (p10-2)**: YAML/k8s → `serde_yaml` + `k8s-manifest-resource-v1` (apiVersion+kind per resource), Dockerfile → `dockerfile-file-v1` (whole-file), Cargo.toml/go.mod/.json/.xml/.groovy → `manifest-file-v1` (whole-file). Tier 2 chunkers live in `kebab-chunk`; no tree-sitter grammar needed (structure from file type, not AST). **Tier 3 (p10-3)**: shell scripts (`.sh`/`.bash`/`.zsh`) direct → `code-text-paragraph-v1` (blank-line paragraph segmentation + 80-line / 20-overlap line-window for oversize). Same chunker also serves as fallback when Tier 1/2 emit 0 chunks or Err — non-k8s YAML / invalid YAML / AST extractor failures all picked up. symbol = None; lang preserved from input doc. **Tier 1 family complete (p10-1D)**: C (`tree-sitter-c`, `code-c-ast-v1`, `.c`/`.h`) + C++ (`tree-sitter-cpp`, `code-cpp-ast-v1`, `.cpp`/`.cc`/`.cxx`/`.hpp`/`.hh`/`.hxx`). C symbol = function name only; C++ symbol = `namespace::Class::method` (recursive nesting). `.h` 가 C++ syntax 만나면 tree-sitter-c parse 실패 → Tier 3 fallback. |
|
||||
| code parser | `tree-sitter` + `tree-sitter-rust` / `tree-sitter-python` / `tree-sitter-typescript` / `tree-sitter-javascript` / `tree-sitter-go` / `tree-sitter-java` / `tree-sitter-kotlin-ng` — **parser-side** (`kebab-parse-code`), chunker-side 아님 (design §6.3). chunker versions: Rust = `code-rust-ast-v1`, Python = `code-python-ast-v1`, TypeScript = `code-ts-ast-v1`, JavaScript = `code-js-ast-v1`, Go = `code-go-ast-v1`, Java = `code-java-ast-v1`, Kotlin = `code-kotlin-ast-v1`. `ast_chunk_max_lines = 200` 상수 고정 (HOTFIXES 2026-05-19 — Chunker trait 이 per-medium config 미노출). Kotlin grammar 은 `tree-sitter-kotlin-ng` 사용 — bare `tree-sitter-kotlin` 은 tree-sitter 0.21–0.23 에 고착되어 있어 사용 불가. **Tier 2 (p10-2)**: YAML/k8s → `serde_yaml_ng` + `k8s-manifest-resource-v1` (apiVersion+kind per resource), Dockerfile → `dockerfile-file-v1` (whole-file), Cargo.toml/go.mod/.json/.xml/.groovy → `manifest-file-v1` (whole-file). Tier 2 chunkers live in `kebab-chunk`; no tree-sitter grammar needed (structure from file type, not AST). **Tier 3 (p10-3)**: shell scripts (`.sh`/`.bash`/`.zsh`) direct → `code-text-paragraph-v1` (blank-line paragraph segmentation + 80-line / 20-overlap line-window for oversize). Same chunker also serves as fallback when Tier 1/2 emit 0 chunks or Err — non-k8s YAML / invalid YAML / AST extractor failures all picked up. symbol = None; lang preserved from input doc. **Tier 1 family complete (p10-1D)**: C (`tree-sitter-c`, `code-c-ast-v1`, `.c`/`.h`) + C++ (`tree-sitter-cpp`, `code-cpp-ast-v1`, `.cpp`/`.cc`/`.cxx`/`.hpp`/`.hh`/`.hxx`). C symbol = function name only; C++ symbol = `namespace::Class::method` (recursive nesting). `.h` 가 C++ syntax 만나면 tree-sitter-c parse 실패 → Tier 3 fallback. |
|
||||
| symbol path 형식 | workspace path → module path: Python = dotted prefix (`kebab_eval.metrics.compute_mrr`), TypeScript/JavaScript = slash-style prefix (`src/Foo.Foo.search`), Go = `package.Func` / `package.(*Receiver).Method`, Java/Kotlin = `com.foo.Foo.bar` (패키지+클래스+메서드/필드), C = 함수명, C++ = `namespace::Class::method`. Rust 1A-2 는 file-scope nesting 만 (workspace prefix 없음, 비일관 수용 — HOTFIXES 2026-05-20). code chunk 은 `citation.kind = "code"` + `citation.lang` + `symbol` + line range, SearchHit 에 `code_lang` + `repo`(`.git` walk-up 디렉토리명) backfill. |
|
||||
| Desktop | Tauri 2 + `pdfjs-dist` (native PDF render backend 금지) — P9-5 |
|
||||
| citation 형식 | URI fragment (`path#L12-L34` / `path#p=12` / `path#xywh=0,0,100,50`, W3C Media Fragments) |
|
||||
|
||||
@@ -105,7 +105,7 @@ flowchart LR
|
||||
## 주요 type / trait / 함수
|
||||
|
||||
**Loader** (`kebab-eval::loader`):
|
||||
- `load_golden_set(path: &Path) -> Result<Vec<GoldenQuery>>` — `serde_yaml` 위 YAML 파싱.
|
||||
- `load_golden_set(path: &Path) -> Result<Vec<GoldenQuery>>` — `serde_yaml_ng` 위 YAML 파싱.
|
||||
- `GoldenQuery { id, query, mode, k, must_contain, expected_doc_ids, ask }` — fixture 한 entry. `must_contain` 가 case-sensitive substring 검사.
|
||||
|
||||
**Runner** (`kebab-eval::runner`):
|
||||
@@ -133,7 +133,7 @@ flowchart LR
|
||||
## 외부 의존
|
||||
|
||||
- crate dep: `kebab-core` + `kebab-config` + `kebab-app` (facade only) + `kebab-store-sqlite` (SQLite 직접 read/write — `eval_runs` / `eval_query_results` 측). retrieval / embedding / LLM crate 직접 import **금지**.
|
||||
- 외부 lib: `serde_yaml` (golden YAML), `serde_json`, `uuid` (v7), `time`, `tracing`, `anyhow`.
|
||||
- 외부 lib: `serde_yaml_ng` (golden YAML), `serde_json`, `uuid` (v7), `time`, `tracing`, `anyhow`.
|
||||
- 외부 서비스: 없음 (facade 가 가져옴).
|
||||
|
||||
## 핵심 결정
|
||||
|
||||
@@ -114,7 +114,7 @@ flowchart LR
|
||||
|
||||
- crate dep:
|
||||
- 모든 parser → `kebab-core` (`Extractor` trait, `Block`, `Metadata`, `id_for_*`).
|
||||
- `kebab-parse-md` → `kebab-parse-types` (`ParsedBlock`/`ParsedPayload`/`Warning`), `pulldown-cmark`, `serde_yaml`.
|
||||
- `kebab-parse-md` → `kebab-parse-types` (`ParsedBlock`/`ParsedPayload`/`Warning`), `pulldown-cmark`, `serde_yaml_ng`.
|
||||
- `kebab-parse-pdf` → `lopdf`.
|
||||
- `kebab-parse-image` → `image` (decode), `kamadak-exif` (EXIF), `kebab-core::LanguageModel` (caption).
|
||||
- 외부 서비스:
|
||||
|
||||
@@ -30,16 +30,12 @@ classDiagram
|
||||
}
|
||||
class HybridRetriever {
|
||||
+new(cfg, lexical, vector) Self
|
||||
+with_policy(lex, vec, FusionPolicy, k)
|
||||
+with_policy(lex, vec, k_rrf, default_k)
|
||||
-lexical: Arc~dyn Retriever~
|
||||
-vector: Arc~dyn Retriever~
|
||||
-fusion: FusionPolicy
|
||||
-k_rrf: u32
|
||||
-default_k: usize
|
||||
}
|
||||
class FusionPolicy {
|
||||
<<enum>>
|
||||
Rrf{k_rrf}
|
||||
}
|
||||
class SearchMode {
|
||||
<<enum>>
|
||||
Lexical
|
||||
@@ -51,7 +47,6 @@ classDiagram
|
||||
Retriever <|.. HybridRetriever
|
||||
HybridRetriever --> LexicalRetriever
|
||||
HybridRetriever --> VectorRetriever
|
||||
HybridRetriever ..> FusionPolicy
|
||||
HybridRetriever ..> SearchMode : dispatch
|
||||
```
|
||||
|
||||
@@ -95,7 +90,7 @@ flowchart LR
|
||||
|
||||
**HybridRetriever** (`kebab-search::hybrid`):
|
||||
- `HybridRetriever::new(&Config, Arc<dyn Retriever> lex, Arc<dyn Retriever> vec) -> Self` — `config.search.hybrid_fusion` (`"rrf"`) + `config.search.rrf_k` 읽음. 두 retriever 의 `index_version` 가 다르면 `tracing::warn`.
|
||||
- `FusionPolicy::Rrf { k_rrf }` — default 60. `with_policy` 헬퍼로 explicit 지정 가능.
|
||||
- `k_rrf: u32` — default 60. `with_policy` 헬퍼로 explicit 지정 가능.
|
||||
- 상수: `DEFAULT_K = 10` (query.k == 0 fallback), `DEFAULT_K_RRF = 60`, `HYBRID_FANOUT_MULTIPLIER = 2`.
|
||||
- merge rule: 양측 등장 chunk 의 `snippet` / `citation` / `heading_path` 는 lexical 측에서 가져옴 (FTS5 highlight 가 vector 의 truncated text 보다 user-relevant).
|
||||
|
||||
|
||||
Reference in New Issue
Block a user