From 727648d21dec580b4472bd96386cc9f1c9303ad4 Mon Sep 17 00:00:00 2001 From: altair823 Date: Wed, 24 Jun 2026 02:12:03 +0000 Subject: [PATCH] =?UTF-8?q?fix(config):=20`[ingest.chunking]`=20budget=20f?= =?UTF-8?q?loor=20=EA=B2=80=EC=A6=9D=20(validate=5Fchunking)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Config::from_file` 에 `validate_chunking()` 추가 — 청킹 budget 의 명백히 깨진 조합을 load 시점에 reject(`validate_sources` 와 동일 패턴, `ConfigInvalid`): - `target_tokens ≥ 16` (`MIN_CHUNK_TOKENS`) - `overlap_tokens < target_tokens` - `max_chunk_tokens ≥ target_tokens` 동기: md-heading-v2(PR #209)의 `max_chunk_tokens` 는 검증이 없어 `0` 같은 오설정이 청커 내부 `budget.max(1)` 클램프에 흡수돼 3-byte 청크 폭주(인덱스 bloat, 무에러)를 냈다 — reviewer 지적. 기존 `target_tokens`/`overlap_tokens` 도 미검증이라 세 필드를 한 번에 floor + 상호 제약으로 막는다. valid config 무영향(동작·결과 불변), 깨진 config 만 명확한 메시지로 load 실패. 새 config 키·migration·동작 변경 없음 → patch-level. 테스트 6종(defaults pass + reject 4 + e2e from_file). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_012Mc6W1fgsrbFKTsqA6P8La --- crates/kebab-config/src/lib.rs | 110 +++++++++++++++++++++++++++++++++ tasks/HOTFIXES.md | 17 +++++ 2 files changed, 127 insertions(+) diff --git a/crates/kebab-config/src/lib.rs b/crates/kebab-config/src/lib.rs index 1d72d4f..a510f0b 100644 --- a/crates/kebab-config/src/lib.rs +++ b/crates/kebab-config/src/lib.rs @@ -196,6 +196,13 @@ fn default_max_chunk_tokens() -> usize { 4000 } +/// Floor for `[ingest.chunking].target_tokens` (and, transitively, the +/// per-chunk cap `max_chunk_tokens ≥ target_tokens`). A chunk target +/// below this is never a real config — it only ever appears as a typo / +/// zero that would shatter the corpus into useless fragments. Enforced +/// by [`Config::validate_chunking`]. +const MIN_CHUNK_TOKENS: usize = 16; + impl ChunkingCfg { pub fn defaults() -> Self { Self { @@ -1186,6 +1193,12 @@ impl Config { cause, }) })?; + cfg.validate_chunking().map_err(|cause| { + anyhow::Error::new(ConfigInvalid { + path: path.to_path_buf(), + cause, + }) + })?; cfg.source_dir = path.parent().map(Path::to_path_buf); Ok(cfg) } @@ -1215,6 +1228,38 @@ impl Config { Ok(()) } + /// Validate `[ingest.chunking]` budget fields. A misconfigured `0` + /// (or any nonsensical combination) would otherwise be silently + /// absorbed by the chunker's internal `budget.max(1)` clamp into a + /// flood of 3-byte chunks — catastrophic index bloat, no error. The + /// floors below reject the clearly-broken configs at load time with a + /// clear cause, mirroring [`validate_sources`](Self::validate_sources). + fn validate_chunking(&self) -> Result<(), String> { + let c = &self.ingest.chunking; + if c.target_tokens < MIN_CHUNK_TOKENS { + return Err(format!( + "ingest.chunking.target_tokens = {} is too small (min {MIN_CHUNK_TOKENS}); \ + a chunk target this low fragments every document", + c.target_tokens + )); + } + if c.overlap_tokens >= c.target_tokens { + return Err(format!( + "ingest.chunking.overlap_tokens = {} must be < target_tokens = {} \ + (overlap ≥ target loops / duplicates content)", + c.overlap_tokens, c.target_tokens + )); + } + if c.max_chunk_tokens < c.target_tokens { + return Err(format!( + "ingest.chunking.max_chunk_tokens = {} must be ≥ target_tokens = {} \ + (a per-chunk cap below the soft target would split every normal chunk)", + c.max_chunk_tokens, c.target_tokens + )); + } + Ok(()) + } + /// Apply `KEBAB_
_` env overrides. Unknown keys are ignored. /// /// The mapping is an explicit grep-friendly whitelist — one match arm @@ -2320,6 +2365,71 @@ max_context_tokens = 8000 ); } + #[test] + fn defaults_pass_chunking_validation() { + Config::defaults() + .validate_chunking() + .expect("default chunking config must be valid"); + } + + #[test] + fn rejects_target_tokens_below_floor() { + let mut cfg = Config::defaults(); + cfg.ingest.chunking.target_tokens = MIN_CHUNK_TOKENS - 1; + // keep max_chunk ≥ target so target is the failing constraint + cfg.ingest.chunking.overlap_tokens = 0; + let err = cfg.validate_chunking().unwrap_err(); + assert!(err.contains("target_tokens"), "got: {err}"); + } + + #[test] + fn rejects_zero_target_tokens() { + let mut cfg = Config::defaults(); + cfg.ingest.chunking.target_tokens = 0; + cfg.ingest.chunking.overlap_tokens = 0; + assert!(cfg.validate_chunking().is_err()); + } + + #[test] + fn rejects_overlap_ge_target() { + let mut cfg = Config::defaults(); + cfg.ingest.chunking.target_tokens = 100; + cfg.ingest.chunking.overlap_tokens = 100; // == target → invalid + cfg.ingest.chunking.max_chunk_tokens = 4000; + let err = cfg.validate_chunking().unwrap_err(); + assert!(err.contains("overlap_tokens"), "got: {err}"); + } + + #[test] + fn rejects_max_chunk_below_target() { + let mut cfg = Config::defaults(); + cfg.ingest.chunking.target_tokens = 500; + cfg.ingest.chunking.overlap_tokens = 80; + cfg.ingest.chunking.max_chunk_tokens = 400; // < target → invalid + let err = cfg.validate_chunking().unwrap_err(); + assert!(err.contains("max_chunk_tokens"), "got: {err}"); + } + + #[test] + fn from_file_rejects_invalid_chunking() { + // End-to-end: validate_chunking is wired into from_file, so a + // config whose max_chunk_tokens is below target must fail to load. + let mut cfg = Config::defaults(); + cfg.workspace.root = Some("/tmp/kb".to_string()); + cfg.ingest.chunking.max_chunk_tokens = 10; // < target 500 + let toml_text = toml::to_string(&cfg).expect("serialize"); + let dir = std::env::temp_dir().join(format!("kebab-cfgtest-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let p = dir.join("config.toml"); + std::fs::write(&p, &toml_text).unwrap(); + let res = Config::from_file(&p); + let _ = std::fs::remove_dir_all(&dir); + assert!( + res.is_err(), + "from_file must reject max_chunk_tokens < target_tokens" + ); + } + fn source_cfg(id: &str, root: &str) -> SourceCfg { SourceCfg { id: id.to_string(), diff --git a/tasks/HOTFIXES.md b/tasks/HOTFIXES.md index ce81758..cf4132d 100644 --- a/tasks/HOTFIXES.md +++ b/tasks/HOTFIXES.md @@ -14,6 +14,23 @@ historical contract that was implemented; this file accumulates the deltas so phase 5+ readers can find the live behavior without diffing git history. +## 2026-06-24 — config: `[ingest.chunking]` budget floor 검증 + +**무엇을 바꿨나.** `Config::from_file` 에 `validate_chunking()` 을 추가해 +청킹 budget 의 명백히 깨진 조합을 **load 시점에 reject**(`validate_sources` +와 동일 패턴, `ConfigInvalid`). 규칙: `target_tokens ≥ 16`(`MIN_CHUNK_TOKENS`), +`overlap_tokens < target_tokens`, `max_chunk_tokens ≥ target_tokens`. + +**왜.** md-heading-v2(PR #209) 의 `max_chunk_tokens` 는 검증이 없어, `0` 같은 +오설정이 청커 내부 `budget.max(1)` 클램프에 흡수돼 **3-byte 청크 폭주**(인덱스 +bloat, 에러 없음)를 냈다. 기존 `target_tokens`/`overlap_tokens` 도 동일하게 +미검증이었다(reviewer 지적). 세 필드를 한 번에 floor + 상호 제약으로 막는다. + +**영향.** valid config 는 무영향(동작·결과 불변). 깨진 config(예: `max_chunk +< target`, `overlap ≥ target`)만 명확한 메시지로 load 실패. 새 config 키·migration +없음, 동작 변경 없음 → patch-level. 테스트: `defaults_pass_chunking_validation` ++ reject 4종 + `from_file_rejects_invalid_chunking`(e2e). + ## 2026-06-24 — md-heading-v2: 예산 초과 청크 일반 분할 (oversize-chunk split) (v0.30.0) **무엇을 바꿨나.** markdown 청커에 새 변종 `md-heading-v2` 를 추가하고