From ed8ab7cdbeac9f75dc19d5d695264edf31010d11 Mon Sep 17 00:00:00 2001 From: altair823 Date: Wed, 24 Jun 2026 03:09:13 +0000 Subject: [PATCH] =?UTF-8?q?feat(chunk):=20pdf-page-v1.2=20=E2=80=94=20PDF?= =?UTF-8?q?=20=ED=8E=98=EC=9D=B4=EC=A7=80=20oversize=20=EB=B6=84=ED=95=A0?= =?UTF-8?q?=20+=20=EA=B3=B5=EC=9C=A0=20oversize=20=EB=AA=A8=EB=93=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit md-heading-v2(PR #209)의 oversize 분할을 PDF 청커에도 적용. v1.1 의 chunk_page 는 문장/문단 경계로만 잘라서 경계 없는 거대 페이지(빽빽한 scanned page 한 줄 OCR)가 통째로 한 청크 → strict 임베더 실패. v1.2 는 2-tier: tier-1(문장/문단 greedy + overlap) 후 segment 가 max_chunk_tokens 초과면 tier-2 가 공유 text_pieces 로 재분할 → 모든 PDF 청크 ≤ 예산. - 신규 공유 모듈 crate::oversize (text_pieces/char_pieces/BYTES_PER_TOKEN) — md 와 PDF 가 공유(단일 진실 공급원). md-heading-v2 는 호출만, 출력 byte-identical(md 라벨·동작 불변, parity 테스트 전부 통과). - PdfPageV1Chunker { max_chunk_tokens } + policy_hash budget fold(md 동형) + pdf_chunker_from_config(kebab-app). 신규 config 키 없음. - 분할 조각 chunk_id 는 #c{segment_start}s{i}(미분할 단일 segment 는 bare #c{segment_start} 유지 → 공통 경우 v1.1 동일). - Page span: 분할 조각은 부모 segment 의 char 범위를 그대로 가짐(segment-granular, md 동형). per-piece narrowing 은 text_pieces 의 줄 구분자 소실로 drift 하는 버그라 코드 리뷰 후 제거 — 회귀 테스트 oversize_pdf_page_with_newlines_splits_without_span_drift 로 잠금. - chunker_version v1.1→v1.2 → 다음 ingest 에서 PDF 1회 자동 재청크(md/code 무영향). 검증: kebab-chunk lib 93 pass(span 회귀 포함), kebab-app green, clippy 0. 도그푸딩 (실험 KB, scanned PDF + arctic@Lemonade, budget 200): 625/625 errors=0, scanned_page1 1→3 청크·scanned_page2 3→7 청크(둘 다 pdf-page-v1.2), 전 코퍼스 10215 청크 전부 ≤200. 버전 bump 은 follow-up 들과 함께 배치 릴리스에서 일괄. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_012Mc6W1fgsrbFKTsqA6P8La --- crates/kebab-app/src/lib.rs | 22 +- ...canned_pdf_ingest_no_chunk_id_collision.rs | 17 +- crates/kebab-app/tests/pdf_pipeline.rs | 4 +- crates/kebab-chunk/src/lib.rs | 1 + crates/kebab-chunk/src/md_heading_v2.rs | 152 +----- crates/kebab-chunk/src/oversize.rs | 162 +++++++ crates/kebab-chunk/src/pdf_page_v1.rs | 452 +++++++++++++++--- docs/components/normalize-chunk/README.md | 5 +- ...2026-06-24-pdf-page-v1.2-oversize-split.md | 89 ++++ tasks/HOTFIXES.md | 39 ++ 10 files changed, 727 insertions(+), 216 deletions(-) create mode 100644 crates/kebab-chunk/src/oversize.rs create mode 100644 docs/superpowers/plans/2026-06-24-pdf-page-v1.2-oversize-split.md diff --git a/crates/kebab-app/src/lib.rs b/crates/kebab-app/src/lib.rs index 7ed4a4e..dc01c60 100644 --- a/crates/kebab-app/src/lib.rs +++ b/crates/kebab-app/src/lib.rs @@ -2270,7 +2270,7 @@ fn ingest_one_pdf_asset( app, asset, &eff_parser_version, - &PdfPageV1Chunker.chunker_version(), + &pdf_chunker_from_config(&app.config).chunker_version(), embedder.map(|e| e.model_version()).as_ref(), force_reingest, None, @@ -2434,8 +2434,10 @@ fn ingest_one_pdf_asset( // Per-medium chunker selection: PDF docs always use pdf-page-v1 // regardless of `config.ingest.chunking.chunker_version`. The chunker // validates every block carries `SourceSpan::Page`; failure here - // means the parser drifted from its contract. - let chunker = PdfPageV1Chunker; + // means the parser drifted from its contract. v1.2: the tier-2 oversize + // split budget is threaded from `config.ingest.chunking.max_chunk_tokens` + // (no new config key — same one md uses). + let chunker = pdf_chunker_from_config(&app.config); let t_chunk = std::time::Instant::now(); let chunks = chunker .chunk(&canonical, chunk_policy) @@ -3170,6 +3172,20 @@ fn md_chunker_from_config(config: &kebab_config::Config) -> MdHeadingV2Chunker { } } +/// Construct the PDF chunker (`pdf-page-v1.2`) with the tier-2 oversize +/// split budget threaded from config — mirrors [`md_chunker_from_config`]. +/// The PDF path stays pinned to `pdf-page-v1` regardless of +/// `config.ingest.chunking.chunker_version`; only the tier-2 budget is +/// config-driven (no new config key — it reuses `max_chunk_tokens`, already +/// folded into `ingest_config_signature` so a budget change re-indexes PDFs +/// without `--force-reingest`). The budget also folds into the v1.2 +/// `policy_hash`, aligning the PDF chunk_id cascade with markdown. +fn pdf_chunker_from_config(config: &kebab_config::Config) -> PdfPageV1Chunker { + PdfPageV1Chunker { + max_chunk_tokens: config.ingest.chunking.max_chunk_tokens, + } +} + /// v0.26.2: deterministic signature of the **ingest-output-affecting** /// config for an asset's media type, folded into the effective /// `parser_version` (both the `try_skip_unchanged` compare field AND the diff --git a/crates/kebab-app/tests/multi_scanned_pdf_ingest_no_chunk_id_collision.rs b/crates/kebab-app/tests/multi_scanned_pdf_ingest_no_chunk_id_collision.rs index b90730f..d575464 100644 --- a/crates/kebab-app/tests/multi_scanned_pdf_ingest_no_chunk_id_collision.rs +++ b/crates/kebab-app/tests/multi_scanned_pdf_ingest_no_chunk_id_collision.rs @@ -85,19 +85,22 @@ fn multi_scanned_pdf_ingest_no_chunk_id_collision() { let f1_canonical = extract_and_ocr(&f1_bytes, "page1.pdf", '1', &f1_engine); let f2_canonical = extract_and_ocr(&f2_bytes, "page2.pdf", '2', &f2_engine); + // v1.2: PdfPageV1Chunker carries a tier-2 oversize budget. Use a + // generous budget so the tier-2 split never fires — this test exercises + // the tier-1 sentence/paragraph `#c{segment_start}` collision-avoidance, + // which is unchanged from v1.1. + let chunker = PdfPageV1Chunker { + max_chunk_tokens: 100_000, + }; let chunk_policy = ChunkPolicy { target_tokens: 500, overlap_tokens: 80, respect_markdown_headings: false, - chunker_version: PdfPageV1Chunker.chunker_version(), + chunker_version: chunker.chunker_version(), }; - let f1_chunks = PdfPageV1Chunker - .chunk(&f1_canonical, &chunk_policy) - .unwrap(); - let f2_chunks = PdfPageV1Chunker - .chunk(&f2_canonical, &chunk_policy) - .unwrap(); + let f1_chunks = chunker.chunk(&f1_canonical, &chunk_policy).unwrap(); + let f2_chunks = chunker.chunk(&f2_canonical, &chunk_policy).unwrap(); assert!( f2_chunks.len() >= 2, diff --git a/crates/kebab-app/tests/pdf_pipeline.rs b/crates/kebab-app/tests/pdf_pipeline.rs index 732e741..aaae43c 100644 --- a/crates/kebab-app/tests/pdf_pipeline.rs +++ b/crates/kebab-app/tests/pdf_pipeline.rs @@ -169,7 +169,7 @@ fn ingest_3_page_pdf_produces_one_doc_and_per_page_chunks() { ); assert_eq!( pdf_item.chunker_version.as_ref().map(|c| c.0.as_str()), - Some("pdf-page-v1.1") + Some("pdf-page-v1.2") ); // Inspect the stored doc to confirm SourceSpan::Page round-trip. @@ -363,7 +363,7 @@ fn mixed_page_pdf_stores_asset_with_scanned_candidate_warning() { assert_eq!( pdf_item.chunk_count, Some(2), - "pdf-page-v1.1 emits 0 chunks for the empty page; total = 2" + "pdf-page-v1.2 emits 0 chunks for the empty page; total = 2" ); let doc = kebab_app::inspect_doc_with_config(cfg, pdf_item.doc_id.as_ref().unwrap()).unwrap(); diff --git a/crates/kebab-chunk/src/lib.rs b/crates/kebab-chunk/src/lib.rs index e3ae88d..26c1977 100644 --- a/crates/kebab-chunk/src/lib.rs +++ b/crates/kebab-chunk/src/lib.rs @@ -37,6 +37,7 @@ pub mod k8s_manifest_resource_v1; pub mod manifest_file_v1; mod md_heading_v1; mod md_heading_v2; +mod oversize; mod pdf_page_v1; mod tier2_shared; diff --git a/crates/kebab-chunk/src/md_heading_v2.rs b/crates/kebab-chunk/src/md_heading_v2.rs index 274a896..df0f476 100644 --- a/crates/kebab-chunk/src/md_heading_v2.rs +++ b/crates/kebab-chunk/src/md_heading_v2.rs @@ -42,6 +42,13 @@ //! granular citation). This is intentional: we do not have sub-line source //! map data for arbitrary block kinds, and a citation to the enclosing //! block/region is always correct — never wrong, just not sub-line-precise. +//! +//! ## Shared split primitive +//! +//! The text-decomposition itself (line split → char-split fallback) lives +//! in [`crate::oversize`], shared with `pdf-page-v1.2`'s tier-2 fallback. +//! Only the md-specific `#seg{i}` chunk-id recipe + `source_spans` clone +//! stay here. use kebab_core::{ Block, BlockId, CanonicalDocument, Chunk, ChunkPolicy, Chunker, ChunkerVersion, DocumentId, @@ -53,10 +60,12 @@ use kebab_core::{ /// markdown doc on the first ingest after v2 becomes the default. const VERSION_LABEL: &str = "md-heading-v2"; -/// Bytes-per-token proxy — identical to v1. 3 bytes/token over-estimates -/// token count for both Korean (E5 ≈ 3) and English (BPE ≈ 4) so chunks -/// sized against this proxy always fit a real tokenizer's budget. -const BYTES_PER_TOKEN: usize = 3; +/// Bytes-per-token proxy — re-exported from the shared [`crate::oversize`] +/// module so md-heading-v2 and pdf-page-v1.2 cannot drift. 3 bytes/token +/// over-estimates token count for both Korean (E5 ≈ 3) and English +/// (BPE ≈ 4) so chunks sized against this proxy always fit a real +/// tokenizer's budget. +use crate::oversize::BYTES_PER_TOKEN; /// Maximum hex characters of the policy hash. 16 hex = 64 bits, matching v1. const POLICY_HASH_HEX_LEN: usize = 16; @@ -253,7 +262,7 @@ fn split_oversize_chunk( base_policy_hash: &str, ) -> Vec { // Collect all sub-piece texts first. - let pieces: Vec = text_pieces(&chunk.text, budget); + let pieces: Vec = crate::oversize::text_pieces(&chunk.text, budget); // Safety invariant: split must produce ≥1 piece. debug_assert!(!pieces.is_empty(), "text_pieces must return ≥1 piece"); @@ -287,92 +296,6 @@ fn split_oversize_chunk( .collect() } -/// Decompose `text` into sub-pieces each with `len().div_ceil(BYTES_PER_TOKEN) -/// <= budget`. Returns ≥1 piece. The pieces join back to the original with -/// `\n` (line splits) or direct concatenation (char splits within a single -/// line), preserving the full text. -/// -/// The returned vec is ordered; concatenating the pieces in order with `\n` -/// reconstructs `text` exactly when `text` contains newlines. For a -/// single-line (newline-free) text, the pieces concatenate directly. -fn text_pieces(text: &str, budget: usize) -> Vec { - // A budget of 0 is degenerate — treat as 1 to avoid infinite loops. - let budget = budget.max(1); - - // Split on '\n' first. A trailing '\n' yields a final empty element - // which we preserve so joining with '\n' reconstructs the original. - let lines: Vec<&str> = text.split('\n').collect(); - - let mut result: Vec = Vec::new(); - let mut current_piece: Vec<&str> = Vec::new(); - let mut current_bytes: usize = 0; - - for line in &lines { - let line_bytes = line.len(); - // +1 for the '\n' that re-joins this line to the previous one. - let sep_bytes = usize::from(!current_piece.is_empty()); - - if line_bytes.div_ceil(BYTES_PER_TOKEN) > budget { - // This single line alone exceeds the budget → flush any - // accumulated piece, then char-split the line. - if !current_piece.is_empty() { - result.push(current_piece.join("\n")); - current_piece.clear(); - current_bytes = 0; - } - result.extend(char_pieces(line, budget)); - } else if !current_piece.is_empty() - && (current_bytes + sep_bytes + line_bytes).div_ceil(BYTES_PER_TOKEN) > budget - { - // Adding this line would push the current piece over budget - // → flush, then start a new piece with this line. - result.push(current_piece.join("\n")); - current_piece = vec![line]; - current_bytes = line_bytes; - } else { - // Fits: accumulate. - current_bytes += sep_bytes + line_bytes; - current_piece.push(line); - } - } - if !current_piece.is_empty() { - result.push(current_piece.join("\n")); - } - if result.is_empty() { - // Degenerate (empty text) — return one empty piece so the caller - // always gets ≥1 chunk. - result.push(String::new()); - } - result -} - -/// Char-split a single newline-free string `s` into sub-pieces each with -/// `len() <= budget * BYTES_PER_TOKEN`, cutting at UTF-8 char boundaries. -/// Returns ≥1 piece; direct concatenation of all pieces reconstructs `s`. -fn char_pieces(s: &str, budget: usize) -> Vec { - let byte_budget = budget * BYTES_PER_TOKEN; - let mut result: Vec = Vec::new(); - let mut piece_start = 0usize; - let mut piece_bytes = 0usize; - - for (byte_idx, ch) in s.char_indices() { - let ch_bytes = ch.len_utf8(); - if piece_bytes > 0 && piece_bytes + ch_bytes > byte_budget { - // Flush current piece. - result.push(s[piece_start..byte_idx].to_string()); - piece_start = byte_idx; - piece_bytes = 0; - } - piece_bytes += ch_bytes; - } - // Remaining tail. - result.push(s[piece_start..].to_string()); - if result.is_empty() { - result.push(String::new()); - } - result -} - // ── v1-equivalent helpers (verbatim from md_heading_v1) ───────────────────── #[derive(Default)] @@ -1033,46 +956,9 @@ mod tests { ); } - // ── Unit tests for the split helpers ───────────────────────────────── - - /// `text_pieces` on a multi-line string reconstructs with join("\n"). - #[test] - fn text_pieces_multiline_roundtrip() { - let lines: Vec = (0..20) - .map(|i| format!("line {i:02}: some content here")) - .collect(); - let text = lines.join("\n"); - let budget = 10usize; // small to force splits - let pieces = text_pieces(&text, budget); - assert!(pieces.len() >= 2, "must split multi-line text"); - for p in &pieces { - assert!( - p.len().div_ceil(BYTES_PER_TOKEN) <= budget, - "piece exceeds budget: {} bytes / 3 = {} > {budget}", - p.len(), - p.len().div_ceil(BYTES_PER_TOKEN) - ); - } - assert_eq!(pieces.join("\n"), text, "pieces must reconstruct original"); - } - - /// `char_pieces` on a newline-free string reconstructs by concatenation. - #[test] - fn char_pieces_utf8_roundtrip() { - // Mix of ASCII and 3-byte Korean. - let s = "hello가나다world마바사".repeat(10); - let budget = 5usize; - let pieces = char_pieces(&s, budget); - assert!(pieces.len() >= 2); - for p in &pieces { - assert!( - p.len() <= budget * BYTES_PER_TOKEN, - "char piece too long: {} > {}", - p.len(), - budget * BYTES_PER_TOKEN - ); - assert!(std::str::from_utf8(p.as_bytes()).is_ok(), "not valid UTF-8"); - } - assert_eq!(pieces.concat(), s, "char pieces must reconstruct original"); - } + // NOTE: the `text_pieces` / `char_pieces` unit roundtrip tests moved to + // `crate::oversize` along with the functions themselves (shared with + // pdf-page-v1.2). md-heading-v2's split behavior is still covered + // end-to-end by the `oversize_*` tests above, which exercise the same + // primitive through `split_oversize_chunk`. } diff --git a/crates/kebab-chunk/src/oversize.rs b/crates/kebab-chunk/src/oversize.rs new file mode 100644 index 0000000..e1113ca --- /dev/null +++ b/crates/kebab-chunk/src/oversize.rs @@ -0,0 +1,162 @@ +//! Shared oversize-chunk split primitives. +//! +//! Two chunkers need to guarantee that no emitted chunk exceeds the +//! configured `max_chunk_tokens` budget (byte/3 proxy): +//! +//! * `md-heading-v2` ([`crate::md_heading_v2`]) — a generic post-pass over +//! every block kind (list, code, paragraph, table, image-OCR). +//! * `pdf-page-v1.2` ([`crate::pdf_page_v1`]) — a tier-2 fallback for a +//! dense scanned page OCR'd into one over-budget run with no sentence / +//! paragraph boundary for the tier-1 greedy splitter to cut on. +//! +//! Both share the SAME two-tier splitting primitive so the byte-budget +//! bound (and its char-boundary fallback for a no-whitespace page) is +//! defined in exactly one place. This module is the single source of +//! [`BYTES_PER_TOKEN`] for the chunk crate's size proxy — both callers use +//! `crate::oversize::BYTES_PER_TOKEN` so a third divergent copy can't drift. +//! +//! The split disambiguation (chunk_id suffix recipe) and the source-span +//! handling stay with each caller, because they differ: md clones the +//! original chunk's block-granular `source_spans` while pdf narrows the +//! page-relative char range per sub-piece (Page spans are char-indexed, so +//! pdf can be strictly more precise). Only the text-decomposition logic — +//! which is identical — lives here. + +/// Bytes-per-token proxy — single source for the chunk crate. 3 bytes/token +/// over-estimates token count for both Korean (E5 ≈ 3) and English +/// (BPE ≈ 4) so chunks sized against this proxy always fit a real +/// tokenizer's budget. Mirrors the `md-heading-v1` calibration; both +/// `md-heading-v2` and `pdf-page-v1.2` reference this constant. +pub(crate) const BYTES_PER_TOKEN: usize = 3; + +/// Decompose `text` into sub-pieces each with `len().div_ceil(BYTES_PER_TOKEN) +/// <= budget`. Returns ≥1 piece. The pieces join back to the original with +/// `\n` (line splits) or direct concatenation (char splits within a single +/// line), preserving the full text. +/// +/// The returned vec is ordered; concatenating the pieces in order with `\n` +/// reconstructs `text` exactly when `text` contains newlines. For a +/// single-line (newline-free) text, the pieces concatenate directly. +pub(crate) fn text_pieces(text: &str, budget: usize) -> Vec { + // A budget of 0 is degenerate — treat as 1 to avoid infinite loops. + let budget = budget.max(1); + + // Split on '\n' first. A trailing '\n' yields a final empty element + // which we preserve so joining with '\n' reconstructs the original. + let lines: Vec<&str> = text.split('\n').collect(); + + let mut result: Vec = Vec::new(); + let mut current_piece: Vec<&str> = Vec::new(); + let mut current_bytes: usize = 0; + + for line in &lines { + let line_bytes = line.len(); + // +1 for the '\n' that re-joins this line to the previous one. + let sep_bytes = usize::from(!current_piece.is_empty()); + + if line_bytes.div_ceil(BYTES_PER_TOKEN) > budget { + // This single line alone exceeds the budget → flush any + // accumulated piece, then char-split the line. + if !current_piece.is_empty() { + result.push(current_piece.join("\n")); + current_piece.clear(); + current_bytes = 0; + } + result.extend(char_pieces(line, budget)); + } else if !current_piece.is_empty() + && (current_bytes + sep_bytes + line_bytes).div_ceil(BYTES_PER_TOKEN) > budget + { + // Adding this line would push the current piece over budget + // → flush, then start a new piece with this line. + result.push(current_piece.join("\n")); + current_piece = vec![line]; + current_bytes = line_bytes; + } else { + // Fits: accumulate. + current_bytes += sep_bytes + line_bytes; + current_piece.push(line); + } + } + if !current_piece.is_empty() { + result.push(current_piece.join("\n")); + } + if result.is_empty() { + // Degenerate (empty text) — return one empty piece so the caller + // always gets ≥1 chunk. + result.push(String::new()); + } + result +} + +/// Char-split a single newline-free string `s` into sub-pieces each with +/// `len() <= budget * BYTES_PER_TOKEN`, cutting at UTF-8 char boundaries. +/// Returns ≥1 piece; direct concatenation of all pieces reconstructs `s`. +pub(crate) fn char_pieces(s: &str, budget: usize) -> Vec { + let byte_budget = budget * BYTES_PER_TOKEN; + let mut result: Vec = Vec::new(); + let mut piece_start = 0usize; + let mut piece_bytes = 0usize; + + for (byte_idx, ch) in s.char_indices() { + let ch_bytes = ch.len_utf8(); + if piece_bytes > 0 && piece_bytes + ch_bytes > byte_budget { + // Flush current piece. + result.push(s[piece_start..byte_idx].to_string()); + piece_start = byte_idx; + piece_bytes = 0; + } + piece_bytes += ch_bytes; + } + // Remaining tail. + result.push(s[piece_start..].to_string()); + if result.is_empty() { + result.push(String::new()); + } + result +} + +#[cfg(test)] +mod tests { + use super::*; + + /// `text_pieces` on a multi-line string reconstructs with join("\n"). + #[test] + fn text_pieces_multiline_roundtrip() { + let lines: Vec = (0..20) + .map(|i| format!("line {i:02}: some content here")) + .collect(); + let text = lines.join("\n"); + let budget = 10usize; // small to force splits + let pieces = text_pieces(&text, budget); + assert!(pieces.len() >= 2, "must split multi-line text"); + for p in &pieces { + assert!( + p.len().div_ceil(BYTES_PER_TOKEN) <= budget, + "piece exceeds budget: {} bytes / 3 = {} > {budget}", + p.len(), + p.len().div_ceil(BYTES_PER_TOKEN) + ); + } + assert_eq!(pieces.join("\n"), text, "pieces must reconstruct original"); + } + + /// `char_pieces` on a newline-free string reconstructs by concatenation. + #[test] + fn char_pieces_utf8_roundtrip() { + // Mix of ASCII and 3-byte Korean. + let s = "hello가나다world마바사".repeat(10); + let budget = 5usize; + let pieces = char_pieces(&s, budget); + assert!(pieces.len() >= 2); + for p in &pieces { + assert!( + p.len() <= budget * BYTES_PER_TOKEN, + "char piece too long: {} > {}", + p.len(), + budget * BYTES_PER_TOKEN + ); + assert!(std::str::from_utf8(p.as_bytes()).is_ok(), "not valid UTF-8"); + } + assert_eq!(pieces.concat(), s, "char pieces must reconstruct original"); + } +} diff --git a/crates/kebab-chunk/src/pdf_page_v1.rs b/crates/kebab-chunk/src/pdf_page_v1.rs index dfcdac9..40ecc6b 100644 --- a/crates/kebab-chunk/src/pdf_page_v1.rs +++ b/crates/kebab-chunk/src/pdf_page_v1.rs @@ -12,7 +12,9 @@ //! Per design §3.5 (Chunk), §4.2 (chunk_id recipe — see deviation note //! below), §0 Q3 (citation), §9 (versioning). //! -//! ## Splitting policy +//! ## Splitting policy (two tiers, `pdf-page-v1.2`) +//! +//! **Tier 1 — sentence / paragraph greedy split (unchanged from v1.1):** //! //! - If a page's bytes fit under `policy.target_tokens * BYTES_PER_TOKEN` //! the entire page is a single chunk. @@ -23,11 +25,6 @@ //! prefix is seeded with the trailing `policy.overlap_tokens * //! BYTES_PER_TOKEN` bytes of the prior chunk so retrieval handles //! queries that fall on the boundary. -//! - A page with no qualifying segment boundary AND text exceeding the -//! budget (e.g. a 5,000-byte single sentence) emits one oversized -//! chunk rather than hard-splitting mid-word — a real tokenizer slot -//! in P+ replaces this proxy and can do better mid-sentence splitting -//! when needed. //! - Common English abbreviations (`Mr.`, `i.e.`, `e.g.`, `Fig. 3`) //! trip the sentence-end heuristic and produce spurious boundaries — //! accepted as a v1 limit. A real sentence segmenter lands with the @@ -37,12 +34,27 @@ //! make a chunk fully re-emit the previous chunk's text. Same guard //! pattern as `md-heading-v1::collect_overlap_seed`. //! +//! **Tier 2 — generic oversize fallback (NEW in `pdf-page-v1.2`):** +//! +//! v1.1 had an ACCEPTED HOLE: a page with no qualifying segment boundary +//! AND text exceeding the budget (e.g. a dense scanned page OCR'd into one +//! 5,000-byte run with no sentence/paragraph break) emitted ONE oversized +//! chunk regardless of budget. That single over-budget chunk overflows a +//! strict embedder (e.g. AMD Lemonade). v1.2 closes the hole: every tier-1 +//! segment whose byte/3 estimate still exceeds `self.max_chunk_tokens` is +//! handed to [`crate::oversize::text_pieces`] (line split → UTF-8 char +//! fallback), so each emitted chunk is GUARANTEED ≤ budget. The char +//! fallback bounds even a no-whitespace page that the tier-1 greedy +//! splitter cannot cut. This is the same generic post-pass `md-heading-v2` +//! applies — shared via [`crate::oversize`]. +//! //! ## `BYTES_PER_TOKEN` //! //! 3 — same calibration as `md-heading-v1` (covers Korean ≈ 3 b/tok and //! over-estimates English ≈ 4 b/tok). The original p7-2 spec literal said //! `× 4`, but cross-chunker comparability outweighs the spec literal here. -//! Logged in `tasks/HOTFIXES.md`. +//! Logged in `tasks/HOTFIXES.md`. Sourced from [`crate::oversize`] so the +//! proxy can't drift between this chunker and `md-heading-v2`. //! //! ## `chunk_id` collision deviation //! @@ -61,33 +73,76 @@ //! `Chunk.policy_hash` so the field still answers "what policy was //! active". v1.1 second-iteration patch — logged in //! `tasks/HOTFIXES.md` (2026-05-27). +//! +//! v1.2 extends the suffix for tier-2 sub-pieces: `#c{segment_start}s{i}` +//! for sub-piece `i` (0-based) of a tier-1 segment that had to be +//! oversize-split. A single-piece (non-oversize) segment keeps the bare +//! `#c{segment_start}` so common-case chunk_ids are byte-identical to the +//! pre-pass. `segment_start` is per-segment-unique and `i` disambiguates +//! within a segment, so the full suffix is strictly unique across the page. +//! +//! ## Budget in `policy_hash` (NEW in `pdf-page-v1.2`) +//! +//! Because the tier-2 split is keyed on `self.max_chunk_tokens`, changing +//! the budget changes the produced chunks — so the budget MUST participate +//! in the chunk-id cascade (design §9), exactly as `md-heading-v2` does. +//! `policy_hash()` folds the 8 LE bytes of `self.max_chunk_tokens` after +//! the canonical `ChunkPolicy` bytes. This moves PDF chunk_ids on a budget +//! change, consistent with markdown. (The `ingest_config_signature` +//! already folds `max_chunk_tokens` into the skip-check, so the no-`--force` +//! re-index already worked; this aligns the chunk_id cascade with it.) use kebab_core::{ Block, BlockId, CanonicalDocument, Chunk, ChunkPolicy, Chunker, ChunkerVersion, DocumentId, SourceSpan, id_for_chunk, }; -const VERSION_LABEL: &str = "pdf-page-v1.1"; -const BYTES_PER_TOKEN: usize = 3; +use crate::oversize::{BYTES_PER_TOKEN, text_pieces}; + +const VERSION_LABEL: &str = "pdf-page-v1.2"; const POLICY_HASH_HEX_LEN: usize = 16; -/// Page-aware PDF chunker. See module docs for the splitting policy and -/// the `chunk_id` collision-avoidance deviation. -#[derive(Clone, Copy, Debug, Default)] -pub struct PdfPageV1Chunker; +/// Page-aware PDF chunker. See module docs for the two-tier splitting +/// policy and the `chunk_id` collision-avoidance deviation. +/// +/// Not a unit struct as of v1.2 — it carries the tier-2 split budget +/// threaded from `config.ingest.chunking.max_chunk_tokens` (mirrors +/// `MdHeadingV2Chunker`). The budget folds into `policy_hash` so a change +/// re-chunks every PDF via the cascade (design §9). +#[derive(Clone, Copy, Debug)] +pub struct PdfPageV1Chunker { + /// Max byte/3 token estimate per emitted chunk. Any tier-1 segment whose + /// `token_estimate` exceeds this is oversize-split at line (then UTF-8 + /// char) boundaries by the shared [`crate::oversize`] primitive. Folded + /// into `policy_hash` so changing this budget triggers a re-chunk via + /// the cascade (design §9). + pub max_chunk_tokens: usize, +} impl Chunker for PdfPageV1Chunker { fn chunker_version(&self) -> ChunkerVersion { ChunkerVersion(VERSION_LABEL.to_string()) } - /// blake3(canonical_json(policy)) truncated to 16 hex chars. Matches - /// the `md-heading-v1` recipe so a workspace-wide policy hash lookup - /// (e.g. for invalidation reports) yields the same digest across - /// chunkers. + /// Policy hash with the v1.2 tier-2 budget folded in. + /// + /// We append the 8 LE bytes of `self.max_chunk_tokens` to the canonical + /// `ChunkPolicy` JSON before hashing. The canonical JSON is + /// self-delimiting (a balanced JSON object), so there is no structural + /// ambiguity between the policy bytes and the trailing 8 budget bytes. + /// Same recipe as [`crate::md_heading_v2::MdHeadingV2Chunker::policy_hash`] + /// — so two PDF instances with different `max_chunk_tokens` produce + /// different `policy_hash` (and chunk_ids), re-indexing all PDFs on a + /// budget change rather than leaving stale oversized chunks. + /// + /// # Panics + /// + /// Panics if canonical JSON serialization of `ChunkPolicy` fails — + /// unreachable in practice. fn policy_hash(&self, policy: &ChunkPolicy) -> String { - let bytes = serde_json_canonicalizer::to_vec(policy) + let mut bytes = serde_json_canonicalizer::to_vec(policy) .expect("canonical JSON serialization of ChunkPolicy must not fail"); + bytes.extend_from_slice(&self.max_chunk_tokens.to_le_bytes()); let hex = blake3::hash(&bytes).to_hex().to_string(); hex[..POLICY_HASH_HEX_LEN].to_string() } @@ -140,44 +195,85 @@ impl Chunker for PdfPageV1Chunker { continue; } - for (segment_start, char_start, char_end, slice) in + for (segment_start, char_start, seg_char_end, slice) in chunk_page(&p.text, target_bytes, overlap_bytes) { - // PDF chars-per-page comfortably fits in u32 (a single - // page maxes out around ~10k chars even for dense - // typography); silent `as u32` truncation would only - // surface on corrupted input, where an explicit panic - // is preferable to an off-by-2^32 span. - let char_start_u32 = u32::try_from(char_start).expect("page chars fit in u32"); - let char_end_u32 = u32::try_from(char_end).expect("page chars fit in u32"); + // ── Tier 2: oversize fallback (pdf-page-v1.2) ──────────── + // The tier-1 greedy splitter (`chunk_page`) can still hand + // back an over-budget slice when a page has no qualifying + // sentence/paragraph boundary (e.g. a dense scanned page + // OCR'd into one long run). Split such a slice into + // sub-pieces each ≤ budget via the shared primitive; the + // common case (slice ≤ budget) returns a single piece and + // its chunk_id / span are byte-identical to the v1.1 + // pre-pass. + let pieces: Vec = if slice.len().div_ceil(BYTES_PER_TOKEN) + > self.max_chunk_tokens + { + text_pieces(&slice, self.max_chunk_tokens) + } else { + vec![slice] + }; + let single_piece = pieces.len() == 1; + + // Page span for the sub-pieces. Exact per-sub-piece char + // narrowing is NOT recoverable from `text_pieces`' `Vec` + // alone: line splits drop the inter-piece '\n' separators (each + // is consumed at the split boundary, present in neither adjacent + // piece), so summing piece char counts drifts earlier by the + // number of those boundaries. So every sub-piece carries the + // PARENT SEGMENT's char range (`char_start..seg_char_end`) — the + // md-heading-v2 block-granular approach. A citation points at the + // correct page region, never a drifted offset; it is coarser than + // per-sub-piece, never wrong. The common single-piece case is the + // whole segment span → byte-identical to the v1.1 pre-pass. + // + // PDF chars-per-page comfortably fits in u32 (~10k chars even for + // dense typography); an explicit panic beats a silent off-by-2^32. + let seg_char_start_u32 = + u32::try_from(char_start).expect("page chars fit in u32"); + let seg_char_end_u32 = + u32::try_from(seg_char_end).expect("page chars fit in u32"); let span = SourceSpan::Page { page: page_num, - char_start: Some(char_start_u32), - char_end: Some(char_end_u32), + char_start: Some(seg_char_start_u32), + char_end: Some(seg_char_end_u32), }; - let block_ids: Vec = vec![p.common.block_id.clone()]; - // v0.20.0 sub-item 1 bugfix (#3): per-chunk policy_hash - // variant uses `segment_start` (pre-overlap boundary, - // strictly increasing) instead of `char_start` (post- - // overlap, may collapse to prev_min). See module docs + - // spec §4.1 root cause + HOTFIXES.md 2026-05-27. - let per_chunk_hash = format!("{base_policy_hash}#c{segment_start}"); - let chunk_id = - id_for_chunk(&doc.doc_id, &chunker_version, &block_ids, &per_chunk_hash); - let token_estimate = slice.len().div_ceil(BYTES_PER_TOKEN); - out.push(Chunk { - chunk_id, - doc_id: DocumentId(doc.doc_id.0.clone()), - block_ids, - tokenized_korean_text: crate::tokenize_korean_morphological(&slice), - text: slice, - heading_path: Vec::new(), - source_spans: vec![span], - token_estimate, - chunker_version: chunker_version.clone(), - policy_hash: base_policy_hash.clone(), - }); + for (i, piece) in pieces.into_iter().enumerate() { + let block_ids: Vec = vec![p.common.block_id.clone()]; + // v0.20.0 sub-item 1 bugfix (#3): per-chunk policy_hash + // variant uses `segment_start` (pre-overlap boundary, + // strictly increasing) instead of `char_start` (post- + // overlap, may collapse to prev_min). See module docs + + // spec §4.1 root cause + HOTFIXES.md 2026-05-27. + // + // v1.2: append `s{i}` for tier-2 sub-pieces so each + // oversize-split piece gets a unique id, while a single + // (non-oversize) piece keeps the bare `#c{segment_start}` + // — common-case chunk_ids stay byte-identical to v1.1. + let per_chunk_hash = if single_piece { + format!("{base_policy_hash}#c{segment_start}") + } else { + format!("{base_policy_hash}#c{segment_start}s{i}") + }; + let chunk_id = + id_for_chunk(&doc.doc_id, &chunker_version, &block_ids, &per_chunk_hash); + let token_estimate = piece.len().div_ceil(BYTES_PER_TOKEN); + + out.push(Chunk { + chunk_id, + doc_id: DocumentId(doc.doc_id.0.clone()), + block_ids, + tokenized_korean_text: crate::tokenize_korean_morphological(&piece), + text: piece, + heading_path: Vec::new(), + source_spans: vec![span.clone()], + token_estimate, + chunker_version: chunker_version.clone(), + policy_hash: base_policy_hash.clone(), + }); + } } } @@ -375,10 +471,21 @@ mod tests { } } + /// Default test budget large enough that the tier-2 oversize fallback + /// never fires — so tests targeting the tier-1 sentence/paragraph + /// splitter keep their pre-v1.2 behavior. Tier-2 tests pass an explicit + /// small budget. + const BIG_BUDGET: usize = 100_000; + + /// Construct the chunker with an explicit tier-2 budget. + fn chunker(max_chunk_tokens: usize) -> PdfPageV1Chunker { + PdfPageV1Chunker { max_chunk_tokens } + } + #[test] fn chunker_version_is_pdf_page_v1() { assert_eq!( - PdfPageV1Chunker.chunker_version(), + chunker(BIG_BUDGET).chunker_version(), ChunkerVersion(VERSION_LABEL.to_string()) ); } @@ -386,7 +493,7 @@ mod tests { #[test] fn three_page_small_emits_one_chunk_per_page() { let doc = make_pdf_doc(&["page one", "page two", "page three"]); - let chunks = PdfPageV1Chunker + let chunks = chunker(BIG_BUDGET) .chunk(&doc, &default_policy(500, 80)) .unwrap(); assert_eq!(chunks.len(), 3); @@ -423,7 +530,7 @@ mod tests { .collect::>() .join("\n\n"); let doc = make_pdf_doc(&[&page_text]); - let chunks = PdfPageV1Chunker + let chunks = chunker(BIG_BUDGET) .chunk(&doc, &default_policy(50, 20)) .unwrap(); assert!( @@ -473,7 +580,7 @@ mod tests { #[test] fn empty_page_produces_no_chunks_for_that_page() { let doc = make_pdf_doc(&["page one", "", "page three"]); - let chunks = PdfPageV1Chunker + let chunks = chunker(BIG_BUDGET) .chunk(&doc, &default_policy(500, 80)) .unwrap(); assert_eq!(chunks.len(), 2); @@ -490,7 +597,7 @@ mod tests { #[test] fn whitespace_only_page_skipped_too() { let doc = make_pdf_doc(&["page one", " \n ", "page three"]); - let chunks = PdfPageV1Chunker + let chunks = chunker(BIG_BUDGET) .chunk(&doc, &default_policy(500, 80)) .unwrap(); assert_eq!(chunks.len(), 2); @@ -543,7 +650,7 @@ mod tests { last_chunker_version: None, last_embedding_version: None, }; - let err = PdfPageV1Chunker + let err = chunker(BIG_BUDGET) .chunk(&doc, &default_policy(500, 80)) .expect_err("non-PDF doc must error"); assert!( @@ -565,7 +672,7 @@ mod tests { big_y.as_str(), ]; let doc = make_pdf_doc(&pages); - let chunks = PdfPageV1Chunker + let chunks = chunker(BIG_BUDGET) .chunk(&doc, &default_policy(50, 10)) .unwrap(); for c in &chunks { @@ -595,14 +702,14 @@ mod tests { &("xyz ".repeat(500)), ]); let policy = default_policy(80, 20); - let baseline: Vec = PdfPageV1Chunker + let baseline: Vec = chunker(BIG_BUDGET) .chunk(&doc, &policy) .unwrap() .into_iter() .map(|c| c.chunk_id.0) .collect(); for _ in 0..1000 { - let again: Vec = PdfPageV1Chunker + let again: Vec = chunker(BIG_BUDGET) .chunk(&doc, &policy) .unwrap() .into_iter() @@ -619,7 +726,7 @@ mod tests { "Hello page 2 with some more body text.", "Hello page 3.", ]); - let chunks = PdfPageV1Chunker + let chunks = chunker(BIG_BUDGET) .chunk(&doc, &default_policy(500, 80)) .unwrap(); assert_eq!(chunks.len(), 3); @@ -661,7 +768,7 @@ mod tests { respect_markdown_headings: false, chunker_version: ChunkerVersion(VERSION_LABEL.into()), }; - let chunks = PdfPageV1Chunker.chunk(&doc, &policy).unwrap(); + let chunks = chunker(BIG_BUDGET).chunk(&doc, &policy).unwrap(); // For each consecutive pair, the new chunk's actual_start must // be strictly greater than the previous chunk's actual_start // (no full re-emission). Without the clamp, equality (full @@ -712,7 +819,7 @@ mod tests { let doc = make_pdf_doc(&[&page_text]); let policy = default_policy(500, 80); // target=1500 byte, overlap=240 byte - let chunks = PdfPageV1Chunker.chunk(&doc, &policy).unwrap(); + let chunks = chunker(BIG_BUDGET).chunk(&doc, &policy).unwrap(); assert!( chunks.len() >= 2, @@ -732,14 +839,221 @@ mod tests { ); } + /// Retargeted from `policy_hash_matches_md_heading_v1_for_identical_policy`. + /// + /// v1.1's PDF `policy_hash` matched `md-heading-v1` (neither folded a + /// budget). v1.2 folds `max_chunk_tokens` into the PDF hash exactly as + /// `md-heading-v2` does — so the meaningful cross-chunker fingerprint + /// identity is now PDF-v1.2 ≡ md-heading-v2 for the SAME budget. We + /// assert that identity (both append the same 8 budget bytes to the same + /// canonical `ChunkPolicy` JSON), preserving the "one policy_hash query + /// covers both chunkers" property across the budget-aware chunkers. #[test] - fn policy_hash_matches_md_heading_v1_for_identical_policy() { - // Cross-chunker policy fingerprint identity — important so a - // workspace-wide "show me chunks with policy_hash = X" query - // covers both chunkers without per-chunker logic. + fn policy_hash_matches_md_heading_v2_for_identical_policy_and_budget() { let p = default_policy(500, 80); - let pdf = PdfPageV1Chunker.policy_hash(&p); - let md = crate::MdHeadingV1Chunker.policy_hash(&p); - assert_eq!(pdf, md); + let budget = 4000; + let pdf = chunker(budget).policy_hash(&p); + let md = crate::MdHeadingV2Chunker { + max_chunk_tokens: budget, + } + .policy_hash(&p); + assert_eq!( + pdf, md, + "pdf-page-v1.2 and md-heading-v2 must share policy_hash for an identical policy + budget" + ); + } + + /// Budget-sensitivity: two PDF instances with different `max_chunk_tokens` + /// must produce different `policy_hash` (and therefore different + /// chunk_ids), so a budget change re-indexes all PDFs via the cascade + /// (design §9). Mirrors `md_heading_v2::budget_in_policy_hash`. + #[test] + fn budget_in_policy_hash() { + let p = default_policy(500, 80); + let a = chunker(100).policy_hash(&p); + let b = chunker(200).policy_hash(&p); + assert_ne!(a, b, "different budgets must yield different policy_hash"); + } + + /// NEW (pdf-page-v1.2): a single page that exceeds a small budget with + /// NO sentence/paragraph boundary (one long run) — the exact v1.1 hole + /// — must now tier-2 split into ≥2 chunks, each ≤ budget. Also asserts + /// text reconstruction, chunk_id uniqueness, and that every sub-piece + /// carries the PARENT SEGMENT's Page char span (segment-granular). + #[test] + fn oversize_pdf_page_splits() { + // 600 chars of "x" with NO whitespace / sentence end / paragraph + // break. target_tokens=500 → 1500 byte tier-1 budget, so tier-1 + // (`chunk_page`) returns the 600-byte page as a SINGLE segment + // (v1.1 would emit one 600-byte chunk — the hole). budget = 20 + // tokens = 60 bytes, so tier-2 must char-split it into ≥10 pieces. + let page_text = "x".repeat(600); + let doc = make_pdf_doc(&[&page_text]); + let budget = 20; + let policy = default_policy(500, 80); + let chunks = chunker(budget).chunk(&doc, &policy).unwrap(); + + // ≥2 chunks (the hole is closed). + assert!( + chunks.len() >= 2, + "oversize no-boundary page must tier-2 split, got {}", + chunks.len() + ); + + // Every chunk ≤ budget. + for c in &chunks { + assert!( + c.text.len().div_ceil(BYTES_PER_TOKEN) <= budget, + "piece exceeds budget: {} > {budget}", + c.text.len().div_ceil(BYTES_PER_TOKEN) + ); + } + + // Concatenation reconstructs the page text (single tier-1 segment, + // char-split → direct concat). + let rejoined: String = chunks.iter().map(|c| c.text.as_str()).collect(); + assert_eq!(rejoined, page_text, "sub-pieces must reconstruct the page text"); + + // chunk_ids unique. + let mut ids: Vec<&str> = chunks.iter().map(|c| c.chunk_id.0.as_str()).collect(); + let total = ids.len(); + ids.sort_unstable(); + ids.dedup(); + assert_eq!(ids.len(), total, "all sub-piece chunk_ids must be unique"); + + // Every sub-piece carries the PARENT SEGMENT's Page char span + // (segment-granular, md-style). This page is a single tier-1 segment, + // so every tier-2 sub-piece spans the whole page `[0, page_chars]` — + // a citation points at the right page region, never a drifted offset. + let page_chars = page_text.chars().count() as u32; + for c in &chunks { + match c.source_spans[0] { + SourceSpan::Page { + page, + char_start: Some(s), + char_end: Some(e), + } => { + assert_eq!(page, 1, "all pieces on page 1"); + assert_eq!(s, 0, "sub-piece span = parent segment start (0)"); + assert_eq!(e, page_chars, "sub-piece span = parent segment end"); + } + ref other => panic!("expected fully-populated Page span, got {other:?}"), + } + } + } + + /// Regression for the v1.2 span-drift bug: a tier-2 split whose oversize + /// slice CONTAINS `\n` line boundaries (the realistic dense-OCR shape). + /// The earlier per-piece char-narrowing summed piece char counts, but + /// `text_pieces` consumes the inter-piece `\n` separators, so the running + /// offset drifted earlier on every line boundary. The segment-granular + /// span (every piece = parent segment range) sidesteps that entirely. + #[test] + fn oversize_pdf_page_with_newlines_splits_without_span_drift() { + // A page of newline-separated short lines, no sentence-end / blank + // line, so tier-1 keeps it as one segment; total > the small budget + // so tier-2 line-splits it into multiple pieces. + let page_text = std::iter::repeat_n("wordword", 80) + .collect::>() + .join("\n"); + let doc = make_pdf_doc(&[&page_text]); + let budget = 20; // 60 bytes/piece → forces a multi-piece line split + let policy = default_policy(500, 80); + let chunks = chunker(budget).chunk(&doc, &policy).unwrap(); + + assert!(chunks.len() >= 2, "newline page must tier-2 split"); + + // Every piece ≤ budget. + for c in &chunks { + assert!( + c.text.len().div_ceil(BYTES_PER_TOKEN) <= budget, + "piece exceeds budget" + ); + } + // '\n'-join reconstructs the page (line splits drop the separator, + // re-added by the join — proves no content is lost or duplicated). + let rejoined = chunks + .iter() + .map(|c| c.text.as_str()) + .collect::>() + .join("\n"); + assert_eq!(rejoined, page_text, "line-split pieces must reconstruct the page"); + + // Every piece carries the parent-segment span [0, page_chars] — no + // drift (the old running-offset would have under-counted by the lost + // '\n' separators and produced char_end < page_chars). + let page_chars = page_text.chars().count() as u32; + for c in &chunks { + match c.source_spans[0] { + SourceSpan::Page { + char_start: Some(s), + char_end: Some(e), + .. + } => { + assert_eq!(s, 0, "segment start"); + assert_eq!(e, page_chars, "segment end (no drift)"); + } + ref other => panic!("expected Page span, got {other:?}"), + } + } + // chunk_ids unique. + let mut ids: Vec<&str> = chunks.iter().map(|c| c.chunk_id.0.as_str()).collect(); + let total = ids.len(); + ids.sort_unstable(); + ids.dedup(); + assert_eq!(ids.len(), total, "sub-piece chunk_ids unique"); + } + + /// NEW (pdf-page-v1.2): a page comfortably under budget must produce + /// output byte-identical to the v1.1 pre-pass — one chunk, full text, + /// full-page char span, and the BARE `#c{segment_start}` id form (no + /// `s{i}` suffix). This locks the common case to "unchanged from v1.1". + #[test] + fn non_oversize_pdf_unchanged() { + let page_text = "A short page that fits well under the budget."; + let doc = make_pdf_doc(&[page_text]); + let policy = default_policy(500, 80); + + // Generous budget — tier-2 never fires. + let v12 = chunker(BIG_BUDGET).chunk(&doc, &policy).unwrap(); + assert_eq!(v12.len(), 1, "under-budget page is a single chunk"); + let c = &v12[0]; + assert_eq!(c.text, page_text, "text is the full page"); + assert_eq!(c.token_estimate, page_text.len().div_ceil(BYTES_PER_TOKEN)); + assert_eq!(c.heading_path, Vec::::new()); + + // Full-page char span (char_start = 0, char_end = page char count). + match c.source_spans[0] { + SourceSpan::Page { + page, + char_start, + char_end, + } => { + assert_eq!(page, 1); + assert_eq!(char_start, Some(0)); + assert_eq!(char_end, Some(page_text.chars().count() as u32)); + } + ref other => panic!("expected Page span, got {other:?}"), + } + + // chunk_id uses the bare `#c0` id form (segment_start = 0, no `s{i}` + // suffix). We reconstruct the expected id with the v1.1 recipe and + // compare — proving the common-case id is byte-identical to a + // single-piece (non-oversize) emission. + let base = chunker(BIG_BUDGET).policy_hash(&policy); + let block_id = match &doc.blocks[0] { + Block::Paragraph(p) => p.common.block_id.clone(), + _ => unreachable!(), + }; + let expected_id = id_for_chunk( + &doc.doc_id, + &chunker(BIG_BUDGET).chunker_version(), + &[block_id], + &format!("{base}#c0"), + ); + assert_eq!( + c.chunk_id, expected_id, + "single-piece chunk_id must use the bare #c{{segment_start}} form (no s-suffix)" + ); } } diff --git a/docs/components/normalize-chunk/README.md b/docs/components/normalize-chunk/README.md index 16cc04a..c0bf05f 100644 --- a/docs/components/normalize-chunk/README.md +++ b/docs/components/normalize-chunk/README.md @@ -7,7 +7,7 @@ | Crate | 역할 | |-------|------| | `kebab-normalize` | `ParsedBlock` (markdown only) → `CanonicalDocument` lift. NFC + heading-path ordinal + provenance 합성 + title fallback chain (p9-fb-07). | -| `kebab-chunk` | `CanonicalDocument` → `Vec`. markdown 기본 `md-heading-v2` (v1 + 예산 초과 청크 일반 분할; v0.30.0), `pdf-page-v1` (PDF). `md-heading-v1` 은 historical 변종으로 잔존. | +| `kebab-chunk` | `CanonicalDocument` → `Vec`. markdown 기본 `md-heading-v2`, PDF `pdf-page-v1.2` — 둘 다 공유 `crate::oversize` (line→char 분할)로 예산 초과 청크를 잘라 모든 청크 ≤ `max_chunk_tokens`. `md-heading-v1`/`pdf-page-v1.1` 은 historical 변종으로 잔존. | ## 구조 @@ -36,7 +36,8 @@ classDiagram split_oversize_chunk(line→char) } class PdfPageV1Chunker { - VERSION = "pdf-page-v1" + VERSION = "pdf-page-v1.2" + max_chunk_tokens BYTES_PER_TOKEN = 3 POLICY_HASH_HEX_LEN = 16 } diff --git a/docs/superpowers/plans/2026-06-24-pdf-page-v1.2-oversize-split.md b/docs/superpowers/plans/2026-06-24-pdf-page-v1.2-oversize-split.md new file mode 100644 index 0000000..65630ab --- /dev/null +++ b/docs/superpowers/plans/2026-06-24-pdf-page-v1.2-oversize-split.md @@ -0,0 +1,89 @@ +--- +title: "pdf-page-v1.2 — PDF 페이지 oversize 청크 분할 (shared oversize module)" +created: 2026-06-24 +status: implemented +extends: tasks/p7/ (pdf-page-v1), HOTFIXES 2026-05-27 (pdf-page-v1.1) +follows: docs/superpowers/plans/2026-06-24-md-heading-v2-oversize-split.md +contract_sections: [§3.5 Chunk, §4.2 chunk_id recipe, §7.2 Chunker, §9 versioning] +design_doc_change: none +--- + +# pdf-page-v1.2 — PDF oversize 청크 분할 + +## 문제 + +md-heading-v2(PR #209)가 markdown/이미지-OCR 텍스트의 예산 초과 청크를 분할하게 +했지만, **PDF 는 별도 청커 `pdf-page-v1.1`** 을 쓴다. v1.1 의 `chunk_page` 는 +문장(`.?!`)/문단(`\n\n`) 경계로만 자르므로, **경계 없는 거대 페이지**(빽빽한 +scanned page 가 한 줄로 OCR 된 경우)는 통째로 한 청크가 되어 strict 임베더(AMD +Lemonade)에서 임베드 실패 — md 와 동일한 hole 이 PDF 에 잔존했다(v1.1 module +doc 에 "accepted limit" 으로 명시돼 있었음). + +## 설계 — 공유 모듈 + 2-tier + +### 1. 공유 `crate::oversize` 모듈 + +md-heading-v2 의 분할 primitive `text_pieces`(줄 경계)·`char_pieces`(UTF-8 char +경계)·`BYTES_PER_TOKEN=3` 을 `crates/kebab-chunk/src/oversize.rs` 로 추출, +`pub(crate)` 로 두 청커가 공유. md-heading-v2 는 이제 `crate::oversize::` 를 호출 +— **출력 byte-identical**(md parity 테스트 전부 통과, md 라벨 불변). 단일 진실 +공급원(DRY)으로 세 번째 복사 방지. + +### 2. pdf-page-v1.2 의 2-tier + +- **Tier 1**(기존): `chunk_page` 의 문장/문단 greedy + overlap. 경계 풍부한 + 페이지는 그대로 — chunk_id 안정. +- **Tier 2**(신규): tier-1 이 내준 segment slice 가 `max_chunk_tokens`(공유 config, + byte/3, default 4000)를 넘으면 `crate::oversize::text_pieces` 로 재분할 → + **모든 PDF 청크 ≤ 예산** 보장(char fallback 이 경계 없는 페이지도 bound). +- `PdfPageV1Chunker { max_chunk_tokens }`(이전 unit struct), `policy_hash()` 에 + budget fold(md 와 동일 8-LE-byte append), `pdf_chunker_from_config`(kebab-app)로 + config 주입. 신규 config 키 없음 — `max_chunk_tokens` 공유. + +### 3. chunk_id sub-piece 스킴 + +분할 조각은 동일 page block_id 공유 → id 충돌. 기존 `#c{segment_start}` 접미사를 +tier-2 에서 `#c{segment_start}s{i}` 로 확장(i = segment 내 0-based). **단일(미분할) +segment 는 bare `#c{segment_start}` 유지** → 미분할 PDF 청크의 hash 컴포넌트는 +v1.1 과 동일(공통 경우 churn 최소). `segment_start` 가 segment-unique + `i` 가 +segment 내 unique → 페이지 전역 고유. + +### 4. Page source_span — segment-granular (중요한 설계 결정) + +분할 조각의 `SourceSpan::Page` char 범위를 **per-piece 로 정밀 narrow 하려던 최초 +구현은 버그였다**: `text_pieces` 가 줄 분할 시 piece 사이의 `'\n'` 구분자를 +**소실**(각 split 경계에서 consume, 인접 piece 어디에도 없음)시키므로, piece char +수를 합산하는 running offset 이 줄 경계마다 1씩 **earlier 로 drift**(코드 리뷰 +실증: `"aaaa\nbbbb\ncccc"` → drift 2). 게다가 한 줄이 char-split 되면 그 piece 들 +사이엔 구분자가 없어 "boundary 당 +1" 보정도 불가능(`Vec` 만으로 복구 +불가). → **md-heading-v2 와 동일하게 모든 sub-piece 가 부모 segment 의 +`char_start..seg_char_end` 를 그대로 갖는다**(segment-granular). citation 은 올바른 +페이지 영역을 가리키며 **절대 drift 하지 않는다**(per-piece 보다 coarse 하나 +never wrong). 미분할 단일 piece 는 정확히 segment span → v1.1 과 동일. + +### 5. 버전 cascade + +`VERSION_LABEL` `pdf-page-v1.1` → **`pdf-page-v1.2`** → 다음 plain `kebab ingest` +에서 PDF 자산 1회 자동 재청크(skip-check mismatch). markdown/code 무영향. +`max_chunk_tokens` 는 이미 `ingest_config_signature` 공통 prefix 에 있어 budget +변경 시 PDF 재색인이 작동(단 v1.1 은 값을 무시했음 → 이제 실효). + +## 검증 + +단위 테스트(`crates/kebab-chunk/src/pdf_page_v1.rs`): `oversize_pdf_page_splits` +(경계 없는 페이지 → tier-2 char-split, 각 ≤ budget), +`oversize_pdf_page_with_newlines_splits_without_span_drift`(줄 포함 페이지 — span +drift 회귀 잠금), `non_oversize_pdf_unchanged`(미분할 = v1.1 동일), +`policy_hash_matches_md_heading_v2_for_identical_policy_and_budget`, +`budget_in_policy_hash`. `crate::oversize` roundtrip 테스트 2종. md parity 테스트 +전부 통과(md 무변경). kebab-chunk lib 93 pass / kebab-app green / clippy `-D +warnings` 0. + +도그푸딩(실험 KB, scanned PDF + arctic@Lemonade): [HOTFIXES 2026-06-24 pdf-page-v1.2 +entry 참조]. + +## 버전 + +`Cargo.toml` workspace version: minor bump(사용자-visible — 빽빽한 scanned PDF 가 +분할되어 검색 hit 분리 + strict 임베더 호환). follow-up #1/#2/#3 와 함께 배치 +릴리스에서 일괄. diff --git a/tasks/HOTFIXES.md b/tasks/HOTFIXES.md index cf4132d..f64f788 100644 --- a/tasks/HOTFIXES.md +++ b/tasks/HOTFIXES.md @@ -14,6 +14,44 @@ historical contract that was implemented; this file accumulates the deltas so phase 5+ readers can find the live behavior without diffing git history. +## 2026-06-24 — pdf-page-v1.2: PDF 페이지 oversize 분할 + 공유 `crate::oversize` 모듈 + +**무엇을 바꿨나.** md-heading-v2 의 oversize 분할 primitive(`text_pieces`/ +`char_pieces`/`BYTES_PER_TOKEN`)를 `crates/kebab-chunk/src/oversize.rs` 공유 모듈로 +추출하고, **PDF 청커를 `pdf-page-v1.1` → `pdf-page-v1.2`** 로 올려 같은 분할을 +적용했다. v1.1 의 `chunk_page` 는 문장/문단 경계로만 잘라서, 경계 없는 거대 +페이지(빽빽한 scanned page 가 한 줄로 OCR 된 경우)가 통째로 한 청크 → strict +임베더에서 실패하는 hole 이 PDF 에 잔존했다(md 와 동형). v1.2 는 **2-tier**: +tier-1(문장/문단 greedy + overlap) 후, segment 가 `max_chunk_tokens`(공유 config, +default 4000) 초과면 tier-2 가 `text_pieces` 로 재분할 → 모든 PDF 청크 ≤ 예산. + +**구현.** `PdfPageV1Chunker { max_chunk_tokens }`(이전 unit struct), `policy_hash` +budget fold(md 와 동일), `pdf_chunker_from_config`(kebab-app)로 config 주입(신규 +config 키 없음). 분할 조각 chunk_id 는 `#c{segment_start}s{i}`(미분할 단일 segment +는 bare `#c{segment_start}` 유지 → 공통 경우 hash 컴포넌트 v1.1 동일). md-heading-v2 +는 공유 모듈을 호출만 하고 **출력 byte-identical**(md 라벨·동작 불변, parity 테스트 +전부 통과). + +**span 버그 발견·수정(코드 리뷰).** 최초 구현은 분할 조각 Page span 의 +char_start/char_end 를 per-piece 로 정밀 narrow 하려 했으나, `text_pieces` 가 줄 +분할 시 piece 사이 `'\n'` 구분자를 소실시켜 running offset 이 줄 경계마다 1씩 +drift(`"aaaa\nbbbb\ncccc"` → drift 2). `Vec` 만으로 복구 불가 → +**md 와 동일하게 부모 segment span(`char_start..seg_char_end`)을 모든 sub-piece 에 +적용**(segment-granular, 절대 drift 없음, never wrong). 회귀 테스트 +`oversize_pdf_page_with_newlines_splits_without_span_drift` 로 잠금. + +**cascade.** `chunker_version` v1.1→v1.2 → 다음 plain `kebab ingest` 에서 PDF 자산 +1회 자동 재청크(markdown/code 무영향). wire/CLI/포맷 불변(검색 hit 의 거대 PDF +페이지가 여러 hit 로 나뉠 수 있음). + +**도그푸딩 evidence**(실험 KB, scanned PDF + paddle-onnx OCR + arctic@Lemonade, +budget 200 으로 tier-2 강제). 625/625 errors=0. scanned_page1.pdf 1→**3 청크** +(max 190 ≤200), scanned_page2.pdf 3→**7 청크**(max 197 ≤200), 둘 다 +`chunker_version=pdf-page-v1.2` 스탬프. 전 코퍼스(markdown+이미지OCR+PDF) **10215 +청크 전부 ≤200, 초과 0**. 단위: kebab-chunk lib 93 pass(span 회귀 테스트 포함), +kebab-app green, clippy `-D warnings` 0. 설계: +`docs/superpowers/plans/2026-06-24-pdf-page-v1.2-oversize-split.md`. + ## 2026-06-24 — config: `[ingest.chunking]` budget floor 검증 **무엇을 바꿨나.** `Config::from_file` 에 `validate_chunking()` 을 추가해 @@ -31,6 +69,7 @@ bloat, 에러 없음)를 냈다. 기존 `target_tokens`/`overlap_tokens` 도 동 없음, 동작 변경 없음 → patch-level. 테스트: `defaults_pass_chunking_validation` + reject 4종 + `from_file_rejects_invalid_chunking`(e2e). + ## 2026-06-24 — md-heading-v2: 예산 초과 청크 일반 분할 (oversize-chunk split) (v0.30.0) **무엇을 바꿨나.** markdown 청커에 새 변종 `md-heading-v2` 를 추가하고