diff --git a/Cargo.lock b/Cargo.lock index 5e3e091..25bf84e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1065,6 +1065,26 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "console_error_panic_hook" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06aeb73f470f66dcdbf7223caeebb85984942f22f1adb2a088cf9668146bbbc" +dependencies = [ + "cfg-if", + "wasm-bindgen", +] + +[[package]] +name = "console_log" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86919cef3e37b9356ccf54d4421208c17ecfda01beae61393e7ffd72916c0ef1" +dependencies = [ + "log", + "web-sys", +] + [[package]] name = "const-random" version = "0.1.18" @@ -4516,9 +4536,11 @@ version = "0.32.0" dependencies = [ "anyhow", "blake3", + "image", "kebab-core", "kebab-parse-image", "lopdf", + "pdfium-render", "serde_json", "strsim", "time", @@ -5283,6 +5305,16 @@ dependencies = [ "cc", ] +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + [[package]] name = "libm" version = "0.2.16" @@ -5638,6 +5670,12 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "maybe-owned" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4facc753ae494aeb6e3c22f839b158aebd4f9270f55cd3c79906c45476c47ab4" + [[package]] name = "maybe-rayon" version = "0.1.1" @@ -6364,6 +6402,32 @@ dependencies = [ "stfu8", ] +[[package]] +name = "pdfium-render" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "826f8f64bc88cb15381cbb5aa245c1500632cd8a453b244bc2b0cbaff7098077" +dependencies = [ + "bitflags 2.11.1", + "bytemuck", + "bytes", + "chrono", + "console_error_panic_hook", + "console_log", + "image", + "itertools 0.14.0", + "js-sys", + "libloading", + "log", + "maybe-owned", + "once_cell", + "utf16string", + "vecmath", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -6442,6 +6506,12 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "piston-float" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad78bf43dcf80e8f950c92b84f938a0fc7590b7f6866fbcbeca781609c115590" + [[package]] name = "pkg-config" version = "0.3.33" @@ -9110,6 +9180,15 @@ dependencies = [ "serde", ] +[[package]] +name = "utf16string" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b62a1e85e12d5d712bf47a85f426b73d303e2d00a90de5f3004df3596e9d216" +dependencies = [ + "byteorder", +] + [[package]] name = "utf8-ranges" version = "1.0.5" @@ -9163,6 +9242,15 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "vecmath" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "956ae1e0d85bca567dee1dcf87fb1ca2e792792f66f87dced8381f99cd91156a" +dependencies = [ + "piston-float", +] + [[package]] name = "version_check" version = "0.9.5" diff --git a/README.md b/README.md index 9e8d705..1473604 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ rsync -a /lancedb/ user@server:/lancedb/ ### 멀티미디어 색인 -Markdown · PDF · 이미지(OCR + caption) · 소스코드(Rust/Python/TS/JS/Go/Java/Kotlin/C/C++ AST) · 리소스(YAML/Dockerfile/TOML/JSON/XML 등)를 확장자에 따라 자동으로 적절한 chunker 에 라우팅한다. embedded text 가 없는 scanned PDF 는 `[ingest.pdf.ocr]` 로 page-단위 OCR (opt-in). 전체 확장자→chunker 매핑은 [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md). +Markdown · PDF · 이미지(OCR + caption) · 소스코드(Rust/Python/TS/JS/Go/Java/Kotlin/C/C++ AST) · 리소스(YAML/Dockerfile/TOML/JSON/XML 등)를 확장자에 따라 자동으로 적절한 chunker 에 라우팅한다. embedded text 가 없는 scanned PDF 는 `[ingest.pdf.ocr]` 로 page-단위 OCR (opt-in; 인코딩 무관하게 읽으려면 `render_library` 에 libpdfium 지정). 전체 확장자→chunker 매핑은 [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md). ### RAG (근거 인용 + 거절) @@ -91,7 +91,7 @@ Markdown · PDF · 이미지(OCR + caption) · 소스코드(Rust/Python/TS/JS/Go | `kebab fetch chunk\|doc\|span [flags]` | indexed corpus 에서 verbatim text fetch | | `kebab eval run \| aggregate \| compare \| variants` | golden query 회귀 측정 + 변형 일관성 진단. `compare --max-drop <낙폭>` 은 어떤 지표든 그 이상 **떨어지면** exit 1 — 절대 하한이 아니라 델타 예산이다 (없으면 delta 만 출력하고 항상 exit 0) | | `kebab schema [--json]` | introspection — wire schemas / capabilities / models / stats | -| `kebab doctor` | 설정 / 모델 / DB 헬스 체크. `vector_store` 체크는 Lance fragment·버전 수를 정보성으로 보여준다(종료 코드에 영향 없음). `fts_shadow` 체크는 어휘 인덱스가 원본과 어긋났는지 보며, **어긋나면 exit 3** — 이 상태에서는 문서 삭제가 엉뚱한 인덱스 행을 지운다 | +| `kebab doctor` | 설정 / 모델 / DB 헬스 체크. `vector_store` 체크는 Lance fragment·버전 수를 정보성으로 보여준다(종료 코드에 영향 없음). `fts_shadow` 체크는 어휘 인덱스가 원본과 어긋났는지 보며, **어긋나면 exit 3** — 이 상태에서는 문서 삭제가 엉뚱한 인덱스 행을 지운다. `pdf_render` 체크는 스캔 PDF 페이지 렌더러(pdfium)가 있는지 정보성으로 알려준다 | | `kebab mcp` | MCP stdio server (`search` / `bulk_search` / `ask` / `fetch` / `schema` / `doctor` / `ingest_file` / `ingest_stdin`) | | `kebab reset [--all \| --data-only \| --vector-only \| --config-only \| --orphans-only] [--yes]` | XDG 데이터 wipe (**irreversible**) | @@ -171,6 +171,8 @@ nli_threshold = 0.0 # >0 (예: 0.5) 면 mDeBERTa XNLI groundedn - **`[ingest.ocr]`** (config schema v5) — image/pdf OCR 가 공유하는 **엔진** 설정의 단일 출처 (`engine`/`model`/`endpoint`/`languages`/`max_pixels`/`request_timeout_secs` + paddle 모델 경로·튜닝 키). 여기에 한 번 적어 두면 image·pdf 양쪽에 적용되고, 각 미디어 블록(`[ingest.image.ocr]`/`[ingest.pdf.ocr]`)이 자기 키로 override 한다 (우선순위: 미디어 블록 > `[ingest.ocr]` > 내장 기본값). 옛 v4 `config.toml` 의 image/pdf 에 중복돼 있던 OCR 엔진 키는 로드 시 자동으로 이 블록으로 통합된다 (effective 값 불변, 자동 재색인 없음). env override 도 `KEBAB_OCR_*` 하나로 통합 (양쪽 미디어에 적용). - **`[ingest.image.ocr]`** — 이미지 OCR. on/off 토글(`enabled`, default off / opt-in)은 미디어별이며, 엔진 설정은 `[ingest.ocr]` 에서 상속하되 이 블록에서 override 할 수 있다. `engine` 으로 백엔드 선택: `"ollama-vision"` (default, 원격 vision LM) 또는 `"paddle-onnx"` (PP-OCRv5 ONNX 를 in-process 로 실행, Python 런타임 불필요, 큰 페이지 CPU <4초, 오프라인). `paddle-onnx` 는 워크스페이스에 번들된 모델을 쓰며 `det_model`/`rec_model`/`dict` 로 경로 override, `score_thresh`(0.3)/`unclip_ratio`(1.5)/`max_boxes`(1000) 로 검출 튜닝 가능. engine 또는 모델을 바꾸면 영향 이미지가 자동 재색인된다. - **`[ingest.pdf.ocr]`** — scanned PDF 의 page-단위 OCR (default off / opt-in, page 당 ~수십 초 cost). on/off 토글(`enabled`/`always_on`)과 PDF 고유 키(`valid_ratio_threshold`/`min_char_count`/`lang_hint`)는 미디어별이고, 엔진 설정은 `[ingest.ocr]` 에서 상속하되 이 블록에서 override 한다(PDF 기본 모델은 `qwen2.5vl:3b`, 이미지의 `gemma4:e4b` 와 다름 — 미디어별 기본값 보존). 활성화 후 옛 색인분은 `kebab ingest --force-reingest` 로 재처리. + + **스캔본을 제대로 읽으려면 페이지 렌더러가 필요하다.** `render_library` 에 `libpdfium` 경로를 적거나 로더가 찾는 곳에 두면, 페이지 이미지가 어떤 인코딩이든(CCITTFax·JBIG2·Flate·JPX, 배경+마스크 분리 구조 포함) OCR 된다. 없으면 **단일 DCTDecode(JPEG) 이미지 페이지만** OCR 되고 나머지는 본문 없이 색인되며, 그 건수가 ingest 요약의 `ocr-skipped` 와 `--json` 의 `ocr_skipped_pages` 에 찍힌다. `kebab doctor` 의 `pdf_render` 가 지금 어느 쪽인지 알려준다. `render_dpi` 는 렌더 해상도 **요청**(기본 300)이고 실제로는 `max_pixels` 가 이긴다 — PDF 기본값 `max_pixels = 2048` 이면 A4 는 175 DPI 언저리에서 잘린다. 300 을 실제로 쓰려면 `max_pixels` 를 3500 이상으로 올려야 하고, 그만큼 큰 이미지를 OCR 엔진이 받는다. 다만 OCR 엔진(ollama-vision·paddle-onnx 둘 다)이 `max_pixels` 를 256~4096 으로 다시 조인다 — 그보다 크게 적어도 4096 이 상한이다. pdfium 은 공유 라이브러리로만 배포돼서 바이너리에 넣지 않았다 — kebab 자체는 단일 실행 파일 그대로다. - **`--config `** — 임시 워크스페이스 / 격리 테스트용 (CLI honor). - **`kebab config migrate`** — 새 버전에서 추가된 config 섹션을 기존 `config.toml` 에 설명 주석과 함께 채워 넣는다 (사용자가 손본 값·주석·순서는 보존, 멱등, 변경 시 자동 `.bak` 백업). `--dry-run` 으로 변경 미리보기. `kebab doctor` 가 갱신 필요 시 안내한다. `kebab init` 으로 새로 생성되는 config.toml 도 섹션별 주석을 포함한다. - **`KEBAB_*` env** — 런타임 override용 ~22개 키만 노출. 엔드포인트(`KEBAB_MODELS_LLM_ENDPOINT`, `KEBAB_MODELS_EMBEDDING_ENDPOINT`, `KEBAB_OCR_ENDPOINT`), 모델명/프로바이더(`KEBAB_MODELS_LLM_MODEL`, `KEBAB_MODELS_EMBEDDING_MODEL`, `KEBAB_MODELS_EMBEDDING_PROVIDER`, `KEBAB_MODELS_LLM_PROVIDER`, `KEBAB_MODELS_NLI_MODEL`), 경로(`KEBAB_WORKSPACE_ROOT`, `KEBAB_STORAGE_DATA_DIR`), 병렬도(`KEBAB_INDEXING_MAX_PARALLEL_EXTRACTORS`, `KEBAB_INDEXING_MAX_PARALLEL_EMBEDDINGS`), 청킹(`KEBAB_CHUNKING_TARGET_TOKENS`, `KEBAB_CHUNKING_OVERLAP_TOKENS`), OCR 토글/엔진/언어(`KEBAB_IMAGE_OCR_ENABLED`, `KEBAB_PDF_OCR_ENABLED`, `KEBAB_OCR_ENGINE`, `KEBAB_OCR_MODEL`, `KEBAB_OCR_LANGUAGES`), 기타(`KEBAB_IMAGE_CAPTION_ENABLED`, `KEBAB_SEARCH_DEFAULT_K`, `KEBAB_RAG_PROMPT_TEMPLATE_VERSION`). 나머지 세부 튜닝 키(score_gate, rrf_k, temperature 등)는 `config.toml` 전용. 특수: `KEBAB_READONLY=1`(write-path 비활성), `KEBAB_PROGRESS=plain`(non-TTY 진행 출력), `KEBAB_EVAL_GOLDEN`(eval golden set 경로). diff --git a/crates/kebab-app/src/ingest.rs b/crates/kebab-app/src/ingest.rs index 2bb7638..bb12d78 100644 --- a/crates/kebab-app/src/ingest.rs +++ b/crates/kebab-app/src/ingest.rs @@ -130,6 +130,10 @@ pub fn ingest_with_config( let ocr_ms_samples: Arc>> = Arc::new(Mutex::new(Vec::new())); let ocr_pages_cnt: Arc> = Arc::new(Mutex::new(0u32)); let ocr_failures_cnt: Arc> = Arc::new(Mutex::new(0u32)); + // Pages the text gate wanted OCR'd but that produced no raster + // (issue #232). Surfaced on `IngestReport` so a scan that indexed + // nothing is visible in the run's own output, not just in stderr. + let ocr_skipped_cnt: Arc> = Arc::new(Mutex::new(0u32)); // v0.20.x r2: prune stale pdf_ocr_events rows once per ingest run. let _pruned = app @@ -271,6 +275,44 @@ pub fn ingest_with_config( None }; + // Page renderer for scanned PDFs (issue #232). Bound once per run — + // binding walks the loader path — and shared by every PDF. + // + // Not fail-fast, unlike the OCR engine above: a missing pdfium is a + // narrower capability than a missing OCR engine. Ingest still reads + // text PDFs and still OCRs pages that are a single embedded JPEG, so + // aborting the run would cost the user more than the gap does. The + // warning names the config key, and `kebab doctor` reports the mode. + let pdf_renderer: Option> = + if app.config.pdf_ocr().enabled || app.config.pdf_ocr().always_on { + let configured = app.config.pdf_ocr().render_library.clone(); + let path = configured + .as_deref() + .map(|p| kebab_config::expand_path(p, "")); + match kebab_parse_pdf::PageRenderer::bind(path.as_deref()) { + Ok(r) => { + tracing::info!( + target: "kebab-app", + source = r.source(), + "pdf page renderer ready — scanned pages of any encoding will be OCR'd" + ); + Some(Arc::new(r)) + } + Err(e) => { + tracing::warn!( + target: "kebab-app", + error = %e, + "no pdf page renderer — only pages that are a single DCTDecode image \ + can be OCR'd. Set `[ingest.pdf.ocr] render_library` to a libpdfium \ + to cover CCITTFax / JBIG2 / Flate / JPX scans" + ); + None + } + } + } else { + None + }; + // Pre-load every existing doc_id so we can label `IngestItem.kind` // as `New` vs `Updated` correctly. `list_documents` returns one // row per `(workspace_path, asset_id)` — index by the deterministic @@ -383,12 +425,14 @@ pub fn ingest_with_config( &image_pipeline, force_reingest, pdf_ocr_engine.as_deref(), + pdf_renderer.as_ref(), progress, opts.cancel.as_ref(), log_writer.clone(), ocr_ms_samples.clone(), ocr_pages_cnt.clone(), ocr_failures_cnt.clone(), + ocr_skipped_cnt.clone(), ); let item = match item { @@ -675,6 +719,7 @@ pub fn ingest_with_config( skipped_size_exceeded: fs_skips.skipped_size_exceeded, skip_examples: fs_skips.skip_examples, purged_deleted_files, + ocr_skipped_pages: ocr_skipped_cnt.lock().map_or(0, |v| *v), items: if opts.summary_only { None } else { Some(items) }, }) } @@ -1145,12 +1190,14 @@ fn ingest_one_asset( image_pipeline: &ImagePipeline<'_>, force_reingest: bool, pdf_ocr_engine: Option<&dyn OcrEngine>, + pdf_renderer: Option<&Arc>, progress: Option<&std::sync::mpsc::Sender>, cancel: Option<&std::sync::Arc>, log_writer: Option>>, ocr_ms_samples: Arc>>, ocr_pages_cnt: Arc>, ocr_failures_cnt: Arc>, + ocr_skipped_cnt: Arc>, ) -> anyhow::Result { tracing::debug!( target: "kebab-app::ingest", @@ -1194,12 +1241,14 @@ fn ingest_one_asset( source_id, force_reingest, pdf_ocr_engine, + pdf_renderer, progress, cancel, log_writer, ocr_ms_samples, ocr_pages_cnt, ocr_failures_cnt, + ocr_skipped_cnt, ); } // p10-1A-2 / 1B: code ingest dispatch. p10-2: Tier 2 langs added. p10-3: shell added. p10-1D: c/cpp added. @@ -2462,12 +2511,14 @@ fn ingest_one_pdf_asset( source_id: &str, force_reingest: bool, pdf_ocr_engine: Option<&dyn OcrEngine>, + pdf_renderer: Option<&Arc>, progress: Option<&std::sync::mpsc::Sender>, cancel: Option<&std::sync::Arc>, log_writer: Option>>, ocr_ms_samples: Arc>>, ocr_pages_cnt: Arc>, ocr_failures_cnt: Arc>, + ocr_skipped_cnt: Arc>, ) -> anyhow::Result { let path = match &asset.source_uri { SourceUri::File(p) => p.clone(), @@ -2490,7 +2541,7 @@ fn ingest_one_pdf_asset( } }; // p9-fb-23 task 7: incremental-ingest early-skip for the PDF flow. - // PDF docs use `pdf-text-v1` as the parser_version and `PdfPageV1Chunker` + // PDF docs use `pdf-text-v2` as the parser_version and `PdfPageV1Chunker` // as the chunker — both pinned per-medium today (no config knob). // v0.26.2: composite parser_version folds pdf.ocr (enabled/always_on/ // model) + chunking, so enabling scanned-PDF OCR auto-re-indexes PDFs. @@ -2527,7 +2578,7 @@ fn ingest_one_pdf_asset( let mut canonical = app .extract_for(&asset.media_type, &ctx, &bytes) .context("kb-app::extract_for (pdf)")?; - // v0.26.2: store the composite parser_version (base `pdf-text-v1` already + // v0.26.2: store the composite parser_version (base `pdf-text-v2` already // fixed doc_id) so the next run's skip compare matches. canonical.parser_version = eff_parser_version.clone(); // `[[workspace.sources]]`: stamp the owning source id (pdf extractor @@ -2561,6 +2612,9 @@ fn ingest_one_pdf_asset( cancel: cancel.cloned(), ocr_cache: Some(Arc::clone(&app.sqlite)), ocr_version_key: pdf_ocr_vkey, + renderer: pdf_renderer.cloned(), + render_dpi: pdf_ocr.render_dpi, + max_pixels: pdf_ocr.max_pixels, }; // v0.20.x Hook 2: pre-clone Arcs for capture by OCR closure. let lw_for_ocr = log_writer.clone(); @@ -2669,6 +2723,11 @@ fn ingest_one_pdf_asset( } }, )?; + if summary.pages_skipped > 0 + && let Ok(mut s) = ocr_skipped_cnt.lock() + { + *s = s.saturating_add(summary.pages_skipped); + } (Some(summary.pages_ocrd), Some(summary.ms_total)) } None => (Some(0), Some(0)), diff --git a/crates/kebab-app/src/lib.rs b/crates/kebab-app/src/lib.rs index 040bafb..8e967e2 100644 --- a/crates/kebab-app/src/lib.rs +++ b/crates/kebab-app/src/lib.rs @@ -445,6 +445,49 @@ pub fn doctor_with_config_path( } } + // pdf_render — is a page renderer available (issue #232). + // + // Reported because the capability is invisible otherwise: without a + // renderer, a scanned PDF still ingests "successfully" and simply + // contains no text, and the user finds out when a search comes back + // empty. Informational rather than a failure — a corpus with no + // scanned PDFs loses nothing by not having pdfium, and making doctor + // exit 3 over an unused capability would be noise. + { + let cfg = match loaded_cfg.as_ref() { + Some(c) => { + let env: std::collections::HashMap = std::env::vars().collect(); + c.clone().apply_env(&env) + } + None => kebab_config::Config::defaults(), + }; + let configured = cfg.pdf_ocr().render_library.clone(); + let path = configured + .as_deref() + .map(|p| kebab_config::expand_path(p, "")); + let (detail, hint) = match kebab_parse_pdf::PageRenderer::bind(path.as_deref()) { + Ok(r) => ( + format!( + "pdfium ({}) — 모든 인코딩의 스캔 페이지 OCR 가능", + r.source() + ), + None, + ), + Err(e) => ( + "페이지 렌더러 없음 — 단일 DCTDecode 이미지 페이지만 OCR 된다".to_string(), + Some(format!( + "CCITTFax / JBIG2 / Flate / JPX 스캔은 본문 없이 색인된다. libpdfium 을 로더 경로에 두거나 `[ingest.pdf.ocr] render_library` 로 지정하라 ({e})" + )), + ), + }; + checks.push(DoctorCheck { + name: "pdf_render".to_string(), + ok: true, + detail, + hint, + }); + } + // fts_shadow — chunks_fts rowid alignment (issue #229 / V016). // // V016 made the FTS delete trigger address rows by rowid instead of diff --git a/crates/kebab-app/src/pdf_ocr_apply.rs b/crates/kebab-app/src/pdf_ocr_apply.rs index f0601f2..afaebeb 100644 --- a/crates/kebab-app/src/pdf_ocr_apply.rs +++ b/crates/kebab-app/src/pdf_ocr_apply.rs @@ -66,6 +66,20 @@ pub struct PdfOcrOpts { /// Version key folded into the per-page OCR cache key (§3.3). Empty when /// `ocr_cache` is `None`. pub ocr_version_key: String, + /// Page renderer (issue #232). `Some` → every page rasterizes + /// regardless of how its images are encoded. `None` → fall back to + /// pulling an embedded DCTDecode JPEG, which covers only pages that + /// are exactly one JPEG and silently skips CCITTFax / JBIG2 / Flate / + /// JPX scans and pages that split content across a mask. + /// + /// Optional because pdfium ships only as a shared library; see + /// `PdfOcrCfg::render_library`. + pub renderer: Option>, + /// Rendering resolution (DPI) when `renderer` is `Some`. + pub render_dpi: u32, + /// Long-edge pixel ceiling the OCR engine will accept. Caps the + /// rendered size no matter what `render_dpi` asks for. + pub max_pixels: u32, } /// OCR run summary returned by [`apply_ocr_to_pdf_pages`] for the caller's @@ -78,6 +92,126 @@ pub struct PdfOcrSummary { /// engine call on a cache miss, or the (sub-ms) cache read on a hit. /// `saturating_add` 사용 — 24-day cumulative 까지 overflow-safe. pub ms_total: u64, + /// Pages the text gate said needed OCR but that produced no raster to + /// OCR (issue #232). Counted so the caller can surface it — a run + /// where every page of a scan lands here indexes an empty document + /// and used to report success. + pub pages_skipped: u32, +} + +/// Why a page produced no raster to OCR. +/// +/// Issue #232 asked for this to reach the user: the wire event already +/// carried a `failure_reason`, but everything funnelled into one +/// "no DCTDecode or engine fail" string, so a page dropped for its +/// encoding was indistinguishable from one the OCR engine choked on. +enum RasterFailure { + /// No renderer configured, and the page is not a single embedded + /// JPEG. Carries the `/Filter` names actually present, because that + /// is the fact that explains the skip. + NoRenderer(Vec), + /// A renderer was configured and rasterizing this page failed. + Render(String), + /// A renderer was configured but could not open this PDF at all, so + /// no page of it can be rendered. Distinct from `NoRenderer` because + /// telling a user who already configured a renderer to configure one + /// is the wrong instruction. + Unopenable, +} + +impl RasterFailure { + /// Stable code for the wire event, so agents can branch on the cause + /// rather than parse prose. + fn reason_code(&self) -> &'static str { + match self { + Self::NoRenderer(_) => "no_renderer", + Self::Render(_) => "render_error", + Self::Unopenable => "unopenable_pdf", + } + } + + fn describe(&self) -> String { + match self { + Self::NoRenderer(filters) => { + let seen = if filters.is_empty() { + "no image XObject (vector-only page)".to_string() + } else { + format!("/Filter={}", filters.join(", ")) + }; + format!( + "{seen} — without a page renderer only a single DCTDecode image can be read. \ + Set `[ingest.pdf.ocr] render_library` to a libpdfium, or install one where the \ + loader finds it; `kebab doctor` reports which mode is active." + ) + } + Self::Render(e) => format!("page renderer failed: {e}"), + Self::Unopenable => { + "the page renderer could not open this PDF, so no page of it could be \ + rasterized; the file may be malformed or encrypted" + .to_string() + } + } + } +} + +/// The `/Filter` names on a page's image XObjects, in a stable order. +/// +/// Only used to explain a skip. Sorted and deduplicated so the message is +/// the same across runs — lopdf's dictionary iteration order is not. +fn page_filters(doc: &LopdfDocument, page_num: u32) -> Vec { + use lopdf::Object; + + let Some(&page_oid) = doc.get_pages().get(&page_num) else { + return Vec::new(); + }; + let Ok(page) = doc.get_dictionary(page_oid) else { + return Vec::new(); + }; + let resources = match page.get(b"Resources").ok() { + Some(Object::Dictionary(d)) => d.clone(), + Some(Object::Reference(r)) => match doc.get_dictionary(*r) { + Ok(d) => d.clone(), + Err(_) => return Vec::new(), + }, + _ => return Vec::new(), + }; + let xobject = match resources.get(b"XObject").ok() { + Some(Object::Dictionary(d)) => d.clone(), + Some(Object::Reference(r)) => match doc.get_dictionary(*r) { + Ok(d) => d.clone(), + Err(_) => return Vec::new(), + }, + _ => return Vec::new(), + }; + + let mut names = Vec::new(); + for (_key, obj) in xobject.iter() { + let Object::Reference(r) = obj else { continue }; + let Ok(Object::Stream(stream)) = doc.get_object(*r) else { + continue; + }; + let is_image = matches!( + stream.dict.get(b"Subtype").ok(), + Some(Object::Name(n)) if n.as_slice() == b"Image" + ); + if !is_image { + continue; + } + match stream.dict.get(b"Filter").ok() { + Some(Object::Name(n)) => names.push(String::from_utf8_lossy(n).into_owned()), + Some(Object::Array(arr)) => { + for f in arr { + if let Object::Name(n) = f { + names.push(String::from_utf8_lossy(n).into_owned()); + } + } + } + _ => {} + } + } + names.sort_unstable(); + names.dedup(); + names } /// Post-extract OCR enrichment for PDF. Walks `canonical.blocks` page-by-page, @@ -111,15 +245,37 @@ where return Ok(PdfOcrSummary { pages_ocrd: 0, ms_total: 0, + pages_skipped: 0, }); } let pdf_doc = LopdfDocument::load_mem(pdf_bytes) .context("kb-app::pdf_ocr_apply: re-parse PDF for image extract")?; let page_count = pdf_doc.get_pages().len() as u32; + // Open the PDF once through pdfium when a renderer is configured. A + // failure here is not fatal: the DCTDecode path still reads the pages + // that are a single JPEG, and reporting per page is what tells the + // user which pages lost content and why. + let renderer_configured = opts.renderer.is_some(); + let rendered = opts + .renderer + .as_ref() + .and_then(|r| match r.open(pdf_bytes, None) { + Ok(doc) => Some(doc), + Err(e) => { + warn!( + target: "kebab-app", + error = %e, + "pdfium could not open this PDF; page rendering unavailable for it" + ); + None + } + }); + let mut new_events: Vec = Vec::new(); let mut ocr_blocks: Vec = Vec::new(); let mut pages_ocrd: u32 = 0; + let mut pages_skipped: u32 = 0; let mut ms_total: u64 = 0; // canonical.blocks 의 page → block index map (text-detect block 의 in-place @@ -153,30 +309,90 @@ where emit_progress(PdfOcrProgress::Started { page: page_num }); - let page_image_bytes = if let Some(b) = extract_dctdecode_page_image(&pdf_doc, page_num)? { - b - } else { - let note = format!( - "page={page_num} skipped: no DCTDecode image XObject (vector PDF page or unsupported /Filter — v1 supports DCTDecode passthrough only; see release notes for normalization guidance)" - ); - warn!(target: "kebab-app", "{}", note); - new_events.push(ProvenanceEvent { - at: OffsetDateTime::now_utc(), - agent: "kb-parse-pdf".to_string(), - kind: ProvenanceKind::Warning, - note: Some(note), - }); - emit_progress(PdfOcrProgress::Finished { - page: page_num, - ms: 0, - chars: 0, - skipped: true, - image_byte_size: None, - image_width: None, - image_height: None, - failure_reason: None, - }); - continue; + // Read back an embedded JPEG. Only works when the page is exactly + // one DCTDecode image, which is why rendering exists — but it is + // still the right thing to try when rendering is unavailable or + // fails for this particular page. + let dct = |doc: &LopdfDocument| { + extract_dctdecode_page_image(doc, page_num).inspect_err(|e| { + // Swallowed by the callers below on purpose — a page-local + // parse failure must not abort the document. Logged so the + // reason is recoverable, matching the OCR cache GET below. + tracing::debug!( + target: "kebab-app::pdf_ocr", + page = page_num, + error = %e, + "DCTDecode extraction failed; treating as no raster for this page" + ); + }) + }; + + let rasterized = match (rendered.as_ref(), renderer_configured) { + (Some(doc), _) => match doc.render_page_png(page_num, opts.render_dpi, opts.max_pixels) + { + Ok(png) => Ok(png), + // Per-page fallback: one page failing to render must not + // cost content that the DCTDecode path could still read. + // Configuring a renderer should never make a page worse + // off than not having one. + // + // `.ok()` rather than `?`: a page that fails to render is + // often a page whose lopdf dictionary is also malformed, + // and propagating that error here would turn one bad page + // into an aborted document — the opposite of this loop's + // per-page `continue`-on-error discipline, and the same + // silent-total-loss shape this whole change is about. + // + // Defensive rather than demonstrated: the loop only visits + // pages `get_pages()` lists, which is the same lookup that + // makes `extract_dctdecode_page_image` fail, so no fixture + // reaches this arm's error path. An attempt at a test for + // it passed while exercising nothing and was removed + // rather than kept as false coverage. + Err(e) => match dct(&pdf_doc).ok().flatten() { + Some(b) => Ok(b), + None => Err(RasterFailure::Render(e.to_string())), + }, + }, + // A renderer is configured but could not open this PDF. Same + // reasoning as above and stronger: reaching here means pdfium + // rejected the whole document, so the odds that lopdf also + // stumbles on it are at their highest — exactly where an `?` + // would turn one file into an aborted run. + (None, true) => match dct(&pdf_doc).ok().flatten() { + Some(b) => Ok(b), + None => Err(RasterFailure::Unopenable), + }, + (None, false) => match dct(&pdf_doc)? { + Some(b) => Ok(b), + None => Err(RasterFailure::NoRenderer(page_filters(&pdf_doc, page_num))), + }, + }; + + let page_image_bytes = match rasterized { + Ok(b) => b, + Err(why) => { + let note = format!("page={page_num} skipped: {}", why.describe()); + warn!(target: "kebab-app", "{}", note); + new_events.push(ProvenanceEvent { + at: OffsetDateTime::now_utc(), + agent: "kb-parse-pdf".to_string(), + kind: ProvenanceKind::Warning, + note: Some(note), + }); + pages_skipped = pages_skipped.saturating_add(1); + emit_progress(PdfOcrProgress::Finished { + page: page_num, + ms: 0, + chars: 0, + skipped: true, + image_byte_size: None, + image_width: None, + image_height: None, + failure_reason: Some(why.reason_code().to_string()), + }); + continue; + } }; let start = Instant::now(); @@ -360,6 +576,7 @@ where Ok(PdfOcrSummary { pages_ocrd, ms_total, + pages_skipped, }) } diff --git a/crates/kebab-app/tests/common/mod.rs b/crates/kebab-app/tests/common/mod.rs index 286150b..672f8f9 100644 --- a/crates/kebab-app/tests/common/mod.rs +++ b/crates/kebab-app/tests/common/mod.rs @@ -107,8 +107,15 @@ pub fn ingest_md(env: &TestEnv, relative_path: &str, content: &str) { std::fs::create_dir_all(parent).expect("create parent dirs"); } std::fs::write(&path, content).expect("write workspace file"); - kebab_app::ingest_with_config(env.config.clone(), env.scope(), kebab_app::IngestOpts { summary_only: true, ..Default::default() }) - .expect("ingest_with_config"); + kebab_app::ingest_with_config( + env.config.clone(), + env.scope(), + kebab_app::IngestOpts { + summary_only: true, + ..Default::default() + }, + ) + .expect("ingest_with_config"); } /// Test helper: build a `SearchQuery` for lexical mode at k=10. Used diff --git a/crates/kebab-app/tests/multi_scanned_pdf_ingest_no_chunk_id_collision.rs b/crates/kebab-app/tests/multi_scanned_pdf_ingest_no_chunk_id_collision.rs index 425592b..7b04a90 100644 --- a/crates/kebab-app/tests/multi_scanned_pdf_ingest_no_chunk_id_collision.rs +++ b/crates/kebab-app/tests/multi_scanned_pdf_ingest_no_chunk_id_collision.rs @@ -65,6 +65,9 @@ fn extract_and_ocr( cancel: None, ocr_cache: None, ocr_version_key: String::new(), + renderer: None, + render_dpi: 300, + max_pixels: 4096, }; apply_ocr_to_pdf_pages(&mut canonical, engine, bytes, &opts, |_| {}).unwrap(); canonical diff --git a/crates/kebab-app/tests/ocr_caption_cache.rs b/crates/kebab-app/tests/ocr_caption_cache.rs index f6776ba..69df6f0 100644 --- a/crates/kebab-app/tests/ocr_caption_cache.rs +++ b/crates/kebab-app/tests/ocr_caption_cache.rs @@ -108,6 +108,9 @@ fn opts_with_cache(store: &Arc, version_key: &str) -> PdfOcrOpts { cancel: None, ocr_cache: Some(Arc::clone(store)), ocr_version_key: version_key.to_string(), + renderer: None, + render_dpi: 300, + max_pixels: 4096, } } diff --git a/crates/kebab-app/tests/pdf_ocr_apply.rs b/crates/kebab-app/tests/pdf_ocr_apply.rs index c9a1cdb..a6e6848 100644 --- a/crates/kebab-app/tests/pdf_ocr_apply.rs +++ b/crates/kebab-app/tests/pdf_ocr_apply.rs @@ -101,6 +101,11 @@ fn default_opts(enabled: bool) -> PdfOcrOpts { cancel: None, ocr_cache: None, ocr_version_key: String::new(), + // No renderer in these unit tests: they pin the fallback path's + // behavior, which is what a machine without pdfium gets. + renderer: None, + render_dpi: 300, + max_pixels: 4096, } } @@ -121,6 +126,11 @@ fn f1_input_with_ocr_enabled_replaces_empty_block() { cancel: None, ocr_cache: None, ocr_version_key: String::new(), + // No renderer in these unit tests: they pin the fallback path's + // behavior, which is what a machine without pdfium gets. + renderer: None, + render_dpi: 300, + max_pixels: 4096, }; let summary = apply_ocr_to_pdf_pages(&mut canonical, &engine, &bytes, &opts, |_| {}).unwrap(); @@ -184,6 +194,11 @@ fn f4_input_with_ocr_enabled_replaces_mojibake_block() { cancel: None, ocr_cache: None, ocr_version_key: String::new(), + // No renderer in these unit tests: they pin the fallback path's + // behavior, which is what a machine without pdfium gets. + renderer: None, + render_dpi: 300, + max_pixels: 4096, }; let summary = apply_ocr_to_pdf_pages(&mut canonical, &engine, &bytes, &opts, |_| {}).unwrap(); @@ -215,6 +230,11 @@ fn f3_input_with_always_on_pushes_dual_blocks() { cancel: None, ocr_cache: None, ocr_version_key: String::new(), + // No renderer in these unit tests: they pin the fallback path's + // behavior, which is what a machine without pdfium gets. + renderer: None, + render_dpi: 300, + max_pixels: 4096, }; let summary = apply_ocr_to_pdf_pages(&mut canonical, &engine, &bytes, &opts, |_| {}).unwrap(); @@ -322,6 +342,11 @@ fn dual_block_ordinals_are_deterministic_and_unique() { cancel: None, ocr_cache: None, ocr_version_key: String::new(), + // No renderer in these unit tests: they pin the fallback path's + // behavior, which is what a machine without pdfium gets. + renderer: None, + render_dpi: 300, + max_pixels: 4096, }; apply_ocr_to_pdf_pages(&mut canonical, &engine, &bytes, &opts, |_| {}).unwrap(); @@ -361,6 +386,11 @@ fn cancel_handle_aborts_mid_pdf() { cancel: Some(cancel.clone()), ocr_cache: None, ocr_version_key: String::new(), + // No renderer in these unit tests: they pin the fallback path's + // behavior, which is what a machine without pdfium gets. + renderer: None, + render_dpi: 300, + max_pixels: 4096, }; let result = apply_ocr_to_pdf_pages(&mut canonical, &engine, &bytes, &opts, |_| {}); @@ -370,3 +400,143 @@ fn cancel_handle_aborts_mid_pdf() { "error message 가 'cancelled mid-PDF' 포함: {err}" ); } + +// ── Renderer path (issue #232) ──────────────────────────────────────────── +// +// The tests above pin what a machine *without* pdfium does. These pin the +// other half. They are `#[ignore]`d behind `KEBAB_TEST_PDFIUM` for the +// same reason the renderer itself is optional — a lane without the +// library must not report a failure it cannot act on: +// +// KEBAB_TEST_PDFIUM=/path/to/libpdfium.so \ +// cargo test -p kebab-app --test pdf_ocr_apply -- --ignored +// +// The OCR engine is mocked, so these exercise rasterization and the +// branch that chooses it without touching a network or a model. + +/// Bound once for the binary: pdfium's initialization is not reentrant +/// and cargo runs tests in parallel. +fn test_renderer() -> Arc { + static SHARED: std::sync::OnceLock> = + std::sync::OnceLock::new(); + SHARED + .get_or_init(|| { + let explicit = std::env::var("KEBAB_TEST_PDFIUM").ok(); + let path = explicit.as_deref().map(Path::new); + Arc::new( + kebab_parse_pdf::PageRenderer::bind(path) + .expect("these tests require libpdfium; point KEBAB_TEST_PDFIUM at one"), + ) + }) + .clone() +} + +fn opts_with_renderer() -> PdfOcrOpts { + PdfOcrOpts { + renderer: Some(test_renderer()), + ..default_opts(true) + } +} + +/// The whole point of issue #232. `f7_ccittfax_skipped_with_warning` +/// above pins that this exact fixture is skipped with no renderer; with +/// one, the same bytes must reach the OCR engine instead. +#[test] +#[ignore = "requires libpdfium"] +fn a_ccitt_page_reaches_the_ocr_engine_once_a_renderer_is_configured() { + let bytes = + std::fs::read("../kebab-parse-pdf/tests/fixtures/ccitt.pdf").expect("F7 fixture missing"); + let mut canonical = canonical_with_empty_block(); + let engine = MockOcrEngine::single("RASTERIZED AND READ", false); + + let summary = apply_ocr_to_pdf_pages( + &mut canonical, + &engine, + &bytes, + &opts_with_renderer(), + |_| {}, + ) + .unwrap(); + + assert_eq!( + summary.pages_ocrd, 1, + "the CCITT page must be OCR'd, not skipped: {summary:?}" + ); + assert_eq!( + summary.pages_skipped, 0, + "and must not be counted as a page with no raster" + ); +} + +/// A renderer must not cost the DCTDecode path its coverage. Same +/// fixture as `f1_enabled_true_mutates_block_in_place`, with a renderer +/// added — the outcome has to be the same. +#[test] +#[ignore = "requires libpdfium"] +fn a_dctdecode_page_still_works_with_a_renderer_configured() { + let bytes = f1_pdf_bytes(); + let mut canonical = canonical_with_empty_block(); + let engine = MockOcrEngine::single("STILL READ", false); + + let summary = apply_ocr_to_pdf_pages( + &mut canonical, + &engine, + &bytes, + &opts_with_renderer(), + |_| {}, + ) + .unwrap(); + + assert_eq!(summary.pages_ocrd, 1); + assert_eq!(summary.pages_skipped, 0); +} + +/// A PDF that lopdf parses but pdfium refuses. The renderer is +/// configured and working — it simply cannot open *this* file — so the +/// page must fall through to the DCTDecode path and, when that also has +/// nothing, be reported as `unopenable_pdf` rather than `no_renderer`. +/// Telling a user who already configured a renderer to configure one is +/// the wrong instruction. +/// +/// The fixture is an encrypted PDF with no password: lopdf reads the +/// object graph without decrypting, pdfium refuses outright +/// (`PasswordError`). A first attempt at this test used truncated bytes, +/// which lopdf rejected before the branch was ever reached — the test +/// passed while exercising nothing. +#[test] +#[ignore = "requires libpdfium"] +fn a_pdf_the_renderer_cannot_open_falls_back_instead_of_blaming_config() { + let bytes = std::fs::read("../kebab-parse-pdf/tests/fixtures/encrypted_no_password.pdf") + .expect("encrypted fixture missing"); + let mut canonical = canonical_with_empty_block(); + let engine = MockOcrEngine::single("SHOULD_NOT_BE_CALLED", false); + + let mut reasons = Vec::new(); + let summary = apply_ocr_to_pdf_pages( + &mut canonical, + &engine, + &bytes, + &opts_with_renderer(), + |p| { + if let kebab_app::pdf_ocr_apply::PdfOcrProgress::Finished { + failure_reason: Some(r), + .. + } = p + { + reasons.push(r); + } + }, + ) + .expect("a PDF the renderer cannot open must not abort the run"); + + assert_eq!(summary.pages_ocrd, 0, "nothing could be rasterized"); + assert_eq!( + summary.pages_skipped, 1, + "and the page is counted as skipped" + ); + assert_eq!( + reasons, + vec!["unopenable_pdf".to_string()], + "the reason must name the real problem, not a missing renderer" + ); +} diff --git a/crates/kebab-app/tests/pdf_pipeline.rs b/crates/kebab-app/tests/pdf_pipeline.rs index bb84dd1..ad7ca67 100644 --- a/crates/kebab-app/tests/pdf_pipeline.rs +++ b/crates/kebab-app/tests/pdf_pipeline.rs @@ -165,7 +165,7 @@ fn ingest_3_page_pdf_produces_one_doc_and_per_page_chunks() { pdf_item.parser_version .as_ref() .map(|p| p.0.split('|').next().unwrap()), - Some("pdf-text-v1") + Some("pdf-text-v2") ); assert_eq!( pdf_item.chunker_version.as_ref().map(|c| c.0.as_str()), @@ -479,10 +479,10 @@ fn inspect_doc_surfaces_page_spans() { .find(|i| i.doc_path.0.ends_with("inspect.pdf")) .unwrap(); let doc = kebab_app::inspect_doc_with_config(cfg, pdf_item.doc_id.as_ref().unwrap()).unwrap(); - // v0.26.2: stored parser_version is now `pdf-text-v1|` + // v0.26.2: stored parser_version is now `pdf-text-v2|` // (the signature folds chunking / pdf.ocr settings for skip detection). // Assert the base identity by taking the prefix before the first '|'. - assert_eq!(doc.parser_version.0.split('|').next().unwrap(), "pdf-text-v1"); + assert_eq!(doc.parser_version.0.split('|').next().unwrap(), "pdf-text-v2"); assert_eq!(doc.blocks.len(), 3); for block in &doc.blocks { match block { diff --git a/crates/kebab-chunk/src/pdf_page_v1.rs b/crates/kebab-chunk/src/pdf_page_v1.rs index 485ae65..601c789 100644 --- a/crates/kebab-chunk/src/pdf_page_v1.rs +++ b/crates/kebab-chunk/src/pdf_page_v1.rs @@ -400,7 +400,7 @@ mod tests { fn make_pdf_doc(pages: &[&str]) -> CanonicalDocument { let workspace_path = WorkspacePath::new("docs/test.pdf".into()).unwrap(); let asset_id = AssetId("a".repeat(64)); - let parser_version = ParserVersion("pdf-text-v1".into()); + let parser_version = ParserVersion("pdf-text-v2".into()); let doc_id = id_for_doc(&workspace_path, &asset_id, &parser_version); let mut blocks: Vec = Vec::new(); diff --git a/crates/kebab-cli/src/main.rs b/crates/kebab-cli/src/main.rs index 61978db..94a622c 100644 --- a/crates/kebab-cli/src/main.rs +++ b/crates/kebab-cli/src/main.rs @@ -700,8 +700,17 @@ fn run(cli: &Cli) -> anyhow::Result<()> { } else { String::new() }; + // Issue #232: a scan whose pages produced no raster used to + // finish with a clean summary and an empty document, and + // the user found out when a search came back empty. Put + // the count where the run's result is read. + let ocr_skipped_suffix = if report.ocr_skipped_pages > 0 { + format!(" ocr-skipped {}", report.ocr_skipped_pages) + } else { + String::new() + }; println!( - "scanned {} new {} updated {} skipped {}{} errors {}{} ({} ms)", + "scanned {} new {} updated {} skipped {}{} errors {}{}{} ({} ms)", report.scanned, report.new, report.updated, @@ -709,6 +718,7 @@ fn run(cli: &Cli) -> anyhow::Result<()> { skipped_breakdown, report.errors, purged_suffix, + ocr_skipped_suffix, report.duration_ms ); } diff --git a/crates/kebab-cli/src/progress.rs b/crates/kebab-cli/src/progress.rs index 7b35723..5654ddf 100644 --- a/crates/kebab-cli/src/progress.rs +++ b/crates/kebab-cli/src/progress.rs @@ -463,15 +463,37 @@ impl ProgressDisplay { chars, ocr_engine, skipped, + failure_reason, .. } => { if !quiet { let mut err = std::io::stderr().lock(); if *skipped { - let _ = writeln!( - err, - " ⊘ OCR page {page} skipped (no DCTDecode or engine fail, {ms}ms)" - ); + // The wire event distinguishes why (issue #232) — + // a page whose encoding could not be rasterized is + // a different problem, with a different fix, from + // one the OCR engine failed on. Collapsing both + // into "no DCTDecode or engine fail" told the user + // neither. + // Two different failures wear the same ⊘ here, and + // only one of them is counted in the run summary's + // `ocr-skipped`. Say which, so the line and the + // count can be reconciled: the page never produced + // an image to read, or the engine failed to read + // one that it did get. + let why = match failure_reason.as_deref() { + Some("no_renderer") => { + "래스터 없음 — 이 페이지의 인코딩은 페이지 렌더러 없이 읽을 수 없다" + } + Some("render_error") => "래스터 없음 — 페이지 렌더링 실패", + Some("unopenable_pdf") => { + "래스터 없음 — 렌더러가 이 PDF 를 열지 못했다" + } + Some("ocr_error") => "OCR 엔진 실패", + Some(other) => other, + None => "사유 미상", + }; + let _ = writeln!(err, " ⊘ OCR page {page} 건너뜀 — {why} ({ms}ms)"); } else { let _ = writeln!( err, diff --git a/crates/kebab-cli/src/wire.rs b/crates/kebab-cli/src/wire.rs index c099fa3..5cc788e 100644 --- a/crates/kebab-cli/src/wire.rs +++ b/crates/kebab-cli/src/wire.rs @@ -277,6 +277,7 @@ mod tests { skipped_size_exceeded: 0, skip_examples: SkipExamples::default(), purged_deleted_files: 0, + ocr_skipped_pages: 0, items: None, }; let v = wire_ingest(&r); diff --git a/crates/kebab-config/src/lib.rs b/crates/kebab-config/src/lib.rs index 2b38b08..e1ccfa0 100644 --- a/crates/kebab-config/src/lib.rs +++ b/crates/kebab-config/src/lib.rs @@ -817,6 +817,35 @@ pub struct PdfOcrCfg { /// Hard cap on detected boxes per page (runaway guard). Default `1000`. #[serde(default = "default_ocr_max_boxes")] pub max_boxes: usize, + + // ── page rendering (issue #232) ───────────────────────────────────── + /// Path to `libpdfium`. `None` → search the platform's library path. + /// + /// Rendering the page is what makes OCR work on scans that are not a + /// single DCTDecode image — CCITTFax, JBIG2, Flate, JPX, and pages + /// that split content across a background plus a mask. Without a + /// renderer those pages produce no raster and are silently skipped. + /// + /// It is a path rather than a bundled library because pdfium ships + /// only as a shared object; linking it would end kebab's + /// single-binary property. `kebab doctor` reports which mode is live. + #[serde(default)] + pub render_library: Option, + /// Requested rendering resolution in DPI. Default `300` — the + /// scanning convention, and what OCR engines are tuned for. + /// + /// **`max_pixels` wins.** It is the ceiling the OCR engine will + /// accept on either side, so the effective resolution is whatever + /// fits: an A4 page at the PDF default `max_pixels = 2048` tops out + /// near 175 DPI no matter what is asked for here. Raise `max_pixels` + /// to actually reach 300 (A4 needs ~3500), at the cost of a larger + /// image for the engine to chew on. + #[serde(default = "default_pdf_render_dpi")] + pub render_dpi: u32, +} + +fn default_pdf_render_dpi() -> u32 { + 300 } impl PdfOcrCfg { @@ -839,6 +868,8 @@ impl PdfOcrCfg { score_thresh: default_ocr_score_thresh(), unclip_ratio: default_ocr_unclip_ratio(), max_boxes: default_ocr_max_boxes(), + render_library: None, + render_dpi: default_pdf_render_dpi(), } } } diff --git a/crates/kebab-config/src/migrate.rs b/crates/kebab-config/src/migrate.rs index 91b5af6..d3522d7 100644 --- a/crates/kebab-config/src/migrate.rs +++ b/crates/kebab-config/src/migrate.rs @@ -126,6 +126,10 @@ fn key_comment(path: &str) -> Option<&'static str> { "ingest.pdf.ocr.model" => "ollama-vision 전용. paddle-onnx 는 번들 모델 사용.", "ingest.pdf.ocr.valid_ratio_threshold" => "유효문자 비율 < 이면 scanned 판정.", "ingest.pdf.ocr.min_char_count" => "page 문자수 < 이면 auto-scanned.", + "ingest.pdf.ocr.render_dpi" => "스캔 page 렌더 해상도 요청(DPI). max_pixels 가 이기므로 실효 DPI 는 그쪽에 달렸다.", + "ingest.pdf.ocr.render_library" => { + "libpdfium 경로. 지정하면 CCITTFax/JBIG2/Flate/JPX 스캔도 OCR. 비우면 로더 경로 탐색." + } "ingest.pdf.ocr.request_timeout_secs" => "0=즉시실패(비활성화 아님).", "rag.score_gate" => "검색 점수 게이트.", "rag.nli_threshold" => "0=NLI 게이트 off.", diff --git a/crates/kebab-core/src/ingest.rs b/crates/kebab-core/src/ingest.rs index c3c5f3a..e91f420 100644 --- a/crates/kebab-core/src/ingest.rs +++ b/crates/kebab-core/src/ingest.rs @@ -53,6 +53,20 @@ pub struct IngestReport { /// `#[serde(default)]`. #[serde(default)] pub purged_deleted_files: u32, + /// PDF pages the text gate classified as scans but which produced no + /// raster to OCR, so their content is not indexed (issue #232). + /// + /// The condition used to reach the user as one stderr line per page + /// and nothing else: the run reported success, and the absence only + /// showed up later as a search that found nothing. Additive field — + /// older wire consumers read it as 0 via `#[serde(default)]`. + /// + /// Counts pages with no image to read, not pages the OCR engine + /// failed on — those are engine errors, already tracked separately, + /// and lumping them together would hide which of the two a run hit. + /// The progress line names the cause for each. + #[serde(default)] + pub ocr_skipped_pages: u32, /// `None` ↔ wire `items: null` (`--summary-only`). pub items: Option>, } @@ -149,6 +163,7 @@ mod tests { gitignore: vec![], }, purged_deleted_files: 0, + ocr_skipped_pages: 0, items: None, }; let v = serde_json::to_value(&r).unwrap(); diff --git a/crates/kebab-parse-pdf/Cargo.toml b/crates/kebab-parse-pdf/Cargo.toml index e122918..77305b0 100644 --- a/crates/kebab-parse-pdf/Cargo.toml +++ b/crates/kebab-parse-pdf/Cargo.toml @@ -21,12 +21,27 @@ tracing = { workspace = true } # at v1 (we don't call its whole-doc API), and the future scanned-PDF # OCR fallback can re-add it when it actually needs it. lopdf = { workspace = true } +# Page rasterization for scanned PDFs (issue #232). `dynamic_bindings` +# only: pdfium ships as a shared library and publishes no static build, +# so linking it would end kebab's single-binary property. The renderer +# binds at run time instead — present and every scanned PDF is covered, +# absent and ingest falls back to the DCTDecode passthrough below. +# Default features are off because they pull the `libloading`-free +# static path we deliberately do not use. +pdfium-render = { version = "0.9", default-features = false, features = [ + "image_025", + "pdfium_latest", + "thread_safe", +] } +image = { version = "0.25", default-features = false, features = ["png"] } [dev-dependencies] anyhow = { workspace = true } blake3 = { workspace = true } kebab-parse-image = { path = "../kebab-parse-image" } strsim = "0.11" +# `tests/page_render.rs` decodes a render back to check its dimensions. +image = { version = "0.25", default-features = false, features = ["png"] } [lints] workspace = true diff --git a/crates/kebab-parse-pdf/src/lib.rs b/crates/kebab-parse-pdf/src/lib.rs index 630f048..986897a 100644 --- a/crates/kebab-parse-pdf/src/lib.rs +++ b/crates/kebab-parse-pdf/src/lib.rs @@ -19,10 +19,12 @@ mod info; mod page_image; +mod page_render; mod page_text; mod text_quality; pub use page_image::extract_dctdecode_page_image; +pub use page_render::{PageRenderer, RenderedPdf}; pub use text_quality::compute_valid_char_ratio; use anyhow::{Context, Result}; @@ -34,7 +36,18 @@ use kebab_core::{ use serde_json::{Map, Value}; use time::OffsetDateTime; -pub const PARSER_VERSION: &str = "pdf-text-v1"; +/// Bumped to v2 for issue #232 (2026-08-17): scanned pages are now +/// rasterized by rendering rather than by pulling out an embedded JPEG, +/// so pages encoded with CCITTFax / JBIG2 / Flate / JPX — previously +/// dropped without a raster and therefore never OCR'd — now carry text. +/// +/// The bump is what makes that reach existing stores. `try_skip_unchanged` +/// treats an asset as Unchanged when its content hash *and* all four +/// version inputs match; a PDF on disk has not changed, so without this +/// every already-indexed scan would keep its empty extraction until the +/// user thought to pass `--force-reingest`. Per CLAUDE.md §Versioning +/// cascade, changing this invalidates downstream PDF records. +pub const PARSER_VERSION: &str = "pdf-text-v2"; /// Text-PDF extractor. Per-page text via `lopdf::Document::extract_text` /// (the only stable per-page API in the lopdf / pdf-extract pair — diff --git a/crates/kebab-parse-pdf/src/page_render.rs b/crates/kebab-parse-pdf/src/page_render.rs new file mode 100644 index 0000000..b95c4a7 --- /dev/null +++ b/crates/kebab-parse-pdf/src/page_render.rs @@ -0,0 +1,224 @@ +//! Rasterize PDF pages for OCR, whatever the page is made of. +//! +//! [`page_image`](crate::page_image) pulls a page's embedded JPEG out +//! verbatim, which only works when the page *is* one DCTDecode image. +//! Issue #232: real scanners emit CCITTFaxDecode, JBIG2Decode, +//! FlateDecode and JPXDecode, and Internet Archive scans split the page +//! across a background plus an `/ImageMask`. Every one of those was +//! dropped silently — the text gate correctly said "this page is a scan, +//! OCR it" and then no raster came back. +//! +//! Rendering the page sidesteps the whole question. There is no filter to +//! support, no XObject to pick between, and a page that mixes vector +//! drawing with images comes out as the reader sees it. +//! +//! # Why this is optional +//! +//! pdfium ships as a shared library and no static build is published, so +//! requiring it would end kebab's single-binary property. Instead the +//! renderer binds at runtime: present, and every scanned PDF is covered; +//! absent, and ingest falls back to the DCTDecode path exactly as before +//! and says so. `kebab doctor` reports which of the two is in effect. + +use std::path::Path; + +use anyhow::{Context, Result}; +use pdfium_render::prelude::*; + +/// A bound pdfium library. Construct once per run — binding hits the +/// filesystem and the dynamic loader. +pub struct PageRenderer { + pdfium: Pdfium, + source: String, + /// Serializes use of `pdfium`. + /// + /// `pdfium-render`'s `thread_safe` feature was not enough on its own: + /// driving one bound instance from several threads at once aborted + /// the process with a double-free. Since the ingest loop handles one + /// PDF at a time anyway, the lock costs nothing today and keeps that + /// from becoming a memory-safety bug the day the loop parallelizes. + /// Held by [`RenderedPdf`] for as long as the document is open. + lock: std::sync::Mutex<()>, +} + +impl PageRenderer { + /// Bind to `libpdfium`, either at an explicit path or wherever the + /// platform's loader finds it. + /// + /// Returns `Err` when the library is missing or unloadable; callers + /// treat that as "no renderer" rather than a failure, because the + /// DCTDecode path still works. + /// + /// # Bind once + /// + /// Call this once and share the result. Binding concurrently from + /// several threads fails — pdfium's initialization is not reentrant — + /// and binding repeatedly re-walks the loader path for nothing. The + /// ingest path binds once per run and hands out an `Arc`. + pub fn bind(library: Option<&Path>) -> Result { + let (bindings, source) = match library { + Some(path) => ( + Pdfium::bind_to_library(path) + .with_context(|| format!("bind pdfium at {}", path.display()))?, + path.display().to_string(), + ), + None => ( + Pdfium::bind_to_system_library() + .context("bind pdfium from the system library path")?, + "system".to_string(), + ), + }; + Ok(Self { + pdfium: Pdfium::new(bindings), + source, + lock: std::sync::Mutex::new(()), + }) + } + + /// Where the bound library came from — an explicit path, or `system`. + /// Reported by `kebab doctor` so a user can tell which copy is live. + pub fn source(&self) -> &str { + &self.source + } + + /// Open a PDF held in memory. The returned handle borrows `self`, so + /// one bound library serves every document in a run. + /// + /// Takes the renderer's lock and holds it until the handle drops, so + /// a second caller waits rather than racing. One PDF at a time is + /// what the ingest loop does regardless. + pub fn open<'a>(&'a self, bytes: &'a [u8], password: Option<&str>) -> Result> { + // A poisoned lock means some other thread panicked mid-render. + // pdfium state is not ours to reason about after that, but + // refusing every later page would turn one bad PDF into a dead + // capability for the rest of the run — take the guard and carry + // on, which is what `PoisonError::into_inner` is for. + let guard = self + .lock + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let doc = self + .pdfium + .load_pdf_from_byte_slice(bytes, password) + .context("pdfium: load PDF")?; + Ok(RenderedPdf { doc, _guard: guard }) + } +} + +/// A PDF opened through pdfium, ready to rasterize pages. +/// +/// Holds the renderer's lock for its lifetime. Field order is load- +/// bearing: `doc` is declared first so it drops before the guard, +/// because releasing the lock while a document handle is still alive is +/// exactly the race the lock exists to prevent. +pub struct RenderedPdf<'a> { + doc: PdfDocument<'a>, + _guard: std::sync::MutexGuard<'a, ()>, +} + +impl RenderedPdf<'_> { + pub fn page_count(&self) -> u32 { + u32::try_from(self.doc.pages().len()).unwrap_or(0) + } + + /// Rasterize a 1-based page and return PNG bytes. + /// + /// PNG rather than JPEG: OCR reads glyph edges, and JPEG ringing on + /// the high-contrast black-on-white of a scan is exactly the artifact + /// that costs recognition accuracy. The bytes go straight to an OCR + /// engine and are never stored, so the size difference is transient. + /// + /// `dpi` is the requested resolution; `max_px` is the ceiling the OCR + /// engine will accept on either side, and it wins. The page's own + /// size comes from pdfium, which already parsed it — asking the PDF + /// dictionary ourselves would mean reimplementing `/MediaBox` + /// inheritance and `/UserUnit`, and getting it wrong silently + /// produces a differently-sized render. + pub fn render_page_png(&self, page_num: u32, dpi: u32, max_px: u32) -> Result> { + let index = i32::try_from(page_num.saturating_sub(1)) + .with_context(|| format!("page {page_num} out of pdfium's index range"))?; + let page = self + .doc + .pages() + .get(index) + .with_context(|| format!("pdfium: get page {page_num}"))?; + + let long_edge_pt = page.width().value.max(page.height().value); + let cap = i32::try_from(long_edge_for_dpi(long_edge_pt, dpi, max_px)).unwrap_or(i32::MAX); + + // All three, and the combination is the point. + // + // `set_maximum_*` alone does not scale — it only clamps a render + // that would otherwise exceed it. With no target and no scale + // factor pdfium renders at 1pt-to-1px, i.e. 72 DPI, and + // `render_dpi` silently does nothing. A target alone is what + // makes a long scan allocate past what pdfium will take: it + // throws `length_error` from C++ with exceptions disabled, which + // aborts the process rather than returning an error we could + // downgrade to a skip. Target sets the scale, the maxima bound + // it, and together they also keep the aspect ratio — the + // clamp-only path sets `do_maintain_aspect_ratio = false` and + // squashes the page. + let config = PdfRenderConfig::new() + .set_target_width(cap) + .set_maximum_width(cap) + .set_maximum_height(cap); + + let image = page + .render_with_config(&config) + .with_context(|| format!("pdfium: render page {page_num}"))? + .as_image() + .with_context(|| format!("pdfium: page {page_num} bitmap to image"))? + .into_rgb8(); + + let mut png = Vec::new(); + image + .write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png) + .with_context(|| format!("encode page {page_num} as PNG"))?; + Ok(png) + } +} + +/// Long edge in pixels for a page rendered at `dpi`. +/// +/// PDF user space is 72 units to the inch, so the multiplier is +/// `dpi / 72`. Clamped to `max_px` because the OCR engine's own pixel +/// budget is the real ceiling — raising `render_dpi` past what the engine +/// accepts would only cost time. +fn long_edge_for_dpi(page_long_edge_pt: f32, dpi: u32, max_px: u32) -> u32 { + let scaled = (page_long_edge_pt * dpi as f32 / 72.0).round(); + let scaled = if scaled.is_finite() && scaled >= 1.0 { + scaled as u32 + } else { + 1 + }; + scaled.min(max_px.max(1)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn dpi_scales_from_pdf_user_space() { + // A4 is 842pt on the long edge; 300dpi is the scanning default. + assert_eq!(long_edge_for_dpi(842.0, 300, 10_000), 3508); + assert_eq!(long_edge_for_dpi(842.0, 72, 10_000), 842); + } + + #[test] + fn dpi_is_capped_by_the_engines_pixel_budget() { + // The OCR engine will not read more than it will read, so a high + // render_dpi must not turn into an oversized image it rejects. + assert_eq!(long_edge_for_dpi(842.0, 600, 2_048), 2_048); + } + + #[test] + fn degenerate_inputs_still_produce_a_renderable_size() { + // A malformed page box must not become a zero-pixel render + // request, which pdfium rejects outright. + assert_eq!(long_edge_for_dpi(0.0, 300, 4_096), 1); + assert_eq!(long_edge_for_dpi(f32::NAN, 300, 4_096), 1); + assert_eq!(long_edge_for_dpi(842.0, 300, 0), 1); + } +} diff --git a/crates/kebab-parse-pdf/tests/extractor.rs b/crates/kebab-parse-pdf/tests/extractor.rs index 755039b..1645533 100644 --- a/crates/kebab-parse-pdf/tests/extractor.rs +++ b/crates/kebab-parse-pdf/tests/extractor.rs @@ -267,7 +267,7 @@ fn snapshot_three_page_canonical_document_stable() { // golden file (the full JSON contains BLAKE3 ids that would // change if `id_from(...)`'s tuple shape ever shifts — that would // be a separate, intentional break). - assert_eq!(json["parser_version"], Value::String("pdf-text-v1".into())); + assert_eq!(json["parser_version"], Value::String("pdf-text-v2".into())); assert_eq!(json["lang"], Value::String("und".into())); assert_eq!(json["schema_version"], Value::Number(1.into())); assert_eq!(json["doc_version"], Value::Number(1.into())); diff --git a/crates/kebab-parse-pdf/tests/fixtures/encrypted_no_password.pdf b/crates/kebab-parse-pdf/tests/fixtures/encrypted_no_password.pdf new file mode 100644 index 0000000..96394b2 --- /dev/null +++ b/crates/kebab-parse-pdf/tests/fixtures/encrypted_no_password.pdf @@ -0,0 +1,25 @@ +%PDF-1.4 +1 0 obj +<> +endobj +2 0 obj +<> +endobj +3 0 obj +<> +endobj +4 0 obj +</U<0000000000000000000000000000000000000000000000000000000000000000>>> +endobj +xref +0 5 +0000000000 65535 f +0000000009 00000 n +0000000054 00000 n +0000000105 00000 n +0000000170 00000 n +trailer +<<02>]>> +startxref +366 +%%EOF diff --git a/crates/kebab-parse-pdf/tests/page_render.rs b/crates/kebab-parse-pdf/tests/page_render.rs new file mode 100644 index 0000000..8d1a4fb --- /dev/null +++ b/crates/kebab-parse-pdf/tests/page_render.rs @@ -0,0 +1,176 @@ +//! Issue #232: every scanned page must produce a raster, whatever its +//! images are encoded with. +//! +//! These tests need a `libpdfium` and are `#[ignore]`d for the same +//! reason `kebab-store-vector`'s AVX suite is: the capability is +//! genuinely optional, and a lane without it must not report a failure +//! it cannot act on. Opt in with +//! +//! ```text +//! KEBAB_TEST_PDFIUM=/path/to/libpdfium.so \ +//! cargo test -p kebab-parse-pdf --test page_render -- --ignored +//! ``` +//! +//! The fallback path — what a machine *without* pdfium does — is covered +//! by `page_image.rs` and needs no library, so the behaviour that ships +//! to a bare install is tested unconditionally. + +use kebab_parse_pdf::PageRenderer; + +/// Bind to the library named by `KEBAB_TEST_PDFIUM`, or the system one. +/// +/// Bound once for the whole binary, which is both what production does +/// (one renderer per ingest run) and what pdfium requires — binding it +/// from several threads at once fails, and cargo runs tests in parallel. +/// +/// Panics rather than skipping: an `--ignored` run that silently passes +/// without exercising the renderer is worse than no test. +fn renderer() -> &'static PageRenderer { + static SHARED: std::sync::OnceLock = std::sync::OnceLock::new(); + SHARED.get_or_init(|| { + let explicit = std::env::var("KEBAB_TEST_PDFIUM").ok(); + let path = explicit.as_deref().map(std::path::Path::new); + PageRenderer::bind(path).expect( + "these tests require libpdfium; point KEBAB_TEST_PDFIUM at one or \ + install it where the loader finds it", + ) + }) +} + +/// PNG signature, so a test can tell "we got an image" from "we got +/// bytes". +const PNG_MAGIC: &[u8] = b"\x89PNG\r\n\x1a\n"; + +/// The whole point of the change: a page whose image is **not** a single +/// DCTDecode JPEG still rasterizes. `ccitt.pdf` is the fixture the +/// DCTDecode path returns `None` for (see `page_image.rs`), which is +/// exactly the silent content loss issue #232 reported. +#[test] +#[ignore = "requires libpdfium"] +fn a_ccitt_page_rasterizes_even_though_dctdecode_extraction_cannot() { + let bytes = include_bytes!("fixtures/ccitt.pdf"); + + // Precondition: this fixture is one the old path gives up on. + let doc = lopdf::Document::load_mem(bytes).unwrap(); + assert!( + kebab_parse_pdf::extract_dctdecode_page_image(&doc, 1) + .unwrap() + .is_none(), + "fixture no longer exercises the gap — pick one the DCTDecode path drops" + ); + + let r = renderer(); + let pdf = r.open(bytes, None).expect("open ccitt.pdf"); + let png = pdf.render_page_png(1, 300, 1_000).expect("render page 1"); + assert!(png.starts_with(PNG_MAGIC), "rendered bytes are not a PNG"); + assert!( + png.len() > 1_000, + "suspiciously small render: {} B", + png.len() + ); +} + +/// FlateDecode raw pixels — the other encoding `page_image.rs` pins as +/// unreadable. +#[test] +#[ignore = "requires libpdfium"] +fn a_flate_page_rasterizes_too() { + let bytes = include_bytes!("fixtures/flate_raw.pdf"); + let r = renderer(); + let pdf = r.open(bytes, None).expect("open flate_raw.pdf"); + let png = pdf.render_page_png(1, 300, 1_000).expect("render page 1"); + assert!(png.starts_with(PNG_MAGIC)); +} + +/// The DCTDecode case must not regress: rendering has to cover what the +/// passthrough already covered, or the change trades one gap for another. +#[test] +#[ignore = "requires libpdfium"] +fn the_dctdecode_case_still_works_through_the_renderer() { + let bytes = include_bytes!("fixtures/scanned_page1.pdf"); + let r = renderer(); + let pdf = r.open(bytes, None).expect("open scanned_page1.pdf"); + let png = pdf.render_page_png(1, 300, 1_000).expect("render page 1"); + assert!(png.starts_with(PNG_MAGIC)); +} + +/// The pixel budget is a ceiling, not a suggestion — an OCR engine that +/// refuses oversized input would otherwise turn a render into a failure. +/// And the page must not be squashed to fit it: pdfium's clamp-only path +/// applies width and height independently and silently changes the +/// aspect ratio, which is not something OCR recovers from. +#[test] +#[ignore = "requires libpdfium"] +fn the_pixel_budget_is_respected_without_distorting_the_page() { + let r = renderer(); + for fixture in [ + &include_bytes!("fixtures/scanned_page1.pdf")[..], + &include_bytes!("fixtures/ccitt.pdf")[..], + ] { + let pdf = r.open(fixture, None).expect("open"); + let png = pdf.render_page_png(1, 600, 600).expect("render"); + let img = image::load_from_memory(&png).expect("decode render"); + assert!( + img.width().max(img.height()) <= 600, + "long edge {} exceeds the 600px budget", + img.width().max(img.height()) + ); + // Both fixtures are portrait, so a render that kept the shape is + // taller than it is wide. A square output means the clamp ran + // without a scale and each side was cut to the cap on its own. + assert!( + img.height() > img.width(), + "portrait page came back {}x{} — aspect ratio was not preserved", + img.width(), + img.height() + ); + } +} + +/// `render_dpi` has to actually change the render. Clamping alone leaves +/// pdfium at 1pt-to-1px (72 DPI) no matter what is asked for, and the +/// knob reads as working because a render still comes back. +#[test] +#[ignore = "requires libpdfium"] +fn render_dpi_changes_the_rendered_size() { + let bytes = include_bytes!("fixtures/scanned_page1.pdf"); + let r = renderer(); + let pdf = r.open(bytes, None).expect("open"); + + let size_at = |dpi: u32| { + let png = pdf.render_page_png(1, dpi, 10_000).expect("render"); + let img = image::load_from_memory(&png).expect("decode"); + img.width().max(img.height()) + }; + + let at_72 = size_at(72); + let at_300 = size_at(300); + // A4-ish page at 72 DPI is its point size; at 300 it is ~4.17x that. + assert!( + at_300 > at_72 * 3, + "300 DPI produced {at_300}px against 72 DPI's {at_72}px — \ + the dpi argument is not reaching the renderer" + ); +} + +/// A page number past the end is a caller error, not a panic. The OCR +/// loop walks pages from lopdf's count, and the two libraries disagreeing +/// about page count must degrade to a skip. +#[test] +#[ignore = "requires libpdfium"] +fn a_page_past_the_end_errors_rather_than_panicking() { + let bytes = include_bytes!("fixtures/scanned_page1.pdf"); + let r = renderer(); + let pdf = r.open(bytes, None).expect("open"); + assert!(pdf.render_page_png(9_999, 300, 600).is_err()); +} + +/// Bytes that are not a PDF must come back as an error from `open`, so +/// the caller falls through to the DCTDecode path instead of aborting +/// the ingest. +#[test] +#[ignore = "requires libpdfium"] +fn garbage_input_is_an_error_not_a_crash() { + let r = renderer(); + assert!(r.open(b"this is not a pdf at all", None).is_err()); +} diff --git a/crates/kebab-parse-pdf/tests/snapshots/vector_pdf_canonical.json b/crates/kebab-parse-pdf/tests/snapshots/vector_pdf_canonical.json index d33a66c..e970c5d 100644 --- a/crates/kebab-parse-pdf/tests/snapshots/vector_pdf_canonical.json +++ b/crates/kebab-parse-pdf/tests/snapshots/vector_pdf_canonical.json @@ -1,5 +1,5 @@ { - "doc_id": "c90fae7576fe514fb08190cb29d1ef5d", + "doc_id": "bd04fa013899592afcf671013404ed68", "source_asset_id": "babe9824b6b28237c0898575a40ba48d", "workspace_path": "mojibake.pdf", "title": "untitled", @@ -8,7 +8,7 @@ { "kind": "paragraph", "common": { - "block_id": "22bb97fc37da5c55c099e2763f95ffd9", + "block_id": "964162ec3cf0c191849f5a5cf8a3b675", "heading_path": [], "source_span": { "kind": "page", @@ -54,11 +54,11 @@ "at": "1970-01-01T00:00:00Z", "agent": "kb-parse-pdf", "kind": "parsed", - "note": "parser_version=pdf-text-v1; page_count=1" + "note": "parser_version=pdf-text-v2; page_count=1" } ] }, - "parser_version": "pdf-text-v1", + "parser_version": "pdf-text-v2", "schema_version": 1, "doc_version": 1, "last_chunker_version": null, diff --git a/crates/kebab-store-sqlite/snapshots/ingest_report.snapshot.json b/crates/kebab-store-sqlite/snapshots/ingest_report.snapshot.json index a2162ba..f3f56b6 100644 --- a/crates/kebab-store-sqlite/snapshots/ingest_report.snapshot.json +++ b/crates/kebab-store-sqlite/snapshots/ingest_report.snapshot.json @@ -36,6 +36,8 @@ } ], "new": 2, + "ocr_skipped_pages": 0, + "purged_deleted_files": 0, "scanned": 3, "scope": { "exclude": [ @@ -60,6 +62,5 @@ "skipped_kebabignore": 0, "skipped_size_exceeded": 0, "unchanged": 0, - "purged_deleted_files": 0, "updated": 1 } diff --git a/crates/kebab-store-sqlite/tests/ingest_report_snapshot.rs b/crates/kebab-store-sqlite/tests/ingest_report_snapshot.rs index 91741fb..05a5d1e 100644 --- a/crates/kebab-store-sqlite/tests/ingest_report_snapshot.rs +++ b/crates/kebab-store-sqlite/tests/ingest_report_snapshot.rs @@ -42,6 +42,7 @@ fn fixture_report() -> IngestReport { skipped_size_exceeded: 0, skip_examples: kebab_core::SkipExamples::default(), purged_deleted_files: 0, + ocr_skipped_pages: 0, items: Some(vec![ IngestItem { kind: IngestItemKind::New, diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 5cd7034..3c6b644 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -24,7 +24,7 @@ Cargo workspace, 함수 호출 기반 모듈러 모놀리스. UI binary (`kebab- | OCR (PDF, v0.20.0+) | Ollama vision LM (default `qwen2.5vl:3b`) — post-extract enrichment via `kebab-app::pdf_ocr_apply` (H-1 resolution). DCTDecode-only v1 (FlateDecode/CCITTFax skip + warning). family asymmetry vs image OCR: PoC alnum 94.79% (qwen2.5vl) >> 27% (gemma4:e4b 받침), 본 단계에서 PDF OCR 만 qwen2.5vl. | | Image caption | Ollama vision LM, runtime gate `image.caption.enabled` (default OFF) | | RAG groundedness 검증 | `kebab-nli` 의 mDeBERTa-v3 XNLI 가 `(packed_chunks, generated_answer)` entailment 검사 (fb-41). `[rag] nli_threshold > 0` (default 0 = disabled, production 권장 0.5) 일 때 활성 — 미달 시 `refusal_reason = nli_verification_failed` (LLM self-judge ceiling 보완). 첫 호출 시 ~280 MB ONNX 자동 다운로드 | -| PDF parser | `lopdf` per-page 텍스트 + scanned-page image extract (`page_image::extract_dctdecode_page_image`, v0.20.0). `chunker_version = "pdf-page-v1"` 하드코딩 (HOTFIXES P7-3). `parser_version = "pdf-text-v1"` 보존 (v0.20 OCR 후에도) — provenance event 로 OCR 사용 차별화. force-reingest 가 v0.19 indexed scanned PDF 의 재처리에 필요. | +| PDF parser | `lopdf` per-page 텍스트 + 스캔 페이지 래스터화. 래스터는 **pdfium 페이지 렌더링**(`page_render::PageRenderer`, issue #232) 이 1순위 — 필터·XObject 구성과 무관하게 페이지를 그린다. pdfium 이 없으면 `page_image::extract_dctdecode_page_image` 로 떨어지며 그 경우 단일 DCTDecode 이미지 페이지만 OCR 된다. pdfium 은 공유 라이브러리로만 배포돼 링크하면 단일 바이너리 원칙이 깨지므로 **런타임 바인딩**이고, `[ingest.pdf.ocr] render_library` 로 경로를 지정하거나 로더 경로에 두면 된다. `kebab doctor` 의 `pdf_render` 가 어느 쪽인지 보고한다. `chunker_version = "pdf-page-v1"` 하드코딩 (HOTFIXES P7-3). `parser_version = "pdf-text-v2"` (issue #232 에서 v1 → v2, 기존 색인 스캔본 재처리 유발). | | code parser | `tree-sitter` + `tree-sitter-rust` / `tree-sitter-python` / `tree-sitter-typescript` / `tree-sitter-javascript` / `tree-sitter-go` / `tree-sitter-java` / `tree-sitter-kotlin-ng` — **parser-side** (`kebab-parse-code`), chunker-side 아님 (design §6.3). chunker versions: Rust = `code-rust-ast-v1`, Python = `code-python-ast-v1`, TypeScript = `code-ts-ast-v1`, JavaScript = `code-js-ast-v1`, Go = `code-go-ast-v1`, Java = `code-java-ast-v1`, Kotlin = `code-kotlin-ast-v1`. (v0.32.0 #220: 9개 언어 chunker 가 단일 `CodeAstV1Chunker` 로 통합 — `for_lang(lang)` 가 per-lang `chunker_version` 라벨을 verbatim 매핑. chunker 는 tree-sitter 미사용·`lang` 은 `SourceSpan::Code` 데이터에서 흐르므로 9개 struct 차이는 `VERSION_LABEL` 문자열뿐이었음 → chunk_id byte-identical.) `ast_chunk_max_lines = 200` 상수 고정 (HOTFIXES 2026-05-19 — Chunker trait 이 per-medium config 미노출). Kotlin grammar 은 `tree-sitter-kotlin-ng` 사용 — bare `tree-sitter-kotlin` 은 tree-sitter 0.21–0.23 에 고착되어 있어 사용 불가. **Tier 2 (p10-2)**: YAML/k8s → `serde_yaml_ng` + `k8s-manifest-resource-v1` (apiVersion+kind per resource), Dockerfile → `dockerfile-file-v1` (whole-file), Cargo.toml/go.mod/.json/.xml/.groovy → `manifest-file-v1` (whole-file). Tier 2 chunkers live in `kebab-chunk`; no tree-sitter grammar needed (structure from file type, not AST). **Tier 3 (p10-3)**: shell scripts (`.sh`/`.bash`/`.zsh`) direct → `code-text-paragraph-v1` (blank-line paragraph segmentation + 80-line / 20-overlap line-window for oversize). Same chunker also serves as fallback when Tier 1/2 emit 0 chunks or Err — non-k8s YAML / invalid YAML / AST extractor failures all picked up. symbol = None; lang preserved from input doc. **Tier 1 family complete (p10-1D)**: C (`tree-sitter-c`, `code-c-ast-v1`, `.c`/`.h`) + C++ (`tree-sitter-cpp`, `code-cpp-ast-v1`, `.cpp`/`.cc`/`.cxx`/`.hpp`/`.hh`/`.hxx`). C symbol = function name only; C++ symbol = `namespace::Class::method` (recursive nesting). `.h` 가 C++ syntax 만나면 tree-sitter-c parse 실패 → Tier 3 fallback. | | symbol path 형식 | workspace path → module path: Python = dotted prefix (`kebab_eval.metrics.compute_mrr`), TypeScript/JavaScript = slash-style prefix (`src/Foo.Foo.search`), Go = `package.Func` / `package.(*Receiver).Method`, Java/Kotlin = `com.foo.Foo.bar` (패키지+클래스+메서드/필드), C = 함수명, C++ = `namespace::Class::method`. Rust 1A-2 는 file-scope nesting 만 (workspace prefix 없음, 비일관 수용 — HOTFIXES 2026-05-20). code chunk 은 `citation.kind = "code"` + `citation.lang` + `symbol` + line range, SearchHit 에 `code_lang` + `repo`(`.git` walk-up 디렉토리명) backfill. | | Desktop | Tauri 2 + `pdfjs-dist` (native PDF render backend 금지) — P9-5 | diff --git a/docs/DOGFOOD.md b/docs/DOGFOOD.md index 3fc2be0..61ea286 100644 --- a/docs/DOGFOOD.md +++ b/docs/DOGFOOD.md @@ -141,7 +141,7 @@ enabled = false # opt-in - 1 `Block::Paragraph` per page (P7-1 invariant). **verify**: -- `parser_version = "pdf-text-v1"`. +- `parser_version = "pdf-text-v2"`. - `chunker_version = "pdf-page-v1"` (또는 `"pdf-page-v1.1"` from v0.20.1). - `block_count` ≥ page count. @@ -166,16 +166,25 @@ valid_ratio_threshold = 0.5 min_char_count = 20 ``` +**Config (v0.33.0~)**: 위 블록은 `[ingest.pdf.ocr]` 로 옮겨졌고, 스캔본을 제대로 읽으려면 `render_library` 가 필요하다. +```toml +render_library = "/path/to/libpdfium.so" +render_dpi = 300 +``` + **verify**: - IngestEvent::PdfOcrStarted / PdfOcrFinished emit. - `IngestItem.pdf_ocr_pages > 0` for scanned PDF. - `IngestItem.pdf_ocr_ms_total > 0`. - CLI printer: `📷 OCR page N...` / `✓ OCR page N (chars chars, msms via ollama-vision)`. +- **렌더러 유무로 갈리는 지점** (issue #232): `render_library` 없이 CCITTFax·JBIG2·Flate·JPX 스캔을 넣으면 `⊘ OCR page N 건너뜀 — 이 페이지의 인코딩은 페이지 렌더러 없이 읽을 수 없다` 가 찍히고, 요약에 `ocr-skipped N`, `--json` 의 `ingest_report.ocr_skipped_pages` 와 `pdf_ocr_finished.failure_reason = "no_renderer"` 로 나온다. 렌더러를 주면 같은 파일이 실제 텍스트를 낸다. +- `kebab doctor` 의 `pdf_render` 가 바인딩된 라이브러리 경로를 보고하는가. **scenarios** (이전 dogfood report 의 9 시나리오): - 1.4.a scanned 한국어 (F1 / F2) → OCR text indexed + search hit. - 1.4.b multi-scanned PDF (5+ files) → chunk_id collision 0 (Bug #3 fix verify). - 1.4.c always_on=true → vector PDF page 도 dual-block OCR. +- 1.4.d **필터별 커버리지** (issue #232): `corpus/pdf/` 의 `scanned-ccitt` / `korean-scan`(JBIG2·DCT) / `scanned-flate` / `scanned-jpx` / `scanned-mixed` 를 렌더러 있음/없음 두 조건으로 색인해 `ocr-skipped` 와 색인된 글자 수를 비교. - 1.4.d valid_ratio_threshold variation (0.3 / 0.5 / 0.8) → mojibake / 정상 page 분류. - 1.4.e min_char_count variation (5 / 20 / 100) → 짧은 page OCR 호출. - 1.4.f DCTDecode-only skip (F6 FlateDecode / F7 CCITTFax) → warning + skip. diff --git a/docs/SMOKE.md b/docs/SMOKE.md index 416916b..778402a 100644 --- a/docs/SMOKE.md +++ b/docs/SMOKE.md @@ -348,8 +348,12 @@ request_timeout_secs = 600 valid_ratio_threshold = 0.5 # PDF 고유 키 (image 에 없음) min_char_count = 20 lang_hint = "kor" +# render_library = "/usr/lib/libpdfium.so" # 있으면 모든 인코딩의 스캔 페이지 OCR +render_dpi = 300 # 렌더 해상도 요청. 실효값은 max_pixels 가 결정 ``` +> `render_library` 를 비워 두면 로더 경로에서 찾고, 못 찾으면 **단일 DCTDecode(JPEG) 페이지만** OCR 된다. CCITTFax·JBIG2·Flate·JPX 스캔은 본문 없이 색인되고 그 건수가 `ocr-skipped` 로 찍힌다. `kebab doctor` 의 `pdf_render` 로 확인. + > env override: 엔진 설정은 `KEBAB_OCR_*` (예: `KEBAB_OCR_ENDPOINT`, `KEBAB_OCR_MODEL`) 하나로 image·pdf 양쪽에 적용. on/off 는 `KEBAB_IMAGE_OCR_ENABLED` / `KEBAB_PDF_OCR_ENABLED` 로 미디어별. (config schema v5 — 옛 `KEBAB_IMAGE_OCR_*` / `KEBAB_PDF_OCR_*` 엔진 키는 `KEBAB_OCR_*` 로 통합.) 이미지 자산 한 장당 OCR 1 호출 + Caption 1 호출 → ~3-6초 (`gemma4:e4b` 기준). 다이어그램 / 카메라 사진 / 스크린샷 위주 워크스페이스에 권장. 책 / 스캔본은 P7 PDF 라인으로. @@ -709,7 +713,7 @@ KB --json schema | jq '.stats.code_lang_breakdown' - 코퍼스에 없는 주제로 `kebab ask` → `refusal_reason: "llm_self_judge"` (또는 `no_chunks` / `score_gate`) + `grounded: false`. - (P6-4) `image.ocr.enabled = true` 로 PNG 자산을 ingest 하면 `kebab list docs` 가 markdown 옆에 image doc 도 출력 (`workspace_path` 가 `*.png`). `kebab inspect doc ` 의 `block.ocr.joined` 가 vision LM 의 OCR 결과 (예: 스크린샷 안의 텍스트). `kebab search --mode lexical ""` 가 그 image chunk 를 반환하면 wiring 정상. - OCR / caption 부분 실패는 `errors` 카운터 미증가 — `kebab inspect doc ` 의 Provenance Warning 이벤트 또는 `--debug` 로그에서만 확인. -- (P7-3) `*.pdf` 자산을 워크스페이스에 두면 `kebab ingest` 출력에 PDF 도 `new` 카운터에 포함. `kebab inspect doc ` 가 `parser_version = "pdf-text-v1"` + 페이지마다 `Block::Paragraph` + `SourceSpan::Page { page, char_start, char_end }`. 본문에 등장하는 단어로 `kebab search --mode hybrid` 시 PDF chunk 가 결과에 포함되고 `source_span.kind = "page"` 면 wiring 정상. 암호화 PDF 는 `errors+=1` 로 분류되며 `error` 필드에 `qpdf --decrypt` 안내 보존. 빈/스캔 페이지 (PDF 가 텍스트를 추출하지 못한 페이지) 는 0 chunk + `Provenance::Warning` ("scanned candidate") 로 표시 — P+ scanned-PDF OCR fallback 까지는 검색 불가. +- (P7-3) `*.pdf` 자산을 워크스페이스에 두면 `kebab ingest` 출력에 PDF 도 `new` 카운터에 포함. `kebab inspect doc ` 가 `parser_version = "pdf-text-v2"` + 페이지마다 `Block::Paragraph` + `SourceSpan::Page { page, char_start, char_end }`. 본문에 등장하는 단어로 `kebab search --mode hybrid` 시 PDF chunk 가 결과에 포함되고 `source_span.kind = "page"` 면 wiring 정상. 암호화 PDF 는 `errors+=1` 로 분류되며 `error` 필드에 `qpdf --decrypt` 안내 보존. 빈/스캔 페이지 (PDF 가 텍스트를 추출하지 못한 페이지) 는 0 chunk + `Provenance::Warning` ("scanned candidate") 로 표시 — P+ scanned-PDF OCR fallback 까지는 검색 불가. ## config migrate (마이그레이션) diff --git a/docs/components/parse/README.md b/docs/components/parse/README.md index a39ba91..67d47f0 100644 --- a/docs/components/parse/README.md +++ b/docs/components/parse/README.md @@ -26,7 +26,7 @@ classDiagram parse_blocks(body) (Vec~ParsedBlock~, Warnings) } class PdfTextExtractor { - PARSER_VERSION = "pdf-text-v1" + PARSER_VERSION = "pdf-text-v2" new() Self } class ImageExtractor { @@ -101,7 +101,7 @@ flowchart LR **PDF** (`kebab-parse-pdf`): - `PdfTextExtractor` — `Extractor` 구현체. `lopdf::Document::load_mem` 로 한 번 파싱, encrypted 면 즉시 bail. -- `PARSER_VERSION = "pdf-text-v1"` — version cascade entry. (HOTFIXES P7-2 의 chunker_version `pdf-page-v1` 와 별개.) +- `PARSER_VERSION = "pdf-text-v2"` — version cascade entry (issue #232 에서 v1 → v2, 페이지 렌더링 도입으로 기존 색인 스캔본 재처리 유발). (HOTFIXES P7-2 의 chunker_version `pdf-page-v1` 와 별개.) - 빈 페이지 / extract 실패 → `Block::Paragraph` 빈 inlines + `ProvenanceKind::Warning("scanned candidate")`. OCR fallback 미구현. **Image** (`kebab-parse-image`): diff --git a/docs/wire-schema/v1/ingest_progress.schema.json b/docs/wire-schema/v1/ingest_progress.schema.json index a438bcd..d07f9b7 100644 --- a/docs/wire-schema/v1/ingest_progress.schema.json +++ b/docs/wire-schema/v1/ingest_progress.schema.json @@ -172,7 +172,7 @@ }, "failure_reason": { "type": "string", - "description": "pdf_ocr_finished (optional, v0.20.x): OCR failure reason. Present iff skipped=true due to engine error. Values: timeout | ocr_error | network_error | other." + "description": "pdf_ocr_finished: why the page was skipped. \"no_renderer\" — no page renderer configured and the page is not a single DCTDecode image, so no raster could be produced (issue #232). \"render_error\" — a renderer was configured and rasterizing this page failed. \"unopenable_pdf\" — a renderer was configured but could not open the PDF at all. \"ocr_error\" — a raster was produced but the OCR engine failed on it. Absent when the page succeeded. The first three are the ones counted in ingest_report.ocr_skipped_pages." }, "counts": { "type": "object", diff --git a/docs/wire-schema/v1/ingest_report.schema.json b/docs/wire-schema/v1/ingest_report.schema.json index f2ee803..9bfcfb6 100644 --- a/docs/wire-schema/v1/ingest_report.schema.json +++ b/docs/wire-schema/v1/ingest_report.schema.json @@ -17,19 +17,41 @@ "skipped_by_extension" ], "properties": { - "schema_version": { "const": "ingest_report.v1" }, - "scope": { "type": "object" }, - "scanned": { "type": "integer", "minimum": 0 }, - "new": { "type": "integer", "minimum": 0 }, - "updated": { "type": "integer", "minimum": 0 }, - "skipped": { "type": "integer", "minimum": 0 }, + "schema_version": { + "const": "ingest_report.v1" + }, + "scope": { + "type": "object" + }, + "scanned": { + "type": "integer", + "minimum": 0 + }, + "new": { + "type": "integer", + "minimum": 0 + }, + "updated": { + "type": "integer", + "minimum": 0 + }, + "skipped": { + "type": "integer", + "minimum": 0 + }, "unchanged": { "type": "integer", "minimum": 0, "description": "p9-fb-23: assets whose checksum + parser_version + chunker_version + embedding_version all matched the existing record. Parse / chunk / embed / vector upsert all skipped." }, - "errors": { "type": "integer", "minimum": 0 }, - "duration_ms": { "type": "integer", "minimum": 0 }, + "errors": { + "type": "integer", + "minimum": 0 + }, + "duration_ms": { + "type": "integer", + "minimum": 0 + }, "skipped_by_extension": { "type": "object", "additionalProperties": { @@ -39,40 +61,163 @@ "description": "p9-fb-25: per-extension skip count. Key = lowercase extension without leading dot (e.g. 'docx'). Files without extension key under ''." }, "items": { - "type": ["array", "null"], + "type": [ + "array", + "null" + ], "items": { "type": "object", - "required": ["kind", "doc_path"], + "required": [ + "kind", + "doc_path" + ], "properties": { - "kind": { "type": "string", "enum": ["new", "updated", "skipped", "unchanged", "error"] }, - "doc_id": { "type": ["string", "null"] }, - "doc_path": { "type": "string" }, - "asset_id": { "type": ["string", "null"] }, - "byte_len": { "type": ["integer", "null"], "minimum": 0 }, - "block_count": { "type": ["integer", "null"], "minimum": 0 }, - "chunk_count": { "type": ["integer", "null"], "minimum": 0 }, - "parser_version": { "type": ["string", "null"] }, - "chunker_version": { "type": ["string", "null"] }, - "warnings": { "type": "array", "items": { "type": "string" } }, - "pdf_ocr_pages": { "type": ["integer", "null"], "minimum": 0, "description": "v0.20.0 sub-item 1: number of PDF pages 가 OCR pipeline 통과. null = OCR disabled or non-PDF asset." }, - "pdf_ocr_ms_total": { "type": ["integer", "null"], "minimum": 0, "description": "v0.20.0 sub-item 1: cumulative OCR engine wall-clock duration (ms). null = OCR disabled or non-PDF asset." }, - "error": { "type": ["string", "null"] } + "kind": { + "type": "string", + "enum": [ + "new", + "updated", + "skipped", + "unchanged", + "error" + ] + }, + "doc_id": { + "type": [ + "string", + "null" + ] + }, + "doc_path": { + "type": "string" + }, + "asset_id": { + "type": [ + "string", + "null" + ] + }, + "byte_len": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "block_count": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "chunk_count": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "parser_version": { + "type": [ + "string", + "null" + ] + }, + "chunker_version": { + "type": [ + "string", + "null" + ] + }, + "warnings": { + "type": "array", + "items": { + "type": "string" + } + }, + "pdf_ocr_pages": { + "type": [ + "integer", + "null" + ], + "minimum": 0, + "description": "v0.20.0 sub-item 1: number of PDF pages 가 OCR pipeline 통과. null = OCR disabled or non-PDF asset." + }, + "pdf_ocr_ms_total": { + "type": [ + "integer", + "null" + ], + "minimum": 0, + "description": "v0.20.0 sub-item 1: cumulative OCR engine wall-clock duration (ms). null = OCR disabled or non-PDF asset." + }, + "error": { + "type": [ + "string", + "null" + ] + } } } }, - "skipped_gitignore": { "type": "integer", "minimum": 0 }, - "skipped_kebabignore": { "type": "integer", "minimum": 0 }, - "skipped_builtin_blacklist": { "type": "integer", "minimum": 0 }, - "skipped_generated": { "type": "integer", "minimum": 0 }, - "skipped_size_exceeded": { "type": "integer", "minimum": 0 }, + "skipped_gitignore": { + "type": "integer", + "minimum": 0 + }, + "skipped_kebabignore": { + "type": "integer", + "minimum": 0 + }, + "skipped_builtin_blacklist": { + "type": "integer", + "minimum": 0 + }, + "skipped_generated": { + "type": "integer", + "minimum": 0 + }, + "skipped_size_exceeded": { + "type": "integer", + "minimum": 0 + }, "skip_examples": { "type": "object", "properties": { - "generated": { "type": "array", "items": { "type": "string" }, "maxItems": 5 }, - "size_exceeded": { "type": "array", "items": { "type": "string" }, "maxItems": 5 }, - "builtin_blacklist": { "type": "array", "items": { "type": "string" }, "maxItems": 5 }, - "gitignore": { "type": "array", "items": { "type": "string" }, "maxItems": 5 } + "generated": { + "type": "array", + "items": { + "type": "string" + }, + "maxItems": 5 + }, + "size_exceeded": { + "type": "array", + "items": { + "type": "string" + }, + "maxItems": 5 + }, + "builtin_blacklist": { + "type": "array", + "items": { + "type": "string" + }, + "maxItems": 5 + }, + "gitignore": { + "type": "array", + "items": { + "type": "string" + }, + "maxItems": 5 + } } + }, + "ocr_skipped_pages": { + "type": "integer", + "minimum": 0, + "description": "PDF pages the text gate classified as scans but which produced no raster to OCR, so their content is not indexed (issue #232). Counts pages with no image to read (pdf_ocr_finished.failure_reason of no_renderer / render_error / unopenable_pdf), not pages the OCR engine failed on. Additive; absent in pre-v0.33 output and read as 0." } } } diff --git a/tasks/HOTFIXES.md b/tasks/HOTFIXES.md index bc3368e..236accf 100644 --- a/tasks/HOTFIXES.md +++ b/tasks/HOTFIXES.md @@ -14,6 +14,134 @@ historical contract that was implemented; this file accumulates the deltas so phase 5+ readers can find the live behavior without diffing git history. +## 2026-08-17 — #232 PDF OCR 이 DCTDecode 아닌 스캔본을 전량 건너뜀 (페이지 렌더링) + +### 무엇이 문제였나 + +`extract_dctdecode_page_image` 는 페이지의 image XObject 중 `/Filter` 가 **정확히 DCTDecode 인 것 하나**만 받는다. 실제 스캔본에서 흔한 CCITTFaxDecode·JBIG2Decode·FlateDecode·JPXDecode, `[FlateDecode, DCTDecode]` 같은 필터 체인, 그리고 Internet Archive 계열의 "배경 + `/ImageMask`" 분리 구조가 전부 걸러진다. + +주목할 점은 **텍스트 게이트가 정상 동작했다**는 것이다. `needs_ocr` 판정을 통과했다는 건 kebab 이 "이 페이지는 스캔본이라 OCR 이 필요하다"고 올바르게 본 것이다. 판정은 맞았고 래스터를 못 꺼냈을 뿐인데, 그 결과가 **조용한 내용 손실**이었다 — 색인은 "성공"으로 끝나고, 검색이 안 되는 시점에야 알게 되며, 그때 원인이 PDF 인코더라는 걸 역추적할 방법이 없다. + +### 무엇을 고쳤나 + +페이지를 **렌더링**한다 (`page_render::PageRenderer`, pdfium). 필터를 지원할 것도, XObject 중에 고를 것도 없고, 벡터 드로잉과 이미지가 섞인 페이지도 리더가 보는 대로 나온다. 부수적으로 이슈가 지적한 "페이지 내 image XObject 선택이 비결정적" 문제도 렌더링 경로에서는 성립하지 않는다. + +이슈는 **교체**를 권했지만 **렌더러 우선 + DCTDecode 폴백**으로 갔다. 배포 형태 때문이다 — pdfium 은 공유 라이브러리로만 배포되고 정적 빌드가 없어서, 링크하면 CLAUDE.md 가 규정한 "단일 바이너리" 가 깨진다. 사용자와 상의해 정한 결론이다: + +- 런타임 바인딩. 있으면 모든 인코딩 커버, 없으면 오늘 동작 그대로 + **왜 건너뛰었는지 명시**. +- `[ingest.pdf.ocr] render_library` 로 경로 지정, 비우면 로더 경로 탐색. +- `kebab doctor` 의 `pdf_render` 가 지금 어느 쪽인지 보고. +- 바이너리는 392.9 MB → 399.3 MB (+6.4 MB, 바인딩 글루). `ldd` 에 pdfium 없음 — 단일 실행 파일 유지. + +### 조용한 손실을 시끄럽게 + +이슈 부수 제안 2·3 이다. + +`failure_reason` 이 CLI 에서 버려지고 있었다(`..` 로 폐기). wire 이벤트는 원인을 구분해 싣는데 사람이 보는 출력이 "no DCTDecode or engine fail" 로 뭉갰다. 이제 `no_renderer` / `render_error` / `ocr_error` 를 구분해 찍는다. + +`IngestReport.ocr_skipped_pages` 를 추가하고(additive) 사람용 요약에도 `ocr-skipped N` 으로 낸다. stderr 한 줄로 흘려보내면 대량 ingest 에서 지나간다는 게 이슈의 지적이었다. + +### parser_version cascade + +`pdf-text-v1` → **`pdf-text-v2`**. 안 올리면 이미 색인된 스캔본에 적용되지 않는다 — PDF 파일 자체는 안 바뀌었으니 해시가 같고 `try_skip_unchanged` 가 Unchanged 로 건너뛴다. 사용자가 `--force-reingest` 를 떠올려야만 고쳐지는 수정은 고쳐진 게 아니다. + +스냅샷 두 개가 따라 움직였고, 바뀐 것이 파생 식별자뿐임을 확인했다: `vector_pdf_canonical.json` 의 `doc_id`/`block_id`/`parser_version`/provenance note 만 바뀌고 **본문 텍스트·inlines·source_span·metadata 는 동일**, `ingest_report.snapshot.json` 은 `ocr_skipped_pages` 키 하나만 추가. + +### 리뷰가 잡은 것 — render_dpi 가 아무 일도 안 하고 있었다 + +초안은 `set_maximum_width` / `set_maximum_height` 만 걸었다. 그런데 pdfium-render 에서 `maximum_*` 은 **초과할 때만 줄이는 클램프**이고 스케일이 아니다. 타깃도 배율도 없으면 `width_scale = height_scale = 1.0` 로 떨어져 **1 pt → 1 px, 즉 72 DPI** 로 렌더된다. `render_dpi` 를 300 으로 주든 1200 으로 주든 산출물이 같았다. + +렌더가 실패하지 않으니 도그푸딩도 통과해 버렸다. 신규 config 키가 선언한 해상도와 실제가 다르고 그만큼 인식률을 손해 보는 상태였다. + +실측으로 확인했다 (govdocs1-000157-ccitt.pdf 5쪽): + +| 설정 | 결과 | +|---|---| +| `maximum_*` 만 (초안) | 621×801 px — **72 DPI** | +| `target + maximum_*` (수정) | 1588×2048 px — **184 DPI** | + +같은 뿌리의 문제가 하나 더 있었다. 클램프만 걸리는 경로는 `do_maintain_aspect_ratio = false` 를 함께 세팅해서 가로·세로가 **독립적으로** 잘린다. `ccitt.pdf`(600×800pt)를 600px 예산으로 렌더하면 600×600 으로 세로가 25% 눌린 채 나왔고, 긴 변만 보던 테스트는 초록불이었다. 테스트에 종횡비 단언을 넣었다. + +`render_dpi_changes_the_rendered_size` 와 `the_pixel_budget_is_respected_without_distorting_the_page` 로 고정했다. `set_target_width` 한 줄을 되돌리면 둘 다 실패하는 것을 확인했다. + +덧붙여 `render_dpi` 는 **요청**이고 `max_pixels` 가 이긴다. PDF 기본값 `max_pixels = 2048` 이면 A4 는 175 DPI 언저리에서 잘린다. 기본값 300 이 그대로 나오지 않는다는 뜻이라 config·README·SMOKE 문구를 실제와 맞췄다. + +### 리뷰가 잡은 것 — 렌더러가 있으면 오히려 손해 보는 경우 + +페이지 하나만 렌더에 실패하면 곧장 skip 이었고, DCTDecode 경로를 시도하지 않았다. "렌더러 우선 + 폴백" 이 렌더러 **유무** 수준에서만 성립했던 것이다. pdfium 을 설치한 쪽이 그 페이지에서는 손해를 보는 셈이라, 페이지 단위 폴백을 넣었다. + +pdfium 이 PDF 자체를 못 열면 모든 페이지가 `no_renderer` 로 보고되면서 "`render_library` 를 지정하라" 고 안내했다. 이미 제대로 설정한 사용자에게는 오답이라 `unopenable_pdf` 로 갈랐다. + +`ocr-skipped` 카운트는 래스터 실패만 세는데, OCR 엔진 실패도 화면에는 똑같이 `⊘` 로 찍혀서 줄 수와 카운트가 안 맞았다. 사유를 라벨에 적어 둘을 구분한다. + +### `/MediaBox` 를 직접 파싱하지 않기로 + +초안은 lopdf 로 `/MediaBox` 를 읽어 페이지 크기를 구했다. 리뷰가 지적했듯 `/MediaBox` 는 **상속 속성**이고 대부분의 생산자가 `/Pages` 노드에 한 번만 쓴다 — lopdf 0.32 에는 상속 해석 헬퍼가 없어서 그런 PDF 는 전부 조용히 A4 폴백을 탄다. `/UserUnit` 도 미반영이었다. + +pdfium 이 이미 페이지 크기를 알고 있으므로 거기서 받는다. 40여 줄이 사라졌고 상속·UserUnit 문제가 함께 없어졌으며, kebab-app 이 lopdf 딕셔너리를 뒤지던 레이어링도 정리됐다. + +### 구현 중 발견한 것 — pdfium 은 동시 사용이 안전하지 않다 + +테스트를 병렬로 돌리자 `double free or corruption` 으로 프로세스가 죽었다. `pdfium-render` 의 `thread_safe` 기능만으로는 부족하다. 단일 스레드에서는 6개 테스트가 전부 통과한다. + +ingest 는 PDF 를 한 번에 하나씩 처리하므로 오늘은 문제가 없지만, `Arc` 는 "공유해도 된다"고 광고하는 타입이다. `PageRenderer` 안에 뮤텍스를 두고 `RenderedPdf` 가 문서 수명 동안 잡고 있게 했다 — 필드 선언 순서가 load-bearing 이다(`doc` 이 guard 보다 먼저 드롭돼야 한다). 지금 비용은 0 이고, ingest 루프가 병렬화되는 날 메모리 손상 대신 대기가 된다. + +`set_target_width` 만 주면 긴 스캔에서 pdfium 이 C++ `length_error` 로 프로세스를 죽인다(exceptions 비활성 빌드라 Err 로 못 받는다). 양변을 `set_maximum_*` 으로 묶었다. + +바인딩도 한 번만 해야 한다 — 여러 스레드에서 동시에 바인딩하면 실패한다. ingest 는 run 당 1회 바인딩해 `Arc` 로 공유한다. + +### 실측 + +`govdocs1-000157-ccitt.pdf` (22쪽, 그중 1쪽이 CCITT 스캔), gemma3:4b vision: + +| | 렌더러 없음 | 렌더러 있음 | +|---|---|---| +| OCR 결과 | `⊘ 건너뜀 — 인코딩을 읽을 수 없다` | `✓ 101 chars, 6489ms` | +| 색인 chunk | 35 | **36** | +| 색인 글자 수 | 35,994 | **36,095** | +| 요약 | `ocr-skipped 1` | (없음) | + +렌더링 자체는 스파이크에서 여섯 필터 계열 전부 확인했다 — CCITT / JBIG2 / Flate / JPX / 혼합(DCT+CCITT+JBIG2+Flate) / DCT, 페이지당 40~145 ms. OCR 호출(초 단위)에 묻히는 비용이다. (그 측정은 72 DPI 버그가 있던 상태라 실제 해상도가 요청보다 낮았다 — 수정 후에는 더 걸리지만 여전히 OCR 호출에 묻힌다.) + +### 정답 있는 한국어 스캔으로 잰 인식률 + +도그푸딩 store 의 합성 픽스처(나무위키 문서를 조판→PDF→이미지로 구워 텍스트 레이어를 없앤 것, 정답 텍스트 동봉) 중 **CCITT 인코딩 3건**. 렌더러 없이는 전 페이지가 건너뛰어져 색인 내용이 0 이던 파일들이다. 엔진은 config 기본값인 `qwen2.5vl:3b`. + +| 문서 | 페이지 | CER | +|---|---|---| +| namu-beulenda-me-ijeu-leoneo-silijeu | 8 | **15.65%** | +| namu-bihaengdae-seutoli | 8 | **12.55%** | +| namu-gu-anoli-en | 6 | **15.08%** | + +22 페이지 전부 OCR 성공, 건너뜀 0. **이 PR 이전에는** 세 문서 모두 본문 0 자였으므로(CCITT 는 DCTDecode 경로가 못 읽는다) 비교 대상 CER 은 100% 다. + +첫 문단 대조 (읽히는 수준인지 확인용): + +``` +OCR 브렌다(메이즈 러너 시리즈) / 개요 / 원작 소설과 영화 메이즈 러너의 + 등장인물이자 원작에서는 진 히로인. 갈색 긴 머리의 미소녀로, 영화에서는 손이… +정답 브렌다(메이즈 러너 시리즈) / 개요 / 원작 소설과 영화 메이즈 러너의 + 등장인물이자 원작에서는 진 히로인. 갈색 긴 머리의 미소녀로, 영화에서는 숏컷이… +``` + +**엔진 선택이 결과를 가른다.** 처음에는 이 머신에 있던 `gemma3:4b` 로 쟀는데, 래스터는 정상이었지만 출력이 원문과 무관한 환각이었다("이 문서에 스무어라가 포함되어 있습니다"). 게다가 해상도가 올라가자 밀집 한국어 페이지에서 180 초 타임아웃이 났다. 범용 멀티모달 모델은 OCR 엔진이 아니다 — config 기본값이 `qwen2.5vl:3b` 인 이유가 이것이고, 릴리스 노트에 적어 둘 만하다. + +이때 새 라벨이 제 역할을 했다. 타임아웃 페이지가 "래스터 없음" 이 아니라 **"OCR 엔진 실패"** 로 찍혀서, 렌더링 문제가 아니라 엔진 문제라는 게 로그만 보고 갈렸다. + +### 리뷰가 잡은 것 — 내 테스트가 아무것도 검증하지 않고 있었다 + +2회차에서 새 세 분기에 테스트를 넣었는데, 그중 "렌더러가 PDF 를 못 여는 경우" 테스트는 **잘린 바이트를 썼고 lopdf 가 그걸 먼저 거부했다**. 함수 초반 `load_mem(...)?` 에서 리턴되므로 pdfium 은 호출조차 안 됐고, 검증 대상 분기는 한 번도 실행되지 않았다. 통과하는데 아무것도 지키지 않는 테스트였다. + +**암호 없는 암호화 PDF** 로 픽스처를 만들어 고쳤다 — lopdf 는 객체 그래프를 읽고 pdfium 은 `PasswordError` 로 거부한다. 정확히 원하던 불일치다. `unopenable_pdf` 를 `no_renderer` 로 되돌리면 실패하는 것을 확인했다. + +같은 회차에서 `(None, true)` 분기에도 `?` 가 남아 있던 것을 고쳤다. `(Some, _)` 와 근거가 같고 오히려 더 강하다 — pdfium 이 문서 전체를 열지 못한 상태라 lopdf 도 깨져 있을 상관관계가 최대인 지점이다. + +`(Some, _)` 쪽 `?` 수정은 **테스트 없이 남긴다**. 루프가 `get_pages()` 가 나열한 페이지만 도는데 그 조회가 곧 `extract_dctdecode_page_image` 가 실패하는 조건이라, 이 arm 의 에러 경로에 닿는 픽스처를 만들 수 없다. 0-페이지 PDF 로 시도했다가 루프 자체가 안 도는 것을 확인하고 접었다 — 통과하지만 아무것도 안 지키는 테스트를 또 만드는 것보다 없는 편이 정직하다. 근거를 코드 주석에 남겼다. + +### 범위 밖 + +이슈가 권한 "DCTDecode 고속 경로를 남기지 말 것" 은 따르지 않았다. 폴백이 곧 그 경로이고, 폴백을 두는 것이 배포 결정의 귀결이다. 다만 렌더러가 있으면 그 경로는 타지 않는다. + ## 2026-08-16 — #231 derivation_cache: 이슈 가설이 재현되지 않음 + 계측 노출 ### 이슈가 요청한 실측을 채웠다