feat(kebab-parse-pdf): P7-1 text PDF extractor — per-page CanonicalDocument
`PdfTextExtractor`(MediaType::Pdf) lopdf 기반 per-page 텍스트 추출.
페이지마다 `Block::Paragraph` + `SourceSpan::Page { page, char_start, char_end }`
emit. 본문이 비거나 추출 panic 인 페이지는 빈 paragraph + `Provenance::Warning`
("scanned candidate") 로 표시 — 이후 OCR fallback (별도 task) 의 입력.
핵심 동작:
- `lopdf::Document::load_mem` + `is_encrypted()` → 암호화 PDF 는 명시 에러
(`qpdf --decrypt` 안내).
- 페이지 단위 `extract_text(&[page])` 를 `catch_unwind` 로 감싸 malformed
page panic 을 recoverable warning 으로 변환.
- `/Info` dict 에서 Title/Producer/Creator best-effort 추출. UTF-16BE BOM
prefixed 문자열도 디코드 (한국어 등 non-ASCII Title 정상 처리).
- 9개 통합 테스트: 3-page emit, scanned-mixed warning, encrypted refuse,
corrupt header error, page_count 메타, UTF-16BE Title, filename
fallback, determinism, snapshot.
`parser_version = "pdf-text-v1"`. Allowed deps: `lopdf 0.32` + `pdf-extract 0.7`
(원본 spec 그대로). 본문 다국어 OCR fallback 은 §9.2 후속 task (out of scope).
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
248
crates/kebab-parse-pdf/tests/extractor.rs
Normal file
248
crates/kebab-parse-pdf/tests/extractor.rs
Normal file
@@ -0,0 +1,248 @@
|
||||
//! Integration tests for `kebab_parse_pdf::PdfTextExtractor` (P7-1).
|
||||
|
||||
mod common;
|
||||
|
||||
use kebab_core::{Block, Extractor, ProvenanceKind, SourceSpan};
|
||||
use kebab_parse_pdf::PdfTextExtractor;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::common::{
|
||||
InfoDict, build_text_pdf, build_text_pdf_with_info, corrupt_pdf, fixture_for,
|
||||
make_encrypted_pdf, strip_dynamic_at, utf16be_bom,
|
||||
};
|
||||
|
||||
fn paragraph_blocks(doc: &kebab_core::CanonicalDocument) -> Vec<&kebab_core::TextBlock> {
|
||||
doc.blocks
|
||||
.iter()
|
||||
.map(|b| match b {
|
||||
Block::Paragraph(t) => t,
|
||||
other => panic!("expected Paragraph, got {other:?}"),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn three_page_pdf_emits_one_paragraph_block_per_page() {
|
||||
let bytes = build_text_pdf(&[
|
||||
Some("Hello page 1"),
|
||||
Some("Hello page 2"),
|
||||
Some("Hello page 3"),
|
||||
]);
|
||||
let fx = fixture_for("docs/three.pdf", &bytes);
|
||||
let doc = PdfTextExtractor::new()
|
||||
.extract(&fx.ctx(), &bytes)
|
||||
.expect("3-page extraction must succeed");
|
||||
|
||||
assert_eq!(doc.title, "three");
|
||||
assert_eq!(doc.lang.0, "und");
|
||||
assert_eq!(doc.parser_version.0, kebab_parse_pdf::PARSER_VERSION);
|
||||
assert_eq!(doc.metadata.user["pdf"]["page_count"], Value::Number(3.into()));
|
||||
|
||||
let blocks = paragraph_blocks(&doc);
|
||||
assert_eq!(blocks.len(), 3);
|
||||
for (i, b) in blocks.iter().enumerate() {
|
||||
let want_page = (i as u32) + 1;
|
||||
match b.common.source_span {
|
||||
SourceSpan::Page {
|
||||
page,
|
||||
char_start,
|
||||
char_end,
|
||||
} => {
|
||||
assert_eq!(page, want_page);
|
||||
assert_eq!(char_start, Some(0));
|
||||
let chars = b.text.chars().count() as u32;
|
||||
assert_eq!(char_end, Some(chars));
|
||||
}
|
||||
ref other => panic!("expected Page span, got {other:?}"),
|
||||
}
|
||||
assert!(
|
||||
b.text.contains(&format!("Hello page {want_page}")),
|
||||
"page {want_page} text mismatch: {:?}",
|
||||
b.text
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_page_emits_warning_and_empty_paragraph() {
|
||||
let bytes = build_text_pdf(&[Some("page one text"), None, Some("page three text")]);
|
||||
let fx = fixture_for("docs/scanned-mixed.pdf", &bytes);
|
||||
let doc = PdfTextExtractor::new()
|
||||
.extract(&fx.ctx(), &bytes)
|
||||
.expect("scanned-mixed extraction must succeed");
|
||||
|
||||
let blocks = paragraph_blocks(&doc);
|
||||
assert_eq!(blocks.len(), 3);
|
||||
assert!(blocks[1].text.is_empty(), "page 2 should have empty text");
|
||||
assert!(
|
||||
blocks[1].inlines.is_empty(),
|
||||
"page 2 inlines should be empty"
|
||||
);
|
||||
match blocks[1].common.source_span {
|
||||
SourceSpan::Page {
|
||||
page,
|
||||
char_start,
|
||||
char_end,
|
||||
} => {
|
||||
assert_eq!(page, 2);
|
||||
assert_eq!(char_start, Some(0));
|
||||
assert_eq!(char_end, Some(0));
|
||||
}
|
||||
ref other => panic!("expected Page, got {other:?}"),
|
||||
}
|
||||
|
||||
let warnings: Vec<_> = doc
|
||||
.provenance
|
||||
.events
|
||||
.iter()
|
||||
.filter(|e| e.kind == ProvenanceKind::Warning)
|
||||
.collect();
|
||||
assert_eq!(warnings.len(), 1, "exactly one warning for the empty page");
|
||||
assert!(
|
||||
warnings[0]
|
||||
.note
|
||||
.as_deref()
|
||||
.unwrap_or("")
|
||||
.contains("page2 empty (scanned candidate)"),
|
||||
"warning note must mark page 2 as scanned candidate: {:?}",
|
||||
warnings[0].note
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encrypted_pdf_returns_helpful_error() {
|
||||
let bytes = make_encrypted_pdf();
|
||||
let fx = fixture_for("docs/encrypted.pdf", &bytes);
|
||||
let err = PdfTextExtractor::new()
|
||||
.extract(&fx.ctx(), &bytes)
|
||||
.expect_err("encrypted PDF must be refused");
|
||||
let msg = format!("{err:#}");
|
||||
assert!(
|
||||
msg.contains("encrypted"),
|
||||
"error must mention encryption: {msg}"
|
||||
);
|
||||
assert!(
|
||||
msg.contains("qpdf") || msg.contains("decrypt"),
|
||||
"error should point at remediation: {msg}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn corrupt_header_returns_error() {
|
||||
let bytes = corrupt_pdf();
|
||||
let fx = fixture_for("docs/corrupt.pdf", &bytes);
|
||||
let err = PdfTextExtractor::new()
|
||||
.extract(&fx.ctx(), &bytes)
|
||||
.expect_err("corrupt PDF must error");
|
||||
let msg = format!("{err:#}");
|
||||
assert!(
|
||||
msg.to_lowercase().contains("pdf") || msg.contains("parse"),
|
||||
"error must mention PDF parse failure: {msg}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn page_count_matches_actual_count() {
|
||||
let bytes = build_text_pdf(&[Some("a"), Some("b"), Some("c"), Some("d"), Some("e")]);
|
||||
let fx = fixture_for("docs/five.pdf", &bytes);
|
||||
let doc = PdfTextExtractor::new()
|
||||
.extract(&fx.ctx(), &bytes)
|
||||
.expect("5-page extraction must succeed");
|
||||
|
||||
assert_eq!(doc.metadata.user["pdf"]["page_count"], Value::Number(5.into()));
|
||||
assert_eq!(doc.blocks.len(), 5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn info_dict_title_utf16be_bom_decoded() {
|
||||
// Korean Title encoded as UTF-16BE with BOM is the standard PDF
|
||||
// path for any non-ASCII metadata. We don't try to decode the
|
||||
// body text in non-Latin scripts here (CID font support is out
|
||||
// of scope for v1) — but the metadata path is in scope.
|
||||
let info = InfoDict {
|
||||
title: Some(utf16be_bom("케밥 문서")),
|
||||
producer: Some("kebab-test"),
|
||||
creator: None,
|
||||
};
|
||||
let bytes = build_text_pdf_with_info(&[Some("body")], &info);
|
||||
let fx = fixture_for("docs/korean-title.pdf", &bytes);
|
||||
let doc = PdfTextExtractor::new()
|
||||
.extract(&fx.ctx(), &bytes)
|
||||
.expect("PDF with UTF-16BE Title must extract");
|
||||
|
||||
assert_eq!(doc.title, "케밥 문서");
|
||||
assert_eq!(
|
||||
doc.metadata.user["pdf"]["producer"],
|
||||
Value::String("kebab-test".into())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn info_dict_title_falls_back_to_filename_when_missing() {
|
||||
let bytes = build_text_pdf(&[Some("body")]);
|
||||
let fx = fixture_for("docs/no-info.pdf", &bytes);
|
||||
let doc = PdfTextExtractor::new()
|
||||
.extract(&fx.ctx(), &bytes)
|
||||
.expect("no-info PDF must extract");
|
||||
assert_eq!(doc.title, "no-info");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn determinism_identical_bytes_produce_identical_documents() {
|
||||
let bytes = build_text_pdf(&[Some("alpha"), Some("beta"), Some("gamma")]);
|
||||
let fx = fixture_for("docs/det.pdf", &bytes);
|
||||
|
||||
let mut a = serde_json::to_value(
|
||||
PdfTextExtractor::new()
|
||||
.extract(&fx.ctx(), &bytes)
|
||||
.expect("first extract"),
|
||||
)
|
||||
.unwrap();
|
||||
let mut b = serde_json::to_value(
|
||||
PdfTextExtractor::new()
|
||||
.extract(&fx.ctx(), &bytes)
|
||||
.expect("second extract"),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
strip_dynamic_at(&mut a);
|
||||
strip_dynamic_at(&mut b);
|
||||
assert_eq!(a, b, "two extracts of identical bytes must be byte-equal");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn snapshot_three_page_canonical_document_stable() {
|
||||
let bytes = build_text_pdf(&[Some("p1"), Some("p2"), Some("p3")]);
|
||||
let fx = fixture_for("docs/snapshot.pdf", &bytes);
|
||||
let doc = PdfTextExtractor::new()
|
||||
.extract(&fx.ctx(), &bytes)
|
||||
.expect("snapshot extract");
|
||||
let mut json = serde_json::to_value(&doc).unwrap();
|
||||
strip_dynamic_at(&mut json);
|
||||
|
||||
// Spot-check the load-bearing shape rather than committing a full
|
||||
// golden file (the full JSON contains BLAKE3 ids that would
|
||||
// change if `id_from(...)`'s tuple shape ever shifts — that would
|
||||
// be a separate, intentional break).
|
||||
assert_eq!(json["parser_version"], Value::String("pdf-text-v1".into()));
|
||||
assert_eq!(json["lang"], Value::String("und".into()));
|
||||
assert_eq!(json["schema_version"], Value::Number(1.into()));
|
||||
assert_eq!(json["doc_version"], Value::Number(1.into()));
|
||||
assert_eq!(json["blocks"].as_array().unwrap().len(), 3);
|
||||
for (i, block) in json["blocks"].as_array().unwrap().iter().enumerate() {
|
||||
assert_eq!(block["kind"], Value::String("paragraph".into()));
|
||||
assert_eq!(
|
||||
block["common"]["source_span"]["kind"],
|
||||
Value::String("page".into())
|
||||
);
|
||||
assert_eq!(
|
||||
block["common"]["source_span"]["page"],
|
||||
Value::Number(((i as u64) + 1).into())
|
||||
);
|
||||
}
|
||||
assert_eq!(json["metadata"]["source_type"], Value::String("paper".into()));
|
||||
assert_eq!(
|
||||
json["metadata"]["trust_level"],
|
||||
Value::String("primary".into())
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user