feat(kebab-parse-pdf): P7-1 text PDF extractor — per-page CanonicalDocument
`PdfTextExtractor`(MediaType::Pdf) lopdf 기반 per-page 텍스트 추출.
페이지마다 `Block::Paragraph` + `SourceSpan::Page { page, char_start, char_end }`
emit. 본문이 비거나 추출 panic 인 페이지는 빈 paragraph + `Provenance::Warning`
("scanned candidate") 로 표시 — 이후 OCR fallback (별도 task) 의 입력.
핵심 동작:
- `lopdf::Document::load_mem` + `is_encrypted()` → 암호화 PDF 는 명시 에러
(`qpdf --decrypt` 안내).
- 페이지 단위 `extract_text(&[page])` 를 `catch_unwind` 로 감싸 malformed
page panic 을 recoverable warning 으로 변환.
- `/Info` dict 에서 Title/Producer/Creator best-effort 추출. UTF-16BE BOM
prefixed 문자열도 디코드 (한국어 등 non-ASCII Title 정상 처리).
- 9개 통합 테스트: 3-page emit, scanned-mixed warning, encrypted refuse,
corrupt header error, page_count 메타, UTF-16BE Title, filename
fallback, determinism, snapshot.
`parser_version = "pdf-text-v1"`. Allowed deps: `lopdf 0.32` + `pdf-extract 0.7`
(원본 spec 그대로). 본문 다국어 OCR fallback 은 §9.2 후속 task (out of scope).
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
228
crates/kebab-parse-pdf/src/lib.rs
Normal file
228
crates/kebab-parse-pdf/src/lib.rs
Normal file
@@ -0,0 +1,228 @@
|
||||
//! `kebab-parse-pdf` — text PDF extractor (P7-1).
|
||||
//!
|
||||
//! Implements [`kebab_core::Extractor`] for [`MediaType::Pdf`]. Extracts
|
||||
//! text page-by-page via `lopdf`'s per-page API and emits one
|
||||
//! [`Block::Paragraph`] per page with [`SourceSpan::Page`] (1-based page,
|
||||
//! `char_start = 0`, `char_end = chars().count()`).
|
||||
//!
|
||||
//! Pages where text extraction fails or returns empty get an empty
|
||||
//! `Block::Paragraph` plus a `Provenance::Warning` flagging the page as
|
||||
//! a "scanned candidate" — out-of-scope OCR fallback can pick those up.
|
||||
//!
|
||||
//! Scope is intentionally narrow: page text + page numbers. Layout
|
||||
//! reconstruction (multi-column reading order, tables, math), form
|
||||
//! fields, bookmarks, and OCR for scanned PDFs are explicitly **not**
|
||||
//! in this task. See `tasks/p7/p7-1-pdf-text-extractor.md`.
|
||||
//!
|
||||
//! Per design §3.4 (`SourceSpan::Page` / `Block::Paragraph`),
|
||||
//! §9.2 (PDF text extraction), §9 versioning.
|
||||
|
||||
mod info;
|
||||
mod page_text;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use kebab_core::{
|
||||
Block, CanonicalDocument, CommonBlock, Extractor, Inline, Lang, MediaType, Metadata,
|
||||
ParserVersion, Provenance, ProvenanceEvent, ProvenanceKind, SourceSpan, SourceType, TextBlock,
|
||||
TrustLevel, id_for_block, id_for_doc,
|
||||
};
|
||||
use serde_json::{Map, Value};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
pub const PARSER_VERSION: &str = "pdf-text-v1";
|
||||
|
||||
/// Text-PDF extractor. Per-page text via `lopdf::Document::extract_text`
|
||||
/// (the only stable per-page API in the lopdf / pdf-extract pair —
|
||||
/// pdf-extract 0.7 only exposes whole-document calls).
|
||||
pub struct PdfTextExtractor;
|
||||
|
||||
impl PdfTextExtractor {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for PdfTextExtractor {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl Extractor for PdfTextExtractor {
|
||||
fn supports(&self, m: &MediaType) -> bool {
|
||||
matches!(m, MediaType::Pdf)
|
||||
}
|
||||
|
||||
fn parser_version(&self) -> ParserVersion {
|
||||
ParserVersion(PARSER_VERSION.to_string())
|
||||
}
|
||||
|
||||
fn extract(
|
||||
&self,
|
||||
ctx: &kebab_core::ExtractContext<'_>,
|
||||
bytes: &[u8],
|
||||
) -> Result<CanonicalDocument> {
|
||||
let asset = ctx.asset;
|
||||
if !self.supports(&asset.media_type) {
|
||||
anyhow::bail!(
|
||||
"kebab-parse-pdf: unsupported media_type for PdfTextExtractor: {:?}",
|
||||
asset.media_type
|
||||
);
|
||||
}
|
||||
|
||||
let parser_version = self.parser_version();
|
||||
let doc_id = id_for_doc(&asset.workspace_path, &asset.asset_id, &parser_version);
|
||||
|
||||
// Catastrophic-decode guard via lopdf. `pdf-extract` is intentionally
|
||||
// not used for parsing here — it only exposes whole-doc text and
|
||||
// would re-parse the bytes a second time.
|
||||
let pdf_doc = lopdf::Document::load_mem(bytes)
|
||||
.context("kebab-parse-pdf: failed to parse PDF (corrupt header or not a PDF)")?;
|
||||
|
||||
if pdf_doc.is_encrypted() {
|
||||
anyhow::bail!(
|
||||
"kebab-parse-pdf: encrypted PDF; remove encryption (e.g. `qpdf --decrypt`) before ingest"
|
||||
);
|
||||
}
|
||||
|
||||
let info = info::extract_info(&pdf_doc);
|
||||
// `get_pages()` returns BTreeMap<u32, ObjectId> with 1-based page
|
||||
// numbers. We iterate keys in BTreeMap natural order so output is
|
||||
// deterministic.
|
||||
let pages = pdf_doc.get_pages();
|
||||
let page_count = pages.len() as u32;
|
||||
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let mut events: Vec<ProvenanceEvent> = Vec::with_capacity(2 + pages.len());
|
||||
events.push(ProvenanceEvent {
|
||||
at: asset.discovered_at,
|
||||
agent: "kb-source-fs".to_string(),
|
||||
kind: ProvenanceKind::Discovered,
|
||||
note: None,
|
||||
});
|
||||
events.push(ProvenanceEvent {
|
||||
at: now,
|
||||
agent: "kb-parse-pdf".to_string(),
|
||||
kind: ProvenanceKind::Parsed,
|
||||
note: Some(format!(
|
||||
"parser_version={}; page_count={}",
|
||||
parser_version.0, page_count
|
||||
)),
|
||||
});
|
||||
|
||||
let mut blocks: Vec<Block> = Vec::with_capacity(pages.len());
|
||||
for (&page_num, _) in pages.iter() {
|
||||
let (text, warning) = match page_text::extract_one(&pdf_doc, page_num) {
|
||||
Ok(t) if !t.trim().is_empty() => (t, None),
|
||||
Ok(_) => (
|
||||
String::new(),
|
||||
Some(format!("page{page_num} empty (scanned candidate)")),
|
||||
),
|
||||
Err(e) => (
|
||||
String::new(),
|
||||
Some(format!(
|
||||
"page{page_num} extract failed: {e} (scanned candidate)"
|
||||
)),
|
||||
),
|
||||
};
|
||||
let char_count = text.chars().count() as u32;
|
||||
let span = SourceSpan::Page {
|
||||
page: page_num,
|
||||
char_start: Some(0),
|
||||
char_end: Some(char_count),
|
||||
};
|
||||
// ordinal = page - 1; saturating_sub guards the (shouldn't-happen)
|
||||
// case where lopdf hands back a 0-indexed page key.
|
||||
let ordinal = page_num.saturating_sub(1);
|
||||
let block_id = id_for_block(&doc_id, "paragraph", &[], ordinal, &span);
|
||||
let common = CommonBlock {
|
||||
block_id,
|
||||
heading_path: Vec::new(),
|
||||
source_span: span,
|
||||
};
|
||||
let inlines = if text.is_empty() {
|
||||
Vec::new()
|
||||
} else {
|
||||
vec![Inline::Text { text: text.clone() }]
|
||||
};
|
||||
blocks.push(Block::Paragraph(TextBlock {
|
||||
common,
|
||||
text,
|
||||
inlines,
|
||||
}));
|
||||
if let Some(note) = warning {
|
||||
events.push(ProvenanceEvent {
|
||||
at: now,
|
||||
agent: "kb-parse-pdf".to_string(),
|
||||
kind: ProvenanceKind::Warning,
|
||||
note: Some(note),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let title = info
|
||||
.title
|
||||
.clone()
|
||||
.filter(|t| !t.trim().is_empty())
|
||||
.unwrap_or_else(|| {
|
||||
let fname = filename_from_workspace_path(&asset.workspace_path.0);
|
||||
strip_extension(&fname)
|
||||
});
|
||||
|
||||
let mut user = Map::new();
|
||||
let mut pdf_meta = Map::new();
|
||||
pdf_meta.insert("page_count".into(), Value::Number(page_count.into()));
|
||||
if let Some(p) = &info.producer {
|
||||
pdf_meta.insert("producer".into(), Value::String(p.clone()));
|
||||
}
|
||||
if let Some(c) = &info.creator {
|
||||
pdf_meta.insert("creator".into(), Value::String(c.clone()));
|
||||
}
|
||||
user.insert("pdf".into(), Value::Object(pdf_meta));
|
||||
|
||||
let metadata = Metadata {
|
||||
aliases: Vec::new(),
|
||||
tags: Vec::new(),
|
||||
created_at: asset.discovered_at,
|
||||
updated_at: asset.discovered_at,
|
||||
source_type: SourceType::Paper,
|
||||
trust_level: TrustLevel::Primary,
|
||||
user_id_alias: None,
|
||||
user,
|
||||
};
|
||||
|
||||
tracing::debug!(
|
||||
target: "kebab-parse-pdf",
|
||||
"extracted PDF doc_id={} workspace_path={} pages={}",
|
||||
doc_id.0,
|
||||
asset.workspace_path.0,
|
||||
page_count
|
||||
);
|
||||
|
||||
Ok(CanonicalDocument {
|
||||
doc_id,
|
||||
source_asset_id: asset.asset_id.clone(),
|
||||
workspace_path: asset.workspace_path.clone(),
|
||||
title,
|
||||
lang: Lang("und".to_string()),
|
||||
blocks,
|
||||
metadata,
|
||||
provenance: Provenance { events },
|
||||
parser_version,
|
||||
schema_version: 1,
|
||||
doc_version: 1,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn filename_from_workspace_path(p: &str) -> String {
|
||||
p.rsplit('/').next().unwrap_or(p).to_string()
|
||||
}
|
||||
|
||||
fn strip_extension(filename: &str) -> String {
|
||||
match filename.rfind('.') {
|
||||
Some(0) => filename.to_string(),
|
||||
Some(idx) => filename[..idx].to_string(),
|
||||
None => filename.to_string(),
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user