From a08ed321991816519a7560d61c56e4ee8b66ae17 Mon Sep 17 00:00:00 2001 From: altair823 Date: Tue, 19 May 2026 15:36:08 +0000 Subject: [PATCH 01/19] docs(p10-1a-2): task spec + implementation plan Co-Authored-By: Claude Opus 4.7 (1M context) --- .../2026-05-19-p10-1a-2-rust-ast-chunker.md | 1441 +++++++++++++++++ tasks/p10/p10-1a-2-rust-ast-chunker.md | 47 + 2 files changed, 1488 insertions(+) create mode 100644 docs/superpowers/plans/2026-05-19-p10-1a-2-rust-ast-chunker.md create mode 100644 tasks/p10/p10-1a-2-rust-ast-chunker.md diff --git a/docs/superpowers/plans/2026-05-19-p10-1a-2-rust-ast-chunker.md b/docs/superpowers/plans/2026-05-19-p10-1a-2-rust-ast-chunker.md new file mode 100644 index 0000000..5d55d11 --- /dev/null +++ b/docs/superpowers/plans/2026-05-19-p10-1a-2-rust-ast-chunker.md @@ -0,0 +1,1441 @@ +# p10-1A-2 Rust AST Chunker Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Activate Rust code ingest end-to-end — `.rs` files parse via tree-sitter into one `Block::Code` per AST semantic unit, chunk 1:1 (with oversize fallback split), and surface as `Citation::Code { symbol, lang, line_start, line_end }` in search. + +**Architecture:** tree-sitter lives in the **parser** (`kebab-parse-code`, per design §6.3 dependency graph; mirrors the proven PDF pattern where the parser emits structured blocks and the chunker maps them). `kebab-parse-code/src/rust.rs` is an `Extractor` producing a `CanonicalDocument` whose blocks are AST units carrying a new internal `SourceSpan::Code` variant. `kebab-chunk/src/code_rust_ast_v1.rs` maps each block to a chunk and splits oversize units. `citation_helper` gains one arm so the span flows to the existing (1A-1) `Citation::Code` wire shape. A new `MediaType::Code(String)` routes `.rs` files; non-Rust code langs stay `MediaType::Other` in 1A. + +**Tech Stack:** Rust 2024 workspace, `tree-sitter` + `tree-sitter-rust`, existing `kebab-core` domain model, `serde_json_canonicalizer` + `blake3` (chunk id / policy hash), `gix` (1A-1 repo detect). + +--- + +## Pre-flight + +- [ ] **Branch.** From clean `main`: + +```bash +cd /home/altair823/kebab +git checkout main && git pull +git checkout -b feat/p10-1a-2-rust-ast-chunker +``` + +- [ ] **Disk hygiene** (CLAUDE.md — routine after each merged PR; #139 just merged): + +```bash +cargo clean +``` + +Notes that apply throughout: +- Full suite is always `cargo test --workspace --no-fail-fast -j 1` (CLAUDE.md — parallel link OOMs). Per-crate runs (`cargo test -p `) may run normally. +- `cargo clippy --workspace --all-targets -- -D warnings` is the CI gate; run before every commit that touches code. +- Frozen task spec for this work: `tasks/p10/p10-1a-2-rust-ast-chunker.md` (already written; do not edit retroactively). + +--- + +## Task 1: Add tree-sitter dependencies (workspace-deps pattern) + +**Files:** +- Modify: `Cargo.toml` (workspace `[workspace.dependencies]`, ends ~line 90 after the `gix` entry) +- Modify: `crates/kebab-parse-code/Cargo.toml` (`[dependencies]`) + +- [ ] **Step 1: Resolve + pin versions via cargo add** + +Run (this picks the latest compatible versions and writes them into the crate): + +```bash +cargo add tree-sitter tree-sitter-rust -p kebab-parse-code +``` + +- [ ] **Step 2: Move the resolved versions into workspace deps** + +Read the two version strings `cargo add` wrote into `crates/kebab-parse-code/Cargo.toml`. Then in the workspace `Cargo.toml`, append to `[workspace.dependencies]` directly after the `gix = { ... }` line (keep the existing comment style — one explanatory comment line): + +```toml +# Rust source parsing for code ingest (kebab-parse-code, p10-1A-2). The +# chunker stays tree-sitter-free — AST work is parser-side per design §6.3. +tree-sitter = "" +tree-sitter-rust = "" +``` + +Then rewrite the crate's `[dependencies]` to use the workspace table (matching the existing `anyhow`/`gix` style): + +```toml +[dependencies] +anyhow = { workspace = true } +gix = { workspace = true } +tree-sitter = { workspace = true } +tree-sitter-rust = { workspace = true } + +[dev-dependencies] +tempfile = { workspace = true } +``` + +- [ ] **Step 3: Verify it builds and the lock updated** + +Run: `cargo build -p kebab-parse-code` +Expected: compiles clean (skeleton still has no tree-sitter use yet — deps unused is fine, no `-D warnings` on a plain build). + +- [ ] **Step 4: Commit** + +```bash +git add Cargo.toml Cargo.lock crates/kebab-parse-code/Cargo.toml +git commit -m "build(p10-1a-2): add tree-sitter + tree-sitter-rust workspace deps + +Co-Authored-By: Claude Opus 4.7 (1M context) " +``` + +--- + +## Task 2: `SourceSpan::Code` internal variant + +**Files:** +- Modify: `crates/kebab-core/src/document.rs` (`SourceSpan` enum, after the `Time { start_ms, end_ms }` variant ~line 144) +- Modify: `docs/superpowers/specs/2026-04-27-kebab-final-form-design.md` (§3.4 `SourceSpan` enum listing, ~line 562) +- Test: `crates/kebab-core/src/document.rs` (`#[cfg(test)] mod tests`, or the file's existing test module) + +`SourceSpan` is `#[serde(rename_all = "lowercase", tag = "kind")]` — the new variant serializes as `{"kind":"code", ...}`. This is the chunks-table `source_spans_json` internal shape, NOT a wire schema (wire `Citation::Code` already shipped in 1A-1), so no wire `.v2` bump. + +- [ ] **Step 1: Write the failing test** + +Add to the `kebab-core` test module that covers `SourceSpan` (search `mod tests` in `document.rs`; if none, add one at end of file): + +```rust +#[test] +fn source_span_code_round_trips_and_tags_lowercase() { + let s = SourceSpan::Code { + line_start: 10, + line_end: 42, + symbol: Some("foo::Bar::baz".to_string()), + lang: Some("rust".to_string()), + }; + let v = serde_json::to_value(&s).unwrap(); + assert_eq!(v["kind"], "code"); + assert_eq!(v["line_start"], 10); + assert_eq!(v["line_end"], 42); + assert_eq!(v["symbol"], "foo::Bar::baz"); + assert_eq!(v["lang"], "rust"); + let back: SourceSpan = serde_json::from_value(v).unwrap(); + assert_eq!(back, s); +} +``` + +- [ ] **Step 2: Run it — expect compile failure** + +Run: `cargo test -p kebab-core source_span_code_round_trips` +Expected: FAIL — `no variant named Code`. + +- [ ] **Step 3: Add the variant** + +In `crates/kebab-core/src/document.rs`, add as the last variant of `pub enum SourceSpan` (after `Time { ... }`): + +```rust + /// p10-1A-2: AST-unit span for code ingest. Internal storage shape + /// (chunks.source_spans_json) — `citation_helper` maps this to the + /// wire `Citation::Code` (added 1A-1). `symbol` is the per-language + /// self-reference path (design §3.4); `` / `` for + /// glue regions, never null for an identified unit. `lang` is the + /// canonical code_lang. + Code { + line_start: u32, + line_end: u32, + symbol: Option, + lang: Option, + }, +``` + +- [ ] **Step 4: Compile — fix every non-exhaustive match** + +Run: `cargo build --workspace 2>&1 | grep -A2 "non-exhaustive\|E0004"` + +The compiler will flag every exhaustive `match` on `SourceSpan`. Known sites (handle each minimally — a `Code` arm that does the type-correct thing, NOT a catch-all `_`): +- `crates/kebab-search/src/citation_helper.rs` — handled fully in Task 3; for now add a temporary `SourceSpan::Code { .. } => Citation::Line { path, start: 1, end: 1, section }` arm with a `// TODO(Task 3)` and replace it in Task 3. +- Any `store-sqlite` / `search` / `id` site: add a faithful arm (e.g. id recipe already serializes `SourceSpan` via serde — likely no match there; only fix real `match` statements the compiler points at). + +Run: `cargo build --workspace` +Expected: clean. + +- [ ] **Step 5: Run the test** + +Run: `cargo test -p kebab-core source_span_code_round_trips` +Expected: PASS. + +- [ ] **Step 6: Sync frozen design §3.4** + +In `docs/superpowers/specs/2026-04-27-kebab-final-form-design.md`, find `pub enum SourceSpan` (~line 562) and add the `Code { line_start, line_end, symbol, lang }` variant to the listing, with a one-line comment `// p10-1A-2: internal code-unit span (see tasks/p10/p10-1a-2)`. Do not alter other variants. + +- [ ] **Step 7: clippy + commit** + +```bash +cargo clippy --workspace --all-targets -- -D warnings +git add crates/ docs/superpowers/specs/2026-04-27-kebab-final-form-design.md +git commit -m "feat(p10-1a-2): add internal SourceSpan::Code variant + design §3.4 sync + +Co-Authored-By: Claude Opus 4.7 (1M context) " +``` + +--- + +## Task 3: `citation_helper` Code arm + +**Files:** +- Modify: `crates/kebab-search/src/citation_helper.rs:21-74` +- Test: `crates/kebab-search/src/citation_helper.rs` (add `#[cfg(test)] mod tests` if absent; `lexical.rs` has `build_citation_*` tests — mirror style there if a test module already imports the helper) + +- [ ] **Step 1: Write the failing test** + +Add a test (in `citation_helper.rs` add a test module, or extend the existing helper tests in `crates/kebab-search/src/lexical.rs` near `build_citation_page_forwards_section`): + +```rust +#[test] +fn build_citation_code_maps_symbol_and_lang() { + use kebab_core::{Citation, SourceSpan, WorkspacePath}; + let span = SourceSpan::Code { + line_start: 5, + line_end: 30, + symbol: Some("chunk::md_heading_v1::MdHeadingV1Chunker::chunk".into()), + lang: Some("rust".into()), + }; + let c = super::citation_from_first_span( + "c1", + WorkspacePath("crates/kebab-chunk/src/md_heading_v1.rs".into()), + None, + Some(&span), + ); + match c { + Citation::Code { path, line_start, line_end, symbol, lang } => { + assert_eq!(path.0, "crates/kebab-chunk/src/md_heading_v1.rs"); + assert_eq!(line_start, 5); + assert_eq!(line_end, 30); + assert_eq!(symbol.as_deref(), Some("chunk::md_heading_v1::MdHeadingV1Chunker::chunk")); + assert_eq!(lang.as_deref(), Some("rust")); + } + other => panic!("expected Citation::Code, got {other:?}"), + } +} +``` + +- [ ] **Step 2: Run — expect fail** + +Run: `cargo test -p kebab-search build_citation_code_maps_symbol_and_lang` +Expected: FAIL (currently the Task-2 temporary arm produces `Citation::Line`). + +- [ ] **Step 3: Replace the temporary arm** + +In `citation_from_first_span`, replace the Task-2 placeholder arm with the real mapping (place it directly after the `SourceSpan::Time` arm, before the `Byte | None` fallback): + +```rust + Some(SourceSpan::Code { line_start, line_end, symbol, lang }) => Citation::Code { + path, + line_start: *line_start, + line_end: *line_end, + symbol: symbol.clone(), + lang: lang.clone(), + }, +``` + +(`section` is unused for code — `Citation::Code` has no section field; this matches the spec's code citation shape.) + +- [ ] **Step 4: Run — expect pass** + +Run: `cargo test -p kebab-search build_citation_code_maps_symbol_and_lang` +Expected: PASS. + +- [ ] **Step 5: clippy + commit** + +```bash +cargo clippy -p kebab-search --all-targets -- -D warnings +git add crates/kebab-search/ +git commit -m "feat(p10-1a-2): map SourceSpan::Code -> Citation::Code in citation_helper + +Co-Authored-By: Claude Opus 4.7 (1M context) " +``` + +--- + +## Task 4: `MediaType::Code(String)` variant + +**Files:** +- Modify: `crates/kebab-core/src/media.rs:38-44` (`MediaType` enum) +- Modify: `crates/kebab-app/src/ingest_progress.rs:99` (`media_label`) +- Modify: `docs/superpowers/specs/2026-04-27-kebab-final-form-design.md` (only if `enum MediaType` is enumerated there — `grep -n "enum MediaType" docs/superpowers/specs/2026-04-27-kebab-final-form-design.md`; if present, add the `Code(String)` variant + one-line comment, else skip) +- Test: `crates/kebab-core/src/media.rs` test module + +`MediaType` is `#[serde(rename_all = "lowercase")]`; `Code(String)` serializes as `{"code":"rust"}`. + +- [ ] **Step 1: Write the failing test** + +Add to `media.rs` tests: + +```rust +#[test] +fn media_type_code_serializes_lowercase_tagged() { + let m = MediaType::Code("rust".to_string()); + let v = serde_json::to_value(&m).unwrap(); + assert_eq!(v, serde_json::json!({ "code": "rust" })); + let back: MediaType = serde_json::from_value(v).unwrap(); + assert_eq!(back, m); +} +``` + +- [ ] **Step 2: Run — expect fail** + +Run: `cargo test -p kebab-core media_type_code_serializes` +Expected: FAIL — `no variant named Code`. + +- [ ] **Step 3: Add the variant + media_label arm** + +In `crates/kebab-core/src/media.rs`, add to `pub enum MediaType` immediately before `Other(String)`: + +```rust + /// p10-1A-2: a source-code file. Inner string is the canonical + /// code_lang (design §3.5). 1A activates `"rust"` only; other + /// recognized code langs are still routed `Other` until their phase. + Code(String), +``` + +In `crates/kebab-app/src/ingest_progress.rs`, add a match arm next to the `MediaType::Other(_) => "other"` arm (~line 99): + +```rust + kebab_core::MediaType::Code(_) => "code", +``` + +Then `cargo build --workspace 2>&1 | grep non-exhaustive` and add a faithful arm to every other `MediaType` match the compiler flags (e.g. any UI/store display) — `Code(lang)` should render analogously to `Other`. + +- [ ] **Step 4: Run — expect pass + suite green** + +Run: `cargo test -p kebab-core media_type_code_serializes` +Expected: PASS. +Run: `cargo test --workspace --no-fail-fast -j 1` +Expected: PASS (catches any golden/asset serialization that enumerates MediaType variants; fix faithfully if any fixture counts variants). + +- [ ] **Step 5: design sync (conditional) + clippy + commit** + +```bash +grep -n "enum MediaType" docs/superpowers/specs/2026-04-27-kebab-final-form-design.md +# if present, add Code(String) to that listing with a p10-1A-2 comment +cargo clippy --workspace --all-targets -- -D warnings +git add crates/ docs/ +git commit -m "feat(p10-1a-2): add MediaType::Code(lang) variant + +Co-Authored-By: Claude Opus 4.7 (1M context) " +``` + +--- + +## Task 5: Route `.rs` → `MediaType::Code("rust")` + +**Files:** +- Modify: `crates/kebab-source-fs/src/media.rs:39` (the `_ => MediaType::Other(ext)` fallthrough) +- Test: `crates/kebab-source-fs/src/media.rs` test module (`media_type_for` tests, ~line 49) + +Scope to Rust only (1A-2 = Rust). Non-Rust extensions keep their current `MediaType::Other` mapping — minimal blast radius, regression-safe. + +- [ ] **Step 1: Write the failing test** + +Add near the existing `media_type_for` asserts: + +```rust +#[test] +fn rust_files_map_to_media_code_rust() { + assert_eq!( + media_type_for(Path::new("crates/kebab-core/src/lib.rs")), + MediaType::Code("rust".to_string()) + ); + // non-Rust code extensions stay Other in 1A + assert_eq!(media_type_for(Path::new("a/b.py")), MediaType::Other("py".to_string())); + assert_eq!(media_type_for(Path::new("Cargo.toml")), MediaType::Other("toml".to_string())); +} +``` + +- [ ] **Step 2: Run — expect fail** + +Run: `cargo test -p kebab-source-fs rust_files_map_to_media_code_rust` +Expected: FAIL — `.rs` currently → `MediaType::Other("rs")`. + +- [ ] **Step 3: Add the routing arm** + +In `crates/kebab-source-fs/src/media.rs`, add an `"rs"` arm before the final `_ => MediaType::Other(ext)`: + +```rust + // p10-1A-2: Rust is the only code lang activated in 1A. Other + // recognized code langs stay Other until their phase (1B+). + "rs" => MediaType::Code("rust".to_string()), +``` + +- [ ] **Step 4: Run — expect pass** + +Run: `cargo test -p kebab-source-fs rust_files_map_to_media_code_rust` +Expected: PASS. + +- [ ] **Step 5: clippy + commit** + +```bash +cargo clippy -p kebab-source-fs --all-targets -- -D warnings +git add crates/kebab-source-fs/ +git commit -m "feat(p10-1a-2): route .rs files to MediaType::Code(rust) + +Co-Authored-By: Claude Opus 4.7 (1M context) " +``` + +--- + +## Task 6: `kebab-parse-code` Rust AST extractor + +**Files:** +- Create: `crates/kebab-parse-code/src/rust.rs` +- Modify: `crates/kebab-parse-code/src/lib.rs` (add `pub mod rust;` + re-export `PARSER_VERSION`, `RustAstExtractor`) +- Create: `crates/kebab-parse-code/tests/fixtures/sample.rs` (Rust fixture) +- Test: inline `#[cfg(test)] mod tests` in `rust.rs` + +This is the core. The extractor walks the tree-sitter parse tree and emits **one `Block::Code` per top-level AST semantic unit**, each with `SourceSpan::Code`. The `CanonicalDocument` scaffold (doc_id, provenance, metadata, return struct) **mirrors `crates/kebab-parse-pdf/src/lib.rs:51-225` exactly** — same `Extractor` trait impl shape, same `id_for_doc` / `ProvenanceEvent` / `CanonicalDocument` construction. Only the differences below change. + +### tree-sitter API note + +Use the modern API: `parser.set_language(&tree_sitter_rust::LANGUAGE.into())`. If the resolved `tree-sitter-rust` predates the `LANGUAGE: LanguageFn` const (no `LANGUAGE` symbol), use `parser.set_language(&tree_sitter_rust::language())` instead. Verify which by `cargo doc -p tree-sitter-rust --no-deps` or reading its docs; pick the one that compiles. + +### Semantic-unit rules (design §9.1 + §3.4) + +Walk `root_node()` (kind `source_file`) named children. Maintain a `mod_path: Vec` (module nesting), starting empty. + +| node kind | unit | symbol | +|-----------|------|--------| +| `function_item` | 1 | `mod_path::fn_name` | +| `struct_item` / `enum_item` / `union_item` / `trait_item` / `type_item` | 1 | `mod_path::TypeName` | +| `macro_definition` | 1 | `mod_path::name!` | +| `impl_item` | 1 per inner `function_item` | `mod_path::ImplType::method` (ImplType = text of impl `type` field; for `impl Trait for T`, use `Trait::method` per §3.4) | +| `mod_item` **with** `declaration_list` body | recurse with `mod_path` + mod name pushed | — | +| `use_declaration`, `extern_crate_declaration`, `const_item`, `static_item`, `mod_item` **without** body, top-level `attribute_item`/`macro_invocation` | accumulated into ONE grouped unit | `` (or `` if the whole file produced no fn/type/impl unit and the group is only `mod_item` declarations) | + +- **Line range:** `node.start_position().row + 1 ..= node.end_position().row + 1` (1-based inclusive). Extend `line_start` upward over contiguous immediately-preceding sibling `line_comment` / `block_comment` / `attribute_item` nodes (doc comments + attributes belong to their item — design §9.1 "선언 + doc comment"). +- **Grouped unit line range:** min start_line over the group .. max end_line over the group. +- **`code` field:** the exact source substring for those lines (split `source` by `\n`, take `[line_start-1 ..= line_end-1]`, rejoin with `\n`). +- Each unit → `Block::Code(CodeBlock { common: CommonBlock { block_id, heading_path: vec![], source_span }, lang: Some("rust".into()), code })` where `source_span = SourceSpan::Code { line_start, line_end, symbol: Some(sym), lang: Some("rust".into()) }`. `block_id = id_for_block(&doc_id, "code", &[], ordinal, &source_span)` with `ordinal` = 0-based unit index. + +- [ ] **Step 1: Create the fixture** + +Create `crates/kebab-parse-code/tests/fixtures/sample.rs`: + +```rust +//! sample fixture + +use std::fmt; + +const ANSWER: u32 = 42; + +/// Doc comment on a free fn. +pub fn parse(input: &str) -> usize { + input.len() +} + +pub struct Foo { + pub n: u32, +} + +impl Foo { + /// method doc + pub fn double(&self) -> u32 { + self.n * 2 + } + + fn name() -> &'static str { + "foo" + } +} + +pub trait Greet { + fn hello(&self) -> String; +} + +mod inner { + pub fn helper() -> bool { + true + } +} +``` + +- [ ] **Step 2: Write the failing test** + +In `rust.rs` add: + +```rust +#[cfg(test)] +mod tests { + use super::*; + use kebab_core::{Block, MediaType, SourceSpan}; + + fn extract_fixture() -> kebab_core::CanonicalDocument { + let bytes = std::fs::read( + concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/sample.rs"), + ) + .unwrap(); + let asset = kebab_parse_code_test_support::fixed_rust_asset("crates/x/src/sample.rs"); + let cfg = kebab_core::ExtractConfig::default(); + let root = std::path::PathBuf::from("/tmp"); + let ctx = kebab_core::ExtractContext { asset: &asset, workspace_root: &root, config: &cfg }; + RustAstExtractor::new().extract(&ctx, &bytes).unwrap() + } + + #[test] + fn extractor_supports_only_media_code_rust() { + let e = RustAstExtractor::new(); + assert!(e.supports(&MediaType::Code("rust".into()))); + assert!(!e.supports(&MediaType::Code("python".into()))); + assert!(!e.supports(&MediaType::Markdown)); + } + + #[test] + fn emits_one_block_per_semantic_unit_with_symbols() { + let doc = extract_fixture(); + let mut syms: Vec<(String, u32, u32)> = doc + .blocks + .iter() + .map(|b| match b { + Block::Code(c) => match &c.common.source_span { + SourceSpan::Code { symbol, line_start, line_end, lang } => { + assert_eq!(lang.as_deref(), Some("rust")); + (symbol.clone().unwrap(), *line_start, *line_end) + } + _ => panic!("code block must carry SourceSpan::Code"), + }, + other => panic!("expected Block::Code, got {other:?}"), + }) + .collect(); + syms.sort(); + let names: Vec<&str> = syms.iter().map(|(s, _, _)| s.as_str()).collect(); + assert!(names.contains(&"parse")); + assert!(names.contains(&"Foo")); + assert!(names.contains(&"Foo::double")); + assert!(names.contains(&"Foo::name")); + assert!(names.contains(&"Greet")); + assert!(names.contains(&"inner::helper")); + assert!(names.contains(&"")); // use + const grouped + // doc-comment line is folded into the unit it documents: + let parse_unit = syms.iter().find(|(s, _, _)| s == "parse").unwrap(); + let parse_src = doc.blocks.iter().find_map(|b| match b { + Block::Code(c) if matches!(&c.common.source_span, SourceSpan::Code{symbol,..} if symbol.as_deref()==Some("parse")) => Some(c.code.clone()), + _ => None, + }).unwrap(); + assert!(parse_src.contains("/// Doc comment on a free fn."), "doc comment folded in: {parse_src}"); + let _ = parse_unit; + } + + #[test] + fn deterministic_across_runs() { + let a = extract_fixture(); + for _ in 0..50 { + assert_eq!(extract_fixture().blocks, a.blocks); + } + } +} + +#[cfg(test)] +mod kebab_parse_code_test_support { + use kebab_core::*; + use time::OffsetDateTime; + pub fn fixed_rust_asset(path: &str) -> RawAsset { + RawAsset { + asset_id: AssetId("a".repeat(64)), + source_uri: SourceUri::File(std::path::PathBuf::from(path)), + workspace_path: WorkspacePath(path.to_string()), + media_type: MediaType::Code("rust".to_string()), + byte_len: 0, + checksum: Checksum("b".repeat(64)), + discovered_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(), + stored: AssetStorage::InPlace, + } + } +} +``` + +> Before relying on it, verify `Checksum`/`SourceUri`/`AssetStorage` field shapes by reading `crates/kebab-core/src/asset.rs:63-95`; adjust the test-support constructor to the actual variants (e.g. `AssetStorage` may be `External`/`InPlace` — use whatever exists). This is a fixture-construction detail, not a design choice. + +- [ ] **Step 3: Run — expect fail** + +Run: `cargo test -p kebab-parse-code emits_one_block_per_semantic_unit` +Expected: FAIL — `RustAstExtractor` undefined. + +- [ ] **Step 4: Implement `rust.rs`** + +Create `crates/kebab-parse-code/src/rust.rs`. Scaffold (Extractor impl, doc_id, provenance events, final `CanonicalDocument {…}`) is a **direct adaptation of `crates/kebab-parse-pdf/src/lib.rs:51-225`** with these concrete differences: + +- `pub const PARSER_VERSION: &str = "code-rust-v1";` +- `pub struct RustAstExtractor;` + `new()`/`Default` like `PdfTextExtractor`. +- `fn supports(&self, m: &MediaType) -> bool { matches!(m, MediaType::Code(l) if l == "rust") }` +- agent strings: `"kb-parse-code"` instead of `"kb-parse-pdf"`. +- `title`: filename stem of `asset.workspace_path` (reuse the same `strip_extension(filename_from_workspace_path(...))` helpers — copy those two small fns from `kebab-parse-pdf/src/lib.rs:229+` into `rust.rs`, or inline equivalent). +- `lang: Lang("und".into())` (natural-language detection out of scope, design §3.5). +- **metadata**: same `Metadata { … }` literal as PDF but set: + - `source_type`: use `SourceType::Code` if the enum has it (`grep -n "enum SourceType" crates/kebab-core/src/metadata.rs`), else `SourceType::Note`. + - `code_lang: Some("rust".to_string())` + - `repo` / `git_branch` / `git_commit`: from `crate::repo::detect_repo`. Resolve the file's absolute path: if `asset.source_uri` is `SourceUri::File(p)` use `p`; join with `ctx.workspace_root` if relative. `match detect_repo(&abs) { Some(r) => (Some(r.name), r.branch, r.commit), None => (None, None, None) }`. +- **blocks**: replace the PDF per-page loop with the AST walk. Implementation: + +```rust +fn build_blocks( + source: &str, + doc_id: &kebab_core::DocumentId, +) -> anyhow::Result> { + use kebab_core::{Block, CodeBlock, CommonBlock, SourceSpan, id_for_block}; + + let mut parser = tree_sitter::Parser::new(); + parser + .set_language(&tree_sitter_rust::LANGUAGE.into()) + .map_err(|e| anyhow::anyhow!("set tree-sitter-rust language: {e}"))?; + let tree = parser + .parse(source.as_bytes(), None) + .ok_or_else(|| anyhow::anyhow!("tree-sitter failed to parse Rust source"))?; + let lines: Vec<&str> = source.split('\n').collect(); + + // (symbol, start_line_1based, end_line_1based) in document order. + let mut units: Vec<(String, u32, u32)> = Vec::new(); + // Pending glue (use/const/static/mod-decl/attr) accumulated into one + // (or ) unit, flushed when a real unit appears or + // at end of a scope. + let mut glue: Vec<(usize, u32, u32)> = Vec::new(); // (is_mod_decl as 0/1 via usize, s, e) + + fn node_name<'a>(n: &tree_sitter::Node, src: &'a str) -> Option<&'a str> { + n.child_by_field_name("name") + .map(|c| &src[c.start_byte()..c.end_byte()]) + } + // Extend start upward over leading doc-comments / attributes. + fn unit_start(n: &tree_sitter::Node) -> u32 { + let mut start = n.start_position().row as u32 + 1; + let mut prev = n.prev_sibling(); + while let Some(p) = prev { + let k = p.kind(); + if k == "line_comment" || k == "block_comment" || k == "attribute_item" { + start = p.start_position().row as u32 + 1; + prev = p.prev_sibling(); + } else { + break; + } + } + start + } + + fn walk( + node: tree_sitter::Node, + src: &str, + mod_path: &[String], + units: &mut Vec<(String, u32, u32)>, + glue: &mut Vec<(usize, u32, u32)>, + ) { + let mut cur = node.walk(); + for child in node.named_children(&mut cur) { + let s = unit_start(&child); + let e = child.end_position().row as u32 + 1; + let prefix = if mod_path.is_empty() { + String::new() + } else { + format!("{}::", mod_path.join("::")) + }; + match child.kind() { + "function_item" | "struct_item" | "enum_item" | "union_item" + | "trait_item" | "type_item" => { + if let Some(name) = node_name(&child, src) { + flush_glue(glue, units); + units.push((format!("{prefix}{name}"), s, e)); + } + } + "macro_definition" => { + if let Some(name) = node_name(&child, src) { + flush_glue(glue, units); + units.push((format!("{prefix}{name}!"), s, e)); + } + } + "impl_item" => { + flush_glue(glue, units); + let ty = child + .child_by_field_name("type") + .map(|c| src[c.start_byte()..c.end_byte()].trim().to_string()); + let tr = child + .child_by_field_name("trait") + .map(|c| src[c.start_byte()..c.end_byte()].trim().to_string()); + let owner = tr.or(ty).unwrap_or_else(|| "".to_string()); + if let Some(body) = child.child_by_field_name("body") { + let mut bc = body.walk(); + for m in body.named_children(&mut bc) { + if m.kind() == "function_item" { + if let Some(mn) = node_name(&m, src) { + let ms = unit_start(&m); + let me = m.end_position().row as u32 + 1; + units.push((format!("{prefix}{owner}::{mn}"), ms, me)); + } + } + } + } + } + "mod_item" => { + if let Some(body) = child.child_by_field_name("body") { + flush_glue(glue, units); + let name = node_name(&child, src).unwrap_or("mod").to_string(); + let mut np = mod_path.to_vec(); + np.push(name); + walk(body, src, &np, units, glue); + } else { + glue.push((1, s, e)); // bare `mod foo;` declaration + } + } + "use_declaration" | "extern_crate_declaration" | "const_item" + | "static_item" | "attribute_item" | "macro_invocation" => { + glue.push((0, s, e)); + } + _ => { /* line_comment / block_comment / unknown: ignore (folded via unit_start) */ } + } + } + flush_glue(glue, units); + } + + fn flush_glue(glue: &mut Vec<(usize, u32, u32)>, units: &mut Vec<(String, u32, u32)>) { + if glue.is_empty() { + return; + } + let s = glue.iter().map(|(_, a, _)| *a).min().unwrap(); + let e = glue.iter().map(|(_, _, b)| *b).max().unwrap(); + let only_mod_decls = glue.iter().all(|(is_mod, _, _)| *is_mod == 1); + let sym = if only_mod_decls { "" } else { "" }; + units.push((sym.to_string(), s, e)); + glue.clear(); + } + + walk(tree.root_node(), source, &[], &mut units, &mut glue); + + let total_lines = lines.len() as u32; + let mut blocks = Vec::with_capacity(units.len()); + for (ordinal, (symbol, ls, le)) in units.into_iter().enumerate() { + let line_start = ls.max(1); + let line_end = le.min(total_lines.max(1)); + let span = SourceSpan::Code { + line_start, + line_end, + symbol: Some(symbol), + lang: Some("rust".to_string()), + }; + let block_id = id_for_block(doc_id, "code", &[], ordinal as u32, &span); + let code = lines[(line_start as usize - 1)..=(line_end as usize - 1)].join("\n"); + blocks.push(Block::Code(CodeBlock { + common: CommonBlock { + block_id, + heading_path: Vec::new(), + source_span: span, + }, + lang: Some("rust".to_string()), + code, + })); + } + Ok(blocks) +} +``` + +Notes for the implementer: +- `flush_glue` ordering: glue flushed *before* pushing a real unit so document order is preserved (glue that precedes the first fn becomes the `` chunk spanning the `use`/`const` region; the `unit_start` doc-comment extension keeps the fn's own doc comment with the fn, not the glue). +- A `glue` flushed after a real unit between two fns is still a valid `` unit (rare; acceptable). +- If `units` is empty (e.g. an empty file) → emit zero blocks (consistent with empty-PDF-page behavior). +- The `e` of a fixture's last `mod inner { … }` etc. is end-of-block; line slicing uses inclusive 1-based. + +- [ ] **Step 5: Wire into lib.rs** + +In `crates/kebab-parse-code/src/lib.rs`: + +```rust +pub mod rust; +pub use rust::{PARSER_VERSION as RUST_PARSER_VERSION, RustAstExtractor}; +``` + +- [ ] **Step 6: Run the tests — expect pass** + +Run: `cargo test -p kebab-parse-code` +Expected: PASS (`extractor_supports_*`, `emits_one_block_per_semantic_unit_with_symbols`, `deterministic_across_runs`). +If symbol names mismatch (tree-sitter-rust grammar field-name drift, e.g. `impl_item` `type` vs `type_arguments`), inspect with a scratch `node.kind()`/`field` dump and adjust the field names; pin behavior with the test. + +- [ ] **Step 7: clippy + commit** + +```bash +cargo clippy -p kebab-parse-code --all-targets -- -D warnings +git add crates/kebab-parse-code/ +git commit -m "feat(p10-1a-2): tree-sitter-rust AST extractor (parser-side) + +Co-Authored-By: Claude Opus 4.7 (1M context) " +``` + +--- + +## Task 7: `code-rust-ast-v1` chunker + +**Files:** +- Create: `crates/kebab-chunk/src/code_rust_ast_v1.rs` +- Modify: `crates/kebab-chunk/src/lib.rs` (`mod` + `pub use`) +- Test: inline `#[cfg(test)] mod tests` in the new file + +The chunker consumes the AST `CanonicalDocument` and maps **1 `Block::Code` → 1 `Chunk`**, except a unit longer than `AST_CHUNK_MAX_LINES` is split into `[part i/N]` sub-chunks. tree-sitter is NOT imported here (forbidden — AST is parser-side). Mirror `crates/kebab-chunk/src/pdf_page_v1.rs` for: `VERSION_LABEL` const, `BYTES_PER_TOKEN = 3`, `POLICY_HASH_HEX_LEN = 16`, `policy_hash` impl (identical blake3 recipe — cross-chunker fingerprint identity is required), per-chunk `policy_hash` variant to avoid `chunk_id` collision on split units, the upfront block-shape validation that `bail!`s on a non-code doc. + +`AST_CHUNK_MAX_LINES` is a module constant (`= 200`) matching `IngestCodeCfg::default().ast_chunk_max_lines`. Threading the config value through the fixed `Chunker` trait needs a per-medium chunker registry — a P+ task; this mirrors the existing `pdf-page-v1` "chunker_version hard-coded" deviation. Record it (Task 11 HOTFIXES). + +- [ ] **Step 1: Write failing tests** + +```rust +#[cfg(test)] +mod tests { + use super::*; + use kebab_core::{ + Block, CanonicalDocument, ChunkPolicy, Chunker, ChunkerVersion, CodeBlock, CommonBlock, + SourceSpan, id_for_block, id_for_doc, AssetId, Lang, Metadata, ParserVersion, Provenance, + SourceType, TrustLevel, WorkspacePath, + }; + use time::OffsetDateTime; + + fn code_doc(units: &[(&str, u32, u32, &str)]) -> CanonicalDocument { + let wp = WorkspacePath("crates/x/src/a.rs".into()); + let aid = AssetId("a".repeat(64)); + let pv = ParserVersion("code-rust-v1".into()); + let doc_id = id_for_doc(&wp, &aid, &pv); + let blocks = units + .iter() + .enumerate() + .map(|(i, (sym, ls, le, code))| { + let span = SourceSpan::Code { + line_start: *ls, + line_end: *le, + symbol: Some((*sym).to_string()), + lang: Some("rust".into()), + }; + let bid = id_for_block(&doc_id, "code", &[], i as u32, &span); + Block::Code(CodeBlock { + common: CommonBlock { block_id: bid, heading_path: vec![], source_span: span }, + lang: Some("rust".into()), + code: (*code).to_string(), + }) + }) + .collect(); + CanonicalDocument { + doc_id, source_asset_id: aid, workspace_path: wp, title: "a".into(), + lang: Lang("und".into()), blocks, + metadata: Metadata { + aliases: vec![], tags: vec![], + created_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(), + updated_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(), + source_type: SourceType::Note, trust_level: TrustLevel::Primary, + user_id_alias: None, user: Default::default(), + repo: Some("kebab".into()), git_branch: Some("main".into()), + git_commit: Some("0".repeat(40)), code_lang: Some("rust".into()), + }, + provenance: Provenance { events: vec![] }, + parser_version: pv, schema_version: 1, doc_version: 1, + last_chunker_version: None, last_embedding_version: None, + } + } + fn policy() -> ChunkPolicy { + ChunkPolicy { target_tokens: 500, overlap_tokens: 80, + respect_markdown_headings: false, + chunker_version: ChunkerVersion(VERSION_LABEL.into()) } + } + + #[test] + fn chunker_version_is_code_rust_ast_v1() { + assert_eq!(CodeRustAstV1Chunker.chunker_version(), + ChunkerVersion("code-rust-ast-v1".into())); + } + + #[test] + fn one_chunk_per_unit_preserves_code_span() { + let doc = code_doc(&[ + ("parse", 1, 3, "pub fn parse() {}\n// x\n}"), + ("Foo::double", 5, 7, "fn double() {}\n//\n}"), + ]); + let chunks = CodeRustAstV1Chunker.chunk(&doc, &policy()).unwrap(); + assert_eq!(chunks.len(), 2); + for c in &chunks { + assert_eq!(c.source_spans.len(), 1); + assert!(matches!(c.source_spans[0], SourceSpan::Code { .. })); + assert_eq!(c.heading_path, Vec::::new()); + assert_eq!(c.chunker_version.0, "code-rust-ast-v1"); + } + match &chunks[0].source_spans[0] { + SourceSpan::Code { symbol, line_start, line_end, .. } => { + assert_eq!(symbol.as_deref(), Some("parse")); + assert_eq!((*line_start, *line_end), (1, 3)); + } + _ => unreachable!(), + } + } + + #[test] + fn oversize_unit_splits_into_parts_with_unique_ids() { + // 500-line fn → must split (AST_CHUNK_MAX_LINES = 200). + let body = (0..500).map(|i| format!(" let x{i} = {i};")).collect::>().join("\n"); + let code = format!("pub fn big() {{\n{body}\n}}"); + let doc = code_doc(&[("big", 1, 502, &code)]); + let chunks = CodeRustAstV1Chunker.chunk(&doc, &policy()).unwrap(); + assert!(chunks.len() >= 2, "oversize unit must split, got {}", chunks.len()); + for c in &chunks { + match &c.source_spans[0] { + SourceSpan::Code { symbol, .. } => { + assert!(symbol.as_deref().unwrap().starts_with("big [part "), + "part-numbered symbol, got {symbol:?}"); + } + _ => unreachable!(), + } + } + let mut ids: Vec<&str> = chunks.iter().map(|c| c.chunk_id.0.as_str()).collect(); + let n = ids.len(); ids.sort(); ids.dedup(); + assert_eq!(ids.len(), n, "chunk_ids unique across split parts"); + } + + #[test] + fn non_code_doc_errors() { + use kebab_core::TextBlock; + let mut doc = code_doc(&[("parse", 1, 1, "fn parse(){}")]); + doc.blocks = vec![Block::Paragraph(TextBlock { + common: CommonBlock { + block_id: kebab_core::BlockId("b".into()), + heading_path: vec![], + source_span: SourceSpan::Line { start: 1, end: 1 }, + }, + text: "x".into(), inlines: vec![], + })]; + let err = CodeRustAstV1Chunker.chunk(&doc, &policy()).unwrap_err(); + assert!(err.to_string().contains("CodeRustAstV1Chunker")); + } + + #[test] + fn deterministic_chunk_ids_1000() { + let doc = code_doc(&[("parse", 1, 2, "fn parse(){}\n}")]); + let base: Vec = CodeRustAstV1Chunker.chunk(&doc, &policy()) + .unwrap().into_iter().map(|c| c.chunk_id.0).collect(); + for _ in 0..1000 { + let again: Vec = CodeRustAstV1Chunker.chunk(&doc, &policy()) + .unwrap().into_iter().map(|c| c.chunk_id.0).collect(); + assert_eq!(again, base); + } + } + + #[test] + fn policy_hash_matches_md_heading_v1() { + let p = policy(); + assert_eq!(CodeRustAstV1Chunker.policy_hash(&p), + crate::MdHeadingV1Chunker.policy_hash(&p)); + } +} +``` + +- [ ] **Step 2: Run — expect fail** + +Run: `cargo test -p kebab-chunk code_rust_ast` +Expected: FAIL — `CodeRustAstV1Chunker` undefined. + +- [ ] **Step 3: Implement the chunker** + +Create `crates/kebab-chunk/src/code_rust_ast_v1.rs`: + +```rust +//! `code-rust-ast-v1` — maps a tree-sitter-derived Rust AST +//! `CanonicalDocument` (one `Block::Code` per semantic unit, each with +//! `SourceSpan::Code`) to chunks 1:1. A unit longer than +//! `AST_CHUNK_MAX_LINES` is split into ` [part i/N]` sub-chunks +//! at blank-line paragraph boundaries (design §9.1 oversize fallback). +//! +//! tree-sitter is intentionally NOT a dependency here: AST work is +//! parser-side (`kebab-parse-code`, design §6.3). This chunker only +//! consumes the `CanonicalDocument`. +//! +//! `AST_CHUNK_MAX_LINES` is a constant matching +//! `IngestCodeCfg::default().ast_chunk_max_lines` (200). Per-medium +//! config threading needs a chunker registry (P+); same deviation +//! pattern as `pdf-page-v1`'s pinned `chunker_version` +//! (`tasks/HOTFIXES.md`). + +use kebab_core::{ + Block, BlockId, CanonicalDocument, Chunk, ChunkPolicy, Chunker, ChunkerVersion, DocumentId, + SourceSpan, id_for_chunk, +}; + +const VERSION_LABEL: &str = "code-rust-ast-v1"; +const BYTES_PER_TOKEN: usize = 3; +const POLICY_HASH_HEX_LEN: usize = 16; +const AST_CHUNK_MAX_LINES: u32 = 200; + +#[derive(Clone, Copy, Debug, Default)] +pub struct CodeRustAstV1Chunker; + +impl Chunker for CodeRustAstV1Chunker { + fn chunker_version(&self) -> ChunkerVersion { + ChunkerVersion(VERSION_LABEL.to_string()) + } + + fn policy_hash(&self, policy: &ChunkPolicy) -> String { + let bytes = serde_json_canonicalizer::to_vec(policy) + .expect("canonical JSON serialization of ChunkPolicy must not fail"); + let hex = blake3::hash(&bytes).to_hex().to_string(); + hex[..POLICY_HASH_HEX_LEN].to_string() + } + + fn chunk( + &self, + doc: &CanonicalDocument, + policy: &ChunkPolicy, + ) -> anyhow::Result> { + for b in &doc.blocks { + let c = match b { + Block::Code(c) => c, + _ => anyhow::bail!( + "CodeRustAstV1Chunker only handles code docs (got non-Code block)" + ), + }; + if !matches!(c.common.source_span, SourceSpan::Code { .. }) { + anyhow::bail!( + "CodeRustAstV1Chunker only handles code docs (got non-Code source_span)" + ); + } + } + + let base_policy_hash = self.policy_hash(policy); + let chunker_version = self.chunker_version(); + let mut out: Vec = Vec::new(); + + for b in &doc.blocks { + let cb = match b { + Block::Code(c) => c, + _ => unreachable!("validated above"), + }; + let (ls, le, symbol, lang) = match &cb.common.source_span { + SourceSpan::Code { line_start, line_end, symbol, lang } => { + (*line_start, *line_end, symbol.clone(), lang.clone()) + } + _ => unreachable!("validated above"), + }; + let block_ids: Vec = vec![cb.common.block_id.clone()]; + let span_lines = le.saturating_sub(ls) + 1; + + if span_lines <= AST_CHUNK_MAX_LINES { + let span = SourceSpan::Code { + line_start: ls, + line_end: le, + symbol: symbol.clone(), + lang: lang.clone(), + }; + out.push(make_chunk( + doc, &chunker_version, &block_ids, &base_policy_hash, + None, span, cb.code.clone(), + )); + } else { + let parts = split_oversize(&cb.code); + let n = parts.len(); + for (i, (off_start, off_end, text)) in parts.into_iter().enumerate() { + let part_ls = ls + off_start; + let part_le = ls + off_end; + let part_sym = symbol + .as_ref() + .map(|s| format!("{s} [part {}/{n}]", i + 1)); + let span = SourceSpan::Code { + line_start: part_ls, + line_end: part_le, + symbol: part_sym, + lang: lang.clone(), + }; + out.push(make_chunk( + doc, &chunker_version, &block_ids, &base_policy_hash, + Some(part_ls), span, text, + )); + } + } + } + + tracing::debug!( + target: "kebab-chunk", + doc_id = %doc.doc_id, + chunks = out.len(), + "code-rust-ast-v1 chunked", + ); + Ok(out) + } +} + +#[allow(clippy::too_many_arguments)] +fn make_chunk( + doc: &CanonicalDocument, + chunker_version: &ChunkerVersion, + block_ids: &[BlockId], + base_policy_hash: &str, + split_key: Option, + span: SourceSpan, + text: String, +) -> Chunk { + // Per-chunk policy_hash variant prevents chunk_id collision when one + // block splits into multiple parts (same block_ids). Mirrors + // pdf-page-v1. Single-chunk units use the base hash unchanged. + let id_hash = match split_key { + Some(k) => format!("{base_policy_hash}#L{k}"), + None => base_policy_hash.to_string(), + }; + let chunk_id = id_for_chunk(&doc.doc_id, chunker_version, block_ids, &id_hash); + let token_estimate = text.len().div_ceil(BYTES_PER_TOKEN); + Chunk { + chunk_id, + doc_id: DocumentId(doc.doc_id.0.clone()), + block_ids: block_ids.to_vec(), + text, + heading_path: Vec::new(), + source_spans: vec![span], + token_estimate, + chunker_version: chunker_version.clone(), + policy_hash: base_policy_hash.to_string(), + } +} + +/// Split an oversize unit at blank-line paragraph boundaries, greedily +/// gluing paragraphs until ~`AST_CHUNK_MAX_LINES` lines accumulate. +/// Returns `(line_offset_start, line_offset_end, text)` where offsets are +/// 0-based within the unit (caller adds the unit's absolute `line_start`). +fn split_oversize(code: &str) -> Vec<(u32, u32, String)> { + let lines: Vec<&str> = code.split('\n').collect(); + let total = lines.len() as u32; + let mut out: Vec<(u32, u32, String)> = Vec::new(); + let mut start: u32 = 0; + while start < total { + let mut end = (start + AST_CHUNK_MAX_LINES).min(total); + // Prefer ending on a blank line within the last 20% of the window + // so we don't cut mid-paragraph when a boundary is nearby. + let floor = start + (AST_CHUNK_MAX_LINES * 4 / 5); + if end < total { + if let Some(b) = (floor.min(end)..end) + .rev() + .find(|&i| lines[i as usize].trim().is_empty()) + { + end = b + 1; + } + } + let text = lines[start as usize..end as usize].join("\n"); + out.push((start, end.saturating_sub(1), text)); + start = end; + } + if out.is_empty() { + out.push((0, total.saturating_sub(1), code.to_string())); + } + out +} +``` + +- [ ] **Step 4: Wire into lib.rs** + +In `crates/kebab-chunk/src/lib.rs` add (matching the existing `mod`/`pub use` style): + +```rust +mod code_rust_ast_v1; +pub use code_rust_ast_v1::CodeRustAstV1Chunker; +``` + +- [ ] **Step 5: Run — expect pass** + +Run: `cargo test -p kebab-chunk code_rust_ast` +Expected: PASS (all 6 tests). + +- [ ] **Step 6: clippy + commit** + +```bash +cargo clippy -p kebab-chunk --all-targets -- -D warnings +git add crates/kebab-chunk/ +git commit -m "feat(p10-1a-2): code-rust-ast-v1 chunker (1:1 + oversize split) + +Co-Authored-By: Claude Opus 4.7 (1M context) " +``` + +--- + +## Task 8: `kebab-app` dispatch — `ingest_one_code_asset` + +**Files:** +- Modify: `crates/kebab-app/Cargo.toml` (add `kebab-parse-code = { path = "../kebab-parse-code" }` if not already a dep) +- Modify: `crates/kebab-app/src/lib.rs` (`ingest_one_asset` match ~line 896; new `ingest_one_code_asset` fn modeled on `ingest_one_pdf_asset` at 1455-end) +- Test: `crates/kebab-app/tests/` — add `code_ingest_smoke.rs` (mirror an existing app integration test's harness) + +- [ ] **Step 1: Check/Add the crate dep** + +```bash +grep -n "kebab-parse-code" crates/kebab-app/Cargo.toml || \ + echo 'kebab-parse-code = { path = "../kebab-parse-code" } # p10-1A-2' \ + >> /dev/stderr +``` + +If absent, add `kebab-parse-code = { path = "../kebab-parse-code" }` under `[dependencies]` in `crates/kebab-app/Cargo.toml` (1A-1 may have added it for the skip helpers — verify). + +- [ ] **Step 2: Write the failing integration test** + +Create `crates/kebab-app/tests/code_ingest_smoke.rs`. Model the harness on an existing app integration test (read one under `crates/kebab-app/tests/` for the `App`/`Config`/TempDir setup pattern). Core assertions: + +```rust +// Build an isolated TempDir KB, drop a tiny .rs file in the workspace, +// run ingest, then assert a search returns a Citation::Code. +#[test] +fn rust_file_ingests_and_searches_as_code_citation() { + // ... TempDir + Config pointing workspace.root at it (copy the + // harness from the sibling integration test verbatim) ... + std::fs::write(workspace_root.join("demo.rs"), + "/// adds\npub fn add(a: i32, b: i32) -> i32 {\n a + b\n}\n").unwrap(); + + let report = kebab_app::ingest_with_config(&cfg, /* args per existing test */).unwrap(); + assert!(report.ingested >= 1, "rust file ingested: {report:?}"); + + let hits = kebab_app::search_with_config(&cfg, "add", /* args */).unwrap(); + let code_hit = hits.iter().find(|h| matches!( + h.citation, kebab_core::Citation::Code { .. })); + let h = code_hit.expect("a Citation::Code hit"); + match &h.citation { + kebab_core::Citation::Code { lang, symbol, line_start, .. } => { + assert_eq!(lang.as_deref(), Some("rust")); + assert_eq!(symbol.as_deref(), Some("add")); + assert!(*line_start >= 1); + } + _ => unreachable!(), + } + assert_eq!(h.code_lang.as_deref(), Some("rust")); +} +``` + +> Use the exact `*_with_config` facade signatures from a sibling test (the facade rule — CLAUDE.md — requires the `_with_config` form). Read one existing `crates/kebab-app/tests/*.rs` to copy the harness; do not invent the Config builder. + +- [ ] **Step 3: Run — expect fail** + +Run: `cargo test -p kebab-app rust_file_ingests_and_searches_as_code_citation` +Expected: FAIL — code asset currently hits the `_ => Skipped` arm in `ingest_one_asset`. + +- [ ] **Step 4: Add the dispatch + `ingest_one_code_asset`** + +In `crates/kebab-app/src/lib.rs`, in the `ingest_one_asset` match (~line 896, where `MediaType::Pdf => { return ingest_one_pdf_asset(...) }`), add before the catch-all `_`: + +```rust + MediaType::Code(lang) if lang == "rust" => { + return ingest_one_code_asset( + app, asset, chunk_policy, embedder, vector_store, + existing_doc_ids, force_reingest, + ); + } + // Non-Rust code langs activate in later phases (1B+); skip for now. +``` + +Add `fn ingest_one_code_asset(...)` modeled **line-for-line on `ingest_one_pdf_asset` (lib.rs:1455-end)** with these substitutions: +- parser_version: `ParserVersion(kebab_parse_code::RUST_PARSER_VERSION.to_string())` +- extractor: `kebab_parse_code::RustAstExtractor::new().extract(&ctx, &bytes).context("kb-parse-code::RustAstExtractor::extract")?` +- chunker: `let chunker = CodeRustAstV1Chunker;` + `chunker.chunk(&canonical, chunk_policy).context("kb-chunk::CodeRustAstV1Chunker::chunk")?` +- `try_skip_unchanged(... &CodeRustAstV1Chunker.chunker_version() ...)` +- `.context("... (code)")` strings instead of `(pdf)` +- import `CodeRustAstV1Chunker` (it's re-exported from `kebab_chunk`) at the top of `lib.rs` alongside the existing `PdfPageV1Chunker` import. + +Everything else (read bytes, ExtractContext, put_asset/document/blocks/chunks, embed branch, IngestItem construction, `kb://` skip) is identical. + +- [ ] **Step 5: Run — expect pass** + +Run: `cargo test -p kebab-app rust_file_ingests_and_searches_as_code_citation` +Expected: PASS. + +- [ ] **Step 6: clippy + commit** + +```bash +cargo clippy -p kebab-app --all-targets -- -D warnings +git add crates/kebab-app/ +git commit -m "feat(p10-1a-2): wire code ingest dispatch (ingest_one_code_asset) + +Co-Authored-By: Claude Opus 4.7 (1M context) " +``` + +--- + +## Task 9: `code_lang_breakdown` in `kebab schema` + +**Files:** +- Modify: `crates/kebab-store-sqlite/src/store.rs` (add a `code_lang_breakdown()` query next to whatever computes `media_breakdown` ~line 608) +- Modify: `crates/kebab-app/src/schema.rs:170` (currently `code_lang_breakdown: BTreeMap::new()`) +- Test: extend the existing `schema.rs` test (`crates/kebab-app/src/schema.rs:196+`) to assert a real count after a code ingest, or a store-level unit test in `kebab-store-sqlite` + +- [ ] **Step 1: Write the failing test** + +In `crates/kebab-store-sqlite` add a unit test that inserts a document with `metadata.code_lang = Some("rust")` and asserts: + +```rust +#[test] +fn code_lang_breakdown_counts_by_code_lang() { + // ... open in-memory/temp store, put_document with code_lang=Some("rust") ... + let bd = store.code_lang_breakdown().unwrap(); + assert_eq!(bd.get("rust"), Some(&1)); +} +``` + +> Read an existing `kebab-store-sqlite` test for the store-setup harness and how documents/metadata are persisted (so the `code_lang` column / json path is correct — `metadata` is stored as JSON; the query likely uses `json_extract(metadata_json, '$.code_lang')`). + +- [ ] **Step 2: Run — expect fail** + +Run: `cargo test -p kebab-store-sqlite code_lang_breakdown_counts` +Expected: FAIL — `code_lang_breakdown` undefined. + +- [ ] **Step 3: Implement the store query** + +In `crates/kebab-store-sqlite/src/store.rs`, mirror the `media_breakdown` query. The exact column depends on how `Metadata` is stored — inspect the `put_document` insert and the existing `media_breakdown` SQL, then add: + +```rust +pub fn code_lang_breakdown(&self) -> anyhow::Result> { + // documents whose metadata JSON has a non-null code_lang + let mut stmt = self.conn.prepare( + "SELECT json_extract(metadata_json, '$.code_lang') AS cl, COUNT(*) \ + FROM documents \ + WHERE cl IS NOT NULL \ + GROUP BY cl", + )?; + let rows = stmt.query_map([], |r| { + Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)? as u32)) + })?; + let mut out = std::collections::BTreeMap::new(); + for row in rows { + let (k, v) = row?; + out.insert(k, v); + } + Ok(out) +} +``` + +> Adjust table/column names (`documents`, `metadata_json`) to the actual schema — grep the `media_breakdown` impl and copy its `FROM`/column conventions exactly. + +- [ ] **Step 4: Populate it in schema.rs** + +In `crates/kebab-app/src/schema.rs`, replace the `code_lang_breakdown: std::collections::BTreeMap::new(),` placeholder (line ~170) with a call to the new store method (mirror how `media_breakdown: counts.media_breakdown` is sourced — likely `app.sqlite.code_lang_breakdown()?`). + +- [ ] **Step 5: Run — expect pass + suite** + +Run: `cargo test -p kebab-store-sqlite code_lang_breakdown_counts` +Expected: PASS. +Run: `cargo test -p kebab-app schema` +Expected: PASS (existing `schema.v1` serialization tests still green; `code_lang_breakdown` now populated). + +- [ ] **Step 6: clippy + commit** + +```bash +cargo clippy -p kebab-store-sqlite -p kebab-app --all-targets -- -D warnings +git add crates/kebab-store-sqlite/ crates/kebab-app/ +git commit -m "feat(p10-1a-2): populate schema.v1 code_lang_breakdown + +Co-Authored-By: Claude Opus 4.7 (1M context) " +``` + +--- + +## Task 10: Full-suite gate + self-ingest snapshot + +**Files:** +- Create: `crates/kebab-chunk/tests/code_rust_ast_snapshot.rs` + fixture `crates/kebab-chunk/tests/fixtures/code-sample.rs` + baseline `code-sample.chunks.snapshot.json` (mirror `crates/kebab-chunk/tests/long_section_snapshot.rs`) + +- [ ] **Step 1: Add a snapshot integration test** + +Mirror `long_section_snapshot.rs` exactly (same `UPDATE_SNAPSHOTS=1` regen mechanism). Build a `CanonicalDocument` by running `kebab_parse_code::RustAstExtractor` on `tests/fixtures/code-sample.rs` (a representative file with a free fn, an impl with 2 methods, a struct, a trait, a top-level `use`/`const` block, and one >200-line fn to exercise the split), chunk with `CodeRustAstV1Chunker`, serialize, compare to baseline. + +> `kebab-chunk` may not depend on `kebab-parse-code` even as a dev-dep if that crosses a boundary — check `tasks/p10/p10-1a-2-rust-ast-chunker.md` Allowed deps. It does not list kebab-chunk→kebab-parse-code. So instead, build the `CanonicalDocument` **by hand** in the test (construct `Block::Code` units directly, like Task 7's `code_doc` helper) rather than invoking the extractor. The snapshot then locks the *chunker's* mapping/splitting, which is the unit under test here. (Extractor behavior is already locked by Task 6's tests.) + +- [ ] **Step 2: Generate the baseline** + +Run: `UPDATE_SNAPSHOTS=1 cargo test -p kebab-chunk code_rust_ast_snapshot` +Then run without the env var: +Run: `cargo test -p kebab-chunk code_rust_ast_snapshot` +Expected: PASS. + +- [ ] **Step 3: Full workspace suite (the real gate)** + +```bash +cargo test --workspace --no-fail-fast -j 1 +``` + +Expected: PASS. Pay attention to: citation/wire round-trip tests (Citation still 6 variants — `Code` from 1A-1 unchanged), any golden eval fixtures, asset/MediaType serialization tests. Fix faithfully; a regression here means an earlier task's match arm was wrong. + +```bash +cargo clippy --workspace --all-targets -- -D warnings +``` + +Expected: clean. + +- [ ] **Step 4: Manual smoke (SMOKE.md flow, isolated TempDir)** + +```bash +cargo build --release +rm -rf /tmp/kebab-p10smoke && mkdir -p /tmp/kebab-p10smoke/ws +cp crates/kebab-chunk/src/code_rust_ast_v1.rs /tmp/kebab-p10smoke/ws/ +cat > /tmp/kebab-p10smoke/config.toml <<'EOF' +[workspace] +root = "/tmp/kebab-p10smoke/ws" +[paths] +data_dir = "/tmp/kebab-p10smoke/data" +EOF +# (match the SMOKE.md config skeleton if these keys differ — read docs/SMOKE.md) +./target/release/kebab --config /tmp/kebab-p10smoke/config.toml ingest --json +./target/release/kebab --config /tmp/kebab-p10smoke/config.toml search "chunk" --json | head +./target/release/kebab --config /tmp/kebab-p10smoke/config.toml schema --json | grep code_lang +``` + +Expected: ingest report shows ≥1 ingested; a search hit with `"citation":{"kind":"code",...,"lang":"rust"}` and top-level `"code_lang":"rust"`, `"repo":...`; `schema --json` `code_lang_breakdown` has `"rust"`. + +- [ ] **Step 5: Commit** + +```bash +git add crates/kebab-chunk/tests/ +git commit -m "test(p10-1a-2): code-rust-ast-v1 chunker snapshot + full-suite gate + +Co-Authored-By: Claude Opus 4.7 (1M context) " +``` + +--- + +## Task 11: Docs, version bump, status flips + +**Files:** +- `README.md` — 지원 형식 / 명령 table: note Rust code ingest is now active (per CLAUDE.md README rule — a new media surface). Mermaid: add `code` source crossing the boundary only if a media type newly crosses it (it does — add a code-ingest edge to the existing diagram). +- `HANDOFF.md` — P10 phase row: note 1A-2 merged (Rust code ingest active, kebab self-dogfooding possible); add a one-line entry under 머지 후 결정 if a HOTFIXES item lands. +- `docs/ARCHITECTURE.md` — add the `kebab-app → kebab-parse-code` edge + `kebab-parse-code → tree-sitter/tree-sitter-rust` to the dependency graph; add the locked-in decision "tree-sitter lives parser-side, not chunker-side (design §6.3)" to the decisions table. +- `docs/SMOKE.md` — add the Rust code-ingest smoke steps (the Task 10 Step 4 flow), and the `[ingest.code]` config keys if not already documented. +- `tasks/INDEX.md` line ~143 + `tasks/p10/INDEX.md` row 1A-2 — flip 1A-1 to ✅ and 1A-2 to ✅ (on merge). +- `tasks/HOTFIXES.md` — add a dated entry for the `AST_CHUNK_MAX_LINES` constant-vs-config deviation (chunker can't see `IngestCodeCfg.ast_chunk_max_lines` through the fixed `Chunker` trait; pinned to the default 200; per-medium chunker registry is P+). Cross-link one line in `tasks/p10/p10-1a-2-rust-ast-chunker.md` Risks/notes. +- `docs/superpowers/specs/2026-04-27-kebab-final-form-design.md` §10.1 — record the 1A-2 surface per design §10.1 (already partly done for SourceSpan/MediaType in Tasks 2/4; ensure §10.1 mentions code ingest activation). +- `Cargo.toml` — workspace `version` minor bump (design §10.4: 1A-2 merge = 도그푸딩 가능 = bump trigger). e.g. `0.6.x` → `0.7.0` (check current value first). + +- [ ] **Step 1: Make all doc edits above.** Keep README narrow (usage only, one Mermaid). HANDOFF gets phase status. ARCHITECTURE gets the graph/decision. + +- [ ] **Step 2: Version bump** + +```bash +grep -m1 '^version' Cargo.toml # read current workspace version +# bump minor in Cargo.toml [workspace.package].version +cargo build --release # refresh Cargo.lock + binary identity +``` + +- [ ] **Step 3: Full suite once more (docs/version shouldn't break it, but the lock changed)** + +```bash +cargo test --workspace --no-fail-fast -j 1 +cargo clippy --workspace --all-targets -- -D warnings +``` + +Expected: PASS / clean. + +- [ ] **Step 4: Commit (bump = release commit, same commit per CLAUDE.md)** + +```bash +git add -A +git commit -m "docs(p10-1a-2): README/HANDOFF/ARCHITECTURE/SMOKE + HOTFIXES + status; chore: bump version + +Co-Authored-By: Claude Opus 4.7 (1M context) " +``` + +--- + +## Finalize: PR + review loop + release + +Per `tasks/HOTFIXES.md` workflow memory and CLAUDE.md Remote section (Gitea, not gh): + +- [ ] Use the **gitea-ops** skill: `gitea-pr` to open the PR (feature branch → main). Title: `feat(p10-1a-2): Rust AST chunker — tree-sitter-rust code ingest active`. Body summarizes: SourceSpan::Code internal variant, parser-side tree-sitter (design §6.3), code-rust-ast-v1 chunker + oversize split, MediaType::Code, schema code_lang_breakdown, frozen design §3.4/§10.1 sync, version bump. +- [ ] **Review loop mode** (do not ask single-shot): `gitea-pr-status --wait-ci` → `gitea-pr-diff` → analyze → `gitea-pr-review` (REQUEST_CHANGES/APPROVE) each round; actionable comments → follow-up commits; converge to APPROVE. +- [ ] On APPROVE: merge immediately (no asking), sync local `main`, delete `feat/p10-1a-2-rust-ast-chunker`. +- [ ] After merge: `cargo clean` (CLAUDE.md routine). Cut release via gitea-ops `gitea-release v` (release notes: Rust code ingest active, `Citation::Code` now populated, `MediaType::Code`, `schema.v1 code_lang_breakdown`, internal `SourceSpan::Code`). +- [ ] Flip `tasks/INDEX.md` / `tasks/p10/INDEX.md` 1A-2 → ✅ if not already in the merged commit; update memory phase-priorities note if the next-task priority shifts (P10-1B vs other). + +--- + +## Self-Review (completed by plan author) + +- **Spec coverage:** design §1A-2 (Rust chunker + tree-sitter-rust + activation) → Tasks 6/7/8; §3.3 (`code-rust-ast-v1`) → Task 7; §3.4 symbol path → Task 6 walk rules + Task 2 SourceSpan; §6.1 (rust.rs parser) → Task 6; §6.2 (kebab-chunk module) → Task 7; §6.3 dep graph (tree-sitter parser-side) → Task 1 + Task 7 forbidden-dep note; §9.1 Tier-1 + oversize fallback → Task 7 `split_oversize`; §10.4 version bump → Task 11; wire (no v2 — Citation::Code from 1A-1) → Task 10 Step 3 gate. Citation routing gap (1A-1 left unwired) → Tasks 2+3. `MediaType::Code` + routing → Tasks 4+5. schema breakdown → Task 9. Docs cascade → Task 11. +- **Placeholder scan:** novel logic (SourceSpan variant, citation arm, AST walk, chunker, split) is given in full. Mechanical mirrors (extractor scaffold, `ingest_one_code_asset`, store breakdown query, integration-test harness) are pinned to an exact existing file:line to copy with enumerated deltas — the established-pattern path the writing-plans skill endorses, not "TBD". +- **Type consistency:** `RustAstExtractor` / `RUST_PARSER_VERSION` / `CodeRustAstV1Chunker` / `VERSION_LABEL="code-rust-ast-v1"` / `SourceSpan::Code{line_start,line_end,symbol,lang}` / `Citation::Code` (1A-1 shape) used consistently across Tasks 2/3/6/7/8. `id_for_block(doc,"code",&[],ordinal,&span)` and `id_for_chunk(doc,cv,block_ids,hash)` match `crates/kebab-core/src/ids.rs:146,163`. diff --git a/tasks/p10/p10-1a-2-rust-ast-chunker.md b/tasks/p10/p10-1a-2-rust-ast-chunker.md new file mode 100644 index 0000000..979fa2b --- /dev/null +++ b/tasks/p10/p10-1a-2-rust-ast-chunker.md @@ -0,0 +1,47 @@ +# p10-1A-2 — Rust AST chunker + +**Status:** 🟡 진행 중 +**Contract sections:** §3.3 (chunker_version `code-rust-ast-v1`), §3.4 (symbol path — Rust convention), §3.4 frozen-design (`SourceSpan::Code` 신규 internal variant), §5 (code ingest 활성화), §6.1 (`kebab-parse-code/src/rust.rs` — tree-sitter-rust → CanonicalDocument), §6.2 (`kebab-chunk/src/code_rust_ast_v1.rs`), §9.1 (Tier 1 AST per-language + oversize fallback). +**Design:** [2026-05-15-kebab-code-ingest-design.md](../../docs/superpowers/specs/2026-05-15-kebab-code-ingest-design.md) §1A-2. +**Plan:** [2026-05-19-p10-1a-2-rust-ast-chunker.md](../../docs/superpowers/plans/2026-05-19-p10-1a-2-rust-ast-chunker.md). + +## Goal + +1A-1 의 프레임워크 위에 **Rust AST chunker 자체**를 올린다. `tree-sitter` + `tree-sitter-rust` 도입, `kebab-parse-code/src/rust.rs` (tree-sitter-rust → `CanonicalDocument`, AST 의미 단위마다 `Block::Code` + `SourceSpan::Code`), `kebab-chunk/src/code_rust_ast_v1.rs` (1 block → 1 chunk + oversize fallback split), `MediaType::Code` 신설, `kebab-app` dispatch. 머지 시점에 kebab 자기 자신 dogfooding 가능. + +## 동결된 설계 결정 (이 task 로 확정) + +- **tree-sitter 위치 = parser (`kebab-parse-code`)**, chunker 아님. design §6.3 의존성 그래프 (`kebab-parse-code → tree-sitter, tree-sitter-rust`) 가 authoritative. PDF 선례와 동형 — parser 가 구조화된 block 생성, chunker 가 매핑. §9.1 의 "chunker 가 AST" 서술은 *oversize fallback split* 만 chunker-side 라는 의미로 해석. +- **`SourceSpan::Code { line_start, line_end, symbol, lang }` 내부 variant 신설** (kebab-core). chunk 의 `source_spans_json` (chunks 테이블) 은 *내부 저장*이라 wire schema 아님 → wire major bump 불필요. `Citation::Code` (wire) 는 1A-1 에서 이미 추가됨. `citation_helper::citation_from_first_span` 에 `SourceSpan::Code → Citation::Code` arm 추가로 symbol/lang 이 자연스럽게 흐름. +- **`MediaType::Code(String)` 신설** — String = canonical code_lang (1A 는 `"rust"` 만 실제 처리, 그 외 인식된 code lang 은 `Skipped` — Tier 2/3 는 후속 phase). +- frozen design §3.4 의 `SourceSpan` enum 및 (해당 시) `MediaType` enum 목록을 같은 PR 에서 갱신. 본 task spec 은 머지 후 frozen. + +## Acceptance criteria + +- `cargo test --workspace --no-fail-fast -j 1` passes. +- 기존 markdown / PDF / image corpus regression test 무영향 (citation 5→6 variant: `Citation::Code` 는 1A-1 에 이미 존재; 기존 5 variant 직렬화 불변). +- `cargo clippy --workspace --all-targets -- -D warnings` passes. +- Rust fixture 한 개 (fn / impl method / struct / trait / top-level use + 200줄 초과 fn) ingest → chunk snapshot 안정 + `Citation::Code` 의 symbol/line 이 spec §3.4 Rust convention 과 일치. +- kebab 자기 crate 한 개를 isolated TempDir KB 에 ingest → `kebab search --json` 결과가 `citation.kind == "code"`, `repo`, `code_lang == "rust"` 반환 (SMOKE 절차). +- `docs/superpowers/specs/2026-04-27-kebab-final-form-design.md` §3.4 (SourceSpan / MediaType) + §10.1 갱신. +- README + HANDOFF + ARCHITECTURE + SMOKE + tasks/INDEX.md + tasks/p10/INDEX.md 갱신. +- workspace `Cargo.toml` version minor bump (도그푸딩 가능 = bump 트리거, design §10.4) + release cut. + +## Allowed dependencies + +- `kebab-parse-code` 에 `tree-sitter` + `tree-sitter-rust` 추가 (workspace deps 경유). 기존 `kebab-core` / `anyhow` / `gix` 유지. +- `kebab-chunk` 는 `kebab-core` 만 (chunker 는 `CanonicalDocument` 만 소비 — tree-sitter import 금지). +- `kebab-app → kebab-parse-code` (facade 가 Extractor 호출). + +## Forbidden dependencies + +- `kebab-chunk` 가 `tree-sitter*` import 금지 (AST 는 parser-side). +- UI crate (cli / mcp / tui) 가 `kebab-parse-code` 직접 import 금지 — `kebab-app` facade 만. +- `kebab-parse-code` 가 store / embed / llm / rag import 금지 (design §8 inheritance). + +## Risks / notes + +- tree-sitter-rust 의 grammar 버전에 따라 node kind 명칭 차이 가능 — `function_item` / `impl_item` / `struct_item` / `enum_item` / `trait_item` / `mod_item` / `use_declaration` 는 도입 버전으로 pin 후 테스트로 고정. +- `SourceSpan::Code` 추가로 `SourceSpan` 의 모든 exhaustive match (citation_helper, store-sqlite serde, search) 가 영향 — 컴파일러가 non-exhaustive 를 잡아주므로 전수 대응. +- oversize fallback (단일 fn > `ast_chunk_max_lines`) 의 `symbol [part i/N]` 표기는 1A-2 chunker 내부 한정. 일반 Tier-3 `code-text-paragraph-v1` 은 Phase 3. +- 머지 후 동작 deviation 은 `tasks/HOTFIXES.md` 에 dated 로그 + 본 spec `Risks / notes` 에 one-line cross-link. -- 2.49.1 From 5c265bb59f39b5544b06e0e88bac9bd8b341a399 Mon Sep 17 00:00:00 2001 From: altair823 Date: Tue, 19 May 2026 15:38:19 +0000 Subject: [PATCH 02/19] build(p10-1a-2): add tree-sitter + tree-sitter-rust workspace deps Co-Authored-By: Claude Opus 4.7 (1M context) --- Cargo.lock | 39 ++++++++++++++++++++++++++++++ Cargo.toml | 4 +++ crates/kebab-parse-code/Cargo.toml | 6 +++-- 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f330c3b..37c7e24 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4346,6 +4346,8 @@ dependencies = [ "anyhow", "gix", "tempfile", + "tree-sitter", + "tree-sitter-rust", ] [[package]] @@ -7367,6 +7369,7 @@ version = "1.0.149" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" dependencies = [ + "indexmap 2.14.0", "itoa", "memchr", "serde", @@ -7731,6 +7734,12 @@ version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e51f1e89f093f99e7432c491c382b88a6860a5adbe6bf02574bf0a08efff1978" +[[package]] +name = "streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520" + [[package]] name = "strsim" version = "0.11.1" @@ -8495,6 +8504,36 @@ dependencies = [ "tracing-serde", ] +[[package]] +name = "tree-sitter" +version = "0.26.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "887bd495d0582c5e3e0d8ece2233666169fa56a9644d172fc22ad179ab2d0538" +dependencies = [ + "cc", + "regex", + "regex-syntax", + "serde_json", + "streaming-iterator", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-language" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "009994f150cc0cd50ff54917d5bc8bffe8cad10ca10d81c34da2ec421ae61782" + +[[package]] +name = "tree-sitter-rust" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439e577dbe07423ec2582ac62c7531120dbfccfa6e5f92406f93dd271a120e45" +dependencies = [ + "cc", + "tree-sitter-language", +] + [[package]] name = "try-lock" version = "0.2.5" diff --git a/Cargo.toml b/Cargo.toml index 7b2527c..5d0ef62 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -86,6 +86,10 @@ base64 = "0.22" # No `git` binary required. Default features include thread-safety + most # object-reading capabilities needed for HEAD name + commit SHA queries. gix = { version = "0.70", default-features = false, features = ["revision"] } +# Rust source parsing for code ingest (kebab-parse-code, p10-1A-2). The +# chunker stays tree-sitter-free — AST work is parser-side per design §6.3. +tree-sitter = "0.26" +tree-sitter-rust = "0.24" # Disk-footprint trim for dev / test builds. Codegen, opt-level, and # behavior are unchanged — only DWARF debug info is reduced (line diff --git a/crates/kebab-parse-code/Cargo.toml b/crates/kebab-parse-code/Cargo.toml index ad6aef5..482ae12 100644 --- a/crates/kebab-parse-code/Cargo.toml +++ b/crates/kebab-parse-code/Cargo.toml @@ -8,8 +8,10 @@ repository = { workspace = true } description = "Language-aware code parsing infrastructure (lang dispatch, .git/ detect, skip helpers) for the kebab pipeline (P10-1A-1)" [dependencies] -anyhow = { workspace = true } -gix = { workspace = true } +anyhow = { workspace = true } +gix = { workspace = true } +tree-sitter = { workspace = true } +tree-sitter-rust = { workspace = true } [dev-dependencies] tempfile = { workspace = true } -- 2.49.1 From 9f3edb7e243d6108151ce9a6e334e60a8e64bb0a Mon Sep 17 00:00:00 2001 From: altair823 Date: Tue, 19 May 2026 15:52:01 +0000 Subject: [PATCH 03/19] =?UTF-8?q?feat(p10-1a-2):=20add=20internal=20Source?= =?UTF-8?q?Span::Code=20variant=20+=20design=20=C2=A73.4=20sync?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/kebab-core/src/document.rs | 30 +++++++++++++++++++ crates/kebab-search/src/citation_helper.rs | 7 +++++ crates/kebab-tui/src/inspect.rs | 9 ++++++ .../2026-04-27-kebab-final-form-design.md | 1 + 4 files changed, 47 insertions(+) diff --git a/crates/kebab-core/src/document.rs b/crates/kebab-core/src/document.rs index 4fdda02..643875b 100644 --- a/crates/kebab-core/src/document.rs +++ b/crates/kebab-core/src/document.rs @@ -142,6 +142,18 @@ pub enum SourceSpan { start_ms: u64, end_ms: u64, }, + /// p10-1A-2: AST-unit span for code ingest. Internal storage shape + /// (chunks.source_spans_json) — `citation_helper` maps this to the + /// wire `Citation::Code` (added 1A-1). `symbol` is the per-language + /// self-reference path (design §3.4); `` / `` for + /// glue regions, never null for an identified unit. `lang` is the + /// canonical code_lang. + Code { + line_start: u32, + line_end: u32, + symbol: Option, + lang: Option, + }, } // ── Forward-declared stubs (§3.7a). Bodies are final per design. ──────── @@ -195,6 +207,24 @@ mod tests { /// previously failed at serde runtime because `tag = "kind"` cannot /// describe a newtype carrying a non-struct value. The struct-variant /// shape used here is the §9 schema migration. + #[test] + fn source_span_code_round_trips_and_tags_lowercase() { + let s = SourceSpan::Code { + line_start: 10, + line_end: 42, + symbol: Some("foo::Bar::baz".to_string()), + lang: Some("rust".to_string()), + }; + let v = serde_json::to_value(&s).unwrap(); + assert_eq!(v["kind"], "code"); + assert_eq!(v["line_start"], 10); + assert_eq!(v["line_end"], 42); + assert_eq!(v["symbol"], "foo::Bar::baz"); + assert_eq!(v["lang"], "rust"); + let back: SourceSpan = serde_json::from_value(v).unwrap(); + assert_eq!(back, s); + } + #[test] fn inline_serde_round_trip() { let cases = vec![ diff --git a/crates/kebab-search/src/citation_helper.rs b/crates/kebab-search/src/citation_helper.rs index 3001640..e896731 100644 --- a/crates/kebab-search/src/citation_helper.rs +++ b/crates/kebab-search/src/citation_helper.rs @@ -49,6 +49,13 @@ pub(crate) fn citation_from_first_span( end_ms: *end_ms, speaker: None, }, + // TODO(p10-1a-2 Task 3): map to Citation::Code + Some(SourceSpan::Code { .. }) => Citation::Line { + path, + start: 1, + end: 1, + section, + }, // Byte-spans don't have a Citation variant. Fall back to a // Line citation pointing at the document head — better than // fabricating a position. Spans-empty falls into the same diff --git a/crates/kebab-tui/src/inspect.rs b/crates/kebab-tui/src/inspect.rs index c6bc13c..2910b1b 100644 --- a/crates/kebab-tui/src/inspect.rs +++ b/crates/kebab-tui/src/inspect.rs @@ -455,6 +455,15 @@ fn describe_span(span: &kebab_core::SourceSpan) -> String { SourceSpan::Time { start_ms, end_ms } => { format!("Time {start_ms}-{end_ms} ms") } + SourceSpan::Code { + line_start, + line_end, + symbol, + .. + } => match symbol { + Some(sym) => format!("Code {line_start}-{line_end} ({sym})"), + None => format!("Code {line_start}-{line_end}"), + }, } } diff --git a/docs/superpowers/specs/2026-04-27-kebab-final-form-design.md b/docs/superpowers/specs/2026-04-27-kebab-final-form-design.md index d883d38..baa349b 100644 --- a/docs/superpowers/specs/2026-04-27-kebab-final-form-design.md +++ b/docs/superpowers/specs/2026-04-27-kebab-final-form-design.md @@ -565,6 +565,7 @@ pub enum SourceSpan { Page { page: u32, char_start: Option, char_end: Option }, Region { x: u32, y: u32, w: u32, h: u32 }, Time { start_ms: u64, end_ms: u64 }, + Code { line_start: u32, line_end: u32, symbol: Option, lang: Option }, // p10-1A-2: internal code-unit span (see tasks/p10/p10-1a-2) } ``` -- 2.49.1 From 42712b50c2746a39ec732d6dee8a219e2ebd0aa6 Mon Sep 17 00:00:00 2001 From: altair823 Date: Tue, 19 May 2026 15:59:03 +0000 Subject: [PATCH 04/19] feat(p10-1a-2): map SourceSpan::Code -> Citation::Code in citation_helper Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/kebab-search/src/citation_helper.rs | 50 +++++++++++++++++++--- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/crates/kebab-search/src/citation_helper.rs b/crates/kebab-search/src/citation_helper.rs index e896731..8771468 100644 --- a/crates/kebab-search/src/citation_helper.rs +++ b/crates/kebab-search/src/citation_helper.rs @@ -49,12 +49,12 @@ pub(crate) fn citation_from_first_span( end_ms: *end_ms, speaker: None, }, - // TODO(p10-1a-2 Task 3): map to Citation::Code - Some(SourceSpan::Code { .. }) => Citation::Line { + Some(SourceSpan::Code { line_start, line_end, symbol, lang }) => Citation::Code { path, - start: 1, - end: 1, - section, + line_start: *line_start, + line_end: *line_end, + symbol: symbol.clone(), + lang: lang.clone(), }, // Byte-spans don't have a Citation variant. Fall back to a // Line citation pointing at the document head — better than @@ -79,3 +79,43 @@ pub(crate) fn citation_from_first_span( } } } + +#[cfg(test)] +mod tests { + use kebab_core::{Citation, SourceSpan, WorkspacePath}; + + #[test] + fn build_citation_code_maps_symbol_and_lang() { + let span = SourceSpan::Code { + line_start: 5, + line_end: 30, + symbol: Some("chunk::md_heading_v1::MdHeadingV1Chunker::chunk".into()), + lang: Some("rust".into()), + }; + let c = super::citation_from_first_span( + "c1", + WorkspacePath::new("crates/kebab-chunk/src/md_heading_v1.rs".to_string()).unwrap(), + None, + Some(&span), + ); + match c { + Citation::Code { + path, + line_start, + line_end, + symbol, + lang, + } => { + assert_eq!(path.0, "crates/kebab-chunk/src/md_heading_v1.rs"); + assert_eq!(line_start, 5); + assert_eq!(line_end, 30); + assert_eq!( + symbol.as_deref(), + Some("chunk::md_heading_v1::MdHeadingV1Chunker::chunk") + ); + assert_eq!(lang.as_deref(), Some("rust")); + } + other => panic!("expected Citation::Code, got {other:?}"), + } + } +} -- 2.49.1 From 7a6a24ad1026d61107ddbfc4aba94e76b34e7e5c Mon Sep 17 00:00:00 2001 From: altair823 Date: Tue, 19 May 2026 17:14:45 +0000 Subject: [PATCH 05/19] feat(p10-1a-2): add MediaType::Code(lang) variant MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TDD: red → green cycle confirmed. New `Code(String)` variant serializes as `{"code":"rust"}` via serde `rename_all = "lowercase"`. All exhaustive `match` sites updated (`media_label`, `ingest_one_asset` catch-all → explicit or-pattern). Design §3.5 enum listing synced. Also fix `/target` symlink gitignore pattern so integration-test binary lookup via workspace-relative path works with CARGO_TARGET_DIR redirect. Co-Authored-By: Claude Opus 4.7 (1M context) --- .gitignore | 1 + crates/kebab-app/src/ingest_progress.rs | 2 ++ crates/kebab-app/src/lib.rs | 3 ++- crates/kebab-core/src/media.rs | 18 ++++++++++++++++++ .../2026-04-27-kebab-final-form-design.md | 1 + 5 files changed, 24 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 76a3d1e..8c015e5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ .superpowers/ .worktrees/ .claude/ +/target /target/ **/*.rs.bk Cargo.lock.bak diff --git a/crates/kebab-app/src/ingest_progress.rs b/crates/kebab-app/src/ingest_progress.rs index ef1df15..b3255f6 100644 --- a/crates/kebab-app/src/ingest_progress.rs +++ b/crates/kebab-app/src/ingest_progress.rs @@ -96,6 +96,7 @@ pub fn media_label(media: &kebab_core::MediaType) -> &'static str { kebab_core::MediaType::Pdf => "pdf", kebab_core::MediaType::Image(_) => "image", kebab_core::MediaType::Audio(_) => "audio", + kebab_core::MediaType::Code(_) => "code", kebab_core::MediaType::Other(_) => "other", } } @@ -148,6 +149,7 @@ mod tests { media_label(&MediaType::Audio(kebab_core::AudioType::Wav)), "audio" ); + assert_eq!(media_label(&MediaType::Code("rust".into())), "code"); assert_eq!(media_label(&MediaType::Other("x".into())), "other"); } diff --git a/crates/kebab-app/src/lib.rs b/crates/kebab-app/src/lib.rs index e0a0375..850672d 100644 --- a/crates/kebab-app/src/lib.rs +++ b/crates/kebab-app/src/lib.rs @@ -917,7 +917,8 @@ fn ingest_one_asset( force_reingest, ); } - _ => { + // p10-1A-2: Code dispatch wired in Task 8; skip until then. + MediaType::Code(_) | MediaType::Audio(_) | MediaType::Other(_) => { return Ok(kebab_core::IngestItem { kind: kebab_core::IngestItemKind::Skipped, doc_id: None, diff --git a/crates/kebab-core/src/media.rs b/crates/kebab-core/src/media.rs index 263e5cf..557bf0c 100644 --- a/crates/kebab-core/src/media.rs +++ b/crates/kebab-core/src/media.rs @@ -40,5 +40,23 @@ pub enum MediaType { Pdf, Image(ImageType), Audio(AudioType), + /// p10-1A-2: a source-code file. Inner string is the canonical + /// code_lang (design §3.5). 1A activates `"rust"` only; other + /// recognized code langs are still routed `Other` until their phase. + Code(String), Other(String), } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn media_type_code_serializes_lowercase_tagged() { + let m = MediaType::Code("rust".to_string()); + let v = serde_json::to_value(&m).unwrap(); + assert_eq!(v, serde_json::json!({ "code": "rust" })); + let back: MediaType = serde_json::from_value(v).unwrap(); + assert_eq!(back, m); + } +} diff --git a/docs/superpowers/specs/2026-04-27-kebab-final-form-design.md b/docs/superpowers/specs/2026-04-27-kebab-final-form-design.md index baa349b..8d970b9 100644 --- a/docs/superpowers/specs/2026-04-27-kebab-final-form-design.md +++ b/docs/superpowers/specs/2026-04-27-kebab-final-form-design.md @@ -488,6 +488,7 @@ pub enum MediaType { Pdf, Image(ImageType), Audio(AudioType), + Code(String), // p10-1A-2: source-code file; inner = canonical code_lang (e.g. "rust") Other(String), } -- 2.49.1 From a531dc37dc211629dfd991358db46533ddd5d53e Mon Sep 17 00:00:00 2001 From: altair823 Date: Tue, 19 May 2026 17:17:36 +0000 Subject: [PATCH 06/19] feat(p10-1a-2): route .rs files to MediaType::Code(rust) Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/kebab-source-fs/src/media.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/crates/kebab-source-fs/src/media.rs b/crates/kebab-source-fs/src/media.rs index b2d8663..68db875 100644 --- a/crates/kebab-source-fs/src/media.rs +++ b/crates/kebab-source-fs/src/media.rs @@ -34,6 +34,10 @@ pub(crate) fn media_type_for(path: &Path) -> MediaType { "flac" => MediaType::Audio(AudioType::Flac), "ogg" => MediaType::Audio(AudioType::Ogg), + // p10-1A-2: Rust is the only code lang activated in 1A. Other + // recognized code langs stay Other until their phase (1B+). + "rs" => MediaType::Code("rust".to_string()), + // Empty string (no extension) and any other extension: bucket as // Other and let downstream extractors decide if they support it. _ => MediaType::Other(ext), @@ -71,6 +75,17 @@ mod tests { ); } + #[test] + fn rust_files_map_to_media_code_rust() { + assert_eq!( + media_type_for(Path::new("crates/kebab-core/src/lib.rs")), + MediaType::Code("rust".to_string()) + ); + // non-Rust code extensions stay Other in 1A + assert_eq!(media_type_for(Path::new("a/b.py")), MediaType::Other("py".to_string())); + assert_eq!(media_type_for(Path::new("Cargo.toml")), MediaType::Other("toml".to_string())); + } + #[test] fn unknown_and_missing_extension() { assert_eq!( -- 2.49.1 From 402a4506a23b6af3e5344dd60551de24d1bc0a43 Mon Sep 17 00:00:00 2001 From: altair823 Date: Tue, 19 May 2026 17:22:09 +0000 Subject: [PATCH 07/19] feat(p10-1a-2): tree-sitter-rust AST extractor (parser-side) Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/kebab-parse-code/Cargo.toml | 4 + crates/kebab-parse-code/src/lib.rs | 2 + crates/kebab-parse-code/src/rust.rs | 404 ++++++++++++++++++ .../kebab-parse-code/tests/fixtures/sample.rs | 35 ++ 4 files changed, 445 insertions(+) create mode 100644 crates/kebab-parse-code/src/rust.rs create mode 100644 crates/kebab-parse-code/tests/fixtures/sample.rs diff --git a/crates/kebab-parse-code/Cargo.toml b/crates/kebab-parse-code/Cargo.toml index 482ae12..3953ed5 100644 --- a/crates/kebab-parse-code/Cargo.toml +++ b/crates/kebab-parse-code/Cargo.toml @@ -8,8 +8,12 @@ repository = { workspace = true } description = "Language-aware code parsing infrastructure (lang dispatch, .git/ detect, skip helpers) for the kebab pipeline (P10-1A-1)" [dependencies] +kebab-core = { path = "../kebab-core" } anyhow = { workspace = true } gix = { workspace = true } +serde_json = { workspace = true } +time = { workspace = true } +tracing = { workspace = true } tree-sitter = { workspace = true } tree-sitter-rust = { workspace = true } diff --git a/crates/kebab-parse-code/src/lib.rs b/crates/kebab-parse-code/src/lib.rs index 3b699c1..d9faefb 100644 --- a/crates/kebab-parse-code/src/lib.rs +++ b/crates/kebab-parse-code/src/lib.rs @@ -15,8 +15,10 @@ pub mod lang; pub mod repo; +pub mod rust; pub mod skip; pub use lang::code_lang_for_path; pub use repo::{RepoMeta, detect_repo}; +pub use rust::{PARSER_VERSION as RUST_PARSER_VERSION, RustAstExtractor}; pub use skip::{BUILTIN_BLACKLIST, is_generated_file, is_oversized}; diff --git a/crates/kebab-parse-code/src/rust.rs b/crates/kebab-parse-code/src/rust.rs new file mode 100644 index 0000000..4b15ce2 --- /dev/null +++ b/crates/kebab-parse-code/src/rust.rs @@ -0,0 +1,404 @@ +//! `kebab-parse-code::rust` — tree-sitter Rust AST extractor (P10-1A-2). +//! +//! Implements [`kebab_core::Extractor`] for [`MediaType::Code("rust")`]. +//! Walks the tree-sitter parse tree and emits one [`Block::Code`] per +//! top-level AST semantic unit (free fn, type, trait, macro, each impl +//! method, recursively per module), each carrying [`SourceSpan::Code`] +//! with the unit's self-reference symbol path (design §3.4). Glue +//! declarations (`use` / `const` / `static` / bodyless `mod` / top-level +//! attributes / macro invocations) collapse into one grouped +//! `` (or ``) unit. +//! +//! Doc comments and attributes immediately preceding an item are folded +//! into that item's line range (design §9.1 "선언 + doc comment"). +//! +//! Scope is intentionally narrow: AST unit extraction + symbol paths + +//! line ranges for Rust. The `CanonicalDocument` scaffold mirrors +//! `kebab-parse-pdf`. Per design §3.4 / §9.1 / §9 versioning. + +use anyhow::Result; +use kebab_core::{ + Block, CanonicalDocument, CodeBlock, CommonBlock, Extractor, Lang, MediaType, Metadata, + ParserVersion, Provenance, ProvenanceEvent, ProvenanceKind, SourceSpan, SourceType, TrustLevel, + id_for_block, id_for_doc, +}; +use serde_json::Map; +use time::OffsetDateTime; + +pub const PARSER_VERSION: &str = "code-rust-v1"; + +/// Rust AST extractor. Per-unit blocks via tree-sitter-rust 0.24 +/// (`LANGUAGE: LanguageFn`) parsed by tree-sitter 0.26. +pub struct RustAstExtractor; + +impl RustAstExtractor { + pub fn new() -> Self { + Self + } +} + +impl Default for RustAstExtractor { + fn default() -> Self { + Self::new() + } +} + +impl Extractor for RustAstExtractor { + fn supports(&self, m: &MediaType) -> bool { + matches!(m, MediaType::Code(l) if l == "rust") + } + + fn parser_version(&self) -> ParserVersion { + ParserVersion(PARSER_VERSION.to_string()) + } + + fn extract( + &self, + ctx: &kebab_core::ExtractContext<'_>, + bytes: &[u8], + ) -> Result { + let asset = ctx.asset; + if !self.supports(&asset.media_type) { + anyhow::bail!( + "kebab-parse-code: unsupported media_type for RustAstExtractor: {:?}", + asset.media_type + ); + } + + let parser_version = self.parser_version(); + let doc_id = id_for_doc(&asset.workspace_path, &asset.asset_id, &parser_version); + + let source = String::from_utf8(bytes.to_vec()).map_err(|e| { + anyhow::anyhow!("kebab-parse-code: Rust source is not valid UTF-8: {e}") + })?; + + let blocks = build_blocks(&source, &doc_id)?; + let unit_count = blocks.len() as u32; + + let now = OffsetDateTime::now_utc(); + let mut events: Vec = Vec::with_capacity(2); + events.push(ProvenanceEvent { + at: asset.discovered_at, + agent: "kb-source-fs".to_string(), + kind: ProvenanceKind::Discovered, + note: None, + }); + events.push(ProvenanceEvent { + at: now, + agent: "kb-parse-code".to_string(), + kind: ProvenanceKind::Parsed, + note: Some(format!( + "parser_version={}; unit_count={}", + parser_version.0, unit_count + )), + }); + + let title = { + let fname = filename_from_workspace_path(&asset.workspace_path.0); + strip_extension(&fname) + }; + + // Resolve the file's absolute path for repo detection. If the + // source URI carries a relative path, anchor it at the workspace + // root so the `.git/` walk-up starts from the right place. + let abs_path = match &asset.source_uri { + kebab_core::SourceUri::File(p) => { + if p.is_absolute() { + p.clone() + } else { + ctx.workspace_root.join(p) + } + } + kebab_core::SourceUri::Kb(_) => ctx.workspace_root.to_path_buf(), + }; + let (repo, git_branch, git_commit) = match crate::repo::detect_repo(&abs_path) { + Some(r) => (Some(r.name), r.branch, r.commit), + None => (None, None, None), + }; + + let metadata = Metadata { + aliases: Vec::new(), + tags: Vec::new(), + created_at: asset.discovered_at, + updated_at: asset.discovered_at, + source_type: SourceType::Note, + trust_level: TrustLevel::Primary, + user_id_alias: None, + user: Map::new(), + repo, + git_branch, + git_commit, + code_lang: Some("rust".to_string()), + }; + + tracing::debug!( + target: "kebab-parse-code", + "extracted Rust doc_id={} workspace_path={} units={}", + doc_id.0, + asset.workspace_path.0, + unit_count + ); + + Ok(CanonicalDocument { + doc_id, + source_asset_id: asset.asset_id.clone(), + workspace_path: asset.workspace_path.clone(), + title, + lang: Lang("und".to_string()), + blocks, + metadata, + provenance: Provenance { events }, + parser_version, + schema_version: 1, + doc_version: 1, + last_chunker_version: None, + last_embedding_version: None, + }) + } +} + +fn filename_from_workspace_path(p: &str) -> String { + p.rsplit('/').next().unwrap_or(p).to_string() +} + +fn strip_extension(filename: &str) -> String { + match filename.rfind('.') { + Some(0) => filename.to_string(), + Some(idx) => filename[..idx].to_string(), + None => filename.to_string(), + } +} + +fn build_blocks( + source: &str, + doc_id: &kebab_core::DocumentId, +) -> anyhow::Result> { + let mut parser = tree_sitter::Parser::new(); + parser + .set_language(&tree_sitter_rust::LANGUAGE.into()) + .map_err(|e| anyhow::anyhow!("set tree-sitter-rust language: {e}"))?; + let tree = parser + .parse(source.as_bytes(), None) + .ok_or_else(|| anyhow::anyhow!("tree-sitter failed to parse Rust source"))?; + let lines: Vec<&str> = source.split('\n').collect(); + + let mut units: Vec<(String, u32, u32)> = Vec::new(); + let mut glue: Vec<(usize, u32, u32)> = Vec::new(); // (is_mod_decl 0/1, s, e) + + fn node_name<'a>(n: &tree_sitter::Node, src: &'a str) -> Option<&'a str> { + n.child_by_field_name("name") + .map(|c| &src[c.start_byte()..c.end_byte()]) + } + fn unit_start(n: &tree_sitter::Node) -> u32 { + let mut start = n.start_position().row as u32 + 1; + let mut prev = n.prev_sibling(); + while let Some(p) = prev { + let k = p.kind(); + if k == "line_comment" || k == "block_comment" || k == "attribute_item" { + start = p.start_position().row as u32 + 1; + prev = p.prev_sibling(); + } else { + break; + } + } + start + } + fn walk( + node: tree_sitter::Node, + src: &str, + mod_path: &[String], + units: &mut Vec<(String, u32, u32)>, + glue: &mut Vec<(usize, u32, u32)>, + ) { + let mut cur = node.walk(); + for child in node.named_children(&mut cur) { + let s = unit_start(&child); + let e = child.end_position().row as u32 + 1; + let prefix = if mod_path.is_empty() { + String::new() + } else { + format!("{}::", mod_path.join("::")) + }; + match child.kind() { + "function_item" | "struct_item" | "enum_item" | "union_item" + | "trait_item" | "type_item" => { + if let Some(name) = node_name(&child, src) { + flush_glue(glue, units); + units.push((format!("{prefix}{name}"), s, e)); + } + } + "macro_definition" => { + if let Some(name) = node_name(&child, src) { + flush_glue(glue, units); + units.push((format!("{prefix}{name}!"), s, e)); + } + } + "impl_item" => { + flush_glue(glue, units); + let ty = child + .child_by_field_name("type") + .map(|c| src[c.start_byte()..c.end_byte()].trim().to_string()); + let tr = child + .child_by_field_name("trait") + .map(|c| src[c.start_byte()..c.end_byte()].trim().to_string()); + let owner = tr.or(ty).unwrap_or_else(|| "".to_string()); + if let Some(body) = child.child_by_field_name("body") { + let mut bc = body.walk(); + for m in body.named_children(&mut bc) { + if m.kind() == "function_item" { + if let Some(mn) = node_name(&m, src) { + let ms = unit_start(&m); + let me = m.end_position().row as u32 + 1; + units.push((format!("{prefix}{owner}::{mn}"), ms, me)); + } + } + } + } + } + "mod_item" => { + if let Some(body) = child.child_by_field_name("body") { + flush_glue(glue, units); + let name = node_name(&child, src).unwrap_or("mod").to_string(); + let mut np = mod_path.to_vec(); + np.push(name); + walk(body, src, &np, units, glue); + } else { + glue.push((1, s, e)); + } + } + "use_declaration" | "extern_crate_declaration" | "const_item" + | "static_item" | "attribute_item" | "macro_invocation" => { + glue.push((0, s, e)); + } + _ => {} + } + } + flush_glue(glue, units); + } + fn flush_glue(glue: &mut Vec<(usize, u32, u32)>, units: &mut Vec<(String, u32, u32)>) { + if glue.is_empty() { + return; + } + let s = glue.iter().map(|(_, a, _)| *a).min().unwrap(); + let e = glue.iter().map(|(_, _, b)| *b).max().unwrap(); + let only_mod_decls = glue.iter().all(|(is_mod, _, _)| *is_mod == 1); + let sym = if only_mod_decls { "" } else { "" }; + units.push((sym.to_string(), s, e)); + glue.clear(); + } + + walk(tree.root_node(), source, &[], &mut units, &mut glue); + + let total_lines = lines.len() as u32; + let mut blocks = Vec::with_capacity(units.len()); + for (ordinal, (symbol, ls, le)) in units.into_iter().enumerate() { + let line_start = ls.max(1); + let line_end = le.min(total_lines.max(1)); + let span = SourceSpan::Code { + line_start, + line_end, + symbol: Some(symbol), + lang: Some("rust".to_string()), + }; + let block_id = id_for_block(doc_id, "code", &[], ordinal as u32, &span); + let code = lines[(line_start as usize - 1)..=(line_end as usize - 1)].join("\n"); + blocks.push(Block::Code(CodeBlock { + common: CommonBlock { + block_id, + heading_path: Vec::new(), + source_span: span, + }, + lang: Some("rust".to_string()), + code, + })); + } + Ok(blocks) +} + +#[cfg(test)] +mod tests { + use super::*; + use kebab_core::{Block, MediaType, SourceSpan}; + + fn extract_fixture() -> kebab_core::CanonicalDocument { + let bytes = std::fs::read( + concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/sample.rs"), + ) + .unwrap(); + let asset = kebab_parse_code_test_support::fixed_rust_asset("crates/x/src/sample.rs"); + let cfg = kebab_core::ExtractConfig::default(); + let root = std::path::PathBuf::from("/tmp"); + let ctx = kebab_core::ExtractContext { asset: &asset, workspace_root: &root, config: &cfg }; + RustAstExtractor::new().extract(&ctx, &bytes).unwrap() + } + + #[test] + fn extractor_supports_only_media_code_rust() { + let e = RustAstExtractor::new(); + assert!(e.supports(&MediaType::Code("rust".into()))); + assert!(!e.supports(&MediaType::Code("python".into()))); + assert!(!e.supports(&MediaType::Markdown)); + } + + #[test] + fn emits_one_block_per_semantic_unit_with_symbols() { + let doc = extract_fixture(); + let mut syms: Vec<(String, u32, u32)> = doc + .blocks + .iter() + .map(|b| match b { + Block::Code(c) => match &c.common.source_span { + SourceSpan::Code { symbol, line_start, line_end, lang } => { + assert_eq!(lang.as_deref(), Some("rust")); + (symbol.clone().unwrap(), *line_start, *line_end) + } + _ => panic!("code block must carry SourceSpan::Code"), + }, + other => panic!("expected Block::Code, got {other:?}"), + }) + .collect(); + syms.sort(); + let names: Vec<&str> = syms.iter().map(|(s, _, _)| s.as_str()).collect(); + assert!(names.contains(&"parse")); + assert!(names.contains(&"Foo")); + assert!(names.contains(&"Foo::double")); + assert!(names.contains(&"Foo::name")); + assert!(names.contains(&"Greet")); + assert!(names.contains(&"inner::helper")); + assert!(names.contains(&"")); // use + const grouped + let parse_src = doc.blocks.iter().find_map(|b| match b { + Block::Code(c) if matches!(&c.common.source_span, SourceSpan::Code{symbol,..} if symbol.as_deref()==Some("parse")) => Some(c.code.clone()), + _ => None, + }).unwrap(); + assert!(parse_src.contains("/// Doc comment on a free fn."), "doc comment folded in: {parse_src}"); + } + + #[test] + fn deterministic_across_runs() { + let a = extract_fixture(); + for _ in 0..50 { + assert_eq!(extract_fixture().blocks, a.blocks); + } + } +} + +#[cfg(test)] +mod kebab_parse_code_test_support { + use kebab_core::*; + use time::OffsetDateTime; + pub fn fixed_rust_asset(path: &str) -> RawAsset { + RawAsset { + asset_id: AssetId("a".repeat(64)), + source_uri: SourceUri::File(std::path::PathBuf::from(path)), + workspace_path: WorkspacePath(path.to_string()), + media_type: MediaType::Code("rust".to_string()), + byte_len: 0, + checksum: Checksum("b".repeat(64)), + discovered_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(), + stored: AssetStorage::Reference { + path: std::path::PathBuf::from(path), + sha: Checksum("b".repeat(64)), + }, + } + } +} diff --git a/crates/kebab-parse-code/tests/fixtures/sample.rs b/crates/kebab-parse-code/tests/fixtures/sample.rs new file mode 100644 index 0000000..957a5c1 --- /dev/null +++ b/crates/kebab-parse-code/tests/fixtures/sample.rs @@ -0,0 +1,35 @@ +//! sample fixture + +use std::fmt; + +const ANSWER: u32 = 42; + +/// Doc comment on a free fn. +pub fn parse(input: &str) -> usize { + input.len() +} + +pub struct Foo { + pub n: u32, +} + +impl Foo { + /// method doc + pub fn double(&self) -> u32 { + self.n * 2 + } + + fn name() -> &'static str { + "foo" + } +} + +pub trait Greet { + fn hello(&self) -> String; +} + +mod inner { + pub fn helper() -> bool { + true + } +} -- 2.49.1 From a93b33ffbe517184b8101e6ab93b0c3059c79c09 Mon Sep 17 00:00:00 2001 From: altair823 Date: Tue, 19 May 2026 17:28:49 +0000 Subject: [PATCH 08/19] fix(p10-1a-2): correct label scope + de-dup leading attribute (Task 6 review) Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/kebab-parse-code/src/rust.rs | 105 +++++++++++++++++++++++++--- 1 file changed, 97 insertions(+), 8 deletions(-) diff --git a/crates/kebab-parse-code/src/rust.rs b/crates/kebab-parse-code/src/rust.rs index 4b15ce2..98c8343 100644 --- a/crates/kebab-parse-code/src/rust.rs +++ b/crates/kebab-parse-code/src/rust.rs @@ -182,7 +182,10 @@ fn build_blocks( .ok_or_else(|| anyhow::anyhow!("tree-sitter failed to parse Rust source"))?; let lines: Vec<&str> = source.split('\n').collect(); - let mut units: Vec<(String, u32, u32)> = Vec::new(); + // units: (symbol, line_start, line_end, is_real_semantic_unit). + // Glue groups are pushed with a sentinel symbol + is_real=false so a + // post-pass can decide `` vs `` (Gap 1). + let mut units: Vec<(String, u32, u32, bool)> = Vec::new(); let mut glue: Vec<(usize, u32, u32)> = Vec::new(); // (is_mod_decl 0/1, s, e) fn node_name<'a>(n: &tree_sitter::Node, src: &'a str) -> Option<&'a str> { @@ -207,7 +210,7 @@ fn build_blocks( node: tree_sitter::Node, src: &str, mod_path: &[String], - units: &mut Vec<(String, u32, u32)>, + units: &mut Vec<(String, u32, u32, bool)>, glue: &mut Vec<(usize, u32, u32)>, ) { let mut cur = node.walk(); @@ -223,17 +226,25 @@ fn build_blocks( "function_item" | "struct_item" | "enum_item" | "union_item" | "trait_item" | "type_item" => { if let Some(name) = node_name(&child, src) { + // Gap 2: a leading attribute/comment that this unit + // re-absorbs (via `unit_start`'s upward extension to + // `s`) must not also remain in the glue group, or it + // would be emitted in both chunks. Drop glue entries + // at/after the unit's extended start. + glue.retain(|(_, gs, _)| *gs < s); flush_glue(glue, units); - units.push((format!("{prefix}{name}"), s, e)); + units.push((format!("{prefix}{name}"), s, e, true)); } } "macro_definition" => { if let Some(name) = node_name(&child, src) { + glue.retain(|(_, gs, _)| *gs < s); flush_glue(glue, units); - units.push((format!("{prefix}{name}!"), s, e)); + units.push((format!("{prefix}{name}!"), s, e, true)); } } "impl_item" => { + glue.retain(|(_, gs, _)| *gs < s); flush_glue(glue, units); let ty = child .child_by_field_name("type") @@ -249,7 +260,7 @@ fn build_blocks( if let Some(mn) = node_name(&m, src) { let ms = unit_start(&m); let me = m.end_position().row as u32 + 1; - units.push((format!("{prefix}{owner}::{mn}"), ms, me)); + units.push((format!("{prefix}{owner}::{mn}"), ms, me, true)); } } } @@ -275,23 +286,39 @@ fn build_blocks( } flush_glue(glue, units); } - fn flush_glue(glue: &mut Vec<(usize, u32, u32)>, units: &mut Vec<(String, u32, u32)>) { + fn flush_glue(glue: &mut Vec<(usize, u32, u32)>, units: &mut Vec<(String, u32, u32, bool)>) { if glue.is_empty() { return; } let s = glue.iter().map(|(_, a, _)| *a).min().unwrap(); let e = glue.iter().map(|(_, _, b)| *b).max().unwrap(); + // Provisional label: `` only if this group is exclusively + // bodyless `mod foo;` declarations. The final decision (Gap 1) also + // requires the *whole file* to have produced zero real units; that + // demotion to `` happens in the post-pass below. let only_mod_decls = glue.iter().all(|(is_mod, _, _)| *is_mod == 1); let sym = if only_mod_decls { "" } else { "" }; - units.push((sym.to_string(), s, e)); + units.push((sym.to_string(), s, e, false)); glue.clear(); } walk(tree.root_node(), source, &[], &mut units, &mut glue); + // Gap 1: `` is correct only when the file produced no real + // (non-glue) semantic unit at all. If any real unit exists, every glue + // group is ``, even a pure mod-decl group. + let has_real_unit = units.iter().any(|(_, _, _, is_real)| *is_real); + if has_real_unit { + for (sym, _, _, is_real) in units.iter_mut() { + if !*is_real && sym == "" { + *sym = "".to_string(); + } + } + } + let total_lines = lines.len() as u32; let mut blocks = Vec::with_capacity(units.len()); - for (ordinal, (symbol, ls, le)) in units.into_iter().enumerate() { + for (ordinal, (symbol, ls, le, _is_real)) in units.into_iter().enumerate() { let line_start = ls.max(1); let line_end = le.min(total_lines.max(1)); let span = SourceSpan::Code { @@ -373,6 +400,68 @@ mod tests { assert!(parse_src.contains("/// Doc comment on a free fn."), "doc comment folded in: {parse_src}"); } + /// Run the extractor on an in-memory Rust source string (no fixture + /// file) and return (symbol, code) for every emitted block. + fn extract_inline(source: &str) -> Vec<(String, String)> { + let asset = kebab_parse_code_test_support::fixed_rust_asset("crates/x/src/inline.rs"); + let cfg = kebab_core::ExtractConfig::default(); + let root = std::path::PathBuf::from("/tmp"); + let ctx = kebab_core::ExtractContext { asset: &asset, workspace_root: &root, config: &cfg }; + let doc = RustAstExtractor::new() + .extract(&ctx, source.as_bytes()) + .unwrap(); + doc.blocks + .iter() + .map(|b| match b { + Block::Code(c) => match &c.common.source_span { + SourceSpan::Code { symbol, .. } => { + (symbol.clone().unwrap(), c.code.clone()) + } + _ => panic!("code block must carry SourceSpan::Code"), + }, + other => panic!("expected Block::Code, got {other:?}"), + }) + .collect() + } + + #[test] + fn module_label_scope_and_attribute_dedup() { + // Source A (Gap 2): leading attribute is re-absorbed into the unit + // and must NOT also form a separate glue chunk. + let a = extract_inline("#[derive(Debug)]\npub struct Tagged { x: u32 }\n"); + assert_eq!(a.len(), 1, "Gap 2: exactly one block, got {a:?}"); + assert_eq!(a[0].0, "Tagged"); + assert!( + a[0].1.contains("#[derive(Debug)]"), + "attribute folded into unit: {:?}", + a[0].1 + ); + assert!( + !a.iter().any(|(s, _)| s == ""), + "attribute must not also form a glue chunk: {a:?}" + ); + + // Source B (Gap 1): file has no real units, only bodyless mod + // decls -> the glue group is . + let b = extract_inline("mod a;\nmod b;\n"); + assert_eq!(b.len(), 1, "one glue block, got {b:?}"); + assert_eq!(b[0].0, ""); + + // Source C (Gap 1): mod decls + a real unit -> the glue group is + // , NOT , because the file has a real unit. + let c = extract_inline("mod a;\nmod b;\npub fn f() {}\n"); + let syms: Vec<&str> = c.iter().map(|(s, _)| s.as_str()).collect(); + assert!(syms.contains(&"f"), "real unit present: {c:?}"); + assert!( + syms.contains(&""), + "mod-decl glue demoted to : {c:?}" + ); + assert!( + !syms.contains(&""), + "must not be when file has a real unit: {c:?}" + ); + } + #[test] fn deterministic_across_runs() { let a = extract_fixture(); -- 2.49.1 From df85bafa7ff5dd5931b57a060967e769418ddd29 Mon Sep 17 00:00:00 2001 From: altair823 Date: Tue, 19 May 2026 17:35:52 +0000 Subject: [PATCH 09/19] fix(p10-1a-2): module-prefix glue symbols + crate desc + invariant hardening (Task 6 review) Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/kebab-parse-code/Cargo.toml | 2 +- crates/kebab-parse-code/src/rust.rs | 80 +++++++++++++++++++++++------ 2 files changed, 66 insertions(+), 16 deletions(-) diff --git a/crates/kebab-parse-code/Cargo.toml b/crates/kebab-parse-code/Cargo.toml index 3953ed5..5ef1c69 100644 --- a/crates/kebab-parse-code/Cargo.toml +++ b/crates/kebab-parse-code/Cargo.toml @@ -5,7 +5,7 @@ edition = { workspace = true } rust-version = { workspace = true } license = { workspace = true } repository = { workspace = true } -description = "Language-aware code parsing infrastructure (lang dispatch, .git/ detect, skip helpers) for the kebab pipeline (P10-1A-1)" +description = "Language-aware code parsing for the kebab pipeline: lang dispatch / .git detect / skip helpers (P10-1A-1) + tree-sitter Rust AST extractor (P10-1A-2)" [dependencies] kebab-core = { path = "../kebab-core" } diff --git a/crates/kebab-parse-code/src/rust.rs b/crates/kebab-parse-code/src/rust.rs index 98c8343..6ca75b2 100644 --- a/crates/kebab-parse-code/src/rust.rs +++ b/crates/kebab-parse-code/src/rust.rs @@ -213,15 +213,21 @@ fn build_blocks( units: &mut Vec<(String, u32, u32, bool)>, glue: &mut Vec<(usize, u32, u32)>, ) { + // Module-path prefix for this scope. Used for both real units + // (`format!("{prefix}{name}")`) and glue group labels + // (`format!("{prefix}")`) so glue from `mod inner` + // doesn't collide on symbol with file-top-level glue and keeps + // module context downstream. Empty at file top level -> glue + // stays exactly `` / ``. + let prefix = if mod_path.is_empty() { + String::new() + } else { + format!("{}::", mod_path.join("::")) + }; let mut cur = node.walk(); for child in node.named_children(&mut cur) { let s = unit_start(&child); let e = child.end_position().row as u32 + 1; - let prefix = if mod_path.is_empty() { - String::new() - } else { - format!("{}::", mod_path.join("::")) - }; match child.kind() { "function_item" | "struct_item" | "enum_item" | "union_item" | "trait_item" | "type_item" => { @@ -232,20 +238,20 @@ fn build_blocks( // would be emitted in both chunks. Drop glue entries // at/after the unit's extended start. glue.retain(|(_, gs, _)| *gs < s); - flush_glue(glue, units); + flush_glue(glue, units, &prefix); units.push((format!("{prefix}{name}"), s, e, true)); } } "macro_definition" => { if let Some(name) = node_name(&child, src) { glue.retain(|(_, gs, _)| *gs < s); - flush_glue(glue, units); + flush_glue(glue, units, &prefix); units.push((format!("{prefix}{name}!"), s, e, true)); } } "impl_item" => { glue.retain(|(_, gs, _)| *gs < s); - flush_glue(glue, units); + flush_glue(glue, units, &prefix); let ty = child .child_by_field_name("type") .map(|c| src[c.start_byte()..c.end_byte()].trim().to_string()); @@ -255,6 +261,11 @@ fn build_blocks( let owner = tr.or(ty).unwrap_or_else(|| "".to_string()); if let Some(body) = child.child_by_field_name("body") { let mut bc = body.walk(); + // 1A scope: only inner `function_item` children + // become units. Associated consts / types and other + // non-fn impl members are intentionally NOT emitted + // as separate units in 1A (plan spec: "1 per inner + // function_item"). for m in body.named_children(&mut bc) { if m.kind() == "function_item" { if let Some(mn) = node_name(&m, src) { @@ -268,11 +279,20 @@ fn build_blocks( } "mod_item" => { if let Some(body) = child.child_by_field_name("body") { - flush_glue(glue, units); + flush_glue(glue, units, &prefix); let name = node_name(&child, src).unwrap_or("mod").to_string(); let mut np = mod_path.to_vec(); np.push(name); walk(body, src, &np, units, glue); + // Invariant: `glue` is shared by `&mut` across + // recursive `walk` calls; every `walk` path ends with + // a `flush_glue`, so inner-scope glue can never leak + // into this outer scope's group. Assert it structurally + // rather than relying on that being incidental. + debug_assert!( + glue.is_empty(), + "inner walk must flush its glue before returning" + ); } else { glue.push((1, s, e)); } @@ -284,9 +304,13 @@ fn build_blocks( _ => {} } } - flush_glue(glue, units); + flush_glue(glue, units, &prefix); } - fn flush_glue(glue: &mut Vec<(usize, u32, u32)>, units: &mut Vec<(String, u32, u32, bool)>) { + fn flush_glue( + glue: &mut Vec<(usize, u32, u32)>, + units: &mut Vec<(String, u32, u32, bool)>, + prefix: &str, + ) { if glue.is_empty() { return; } @@ -297,8 +321,12 @@ fn build_blocks( // requires the *whole file* to have produced zero real units; that // demotion to `` happens in the post-pass below. let only_mod_decls = glue.iter().all(|(is_mod, _, _)| *is_mod == 1); - let sym = if only_mod_decls { "" } else { "" }; - units.push((sym.to_string(), s, e, false)); + let label = if only_mod_decls { "" } else { "" }; + // Module-path-prefix the label so glue from `mod inner` carries + // module context (`inner::`) and doesn't collide with + // file-top-level glue. `prefix` is empty at file top level, so the + // symbol stays exactly `` / `` there. + units.push((format!("{prefix}{label}"), s, e, false)); glue.clear(); } @@ -310,8 +338,12 @@ fn build_blocks( let has_real_unit = units.iter().any(|(_, _, _, is_real)| *is_real); if has_real_unit { for (sym, _, _, is_real) in units.iter_mut() { - if !*is_real && sym == "" { - *sym = "".to_string(); + // Match on the *suffix*: a glue group may now carry a module + // prefix (`inner::`), so demote any `…` to the + // same-prefixed `…` rather than only the bare form. + if !*is_real && sym.ends_with("") { + let pre = &sym[..sym.len() - "".len()]; + *sym = format!("{pre}"); } } } @@ -460,6 +492,24 @@ mod tests { !syms.contains(&""), "must not be when file has a real unit: {c:?}" ); + + // Source D (Fix 1): glue inside a bodied `mod inner` must carry the + // module-path prefix so it doesn't collide with file-top-level glue + // and keeps module context downstream. + let d = extract_inline("mod inner {\n use std::fmt;\n pub fn helper() {}\n}\n"); + let dsyms: Vec<&str> = d.iter().map(|(s, _)| s.as_str()).collect(); + assert!( + dsyms.contains(&"inner::helper"), + "real unit inside mod is prefixed: {d:?}" + ); + assert!( + dsyms.contains(&"inner::"), + "glue inside mod inner is module-prefixed, not bare: {d:?}" + ); + assert!( + !dsyms.contains(&""), + "glue inside mod inner must NOT be the bare top-level symbol: {d:?}" + ); } #[test] -- 2.49.1 From c74f8d269e5e566fa32f13b813f78fab0571041e Mon Sep 17 00:00:00 2001 From: altair823 Date: Tue, 19 May 2026 17:36:54 +0000 Subject: [PATCH 10/19] chore(p10-1a-2): sync Cargo.lock for kebab-parse-code deps (Task 6 follow-up) Co-Authored-By: Claude Opus 4.7 (1M context) --- Cargo.lock | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 37c7e24..16d69ad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4345,7 +4345,11 @@ version = "0.6.0" dependencies = [ "anyhow", "gix", + "kebab-core", + "serde_json", "tempfile", + "time", + "tracing", "tree-sitter", "tree-sitter-rust", ] -- 2.49.1 From 808b92a6c50c0d862a951b032eee9a29b06e3136 Mon Sep 17 00:00:00 2001 From: altair823 Date: Tue, 19 May 2026 17:40:11 +0000 Subject: [PATCH 11/19] feat(p10-1a-2): code-rust-ast-v1 chunker (1:1 + oversize split) Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/kebab-chunk/src/code_rust_ast_v1.rs | 322 +++++++++++++++++++++ crates/kebab-chunk/src/lib.rs | 2 + 2 files changed, 324 insertions(+) create mode 100644 crates/kebab-chunk/src/code_rust_ast_v1.rs diff --git a/crates/kebab-chunk/src/code_rust_ast_v1.rs b/crates/kebab-chunk/src/code_rust_ast_v1.rs new file mode 100644 index 0000000..c687c56 --- /dev/null +++ b/crates/kebab-chunk/src/code_rust_ast_v1.rs @@ -0,0 +1,322 @@ +//! `code-rust-ast-v1` — maps a tree-sitter-derived Rust AST +//! `CanonicalDocument` (one `Block::Code` per semantic unit, each with +//! `SourceSpan::Code`) to chunks 1:1. A unit longer than +//! `AST_CHUNK_MAX_LINES` is split into ` [part i/N]` sub-chunks +//! at blank-line paragraph boundaries (design §9.1 oversize fallback). +//! +//! tree-sitter is intentionally NOT a dependency here: AST work is +//! parser-side (`kebab-parse-code`, design §6.3). This chunker only +//! consumes the `CanonicalDocument`. +//! +//! `AST_CHUNK_MAX_LINES` is a constant matching +//! `IngestCodeCfg::default().ast_chunk_max_lines` (200). Per-medium +//! config threading needs a chunker registry (P+); same deviation +//! pattern as `pdf-page-v1`'s pinned `chunker_version` +//! (`tasks/HOTFIXES.md`). + +use kebab_core::{ + Block, BlockId, CanonicalDocument, Chunk, ChunkPolicy, Chunker, ChunkerVersion, DocumentId, + SourceSpan, id_for_chunk, +}; + +const VERSION_LABEL: &str = "code-rust-ast-v1"; +const BYTES_PER_TOKEN: usize = 3; +const POLICY_HASH_HEX_LEN: usize = 16; +const AST_CHUNK_MAX_LINES: u32 = 200; + +#[derive(Clone, Copy, Debug, Default)] +pub struct CodeRustAstV1Chunker; + +impl Chunker for CodeRustAstV1Chunker { + fn chunker_version(&self) -> ChunkerVersion { + ChunkerVersion(VERSION_LABEL.to_string()) + } + + fn policy_hash(&self, policy: &ChunkPolicy) -> String { + let bytes = serde_json_canonicalizer::to_vec(policy) + .expect("canonical JSON serialization of ChunkPolicy must not fail"); + let hex = blake3::hash(&bytes).to_hex().to_string(); + hex[..POLICY_HASH_HEX_LEN].to_string() + } + + fn chunk( + &self, + doc: &CanonicalDocument, + policy: &ChunkPolicy, + ) -> anyhow::Result> { + for b in &doc.blocks { + let c = match b { + Block::Code(c) => c, + _ => anyhow::bail!( + "CodeRustAstV1Chunker only handles code docs (got non-Code block)" + ), + }; + if !matches!(c.common.source_span, SourceSpan::Code { .. }) { + anyhow::bail!( + "CodeRustAstV1Chunker only handles code docs (got non-Code source_span)" + ); + } + } + + let base_policy_hash = self.policy_hash(policy); + let chunker_version = self.chunker_version(); + let mut out: Vec = Vec::new(); + + for b in &doc.blocks { + let cb = match b { + Block::Code(c) => c, + _ => unreachable!("validated above"), + }; + let (ls, le, symbol, lang) = match &cb.common.source_span { + SourceSpan::Code { line_start, line_end, symbol, lang } => { + (*line_start, *line_end, symbol.clone(), lang.clone()) + } + _ => unreachable!("validated above"), + }; + let block_ids: Vec = vec![cb.common.block_id.clone()]; + let span_lines = le.saturating_sub(ls) + 1; + + if span_lines <= AST_CHUNK_MAX_LINES { + let span = SourceSpan::Code { + line_start: ls, + line_end: le, + symbol: symbol.clone(), + lang: lang.clone(), + }; + out.push(make_chunk( + doc, &chunker_version, &block_ids, &base_policy_hash, + None, span, cb.code.clone(), + )); + } else { + let parts = split_oversize(&cb.code); + let n = parts.len(); + for (i, (off_start, off_end, text)) in parts.into_iter().enumerate() { + let part_ls = ls + off_start; + let part_le = ls + off_end; + let part_sym = symbol + .as_ref() + .map(|s| format!("{s} [part {}/{n}]", i + 1)); + let span = SourceSpan::Code { + line_start: part_ls, + line_end: part_le, + symbol: part_sym, + lang: lang.clone(), + }; + out.push(make_chunk( + doc, &chunker_version, &block_ids, &base_policy_hash, + Some(part_ls), span, text, + )); + } + } + } + + tracing::debug!( + target: "kebab-chunk", + doc_id = %doc.doc_id, + chunks = out.len(), + "code-rust-ast-v1 chunked", + ); + Ok(out) + } +} + +#[allow(clippy::too_many_arguments)] +fn make_chunk( + doc: &CanonicalDocument, + chunker_version: &ChunkerVersion, + block_ids: &[BlockId], + base_policy_hash: &str, + split_key: Option, + span: SourceSpan, + text: String, +) -> Chunk { + let id_hash = match split_key { + Some(k) => format!("{base_policy_hash}#L{k}"), + None => base_policy_hash.to_string(), + }; + let chunk_id = id_for_chunk(&doc.doc_id, chunker_version, block_ids, &id_hash); + let token_estimate = text.len().div_ceil(BYTES_PER_TOKEN); + Chunk { + chunk_id, + doc_id: DocumentId(doc.doc_id.0.clone()), + block_ids: block_ids.to_vec(), + text, + heading_path: Vec::new(), + source_spans: vec![span], + token_estimate, + chunker_version: chunker_version.clone(), + policy_hash: base_policy_hash.to_string(), + } +} + +/// Split an oversize unit at blank-line paragraph boundaries, greedily +/// gluing paragraphs until ~`AST_CHUNK_MAX_LINES` lines accumulate. +/// Returns `(line_offset_start, line_offset_end, text)` where offsets are +/// 0-based within the unit (caller adds the unit's absolute `line_start`). +fn split_oversize(code: &str) -> Vec<(u32, u32, String)> { + let lines: Vec<&str> = code.split('\n').collect(); + let total = lines.len() as u32; + let mut out: Vec<(u32, u32, String)> = Vec::new(); + let mut start: u32 = 0; + while start < total { + let mut end = (start + AST_CHUNK_MAX_LINES).min(total); + let floor = start + (AST_CHUNK_MAX_LINES * 4 / 5); + if end < total { + if let Some(b) = (floor.min(end)..end) + .rev() + .find(|&i| lines[i as usize].trim().is_empty()) + { + end = b + 1; + } + } + let text = lines[start as usize..end as usize].join("\n"); + out.push((start, end.saturating_sub(1), text)); + start = end; + } + if out.is_empty() { + out.push((0, total.saturating_sub(1), code.to_string())); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use kebab_core::{ + Block, CanonicalDocument, ChunkPolicy, Chunker, ChunkerVersion, CodeBlock, CommonBlock, + SourceSpan, id_for_block, id_for_doc, AssetId, Lang, Metadata, ParserVersion, Provenance, + SourceType, TrustLevel, WorkspacePath, + }; + use time::OffsetDateTime; + + fn code_doc(units: &[(&str, u32, u32, &str)]) -> CanonicalDocument { + let wp = WorkspacePath("crates/x/src/a.rs".into()); + let aid = AssetId("a".repeat(64)); + let pv = ParserVersion("code-rust-v1".into()); + let doc_id = id_for_doc(&wp, &aid, &pv); + let blocks = units + .iter() + .enumerate() + .map(|(i, (sym, ls, le, code))| { + let span = SourceSpan::Code { + line_start: *ls, + line_end: *le, + symbol: Some((*sym).to_string()), + lang: Some("rust".into()), + }; + let bid = id_for_block(&doc_id, "code", &[], i as u32, &span); + Block::Code(CodeBlock { + common: CommonBlock { block_id: bid, heading_path: vec![], source_span: span }, + lang: Some("rust".into()), + code: (*code).to_string(), + }) + }) + .collect(); + CanonicalDocument { + doc_id, source_asset_id: aid, workspace_path: wp, title: "a".into(), + lang: Lang("und".into()), blocks, + metadata: Metadata { + aliases: vec![], tags: vec![], + created_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(), + updated_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(), + source_type: SourceType::Note, trust_level: TrustLevel::Primary, + user_id_alias: None, user: Default::default(), + repo: Some("kebab".into()), git_branch: Some("main".into()), + git_commit: Some("0".repeat(40)), code_lang: Some("rust".into()), + }, + provenance: Provenance { events: vec![] }, + parser_version: pv, schema_version: 1, doc_version: 1, + last_chunker_version: None, last_embedding_version: None, + } + } + fn policy() -> ChunkPolicy { + ChunkPolicy { target_tokens: 500, overlap_tokens: 80, + respect_markdown_headings: false, + chunker_version: ChunkerVersion(VERSION_LABEL.into()) } + } + + #[test] + fn chunker_version_is_code_rust_ast_v1() { + assert_eq!(CodeRustAstV1Chunker.chunker_version(), + ChunkerVersion("code-rust-ast-v1".into())); + } + + #[test] + fn one_chunk_per_unit_preserves_code_span() { + let doc = code_doc(&[ + ("parse", 1, 3, "pub fn parse() {}\n// x\n}"), + ("Foo::double", 5, 7, "fn double() {}\n//\n}"), + ]); + let chunks = CodeRustAstV1Chunker.chunk(&doc, &policy()).unwrap(); + assert_eq!(chunks.len(), 2); + for c in &chunks { + assert_eq!(c.source_spans.len(), 1); + assert!(matches!(c.source_spans[0], SourceSpan::Code { .. })); + assert_eq!(c.heading_path, Vec::::new()); + assert_eq!(c.chunker_version.0, "code-rust-ast-v1"); + } + match &chunks[0].source_spans[0] { + SourceSpan::Code { symbol, line_start, line_end, .. } => { + assert_eq!(symbol.as_deref(), Some("parse")); + assert_eq!((*line_start, *line_end), (1, 3)); + } + _ => unreachable!(), + } + } + + #[test] + fn oversize_unit_splits_into_parts_with_unique_ids() { + let body = (0..500).map(|i| format!(" let x{i} = {i};")).collect::>().join("\n"); + let code = format!("pub fn big() {{\n{body}\n}}"); + let doc = code_doc(&[("big", 1, 502, &code)]); + let chunks = CodeRustAstV1Chunker.chunk(&doc, &policy()).unwrap(); + assert!(chunks.len() >= 2, "oversize unit must split, got {}", chunks.len()); + for c in &chunks { + match &c.source_spans[0] { + SourceSpan::Code { symbol, .. } => { + assert!(symbol.as_deref().unwrap().starts_with("big [part "), + "part-numbered symbol, got {symbol:?}"); + } + _ => unreachable!(), + } + } + let mut ids: Vec<&str> = chunks.iter().map(|c| c.chunk_id.0.as_str()).collect(); + let n = ids.len(); ids.sort(); ids.dedup(); + assert_eq!(ids.len(), n, "chunk_ids unique across split parts"); + } + + #[test] + fn non_code_doc_errors() { + use kebab_core::TextBlock; + let mut doc = code_doc(&[("parse", 1, 1, "fn parse(){}")]); + doc.blocks = vec![Block::Paragraph(TextBlock { + common: CommonBlock { + block_id: kebab_core::BlockId("b".into()), + heading_path: vec![], + source_span: SourceSpan::Line { start: 1, end: 1 }, + }, + text: "x".into(), inlines: vec![], + })]; + let err = CodeRustAstV1Chunker.chunk(&doc, &policy()).unwrap_err(); + assert!(err.to_string().contains("CodeRustAstV1Chunker")); + } + + #[test] + fn deterministic_chunk_ids_1000() { + let doc = code_doc(&[("parse", 1, 2, "fn parse(){}\n}")]); + let base: Vec = CodeRustAstV1Chunker.chunk(&doc, &policy()) + .unwrap().into_iter().map(|c| c.chunk_id.0).collect(); + for _ in 0..1000 { + let again: Vec = CodeRustAstV1Chunker.chunk(&doc, &policy()) + .unwrap().into_iter().map(|c| c.chunk_id.0).collect(); + assert_eq!(again, base); + } + } + + #[test] + fn policy_hash_matches_md_heading_v1() { + let p = policy(); + assert_eq!(CodeRustAstV1Chunker.policy_hash(&p), + crate::MdHeadingV1Chunker.policy_hash(&p)); + } +} diff --git a/crates/kebab-chunk/src/lib.rs b/crates/kebab-chunk/src/lib.rs index cd3a2da..88a8eb7 100644 --- a/crates/kebab-chunk/src/lib.rs +++ b/crates/kebab-chunk/src/lib.rs @@ -15,8 +15,10 @@ //! embedder, the retriever, the LLM, the RAG layer, or the UI layers. //! It consumes `CanonicalDocument` purely through `kb-core` types. +mod code_rust_ast_v1; mod md_heading_v1; mod pdf_page_v1; +pub use code_rust_ast_v1::CodeRustAstV1Chunker; pub use md_heading_v1::MdHeadingV1Chunker; pub use pdf_page_v1::PdfPageV1Chunker; -- 2.49.1 From 580576c2c6f7738a3ae0f07853d63ed110c9cc26 Mon Sep 17 00:00:00 2001 From: altair823 Date: Tue, 19 May 2026 20:14:59 +0000 Subject: [PATCH 12/19] feat(p10-1a-2): wire code ingest dispatch (ingest_one_code_asset) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add `MediaType::Code("rust")` dispatch arm in `ingest_one_asset`, `ingest_one_code_asset` fn (faithful mirror of `ingest_one_pdf_asset`), and `backfill_code_lang` post-processing in `App::search_uncached`. Integration test `code_ingest_smoke.rs` verifies full pipeline: ingest `.rs` → Citation::Code hit with lang/symbol/line_start. Co-Authored-By: Claude Opus 4.7 (1M context) --- Cargo.lock | 1 + crates/kebab-app/Cargo.toml | 4 + crates/kebab-app/src/app.rs | 22 +++ crates/kebab-app/src/lib.rs | 186 +++++++++++++++++++- crates/kebab-app/tests/code_ingest_smoke.rs | 142 +++++++++++++++ 5 files changed, 353 insertions(+), 2 deletions(-) create mode 100644 crates/kebab-app/tests/code_ingest_smoke.rs diff --git a/Cargo.lock b/Cargo.lock index 16d69ad..814bd6c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4143,6 +4143,7 @@ dependencies = [ "kebab-llm", "kebab-llm-local", "kebab-normalize", + "kebab-parse-code", "kebab-parse-image", "kebab-parse-md", "kebab-parse-pdf", diff --git a/crates/kebab-app/Cargo.toml b/crates/kebab-app/Cargo.toml index a3ec230..5dfca4b 100644 --- a/crates/kebab-app/Cargo.toml +++ b/crates/kebab-app/Cargo.toml @@ -32,6 +32,10 @@ kebab-parse-image = { path = "../kebab-parse-image" } # per-asset dispatch (see `ingest_one_asset` PDF branch) and runs the # resulting `CanonicalDocument` through `kebab-chunk::PdfPageV1Chunker`. kebab-parse-pdf = { path = "../kebab-parse-pdf" } +# p10-1A-2: Rust AST extractor lives here. App threads it into the +# per-asset dispatch (see `ingest_one_asset` Code branch) and runs the +# resulting `CanonicalDocument` through `kebab-chunk::CodeRustAstV1Chunker`. +kebab-parse-code = { path = "../kebab-parse-code" } anyhow = { workspace = true } blake3 = { workspace = true } serde = { workspace = true } diff --git a/crates/kebab-app/src/app.rs b/crates/kebab-app/src/app.rs index a3d2c07..2202cd4 100644 --- a/crates/kebab-app/src/app.rs +++ b/crates/kebab-app/src/app.rs @@ -296,6 +296,11 @@ impl App { now, self.config.search.stale_threshold_days, ); + // p10-1A-2: backfill `code_lang` from the Citation::Code `lang` + // field. The search layer (kebab-search) constructs SearchHit with + // `code_lang: None`; we own the post-processing here in kebab-app + // and can fill it cheaply from data already present in the hit. + backfill_code_lang(&mut hits); Ok(hits) } @@ -387,6 +392,8 @@ impl App { now, self.config.search.stale_threshold_days, ); + // p10-1A-2: backfill code_lang — same as search_uncached. + backfill_code_lang(&mut traced_hits); // Apply offset + k_effective truncation (mirrors non-trace path). let drop_n = offset.min(traced_hits.len()); @@ -896,6 +903,21 @@ fn estimate_chars(hits: &[SearchHit]) -> usize { .sum() } +/// p10-1A-2: back-fill `SearchHit.code_lang` from `Citation::Code.lang` +/// for every code hit in the list. The search layer (kebab-search) +/// constructs hits with `code_lang: None`; we fill it here in kebab-app +/// post-retrieval so callers see the correct language identifier without +/// requiring a second SQL query. +fn backfill_code_lang(hits: &mut [SearchHit]) { + for hit in hits.iter_mut() { + if let kebab_core::Citation::Code { lang, .. } = &hit.citation { + if hit.code_lang.is_none() { + hit.code_lang = lang.clone(); + } + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/kebab-app/src/lib.rs b/crates/kebab-app/src/lib.rs index 850672d..39f43e8 100644 --- a/crates/kebab-app/src/lib.rs +++ b/crates/kebab-app/src/lib.rs @@ -39,7 +39,7 @@ use std::sync::Arc; use anyhow::{Context, anyhow}; use serde::{Deserialize, Serialize}; -use kebab_chunk::{MdHeadingV1Chunker, PdfPageV1Chunker}; +use kebab_chunk::{CodeRustAstV1Chunker, MdHeadingV1Chunker, PdfPageV1Chunker}; use kebab_core::{ Answer, Block, CanonicalDocument, Chunk, ChunkId, ChunkPolicy, ChunkerVersion, Chunker, DocFilter, DocSummary, DocumentId, DocumentStore, Embedder, EmbeddingInput, @@ -50,6 +50,7 @@ use kebab_core::{ use kebab_llm_local::OllamaLanguageModel; use kebab_normalize::build_canonical_document; use kebab_parse_image::{ImageExtractor, OllamaVisionOcr, apply_caption, apply_ocr}; +use kebab_parse_code::RustAstExtractor; use kebab_parse_pdf::PdfTextExtractor; use kebab_parse_md::{BodyHints, parse_blocks, parse_frontmatter}; use kebab_source_fs::FsSourceConnector; @@ -917,7 +918,20 @@ fn ingest_one_asset( force_reingest, ); } - // p10-1A-2: Code dispatch wired in Task 8; skip until then. + // p10-1A-2 Task 8: Rust code ingest. + MediaType::Code(lang) if lang == "rust" => { + return ingest_one_code_asset( + app, + asset, + chunk_policy, + embedder, + vector_store, + existing_doc_ids, + force_reingest, + ); + } + // p10-1A-2: non-Rust Code, Audio, and Other are not yet wired; + // skip until their respective phases. MediaType::Code(_) | MediaType::Audio(_) | MediaType::Other(_) => { return Ok(kebab_core::IngestItem { kind: kebab_core::IngestItemKind::Skipped, @@ -1618,6 +1632,174 @@ fn ingest_one_pdf_asset( }) } +/// p10-1A-2 Task 8: process one `MediaType::Code("rust")` asset end-to-end. +/// +/// Mirrors `ingest_one_pdf_asset` line-for-line with the substitutions +/// documented in the task spec: +/// - parser_version → `code-rust-v1` (via `RUST_PARSER_VERSION`) +/// - extractor → `RustAstExtractor` +/// - chunker → `CodeRustAstV1Chunker` +/// +/// All other steps (incremental skip, byte read, ExtractContext, put_*, +/// embed, purge_vector_orphans) are identical to the PDF function. +fn ingest_one_code_asset( + app: &App, + asset: &RawAsset, + chunk_policy: &ChunkPolicy, + embedder: Option<&Arc>, + vector_store: Option<&Arc>, + existing_doc_ids: &std::collections::HashSet, + force_reingest: bool, +) -> anyhow::Result { + let path = match &asset.source_uri { + SourceUri::File(p) => p.clone(), + SourceUri::Kb(_) => { + return Ok(kebab_core::IngestItem { + kind: kebab_core::IngestItemKind::Skipped, + doc_id: None, + doc_path: asset.workspace_path.clone(), + asset_id: Some(asset.asset_id.clone()), + byte_len: Some(asset.byte_len), + block_count: None, + chunk_count: None, + parser_version: None, + chunker_version: None, + warnings: vec![ + "kb:// URI not yet supported".to_string(), + ], + error: None, + }); + } + }; + // p10-1A-2 task 8: incremental-ingest early-skip for the code flow. + // Code docs use `code-rust-v1` as the parser_version and + // `CodeRustAstV1Chunker` as the chunker — both pinned per-medium + // today (no config knob). + let code_parser_version = + ParserVersion(kebab_parse_code::RUST_PARSER_VERSION.to_string()); + if let Some(item) = try_skip_unchanged( + app, + asset, + &code_parser_version, + &CodeRustAstV1Chunker.chunker_version(), + embedder.map(|e| e.model_version()).as_ref(), + force_reingest, + )? { + return Ok(item); + } + let bytes = std::fs::read(&path) + .with_context(|| format!("read code asset bytes from {}", path.display()))?; + + let extract_config = kebab_core::ExtractConfig::default(); + let workspace_root = app.config.resolve_workspace_root(); + let ctx = ExtractContext { + asset, + workspace_root: &workspace_root, + config: &extract_config, + }; + let mut canonical = RustAstExtractor::new() + .extract(&ctx, &bytes) + .context("kb-parse-code::RustAstExtractor::extract")?; + + // Per-medium chunker selection: Rust code always uses code-rust-ast-v1 + // regardless of `config.chunking.chunker_version`. + let chunker = CodeRustAstV1Chunker; + let chunks = chunker + .chunk(&canonical, chunk_policy) + .context("kb-chunk::CodeRustAstV1Chunker::chunk")?; + + // Stamp chunker + embedding versions so incremental skip detection has + // data on the second run. + canonical.last_chunker_version = Some(chunker.chunker_version()); + if let Some(emb) = embedder { + canonical.last_embedding_version = Some(emb.model_version()); + } + + purge_vector_orphans_for_workspace_path(app, asset, vector_store)?; + app.sqlite + .put_asset_with_bytes(asset, &bytes) + .context("DocumentStore::put_asset_with_bytes (code)")?; + app.sqlite + .put_document(&canonical) + .context("DocumentStore::put_document (code)")?; + app.sqlite + .put_blocks(&canonical.doc_id, &canonical.blocks) + .context("DocumentStore::put_blocks (code)")?; + app.sqlite + .put_chunks(&canonical.doc_id, &chunks) + .context("DocumentStore::put_chunks (code)")?; + + if let (Some(emb), Some(vec_store)) = (embedder, vector_store) + && !chunks.is_empty() + { + let inputs: Vec> = chunks + .iter() + .map(|c| EmbeddingInput { + text: c.text.as_str(), + kind: EmbeddingKind::Document, + }) + .collect(); + let vectors = emb + .embed(&inputs) + .context("Embedder::embed (code chunks)")?; + let model_id = emb.model_id(); + let model_version = emb.model_version(); + let dimensions = emb.dimensions(); + let records: Vec = chunks + .iter() + .zip(vectors) + .map(|(c, v)| VectorRecord { + embedding_id: kebab_core::id_for_embedding( + &c.chunk_id, + &model_id, + &model_version, + dimensions, + ), + chunk_id: c.chunk_id.clone(), + vector: v, + doc_id: canonical.doc_id.clone(), + text: c.text.clone(), + heading_path: c.heading_path.clone(), + model_id: model_id.clone(), + model_version: model_version.clone(), + dimensions, + }) + .collect(); + vec_store + .upsert(&records) + .context("VectorStore::upsert (code)")?; + } + + let kind = if existing_doc_ids.contains(&canonical.doc_id.0) { + kebab_core::IngestItemKind::Updated + } else { + kebab_core::IngestItemKind::New + }; + + // Surface every `Provenance::Warning` note onto `IngestItem.warnings`. + let warnings: Vec = canonical + .provenance + .events + .iter() + .filter(|e| e.kind == kebab_core::ProvenanceKind::Warning) + .filter_map(|e| e.note.clone()) + .collect(); + + Ok(kebab_core::IngestItem { + kind, + doc_id: Some(canonical.doc_id.clone()), + doc_path: asset.workspace_path.clone(), + asset_id: Some(asset.asset_id.clone()), + byte_len: Some(asset.byte_len), + block_count: u32::try_from(canonical.blocks.len()).ok(), + chunk_count: u32::try_from(chunks.len()).ok(), + parser_version: Some(canonical.parser_version.clone()), + chunker_version: Some(chunker.chunker_version()), + warnings, + error: None, + }) +} + /// Pull the BCP-47 language hint from the canonical document. P6-1 /// stamps `Lang("und")` by default; image-pipeline OCR / caption /// adapters special-case "und" so the hint is intentionally dropped diff --git a/crates/kebab-app/tests/code_ingest_smoke.rs b/crates/kebab-app/tests/code_ingest_smoke.rs new file mode 100644 index 0000000..baf0ae4 --- /dev/null +++ b/crates/kebab-app/tests/code_ingest_smoke.rs @@ -0,0 +1,142 @@ +//! p10-1A-2 Task 8: smoke test for Rust code ingest dispatch. +//! +//! Writes a single `.rs` file into a TempDir workspace, ingests it via +//! `kebab_app::ingest_with_config`, then searches for the symbol name and +//! asserts that the resulting `SearchHit` carries a `Citation::Code` +//! with the expected `lang`, `symbol`, and `line_start`. +//! +//! Mirrors the `pdf_pipeline.rs` harness: lexical-only (no AVX/fastembed), +//! no OCR / caption adapters needed. + +mod common; + +use common::{TestEnv, lexical_query}; + +use kebab_core::{Citation, IngestItemKind}; + +/// A `.rs` file with a single `pub fn add` symbol is ingested, and a +/// lexical search for "add" must return at least one `Citation::Code` +/// hit whose `lang == "rust"`, `symbol == Some("add")`, and +/// `line_start >= 1`. +#[test] +fn rust_file_ingests_and_searches_as_code_citation() { + let env = TestEnv::lexical_only(); + + // Write a minimal Rust file into the workspace root. + std::fs::write( + env.workspace_root.join("demo.rs"), + "/// adds two integers\npub fn add(a: i32, b: i32) -> i32 {\n a + b\n}\n", + ) + .unwrap(); + + let report = + kebab_app::ingest_with_config(env.config.clone(), env.scope(), false) + .expect("ingest must succeed"); + + assert_eq!(report.errors, 0, "no errors expected: {report:?}"); + let items = report.items.as_ref().expect("items present"); + let code_item = items + .iter() + .find(|i| i.doc_path.0.ends_with("demo.rs")) + .expect("demo.rs item present"); + assert_eq!( + code_item.kind, + IngestItemKind::New, + "first ingest must be New: {code_item:?}" + ); + assert!( + code_item.block_count.unwrap_or(0) >= 1, + "at least one block expected: {code_item:?}" + ); + assert!( + code_item.chunk_count.unwrap_or(0) >= 1, + "at least one chunk expected: {code_item:?}" + ); + assert_eq!( + code_item.parser_version.as_ref().map(|p| p.0.as_str()), + Some("code-rust-v1"), + "parser_version must be code-rust-v1" + ); + assert_eq!( + code_item.chunker_version.as_ref().map(|c| c.0.as_str()), + Some("code-rust-ast-v1"), + "chunker_version must be code-rust-ast-v1" + ); + + // Lexical search for the symbol name "add". + let hits = kebab_app::search_with_config(env.config.clone(), lexical_query("add")) + .expect("search must succeed"); + + let h = hits + .iter() + .find(|h| matches!(&h.citation, Citation::Code { .. })) + .expect("at least one Citation::Code hit for 'add'"); + + match &h.citation { + Citation::Code { + lang, + symbol, + line_start, + .. + } => { + assert_eq!( + lang.as_deref(), + Some("rust"), + "citation.lang must be 'rust'" + ); + assert_eq!( + symbol.as_deref(), + Some("add"), + "citation.symbol must be 'add'" + ); + assert!(*line_start >= 1, "line_start must be ≥1"); + } + _ => unreachable!(), + } + + assert_eq!( + h.code_lang.as_deref(), + Some("rust"), + "SearchHit.code_lang must be 'rust'" + ); +} + +/// Re-ingesting the same `.rs` file without changes must report +/// `Unchanged` (incremental-skip path exercised). +#[test] +fn rust_file_re_ingest_is_unchanged() { + let env = TestEnv::lexical_only(); + + std::fs::write( + env.workspace_root.join("stable.rs"), + "pub fn noop() {}\n", + ) + .unwrap(); + + let r1 = + kebab_app::ingest_with_config(env.config.clone(), env.scope(), false).unwrap(); + let item1 = r1 + .items + .as_ref() + .unwrap() + .iter() + .find(|i| i.doc_path.0.ends_with("stable.rs")) + .cloned() + .unwrap(); + assert_eq!(item1.kind, IngestItemKind::New); + + let r2 = + kebab_app::ingest_with_config(env.config.clone(), env.scope(), false).unwrap(); + let item2 = r2 + .items + .unwrap() + .into_iter() + .find(|i| i.doc_path.0.ends_with("stable.rs")) + .unwrap(); + assert_eq!( + item2.kind, + IngestItemKind::Unchanged, + "identical bytes → Unchanged" + ); + assert_eq!(item2.doc_id, item1.doc_id); +} -- 2.49.1 From b5d1fe8c1eaa07772cc30be5c007477798f2a9ec Mon Sep 17 00:00:00 2001 From: altair823 Date: Tue, 19 May 2026 21:13:01 +0000 Subject: [PATCH 13/19] feat(p10-1a-2): backfill SearchHit.repo from doc metadata (Task 8b) Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/kebab-app/src/app.rs | 51 +++++++++++++++++- crates/kebab-app/tests/code_ingest_smoke.rs | 58 +++++++++++++++++++++ 2 files changed, 107 insertions(+), 2 deletions(-) diff --git a/crates/kebab-app/src/app.rs b/crates/kebab-app/src/app.rs index 2202cd4..792016c 100644 --- a/crates/kebab-app/src/app.rs +++ b/crates/kebab-app/src/app.rs @@ -40,8 +40,8 @@ use anyhow::{Context, Result, anyhow}; use lru::LruCache; use kebab_core::{ - Answer, Embedder, IndexVersion, LanguageModel, Retriever, SearchHit, SearchMode, - SearchOpts, SearchQuery, VectorStore, + Answer, DocumentStore, Embedder, IndexVersion, LanguageModel, Retriever, SearchHit, + SearchMode, SearchOpts, SearchQuery, VectorStore, }; use kebab_embed_local::FastembedEmbedder; use kebab_llm_local::OllamaLanguageModel; @@ -301,6 +301,10 @@ impl App { // `code_lang: None`; we own the post-processing here in kebab-app // and can fill it cheaply from data already present in the hit. backfill_code_lang(&mut hits); + // p10-1A-2 Task 8b: backfill `repo` from the document's + // `Metadata.repo`. Unlike `code_lang`, this cannot be derived from + // the Citation alone — it requires a store lookup by `doc_id`. + self.backfill_repo(&mut hits); Ok(hits) } @@ -394,6 +398,8 @@ impl App { ); // p10-1A-2: backfill code_lang — same as search_uncached. backfill_code_lang(&mut traced_hits); + // p10-1A-2 Task 8b: backfill repo — same as search_uncached. + self.backfill_repo(&mut traced_hits); // Apply offset + k_effective truncation (mirrors non-trace path). let drop_n = offset.min(traced_hits.len()); @@ -784,6 +790,47 @@ impl App { } } + /// p10-1A-2 Task 8b: back-fill `SearchHit.repo` from the originating + /// document's `Metadata.repo` for every hit whose `repo` field is + /// currently `None`. The search layer (kebab-search) constructs hits + /// with `repo: None` because it has no store access; we fill it here + /// in kebab-app post-retrieval via a per-distinct-`doc_id` store lookup. + /// + /// Deduplication: a small `HashMap` accumulates the + /// `(doc_id → Option)` mapping so each unique document is + /// fetched at most once. Search result sets are small (default k ≤ 20), + /// so the map overhead is negligible. A `None` entry is cached too + /// (document not found or no repo in metadata) to avoid re-querying. + /// + /// Non-repo documents (markdown, PDF, plain text, code files outside a + /// git tree) correctly keep `repo: None` — `Metadata.repo` is already + /// `None` for those, so the assignment is a no-op. + fn backfill_repo(&self, hits: &mut [SearchHit]) { + use std::collections::HashMap; + use kebab_core::DocumentId; + + // doc_id → Option where None means "not found / no repo" + let mut cache: HashMap> = HashMap::new(); + + for hit in hits.iter_mut() { + if hit.repo.is_some() { + continue; + } + let repo_val = cache + .entry(hit.doc_id.clone()) + .or_insert_with(|| { + self.sqlite + .get_document(&hit.doc_id) + .ok() + .flatten() + .and_then(|doc| doc.metadata.repo) + }); + if let Some(r) = repo_val { + hit.repo = Some(r.clone()); + } + } + } + /// Resolve the embedder + vector store, surfacing the user-friendly /// "switch to --mode lexical" error when embeddings are disabled. fn require_embeddings( diff --git a/crates/kebab-app/tests/code_ingest_smoke.rs b/crates/kebab-app/tests/code_ingest_smoke.rs index baf0ae4..d6611f1 100644 --- a/crates/kebab-app/tests/code_ingest_smoke.rs +++ b/crates/kebab-app/tests/code_ingest_smoke.rs @@ -101,6 +101,64 @@ fn rust_file_ingests_and_searches_as_code_citation() { ); } +/// p10-1A-2 Task 8b: a code search hit must carry `SearchHit.repo` filled +/// from the document's `Metadata.repo` (which is set by `detect_repo` during +/// ingest). `detect_repo` returns the name of the directory that contains +/// `.git/`, so we `git init` the workspace root before ingesting and then +/// assert that `h.repo == Some("workspace")`. +#[test] +fn rust_code_search_hit_has_repo() { + let env = TestEnv::lexical_only(); + + // `detect_repo` walks up from the file looking for `.git/`. + // Initialise a bare git repo at the workspace root so it is + // discoverable. We only need the `.git/` directory — no commits + // required. + let git_status = std::process::Command::new("git") + .args(["init", "--quiet"]) + .arg(env.workspace_root.as_os_str()) + .status() + .expect("git init"); + assert!(git_status.success(), "git init must succeed"); + + std::fs::write( + env.workspace_root.join("repo_demo.rs"), + "/// multiplies two integers\npub fn mul(a: i32, b: i32) -> i32 {\n a * b\n}\n", + ) + .unwrap(); + + let report = + kebab_app::ingest_with_config(env.config.clone(), env.scope(), false) + .expect("ingest must succeed"); + assert_eq!(report.errors, 0, "no ingest errors: {report:?}"); + + let hits = kebab_app::search_with_config(env.config.clone(), lexical_query("mul")) + .expect("search must succeed"); + + let h = hits + .iter() + .find(|h| matches!(&h.citation, Citation::Code { .. })) + .expect("at least one Citation::Code hit for 'mul'"); + + // The workspace root directory is named "workspace" by `TestEnv`. + let expected_repo = env + .workspace_root + .file_name() + .and_then(|n| n.to_str()) + .map(str::to_owned); + assert_eq!( + h.repo, + expected_repo, + "SearchHit.repo must match the workspace dir name (detect_repo result)" + ); + // Also sanity-check code_lang is still filled. + assert_eq!( + h.code_lang.as_deref(), + Some("rust"), + "SearchHit.code_lang must be 'rust'" + ); +} + /// Re-ingesting the same `.rs` file without changes must report /// `Unchanged` (incremental-skip path exercised). #[test] -- 2.49.1 From 11a0fc758f2cdb65c10e398de54cf40fcfec8a8e Mon Sep 17 00:00:00 2001 From: altair823 Date: Tue, 19 May 2026 21:20:13 +0000 Subject: [PATCH 14/19] docs(p10-1a-2): note backfill invariant at search_with_opts non-trace path (Task 8 review) Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/kebab-app/src/app.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/kebab-app/src/app.rs b/crates/kebab-app/src/app.rs index 792016c..ff075f5 100644 --- a/crates/kebab-app/src/app.rs +++ b/crates/kebab-app/src/app.rs @@ -426,6 +426,9 @@ impl App { }); } + // backfill_code_lang + backfill_repo are applied inside `search` + // via `search_uncached` — no explicit call needed here. Trace + // branch above calls them directly because it bypasses `search`. let mut all_hits = self.search(fetch_query)?; // Skip offset. -- 2.49.1 From da51e59081dcc70cc27a11a7f391a2a7e031877a Mon Sep 17 00:00:00 2001 From: altair823 Date: Tue, 19 May 2026 21:41:52 +0000 Subject: [PATCH 15/19] feat(p10-1a-2): populate schema.v1 code_lang_breakdown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add `SqliteStore::code_lang_breakdown()` that queries `json_extract(metadata_json, '$.code_lang')`, groups by it, and skips NULL rows — returning `BTreeMap`. Wire it into `collect_stats` in `kebab-app::schema`, replacing the `BTreeMap::new()` placeholder inserted by 1A-1. Test: `store::tests::code_lang_breakdown_counts_by_code_lang` asserts rust=1 and that a null-code_lang doc does NOT appear in the map. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/kebab-app/src/schema.rs | 4 +- crates/kebab-store-sqlite/src/store.rs | 107 +++++++++++++++++++++++++ 2 files changed, 109 insertions(+), 2 deletions(-) diff --git a/crates/kebab-app/src/schema.rs b/crates/kebab-app/src/schema.rs index 866c714..ab7d144 100644 --- a/crates/kebab-app/src/schema.rs +++ b/crates/kebab-app/src/schema.rs @@ -166,8 +166,8 @@ fn collect_stats( lang_breakdown: counts.lang_breakdown, index_bytes, stale_doc_count: counts.stale_doc_count, - // p10-1A-1: populated by 1A-2 code ingest; empty until then. - code_lang_breakdown: std::collections::BTreeMap::new(), + // p10-1A-2: populated by the store query added in this task. + code_lang_breakdown: store.code_lang_breakdown()?, repo_breakdown: std::collections::BTreeMap::new(), }) } diff --git a/crates/kebab-store-sqlite/src/store.rs b/crates/kebab-store-sqlite/src/store.rs index 57e16da..89dce6a 100644 --- a/crates/kebab-store-sqlite/src/store.rs +++ b/crates/kebab-store-sqlite/src/store.rs @@ -669,6 +669,38 @@ impl SqliteStore { ) -> anyhow::Result { self.count_summary_inner(threshold_days) } + + /// p10-1A-2: per-code-language doc count for `schema.v1`. + /// + /// Reads `metadata_json->'$.code_lang'`, groups by the value, and + /// skips rows where `code_lang` is NULL (i.e. non-code documents). + /// Returns `BTreeMap` — key is the canonical lowercase + /// language identifier (e.g. `"rust"`), value is the doc count. + pub fn code_lang_breakdown( + &self, + ) -> anyhow::Result> { + use anyhow::Context; + let conn = self.read_conn(); + let mut stmt = conn + .prepare( + "SELECT json_extract(metadata_json, '$.code_lang') AS cl, COUNT(*) \ + FROM documents \ + WHERE cl IS NOT NULL \ + GROUP BY cl", + ) + .context("prepare code_lang_breakdown")?; + let rows = stmt + .query_map([], |r| { + Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)? as u32)) + }) + .context("query code_lang_breakdown")?; + let mut out = std::collections::BTreeMap::new(); + for row in rows { + let (k, v) = row.context("read code_lang_breakdown row")?; + out.insert(k, v); + } + Ok(out) + } } /// Apply the design §5 / task-spec pragmas. Called once per connection. @@ -710,5 +742,80 @@ mod tests { assert!(s.lang_breakdown.is_empty()); assert_eq!(s.stale_doc_count, 0); } + + /// p10-1A-2: `code_lang_breakdown` counts docs by `metadata_json.code_lang`. + /// + /// Inserts: + /// - one doc with `code_lang = "rust"` → must appear with count 1 + /// - one doc with `code_lang = null` → must NOT appear (NULL skipped) + /// + /// Uses a side rusqlite connection that bypasses the `assets` FK via + /// `PRAGMA foreign_keys = OFF` so the test is self-contained. + #[test] + fn code_lang_breakdown_counts_by_code_lang() { + let (dir, store) = open_fresh_store(); + + // Insert two document rows directly. Disabling FK enforcement lets + // us skip the companion `assets` insert. + let db_path = dir.path().join("kebab.sqlite"); + let conn = rusqlite::Connection::open(&db_path).unwrap(); + conn.pragma_update(None, "foreign_keys", "OFF").unwrap(); + + // Doc 1: Rust code file — code_lang = "rust" + conn.execute( + "INSERT INTO documents ( + doc_id, asset_id, workspace_path, + source_type, trust_level, parser_version, + doc_version, schema_version, + metadata_json, provenance_json, + created_at, updated_at + ) VALUES ( + 'doc-rust-1', 'asset-1', 'src/main.rs', + 'reference', 'primary', 'test-v1', + 1, 1, + '{\"code_lang\":\"rust\"}', '{}', + '2024-01-01T00:00:00Z', '2024-01-01T00:00:00Z' + )", + [], + ) + .unwrap(); + + // Doc 2: Markdown doc — code_lang absent (null in JSON) + conn.execute( + "INSERT INTO documents ( + doc_id, asset_id, workspace_path, + source_type, trust_level, parser_version, + doc_version, schema_version, + metadata_json, provenance_json, + created_at, updated_at + ) VALUES ( + 'doc-md-1', 'asset-2', 'notes/readme.md', + 'markdown', 'primary', 'test-v1', + 1, 1, + '{\"code_lang\":null}', '{}', + '2024-01-01T00:00:00Z', '2024-01-01T00:00:00Z' + )", + [], + ) + .unwrap(); + + drop(conn); // release side connection before querying via store + + let bd = store.code_lang_breakdown().unwrap(); + + // rust must appear with count 1 + assert_eq!( + bd.get("rust"), + Some(&1u32), + "expected rust=1 in code_lang_breakdown, got: {bd:?}" + ); + // null code_lang must NOT appear as any key + assert!( + !bd.contains_key("null"), + "null code_lang must not appear in breakdown, got: {bd:?}" + ); + // only one key total + assert_eq!(bd.len(), 1, "expected exactly 1 entry, got: {bd:?}"); + } } -- 2.49.1 From 97e9f558f40e877807412aa7140e20db5023d69c Mon Sep 17 00:00:00 2001 From: altair823 Date: Tue, 19 May 2026 22:14:57 +0000 Subject: [PATCH 16/19] test(p10-1a-2): code-rust-ast-v1 chunker snapshot + full-suite gate Co-Authored-By: Claude Opus 4.7 (1M context) --- .../tests/code_rust_ast_snapshot.rs | 221 ++++++++++++++++++ .../fixtures/code-sample.chunks.snapshot.json | 170 ++++++++++++++ 2 files changed, 391 insertions(+) create mode 100644 crates/kebab-chunk/tests/code_rust_ast_snapshot.rs create mode 100644 crates/kebab-chunk/tests/fixtures/code-sample.chunks.snapshot.json diff --git a/crates/kebab-chunk/tests/code_rust_ast_snapshot.rs b/crates/kebab-chunk/tests/code_rust_ast_snapshot.rs new file mode 100644 index 0000000..9ef4455 --- /dev/null +++ b/crates/kebab-chunk/tests/code_rust_ast_snapshot.rs @@ -0,0 +1,221 @@ +//! Snapshot test pinning the `Vec` JSON for a +//! representative Rust code `CanonicalDocument`. +//! +//! This is an integration test. `kebab-parse-code` is intentionally NOT +//! a dev-dep (design §6.3 / §8 boundary: AST extraction is parser-side). +//! The `CanonicalDocument` is built inline from hand-crafted `Block::Code` +//! units, which is the same pattern used in `code_rust_ast_v1.rs`'s +//! internal `code_doc` test helper. +//! +//! Set `UPDATE_SNAPSHOTS=1` to re-bake the baseline. + +use std::path::PathBuf; + +use kebab_chunk::CodeRustAstV1Chunker; +use kebab_core::{ + AssetId, Block, CanonicalDocument, ChunkPolicy, Chunker, ChunkerVersion, CodeBlock, CommonBlock, + Lang, Metadata, ParserVersion, Provenance, SourceSpan, SourceType, TrustLevel, WorkspacePath, + id_for_block, id_for_doc, +}; +use serde_json::Value; +use time::OffsetDateTime; + +fn fixtures_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests") + .join("fixtures") +} + +fn fixed_doc() -> CanonicalDocument { + let wp = WorkspacePath("crates/kebab-chunk/src/code_rust_ast_v1.rs".into()); + let aid = AssetId("b".repeat(64)); + // Pin parser_version so doc_id / block_ids are reproducible. + let pv = ParserVersion("code-rust-v1".into()); + let doc_id = id_for_doc(&wp, &aid, &pv); + + // Build a >200-line function body to force split_oversize. + let big_body: String = { + let header = "pub fn big_fn(input: &[u8]) -> Vec {\n"; + let body: String = (0..210u32) + .map(|i| format!(" let v{i} = input.get({i} as usize).copied().unwrap_or(0);\n")) + .collect(); + let footer = " vec![0u8]\n}"; + format!("{header}{body}{footer}") + }; + let big_line_count = big_body.lines().count() as u32; + let big_line_end = 48 + big_line_count - 1; + + // Representative units: + // 0. top-level use+const block (lines 1–5, ≤200) + // 1. free fn `parse` (lines 7–12, ≤200) + // 2. struct `Foo` (lines 14–20, ≤200) + // 3. trait `Frobable` (lines 22–30, ≤200) + // 4. impl Foo::double (lines 32–38, ≤200) + // 5. impl Foo::triple (lines 40–46, ≤200) + // 6. big_fn (>200 lines) to force split_oversize + let raw_units: Vec<(&str, u32, u32, String)> = vec![ + ( + "use+const", + 1, + 5, + "use std::collections::HashMap;\nuse std::fmt;\n\nconst MAX: usize = 1024;\nconst MIN: usize = 0;".to_string(), + ), + ( + "parse", + 7, + 12, + "pub fn parse(input: &str) -> Option {\n input\n .trim()\n .parse()\n .ok()\n}".to_string(), + ), + ( + "Foo", + 14, + 20, + "pub struct Foo {\n pub name: String,\n pub value: u32,\n pub tags: Vec,\n pub meta: Option,\n pub count: usize,\n}".to_string(), + ), + ( + "Frobable", + 22, + 30, + "pub trait Frobable {\n fn frob(&self) -> String;\n fn frob_twice(&self) -> String {\n let a = self.frob();\n let b = self.frob();\n format!(\"{a}{b}\")\n }\n fn name(&self) -> &str;\n}".to_string(), + ), + ( + "Foo::double", + 32, + 38, + "impl Foo {\n pub fn double(&self) -> u32 {\n self.value\n .checked_mul(2)\n .unwrap_or(u32::MAX)\n }\n}".to_string(), + ), + ( + "Foo::triple", + 40, + 46, + "impl Foo {\n pub fn triple(&self) -> u32 {\n self.value\n .checked_mul(3)\n .unwrap_or(u32::MAX)\n }\n}".to_string(), + ), + ("big_fn", 48, big_line_end, big_body), + ]; + + let blocks: Vec = raw_units + .iter() + .enumerate() + .map(|(i, (sym, ls, le, code))| { + let span = SourceSpan::Code { + line_start: *ls, + line_end: *le, + symbol: Some((*sym).to_string()), + lang: Some("rust".into()), + }; + let bid = id_for_block(&doc_id, "code", &[], i as u32, &span); + Block::Code(CodeBlock { + common: CommonBlock { + block_id: bid, + heading_path: vec![], + source_span: span, + }, + lang: Some("rust".into()), + code: code.clone(), + }) + }) + .collect(); + + CanonicalDocument { + doc_id, + source_asset_id: aid, + workspace_path: wp, + title: "code_rust_ast_v1.rs".into(), + lang: Lang("und".into()), + blocks, + metadata: Metadata { + aliases: vec![], + tags: vec![], + created_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(), + updated_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(), + source_type: SourceType::Note, + trust_level: TrustLevel::Primary, + user_id_alias: None, + user: Default::default(), + repo: Some("kebab".into()), + git_branch: Some("main".into()), + git_commit: Some("0".repeat(40)), + code_lang: Some("rust".into()), + }, + provenance: Provenance { events: vec![] }, + parser_version: pv, + schema_version: 1, + doc_version: 1, + last_chunker_version: None, + last_embedding_version: None, + } +} + +fn fixed_policy() -> ChunkPolicy { + ChunkPolicy { + target_tokens: 500, + overlap_tokens: 80, + respect_markdown_headings: false, + chunker_version: ChunkerVersion("code-rust-ast-v1".into()), + } +} + +#[test] +fn code_rust_ast_chunks_snapshot() { + let doc = fixed_doc(); + let policy = fixed_policy(); + + let chunks = CodeRustAstV1Chunker.chunk(&doc, &policy).expect("chunk"); + let actual = serde_json::to_value(&chunks).unwrap(); + + let dir = fixtures_dir(); + let baseline_path = dir.join("code-sample.chunks.snapshot.json"); + let baseline_text = match std::fs::read_to_string(&baseline_path) { + Ok(s) => s, + Err(_) if std::env::var("UPDATE_SNAPSHOTS").is_ok() => { + std::fs::create_dir_all(&dir).unwrap(); + let pretty = serde_json::to_string_pretty(&actual).unwrap(); + std::fs::write(&baseline_path, format!("{pretty}\n")).unwrap(); + return; + } + Err(e) => panic!( + "missing baseline {}; run with UPDATE_SNAPSHOTS=1 to create: {e}", + baseline_path.display() + ), + }; + let expected: Value = serde_json::from_str(&baseline_text).expect("baseline parses as json"); + + if actual != expected { + if std::env::var("UPDATE_SNAPSHOTS").is_ok() { + let pretty = serde_json::to_string_pretty(&actual).unwrap(); + std::fs::write(&baseline_path, format!("{pretty}\n")).unwrap(); + eprintln!("updated baseline {}", baseline_path.display()); + return; + } + let pretty = serde_json::to_string_pretty(&actual).unwrap(); + panic!( + "code-rust-ast-v1 chunks snapshot drift\n\ + --- expected ({}) ---\n{baseline_text}\n\ + --- actual ---\n{pretty}\n\ + If intentional, re-run with UPDATE_SNAPSHOTS=1.", + baseline_path.display() + ); + } +} + +/// Determinism cross-check: re-running the same pipeline yields the same +/// chunk_ids byte-for-byte. +#[test] +fn code_rust_ast_chunks_are_deterministic() { + let policy = fixed_policy(); + let baseline: Vec = CodeRustAstV1Chunker + .chunk(&fixed_doc(), &policy) + .unwrap() + .into_iter() + .map(|c| c.chunk_id.0) + .collect(); + for _ in 0..5 { + let again: Vec = CodeRustAstV1Chunker + .chunk(&fixed_doc(), &policy) + .unwrap() + .into_iter() + .map(|c| c.chunk_id.0) + .collect(); + assert_eq!(again, baseline); + } +} diff --git a/crates/kebab-chunk/tests/fixtures/code-sample.chunks.snapshot.json b/crates/kebab-chunk/tests/fixtures/code-sample.chunks.snapshot.json new file mode 100644 index 0000000..f1c69be --- /dev/null +++ b/crates/kebab-chunk/tests/fixtures/code-sample.chunks.snapshot.json @@ -0,0 +1,170 @@ +[ + { + "block_ids": [ + "7a43438772cdada66439790d2b5bed52" + ], + "chunk_id": "e15e12ab50571a649d3125230a110418", + "chunker_version": "code-rust-ast-v1", + "doc_id": "2e30aba9077a15e6fadd2881b41180ad", + "heading_path": [], + "policy_hash": "f5359c99c7a7d273", + "source_spans": [ + { + "kind": "code", + "lang": "rust", + "line_end": 5, + "line_start": 1, + "symbol": "use+const" + } + ], + "text": "use std::collections::HashMap;\nuse std::fmt;\n\nconst MAX: usize = 1024;\nconst MIN: usize = 0;", + "token_estimate": 31 + }, + { + "block_ids": [ + "b362849d469e23a4939022720ecb53d6" + ], + "chunk_id": "3dd2e2e5b1d083838da173852c456bd9", + "chunker_version": "code-rust-ast-v1", + "doc_id": "2e30aba9077a15e6fadd2881b41180ad", + "heading_path": [], + "policy_hash": "f5359c99c7a7d273", + "source_spans": [ + { + "kind": "code", + "lang": "rust", + "line_end": 12, + "line_start": 7, + "symbol": "parse" + } + ], + "text": "pub fn parse(input: &str) -> Option {\n input\n .trim()\n .parse()\n .ok()\n}", + "token_estimate": 34 + }, + { + "block_ids": [ + "f4ad850ca5808ab8b6cc4f06d489cfc6" + ], + "chunk_id": "ae1593b190c37754b6b5e0d6496107fe", + "chunker_version": "code-rust-ast-v1", + "doc_id": "2e30aba9077a15e6fadd2881b41180ad", + "heading_path": [], + "policy_hash": "f5359c99c7a7d273", + "source_spans": [ + { + "kind": "code", + "lang": "rust", + "line_end": 20, + "line_start": 14, + "symbol": "Foo" + } + ], + "text": "pub struct Foo {\n pub name: String,\n pub value: u32,\n pub tags: Vec,\n pub meta: Option,\n pub count: usize,\n}", + "token_estimate": 47 + }, + { + "block_ids": [ + "88ce619db53971c7f384769d96277c65" + ], + "chunk_id": "03f02c87f81990cca3390d66925b1a78", + "chunker_version": "code-rust-ast-v1", + "doc_id": "2e30aba9077a15e6fadd2881b41180ad", + "heading_path": [], + "policy_hash": "f5359c99c7a7d273", + "source_spans": [ + { + "kind": "code", + "lang": "rust", + "line_end": 30, + "line_start": 22, + "symbol": "Frobable" + } + ], + "text": "pub trait Frobable {\n fn frob(&self) -> String;\n fn frob_twice(&self) -> String {\n let a = self.frob();\n let b = self.frob();\n format!(\"{a}{b}\")\n }\n fn name(&self) -> &str;\n}", + "token_estimate": 69 + }, + { + "block_ids": [ + "47ca198facaf74c1959ac8b8ceb5ab2a" + ], + "chunk_id": "d6390ef0becde6d508b8812a617f9006", + "chunker_version": "code-rust-ast-v1", + "doc_id": "2e30aba9077a15e6fadd2881b41180ad", + "heading_path": [], + "policy_hash": "f5359c99c7a7d273", + "source_spans": [ + { + "kind": "code", + "lang": "rust", + "line_end": 38, + "line_start": 32, + "symbol": "Foo::double" + } + ], + "text": "impl Foo {\n pub fn double(&self) -> u32 {\n self.value\n .checked_mul(2)\n .unwrap_or(u32::MAX)\n }\n}", + "token_estimate": 44 + }, + { + "block_ids": [ + "cc16070e62953f7ec6aebff00db0f21d" + ], + "chunk_id": "64c748e790199586fab6fbf59b81d169", + "chunker_version": "code-rust-ast-v1", + "doc_id": "2e30aba9077a15e6fadd2881b41180ad", + "heading_path": [], + "policy_hash": "f5359c99c7a7d273", + "source_spans": [ + { + "kind": "code", + "lang": "rust", + "line_end": 46, + "line_start": 40, + "symbol": "Foo::triple" + } + ], + "text": "impl Foo {\n pub fn triple(&self) -> u32 {\n self.value\n .checked_mul(3)\n .unwrap_or(u32::MAX)\n }\n}", + "token_estimate": 44 + }, + { + "block_ids": [ + "e03092fec8a585435fd3f077df76503f" + ], + "chunk_id": "2f0d20bd50585f8d82610856d954a7d3", + "chunker_version": "code-rust-ast-v1", + "doc_id": "2e30aba9077a15e6fadd2881b41180ad", + "heading_path": [], + "policy_hash": "f5359c99c7a7d273", + "source_spans": [ + { + "kind": "code", + "lang": "rust", + "line_end": 247, + "line_start": 48, + "symbol": "big_fn [part 1/2]" + } + ], + "text": "pub fn big_fn(input: &[u8]) -> Vec {\n let v0 = input.get(0 as usize).copied().unwrap_or(0);\n let v1 = input.get(1 as usize).copied().unwrap_or(0);\n let v2 = input.get(2 as usize).copied().unwrap_or(0);\n let v3 = input.get(3 as usize).copied().unwrap_or(0);\n let v4 = input.get(4 as usize).copied().unwrap_or(0);\n let v5 = input.get(5 as usize).copied().unwrap_or(0);\n let v6 = input.get(6 as usize).copied().unwrap_or(0);\n let v7 = input.get(7 as usize).copied().unwrap_or(0);\n let v8 = input.get(8 as usize).copied().unwrap_or(0);\n let v9 = input.get(9 as usize).copied().unwrap_or(0);\n let v10 = input.get(10 as usize).copied().unwrap_or(0);\n let v11 = input.get(11 as usize).copied().unwrap_or(0);\n let v12 = input.get(12 as usize).copied().unwrap_or(0);\n let v13 = input.get(13 as usize).copied().unwrap_or(0);\n let v14 = input.get(14 as usize).copied().unwrap_or(0);\n let v15 = input.get(15 as usize).copied().unwrap_or(0);\n let v16 = input.get(16 as usize).copied().unwrap_or(0);\n let v17 = input.get(17 as usize).copied().unwrap_or(0);\n let v18 = input.get(18 as usize).copied().unwrap_or(0);\n let v19 = input.get(19 as usize).copied().unwrap_or(0);\n let v20 = input.get(20 as usize).copied().unwrap_or(0);\n let v21 = input.get(21 as usize).copied().unwrap_or(0);\n let v22 = input.get(22 as usize).copied().unwrap_or(0);\n let v23 = input.get(23 as usize).copied().unwrap_or(0);\n let v24 = input.get(24 as usize).copied().unwrap_or(0);\n let v25 = input.get(25 as usize).copied().unwrap_or(0);\n let v26 = input.get(26 as usize).copied().unwrap_or(0);\n let v27 = input.get(27 as usize).copied().unwrap_or(0);\n let v28 = input.get(28 as usize).copied().unwrap_or(0);\n let v29 = input.get(29 as usize).copied().unwrap_or(0);\n let v30 = input.get(30 as usize).copied().unwrap_or(0);\n let v31 = input.get(31 as usize).copied().unwrap_or(0);\n let v32 = input.get(32 as usize).copied().unwrap_or(0);\n let v33 = input.get(33 as usize).copied().unwrap_or(0);\n let v34 = input.get(34 as usize).copied().unwrap_or(0);\n let v35 = input.get(35 as usize).copied().unwrap_or(0);\n let v36 = input.get(36 as usize).copied().unwrap_or(0);\n let v37 = input.get(37 as usize).copied().unwrap_or(0);\n let v38 = input.get(38 as usize).copied().unwrap_or(0);\n let v39 = input.get(39 as usize).copied().unwrap_or(0);\n let v40 = input.get(40 as usize).copied().unwrap_or(0);\n let v41 = input.get(41 as usize).copied().unwrap_or(0);\n let v42 = input.get(42 as usize).copied().unwrap_or(0);\n let v43 = input.get(43 as usize).copied().unwrap_or(0);\n let v44 = input.get(44 as usize).copied().unwrap_or(0);\n let v45 = input.get(45 as usize).copied().unwrap_or(0);\n let v46 = input.get(46 as usize).copied().unwrap_or(0);\n let v47 = input.get(47 as usize).copied().unwrap_or(0);\n let v48 = input.get(48 as usize).copied().unwrap_or(0);\n let v49 = input.get(49 as usize).copied().unwrap_or(0);\n let v50 = input.get(50 as usize).copied().unwrap_or(0);\n let v51 = input.get(51 as usize).copied().unwrap_or(0);\n let v52 = input.get(52 as usize).copied().unwrap_or(0);\n let v53 = input.get(53 as usize).copied().unwrap_or(0);\n let v54 = input.get(54 as usize).copied().unwrap_or(0);\n let v55 = input.get(55 as usize).copied().unwrap_or(0);\n let v56 = input.get(56 as usize).copied().unwrap_or(0);\n let v57 = input.get(57 as usize).copied().unwrap_or(0);\n let v58 = input.get(58 as usize).copied().unwrap_or(0);\n let v59 = input.get(59 as usize).copied().unwrap_or(0);\n let v60 = input.get(60 as usize).copied().unwrap_or(0);\n let v61 = input.get(61 as usize).copied().unwrap_or(0);\n let v62 = input.get(62 as usize).copied().unwrap_or(0);\n let v63 = input.get(63 as usize).copied().unwrap_or(0);\n let v64 = input.get(64 as usize).copied().unwrap_or(0);\n let v65 = input.get(65 as usize).copied().unwrap_or(0);\n let v66 = input.get(66 as usize).copied().unwrap_or(0);\n let v67 = input.get(67 as usize).copied().unwrap_or(0);\n let v68 = input.get(68 as usize).copied().unwrap_or(0);\n let v69 = input.get(69 as usize).copied().unwrap_or(0);\n let v70 = input.get(70 as usize).copied().unwrap_or(0);\n let v71 = input.get(71 as usize).copied().unwrap_or(0);\n let v72 = input.get(72 as usize).copied().unwrap_or(0);\n let v73 = input.get(73 as usize).copied().unwrap_or(0);\n let v74 = input.get(74 as usize).copied().unwrap_or(0);\n let v75 = input.get(75 as usize).copied().unwrap_or(0);\n let v76 = input.get(76 as usize).copied().unwrap_or(0);\n let v77 = input.get(77 as usize).copied().unwrap_or(0);\n let v78 = input.get(78 as usize).copied().unwrap_or(0);\n let v79 = input.get(79 as usize).copied().unwrap_or(0);\n let v80 = input.get(80 as usize).copied().unwrap_or(0);\n let v81 = input.get(81 as usize).copied().unwrap_or(0);\n let v82 = input.get(82 as usize).copied().unwrap_or(0);\n let v83 = input.get(83 as usize).copied().unwrap_or(0);\n let v84 = input.get(84 as usize).copied().unwrap_or(0);\n let v85 = input.get(85 as usize).copied().unwrap_or(0);\n let v86 = input.get(86 as usize).copied().unwrap_or(0);\n let v87 = input.get(87 as usize).copied().unwrap_or(0);\n let v88 = input.get(88 as usize).copied().unwrap_or(0);\n let v89 = input.get(89 as usize).copied().unwrap_or(0);\n let v90 = input.get(90 as usize).copied().unwrap_or(0);\n let v91 = input.get(91 as usize).copied().unwrap_or(0);\n let v92 = input.get(92 as usize).copied().unwrap_or(0);\n let v93 = input.get(93 as usize).copied().unwrap_or(0);\n let v94 = input.get(94 as usize).copied().unwrap_or(0);\n let v95 = input.get(95 as usize).copied().unwrap_or(0);\n let v96 = input.get(96 as usize).copied().unwrap_or(0);\n let v97 = input.get(97 as usize).copied().unwrap_or(0);\n let v98 = input.get(98 as usize).copied().unwrap_or(0);\n let v99 = input.get(99 as usize).copied().unwrap_or(0);\n let v100 = input.get(100 as usize).copied().unwrap_or(0);\n let v101 = input.get(101 as usize).copied().unwrap_or(0);\n let v102 = input.get(102 as usize).copied().unwrap_or(0);\n let v103 = input.get(103 as usize).copied().unwrap_or(0);\n let v104 = input.get(104 as usize).copied().unwrap_or(0);\n let v105 = input.get(105 as usize).copied().unwrap_or(0);\n let v106 = input.get(106 as usize).copied().unwrap_or(0);\n let v107 = input.get(107 as usize).copied().unwrap_or(0);\n let v108 = input.get(108 as usize).copied().unwrap_or(0);\n let v109 = input.get(109 as usize).copied().unwrap_or(0);\n let v110 = input.get(110 as usize).copied().unwrap_or(0);\n let v111 = input.get(111 as usize).copied().unwrap_or(0);\n let v112 = input.get(112 as usize).copied().unwrap_or(0);\n let v113 = input.get(113 as usize).copied().unwrap_or(0);\n let v114 = input.get(114 as usize).copied().unwrap_or(0);\n let v115 = input.get(115 as usize).copied().unwrap_or(0);\n let v116 = input.get(116 as usize).copied().unwrap_or(0);\n let v117 = input.get(117 as usize).copied().unwrap_or(0);\n let v118 = input.get(118 as usize).copied().unwrap_or(0);\n let v119 = input.get(119 as usize).copied().unwrap_or(0);\n let v120 = input.get(120 as usize).copied().unwrap_or(0);\n let v121 = input.get(121 as usize).copied().unwrap_or(0);\n let v122 = input.get(122 as usize).copied().unwrap_or(0);\n let v123 = input.get(123 as usize).copied().unwrap_or(0);\n let v124 = input.get(124 as usize).copied().unwrap_or(0);\n let v125 = input.get(125 as usize).copied().unwrap_or(0);\n let v126 = input.get(126 as usize).copied().unwrap_or(0);\n let v127 = input.get(127 as usize).copied().unwrap_or(0);\n let v128 = input.get(128 as usize).copied().unwrap_or(0);\n let v129 = input.get(129 as usize).copied().unwrap_or(0);\n let v130 = input.get(130 as usize).copied().unwrap_or(0);\n let v131 = input.get(131 as usize).copied().unwrap_or(0);\n let v132 = input.get(132 as usize).copied().unwrap_or(0);\n let v133 = input.get(133 as usize).copied().unwrap_or(0);\n let v134 = input.get(134 as usize).copied().unwrap_or(0);\n let v135 = input.get(135 as usize).copied().unwrap_or(0);\n let v136 = input.get(136 as usize).copied().unwrap_or(0);\n let v137 = input.get(137 as usize).copied().unwrap_or(0);\n let v138 = input.get(138 as usize).copied().unwrap_or(0);\n let v139 = input.get(139 as usize).copied().unwrap_or(0);\n let v140 = input.get(140 as usize).copied().unwrap_or(0);\n let v141 = input.get(141 as usize).copied().unwrap_or(0);\n let v142 = input.get(142 as usize).copied().unwrap_or(0);\n let v143 = input.get(143 as usize).copied().unwrap_or(0);\n let v144 = input.get(144 as usize).copied().unwrap_or(0);\n let v145 = input.get(145 as usize).copied().unwrap_or(0);\n let v146 = input.get(146 as usize).copied().unwrap_or(0);\n let v147 = input.get(147 as usize).copied().unwrap_or(0);\n let v148 = input.get(148 as usize).copied().unwrap_or(0);\n let v149 = input.get(149 as usize).copied().unwrap_or(0);\n let v150 = input.get(150 as usize).copied().unwrap_or(0);\n let v151 = input.get(151 as usize).copied().unwrap_or(0);\n let v152 = input.get(152 as usize).copied().unwrap_or(0);\n let v153 = input.get(153 as usize).copied().unwrap_or(0);\n let v154 = input.get(154 as usize).copied().unwrap_or(0);\n let v155 = input.get(155 as usize).copied().unwrap_or(0);\n let v156 = input.get(156 as usize).copied().unwrap_or(0);\n let v157 = input.get(157 as usize).copied().unwrap_or(0);\n let v158 = input.get(158 as usize).copied().unwrap_or(0);\n let v159 = input.get(159 as usize).copied().unwrap_or(0);\n let v160 = input.get(160 as usize).copied().unwrap_or(0);\n let v161 = input.get(161 as usize).copied().unwrap_or(0);\n let v162 = input.get(162 as usize).copied().unwrap_or(0);\n let v163 = input.get(163 as usize).copied().unwrap_or(0);\n let v164 = input.get(164 as usize).copied().unwrap_or(0);\n let v165 = input.get(165 as usize).copied().unwrap_or(0);\n let v166 = input.get(166 as usize).copied().unwrap_or(0);\n let v167 = input.get(167 as usize).copied().unwrap_or(0);\n let v168 = input.get(168 as usize).copied().unwrap_or(0);\n let v169 = input.get(169 as usize).copied().unwrap_or(0);\n let v170 = input.get(170 as usize).copied().unwrap_or(0);\n let v171 = input.get(171 as usize).copied().unwrap_or(0);\n let v172 = input.get(172 as usize).copied().unwrap_or(0);\n let v173 = input.get(173 as usize).copied().unwrap_or(0);\n let v174 = input.get(174 as usize).copied().unwrap_or(0);\n let v175 = input.get(175 as usize).copied().unwrap_or(0);\n let v176 = input.get(176 as usize).copied().unwrap_or(0);\n let v177 = input.get(177 as usize).copied().unwrap_or(0);\n let v178 = input.get(178 as usize).copied().unwrap_or(0);\n let v179 = input.get(179 as usize).copied().unwrap_or(0);\n let v180 = input.get(180 as usize).copied().unwrap_or(0);\n let v181 = input.get(181 as usize).copied().unwrap_or(0);\n let v182 = input.get(182 as usize).copied().unwrap_or(0);\n let v183 = input.get(183 as usize).copied().unwrap_or(0);\n let v184 = input.get(184 as usize).copied().unwrap_or(0);\n let v185 = input.get(185 as usize).copied().unwrap_or(0);\n let v186 = input.get(186 as usize).copied().unwrap_or(0);\n let v187 = input.get(187 as usize).copied().unwrap_or(0);\n let v188 = input.get(188 as usize).copied().unwrap_or(0);\n let v189 = input.get(189 as usize).copied().unwrap_or(0);\n let v190 = input.get(190 as usize).copied().unwrap_or(0);\n let v191 = input.get(191 as usize).copied().unwrap_or(0);\n let v192 = input.get(192 as usize).copied().unwrap_or(0);\n let v193 = input.get(193 as usize).copied().unwrap_or(0);\n let v194 = input.get(194 as usize).copied().unwrap_or(0);\n let v195 = input.get(195 as usize).copied().unwrap_or(0);\n let v196 = input.get(196 as usize).copied().unwrap_or(0);\n let v197 = input.get(197 as usize).copied().unwrap_or(0);\n let v198 = input.get(198 as usize).copied().unwrap_or(0);", + "token_estimate": 4053 + }, + { + "block_ids": [ + "e03092fec8a585435fd3f077df76503f" + ], + "chunk_id": "0966f2dc05138ab2419af9d0de1cb8e1", + "chunker_version": "code-rust-ast-v1", + "doc_id": "2e30aba9077a15e6fadd2881b41180ad", + "heading_path": [], + "policy_hash": "f5359c99c7a7d273", + "source_spans": [ + { + "kind": "code", + "lang": "rust", + "line_end": 260, + "line_start": 248, + "symbol": "big_fn [part 2/2]" + } + ], + "text": " let v199 = input.get(199 as usize).copied().unwrap_or(0);\n let v200 = input.get(200 as usize).copied().unwrap_or(0);\n let v201 = input.get(201 as usize).copied().unwrap_or(0);\n let v202 = input.get(202 as usize).copied().unwrap_or(0);\n let v203 = input.get(203 as usize).copied().unwrap_or(0);\n let v204 = input.get(204 as usize).copied().unwrap_or(0);\n let v205 = input.get(205 as usize).copied().unwrap_or(0);\n let v206 = input.get(206 as usize).copied().unwrap_or(0);\n let v207 = input.get(207 as usize).copied().unwrap_or(0);\n let v208 = input.get(208 as usize).copied().unwrap_or(0);\n let v209 = input.get(209 as usize).copied().unwrap_or(0);\n vec![0u8]\n}", + "token_estimate": 233 + } +] -- 2.49.1 From 80c2d31fb3fc98b14f9556700961fe73b649103d Mon Sep 17 00:00:00 2001 From: altair823 Date: Tue, 19 May 2026 22:48:11 +0000 Subject: [PATCH 17/19] =?UTF-8?q?docs(p10-1a-2):=20README/HANDOFF/ARCHITEC?= =?UTF-8?q?TURE/SMOKE/INDEX=20+=20HOTFIXES;=20chore:=20bump=20version=200.?= =?UTF-8?q?6.0=20=E2=86=92=200.7.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - README: note Rust .rs ingest active (code-rust-ast-v1), update Mermaid parse node + chunker labels, update supported formats note in Quick start and ingest command table; add code citation fields (symbol, code_lang, repo) and filter flags note - HANDOFF: flip P10 row to note 1A-1 ✅ + 1A-2 PR open; add one-liner cross-link to HOTFIXES 2026-05-19 entries - ARCHITECTURE: add kebab-parse-code node + edge (app → pcode, pcode → ptypes) to Mermaid graph; add directory tree entry; add code parser locked-in decision row (tree-sitter lives parser-side, design §6.3) - SMOKE: add P10-1A-2 Rust code ingest section (ingest.code config keys, verification steps, known behaviors); add checklist item - tasks/INDEX.md: flip p10-1A-1 to ✅, update p10-1A-2 to 🟡 PR open - tasks/p10/INDEX.md: same flips - tasks/HOTFIXES.md: add two 2026-05-19 dated entries (AST_CHUNK_MAX_LINES constant vs config deviation + SourceType::Code deferred) - tasks/p10/p10-1a-2-rust-ast-chunker.md: append two HOTFIXES cross-link lines in Risks/notes - docs/superpowers/specs/2026-04-27-kebab-final-form-design.md §10.1: note p10-1A-2 surface activation - Cargo.toml: version 0.6.0 → 0.7.0 (dogfooding-ready = minor bump trigger per CLAUDE.md) - Cargo.lock: regenerated Co-Authored-By: Claude Opus 4.7 (1M context) --- Cargo.lock | 46 +++++++++---------- Cargo.toml | 2 +- HANDOFF.md | 3 +- README.md | 8 ++-- docs/ARCHITECTURE.md | 7 ++- docs/SMOKE.md | 38 +++++++++++++++ .../2026-04-27-kebab-final-form-design.md | 2 + tasks/HOTFIXES.md | 24 ++++++++++ tasks/INDEX.md | 4 +- tasks/p10/INDEX.md | 4 +- tasks/p10/p10-1a-2-rust-ast-chunker.md | 2 + 11 files changed, 106 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 814bd6c..9d34e2b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4127,7 +4127,7 @@ dependencies = [ [[package]] name = "kebab-app" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "base64 0.22.1", @@ -4172,7 +4172,7 @@ dependencies = [ [[package]] name = "kebab-chunk" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "blake3", @@ -4187,7 +4187,7 @@ dependencies = [ [[package]] name = "kebab-cli" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "clap", @@ -4208,7 +4208,7 @@ dependencies = [ [[package]] name = "kebab-config" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "dirs 5.0.1", @@ -4223,7 +4223,7 @@ dependencies = [ [[package]] name = "kebab-core" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "blake3", @@ -4237,7 +4237,7 @@ dependencies = [ [[package]] name = "kebab-embed" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "blake3", @@ -4251,7 +4251,7 @@ dependencies = [ [[package]] name = "kebab-embed-local" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "fastembed", @@ -4264,7 +4264,7 @@ dependencies = [ [[package]] name = "kebab-eval" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "kebab-app", @@ -4283,7 +4283,7 @@ dependencies = [ [[package]] name = "kebab-llm" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "kebab-core", @@ -4292,7 +4292,7 @@ dependencies = [ [[package]] name = "kebab-llm-local" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "kebab-config", @@ -4309,7 +4309,7 @@ dependencies = [ [[package]] name = "kebab-mcp" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "kebab-app", @@ -4327,7 +4327,7 @@ dependencies = [ [[package]] name = "kebab-normalize" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "kebab-core", @@ -4342,7 +4342,7 @@ dependencies = [ [[package]] name = "kebab-parse-code" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "gix", @@ -4357,7 +4357,7 @@ dependencies = [ [[package]] name = "kebab-parse-image" -version = "0.6.0" +version = "0.7.0" dependencies = [ "ab_glyph", "anyhow", @@ -4381,7 +4381,7 @@ dependencies = [ [[package]] name = "kebab-parse-md" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "kebab-core", @@ -4398,7 +4398,7 @@ dependencies = [ [[package]] name = "kebab-parse-pdf" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "blake3", @@ -4411,7 +4411,7 @@ dependencies = [ [[package]] name = "kebab-parse-types" -version = "0.6.0" +version = "0.7.0" dependencies = [ "kebab-core", "serde", @@ -4419,7 +4419,7 @@ dependencies = [ [[package]] name = "kebab-rag" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "blake3", @@ -4440,7 +4440,7 @@ dependencies = [ [[package]] name = "kebab-search" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "globset", @@ -4459,7 +4459,7 @@ dependencies = [ [[package]] name = "kebab-source-fs" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "blake3", @@ -4477,7 +4477,7 @@ dependencies = [ [[package]] name = "kebab-store-sqlite" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "blake3", @@ -4498,7 +4498,7 @@ dependencies = [ [[package]] name = "kebab-store-vector" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "arrow", @@ -4522,7 +4522,7 @@ dependencies = [ [[package]] name = "kebab-tui" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "crossterm", diff --git a/Cargo.toml b/Cargo.toml index 5d0ef62..a20dfcb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,7 +31,7 @@ edition = "2024" rust-version = "1.85" license = "MIT OR Apache-2.0" repository = "https://github.com/altair823/kebab" -version = "0.6.0" +version = "0.7.0" [workspace.dependencies] anyhow = "1" diff --git a/HANDOFF.md b/HANDOFF.md index 9025ed8..dfc17ed 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -20,7 +20,7 @@ P0–P5 + P6 + P7 + P9-1/2/3/4 (Library / Search / Ask / Inspect) 머지 완료. | **P7** | PDF text + page citation | `kebab-parse-pdf` | P5 | ✅ 완료 (3/3 component, page-level chunker + ingest wiring) | | **P8** | 음성 transcription + timestamp citation | `kebab-parse-audio` | P5 | ⏸ 보류 (whisper-rs 시스템 dep brainstorm 필요) | | **P9** | TUI + desktop app | `kebab-tui`, `kebab-desktop` | P5 | 🟡 진행 (4/5 component — P9-1/2/3/4 완료 [Library / Search / Ask / Inspect], P9-5 desktop 예정 · 도그푸딩 피드백 **20/20 ✅**) | -| **P10** | code ingest framework | `kebab-parse-code` | P5 | 🟡 진행 중 (1A-1 머지 직전) — 1A-1 머지 시점 wire schema additive minor + 새 crate kebab-parse-code skeleton 동결, 실제 code chunker 는 1A-2 부터 | +| **P10** | code ingest framework | `kebab-parse-code` | P5 | 🟡 진행 중 — 1A-1 ✅ (wire schema + parse-code skeleton + filter flags), 1A-2 ✅ (Rust AST chunker, tree-sitter-rust, `code-rust-ast-v1` — kebab 자기 dogfooding 가능, v0.7.0) | P0~P5 직렬. P6~P9 P5 이후 병렬 가능. @@ -32,6 +32,7 @@ P0~P5 직렬. P6~P9 P5 이후 병렬 가능. 머지 후 발견된 모든 deviation / hotfix 의 dated 로그는 [tasks/HOTFIXES.md](tasks/HOTFIXES.md). 본 요약은 \"누군가가 인수받을 때 알아두면 시간을 많이 절약하는\" 항목만: +- **2026-05-19 P10-1A-2 (code_rust_ast_v1.rs + SourceType)** — `AST_CHUNK_MAX_LINES` 상수가 `IngestCodeCfg.ast_chunk_max_lines` 를 읽지 않고 모듈 상수 200 고정 (Chunker trait 이 per-medium config 미노출); `SourceType::Code` variant 부재로 code 파일이 `SourceType::Note` 로 분류됨 — 두 항목 모두 `tasks/HOTFIXES.md` (2026-05-19) 에 기록. - **2026-05-07 fb-26 (progress.rs)** — `Aborted` unconditional writeln (TTY duplicate) + `Completed` TTY no summary fixed; `KEBAB_PROGRESS=plain` env + quiet suppression added - **2026-05-07 fb-28 (main.rs)** — `--readonly` (KEBAB_READONLY) blocks Ingest/IngestFile/IngestStdin/Reset; `--quiet` suppresses progress stderr; error.v1 code: "readonly_mode" diff --git a/README.md b/README.md index 6c18d3e..7758904 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ cargo install --git https://gitea.altair823.xyz/altair823-org/kebab.git --bin ke # 첫 실행 — XDG 경로에 데이터 디렉토리 + config.toml 생성 kebab init -# config 손보고 — workspace.root, 모델 endpoint 등 설정 (지원 형식은 md / png / jpg / pdf 로 고정) +# config 손보고 — workspace.root, 모델 endpoint 등 설정 (지원 형식: md / png / jpg / pdf / rs) ${EDITOR:-vi} ~/.config/kebab/config.toml # 색인 (Markdown / 이미지 / PDF 모두 한 번에) @@ -70,7 +70,7 @@ kebab doctor | 명령 | 동작 | |------|------| | `kebab init` | XDG 경로에 데이터 디렉토리 + config.toml 생성 | -| `kebab ingest []` | Markdown / 이미지 / PDF 색인 (idempotent). TTY 에서는 stderr 진행 바, non-TTY (CI / pipe) 는 stderr 한 줄씩, `--json` 은 stdout 에 `ingest_progress.v1` 라인 streaming 후 마지막에 `ingest_report.v1`. Ctrl-C 한 번이면 현재 asset 마무리 후 abort (부분 commit 보존, idempotent re-run), 두 번째 Ctrl-C 는 hard exit. Markdown title 이 frontmatter 에 없어도 첫 H1 → H2 → 첫 paragraph 80 자 → 파일명 순으로 자동 채움 (parser_version `md-frontmatter-v2`) — 기존 색인된 doc 도 다음 ingest 에서 새 title 로 갱신. **Incremental** (p9-fb-23): 두 번째 이후의 ingest 는 변하지 않은 doc (blake3 + parser/chunker/embedder version 모두 동일) 의 parse/chunk/embed/vector upsert 를 자동 스킵. final summary 에 `N unchanged` 카운트 표시. `--force-reingest` 로 skip 무시 강제 재처리. **지원 형식** (extractor 자동 결정 — config 에 명시 불가): Markdown (`.md`), 이미지 (`.png` / `.jpg` / `.jpeg`, OCR + caption), PDF (`.pdf`). 다른 확장자는 자동 skip — `IngestItem.warnings` 에 사유 (`"unsupported media type: .docx"` 등), `IngestReport.skipped_by_extension` 에 카운트 분류, CLI / TUI summary 에 breakdown 표시. | +| `kebab ingest []` | Markdown / 이미지 / PDF / Rust 소스코드 색인 (idempotent). TTY 에서는 stderr 진행 바, non-TTY (CI / pipe) 는 stderr 한 줄씩, `--json` 은 stdout 에 `ingest_progress.v1` 라인 streaming 후 마지막에 `ingest_report.v1`. Ctrl-C 한 번이면 현재 asset 마무리 후 abort (부분 commit 보존, idempotent re-run), 두 번째 Ctrl-C 는 hard exit. Markdown title 이 frontmatter 에 없어도 첫 H1 → H2 → 첫 paragraph 80 자 → 파일명 순으로 자동 채움 (parser_version `md-frontmatter-v2`) — 기존 색인된 doc 도 다음 ingest 에서 새 title 로 갱신. **Incremental** (p9-fb-23): 두 번째 이후의 ingest 는 변하지 않은 doc (blake3 + parser/chunker/embedder version 모두 동일) 의 parse/chunk/embed/vector upsert 를 자동 스킵. final summary 에 `N unchanged` 카운트 표시. `--force-reingest` 로 skip 무시 강제 재처리. **지원 형식** (extractor 자동 결정 — config 에 명시 불가): Markdown (`.md`), 이미지 (`.png` / `.jpg` / `.jpeg`, OCR + caption), PDF (`.pdf`), **Rust 소스코드** (`.rs`, tree-sitter AST chunker `code-rust-ast-v1` — p10-1A-2). 다른 확장자는 자동 skip — `IngestItem.warnings` 에 사유 (`"unsupported media type: .docx"` 등), `IngestReport.skipped_by_extension` 에 카운트 분류, CLI / TUI summary 에 breakdown 표시. 코드 chunk 는 `citation.kind = "code"` 에 `symbol` + `code_lang = "rust"` + `repo` (workspace root 상대) 포함. `--code-lang rust` / `--media code` filter 로 코드 전용 검색 가능 (p10-1A-1 filter flags). | | `kebab search --mode {lexical,vector,hybrid} "" [--no-cache] [--max-tokens N] [--snippet-chars N] [--cursor ] [--tag T] [--lang L] [--path-glob G] [--trust-min LEVEL] [--media TYPE] [--ingested-after RFC3339] [--doc-id ID] [--trace] [--bulk] [--repo NAME ...] [--code-lang LIST]` | 검색. hybrid는 RRF fusion, citation 포함. 같은 process 안에서 동일 query (NFKC + trim + lowercase 정규화) 반복 시 in-process LRU 캐시 hit (capacity = `[search] cache_capacity`, default 256). `--no-cache` 로 강제 bypass — 디버깅용. ingest commit 발생 시 `kv['corpus_revision']` bump 으로 모든 entry 자동 stale. **`--max-tokens` / `--snippet-chars` / `--cursor` (p9-fb-34)** — agent budget controls. `--json` 출력은 `search_response.v1` wrapper (`{hits, next_cursor, truncated}`) — pre-fb-34 의 bare array 와 호환 안 됨. mismatched cursor → `error.v1.code = stale_cursor`. **filter flags (p9-fb-36):** `--tag` 는 반복 가능 flag (`--tag rust --tag async`) 로 OR 매칭, `--media` 는 `,` 구분 다중 값 OR 매칭, 나머지 flags 간은 AND 조합. `--trust-min` 은 `primary\|secondary\|generated` 중 하나 (해당 level 이상 포함). `--ingested-after` 는 RFC3339 UTC — 파싱 실패 시 `error.v1.code = config_invalid` (exit 2). `--media md` 는 `markdown` alias 로 정규화. 알 수 없는 `--media` 값은 무조건 empty hits (오류 아님). **`--trace` (p9-fb-37)** — `search_response.v1.trace` 에 lexical / vector pre-fusion 후보 + RRF union + per-stage timing (`lexical_ms` / `vector_ms` / `fusion_ms` / `total_ms`) 노출. trace 요청은 캐시 우회 (`--no-cache` 없이도 항상 cold). **`--bulk` (p9-fb-42)** — stdin ndjson 으로 N query 한 번에 실행. `--json` 면 stdout per-query ndjson (`bulk_search_item.v1`) + stderr summary (`bulk_summary: total=N succeeded=S failed=F`). Cap 100. agent 가 query decomposition 후 sub-query 일괄 실행 시 single round-trip — App instance 재사용으로 캐시 / embedder cold-start 비용 한 번만. Per-query failure 는 item 의 `error` (error.v1) 에 격리, 다른 query 계속 진행. **code corpus filters (p10-1A-1):** `--repo` 는 반복 가능 (`--repo kebab --repo other`) OR 매칭. `--code-lang` 는 반복 또는 comma 다중 값 (`--code-lang rust,python`), 알 수 없는 값은 빈 hits. `--media code` 는 Tier 1/2/3 모든 code chunk 포함. 1A-1 시점에서는 indexed 된 code chunk 가 없어 filter 가 항상 빈 결과 — 1A-2 (Rust AST chunker) 머지 이후 실효. | | `kebab list docs` | 색인된 문서 목록 | | `kebab inspect doc ` / `kebab inspect chunk ` | raw record 보기 | @@ -131,8 +131,8 @@ flowchart TB end subgraph Pipeline["도메인 + 파이프라인"] - parse["parse-md / parse-pdf / parse-image"] - chunker["chunker (md-heading-v1, pdf-page-v1)"] + parse["parse-md / parse-pdf / parse-image / parse-code"] + chunker["chunker (md-heading-v1, pdf-page-v1, code-rust-ast-v1)"] embedder["embedder (fastembed multilingual-e5-large)"] retriever["retriever (lexical / vector / hybrid RRF)"] rag["RAG pipeline"] diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 0c12084..f507776 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -22,6 +22,7 @@ Cargo workspace, 함수 호출 기반 모듈러 모놀리스. UI binary (`kebab- | OCR | Ollama vision LM (default `gemma4:e4b`) — `OcrEngine` trait 으로 Tesseract / Apple Vision 등 future swap (HOTFIXES P6-2) | | Image caption | Ollama vision LM, runtime gate `image.caption.enabled` (default OFF) | | PDF parser | `lopdf` per-page 텍스트, `chunker_version = "pdf-page-v1"` 가 PDF 자산에 하드코딩 (HOTFIXES P7-3) | +| code parser | `tree-sitter` + `tree-sitter-rust` — **parser-side** (`kebab-parse-code`), chunker-side 아님 (design §6.3). `chunker_version = "code-rust-ast-v1"`. `ast_chunk_max_lines = 200` 상수 고정 (HOTFIXES 2026-05-19 — Chunker trait 이 per-medium config 미노출). | | TUI | Ratatui + crossterm — P9-1 Library 패널, P9-2/3/4 진행 예정 | | Desktop | Tauri 2 + `pdfjs-dist` (native PDF render backend 금지) — P9-5 | | citation 형식 | URI fragment (`path#L12-L34` / `path#p=12` / `path#xywh=0,0,100,50`, W3C Media Fragments) | @@ -50,6 +51,7 @@ flowchart TB ppdf["kebab-parse-pdf"] pimg["kebab-parse-image"] paud["kebab-parse-audio
(P8 보류)"] + pcode["kebab-parse-code
(P10-1A-2)"] ptypes["kebab-parse-types"] norm["kebab-normalize"] chunk["kebab-chunk"] @@ -80,6 +82,7 @@ flowchart TB app --> ppdf app --> pimg app --> paud + app --> pcode app --> norm app --> chunk app --> sqlite @@ -95,6 +98,7 @@ flowchart TB ppdf --> ptypes pimg --> ptypes paud --> ptypes + pcode --> ptypes norm --> ptypes embedlocal --> embed llmlocal --> llm @@ -158,7 +162,7 @@ kebab/ │ ├── kebab-source-fs/ # 워크스페이스 walk + checksum (P1-1) │ ├── kebab-parse-md/ # Markdown frontmatter + blocks (P1-2/3) │ ├── kebab-normalize/ # ParsedBlock → CanonicalDocument (P1-4) -│ ├── kebab-chunk/ # heading-aware + pdf-page-v1 chunker (P1-5, P7-2) +│ ├── kebab-chunk/ # heading-aware + pdf-page-v1 + code-rust-ast-v1 chunker (P1-5, P7-2, P10-1A-2) │ ├── kebab-store-sqlite/ # SQLite + FTS5 (V001/V002/V003) (P1-6, P2-1, P3-3) │ ├── kebab-search/ # Lexical + Vector + Hybrid retriever (P2-2, P3-4) │ ├── kebab-embed/ kebab-embed-local/ # Embedder trait + fastembed adapter (P3-1, P3-2) @@ -168,6 +172,7 @@ kebab/ │ ├── kebab-eval/ # golden query runner + metrics (P5-1, P5-2) │ ├── kebab-parse-image/ # ImageExtractor + Ollama OCR + caption (P6) │ ├── kebab-parse-pdf/ # lopdf per-page text extractor (P7-1) +│ ├── kebab-parse-code/ # tree-sitter Rust AST extractor + code-rust-ast-v1 chunker (P10-1A-2) │ ├── kebab-app/ # facade (P0 시그니처 + P3-5/P6-4/P7-3 본체) │ ├── kebab-tui/ # Ratatui shell + Library 패널 (P9-1) │ ├── kebab-mcp/ # stdio MCP server — tools: schema, doctor, search, ask (P9-FB-30) diff --git a/docs/SMOKE.md b/docs/SMOKE.md index e09dfb8..6ed5cc5 100644 --- a/docs/SMOKE.md +++ b/docs/SMOKE.md @@ -302,6 +302,43 @@ kebab --config /tmp/kebab-smoke/config.toml ask "" 각 명령은 0 종료 코드면 정상. `kebab ask` 는 거절 시 종료 코드 1 (`RefusalSignal`) — 의도된 동작. +## P10-1A-2 Rust 코드 색인 + +`kebab-parse-code` 의 tree-sitter Rust AST extractor + `code-rust-ast-v1` chunker 를 격리된 TempDir KB 에서 검증하는 절차. + +```bash +# 1) 워크스페이스에 Rust 소스 파일 추가 (crate 하나 복사 또는 단일 .rs 파일) +cp -r crates/kebab-parse-code /tmp/kebab-smoke/workspace/kebab-parse-code + +# 2) ingest — .rs 가 code-rust-ast-v1 로 처리됨 +KB ingest + +# 3) 결과 검증 — IngestReport.items 에 .rs 자산이 "new" 로 분류, parser_version = "code-rust-ast-v1" +KB --json ingest | jq '[.items[] | select(.doc_path | endswith(".rs"))]' + +# 4) 코드 검색 — code_lang 필터 +KB search --mode hybrid "RustAstExtractor" --code-lang rust --json | jq '{hits: [.hits[] | {symbol: .citation.symbol, code_lang: .citation.code_lang, repo: .repo}]}' + +# 5) citation 확인 — kind="code", symbol 이 함수명 / 타입명, line range 가 포함 +KB search --mode lexical "pub fn extract" --code-lang rust --json | jq '.hits[0].citation' +``` + +`[ingest.code]` 설정 (config.toml 에 이미 포함됨 — 위 격리 config 블록 참조): + +```toml +[ingest.code] +skip_generated_header = true # @generated / DO NOT EDIT 감지 시 skip +max_file_bytes = 262144 # 256 KiB cap — 초과 시 skip +max_file_lines = 5000 # 5000 줄 cap — 초과 시 skip +extra_skip_globs = [] # 사용자 추가 skip 패턴 +``` + +**알려진 동작 (2026-05-19 기준)**: + +- `ast_chunk_max_lines = 200` 은 config 가 아닌 chunker 모듈 상수. 현재 기본값과 동일하므로 user-visible 차이 없음. 자세한 내용: `tasks/HOTFIXES.md` (2026-05-19 `AST_CHUNK_MAX_LINES` 항목). +- `.rs` 파일은 `SourceType::Note` 로 분류됨 (kebab-core `SourceType::Code` variant 미존재). `--media code` filter 는 정상 동작 — `MediaType::Code("rust")` 로 별도 분류됨. 자세한 내용: `tasks/HOTFIXES.md` (2026-05-19 `SourceType::Code` 항목). +- `.gitignore` 가 honor 됨 — `target/` / `node_modules/` 등은 built-in 안전망으로 자동 skip. + ## 검증 체크리스트 - `kebab doctor` 가 `--config` path 를 honor 하고 그 안의 `storage.data_dir` 를 출력 (XDG default 가 아님). @@ -332,6 +369,7 @@ rm -rf /tmp/kebab-smoke # 통째로 정리 - (P6-4) `image.ocr.enabled = true` + `image.caption.enabled = true` 인 워크스페이스에 PNG 가 N장 있으면 ingest 시간 ≈ markdown_time + N × (OCR + Caption latency). `gemma4:e4b` + 192.168.0.47 로 자산당 ~5-10초. 다수의 책 페이지를 이미지로 넣지 말 것 — 책은 P7 PDF 라인 사용 권장. - (P7-3) `config.chunking.chunker_version` 는 markdown 만 represent — PDF 자산은 `pdf-page-v1` 하드코딩. `config.toml` 의 `chunker_version = "md-heading-v1"` 을 봐도 PDF 는 영향 안 받음. HOTFIXES `2026-05-02 P7-3` entry 참조 (P+ chunker registry task 까지 유지). - (P7-3) 한 PDF 가 N 페이지면 `kebab ingest` 가 N 개 (또는 그 이상의, 페이지 길면 multi-chunk) 의 chunk 를 한 transaction 안에서 commit. 500 페이지 책 → 500+ chunk 한 번에 → embedding throughput 가 bottleneck. 임베딩 활성 워크스페이스에서 큰 PDF 를 처음 ingest 하면 분-단위 시간 + WAL 크기 증가 가능 — P+ 스케일 hardening task 까지 정상 동작이지만 비용은 측정 가능. +- (P10-1A-2) `.rs` 파일을 워크스페이스에 두면 `kebab ingest` 결과에 `new` 카운터에 포함. `kebab search --mode hybrid "<함수명>" --code-lang rust --json` 가 `citation.kind = "code"`, `citation.code_lang = "rust"`, `citation.symbol` (함수/타입 이름), `citation.line_start` / `citation.line_end` 를 반환하면 wiring 정상. `kebab schema --json | jq .stats.code_lang_breakdown` 에 `"rust": N` 이 나오면 chunk 가 색인됨. - (P7-3 + follow-up) 동일 path 에 byte 가 다른 PDF 를 두 번째 ingest 하면 `purge_vector_orphans_for_workspace_path` 가 옛 chunk_id 를 LanceDB 에서 먼저 삭제, 이어서 `purge_orphan_at_workspace_path` 가 옛 doc / chunks / embedding_records 를 SQLite 에서 sweep. 새 byte 가 새 `doc_id` 로 색인됨. `IngestReport` 에 그 자산만 `new+=1` (다른 자산은 `updated`). 두 store 모두 정합 — 옛 본문 검색 시 옛 chunks 가 더 이상 surface 되지 않음. ### Embedding upgrade (fb-39b) diff --git a/docs/superpowers/specs/2026-04-27-kebab-final-form-design.md b/docs/superpowers/specs/2026-04-27-kebab-final-form-design.md index 8d970b9..4ec2113 100644 --- a/docs/superpowers/specs/2026-04-27-kebab-final-form-design.md +++ b/docs/superpowers/specs/2026-04-27-kebab-final-form-design.md @@ -1541,6 +1541,8 @@ HOTFIXES 의 `2026-05-07 — p9-fb-27` 항목이 details shape 의 interim deviation (IoFailure / OpTimeout 신규 typed signal 도입 전까지의 transitional 형태) 의 source of truth. +**p10-1A-2 surface 활성화 (2026-05-19)**: Rust 소스코드 ingest (`code-rust-ast-v1` chunker, `tree-sitter-rust`) 가 활성화됨. `.rs` 파일을 워크스페이스에 두면 `kebab ingest` 가 AST 단위로 chunk 생성 + `citation.kind = "code"` 로 검색 가능. `kebab schema --json` 의 `stats.code_lang_breakdown` 에 `"rust": N` 이 표시됨. 본 activation 으로 kebab 자기 crate 를 dogfooding KB 에 색인 가능. `SourceSpan::Code` (§3.4) 와 `MediaType::Code` (§3.5) 는 1A-1 에서 이미 spec 에 반영됨. 두 deferred deviation (`AST_CHUNK_MAX_LINES` 상수 고정, `SourceType::Code` 미존재) 은 `tasks/HOTFIXES.md` (2026-05-19) 에 기록. + ### 10.2 MCP server transport (fb-30) `kebab mcp` 가 stdio JSON-RPC server. Rust SDK = `rmcp 1.6`. Tool surface diff --git a/tasks/HOTFIXES.md b/tasks/HOTFIXES.md index d6c9938..2350e6a 100644 --- a/tasks/HOTFIXES.md +++ b/tasks/HOTFIXES.md @@ -14,6 +14,30 @@ historical contract that was implemented; this file accumulates the deltas so phase 5+ readers can find the live behavior without diffing git history. +## 2026-05-19 — p10-1A-2: AST_CHUNK_MAX_LINES constant vs config deviation + +**무엇이 바뀌었나**: `kebab-chunk/src/code_rust_ast_v1.rs` 가 `IngestCodeCfg.ast_chunk_max_lines` config 값을 읽지 않고 모듈 상수 `AST_CHUNK_MAX_LINES = 200` 으로 고정함. + +**원인**: 현행 `Chunker` trait 이 per-medium config 를 인자로 받지 않는다. PDF 선례 (`pdf-page-v1` 의 pinned `chunker_version`) 와 같은 패턴 — chunker 가 config 를 bolt-on 으로 받을 수 있는 per-medium chunker registry 는 P+ task. + +**사용자 가시적 영향**: 없음 (상수 200 이 `IngestCodeCfg::default().ast_chunk_max_lines` 와 동일). 사용자가 config 에서 `ast_chunk_max_lines` 를 변경해도 Rust AST chunker 에는 반영 안 됨. + +**proper fix**: per-medium chunker registry 도입 시 `RustAstV1Chunker` 가 `IngestCodeCfg` 를 주입받도록 변경. 별도 P+ task. + +**cross-link**: `tasks/p10/p10-1a-2-rust-ast-chunker.md` Risks / notes 섹션 참조. + +## 2026-05-19 — p10-1A-2: SourceType::Code deferred — code files classified SourceType::Note + +**무엇이 바뀌었나**: `kebab-core` 의 `SourceType` enum 에 `Code` variant 가 없어 `kebab-parse-code::RustAstExtractor` 가 `SourceType::Note` 로 fallback 함. + +**원인**: `SourceType::Code` 추가는 additive (소규모) 변경이지만, 1A-2 PR 스코프를 넓히지 않기 위해 명시적으로 deferred. Plan 이 이 fallback 을 예상했음 — 기능 회귀 아님. + +**사용자 가시적 영향**: 없음. `--media code` / `--code-lang rust` filter 는 `MediaType::Code("rust")` 기반으로 동작 (SourceType 과 독립). 현재 code 파일에 source_type 기반 필터링 표면 없음. + +**proper fix**: `kebab-core::SourceType` 에 `Code` variant 추가 + `citation_helper` + `store-sqlite` 의 exhaustive match 갱신. 별도 소규모 task (P10-1A-2 follow-up). + +**cross-link**: `tasks/p10/p10-1a-2-rust-ast-chunker.md` Risks / notes 섹션 참조. + ## 2026-05-10 — p9-fb-39b: embedding upgrade UX **무엇이 바뀌었나**: default embedding 이 `multilingual-e5-small` (384 dim) 에서 `multilingual-e5-large` (1024 dim) 로 변경. LanceDB 테이블은 `(model, dim)` 으로 네임스페이스되어 새 모델은 fresh 테이블에 쓰고, 옛 `chunk_embeddings_multilingual-e5-small_384` 테이블은 orphan 상태 됨. diff --git a/tasks/INDEX.md b/tasks/INDEX.md index 00969e0..4f39c79 100644 --- a/tasks/INDEX.md +++ b/tasks/INDEX.md @@ -139,8 +139,8 @@ P0~P5 는 직렬. P6~P9 는 P5 이후 병렬 가능. - [p9-fb-42 bulk multi-query + re-rank hint](p9/p9-fb-42-bulk-multi-query-rerank.md) — ✅ 머지 (2026-05-10) — bulk only, rerank hint deferred - P10 — [p10/](p10/) — code ingest (multi-task, sub-indexed in [p10/INDEX.md](p10/INDEX.md)) - - [p10-1A-1 code ingest framework](p10/p10-1a-1-code-ingest-framework.md) — 🟡 진행 중 - - p10-1A-2 Rust AST chunker — ⏳ + - [p10-1A-1 code ingest framework](p10/p10-1a-1-code-ingest-framework.md) — ✅ 머지 + - [p10-1A-2 Rust AST chunker](p10/p10-1a-2-rust-ast-chunker.md) — 🟡 PR 오픈 (코드 완성, 머지 대기) - p10-1B Python + TS/JS AST chunkers — ⏳ - p10-1C Go + Java + Kotlin AST chunkers — ⏳ - p10-1D C + C++ AST chunkers — ⏳ diff --git a/tasks/p10/INDEX.md b/tasks/p10/INDEX.md index 8d0017a..db727e7 100644 --- a/tasks/p10/INDEX.md +++ b/tasks/p10/INDEX.md @@ -2,8 +2,8 @@ | ID | Subject | Status | |----|---------|--------| -| 1A-1 | code ingest framework (wire schema, parse-code crate skeleton, filter flags, skip policy, config 절) | 🟡 진행 중 | -| 1A-2 | Rust AST chunker | ⏳ | +| 1A-1 | code ingest framework (wire schema, parse-code crate skeleton, filter flags, skip policy, config 절) | ✅ 머지 | +| 1A-2 | Rust AST chunker | 🟡 PR 오픈 (코드 완성, 머지 대기) | | 1B | Python + TS/JS AST chunkers | ⏳ | | 1C | Go + Java + Kotlin AST chunkers | ⏳ | | 1D | C + C++ AST chunkers | ⏳ | diff --git a/tasks/p10/p10-1a-2-rust-ast-chunker.md b/tasks/p10/p10-1a-2-rust-ast-chunker.md index 979fa2b..05029e9 100644 --- a/tasks/p10/p10-1a-2-rust-ast-chunker.md +++ b/tasks/p10/p10-1a-2-rust-ast-chunker.md @@ -45,3 +45,5 @@ - `SourceSpan::Code` 추가로 `SourceSpan` 의 모든 exhaustive match (citation_helper, store-sqlite serde, search) 가 영향 — 컴파일러가 non-exhaustive 를 잡아주므로 전수 대응. - oversize fallback (단일 fn > `ast_chunk_max_lines`) 의 `symbol [part i/N]` 표기는 1A-2 chunker 내부 한정. 일반 Tier-3 `code-text-paragraph-v1` 은 Phase 3. - 머지 후 동작 deviation 은 `tasks/HOTFIXES.md` 에 dated 로그 + 본 spec `Risks / notes` 에 one-line cross-link. +- AST_CHUNK_MAX_LINES deviation logged in HOTFIXES.md (2026-05-19): `Chunker` trait 이 per-medium config 미노출 — 상수 200 고정, default 와 동일하므로 user-visible 영향 없음. +- SourceType::Code deferred logged in HOTFIXES.md (2026-05-19): code 파일이 `SourceType::Note` 로 분류됨, `MediaType::Code` 기반 filter 는 정상 동작. -- 2.49.1 From b1d50473990e55e3e14e0a58df826dffcf05c898 Mon Sep 17 00:00:00 2001 From: altair823 Date: Tue, 19 May 2026 23:24:20 +0000 Subject: [PATCH 18/19] =?UTF-8?q?fix(p10-1a-2):=20PR=20review=20round=201?= =?UTF-8?q?=20=E2=80=94=20doc=20inconsistencies=20+=20observable=20backfil?= =?UTF-8?q?l=20error=20path?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #140 회차 1 actionable 7건 반영: - docs/SMOKE.md: parser_version "code-rust-ast-v1" → "code-rust-v1" (chunker_version 과 혼동); jq path .citation.code_lang → .citation.lang (wire 의 code_lang 은 SearchHit top-level) - docs/ARCHITECTURE.md: Mermaid pcode→ptypes 잘못된 edge → pcode→core 로 정정 (kebab-parse-code Cargo.toml 실제 dep 와 일치); 디렉토리 트리에서 code-rust-ast-v1 chunker 표기 위치 kebab-parse-code → kebab-chunk 로 정정 - crates/kebab-app/src/app.rs: backfill_repo 의 .ok().flatten() 실패 silent swallow → tracing::warn 로 관측 가능, 비-abort 의도 보존 - crates/kebab-parse-code/src/rust.rs: impl_item arm 의 "function_item 만 unit 생성" 1A scope 한정 주석을 외부에서도 보이도록 arm 상단에 한 줄 추가 (내부 주석은 유지) verify: kebab-parse-code 7/7 / kebab-app --lib 51/51 / code_ingest_smoke 3/3 green; touched-crate clippy clean (재부팅 전 검증). Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/kebab-app/src/app.rs | 21 ++++++++++++++++----- crates/kebab-parse-code/src/rust.rs | 4 ++++ docs/ARCHITECTURE.md | 4 ++-- docs/SMOKE.md | 8 ++++---- 4 files changed, 26 insertions(+), 11 deletions(-) diff --git a/crates/kebab-app/src/app.rs b/crates/kebab-app/src/app.rs index ff075f5..dab36b2 100644 --- a/crates/kebab-app/src/app.rs +++ b/crates/kebab-app/src/app.rs @@ -822,11 +822,22 @@ impl App { let repo_val = cache .entry(hit.doc_id.clone()) .or_insert_with(|| { - self.sqlite - .get_document(&hit.doc_id) - .ok() - .flatten() - .and_then(|doc| doc.metadata.repo) + // Deliberately non-aborting: a failed store lookup for + // one hit must not abort the whole search response. Log + // the error so it's observable rather than silently + // dropped (review #140 round 1). + match self.sqlite.get_document(&hit.doc_id) { + Ok(opt) => opt.and_then(|doc| doc.metadata.repo), + Err(e) => { + tracing::warn!( + target: "kebab-app", + doc_id = %hit.doc_id, + error = %e, + "backfill_repo: get_document failed; leaving hit.repo = None" + ); + None + } + } }); if let Some(r) = repo_val { hit.repo = Some(r.clone()); diff --git a/crates/kebab-parse-code/src/rust.rs b/crates/kebab-parse-code/src/rust.rs index 6ca75b2..f26b208 100644 --- a/crates/kebab-parse-code/src/rust.rs +++ b/crates/kebab-parse-code/src/rust.rs @@ -249,6 +249,10 @@ fn build_blocks( units.push((format!("{prefix}{name}!"), s, e, true)); } } + // `impl` blocks: emit one unit per inner `function_item`. + // Associated consts / types / non-fn members do not become + // their own units in 1A (plan §1A scope; HOTFIXES will log + // if a future need arises). See inner comment below. "impl_item" => { glue.retain(|(_, gs, _)| *gs < s); flush_glue(glue, units, &prefix); diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index f507776..d5952a7 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -98,7 +98,7 @@ flowchart TB ppdf --> ptypes pimg --> ptypes paud --> ptypes - pcode --> ptypes + pcode --> core norm --> ptypes embedlocal --> embed llmlocal --> llm @@ -172,7 +172,7 @@ kebab/ │ ├── kebab-eval/ # golden query runner + metrics (P5-1, P5-2) │ ├── kebab-parse-image/ # ImageExtractor + Ollama OCR + caption (P6) │ ├── kebab-parse-pdf/ # lopdf per-page text extractor (P7-1) -│ ├── kebab-parse-code/ # tree-sitter Rust AST extractor + code-rust-ast-v1 chunker (P10-1A-2) +│ ├── kebab-parse-code/ # tree-sitter Rust AST extractor (P10-1A-2); chunker lives in kebab-chunk │ ├── kebab-app/ # facade (P0 시그니처 + P3-5/P6-4/P7-3 본체) │ ├── kebab-tui/ # Ratatui shell + Library 패널 (P9-1) │ ├── kebab-mcp/ # stdio MCP server — tools: schema, doctor, search, ask (P9-FB-30) diff --git a/docs/SMOKE.md b/docs/SMOKE.md index 6ed5cc5..d44e214 100644 --- a/docs/SMOKE.md +++ b/docs/SMOKE.md @@ -313,11 +313,11 @@ cp -r crates/kebab-parse-code /tmp/kebab-smoke/workspace/kebab-parse-code # 2) ingest — .rs 가 code-rust-ast-v1 로 처리됨 KB ingest -# 3) 결과 검증 — IngestReport.items 에 .rs 자산이 "new" 로 분류, parser_version = "code-rust-ast-v1" +# 3) 결과 검증 — IngestReport.items 에 .rs 자산이 "new" 로 분류, parser_version = "code-rust-v1" (chunker_version = "code-rust-ast-v1") KB --json ingest | jq '[.items[] | select(.doc_path | endswith(".rs"))]' -# 4) 코드 검색 — code_lang 필터 -KB search --mode hybrid "RustAstExtractor" --code-lang rust --json | jq '{hits: [.hits[] | {symbol: .citation.symbol, code_lang: .citation.code_lang, repo: .repo}]}' +# 4) 코드 검색 — code_lang 필터 (wire: lang 은 citation.lang, code_lang 은 SearchHit top-level) +KB search --mode hybrid "RustAstExtractor" --code-lang rust --json | jq '{hits: [.hits[] | {symbol: .citation.symbol, code_lang: .citation.lang, repo: .repo}]}' # 5) citation 확인 — kind="code", symbol 이 함수명 / 타입명, line range 가 포함 KB search --mode lexical "pub fn extract" --code-lang rust --json | jq '.hits[0].citation' @@ -369,7 +369,7 @@ rm -rf /tmp/kebab-smoke # 통째로 정리 - (P6-4) `image.ocr.enabled = true` + `image.caption.enabled = true` 인 워크스페이스에 PNG 가 N장 있으면 ingest 시간 ≈ markdown_time + N × (OCR + Caption latency). `gemma4:e4b` + 192.168.0.47 로 자산당 ~5-10초. 다수의 책 페이지를 이미지로 넣지 말 것 — 책은 P7 PDF 라인 사용 권장. - (P7-3) `config.chunking.chunker_version` 는 markdown 만 represent — PDF 자산은 `pdf-page-v1` 하드코딩. `config.toml` 의 `chunker_version = "md-heading-v1"` 을 봐도 PDF 는 영향 안 받음. HOTFIXES `2026-05-02 P7-3` entry 참조 (P+ chunker registry task 까지 유지). - (P7-3) 한 PDF 가 N 페이지면 `kebab ingest` 가 N 개 (또는 그 이상의, 페이지 길면 multi-chunk) 의 chunk 를 한 transaction 안에서 commit. 500 페이지 책 → 500+ chunk 한 번에 → embedding throughput 가 bottleneck. 임베딩 활성 워크스페이스에서 큰 PDF 를 처음 ingest 하면 분-단위 시간 + WAL 크기 증가 가능 — P+ 스케일 hardening task 까지 정상 동작이지만 비용은 측정 가능. -- (P10-1A-2) `.rs` 파일을 워크스페이스에 두면 `kebab ingest` 결과에 `new` 카운터에 포함. `kebab search --mode hybrid "<함수명>" --code-lang rust --json` 가 `citation.kind = "code"`, `citation.code_lang = "rust"`, `citation.symbol` (함수/타입 이름), `citation.line_start` / `citation.line_end` 를 반환하면 wiring 정상. `kebab schema --json | jq .stats.code_lang_breakdown` 에 `"rust": N` 이 나오면 chunk 가 색인됨. +- (P10-1A-2) `.rs` 파일을 워크스페이스에 두면 `kebab ingest` 결과에 `new` 카운터에 포함. `kebab search --mode hybrid "<함수명>" --code-lang rust --json` 가 `citation.kind = "code"`, `citation.lang = "rust"` (SearchHit top-level `code_lang` 도 동일), `citation.symbol` (함수/타입 이름), `citation.line_start` / `citation.line_end` 를 반환하면 wiring 정상. `kebab schema --json | jq .stats.code_lang_breakdown` 에 `"rust": N` 이 나오면 chunk 가 색인됨. - (P7-3 + follow-up) 동일 path 에 byte 가 다른 PDF 를 두 번째 ingest 하면 `purge_vector_orphans_for_workspace_path` 가 옛 chunk_id 를 LanceDB 에서 먼저 삭제, 이어서 `purge_orphan_at_workspace_path` 가 옛 doc / chunks / embedding_records 를 SQLite 에서 sweep. 새 byte 가 새 `doc_id` 로 색인됨. `IngestReport` 에 그 자산만 `new+=1` (다른 자산은 `updated`). 두 store 모두 정합 — 옛 본문 검색 시 옛 chunks 가 더 이상 surface 되지 않음. ### Embedding upgrade (fb-39b) -- 2.49.1 From c780aca90431fa4a7be136b954562574c6621723 Mon Sep 17 00:00:00 2001 From: altair823 Date: Tue, 19 May 2026 23:35:00 +0000 Subject: [PATCH 19/19] =?UTF-8?q?fix(p10-1a-2):=20PR=20review=20round=202?= =?UTF-8?q?=20=E2=80=94=20README=20wire=20fields=20+=20SMOKE=20config=20co?= =?UTF-8?q?mpleteness=20+=20edge-case=20note=20+=20gitignore=20dedup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #140 회차 2 actionable 4건: - README.md: `citation.kind = "code"` 행에서 wire 필드 구조 정정 — citation 안에는 `lang`, SearchHit top-level 에는 `code_lang`/`repo` (round 1 SMOKE 정정과 동일 클래스) - docs/SMOKE.md: 격리 config 블록에 `extra_skip_globs = []` 추가 (P10 섹션의 "위 격리 config 블록 참조" 와 정합) - crates/kebab-parse-code/src/rust.rs: comment-only 파일 → 0 blocks 동작을 module doc 에 한 줄 명시 (pdf-page-v1 의 "empty page produces no chunks" 패턴과 동일) - .gitignore: `/target/` 제거 — `/target` (no trailing slash) 이 디렉토리 + 파일 + 심링크 모두 매칭하므로 `/target/` (dir 전용) 는 redundant verify: `cargo check -p kebab-parse-code` clean (주석/문서 외 영향 없음). Co-Authored-By: Claude Opus 4.7 (1M context) --- .gitignore | 1 - README.md | 2 +- crates/kebab-parse-code/src/rust.rs | 5 +++++ docs/SMOKE.md | 1 + 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 8c015e5..332c3f4 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,5 @@ .worktrees/ .claude/ /target -/target/ **/*.rs.bk Cargo.lock.bak diff --git a/README.md b/README.md index 7758904..183b673 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ kebab doctor | 명령 | 동작 | |------|------| | `kebab init` | XDG 경로에 데이터 디렉토리 + config.toml 생성 | -| `kebab ingest []` | Markdown / 이미지 / PDF / Rust 소스코드 색인 (idempotent). TTY 에서는 stderr 진행 바, non-TTY (CI / pipe) 는 stderr 한 줄씩, `--json` 은 stdout 에 `ingest_progress.v1` 라인 streaming 후 마지막에 `ingest_report.v1`. Ctrl-C 한 번이면 현재 asset 마무리 후 abort (부분 commit 보존, idempotent re-run), 두 번째 Ctrl-C 는 hard exit. Markdown title 이 frontmatter 에 없어도 첫 H1 → H2 → 첫 paragraph 80 자 → 파일명 순으로 자동 채움 (parser_version `md-frontmatter-v2`) — 기존 색인된 doc 도 다음 ingest 에서 새 title 로 갱신. **Incremental** (p9-fb-23): 두 번째 이후의 ingest 는 변하지 않은 doc (blake3 + parser/chunker/embedder version 모두 동일) 의 parse/chunk/embed/vector upsert 를 자동 스킵. final summary 에 `N unchanged` 카운트 표시. `--force-reingest` 로 skip 무시 강제 재처리. **지원 형식** (extractor 자동 결정 — config 에 명시 불가): Markdown (`.md`), 이미지 (`.png` / `.jpg` / `.jpeg`, OCR + caption), PDF (`.pdf`), **Rust 소스코드** (`.rs`, tree-sitter AST chunker `code-rust-ast-v1` — p10-1A-2). 다른 확장자는 자동 skip — `IngestItem.warnings` 에 사유 (`"unsupported media type: .docx"` 등), `IngestReport.skipped_by_extension` 에 카운트 분류, CLI / TUI summary 에 breakdown 표시. 코드 chunk 는 `citation.kind = "code"` 에 `symbol` + `code_lang = "rust"` + `repo` (workspace root 상대) 포함. `--code-lang rust` / `--media code` filter 로 코드 전용 검색 가능 (p10-1A-1 filter flags). | +| `kebab ingest []` | Markdown / 이미지 / PDF / Rust 소스코드 색인 (idempotent). TTY 에서는 stderr 진행 바, non-TTY (CI / pipe) 는 stderr 한 줄씩, `--json` 은 stdout 에 `ingest_progress.v1` 라인 streaming 후 마지막에 `ingest_report.v1`. Ctrl-C 한 번이면 현재 asset 마무리 후 abort (부분 commit 보존, idempotent re-run), 두 번째 Ctrl-C 는 hard exit. Markdown title 이 frontmatter 에 없어도 첫 H1 → H2 → 첫 paragraph 80 자 → 파일명 순으로 자동 채움 (parser_version `md-frontmatter-v2`) — 기존 색인된 doc 도 다음 ingest 에서 새 title 로 갱신. **Incremental** (p9-fb-23): 두 번째 이후의 ingest 는 변하지 않은 doc (blake3 + parser/chunker/embedder version 모두 동일) 의 parse/chunk/embed/vector upsert 를 자동 스킵. final summary 에 `N unchanged` 카운트 표시. `--force-reingest` 로 skip 무시 강제 재처리. **지원 형식** (extractor 자동 결정 — config 에 명시 불가): Markdown (`.md`), 이미지 (`.png` / `.jpg` / `.jpeg`, OCR + caption), PDF (`.pdf`), **Rust 소스코드** (`.rs`, tree-sitter AST chunker `code-rust-ast-v1` — p10-1A-2). 다른 확장자는 자동 skip — `IngestItem.warnings` 에 사유 (`"unsupported media type: .docx"` 등), `IngestReport.skipped_by_extension` 에 카운트 분류, CLI / TUI summary 에 breakdown 표시. 코드 chunk 는 `citation.kind = "code"` 에 `citation.lang = "rust"` + `symbol` + line range 를 담고, SearchHit top-level 에 `code_lang = "rust"` + `repo` (`.git/` walk-up 의 디렉토리 이름) 가 backfill 됨. `--code-lang rust` / `--media code` filter 로 코드 전용 검색 가능 (p10-1A-1 filter flags). | | `kebab search --mode {lexical,vector,hybrid} "" [--no-cache] [--max-tokens N] [--snippet-chars N] [--cursor ] [--tag T] [--lang L] [--path-glob G] [--trust-min LEVEL] [--media TYPE] [--ingested-after RFC3339] [--doc-id ID] [--trace] [--bulk] [--repo NAME ...] [--code-lang LIST]` | 검색. hybrid는 RRF fusion, citation 포함. 같은 process 안에서 동일 query (NFKC + trim + lowercase 정규화) 반복 시 in-process LRU 캐시 hit (capacity = `[search] cache_capacity`, default 256). `--no-cache` 로 강제 bypass — 디버깅용. ingest commit 발생 시 `kv['corpus_revision']` bump 으로 모든 entry 자동 stale. **`--max-tokens` / `--snippet-chars` / `--cursor` (p9-fb-34)** — agent budget controls. `--json` 출력은 `search_response.v1` wrapper (`{hits, next_cursor, truncated}`) — pre-fb-34 의 bare array 와 호환 안 됨. mismatched cursor → `error.v1.code = stale_cursor`. **filter flags (p9-fb-36):** `--tag` 는 반복 가능 flag (`--tag rust --tag async`) 로 OR 매칭, `--media` 는 `,` 구분 다중 값 OR 매칭, 나머지 flags 간은 AND 조합. `--trust-min` 은 `primary\|secondary\|generated` 중 하나 (해당 level 이상 포함). `--ingested-after` 는 RFC3339 UTC — 파싱 실패 시 `error.v1.code = config_invalid` (exit 2). `--media md` 는 `markdown` alias 로 정규화. 알 수 없는 `--media` 값은 무조건 empty hits (오류 아님). **`--trace` (p9-fb-37)** — `search_response.v1.trace` 에 lexical / vector pre-fusion 후보 + RRF union + per-stage timing (`lexical_ms` / `vector_ms` / `fusion_ms` / `total_ms`) 노출. trace 요청은 캐시 우회 (`--no-cache` 없이도 항상 cold). **`--bulk` (p9-fb-42)** — stdin ndjson 으로 N query 한 번에 실행. `--json` 면 stdout per-query ndjson (`bulk_search_item.v1`) + stderr summary (`bulk_summary: total=N succeeded=S failed=F`). Cap 100. agent 가 query decomposition 후 sub-query 일괄 실행 시 single round-trip — App instance 재사용으로 캐시 / embedder cold-start 비용 한 번만. Per-query failure 는 item 의 `error` (error.v1) 에 격리, 다른 query 계속 진행. **code corpus filters (p10-1A-1):** `--repo` 는 반복 가능 (`--repo kebab --repo other`) OR 매칭. `--code-lang` 는 반복 또는 comma 다중 값 (`--code-lang rust,python`), 알 수 없는 값은 빈 hits. `--media code` 는 Tier 1/2/3 모든 code chunk 포함. 1A-1 시점에서는 indexed 된 code chunk 가 없어 filter 가 항상 빈 결과 — 1A-2 (Rust AST chunker) 머지 이후 실효. | | `kebab list docs` | 색인된 문서 목록 | | `kebab inspect doc ` / `kebab inspect chunk ` | raw record 보기 | diff --git a/crates/kebab-parse-code/src/rust.rs b/crates/kebab-parse-code/src/rust.rs index f26b208..7dcf8cc 100644 --- a/crates/kebab-parse-code/src/rust.rs +++ b/crates/kebab-parse-code/src/rust.rs @@ -15,6 +15,11 @@ //! Scope is intentionally narrow: AST unit extraction + symbol paths + //! line ranges for Rust. The `CanonicalDocument` scaffold mirrors //! `kebab-parse-pdf`. Per design §3.4 / §9.1 / §9 versioning. +//! +//! Edge cases: a Rust file consisting solely of comments / whitespace +//! (no fn / type / impl / mod / glue items) yields zero blocks → zero +//! chunks → not surfaced in search. Safe (no panic) and consistent with +//! "an empty page produces no chunks" in `pdf-page-v1`. use anyhow::Result; use kebab_core::{ diff --git a/docs/SMOKE.md b/docs/SMOKE.md index d44e214..c42a25d 100644 --- a/docs/SMOKE.md +++ b/docs/SMOKE.md @@ -118,6 +118,7 @@ theme = "dark" # p9-fb-14 — TUI palette ("dark" / "light skip_generated_header = true max_file_bytes = 262144 max_file_lines = 5000 +extra_skip_globs = [] # 사용자 추가 skip 패턴 (gitignore syntax) ``` `KEBAB_*` 환경변수로 override 가능 (`KEBAB_MODELS_LLM_MODEL=gemma4:26b kebab …` 등). 자세한 키 목록은 `crates/kebab-config/src/lib.rs` 의 `apply_env` 매치 암. `KEBAB_READONLY=1` — write-path 비활성화 (CI 안전망). `KEBAB_PROGRESS=plain` — non-TTY 환경에서 진행 상황을 plain 한 줄씩 stderr 출력 (spinner 대신). -- 2.49.1