Compare commits
22 Commits
7bbd2c0cbf
...
v0.7.0
| Author | SHA1 | Date | |
|---|---|---|---|
| 73e5b359d8 | |||
| c780aca904 | |||
| b1d5047399 | |||
| 80c2d31fb3 | |||
| 97e9f558f4 | |||
| da51e59081 | |||
| 11a0fc758f | |||
| b5d1fe8c1e | |||
| 580576c2c6 | |||
| 808b92a6c5 | |||
| c74f8d269e | |||
| df85bafa7f | |||
| a93b33ffbe | |||
| 402a4506a2 | |||
| a531dc37dc | |||
| 7a6a24ad10 | |||
| 42712b50c2 | |||
| 9f3edb7e24 | |||
| 5c265bb59f | |||
| a08ed32199 | |||
| 9362cd0aae | |||
|
|
7961f8813d |
2
.gitignore
vendored
2
.gitignore
vendored
@@ -1,6 +1,6 @@
|
||||
.superpowers/
|
||||
.worktrees/
|
||||
.claude/
|
||||
/target/
|
||||
/target
|
||||
**/*.rs.bk
|
||||
Cargo.lock.bak
|
||||
|
||||
90
Cargo.lock
generated
90
Cargo.lock
generated
@@ -4127,7 +4127,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-app"
|
||||
version = "0.6.0"
|
||||
version = "0.7.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"base64 0.22.1",
|
||||
@@ -4143,6 +4143,7 @@ dependencies = [
|
||||
"kebab-llm",
|
||||
"kebab-llm-local",
|
||||
"kebab-normalize",
|
||||
"kebab-parse-code",
|
||||
"kebab-parse-image",
|
||||
"kebab-parse-md",
|
||||
"kebab-parse-pdf",
|
||||
@@ -4171,7 +4172,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-chunk"
|
||||
version = "0.6.0"
|
||||
version = "0.7.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"blake3",
|
||||
@@ -4186,7 +4187,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-cli"
|
||||
version = "0.6.0"
|
||||
version = "0.7.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"clap",
|
||||
@@ -4207,7 +4208,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-config"
|
||||
version = "0.6.0"
|
||||
version = "0.7.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"dirs 5.0.1",
|
||||
@@ -4222,7 +4223,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-core"
|
||||
version = "0.6.0"
|
||||
version = "0.7.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"blake3",
|
||||
@@ -4236,7 +4237,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-embed"
|
||||
version = "0.6.0"
|
||||
version = "0.7.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"blake3",
|
||||
@@ -4250,7 +4251,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-embed-local"
|
||||
version = "0.6.0"
|
||||
version = "0.7.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"fastembed",
|
||||
@@ -4263,7 +4264,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-eval"
|
||||
version = "0.6.0"
|
||||
version = "0.7.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"kebab-app",
|
||||
@@ -4282,7 +4283,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-llm"
|
||||
version = "0.6.0"
|
||||
version = "0.7.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"kebab-core",
|
||||
@@ -4291,7 +4292,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-llm-local"
|
||||
version = "0.6.0"
|
||||
version = "0.7.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"kebab-config",
|
||||
@@ -4308,7 +4309,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-mcp"
|
||||
version = "0.6.0"
|
||||
version = "0.7.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"kebab-app",
|
||||
@@ -4326,7 +4327,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-normalize"
|
||||
version = "0.6.0"
|
||||
version = "0.7.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"kebab-core",
|
||||
@@ -4341,16 +4342,22 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-parse-code"
|
||||
version = "0.6.0"
|
||||
version = "0.7.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"gix",
|
||||
"kebab-core",
|
||||
"serde_json",
|
||||
"tempfile",
|
||||
"time",
|
||||
"tracing",
|
||||
"tree-sitter",
|
||||
"tree-sitter-rust",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "kebab-parse-image"
|
||||
version = "0.6.0"
|
||||
version = "0.7.0"
|
||||
dependencies = [
|
||||
"ab_glyph",
|
||||
"anyhow",
|
||||
@@ -4374,7 +4381,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-parse-md"
|
||||
version = "0.6.0"
|
||||
version = "0.7.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"kebab-core",
|
||||
@@ -4391,7 +4398,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-parse-pdf"
|
||||
version = "0.6.0"
|
||||
version = "0.7.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"blake3",
|
||||
@@ -4404,7 +4411,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-parse-types"
|
||||
version = "0.6.0"
|
||||
version = "0.7.0"
|
||||
dependencies = [
|
||||
"kebab-core",
|
||||
"serde",
|
||||
@@ -4412,7 +4419,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-rag"
|
||||
version = "0.6.0"
|
||||
version = "0.7.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"blake3",
|
||||
@@ -4433,7 +4440,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-search"
|
||||
version = "0.6.0"
|
||||
version = "0.7.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"globset",
|
||||
@@ -4452,7 +4459,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-source-fs"
|
||||
version = "0.6.0"
|
||||
version = "0.7.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"blake3",
|
||||
@@ -4470,7 +4477,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-store-sqlite"
|
||||
version = "0.6.0"
|
||||
version = "0.7.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"blake3",
|
||||
@@ -4491,7 +4498,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-store-vector"
|
||||
version = "0.6.0"
|
||||
version = "0.7.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"arrow",
|
||||
@@ -4515,7 +4522,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-tui"
|
||||
version = "0.6.0"
|
||||
version = "0.7.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"crossterm",
|
||||
@@ -7367,6 +7374,7 @@ version = "1.0.149"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86"
|
||||
dependencies = [
|
||||
"indexmap 2.14.0",
|
||||
"itoa",
|
||||
"memchr",
|
||||
"serde",
|
||||
@@ -7731,6 +7739,12 @@ version = "0.2.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e51f1e89f093f99e7432c491c382b88a6860a5adbe6bf02574bf0a08efff1978"
|
||||
|
||||
[[package]]
|
||||
name = "streaming-iterator"
|
||||
version = "0.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520"
|
||||
|
||||
[[package]]
|
||||
name = "strsim"
|
||||
version = "0.11.1"
|
||||
@@ -8495,6 +8509,36 @@ dependencies = [
|
||||
"tracing-serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tree-sitter"
|
||||
version = "0.26.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "887bd495d0582c5e3e0d8ece2233666169fa56a9644d172fc22ad179ab2d0538"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"regex",
|
||||
"regex-syntax",
|
||||
"serde_json",
|
||||
"streaming-iterator",
|
||||
"tree-sitter-language",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tree-sitter-language"
|
||||
version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "009994f150cc0cd50ff54917d5bc8bffe8cad10ca10d81c34da2ec421ae61782"
|
||||
|
||||
[[package]]
|
||||
name = "tree-sitter-rust"
|
||||
version = "0.24.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "439e577dbe07423ec2582ac62c7531120dbfccfa6e5f92406f93dd271a120e45"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"tree-sitter-language",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "try-lock"
|
||||
version = "0.2.5"
|
||||
|
||||
@@ -31,7 +31,7 @@ edition = "2024"
|
||||
rust-version = "1.85"
|
||||
license = "MIT OR Apache-2.0"
|
||||
repository = "https://github.com/altair823/kebab"
|
||||
version = "0.6.0"
|
||||
version = "0.7.0"
|
||||
|
||||
[workspace.dependencies]
|
||||
anyhow = "1"
|
||||
@@ -86,6 +86,10 @@ base64 = "0.22"
|
||||
# No `git` binary required. Default features include thread-safety + most
|
||||
# object-reading capabilities needed for HEAD name + commit SHA queries.
|
||||
gix = { version = "0.70", default-features = false, features = ["revision"] }
|
||||
# Rust source parsing for code ingest (kebab-parse-code, p10-1A-2). The
|
||||
# chunker stays tree-sitter-free — AST work is parser-side per design §6.3.
|
||||
tree-sitter = "0.26"
|
||||
tree-sitter-rust = "0.24"
|
||||
|
||||
# Disk-footprint trim for dev / test builds. Codegen, opt-level, and
|
||||
# behavior are unchanged — only DWARF debug info is reduced (line
|
||||
|
||||
@@ -20,7 +20,7 @@ P0–P5 + P6 + P7 + P9-1/2/3/4 (Library / Search / Ask / Inspect) 머지 완료.
|
||||
| **P7** | PDF text + page citation | `kebab-parse-pdf` | P5 | ✅ 완료 (3/3 component, page-level chunker + ingest wiring) |
|
||||
| **P8** | 음성 transcription + timestamp citation | `kebab-parse-audio` | P5 | ⏸ 보류 (whisper-rs 시스템 dep brainstorm 필요) |
|
||||
| **P9** | TUI + desktop app | `kebab-tui`, `kebab-desktop` | P5 | 🟡 진행 (4/5 component — P9-1/2/3/4 완료 [Library / Search / Ask / Inspect], P9-5 desktop 예정 · 도그푸딩 피드백 **20/20 ✅**) |
|
||||
| **10** | code ingest framework | `kebab-parse-code` | P5 | 🟡 진행 중 (1A-1 머지 직전) — 1A-1 머지 시점 wire schema additive minor + 새 crate kebab-parse-code skeleton 동결, 실제 code chunker 는 1A-2 부터 |
|
||||
| **P10** | code ingest framework | `kebab-parse-code` | P5 | 🟡 진행 중 — 1A-1 ✅ (wire schema + parse-code skeleton + filter flags), 1A-2 ✅ (Rust AST chunker, tree-sitter-rust, `code-rust-ast-v1` — kebab 자기 dogfooding 가능, v0.7.0) |
|
||||
|
||||
P0~P5 직렬. P6~P9 P5 이후 병렬 가능.
|
||||
|
||||
@@ -32,6 +32,7 @@ P0~P5 직렬. P6~P9 P5 이후 병렬 가능.
|
||||
|
||||
머지 후 발견된 모든 deviation / hotfix 의 dated 로그는 [tasks/HOTFIXES.md](tasks/HOTFIXES.md). 본 요약은 \"누군가가 인수받을 때 알아두면 시간을 많이 절약하는\" 항목만:
|
||||
|
||||
- **2026-05-19 P10-1A-2 (code_rust_ast_v1.rs + SourceType)** — `AST_CHUNK_MAX_LINES` 상수가 `IngestCodeCfg.ast_chunk_max_lines` 를 읽지 않고 모듈 상수 200 고정 (Chunker trait 이 per-medium config 미노출); `SourceType::Code` variant 부재로 code 파일이 `SourceType::Note` 로 분류됨 — 두 항목 모두 `tasks/HOTFIXES.md` (2026-05-19) 에 기록.
|
||||
- **2026-05-07 fb-26 (progress.rs)** — `Aborted` unconditional writeln (TTY duplicate) + `Completed` TTY no summary fixed; `KEBAB_PROGRESS=plain` env + quiet suppression added
|
||||
- **2026-05-07 fb-28 (main.rs)** — `--readonly` (KEBAB_READONLY) blocks Ingest/IngestFile/IngestStdin/Reset; `--quiet` suppresses progress stderr; error.v1 code: "readonly_mode"
|
||||
|
||||
|
||||
10
README.md
10
README.md
@@ -42,7 +42,7 @@ cargo install --git https://gitea.altair823.xyz/altair823-org/kebab.git --bin ke
|
||||
# 첫 실행 — XDG 경로에 데이터 디렉토리 + config.toml 생성
|
||||
kebab init
|
||||
|
||||
# config 손보고 — workspace.root, 모델 endpoint 등 설정 (지원 형식은 md / png / jpg / pdf 로 고정)
|
||||
# config 손보고 — workspace.root, 모델 endpoint 등 설정 (지원 형식: md / png / jpg / pdf / rs)
|
||||
${EDITOR:-vi} ~/.config/kebab/config.toml
|
||||
|
||||
# 색인 (Markdown / 이미지 / PDF 모두 한 번에)
|
||||
@@ -70,8 +70,8 @@ kebab doctor
|
||||
| 명령 | 동작 |
|
||||
|------|------|
|
||||
| `kebab init` | XDG 경로에 데이터 디렉토리 + config.toml 생성 |
|
||||
| `kebab ingest [<path>]` | Markdown / 이미지 / PDF 색인 (idempotent). TTY 에서는 stderr 진행 바, non-TTY (CI / pipe) 는 stderr 한 줄씩, `--json` 은 stdout 에 `ingest_progress.v1` 라인 streaming 후 마지막에 `ingest_report.v1`. Ctrl-C 한 번이면 현재 asset 마무리 후 abort (부분 commit 보존, idempotent re-run), 두 번째 Ctrl-C 는 hard exit. Markdown title 이 frontmatter 에 없어도 첫 H1 → H2 → 첫 paragraph 80 자 → 파일명 순으로 자동 채움 (parser_version `md-frontmatter-v2`) — 기존 색인된 doc 도 다음 ingest 에서 새 title 로 갱신. **Incremental** (p9-fb-23): 두 번째 이후의 ingest 는 변하지 않은 doc (blake3 + parser/chunker/embedder version 모두 동일) 의 parse/chunk/embed/vector upsert 를 자동 스킵. final summary 에 `N unchanged` 카운트 표시. `--force-reingest` 로 skip 무시 강제 재처리. **지원 형식** (extractor 자동 결정 — config 에 명시 불가): Markdown (`.md`), 이미지 (`.png` / `.jpg` / `.jpeg`, OCR + caption), PDF (`.pdf`). 다른 확장자는 자동 skip — `IngestItem.warnings` 에 사유 (`"unsupported media type: .docx"` 등), `IngestReport.skipped_by_extension` 에 카운트 분류, CLI / TUI summary 에 breakdown 표시. |
|
||||
| `kebab search --mode {lexical,vector,hybrid} "<query>" [--no-cache] [--max-tokens N] [--snippet-chars N] [--cursor <opaque>] [--tag T] [--lang L] [--path-glob G] [--trust-min LEVEL] [--media TYPE] [--ingested-after RFC3339] [--doc-id ID] [--trace] [--bulk] [--repo NAME ...] [--code-lang LIST] [--media code]` | 검색. hybrid는 RRF fusion, citation 포함. 같은 process 안에서 동일 query (NFKC + trim + lowercase 정규화) 반복 시 in-process LRU 캐시 hit (capacity = `[search] cache_capacity`, default 256). `--no-cache` 로 강제 bypass — 디버깅용. ingest commit 발생 시 `kv['corpus_revision']` bump 으로 모든 entry 자동 stale. **`--max-tokens` / `--snippet-chars` / `--cursor` (p9-fb-34)** — agent budget controls. `--json` 출력은 `search_response.v1` wrapper (`{hits, next_cursor, truncated}`) — pre-fb-34 의 bare array 와 호환 안 됨. mismatched cursor → `error.v1.code = stale_cursor`. **filter flags (p9-fb-36):** `--tag` 는 반복 가능 flag (`--tag rust --tag async`) 로 OR 매칭, `--media` 는 `,` 구분 다중 값 OR 매칭, 나머지 flags 간은 AND 조합. `--trust-min` 은 `primary\|secondary\|generated` 중 하나 (해당 level 이상 포함). `--ingested-after` 는 RFC3339 UTC — 파싱 실패 시 `error.v1.code = config_invalid` (exit 2). `--media md` 는 `markdown` alias 로 정규화. 알 수 없는 `--media` 값은 무조건 empty hits (오류 아님). **`--trace` (p9-fb-37)** — `search_response.v1.trace` 에 lexical / vector pre-fusion 후보 + RRF union + per-stage timing (`lexical_ms` / `vector_ms` / `fusion_ms` / `total_ms`) 노출. trace 요청은 캐시 우회 (`--no-cache` 없이도 항상 cold). **`--bulk` (p9-fb-42)** — stdin ndjson 으로 N query 한 번에 실행. `--json` 면 stdout per-query ndjson (`bulk_search_item.v1`) + stderr summary (`bulk_summary: total=N succeeded=S failed=F`). Cap 100. agent 가 query decomposition 후 sub-query 일괄 실행 시 single round-trip — App instance 재사용으로 캐시 / embedder cold-start 비용 한 번만. Per-query failure 는 item 의 `error` (error.v1) 에 격리, 다른 query 계속 진행. **code corpus filters (p10-1A-1):** `--repo` 는 반복 가능 (`--repo kebab --repo other`) OR 매칭. `--code-lang` 는 반복 또는 comma 다중 값 (`--code-lang rust,python`), 알 수 없는 값은 빈 hits. `--media code` 는 Tier 1/2/3 모든 code chunk 포함. 1A-1 시점에서는 indexed 된 code chunk 가 없어 filter 가 항상 빈 결과 — 1A-2 (Rust AST chunker) 머지 이후 실효. |
|
||||
| `kebab ingest [<path>]` | Markdown / 이미지 / PDF / Rust 소스코드 색인 (idempotent). TTY 에서는 stderr 진행 바, non-TTY (CI / pipe) 는 stderr 한 줄씩, `--json` 은 stdout 에 `ingest_progress.v1` 라인 streaming 후 마지막에 `ingest_report.v1`. Ctrl-C 한 번이면 현재 asset 마무리 후 abort (부분 commit 보존, idempotent re-run), 두 번째 Ctrl-C 는 hard exit. Markdown title 이 frontmatter 에 없어도 첫 H1 → H2 → 첫 paragraph 80 자 → 파일명 순으로 자동 채움 (parser_version `md-frontmatter-v2`) — 기존 색인된 doc 도 다음 ingest 에서 새 title 로 갱신. **Incremental** (p9-fb-23): 두 번째 이후의 ingest 는 변하지 않은 doc (blake3 + parser/chunker/embedder version 모두 동일) 의 parse/chunk/embed/vector upsert 를 자동 스킵. final summary 에 `N unchanged` 카운트 표시. `--force-reingest` 로 skip 무시 강제 재처리. **지원 형식** (extractor 자동 결정 — config 에 명시 불가): Markdown (`.md`), 이미지 (`.png` / `.jpg` / `.jpeg`, OCR + caption), PDF (`.pdf`), **Rust 소스코드** (`.rs`, tree-sitter AST chunker `code-rust-ast-v1` — p10-1A-2). 다른 확장자는 자동 skip — `IngestItem.warnings` 에 사유 (`"unsupported media type: .docx"` 등), `IngestReport.skipped_by_extension` 에 카운트 분류, CLI / TUI summary 에 breakdown 표시. 코드 chunk 는 `citation.kind = "code"` 에 `citation.lang = "rust"` + `symbol` + line range 를 담고, SearchHit top-level 에 `code_lang = "rust"` + `repo` (`.git/` walk-up 의 디렉토리 이름) 가 backfill 됨. `--code-lang rust` / `--media code` filter 로 코드 전용 검색 가능 (p10-1A-1 filter flags). |
|
||||
| `kebab search --mode {lexical,vector,hybrid} "<query>" [--no-cache] [--max-tokens N] [--snippet-chars N] [--cursor <opaque>] [--tag T] [--lang L] [--path-glob G] [--trust-min LEVEL] [--media TYPE] [--ingested-after RFC3339] [--doc-id ID] [--trace] [--bulk] [--repo NAME ...] [--code-lang LIST]` | 검색. hybrid는 RRF fusion, citation 포함. 같은 process 안에서 동일 query (NFKC + trim + lowercase 정규화) 반복 시 in-process LRU 캐시 hit (capacity = `[search] cache_capacity`, default 256). `--no-cache` 로 강제 bypass — 디버깅용. ingest commit 발생 시 `kv['corpus_revision']` bump 으로 모든 entry 자동 stale. **`--max-tokens` / `--snippet-chars` / `--cursor` (p9-fb-34)** — agent budget controls. `--json` 출력은 `search_response.v1` wrapper (`{hits, next_cursor, truncated}`) — pre-fb-34 의 bare array 와 호환 안 됨. mismatched cursor → `error.v1.code = stale_cursor`. **filter flags (p9-fb-36):** `--tag` 는 반복 가능 flag (`--tag rust --tag async`) 로 OR 매칭, `--media` 는 `,` 구분 다중 값 OR 매칭, 나머지 flags 간은 AND 조합. `--trust-min` 은 `primary\|secondary\|generated` 중 하나 (해당 level 이상 포함). `--ingested-after` 는 RFC3339 UTC — 파싱 실패 시 `error.v1.code = config_invalid` (exit 2). `--media md` 는 `markdown` alias 로 정규화. 알 수 없는 `--media` 값은 무조건 empty hits (오류 아님). **`--trace` (p9-fb-37)** — `search_response.v1.trace` 에 lexical / vector pre-fusion 후보 + RRF union + per-stage timing (`lexical_ms` / `vector_ms` / `fusion_ms` / `total_ms`) 노출. trace 요청은 캐시 우회 (`--no-cache` 없이도 항상 cold). **`--bulk` (p9-fb-42)** — stdin ndjson 으로 N query 한 번에 실행. `--json` 면 stdout per-query ndjson (`bulk_search_item.v1`) + stderr summary (`bulk_summary: total=N succeeded=S failed=F`). Cap 100. agent 가 query decomposition 후 sub-query 일괄 실행 시 single round-trip — App instance 재사용으로 캐시 / embedder cold-start 비용 한 번만. Per-query failure 는 item 의 `error` (error.v1) 에 격리, 다른 query 계속 진행. **code corpus filters (p10-1A-1):** `--repo` 는 반복 가능 (`--repo kebab --repo other`) OR 매칭. `--code-lang` 는 반복 또는 comma 다중 값 (`--code-lang rust,python`), 알 수 없는 값은 빈 hits. `--media code` 는 Tier 1/2/3 모든 code chunk 포함. 1A-1 시점에서는 indexed 된 code chunk 가 없어 filter 가 항상 빈 결과 — 1A-2 (Rust AST chunker) 머지 이후 실효. |
|
||||
| `kebab list docs` | 색인된 문서 목록 |
|
||||
| `kebab inspect doc <id>` / `kebab inspect chunk <id>` | raw record 보기 |
|
||||
| `kebab fetch chunk <id> [--context N]` / `kebab fetch doc <id> [--max-tokens N]` / `kebab fetch span <doc_id> <ls> <le> [--max-tokens N]` | (p9-fb-35) verbatim text fetch from indexed corpus. wire = `fetch_result.v1` (kind discriminator). chunk: target + ±N ordinal-context chunks. doc: full normalized markdown. span: 1-based line range (PDF/audio rejected as `error.v1.code = span_not_supported`). chars/4 budget on doc/span. |
|
||||
@@ -131,8 +131,8 @@ flowchart TB
|
||||
end
|
||||
|
||||
subgraph Pipeline["도메인 + 파이프라인"]
|
||||
parse["parse-md / parse-pdf / parse-image"]
|
||||
chunker["chunker (md-heading-v1, pdf-page-v1)"]
|
||||
parse["parse-md / parse-pdf / parse-image / parse-code"]
|
||||
chunker["chunker (md-heading-v1, pdf-page-v1, code-rust-ast-v1)"]
|
||||
embedder["embedder (fastembed multilingual-e5-large)"]
|
||||
retriever["retriever (lexical / vector / hybrid RRF)"]
|
||||
rag["RAG pipeline"]
|
||||
|
||||
@@ -32,6 +32,10 @@ kebab-parse-image = { path = "../kebab-parse-image" }
|
||||
# per-asset dispatch (see `ingest_one_asset` PDF branch) and runs the
|
||||
# resulting `CanonicalDocument` through `kebab-chunk::PdfPageV1Chunker`.
|
||||
kebab-parse-pdf = { path = "../kebab-parse-pdf" }
|
||||
# p10-1A-2: Rust AST extractor lives here. App threads it into the
|
||||
# per-asset dispatch (see `ingest_one_asset` Code branch) and runs the
|
||||
# resulting `CanonicalDocument` through `kebab-chunk::CodeRustAstV1Chunker`.
|
||||
kebab-parse-code = { path = "../kebab-parse-code" }
|
||||
anyhow = { workspace = true }
|
||||
blake3 = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
|
||||
@@ -40,8 +40,8 @@ use anyhow::{Context, Result, anyhow};
|
||||
use lru::LruCache;
|
||||
|
||||
use kebab_core::{
|
||||
Answer, Embedder, IndexVersion, LanguageModel, Retriever, SearchHit, SearchMode,
|
||||
SearchOpts, SearchQuery, VectorStore,
|
||||
Answer, DocumentStore, Embedder, IndexVersion, LanguageModel, Retriever, SearchHit,
|
||||
SearchMode, SearchOpts, SearchQuery, VectorStore,
|
||||
};
|
||||
use kebab_embed_local::FastembedEmbedder;
|
||||
use kebab_llm_local::OllamaLanguageModel;
|
||||
@@ -296,6 +296,15 @@ impl App {
|
||||
now,
|
||||
self.config.search.stale_threshold_days,
|
||||
);
|
||||
// p10-1A-2: backfill `code_lang` from the Citation::Code `lang`
|
||||
// field. The search layer (kebab-search) constructs SearchHit with
|
||||
// `code_lang: None`; we own the post-processing here in kebab-app
|
||||
// and can fill it cheaply from data already present in the hit.
|
||||
backfill_code_lang(&mut hits);
|
||||
// p10-1A-2 Task 8b: backfill `repo` from the document's
|
||||
// `Metadata.repo`. Unlike `code_lang`, this cannot be derived from
|
||||
// the Citation alone — it requires a store lookup by `doc_id`.
|
||||
self.backfill_repo(&mut hits);
|
||||
Ok(hits)
|
||||
}
|
||||
|
||||
@@ -387,6 +396,10 @@ impl App {
|
||||
now,
|
||||
self.config.search.stale_threshold_days,
|
||||
);
|
||||
// p10-1A-2: backfill code_lang — same as search_uncached.
|
||||
backfill_code_lang(&mut traced_hits);
|
||||
// p10-1A-2 Task 8b: backfill repo — same as search_uncached.
|
||||
self.backfill_repo(&mut traced_hits);
|
||||
|
||||
// Apply offset + k_effective truncation (mirrors non-trace path).
|
||||
let drop_n = offset.min(traced_hits.len());
|
||||
@@ -413,6 +426,9 @@ impl App {
|
||||
});
|
||||
}
|
||||
|
||||
// backfill_code_lang + backfill_repo are applied inside `search`
|
||||
// via `search_uncached` — no explicit call needed here. Trace
|
||||
// branch above calls them directly because it bypasses `search`.
|
||||
let mut all_hits = self.search(fetch_query)?;
|
||||
|
||||
// Skip offset.
|
||||
@@ -777,6 +793,58 @@ impl App {
|
||||
}
|
||||
}
|
||||
|
||||
/// p10-1A-2 Task 8b: back-fill `SearchHit.repo` from the originating
|
||||
/// document's `Metadata.repo` for every hit whose `repo` field is
|
||||
/// currently `None`. The search layer (kebab-search) constructs hits
|
||||
/// with `repo: None` because it has no store access; we fill it here
|
||||
/// in kebab-app post-retrieval via a per-distinct-`doc_id` store lookup.
|
||||
///
|
||||
/// Deduplication: a small `HashMap` accumulates the
|
||||
/// `(doc_id → Option<String>)` mapping so each unique document is
|
||||
/// fetched at most once. Search result sets are small (default k ≤ 20),
|
||||
/// so the map overhead is negligible. A `None` entry is cached too
|
||||
/// (document not found or no repo in metadata) to avoid re-querying.
|
||||
///
|
||||
/// Non-repo documents (markdown, PDF, plain text, code files outside a
|
||||
/// git tree) correctly keep `repo: None` — `Metadata.repo` is already
|
||||
/// `None` for those, so the assignment is a no-op.
|
||||
fn backfill_repo(&self, hits: &mut [SearchHit]) {
|
||||
use std::collections::HashMap;
|
||||
use kebab_core::DocumentId;
|
||||
|
||||
// doc_id → Option<String> where None means "not found / no repo"
|
||||
let mut cache: HashMap<DocumentId, Option<String>> = HashMap::new();
|
||||
|
||||
for hit in hits.iter_mut() {
|
||||
if hit.repo.is_some() {
|
||||
continue;
|
||||
}
|
||||
let repo_val = cache
|
||||
.entry(hit.doc_id.clone())
|
||||
.or_insert_with(|| {
|
||||
// Deliberately non-aborting: a failed store lookup for
|
||||
// one hit must not abort the whole search response. Log
|
||||
// the error so it's observable rather than silently
|
||||
// dropped (review #140 round 1).
|
||||
match self.sqlite.get_document(&hit.doc_id) {
|
||||
Ok(opt) => opt.and_then(|doc| doc.metadata.repo),
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
target: "kebab-app",
|
||||
doc_id = %hit.doc_id,
|
||||
error = %e,
|
||||
"backfill_repo: get_document failed; leaving hit.repo = None"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
});
|
||||
if let Some(r) = repo_val {
|
||||
hit.repo = Some(r.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve the embedder + vector store, surfacing the user-friendly
|
||||
/// "switch to --mode lexical" error when embeddings are disabled.
|
||||
fn require_embeddings(
|
||||
@@ -896,6 +964,21 @@ fn estimate_chars(hits: &[SearchHit]) -> usize {
|
||||
.sum()
|
||||
}
|
||||
|
||||
/// p10-1A-2: back-fill `SearchHit.code_lang` from `Citation::Code.lang`
|
||||
/// for every code hit in the list. The search layer (kebab-search)
|
||||
/// constructs hits with `code_lang: None`; we fill it here in kebab-app
|
||||
/// post-retrieval so callers see the correct language identifier without
|
||||
/// requiring a second SQL query.
|
||||
fn backfill_code_lang(hits: &mut [SearchHit]) {
|
||||
for hit in hits.iter_mut() {
|
||||
if let kebab_core::Citation::Code { lang, .. } = &hit.citation {
|
||||
if hit.code_lang.is_none() {
|
||||
hit.code_lang = lang.clone();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -96,6 +96,7 @@ pub fn media_label(media: &kebab_core::MediaType) -> &'static str {
|
||||
kebab_core::MediaType::Pdf => "pdf",
|
||||
kebab_core::MediaType::Image(_) => "image",
|
||||
kebab_core::MediaType::Audio(_) => "audio",
|
||||
kebab_core::MediaType::Code(_) => "code",
|
||||
kebab_core::MediaType::Other(_) => "other",
|
||||
}
|
||||
}
|
||||
@@ -148,6 +149,7 @@ mod tests {
|
||||
media_label(&MediaType::Audio(kebab_core::AudioType::Wav)),
|
||||
"audio"
|
||||
);
|
||||
assert_eq!(media_label(&MediaType::Code("rust".into())), "code");
|
||||
assert_eq!(media_label(&MediaType::Other("x".into())), "other");
|
||||
}
|
||||
|
||||
|
||||
@@ -39,7 +39,7 @@ use std::sync::Arc;
|
||||
use anyhow::{Context, anyhow};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use kebab_chunk::{MdHeadingV1Chunker, PdfPageV1Chunker};
|
||||
use kebab_chunk::{CodeRustAstV1Chunker, MdHeadingV1Chunker, PdfPageV1Chunker};
|
||||
use kebab_core::{
|
||||
Answer, Block, CanonicalDocument, Chunk, ChunkId, ChunkPolicy, ChunkerVersion, Chunker,
|
||||
DocFilter, DocSummary, DocumentId, DocumentStore, Embedder, EmbeddingInput,
|
||||
@@ -50,6 +50,7 @@ use kebab_core::{
|
||||
use kebab_llm_local::OllamaLanguageModel;
|
||||
use kebab_normalize::build_canonical_document;
|
||||
use kebab_parse_image::{ImageExtractor, OllamaVisionOcr, apply_caption, apply_ocr};
|
||||
use kebab_parse_code::RustAstExtractor;
|
||||
use kebab_parse_pdf::PdfTextExtractor;
|
||||
use kebab_parse_md::{BodyHints, parse_blocks, parse_frontmatter};
|
||||
use kebab_source_fs::FsSourceConnector;
|
||||
@@ -917,7 +918,21 @@ fn ingest_one_asset(
|
||||
force_reingest,
|
||||
);
|
||||
}
|
||||
_ => {
|
||||
// p10-1A-2 Task 8: Rust code ingest.
|
||||
MediaType::Code(lang) if lang == "rust" => {
|
||||
return ingest_one_code_asset(
|
||||
app,
|
||||
asset,
|
||||
chunk_policy,
|
||||
embedder,
|
||||
vector_store,
|
||||
existing_doc_ids,
|
||||
force_reingest,
|
||||
);
|
||||
}
|
||||
// p10-1A-2: non-Rust Code, Audio, and Other are not yet wired;
|
||||
// skip until their respective phases.
|
||||
MediaType::Code(_) | MediaType::Audio(_) | MediaType::Other(_) => {
|
||||
return Ok(kebab_core::IngestItem {
|
||||
kind: kebab_core::IngestItemKind::Skipped,
|
||||
doc_id: None,
|
||||
@@ -1617,6 +1632,174 @@ fn ingest_one_pdf_asset(
|
||||
})
|
||||
}
|
||||
|
||||
/// p10-1A-2 Task 8: process one `MediaType::Code("rust")` asset end-to-end.
|
||||
///
|
||||
/// Mirrors `ingest_one_pdf_asset` line-for-line with the substitutions
|
||||
/// documented in the task spec:
|
||||
/// - parser_version → `code-rust-v1` (via `RUST_PARSER_VERSION`)
|
||||
/// - extractor → `RustAstExtractor`
|
||||
/// - chunker → `CodeRustAstV1Chunker`
|
||||
///
|
||||
/// All other steps (incremental skip, byte read, ExtractContext, put_*,
|
||||
/// embed, purge_vector_orphans) are identical to the PDF function.
|
||||
fn ingest_one_code_asset(
|
||||
app: &App,
|
||||
asset: &RawAsset,
|
||||
chunk_policy: &ChunkPolicy,
|
||||
embedder: Option<&Arc<dyn Embedder + Send + Sync>>,
|
||||
vector_store: Option<&Arc<kebab_store_vector::LanceVectorStore>>,
|
||||
existing_doc_ids: &std::collections::HashSet<String>,
|
||||
force_reingest: bool,
|
||||
) -> anyhow::Result<kebab_core::IngestItem> {
|
||||
let path = match &asset.source_uri {
|
||||
SourceUri::File(p) => p.clone(),
|
||||
SourceUri::Kb(_) => {
|
||||
return Ok(kebab_core::IngestItem {
|
||||
kind: kebab_core::IngestItemKind::Skipped,
|
||||
doc_id: None,
|
||||
doc_path: asset.workspace_path.clone(),
|
||||
asset_id: Some(asset.asset_id.clone()),
|
||||
byte_len: Some(asset.byte_len),
|
||||
block_count: None,
|
||||
chunk_count: None,
|
||||
parser_version: None,
|
||||
chunker_version: None,
|
||||
warnings: vec![
|
||||
"kb:// URI not yet supported".to_string(),
|
||||
],
|
||||
error: None,
|
||||
});
|
||||
}
|
||||
};
|
||||
// p10-1A-2 task 8: incremental-ingest early-skip for the code flow.
|
||||
// Code docs use `code-rust-v1` as the parser_version and
|
||||
// `CodeRustAstV1Chunker` as the chunker — both pinned per-medium
|
||||
// today (no config knob).
|
||||
let code_parser_version =
|
||||
ParserVersion(kebab_parse_code::RUST_PARSER_VERSION.to_string());
|
||||
if let Some(item) = try_skip_unchanged(
|
||||
app,
|
||||
asset,
|
||||
&code_parser_version,
|
||||
&CodeRustAstV1Chunker.chunker_version(),
|
||||
embedder.map(|e| e.model_version()).as_ref(),
|
||||
force_reingest,
|
||||
)? {
|
||||
return Ok(item);
|
||||
}
|
||||
let bytes = std::fs::read(&path)
|
||||
.with_context(|| format!("read code asset bytes from {}", path.display()))?;
|
||||
|
||||
let extract_config = kebab_core::ExtractConfig::default();
|
||||
let workspace_root = app.config.resolve_workspace_root();
|
||||
let ctx = ExtractContext {
|
||||
asset,
|
||||
workspace_root: &workspace_root,
|
||||
config: &extract_config,
|
||||
};
|
||||
let mut canonical = RustAstExtractor::new()
|
||||
.extract(&ctx, &bytes)
|
||||
.context("kb-parse-code::RustAstExtractor::extract")?;
|
||||
|
||||
// Per-medium chunker selection: Rust code always uses code-rust-ast-v1
|
||||
// regardless of `config.chunking.chunker_version`.
|
||||
let chunker = CodeRustAstV1Chunker;
|
||||
let chunks = chunker
|
||||
.chunk(&canonical, chunk_policy)
|
||||
.context("kb-chunk::CodeRustAstV1Chunker::chunk")?;
|
||||
|
||||
// Stamp chunker + embedding versions so incremental skip detection has
|
||||
// data on the second run.
|
||||
canonical.last_chunker_version = Some(chunker.chunker_version());
|
||||
if let Some(emb) = embedder {
|
||||
canonical.last_embedding_version = Some(emb.model_version());
|
||||
}
|
||||
|
||||
purge_vector_orphans_for_workspace_path(app, asset, vector_store)?;
|
||||
app.sqlite
|
||||
.put_asset_with_bytes(asset, &bytes)
|
||||
.context("DocumentStore::put_asset_with_bytes (code)")?;
|
||||
app.sqlite
|
||||
.put_document(&canonical)
|
||||
.context("DocumentStore::put_document (code)")?;
|
||||
app.sqlite
|
||||
.put_blocks(&canonical.doc_id, &canonical.blocks)
|
||||
.context("DocumentStore::put_blocks (code)")?;
|
||||
app.sqlite
|
||||
.put_chunks(&canonical.doc_id, &chunks)
|
||||
.context("DocumentStore::put_chunks (code)")?;
|
||||
|
||||
if let (Some(emb), Some(vec_store)) = (embedder, vector_store)
|
||||
&& !chunks.is_empty()
|
||||
{
|
||||
let inputs: Vec<EmbeddingInput<'_>> = chunks
|
||||
.iter()
|
||||
.map(|c| EmbeddingInput {
|
||||
text: c.text.as_str(),
|
||||
kind: EmbeddingKind::Document,
|
||||
})
|
||||
.collect();
|
||||
let vectors = emb
|
||||
.embed(&inputs)
|
||||
.context("Embedder::embed (code chunks)")?;
|
||||
let model_id = emb.model_id();
|
||||
let model_version = emb.model_version();
|
||||
let dimensions = emb.dimensions();
|
||||
let records: Vec<VectorRecord> = chunks
|
||||
.iter()
|
||||
.zip(vectors)
|
||||
.map(|(c, v)| VectorRecord {
|
||||
embedding_id: kebab_core::id_for_embedding(
|
||||
&c.chunk_id,
|
||||
&model_id,
|
||||
&model_version,
|
||||
dimensions,
|
||||
),
|
||||
chunk_id: c.chunk_id.clone(),
|
||||
vector: v,
|
||||
doc_id: canonical.doc_id.clone(),
|
||||
text: c.text.clone(),
|
||||
heading_path: c.heading_path.clone(),
|
||||
model_id: model_id.clone(),
|
||||
model_version: model_version.clone(),
|
||||
dimensions,
|
||||
})
|
||||
.collect();
|
||||
vec_store
|
||||
.upsert(&records)
|
||||
.context("VectorStore::upsert (code)")?;
|
||||
}
|
||||
|
||||
let kind = if existing_doc_ids.contains(&canonical.doc_id.0) {
|
||||
kebab_core::IngestItemKind::Updated
|
||||
} else {
|
||||
kebab_core::IngestItemKind::New
|
||||
};
|
||||
|
||||
// Surface every `Provenance::Warning` note onto `IngestItem.warnings`.
|
||||
let warnings: Vec<String> = canonical
|
||||
.provenance
|
||||
.events
|
||||
.iter()
|
||||
.filter(|e| e.kind == kebab_core::ProvenanceKind::Warning)
|
||||
.filter_map(|e| e.note.clone())
|
||||
.collect();
|
||||
|
||||
Ok(kebab_core::IngestItem {
|
||||
kind,
|
||||
doc_id: Some(canonical.doc_id.clone()),
|
||||
doc_path: asset.workspace_path.clone(),
|
||||
asset_id: Some(asset.asset_id.clone()),
|
||||
byte_len: Some(asset.byte_len),
|
||||
block_count: u32::try_from(canonical.blocks.len()).ok(),
|
||||
chunk_count: u32::try_from(chunks.len()).ok(),
|
||||
parser_version: Some(canonical.parser_version.clone()),
|
||||
chunker_version: Some(chunker.chunker_version()),
|
||||
warnings,
|
||||
error: None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Pull the BCP-47 language hint from the canonical document. P6-1
|
||||
/// stamps `Lang("und")` by default; image-pipeline OCR / caption
|
||||
/// adapters special-case "und" so the hint is intentionally dropped
|
||||
|
||||
@@ -166,8 +166,8 @@ fn collect_stats(
|
||||
lang_breakdown: counts.lang_breakdown,
|
||||
index_bytes,
|
||||
stale_doc_count: counts.stale_doc_count,
|
||||
// p10-1A-1: populated by 1A-2 code ingest; empty until then.
|
||||
code_lang_breakdown: std::collections::BTreeMap::new(),
|
||||
// p10-1A-2: populated by the store query added in this task.
|
||||
code_lang_breakdown: store.code_lang_breakdown()?,
|
||||
repo_breakdown: std::collections::BTreeMap::new(),
|
||||
})
|
||||
}
|
||||
|
||||
200
crates/kebab-app/tests/code_ingest_smoke.rs
Normal file
200
crates/kebab-app/tests/code_ingest_smoke.rs
Normal file
@@ -0,0 +1,200 @@
|
||||
//! p10-1A-2 Task 8: smoke test for Rust code ingest dispatch.
|
||||
//!
|
||||
//! Writes a single `.rs` file into a TempDir workspace, ingests it via
|
||||
//! `kebab_app::ingest_with_config`, then searches for the symbol name and
|
||||
//! asserts that the resulting `SearchHit` carries a `Citation::Code`
|
||||
//! with the expected `lang`, `symbol`, and `line_start`.
|
||||
//!
|
||||
//! Mirrors the `pdf_pipeline.rs` harness: lexical-only (no AVX/fastembed),
|
||||
//! no OCR / caption adapters needed.
|
||||
|
||||
mod common;
|
||||
|
||||
use common::{TestEnv, lexical_query};
|
||||
|
||||
use kebab_core::{Citation, IngestItemKind};
|
||||
|
||||
/// A `.rs` file with a single `pub fn add` symbol is ingested, and a
|
||||
/// lexical search for "add" must return at least one `Citation::Code`
|
||||
/// hit whose `lang == "rust"`, `symbol == Some("add")`, and
|
||||
/// `line_start >= 1`.
|
||||
#[test]
|
||||
fn rust_file_ingests_and_searches_as_code_citation() {
|
||||
let env = TestEnv::lexical_only();
|
||||
|
||||
// Write a minimal Rust file into the workspace root.
|
||||
std::fs::write(
|
||||
env.workspace_root.join("demo.rs"),
|
||||
"/// adds two integers\npub fn add(a: i32, b: i32) -> i32 {\n a + b\n}\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let report =
|
||||
kebab_app::ingest_with_config(env.config.clone(), env.scope(), false)
|
||||
.expect("ingest must succeed");
|
||||
|
||||
assert_eq!(report.errors, 0, "no errors expected: {report:?}");
|
||||
let items = report.items.as_ref().expect("items present");
|
||||
let code_item = items
|
||||
.iter()
|
||||
.find(|i| i.doc_path.0.ends_with("demo.rs"))
|
||||
.expect("demo.rs item present");
|
||||
assert_eq!(
|
||||
code_item.kind,
|
||||
IngestItemKind::New,
|
||||
"first ingest must be New: {code_item:?}"
|
||||
);
|
||||
assert!(
|
||||
code_item.block_count.unwrap_or(0) >= 1,
|
||||
"at least one block expected: {code_item:?}"
|
||||
);
|
||||
assert!(
|
||||
code_item.chunk_count.unwrap_or(0) >= 1,
|
||||
"at least one chunk expected: {code_item:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
code_item.parser_version.as_ref().map(|p| p.0.as_str()),
|
||||
Some("code-rust-v1"),
|
||||
"parser_version must be code-rust-v1"
|
||||
);
|
||||
assert_eq!(
|
||||
code_item.chunker_version.as_ref().map(|c| c.0.as_str()),
|
||||
Some("code-rust-ast-v1"),
|
||||
"chunker_version must be code-rust-ast-v1"
|
||||
);
|
||||
|
||||
// Lexical search for the symbol name "add".
|
||||
let hits = kebab_app::search_with_config(env.config.clone(), lexical_query("add"))
|
||||
.expect("search must succeed");
|
||||
|
||||
let h = hits
|
||||
.iter()
|
||||
.find(|h| matches!(&h.citation, Citation::Code { .. }))
|
||||
.expect("at least one Citation::Code hit for 'add'");
|
||||
|
||||
match &h.citation {
|
||||
Citation::Code {
|
||||
lang,
|
||||
symbol,
|
||||
line_start,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(
|
||||
lang.as_deref(),
|
||||
Some("rust"),
|
||||
"citation.lang must be 'rust'"
|
||||
);
|
||||
assert_eq!(
|
||||
symbol.as_deref(),
|
||||
Some("add"),
|
||||
"citation.symbol must be 'add'"
|
||||
);
|
||||
assert!(*line_start >= 1, "line_start must be ≥1");
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
h.code_lang.as_deref(),
|
||||
Some("rust"),
|
||||
"SearchHit.code_lang must be 'rust'"
|
||||
);
|
||||
}
|
||||
|
||||
/// p10-1A-2 Task 8b: a code search hit must carry `SearchHit.repo` filled
|
||||
/// from the document's `Metadata.repo` (which is set by `detect_repo` during
|
||||
/// ingest). `detect_repo` returns the name of the directory that contains
|
||||
/// `.git/`, so we `git init` the workspace root before ingesting and then
|
||||
/// assert that `h.repo == Some("workspace")`.
|
||||
#[test]
|
||||
fn rust_code_search_hit_has_repo() {
|
||||
let env = TestEnv::lexical_only();
|
||||
|
||||
// `detect_repo` walks up from the file looking for `.git/`.
|
||||
// Initialise a bare git repo at the workspace root so it is
|
||||
// discoverable. We only need the `.git/` directory — no commits
|
||||
// required.
|
||||
let git_status = std::process::Command::new("git")
|
||||
.args(["init", "--quiet"])
|
||||
.arg(env.workspace_root.as_os_str())
|
||||
.status()
|
||||
.expect("git init");
|
||||
assert!(git_status.success(), "git init must succeed");
|
||||
|
||||
std::fs::write(
|
||||
env.workspace_root.join("repo_demo.rs"),
|
||||
"/// multiplies two integers\npub fn mul(a: i32, b: i32) -> i32 {\n a * b\n}\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let report =
|
||||
kebab_app::ingest_with_config(env.config.clone(), env.scope(), false)
|
||||
.expect("ingest must succeed");
|
||||
assert_eq!(report.errors, 0, "no ingest errors: {report:?}");
|
||||
|
||||
let hits = kebab_app::search_with_config(env.config.clone(), lexical_query("mul"))
|
||||
.expect("search must succeed");
|
||||
|
||||
let h = hits
|
||||
.iter()
|
||||
.find(|h| matches!(&h.citation, Citation::Code { .. }))
|
||||
.expect("at least one Citation::Code hit for 'mul'");
|
||||
|
||||
// The workspace root directory is named "workspace" by `TestEnv`.
|
||||
let expected_repo = env
|
||||
.workspace_root
|
||||
.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.map(str::to_owned);
|
||||
assert_eq!(
|
||||
h.repo,
|
||||
expected_repo,
|
||||
"SearchHit.repo must match the workspace dir name (detect_repo result)"
|
||||
);
|
||||
// Also sanity-check code_lang is still filled.
|
||||
assert_eq!(
|
||||
h.code_lang.as_deref(),
|
||||
Some("rust"),
|
||||
"SearchHit.code_lang must be 'rust'"
|
||||
);
|
||||
}
|
||||
|
||||
/// Re-ingesting the same `.rs` file without changes must report
|
||||
/// `Unchanged` (incremental-skip path exercised).
|
||||
#[test]
|
||||
fn rust_file_re_ingest_is_unchanged() {
|
||||
let env = TestEnv::lexical_only();
|
||||
|
||||
std::fs::write(
|
||||
env.workspace_root.join("stable.rs"),
|
||||
"pub fn noop() {}\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let r1 =
|
||||
kebab_app::ingest_with_config(env.config.clone(), env.scope(), false).unwrap();
|
||||
let item1 = r1
|
||||
.items
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.find(|i| i.doc_path.0.ends_with("stable.rs"))
|
||||
.cloned()
|
||||
.unwrap();
|
||||
assert_eq!(item1.kind, IngestItemKind::New);
|
||||
|
||||
let r2 =
|
||||
kebab_app::ingest_with_config(env.config.clone(), env.scope(), false).unwrap();
|
||||
let item2 = r2
|
||||
.items
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.find(|i| i.doc_path.0.ends_with("stable.rs"))
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
item2.kind,
|
||||
IngestItemKind::Unchanged,
|
||||
"identical bytes → Unchanged"
|
||||
);
|
||||
assert_eq!(item2.doc_id, item1.doc_id);
|
||||
}
|
||||
322
crates/kebab-chunk/src/code_rust_ast_v1.rs
Normal file
322
crates/kebab-chunk/src/code_rust_ast_v1.rs
Normal file
@@ -0,0 +1,322 @@
|
||||
//! `code-rust-ast-v1` — maps a tree-sitter-derived Rust AST
|
||||
//! `CanonicalDocument` (one `Block::Code` per semantic unit, each with
|
||||
//! `SourceSpan::Code`) to chunks 1:1. A unit longer than
|
||||
//! `AST_CHUNK_MAX_LINES` is split into `<symbol> [part i/N]` sub-chunks
|
||||
//! at blank-line paragraph boundaries (design §9.1 oversize fallback).
|
||||
//!
|
||||
//! tree-sitter is intentionally NOT a dependency here: AST work is
|
||||
//! parser-side (`kebab-parse-code`, design §6.3). This chunker only
|
||||
//! consumes the `CanonicalDocument`.
|
||||
//!
|
||||
//! `AST_CHUNK_MAX_LINES` is a constant matching
|
||||
//! `IngestCodeCfg::default().ast_chunk_max_lines` (200). Per-medium
|
||||
//! config threading needs a chunker registry (P+); same deviation
|
||||
//! pattern as `pdf-page-v1`'s pinned `chunker_version`
|
||||
//! (`tasks/HOTFIXES.md`).
|
||||
|
||||
use kebab_core::{
|
||||
Block, BlockId, CanonicalDocument, Chunk, ChunkPolicy, Chunker, ChunkerVersion, DocumentId,
|
||||
SourceSpan, id_for_chunk,
|
||||
};
|
||||
|
||||
const VERSION_LABEL: &str = "code-rust-ast-v1";
|
||||
const BYTES_PER_TOKEN: usize = 3;
|
||||
const POLICY_HASH_HEX_LEN: usize = 16;
|
||||
const AST_CHUNK_MAX_LINES: u32 = 200;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
pub struct CodeRustAstV1Chunker;
|
||||
|
||||
impl Chunker for CodeRustAstV1Chunker {
|
||||
fn chunker_version(&self) -> ChunkerVersion {
|
||||
ChunkerVersion(VERSION_LABEL.to_string())
|
||||
}
|
||||
|
||||
fn policy_hash(&self, policy: &ChunkPolicy) -> String {
|
||||
let bytes = serde_json_canonicalizer::to_vec(policy)
|
||||
.expect("canonical JSON serialization of ChunkPolicy must not fail");
|
||||
let hex = blake3::hash(&bytes).to_hex().to_string();
|
||||
hex[..POLICY_HASH_HEX_LEN].to_string()
|
||||
}
|
||||
|
||||
fn chunk(
|
||||
&self,
|
||||
doc: &CanonicalDocument,
|
||||
policy: &ChunkPolicy,
|
||||
) -> anyhow::Result<Vec<Chunk>> {
|
||||
for b in &doc.blocks {
|
||||
let c = match b {
|
||||
Block::Code(c) => c,
|
||||
_ => anyhow::bail!(
|
||||
"CodeRustAstV1Chunker only handles code docs (got non-Code block)"
|
||||
),
|
||||
};
|
||||
if !matches!(c.common.source_span, SourceSpan::Code { .. }) {
|
||||
anyhow::bail!(
|
||||
"CodeRustAstV1Chunker only handles code docs (got non-Code source_span)"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let base_policy_hash = self.policy_hash(policy);
|
||||
let chunker_version = self.chunker_version();
|
||||
let mut out: Vec<Chunk> = Vec::new();
|
||||
|
||||
for b in &doc.blocks {
|
||||
let cb = match b {
|
||||
Block::Code(c) => c,
|
||||
_ => unreachable!("validated above"),
|
||||
};
|
||||
let (ls, le, symbol, lang) = match &cb.common.source_span {
|
||||
SourceSpan::Code { line_start, line_end, symbol, lang } => {
|
||||
(*line_start, *line_end, symbol.clone(), lang.clone())
|
||||
}
|
||||
_ => unreachable!("validated above"),
|
||||
};
|
||||
let block_ids: Vec<BlockId> = vec![cb.common.block_id.clone()];
|
||||
let span_lines = le.saturating_sub(ls) + 1;
|
||||
|
||||
if span_lines <= AST_CHUNK_MAX_LINES {
|
||||
let span = SourceSpan::Code {
|
||||
line_start: ls,
|
||||
line_end: le,
|
||||
symbol: symbol.clone(),
|
||||
lang: lang.clone(),
|
||||
};
|
||||
out.push(make_chunk(
|
||||
doc, &chunker_version, &block_ids, &base_policy_hash,
|
||||
None, span, cb.code.clone(),
|
||||
));
|
||||
} else {
|
||||
let parts = split_oversize(&cb.code);
|
||||
let n = parts.len();
|
||||
for (i, (off_start, off_end, text)) in parts.into_iter().enumerate() {
|
||||
let part_ls = ls + off_start;
|
||||
let part_le = ls + off_end;
|
||||
let part_sym = symbol
|
||||
.as_ref()
|
||||
.map(|s| format!("{s} [part {}/{n}]", i + 1));
|
||||
let span = SourceSpan::Code {
|
||||
line_start: part_ls,
|
||||
line_end: part_le,
|
||||
symbol: part_sym,
|
||||
lang: lang.clone(),
|
||||
};
|
||||
out.push(make_chunk(
|
||||
doc, &chunker_version, &block_ids, &base_policy_hash,
|
||||
Some(part_ls), span, text,
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tracing::debug!(
|
||||
target: "kebab-chunk",
|
||||
doc_id = %doc.doc_id,
|
||||
chunks = out.len(),
|
||||
"code-rust-ast-v1 chunked",
|
||||
);
|
||||
Ok(out)
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn make_chunk(
|
||||
doc: &CanonicalDocument,
|
||||
chunker_version: &ChunkerVersion,
|
||||
block_ids: &[BlockId],
|
||||
base_policy_hash: &str,
|
||||
split_key: Option<u32>,
|
||||
span: SourceSpan,
|
||||
text: String,
|
||||
) -> Chunk {
|
||||
let id_hash = match split_key {
|
||||
Some(k) => format!("{base_policy_hash}#L{k}"),
|
||||
None => base_policy_hash.to_string(),
|
||||
};
|
||||
let chunk_id = id_for_chunk(&doc.doc_id, chunker_version, block_ids, &id_hash);
|
||||
let token_estimate = text.len().div_ceil(BYTES_PER_TOKEN);
|
||||
Chunk {
|
||||
chunk_id,
|
||||
doc_id: DocumentId(doc.doc_id.0.clone()),
|
||||
block_ids: block_ids.to_vec(),
|
||||
text,
|
||||
heading_path: Vec::new(),
|
||||
source_spans: vec![span],
|
||||
token_estimate,
|
||||
chunker_version: chunker_version.clone(),
|
||||
policy_hash: base_policy_hash.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Split an oversize unit at blank-line paragraph boundaries, greedily
|
||||
/// gluing paragraphs until ~`AST_CHUNK_MAX_LINES` lines accumulate.
|
||||
/// Returns `(line_offset_start, line_offset_end, text)` where offsets are
|
||||
/// 0-based within the unit (caller adds the unit's absolute `line_start`).
|
||||
fn split_oversize(code: &str) -> Vec<(u32, u32, String)> {
|
||||
let lines: Vec<&str> = code.split('\n').collect();
|
||||
let total = lines.len() as u32;
|
||||
let mut out: Vec<(u32, u32, String)> = Vec::new();
|
||||
let mut start: u32 = 0;
|
||||
while start < total {
|
||||
let mut end = (start + AST_CHUNK_MAX_LINES).min(total);
|
||||
let floor = start + (AST_CHUNK_MAX_LINES * 4 / 5);
|
||||
if end < total {
|
||||
if let Some(b) = (floor.min(end)..end)
|
||||
.rev()
|
||||
.find(|&i| lines[i as usize].trim().is_empty())
|
||||
{
|
||||
end = b + 1;
|
||||
}
|
||||
}
|
||||
let text = lines[start as usize..end as usize].join("\n");
|
||||
out.push((start, end.saturating_sub(1), text));
|
||||
start = end;
|
||||
}
|
||||
if out.is_empty() {
|
||||
out.push((0, total.saturating_sub(1), code.to_string()));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use kebab_core::{
|
||||
Block, CanonicalDocument, ChunkPolicy, Chunker, ChunkerVersion, CodeBlock, CommonBlock,
|
||||
SourceSpan, id_for_block, id_for_doc, AssetId, Lang, Metadata, ParserVersion, Provenance,
|
||||
SourceType, TrustLevel, WorkspacePath,
|
||||
};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
fn code_doc(units: &[(&str, u32, u32, &str)]) -> CanonicalDocument {
|
||||
let wp = WorkspacePath("crates/x/src/a.rs".into());
|
||||
let aid = AssetId("a".repeat(64));
|
||||
let pv = ParserVersion("code-rust-v1".into());
|
||||
let doc_id = id_for_doc(&wp, &aid, &pv);
|
||||
let blocks = units
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, (sym, ls, le, code))| {
|
||||
let span = SourceSpan::Code {
|
||||
line_start: *ls,
|
||||
line_end: *le,
|
||||
symbol: Some((*sym).to_string()),
|
||||
lang: Some("rust".into()),
|
||||
};
|
||||
let bid = id_for_block(&doc_id, "code", &[], i as u32, &span);
|
||||
Block::Code(CodeBlock {
|
||||
common: CommonBlock { block_id: bid, heading_path: vec![], source_span: span },
|
||||
lang: Some("rust".into()),
|
||||
code: (*code).to_string(),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
CanonicalDocument {
|
||||
doc_id, source_asset_id: aid, workspace_path: wp, title: "a".into(),
|
||||
lang: Lang("und".into()), blocks,
|
||||
metadata: Metadata {
|
||||
aliases: vec![], tags: vec![],
|
||||
created_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
updated_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
source_type: SourceType::Note, trust_level: TrustLevel::Primary,
|
||||
user_id_alias: None, user: Default::default(),
|
||||
repo: Some("kebab".into()), git_branch: Some("main".into()),
|
||||
git_commit: Some("0".repeat(40)), code_lang: Some("rust".into()),
|
||||
},
|
||||
provenance: Provenance { events: vec![] },
|
||||
parser_version: pv, schema_version: 1, doc_version: 1,
|
||||
last_chunker_version: None, last_embedding_version: None,
|
||||
}
|
||||
}
|
||||
fn policy() -> ChunkPolicy {
|
||||
ChunkPolicy { target_tokens: 500, overlap_tokens: 80,
|
||||
respect_markdown_headings: false,
|
||||
chunker_version: ChunkerVersion(VERSION_LABEL.into()) }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chunker_version_is_code_rust_ast_v1() {
|
||||
assert_eq!(CodeRustAstV1Chunker.chunker_version(),
|
||||
ChunkerVersion("code-rust-ast-v1".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn one_chunk_per_unit_preserves_code_span() {
|
||||
let doc = code_doc(&[
|
||||
("parse", 1, 3, "pub fn parse() {}\n// x\n}"),
|
||||
("Foo::double", 5, 7, "fn double() {}\n//\n}"),
|
||||
]);
|
||||
let chunks = CodeRustAstV1Chunker.chunk(&doc, &policy()).unwrap();
|
||||
assert_eq!(chunks.len(), 2);
|
||||
for c in &chunks {
|
||||
assert_eq!(c.source_spans.len(), 1);
|
||||
assert!(matches!(c.source_spans[0], SourceSpan::Code { .. }));
|
||||
assert_eq!(c.heading_path, Vec::<String>::new());
|
||||
assert_eq!(c.chunker_version.0, "code-rust-ast-v1");
|
||||
}
|
||||
match &chunks[0].source_spans[0] {
|
||||
SourceSpan::Code { symbol, line_start, line_end, .. } => {
|
||||
assert_eq!(symbol.as_deref(), Some("parse"));
|
||||
assert_eq!((*line_start, *line_end), (1, 3));
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oversize_unit_splits_into_parts_with_unique_ids() {
|
||||
let body = (0..500).map(|i| format!(" let x{i} = {i};")).collect::<Vec<_>>().join("\n");
|
||||
let code = format!("pub fn big() {{\n{body}\n}}");
|
||||
let doc = code_doc(&[("big", 1, 502, &code)]);
|
||||
let chunks = CodeRustAstV1Chunker.chunk(&doc, &policy()).unwrap();
|
||||
assert!(chunks.len() >= 2, "oversize unit must split, got {}", chunks.len());
|
||||
for c in &chunks {
|
||||
match &c.source_spans[0] {
|
||||
SourceSpan::Code { symbol, .. } => {
|
||||
assert!(symbol.as_deref().unwrap().starts_with("big [part "),
|
||||
"part-numbered symbol, got {symbol:?}");
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
let mut ids: Vec<&str> = chunks.iter().map(|c| c.chunk_id.0.as_str()).collect();
|
||||
let n = ids.len(); ids.sort(); ids.dedup();
|
||||
assert_eq!(ids.len(), n, "chunk_ids unique across split parts");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_code_doc_errors() {
|
||||
use kebab_core::TextBlock;
|
||||
let mut doc = code_doc(&[("parse", 1, 1, "fn parse(){}")]);
|
||||
doc.blocks = vec![Block::Paragraph(TextBlock {
|
||||
common: CommonBlock {
|
||||
block_id: kebab_core::BlockId("b".into()),
|
||||
heading_path: vec![],
|
||||
source_span: SourceSpan::Line { start: 1, end: 1 },
|
||||
},
|
||||
text: "x".into(), inlines: vec![],
|
||||
})];
|
||||
let err = CodeRustAstV1Chunker.chunk(&doc, &policy()).unwrap_err();
|
||||
assert!(err.to_string().contains("CodeRustAstV1Chunker"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deterministic_chunk_ids_1000() {
|
||||
let doc = code_doc(&[("parse", 1, 2, "fn parse(){}\n}")]);
|
||||
let base: Vec<String> = CodeRustAstV1Chunker.chunk(&doc, &policy())
|
||||
.unwrap().into_iter().map(|c| c.chunk_id.0).collect();
|
||||
for _ in 0..1000 {
|
||||
let again: Vec<String> = CodeRustAstV1Chunker.chunk(&doc, &policy())
|
||||
.unwrap().into_iter().map(|c| c.chunk_id.0).collect();
|
||||
assert_eq!(again, base);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn policy_hash_matches_md_heading_v1() {
|
||||
let p = policy();
|
||||
assert_eq!(CodeRustAstV1Chunker.policy_hash(&p),
|
||||
crate::MdHeadingV1Chunker.policy_hash(&p));
|
||||
}
|
||||
}
|
||||
@@ -15,8 +15,10 @@
|
||||
//! embedder, the retriever, the LLM, the RAG layer, or the UI layers.
|
||||
//! It consumes `CanonicalDocument` purely through `kb-core` types.
|
||||
|
||||
mod code_rust_ast_v1;
|
||||
mod md_heading_v1;
|
||||
mod pdf_page_v1;
|
||||
|
||||
pub use code_rust_ast_v1::CodeRustAstV1Chunker;
|
||||
pub use md_heading_v1::MdHeadingV1Chunker;
|
||||
pub use pdf_page_v1::PdfPageV1Chunker;
|
||||
|
||||
221
crates/kebab-chunk/tests/code_rust_ast_snapshot.rs
Normal file
221
crates/kebab-chunk/tests/code_rust_ast_snapshot.rs
Normal file
@@ -0,0 +1,221 @@
|
||||
//! Snapshot test pinning the `Vec<Chunk>` JSON for a
|
||||
//! representative Rust code `CanonicalDocument`.
|
||||
//!
|
||||
//! This is an integration test. `kebab-parse-code` is intentionally NOT
|
||||
//! a dev-dep (design §6.3 / §8 boundary: AST extraction is parser-side).
|
||||
//! The `CanonicalDocument` is built inline from hand-crafted `Block::Code`
|
||||
//! units, which is the same pattern used in `code_rust_ast_v1.rs`'s
|
||||
//! internal `code_doc` test helper.
|
||||
//!
|
||||
//! Set `UPDATE_SNAPSHOTS=1` to re-bake the baseline.
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use kebab_chunk::CodeRustAstV1Chunker;
|
||||
use kebab_core::{
|
||||
AssetId, Block, CanonicalDocument, ChunkPolicy, Chunker, ChunkerVersion, CodeBlock, CommonBlock,
|
||||
Lang, Metadata, ParserVersion, Provenance, SourceSpan, SourceType, TrustLevel, WorkspacePath,
|
||||
id_for_block, id_for_doc,
|
||||
};
|
||||
use serde_json::Value;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
fn fixtures_dir() -> PathBuf {
|
||||
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("tests")
|
||||
.join("fixtures")
|
||||
}
|
||||
|
||||
fn fixed_doc() -> CanonicalDocument {
|
||||
let wp = WorkspacePath("crates/kebab-chunk/src/code_rust_ast_v1.rs".into());
|
||||
let aid = AssetId("b".repeat(64));
|
||||
// Pin parser_version so doc_id / block_ids are reproducible.
|
||||
let pv = ParserVersion("code-rust-v1".into());
|
||||
let doc_id = id_for_doc(&wp, &aid, &pv);
|
||||
|
||||
// Build a >200-line function body to force split_oversize.
|
||||
let big_body: String = {
|
||||
let header = "pub fn big_fn(input: &[u8]) -> Vec<u8> {\n";
|
||||
let body: String = (0..210u32)
|
||||
.map(|i| format!(" let v{i} = input.get({i} as usize).copied().unwrap_or(0);\n"))
|
||||
.collect();
|
||||
let footer = " vec![0u8]\n}";
|
||||
format!("{header}{body}{footer}")
|
||||
};
|
||||
let big_line_count = big_body.lines().count() as u32;
|
||||
let big_line_end = 48 + big_line_count - 1;
|
||||
|
||||
// Representative units:
|
||||
// 0. top-level use+const block (lines 1–5, ≤200)
|
||||
// 1. free fn `parse` (lines 7–12, ≤200)
|
||||
// 2. struct `Foo` (lines 14–20, ≤200)
|
||||
// 3. trait `Frobable` (lines 22–30, ≤200)
|
||||
// 4. impl Foo::double (lines 32–38, ≤200)
|
||||
// 5. impl Foo::triple (lines 40–46, ≤200)
|
||||
// 6. big_fn (>200 lines) to force split_oversize
|
||||
let raw_units: Vec<(&str, u32, u32, String)> = vec![
|
||||
(
|
||||
"use+const",
|
||||
1,
|
||||
5,
|
||||
"use std::collections::HashMap;\nuse std::fmt;\n\nconst MAX: usize = 1024;\nconst MIN: usize = 0;".to_string(),
|
||||
),
|
||||
(
|
||||
"parse",
|
||||
7,
|
||||
12,
|
||||
"pub fn parse(input: &str) -> Option<u32> {\n input\n .trim()\n .parse()\n .ok()\n}".to_string(),
|
||||
),
|
||||
(
|
||||
"Foo",
|
||||
14,
|
||||
20,
|
||||
"pub struct Foo {\n pub name: String,\n pub value: u32,\n pub tags: Vec<String>,\n pub meta: Option<String>,\n pub count: usize,\n}".to_string(),
|
||||
),
|
||||
(
|
||||
"Frobable",
|
||||
22,
|
||||
30,
|
||||
"pub trait Frobable {\n fn frob(&self) -> String;\n fn frob_twice(&self) -> String {\n let a = self.frob();\n let b = self.frob();\n format!(\"{a}{b}\")\n }\n fn name(&self) -> &str;\n}".to_string(),
|
||||
),
|
||||
(
|
||||
"Foo::double",
|
||||
32,
|
||||
38,
|
||||
"impl Foo {\n pub fn double(&self) -> u32 {\n self.value\n .checked_mul(2)\n .unwrap_or(u32::MAX)\n }\n}".to_string(),
|
||||
),
|
||||
(
|
||||
"Foo::triple",
|
||||
40,
|
||||
46,
|
||||
"impl Foo {\n pub fn triple(&self) -> u32 {\n self.value\n .checked_mul(3)\n .unwrap_or(u32::MAX)\n }\n}".to_string(),
|
||||
),
|
||||
("big_fn", 48, big_line_end, big_body),
|
||||
];
|
||||
|
||||
let blocks: Vec<Block> = raw_units
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, (sym, ls, le, code))| {
|
||||
let span = SourceSpan::Code {
|
||||
line_start: *ls,
|
||||
line_end: *le,
|
||||
symbol: Some((*sym).to_string()),
|
||||
lang: Some("rust".into()),
|
||||
};
|
||||
let bid = id_for_block(&doc_id, "code", &[], i as u32, &span);
|
||||
Block::Code(CodeBlock {
|
||||
common: CommonBlock {
|
||||
block_id: bid,
|
||||
heading_path: vec![],
|
||||
source_span: span,
|
||||
},
|
||||
lang: Some("rust".into()),
|
||||
code: code.clone(),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
CanonicalDocument {
|
||||
doc_id,
|
||||
source_asset_id: aid,
|
||||
workspace_path: wp,
|
||||
title: "code_rust_ast_v1.rs".into(),
|
||||
lang: Lang("und".into()),
|
||||
blocks,
|
||||
metadata: Metadata {
|
||||
aliases: vec![],
|
||||
tags: vec![],
|
||||
created_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
updated_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
source_type: SourceType::Note,
|
||||
trust_level: TrustLevel::Primary,
|
||||
user_id_alias: None,
|
||||
user: Default::default(),
|
||||
repo: Some("kebab".into()),
|
||||
git_branch: Some("main".into()),
|
||||
git_commit: Some("0".repeat(40)),
|
||||
code_lang: Some("rust".into()),
|
||||
},
|
||||
provenance: Provenance { events: vec![] },
|
||||
parser_version: pv,
|
||||
schema_version: 1,
|
||||
doc_version: 1,
|
||||
last_chunker_version: None,
|
||||
last_embedding_version: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn fixed_policy() -> ChunkPolicy {
|
||||
ChunkPolicy {
|
||||
target_tokens: 500,
|
||||
overlap_tokens: 80,
|
||||
respect_markdown_headings: false,
|
||||
chunker_version: ChunkerVersion("code-rust-ast-v1".into()),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn code_rust_ast_chunks_snapshot() {
|
||||
let doc = fixed_doc();
|
||||
let policy = fixed_policy();
|
||||
|
||||
let chunks = CodeRustAstV1Chunker.chunk(&doc, &policy).expect("chunk");
|
||||
let actual = serde_json::to_value(&chunks).unwrap();
|
||||
|
||||
let dir = fixtures_dir();
|
||||
let baseline_path = dir.join("code-sample.chunks.snapshot.json");
|
||||
let baseline_text = match std::fs::read_to_string(&baseline_path) {
|
||||
Ok(s) => s,
|
||||
Err(_) if std::env::var("UPDATE_SNAPSHOTS").is_ok() => {
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let pretty = serde_json::to_string_pretty(&actual).unwrap();
|
||||
std::fs::write(&baseline_path, format!("{pretty}\n")).unwrap();
|
||||
return;
|
||||
}
|
||||
Err(e) => panic!(
|
||||
"missing baseline {}; run with UPDATE_SNAPSHOTS=1 to create: {e}",
|
||||
baseline_path.display()
|
||||
),
|
||||
};
|
||||
let expected: Value = serde_json::from_str(&baseline_text).expect("baseline parses as json");
|
||||
|
||||
if actual != expected {
|
||||
if std::env::var("UPDATE_SNAPSHOTS").is_ok() {
|
||||
let pretty = serde_json::to_string_pretty(&actual).unwrap();
|
||||
std::fs::write(&baseline_path, format!("{pretty}\n")).unwrap();
|
||||
eprintln!("updated baseline {}", baseline_path.display());
|
||||
return;
|
||||
}
|
||||
let pretty = serde_json::to_string_pretty(&actual).unwrap();
|
||||
panic!(
|
||||
"code-rust-ast-v1 chunks snapshot drift\n\
|
||||
--- expected ({}) ---\n{baseline_text}\n\
|
||||
--- actual ---\n{pretty}\n\
|
||||
If intentional, re-run with UPDATE_SNAPSHOTS=1.",
|
||||
baseline_path.display()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Determinism cross-check: re-running the same pipeline yields the same
|
||||
/// chunk_ids byte-for-byte.
|
||||
#[test]
|
||||
fn code_rust_ast_chunks_are_deterministic() {
|
||||
let policy = fixed_policy();
|
||||
let baseline: Vec<String> = CodeRustAstV1Chunker
|
||||
.chunk(&fixed_doc(), &policy)
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|c| c.chunk_id.0)
|
||||
.collect();
|
||||
for _ in 0..5 {
|
||||
let again: Vec<String> = CodeRustAstV1Chunker
|
||||
.chunk(&fixed_doc(), &policy)
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|c| c.chunk_id.0)
|
||||
.collect();
|
||||
assert_eq!(again, baseline);
|
||||
}
|
||||
}
|
||||
170
crates/kebab-chunk/tests/fixtures/code-sample.chunks.snapshot.json
vendored
Normal file
170
crates/kebab-chunk/tests/fixtures/code-sample.chunks.snapshot.json
vendored
Normal file
File diff suppressed because one or more lines are too long
@@ -142,6 +142,18 @@ pub enum SourceSpan {
|
||||
start_ms: u64,
|
||||
end_ms: u64,
|
||||
},
|
||||
/// p10-1A-2: AST-unit span for code ingest. Internal storage shape
|
||||
/// (chunks.source_spans_json) — `citation_helper` maps this to the
|
||||
/// wire `Citation::Code` (added 1A-1). `symbol` is the per-language
|
||||
/// self-reference path (design §3.4); `<top-level>` / `<module>` for
|
||||
/// glue regions, never null for an identified unit. `lang` is the
|
||||
/// canonical code_lang.
|
||||
Code {
|
||||
line_start: u32,
|
||||
line_end: u32,
|
||||
symbol: Option<String>,
|
||||
lang: Option<String>,
|
||||
},
|
||||
}
|
||||
|
||||
// ── Forward-declared stubs (§3.7a). Bodies are final per design. ────────
|
||||
@@ -195,6 +207,24 @@ mod tests {
|
||||
/// previously failed at serde runtime because `tag = "kind"` cannot
|
||||
/// describe a newtype carrying a non-struct value. The struct-variant
|
||||
/// shape used here is the §9 schema migration.
|
||||
#[test]
|
||||
fn source_span_code_round_trips_and_tags_lowercase() {
|
||||
let s = SourceSpan::Code {
|
||||
line_start: 10,
|
||||
line_end: 42,
|
||||
symbol: Some("foo::Bar::baz".to_string()),
|
||||
lang: Some("rust".to_string()),
|
||||
};
|
||||
let v = serde_json::to_value(&s).unwrap();
|
||||
assert_eq!(v["kind"], "code");
|
||||
assert_eq!(v["line_start"], 10);
|
||||
assert_eq!(v["line_end"], 42);
|
||||
assert_eq!(v["symbol"], "foo::Bar::baz");
|
||||
assert_eq!(v["lang"], "rust");
|
||||
let back: SourceSpan = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(back, s);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inline_serde_round_trip() {
|
||||
let cases = vec![
|
||||
|
||||
@@ -40,5 +40,23 @@ pub enum MediaType {
|
||||
Pdf,
|
||||
Image(ImageType),
|
||||
Audio(AudioType),
|
||||
/// p10-1A-2: a source-code file. Inner string is the canonical
|
||||
/// code_lang (design §3.5). 1A activates `"rust"` only; other
|
||||
/// recognized code langs are still routed `Other` until their phase.
|
||||
Code(String),
|
||||
Other(String),
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn media_type_code_serializes_lowercase_tagged() {
|
||||
let m = MediaType::Code("rust".to_string());
|
||||
let v = serde_json::to_value(&m).unwrap();
|
||||
assert_eq!(v, serde_json::json!({ "code": "rust" }));
|
||||
let back: MediaType = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(back, m);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,11 +5,17 @@ edition = { workspace = true }
|
||||
rust-version = { workspace = true }
|
||||
license = { workspace = true }
|
||||
repository = { workspace = true }
|
||||
description = "Language-aware code parsing infrastructure (lang dispatch, .git/ detect, skip helpers) for the kebab pipeline (P10-1A-1)"
|
||||
description = "Language-aware code parsing for the kebab pipeline: lang dispatch / .git detect / skip helpers (P10-1A-1) + tree-sitter Rust AST extractor (P10-1A-2)"
|
||||
|
||||
[dependencies]
|
||||
anyhow = { workspace = true }
|
||||
gix = { workspace = true }
|
||||
kebab-core = { path = "../kebab-core" }
|
||||
anyhow = { workspace = true }
|
||||
gix = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
time = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
tree-sitter = { workspace = true }
|
||||
tree-sitter-rust = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = { workspace = true }
|
||||
|
||||
@@ -15,8 +15,10 @@
|
||||
|
||||
pub mod lang;
|
||||
pub mod repo;
|
||||
pub mod rust;
|
||||
pub mod skip;
|
||||
|
||||
pub use lang::code_lang_for_path;
|
||||
pub use repo::{RepoMeta, detect_repo};
|
||||
pub use rust::{PARSER_VERSION as RUST_PARSER_VERSION, RustAstExtractor};
|
||||
pub use skip::{BUILTIN_BLACKLIST, is_generated_file, is_oversized};
|
||||
|
||||
552
crates/kebab-parse-code/src/rust.rs
Normal file
552
crates/kebab-parse-code/src/rust.rs
Normal file
@@ -0,0 +1,552 @@
|
||||
//! `kebab-parse-code::rust` — tree-sitter Rust AST extractor (P10-1A-2).
|
||||
//!
|
||||
//! Implements [`kebab_core::Extractor`] for [`MediaType::Code("rust")`].
|
||||
//! Walks the tree-sitter parse tree and emits one [`Block::Code`] per
|
||||
//! top-level AST semantic unit (free fn, type, trait, macro, each impl
|
||||
//! method, recursively per module), each carrying [`SourceSpan::Code`]
|
||||
//! with the unit's self-reference symbol path (design §3.4). Glue
|
||||
//! declarations (`use` / `const` / `static` / bodyless `mod` / top-level
|
||||
//! attributes / macro invocations) collapse into one grouped
|
||||
//! `<top-level>` (or `<module>`) unit.
|
||||
//!
|
||||
//! Doc comments and attributes immediately preceding an item are folded
|
||||
//! into that item's line range (design §9.1 "선언 + doc comment").
|
||||
//!
|
||||
//! Scope is intentionally narrow: AST unit extraction + symbol paths +
|
||||
//! line ranges for Rust. The `CanonicalDocument` scaffold mirrors
|
||||
//! `kebab-parse-pdf`. Per design §3.4 / §9.1 / §9 versioning.
|
||||
//!
|
||||
//! Edge cases: a Rust file consisting solely of comments / whitespace
|
||||
//! (no fn / type / impl / mod / glue items) yields zero blocks → zero
|
||||
//! chunks → not surfaced in search. Safe (no panic) and consistent with
|
||||
//! "an empty page produces no chunks" in `pdf-page-v1`.
|
||||
|
||||
use anyhow::Result;
|
||||
use kebab_core::{
|
||||
Block, CanonicalDocument, CodeBlock, CommonBlock, Extractor, Lang, MediaType, Metadata,
|
||||
ParserVersion, Provenance, ProvenanceEvent, ProvenanceKind, SourceSpan, SourceType, TrustLevel,
|
||||
id_for_block, id_for_doc,
|
||||
};
|
||||
use serde_json::Map;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
pub const PARSER_VERSION: &str = "code-rust-v1";
|
||||
|
||||
/// Rust AST extractor. Per-unit blocks via tree-sitter-rust 0.24
|
||||
/// (`LANGUAGE: LanguageFn`) parsed by tree-sitter 0.26.
|
||||
pub struct RustAstExtractor;
|
||||
|
||||
impl RustAstExtractor {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for RustAstExtractor {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl Extractor for RustAstExtractor {
|
||||
fn supports(&self, m: &MediaType) -> bool {
|
||||
matches!(m, MediaType::Code(l) if l == "rust")
|
||||
}
|
||||
|
||||
fn parser_version(&self) -> ParserVersion {
|
||||
ParserVersion(PARSER_VERSION.to_string())
|
||||
}
|
||||
|
||||
fn extract(
|
||||
&self,
|
||||
ctx: &kebab_core::ExtractContext<'_>,
|
||||
bytes: &[u8],
|
||||
) -> Result<CanonicalDocument> {
|
||||
let asset = ctx.asset;
|
||||
if !self.supports(&asset.media_type) {
|
||||
anyhow::bail!(
|
||||
"kebab-parse-code: unsupported media_type for RustAstExtractor: {:?}",
|
||||
asset.media_type
|
||||
);
|
||||
}
|
||||
|
||||
let parser_version = self.parser_version();
|
||||
let doc_id = id_for_doc(&asset.workspace_path, &asset.asset_id, &parser_version);
|
||||
|
||||
let source = String::from_utf8(bytes.to_vec()).map_err(|e| {
|
||||
anyhow::anyhow!("kebab-parse-code: Rust source is not valid UTF-8: {e}")
|
||||
})?;
|
||||
|
||||
let blocks = build_blocks(&source, &doc_id)?;
|
||||
let unit_count = blocks.len() as u32;
|
||||
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let mut events: Vec<ProvenanceEvent> = Vec::with_capacity(2);
|
||||
events.push(ProvenanceEvent {
|
||||
at: asset.discovered_at,
|
||||
agent: "kb-source-fs".to_string(),
|
||||
kind: ProvenanceKind::Discovered,
|
||||
note: None,
|
||||
});
|
||||
events.push(ProvenanceEvent {
|
||||
at: now,
|
||||
agent: "kb-parse-code".to_string(),
|
||||
kind: ProvenanceKind::Parsed,
|
||||
note: Some(format!(
|
||||
"parser_version={}; unit_count={}",
|
||||
parser_version.0, unit_count
|
||||
)),
|
||||
});
|
||||
|
||||
let title = {
|
||||
let fname = filename_from_workspace_path(&asset.workspace_path.0);
|
||||
strip_extension(&fname)
|
||||
};
|
||||
|
||||
// Resolve the file's absolute path for repo detection. If the
|
||||
// source URI carries a relative path, anchor it at the workspace
|
||||
// root so the `.git/` walk-up starts from the right place.
|
||||
let abs_path = match &asset.source_uri {
|
||||
kebab_core::SourceUri::File(p) => {
|
||||
if p.is_absolute() {
|
||||
p.clone()
|
||||
} else {
|
||||
ctx.workspace_root.join(p)
|
||||
}
|
||||
}
|
||||
kebab_core::SourceUri::Kb(_) => ctx.workspace_root.to_path_buf(),
|
||||
};
|
||||
let (repo, git_branch, git_commit) = match crate::repo::detect_repo(&abs_path) {
|
||||
Some(r) => (Some(r.name), r.branch, r.commit),
|
||||
None => (None, None, None),
|
||||
};
|
||||
|
||||
let metadata = Metadata {
|
||||
aliases: Vec::new(),
|
||||
tags: Vec::new(),
|
||||
created_at: asset.discovered_at,
|
||||
updated_at: asset.discovered_at,
|
||||
source_type: SourceType::Note,
|
||||
trust_level: TrustLevel::Primary,
|
||||
user_id_alias: None,
|
||||
user: Map::new(),
|
||||
repo,
|
||||
git_branch,
|
||||
git_commit,
|
||||
code_lang: Some("rust".to_string()),
|
||||
};
|
||||
|
||||
tracing::debug!(
|
||||
target: "kebab-parse-code",
|
||||
"extracted Rust doc_id={} workspace_path={} units={}",
|
||||
doc_id.0,
|
||||
asset.workspace_path.0,
|
||||
unit_count
|
||||
);
|
||||
|
||||
Ok(CanonicalDocument {
|
||||
doc_id,
|
||||
source_asset_id: asset.asset_id.clone(),
|
||||
workspace_path: asset.workspace_path.clone(),
|
||||
title,
|
||||
lang: Lang("und".to_string()),
|
||||
blocks,
|
||||
metadata,
|
||||
provenance: Provenance { events },
|
||||
parser_version,
|
||||
schema_version: 1,
|
||||
doc_version: 1,
|
||||
last_chunker_version: None,
|
||||
last_embedding_version: None,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn filename_from_workspace_path(p: &str) -> String {
|
||||
p.rsplit('/').next().unwrap_or(p).to_string()
|
||||
}
|
||||
|
||||
fn strip_extension(filename: &str) -> String {
|
||||
match filename.rfind('.') {
|
||||
Some(0) => filename.to_string(),
|
||||
Some(idx) => filename[..idx].to_string(),
|
||||
None => filename.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn build_blocks(
|
||||
source: &str,
|
||||
doc_id: &kebab_core::DocumentId,
|
||||
) -> anyhow::Result<Vec<kebab_core::Block>> {
|
||||
let mut parser = tree_sitter::Parser::new();
|
||||
parser
|
||||
.set_language(&tree_sitter_rust::LANGUAGE.into())
|
||||
.map_err(|e| anyhow::anyhow!("set tree-sitter-rust language: {e}"))?;
|
||||
let tree = parser
|
||||
.parse(source.as_bytes(), None)
|
||||
.ok_or_else(|| anyhow::anyhow!("tree-sitter failed to parse Rust source"))?;
|
||||
let lines: Vec<&str> = source.split('\n').collect();
|
||||
|
||||
// units: (symbol, line_start, line_end, is_real_semantic_unit).
|
||||
// Glue groups are pushed with a sentinel symbol + is_real=false so a
|
||||
// post-pass can decide `<module>` vs `<top-level>` (Gap 1).
|
||||
let mut units: Vec<(String, u32, u32, bool)> = Vec::new();
|
||||
let mut glue: Vec<(usize, u32, u32)> = Vec::new(); // (is_mod_decl 0/1, s, e)
|
||||
|
||||
fn node_name<'a>(n: &tree_sitter::Node, src: &'a str) -> Option<&'a str> {
|
||||
n.child_by_field_name("name")
|
||||
.map(|c| &src[c.start_byte()..c.end_byte()])
|
||||
}
|
||||
fn unit_start(n: &tree_sitter::Node) -> u32 {
|
||||
let mut start = n.start_position().row as u32 + 1;
|
||||
let mut prev = n.prev_sibling();
|
||||
while let Some(p) = prev {
|
||||
let k = p.kind();
|
||||
if k == "line_comment" || k == "block_comment" || k == "attribute_item" {
|
||||
start = p.start_position().row as u32 + 1;
|
||||
prev = p.prev_sibling();
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
start
|
||||
}
|
||||
fn walk(
|
||||
node: tree_sitter::Node,
|
||||
src: &str,
|
||||
mod_path: &[String],
|
||||
units: &mut Vec<(String, u32, u32, bool)>,
|
||||
glue: &mut Vec<(usize, u32, u32)>,
|
||||
) {
|
||||
// Module-path prefix for this scope. Used for both real units
|
||||
// (`format!("{prefix}{name}")`) and glue group labels
|
||||
// (`format!("{prefix}<top-level>")`) so glue from `mod inner`
|
||||
// doesn't collide on symbol with file-top-level glue and keeps
|
||||
// module context downstream. Empty at file top level -> glue
|
||||
// stays exactly `<top-level>` / `<module>`.
|
||||
let prefix = if mod_path.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!("{}::", mod_path.join("::"))
|
||||
};
|
||||
let mut cur = node.walk();
|
||||
for child in node.named_children(&mut cur) {
|
||||
let s = unit_start(&child);
|
||||
let e = child.end_position().row as u32 + 1;
|
||||
match child.kind() {
|
||||
"function_item" | "struct_item" | "enum_item" | "union_item"
|
||||
| "trait_item" | "type_item" => {
|
||||
if let Some(name) = node_name(&child, src) {
|
||||
// Gap 2: a leading attribute/comment that this unit
|
||||
// re-absorbs (via `unit_start`'s upward extension to
|
||||
// `s`) must not also remain in the glue group, or it
|
||||
// would be emitted in both chunks. Drop glue entries
|
||||
// at/after the unit's extended start.
|
||||
glue.retain(|(_, gs, _)| *gs < s);
|
||||
flush_glue(glue, units, &prefix);
|
||||
units.push((format!("{prefix}{name}"), s, e, true));
|
||||
}
|
||||
}
|
||||
"macro_definition" => {
|
||||
if let Some(name) = node_name(&child, src) {
|
||||
glue.retain(|(_, gs, _)| *gs < s);
|
||||
flush_glue(glue, units, &prefix);
|
||||
units.push((format!("{prefix}{name}!"), s, e, true));
|
||||
}
|
||||
}
|
||||
// `impl` blocks: emit one unit per inner `function_item`.
|
||||
// Associated consts / types / non-fn members do not become
|
||||
// their own units in 1A (plan §1A scope; HOTFIXES will log
|
||||
// if a future need arises). See inner comment below.
|
||||
"impl_item" => {
|
||||
glue.retain(|(_, gs, _)| *gs < s);
|
||||
flush_glue(glue, units, &prefix);
|
||||
let ty = child
|
||||
.child_by_field_name("type")
|
||||
.map(|c| src[c.start_byte()..c.end_byte()].trim().to_string());
|
||||
let tr = child
|
||||
.child_by_field_name("trait")
|
||||
.map(|c| src[c.start_byte()..c.end_byte()].trim().to_string());
|
||||
let owner = tr.or(ty).unwrap_or_else(|| "<impl>".to_string());
|
||||
if let Some(body) = child.child_by_field_name("body") {
|
||||
let mut bc = body.walk();
|
||||
// 1A scope: only inner `function_item` children
|
||||
// become units. Associated consts / types and other
|
||||
// non-fn impl members are intentionally NOT emitted
|
||||
// as separate units in 1A (plan spec: "1 per inner
|
||||
// function_item").
|
||||
for m in body.named_children(&mut bc) {
|
||||
if m.kind() == "function_item" {
|
||||
if let Some(mn) = node_name(&m, src) {
|
||||
let ms = unit_start(&m);
|
||||
let me = m.end_position().row as u32 + 1;
|
||||
units.push((format!("{prefix}{owner}::{mn}"), ms, me, true));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
"mod_item" => {
|
||||
if let Some(body) = child.child_by_field_name("body") {
|
||||
flush_glue(glue, units, &prefix);
|
||||
let name = node_name(&child, src).unwrap_or("mod").to_string();
|
||||
let mut np = mod_path.to_vec();
|
||||
np.push(name);
|
||||
walk(body, src, &np, units, glue);
|
||||
// Invariant: `glue` is shared by `&mut` across
|
||||
// recursive `walk` calls; every `walk` path ends with
|
||||
// a `flush_glue`, so inner-scope glue can never leak
|
||||
// into this outer scope's group. Assert it structurally
|
||||
// rather than relying on that being incidental.
|
||||
debug_assert!(
|
||||
glue.is_empty(),
|
||||
"inner walk must flush its glue before returning"
|
||||
);
|
||||
} else {
|
||||
glue.push((1, s, e));
|
||||
}
|
||||
}
|
||||
"use_declaration" | "extern_crate_declaration" | "const_item"
|
||||
| "static_item" | "attribute_item" | "macro_invocation" => {
|
||||
glue.push((0, s, e));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
flush_glue(glue, units, &prefix);
|
||||
}
|
||||
fn flush_glue(
|
||||
glue: &mut Vec<(usize, u32, u32)>,
|
||||
units: &mut Vec<(String, u32, u32, bool)>,
|
||||
prefix: &str,
|
||||
) {
|
||||
if glue.is_empty() {
|
||||
return;
|
||||
}
|
||||
let s = glue.iter().map(|(_, a, _)| *a).min().unwrap();
|
||||
let e = glue.iter().map(|(_, _, b)| *b).max().unwrap();
|
||||
// Provisional label: `<module>` only if this group is exclusively
|
||||
// bodyless `mod foo;` declarations. The final decision (Gap 1) also
|
||||
// requires the *whole file* to have produced zero real units; that
|
||||
// demotion to `<top-level>` happens in the post-pass below.
|
||||
let only_mod_decls = glue.iter().all(|(is_mod, _, _)| *is_mod == 1);
|
||||
let label = if only_mod_decls { "<module>" } else { "<top-level>" };
|
||||
// Module-path-prefix the label so glue from `mod inner` carries
|
||||
// module context (`inner::<top-level>`) and doesn't collide with
|
||||
// file-top-level glue. `prefix` is empty at file top level, so the
|
||||
// symbol stays exactly `<top-level>` / `<module>` there.
|
||||
units.push((format!("{prefix}{label}"), s, e, false));
|
||||
glue.clear();
|
||||
}
|
||||
|
||||
walk(tree.root_node(), source, &[], &mut units, &mut glue);
|
||||
|
||||
// Gap 1: `<module>` is correct only when the file produced no real
|
||||
// (non-glue) semantic unit at all. If any real unit exists, every glue
|
||||
// group is `<top-level>`, even a pure mod-decl group.
|
||||
let has_real_unit = units.iter().any(|(_, _, _, is_real)| *is_real);
|
||||
if has_real_unit {
|
||||
for (sym, _, _, is_real) in units.iter_mut() {
|
||||
// Match on the *suffix*: a glue group may now carry a module
|
||||
// prefix (`inner::<module>`), so demote any `…<module>` to the
|
||||
// same-prefixed `…<top-level>` rather than only the bare form.
|
||||
if !*is_real && sym.ends_with("<module>") {
|
||||
let pre = &sym[..sym.len() - "<module>".len()];
|
||||
*sym = format!("{pre}<top-level>");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let total_lines = lines.len() as u32;
|
||||
let mut blocks = Vec::with_capacity(units.len());
|
||||
for (ordinal, (symbol, ls, le, _is_real)) in units.into_iter().enumerate() {
|
||||
let line_start = ls.max(1);
|
||||
let line_end = le.min(total_lines.max(1));
|
||||
let span = SourceSpan::Code {
|
||||
line_start,
|
||||
line_end,
|
||||
symbol: Some(symbol),
|
||||
lang: Some("rust".to_string()),
|
||||
};
|
||||
let block_id = id_for_block(doc_id, "code", &[], ordinal as u32, &span);
|
||||
let code = lines[(line_start as usize - 1)..=(line_end as usize - 1)].join("\n");
|
||||
blocks.push(Block::Code(CodeBlock {
|
||||
common: CommonBlock {
|
||||
block_id,
|
||||
heading_path: Vec::new(),
|
||||
source_span: span,
|
||||
},
|
||||
lang: Some("rust".to_string()),
|
||||
code,
|
||||
}));
|
||||
}
|
||||
Ok(blocks)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use kebab_core::{Block, MediaType, SourceSpan};
|
||||
|
||||
fn extract_fixture() -> kebab_core::CanonicalDocument {
|
||||
let bytes = std::fs::read(
|
||||
concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/sample.rs"),
|
||||
)
|
||||
.unwrap();
|
||||
let asset = kebab_parse_code_test_support::fixed_rust_asset("crates/x/src/sample.rs");
|
||||
let cfg = kebab_core::ExtractConfig::default();
|
||||
let root = std::path::PathBuf::from("/tmp");
|
||||
let ctx = kebab_core::ExtractContext { asset: &asset, workspace_root: &root, config: &cfg };
|
||||
RustAstExtractor::new().extract(&ctx, &bytes).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extractor_supports_only_media_code_rust() {
|
||||
let e = RustAstExtractor::new();
|
||||
assert!(e.supports(&MediaType::Code("rust".into())));
|
||||
assert!(!e.supports(&MediaType::Code("python".into())));
|
||||
assert!(!e.supports(&MediaType::Markdown));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn emits_one_block_per_semantic_unit_with_symbols() {
|
||||
let doc = extract_fixture();
|
||||
let mut syms: Vec<(String, u32, u32)> = doc
|
||||
.blocks
|
||||
.iter()
|
||||
.map(|b| match b {
|
||||
Block::Code(c) => match &c.common.source_span {
|
||||
SourceSpan::Code { symbol, line_start, line_end, lang } => {
|
||||
assert_eq!(lang.as_deref(), Some("rust"));
|
||||
(symbol.clone().unwrap(), *line_start, *line_end)
|
||||
}
|
||||
_ => panic!("code block must carry SourceSpan::Code"),
|
||||
},
|
||||
other => panic!("expected Block::Code, got {other:?}"),
|
||||
})
|
||||
.collect();
|
||||
syms.sort();
|
||||
let names: Vec<&str> = syms.iter().map(|(s, _, _)| s.as_str()).collect();
|
||||
assert!(names.contains(&"parse"));
|
||||
assert!(names.contains(&"Foo"));
|
||||
assert!(names.contains(&"Foo::double"));
|
||||
assert!(names.contains(&"Foo::name"));
|
||||
assert!(names.contains(&"Greet"));
|
||||
assert!(names.contains(&"inner::helper"));
|
||||
assert!(names.contains(&"<top-level>")); // use + const grouped
|
||||
let parse_src = doc.blocks.iter().find_map(|b| match b {
|
||||
Block::Code(c) if matches!(&c.common.source_span, SourceSpan::Code{symbol,..} if symbol.as_deref()==Some("parse")) => Some(c.code.clone()),
|
||||
_ => None,
|
||||
}).unwrap();
|
||||
assert!(parse_src.contains("/// Doc comment on a free fn."), "doc comment folded in: {parse_src}");
|
||||
}
|
||||
|
||||
/// Run the extractor on an in-memory Rust source string (no fixture
|
||||
/// file) and return (symbol, code) for every emitted block.
|
||||
fn extract_inline(source: &str) -> Vec<(String, String)> {
|
||||
let asset = kebab_parse_code_test_support::fixed_rust_asset("crates/x/src/inline.rs");
|
||||
let cfg = kebab_core::ExtractConfig::default();
|
||||
let root = std::path::PathBuf::from("/tmp");
|
||||
let ctx = kebab_core::ExtractContext { asset: &asset, workspace_root: &root, config: &cfg };
|
||||
let doc = RustAstExtractor::new()
|
||||
.extract(&ctx, source.as_bytes())
|
||||
.unwrap();
|
||||
doc.blocks
|
||||
.iter()
|
||||
.map(|b| match b {
|
||||
Block::Code(c) => match &c.common.source_span {
|
||||
SourceSpan::Code { symbol, .. } => {
|
||||
(symbol.clone().unwrap(), c.code.clone())
|
||||
}
|
||||
_ => panic!("code block must carry SourceSpan::Code"),
|
||||
},
|
||||
other => panic!("expected Block::Code, got {other:?}"),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn module_label_scope_and_attribute_dedup() {
|
||||
// Source A (Gap 2): leading attribute is re-absorbed into the unit
|
||||
// and must NOT also form a separate <top-level> glue chunk.
|
||||
let a = extract_inline("#[derive(Debug)]\npub struct Tagged { x: u32 }\n");
|
||||
assert_eq!(a.len(), 1, "Gap 2: exactly one block, got {a:?}");
|
||||
assert_eq!(a[0].0, "Tagged");
|
||||
assert!(
|
||||
a[0].1.contains("#[derive(Debug)]"),
|
||||
"attribute folded into unit: {:?}",
|
||||
a[0].1
|
||||
);
|
||||
assert!(
|
||||
!a.iter().any(|(s, _)| s == "<top-level>"),
|
||||
"attribute must not also form a glue chunk: {a:?}"
|
||||
);
|
||||
|
||||
// Source B (Gap 1): file has no real units, only bodyless mod
|
||||
// decls -> the glue group is <module>.
|
||||
let b = extract_inline("mod a;\nmod b;\n");
|
||||
assert_eq!(b.len(), 1, "one glue block, got {b:?}");
|
||||
assert_eq!(b[0].0, "<module>");
|
||||
|
||||
// Source C (Gap 1): mod decls + a real unit -> the glue group is
|
||||
// <top-level>, NOT <module>, because the file has a real unit.
|
||||
let c = extract_inline("mod a;\nmod b;\npub fn f() {}\n");
|
||||
let syms: Vec<&str> = c.iter().map(|(s, _)| s.as_str()).collect();
|
||||
assert!(syms.contains(&"f"), "real unit present: {c:?}");
|
||||
assert!(
|
||||
syms.contains(&"<top-level>"),
|
||||
"mod-decl glue demoted to <top-level>: {c:?}"
|
||||
);
|
||||
assert!(
|
||||
!syms.contains(&"<module>"),
|
||||
"must not be <module> when file has a real unit: {c:?}"
|
||||
);
|
||||
|
||||
// Source D (Fix 1): glue inside a bodied `mod inner` must carry the
|
||||
// module-path prefix so it doesn't collide with file-top-level glue
|
||||
// and keeps module context downstream.
|
||||
let d = extract_inline("mod inner {\n use std::fmt;\n pub fn helper() {}\n}\n");
|
||||
let dsyms: Vec<&str> = d.iter().map(|(s, _)| s.as_str()).collect();
|
||||
assert!(
|
||||
dsyms.contains(&"inner::helper"),
|
||||
"real unit inside mod is prefixed: {d:?}"
|
||||
);
|
||||
assert!(
|
||||
dsyms.contains(&"inner::<top-level>"),
|
||||
"glue inside mod inner is module-prefixed, not bare: {d:?}"
|
||||
);
|
||||
assert!(
|
||||
!dsyms.contains(&"<top-level>"),
|
||||
"glue inside mod inner must NOT be the bare top-level symbol: {d:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deterministic_across_runs() {
|
||||
let a = extract_fixture();
|
||||
for _ in 0..50 {
|
||||
assert_eq!(extract_fixture().blocks, a.blocks);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod kebab_parse_code_test_support {
|
||||
use kebab_core::*;
|
||||
use time::OffsetDateTime;
|
||||
pub fn fixed_rust_asset(path: &str) -> RawAsset {
|
||||
RawAsset {
|
||||
asset_id: AssetId("a".repeat(64)),
|
||||
source_uri: SourceUri::File(std::path::PathBuf::from(path)),
|
||||
workspace_path: WorkspacePath(path.to_string()),
|
||||
media_type: MediaType::Code("rust".to_string()),
|
||||
byte_len: 0,
|
||||
checksum: Checksum("b".repeat(64)),
|
||||
discovered_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
stored: AssetStorage::Reference {
|
||||
path: std::path::PathBuf::from(path),
|
||||
sha: Checksum("b".repeat(64)),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
35
crates/kebab-parse-code/tests/fixtures/sample.rs
vendored
Normal file
35
crates/kebab-parse-code/tests/fixtures/sample.rs
vendored
Normal file
@@ -0,0 +1,35 @@
|
||||
//! sample fixture
|
||||
|
||||
use std::fmt;
|
||||
|
||||
const ANSWER: u32 = 42;
|
||||
|
||||
/// Doc comment on a free fn.
|
||||
pub fn parse(input: &str) -> usize {
|
||||
input.len()
|
||||
}
|
||||
|
||||
pub struct Foo {
|
||||
pub n: u32,
|
||||
}
|
||||
|
||||
impl Foo {
|
||||
/// method doc
|
||||
pub fn double(&self) -> u32 {
|
||||
self.n * 2
|
||||
}
|
||||
|
||||
fn name() -> &'static str {
|
||||
"foo"
|
||||
}
|
||||
}
|
||||
|
||||
pub trait Greet {
|
||||
fn hello(&self) -> String;
|
||||
}
|
||||
|
||||
mod inner {
|
||||
pub fn helper() -> bool {
|
||||
true
|
||||
}
|
||||
}
|
||||
@@ -49,6 +49,13 @@ pub(crate) fn citation_from_first_span(
|
||||
end_ms: *end_ms,
|
||||
speaker: None,
|
||||
},
|
||||
Some(SourceSpan::Code { line_start, line_end, symbol, lang }) => Citation::Code {
|
||||
path,
|
||||
line_start: *line_start,
|
||||
line_end: *line_end,
|
||||
symbol: symbol.clone(),
|
||||
lang: lang.clone(),
|
||||
},
|
||||
// Byte-spans don't have a Citation variant. Fall back to a
|
||||
// Line citation pointing at the document head — better than
|
||||
// fabricating a position. Spans-empty falls into the same
|
||||
@@ -72,3 +79,43 @@ pub(crate) fn citation_from_first_span(
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use kebab_core::{Citation, SourceSpan, WorkspacePath};
|
||||
|
||||
#[test]
|
||||
fn build_citation_code_maps_symbol_and_lang() {
|
||||
let span = SourceSpan::Code {
|
||||
line_start: 5,
|
||||
line_end: 30,
|
||||
symbol: Some("chunk::md_heading_v1::MdHeadingV1Chunker::chunk".into()),
|
||||
lang: Some("rust".into()),
|
||||
};
|
||||
let c = super::citation_from_first_span(
|
||||
"c1",
|
||||
WorkspacePath::new("crates/kebab-chunk/src/md_heading_v1.rs".to_string()).unwrap(),
|
||||
None,
|
||||
Some(&span),
|
||||
);
|
||||
match c {
|
||||
Citation::Code {
|
||||
path,
|
||||
line_start,
|
||||
line_end,
|
||||
symbol,
|
||||
lang,
|
||||
} => {
|
||||
assert_eq!(path.0, "crates/kebab-chunk/src/md_heading_v1.rs");
|
||||
assert_eq!(line_start, 5);
|
||||
assert_eq!(line_end, 30);
|
||||
assert_eq!(
|
||||
symbol.as_deref(),
|
||||
Some("chunk::md_heading_v1::MdHeadingV1Chunker::chunk")
|
||||
);
|
||||
assert_eq!(lang.as_deref(), Some("rust"));
|
||||
}
|
||||
other => panic!("expected Citation::Code, got {other:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,6 +34,10 @@ pub(crate) fn media_type_for(path: &Path) -> MediaType {
|
||||
"flac" => MediaType::Audio(AudioType::Flac),
|
||||
"ogg" => MediaType::Audio(AudioType::Ogg),
|
||||
|
||||
// p10-1A-2: Rust is the only code lang activated in 1A. Other
|
||||
// recognized code langs stay Other until their phase (1B+).
|
||||
"rs" => MediaType::Code("rust".to_string()),
|
||||
|
||||
// Empty string (no extension) and any other extension: bucket as
|
||||
// Other and let downstream extractors decide if they support it.
|
||||
_ => MediaType::Other(ext),
|
||||
@@ -71,6 +75,17 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rust_files_map_to_media_code_rust() {
|
||||
assert_eq!(
|
||||
media_type_for(Path::new("crates/kebab-core/src/lib.rs")),
|
||||
MediaType::Code("rust".to_string())
|
||||
);
|
||||
// non-Rust code extensions stay Other in 1A
|
||||
assert_eq!(media_type_for(Path::new("a/b.py")), MediaType::Other("py".to_string()));
|
||||
assert_eq!(media_type_for(Path::new("Cargo.toml")), MediaType::Other("toml".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_and_missing_extension() {
|
||||
assert_eq!(
|
||||
|
||||
@@ -669,6 +669,38 @@ impl SqliteStore {
|
||||
) -> anyhow::Result<CountSummary> {
|
||||
self.count_summary_inner(threshold_days)
|
||||
}
|
||||
|
||||
/// p10-1A-2: per-code-language doc count for `schema.v1`.
|
||||
///
|
||||
/// Reads `metadata_json->'$.code_lang'`, groups by the value, and
|
||||
/// skips rows where `code_lang` is NULL (i.e. non-code documents).
|
||||
/// Returns `BTreeMap<String, u32>` — key is the canonical lowercase
|
||||
/// language identifier (e.g. `"rust"`), value is the doc count.
|
||||
pub fn code_lang_breakdown(
|
||||
&self,
|
||||
) -> anyhow::Result<std::collections::BTreeMap<String, u32>> {
|
||||
use anyhow::Context;
|
||||
let conn = self.read_conn();
|
||||
let mut stmt = conn
|
||||
.prepare(
|
||||
"SELECT json_extract(metadata_json, '$.code_lang') AS cl, COUNT(*) \
|
||||
FROM documents \
|
||||
WHERE cl IS NOT NULL \
|
||||
GROUP BY cl",
|
||||
)
|
||||
.context("prepare code_lang_breakdown")?;
|
||||
let rows = stmt
|
||||
.query_map([], |r| {
|
||||
Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)? as u32))
|
||||
})
|
||||
.context("query code_lang_breakdown")?;
|
||||
let mut out = std::collections::BTreeMap::new();
|
||||
for row in rows {
|
||||
let (k, v) = row.context("read code_lang_breakdown row")?;
|
||||
out.insert(k, v);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
}
|
||||
|
||||
/// Apply the design §5 / task-spec pragmas. Called once per connection.
|
||||
@@ -710,5 +742,80 @@ mod tests {
|
||||
assert!(s.lang_breakdown.is_empty());
|
||||
assert_eq!(s.stale_doc_count, 0);
|
||||
}
|
||||
|
||||
/// p10-1A-2: `code_lang_breakdown` counts docs by `metadata_json.code_lang`.
|
||||
///
|
||||
/// Inserts:
|
||||
/// - one doc with `code_lang = "rust"` → must appear with count 1
|
||||
/// - one doc with `code_lang = null` → must NOT appear (NULL skipped)
|
||||
///
|
||||
/// Uses a side rusqlite connection that bypasses the `assets` FK via
|
||||
/// `PRAGMA foreign_keys = OFF` so the test is self-contained.
|
||||
#[test]
|
||||
fn code_lang_breakdown_counts_by_code_lang() {
|
||||
let (dir, store) = open_fresh_store();
|
||||
|
||||
// Insert two document rows directly. Disabling FK enforcement lets
|
||||
// us skip the companion `assets` insert.
|
||||
let db_path = dir.path().join("kebab.sqlite");
|
||||
let conn = rusqlite::Connection::open(&db_path).unwrap();
|
||||
conn.pragma_update(None, "foreign_keys", "OFF").unwrap();
|
||||
|
||||
// Doc 1: Rust code file — code_lang = "rust"
|
||||
conn.execute(
|
||||
"INSERT INTO documents (
|
||||
doc_id, asset_id, workspace_path,
|
||||
source_type, trust_level, parser_version,
|
||||
doc_version, schema_version,
|
||||
metadata_json, provenance_json,
|
||||
created_at, updated_at
|
||||
) VALUES (
|
||||
'doc-rust-1', 'asset-1', 'src/main.rs',
|
||||
'reference', 'primary', 'test-v1',
|
||||
1, 1,
|
||||
'{\"code_lang\":\"rust\"}', '{}',
|
||||
'2024-01-01T00:00:00Z', '2024-01-01T00:00:00Z'
|
||||
)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// Doc 2: Markdown doc — code_lang absent (null in JSON)
|
||||
conn.execute(
|
||||
"INSERT INTO documents (
|
||||
doc_id, asset_id, workspace_path,
|
||||
source_type, trust_level, parser_version,
|
||||
doc_version, schema_version,
|
||||
metadata_json, provenance_json,
|
||||
created_at, updated_at
|
||||
) VALUES (
|
||||
'doc-md-1', 'asset-2', 'notes/readme.md',
|
||||
'markdown', 'primary', 'test-v1',
|
||||
1, 1,
|
||||
'{\"code_lang\":null}', '{}',
|
||||
'2024-01-01T00:00:00Z', '2024-01-01T00:00:00Z'
|
||||
)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
drop(conn); // release side connection before querying via store
|
||||
|
||||
let bd = store.code_lang_breakdown().unwrap();
|
||||
|
||||
// rust must appear with count 1
|
||||
assert_eq!(
|
||||
bd.get("rust"),
|
||||
Some(&1u32),
|
||||
"expected rust=1 in code_lang_breakdown, got: {bd:?}"
|
||||
);
|
||||
// null code_lang must NOT appear as any key
|
||||
assert!(
|
||||
!bd.contains_key("null"),
|
||||
"null code_lang must not appear in breakdown, got: {bd:?}"
|
||||
);
|
||||
// only one key total
|
||||
assert_eq!(bd.len(), 1, "expected exactly 1 entry, got: {bd:?}");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -455,6 +455,15 @@ fn describe_span(span: &kebab_core::SourceSpan) -> String {
|
||||
SourceSpan::Time { start_ms, end_ms } => {
|
||||
format!("Time {start_ms}-{end_ms} ms")
|
||||
}
|
||||
SourceSpan::Code {
|
||||
line_start,
|
||||
line_end,
|
||||
symbol,
|
||||
..
|
||||
} => match symbol {
|
||||
Some(sym) => format!("Code {line_start}-{line_end} ({sym})"),
|
||||
None => format!("Code {line_start}-{line_end}"),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -22,6 +22,7 @@ Cargo workspace, 함수 호출 기반 모듈러 모놀리스. UI binary (`kebab-
|
||||
| OCR | Ollama vision LM (default `gemma4:e4b`) — `OcrEngine` trait 으로 Tesseract / Apple Vision 등 future swap (HOTFIXES P6-2) |
|
||||
| Image caption | Ollama vision LM, runtime gate `image.caption.enabled` (default OFF) |
|
||||
| PDF parser | `lopdf` per-page 텍스트, `chunker_version = "pdf-page-v1"` 가 PDF 자산에 하드코딩 (HOTFIXES P7-3) |
|
||||
| code parser | `tree-sitter` + `tree-sitter-rust` — **parser-side** (`kebab-parse-code`), chunker-side 아님 (design §6.3). `chunker_version = "code-rust-ast-v1"`. `ast_chunk_max_lines = 200` 상수 고정 (HOTFIXES 2026-05-19 — Chunker trait 이 per-medium config 미노출). |
|
||||
| TUI | Ratatui + crossterm — P9-1 Library 패널, P9-2/3/4 진행 예정 |
|
||||
| Desktop | Tauri 2 + `pdfjs-dist` (native PDF render backend 금지) — P9-5 |
|
||||
| citation 형식 | URI fragment (`path#L12-L34` / `path#p=12` / `path#xywh=0,0,100,50`, W3C Media Fragments) |
|
||||
@@ -50,6 +51,7 @@ flowchart TB
|
||||
ppdf["kebab-parse-pdf"]
|
||||
pimg["kebab-parse-image"]
|
||||
paud["kebab-parse-audio<br/>(P8 보류)"]
|
||||
pcode["kebab-parse-code<br/>(P10-1A-2)"]
|
||||
ptypes["kebab-parse-types"]
|
||||
norm["kebab-normalize"]
|
||||
chunk["kebab-chunk"]
|
||||
@@ -80,6 +82,7 @@ flowchart TB
|
||||
app --> ppdf
|
||||
app --> pimg
|
||||
app --> paud
|
||||
app --> pcode
|
||||
app --> norm
|
||||
app --> chunk
|
||||
app --> sqlite
|
||||
@@ -95,6 +98,7 @@ flowchart TB
|
||||
ppdf --> ptypes
|
||||
pimg --> ptypes
|
||||
paud --> ptypes
|
||||
pcode --> core
|
||||
norm --> ptypes
|
||||
embedlocal --> embed
|
||||
llmlocal --> llm
|
||||
@@ -158,7 +162,7 @@ kebab/
|
||||
│ ├── kebab-source-fs/ # 워크스페이스 walk + checksum (P1-1)
|
||||
│ ├── kebab-parse-md/ # Markdown frontmatter + blocks (P1-2/3)
|
||||
│ ├── kebab-normalize/ # ParsedBlock → CanonicalDocument (P1-4)
|
||||
│ ├── kebab-chunk/ # heading-aware + pdf-page-v1 chunker (P1-5, P7-2)
|
||||
│ ├── kebab-chunk/ # heading-aware + pdf-page-v1 + code-rust-ast-v1 chunker (P1-5, P7-2, P10-1A-2)
|
||||
│ ├── kebab-store-sqlite/ # SQLite + FTS5 (V001/V002/V003) (P1-6, P2-1, P3-3)
|
||||
│ ├── kebab-search/ # Lexical + Vector + Hybrid retriever (P2-2, P3-4)
|
||||
│ ├── kebab-embed/ kebab-embed-local/ # Embedder trait + fastembed adapter (P3-1, P3-2)
|
||||
@@ -168,6 +172,7 @@ kebab/
|
||||
│ ├── kebab-eval/ # golden query runner + metrics (P5-1, P5-2)
|
||||
│ ├── kebab-parse-image/ # ImageExtractor + Ollama OCR + caption (P6)
|
||||
│ ├── kebab-parse-pdf/ # lopdf per-page text extractor (P7-1)
|
||||
│ ├── kebab-parse-code/ # tree-sitter Rust AST extractor (P10-1A-2); chunker lives in kebab-chunk
|
||||
│ ├── kebab-app/ # facade (P0 시그니처 + P3-5/P6-4/P7-3 본체)
|
||||
│ ├── kebab-tui/ # Ratatui shell + Library 패널 (P9-1)
|
||||
│ ├── kebab-mcp/ # stdio MCP server — tools: schema, doctor, search, ask (P9-FB-30)
|
||||
|
||||
@@ -118,6 +118,7 @@ theme = "dark" # p9-fb-14 — TUI palette ("dark" / "light
|
||||
skip_generated_header = true
|
||||
max_file_bytes = 262144
|
||||
max_file_lines = 5000
|
||||
extra_skip_globs = [] # 사용자 추가 skip 패턴 (gitignore syntax)
|
||||
```
|
||||
|
||||
`KEBAB_*` 환경변수로 override 가능 (`KEBAB_MODELS_LLM_MODEL=gemma4:26b kebab …` 등). 자세한 키 목록은 `crates/kebab-config/src/lib.rs` 의 `apply_env` 매치 암. `KEBAB_READONLY=1` — write-path 비활성화 (CI 안전망). `KEBAB_PROGRESS=plain` — non-TTY 환경에서 진행 상황을 plain 한 줄씩 stderr 출력 (spinner 대신).
|
||||
@@ -302,6 +303,43 @@ kebab --config /tmp/kebab-smoke/config.toml ask "<PDF 본문에 관한 질문>"
|
||||
|
||||
각 명령은 0 종료 코드면 정상. `kebab ask` 는 거절 시 종료 코드 1 (`RefusalSignal`) — 의도된 동작.
|
||||
|
||||
## P10-1A-2 Rust 코드 색인
|
||||
|
||||
`kebab-parse-code` 의 tree-sitter Rust AST extractor + `code-rust-ast-v1` chunker 를 격리된 TempDir KB 에서 검증하는 절차.
|
||||
|
||||
```bash
|
||||
# 1) 워크스페이스에 Rust 소스 파일 추가 (crate 하나 복사 또는 단일 .rs 파일)
|
||||
cp -r crates/kebab-parse-code /tmp/kebab-smoke/workspace/kebab-parse-code
|
||||
|
||||
# 2) ingest — .rs 가 code-rust-ast-v1 로 처리됨
|
||||
KB ingest
|
||||
|
||||
# 3) 결과 검증 — IngestReport.items 에 .rs 자산이 "new" 로 분류, parser_version = "code-rust-v1" (chunker_version = "code-rust-ast-v1")
|
||||
KB --json ingest | jq '[.items[] | select(.doc_path | endswith(".rs"))]'
|
||||
|
||||
# 4) 코드 검색 — code_lang 필터 (wire: lang 은 citation.lang, code_lang 은 SearchHit top-level)
|
||||
KB search --mode hybrid "RustAstExtractor" --code-lang rust --json | jq '{hits: [.hits[] | {symbol: .citation.symbol, code_lang: .citation.lang, repo: .repo}]}'
|
||||
|
||||
# 5) citation 확인 — kind="code", symbol 이 함수명 / 타입명, line range 가 포함
|
||||
KB search --mode lexical "pub fn extract" --code-lang rust --json | jq '.hits[0].citation'
|
||||
```
|
||||
|
||||
`[ingest.code]` 설정 (config.toml 에 이미 포함됨 — 위 격리 config 블록 참조):
|
||||
|
||||
```toml
|
||||
[ingest.code]
|
||||
skip_generated_header = true # @generated / DO NOT EDIT 감지 시 skip
|
||||
max_file_bytes = 262144 # 256 KiB cap — 초과 시 skip
|
||||
max_file_lines = 5000 # 5000 줄 cap — 초과 시 skip
|
||||
extra_skip_globs = [] # 사용자 추가 skip 패턴
|
||||
```
|
||||
|
||||
**알려진 동작 (2026-05-19 기준)**:
|
||||
|
||||
- `ast_chunk_max_lines = 200` 은 config 가 아닌 chunker 모듈 상수. 현재 기본값과 동일하므로 user-visible 차이 없음. 자세한 내용: `tasks/HOTFIXES.md` (2026-05-19 `AST_CHUNK_MAX_LINES` 항목).
|
||||
- `.rs` 파일은 `SourceType::Note` 로 분류됨 (kebab-core `SourceType::Code` variant 미존재). `--media code` filter 는 정상 동작 — `MediaType::Code("rust")` 로 별도 분류됨. 자세한 내용: `tasks/HOTFIXES.md` (2026-05-19 `SourceType::Code` 항목).
|
||||
- `.gitignore` 가 honor 됨 — `target/` / `node_modules/` 등은 built-in 안전망으로 자동 skip.
|
||||
|
||||
## 검증 체크리스트
|
||||
|
||||
- `kebab doctor` 가 `--config` path 를 honor 하고 그 안의 `storage.data_dir` 를 출력 (XDG default 가 아님).
|
||||
@@ -332,6 +370,7 @@ rm -rf /tmp/kebab-smoke # 통째로 정리
|
||||
- (P6-4) `image.ocr.enabled = true` + `image.caption.enabled = true` 인 워크스페이스에 PNG 가 N장 있으면 ingest 시간 ≈ markdown_time + N × (OCR + Caption latency). `gemma4:e4b` + 192.168.0.47 로 자산당 ~5-10초. 다수의 책 페이지를 이미지로 넣지 말 것 — 책은 P7 PDF 라인 사용 권장.
|
||||
- (P7-3) `config.chunking.chunker_version` 는 markdown 만 represent — PDF 자산은 `pdf-page-v1` 하드코딩. `config.toml` 의 `chunker_version = "md-heading-v1"` 을 봐도 PDF 는 영향 안 받음. HOTFIXES `2026-05-02 P7-3` entry 참조 (P+ chunker registry task 까지 유지).
|
||||
- (P7-3) 한 PDF 가 N 페이지면 `kebab ingest` 가 N 개 (또는 그 이상의, 페이지 길면 multi-chunk) 의 chunk 를 한 transaction 안에서 commit. 500 페이지 책 → 500+ chunk 한 번에 → embedding throughput 가 bottleneck. 임베딩 활성 워크스페이스에서 큰 PDF 를 처음 ingest 하면 분-단위 시간 + WAL 크기 증가 가능 — P+ 스케일 hardening task 까지 정상 동작이지만 비용은 측정 가능.
|
||||
- (P10-1A-2) `.rs` 파일을 워크스페이스에 두면 `kebab ingest` 결과에 `new` 카운터에 포함. `kebab search --mode hybrid "<함수명>" --code-lang rust --json` 가 `citation.kind = "code"`, `citation.lang = "rust"` (SearchHit top-level `code_lang` 도 동일), `citation.symbol` (함수/타입 이름), `citation.line_start` / `citation.line_end` 를 반환하면 wiring 정상. `kebab schema --json | jq .stats.code_lang_breakdown` 에 `"rust": N` 이 나오면 chunk 가 색인됨.
|
||||
- (P7-3 + follow-up) 동일 path 에 byte 가 다른 PDF 를 두 번째 ingest 하면 `purge_vector_orphans_for_workspace_path` 가 옛 chunk_id 를 LanceDB 에서 먼저 삭제, 이어서 `purge_orphan_at_workspace_path` 가 옛 doc / chunks / embedding_records 를 SQLite 에서 sweep. 새 byte 가 새 `doc_id` 로 색인됨. `IngestReport` 에 그 자산만 `new+=1` (다른 자산은 `updated`). 두 store 모두 정합 — 옛 본문 검색 시 옛 chunks 가 더 이상 surface 되지 않음.
|
||||
|
||||
### Embedding upgrade (fb-39b)
|
||||
|
||||
1441
docs/superpowers/plans/2026-05-19-p10-1a-2-rust-ast-chunker.md
Normal file
1441
docs/superpowers/plans/2026-05-19-p10-1a-2-rust-ast-chunker.md
Normal file
File diff suppressed because it is too large
Load Diff
@@ -182,12 +182,15 @@ $ kebab search "Markdown chunking 규칙"
|
||||
"page": { "page": 13, "section": "Experiment Setup" },
|
||||
"region": { "x": 120, "y": 40, "w": 520, "h": 180 },
|
||||
"caption": { "model": "qwen2.5-vl:7b" },
|
||||
"time": { "start_ms": 822000, "end_ms": 850000, "speaker": "S1" },
|
||||
"code": { "start": 10, "end": 42, "lang": "rust", "repo": "kebab", "symbol": "fn ingest" }
|
||||
"time": { "start_ms": 822000, "end_ms": 850000, "speaker": "S1" }
|
||||
}
|
||||
```
|
||||
|
||||
code variant example (p10-1A-1):
|
||||
variant 별 해당 키만 채움. `path` 와 `uri` 는 항상 채움 (`uri` 는 path + W3C Media Fragments 합본).
|
||||
|
||||
**구현 노트 (wire 실제 형태):** 위 nested form 은 illustrative 구조. 실제 wire 는 `#[serde(tag = "kind")]` 외부 tag enum 이라 variant 별 필드가 *top-level* 에 들어감 (e.g. `Line` → `{"kind":"line", "start":12, "end":34, ...}`, nested 형태 아님). 모든 6 variant 동일.
|
||||
|
||||
**code variant (p10-1A-1, flat wire form):** 자세한 contract 은 2026-05-15 code ingest spec §3.1 참조. 5 필드 — `path`, `line_start`, `line_end`, `symbol` (Option<String>, AST 결과면 채움), `lang` (Option<String>, lowercase canonical). `repo` 는 Citation 이 아니라 `SearchHit` / `Metadata` 에 surface.
|
||||
|
||||
```json
|
||||
{
|
||||
@@ -195,12 +198,13 @@ code variant example (p10-1A-1):
|
||||
"kind": "code",
|
||||
"path": "crates/kebab-app/src/ingest.rs",
|
||||
"uri": "crates/kebab-app/src/ingest.rs#L10-L42",
|
||||
"code": { "start": 10, "end": 42, "lang": "rust", "repo": "kebab", "symbol": "fn ingest" }
|
||||
"line_start": 10,
|
||||
"line_end": 42,
|
||||
"symbol": "fn ingest",
|
||||
"lang": "rust"
|
||||
}
|
||||
```
|
||||
|
||||
variant 별 해당 키만 채움. `path` 와 `uri` 는 항상 채움 (`uri` 는 path + W3C Media Fragments 합본).
|
||||
|
||||
### 2.2 SearchHit
|
||||
|
||||
```json
|
||||
@@ -227,9 +231,8 @@ variant 별 해당 키만 채움. `path` 와 `uri` 는 항상 채움 (`uri` 는
|
||||
},
|
||||
"index_version": "v1.0",
|
||||
"embedding_model": "multilingual-e5-large",
|
||||
"chunker_version": "md-heading-v1",
|
||||
"repo": null, // p10-1A-1: optional, omitted when null (code corpus only)
|
||||
"code_lang": null // p10-1A-1: optional, omitted when null (code corpus only)
|
||||
"chunker_version": "md-heading-v1"
|
||||
// p10-1A-1: 코드 hit 에만 surface — `"repo": "kebab"` / `"code_lang": "rust"` 같은 키 추가됨. markdown hit 에는 키 자체 absent (skip_serializing_if).
|
||||
}
|
||||
```
|
||||
|
||||
@@ -485,6 +488,7 @@ pub enum MediaType {
|
||||
Pdf,
|
||||
Image(ImageType),
|
||||
Audio(AudioType),
|
||||
Code(String), // p10-1A-2: source-code file; inner = canonical code_lang (e.g. "rust")
|
||||
Other(String),
|
||||
}
|
||||
|
||||
@@ -562,6 +566,7 @@ pub enum SourceSpan {
|
||||
Page { page: u32, char_start: Option<u32>, char_end: Option<u32> },
|
||||
Region { x: u32, y: u32, w: u32, h: u32 },
|
||||
Time { start_ms: u64, end_ms: u64 },
|
||||
Code { line_start: u32, line_end: u32, symbol: Option<String>, lang: Option<String> }, // p10-1A-2: internal code-unit span (see tasks/p10/p10-1a-2)
|
||||
}
|
||||
```
|
||||
|
||||
@@ -1536,6 +1541,8 @@ HOTFIXES 의 `2026-05-07 — p9-fb-27` 항목이 details shape 의
|
||||
interim deviation (IoFailure / OpTimeout 신규 typed signal 도입 전까지의
|
||||
transitional 형태) 의 source of truth.
|
||||
|
||||
**p10-1A-2 surface 활성화 (2026-05-19)**: Rust 소스코드 ingest (`code-rust-ast-v1` chunker, `tree-sitter-rust`) 가 활성화됨. `.rs` 파일을 워크스페이스에 두면 `kebab ingest` 가 AST 단위로 chunk 생성 + `citation.kind = "code"` 로 검색 가능. `kebab schema --json` 의 `stats.code_lang_breakdown` 에 `"rust": N` 이 표시됨. 본 activation 으로 kebab 자기 crate 를 dogfooding KB 에 색인 가능. `SourceSpan::Code` (§3.4) 와 `MediaType::Code` (§3.5) 는 1A-1 에서 이미 spec 에 반영됨. 두 deferred deviation (`AST_CHUNK_MAX_LINES` 상수 고정, `SourceType::Code` 미존재) 은 `tasks/HOTFIXES.md` (2026-05-19) 에 기록.
|
||||
|
||||
### 10.2 MCP server transport (fb-30)
|
||||
|
||||
`kebab mcp` 가 stdio JSON-RPC server. Rust SDK = `rmcp 1.6`. Tool surface
|
||||
|
||||
@@ -14,6 +14,30 @@ historical contract that was implemented; this file accumulates the
|
||||
deltas so phase 5+ readers can find the live behavior without diffing
|
||||
git history.
|
||||
|
||||
## 2026-05-19 — p10-1A-2: AST_CHUNK_MAX_LINES constant vs config deviation
|
||||
|
||||
**무엇이 바뀌었나**: `kebab-chunk/src/code_rust_ast_v1.rs` 가 `IngestCodeCfg.ast_chunk_max_lines` config 값을 읽지 않고 모듈 상수 `AST_CHUNK_MAX_LINES = 200` 으로 고정함.
|
||||
|
||||
**원인**: 현행 `Chunker` trait 이 per-medium config 를 인자로 받지 않는다. PDF 선례 (`pdf-page-v1` 의 pinned `chunker_version`) 와 같은 패턴 — chunker 가 config 를 bolt-on 으로 받을 수 있는 per-medium chunker registry 는 P+ task.
|
||||
|
||||
**사용자 가시적 영향**: 없음 (상수 200 이 `IngestCodeCfg::default().ast_chunk_max_lines` 와 동일). 사용자가 config 에서 `ast_chunk_max_lines` 를 변경해도 Rust AST chunker 에는 반영 안 됨.
|
||||
|
||||
**proper fix**: per-medium chunker registry 도입 시 `RustAstV1Chunker` 가 `IngestCodeCfg` 를 주입받도록 변경. 별도 P+ task.
|
||||
|
||||
**cross-link**: `tasks/p10/p10-1a-2-rust-ast-chunker.md` Risks / notes 섹션 참조.
|
||||
|
||||
## 2026-05-19 — p10-1A-2: SourceType::Code deferred — code files classified SourceType::Note
|
||||
|
||||
**무엇이 바뀌었나**: `kebab-core` 의 `SourceType` enum 에 `Code` variant 가 없어 `kebab-parse-code::RustAstExtractor` 가 `SourceType::Note` 로 fallback 함.
|
||||
|
||||
**원인**: `SourceType::Code` 추가는 additive (소규모) 변경이지만, 1A-2 PR 스코프를 넓히지 않기 위해 명시적으로 deferred. Plan 이 이 fallback 을 예상했음 — 기능 회귀 아님.
|
||||
|
||||
**사용자 가시적 영향**: 없음. `--media code` / `--code-lang rust` filter 는 `MediaType::Code("rust")` 기반으로 동작 (SourceType 과 독립). 현재 code 파일에 source_type 기반 필터링 표면 없음.
|
||||
|
||||
**proper fix**: `kebab-core::SourceType` 에 `Code` variant 추가 + `citation_helper` + `store-sqlite` 의 exhaustive match 갱신. 별도 소규모 task (P10-1A-2 follow-up).
|
||||
|
||||
**cross-link**: `tasks/p10/p10-1a-2-rust-ast-chunker.md` Risks / notes 섹션 참조.
|
||||
|
||||
## 2026-05-10 — p9-fb-39b: embedding upgrade UX
|
||||
|
||||
**무엇이 바뀌었나**: default embedding 이 `multilingual-e5-small` (384 dim) 에서 `multilingual-e5-large` (1024 dim) 로 변경. LanceDB 테이블은 `(model, dim)` 으로 네임스페이스되어 새 모델은 fresh 테이블에 쓰고, 옛 `chunk_embeddings_multilingual-e5-small_384` 테이블은 orphan 상태 됨.
|
||||
|
||||
@@ -139,8 +139,8 @@ P0~P5 는 직렬. P6~P9 는 P5 이후 병렬 가능.
|
||||
- [p9-fb-42 bulk multi-query + re-rank hint](p9/p9-fb-42-bulk-multi-query-rerank.md) — ✅ 머지 (2026-05-10) — bulk only, rerank hint deferred
|
||||
|
||||
- P10 — [p10/](p10/) — code ingest (multi-task, sub-indexed in [p10/INDEX.md](p10/INDEX.md))
|
||||
- [p10-1A-1 code ingest framework](p10/p10-1a-1-code-ingest-framework.md) — 🟡 진행 중
|
||||
- p10-1A-2 Rust AST chunker — ⏳
|
||||
- [p10-1A-1 code ingest framework](p10/p10-1a-1-code-ingest-framework.md) — ✅ 머지
|
||||
- [p10-1A-2 Rust AST chunker](p10/p10-1a-2-rust-ast-chunker.md) — 🟡 PR 오픈 (코드 완성, 머지 대기)
|
||||
- p10-1B Python + TS/JS AST chunkers — ⏳
|
||||
- p10-1C Go + Java + Kotlin AST chunkers — ⏳
|
||||
- p10-1D C + C++ AST chunkers — ⏳
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
|
||||
| ID | Subject | Status |
|
||||
|----|---------|--------|
|
||||
| 1A-1 | code ingest framework (wire schema, parse-code crate skeleton, filter flags, skip policy, config 절) | 🟡 진행 중 |
|
||||
| 1A-2 | Rust AST chunker | ⏳ |
|
||||
| 1A-1 | code ingest framework (wire schema, parse-code crate skeleton, filter flags, skip policy, config 절) | ✅ 머지 |
|
||||
| 1A-2 | Rust AST chunker | 🟡 PR 오픈 (코드 완성, 머지 대기) |
|
||||
| 1B | Python + TS/JS AST chunkers | ⏳ |
|
||||
| 1C | Go + Java + Kotlin AST chunkers | ⏳ |
|
||||
| 1D | C + C++ AST chunkers | ⏳ |
|
||||
|
||||
49
tasks/p10/p10-1a-2-rust-ast-chunker.md
Normal file
49
tasks/p10/p10-1a-2-rust-ast-chunker.md
Normal file
@@ -0,0 +1,49 @@
|
||||
# p10-1A-2 — Rust AST chunker
|
||||
|
||||
**Status:** 🟡 진행 중
|
||||
**Contract sections:** §3.3 (chunker_version `code-rust-ast-v1`), §3.4 (symbol path — Rust convention), §3.4 frozen-design (`SourceSpan::Code` 신규 internal variant), §5 (code ingest 활성화), §6.1 (`kebab-parse-code/src/rust.rs` — tree-sitter-rust → CanonicalDocument), §6.2 (`kebab-chunk/src/code_rust_ast_v1.rs`), §9.1 (Tier 1 AST per-language + oversize fallback).
|
||||
**Design:** [2026-05-15-kebab-code-ingest-design.md](../../docs/superpowers/specs/2026-05-15-kebab-code-ingest-design.md) §1A-2.
|
||||
**Plan:** [2026-05-19-p10-1a-2-rust-ast-chunker.md](../../docs/superpowers/plans/2026-05-19-p10-1a-2-rust-ast-chunker.md).
|
||||
|
||||
## Goal
|
||||
|
||||
1A-1 의 프레임워크 위에 **Rust AST chunker 자체**를 올린다. `tree-sitter` + `tree-sitter-rust` 도입, `kebab-parse-code/src/rust.rs` (tree-sitter-rust → `CanonicalDocument`, AST 의미 단위마다 `Block::Code` + `SourceSpan::Code`), `kebab-chunk/src/code_rust_ast_v1.rs` (1 block → 1 chunk + oversize fallback split), `MediaType::Code` 신설, `kebab-app` dispatch. 머지 시점에 kebab 자기 자신 dogfooding 가능.
|
||||
|
||||
## 동결된 설계 결정 (이 task 로 확정)
|
||||
|
||||
- **tree-sitter 위치 = parser (`kebab-parse-code`)**, chunker 아님. design §6.3 의존성 그래프 (`kebab-parse-code → tree-sitter, tree-sitter-rust`) 가 authoritative. PDF 선례와 동형 — parser 가 구조화된 block 생성, chunker 가 매핑. §9.1 의 "chunker 가 AST" 서술은 *oversize fallback split* 만 chunker-side 라는 의미로 해석.
|
||||
- **`SourceSpan::Code { line_start, line_end, symbol, lang }` 내부 variant 신설** (kebab-core). chunk 의 `source_spans_json` (chunks 테이블) 은 *내부 저장*이라 wire schema 아님 → wire major bump 불필요. `Citation::Code` (wire) 는 1A-1 에서 이미 추가됨. `citation_helper::citation_from_first_span` 에 `SourceSpan::Code → Citation::Code` arm 추가로 symbol/lang 이 자연스럽게 흐름.
|
||||
- **`MediaType::Code(String)` 신설** — String = canonical code_lang (1A 는 `"rust"` 만 실제 처리, 그 외 인식된 code lang 은 `Skipped` — Tier 2/3 는 후속 phase).
|
||||
- frozen design §3.4 의 `SourceSpan` enum 및 (해당 시) `MediaType` enum 목록을 같은 PR 에서 갱신. 본 task spec 은 머지 후 frozen.
|
||||
|
||||
## Acceptance criteria
|
||||
|
||||
- `cargo test --workspace --no-fail-fast -j 1` passes.
|
||||
- 기존 markdown / PDF / image corpus regression test 무영향 (citation 5→6 variant: `Citation::Code` 는 1A-1 에 이미 존재; 기존 5 variant 직렬화 불변).
|
||||
- `cargo clippy --workspace --all-targets -- -D warnings` passes.
|
||||
- Rust fixture 한 개 (fn / impl method / struct / trait / top-level use + 200줄 초과 fn) ingest → chunk snapshot 안정 + `Citation::Code` 의 symbol/line 이 spec §3.4 Rust convention 과 일치.
|
||||
- kebab 자기 crate 한 개를 isolated TempDir KB 에 ingest → `kebab search --json` 결과가 `citation.kind == "code"`, `repo`, `code_lang == "rust"` 반환 (SMOKE 절차).
|
||||
- `docs/superpowers/specs/2026-04-27-kebab-final-form-design.md` §3.4 (SourceSpan / MediaType) + §10.1 갱신.
|
||||
- README + HANDOFF + ARCHITECTURE + SMOKE + tasks/INDEX.md + tasks/p10/INDEX.md 갱신.
|
||||
- workspace `Cargo.toml` version minor bump (도그푸딩 가능 = bump 트리거, design §10.4) + release cut.
|
||||
|
||||
## Allowed dependencies
|
||||
|
||||
- `kebab-parse-code` 에 `tree-sitter` + `tree-sitter-rust` 추가 (workspace deps 경유). 기존 `kebab-core` / `anyhow` / `gix` 유지.
|
||||
- `kebab-chunk` 는 `kebab-core` 만 (chunker 는 `CanonicalDocument` 만 소비 — tree-sitter import 금지).
|
||||
- `kebab-app → kebab-parse-code` (facade 가 Extractor 호출).
|
||||
|
||||
## Forbidden dependencies
|
||||
|
||||
- `kebab-chunk` 가 `tree-sitter*` import 금지 (AST 는 parser-side).
|
||||
- UI crate (cli / mcp / tui) 가 `kebab-parse-code` 직접 import 금지 — `kebab-app` facade 만.
|
||||
- `kebab-parse-code` 가 store / embed / llm / rag import 금지 (design §8 inheritance).
|
||||
|
||||
## Risks / notes
|
||||
|
||||
- tree-sitter-rust 의 grammar 버전에 따라 node kind 명칭 차이 가능 — `function_item` / `impl_item` / `struct_item` / `enum_item` / `trait_item` / `mod_item` / `use_declaration` 는 도입 버전으로 pin 후 테스트로 고정.
|
||||
- `SourceSpan::Code` 추가로 `SourceSpan` 의 모든 exhaustive match (citation_helper, store-sqlite serde, search) 가 영향 — 컴파일러가 non-exhaustive 를 잡아주므로 전수 대응.
|
||||
- oversize fallback (단일 fn > `ast_chunk_max_lines`) 의 `symbol [part i/N]` 표기는 1A-2 chunker 내부 한정. 일반 Tier-3 `code-text-paragraph-v1` 은 Phase 3.
|
||||
- 머지 후 동작 deviation 은 `tasks/HOTFIXES.md` 에 dated 로그 + 본 spec `Risks / notes` 에 one-line cross-link.
|
||||
- AST_CHUNK_MAX_LINES deviation logged in HOTFIXES.md (2026-05-19): `Chunker` trait 이 per-medium config 미노출 — 상수 200 고정, default 와 동일하므로 user-visible 영향 없음.
|
||||
- SourceType::Code deferred logged in HOTFIXES.md (2026-05-19): code 파일이 `SourceType::Note` 로 분류됨, `MediaType::Code` 기반 filter 는 정상 동작.
|
||||
Reference in New Issue
Block a user