Compare commits
72 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 1e4cff879b | |||
| 2d7a566624 | |||
| 813bdd1a16 | |||
| ff1bedbef5 | |||
| 30e03c7a12 | |||
| 2ce6ae47c5 | |||
| ebc4ef2eea | |||
| 7bda1509b7 | |||
| 61d48d67a3 | |||
| f4c840b994 | |||
| 15244b7494 | |||
| a7f7ab9f93 | |||
| 1b19e33a4f | |||
| 9c9e391b15 | |||
| f95cd55484 | |||
| ab288135e9 | |||
| c19aa006d0 | |||
| f1a4f67e12 | |||
| 6463c52827 | |||
| 2559d0d95a | |||
| 4524830306 | |||
| 8cdd3903c7 | |||
| 8b89961ada | |||
| eec90996aa | |||
| ce1c778b4a | |||
| 453ec15df4 | |||
| 1e6de9fe9f | |||
| 9fa2a1ebac | |||
| 749c6ae240 | |||
| 5f2bd9e97e | |||
| 1ce06c1e2d | |||
| d26efe167f | |||
| d6d165df01 | |||
| 2baa846c6b | |||
| 27baec82ea | |||
| acf8cf3be2 | |||
| ea5f7b22c8 | |||
| 5497c6e7b5 | |||
| 5a90940f1c | |||
| 4389b887f0 | |||
| 360f825f3a | |||
| 641b92af7d | |||
| 08fb743598 | |||
| 0a2a7ae214 | |||
| 803d02b68b | |||
| 4e8b84c4e0 | |||
| 16dc02cfa2 | |||
| 74f1b0571b | |||
| 918ee6c0be | |||
| 68ada396f3 | |||
| 23c4ad97b9 | |||
| 1f566b8bfa | |||
| 26562588e3 | |||
| 4503b5b12f | |||
| 44813df052 | |||
| d6bb6cfd3b | |||
| d53995a6d4 | |||
| c215034653 | |||
| 31245a4328 | |||
| acb61b6830 | |||
| 20feb3133e | |||
| de63f161ac | |||
| 1815091247 | |||
| 6a0b340941 | |||
| 9664e97497 | |||
| 8bdb3e8090 | |||
| dcad9ccda2 | |||
| ed0f4769b3 | |||
| 0c61758931 | |||
| 39b766ea59 | |||
| 7f287abacb | |||
| d715631928 |
113
Cargo.lock
generated
113
Cargo.lock
generated
@@ -4127,7 +4127,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-app"
|
||||
version = "0.7.0"
|
||||
version = "0.13.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"base64 0.22.1",
|
||||
@@ -4172,7 +4172,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-chunk"
|
||||
version = "0.7.0"
|
||||
version = "0.13.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"blake3",
|
||||
@@ -4187,7 +4187,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-cli"
|
||||
version = "0.7.0"
|
||||
version = "0.13.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"clap",
|
||||
@@ -4208,7 +4208,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-config"
|
||||
version = "0.7.0"
|
||||
version = "0.13.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"dirs 5.0.1",
|
||||
@@ -4223,7 +4223,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-core"
|
||||
version = "0.7.0"
|
||||
version = "0.13.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"blake3",
|
||||
@@ -4237,7 +4237,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-embed"
|
||||
version = "0.7.0"
|
||||
version = "0.13.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"blake3",
|
||||
@@ -4251,7 +4251,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-embed-local"
|
||||
version = "0.7.0"
|
||||
version = "0.13.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"fastembed",
|
||||
@@ -4264,7 +4264,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-eval"
|
||||
version = "0.7.0"
|
||||
version = "0.13.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"kebab-app",
|
||||
@@ -4283,7 +4283,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-llm"
|
||||
version = "0.7.0"
|
||||
version = "0.13.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"kebab-core",
|
||||
@@ -4292,7 +4292,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-llm-local"
|
||||
version = "0.7.0"
|
||||
version = "0.13.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"kebab-config",
|
||||
@@ -4309,7 +4309,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-mcp"
|
||||
version = "0.7.0"
|
||||
version = "0.13.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"kebab-app",
|
||||
@@ -4327,7 +4327,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-normalize"
|
||||
version = "0.7.0"
|
||||
version = "0.13.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"kebab-core",
|
||||
@@ -4342,7 +4342,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-parse-code"
|
||||
version = "0.7.0"
|
||||
version = "0.13.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"gix",
|
||||
@@ -4352,12 +4352,18 @@ dependencies = [
|
||||
"time",
|
||||
"tracing",
|
||||
"tree-sitter",
|
||||
"tree-sitter-go",
|
||||
"tree-sitter-java",
|
||||
"tree-sitter-javascript",
|
||||
"tree-sitter-kotlin-ng",
|
||||
"tree-sitter-python",
|
||||
"tree-sitter-rust",
|
||||
"tree-sitter-typescript",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "kebab-parse-image"
|
||||
version = "0.7.0"
|
||||
version = "0.13.0"
|
||||
dependencies = [
|
||||
"ab_glyph",
|
||||
"anyhow",
|
||||
@@ -4381,7 +4387,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-parse-md"
|
||||
version = "0.7.0"
|
||||
version = "0.13.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"kebab-core",
|
||||
@@ -4398,7 +4404,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-parse-pdf"
|
||||
version = "0.7.0"
|
||||
version = "0.13.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"blake3",
|
||||
@@ -4411,7 +4417,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-parse-types"
|
||||
version = "0.7.0"
|
||||
version = "0.13.0"
|
||||
dependencies = [
|
||||
"kebab-core",
|
||||
"serde",
|
||||
@@ -4419,7 +4425,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-rag"
|
||||
version = "0.7.0"
|
||||
version = "0.13.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"blake3",
|
||||
@@ -4440,7 +4446,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-search"
|
||||
version = "0.7.0"
|
||||
version = "0.13.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"globset",
|
||||
@@ -4459,10 +4465,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-source-fs"
|
||||
version = "0.7.0"
|
||||
version = "0.13.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"blake3",
|
||||
"globset",
|
||||
"ignore",
|
||||
"kebab-config",
|
||||
"kebab-core",
|
||||
@@ -4477,7 +4484,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-store-sqlite"
|
||||
version = "0.7.0"
|
||||
version = "0.13.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"blake3",
|
||||
@@ -4498,7 +4505,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-store-vector"
|
||||
version = "0.7.0"
|
||||
version = "0.13.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"arrow",
|
||||
@@ -4522,7 +4529,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-tui"
|
||||
version = "0.7.0"
|
||||
version = "0.13.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"crossterm",
|
||||
@@ -8523,12 +8530,62 @@ dependencies = [
|
||||
"tree-sitter-language",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tree-sitter-go"
|
||||
version = "0.25.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c8560a4d2f835cc0d4d2c2e03cbd0dde2f6114b43bc491164238d333e28b16ea"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"tree-sitter-language",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tree-sitter-java"
|
||||
version = "0.23.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0aa6cbcdc8c679b214e616fd3300da67da0e492e066df01bcf5a5921a71e90d6"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"tree-sitter-language",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tree-sitter-javascript"
|
||||
version = "0.25.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "68204f2abc0627a90bdf06e605f5c470aa26fdcb2081ea553a04bdad756693f5"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"tree-sitter-language",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tree-sitter-kotlin-ng"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e800ebbda938acfbf224f4d2c34947a31994b1295ee6e819b65226c7b51b4450"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"tree-sitter-language",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tree-sitter-language"
|
||||
version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "009994f150cc0cd50ff54917d5bc8bffe8cad10ca10d81c34da2ec421ae61782"
|
||||
|
||||
[[package]]
|
||||
name = "tree-sitter-python"
|
||||
version = "0.25.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6bf85fd39652e740bf60f46f4cda9492c3a9ad75880575bf14960f775cb74a1c"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"tree-sitter-language",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tree-sitter-rust"
|
||||
version = "0.24.2"
|
||||
@@ -8539,6 +8596,16 @@ dependencies = [
|
||||
"tree-sitter-language",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tree-sitter-typescript"
|
||||
version = "0.23.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6c5f76ed8d947a75cc446d5fccd8b602ebf0cde64ccf2ffa434d873d7a575eff"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"tree-sitter-language",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "try-lock"
|
||||
version = "0.2.5"
|
||||
|
||||
11
Cargo.toml
11
Cargo.toml
@@ -31,7 +31,7 @@ edition = "2024"
|
||||
rust-version = "1.85"
|
||||
license = "MIT OR Apache-2.0"
|
||||
repository = "https://github.com/altair823/kebab"
|
||||
version = "0.7.0"
|
||||
version = "0.13.0"
|
||||
|
||||
[workspace.dependencies]
|
||||
anyhow = "1"
|
||||
@@ -90,6 +90,15 @@ gix = { version = "0.70", default-features = false, features = ["revisi
|
||||
# chunker stays tree-sitter-free — AST work is parser-side per design §6.3.
|
||||
tree-sitter = "0.26"
|
||||
tree-sitter-rust = "0.24"
|
||||
# Python / TS / JS grammars for code ingest (kebab-parse-code, p10-1B).
|
||||
tree-sitter-python = "0.25.0"
|
||||
tree-sitter-typescript = "0.23.2"
|
||||
tree-sitter-javascript = "0.25.0"
|
||||
# Go grammar for code ingest (kebab-parse-code, p10-1C-Go).
|
||||
tree-sitter-go = "0.25.0"
|
||||
# JVM family grammars for code ingest (kebab-parse-code, p10-1C-JK).
|
||||
tree-sitter-java = "0.23.5"
|
||||
tree-sitter-kotlin-ng = "1.1.0" # bare tree-sitter-kotlin requires ts <0.23; -ng uses tree-sitter-language 0.1 (ts 0.26 compat)
|
||||
|
||||
# Disk-footprint trim for dev / test builds. Codegen, opt-level, and
|
||||
# behavior are unchanged — only DWARF debug info is reduced (line
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
## 한 줄 요약
|
||||
|
||||
P0–P5 + P6 + P7 + P9-1/2/3/4 (Library / Search / Ask / Inspect) 머지 완료. `kebab ingest` 가 markdown / image / PDF 모두 처리. `kebab search` / `kebab ask` 가 매체 가로질러 결과 + page citation 반환. `kebab tui` 가 4 패널 (Library + Search + Ask + Inspect) 제공 — 사용자가 `?` 로 ask, `/` 로 search, Library Enter / Search `i` 로 inspect, Search `g` 로 editor jump. 다음 후보 = P9-5 (desktop tauri) 또는 보류 중인 P8 (audio) 의 시스템 dep brainstorm.
|
||||
P0–P5 + P6 + P7 + P9-1/2/3/4 (Library / Search / Ask / Inspect) 머지 완료. `kebab ingest` 가 markdown / image / PDF / 소스코드 (Rust / Python / TS / JS / Go / Java / Kotlin) 처리. `kebab search` / `kebab ask` 가 매체 가로질러 결과 + page / code citation 반환. `kebab tui` 가 4 패널 (Library + Search + Ask + Inspect) 제공. P10-1C (Go + Java + Kotlin) 완료 — 다음 후보 = P10-1D (C/C++) 또는 P9-5 (desktop tauri) 또는 보류 중인 P8 (audio).
|
||||
|
||||
## Phase 로드맵
|
||||
|
||||
@@ -20,7 +20,7 @@ P0–P5 + P6 + P7 + P9-1/2/3/4 (Library / Search / Ask / Inspect) 머지 완료.
|
||||
| **P7** | PDF text + page citation | `kebab-parse-pdf` | P5 | ✅ 완료 (3/3 component, page-level chunker + ingest wiring) |
|
||||
| **P8** | 음성 transcription + timestamp citation | `kebab-parse-audio` | P5 | ⏸ 보류 (whisper-rs 시스템 dep brainstorm 필요) |
|
||||
| **P9** | TUI + desktop app | `kebab-tui`, `kebab-desktop` | P5 | 🟡 진행 (4/5 component — P9-1/2/3/4 완료 [Library / Search / Ask / Inspect], P9-5 desktop 예정 · 도그푸딩 피드백 **20/20 ✅**) |
|
||||
| **P10** | code ingest framework | `kebab-parse-code` | P5 | 🟡 진행 중 — 1A-1 ✅ (wire schema + parse-code skeleton + filter flags), 1A-2 ✅ (Rust AST chunker, tree-sitter-rust, `code-rust-ast-v1` — kebab 자기 dogfooding 가능, v0.7.0) |
|
||||
| **P10** | code ingest framework | `kebab-parse-code` | P5 | 🟡 진행 중 — 1A-1 ✅ (wire schema + parse-code skeleton + filter flags), 1A-2 ✅ (Rust AST chunker, `code-rust-ast-v1` — v0.7.0), 1B ✅ (Python/TS/JS AST chunkers — v0.8.0 이후), **1C-Go ✅ (Go AST chunker, `code-go-ast-v1` — v0.12.0)**, **1C-JavaKotlin ✅ (Java + Kotlin AST chunkers, `code-java-ast-v1` / `code-kotlin-ast-v1` — v0.13.0)** |
|
||||
|
||||
P0~P5 직렬. P6~P9 P5 이후 병렬 가능.
|
||||
|
||||
@@ -32,6 +32,7 @@ P0~P5 직렬. P6~P9 P5 이후 병렬 가능.
|
||||
|
||||
머지 후 발견된 모든 deviation / hotfix 의 dated 로그는 [tasks/HOTFIXES.md](tasks/HOTFIXES.md). 본 요약은 \"누군가가 인수받을 때 알아두면 시간을 많이 절약하는\" 항목만:
|
||||
|
||||
- **2026-05-20 P10-1B (Rust 1A symbol path 비일관 + expression-level 함수 미방출)** — (a) Rust `code-rust-ast-v1` 은 file-scope nesting 만 (workspace path prefix 없음), 1B 의 Python/TypeScript/JavaScript 는 workspace 경로 → module path prefix 사용 (비일관 수용, retrofit = chunker_version bump + reindex 필요, 사용자 명시 요청까지 보류); (b) TS/JS 의 `const foo = () => {...}` 같은 expression-level 함수는 `<top-level>` glue 로 처리됨 (declaration-level 단위만 1B 1차 범위). 자세한 내용: `tasks/HOTFIXES.md` (2026-05-20) 두 항목.
|
||||
- **2026-05-19 P10-1A-2 (code_rust_ast_v1.rs + SourceType)** — `AST_CHUNK_MAX_LINES` 상수가 `IngestCodeCfg.ast_chunk_max_lines` 를 읽지 않고 모듈 상수 200 고정 (Chunker trait 이 per-medium config 미노출); `SourceType::Code` variant 부재로 code 파일이 `SourceType::Note` 로 분류됨 — 두 항목 모두 `tasks/HOTFIXES.md` (2026-05-19) 에 기록.
|
||||
- **2026-05-07 fb-26 (progress.rs)** — `Aborted` unconditional writeln (TTY duplicate) + `Completed` TTY no summary fixed; `KEBAB_PROGRESS=plain` env + quiet suppression added
|
||||
- **2026-05-07 fb-28 (main.rs)** — `--readonly` (KEBAB_READONLY) blocks Ingest/IngestFile/IngestStdin/Reset; `--quiet` suppresses progress stderr; error.v1 code: "readonly_mode"
|
||||
|
||||
@@ -34,7 +34,7 @@ cargo install --git https://gitea.altair823.xyz/altair823-org/kebab.git --bin ke
|
||||
|
||||
업데이트는 `git pull && cargo install --path crates/kebab-cli --locked --force` 또는 git URL 형식의 경우 `cargo install --git ... --force`.
|
||||
|
||||
제거는 `cargo uninstall kebab-cli`. 이 명령은 binary 만 지우고 워크스페이스 데이터는 그대로 남는다. 데이터까지 정리하려면 `kebab reset --all --yes` (config + data + cache + state 4 개 XDG 경로 모두 wipe — **irreversible**, 재시작 시 `kebab init` 다시 실행). 부분 wipe 는 `kebab reset --data-only` (config 보존), `kebab reset --vector-only` (Lance + `embedding_records` 만, 다음 ingest 가 re-embed) 등.
|
||||
제거는 `cargo uninstall kebab-cli`. 이 명령은 binary 만 지우고 워크스페이스 데이터는 그대로 남는다. 데이터까지 정리하려면 `kebab reset --all --yes` (config + data + cache + state 4 개 XDG 경로 모두 wipe — **irreversible**, 재시작 시 `kebab init` 다시 실행). 부분 wipe 는 `kebab reset --data-only` (config 보존), `kebab reset --vector-only` (Lance + `embedding_records` 만, 다음 ingest 가 re-embed), **`kebab reset --orphans-only`** (현재 walker scope 밖에 있는 stored doc 만 정리 — `config.workspace.include` 좁히거나 sub-dir 옮긴 후 explicit reconcile; fs 의 file 은 건드리지 않음) 등.
|
||||
|
||||
## Quick start
|
||||
|
||||
@@ -42,7 +42,7 @@ cargo install --git https://gitea.altair823.xyz/altair823-org/kebab.git --bin ke
|
||||
# 첫 실행 — XDG 경로에 데이터 디렉토리 + config.toml 생성
|
||||
kebab init
|
||||
|
||||
# config 손보고 — workspace.root, 모델 endpoint 등 설정 (지원 형식: md / png / jpg / pdf / rs)
|
||||
# config 손보고 — workspace.root, 모델 endpoint 등 설정 (지원 형식: md / png / jpg / pdf / rs / py / ts / js / go)
|
||||
${EDITOR:-vi} ~/.config/kebab/config.toml
|
||||
|
||||
# 색인 (Markdown / 이미지 / PDF 모두 한 번에)
|
||||
@@ -70,7 +70,7 @@ kebab doctor
|
||||
| 명령 | 동작 |
|
||||
|------|------|
|
||||
| `kebab init` | XDG 경로에 데이터 디렉토리 + config.toml 생성 |
|
||||
| `kebab ingest [<path>]` | Markdown / 이미지 / PDF / Rust 소스코드 색인 (idempotent). TTY 에서는 stderr 진행 바, non-TTY (CI / pipe) 는 stderr 한 줄씩, `--json` 은 stdout 에 `ingest_progress.v1` 라인 streaming 후 마지막에 `ingest_report.v1`. Ctrl-C 한 번이면 현재 asset 마무리 후 abort (부분 commit 보존, idempotent re-run), 두 번째 Ctrl-C 는 hard exit. Markdown title 이 frontmatter 에 없어도 첫 H1 → H2 → 첫 paragraph 80 자 → 파일명 순으로 자동 채움 (parser_version `md-frontmatter-v2`) — 기존 색인된 doc 도 다음 ingest 에서 새 title 로 갱신. **Incremental** (p9-fb-23): 두 번째 이후의 ingest 는 변하지 않은 doc (blake3 + parser/chunker/embedder version 모두 동일) 의 parse/chunk/embed/vector upsert 를 자동 스킵. final summary 에 `N unchanged` 카운트 표시. `--force-reingest` 로 skip 무시 강제 재처리. **지원 형식** (extractor 자동 결정 — config 에 명시 불가): Markdown (`.md`), 이미지 (`.png` / `.jpg` / `.jpeg`, OCR + caption), PDF (`.pdf`), **Rust 소스코드** (`.rs`, tree-sitter AST chunker `code-rust-ast-v1` — p10-1A-2). 다른 확장자는 자동 skip — `IngestItem.warnings` 에 사유 (`"unsupported media type: .docx"` 등), `IngestReport.skipped_by_extension` 에 카운트 분류, CLI / TUI summary 에 breakdown 표시. 코드 chunk 는 `citation.kind = "code"` 에 `citation.lang = "rust"` + `symbol` + line range 를 담고, SearchHit top-level 에 `code_lang = "rust"` + `repo` (`.git/` walk-up 의 디렉토리 이름) 가 backfill 됨. `--code-lang rust` / `--media code` filter 로 코드 전용 검색 가능 (p10-1A-1 filter flags). |
|
||||
| `kebab ingest [<path>]` | Markdown / 이미지 / PDF / Rust 소스코드 색인 (idempotent). TTY 에서는 stderr 진행 바, non-TTY (CI / pipe) 는 stderr 한 줄씩, `--json` 은 stdout 에 `ingest_progress.v1` 라인 streaming 후 마지막에 `ingest_report.v1`. Ctrl-C 한 번이면 현재 asset 마무리 후 abort (부분 commit 보존, idempotent re-run), 두 번째 Ctrl-C 는 hard exit. Markdown title 이 frontmatter 에 없어도 첫 H1 → H2 → 첫 paragraph 80 자 → 파일명 순으로 자동 채움 (parser_version `md-frontmatter-v2`) — 기존 색인된 doc 도 다음 ingest 에서 새 title 로 갱신. **Incremental** (p9-fb-23): 두 번째 이후의 ingest 는 변하지 않은 doc (blake3 + parser/chunker/embedder version 모두 동일) 의 parse/chunk/embed/vector upsert 를 자동 스킵. final summary 에 `N unchanged` 카운트 표시. `--force-reingest` 로 skip 무시 강제 재처리. **지원 형식** (extractor 자동 결정 — config 에 명시 불가): Markdown (`.md`), 이미지 (`.png` / `.jpg` / `.jpeg`, OCR + caption), PDF (`.pdf`), **소스코드** (`.rs` → `code-rust-ast-v1`, `.py` → `code-python-ast-v1`, `.ts`/`.tsx` → `code-ts-ast-v1`, `.js`/`.mjs`/`.cjs`/`.jsx` → `code-js-ast-v1`, `.go` → `code-go-ast-v1`, `.java` → `code-java-ast-v1`, `.kt`/`.kts` → `code-kotlin-ast-v1` — 모두 tree-sitter AST chunker). 다른 확장자는 자동 skip — `IngestItem.warnings` 에 사유 (`"unsupported media type: .docx"` 등), `IngestReport.skipped_by_extension` 에 카운트 분류, CLI / TUI summary 에 breakdown 표시. 코드 chunk 는 `citation.kind = "code"` 에 `citation.lang = "<lang>"` + `symbol` + line range 를 담고, SearchHit top-level 에 `code_lang` + `repo` (`.git/` walk-up 의 디렉토리 이름) 가 backfill 됨. `--code-lang rust` / `--code-lang python` / `--code-lang typescript` / `--code-lang javascript` / `--code-lang go` / `--code-lang java` / `--code-lang kotlin` / `--media code` filter 로 언어별·코드 전용 검색 가능 (p10-1A-1 filter flags). Python symbol 은 workspace 경로 → dotted module path prefix (예: `kebab_eval.metrics.compute_mrr`), TS/JS symbol 은 slash-style module path prefix (예: `src/Foo.Foo.search`), Go symbol 은 `package.Func` / `package.(*Receiver).Method` 형식, Java / Kotlin symbol 은 `com.foo.Foo.bar` 형식 (패키지 + 클래스 + 메서드/필드). |
|
||||
| `kebab search --mode {lexical,vector,hybrid} "<query>" [--no-cache] [--max-tokens N] [--snippet-chars N] [--cursor <opaque>] [--tag T] [--lang L] [--path-glob G] [--trust-min LEVEL] [--media TYPE] [--ingested-after RFC3339] [--doc-id ID] [--trace] [--bulk] [--repo NAME ...] [--code-lang LIST]` | 검색. hybrid는 RRF fusion, citation 포함. 같은 process 안에서 동일 query (NFKC + trim + lowercase 정규화) 반복 시 in-process LRU 캐시 hit (capacity = `[search] cache_capacity`, default 256). `--no-cache` 로 강제 bypass — 디버깅용. ingest commit 발생 시 `kv['corpus_revision']` bump 으로 모든 entry 자동 stale. **`--max-tokens` / `--snippet-chars` / `--cursor` (p9-fb-34)** — agent budget controls. `--json` 출력은 `search_response.v1` wrapper (`{hits, next_cursor, truncated}`) — pre-fb-34 의 bare array 와 호환 안 됨. mismatched cursor → `error.v1.code = stale_cursor`. **filter flags (p9-fb-36):** `--tag` 는 반복 가능 flag (`--tag rust --tag async`) 로 OR 매칭, `--media` 는 `,` 구분 다중 값 OR 매칭, 나머지 flags 간은 AND 조합. `--trust-min` 은 `primary\|secondary\|generated` 중 하나 (해당 level 이상 포함). `--ingested-after` 는 RFC3339 UTC — 파싱 실패 시 `error.v1.code = config_invalid` (exit 2). `--media md` 는 `markdown` alias 로 정규화. 알 수 없는 `--media` 값은 무조건 empty hits (오류 아님). **`--trace` (p9-fb-37)** — `search_response.v1.trace` 에 lexical / vector pre-fusion 후보 + RRF union + per-stage timing (`lexical_ms` / `vector_ms` / `fusion_ms` / `total_ms`) 노출. trace 요청은 캐시 우회 (`--no-cache` 없이도 항상 cold). **`--bulk` (p9-fb-42)** — stdin ndjson 으로 N query 한 번에 실행. `--json` 면 stdout per-query ndjson (`bulk_search_item.v1`) + stderr summary (`bulk_summary: total=N succeeded=S failed=F`). Cap 100. agent 가 query decomposition 후 sub-query 일괄 실행 시 single round-trip — App instance 재사용으로 캐시 / embedder cold-start 비용 한 번만. Per-query failure 는 item 의 `error` (error.v1) 에 격리, 다른 query 계속 진행. **code corpus filters (p10-1A-1):** `--repo` 는 반복 가능 (`--repo kebab --repo other`) OR 매칭. `--code-lang` 는 반복 또는 comma 다중 값 (`--code-lang rust,python`), 알 수 없는 값은 빈 hits. `--media code` 는 Tier 1/2/3 모든 code chunk 포함. 1A-1 시점에서는 indexed 된 code chunk 가 없어 filter 가 항상 빈 결과 — 1A-2 (Rust AST chunker) 머지 이후 실효. |
|
||||
| `kebab list docs` | 색인된 문서 목록 |
|
||||
| `kebab inspect doc <id>` / `kebab inspect chunk <id>` | raw record 보기 |
|
||||
@@ -132,7 +132,7 @@ flowchart TB
|
||||
|
||||
subgraph Pipeline["도메인 + 파이프라인"]
|
||||
parse["parse-md / parse-pdf / parse-image / parse-code"]
|
||||
chunker["chunker (md-heading-v1, pdf-page-v1, code-rust-ast-v1)"]
|
||||
chunker["chunker (md-heading-v1, pdf-page-v1, code-rust-ast-v1, code-python-ast-v1, code-ts-ast-v1, code-js-ast-v1, code-go-ast-v1)"]
|
||||
embedder["embedder (fastembed multilingual-e5-large)"]
|
||||
retriever["retriever (lexical / vector / hybrid RRF)"]
|
||||
rag["RAG pipeline"]
|
||||
|
||||
@@ -189,10 +189,12 @@ fn fetch_span(
|
||||
// (markdown / note / paper / reference / inbox) is the *user-facing*
|
||||
// category, not the rendering format — the actual byte-level format
|
||||
// lives on the source `RawAsset.media_type`. Look it up via
|
||||
// workspace_path (unique key per asset).
|
||||
if let Some(asset) = <kebab_store_sqlite::SqliteStore as DocumentStore>::get_asset_by_workspace_path(
|
||||
// doc.source_asset_id (PRIMARY KEY) so twin files (identical content
|
||||
// at different paths) always read *this* document's own asset row,
|
||||
// not whichever twin last wrote `assets.workspace_path`.
|
||||
if let Some(asset) = <kebab_store_sqlite::SqliteStore as DocumentStore>::get_asset(
|
||||
&app.sqlite,
|
||||
&doc.workspace_path,
|
||||
&doc.source_asset_id,
|
||||
)? {
|
||||
if matches!(
|
||||
asset.media_type,
|
||||
|
||||
@@ -39,7 +39,7 @@ use std::sync::Arc;
|
||||
use anyhow::{Context, anyhow};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use kebab_chunk::{CodeRustAstV1Chunker, MdHeadingV1Chunker, PdfPageV1Chunker};
|
||||
use kebab_chunk::{CodeGoAstV1Chunker, CodeJavaAstV1Chunker, CodeJsAstV1Chunker, CodeKotlinAstV1Chunker, CodePythonAstV1Chunker, CodeRustAstV1Chunker, CodeTsAstV1Chunker, MdHeadingV1Chunker, PdfPageV1Chunker};
|
||||
use kebab_core::{
|
||||
Answer, Block, CanonicalDocument, Chunk, ChunkId, ChunkPolicy, ChunkerVersion, Chunker,
|
||||
DocFilter, DocSummary, DocumentId, DocumentStore, Embedder, EmbeddingInput,
|
||||
@@ -50,7 +50,7 @@ use kebab_core::{
|
||||
use kebab_llm_local::OllamaLanguageModel;
|
||||
use kebab_normalize::build_canonical_document;
|
||||
use kebab_parse_image::{ImageExtractor, OllamaVisionOcr, apply_caption, apply_ocr};
|
||||
use kebab_parse_code::RustAstExtractor;
|
||||
use kebab_parse_code::{GoAstExtractor, JavaAstExtractor, JavascriptAstExtractor, KotlinAstExtractor, PythonAstExtractor, RustAstExtractor, TypescriptAstExtractor};
|
||||
use kebab_parse_pdf::PdfTextExtractor;
|
||||
use kebab_parse_md::{BodyHints, parse_blocks, parse_frontmatter};
|
||||
use kebab_source_fs::FsSourceConnector;
|
||||
@@ -71,7 +71,7 @@ mod staleness;
|
||||
|
||||
pub use app::{App, SearchResponse};
|
||||
pub use ingest_progress::{AggregateCounts, IngestEvent, render_skipped_breakdown};
|
||||
pub use reset::{ResetReport, ResetScope};
|
||||
pub use reset::{ResetReport, ResetScope, enumerate_orphans};
|
||||
pub use error_wire::{ERROR_V1_ID, ErrorV1, StructuredError, classify};
|
||||
pub use fetch::fetch_with_config;
|
||||
#[doc(hidden)]
|
||||
@@ -375,6 +375,28 @@ pub fn ingest_with_config_opts(
|
||||
.map(|d| d.doc_id.0)
|
||||
.collect();
|
||||
|
||||
// Dogfood: post-walker sweep to remove stored docs whose source
|
||||
// file has been deleted from the filesystem. Must run BEFORE the
|
||||
// per-asset loop so the loop's New/Updated labelling is based on
|
||||
// the post-purge store state (the purged doc_ids won't be in
|
||||
// `existing_doc_ids` above — they were already removed, OR the
|
||||
// sweep here removes them before we start counting).
|
||||
//
|
||||
// Critical design invariant: only purge when the file is TRULY
|
||||
// absent from disk. A file that is still on disk but outside the
|
||||
// current walker scope (config narrowing / include-glob change) is
|
||||
// NOT purged — we leave it in place to protect against accidental
|
||||
// data loss via config edits.
|
||||
let scanned_paths: std::collections::HashSet<kebab_core::WorkspacePath> = assets
|
||||
.iter()
|
||||
.map(|a| a.workspace_path.clone())
|
||||
.collect();
|
||||
let purged_deleted_files = sweep_deleted_files(
|
||||
&app,
|
||||
&scanned_paths,
|
||||
vector_store.as_ref().map(|v| v.as_ref()),
|
||||
)?;
|
||||
|
||||
let started_at = time::OffsetDateTime::now_utc();
|
||||
|
||||
let mut items: Vec<kebab_core::IngestItem> = Vec::new();
|
||||
@@ -647,11 +669,11 @@ pub fn ingest_with_config_opts(
|
||||
crate::ingest_progress::emit(progress, terminal_event);
|
||||
|
||||
// p9-fb-19: bump the persistent corpus_revision counter when a
|
||||
// commit landed (any new / updated). This invalidates every
|
||||
// commit landed (any new / updated / purged). This invalidates every
|
||||
// entry in any in-process LRU search cache (in this process or
|
||||
// a sibling) on the next lookup. No-op when nothing changed
|
||||
// (skipped-only run) — the cache stays valid.
|
||||
if new_count > 0 || updated_count > 0 {
|
||||
if new_count > 0 || updated_count > 0 || purged_deleted_files > 0 {
|
||||
match app.sqlite.bump_corpus_revision() {
|
||||
Ok(rev) => tracing::debug!(
|
||||
target: "kebab-app",
|
||||
@@ -682,6 +704,7 @@ pub fn ingest_with_config_opts(
|
||||
skipped_generated: fs_skips.skipped_generated,
|
||||
skipped_size_exceeded: fs_skips.skipped_size_exceeded,
|
||||
skip_examples: fs_skips.skip_examples,
|
||||
purged_deleted_files,
|
||||
items: if summary_only { None } else { Some(items) },
|
||||
})
|
||||
}
|
||||
@@ -748,15 +771,18 @@ struct ImagePipeline<'a> {
|
||||
/// hold (per design §9 cascade rule):
|
||||
///
|
||||
/// 1. `force_reingest == false` — caller hasn't asked to bypass skip.
|
||||
/// 2. The freshly-scanned asset's blake3 checksum equals what the
|
||||
/// existing `assets` row stores at the same `workspace_path`.
|
||||
/// 3. The doc keyed on `(workspace_path, asset_id, current_parser_version)`
|
||||
/// exists. If the parser_version changed, `id_for_doc` produces a
|
||||
/// different `doc_id` so the lookup misses → no skip → re-process.
|
||||
/// 4. The existing doc's stamped `last_chunker_version` AND
|
||||
/// `last_embedding_version` match the values the caller is about
|
||||
/// to use (`Some(v) == Some(v)` and `None == None` — see design
|
||||
/// doc for the `None == None` rule when no embedder is configured).
|
||||
/// 2. A document already exists at this `workspace_path`
|
||||
/// (`get_document_by_workspace_path`). The lookup is document-side, not
|
||||
/// asset-side, so twin files (identical content at different paths) each
|
||||
/// hit their own stable doc row — `documents.workspace_path` is UNIQUE
|
||||
/// while `assets` may dedupe content into a single row with a flip-flop
|
||||
/// `workspace_path` column (dogfood bug #4, see `tasks/HOTFIXES.md`).
|
||||
/// 3. The existing doc's `source_asset_id` equals the freshly-scanned
|
||||
/// asset's blake3 checksum (content unchanged).
|
||||
/// 4. The existing doc's `parser_version` matches the current extractor's
|
||||
/// `parser_version` (extractor not upgraded). Combined with `chunker_version`
|
||||
/// and `last_embedding_version` checks immediately below — full cascade
|
||||
/// per design §9.
|
||||
///
|
||||
/// Returns `Ok(None)` (proceed with full re-process) when any check
|
||||
/// fails or any DB read errors out — the skip path is opportunistic;
|
||||
@@ -773,31 +799,19 @@ fn try_skip_unchanged(
|
||||
if force_reingest {
|
||||
return Ok(None);
|
||||
}
|
||||
let existing_asset = match app
|
||||
// Document-centric skip: look up the existing document row by
|
||||
// workspace_path directly. This avoids the twin-file flip-flop
|
||||
// that the old asset-side lookup suffers from — multiple files
|
||||
// with identical content share one `assets` row whose
|
||||
// `workspace_path` is overwritten on every UPSERT, so
|
||||
// `get_asset_by_workspace_path(path1)` could return the OTHER
|
||||
// twin's path (or None) after any ingest of the twin. The
|
||||
// `documents` table has a UNIQUE index on `workspace_path` (V001),
|
||||
// so each twin has its own stable row regardless of asset de-dup.
|
||||
let existing_doc = match app
|
||||
.sqlite
|
||||
.get_asset_by_workspace_path(&asset.workspace_path)
|
||||
.get_document_by_workspace_path(&asset.workspace_path)
|
||||
{
|
||||
Ok(Some(a)) => a,
|
||||
Ok(None) => return Ok(None),
|
||||
Err(e) => {
|
||||
tracing::debug!(
|
||||
target: "kebab-app",
|
||||
path = %asset.workspace_path.0,
|
||||
error = %e,
|
||||
"skip-check: get_asset_by_workspace_path failed; falling through to re-process"
|
||||
);
|
||||
return Ok(None);
|
||||
}
|
||||
};
|
||||
if existing_asset.checksum != asset.checksum {
|
||||
return Ok(None);
|
||||
}
|
||||
let candidate_doc_id = kebab_core::id_for_doc(
|
||||
&asset.workspace_path,
|
||||
&asset.asset_id,
|
||||
current_parser_version,
|
||||
);
|
||||
let existing_doc = match app.sqlite.get_document(&candidate_doc_id) {
|
||||
Ok(Some(d)) => d,
|
||||
Ok(None) => return Ok(None),
|
||||
Err(e) => {
|
||||
@@ -805,21 +819,37 @@ fn try_skip_unchanged(
|
||||
target: "kebab-app",
|
||||
path = %asset.workspace_path.0,
|
||||
error = %e,
|
||||
"skip-check: get_document failed; falling through to re-process"
|
||||
"skip-check: get_document_by_workspace_path failed; falling through to re-process"
|
||||
);
|
||||
return Ok(None);
|
||||
}
|
||||
};
|
||||
// 1. Content unchanged: the freshly-computed asset_id (blake3
|
||||
// content hash) must match what this document was ingested from.
|
||||
if existing_doc.source_asset_id != asset.asset_id {
|
||||
return Ok(None);
|
||||
}
|
||||
// 2. Parser unchanged: parser_version is baked into id_for_doc so
|
||||
// a version bump yields a different doc_id and the row above
|
||||
// would have been missing. Checking here explicitly keeps the
|
||||
// logic self-documenting and guards against future id_for_doc
|
||||
// changes.
|
||||
if existing_doc.parser_version != *current_parser_version {
|
||||
return Ok(None);
|
||||
}
|
||||
// 3. Chunker unchanged.
|
||||
let chunker_match = existing_doc.last_chunker_version.as_ref()
|
||||
== Some(current_chunker_version);
|
||||
if !chunker_match {
|
||||
return Ok(None);
|
||||
}
|
||||
// 4. Embedder unchanged.
|
||||
let embedder_match = existing_doc.last_embedding_version.as_ref()
|
||||
== current_embedding_version;
|
||||
if !embedder_match {
|
||||
return Ok(None);
|
||||
}
|
||||
let candidate_doc_id = existing_doc.doc_id.clone();
|
||||
tracing::debug!(
|
||||
target: "kebab-app::ingest",
|
||||
path = %asset.workspace_path.0,
|
||||
@@ -918,8 +948,11 @@ fn ingest_one_asset(
|
||||
force_reingest,
|
||||
);
|
||||
}
|
||||
// p10-1A-2 Task 8: Rust code ingest.
|
||||
MediaType::Code(lang) if lang == "rust" => {
|
||||
// p10-1A-2 / 1B: code ingest dispatch.
|
||||
MediaType::Code(lang)
|
||||
if matches!(lang.as_str(),
|
||||
"rust" | "python" | "typescript" | "javascript" | "go" | "java" | "kotlin") =>
|
||||
{
|
||||
return ingest_one_code_asset(
|
||||
app,
|
||||
asset,
|
||||
@@ -928,6 +961,7 @@ fn ingest_one_asset(
|
||||
vector_store,
|
||||
existing_doc_ids,
|
||||
force_reingest,
|
||||
lang.as_str(),
|
||||
);
|
||||
}
|
||||
// p10-1A-2: non-Rust Code, Audio, and Other are not yet wired;
|
||||
@@ -1443,6 +1477,120 @@ fn purge_vector_orphans_for_workspace_path(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Dogfood: post-walker sweep that purges stored documents whose source
|
||||
/// file has been physically deleted from the filesystem.
|
||||
///
|
||||
/// Algorithm:
|
||||
/// 1. Query `documents` for every `workspace_path` currently stored.
|
||||
/// 2. Compute `orphan_candidates = stored_paths - scanned_paths`.
|
||||
/// 3. For each candidate: resolve to an absolute path and call
|
||||
/// `Path::try_exists().unwrap_or(true)` — transient FS errors
|
||||
/// (EACCES, NFS hiccup, ownership change) conservatively count as
|
||||
/// "still present" so we never purge on uncertain signal. If the
|
||||
/// file still exists on disk it was merely out-of-scope this run
|
||||
/// (config narrowing / include-glob change) — leave it alone. Only
|
||||
/// files that are truly absent trigger a purge.
|
||||
/// 4. For absent files: call `purge_deleted_workspace_path` (SQLite
|
||||
/// cascade delete + optional copied-asset file removal) and, if a
|
||||
/// vector store is present, delete the associated vectors.
|
||||
///
|
||||
/// Returns the number of documents purged.
|
||||
///
|
||||
/// Non-fatal design: individual purge failures are logged and counted
|
||||
/// as errors on the per-file level but do NOT abort the sweep — a
|
||||
/// partial failure is preferable to blocking the rest of ingest. The
|
||||
/// return value only counts successful purges.
|
||||
fn sweep_deleted_files(
|
||||
app: &App,
|
||||
scanned_paths: &std::collections::HashSet<kebab_core::WorkspacePath>,
|
||||
vector_store: Option<&kebab_store_vector::LanceVectorStore>,
|
||||
) -> anyhow::Result<u32> {
|
||||
use kebab_core::DocumentStore as _;
|
||||
|
||||
let stored_paths = app
|
||||
.sqlite
|
||||
.all_workspace_paths()
|
||||
.context("sweep_deleted_files: all_workspace_paths")?;
|
||||
|
||||
if stored_paths.is_empty() {
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
let workspace_root = app.config.resolve_workspace_root();
|
||||
let mut purged: u32 = 0;
|
||||
|
||||
for stored_path in stored_paths {
|
||||
if scanned_paths.contains(&stored_path) {
|
||||
continue; // still in scope — skip
|
||||
}
|
||||
|
||||
// Resolve to an absolute path and check existence on disk.
|
||||
// Use `try_exists` + `unwrap_or(true)` so transient FS errors
|
||||
// (EACCES on a path we lack read on, NFS hiccups, ownership
|
||||
// change) are CONSERVATIVELY treated as "file still present" —
|
||||
// never purge on uncertain signal (data-safety: PR #148 review).
|
||||
// `exists()` would return false on Err and trigger a wrongful
|
||||
// purge. Files whose path cannot be joined (theoretically
|
||||
// impossible for non-empty workspace_path strings, but
|
||||
// defense-in-depth) are likewise treated as still present.
|
||||
let abs = workspace_root.join(&stored_path.0);
|
||||
if abs.try_exists().unwrap_or(true) {
|
||||
// File is on disk but not in this scan's scope (config
|
||||
// narrowing). DO NOT purge — critical design constraint.
|
||||
tracing::debug!(
|
||||
target: "kebab-app",
|
||||
path = %stored_path.0,
|
||||
"sweep_deleted_files: file on disk but out of scope — leaving in store"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
// File is truly absent → purge.
|
||||
let chunk_ids = match kebab_store_sqlite::purge_deleted_workspace_path(
|
||||
&app.sqlite,
|
||||
&stored_path,
|
||||
) {
|
||||
Ok(ids) => ids,
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
target: "kebab-app",
|
||||
path = %stored_path.0,
|
||||
error = %e,
|
||||
"sweep_deleted_files: purge failed; skipping this path"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
// Purge associated vectors (best-effort; partial failure
|
||||
// acceptable — orphan vectors get cleaned by `kebab reset
|
||||
// --vector-only` if they accumulate).
|
||||
if let Some(vec) = vector_store {
|
||||
if !chunk_ids.is_empty() {
|
||||
use kebab_core::VectorStore as _;
|
||||
if let Err(e) = vec.delete_by_chunk_ids(&chunk_ids) {
|
||||
tracing::warn!(
|
||||
target: "kebab-app",
|
||||
path = %stored_path.0,
|
||||
count = chunk_ids.len(),
|
||||
error = %e,
|
||||
"sweep_deleted_files: vector delete failed; SQLite side already cleaned"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tracing::info!(
|
||||
target: "kebab-app",
|
||||
path = %stored_path.0,
|
||||
"sweep_deleted_files: purged document for deleted file"
|
||||
);
|
||||
purged = purged.saturating_add(1);
|
||||
}
|
||||
|
||||
Ok(purged)
|
||||
}
|
||||
|
||||
/// P7-3: process one `MediaType::Pdf` asset end-to-end.
|
||||
///
|
||||
/// - Reads bytes from disk.
|
||||
@@ -1642,6 +1790,7 @@ fn ingest_one_pdf_asset(
|
||||
///
|
||||
/// All other steps (incremental skip, byte read, ExtractContext, put_*,
|
||||
/// embed, purge_vector_orphans) are identical to the PDF function.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn ingest_one_code_asset(
|
||||
app: &App,
|
||||
asset: &RawAsset,
|
||||
@@ -1650,6 +1799,7 @@ fn ingest_one_code_asset(
|
||||
vector_store: Option<&Arc<kebab_store_vector::LanceVectorStore>>,
|
||||
existing_doc_ids: &std::collections::HashSet<String>,
|
||||
force_reingest: bool,
|
||||
code_lang: &str, // <-- NEW (p10-1b Task D)
|
||||
) -> anyhow::Result<kebab_core::IngestItem> {
|
||||
let path = match &asset.source_uri {
|
||||
SourceUri::File(p) => p.clone(),
|
||||
@@ -1671,17 +1821,36 @@ fn ingest_one_code_asset(
|
||||
});
|
||||
}
|
||||
};
|
||||
// p10-1A-2 task 8: incremental-ingest early-skip for the code flow.
|
||||
// Code docs use `code-rust-v1` as the parser_version and
|
||||
// `CodeRustAstV1Chunker` as the chunker — both pinned per-medium
|
||||
// today (no config knob).
|
||||
let code_parser_version =
|
||||
ParserVersion(kebab_parse_code::RUST_PARSER_VERSION.to_string());
|
||||
|
||||
// p10-1b Task D/G/J: parser_version per-lang.
|
||||
let parser_version = match code_lang {
|
||||
"rust" => ParserVersion(kebab_parse_code::RUST_PARSER_VERSION.to_string()),
|
||||
"python" => ParserVersion(kebab_parse_code::PYTHON_PARSER_VERSION.to_string()),
|
||||
"typescript" => ParserVersion(kebab_parse_code::TS_PARSER_VERSION.to_string()),
|
||||
"javascript" => ParserVersion(kebab_parse_code::JS_PARSER_VERSION.to_string()),
|
||||
"go" => ParserVersion(kebab_parse_code::GO_PARSER_VERSION.to_string()),
|
||||
"java" => ParserVersion(kebab_parse_code::JAVA_PARSER_VERSION.to_string()),
|
||||
"kotlin" => ParserVersion(kebab_parse_code::KOTLIN_PARSER_VERSION.to_string()),
|
||||
other => anyhow::bail!("unsupported code_lang: {other}"),
|
||||
};
|
||||
|
||||
// p10-1b Task D/G/J/L: chunker_version per-lang.
|
||||
let chunker_version = match code_lang {
|
||||
"rust" => CodeRustAstV1Chunker.chunker_version(),
|
||||
"python" => CodePythonAstV1Chunker.chunker_version(),
|
||||
"typescript" => CodeTsAstV1Chunker.chunker_version(),
|
||||
"javascript" => CodeJsAstV1Chunker.chunker_version(),
|
||||
"go" => CodeGoAstV1Chunker.chunker_version(),
|
||||
"java" => CodeJavaAstV1Chunker.chunker_version(),
|
||||
"kotlin" => CodeKotlinAstV1Chunker.chunker_version(),
|
||||
other => anyhow::bail!("unreachable chunker_version: {other}"),
|
||||
};
|
||||
|
||||
if let Some(item) = try_skip_unchanged(
|
||||
app,
|
||||
asset,
|
||||
&code_parser_version,
|
||||
&CodeRustAstV1Chunker.chunker_version(),
|
||||
&parser_version,
|
||||
&chunker_version,
|
||||
embedder.map(|e| e.model_version()).as_ref(),
|
||||
force_reingest,
|
||||
)? {
|
||||
@@ -1697,20 +1866,62 @@ fn ingest_one_code_asset(
|
||||
workspace_root: &workspace_root,
|
||||
config: &extract_config,
|
||||
};
|
||||
let mut canonical = RustAstExtractor::new()
|
||||
.extract(&ctx, &bytes)
|
||||
.context("kb-parse-code::RustAstExtractor::extract")?;
|
||||
|
||||
// Per-medium chunker selection: Rust code always uses code-rust-ast-v1
|
||||
// regardless of `config.chunking.chunker_version`.
|
||||
let chunker = CodeRustAstV1Chunker;
|
||||
let chunks = chunker
|
||||
.chunk(&canonical, chunk_policy)
|
||||
.context("kb-chunk::CodeRustAstV1Chunker::chunk")?;
|
||||
// p10-1b Task D/G/J/L: extractor per-lang.
|
||||
let mut canonical = match code_lang {
|
||||
"rust" => RustAstExtractor::new()
|
||||
.extract(&ctx, &bytes)
|
||||
.context("kb-parse-code::RustAstExtractor::extract (code:rust)")?,
|
||||
"python" => PythonAstExtractor::new()
|
||||
.extract(&ctx, &bytes)
|
||||
.context("kb-parse-code::PythonAstExtractor::extract (code:python)")?,
|
||||
"typescript" => TypescriptAstExtractor::new()
|
||||
.extract(&ctx, &bytes)
|
||||
.context("kb-parse-code::TypescriptAstExtractor::extract (code:typescript)")?,
|
||||
"javascript" => JavascriptAstExtractor::new()
|
||||
.extract(&ctx, &bytes)
|
||||
.context("kb-parse-code::JavascriptAstExtractor::extract (code:javascript)")?,
|
||||
"go" => GoAstExtractor::new()
|
||||
.extract(&ctx, &bytes)
|
||||
.context("kb-parse-code::GoAstExtractor::extract (code:go)")?,
|
||||
"java" => JavaAstExtractor::new()
|
||||
.extract(&ctx, &bytes)
|
||||
.context("kb-parse-code::JavaAstExtractor::extract (code:java)")?,
|
||||
"kotlin" => KotlinAstExtractor::new()
|
||||
.extract(&ctx, &bytes)
|
||||
.context("kb-parse-code::KotlinAstExtractor::extract (code:kotlin)")?,
|
||||
other => anyhow::bail!("unreachable (extract): {other}"),
|
||||
};
|
||||
|
||||
// p10-1b Task D/G/J/L: chunker per-lang.
|
||||
let chunks = match code_lang {
|
||||
"rust" => CodeRustAstV1Chunker
|
||||
.chunk(&canonical, chunk_policy)
|
||||
.context("kb-chunk::CodeRustAstV1Chunker::chunk (code:rust)")?,
|
||||
"python" => CodePythonAstV1Chunker
|
||||
.chunk(&canonical, chunk_policy)
|
||||
.context("kb-chunk::CodePythonAstV1Chunker::chunk (code:python)")?,
|
||||
"typescript" => CodeTsAstV1Chunker
|
||||
.chunk(&canonical, chunk_policy)
|
||||
.context("kb-chunk::CodeTsAstV1Chunker::chunk (code:typescript)")?,
|
||||
"javascript" => CodeJsAstV1Chunker
|
||||
.chunk(&canonical, chunk_policy)
|
||||
.context("kb-chunk::CodeJsAstV1Chunker::chunk (code:javascript)")?,
|
||||
"go" => CodeGoAstV1Chunker
|
||||
.chunk(&canonical, chunk_policy)
|
||||
.context("kb-chunk::CodeGoAstV1Chunker::chunk (code:go)")?,
|
||||
"java" => CodeJavaAstV1Chunker
|
||||
.chunk(&canonical, chunk_policy)
|
||||
.context("kb-chunk::CodeJavaAstV1Chunker::chunk (code:java)")?,
|
||||
"kotlin" => CodeKotlinAstV1Chunker
|
||||
.chunk(&canonical, chunk_policy)
|
||||
.context("kb-chunk::CodeKotlinAstV1Chunker::chunk (code:kotlin)")?,
|
||||
other => anyhow::bail!("unreachable (chunk): {other}"),
|
||||
};
|
||||
|
||||
// Stamp chunker + embedding versions so incremental skip detection has
|
||||
// data on the second run.
|
||||
canonical.last_chunker_version = Some(chunker.chunker_version());
|
||||
canonical.last_chunker_version = Some(chunker_version.clone());
|
||||
if let Some(emb) = embedder {
|
||||
canonical.last_embedding_version = Some(emb.model_version());
|
||||
}
|
||||
@@ -1794,7 +2005,7 @@ fn ingest_one_code_asset(
|
||||
block_count: u32::try_from(canonical.blocks.len()).ok(),
|
||||
chunk_count: u32::try_from(chunks.len()).ok(),
|
||||
parser_version: Some(canonical.parser_version.clone()),
|
||||
chunker_version: Some(chunker.chunker_version()),
|
||||
chunker_version: Some(chunker_version),
|
||||
warnings,
|
||||
error: None,
|
||||
})
|
||||
|
||||
@@ -9,13 +9,19 @@
|
||||
//!
|
||||
//! `--vector-only` additionally truncates `embedding_records` in SQLite
|
||||
//! so the next `kebab ingest` re-embeds cleanly without orphan rows.
|
||||
//!
|
||||
//! `--orphans-only` purges stored docs that are outside the current walker
|
||||
//! scope (config narrowing / removed sub-directory). No filesystem paths are
|
||||
//! removed — this is purely a store-level reconciliation.
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use kebab_config::{Config, expand_path};
|
||||
use kebab_core::WorkspacePath;
|
||||
|
||||
/// What the user asked to remove. Mutually exclusive — picked by the CLI
|
||||
/// from a clap `ArgGroup`.
|
||||
@@ -32,6 +38,13 @@ pub enum ResetScope {
|
||||
VectorOnly,
|
||||
/// Wipe only the config dir.
|
||||
ConfigOnly,
|
||||
/// Purge stored docs that are outside the current walker scope (no
|
||||
/// filesystem paths are removed). Filesystem existence is NOT checked —
|
||||
/// anything the current walker would not visit is considered an orphan.
|
||||
/// The explicit complement to the conservative `sweep_deleted_files`
|
||||
/// that runs during ingest (which leaves on-disk-but-out-of-scope docs
|
||||
/// alone for data safety).
|
||||
OrphansOnly,
|
||||
}
|
||||
|
||||
/// Result of a successful wipe — emitted as `reset_report.v1` by the
|
||||
@@ -41,6 +54,16 @@ pub struct ResetReport {
|
||||
pub scope: ResetScope,
|
||||
pub removed_paths: Vec<PathBuf>,
|
||||
pub embedding_rows_truncated: u64,
|
||||
/// Number of stored docs purged because they are outside the current
|
||||
/// walker scope. Non-zero only when `scope == OrphansOnly`.
|
||||
/// `#[serde(default)]` preserves back-compat with older callers that
|
||||
/// do not include this field.
|
||||
#[serde(default)]
|
||||
pub orphans_purged: u32,
|
||||
/// Paths of the orphaned docs that were purged. Sorted for deterministic
|
||||
/// output. Non-empty only when `scope == OrphansOnly`.
|
||||
#[serde(default)]
|
||||
pub purged_paths: Vec<WorkspacePath>,
|
||||
}
|
||||
|
||||
/// Compute the absolute on-disk paths a given scope will wipe, given a
|
||||
@@ -67,6 +90,10 @@ pub fn enumerate_paths(scope: ResetScope, cfg: &Config) -> Vec<PathBuf> {
|
||||
vec![vector_dir]
|
||||
}
|
||||
ResetScope::ConfigOnly => vec![cfg_dir],
|
||||
// OrphansOnly operates purely at the store level — no filesystem paths
|
||||
// are removed. Return empty so `estimate_size_bytes` stays zero and
|
||||
// the existing confirm UI path for directory wipes is skipped.
|
||||
ResetScope::OrphansOnly => vec![],
|
||||
}
|
||||
}
|
||||
|
||||
@@ -96,16 +123,82 @@ pub fn estimate_size_bytes(paths: &[PathBuf]) -> u64 {
|
||||
paths.iter().map(|p| walk(p)).sum()
|
||||
}
|
||||
|
||||
/// Compute the workspace paths stored in SQLite that are NOT visited by
|
||||
/// the current walker scope (i.e. they are "orphans" — on disk but
|
||||
/// outside the configured include/exclude rules, or from a sub-directory
|
||||
/// that has since been removed from the workspace).
|
||||
///
|
||||
/// Does NOT check filesystem existence — `OrphansOnly` is the explicit
|
||||
/// "I know what I'm doing" variant; callers that want the conservative
|
||||
/// fs-aware sweep should use `sweep_deleted_files` inside ingest.
|
||||
///
|
||||
/// Returns the list sorted for deterministic output. Called twice by the
|
||||
/// CLI path (once for the confirm UI preview, once inside `execute`);
|
||||
/// the double scan is acceptable for a rare destructive operation.
|
||||
pub fn enumerate_orphans(cfg: &Config) -> Result<Vec<WorkspacePath>> {
|
||||
use kebab_core::DocumentStore as _;
|
||||
use kebab_source_fs::FsSourceConnector;
|
||||
use kebab_core::SourceScope;
|
||||
|
||||
let store = kebab_store_sqlite::SqliteStore::open(cfg)
|
||||
.context("enumerate_orphans: open SqliteStore")?;
|
||||
|
||||
let stored = store
|
||||
.all_workspace_paths()
|
||||
.context("enumerate_orphans: all_workspace_paths")?;
|
||||
|
||||
if stored.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
// Build the same SourceScope the CLI's ingest path uses: root from
|
||||
// config, exclude list from config, no include override (full scope).
|
||||
let root = cfg.resolve_workspace_root();
|
||||
let scope = SourceScope {
|
||||
root: root.clone(),
|
||||
exclude: cfg.workspace.exclude.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let connector = FsSourceConnector::new(cfg)
|
||||
.context("enumerate_orphans: build FsSourceConnector")?;
|
||||
let (assets, _skips) = connector
|
||||
.scan_with_skips(&scope)
|
||||
.context("enumerate_orphans: scan workspace")?;
|
||||
|
||||
let scanned: HashSet<WorkspacePath> = assets
|
||||
.into_iter()
|
||||
.map(|a| a.workspace_path)
|
||||
.collect();
|
||||
|
||||
let mut orphans: Vec<WorkspacePath> = stored
|
||||
.into_iter()
|
||||
.filter(|p| !scanned.contains(p))
|
||||
.collect();
|
||||
orphans.sort_by(|a, b| a.0.cmp(&b.0));
|
||||
Ok(orphans)
|
||||
}
|
||||
|
||||
/// Wipe every path from `enumerate_paths(scope, cfg)`. For
|
||||
/// `ResetScope::VectorOnly`, also truncates the SQLite
|
||||
/// `embedding_records` table so the store doesn't point at the Lance
|
||||
/// rows we just removed off-disk.
|
||||
///
|
||||
/// For `ResetScope::OrphansOnly`, no filesystem directories are removed.
|
||||
/// Instead the store is reconciled: stored docs outside the current walker
|
||||
/// scope are purged from SQLite (+ vector store when configured). The
|
||||
/// caller is expected to have already shown the confirm UI using
|
||||
/// `enumerate_orphans`.
|
||||
///
|
||||
/// Idempotent: a missing path is treated as already-removed (success).
|
||||
/// Returns a `ResetReport` listing exactly what was removed (paths that
|
||||
/// existed before the call) so `--json` callers see the truth, not the
|
||||
/// request.
|
||||
pub fn execute(scope: ResetScope, cfg: &Config) -> Result<ResetReport> {
|
||||
if matches!(scope, ResetScope::OrphansOnly) {
|
||||
return execute_orphans_only(cfg);
|
||||
}
|
||||
|
||||
let paths = enumerate_paths(scope, cfg);
|
||||
let mut removed = Vec::new();
|
||||
|
||||
@@ -128,9 +221,100 @@ pub fn execute(scope: ResetScope, cfg: &Config) -> Result<ResetReport> {
|
||||
scope,
|
||||
removed_paths: removed,
|
||||
embedding_rows_truncated,
|
||||
orphans_purged: 0,
|
||||
purged_paths: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Execute the `OrphansOnly` variant: reconcile stored docs against the
|
||||
/// current walker scope without touching any filesystem directory.
|
||||
fn execute_orphans_only(cfg: &Config) -> Result<ResetReport> {
|
||||
let orphans = enumerate_orphans(cfg)
|
||||
.context("execute_orphans_only: enumerate orphans")?;
|
||||
|
||||
if orphans.is_empty() {
|
||||
return Ok(ResetReport {
|
||||
scope: ResetScope::OrphansOnly,
|
||||
removed_paths: Vec::new(),
|
||||
embedding_rows_truncated: 0,
|
||||
orphans_purged: 0,
|
||||
purged_paths: Vec::new(),
|
||||
});
|
||||
}
|
||||
|
||||
let store = std::sync::Arc::new(
|
||||
kebab_store_sqlite::SqliteStore::open(cfg)
|
||||
.context("execute_orphans_only: open SqliteStore")?,
|
||||
);
|
||||
|
||||
// Open vector store if configured. Mirror the same guard the ingest
|
||||
// path uses: only construct when the provider is not "none" / dims > 0.
|
||||
let vector_store: Option<kebab_store_vector::LanceVectorStore> =
|
||||
open_vector_store_if_configured(cfg, store.clone())?;
|
||||
|
||||
let mut purged_paths: Vec<WorkspacePath> = Vec::new();
|
||||
|
||||
for path in &orphans {
|
||||
let chunk_ids = kebab_store_sqlite::purge_deleted_workspace_path(&store, path)
|
||||
.with_context(|| format!("execute_orphans_only: purge {}", path.0))?;
|
||||
|
||||
if let Some(ref vs) = vector_store {
|
||||
if !chunk_ids.is_empty() {
|
||||
use kebab_core::VectorStore as _;
|
||||
if let Err(e) = vs.delete_by_chunk_ids(&chunk_ids) {
|
||||
tracing::warn!(
|
||||
target: "kebab-app",
|
||||
path = %path.0,
|
||||
count = chunk_ids.len(),
|
||||
error = %e,
|
||||
"reset --orphans-only: vector delete failed; SQLite side already cleaned"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tracing::info!(
|
||||
target: "kebab-app",
|
||||
path = %path.0,
|
||||
"reset --orphans-only: purged orphan document"
|
||||
);
|
||||
purged_paths.push(path.clone());
|
||||
}
|
||||
|
||||
let orphans_purged = u32::try_from(purged_paths.len()).unwrap_or(u32::MAX);
|
||||
|
||||
Ok(ResetReport {
|
||||
scope: ResetScope::OrphansOnly,
|
||||
removed_paths: Vec::new(),
|
||||
embedding_rows_truncated: 0,
|
||||
orphans_purged,
|
||||
purged_paths,
|
||||
})
|
||||
}
|
||||
|
||||
/// Open the Lance vector store if the configured embedding provider is
|
||||
/// active (non-"none", dimensions > 0). Returns `None` for lexical-only
|
||||
/// configs. Mirrors the guard in `App::vector`.
|
||||
fn open_vector_store_if_configured(
|
||||
cfg: &Config,
|
||||
store: std::sync::Arc<kebab_store_sqlite::SqliteStore>,
|
||||
) -> Result<Option<kebab_store_vector::LanceVectorStore>> {
|
||||
if cfg.models.embedding.provider == "none" || cfg.models.embedding.dimensions == 0 {
|
||||
return Ok(None);
|
||||
}
|
||||
match kebab_store_vector::LanceVectorStore::new(cfg, store) {
|
||||
Ok(vs) => Ok(Some(vs)),
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
target: "kebab-app",
|
||||
error = %e,
|
||||
"reset --orphans-only: could not open vector store; skipping vector delete"
|
||||
);
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Open the SQLite store at the configured path and run
|
||||
/// `truncate_embedding_records`. Returns the count of truncated rows
|
||||
/// (the helper itself reports `DELETE` rowcount). If the SQLite file
|
||||
@@ -200,4 +384,14 @@ mod tests {
|
||||
let bytes = estimate_size_bytes(&[dir.path().to_path_buf()]);
|
||||
assert_eq!(bytes, 5 + 6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn enumerate_orphans_only_returns_empty_paths() {
|
||||
let cfg = Config::defaults();
|
||||
let paths = enumerate_paths(ResetScope::OrphansOnly, &cfg);
|
||||
assert!(
|
||||
paths.is_empty(),
|
||||
"OrphansOnly must return empty vec from enumerate_paths"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -168,7 +168,9 @@ fn collect_stats(
|
||||
stale_doc_count: counts.stale_doc_count,
|
||||
// p10-1A-2: populated by the store query added in this task.
|
||||
code_lang_breakdown: store.code_lang_breakdown()?,
|
||||
repo_breakdown: std::collections::BTreeMap::new(),
|
||||
// p10-1A-2 follow-up: dogfooding (2026-05-20) revealed this was a
|
||||
// placeholder — mirror of code_lang_breakdown for the repo field.
|
||||
repo_breakdown: store.repo_breakdown()?,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -159,6 +159,450 @@ fn rust_code_search_hit_has_repo() {
|
||||
);
|
||||
}
|
||||
|
||||
/// p10-1b Task G: a `.py` file in a sub-directory is ingested and the
|
||||
/// resulting `Citation::Code` hit must carry `lang="python"`,
|
||||
/// `symbol="kebab_eval.metrics.compute_mrr"`, and `line_start >= 1`.
|
||||
/// The sub-directory (`kebab_eval/`) ensures `module_path_for_python`
|
||||
/// produces a non-empty prefix so the fully-qualified symbol assertion
|
||||
/// exercises the prefix wiring end-to-end.
|
||||
#[test]
|
||||
fn python_file_ingests_and_searches_as_code_citation() {
|
||||
let env = TestEnv::lexical_only();
|
||||
|
||||
let module_dir = env.workspace_root.join("kebab_eval");
|
||||
std::fs::create_dir_all(&module_dir).unwrap();
|
||||
std::fs::write(
|
||||
module_dir.join("metrics.py"),
|
||||
"\"\"\"compute metrics.\"\"\"\ndef compute_mrr(scores):\n return sum(scores) / max(len(scores), 1)\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let report =
|
||||
kebab_app::ingest_with_config(env.config.clone(), env.scope(), false)
|
||||
.expect("ingest must succeed");
|
||||
|
||||
assert!(report.new >= 1, "python file ingested: {report:?}");
|
||||
|
||||
let items = report.items.as_ref().expect("items present");
|
||||
let py_item = items
|
||||
.iter()
|
||||
.find(|i| i.doc_path.0.ends_with("metrics.py"))
|
||||
.expect("metrics.py item");
|
||||
assert_eq!(
|
||||
py_item.parser_version.as_ref().map(|p| p.0.as_str()),
|
||||
Some("code-python-v1"),
|
||||
"parser_version must be code-python-v1"
|
||||
);
|
||||
assert_eq!(
|
||||
py_item.chunker_version.as_ref().map(|c| c.0.as_str()),
|
||||
Some("code-python-ast-v1"),
|
||||
"chunker_version must be code-python-ast-v1"
|
||||
);
|
||||
|
||||
let hits = kebab_app::search_with_config(env.config.clone(), lexical_query("compute_mrr"))
|
||||
.expect("search must succeed");
|
||||
|
||||
let h = hits
|
||||
.iter()
|
||||
.find(|h| matches!(&h.citation, Citation::Code { .. }))
|
||||
.expect("at least one Citation::Code hit for 'compute_mrr'");
|
||||
|
||||
match &h.citation {
|
||||
Citation::Code {
|
||||
lang,
|
||||
symbol,
|
||||
line_start,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(
|
||||
lang.as_deref(),
|
||||
Some("python"),
|
||||
"citation.lang must be 'python'"
|
||||
);
|
||||
assert_eq!(
|
||||
symbol.as_deref(),
|
||||
Some("kebab_eval.metrics.compute_mrr"),
|
||||
"citation.symbol must be 'kebab_eval.metrics.compute_mrr'"
|
||||
);
|
||||
assert!(*line_start >= 1, "line_start must be >=1");
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
h.code_lang.as_deref(),
|
||||
Some("python"),
|
||||
"SearchHit.code_lang must be 'python'"
|
||||
);
|
||||
}
|
||||
|
||||
/// p10-1b Task J: a `.ts` file in a sub-directory is ingested and the
|
||||
/// resulting `Citation::Code` hit must carry `lang="typescript"`,
|
||||
/// `symbol="src/Foo.Foo.bar"`, and `line_start >= 1`.
|
||||
/// The sub-directory (`src/`) ensures `module_path_for_tsjs` produces
|
||||
/// a non-empty prefix so the fully-qualified symbol assertion exercises
|
||||
/// the prefix wiring end-to-end.
|
||||
#[test]
|
||||
fn typescript_file_ingests_and_searches_as_code_citation() {
|
||||
let env = TestEnv::lexical_only();
|
||||
|
||||
let src_dir = env.workspace_root.join("src");
|
||||
std::fs::create_dir_all(&src_dir).unwrap();
|
||||
std::fs::write(
|
||||
src_dir.join("Foo.ts"),
|
||||
"export class Foo {\n bar(): number { return 42; }\n}\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let report =
|
||||
kebab_app::ingest_with_config(env.config.clone(), env.scope(), false)
|
||||
.expect("ingest must succeed");
|
||||
|
||||
assert!(report.new >= 1, "ts file ingested: {report:?}");
|
||||
|
||||
let items = report.items.as_ref().expect("items present");
|
||||
let ts_item = items
|
||||
.iter()
|
||||
.find(|i| i.doc_path.0.ends_with("Foo.ts"))
|
||||
.expect("Foo.ts item");
|
||||
assert_eq!(
|
||||
ts_item.parser_version.as_ref().map(|p| p.0.as_str()),
|
||||
Some("code-ts-v1"),
|
||||
"parser_version must be code-ts-v1"
|
||||
);
|
||||
assert_eq!(
|
||||
ts_item.chunker_version.as_ref().map(|c| c.0.as_str()),
|
||||
Some("code-ts-ast-v1"),
|
||||
"chunker_version must be code-ts-ast-v1"
|
||||
);
|
||||
|
||||
let hits = kebab_app::search_with_config(env.config.clone(), lexical_query("bar"))
|
||||
.expect("search must succeed");
|
||||
|
||||
let h = hits
|
||||
.iter()
|
||||
.find(|h| matches!(&h.citation, Citation::Code { .. }))
|
||||
.expect("at least one Citation::Code hit for 'bar'");
|
||||
|
||||
match &h.citation {
|
||||
Citation::Code {
|
||||
lang,
|
||||
symbol,
|
||||
line_start,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(
|
||||
lang.as_deref(),
|
||||
Some("typescript"),
|
||||
"citation.lang must be 'typescript'"
|
||||
);
|
||||
assert_eq!(
|
||||
symbol.as_deref(),
|
||||
Some("src/Foo.Foo.bar"),
|
||||
"citation.symbol must be 'src/Foo.Foo.bar'"
|
||||
);
|
||||
assert!(*line_start >= 1, "line_start must be >=1");
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
h.code_lang.as_deref(),
|
||||
Some("typescript"),
|
||||
"SearchHit.code_lang must be 'typescript'"
|
||||
);
|
||||
}
|
||||
|
||||
/// p10-1b Task L: a `.js` file in a sub-directory is ingested and the
|
||||
/// resulting `Citation::Code` hit must carry `lang="javascript"`,
|
||||
/// `symbol="src/Bar.Bar.baz"`, and `line_start >= 1`.
|
||||
/// The sub-directory (`src/`) ensures `module_path_for_tsjs` produces
|
||||
/// a non-empty prefix so the fully-qualified symbol assertion exercises
|
||||
/// the prefix wiring end-to-end.
|
||||
#[test]
|
||||
fn javascript_file_ingests_and_searches_as_code_citation() {
|
||||
let env = TestEnv::lexical_only();
|
||||
|
||||
let src_dir = env.workspace_root.join("src");
|
||||
std::fs::create_dir_all(&src_dir).unwrap();
|
||||
std::fs::write(
|
||||
src_dir.join("Bar.js"),
|
||||
"export class Bar {\n baz() { return 7; }\n}\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let report =
|
||||
kebab_app::ingest_with_config(env.config.clone(), env.scope(), false)
|
||||
.expect("ingest must succeed");
|
||||
|
||||
assert!(report.new >= 1, "js file ingested: {report:?}");
|
||||
|
||||
let items = report.items.as_ref().expect("items present");
|
||||
let js_item = items
|
||||
.iter()
|
||||
.find(|i| i.doc_path.0.ends_with("Bar.js"))
|
||||
.expect("Bar.js item");
|
||||
assert_eq!(
|
||||
js_item.parser_version.as_ref().map(|p| p.0.as_str()),
|
||||
Some("code-js-v1"),
|
||||
"parser_version must be code-js-v1"
|
||||
);
|
||||
assert_eq!(
|
||||
js_item.chunker_version.as_ref().map(|c| c.0.as_str()),
|
||||
Some("code-js-ast-v1"),
|
||||
"chunker_version must be code-js-ast-v1"
|
||||
);
|
||||
|
||||
let hits = kebab_app::search_with_config(env.config.clone(), lexical_query("baz"))
|
||||
.expect("search must succeed");
|
||||
|
||||
let h = hits
|
||||
.iter()
|
||||
.find(|h| matches!(&h.citation, Citation::Code { .. }))
|
||||
.expect("at least one Citation::Code hit for 'baz'");
|
||||
|
||||
match &h.citation {
|
||||
Citation::Code {
|
||||
lang,
|
||||
symbol,
|
||||
line_start,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(
|
||||
lang.as_deref(),
|
||||
Some("javascript"),
|
||||
"citation.lang must be 'javascript'"
|
||||
);
|
||||
assert_eq!(
|
||||
symbol.as_deref(),
|
||||
Some("src/Bar.Bar.baz"),
|
||||
"citation.symbol must be 'src/Bar.Bar.baz'"
|
||||
);
|
||||
assert!(*line_start >= 1, "line_start must be >=1");
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
h.code_lang.as_deref(),
|
||||
Some("javascript"),
|
||||
"SearchHit.code_lang must be 'javascript'"
|
||||
);
|
||||
}
|
||||
|
||||
/// p10-1c-go Task F: a `.go` file in a sub-directory is ingested and the
|
||||
/// resulting `Citation::Code` hit must carry `lang="go"`,
|
||||
/// `symbol="chunk.ParseDoc"`, and `line_start >= 1`.
|
||||
/// The sub-directory (`chunk/`) ensures the Go package-prefix wiring
|
||||
/// produces a non-empty module prefix so the fully-qualified symbol assertion
|
||||
/// exercises that path end-to-end.
|
||||
#[test]
|
||||
fn go_file_ingests_and_searches_as_code_citation() {
|
||||
let env = TestEnv::lexical_only();
|
||||
|
||||
let pkg_dir = env.workspace_root.join("chunk");
|
||||
std::fs::create_dir_all(&pkg_dir).unwrap();
|
||||
std::fs::write(
|
||||
pkg_dir.join("ast.go"),
|
||||
"package chunk\n\nfunc ParseDoc(input string) string {\n return input\n}\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let report = kebab_app::ingest_with_config(env.config.clone(), env.scope(), false)
|
||||
.expect("ingest must succeed");
|
||||
assert_eq!(report.errors, 0);
|
||||
assert!(report.new >= 1);
|
||||
|
||||
let go_item = report
|
||||
.items
|
||||
.as_ref()
|
||||
.expect("items present")
|
||||
.iter()
|
||||
.find(|i| i.doc_path.0.ends_with("ast.go"))
|
||||
.expect("ast.go item present");
|
||||
assert_eq!(
|
||||
go_item.parser_version.as_ref().map(|p| p.0.as_str()),
|
||||
Some("code-go-v1"),
|
||||
"parser_version must be code-go-v1"
|
||||
);
|
||||
assert_eq!(
|
||||
go_item.chunker_version.as_ref().map(|c| c.0.as_str()),
|
||||
Some("code-go-ast-v1"),
|
||||
"chunker_version must be code-go-ast-v1"
|
||||
);
|
||||
|
||||
let hits = kebab_app::search_with_config(env.config.clone(), lexical_query("ParseDoc"))
|
||||
.expect("search must succeed");
|
||||
let h = hits
|
||||
.iter()
|
||||
.find(|h| matches!(&h.citation, kebab_core::Citation::Code { .. }))
|
||||
.expect("Citation::Code hit");
|
||||
match &h.citation {
|
||||
kebab_core::Citation::Code {
|
||||
lang,
|
||||
symbol,
|
||||
line_start,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(lang.as_deref(), Some("go"), "citation.lang must be 'go'");
|
||||
assert_eq!(
|
||||
symbol.as_deref(),
|
||||
Some("chunk.ParseDoc"),
|
||||
"citation.symbol must be 'chunk.ParseDoc'"
|
||||
);
|
||||
assert!(*line_start >= 1, "line_start must be >=1");
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
assert_eq!(
|
||||
h.code_lang.as_deref(),
|
||||
Some("go"),
|
||||
"SearchHit.code_lang must be 'go'"
|
||||
);
|
||||
}
|
||||
|
||||
/// p10-1c-jk Task F: a `.java` file in a package directory is ingested and the
|
||||
/// resulting `Citation::Code` hit must carry `lang="java"`,
|
||||
/// `symbol="com.foo.Foo.bar"`, and `line_start >= 1`.
|
||||
/// The sub-directory (`com/foo/`) ensures the Java package-prefix wiring
|
||||
/// produces a non-empty module prefix so the fully-qualified symbol assertion
|
||||
/// exercises that path end-to-end.
|
||||
#[test]
|
||||
fn java_file_ingests_and_searches_as_code_citation() {
|
||||
let env = TestEnv::lexical_only();
|
||||
|
||||
let pkg_dir = env.workspace_root.join("com").join("foo");
|
||||
std::fs::create_dir_all(&pkg_dir).unwrap();
|
||||
std::fs::write(
|
||||
pkg_dir.join("Foo.java"),
|
||||
"package com.foo;\n\npublic class Foo {\n public String bar() { return \"x\"; }\n}\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let report = kebab_app::ingest_with_config(env.config.clone(), env.scope(), false)
|
||||
.expect("ingest must succeed");
|
||||
assert_eq!(report.errors, 0);
|
||||
assert!(report.new >= 1);
|
||||
|
||||
let java_item = report
|
||||
.items
|
||||
.as_ref()
|
||||
.expect("items present")
|
||||
.iter()
|
||||
.find(|i| i.doc_path.0.ends_with("Foo.java"))
|
||||
.expect("Foo.java item present");
|
||||
assert_eq!(
|
||||
java_item.parser_version.as_ref().map(|p| p.0.as_str()),
|
||||
Some("code-java-v1"),
|
||||
"parser_version must be code-java-v1"
|
||||
);
|
||||
assert_eq!(
|
||||
java_item.chunker_version.as_ref().map(|c| c.0.as_str()),
|
||||
Some("code-java-ast-v1"),
|
||||
"chunker_version must be code-java-ast-v1"
|
||||
);
|
||||
|
||||
let hits = kebab_app::search_with_config(env.config.clone(), lexical_query("bar"))
|
||||
.expect("search must succeed");
|
||||
let h = hits
|
||||
.iter()
|
||||
.find(|h| matches!(&h.citation, kebab_core::Citation::Code { .. }))
|
||||
.expect("Citation::Code hit");
|
||||
match &h.citation {
|
||||
kebab_core::Citation::Code {
|
||||
lang,
|
||||
symbol,
|
||||
line_start,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(lang.as_deref(), Some("java"), "citation.lang must be 'java'");
|
||||
assert_eq!(
|
||||
symbol.as_deref(),
|
||||
Some("com.foo.Foo.bar"),
|
||||
"citation.symbol must be 'com.foo.Foo.bar'"
|
||||
);
|
||||
assert!(*line_start >= 1, "line_start must be >=1");
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
assert_eq!(
|
||||
h.code_lang.as_deref(),
|
||||
Some("java"),
|
||||
"SearchHit.code_lang must be 'java'"
|
||||
);
|
||||
}
|
||||
|
||||
/// p10-1c-jk Task I: a `.kt` file in a package directory is ingested and the
|
||||
/// resulting `Citation::Code` hit must carry `lang="kotlin"`,
|
||||
/// `symbol="com.foo.Foo.bar"`, and `line_start >= 1`.
|
||||
/// The sub-directory (`com/foo/`) ensures the Kotlin package-prefix wiring
|
||||
/// produces a non-empty module prefix so the fully-qualified symbol assertion
|
||||
/// exercises that path end-to-end.
|
||||
#[test]
|
||||
fn kotlin_file_ingests_and_searches_as_code_citation() {
|
||||
let env = TestEnv::lexical_only();
|
||||
|
||||
let pkg_dir = env.workspace_root.join("com").join("foo");
|
||||
std::fs::create_dir_all(&pkg_dir).unwrap();
|
||||
std::fs::write(
|
||||
pkg_dir.join("Foo.kt"),
|
||||
"package com.foo\n\nclass Foo {\n fun bar(): String = \"x\"\n}\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let report = kebab_app::ingest_with_config(env.config.clone(), env.scope(), false)
|
||||
.expect("ingest must succeed");
|
||||
assert_eq!(report.errors, 0);
|
||||
assert!(report.new >= 1);
|
||||
|
||||
let kt_item = report
|
||||
.items
|
||||
.as_ref()
|
||||
.expect("items present")
|
||||
.iter()
|
||||
.find(|i| i.doc_path.0.ends_with("Foo.kt"))
|
||||
.expect("Foo.kt item present");
|
||||
assert_eq!(
|
||||
kt_item.parser_version.as_ref().map(|p| p.0.as_str()),
|
||||
Some("code-kotlin-v1"),
|
||||
"parser_version must be code-kotlin-v1"
|
||||
);
|
||||
assert_eq!(
|
||||
kt_item.chunker_version.as_ref().map(|c| c.0.as_str()),
|
||||
Some("code-kotlin-ast-v1"),
|
||||
"chunker_version must be code-kotlin-ast-v1"
|
||||
);
|
||||
|
||||
let hits = kebab_app::search_with_config(env.config.clone(), lexical_query("bar"))
|
||||
.expect("search must succeed");
|
||||
let h = hits
|
||||
.iter()
|
||||
.find(|h| matches!(&h.citation, kebab_core::Citation::Code { .. }))
|
||||
.expect("Citation::Code hit");
|
||||
match &h.citation {
|
||||
kebab_core::Citation::Code {
|
||||
lang,
|
||||
symbol,
|
||||
line_start,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(lang.as_deref(), Some("kotlin"), "citation.lang must be 'kotlin'");
|
||||
assert_eq!(
|
||||
symbol.as_deref(),
|
||||
Some("com.foo.Foo.bar"),
|
||||
"citation.symbol must be 'com.foo.Foo.bar'"
|
||||
);
|
||||
assert!(*line_start >= 1, "line_start must be >=1");
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
assert_eq!(
|
||||
h.code_lang.as_deref(),
|
||||
Some("kotlin"),
|
||||
"SearchHit.code_lang must be 'kotlin'"
|
||||
);
|
||||
}
|
||||
|
||||
/// Re-ingesting the same `.rs` file without changes must report
|
||||
/// `Unchanged` (incremental-skip path exercised).
|
||||
#[test]
|
||||
|
||||
178
crates/kebab-app/tests/file_deletion_auto_purge.rs
Normal file
178
crates/kebab-app/tests/file_deletion_auto_purge.rs
Normal file
@@ -0,0 +1,178 @@
|
||||
//! Dogfood: auto-purge stored docs for filesystem-deleted files.
|
||||
//!
|
||||
//! Two tests:
|
||||
//!
|
||||
//! 1. `file_deletion_auto_purge` — ingest 2 files, delete one, re-ingest.
|
||||
//! The re-ingest must report `purged_deleted_files = 1`, the deleted
|
||||
//! file must no longer appear in `list_docs`, and lexical search for
|
||||
//! its unique content must return no hits.
|
||||
//!
|
||||
//! 2. `include_scope_narrowing_does_not_purge` — ingest 2 files under a
|
||||
//! wide glob, narrow the walker scope to only one file, re-ingest.
|
||||
//! The narrowed ingest must NOT purge the out-of-scope file because
|
||||
//! the file is still on disk (just excluded from this run). Protects
|
||||
//! users against accidental data loss via config edits.
|
||||
|
||||
mod common;
|
||||
|
||||
use common::TestEnv;
|
||||
use kebab_app::ingest_with_config_opts;
|
||||
use kebab_app::IngestOpts;
|
||||
use kebab_core::{DocFilter, DocumentStore, SearchMode, SearchQuery, SourceScope};
|
||||
|
||||
/// Helper: open the store via `TestEnv` and run `list_documents`.
|
||||
fn list_doc_paths(env: &TestEnv) -> Vec<String> {
|
||||
use kebab_store_sqlite::SqliteStore;
|
||||
let store = SqliteStore::open(&env.config).unwrap();
|
||||
store.run_migrations().unwrap();
|
||||
store
|
||||
.list_documents(&DocFilter::default())
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|d| d.doc_path.0)
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn file_deletion_auto_purge() {
|
||||
let env = TestEnv::lexical_only();
|
||||
|
||||
// Write two .rs files into the workspace.
|
||||
let a_path = env.workspace_root.join("a.rs");
|
||||
let b_path = env.workspace_root.join("b.rs");
|
||||
std::fs::write(&a_path, "// file a\nfn alpha() {}\n").unwrap();
|
||||
std::fs::write(&b_path, "// file b\nfn bravo() {}\n").unwrap();
|
||||
|
||||
// First ingest — both must be New.
|
||||
let first = ingest_with_config_opts(
|
||||
env.config.clone(),
|
||||
env.scope(),
|
||||
false,
|
||||
IngestOpts::default(),
|
||||
)
|
||||
.expect("first ingest must succeed");
|
||||
// Only count the .rs files we added (there may be fixture files too).
|
||||
let first_new = first.new;
|
||||
assert!(first_new >= 2, "expected at least 2 new docs: {first:?}");
|
||||
assert_eq!(
|
||||
first.purged_deleted_files, 0,
|
||||
"no purges on first ingest: {first:?}"
|
||||
);
|
||||
assert_eq!(first.errors, 0, "no errors on first ingest: {first:?}");
|
||||
|
||||
// Delete one file from the filesystem.
|
||||
std::fs::remove_file(&b_path).expect("remove b.rs");
|
||||
|
||||
// Second ingest — scanned count drops by 1; b.rs should be purged.
|
||||
let second = ingest_with_config_opts(
|
||||
env.config.clone(),
|
||||
env.scope(),
|
||||
false,
|
||||
IngestOpts::default(),
|
||||
)
|
||||
.expect("second ingest must succeed");
|
||||
|
||||
assert_eq!(
|
||||
second.purged_deleted_files, 1,
|
||||
"exactly 1 file should be purged: {second:?}"
|
||||
);
|
||||
assert_eq!(second.new, 0, "no new docs after deletion: {second:?}");
|
||||
assert_eq!(second.updated, 0, "no updated docs: {second:?}");
|
||||
assert_eq!(second.errors, 0, "no errors: {second:?}");
|
||||
|
||||
// b.rs must no longer appear in list_docs.
|
||||
let doc_paths = list_doc_paths(&env);
|
||||
let b_ws_path = "b.rs";
|
||||
assert!(
|
||||
!doc_paths.iter().any(|p| p == b_ws_path),
|
||||
"b.rs must be gone from list_docs; got: {doc_paths:?}"
|
||||
);
|
||||
// a.rs must still be present.
|
||||
let a_ws_path = "a.rs";
|
||||
assert!(
|
||||
doc_paths.iter().any(|p| p == a_ws_path),
|
||||
"a.rs must still be in list_docs; got: {doc_paths:?}"
|
||||
);
|
||||
|
||||
// Lexical search for b.rs's unique content returns no hits.
|
||||
let app = env.app();
|
||||
let query = SearchQuery {
|
||||
text: "bravo".to_string(),
|
||||
mode: SearchMode::Lexical,
|
||||
k: 10,
|
||||
filters: kebab_core::SearchFilters::default(),
|
||||
};
|
||||
let hits = app.search(query).expect("search must not error");
|
||||
assert!(
|
||||
hits.is_empty(),
|
||||
"search for deleted file's content must return no hits; got: {hits:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn include_scope_narrowing_does_not_purge() {
|
||||
let env = TestEnv::lexical_only();
|
||||
|
||||
// Write two .rs files.
|
||||
let a_path = env.workspace_root.join("a_narrow.rs");
|
||||
let b_path = env.workspace_root.join("b_narrow.rs");
|
||||
std::fs::write(&a_path, "// narrow a\nfn alpha_narrow() {}\n").unwrap();
|
||||
std::fs::write(&b_path, "// narrow b\nfn bravo_narrow() {}\n").unwrap();
|
||||
|
||||
// Wide scope: first ingest — both must be New.
|
||||
let wide_scope = SourceScope {
|
||||
root: env.workspace_root.clone(),
|
||||
include: vec!["**/*.rs".to_string()],
|
||||
exclude: env.config.workspace.exclude.clone(),
|
||||
};
|
||||
let first = ingest_with_config_opts(
|
||||
env.config.clone(),
|
||||
wide_scope,
|
||||
false,
|
||||
IngestOpts::default(),
|
||||
)
|
||||
.expect("first ingest (wide) must succeed");
|
||||
assert!(
|
||||
first.new >= 2,
|
||||
"expected at least 2 new docs: {first:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
first.purged_deleted_files, 0,
|
||||
"no purges on first ingest: {first:?}"
|
||||
);
|
||||
|
||||
// Narrow scope: only a_narrow.rs in include — b_narrow.rs is still
|
||||
// on disk but excluded from the walker scope.
|
||||
let narrow_scope = SourceScope {
|
||||
root: env.workspace_root.clone(),
|
||||
include: vec!["a_narrow.rs".to_string()],
|
||||
exclude: env.config.workspace.exclude.clone(),
|
||||
};
|
||||
let second = ingest_with_config_opts(
|
||||
env.config.clone(),
|
||||
narrow_scope,
|
||||
false,
|
||||
IngestOpts::default(),
|
||||
)
|
||||
.expect("second ingest (narrow) must succeed");
|
||||
|
||||
// CRITICAL: b_narrow.rs is still on disk — must NOT be purged.
|
||||
assert_eq!(
|
||||
second.purged_deleted_files, 0,
|
||||
"scope-narrowing must NOT purge on-disk files; got: {second:?}"
|
||||
);
|
||||
assert_eq!(second.errors, 0, "no errors: {second:?}");
|
||||
|
||||
// b_narrow.rs must still exist in the store.
|
||||
let doc_paths = list_doc_paths(&env);
|
||||
let b_ws_path = "b_narrow.rs";
|
||||
assert!(
|
||||
doc_paths.iter().any(|p| p == b_ws_path),
|
||||
"b_narrow.rs must still be in list_docs after scope narrowing; got: {doc_paths:?}"
|
||||
);
|
||||
// And the file must still be on disk.
|
||||
assert!(
|
||||
b_path.exists(),
|
||||
"b_narrow.rs must still be on disk (we didn't delete it)"
|
||||
);
|
||||
}
|
||||
141
crates/kebab-app/tests/reset_orphans.rs
Normal file
141
crates/kebab-app/tests/reset_orphans.rs
Normal file
@@ -0,0 +1,141 @@
|
||||
//! Integration test for `kebab reset --orphans-only`.
|
||||
//!
|
||||
//! Verifies that stored docs outside the current walker scope are purged
|
||||
//! from the store without removing any files from the filesystem.
|
||||
//!
|
||||
//! Test outline:
|
||||
//! 1. Ingest 3 .rs files (a.rs, b.rs, c.rs) — all New.
|
||||
//! 2. Narrow the config `include` to `["a.rs"]` only; b.rs and c.rs are
|
||||
//! still on disk but outside the walker scope.
|
||||
//! 3. Run `execute(ResetScope::OrphansOnly, &cfg)` — report must show
|
||||
//! `orphans_purged == 2` and `purged_paths` contains b.rs + c.rs.
|
||||
//! 4. `list docs` must show only a.rs.
|
||||
//! 5. b.rs and c.rs must still exist on disk (no filesystem removal).
|
||||
//! 6. Second reset → `orphans_purged == 0` (idempotent).
|
||||
|
||||
mod common;
|
||||
|
||||
use common::TestEnv;
|
||||
use kebab_app::IngestOpts;
|
||||
use kebab_app::reset::{ResetScope, execute};
|
||||
use kebab_core::{DocFilter, DocumentStore, SourceScope};
|
||||
|
||||
/// Open the SqliteStore and list all `workspace_path` values.
|
||||
fn list_doc_paths(env: &TestEnv) -> Vec<String> {
|
||||
use kebab_store_sqlite::SqliteStore;
|
||||
let store = SqliteStore::open(&env.config).unwrap();
|
||||
store.run_migrations().unwrap();
|
||||
store
|
||||
.list_documents(&DocFilter::default())
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|d| d.doc_path.0)
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reset_orphans_only_purges_out_of_scope_docs() {
|
||||
let env = TestEnv::lexical_only();
|
||||
|
||||
// Write three .rs files into the workspace.
|
||||
let a_path = env.workspace_root.join("a.rs");
|
||||
let b_path = env.workspace_root.join("b.rs");
|
||||
let c_path = env.workspace_root.join("c.rs");
|
||||
std::fs::write(&a_path, "// file a\nfn alpha() {}\n").unwrap();
|
||||
std::fs::write(&b_path, "// file b\nfn bravo() {}\n").unwrap();
|
||||
std::fs::write(&c_path, "// file c\nfn charlie() {}\n").unwrap();
|
||||
|
||||
// Ingest all three with a wide scope.
|
||||
let wide_scope = SourceScope {
|
||||
root: env.workspace_root.clone(),
|
||||
include: vec!["**/*.rs".to_string()],
|
||||
exclude: env.config.workspace.exclude.clone(),
|
||||
};
|
||||
let first = kebab_app::ingest_with_config_opts(
|
||||
env.config.clone(),
|
||||
wide_scope,
|
||||
false,
|
||||
IngestOpts::default(),
|
||||
)
|
||||
.expect("first ingest must succeed");
|
||||
// The fixture workspace may contain other .rs files — just assert we
|
||||
// got at least 3 new docs (our a.rs, b.rs, c.rs).
|
||||
assert!(first.new >= 3, "expected at least 3 new docs: {first:?}");
|
||||
assert_eq!(first.errors, 0, "no errors on first ingest");
|
||||
|
||||
// Narrow config to include only a.rs; b.rs + c.rs are still on disk.
|
||||
let mut narrow_cfg = env.config.clone();
|
||||
narrow_cfg.workspace.exclude.clear();
|
||||
// Re-point workspace root (already correct) and restrict include via
|
||||
// the SourceScope in the connector. The config's `workspace.root` is
|
||||
// used by `enumerate_orphans` to build its scope — we keep that
|
||||
// pointing at the workspace root. We simulate narrowing by setting a
|
||||
// glob that only matches a.rs.
|
||||
//
|
||||
// NOTE: `kebab_config::WorkspaceCfg` does not have an `include` field
|
||||
// (it was removed in p9-fb-25). We narrow the scope via the walker
|
||||
// exclude list: exclude b.rs and c.rs explicitly.
|
||||
narrow_cfg.workspace.exclude = vec!["b.rs".to_string(), "c.rs".to_string()];
|
||||
|
||||
// Run orphans-only reset.
|
||||
let report = execute(ResetScope::OrphansOnly, &narrow_cfg)
|
||||
.expect("orphans-only reset must succeed");
|
||||
|
||||
assert_eq!(
|
||||
report.orphans_purged, 2,
|
||||
"expected 2 orphans purged (b.rs + c.rs): {report:?}"
|
||||
);
|
||||
|
||||
let mut purged: Vec<String> = report
|
||||
.purged_paths
|
||||
.iter()
|
||||
.map(|p| p.0.clone())
|
||||
.collect();
|
||||
purged.sort();
|
||||
assert_eq!(
|
||||
purged,
|
||||
vec!["b.rs".to_string(), "c.rs".to_string()],
|
||||
"purged_paths must list b.rs and c.rs in sorted order: {purged:?}"
|
||||
);
|
||||
|
||||
// list docs must show only a.rs (and any pre-existing fixture files
|
||||
// that are not excluded by the narrow config).
|
||||
let doc_paths = list_doc_paths(&env);
|
||||
// The narrow_cfg excludes b.rs + c.rs — they must no longer be in store.
|
||||
assert!(
|
||||
!doc_paths.iter().any(|p| p == "b.rs"),
|
||||
"b.rs must be gone from store after orphans-only reset; got: {doc_paths:?}"
|
||||
);
|
||||
assert!(
|
||||
!doc_paths.iter().any(|p| p == "c.rs"),
|
||||
"c.rs must be gone from store after orphans-only reset; got: {doc_paths:?}"
|
||||
);
|
||||
assert!(
|
||||
doc_paths.iter().any(|p| p == "a.rs"),
|
||||
"a.rs must still be in store; got: {doc_paths:?}"
|
||||
);
|
||||
|
||||
// Both b.rs and c.rs must still exist on the filesystem — no file
|
||||
// removal is performed by orphans-only.
|
||||
assert!(
|
||||
b_path.exists(),
|
||||
"b.rs must still be on disk after orphans-only reset"
|
||||
);
|
||||
assert!(
|
||||
c_path.exists(),
|
||||
"c.rs must still be on disk after orphans-only reset"
|
||||
);
|
||||
|
||||
// Second reset must be idempotent: nothing left to purge.
|
||||
let second = execute(ResetScope::OrphansOnly, &narrow_cfg)
|
||||
.expect("second orphans-only reset must succeed");
|
||||
assert_eq!(
|
||||
second.orphans_purged, 0,
|
||||
"second reset must be idempotent (orphans_purged == 0): {second:?}"
|
||||
);
|
||||
assert!(
|
||||
second.purged_paths.is_empty(),
|
||||
"second reset purged_paths must be empty: {:?}",
|
||||
second.purged_paths
|
||||
);
|
||||
}
|
||||
176
crates/kebab-app/tests/twin_files_fetch_span.rs
Normal file
176
crates/kebab-app/tests/twin_files_fetch_span.rs
Normal file
@@ -0,0 +1,176 @@
|
||||
//! Regression test for the twin-file fetch_span media-type lookup bug.
|
||||
//!
|
||||
//! Twin files (identical content at different workspace paths) share one
|
||||
//! `assets` row whose PRIMARY KEY is the blake3 content hash. The old
|
||||
//! `fetch_span` implementation called
|
||||
//! `get_asset_by_workspace_path(&doc.workspace_path)` to check whether the
|
||||
//! media type was PDF/audio (and therefore reject span fetch). For a twin
|
||||
//! file that lookup could silently return the *other* twin's asset row if
|
||||
//! `assets.workspace_path` had been overwritten on the most recent ingest of
|
||||
//! the sibling — making the media-type branch decision incorrect.
|
||||
//!
|
||||
//! Fix: `fetch_span` now uses the 2-step lookup
|
||||
//! `get_document_by_workspace_path` → `doc.source_asset_id` → `get_asset`
|
||||
//! so the result is always anchored to the requesting document, not
|
||||
//! whichever twin last updated `assets.workspace_path`.
|
||||
//!
|
||||
//! This test builds a twin-file scenario (two .md files at different paths
|
||||
//! with identical content), ingests both, then calls `fetch_span` on each
|
||||
//! twin's `doc_id` and asserts it succeeds. Before the fix, if the asset
|
||||
//! row's workspace_path happened to point at the wrong twin the span could
|
||||
//! return an incorrect `span_not_supported` for a non-PDF/audio file, or
|
||||
//! conversely allow span on a PDF twin by accident. After the fix, the
|
||||
//! lookup is always doc-specific.
|
||||
|
||||
mod common;
|
||||
|
||||
use common::TestEnv;
|
||||
use kebab_app::ingest_with_config;
|
||||
use kebab_core::{DocumentStore, FetchKind, FetchOpts, FetchQuery, IngestItemKind};
|
||||
|
||||
#[test]
|
||||
fn twin_files_fetch_span_uses_correct_asset() {
|
||||
let env = TestEnv::lexical_only();
|
||||
|
||||
// Write two markdown files with identical content at different paths.
|
||||
let dir_a = env.workspace_root.join("src_a");
|
||||
let dir_b = env.workspace_root.join("src_b");
|
||||
std::fs::create_dir_all(&dir_a).unwrap();
|
||||
std::fs::create_dir_all(&dir_b).unwrap();
|
||||
|
||||
// The content must produce at least 1 line so span fetch is non-trivial.
|
||||
let content = "# Twin\n\nLine one.\n\nLine two.\n\nLine three.\n";
|
||||
std::fs::write(dir_a.join("note.md"), content).unwrap();
|
||||
std::fs::write(dir_b.join("note.md"), content).unwrap();
|
||||
|
||||
// Ingest all files (fixture workspace + our two new twins).
|
||||
let report = ingest_with_config(env.config.clone(), env.scope(), false)
|
||||
.expect("ingest must succeed");
|
||||
assert_eq!(report.errors, 0, "no ingest errors; report={report:?}");
|
||||
|
||||
// Both twin paths must appear as New in the report.
|
||||
let items = report.items.as_ref().expect("items must be present");
|
||||
let twin_items: Vec<_> = items
|
||||
.iter()
|
||||
.filter(|i| {
|
||||
i.doc_path.0.ends_with("src_a/note.md")
|
||||
|| i.doc_path.0.ends_with("src_b/note.md")
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(
|
||||
twin_items.len(),
|
||||
2,
|
||||
"exactly 2 twin items expected; items={items:?}"
|
||||
);
|
||||
for item in &twin_items {
|
||||
assert_eq!(
|
||||
item.kind,
|
||||
IngestItemKind::New,
|
||||
"each twin must be New; item={item:?}"
|
||||
);
|
||||
}
|
||||
|
||||
// Resolve doc_ids for both workspace paths.
|
||||
// The ingest layer normalises workspace_path to the path relative to
|
||||
// workspace_root (e.g. "src_a/note.md"), so we look up by that form.
|
||||
let store = kebab_store_sqlite::SqliteStore::open(&env.config).unwrap();
|
||||
store.run_migrations().unwrap();
|
||||
|
||||
// Find the twin items by matching on suffix so the test is robust to
|
||||
// however the workspace root is represented.
|
||||
let items = report.items.as_ref().expect("items must be present");
|
||||
let path_a_str = items
|
||||
.iter()
|
||||
.find(|i| i.doc_path.0.ends_with("src_a/note.md"))
|
||||
.map(|i| i.doc_path.0.clone())
|
||||
.expect("src_a/note.md must appear in ingest report");
|
||||
let path_b_str = items
|
||||
.iter()
|
||||
.find(|i| i.doc_path.0.ends_with("src_b/note.md"))
|
||||
.map(|i| i.doc_path.0.clone())
|
||||
.expect("src_b/note.md must appear in ingest report");
|
||||
|
||||
let path_a = kebab_core::WorkspacePath(path_a_str);
|
||||
let path_b = kebab_core::WorkspacePath(path_b_str);
|
||||
|
||||
let doc_a = store
|
||||
.get_document_by_workspace_path(&path_a)
|
||||
.expect("get_document_by_workspace_path path_a")
|
||||
.expect("doc_a must exist after ingest");
|
||||
let doc_b = store
|
||||
.get_document_by_workspace_path(&path_b)
|
||||
.expect("get_document_by_workspace_path path_b")
|
||||
.expect("doc_b must exist after ingest");
|
||||
|
||||
// Both twins share one asset_id (same content hash).
|
||||
assert_eq!(
|
||||
doc_a.source_asset_id, doc_b.source_asset_id,
|
||||
"twin files must share one asset_id"
|
||||
);
|
||||
|
||||
// Open App and issue span fetch on each twin's doc_id.
|
||||
let app = env.app();
|
||||
|
||||
let result_a = app
|
||||
.fetch(
|
||||
FetchQuery::Span {
|
||||
doc_id: doc_a.doc_id.clone(),
|
||||
line_start: 1,
|
||||
line_end: 2,
|
||||
},
|
||||
FetchOpts::default(),
|
||||
)
|
||||
.expect("fetch_span on twin A must succeed for a markdown file");
|
||||
assert_eq!(result_a.kind, FetchKind::Span);
|
||||
assert!(
|
||||
result_a.text.as_deref().is_some_and(|t| !t.is_empty()),
|
||||
"span text for twin A must not be empty"
|
||||
);
|
||||
|
||||
let result_b = app
|
||||
.fetch(
|
||||
FetchQuery::Span {
|
||||
doc_id: doc_b.doc_id.clone(),
|
||||
line_start: 1,
|
||||
line_end: 2,
|
||||
},
|
||||
FetchOpts::default(),
|
||||
)
|
||||
.expect("fetch_span on twin B must succeed for a markdown file");
|
||||
assert_eq!(result_b.kind, FetchKind::Span);
|
||||
assert!(
|
||||
result_b.text.as_deref().is_some_and(|t| !t.is_empty()),
|
||||
"span text for twin B must not be empty"
|
||||
);
|
||||
|
||||
// Ingest again to force the asset.workspace_path flip-flop, then
|
||||
// re-check. Pre-fix this was the scenario that triggered the bug:
|
||||
// after the second ingest the asset row's workspace_path could point
|
||||
// at either twin, making one twin's span fetch behave incorrectly.
|
||||
let report2 = ingest_with_config(env.config.clone(), env.scope(), false)
|
||||
.expect("second ingest must succeed");
|
||||
assert_eq!(report2.errors, 0, "no ingest errors on second run; report={report2:?}");
|
||||
|
||||
// Re-open app after second ingest and verify span still works on both.
|
||||
let app2 = env.app();
|
||||
|
||||
app2.fetch(
|
||||
FetchQuery::Span {
|
||||
doc_id: doc_a.doc_id.clone(),
|
||||
line_start: 1,
|
||||
line_end: 3,
|
||||
},
|
||||
FetchOpts::default(),
|
||||
)
|
||||
.expect("fetch_span on twin A after flip-flop must still succeed");
|
||||
|
||||
app2.fetch(
|
||||
FetchQuery::Span {
|
||||
doc_id: doc_b.doc_id.clone(),
|
||||
line_start: 1,
|
||||
line_end: 3,
|
||||
},
|
||||
FetchOpts::default(),
|
||||
)
|
||||
.expect("fetch_span on twin B after flip-flop must still succeed");
|
||||
}
|
||||
90
crates/kebab-app/tests/twin_files_idempotent.rs
Normal file
90
crates/kebab-app/tests/twin_files_idempotent.rs
Normal file
@@ -0,0 +1,90 @@
|
||||
//! Regression test for the twin-file idempotency bug.
|
||||
//!
|
||||
//! Identical-content files at different workspace paths share one
|
||||
//! `assets` row (`asset_id` = blake3 content hash, PRIMARY KEY). The
|
||||
//! old UPSERT `ON CONFLICT(asset_id) DO UPDATE SET workspace_path =
|
||||
//! excluded.workspace_path` made each twin overwrite the other's path
|
||||
//! on every ingest, so `get_asset_by_workspace_path(path1)` returned
|
||||
//! None (or the wrong twin) → re-process every time.
|
||||
//!
|
||||
//! Fix: `try_skip_unchanged` now uses `get_document_by_workspace_path`
|
||||
//! instead. `documents.workspace_path` is UNIQUE (V001) so each twin
|
||||
//! has its own stable document row.
|
||||
//!
|
||||
//! Assertion contract:
|
||||
//! 1st ingest → 2 New (one per twin)
|
||||
//! 2nd ingest → 0 New, 0 Updated, 2 Unchanged
|
||||
|
||||
mod common;
|
||||
|
||||
use common::TestEnv;
|
||||
use kebab_app::ingest_with_config;
|
||||
use kebab_core::IngestItemKind;
|
||||
|
||||
#[test]
|
||||
fn twin_files_second_ingest_is_unchanged() {
|
||||
let env = TestEnv::lexical_only();
|
||||
|
||||
// Write two files with identical content at different paths.
|
||||
let pkg_a = env.workspace_root.join("pkg_a");
|
||||
let pkg_b = env.workspace_root.join("pkg_b");
|
||||
std::fs::create_dir_all(&pkg_a).unwrap();
|
||||
std::fs::create_dir_all(&pkg_b).unwrap();
|
||||
|
||||
let content = b"# shared\nThis content is identical in both files.\n";
|
||||
std::fs::write(pkg_a.join("__init__.py"), content).unwrap();
|
||||
std::fs::write(pkg_b.join("__init__.py"), content).unwrap();
|
||||
|
||||
// First ingest — both files must be New.
|
||||
let first = ingest_with_config(env.config.clone(), env.scope(), false)
|
||||
.expect("first ingest must succeed");
|
||||
assert_eq!(first.errors, 0, "first ingest: no errors; report={first:?}");
|
||||
|
||||
let items = first.items.as_ref().expect("items must be present");
|
||||
let twin_items: Vec<_> = items
|
||||
.iter()
|
||||
.filter(|i| {
|
||||
i.doc_path.0.ends_with("__init__.py")
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(
|
||||
twin_items.len(),
|
||||
2,
|
||||
"first ingest: expected exactly 2 __init__.py items; items={items:?}"
|
||||
);
|
||||
for item in &twin_items {
|
||||
assert_eq!(
|
||||
item.kind,
|
||||
IngestItemKind::New,
|
||||
"first ingest: each twin must be New; item={item:?}"
|
||||
);
|
||||
}
|
||||
|
||||
// Second ingest — same files, same content → both must be Unchanged.
|
||||
let second = ingest_with_config(env.config.clone(), env.scope(), false)
|
||||
.expect("second ingest must succeed");
|
||||
assert_eq!(second.errors, 0, "second ingest: no errors; report={second:?}");
|
||||
assert_eq!(second.new, 0, "second ingest: no new docs; report={second:?}");
|
||||
assert_eq!(
|
||||
second.updated, 0,
|
||||
"second ingest: no updated docs (twin-file bug would set this to 2); report={second:?}"
|
||||
);
|
||||
|
||||
let second_items = second.items.as_ref().expect("items must be present");
|
||||
let twin_items2: Vec<_> = second_items
|
||||
.iter()
|
||||
.filter(|i| i.doc_path.0.ends_with("__init__.py"))
|
||||
.collect();
|
||||
assert_eq!(
|
||||
twin_items2.len(),
|
||||
2,
|
||||
"second ingest: expected exactly 2 __init__.py items; items={second_items:?}"
|
||||
);
|
||||
for item in &twin_items2 {
|
||||
assert_eq!(
|
||||
item.kind,
|
||||
IngestItemKind::Unchanged,
|
||||
"second ingest: each twin must be Unchanged; item={item:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
322
crates/kebab-chunk/src/code_go_ast_v1.rs
Normal file
322
crates/kebab-chunk/src/code_go_ast_v1.rs
Normal file
@@ -0,0 +1,322 @@
|
||||
//! `code-go-ast-v1` — maps a tree-sitter-derived Go AST
|
||||
//! `CanonicalDocument` (one `Block::Code` per semantic unit, each with
|
||||
//! `SourceSpan::Code`) to chunks 1:1. A unit longer than
|
||||
//! `AST_CHUNK_MAX_LINES` is split into `<symbol> [part i/N]` sub-chunks
|
||||
//! at blank-line paragraph boundaries (design §9.1 oversize fallback).
|
||||
//!
|
||||
//! tree-sitter is intentionally NOT a dependency here: AST work is
|
||||
//! parser-side (`kebab-parse-code`, design §6.3). This chunker only
|
||||
//! consumes the `CanonicalDocument`.
|
||||
//!
|
||||
//! `AST_CHUNK_MAX_LINES` is a constant matching
|
||||
//! `IngestCodeCfg::default().ast_chunk_max_lines` (200). Per-medium
|
||||
//! config threading needs a chunker registry (P+); same deviation
|
||||
//! pattern as `pdf-page-v1`'s pinned `chunker_version`
|
||||
//! (`tasks/HOTFIXES.md`).
|
||||
|
||||
use kebab_core::{
|
||||
Block, BlockId, CanonicalDocument, Chunk, ChunkPolicy, Chunker, ChunkerVersion, DocumentId,
|
||||
SourceSpan, id_for_chunk,
|
||||
};
|
||||
|
||||
const VERSION_LABEL: &str = "code-go-ast-v1";
|
||||
const BYTES_PER_TOKEN: usize = 3;
|
||||
const POLICY_HASH_HEX_LEN: usize = 16;
|
||||
const AST_CHUNK_MAX_LINES: u32 = 200;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
pub struct CodeGoAstV1Chunker;
|
||||
|
||||
impl Chunker for CodeGoAstV1Chunker {
|
||||
fn chunker_version(&self) -> ChunkerVersion {
|
||||
ChunkerVersion(VERSION_LABEL.to_string())
|
||||
}
|
||||
|
||||
fn policy_hash(&self, policy: &ChunkPolicy) -> String {
|
||||
let bytes = serde_json_canonicalizer::to_vec(policy)
|
||||
.expect("canonical JSON serialization of ChunkPolicy must not fail");
|
||||
let hex = blake3::hash(&bytes).to_hex().to_string();
|
||||
hex[..POLICY_HASH_HEX_LEN].to_string()
|
||||
}
|
||||
|
||||
fn chunk(
|
||||
&self,
|
||||
doc: &CanonicalDocument,
|
||||
policy: &ChunkPolicy,
|
||||
) -> anyhow::Result<Vec<Chunk>> {
|
||||
for b in &doc.blocks {
|
||||
let c = match b {
|
||||
Block::Code(c) => c,
|
||||
_ => anyhow::bail!(
|
||||
"CodeGoAstV1Chunker only handles code docs (got non-Code block)"
|
||||
),
|
||||
};
|
||||
if !matches!(c.common.source_span, SourceSpan::Code { .. }) {
|
||||
anyhow::bail!(
|
||||
"CodeGoAstV1Chunker only handles code docs (got non-Code source_span)"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let base_policy_hash = self.policy_hash(policy);
|
||||
let chunker_version = self.chunker_version();
|
||||
let mut out: Vec<Chunk> = Vec::new();
|
||||
|
||||
for b in &doc.blocks {
|
||||
let cb = match b {
|
||||
Block::Code(c) => c,
|
||||
_ => unreachable!("validated above"),
|
||||
};
|
||||
let (ls, le, symbol, lang) = match &cb.common.source_span {
|
||||
SourceSpan::Code { line_start, line_end, symbol, lang } => {
|
||||
(*line_start, *line_end, symbol.clone(), lang.clone())
|
||||
}
|
||||
_ => unreachable!("validated above"),
|
||||
};
|
||||
let block_ids: Vec<BlockId> = vec![cb.common.block_id.clone()];
|
||||
let span_lines = le.saturating_sub(ls) + 1;
|
||||
|
||||
if span_lines <= AST_CHUNK_MAX_LINES {
|
||||
let span = SourceSpan::Code {
|
||||
line_start: ls,
|
||||
line_end: le,
|
||||
symbol: symbol.clone(),
|
||||
lang: lang.clone(),
|
||||
};
|
||||
out.push(make_chunk(
|
||||
doc, &chunker_version, &block_ids, &base_policy_hash,
|
||||
None, span, cb.code.clone(),
|
||||
));
|
||||
} else {
|
||||
let parts = split_oversize(&cb.code);
|
||||
let n = parts.len();
|
||||
for (i, (off_start, off_end, text)) in parts.into_iter().enumerate() {
|
||||
let part_ls = ls + off_start;
|
||||
let part_le = ls + off_end;
|
||||
let part_sym = symbol
|
||||
.as_ref()
|
||||
.map(|s| format!("{s} [part {}/{n}]", i + 1));
|
||||
let span = SourceSpan::Code {
|
||||
line_start: part_ls,
|
||||
line_end: part_le,
|
||||
symbol: part_sym,
|
||||
lang: lang.clone(),
|
||||
};
|
||||
out.push(make_chunk(
|
||||
doc, &chunker_version, &block_ids, &base_policy_hash,
|
||||
Some(part_ls), span, text,
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tracing::debug!(
|
||||
target: "kebab-chunk",
|
||||
doc_id = %doc.doc_id,
|
||||
chunks = out.len(),
|
||||
"code-go-ast-v1 chunked",
|
||||
);
|
||||
Ok(out)
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn make_chunk(
|
||||
doc: &CanonicalDocument,
|
||||
chunker_version: &ChunkerVersion,
|
||||
block_ids: &[BlockId],
|
||||
base_policy_hash: &str,
|
||||
split_key: Option<u32>,
|
||||
span: SourceSpan,
|
||||
text: String,
|
||||
) -> Chunk {
|
||||
let id_hash = match split_key {
|
||||
Some(k) => format!("{base_policy_hash}#L{k}"),
|
||||
None => base_policy_hash.to_string(),
|
||||
};
|
||||
let chunk_id = id_for_chunk(&doc.doc_id, chunker_version, block_ids, &id_hash);
|
||||
let token_estimate = text.len().div_ceil(BYTES_PER_TOKEN);
|
||||
Chunk {
|
||||
chunk_id,
|
||||
doc_id: DocumentId(doc.doc_id.0.clone()),
|
||||
block_ids: block_ids.to_vec(),
|
||||
text,
|
||||
heading_path: Vec::new(),
|
||||
source_spans: vec![span],
|
||||
token_estimate,
|
||||
chunker_version: chunker_version.clone(),
|
||||
policy_hash: base_policy_hash.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Split an oversize unit at blank-line paragraph boundaries, greedily
|
||||
/// gluing paragraphs until ~`AST_CHUNK_MAX_LINES` lines accumulate.
|
||||
/// Returns `(line_offset_start, line_offset_end, text)` where offsets are
|
||||
/// 0-based within the unit (caller adds the unit's absolute `line_start`).
|
||||
fn split_oversize(code: &str) -> Vec<(u32, u32, String)> {
|
||||
let lines: Vec<&str> = code.split('\n').collect();
|
||||
let total = lines.len() as u32;
|
||||
let mut out: Vec<(u32, u32, String)> = Vec::new();
|
||||
let mut start: u32 = 0;
|
||||
while start < total {
|
||||
let mut end = (start + AST_CHUNK_MAX_LINES).min(total);
|
||||
let floor = start + (AST_CHUNK_MAX_LINES * 4 / 5);
|
||||
if end < total {
|
||||
if let Some(b) = (floor.min(end)..end)
|
||||
.rev()
|
||||
.find(|&i| lines[i as usize].trim().is_empty())
|
||||
{
|
||||
end = b + 1;
|
||||
}
|
||||
}
|
||||
let text = lines[start as usize..end as usize].join("\n");
|
||||
out.push((start, end.saturating_sub(1), text));
|
||||
start = end;
|
||||
}
|
||||
if out.is_empty() {
|
||||
out.push((0, total.saturating_sub(1), code.to_string()));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use kebab_core::{
|
||||
Block, CanonicalDocument, ChunkPolicy, Chunker, ChunkerVersion, CodeBlock, CommonBlock,
|
||||
SourceSpan, id_for_block, id_for_doc, AssetId, Lang, Metadata, ParserVersion, Provenance,
|
||||
SourceType, TrustLevel, WorkspacePath,
|
||||
};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
fn code_doc(units: &[(&str, u32, u32, &str)]) -> CanonicalDocument {
|
||||
let wp = WorkspacePath("crates/x/src/a.go".into());
|
||||
let aid = AssetId("a".repeat(64));
|
||||
let pv = ParserVersion("code-go-v1".into());
|
||||
let doc_id = id_for_doc(&wp, &aid, &pv);
|
||||
let blocks = units
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, (sym, ls, le, code))| {
|
||||
let span = SourceSpan::Code {
|
||||
line_start: *ls,
|
||||
line_end: *le,
|
||||
symbol: Some((*sym).to_string()),
|
||||
lang: Some("go".into()),
|
||||
};
|
||||
let bid = id_for_block(&doc_id, "code", &[], i as u32, &span);
|
||||
Block::Code(CodeBlock {
|
||||
common: CommonBlock { block_id: bid, heading_path: vec![], source_span: span },
|
||||
lang: Some("go".into()),
|
||||
code: (*code).to_string(),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
CanonicalDocument {
|
||||
doc_id, source_asset_id: aid, workspace_path: wp, title: "a".into(),
|
||||
lang: Lang("und".into()), blocks,
|
||||
metadata: Metadata {
|
||||
aliases: vec![], tags: vec![],
|
||||
created_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
updated_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
source_type: SourceType::Note, trust_level: TrustLevel::Primary,
|
||||
user_id_alias: None, user: Default::default(),
|
||||
repo: Some("kebab".into()), git_branch: Some("main".into()),
|
||||
git_commit: Some("0".repeat(40)), code_lang: Some("go".into()),
|
||||
},
|
||||
provenance: Provenance { events: vec![] },
|
||||
parser_version: pv, schema_version: 1, doc_version: 1,
|
||||
last_chunker_version: None, last_embedding_version: None,
|
||||
}
|
||||
}
|
||||
fn policy() -> ChunkPolicy {
|
||||
ChunkPolicy { target_tokens: 500, overlap_tokens: 80,
|
||||
respect_markdown_headings: false,
|
||||
chunker_version: ChunkerVersion(VERSION_LABEL.into()) }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chunker_version_is_code_go_ast_v1() {
|
||||
assert_eq!(CodeGoAstV1Chunker.chunker_version(),
|
||||
ChunkerVersion("code-go-ast-v1".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn one_chunk_per_unit_preserves_code_span() {
|
||||
let doc = code_doc(&[
|
||||
("parse", 1, 3, "func parse() {\n\t// x\n}"),
|
||||
("Foo.double", 5, 7, "func double() int {\n\t//\n\treturn 0\n}"),
|
||||
]);
|
||||
let chunks = CodeGoAstV1Chunker.chunk(&doc, &policy()).unwrap();
|
||||
assert_eq!(chunks.len(), 2);
|
||||
for c in &chunks {
|
||||
assert_eq!(c.source_spans.len(), 1);
|
||||
assert!(matches!(c.source_spans[0], SourceSpan::Code { .. }));
|
||||
assert_eq!(c.heading_path, Vec::<String>::new());
|
||||
assert_eq!(c.chunker_version.0, "code-go-ast-v1");
|
||||
}
|
||||
match &chunks[0].source_spans[0] {
|
||||
SourceSpan::Code { symbol, line_start, line_end, .. } => {
|
||||
assert_eq!(symbol.as_deref(), Some("parse"));
|
||||
assert_eq!((*line_start, *line_end), (1, 3));
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oversize_unit_splits_into_parts_with_unique_ids() {
|
||||
let body = (0..500).map(|i| format!("\tx{i} := {i}")).collect::<Vec<_>>().join("\n");
|
||||
let code = format!("func big() {{\n{body}\n}}");
|
||||
let doc = code_doc(&[("big", 1, 502, &code)]);
|
||||
let chunks = CodeGoAstV1Chunker.chunk(&doc, &policy()).unwrap();
|
||||
assert!(chunks.len() >= 2, "oversize unit must split, got {}", chunks.len());
|
||||
for c in &chunks {
|
||||
match &c.source_spans[0] {
|
||||
SourceSpan::Code { symbol, .. } => {
|
||||
assert!(symbol.as_deref().unwrap().starts_with("big [part "),
|
||||
"part-numbered symbol, got {symbol:?}");
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
let mut ids: Vec<&str> = chunks.iter().map(|c| c.chunk_id.0.as_str()).collect();
|
||||
let n = ids.len(); ids.sort(); ids.dedup();
|
||||
assert_eq!(ids.len(), n, "chunk_ids unique across split parts");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_code_doc_errors() {
|
||||
use kebab_core::TextBlock;
|
||||
let mut doc = code_doc(&[("parse", 1, 1, "func parse() {}")]);
|
||||
doc.blocks = vec![Block::Paragraph(TextBlock {
|
||||
common: CommonBlock {
|
||||
block_id: kebab_core::BlockId("b".into()),
|
||||
heading_path: vec![],
|
||||
source_span: SourceSpan::Line { start: 1, end: 1 },
|
||||
},
|
||||
text: "x".into(), inlines: vec![],
|
||||
})];
|
||||
let err = CodeGoAstV1Chunker.chunk(&doc, &policy()).unwrap_err();
|
||||
assert!(err.to_string().contains("CodeGoAstV1Chunker"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deterministic_chunk_ids_1000() {
|
||||
let doc = code_doc(&[("parse", 1, 2, "func parse() {}\n")]);
|
||||
let base: Vec<String> = CodeGoAstV1Chunker.chunk(&doc, &policy())
|
||||
.unwrap().into_iter().map(|c| c.chunk_id.0).collect();
|
||||
for _ in 0..1000 {
|
||||
let again: Vec<String> = CodeGoAstV1Chunker.chunk(&doc, &policy())
|
||||
.unwrap().into_iter().map(|c| c.chunk_id.0).collect();
|
||||
assert_eq!(again, base);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn policy_hash_matches_md_heading_v1() {
|
||||
let p = policy();
|
||||
assert_eq!(CodeGoAstV1Chunker.policy_hash(&p),
|
||||
crate::MdHeadingV1Chunker.policy_hash(&p));
|
||||
}
|
||||
}
|
||||
322
crates/kebab-chunk/src/code_java_ast_v1.rs
Normal file
322
crates/kebab-chunk/src/code_java_ast_v1.rs
Normal file
@@ -0,0 +1,322 @@
|
||||
//! `code-java-ast-v1` — maps a tree-sitter-derived Java AST
|
||||
//! `CanonicalDocument` (one `Block::Code` per semantic unit, each with
|
||||
//! `SourceSpan::Code`) to chunks 1:1. A unit longer than
|
||||
//! `AST_CHUNK_MAX_LINES` is split into `<symbol> [part i/N]` sub-chunks
|
||||
//! at blank-line paragraph boundaries (design §9.1 oversize fallback).
|
||||
//!
|
||||
//! tree-sitter is intentionally NOT a dependency here: AST work is
|
||||
//! parser-side (`kebab-parse-code`, design §6.3). This chunker only
|
||||
//! consumes the `CanonicalDocument`.
|
||||
//!
|
||||
//! `AST_CHUNK_MAX_LINES` is a constant matching
|
||||
//! `IngestCodeCfg::default().ast_chunk_max_lines` (200). Per-medium
|
||||
//! config threading needs a chunker registry (P+); same deviation
|
||||
//! pattern as `pdf-page-v1`'s pinned `chunker_version`
|
||||
//! (`tasks/HOTFIXES.md`).
|
||||
|
||||
use kebab_core::{
|
||||
Block, BlockId, CanonicalDocument, Chunk, ChunkPolicy, Chunker, ChunkerVersion, DocumentId,
|
||||
SourceSpan, id_for_chunk,
|
||||
};
|
||||
|
||||
const VERSION_LABEL: &str = "code-java-ast-v1";
|
||||
const BYTES_PER_TOKEN: usize = 3;
|
||||
const POLICY_HASH_HEX_LEN: usize = 16;
|
||||
const AST_CHUNK_MAX_LINES: u32 = 200;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
pub struct CodeJavaAstV1Chunker;
|
||||
|
||||
impl Chunker for CodeJavaAstV1Chunker {
|
||||
fn chunker_version(&self) -> ChunkerVersion {
|
||||
ChunkerVersion(VERSION_LABEL.to_string())
|
||||
}
|
||||
|
||||
fn policy_hash(&self, policy: &ChunkPolicy) -> String {
|
||||
let bytes = serde_json_canonicalizer::to_vec(policy)
|
||||
.expect("canonical JSON serialization of ChunkPolicy must not fail");
|
||||
let hex = blake3::hash(&bytes).to_hex().to_string();
|
||||
hex[..POLICY_HASH_HEX_LEN].to_string()
|
||||
}
|
||||
|
||||
fn chunk(
|
||||
&self,
|
||||
doc: &CanonicalDocument,
|
||||
policy: &ChunkPolicy,
|
||||
) -> anyhow::Result<Vec<Chunk>> {
|
||||
for b in &doc.blocks {
|
||||
let c = match b {
|
||||
Block::Code(c) => c,
|
||||
_ => anyhow::bail!(
|
||||
"CodeJavaAstV1Chunker only handles code docs (got non-Code block)"
|
||||
),
|
||||
};
|
||||
if !matches!(c.common.source_span, SourceSpan::Code { .. }) {
|
||||
anyhow::bail!(
|
||||
"CodeJavaAstV1Chunker only handles code docs (got non-Code source_span)"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let base_policy_hash = self.policy_hash(policy);
|
||||
let chunker_version = self.chunker_version();
|
||||
let mut out: Vec<Chunk> = Vec::new();
|
||||
|
||||
for b in &doc.blocks {
|
||||
let cb = match b {
|
||||
Block::Code(c) => c,
|
||||
_ => unreachable!("validated above"),
|
||||
};
|
||||
let (ls, le, symbol, lang) = match &cb.common.source_span {
|
||||
SourceSpan::Code { line_start, line_end, symbol, lang } => {
|
||||
(*line_start, *line_end, symbol.clone(), lang.clone())
|
||||
}
|
||||
_ => unreachable!("validated above"),
|
||||
};
|
||||
let block_ids: Vec<BlockId> = vec![cb.common.block_id.clone()];
|
||||
let span_lines = le.saturating_sub(ls) + 1;
|
||||
|
||||
if span_lines <= AST_CHUNK_MAX_LINES {
|
||||
let span = SourceSpan::Code {
|
||||
line_start: ls,
|
||||
line_end: le,
|
||||
symbol: symbol.clone(),
|
||||
lang: lang.clone(),
|
||||
};
|
||||
out.push(make_chunk(
|
||||
doc, &chunker_version, &block_ids, &base_policy_hash,
|
||||
None, span, cb.code.clone(),
|
||||
));
|
||||
} else {
|
||||
let parts = split_oversize(&cb.code);
|
||||
let n = parts.len();
|
||||
for (i, (off_start, off_end, text)) in parts.into_iter().enumerate() {
|
||||
let part_ls = ls + off_start;
|
||||
let part_le = ls + off_end;
|
||||
let part_sym = symbol
|
||||
.as_ref()
|
||||
.map(|s| format!("{s} [part {}/{n}]", i + 1));
|
||||
let span = SourceSpan::Code {
|
||||
line_start: part_ls,
|
||||
line_end: part_le,
|
||||
symbol: part_sym,
|
||||
lang: lang.clone(),
|
||||
};
|
||||
out.push(make_chunk(
|
||||
doc, &chunker_version, &block_ids, &base_policy_hash,
|
||||
Some(part_ls), span, text,
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tracing::debug!(
|
||||
target: "kebab-chunk",
|
||||
doc_id = %doc.doc_id,
|
||||
chunks = out.len(),
|
||||
"code-java-ast-v1 chunked",
|
||||
);
|
||||
Ok(out)
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn make_chunk(
|
||||
doc: &CanonicalDocument,
|
||||
chunker_version: &ChunkerVersion,
|
||||
block_ids: &[BlockId],
|
||||
base_policy_hash: &str,
|
||||
split_key: Option<u32>,
|
||||
span: SourceSpan,
|
||||
text: String,
|
||||
) -> Chunk {
|
||||
let id_hash = match split_key {
|
||||
Some(k) => format!("{base_policy_hash}#L{k}"),
|
||||
None => base_policy_hash.to_string(),
|
||||
};
|
||||
let chunk_id = id_for_chunk(&doc.doc_id, chunker_version, block_ids, &id_hash);
|
||||
let token_estimate = text.len().div_ceil(BYTES_PER_TOKEN);
|
||||
Chunk {
|
||||
chunk_id,
|
||||
doc_id: DocumentId(doc.doc_id.0.clone()),
|
||||
block_ids: block_ids.to_vec(),
|
||||
text,
|
||||
heading_path: Vec::new(),
|
||||
source_spans: vec![span],
|
||||
token_estimate,
|
||||
chunker_version: chunker_version.clone(),
|
||||
policy_hash: base_policy_hash.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Split an oversize unit at blank-line paragraph boundaries, greedily
|
||||
/// gluing paragraphs until ~`AST_CHUNK_MAX_LINES` lines accumulate.
|
||||
/// Returns `(line_offset_start, line_offset_end, text)` where offsets are
|
||||
/// 0-based within the unit (caller adds the unit's absolute `line_start`).
|
||||
fn split_oversize(code: &str) -> Vec<(u32, u32, String)> {
|
||||
let lines: Vec<&str> = code.split('\n').collect();
|
||||
let total = lines.len() as u32;
|
||||
let mut out: Vec<(u32, u32, String)> = Vec::new();
|
||||
let mut start: u32 = 0;
|
||||
while start < total {
|
||||
let mut end = (start + AST_CHUNK_MAX_LINES).min(total);
|
||||
let floor = start + (AST_CHUNK_MAX_LINES * 4 / 5);
|
||||
if end < total {
|
||||
if let Some(b) = (floor.min(end)..end)
|
||||
.rev()
|
||||
.find(|&i| lines[i as usize].trim().is_empty())
|
||||
{
|
||||
end = b + 1;
|
||||
}
|
||||
}
|
||||
let text = lines[start as usize..end as usize].join("\n");
|
||||
out.push((start, end.saturating_sub(1), text));
|
||||
start = end;
|
||||
}
|
||||
if out.is_empty() {
|
||||
out.push((0, total.saturating_sub(1), code.to_string()));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use kebab_core::{
|
||||
Block, CanonicalDocument, ChunkPolicy, Chunker, ChunkerVersion, CodeBlock, CommonBlock,
|
||||
SourceSpan, id_for_block, id_for_doc, AssetId, Lang, Metadata, ParserVersion, Provenance,
|
||||
SourceType, TrustLevel, WorkspacePath,
|
||||
};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
fn code_doc(units: &[(&str, u32, u32, &str)]) -> CanonicalDocument {
|
||||
let wp = WorkspacePath("crates/x/src/Main.java".into());
|
||||
let aid = AssetId("a".repeat(64));
|
||||
let pv = ParserVersion("code-java-v1".into());
|
||||
let doc_id = id_for_doc(&wp, &aid, &pv);
|
||||
let blocks = units
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, (sym, ls, le, code))| {
|
||||
let span = SourceSpan::Code {
|
||||
line_start: *ls,
|
||||
line_end: *le,
|
||||
symbol: Some((*sym).to_string()),
|
||||
lang: Some("java".into()),
|
||||
};
|
||||
let bid = id_for_block(&doc_id, "code", &[], i as u32, &span);
|
||||
Block::Code(CodeBlock {
|
||||
common: CommonBlock { block_id: bid, heading_path: vec![], source_span: span },
|
||||
lang: Some("java".into()),
|
||||
code: (*code).to_string(),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
CanonicalDocument {
|
||||
doc_id, source_asset_id: aid, workspace_path: wp, title: "a".into(),
|
||||
lang: Lang("und".into()), blocks,
|
||||
metadata: Metadata {
|
||||
aliases: vec![], tags: vec![],
|
||||
created_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
updated_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
source_type: SourceType::Note, trust_level: TrustLevel::Primary,
|
||||
user_id_alias: None, user: Default::default(),
|
||||
repo: Some("kebab".into()), git_branch: Some("main".into()),
|
||||
git_commit: Some("0".repeat(40)), code_lang: Some("java".into()),
|
||||
},
|
||||
provenance: Provenance { events: vec![] },
|
||||
parser_version: pv, schema_version: 1, doc_version: 1,
|
||||
last_chunker_version: None, last_embedding_version: None,
|
||||
}
|
||||
}
|
||||
fn policy() -> ChunkPolicy {
|
||||
ChunkPolicy { target_tokens: 500, overlap_tokens: 80,
|
||||
respect_markdown_headings: false,
|
||||
chunker_version: ChunkerVersion(VERSION_LABEL.into()) }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chunker_version_is_code_java_ast_v1() {
|
||||
assert_eq!(CodeJavaAstV1Chunker.chunker_version(),
|
||||
ChunkerVersion("code-java-ast-v1".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn one_chunk_per_unit_preserves_code_span() {
|
||||
let doc = code_doc(&[
|
||||
("parse", 1, 3, "void parse() {\n\t// x\n}"),
|
||||
("Foo.double", 5, 7, "int double() {\n\t//\n\treturn 0;\n}"),
|
||||
]);
|
||||
let chunks = CodeJavaAstV1Chunker.chunk(&doc, &policy()).unwrap();
|
||||
assert_eq!(chunks.len(), 2);
|
||||
for c in &chunks {
|
||||
assert_eq!(c.source_spans.len(), 1);
|
||||
assert!(matches!(c.source_spans[0], SourceSpan::Code { .. }));
|
||||
assert_eq!(c.heading_path, Vec::<String>::new());
|
||||
assert_eq!(c.chunker_version.0, "code-java-ast-v1");
|
||||
}
|
||||
match &chunks[0].source_spans[0] {
|
||||
SourceSpan::Code { symbol, line_start, line_end, .. } => {
|
||||
assert_eq!(symbol.as_deref(), Some("parse"));
|
||||
assert_eq!((*line_start, *line_end), (1, 3));
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oversize_unit_splits_into_parts_with_unique_ids() {
|
||||
let body = (0..500).map(|i| format!("\tint x{i} = {i};")).collect::<Vec<_>>().join("\n");
|
||||
let code = format!("void big() {{\n{body}\n}}");
|
||||
let doc = code_doc(&[("big", 1, 502, &code)]);
|
||||
let chunks = CodeJavaAstV1Chunker.chunk(&doc, &policy()).unwrap();
|
||||
assert!(chunks.len() >= 2, "oversize unit must split, got {}", chunks.len());
|
||||
for c in &chunks {
|
||||
match &c.source_spans[0] {
|
||||
SourceSpan::Code { symbol, .. } => {
|
||||
assert!(symbol.as_deref().unwrap().starts_with("big [part "),
|
||||
"part-numbered symbol, got {symbol:?}");
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
let mut ids: Vec<&str> = chunks.iter().map(|c| c.chunk_id.0.as_str()).collect();
|
||||
let n = ids.len(); ids.sort(); ids.dedup();
|
||||
assert_eq!(ids.len(), n, "chunk_ids unique across split parts");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_code_doc_errors() {
|
||||
use kebab_core::TextBlock;
|
||||
let mut doc = code_doc(&[("parse", 1, 1, "void parse() {}")]);
|
||||
doc.blocks = vec![Block::Paragraph(TextBlock {
|
||||
common: CommonBlock {
|
||||
block_id: kebab_core::BlockId("b".into()),
|
||||
heading_path: vec![],
|
||||
source_span: SourceSpan::Line { start: 1, end: 1 },
|
||||
},
|
||||
text: "x".into(), inlines: vec![],
|
||||
})];
|
||||
let err = CodeJavaAstV1Chunker.chunk(&doc, &policy()).unwrap_err();
|
||||
assert!(err.to_string().contains("CodeJavaAstV1Chunker"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deterministic_chunk_ids_1000() {
|
||||
let doc = code_doc(&[("parse", 1, 2, "void parse() {}\n")]);
|
||||
let base: Vec<String> = CodeJavaAstV1Chunker.chunk(&doc, &policy())
|
||||
.unwrap().into_iter().map(|c| c.chunk_id.0).collect();
|
||||
for _ in 0..1000 {
|
||||
let again: Vec<String> = CodeJavaAstV1Chunker.chunk(&doc, &policy())
|
||||
.unwrap().into_iter().map(|c| c.chunk_id.0).collect();
|
||||
assert_eq!(again, base);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn policy_hash_matches_md_heading_v1() {
|
||||
let p = policy();
|
||||
assert_eq!(CodeJavaAstV1Chunker.policy_hash(&p),
|
||||
crate::MdHeadingV1Chunker.policy_hash(&p));
|
||||
}
|
||||
}
|
||||
322
crates/kebab-chunk/src/code_js_ast_v1.rs
Normal file
322
crates/kebab-chunk/src/code_js_ast_v1.rs
Normal file
@@ -0,0 +1,322 @@
|
||||
//! `code-js-ast-v1` — maps a tree-sitter-derived JavaScript AST
|
||||
//! `CanonicalDocument` (one `Block::Code` per semantic unit, each with
|
||||
//! `SourceSpan::Code`) to chunks 1:1. A unit longer than
|
||||
//! `AST_CHUNK_MAX_LINES` is split into `<symbol> [part i/N]` sub-chunks
|
||||
//! at blank-line paragraph boundaries (design §9.1 oversize fallback).
|
||||
//!
|
||||
//! tree-sitter is intentionally NOT a dependency here: AST work is
|
||||
//! parser-side (`kebab-parse-code`, design §6.3). This chunker only
|
||||
//! consumes the `CanonicalDocument`.
|
||||
//!
|
||||
//! `AST_CHUNK_MAX_LINES` is a constant matching
|
||||
//! `IngestCodeCfg::default().ast_chunk_max_lines` (200). Per-medium
|
||||
//! config threading needs a chunker registry (P+); same deviation
|
||||
//! pattern as `pdf-page-v1`'s pinned `chunker_version`
|
||||
//! (`tasks/HOTFIXES.md`).
|
||||
|
||||
use kebab_core::{
|
||||
Block, BlockId, CanonicalDocument, Chunk, ChunkPolicy, Chunker, ChunkerVersion, DocumentId,
|
||||
SourceSpan, id_for_chunk,
|
||||
};
|
||||
|
||||
const VERSION_LABEL: &str = "code-js-ast-v1";
|
||||
const BYTES_PER_TOKEN: usize = 3;
|
||||
const POLICY_HASH_HEX_LEN: usize = 16;
|
||||
const AST_CHUNK_MAX_LINES: u32 = 200;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
pub struct CodeJsAstV1Chunker;
|
||||
|
||||
impl Chunker for CodeJsAstV1Chunker {
|
||||
fn chunker_version(&self) -> ChunkerVersion {
|
||||
ChunkerVersion(VERSION_LABEL.to_string())
|
||||
}
|
||||
|
||||
fn policy_hash(&self, policy: &ChunkPolicy) -> String {
|
||||
let bytes = serde_json_canonicalizer::to_vec(policy)
|
||||
.expect("canonical JSON serialization of ChunkPolicy must not fail");
|
||||
let hex = blake3::hash(&bytes).to_hex().to_string();
|
||||
hex[..POLICY_HASH_HEX_LEN].to_string()
|
||||
}
|
||||
|
||||
fn chunk(
|
||||
&self,
|
||||
doc: &CanonicalDocument,
|
||||
policy: &ChunkPolicy,
|
||||
) -> anyhow::Result<Vec<Chunk>> {
|
||||
for b in &doc.blocks {
|
||||
let c = match b {
|
||||
Block::Code(c) => c,
|
||||
_ => anyhow::bail!(
|
||||
"CodeJsAstV1Chunker only handles code docs (got non-Code block)"
|
||||
),
|
||||
};
|
||||
if !matches!(c.common.source_span, SourceSpan::Code { .. }) {
|
||||
anyhow::bail!(
|
||||
"CodeJsAstV1Chunker only handles code docs (got non-Code source_span)"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let base_policy_hash = self.policy_hash(policy);
|
||||
let chunker_version = self.chunker_version();
|
||||
let mut out: Vec<Chunk> = Vec::new();
|
||||
|
||||
for b in &doc.blocks {
|
||||
let cb = match b {
|
||||
Block::Code(c) => c,
|
||||
_ => unreachable!("validated above"),
|
||||
};
|
||||
let (ls, le, symbol, lang) = match &cb.common.source_span {
|
||||
SourceSpan::Code { line_start, line_end, symbol, lang } => {
|
||||
(*line_start, *line_end, symbol.clone(), lang.clone())
|
||||
}
|
||||
_ => unreachable!("validated above"),
|
||||
};
|
||||
let block_ids: Vec<BlockId> = vec![cb.common.block_id.clone()];
|
||||
let span_lines = le.saturating_sub(ls) + 1;
|
||||
|
||||
if span_lines <= AST_CHUNK_MAX_LINES {
|
||||
let span = SourceSpan::Code {
|
||||
line_start: ls,
|
||||
line_end: le,
|
||||
symbol: symbol.clone(),
|
||||
lang: lang.clone(),
|
||||
};
|
||||
out.push(make_chunk(
|
||||
doc, &chunker_version, &block_ids, &base_policy_hash,
|
||||
None, span, cb.code.clone(),
|
||||
));
|
||||
} else {
|
||||
let parts = split_oversize(&cb.code);
|
||||
let n = parts.len();
|
||||
for (i, (off_start, off_end, text)) in parts.into_iter().enumerate() {
|
||||
let part_ls = ls + off_start;
|
||||
let part_le = ls + off_end;
|
||||
let part_sym = symbol
|
||||
.as_ref()
|
||||
.map(|s| format!("{s} [part {}/{n}]", i + 1));
|
||||
let span = SourceSpan::Code {
|
||||
line_start: part_ls,
|
||||
line_end: part_le,
|
||||
symbol: part_sym,
|
||||
lang: lang.clone(),
|
||||
};
|
||||
out.push(make_chunk(
|
||||
doc, &chunker_version, &block_ids, &base_policy_hash,
|
||||
Some(part_ls), span, text,
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tracing::debug!(
|
||||
target: "kebab-chunk",
|
||||
doc_id = %doc.doc_id,
|
||||
chunks = out.len(),
|
||||
"code-js-ast-v1 chunked",
|
||||
);
|
||||
Ok(out)
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn make_chunk(
|
||||
doc: &CanonicalDocument,
|
||||
chunker_version: &ChunkerVersion,
|
||||
block_ids: &[BlockId],
|
||||
base_policy_hash: &str,
|
||||
split_key: Option<u32>,
|
||||
span: SourceSpan,
|
||||
text: String,
|
||||
) -> Chunk {
|
||||
let id_hash = match split_key {
|
||||
Some(k) => format!("{base_policy_hash}#L{k}"),
|
||||
None => base_policy_hash.to_string(),
|
||||
};
|
||||
let chunk_id = id_for_chunk(&doc.doc_id, chunker_version, block_ids, &id_hash);
|
||||
let token_estimate = text.len().div_ceil(BYTES_PER_TOKEN);
|
||||
Chunk {
|
||||
chunk_id,
|
||||
doc_id: DocumentId(doc.doc_id.0.clone()),
|
||||
block_ids: block_ids.to_vec(),
|
||||
text,
|
||||
heading_path: Vec::new(),
|
||||
source_spans: vec![span],
|
||||
token_estimate,
|
||||
chunker_version: chunker_version.clone(),
|
||||
policy_hash: base_policy_hash.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Split an oversize unit at blank-line paragraph boundaries, greedily
|
||||
/// gluing paragraphs until ~`AST_CHUNK_MAX_LINES` lines accumulate.
|
||||
/// Returns `(line_offset_start, line_offset_end, text)` where offsets are
|
||||
/// 0-based within the unit (caller adds the unit's absolute `line_start`).
|
||||
fn split_oversize(code: &str) -> Vec<(u32, u32, String)> {
|
||||
let lines: Vec<&str> = code.split('\n').collect();
|
||||
let total = lines.len() as u32;
|
||||
let mut out: Vec<(u32, u32, String)> = Vec::new();
|
||||
let mut start: u32 = 0;
|
||||
while start < total {
|
||||
let mut end = (start + AST_CHUNK_MAX_LINES).min(total);
|
||||
let floor = start + (AST_CHUNK_MAX_LINES * 4 / 5);
|
||||
if end < total {
|
||||
if let Some(b) = (floor.min(end)..end)
|
||||
.rev()
|
||||
.find(|&i| lines[i as usize].trim().is_empty())
|
||||
{
|
||||
end = b + 1;
|
||||
}
|
||||
}
|
||||
let text = lines[start as usize..end as usize].join("\n");
|
||||
out.push((start, end.saturating_sub(1), text));
|
||||
start = end;
|
||||
}
|
||||
if out.is_empty() {
|
||||
out.push((0, total.saturating_sub(1), code.to_string()));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use kebab_core::{
|
||||
Block, CanonicalDocument, ChunkPolicy, Chunker, ChunkerVersion, CodeBlock, CommonBlock,
|
||||
SourceSpan, id_for_block, id_for_doc, AssetId, Lang, Metadata, ParserVersion, Provenance,
|
||||
SourceType, TrustLevel, WorkspacePath,
|
||||
};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
fn code_doc(units: &[(&str, u32, u32, &str)]) -> CanonicalDocument {
|
||||
let wp = WorkspacePath("crates/x/src/a.js".into());
|
||||
let aid = AssetId("a".repeat(64));
|
||||
let pv = ParserVersion("code-js-v1".into());
|
||||
let doc_id = id_for_doc(&wp, &aid, &pv);
|
||||
let blocks = units
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, (sym, ls, le, code))| {
|
||||
let span = SourceSpan::Code {
|
||||
line_start: *ls,
|
||||
line_end: *le,
|
||||
symbol: Some((*sym).to_string()),
|
||||
lang: Some("javascript".into()),
|
||||
};
|
||||
let bid = id_for_block(&doc_id, "code", &[], i as u32, &span);
|
||||
Block::Code(CodeBlock {
|
||||
common: CommonBlock { block_id: bid, heading_path: vec![], source_span: span },
|
||||
lang: Some("javascript".into()),
|
||||
code: (*code).to_string(),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
CanonicalDocument {
|
||||
doc_id, source_asset_id: aid, workspace_path: wp, title: "a".into(),
|
||||
lang: Lang("und".into()), blocks,
|
||||
metadata: Metadata {
|
||||
aliases: vec![], tags: vec![],
|
||||
created_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
updated_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
source_type: SourceType::Note, trust_level: TrustLevel::Primary,
|
||||
user_id_alias: None, user: Default::default(),
|
||||
repo: Some("kebab".into()), git_branch: Some("main".into()),
|
||||
git_commit: Some("0".repeat(40)), code_lang: Some("javascript".into()),
|
||||
},
|
||||
provenance: Provenance { events: vec![] },
|
||||
parser_version: pv, schema_version: 1, doc_version: 1,
|
||||
last_chunker_version: None, last_embedding_version: None,
|
||||
}
|
||||
}
|
||||
fn policy() -> ChunkPolicy {
|
||||
ChunkPolicy { target_tokens: 500, overlap_tokens: 80,
|
||||
respect_markdown_headings: false,
|
||||
chunker_version: ChunkerVersion(VERSION_LABEL.into()) }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chunker_version_is_code_js_ast_v1() {
|
||||
assert_eq!(CodeJsAstV1Chunker.chunker_version(),
|
||||
ChunkerVersion("code-js-ast-v1".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn one_chunk_per_unit_preserves_code_span() {
|
||||
let doc = code_doc(&[
|
||||
("parse", 1, 3, "function parse() {\n // x\n}"),
|
||||
("Foo.double", 5, 7, "function double() {\n //\n return 0;\n}"),
|
||||
]);
|
||||
let chunks = CodeJsAstV1Chunker.chunk(&doc, &policy()).unwrap();
|
||||
assert_eq!(chunks.len(), 2);
|
||||
for c in &chunks {
|
||||
assert_eq!(c.source_spans.len(), 1);
|
||||
assert!(matches!(c.source_spans[0], SourceSpan::Code { .. }));
|
||||
assert_eq!(c.heading_path, Vec::<String>::new());
|
||||
assert_eq!(c.chunker_version.0, "code-js-ast-v1");
|
||||
}
|
||||
match &chunks[0].source_spans[0] {
|
||||
SourceSpan::Code { symbol, line_start, line_end, .. } => {
|
||||
assert_eq!(symbol.as_deref(), Some("parse"));
|
||||
assert_eq!((*line_start, *line_end), (1, 3));
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oversize_unit_splits_into_parts_with_unique_ids() {
|
||||
let body = (0..500).map(|i| format!(" const x{i} = {i};")).collect::<Vec<_>>().join("\n");
|
||||
let code = format!("function big() {{\n{body}\n}}");
|
||||
let doc = code_doc(&[("big", 1, 502, &code)]);
|
||||
let chunks = CodeJsAstV1Chunker.chunk(&doc, &policy()).unwrap();
|
||||
assert!(chunks.len() >= 2, "oversize unit must split, got {}", chunks.len());
|
||||
for c in &chunks {
|
||||
match &c.source_spans[0] {
|
||||
SourceSpan::Code { symbol, .. } => {
|
||||
assert!(symbol.as_deref().unwrap().starts_with("big [part "),
|
||||
"part-numbered symbol, got {symbol:?}");
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
let mut ids: Vec<&str> = chunks.iter().map(|c| c.chunk_id.0.as_str()).collect();
|
||||
let n = ids.len(); ids.sort(); ids.dedup();
|
||||
assert_eq!(ids.len(), n, "chunk_ids unique across split parts");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_code_doc_errors() {
|
||||
use kebab_core::TextBlock;
|
||||
let mut doc = code_doc(&[("parse", 1, 1, "function parse() {}")]);
|
||||
doc.blocks = vec![Block::Paragraph(TextBlock {
|
||||
common: CommonBlock {
|
||||
block_id: kebab_core::BlockId("b".into()),
|
||||
heading_path: vec![],
|
||||
source_span: SourceSpan::Line { start: 1, end: 1 },
|
||||
},
|
||||
text: "x".into(), inlines: vec![],
|
||||
})];
|
||||
let err = CodeJsAstV1Chunker.chunk(&doc, &policy()).unwrap_err();
|
||||
assert!(err.to_string().contains("CodeJsAstV1Chunker"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deterministic_chunk_ids_1000() {
|
||||
let doc = code_doc(&[("parse", 1, 2, "function parse() {}\n")]);
|
||||
let base: Vec<String> = CodeJsAstV1Chunker.chunk(&doc, &policy())
|
||||
.unwrap().into_iter().map(|c| c.chunk_id.0).collect();
|
||||
for _ in 0..1000 {
|
||||
let again: Vec<String> = CodeJsAstV1Chunker.chunk(&doc, &policy())
|
||||
.unwrap().into_iter().map(|c| c.chunk_id.0).collect();
|
||||
assert_eq!(again, base);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn policy_hash_matches_md_heading_v1() {
|
||||
let p = policy();
|
||||
assert_eq!(CodeJsAstV1Chunker.policy_hash(&p),
|
||||
crate::MdHeadingV1Chunker.policy_hash(&p));
|
||||
}
|
||||
}
|
||||
322
crates/kebab-chunk/src/code_kotlin_ast_v1.rs
Normal file
322
crates/kebab-chunk/src/code_kotlin_ast_v1.rs
Normal file
@@ -0,0 +1,322 @@
|
||||
//! `code-kotlin-ast-v1` — maps a tree-sitter-derived Kotlin AST
|
||||
//! `CanonicalDocument` (one `Block::Code` per semantic unit, each with
|
||||
//! `SourceSpan::Code`) to chunks 1:1. A unit longer than
|
||||
//! `AST_CHUNK_MAX_LINES` is split into `<symbol> [part i/N]` sub-chunks
|
||||
//! at blank-line paragraph boundaries (design §9.1 oversize fallback).
|
||||
//!
|
||||
//! tree-sitter is intentionally NOT a dependency here: AST work is
|
||||
//! parser-side (`kebab-parse-code`, design §6.3). This chunker only
|
||||
//! consumes the `CanonicalDocument`.
|
||||
//!
|
||||
//! `AST_CHUNK_MAX_LINES` is a constant matching
|
||||
//! `IngestCodeCfg::default().ast_chunk_max_lines` (200). Per-medium
|
||||
//! config threading needs a chunker registry (P+); same deviation
|
||||
//! pattern as `pdf-page-v1`'s pinned `chunker_version`
|
||||
//! (`tasks/HOTFIXES.md`).
|
||||
|
||||
use kebab_core::{
|
||||
Block, BlockId, CanonicalDocument, Chunk, ChunkPolicy, Chunker, ChunkerVersion, DocumentId,
|
||||
SourceSpan, id_for_chunk,
|
||||
};
|
||||
|
||||
const VERSION_LABEL: &str = "code-kotlin-ast-v1";
|
||||
const BYTES_PER_TOKEN: usize = 3;
|
||||
const POLICY_HASH_HEX_LEN: usize = 16;
|
||||
const AST_CHUNK_MAX_LINES: u32 = 200;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
pub struct CodeKotlinAstV1Chunker;
|
||||
|
||||
impl Chunker for CodeKotlinAstV1Chunker {
|
||||
fn chunker_version(&self) -> ChunkerVersion {
|
||||
ChunkerVersion(VERSION_LABEL.to_string())
|
||||
}
|
||||
|
||||
fn policy_hash(&self, policy: &ChunkPolicy) -> String {
|
||||
let bytes = serde_json_canonicalizer::to_vec(policy)
|
||||
.expect("canonical JSON serialization of ChunkPolicy must not fail");
|
||||
let hex = blake3::hash(&bytes).to_hex().to_string();
|
||||
hex[..POLICY_HASH_HEX_LEN].to_string()
|
||||
}
|
||||
|
||||
fn chunk(
|
||||
&self,
|
||||
doc: &CanonicalDocument,
|
||||
policy: &ChunkPolicy,
|
||||
) -> anyhow::Result<Vec<Chunk>> {
|
||||
for b in &doc.blocks {
|
||||
let c = match b {
|
||||
Block::Code(c) => c,
|
||||
_ => anyhow::bail!(
|
||||
"CodeKotlinAstV1Chunker only handles code docs (got non-Code block)"
|
||||
),
|
||||
};
|
||||
if !matches!(c.common.source_span, SourceSpan::Code { .. }) {
|
||||
anyhow::bail!(
|
||||
"CodeKotlinAstV1Chunker only handles code docs (got non-Code source_span)"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let base_policy_hash = self.policy_hash(policy);
|
||||
let chunker_version = self.chunker_version();
|
||||
let mut out: Vec<Chunk> = Vec::new();
|
||||
|
||||
for b in &doc.blocks {
|
||||
let cb = match b {
|
||||
Block::Code(c) => c,
|
||||
_ => unreachable!("validated above"),
|
||||
};
|
||||
let (ls, le, symbol, lang) = match &cb.common.source_span {
|
||||
SourceSpan::Code { line_start, line_end, symbol, lang } => {
|
||||
(*line_start, *line_end, symbol.clone(), lang.clone())
|
||||
}
|
||||
_ => unreachable!("validated above"),
|
||||
};
|
||||
let block_ids: Vec<BlockId> = vec![cb.common.block_id.clone()];
|
||||
let span_lines = le.saturating_sub(ls) + 1;
|
||||
|
||||
if span_lines <= AST_CHUNK_MAX_LINES {
|
||||
let span = SourceSpan::Code {
|
||||
line_start: ls,
|
||||
line_end: le,
|
||||
symbol: symbol.clone(),
|
||||
lang: lang.clone(),
|
||||
};
|
||||
out.push(make_chunk(
|
||||
doc, &chunker_version, &block_ids, &base_policy_hash,
|
||||
None, span, cb.code.clone(),
|
||||
));
|
||||
} else {
|
||||
let parts = split_oversize(&cb.code);
|
||||
let n = parts.len();
|
||||
for (i, (off_start, off_end, text)) in parts.into_iter().enumerate() {
|
||||
let part_ls = ls + off_start;
|
||||
let part_le = ls + off_end;
|
||||
let part_sym = symbol
|
||||
.as_ref()
|
||||
.map(|s| format!("{s} [part {}/{n}]", i + 1));
|
||||
let span = SourceSpan::Code {
|
||||
line_start: part_ls,
|
||||
line_end: part_le,
|
||||
symbol: part_sym,
|
||||
lang: lang.clone(),
|
||||
};
|
||||
out.push(make_chunk(
|
||||
doc, &chunker_version, &block_ids, &base_policy_hash,
|
||||
Some(part_ls), span, text,
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tracing::debug!(
|
||||
target: "kebab-chunk",
|
||||
doc_id = %doc.doc_id,
|
||||
chunks = out.len(),
|
||||
"code-kotlin-ast-v1 chunked",
|
||||
);
|
||||
Ok(out)
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn make_chunk(
|
||||
doc: &CanonicalDocument,
|
||||
chunker_version: &ChunkerVersion,
|
||||
block_ids: &[BlockId],
|
||||
base_policy_hash: &str,
|
||||
split_key: Option<u32>,
|
||||
span: SourceSpan,
|
||||
text: String,
|
||||
) -> Chunk {
|
||||
let id_hash = match split_key {
|
||||
Some(k) => format!("{base_policy_hash}#L{k}"),
|
||||
None => base_policy_hash.to_string(),
|
||||
};
|
||||
let chunk_id = id_for_chunk(&doc.doc_id, chunker_version, block_ids, &id_hash);
|
||||
let token_estimate = text.len().div_ceil(BYTES_PER_TOKEN);
|
||||
Chunk {
|
||||
chunk_id,
|
||||
doc_id: DocumentId(doc.doc_id.0.clone()),
|
||||
block_ids: block_ids.to_vec(),
|
||||
text,
|
||||
heading_path: Vec::new(),
|
||||
source_spans: vec![span],
|
||||
token_estimate,
|
||||
chunker_version: chunker_version.clone(),
|
||||
policy_hash: base_policy_hash.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Split an oversize unit at blank-line paragraph boundaries, greedily
|
||||
/// gluing paragraphs until ~`AST_CHUNK_MAX_LINES` lines accumulate.
|
||||
/// Returns `(line_offset_start, line_offset_end, text)` where offsets are
|
||||
/// 0-based within the unit (caller adds the unit's absolute `line_start`).
|
||||
fn split_oversize(code: &str) -> Vec<(u32, u32, String)> {
|
||||
let lines: Vec<&str> = code.split('\n').collect();
|
||||
let total = lines.len() as u32;
|
||||
let mut out: Vec<(u32, u32, String)> = Vec::new();
|
||||
let mut start: u32 = 0;
|
||||
while start < total {
|
||||
let mut end = (start + AST_CHUNK_MAX_LINES).min(total);
|
||||
let floor = start + (AST_CHUNK_MAX_LINES * 4 / 5);
|
||||
if end < total {
|
||||
if let Some(b) = (floor.min(end)..end)
|
||||
.rev()
|
||||
.find(|&i| lines[i as usize].trim().is_empty())
|
||||
{
|
||||
end = b + 1;
|
||||
}
|
||||
}
|
||||
let text = lines[start as usize..end as usize].join("\n");
|
||||
out.push((start, end.saturating_sub(1), text));
|
||||
start = end;
|
||||
}
|
||||
if out.is_empty() {
|
||||
out.push((0, total.saturating_sub(1), code.to_string()));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use kebab_core::{
|
||||
Block, CanonicalDocument, ChunkPolicy, Chunker, ChunkerVersion, CodeBlock, CommonBlock,
|
||||
SourceSpan, id_for_block, id_for_doc, AssetId, Lang, Metadata, ParserVersion, Provenance,
|
||||
SourceType, TrustLevel, WorkspacePath,
|
||||
};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
fn code_doc(units: &[(&str, u32, u32, &str)]) -> CanonicalDocument {
|
||||
let wp = WorkspacePath("crates/x/src/Main.kt".into());
|
||||
let aid = AssetId("a".repeat(64));
|
||||
let pv = ParserVersion("code-kotlin-v1".into());
|
||||
let doc_id = id_for_doc(&wp, &aid, &pv);
|
||||
let blocks = units
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, (sym, ls, le, code))| {
|
||||
let span = SourceSpan::Code {
|
||||
line_start: *ls,
|
||||
line_end: *le,
|
||||
symbol: Some((*sym).to_string()),
|
||||
lang: Some("kotlin".into()),
|
||||
};
|
||||
let bid = id_for_block(&doc_id, "code", &[], i as u32, &span);
|
||||
Block::Code(CodeBlock {
|
||||
common: CommonBlock { block_id: bid, heading_path: vec![], source_span: span },
|
||||
lang: Some("kotlin".into()),
|
||||
code: (*code).to_string(),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
CanonicalDocument {
|
||||
doc_id, source_asset_id: aid, workspace_path: wp, title: "a".into(),
|
||||
lang: Lang("und".into()), blocks,
|
||||
metadata: Metadata {
|
||||
aliases: vec![], tags: vec![],
|
||||
created_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
updated_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
source_type: SourceType::Note, trust_level: TrustLevel::Primary,
|
||||
user_id_alias: None, user: Default::default(),
|
||||
repo: Some("kebab".into()), git_branch: Some("main".into()),
|
||||
git_commit: Some("0".repeat(40)), code_lang: Some("kotlin".into()),
|
||||
},
|
||||
provenance: Provenance { events: vec![] },
|
||||
parser_version: pv, schema_version: 1, doc_version: 1,
|
||||
last_chunker_version: None, last_embedding_version: None,
|
||||
}
|
||||
}
|
||||
fn policy() -> ChunkPolicy {
|
||||
ChunkPolicy { target_tokens: 500, overlap_tokens: 80,
|
||||
respect_markdown_headings: false,
|
||||
chunker_version: ChunkerVersion(VERSION_LABEL.into()) }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chunker_version_is_code_kotlin_ast_v1() {
|
||||
assert_eq!(CodeKotlinAstV1Chunker.chunker_version(),
|
||||
ChunkerVersion("code-kotlin-ast-v1".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn one_chunk_per_unit_preserves_code_span() {
|
||||
let doc = code_doc(&[
|
||||
("parse", 1, 3, "fun parse() {\n\t// x\n}"),
|
||||
("Foo.double", 5, 7, "fun double(): Int {\n\t//\n\treturn 0\n}"),
|
||||
]);
|
||||
let chunks = CodeKotlinAstV1Chunker.chunk(&doc, &policy()).unwrap();
|
||||
assert_eq!(chunks.len(), 2);
|
||||
for c in &chunks {
|
||||
assert_eq!(c.source_spans.len(), 1);
|
||||
assert!(matches!(c.source_spans[0], SourceSpan::Code { .. }));
|
||||
assert_eq!(c.heading_path, Vec::<String>::new());
|
||||
assert_eq!(c.chunker_version.0, "code-kotlin-ast-v1");
|
||||
}
|
||||
match &chunks[0].source_spans[0] {
|
||||
SourceSpan::Code { symbol, line_start, line_end, .. } => {
|
||||
assert_eq!(symbol.as_deref(), Some("parse"));
|
||||
assert_eq!((*line_start, *line_end), (1, 3));
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oversize_unit_splits_into_parts_with_unique_ids() {
|
||||
let body = (0..500).map(|i| format!("\tval x{i} = {i}")).collect::<Vec<_>>().join("\n");
|
||||
let code = format!("fun big() {{\n{body}\n}}");
|
||||
let doc = code_doc(&[("big", 1, 502, &code)]);
|
||||
let chunks = CodeKotlinAstV1Chunker.chunk(&doc, &policy()).unwrap();
|
||||
assert!(chunks.len() >= 2, "oversize unit must split, got {}", chunks.len());
|
||||
for c in &chunks {
|
||||
match &c.source_spans[0] {
|
||||
SourceSpan::Code { symbol, .. } => {
|
||||
assert!(symbol.as_deref().unwrap().starts_with("big [part "),
|
||||
"part-numbered symbol, got {symbol:?}");
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
let mut ids: Vec<&str> = chunks.iter().map(|c| c.chunk_id.0.as_str()).collect();
|
||||
let n = ids.len(); ids.sort(); ids.dedup();
|
||||
assert_eq!(ids.len(), n, "chunk_ids unique across split parts");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_code_doc_errors() {
|
||||
use kebab_core::TextBlock;
|
||||
let mut doc = code_doc(&[("parse", 1, 1, "fun parse() {}")]);
|
||||
doc.blocks = vec![Block::Paragraph(TextBlock {
|
||||
common: CommonBlock {
|
||||
block_id: kebab_core::BlockId("b".into()),
|
||||
heading_path: vec![],
|
||||
source_span: SourceSpan::Line { start: 1, end: 1 },
|
||||
},
|
||||
text: "x".into(), inlines: vec![],
|
||||
})];
|
||||
let err = CodeKotlinAstV1Chunker.chunk(&doc, &policy()).unwrap_err();
|
||||
assert!(err.to_string().contains("CodeKotlinAstV1Chunker"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deterministic_chunk_ids_1000() {
|
||||
let doc = code_doc(&[("parse", 1, 2, "fun parse() {}\n")]);
|
||||
let base: Vec<String> = CodeKotlinAstV1Chunker.chunk(&doc, &policy())
|
||||
.unwrap().into_iter().map(|c| c.chunk_id.0).collect();
|
||||
for _ in 0..1000 {
|
||||
let again: Vec<String> = CodeKotlinAstV1Chunker.chunk(&doc, &policy())
|
||||
.unwrap().into_iter().map(|c| c.chunk_id.0).collect();
|
||||
assert_eq!(again, base);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn policy_hash_matches_md_heading_v1() {
|
||||
let p = policy();
|
||||
assert_eq!(CodeKotlinAstV1Chunker.policy_hash(&p),
|
||||
crate::MdHeadingV1Chunker.policy_hash(&p));
|
||||
}
|
||||
}
|
||||
322
crates/kebab-chunk/src/code_python_ast_v1.rs
Normal file
322
crates/kebab-chunk/src/code_python_ast_v1.rs
Normal file
@@ -0,0 +1,322 @@
|
||||
//! `code-python-ast-v1` — maps a tree-sitter-derived Python AST
|
||||
//! `CanonicalDocument` (one `Block::Code` per semantic unit, each with
|
||||
//! `SourceSpan::Code`) to chunks 1:1. A unit longer than
|
||||
//! `AST_CHUNK_MAX_LINES` is split into `<symbol> [part i/N]` sub-chunks
|
||||
//! at blank-line paragraph boundaries (design §9.1 oversize fallback).
|
||||
//!
|
||||
//! tree-sitter is intentionally NOT a dependency here: AST work is
|
||||
//! parser-side (`kebab-parse-code`, design §6.3). This chunker only
|
||||
//! consumes the `CanonicalDocument`.
|
||||
//!
|
||||
//! `AST_CHUNK_MAX_LINES` is a constant matching
|
||||
//! `IngestCodeCfg::default().ast_chunk_max_lines` (200). Per-medium
|
||||
//! config threading needs a chunker registry (P+); same deviation
|
||||
//! pattern as `pdf-page-v1`'s pinned `chunker_version`
|
||||
//! (`tasks/HOTFIXES.md`).
|
||||
|
||||
use kebab_core::{
|
||||
Block, BlockId, CanonicalDocument, Chunk, ChunkPolicy, Chunker, ChunkerVersion, DocumentId,
|
||||
SourceSpan, id_for_chunk,
|
||||
};
|
||||
|
||||
const VERSION_LABEL: &str = "code-python-ast-v1";
|
||||
const BYTES_PER_TOKEN: usize = 3;
|
||||
const POLICY_HASH_HEX_LEN: usize = 16;
|
||||
const AST_CHUNK_MAX_LINES: u32 = 200;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
pub struct CodePythonAstV1Chunker;
|
||||
|
||||
impl Chunker for CodePythonAstV1Chunker {
|
||||
fn chunker_version(&self) -> ChunkerVersion {
|
||||
ChunkerVersion(VERSION_LABEL.to_string())
|
||||
}
|
||||
|
||||
fn policy_hash(&self, policy: &ChunkPolicy) -> String {
|
||||
let bytes = serde_json_canonicalizer::to_vec(policy)
|
||||
.expect("canonical JSON serialization of ChunkPolicy must not fail");
|
||||
let hex = blake3::hash(&bytes).to_hex().to_string();
|
||||
hex[..POLICY_HASH_HEX_LEN].to_string()
|
||||
}
|
||||
|
||||
fn chunk(
|
||||
&self,
|
||||
doc: &CanonicalDocument,
|
||||
policy: &ChunkPolicy,
|
||||
) -> anyhow::Result<Vec<Chunk>> {
|
||||
for b in &doc.blocks {
|
||||
let c = match b {
|
||||
Block::Code(c) => c,
|
||||
_ => anyhow::bail!(
|
||||
"CodePythonAstV1Chunker only handles code docs (got non-Code block)"
|
||||
),
|
||||
};
|
||||
if !matches!(c.common.source_span, SourceSpan::Code { .. }) {
|
||||
anyhow::bail!(
|
||||
"CodePythonAstV1Chunker only handles code docs (got non-Code source_span)"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let base_policy_hash = self.policy_hash(policy);
|
||||
let chunker_version = self.chunker_version();
|
||||
let mut out: Vec<Chunk> = Vec::new();
|
||||
|
||||
for b in &doc.blocks {
|
||||
let cb = match b {
|
||||
Block::Code(c) => c,
|
||||
_ => unreachable!("validated above"),
|
||||
};
|
||||
let (ls, le, symbol, lang) = match &cb.common.source_span {
|
||||
SourceSpan::Code { line_start, line_end, symbol, lang } => {
|
||||
(*line_start, *line_end, symbol.clone(), lang.clone())
|
||||
}
|
||||
_ => unreachable!("validated above"),
|
||||
};
|
||||
let block_ids: Vec<BlockId> = vec![cb.common.block_id.clone()];
|
||||
let span_lines = le.saturating_sub(ls) + 1;
|
||||
|
||||
if span_lines <= AST_CHUNK_MAX_LINES {
|
||||
let span = SourceSpan::Code {
|
||||
line_start: ls,
|
||||
line_end: le,
|
||||
symbol: symbol.clone(),
|
||||
lang: lang.clone(),
|
||||
};
|
||||
out.push(make_chunk(
|
||||
doc, &chunker_version, &block_ids, &base_policy_hash,
|
||||
None, span, cb.code.clone(),
|
||||
));
|
||||
} else {
|
||||
let parts = split_oversize(&cb.code);
|
||||
let n = parts.len();
|
||||
for (i, (off_start, off_end, text)) in parts.into_iter().enumerate() {
|
||||
let part_ls = ls + off_start;
|
||||
let part_le = ls + off_end;
|
||||
let part_sym = symbol
|
||||
.as_ref()
|
||||
.map(|s| format!("{s} [part {}/{n}]", i + 1));
|
||||
let span = SourceSpan::Code {
|
||||
line_start: part_ls,
|
||||
line_end: part_le,
|
||||
symbol: part_sym,
|
||||
lang: lang.clone(),
|
||||
};
|
||||
out.push(make_chunk(
|
||||
doc, &chunker_version, &block_ids, &base_policy_hash,
|
||||
Some(part_ls), span, text,
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tracing::debug!(
|
||||
target: "kebab-chunk",
|
||||
doc_id = %doc.doc_id,
|
||||
chunks = out.len(),
|
||||
"code-python-ast-v1 chunked",
|
||||
);
|
||||
Ok(out)
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn make_chunk(
|
||||
doc: &CanonicalDocument,
|
||||
chunker_version: &ChunkerVersion,
|
||||
block_ids: &[BlockId],
|
||||
base_policy_hash: &str,
|
||||
split_key: Option<u32>,
|
||||
span: SourceSpan,
|
||||
text: String,
|
||||
) -> Chunk {
|
||||
let id_hash = match split_key {
|
||||
Some(k) => format!("{base_policy_hash}#L{k}"),
|
||||
None => base_policy_hash.to_string(),
|
||||
};
|
||||
let chunk_id = id_for_chunk(&doc.doc_id, chunker_version, block_ids, &id_hash);
|
||||
let token_estimate = text.len().div_ceil(BYTES_PER_TOKEN);
|
||||
Chunk {
|
||||
chunk_id,
|
||||
doc_id: DocumentId(doc.doc_id.0.clone()),
|
||||
block_ids: block_ids.to_vec(),
|
||||
text,
|
||||
heading_path: Vec::new(),
|
||||
source_spans: vec![span],
|
||||
token_estimate,
|
||||
chunker_version: chunker_version.clone(),
|
||||
policy_hash: base_policy_hash.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Split an oversize unit at blank-line paragraph boundaries, greedily
|
||||
/// gluing paragraphs until ~`AST_CHUNK_MAX_LINES` lines accumulate.
|
||||
/// Returns `(line_offset_start, line_offset_end, text)` where offsets are
|
||||
/// 0-based within the unit (caller adds the unit's absolute `line_start`).
|
||||
fn split_oversize(code: &str) -> Vec<(u32, u32, String)> {
|
||||
let lines: Vec<&str> = code.split('\n').collect();
|
||||
let total = lines.len() as u32;
|
||||
let mut out: Vec<(u32, u32, String)> = Vec::new();
|
||||
let mut start: u32 = 0;
|
||||
while start < total {
|
||||
let mut end = (start + AST_CHUNK_MAX_LINES).min(total);
|
||||
let floor = start + (AST_CHUNK_MAX_LINES * 4 / 5);
|
||||
if end < total {
|
||||
if let Some(b) = (floor.min(end)..end)
|
||||
.rev()
|
||||
.find(|&i| lines[i as usize].trim().is_empty())
|
||||
{
|
||||
end = b + 1;
|
||||
}
|
||||
}
|
||||
let text = lines[start as usize..end as usize].join("\n");
|
||||
out.push((start, end.saturating_sub(1), text));
|
||||
start = end;
|
||||
}
|
||||
if out.is_empty() {
|
||||
out.push((0, total.saturating_sub(1), code.to_string()));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use kebab_core::{
|
||||
Block, CanonicalDocument, ChunkPolicy, Chunker, ChunkerVersion, CodeBlock, CommonBlock,
|
||||
SourceSpan, id_for_block, id_for_doc, AssetId, Lang, Metadata, ParserVersion, Provenance,
|
||||
SourceType, TrustLevel, WorkspacePath,
|
||||
};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
fn code_doc(units: &[(&str, u32, u32, &str)]) -> CanonicalDocument {
|
||||
let wp = WorkspacePath("crates/x/src/a.py".into());
|
||||
let aid = AssetId("a".repeat(64));
|
||||
let pv = ParserVersion("code-python-v1".into());
|
||||
let doc_id = id_for_doc(&wp, &aid, &pv);
|
||||
let blocks = units
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, (sym, ls, le, code))| {
|
||||
let span = SourceSpan::Code {
|
||||
line_start: *ls,
|
||||
line_end: *le,
|
||||
symbol: Some((*sym).to_string()),
|
||||
lang: Some("python".into()),
|
||||
};
|
||||
let bid = id_for_block(&doc_id, "code", &[], i as u32, &span);
|
||||
Block::Code(CodeBlock {
|
||||
common: CommonBlock { block_id: bid, heading_path: vec![], source_span: span },
|
||||
lang: Some("python".into()),
|
||||
code: (*code).to_string(),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
CanonicalDocument {
|
||||
doc_id, source_asset_id: aid, workspace_path: wp, title: "a".into(),
|
||||
lang: Lang("und".into()), blocks,
|
||||
metadata: Metadata {
|
||||
aliases: vec![], tags: vec![],
|
||||
created_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
updated_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
source_type: SourceType::Note, trust_level: TrustLevel::Primary,
|
||||
user_id_alias: None, user: Default::default(),
|
||||
repo: Some("kebab".into()), git_branch: Some("main".into()),
|
||||
git_commit: Some("0".repeat(40)), code_lang: Some("python".into()),
|
||||
},
|
||||
provenance: Provenance { events: vec![] },
|
||||
parser_version: pv, schema_version: 1, doc_version: 1,
|
||||
last_chunker_version: None, last_embedding_version: None,
|
||||
}
|
||||
}
|
||||
fn policy() -> ChunkPolicy {
|
||||
ChunkPolicy { target_tokens: 500, overlap_tokens: 80,
|
||||
respect_markdown_headings: false,
|
||||
chunker_version: ChunkerVersion(VERSION_LABEL.into()) }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chunker_version_is_code_python_ast_v1() {
|
||||
assert_eq!(CodePythonAstV1Chunker.chunker_version(),
|
||||
ChunkerVersion("code-python-ast-v1".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn one_chunk_per_unit_preserves_code_span() {
|
||||
let doc = code_doc(&[
|
||||
("parse", 1, 3, "def parse():\n pass\n # x"),
|
||||
("Foo.double", 5, 7, "def double():\n #\n pass"),
|
||||
]);
|
||||
let chunks = CodePythonAstV1Chunker.chunk(&doc, &policy()).unwrap();
|
||||
assert_eq!(chunks.len(), 2);
|
||||
for c in &chunks {
|
||||
assert_eq!(c.source_spans.len(), 1);
|
||||
assert!(matches!(c.source_spans[0], SourceSpan::Code { .. }));
|
||||
assert_eq!(c.heading_path, Vec::<String>::new());
|
||||
assert_eq!(c.chunker_version.0, "code-python-ast-v1");
|
||||
}
|
||||
match &chunks[0].source_spans[0] {
|
||||
SourceSpan::Code { symbol, line_start, line_end, .. } => {
|
||||
assert_eq!(symbol.as_deref(), Some("parse"));
|
||||
assert_eq!((*line_start, *line_end), (1, 3));
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oversize_unit_splits_into_parts_with_unique_ids() {
|
||||
let body = (0..500).map(|i| format!(" x{i} = {i}")).collect::<Vec<_>>().join("\n");
|
||||
let code = format!("def big():\n{body}\n");
|
||||
let doc = code_doc(&[("big", 1, 502, &code)]);
|
||||
let chunks = CodePythonAstV1Chunker.chunk(&doc, &policy()).unwrap();
|
||||
assert!(chunks.len() >= 2, "oversize unit must split, got {}", chunks.len());
|
||||
for c in &chunks {
|
||||
match &c.source_spans[0] {
|
||||
SourceSpan::Code { symbol, .. } => {
|
||||
assert!(symbol.as_deref().unwrap().starts_with("big [part "),
|
||||
"part-numbered symbol, got {symbol:?}");
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
let mut ids: Vec<&str> = chunks.iter().map(|c| c.chunk_id.0.as_str()).collect();
|
||||
let n = ids.len(); ids.sort(); ids.dedup();
|
||||
assert_eq!(ids.len(), n, "chunk_ids unique across split parts");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_code_doc_errors() {
|
||||
use kebab_core::TextBlock;
|
||||
let mut doc = code_doc(&[("parse", 1, 1, "def parse(): pass")]);
|
||||
doc.blocks = vec![Block::Paragraph(TextBlock {
|
||||
common: CommonBlock {
|
||||
block_id: kebab_core::BlockId("b".into()),
|
||||
heading_path: vec![],
|
||||
source_span: SourceSpan::Line { start: 1, end: 1 },
|
||||
},
|
||||
text: "x".into(), inlines: vec![],
|
||||
})];
|
||||
let err = CodePythonAstV1Chunker.chunk(&doc, &policy()).unwrap_err();
|
||||
assert!(err.to_string().contains("CodePythonAstV1Chunker"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deterministic_chunk_ids_1000() {
|
||||
let doc = code_doc(&[("parse", 1, 2, "def parse(): pass\n")]);
|
||||
let base: Vec<String> = CodePythonAstV1Chunker.chunk(&doc, &policy())
|
||||
.unwrap().into_iter().map(|c| c.chunk_id.0).collect();
|
||||
for _ in 0..1000 {
|
||||
let again: Vec<String> = CodePythonAstV1Chunker.chunk(&doc, &policy())
|
||||
.unwrap().into_iter().map(|c| c.chunk_id.0).collect();
|
||||
assert_eq!(again, base);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn policy_hash_matches_md_heading_v1() {
|
||||
let p = policy();
|
||||
assert_eq!(CodePythonAstV1Chunker.policy_hash(&p),
|
||||
crate::MdHeadingV1Chunker.policy_hash(&p));
|
||||
}
|
||||
}
|
||||
322
crates/kebab-chunk/src/code_ts_ast_v1.rs
Normal file
322
crates/kebab-chunk/src/code_ts_ast_v1.rs
Normal file
@@ -0,0 +1,322 @@
|
||||
//! `code-ts-ast-v1` — maps a tree-sitter-derived TypeScript AST
|
||||
//! `CanonicalDocument` (one `Block::Code` per semantic unit, each with
|
||||
//! `SourceSpan::Code`) to chunks 1:1. A unit longer than
|
||||
//! `AST_CHUNK_MAX_LINES` is split into `<symbol> [part i/N]` sub-chunks
|
||||
//! at blank-line paragraph boundaries (design §9.1 oversize fallback).
|
||||
//!
|
||||
//! tree-sitter is intentionally NOT a dependency here: AST work is
|
||||
//! parser-side (`kebab-parse-code`, design §6.3). This chunker only
|
||||
//! consumes the `CanonicalDocument`.
|
||||
//!
|
||||
//! `AST_CHUNK_MAX_LINES` is a constant matching
|
||||
//! `IngestCodeCfg::default().ast_chunk_max_lines` (200). Per-medium
|
||||
//! config threading needs a chunker registry (P+); same deviation
|
||||
//! pattern as `pdf-page-v1`'s pinned `chunker_version`
|
||||
//! (`tasks/HOTFIXES.md`).
|
||||
|
||||
use kebab_core::{
|
||||
Block, BlockId, CanonicalDocument, Chunk, ChunkPolicy, Chunker, ChunkerVersion, DocumentId,
|
||||
SourceSpan, id_for_chunk,
|
||||
};
|
||||
|
||||
const VERSION_LABEL: &str = "code-ts-ast-v1";
|
||||
const BYTES_PER_TOKEN: usize = 3;
|
||||
const POLICY_HASH_HEX_LEN: usize = 16;
|
||||
const AST_CHUNK_MAX_LINES: u32 = 200;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
pub struct CodeTsAstV1Chunker;
|
||||
|
||||
impl Chunker for CodeTsAstV1Chunker {
|
||||
fn chunker_version(&self) -> ChunkerVersion {
|
||||
ChunkerVersion(VERSION_LABEL.to_string())
|
||||
}
|
||||
|
||||
fn policy_hash(&self, policy: &ChunkPolicy) -> String {
|
||||
let bytes = serde_json_canonicalizer::to_vec(policy)
|
||||
.expect("canonical JSON serialization of ChunkPolicy must not fail");
|
||||
let hex = blake3::hash(&bytes).to_hex().to_string();
|
||||
hex[..POLICY_HASH_HEX_LEN].to_string()
|
||||
}
|
||||
|
||||
fn chunk(
|
||||
&self,
|
||||
doc: &CanonicalDocument,
|
||||
policy: &ChunkPolicy,
|
||||
) -> anyhow::Result<Vec<Chunk>> {
|
||||
for b in &doc.blocks {
|
||||
let c = match b {
|
||||
Block::Code(c) => c,
|
||||
_ => anyhow::bail!(
|
||||
"CodeTsAstV1Chunker only handles code docs (got non-Code block)"
|
||||
),
|
||||
};
|
||||
if !matches!(c.common.source_span, SourceSpan::Code { .. }) {
|
||||
anyhow::bail!(
|
||||
"CodeTsAstV1Chunker only handles code docs (got non-Code source_span)"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let base_policy_hash = self.policy_hash(policy);
|
||||
let chunker_version = self.chunker_version();
|
||||
let mut out: Vec<Chunk> = Vec::new();
|
||||
|
||||
for b in &doc.blocks {
|
||||
let cb = match b {
|
||||
Block::Code(c) => c,
|
||||
_ => unreachable!("validated above"),
|
||||
};
|
||||
let (ls, le, symbol, lang) = match &cb.common.source_span {
|
||||
SourceSpan::Code { line_start, line_end, symbol, lang } => {
|
||||
(*line_start, *line_end, symbol.clone(), lang.clone())
|
||||
}
|
||||
_ => unreachable!("validated above"),
|
||||
};
|
||||
let block_ids: Vec<BlockId> = vec![cb.common.block_id.clone()];
|
||||
let span_lines = le.saturating_sub(ls) + 1;
|
||||
|
||||
if span_lines <= AST_CHUNK_MAX_LINES {
|
||||
let span = SourceSpan::Code {
|
||||
line_start: ls,
|
||||
line_end: le,
|
||||
symbol: symbol.clone(),
|
||||
lang: lang.clone(),
|
||||
};
|
||||
out.push(make_chunk(
|
||||
doc, &chunker_version, &block_ids, &base_policy_hash,
|
||||
None, span, cb.code.clone(),
|
||||
));
|
||||
} else {
|
||||
let parts = split_oversize(&cb.code);
|
||||
let n = parts.len();
|
||||
for (i, (off_start, off_end, text)) in parts.into_iter().enumerate() {
|
||||
let part_ls = ls + off_start;
|
||||
let part_le = ls + off_end;
|
||||
let part_sym = symbol
|
||||
.as_ref()
|
||||
.map(|s| format!("{s} [part {}/{n}]", i + 1));
|
||||
let span = SourceSpan::Code {
|
||||
line_start: part_ls,
|
||||
line_end: part_le,
|
||||
symbol: part_sym,
|
||||
lang: lang.clone(),
|
||||
};
|
||||
out.push(make_chunk(
|
||||
doc, &chunker_version, &block_ids, &base_policy_hash,
|
||||
Some(part_ls), span, text,
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tracing::debug!(
|
||||
target: "kebab-chunk",
|
||||
doc_id = %doc.doc_id,
|
||||
chunks = out.len(),
|
||||
"code-ts-ast-v1 chunked",
|
||||
);
|
||||
Ok(out)
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn make_chunk(
|
||||
doc: &CanonicalDocument,
|
||||
chunker_version: &ChunkerVersion,
|
||||
block_ids: &[BlockId],
|
||||
base_policy_hash: &str,
|
||||
split_key: Option<u32>,
|
||||
span: SourceSpan,
|
||||
text: String,
|
||||
) -> Chunk {
|
||||
let id_hash = match split_key {
|
||||
Some(k) => format!("{base_policy_hash}#L{k}"),
|
||||
None => base_policy_hash.to_string(),
|
||||
};
|
||||
let chunk_id = id_for_chunk(&doc.doc_id, chunker_version, block_ids, &id_hash);
|
||||
let token_estimate = text.len().div_ceil(BYTES_PER_TOKEN);
|
||||
Chunk {
|
||||
chunk_id,
|
||||
doc_id: DocumentId(doc.doc_id.0.clone()),
|
||||
block_ids: block_ids.to_vec(),
|
||||
text,
|
||||
heading_path: Vec::new(),
|
||||
source_spans: vec![span],
|
||||
token_estimate,
|
||||
chunker_version: chunker_version.clone(),
|
||||
policy_hash: base_policy_hash.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Split an oversize unit at blank-line paragraph boundaries, greedily
|
||||
/// gluing paragraphs until ~`AST_CHUNK_MAX_LINES` lines accumulate.
|
||||
/// Returns `(line_offset_start, line_offset_end, text)` where offsets are
|
||||
/// 0-based within the unit (caller adds the unit's absolute `line_start`).
|
||||
fn split_oversize(code: &str) -> Vec<(u32, u32, String)> {
|
||||
let lines: Vec<&str> = code.split('\n').collect();
|
||||
let total = lines.len() as u32;
|
||||
let mut out: Vec<(u32, u32, String)> = Vec::new();
|
||||
let mut start: u32 = 0;
|
||||
while start < total {
|
||||
let mut end = (start + AST_CHUNK_MAX_LINES).min(total);
|
||||
let floor = start + (AST_CHUNK_MAX_LINES * 4 / 5);
|
||||
if end < total {
|
||||
if let Some(b) = (floor.min(end)..end)
|
||||
.rev()
|
||||
.find(|&i| lines[i as usize].trim().is_empty())
|
||||
{
|
||||
end = b + 1;
|
||||
}
|
||||
}
|
||||
let text = lines[start as usize..end as usize].join("\n");
|
||||
out.push((start, end.saturating_sub(1), text));
|
||||
start = end;
|
||||
}
|
||||
if out.is_empty() {
|
||||
out.push((0, total.saturating_sub(1), code.to_string()));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use kebab_core::{
|
||||
Block, CanonicalDocument, ChunkPolicy, Chunker, ChunkerVersion, CodeBlock, CommonBlock,
|
||||
SourceSpan, id_for_block, id_for_doc, AssetId, Lang, Metadata, ParserVersion, Provenance,
|
||||
SourceType, TrustLevel, WorkspacePath,
|
||||
};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
fn code_doc(units: &[(&str, u32, u32, &str)]) -> CanonicalDocument {
|
||||
let wp = WorkspacePath("crates/x/src/a.ts".into());
|
||||
let aid = AssetId("a".repeat(64));
|
||||
let pv = ParserVersion("code-ts-v1".into());
|
||||
let doc_id = id_for_doc(&wp, &aid, &pv);
|
||||
let blocks = units
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, (sym, ls, le, code))| {
|
||||
let span = SourceSpan::Code {
|
||||
line_start: *ls,
|
||||
line_end: *le,
|
||||
symbol: Some((*sym).to_string()),
|
||||
lang: Some("typescript".into()),
|
||||
};
|
||||
let bid = id_for_block(&doc_id, "code", &[], i as u32, &span);
|
||||
Block::Code(CodeBlock {
|
||||
common: CommonBlock { block_id: bid, heading_path: vec![], source_span: span },
|
||||
lang: Some("typescript".into()),
|
||||
code: (*code).to_string(),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
CanonicalDocument {
|
||||
doc_id, source_asset_id: aid, workspace_path: wp, title: "a".into(),
|
||||
lang: Lang("und".into()), blocks,
|
||||
metadata: Metadata {
|
||||
aliases: vec![], tags: vec![],
|
||||
created_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
updated_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
source_type: SourceType::Note, trust_level: TrustLevel::Primary,
|
||||
user_id_alias: None, user: Default::default(),
|
||||
repo: Some("kebab".into()), git_branch: Some("main".into()),
|
||||
git_commit: Some("0".repeat(40)), code_lang: Some("typescript".into()),
|
||||
},
|
||||
provenance: Provenance { events: vec![] },
|
||||
parser_version: pv, schema_version: 1, doc_version: 1,
|
||||
last_chunker_version: None, last_embedding_version: None,
|
||||
}
|
||||
}
|
||||
fn policy() -> ChunkPolicy {
|
||||
ChunkPolicy { target_tokens: 500, overlap_tokens: 80,
|
||||
respect_markdown_headings: false,
|
||||
chunker_version: ChunkerVersion(VERSION_LABEL.into()) }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chunker_version_is_code_ts_ast_v1() {
|
||||
assert_eq!(CodeTsAstV1Chunker.chunker_version(),
|
||||
ChunkerVersion("code-ts-ast-v1".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn one_chunk_per_unit_preserves_code_span() {
|
||||
let doc = code_doc(&[
|
||||
("parse", 1, 3, "function parse(): void {\n // x\n}"),
|
||||
("Foo.double", 5, 7, "function double(): number {\n //\n return 0;\n}"),
|
||||
]);
|
||||
let chunks = CodeTsAstV1Chunker.chunk(&doc, &policy()).unwrap();
|
||||
assert_eq!(chunks.len(), 2);
|
||||
for c in &chunks {
|
||||
assert_eq!(c.source_spans.len(), 1);
|
||||
assert!(matches!(c.source_spans[0], SourceSpan::Code { .. }));
|
||||
assert_eq!(c.heading_path, Vec::<String>::new());
|
||||
assert_eq!(c.chunker_version.0, "code-ts-ast-v1");
|
||||
}
|
||||
match &chunks[0].source_spans[0] {
|
||||
SourceSpan::Code { symbol, line_start, line_end, .. } => {
|
||||
assert_eq!(symbol.as_deref(), Some("parse"));
|
||||
assert_eq!((*line_start, *line_end), (1, 3));
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oversize_unit_splits_into_parts_with_unique_ids() {
|
||||
let body = (0..500).map(|i| format!(" const x{i} = {i};")).collect::<Vec<_>>().join("\n");
|
||||
let code = format!("function big(): void {{\n{body}\n}}");
|
||||
let doc = code_doc(&[("big", 1, 502, &code)]);
|
||||
let chunks = CodeTsAstV1Chunker.chunk(&doc, &policy()).unwrap();
|
||||
assert!(chunks.len() >= 2, "oversize unit must split, got {}", chunks.len());
|
||||
for c in &chunks {
|
||||
match &c.source_spans[0] {
|
||||
SourceSpan::Code { symbol, .. } => {
|
||||
assert!(symbol.as_deref().unwrap().starts_with("big [part "),
|
||||
"part-numbered symbol, got {symbol:?}");
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
let mut ids: Vec<&str> = chunks.iter().map(|c| c.chunk_id.0.as_str()).collect();
|
||||
let n = ids.len(); ids.sort(); ids.dedup();
|
||||
assert_eq!(ids.len(), n, "chunk_ids unique across split parts");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_code_doc_errors() {
|
||||
use kebab_core::TextBlock;
|
||||
let mut doc = code_doc(&[("parse", 1, 1, "function parse(): void {}")]);
|
||||
doc.blocks = vec![Block::Paragraph(TextBlock {
|
||||
common: CommonBlock {
|
||||
block_id: kebab_core::BlockId("b".into()),
|
||||
heading_path: vec![],
|
||||
source_span: SourceSpan::Line { start: 1, end: 1 },
|
||||
},
|
||||
text: "x".into(), inlines: vec![],
|
||||
})];
|
||||
let err = CodeTsAstV1Chunker.chunk(&doc, &policy()).unwrap_err();
|
||||
assert!(err.to_string().contains("CodeTsAstV1Chunker"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deterministic_chunk_ids_1000() {
|
||||
let doc = code_doc(&[("parse", 1, 2, "function parse(): void {}\n")]);
|
||||
let base: Vec<String> = CodeTsAstV1Chunker.chunk(&doc, &policy())
|
||||
.unwrap().into_iter().map(|c| c.chunk_id.0).collect();
|
||||
for _ in 0..1000 {
|
||||
let again: Vec<String> = CodeTsAstV1Chunker.chunk(&doc, &policy())
|
||||
.unwrap().into_iter().map(|c| c.chunk_id.0).collect();
|
||||
assert_eq!(again, base);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn policy_hash_matches_md_heading_v1() {
|
||||
let p = policy();
|
||||
assert_eq!(CodeTsAstV1Chunker.policy_hash(&p),
|
||||
crate::MdHeadingV1Chunker.policy_hash(&p));
|
||||
}
|
||||
}
|
||||
@@ -15,10 +15,22 @@
|
||||
//! embedder, the retriever, the LLM, the RAG layer, or the UI layers.
|
||||
//! It consumes `CanonicalDocument` purely through `kb-core` types.
|
||||
|
||||
mod code_go_ast_v1;
|
||||
mod code_java_ast_v1;
|
||||
mod code_js_ast_v1;
|
||||
mod code_kotlin_ast_v1;
|
||||
mod code_python_ast_v1;
|
||||
mod code_rust_ast_v1;
|
||||
mod code_ts_ast_v1;
|
||||
mod md_heading_v1;
|
||||
mod pdf_page_v1;
|
||||
|
||||
pub use code_go_ast_v1::CodeGoAstV1Chunker;
|
||||
pub use code_java_ast_v1::CodeJavaAstV1Chunker;
|
||||
pub use code_js_ast_v1::CodeJsAstV1Chunker;
|
||||
pub use code_kotlin_ast_v1::CodeKotlinAstV1Chunker;
|
||||
pub use code_python_ast_v1::CodePythonAstV1Chunker;
|
||||
pub use code_rust_ast_v1::CodeRustAstV1Chunker;
|
||||
pub use code_ts_ast_v1::CodeTsAstV1Chunker;
|
||||
pub use md_heading_v1::MdHeadingV1Chunker;
|
||||
pub use pdf_page_v1::PdfPageV1Chunker;
|
||||
|
||||
221
crates/kebab-chunk/tests/code_go_ast_snapshot.rs
Normal file
221
crates/kebab-chunk/tests/code_go_ast_snapshot.rs
Normal file
@@ -0,0 +1,221 @@
|
||||
//! Snapshot test pinning the `Vec<Chunk>` JSON for a
|
||||
//! representative Go code `CanonicalDocument`.
|
||||
//!
|
||||
//! This is an integration test. `kebab-parse-code` is intentionally NOT
|
||||
//! a dev-dep (design §6.3 / §8 boundary: AST extraction is parser-side).
|
||||
//! The `CanonicalDocument` is built inline from hand-crafted `Block::Code`
|
||||
//! units, which is the same pattern used in `code_rust_ast_v1.rs`'s
|
||||
//! internal `code_doc` test helper.
|
||||
//!
|
||||
//! Set `UPDATE_SNAPSHOTS=1` to re-bake the baseline.
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use kebab_chunk::CodeGoAstV1Chunker;
|
||||
use kebab_core::{
|
||||
AssetId, Block, CanonicalDocument, ChunkPolicy, Chunker, ChunkerVersion, CodeBlock, CommonBlock,
|
||||
Lang, Metadata, ParserVersion, Provenance, SourceSpan, SourceType, TrustLevel, WorkspacePath,
|
||||
id_for_block, id_for_doc,
|
||||
};
|
||||
use serde_json::Value;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
fn fixtures_dir() -> PathBuf {
|
||||
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("tests")
|
||||
.join("fixtures")
|
||||
}
|
||||
|
||||
fn fixed_doc() -> CanonicalDocument {
|
||||
let wp = WorkspacePath("kebab_eval/metrics.go".into());
|
||||
let aid = AssetId("b".repeat(64));
|
||||
// Pin parser_version so doc_id / block_ids are reproducible.
|
||||
let pv = ParserVersion("code-go-v1".into());
|
||||
let doc_id = id_for_doc(&wp, &aid, &pv);
|
||||
|
||||
// Build a >200-line function body to force split_oversize.
|
||||
let big_body: String = {
|
||||
let header = "func BigCompute(data []int) int {\n";
|
||||
let body: String = (0..210u32)
|
||||
.map(|i| format!("\tv{i} := 0\n\tif {i} < len(data) {{\n\t\tv{i} = data[{i}]\n\t}}\n"))
|
||||
.collect();
|
||||
let footer = "\treturn len(data)\n}";
|
||||
format!("{header}{body}{footer}")
|
||||
};
|
||||
let big_line_count = big_body.lines().count() as u32;
|
||||
let big_line_end = 48 + big_line_count - 1;
|
||||
|
||||
// Representative units:
|
||||
// 0. import block (lines 1–5, ≤200)
|
||||
// 1. free fn `ComputeMRR` (lines 7–12, ≤200)
|
||||
// 2. struct `MetricsCollector` (lines 14–20, ≤200)
|
||||
// 3. struct `BaseEvaluator` (lines 22–30, ≤200)
|
||||
// 4. method `Run` (lines 32–38, ≤200)
|
||||
// 5. method `Report` (lines 40–46, ≤200)
|
||||
// 6. BigCompute (>200 lines) to force split_oversize
|
||||
let raw_units: Vec<(&str, u32, u32, String)> = vec![
|
||||
(
|
||||
"imports",
|
||||
1,
|
||||
5,
|
||||
"import (\n\t\"fmt\"\n\t\"os\"\n\t\"strings\"\n)".to_string(),
|
||||
),
|
||||
(
|
||||
"ComputeMRR",
|
||||
7,
|
||||
12,
|
||||
"func ComputeMRR(scores []float64) float64 {\n\tif len(scores) == 0 {\n\t\treturn 0.0\n\t}\n\t_ = fmt.Sprintf(\"%v\", scores)\n\treturn 1.0 / float64(len(scores))\n}".to_string(),
|
||||
),
|
||||
(
|
||||
"MetricsCollector",
|
||||
14,
|
||||
20,
|
||||
"type MetricsCollector struct {\n\tScores []float64\n\tLabels []string\n\tCounts map[string]int\n\tTotals map[string]float64\n\tTags []string\n}".to_string(),
|
||||
),
|
||||
(
|
||||
"BaseEvaluator",
|
||||
22,
|
||||
30,
|
||||
"type BaseEvaluator struct {\n\tName string\n}\n\nfunc (e *BaseEvaluator) Evaluate(data []string) error {\n\t_ = os.Stderr\n\t_ = strings.Join(data, \",\")\n\treturn nil\n}".to_string(),
|
||||
),
|
||||
(
|
||||
"MetricsCollector.Run",
|
||||
32,
|
||||
38,
|
||||
"func (m *MetricsCollector) Run(inputs []float64) {\n\tfor _, inp := range inputs {\n\t\tm.Scores = append(\n\t\t\tm.Scores,\n\t\t\tinp,\n\t\t)\n\t}\n}".to_string(),
|
||||
),
|
||||
(
|
||||
"MetricsCollector.Report",
|
||||
40,
|
||||
46,
|
||||
"func (m *MetricsCollector) Report() map[string]interface{} {\n\treturn map[string]interface{}{\n\t\t\"mean\": 0.0,\n\t\t\"count\": len(m.Scores),\n\t\t\"tags\": m.Tags,\n\t}\n}".to_string(),
|
||||
),
|
||||
("BigCompute", 48, big_line_end, big_body),
|
||||
];
|
||||
|
||||
let blocks: Vec<Block> = raw_units
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, (sym, ls, le, code))| {
|
||||
let span = SourceSpan::Code {
|
||||
line_start: *ls,
|
||||
line_end: *le,
|
||||
symbol: Some((*sym).to_string()),
|
||||
lang: Some("go".into()),
|
||||
};
|
||||
let bid = id_for_block(&doc_id, "code", &[], i as u32, &span);
|
||||
Block::Code(CodeBlock {
|
||||
common: CommonBlock {
|
||||
block_id: bid,
|
||||
heading_path: vec![],
|
||||
source_span: span,
|
||||
},
|
||||
lang: Some("go".into()),
|
||||
code: code.clone(),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
CanonicalDocument {
|
||||
doc_id,
|
||||
source_asset_id: aid,
|
||||
workspace_path: wp,
|
||||
title: "metrics.go".into(),
|
||||
lang: Lang("und".into()),
|
||||
blocks,
|
||||
metadata: Metadata {
|
||||
aliases: vec![],
|
||||
tags: vec![],
|
||||
created_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
updated_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
source_type: SourceType::Note,
|
||||
trust_level: TrustLevel::Primary,
|
||||
user_id_alias: None,
|
||||
user: Default::default(),
|
||||
repo: Some("kebab".into()),
|
||||
git_branch: Some("main".into()),
|
||||
git_commit: Some("0".repeat(40)),
|
||||
code_lang: Some("go".into()),
|
||||
},
|
||||
provenance: Provenance { events: vec![] },
|
||||
parser_version: pv,
|
||||
schema_version: 1,
|
||||
doc_version: 1,
|
||||
last_chunker_version: None,
|
||||
last_embedding_version: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn fixed_policy() -> ChunkPolicy {
|
||||
ChunkPolicy {
|
||||
target_tokens: 500,
|
||||
overlap_tokens: 80,
|
||||
respect_markdown_headings: false,
|
||||
chunker_version: ChunkerVersion("code-go-ast-v1".into()),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn code_go_ast_chunks_snapshot() {
|
||||
let doc = fixed_doc();
|
||||
let policy = fixed_policy();
|
||||
|
||||
let chunks = CodeGoAstV1Chunker.chunk(&doc, &policy).expect("chunk");
|
||||
let actual = serde_json::to_value(&chunks).unwrap();
|
||||
|
||||
let dir = fixtures_dir();
|
||||
let baseline_path = dir.join("code-sample.go.chunks.snapshot.json");
|
||||
let baseline_text = match std::fs::read_to_string(&baseline_path) {
|
||||
Ok(s) => s,
|
||||
Err(_) if std::env::var("UPDATE_SNAPSHOTS").is_ok() => {
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let pretty = serde_json::to_string_pretty(&actual).unwrap();
|
||||
std::fs::write(&baseline_path, format!("{pretty}\n")).unwrap();
|
||||
return;
|
||||
}
|
||||
Err(e) => panic!(
|
||||
"missing baseline {}; run with UPDATE_SNAPSHOTS=1 to create: {e}",
|
||||
baseline_path.display()
|
||||
),
|
||||
};
|
||||
let expected: Value = serde_json::from_str(&baseline_text).expect("baseline parses as json");
|
||||
|
||||
if actual != expected {
|
||||
if std::env::var("UPDATE_SNAPSHOTS").is_ok() {
|
||||
let pretty = serde_json::to_string_pretty(&actual).unwrap();
|
||||
std::fs::write(&baseline_path, format!("{pretty}\n")).unwrap();
|
||||
eprintln!("updated baseline {}", baseline_path.display());
|
||||
return;
|
||||
}
|
||||
let pretty = serde_json::to_string_pretty(&actual).unwrap();
|
||||
panic!(
|
||||
"code-go-ast-v1 chunks snapshot drift\n\
|
||||
--- expected ({}) ---\n{baseline_text}\n\
|
||||
--- actual ---\n{pretty}\n\
|
||||
If intentional, re-run with UPDATE_SNAPSHOTS=1.",
|
||||
baseline_path.display()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Determinism cross-check: re-running the same pipeline yields the same
|
||||
/// chunk_ids byte-for-byte.
|
||||
#[test]
|
||||
fn code_go_ast_chunks_are_deterministic() {
|
||||
let policy = fixed_policy();
|
||||
let baseline: Vec<String> = CodeGoAstV1Chunker
|
||||
.chunk(&fixed_doc(), &policy)
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|c| c.chunk_id.0)
|
||||
.collect();
|
||||
for _ in 0..5 {
|
||||
let again: Vec<String> = CodeGoAstV1Chunker
|
||||
.chunk(&fixed_doc(), &policy)
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|c| c.chunk_id.0)
|
||||
.collect();
|
||||
assert_eq!(again, baseline);
|
||||
}
|
||||
}
|
||||
221
crates/kebab-chunk/tests/code_java_ast_snapshot.rs
Normal file
221
crates/kebab-chunk/tests/code_java_ast_snapshot.rs
Normal file
@@ -0,0 +1,221 @@
|
||||
//! Snapshot test pinning the `Vec<Chunk>` JSON for a
|
||||
//! representative Java code `CanonicalDocument`.
|
||||
//!
|
||||
//! This is an integration test. `kebab-parse-code` is intentionally NOT
|
||||
//! a dev-dep (design §6.3 / §8 boundary: AST extraction is parser-side).
|
||||
//! The `CanonicalDocument` is built inline from hand-crafted `Block::Code`
|
||||
//! units, which is the same pattern used in `code_rust_ast_v1.rs`'s
|
||||
//! internal `code_doc` test helper.
|
||||
//!
|
||||
//! Set `UPDATE_SNAPSHOTS=1` to re-bake the baseline.
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use kebab_chunk::CodeJavaAstV1Chunker;
|
||||
use kebab_core::{
|
||||
AssetId, Block, CanonicalDocument, ChunkPolicy, Chunker, ChunkerVersion, CodeBlock, CommonBlock,
|
||||
Lang, Metadata, ParserVersion, Provenance, SourceSpan, SourceType, TrustLevel, WorkspacePath,
|
||||
id_for_block, id_for_doc,
|
||||
};
|
||||
use serde_json::Value;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
fn fixtures_dir() -> PathBuf {
|
||||
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("tests")
|
||||
.join("fixtures")
|
||||
}
|
||||
|
||||
fn fixed_doc() -> CanonicalDocument {
|
||||
let wp = WorkspacePath("src/main/java/com/example/Metrics.java".into());
|
||||
let aid = AssetId("b".repeat(64));
|
||||
// Pin parser_version so doc_id / block_ids are reproducible.
|
||||
let pv = ParserVersion("code-java-v1".into());
|
||||
let doc_id = id_for_doc(&wp, &aid, &pv);
|
||||
|
||||
// Build a >200-line method body to force split_oversize.
|
||||
let big_body: String = {
|
||||
let header = "public class BigCompute {\n public int compute(int[] data) {\n";
|
||||
let body: String = (0..210u32)
|
||||
.map(|i| format!(" int v{i} = {i} < data.length ? data[{i}] : 0;\n"))
|
||||
.collect();
|
||||
let footer = " return data.length;\n }\n}";
|
||||
format!("{header}{body}{footer}")
|
||||
};
|
||||
let big_line_count = big_body.lines().count() as u32;
|
||||
let big_line_end = 48 + big_line_count - 1;
|
||||
|
||||
// Representative units:
|
||||
// 0. import block (lines 1–5, ≤200)
|
||||
// 1. free method `computeMRR` (lines 7–12, ≤200)
|
||||
// 2. class `MetricsCollector` (lines 14–20, ≤200)
|
||||
// 3. class `BaseEvaluator` (lines 22–30, ≤200)
|
||||
// 4. method `MetricsCollector.run` (lines 32–38, ≤200)
|
||||
// 5. method `MetricsCollector.report` (lines 40–46, ≤200)
|
||||
// 6. BigCompute (>200 lines) to force split_oversize
|
||||
let raw_units: Vec<(&str, u32, u32, String)> = vec![
|
||||
(
|
||||
"imports",
|
||||
1,
|
||||
5,
|
||||
"import java.util.List;\nimport java.util.Map;\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.stream.Collectors;".to_string(),
|
||||
),
|
||||
(
|
||||
"computeMRR",
|
||||
7,
|
||||
12,
|
||||
"public static double computeMRR(List<Double> scores) {\n if (scores.isEmpty()) {\n return 0.0;\n }\n return 1.0 / scores.size();\n}".to_string(),
|
||||
),
|
||||
(
|
||||
"MetricsCollector",
|
||||
14,
|
||||
20,
|
||||
"public class MetricsCollector {\n private List<Double> scores;\n private List<String> labels;\n private Map<String, Integer> counts;\n private Map<String, Double> totals;\n private List<String> tags;\n}".to_string(),
|
||||
),
|
||||
(
|
||||
"BaseEvaluator",
|
||||
22,
|
||||
30,
|
||||
"public class BaseEvaluator {\n private String name;\n\n public BaseEvaluator(String name) {\n this.name = name;\n }\n\n public void evaluate(List<String> data) throws Exception {\n String joined = String.join(\",\", data);\n }\n}".to_string(),
|
||||
),
|
||||
(
|
||||
"MetricsCollector.run",
|
||||
32,
|
||||
38,
|
||||
"public void run(List<Double> inputs) {\n for (Double inp : inputs) {\n scores.add(\n inp\n );\n }\n}".to_string(),
|
||||
),
|
||||
(
|
||||
"MetricsCollector.report",
|
||||
40,
|
||||
46,
|
||||
"public Map<String, Object> report() {\n Map<String, Object> result = new HashMap<>();\n result.put(\"mean\", 0.0);\n result.put(\"count\", scores.size());\n result.put(\"tags\", tags);\n return result;\n}".to_string(),
|
||||
),
|
||||
("BigCompute", 48, big_line_end, big_body),
|
||||
];
|
||||
|
||||
let blocks: Vec<Block> = raw_units
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, (sym, ls, le, code))| {
|
||||
let span = SourceSpan::Code {
|
||||
line_start: *ls,
|
||||
line_end: *le,
|
||||
symbol: Some((*sym).to_string()),
|
||||
lang: Some("java".into()),
|
||||
};
|
||||
let bid = id_for_block(&doc_id, "code", &[], i as u32, &span);
|
||||
Block::Code(CodeBlock {
|
||||
common: CommonBlock {
|
||||
block_id: bid,
|
||||
heading_path: vec![],
|
||||
source_span: span,
|
||||
},
|
||||
lang: Some("java".into()),
|
||||
code: code.clone(),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
CanonicalDocument {
|
||||
doc_id,
|
||||
source_asset_id: aid,
|
||||
workspace_path: wp,
|
||||
title: "Metrics.java".into(),
|
||||
lang: Lang("und".into()),
|
||||
blocks,
|
||||
metadata: Metadata {
|
||||
aliases: vec![],
|
||||
tags: vec![],
|
||||
created_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
updated_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
source_type: SourceType::Note,
|
||||
trust_level: TrustLevel::Primary,
|
||||
user_id_alias: None,
|
||||
user: Default::default(),
|
||||
repo: Some("kebab".into()),
|
||||
git_branch: Some("main".into()),
|
||||
git_commit: Some("0".repeat(40)),
|
||||
code_lang: Some("java".into()),
|
||||
},
|
||||
provenance: Provenance { events: vec![] },
|
||||
parser_version: pv,
|
||||
schema_version: 1,
|
||||
doc_version: 1,
|
||||
last_chunker_version: None,
|
||||
last_embedding_version: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn fixed_policy() -> ChunkPolicy {
|
||||
ChunkPolicy {
|
||||
target_tokens: 500,
|
||||
overlap_tokens: 80,
|
||||
respect_markdown_headings: false,
|
||||
chunker_version: ChunkerVersion("code-java-ast-v1".into()),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn code_java_ast_chunks_snapshot() {
|
||||
let doc = fixed_doc();
|
||||
let policy = fixed_policy();
|
||||
|
||||
let chunks = CodeJavaAstV1Chunker.chunk(&doc, &policy).expect("chunk");
|
||||
let actual = serde_json::to_value(&chunks).unwrap();
|
||||
|
||||
let dir = fixtures_dir();
|
||||
let baseline_path = dir.join("code-sample.java.chunks.snapshot.json");
|
||||
let baseline_text = match std::fs::read_to_string(&baseline_path) {
|
||||
Ok(s) => s,
|
||||
Err(_) if std::env::var("UPDATE_SNAPSHOTS").is_ok() => {
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let pretty = serde_json::to_string_pretty(&actual).unwrap();
|
||||
std::fs::write(&baseline_path, format!("{pretty}\n")).unwrap();
|
||||
return;
|
||||
}
|
||||
Err(e) => panic!(
|
||||
"missing baseline {}; run with UPDATE_SNAPSHOTS=1 to create: {e}",
|
||||
baseline_path.display()
|
||||
),
|
||||
};
|
||||
let expected: Value = serde_json::from_str(&baseline_text).expect("baseline parses as json");
|
||||
|
||||
if actual != expected {
|
||||
if std::env::var("UPDATE_SNAPSHOTS").is_ok() {
|
||||
let pretty = serde_json::to_string_pretty(&actual).unwrap();
|
||||
std::fs::write(&baseline_path, format!("{pretty}\n")).unwrap();
|
||||
eprintln!("updated baseline {}", baseline_path.display());
|
||||
return;
|
||||
}
|
||||
let pretty = serde_json::to_string_pretty(&actual).unwrap();
|
||||
panic!(
|
||||
"code-java-ast-v1 chunks snapshot drift\n\
|
||||
--- expected ({}) ---\n{baseline_text}\n\
|
||||
--- actual ---\n{pretty}\n\
|
||||
If intentional, re-run with UPDATE_SNAPSHOTS=1.",
|
||||
baseline_path.display()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Determinism cross-check: re-running the same pipeline yields the same
|
||||
/// chunk_ids byte-for-byte.
|
||||
#[test]
|
||||
fn code_java_ast_chunks_are_deterministic() {
|
||||
let policy = fixed_policy();
|
||||
let baseline: Vec<String> = CodeJavaAstV1Chunker
|
||||
.chunk(&fixed_doc(), &policy)
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|c| c.chunk_id.0)
|
||||
.collect();
|
||||
for _ in 0..5 {
|
||||
let again: Vec<String> = CodeJavaAstV1Chunker
|
||||
.chunk(&fixed_doc(), &policy)
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|c| c.chunk_id.0)
|
||||
.collect();
|
||||
assert_eq!(again, baseline);
|
||||
}
|
||||
}
|
||||
221
crates/kebab-chunk/tests/code_js_ast_snapshot.rs
Normal file
221
crates/kebab-chunk/tests/code_js_ast_snapshot.rs
Normal file
@@ -0,0 +1,221 @@
|
||||
//! Snapshot test pinning the `Vec<Chunk>` JSON for a
|
||||
//! representative JavaScript code `CanonicalDocument`.
|
||||
//!
|
||||
//! This is an integration test. `kebab-parse-code` is intentionally NOT
|
||||
//! a dev-dep (design §6.3 / §8 boundary: AST extraction is parser-side).
|
||||
//! The `CanonicalDocument` is built inline from hand-crafted `Block::Code`
|
||||
//! units, which is the same pattern used in `code_rust_ast_v1.rs`'s
|
||||
//! internal `code_doc` test helper.
|
||||
//!
|
||||
//! Set `UPDATE_SNAPSHOTS=1` to re-bake the baseline.
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use kebab_chunk::CodeJsAstV1Chunker;
|
||||
use kebab_core::{
|
||||
AssetId, Block, CanonicalDocument, ChunkPolicy, Chunker, ChunkerVersion, CodeBlock, CommonBlock,
|
||||
Lang, Metadata, ParserVersion, Provenance, SourceSpan, SourceType, TrustLevel, WorkspacePath,
|
||||
id_for_block, id_for_doc,
|
||||
};
|
||||
use serde_json::Value;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
fn fixtures_dir() -> PathBuf {
|
||||
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("tests")
|
||||
.join("fixtures")
|
||||
}
|
||||
|
||||
fn fixed_doc() -> CanonicalDocument {
|
||||
let wp = WorkspacePath("src/bar.js".into());
|
||||
let aid = AssetId("b".repeat(64));
|
||||
// Pin parser_version so doc_id / block_ids are reproducible.
|
||||
let pv = ParserVersion("code-js-v1".into());
|
||||
let doc_id = id_for_doc(&wp, &aid, &pv);
|
||||
|
||||
// Build a >200-line function body to force split_oversize.
|
||||
let big_body: String = {
|
||||
let header = "function bigTransform(items) {\n";
|
||||
let body: String = (0..210u32)
|
||||
.map(|i| format!(" const v{i} = items[{i}] !== undefined ? items[{i}] : null;\n"))
|
||||
.collect();
|
||||
let footer = " return items;\n}";
|
||||
format!("{header}{body}{footer}")
|
||||
};
|
||||
let big_line_count = big_body.lines().count() as u32;
|
||||
let big_line_end = 48 + big_line_count - 1;
|
||||
|
||||
// Representative units:
|
||||
// 0. require/import block (lines 1–5, ≤200)
|
||||
// 1. free fn `add` (lines 7–12, ≤200)
|
||||
// 2. class `EventBus` (lines 14–20, ≤200)
|
||||
// 3. class `BaseHandler` (lines 22–30, ≤200)
|
||||
// 4. method `EventBus.emit` (lines 32–38, ≤200)
|
||||
// 5. method `EventBus.on` (lines 40–46, ≤200)
|
||||
// 6. bigTransform (>200 lines) to force split_oversize
|
||||
let raw_units: Vec<(&str, u32, u32, String)> = vec![
|
||||
(
|
||||
"requires",
|
||||
1,
|
||||
5,
|
||||
"const fs = require('fs');\nconst path = require('path');\nconst { EventEmitter } = require('events');\nconst assert = require('assert');\nconst crypto = require('crypto');".to_string(),
|
||||
),
|
||||
(
|
||||
"add",
|
||||
7,
|
||||
12,
|
||||
"export function add(a, b) {\n if (typeof a !== 'number') throw new TypeError('a');\n if (typeof b !== 'number') throw new TypeError('b');\n const result = a + b;\n assert(isFinite(result));\n return result;\n}".to_string(),
|
||||
),
|
||||
(
|
||||
"EventBus",
|
||||
14,
|
||||
20,
|
||||
"class EventBus {\n constructor() {\n this._handlers = new Map();\n this._history = [];\n this._maxHistory = 100;\n this._seq = 0;\n }\n}".to_string(),
|
||||
),
|
||||
(
|
||||
"BaseHandler",
|
||||
22,
|
||||
30,
|
||||
"class BaseHandler {\n handle(event) {\n throw new Error('not implemented');\n }\n batchHandle(events) {\n const results = [];\n for (const ev of events) {\n results.push(this.handle(ev));\n }\n return results;\n }\n}".to_string(),
|
||||
),
|
||||
(
|
||||
"EventBus.emit",
|
||||
32,
|
||||
38,
|
||||
"class EventBus {\n emit(name, payload) {\n const handlers = this._handlers.get(name) ?? [];\n for (const h of handlers) {\n h(payload);\n }\n return this;\n }\n}".to_string(),
|
||||
),
|
||||
(
|
||||
"EventBus.on",
|
||||
40,
|
||||
46,
|
||||
"class EventBus {\n on(name, handler) {\n if (!this._handlers.has(name)) {\n this._handlers.set(name, []);\n }\n this._handlers.get(name).push(handler);\n return this;\n }\n}".to_string(),
|
||||
),
|
||||
("bigTransform", 48, big_line_end, big_body),
|
||||
];
|
||||
|
||||
let blocks: Vec<Block> = raw_units
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, (sym, ls, le, code))| {
|
||||
let span = SourceSpan::Code {
|
||||
line_start: *ls,
|
||||
line_end: *le,
|
||||
symbol: Some((*sym).to_string()),
|
||||
lang: Some("javascript".into()),
|
||||
};
|
||||
let bid = id_for_block(&doc_id, "code", &[], i as u32, &span);
|
||||
Block::Code(CodeBlock {
|
||||
common: CommonBlock {
|
||||
block_id: bid,
|
||||
heading_path: vec![],
|
||||
source_span: span,
|
||||
},
|
||||
lang: Some("javascript".into()),
|
||||
code: code.clone(),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
CanonicalDocument {
|
||||
doc_id,
|
||||
source_asset_id: aid,
|
||||
workspace_path: wp,
|
||||
title: "bar.js".into(),
|
||||
lang: Lang("und".into()),
|
||||
blocks,
|
||||
metadata: Metadata {
|
||||
aliases: vec![],
|
||||
tags: vec![],
|
||||
created_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
updated_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
source_type: SourceType::Note,
|
||||
trust_level: TrustLevel::Primary,
|
||||
user_id_alias: None,
|
||||
user: Default::default(),
|
||||
repo: Some("kebab".into()),
|
||||
git_branch: Some("main".into()),
|
||||
git_commit: Some("0".repeat(40)),
|
||||
code_lang: Some("javascript".into()),
|
||||
},
|
||||
provenance: Provenance { events: vec![] },
|
||||
parser_version: pv,
|
||||
schema_version: 1,
|
||||
doc_version: 1,
|
||||
last_chunker_version: None,
|
||||
last_embedding_version: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn fixed_policy() -> ChunkPolicy {
|
||||
ChunkPolicy {
|
||||
target_tokens: 500,
|
||||
overlap_tokens: 80,
|
||||
respect_markdown_headings: false,
|
||||
chunker_version: ChunkerVersion("code-js-ast-v1".into()),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn code_js_ast_chunks_snapshot() {
|
||||
let doc = fixed_doc();
|
||||
let policy = fixed_policy();
|
||||
|
||||
let chunks = CodeJsAstV1Chunker.chunk(&doc, &policy).expect("chunk");
|
||||
let actual = serde_json::to_value(&chunks).unwrap();
|
||||
|
||||
let dir = fixtures_dir();
|
||||
let baseline_path = dir.join("code-sample.js.chunks.snapshot.json");
|
||||
let baseline_text = match std::fs::read_to_string(&baseline_path) {
|
||||
Ok(s) => s,
|
||||
Err(_) if std::env::var("UPDATE_SNAPSHOTS").is_ok() => {
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let pretty = serde_json::to_string_pretty(&actual).unwrap();
|
||||
std::fs::write(&baseline_path, format!("{pretty}\n")).unwrap();
|
||||
return;
|
||||
}
|
||||
Err(e) => panic!(
|
||||
"missing baseline {}; run with UPDATE_SNAPSHOTS=1 to create: {e}",
|
||||
baseline_path.display()
|
||||
),
|
||||
};
|
||||
let expected: Value = serde_json::from_str(&baseline_text).expect("baseline parses as json");
|
||||
|
||||
if actual != expected {
|
||||
if std::env::var("UPDATE_SNAPSHOTS").is_ok() {
|
||||
let pretty = serde_json::to_string_pretty(&actual).unwrap();
|
||||
std::fs::write(&baseline_path, format!("{pretty}\n")).unwrap();
|
||||
eprintln!("updated baseline {}", baseline_path.display());
|
||||
return;
|
||||
}
|
||||
let pretty = serde_json::to_string_pretty(&actual).unwrap();
|
||||
panic!(
|
||||
"code-js-ast-v1 chunks snapshot drift\n\
|
||||
--- expected ({}) ---\n{baseline_text}\n\
|
||||
--- actual ---\n{pretty}\n\
|
||||
If intentional, re-run with UPDATE_SNAPSHOTS=1.",
|
||||
baseline_path.display()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Determinism cross-check: re-running the same pipeline yields the same
|
||||
/// chunk_ids byte-for-byte.
|
||||
#[test]
|
||||
fn code_js_ast_chunks_are_deterministic() {
|
||||
let policy = fixed_policy();
|
||||
let baseline: Vec<String> = CodeJsAstV1Chunker
|
||||
.chunk(&fixed_doc(), &policy)
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|c| c.chunk_id.0)
|
||||
.collect();
|
||||
for _ in 0..5 {
|
||||
let again: Vec<String> = CodeJsAstV1Chunker
|
||||
.chunk(&fixed_doc(), &policy)
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|c| c.chunk_id.0)
|
||||
.collect();
|
||||
assert_eq!(again, baseline);
|
||||
}
|
||||
}
|
||||
221
crates/kebab-chunk/tests/code_kotlin_ast_snapshot.rs
Normal file
221
crates/kebab-chunk/tests/code_kotlin_ast_snapshot.rs
Normal file
@@ -0,0 +1,221 @@
|
||||
//! Snapshot test pinning the `Vec<Chunk>` JSON for a
|
||||
//! representative Kotlin code `CanonicalDocument`.
|
||||
//!
|
||||
//! This is an integration test. `kebab-parse-code` is intentionally NOT
|
||||
//! a dev-dep (design §6.3 / §8 boundary: AST extraction is parser-side).
|
||||
//! The `CanonicalDocument` is built inline from hand-crafted `Block::Code`
|
||||
//! units, which is the same pattern used in `code_rust_ast_v1.rs`'s
|
||||
//! internal `code_doc` test helper.
|
||||
//!
|
||||
//! Set `UPDATE_SNAPSHOTS=1` to re-bake the baseline.
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use kebab_chunk::CodeKotlinAstV1Chunker;
|
||||
use kebab_core::{
|
||||
AssetId, Block, CanonicalDocument, ChunkPolicy, Chunker, ChunkerVersion, CodeBlock, CommonBlock,
|
||||
Lang, Metadata, ParserVersion, Provenance, SourceSpan, SourceType, TrustLevel, WorkspacePath,
|
||||
id_for_block, id_for_doc,
|
||||
};
|
||||
use serde_json::Value;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
fn fixtures_dir() -> PathBuf {
|
||||
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("tests")
|
||||
.join("fixtures")
|
||||
}
|
||||
|
||||
fn fixed_doc() -> CanonicalDocument {
|
||||
let wp = WorkspacePath("src/main/kotlin/com/example/Metrics.kt".into());
|
||||
let aid = AssetId("b".repeat(64));
|
||||
// Pin parser_version so doc_id / block_ids are reproducible.
|
||||
let pv = ParserVersion("code-kotlin-v1".into());
|
||||
let doc_id = id_for_doc(&wp, &aid, &pv);
|
||||
|
||||
// Build a >200-line function body to force split_oversize.
|
||||
let big_body: String = {
|
||||
let header = "class BigCompute {\n fun compute(data: IntArray): Int {\n";
|
||||
let body: String = (0..210u32)
|
||||
.map(|i| format!(" val v{i} = if ({i} < data.size) data[{i}] else 0\n"))
|
||||
.collect();
|
||||
let footer = " return data.size\n }\n}";
|
||||
format!("{header}{body}{footer}")
|
||||
};
|
||||
let big_line_count = big_body.lines().count() as u32;
|
||||
let big_line_end = 48 + big_line_count - 1;
|
||||
|
||||
// Representative units:
|
||||
// 0. import block (lines 1–5, ≤200)
|
||||
// 1. top-level fn `computeMRR` (lines 7–12, ≤200)
|
||||
// 2. data class `MetricsCollector` (lines 14–20, ≤200)
|
||||
// 3. class `BaseEvaluator` (lines 22–30, ≤200)
|
||||
// 4. method `MetricsCollector.run` (lines 32–38, ≤200)
|
||||
// 5. method `MetricsCollector.report` (lines 40–46, ≤200)
|
||||
// 6. BigCompute (>200 lines) to force split_oversize
|
||||
let raw_units: Vec<(&str, u32, u32, String)> = vec![
|
||||
(
|
||||
"imports",
|
||||
1,
|
||||
5,
|
||||
"import kotlin.collections.List\nimport kotlin.collections.Map\nimport kotlin.collections.MutableList\nimport kotlin.collections.MutableMap\nimport kotlin.collections.mutableListOf".to_string(),
|
||||
),
|
||||
(
|
||||
"computeMRR",
|
||||
7,
|
||||
12,
|
||||
"fun computeMRR(scores: List<Double>): Double {\n if (scores.isEmpty()) {\n return 0.0\n }\n return 1.0 / scores.size\n}".to_string(),
|
||||
),
|
||||
(
|
||||
"MetricsCollector",
|
||||
14,
|
||||
20,
|
||||
"data class MetricsCollector(\n val scores: MutableList<Double> = mutableListOf(),\n val labels: MutableList<String> = mutableListOf(),\n val counts: MutableMap<String, Int> = mutableMapOf(),\n val totals: MutableMap<String, Double> = mutableMapOf(),\n val tags: MutableList<String> = mutableListOf(),\n)".to_string(),
|
||||
),
|
||||
(
|
||||
"BaseEvaluator",
|
||||
22,
|
||||
30,
|
||||
"open class BaseEvaluator(val name: String) {\n\n fun evaluate(data: List<String>) {\n val joined = data.joinToString(\",\")\n println(joined)\n }\n\n open fun describe(): String = name\n}".to_string(),
|
||||
),
|
||||
(
|
||||
"MetricsCollector.run",
|
||||
32,
|
||||
38,
|
||||
"fun MetricsCollector.run(inputs: List<Double>) {\n for (inp in inputs) {\n scores.add(\n inp\n )\n }\n}".to_string(),
|
||||
),
|
||||
(
|
||||
"MetricsCollector.report",
|
||||
40,
|
||||
46,
|
||||
"fun MetricsCollector.report(): Map<String, Any> {\n return mapOf(\n \"mean\" to 0.0,\n \"count\" to scores.size,\n \"tags\" to tags,\n )\n}".to_string(),
|
||||
),
|
||||
("BigCompute", 48, big_line_end, big_body),
|
||||
];
|
||||
|
||||
let blocks: Vec<Block> = raw_units
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, (sym, ls, le, code))| {
|
||||
let span = SourceSpan::Code {
|
||||
line_start: *ls,
|
||||
line_end: *le,
|
||||
symbol: Some((*sym).to_string()),
|
||||
lang: Some("kotlin".into()),
|
||||
};
|
||||
let bid = id_for_block(&doc_id, "code", &[], i as u32, &span);
|
||||
Block::Code(CodeBlock {
|
||||
common: CommonBlock {
|
||||
block_id: bid,
|
||||
heading_path: vec![],
|
||||
source_span: span,
|
||||
},
|
||||
lang: Some("kotlin".into()),
|
||||
code: code.clone(),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
CanonicalDocument {
|
||||
doc_id,
|
||||
source_asset_id: aid,
|
||||
workspace_path: wp,
|
||||
title: "Metrics.kt".into(),
|
||||
lang: Lang("und".into()),
|
||||
blocks,
|
||||
metadata: Metadata {
|
||||
aliases: vec![],
|
||||
tags: vec![],
|
||||
created_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
updated_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
source_type: SourceType::Note,
|
||||
trust_level: TrustLevel::Primary,
|
||||
user_id_alias: None,
|
||||
user: Default::default(),
|
||||
repo: Some("kebab".into()),
|
||||
git_branch: Some("main".into()),
|
||||
git_commit: Some("0".repeat(40)),
|
||||
code_lang: Some("kotlin".into()),
|
||||
},
|
||||
provenance: Provenance { events: vec![] },
|
||||
parser_version: pv,
|
||||
schema_version: 1,
|
||||
doc_version: 1,
|
||||
last_chunker_version: None,
|
||||
last_embedding_version: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn fixed_policy() -> ChunkPolicy {
|
||||
ChunkPolicy {
|
||||
target_tokens: 500,
|
||||
overlap_tokens: 80,
|
||||
respect_markdown_headings: false,
|
||||
chunker_version: ChunkerVersion("code-kotlin-ast-v1".into()),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn code_kotlin_ast_chunks_snapshot() {
|
||||
let doc = fixed_doc();
|
||||
let policy = fixed_policy();
|
||||
|
||||
let chunks = CodeKotlinAstV1Chunker.chunk(&doc, &policy).expect("chunk");
|
||||
let actual = serde_json::to_value(&chunks).unwrap();
|
||||
|
||||
let dir = fixtures_dir();
|
||||
let baseline_path = dir.join("code-sample.kt.chunks.snapshot.json");
|
||||
let baseline_text = match std::fs::read_to_string(&baseline_path) {
|
||||
Ok(s) => s,
|
||||
Err(_) if std::env::var("UPDATE_SNAPSHOTS").is_ok() => {
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let pretty = serde_json::to_string_pretty(&actual).unwrap();
|
||||
std::fs::write(&baseline_path, format!("{pretty}\n")).unwrap();
|
||||
return;
|
||||
}
|
||||
Err(e) => panic!(
|
||||
"missing baseline {}; run with UPDATE_SNAPSHOTS=1 to create: {e}",
|
||||
baseline_path.display()
|
||||
),
|
||||
};
|
||||
let expected: Value = serde_json::from_str(&baseline_text).expect("baseline parses as json");
|
||||
|
||||
if actual != expected {
|
||||
if std::env::var("UPDATE_SNAPSHOTS").is_ok() {
|
||||
let pretty = serde_json::to_string_pretty(&actual).unwrap();
|
||||
std::fs::write(&baseline_path, format!("{pretty}\n")).unwrap();
|
||||
eprintln!("updated baseline {}", baseline_path.display());
|
||||
return;
|
||||
}
|
||||
let pretty = serde_json::to_string_pretty(&actual).unwrap();
|
||||
panic!(
|
||||
"code-kotlin-ast-v1 chunks snapshot drift\n\
|
||||
--- expected ({}) ---\n{baseline_text}\n\
|
||||
--- actual ---\n{pretty}\n\
|
||||
If intentional, re-run with UPDATE_SNAPSHOTS=1.",
|
||||
baseline_path.display()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Determinism cross-check: re-running the same pipeline yields the same
|
||||
/// chunk_ids byte-for-byte.
|
||||
#[test]
|
||||
fn code_kotlin_ast_chunks_are_deterministic() {
|
||||
let policy = fixed_policy();
|
||||
let baseline: Vec<String> = CodeKotlinAstV1Chunker
|
||||
.chunk(&fixed_doc(), &policy)
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|c| c.chunk_id.0)
|
||||
.collect();
|
||||
for _ in 0..5 {
|
||||
let again: Vec<String> = CodeKotlinAstV1Chunker
|
||||
.chunk(&fixed_doc(), &policy)
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|c| c.chunk_id.0)
|
||||
.collect();
|
||||
assert_eq!(again, baseline);
|
||||
}
|
||||
}
|
||||
221
crates/kebab-chunk/tests/code_python_ast_snapshot.rs
Normal file
221
crates/kebab-chunk/tests/code_python_ast_snapshot.rs
Normal file
@@ -0,0 +1,221 @@
|
||||
//! Snapshot test pinning the `Vec<Chunk>` JSON for a
|
||||
//! representative Python code `CanonicalDocument`.
|
||||
//!
|
||||
//! This is an integration test. `kebab-parse-code` is intentionally NOT
|
||||
//! a dev-dep (design §6.3 / §8 boundary: AST extraction is parser-side).
|
||||
//! The `CanonicalDocument` is built inline from hand-crafted `Block::Code`
|
||||
//! units, which is the same pattern used in `code_rust_ast_v1.rs`'s
|
||||
//! internal `code_doc` test helper.
|
||||
//!
|
||||
//! Set `UPDATE_SNAPSHOTS=1` to re-bake the baseline.
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use kebab_chunk::CodePythonAstV1Chunker;
|
||||
use kebab_core::{
|
||||
AssetId, Block, CanonicalDocument, ChunkPolicy, Chunker, ChunkerVersion, CodeBlock, CommonBlock,
|
||||
Lang, Metadata, ParserVersion, Provenance, SourceSpan, SourceType, TrustLevel, WorkspacePath,
|
||||
id_for_block, id_for_doc,
|
||||
};
|
||||
use serde_json::Value;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
fn fixtures_dir() -> PathBuf {
|
||||
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("tests")
|
||||
.join("fixtures")
|
||||
}
|
||||
|
||||
fn fixed_doc() -> CanonicalDocument {
|
||||
let wp = WorkspacePath("kebab_eval/metrics.py".into());
|
||||
let aid = AssetId("b".repeat(64));
|
||||
// Pin parser_version so doc_id / block_ids are reproducible.
|
||||
let pv = ParserVersion("code-python-v1".into());
|
||||
let doc_id = id_for_doc(&wp, &aid, &pv);
|
||||
|
||||
// Build a >200-line function body to force split_oversize.
|
||||
let big_body: String = {
|
||||
let header = "def big_compute(data):\n";
|
||||
let body: String = (0..210u32)
|
||||
.map(|i| format!(" v{i} = data[{i}] if {i} < len(data) else 0\n"))
|
||||
.collect();
|
||||
let footer = " return sum(data)";
|
||||
format!("{header}{body}{footer}")
|
||||
};
|
||||
let big_line_count = big_body.lines().count() as u32;
|
||||
let big_line_end = 48 + big_line_count - 1;
|
||||
|
||||
// Representative units:
|
||||
// 0. import block (lines 1–5, ≤200)
|
||||
// 1. free fn `compute_mrr` (lines 7–12, ≤200)
|
||||
// 2. class `MetricsCollector` (lines 14–20, ≤200)
|
||||
// 3. class `BaseEvaluator` (lines 22–30, ≤200)
|
||||
// 4. method `run` (lines 32–38, ≤200)
|
||||
// 5. method `report` (lines 40–46, ≤200)
|
||||
// 6. big_compute (>200 lines) to force split_oversize
|
||||
let raw_units: Vec<(&str, u32, u32, String)> = vec![
|
||||
(
|
||||
"imports",
|
||||
1,
|
||||
5,
|
||||
"import os\nimport sys\nfrom typing import List\nfrom pathlib import Path\nfrom collections import defaultdict".to_string(),
|
||||
),
|
||||
(
|
||||
"compute_mrr",
|
||||
7,
|
||||
12,
|
||||
"def compute_mrr(scores):\n if not scores:\n return 0.0\n return sum(\n 1.0 / r for r in scores\n ) / len(scores)".to_string(),
|
||||
),
|
||||
(
|
||||
"MetricsCollector",
|
||||
14,
|
||||
20,
|
||||
"class MetricsCollector:\n def __init__(self):\n self.scores = []\n self.labels = []\n self.counts = defaultdict(int)\n self.totals = defaultdict(float)\n self.tags = []".to_string(),
|
||||
),
|
||||
(
|
||||
"BaseEvaluator",
|
||||
22,
|
||||
30,
|
||||
"class BaseEvaluator:\n def evaluate(self, data):\n raise NotImplementedError\n def batch_evaluate(self, items):\n results = []\n for item in items:\n results.append(self.evaluate(item))\n return results\n def name(self):\n return type(self).__name__".to_string(),
|
||||
),
|
||||
(
|
||||
"MetricsCollector.run",
|
||||
32,
|
||||
38,
|
||||
"class MetricsCollector:\n def run(self, inputs):\n for inp in inputs:\n score = self._score(inp)\n self.scores.append(\n score\n )".to_string(),
|
||||
),
|
||||
(
|
||||
"MetricsCollector.report",
|
||||
40,
|
||||
46,
|
||||
"class MetricsCollector:\n def report(self):\n return {\n 'mean': sum(self.scores) / max(len(self.scores), 1),\n 'count': len(self.scores),\n 'tags': self.tags,\n }".to_string(),
|
||||
),
|
||||
("big_compute", 48, big_line_end, big_body),
|
||||
];
|
||||
|
||||
let blocks: Vec<Block> = raw_units
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, (sym, ls, le, code))| {
|
||||
let span = SourceSpan::Code {
|
||||
line_start: *ls,
|
||||
line_end: *le,
|
||||
symbol: Some((*sym).to_string()),
|
||||
lang: Some("python".into()),
|
||||
};
|
||||
let bid = id_for_block(&doc_id, "code", &[], i as u32, &span);
|
||||
Block::Code(CodeBlock {
|
||||
common: CommonBlock {
|
||||
block_id: bid,
|
||||
heading_path: vec![],
|
||||
source_span: span,
|
||||
},
|
||||
lang: Some("python".into()),
|
||||
code: code.clone(),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
CanonicalDocument {
|
||||
doc_id,
|
||||
source_asset_id: aid,
|
||||
workspace_path: wp,
|
||||
title: "metrics.py".into(),
|
||||
lang: Lang("und".into()),
|
||||
blocks,
|
||||
metadata: Metadata {
|
||||
aliases: vec![],
|
||||
tags: vec![],
|
||||
created_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
updated_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
source_type: SourceType::Note,
|
||||
trust_level: TrustLevel::Primary,
|
||||
user_id_alias: None,
|
||||
user: Default::default(),
|
||||
repo: Some("kebab".into()),
|
||||
git_branch: Some("main".into()),
|
||||
git_commit: Some("0".repeat(40)),
|
||||
code_lang: Some("python".into()),
|
||||
},
|
||||
provenance: Provenance { events: vec![] },
|
||||
parser_version: pv,
|
||||
schema_version: 1,
|
||||
doc_version: 1,
|
||||
last_chunker_version: None,
|
||||
last_embedding_version: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn fixed_policy() -> ChunkPolicy {
|
||||
ChunkPolicy {
|
||||
target_tokens: 500,
|
||||
overlap_tokens: 80,
|
||||
respect_markdown_headings: false,
|
||||
chunker_version: ChunkerVersion("code-python-ast-v1".into()),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn code_python_ast_chunks_snapshot() {
|
||||
let doc = fixed_doc();
|
||||
let policy = fixed_policy();
|
||||
|
||||
let chunks = CodePythonAstV1Chunker.chunk(&doc, &policy).expect("chunk");
|
||||
let actual = serde_json::to_value(&chunks).unwrap();
|
||||
|
||||
let dir = fixtures_dir();
|
||||
let baseline_path = dir.join("code-sample.py.chunks.snapshot.json");
|
||||
let baseline_text = match std::fs::read_to_string(&baseline_path) {
|
||||
Ok(s) => s,
|
||||
Err(_) if std::env::var("UPDATE_SNAPSHOTS").is_ok() => {
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let pretty = serde_json::to_string_pretty(&actual).unwrap();
|
||||
std::fs::write(&baseline_path, format!("{pretty}\n")).unwrap();
|
||||
return;
|
||||
}
|
||||
Err(e) => panic!(
|
||||
"missing baseline {}; run with UPDATE_SNAPSHOTS=1 to create: {e}",
|
||||
baseline_path.display()
|
||||
),
|
||||
};
|
||||
let expected: Value = serde_json::from_str(&baseline_text).expect("baseline parses as json");
|
||||
|
||||
if actual != expected {
|
||||
if std::env::var("UPDATE_SNAPSHOTS").is_ok() {
|
||||
let pretty = serde_json::to_string_pretty(&actual).unwrap();
|
||||
std::fs::write(&baseline_path, format!("{pretty}\n")).unwrap();
|
||||
eprintln!("updated baseline {}", baseline_path.display());
|
||||
return;
|
||||
}
|
||||
let pretty = serde_json::to_string_pretty(&actual).unwrap();
|
||||
panic!(
|
||||
"code-python-ast-v1 chunks snapshot drift\n\
|
||||
--- expected ({}) ---\n{baseline_text}\n\
|
||||
--- actual ---\n{pretty}\n\
|
||||
If intentional, re-run with UPDATE_SNAPSHOTS=1.",
|
||||
baseline_path.display()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Determinism cross-check: re-running the same pipeline yields the same
|
||||
/// chunk_ids byte-for-byte.
|
||||
#[test]
|
||||
fn code_python_ast_chunks_are_deterministic() {
|
||||
let policy = fixed_policy();
|
||||
let baseline: Vec<String> = CodePythonAstV1Chunker
|
||||
.chunk(&fixed_doc(), &policy)
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|c| c.chunk_id.0)
|
||||
.collect();
|
||||
for _ in 0..5 {
|
||||
let again: Vec<String> = CodePythonAstV1Chunker
|
||||
.chunk(&fixed_doc(), &policy)
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|c| c.chunk_id.0)
|
||||
.collect();
|
||||
assert_eq!(again, baseline);
|
||||
}
|
||||
}
|
||||
221
crates/kebab-chunk/tests/code_ts_ast_snapshot.rs
Normal file
221
crates/kebab-chunk/tests/code_ts_ast_snapshot.rs
Normal file
@@ -0,0 +1,221 @@
|
||||
//! Snapshot test pinning the `Vec<Chunk>` JSON for a
|
||||
//! representative TypeScript code `CanonicalDocument`.
|
||||
//!
|
||||
//! This is an integration test. `kebab-parse-code` is intentionally NOT
|
||||
//! a dev-dep (design §6.3 / §8 boundary: AST extraction is parser-side).
|
||||
//! The `CanonicalDocument` is built inline from hand-crafted `Block::Code`
|
||||
//! units, which is the same pattern used in `code_rust_ast_v1.rs`'s
|
||||
//! internal `code_doc` test helper.
|
||||
//!
|
||||
//! Set `UPDATE_SNAPSHOTS=1` to re-bake the baseline.
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use kebab_chunk::CodeTsAstV1Chunker;
|
||||
use kebab_core::{
|
||||
AssetId, Block, CanonicalDocument, ChunkPolicy, Chunker, ChunkerVersion, CodeBlock, CommonBlock,
|
||||
Lang, Metadata, ParserVersion, Provenance, SourceSpan, SourceType, TrustLevel, WorkspacePath,
|
||||
id_for_block, id_for_doc,
|
||||
};
|
||||
use serde_json::Value;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
fn fixtures_dir() -> PathBuf {
|
||||
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("tests")
|
||||
.join("fixtures")
|
||||
}
|
||||
|
||||
fn fixed_doc() -> CanonicalDocument {
|
||||
let wp = WorkspacePath("src/Foo.ts".into());
|
||||
let aid = AssetId("b".repeat(64));
|
||||
// Pin parser_version so doc_id / block_ids are reproducible.
|
||||
let pv = ParserVersion("code-ts-v1".into());
|
||||
let doc_id = id_for_doc(&wp, &aid, &pv);
|
||||
|
||||
// Build a >200-line method body to force split_oversize.
|
||||
let big_body: String = {
|
||||
let header = "export class BigProcessor {\n process(items: string[]): string[] {\n";
|
||||
let body: String = (0..210u32)
|
||||
.map(|i| format!(" const v{i} = items[{i}] ?? '';\n"))
|
||||
.collect();
|
||||
let footer = " return items;\n }\n}";
|
||||
format!("{header}{body}{footer}")
|
||||
};
|
||||
let big_line_count = big_body.lines().count() as u32;
|
||||
let big_line_end = 48 + big_line_count - 1;
|
||||
|
||||
// Representative units:
|
||||
// 0. import block (lines 1–5, ≤200)
|
||||
// 1. free fn `parseInput` (lines 7–12, ≤200)
|
||||
// 2. interface `Frobable` (lines 14–20, ≤200)
|
||||
// 3. class `Foo` (lines 22–30, ≤200)
|
||||
// 4. method `Foo.double` (lines 32–38, ≤200)
|
||||
// 5. method `Foo.triple` (lines 40–46, ≤200)
|
||||
// 6. BigProcessor (>200 lines) to force split_oversize
|
||||
let raw_units: Vec<(&str, u32, u32, String)> = vec![
|
||||
(
|
||||
"imports",
|
||||
1,
|
||||
5,
|
||||
"import { readFileSync } from 'fs';\nimport { join } from 'path';\nimport type { Config } from './config';\nimport { Logger } from './logger';\nimport { EventEmitter } from 'events';".to_string(),
|
||||
),
|
||||
(
|
||||
"parseInput",
|
||||
7,
|
||||
12,
|
||||
"export function parseInput(raw: string): number | null {\n const trimmed = raw.trim();\n const n = Number(trimmed);\n if (isNaN(n)) return null;\n return n;\n}".to_string(),
|
||||
),
|
||||
(
|
||||
"Frobable",
|
||||
14,
|
||||
20,
|
||||
"export interface Frobable {\n frob(): string;\n frobTwice(): string;\n readonly name: string;\n readonly tags: string[];\n count: number;\n reset(): void;\n}".to_string(),
|
||||
),
|
||||
(
|
||||
"Foo",
|
||||
22,
|
||||
30,
|
||||
"export class Foo implements Frobable {\n constructor(\n public readonly name: string,\n public value: number,\n public tags: string[] = [],\n ) {}\n frob(): string { return this.name; }\n frobTwice(): string { return this.name.repeat(2); }\n reset(): void { this.value = 0; }\n}".to_string(),
|
||||
),
|
||||
(
|
||||
"Foo.double",
|
||||
32,
|
||||
38,
|
||||
"export class Foo {\n double(): number {\n const result = this.value * 2;\n if (result > Number.MAX_SAFE_INTEGER) {\n return Number.MAX_SAFE_INTEGER;\n }\n return result;\n }\n}".to_string(),
|
||||
),
|
||||
(
|
||||
"Foo.triple",
|
||||
40,
|
||||
46,
|
||||
"export class Foo {\n triple(): number {\n const result = this.value * 3;\n if (result > Number.MAX_SAFE_INTEGER) {\n return Number.MAX_SAFE_INTEGER;\n }\n return result;\n }\n}".to_string(),
|
||||
),
|
||||
("BigProcessor", 48, big_line_end, big_body),
|
||||
];
|
||||
|
||||
let blocks: Vec<Block> = raw_units
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, (sym, ls, le, code))| {
|
||||
let span = SourceSpan::Code {
|
||||
line_start: *ls,
|
||||
line_end: *le,
|
||||
symbol: Some((*sym).to_string()),
|
||||
lang: Some("typescript".into()),
|
||||
};
|
||||
let bid = id_for_block(&doc_id, "code", &[], i as u32, &span);
|
||||
Block::Code(CodeBlock {
|
||||
common: CommonBlock {
|
||||
block_id: bid,
|
||||
heading_path: vec![],
|
||||
source_span: span,
|
||||
},
|
||||
lang: Some("typescript".into()),
|
||||
code: code.clone(),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
CanonicalDocument {
|
||||
doc_id,
|
||||
source_asset_id: aid,
|
||||
workspace_path: wp,
|
||||
title: "Foo.ts".into(),
|
||||
lang: Lang("und".into()),
|
||||
blocks,
|
||||
metadata: Metadata {
|
||||
aliases: vec![],
|
||||
tags: vec![],
|
||||
created_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
updated_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
source_type: SourceType::Note,
|
||||
trust_level: TrustLevel::Primary,
|
||||
user_id_alias: None,
|
||||
user: Default::default(),
|
||||
repo: Some("kebab".into()),
|
||||
git_branch: Some("main".into()),
|
||||
git_commit: Some("0".repeat(40)),
|
||||
code_lang: Some("typescript".into()),
|
||||
},
|
||||
provenance: Provenance { events: vec![] },
|
||||
parser_version: pv,
|
||||
schema_version: 1,
|
||||
doc_version: 1,
|
||||
last_chunker_version: None,
|
||||
last_embedding_version: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn fixed_policy() -> ChunkPolicy {
|
||||
ChunkPolicy {
|
||||
target_tokens: 500,
|
||||
overlap_tokens: 80,
|
||||
respect_markdown_headings: false,
|
||||
chunker_version: ChunkerVersion("code-ts-ast-v1".into()),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn code_ts_ast_chunks_snapshot() {
|
||||
let doc = fixed_doc();
|
||||
let policy = fixed_policy();
|
||||
|
||||
let chunks = CodeTsAstV1Chunker.chunk(&doc, &policy).expect("chunk");
|
||||
let actual = serde_json::to_value(&chunks).unwrap();
|
||||
|
||||
let dir = fixtures_dir();
|
||||
let baseline_path = dir.join("code-sample.ts.chunks.snapshot.json");
|
||||
let baseline_text = match std::fs::read_to_string(&baseline_path) {
|
||||
Ok(s) => s,
|
||||
Err(_) if std::env::var("UPDATE_SNAPSHOTS").is_ok() => {
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let pretty = serde_json::to_string_pretty(&actual).unwrap();
|
||||
std::fs::write(&baseline_path, format!("{pretty}\n")).unwrap();
|
||||
return;
|
||||
}
|
||||
Err(e) => panic!(
|
||||
"missing baseline {}; run with UPDATE_SNAPSHOTS=1 to create: {e}",
|
||||
baseline_path.display()
|
||||
),
|
||||
};
|
||||
let expected: Value = serde_json::from_str(&baseline_text).expect("baseline parses as json");
|
||||
|
||||
if actual != expected {
|
||||
if std::env::var("UPDATE_SNAPSHOTS").is_ok() {
|
||||
let pretty = serde_json::to_string_pretty(&actual).unwrap();
|
||||
std::fs::write(&baseline_path, format!("{pretty}\n")).unwrap();
|
||||
eprintln!("updated baseline {}", baseline_path.display());
|
||||
return;
|
||||
}
|
||||
let pretty = serde_json::to_string_pretty(&actual).unwrap();
|
||||
panic!(
|
||||
"code-ts-ast-v1 chunks snapshot drift\n\
|
||||
--- expected ({}) ---\n{baseline_text}\n\
|
||||
--- actual ---\n{pretty}\n\
|
||||
If intentional, re-run with UPDATE_SNAPSHOTS=1.",
|
||||
baseline_path.display()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Determinism cross-check: re-running the same pipeline yields the same
|
||||
/// chunk_ids byte-for-byte.
|
||||
#[test]
|
||||
fn code_ts_ast_chunks_are_deterministic() {
|
||||
let policy = fixed_policy();
|
||||
let baseline: Vec<String> = CodeTsAstV1Chunker
|
||||
.chunk(&fixed_doc(), &policy)
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|c| c.chunk_id.0)
|
||||
.collect();
|
||||
for _ in 0..5 {
|
||||
let again: Vec<String> = CodeTsAstV1Chunker
|
||||
.chunk(&fixed_doc(), &policy)
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|c| c.chunk_id.0)
|
||||
.collect();
|
||||
assert_eq!(again, baseline);
|
||||
}
|
||||
}
|
||||
233
crates/kebab-chunk/tests/fixtures/code-sample.go.chunks.snapshot.json
vendored
Normal file
233
crates/kebab-chunk/tests/fixtures/code-sample.go.chunks.snapshot.json
vendored
Normal file
@@ -0,0 +1,233 @@
|
||||
[
|
||||
{
|
||||
"block_ids": [
|
||||
"c182bf37e32c7fc1b868bd617f8eaf66"
|
||||
],
|
||||
"chunk_id": "43de518d946dc18ec040ae20d74e0cff",
|
||||
"chunker_version": "code-go-ast-v1",
|
||||
"doc_id": "83daba5fbb026e7a400d68a1c4bd36db",
|
||||
"heading_path": [],
|
||||
"policy_hash": "6cfe77abe2b0e5c3",
|
||||
"source_spans": [
|
||||
{
|
||||
"kind": "code",
|
||||
"lang": "go",
|
||||
"line_end": 5,
|
||||
"line_start": 1,
|
||||
"symbol": "imports"
|
||||
}
|
||||
],
|
||||
"text": "import (\n\t\"fmt\"\n\t\"os\"\n\t\"strings\"\n)",
|
||||
"token_estimate": 12
|
||||
},
|
||||
{
|
||||
"block_ids": [
|
||||
"c9992cdcfdf3c2a7700a4abc4782a8a4"
|
||||
],
|
||||
"chunk_id": "af4c382a83f1e8cdea495d8b33c11abc",
|
||||
"chunker_version": "code-go-ast-v1",
|
||||
"doc_id": "83daba5fbb026e7a400d68a1c4bd36db",
|
||||
"heading_path": [],
|
||||
"policy_hash": "6cfe77abe2b0e5c3",
|
||||
"source_spans": [
|
||||
{
|
||||
"kind": "code",
|
||||
"lang": "go",
|
||||
"line_end": 12,
|
||||
"line_start": 7,
|
||||
"symbol": "ComputeMRR"
|
||||
}
|
||||
],
|
||||
"text": "func ComputeMRR(scores []float64) float64 {\n\tif len(scores) == 0 {\n\t\treturn 0.0\n\t}\n\t_ = fmt.Sprintf(\"%v\", scores)\n\treturn 1.0 / float64(len(scores))\n}",
|
||||
"token_estimate": 50
|
||||
},
|
||||
{
|
||||
"block_ids": [
|
||||
"5f18dc3e79fe946ba05d32c3bfc00684"
|
||||
],
|
||||
"chunk_id": "4be6d8f180bc19b8651877e5264852ac",
|
||||
"chunker_version": "code-go-ast-v1",
|
||||
"doc_id": "83daba5fbb026e7a400d68a1c4bd36db",
|
||||
"heading_path": [],
|
||||
"policy_hash": "6cfe77abe2b0e5c3",
|
||||
"source_spans": [
|
||||
{
|
||||
"kind": "code",
|
||||
"lang": "go",
|
||||
"line_end": 20,
|
||||
"line_start": 14,
|
||||
"symbol": "MetricsCollector"
|
||||
}
|
||||
],
|
||||
"text": "type MetricsCollector struct {\n\tScores []float64\n\tLabels []string\n\tCounts map[string]int\n\tTotals map[string]float64\n\tTags []string\n}",
|
||||
"token_estimate": 45
|
||||
},
|
||||
{
|
||||
"block_ids": [
|
||||
"3009cc022ca832c323393e4f9bcdb388"
|
||||
],
|
||||
"chunk_id": "3ae182f4c6d304ee7f0aaf447142f948",
|
||||
"chunker_version": "code-go-ast-v1",
|
||||
"doc_id": "83daba5fbb026e7a400d68a1c4bd36db",
|
||||
"heading_path": [],
|
||||
"policy_hash": "6cfe77abe2b0e5c3",
|
||||
"source_spans": [
|
||||
{
|
||||
"kind": "code",
|
||||
"lang": "go",
|
||||
"line_end": 30,
|
||||
"line_start": 22,
|
||||
"symbol": "BaseEvaluator"
|
||||
}
|
||||
],
|
||||
"text": "type BaseEvaluator struct {\n\tName string\n}\n\nfunc (e *BaseEvaluator) Evaluate(data []string) error {\n\t_ = os.Stderr\n\t_ = strings.Join(data, \",\")\n\treturn nil\n}",
|
||||
"token_estimate": 53
|
||||
},
|
||||
{
|
||||
"block_ids": [
|
||||
"e0e83d1d7f9327a1902ae9a8f67c1f1c"
|
||||
],
|
||||
"chunk_id": "b962f14980e756bb8ba514e2282756cd",
|
||||
"chunker_version": "code-go-ast-v1",
|
||||
"doc_id": "83daba5fbb026e7a400d68a1c4bd36db",
|
||||
"heading_path": [],
|
||||
"policy_hash": "6cfe77abe2b0e5c3",
|
||||
"source_spans": [
|
||||
{
|
||||
"kind": "code",
|
||||
"lang": "go",
|
||||
"line_end": 38,
|
||||
"line_start": 32,
|
||||
"symbol": "MetricsCollector.Run"
|
||||
}
|
||||
],
|
||||
"text": "func (m *MetricsCollector) Run(inputs []float64) {\n\tfor _, inp := range inputs {\n\t\tm.Scores = append(\n\t\t\tm.Scores,\n\t\t\tinp,\n\t\t)\n\t}\n}",
|
||||
"token_estimate": 44
|
||||
},
|
||||
{
|
||||
"block_ids": [
|
||||
"0e6a572bc3fe2bd6d173fe614bd1b763"
|
||||
],
|
||||
"chunk_id": "441c695e990e7f49188068433e313e87",
|
||||
"chunker_version": "code-go-ast-v1",
|
||||
"doc_id": "83daba5fbb026e7a400d68a1c4bd36db",
|
||||
"heading_path": [],
|
||||
"policy_hash": "6cfe77abe2b0e5c3",
|
||||
"source_spans": [
|
||||
{
|
||||
"kind": "code",
|
||||
"lang": "go",
|
||||
"line_end": 46,
|
||||
"line_start": 40,
|
||||
"symbol": "MetricsCollector.Report"
|
||||
}
|
||||
],
|
||||
"text": "func (m *MetricsCollector) Report() map[string]interface{} {\n\treturn map[string]interface{}{\n\t\t\"mean\": 0.0,\n\t\t\"count\": len(m.Scores),\n\t\t\"tags\": m.Tags,\n\t}\n}",
|
||||
"token_estimate": 53
|
||||
},
|
||||
{
|
||||
"block_ids": [
|
||||
"5d269745b2e5dbdcbef0c09ba54b0bd6"
|
||||
],
|
||||
"chunk_id": "7a942d871c588ec69426290561f05179",
|
||||
"chunker_version": "code-go-ast-v1",
|
||||
"doc_id": "83daba5fbb026e7a400d68a1c4bd36db",
|
||||
"heading_path": [],
|
||||
"policy_hash": "6cfe77abe2b0e5c3",
|
||||
"source_spans": [
|
||||
{
|
||||
"kind": "code",
|
||||
"lang": "go",
|
||||
"line_end": 247,
|
||||
"line_start": 48,
|
||||
"symbol": "BigCompute [part 1/5]"
|
||||
}
|
||||
],
|
||||
"text": "func BigCompute(data []int) int {\n\tv0 := 0\n\tif 0 < len(data) {\n\t\tv0 = data[0]\n\t}\n\tv1 := 0\n\tif 1 < len(data) {\n\t\tv1 = data[1]\n\t}\n\tv2 := 0\n\tif 2 < len(data) {\n\t\tv2 = data[2]\n\t}\n\tv3 := 0\n\tif 3 < len(data) {\n\t\tv3 = data[3]\n\t}\n\tv4 := 0\n\tif 4 < len(data) {\n\t\tv4 = data[4]\n\t}\n\tv5 := 0\n\tif 5 < len(data) {\n\t\tv5 = data[5]\n\t}\n\tv6 := 0\n\tif 6 < len(data) {\n\t\tv6 = data[6]\n\t}\n\tv7 := 0\n\tif 7 < len(data) {\n\t\tv7 = data[7]\n\t}\n\tv8 := 0\n\tif 8 < len(data) {\n\t\tv8 = data[8]\n\t}\n\tv9 := 0\n\tif 9 < len(data) {\n\t\tv9 = data[9]\n\t}\n\tv10 := 0\n\tif 10 < len(data) {\n\t\tv10 = data[10]\n\t}\n\tv11 := 0\n\tif 11 < len(data) {\n\t\tv11 = data[11]\n\t}\n\tv12 := 0\n\tif 12 < len(data) {\n\t\tv12 = data[12]\n\t}\n\tv13 := 0\n\tif 13 < len(data) {\n\t\tv13 = data[13]\n\t}\n\tv14 := 0\n\tif 14 < len(data) {\n\t\tv14 = data[14]\n\t}\n\tv15 := 0\n\tif 15 < len(data) {\n\t\tv15 = data[15]\n\t}\n\tv16 := 0\n\tif 16 < len(data) {\n\t\tv16 = data[16]\n\t}\n\tv17 := 0\n\tif 17 < len(data) {\n\t\tv17 = data[17]\n\t}\n\tv18 := 0\n\tif 18 < len(data) {\n\t\tv18 = data[18]\n\t}\n\tv19 := 0\n\tif 19 < len(data) {\n\t\tv19 = data[19]\n\t}\n\tv20 := 0\n\tif 20 < len(data) {\n\t\tv20 = data[20]\n\t}\n\tv21 := 0\n\tif 21 < len(data) {\n\t\tv21 = data[21]\n\t}\n\tv22 := 0\n\tif 22 < len(data) {\n\t\tv22 = data[22]\n\t}\n\tv23 := 0\n\tif 23 < len(data) {\n\t\tv23 = data[23]\n\t}\n\tv24 := 0\n\tif 24 < len(data) {\n\t\tv24 = data[24]\n\t}\n\tv25 := 0\n\tif 25 < len(data) {\n\t\tv25 = data[25]\n\t}\n\tv26 := 0\n\tif 26 < len(data) {\n\t\tv26 = data[26]\n\t}\n\tv27 := 0\n\tif 27 < len(data) {\n\t\tv27 = data[27]\n\t}\n\tv28 := 0\n\tif 28 < len(data) {\n\t\tv28 = data[28]\n\t}\n\tv29 := 0\n\tif 29 < len(data) {\n\t\tv29 = data[29]\n\t}\n\tv30 := 0\n\tif 30 < len(data) {\n\t\tv30 = data[30]\n\t}\n\tv31 := 0\n\tif 31 < len(data) {\n\t\tv31 = data[31]\n\t}\n\tv32 := 0\n\tif 32 < len(data) {\n\t\tv32 = data[32]\n\t}\n\tv33 := 0\n\tif 33 < len(data) {\n\t\tv33 = data[33]\n\t}\n\tv34 := 0\n\tif 34 < len(data) {\n\t\tv34 = data[34]\n\t}\n\tv35 := 0\n\tif 35 < len(data) {\n\t\tv35 = data[35]\n\t}\n\tv36 := 0\n\tif 36 < len(data) {\n\t\tv36 = data[36]\n\t}\n\tv37 := 0\n\tif 37 < len(data) {\n\t\tv37 = data[37]\n\t}\n\tv38 := 0\n\tif 38 < len(data) {\n\t\tv38 = data[38]\n\t}\n\tv39 := 0\n\tif 39 < len(data) {\n\t\tv39 = data[39]\n\t}\n\tv40 := 0\n\tif 40 < len(data) {\n\t\tv40 = data[40]\n\t}\n\tv41 := 0\n\tif 41 < len(data) {\n\t\tv41 = data[41]\n\t}\n\tv42 := 0\n\tif 42 < len(data) {\n\t\tv42 = data[42]\n\t}\n\tv43 := 0\n\tif 43 < len(data) {\n\t\tv43 = data[43]\n\t}\n\tv44 := 0\n\tif 44 < len(data) {\n\t\tv44 = data[44]\n\t}\n\tv45 := 0\n\tif 45 < len(data) {\n\t\tv45 = data[45]\n\t}\n\tv46 := 0\n\tif 46 < len(data) {\n\t\tv46 = data[46]\n\t}\n\tv47 := 0\n\tif 47 < len(data) {\n\t\tv47 = data[47]\n\t}\n\tv48 := 0\n\tif 48 < len(data) {\n\t\tv48 = data[48]\n\t}\n\tv49 := 0\n\tif 49 < len(data) {\n\t\tv49 = data[49]",
|
||||
"token_estimate": 847
|
||||
},
|
||||
{
|
||||
"block_ids": [
|
||||
"5d269745b2e5dbdcbef0c09ba54b0bd6"
|
||||
],
|
||||
"chunk_id": "3f44ba43c9415652e2705bb667776e76",
|
||||
"chunker_version": "code-go-ast-v1",
|
||||
"doc_id": "83daba5fbb026e7a400d68a1c4bd36db",
|
||||
"heading_path": [],
|
||||
"policy_hash": "6cfe77abe2b0e5c3",
|
||||
"source_spans": [
|
||||
{
|
||||
"kind": "code",
|
||||
"lang": "go",
|
||||
"line_end": 447,
|
||||
"line_start": 248,
|
||||
"symbol": "BigCompute [part 2/5]"
|
||||
}
|
||||
],
|
||||
"text": "\t}\n\tv50 := 0\n\tif 50 < len(data) {\n\t\tv50 = data[50]\n\t}\n\tv51 := 0\n\tif 51 < len(data) {\n\t\tv51 = data[51]\n\t}\n\tv52 := 0\n\tif 52 < len(data) {\n\t\tv52 = data[52]\n\t}\n\tv53 := 0\n\tif 53 < len(data) {\n\t\tv53 = data[53]\n\t}\n\tv54 := 0\n\tif 54 < len(data) {\n\t\tv54 = data[54]\n\t}\n\tv55 := 0\n\tif 55 < len(data) {\n\t\tv55 = data[55]\n\t}\n\tv56 := 0\n\tif 56 < len(data) {\n\t\tv56 = data[56]\n\t}\n\tv57 := 0\n\tif 57 < len(data) {\n\t\tv57 = data[57]\n\t}\n\tv58 := 0\n\tif 58 < len(data) {\n\t\tv58 = data[58]\n\t}\n\tv59 := 0\n\tif 59 < len(data) {\n\t\tv59 = data[59]\n\t}\n\tv60 := 0\n\tif 60 < len(data) {\n\t\tv60 = data[60]\n\t}\n\tv61 := 0\n\tif 61 < len(data) {\n\t\tv61 = data[61]\n\t}\n\tv62 := 0\n\tif 62 < len(data) {\n\t\tv62 = data[62]\n\t}\n\tv63 := 0\n\tif 63 < len(data) {\n\t\tv63 = data[63]\n\t}\n\tv64 := 0\n\tif 64 < len(data) {\n\t\tv64 = data[64]\n\t}\n\tv65 := 0\n\tif 65 < len(data) {\n\t\tv65 = data[65]\n\t}\n\tv66 := 0\n\tif 66 < len(data) {\n\t\tv66 = data[66]\n\t}\n\tv67 := 0\n\tif 67 < len(data) {\n\t\tv67 = data[67]\n\t}\n\tv68 := 0\n\tif 68 < len(data) {\n\t\tv68 = data[68]\n\t}\n\tv69 := 0\n\tif 69 < len(data) {\n\t\tv69 = data[69]\n\t}\n\tv70 := 0\n\tif 70 < len(data) {\n\t\tv70 = data[70]\n\t}\n\tv71 := 0\n\tif 71 < len(data) {\n\t\tv71 = data[71]\n\t}\n\tv72 := 0\n\tif 72 < len(data) {\n\t\tv72 = data[72]\n\t}\n\tv73 := 0\n\tif 73 < len(data) {\n\t\tv73 = data[73]\n\t}\n\tv74 := 0\n\tif 74 < len(data) {\n\t\tv74 = data[74]\n\t}\n\tv75 := 0\n\tif 75 < len(data) {\n\t\tv75 = data[75]\n\t}\n\tv76 := 0\n\tif 76 < len(data) {\n\t\tv76 = data[76]\n\t}\n\tv77 := 0\n\tif 77 < len(data) {\n\t\tv77 = data[77]\n\t}\n\tv78 := 0\n\tif 78 < len(data) {\n\t\tv78 = data[78]\n\t}\n\tv79 := 0\n\tif 79 < len(data) {\n\t\tv79 = data[79]\n\t}\n\tv80 := 0\n\tif 80 < len(data) {\n\t\tv80 = data[80]\n\t}\n\tv81 := 0\n\tif 81 < len(data) {\n\t\tv81 = data[81]\n\t}\n\tv82 := 0\n\tif 82 < len(data) {\n\t\tv82 = data[82]\n\t}\n\tv83 := 0\n\tif 83 < len(data) {\n\t\tv83 = data[83]\n\t}\n\tv84 := 0\n\tif 84 < len(data) {\n\t\tv84 = data[84]\n\t}\n\tv85 := 0\n\tif 85 < len(data) {\n\t\tv85 = data[85]\n\t}\n\tv86 := 0\n\tif 86 < len(data) {\n\t\tv86 = data[86]\n\t}\n\tv87 := 0\n\tif 87 < len(data) {\n\t\tv87 = data[87]\n\t}\n\tv88 := 0\n\tif 88 < len(data) {\n\t\tv88 = data[88]\n\t}\n\tv89 := 0\n\tif 89 < len(data) {\n\t\tv89 = data[89]\n\t}\n\tv90 := 0\n\tif 90 < len(data) {\n\t\tv90 = data[90]\n\t}\n\tv91 := 0\n\tif 91 < len(data) {\n\t\tv91 = data[91]\n\t}\n\tv92 := 0\n\tif 92 < len(data) {\n\t\tv92 = data[92]\n\t}\n\tv93 := 0\n\tif 93 < len(data) {\n\t\tv93 = data[93]\n\t}\n\tv94 := 0\n\tif 94 < len(data) {\n\t\tv94 = data[94]\n\t}\n\tv95 := 0\n\tif 95 < len(data) {\n\t\tv95 = data[95]\n\t}\n\tv96 := 0\n\tif 96 < len(data) {\n\t\tv96 = data[96]\n\t}\n\tv97 := 0\n\tif 97 < len(data) {\n\t\tv97 = data[97]\n\t}\n\tv98 := 0\n\tif 98 < len(data) {\n\t\tv98 = data[98]\n\t}\n\tv99 := 0\n\tif 99 < len(data) {\n\t\tv99 = data[99]",
|
||||
"token_estimate": 850
|
||||
},
|
||||
{
|
||||
"block_ids": [
|
||||
"5d269745b2e5dbdcbef0c09ba54b0bd6"
|
||||
],
|
||||
"chunk_id": "e4763e10f059d97f40c2932761b56c3e",
|
||||
"chunker_version": "code-go-ast-v1",
|
||||
"doc_id": "83daba5fbb026e7a400d68a1c4bd36db",
|
||||
"heading_path": [],
|
||||
"policy_hash": "6cfe77abe2b0e5c3",
|
||||
"source_spans": [
|
||||
{
|
||||
"kind": "code",
|
||||
"lang": "go",
|
||||
"line_end": 647,
|
||||
"line_start": 448,
|
||||
"symbol": "BigCompute [part 3/5]"
|
||||
}
|
||||
],
|
||||
"text": "\t}\n\tv100 := 0\n\tif 100 < len(data) {\n\t\tv100 = data[100]\n\t}\n\tv101 := 0\n\tif 101 < len(data) {\n\t\tv101 = data[101]\n\t}\n\tv102 := 0\n\tif 102 < len(data) {\n\t\tv102 = data[102]\n\t}\n\tv103 := 0\n\tif 103 < len(data) {\n\t\tv103 = data[103]\n\t}\n\tv104 := 0\n\tif 104 < len(data) {\n\t\tv104 = data[104]\n\t}\n\tv105 := 0\n\tif 105 < len(data) {\n\t\tv105 = data[105]\n\t}\n\tv106 := 0\n\tif 106 < len(data) {\n\t\tv106 = data[106]\n\t}\n\tv107 := 0\n\tif 107 < len(data) {\n\t\tv107 = data[107]\n\t}\n\tv108 := 0\n\tif 108 < len(data) {\n\t\tv108 = data[108]\n\t}\n\tv109 := 0\n\tif 109 < len(data) {\n\t\tv109 = data[109]\n\t}\n\tv110 := 0\n\tif 110 < len(data) {\n\t\tv110 = data[110]\n\t}\n\tv111 := 0\n\tif 111 < len(data) {\n\t\tv111 = data[111]\n\t}\n\tv112 := 0\n\tif 112 < len(data) {\n\t\tv112 = data[112]\n\t}\n\tv113 := 0\n\tif 113 < len(data) {\n\t\tv113 = data[113]\n\t}\n\tv114 := 0\n\tif 114 < len(data) {\n\t\tv114 = data[114]\n\t}\n\tv115 := 0\n\tif 115 < len(data) {\n\t\tv115 = data[115]\n\t}\n\tv116 := 0\n\tif 116 < len(data) {\n\t\tv116 = data[116]\n\t}\n\tv117 := 0\n\tif 117 < len(data) {\n\t\tv117 = data[117]\n\t}\n\tv118 := 0\n\tif 118 < len(data) {\n\t\tv118 = data[118]\n\t}\n\tv119 := 0\n\tif 119 < len(data) {\n\t\tv119 = data[119]\n\t}\n\tv120 := 0\n\tif 120 < len(data) {\n\t\tv120 = data[120]\n\t}\n\tv121 := 0\n\tif 121 < len(data) {\n\t\tv121 = data[121]\n\t}\n\tv122 := 0\n\tif 122 < len(data) {\n\t\tv122 = data[122]\n\t}\n\tv123 := 0\n\tif 123 < len(data) {\n\t\tv123 = data[123]\n\t}\n\tv124 := 0\n\tif 124 < len(data) {\n\t\tv124 = data[124]\n\t}\n\tv125 := 0\n\tif 125 < len(data) {\n\t\tv125 = data[125]\n\t}\n\tv126 := 0\n\tif 126 < len(data) {\n\t\tv126 = data[126]\n\t}\n\tv127 := 0\n\tif 127 < len(data) {\n\t\tv127 = data[127]\n\t}\n\tv128 := 0\n\tif 128 < len(data) {\n\t\tv128 = data[128]\n\t}\n\tv129 := 0\n\tif 129 < len(data) {\n\t\tv129 = data[129]\n\t}\n\tv130 := 0\n\tif 130 < len(data) {\n\t\tv130 = data[130]\n\t}\n\tv131 := 0\n\tif 131 < len(data) {\n\t\tv131 = data[131]\n\t}\n\tv132 := 0\n\tif 132 < len(data) {\n\t\tv132 = data[132]\n\t}\n\tv133 := 0\n\tif 133 < len(data) {\n\t\tv133 = data[133]\n\t}\n\tv134 := 0\n\tif 134 < len(data) {\n\t\tv134 = data[134]\n\t}\n\tv135 := 0\n\tif 135 < len(data) {\n\t\tv135 = data[135]\n\t}\n\tv136 := 0\n\tif 136 < len(data) {\n\t\tv136 = data[136]\n\t}\n\tv137 := 0\n\tif 137 < len(data) {\n\t\tv137 = data[137]\n\t}\n\tv138 := 0\n\tif 138 < len(data) {\n\t\tv138 = data[138]\n\t}\n\tv139 := 0\n\tif 139 < len(data) {\n\t\tv139 = data[139]\n\t}\n\tv140 := 0\n\tif 140 < len(data) {\n\t\tv140 = data[140]\n\t}\n\tv141 := 0\n\tif 141 < len(data) {\n\t\tv141 = data[141]\n\t}\n\tv142 := 0\n\tif 142 < len(data) {\n\t\tv142 = data[142]\n\t}\n\tv143 := 0\n\tif 143 < len(data) {\n\t\tv143 = data[143]\n\t}\n\tv144 := 0\n\tif 144 < len(data) {\n\t\tv144 = data[144]\n\t}\n\tv145 := 0\n\tif 145 < len(data) {\n\t\tv145 = data[145]\n\t}\n\tv146 := 0\n\tif 146 < len(data) {\n\t\tv146 = data[146]\n\t}\n\tv147 := 0\n\tif 147 < len(data) {\n\t\tv147 = data[147]\n\t}\n\tv148 := 0\n\tif 148 < len(data) {\n\t\tv148 = data[148]\n\t}\n\tv149 := 0\n\tif 149 < len(data) {\n\t\tv149 = data[149]",
|
||||
"token_estimate": 917
|
||||
},
|
||||
{
|
||||
"block_ids": [
|
||||
"5d269745b2e5dbdcbef0c09ba54b0bd6"
|
||||
],
|
||||
"chunk_id": "24176c911d0bacf9a29fa7f8251f5036",
|
||||
"chunker_version": "code-go-ast-v1",
|
||||
"doc_id": "83daba5fbb026e7a400d68a1c4bd36db",
|
||||
"heading_path": [],
|
||||
"policy_hash": "6cfe77abe2b0e5c3",
|
||||
"source_spans": [
|
||||
{
|
||||
"kind": "code",
|
||||
"lang": "go",
|
||||
"line_end": 847,
|
||||
"line_start": 648,
|
||||
"symbol": "BigCompute [part 4/5]"
|
||||
}
|
||||
],
|
||||
"text": "\t}\n\tv150 := 0\n\tif 150 < len(data) {\n\t\tv150 = data[150]\n\t}\n\tv151 := 0\n\tif 151 < len(data) {\n\t\tv151 = data[151]\n\t}\n\tv152 := 0\n\tif 152 < len(data) {\n\t\tv152 = data[152]\n\t}\n\tv153 := 0\n\tif 153 < len(data) {\n\t\tv153 = data[153]\n\t}\n\tv154 := 0\n\tif 154 < len(data) {\n\t\tv154 = data[154]\n\t}\n\tv155 := 0\n\tif 155 < len(data) {\n\t\tv155 = data[155]\n\t}\n\tv156 := 0\n\tif 156 < len(data) {\n\t\tv156 = data[156]\n\t}\n\tv157 := 0\n\tif 157 < len(data) {\n\t\tv157 = data[157]\n\t}\n\tv158 := 0\n\tif 158 < len(data) {\n\t\tv158 = data[158]\n\t}\n\tv159 := 0\n\tif 159 < len(data) {\n\t\tv159 = data[159]\n\t}\n\tv160 := 0\n\tif 160 < len(data) {\n\t\tv160 = data[160]\n\t}\n\tv161 := 0\n\tif 161 < len(data) {\n\t\tv161 = data[161]\n\t}\n\tv162 := 0\n\tif 162 < len(data) {\n\t\tv162 = data[162]\n\t}\n\tv163 := 0\n\tif 163 < len(data) {\n\t\tv163 = data[163]\n\t}\n\tv164 := 0\n\tif 164 < len(data) {\n\t\tv164 = data[164]\n\t}\n\tv165 := 0\n\tif 165 < len(data) {\n\t\tv165 = data[165]\n\t}\n\tv166 := 0\n\tif 166 < len(data) {\n\t\tv166 = data[166]\n\t}\n\tv167 := 0\n\tif 167 < len(data) {\n\t\tv167 = data[167]\n\t}\n\tv168 := 0\n\tif 168 < len(data) {\n\t\tv168 = data[168]\n\t}\n\tv169 := 0\n\tif 169 < len(data) {\n\t\tv169 = data[169]\n\t}\n\tv170 := 0\n\tif 170 < len(data) {\n\t\tv170 = data[170]\n\t}\n\tv171 := 0\n\tif 171 < len(data) {\n\t\tv171 = data[171]\n\t}\n\tv172 := 0\n\tif 172 < len(data) {\n\t\tv172 = data[172]\n\t}\n\tv173 := 0\n\tif 173 < len(data) {\n\t\tv173 = data[173]\n\t}\n\tv174 := 0\n\tif 174 < len(data) {\n\t\tv174 = data[174]\n\t}\n\tv175 := 0\n\tif 175 < len(data) {\n\t\tv175 = data[175]\n\t}\n\tv176 := 0\n\tif 176 < len(data) {\n\t\tv176 = data[176]\n\t}\n\tv177 := 0\n\tif 177 < len(data) {\n\t\tv177 = data[177]\n\t}\n\tv178 := 0\n\tif 178 < len(data) {\n\t\tv178 = data[178]\n\t}\n\tv179 := 0\n\tif 179 < len(data) {\n\t\tv179 = data[179]\n\t}\n\tv180 := 0\n\tif 180 < len(data) {\n\t\tv180 = data[180]\n\t}\n\tv181 := 0\n\tif 181 < len(data) {\n\t\tv181 = data[181]\n\t}\n\tv182 := 0\n\tif 182 < len(data) {\n\t\tv182 = data[182]\n\t}\n\tv183 := 0\n\tif 183 < len(data) {\n\t\tv183 = data[183]\n\t}\n\tv184 := 0\n\tif 184 < len(data) {\n\t\tv184 = data[184]\n\t}\n\tv185 := 0\n\tif 185 < len(data) {\n\t\tv185 = data[185]\n\t}\n\tv186 := 0\n\tif 186 < len(data) {\n\t\tv186 = data[186]\n\t}\n\tv187 := 0\n\tif 187 < len(data) {\n\t\tv187 = data[187]\n\t}\n\tv188 := 0\n\tif 188 < len(data) {\n\t\tv188 = data[188]\n\t}\n\tv189 := 0\n\tif 189 < len(data) {\n\t\tv189 = data[189]\n\t}\n\tv190 := 0\n\tif 190 < len(data) {\n\t\tv190 = data[190]\n\t}\n\tv191 := 0\n\tif 191 < len(data) {\n\t\tv191 = data[191]\n\t}\n\tv192 := 0\n\tif 192 < len(data) {\n\t\tv192 = data[192]\n\t}\n\tv193 := 0\n\tif 193 < len(data) {\n\t\tv193 = data[193]\n\t}\n\tv194 := 0\n\tif 194 < len(data) {\n\t\tv194 = data[194]\n\t}\n\tv195 := 0\n\tif 195 < len(data) {\n\t\tv195 = data[195]\n\t}\n\tv196 := 0\n\tif 196 < len(data) {\n\t\tv196 = data[196]\n\t}\n\tv197 := 0\n\tif 197 < len(data) {\n\t\tv197 = data[197]\n\t}\n\tv198 := 0\n\tif 198 < len(data) {\n\t\tv198 = data[198]\n\t}\n\tv199 := 0\n\tif 199 < len(data) {\n\t\tv199 = data[199]",
|
||||
"token_estimate": 917
|
||||
},
|
||||
{
|
||||
"block_ids": [
|
||||
"5d269745b2e5dbdcbef0c09ba54b0bd6"
|
||||
],
|
||||
"chunk_id": "438127626378632c03780d10603de32c",
|
||||
"chunker_version": "code-go-ast-v1",
|
||||
"doc_id": "83daba5fbb026e7a400d68a1c4bd36db",
|
||||
"heading_path": [],
|
||||
"policy_hash": "6cfe77abe2b0e5c3",
|
||||
"source_spans": [
|
||||
{
|
||||
"kind": "code",
|
||||
"lang": "go",
|
||||
"line_end": 890,
|
||||
"line_start": 848,
|
||||
"symbol": "BigCompute [part 5/5]"
|
||||
}
|
||||
],
|
||||
"text": "\t}\n\tv200 := 0\n\tif 200 < len(data) {\n\t\tv200 = data[200]\n\t}\n\tv201 := 0\n\tif 201 < len(data) {\n\t\tv201 = data[201]\n\t}\n\tv202 := 0\n\tif 202 < len(data) {\n\t\tv202 = data[202]\n\t}\n\tv203 := 0\n\tif 203 < len(data) {\n\t\tv203 = data[203]\n\t}\n\tv204 := 0\n\tif 204 < len(data) {\n\t\tv204 = data[204]\n\t}\n\tv205 := 0\n\tif 205 < len(data) {\n\t\tv205 = data[205]\n\t}\n\tv206 := 0\n\tif 206 < len(data) {\n\t\tv206 = data[206]\n\t}\n\tv207 := 0\n\tif 207 < len(data) {\n\t\tv207 = data[207]\n\t}\n\tv208 := 0\n\tif 208 < len(data) {\n\t\tv208 = data[208]\n\t}\n\tv209 := 0\n\tif 209 < len(data) {\n\t\tv209 = data[209]\n\t}\n\treturn len(data)\n}",
|
||||
"token_estimate": 191
|
||||
}
|
||||
]
|
||||
170
crates/kebab-chunk/tests/fixtures/code-sample.java.chunks.snapshot.json
vendored
Normal file
170
crates/kebab-chunk/tests/fixtures/code-sample.java.chunks.snapshot.json
vendored
Normal file
File diff suppressed because one or more lines are too long
170
crates/kebab-chunk/tests/fixtures/code-sample.js.chunks.snapshot.json
vendored
Normal file
170
crates/kebab-chunk/tests/fixtures/code-sample.js.chunks.snapshot.json
vendored
Normal file
File diff suppressed because one or more lines are too long
170
crates/kebab-chunk/tests/fixtures/code-sample.kt.chunks.snapshot.json
vendored
Normal file
170
crates/kebab-chunk/tests/fixtures/code-sample.kt.chunks.snapshot.json
vendored
Normal file
File diff suppressed because one or more lines are too long
170
crates/kebab-chunk/tests/fixtures/code-sample.py.chunks.snapshot.json
vendored
Normal file
170
crates/kebab-chunk/tests/fixtures/code-sample.py.chunks.snapshot.json
vendored
Normal file
File diff suppressed because one or more lines are too long
170
crates/kebab-chunk/tests/fixtures/code-sample.ts.chunks.snapshot.json
vendored
Normal file
170
crates/kebab-chunk/tests/fixtures/code-sample.ts.chunks.snapshot.json
vendored
Normal file
File diff suppressed because one or more lines are too long
@@ -275,6 +275,14 @@ enum Cmd {
|
||||
#[arg(long, group = "reset_scope")]
|
||||
config_only: bool,
|
||||
|
||||
/// Purge stored docs that are outside the current walker scope
|
||||
/// (config narrowing / removed sub-directory). No filesystem paths
|
||||
/// are removed — this is purely a store-level reconciliation.
|
||||
/// Filesystem existence is NOT checked; anything the current walker
|
||||
/// would not visit is considered an orphan and removed from the store.
|
||||
#[arg(long, group = "reset_scope")]
|
||||
orphans_only: bool,
|
||||
|
||||
/// Skip the interactive confirm. Required in non-interactive
|
||||
/// contexts (CI, pipes).
|
||||
#[arg(long)]
|
||||
@@ -595,14 +603,20 @@ fn run(cli: &Cli) -> anyhow::Result<()> {
|
||||
println!("{}", serde_json::to_string(&wire::wire_ingest(&report))?);
|
||||
} else {
|
||||
let skipped_breakdown = kebab_app::render_skipped_breakdown(&report.skipped_by_extension);
|
||||
let purged_suffix = if report.purged_deleted_files > 0 {
|
||||
format!(" purged {}", report.purged_deleted_files)
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
println!(
|
||||
"scanned {} new {} updated {} skipped {}{} errors {} ({} ms)",
|
||||
"scanned {} new {} updated {} skipped {}{} errors {}{} ({} ms)",
|
||||
report.scanned,
|
||||
report.new,
|
||||
report.updated,
|
||||
report.skipped,
|
||||
skipped_breakdown,
|
||||
report.errors,
|
||||
purged_suffix,
|
||||
report.duration_ms
|
||||
);
|
||||
}
|
||||
@@ -1088,6 +1102,7 @@ fn run(cli: &Cli) -> anyhow::Result<()> {
|
||||
data_only: _,
|
||||
vector_only,
|
||||
config_only,
|
||||
orphans_only,
|
||||
yes,
|
||||
} => {
|
||||
use kebab_app::ResetScope;
|
||||
@@ -1101,11 +1116,50 @@ fn run(cli: &Cli) -> anyhow::Result<()> {
|
||||
ResetScope::VectorOnly
|
||||
} else if *config_only {
|
||||
ResetScope::ConfigOnly
|
||||
} else if *orphans_only {
|
||||
ResetScope::OrphansOnly
|
||||
} else {
|
||||
ResetScope::DataOnly
|
||||
};
|
||||
|
||||
let cfg = kebab_config::Config::load(cli.config.as_deref())?;
|
||||
|
||||
if matches!(scope, ResetScope::OrphansOnly) {
|
||||
// OrphansOnly: confirm UI shows orphan count + sample paths
|
||||
// rather than on-disk directory sizes.
|
||||
let orphan_paths = kebab_app::enumerate_orphans(&cfg)?;
|
||||
|
||||
if !*yes {
|
||||
use std::io::IsTerminal;
|
||||
if !std::io::stdin().is_terminal() {
|
||||
anyhow::bail!(
|
||||
"reset --orphans-only is destructive and stdin is non-interactive — pass --yes to proceed"
|
||||
);
|
||||
}
|
||||
if !confirm_orphans_only(&orphan_paths)? {
|
||||
if !cli.quiet {
|
||||
eprintln!("aborted.");
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
|
||||
let report = kebab_app::reset::execute(scope, &cfg)?;
|
||||
if cli.json {
|
||||
println!("{}", serde_json::to_string(&wire::wire_reset(&report))?);
|
||||
} else {
|
||||
if report.orphans_purged > 0 {
|
||||
println!("orphans purged: {}", report.orphans_purged);
|
||||
for p in &report.purged_paths {
|
||||
println!(" - {}", p.0);
|
||||
}
|
||||
} else {
|
||||
println!("no orphaned docs found — store is already in sync with walker scope");
|
||||
}
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let paths = kebab_app::reset::enumerate_paths(scope, &cfg);
|
||||
let bytes = kebab_app::reset::estimate_size_bytes(&paths);
|
||||
|
||||
@@ -1444,6 +1498,46 @@ fn confirm_destructive(
|
||||
Ok(matches!(s.as_str(), "y" | "yes"))
|
||||
}
|
||||
|
||||
/// Confirm prompt for `--orphans-only`: shows the orphan count + a
|
||||
/// sample of up to 5 paths so the user knows what will be purged before
|
||||
/// committing. No filesystem paths are removed — only store records.
|
||||
fn confirm_orphans_only(
|
||||
orphan_paths: &[kebab_core::WorkspacePath],
|
||||
) -> anyhow::Result<bool> {
|
||||
use std::io::Write;
|
||||
let n = orphan_paths.len();
|
||||
let mut out = std::io::stderr().lock();
|
||||
|
||||
if n == 0 {
|
||||
writeln!(out, "no orphaned docs found — nothing to purge.")?;
|
||||
out.flush()?;
|
||||
// Nothing to do; treat as confirmed so the caller can emit the
|
||||
// "no orphans" report without prompting.
|
||||
return Ok(true);
|
||||
}
|
||||
|
||||
let sample: Vec<&str> = orphan_paths
|
||||
.iter()
|
||||
.take(5)
|
||||
.map(|p| p.0.as_str())
|
||||
.collect();
|
||||
let sample_str = sample.join(", ");
|
||||
let ellipsis = if n > 5 { ", …" } else { "" };
|
||||
|
||||
writeln!(
|
||||
out,
|
||||
"Purge {n} stored doc(s) outside the current walker scope? (no filesystem paths removed)"
|
||||
)?;
|
||||
writeln!(out, " sample: {sample_str}{ellipsis}")?;
|
||||
write!(out, "[y/N] ")?;
|
||||
out.flush()?;
|
||||
|
||||
let mut line = String::new();
|
||||
std::io::stdin().read_line(&mut line)?;
|
||||
let s = line.trim().to_ascii_lowercase();
|
||||
Ok(matches!(s.as_str(), "y" | "yes"))
|
||||
}
|
||||
|
||||
/// p9-fb-35: human-friendly plain output for `kebab fetch`.
|
||||
fn render_fetch_plain(r: &kebab_core::FetchResult) {
|
||||
println!("# {} ({})", r.doc_path.0, format_kind(r.kind));
|
||||
|
||||
@@ -260,6 +260,7 @@ mod tests {
|
||||
skipped_generated: 0,
|
||||
skipped_size_exceeded: 0,
|
||||
skip_examples: SkipExamples::default(),
|
||||
purged_deleted_files: 0,
|
||||
items: None,
|
||||
};
|
||||
let v = wire_ingest(&r);
|
||||
@@ -364,6 +365,8 @@ mod tests {
|
||||
scope: kebab_app::ResetScope::DataOnly,
|
||||
removed_paths: vec![std::path::PathBuf::from("/tmp/x")],
|
||||
embedding_rows_truncated: 0,
|
||||
orphans_purged: 0,
|
||||
purged_paths: vec![],
|
||||
};
|
||||
let v = wire_reset(&r);
|
||||
assert_eq!(schema_of(&v), Some("reset_report.v1"));
|
||||
|
||||
@@ -47,6 +47,12 @@ pub struct IngestReport {
|
||||
/// p10-1A-1: sample file paths per skip category (≤ 5 each).
|
||||
#[serde(default)]
|
||||
pub skip_examples: SkipExamples,
|
||||
/// Dogfood: docs whose on-disk file was deleted since the last ingest
|
||||
/// and were therefore removed from the store. Additive field — older
|
||||
/// wire consumers that pre-date this field read it as 0 via
|
||||
/// `#[serde(default)]`.
|
||||
#[serde(default)]
|
||||
pub purged_deleted_files: u32,
|
||||
/// `None` ↔ wire `items: null` (`--summary-only`).
|
||||
pub items: Option<Vec<IngestItem>>,
|
||||
}
|
||||
@@ -136,6 +142,7 @@ mod tests {
|
||||
builtin_blacklist: vec!["node_modules/x.js".into()],
|
||||
gitignore: vec![],
|
||||
},
|
||||
purged_deleted_files: 0,
|
||||
items: None,
|
||||
};
|
||||
let v = serde_json::to_value(&r).unwrap();
|
||||
|
||||
@@ -8,7 +8,7 @@ use serde_json::Value;
|
||||
use crate::asset::{RawAsset, WorkspacePath};
|
||||
use crate::chunk::Chunk;
|
||||
use crate::document::{Block, CanonicalDocument};
|
||||
use crate::ids::{ChunkId, DocumentId};
|
||||
use crate::ids::{AssetId, ChunkId, DocumentId};
|
||||
use crate::jobs::{JobFilter, JobId, JobKind, JobRow, JobStatus};
|
||||
use crate::media::MediaType;
|
||||
use crate::search::{DocFilter, DocSummary, SearchFilters, SearchHit, SearchQuery};
|
||||
@@ -161,14 +161,51 @@ pub trait DocumentStore {
|
||||
fn get_document(&self, id: &DocumentId) -> anyhow::Result<Option<CanonicalDocument>>;
|
||||
fn get_chunk(&self, id: &ChunkId) -> anyhow::Result<Option<Chunk>>;
|
||||
fn list_documents(&self, filter: &DocFilter) -> anyhow::Result<Vec<DocSummary>>;
|
||||
/// Look up an asset row by its `asset_id` (PRIMARY KEY = blake3
|
||||
/// content hash). Twin-file safe: asset_id is PK so there is
|
||||
/// exactly one row per unique content hash, regardless of how many
|
||||
/// `documents` rows share it. Use this instead of
|
||||
/// `get_asset_by_workspace_path` when you already have a
|
||||
/// `CanonicalDocument` (which carries `source_asset_id`).
|
||||
fn get_asset(&self, id: &AssetId) -> anyhow::Result<Option<RawAsset>>;
|
||||
|
||||
/// p9-fb-23: look up an asset row by its workspace path. Used by
|
||||
/// the incremental-ingest skip path to compare the freshly
|
||||
/// computed blake3 checksum against what's already in SQLite. The
|
||||
/// schema enforces a unique workspace_path per asset.
|
||||
///
|
||||
/// NOTE: for twin files (identical content at different paths),
|
||||
/// `assets.workspace_path` is "last-registered path" — it
|
||||
/// flip-flops on every ingest. Prefer `get_asset` (by asset_id)
|
||||
/// when you have a `CanonicalDocument.source_asset_id`.
|
||||
fn get_asset_by_workspace_path(
|
||||
&self,
|
||||
path: &WorkspacePath,
|
||||
) -> anyhow::Result<Option<RawAsset>>;
|
||||
|
||||
/// Look up a document row by its workspace path. Used by the
|
||||
/// document-centric skip path in `try_skip_unchanged` to avoid the
|
||||
/// twin-file flip-flop that the asset-side lookup suffers from
|
||||
/// (multiple files with identical content share one `assets` row
|
||||
/// whose `workspace_path` is overwritten on every UPSERT, so
|
||||
/// `get_asset_by_workspace_path` returns the wrong twin's path).
|
||||
///
|
||||
/// `documents.workspace_path` is UNIQUE (V001), so each twin has
|
||||
/// its own stable document row regardless of the asset de-dup.
|
||||
fn get_document_by_workspace_path(
|
||||
&self,
|
||||
path: &WorkspacePath,
|
||||
) -> anyhow::Result<Option<CanonicalDocument>>;
|
||||
|
||||
/// Return every `workspace_path` stored in the `documents` table.
|
||||
///
|
||||
/// Used by the post-walker sweep in `kebab-app::ingest` to detect
|
||||
/// documents whose source file has been deleted from the filesystem.
|
||||
/// The set difference `(stored - scanned)` yields orphan candidates;
|
||||
/// each candidate is then existence-checked on disk so that
|
||||
/// out-of-scope files (config narrowing) are NOT purged — only truly
|
||||
/// absent files trigger the purge.
|
||||
fn all_workspace_paths(&self) -> anyhow::Result<Vec<WorkspacePath>>;
|
||||
}
|
||||
|
||||
pub trait VectorStore {
|
||||
|
||||
@@ -336,21 +336,29 @@ fn runner_lexical_is_deterministic_per_query_payload() {
|
||||
"- id: q1\n query: ownership\n- id: q2\n query: heading\n",
|
||||
);
|
||||
|
||||
let run_a = run_with_golden(&yaml, || {
|
||||
let mut run_a = run_with_golden(&yaml, || {
|
||||
run_eval_with_config(&env.config, &lexical_opts()).unwrap()
|
||||
});
|
||||
let run_b = run_with_golden(&yaml, || {
|
||||
let mut run_b = run_with_golden(&yaml, || {
|
||||
run_eval_with_config(&env.config, &lexical_opts()).unwrap()
|
||||
});
|
||||
|
||||
// Run-level fields (`run_id`, `created_at`) intentionally diverge;
|
||||
// the per-query payload (which is what the snapshot fixture pins)
|
||||
// must be byte-identical.
|
||||
// must be byte-identical EXCEPT for `elapsed_ms`. Timing-sensitive
|
||||
// fields aren't determinism signals — they're µs-scale wall-clock
|
||||
// jitter and would otherwise make this assertion a flaky one (a 0
|
||||
// vs 1 ms divergence was observed under contended-CI load). Normalize
|
||||
// before comparing; see test #7 for the same exclusion done via a
|
||||
// projection.
|
||||
for qr in run_a.per_query.iter_mut().chain(run_b.per_query.iter_mut()) {
|
||||
qr.elapsed_ms = 0;
|
||||
}
|
||||
let a_json = serde_json::to_string(&run_a.per_query).unwrap();
|
||||
let b_json = serde_json::to_string(&run_b.per_query).unwrap();
|
||||
assert_eq!(
|
||||
a_json, b_json,
|
||||
"lexical-only per_query payload must be byte-identical across runs"
|
||||
"lexical-only per_query payload must be byte-identical across runs (timing normalized)"
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -16,6 +16,12 @@ time = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
tree-sitter = { workspace = true }
|
||||
tree-sitter-rust = { workspace = true }
|
||||
tree-sitter-python = { workspace = true }
|
||||
tree-sitter-typescript = { workspace = true }
|
||||
tree-sitter-javascript = { workspace = true }
|
||||
tree-sitter-go = { workspace = true }
|
||||
tree-sitter-java = { workspace = true }
|
||||
tree-sitter-kotlin-ng = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = { workspace = true }
|
||||
|
||||
451
crates/kebab-parse-code/src/go.rs
Normal file
451
crates/kebab-parse-code/src/go.rs
Normal file
@@ -0,0 +1,451 @@
|
||||
//! `kebab-parse-code::go` — tree-sitter Go AST extractor (P10-1C-Go Task D).
|
||||
//!
|
||||
//! Implements [`kebab_core::Extractor`] for [`MediaType::Code("go")`].
|
||||
//! Walks the tree-sitter parse tree and emits one [`Block::Code`] per
|
||||
//! top-level AST semantic unit (free fn, method, each type spec) carrying
|
||||
//! [`SourceSpan::Code`] with the unit's self-reference symbol path
|
||||
//! (design §3.4 Go row). Glue declarations (`import` / `const` / `var`)
|
||||
//! collapse into one grouped `<top-level>` (or `<module>`) unit.
|
||||
//!
|
||||
//! Unlike the Python/TS/JS extractors which path-derive their module
|
||||
//! prefix from the workspace file path, Go's package identity comes from
|
||||
//! the source itself (the leading `package` clause) — `extract_package`
|
||||
//! reads it from the AST. If the `package_clause` is missing (invalid Go
|
||||
//! in practice) the prefix falls back to `"<unknown>"`.
|
||||
//!
|
||||
//! Doc comments immediately preceding an item are folded into that
|
||||
//! item's line range via `unit_start` (1B pattern). Go has no separate
|
||||
//! attribute/decorator AST nodes.
|
||||
//!
|
||||
//! Per design §3.4 / §9.1 / §9 versioning.
|
||||
|
||||
use anyhow::Result;
|
||||
use kebab_core::{
|
||||
Block, CanonicalDocument, CodeBlock, CommonBlock, Extractor, Lang, MediaType, Metadata,
|
||||
ParserVersion, Provenance, ProvenanceEvent, ProvenanceKind, SourceSpan, SourceType, TrustLevel,
|
||||
id_for_block, id_for_doc,
|
||||
};
|
||||
use serde_json::Map;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
use crate::scaffold::{filename_from_workspace_path, join_symbol, strip_extension};
|
||||
|
||||
pub const PARSER_VERSION: &str = "code-go-v1";
|
||||
|
||||
/// Go AST extractor. Per-unit blocks via tree-sitter-go 0.25
|
||||
/// (`LANGUAGE: LanguageFn`) parsed by tree-sitter 0.26.
|
||||
pub struct GoAstExtractor;
|
||||
|
||||
impl GoAstExtractor {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for GoAstExtractor {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl Extractor for GoAstExtractor {
|
||||
fn supports(&self, m: &MediaType) -> bool {
|
||||
matches!(m, MediaType::Code(l) if l == "go")
|
||||
}
|
||||
|
||||
fn parser_version(&self) -> ParserVersion {
|
||||
ParserVersion(PARSER_VERSION.to_string())
|
||||
}
|
||||
|
||||
fn extract(
|
||||
&self,
|
||||
ctx: &kebab_core::ExtractContext<'_>,
|
||||
bytes: &[u8],
|
||||
) -> Result<CanonicalDocument> {
|
||||
let asset = ctx.asset;
|
||||
if !self.supports(&asset.media_type) {
|
||||
anyhow::bail!(
|
||||
"kebab-parse-code: unsupported media_type for GoAstExtractor: {:?}",
|
||||
asset.media_type
|
||||
);
|
||||
}
|
||||
|
||||
let parser_version = self.parser_version();
|
||||
let doc_id = id_for_doc(&asset.workspace_path, &asset.asset_id, &parser_version);
|
||||
|
||||
let source = String::from_utf8(bytes.to_vec())
|
||||
.map_err(|e| anyhow::anyhow!("kebab-parse-code: Go source is not valid UTF-8: {e}"))?;
|
||||
|
||||
let blocks = build_blocks(&source, &doc_id)?;
|
||||
let unit_count = blocks.len() as u32;
|
||||
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let mut events: Vec<ProvenanceEvent> = Vec::with_capacity(2);
|
||||
events.push(ProvenanceEvent {
|
||||
at: asset.discovered_at,
|
||||
agent: "kb-source-fs".to_string(),
|
||||
kind: ProvenanceKind::Discovered,
|
||||
note: None,
|
||||
});
|
||||
events.push(ProvenanceEvent {
|
||||
at: now,
|
||||
agent: "kb-parse-code".to_string(),
|
||||
kind: ProvenanceKind::Parsed,
|
||||
note: Some(format!(
|
||||
"parser_version={}; unit_count={}",
|
||||
parser_version.0, unit_count
|
||||
)),
|
||||
});
|
||||
|
||||
let title = {
|
||||
let fname = filename_from_workspace_path(&asset.workspace_path.0);
|
||||
strip_extension(&fname)
|
||||
};
|
||||
|
||||
// Resolve the file's absolute path for repo detection. If the
|
||||
// source URI carries a relative path, anchor it at the workspace
|
||||
// root so the `.git/` walk-up starts from the right place.
|
||||
let abs_path = match &asset.source_uri {
|
||||
kebab_core::SourceUri::File(p) => {
|
||||
if p.is_absolute() {
|
||||
p.clone()
|
||||
} else {
|
||||
ctx.workspace_root.join(p)
|
||||
}
|
||||
}
|
||||
kebab_core::SourceUri::Kb(_) => ctx.workspace_root.to_path_buf(),
|
||||
};
|
||||
let (repo, git_branch, git_commit) = match crate::repo::detect_repo(&abs_path) {
|
||||
Some(r) => (Some(r.name), r.branch, r.commit),
|
||||
None => (None, None, None),
|
||||
};
|
||||
|
||||
let metadata = Metadata {
|
||||
aliases: Vec::new(),
|
||||
tags: Vec::new(),
|
||||
created_at: asset.discovered_at,
|
||||
updated_at: asset.discovered_at,
|
||||
source_type: SourceType::Note,
|
||||
trust_level: TrustLevel::Primary,
|
||||
user_id_alias: None,
|
||||
user: Map::new(),
|
||||
repo,
|
||||
git_branch,
|
||||
git_commit,
|
||||
code_lang: Some("go".to_string()),
|
||||
};
|
||||
|
||||
tracing::debug!(
|
||||
target: "kebab-parse-code",
|
||||
"extracted Go doc_id={} workspace_path={} units={}",
|
||||
doc_id.0,
|
||||
asset.workspace_path.0,
|
||||
unit_count
|
||||
);
|
||||
|
||||
Ok(CanonicalDocument {
|
||||
doc_id,
|
||||
source_asset_id: asset.asset_id.clone(),
|
||||
workspace_path: asset.workspace_path.clone(),
|
||||
title,
|
||||
lang: Lang("und".to_string()),
|
||||
blocks,
|
||||
metadata,
|
||||
provenance: Provenance { events },
|
||||
parser_version,
|
||||
schema_version: 1,
|
||||
doc_version: 1,
|
||||
last_chunker_version: None,
|
||||
last_embedding_version: None,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// p10-1C-Go: extract `package` declaration text from a tree-sitter-go
|
||||
/// `source_file`. Returns `None` if no `package_clause` (invalid Go in
|
||||
/// practice but defense-in-depth). Per design §3.4 Go row.
|
||||
fn extract_package(root: tree_sitter::Node, src: &str) -> Option<String> {
|
||||
let mut cur = root.walk();
|
||||
for child in root.named_children(&mut cur) {
|
||||
if child.kind() == "package_clause" {
|
||||
let mut c2 = child.walk();
|
||||
for sub in child.named_children(&mut c2) {
|
||||
if sub.kind() == "package_identifier" {
|
||||
return Some(src[sub.start_byte()..sub.end_byte()].to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn build_blocks(
|
||||
source: &str,
|
||||
doc_id: &kebab_core::DocumentId,
|
||||
) -> anyhow::Result<Vec<kebab_core::Block>> {
|
||||
let mut parser = tree_sitter::Parser::new();
|
||||
parser
|
||||
.set_language(&tree_sitter_go::LANGUAGE.into())
|
||||
.map_err(|e| anyhow::anyhow!("set tree-sitter-go language: {e}"))?;
|
||||
let tree = parser
|
||||
.parse(source.as_bytes(), None)
|
||||
.ok_or_else(|| anyhow::anyhow!("tree-sitter failed to parse Go source"))?;
|
||||
let lines: Vec<&str> = source.split('\n').collect();
|
||||
|
||||
let root = tree.root_node();
|
||||
let mod_prefix = extract_package(root, source).unwrap_or_else(|| "<unknown>".to_string());
|
||||
|
||||
// units: (symbol, line_start, line_end, is_real_semantic_unit).
|
||||
// Glue groups are pushed with a sentinel symbol + is_real=false so a
|
||||
// post-pass can decide `<module>` vs `<top-level>` (1B post-pass
|
||||
// mirror).
|
||||
let mut units: Vec<(String, u32, u32, bool)> = Vec::new();
|
||||
// (is_import 0/1, s, e). `is_import` flags `import_declaration` —
|
||||
// used by the glue flush to pick `<module>` vs `<top-level>`
|
||||
// provisional label.
|
||||
let mut glue: Vec<(usize, u32, u32)> = Vec::new();
|
||||
|
||||
fn node_name_text<'a>(n: &tree_sitter::Node, src: &'a str) -> Option<&'a str> {
|
||||
n.child_by_field_name("name")
|
||||
.map(|c| &src[c.start_byte()..c.end_byte()])
|
||||
}
|
||||
/// Walk preceding `comment` siblings to extend the unit's line range
|
||||
/// upward, folding leading doc / line comments into the unit. Go has
|
||||
/// no decorator/attribute nodes — doc comments are simply preceding
|
||||
/// `comment` siblings (the 1B pattern).
|
||||
fn unit_start(n: &tree_sitter::Node) -> u32 {
|
||||
let mut start = n.start_position().row as u32 + 1;
|
||||
let mut prev = n.prev_sibling();
|
||||
while let Some(p) = prev {
|
||||
if p.kind() == "comment" {
|
||||
start = p.start_position().row as u32 + 1;
|
||||
prev = p.prev_sibling();
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
start
|
||||
}
|
||||
|
||||
/// Extract the receiver type text for a `method_declaration`. The
|
||||
/// returned slice INCLUDES the leading `*` for pointer receivers
|
||||
/// (`(*Foo).Bar`) per design §3.4 Go row example. Returns `None` if
|
||||
/// the receiver is malformed (defense in depth).
|
||||
fn receiver_type_text<'a>(method_node: &tree_sitter::Node, src: &'a str) -> Option<&'a str> {
|
||||
let recv = method_node.child_by_field_name("receiver")?;
|
||||
let mut cw = recv.walk();
|
||||
for p in recv.named_children(&mut cw) {
|
||||
if p.kind() == "parameter_declaration" {
|
||||
if let Some(ty) = p.child_by_field_name("type") {
|
||||
return Some(&src[ty.start_byte()..ty.end_byte()]);
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
let mut cur = root.walk();
|
||||
for child in root.named_children(&mut cur) {
|
||||
let s = unit_start(&child);
|
||||
let e = child.end_position().row as u32 + 1;
|
||||
match child.kind() {
|
||||
"function_declaration" => {
|
||||
if let Some(name) = node_name_text(&child, source) {
|
||||
glue.retain(|(_, gs, _)| *gs < s);
|
||||
flush_glue(&mut glue, &mut units, &mod_prefix);
|
||||
let sym = join_symbol(&mod_prefix, &[], name);
|
||||
units.push((sym, s, e, true));
|
||||
}
|
||||
}
|
||||
"method_declaration" => {
|
||||
if let Some(name_node) = child.child_by_field_name("name") {
|
||||
glue.retain(|(_, gs, _)| *gs < s);
|
||||
flush_glue(&mut glue, &mut units, &mod_prefix);
|
||||
let owner = receiver_type_text(&child, source).unwrap_or("<unknown>");
|
||||
let method_name = &source[name_node.start_byte()..name_node.end_byte()];
|
||||
let sym = format!("{mod_prefix}.({owner}).{method_name}");
|
||||
units.push((sym, s, e, true));
|
||||
}
|
||||
}
|
||||
"type_declaration" => {
|
||||
// One unit per inner `type_spec`. Each type_spec gets
|
||||
// the type_declaration's whole upward-folded `s` range
|
||||
// start so doc comments are attached to the first spec;
|
||||
// subsequent specs use their own start. Match 1B
|
||||
// pattern: keep the outer `s` only when there's a single
|
||||
// spec; otherwise use the spec's own start.
|
||||
let mut tcur = child.walk();
|
||||
let specs: Vec<tree_sitter::Node> = child
|
||||
.named_children(&mut tcur)
|
||||
.filter(|c| c.kind() == "type_spec")
|
||||
.collect();
|
||||
let single = specs.len() == 1;
|
||||
for spec in specs {
|
||||
let name_node = match spec.child_by_field_name("name") {
|
||||
Some(n) => n,
|
||||
None => continue,
|
||||
};
|
||||
let spec_s = if single {
|
||||
s
|
||||
} else {
|
||||
spec.start_position().row as u32 + 1
|
||||
};
|
||||
let spec_e = spec.end_position().row as u32 + 1;
|
||||
glue.retain(|(_, gs, _)| *gs < spec_s);
|
||||
flush_glue(&mut glue, &mut units, &mod_prefix);
|
||||
let name = &source[name_node.start_byte()..name_node.end_byte()];
|
||||
let sym = join_symbol(&mod_prefix, &[], name);
|
||||
units.push((sym, spec_s, spec_e, true));
|
||||
}
|
||||
}
|
||||
"import_declaration" => {
|
||||
glue.push((1, s, e));
|
||||
}
|
||||
"const_declaration" | "var_declaration" => {
|
||||
glue.push((0, s, e));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
flush_glue(&mut glue, &mut units, &mod_prefix);
|
||||
|
||||
// `<module>` is correct only when the file produced no real unit.
|
||||
// Otherwise the import/const/var-only group becomes `<top-level>`
|
||||
// (same post-pass as 1B). Match on the suffix so the demotion stays
|
||||
// mod-prefix-agnostic.
|
||||
let has_real_unit = units.iter().any(|(_, _, _, is_real)| *is_real);
|
||||
if has_real_unit {
|
||||
for (sym, _, _, is_real) in units.iter_mut() {
|
||||
if !*is_real && sym.ends_with("<module>") {
|
||||
let pre = &sym[..sym.len() - "<module>".len()];
|
||||
*sym = format!("{pre}<top-level>");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let total_lines = lines.len() as u32;
|
||||
let mut blocks = Vec::with_capacity(units.len());
|
||||
for (ordinal, (symbol, ls, le, _is_real)) in units.into_iter().enumerate() {
|
||||
let line_start = ls.max(1);
|
||||
let line_end = le.min(total_lines.max(1));
|
||||
let span = SourceSpan::Code {
|
||||
line_start,
|
||||
line_end,
|
||||
symbol: Some(symbol),
|
||||
lang: Some("go".to_string()),
|
||||
};
|
||||
let block_id = id_for_block(doc_id, "code", &[], ordinal as u32, &span);
|
||||
let code = lines[(line_start as usize - 1)..=(line_end as usize - 1)].join("\n");
|
||||
blocks.push(Block::Code(CodeBlock {
|
||||
common: CommonBlock {
|
||||
block_id,
|
||||
heading_path: Vec::new(),
|
||||
source_span: span,
|
||||
},
|
||||
lang: Some("go".to_string()),
|
||||
code,
|
||||
}));
|
||||
}
|
||||
Ok(blocks)
|
||||
}
|
||||
|
||||
fn flush_glue(
|
||||
glue: &mut Vec<(usize, u32, u32)>,
|
||||
units: &mut Vec<(String, u32, u32, bool)>,
|
||||
mod_prefix: &str,
|
||||
) {
|
||||
if glue.is_empty() {
|
||||
return;
|
||||
}
|
||||
let s = glue.iter().map(|(_, a, _)| *a).min().unwrap();
|
||||
let e = glue.iter().map(|(_, _, b)| *b).max().unwrap();
|
||||
// Provisional label: `<module>` only if the group is exclusively
|
||||
// imports (1A's `only_mod_decls` analog). The post-pass demotes any
|
||||
// `<module>` to `<top-level>` if the file produced any real unit.
|
||||
let only_imports = glue.iter().all(|(is_import, _, _)| *is_import == 1);
|
||||
let label = if only_imports { "<module>" } else { "<top-level>" };
|
||||
units.push((join_symbol(mod_prefix, &[], label), s, e, false));
|
||||
glue.clear();
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use kebab_core::{Block, MediaType, SourceSpan};
|
||||
|
||||
fn extract_fixture() -> kebab_core::CanonicalDocument {
|
||||
let bytes = std::fs::read(concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/tests/fixtures/sample.go"
|
||||
))
|
||||
.unwrap();
|
||||
// Reuse the cross-language test-support helper promoted in 1B.
|
||||
let asset = crate::rust::tests_support::fixed_code_asset("crates/x/src/sample.go", "go");
|
||||
let cfg = kebab_core::ExtractConfig::default();
|
||||
let root = std::path::PathBuf::from("/tmp");
|
||||
let ctx = kebab_core::ExtractContext {
|
||||
asset: &asset,
|
||||
workspace_root: &root,
|
||||
config: &cfg,
|
||||
};
|
||||
GoAstExtractor::new().extract(&ctx, &bytes).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extractor_supports_only_media_code_go() {
|
||||
let e = GoAstExtractor::new();
|
||||
assert!(e.supports(&MediaType::Code("go".into())));
|
||||
assert!(!e.supports(&MediaType::Code("rust".into())));
|
||||
assert!(!e.supports(&MediaType::Markdown));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn go_units_match_design_3_4_symbols() {
|
||||
let doc = extract_fixture();
|
||||
let mut syms: Vec<String> = doc
|
||||
.blocks
|
||||
.iter()
|
||||
.filter_map(|b| match b {
|
||||
Block::Code(c) => match &c.common.source_span {
|
||||
SourceSpan::Code { symbol, lang, .. } => {
|
||||
assert_eq!(lang.as_deref(), Some("go"));
|
||||
symbol.clone()
|
||||
}
|
||||
_ => None,
|
||||
},
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
syms.sort();
|
||||
assert!(syms.iter().any(|s| s == "chunk.Free"), "got {syms:?}");
|
||||
assert!(syms.iter().any(|s| s == "chunk.init"), "got {syms:?}");
|
||||
assert!(
|
||||
syms.iter().any(|s| s == "chunk.MdHeadingV1Chunker"),
|
||||
"got {syms:?}"
|
||||
);
|
||||
assert!(
|
||||
syms.iter()
|
||||
.any(|s| s == "chunk.(*MdHeadingV1Chunker).ChunkDoc"),
|
||||
"got {syms:?}"
|
||||
);
|
||||
assert!(
|
||||
syms.iter()
|
||||
.any(|s| s == "chunk.(MdHeadingV1Chunker).Name2"),
|
||||
"got {syms:?}"
|
||||
);
|
||||
assert!(syms.iter().any(|s| s == "chunk.Stringer"), "got {syms:?}");
|
||||
// import + const grouped into one glue unit (no isolated `<module>`).
|
||||
assert!(
|
||||
syms.iter().any(|s| s == "chunk.<top-level>"),
|
||||
"got {syms:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deterministic_across_runs() {
|
||||
let a = extract_fixture();
|
||||
for _ in 0..50 {
|
||||
assert_eq!(extract_fixture().blocks, a.blocks);
|
||||
}
|
||||
}
|
||||
}
|
||||
543
crates/kebab-parse-code/src/java.rs
Normal file
543
crates/kebab-parse-code/src/java.rs
Normal file
@@ -0,0 +1,543 @@
|
||||
//! `kebab-parse-code::java` — tree-sitter Java AST extractor (P10-1C-JK Task D).
|
||||
//!
|
||||
//! Implements [`kebab_core::Extractor`] for [`MediaType::Code("java")`].
|
||||
//! Walks the tree-sitter parse tree and emits one [`Block::Code`] per
|
||||
//! top-level AST semantic unit (class / interface / enum / record /
|
||||
//! annotation-type at any nesting level, plus methods + constructors
|
||||
//! inside class / interface / record bodies), each carrying
|
||||
//! [`SourceSpan::Code`] with the unit's dotted self-reference symbol
|
||||
//! path (design §3.4 Java row). Glue declarations (`import`) collapse
|
||||
//! into one grouped `<top-level>` (or `<module>`) unit.
|
||||
//!
|
||||
//! Like the Go extractor, Java's package identity comes from the
|
||||
//! source itself (the `package_declaration` clause), not from the
|
||||
//! workspace file path — `extract_package` reads it from the AST. If
|
||||
//! the clause is missing the prefix falls back to `"<unknown>"`.
|
||||
//!
|
||||
//! Class/interface/record bodies are recursed (1B Python pattern):
|
||||
//! the type name is pushed onto `mod_path` so methods and nested
|
||||
//! types become `<pkg>.<Outer>.<Inner>.<method>`. Constructors use
|
||||
//! the Java convention `<pkg>.<...>.<Class>.<ClassName>` (name
|
||||
//! duplicated, per design §3.4). Enum bodies are not recursed for
|
||||
//! the 1차 cut — enum constants are not emitted as units.
|
||||
//!
|
||||
//! Javadoc (`/** ... */` → `block_comment`) and line comments
|
||||
//! immediately preceding an item are folded into that item's line
|
||||
//! range via `unit_start` (1B pattern). Annotations are children of
|
||||
//! the declaration node itself (inside `modifiers`), so they are
|
||||
//! already part of the declaration's span — no separate unwrap arm.
|
||||
//!
|
||||
//! Per design §3.4 / §9.1 / §9 versioning.
|
||||
|
||||
use anyhow::Result;
|
||||
use kebab_core::{
|
||||
Block, CanonicalDocument, CodeBlock, CommonBlock, Extractor, Lang, MediaType, Metadata,
|
||||
ParserVersion, Provenance, ProvenanceEvent, ProvenanceKind, SourceSpan, SourceType, TrustLevel,
|
||||
id_for_block, id_for_doc,
|
||||
};
|
||||
use serde_json::Map;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
use crate::scaffold::{filename_from_workspace_path, join_symbol, strip_extension};
|
||||
|
||||
pub const PARSER_VERSION: &str = "code-java-v1";
|
||||
|
||||
/// Java AST extractor. Per-unit blocks via tree-sitter-java 0.23
|
||||
/// (`LANGUAGE: LanguageFn`) parsed by tree-sitter 0.26.
|
||||
pub struct JavaAstExtractor;
|
||||
|
||||
impl JavaAstExtractor {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for JavaAstExtractor {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl Extractor for JavaAstExtractor {
|
||||
fn supports(&self, m: &MediaType) -> bool {
|
||||
matches!(m, MediaType::Code(l) if l == "java")
|
||||
}
|
||||
|
||||
fn parser_version(&self) -> ParserVersion {
|
||||
ParserVersion(PARSER_VERSION.to_string())
|
||||
}
|
||||
|
||||
fn extract(
|
||||
&self,
|
||||
ctx: &kebab_core::ExtractContext<'_>,
|
||||
bytes: &[u8],
|
||||
) -> Result<CanonicalDocument> {
|
||||
let asset = ctx.asset;
|
||||
if !self.supports(&asset.media_type) {
|
||||
anyhow::bail!(
|
||||
"kebab-parse-code: unsupported media_type for JavaAstExtractor: {:?}",
|
||||
asset.media_type
|
||||
);
|
||||
}
|
||||
|
||||
let parser_version = self.parser_version();
|
||||
let doc_id = id_for_doc(&asset.workspace_path, &asset.asset_id, &parser_version);
|
||||
|
||||
let source = String::from_utf8(bytes.to_vec())
|
||||
.map_err(|e| anyhow::anyhow!("kebab-parse-code: Java source is not valid UTF-8: {e}"))?;
|
||||
|
||||
let blocks = build_blocks(&source, &doc_id)?;
|
||||
let unit_count = blocks.len() as u32;
|
||||
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let mut events: Vec<ProvenanceEvent> = Vec::with_capacity(2);
|
||||
events.push(ProvenanceEvent {
|
||||
at: asset.discovered_at,
|
||||
agent: "kb-source-fs".to_string(),
|
||||
kind: ProvenanceKind::Discovered,
|
||||
note: None,
|
||||
});
|
||||
events.push(ProvenanceEvent {
|
||||
at: now,
|
||||
agent: "kb-parse-code".to_string(),
|
||||
kind: ProvenanceKind::Parsed,
|
||||
note: Some(format!(
|
||||
"parser_version={}; unit_count={}",
|
||||
parser_version.0, unit_count
|
||||
)),
|
||||
});
|
||||
|
||||
let title = {
|
||||
let fname = filename_from_workspace_path(&asset.workspace_path.0);
|
||||
strip_extension(&fname)
|
||||
};
|
||||
|
||||
// Resolve the file's absolute path for repo detection. If the
|
||||
// source URI carries a relative path, anchor it at the workspace
|
||||
// root so the `.git/` walk-up starts from the right place.
|
||||
let abs_path = match &asset.source_uri {
|
||||
kebab_core::SourceUri::File(p) => {
|
||||
if p.is_absolute() {
|
||||
p.clone()
|
||||
} else {
|
||||
ctx.workspace_root.join(p)
|
||||
}
|
||||
}
|
||||
kebab_core::SourceUri::Kb(_) => ctx.workspace_root.to_path_buf(),
|
||||
};
|
||||
let (repo, git_branch, git_commit) = match crate::repo::detect_repo(&abs_path) {
|
||||
Some(r) => (Some(r.name), r.branch, r.commit),
|
||||
None => (None, None, None),
|
||||
};
|
||||
|
||||
let metadata = Metadata {
|
||||
aliases: Vec::new(),
|
||||
tags: Vec::new(),
|
||||
created_at: asset.discovered_at,
|
||||
updated_at: asset.discovered_at,
|
||||
source_type: SourceType::Note,
|
||||
trust_level: TrustLevel::Primary,
|
||||
user_id_alias: None,
|
||||
user: Map::new(),
|
||||
repo,
|
||||
git_branch,
|
||||
git_commit,
|
||||
code_lang: Some("java".to_string()),
|
||||
};
|
||||
|
||||
tracing::debug!(
|
||||
target: "kebab-parse-code",
|
||||
"extracted Java doc_id={} workspace_path={} units={}",
|
||||
doc_id.0,
|
||||
asset.workspace_path.0,
|
||||
unit_count
|
||||
);
|
||||
|
||||
Ok(CanonicalDocument {
|
||||
doc_id,
|
||||
source_asset_id: asset.asset_id.clone(),
|
||||
workspace_path: asset.workspace_path.clone(),
|
||||
title,
|
||||
lang: Lang("und".to_string()),
|
||||
blocks,
|
||||
metadata,
|
||||
provenance: Provenance { events },
|
||||
parser_version,
|
||||
schema_version: 1,
|
||||
doc_version: 1,
|
||||
last_chunker_version: None,
|
||||
last_embedding_version: None,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// p10-1C-JK: extract `package` declaration text from a tree-sitter-java
|
||||
/// `program`. Returns `None` if no `package_declaration` (default-package
|
||||
/// Java file). The package_declaration's named children are either a
|
||||
/// single `identifier` (single-segment package, rare) or a
|
||||
/// `scoped_identifier` (dotted, common). Per design §3.4 Java row.
|
||||
fn extract_package(root: tree_sitter::Node, src: &str) -> Option<String> {
|
||||
let mut cur = root.walk();
|
||||
for child in root.named_children(&mut cur) {
|
||||
if child.kind() == "package_declaration" {
|
||||
let mut c2 = child.walk();
|
||||
for sub in child.named_children(&mut c2) {
|
||||
if sub.kind() == "scoped_identifier" || sub.kind() == "identifier" {
|
||||
return Some(src[sub.start_byte()..sub.end_byte()].to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Walk preceding `line_comment` / `block_comment` siblings to extend
|
||||
/// the unit's line range upward, folding leading Javadoc / line
|
||||
/// comments into the unit. Annotations live INSIDE `modifiers` on the
|
||||
/// declaration node itself, so their lines are already inside
|
||||
/// `n.start_position()` — no separate unwrap arm is needed for them.
|
||||
fn unit_start(n: &tree_sitter::Node) -> u32 {
|
||||
let mut start = n.start_position().row as u32 + 1;
|
||||
let mut prev = n.prev_sibling();
|
||||
while let Some(p) = prev {
|
||||
let k = p.kind();
|
||||
if k == "line_comment" || k == "block_comment" {
|
||||
start = p.start_position().row as u32 + 1;
|
||||
prev = p.prev_sibling();
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
start
|
||||
}
|
||||
|
||||
fn node_name_text<'a>(n: &tree_sitter::Node, src: &'a str) -> Option<&'a str> {
|
||||
n.child_by_field_name("name")
|
||||
.map(|c| &src[c.start_byte()..c.end_byte()])
|
||||
}
|
||||
|
||||
fn build_blocks(
|
||||
source: &str,
|
||||
doc_id: &kebab_core::DocumentId,
|
||||
) -> anyhow::Result<Vec<kebab_core::Block>> {
|
||||
let mut parser = tree_sitter::Parser::new();
|
||||
parser
|
||||
.set_language(&tree_sitter_java::LANGUAGE.into())
|
||||
.map_err(|e| anyhow::anyhow!("set tree-sitter-java language: {e}"))?;
|
||||
let tree = parser
|
||||
.parse(source.as_bytes(), None)
|
||||
.ok_or_else(|| anyhow::anyhow!("tree-sitter failed to parse Java source"))?;
|
||||
let lines: Vec<&str> = source.split('\n').collect();
|
||||
|
||||
let root = tree.root_node();
|
||||
let mod_prefix = extract_package(root, source).unwrap_or_else(|| "<unknown>".to_string());
|
||||
|
||||
// units: (symbol, line_start, line_end, is_real_semantic_unit).
|
||||
// Glue groups are pushed with a sentinel symbol + is_real=false so a
|
||||
// post-pass can decide `<module>` vs `<top-level>` (1B/1C-Go pattern).
|
||||
let mut units: Vec<(String, u32, u32, bool)> = Vec::new();
|
||||
// (is_import 0/1, s, e). `is_import` flags `import_declaration` —
|
||||
// used by the glue flush to pick `<module>` vs `<top-level>`
|
||||
// provisional label.
|
||||
let mut glue: Vec<(usize, u32, u32)> = Vec::new();
|
||||
|
||||
walk_top(root, source, &mod_prefix, &mut units, &mut glue);
|
||||
|
||||
// `<module>` is correct only when the file produced no real unit.
|
||||
// Otherwise the import-only group becomes `<top-level>` (same
|
||||
// post-pass as 1B / 1C-Go).
|
||||
let has_real_unit = units.iter().any(|(_, _, _, is_real)| *is_real);
|
||||
if has_real_unit {
|
||||
for (sym, _, _, is_real) in units.iter_mut() {
|
||||
if !*is_real && sym.ends_with("<module>") {
|
||||
let pre = &sym[..sym.len() - "<module>".len()];
|
||||
*sym = format!("{pre}<top-level>");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let total_lines = lines.len() as u32;
|
||||
let mut blocks = Vec::with_capacity(units.len());
|
||||
for (ordinal, (symbol, ls, le, _is_real)) in units.into_iter().enumerate() {
|
||||
let line_start = ls.max(1);
|
||||
let line_end = le.min(total_lines.max(1));
|
||||
let span = SourceSpan::Code {
|
||||
line_start,
|
||||
line_end,
|
||||
symbol: Some(symbol),
|
||||
lang: Some("java".to_string()),
|
||||
};
|
||||
let block_id = id_for_block(doc_id, "code", &[], ordinal as u32, &span);
|
||||
let code = lines[(line_start as usize - 1)..=(line_end as usize - 1)].join("\n");
|
||||
blocks.push(Block::Code(CodeBlock {
|
||||
common: CommonBlock {
|
||||
block_id,
|
||||
heading_path: Vec::new(),
|
||||
source_span: span,
|
||||
},
|
||||
lang: Some("java".to_string()),
|
||||
code,
|
||||
}));
|
||||
}
|
||||
Ok(blocks)
|
||||
}
|
||||
|
||||
/// Walk the file's top-level children — `program` named children:
|
||||
/// `package_declaration` (handled by `extract_package`), `import_declaration`
|
||||
/// (glue), and the five type declarations (`class` / `interface` /
|
||||
/// `enum` / `record` / `annotation_type`). Type-declaration bodies
|
||||
/// are recursed via [`walk_body`] with the type name pushed onto
|
||||
/// `mod_path` (1B Python pattern). Enum bodies are NOT recursed
|
||||
/// (1차 cut — see module-level doc).
|
||||
fn walk_top(
|
||||
node: tree_sitter::Node,
|
||||
src: &str,
|
||||
mod_prefix: &str,
|
||||
units: &mut Vec<(String, u32, u32, bool)>,
|
||||
glue: &mut Vec<(usize, u32, u32)>,
|
||||
) {
|
||||
let mod_path: &[String] = &[];
|
||||
let mut cur = node.walk();
|
||||
for child in node.named_children(&mut cur) {
|
||||
let s = unit_start(&child);
|
||||
let e = child.end_position().row as u32 + 1;
|
||||
match child.kind() {
|
||||
"class_declaration"
|
||||
| "interface_declaration"
|
||||
| "record_declaration" => {
|
||||
if let Some(name) = node_name_text(&child, src) {
|
||||
glue.retain(|(_, gs, _)| *gs < s);
|
||||
flush_glue(glue, units, mod_prefix, mod_path);
|
||||
let sym = join_symbol(mod_prefix, mod_path, name);
|
||||
units.push((sym, s, e, true));
|
||||
if let Some(body) = child.child_by_field_name("body") {
|
||||
let np: Vec<String> = vec![name.to_string()];
|
||||
walk_body(body, src, mod_prefix, &np, units);
|
||||
}
|
||||
}
|
||||
}
|
||||
"enum_declaration" => {
|
||||
if let Some(name) = node_name_text(&child, src) {
|
||||
glue.retain(|(_, gs, _)| *gs < s);
|
||||
flush_glue(glue, units, mod_prefix, mod_path);
|
||||
let sym = join_symbol(mod_prefix, mod_path, name);
|
||||
units.push((sym, s, e, true));
|
||||
// Enum body NOT recursed for 1차 — enum constants are
|
||||
// not emitted as units, and method declarations inside
|
||||
// enum bodies (rare) live under `enum_body_declarations`
|
||||
// not `class_body`. Skip per design §3.4 1차 scope.
|
||||
}
|
||||
}
|
||||
"annotation_type_declaration" => {
|
||||
if let Some(name) = node_name_text(&child, src) {
|
||||
glue.retain(|(_, gs, _)| *gs < s);
|
||||
flush_glue(glue, units, mod_prefix, mod_path);
|
||||
let sym = join_symbol(mod_prefix, mod_path, name);
|
||||
units.push((sym, s, e, true));
|
||||
}
|
||||
}
|
||||
"import_declaration" => {
|
||||
glue.push((1, s, e));
|
||||
}
|
||||
// package_declaration is handled by `extract_package`; no
|
||||
// glue entry — it's structural metadata, not a unit.
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
flush_glue(glue, units, mod_prefix, mod_path);
|
||||
}
|
||||
|
||||
/// Walk a `class_body` / `interface_body` (or record's `class_body`).
|
||||
/// Emits one unit per method / constructor, and recurses into nested
|
||||
/// type declarations. Field declarations are NOT emitted (would
|
||||
/// explode unit count). `compact_constructor_declaration` (records)
|
||||
/// is handled the same as `constructor_declaration`.
|
||||
///
|
||||
/// No `glue` parameter: Java does not have imports inside type
|
||||
/// bodies — they only appear at file top level, handled by
|
||||
/// [`walk_top`].
|
||||
fn walk_body(
|
||||
body: tree_sitter::Node,
|
||||
src: &str,
|
||||
mod_prefix: &str,
|
||||
mod_path: &[String],
|
||||
units: &mut Vec<(String, u32, u32, bool)>,
|
||||
) {
|
||||
let mut cur = body.walk();
|
||||
for child in body.named_children(&mut cur) {
|
||||
let s = unit_start(&child);
|
||||
let e = child.end_position().row as u32 + 1;
|
||||
match child.kind() {
|
||||
"method_declaration"
|
||||
| "constructor_declaration"
|
||||
| "compact_constructor_declaration" => {
|
||||
// Constructor: name field equals the class name. Per
|
||||
// design §3.4 Java convention, symbol is
|
||||
// `<pkg>.<mod_path>.<ClassName>` with the constructor
|
||||
// name (== class name) as the trailing segment. This
|
||||
// means the symbol duplicates the class name (e.g.
|
||||
// `com.x.Foo.Foo`), which is the documented convention.
|
||||
if let Some(name) = node_name_text(&child, src) {
|
||||
let sym = join_symbol(mod_prefix, mod_path, name);
|
||||
units.push((sym, s, e, true));
|
||||
}
|
||||
}
|
||||
"class_declaration"
|
||||
| "interface_declaration"
|
||||
| "record_declaration"
|
||||
| "enum_declaration"
|
||||
| "annotation_type_declaration" => {
|
||||
// Nested type — emit unit, then recurse into its body
|
||||
// (skipped for enum + annotation_type per 1차 scope).
|
||||
let name = match node_name_text(&child, src) {
|
||||
Some(n) => n,
|
||||
None => continue,
|
||||
};
|
||||
let sym = join_symbol(mod_prefix, mod_path, name);
|
||||
units.push((sym, s, e, true));
|
||||
if child.kind() != "enum_declaration"
|
||||
&& child.kind() != "annotation_type_declaration"
|
||||
{
|
||||
if let Some(inner_body) = child.child_by_field_name("body") {
|
||||
let mut np = mod_path.to_vec();
|
||||
np.push(name.to_string());
|
||||
walk_body(inner_body, src, mod_prefix, &np, units);
|
||||
}
|
||||
}
|
||||
}
|
||||
// field_declaration, static_initializer, block: NOT emitted.
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn flush_glue(
|
||||
glue: &mut Vec<(usize, u32, u32)>,
|
||||
units: &mut Vec<(String, u32, u32, bool)>,
|
||||
mod_prefix: &str,
|
||||
mod_path: &[String],
|
||||
) {
|
||||
if glue.is_empty() {
|
||||
return;
|
||||
}
|
||||
let s = glue.iter().map(|(_, a, _)| *a).min().unwrap();
|
||||
let e = glue.iter().map(|(_, _, b)| *b).max().unwrap();
|
||||
// Provisional label: `<module>` only if the group is exclusively
|
||||
// imports (1A's `only_mod_decls` analog). The post-pass demotes any
|
||||
// `<module>` to `<top-level>` if the file produced any real unit.
|
||||
let only_imports = glue.iter().all(|(is_import, _, _)| *is_import == 1);
|
||||
let label = if only_imports { "<module>" } else { "<top-level>" };
|
||||
units.push((join_symbol(mod_prefix, mod_path, label), s, e, false));
|
||||
glue.clear();
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use kebab_core::{Block, MediaType, SourceSpan};
|
||||
|
||||
fn extract_fixture() -> kebab_core::CanonicalDocument {
|
||||
let bytes = std::fs::read(concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/tests/fixtures/sample.java"
|
||||
))
|
||||
.unwrap();
|
||||
let asset =
|
||||
crate::rust::tests_support::fixed_code_asset("crates/x/src/sample.java", "java");
|
||||
let cfg = kebab_core::ExtractConfig::default();
|
||||
let root = std::path::PathBuf::from("/tmp");
|
||||
let ctx = kebab_core::ExtractContext {
|
||||
asset: &asset,
|
||||
workspace_root: &root,
|
||||
config: &cfg,
|
||||
};
|
||||
JavaAstExtractor::new().extract(&ctx, &bytes).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extractor_supports_only_media_code_java() {
|
||||
let e = JavaAstExtractor::new();
|
||||
assert!(e.supports(&MediaType::Code("java".into())));
|
||||
assert!(!e.supports(&MediaType::Code("rust".into())));
|
||||
assert!(!e.supports(&MediaType::Markdown));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn java_units_match_design_3_4_symbols() {
|
||||
let doc = extract_fixture();
|
||||
let mut syms: Vec<String> = doc
|
||||
.blocks
|
||||
.iter()
|
||||
.filter_map(|b| match b {
|
||||
Block::Code(c) => match &c.common.source_span {
|
||||
SourceSpan::Code { symbol, lang, .. } => {
|
||||
assert_eq!(lang.as_deref(), Some("java"));
|
||||
symbol.clone()
|
||||
}
|
||||
_ => None,
|
||||
},
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
syms.sort();
|
||||
// package extracted from source = com.kebab.chunk
|
||||
assert!(
|
||||
syms.iter().any(|s| s == "com.kebab.chunk.MdHeadingV1Chunker"),
|
||||
"got {syms:?}"
|
||||
);
|
||||
// constructor — Java convention is class-name-as-method-name
|
||||
assert!(
|
||||
syms.iter()
|
||||
.any(|s| s == "com.kebab.chunk.MdHeadingV1Chunker.MdHeadingV1Chunker"),
|
||||
"got {syms:?}"
|
||||
);
|
||||
assert!(
|
||||
syms.iter()
|
||||
.any(|s| s == "com.kebab.chunk.MdHeadingV1Chunker.chunkDoc"),
|
||||
"got {syms:?}"
|
||||
);
|
||||
assert!(
|
||||
syms.iter()
|
||||
.any(|s| s == "com.kebab.chunk.MdHeadingV1Chunker.getName"),
|
||||
"got {syms:?}"
|
||||
);
|
||||
// static nested class
|
||||
assert!(
|
||||
syms.iter()
|
||||
.any(|s| s == "com.kebab.chunk.MdHeadingV1Chunker.Builder"),
|
||||
"got {syms:?}"
|
||||
);
|
||||
assert!(
|
||||
syms.iter()
|
||||
.any(|s| s == "com.kebab.chunk.MdHeadingV1Chunker.Builder.withName"),
|
||||
"got {syms:?}"
|
||||
);
|
||||
assert!(
|
||||
syms.iter()
|
||||
.any(|s| s == "com.kebab.chunk.MdHeadingV1Chunker.Builder.build"),
|
||||
"got {syms:?}"
|
||||
);
|
||||
// package-private interface + enum
|
||||
assert!(
|
||||
syms.iter().any(|s| s == "com.kebab.chunk.Stringer"),
|
||||
"got {syms:?}"
|
||||
);
|
||||
assert!(
|
||||
syms.iter().any(|s| s == "com.kebab.chunk.Mode"),
|
||||
"got {syms:?}"
|
||||
);
|
||||
// import grouped as <top-level>
|
||||
assert!(
|
||||
syms.iter().any(|s| s == "com.kebab.chunk.<top-level>"),
|
||||
"got {syms:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deterministic_across_runs() {
|
||||
let a = extract_fixture();
|
||||
for _ in 0..50 {
|
||||
assert_eq!(extract_fixture().blocks, a.blocks);
|
||||
}
|
||||
}
|
||||
}
|
||||
574
crates/kebab-parse-code/src/javascript.rs
Normal file
574
crates/kebab-parse-code/src/javascript.rs
Normal file
@@ -0,0 +1,574 @@
|
||||
//! `kebab-parse-code::javascript` — tree-sitter JavaScript / JSX AST
|
||||
//! extractor (P10-1B Task K).
|
||||
//!
|
||||
//! Implements [`kebab_core::Extractor`] for [`MediaType::Code("javascript")`].
|
||||
//! Walks the tree-sitter parse tree (single grammar
|
||||
//! [`tree_sitter_javascript::LANGUAGE`] — the JS grammar handles `.jsx`
|
||||
//! as well, no second grammar needed) and emits one [`Block::Code`] per
|
||||
//! top-level AST semantic unit (free fn, class, each method,
|
||||
//! recursively per nested class), each carrying [`SourceSpan::Code`]
|
||||
//! with the unit's dotted symbol path prefixed by
|
||||
//! [`module_path_for_tsjs`].
|
||||
//!
|
||||
//! Glue declarations (`import_statement`, bare `export_statement`
|
||||
//! re-exports, `lexical_declaration` / `variable_declaration` at the
|
||||
//! module level, etc.) collapse into one grouped `<top-level>` (or
|
||||
//! `<module>`) unit.
|
||||
//!
|
||||
//! `export_statement` is unwrapped: an `export function|class` is
|
||||
//! treated as the inner declaration arm but the unit's line range
|
||||
//! comes from the OUTER `export_statement` so the `export ` prefix is
|
||||
//! folded in. `export default function () {}` / `export default class
|
||||
//! {}` (no `name` field) emits `default` as the symbol name.
|
||||
//!
|
||||
//! Differs from `typescript.rs` only by: single-grammar (no
|
||||
//! TS/TSX selection) and no `interface_declaration` /
|
||||
//! `type_alias_declaration` / `enum_declaration` arms (TS-only). All
|
||||
//! other walker behavior (export unwrap with `value`-field quirk for
|
||||
//! default-exported anonymous function/class, class-body method walk,
|
||||
//! glue flush, post-pass `<module>` → `<top-level>` rewrite) is
|
||||
//! identical.
|
||||
//!
|
||||
//! Scope follows 1A-2 / 1B Task K: AST unit extraction + dotted symbol
|
||||
//! paths + line ranges. Per design §3.4 / §9.1 / §9 versioning.
|
||||
|
||||
use anyhow::Result;
|
||||
use kebab_core::{
|
||||
Block, CanonicalDocument, CodeBlock, CommonBlock, Extractor, Lang, MediaType, Metadata,
|
||||
ParserVersion, Provenance, ProvenanceEvent, ProvenanceKind, SourceSpan, SourceType, TrustLevel,
|
||||
id_for_block, id_for_doc,
|
||||
};
|
||||
use serde_json::Map;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
use crate::scaffold::{filename_from_workspace_path, join_symbol, strip_extension};
|
||||
|
||||
pub const PARSER_VERSION: &str = "code-js-v1";
|
||||
|
||||
/// JavaScript / JSX AST extractor. Per-unit blocks via
|
||||
/// tree-sitter-javascript 0.25 (single `LANGUAGE` `LanguageFn` — the
|
||||
/// JS grammar covers `.jsx` natively, no second grammar) parsed by
|
||||
/// tree-sitter 0.26.
|
||||
pub struct JavascriptAstExtractor;
|
||||
|
||||
impl JavascriptAstExtractor {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for JavascriptAstExtractor {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl Extractor for JavascriptAstExtractor {
|
||||
fn supports(&self, m: &MediaType) -> bool {
|
||||
matches!(m, MediaType::Code(l) if l == "javascript")
|
||||
}
|
||||
|
||||
fn parser_version(&self) -> ParserVersion {
|
||||
ParserVersion(PARSER_VERSION.to_string())
|
||||
}
|
||||
|
||||
fn extract(
|
||||
&self,
|
||||
ctx: &kebab_core::ExtractContext<'_>,
|
||||
bytes: &[u8],
|
||||
) -> Result<CanonicalDocument> {
|
||||
let asset = ctx.asset;
|
||||
if !self.supports(&asset.media_type) {
|
||||
anyhow::bail!(
|
||||
"kebab-parse-code: unsupported media_type for JavascriptAstExtractor: {:?}",
|
||||
asset.media_type
|
||||
);
|
||||
}
|
||||
|
||||
let parser_version = self.parser_version();
|
||||
let doc_id = id_for_doc(&asset.workspace_path, &asset.asset_id, &parser_version);
|
||||
|
||||
let source = String::from_utf8(bytes.to_vec()).map_err(|e| {
|
||||
anyhow::anyhow!("kebab-parse-code: JavaScript source is not valid UTF-8: {e}")
|
||||
})?;
|
||||
|
||||
let mod_prefix = crate::lang::module_path_for_tsjs(&asset.workspace_path.0);
|
||||
let language: tree_sitter::Language = tree_sitter_javascript::LANGUAGE.into();
|
||||
let blocks = build_blocks(&source, &doc_id, &mod_prefix, language)?;
|
||||
let unit_count = blocks.len() as u32;
|
||||
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let mut events: Vec<ProvenanceEvent> = Vec::with_capacity(2);
|
||||
events.push(ProvenanceEvent {
|
||||
at: asset.discovered_at,
|
||||
agent: "kb-source-fs".to_string(),
|
||||
kind: ProvenanceKind::Discovered,
|
||||
note: None,
|
||||
});
|
||||
events.push(ProvenanceEvent {
|
||||
at: now,
|
||||
agent: "kb-parse-code".to_string(),
|
||||
kind: ProvenanceKind::Parsed,
|
||||
note: Some(format!(
|
||||
"parser_version={}; unit_count={}",
|
||||
parser_version.0, unit_count
|
||||
)),
|
||||
});
|
||||
|
||||
let title = {
|
||||
let fname = filename_from_workspace_path(&asset.workspace_path.0);
|
||||
strip_extension(&fname)
|
||||
};
|
||||
|
||||
// Resolve the file's absolute path for repo detection. If the
|
||||
// source URI carries a relative path, anchor it at the workspace
|
||||
// root so the `.git/` walk-up starts from the right place.
|
||||
let abs_path = match &asset.source_uri {
|
||||
kebab_core::SourceUri::File(p) => {
|
||||
if p.is_absolute() {
|
||||
p.clone()
|
||||
} else {
|
||||
ctx.workspace_root.join(p)
|
||||
}
|
||||
}
|
||||
kebab_core::SourceUri::Kb(_) => ctx.workspace_root.to_path_buf(),
|
||||
};
|
||||
let (repo, git_branch, git_commit) = match crate::repo::detect_repo(&abs_path) {
|
||||
Some(r) => (Some(r.name), r.branch, r.commit),
|
||||
None => (None, None, None),
|
||||
};
|
||||
|
||||
let metadata = Metadata {
|
||||
aliases: Vec::new(),
|
||||
tags: Vec::new(),
|
||||
created_at: asset.discovered_at,
|
||||
updated_at: asset.discovered_at,
|
||||
source_type: SourceType::Note,
|
||||
trust_level: TrustLevel::Primary,
|
||||
user_id_alias: None,
|
||||
user: Map::new(),
|
||||
repo,
|
||||
git_branch,
|
||||
git_commit,
|
||||
code_lang: Some("javascript".to_string()),
|
||||
};
|
||||
|
||||
tracing::debug!(
|
||||
target: "kebab-parse-code",
|
||||
"extracted JavaScript doc_id={} workspace_path={} units={}",
|
||||
doc_id.0,
|
||||
asset.workspace_path.0,
|
||||
unit_count
|
||||
);
|
||||
|
||||
Ok(CanonicalDocument {
|
||||
doc_id,
|
||||
source_asset_id: asset.asset_id.clone(),
|
||||
workspace_path: asset.workspace_path.clone(),
|
||||
title,
|
||||
lang: Lang("und".to_string()),
|
||||
blocks,
|
||||
metadata,
|
||||
provenance: Provenance { events },
|
||||
parser_version,
|
||||
schema_version: 1,
|
||||
doc_version: 1,
|
||||
last_chunker_version: None,
|
||||
last_embedding_version: None,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn build_blocks(
|
||||
source: &str,
|
||||
doc_id: &kebab_core::DocumentId,
|
||||
mod_prefix: &str,
|
||||
language: tree_sitter::Language,
|
||||
) -> anyhow::Result<Vec<kebab_core::Block>> {
|
||||
let mut parser = tree_sitter::Parser::new();
|
||||
parser
|
||||
.set_language(&language)
|
||||
.map_err(|e| anyhow::anyhow!("set tree-sitter-javascript language: {e}"))?;
|
||||
let tree = parser
|
||||
.parse(source.as_bytes(), None)
|
||||
.ok_or_else(|| anyhow::anyhow!("tree-sitter failed to parse JavaScript source"))?;
|
||||
let lines: Vec<&str> = source.split('\n').collect();
|
||||
|
||||
// units: (symbol, line_start, line_end, is_real_semantic_unit).
|
||||
// Glue groups are pushed with a sentinel symbol + is_real=false so a
|
||||
// post-pass can decide `<module>` vs `<top-level>` (same algorithm
|
||||
// as 1A Gap 1 / 1B Python / 1B TS).
|
||||
let mut units: Vec<(String, u32, u32, bool)> = Vec::new();
|
||||
// (is_module_only_kind 0/1, s, e). `is_module_only_kind` flags
|
||||
// `import_statement` and bare re-export `export_statement`s — used by
|
||||
// the glue flush to pick `<module>` vs `<top-level>` provisional
|
||||
// label (1A's `is_mod_decl` analog).
|
||||
let mut glue: Vec<(usize, u32, u32)> = Vec::new();
|
||||
|
||||
/// Walk preceding `comment` siblings to extend the unit's line range
|
||||
/// upward, folding leading doc / line comments into the unit.
|
||||
fn unit_start(n: &tree_sitter::Node) -> u32 {
|
||||
let mut start = n.start_position().row as u32 + 1;
|
||||
let mut prev = n.prev_sibling();
|
||||
while let Some(p) = prev {
|
||||
if p.kind() == "comment" {
|
||||
start = p.start_position().row as u32 + 1;
|
||||
prev = p.prev_sibling();
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
start
|
||||
}
|
||||
fn name_text<'a>(n: &tree_sitter::Node, src: &'a str) -> Option<&'a str> {
|
||||
n.child_by_field_name("name")
|
||||
.map(|c| &src[c.start_byte()..c.end_byte()])
|
||||
}
|
||||
/// Walk a class body, emitting one unit per `method_definition`.
|
||||
/// Class names already pushed onto `mod_path` by the caller, so
|
||||
/// method symbols come out as `<mod_prefix>.<Class>.<method>`.
|
||||
fn walk_class_body(
|
||||
body: tree_sitter::Node,
|
||||
src: &str,
|
||||
mod_prefix: &str,
|
||||
mod_path: &[String],
|
||||
units: &mut Vec<(String, u32, u32, bool)>,
|
||||
) {
|
||||
let mut cur = body.walk();
|
||||
for child in body.named_children(&mut cur) {
|
||||
if child.kind() == "method_definition" {
|
||||
if let Some(name) = name_text(&child, src) {
|
||||
let s = unit_start(&child);
|
||||
let e = child.end_position().row as u32 + 1;
|
||||
let sym = join_symbol(mod_prefix, mod_path, name);
|
||||
units.push((sym, s, e, true));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
fn walk(
|
||||
node: tree_sitter::Node,
|
||||
src: &str,
|
||||
mod_prefix: &str,
|
||||
mod_path: &[String],
|
||||
units: &mut Vec<(String, u32, u32, bool)>,
|
||||
glue: &mut Vec<(usize, u32, u32)>,
|
||||
) {
|
||||
let mut cur = node.walk();
|
||||
for child in node.named_children(&mut cur) {
|
||||
let s = unit_start(&child);
|
||||
let e = child.end_position().row as u32 + 1;
|
||||
match child.kind() {
|
||||
"function_declaration" => {
|
||||
if let Some(name) = name_text(&child, src) {
|
||||
glue.retain(|(_, gs, _)| *gs < s);
|
||||
flush_glue(glue, units, mod_prefix, mod_path);
|
||||
let sym = join_symbol(mod_prefix, mod_path, name);
|
||||
units.push((sym, s, e, true));
|
||||
}
|
||||
}
|
||||
"class_declaration" => {
|
||||
if let Some(name) = name_text(&child, src) {
|
||||
glue.retain(|(_, gs, _)| *gs < s);
|
||||
flush_glue(glue, units, mod_prefix, mod_path);
|
||||
let sym = join_symbol(mod_prefix, mod_path, name);
|
||||
units.push((sym, s, e, true));
|
||||
if let Some(body) = child.child_by_field_name("body") {
|
||||
let mut np = mod_path.to_vec();
|
||||
np.push(name.to_string());
|
||||
walk_class_body(body, src, mod_prefix, &np, units);
|
||||
}
|
||||
}
|
||||
}
|
||||
"export_statement" => {
|
||||
// Try field "declaration" first (export class /
|
||||
// function). If absent, fall back to "value" —
|
||||
// `export default function () {}` / `export default
|
||||
// class {}` expose the anonymous function_expression
|
||||
// / class under the `value` field (same grammar
|
||||
// quirk as TS 0.23).
|
||||
let outer_s = s; // includes `export ` prefix line
|
||||
let outer_e = e;
|
||||
if let Some(inner) = child.child_by_field_name("declaration") {
|
||||
let inner_kind = inner.kind();
|
||||
match inner_kind {
|
||||
"function_declaration" | "class_declaration" => {
|
||||
let name_opt = name_text(&inner, src).map(|s| s.to_string());
|
||||
if let Some(name) = name_opt {
|
||||
glue.retain(|(_, gs, _)| *gs < outer_s);
|
||||
flush_glue(glue, units, mod_prefix, mod_path);
|
||||
let sym = join_symbol(mod_prefix, mod_path, &name);
|
||||
units.push((sym, outer_s, outer_e, true));
|
||||
if inner_kind == "class_declaration" {
|
||||
if let Some(body) = inner.child_by_field_name("body") {
|
||||
let mut np = mod_path.to_vec();
|
||||
np.push(name);
|
||||
walk_class_body(body, src, mod_prefix, &np, units);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Defensive: `export default` with a
|
||||
// function_declaration that somehow
|
||||
// lacks `name`. Emit `default`.
|
||||
glue.retain(|(_, gs, _)| *gs < outer_s);
|
||||
flush_glue(glue, units, mod_prefix, mod_path);
|
||||
let sym = join_symbol(mod_prefix, mod_path, "default");
|
||||
units.push((sym, outer_s, outer_e, true));
|
||||
}
|
||||
}
|
||||
// `lexical_declaration` etc. wrapped in
|
||||
// export: treat as glue (assigned arrow
|
||||
// fns / consts don't get their own unit).
|
||||
_ => {
|
||||
glue.push((0, s, e));
|
||||
}
|
||||
}
|
||||
} else if let Some(value) = child.child_by_field_name("value") {
|
||||
// `export default <expr>`. We emit a unit only
|
||||
// for the function / class shapes (named or
|
||||
// anonymous); other value shapes are glue.
|
||||
match value.kind() {
|
||||
"function_expression"
|
||||
| "function_declaration"
|
||||
| "class"
|
||||
| "class_declaration" => {
|
||||
let name_opt = name_text(&value, src).map(|s| s.to_string());
|
||||
let leaf =
|
||||
name_opt.as_deref().unwrap_or("default").to_string();
|
||||
glue.retain(|(_, gs, _)| *gs < outer_s);
|
||||
flush_glue(glue, units, mod_prefix, mod_path);
|
||||
let sym = join_symbol(mod_prefix, mod_path, &leaf);
|
||||
units.push((sym, outer_s, outer_e, true));
|
||||
// Recurse into class body if we have one.
|
||||
if matches!(value.kind(), "class" | "class_declaration") {
|
||||
if let Some(body) = value.child_by_field_name("body") {
|
||||
let mut np = mod_path.to_vec();
|
||||
np.push(leaf);
|
||||
walk_class_body(body, src, mod_prefix, &np, units);
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
glue.push((0, s, e));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Bare `export { x };` / `export * from "..."` —
|
||||
// a re-export, glue with module-only flag set
|
||||
// (we have no `declaration` / `value` field for
|
||||
// it).
|
||||
glue.push((1, s, e));
|
||||
}
|
||||
}
|
||||
"import_statement" => {
|
||||
glue.push((1, s, e));
|
||||
}
|
||||
"lexical_declaration" | "variable_declaration" => {
|
||||
glue.push((0, s, e));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
flush_glue(glue, units, mod_prefix, mod_path);
|
||||
}
|
||||
fn flush_glue(
|
||||
glue: &mut Vec<(usize, u32, u32)>,
|
||||
units: &mut Vec<(String, u32, u32, bool)>,
|
||||
mod_prefix: &str,
|
||||
mod_path: &[String],
|
||||
) {
|
||||
if glue.is_empty() {
|
||||
return;
|
||||
}
|
||||
let s = glue.iter().map(|(_, a, _)| *a).min().unwrap();
|
||||
let e = glue.iter().map(|(_, _, b)| *b).max().unwrap();
|
||||
let only_module = glue.iter().all(|(is_mod, _, _)| *is_mod == 1);
|
||||
let label = if only_module { "<module>" } else { "<top-level>" };
|
||||
units.push((join_symbol(mod_prefix, mod_path, label), s, e, false));
|
||||
glue.clear();
|
||||
}
|
||||
|
||||
walk(
|
||||
tree.root_node(),
|
||||
source,
|
||||
mod_prefix,
|
||||
&[],
|
||||
&mut units,
|
||||
&mut glue,
|
||||
);
|
||||
|
||||
// `<module>` is correct only when the file produced no real unit.
|
||||
// Otherwise the import-only group becomes `<top-level>` (same
|
||||
// post-pass as 1A Gap 1 / Python / TS).
|
||||
let has_real_unit = units.iter().any(|(_, _, _, is_real)| *is_real);
|
||||
if has_real_unit {
|
||||
for (sym, _, _, is_real) in units.iter_mut() {
|
||||
if !*is_real && sym.ends_with("<module>") {
|
||||
let pre = &sym[..sym.len() - "<module>".len()];
|
||||
*sym = format!("{pre}<top-level>");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let total_lines = lines.len() as u32;
|
||||
let mut blocks = Vec::with_capacity(units.len());
|
||||
for (ordinal, (symbol, ls, le, _is_real)) in units.into_iter().enumerate() {
|
||||
let line_start = ls.max(1);
|
||||
let line_end = le.min(total_lines.max(1));
|
||||
let span = SourceSpan::Code {
|
||||
line_start,
|
||||
line_end,
|
||||
symbol: Some(symbol),
|
||||
lang: Some("javascript".to_string()),
|
||||
};
|
||||
let block_id = id_for_block(doc_id, "code", &[], ordinal as u32, &span);
|
||||
let code = lines[(line_start as usize - 1)..=(line_end as usize - 1)].join("\n");
|
||||
blocks.push(Block::Code(CodeBlock {
|
||||
common: CommonBlock {
|
||||
block_id,
|
||||
heading_path: Vec::new(),
|
||||
source_span: span,
|
||||
},
|
||||
lang: Some("javascript".to_string()),
|
||||
code,
|
||||
}));
|
||||
}
|
||||
Ok(blocks)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use kebab_core::{Block, MediaType, SourceSpan};
|
||||
|
||||
fn extract_fixture(workspace_path: &str) -> kebab_core::CanonicalDocument {
|
||||
let bytes = std::fs::read(
|
||||
concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/sample.js"),
|
||||
)
|
||||
.unwrap();
|
||||
let asset = crate::rust::tests_support::fixed_code_asset(workspace_path, "javascript");
|
||||
let cfg = kebab_core::ExtractConfig::default();
|
||||
let root = std::path::PathBuf::from("/tmp");
|
||||
let ctx = kebab_core::ExtractContext {
|
||||
asset: &asset,
|
||||
workspace_root: &root,
|
||||
config: &cfg,
|
||||
};
|
||||
JavascriptAstExtractor::new().extract(&ctx, &bytes).unwrap()
|
||||
}
|
||||
fn symbols(doc: &kebab_core::CanonicalDocument) -> Vec<String> {
|
||||
let mut s: Vec<String> = doc
|
||||
.blocks
|
||||
.iter()
|
||||
.filter_map(|b| match b {
|
||||
Block::Code(c) => match &c.common.source_span {
|
||||
SourceSpan::Code { symbol, lang, .. } => {
|
||||
assert_eq!(lang.as_deref(), Some("javascript"));
|
||||
symbol.clone()
|
||||
}
|
||||
_ => None,
|
||||
},
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
s.sort();
|
||||
s
|
||||
}
|
||||
#[test]
|
||||
fn extractor_supports_only_media_code_javascript() {
|
||||
let e = JavascriptAstExtractor::new();
|
||||
assert!(e.supports(&MediaType::Code("javascript".into())));
|
||||
assert!(!e.supports(&MediaType::Code("typescript".into())));
|
||||
assert!(!e.supports(&MediaType::Markdown));
|
||||
}
|
||||
#[test]
|
||||
fn js_units_match_design_3_4_symbols() {
|
||||
let doc = extract_fixture("src/sample.js");
|
||||
let syms = symbols(&doc);
|
||||
assert!(syms.iter().any(|s| s == "src/sample.add"), "got {syms:?}");
|
||||
assert!(syms.iter().any(|s| s == "src/sample.Retriever"));
|
||||
assert!(syms.iter().any(|s| s == "src/sample.Retriever.search"));
|
||||
assert!(syms.iter().any(|s| s == "src/sample.Retriever.create"));
|
||||
assert!(syms.iter().any(|s| s == "src/sample.default"));
|
||||
assert!(syms.iter().any(|s| s == "src/sample.<top-level>"));
|
||||
}
|
||||
#[test]
|
||||
fn jsx_via_js_grammar() {
|
||||
// tree-sitter-javascript handles .jsx via the same single grammar.
|
||||
let bytes = b"export function App() { return null; }\n";
|
||||
let asset = crate::rust::tests_support::fixed_code_asset("src/App.jsx", "javascript");
|
||||
let cfg = kebab_core::ExtractConfig::default();
|
||||
let root = std::path::PathBuf::from("/tmp");
|
||||
let ctx = kebab_core::ExtractContext {
|
||||
asset: &asset,
|
||||
workspace_root: &root,
|
||||
config: &cfg,
|
||||
};
|
||||
let doc = JavascriptAstExtractor::new().extract(&ctx, bytes).unwrap();
|
||||
let syms = symbols(&doc);
|
||||
assert!(syms.iter().any(|s| s == "src/App.App"), "got {syms:?}");
|
||||
}
|
||||
#[test]
|
||||
fn deterministic_across_runs() {
|
||||
let a = extract_fixture("src/sample.js");
|
||||
for _ in 0..30 {
|
||||
assert_eq!(extract_fixture("src/sample.js").blocks, a.blocks);
|
||||
}
|
||||
}
|
||||
|
||||
/// In tree-sitter-javascript, `decorator` is a CHILD of
|
||||
/// `method_definition` (stored in the `decorator` field), so
|
||||
/// `method_definition.start_row` already covers the decorator line
|
||||
/// without any sibling walk. Verify that the emitted unit already
|
||||
/// includes the decorator line and line_start is 2 (the @Log() line).
|
||||
#[test]
|
||||
fn js_class_method_decorator_already_folded_by_grammar() {
|
||||
// Line 1 (1-indexed): "class Foo {"
|
||||
// Line 2: " @Log()" <- decorator (child of method_definition in JS grammar)
|
||||
// Line 3: " bar() { return 1; }"
|
||||
// Line 4: "}"
|
||||
let bytes = b"class Foo {\n @Log()\n bar() { return 1; }\n}\n";
|
||||
let asset = crate::rust::tests_support::fixed_code_asset("src/foo.js", "javascript");
|
||||
let cfg = kebab_core::ExtractConfig::default();
|
||||
let root = std::path::PathBuf::from("/tmp");
|
||||
let ctx = kebab_core::ExtractContext {
|
||||
asset: &asset,
|
||||
workspace_root: &root,
|
||||
config: &cfg,
|
||||
};
|
||||
let doc = JavascriptAstExtractor::new().extract(&ctx, bytes).unwrap();
|
||||
|
||||
let bar_block = doc
|
||||
.blocks
|
||||
.iter()
|
||||
.find_map(|b| match b {
|
||||
Block::Code(c) => match &c.common.source_span {
|
||||
SourceSpan::Code { symbol, .. }
|
||||
if symbol.as_deref() == Some("src/foo.Foo.bar") =>
|
||||
{
|
||||
Some(c)
|
||||
}
|
||||
_ => None,
|
||||
},
|
||||
_ => None,
|
||||
})
|
||||
.expect("src/foo.Foo.bar block should be present");
|
||||
|
||||
// JS grammar: method_definition.start_row == decorator row, so
|
||||
// no sibling walk change needed -- decorator is already included.
|
||||
assert!(
|
||||
bar_block.code.contains("@Log()"),
|
||||
"JS method unit must include decorator (grammar folds it natively); got: {:?}",
|
||||
bar_block.code
|
||||
);
|
||||
match &bar_block.common.source_span {
|
||||
SourceSpan::Code { line_start, .. } => {
|
||||
assert_eq!(
|
||||
*line_start, 2,
|
||||
"JS line_start must cover the @Log() decorator line (got {line_start})"
|
||||
);
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
}
|
||||
627
crates/kebab-parse-code/src/kotlin.rs
Normal file
627
crates/kebab-parse-code/src/kotlin.rs
Normal file
@@ -0,0 +1,627 @@
|
||||
//! `kebab-parse-code::kotlin` — tree-sitter Kotlin AST extractor (P10-1C-JK Task G).
|
||||
//!
|
||||
//! Implements [`kebab_core::Extractor`] for [`MediaType::Code("kotlin")`].
|
||||
//! Mirrors the Java extractor (JVM family, source-side `package` extraction +
|
||||
//! class-nesting) with Kotlin-specific adjustments:
|
||||
//!
|
||||
//! * Root is `source_file` (not `program`).
|
||||
//! * `package_header` carries a single `qualified_identifier` child whose
|
||||
//! slice text IS the dotted package path — never a bare `identifier`
|
||||
//! sub-form for the package (the grammar always wraps a single segment
|
||||
//! in `qualified_identifier` too).
|
||||
//! * `class_declaration` covers `class`, `data class`, `sealed class`,
|
||||
//! `enum class`, AND `interface` — Kotlin uses ONE node kind with a
|
||||
//! `modifiers` child rather than separate `interface_declaration` /
|
||||
//! `enum_declaration` nodes (verified via tree-sitter-kotlin-ng
|
||||
//! `node-types.json`).
|
||||
//! * The body child of `class_declaration` is either `class_body` (normal
|
||||
//! classes / interfaces) OR `enum_class_body` (enum class). Neither
|
||||
//! carries a `body` field name, so it is matched by kind, not by
|
||||
//! `child_by_field_name("body")`.
|
||||
//! * `companion_object` is a SEPARATE node kind (not `object_declaration`
|
||||
//! with a modifier). Its `name` field is OPTIONAL — when omitted (the
|
||||
//! common case `companion object { ... }`) the symbol uses the
|
||||
//! implicit Kotlin convention name `Companion`.
|
||||
//! * `object_declaration` (named singleton) carries a `name` field and a
|
||||
//! `class_body` child.
|
||||
//! * `function_declaration` may appear at top level (Kotlin top-level
|
||||
//! function) AND inside `class_body` — same node kind, the
|
||||
//! `mod_path` state distinguishes the two emit forms.
|
||||
//!
|
||||
//! Enum bodies (`enum_class_body`) are NOT recursed for the 1차 cut —
|
||||
//! `enum_entry` declarations are not emitted as units, matching the
|
||||
//! Java extractor's enum policy (design §3.4 1차 scope).
|
||||
//!
|
||||
//! Per design §3.4 / §9.1 / §9 versioning.
|
||||
|
||||
use anyhow::Result;
|
||||
use kebab_core::{
|
||||
Block, CanonicalDocument, CodeBlock, CommonBlock, Extractor, Lang, MediaType, Metadata,
|
||||
ParserVersion, Provenance, ProvenanceEvent, ProvenanceKind, SourceSpan, SourceType, TrustLevel,
|
||||
id_for_block, id_for_doc,
|
||||
};
|
||||
use serde_json::Map;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
use crate::scaffold::{filename_from_workspace_path, join_symbol, strip_extension};
|
||||
|
||||
pub const PARSER_VERSION: &str = "code-kotlin-v1";
|
||||
|
||||
/// Kotlin AST extractor. Per-unit blocks via tree-sitter-kotlin-ng 1.1
|
||||
/// (`LANGUAGE: LanguageFn`) parsed by tree-sitter 0.26.
|
||||
pub struct KotlinAstExtractor;
|
||||
|
||||
impl KotlinAstExtractor {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for KotlinAstExtractor {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl Extractor for KotlinAstExtractor {
|
||||
fn supports(&self, m: &MediaType) -> bool {
|
||||
matches!(m, MediaType::Code(l) if l == "kotlin")
|
||||
}
|
||||
|
||||
fn parser_version(&self) -> ParserVersion {
|
||||
ParserVersion(PARSER_VERSION.to_string())
|
||||
}
|
||||
|
||||
fn extract(
|
||||
&self,
|
||||
ctx: &kebab_core::ExtractContext<'_>,
|
||||
bytes: &[u8],
|
||||
) -> Result<CanonicalDocument> {
|
||||
let asset = ctx.asset;
|
||||
if !self.supports(&asset.media_type) {
|
||||
anyhow::bail!(
|
||||
"kebab-parse-code: unsupported media_type for KotlinAstExtractor: {:?}",
|
||||
asset.media_type
|
||||
);
|
||||
}
|
||||
|
||||
let parser_version = self.parser_version();
|
||||
let doc_id = id_for_doc(&asset.workspace_path, &asset.asset_id, &parser_version);
|
||||
|
||||
let source = String::from_utf8(bytes.to_vec()).map_err(|e| {
|
||||
anyhow::anyhow!("kebab-parse-code: Kotlin source is not valid UTF-8: {e}")
|
||||
})?;
|
||||
|
||||
let blocks = build_blocks(&source, &doc_id)?;
|
||||
let unit_count = blocks.len() as u32;
|
||||
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let mut events: Vec<ProvenanceEvent> = Vec::with_capacity(2);
|
||||
events.push(ProvenanceEvent {
|
||||
at: asset.discovered_at,
|
||||
agent: "kb-source-fs".to_string(),
|
||||
kind: ProvenanceKind::Discovered,
|
||||
note: None,
|
||||
});
|
||||
events.push(ProvenanceEvent {
|
||||
at: now,
|
||||
agent: "kb-parse-code".to_string(),
|
||||
kind: ProvenanceKind::Parsed,
|
||||
note: Some(format!(
|
||||
"parser_version={}; unit_count={}",
|
||||
parser_version.0, unit_count
|
||||
)),
|
||||
});
|
||||
|
||||
let title = {
|
||||
let fname = filename_from_workspace_path(&asset.workspace_path.0);
|
||||
strip_extension(&fname)
|
||||
};
|
||||
|
||||
// Resolve the file's absolute path for repo detection. If the
|
||||
// source URI carries a relative path, anchor it at the workspace
|
||||
// root so the `.git/` walk-up starts from the right place.
|
||||
let abs_path = match &asset.source_uri {
|
||||
kebab_core::SourceUri::File(p) => {
|
||||
if p.is_absolute() {
|
||||
p.clone()
|
||||
} else {
|
||||
ctx.workspace_root.join(p)
|
||||
}
|
||||
}
|
||||
kebab_core::SourceUri::Kb(_) => ctx.workspace_root.to_path_buf(),
|
||||
};
|
||||
let (repo, git_branch, git_commit) = match crate::repo::detect_repo(&abs_path) {
|
||||
Some(r) => (Some(r.name), r.branch, r.commit),
|
||||
None => (None, None, None),
|
||||
};
|
||||
|
||||
let metadata = Metadata {
|
||||
aliases: Vec::new(),
|
||||
tags: Vec::new(),
|
||||
created_at: asset.discovered_at,
|
||||
updated_at: asset.discovered_at,
|
||||
source_type: SourceType::Note,
|
||||
trust_level: TrustLevel::Primary,
|
||||
user_id_alias: None,
|
||||
user: Map::new(),
|
||||
repo,
|
||||
git_branch,
|
||||
git_commit,
|
||||
code_lang: Some("kotlin".to_string()),
|
||||
};
|
||||
|
||||
tracing::debug!(
|
||||
target: "kebab-parse-code",
|
||||
"extracted Kotlin doc_id={} workspace_path={} units={}",
|
||||
doc_id.0,
|
||||
asset.workspace_path.0,
|
||||
unit_count
|
||||
);
|
||||
|
||||
Ok(CanonicalDocument {
|
||||
doc_id,
|
||||
source_asset_id: asset.asset_id.clone(),
|
||||
workspace_path: asset.workspace_path.clone(),
|
||||
title,
|
||||
lang: Lang("und".to_string()),
|
||||
blocks,
|
||||
metadata,
|
||||
provenance: Provenance { events },
|
||||
parser_version,
|
||||
schema_version: 1,
|
||||
doc_version: 1,
|
||||
last_chunker_version: None,
|
||||
last_embedding_version: None,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// p10-1C-JK: extract `package` declaration text from a tree-sitter-kotlin
|
||||
/// `source_file`. Returns `None` if no `package_header` (default-package
|
||||
/// Kotlin file). The package_header's single named child is a
|
||||
/// `qualified_identifier`; its slice text is the dotted path. Per design
|
||||
/// §3.4 Kotlin row.
|
||||
fn extract_package(root: tree_sitter::Node, src: &str) -> Option<String> {
|
||||
let mut cur = root.walk();
|
||||
for child in root.named_children(&mut cur) {
|
||||
if child.kind() == "package_header" {
|
||||
let mut c2 = child.walk();
|
||||
for sub in child.named_children(&mut c2) {
|
||||
let k = sub.kind();
|
||||
if k == "qualified_identifier" || k == "identifier" {
|
||||
return Some(src[sub.start_byte()..sub.end_byte()].to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Walk preceding `line_comment` / `block_comment` siblings to extend
|
||||
/// the unit's line range upward, folding leading KDoc / line comments
|
||||
/// into the unit. Modifiers / annotations live INSIDE the declaration
|
||||
/// node itself, so their lines are already inside `n.start_position()`.
|
||||
fn unit_start(n: &tree_sitter::Node) -> u32 {
|
||||
let mut start = n.start_position().row as u32 + 1;
|
||||
let mut prev = n.prev_sibling();
|
||||
while let Some(p) = prev {
|
||||
let k = p.kind();
|
||||
if k == "line_comment" || k == "block_comment" {
|
||||
start = p.start_position().row as u32 + 1;
|
||||
prev = p.prev_sibling();
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
start
|
||||
}
|
||||
|
||||
fn node_name_text<'a>(n: &tree_sitter::Node, src: &'a str) -> Option<&'a str> {
|
||||
n.child_by_field_name("name")
|
||||
.map(|c| &src[c.start_byte()..c.end_byte()])
|
||||
}
|
||||
|
||||
/// Find the first child of a node with one of the given kinds. Used to
|
||||
/// locate `class_body` / `enum_class_body` on `class_declaration` since
|
||||
/// the kotlin grammar attaches them without a `body` field name.
|
||||
fn first_child_of_kinds<'a>(
|
||||
n: &tree_sitter::Node<'a>,
|
||||
kinds: &[&str],
|
||||
) -> Option<tree_sitter::Node<'a>> {
|
||||
let mut cur = n.walk();
|
||||
n.named_children(&mut cur)
|
||||
.find(|child| kinds.contains(&child.kind()))
|
||||
}
|
||||
|
||||
/// `true` iff a `class_declaration` carries the `enum` class modifier.
|
||||
/// Detected by walking `modifiers` → `class_modifier` and checking the
|
||||
/// child text. The grammar exposes "enum" / "sealed" / "data" /
|
||||
/// "annotation" / "inner" as named `class_modifier` children of
|
||||
/// `modifiers`. We only need to know about "enum" to decide whether to
|
||||
/// look for `class_body` or `enum_class_body` and whether to skip body
|
||||
/// recursion.
|
||||
fn class_decl_is_enum(n: &tree_sitter::Node, src: &str) -> bool {
|
||||
let mut cur = n.walk();
|
||||
for child in n.named_children(&mut cur) {
|
||||
if child.kind() == "modifiers" {
|
||||
let mut c2 = child.walk();
|
||||
for sub in child.named_children(&mut c2) {
|
||||
if sub.kind() == "class_modifier" {
|
||||
let text = &src[sub.start_byte()..sub.end_byte()];
|
||||
if text == "enum" {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
fn build_blocks(
|
||||
source: &str,
|
||||
doc_id: &kebab_core::DocumentId,
|
||||
) -> anyhow::Result<Vec<kebab_core::Block>> {
|
||||
let mut parser = tree_sitter::Parser::new();
|
||||
parser
|
||||
.set_language(&tree_sitter_kotlin_ng::LANGUAGE.into())
|
||||
.map_err(|e| anyhow::anyhow!("set tree-sitter-kotlin-ng language: {e}"))?;
|
||||
let tree = parser
|
||||
.parse(source.as_bytes(), None)
|
||||
.ok_or_else(|| anyhow::anyhow!("tree-sitter failed to parse Kotlin source"))?;
|
||||
let lines: Vec<&str> = source.split('\n').collect();
|
||||
|
||||
let root = tree.root_node();
|
||||
let mod_prefix = extract_package(root, source).unwrap_or_else(|| "<unknown>".to_string());
|
||||
|
||||
// units: (symbol, line_start, line_end, is_real_semantic_unit).
|
||||
// Glue groups are pushed with a sentinel symbol + is_real=false so a
|
||||
// post-pass can decide `<module>` vs `<top-level>` (JVM family pattern).
|
||||
let mut units: Vec<(String, u32, u32, bool)> = Vec::new();
|
||||
// (is_import 0/1, s, e). `is_import` flags `import` — used by the
|
||||
// glue flush to pick `<module>` vs `<top-level>` provisional label.
|
||||
let mut glue: Vec<(usize, u32, u32)> = Vec::new();
|
||||
|
||||
walk_top(root, source, &mod_prefix, &mut units, &mut glue);
|
||||
|
||||
// `<module>` is correct only when the file produced no real unit.
|
||||
// Otherwise the import-only group becomes `<top-level>` (same
|
||||
// post-pass as 1B / 1C-Go / Java).
|
||||
let has_real_unit = units.iter().any(|(_, _, _, is_real)| *is_real);
|
||||
if has_real_unit {
|
||||
for (sym, _, _, is_real) in units.iter_mut() {
|
||||
if !*is_real && sym.ends_with("<module>") {
|
||||
let pre = &sym[..sym.len() - "<module>".len()];
|
||||
*sym = format!("{pre}<top-level>");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let total_lines = lines.len() as u32;
|
||||
let mut blocks = Vec::with_capacity(units.len());
|
||||
for (ordinal, (symbol, ls, le, _is_real)) in units.into_iter().enumerate() {
|
||||
let line_start = ls.max(1);
|
||||
let line_end = le.min(total_lines.max(1));
|
||||
let span = SourceSpan::Code {
|
||||
line_start,
|
||||
line_end,
|
||||
symbol: Some(symbol),
|
||||
lang: Some("kotlin".to_string()),
|
||||
};
|
||||
let block_id = id_for_block(doc_id, "code", &[], ordinal as u32, &span);
|
||||
let code = lines[(line_start as usize - 1)..=(line_end as usize - 1)].join("\n");
|
||||
blocks.push(Block::Code(CodeBlock {
|
||||
common: CommonBlock {
|
||||
block_id,
|
||||
heading_path: Vec::new(),
|
||||
source_span: span,
|
||||
},
|
||||
lang: Some("kotlin".to_string()),
|
||||
code,
|
||||
}));
|
||||
}
|
||||
Ok(blocks)
|
||||
}
|
||||
|
||||
/// Walk the file's top-level children — `source_file` named children:
|
||||
/// `package_header` (handled by `extract_package`), `import` (glue),
|
||||
/// `class_declaration` (class / interface / enum class), `object_declaration`,
|
||||
/// `function_declaration` (top-level), `property_declaration` (top-level),
|
||||
/// `type_alias` (currently treated as glue). Class / object bodies are
|
||||
/// recursed via [`walk_body`] with the type name pushed onto `mod_path`
|
||||
/// (JVM family pattern). Enum bodies are NOT recursed (1차 cut).
|
||||
fn walk_top(
|
||||
node: tree_sitter::Node,
|
||||
src: &str,
|
||||
mod_prefix: &str,
|
||||
units: &mut Vec<(String, u32, u32, bool)>,
|
||||
glue: &mut Vec<(usize, u32, u32)>,
|
||||
) {
|
||||
let mod_path: &[String] = &[];
|
||||
let mut cur = node.walk();
|
||||
for child in node.named_children(&mut cur) {
|
||||
let s = unit_start(&child);
|
||||
let e = child.end_position().row as u32 + 1;
|
||||
match child.kind() {
|
||||
"class_declaration" => {
|
||||
// Covers class / data class / sealed class / interface /
|
||||
// enum class — single grammar node, the modifiers child
|
||||
// distinguishes them. The body is `class_body` for
|
||||
// non-enum and `enum_class_body` for enum class; both
|
||||
// attach without a `body` field name.
|
||||
if let Some(name) = node_name_text(&child, src) {
|
||||
glue.retain(|(_, gs, _)| *gs < s);
|
||||
flush_glue(glue, units, mod_prefix, mod_path);
|
||||
let sym = join_symbol(mod_prefix, mod_path, name);
|
||||
units.push((sym, s, e, true));
|
||||
let is_enum = class_decl_is_enum(&child, src);
|
||||
if !is_enum {
|
||||
if let Some(body) = first_child_of_kinds(&child, &["class_body"]) {
|
||||
let np: Vec<String> = vec![name.to_string()];
|
||||
walk_body(body, src, mod_prefix, &np, units);
|
||||
}
|
||||
}
|
||||
// enum_class_body NOT recursed — enum constants are
|
||||
// not emitted as units (1차 scope, matches Java).
|
||||
}
|
||||
}
|
||||
"object_declaration" => {
|
||||
// Singleton object — name field is required by the grammar.
|
||||
if let Some(name) = node_name_text(&child, src) {
|
||||
glue.retain(|(_, gs, _)| *gs < s);
|
||||
flush_glue(glue, units, mod_prefix, mod_path);
|
||||
let sym = join_symbol(mod_prefix, mod_path, name);
|
||||
units.push((sym, s, e, true));
|
||||
if let Some(body) = first_child_of_kinds(&child, &["class_body"]) {
|
||||
let np: Vec<String> = vec![name.to_string()];
|
||||
walk_body(body, src, mod_prefix, &np, units);
|
||||
}
|
||||
}
|
||||
}
|
||||
"function_declaration" => {
|
||||
// Top-level Kotlin function (unlike Java).
|
||||
if let Some(name) = node_name_text(&child, src) {
|
||||
glue.retain(|(_, gs, _)| *gs < s);
|
||||
flush_glue(glue, units, mod_prefix, mod_path);
|
||||
let sym = join_symbol(mod_prefix, mod_path, name);
|
||||
units.push((sym, s, e, true));
|
||||
}
|
||||
}
|
||||
"import" => {
|
||||
glue.push((1, s, e));
|
||||
}
|
||||
// `property_declaration` (top-level val/var) and `type_alias`
|
||||
// are not emitted as standalone units in the 1차 cut — they
|
||||
// glue into the import group instead. `package_header` is
|
||||
// handled by `extract_package` (structural metadata, not a
|
||||
// unit).
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
flush_glue(glue, units, mod_prefix, mod_path);
|
||||
}
|
||||
|
||||
/// Walk a `class_body` (or object's `class_body`). Emits one unit per
|
||||
/// method / secondary constructor and recurses into nested type
|
||||
/// declarations + companion objects. Property declarations are NOT
|
||||
/// emitted (would explode unit count, parallel to Java field policy).
|
||||
///
|
||||
/// `companion_object` carries an optional `name` field — when omitted
|
||||
/// (the common case `companion object { ... }`) the implicit Kotlin
|
||||
/// convention name `Companion` is used.
|
||||
///
|
||||
/// No `glue` parameter: Kotlin imports are file-level only.
|
||||
fn walk_body(
|
||||
body: tree_sitter::Node,
|
||||
src: &str,
|
||||
mod_prefix: &str,
|
||||
mod_path: &[String],
|
||||
units: &mut Vec<(String, u32, u32, bool)>,
|
||||
) {
|
||||
let mut cur = body.walk();
|
||||
for child in body.named_children(&mut cur) {
|
||||
let s = unit_start(&child);
|
||||
let e = child.end_position().row as u32 + 1;
|
||||
match child.kind() {
|
||||
"function_declaration" => {
|
||||
if let Some(name) = node_name_text(&child, src) {
|
||||
let sym = join_symbol(mod_prefix, mod_path, name);
|
||||
units.push((sym, s, e, true));
|
||||
}
|
||||
}
|
||||
"secondary_constructor" => {
|
||||
// Kotlin secondary constructor — no `name` field on the
|
||||
// grammar node. Per design §3.4 (Java JVM convention) the
|
||||
// symbol uses the enclosing class name as the trailing
|
||||
// segment (matches the Java `<pkg>.<...>.<Class>.<Class>`
|
||||
// duplication for constructors).
|
||||
if let Some(class_name) = mod_path.last() {
|
||||
let sym = join_symbol(mod_prefix, mod_path, class_name);
|
||||
units.push((sym, s, e, true));
|
||||
}
|
||||
}
|
||||
"companion_object" => {
|
||||
// Companion's name field is OPTIONAL — fall back to the
|
||||
// Kotlin implicit name `Companion`.
|
||||
let name: &str = node_name_text(&child, src).unwrap_or("Companion");
|
||||
let sym = join_symbol(mod_prefix, mod_path, name);
|
||||
units.push((sym, s, e, true));
|
||||
if let Some(inner_body) = first_child_of_kinds(&child, &["class_body"]) {
|
||||
let mut np = mod_path.to_vec();
|
||||
np.push(name.to_string());
|
||||
walk_body(inner_body, src, mod_prefix, &np, units);
|
||||
}
|
||||
}
|
||||
"class_declaration" => {
|
||||
let name = match node_name_text(&child, src) {
|
||||
Some(n) => n,
|
||||
None => continue,
|
||||
};
|
||||
let sym = join_symbol(mod_prefix, mod_path, name);
|
||||
units.push((sym, s, e, true));
|
||||
let is_enum = class_decl_is_enum(&child, src);
|
||||
if !is_enum {
|
||||
if let Some(inner_body) = first_child_of_kinds(&child, &["class_body"]) {
|
||||
let mut np = mod_path.to_vec();
|
||||
np.push(name.to_string());
|
||||
walk_body(inner_body, src, mod_prefix, &np, units);
|
||||
}
|
||||
}
|
||||
}
|
||||
"object_declaration" => {
|
||||
let name = match node_name_text(&child, src) {
|
||||
Some(n) => n,
|
||||
None => continue,
|
||||
};
|
||||
let sym = join_symbol(mod_prefix, mod_path, name);
|
||||
units.push((sym, s, e, true));
|
||||
if let Some(inner_body) = first_child_of_kinds(&child, &["class_body"]) {
|
||||
let mut np = mod_path.to_vec();
|
||||
np.push(name.to_string());
|
||||
walk_body(inner_body, src, mod_prefix, &np, units);
|
||||
}
|
||||
}
|
||||
// property_declaration, anonymous_initializer: NOT emitted.
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn flush_glue(
|
||||
glue: &mut Vec<(usize, u32, u32)>,
|
||||
units: &mut Vec<(String, u32, u32, bool)>,
|
||||
mod_prefix: &str,
|
||||
mod_path: &[String],
|
||||
) {
|
||||
if glue.is_empty() {
|
||||
return;
|
||||
}
|
||||
let s = glue.iter().map(|(_, a, _)| *a).min().unwrap();
|
||||
let e = glue.iter().map(|(_, _, b)| *b).max().unwrap();
|
||||
// Provisional label: `<module>` only if the group is exclusively
|
||||
// imports. The post-pass demotes any `<module>` to `<top-level>` if
|
||||
// the file produced any real unit.
|
||||
let only_imports = glue.iter().all(|(is_import, _, _)| *is_import == 1);
|
||||
let label = if only_imports { "<module>" } else { "<top-level>" };
|
||||
units.push((join_symbol(mod_prefix, mod_path, label), s, e, false));
|
||||
glue.clear();
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use kebab_core::{Block, MediaType, SourceSpan};
|
||||
|
||||
fn extract_fixture() -> kebab_core::CanonicalDocument {
|
||||
let bytes = std::fs::read(concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/tests/fixtures/sample.kt"
|
||||
))
|
||||
.unwrap();
|
||||
let asset =
|
||||
crate::rust::tests_support::fixed_code_asset("crates/x/src/sample.kt", "kotlin");
|
||||
let cfg = kebab_core::ExtractConfig::default();
|
||||
let root = std::path::PathBuf::from("/tmp");
|
||||
let ctx = kebab_core::ExtractContext {
|
||||
asset: &asset,
|
||||
workspace_root: &root,
|
||||
config: &cfg,
|
||||
};
|
||||
KotlinAstExtractor::new().extract(&ctx, &bytes).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extractor_supports_only_media_code_kotlin() {
|
||||
let e = KotlinAstExtractor::new();
|
||||
assert!(e.supports(&MediaType::Code("kotlin".into())));
|
||||
assert!(!e.supports(&MediaType::Code("java".into())));
|
||||
assert!(!e.supports(&MediaType::Code("rust".into())));
|
||||
assert!(!e.supports(&MediaType::Markdown));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kotlin_units_match_design_3_4_symbols() {
|
||||
let doc = extract_fixture();
|
||||
let mut syms: Vec<String> = doc
|
||||
.blocks
|
||||
.iter()
|
||||
.filter_map(|b| match b {
|
||||
Block::Code(c) => match &c.common.source_span {
|
||||
SourceSpan::Code { symbol, lang, .. } => {
|
||||
assert_eq!(lang.as_deref(), Some("kotlin"));
|
||||
symbol.clone()
|
||||
}
|
||||
_ => None,
|
||||
},
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
syms.sort();
|
||||
// package extracted from source = com.kebab.chunk
|
||||
assert!(
|
||||
syms.iter()
|
||||
.any(|s| s == "com.kebab.chunk.MdHeadingV1Chunker"),
|
||||
"got {syms:?}"
|
||||
);
|
||||
assert!(
|
||||
syms.iter()
|
||||
.any(|s| s == "com.kebab.chunk.MdHeadingV1Chunker.chunkDoc"),
|
||||
"got {syms:?}"
|
||||
);
|
||||
assert!(
|
||||
syms.iter()
|
||||
.any(|s| s == "com.kebab.chunk.MdHeadingV1Chunker.getName"),
|
||||
"got {syms:?}"
|
||||
);
|
||||
// Implicit companion object name = Companion (grammar leaves the
|
||||
// name field unset; the extractor fills it in).
|
||||
assert!(
|
||||
syms.iter()
|
||||
.any(|s| s == "com.kebab.chunk.MdHeadingV1Chunker.Companion"),
|
||||
"got {syms:?}"
|
||||
);
|
||||
assert!(
|
||||
syms.iter()
|
||||
.any(|s| s == "com.kebab.chunk.MdHeadingV1Chunker.Companion.withName"),
|
||||
"got {syms:?}"
|
||||
);
|
||||
// interface — also via class_declaration in the grammar
|
||||
assert!(
|
||||
syms.iter().any(|s| s == "com.kebab.chunk.Stringer"),
|
||||
"got {syms:?}"
|
||||
);
|
||||
// enum class — also via class_declaration; body NOT recursed
|
||||
assert!(
|
||||
syms.iter().any(|s| s == "com.kebab.chunk.Mode"),
|
||||
"got {syms:?}"
|
||||
);
|
||||
// Kotlin top-level fn — unlike Java
|
||||
assert!(
|
||||
syms.iter().any(|s| s == "com.kebab.chunk.freeFunction"),
|
||||
"got {syms:?}"
|
||||
);
|
||||
// Singleton object + its method
|
||||
assert!(
|
||||
syms.iter().any(|s| s == "com.kebab.chunk.Singleton"),
|
||||
"got {syms:?}"
|
||||
);
|
||||
assert!(
|
||||
syms.iter().any(|s| s == "com.kebab.chunk.Singleton.ping"),
|
||||
"got {syms:?}"
|
||||
);
|
||||
// import grouped as <top-level>
|
||||
assert!(
|
||||
syms.iter().any(|s| s == "com.kebab.chunk.<top-level>"),
|
||||
"got {syms:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deterministic_across_runs() {
|
||||
let a = extract_fixture();
|
||||
for _ in 0..50 {
|
||||
assert_eq!(extract_fixture().blocks, a.blocks);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -24,7 +24,7 @@ pub fn code_lang_for_path(path: &Path) -> Option<&'static str> {
|
||||
match ext.as_str() {
|
||||
"rs" => Some("rust"),
|
||||
"py" | "pyi" => Some("python"),
|
||||
"ts" | "tsx" => Some("typescript"),
|
||||
"ts" | "tsx" | "mts" | "cts" => Some("typescript"),
|
||||
"js" | "mjs" | "cjs" | "jsx" => Some("javascript"),
|
||||
"go" => Some("go"),
|
||||
"java" => Some("java"),
|
||||
@@ -40,3 +40,82 @@ pub fn code_lang_for_path(path: &Path) -> Option<&'static str> {
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// p10-1B: workspace-relative Python file path → dotted module-path prefix.
|
||||
/// See plan §Task C for the exact rules + tasks/p10/p10-1b for the §3.4
|
||||
/// design contract.
|
||||
///
|
||||
/// Stripped source-roots: `src/`, `lib/`, and `crates/<crate>/src/`.
|
||||
/// `tests/`, `examples/`, and `benches/` are intentionally NOT stripped —
|
||||
/// they appear in test/example/bench namespaces and dropping them would
|
||||
/// conflate identical symbol names across conventional Python directories
|
||||
/// (e.g. `tests/test_foo.py` → `tests.test_foo`, not `test_foo`).
|
||||
pub fn module_path_for_python(workspace_path: &str) -> String {
|
||||
let mut p: &str = workspace_path;
|
||||
if let Some(rest) = p.strip_prefix("crates/") {
|
||||
if let Some(slash) = rest.find('/') {
|
||||
let after = &rest[slash + 1..];
|
||||
if let Some(stripped) = after.strip_prefix("src/") {
|
||||
p = stripped;
|
||||
}
|
||||
}
|
||||
} else if let Some(stripped) = p.strip_prefix("src/") {
|
||||
p = stripped;
|
||||
} else if let Some(stripped) = p.strip_prefix("lib/") {
|
||||
p = stripped;
|
||||
}
|
||||
let p = match p.strip_suffix(".py") {
|
||||
Some(s) => s,
|
||||
None => p.strip_suffix(".pyi").unwrap_or(p),
|
||||
};
|
||||
let p = if let Some(parent) = p.strip_suffix("/__init__") {
|
||||
parent
|
||||
} else if p == "__init__" {
|
||||
""
|
||||
} else {
|
||||
p
|
||||
};
|
||||
p.replace('/', ".")
|
||||
}
|
||||
|
||||
/// p10-1B: workspace-relative TS/JS file path → path-style prefix
|
||||
/// (no slash replacement, no source-root strip). See plan §Task C.
|
||||
pub fn module_path_for_tsjs(workspace_path: &str) -> String {
|
||||
let p = workspace_path;
|
||||
for ext in [".tsx", ".mts", ".cts", ".ts", ".jsx", ".mjs", ".cjs", ".js"] {
|
||||
if let Some(stripped) = p.strip_suffix(ext) {
|
||||
return stripped.to_string();
|
||||
}
|
||||
}
|
||||
p.to_string()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn module_path_for_python_strips_src_roots_and_extensions() {
|
||||
assert_eq!(module_path_for_python("kebab_eval/metrics.py"), "kebab_eval.metrics");
|
||||
assert_eq!(module_path_for_python("kebab_eval/__init__.py"), "kebab_eval");
|
||||
assert_eq!(module_path_for_python("src/foo/bar.py"), "foo.bar");
|
||||
assert_eq!(module_path_for_python("crates/x/src/foo/bar.py"), "foo.bar");
|
||||
assert_eq!(module_path_for_python("a/b/c.pyi"), "a.b.c");
|
||||
assert_eq!(module_path_for_python("standalone.py"), "standalone");
|
||||
assert_eq!(module_path_for_python("src/__init__.py"), "");
|
||||
// `tests/` is NOT a stripped source-root — it is preserved as
|
||||
// part of the module path so test symbols stay namespaced.
|
||||
assert_eq!(module_path_for_python("tests/test_foo.py"), "tests.test_foo");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn module_path_for_tsjs_keeps_slashes_and_strips_ext() {
|
||||
for ext in ["ts", "tsx", "mts", "cts", "js", "jsx", "mjs", "cjs"] {
|
||||
let p = format!("src/search/retriever/Retriever.{ext}");
|
||||
assert_eq!(module_path_for_tsjs(&p), "src/search/retriever/Retriever");
|
||||
}
|
||||
assert_eq!(module_path_for_tsjs("foo.ts"), "foo");
|
||||
assert_eq!(module_path_for_tsjs("a/b/c.ts"), "a/b/c");
|
||||
assert_eq!(module_path_for_tsjs("packages/x/src/Foo.ts"), "packages/x/src/Foo");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,12 +13,25 @@
|
||||
//! `kebab-parse-*` crates per design §8: must NOT depend on store / embed
|
||||
//! / llm / rag.
|
||||
|
||||
pub mod go;
|
||||
pub mod java;
|
||||
pub mod javascript;
|
||||
pub mod kotlin;
|
||||
pub mod lang;
|
||||
pub mod python;
|
||||
pub mod repo;
|
||||
pub mod rust;
|
||||
pub(crate) mod scaffold;
|
||||
pub mod skip;
|
||||
pub mod typescript;
|
||||
|
||||
pub use lang::code_lang_for_path;
|
||||
pub use go::{PARSER_VERSION as GO_PARSER_VERSION, GoAstExtractor};
|
||||
pub use java::{PARSER_VERSION as JAVA_PARSER_VERSION, JavaAstExtractor};
|
||||
pub use javascript::{PARSER_VERSION as JS_PARSER_VERSION, JavascriptAstExtractor};
|
||||
pub use kotlin::{PARSER_VERSION as KOTLIN_PARSER_VERSION, KotlinAstExtractor};
|
||||
pub use lang::{code_lang_for_path, module_path_for_python, module_path_for_tsjs};
|
||||
pub use python::{PARSER_VERSION as PYTHON_PARSER_VERSION, PythonAstExtractor};
|
||||
pub use repo::{RepoMeta, detect_repo};
|
||||
pub use rust::{PARSER_VERSION as RUST_PARSER_VERSION, RustAstExtractor};
|
||||
pub use skip::{BUILTIN_BLACKLIST, is_generated_file, is_oversized};
|
||||
pub use typescript::{PARSER_VERSION as TS_PARSER_VERSION, TypescriptAstExtractor};
|
||||
|
||||
437
crates/kebab-parse-code/src/python.rs
Normal file
437
crates/kebab-parse-code/src/python.rs
Normal file
@@ -0,0 +1,437 @@
|
||||
//! `kebab-parse-code::python` — tree-sitter Python AST extractor (P10-1B Task E).
|
||||
//!
|
||||
//! Implements [`kebab_core::Extractor`] for [`MediaType::Code("python")`].
|
||||
//! Walks the tree-sitter parse tree and emits one [`Block::Code`] per
|
||||
//! top-level AST semantic unit (free fn, class, each method, recursively
|
||||
//! per nested class), each carrying [`SourceSpan::Code`] with the unit's
|
||||
//! dotted self-reference symbol path prefixed by `module_path_for_python`
|
||||
//! (design §3.4). Glue declarations (`import` / `import from` /
|
||||
//! `expression_statement` / `assignment` / `global_statement` /
|
||||
//! `future_import_statement`) collapse into one grouped `<top-level>`
|
||||
//! (or `<module>`) unit.
|
||||
//!
|
||||
//! Decorators are folded into the decorated unit's line range via the
|
||||
//! `decorated_definition` unwrap arm (analog of the Rust `attribute_item`
|
||||
//! re-absorption in 1A — see §9.1).
|
||||
//!
|
||||
//! Scope follows 1A: AST unit extraction + dotted symbol paths + line
|
||||
//! ranges. Per design §3.4 / §9.1 / §9 versioning.
|
||||
|
||||
use anyhow::Result;
|
||||
use kebab_core::{
|
||||
Block, CanonicalDocument, CodeBlock, CommonBlock, Extractor, Lang, MediaType, Metadata,
|
||||
ParserVersion, Provenance, ProvenanceEvent, ProvenanceKind, SourceSpan, SourceType, TrustLevel,
|
||||
id_for_block, id_for_doc,
|
||||
};
|
||||
use serde_json::Map;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
use crate::scaffold::{filename_from_workspace_path, join_symbol, strip_extension};
|
||||
|
||||
pub const PARSER_VERSION: &str = "code-python-v1";
|
||||
|
||||
/// Python AST extractor. Per-unit blocks via tree-sitter-python 0.25
|
||||
/// (`LANGUAGE: LanguageFn`) parsed by tree-sitter 0.26.
|
||||
pub struct PythonAstExtractor;
|
||||
|
||||
impl PythonAstExtractor {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for PythonAstExtractor {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl Extractor for PythonAstExtractor {
|
||||
fn supports(&self, m: &MediaType) -> bool {
|
||||
matches!(m, MediaType::Code(l) if l == "python")
|
||||
}
|
||||
|
||||
fn parser_version(&self) -> ParserVersion {
|
||||
ParserVersion(PARSER_VERSION.to_string())
|
||||
}
|
||||
|
||||
fn extract(
|
||||
&self,
|
||||
ctx: &kebab_core::ExtractContext<'_>,
|
||||
bytes: &[u8],
|
||||
) -> Result<CanonicalDocument> {
|
||||
let asset = ctx.asset;
|
||||
if !self.supports(&asset.media_type) {
|
||||
anyhow::bail!(
|
||||
"kebab-parse-code: unsupported media_type for PythonAstExtractor: {:?}",
|
||||
asset.media_type
|
||||
);
|
||||
}
|
||||
|
||||
let parser_version = self.parser_version();
|
||||
let doc_id = id_for_doc(&asset.workspace_path, &asset.asset_id, &parser_version);
|
||||
|
||||
let source = String::from_utf8(bytes.to_vec()).map_err(|e| {
|
||||
anyhow::anyhow!("kebab-parse-code: Python source is not valid UTF-8: {e}")
|
||||
})?;
|
||||
|
||||
let mod_prefix = crate::lang::module_path_for_python(&asset.workspace_path.0);
|
||||
let blocks = build_blocks(&source, &doc_id, &mod_prefix)?;
|
||||
let unit_count = blocks.len() as u32;
|
||||
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let mut events: Vec<ProvenanceEvent> = Vec::with_capacity(2);
|
||||
events.push(ProvenanceEvent {
|
||||
at: asset.discovered_at,
|
||||
agent: "kb-source-fs".to_string(),
|
||||
kind: ProvenanceKind::Discovered,
|
||||
note: None,
|
||||
});
|
||||
events.push(ProvenanceEvent {
|
||||
at: now,
|
||||
agent: "kb-parse-code".to_string(),
|
||||
kind: ProvenanceKind::Parsed,
|
||||
note: Some(format!(
|
||||
"parser_version={}; unit_count={}",
|
||||
parser_version.0, unit_count
|
||||
)),
|
||||
});
|
||||
|
||||
let title = {
|
||||
let fname = filename_from_workspace_path(&asset.workspace_path.0);
|
||||
strip_extension(&fname)
|
||||
};
|
||||
|
||||
// Resolve the file's absolute path for repo detection. If the
|
||||
// source URI carries a relative path, anchor it at the workspace
|
||||
// root so the `.git/` walk-up starts from the right place.
|
||||
let abs_path = match &asset.source_uri {
|
||||
kebab_core::SourceUri::File(p) => {
|
||||
if p.is_absolute() {
|
||||
p.clone()
|
||||
} else {
|
||||
ctx.workspace_root.join(p)
|
||||
}
|
||||
}
|
||||
kebab_core::SourceUri::Kb(_) => ctx.workspace_root.to_path_buf(),
|
||||
};
|
||||
let (repo, git_branch, git_commit) = match crate::repo::detect_repo(&abs_path) {
|
||||
Some(r) => (Some(r.name), r.branch, r.commit),
|
||||
None => (None, None, None),
|
||||
};
|
||||
|
||||
let metadata = Metadata {
|
||||
aliases: Vec::new(),
|
||||
tags: Vec::new(),
|
||||
created_at: asset.discovered_at,
|
||||
updated_at: asset.discovered_at,
|
||||
source_type: SourceType::Note,
|
||||
trust_level: TrustLevel::Primary,
|
||||
user_id_alias: None,
|
||||
user: Map::new(),
|
||||
repo,
|
||||
git_branch,
|
||||
git_commit,
|
||||
code_lang: Some("python".to_string()),
|
||||
};
|
||||
|
||||
tracing::debug!(
|
||||
target: "kebab-parse-code",
|
||||
"extracted Python doc_id={} workspace_path={} units={}",
|
||||
doc_id.0,
|
||||
asset.workspace_path.0,
|
||||
unit_count
|
||||
);
|
||||
|
||||
Ok(CanonicalDocument {
|
||||
doc_id,
|
||||
source_asset_id: asset.asset_id.clone(),
|
||||
workspace_path: asset.workspace_path.clone(),
|
||||
title,
|
||||
lang: Lang("und".to_string()),
|
||||
blocks,
|
||||
metadata,
|
||||
provenance: Provenance { events },
|
||||
parser_version,
|
||||
schema_version: 1,
|
||||
doc_version: 1,
|
||||
last_chunker_version: None,
|
||||
last_embedding_version: None,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn build_blocks(
|
||||
source: &str,
|
||||
doc_id: &kebab_core::DocumentId,
|
||||
mod_prefix: &str,
|
||||
) -> anyhow::Result<Vec<kebab_core::Block>> {
|
||||
let mut parser = tree_sitter::Parser::new();
|
||||
parser
|
||||
.set_language(&tree_sitter_python::LANGUAGE.into())
|
||||
.map_err(|e| anyhow::anyhow!("set tree-sitter-python language: {e}"))?;
|
||||
let tree = parser
|
||||
.parse(source.as_bytes(), None)
|
||||
.ok_or_else(|| anyhow::anyhow!("tree-sitter failed to parse Python source"))?;
|
||||
let lines: Vec<&str> = source.split('\n').collect();
|
||||
|
||||
// units: (symbol, line_start, line_end, is_real_semantic_unit).
|
||||
// Glue groups are pushed with a sentinel symbol + is_real=false so a
|
||||
// post-pass can decide `<module>` vs `<top-level>` (same algorithm
|
||||
// as 1A Gap 1).
|
||||
let mut units: Vec<(String, u32, u32, bool)> = Vec::new();
|
||||
// (is_import 0/1, s, e). `is_import` flags `import_statement` /
|
||||
// `import_from_statement` / `future_import_statement` — used by the
|
||||
// glue flush to pick `<module>` vs `<top-level>` provisional label
|
||||
// (1A's `is_mod_decl` analog).
|
||||
let mut glue: Vec<(usize, u32, u32)> = Vec::new();
|
||||
|
||||
fn node_name<'a>(n: &tree_sitter::Node, src: &'a str) -> Option<&'a str> {
|
||||
n.child_by_field_name("name")
|
||||
.map(|c| &src[c.start_byte()..c.end_byte()])
|
||||
}
|
||||
/// Walk preceding `comment` siblings to extend the unit's line range
|
||||
/// upward, folding leading doc / line comments into the unit. Note
|
||||
/// that Python decorators are NOT preceding siblings — they live
|
||||
/// INSIDE a `decorated_definition` parent — so they are handled by
|
||||
/// the unwrap arm below, not here.
|
||||
fn unit_start(n: &tree_sitter::Node) -> u32 {
|
||||
let mut start = n.start_position().row as u32 + 1;
|
||||
let mut prev = n.prev_sibling();
|
||||
while let Some(p) = prev {
|
||||
if p.kind() == "comment" {
|
||||
start = p.start_position().row as u32 + 1;
|
||||
prev = p.prev_sibling();
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
start
|
||||
}
|
||||
fn walk(
|
||||
node: tree_sitter::Node,
|
||||
src: &str,
|
||||
mod_prefix: &str,
|
||||
mod_path: &[String],
|
||||
units: &mut Vec<(String, u32, u32, bool)>,
|
||||
glue: &mut Vec<(usize, u32, u32)>,
|
||||
) {
|
||||
let mut cur = node.walk();
|
||||
for child in node.named_children(&mut cur) {
|
||||
// Default unit line range — overridden by the
|
||||
// `decorated_definition` unwrap arm so decorator lines are
|
||||
// included.
|
||||
let s = unit_start(&child);
|
||||
let e = child.end_position().row as u32 + 1;
|
||||
match child.kind() {
|
||||
"function_definition" => {
|
||||
if let Some(name) = node_name(&child, src) {
|
||||
glue.retain(|(_, gs, _)| *gs < s);
|
||||
flush_glue(glue, units, mod_prefix, mod_path);
|
||||
let sym = join_symbol(mod_prefix, mod_path, name);
|
||||
units.push((sym, s, e, true));
|
||||
}
|
||||
}
|
||||
"class_definition" => {
|
||||
if let Some(name) = node_name(&child, src) {
|
||||
glue.retain(|(_, gs, _)| *gs < s);
|
||||
flush_glue(glue, units, mod_prefix, mod_path);
|
||||
let sym = join_symbol(mod_prefix, mod_path, name);
|
||||
units.push((sym, s, e, true));
|
||||
// Recurse into the class body with the class
|
||||
// name pushed onto mod_path; methods become
|
||||
// `<...>.<ClassName>.<method>` and nested
|
||||
// classes recurse further with both names.
|
||||
if let Some(body) = child.child_by_field_name("body") {
|
||||
let mut np = mod_path.to_vec();
|
||||
np.push(name.to_string());
|
||||
walk(body, src, mod_prefix, &np, units, glue);
|
||||
debug_assert!(
|
||||
glue.is_empty(),
|
||||
"inner walk must flush its glue before returning"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
"decorated_definition" => {
|
||||
// Unwrap: the inner definition supplies the symbol
|
||||
// name, but the unit's line range comes from the
|
||||
// OUTER `decorated_definition` so decorator lines
|
||||
// are folded in (analog of `attribute_item`
|
||||
// re-absorption in 1A — see plan §Task E note (b)).
|
||||
if let Some(inner) = child.child_by_field_name("definition") {
|
||||
let outer_s = s; // already includes decorators
|
||||
let outer_e = e;
|
||||
match inner.kind() {
|
||||
"function_definition" => {
|
||||
if let Some(name) = node_name(&inner, src) {
|
||||
glue.retain(|(_, gs, _)| *gs < outer_s);
|
||||
flush_glue(glue, units, mod_prefix, mod_path);
|
||||
let sym = join_symbol(mod_prefix, mod_path, name);
|
||||
units.push((sym, outer_s, outer_e, true));
|
||||
}
|
||||
}
|
||||
"class_definition" => {
|
||||
if let Some(name) = node_name(&inner, src) {
|
||||
glue.retain(|(_, gs, _)| *gs < outer_s);
|
||||
flush_glue(glue, units, mod_prefix, mod_path);
|
||||
let sym = join_symbol(mod_prefix, mod_path, name);
|
||||
units.push((sym, outer_s, outer_e, true));
|
||||
if let Some(body) = inner.child_by_field_name("body") {
|
||||
let mut np = mod_path.to_vec();
|
||||
np.push(name.to_string());
|
||||
walk(body, src, mod_prefix, &np, units, glue);
|
||||
debug_assert!(
|
||||
glue.is_empty(),
|
||||
"inner walk must flush its glue before returning"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
"import_statement" | "import_from_statement" | "future_import_statement" => {
|
||||
glue.push((1, s, e));
|
||||
}
|
||||
"expression_statement" | "assignment" | "global_statement" => {
|
||||
glue.push((0, s, e));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
flush_glue(glue, units, mod_prefix, mod_path);
|
||||
}
|
||||
fn flush_glue(
|
||||
glue: &mut Vec<(usize, u32, u32)>,
|
||||
units: &mut Vec<(String, u32, u32, bool)>,
|
||||
mod_prefix: &str,
|
||||
mod_path: &[String],
|
||||
) {
|
||||
if glue.is_empty() {
|
||||
return;
|
||||
}
|
||||
let s = glue.iter().map(|(_, a, _)| *a).min().unwrap();
|
||||
let e = glue.iter().map(|(_, _, b)| *b).max().unwrap();
|
||||
// Provisional label: `<module>` only if the group is exclusively
|
||||
// imports (1A's `only_mod_decls` analog). The post-pass below
|
||||
// demotes any `<module>` to `<top-level>` if the file produced
|
||||
// any real unit.
|
||||
let only_imports = glue.iter().all(|(is_import, _, _)| *is_import == 1);
|
||||
let label = if only_imports { "<module>" } else { "<top-level>" };
|
||||
units.push((join_symbol(mod_prefix, mod_path, label), s, e, false));
|
||||
glue.clear();
|
||||
}
|
||||
|
||||
walk(tree.root_node(), source, mod_prefix, &[], &mut units, &mut glue);
|
||||
|
||||
// `<module>` is correct only when the file produced no real unit.
|
||||
// Otherwise the import-only group becomes `<top-level>` (same
|
||||
// algorithm as 1A Gap 1). Match on the suffix so a class-nested
|
||||
// glue group (which doesn't exist in current Python AST but is
|
||||
// future-proofed) still demotes correctly.
|
||||
let has_real_unit = units.iter().any(|(_, _, _, is_real)| *is_real);
|
||||
if has_real_unit {
|
||||
for (sym, _, _, is_real) in units.iter_mut() {
|
||||
if !*is_real && sym.ends_with("<module>") {
|
||||
let pre = &sym[..sym.len() - "<module>".len()];
|
||||
*sym = format!("{pre}<top-level>");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let total_lines = lines.len() as u32;
|
||||
let mut blocks = Vec::with_capacity(units.len());
|
||||
for (ordinal, (symbol, ls, le, _is_real)) in units.into_iter().enumerate() {
|
||||
let line_start = ls.max(1);
|
||||
let line_end = le.min(total_lines.max(1));
|
||||
let span = SourceSpan::Code {
|
||||
line_start,
|
||||
line_end,
|
||||
symbol: Some(symbol),
|
||||
lang: Some("python".to_string()),
|
||||
};
|
||||
let block_id = id_for_block(doc_id, "code", &[], ordinal as u32, &span);
|
||||
let code = lines[(line_start as usize - 1)..=(line_end as usize - 1)].join("\n");
|
||||
blocks.push(Block::Code(CodeBlock {
|
||||
common: CommonBlock {
|
||||
block_id,
|
||||
heading_path: Vec::new(),
|
||||
source_span: span,
|
||||
},
|
||||
lang: Some("python".to_string()),
|
||||
code,
|
||||
}));
|
||||
}
|
||||
Ok(blocks)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use kebab_core::{Block, MediaType, SourceSpan};
|
||||
|
||||
fn extract_fixture() -> kebab_core::CanonicalDocument {
|
||||
let bytes = std::fs::read(
|
||||
concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/sample.py"),
|
||||
)
|
||||
.unwrap();
|
||||
let asset = crate::rust::tests_support::fixed_code_asset(
|
||||
"kebab_eval/metrics.py", "python",
|
||||
);
|
||||
let cfg = kebab_core::ExtractConfig::default();
|
||||
let root = std::path::PathBuf::from("/tmp");
|
||||
let ctx = kebab_core::ExtractContext {
|
||||
asset: &asset, workspace_root: &root, config: &cfg,
|
||||
};
|
||||
PythonAstExtractor::new().extract(&ctx, &bytes).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extractor_supports_only_media_code_python() {
|
||||
let e = PythonAstExtractor::new();
|
||||
assert!(e.supports(&MediaType::Code("python".into())));
|
||||
assert!(!e.supports(&MediaType::Code("rust".into())));
|
||||
assert!(!e.supports(&MediaType::Markdown));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn python_units_carry_module_prefixed_symbols() {
|
||||
let doc = extract_fixture();
|
||||
let mut syms: Vec<String> = doc.blocks.iter().map(|b| match b {
|
||||
Block::Code(c) => match &c.common.source_span {
|
||||
SourceSpan::Code { symbol, lang, .. } => {
|
||||
assert_eq!(lang.as_deref(), Some("python"));
|
||||
symbol.clone().unwrap()
|
||||
}
|
||||
_ => panic!("expected SourceSpan::Code"),
|
||||
},
|
||||
other => panic!("expected Block::Code, got {other:?}"),
|
||||
}).collect();
|
||||
syms.sort();
|
||||
assert!(syms.iter().any(|s| s == "kebab_eval.metrics.free"));
|
||||
assert!(syms.iter().any(|s| s == "kebab_eval.metrics.Foo"));
|
||||
assert!(syms.iter().any(|s| s == "kebab_eval.metrics.Foo.double"));
|
||||
assert!(syms.iter().any(|s| s == "kebab_eval.metrics.Foo.name"));
|
||||
assert!(syms.iter().any(|s| s == "kebab_eval.metrics.Outer"));
|
||||
assert!(syms.iter().any(|s| s == "kebab_eval.metrics.Outer.Inner"));
|
||||
assert!(syms.iter().any(|s| s == "kebab_eval.metrics.Outer.Inner.helper"));
|
||||
assert!(syms.iter().any(|s| s == "kebab_eval.metrics.with_decorator"));
|
||||
assert!(syms.iter().any(|s| s == "kebab_eval.metrics.<top-level>"));
|
||||
// The `@no_type_check` decorator on `free` is folded into its
|
||||
// unit's line range (decorated_definition unwrap).
|
||||
let free_src = doc.blocks.iter().find_map(|b| match b {
|
||||
Block::Code(c) if matches!(&c.common.source_span,
|
||||
SourceSpan::Code{symbol,..} if symbol.as_deref()==Some("kebab_eval.metrics.free")) => Some(c.code.clone()),
|
||||
_ => None,
|
||||
}).unwrap();
|
||||
assert!(free_src.contains("@no_type_check"), "decorator folded in: {free_src}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deterministic_across_runs() {
|
||||
let a = extract_fixture();
|
||||
for _ in 0..50 { assert_eq!(extract_fixture().blocks, a.blocks); }
|
||||
}
|
||||
}
|
||||
@@ -30,6 +30,8 @@ use kebab_core::{
|
||||
use serde_json::Map;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
use crate::scaffold::{filename_from_workspace_path, strip_extension};
|
||||
|
||||
pub const PARSER_VERSION: &str = "code-rust-v1";
|
||||
|
||||
/// Rust AST extractor. Per-unit blocks via tree-sitter-rust 0.24
|
||||
@@ -162,18 +164,6 @@ impl Extractor for RustAstExtractor {
|
||||
}
|
||||
}
|
||||
|
||||
fn filename_from_workspace_path(p: &str) -> String {
|
||||
p.rsplit('/').next().unwrap_or(p).to_string()
|
||||
}
|
||||
|
||||
fn strip_extension(filename: &str) -> String {
|
||||
match filename.rfind('.') {
|
||||
Some(0) => filename.to_string(),
|
||||
Some(idx) => filename[..idx].to_string(),
|
||||
None => filename.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn build_blocks(
|
||||
source: &str,
|
||||
doc_id: &kebab_core::DocumentId,
|
||||
@@ -393,7 +383,7 @@ mod tests {
|
||||
concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/sample.rs"),
|
||||
)
|
||||
.unwrap();
|
||||
let asset = kebab_parse_code_test_support::fixed_rust_asset("crates/x/src/sample.rs");
|
||||
let asset = tests_support::fixed_code_asset("crates/x/src/sample.rs", "rust");
|
||||
let cfg = kebab_core::ExtractConfig::default();
|
||||
let root = std::path::PathBuf::from("/tmp");
|
||||
let ctx = kebab_core::ExtractContext { asset: &asset, workspace_root: &root, config: &cfg };
|
||||
@@ -444,7 +434,7 @@ mod tests {
|
||||
/// Run the extractor on an in-memory Rust source string (no fixture
|
||||
/// file) and return (symbol, code) for every emitted block.
|
||||
fn extract_inline(source: &str) -> Vec<(String, String)> {
|
||||
let asset = kebab_parse_code_test_support::fixed_rust_asset("crates/x/src/inline.rs");
|
||||
let asset = tests_support::fixed_code_asset("crates/x/src/inline.rs", "rust");
|
||||
let cfg = kebab_core::ExtractConfig::default();
|
||||
let root = std::path::PathBuf::from("/tmp");
|
||||
let ctx = kebab_core::ExtractContext { asset: &asset, workspace_root: &root, config: &cfg };
|
||||
@@ -531,20 +521,23 @@ mod tests {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod kebab_parse_code_test_support {
|
||||
pub(crate) mod tests_support {
|
||||
use kebab_core::*;
|
||||
use time::OffsetDateTime;
|
||||
pub fn fixed_rust_asset(path: &str) -> RawAsset {
|
||||
/// Test-only `RawAsset` builder for any tree-sitter language. Shared
|
||||
/// across `rust.rs` / `python.rs` / future TS+JS extractor tests so all
|
||||
/// in-crate code-extractor tests use a single canonical fixture shape.
|
||||
pub fn fixed_code_asset(workspace_path: &str, code_lang: &str) -> RawAsset {
|
||||
RawAsset {
|
||||
asset_id: AssetId("a".repeat(64)),
|
||||
source_uri: SourceUri::File(std::path::PathBuf::from(path)),
|
||||
workspace_path: WorkspacePath(path.to_string()),
|
||||
media_type: MediaType::Code("rust".to_string()),
|
||||
source_uri: SourceUri::File(std::path::PathBuf::from(workspace_path)),
|
||||
workspace_path: WorkspacePath(workspace_path.to_string()),
|
||||
media_type: MediaType::Code(code_lang.to_string()),
|
||||
byte_len: 0,
|
||||
checksum: Checksum("b".repeat(64)),
|
||||
discovered_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
stored: AssetStorage::Reference {
|
||||
path: std::path::PathBuf::from(path),
|
||||
path: std::path::PathBuf::from(workspace_path),
|
||||
sha: Checksum("b".repeat(64)),
|
||||
},
|
||||
}
|
||||
|
||||
45
crates/kebab-parse-code/src/scaffold.rs
Normal file
45
crates/kebab-parse-code/src/scaffold.rs
Normal file
@@ -0,0 +1,45 @@
|
||||
//! `kebab-parse-code::scaffold` — shared pure helpers used by all
|
||||
//! per-language extractor modules.
|
||||
//!
|
||||
//! These are `pub(crate)` utilities extracted from the four extractor
|
||||
//! modules (rust / python / typescript / javascript) where identical
|
||||
//! copies existed. Keeping them here is the single source of truth.
|
||||
|
||||
/// Extract the last path component (filename) from a `/`-separated
|
||||
/// workspace path string.
|
||||
/// For a path like `crates/x/src/foo.rs` this returns `foo.rs`.
|
||||
pub(crate) fn filename_from_workspace_path(p: &str) -> String {
|
||||
p.rsplit('/').next().unwrap_or(p).to_string()
|
||||
}
|
||||
|
||||
/// Strip the last dot-extension from a filename string.
|
||||
/// A leading dot (hidden-file convention) is preserved as-is.
|
||||
/// `foo.rs` → `foo`, `.hidden` → `.hidden`, `noext` → `noext`.
|
||||
pub(crate) fn strip_extension(filename: &str) -> String {
|
||||
match filename.rfind('.') {
|
||||
Some(0) => filename.to_string(),
|
||||
Some(idx) => filename[..idx].to_string(),
|
||||
None => filename.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Join `(mod_prefix, mod_path, name)` into a dotted symbol string.
|
||||
///
|
||||
/// Used by Python / TypeScript / JavaScript extractors. Rust uses
|
||||
/// `::` separators instead and builds symbols inline; this helper
|
||||
/// covers the `.`-joined languages.
|
||||
///
|
||||
/// Empty `mod_prefix` (e.g. file is `__init__.py` at workspace root)
|
||||
/// drops the leading prefix segment; empty `mod_path` (file top-level)
|
||||
/// drops the class-nesting middle segment.
|
||||
pub(crate) fn join_symbol(mod_prefix: &str, mod_path: &[String], name: &str) -> String {
|
||||
let mut parts: Vec<&str> = Vec::with_capacity(mod_path.len() + 2);
|
||||
if !mod_prefix.is_empty() {
|
||||
parts.push(mod_prefix);
|
||||
}
|
||||
for p in mod_path {
|
||||
parts.push(p.as_str());
|
||||
}
|
||||
parts.push(name);
|
||||
parts.join(".")
|
||||
}
|
||||
691
crates/kebab-parse-code/src/typescript.rs
Normal file
691
crates/kebab-parse-code/src/typescript.rs
Normal file
@@ -0,0 +1,691 @@
|
||||
//! `kebab-parse-code::typescript` — tree-sitter TypeScript / TSX AST
|
||||
//! extractor (P10-1B Task H).
|
||||
//!
|
||||
//! Implements [`kebab_core::Extractor`] for [`MediaType::Code("typescript")`].
|
||||
//! Walks the tree-sitter parse tree (one of two grammars selected by the
|
||||
//! workspace path's extension — `.tsx` uses [`tree_sitter_typescript::LANGUAGE_TSX`],
|
||||
//! everything else uses [`tree_sitter_typescript::LANGUAGE_TYPESCRIPT`]) and
|
||||
//! emits one [`Block::Code`] per top-level AST semantic unit (free fn,
|
||||
//! class, each method, interface, type alias, enum, recursively per
|
||||
//! nested class), each carrying [`SourceSpan::Code`] with the unit's
|
||||
//! dotted symbol path prefixed by [`module_path_for_tsjs`].
|
||||
//!
|
||||
//! Glue declarations (`import_statement`, bare `export_statement`
|
||||
//! re-exports, `lexical_declaration` / `variable_declaration` at the
|
||||
//! module level, namespace / module declarations, etc.) collapse into
|
||||
//! one grouped `<top-level>` (or `<module>`) unit.
|
||||
//!
|
||||
//! `export_statement` is unwrapped: an `export function|class|interface
|
||||
//! |type|enum` is treated as the inner declaration arm but the unit's
|
||||
//! line range comes from the OUTER `export_statement` so the `export `
|
||||
//! prefix is folded in. `export default function () {}` / `export
|
||||
//! default class {}` (no `name` field) emits `default` as the symbol
|
||||
//! name.
|
||||
//!
|
||||
//! Scope follows 1A-2 / 1B Task E: AST unit extraction + dotted symbol
|
||||
//! paths + line ranges. Per design §3.4 / §9.1 / §9 versioning.
|
||||
|
||||
use anyhow::Result;
|
||||
use kebab_core::{
|
||||
Block, CanonicalDocument, CodeBlock, CommonBlock, Extractor, Lang, MediaType, Metadata,
|
||||
ParserVersion, Provenance, ProvenanceEvent, ProvenanceKind, SourceSpan, SourceType, TrustLevel,
|
||||
id_for_block, id_for_doc,
|
||||
};
|
||||
use serde_json::Map;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
use crate::scaffold::{filename_from_workspace_path, join_symbol, strip_extension};
|
||||
|
||||
pub const PARSER_VERSION: &str = "code-ts-v1";
|
||||
|
||||
/// TypeScript / TSX AST extractor. Per-unit blocks via
|
||||
/// tree-sitter-typescript 0.23 (`LANGUAGE_TYPESCRIPT` / `LANGUAGE_TSX`
|
||||
/// — two `LanguageFn`s, selected by extension) parsed by tree-sitter
|
||||
/// 0.26.
|
||||
pub struct TypescriptAstExtractor;
|
||||
|
||||
impl TypescriptAstExtractor {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for TypescriptAstExtractor {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl Extractor for TypescriptAstExtractor {
|
||||
fn supports(&self, m: &MediaType) -> bool {
|
||||
matches!(m, MediaType::Code(l) if l == "typescript")
|
||||
}
|
||||
|
||||
fn parser_version(&self) -> ParserVersion {
|
||||
ParserVersion(PARSER_VERSION.to_string())
|
||||
}
|
||||
|
||||
fn extract(
|
||||
&self,
|
||||
ctx: &kebab_core::ExtractContext<'_>,
|
||||
bytes: &[u8],
|
||||
) -> Result<CanonicalDocument> {
|
||||
let asset = ctx.asset;
|
||||
if !self.supports(&asset.media_type) {
|
||||
anyhow::bail!(
|
||||
"kebab-parse-code: unsupported media_type for TypescriptAstExtractor: {:?}",
|
||||
asset.media_type
|
||||
);
|
||||
}
|
||||
|
||||
let parser_version = self.parser_version();
|
||||
let doc_id = id_for_doc(&asset.workspace_path, &asset.asset_id, &parser_version);
|
||||
|
||||
let source = String::from_utf8(bytes.to_vec()).map_err(|e| {
|
||||
anyhow::anyhow!("kebab-parse-code: TypeScript source is not valid UTF-8: {e}")
|
||||
})?;
|
||||
|
||||
let mod_prefix = crate::lang::module_path_for_tsjs(&asset.workspace_path.0);
|
||||
let language = select_grammar(&asset.workspace_path.0);
|
||||
let blocks = build_blocks(&source, &doc_id, &mod_prefix, language)?;
|
||||
let unit_count = blocks.len() as u32;
|
||||
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let mut events: Vec<ProvenanceEvent> = Vec::with_capacity(2);
|
||||
events.push(ProvenanceEvent {
|
||||
at: asset.discovered_at,
|
||||
agent: "kb-source-fs".to_string(),
|
||||
kind: ProvenanceKind::Discovered,
|
||||
note: None,
|
||||
});
|
||||
events.push(ProvenanceEvent {
|
||||
at: now,
|
||||
agent: "kb-parse-code".to_string(),
|
||||
kind: ProvenanceKind::Parsed,
|
||||
note: Some(format!(
|
||||
"parser_version={}; unit_count={}",
|
||||
parser_version.0, unit_count
|
||||
)),
|
||||
});
|
||||
|
||||
let title = {
|
||||
let fname = filename_from_workspace_path(&asset.workspace_path.0);
|
||||
strip_extension(&fname)
|
||||
};
|
||||
|
||||
// Resolve the file's absolute path for repo detection. If the
|
||||
// source URI carries a relative path, anchor it at the workspace
|
||||
// root so the `.git/` walk-up starts from the right place.
|
||||
let abs_path = match &asset.source_uri {
|
||||
kebab_core::SourceUri::File(p) => {
|
||||
if p.is_absolute() {
|
||||
p.clone()
|
||||
} else {
|
||||
ctx.workspace_root.join(p)
|
||||
}
|
||||
}
|
||||
kebab_core::SourceUri::Kb(_) => ctx.workspace_root.to_path_buf(),
|
||||
};
|
||||
let (repo, git_branch, git_commit) = match crate::repo::detect_repo(&abs_path) {
|
||||
Some(r) => (Some(r.name), r.branch, r.commit),
|
||||
None => (None, None, None),
|
||||
};
|
||||
|
||||
let metadata = Metadata {
|
||||
aliases: Vec::new(),
|
||||
tags: Vec::new(),
|
||||
created_at: asset.discovered_at,
|
||||
updated_at: asset.discovered_at,
|
||||
source_type: SourceType::Note,
|
||||
trust_level: TrustLevel::Primary,
|
||||
user_id_alias: None,
|
||||
user: Map::new(),
|
||||
repo,
|
||||
git_branch,
|
||||
git_commit,
|
||||
code_lang: Some("typescript".to_string()),
|
||||
};
|
||||
|
||||
tracing::debug!(
|
||||
target: "kebab-parse-code",
|
||||
"extracted TypeScript doc_id={} workspace_path={} units={}",
|
||||
doc_id.0,
|
||||
asset.workspace_path.0,
|
||||
unit_count
|
||||
);
|
||||
|
||||
Ok(CanonicalDocument {
|
||||
doc_id,
|
||||
source_asset_id: asset.asset_id.clone(),
|
||||
workspace_path: asset.workspace_path.clone(),
|
||||
title,
|
||||
lang: Lang("und".to_string()),
|
||||
blocks,
|
||||
metadata,
|
||||
provenance: Provenance { events },
|
||||
parser_version,
|
||||
schema_version: 1,
|
||||
doc_version: 1,
|
||||
last_chunker_version: None,
|
||||
last_embedding_version: None,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Select the tree-sitter grammar based on the workspace path's
|
||||
/// extension. `.tsx` → TSX grammar; everything else (`.ts`, `.mts`,
|
||||
/// `.cts`, `.d.ts`, missing extension) → TypeScript grammar (the JSX-
|
||||
/// agnostic variants all share one grammar in tree-sitter-typescript 0.23).
|
||||
fn select_grammar(workspace_path: &str) -> tree_sitter::Language {
|
||||
if workspace_path.ends_with(".tsx") {
|
||||
tree_sitter_typescript::LANGUAGE_TSX.into()
|
||||
} else {
|
||||
tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into()
|
||||
}
|
||||
}
|
||||
|
||||
fn build_blocks(
|
||||
source: &str,
|
||||
doc_id: &kebab_core::DocumentId,
|
||||
mod_prefix: &str,
|
||||
language: tree_sitter::Language,
|
||||
) -> anyhow::Result<Vec<kebab_core::Block>> {
|
||||
let mut parser = tree_sitter::Parser::new();
|
||||
parser
|
||||
.set_language(&language)
|
||||
.map_err(|e| anyhow::anyhow!("set tree-sitter-typescript language: {e}"))?;
|
||||
let tree = parser
|
||||
.parse(source.as_bytes(), None)
|
||||
.ok_or_else(|| anyhow::anyhow!("tree-sitter failed to parse TypeScript source"))?;
|
||||
let lines: Vec<&str> = source.split('\n').collect();
|
||||
|
||||
// units: (symbol, line_start, line_end, is_real_semantic_unit).
|
||||
// Glue groups are pushed with a sentinel symbol + is_real=false so a
|
||||
// post-pass can decide `<module>` vs `<top-level>` (same algorithm
|
||||
// as 1A Gap 1 / 1B Python).
|
||||
let mut units: Vec<(String, u32, u32, bool)> = Vec::new();
|
||||
// (is_module_only_kind 0/1, s, e). `is_module_only_kind` flags
|
||||
// `import_statement` and bare re-export `export_statement`s — used by
|
||||
// the glue flush to pick `<module>` vs `<top-level>` provisional
|
||||
// label (1A's `is_mod_decl` analog).
|
||||
let mut glue: Vec<(usize, u32, u32)> = Vec::new();
|
||||
|
||||
/// Walk preceding `comment` and `decorator` siblings to extend the
|
||||
/// unit's line range upward, folding leading doc/line comments and
|
||||
/// decorators into the unit.
|
||||
///
|
||||
/// In tree-sitter-typescript 0.23, TS class-method decorators (and
|
||||
/// class-level decorators) are **`class_body` siblings** that
|
||||
/// immediately precede the `method_definition` node — they are NOT
|
||||
/// children of `method_definition`. (Contrast with
|
||||
/// tree-sitter-javascript, where the `decorator` IS stored inside
|
||||
/// `method_definition` as a named child via the `decorator` field, so
|
||||
/// `method_definition.start_row` already covers the decorator line
|
||||
/// there — no sibling walk needed in `javascript.rs`.)
|
||||
///
|
||||
/// Extending backward over `decorator` siblings here matches Python's
|
||||
/// `decorated_definition` arm behavior: the decorator line is folded
|
||||
/// into the emitted unit's line range.
|
||||
fn unit_start(n: &tree_sitter::Node) -> u32 {
|
||||
let mut start = n.start_position().row as u32 + 1;
|
||||
let mut prev = n.prev_sibling();
|
||||
while let Some(p) = prev {
|
||||
if p.kind() == "comment" || p.kind() == "decorator" {
|
||||
start = p.start_position().row as u32 + 1;
|
||||
prev = p.prev_sibling();
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
start
|
||||
}
|
||||
fn name_text<'a>(n: &tree_sitter::Node, src: &'a str) -> Option<&'a str> {
|
||||
n.child_by_field_name("name")
|
||||
.map(|c| &src[c.start_byte()..c.end_byte()])
|
||||
}
|
||||
/// Walk a class body, emitting one unit per `method_definition`.
|
||||
/// Class names already pushed onto `mod_path` by the caller, so
|
||||
/// method symbols come out as `<mod_prefix>.<Class>.<method>`.
|
||||
fn walk_class_body(
|
||||
body: tree_sitter::Node,
|
||||
src: &str,
|
||||
mod_prefix: &str,
|
||||
mod_path: &[String],
|
||||
units: &mut Vec<(String, u32, u32, bool)>,
|
||||
) {
|
||||
let mut cur = body.walk();
|
||||
for child in body.named_children(&mut cur) {
|
||||
if child.kind() == "method_definition" {
|
||||
if let Some(name) = name_text(&child, src) {
|
||||
let s = unit_start(&child);
|
||||
let e = child.end_position().row as u32 + 1;
|
||||
let sym = join_symbol(mod_prefix, mod_path, name);
|
||||
units.push((sym, s, e, true));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
fn walk(
|
||||
node: tree_sitter::Node,
|
||||
src: &str,
|
||||
mod_prefix: &str,
|
||||
mod_path: &[String],
|
||||
units: &mut Vec<(String, u32, u32, bool)>,
|
||||
glue: &mut Vec<(usize, u32, u32)>,
|
||||
) {
|
||||
let mut cur = node.walk();
|
||||
for child in node.named_children(&mut cur) {
|
||||
let s = unit_start(&child);
|
||||
let e = child.end_position().row as u32 + 1;
|
||||
match child.kind() {
|
||||
"function_declaration" => {
|
||||
if let Some(name) = name_text(&child, src) {
|
||||
glue.retain(|(_, gs, _)| *gs < s);
|
||||
flush_glue(glue, units, mod_prefix, mod_path);
|
||||
let sym = join_symbol(mod_prefix, mod_path, name);
|
||||
units.push((sym, s, e, true));
|
||||
}
|
||||
}
|
||||
"class_declaration" => {
|
||||
if let Some(name) = name_text(&child, src) {
|
||||
glue.retain(|(_, gs, _)| *gs < s);
|
||||
flush_glue(glue, units, mod_prefix, mod_path);
|
||||
let sym = join_symbol(mod_prefix, mod_path, name);
|
||||
units.push((sym, s, e, true));
|
||||
if let Some(body) = child.child_by_field_name("body") {
|
||||
let mut np = mod_path.to_vec();
|
||||
np.push(name.to_string());
|
||||
walk_class_body(body, src, mod_prefix, &np, units);
|
||||
}
|
||||
}
|
||||
}
|
||||
"interface_declaration"
|
||||
| "type_alias_declaration"
|
||||
| "enum_declaration" => {
|
||||
if let Some(name) = name_text(&child, src) {
|
||||
glue.retain(|(_, gs, _)| *gs < s);
|
||||
flush_glue(glue, units, mod_prefix, mod_path);
|
||||
let sym = join_symbol(mod_prefix, mod_path, name);
|
||||
units.push((sym, s, e, true));
|
||||
}
|
||||
}
|
||||
"export_statement" => {
|
||||
// Try field "declaration" first (export class /
|
||||
// function / interface / type / enum). If absent,
|
||||
// fall back to "value" — `export default function
|
||||
// () {}` / `export default class {}` expose the
|
||||
// anonymous function_expression / class under the
|
||||
// `value` field (TS grammar 0.23).
|
||||
let outer_s = s; // includes `export ` prefix line
|
||||
let outer_e = e;
|
||||
if let Some(inner) = child.child_by_field_name("declaration") {
|
||||
let inner_kind = inner.kind();
|
||||
match inner_kind {
|
||||
"function_declaration"
|
||||
| "class_declaration"
|
||||
| "interface_declaration"
|
||||
| "type_alias_declaration"
|
||||
| "enum_declaration" => {
|
||||
let name_opt = name_text(&inner, src).map(|s| s.to_string());
|
||||
if let Some(name) = name_opt {
|
||||
glue.retain(|(_, gs, _)| *gs < outer_s);
|
||||
flush_glue(glue, units, mod_prefix, mod_path);
|
||||
let sym =
|
||||
join_symbol(mod_prefix, mod_path, &name);
|
||||
units.push((sym, outer_s, outer_e, true));
|
||||
if inner_kind == "class_declaration" {
|
||||
if let Some(body) =
|
||||
inner.child_by_field_name("body")
|
||||
{
|
||||
let mut np = mod_path.to_vec();
|
||||
np.push(name);
|
||||
walk_class_body(
|
||||
body, src, mod_prefix, &np, units,
|
||||
);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// `export default function foo() {}`
|
||||
// path is covered by name_opt =
|
||||
// Some(_) above; the no-name path
|
||||
// here is `export default` with a
|
||||
// function_declaration that
|
||||
// somehow lacks `name`. Emit
|
||||
// `default` defensively.
|
||||
glue.retain(|(_, gs, _)| *gs < outer_s);
|
||||
flush_glue(glue, units, mod_prefix, mod_path);
|
||||
let sym =
|
||||
join_symbol(mod_prefix, mod_path, "default");
|
||||
units.push((sym, outer_s, outer_e, true));
|
||||
}
|
||||
}
|
||||
// `lexical_declaration` etc. wrapped in
|
||||
// export: treat as glue (assigned arrow
|
||||
// fns / consts don't get their own unit).
|
||||
_ => {
|
||||
glue.push((0, s, e));
|
||||
}
|
||||
}
|
||||
} else if let Some(value) = child.child_by_field_name("value") {
|
||||
// `export default <expr>`. We emit a unit only
|
||||
// for the function / class shapes (named or
|
||||
// anonymous); other value shapes are glue.
|
||||
match value.kind() {
|
||||
"function_expression"
|
||||
| "function_declaration"
|
||||
| "class"
|
||||
| "class_declaration" => {
|
||||
let name_opt =
|
||||
name_text(&value, src).map(|s| s.to_string());
|
||||
let leaf = name_opt
|
||||
.as_deref()
|
||||
.unwrap_or("default")
|
||||
.to_string();
|
||||
glue.retain(|(_, gs, _)| *gs < outer_s);
|
||||
flush_glue(glue, units, mod_prefix, mod_path);
|
||||
let sym = join_symbol(mod_prefix, mod_path, &leaf);
|
||||
units.push((sym, outer_s, outer_e, true));
|
||||
// Recurse into class body if we have one.
|
||||
if matches!(
|
||||
value.kind(),
|
||||
"class" | "class_declaration"
|
||||
) {
|
||||
if let Some(body) =
|
||||
value.child_by_field_name("body")
|
||||
{
|
||||
let mut np = mod_path.to_vec();
|
||||
np.push(leaf);
|
||||
walk_class_body(
|
||||
body, src, mod_prefix, &np, units,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
glue.push((0, s, e));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Bare `export { x };` / `export * from "..."` —
|
||||
// a re-export, glue with module-only flag set
|
||||
// (we have no `declaration` / `value` field for
|
||||
// it).
|
||||
glue.push((1, s, e));
|
||||
}
|
||||
}
|
||||
"import_statement" => {
|
||||
glue.push((1, s, e));
|
||||
}
|
||||
"lexical_declaration" | "variable_declaration" => {
|
||||
glue.push((0, s, e));
|
||||
}
|
||||
// Namespace / module declarations (rare in app code,
|
||||
// common in `.d.ts`): treat as glue per plan §Task H
|
||||
// (1B 1차 scope; documented under spec Risks).
|
||||
"internal_module" | "module" | "ambient_declaration" => {
|
||||
glue.push((0, s, e));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
flush_glue(glue, units, mod_prefix, mod_path);
|
||||
}
|
||||
fn flush_glue(
|
||||
glue: &mut Vec<(usize, u32, u32)>,
|
||||
units: &mut Vec<(String, u32, u32, bool)>,
|
||||
mod_prefix: &str,
|
||||
mod_path: &[String],
|
||||
) {
|
||||
if glue.is_empty() {
|
||||
return;
|
||||
}
|
||||
let s = glue.iter().map(|(_, a, _)| *a).min().unwrap();
|
||||
let e = glue.iter().map(|(_, _, b)| *b).max().unwrap();
|
||||
let only_module = glue.iter().all(|(is_mod, _, _)| *is_mod == 1);
|
||||
let label = if only_module { "<module>" } else { "<top-level>" };
|
||||
units.push((join_symbol(mod_prefix, mod_path, label), s, e, false));
|
||||
glue.clear();
|
||||
}
|
||||
|
||||
walk(
|
||||
tree.root_node(),
|
||||
source,
|
||||
mod_prefix,
|
||||
&[],
|
||||
&mut units,
|
||||
&mut glue,
|
||||
);
|
||||
|
||||
// `<module>` is correct only when the file produced no real unit.
|
||||
// Otherwise the import-only group becomes `<top-level>` (same
|
||||
// post-pass as 1A Gap 1 / Python).
|
||||
let has_real_unit = units.iter().any(|(_, _, _, is_real)| *is_real);
|
||||
if has_real_unit {
|
||||
for (sym, _, _, is_real) in units.iter_mut() {
|
||||
if !*is_real && sym.ends_with("<module>") {
|
||||
let pre = &sym[..sym.len() - "<module>".len()];
|
||||
*sym = format!("{pre}<top-level>");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let total_lines = lines.len() as u32;
|
||||
let mut blocks = Vec::with_capacity(units.len());
|
||||
for (ordinal, (symbol, ls, le, _is_real)) in units.into_iter().enumerate() {
|
||||
let line_start = ls.max(1);
|
||||
let line_end = le.min(total_lines.max(1));
|
||||
let span = SourceSpan::Code {
|
||||
line_start,
|
||||
line_end,
|
||||
symbol: Some(symbol),
|
||||
lang: Some("typescript".to_string()),
|
||||
};
|
||||
let block_id = id_for_block(doc_id, "code", &[], ordinal as u32, &span);
|
||||
let code = lines[(line_start as usize - 1)..=(line_end as usize - 1)].join("\n");
|
||||
blocks.push(Block::Code(CodeBlock {
|
||||
common: CommonBlock {
|
||||
block_id,
|
||||
heading_path: Vec::new(),
|
||||
source_span: span,
|
||||
},
|
||||
lang: Some("typescript".to_string()),
|
||||
code,
|
||||
}));
|
||||
}
|
||||
Ok(blocks)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use kebab_core::{Block, MediaType, SourceSpan};
|
||||
|
||||
fn extract_fixture(name: &str, workspace_path: &str) -> kebab_core::CanonicalDocument {
|
||||
let bytes = std::fs::read(format!(
|
||||
concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/{}"),
|
||||
name
|
||||
))
|
||||
.unwrap();
|
||||
let asset = crate::rust::tests_support::fixed_code_asset(workspace_path, "typescript");
|
||||
let cfg = kebab_core::ExtractConfig::default();
|
||||
let root = std::path::PathBuf::from("/tmp");
|
||||
let ctx = kebab_core::ExtractContext {
|
||||
asset: &asset,
|
||||
workspace_root: &root,
|
||||
config: &cfg,
|
||||
};
|
||||
TypescriptAstExtractor::new()
|
||||
.extract(&ctx, &bytes)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn symbols(doc: &kebab_core::CanonicalDocument) -> Vec<String> {
|
||||
let mut s: Vec<String> = doc
|
||||
.blocks
|
||||
.iter()
|
||||
.filter_map(|b| match b {
|
||||
Block::Code(c) => match &c.common.source_span {
|
||||
SourceSpan::Code { symbol, lang, .. } => {
|
||||
assert_eq!(lang.as_deref(), Some("typescript"));
|
||||
symbol.clone()
|
||||
}
|
||||
_ => None,
|
||||
},
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
s.sort();
|
||||
s
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extractor_supports_only_media_code_typescript() {
|
||||
let e = TypescriptAstExtractor::new();
|
||||
assert!(e.supports(&MediaType::Code("typescript".into())));
|
||||
assert!(!e.supports(&MediaType::Code("rust".into())));
|
||||
assert!(!e.supports(&MediaType::Markdown));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ts_units_match_design_3_4_symbols() {
|
||||
// workspace_path `src/sample.ts` → mod_prefix `src/sample`
|
||||
let doc = extract_fixture("sample.ts", "src/sample.ts");
|
||||
let syms = symbols(&doc);
|
||||
assert!(syms.iter().any(|s| s == "src/sample.add"), "got {syms:?}");
|
||||
assert!(syms.iter().any(|s| s == "src/sample.Greet"));
|
||||
assert!(syms.iter().any(|s| s == "src/sample.Maybe"));
|
||||
assert!(syms.iter().any(|s| s == "src/sample.Retriever"));
|
||||
assert!(syms.iter().any(|s| s == "src/sample.Retriever.search"));
|
||||
assert!(syms.iter().any(|s| s == "src/sample.Retriever.create"));
|
||||
assert!(syms.iter().any(|s| s == "src/sample.default"));
|
||||
assert!(syms.iter().any(|s| s == "src/sample.<top-level>"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tsx_uses_tsx_grammar_and_emits_units() {
|
||||
let doc = extract_fixture("sample.tsx", "src/sample.tsx");
|
||||
let syms = symbols(&doc);
|
||||
assert!(
|
||||
syms.iter().any(|s| s == "src/sample.Hello"),
|
||||
"got {syms:?}"
|
||||
);
|
||||
assert!(
|
||||
syms.iter().any(|s| s == "src/sample.<top-level>"),
|
||||
"arrow fn + import should roll into top-level glue"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deterministic_across_runs() {
|
||||
let a = extract_fixture("sample.ts", "src/sample.ts");
|
||||
for _ in 0..30 {
|
||||
assert_eq!(extract_fixture("sample.ts", "src/sample.ts").blocks, a.blocks);
|
||||
}
|
||||
}
|
||||
|
||||
/// Regression: TS class-method decorators are `class_body` preceding
|
||||
/// siblings (not children of `method_definition`). The `unit_start`
|
||||
/// backward walk must fold the decorator line into the emitted unit's
|
||||
/// line range, matching Python's `decorated_definition` behavior.
|
||||
#[test]
|
||||
fn class_method_decorator_folded_into_method_unit() {
|
||||
// Line 1 (1-indexed): "class Foo {"
|
||||
// Line 2: " @Log()" <- decorator
|
||||
// Line 3: " bar() { return 1; }"
|
||||
// Line 4: "}"
|
||||
let bytes = b"class Foo {\n @Log()\n bar() { return 1; }\n}\n";
|
||||
let asset = crate::rust::tests_support::fixed_code_asset("src/foo.ts", "typescript");
|
||||
let cfg = kebab_core::ExtractConfig::default();
|
||||
let root = std::path::PathBuf::from("/tmp");
|
||||
let ctx = kebab_core::ExtractContext {
|
||||
asset: &asset,
|
||||
workspace_root: &root,
|
||||
config: &cfg,
|
||||
};
|
||||
let doc = TypescriptAstExtractor::new().extract(&ctx, bytes).unwrap();
|
||||
|
||||
let bar_block = doc
|
||||
.blocks
|
||||
.iter()
|
||||
.find_map(|b| match b {
|
||||
Block::Code(c) => match &c.common.source_span {
|
||||
SourceSpan::Code { symbol, .. }
|
||||
if symbol.as_deref() == Some("src/foo.Foo.bar") =>
|
||||
{
|
||||
Some(c)
|
||||
}
|
||||
_ => None,
|
||||
},
|
||||
_ => None,
|
||||
})
|
||||
.expect("src/foo.Foo.bar block should be present");
|
||||
|
||||
// After the fix, the unit MUST include the @Log() decorator line.
|
||||
assert!(
|
||||
bar_block.code.contains("@Log()"),
|
||||
"decorator must be folded into class-method unit (Python parity); got code: {:?}",
|
||||
bar_block.code
|
||||
);
|
||||
|
||||
// line_start must be 2 (the @Log() line), NOT 3 (the bar() line).
|
||||
match &bar_block.common.source_span {
|
||||
SourceSpan::Code { line_start, .. } => {
|
||||
assert_eq!(
|
||||
*line_start, 2,
|
||||
"line_start must cover the @Log() decorator line (got {line_start})"
|
||||
);
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Class-level decorator (preceding sibling of `class_declaration` in
|
||||
/// the module root): same `unit_start` backward walk folds it in.
|
||||
/// Line 1: "@Injectable()"
|
||||
/// Line 2: "class Service {"
|
||||
/// Line 3: "}"
|
||||
#[test]
|
||||
fn ts_class_decorator_folded_into_class_unit() {
|
||||
let bytes = b"@Injectable()\nclass Service {\n}\n";
|
||||
let asset = crate::rust::tests_support::fixed_code_asset("src/svc.ts", "typescript");
|
||||
let cfg = kebab_core::ExtractConfig::default();
|
||||
let root = std::path::PathBuf::from("/tmp");
|
||||
let ctx = kebab_core::ExtractContext {
|
||||
asset: &asset,
|
||||
workspace_root: &root,
|
||||
config: &cfg,
|
||||
};
|
||||
let doc = TypescriptAstExtractor::new().extract(&ctx, bytes).unwrap();
|
||||
|
||||
let svc_block = doc
|
||||
.blocks
|
||||
.iter()
|
||||
.find_map(|b| match b {
|
||||
Block::Code(c) => match &c.common.source_span {
|
||||
SourceSpan::Code { symbol, .. }
|
||||
if symbol.as_deref() == Some("src/svc.Service") =>
|
||||
{
|
||||
Some(c)
|
||||
}
|
||||
_ => None,
|
||||
},
|
||||
_ => None,
|
||||
})
|
||||
.expect("src/svc.Service block should be present");
|
||||
|
||||
assert!(
|
||||
svc_block.code.contains("@Injectable()"),
|
||||
"class-level decorator must be folded into the class unit; got code: {:?}",
|
||||
svc_block.code
|
||||
);
|
||||
match &svc_block.common.source_span {
|
||||
SourceSpan::Code { line_start, .. } => {
|
||||
assert_eq!(
|
||||
*line_start, 1,
|
||||
"line_start must cover the @Injectable() line (got {line_start})"
|
||||
);
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
}
|
||||
34
crates/kebab-parse-code/tests/fixtures/sample.go
vendored
Normal file
34
crates/kebab-parse-code/tests/fixtures/sample.go
vendored
Normal file
@@ -0,0 +1,34 @@
|
||||
// sample.go
|
||||
package chunk
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const Version = "v1"
|
||||
|
||||
type MdHeadingV1Chunker struct {
|
||||
Name string
|
||||
}
|
||||
|
||||
// ChunkDoc returns a stub list of strings.
|
||||
func (m *MdHeadingV1Chunker) ChunkDoc(input string) []string {
|
||||
return []string{m.Name}
|
||||
}
|
||||
|
||||
func (m MdHeadingV1Chunker) Name2() string {
|
||||
return m.Name
|
||||
}
|
||||
|
||||
type Stringer interface {
|
||||
String() string
|
||||
}
|
||||
|
||||
func Free(x int) int {
|
||||
return x + 1
|
||||
}
|
||||
|
||||
func init() {
|
||||
fmt.Println(strings.ToUpper("init"))
|
||||
}
|
||||
36
crates/kebab-parse-code/tests/fixtures/sample.java
vendored
Normal file
36
crates/kebab-parse-code/tests/fixtures/sample.java
vendored
Normal file
@@ -0,0 +1,36 @@
|
||||
// sample.java
|
||||
package com.kebab.chunk;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* Heading-aware Markdown chunker.
|
||||
*/
|
||||
public class MdHeadingV1Chunker {
|
||||
private final String name;
|
||||
|
||||
public MdHeadingV1Chunker(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
public List<String> chunkDoc(String input) {
|
||||
return List.of(name, input);
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
private String name;
|
||||
public Builder withName(String n) { this.name = n; return this; }
|
||||
public MdHeadingV1Chunker build() { return new MdHeadingV1Chunker(name); }
|
||||
}
|
||||
}
|
||||
|
||||
interface Stringer {
|
||||
String asString();
|
||||
}
|
||||
|
||||
enum Mode { DEFAULT, FAST }
|
||||
9
crates/kebab-parse-code/tests/fixtures/sample.js
vendored
Normal file
9
crates/kebab-parse-code/tests/fixtures/sample.js
vendored
Normal file
@@ -0,0 +1,9 @@
|
||||
// sample.js
|
||||
import { x } from "./other";
|
||||
const ANSWER = 42;
|
||||
export function add(a, b) { return a + b; }
|
||||
export class Retriever {
|
||||
search(q) { return []; }
|
||||
static create() { return new Retriever(); }
|
||||
}
|
||||
export default function () { return 1; }
|
||||
29
crates/kebab-parse-code/tests/fixtures/sample.kt
vendored
Normal file
29
crates/kebab-parse-code/tests/fixtures/sample.kt
vendored
Normal file
@@ -0,0 +1,29 @@
|
||||
// sample.kt
|
||||
package com.kebab.chunk
|
||||
|
||||
import java.util.List
|
||||
|
||||
/**
|
||||
* Heading-aware Markdown chunker.
|
||||
*/
|
||||
class MdHeadingV1Chunker(val name: String) {
|
||||
fun chunkDoc(input: String): List<String> = listOf(name, input)
|
||||
|
||||
fun getName(): String = name
|
||||
|
||||
companion object {
|
||||
fun withName(n: String): MdHeadingV1Chunker = MdHeadingV1Chunker(n)
|
||||
}
|
||||
}
|
||||
|
||||
interface Stringer {
|
||||
fun asString(): String
|
||||
}
|
||||
|
||||
enum class Mode { DEFAULT, FAST }
|
||||
|
||||
fun freeFunction(x: Int): Int = x + 1
|
||||
|
||||
object Singleton {
|
||||
fun ping(): String = "pong"
|
||||
}
|
||||
26
crates/kebab-parse-code/tests/fixtures/sample.py
vendored
Normal file
26
crates/kebab-parse-code/tests/fixtures/sample.py
vendored
Normal file
@@ -0,0 +1,26 @@
|
||||
"""sample fixture."""
|
||||
import os
|
||||
|
||||
ANSWER = 42
|
||||
|
||||
@no_type_check
|
||||
def free(x):
|
||||
"""free fn."""
|
||||
return x + 1
|
||||
|
||||
class Foo:
|
||||
"""doc."""
|
||||
def double(self, n):
|
||||
return n * 2
|
||||
|
||||
@classmethod
|
||||
def name(cls):
|
||||
return "foo"
|
||||
|
||||
class Outer:
|
||||
class Inner:
|
||||
def helper(self):
|
||||
return True
|
||||
|
||||
def with_decorator():
|
||||
pass
|
||||
11
crates/kebab-parse-code/tests/fixtures/sample.ts
vendored
Normal file
11
crates/kebab-parse-code/tests/fixtures/sample.ts
vendored
Normal file
@@ -0,0 +1,11 @@
|
||||
// sample.ts
|
||||
import { x } from "./other";
|
||||
const ANSWER = 42;
|
||||
export interface Greet { hello(): string; }
|
||||
export type Maybe<T> = T | null;
|
||||
export function add(a: number, b: number): number { return a + b; }
|
||||
export class Retriever {
|
||||
search(q: string): string[] { return []; }
|
||||
static create(): Retriever { return new Retriever(); }
|
||||
}
|
||||
export default function () { return 1; }
|
||||
4
crates/kebab-parse-code/tests/fixtures/sample.tsx
vendored
Normal file
4
crates/kebab-parse-code/tests/fixtures/sample.tsx
vendored
Normal file
@@ -0,0 +1,4 @@
|
||||
// sample.tsx
|
||||
import React from "react";
|
||||
export function Hello({ name }: { name: string }) { return <span>{name}</span>; }
|
||||
export const App = () => <Hello name="x" />; // arrow fn assigned → glue
|
||||
@@ -9,6 +9,8 @@ fn known_extensions_map_to_canonical_identifiers() {
|
||||
("foo.pyi", Some("python")),
|
||||
("foo.ts", Some("typescript")),
|
||||
("foo.tsx", Some("typescript")),
|
||||
("foo.mts", Some("typescript")), // ESM TS — same grammar
|
||||
("foo.cts", Some("typescript")), // CommonJS TS — same grammar
|
||||
("foo.js", Some("javascript")),
|
||||
("foo.mjs", Some("javascript")),
|
||||
("foo.cjs", Some("javascript")),
|
||||
|
||||
@@ -346,6 +346,34 @@ fn run_query(
|
||||
}
|
||||
}
|
||||
|
||||
// p10-1A-1 fix (dogfood-discovered 2026-05-20): code_lang filter
|
||||
// (IN-list on metadata_json.$.code_lang). Empty Vec = no filter.
|
||||
if !filters.code_lang.is_empty() {
|
||||
let placeholders = std::iter::repeat_n("?", filters.code_lang.len())
|
||||
.collect::<Vec<_>>()
|
||||
.join(",");
|
||||
sql.push_str(&format!(
|
||||
" AND json_extract(d.metadata_json, '$.code_lang') IN ({placeholders})"
|
||||
));
|
||||
for lang in &filters.code_lang {
|
||||
params.push(Box::new(lang.clone()));
|
||||
}
|
||||
}
|
||||
|
||||
// p10-1A-1 fix (dogfood-discovered 2026-05-20): repo filter
|
||||
// (IN-list on metadata_json.$.repo). Empty Vec = no filter.
|
||||
if !filters.repo.is_empty() {
|
||||
let placeholders = std::iter::repeat_n("?", filters.repo.len())
|
||||
.collect::<Vec<_>>()
|
||||
.join(",");
|
||||
sql.push_str(&format!(
|
||||
" AND json_extract(d.metadata_json, '$.repo') IN ({placeholders})"
|
||||
));
|
||||
for repo in &filters.repo {
|
||||
params.push(Box::new(repo.clone()));
|
||||
}
|
||||
}
|
||||
|
||||
// p9-fb-36: ingested_after filter.
|
||||
// `documents.updated_at` is RFC3339 stored as TEXT (always UTC `Z` per
|
||||
// fb-32 ingest path), so lexicographic >= compare is correct — but only
|
||||
|
||||
@@ -785,6 +785,19 @@ impl TestEnv {
|
||||
body: &str,
|
||||
media: MediaType,
|
||||
updated_at: OffsetDateTime,
|
||||
) -> DocumentId {
|
||||
self.insert_doc_full_with_metadata(path, body, media, updated_at, "{}")
|
||||
}
|
||||
|
||||
/// Like `insert_doc_full` but accepts an explicit `metadata_json` string
|
||||
/// so p10-1A-1 filter tests can set `metadata.code_lang` / `metadata.repo`.
|
||||
fn insert_doc_full_with_metadata(
|
||||
&self,
|
||||
path: &str,
|
||||
body: &str,
|
||||
media: MediaType,
|
||||
updated_at: OffsetDateTime,
|
||||
metadata_json: &str,
|
||||
) -> DocumentId {
|
||||
use time::format_description::well_known::Rfc3339;
|
||||
let doc_id = self.next_id("doc");
|
||||
@@ -810,10 +823,10 @@ impl TestEnv {
|
||||
source_type, trust_level, parser_version,
|
||||
doc_version, schema_version, metadata_json,
|
||||
provenance_json, created_at, updated_at
|
||||
) VALUES (?, ?, ?, NULL, 'en', 'markdown', 'primary', 'pv1', 1, 1,
|
||||
'{}', '{\"events\":[]}',
|
||||
) VALUES (?, ?, ?, NULL, 'en', 'code', 'primary', 'pv1', 1, 1,
|
||||
?, '{\"events\":[]}',
|
||||
'2024-01-01T00:00:00Z', ?)",
|
||||
rusqlite::params![doc_id, asset_id, path, updated_at_str],
|
||||
rusqlite::params![doc_id, asset_id, path, metadata_json, updated_at_str],
|
||||
)
|
||||
.expect("insert document");
|
||||
|
||||
@@ -834,6 +847,21 @@ impl TestEnv {
|
||||
DocumentId(doc_id)
|
||||
}
|
||||
|
||||
/// Insert a code doc with explicit `code_lang` and optional `repo` in metadata.
|
||||
fn insert_code_doc(&self, path: &str, body: &str, code_lang: &str, repo: Option<&str>) -> DocumentId {
|
||||
let metadata_json = match repo {
|
||||
Some(r) => format!(r#"{{"code_lang":"{code_lang}","repo":"{r}"}}"#),
|
||||
None => format!(r#"{{"code_lang":"{code_lang}"}}"#),
|
||||
};
|
||||
self.insert_doc_full_with_metadata(
|
||||
path,
|
||||
body,
|
||||
MediaType::Markdown,
|
||||
OffsetDateTime::now_utc(),
|
||||
&metadata_json,
|
||||
)
|
||||
}
|
||||
|
||||
fn run_search(&self, query: &str, filters: &SearchFilters) -> Vec<SearchHit> {
|
||||
let r = self.inner.retriever();
|
||||
let q = SearchQuery {
|
||||
@@ -934,6 +962,52 @@ fn lexical_empty_filters_match_default_behavior() {
|
||||
assert!(!with_default.is_empty());
|
||||
}
|
||||
|
||||
// ── p10-1A-1 filter tests ────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn lexical_filter_by_code_lang() {
|
||||
// Three docs: python code, rust code, markdown (no code_lang).
|
||||
// Filter code_lang=["python"] → only the python doc should match.
|
||||
let env = TestEnv::new();
|
||||
env.insert_code_doc("src/main.py", "AsyncClient session", "python", None);
|
||||
env.insert_code_doc("src/lib.rs", "AsyncClient session", "rust", None);
|
||||
env.insert_doc("docs/guide.md", "AsyncClient session");
|
||||
|
||||
let filters = SearchFilters {
|
||||
code_lang: vec!["python".to_string()],
|
||||
..Default::default()
|
||||
};
|
||||
let hits = env.run_search("AsyncClient", &filters);
|
||||
assert_eq!(hits.len(), 1, "only python doc should match code_lang filter");
|
||||
assert!(
|
||||
hits[0].doc_path.0.ends_with(".py"),
|
||||
"expected python path, got: {}",
|
||||
hits[0].doc_path.0
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lexical_filter_by_repo() {
|
||||
// Three docs: one in repo "httpx", one in repo "requests", one with no repo.
|
||||
// Filter repo=["httpx"] → only the httpx doc should match.
|
||||
let env = TestEnv::new();
|
||||
env.insert_code_doc("httpx/client.py", "session send request", "python", Some("httpx"));
|
||||
env.insert_code_doc("requests/api.py", "session send request", "python", Some("requests"));
|
||||
env.insert_code_doc("standalone.py", "session send request", "python", None);
|
||||
|
||||
let filters = SearchFilters {
|
||||
repo: vec!["httpx".to_string()],
|
||||
..Default::default()
|
||||
};
|
||||
let hits = env.run_search("session", &filters);
|
||||
assert_eq!(hits.len(), 1, "only httpx doc should match repo filter");
|
||||
assert!(
|
||||
hits[0].doc_path.0.starts_with("httpx/"),
|
||||
"expected httpx path, got: {}",
|
||||
hits[0].doc_path.0
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lexical_snapshot_run_1() {
|
||||
// Pinned snapshot. A small, deterministic corpus; the JSON shape of
|
||||
|
||||
@@ -18,6 +18,7 @@ blake3 = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
walkdir = "2"
|
||||
ignore = "0.4"
|
||||
globset = "0.4"
|
||||
|
||||
[dev-dependencies]
|
||||
serde_json = { workspace = true }
|
||||
|
||||
@@ -86,7 +86,7 @@ impl FsSourceConnector {
|
||||
excludes.extend(scope.exclude.iter().cloned());
|
||||
let kbignore = read_kbignore(&root)?;
|
||||
|
||||
let overrides = build_overrides(&root, &excludes, &kbignore)?;
|
||||
let overrides = build_overrides(&root, &excludes, &kbignore, &scope.include)?;
|
||||
Ok((root, overrides))
|
||||
}
|
||||
|
||||
@@ -103,8 +103,6 @@ impl FsSourceConnector {
|
||||
) -> Result<(Vec<RawAsset>, FsScanSkips)> {
|
||||
let (root, overrides) = self.resolve_scan_params(scope)?;
|
||||
|
||||
log_scope_include_warning(scope);
|
||||
|
||||
let (files, skipped_entries) = walk_files_with_skips(&root, &overrides)?;
|
||||
|
||||
// Accumulate per-category skip counts and sample paths.
|
||||
@@ -284,14 +282,6 @@ fn build_assets(
|
||||
Ok(assets)
|
||||
}
|
||||
|
||||
fn log_scope_include_warning(scope: &SourceScope) {
|
||||
if !scope.include.is_empty() {
|
||||
tracing::debug!(
|
||||
count = scope.include.len(),
|
||||
"FsSourceConnector ignores scope.include — handled by extractor router"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
impl SourceConnector for FsSourceConnector {
|
||||
fn scan(&self, scope: &SourceScope) -> Result<Vec<RawAsset>> {
|
||||
|
||||
@@ -19,7 +19,9 @@ pub(crate) fn media_type_for(path: &Path) -> MediaType {
|
||||
.unwrap_or_default();
|
||||
|
||||
match ext.as_str() {
|
||||
"md" => MediaType::Markdown,
|
||||
// Markdown + MDX (markdown + JSX, treated as plain markdown — the
|
||||
// JSX islands are folded into raw passthrough by the md parser).
|
||||
"md" | "mdx" => MediaType::Markdown,
|
||||
"pdf" => MediaType::Pdf,
|
||||
|
||||
"png" => MediaType::Image(ImageType::Png),
|
||||
@@ -38,6 +40,19 @@ pub(crate) fn media_type_for(path: &Path) -> MediaType {
|
||||
// recognized code langs stay Other until their phase (1B+).
|
||||
"rs" => MediaType::Code("rust".to_string()),
|
||||
|
||||
// p10-1B: Python / TS / JS AST chunkers active.
|
||||
"py" | "pyi" => MediaType::Code("python".into()),
|
||||
// .mts / .cts are TypeScript ESM / CommonJS variants — same grammar.
|
||||
"ts" | "tsx" | "mts" | "cts" => MediaType::Code("typescript".into()),
|
||||
"js" | "mjs" | "cjs" | "jsx" => MediaType::Code("javascript".into()),
|
||||
|
||||
// p10-1C-Go: Go ingest activated.
|
||||
"go" => MediaType::Code("go".into()),
|
||||
|
||||
// p10-1C-JK: JVM family (Java + Kotlin) ingest activated.
|
||||
"java" => MediaType::Code("java".into()),
|
||||
"kt" | "kts" => MediaType::Code("kotlin".into()),
|
||||
|
||||
// Empty string (no extension) and any other extension: bucket as
|
||||
// Other and let downstream extractors decide if they support it.
|
||||
_ => MediaType::Other(ext),
|
||||
@@ -81,11 +96,48 @@ mod tests {
|
||||
media_type_for(Path::new("crates/kebab-core/src/lib.rs")),
|
||||
MediaType::Code("rust".to_string())
|
||||
);
|
||||
// non-Rust code extensions stay Other in 1A
|
||||
assert_eq!(media_type_for(Path::new("a/b.py")), MediaType::Other("py".to_string()));
|
||||
assert_eq!(media_type_for(Path::new("Cargo.toml")), MediaType::Other("toml".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn py_ts_js_files_map_to_media_code() {
|
||||
assert_eq!(media_type_for(Path::new("a/b.py")), MediaType::Code("python".into()));
|
||||
assert_eq!(media_type_for(Path::new("a/b.pyi")), MediaType::Code("python".into()));
|
||||
assert_eq!(media_type_for(Path::new("a/b.ts")), MediaType::Code("typescript".into()));
|
||||
assert_eq!(media_type_for(Path::new("a/b.tsx")), MediaType::Code("typescript".into()));
|
||||
assert_eq!(media_type_for(Path::new("a/b.js")), MediaType::Code("javascript".into()));
|
||||
assert_eq!(media_type_for(Path::new("a/b.mjs")), MediaType::Code("javascript".into()));
|
||||
assert_eq!(media_type_for(Path::new("a/b.cjs")), MediaType::Code("javascript".into()));
|
||||
assert_eq!(media_type_for(Path::new("a/b.jsx")), MediaType::Code("javascript".into()));
|
||||
assert_eq!(media_type_for(Path::new("a/b.rs")), MediaType::Code("rust".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ts_variants_mts_cts() {
|
||||
// .mts / .cts are TypeScript ESM / CommonJS — same grammar as .ts.
|
||||
assert_eq!(media_type_for(Path::new("a/b.mts")), MediaType::Code("typescript".into()));
|
||||
assert_eq!(media_type_for(Path::new("a/b.cts")), MediaType::Code("typescript".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mdx_routes_to_markdown() {
|
||||
// MDX is markdown with JSX islands; the md parser folds the JSX
|
||||
// through as raw passthrough.
|
||||
assert_eq!(media_type_for(Path::new("docs/page.mdx")), MediaType::Markdown);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn go_files_map_to_media_code_go() {
|
||||
assert_eq!(media_type_for(Path::new("a/b.go")), MediaType::Code("go".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn java_kotlin_files_map_to_media_code() {
|
||||
assert_eq!(media_type_for(Path::new("a/b.java")), MediaType::Code("java".into()));
|
||||
assert_eq!(media_type_for(Path::new("a/b.kt")), MediaType::Code("kotlin".into()));
|
||||
assert_eq!(media_type_for(Path::new("a/b.kts")), MediaType::Code("kotlin".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_and_missing_extension() {
|
||||
assert_eq!(
|
||||
|
||||
@@ -44,6 +44,7 @@ use std::collections::HashSet;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use globset::{GlobBuilder, GlobSet, GlobSetBuilder};
|
||||
use ignore::overrides::{Override, OverrideBuilder};
|
||||
use walkdir::WalkDir;
|
||||
|
||||
@@ -69,6 +70,11 @@ const DEFAULT_EXCLUDES: &[&str] = &[
|
||||
///
|
||||
/// `default_and_config` covers DEFAULT_EXCLUDES + `config.workspace.exclude`
|
||||
/// — these do NOT map to any of the three named `IngestReport` counters.
|
||||
///
|
||||
/// `include` is the compiled `scope.include` allow-list. When the set is
|
||||
/// empty (no patterns) every file passes; when non-empty a file must match
|
||||
/// at least one pattern to be accepted (directories always pass, so the
|
||||
/// walker can still descend into them).
|
||||
pub(crate) struct WalkOverrides {
|
||||
/// Merged matcher — same as today's `Override`; used for the walk decision.
|
||||
pub combined: Override,
|
||||
@@ -78,6 +84,8 @@ pub(crate) struct WalkOverrides {
|
||||
pub kebabignore: Override,
|
||||
/// Matcher built from `kebab_parse_code::BUILTIN_BLACKLIST` only.
|
||||
pub builtin: Override,
|
||||
/// Compiled allow-list from `scope.include`. Empty set = pass all.
|
||||
pub include: GlobSet,
|
||||
}
|
||||
|
||||
/// Skip attribution category. Used by the connector when counting per-source
|
||||
@@ -161,10 +169,15 @@ fn build_single_matcher_owned(root: &Path, patterns: &[String]) -> Result<Overri
|
||||
/// The three per-source matchers (`gitignore`, `kebabignore`, `builtin`) are
|
||||
/// built in addition to the combined one so the connector can attribute skips
|
||||
/// to the correct `IngestReport` counter without a second walker pass.
|
||||
///
|
||||
/// `include_patterns` (from `scope.include`) are compiled into an allow-list
|
||||
/// `GlobSet`. Empty slice → pass-all (backward-compat); non-empty → file
|
||||
/// must match at least one pattern to be accepted.
|
||||
pub(crate) fn build_overrides(
|
||||
root: &Path,
|
||||
config_exclude: &[String],
|
||||
kbignore_patterns: &[String],
|
||||
include_patterns: &[String],
|
||||
) -> Result<WalkOverrides> {
|
||||
let gitignore_patterns = read_gitignore(root)?;
|
||||
|
||||
@@ -209,14 +222,41 @@ pub(crate) fn build_overrides(
|
||||
.build()
|
||||
.context("failed to compile combined override set")?;
|
||||
|
||||
// Allow-list GlobSet: empty Vec → matches nothing (= pass all); non-empty
|
||||
// → file must match at least one glob to be accepted. We compile with
|
||||
// `case_insensitive=false` to keep the semantics consistent with the
|
||||
// OverrideBuilder exclude patterns above.
|
||||
let include = build_include_globset(include_patterns)?;
|
||||
|
||||
Ok(WalkOverrides {
|
||||
combined,
|
||||
gitignore,
|
||||
kebabignore,
|
||||
builtin,
|
||||
include,
|
||||
})
|
||||
}
|
||||
|
||||
/// Compile `scope.include` patterns into a `GlobSet` allow-list.
|
||||
///
|
||||
/// Each pattern uses `GlobBuilder` with `literal_separator = true` so that
|
||||
/// `**` can cross directory boundaries while `*` stops at `/`, matching the
|
||||
/// gitignore convention used throughout the rest of the walker.
|
||||
///
|
||||
/// An empty slice produces an empty `GlobSet` — callers interpret that as
|
||||
/// "pass all files" (no allow-list constraint).
|
||||
fn build_include_globset(patterns: &[String]) -> Result<GlobSet> {
|
||||
let mut builder = GlobSetBuilder::new();
|
||||
for pat in patterns {
|
||||
let glob = GlobBuilder::new(pat)
|
||||
.literal_separator(true)
|
||||
.build()
|
||||
.with_context(|| format!("invalid include pattern: {pat}"))?;
|
||||
builder.add(glob);
|
||||
}
|
||||
builder.build().context("failed to compile include globset")
|
||||
}
|
||||
|
||||
/// Classify why a path was excluded, using per-source matchers in spec §5.2
|
||||
/// priority order: built-in > gitignore > kebabignore > other.
|
||||
///
|
||||
@@ -391,6 +431,13 @@ pub(crate) fn walk_files_with_skips(
|
||||
}
|
||||
|
||||
if entry.file_type().is_file() {
|
||||
// Apply include allow-list: if non-empty, the file's path
|
||||
// relative to root must match at least one pattern.
|
||||
if !overrides.include.is_empty() && !overrides.include.is_match(rel) {
|
||||
// Not in the allow-list — silently drop (no skip counter;
|
||||
// the include filter is not a "skip" source in IngestReport).
|
||||
continue;
|
||||
}
|
||||
accepted.push(path.to_path_buf());
|
||||
}
|
||||
}
|
||||
@@ -406,7 +453,7 @@ mod tests {
|
||||
#[test]
|
||||
fn empty_inputs_compile_into_an_override() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let ov = build_overrides(dir.path(), &[], &[]).unwrap();
|
||||
let ov = build_overrides(dir.path(), &[], &[], &[]).unwrap();
|
||||
// Default-excludes only; non-special files should not match.
|
||||
let m = ov.combined.matched(Path::new("notes/alpha.md"), false);
|
||||
assert!(!m.is_ignore());
|
||||
@@ -415,7 +462,7 @@ mod tests {
|
||||
#[test]
|
||||
fn default_excludes_ds_store_and_resource_forks() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let ov = build_overrides(dir.path(), &[], &[]).unwrap();
|
||||
let ov = build_overrides(dir.path(), &[], &[], &[]).unwrap();
|
||||
assert!(ov.combined.matched(Path::new(".DS_Store"), false).is_ignore());
|
||||
assert!(
|
||||
ov.combined.matched(Path::new("notes/.DS_Store"), false).is_ignore()
|
||||
@@ -433,6 +480,7 @@ mod tests {
|
||||
dir.path(),
|
||||
&["*.tmp".to_string(), "node_modules/**".to_string()],
|
||||
&[],
|
||||
&[],
|
||||
)
|
||||
.unwrap();
|
||||
assert!(ov.combined.matched(Path::new("a.tmp"), false).is_ignore());
|
||||
@@ -452,6 +500,7 @@ mod tests {
|
||||
dir.path(),
|
||||
&["*.tmp".to_string()],
|
||||
&["secret/**".to_string()],
|
||||
&[],
|
||||
)
|
||||
.unwrap();
|
||||
assert!(ov.combined.matched(Path::new("a.tmp"), false).is_ignore());
|
||||
@@ -491,7 +540,7 @@ mod tests {
|
||||
fs::write(root.join("src/main.rs"), "x").unwrap();
|
||||
fs::write(root.join("node_modules/foo/bar.js"), "x").unwrap();
|
||||
|
||||
let overrides = build_overrides(root, &[], &[]).unwrap();
|
||||
let overrides = build_overrides(root, &[], &[], &[]).unwrap();
|
||||
// Override::matched expects paths relative to the builder's root.
|
||||
let m_in = overrides.combined.matched(Path::new("src/main.rs"), false);
|
||||
let m_out = overrides.combined.matched(Path::new("node_modules/foo/bar.js"), false);
|
||||
@@ -514,7 +563,7 @@ mod tests {
|
||||
fs::create_dir_all(root.join("ok")).unwrap();
|
||||
fs::write(root.join("ok/z.txt"), "z").unwrap();
|
||||
|
||||
let overrides = build_overrides(root, &[], &[]).unwrap();
|
||||
let overrides = build_overrides(root, &[], &[], &[]).unwrap();
|
||||
// Override::matched expects paths relative to the builder's root.
|
||||
for blacklisted in [
|
||||
"target/x/y.txt",
|
||||
@@ -544,7 +593,7 @@ mod tests {
|
||||
fs::create_dir_all(root.join("dist")).unwrap();
|
||||
fs::write(root.join("dist/bundle.js"), "x").unwrap();
|
||||
|
||||
let overrides = build_overrides(root, &[], &[]).unwrap();
|
||||
let overrides = build_overrides(root, &[], &[], &[]).unwrap();
|
||||
assert!(overrides.combined.matched(Path::new("a.log"), false).is_ignore());
|
||||
assert!(overrides.combined.matched(Path::new("dist/bundle.js"), false).is_ignore());
|
||||
assert!(!overrides.combined.matched(Path::new("src/main.rs"), false).is_ignore());
|
||||
@@ -562,7 +611,7 @@ mod tests {
|
||||
fs::write(root.join("src/main.rs"), "x").unwrap();
|
||||
|
||||
// No .gitignore present — patterns from .gitignore should not affect overrides.
|
||||
let overrides = build_overrides(root, &[], &[]).unwrap();
|
||||
let overrides = build_overrides(root, &[], &[], &[]).unwrap();
|
||||
assert!(!overrides.combined.matched(Path::new("a.log"), false).is_ignore());
|
||||
assert!(!overrides.combined.matched(Path::new("src/main.rs"), false).is_ignore());
|
||||
}
|
||||
@@ -577,7 +626,7 @@ mod tests {
|
||||
// semantics, but at minimum it must not produce double-`!` corruption.
|
||||
fs::write(root.join(".gitignore"), "!keep/\n").unwrap();
|
||||
// Just verify build_overrides doesn't error.
|
||||
let result = build_overrides(root, &[], &[]);
|
||||
let result = build_overrides(root, &[], &[], &[]);
|
||||
assert!(result.is_ok(), "should not error on negation pattern: {:?}", result.err());
|
||||
}
|
||||
|
||||
@@ -594,7 +643,7 @@ mod tests {
|
||||
// .gitignore entry. Builtin must win (priority order §5.2).
|
||||
fs::write(root.join(".gitignore"), "node_modules/\n").unwrap();
|
||||
|
||||
let ov = build_overrides(root, &[], &[]).unwrap();
|
||||
let ov = build_overrides(root, &[], &[], &[]).unwrap();
|
||||
// node_modules/ dir itself
|
||||
let cat = classify_skip(Path::new("node_modules"), true, &ov);
|
||||
assert_eq!(cat, SkipCategory::BuiltinBlacklist, "builtin must have priority");
|
||||
@@ -609,7 +658,7 @@ mod tests {
|
||||
let root = tmp.path();
|
||||
fs::write(root.join(".gitignore"), "*.log\n").unwrap();
|
||||
|
||||
let ov = build_overrides(root, &[], &[]).unwrap();
|
||||
let ov = build_overrides(root, &[], &[], &[]).unwrap();
|
||||
let cat = classify_skip(Path::new("app.log"), false, &ov);
|
||||
assert_eq!(cat, SkipCategory::Gitignore);
|
||||
}
|
||||
@@ -621,7 +670,7 @@ mod tests {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let root = tmp.path();
|
||||
|
||||
let ov = build_overrides(root, &[], &["*.secret".to_string()]).unwrap();
|
||||
let ov = build_overrides(root, &[], &["*.secret".to_string()], &[]).unwrap();
|
||||
let cat = classify_skip(Path::new("creds.secret"), false, &ov);
|
||||
assert_eq!(cat, SkipCategory::Kebabignore);
|
||||
}
|
||||
@@ -637,7 +686,7 @@ mod tests {
|
||||
fs::write(root.join("ok.md"), "# ok").unwrap();
|
||||
fs::write(root.join("skipme.log"), "x").unwrap();
|
||||
|
||||
let ov = build_overrides(root, &[], &[]).unwrap();
|
||||
let ov = build_overrides(root, &[], &[], &[]).unwrap();
|
||||
let (accepted, skipped_entries) = walk_files_with_skips(root, &ov).unwrap();
|
||||
|
||||
let accepted_names: Vec<_> = accepted
|
||||
@@ -677,7 +726,7 @@ mod tests {
|
||||
fs::write(root.join("node_modules/foo/bar.js"), "x").unwrap();
|
||||
fs::write(root.join("ok.md"), "# ok").unwrap();
|
||||
|
||||
let ov = build_overrides(root, &[], &[]).unwrap();
|
||||
let ov = build_overrides(root, &[], &[], &[]).unwrap();
|
||||
let (accepted, skipped_entries) = walk_files_with_skips(root, &ov).unwrap();
|
||||
|
||||
let accepted_names: Vec<_> = accepted
|
||||
|
||||
111
crates/kebab-source-fs/tests/include_allowlist.rs
Normal file
111
crates/kebab-source-fs/tests/include_allowlist.rs
Normal file
@@ -0,0 +1,111 @@
|
||||
//! Integration test: `scope.include` enforces an allow-list.
|
||||
//!
|
||||
//! Semantics (gitignore convention):
|
||||
//! - `include` is empty Vec → all files pass through (backward-compat).
|
||||
//! - `include` is non-empty → only files matching at least one pattern
|
||||
//! are accepted. `exclude` rules still apply after include.
|
||||
//!
|
||||
//! Layout (built per-test in a TempDir):
|
||||
//! root/
|
||||
//! ├── a.md
|
||||
//! ├── b.py
|
||||
//! ├── c.png
|
||||
//! └── d.pdf
|
||||
|
||||
use std::fs;
|
||||
|
||||
use kebab_config::Config;
|
||||
use kebab_core::{SourceConnector, SourceScope};
|
||||
use kebab_source_fs::FsSourceConnector;
|
||||
|
||||
fn cfg_with_root(root: &str) -> Config {
|
||||
let mut c = Config::defaults();
|
||||
c.workspace.root = root.to_string();
|
||||
c.workspace.exclude.clear();
|
||||
// Disable size / generated caps so small test files always pass.
|
||||
c.ingest.code.max_file_bytes = u64::MAX;
|
||||
c.ingest.code.max_file_lines = u32::MAX;
|
||||
c.ingest.code.skip_generated_header = false;
|
||||
c
|
||||
}
|
||||
|
||||
fn setup_mixed_dir() -> tempfile::TempDir {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let root = dir.path();
|
||||
fs::write(root.join("a.md"), b"md").unwrap();
|
||||
fs::write(root.join("b.py"), b"py").unwrap();
|
||||
fs::write(root.join("c.png"), b"\x89PNG").unwrap();
|
||||
fs::write(root.join("d.pdf"), b"%PDF").unwrap();
|
||||
dir
|
||||
}
|
||||
|
||||
/// Empty include → all 4 files pass (backward-compat).
|
||||
#[test]
|
||||
fn include_empty_accepts_all_files() {
|
||||
let dir = setup_mixed_dir();
|
||||
let conn = FsSourceConnector::new(&cfg_with_root(dir.path().to_str().unwrap())).unwrap();
|
||||
let scope = SourceScope {
|
||||
include: vec![],
|
||||
..SourceScope::default()
|
||||
};
|
||||
let assets = conn.scan(&scope).unwrap();
|
||||
let names: Vec<_> = assets.iter().map(|a| a.workspace_path.0.clone()).collect();
|
||||
assert!(names.contains(&"a.md".to_string()), "a.md missing; got: {names:?}");
|
||||
assert!(names.contains(&"b.py".to_string()), "b.py missing; got: {names:?}");
|
||||
assert!(names.contains(&"c.png".to_string()), "c.png missing; got: {names:?}");
|
||||
assert!(names.contains(&"d.pdf".to_string()), "d.pdf missing; got: {names:?}");
|
||||
assert_eq!(names.len(), 4, "expected exactly 4 files; got: {names:?}");
|
||||
}
|
||||
|
||||
/// Non-empty include → only md + py come back; png + pdf are excluded.
|
||||
#[test]
|
||||
fn include_nonempty_is_allowlist() {
|
||||
let dir = setup_mixed_dir();
|
||||
let conn = FsSourceConnector::new(&cfg_with_root(dir.path().to_str().unwrap())).unwrap();
|
||||
let scope = SourceScope {
|
||||
include: vec!["**/*.md".to_string(), "**/*.py".to_string()],
|
||||
..SourceScope::default()
|
||||
};
|
||||
let assets = conn.scan(&scope).unwrap();
|
||||
let names: Vec<_> = assets.iter().map(|a| a.workspace_path.0.clone()).collect();
|
||||
assert!(names.contains(&"a.md".to_string()), "a.md should be accepted; got: {names:?}");
|
||||
assert!(names.contains(&"b.py".to_string()), "b.py should be accepted; got: {names:?}");
|
||||
assert!(
|
||||
!names.contains(&"c.png".to_string()),
|
||||
"c.png must be rejected by include allowlist; got: {names:?}"
|
||||
);
|
||||
assert!(
|
||||
!names.contains(&"d.pdf".to_string()),
|
||||
"d.pdf must be rejected by include allowlist; got: {names:?}"
|
||||
);
|
||||
assert_eq!(names.len(), 2, "expected exactly 2 files; got: {names:?}");
|
||||
}
|
||||
|
||||
/// include + exclude are ANDed: a file matching include but also matching
|
||||
/// exclude must be rejected.
|
||||
#[test]
|
||||
fn include_and_exclude_are_anded() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let root = dir.path();
|
||||
fs::write(root.join("keep.md"), b"keep").unwrap();
|
||||
fs::write(root.join("drop.md"), b"drop").unwrap();
|
||||
fs::write(root.join("other.py"), b"py").unwrap();
|
||||
|
||||
let conn = FsSourceConnector::new(&cfg_with_root(root.to_str().unwrap())).unwrap();
|
||||
let scope = SourceScope {
|
||||
include: vec!["**/*.md".to_string()],
|
||||
exclude: vec!["drop.md".to_string()],
|
||||
..SourceScope::default()
|
||||
};
|
||||
let assets = conn.scan(&scope).unwrap();
|
||||
let names: Vec<_> = assets.iter().map(|a| a.workspace_path.0.clone()).collect();
|
||||
assert!(names.contains(&"keep.md".to_string()), "keep.md should be accepted; got: {names:?}");
|
||||
assert!(
|
||||
!names.contains(&"drop.md".to_string()),
|
||||
"drop.md should be excluded (matched exclude); got: {names:?}"
|
||||
);
|
||||
assert!(
|
||||
!names.contains(&"other.py".to_string()),
|
||||
"other.py should be excluded (not in include); got: {names:?}"
|
||||
);
|
||||
}
|
||||
@@ -56,5 +56,6 @@
|
||||
"skipped_kebabignore": 0,
|
||||
"skipped_size_exceeded": 0,
|
||||
"unchanged": 0,
|
||||
"purged_deleted_files": 0,
|
||||
"updated": 1
|
||||
}
|
||||
|
||||
@@ -264,6 +264,28 @@ impl kebab_core::DocumentStore for SqliteStore {
|
||||
}))
|
||||
}
|
||||
|
||||
fn get_asset(
|
||||
&self,
|
||||
id: &kebab_core::AssetId,
|
||||
) -> Result<Option<kebab_core::RawAsset>> {
|
||||
let conn = self.lock_conn();
|
||||
let result = conn.query_row(
|
||||
r#"SELECT
|
||||
asset_id, source_uri, workspace_path, media_type,
|
||||
byte_len, checksum, storage_kind, storage_path,
|
||||
discovered_at
|
||||
FROM assets
|
||||
WHERE asset_id = ?"#,
|
||||
rusqlite::params![id.0.as_str()],
|
||||
asset_from_row,
|
||||
);
|
||||
match result {
|
||||
Ok(asset) => Ok(Some(asset)),
|
||||
Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
|
||||
Err(e) => Err(e.into()),
|
||||
}
|
||||
}
|
||||
|
||||
fn get_asset_by_workspace_path(
|
||||
&self,
|
||||
path: &kebab_core::WorkspacePath,
|
||||
@@ -286,6 +308,88 @@ impl kebab_core::DocumentStore for SqliteStore {
|
||||
}
|
||||
}
|
||||
|
||||
fn get_document_by_workspace_path(
|
||||
&self,
|
||||
path: &kebab_core::WorkspacePath,
|
||||
) -> Result<Option<kebab_core::CanonicalDocument>> {
|
||||
let conn = self.lock_conn();
|
||||
let row: Option<DocumentRow> = conn
|
||||
.query_row(
|
||||
"SELECT
|
||||
doc_id, asset_id, workspace_path, title, lang,
|
||||
source_type, trust_level, parser_version,
|
||||
doc_version, schema_version, metadata_json,
|
||||
provenance_json, created_at, updated_at,
|
||||
last_chunker_version, last_embedding_version
|
||||
FROM documents WHERE workspace_path = ?",
|
||||
params![path.0],
|
||||
document_row_from_sql,
|
||||
)
|
||||
.map(Some)
|
||||
.or_else(rows_optional)
|
||||
.map_err(StoreError::from)?;
|
||||
let Some(row) = row else { return Ok(None) };
|
||||
|
||||
let doc_id = kebab_core::DocumentId(row.doc_id.clone());
|
||||
let mut blocks_stmt = conn
|
||||
.prepare(
|
||||
"SELECT payload_json FROM blocks
|
||||
WHERE doc_id = ? ORDER BY ordinal ASC",
|
||||
)
|
||||
.map_err(StoreError::from)?;
|
||||
let block_rows = blocks_stmt
|
||||
.query_map(params![row.doc_id], |r| {
|
||||
let payload_json: String = r.get(0)?;
|
||||
Ok(payload_json)
|
||||
})
|
||||
.map_err(StoreError::from)?;
|
||||
let mut blocks: Vec<kebab_core::Block> = Vec::new();
|
||||
for block_row in block_rows {
|
||||
let payload_json = block_row.map_err(StoreError::from)?;
|
||||
let block: kebab_core::Block = serde_json::from_str(&payload_json)
|
||||
.context("deserialize block payload_json")?;
|
||||
blocks.push(block);
|
||||
}
|
||||
|
||||
let metadata: kebab_core::Metadata = serde_json::from_str(&row.metadata_json)
|
||||
.context("deserialize metadata_json")?;
|
||||
let provenance: kebab_core::Provenance =
|
||||
serde_json::from_str(&row.provenance_json)
|
||||
.context("deserialize provenance_json")?;
|
||||
|
||||
Ok(Some(kebab_core::CanonicalDocument {
|
||||
doc_id,
|
||||
source_asset_id: kebab_core::AssetId(row.asset_id),
|
||||
workspace_path: kebab_core::WorkspacePath(row.workspace_path),
|
||||
title: row.title.unwrap_or_default(),
|
||||
lang: kebab_core::Lang(row.lang.unwrap_or_default()),
|
||||
blocks,
|
||||
metadata,
|
||||
provenance,
|
||||
parser_version: kebab_core::ParserVersion(row.parser_version),
|
||||
schema_version: row.schema_version as u32,
|
||||
doc_version: row.doc_version as u32,
|
||||
last_chunker_version: row.last_chunker_version.map(kebab_core::ChunkerVersion),
|
||||
last_embedding_version: row.last_embedding_version.map(kebab_core::EmbeddingVersion),
|
||||
}))
|
||||
}
|
||||
|
||||
fn all_workspace_paths(&self) -> Result<Vec<kebab_core::WorkspacePath>> {
|
||||
let conn = self.lock_conn();
|
||||
let mut stmt = conn
|
||||
.prepare("SELECT workspace_path FROM documents")
|
||||
.map_err(StoreError::from)?;
|
||||
let rows = stmt
|
||||
.query_map([], |r| r.get::<_, String>(0))
|
||||
.map_err(StoreError::from)?;
|
||||
let mut out = Vec::new();
|
||||
for row in rows {
|
||||
let path = row.map_err(StoreError::from)?;
|
||||
out.push(kebab_core::WorkspacePath(path));
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn list_documents(
|
||||
&self,
|
||||
filter: &kebab_core::DocFilter,
|
||||
@@ -550,7 +654,8 @@ fn rows_optional<T>(err: rusqlite::Error) -> rusqlite::Result<Option<T>> {
|
||||
|
||||
/// Reconstruct a [`kebab_core::RawAsset`] from one `assets` row.
|
||||
/// Row mapper for `RawAsset`. Column names are self-documenting; the
|
||||
/// SELECT in [`DocumentStore::get_asset_by_workspace_path`] must include
|
||||
/// SELECTs in [`DocumentStore::get_asset`] and
|
||||
/// [`DocumentStore::get_asset_by_workspace_path`] must both include
|
||||
/// all nine columns by their schema names.
|
||||
fn asset_from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result<kebab_core::RawAsset> {
|
||||
use std::path::PathBuf;
|
||||
|
||||
@@ -153,6 +153,34 @@ impl SqliteStore {
|
||||
}
|
||||
}
|
||||
|
||||
// p10-1A-1 fix (dogfood-discovered 2026-05-20): code_lang filter
|
||||
// (IN-list on metadata_json.$.code_lang). Empty Vec = no filter.
|
||||
if !filters.code_lang.is_empty() {
|
||||
let placeholders = std::iter::repeat_n("?", filters.code_lang.len())
|
||||
.collect::<Vec<_>>()
|
||||
.join(",");
|
||||
sql.push_str(&format!(
|
||||
" AND json_extract(d.metadata_json, '$.code_lang') IN ({placeholders})"
|
||||
));
|
||||
for lang in &filters.code_lang {
|
||||
bind.push(Box::new(lang.clone()));
|
||||
}
|
||||
}
|
||||
|
||||
// p10-1A-1 fix (dogfood-discovered 2026-05-20): repo filter
|
||||
// (IN-list on metadata_json.$.repo). Empty Vec = no filter.
|
||||
if !filters.repo.is_empty() {
|
||||
let placeholders = std::iter::repeat_n("?", filters.repo.len())
|
||||
.collect::<Vec<_>>()
|
||||
.join(",");
|
||||
sql.push_str(&format!(
|
||||
" AND json_extract(d.metadata_json, '$.repo') IN ({placeholders})"
|
||||
));
|
||||
for repo in &filters.repo {
|
||||
bind.push(Box::new(repo.clone()));
|
||||
}
|
||||
}
|
||||
|
||||
// p9-fb-36: ingested_after filter.
|
||||
// `documents.updated_at` is RFC3339 TEXT (UTC `Z` per fb-32);
|
||||
// lexicographic >= compare is correct — but only when the filter
|
||||
@@ -408,6 +436,78 @@ mod tests {
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
/// Variant of `seed_committed_full` that additionally accepts a
|
||||
/// `metadata_json` string so p10-1A-1 filter tests can set
|
||||
/// `metadata.code_lang` / `metadata.repo` without going through the
|
||||
/// full ingest pipeline.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn seed_committed_with_metadata(
|
||||
store: &SqliteStore,
|
||||
chunk_id: &str,
|
||||
doc_id: &str,
|
||||
workspace_path: &str,
|
||||
media_type_json: &str,
|
||||
metadata_json: &str,
|
||||
) {
|
||||
let asset_id = format!("a{}", &doc_id[..31]);
|
||||
{
|
||||
let conn = store.lock_conn();
|
||||
conn.execute(
|
||||
"INSERT INTO assets (
|
||||
asset_id, source_uri, workspace_path, media_type, byte_len,
|
||||
checksum, storage_kind, storage_path, discovered_at
|
||||
) VALUES (?, ?, ?, ?, 0, 'deadbeefdeadbeefdeadbeefdeadbeef',
|
||||
'reference', ?, '1970-01-01T00:00:00Z')",
|
||||
params![
|
||||
asset_id,
|
||||
format!("file://{workspace_path}"),
|
||||
workspace_path,
|
||||
media_type_json,
|
||||
workspace_path,
|
||||
],
|
||||
)
|
||||
.unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO documents (
|
||||
doc_id, asset_id, workspace_path, title, lang, source_type,
|
||||
trust_level, parser_version, doc_version, schema_version,
|
||||
metadata_json, provenance_json, created_at, updated_at
|
||||
) VALUES (?, ?, ?, NULL, 'en', 'code', 'primary', 'v1', 1, 1,
|
||||
?, '{}', '1970-01-01T00:00:00Z', '1970-01-01T00:00:00Z')",
|
||||
params![doc_id, asset_id, workspace_path, metadata_json],
|
||||
)
|
||||
.unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO chunks (
|
||||
chunk_id, doc_id, text, heading_path_json, section_label,
|
||||
source_spans_json, token_estimate, chunker_version,
|
||||
policy_hash, block_ids_json, created_at
|
||||
) VALUES (?, ?, 'code snippet', '[]', NULL, '[]', 1, 'v1', 'h', '[]',
|
||||
'1970-01-01T00:00:00Z')",
|
||||
params![chunk_id, doc_id],
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let embed_row = EmbeddingRecordRow {
|
||||
embedding_id: format!("e{}", &chunk_id[..31]),
|
||||
chunk_id: chunk_id.to_string(),
|
||||
model_id: "m".to_string(),
|
||||
model_version: "v1".to_string(),
|
||||
dimensions: 4,
|
||||
lance_table: "t".to_string(),
|
||||
created_at: OffsetDateTime::UNIX_EPOCH,
|
||||
};
|
||||
store
|
||||
.put_embedding_records_pending(std::slice::from_ref(&embed_row))
|
||||
.unwrap();
|
||||
store
|
||||
.mark_embedding_records_committed(std::slice::from_ref(
|
||||
&embed_row.embedding_id,
|
||||
))
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
fn cid(s: &str) -> ChunkId {
|
||||
ChunkId(s.to_string())
|
||||
}
|
||||
@@ -671,6 +771,78 @@ mod tests {
|
||||
assert_eq!(out, vec![cid(c1)], "doc_id filter must scope to the target doc only");
|
||||
}
|
||||
|
||||
// ── p10-1A-1 new filter arms ─────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn filter_chunks_code_lang_keeps_matching_lang() {
|
||||
// c1 = python, c2 = rust, c3 = markdown (no code_lang).
|
||||
// Filter code_lang=["python"] → only c1 survives.
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let store = open_store(&tmp);
|
||||
let c1 = "11111111111111111111111111111111";
|
||||
let c2 = "22222222222222222222222222222222";
|
||||
let c3 = "33333333333333333333333333333333";
|
||||
seed_committed_with_metadata(
|
||||
&store, c1, "d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1",
|
||||
"src/main.py", r#""code""#,
|
||||
r#"{"code_lang":"python"}"#,
|
||||
);
|
||||
seed_committed_with_metadata(
|
||||
&store, c2, "d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2",
|
||||
"src/lib.rs", r#""code""#,
|
||||
r#"{"code_lang":"rust"}"#,
|
||||
);
|
||||
seed_committed_with_metadata(
|
||||
&store, c3, "d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3",
|
||||
"README.md", r#""markdown""#,
|
||||
r#"{}"#,
|
||||
);
|
||||
|
||||
let f = SearchFilters {
|
||||
code_lang: vec!["python".to_string()],
|
||||
..Default::default()
|
||||
};
|
||||
let out = store
|
||||
.filter_chunks(&[cid(c1), cid(c2), cid(c3)], &f)
|
||||
.unwrap();
|
||||
assert_eq!(out, vec![cid(c1)], "only python chunk should survive code_lang filter");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn filter_chunks_repo_keeps_matching_repo() {
|
||||
// c1 = repo "httpx", c2 = repo "requests", c3 = no repo.
|
||||
// Filter repo=["httpx"] → only c1 survives.
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let store = open_store(&tmp);
|
||||
let c1 = "11111111111111111111111111111111";
|
||||
let c2 = "22222222222222222222222222222222";
|
||||
let c3 = "33333333333333333333333333333333";
|
||||
seed_committed_with_metadata(
|
||||
&store, c1, "d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1",
|
||||
"httpx/client.py", r#""code""#,
|
||||
r#"{"repo":"httpx","code_lang":"python"}"#,
|
||||
);
|
||||
seed_committed_with_metadata(
|
||||
&store, c2, "d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2",
|
||||
"requests/api.py", r#""code""#,
|
||||
r#"{"repo":"requests","code_lang":"python"}"#,
|
||||
);
|
||||
seed_committed_with_metadata(
|
||||
&store, c3, "d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3",
|
||||
"standalone.py", r#""code""#,
|
||||
r#"{"code_lang":"python"}"#,
|
||||
);
|
||||
|
||||
let f = SearchFilters {
|
||||
repo: vec!["httpx".to_string()],
|
||||
..Default::default()
|
||||
};
|
||||
let out = store
|
||||
.filter_chunks(&[cid(c1), cid(c2), cid(c3)], &f)
|
||||
.unwrap();
|
||||
assert_eq!(out, vec![cid(c1)], "only httpx chunk should survive repo filter");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn filter_chunks_ingested_after_non_utc_offset_compares_as_instant() {
|
||||
// Regression test for the non-UTC offset lex-compare bug.
|
||||
|
||||
@@ -35,4 +35,4 @@ pub use error::StoreError;
|
||||
pub use eval::{EvalQueryResultRecord, EvalRunRecord, EvalRunRow};
|
||||
pub use fts::rebuild_chunks_fts;
|
||||
pub use jobs::IngestRunRow;
|
||||
pub use store::{CountSummary, NotIndexed, SqliteStore};
|
||||
pub use store::{CountSummary, NotIndexed, SqliteStore, purge_deleted_workspace_path};
|
||||
|
||||
@@ -540,10 +540,132 @@ pub(crate) fn purge_orphan_at_workspace_path(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Purge all stored data for a document whose on-disk file has been
|
||||
/// deleted (as opposed to content-changed, which is handled by
|
||||
/// `purge_orphan_at_workspace_path`).
|
||||
///
|
||||
/// Returns the `chunk_id`s that were associated with the document so
|
||||
/// the caller can issue a matching `VectorStore::delete_by_chunk_ids`
|
||||
/// on the LanceDB side.
|
||||
///
|
||||
/// Deletion order:
|
||||
/// 1. Collect chunk_ids (before cascade removes them).
|
||||
/// 2. DELETE the `documents` row → CASCADE clears `blocks`, `chunks`,
|
||||
/// `embedding_records`.
|
||||
/// 3. DELETE the `assets` row **only if no other document still
|
||||
/// references it** (twin-file protection — `assets` can be shared
|
||||
/// across identical-content files via the blake3 PK).
|
||||
/// 4. If the asset was `storage_kind = 'copied'`, best-effort delete
|
||||
/// the on-disk byte file at `storage_path`.
|
||||
///
|
||||
/// Returns `Ok(vec![])` when no document exists at `workspace_path`
|
||||
/// (idempotent — caller doesn't need to pre-check).
|
||||
pub fn purge_deleted_workspace_path(
|
||||
store: &SqliteStore,
|
||||
workspace_path: &kebab_core::WorkspacePath,
|
||||
) -> anyhow::Result<Vec<kebab_core::ChunkId>> {
|
||||
let conn = store.lock_conn();
|
||||
|
||||
// Look up the document + its asset_id.
|
||||
let doc_row: Option<(String, String)> = conn
|
||||
.query_row(
|
||||
"SELECT doc_id, asset_id FROM documents WHERE workspace_path = ?",
|
||||
rusqlite::params![workspace_path.0],
|
||||
|r| Ok((r.get(0)?, r.get(1)?)),
|
||||
)
|
||||
.optional()
|
||||
.map_err(StoreError::from)?;
|
||||
|
||||
let Some((doc_id, asset_id)) = doc_row else {
|
||||
return Ok(Vec::new());
|
||||
};
|
||||
|
||||
// 1. Collect chunk_ids before CASCADE removes them.
|
||||
let mut stmt = conn
|
||||
.prepare("SELECT chunk_id FROM chunks WHERE doc_id = ?")
|
||||
.map_err(StoreError::from)?;
|
||||
let rows = stmt
|
||||
.query_map(rusqlite::params![doc_id], |r| r.get::<_, String>(0))
|
||||
.map_err(StoreError::from)?;
|
||||
let chunk_ids: Vec<kebab_core::ChunkId> = rows
|
||||
.map(|r| r.map(kebab_core::ChunkId))
|
||||
.collect::<rusqlite::Result<Vec<_>>>()
|
||||
.map_err(StoreError::from)?;
|
||||
drop(stmt);
|
||||
|
||||
// 2. DELETE the document row (CASCADE clears blocks / chunks /
|
||||
// embedding_records via the FK constraints in V001).
|
||||
conn.execute(
|
||||
"DELETE FROM documents WHERE doc_id = ?",
|
||||
rusqlite::params![doc_id],
|
||||
)
|
||||
.map_err(StoreError::from)?;
|
||||
|
||||
// 3. Delete the asset row only when no other document still
|
||||
// references it (twin-file safety: two files with identical
|
||||
// bytes share a single asset row via the blake3 PK).
|
||||
let remaining_refs: i64 = conn
|
||||
.query_row(
|
||||
"SELECT COUNT(*) FROM documents WHERE asset_id = ?",
|
||||
rusqlite::params![asset_id],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.map_err(StoreError::from)?;
|
||||
|
||||
if remaining_refs == 0 {
|
||||
// 4. Capture storage details before deleting the row.
|
||||
let asset_storage: Option<(String, String)> = conn
|
||||
.query_row(
|
||||
"SELECT storage_kind, storage_path FROM assets WHERE asset_id = ?",
|
||||
rusqlite::params![asset_id],
|
||||
|r| Ok((r.get(0)?, r.get(1)?)),
|
||||
)
|
||||
.optional()
|
||||
.map_err(StoreError::from)?;
|
||||
|
||||
conn.execute(
|
||||
"DELETE FROM assets WHERE asset_id = ?",
|
||||
rusqlite::params![asset_id],
|
||||
)
|
||||
.map_err(StoreError::from)?;
|
||||
|
||||
// 5. Best-effort: remove the on-disk copied asset file.
|
||||
if let Some((storage_kind, storage_path)) = asset_storage {
|
||||
if storage_kind == "copied" {
|
||||
let _ = std::fs::remove_file(&storage_path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tracing::debug!(
|
||||
target: "kebab-store-sqlite",
|
||||
workspace_path = %workspace_path.0,
|
||||
doc_id = %doc_id,
|
||||
chunk_count = chunk_ids.len(),
|
||||
"purged deleted-file document from store"
|
||||
);
|
||||
|
||||
Ok(chunk_ids)
|
||||
}
|
||||
|
||||
/// UPSERT a row into `assets`. Used by both the `put_asset_with_bytes`
|
||||
/// path (which has bytes + computed `storage_kind/path`) and the
|
||||
/// `DocumentStore::put_asset` path (which only has the `RawAsset` and
|
||||
/// reads `storage_kind/path` from `asset.stored`).
|
||||
///
|
||||
/// **`assets.workspace_path` is "last-registered path" semantics for
|
||||
/// twin files** (two source files with identical content share one
|
||||
/// `assets` row keyed on `asset_id = blake3(content)`). Each ingest
|
||||
/// of either twin overwrites `workspace_path` with whichever path was
|
||||
/// seen most recently — this is intentional and correct after PR #146
|
||||
/// made `try_skip_unchanged` document-centric (uses
|
||||
/// `get_document_by_workspace_path`, not `get_asset_by_workspace_path`)
|
||||
/// and PR #149 made `reset --orphans-only` document-centric too.
|
||||
/// Do NOT "fix" the flip-flop by adding a UNIQUE constraint on
|
||||
/// `workspace_path` in the `assets` table — twin de-dup is load-bearing.
|
||||
/// When you need media_type for a known document, use the 2-step lookup
|
||||
/// `get_document_by_workspace_path` → `doc.source_asset_id` →
|
||||
/// `get_asset(asset_id)` so the result is twin-safe.
|
||||
pub(crate) fn upsert_asset_row(
|
||||
conn: &Connection,
|
||||
asset: &kebab_core::RawAsset,
|
||||
@@ -701,6 +823,39 @@ impl SqliteStore {
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// p10-1A-2 follow-up (dogfooding 2026-05-20): per-repo doc count for
|
||||
/// `schema.v1`.
|
||||
///
|
||||
/// Reads `metadata_json->'$.repo'`, groups by the value, and skips rows
|
||||
/// where `repo` is NULL (documents without an explicit repo tag).
|
||||
/// Returns `BTreeMap<String, u32>` — key is the repo name as stored in
|
||||
/// frontmatter, value is the doc count.
|
||||
pub fn repo_breakdown(
|
||||
&self,
|
||||
) -> anyhow::Result<std::collections::BTreeMap<String, u32>> {
|
||||
use anyhow::Context;
|
||||
let conn = self.read_conn();
|
||||
let mut stmt = conn
|
||||
.prepare(
|
||||
"SELECT json_extract(metadata_json, '$.repo') AS rp, COUNT(*) \
|
||||
FROM documents \
|
||||
WHERE rp IS NOT NULL \
|
||||
GROUP BY rp",
|
||||
)
|
||||
.context("prepare repo_breakdown")?;
|
||||
let rows = stmt
|
||||
.query_map([], |r| {
|
||||
Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)? as u32))
|
||||
})
|
||||
.context("query repo_breakdown")?;
|
||||
let mut out = std::collections::BTreeMap::new();
|
||||
for row in rows {
|
||||
let (k, v) = row.context("read repo_breakdown row")?;
|
||||
out.insert(k, v);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
}
|
||||
|
||||
/// Apply the design §5 / task-spec pragmas. Called once per connection.
|
||||
@@ -817,5 +972,79 @@ mod tests {
|
||||
// only one key total
|
||||
assert_eq!(bd.len(), 1, "expected exactly 1 entry, got: {bd:?}");
|
||||
}
|
||||
|
||||
/// p10-1A-2 follow-up: `repo_breakdown` counts docs by
|
||||
/// `metadata_json.repo`.
|
||||
///
|
||||
/// Inserts:
|
||||
/// - one doc with `repo = "my-repo"` → must appear with count 1
|
||||
/// - one doc with `repo = null` → must NOT appear (NULL skipped)
|
||||
///
|
||||
/// Uses a side rusqlite connection that bypasses the `assets` FK via
|
||||
/// `PRAGMA foreign_keys = OFF` so the test is self-contained.
|
||||
#[test]
|
||||
fn repo_breakdown_counts_by_repo() {
|
||||
let (dir, store) = open_fresh_store();
|
||||
|
||||
let db_path = dir.path().join("kebab.sqlite");
|
||||
let conn = rusqlite::Connection::open(&db_path).unwrap();
|
||||
conn.pragma_update(None, "foreign_keys", "OFF").unwrap();
|
||||
|
||||
// Doc 1: doc with repo = "my-repo"
|
||||
conn.execute(
|
||||
"INSERT INTO documents (
|
||||
doc_id, asset_id, workspace_path,
|
||||
source_type, trust_level, parser_version,
|
||||
doc_version, schema_version,
|
||||
metadata_json, provenance_json,
|
||||
created_at, updated_at
|
||||
) VALUES (
|
||||
'doc-repo-1', 'asset-r1', 'my-repo/README.md',
|
||||
'markdown', 'primary', 'test-v1',
|
||||
1, 1,
|
||||
'{\"repo\":\"my-repo\"}', '{}',
|
||||
'2024-01-01T00:00:00Z', '2024-01-01T00:00:00Z'
|
||||
)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// Doc 2: doc with repo absent (null in JSON)
|
||||
conn.execute(
|
||||
"INSERT INTO documents (
|
||||
doc_id, asset_id, workspace_path,
|
||||
source_type, trust_level, parser_version,
|
||||
doc_version, schema_version,
|
||||
metadata_json, provenance_json,
|
||||
created_at, updated_at
|
||||
) VALUES (
|
||||
'doc-norepo-1', 'asset-r2', 'standalone/notes.md',
|
||||
'markdown', 'primary', 'test-v1',
|
||||
1, 1,
|
||||
'{\"repo\":null}', '{}',
|
||||
'2024-01-01T00:00:00Z', '2024-01-01T00:00:00Z'
|
||||
)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
drop(conn); // release side connection before querying via store
|
||||
|
||||
let bd = store.repo_breakdown().unwrap();
|
||||
|
||||
// "my-repo" must appear with count 1
|
||||
assert_eq!(
|
||||
bd.get("my-repo"),
|
||||
Some(&1u32),
|
||||
"expected my-repo=1 in repo_breakdown, got: {bd:?}"
|
||||
);
|
||||
// null repo must NOT appear as any key
|
||||
assert!(
|
||||
!bd.contains_key("null"),
|
||||
"null repo must not appear in breakdown, got: {bd:?}"
|
||||
);
|
||||
// only one key total
|
||||
assert_eq!(bd.len(), 1, "expected exactly 1 entry, got: {bd:?}");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -41,6 +41,7 @@ fn fixture_report() -> IngestReport {
|
||||
skipped_generated: 0,
|
||||
skipped_size_exceeded: 0,
|
||||
skip_examples: kebab_core::SkipExamples::default(),
|
||||
purged_deleted_files: 0,
|
||||
items: Some(vec![
|
||||
IngestItem {
|
||||
kind: IngestItemKind::New,
|
||||
|
||||
@@ -22,7 +22,8 @@ Cargo workspace, 함수 호출 기반 모듈러 모놀리스. UI binary (`kebab-
|
||||
| OCR | Ollama vision LM (default `gemma4:e4b`) — `OcrEngine` trait 으로 Tesseract / Apple Vision 등 future swap (HOTFIXES P6-2) |
|
||||
| Image caption | Ollama vision LM, runtime gate `image.caption.enabled` (default OFF) |
|
||||
| PDF parser | `lopdf` per-page 텍스트, `chunker_version = "pdf-page-v1"` 가 PDF 자산에 하드코딩 (HOTFIXES P7-3) |
|
||||
| code parser | `tree-sitter` + `tree-sitter-rust` — **parser-side** (`kebab-parse-code`), chunker-side 아님 (design §6.3). `chunker_version = "code-rust-ast-v1"`. `ast_chunk_max_lines = 200` 상수 고정 (HOTFIXES 2026-05-19 — Chunker trait 이 per-medium config 미노출). |
|
||||
| code parser | `tree-sitter` + `tree-sitter-rust` / `tree-sitter-python` / `tree-sitter-typescript` / `tree-sitter-javascript` / `tree-sitter-go` / `tree-sitter-java` / `tree-sitter-kotlin-ng` — **parser-side** (`kebab-parse-code`), chunker-side 아님 (design §6.3). chunker versions: Rust = `code-rust-ast-v1`, Python = `code-python-ast-v1`, TypeScript = `code-ts-ast-v1`, JavaScript = `code-js-ast-v1`, Go = `code-go-ast-v1`, Java = `code-java-ast-v1`, Kotlin = `code-kotlin-ast-v1`. `ast_chunk_max_lines = 200` 상수 고정 (HOTFIXES 2026-05-19 — Chunker trait 이 per-medium config 미노출). Kotlin grammar 은 `tree-sitter-kotlin-ng` 사용 — bare `tree-sitter-kotlin` 은 tree-sitter 0.21–0.23 에 고착되어 있어 사용 불가. |
|
||||
| 1B symbol path | workspace path → module path: Python = dotted prefix (`kebab_eval.metrics.compute_mrr`), TypeScript/JavaScript = slash-style prefix (`src/Foo.Foo.search`). Rust 1A-2 는 file-scope nesting 만 (workspace prefix 없음, 비일관 수용 — HOTFIXES 2026-05-20). |
|
||||
| TUI | Ratatui + crossterm — P9-1 Library 패널, P9-2/3/4 진행 예정 |
|
||||
| Desktop | Tauri 2 + `pdfjs-dist` (native PDF render backend 금지) — P9-5 |
|
||||
| citation 형식 | URI fragment (`path#L12-L34` / `path#p=12` / `path#xywh=0,0,100,50`, W3C Media Fragments) |
|
||||
@@ -51,7 +52,7 @@ flowchart TB
|
||||
ppdf["kebab-parse-pdf"]
|
||||
pimg["kebab-parse-image"]
|
||||
paud["kebab-parse-audio<br/>(P8 보류)"]
|
||||
pcode["kebab-parse-code<br/>(P10-1A-2)"]
|
||||
pcode["kebab-parse-code<br/>(P10-1A-2 + P10-1B + P10-1C-Go + P10-1C-JK)"]
|
||||
ptypes["kebab-parse-types"]
|
||||
norm["kebab-normalize"]
|
||||
chunk["kebab-chunk"]
|
||||
@@ -126,6 +127,8 @@ flowchart TB
|
||||
|
||||
UI → store/llm/parse 직접 의존 금지. 모든 user-facing 진입은 `kebab-app` facade 만 통한다 (frozen 설계 §8). `kebab-cli` 가 `--config <path>` flag 를 honor 하려면 `kebab_app::*_with_config(cfg, …)` companion 을 통해 Config 을 명시적으로 thread 하는 패턴 — 자세한 이유는 [tasks/HOTFIXES.md](../tasks/HOTFIXES.md) 의 `--config` 항목.
|
||||
|
||||
`kebab-parse-code` 의 외부 tree-sitter grammar crate 의존: P10-1A-2 에서 `tree-sitter-rust` 추가, P10-1B 에서 `tree-sitter-python` / `tree-sitter-typescript` / `tree-sitter-javascript` 추가, P10-1C-Go 에서 `tree-sitter-go` 추가, P10-1C-JK 에서 `tree-sitter-java` / `tree-sitter-kotlin-ng` 추가. 모두 `kebab-parse-code` 에만 격리 (facade 룰 — UI crate / chunker 가 직접 import 금지). Kotlin 은 `tree-sitter-kotlin-ng` 사용 (bare `tree-sitter-kotlin` 은 tree-sitter 0.21–0.23 에 고착 — 사용 불가).
|
||||
|
||||
## 디렉토리 구조
|
||||
|
||||
```text
|
||||
@@ -162,7 +165,7 @@ kebab/
|
||||
│ ├── kebab-source-fs/ # 워크스페이스 walk + checksum (P1-1)
|
||||
│ ├── kebab-parse-md/ # Markdown frontmatter + blocks (P1-2/3)
|
||||
│ ├── kebab-normalize/ # ParsedBlock → CanonicalDocument (P1-4)
|
||||
│ ├── kebab-chunk/ # heading-aware + pdf-page-v1 + code-rust-ast-v1 chunker (P1-5, P7-2, P10-1A-2)
|
||||
│ ├── kebab-chunk/ # heading-aware + pdf-page-v1 + code-rust-ast-v1 + code-python-ast-v1 + code-ts-ast-v1 + code-js-ast-v1 + code-go-ast-v1 + code-java-ast-v1 + code-kotlin-ast-v1 chunker (P1-5, P7-2, P10-1A-2, P10-1B, P10-1C-Go, P10-1C-JK)
|
||||
│ ├── kebab-store-sqlite/ # SQLite + FTS5 (V001/V002/V003) (P1-6, P2-1, P3-3)
|
||||
│ ├── kebab-search/ # Lexical + Vector + Hybrid retriever (P2-2, P3-4)
|
||||
│ ├── kebab-embed/ kebab-embed-local/ # Embedder trait + fastembed adapter (P3-1, P3-2)
|
||||
@@ -172,7 +175,7 @@ kebab/
|
||||
│ ├── kebab-eval/ # golden query runner + metrics (P5-1, P5-2)
|
||||
│ ├── kebab-parse-image/ # ImageExtractor + Ollama OCR + caption (P6)
|
||||
│ ├── kebab-parse-pdf/ # lopdf per-page text extractor (P7-1)
|
||||
│ ├── kebab-parse-code/ # tree-sitter Rust AST extractor (P10-1A-2); chunker lives in kebab-chunk
|
||||
│ ├── kebab-parse-code/ # tree-sitter AST extractors: Rust (P10-1A-2), Python + TypeScript + JavaScript (P10-1B), Go (P10-1C-Go), Java + Kotlin (P10-1C-JK — java.rs + kotlin.rs); chunker lives in kebab-chunk
|
||||
│ ├── kebab-app/ # facade (P0 시그니처 + P3-5/P6-4/P7-3 본체)
|
||||
│ ├── kebab-tui/ # Ratatui shell + Library 패널 (P9-1)
|
||||
│ ├── kebab-mcp/ # stdio MCP server — tools: schema, doctor, search, ask (P9-FB-30)
|
||||
|
||||
@@ -340,6 +340,88 @@ extra_skip_globs = [] # 사용자 추가 skip 패턴
|
||||
- `.rs` 파일은 `SourceType::Note` 로 분류됨 (kebab-core `SourceType::Code` variant 미존재). `--media code` filter 는 정상 동작 — `MediaType::Code("rust")` 로 별도 분류됨. 자세한 내용: `tasks/HOTFIXES.md` (2026-05-19 `SourceType::Code` 항목).
|
||||
- `.gitignore` 가 honor 됨 — `target/` / `node_modules/` 등은 built-in 안전망으로 자동 skip.
|
||||
|
||||
## P10-1B Python / TypeScript / JavaScript 코드 색인
|
||||
|
||||
P10-1A-2 와 동일한 격리 KB 설정으로 Python / TypeScript / JavaScript 3 언어를 검증한다. 설정 블록은 P10-1A-2 와 동일 (`[ingest.code]` 절 포함).
|
||||
|
||||
```bash
|
||||
# 1) 워크스페이스에 Python / TS / JS 파일 추가 (소규모 샘플로 충분)
|
||||
mkdir -p /tmp/kebab-smoke/workspace/sample_code
|
||||
# Python 예시
|
||||
cat > /tmp/kebab-smoke/workspace/sample_code/metrics.py <<'EOF'
|
||||
def compute_mrr(results):
|
||||
"""Mean Reciprocal Rank."""
|
||||
total = 0.0
|
||||
for i, hit in enumerate(results, 1):
|
||||
if hit:
|
||||
total += 1.0 / i
|
||||
break
|
||||
return total
|
||||
EOF
|
||||
# TypeScript 예시
|
||||
cat > /tmp/kebab-smoke/workspace/sample_code/searcher.ts <<'EOF'
|
||||
export class Searcher {
|
||||
search(query: string): string[] {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
EOF
|
||||
# JavaScript 예시
|
||||
cat > /tmp/kebab-smoke/workspace/sample_code/utils.js <<'EOF'
|
||||
function formatResult(hit) {
|
||||
return `${hit.score}: ${hit.path}`;
|
||||
}
|
||||
module.exports = { formatResult };
|
||||
EOF
|
||||
|
||||
# 2) ingest
|
||||
KB ingest
|
||||
|
||||
# 3) 언어별 검색 (symbol + module path prefix 확인)
|
||||
KB search --mode hybrid "compute_mrr" --code-lang python --json | \
|
||||
jq '{hits: [.hits[] | {symbol: .citation.symbol, lang: .citation.lang}]}'
|
||||
KB search --mode hybrid "search" --code-lang typescript --json | \
|
||||
jq '{hits: [.hits[] | {symbol: .citation.symbol, lang: .citation.lang}]}'
|
||||
KB search --mode hybrid "formatResult" --code-lang javascript --json | \
|
||||
jq '{hits: [.hits[] | {symbol: .citation.symbol, lang: .citation.lang}]}'
|
||||
|
||||
# 4) schema stats 에 3 언어 카운트 확인
|
||||
KB --json schema | jq '.stats.code_lang_breakdown'
|
||||
# 기대: {"python": N, "typescript": N, "javascript": N, "rust": M, ...}
|
||||
```
|
||||
|
||||
**Symbol path 컨벤션 (2026-05-20 기준)**:
|
||||
|
||||
- **Python**: workspace 경로 → dotted module path prefix. `sample_code/metrics.py` 의 `compute_mrr` → symbol `sample_code.metrics.compute_mrr`.
|
||||
- **TypeScript / JavaScript**: workspace 경로 → slash-style module path prefix. `sample_code/searcher.ts` 의 `search` → `sample_code/searcher.Searcher.search`. `.tsx` / `.mjs` / `.cjs` / `.jsx` 도 동일 처리.
|
||||
- **Rust** (1A-2): file-scope nesting 만, workspace path prefix 없음 (예: `Foo::double`). Python/TS/JS 와 비일관 — HOTFIXES 2026-05-20 참조.
|
||||
|
||||
**알려진 동작**:
|
||||
|
||||
- `const foo = () => {...}` 같은 expression-level 함수는 `<top-level>` glue 로 잡힘 (declaration-level 단위만 1B 1차 범위). 자세한 내용: `tasks/HOTFIXES.md` (2026-05-20).
|
||||
- `.gitignore` honor — `node_modules/` / `__pycache__/` / `.venv/` 등 built-in 안전망 자동 skip.
|
||||
|
||||
## P10-1C-Go Go 코드 색인
|
||||
|
||||
P10-1B 와 동일한 격리 KB 설정. `.go` 파일을 워크스페이스에 두고 ingest 하면 `code-go-ast-v1` chunker 가 package 단위 AST 로 처리한다.
|
||||
|
||||
```bash
|
||||
cat > /tmp/kebab-smoke/workspace/sample_code/hello.go <<'EOF'
|
||||
package main
|
||||
|
||||
import "fmt"
|
||||
|
||||
func Hello(name string) string {
|
||||
return fmt.Sprintf("Hello, %s!", name)
|
||||
}
|
||||
EOF
|
||||
|
||||
KB ingest
|
||||
KB search --mode hybrid "Hello" --code-lang go --json | \
|
||||
jq '{hits: [.hits[] | {symbol: .citation.symbol, lang: .citation.lang}]}'
|
||||
# 기대: symbol = "main.Hello", lang = "go"
|
||||
```
|
||||
|
||||
## 검증 체크리스트
|
||||
|
||||
- `kebab doctor` 가 `--config` path 를 honor 하고 그 안의 `storage.data_dir` 를 출력 (XDG default 가 아님).
|
||||
@@ -371,6 +453,9 @@ rm -rf /tmp/kebab-smoke # 통째로 정리
|
||||
- (P7-3) `config.chunking.chunker_version` 는 markdown 만 represent — PDF 자산은 `pdf-page-v1` 하드코딩. `config.toml` 의 `chunker_version = "md-heading-v1"` 을 봐도 PDF 는 영향 안 받음. HOTFIXES `2026-05-02 P7-3` entry 참조 (P+ chunker registry task 까지 유지).
|
||||
- (P7-3) 한 PDF 가 N 페이지면 `kebab ingest` 가 N 개 (또는 그 이상의, 페이지 길면 multi-chunk) 의 chunk 를 한 transaction 안에서 commit. 500 페이지 책 → 500+ chunk 한 번에 → embedding throughput 가 bottleneck. 임베딩 활성 워크스페이스에서 큰 PDF 를 처음 ingest 하면 분-단위 시간 + WAL 크기 증가 가능 — P+ 스케일 hardening task 까지 정상 동작이지만 비용은 측정 가능.
|
||||
- (P10-1A-2) `.rs` 파일을 워크스페이스에 두면 `kebab ingest` 결과에 `new` 카운터에 포함. `kebab search --mode hybrid "<함수명>" --code-lang rust --json` 가 `citation.kind = "code"`, `citation.lang = "rust"` (SearchHit top-level `code_lang` 도 동일), `citation.symbol` (함수/타입 이름), `citation.line_start` / `citation.line_end` 를 반환하면 wiring 정상. `kebab schema --json | jq .stats.code_lang_breakdown` 에 `"rust": N` 이 나오면 chunk 가 색인됨.
|
||||
- (P10-1B) `.py` / `.ts` / `.tsx` / `.js` / `.mjs` / `.cjs` / `.jsx` 파일을 워크스페이스에 두면 `kebab ingest` 결과에 `new` 카운터에 포함. `--code-lang python` / `--code-lang typescript` / `--code-lang javascript` 검색이 `citation.symbol` 에 module path prefix 를 포함한 결과를 반환하면 wiring 정상. `kebab schema --json | jq .stats.code_lang_breakdown` 에 해당 언어 카운트 등장 확인.
|
||||
- (P10-1C-Go) `.go` 파일을 워크스페이스에 두면 `kebab ingest` 가 `code-go-ast-v1` 로 처리. `--code-lang go` 검색이 `citation.symbol` 에 `<package>.<Func>` / `<package>.(*Receiver).<Method>` 형식 결과를 반환하면 wiring 정상. `kebab schema --json | jq .stats.code_lang_breakdown` 에 `"go": N` 등장 확인.
|
||||
- (P10-1C-JK) `.java` 파일은 `code-java-ast-v1`, `.kt`/`.kts` 파일은 `code-kotlin-ast-v1` 로 처리. `--code-lang java` / `--code-lang kotlin` 검색이 `citation.symbol` 에 `com.foo.Foo.bar` 형식 결과를 반환하면 wiring 정상. `kebab schema --json | jq .stats.code_lang_breakdown` 에 `"java": N` / `"kotlin": N` 등장 확인.
|
||||
- (P7-3 + follow-up) 동일 path 에 byte 가 다른 PDF 를 두 번째 ingest 하면 `purge_vector_orphans_for_workspace_path` 가 옛 chunk_id 를 LanceDB 에서 먼저 삭제, 이어서 `purge_orphan_at_workspace_path` 가 옛 doc / chunks / embedding_records 를 SQLite 에서 sweep. 새 byte 가 새 `doc_id` 로 색인됨. `IngestReport` 에 그 자산만 `new+=1` (다른 자산은 `updated`). 두 store 모두 정합 — 옛 본문 검색 시 옛 chunks 가 더 이상 surface 되지 않음.
|
||||
|
||||
### Embedding upgrade (fb-39b)
|
||||
|
||||
@@ -0,0 +1,741 @@
|
||||
# p10-1B Python + TS/JS AST Chunkers Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development to implement this plan task-by-task.
|
||||
|
||||
**Goal:** Activate Python / TypeScript / JavaScript code ingest end-to-end on top of 1A-2's infrastructure — 3 new tree-sitter grammars, 3 new Extractors, 3 new chunkers (`code-{python,ts,js}-ast-v1`), a `module_path_for_*` helper for workspace-path → module-path conversion, and a small app-dispatch generalization. Wire `code_lang` filter / breakdown / Citation::Code surface activate automatically.
|
||||
|
||||
**Architecture:** Mirror 1A-2 exactly per language. Each Extractor in `kebab-parse-code/src/{python,typescript,javascript}.rs` calls its tree-sitter grammar and emits one `Block::Code` per top-level AST semantic unit with `SourceSpan::Code { line_start, line_end, symbol, lang }`. Symbol = `module_path` (from workspace_path) `+` per-language join (`.` for Python, `/.../basename.symbol` for TS/JS). Each chunker is a near-duplicate of `code-rust-ast-v1` (1:1 + oversize split). App dispatch becomes `match lang { "rust" | "python" | "typescript" | "javascript" }`.
|
||||
|
||||
**Tech Stack:** Rust 2024 workspace, `tree-sitter` 0.26, `tree-sitter-python` / `tree-sitter-typescript` / `tree-sitter-javascript`, existing 1A-2 infrastructure (citation_helper Code arm, backfill, schema breakdown).
|
||||
|
||||
**Memory note:** Host was OOM-killed earlier in this branch's history. Prefer `cargo test -p <crate>` and `cargo check -p <crate>`; the only `cargo test --workspace -j 1` call is the Task L full-suite gate. Never run cargo invocations in parallel.
|
||||
|
||||
---
|
||||
|
||||
## Pre-flight
|
||||
|
||||
Branch `feat/p10-1b-py-ts-js` already exists on main (`git checkout feat/p10-1b-py-ts-js`).
|
||||
|
||||
- [ ] **Disk hygiene**: `cargo clean`.
|
||||
|
||||
Reference files (read before touching the corresponding 1B file):
|
||||
- 1A-2 Rust extractor: `crates/kebab-parse-code/src/rust.rs` — the scaffold every per-lang extractor mirrors.
|
||||
- 1A-2 Rust chunker: `crates/kebab-chunk/src/code_rust_ast_v1.rs` — the scaffold every per-lang chunker mirrors.
|
||||
- 1A-2 app dispatch: `crates/kebab-app/src/lib.rs` `ingest_one_code_asset` (~line 1645).
|
||||
- 1A-2 source-fs routing: `crates/kebab-source-fs/src/media.rs:39` (the `"rs" =>` arm).
|
||||
- 1A-2 lang dispatch: `crates/kebab-parse-code/src/lang.rs::code_lang_for_path`.
|
||||
|
||||
---
|
||||
|
||||
## Task A: Workspace deps
|
||||
|
||||
**Files:**
|
||||
- Modify: `Cargo.toml` (workspace `[workspace.dependencies]`, after the existing `tree-sitter-rust` entry)
|
||||
- Modify: `crates/kebab-parse-code/Cargo.toml` (`[dependencies]`)
|
||||
|
||||
- [ ] **Step 1**: Resolve versions: `cargo add tree-sitter-python tree-sitter-typescript tree-sitter-javascript -p kebab-parse-code`.
|
||||
|
||||
- [ ] **Step 2**: Lift the three resolved versions into `[workspace.dependencies]` in the root `Cargo.toml`, immediately after the `tree-sitter-rust` line. Single-line comment first:
|
||||
|
||||
```toml
|
||||
# Python / TS / JS grammars for code ingest (kebab-parse-code, p10-1B).
|
||||
tree-sitter-python = "<resolved>"
|
||||
tree-sitter-typescript = "<resolved>"
|
||||
tree-sitter-javascript = "<resolved>"
|
||||
```
|
||||
|
||||
Then change the crate's `[dependencies]` entries to `{ workspace = true }` matching the existing `tree-sitter` / `tree-sitter-rust` style.
|
||||
|
||||
- [ ] **Step 3**: `cargo build -p kebab-parse-code` → clean (unused deps OK; warnings appear when actually imported in later tasks).
|
||||
|
||||
- [ ] **Step 4**: Commit.
|
||||
|
||||
```bash
|
||||
git add Cargo.toml Cargo.lock crates/kebab-parse-code/Cargo.toml
|
||||
git commit -m "build(p10-1b): add tree-sitter-python/-typescript/-javascript workspace deps
|
||||
|
||||
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task B: source-fs media routing for `.py`/`.pyi`/`.ts`/`.tsx`/`.js`/`.mjs`/`.cjs`/`.jsx`
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/kebab-source-fs/src/media.rs` (add 3 arms next to the existing `"rs"` arm at L39)
|
||||
- Test: same file's test module
|
||||
|
||||
- [ ] **Step 1 (failing test)**:
|
||||
|
||||
```rust
|
||||
#[test]
|
||||
fn py_ts_js_files_map_to_media_code() {
|
||||
assert_eq!(media_type_for(Path::new("a/b.py")), MediaType::Code("python".into()));
|
||||
assert_eq!(media_type_for(Path::new("a/b.pyi")), MediaType::Code("python".into()));
|
||||
assert_eq!(media_type_for(Path::new("a/b.ts")), MediaType::Code("typescript".into()));
|
||||
assert_eq!(media_type_for(Path::new("a/b.tsx")), MediaType::Code("typescript".into()));
|
||||
assert_eq!(media_type_for(Path::new("a/b.js")), MediaType::Code("javascript".into()));
|
||||
assert_eq!(media_type_for(Path::new("a/b.mjs")), MediaType::Code("javascript".into()));
|
||||
assert_eq!(media_type_for(Path::new("a/b.cjs")), MediaType::Code("javascript".into()));
|
||||
assert_eq!(media_type_for(Path::new("a/b.jsx")), MediaType::Code("javascript".into()));
|
||||
// Rust 1A-2 arm still works
|
||||
assert_eq!(media_type_for(Path::new("a/b.rs")), MediaType::Code("rust".into()));
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2**: Run → FAIL.
|
||||
|
||||
- [ ] **Step 3**: Add the three arms before the `_ => MediaType::Other(ext)` fallback. Match existing style and order extensions logically (most common first within each language):
|
||||
|
||||
```rust
|
||||
// p10-1B: Python / TS / JS AST chunkers active.
|
||||
"py" | "pyi" => MediaType::Code("python".into()),
|
||||
"ts" | "tsx" => MediaType::Code("typescript".into()),
|
||||
"js" | "mjs" | "cjs" | "jsx" => MediaType::Code("javascript".into()),
|
||||
```
|
||||
|
||||
- [ ] **Step 4**: Run → PASS. Then `cargo test -p kebab-source-fs` → no regression.
|
||||
|
||||
- [ ] **Step 5**: `cargo clippy -p kebab-source-fs --all-targets -- -D warnings` clean. Commit.
|
||||
|
||||
```bash
|
||||
git add crates/kebab-source-fs/
|
||||
git commit -m "feat(p10-1b): route .py/.pyi/.ts/.tsx/.js/.mjs/.cjs/.jsx to MediaType::Code
|
||||
|
||||
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task C: `module_path_for_python` + `module_path_for_tsjs` helpers
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/kebab-parse-code/src/lang.rs` (add 2 pub fns + tests)
|
||||
- Modify: `crates/kebab-parse-code/src/lib.rs` (re-export the 2 fns)
|
||||
|
||||
These convert a `WorkspacePath` into a module-path prefix for symbol formatting. Single source of truth — used by all per-language extractors.
|
||||
|
||||
### Rules
|
||||
|
||||
**`module_path_for_python(workspace_path: &str) -> String`**:
|
||||
1. Strip a leading well-known "source root" prefix from a small allowlist if present (in order): `crates/<name>/src/`, `src/`, `lib/`. (Use a single small `for` loop over the allowlist; stop at first prefix match.) Rationale: avoid noisy `crates.x.src.foo.bar` symbols when the user has a conventional layout, while leaving non-conventional paths untouched.
|
||||
2. Strip trailing `.py` or `.pyi` extension. If the basename (after extension strip) is `__init__`, drop it (and the preceding `/`) so `pkg/__init__.py` → `pkg`.
|
||||
3. Replace `/` with `.`.
|
||||
4. Result is the dotted module prefix. Symbols are joined with `.` (e.g. `module_path + "." + sym`). Empty result (file is at workspace root without prefix) → use empty string → symbol is the unit name alone.
|
||||
|
||||
**`module_path_for_tsjs(workspace_path: &str) -> String`**:
|
||||
1. Strip extension if it's one of `.ts` / `.tsx` / `.js` / `.jsx` / `.mjs` / `.cjs`.
|
||||
2. Do NOT replace `/` (TS/JS convention is path-like). Do NOT strip any source root (TS/JS layouts vary too widely).
|
||||
3. Result is the path-style prefix (e.g. `src/search/retriever/Retriever`). Symbols join with `.` (`prefix + "." + sym`, e.g. `src/search/retriever/Retriever.search`).
|
||||
|
||||
- [ ] **Step 1 (failing tests)** — add to existing `mod tests` (or create one) in `lang.rs`:
|
||||
|
||||
```rust
|
||||
#[test]
|
||||
fn module_path_for_python_strips_src_roots_and_extensions() {
|
||||
assert_eq!(module_path_for_python("kebab_eval/metrics.py"), "kebab_eval.metrics");
|
||||
assert_eq!(module_path_for_python("kebab_eval/__init__.py"), "kebab_eval");
|
||||
assert_eq!(module_path_for_python("src/foo/bar.py"), "foo.bar");
|
||||
assert_eq!(module_path_for_python("crates/x/src/foo/bar.py"), "foo.bar");
|
||||
assert_eq!(module_path_for_python("a/b/c.pyi"), "a.b.c");
|
||||
assert_eq!(module_path_for_python("standalone.py"), "standalone");
|
||||
assert_eq!(module_path_for_python("src/__init__.py"), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn module_path_for_tsjs_keeps_slashes_and_strips_ext() {
|
||||
for ext in ["ts", "tsx", "js", "jsx", "mjs", "cjs"] {
|
||||
let p = format!("src/search/retriever/Retriever.{ext}");
|
||||
assert_eq!(module_path_for_tsjs(&p), "src/search/retriever/Retriever");
|
||||
}
|
||||
assert_eq!(module_path_for_tsjs("foo.ts"), "foo");
|
||||
assert_eq!(module_path_for_tsjs("a/b/c.ts"), "a/b/c");
|
||||
// No `src/` strip — TS layouts vary.
|
||||
assert_eq!(module_path_for_tsjs("packages/x/src/Foo.ts"), "packages/x/src/Foo");
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2**: Run → FAIL (helpers not defined).
|
||||
|
||||
- [ ] **Step 3**: Implement both in `lang.rs`. Suggested implementation (refine if a test points out a missed edge case):
|
||||
|
||||
```rust
|
||||
/// p10-1B: workspace-relative Python file path → dotted module-path prefix.
|
||||
/// See plan §Task C for the exact rules.
|
||||
pub fn module_path_for_python(workspace_path: &str) -> String {
|
||||
let mut p: &str = workspace_path;
|
||||
// Strip a known source-root prefix. Allowlist + `starts_with` over a
|
||||
// pattern with a glob in the middle would be a pain; treat
|
||||
// `crates/*/src/` by string-walking.
|
||||
if let Some(rest) = p.strip_prefix("crates/") {
|
||||
if let Some(slash) = rest.find('/') {
|
||||
let after = &rest[slash + 1..];
|
||||
if let Some(stripped) = after.strip_prefix("src/") {
|
||||
p = stripped;
|
||||
}
|
||||
}
|
||||
} else if let Some(stripped) = p.strip_prefix("src/") {
|
||||
p = stripped;
|
||||
} else if let Some(stripped) = p.strip_prefix("lib/") {
|
||||
p = stripped;
|
||||
}
|
||||
// Strip extension.
|
||||
let p = p
|
||||
.strip_suffix(".py")
|
||||
.or_else(|| p.strip_suffix(".pyi"))
|
||||
.unwrap_or(p);
|
||||
// __init__ → drop it (and the preceding `/`).
|
||||
let p = if let Some(parent) = p.strip_suffix("/__init__") {
|
||||
parent
|
||||
} else if p == "__init__" {
|
||||
""
|
||||
} else {
|
||||
p
|
||||
};
|
||||
p.replace('/', ".")
|
||||
}
|
||||
|
||||
/// p10-1B: workspace-relative TS/JS file path → path-style prefix
|
||||
/// (no slash replacement). See plan §Task C.
|
||||
pub fn module_path_for_tsjs(workspace_path: &str) -> String {
|
||||
let p = workspace_path;
|
||||
for ext in [".tsx", ".ts", ".jsx", ".mjs", ".cjs", ".js"] {
|
||||
if let Some(stripped) = p.strip_suffix(ext) {
|
||||
return stripped.to_string();
|
||||
}
|
||||
}
|
||||
p.to_string()
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4**: Re-export both from `lib.rs` (next to the existing `pub use lang::code_lang_for_path`):
|
||||
|
||||
```rust
|
||||
pub use lang::{code_lang_for_path, module_path_for_python, module_path_for_tsjs};
|
||||
```
|
||||
|
||||
- [ ] **Step 5**: Run → PASS. clippy clean.
|
||||
|
||||
- [ ] **Step 6**: Commit.
|
||||
|
||||
```bash
|
||||
git add crates/kebab-parse-code/
|
||||
git commit -m "feat(p10-1b): module_path_for_python / _tsjs helpers (workspace path → module prefix)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task D: App dispatch generalization
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/kebab-app/src/lib.rs`
|
||||
|
||||
Today's `ingest_one_code_asset` (~L1645) hardcodes `RustAstExtractor` + `CodeRustAstV1Chunker`. 1B needs to dispatch by `lang`. Cleanest minimal change: keep the same function signature but take `code_lang: &str` and `match` it internally onto an `Extractor` + `Chunker` pair. Rust path keeps the same observable behavior.
|
||||
|
||||
Two equivalent dispatch shapes — pick the one with the smallest diff:
|
||||
|
||||
**Shape 1 (recommended — fewest lines changed):** factor extractor invocation + chunker invocation into a small `match code_lang` *inside* `ingest_one_code_asset`. The `parser_version` constant lookup also branches. Everything else (read bytes, ExtractContext, put_*, embed, IngestItem) stays a single non-branched flow.
|
||||
|
||||
**Shape 2:** introduce a tiny enum `CodeLangKind { Rust, Python, Typescript, Javascript }` + an `impl CodeLangKind { fn extract(...) -> CanonicalDocument; fn chunk(...) -> Vec<Chunk>; fn parser_version() -> ParserVersion; fn chunker_version() -> ChunkerVersion; }`. More structure, but better insulates the function body.
|
||||
|
||||
Use Shape 1 for this task (less risk). A future C/D phase can refactor to Shape 2 if the dispatch grows.
|
||||
|
||||
- [ ] **Step 1 (failing test)** — add a Python smoke as the failing test (TS/JS land later in this PR; one failing-then-passing TDD cycle is enough to lock the dispatch contract):
|
||||
|
||||
In `crates/kebab-app/tests/code_ingest_smoke.rs` add:
|
||||
|
||||
```rust
|
||||
#[test]
|
||||
fn python_file_ingests_and_searches_as_code_citation() {
|
||||
// Mirror rust_file_ingests_and_searches_as_code_citation exactly,
|
||||
// but write `kebab_eval/metrics.py` (in the temp workspace root) with:
|
||||
// def compute_mrr(): return 1.0
|
||||
// and assert h.code_lang == Some("python"), citation.lang == Some("python"),
|
||||
// citation.symbol == Some("kebab_eval.metrics.compute_mrr"), parser_version "code-python-v1",
|
||||
// chunker_version "code-python-ast-v1".
|
||||
// ...
|
||||
}
|
||||
```
|
||||
|
||||
(Spec shape ONLY — the actual extractor + chunker land in Tasks F + G. This test compiles but FAILS at runtime until those land. Mark it `#[ignore]` if it would otherwise break TDD ordering — un-`#[ignore]` it in Task G's commit. Alternative: skip this step here and rely on the per-extractor unit tests in Task F + Task G; that is the cleaner TDD ordering. Choose either; document the choice in the commit message.)
|
||||
|
||||
- [ ] **Step 2**: Update `ingest_one_asset` dispatch match arm to accept all four code languages with a `lang` capture passed through:
|
||||
|
||||
```rust
|
||||
// p10-1A-2 / 1B: code ingest dispatch.
|
||||
MediaType::Code(lang)
|
||||
if matches!(lang.as_str(), "rust" | "python" | "typescript" | "javascript") =>
|
||||
{
|
||||
return ingest_one_code_asset(
|
||||
app, asset, chunk_policy, embedder, vector_store,
|
||||
existing_doc_ids, force_reingest, lang.as_str(),
|
||||
);
|
||||
}
|
||||
```
|
||||
|
||||
(Keep the trailing `MediaType::Code(_) | MediaType::Audio(_) | MediaType::Other(_)` or-pattern as the Skipped fallback — non-allowlisted code langs route there.)
|
||||
|
||||
- [ ] **Step 3**: Update `ingest_one_code_asset` signature to take `code_lang: &str` and dispatch internally. Keep all I/O / persistence / embed code unchanged. Per the Shape-1 recipe:
|
||||
- `let parser_version = match code_lang { "rust" => ParserVersion(kebab_parse_code::RUST_PARSER_VERSION.into()), "python" => ParserVersion(kebab_parse_code::PYTHON_PARSER_VERSION.into()), "typescript" => ParserVersion(kebab_parse_code::TS_PARSER_VERSION.into()), "javascript" => ParserVersion(kebab_parse_code::JS_PARSER_VERSION.into()), _ => unreachable!(), };`
|
||||
- The `try_skip_unchanged` call's chunker_version arg branches the same way (different chunker per lang).
|
||||
- The extract call branches: `match code_lang { "rust" => RustAstExtractor::new().extract(...), "python" => PythonAstExtractor::new().extract(...), ... }`.
|
||||
- The chunk call branches: `match code_lang { "rust" => CodeRustAstV1Chunker.chunk(...), "python" => CodePythonAstV1Chunker.chunk(...), ... }`.
|
||||
- All other lines (purge_vector_orphans / put_asset_with_bytes / put_document / put_blocks / put_chunks / embed branch / IngestItem) unchanged.
|
||||
|
||||
At this point Python/TS/JS extractors + chunkers don't exist yet → compile FAILS on the references. Acceptable — Task E/F/G/H/I add them. To stage compile-cleanly: gate the Python/TS/JS arms behind `unimplemented!()` for now (returns an error path) and let Tasks F/G/H/I/J/K replace them. Recommended: leave the dispatch fully written but use `anyhow::bail!("not yet activated in this commit")` for the three non-Rust arms, with a `TODO(p10-1b Task X)` comment per arm. They flip to real calls when each language's extractor + chunker land.
|
||||
|
||||
- [ ] **Step 4**: `cargo test -p kebab-app --lib` (lib-only is enough — integration tests for the non-Rust paths land later). Existing Rust path tests must stay green.
|
||||
|
||||
- [ ] **Step 5**: clippy clean, commit.
|
||||
|
||||
```bash
|
||||
git add crates/kebab-app/
|
||||
git commit -m "refactor(p10-1b): generalize ingest_one_code_asset for multi-language dispatch
|
||||
|
||||
Rust path unchanged (verified by existing code_ingest_smoke tests). Python/TS/JS arms
|
||||
bail with TODO; per-lang extractor + chunker land in subsequent tasks.
|
||||
|
||||
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task E: Python Extractor (`kebab-parse-code/src/python.rs`)
|
||||
|
||||
**Files:**
|
||||
- Create: `crates/kebab-parse-code/src/python.rs`
|
||||
- Modify: `crates/kebab-parse-code/src/lib.rs` (`pub mod python` + re-exports `PYTHON_PARSER_VERSION` and `PythonAstExtractor`)
|
||||
- Create: `crates/kebab-parse-code/tests/fixtures/sample.py`
|
||||
|
||||
Scaffold MIRRORS `crates/kebab-parse-code/src/rust.rs` line-for-line (read it first). Only the AST walk + the symbol prefix differ.
|
||||
|
||||
### Python AST mapping
|
||||
|
||||
tree-sitter-python language: `tree_sitter_python::LANGUAGE` (LanguageFn). Set via `parser.set_language(&tree_sitter_python::LANGUAGE.into())`.
|
||||
|
||||
Walk `module` (root) named children. Maintain `mod_path: Vec<String>` — but for Python we DO NOT push class names onto `mod_path` (class members get `Class.method` form via the class arm directly; nested classes recurse with the class name appended).
|
||||
|
||||
| node kind | unit | symbol (joined with `.`) |
|
||||
|-----------|------|--------------------------|
|
||||
| `function_definition` (name field) | 1 | `<module_prefix>.<fn_name>` (or `<fn_name>` if module_prefix empty) |
|
||||
| `class_definition` (name) — emit ONE unit for the class definition itself (symbol `<module_prefix>.<ClassName>`), then recurse into its `block` body: each inner `function_definition` → unit with symbol `<module_prefix>.<ClassName>.<method_name>`; nested `class_definition` recurses with parent class prepended. | 1 per class + 1 per method (etc.) | as above |
|
||||
| `decorated_definition` | unwrap — process its inner `definition` (either function_definition or class_definition) as if at the same level. `unit_start`'s backward extension over `decorator` siblings folds them into the unit. | n/a | n/a |
|
||||
| `import_statement`, `import_from_statement`, `expression_statement`, `assignment`, `global_statement`, `future_import_statement` at module level | glue | `<top-level>` (with `module_prefix` prefix if non-empty: `<module_prefix>.<top-level>`) |
|
||||
|
||||
`unit_start` (backward extension) covers `comment` siblings + `decorator` siblings (decorators in tree-sitter-python appear as children of `decorated_definition`, NOT as siblings — so the `unwrap decorated_definition` arm above is what brings them in; `comment` siblings still need backward extension). Adapt `unit_start` for the Python flavor: extend over `comment` siblings only (decorators are already covered by unwrapping `decorated_definition`).
|
||||
|
||||
Module-prefix application: at extract time, compute `let mod_prefix = kebab_parse_code::module_path_for_python(&asset.workspace_path.0);`. The walk builds symbols using `mod_prefix` (joined with `.` if non-empty; the bare name if empty). Glue group: if `mod_prefix` non-empty, symbol = `format!("{mod_prefix}.<top-level>")`; else `<top-level>`. `<module>` glue label (file contains only `import`s and no real unit) follows the same prefix rule.
|
||||
|
||||
### Scaffold differences from rust.rs
|
||||
|
||||
- `pub const PARSER_VERSION: &str = "code-python-v1";`
|
||||
- `pub struct PythonAstExtractor;` + `new()`/`Default`.
|
||||
- `fn supports(&self, m: &MediaType) -> bool { matches!(m, MediaType::Code(l) if l == "python") }`
|
||||
- Agent string `"kb-parse-code"` (unchanged).
|
||||
- `metadata.code_lang = Some("python".to_string())`.
|
||||
- `repo` / `git_branch` / `git_commit` from `crate::repo::detect_repo` (same as Rust).
|
||||
- The AST walk is its own `build_blocks` function — DO NOT generalize across languages in this task (each grammar's node names differ enough that polymorphism hurts more than helps; a future refactor task can extract common helpers if patterns converge).
|
||||
|
||||
### Step list (TDD)
|
||||
|
||||
- [ ] **Step 1**: Create `tests/fixtures/sample.py`:
|
||||
|
||||
```python
|
||||
"""sample fixture."""
|
||||
import os
|
||||
|
||||
ANSWER = 42
|
||||
|
||||
@staticmethod
|
||||
def free(x):
|
||||
"""free fn."""
|
||||
return x + 1
|
||||
|
||||
class Foo:
|
||||
"""doc."""
|
||||
def double(self, n):
|
||||
return n * 2
|
||||
|
||||
@classmethod
|
||||
def name(cls):
|
||||
return "foo"
|
||||
|
||||
class Outer:
|
||||
class Inner:
|
||||
def helper(self):
|
||||
return True
|
||||
|
||||
def with_decorator():
|
||||
pass
|
||||
```
|
||||
|
||||
- [ ] **Step 2 (failing test)** in `python.rs`:
|
||||
|
||||
```rust
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use kebab_core::{Block, MediaType, SourceSpan};
|
||||
fn extract_fixture() -> kebab_core::CanonicalDocument {
|
||||
let bytes = std::fs::read(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/sample.py")).unwrap();
|
||||
// Reuse the test-support helper added in Task 6 of 1A-2 (rust.rs tests):
|
||||
// adjust `fixed_rust_asset` to a generic `fixed_code_asset(workspace_path, code_lang)`
|
||||
// OR inline a per-test asset constructor that matches its kebab-core types.
|
||||
let asset = crate::rust::tests_support::fixed_code_asset(
|
||||
"kebab_eval/metrics.py", "python");
|
||||
let cfg = kebab_core::ExtractConfig::default();
|
||||
let root = std::path::PathBuf::from("/tmp");
|
||||
let ctx = kebab_core::ExtractContext { asset: &asset, workspace_root: &root, config: &cfg };
|
||||
PythonAstExtractor::new().extract(&ctx, &bytes).unwrap()
|
||||
}
|
||||
#[test]
|
||||
fn extractor_supports_only_media_code_python() {
|
||||
let e = PythonAstExtractor::new();
|
||||
assert!(e.supports(&MediaType::Code("python".into())));
|
||||
assert!(!e.supports(&MediaType::Code("rust".into())));
|
||||
assert!(!e.supports(&MediaType::Markdown));
|
||||
}
|
||||
#[test]
|
||||
fn python_units_carry_module_prefixed_symbols() {
|
||||
let doc = extract_fixture();
|
||||
let mut syms: Vec<String> = doc.blocks.iter().map(|b| match b {
|
||||
Block::Code(c) => match &c.common.source_span {
|
||||
SourceSpan::Code { symbol, lang, .. } => {
|
||||
assert_eq!(lang.as_deref(), Some("python"));
|
||||
symbol.clone().unwrap()
|
||||
}
|
||||
_ => panic!("expected SourceSpan::Code"),
|
||||
},
|
||||
other => panic!("expected Block::Code, got {other:?}"),
|
||||
}).collect();
|
||||
syms.sort();
|
||||
// workspace_path `kebab_eval/metrics.py` → mod_prefix `kebab_eval.metrics`
|
||||
assert!(syms.iter().any(|s| s == "kebab_eval.metrics.free"));
|
||||
assert!(syms.iter().any(|s| s == "kebab_eval.metrics.Foo"));
|
||||
assert!(syms.iter().any(|s| s == "kebab_eval.metrics.Foo.double"));
|
||||
assert!(syms.iter().any(|s| s == "kebab_eval.metrics.Foo.name"));
|
||||
assert!(syms.iter().any(|s| s == "kebab_eval.metrics.Outer"));
|
||||
assert!(syms.iter().any(|s| s == "kebab_eval.metrics.Outer.Inner"));
|
||||
assert!(syms.iter().any(|s| s == "kebab_eval.metrics.Outer.Inner.helper"));
|
||||
assert!(syms.iter().any(|s| s == "kebab_eval.metrics.with_decorator"));
|
||||
assert!(syms.iter().any(|s| s == "kebab_eval.metrics.<top-level>")); // import + assignment
|
||||
}
|
||||
#[test]
|
||||
fn deterministic_across_runs() {
|
||||
let a = extract_fixture();
|
||||
for _ in 0..50 { assert_eq!(extract_fixture().blocks, a.blocks); }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
(`tests_support::fixed_code_asset` — promote 1A-2's `fixed_rust_asset` to a generic helper that takes the lang string and sets `media_type: MediaType::Code(lang.to_string())`. Move it to a new `pub(crate) mod tests_support` in `rust.rs` so it's reachable from `python.rs::tests`, OR duplicate it inline — pick the smaller diff. Keep the helper `#[cfg(test)]`.)
|
||||
|
||||
- [ ] **Step 3**: Run → FAIL (`PythonAstExtractor` undefined).
|
||||
|
||||
- [ ] **Step 4**: Implement `python.rs`. Scaffold mirrors `rust.rs`; the AST walk follows the table above. The `mod_path: Vec<String>` for Python tracks **class nesting** (so methods get `Class.method`, nested classes get `Outer.Inner`). `Vec` empty at function-level. Glue grouping mirrors Rust's. Apply `mod_prefix` from `module_path_for_python(&asset.workspace_path.0)` to all unit symbols: `if mod_prefix.is_empty() { sym } else { format!("{mod_prefix}.{sym}") }`. The `<top-level>` / `<module>` label inherits the same prefixing.
|
||||
|
||||
- [ ] **Step 5**: Wire into `lib.rs`:
|
||||
|
||||
```rust
|
||||
pub mod python;
|
||||
pub use python::{PARSER_VERSION as PYTHON_PARSER_VERSION, PythonAstExtractor};
|
||||
```
|
||||
|
||||
- [ ] **Step 6**: `cargo test -p kebab-parse-code python` → all pass.
|
||||
|
||||
- [ ] **Step 7**: clippy clean, commit.
|
||||
|
||||
```bash
|
||||
git add crates/kebab-parse-code/
|
||||
git commit -m "feat(p10-1b): tree-sitter-python AST extractor (PythonAstExtractor)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task F: Python chunker (`code-python-ast-v1`)
|
||||
|
||||
**Files:**
|
||||
- Create: `crates/kebab-chunk/src/code_python_ast_v1.rs`
|
||||
- Modify: `crates/kebab-chunk/src/lib.rs` (`mod` + `pub use`)
|
||||
|
||||
NEAR-DUPLICATE of `crates/kebab-chunk/src/code_rust_ast_v1.rs`. ONLY differences:
|
||||
- `const VERSION_LABEL: &str = "code-python-ast-v1";`
|
||||
- struct name `CodePythonAstV1Chunker`
|
||||
- The validation message says "code-python-ast-v1 only handles..."
|
||||
|
||||
`split_oversize` + `make_chunk` + `AST_CHUNK_MAX_LINES` + `BYTES_PER_TOKEN` + `POLICY_HASH_HEX_LEN` IDENTICAL (these are language-agnostic).
|
||||
|
||||
- [ ] **Step 1 (failing tests)**: Copy the entire `#[cfg(test)] mod tests` from `code_rust_ast_v1.rs` and substitute `Rust` → `Python` / `code-rust-ast-v1` → `code-python-ast-v1`. Use the same in-memory `code_doc` helper — it doesn't care about the actual language. Add one extra test specifically asserting the `policy_hash` equals the Rust chunker's (cross-chunker fingerprint identity is a 1A-2 invariant — must hold for new chunkers too).
|
||||
|
||||
- [ ] **Step 2**: Run → FAIL.
|
||||
|
||||
- [ ] **Step 3**: Copy `code_rust_ast_v1.rs` to `code_python_ast_v1.rs` and apply the substitutions above. Keep the `tree-sitter is intentionally NOT a dependency here` comment (still true).
|
||||
|
||||
- [ ] **Step 4**: Wire into `lib.rs`:
|
||||
|
||||
```rust
|
||||
mod code_python_ast_v1;
|
||||
pub use code_python_ast_v1::CodePythonAstV1Chunker;
|
||||
```
|
||||
|
||||
- [ ] **Step 5**: `cargo test -p kebab-chunk code_python_ast` → pass. Full per-crate suite stays green.
|
||||
|
||||
- [ ] **Step 6**: clippy clean, commit.
|
||||
|
||||
```bash
|
||||
git add crates/kebab-chunk/
|
||||
git commit -m "feat(p10-1b): code-python-ast-v1 chunker (1:1 + oversize split)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task G: Activate Python in app dispatch
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/kebab-app/src/lib.rs` (replace the Python `bail!` arm with real calls)
|
||||
- Modify: `crates/kebab-app/tests/code_ingest_smoke.rs` (un-`#[ignore]` the Python test, OR add it now if you deferred in Task D)
|
||||
|
||||
- [ ] **Step 1**: Replace the Python arm's `bail!` with `PythonAstExtractor::new().extract(...)` + `CodePythonAstV1Chunker.chunk(...)` calls (mirror the Rust arm exactly). Set parser_version / chunker_version per Python.
|
||||
|
||||
- [ ] **Step 2**: Un-ignore / add `python_file_ingests_and_searches_as_code_citation`. Test asserts the full pipeline produces a `Citation::Code { lang: Some("python"), symbol: Some("kebab_eval.metrics.compute_mrr"), .. }` for a `kebab_eval/metrics.py` written into the temp workspace.
|
||||
|
||||
- [ ] **Step 3**: `cargo test -p kebab-app code_ingest_smoke python_file_ingests` → pass. Existing Rust test stays green.
|
||||
|
||||
- [ ] **Step 4**: clippy clean, commit.
|
||||
|
||||
```bash
|
||||
git add crates/kebab-app/
|
||||
git commit -m "feat(p10-1b): activate Python in ingest_one_code_asset dispatch
|
||||
|
||||
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task H: TypeScript Extractor (`kebab-parse-code/src/typescript.rs`)
|
||||
|
||||
**Files:**
|
||||
- Create: `crates/kebab-parse-code/src/typescript.rs`
|
||||
- Modify: `crates/kebab-parse-code/src/lib.rs`
|
||||
- Create: `crates/kebab-parse-code/tests/fixtures/sample.ts` + `sample.tsx`
|
||||
|
||||
Scaffold mirrors `rust.rs`/`python.rs`. Grammar selection: `tree_sitter_typescript::LANGUAGE_TYPESCRIPT` for `.ts`, `LANGUAGE_TSX` for `.tsx`. Decide inside `extract` by inspecting `asset.workspace_path.0` extension (a tiny helper local to this module is fine).
|
||||
|
||||
### TypeScript AST mapping
|
||||
|
||||
| node kind | unit | symbol (joined with `.`) |
|
||||
|-----------|------|--------------------------|
|
||||
| `function_declaration` (name) | 1 | `<mod>.<fn>` |
|
||||
| `class_declaration` (name) — recurse into `class_body`: each `method_definition` (name) → unit `<mod>.<Class>.<method>` | 1 + 1 per method | as above |
|
||||
| `interface_declaration` (name), `type_alias_declaration` (name), `enum_declaration` (name) | 1 | `<mod>.<Name>` |
|
||||
| `export_statement` wrapping any of the above | unwrap to inner declaration; if the inner is `class_declaration` / `function_declaration` / `interface_declaration` / `type_alias_declaration` / `enum_declaration`, treat as that arm. If `export_statement` itself contains a default (i.e., `export default function () {...}` with no name field), emit unit symbol `<mod>.default`. | unwrapped as above, OR `<mod>.default` for nameless default |
|
||||
| `lexical_declaration` / `variable_declaration` at top level (`const`/`let`/`var`) | glue | `<top-level>` (prefixed) |
|
||||
| `import_statement`, `export_statement` of bare values | glue | as above |
|
||||
|
||||
`mod_path` for TS is empty (TS modules are file-level, not nested class/namespace at the symbol level — interfaces/types DO live in module scope but their names are unit-level, not parent context). Skip TS `namespace` / `module` declarations: emit them as glue for 1B (the explicit-namespace case is rare in modern TS; documented in 1B Risks).
|
||||
|
||||
Module prefix: `mod_prefix = module_path_for_tsjs(&asset.workspace_path.0)`. Join with `.` for symbol.
|
||||
|
||||
### Steps
|
||||
|
||||
- [ ] **Step 1 (fixtures)**:
|
||||
|
||||
```typescript
|
||||
// sample.ts
|
||||
import { x } from "./other";
|
||||
const ANSWER = 42;
|
||||
export interface Greet { hello(): string; }
|
||||
export type Maybe<T> = T | null;
|
||||
export function add(a: number, b: number): number { return a + b; }
|
||||
export class Retriever {
|
||||
search(q: string): string[] { return []; }
|
||||
static create(): Retriever { return new Retriever(); }
|
||||
}
|
||||
export default function () { return 1; }
|
||||
```
|
||||
|
||||
```tsx
|
||||
// sample.tsx
|
||||
import React from "react";
|
||||
export function Hello({ name }: { name: string }) { return <span>{name}</span>; }
|
||||
export const App = () => <Hello name="x" />; // arrow fn assigned → glue in 1B
|
||||
```
|
||||
|
||||
- [ ] **Step 2 (failing tests)**: 2 fixture-based tests asserting per-fixture symbols. Asserted symbols (sample.ts):
|
||||
- `src/sample.add` (if workspace_path is `src/sample.ts`)
|
||||
- `src/sample.Greet`, `src/sample.Maybe`, `src/sample.Retriever`, `src/sample.Retriever.search`, `src/sample.Retriever.create`, `src/sample.default`, `src/sample.<top-level>`.
|
||||
- For sample.tsx (workspace_path `src/sample.tsx`): `src/sample.Hello`, `src/sample.<top-level>` (App arrow fn rolled into glue).
|
||||
- Also: `extractor_supports_only_media_code_typescript`, `deterministic_across_runs`.
|
||||
|
||||
- [ ] **Step 3**: Run → FAIL.
|
||||
|
||||
- [ ] **Step 4**: Implement `typescript.rs` mirroring `rust.rs` scaffold. Grammar selection by file extension. AST walk per the table above. Module prefix application same shape as Python (prefix joined with `.`).
|
||||
|
||||
- [ ] **Step 5**: Wire into `lib.rs`:
|
||||
|
||||
```rust
|
||||
pub mod typescript;
|
||||
pub use typescript::{PARSER_VERSION as TS_PARSER_VERSION, TypescriptAstExtractor};
|
||||
```
|
||||
|
||||
- [ ] **Step 6**: Tests pass, clippy clean, commit.
|
||||
|
||||
---
|
||||
|
||||
## Task I: TS chunker (`code-ts-ast-v1`)
|
||||
|
||||
Pattern identical to Task F — duplicate `code_rust_ast_v1.rs` with substitutions (`VERSION_LABEL = "code-ts-ast-v1"`, struct `CodeTsAstV1Chunker`, error message). Test module copies the Rust chunker tests with name substitutions + adds `policy_hash_matches_md_heading_v1`.
|
||||
|
||||
Commit:
|
||||
|
||||
```
|
||||
feat(p10-1b): code-ts-ast-v1 chunker (1:1 + oversize split)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task J: Activate TypeScript in app dispatch
|
||||
|
||||
Mirror Task G. Replace TS `bail!` arm with real calls. Add `typescript_file_ingests_and_searches_as_code_citation` integration test using a `src/Foo.ts` fixture.
|
||||
|
||||
Commit:
|
||||
|
||||
```
|
||||
feat(p10-1b): activate TypeScript in ingest_one_code_asset dispatch
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task K: JavaScript Extractor (`javascript.rs`)
|
||||
|
||||
Mirror Task H. tree-sitter-javascript single LanguageFn. AST mapping similar to TS but without `interface_declaration` / `type_alias_declaration` / `enum_declaration`. Module prefix via `module_path_for_tsjs`.
|
||||
|
||||
Test fixture `sample.js`:
|
||||
|
||||
```javascript
|
||||
// sample.js
|
||||
import { x } from "./other";
|
||||
const ANSWER = 42;
|
||||
export function add(a, b) { return a + b; }
|
||||
export class Retriever {
|
||||
search(q) { return []; }
|
||||
static create() { return new Retriever(); }
|
||||
}
|
||||
export default function () { return 1; }
|
||||
```
|
||||
|
||||
Asserted symbols: `src/sample.add`, `src/sample.Retriever`, `src/sample.Retriever.search`, `src/sample.Retriever.create`, `src/sample.default`, `src/sample.<top-level>`.
|
||||
|
||||
Wire into `lib.rs`:
|
||||
|
||||
```rust
|
||||
pub mod javascript;
|
||||
pub use javascript::{PARSER_VERSION as JS_PARSER_VERSION, JavascriptAstExtractor};
|
||||
```
|
||||
|
||||
Commits:
|
||||
|
||||
```
|
||||
feat(p10-1b): tree-sitter-javascript AST extractor (JavascriptAstExtractor)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task L: JS chunker (`code-js-ast-v1`) + Activate JS in app dispatch
|
||||
|
||||
Combine Task F + Task G shape for JS in a single commit (less ceremony than splitting since the diffs are tiny):
|
||||
|
||||
- Chunker: duplicate-with-substitution from `code_rust_ast_v1.rs`. `VERSION_LABEL = "code-js-ast-v1"`, struct `CodeJsAstV1Chunker`.
|
||||
- App dispatch: replace JS `bail!` with real calls.
|
||||
- Integration test: `javascript_file_ingests_and_searches_as_code_citation`.
|
||||
|
||||
Commit:
|
||||
|
||||
```
|
||||
feat(p10-1b): code-js-ast-v1 chunker + activate JS in app dispatch
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task M: Snapshots + full-suite gate + manual SMOKE
|
||||
|
||||
**Files:**
|
||||
- Create: `crates/kebab-chunk/tests/code_python_ast_snapshot.rs` + fixture `tests/fixtures/code-sample.py` + baseline `code-sample.chunks.snapshot.json`
|
||||
- Create: same for TS (`code_ts_ast_snapshot.rs` + fixture `.ts` + baseline)
|
||||
- Create: same for JS (`code_js_ast_snapshot.rs` + fixture `.js` + baseline)
|
||||
|
||||
Mirror `crates/kebab-chunk/tests/code_rust_ast_snapshot.rs` exactly for each language. Build the `CanonicalDocument` IN-MEMORY (no `kebab-parse-code` dep crossing the chunk boundary).
|
||||
|
||||
- [ ] **Step 1**: Add the 3 snapshot tests. Generate baselines: `UPDATE_SNAPSHOTS=1 cargo test -p kebab-chunk code_{python,ts,js}_ast_snapshot`. Re-run without env var → PASS.
|
||||
|
||||
- [ ] **Step 2**: Full-suite gate (memory-conscious):
|
||||
- `cargo clippy --workspace --all-targets -- -D warnings` (one invocation, no parallel).
|
||||
- `cargo test --workspace --no-fail-fast -j 1` (the `-j 1` is mandatory). If the pre-existing `runner_lexical_is_deterministic_per_query_payload` flake reappears (unlikely — was fixed in PR #141 on main and merged before 1B branch was cut), re-run that single test once.
|
||||
|
||||
- [ ] **Step 3**: Manual SMOKE (mirror `docs/SMOKE.md` P10-1A-2 flow for each language):
|
||||
|
||||
```bash
|
||||
cargo build --release
|
||||
rm -rf /tmp/kebab-1bsmoke && mkdir -p /tmp/kebab-1bsmoke/ws/{kebab_eval,src}
|
||||
echo 'def compute_mrr(): return 1.0' > /tmp/kebab-1bsmoke/ws/kebab_eval/metrics.py
|
||||
echo 'export function add(a,b){return a+b;}' > /tmp/kebab-1bsmoke/ws/src/foo.ts
|
||||
echo 'export function sub(a,b){return a-b;}' > /tmp/kebab-1bsmoke/ws/src/bar.js
|
||||
# (match isolated config block format from docs/SMOKE.md)
|
||||
./target/release/kebab --config /tmp/kebab-1bsmoke/config.toml ingest --json | jq '.items[].parser_version' | sort -u
|
||||
./target/release/kebab --config /tmp/kebab-1bsmoke/config.toml search "compute_mrr" --code-lang python --json | jq '.hits[0]'
|
||||
./target/release/kebab --config /tmp/kebab-1bsmoke/config.toml schema --json | jq '.stats.code_lang_breakdown'
|
||||
```
|
||||
|
||||
Expected: parser_versions include `code-python-v1`, `code-ts-v1`, `code-js-v1`. Search returns `Citation::Code { lang: "python", symbol: "kebab_eval.metrics.compute_mrr" }`. `code_lang_breakdown` includes all four langs (rust may be 0 unless you also added a .rs).
|
||||
|
||||
- [ ] **Step 4**: Commit (snapshot files + any harness tweaks).
|
||||
|
||||
```bash
|
||||
git add crates/kebab-chunk/tests/
|
||||
git commit -m "test(p10-1b): per-language chunker snapshots + full-suite gate"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task N: Docs + HOTFIXES + version bump
|
||||
|
||||
- README: 지원 형식 / 명령 table row adds Python / TypeScript / JavaScript next to Rust. Mermaid stays unchanged (no new external surface crosses the diagram).
|
||||
- HANDOFF: P10 row notes 1B merged (3 langs active). Add a one-line entry under 머지 후 결정 cross-linking the HOTFIXES entries.
|
||||
- ARCHITECTURE: dependency-graph edge `pcode → core` already present. The new tree-sitter-{python,typescript,javascript} edges to `pcode` add to the description text. Locked-in decisions table: add "1B symbol path: workspace path → module path (Python dotted, TS/JS slash-style); Rust 1A keeps file-scope nesting only — HOTFIXES 2026-05-20".
|
||||
- SMOKE: add 1B section mirroring the 1A-2 P10 section structure (config block, ingest / search / schema verification commands) for Python and TS/JS. Compact — one shared section for all three.
|
||||
- tasks/INDEX + tasks/p10/INDEX: flip 1B row 🟡→🟢 (on PR open; ✅ on merge).
|
||||
- tasks/HOTFIXES.md: TWO dated 2026-05-20 entries:
|
||||
1. **Rust 1A-2 symbol path is file-scope-only; 1B+ uses workspace path → module prefix**. Cross-link to design §3.4. Acceptable inconsistency for now (cost of 1A retrofit = chunker_version bump + reindex for every existing Rust corpus). User-requested retrofit triggers a separate task.
|
||||
2. **Expression-level functions (arrow fn / function expression assigned to const) NOT emitted as separate units in 1B 1차**. They fold into the `<top-level>` glue. Documented limit; future phase may add `lexical_declaration` → inner-expression unwrap.
|
||||
Cross-link both in `tasks/p10/p10-1b-py-ts-js-ast-chunkers.md` Risks/notes.
|
||||
- `docs/superpowers/specs/2026-04-27-kebab-final-form-design.md` §10.1: add a one-liner — "p10-1B 활성화 (Python / TypeScript / JavaScript)".
|
||||
- `Cargo.toml`: workspace version `0.7.0 → 0.8.0`. `cargo build --release` refreshes Cargo.lock.
|
||||
- One commit:
|
||||
|
||||
```bash
|
||||
git add -A
|
||||
git commit -m "docs(p10-1b): README/HANDOFF/ARCHITECTURE/SMOKE/INDEX + HOTFIXES; chore: bump version 0.7.0 → 0.8.0
|
||||
|
||||
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Finalize
|
||||
|
||||
- `gitea-pr` open the PR (gitea-ops skill) — title `feat(p10-1B): Python + TS/JS AST chunkers — tree-sitter-{python,typescript,javascript} 코드 색인 활성화`.
|
||||
- **Review loop mode** (fixed per workflow memory) until APPROVE → merge → main pull → branch cleanup → `cargo clean` → `gitea-release v0.8.0`.
|
||||
|
||||
---
|
||||
|
||||
## Self-review checklist (filled by plan author)
|
||||
|
||||
- **Spec coverage**: every row of design §1B has a task; §3.4 symbol path covered by Task C + per-language extractors + integration tests; §6.1/§6.2 module structure covered by Tasks E/F/H/I/K/L; §9.1 Tier-1 + oversize fallback inherited from 1A-2 chunker pattern (Tasks F/I/L); §3.5 code_lang already in 1A-2 helper, extended in Task B routing; §5 dispatch covered by Task D; cascade rule (versioning §9) — chunker versions are per-language, fixture snapshots lock behavior.
|
||||
- **No placeholders**: all novel logic (module_path helpers, app dispatch generalization, Python AST walk rules) given concretely with full code or exact deltas vs 1A-2. The per-language chunkers are explicit "duplicate code_rust_ast_v1.rs with substitution X/Y/Z" — concrete and verifiable, not vague.
|
||||
- **Type consistency**: parser_version constants (`code-{rust,python,ts,js}-v1`) and chunker_version labels (`code-{rust,python,ts,js}-ast-v1`) used consistently across Tasks D/E/F/G/H/I/J/K/L. `module_path_for_python` / `module_path_for_tsjs` referenced consistently as the source of truth for prefixing.
|
||||
540
docs/superpowers/plans/2026-05-20-p10-1c-go-ast-chunker.md
Normal file
540
docs/superpowers/plans/2026-05-20-p10-1c-go-ast-chunker.md
Normal file
@@ -0,0 +1,540 @@
|
||||
# p10-1C-Go Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development to implement this plan task-by-task.
|
||||
|
||||
**Goal:** Activate Go code ingest end-to-end on top of 1A-2 (Rust) + 1B (Python/TS/JS) infrastructure. Add `tree-sitter-go` grammar + `GoAstExtractor` + `code-go-ast-v1` chunker + media routing + app dispatch arm.
|
||||
|
||||
**Architecture:** Mirror 1A-2 / 1B exactly. `kebab-parse-code/src/go.rs` walks tree-sitter-go parse tree; emits one `Block::Code` per top-level AST semantic unit with `SourceSpan::Code { symbol, lang: Some("go") }`. Symbol prefix = **source-extracted package name** (from `package_clause` AST node — design §3.4 Go row). `kebab-chunk/src/code_go_ast_v1.rs` is a near-duplicate of `code-rust-ast-v1`. App dispatch's `ingest_one_code_asset` (PR #142 generalized 4-arm match) gets a 5th arm.
|
||||
|
||||
**Tech Stack:** Rust 2024 workspace, `tree-sitter` 0.26 (already in workspace), `tree-sitter-go` (NEW), 1A-2/1B infrastructure unchanged.
|
||||
|
||||
**Memory note:** Host has been OOM-killed previously. Use `cargo test -p <crate>` and `cargo check -p <crate>` only. ONE full-suite invocation reserved for Task G gate.
|
||||
|
||||
---
|
||||
|
||||
## Pre-flight
|
||||
|
||||
Branch `feat/p10-1c-go` already exists.
|
||||
|
||||
- [ ] **Disk hygiene**: `cargo clean` if previous artifacts are bloated. Skip if disk is comfortable (`df -h /`).
|
||||
|
||||
Reference files:
|
||||
- 1A-2 Rust extractor: `crates/kebab-parse-code/src/rust.rs` — closest single-language scaffold template.
|
||||
- 1B Python extractor (closest analog for "class-nesting recursion" — Go doesn't have classes but has package as the single prefix): `crates/kebab-parse-code/src/python.rs`.
|
||||
- 1A-2 chunker scaffold: `crates/kebab-chunk/src/code_rust_ast_v1.rs`.
|
||||
- 1B dispatch generalization: `crates/kebab-app/src/lib.rs::ingest_one_code_asset` (~L1645, 4-arm match).
|
||||
- 1A-2 source-fs routing: `crates/kebab-source-fs/src/media.rs` `"rs" =>` arm.
|
||||
|
||||
---
|
||||
|
||||
## Task A: Workspace dep `tree-sitter-go`
|
||||
|
||||
**Files:**
|
||||
- Modify: `Cargo.toml` (workspace `[workspace.dependencies]`, after `tree-sitter-javascript` line)
|
||||
- Modify: `crates/kebab-parse-code/Cargo.toml`
|
||||
|
||||
- [ ] **Step 1**: `cargo add tree-sitter-go -p kebab-parse-code` to resolve version.
|
||||
|
||||
- [ ] **Step 2**: Lift the resolved version into `[workspace.dependencies]` after `tree-sitter-javascript`:
|
||||
|
||||
```toml
|
||||
# Go grammar for code ingest (kebab-parse-code, p10-1C).
|
||||
tree-sitter-go = "<resolved>"
|
||||
```
|
||||
|
||||
Switch the crate's entry to `{ workspace = true }` matching existing tree-sitter-* style.
|
||||
|
||||
- [ ] **Step 3**: `cargo build -p kebab-parse-code` → clean. Unused dep warning is fine.
|
||||
|
||||
- [ ] **Step 4**: Commit:
|
||||
|
||||
```bash
|
||||
git add Cargo.toml Cargo.lock crates/kebab-parse-code/Cargo.toml
|
||||
git commit -m "build(p10-1c-go): add tree-sitter-go workspace dep
|
||||
|
||||
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task B: source-fs media routing `.go` → `MediaType::Code("go")`
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/kebab-source-fs/src/media.rs` (add arm after the existing JS arm at ~L44)
|
||||
- Test: same file's test module
|
||||
|
||||
- [ ] **Step 1 (failing test)** — add to existing tests near `py_ts_js_files_map_to_media_code`:
|
||||
|
||||
```rust
|
||||
#[test]
|
||||
fn go_files_map_to_media_code_go() {
|
||||
assert_eq!(media_type_for(Path::new("a/b.go")), MediaType::Code("go".into()));
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2**: Run → FAIL.
|
||||
|
||||
- [ ] **Step 3**: Add the arm before the catch-all `_ => MediaType::Other(ext)`:
|
||||
|
||||
```rust
|
||||
// p10-1C-Go: Go ingest activated.
|
||||
"go" => MediaType::Code("go".into()),
|
||||
```
|
||||
|
||||
- [ ] **Step 4**: Run → PASS. `cargo test -p kebab-source-fs` → no regression.
|
||||
|
||||
- [ ] **Step 5**: clippy clean, commit:
|
||||
|
||||
```bash
|
||||
git add crates/kebab-source-fs/
|
||||
git commit -m "feat(p10-1c-go): route .go to MediaType::Code(go)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task C: App dispatch allowlist + bail arm for "go"
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/kebab-app/src/lib.rs` (dispatch match guard + 4 internal match arms in `ingest_one_code_asset`)
|
||||
|
||||
- [ ] **Step 1**: Find the `MediaType::Code(lang) if matches!(lang.as_str(), "rust" | "python" | "typescript" | "javascript")` arm (~L953). Add `"go"` to the allowlist:
|
||||
|
||||
```rust
|
||||
MediaType::Code(lang)
|
||||
if matches!(lang.as_str(), "rust" | "python" | "typescript" | "javascript" | "go") =>
|
||||
{
|
||||
```
|
||||
|
||||
- [ ] **Step 2**: In `ingest_one_code_asset`'s 4 `match code_lang` blocks (parser_version, chunker_version, extract, chunk), add a "go" arm that `bail!()`s for now (extractor + chunker land in Task D/E). Mirror the Python/TS/JS bail-then-activate pattern:
|
||||
|
||||
```rust
|
||||
let parser_version = match code_lang {
|
||||
// ... existing arms ...
|
||||
"go" => anyhow::bail!("go ingest not yet wired (p10-1c-go Task F)"),
|
||||
other => anyhow::bail!("unsupported code_lang: {other}"),
|
||||
};
|
||||
// similar for chunker_version / extract / chunk matches
|
||||
```
|
||||
|
||||
- [ ] **Step 3**: `cargo test -p kebab-app --lib` → existing 52 lib tests stay green. `cargo test -p kebab-app --test code_ingest_smoke` → 6 stay green (Rust path unaffected).
|
||||
|
||||
- [ ] **Step 4**: clippy clean, commit:
|
||||
|
||||
```bash
|
||||
git add crates/kebab-app/
|
||||
git commit -m "refactor(p10-1c-go): add go to ingest dispatch allowlist (bail until Task F)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task D: `GoAstExtractor` (`kebab-parse-code/src/go.rs`)
|
||||
|
||||
**Files:**
|
||||
- Create: `crates/kebab-parse-code/src/go.rs`
|
||||
- Modify: `crates/kebab-parse-code/src/lib.rs` (`pub mod go;` + re-exports `GO_PARSER_VERSION`, `GoAstExtractor`)
|
||||
- Create: `crates/kebab-parse-code/tests/fixtures/sample.go`
|
||||
|
||||
Scaffold mirrors `crates/kebab-parse-code/src/rust.rs` line-for-line for the `CanonicalDocument` skeleton (Extractor trait impl, `id_for_doc`, ProvenanceEvent, final `CanonicalDocument` literal). The novel parts:
|
||||
|
||||
### Constants
|
||||
|
||||
```rust
|
||||
pub const PARSER_VERSION: &str = "code-go-v1";
|
||||
|
||||
pub struct GoAstExtractor;
|
||||
// new() + Default
|
||||
// supports: matches!(m, MediaType::Code(l) if l == "go")
|
||||
// agent = "kb-parse-code"
|
||||
// metadata.code_lang = Some("go")
|
||||
// SourceType::Note (no SourceType::Code variant)
|
||||
// repo/git_branch/git_commit via detect_repo
|
||||
```
|
||||
|
||||
### Package extraction
|
||||
|
||||
Unlike 1B's path-based `module_path_for_python` / `_for_tsjs`, the Go package prefix comes from the **source code's `package` declaration** (design §3.4). tree-sitter-go's grammar:
|
||||
|
||||
- Root: `source_file`
|
||||
- First named child is typically `package_clause` → contains `package_identifier` child whose text is the package name.
|
||||
|
||||
Helper (local to `go.rs`):
|
||||
|
||||
```rust
|
||||
/// Returns the package name from a tree-sitter-go `source_file`, or
|
||||
/// `None` if the file has no `package_clause` (invalid Go in practice,
|
||||
/// but be defensive).
|
||||
fn extract_package(root: tree_sitter::Node, src: &str) -> Option<String> {
|
||||
let mut cur = root.walk();
|
||||
for child in root.named_children(&mut cur) {
|
||||
if child.kind() == "package_clause" {
|
||||
// `package_clause` has a `package_identifier` named child.
|
||||
let mut c2 = child.walk();
|
||||
for sub in child.named_children(&mut c2) {
|
||||
if sub.kind() == "package_identifier" {
|
||||
return Some(src[sub.start_byte()..sub.end_byte()].to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
```
|
||||
|
||||
### Semantic-unit rules
|
||||
|
||||
| node kind | unit | symbol |
|
||||
|-----------|------|--------|
|
||||
| `function_declaration` (name field) | 1 | `<pkg>.<fn_name>` |
|
||||
| `method_declaration` | 1 | `<pkg>.(<TypeText>).<MethodName>` where `<TypeText>` includes a leading `*` if the receiver is `pointer_type`. Examples: `chunk.(*MdHeadingV1Chunker).ChunkDoc`, `chunk.(Foo).Bar`. |
|
||||
| `type_declaration` (struct / interface / type alias) | 1 per inner `type_spec` | `<pkg>.<TypeName>` |
|
||||
| `const_declaration`, `var_declaration`, `import_declaration` (single or block) | glue | `<pkg>.<top-level>` (or `<pkg>.<package>` if file has ZERO real units AND glue is import-only — same `<module>` post-pass pattern as 1B Python, renamed to `<package>` to avoid colliding with Go's `package` keyword? — actually use `<module>` per design §3.4 — see "module / namespace 만 있고 symbol 없는 경우" line) |
|
||||
|
||||
`unit_start` walks `comment` siblings (same as 1B). Go doesn't have separate attribute / decorator nodes.
|
||||
|
||||
Method receiver pointer detection:
|
||||
|
||||
```rust
|
||||
// In the method_declaration arm:
|
||||
let receiver = child.child_by_field_name("receiver"); // parameter_list
|
||||
let receiver_type_text = receiver.and_then(|r| {
|
||||
let mut cw = r.walk();
|
||||
for p in r.named_children(&mut cw) {
|
||||
if p.kind() == "parameter_declaration" {
|
||||
// type field is either type_identifier (value) or pointer_type (ptr)
|
||||
if let Some(ty) = p.child_by_field_name("type") {
|
||||
let s = &src[ty.start_byte()..ty.end_byte()];
|
||||
return Some(s.to_string()); // includes leading "*" if pointer_type
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
});
|
||||
// Format: "(*Foo)" or "(Foo)" — wrap in parens, preserve leading "*" if any.
|
||||
let owner = receiver_type_text
|
||||
.map(|t| format!("({t})"))
|
||||
.unwrap_or_else(|| "()".to_string());
|
||||
let method_name = name_text(&child, src);
|
||||
// symbol = format!("{pkg}.{owner}.{method_name}")
|
||||
```
|
||||
|
||||
Read tree-sitter-go's grammar.json or node-types.json (in the registry source) if any field name above differs in the resolved crate version.
|
||||
|
||||
### Fixture `tests/fixtures/sample.go`:
|
||||
|
||||
```go
|
||||
// sample.go
|
||||
package chunk
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const Version = "v1"
|
||||
|
||||
type MdHeadingV1Chunker struct {
|
||||
Name string
|
||||
}
|
||||
|
||||
// ChunkDoc returns a stub list of strings.
|
||||
func (m *MdHeadingV1Chunker) ChunkDoc(input string) []string {
|
||||
return []string{m.Name}
|
||||
}
|
||||
|
||||
func (m MdHeadingV1Chunker) Name2() string {
|
||||
return m.Name
|
||||
}
|
||||
|
||||
type Stringer interface {
|
||||
String() string
|
||||
}
|
||||
|
||||
func Free(x int) int {
|
||||
return x + 1
|
||||
}
|
||||
|
||||
func init() {
|
||||
fmt.Println(strings.ToUpper("init"))
|
||||
}
|
||||
```
|
||||
|
||||
### Test module
|
||||
|
||||
Mirror Python's test shape (use `crate::rust::tests_support::fixed_code_asset` from 1B):
|
||||
|
||||
```rust
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use kebab_core::{Block, MediaType, SourceSpan};
|
||||
|
||||
fn extract_fixture() -> kebab_core::CanonicalDocument {
|
||||
let bytes = std::fs::read(
|
||||
concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/sample.go"),
|
||||
).unwrap();
|
||||
let asset = crate::rust::tests_support::fixed_code_asset(
|
||||
"crates/x/src/sample.go", "go",
|
||||
);
|
||||
let cfg = kebab_core::ExtractConfig::default();
|
||||
let root = std::path::PathBuf::from("/tmp");
|
||||
let ctx = kebab_core::ExtractContext { asset: &asset, workspace_root: &root, config: &cfg };
|
||||
GoAstExtractor::new().extract(&ctx, &bytes).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extractor_supports_only_media_code_go() {
|
||||
let e = GoAstExtractor::new();
|
||||
assert!(e.supports(&MediaType::Code("go".into())));
|
||||
assert!(!e.supports(&MediaType::Code("rust".into())));
|
||||
assert!(!e.supports(&MediaType::Markdown));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn go_units_match_design_3_4_symbols() {
|
||||
let doc = extract_fixture();
|
||||
let mut syms: Vec<String> = doc.blocks.iter().filter_map(|b| match b {
|
||||
Block::Code(c) => match &c.common.source_span {
|
||||
SourceSpan::Code { symbol, lang, .. } => {
|
||||
assert_eq!(lang.as_deref(), Some("go"));
|
||||
symbol.clone()
|
||||
}
|
||||
_ => None,
|
||||
},
|
||||
_ => None,
|
||||
}).collect();
|
||||
syms.sort();
|
||||
assert!(syms.iter().any(|s| s == "chunk.Free"), "got {syms:?}");
|
||||
assert!(syms.iter().any(|s| s == "chunk.init"));
|
||||
assert!(syms.iter().any(|s| s == "chunk.MdHeadingV1Chunker"));
|
||||
assert!(syms.iter().any(|s| s == "chunk.(*MdHeadingV1Chunker).ChunkDoc"));
|
||||
assert!(syms.iter().any(|s| s == "chunk.(MdHeadingV1Chunker).Name2"));
|
||||
assert!(syms.iter().any(|s| s == "chunk.Stringer"));
|
||||
assert!(syms.iter().any(|s| s == "chunk.<top-level>")); // import + const grouped
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deterministic_across_runs() {
|
||||
let a = extract_fixture();
|
||||
for _ in 0..50 { assert_eq!(extract_fixture().blocks, a.blocks); }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Step list
|
||||
|
||||
- [ ] Step 1: create fixture + test module.
|
||||
- [ ] Step 2: run → FAIL (`GoAstExtractor` undefined).
|
||||
- [ ] Step 3: implement `go.rs`. Scaffold mirrors `python.rs` (Extractor impl + extract scaffold + `build_blocks` returning blocks). `build_blocks` does: extract_package → walk root's named children → branch per node kind per the table above → emit `Block::Code` with `SourceSpan::Code { symbol, lang: Some("go") }`. Use the same `flush_glue` / glue grouping / `<top-level>` vs `<module>` post-pass as Python (rename to `<package>` if user prefers, but spec §3.4 says `<module>` so keep that name for cross-language consistency).
|
||||
- [ ] Step 4: wire into `lib.rs`:
|
||||
|
||||
```rust
|
||||
pub mod go;
|
||||
pub use go::{PARSER_VERSION as GO_PARSER_VERSION, GoAstExtractor};
|
||||
```
|
||||
|
||||
- [ ] Step 5: `cargo test -p kebab-parse-code` → all pass (Rust/Python/TS/JS + new Go). `cargo clippy -p kebab-parse-code --all-targets -- -D warnings` clean.
|
||||
- [ ] Step 6: commit:
|
||||
|
||||
```bash
|
||||
git add crates/kebab-parse-code/
|
||||
git commit -m "feat(p10-1c-go): tree-sitter-go AST extractor (GoAstExtractor)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task E: `code-go-ast-v1` chunker
|
||||
|
||||
**Files:**
|
||||
- Create: `crates/kebab-chunk/src/code_go_ast_v1.rs`
|
||||
- Modify: `crates/kebab-chunk/src/lib.rs`
|
||||
|
||||
Identical pattern to PR #142 Task I (TS) / Task L (JS) — near-duplicate of `code_rust_ast_v1.rs` with substitutions:
|
||||
- `const VERSION_LABEL: &str = "code-go-ast-v1";`
|
||||
- struct name `CodeGoAstV1Chunker`
|
||||
- error message says `"CodeGoAstV1Chunker only handles..."`
|
||||
- module doc-comment prose `Rust` → `Go`, `code-rust-ast-v1` → `code-go-ast-v1`
|
||||
|
||||
`split_oversize` / `make_chunk` / `AST_CHUNK_MAX_LINES = 200` / `BYTES_PER_TOKEN = 3` / `POLICY_HASH_HEX_LEN = 16` IDENTICAL (language-agnostic).
|
||||
|
||||
Test module: copy from `code_ts_ast_v1.rs` and substitute names. KEEP cross-chunker `policy_hash_matches_md_heading_v1`.
|
||||
|
||||
Wire into `crates/kebab-chunk/src/lib.rs`:
|
||||
|
||||
```rust
|
||||
mod code_go_ast_v1;
|
||||
pub use code_go_ast_v1::CodeGoAstV1Chunker;
|
||||
```
|
||||
|
||||
(Alphabetical placement.)
|
||||
|
||||
Verify + commit:
|
||||
- `cargo test -p kebab-chunk code_go_ast` PASS (~6 tests)
|
||||
- `cargo test -p kebab-chunk` full per-crate green
|
||||
- `cargo clippy -p kebab-chunk --all-targets -- -D warnings` clean
|
||||
|
||||
```bash
|
||||
git add crates/kebab-chunk/
|
||||
git commit -m "feat(p10-1c-go): code-go-ast-v1 chunker (1:1 + oversize split)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task F: Activate Go in app dispatch
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/kebab-app/src/lib.rs` (replace 4 "go" bail! arms with real calls)
|
||||
- Modify: `crates/kebab-app/tests/code_ingest_smoke.rs` (add Go integration test)
|
||||
|
||||
Replace the 4 `"go" => anyhow::bail!(...)` arms in `ingest_one_code_asset` (added in Task C) with real:
|
||||
|
||||
```rust
|
||||
"go" => ParserVersion(kebab_parse_code::GO_PARSER_VERSION.to_string()),
|
||||
// ...
|
||||
"go" => CodeGoAstV1Chunker.chunker_version(),
|
||||
// ...
|
||||
"go" => kebab_parse_code::GoAstExtractor::new()
|
||||
.extract(&ctx, &bytes)
|
||||
.context("kb-parse-code::GoAstExtractor::extract (code:go)")?,
|
||||
// ...
|
||||
"go" => CodeGoAstV1Chunker
|
||||
.chunk(&canonical, chunk_policy)
|
||||
.context("kb-chunk::CodeGoAstV1Chunker::chunk (code:go)")?,
|
||||
```
|
||||
|
||||
Add imports at top of lib.rs:
|
||||
- `kebab_chunk::CodeGoAstV1Chunker`
|
||||
- `kebab_parse_code::GoAstExtractor`
|
||||
|
||||
Integration test (mirror PR #142's `python_file_ingests_and_searches_as_code_citation`):
|
||||
|
||||
```rust
|
||||
#[test]
|
||||
fn go_file_ingests_and_searches_as_code_citation() {
|
||||
// ... TempDir + Config harness same as Python/TS test ...
|
||||
let pkg_dir = env.workspace_root.join("chunk");
|
||||
std::fs::create_dir_all(&pkg_dir).unwrap();
|
||||
std::fs::write(
|
||||
pkg_dir.join("ast.go"),
|
||||
"package chunk\n\nfunc ParseDoc(input string) string {\n return input\n}\n",
|
||||
).unwrap();
|
||||
|
||||
let report = kebab_app::ingest_with_config(/* ... */).unwrap();
|
||||
assert!(report.new >= 1);
|
||||
let go_item = report.items.as_ref().unwrap().iter()
|
||||
.find(|i| i.doc_path.0.ends_with("ast.go")).expect("ast.go item");
|
||||
assert_eq!(go_item.parser_version.as_ref().unwrap().0, "code-go-v1");
|
||||
assert_eq!(go_item.chunker_version.as_ref().unwrap().0, "code-go-ast-v1");
|
||||
|
||||
let hits = kebab_app::search_with_config(/* search "ParseDoc" */).unwrap();
|
||||
let h = hits.iter().find(|h| matches!(h.citation, kebab_core::Citation::Code { .. }))
|
||||
.expect("Citation::Code hit");
|
||||
match &h.citation {
|
||||
kebab_core::Citation::Code { lang, symbol, line_start, .. } => {
|
||||
assert_eq!(lang.as_deref(), Some("go"));
|
||||
assert_eq!(symbol.as_deref(), Some("chunk.ParseDoc"));
|
||||
assert!(*line_start >= 1);
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
assert_eq!(h.code_lang.as_deref(), Some("go"));
|
||||
}
|
||||
```
|
||||
|
||||
Verify:
|
||||
- `cargo test -p kebab-app --test code_ingest_smoke` → 7/7 (6 existing + 1 new go)
|
||||
- `cargo test -p kebab-app --lib` → 52/52 (no regression)
|
||||
- clippy clean
|
||||
|
||||
```bash
|
||||
git add crates/kebab-app/
|
||||
git commit -m "feat(p10-1c-go): activate Go in ingest_one_code_asset dispatch
|
||||
|
||||
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task G: Snapshot + full-suite gate + manual SMOKE
|
||||
|
||||
**Files:**
|
||||
- Create: `crates/kebab-chunk/tests/code_go_ast_snapshot.rs` + fixture + baseline (mirror `code_python_ast_snapshot.rs` from PR #142)
|
||||
|
||||
- [ ] **Step 1**: Add snapshot integration test. In-memory `CanonicalDocument` (no kebab-parse-code dep — boundary §6.3). Generate baseline: `UPDATE_SNAPSHOTS=1 cargo test -p kebab-chunk code_go_ast_snapshot` → re-run without env → PASS.
|
||||
|
||||
- [ ] **Step 2**: Full-suite gate (the ONE invocation allowed this PR):
|
||||
|
||||
```bash
|
||||
cargo clippy --workspace --all-targets -- -D warnings
|
||||
cargo test --workspace --no-fail-fast -j 1
|
||||
```
|
||||
|
||||
Both must be CLEAN/GREEN.
|
||||
|
||||
- [ ] **Step 3**: Manual SMOKE (optional but recommended — mirror PR #142 SMOKE):
|
||||
|
||||
```bash
|
||||
cargo build --release # OR debug if RAM-tight
|
||||
rm -rf /tmp/kebab-go-smoke && mkdir -p /tmp/kebab-go-smoke/ws/chunk
|
||||
echo 'package chunk
|
||||
|
||||
func ParseDoc(input string) string { return input }
|
||||
' > /tmp/kebab-go-smoke/ws/chunk/ast.go
|
||||
# adapt isolated config from docs/SMOKE.md
|
||||
./target/release/kebab --config /tmp/kebab-go-smoke/config.toml ingest --json | jq '.items[].parser_version' | sort -u
|
||||
./target/release/kebab --config /tmp/kebab-go-smoke/config.toml search "ParseDoc" --code-lang go --json | jq '.hits[0]'
|
||||
```
|
||||
|
||||
Expected: `code-go-v1` in parser_versions; Citation::Code with symbol `chunk.ParseDoc`.
|
||||
|
||||
- [ ] **Step 4**: Commit snapshot only (full-suite + SMOKE are gates, not commit content):
|
||||
|
||||
```bash
|
||||
git add crates/kebab-chunk/tests/
|
||||
git commit -m "test(p10-1c-go): code-go-ast-v1 chunker snapshot + full-suite gate
|
||||
|
||||
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task H: Docs + version bump
|
||||
|
||||
- README: 지원 형식 row — add Go (`.go`, `code-go-ast-v1`).
|
||||
- HANDOFF: P10 phase row note 1C-Go merged (Go active). Java/Kotlin remain pending.
|
||||
- ARCHITECTURE: directory tree note for kebab-parse-code includes `go.rs` (Java/Kotlin coming in next PR). Decisions table — no new row (1C-Go follows the 1A-2/1B convention).
|
||||
- SMOKE: extend the P10 section with a 1-line note for Go (or compact Go example).
|
||||
- tasks/INDEX + tasks/p10/INDEX: flip the row for 1C-Go to 🟡 (PR open) → ✅ on merge. The 1C row in p10/INDEX may need a split — `p10-1C-Go ⏳ → 🟡` and `p10-1C-JavaKotlin ⏳ unchanged` (since user split into 2 PRs).
|
||||
- frozen design §10.1: add a one-liner — "p10-1C-Go 활성화 (Go)" (Java/Kotlin will get its own line in the next PR).
|
||||
- `Cargo.toml`: workspace version `0.11.1 → 0.12.0` (minor — dogfooding surface 확장, 새 chunker + extractor 활성화).
|
||||
|
||||
```bash
|
||||
git add -A
|
||||
git commit -m "docs(p10-1c-go): README/HANDOFF/ARCHITECTURE/SMOKE/INDEX + chore: bump version 0.11.1 → 0.12.0
|
||||
|
||||
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Finalize: PR + review loop + release
|
||||
|
||||
Per workflow memory (gitea-pr + review loop, no single-shot):
|
||||
|
||||
- [ ] `gitea-pr` → PR title `feat(p10-1C-Go): tree-sitter-go AST extractor + chunker — Go 코드 색인 활성화`
|
||||
- [ ] Review loop until APPROVE → merge → main pull → branch cleanup → `cargo clean` → `gitea-release v0.12.0`.
|
||||
|
||||
---
|
||||
|
||||
## Self-Review (filled by plan author)
|
||||
|
||||
- **Spec coverage**: design §1C Go (extractor + chunker + activation) → Tasks D/E/F; §3.3 (`code-go-ast-v1`) → Task E; §3.4 symbol path → Task D (extract_package + method receiver pointer detection); §6.1 (`kebab-parse-code/src/go.rs`) → Task D; §6.2 (`kebab-chunk/src/code_go_ast_v1.rs`) → Task E; §6.3 dep graph (`tree-sitter-go` parser-side) → Task A; §9.1 Tier-1 + oversize fallback → Task E (1A-2 split_oversize reused identically).
|
||||
- **No placeholders**: novel logic (`extract_package`, method receiver pointer detection, fixture, test assertions, dispatch arm additions) given concretely. Mechanical mirrors (chunker, integration test, snapshot test) pinned to exact existing files with substitutions.
|
||||
- **Type consistency**: `GoAstExtractor` / `GO_PARSER_VERSION = "code-go-v1"` / `CodeGoAstV1Chunker` / `VERSION_LABEL = "code-go-ast-v1"` used consistently across Tasks A-H. `MediaType::Code("go")` in routing + dispatch. `Citation::Code` with `lang: Some("go")` in integration test.
|
||||
494
docs/superpowers/plans/2026-05-20-p10-1c-jk-ast-chunker.md
Normal file
494
docs/superpowers/plans/2026-05-20-p10-1c-jk-ast-chunker.md
Normal file
@@ -0,0 +1,494 @@
|
||||
# p10-1C-JavaKotlin Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development to implement this plan task-by-task.
|
||||
|
||||
**Goal:** Activate Java + Kotlin code ingest end-to-end. Mirror 1C-Go (PR #151 / v0.12.0) for Java (single-language scaffold) and Kotlin (additional top-level fn variant). Both use source-side `package` extraction (design §3.4 JVM convention).
|
||||
|
||||
**Architecture:** Same shape as 1B (multi-language single PR). 2 new tree-sitter grammars + 2 extractors + 2 chunkers + media routing + app dispatch arms. 1C-Go pattern is the closest template for source-side `package` extraction.
|
||||
|
||||
**Tech Stack:** Rust 2024 workspace, `tree-sitter` 0.26 (already), `tree-sitter-java` + `tree-sitter-kotlin` (NEW). 1A-2/1B/1C-Go infrastructure unchanged.
|
||||
|
||||
**Memory note:** Host has been OOM'd previously. Per-crate cargo only. ONE full-suite + clippy invocation in Task J.
|
||||
|
||||
---
|
||||
|
||||
## Pre-flight
|
||||
|
||||
Branch `feat/p10-1c-jk` already exists.
|
||||
|
||||
- [ ] **Disk hygiene**: `cargo clean` if heavy (last cleanup recovered 34 GB).
|
||||
|
||||
Reference files:
|
||||
- 1C-Go extractor: `crates/kebab-parse-code/src/go.rs` — closest template for source-side package extraction.
|
||||
- 1B Python extractor: `crates/kebab-parse-code/src/python.rs` — class-nesting recursion model (relevant for Java/Kotlin).
|
||||
- 1A-2 chunker: `crates/kebab-chunk/src/code_rust_ast_v1.rs` — duplicate-with-substitution.
|
||||
- 1B dispatch generalization: `crates/kebab-app/src/lib.rs::ingest_one_code_asset` 4-arm match (~L1645). 1C-Go already added `"go"`; this PR adds `"java"` + `"kotlin"`.
|
||||
|
||||
---
|
||||
|
||||
## Task A: Workspace deps (tree-sitter-java + tree-sitter-kotlin)
|
||||
|
||||
**Files:**
|
||||
- Modify: `Cargo.toml` (workspace `[workspace.dependencies]`, after `tree-sitter-go` line)
|
||||
- Modify: `crates/kebab-parse-code/Cargo.toml`
|
||||
|
||||
- [ ] **Step 1**: `cargo add tree-sitter-java tree-sitter-kotlin -p kebab-parse-code`. If `tree-sitter-kotlin` resolves to a fork name, verify the actively-maintained crate (e.g. check crates.io page / GitHub stars / last update). Likely `tree-sitter-kotlin` (without fork suffix) is the default.
|
||||
|
||||
- [ ] **Step 2**: Lift the two resolved versions into `[workspace.dependencies]` after `tree-sitter-go`:
|
||||
|
||||
```toml
|
||||
# JVM family grammars for code ingest (kebab-parse-code, p10-1C-JK).
|
||||
tree-sitter-java = "<resolved>"
|
||||
tree-sitter-kotlin = "<resolved>"
|
||||
```
|
||||
|
||||
Switch crate's entries to `{ workspace = true }`.
|
||||
|
||||
- [ ] **Step 3**: `cargo build -p kebab-parse-code` → clean. Unused dep warning is fine.
|
||||
|
||||
- [ ] **Step 4**: Commit:
|
||||
|
||||
```bash
|
||||
git add Cargo.toml Cargo.lock crates/kebab-parse-code/Cargo.toml
|
||||
git commit -m "build(p10-1c-jk): add tree-sitter-java + tree-sitter-kotlin workspace deps
|
||||
|
||||
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
If the kotlin crate has a different actual name (e.g. `tree-sitter-kotlin-ng` or fork suffix), document the choice in the commit body briefly.
|
||||
|
||||
---
|
||||
|
||||
## Task B: source-fs routing `.java` / `.kt` / `.kts`
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/kebab-source-fs/src/media.rs` (add arm after the existing `.go` arm)
|
||||
- Test: same file's test module
|
||||
|
||||
- [ ] **Step 1 (failing test)** — add near `go_files_map_to_media_code_go`:
|
||||
|
||||
```rust
|
||||
#[test]
|
||||
fn java_kotlin_files_map_to_media_code() {
|
||||
assert_eq!(media_type_for(Path::new("a/b.java")), MediaType::Code("java".into()));
|
||||
assert_eq!(media_type_for(Path::new("a/b.kt")), MediaType::Code("kotlin".into()));
|
||||
assert_eq!(media_type_for(Path::new("a/b.kts")), MediaType::Code("kotlin".into()));
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2**: Run → FAIL.
|
||||
|
||||
- [ ] **Step 3**: Add the arms before the `_ => MediaType::Other(ext)` fallback (after `"go" => ...`):
|
||||
|
||||
```rust
|
||||
// p10-1C-JK: JVM family (Java + Kotlin) ingest activated.
|
||||
"java" => MediaType::Code("java".into()),
|
||||
"kt" | "kts" => MediaType::Code("kotlin".into()),
|
||||
```
|
||||
|
||||
- [ ] **Step 4**: Run → PASS. `cargo test -p kebab-source-fs` → no regression.
|
||||
|
||||
- [ ] **Step 5**: clippy clean, commit.
|
||||
|
||||
```bash
|
||||
cargo clippy -p kebab-source-fs --all-targets -- -D warnings
|
||||
git add crates/kebab-source-fs/
|
||||
git commit -m "feat(p10-1c-jk): route .java/.kt/.kts to MediaType::Code
|
||||
|
||||
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task C: App dispatch + bail arms for "java" + "kotlin"
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/kebab-app/src/lib.rs`
|
||||
|
||||
- [ ] **Step 1**: Find the dispatch arm guard (currently `matches!(lang.as_str(), "rust" | "python" | "typescript" | "javascript" | "go")`). Add `"java"` + `"kotlin"`:
|
||||
|
||||
```rust
|
||||
MediaType::Code(lang)
|
||||
if matches!(lang.as_str(),
|
||||
"rust" | "python" | "typescript" | "javascript" | "go" | "java" | "kotlin") =>
|
||||
```
|
||||
|
||||
- [ ] **Step 2**: In `ingest_one_code_asset` the 4 `match code_lang` blocks add `"java"` and `"kotlin"` arms that `bail!()` for now:
|
||||
|
||||
```rust
|
||||
"java" => anyhow::bail!("java ingest not yet wired (p10-1c-jk Task F)"),
|
||||
"kotlin" => anyhow::bail!("kotlin ingest not yet wired (p10-1c-jk Task I)"),
|
||||
```
|
||||
|
||||
(in each of the 4 blocks before the `other =>` catch-all).
|
||||
|
||||
- [ ] **Step 3**: Verify per-crate:
|
||||
- `cargo test -p kebab-app --lib` → 52 stay green
|
||||
- `cargo test -p kebab-app --test code_ingest_smoke` → 7 stay green
|
||||
- `cargo clippy -p kebab-app --all-targets -- -D warnings` clean
|
||||
|
||||
- [ ] **Step 4**: Commit:
|
||||
|
||||
```bash
|
||||
git add crates/kebab-app/
|
||||
git commit -m "refactor(p10-1c-jk): add java + kotlin to ingest dispatch allowlist (bail until Tasks F/I)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task D: `JavaAstExtractor`
|
||||
|
||||
**Files:**
|
||||
- Create: `crates/kebab-parse-code/src/java.rs`
|
||||
- Modify: `crates/kebab-parse-code/src/lib.rs` (`pub mod java;` + re-exports `JAVA_PARSER_VERSION`, `JavaAstExtractor`)
|
||||
- Create: `crates/kebab-parse-code/tests/fixtures/sample.java`
|
||||
|
||||
Scaffold mirrors `crates/kebab-parse-code/src/go.rs` (1C-Go) — single-language with source-side `package` extraction. Differences:
|
||||
|
||||
### Constants
|
||||
|
||||
```rust
|
||||
pub const PARSER_VERSION: &str = "code-java-v1";
|
||||
pub struct JavaAstExtractor;
|
||||
// supports: matches!(m, MediaType::Code(l) if l == "java")
|
||||
// code_lang = Some("java"), SourceType::Note, repo via detect_repo
|
||||
```
|
||||
|
||||
### Package extraction (Java)
|
||||
|
||||
tree-sitter-java grammar:
|
||||
- Root: `program`
|
||||
- `package_declaration` (top-level child) → contains `scoped_identifier` (dotted) OR `identifier` (single-segment)
|
||||
|
||||
```rust
|
||||
fn extract_package(root: tree_sitter::Node, src: &str) -> Option<String> {
|
||||
let mut cur = root.walk();
|
||||
for child in root.named_children(&mut cur) {
|
||||
if child.kind() == "package_declaration" {
|
||||
// package_declaration has scoped_identifier OR identifier as first named child
|
||||
let mut c2 = child.walk();
|
||||
for sub in child.named_children(&mut c2) {
|
||||
if sub.kind() == "scoped_identifier" || sub.kind() == "identifier" {
|
||||
return Some(src[sub.start_byte()..sub.end_byte()].to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
```
|
||||
|
||||
(Verify field names against tree-sitter-java's node-types.json if any field differs.)
|
||||
|
||||
### AST mapping
|
||||
|
||||
| node kind | unit | symbol |
|
||||
|-----------|------|--------|
|
||||
| `class_declaration` (name field) | 1 + recurse body | `<pkg>.<ClassName>` |
|
||||
| `interface_declaration` (name) | 1 + recurse body | `<pkg>.<InterfaceName>` |
|
||||
| `enum_declaration` (name) | 1 | `<pkg>.<EnumName>` |
|
||||
| `record_declaration` (name, Java 14+) | 1 | `<pkg>.<RecordName>` |
|
||||
| `annotation_type_declaration` (name) | 1 | `<pkg>.<AnnotationName>` |
|
||||
| Inside class body: `method_declaration` (name) | 1 | `<pkg>.<Class>.<method>` |
|
||||
| Inside class body: `constructor_declaration` (name = class name) | 1 | `<pkg>.<Class>.<ClassName>` (matches Java convention) |
|
||||
| Nested classes recurse with class name pushed onto mod_path | as above | `<pkg>.<Outer>.<Inner>` etc. |
|
||||
| `import_declaration`, `package_declaration` | glue | `<pkg>.<top-level>` |
|
||||
| `field_declaration` at top of class | NOT a unit in 1C-JK (would explode unit count for value-only fields) | n/a |
|
||||
|
||||
`unit_start` walks `comment` siblings; Java has `@interface` annotations but those are part of `annotation_type_declaration` itself, not separate sibling nodes.
|
||||
|
||||
`mod_path` = class nesting (like 1B Python). Empty at file top level.
|
||||
|
||||
### Fixture `tests/fixtures/sample.java`:
|
||||
|
||||
```java
|
||||
// sample.java
|
||||
package com.kebab.chunk;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* Heading-aware Markdown chunker.
|
||||
*/
|
||||
public class MdHeadingV1Chunker {
|
||||
private final String name;
|
||||
|
||||
public MdHeadingV1Chunker(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
public List<String> chunkDoc(String input) {
|
||||
return List.of(name, input);
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
private String name;
|
||||
public Builder withName(String n) { this.name = n; return this; }
|
||||
public MdHeadingV1Chunker build() { return new MdHeadingV1Chunker(name); }
|
||||
}
|
||||
}
|
||||
|
||||
interface Stringer {
|
||||
String asString();
|
||||
}
|
||||
|
||||
enum Mode { DEFAULT, FAST }
|
||||
```
|
||||
|
||||
### Test module (inline `#[cfg(test)] mod tests`)
|
||||
|
||||
Mirror 1C-Go shape:
|
||||
|
||||
```rust
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use kebab_core::{Block, MediaType, SourceSpan};
|
||||
|
||||
fn extract_fixture() -> kebab_core::CanonicalDocument {
|
||||
let bytes = std::fs::read(
|
||||
concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/sample.java"),
|
||||
).unwrap();
|
||||
let asset = crate::rust::tests_support::fixed_code_asset(
|
||||
"crates/x/src/sample.java", "java",
|
||||
);
|
||||
let cfg = kebab_core::ExtractConfig::default();
|
||||
let root = std::path::PathBuf::from("/tmp");
|
||||
let ctx = kebab_core::ExtractContext { asset: &asset, workspace_root: &root, config: &cfg };
|
||||
JavaAstExtractor::new().extract(&ctx, &bytes).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extractor_supports_only_media_code_java() { /* ... */ }
|
||||
|
||||
#[test]
|
||||
fn java_units_match_design_3_4_symbols() {
|
||||
let doc = extract_fixture();
|
||||
let mut syms: Vec<String> = doc.blocks.iter().filter_map(|b| match b {
|
||||
Block::Code(c) => match &c.common.source_span {
|
||||
SourceSpan::Code { symbol, lang, .. } => {
|
||||
assert_eq!(lang.as_deref(), Some("java"));
|
||||
symbol.clone()
|
||||
}
|
||||
_ => None,
|
||||
},
|
||||
_ => None,
|
||||
}).collect();
|
||||
syms.sort();
|
||||
// workspace path → package extracted from source = com.kebab.chunk
|
||||
assert!(syms.iter().any(|s| s == "com.kebab.chunk.MdHeadingV1Chunker"), "got {syms:?}");
|
||||
assert!(syms.iter().any(|s| s == "com.kebab.chunk.MdHeadingV1Chunker.MdHeadingV1Chunker")); // constructor
|
||||
assert!(syms.iter().any(|s| s == "com.kebab.chunk.MdHeadingV1Chunker.chunkDoc"));
|
||||
assert!(syms.iter().any(|s| s == "com.kebab.chunk.MdHeadingV1Chunker.getName"));
|
||||
assert!(syms.iter().any(|s| s == "com.kebab.chunk.MdHeadingV1Chunker.Builder"));
|
||||
assert!(syms.iter().any(|s| s == "com.kebab.chunk.MdHeadingV1Chunker.Builder.withName"));
|
||||
assert!(syms.iter().any(|s| s == "com.kebab.chunk.MdHeadingV1Chunker.Builder.build"));
|
||||
assert!(syms.iter().any(|s| s == "com.kebab.chunk.Stringer"));
|
||||
assert!(syms.iter().any(|s| s == "com.kebab.chunk.Mode"));
|
||||
assert!(syms.iter().any(|s| s == "com.kebab.chunk.<top-level>"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deterministic_across_runs() {
|
||||
let a = extract_fixture();
|
||||
for _ in 0..50 { assert_eq!(extract_fixture().blocks, a.blocks); }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Wire into lib.rs
|
||||
|
||||
```rust
|
||||
pub mod java;
|
||||
pub use java::{PARSER_VERSION as JAVA_PARSER_VERSION, JavaAstExtractor};
|
||||
```
|
||||
|
||||
### Verify + commit
|
||||
|
||||
- `cargo test -p kebab-parse-code` → all pass
|
||||
- `cargo clippy -p kebab-parse-code --all-targets -- -D warnings` clean
|
||||
- commit `feat(p10-1c-jk): tree-sitter-java AST extractor (JavaAstExtractor)`
|
||||
|
||||
---
|
||||
|
||||
## Task E: `code-java-ast-v1` chunker
|
||||
|
||||
Identical pattern to 1C-Go Task E. Duplicate `code_rust_ast_v1.rs` with substitutions:
|
||||
- `VERSION_LABEL = "code-java-ast-v1"`, struct `CodeJavaAstV1Chunker`
|
||||
- error message + module doc-comment prose
|
||||
- Test module: parser_version `"code-java-v1"`, code_lang `"java"`
|
||||
- Keep cross-chunker `policy_hash_matches_md_heading_v1`
|
||||
|
||||
Wire into `crates/kebab-chunk/src/lib.rs` (alphabetical). Verify + commit.
|
||||
|
||||
---
|
||||
|
||||
## Task F: Activate Java in app dispatch
|
||||
|
||||
Replace the `"java"` `bail!()` arms in `ingest_one_code_asset` with real calls (`JavaAstExtractor` + `CodeJavaAstV1Chunker`). Add integration test `java_file_ingests_and_searches_as_code_citation` (mirror 1C-Go test, fixture `pkg_dir/Foo.java` with `package com.foo;` and `public class Foo { public String bar() { ... } }`, assert symbol `com.foo.Foo.bar`).
|
||||
|
||||
Verify + commit.
|
||||
|
||||
---
|
||||
|
||||
## Task G: `KotlinAstExtractor`
|
||||
|
||||
**Files:**
|
||||
- Create: `crates/kebab-parse-code/src/kotlin.rs`
|
||||
- Modify: `crates/kebab-parse-code/src/lib.rs`
|
||||
- Create: `crates/kebab-parse-code/tests/fixtures/sample.kt`
|
||||
|
||||
Constants: `PARSER_VERSION = "code-kotlin-v1"`, `KotlinAstExtractor`, `code_lang = "kotlin"`.
|
||||
|
||||
### Package extraction (Kotlin)
|
||||
|
||||
tree-sitter-kotlin grammar:
|
||||
- Root: `source_file`
|
||||
- `package_header` (top-level) → contains `identifier` (dotted is single `identifier` node text; verify against node-types.json)
|
||||
|
||||
```rust
|
||||
fn extract_package(root: tree_sitter::Node, src: &str) -> Option<String> {
|
||||
let mut cur = root.walk();
|
||||
for child in root.named_children(&mut cur) {
|
||||
if child.kind() == "package_header" {
|
||||
let mut c2 = child.walk();
|
||||
for sub in child.named_children(&mut c2) {
|
||||
if sub.kind() == "identifier" {
|
||||
return Some(src[sub.start_byte()..sub.end_byte()].to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
```
|
||||
|
||||
(Verify against tree-sitter-kotlin's node-types.json — Kotlin grammar varies more than Java's.)
|
||||
|
||||
### AST mapping (Kotlin)
|
||||
|
||||
| node kind | unit | symbol |
|
||||
|-----------|------|--------|
|
||||
| `class_declaration` (name field) — covers `class`, `data class`, `sealed class`, `enum class`, `interface` (Kotlin's interface is a class_declaration variant) | 1 + recurse body | `<pkg>.<ClassName>` |
|
||||
| `object_declaration` (name) — singleton | 1 + recurse | `<pkg>.<ObjectName>` |
|
||||
| `function_declaration` (name) | 1 | `<pkg>.<fn_name>` (top-level) or `<pkg>.<Class>.<method>` (inside class) |
|
||||
| Inside class body: `function_declaration` → method | 1 | `<pkg>.<Class>.<method>` |
|
||||
| `property_declaration` at top-level (`val` / `var`) | glue | `<top-level>` (Kotlin top-level properties are common — keep as glue not unit) |
|
||||
| `import_header`, `package_header` | glue | `<top-level>` |
|
||||
|
||||
(Detect class-vs-interface via modifier; for 1C 1차 treat both as `class_declaration` arm — symbol differs only via name. If tree-sitter-kotlin exposes `interface` keyword via modifier list, mention in HOTFIXES if special handling needed.)
|
||||
|
||||
### Fixture `sample.kt`:
|
||||
|
||||
```kotlin
|
||||
// sample.kt
|
||||
package com.kebab.chunk
|
||||
|
||||
import java.util.List
|
||||
|
||||
/**
|
||||
* Heading-aware Markdown chunker.
|
||||
*/
|
||||
class MdHeadingV1Chunker(val name: String) {
|
||||
fun chunkDoc(input: String): List<String> = listOf(name, input)
|
||||
|
||||
fun getName(): String = name
|
||||
|
||||
companion object {
|
||||
fun withName(n: String): MdHeadingV1Chunker = MdHeadingV1Chunker(n)
|
||||
}
|
||||
}
|
||||
|
||||
interface Stringer {
|
||||
fun asString(): String
|
||||
}
|
||||
|
||||
enum class Mode { DEFAULT, FAST }
|
||||
|
||||
fun freeFunction(x: Int): Int = x + 1
|
||||
|
||||
object Singleton {
|
||||
fun ping(): String = "pong"
|
||||
}
|
||||
```
|
||||
|
||||
### Test module — assert symbols
|
||||
|
||||
```rust
|
||||
// Asserted symbols:
|
||||
"com.kebab.chunk.MdHeadingV1Chunker"
|
||||
"com.kebab.chunk.MdHeadingV1Chunker.chunkDoc"
|
||||
"com.kebab.chunk.MdHeadingV1Chunker.getName"
|
||||
"com.kebab.chunk.MdHeadingV1Chunker.Companion" // companion object (verify name)
|
||||
"com.kebab.chunk.MdHeadingV1Chunker.Companion.withName" // method on companion
|
||||
"com.kebab.chunk.Stringer"
|
||||
"com.kebab.chunk.Mode"
|
||||
"com.kebab.chunk.freeFunction" // top-level fn (Kotlin-specific!)
|
||||
"com.kebab.chunk.Singleton"
|
||||
"com.kebab.chunk.Singleton.ping"
|
||||
"com.kebab.chunk.<top-level>" // import + property glue
|
||||
```
|
||||
|
||||
(Companion object: tree-sitter-kotlin may use `companion_object` or `object_declaration` with `companion` modifier — verify and adjust the symbol if `Companion` isn't the right name.)
|
||||
|
||||
### Wire into lib.rs
|
||||
|
||||
```rust
|
||||
pub mod kotlin;
|
||||
pub use kotlin::{PARSER_VERSION as KOTLIN_PARSER_VERSION, KotlinAstExtractor};
|
||||
```
|
||||
|
||||
Verify + commit.
|
||||
|
||||
---
|
||||
|
||||
## Task H: `code-kotlin-ast-v1` chunker
|
||||
|
||||
Same pattern as Task E. Substitute kotlin labels. Verify + commit.
|
||||
|
||||
---
|
||||
|
||||
## Task I: Activate Kotlin in app dispatch
|
||||
|
||||
Replace `"kotlin"` bail arms with real calls. Add integration test `kotlin_file_ingests_and_searches_as_code_citation`. Verify + commit.
|
||||
|
||||
---
|
||||
|
||||
## Task J: Snapshots + full-suite + SMOKE
|
||||
|
||||
- Create 2 snapshot tests (`code_java_ast_snapshot.rs`, `code_kotlin_ast_snapshot.rs`) + baselines. Mirror 1C-Go Task G snapshot test.
|
||||
- ONE workspace test + clippy invocation.
|
||||
- Manual SMOKE: write a `.java` and `.kt` file in TempDir, ingest, search.
|
||||
|
||||
Verify + commit (snapshot only).
|
||||
|
||||
---
|
||||
|
||||
## Task K: Docs + version bump
|
||||
|
||||
- README + HANDOFF + ARCHITECTURE + SMOKE + 2 INDEX updates + design §10.1.
|
||||
- `Cargo.toml` version `0.12.0 → 0.13.0` (minor, surface 확장).
|
||||
|
||||
Commit `docs(p10-1c-jk): ... + chore: bump 0.12.0 → 0.13.0`.
|
||||
|
||||
---
|
||||
|
||||
## Finalize
|
||||
|
||||
`gitea-pr` → review loop → merge → main pull → branch cleanup → `cargo clean` → `gitea-release v0.13.0`.
|
||||
|
||||
---
|
||||
|
||||
## Self-Review (filled by plan author)
|
||||
|
||||
- **Spec coverage**: design §1C Java + Kotlin → Tasks D-I; §3.4 symbol path → extractor (Java D, Kotlin G); §6.1/§6.2 module structure → Tasks D/E/G/H; §6.3 dep graph → Task A; §9.1 Tier-1 + oversize fallback → chunkers E/H.
|
||||
- **No placeholders**: novel logic (Java `extract_package`, Kotlin `extract_package`, AST walk arm tables) given concretely. Chunkers (E, H) are explicit "duplicate code_rust_ast_v1.rs with substitution X/Y/Z".
|
||||
- **Type consistency**: `JavaAstExtractor` / `JAVA_PARSER_VERSION` / `CodeJavaAstV1Chunker` + `KotlinAstExtractor` / `KOTLIN_PARSER_VERSION` / `CodeKotlinAstV1Chunker` used consistently. `MediaType::Code("java")` / `("kotlin")` in routing + dispatch.
|
||||
- **Kotlin grammar risk**: noted — tree-sitter-kotlin's exact node kinds (`class_declaration` vs `object_declaration`, `companion_object` vs companion modifier, `package_header` vs `package_directive`) should be verified against the resolved crate's node-types.json. Pin contract via test fixture; HOTFIXES any deviation found during implementation.
|
||||
@@ -1543,6 +1543,12 @@ transitional 형태) 의 source of truth.
|
||||
|
||||
**p10-1A-2 surface 활성화 (2026-05-19)**: Rust 소스코드 ingest (`code-rust-ast-v1` chunker, `tree-sitter-rust`) 가 활성화됨. `.rs` 파일을 워크스페이스에 두면 `kebab ingest` 가 AST 단위로 chunk 생성 + `citation.kind = "code"` 로 검색 가능. `kebab schema --json` 의 `stats.code_lang_breakdown` 에 `"rust": N` 이 표시됨. 본 activation 으로 kebab 자기 crate 를 dogfooding KB 에 색인 가능. `SourceSpan::Code` (§3.4) 와 `MediaType::Code` (§3.5) 는 1A-1 에서 이미 spec 에 반영됨. 두 deferred deviation (`AST_CHUNK_MAX_LINES` 상수 고정, `SourceType::Code` 미존재) 은 `tasks/HOTFIXES.md` (2026-05-19) 에 기록.
|
||||
|
||||
**p10-1B 활성화 (Python / TypeScript / JavaScript) (2026-05-20)**: Python (`code-python-ast-v1`, `.py`), TypeScript (`code-ts-ast-v1`, `.ts`/`.tsx`), JavaScript (`code-js-ast-v1`, `.js`/`.mjs`/`.cjs`/`.jsx`) AST chunker 활성화. symbol path 는 workspace 경로 → module path prefix: Python = dotted (예: `kebab_eval.metrics.compute_mrr`), TypeScript/JavaScript = slash-style (예: `src/Foo.Foo.search`). Rust 1A-2 의 file-scope-only symbol 과 비일관 수용 (HOTFIXES 2026-05-20). expression-level 함수 (`const foo = () => {}`) 는 glue 처리 (HOTFIXES 2026-05-20).
|
||||
|
||||
**p10-1C-Go 활성화 (Go) (2026-05-20)**: Go (`code-go-ast-v1`, `.go`) AST chunker 활성화. symbol = `<package>.<Func>` / `<package>.(*Receiver).<Method>` 형식.
|
||||
|
||||
**p10-1C-JavaKotlin 활성화 (Java + Kotlin) (2026-05-20)**: Java (`code-java-ast-v1`, `.java`) + Kotlin (`code-kotlin-ast-v1`, `.kt`/`.kts`) AST chunker 활성화. symbol = `com.foo.Foo.bar` 형식 (패키지 + 클래스 + 메서드/필드). Kotlin grammar 은 `tree-sitter-kotlin-ng` 사용 (bare `tree-sitter-kotlin` 은 tree-sitter 0.21–0.23 고착으로 사용 불가).
|
||||
|
||||
### 10.2 MCP server transport (fb-30)
|
||||
|
||||
`kebab mcp` 가 stdio JSON-RPC server. Rust SDK = `rmcp 1.6`. Tool surface
|
||||
|
||||
@@ -14,12 +14,18 @@
|
||||
"schema_version": { "const": "reset_report.v1" },
|
||||
"scope": {
|
||||
"type": "string",
|
||||
"enum": ["all", "data_only", "vector_only", "config_only"]
|
||||
"enum": ["all", "data_only", "vector_only", "config_only", "orphans_only"]
|
||||
},
|
||||
"removed_paths": {
|
||||
"type": "array",
|
||||
"items": { "type": "string" }
|
||||
},
|
||||
"embedding_rows_truncated": { "type": "integer", "minimum": 0 }
|
||||
"embedding_rows_truncated": { "type": "integer", "minimum": 0 },
|
||||
"orphans_purged": { "type": "integer", "minimum": 0, "default": 0 },
|
||||
"purged_paths": {
|
||||
"type": "array",
|
||||
"items": { "type": "string" },
|
||||
"default": []
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,6 +14,40 @@ historical contract that was implemented; this file accumulates the
|
||||
deltas so phase 5+ readers can find the live behavior without diffing
|
||||
git history.
|
||||
|
||||
## 2026-05-20 — p10-1B: Rust 1A-2 symbol path is file-scope-only; 1B+ uses workspace path → module prefix
|
||||
|
||||
**무엇이 바뀌었나**: P10-1A-2 의 Rust `code-rust-ast-v1` chunker 가 생성하는 symbol 은 file-scope mod-path nesting 만 사용한다 (예: `Foo::double`). P10-1B 이후 Python / TypeScript / JavaScript 의 symbol 은 workspace 경로 → module path prefix 를 포함한다 (예: `kebab_eval.metrics.compute_mrr`, `src/Foo.Foo.search`).
|
||||
|
||||
**원인**: 1A-2 는 symbol path 컨벤션이 확정되기 전에 구현됐고, 1B spec 에서 workspace path → module prefix 를 명시적 결정으로 확정했다 (p10-1b-py-ts-js-ast-chunkers.md §동결된 설계 결정). 1A-2 retrofit = `chunker_version` bump + Rust corpus 전체 re-ingest 비용이 수반됨.
|
||||
|
||||
**사용자 가시적 영향**: Rust 코드 검색 시 symbol 이 `<ClassName>::<method>` 형태 (workspace prefix 없음). Python/TypeScript/JavaScript 는 `<module.path>.<symbol>` / `<module/path>.<symbol>` 형태. 비일관이지만 각각은 일관되게 동작.
|
||||
|
||||
**proper fix**: Rust AST chunker 에 `module_path_for_rust(workspace_path)` helper 추가 + `chunker_version = "code-rust-ast-v2"` bump → 사용자가 명시 요청할 때까지 보류.
|
||||
|
||||
**cross-link**: `tasks/p10/p10-1b-py-ts-js-ast-chunkers.md` Risks / notes 섹션, design §3.4.
|
||||
|
||||
## 2026-05-20 — p10-1B: module_path_for_python / _tsjs do not sanitize non-ASCII / 공백 / 특수문자 in workspace path
|
||||
|
||||
**동작**: `module_path_for_python` 와 `module_path_for_tsjs` 가 workspace path 의 비-ASCII / 공백 / 따옴표 / 백슬래시 같은 특수문자를 그대로 prefix 에 통과시킨다. 예: `kebab eval/metrics.py` (공백 포함) → module prefix `kebab eval.metrics` — 라이브러리 코드는 동작하지만 symbol 텍스트에 공백이 들어간다.
|
||||
|
||||
**이유**: 1B 1차 단순화. 대다수 코드 베이스가 ASCII identifier + `/` 구분자만 사용하므로 사용자 경험상 영향 미미.
|
||||
|
||||
**해결**: 후속 phase 에서 path-sanitize 추가 검토. NFKC normalize 후 `[^A-Za-z0-9_.\-/]` → `_` 변환 식. 적용 시 chunker_version bump 트리거 (re-ingest cascade 필요).
|
||||
|
||||
**cross-link**: `tasks/p10/p10-1b-py-ts-js-ast-chunkers.md` Risks / notes 섹션 line 55.
|
||||
|
||||
## 2026-05-20 — p10-1B: expression-level functions (arrow fn, function expression assigned to const) NOT emitted as units in 1B 1차
|
||||
|
||||
**무엇이 바뀌었나**: TypeScript / JavaScript 의 `const foo = () => {...}` 또는 `const bar = function() {...}` 같은 expression-level 함수 할당은 `code-ts-ast-v1` / `code-js-ast-v1` 에서 독립 unit 으로 방출되지 않는다. 해당 코드는 가장 가까운 surrounding declaration-level unit (또는 `<top-level>` glue) 에 흡수된다.
|
||||
|
||||
**원인**: `function_declaration` / `class_declaration` / `method_definition` / `interface_declaration` 같은 declaration-level 노드만 unit 으로 선택. `lexical_declaration` (= `const / let / var`) 안의 function / arrow expression 은 별도 unwrap 없이 pass-through. 1B 1차 단순화.
|
||||
|
||||
**사용자 가시적 영향**: expression-level 함수 이름으로 검색 시 함수 body 를 포함하는 glue chunk 가 반환되지만, symbol 이 함수 이름 자체를 가리키지는 않는다. 함수명이 함수 본문 텍스트에 등장하므로 lexical / hybrid 검색으로 일반적으로 찾을 수 있다.
|
||||
|
||||
**proper fix**: `lexical_declaration` visitor 에서 binding value 가 `arrow_function` / `function` expression 인 경우 해당 identifier name 을 symbol 로 사용하는 unwrap 추가. 후속 phase 에서 검토.
|
||||
|
||||
**cross-link**: `tasks/p10/p10-1b-py-ts-js-ast-chunkers.md` Risks / notes 섹션.
|
||||
|
||||
## 2026-05-19 — p10-1A-2: AST_CHUNK_MAX_LINES constant vs config deviation
|
||||
|
||||
**무엇이 바뀌었나**: `kebab-chunk/src/code_rust_ast_v1.rs` 가 `IngestCodeCfg.ast_chunk_max_lines` config 값을 읽지 않고 모듈 상수 `AST_CHUNK_MAX_LINES = 200` 으로 고정함.
|
||||
|
||||
@@ -140,9 +140,10 @@ P0~P5 는 직렬. P6~P9 는 P5 이후 병렬 가능.
|
||||
|
||||
- P10 — [p10/](p10/) — code ingest (multi-task, sub-indexed in [p10/INDEX.md](p10/INDEX.md))
|
||||
- [p10-1A-1 code ingest framework](p10/p10-1a-1-code-ingest-framework.md) — ✅ 머지
|
||||
- [p10-1A-2 Rust AST chunker](p10/p10-1a-2-rust-ast-chunker.md) — 🟡 PR 오픈 (코드 완성, 머지 대기)
|
||||
- p10-1B Python + TS/JS AST chunkers — ⏳
|
||||
- p10-1C Go + Java + Kotlin AST chunkers — ⏳
|
||||
- [p10-1A-2 Rust AST chunker](p10/p10-1a-2-rust-ast-chunker.md) — ✅ 머지
|
||||
- [p10-1B Python + TS/JS AST chunkers](p10/p10-1b-py-ts-js-ast-chunkers.md) — 🟡 PR 오픈 (코드 완성, 머지 대기)
|
||||
- p10-1C-Go Go AST chunker — 🟡 PR 오픈 (v0.12.0, `code-go-ast-v1`)
|
||||
- p10-1C-JavaKotlin Java + Kotlin AST chunkers — 🟢 PR 오픈 (v0.13.0, `code-java-ast-v1` / `code-kotlin-ast-v1`)
|
||||
- p10-1D C + C++ AST chunkers — ⏳
|
||||
- p10-2 Tier 2 resource-aware — ⏳
|
||||
- p10-3 Tier 3 paragraph + line-window fallback — ⏳
|
||||
|
||||
@@ -3,9 +3,10 @@
|
||||
| ID | Subject | Status |
|
||||
|----|---------|--------|
|
||||
| 1A-1 | code ingest framework (wire schema, parse-code crate skeleton, filter flags, skip policy, config 절) | ✅ 머지 |
|
||||
| 1A-2 | Rust AST chunker | 🟡 PR 오픈 (코드 완성, 머지 대기) |
|
||||
| 1B | Python + TS/JS AST chunkers | ⏳ |
|
||||
| 1C | Go + Java + Kotlin AST chunkers | ⏳ |
|
||||
| 1A-2 | Rust AST chunker | ✅ 머지 |
|
||||
| 1B | Python + TS/JS AST chunkers | 🟡 PR 오픈 (코드 완성, 머지 대기) |
|
||||
| 1C-Go | Go AST chunker (`code-go-ast-v1`) | 🟡 PR 오픈 (v0.12.0) |
|
||||
| 1C-JavaKotlin | Java + Kotlin AST chunkers (`code-java-ast-v1` / `code-kotlin-ast-v1`) | 🟢 PR 오픈 (v0.13.0) |
|
||||
| 1D | C + C++ AST chunkers | ⏳ |
|
||||
| 2 | Tier 2 resource-aware (k8s / Dockerfile / manifest) | ⏳ |
|
||||
| 3 | Tier 3 paragraph + line-window fallback | ⏳ |
|
||||
|
||||
60
tasks/p10/p10-1b-py-ts-js-ast-chunkers.md
Normal file
60
tasks/p10/p10-1b-py-ts-js-ast-chunkers.md
Normal file
@@ -0,0 +1,60 @@
|
||||
# p10-1B — Python + TS/JS AST chunkers
|
||||
|
||||
**Status:** 🟡 진행 중
|
||||
**Contract sections:** §3.3 (chunker_version `code-python-ast-v1` / `code-ts-ast-v1` / `code-js-ast-v1`), §3.4 (symbol path — Python `pkg.module.Class.method`, TS/JS `module/Class.method` / `module/default`), §3.5 (code_lang `python` / `typescript` / `javascript`), §5 (확장자 라우팅 활성화), §6.1 (`kebab-parse-code/src/{python,typescript,javascript}.rs`), §6.2 (`kebab-chunk/src/code_{python,ts,js}_ast_v1.rs`), §9.1 (Tier 1 AST per-language + oversize fallback).
|
||||
**Design:** [2026-05-15-kebab-code-ingest-design.md](../../docs/superpowers/specs/2026-05-15-kebab-code-ingest-design.md) §1B.
|
||||
**Plan:** [2026-05-20-p10-1b-py-ts-js-ast-chunkers.md](../../docs/superpowers/plans/2026-05-20-p10-1b-py-ts-js-ast-chunkers.md).
|
||||
|
||||
## Goal
|
||||
|
||||
1A-2 가 깐 인프라 (`SourceSpan::Code`, `MediaType::Code(String)`, `Citation::Code` 매핑, `citation_helper` arm, `backfill_code_lang` + `backfill_repo`, `schema.v1.code_lang_breakdown`, `[ingest.code]` 절, HOTFIXES) 위에 **Python + TypeScript + JavaScript** 3 언어의 extractor + chunker 를 활성화. design §1B 묶음과 일치하는 단일 PR. 머지 시점부터 Python / TS / JS 프로젝트도 dogfooding 가능.
|
||||
|
||||
## 동결된 설계 결정 (이 task 로 확정)
|
||||
|
||||
- **Symbol path 의 module prefix = workspace 경로 → module path 변환** (design §3.4 예시 충실, 사용자 명시 결정):
|
||||
- **Python**: `crates/x/src/foo/bar.py` 같은 workspace_path 를 `/`/`__init__.py` 처리 + `.py`·`.pyi` strip + `/` → `.` 변환 후 dotted prefix 로 사용. 예시: `kebab_eval/metrics.py` 의 `def compute_mrr()` → symbol `kebab_eval.metrics.compute_mrr`. `pkg/__init__.py` 는 module `pkg` 자체. 변환은 `kebab-parse-code::lang::module_path_for_python(workspace_path)` 단일 함수 (source of truth).
|
||||
- **TS/JS**: `src/search/retriever/Retriever.ts` → `src/search/retriever/Retriever` prefix + `/` 구분자 보존 + `.ts`/`.tsx`/`.js`/`.jsx`/`.mjs`/`.cjs` strip. 예시: `src/search/retriever/Retriever.ts` 의 method `search` → `src/search/retriever/Retriever.search`. `module/default` 는 `export default function/class` 경우. 변환은 `module_path_for_tsjs(workspace_path)`.
|
||||
- **Rust 1A-2 는 retrofit 하지 않음** — 1A 는 file-scope nesting 만 사용 (workspace prefix 없음). 비일관 수용; HOTFIXES 2026-05-20 에 기록 + 사용자가 명시 요청 시 retrofit (chunker_version bump + re-ingest cascade 필요).
|
||||
- **TypeScript grammar selection**: `tree-sitter-typescript` crate 의 `LANGUAGE_TYPESCRIPT` 는 `.ts`, `LANGUAGE_TSX` 는 `.tsx` 에 사용. 파일 확장자로 선택. `code-ts-ast-v1` 하나의 chunker 가 둘 다 처리 (parser_version `code-ts-v1`).
|
||||
- **JavaScript grammar**: `tree-sitter-javascript` 단일 LanguageFn 가 `.js` / `.mjs` / `.cjs` / `.jsx` 모두 처리. 별도 분기 불필요.
|
||||
- **Expression-level 함수 (arrow fn / function expression assigned to const)**: 1B 1차에서는 *declaration-level 만* unit (function_declaration / class_declaration / method_definition / interface_declaration / type_alias_declaration / decorated_definition 등). `const foo = () => {...}` 같은 expression-level 은 glue 로 잡힘. HOTFIXES 2026-05-20 기록; 후속 phase 에서 lexical_declaration 안의 함수 표현식 unwrap 추가 검토.
|
||||
- **App dispatch 일반화**: 현재 `ingest_one_code_asset` 은 RustAstExtractor + CodeRustAstV1Chunker 하드코딩. 1B 에서 `lang: &str` 받아 dispatch (Rust 도 동일 함수로 흡수) — Extractor 와 Chunker 를 trait object 가 아니라 enum/match 로 선택 (kebab-app 만 변경, kebab-core/Chunker trait 불변). frozen design 영향 없음.
|
||||
- frozen design 자체는 변경 없음 (§3.4 의 symbol path 예시는 이미 본 결정과 일치). §10.1 (post-merge surface) 에 1B 활성화 한 줄 추가.
|
||||
|
||||
## Acceptance criteria
|
||||
|
||||
- `cargo test --workspace --no-fail-fast -j 1` passes (메모리 의식적으로는 per-crate; full-suite gate 는 Task K 직전 1회).
|
||||
- `cargo clippy --workspace --all-targets -- -D warnings` passes.
|
||||
- 3 언어 각각의 fixture (`tests/fixtures/sample.{py,ts,js}`) ingest → chunk snapshot 안정 + `Citation::Code` 의 symbol/line 이 §3.4 컨벤션 (workspace path → module path) 과 일치.
|
||||
- 격리 TempDir KB 에 Python/TS/JS 파일 하나씩 두고 `kebab search --code-lang {python|typescript|javascript} --json` 가 정상 결과 반환.
|
||||
- `kebab schema --json | jq .stats.code_lang_breakdown` 에 `python`, `typescript`, `javascript` 카운트 등장.
|
||||
- README + HANDOFF + ARCHITECTURE + SMOKE + tasks/INDEX + tasks/p10/INDEX 갱신.
|
||||
- frozen design §10.1 한 줄 추가 (1B 활성화).
|
||||
- HOTFIXES 2026-05-20 에 (a) Rust 1A-2 symbol path 비일관 (1B 와 다름), (b) expression-level 함수 단위 제외 — 두 편차 기록.
|
||||
- workspace `Cargo.toml` minor bump (0.7.0 → 0.8.0) — 도그푸딩 가능 surface 확장.
|
||||
|
||||
## Allowed dependencies
|
||||
|
||||
- `kebab-parse-code` 에 `tree-sitter-python`, `tree-sitter-typescript`, `tree-sitter-javascript` 추가 (workspace deps 경유). 기존 `kebab-core` / `anyhow` / `gix` / `tree-sitter` / `tree-sitter-rust` / `serde_json` / `time` / `tracing` 유지.
|
||||
- `kebab-chunk` 의 새 모듈 3개 (`code_python_ast_v1.rs` / `code_ts_ast_v1.rs` / `code_js_ast_v1.rs`) — 1A-2 chunker 와 동일 dep (kebab-core + serde_json_canonicalizer + blake3 + anyhow + tracing). tree-sitter 절대 import 금지.
|
||||
- `kebab-app` 변경 — 새 crate dep 없음.
|
||||
- `kebab-source-fs` — 확장자 추가만, 새 dep 없음.
|
||||
|
||||
## Forbidden dependencies
|
||||
|
||||
- `kebab-chunk` 가 `tree-sitter-*` 직접 import 금지 (AST 는 parser-side).
|
||||
- UI crate (cli / mcp / tui) 가 `kebab-parse-code` 직접 import 금지.
|
||||
- `kebab-parse-code` 가 store / embed / llm / rag 직접 import 금지 (design §8 inheritance).
|
||||
|
||||
## Risks / notes
|
||||
|
||||
- tree-sitter-typescript 의 `LANGUAGE_TYPESCRIPT` 와 `LANGUAGE_TSX` 가 별도 LanguageFn — 잘못 선택하면 TSX JSX 가 parse 실패. 파일 확장자 기반 선택을 단일 함수에서 결정 (테스트로 고정).
|
||||
- tree-sitter-python 의 `decorated_definition` 노드 처리 — 데코레이터가 wrap 하는 형태라 `function_definition` / `class_definition` 가 child. unwrap 필요 (decorator 라인은 unit_start backward extension 으로 자연스럽게 포함됨).
|
||||
- Python `pkg/__init__.py` 의 module path = `pkg` 자체 (basename 제거). `module_path_for_python` 가 이걸 처리.
|
||||
- TS/JS 의 `export default function/class` — name 이 없을 수 있음 (`export default function () {...}`). symbol `module/default` 로 표기 (design §3.4).
|
||||
- `module_path_for_python` / `module_path_for_tsjs` 가 workspace_path 의 비-ASCII / 공백 / 특수문자 처리 필요. 1B 1차에서는 그대로 전달 (sanitize 없음); HOTFIXES 에 path-sanitize 부재 기록.
|
||||
- 1A-2 `ingest_one_code_asset` 일반화로 인한 dispatch 코드 변경 — Rust 기존 동작 byte-identical 유지를 통합 테스트로 확인.
|
||||
- 머지 후 deviation 은 `tasks/HOTFIXES.md` 에 dated 로그 + 본 spec `Risks / notes` 에 one-line cross-link.
|
||||
- **[HOTFIXES 2026-05-20]** Rust 1A-2 symbol 은 file-scope nesting 만 (workspace prefix 없음); 1B 의 Python/TypeScript/JavaScript 와 비일관 — retrofit 은 사용자 명시 요청 시. 자세한 내용: `tasks/HOTFIXES.md` (2026-05-20, "Rust 1A-2 symbol path").
|
||||
- **[HOTFIXES 2026-05-20]** TypeScript/JavaScript 의 expression-level 함수 (`const foo = () => {}` 등) 는 `<top-level>` glue 로 처리됨, 독립 unit 미방출 — 후속 phase 에서 `lexical_declaration` unwrap 검토. 자세한 내용: `tasks/HOTFIXES.md` (2026-05-20, "expression-level functions").
|
||||
- **[HOTFIXES 2026-05-20]** `module_path_for_python` / `module_path_for_tsjs` 가 path-sanitize 안 함 (특수문자/공백 그대로 prefix 에 들어감) — 후속 phase 에서 NFKC + 사용금지 문자 변환 검토. 자세한 내용: `tasks/HOTFIXES.md` (2026-05-20, "module_path_for_python / _tsjs do not sanitize").
|
||||
54
tasks/p10/p10-1c-go-ast-chunker.md
Normal file
54
tasks/p10/p10-1c-go-ast-chunker.md
Normal file
@@ -0,0 +1,54 @@
|
||||
# p10-1C-Go — Go AST chunker
|
||||
|
||||
**Status:** 🟡 진행 중
|
||||
**Contract sections:** §3.3 (chunker_version `code-go-ast-v1`), §3.4 (symbol path — Go `package.Func` / `package.(*Receiver).Method`), §3.5 (code_lang `go`, ext `.go`), §6.1 (`kebab-parse-code/src/go.rs`), §6.2 (`kebab-chunk/src/code_go_ast_v1.rs`), §9.1 (Tier 1 AST per-language + oversize fallback).
|
||||
**Design:** [2026-05-15-kebab-code-ingest-design.md](../../docs/superpowers/specs/2026-05-15-kebab-code-ingest-design.md) §1C (Go 부분 — Java + Kotlin 은 후속 PR).
|
||||
**Plan:** [2026-05-20-p10-1c-go-ast-chunker.md](../../docs/superpowers/plans/2026-05-20-p10-1c-go-ast-chunker.md).
|
||||
|
||||
## Goal
|
||||
|
||||
1A-2 / 1B 인프라 위에 Go AST chunker 활성화. 사용자 결정으로 1C 의 3 언어 (Go + Java + Kotlin) 를 2 PR 로 분할 — Go 가 method receiver / package convention 면에서 Java/Kotlin (JVM family) 과 다르므로 별 PR. 본 PR 머지 시점부터 Go 프로젝트 dogfooding 가능.
|
||||
|
||||
## 동결된 설계 결정 (이 task 로 확정)
|
||||
|
||||
- **Symbol path 의 package prefix = 소스 코드의 `package` 선언에서 추출** (design §3.4 그대로). 1B 의 workspace-path 변환과 다름 — Go 는 언어 자체에 `package` declaration 이 있어 그게 canonical source. tree-sitter-go 의 `source_file` root 의 첫 named child `package_clause` 에서 추출. 빈 경우 (이론상 invalid Go, 실용엔 거의 없음) `<unknown>` 또는 fallback `<package>` (1A `<module>` 패턴과 유사).
|
||||
- **Method receiver 표현** (design 예시 그대로): `package.(*Receiver).Method` (포인터 receiver), `package.(Receiver).Method` (value receiver). tree-sitter-go 의 `method_declaration` 의 `receiver` field 에서 type + pointer 여부 추출. 예: `func (m *MdHeadingV1Chunker) ChunkDoc(...)` → symbol `chunk.(*MdHeadingV1Chunker).ChunkDoc`.
|
||||
- **Top-level unit 종류**:
|
||||
- `function_declaration` → 1 unit, symbol `package.Func`
|
||||
- `method_declaration` → 1 unit, symbol `package.(*Receiver).Method` / `package.(Receiver).Method`
|
||||
- `type_declaration` (struct / interface / type alias) → 1 unit each, symbol `package.TypeName`
|
||||
- `const_declaration`, `var_declaration`, `import_declaration` (블록 또는 단일) → glue, grouped → `package.<top-level>` (1A/1B 패턴)
|
||||
- **Go 의 generic 처리**: `func Foo[T any](...)` 또는 `type Foo[T any] struct{}` 의 type parameter 는 symbol 에 미포함 (Go 자체도 보통 symbol 에 안 적음). 단순 `package.Foo` 만.
|
||||
- **Test detection**: Go 의 `func TestXxx(t *testing.T)` 는 *일반 fn 으로 emit*. test 감지 boost/penalty 등 ranking 영향은 본 task 범위 밖 (ranking brainstorm 보류 메모리 따름).
|
||||
- frozen design 자체는 변경 없음 (§3.4 의 Go 행이 이미 본 결정과 일치). §10.1 에 1C-Go 활성화 한 줄 추가.
|
||||
|
||||
## Acceptance criteria
|
||||
|
||||
- `cargo test --workspace --no-fail-fast -j 1` passes (memory-conscious: per-crate 위주, full-suite gate 는 docs task 직전 1회).
|
||||
- `cargo clippy --workspace --all-targets -- -D warnings` passes.
|
||||
- Go fixture (`tests/fixtures/sample.go`) ingest → chunk snapshot 안정 + `Citation::Code` 의 symbol 이 §3.4 컨벤션 일치 (`pkg.Func` / `pkg.(*Receiver).Method`).
|
||||
- 격리 TempDir KB 에 Go 파일 두고 `kebab search --code-lang go --json` 가 `Citation::Code { lang: "go", symbol: "...", ... }` 반환.
|
||||
- `kebab schema --json | jq .stats.code_lang_breakdown` 에 `"go"` 카운트.
|
||||
- README + HANDOFF + ARCHITECTURE + SMOKE + tasks/INDEX + tasks/p10/INDEX 갱신.
|
||||
- frozen design §10.1 한 줄 추가.
|
||||
- workspace `Cargo.toml` minor bump (0.11.1 → 0.12.0).
|
||||
|
||||
## Allowed dependencies
|
||||
|
||||
- `kebab-parse-code` 에 `tree-sitter-go` 추가 (workspace deps). 기존 deps 유지.
|
||||
- `kebab-chunk` 의 새 모듈 `code_go_ast_v1.rs` — kebab-core + serde_json_canonicalizer + blake3 + anyhow + tracing. tree-sitter 절대 import 금지.
|
||||
- `kebab-app`, `kebab-source-fs` 변경 — 새 crate dep 없음.
|
||||
|
||||
## Forbidden dependencies
|
||||
|
||||
- `kebab-chunk` 가 `tree-sitter-go` 직접 import 금지.
|
||||
- UI crate 가 `kebab-parse-code` 직접 import 금지.
|
||||
- `kebab-parse-code` 가 store / embed / llm / rag 직접 import 금지.
|
||||
|
||||
## Risks / notes
|
||||
|
||||
- tree-sitter-go 의 `package_clause` node 가 root 의 첫 named child 인지 grammar 버전에 따라 다를 수 있음 — extractor 가 `source_file` 전체를 named_children iterate 하면서 첫 `package_clause` 잡는 방식이 안전.
|
||||
- `method_declaration` 의 receiver pointer 여부: tree-sitter-go AST 에서 receiver type 이 `pointer_type` 노드면 `*Receiver`, 그냥 `type_identifier` 면 `Receiver`. 정확한 텍스트 추출 필요.
|
||||
- Generic type parameter (`[T any]`) 가 method_declaration / function_declaration 의 name field 와 별도 child — name 만 추출하면 generic 부분 자동 제외.
|
||||
- 1B Python/TS/JS 패턴 (helpers from lang.rs) 와 *다른* 모델 — 본 task 의 mod_prefix 는 source-side AST 에서 추출, helper fn 불필요.
|
||||
- 머지 후 deviation 은 `tasks/HOTFIXES.md` 에 dated 로그 + 본 spec `Risks / notes` 에 one-line cross-link.
|
||||
69
tasks/p10/p10-1c-jk-ast-chunker.md
Normal file
69
tasks/p10/p10-1c-jk-ast-chunker.md
Normal file
@@ -0,0 +1,69 @@
|
||||
# p10-1C-JavaKotlin — Java + Kotlin AST chunkers
|
||||
|
||||
**Status:** 🟡 진행 중
|
||||
**Contract sections:** §3.3 (chunker_version `code-java-ast-v1` + `code-kotlin-ast-v1`), §3.4 (symbol path — Java/Kotlin `package.Class.method`), §3.5 (code_lang `java` + `kotlin`, ext `.java` / `.kt` / `.kts`), §6.1 (`kebab-parse-code/src/{java,kotlin}.rs`), §6.2 (`kebab-chunk/src/code_{java,kotlin}_ast_v1.rs`), §9.1 (Tier 1 AST per-language + oversize fallback).
|
||||
**Design:** [2026-05-15-kebab-code-ingest-design.md](../../docs/superpowers/specs/2026-05-15-kebab-code-ingest-design.md) §1C (Java + Kotlin 부분 — Go 는 PR #151 / v0.12.0 별 PR 완료).
|
||||
**Plan:** [2026-05-20-p10-1c-jk-ast-chunker.md](../../docs/superpowers/plans/2026-05-20-p10-1c-jk-ast-chunker.md).
|
||||
|
||||
## Goal
|
||||
|
||||
1C-Go (PR #151 / v0.12.0) 의 자매 PR. 같은 1C phase 의 JVM family (Java + Kotlin) 묶음. 머지 시점부터 `.java` / `.kt` / `.kts` 파일 dogfooding 가능.
|
||||
|
||||
## 동결된 설계 결정 (이 task 로 확정)
|
||||
|
||||
- **Symbol prefix = 소스 코드의 `package` 선언에서 추출** (design §3.4 그대로, 1C-Go 모델과 동일). 1B 의 workspace-path 변환과 다름.
|
||||
- **Java**: tree-sitter-java 의 `package_declaration` → 안의 `scoped_identifier` 또는 `identifier` 텍스트 (e.g. `com.kebab.chunk`). 없으면 `<unknown>`.
|
||||
- **Kotlin**: tree-sitter-kotlin 의 `package_header` → `identifier` 텍스트. 없으면 (default package) `<unknown>`.
|
||||
- **Symbol 형식** (design §3.4): `package.Class.method`. 예시: `com.kebab.chunk.MdHeadingV1Chunker.chunkDoc`.
|
||||
- **Java AST mapping**:
|
||||
- `class_declaration` (name) → 1 unit + recurse body
|
||||
- `interface_declaration` (name) → 1 unit + recurse
|
||||
- `enum_declaration` (name) → 1 unit
|
||||
- `record_declaration` (Java 14+) (name) → 1 unit
|
||||
- `annotation_type_declaration` → 1 unit
|
||||
- Inside class/interface/enum: `method_declaration` (name) → unit `package.Class.method` (class nesting like 1B Python)
|
||||
- `import_declaration`, `package_declaration` 자체 → glue `<top-level>`
|
||||
- Top-level fn 없음 (Java 자체에 없음)
|
||||
- **Kotlin AST mapping**:
|
||||
- `class_declaration` (name) → 1 unit + recurse class_body. `data class` / `sealed class` / `enum class` 도 같은 노드.
|
||||
- `object_declaration` (name) → 1 unit + recurse class_body (singleton)
|
||||
- `function_declaration` (name) — **top-level 가능** → unit `package.fnName`. Class 내부면 `package.Class.method`.
|
||||
- `property_declaration` at top-level → glue
|
||||
- `interface` (in tree-sitter-kotlin 보통 `class_declaration` with `interface` modifier 또는 별 노드) → 1 unit
|
||||
- `import_header`, `package_header` 자체 → glue `<top-level>`
|
||||
- **Glue grouping**: 1B Python / 1C-Go 패턴 동일 — imports + 기타 → 하나의 `<top-level>` (또는 `<module>` post-pass if file has zero real units).
|
||||
- **Tree-sitter Kotlin crate 선택**: tree-sitter-kotlin 의 가장 잘 유지되는 crate 사용 (`tree-sitter-kotlin` 또는 fork). resolve 시 active maintainer 확인.
|
||||
- frozen design 자체 변경 없음 — §10.1 에 1C-JK 활성화 한 줄.
|
||||
|
||||
## Acceptance criteria
|
||||
|
||||
- `cargo test --workspace --no-fail-fast -j 1` passes.
|
||||
- `cargo clippy --workspace --all-targets -- -D warnings` passes.
|
||||
- Java/Kotlin fixture 각각 (`tests/fixtures/sample.java`, `tests/fixtures/sample.kt`) ingest → chunk snapshot 안정 + symbol 이 §3.4 컨벤션 일치.
|
||||
- 격리 TempDir KB 에 `.java` / `.kt` 파일 두고 `kebab search --code-lang java --json` / `--code-lang kotlin --json` 가 `Citation::Code` 반환.
|
||||
- `kebab schema --json | jq .stats.code_lang_breakdown` 에 `"java"` + `"kotlin"` 카운트.
|
||||
- README + HANDOFF + ARCHITECTURE + SMOKE + tasks/INDEX + tasks/p10/INDEX 갱신.
|
||||
- frozen design §10.1 한 줄.
|
||||
- workspace `Cargo.toml` minor bump (0.12.0 → 0.13.0).
|
||||
|
||||
## Allowed dependencies
|
||||
|
||||
- `kebab-parse-code` 에 `tree-sitter-java` + `tree-sitter-kotlin` 추가. 기존 deps 유지.
|
||||
- `kebab-chunk` 의 새 모듈 2개 (`code_java_ast_v1.rs`, `code_kotlin_ast_v1.rs`) — language-agnostic body. tree-sitter import 금지.
|
||||
- `kebab-app`, `kebab-source-fs` — 새 crate dep 없음.
|
||||
|
||||
## Forbidden dependencies
|
||||
|
||||
- `kebab-chunk` 가 tree-sitter-java / tree-sitter-kotlin import 금지 (boundary §6.3).
|
||||
- UI crate 가 `kebab-parse-code` 직접 import 금지.
|
||||
- `kebab-parse-code` 가 store / embed / llm / rag 직접 import 금지.
|
||||
|
||||
## Risks / notes
|
||||
|
||||
- tree-sitter-kotlin: 공식 또는 가장 활발히 유지되는 crate (`tree-sitter-kotlin` 또는 fork) 선택 필요. resolve 시 metadata 확인.
|
||||
- Kotlin 의 grammar 가 다른 tree-sitter-* 보다 update 빈도 낮을 수 있어 grammar field 명 변동 가능 — 테스트 fixture 로 contract 고정.
|
||||
- Java record (Java 14+) — tree-sitter-java 에서 `record_declaration` 노드 (확인 필요).
|
||||
- Kotlin sealed class / data class / object declaration 등 변종 노드 — tree-sitter-kotlin 의 정확한 node kind 명 확인 필요 (grammar.json / node-types.json).
|
||||
- Java class 안의 inner class — Python 패턴 (recursion with class name pushed) 동일 처리.
|
||||
- Kotlin top-level fn 은 1B Python 의 top-level fn 패턴 + 1C-Go 의 package-prefix 패턴 hybrid — `package.fnName`.
|
||||
- 머지 후 deviation 은 `tasks/HOTFIXES.md` dated 로그 + 본 spec `Risks / notes` cross-link.
|
||||
Reference in New Issue
Block a user