Compare commits
221 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 5a90940f1c | |||
| 4389b887f0 | |||
| 360f825f3a | |||
| 641b92af7d | |||
| 08fb743598 | |||
| 0a2a7ae214 | |||
| 803d02b68b | |||
| 4e8b84c4e0 | |||
| 16dc02cfa2 | |||
| 74f1b0571b | |||
| 918ee6c0be | |||
| 68ada396f3 | |||
| 23c4ad97b9 | |||
| 1f566b8bfa | |||
| 26562588e3 | |||
| 4503b5b12f | |||
| 44813df052 | |||
| d6bb6cfd3b | |||
| d53995a6d4 | |||
| c215034653 | |||
| 31245a4328 | |||
| acb61b6830 | |||
| 20feb3133e | |||
| de63f161ac | |||
| 1815091247 | |||
| 6a0b340941 | |||
| 9664e97497 | |||
| 8bdb3e8090 | |||
| dcad9ccda2 | |||
| ed0f4769b3 | |||
| 0c61758931 | |||
| 39b766ea59 | |||
| 7f287abacb | |||
| d715631928 | |||
| 73e5b359d8 | |||
| c780aca904 | |||
| b1d5047399 | |||
| 80c2d31fb3 | |||
| 97e9f558f4 | |||
| da51e59081 | |||
| 11a0fc758f | |||
| b5d1fe8c1e | |||
| 580576c2c6 | |||
| 808b92a6c5 | |||
| c74f8d269e | |||
| df85bafa7f | |||
| a93b33ffbe | |||
| 402a4506a2 | |||
| a531dc37dc | |||
| 7a6a24ad10 | |||
| 42712b50c2 | |||
| 9f3edb7e24 | |||
| 5c265bb59f | |||
| a08ed32199 | |||
| 9362cd0aae | |||
|
|
7961f8813d | ||
|
|
7bbd2c0cbf | ||
|
|
d13f58d28a | ||
|
|
298f4adc81 | ||
|
|
4e8b70a04b | ||
|
|
682f7dd3a2 | ||
|
|
40b3ea8408 | ||
|
|
9fce24b106 | ||
|
|
8bbe25dc10 | ||
|
|
abfdcbd31d | ||
|
|
69d1593bc5 | ||
|
|
2a8451c033 | ||
|
|
ff11f81f7f | ||
|
|
bf4ebf8d2a | ||
|
|
351c7a0826 | ||
|
|
7329ba96ee | ||
|
|
fa4eeb5a87 | ||
|
|
3b1e878aed | ||
|
|
005a9011ea | ||
|
|
c6d61b0b37 | ||
|
|
49487dc46b | ||
| 2c2bf9bac5 | |||
| 72798bd3ff | |||
|
|
c3177561b9 | ||
| a465b71f99 | |||
|
|
787007172a | ||
|
|
b954e9ce66 | ||
|
|
c62a8ff503 | ||
|
|
69c94b6692 | ||
|
|
d5321701ea | ||
|
|
2c3461c465 | ||
| 240120ee80 | |||
|
|
5870a1de15 | ||
|
|
f00fb376fe | ||
|
|
bb0ec0469f | ||
|
|
f303c76f52 | ||
|
|
cd5b1e3bfc | ||
|
|
7c6c2e8102 | ||
| 3a9a52326d | |||
|
|
b53376e96e | ||
|
|
441f1192ee | ||
|
|
e8da415624 | ||
|
|
d8e5f35601 | ||
|
|
6ab0d782ef | ||
|
|
2bbe94eb05 | ||
|
|
9ac13fa256 | ||
|
|
67f2c16cc2 | ||
|
|
1ebbd6b711 | ||
|
|
892175d009 | ||
|
|
de9016fe16 | ||
|
|
35df15df99 | ||
| b0becf43b8 | |||
| 21ecbb00d4 | |||
|
|
8cd21e8342 | ||
|
|
b35f163f56 | ||
|
|
600c6182fc | ||
|
|
0e8b800b6b | ||
|
|
126559ce7a | ||
|
|
137fc4ee31 | ||
|
|
59f01f8185 | ||
|
|
9f70681b77 | ||
|
|
6d6eb442be | ||
|
|
28d3250546 | ||
| 945319ae93 | |||
|
|
c864bd007f | ||
|
|
67aee9f480 | ||
|
|
4440fa6659 | ||
|
|
b51cdb9e8f | ||
|
|
4e739f3cd8 | ||
|
|
3a621bba0d | ||
|
|
3c605b1a5d | ||
|
|
56f20b7235 | ||
|
|
0359bd9682 | ||
| cf3acfc136 | |||
|
|
668e1174cc | ||
| 745a75a82b | |||
|
|
6a33d08aea | ||
|
|
a40593590b | ||
|
|
5687cbc0e2 | ||
|
|
653e432a30 | ||
|
|
f7e2072d66 | ||
|
|
72c227af23 | ||
|
|
69037c313a | ||
|
|
6a067e3ab1 | ||
|
|
231d80e82d | ||
|
|
69c6e23432 | ||
|
|
1e943f21dc | ||
|
|
fb31befef1 | ||
|
|
5f6b2fa259 | ||
| a0497d9c53 | |||
|
|
b221686133 | ||
| a72c6f307c | |||
|
|
84287d0ef6 | ||
|
|
6e7446861b | ||
|
|
b06f4654e7 | ||
|
|
4e0379c04f | ||
|
|
6a18847892 | ||
|
|
c6cc1e2bfe | ||
|
|
86475e5ba2 | ||
|
|
2c80e2ad91 | ||
|
|
d3f38c76e9 | ||
|
|
31c1e05951 | ||
|
|
7210386699 | ||
| a7115be699 | |||
|
|
b86b763dfb | ||
|
|
7dddc1d706 | ||
|
|
2a6b3dc7e6 | ||
|
|
8d8f1c0294 | ||
|
|
77bf19566c | ||
|
|
beb40249a3 | ||
|
|
0fffd69071 | ||
|
|
1b9d89eb3a | ||
|
|
7d1f855f7e | ||
|
|
610d29f053 | ||
|
|
75eeae3933 | ||
|
|
9653592c16 | ||
|
|
353aa5cc78 | ||
|
|
4eda9c317d | ||
| 9817a3de59 | |||
|
|
e084b306e5 | ||
|
|
f485608108 | ||
|
|
9f076003e2 | ||
|
|
e1fcea6313 | ||
|
|
5e0cff1b92 | ||
|
|
603061fb86 | ||
|
|
21220f6d39 | ||
|
|
f25ad31741 | ||
|
|
af80cedd81 | ||
|
|
aabe66f5e2 | ||
|
|
ebbc3a46ae | ||
|
|
e00418537f | ||
|
|
dbb7b54d5d | ||
|
|
a80f65c6f2 | ||
| a9ff122ab2 | |||
|
|
225831ffcd | ||
|
|
a082b78f8e | ||
|
|
e1c6b7055a | ||
|
|
39bf0de949 | ||
|
|
29629e6786 | ||
|
|
e8caf2a57e | ||
|
|
e5c99f5b80 | ||
|
|
307fd8d527 | ||
|
|
31475f0312 | ||
|
|
0ca9b1d5c3 | ||
|
|
4949775c8b | ||
| 877ad18f34 | |||
|
|
df42d8f621 | ||
| 6a01f15261 | |||
|
|
cb04bd8c8d | ||
|
|
efc6b7ebb0 | ||
|
|
1008bca342 | ||
|
|
1f39b6bc2c | ||
|
|
aeee7ed771 | ||
|
|
15cdc97cae | ||
|
|
cc41adabb5 | ||
|
|
16db60f7bd | ||
|
|
e398272a24 | ||
|
|
e891e487cf | ||
|
|
dfef65f196 | ||
|
|
8faad2f407 | ||
|
|
f4ce6652b2 | ||
|
|
922849cd95 | ||
|
|
3a7a28e682 | ||
|
|
8b0f64db6b | ||
|
|
4728a87957 | ||
|
|
401a47fb43 |
2
.gitignore
vendored
2
.gitignore
vendored
@@ -1,6 +1,6 @@
|
||||
.superpowers/
|
||||
.worktrees/
|
||||
.claude/
|
||||
/target/
|
||||
/target
|
||||
**/*.rs.bk
|
||||
Cargo.lock.bak
|
||||
|
||||
@@ -27,7 +27,7 @@ cargo build --release # produces target/release/kebab
|
||||
|
||||
`-j 1` for the full workspace test isn't optional: 18 integration-test binaries each link `lance` + `datafusion` + `arrow` + `tantivy` and the parallel link step exhausts memory (linker gets SIGKILL'd, build silently fails partway). Per-crate runs are fine in parallel.
|
||||
|
||||
`target/` is 6–10 GB after a fresh build (DataFusion + Lance + fastembed + 18 × test-binary debug info). The dev/test profile is already trimmed (`debug = "line-tables-only"`, `split-debuginfo = "unpacked"` — see workspace `Cargo.toml`). Run `cargo clean` after phase merges if disk pressure shows up; backtraces still resolve to function + line.
|
||||
`target/` is 6–10 GB after a fresh build but **balloons to 90+ GB after a few task cycles** (each fb-* batch adds incremental compile artifacts on top of the existing 18 × test-binary debug info). The dev/test profile is already trimmed (`debug = "line-tables-only"`, `split-debuginfo = "unpacked"` — see workspace `Cargo.toml`). Run `cargo clean` **routinely after each merged PR**, not just "if pressure shows up" — disk space is tight and recovery via `cargo clean` is cheap (one re-link per crate on next build). Verified pattern: 92 GB → 0 GB in seconds, backtraces still resolve to function + line.
|
||||
|
||||
## The facade rule
|
||||
|
||||
|
||||
786
Cargo.lock
generated
786
Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
16
Cargo.toml
16
Cargo.toml
@@ -23,6 +23,7 @@ members = [
|
||||
"crates/kebab-parse-pdf",
|
||||
"crates/kebab-tui",
|
||||
"crates/kebab-mcp",
|
||||
"crates/kebab-parse-code",
|
||||
]
|
||||
|
||||
[workspace.package]
|
||||
@@ -30,7 +31,7 @@ edition = "2024"
|
||||
rust-version = "1.85"
|
||||
license = "MIT OR Apache-2.0"
|
||||
repository = "https://github.com/altair823/kebab"
|
||||
version = "0.3.3"
|
||||
version = "0.8.3"
|
||||
|
||||
[workspace.dependencies]
|
||||
anyhow = "1"
|
||||
@@ -80,6 +81,19 @@ rmcp = { version = "1.6", default-features = false, features = ["server"
|
||||
# a tokio runtime to host its mock server (the runtime adapter crate stays
|
||||
# sync via reqwest::blocking — wiremock is dev-only there).
|
||||
wiremock = "0.6"
|
||||
base64 = "0.22"
|
||||
# Pure-Rust git library for repo metadata detection (kebab-parse-code).
|
||||
# No `git` binary required. Default features include thread-safety + most
|
||||
# object-reading capabilities needed for HEAD name + commit SHA queries.
|
||||
gix = { version = "0.70", default-features = false, features = ["revision"] }
|
||||
# Rust source parsing for code ingest (kebab-parse-code, p10-1A-2). The
|
||||
# chunker stays tree-sitter-free — AST work is parser-side per design §6.3.
|
||||
tree-sitter = "0.26"
|
||||
tree-sitter-rust = "0.24"
|
||||
# Python / TS / JS grammars for code ingest (kebab-parse-code, p10-1B).
|
||||
tree-sitter-python = "0.25.0"
|
||||
tree-sitter-typescript = "0.23.2"
|
||||
tree-sitter-javascript = "0.25.0"
|
||||
|
||||
# Disk-footprint trim for dev / test builds. Codegen, opt-level, and
|
||||
# behavior are unchanged — only DWARF debug info is reduced (line
|
||||
|
||||
16
HANDOFF.md
16
HANDOFF.md
@@ -20,6 +20,7 @@ P0–P5 + P6 + P7 + P9-1/2/3/4 (Library / Search / Ask / Inspect) 머지 완료.
|
||||
| **P7** | PDF text + page citation | `kebab-parse-pdf` | P5 | ✅ 완료 (3/3 component, page-level chunker + ingest wiring) |
|
||||
| **P8** | 음성 transcription + timestamp citation | `kebab-parse-audio` | P5 | ⏸ 보류 (whisper-rs 시스템 dep brainstorm 필요) |
|
||||
| **P9** | TUI + desktop app | `kebab-tui`, `kebab-desktop` | P5 | 🟡 진행 (4/5 component — P9-1/2/3/4 완료 [Library / Search / Ask / Inspect], P9-5 desktop 예정 · 도그푸딩 피드백 **20/20 ✅**) |
|
||||
| **P10** | code ingest framework | `kebab-parse-code` | P5 | 🟡 진행 중 — 1A-1 ✅ (wire schema + parse-code skeleton + filter flags), 1A-2 ✅ (Rust AST chunker, tree-sitter-rust, `code-rust-ast-v1` — v0.7.0), **1B 🟡 PR 오픈** (Python `code-python-ast-v1` + TypeScript `code-ts-ast-v1` + JavaScript `code-js-ast-v1` — 3 언어 dogfooding 가능, v0.8.0 대기) |
|
||||
|
||||
P0~P5 직렬. P6~P9 P5 이후 병렬 가능.
|
||||
|
||||
@@ -31,6 +32,8 @@ P0~P5 직렬. P6~P9 P5 이후 병렬 가능.
|
||||
|
||||
머지 후 발견된 모든 deviation / hotfix 의 dated 로그는 [tasks/HOTFIXES.md](tasks/HOTFIXES.md). 본 요약은 \"누군가가 인수받을 때 알아두면 시간을 많이 절약하는\" 항목만:
|
||||
|
||||
- **2026-05-20 P10-1B (Rust 1A symbol path 비일관 + expression-level 함수 미방출)** — (a) Rust `code-rust-ast-v1` 은 file-scope nesting 만 (workspace path prefix 없음), 1B 의 Python/TypeScript/JavaScript 는 workspace 경로 → module path prefix 사용 (비일관 수용, retrofit = chunker_version bump + reindex 필요, 사용자 명시 요청까지 보류); (b) TS/JS 의 `const foo = () => {...}` 같은 expression-level 함수는 `<top-level>` glue 로 처리됨 (declaration-level 단위만 1B 1차 범위). 자세한 내용: `tasks/HOTFIXES.md` (2026-05-20) 두 항목.
|
||||
- **2026-05-19 P10-1A-2 (code_rust_ast_v1.rs + SourceType)** — `AST_CHUNK_MAX_LINES` 상수가 `IngestCodeCfg.ast_chunk_max_lines` 를 읽지 않고 모듈 상수 200 고정 (Chunker trait 이 per-medium config 미노출); `SourceType::Code` variant 부재로 code 파일이 `SourceType::Note` 로 분류됨 — 두 항목 모두 `tasks/HOTFIXES.md` (2026-05-19) 에 기록.
|
||||
- **2026-05-07 fb-26 (progress.rs)** — `Aborted` unconditional writeln (TTY duplicate) + `Completed` TTY no summary fixed; `KEBAB_PROGRESS=plain` env + quiet suppression added
|
||||
- **2026-05-07 fb-28 (main.rs)** — `--readonly` (KEBAB_READONLY) blocks Ingest/IngestFile/IngestStdin/Reset; `--quiet` suppresses progress stderr; error.v1 code: "readonly_mode"
|
||||
|
||||
@@ -86,14 +89,15 @@ P0~P5 직렬. P6~P9 P5 이후 병렬 가능.
|
||||
|
||||
P9-2/3/4 는 P9-1 의 parallel-safety contract (sub-state slot 패턴) 덕에 병렬 진행 가능 — 같은 `App` 손대지 않음.
|
||||
|
||||
### P9 dogfooding 백로그 (fb-26 ~ fb-42) — 4 minor release 분할
|
||||
### P9 dogfooding 백로그 (fb-26 ~ fb-42) — release 분할
|
||||
|
||||
2026-05-06 도그푸딩 누적 피드백 + "AI agent 가 kebab 을 쓰게 한다" 궁극 목표용 surface 확장. 17 항목 모두 **status: open + brainstorm 선행 필요**. 각 spec 상단 banner 명시. cascade 영향 / 분량 고려해 한 minor 에 묶지 않고 4 분할. 2026-05-06 renumber — **번호 = release 순서**:
|
||||
2026-05-06 도그푸딩 누적 피드백 + "AI agent 가 kebab 을 쓰게 한다" 궁극 목표용 surface 확장. cascade 영향 / 분량 고려해 한 minor 에 묶지 않고 분할.
|
||||
|
||||
- **0.3.0+ — agent foundation**: fb-26 (log), fb-27 (introspection/error wire) ✅ 머지 + v0.3.0 cut (2026-05-07), fb-28 (readonly/quiet), ~~fb-29 (daemon)~~ → 🚫 **deferred (2026-05-07 brainstorm)** — fb-30 stdio MCP 가 동일 가치 (agent integration + session 동안 hot cache) 를 daemon 복잡도 (PID file / port lock / loopback security / lifecycle UX) 없이 제공, single-user local-first 환경에 비대. fb-30 (MCP, stdio-only — fb-29 의존 제거 → depends_on `[p9-fb-27]` 만), fb-31 (single-file ingest). 후속 fb 들은 0.3.x patch / 0.4.0 minor 로 누적.
|
||||
- **0.4.0 — agent surface refinement (additive)**: fb-32 (stale), fb-33 (streaming), fb-34 (budget), fb-35 (verbatim fetch), fb-36 (filters), fb-37 (trace/stats).
|
||||
- **0.5.0 — RAG quality (cascade 동반)**: fb-38 (score semantics), fb-39 (precision tuning, embedding_version cascade + V00X), fb-40 (fact-grounded, prompt_template_version cascade).
|
||||
- **0.6.0 또는 P+**: fb-41 (multi-hop, XL), fb-42 (bulk/rerank, Nice).
|
||||
- **0.3.0 — agent foundation** ✅ cut 2026-05-07: fb-26 (log), fb-27 (introspection/error wire), fb-28 (readonly/quiet). ~~fb-29 (daemon)~~ → 🚫 **deferred** — fb-30 stdio MCP 가 동일 가치를 daemon 복잡도 없이 제공.
|
||||
- **0.4.0 — agent integration (MCP)** ✅ cut: fb-30 (MCP stdio), fb-31 (single-file/stdin ingest).
|
||||
- **0.5.0 — agent surface refinement (additive)** ✅ cut 2026-05-10: fb-32 (stale doc indicator), fb-33 (streaming ask), fb-34 (output budget controls), fb-35 (verbatim fetch), fb-36 (search filter args), fb-37 (trace + stats). 모두 wire schema additive minor.
|
||||
- **0.6.0 — RAG quality** 🟡 진행: fb-38 (score semantics) ✅ 머지 (2026-05-10), fb-40 (fact-grounded answer / rag-v2 prompt) ✅ 머지 (2026-05-10), fb-39 (retrieval precision tuning, embedding_version cascade) — 미진행 (eval golden set 선행 필요).
|
||||
- **0.7.0 또는 P+**: fb-41 (multi-hop reasoning, XL), fb-42 (bulk multi-query / rerank, Nice).
|
||||
|
||||
각 fb spec frontmatter 의 `target_version` 필드가 source of truth. INDEX.md 의 release subheader 도 동일 grouping.
|
||||
|
||||
|
||||
62
README.md
62
README.md
@@ -7,7 +7,7 @@
|
||||
- **Rust toolchain** ≥ 1.85 (workspace 가 edition 2024 + resolver 3 사용). [rustup](https://rustup.rs) 권장.
|
||||
- **Ollama** — `kebab ask` 와 이미지 OCR/caption 가 사용. `https://ollama.com/download` 에서 설치 후 `ollama serve` 실행. 기본 LLM 은 gemma4 계열 (`ollama pull gemma4:e4b`) — OCR / caption 도 같은 family 라 모델 하나만 pull 하면 됨. 더 큰 variant 원하면 `gemma4:26b` 등으로 config override. config 의 `[models.llm].endpoint` 에 host:port 명시.
|
||||
- **빌드 디스크** — 첫 빌드 시 `target/` 가 6–10 GB (Lance + DataFusion + fastembed). 여유 확인.
|
||||
- **fastembed 모델** — 첫 `kebab ingest` 시 `multilingual-e5-small` (~470 MB) 자동 다운로드.
|
||||
- **fastembed 모델** — 첫 `kebab ingest` 시 `multilingual-e5-large` (~1.3 GB, fb-39b) 자동 다운로드. `config.toml` 에서 `model = "multilingual-e5-small"` 로 명시하면 이전 모델 사용.
|
||||
|
||||
## 설치
|
||||
|
||||
@@ -42,7 +42,7 @@ cargo install --git https://gitea.altair823.xyz/altair823-org/kebab.git --bin ke
|
||||
# 첫 실행 — XDG 경로에 데이터 디렉토리 + config.toml 생성
|
||||
kebab init
|
||||
|
||||
# config 손보고 — workspace.root, 모델 endpoint 등 설정 (지원 형식은 md / png / jpg / pdf 로 고정)
|
||||
# config 손보고 — workspace.root, 모델 endpoint 등 설정 (지원 형식: md / png / jpg / pdf / rs / py / ts / js)
|
||||
${EDITOR:-vi} ~/.config/kebab/config.toml
|
||||
|
||||
# 색인 (Markdown / 이미지 / PDF 모두 한 번에)
|
||||
@@ -70,24 +70,51 @@ kebab doctor
|
||||
| 명령 | 동작 |
|
||||
|------|------|
|
||||
| `kebab init` | XDG 경로에 데이터 디렉토리 + config.toml 생성 |
|
||||
| `kebab ingest [<path>]` | Markdown / 이미지 / PDF 색인 (idempotent). TTY 에서는 stderr 진행 바, non-TTY (CI / pipe) 는 stderr 한 줄씩, `--json` 은 stdout 에 `ingest_progress.v1` 라인 streaming 후 마지막에 `ingest_report.v1`. Ctrl-C 한 번이면 현재 asset 마무리 후 abort (부분 commit 보존, idempotent re-run), 두 번째 Ctrl-C 는 hard exit. Markdown title 이 frontmatter 에 없어도 첫 H1 → H2 → 첫 paragraph 80 자 → 파일명 순으로 자동 채움 (parser_version `md-frontmatter-v2`) — 기존 색인된 doc 도 다음 ingest 에서 새 title 로 갱신. **Incremental** (p9-fb-23): 두 번째 이후의 ingest 는 변하지 않은 doc (blake3 + parser/chunker/embedder version 모두 동일) 의 parse/chunk/embed/vector upsert 를 자동 스킵. final summary 에 `N unchanged` 카운트 표시. `--force-reingest` 로 skip 무시 강제 재처리. **지원 형식** (extractor 자동 결정 — config 에 명시 불가): Markdown (`.md`), 이미지 (`.png` / `.jpg` / `.jpeg`, OCR + caption), PDF (`.pdf`). 다른 확장자는 자동 skip — `IngestItem.warnings` 에 사유 (`"unsupported media type: .docx"` 등), `IngestReport.skipped_by_extension` 에 카운트 분류, CLI / TUI summary 에 breakdown 표시. |
|
||||
| `kebab search --mode {lexical,vector,hybrid} "<query>" [--no-cache]` | 검색. hybrid는 RRF fusion, citation 포함. 같은 process 안에서 동일 query (NFKC + trim + lowercase 정규화) 반복 시 in-process LRU 캐시 hit (capacity = `[search] cache_capacity`, default 256). `--no-cache` 로 강제 bypass — 디버깅용. ingest commit 발생 시 `kv['corpus_revision']` bump 으로 모든 entry 자동 stale |
|
||||
| `kebab ingest [<path>]` | Markdown / 이미지 / PDF / Rust 소스코드 색인 (idempotent). TTY 에서는 stderr 진행 바, non-TTY (CI / pipe) 는 stderr 한 줄씩, `--json` 은 stdout 에 `ingest_progress.v1` 라인 streaming 후 마지막에 `ingest_report.v1`. Ctrl-C 한 번이면 현재 asset 마무리 후 abort (부분 commit 보존, idempotent re-run), 두 번째 Ctrl-C 는 hard exit. Markdown title 이 frontmatter 에 없어도 첫 H1 → H2 → 첫 paragraph 80 자 → 파일명 순으로 자동 채움 (parser_version `md-frontmatter-v2`) — 기존 색인된 doc 도 다음 ingest 에서 새 title 로 갱신. **Incremental** (p9-fb-23): 두 번째 이후의 ingest 는 변하지 않은 doc (blake3 + parser/chunker/embedder version 모두 동일) 의 parse/chunk/embed/vector upsert 를 자동 스킵. final summary 에 `N unchanged` 카운트 표시. `--force-reingest` 로 skip 무시 강제 재처리. **지원 형식** (extractor 자동 결정 — config 에 명시 불가): Markdown (`.md`), 이미지 (`.png` / `.jpg` / `.jpeg`, OCR + caption), PDF (`.pdf`), **소스코드** (`.rs` → `code-rust-ast-v1`, `.py` → `code-python-ast-v1`, `.ts`/`.tsx` → `code-ts-ast-v1`, `.js`/`.mjs`/`.cjs`/`.jsx` → `code-js-ast-v1` — 모두 tree-sitter AST chunker). 다른 확장자는 자동 skip — `IngestItem.warnings` 에 사유 (`"unsupported media type: .docx"` 등), `IngestReport.skipped_by_extension` 에 카운트 분류, CLI / TUI summary 에 breakdown 표시. 코드 chunk 는 `citation.kind = "code"` 에 `citation.lang = "<lang>"` + `symbol` + line range 를 담고, SearchHit top-level 에 `code_lang` + `repo` (`.git/` walk-up 의 디렉토리 이름) 가 backfill 됨. `--code-lang rust` / `--code-lang python` / `--code-lang typescript` / `--code-lang javascript` / `--media code` filter 로 언어별·코드 전용 검색 가능 (p10-1A-1 filter flags). Python symbol 은 workspace 경로 → dotted module path prefix (예: `kebab_eval.metrics.compute_mrr`), TS/JS symbol 은 slash-style module path prefix (예: `src/Foo.Foo.search`). |
|
||||
| `kebab search --mode {lexical,vector,hybrid} "<query>" [--no-cache] [--max-tokens N] [--snippet-chars N] [--cursor <opaque>] [--tag T] [--lang L] [--path-glob G] [--trust-min LEVEL] [--media TYPE] [--ingested-after RFC3339] [--doc-id ID] [--trace] [--bulk] [--repo NAME ...] [--code-lang LIST]` | 검색. hybrid는 RRF fusion, citation 포함. 같은 process 안에서 동일 query (NFKC + trim + lowercase 정규화) 반복 시 in-process LRU 캐시 hit (capacity = `[search] cache_capacity`, default 256). `--no-cache` 로 강제 bypass — 디버깅용. ingest commit 발생 시 `kv['corpus_revision']` bump 으로 모든 entry 자동 stale. **`--max-tokens` / `--snippet-chars` / `--cursor` (p9-fb-34)** — agent budget controls. `--json` 출력은 `search_response.v1` wrapper (`{hits, next_cursor, truncated}`) — pre-fb-34 의 bare array 와 호환 안 됨. mismatched cursor → `error.v1.code = stale_cursor`. **filter flags (p9-fb-36):** `--tag` 는 반복 가능 flag (`--tag rust --tag async`) 로 OR 매칭, `--media` 는 `,` 구분 다중 값 OR 매칭, 나머지 flags 간은 AND 조합. `--trust-min` 은 `primary\|secondary\|generated` 중 하나 (해당 level 이상 포함). `--ingested-after` 는 RFC3339 UTC — 파싱 실패 시 `error.v1.code = config_invalid` (exit 2). `--media md` 는 `markdown` alias 로 정규화. 알 수 없는 `--media` 값은 무조건 empty hits (오류 아님). **`--trace` (p9-fb-37)** — `search_response.v1.trace` 에 lexical / vector pre-fusion 후보 + RRF union + per-stage timing (`lexical_ms` / `vector_ms` / `fusion_ms` / `total_ms`) 노출. trace 요청은 캐시 우회 (`--no-cache` 없이도 항상 cold). **`--bulk` (p9-fb-42)** — stdin ndjson 으로 N query 한 번에 실행. `--json` 면 stdout per-query ndjson (`bulk_search_item.v1`) + stderr summary (`bulk_summary: total=N succeeded=S failed=F`). Cap 100. agent 가 query decomposition 후 sub-query 일괄 실행 시 single round-trip — App instance 재사용으로 캐시 / embedder cold-start 비용 한 번만. Per-query failure 는 item 의 `error` (error.v1) 에 격리, 다른 query 계속 진행. **code corpus filters (p10-1A-1):** `--repo` 는 반복 가능 (`--repo kebab --repo other`) OR 매칭. `--code-lang` 는 반복 또는 comma 다중 값 (`--code-lang rust,python`), 알 수 없는 값은 빈 hits. `--media code` 는 Tier 1/2/3 모든 code chunk 포함. 1A-1 시점에서는 indexed 된 code chunk 가 없어 filter 가 항상 빈 결과 — 1A-2 (Rust AST chunker) 머지 이후 실효. |
|
||||
| `kebab list docs` | 색인된 문서 목록 |
|
||||
| `kebab inspect doc <id>` / `kebab inspect chunk <id>` | raw record 보기 |
|
||||
| `kebab ask "<query>" [--show-citations / --hide-citations] [--session <id>]` | RAG 답변 + 근거 인용. 답변 후 `근거:` block 으로 full path / line range / score 한 줄씩 (default ON — `--hide-citations` 로 끄기, pipe 시 유용). 근거 부족 시 거절. Ollama 필요. `--session <id>` 로 multi-turn — 첫 호출에서 SQLite `chat_sessions` 에 자동 생성, 이후 호출은 prior turns 를 history 로 받아 follow-up. session id 는 사용자 지정 (e.g. `kb-rust-async-2026-05`) — `kebab reset --data-only` 로 모든 session wipe |
|
||||
| `kebab fetch chunk <id> [--context N]` / `kebab fetch doc <id> [--max-tokens N]` / `kebab fetch span <doc_id> <ls> <le> [--max-tokens N]` | (p9-fb-35) verbatim text fetch from indexed corpus. wire = `fetch_result.v1` (kind discriminator). chunk: target + ±N ordinal-context chunks. doc: full normalized markdown. span: 1-based line range (PDF/audio rejected as `error.v1.code = span_not_supported`). chars/4 budget on doc/span. |
|
||||
| `kebab ask "<query>" [--show-citations / --hide-citations] [--session <id>] [--stream]` | RAG 답변 + 근거 인용. 답변 후 `근거:` block 으로 full path / line range / score 한 줄씩 (default ON — `--hide-citations` 로 끄기, pipe 시 유용). 근거 부족 시 거절. Ollama 필요. `--session <id>` 로 multi-turn — 첫 호출에서 SQLite `chat_sessions` 에 자동 생성, 이후 호출은 prior turns 를 history 로 받아 follow-up. session id 는 사용자 지정 (e.g. `kb-rust-async-2026-05`) — `kebab reset --data-only` 로 모든 session wipe. **`--stream` (p9-fb-33)** 로 ndjson `answer_event.v1` event (retrieval_done → token* → final) 를 stderr 에 흘리고 stdout 마지막 줄에 기존 `answer.v1` — agent 가 token 즉시 소비 가능 |
|
||||
| `kebab doctor` | 설정/모델/DB 헬스 체크 |
|
||||
| `kebab tui` | Ratatui 셸 (Library + Search + Ask + Inspect 패널, desktop 진행 중). Library 에서 `r` 키로 background ingest 시작 — 화면 하단 status bar 가 진행 표시, 완료/abort 시 final 라인 잠시 유지 후 자동 hide. ingest 진행 중 `Esc` / `Ctrl-C` 가 cancel signal (그 외에는 quit). vim-style mode (header 우측 `-- NORMAL --` / `-- INSERT --`) — Library/Inspect 는 자동 NORMAL, Search/Ask 는 자동 INSERT. `i` 로 Normal→Insert (모든 pane — p9-fb-21), `Esc` 로 Insert→Normal 어디서나. mode-authoritative dispatch — Search 의 `j/k/o/g`, Ask 의 `e/j/k` 는 NORMAL 모드에서만 명령으로 동작, INSERT 에서는 입력 문자로 typing. (Search 의 chunk inspect 키는 `i`→`o` 로 rebind — `i` 가 universal Insert toggle.) **`F1` 로 cheatsheet popup** (현재 pane 의 키 매핑 + global 토글 표) — `Esc` / `F1` 로 닫기. Search 패널은 200ms debounce 후 background worker 가 검색 — 키 입력으로 UI freeze 안 됨, 사용자가 계속 타이핑하면 stale 결과 자동 폐기 (generation counter). Ask 패널은 multi-turn — 같은 conversation 안에서 Q1/A1, Q2/A2 transcript 누적, 다음 질문이 이전 턴을 history 로 받아 답변. 답변 본문은 markdown 렌더 (bold/italic/inline code/heading/list/code fence/table/blockquote, raw `**bold**` 가 실제 굵게 표시). `Ctrl-L` 로 새 conversation 시작. Search 의 `g` 키가 `$EDITOR` (기본 `vi`) 로 hit 의 citation 위치 열기 — 종료 후 TUI 화면이 자동으로 깨끗이 redraw. CLI `kebab ask` 는 raw markdown 그대로 (terminal 호환성 위해). Library 의 doc-list 가 한글 / 일본어 / 중국어 (CJK) 제목을 wide-char 정확한 column width 로 truncate — 한글 제목이 한 줄을 넘기지 않음 (CJK 1 자 = 2 col). Search/Ask/Filter 입력의 cursor 가 wide char 위에서 column 단위로 정렬 — 한글 입력 시 caret 이 글자 옆에 정확히 놓임. `← / →` 로 입력 문자열 중간 cursor 이동 (한글 한 글자 = 2 column 이라도 한 번에 이동), `Home / End` 로 양 끝 점프, `Delete` 로 cursor 위치 char 삭제 — 모든 input pane (Ask / Search / Library filter overlay) 동일 (p9-fb-22). Ask 트랜스크립트는 새 답변이 viewport 아래로 누적될 때 자동으로 tail 을 따라감 (auto-scroll); `j` / `k` 로 위로 스크롤하면 freeze, `Shift-G` 로 다시 bottom + auto-tail 재개. 화면 하단 hint line 은 한국어 동사구로 (`"위로"` / `"아래로"` / `"필터"` / `"타이핑 검색어"` / `"Esc 로 NORMAL 모드"` / `"i 입력모드"` 등) + 현재 (pane, mode) 조합에 맞춰 자동 분기, **첫 fragment 가 항상 `F1 도움말`** (cheatsheet 발견성 보장). 모든 모드에서 항상 떠 있는 상태바 — `kebab v<version> │ <pane> │ <docs> docs │ <state>` (state: streaming/searching/indexing/idle, ingest 진행 중에는 progress 가 같은 자리에 흡수됨). Ask 진입 시 conversation id 8 자 prefix 도 함께 표시. Ask 트랜스크립트와 Inspect 양쪽에서 `PgUp / PgDn` 으로 10 줄씩 페이지 스크롤. Library 의 doc list 위에는 `TITLE / TAGS / UPDATED / CHUNKS` 컬럼 헤더 행 표시 (display-width 정렬, Hangul / CJK 안전). |
|
||||
| `kebab reset [--all / --data-only / --vector-only / --config-only] [--yes]` | XDG 데이터 wipe. **Irreversible.** TTY 면 confirm prompt, 아니면 `--yes` 필수. `--vector-only` 는 SQLite `embedding_records` 도 함께 truncate (orphan 방지) |
|
||||
| `kebab eval run / compare` | golden query 회귀 측정 |
|
||||
| `kebab schema [--json]` | introspection — wire schemas / capabilities / models / stats 한 번에. `--json` 은 `schema.v1` wire; 사람 모드는 서식 출력. |
|
||||
| `kebab schema [--json]` | introspection — wire schemas / capabilities / models / stats 한 번에. `--json` 은 `schema.v1` wire; 사람 모드는 서식 출력. **stats 에 (p9-fb-37) `media_breakdown` (5 keys: markdown / pdf / image / audio / other) + `lang_breakdown` (BCP-47 코드, NULL 은 literal `"null"`) + `index_bytes` (sqlite + lancedb on-disk 합계) + `stale_doc_count` (`config.search.stale_threshold_days` 초과 doc 수) 추가.** |
|
||||
| `kebab ingest-file <path>` | 단일 파일 ingest (workspace 외부 가능). 바이트는 `<workspace.root>/_external/<hash12>.<ext>` 로 copy. `.kebabignore` 매치 시 stderr warn 후 진행 (explicit ingest 가 bypass intent). |
|
||||
| `kebab ingest-stdin --title <T> [--source-uri <URI>]` | stdin 의 markdown 본문 ingest. frontmatter (title + source_uri) 자동 prepend. v1 markdown only. |
|
||||
| `kebab mcp` | MCP (Model Context Protocol) stdio server. agent host (Claude Code / Cursor / OpenAI Agents) 가 spawn 하여 tool 호출 (`search` / `ask` / `schema` / `doctor` / `ingest_file` / `ingest_stdin`). `--config` honor. |
|
||||
| `kebab mcp` | MCP (Model Context Protocol) stdio server. agent host (Claude Code / Cursor / OpenAI Agents) 가 spawn 하여 tool 호출 (`search` / `bulk_search` / `ask` / `fetch` / `schema` / `doctor` / `ingest_file` / `ingest_stdin`). `--config` honor. |
|
||||
|
||||
모든 명령에 `--json` 플래그. 출력은 frozen wire schema v1 (`schema_version` 항상 포함, 예: `ingest_report.v1`, `ingest_progress.v1`, `search_hit.v1`, `answer.v1`, `doctor.v1`, `reset_report.v1`, `schema.v1`). `--json` 모드에서 fatal error 는 stderr 에 `error.v1` ndjson 으로 emit (exit code 0/1/2/3 unchanged).
|
||||
|
||||
글로벌 플래그: `--readonly` (또는 `KEBAB_READONLY=1`) — 모든 write-path 명령 (`ingest` / `ingest-file` / `ingest-stdin` / `reset`) 을 비활성화, exit 1. `--quiet` — 진행 바 / hint 등 human-readable stderr 억제 (exit code / stdout 출력은 그대로). `KEBAB_PROGRESS=plain` — TTY 가 없는 환경에서도 진행 상황을 plain-text 한 줄씩 stderr 로 출력 (spinner 대신).
|
||||
|
||||
### Score 해석 (fb-38)
|
||||
|
||||
`search_hit.v1.score` 는 **ranking signal** 이지 confidence 가 아니다. `score_kind` 필드로 의미 선언:
|
||||
|
||||
| `score_kind` | 의미 | 범위 |
|
||||
|--------------|------|------|
|
||||
| `rrf` (hybrid) | RRF normalized | `[0, 1]`, ceiling = 1.0 (양 채널 rank=1) |
|
||||
| `bm25` (lexical) | raw BM25 | unbounded (≥ 0) |
|
||||
| `cosine` (vector) | cosine sim | `[-1, 1]` |
|
||||
|
||||
#### RRF 수식 (hybrid mode)
|
||||
|
||||
```
|
||||
chunk c 의 raw RRF = Σ_m 1 / (k_rrf + rank_m(c))
|
||||
|
||||
여기서 m ∈ {lexical, vector}, k_rrf = config.search.rrf_k (default 60).
|
||||
양 채널 모두 rank=1 일 때 raw RRF = 2 / (k_rrf + 1) ≈ 0.0328.
|
||||
|
||||
normalize: rrf_score = raw_rrf / (2 / (k_rrf + 1))
|
||||
→ rrf_score ∈ [0, 1]. 양쪽 rank=1 → 1.0, 한 쪽만 등장 → ≈ 0.5 천장.
|
||||
```
|
||||
|
||||
`rrf_score = 0.5` 의 의미: chunk 가 한 채널 (lexical 또는 vector) 에서만 rank 1 로 등장. confidence 50% 가 아님 — RRF 수식의 산술적 천장.
|
||||
|
||||
agent 가 trust threshold 가 필요하면 top-level `score` 가 아닌 nested `retrieval.lexical_score` (BM25 raw) / `retrieval.vector_score` (cosine raw) 사용.
|
||||
|
||||
## 논리 아키텍처
|
||||
|
||||
```mermaid
|
||||
@@ -104,9 +131,9 @@ flowchart TB
|
||||
end
|
||||
|
||||
subgraph Pipeline["도메인 + 파이프라인"]
|
||||
parse["parse-md / parse-pdf / parse-image"]
|
||||
chunker["chunker (md-heading-v1, pdf-page-v1)"]
|
||||
embedder["embedder (fastembed multilingual-e5-small)"]
|
||||
parse["parse-md / parse-pdf / parse-image / parse-code"]
|
||||
chunker["chunker (md-heading-v1, pdf-page-v1, code-rust-ast-v1, code-python-ast-v1, code-ts-ast-v1, code-js-ast-v1)"]
|
||||
embedder["embedder (fastembed multilingual-e5-large)"]
|
||||
retriever["retriever (lexical / vector / hybrid RRF)"]
|
||||
rag["RAG pipeline"]
|
||||
end
|
||||
@@ -151,7 +178,18 @@ flowchart TB
|
||||
|
||||
## Configuration
|
||||
|
||||
- `~/.config/kebab/config.toml` — `kebab init` 가 XDG 경로에 생성. `[workspace]` (root, exclude — include 필드는 제거됨, 지원 형식은 자동 결정), `[storage]`, `[chunking]`, `[models.embedding]`, `[models.llm]`, `[image.ocr]`, `[image.caption]`, `[search]`, `[rag]`, `[ui]` 절. `[ui] theme = "dark" | "light"` 로 TUI 팔레트 선택 (default `"dark"`, 알 수 없는 값은 dark fallback). 옛 config 의 `workspace.include = [...]` 은 silently 무시 + 단발 deprecation warning (p9-fb-25).
|
||||
- `~/.config/kebab/config.toml` — `kebab init` 가 XDG 경로에 생성. `[workspace]` (root, exclude — include 필드는 제거됨, 지원 형식은 자동 결정), `[storage]`, `[chunking]`, `[models.embedding]`, `[models.llm]`, `[image.ocr]`, `[image.caption]`, `[search]`, `[rag]`, `[ui]` 절.
|
||||
- `[models.embedding]` —
|
||||
- `model` (default `"multilingual-e5-large"`, fb-39b) — 다국어 sentence embedding 모델. 1024-dim. ONNX (~1.3 GB) 첫 실행 시 fastembed cache (`config.storage.model_dir/fastembed/`) 에 자동 다운로드. `"multilingual-e5-small"` (384 dim) 는 backwards-compat 으로 사용 가능 — TOML 에 명시.
|
||||
- `dimensions` (default `1024`) — 모델의 embedding 차원. config 와 LanceDB stored dim 불일치 시 검색 결과 0 건 (orphan table). 모델 변경 시 `kebab reset --vector-only && kebab ingest` 로 vector index 재구축 권장.
|
||||
- `[ui] theme = "dark" | "light"` 로 TUI 팔레트 선택 (default `"dark"`, 알 수 없는 값은 dark fallback).
|
||||
- `[search] stale_threshold_days = 30` (p9-fb-32) — search hit / RAG citation 의 `stale` 플래그 기준 (default 30 일, `0` 으로 비활성화). 옛 config 의 `workspace.include = [...]` 은 silently 무시 + 단발 deprecation warning (p9-fb-25).
|
||||
- `[ingest.code]` (p10-1A-1) — code ingest 의 skip 정책 + chunker 기본값.
|
||||
- `skip_generated_header = true` — 첫 ~512 byte 의 generated marker (`@generated` / `DO NOT EDIT` 등) 감지 시 skip.
|
||||
- `max_file_bytes = 262144` (256 KiB) / `max_file_lines = 5000` — 파일당 cap, 초과 시 skip.
|
||||
- `extra_skip_globs = []` — 사용자 추가 skip 패턴 (`.gitignore` 문법).
|
||||
- `.gitignore` honor: 자동 적용. `.kebabignore` 는 추가 layer. 우선순위: built-in safety net (`node_modules/` / `target/` / `__pycache__/` / `.venv/` / `venv/` / `env/`) > `.gitignore` > `.kebabignore`.
|
||||
- `[rag] prompt_template_version` (default `"rag-v2"`) — RAG system prompt version. `"rag-v1"` 은 legacy backwards-compat (사용자 명시 시 유지). v2 강화 규칙: (1) fact 인용 시 [#번호] 앞에 chunk 속 원문 큰따옴표 표기, (2) 학습 지식 동원 금지, (3) 근거 모호 시 "확실하지 않다" 명시.
|
||||
- `--config <path>` flag — 임시 워크스페이스 / 격리 테스트 시 사용. CLI / TUI 모두 honor.
|
||||
- `KEBAB_*` env — 일부 키 override (`KEBAB_RAG_SCORE_GATE`, `KEBAB_EVAL_GOLDEN`, `KEBAB_COMMIT_HASH` 등).
|
||||
- XDG layout: `~/.config/kebab/`, `~/.local/share/kebab/`, `~/.cache/kebab/`, `~/.local/state/kebab/`.
|
||||
@@ -170,7 +208,7 @@ config 예시는 [docs/SMOKE.md](docs/SMOKE.md) 의 `/tmp/kebab-smoke/config.tom
|
||||
|
||||
## MCP 사용
|
||||
|
||||
`kebab mcp` 가 stdio MCP server. 6 tool: `search` / `ask` / `schema` / `doctor` / `ingest_file` / `ingest_stdin`.
|
||||
`kebab mcp` 가 stdio MCP server. 8 tool: `search` / `bulk_search` (p9-fb-42 — N query 한 번에) / `ask` / `fetch` (p9-fb-35) / `schema` / `doctor` / `ingest_file` / `ingest_stdin`.
|
||||
|
||||
Claude Code 빠른 등록 (`~/.claude/mcp.json` 또는 host 동등 위치):
|
||||
|
||||
|
||||
@@ -32,6 +32,10 @@ kebab-parse-image = { path = "../kebab-parse-image" }
|
||||
# per-asset dispatch (see `ingest_one_asset` PDF branch) and runs the
|
||||
# resulting `CanonicalDocument` through `kebab-chunk::PdfPageV1Chunker`.
|
||||
kebab-parse-pdf = { path = "../kebab-parse-pdf" }
|
||||
# p10-1A-2: Rust AST extractor lives here. App threads it into the
|
||||
# per-asset dispatch (see `ingest_one_asset` Code branch) and runs the
|
||||
# resulting `CanonicalDocument` through `kebab-chunk::CodeRustAstV1Chunker`.
|
||||
kebab-parse-code = { path = "../kebab-parse-code" }
|
||||
anyhow = { workspace = true }
|
||||
blake3 = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
@@ -52,6 +56,8 @@ unicode-normalization = "0.1"
|
||||
# p9-fb-31: GitignoreBuilder for .kebabignore matching in ingest_file_with_config.
|
||||
# Same version as kebab-source-fs (0.4) to avoid duplicate dep versions.
|
||||
ignore = "0.4"
|
||||
# p9-fb-34: opaque pagination cursor encodes payload as base64.
|
||||
base64 = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
rusqlite = { workspace = true }
|
||||
|
||||
@@ -40,8 +40,8 @@ use anyhow::{Context, Result, anyhow};
|
||||
use lru::LruCache;
|
||||
|
||||
use kebab_core::{
|
||||
Answer, Embedder, IndexVersion, LanguageModel, Retriever, SearchHit, SearchMode,
|
||||
SearchQuery, VectorStore,
|
||||
Answer, DocumentStore, Embedder, IndexVersion, LanguageModel, Retriever, SearchHit,
|
||||
SearchMode, SearchOpts, SearchQuery, VectorStore,
|
||||
};
|
||||
use kebab_embed_local::FastembedEmbedder;
|
||||
use kebab_llm_local::OllamaLanguageModel;
|
||||
@@ -50,6 +50,31 @@ use kebab_search::{HybridRetriever, LexicalRetriever, VectorRetriever};
|
||||
use kebab_store_sqlite::SqliteStore;
|
||||
use kebab_store_vector::LanceVectorStore;
|
||||
|
||||
/// p9-fb-34: top-level wrapper around a paginated, budget-limited
|
||||
/// search result. Mirrors the wire `search_response.v1` shape.
|
||||
///
|
||||
/// `next_cursor` is non-null whenever more hits may be reachable —
|
||||
/// either the retriever filled the page (more behind it), or the
|
||||
/// budget loop popped hits (those popped hits remain fetchable
|
||||
/// from `offset + returned`). It is null only when the retriever
|
||||
/// returned fewer hits than requested AND nothing was popped — i.e.
|
||||
/// the corpus has nothing more for this query.
|
||||
///
|
||||
/// `truncated` is independent of `next_cursor`: it signals that
|
||||
/// the budget loop modified the page (snippet shorten or k pop).
|
||||
/// Caller may either widen `max_tokens` (and re-issue the same
|
||||
/// query) or follow `next_cursor` (to advance through more hits)
|
||||
/// or both.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct SearchResponse {
|
||||
pub hits: Vec<SearchHit>,
|
||||
pub next_cursor: Option<String>,
|
||||
pub truncated: bool,
|
||||
/// p9-fb-37: present when caller passed `SearchOpts.trace = true`.
|
||||
/// Consumers that ignore trace should leave this `None`.
|
||||
pub trace: Option<kebab_core::SearchTrace>,
|
||||
}
|
||||
|
||||
/// Facade state — see module docs for lifetime rules.
|
||||
///
|
||||
/// The struct is public so long-lived callers (kb-eval, the future P9
|
||||
@@ -190,7 +215,21 @@ impl App {
|
||||
corpus_revision = key.corpus_revision,
|
||||
"search served from LRU cache"
|
||||
);
|
||||
return Ok(hits.clone());
|
||||
// p9-fb-32: re-stamp staleness on every cache hit. The cache
|
||||
// entry was stamped at insert time against an older `now`
|
||||
// and an older threshold; if either has shifted (config
|
||||
// reload, time passing) the cached `stale: false` may now
|
||||
// be wrong. Re-stamping is cheap (per-hit comparison) and
|
||||
// avoids invalidating the cache on threshold changes.
|
||||
let mut hits = hits.clone();
|
||||
drop(guard);
|
||||
let now = time::OffsetDateTime::now_utc();
|
||||
crate::staleness::mark_stale_in_place(
|
||||
&mut hits,
|
||||
now,
|
||||
self.config.search.stale_threshold_days,
|
||||
);
|
||||
return Ok(hits);
|
||||
}
|
||||
// Drop the lock before the (potentially slow) retriever call
|
||||
// so other in-flight searches can use the cache concurrently.
|
||||
@@ -205,14 +244,14 @@ impl App {
|
||||
/// Used by `--no-cache` CLI invocations and by `search` itself
|
||||
/// on cache miss. Identical behavior to the pre-fb-19 `search`.
|
||||
pub fn search_uncached(&self, query: SearchQuery) -> Result<Vec<SearchHit>> {
|
||||
match query.mode {
|
||||
let mut hits = match query.mode {
|
||||
SearchMode::Lexical => {
|
||||
let lex = LexicalRetriever::with_settings(
|
||||
self.sqlite.clone(),
|
||||
lexical_index_version(&self.config),
|
||||
self.config.search.snippet_chars,
|
||||
);
|
||||
lex.search(&query)
|
||||
lex.search(&query)?
|
||||
}
|
||||
SearchMode::Vector => {
|
||||
let (emb, vec_store) = self.require_embeddings()?;
|
||||
@@ -226,7 +265,7 @@ impl App {
|
||||
vec_iv,
|
||||
self.config.search.snippet_chars,
|
||||
);
|
||||
retr.search(&query)
|
||||
retr.search(&query)?
|
||||
}
|
||||
SearchMode::Hybrid => {
|
||||
let lex = Arc::new(LexicalRetriever::with_settings(
|
||||
@@ -246,9 +285,232 @@ impl App {
|
||||
self.config.search.snippet_chars,
|
||||
)) as Arc<dyn Retriever>;
|
||||
let hybrid = HybridRetriever::new(&self.config, lex, vec_retr);
|
||||
hybrid.search(&query)
|
||||
hybrid.search(&query)?
|
||||
}
|
||||
};
|
||||
// p9-fb-32: stamp staleness against the freshest possible `now`
|
||||
// and the current threshold. Cheap (per-hit comparison).
|
||||
let now = time::OffsetDateTime::now_utc();
|
||||
crate::staleness::mark_stale_in_place(
|
||||
&mut hits,
|
||||
now,
|
||||
self.config.search.stale_threshold_days,
|
||||
);
|
||||
// p10-1A-2: backfill `code_lang` from the Citation::Code `lang`
|
||||
// field. The search layer (kebab-search) constructs SearchHit with
|
||||
// `code_lang: None`; we own the post-processing here in kebab-app
|
||||
// and can fill it cheaply from data already present in the hit.
|
||||
backfill_code_lang(&mut hits);
|
||||
// p10-1A-2 Task 8b: backfill `repo` from the document's
|
||||
// `Metadata.repo`. Unlike `code_lang`, this cannot be derived from
|
||||
// the Citation alone — it requires a store lookup by `doc_id`.
|
||||
self.backfill_repo(&mut hits);
|
||||
Ok(hits)
|
||||
}
|
||||
|
||||
/// p9-fb-34: budget-aware search facade. Returns hits trimmed to
|
||||
/// `opts.max_tokens` (chars/4 approximation) plus pagination
|
||||
/// metadata. `App::search` is now a thin wrapper that drops the
|
||||
/// metadata for backwards compat.
|
||||
///
|
||||
/// `SearchResponse.next_cursor` and `truncated` are independent
|
||||
/// signals — see `SearchResponse` doc for details.
|
||||
pub fn search_with_opts(
|
||||
&self,
|
||||
query: SearchQuery,
|
||||
opts: SearchOpts,
|
||||
) -> Result<SearchResponse> {
|
||||
use crate::cursor;
|
||||
|
||||
let corpus_revision = self.sqlite.corpus_revision().to_string();
|
||||
let offset = match opts.cursor.as_ref() {
|
||||
// p9-fb-34: wrap the typed ErrorV1 in StructuredError so
|
||||
// anyhow carries the structured payload all the way to
|
||||
// `classify` — string formatting here would degrade
|
||||
// `code = "stale_cursor"` to `code = "generic"` on the wire.
|
||||
Some(c) => cursor::decode(c, &corpus_revision)
|
||||
.map_err(|e| anyhow::Error::new(crate::error_wire::StructuredError(e)))?,
|
||||
None => 0,
|
||||
};
|
||||
|
||||
let snippet_chars = opts
|
||||
.snippet_chars
|
||||
.unwrap_or(self.config.search.snippet_chars);
|
||||
|
||||
// Fetch enough to satisfy offset + the requested page. The
|
||||
// retriever returns at most `fetch_k` hits — we then drop
|
||||
// `offset` and keep the next `k_effective`. `k = 0` is
|
||||
// treated as "use config default" so a caller passing through
|
||||
// a default-constructed `SearchQuery` still gets useful work
|
||||
// out of the budget facade.
|
||||
let k_effective = if query.k == 0 {
|
||||
self.config.search.default_k
|
||||
} else {
|
||||
query.k
|
||||
};
|
||||
let fetch_k = offset.saturating_add(k_effective);
|
||||
let fetch_query = SearchQuery {
|
||||
k: fetch_k,
|
||||
..query.clone()
|
||||
};
|
||||
|
||||
// p9-fb-37: when --trace is requested, bypass the LRU cache and
|
||||
// run through `HybridRetriever::search_with_trace`, which
|
||||
// dispatches by mode internally. Vector / hybrid modes require
|
||||
// embeddings (same as `--mode hybrid`); lexical mode skips
|
||||
// embedder construction via `NoopRetriever` so lexical-only
|
||||
// workspaces (provider = "none") can use `--trace` without
|
||||
// surfacing the "switch to --mode lexical" error.
|
||||
if opts.trace {
|
||||
let lex = Arc::new(LexicalRetriever::with_settings(
|
||||
self.sqlite.clone(),
|
||||
lexical_index_version(&self.config),
|
||||
self.config.search.snippet_chars,
|
||||
)) as Arc<dyn Retriever>;
|
||||
let vec_retr: Arc<dyn Retriever> = if matches!(query.mode, SearchMode::Lexical) {
|
||||
// `HybridRetriever::search_with_trace` never invokes the
|
||||
// vector retriever for `SearchMode::Lexical` (Task 4).
|
||||
// A no-op stand-in lets us avoid the ~470 MB embedder
|
||||
// load when the user only asked for lexical trace.
|
||||
Arc::new(NoopRetriever)
|
||||
} else {
|
||||
let (emb, vec_store) = self.require_embeddings()?;
|
||||
let vec_iv = vector_index_version(emb.as_ref());
|
||||
let vec_dyn: Arc<dyn VectorStore + Send + Sync> = vec_store;
|
||||
let emb_dyn: Arc<dyn Embedder> = emb;
|
||||
Arc::new(VectorRetriever::with_settings(
|
||||
vec_dyn,
|
||||
emb_dyn,
|
||||
self.sqlite.clone(),
|
||||
vec_iv,
|
||||
self.config.search.snippet_chars,
|
||||
)) as Arc<dyn Retriever>
|
||||
};
|
||||
let hybrid = HybridRetriever::new(&self.config, lex, vec_retr);
|
||||
let (mut traced_hits, trace) = hybrid.search_with_trace(&fetch_query)?;
|
||||
|
||||
// Stamp staleness — same as search_uncached.
|
||||
let now = time::OffsetDateTime::now_utc();
|
||||
crate::staleness::mark_stale_in_place(
|
||||
&mut traced_hits,
|
||||
now,
|
||||
self.config.search.stale_threshold_days,
|
||||
);
|
||||
// p10-1A-2: backfill code_lang — same as search_uncached.
|
||||
backfill_code_lang(&mut traced_hits);
|
||||
// p10-1A-2 Task 8b: backfill repo — same as search_uncached.
|
||||
self.backfill_repo(&mut traced_hits);
|
||||
|
||||
// Apply offset + k_effective truncation (mirrors non-trace path).
|
||||
let drop_n = offset.min(traced_hits.len());
|
||||
traced_hits.drain(..drop_n);
|
||||
let mut hits: Vec<SearchHit> =
|
||||
traced_hits.into_iter().take(k_effective).collect();
|
||||
|
||||
// Snippet truncation if opts.snippet_chars set (mirror non-trace path).
|
||||
if opts.snippet_chars.is_some() {
|
||||
for h in hits.iter_mut() {
|
||||
if h.snippet.chars().count() > snippet_chars {
|
||||
h.snippet = trim_to_chars(&h.snippet, snippet_chars);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Trace path skips the budget loop. Caller will inspect
|
||||
// `hits.len()` and `trace.timing` rather than paginate.
|
||||
return Ok(SearchResponse {
|
||||
hits,
|
||||
next_cursor: None,
|
||||
truncated: false,
|
||||
trace: Some(trace),
|
||||
});
|
||||
}
|
||||
|
||||
// backfill_code_lang + backfill_repo are applied inside `search`
|
||||
// via `search_uncached` — no explicit call needed here. Trace
|
||||
// branch above calls them directly because it bypasses `search`.
|
||||
let mut all_hits = self.search(fetch_query)?;
|
||||
|
||||
// Skip offset.
|
||||
let drop_n = offset.min(all_hits.len());
|
||||
all_hits.drain(..drop_n);
|
||||
let mut hits: Vec<SearchHit> =
|
||||
all_hits.into_iter().take(k_effective).collect();
|
||||
|
||||
// Apply snippet_chars override if shorter than what the
|
||||
// retriever returned (retriever already honored
|
||||
// `config.search.snippet_chars`; this only kicks in when the
|
||||
// caller asked for *less*).
|
||||
if opts.snippet_chars.is_some() {
|
||||
for h in hits.iter_mut() {
|
||||
if h.snippet.chars().count() > snippet_chars {
|
||||
h.snippet = trim_to_chars(&h.snippet, snippet_chars);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Budget loop.
|
||||
let mut truncated = false;
|
||||
if let Some(max_tokens) = opts.max_tokens {
|
||||
let max_chars = max_tokens.saturating_mul(4);
|
||||
// Step 1: shorten snippets progressively to a 60-char floor.
|
||||
const SNIPPET_FLOOR: usize = 60;
|
||||
let mut current_snippet_cap = snippet_chars;
|
||||
while estimate_chars(&hits) > max_chars
|
||||
&& current_snippet_cap > SNIPPET_FLOOR
|
||||
{
|
||||
current_snippet_cap =
|
||||
(current_snippet_cap / 2).max(SNIPPET_FLOOR);
|
||||
for h in hits.iter_mut() {
|
||||
if h.snippet.chars().count() > current_snippet_cap {
|
||||
h.snippet =
|
||||
trim_to_chars(&h.snippet, current_snippet_cap);
|
||||
truncated = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Step 2: pop hits from the end until we fit, but always
|
||||
// keep ≥ 1.
|
||||
while estimate_chars(&hits) > max_chars && hits.len() > 1 {
|
||||
hits.pop();
|
||||
truncated = true;
|
||||
}
|
||||
}
|
||||
|
||||
// p9-fb-34: emit cursor whenever more hits may be reachable.
|
||||
// Three cases produce a non-null cursor:
|
||||
// (a) returned == k_effective: retriever filled the page; there
|
||||
// may be more behind it. Speculative — next call may return
|
||||
// an empty page if nothing remains.
|
||||
// (b) truncated by k-pop: returned < k_effective because we
|
||||
// popped hits to fit the budget. Those popped hits live at
|
||||
// offset+returned..; next call (with same or wider budget)
|
||||
// resumes from there.
|
||||
// (c) truncated by snippet-only shrink: returned == k_effective,
|
||||
// falls under (a). Cursor lets caller paginate; widening
|
||||
// --max-tokens lets caller re-fetch fuller snippets at the
|
||||
// same offset.
|
||||
//
|
||||
// No cursor when neither (a) nor (b) applies — i.e. the retriever
|
||||
// returned fewer than k_effective AND we didn't pop. That means
|
||||
// end of available results.
|
||||
let returned = hits.len();
|
||||
let next_cursor = if returned == k_effective || truncated {
|
||||
if offset.saturating_add(returned) > 0 {
|
||||
Some(cursor::encode(offset + returned, &corpus_revision))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
Ok(SearchResponse {
|
||||
hits,
|
||||
next_cursor,
|
||||
truncated,
|
||||
trace: None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Run a RAG `ask` against the configured retriever + LLM. Reuses
|
||||
@@ -531,6 +793,58 @@ impl App {
|
||||
}
|
||||
}
|
||||
|
||||
/// p10-1A-2 Task 8b: back-fill `SearchHit.repo` from the originating
|
||||
/// document's `Metadata.repo` for every hit whose `repo` field is
|
||||
/// currently `None`. The search layer (kebab-search) constructs hits
|
||||
/// with `repo: None` because it has no store access; we fill it here
|
||||
/// in kebab-app post-retrieval via a per-distinct-`doc_id` store lookup.
|
||||
///
|
||||
/// Deduplication: a small `HashMap` accumulates the
|
||||
/// `(doc_id → Option<String>)` mapping so each unique document is
|
||||
/// fetched at most once. Search result sets are small (default k ≤ 20),
|
||||
/// so the map overhead is negligible. A `None` entry is cached too
|
||||
/// (document not found or no repo in metadata) to avoid re-querying.
|
||||
///
|
||||
/// Non-repo documents (markdown, PDF, plain text, code files outside a
|
||||
/// git tree) correctly keep `repo: None` — `Metadata.repo` is already
|
||||
/// `None` for those, so the assignment is a no-op.
|
||||
fn backfill_repo(&self, hits: &mut [SearchHit]) {
|
||||
use std::collections::HashMap;
|
||||
use kebab_core::DocumentId;
|
||||
|
||||
// doc_id → Option<String> where None means "not found / no repo"
|
||||
let mut cache: HashMap<DocumentId, Option<String>> = HashMap::new();
|
||||
|
||||
for hit in hits.iter_mut() {
|
||||
if hit.repo.is_some() {
|
||||
continue;
|
||||
}
|
||||
let repo_val = cache
|
||||
.entry(hit.doc_id.clone())
|
||||
.or_insert_with(|| {
|
||||
// Deliberately non-aborting: a failed store lookup for
|
||||
// one hit must not abort the whole search response. Log
|
||||
// the error so it's observable rather than silently
|
||||
// dropped (review #140 round 1).
|
||||
match self.sqlite.get_document(&hit.doc_id) {
|
||||
Ok(opt) => opt.and_then(|doc| doc.metadata.repo),
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
target: "kebab-app",
|
||||
doc_id = %hit.doc_id,
|
||||
error = %e,
|
||||
"backfill_repo: get_document failed; leaving hit.repo = None"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
});
|
||||
if let Some(r) = repo_val {
|
||||
hit.repo = Some(r.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve the embedder + vector store, surfacing the user-friendly
|
||||
/// "switch to --mode lexical" error when embeddings are disabled.
|
||||
fn require_embeddings(
|
||||
@@ -564,6 +878,24 @@ fn lexical_index_version(config: &kebab_config::Config) -> IndexVersion {
|
||||
IndexVersion(format!("lex:{}", config.chunking.chunker_version))
|
||||
}
|
||||
|
||||
/// p9-fb-37: stand-in for the vector retriever in the trace path when
|
||||
/// `query.mode == SearchMode::Lexical`. `HybridRetriever::search_with_trace`'s
|
||||
/// Lexical branch never calls `vector.search()`, so returning an empty
|
||||
/// hit list here is safe and lets lexical-only workspaces (embedding
|
||||
/// `provider = "none"`) use `--trace` without paying the ~470 MB
|
||||
/// embedder load.
|
||||
struct NoopRetriever;
|
||||
|
||||
impl Retriever for NoopRetriever {
|
||||
fn search(&self, _q: &kebab_core::SearchQuery) -> anyhow::Result<Vec<kebab_core::SearchHit>> {
|
||||
Ok(Vec::new())
|
||||
}
|
||||
|
||||
fn index_version(&self) -> kebab_core::IndexVersion {
|
||||
kebab_core::IndexVersion("noop:trace".into())
|
||||
}
|
||||
}
|
||||
|
||||
/// Compose a stable `IndexVersion` for the vector retriever. Tracks
|
||||
/// `(embedding_model, embedding_version, dimensions)` so a model swap
|
||||
/// flags drift via the existing index_version mismatch warning in
|
||||
@@ -604,6 +936,49 @@ fn blake3_truncate(input: &str) -> u128 {
|
||||
u128::from_be_bytes(buf)
|
||||
}
|
||||
|
||||
/// p9-fb-34: trim `s` to at most `n` Unicode scalar chars. Cheap
|
||||
/// alternative to a `.chars().take(n).collect::<String>()` pattern;
|
||||
/// reserves capacity proportional to UTF-8 worst case (4 bytes / char)
|
||||
/// so the inner push never re-allocates.
|
||||
fn trim_to_chars(s: &str, n: usize) -> String {
|
||||
if s.chars().count() <= n {
|
||||
return s.to_string();
|
||||
}
|
||||
let mut out = String::with_capacity(n.saturating_mul(4));
|
||||
for (i, c) in s.chars().enumerate() {
|
||||
if i >= n {
|
||||
break;
|
||||
}
|
||||
out.push(c);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// p9-fb-34: estimate wire JSON char cost of the hit list. Returns 0
|
||||
/// per-hit when serialization fails — a SearchHit serialization
|
||||
/// failure is an invariant violation; we degrade gracefully (loop
|
||||
/// terminates early) rather than panic in the budget loop.
|
||||
fn estimate_chars(hits: &[SearchHit]) -> usize {
|
||||
hits.iter()
|
||||
.map(|h| serde_json::to_string(h).map(|s| s.len()).unwrap_or(0))
|
||||
.sum()
|
||||
}
|
||||
|
||||
/// p10-1A-2: back-fill `SearchHit.code_lang` from `Citation::Code.lang`
|
||||
/// for every code hit in the list. The search layer (kebab-search)
|
||||
/// constructs hits with `code_lang: None`; we fill it here in kebab-app
|
||||
/// post-retrieval so callers see the correct language identifier without
|
||||
/// requiring a second SQL query.
|
||||
fn backfill_code_lang(hits: &mut [SearchHit]) {
|
||||
for hit in hits.iter_mut() {
|
||||
if let kebab_core::Citation::Code { lang, .. } = &hit.citation {
|
||||
if hit.code_lang.is_none() {
|
||||
hit.code_lang = lang.clone();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -646,3 +1021,59 @@ mod tests {
|
||||
assert_ne!(a, d, "different session_id → different hash");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests_trace {
|
||||
use super::*;
|
||||
use kebab_core::{SearchMode, SearchOpts, SearchQuery};
|
||||
|
||||
fn open_app_with_temp_dir() -> (tempfile::TempDir, App) {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut cfg = kebab_config::Config::defaults();
|
||||
cfg.storage.data_dir = dir.path().to_string_lossy().into_owned();
|
||||
// Bring up migrations.
|
||||
let store = kebab_store_sqlite::SqliteStore::open(&cfg).unwrap();
|
||||
store.run_migrations().unwrap();
|
||||
drop(store);
|
||||
let app = App::open_with_config(cfg).unwrap();
|
||||
(dir, app)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn search_response_trace_none_when_opts_trace_false() {
|
||||
let (_dir, app) = open_app_with_temp_dir();
|
||||
let q = SearchQuery {
|
||||
text: "x".into(),
|
||||
mode: SearchMode::Lexical,
|
||||
k: 1,
|
||||
filters: Default::default(),
|
||||
};
|
||||
let resp = app.search_with_opts(q, SearchOpts::default()).unwrap();
|
||||
assert!(resp.trace.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn search_response_trace_some_when_opts_trace_true_lexical_mode() {
|
||||
// Lexical mode doesn't require embeddings — the trace path
|
||||
// builds HybridRetriever with a `NoopRetriever` stand-in for
|
||||
// the vector side, since `HybridRetriever::search_with_trace`'s
|
||||
// Lexical branch never invokes `vector.search()`. Default
|
||||
// Config has embedding `provider = "none"`, and lexical-mode
|
||||
// trace must succeed under that config (no embedder load).
|
||||
let (_dir, app) = open_app_with_temp_dir();
|
||||
let q = SearchQuery {
|
||||
text: "x".into(),
|
||||
mode: SearchMode::Lexical,
|
||||
k: 1,
|
||||
filters: Default::default(),
|
||||
};
|
||||
let opts = SearchOpts {
|
||||
trace: true,
|
||||
..Default::default()
|
||||
};
|
||||
let resp = app
|
||||
.search_with_opts(q, opts)
|
||||
.expect("lexical-mode trace must succeed without embeddings");
|
||||
assert!(resp.trace.is_some(), "trace populated when opts.trace=true");
|
||||
}
|
||||
}
|
||||
|
||||
298
crates/kebab-app/src/bulk.rs
Normal file
298
crates/kebab-app/src/bulk.rs
Normal file
@@ -0,0 +1,298 @@
|
||||
//! p9-fb-42: bulk multi-query facade. Sequential for-loop reusing
|
||||
//! one App instance so embedder cold-start + LRU cache amortize
|
||||
//! across the N queries.
|
||||
|
||||
use anyhow::Context;
|
||||
use kebab_core::{
|
||||
BulkSearchItem, BulkSearchSummary, DocumentId, Lang, SearchFilters, SearchHit, SearchMode,
|
||||
SearchOpts, SearchQuery, TrustLevel,
|
||||
};
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::{App, SearchResponse};
|
||||
|
||||
/// Hard cap on items per bulk call. Documented in spec — agents that
|
||||
/// hit this should batch-split.
|
||||
pub const BULK_QUERIES_MAX: usize = 100;
|
||||
|
||||
/// p9-fb-42: bulk search facade. Returns `(items, summary)` always
|
||||
/// — per-query failures embed `error.v1` JSON in the item rather
|
||||
/// than aborting the bulk call. Returns `Err` only for input
|
||||
/// validation failures (e.g. >100 queries).
|
||||
#[doc(hidden)]
|
||||
pub fn bulk_search_with_config(
|
||||
config: kebab_config::Config,
|
||||
raw_items: Vec<Value>,
|
||||
) -> anyhow::Result<(Vec<BulkSearchItem>, BulkSearchSummary)> {
|
||||
if raw_items.len() > BULK_QUERIES_MAX {
|
||||
anyhow::bail!(
|
||||
"queries: max {} items, got {}",
|
||||
BULK_QUERIES_MAX,
|
||||
raw_items.len()
|
||||
);
|
||||
}
|
||||
|
||||
let app = App::open_with_config(config).context("kebab-app: open for bulk_search")?;
|
||||
|
||||
let mut results: Vec<BulkSearchItem> = Vec::with_capacity(raw_items.len());
|
||||
let mut succeeded: u32 = 0;
|
||||
let mut failed: u32 = 0;
|
||||
|
||||
for raw in raw_items {
|
||||
let item = run_one(&app, raw);
|
||||
if item.error.is_some() {
|
||||
failed += 1;
|
||||
} else {
|
||||
succeeded += 1;
|
||||
}
|
||||
results.push(item);
|
||||
}
|
||||
|
||||
let summary = BulkSearchSummary {
|
||||
total: succeeded + failed,
|
||||
succeeded,
|
||||
failed,
|
||||
};
|
||||
Ok((results, summary))
|
||||
}
|
||||
|
||||
fn run_one(app: &App, raw: Value) -> BulkSearchItem {
|
||||
let echo = raw.clone();
|
||||
match parse_one(&raw) {
|
||||
Ok((query, opts)) => match app.search_with_opts(query, opts) {
|
||||
Ok(resp) => BulkSearchItem {
|
||||
query: echo,
|
||||
response: Some(serialize_search_response(&resp)),
|
||||
error: None,
|
||||
},
|
||||
Err(e) => BulkSearchItem {
|
||||
query: echo,
|
||||
response: None,
|
||||
error: Some(error_v1_json("retrieval_error", &format!("{e:#}"), None)),
|
||||
},
|
||||
},
|
||||
Err(msg) => BulkSearchItem {
|
||||
query: echo,
|
||||
response: None,
|
||||
error: Some(error_v1_json("invalid_input", &msg, None)),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Mirror of `kebab-cli::wire::wire_search_response` — `SearchResponse`
|
||||
/// itself is not `Serialize`, so we build the `search_response.v1`-shaped
|
||||
/// JSON manually. Each hit also gets `score` promoted from
|
||||
/// `retrieval.fusion_score` per §2.2, matching the CLI wire layer.
|
||||
fn serialize_search_response(r: &SearchResponse) -> Value {
|
||||
let mut v = serde_json::json!({
|
||||
"schema_version": "search_response.v1",
|
||||
"hits": r.hits.iter().map(serialize_search_hit).collect::<Vec<_>>(),
|
||||
"next_cursor": r.next_cursor,
|
||||
"truncated": r.truncated,
|
||||
});
|
||||
if let Value::Object(ref mut map) = v {
|
||||
let trace_v = match &r.trace {
|
||||
Some(t) => serde_json::to_value(t).unwrap_or(Value::Null),
|
||||
None => Value::Null,
|
||||
};
|
||||
map.insert("trace".to_string(), trace_v);
|
||||
}
|
||||
v
|
||||
}
|
||||
|
||||
fn serialize_search_hit(h: &SearchHit) -> Value {
|
||||
let mut v = serde_json::to_value(h).unwrap_or(Value::Null);
|
||||
if let Value::Object(ref mut map) = v {
|
||||
if let Some(Value::Object(retrieval)) = map.get("retrieval") {
|
||||
if let Some(score) = retrieval.get("fusion_score").cloned() {
|
||||
map.insert("score".to_string(), score);
|
||||
}
|
||||
}
|
||||
map.insert(
|
||||
"schema_version".to_string(),
|
||||
Value::String("search_hit.v1".to_string()),
|
||||
);
|
||||
}
|
||||
v
|
||||
}
|
||||
|
||||
fn parse_one(raw: &Value) -> Result<(SearchQuery, SearchOpts), String> {
|
||||
let obj = raw.as_object().ok_or("expected JSON object")?;
|
||||
let text = obj
|
||||
.get("query")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or("missing required field: query")?
|
||||
.to_string();
|
||||
|
||||
let mode = match obj.get("mode").and_then(|v| v.as_str()) {
|
||||
None => SearchMode::Hybrid,
|
||||
Some("hybrid") => SearchMode::Hybrid,
|
||||
Some("lexical") => SearchMode::Lexical,
|
||||
Some("vector") => SearchMode::Vector,
|
||||
Some(other) => return Err(format!("invalid mode: {other:?}")),
|
||||
};
|
||||
|
||||
let k = obj
|
||||
.get("k")
|
||||
.and_then(|v| v.as_u64())
|
||||
.map(|n| n as usize)
|
||||
.unwrap_or(0); // 0 → use config default in app
|
||||
|
||||
let trust_min = match obj.get("trust_min").and_then(|v| v.as_str()) {
|
||||
None => None,
|
||||
Some("primary") => Some(TrustLevel::Primary),
|
||||
Some("secondary") => Some(TrustLevel::Secondary),
|
||||
Some("generated") => Some(TrustLevel::Generated),
|
||||
Some(other) => return Err(format!("invalid trust_min: {other:?}")),
|
||||
};
|
||||
|
||||
let ingested_after = match obj.get("ingested_after").and_then(|v| v.as_str()) {
|
||||
None => None,
|
||||
Some(s) => Some(
|
||||
time::OffsetDateTime::parse(s, &time::format_description::well_known::Rfc3339)
|
||||
.map_err(|e| format!("invalid ingested_after RFC3339 {s:?}: {e}"))?,
|
||||
),
|
||||
};
|
||||
|
||||
let media: Vec<String> = obj
|
||||
.get("media")
|
||||
.and_then(|v| v.as_array())
|
||||
.map(|arr| {
|
||||
arr.iter()
|
||||
.filter_map(|x| x.as_str().map(normalize_media_alias))
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
let tags_any: Vec<String> = obj
|
||||
.get("tag")
|
||||
.and_then(|v| v.as_array())
|
||||
.map(|arr| {
|
||||
arr.iter()
|
||||
.filter_map(|x| x.as_str().map(String::from))
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
let lang = obj
|
||||
.get("lang")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| Lang(s.to_string()));
|
||||
|
||||
let path_glob = obj
|
||||
.get("path_glob")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(String::from);
|
||||
|
||||
let doc_id = obj
|
||||
.get("doc_id")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| DocumentId(s.to_string()));
|
||||
|
||||
let filters = SearchFilters {
|
||||
tags_any,
|
||||
lang,
|
||||
path_glob,
|
||||
trust_min,
|
||||
media,
|
||||
ingested_after,
|
||||
doc_id,
|
||||
repo: vec![],
|
||||
code_lang: vec![],
|
||||
};
|
||||
|
||||
let opts = SearchOpts {
|
||||
max_tokens: obj
|
||||
.get("max_tokens")
|
||||
.and_then(|v| v.as_u64())
|
||||
.map(|n| n as usize),
|
||||
snippet_chars: obj
|
||||
.get("snippet_chars")
|
||||
.and_then(|v| v.as_u64())
|
||||
.map(|n| n as usize),
|
||||
cursor: obj.get("cursor").and_then(|v| v.as_str()).map(String::from),
|
||||
trace: obj.get("trace").and_then(|v| v.as_bool()).unwrap_or(false),
|
||||
};
|
||||
|
||||
Ok((
|
||||
SearchQuery {
|
||||
text,
|
||||
mode,
|
||||
k,
|
||||
filters,
|
||||
},
|
||||
opts,
|
||||
))
|
||||
}
|
||||
|
||||
fn normalize_media_alias(s: &str) -> String {
|
||||
match s.to_ascii_lowercase().as_str() {
|
||||
"md" => "markdown".to_string(),
|
||||
other => other.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn error_v1_json(code: &str, message: &str, hint: Option<&str>) -> Value {
|
||||
serde_json::json!({
|
||||
"schema_version": "error.v1",
|
||||
"code": code,
|
||||
"message": message,
|
||||
"hint": hint,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn open_temp() -> kebab_config::Config {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut cfg = kebab_config::Config::defaults();
|
||||
cfg.storage.data_dir = dir.path().to_string_lossy().into_owned();
|
||||
// Bring up migrations so SqliteStore::open_existing succeeds inside App::open.
|
||||
let store = kebab_store_sqlite::SqliteStore::open(&cfg).unwrap();
|
||||
store.run_migrations().unwrap();
|
||||
drop(store);
|
||||
// Leak the tempdir into a static — tests are short-lived; not worth threading.
|
||||
std::mem::forget(dir);
|
||||
cfg
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_input_returns_empty_summary() {
|
||||
let cfg = open_temp();
|
||||
let (items, summary) = bulk_search_with_config(cfg, vec![]).unwrap();
|
||||
assert!(items.is_empty());
|
||||
assert_eq!(summary.total, 0);
|
||||
assert_eq!(summary.succeeded, 0);
|
||||
assert_eq!(summary.failed, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn over_cap_returns_err() {
|
||||
let cfg = open_temp();
|
||||
let raw: Vec<Value> = (0..101)
|
||||
.map(|_| serde_json::json!({"query": "x"}))
|
||||
.collect();
|
||||
let err = bulk_search_with_config(cfg, raw).unwrap_err();
|
||||
let msg = format!("{err:#}");
|
||||
assert!(msg.contains("max 100"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_item_emits_error_keeps_total_count() {
|
||||
let cfg = open_temp();
|
||||
let raw = vec![
|
||||
serde_json::json!({"query": "ok", "mode": "lexical"}),
|
||||
serde_json::json!({"mode": "lexical"}), // missing required `query`
|
||||
];
|
||||
let (items, summary) = bulk_search_with_config(cfg, raw).unwrap();
|
||||
assert_eq!(items.len(), 2);
|
||||
assert_eq!(summary.total, 2);
|
||||
// First item: lexical mode against empty corpus succeeds with empty hits.
|
||||
assert!(items[0].error.is_none());
|
||||
// Second item: missing required field.
|
||||
assert!(items[1].error.is_some());
|
||||
assert_eq!(items[1].error.as_ref().unwrap()["code"], "invalid_input");
|
||||
}
|
||||
}
|
||||
75
crates/kebab-app/src/cursor.rs
Normal file
75
crates/kebab-app/src/cursor.rs
Normal file
@@ -0,0 +1,75 @@
|
||||
//! p9-fb-34 opaque pagination cursor.
|
||||
//!
|
||||
//! Format: base64(JSON({offset: usize, corpus_revision: string})).
|
||||
//! Opaque to callers — they MUST NOT decode the contents themselves;
|
||||
//! the schema is internal and may change without notice.
|
||||
|
||||
use base64::Engine;
|
||||
use base64::engine::general_purpose::URL_SAFE_NO_PAD;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::error_wire::ErrorV1;
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
struct Payload {
|
||||
offset: usize,
|
||||
corpus_revision: String,
|
||||
}
|
||||
|
||||
/// Encode `(offset, corpus_revision)` as an opaque base64 string.
|
||||
pub fn encode(offset: usize, corpus_revision: &str) -> String {
|
||||
let payload = Payload {
|
||||
offset,
|
||||
corpus_revision: corpus_revision.to_string(),
|
||||
};
|
||||
let json = serde_json::to_vec(&payload).expect("Payload serializes");
|
||||
URL_SAFE_NO_PAD.encode(&json)
|
||||
}
|
||||
|
||||
/// Decode an opaque cursor against the expected `corpus_revision`.
|
||||
/// Mismatch or malformed input returns an `ErrorV1` with
|
||||
/// `code = "stale_cursor"`.
|
||||
//
|
||||
// p9-fb-34: ErrorV1 is the workspace-wide wire error struct (~200B
|
||||
// after monomorphization with Value + String fields). Boxing here
|
||||
// would force every call site to deref through a Box for no win —
|
||||
// the err-path is rare. Single allow at the function level.
|
||||
//
|
||||
// p9-fb-34 round-1 review: differentiate the three failure modes
|
||||
// (base64 / JSON / revision mismatch) with distinct messages — all
|
||||
// keep `code = "stale_cursor"` so the agent's branching logic stays
|
||||
// the same, but humans reading the message get a precise hint.
|
||||
#[allow(clippy::result_large_err)]
|
||||
pub fn decode(s: &str, expected_revision: &str) -> Result<usize, ErrorV1> {
|
||||
let bytes = URL_SAFE_NO_PAD.decode(s.as_bytes()).map_err(|_| ErrorV1 {
|
||||
schema_version: "error.v1".to_string(),
|
||||
code: "stale_cursor".to_string(),
|
||||
message: "cursor is not valid base64. Re-issue search to obtain a fresh cursor."
|
||||
.to_string(),
|
||||
details: Value::Null,
|
||||
hint: None,
|
||||
})?;
|
||||
let payload: Payload = serde_json::from_slice(&bytes).map_err(|_| ErrorV1 {
|
||||
schema_version: "error.v1".to_string(),
|
||||
code: "stale_cursor".to_string(),
|
||||
message: "cursor payload is malformed. Re-issue search to obtain a fresh cursor."
|
||||
.to_string(),
|
||||
details: Value::Null,
|
||||
hint: None,
|
||||
})?;
|
||||
if payload.corpus_revision != expected_revision {
|
||||
return Err(ErrorV1 {
|
||||
schema_version: "error.v1".to_string(),
|
||||
code: "stale_cursor".to_string(),
|
||||
message: format!(
|
||||
"cursor was issued against corpus_revision '{}'; current revision is \
|
||||
'{}'. Re-issue search to obtain a fresh cursor.",
|
||||
payload.corpus_revision, expected_revision
|
||||
),
|
||||
details: Value::Null,
|
||||
hint: None,
|
||||
});
|
||||
}
|
||||
Ok(payload.offset)
|
||||
}
|
||||
@@ -11,6 +11,12 @@ use serde_json::{Value, json};
|
||||
|
||||
use crate::error_signal::{ConfigInvalid, LlmError, NotIndexed};
|
||||
|
||||
// p9-fb-34: `stale_cursor` is constructed directly by `cursor::decode`
|
||||
// and surfaced through `StructuredError` (an anyhow-friendly wrapper
|
||||
// that carries the typed `ErrorV1` payload without lossy string
|
||||
// formatting). `classify` short-circuits on it at the top of the
|
||||
// function so the typed `code = "stale_cursor"` reaches the wire.
|
||||
|
||||
/// Wire schema id for [`ErrorV1`]. Single source of truth — kebab-cli
|
||||
/// + kebab-mcp use this via `kebab_app::ERROR_V1_ID`.
|
||||
pub const ERROR_V1_ID: &str = "error.v1";
|
||||
@@ -24,7 +30,29 @@ pub struct ErrorV1 {
|
||||
pub hint: Option<String>,
|
||||
}
|
||||
|
||||
/// p9-fb-34: typed wrapper around an [`ErrorV1`] so callers that
|
||||
/// surface `anyhow::Error` can downcast back to the structured wire
|
||||
/// payload instead of losing it to string formatting. Constructed by
|
||||
/// the cursor code path (`cursor::decode` → `App::search_with_opts`)
|
||||
/// and short-circuited inside [`classify`].
|
||||
#[derive(Debug)]
|
||||
pub struct StructuredError(pub ErrorV1);
|
||||
|
||||
impl std::fmt::Display for StructuredError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "[{}] {}", self.0.code, self.0.message)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for StructuredError {}
|
||||
|
||||
pub fn classify(err: &anyhow::Error, verbose: bool) -> ErrorV1 {
|
||||
// p9-fb-34: structured wrapper short-circuits — preserves the
|
||||
// typed payload that callers (cursor::decode) constructed
|
||||
// instead of falling through to `code = "generic"`.
|
||||
if let Some(s) = err.downcast_ref::<StructuredError>() {
|
||||
return s.0.clone();
|
||||
}
|
||||
if let Some(s) = err.downcast_ref::<ConfigInvalid>() {
|
||||
return ErrorV1 {
|
||||
schema_version: ERROR_V1_ID.to_string(),
|
||||
@@ -197,4 +225,36 @@ mod tests {
|
||||
let v1 = classify(&err, false);
|
||||
assert_eq!(v1.code, "io_error");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stale_cursor_is_not_routed_through_classify() {
|
||||
use anyhow::anyhow;
|
||||
let err: anyhow::Error = anyhow!("stale_cursor: rev mismatch");
|
||||
let v1 = classify(&err, false);
|
||||
// p9-fb-34: stale_cursor is constructed directly by cursor::decode
|
||||
// (single source of truth). classify must not pattern-match on
|
||||
// anyhow string contents — that would create two sources of
|
||||
// truth. The bare anyhow string falls through to "generic".
|
||||
assert_ne!(v1.code, "stale_cursor", "classify must not produce stale_cursor from bare anyhow string");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stale_cursor_propagates_through_structured_wrapper() {
|
||||
// p9-fb-34: positive-side contract for the structured-wrapper
|
||||
// path. cursor::decode constructs a typed ErrorV1, the call site
|
||||
// wraps it in `StructuredError`, anyhow carries it, and classify
|
||||
// short-circuits via downcast — preserving the typed code +
|
||||
// message instead of falling through to "generic".
|
||||
let original = ErrorV1 {
|
||||
schema_version: ERROR_V1_ID.to_string(),
|
||||
code: "stale_cursor".to_string(),
|
||||
message: "test stale cursor".to_string(),
|
||||
details: Value::Null,
|
||||
hint: None,
|
||||
};
|
||||
let err: anyhow::Error = anyhow::Error::new(StructuredError(original));
|
||||
let v1 = classify(&err, false);
|
||||
assert_eq!(v1.code, "stale_cursor");
|
||||
assert_eq!(v1.message, "test stale cursor");
|
||||
}
|
||||
}
|
||||
|
||||
447
crates/kebab-app/src/fetch.rs
Normal file
447
crates/kebab-app/src/fetch.rs
Normal file
@@ -0,0 +1,447 @@
|
||||
//! p9-fb-35 verbatim fetch implementation.
|
||||
//!
|
||||
//! [`App::fetch`] is the facade entry point. It dispatches on
|
||||
//! [`FetchQuery`] variants:
|
||||
//!
|
||||
//! - `Chunk(id)` — return the chunk row from `chunks.text`, optionally
|
||||
//! with ±N surrounding chunks (`FetchOpts::context`).
|
||||
//! - `Doc(id)` — return the entire document re-serialized to markdown.
|
||||
//! (Implemented in Task 4.)
|
||||
//! - `Span { doc_id, line_start, line_end }` — return a contiguous line
|
||||
//! slice. (Implemented in Task 5.)
|
||||
//!
|
||||
//! Errors are surfaced as [`StructuredError`] (anyhow-friendly wrapper
|
||||
//! around `ErrorV1`) so the CLI / MCP wire layer's `classify` keeps the
|
||||
//! typed `code` (`chunk_not_found` / `doc_not_found` /
|
||||
//! `span_not_supported`) instead of falling through to `code =
|
||||
//! "generic"`.
|
||||
|
||||
use anyhow::Result;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
use kebab_core::{
|
||||
Block, CanonicalDocument, Chunk, ChunkId, DocumentId, DocumentStore, FetchKind, FetchOpts,
|
||||
FetchQuery, FetchResult,
|
||||
};
|
||||
|
||||
use crate::App;
|
||||
use crate::error_wire::{ERROR_V1_ID, ErrorV1, StructuredError};
|
||||
use crate::staleness::compute_stale;
|
||||
|
||||
impl App {
|
||||
/// p9-fb-35: verbatim fetch facade. Returns text from
|
||||
/// `chunks.text` / `CanonicalDocument` based on the requested
|
||||
/// mode. Errors surface as `StructuredError(ErrorV1)` with one
|
||||
/// of `chunk_not_found` / `doc_not_found` / `span_not_supported`
|
||||
/// so the wire-layer classifier preserves the typed code.
|
||||
pub fn fetch(&self, query: FetchQuery, opts: FetchOpts) -> Result<FetchResult> {
|
||||
match query {
|
||||
FetchQuery::Chunk(id) => fetch_chunk(self, id, opts),
|
||||
FetchQuery::Doc(id) => fetch_doc(self, id, opts),
|
||||
FetchQuery::Span {
|
||||
doc_id,
|
||||
line_start,
|
||||
line_end,
|
||||
} => fetch_span(self, doc_id, line_start, line_end, opts),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn fetch_chunk(app: &App, id: ChunkId, opts: FetchOpts) -> Result<FetchResult> {
|
||||
let target = <kebab_store_sqlite::SqliteStore as DocumentStore>::get_chunk(&app.sqlite, &id)?
|
||||
.ok_or_else(|| {
|
||||
anyhow::Error::new(StructuredError(ErrorV1 {
|
||||
schema_version: ERROR_V1_ID.to_string(),
|
||||
code: "chunk_not_found".to_string(),
|
||||
message: format!("chunk_id '{}' not found", id.0),
|
||||
details: serde_json::Value::Null,
|
||||
hint: None,
|
||||
}))
|
||||
})?;
|
||||
|
||||
let doc_id = target.doc_id.clone();
|
||||
let doc =
|
||||
<kebab_store_sqlite::SqliteStore as DocumentStore>::get_document(&app.sqlite, &doc_id)?
|
||||
.ok_or_else(|| {
|
||||
anyhow::Error::new(StructuredError(ErrorV1 {
|
||||
schema_version: ERROR_V1_ID.to_string(),
|
||||
code: "doc_not_found".to_string(),
|
||||
message: format!(
|
||||
"doc_id '{}' (parent of chunk '{}') not found",
|
||||
doc_id.0, id.0
|
||||
),
|
||||
details: serde_json::Value::Null,
|
||||
hint: None,
|
||||
}))
|
||||
})?;
|
||||
|
||||
let (context_before, context_after) = match opts.context {
|
||||
Some(n) if n > 0 => surrounding_chunks(app, &doc_id, &id, n)?,
|
||||
_ => (Vec::new(), Vec::new()),
|
||||
};
|
||||
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let stale = compute_stale(
|
||||
doc_metadata_updated_at(&doc),
|
||||
now,
|
||||
app.config.search.stale_threshold_days,
|
||||
);
|
||||
|
||||
Ok(FetchResult {
|
||||
kind: FetchKind::Chunk,
|
||||
doc_id: doc.doc_id.clone(),
|
||||
doc_path: doc.workspace_path.clone(),
|
||||
indexed_at: doc_metadata_updated_at(&doc),
|
||||
stale,
|
||||
chunk: Some(target),
|
||||
context_before,
|
||||
context_after,
|
||||
text: None,
|
||||
line_start: None,
|
||||
line_end: None,
|
||||
effective_end: None,
|
||||
truncated: false,
|
||||
})
|
||||
}
|
||||
|
||||
fn fetch_doc(app: &App, id: DocumentId, opts: FetchOpts) -> Result<FetchResult> {
|
||||
let doc = <kebab_store_sqlite::SqliteStore as DocumentStore>::get_document(&app.sqlite, &id)?
|
||||
.ok_or_else(|| {
|
||||
anyhow::Error::new(StructuredError(ErrorV1 {
|
||||
schema_version: ERROR_V1_ID.to_string(),
|
||||
code: "doc_not_found".to_string(),
|
||||
message: format!("doc_id '{}' not found", id.0),
|
||||
details: serde_json::Value::Null,
|
||||
hint: None,
|
||||
}))
|
||||
})?;
|
||||
|
||||
let mut text = fmt_canonical_to_markdown(&doc);
|
||||
let mut truncated = false;
|
||||
if let Some(max_tokens) = opts.max_tokens {
|
||||
let max_chars = max_tokens.saturating_mul(4);
|
||||
if text.chars().count() > max_chars {
|
||||
text = trim_to_chars(&text, max_chars);
|
||||
truncated = true;
|
||||
}
|
||||
}
|
||||
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let stale = compute_stale(
|
||||
doc_metadata_updated_at(&doc),
|
||||
now,
|
||||
app.config.search.stale_threshold_days,
|
||||
);
|
||||
|
||||
Ok(FetchResult {
|
||||
kind: FetchKind::Doc,
|
||||
doc_id: doc.doc_id.clone(),
|
||||
doc_path: doc.workspace_path.clone(),
|
||||
indexed_at: doc_metadata_updated_at(&doc),
|
||||
stale,
|
||||
chunk: None,
|
||||
context_before: Vec::new(),
|
||||
context_after: Vec::new(),
|
||||
text: Some(text),
|
||||
line_start: None,
|
||||
line_end: None,
|
||||
effective_end: None,
|
||||
truncated,
|
||||
})
|
||||
}
|
||||
|
||||
/// p9-fb-35: trim string to N chars (Unicode-safe). Mirrors fb-34's
|
||||
/// helper at `crates/kebab-app/src/app.rs` — kept local to avoid
|
||||
/// re-exporting an internal helper.
|
||||
fn trim_to_chars(s: &str, n: usize) -> String {
|
||||
if s.chars().count() <= n {
|
||||
return s.to_string();
|
||||
}
|
||||
let mut out = String::with_capacity(n * 4);
|
||||
for (i, c) in s.chars().enumerate() {
|
||||
if i >= n {
|
||||
break;
|
||||
}
|
||||
out.push(c);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn fetch_span(
|
||||
app: &App,
|
||||
id: DocumentId,
|
||||
line_start: u32,
|
||||
line_end: u32,
|
||||
opts: FetchOpts,
|
||||
) -> Result<FetchResult> {
|
||||
let doc = <kebab_store_sqlite::SqliteStore as DocumentStore>::get_document(&app.sqlite, &id)?
|
||||
.ok_or_else(|| {
|
||||
anyhow::Error::new(StructuredError(ErrorV1 {
|
||||
schema_version: ERROR_V1_ID.to_string(),
|
||||
code: "doc_not_found".to_string(),
|
||||
message: format!("doc_id '{}' not found", id.0),
|
||||
details: serde_json::Value::Null,
|
||||
hint: None,
|
||||
}))
|
||||
})?;
|
||||
|
||||
// Reject line-incompatible media types (PDF / audio). `SourceType`
|
||||
// (markdown / note / paper / reference / inbox) is the *user-facing*
|
||||
// category, not the rendering format — the actual byte-level format
|
||||
// lives on the source `RawAsset.media_type`. Look it up via
|
||||
// workspace_path (unique key per asset).
|
||||
if let Some(asset) = <kebab_store_sqlite::SqliteStore as DocumentStore>::get_asset_by_workspace_path(
|
||||
&app.sqlite,
|
||||
&doc.workspace_path,
|
||||
)? {
|
||||
if matches!(
|
||||
asset.media_type,
|
||||
kebab_core::MediaType::Pdf | kebab_core::MediaType::Audio(_)
|
||||
) {
|
||||
return Err(anyhow::Error::new(StructuredError(ErrorV1 {
|
||||
schema_version: ERROR_V1_ID.to_string(),
|
||||
code: "span_not_supported".to_string(),
|
||||
message: format!(
|
||||
"doc '{}' has media_type {:?}; line-based span fetch unsupported. \
|
||||
Use `fetch chunk` or `fetch doc` instead.",
|
||||
id.0, asset.media_type
|
||||
),
|
||||
details: serde_json::Value::Null,
|
||||
hint: Some("kind = chunk or kind = doc instead".to_string()),
|
||||
})));
|
||||
}
|
||||
}
|
||||
|
||||
if line_start == 0 || line_end == 0 || line_end < line_start {
|
||||
return Err(anyhow::Error::new(StructuredError(ErrorV1 {
|
||||
schema_version: ERROR_V1_ID.to_string(),
|
||||
code: "invalid_input".to_string(),
|
||||
message: format!(
|
||||
"line_start ({line_start}) and line_end ({line_end}) must be 1-based with start <= end"
|
||||
),
|
||||
details: serde_json::Value::Null,
|
||||
hint: None,
|
||||
})));
|
||||
}
|
||||
|
||||
let full = fmt_canonical_to_markdown(&doc);
|
||||
let lines: Vec<&str> = full.lines().collect();
|
||||
let total = lines.len() as u32;
|
||||
|
||||
// p9-fb-35 round-1 review fix: empty / out-of-range request must
|
||||
// not slice. Returning empty text + `effective_end = line_start - 1`
|
||||
// lets the caller detect "no lines fetched" via
|
||||
// `text.is_empty() && effective_end < line_start`. `truncated`
|
||||
// stays false because line-range clamp is NOT a budget event —
|
||||
// budget-driven truncation is the only thing `truncated` signals.
|
||||
if total == 0 || line_start > total {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let stale = compute_stale(
|
||||
doc_metadata_updated_at(&doc),
|
||||
now,
|
||||
app.config.search.stale_threshold_days,
|
||||
);
|
||||
return Ok(FetchResult {
|
||||
kind: FetchKind::Span,
|
||||
doc_id: doc.doc_id.clone(),
|
||||
doc_path: doc.workspace_path.clone(),
|
||||
indexed_at: doc_metadata_updated_at(&doc),
|
||||
stale,
|
||||
chunk: None,
|
||||
context_before: Vec::new(),
|
||||
context_after: Vec::new(),
|
||||
text: Some(String::new()),
|
||||
line_start: Some(line_start),
|
||||
line_end: Some(line_end),
|
||||
// saturating_sub: when line_start = 1 we end at 0, signaling
|
||||
// "no lines fetched" without underflowing u32.
|
||||
effective_end: Some(line_start.saturating_sub(1)),
|
||||
truncated: false,
|
||||
});
|
||||
}
|
||||
|
||||
let effective_end_raw = line_end.min(total);
|
||||
let lo = (line_start - 1) as usize;
|
||||
let hi = effective_end_raw as usize;
|
||||
let mut text = lines[lo..hi].join("\n");
|
||||
|
||||
// p9-fb-35 round-1 review fix: `truncated` is reserved for
|
||||
// budget-driven truncation only. Line-range clamp (line_end >
|
||||
// total) is signaled via `effective_end < line_end`, not via
|
||||
// `truncated`.
|
||||
let mut truncated = false;
|
||||
let mut effective_end = effective_end_raw;
|
||||
if let Some(max_tokens) = opts.max_tokens {
|
||||
let max_chars = max_tokens.saturating_mul(4);
|
||||
if text.chars().count() > max_chars {
|
||||
text = trim_to_chars(&text, max_chars);
|
||||
truncated = true;
|
||||
let kept = text.lines().count() as u32;
|
||||
effective_end = (line_start - 1) + kept;
|
||||
}
|
||||
}
|
||||
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let stale = compute_stale(
|
||||
doc_metadata_updated_at(&doc),
|
||||
now,
|
||||
app.config.search.stale_threshold_days,
|
||||
);
|
||||
|
||||
Ok(FetchResult {
|
||||
kind: FetchKind::Span,
|
||||
doc_id: doc.doc_id.clone(),
|
||||
doc_path: doc.workspace_path.clone(),
|
||||
indexed_at: doc_metadata_updated_at(&doc),
|
||||
stale,
|
||||
chunk: None,
|
||||
context_before: Vec::new(),
|
||||
context_after: Vec::new(),
|
||||
text: Some(text),
|
||||
line_start: Some(line_start),
|
||||
line_end: Some(line_end),
|
||||
effective_end: Some(effective_end),
|
||||
truncated,
|
||||
})
|
||||
}
|
||||
|
||||
/// p9-fb-35: list chunks for a document in ordinal order, return
|
||||
/// `(before, after)` slices around the target chunk_id. `n` caps each
|
||||
/// side independently — the worst case is `2n` total neighbors when
|
||||
/// the target sits in the middle of the doc.
|
||||
fn surrounding_chunks(
|
||||
app: &App,
|
||||
doc_id: &DocumentId,
|
||||
target: &ChunkId,
|
||||
n: u32,
|
||||
) -> Result<(Vec<Chunk>, Vec<Chunk>)> {
|
||||
let chunks = list_chunks_in_order(app, doc_id)?;
|
||||
let target_idx = chunks
|
||||
.iter()
|
||||
.position(|c| c.chunk_id == *target)
|
||||
.ok_or_else(|| anyhow::anyhow!("chunk not found in doc chunk list"))?;
|
||||
let n = n as usize;
|
||||
let lo = target_idx.saturating_sub(n);
|
||||
let hi = target_idx
|
||||
.saturating_add(n)
|
||||
.saturating_add(1)
|
||||
.min(chunks.len());
|
||||
let before: Vec<Chunk> = chunks[lo..target_idx].to_vec();
|
||||
let after: Vec<Chunk> = chunks[target_idx + 1..hi].to_vec();
|
||||
Ok((before, after))
|
||||
}
|
||||
|
||||
/// p9-fb-35: chunks have no explicit ordinal column, so the underlying
|
||||
/// helper sorts by `(created_at, chunk_id)` which matches insertion
|
||||
/// order produced by the chunker (deterministic). The actual SQL lives
|
||||
/// inside `kebab-store-sqlite` (`SqliteStore::list_chunk_ids_for_doc`)
|
||||
/// to keep the facade crate free of direct rusqlite usage.
|
||||
fn list_chunks_in_order(app: &App, doc_id: &DocumentId) -> Result<Vec<Chunk>> {
|
||||
let chunk_ids = app.sqlite.list_chunk_ids_for_doc(doc_id)?;
|
||||
let mut out: Vec<Chunk> = Vec::with_capacity(chunk_ids.len());
|
||||
for cid in chunk_ids {
|
||||
if let Some(chunk) =
|
||||
<kebab_store_sqlite::SqliteStore as DocumentStore>::get_chunk(&app.sqlite, &cid)?
|
||||
{
|
||||
out.push(chunk);
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn doc_metadata_updated_at(doc: &CanonicalDocument) -> OffsetDateTime {
|
||||
doc.metadata.updated_at
|
||||
}
|
||||
|
||||
/// p9-fb-35: serialize a `CanonicalDocument` back to markdown. Best-
|
||||
/// effort round-trip — inline-styled spans (Strong/Emph children)
|
||||
/// flatten to plain text via the already-flattened `TextBlock.text`
|
||||
/// field. Good enough for an agent reading verbatim context. Used by
|
||||
/// Task 4 (doc mode) and Task 5 (span mode).
|
||||
pub(crate) fn fmt_canonical_to_markdown(doc: &CanonicalDocument) -> String {
|
||||
let mut out = String::with_capacity(1024);
|
||||
for (i, block) in doc.blocks.iter().enumerate() {
|
||||
if i > 0 {
|
||||
out.push_str("\n\n");
|
||||
}
|
||||
match block {
|
||||
Block::Heading(h) => {
|
||||
let level = h.level.clamp(1, 6) as usize;
|
||||
for _ in 0..level {
|
||||
out.push('#');
|
||||
}
|
||||
out.push(' ');
|
||||
out.push_str(&h.text);
|
||||
}
|
||||
Block::Paragraph(t) => out.push_str(&t.text),
|
||||
Block::Quote(t) => {
|
||||
// Prefix every line with `> ` so block-quote round-trips.
|
||||
for (li, line) in t.text.split('\n').enumerate() {
|
||||
if li > 0 {
|
||||
out.push('\n');
|
||||
}
|
||||
out.push_str("> ");
|
||||
out.push_str(line);
|
||||
}
|
||||
}
|
||||
Block::List(l) => {
|
||||
for (idx, item) in l.items.iter().enumerate() {
|
||||
if idx > 0 {
|
||||
out.push('\n');
|
||||
}
|
||||
if l.ordered {
|
||||
out.push_str(&format!("{}. {}", idx + 1, item.text));
|
||||
} else {
|
||||
out.push_str(&format!("- {}", item.text));
|
||||
}
|
||||
}
|
||||
}
|
||||
Block::Code(c) => {
|
||||
out.push_str("```");
|
||||
if let Some(lang) = &c.lang {
|
||||
out.push_str(lang);
|
||||
}
|
||||
out.push('\n');
|
||||
out.push_str(&c.code);
|
||||
if !c.code.ends_with('\n') {
|
||||
out.push('\n');
|
||||
}
|
||||
out.push_str("```");
|
||||
}
|
||||
Block::Table(t) => {
|
||||
out.push_str(&t.headers.join(" | "));
|
||||
out.push('\n');
|
||||
// Markdown table separator — N copies of `---|` is
|
||||
// acceptable for a verbatim re-serialization (renderer
|
||||
// tolerates trailing pipe).
|
||||
out.push_str(&"---|".repeat(t.headers.len()));
|
||||
for row in &t.rows {
|
||||
out.push('\n');
|
||||
out.push_str(&row.join(" | "));
|
||||
}
|
||||
}
|
||||
Block::ImageRef(img) => {
|
||||
out.push_str(&format!("", img.alt, img.src));
|
||||
}
|
||||
Block::AudioRef(_a) => {
|
||||
// Canonical doc carries the transcript on AudioRefBlock,
|
||||
// but markdown has no native audio embed. Emit a stub
|
||||
// marker so the agent sees something ran here.
|
||||
out.push_str("(audio reference)");
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// p9-fb-35: free-function entry for CLI / MCP. Mirrors the
|
||||
/// `*_with_config` pattern documented in the kebab-app crate root —
|
||||
/// `kebab-cli` calls this so a `--config <path>` flag is honored.
|
||||
#[doc(hidden)]
|
||||
pub fn fetch_with_config(
|
||||
config: kebab_config::Config,
|
||||
query: FetchQuery,
|
||||
opts: FetchOpts,
|
||||
) -> Result<FetchResult> {
|
||||
App::open_with_config(config)?.fetch(query, opts)
|
||||
}
|
||||
@@ -96,6 +96,7 @@ pub fn media_label(media: &kebab_core::MediaType) -> &'static str {
|
||||
kebab_core::MediaType::Pdf => "pdf",
|
||||
kebab_core::MediaType::Image(_) => "image",
|
||||
kebab_core::MediaType::Audio(_) => "audio",
|
||||
kebab_core::MediaType::Code(_) => "code",
|
||||
kebab_core::MediaType::Other(_) => "other",
|
||||
}
|
||||
}
|
||||
@@ -148,6 +149,7 @@ mod tests {
|
||||
media_label(&MediaType::Audio(kebab_core::AudioType::Wav)),
|
||||
"audio"
|
||||
);
|
||||
assert_eq!(media_label(&MediaType::Code("rust".into())), "code");
|
||||
assert_eq!(media_label(&MediaType::Other("x".into())), "other");
|
||||
}
|
||||
|
||||
|
||||
@@ -39,36 +39,45 @@ use std::sync::Arc;
|
||||
use anyhow::{Context, anyhow};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use kebab_chunk::{MdHeadingV1Chunker, PdfPageV1Chunker};
|
||||
use kebab_chunk::{CodeJsAstV1Chunker, CodePythonAstV1Chunker, CodeRustAstV1Chunker, CodeTsAstV1Chunker, MdHeadingV1Chunker, PdfPageV1Chunker};
|
||||
use kebab_core::{
|
||||
Answer, Block, CanonicalDocument, Chunk, ChunkId, ChunkPolicy, ChunkerVersion, Chunker,
|
||||
DocFilter, DocSummary, DocumentId, DocumentStore, Embedder, EmbeddingInput,
|
||||
EmbeddingKind, ExtractContext, Extractor, IngestReport, Lang, LanguageModel, MediaType,
|
||||
ParserVersion, RawAsset, SearchHit, SearchQuery, SourceConnector, SourceScope,
|
||||
ParserVersion, RawAsset, SearchHit, SearchQuery, SourceScope,
|
||||
SourceUri, VectorRecord, VectorStore,
|
||||
};
|
||||
use kebab_llm_local::OllamaLanguageModel;
|
||||
use kebab_normalize::build_canonical_document;
|
||||
use kebab_parse_image::{ImageExtractor, OllamaVisionOcr, apply_caption, apply_ocr};
|
||||
use kebab_parse_code::{JavascriptAstExtractor, PythonAstExtractor, RustAstExtractor, TypescriptAstExtractor};
|
||||
use kebab_parse_pdf::PdfTextExtractor;
|
||||
use kebab_parse_md::{BodyHints, parse_blocks, parse_frontmatter};
|
||||
use kebab_source_fs::FsSourceConnector;
|
||||
|
||||
mod app;
|
||||
mod bulk;
|
||||
pub mod cursor;
|
||||
pub mod doctor_signal;
|
||||
pub mod error_signal;
|
||||
pub mod error_wire;
|
||||
pub mod external;
|
||||
pub mod fetch;
|
||||
pub mod ingest_progress;
|
||||
pub mod logging;
|
||||
pub mod reset;
|
||||
pub mod schema;
|
||||
mod staleness;
|
||||
|
||||
pub use app::App;
|
||||
pub use app::{App, SearchResponse};
|
||||
pub use ingest_progress::{AggregateCounts, IngestEvent, render_skipped_breakdown};
|
||||
pub use reset::{ResetReport, ResetScope};
|
||||
pub use error_wire::{ERROR_V1_ID, ErrorV1, classify};
|
||||
pub use error_wire::{ERROR_V1_ID, ErrorV1, StructuredError, classify};
|
||||
pub use fetch::fetch_with_config;
|
||||
#[doc(hidden)]
|
||||
pub use bulk::{BULK_QUERIES_MAX, bulk_search_with_config};
|
||||
pub use schema::{Capabilities, Models, SCHEMA_V1_ID, SchemaV1, Stats, WireBlock, schema_with_config};
|
||||
pub use staleness::{compute_stale, mark_stale_in_place};
|
||||
|
||||
/// p9-fb-25: sentinel for files without an extension in
|
||||
/// `IngestReport.skipped_by_extension` keys + `IngestItem.warnings`
|
||||
@@ -83,7 +92,7 @@ pub const NO_EXT_SENTINEL: &str = "<no-ext>";
|
||||
/// `use kebab_app::AskOpts` keeps working without churn. The struct gained
|
||||
/// a `stream_sink` field in P4-3; non-streaming callers (kb-cli today)
|
||||
/// pass `stream_sink: None`.
|
||||
pub use kebab_rag::AskOpts;
|
||||
pub use kebab_rag::{AskOpts, StreamEvent};
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
pub struct DoctorReport {
|
||||
@@ -297,8 +306,8 @@ pub fn ingest_with_config_opts(
|
||||
);
|
||||
let connector = FsSourceConnector::new(&app.config)
|
||||
.context("kb-app::ingest: build FsSourceConnector")?;
|
||||
let assets = connector
|
||||
.scan(&scope)
|
||||
let (assets, fs_skips) = connector
|
||||
.scan_with_skips(&scope)
|
||||
.context("kb-app::ingest: scan workspace")?;
|
||||
crate::ingest_progress::emit(
|
||||
progress,
|
||||
@@ -667,6 +676,12 @@ pub fn ingest_with_config_opts(
|
||||
errors: error_count,
|
||||
duration_ms,
|
||||
skipped_by_extension,
|
||||
skipped_gitignore: fs_skips.skipped_gitignore,
|
||||
skipped_kebabignore: fs_skips.skipped_kebabignore,
|
||||
skipped_builtin_blacklist: fs_skips.skipped_builtin_blacklist,
|
||||
skipped_generated: fs_skips.skipped_generated,
|
||||
skipped_size_exceeded: fs_skips.skipped_size_exceeded,
|
||||
skip_examples: fs_skips.skip_examples,
|
||||
items: if summary_only { None } else { Some(items) },
|
||||
})
|
||||
}
|
||||
@@ -733,15 +748,18 @@ struct ImagePipeline<'a> {
|
||||
/// hold (per design §9 cascade rule):
|
||||
///
|
||||
/// 1. `force_reingest == false` — caller hasn't asked to bypass skip.
|
||||
/// 2. The freshly-scanned asset's blake3 checksum equals what the
|
||||
/// existing `assets` row stores at the same `workspace_path`.
|
||||
/// 3. The doc keyed on `(workspace_path, asset_id, current_parser_version)`
|
||||
/// exists. If the parser_version changed, `id_for_doc` produces a
|
||||
/// different `doc_id` so the lookup misses → no skip → re-process.
|
||||
/// 4. The existing doc's stamped `last_chunker_version` AND
|
||||
/// `last_embedding_version` match the values the caller is about
|
||||
/// to use (`Some(v) == Some(v)` and `None == None` — see design
|
||||
/// doc for the `None == None` rule when no embedder is configured).
|
||||
/// 2. A document already exists at this `workspace_path`
|
||||
/// (`get_document_by_workspace_path`). The lookup is document-side, not
|
||||
/// asset-side, so twin files (identical content at different paths) each
|
||||
/// hit their own stable doc row — `documents.workspace_path` is UNIQUE
|
||||
/// while `assets` may dedupe content into a single row with a flip-flop
|
||||
/// `workspace_path` column (dogfood bug #4, see `tasks/HOTFIXES.md`).
|
||||
/// 3. The existing doc's `source_asset_id` equals the freshly-scanned
|
||||
/// asset's blake3 checksum (content unchanged).
|
||||
/// 4. The existing doc's `parser_version` matches the current extractor's
|
||||
/// `parser_version` (extractor not upgraded). Combined with `chunker_version`
|
||||
/// and `last_embedding_version` checks immediately below — full cascade
|
||||
/// per design §9.
|
||||
///
|
||||
/// Returns `Ok(None)` (proceed with full re-process) when any check
|
||||
/// fails or any DB read errors out — the skip path is opportunistic;
|
||||
@@ -758,31 +776,19 @@ fn try_skip_unchanged(
|
||||
if force_reingest {
|
||||
return Ok(None);
|
||||
}
|
||||
let existing_asset = match app
|
||||
// Document-centric skip: look up the existing document row by
|
||||
// workspace_path directly. This avoids the twin-file flip-flop
|
||||
// that the old asset-side lookup suffers from — multiple files
|
||||
// with identical content share one `assets` row whose
|
||||
// `workspace_path` is overwritten on every UPSERT, so
|
||||
// `get_asset_by_workspace_path(path1)` could return the OTHER
|
||||
// twin's path (or None) after any ingest of the twin. The
|
||||
// `documents` table has a UNIQUE index on `workspace_path` (V001),
|
||||
// so each twin has its own stable row regardless of asset de-dup.
|
||||
let existing_doc = match app
|
||||
.sqlite
|
||||
.get_asset_by_workspace_path(&asset.workspace_path)
|
||||
.get_document_by_workspace_path(&asset.workspace_path)
|
||||
{
|
||||
Ok(Some(a)) => a,
|
||||
Ok(None) => return Ok(None),
|
||||
Err(e) => {
|
||||
tracing::debug!(
|
||||
target: "kebab-app",
|
||||
path = %asset.workspace_path.0,
|
||||
error = %e,
|
||||
"skip-check: get_asset_by_workspace_path failed; falling through to re-process"
|
||||
);
|
||||
return Ok(None);
|
||||
}
|
||||
};
|
||||
if existing_asset.checksum != asset.checksum {
|
||||
return Ok(None);
|
||||
}
|
||||
let candidate_doc_id = kebab_core::id_for_doc(
|
||||
&asset.workspace_path,
|
||||
&asset.asset_id,
|
||||
current_parser_version,
|
||||
);
|
||||
let existing_doc = match app.sqlite.get_document(&candidate_doc_id) {
|
||||
Ok(Some(d)) => d,
|
||||
Ok(None) => return Ok(None),
|
||||
Err(e) => {
|
||||
@@ -790,21 +796,37 @@ fn try_skip_unchanged(
|
||||
target: "kebab-app",
|
||||
path = %asset.workspace_path.0,
|
||||
error = %e,
|
||||
"skip-check: get_document failed; falling through to re-process"
|
||||
"skip-check: get_document_by_workspace_path failed; falling through to re-process"
|
||||
);
|
||||
return Ok(None);
|
||||
}
|
||||
};
|
||||
// 1. Content unchanged: the freshly-computed asset_id (blake3
|
||||
// content hash) must match what this document was ingested from.
|
||||
if existing_doc.source_asset_id != asset.asset_id {
|
||||
return Ok(None);
|
||||
}
|
||||
// 2. Parser unchanged: parser_version is baked into id_for_doc so
|
||||
// a version bump yields a different doc_id and the row above
|
||||
// would have been missing. Checking here explicitly keeps the
|
||||
// logic self-documenting and guards against future id_for_doc
|
||||
// changes.
|
||||
if existing_doc.parser_version != *current_parser_version {
|
||||
return Ok(None);
|
||||
}
|
||||
// 3. Chunker unchanged.
|
||||
let chunker_match = existing_doc.last_chunker_version.as_ref()
|
||||
== Some(current_chunker_version);
|
||||
if !chunker_match {
|
||||
return Ok(None);
|
||||
}
|
||||
// 4. Embedder unchanged.
|
||||
let embedder_match = existing_doc.last_embedding_version.as_ref()
|
||||
== current_embedding_version;
|
||||
if !embedder_match {
|
||||
return Ok(None);
|
||||
}
|
||||
let candidate_doc_id = existing_doc.doc_id.clone();
|
||||
tracing::debug!(
|
||||
target: "kebab-app::ingest",
|
||||
path = %asset.workspace_path.0,
|
||||
@@ -903,7 +925,24 @@ fn ingest_one_asset(
|
||||
force_reingest,
|
||||
);
|
||||
}
|
||||
_ => {
|
||||
// p10-1A-2 / 1B: code ingest dispatch.
|
||||
MediaType::Code(lang)
|
||||
if matches!(lang.as_str(), "rust" | "python" | "typescript" | "javascript") =>
|
||||
{
|
||||
return ingest_one_code_asset(
|
||||
app,
|
||||
asset,
|
||||
chunk_policy,
|
||||
embedder,
|
||||
vector_store,
|
||||
existing_doc_ids,
|
||||
force_reingest,
|
||||
lang.as_str(),
|
||||
);
|
||||
}
|
||||
// p10-1A-2: non-Rust Code, Audio, and Other are not yet wired;
|
||||
// skip until their respective phases.
|
||||
MediaType::Code(_) | MediaType::Audio(_) | MediaType::Other(_) => {
|
||||
return Ok(kebab_core::IngestItem {
|
||||
kind: kebab_core::IngestItemKind::Skipped,
|
||||
doc_id: None,
|
||||
@@ -1603,6 +1642,213 @@ fn ingest_one_pdf_asset(
|
||||
})
|
||||
}
|
||||
|
||||
/// p10-1A-2 Task 8: process one `MediaType::Code("rust")` asset end-to-end.
|
||||
///
|
||||
/// Mirrors `ingest_one_pdf_asset` line-for-line with the substitutions
|
||||
/// documented in the task spec:
|
||||
/// - parser_version → `code-rust-v1` (via `RUST_PARSER_VERSION`)
|
||||
/// - extractor → `RustAstExtractor`
|
||||
/// - chunker → `CodeRustAstV1Chunker`
|
||||
///
|
||||
/// All other steps (incremental skip, byte read, ExtractContext, put_*,
|
||||
/// embed, purge_vector_orphans) are identical to the PDF function.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn ingest_one_code_asset(
|
||||
app: &App,
|
||||
asset: &RawAsset,
|
||||
chunk_policy: &ChunkPolicy,
|
||||
embedder: Option<&Arc<dyn Embedder + Send + Sync>>,
|
||||
vector_store: Option<&Arc<kebab_store_vector::LanceVectorStore>>,
|
||||
existing_doc_ids: &std::collections::HashSet<String>,
|
||||
force_reingest: bool,
|
||||
code_lang: &str, // <-- NEW (p10-1b Task D)
|
||||
) -> anyhow::Result<kebab_core::IngestItem> {
|
||||
let path = match &asset.source_uri {
|
||||
SourceUri::File(p) => p.clone(),
|
||||
SourceUri::Kb(_) => {
|
||||
return Ok(kebab_core::IngestItem {
|
||||
kind: kebab_core::IngestItemKind::Skipped,
|
||||
doc_id: None,
|
||||
doc_path: asset.workspace_path.clone(),
|
||||
asset_id: Some(asset.asset_id.clone()),
|
||||
byte_len: Some(asset.byte_len),
|
||||
block_count: None,
|
||||
chunk_count: None,
|
||||
parser_version: None,
|
||||
chunker_version: None,
|
||||
warnings: vec![
|
||||
"kb:// URI not yet supported".to_string(),
|
||||
],
|
||||
error: None,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
// p10-1b Task D/G/J: parser_version per-lang.
|
||||
let parser_version = match code_lang {
|
||||
"rust" => ParserVersion(kebab_parse_code::RUST_PARSER_VERSION.to_string()),
|
||||
"python" => ParserVersion(kebab_parse_code::PYTHON_PARSER_VERSION.to_string()),
|
||||
"typescript" => ParserVersion(kebab_parse_code::TS_PARSER_VERSION.to_string()),
|
||||
"javascript" => ParserVersion(kebab_parse_code::JS_PARSER_VERSION.to_string()),
|
||||
other => anyhow::bail!("unsupported code_lang: {other}"),
|
||||
};
|
||||
|
||||
// p10-1b Task D/G/J/L: chunker_version per-lang.
|
||||
let chunker_version = match code_lang {
|
||||
"rust" => CodeRustAstV1Chunker.chunker_version(),
|
||||
"python" => CodePythonAstV1Chunker.chunker_version(),
|
||||
"typescript" => CodeTsAstV1Chunker.chunker_version(),
|
||||
"javascript" => CodeJsAstV1Chunker.chunker_version(),
|
||||
other => anyhow::bail!("unreachable chunker_version: {other}"),
|
||||
};
|
||||
|
||||
if let Some(item) = try_skip_unchanged(
|
||||
app,
|
||||
asset,
|
||||
&parser_version,
|
||||
&chunker_version,
|
||||
embedder.map(|e| e.model_version()).as_ref(),
|
||||
force_reingest,
|
||||
)? {
|
||||
return Ok(item);
|
||||
}
|
||||
let bytes = std::fs::read(&path)
|
||||
.with_context(|| format!("read code asset bytes from {}", path.display()))?;
|
||||
|
||||
let extract_config = kebab_core::ExtractConfig::default();
|
||||
let workspace_root = app.config.resolve_workspace_root();
|
||||
let ctx = ExtractContext {
|
||||
asset,
|
||||
workspace_root: &workspace_root,
|
||||
config: &extract_config,
|
||||
};
|
||||
|
||||
// p10-1b Task D/G/J/L: extractor per-lang.
|
||||
let mut canonical = match code_lang {
|
||||
"rust" => RustAstExtractor::new()
|
||||
.extract(&ctx, &bytes)
|
||||
.context("kb-parse-code::RustAstExtractor::extract (code:rust)")?,
|
||||
"python" => PythonAstExtractor::new()
|
||||
.extract(&ctx, &bytes)
|
||||
.context("kb-parse-code::PythonAstExtractor::extract (code:python)")?,
|
||||
"typescript" => TypescriptAstExtractor::new()
|
||||
.extract(&ctx, &bytes)
|
||||
.context("kb-parse-code::TypescriptAstExtractor::extract (code:typescript)")?,
|
||||
"javascript" => JavascriptAstExtractor::new()
|
||||
.extract(&ctx, &bytes)
|
||||
.context("kb-parse-code::JavascriptAstExtractor::extract (code:javascript)")?,
|
||||
other => anyhow::bail!("unreachable (extract): {other}"),
|
||||
};
|
||||
|
||||
// p10-1b Task D/G/J/L: chunker per-lang.
|
||||
let chunks = match code_lang {
|
||||
"rust" => CodeRustAstV1Chunker
|
||||
.chunk(&canonical, chunk_policy)
|
||||
.context("kb-chunk::CodeRustAstV1Chunker::chunk (code:rust)")?,
|
||||
"python" => CodePythonAstV1Chunker
|
||||
.chunk(&canonical, chunk_policy)
|
||||
.context("kb-chunk::CodePythonAstV1Chunker::chunk (code:python)")?,
|
||||
"typescript" => CodeTsAstV1Chunker
|
||||
.chunk(&canonical, chunk_policy)
|
||||
.context("kb-chunk::CodeTsAstV1Chunker::chunk (code:typescript)")?,
|
||||
"javascript" => CodeJsAstV1Chunker
|
||||
.chunk(&canonical, chunk_policy)
|
||||
.context("kb-chunk::CodeJsAstV1Chunker::chunk (code:javascript)")?,
|
||||
other => anyhow::bail!("unreachable (chunk): {other}"),
|
||||
};
|
||||
|
||||
// Stamp chunker + embedding versions so incremental skip detection has
|
||||
// data on the second run.
|
||||
canonical.last_chunker_version = Some(chunker_version.clone());
|
||||
if let Some(emb) = embedder {
|
||||
canonical.last_embedding_version = Some(emb.model_version());
|
||||
}
|
||||
|
||||
purge_vector_orphans_for_workspace_path(app, asset, vector_store)?;
|
||||
app.sqlite
|
||||
.put_asset_with_bytes(asset, &bytes)
|
||||
.context("DocumentStore::put_asset_with_bytes (code)")?;
|
||||
app.sqlite
|
||||
.put_document(&canonical)
|
||||
.context("DocumentStore::put_document (code)")?;
|
||||
app.sqlite
|
||||
.put_blocks(&canonical.doc_id, &canonical.blocks)
|
||||
.context("DocumentStore::put_blocks (code)")?;
|
||||
app.sqlite
|
||||
.put_chunks(&canonical.doc_id, &chunks)
|
||||
.context("DocumentStore::put_chunks (code)")?;
|
||||
|
||||
if let (Some(emb), Some(vec_store)) = (embedder, vector_store)
|
||||
&& !chunks.is_empty()
|
||||
{
|
||||
let inputs: Vec<EmbeddingInput<'_>> = chunks
|
||||
.iter()
|
||||
.map(|c| EmbeddingInput {
|
||||
text: c.text.as_str(),
|
||||
kind: EmbeddingKind::Document,
|
||||
})
|
||||
.collect();
|
||||
let vectors = emb
|
||||
.embed(&inputs)
|
||||
.context("Embedder::embed (code chunks)")?;
|
||||
let model_id = emb.model_id();
|
||||
let model_version = emb.model_version();
|
||||
let dimensions = emb.dimensions();
|
||||
let records: Vec<VectorRecord> = chunks
|
||||
.iter()
|
||||
.zip(vectors)
|
||||
.map(|(c, v)| VectorRecord {
|
||||
embedding_id: kebab_core::id_for_embedding(
|
||||
&c.chunk_id,
|
||||
&model_id,
|
||||
&model_version,
|
||||
dimensions,
|
||||
),
|
||||
chunk_id: c.chunk_id.clone(),
|
||||
vector: v,
|
||||
doc_id: canonical.doc_id.clone(),
|
||||
text: c.text.clone(),
|
||||
heading_path: c.heading_path.clone(),
|
||||
model_id: model_id.clone(),
|
||||
model_version: model_version.clone(),
|
||||
dimensions,
|
||||
})
|
||||
.collect();
|
||||
vec_store
|
||||
.upsert(&records)
|
||||
.context("VectorStore::upsert (code)")?;
|
||||
}
|
||||
|
||||
let kind = if existing_doc_ids.contains(&canonical.doc_id.0) {
|
||||
kebab_core::IngestItemKind::Updated
|
||||
} else {
|
||||
kebab_core::IngestItemKind::New
|
||||
};
|
||||
|
||||
// Surface every `Provenance::Warning` note onto `IngestItem.warnings`.
|
||||
let warnings: Vec<String> = canonical
|
||||
.provenance
|
||||
.events
|
||||
.iter()
|
||||
.filter(|e| e.kind == kebab_core::ProvenanceKind::Warning)
|
||||
.filter_map(|e| e.note.clone())
|
||||
.collect();
|
||||
|
||||
Ok(kebab_core::IngestItem {
|
||||
kind,
|
||||
doc_id: Some(canonical.doc_id.clone()),
|
||||
doc_path: asset.workspace_path.clone(),
|
||||
asset_id: Some(asset.asset_id.clone()),
|
||||
byte_len: Some(asset.byte_len),
|
||||
block_count: u32::try_from(canonical.blocks.len()).ok(),
|
||||
chunk_count: u32::try_from(chunks.len()).ok(),
|
||||
parser_version: Some(canonical.parser_version.clone()),
|
||||
chunker_version: Some(chunker_version),
|
||||
warnings,
|
||||
error: None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Pull the BCP-47 language hint from the canonical document. P6-1
|
||||
/// stamps `Lang("und")` by default; image-pipeline OCR / caption
|
||||
/// adapters special-case "und" so the hint is intentionally dropped
|
||||
@@ -1737,6 +1983,19 @@ pub fn search_uncached_with_config(
|
||||
App::open_with_config(config)?.search_uncached(query)
|
||||
}
|
||||
|
||||
/// p9-fb-34: budget-aware search free function. Mirrors
|
||||
/// [`search_with_config`] but threads `SearchOpts` (max_tokens,
|
||||
/// snippet_chars, cursor) and returns the [`SearchResponse`]
|
||||
/// pagination wrapper. Tasks 6+8 surface this via CLI / MCP.
|
||||
#[doc(hidden)]
|
||||
pub fn search_with_opts_with_config(
|
||||
config: kebab_config::Config,
|
||||
query: kebab_core::SearchQuery,
|
||||
opts: kebab_core::SearchOpts,
|
||||
) -> anyhow::Result<SearchResponse> {
|
||||
App::open_with_config(config)?.search_with_opts(query, opts)
|
||||
}
|
||||
|
||||
// ── ask ──────────────────────────────────────────────────────────────────
|
||||
//
|
||||
// P4-3 wires `ask` end-to-end. The retriever is built per `opts.mode`;
|
||||
|
||||
@@ -32,6 +32,7 @@ pub struct Capabilities {
|
||||
pub http_daemon: bool,
|
||||
pub mcp_server: bool,
|
||||
pub single_file_ingest: bool,
|
||||
pub bulk_search: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
@@ -44,12 +45,32 @@ pub struct Models {
|
||||
pub corpus_revision: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
pub struct Stats {
|
||||
pub doc_count: u64,
|
||||
pub chunk_count: u64,
|
||||
pub asset_count: u64,
|
||||
pub last_ingest_at: Option<String>,
|
||||
/// p9-fb-37: per-media-kind doc count (5 keys, zero-padded).
|
||||
#[serde(default)]
|
||||
pub media_breakdown: std::collections::BTreeMap<String, u64>,
|
||||
/// p9-fb-37: per-language doc count, NULL keyed as `"null"`.
|
||||
#[serde(default)]
|
||||
pub lang_breakdown: std::collections::BTreeMap<String, u64>,
|
||||
/// p9-fb-37: on-disk byte sums.
|
||||
#[serde(default)]
|
||||
pub index_bytes: kebab_core::IndexBytes,
|
||||
/// p9-fb-37: docs whose `updated_at` exceeds the staleness threshold.
|
||||
#[serde(default)]
|
||||
pub stale_doc_count: u64,
|
||||
/// p10-1A-1: code language breakdown (chunk counts by canonical lowercase
|
||||
/// language identifier). Empty until 1A-2 produces code chunks.
|
||||
#[serde(default)]
|
||||
pub code_lang_breakdown: std::collections::BTreeMap<String, u32>,
|
||||
/// p10-1A-1: repo breakdown (chunk counts by `metadata.repo` value).
|
||||
/// Empty until 1A-2 produces code chunks.
|
||||
#[serde(default)]
|
||||
pub repo_breakdown: std::collections::BTreeMap<String, u32>,
|
||||
}
|
||||
|
||||
const KEBAB_VERSION: &str = env!("CARGO_PKG_VERSION");
|
||||
@@ -63,6 +84,7 @@ pub const SCHEMA_V1_ID: &str = "schema.v1";
|
||||
const WIRE_SCHEMAS: &[&str] = &[
|
||||
"answer.v1",
|
||||
"search_hit.v1",
|
||||
"search_response.v1",
|
||||
"doc_summary.v1",
|
||||
"chunk_inspection.v1",
|
||||
"doctor.v1",
|
||||
@@ -72,6 +94,8 @@ const WIRE_SCHEMAS: &[&str] = &[
|
||||
"citation.v1",
|
||||
"schema.v1",
|
||||
"error.v1",
|
||||
"bulk_search_item.v1",
|
||||
"bulk_search_response.v1",
|
||||
];
|
||||
|
||||
/// Build a [`SchemaV1`] introspection report for the given config.
|
||||
@@ -84,7 +108,7 @@ const WIRE_SCHEMAS: &[&str] = &[
|
||||
#[doc(hidden)]
|
||||
pub fn schema_with_config(cfg: &Config) -> anyhow::Result<SchemaV1> {
|
||||
let store = open_store_for_stats(cfg)?;
|
||||
let stats = collect_stats(&store)?;
|
||||
let stats = collect_stats(cfg, &store)?;
|
||||
let models = collect_models(cfg, &store);
|
||||
Ok(SchemaV1 {
|
||||
schema_version: SCHEMA_V1_ID.to_string(),
|
||||
@@ -110,6 +134,7 @@ fn capabilities_snapshot() -> Capabilities {
|
||||
http_daemon: false,
|
||||
mcp_server: true,
|
||||
single_file_ingest: false,
|
||||
bulk_search: true,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -123,13 +148,29 @@ fn open_store_for_stats(cfg: &Config) -> anyhow::Result<kebab_store_sqlite::Sqli
|
||||
kebab_store_sqlite::SqliteStore::open_existing(&db_path)
|
||||
}
|
||||
|
||||
fn collect_stats(store: &kebab_store_sqlite::SqliteStore) -> anyhow::Result<Stats> {
|
||||
let counts = store.count_summary()?;
|
||||
fn collect_stats(
|
||||
cfg: &Config,
|
||||
store: &kebab_store_sqlite::SqliteStore,
|
||||
) -> anyhow::Result<Stats> {
|
||||
let counts = store
|
||||
.count_summary_with_threshold(cfg.search.stale_threshold_days as u64)?;
|
||||
let data_dir = kebab_config::expand_path(&cfg.storage.data_dir, "");
|
||||
let index_bytes = kebab_store_sqlite::stats_ext::index_bytes(&data_dir)
|
||||
.map_err(|e| anyhow::anyhow!("index_bytes: {e}"))?;
|
||||
Ok(Stats {
|
||||
doc_count: counts.doc_count,
|
||||
chunk_count: counts.chunk_count,
|
||||
asset_count: counts.asset_count,
|
||||
last_ingest_at: counts.last_ingest_at,
|
||||
media_breakdown: counts.media_breakdown,
|
||||
lang_breakdown: counts.lang_breakdown,
|
||||
index_bytes,
|
||||
stale_doc_count: counts.stale_doc_count,
|
||||
// p10-1A-2: populated by the store query added in this task.
|
||||
code_lang_breakdown: store.code_lang_breakdown()?,
|
||||
// p10-1A-2 follow-up: dogfooding (2026-05-20) revealed this was a
|
||||
// placeholder — mirror of code_lang_breakdown for the repo field.
|
||||
repo_breakdown: store.repo_breakdown()?,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -149,3 +190,57 @@ fn collect_models(cfg: &Config, store: &kebab_store_sqlite::SqliteStore) -> Mode
|
||||
corpus_revision: store.corpus_revision(),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests_stats_ext {
|
||||
use super::*;
|
||||
|
||||
/// p10-1A-1: Stats must serialize `code_lang_breakdown` and
|
||||
/// `repo_breakdown` so downstream consumers (MCP skill, Claude Code)
|
||||
/// can branch on their presence.
|
||||
#[test]
|
||||
fn stats_includes_code_lang_and_repo_breakdown_fields() {
|
||||
let stats = Stats::default();
|
||||
let v = serde_json::to_value(&stats).unwrap();
|
||||
assert!(
|
||||
v.get("code_lang_breakdown").is_some(),
|
||||
"Stats JSON must include code_lang_breakdown: {v}"
|
||||
);
|
||||
assert!(
|
||||
v.get("repo_breakdown").is_some(),
|
||||
"Stats JSON must include repo_breakdown: {v}"
|
||||
);
|
||||
// Empty BTreeMap serializes as `{}` — confirm it's an object, not null.
|
||||
assert!(
|
||||
v["code_lang_breakdown"].is_object(),
|
||||
"code_lang_breakdown must be an object: {v}"
|
||||
);
|
||||
assert!(
|
||||
v["repo_breakdown"].is_object(),
|
||||
"repo_breakdown must be an object: {v}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stats_includes_breakdowns_and_bytes_on_fresh_corpus() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut cfg = kebab_config::Config::defaults();
|
||||
cfg.storage.data_dir = dir.path().to_string_lossy().into_owned();
|
||||
// Bring up migrations so the sqlite file is created.
|
||||
let store = kebab_store_sqlite::SqliteStore::open(&cfg).unwrap();
|
||||
store.run_migrations().unwrap();
|
||||
drop(store);
|
||||
|
||||
let s = schema_with_config(&cfg).unwrap();
|
||||
// 5 keys padded.
|
||||
assert_eq!(s.stats.media_breakdown.len(), 5);
|
||||
assert_eq!(s.stats.media_breakdown.get("markdown"), Some(&0));
|
||||
assert_eq!(s.stats.media_breakdown.get("pdf"), Some(&0));
|
||||
// lang map empty on empty corpus.
|
||||
assert!(s.stats.lang_breakdown.is_empty());
|
||||
// sqlite bytes positive after migrations, lancedb 0.
|
||||
assert!(s.stats.index_bytes.sqlite > 0);
|
||||
assert_eq!(s.stats.index_bytes.lancedb, 0);
|
||||
assert_eq!(s.stats.stale_doc_count, 0);
|
||||
}
|
||||
}
|
||||
|
||||
77
crates/kebab-app/src/staleness.rs
Normal file
77
crates/kebab-app/src/staleness.rs
Normal file
@@ -0,0 +1,77 @@
|
||||
//! p9-fb-32 staleness helpers.
|
||||
|
||||
use time::{Duration, OffsetDateTime};
|
||||
|
||||
use kebab_core::SearchHit;
|
||||
|
||||
/// Returns `true` iff `now - indexed_at > threshold_days * 24h`.
|
||||
/// `threshold_days = 0` always returns `false` (feature disabled).
|
||||
/// Strict `>` so that exactly `threshold_days` old returns `false`.
|
||||
///
|
||||
/// p9-fb-32: mirrored in `kebab_rag::pipeline::compute_stale` (dep-boundary
|
||||
/// rule prevents `kebab-rag → kebab-app`). Update both together.
|
||||
pub fn compute_stale(
|
||||
indexed_at: OffsetDateTime,
|
||||
now: OffsetDateTime,
|
||||
threshold_days: u32,
|
||||
) -> bool {
|
||||
if threshold_days == 0 {
|
||||
return false;
|
||||
}
|
||||
let threshold = Duration::days(i64::from(threshold_days));
|
||||
(now - indexed_at) > threshold
|
||||
}
|
||||
|
||||
/// Sets `stale` on each hit in place using `compute_stale`.
|
||||
pub fn mark_stale_in_place(
|
||||
hits: &mut [SearchHit],
|
||||
now: OffsetDateTime,
|
||||
threshold_days: u32,
|
||||
) {
|
||||
for h in hits {
|
||||
h.stale = compute_stale(h.indexed_at, now, threshold_days);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use time::macros::datetime;
|
||||
|
||||
fn now() -> OffsetDateTime {
|
||||
datetime!(2026-05-09 12:00:00 UTC)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn threshold_zero_always_fresh() {
|
||||
let very_old = datetime!(2020-01-01 00:00:00 UTC);
|
||||
assert!(!compute_stale(very_old, now(), 0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn just_under_threshold_is_fresh() {
|
||||
// 29 days, 23h, 59m old — under 30d.
|
||||
let indexed = now() - Duration::days(29) - Duration::hours(23) - Duration::minutes(59);
|
||||
assert!(!compute_stale(indexed, now(), 30));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exactly_threshold_is_fresh() {
|
||||
// strict `>` boundary: exactly 30d old is still fresh.
|
||||
let indexed = now() - Duration::days(30);
|
||||
assert!(!compute_stale(indexed, now(), 30));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn one_minute_past_threshold_is_stale() {
|
||||
let indexed = now() - Duration::days(30) - Duration::minutes(1);
|
||||
assert!(compute_stale(indexed, now(), 30));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn future_indexed_at_is_fresh() {
|
||||
// clock skew safety: future timestamps must not be stale.
|
||||
let future = now() + Duration::hours(1);
|
||||
assert!(!compute_stale(future, now(), 30));
|
||||
}
|
||||
}
|
||||
431
crates/kebab-app/tests/code_ingest_smoke.rs
Normal file
431
crates/kebab-app/tests/code_ingest_smoke.rs
Normal file
@@ -0,0 +1,431 @@
|
||||
//! p10-1A-2 Task 8: smoke test for Rust code ingest dispatch.
|
||||
//!
|
||||
//! Writes a single `.rs` file into a TempDir workspace, ingests it via
|
||||
//! `kebab_app::ingest_with_config`, then searches for the symbol name and
|
||||
//! asserts that the resulting `SearchHit` carries a `Citation::Code`
|
||||
//! with the expected `lang`, `symbol`, and `line_start`.
|
||||
//!
|
||||
//! Mirrors the `pdf_pipeline.rs` harness: lexical-only (no AVX/fastembed),
|
||||
//! no OCR / caption adapters needed.
|
||||
|
||||
mod common;
|
||||
|
||||
use common::{TestEnv, lexical_query};
|
||||
|
||||
use kebab_core::{Citation, IngestItemKind};
|
||||
|
||||
/// A `.rs` file with a single `pub fn add` symbol is ingested, and a
|
||||
/// lexical search for "add" must return at least one `Citation::Code`
|
||||
/// hit whose `lang == "rust"`, `symbol == Some("add")`, and
|
||||
/// `line_start >= 1`.
|
||||
#[test]
|
||||
fn rust_file_ingests_and_searches_as_code_citation() {
|
||||
let env = TestEnv::lexical_only();
|
||||
|
||||
// Write a minimal Rust file into the workspace root.
|
||||
std::fs::write(
|
||||
env.workspace_root.join("demo.rs"),
|
||||
"/// adds two integers\npub fn add(a: i32, b: i32) -> i32 {\n a + b\n}\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let report =
|
||||
kebab_app::ingest_with_config(env.config.clone(), env.scope(), false)
|
||||
.expect("ingest must succeed");
|
||||
|
||||
assert_eq!(report.errors, 0, "no errors expected: {report:?}");
|
||||
let items = report.items.as_ref().expect("items present");
|
||||
let code_item = items
|
||||
.iter()
|
||||
.find(|i| i.doc_path.0.ends_with("demo.rs"))
|
||||
.expect("demo.rs item present");
|
||||
assert_eq!(
|
||||
code_item.kind,
|
||||
IngestItemKind::New,
|
||||
"first ingest must be New: {code_item:?}"
|
||||
);
|
||||
assert!(
|
||||
code_item.block_count.unwrap_or(0) >= 1,
|
||||
"at least one block expected: {code_item:?}"
|
||||
);
|
||||
assert!(
|
||||
code_item.chunk_count.unwrap_or(0) >= 1,
|
||||
"at least one chunk expected: {code_item:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
code_item.parser_version.as_ref().map(|p| p.0.as_str()),
|
||||
Some("code-rust-v1"),
|
||||
"parser_version must be code-rust-v1"
|
||||
);
|
||||
assert_eq!(
|
||||
code_item.chunker_version.as_ref().map(|c| c.0.as_str()),
|
||||
Some("code-rust-ast-v1"),
|
||||
"chunker_version must be code-rust-ast-v1"
|
||||
);
|
||||
|
||||
// Lexical search for the symbol name "add".
|
||||
let hits = kebab_app::search_with_config(env.config.clone(), lexical_query("add"))
|
||||
.expect("search must succeed");
|
||||
|
||||
let h = hits
|
||||
.iter()
|
||||
.find(|h| matches!(&h.citation, Citation::Code { .. }))
|
||||
.expect("at least one Citation::Code hit for 'add'");
|
||||
|
||||
match &h.citation {
|
||||
Citation::Code {
|
||||
lang,
|
||||
symbol,
|
||||
line_start,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(
|
||||
lang.as_deref(),
|
||||
Some("rust"),
|
||||
"citation.lang must be 'rust'"
|
||||
);
|
||||
assert_eq!(
|
||||
symbol.as_deref(),
|
||||
Some("add"),
|
||||
"citation.symbol must be 'add'"
|
||||
);
|
||||
assert!(*line_start >= 1, "line_start must be ≥1");
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
h.code_lang.as_deref(),
|
||||
Some("rust"),
|
||||
"SearchHit.code_lang must be 'rust'"
|
||||
);
|
||||
}
|
||||
|
||||
/// p10-1A-2 Task 8b: a code search hit must carry `SearchHit.repo` filled
|
||||
/// from the document's `Metadata.repo` (which is set by `detect_repo` during
|
||||
/// ingest). `detect_repo` returns the name of the directory that contains
|
||||
/// `.git/`, so we `git init` the workspace root before ingesting and then
|
||||
/// assert that `h.repo == Some("workspace")`.
|
||||
#[test]
|
||||
fn rust_code_search_hit_has_repo() {
|
||||
let env = TestEnv::lexical_only();
|
||||
|
||||
// `detect_repo` walks up from the file looking for `.git/`.
|
||||
// Initialise a bare git repo at the workspace root so it is
|
||||
// discoverable. We only need the `.git/` directory — no commits
|
||||
// required.
|
||||
let git_status = std::process::Command::new("git")
|
||||
.args(["init", "--quiet"])
|
||||
.arg(env.workspace_root.as_os_str())
|
||||
.status()
|
||||
.expect("git init");
|
||||
assert!(git_status.success(), "git init must succeed");
|
||||
|
||||
std::fs::write(
|
||||
env.workspace_root.join("repo_demo.rs"),
|
||||
"/// multiplies two integers\npub fn mul(a: i32, b: i32) -> i32 {\n a * b\n}\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let report =
|
||||
kebab_app::ingest_with_config(env.config.clone(), env.scope(), false)
|
||||
.expect("ingest must succeed");
|
||||
assert_eq!(report.errors, 0, "no ingest errors: {report:?}");
|
||||
|
||||
let hits = kebab_app::search_with_config(env.config.clone(), lexical_query("mul"))
|
||||
.expect("search must succeed");
|
||||
|
||||
let h = hits
|
||||
.iter()
|
||||
.find(|h| matches!(&h.citation, Citation::Code { .. }))
|
||||
.expect("at least one Citation::Code hit for 'mul'");
|
||||
|
||||
// The workspace root directory is named "workspace" by `TestEnv`.
|
||||
let expected_repo = env
|
||||
.workspace_root
|
||||
.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.map(str::to_owned);
|
||||
assert_eq!(
|
||||
h.repo,
|
||||
expected_repo,
|
||||
"SearchHit.repo must match the workspace dir name (detect_repo result)"
|
||||
);
|
||||
// Also sanity-check code_lang is still filled.
|
||||
assert_eq!(
|
||||
h.code_lang.as_deref(),
|
||||
Some("rust"),
|
||||
"SearchHit.code_lang must be 'rust'"
|
||||
);
|
||||
}
|
||||
|
||||
/// p10-1b Task G: a `.py` file in a sub-directory is ingested and the
|
||||
/// resulting `Citation::Code` hit must carry `lang="python"`,
|
||||
/// `symbol="kebab_eval.metrics.compute_mrr"`, and `line_start >= 1`.
|
||||
/// The sub-directory (`kebab_eval/`) ensures `module_path_for_python`
|
||||
/// produces a non-empty prefix so the fully-qualified symbol assertion
|
||||
/// exercises the prefix wiring end-to-end.
|
||||
#[test]
|
||||
fn python_file_ingests_and_searches_as_code_citation() {
|
||||
let env = TestEnv::lexical_only();
|
||||
|
||||
let module_dir = env.workspace_root.join("kebab_eval");
|
||||
std::fs::create_dir_all(&module_dir).unwrap();
|
||||
std::fs::write(
|
||||
module_dir.join("metrics.py"),
|
||||
"\"\"\"compute metrics.\"\"\"\ndef compute_mrr(scores):\n return sum(scores) / max(len(scores), 1)\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let report =
|
||||
kebab_app::ingest_with_config(env.config.clone(), env.scope(), false)
|
||||
.expect("ingest must succeed");
|
||||
|
||||
assert!(report.new >= 1, "python file ingested: {report:?}");
|
||||
|
||||
let items = report.items.as_ref().expect("items present");
|
||||
let py_item = items
|
||||
.iter()
|
||||
.find(|i| i.doc_path.0.ends_with("metrics.py"))
|
||||
.expect("metrics.py item");
|
||||
assert_eq!(
|
||||
py_item.parser_version.as_ref().map(|p| p.0.as_str()),
|
||||
Some("code-python-v1"),
|
||||
"parser_version must be code-python-v1"
|
||||
);
|
||||
assert_eq!(
|
||||
py_item.chunker_version.as_ref().map(|c| c.0.as_str()),
|
||||
Some("code-python-ast-v1"),
|
||||
"chunker_version must be code-python-ast-v1"
|
||||
);
|
||||
|
||||
let hits = kebab_app::search_with_config(env.config.clone(), lexical_query("compute_mrr"))
|
||||
.expect("search must succeed");
|
||||
|
||||
let h = hits
|
||||
.iter()
|
||||
.find(|h| matches!(&h.citation, Citation::Code { .. }))
|
||||
.expect("at least one Citation::Code hit for 'compute_mrr'");
|
||||
|
||||
match &h.citation {
|
||||
Citation::Code {
|
||||
lang,
|
||||
symbol,
|
||||
line_start,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(
|
||||
lang.as_deref(),
|
||||
Some("python"),
|
||||
"citation.lang must be 'python'"
|
||||
);
|
||||
assert_eq!(
|
||||
symbol.as_deref(),
|
||||
Some("kebab_eval.metrics.compute_mrr"),
|
||||
"citation.symbol must be 'kebab_eval.metrics.compute_mrr'"
|
||||
);
|
||||
assert!(*line_start >= 1, "line_start must be >=1");
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
h.code_lang.as_deref(),
|
||||
Some("python"),
|
||||
"SearchHit.code_lang must be 'python'"
|
||||
);
|
||||
}
|
||||
|
||||
/// p10-1b Task J: a `.ts` file in a sub-directory is ingested and the
|
||||
/// resulting `Citation::Code` hit must carry `lang="typescript"`,
|
||||
/// `symbol="src/Foo.Foo.bar"`, and `line_start >= 1`.
|
||||
/// The sub-directory (`src/`) ensures `module_path_for_tsjs` produces
|
||||
/// a non-empty prefix so the fully-qualified symbol assertion exercises
|
||||
/// the prefix wiring end-to-end.
|
||||
#[test]
|
||||
fn typescript_file_ingests_and_searches_as_code_citation() {
|
||||
let env = TestEnv::lexical_only();
|
||||
|
||||
let src_dir = env.workspace_root.join("src");
|
||||
std::fs::create_dir_all(&src_dir).unwrap();
|
||||
std::fs::write(
|
||||
src_dir.join("Foo.ts"),
|
||||
"export class Foo {\n bar(): number { return 42; }\n}\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let report =
|
||||
kebab_app::ingest_with_config(env.config.clone(), env.scope(), false)
|
||||
.expect("ingest must succeed");
|
||||
|
||||
assert!(report.new >= 1, "ts file ingested: {report:?}");
|
||||
|
||||
let items = report.items.as_ref().expect("items present");
|
||||
let ts_item = items
|
||||
.iter()
|
||||
.find(|i| i.doc_path.0.ends_with("Foo.ts"))
|
||||
.expect("Foo.ts item");
|
||||
assert_eq!(
|
||||
ts_item.parser_version.as_ref().map(|p| p.0.as_str()),
|
||||
Some("code-ts-v1"),
|
||||
"parser_version must be code-ts-v1"
|
||||
);
|
||||
assert_eq!(
|
||||
ts_item.chunker_version.as_ref().map(|c| c.0.as_str()),
|
||||
Some("code-ts-ast-v1"),
|
||||
"chunker_version must be code-ts-ast-v1"
|
||||
);
|
||||
|
||||
let hits = kebab_app::search_with_config(env.config.clone(), lexical_query("bar"))
|
||||
.expect("search must succeed");
|
||||
|
||||
let h = hits
|
||||
.iter()
|
||||
.find(|h| matches!(&h.citation, Citation::Code { .. }))
|
||||
.expect("at least one Citation::Code hit for 'bar'");
|
||||
|
||||
match &h.citation {
|
||||
Citation::Code {
|
||||
lang,
|
||||
symbol,
|
||||
line_start,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(
|
||||
lang.as_deref(),
|
||||
Some("typescript"),
|
||||
"citation.lang must be 'typescript'"
|
||||
);
|
||||
assert_eq!(
|
||||
symbol.as_deref(),
|
||||
Some("src/Foo.Foo.bar"),
|
||||
"citation.symbol must be 'src/Foo.Foo.bar'"
|
||||
);
|
||||
assert!(*line_start >= 1, "line_start must be >=1");
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
h.code_lang.as_deref(),
|
||||
Some("typescript"),
|
||||
"SearchHit.code_lang must be 'typescript'"
|
||||
);
|
||||
}
|
||||
|
||||
/// p10-1b Task L: a `.js` file in a sub-directory is ingested and the
|
||||
/// resulting `Citation::Code` hit must carry `lang="javascript"`,
|
||||
/// `symbol="src/Bar.Bar.baz"`, and `line_start >= 1`.
|
||||
/// The sub-directory (`src/`) ensures `module_path_for_tsjs` produces
|
||||
/// a non-empty prefix so the fully-qualified symbol assertion exercises
|
||||
/// the prefix wiring end-to-end.
|
||||
#[test]
|
||||
fn javascript_file_ingests_and_searches_as_code_citation() {
|
||||
let env = TestEnv::lexical_only();
|
||||
|
||||
let src_dir = env.workspace_root.join("src");
|
||||
std::fs::create_dir_all(&src_dir).unwrap();
|
||||
std::fs::write(
|
||||
src_dir.join("Bar.js"),
|
||||
"export class Bar {\n baz() { return 7; }\n}\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let report =
|
||||
kebab_app::ingest_with_config(env.config.clone(), env.scope(), false)
|
||||
.expect("ingest must succeed");
|
||||
|
||||
assert!(report.new >= 1, "js file ingested: {report:?}");
|
||||
|
||||
let items = report.items.as_ref().expect("items present");
|
||||
let js_item = items
|
||||
.iter()
|
||||
.find(|i| i.doc_path.0.ends_with("Bar.js"))
|
||||
.expect("Bar.js item");
|
||||
assert_eq!(
|
||||
js_item.parser_version.as_ref().map(|p| p.0.as_str()),
|
||||
Some("code-js-v1"),
|
||||
"parser_version must be code-js-v1"
|
||||
);
|
||||
assert_eq!(
|
||||
js_item.chunker_version.as_ref().map(|c| c.0.as_str()),
|
||||
Some("code-js-ast-v1"),
|
||||
"chunker_version must be code-js-ast-v1"
|
||||
);
|
||||
|
||||
let hits = kebab_app::search_with_config(env.config.clone(), lexical_query("baz"))
|
||||
.expect("search must succeed");
|
||||
|
||||
let h = hits
|
||||
.iter()
|
||||
.find(|h| matches!(&h.citation, Citation::Code { .. }))
|
||||
.expect("at least one Citation::Code hit for 'baz'");
|
||||
|
||||
match &h.citation {
|
||||
Citation::Code {
|
||||
lang,
|
||||
symbol,
|
||||
line_start,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(
|
||||
lang.as_deref(),
|
||||
Some("javascript"),
|
||||
"citation.lang must be 'javascript'"
|
||||
);
|
||||
assert_eq!(
|
||||
symbol.as_deref(),
|
||||
Some("src/Bar.Bar.baz"),
|
||||
"citation.symbol must be 'src/Bar.Bar.baz'"
|
||||
);
|
||||
assert!(*line_start >= 1, "line_start must be >=1");
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
h.code_lang.as_deref(),
|
||||
Some("javascript"),
|
||||
"SearchHit.code_lang must be 'javascript'"
|
||||
);
|
||||
}
|
||||
|
||||
/// Re-ingesting the same `.rs` file without changes must report
|
||||
/// `Unchanged` (incremental-skip path exercised).
|
||||
#[test]
|
||||
fn rust_file_re_ingest_is_unchanged() {
|
||||
let env = TestEnv::lexical_only();
|
||||
|
||||
std::fs::write(
|
||||
env.workspace_root.join("stable.rs"),
|
||||
"pub fn noop() {}\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let r1 =
|
||||
kebab_app::ingest_with_config(env.config.clone(), env.scope(), false).unwrap();
|
||||
let item1 = r1
|
||||
.items
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.find(|i| i.doc_path.0.ends_with("stable.rs"))
|
||||
.cloned()
|
||||
.unwrap();
|
||||
assert_eq!(item1.kind, IngestItemKind::New);
|
||||
|
||||
let r2 =
|
||||
kebab_app::ingest_with_config(env.config.clone(), env.scope(), false).unwrap();
|
||||
let item2 = r2
|
||||
.items
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.find(|i| i.doc_path.0.ends_with("stable.rs"))
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
item2.kind,
|
||||
IngestItemKind::Unchanged,
|
||||
"identical bytes → Unchanged"
|
||||
);
|
||||
assert_eq!(item2.doc_id, item1.doc_id);
|
||||
}
|
||||
@@ -79,6 +79,37 @@ impl TestEnv {
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// p9-fb-34 alias — tests added in fb-34 invoke `TestEnv::new()`
|
||||
/// per the plan; route to the existing lexical-only constructor
|
||||
/// so the lane stays AVX-free without churning all the existing
|
||||
/// callers.
|
||||
pub fn new() -> Self {
|
||||
Self::lexical_only()
|
||||
}
|
||||
|
||||
/// p9-fb-34: open a fresh `App` against this env's config. Used
|
||||
/// by integration tests that need to call `App::search_with_opts`
|
||||
/// directly. Caller can invoke this multiple times to simulate
|
||||
/// re-opening the binary after a corpus revision bump.
|
||||
pub fn app(&self) -> kebab_app::App {
|
||||
kebab_app::App::open_with_config(self.config.clone())
|
||||
.expect("App::open_with_config")
|
||||
}
|
||||
}
|
||||
|
||||
/// p9-fb-34: write `content` into the env's workspace at
|
||||
/// `relative_path`, then run a full ingest so the document is
|
||||
/// searchable. Mirrors the convenience helpers used by other
|
||||
/// `TestEnv`-driven crates.
|
||||
pub fn ingest_md(env: &TestEnv, relative_path: &str, content: &str) {
|
||||
let path = env.workspace_root.join(relative_path);
|
||||
if let Some(parent) = path.parent() {
|
||||
std::fs::create_dir_all(parent).expect("create parent dirs");
|
||||
}
|
||||
std::fs::write(&path, content).expect("write workspace file");
|
||||
kebab_app::ingest_with_config(env.config.clone(), env.scope(), true)
|
||||
.expect("ingest_with_config");
|
||||
}
|
||||
|
||||
/// Test helper: build a `SearchQuery` for lexical mode at k=10. Used
|
||||
@@ -94,6 +125,29 @@ pub fn lexical_query(text: &str) -> kebab_core::SearchQuery {
|
||||
}
|
||||
}
|
||||
|
||||
/// p9-fb-32: rewrite `documents.updated_at` for one workspace path
|
||||
/// to `now - days_ago` (RFC3339 UTC). Used by staleness integration
|
||||
/// tests to simulate aged-out docs without faking system time. Caller
|
||||
/// is responsible for ingesting the doc *before* calling this — the
|
||||
/// row must already exist.
|
||||
pub fn backdate_document_updated_at(env: &TestEnv, workspace_path: &str, days_ago: i64) {
|
||||
let backdated = (time::OffsetDateTime::now_utc() - time::Duration::days(days_ago))
|
||||
.format(&time::format_description::well_known::Rfc3339)
|
||||
.expect("format backdated updated_at");
|
||||
let db_path = PathBuf::from(&env.config.storage.data_dir).join("kebab.sqlite");
|
||||
let conn = rusqlite::Connection::open(&db_path).expect("open kebab.sqlite");
|
||||
let updated = conn
|
||||
.execute(
|
||||
"UPDATE documents SET updated_at = ?1 WHERE workspace_path = ?2",
|
||||
rusqlite::params![backdated, workspace_path],
|
||||
)
|
||||
.expect("UPDATE documents.updated_at");
|
||||
assert_eq!(
|
||||
updated, 1,
|
||||
"backdate_document_updated_at: expected to update exactly 1 row for {workspace_path}, got {updated}"
|
||||
);
|
||||
}
|
||||
|
||||
fn copy_fixture_workspace(dest: &Path) {
|
||||
let src = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("tests")
|
||||
|
||||
24
crates/kebab-app/tests/cursor.rs
Normal file
24
crates/kebab-app/tests/cursor.rs
Normal file
@@ -0,0 +1,24 @@
|
||||
//! p9-fb-34: cursor encode/decode round-trip + corpus_revision mismatch.
|
||||
|
||||
use kebab_app::cursor;
|
||||
|
||||
#[test]
|
||||
fn cursor_roundtrip_preserves_offset() {
|
||||
let encoded = cursor::encode(5, "rev-abc");
|
||||
let offset = cursor::decode(&encoded, "rev-abc").unwrap();
|
||||
assert_eq!(offset, 5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cursor_decode_rejects_mismatched_revision() {
|
||||
let encoded = cursor::encode(7, "rev-old");
|
||||
let err = cursor::decode(&encoded, "rev-new").unwrap_err();
|
||||
assert_eq!(err.code, "stale_cursor");
|
||||
assert!(err.message.contains("rev-old") || err.message.contains("rev-new"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cursor_decode_rejects_garbage_input() {
|
||||
let err = cursor::decode("not-base64!!!", "any").unwrap_err();
|
||||
assert_eq!(err.code, "stale_cursor");
|
||||
}
|
||||
329
crates/kebab-app/tests/fetch_integration.rs
Normal file
329
crates/kebab-app/tests/fetch_integration.rs
Normal file
@@ -0,0 +1,329 @@
|
||||
//! p9-fb-35 App::fetch integration tests.
|
||||
|
||||
mod common;
|
||||
|
||||
use kebab_app::App;
|
||||
use kebab_core::{FetchKind, FetchOpts, FetchQuery};
|
||||
|
||||
fn open(env: &common::TestEnv) -> App {
|
||||
env.app()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fetch_chunk_returns_target_only_when_no_context() {
|
||||
let env = common::TestEnv::new();
|
||||
common::ingest_md(&env, "a.md", "# Title\n\nFirst paragraph.\n\n## Section\n\nSecond.\n");
|
||||
let app = open(&env);
|
||||
|
||||
// Find a chunk via search to obtain its id.
|
||||
let q = kebab_core::SearchQuery {
|
||||
text: "First".to_string(),
|
||||
mode: kebab_core::SearchMode::Lexical,
|
||||
k: 1,
|
||||
filters: kebab_core::SearchFilters::default(),
|
||||
};
|
||||
let hits = app.search(q).unwrap();
|
||||
let chunk_id = hits[0].chunk_id.clone();
|
||||
|
||||
let result = app
|
||||
.fetch(FetchQuery::Chunk(chunk_id), FetchOpts::default())
|
||||
.unwrap();
|
||||
assert_eq!(result.kind, FetchKind::Chunk);
|
||||
assert!(result.chunk.is_some(), "target chunk populated");
|
||||
assert!(result.context_before.is_empty());
|
||||
assert!(result.context_after.is_empty());
|
||||
assert!(!result.truncated);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fetch_chunk_with_context_returns_neighbors() {
|
||||
let env = common::TestEnv::new();
|
||||
let body = "# H1\n\nA1\n\n# H2\n\nA2\n\n# H3\n\nA3\n\n# H4\n\nA4\n\n# H5\n\nA5\n";
|
||||
common::ingest_md(&env, "multi.md", body);
|
||||
let app = env.app();
|
||||
|
||||
let q = kebab_core::SearchQuery {
|
||||
text: "A3".to_string(),
|
||||
mode: kebab_core::SearchMode::Lexical,
|
||||
k: 1,
|
||||
filters: kebab_core::SearchFilters::default(),
|
||||
};
|
||||
let hits = app.search(q).unwrap();
|
||||
let chunk_id = hits[0].chunk_id.clone();
|
||||
|
||||
let result = app
|
||||
.fetch(
|
||||
FetchQuery::Chunk(chunk_id),
|
||||
FetchOpts {
|
||||
context: Some(2),
|
||||
max_tokens: None,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(result.kind, FetchKind::Chunk);
|
||||
assert!(result.chunk.is_some());
|
||||
let total = result.context_before.len() + result.context_after.len();
|
||||
assert!(total >= 1, "at least one neighbor expected");
|
||||
assert!(total <= 4, "context capped at +-2 ⇒ max 4 neighbors");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fetch_chunk_unknown_id_returns_chunk_not_found() {
|
||||
let env = common::TestEnv::new();
|
||||
let app = env.app();
|
||||
let err = app
|
||||
.fetch(
|
||||
FetchQuery::Chunk(kebab_core::ChunkId("nonexistent-id".to_string())),
|
||||
FetchOpts::default(),
|
||||
)
|
||||
.unwrap_err();
|
||||
let msg = err.to_string();
|
||||
assert!(
|
||||
msg.contains("chunk_not_found") || msg.contains("nonexistent-id"),
|
||||
"expected chunk_not_found error, got: {msg}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fetch_doc_returns_serialized_markdown() {
|
||||
let env = common::TestEnv::new();
|
||||
let body = "# Heading One\n\nFirst paragraph.\n\n## Sub\n\nSecond.\n";
|
||||
common::ingest_md(&env, "doc.md", body);
|
||||
let app = env.app();
|
||||
|
||||
// Discover doc_id via search hit (avoids depending on list_docs API shape).
|
||||
let q = kebab_core::SearchQuery {
|
||||
text: "First".to_string(),
|
||||
mode: kebab_core::SearchMode::Lexical,
|
||||
k: 1,
|
||||
filters: kebab_core::SearchFilters::default(),
|
||||
};
|
||||
let hits = app.search(q).unwrap();
|
||||
let doc_id = hits[0].doc_id.clone();
|
||||
|
||||
let result = app
|
||||
.fetch(FetchQuery::Doc(doc_id), FetchOpts::default())
|
||||
.unwrap();
|
||||
assert_eq!(result.kind, FetchKind::Doc);
|
||||
let text = result.text.expect("doc text");
|
||||
assert!(text.contains("Heading One"), "doc text contains heading: {text:?}");
|
||||
assert!(text.contains("First paragraph"), "doc text contains body");
|
||||
assert!(!result.truncated);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fetch_doc_unknown_id_returns_doc_not_found() {
|
||||
let env = common::TestEnv::new();
|
||||
let app = env.app();
|
||||
let err = app
|
||||
.fetch(
|
||||
FetchQuery::Doc(kebab_core::DocumentId("nonexistent-doc".to_string())),
|
||||
FetchOpts::default(),
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(err.to_string().contains("doc_not_found"), "got: {err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fetch_doc_with_max_tokens_truncates() {
|
||||
let env = common::TestEnv::new();
|
||||
let p = "Lorem ipsum dolor sit amet consectetur adipiscing elit. ".repeat(20);
|
||||
let body = format!("# Big\n\n{p}\n");
|
||||
common::ingest_md(&env, "big.md", &body);
|
||||
let app = env.app();
|
||||
let q = kebab_core::SearchQuery {
|
||||
text: "Lorem".to_string(),
|
||||
mode: kebab_core::SearchMode::Lexical,
|
||||
k: 1,
|
||||
filters: kebab_core::SearchFilters::default(),
|
||||
};
|
||||
let hits = app.search(q).unwrap();
|
||||
let doc_id = hits[0].doc_id.clone();
|
||||
|
||||
let result = app
|
||||
.fetch(
|
||||
FetchQuery::Doc(doc_id),
|
||||
FetchOpts {
|
||||
context: None,
|
||||
max_tokens: Some(20), // ~80 chars
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
assert!(result.truncated);
|
||||
let text = result.text.expect("doc text");
|
||||
assert!(text.chars().count() <= 100, "trimmed text len {}", text.chars().count());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fetch_span_returns_line_range() {
|
||||
let env = common::TestEnv::new();
|
||||
// Use a list so the canonical-to-markdown roundtrip emits 5
|
||||
// single-line entries joined by `\n` (paragraphs would be joined by
|
||||
// `\n\n`, and CommonMark soft breaks inside one paragraph collapse to
|
||||
// spaces — see crates/kebab-parse-md/src/blocks.rs `Event::SoftBreak`).
|
||||
let body = "- Line one.\n- Line two.\n- Line three.\n- Line four.\n- Line five.\n";
|
||||
common::ingest_md(&env, "lines.md", body);
|
||||
let app = env.app();
|
||||
|
||||
let q = kebab_core::SearchQuery {
|
||||
text: "Line".to_string(),
|
||||
mode: kebab_core::SearchMode::Lexical,
|
||||
k: 1,
|
||||
filters: kebab_core::SearchFilters::default(),
|
||||
};
|
||||
let hits = app.search(q).unwrap();
|
||||
let doc_id = hits[0].doc_id.clone();
|
||||
|
||||
let result = app
|
||||
.fetch(
|
||||
FetchQuery::Span {
|
||||
doc_id,
|
||||
line_start: 2,
|
||||
line_end: 4,
|
||||
},
|
||||
FetchOpts::default(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(result.kind, FetchKind::Span);
|
||||
let text = result.text.expect("span text");
|
||||
let line_count = text.lines().count();
|
||||
assert_eq!(line_count, 3, "span should be 3 lines: {text:?}");
|
||||
assert_eq!(result.line_start, Some(2));
|
||||
assert_eq!(result.line_end, Some(4));
|
||||
assert_eq!(result.effective_end, Some(4));
|
||||
assert!(!result.truncated);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fetch_span_clamps_line_end_when_out_of_range() {
|
||||
let env = common::TestEnv::new();
|
||||
common::ingest_md(&env, "short.md", "Line one.\nLine two.\n");
|
||||
let app = env.app();
|
||||
let q = kebab_core::SearchQuery {
|
||||
text: "Line".to_string(),
|
||||
mode: kebab_core::SearchMode::Lexical,
|
||||
k: 1,
|
||||
filters: kebab_core::SearchFilters::default(),
|
||||
};
|
||||
let hits = app.search(q).unwrap();
|
||||
let doc_id = hits[0].doc_id.clone();
|
||||
|
||||
let result = app
|
||||
.fetch(
|
||||
FetchQuery::Span {
|
||||
doc_id,
|
||||
line_start: 1,
|
||||
line_end: 999,
|
||||
},
|
||||
FetchOpts::default(),
|
||||
)
|
||||
.unwrap();
|
||||
let text = result.text.expect("span text");
|
||||
let actual_lines = text.lines().count();
|
||||
assert_eq!(result.effective_end, Some(actual_lines as u32));
|
||||
assert!(actual_lines < 999);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fetch_span_invalid_input_when_zero_lines() {
|
||||
let env = common::TestEnv::new();
|
||||
common::ingest_md(&env, "a.md", "Line one.\n");
|
||||
let app = env.app();
|
||||
let q = kebab_core::SearchQuery {
|
||||
text: "Line".to_string(),
|
||||
mode: kebab_core::SearchMode::Lexical,
|
||||
k: 1,
|
||||
filters: kebab_core::SearchFilters::default(),
|
||||
};
|
||||
let hits = app.search(q).unwrap();
|
||||
let doc_id = hits[0].doc_id.clone();
|
||||
|
||||
let err = app
|
||||
.fetch(
|
||||
FetchQuery::Span {
|
||||
doc_id,
|
||||
line_start: 0,
|
||||
line_end: 0,
|
||||
},
|
||||
FetchOpts::default(),
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(err.to_string().contains("invalid_input"), "got: {err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fetch_span_line_start_beyond_total_returns_empty_text() {
|
||||
let env = common::TestEnv::new();
|
||||
let body = "- Line one.\n- Line two.\n";
|
||||
common::ingest_md(&env, "two_lines.md", body);
|
||||
let app = env.app();
|
||||
let q = kebab_core::SearchQuery {
|
||||
text: "Line".to_string(),
|
||||
mode: kebab_core::SearchMode::Lexical,
|
||||
k: 1,
|
||||
filters: kebab_core::SearchFilters::default(),
|
||||
};
|
||||
let hits = app.search(q).unwrap();
|
||||
let doc_id = hits[0].doc_id.clone();
|
||||
|
||||
let result = app
|
||||
.fetch(
|
||||
FetchQuery::Span {
|
||||
doc_id,
|
||||
line_start: 100,
|
||||
line_end: 200,
|
||||
},
|
||||
FetchOpts::default(),
|
||||
)
|
||||
.unwrap();
|
||||
let text = result.text.expect("text field");
|
||||
assert!(text.is_empty(), "out-of-range request returns empty text");
|
||||
assert!(
|
||||
!result.truncated,
|
||||
"out-of-range is NOT truncated (budget-only flag)"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fetch_chunk_context_at_first_chunk_clamps_lower_bound() {
|
||||
let env = common::TestEnv::new();
|
||||
// Multi-chunk markdown so context ±N has neighbors.
|
||||
let body =
|
||||
"# H1\n\nFirst chunk text body.\n\n# H2\n\nSecond chunk.\n\n# H3\n\nThird chunk.\n";
|
||||
common::ingest_md(&env, "boundary.md", body);
|
||||
let app = env.app();
|
||||
let q = kebab_core::SearchQuery {
|
||||
text: "First".to_string(),
|
||||
mode: kebab_core::SearchMode::Lexical,
|
||||
k: 1,
|
||||
filters: kebab_core::SearchFilters::default(),
|
||||
};
|
||||
let hits = app.search(q).unwrap();
|
||||
let chunk_id = hits[0].chunk_id.clone();
|
||||
|
||||
let result = app
|
||||
.fetch(
|
||||
FetchQuery::Chunk(chunk_id),
|
||||
FetchOpts {
|
||||
context: Some(2),
|
||||
max_tokens: None,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
// p9-fb-35 R2: doc has 3 chunks; ±2 should clamp the total
|
||||
// neighbor count to ≤ 2 + 1 (= excludes target).
|
||||
//
|
||||
// ⚠ Strict "first-chunk → context_before is empty" cannot be
|
||||
// asserted here yet because chunks.ordinal column does not exist
|
||||
// — `list_chunk_ids_for_doc` orders by `(created_at, chunk_id)`
|
||||
// and chunk_id is a blake3 hash, so the "First chunk" content
|
||||
// may land at any hash-order position within the doc. The clamp
|
||||
// logic itself is correct (target_idx ± n → [0..len]); we just
|
||||
// can't pin which chunk is hash-order-first. Tracked as
|
||||
// follow-up: V007 chunks.ordinal migration.
|
||||
let total = result.context_before.len() + result.context_after.len();
|
||||
assert!(
|
||||
total <= 2,
|
||||
"doc with 3 chunks ±2 → at most 2 neighbors (excludes target), got {total}"
|
||||
);
|
||||
}
|
||||
165
crates/kebab-app/tests/search_budget_integration.rs
Normal file
165
crates/kebab-app/tests/search_budget_integration.rs
Normal file
@@ -0,0 +1,165 @@
|
||||
//! p9-fb-34: App::search_with_opts integration tests.
|
||||
|
||||
mod common;
|
||||
|
||||
use kebab_app::SearchResponse;
|
||||
use kebab_core::{SearchFilters, SearchMode, SearchOpts, SearchQuery};
|
||||
|
||||
fn lex(text: &str, k: usize) -> SearchQuery {
|
||||
SearchQuery {
|
||||
text: text.to_string(),
|
||||
mode: SearchMode::Lexical,
|
||||
k,
|
||||
filters: SearchFilters::default(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn search_with_opts_no_budget_matches_search() {
|
||||
let env = common::TestEnv::new();
|
||||
common::ingest_md(&env, "a.md", "# T\n\napples are red\n");
|
||||
let app = env.app();
|
||||
|
||||
let baseline = app.search(lex("apples", 5)).unwrap();
|
||||
let resp: SearchResponse = app
|
||||
.search_with_opts(lex("apples", 5), SearchOpts::default())
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.hits.len(), baseline.len());
|
||||
assert!(!resp.truncated);
|
||||
assert!(resp.next_cursor.is_none(), "k=5 against 1 doc → no next page");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn budget_truncates_snippets_when_below_threshold() {
|
||||
let env = common::TestEnv::new();
|
||||
let body: String = "rust ownership is a memory model. ".repeat(10);
|
||||
common::ingest_md(&env, "a.md", &format!("# T\n\n{body}\n"));
|
||||
let app = env.app();
|
||||
|
||||
let unrestricted = app.search(lex("rust", 5)).unwrap();
|
||||
let unrestricted_chars: usize = unrestricted.iter().map(|h| h.snippet.chars().count()).sum();
|
||||
|
||||
let resp = app
|
||||
.search_with_opts(
|
||||
lex("rust", 5),
|
||||
SearchOpts {
|
||||
max_tokens: Some(50),
|
||||
snippet_chars: None,
|
||||
cursor: None,
|
||||
trace: false,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
let limited_chars: usize = resp.hits.iter().map(|h| h.snippet.chars().count()).sum();
|
||||
|
||||
assert!(resp.truncated, "small budget must trip truncation");
|
||||
assert!(limited_chars < unrestricted_chars, "snippet should shrink");
|
||||
assert!(!resp.hits.is_empty(), "always retain ≥1 hit");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cursor_paginates_to_next_page() {
|
||||
let env = common::TestEnv::new();
|
||||
for i in 0..6 {
|
||||
common::ingest_md(&env, &format!("d{i}.md"), &format!("# T{i}\n\nrust topic {i}\n"));
|
||||
}
|
||||
let app = env.app();
|
||||
|
||||
let page1 = app
|
||||
.search_with_opts(lex("rust", 2), SearchOpts::default())
|
||||
.unwrap();
|
||||
assert_eq!(page1.hits.len(), 2);
|
||||
let cursor = page1.next_cursor.expect("more hits available");
|
||||
|
||||
let page2 = app
|
||||
.search_with_opts(
|
||||
lex("rust", 2),
|
||||
SearchOpts {
|
||||
max_tokens: None,
|
||||
snippet_chars: None,
|
||||
cursor: Some(cursor),
|
||||
trace: false,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(page2.hits.len(), 2);
|
||||
let p1_ids: std::collections::HashSet<_> =
|
||||
page1.hits.iter().map(|h| h.chunk_id.0.clone()).collect();
|
||||
let p2_ids: std::collections::HashSet<_> =
|
||||
page2.hits.iter().map(|h| h.chunk_id.0.clone()).collect();
|
||||
assert!(p1_ids.is_disjoint(&p2_ids), "page 2 must not repeat page 1 hits");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cursor_rejected_after_corpus_revision_bump() {
|
||||
let env = common::TestEnv::new();
|
||||
common::ingest_md(&env, "a.md", "# T\n\napples\n");
|
||||
let app = env.app();
|
||||
|
||||
let page1 = app
|
||||
.search_with_opts(lex("apples", 1), SearchOpts::default())
|
||||
.unwrap();
|
||||
// p9-fb-34 round-1 review: replaced silent `if let Some(c) = ...`
|
||||
// with `.expect(...)` so a fixture regression that breaks the
|
||||
// cursor-emission contract fails loudly instead of passing vacuously.
|
||||
let c = page1
|
||||
.next_cursor
|
||||
.expect("k=1 page must emit next_cursor — fixture too small if this fails");
|
||||
|
||||
common::ingest_md(&env, "b.md", "# B\n\nbananas\n");
|
||||
let app2 = env.app();
|
||||
|
||||
let result = app2.search_with_opts(
|
||||
lex("apples", 1),
|
||||
SearchOpts {
|
||||
max_tokens: None,
|
||||
snippet_chars: None,
|
||||
cursor: Some(c),
|
||||
trace: false,
|
||||
},
|
||||
);
|
||||
let err = result.unwrap_err();
|
||||
assert!(
|
||||
err.to_string().contains("stale_cursor"),
|
||||
"must surface stale_cursor: {err}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn max_tokens_zero_returns_one_hit_truncated() {
|
||||
// p9-fb-34 round-1 review: pin the documented "≥1 hit floor"
|
||||
// contract — even with `max_tokens=0` (an absurdly tight budget)
|
||||
// the budget loop must keep one hit and flip `truncated: true`.
|
||||
// Fixture intentionally seeds multiple matches so step 2 of the
|
||||
// budget loop (pop hits to 1) actually fires.
|
||||
let env = common::TestEnv::new();
|
||||
for i in 0..3 {
|
||||
common::ingest_md(
|
||||
&env,
|
||||
&format!("d{i}.md"),
|
||||
&format!("# T{i}\n\napples are red {i}\n"),
|
||||
);
|
||||
}
|
||||
let app = env.app();
|
||||
|
||||
let resp = app
|
||||
.search_with_opts(
|
||||
lex("apples", 5),
|
||||
SearchOpts {
|
||||
max_tokens: Some(0),
|
||||
snippet_chars: None,
|
||||
cursor: None,
|
||||
trace: false,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(resp.hits.len(), 1, "max_tokens=0 collapses to 1-hit floor");
|
||||
assert!(resp.truncated);
|
||||
// p9-fb-34 R2: cursor IS emitted on k-pop case so the popped
|
||||
// hits remain reachable.
|
||||
assert!(
|
||||
resp.next_cursor.is_some(),
|
||||
"k-pop truncation must still emit next_cursor; popped hits at offset+returned"
|
||||
);
|
||||
}
|
||||
87
crates/kebab-app/tests/search_stale_integration.rs
Normal file
87
crates/kebab-app/tests/search_stale_integration.rs
Normal file
@@ -0,0 +1,87 @@
|
||||
//! p9-fb-32: `App::search` end-to-end staleness wiring.
|
||||
//!
|
||||
//! `compute_stale` itself is unit-tested in `kebab_app::staleness`; this
|
||||
//! file proves the post-process actually fires through the full
|
||||
//! retriever stack and that the cache-hit re-stamp respects the
|
||||
//! configured threshold.
|
||||
//!
|
||||
//! All three tests run lexical-only (no AVX, no fastembed download).
|
||||
|
||||
mod common;
|
||||
|
||||
use common::TestEnv;
|
||||
|
||||
fn lexical_query_owner() -> kebab_core::SearchQuery {
|
||||
common::lexical_query("ownership")
|
||||
}
|
||||
|
||||
/// Fresh ingest at default 30-day threshold → no hit can be stale.
|
||||
/// `documents.updated_at` is stamped at ingest time (now), so the
|
||||
/// distance to `now_utc()` is sub-second.
|
||||
#[test]
|
||||
fn fresh_doc_is_not_stale_with_default_threshold() {
|
||||
let env = TestEnv::lexical_only();
|
||||
kebab_app::ingest_with_config(env.config.clone(), env.scope(), true).unwrap();
|
||||
|
||||
let app = kebab_app::App::open_with_config(env.config.clone()).unwrap();
|
||||
let hits = app.search(lexical_query_owner()).unwrap();
|
||||
assert!(!hits.is_empty(), "expected ≥1 hit for 'ownership'");
|
||||
assert!(
|
||||
hits.iter().all(|h| !h.stale),
|
||||
"freshly-ingested doc must not be stale at default 30d threshold: {:?}",
|
||||
hits.iter().map(|h| (h.doc_path.0.clone(), h.stale)).collect::<Vec<_>>()
|
||||
);
|
||||
}
|
||||
|
||||
/// `stale_threshold_days = 0` disables the feature even for very old
|
||||
/// `documents.updated_at`. Backdate the row to a year ago, expect
|
||||
/// `stale: false` on every hit.
|
||||
#[test]
|
||||
fn threshold_zero_disables_staleness() {
|
||||
let mut env = TestEnv::lexical_only();
|
||||
env.config.search.stale_threshold_days = 0;
|
||||
|
||||
kebab_app::ingest_with_config(env.config.clone(), env.scope(), true).unwrap();
|
||||
common::backdate_document_updated_at(&env, "intro.md", 365);
|
||||
|
||||
let app = kebab_app::App::open_with_config(env.config.clone()).unwrap();
|
||||
let hits = app.search(lexical_query_owner()).unwrap();
|
||||
assert!(!hits.is_empty(), "expected ≥1 hit");
|
||||
assert!(
|
||||
hits.iter().all(|h| !h.stale),
|
||||
"threshold=0 disables staleness even for year-old docs: {:?}",
|
||||
hits.iter().map(|h| (h.doc_path.0.clone(), h.stale)).collect::<Vec<_>>()
|
||||
);
|
||||
}
|
||||
|
||||
/// At a 30-day threshold, a 60-day-old `documents.updated_at` must
|
||||
/// surface as stale on the matching hit. (Other hits — fresh fixtures
|
||||
/// not backdated — stay fresh, so we use `any` not `all`.)
|
||||
#[test]
|
||||
fn old_doc_marked_stale() {
|
||||
let mut env = TestEnv::lexical_only();
|
||||
env.config.search.stale_threshold_days = 30;
|
||||
|
||||
kebab_app::ingest_with_config(env.config.clone(), env.scope(), true).unwrap();
|
||||
common::backdate_document_updated_at(&env, "intro.md", 60);
|
||||
|
||||
let app = kebab_app::App::open_with_config(env.config.clone()).unwrap();
|
||||
let hits = app.search(lexical_query_owner()).unwrap();
|
||||
assert!(!hits.is_empty(), "expected ≥1 hit");
|
||||
let intro_hits: Vec<&kebab_core::SearchHit> = hits
|
||||
.iter()
|
||||
.filter(|h| h.doc_path.0.ends_with("intro.md"))
|
||||
.collect();
|
||||
assert!(
|
||||
!intro_hits.is_empty(),
|
||||
"expected ≥1 hit on intro.md (the backdated doc)"
|
||||
);
|
||||
assert!(
|
||||
intro_hits.iter().all(|h| h.stale),
|
||||
"60-day-old intro.md must be stale at 30d threshold: {:?}",
|
||||
intro_hits
|
||||
.iter()
|
||||
.map(|h| (h.doc_path.0.clone(), h.stale))
|
||||
.collect::<Vec<_>>()
|
||||
);
|
||||
}
|
||||
90
crates/kebab-app/tests/twin_files_idempotent.rs
Normal file
90
crates/kebab-app/tests/twin_files_idempotent.rs
Normal file
@@ -0,0 +1,90 @@
|
||||
//! Regression test for the twin-file idempotency bug.
|
||||
//!
|
||||
//! Identical-content files at different workspace paths share one
|
||||
//! `assets` row (`asset_id` = blake3 content hash, PRIMARY KEY). The
|
||||
//! old UPSERT `ON CONFLICT(asset_id) DO UPDATE SET workspace_path =
|
||||
//! excluded.workspace_path` made each twin overwrite the other's path
|
||||
//! on every ingest, so `get_asset_by_workspace_path(path1)` returned
|
||||
//! None (or the wrong twin) → re-process every time.
|
||||
//!
|
||||
//! Fix: `try_skip_unchanged` now uses `get_document_by_workspace_path`
|
||||
//! instead. `documents.workspace_path` is UNIQUE (V001) so each twin
|
||||
//! has its own stable document row.
|
||||
//!
|
||||
//! Assertion contract:
|
||||
//! 1st ingest → 2 New (one per twin)
|
||||
//! 2nd ingest → 0 New, 0 Updated, 2 Unchanged
|
||||
|
||||
mod common;
|
||||
|
||||
use common::TestEnv;
|
||||
use kebab_app::ingest_with_config;
|
||||
use kebab_core::IngestItemKind;
|
||||
|
||||
#[test]
|
||||
fn twin_files_second_ingest_is_unchanged() {
|
||||
let env = TestEnv::lexical_only();
|
||||
|
||||
// Write two files with identical content at different paths.
|
||||
let pkg_a = env.workspace_root.join("pkg_a");
|
||||
let pkg_b = env.workspace_root.join("pkg_b");
|
||||
std::fs::create_dir_all(&pkg_a).unwrap();
|
||||
std::fs::create_dir_all(&pkg_b).unwrap();
|
||||
|
||||
let content = b"# shared\nThis content is identical in both files.\n";
|
||||
std::fs::write(pkg_a.join("__init__.py"), content).unwrap();
|
||||
std::fs::write(pkg_b.join("__init__.py"), content).unwrap();
|
||||
|
||||
// First ingest — both files must be New.
|
||||
let first = ingest_with_config(env.config.clone(), env.scope(), false)
|
||||
.expect("first ingest must succeed");
|
||||
assert_eq!(first.errors, 0, "first ingest: no errors; report={first:?}");
|
||||
|
||||
let items = first.items.as_ref().expect("items must be present");
|
||||
let twin_items: Vec<_> = items
|
||||
.iter()
|
||||
.filter(|i| {
|
||||
i.doc_path.0.ends_with("__init__.py")
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(
|
||||
twin_items.len(),
|
||||
2,
|
||||
"first ingest: expected exactly 2 __init__.py items; items={items:?}"
|
||||
);
|
||||
for item in &twin_items {
|
||||
assert_eq!(
|
||||
item.kind,
|
||||
IngestItemKind::New,
|
||||
"first ingest: each twin must be New; item={item:?}"
|
||||
);
|
||||
}
|
||||
|
||||
// Second ingest — same files, same content → both must be Unchanged.
|
||||
let second = ingest_with_config(env.config.clone(), env.scope(), false)
|
||||
.expect("second ingest must succeed");
|
||||
assert_eq!(second.errors, 0, "second ingest: no errors; report={second:?}");
|
||||
assert_eq!(second.new, 0, "second ingest: no new docs; report={second:?}");
|
||||
assert_eq!(
|
||||
second.updated, 0,
|
||||
"second ingest: no updated docs (twin-file bug would set this to 2); report={second:?}"
|
||||
);
|
||||
|
||||
let second_items = second.items.as_ref().expect("items must be present");
|
||||
let twin_items2: Vec<_> = second_items
|
||||
.iter()
|
||||
.filter(|i| i.doc_path.0.ends_with("__init__.py"))
|
||||
.collect();
|
||||
assert_eq!(
|
||||
twin_items2.len(),
|
||||
2,
|
||||
"second ingest: expected exactly 2 __init__.py items; items={second_items:?}"
|
||||
);
|
||||
for item in &twin_items2 {
|
||||
assert_eq!(
|
||||
item.kind,
|
||||
IngestItemKind::Unchanged,
|
||||
"second ingest: each twin must be Unchanged; item={item:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
322
crates/kebab-chunk/src/code_js_ast_v1.rs
Normal file
322
crates/kebab-chunk/src/code_js_ast_v1.rs
Normal file
@@ -0,0 +1,322 @@
|
||||
//! `code-js-ast-v1` — maps a tree-sitter-derived JavaScript AST
|
||||
//! `CanonicalDocument` (one `Block::Code` per semantic unit, each with
|
||||
//! `SourceSpan::Code`) to chunks 1:1. A unit longer than
|
||||
//! `AST_CHUNK_MAX_LINES` is split into `<symbol> [part i/N]` sub-chunks
|
||||
//! at blank-line paragraph boundaries (design §9.1 oversize fallback).
|
||||
//!
|
||||
//! tree-sitter is intentionally NOT a dependency here: AST work is
|
||||
//! parser-side (`kebab-parse-code`, design §6.3). This chunker only
|
||||
//! consumes the `CanonicalDocument`.
|
||||
//!
|
||||
//! `AST_CHUNK_MAX_LINES` is a constant matching
|
||||
//! `IngestCodeCfg::default().ast_chunk_max_lines` (200). Per-medium
|
||||
//! config threading needs a chunker registry (P+); same deviation
|
||||
//! pattern as `pdf-page-v1`'s pinned `chunker_version`
|
||||
//! (`tasks/HOTFIXES.md`).
|
||||
|
||||
use kebab_core::{
|
||||
Block, BlockId, CanonicalDocument, Chunk, ChunkPolicy, Chunker, ChunkerVersion, DocumentId,
|
||||
SourceSpan, id_for_chunk,
|
||||
};
|
||||
|
||||
const VERSION_LABEL: &str = "code-js-ast-v1";
|
||||
const BYTES_PER_TOKEN: usize = 3;
|
||||
const POLICY_HASH_HEX_LEN: usize = 16;
|
||||
const AST_CHUNK_MAX_LINES: u32 = 200;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
pub struct CodeJsAstV1Chunker;
|
||||
|
||||
impl Chunker for CodeJsAstV1Chunker {
|
||||
fn chunker_version(&self) -> ChunkerVersion {
|
||||
ChunkerVersion(VERSION_LABEL.to_string())
|
||||
}
|
||||
|
||||
fn policy_hash(&self, policy: &ChunkPolicy) -> String {
|
||||
let bytes = serde_json_canonicalizer::to_vec(policy)
|
||||
.expect("canonical JSON serialization of ChunkPolicy must not fail");
|
||||
let hex = blake3::hash(&bytes).to_hex().to_string();
|
||||
hex[..POLICY_HASH_HEX_LEN].to_string()
|
||||
}
|
||||
|
||||
fn chunk(
|
||||
&self,
|
||||
doc: &CanonicalDocument,
|
||||
policy: &ChunkPolicy,
|
||||
) -> anyhow::Result<Vec<Chunk>> {
|
||||
for b in &doc.blocks {
|
||||
let c = match b {
|
||||
Block::Code(c) => c,
|
||||
_ => anyhow::bail!(
|
||||
"CodeJsAstV1Chunker only handles code docs (got non-Code block)"
|
||||
),
|
||||
};
|
||||
if !matches!(c.common.source_span, SourceSpan::Code { .. }) {
|
||||
anyhow::bail!(
|
||||
"CodeJsAstV1Chunker only handles code docs (got non-Code source_span)"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let base_policy_hash = self.policy_hash(policy);
|
||||
let chunker_version = self.chunker_version();
|
||||
let mut out: Vec<Chunk> = Vec::new();
|
||||
|
||||
for b in &doc.blocks {
|
||||
let cb = match b {
|
||||
Block::Code(c) => c,
|
||||
_ => unreachable!("validated above"),
|
||||
};
|
||||
let (ls, le, symbol, lang) = match &cb.common.source_span {
|
||||
SourceSpan::Code { line_start, line_end, symbol, lang } => {
|
||||
(*line_start, *line_end, symbol.clone(), lang.clone())
|
||||
}
|
||||
_ => unreachable!("validated above"),
|
||||
};
|
||||
let block_ids: Vec<BlockId> = vec![cb.common.block_id.clone()];
|
||||
let span_lines = le.saturating_sub(ls) + 1;
|
||||
|
||||
if span_lines <= AST_CHUNK_MAX_LINES {
|
||||
let span = SourceSpan::Code {
|
||||
line_start: ls,
|
||||
line_end: le,
|
||||
symbol: symbol.clone(),
|
||||
lang: lang.clone(),
|
||||
};
|
||||
out.push(make_chunk(
|
||||
doc, &chunker_version, &block_ids, &base_policy_hash,
|
||||
None, span, cb.code.clone(),
|
||||
));
|
||||
} else {
|
||||
let parts = split_oversize(&cb.code);
|
||||
let n = parts.len();
|
||||
for (i, (off_start, off_end, text)) in parts.into_iter().enumerate() {
|
||||
let part_ls = ls + off_start;
|
||||
let part_le = ls + off_end;
|
||||
let part_sym = symbol
|
||||
.as_ref()
|
||||
.map(|s| format!("{s} [part {}/{n}]", i + 1));
|
||||
let span = SourceSpan::Code {
|
||||
line_start: part_ls,
|
||||
line_end: part_le,
|
||||
symbol: part_sym,
|
||||
lang: lang.clone(),
|
||||
};
|
||||
out.push(make_chunk(
|
||||
doc, &chunker_version, &block_ids, &base_policy_hash,
|
||||
Some(part_ls), span, text,
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tracing::debug!(
|
||||
target: "kebab-chunk",
|
||||
doc_id = %doc.doc_id,
|
||||
chunks = out.len(),
|
||||
"code-js-ast-v1 chunked",
|
||||
);
|
||||
Ok(out)
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn make_chunk(
|
||||
doc: &CanonicalDocument,
|
||||
chunker_version: &ChunkerVersion,
|
||||
block_ids: &[BlockId],
|
||||
base_policy_hash: &str,
|
||||
split_key: Option<u32>,
|
||||
span: SourceSpan,
|
||||
text: String,
|
||||
) -> Chunk {
|
||||
let id_hash = match split_key {
|
||||
Some(k) => format!("{base_policy_hash}#L{k}"),
|
||||
None => base_policy_hash.to_string(),
|
||||
};
|
||||
let chunk_id = id_for_chunk(&doc.doc_id, chunker_version, block_ids, &id_hash);
|
||||
let token_estimate = text.len().div_ceil(BYTES_PER_TOKEN);
|
||||
Chunk {
|
||||
chunk_id,
|
||||
doc_id: DocumentId(doc.doc_id.0.clone()),
|
||||
block_ids: block_ids.to_vec(),
|
||||
text,
|
||||
heading_path: Vec::new(),
|
||||
source_spans: vec![span],
|
||||
token_estimate,
|
||||
chunker_version: chunker_version.clone(),
|
||||
policy_hash: base_policy_hash.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Split an oversize unit at blank-line paragraph boundaries, greedily
|
||||
/// gluing paragraphs until ~`AST_CHUNK_MAX_LINES` lines accumulate.
|
||||
/// Returns `(line_offset_start, line_offset_end, text)` where offsets are
|
||||
/// 0-based within the unit (caller adds the unit's absolute `line_start`).
|
||||
fn split_oversize(code: &str) -> Vec<(u32, u32, String)> {
|
||||
let lines: Vec<&str> = code.split('\n').collect();
|
||||
let total = lines.len() as u32;
|
||||
let mut out: Vec<(u32, u32, String)> = Vec::new();
|
||||
let mut start: u32 = 0;
|
||||
while start < total {
|
||||
let mut end = (start + AST_CHUNK_MAX_LINES).min(total);
|
||||
let floor = start + (AST_CHUNK_MAX_LINES * 4 / 5);
|
||||
if end < total {
|
||||
if let Some(b) = (floor.min(end)..end)
|
||||
.rev()
|
||||
.find(|&i| lines[i as usize].trim().is_empty())
|
||||
{
|
||||
end = b + 1;
|
||||
}
|
||||
}
|
||||
let text = lines[start as usize..end as usize].join("\n");
|
||||
out.push((start, end.saturating_sub(1), text));
|
||||
start = end;
|
||||
}
|
||||
if out.is_empty() {
|
||||
out.push((0, total.saturating_sub(1), code.to_string()));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use kebab_core::{
|
||||
Block, CanonicalDocument, ChunkPolicy, Chunker, ChunkerVersion, CodeBlock, CommonBlock,
|
||||
SourceSpan, id_for_block, id_for_doc, AssetId, Lang, Metadata, ParserVersion, Provenance,
|
||||
SourceType, TrustLevel, WorkspacePath,
|
||||
};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
fn code_doc(units: &[(&str, u32, u32, &str)]) -> CanonicalDocument {
|
||||
let wp = WorkspacePath("crates/x/src/a.js".into());
|
||||
let aid = AssetId("a".repeat(64));
|
||||
let pv = ParserVersion("code-js-v1".into());
|
||||
let doc_id = id_for_doc(&wp, &aid, &pv);
|
||||
let blocks = units
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, (sym, ls, le, code))| {
|
||||
let span = SourceSpan::Code {
|
||||
line_start: *ls,
|
||||
line_end: *le,
|
||||
symbol: Some((*sym).to_string()),
|
||||
lang: Some("javascript".into()),
|
||||
};
|
||||
let bid = id_for_block(&doc_id, "code", &[], i as u32, &span);
|
||||
Block::Code(CodeBlock {
|
||||
common: CommonBlock { block_id: bid, heading_path: vec![], source_span: span },
|
||||
lang: Some("javascript".into()),
|
||||
code: (*code).to_string(),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
CanonicalDocument {
|
||||
doc_id, source_asset_id: aid, workspace_path: wp, title: "a".into(),
|
||||
lang: Lang("und".into()), blocks,
|
||||
metadata: Metadata {
|
||||
aliases: vec![], tags: vec![],
|
||||
created_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
updated_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
source_type: SourceType::Note, trust_level: TrustLevel::Primary,
|
||||
user_id_alias: None, user: Default::default(),
|
||||
repo: Some("kebab".into()), git_branch: Some("main".into()),
|
||||
git_commit: Some("0".repeat(40)), code_lang: Some("javascript".into()),
|
||||
},
|
||||
provenance: Provenance { events: vec![] },
|
||||
parser_version: pv, schema_version: 1, doc_version: 1,
|
||||
last_chunker_version: None, last_embedding_version: None,
|
||||
}
|
||||
}
|
||||
fn policy() -> ChunkPolicy {
|
||||
ChunkPolicy { target_tokens: 500, overlap_tokens: 80,
|
||||
respect_markdown_headings: false,
|
||||
chunker_version: ChunkerVersion(VERSION_LABEL.into()) }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chunker_version_is_code_js_ast_v1() {
|
||||
assert_eq!(CodeJsAstV1Chunker.chunker_version(),
|
||||
ChunkerVersion("code-js-ast-v1".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn one_chunk_per_unit_preserves_code_span() {
|
||||
let doc = code_doc(&[
|
||||
("parse", 1, 3, "function parse() {\n // x\n}"),
|
||||
("Foo.double", 5, 7, "function double() {\n //\n return 0;\n}"),
|
||||
]);
|
||||
let chunks = CodeJsAstV1Chunker.chunk(&doc, &policy()).unwrap();
|
||||
assert_eq!(chunks.len(), 2);
|
||||
for c in &chunks {
|
||||
assert_eq!(c.source_spans.len(), 1);
|
||||
assert!(matches!(c.source_spans[0], SourceSpan::Code { .. }));
|
||||
assert_eq!(c.heading_path, Vec::<String>::new());
|
||||
assert_eq!(c.chunker_version.0, "code-js-ast-v1");
|
||||
}
|
||||
match &chunks[0].source_spans[0] {
|
||||
SourceSpan::Code { symbol, line_start, line_end, .. } => {
|
||||
assert_eq!(symbol.as_deref(), Some("parse"));
|
||||
assert_eq!((*line_start, *line_end), (1, 3));
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oversize_unit_splits_into_parts_with_unique_ids() {
|
||||
let body = (0..500).map(|i| format!(" const x{i} = {i};")).collect::<Vec<_>>().join("\n");
|
||||
let code = format!("function big() {{\n{body}\n}}");
|
||||
let doc = code_doc(&[("big", 1, 502, &code)]);
|
||||
let chunks = CodeJsAstV1Chunker.chunk(&doc, &policy()).unwrap();
|
||||
assert!(chunks.len() >= 2, "oversize unit must split, got {}", chunks.len());
|
||||
for c in &chunks {
|
||||
match &c.source_spans[0] {
|
||||
SourceSpan::Code { symbol, .. } => {
|
||||
assert!(symbol.as_deref().unwrap().starts_with("big [part "),
|
||||
"part-numbered symbol, got {symbol:?}");
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
let mut ids: Vec<&str> = chunks.iter().map(|c| c.chunk_id.0.as_str()).collect();
|
||||
let n = ids.len(); ids.sort(); ids.dedup();
|
||||
assert_eq!(ids.len(), n, "chunk_ids unique across split parts");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_code_doc_errors() {
|
||||
use kebab_core::TextBlock;
|
||||
let mut doc = code_doc(&[("parse", 1, 1, "function parse() {}")]);
|
||||
doc.blocks = vec![Block::Paragraph(TextBlock {
|
||||
common: CommonBlock {
|
||||
block_id: kebab_core::BlockId("b".into()),
|
||||
heading_path: vec![],
|
||||
source_span: SourceSpan::Line { start: 1, end: 1 },
|
||||
},
|
||||
text: "x".into(), inlines: vec![],
|
||||
})];
|
||||
let err = CodeJsAstV1Chunker.chunk(&doc, &policy()).unwrap_err();
|
||||
assert!(err.to_string().contains("CodeJsAstV1Chunker"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deterministic_chunk_ids_1000() {
|
||||
let doc = code_doc(&[("parse", 1, 2, "function parse() {}\n")]);
|
||||
let base: Vec<String> = CodeJsAstV1Chunker.chunk(&doc, &policy())
|
||||
.unwrap().into_iter().map(|c| c.chunk_id.0).collect();
|
||||
for _ in 0..1000 {
|
||||
let again: Vec<String> = CodeJsAstV1Chunker.chunk(&doc, &policy())
|
||||
.unwrap().into_iter().map(|c| c.chunk_id.0).collect();
|
||||
assert_eq!(again, base);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn policy_hash_matches_md_heading_v1() {
|
||||
let p = policy();
|
||||
assert_eq!(CodeJsAstV1Chunker.policy_hash(&p),
|
||||
crate::MdHeadingV1Chunker.policy_hash(&p));
|
||||
}
|
||||
}
|
||||
322
crates/kebab-chunk/src/code_python_ast_v1.rs
Normal file
322
crates/kebab-chunk/src/code_python_ast_v1.rs
Normal file
@@ -0,0 +1,322 @@
|
||||
//! `code-python-ast-v1` — maps a tree-sitter-derived Python AST
|
||||
//! `CanonicalDocument` (one `Block::Code` per semantic unit, each with
|
||||
//! `SourceSpan::Code`) to chunks 1:1. A unit longer than
|
||||
//! `AST_CHUNK_MAX_LINES` is split into `<symbol> [part i/N]` sub-chunks
|
||||
//! at blank-line paragraph boundaries (design §9.1 oversize fallback).
|
||||
//!
|
||||
//! tree-sitter is intentionally NOT a dependency here: AST work is
|
||||
//! parser-side (`kebab-parse-code`, design §6.3). This chunker only
|
||||
//! consumes the `CanonicalDocument`.
|
||||
//!
|
||||
//! `AST_CHUNK_MAX_LINES` is a constant matching
|
||||
//! `IngestCodeCfg::default().ast_chunk_max_lines` (200). Per-medium
|
||||
//! config threading needs a chunker registry (P+); same deviation
|
||||
//! pattern as `pdf-page-v1`'s pinned `chunker_version`
|
||||
//! (`tasks/HOTFIXES.md`).
|
||||
|
||||
use kebab_core::{
|
||||
Block, BlockId, CanonicalDocument, Chunk, ChunkPolicy, Chunker, ChunkerVersion, DocumentId,
|
||||
SourceSpan, id_for_chunk,
|
||||
};
|
||||
|
||||
const VERSION_LABEL: &str = "code-python-ast-v1";
|
||||
const BYTES_PER_TOKEN: usize = 3;
|
||||
const POLICY_HASH_HEX_LEN: usize = 16;
|
||||
const AST_CHUNK_MAX_LINES: u32 = 200;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
pub struct CodePythonAstV1Chunker;
|
||||
|
||||
impl Chunker for CodePythonAstV1Chunker {
|
||||
fn chunker_version(&self) -> ChunkerVersion {
|
||||
ChunkerVersion(VERSION_LABEL.to_string())
|
||||
}
|
||||
|
||||
fn policy_hash(&self, policy: &ChunkPolicy) -> String {
|
||||
let bytes = serde_json_canonicalizer::to_vec(policy)
|
||||
.expect("canonical JSON serialization of ChunkPolicy must not fail");
|
||||
let hex = blake3::hash(&bytes).to_hex().to_string();
|
||||
hex[..POLICY_HASH_HEX_LEN].to_string()
|
||||
}
|
||||
|
||||
fn chunk(
|
||||
&self,
|
||||
doc: &CanonicalDocument,
|
||||
policy: &ChunkPolicy,
|
||||
) -> anyhow::Result<Vec<Chunk>> {
|
||||
for b in &doc.blocks {
|
||||
let c = match b {
|
||||
Block::Code(c) => c,
|
||||
_ => anyhow::bail!(
|
||||
"CodePythonAstV1Chunker only handles code docs (got non-Code block)"
|
||||
),
|
||||
};
|
||||
if !matches!(c.common.source_span, SourceSpan::Code { .. }) {
|
||||
anyhow::bail!(
|
||||
"CodePythonAstV1Chunker only handles code docs (got non-Code source_span)"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let base_policy_hash = self.policy_hash(policy);
|
||||
let chunker_version = self.chunker_version();
|
||||
let mut out: Vec<Chunk> = Vec::new();
|
||||
|
||||
for b in &doc.blocks {
|
||||
let cb = match b {
|
||||
Block::Code(c) => c,
|
||||
_ => unreachable!("validated above"),
|
||||
};
|
||||
let (ls, le, symbol, lang) = match &cb.common.source_span {
|
||||
SourceSpan::Code { line_start, line_end, symbol, lang } => {
|
||||
(*line_start, *line_end, symbol.clone(), lang.clone())
|
||||
}
|
||||
_ => unreachable!("validated above"),
|
||||
};
|
||||
let block_ids: Vec<BlockId> = vec![cb.common.block_id.clone()];
|
||||
let span_lines = le.saturating_sub(ls) + 1;
|
||||
|
||||
if span_lines <= AST_CHUNK_MAX_LINES {
|
||||
let span = SourceSpan::Code {
|
||||
line_start: ls,
|
||||
line_end: le,
|
||||
symbol: symbol.clone(),
|
||||
lang: lang.clone(),
|
||||
};
|
||||
out.push(make_chunk(
|
||||
doc, &chunker_version, &block_ids, &base_policy_hash,
|
||||
None, span, cb.code.clone(),
|
||||
));
|
||||
} else {
|
||||
let parts = split_oversize(&cb.code);
|
||||
let n = parts.len();
|
||||
for (i, (off_start, off_end, text)) in parts.into_iter().enumerate() {
|
||||
let part_ls = ls + off_start;
|
||||
let part_le = ls + off_end;
|
||||
let part_sym = symbol
|
||||
.as_ref()
|
||||
.map(|s| format!("{s} [part {}/{n}]", i + 1));
|
||||
let span = SourceSpan::Code {
|
||||
line_start: part_ls,
|
||||
line_end: part_le,
|
||||
symbol: part_sym,
|
||||
lang: lang.clone(),
|
||||
};
|
||||
out.push(make_chunk(
|
||||
doc, &chunker_version, &block_ids, &base_policy_hash,
|
||||
Some(part_ls), span, text,
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tracing::debug!(
|
||||
target: "kebab-chunk",
|
||||
doc_id = %doc.doc_id,
|
||||
chunks = out.len(),
|
||||
"code-python-ast-v1 chunked",
|
||||
);
|
||||
Ok(out)
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn make_chunk(
|
||||
doc: &CanonicalDocument,
|
||||
chunker_version: &ChunkerVersion,
|
||||
block_ids: &[BlockId],
|
||||
base_policy_hash: &str,
|
||||
split_key: Option<u32>,
|
||||
span: SourceSpan,
|
||||
text: String,
|
||||
) -> Chunk {
|
||||
let id_hash = match split_key {
|
||||
Some(k) => format!("{base_policy_hash}#L{k}"),
|
||||
None => base_policy_hash.to_string(),
|
||||
};
|
||||
let chunk_id = id_for_chunk(&doc.doc_id, chunker_version, block_ids, &id_hash);
|
||||
let token_estimate = text.len().div_ceil(BYTES_PER_TOKEN);
|
||||
Chunk {
|
||||
chunk_id,
|
||||
doc_id: DocumentId(doc.doc_id.0.clone()),
|
||||
block_ids: block_ids.to_vec(),
|
||||
text,
|
||||
heading_path: Vec::new(),
|
||||
source_spans: vec![span],
|
||||
token_estimate,
|
||||
chunker_version: chunker_version.clone(),
|
||||
policy_hash: base_policy_hash.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Split an oversize unit at blank-line paragraph boundaries, greedily
|
||||
/// gluing paragraphs until ~`AST_CHUNK_MAX_LINES` lines accumulate.
|
||||
/// Returns `(line_offset_start, line_offset_end, text)` where offsets are
|
||||
/// 0-based within the unit (caller adds the unit's absolute `line_start`).
|
||||
fn split_oversize(code: &str) -> Vec<(u32, u32, String)> {
|
||||
let lines: Vec<&str> = code.split('\n').collect();
|
||||
let total = lines.len() as u32;
|
||||
let mut out: Vec<(u32, u32, String)> = Vec::new();
|
||||
let mut start: u32 = 0;
|
||||
while start < total {
|
||||
let mut end = (start + AST_CHUNK_MAX_LINES).min(total);
|
||||
let floor = start + (AST_CHUNK_MAX_LINES * 4 / 5);
|
||||
if end < total {
|
||||
if let Some(b) = (floor.min(end)..end)
|
||||
.rev()
|
||||
.find(|&i| lines[i as usize].trim().is_empty())
|
||||
{
|
||||
end = b + 1;
|
||||
}
|
||||
}
|
||||
let text = lines[start as usize..end as usize].join("\n");
|
||||
out.push((start, end.saturating_sub(1), text));
|
||||
start = end;
|
||||
}
|
||||
if out.is_empty() {
|
||||
out.push((0, total.saturating_sub(1), code.to_string()));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use kebab_core::{
|
||||
Block, CanonicalDocument, ChunkPolicy, Chunker, ChunkerVersion, CodeBlock, CommonBlock,
|
||||
SourceSpan, id_for_block, id_for_doc, AssetId, Lang, Metadata, ParserVersion, Provenance,
|
||||
SourceType, TrustLevel, WorkspacePath,
|
||||
};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
fn code_doc(units: &[(&str, u32, u32, &str)]) -> CanonicalDocument {
|
||||
let wp = WorkspacePath("crates/x/src/a.py".into());
|
||||
let aid = AssetId("a".repeat(64));
|
||||
let pv = ParserVersion("code-python-v1".into());
|
||||
let doc_id = id_for_doc(&wp, &aid, &pv);
|
||||
let blocks = units
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, (sym, ls, le, code))| {
|
||||
let span = SourceSpan::Code {
|
||||
line_start: *ls,
|
||||
line_end: *le,
|
||||
symbol: Some((*sym).to_string()),
|
||||
lang: Some("python".into()),
|
||||
};
|
||||
let bid = id_for_block(&doc_id, "code", &[], i as u32, &span);
|
||||
Block::Code(CodeBlock {
|
||||
common: CommonBlock { block_id: bid, heading_path: vec![], source_span: span },
|
||||
lang: Some("python".into()),
|
||||
code: (*code).to_string(),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
CanonicalDocument {
|
||||
doc_id, source_asset_id: aid, workspace_path: wp, title: "a".into(),
|
||||
lang: Lang("und".into()), blocks,
|
||||
metadata: Metadata {
|
||||
aliases: vec![], tags: vec![],
|
||||
created_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
updated_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
source_type: SourceType::Note, trust_level: TrustLevel::Primary,
|
||||
user_id_alias: None, user: Default::default(),
|
||||
repo: Some("kebab".into()), git_branch: Some("main".into()),
|
||||
git_commit: Some("0".repeat(40)), code_lang: Some("python".into()),
|
||||
},
|
||||
provenance: Provenance { events: vec![] },
|
||||
parser_version: pv, schema_version: 1, doc_version: 1,
|
||||
last_chunker_version: None, last_embedding_version: None,
|
||||
}
|
||||
}
|
||||
fn policy() -> ChunkPolicy {
|
||||
ChunkPolicy { target_tokens: 500, overlap_tokens: 80,
|
||||
respect_markdown_headings: false,
|
||||
chunker_version: ChunkerVersion(VERSION_LABEL.into()) }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chunker_version_is_code_python_ast_v1() {
|
||||
assert_eq!(CodePythonAstV1Chunker.chunker_version(),
|
||||
ChunkerVersion("code-python-ast-v1".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn one_chunk_per_unit_preserves_code_span() {
|
||||
let doc = code_doc(&[
|
||||
("parse", 1, 3, "def parse():\n pass\n # x"),
|
||||
("Foo.double", 5, 7, "def double():\n #\n pass"),
|
||||
]);
|
||||
let chunks = CodePythonAstV1Chunker.chunk(&doc, &policy()).unwrap();
|
||||
assert_eq!(chunks.len(), 2);
|
||||
for c in &chunks {
|
||||
assert_eq!(c.source_spans.len(), 1);
|
||||
assert!(matches!(c.source_spans[0], SourceSpan::Code { .. }));
|
||||
assert_eq!(c.heading_path, Vec::<String>::new());
|
||||
assert_eq!(c.chunker_version.0, "code-python-ast-v1");
|
||||
}
|
||||
match &chunks[0].source_spans[0] {
|
||||
SourceSpan::Code { symbol, line_start, line_end, .. } => {
|
||||
assert_eq!(symbol.as_deref(), Some("parse"));
|
||||
assert_eq!((*line_start, *line_end), (1, 3));
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oversize_unit_splits_into_parts_with_unique_ids() {
|
||||
let body = (0..500).map(|i| format!(" x{i} = {i}")).collect::<Vec<_>>().join("\n");
|
||||
let code = format!("def big():\n{body}\n");
|
||||
let doc = code_doc(&[("big", 1, 502, &code)]);
|
||||
let chunks = CodePythonAstV1Chunker.chunk(&doc, &policy()).unwrap();
|
||||
assert!(chunks.len() >= 2, "oversize unit must split, got {}", chunks.len());
|
||||
for c in &chunks {
|
||||
match &c.source_spans[0] {
|
||||
SourceSpan::Code { symbol, .. } => {
|
||||
assert!(symbol.as_deref().unwrap().starts_with("big [part "),
|
||||
"part-numbered symbol, got {symbol:?}");
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
let mut ids: Vec<&str> = chunks.iter().map(|c| c.chunk_id.0.as_str()).collect();
|
||||
let n = ids.len(); ids.sort(); ids.dedup();
|
||||
assert_eq!(ids.len(), n, "chunk_ids unique across split parts");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_code_doc_errors() {
|
||||
use kebab_core::TextBlock;
|
||||
let mut doc = code_doc(&[("parse", 1, 1, "def parse(): pass")]);
|
||||
doc.blocks = vec![Block::Paragraph(TextBlock {
|
||||
common: CommonBlock {
|
||||
block_id: kebab_core::BlockId("b".into()),
|
||||
heading_path: vec![],
|
||||
source_span: SourceSpan::Line { start: 1, end: 1 },
|
||||
},
|
||||
text: "x".into(), inlines: vec![],
|
||||
})];
|
||||
let err = CodePythonAstV1Chunker.chunk(&doc, &policy()).unwrap_err();
|
||||
assert!(err.to_string().contains("CodePythonAstV1Chunker"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deterministic_chunk_ids_1000() {
|
||||
let doc = code_doc(&[("parse", 1, 2, "def parse(): pass\n")]);
|
||||
let base: Vec<String> = CodePythonAstV1Chunker.chunk(&doc, &policy())
|
||||
.unwrap().into_iter().map(|c| c.chunk_id.0).collect();
|
||||
for _ in 0..1000 {
|
||||
let again: Vec<String> = CodePythonAstV1Chunker.chunk(&doc, &policy())
|
||||
.unwrap().into_iter().map(|c| c.chunk_id.0).collect();
|
||||
assert_eq!(again, base);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn policy_hash_matches_md_heading_v1() {
|
||||
let p = policy();
|
||||
assert_eq!(CodePythonAstV1Chunker.policy_hash(&p),
|
||||
crate::MdHeadingV1Chunker.policy_hash(&p));
|
||||
}
|
||||
}
|
||||
322
crates/kebab-chunk/src/code_rust_ast_v1.rs
Normal file
322
crates/kebab-chunk/src/code_rust_ast_v1.rs
Normal file
@@ -0,0 +1,322 @@
|
||||
//! `code-rust-ast-v1` — maps a tree-sitter-derived Rust AST
|
||||
//! `CanonicalDocument` (one `Block::Code` per semantic unit, each with
|
||||
//! `SourceSpan::Code`) to chunks 1:1. A unit longer than
|
||||
//! `AST_CHUNK_MAX_LINES` is split into `<symbol> [part i/N]` sub-chunks
|
||||
//! at blank-line paragraph boundaries (design §9.1 oversize fallback).
|
||||
//!
|
||||
//! tree-sitter is intentionally NOT a dependency here: AST work is
|
||||
//! parser-side (`kebab-parse-code`, design §6.3). This chunker only
|
||||
//! consumes the `CanonicalDocument`.
|
||||
//!
|
||||
//! `AST_CHUNK_MAX_LINES` is a constant matching
|
||||
//! `IngestCodeCfg::default().ast_chunk_max_lines` (200). Per-medium
|
||||
//! config threading needs a chunker registry (P+); same deviation
|
||||
//! pattern as `pdf-page-v1`'s pinned `chunker_version`
|
||||
//! (`tasks/HOTFIXES.md`).
|
||||
|
||||
use kebab_core::{
|
||||
Block, BlockId, CanonicalDocument, Chunk, ChunkPolicy, Chunker, ChunkerVersion, DocumentId,
|
||||
SourceSpan, id_for_chunk,
|
||||
};
|
||||
|
||||
const VERSION_LABEL: &str = "code-rust-ast-v1";
|
||||
const BYTES_PER_TOKEN: usize = 3;
|
||||
const POLICY_HASH_HEX_LEN: usize = 16;
|
||||
const AST_CHUNK_MAX_LINES: u32 = 200;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
pub struct CodeRustAstV1Chunker;
|
||||
|
||||
impl Chunker for CodeRustAstV1Chunker {
|
||||
fn chunker_version(&self) -> ChunkerVersion {
|
||||
ChunkerVersion(VERSION_LABEL.to_string())
|
||||
}
|
||||
|
||||
fn policy_hash(&self, policy: &ChunkPolicy) -> String {
|
||||
let bytes = serde_json_canonicalizer::to_vec(policy)
|
||||
.expect("canonical JSON serialization of ChunkPolicy must not fail");
|
||||
let hex = blake3::hash(&bytes).to_hex().to_string();
|
||||
hex[..POLICY_HASH_HEX_LEN].to_string()
|
||||
}
|
||||
|
||||
fn chunk(
|
||||
&self,
|
||||
doc: &CanonicalDocument,
|
||||
policy: &ChunkPolicy,
|
||||
) -> anyhow::Result<Vec<Chunk>> {
|
||||
for b in &doc.blocks {
|
||||
let c = match b {
|
||||
Block::Code(c) => c,
|
||||
_ => anyhow::bail!(
|
||||
"CodeRustAstV1Chunker only handles code docs (got non-Code block)"
|
||||
),
|
||||
};
|
||||
if !matches!(c.common.source_span, SourceSpan::Code { .. }) {
|
||||
anyhow::bail!(
|
||||
"CodeRustAstV1Chunker only handles code docs (got non-Code source_span)"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let base_policy_hash = self.policy_hash(policy);
|
||||
let chunker_version = self.chunker_version();
|
||||
let mut out: Vec<Chunk> = Vec::new();
|
||||
|
||||
for b in &doc.blocks {
|
||||
let cb = match b {
|
||||
Block::Code(c) => c,
|
||||
_ => unreachable!("validated above"),
|
||||
};
|
||||
let (ls, le, symbol, lang) = match &cb.common.source_span {
|
||||
SourceSpan::Code { line_start, line_end, symbol, lang } => {
|
||||
(*line_start, *line_end, symbol.clone(), lang.clone())
|
||||
}
|
||||
_ => unreachable!("validated above"),
|
||||
};
|
||||
let block_ids: Vec<BlockId> = vec![cb.common.block_id.clone()];
|
||||
let span_lines = le.saturating_sub(ls) + 1;
|
||||
|
||||
if span_lines <= AST_CHUNK_MAX_LINES {
|
||||
let span = SourceSpan::Code {
|
||||
line_start: ls,
|
||||
line_end: le,
|
||||
symbol: symbol.clone(),
|
||||
lang: lang.clone(),
|
||||
};
|
||||
out.push(make_chunk(
|
||||
doc, &chunker_version, &block_ids, &base_policy_hash,
|
||||
None, span, cb.code.clone(),
|
||||
));
|
||||
} else {
|
||||
let parts = split_oversize(&cb.code);
|
||||
let n = parts.len();
|
||||
for (i, (off_start, off_end, text)) in parts.into_iter().enumerate() {
|
||||
let part_ls = ls + off_start;
|
||||
let part_le = ls + off_end;
|
||||
let part_sym = symbol
|
||||
.as_ref()
|
||||
.map(|s| format!("{s} [part {}/{n}]", i + 1));
|
||||
let span = SourceSpan::Code {
|
||||
line_start: part_ls,
|
||||
line_end: part_le,
|
||||
symbol: part_sym,
|
||||
lang: lang.clone(),
|
||||
};
|
||||
out.push(make_chunk(
|
||||
doc, &chunker_version, &block_ids, &base_policy_hash,
|
||||
Some(part_ls), span, text,
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tracing::debug!(
|
||||
target: "kebab-chunk",
|
||||
doc_id = %doc.doc_id,
|
||||
chunks = out.len(),
|
||||
"code-rust-ast-v1 chunked",
|
||||
);
|
||||
Ok(out)
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn make_chunk(
|
||||
doc: &CanonicalDocument,
|
||||
chunker_version: &ChunkerVersion,
|
||||
block_ids: &[BlockId],
|
||||
base_policy_hash: &str,
|
||||
split_key: Option<u32>,
|
||||
span: SourceSpan,
|
||||
text: String,
|
||||
) -> Chunk {
|
||||
let id_hash = match split_key {
|
||||
Some(k) => format!("{base_policy_hash}#L{k}"),
|
||||
None => base_policy_hash.to_string(),
|
||||
};
|
||||
let chunk_id = id_for_chunk(&doc.doc_id, chunker_version, block_ids, &id_hash);
|
||||
let token_estimate = text.len().div_ceil(BYTES_PER_TOKEN);
|
||||
Chunk {
|
||||
chunk_id,
|
||||
doc_id: DocumentId(doc.doc_id.0.clone()),
|
||||
block_ids: block_ids.to_vec(),
|
||||
text,
|
||||
heading_path: Vec::new(),
|
||||
source_spans: vec![span],
|
||||
token_estimate,
|
||||
chunker_version: chunker_version.clone(),
|
||||
policy_hash: base_policy_hash.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Split an oversize unit at blank-line paragraph boundaries, greedily
|
||||
/// gluing paragraphs until ~`AST_CHUNK_MAX_LINES` lines accumulate.
|
||||
/// Returns `(line_offset_start, line_offset_end, text)` where offsets are
|
||||
/// 0-based within the unit (caller adds the unit's absolute `line_start`).
|
||||
fn split_oversize(code: &str) -> Vec<(u32, u32, String)> {
|
||||
let lines: Vec<&str> = code.split('\n').collect();
|
||||
let total = lines.len() as u32;
|
||||
let mut out: Vec<(u32, u32, String)> = Vec::new();
|
||||
let mut start: u32 = 0;
|
||||
while start < total {
|
||||
let mut end = (start + AST_CHUNK_MAX_LINES).min(total);
|
||||
let floor = start + (AST_CHUNK_MAX_LINES * 4 / 5);
|
||||
if end < total {
|
||||
if let Some(b) = (floor.min(end)..end)
|
||||
.rev()
|
||||
.find(|&i| lines[i as usize].trim().is_empty())
|
||||
{
|
||||
end = b + 1;
|
||||
}
|
||||
}
|
||||
let text = lines[start as usize..end as usize].join("\n");
|
||||
out.push((start, end.saturating_sub(1), text));
|
||||
start = end;
|
||||
}
|
||||
if out.is_empty() {
|
||||
out.push((0, total.saturating_sub(1), code.to_string()));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use kebab_core::{
|
||||
Block, CanonicalDocument, ChunkPolicy, Chunker, ChunkerVersion, CodeBlock, CommonBlock,
|
||||
SourceSpan, id_for_block, id_for_doc, AssetId, Lang, Metadata, ParserVersion, Provenance,
|
||||
SourceType, TrustLevel, WorkspacePath,
|
||||
};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
fn code_doc(units: &[(&str, u32, u32, &str)]) -> CanonicalDocument {
|
||||
let wp = WorkspacePath("crates/x/src/a.rs".into());
|
||||
let aid = AssetId("a".repeat(64));
|
||||
let pv = ParserVersion("code-rust-v1".into());
|
||||
let doc_id = id_for_doc(&wp, &aid, &pv);
|
||||
let blocks = units
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, (sym, ls, le, code))| {
|
||||
let span = SourceSpan::Code {
|
||||
line_start: *ls,
|
||||
line_end: *le,
|
||||
symbol: Some((*sym).to_string()),
|
||||
lang: Some("rust".into()),
|
||||
};
|
||||
let bid = id_for_block(&doc_id, "code", &[], i as u32, &span);
|
||||
Block::Code(CodeBlock {
|
||||
common: CommonBlock { block_id: bid, heading_path: vec![], source_span: span },
|
||||
lang: Some("rust".into()),
|
||||
code: (*code).to_string(),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
CanonicalDocument {
|
||||
doc_id, source_asset_id: aid, workspace_path: wp, title: "a".into(),
|
||||
lang: Lang("und".into()), blocks,
|
||||
metadata: Metadata {
|
||||
aliases: vec![], tags: vec![],
|
||||
created_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
updated_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
source_type: SourceType::Note, trust_level: TrustLevel::Primary,
|
||||
user_id_alias: None, user: Default::default(),
|
||||
repo: Some("kebab".into()), git_branch: Some("main".into()),
|
||||
git_commit: Some("0".repeat(40)), code_lang: Some("rust".into()),
|
||||
},
|
||||
provenance: Provenance { events: vec![] },
|
||||
parser_version: pv, schema_version: 1, doc_version: 1,
|
||||
last_chunker_version: None, last_embedding_version: None,
|
||||
}
|
||||
}
|
||||
fn policy() -> ChunkPolicy {
|
||||
ChunkPolicy { target_tokens: 500, overlap_tokens: 80,
|
||||
respect_markdown_headings: false,
|
||||
chunker_version: ChunkerVersion(VERSION_LABEL.into()) }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chunker_version_is_code_rust_ast_v1() {
|
||||
assert_eq!(CodeRustAstV1Chunker.chunker_version(),
|
||||
ChunkerVersion("code-rust-ast-v1".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn one_chunk_per_unit_preserves_code_span() {
|
||||
let doc = code_doc(&[
|
||||
("parse", 1, 3, "pub fn parse() {}\n// x\n}"),
|
||||
("Foo::double", 5, 7, "fn double() {}\n//\n}"),
|
||||
]);
|
||||
let chunks = CodeRustAstV1Chunker.chunk(&doc, &policy()).unwrap();
|
||||
assert_eq!(chunks.len(), 2);
|
||||
for c in &chunks {
|
||||
assert_eq!(c.source_spans.len(), 1);
|
||||
assert!(matches!(c.source_spans[0], SourceSpan::Code { .. }));
|
||||
assert_eq!(c.heading_path, Vec::<String>::new());
|
||||
assert_eq!(c.chunker_version.0, "code-rust-ast-v1");
|
||||
}
|
||||
match &chunks[0].source_spans[0] {
|
||||
SourceSpan::Code { symbol, line_start, line_end, .. } => {
|
||||
assert_eq!(symbol.as_deref(), Some("parse"));
|
||||
assert_eq!((*line_start, *line_end), (1, 3));
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oversize_unit_splits_into_parts_with_unique_ids() {
|
||||
let body = (0..500).map(|i| format!(" let x{i} = {i};")).collect::<Vec<_>>().join("\n");
|
||||
let code = format!("pub fn big() {{\n{body}\n}}");
|
||||
let doc = code_doc(&[("big", 1, 502, &code)]);
|
||||
let chunks = CodeRustAstV1Chunker.chunk(&doc, &policy()).unwrap();
|
||||
assert!(chunks.len() >= 2, "oversize unit must split, got {}", chunks.len());
|
||||
for c in &chunks {
|
||||
match &c.source_spans[0] {
|
||||
SourceSpan::Code { symbol, .. } => {
|
||||
assert!(symbol.as_deref().unwrap().starts_with("big [part "),
|
||||
"part-numbered symbol, got {symbol:?}");
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
let mut ids: Vec<&str> = chunks.iter().map(|c| c.chunk_id.0.as_str()).collect();
|
||||
let n = ids.len(); ids.sort(); ids.dedup();
|
||||
assert_eq!(ids.len(), n, "chunk_ids unique across split parts");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_code_doc_errors() {
|
||||
use kebab_core::TextBlock;
|
||||
let mut doc = code_doc(&[("parse", 1, 1, "fn parse(){}")]);
|
||||
doc.blocks = vec![Block::Paragraph(TextBlock {
|
||||
common: CommonBlock {
|
||||
block_id: kebab_core::BlockId("b".into()),
|
||||
heading_path: vec![],
|
||||
source_span: SourceSpan::Line { start: 1, end: 1 },
|
||||
},
|
||||
text: "x".into(), inlines: vec![],
|
||||
})];
|
||||
let err = CodeRustAstV1Chunker.chunk(&doc, &policy()).unwrap_err();
|
||||
assert!(err.to_string().contains("CodeRustAstV1Chunker"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deterministic_chunk_ids_1000() {
|
||||
let doc = code_doc(&[("parse", 1, 2, "fn parse(){}\n}")]);
|
||||
let base: Vec<String> = CodeRustAstV1Chunker.chunk(&doc, &policy())
|
||||
.unwrap().into_iter().map(|c| c.chunk_id.0).collect();
|
||||
for _ in 0..1000 {
|
||||
let again: Vec<String> = CodeRustAstV1Chunker.chunk(&doc, &policy())
|
||||
.unwrap().into_iter().map(|c| c.chunk_id.0).collect();
|
||||
assert_eq!(again, base);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn policy_hash_matches_md_heading_v1() {
|
||||
let p = policy();
|
||||
assert_eq!(CodeRustAstV1Chunker.policy_hash(&p),
|
||||
crate::MdHeadingV1Chunker.policy_hash(&p));
|
||||
}
|
||||
}
|
||||
322
crates/kebab-chunk/src/code_ts_ast_v1.rs
Normal file
322
crates/kebab-chunk/src/code_ts_ast_v1.rs
Normal file
@@ -0,0 +1,322 @@
|
||||
//! `code-ts-ast-v1` — maps a tree-sitter-derived TypeScript AST
|
||||
//! `CanonicalDocument` (one `Block::Code` per semantic unit, each with
|
||||
//! `SourceSpan::Code`) to chunks 1:1. A unit longer than
|
||||
//! `AST_CHUNK_MAX_LINES` is split into `<symbol> [part i/N]` sub-chunks
|
||||
//! at blank-line paragraph boundaries (design §9.1 oversize fallback).
|
||||
//!
|
||||
//! tree-sitter is intentionally NOT a dependency here: AST work is
|
||||
//! parser-side (`kebab-parse-code`, design §6.3). This chunker only
|
||||
//! consumes the `CanonicalDocument`.
|
||||
//!
|
||||
//! `AST_CHUNK_MAX_LINES` is a constant matching
|
||||
//! `IngestCodeCfg::default().ast_chunk_max_lines` (200). Per-medium
|
||||
//! config threading needs a chunker registry (P+); same deviation
|
||||
//! pattern as `pdf-page-v1`'s pinned `chunker_version`
|
||||
//! (`tasks/HOTFIXES.md`).
|
||||
|
||||
use kebab_core::{
|
||||
Block, BlockId, CanonicalDocument, Chunk, ChunkPolicy, Chunker, ChunkerVersion, DocumentId,
|
||||
SourceSpan, id_for_chunk,
|
||||
};
|
||||
|
||||
const VERSION_LABEL: &str = "code-ts-ast-v1";
|
||||
const BYTES_PER_TOKEN: usize = 3;
|
||||
const POLICY_HASH_HEX_LEN: usize = 16;
|
||||
const AST_CHUNK_MAX_LINES: u32 = 200;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
pub struct CodeTsAstV1Chunker;
|
||||
|
||||
impl Chunker for CodeTsAstV1Chunker {
|
||||
fn chunker_version(&self) -> ChunkerVersion {
|
||||
ChunkerVersion(VERSION_LABEL.to_string())
|
||||
}
|
||||
|
||||
fn policy_hash(&self, policy: &ChunkPolicy) -> String {
|
||||
let bytes = serde_json_canonicalizer::to_vec(policy)
|
||||
.expect("canonical JSON serialization of ChunkPolicy must not fail");
|
||||
let hex = blake3::hash(&bytes).to_hex().to_string();
|
||||
hex[..POLICY_HASH_HEX_LEN].to_string()
|
||||
}
|
||||
|
||||
fn chunk(
|
||||
&self,
|
||||
doc: &CanonicalDocument,
|
||||
policy: &ChunkPolicy,
|
||||
) -> anyhow::Result<Vec<Chunk>> {
|
||||
for b in &doc.blocks {
|
||||
let c = match b {
|
||||
Block::Code(c) => c,
|
||||
_ => anyhow::bail!(
|
||||
"CodeTsAstV1Chunker only handles code docs (got non-Code block)"
|
||||
),
|
||||
};
|
||||
if !matches!(c.common.source_span, SourceSpan::Code { .. }) {
|
||||
anyhow::bail!(
|
||||
"CodeTsAstV1Chunker only handles code docs (got non-Code source_span)"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let base_policy_hash = self.policy_hash(policy);
|
||||
let chunker_version = self.chunker_version();
|
||||
let mut out: Vec<Chunk> = Vec::new();
|
||||
|
||||
for b in &doc.blocks {
|
||||
let cb = match b {
|
||||
Block::Code(c) => c,
|
||||
_ => unreachable!("validated above"),
|
||||
};
|
||||
let (ls, le, symbol, lang) = match &cb.common.source_span {
|
||||
SourceSpan::Code { line_start, line_end, symbol, lang } => {
|
||||
(*line_start, *line_end, symbol.clone(), lang.clone())
|
||||
}
|
||||
_ => unreachable!("validated above"),
|
||||
};
|
||||
let block_ids: Vec<BlockId> = vec![cb.common.block_id.clone()];
|
||||
let span_lines = le.saturating_sub(ls) + 1;
|
||||
|
||||
if span_lines <= AST_CHUNK_MAX_LINES {
|
||||
let span = SourceSpan::Code {
|
||||
line_start: ls,
|
||||
line_end: le,
|
||||
symbol: symbol.clone(),
|
||||
lang: lang.clone(),
|
||||
};
|
||||
out.push(make_chunk(
|
||||
doc, &chunker_version, &block_ids, &base_policy_hash,
|
||||
None, span, cb.code.clone(),
|
||||
));
|
||||
} else {
|
||||
let parts = split_oversize(&cb.code);
|
||||
let n = parts.len();
|
||||
for (i, (off_start, off_end, text)) in parts.into_iter().enumerate() {
|
||||
let part_ls = ls + off_start;
|
||||
let part_le = ls + off_end;
|
||||
let part_sym = symbol
|
||||
.as_ref()
|
||||
.map(|s| format!("{s} [part {}/{n}]", i + 1));
|
||||
let span = SourceSpan::Code {
|
||||
line_start: part_ls,
|
||||
line_end: part_le,
|
||||
symbol: part_sym,
|
||||
lang: lang.clone(),
|
||||
};
|
||||
out.push(make_chunk(
|
||||
doc, &chunker_version, &block_ids, &base_policy_hash,
|
||||
Some(part_ls), span, text,
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tracing::debug!(
|
||||
target: "kebab-chunk",
|
||||
doc_id = %doc.doc_id,
|
||||
chunks = out.len(),
|
||||
"code-ts-ast-v1 chunked",
|
||||
);
|
||||
Ok(out)
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn make_chunk(
|
||||
doc: &CanonicalDocument,
|
||||
chunker_version: &ChunkerVersion,
|
||||
block_ids: &[BlockId],
|
||||
base_policy_hash: &str,
|
||||
split_key: Option<u32>,
|
||||
span: SourceSpan,
|
||||
text: String,
|
||||
) -> Chunk {
|
||||
let id_hash = match split_key {
|
||||
Some(k) => format!("{base_policy_hash}#L{k}"),
|
||||
None => base_policy_hash.to_string(),
|
||||
};
|
||||
let chunk_id = id_for_chunk(&doc.doc_id, chunker_version, block_ids, &id_hash);
|
||||
let token_estimate = text.len().div_ceil(BYTES_PER_TOKEN);
|
||||
Chunk {
|
||||
chunk_id,
|
||||
doc_id: DocumentId(doc.doc_id.0.clone()),
|
||||
block_ids: block_ids.to_vec(),
|
||||
text,
|
||||
heading_path: Vec::new(),
|
||||
source_spans: vec![span],
|
||||
token_estimate,
|
||||
chunker_version: chunker_version.clone(),
|
||||
policy_hash: base_policy_hash.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Split an oversize unit at blank-line paragraph boundaries, greedily
|
||||
/// gluing paragraphs until ~`AST_CHUNK_MAX_LINES` lines accumulate.
|
||||
/// Returns `(line_offset_start, line_offset_end, text)` where offsets are
|
||||
/// 0-based within the unit (caller adds the unit's absolute `line_start`).
|
||||
fn split_oversize(code: &str) -> Vec<(u32, u32, String)> {
|
||||
let lines: Vec<&str> = code.split('\n').collect();
|
||||
let total = lines.len() as u32;
|
||||
let mut out: Vec<(u32, u32, String)> = Vec::new();
|
||||
let mut start: u32 = 0;
|
||||
while start < total {
|
||||
let mut end = (start + AST_CHUNK_MAX_LINES).min(total);
|
||||
let floor = start + (AST_CHUNK_MAX_LINES * 4 / 5);
|
||||
if end < total {
|
||||
if let Some(b) = (floor.min(end)..end)
|
||||
.rev()
|
||||
.find(|&i| lines[i as usize].trim().is_empty())
|
||||
{
|
||||
end = b + 1;
|
||||
}
|
||||
}
|
||||
let text = lines[start as usize..end as usize].join("\n");
|
||||
out.push((start, end.saturating_sub(1), text));
|
||||
start = end;
|
||||
}
|
||||
if out.is_empty() {
|
||||
out.push((0, total.saturating_sub(1), code.to_string()));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use kebab_core::{
|
||||
Block, CanonicalDocument, ChunkPolicy, Chunker, ChunkerVersion, CodeBlock, CommonBlock,
|
||||
SourceSpan, id_for_block, id_for_doc, AssetId, Lang, Metadata, ParserVersion, Provenance,
|
||||
SourceType, TrustLevel, WorkspacePath,
|
||||
};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
fn code_doc(units: &[(&str, u32, u32, &str)]) -> CanonicalDocument {
|
||||
let wp = WorkspacePath("crates/x/src/a.ts".into());
|
||||
let aid = AssetId("a".repeat(64));
|
||||
let pv = ParserVersion("code-ts-v1".into());
|
||||
let doc_id = id_for_doc(&wp, &aid, &pv);
|
||||
let blocks = units
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, (sym, ls, le, code))| {
|
||||
let span = SourceSpan::Code {
|
||||
line_start: *ls,
|
||||
line_end: *le,
|
||||
symbol: Some((*sym).to_string()),
|
||||
lang: Some("typescript".into()),
|
||||
};
|
||||
let bid = id_for_block(&doc_id, "code", &[], i as u32, &span);
|
||||
Block::Code(CodeBlock {
|
||||
common: CommonBlock { block_id: bid, heading_path: vec![], source_span: span },
|
||||
lang: Some("typescript".into()),
|
||||
code: (*code).to_string(),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
CanonicalDocument {
|
||||
doc_id, source_asset_id: aid, workspace_path: wp, title: "a".into(),
|
||||
lang: Lang("und".into()), blocks,
|
||||
metadata: Metadata {
|
||||
aliases: vec![], tags: vec![],
|
||||
created_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
updated_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
source_type: SourceType::Note, trust_level: TrustLevel::Primary,
|
||||
user_id_alias: None, user: Default::default(),
|
||||
repo: Some("kebab".into()), git_branch: Some("main".into()),
|
||||
git_commit: Some("0".repeat(40)), code_lang: Some("typescript".into()),
|
||||
},
|
||||
provenance: Provenance { events: vec![] },
|
||||
parser_version: pv, schema_version: 1, doc_version: 1,
|
||||
last_chunker_version: None, last_embedding_version: None,
|
||||
}
|
||||
}
|
||||
fn policy() -> ChunkPolicy {
|
||||
ChunkPolicy { target_tokens: 500, overlap_tokens: 80,
|
||||
respect_markdown_headings: false,
|
||||
chunker_version: ChunkerVersion(VERSION_LABEL.into()) }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chunker_version_is_code_ts_ast_v1() {
|
||||
assert_eq!(CodeTsAstV1Chunker.chunker_version(),
|
||||
ChunkerVersion("code-ts-ast-v1".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn one_chunk_per_unit_preserves_code_span() {
|
||||
let doc = code_doc(&[
|
||||
("parse", 1, 3, "function parse(): void {\n // x\n}"),
|
||||
("Foo.double", 5, 7, "function double(): number {\n //\n return 0;\n}"),
|
||||
]);
|
||||
let chunks = CodeTsAstV1Chunker.chunk(&doc, &policy()).unwrap();
|
||||
assert_eq!(chunks.len(), 2);
|
||||
for c in &chunks {
|
||||
assert_eq!(c.source_spans.len(), 1);
|
||||
assert!(matches!(c.source_spans[0], SourceSpan::Code { .. }));
|
||||
assert_eq!(c.heading_path, Vec::<String>::new());
|
||||
assert_eq!(c.chunker_version.0, "code-ts-ast-v1");
|
||||
}
|
||||
match &chunks[0].source_spans[0] {
|
||||
SourceSpan::Code { symbol, line_start, line_end, .. } => {
|
||||
assert_eq!(symbol.as_deref(), Some("parse"));
|
||||
assert_eq!((*line_start, *line_end), (1, 3));
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oversize_unit_splits_into_parts_with_unique_ids() {
|
||||
let body = (0..500).map(|i| format!(" const x{i} = {i};")).collect::<Vec<_>>().join("\n");
|
||||
let code = format!("function big(): void {{\n{body}\n}}");
|
||||
let doc = code_doc(&[("big", 1, 502, &code)]);
|
||||
let chunks = CodeTsAstV1Chunker.chunk(&doc, &policy()).unwrap();
|
||||
assert!(chunks.len() >= 2, "oversize unit must split, got {}", chunks.len());
|
||||
for c in &chunks {
|
||||
match &c.source_spans[0] {
|
||||
SourceSpan::Code { symbol, .. } => {
|
||||
assert!(symbol.as_deref().unwrap().starts_with("big [part "),
|
||||
"part-numbered symbol, got {symbol:?}");
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
let mut ids: Vec<&str> = chunks.iter().map(|c| c.chunk_id.0.as_str()).collect();
|
||||
let n = ids.len(); ids.sort(); ids.dedup();
|
||||
assert_eq!(ids.len(), n, "chunk_ids unique across split parts");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_code_doc_errors() {
|
||||
use kebab_core::TextBlock;
|
||||
let mut doc = code_doc(&[("parse", 1, 1, "function parse(): void {}")]);
|
||||
doc.blocks = vec![Block::Paragraph(TextBlock {
|
||||
common: CommonBlock {
|
||||
block_id: kebab_core::BlockId("b".into()),
|
||||
heading_path: vec![],
|
||||
source_span: SourceSpan::Line { start: 1, end: 1 },
|
||||
},
|
||||
text: "x".into(), inlines: vec![],
|
||||
})];
|
||||
let err = CodeTsAstV1Chunker.chunk(&doc, &policy()).unwrap_err();
|
||||
assert!(err.to_string().contains("CodeTsAstV1Chunker"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deterministic_chunk_ids_1000() {
|
||||
let doc = code_doc(&[("parse", 1, 2, "function parse(): void {}\n")]);
|
||||
let base: Vec<String> = CodeTsAstV1Chunker.chunk(&doc, &policy())
|
||||
.unwrap().into_iter().map(|c| c.chunk_id.0).collect();
|
||||
for _ in 0..1000 {
|
||||
let again: Vec<String> = CodeTsAstV1Chunker.chunk(&doc, &policy())
|
||||
.unwrap().into_iter().map(|c| c.chunk_id.0).collect();
|
||||
assert_eq!(again, base);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn policy_hash_matches_md_heading_v1() {
|
||||
let p = policy();
|
||||
assert_eq!(CodeTsAstV1Chunker.policy_hash(&p),
|
||||
crate::MdHeadingV1Chunker.policy_hash(&p));
|
||||
}
|
||||
}
|
||||
@@ -15,8 +15,16 @@
|
||||
//! embedder, the retriever, the LLM, the RAG layer, or the UI layers.
|
||||
//! It consumes `CanonicalDocument` purely through `kb-core` types.
|
||||
|
||||
mod code_js_ast_v1;
|
||||
mod code_python_ast_v1;
|
||||
mod code_rust_ast_v1;
|
||||
mod code_ts_ast_v1;
|
||||
mod md_heading_v1;
|
||||
mod pdf_page_v1;
|
||||
|
||||
pub use code_js_ast_v1::CodeJsAstV1Chunker;
|
||||
pub use code_python_ast_v1::CodePythonAstV1Chunker;
|
||||
pub use code_rust_ast_v1::CodeRustAstV1Chunker;
|
||||
pub use code_ts_ast_v1::CodeTsAstV1Chunker;
|
||||
pub use md_heading_v1::MdHeadingV1Chunker;
|
||||
pub use pdf_page_v1::PdfPageV1Chunker;
|
||||
|
||||
@@ -472,6 +472,10 @@ mod tests {
|
||||
trust_level: TrustLevel::Primary,
|
||||
user_id_alias: None,
|
||||
user: Default::default(),
|
||||
repo: None,
|
||||
git_branch: None,
|
||||
git_commit: None,
|
||||
code_lang: None,
|
||||
},
|
||||
provenance: Provenance { events: vec![] },
|
||||
parser_version: kebab_core::ParserVersion("test-parser-0".into()),
|
||||
|
||||
@@ -347,6 +347,10 @@ mod tests {
|
||||
trust_level: TrustLevel::Primary,
|
||||
user_id_alias: None,
|
||||
user: Default::default(),
|
||||
repo: None,
|
||||
git_branch: None,
|
||||
git_commit: None,
|
||||
code_lang: None,
|
||||
},
|
||||
provenance: Provenance { events: vec![] },
|
||||
parser_version,
|
||||
@@ -512,6 +516,10 @@ mod tests {
|
||||
trust_level: TrustLevel::Primary,
|
||||
user_id_alias: None,
|
||||
user: Default::default(),
|
||||
repo: None,
|
||||
git_branch: None,
|
||||
git_commit: None,
|
||||
code_lang: None,
|
||||
},
|
||||
provenance: Provenance { events: vec![] },
|
||||
parser_version,
|
||||
|
||||
221
crates/kebab-chunk/tests/code_js_ast_snapshot.rs
Normal file
221
crates/kebab-chunk/tests/code_js_ast_snapshot.rs
Normal file
@@ -0,0 +1,221 @@
|
||||
//! Snapshot test pinning the `Vec<Chunk>` JSON for a
|
||||
//! representative JavaScript code `CanonicalDocument`.
|
||||
//!
|
||||
//! This is an integration test. `kebab-parse-code` is intentionally NOT
|
||||
//! a dev-dep (design §6.3 / §8 boundary: AST extraction is parser-side).
|
||||
//! The `CanonicalDocument` is built inline from hand-crafted `Block::Code`
|
||||
//! units, which is the same pattern used in `code_rust_ast_v1.rs`'s
|
||||
//! internal `code_doc` test helper.
|
||||
//!
|
||||
//! Set `UPDATE_SNAPSHOTS=1` to re-bake the baseline.
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use kebab_chunk::CodeJsAstV1Chunker;
|
||||
use kebab_core::{
|
||||
AssetId, Block, CanonicalDocument, ChunkPolicy, Chunker, ChunkerVersion, CodeBlock, CommonBlock,
|
||||
Lang, Metadata, ParserVersion, Provenance, SourceSpan, SourceType, TrustLevel, WorkspacePath,
|
||||
id_for_block, id_for_doc,
|
||||
};
|
||||
use serde_json::Value;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
fn fixtures_dir() -> PathBuf {
|
||||
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("tests")
|
||||
.join("fixtures")
|
||||
}
|
||||
|
||||
fn fixed_doc() -> CanonicalDocument {
|
||||
let wp = WorkspacePath("src/bar.js".into());
|
||||
let aid = AssetId("b".repeat(64));
|
||||
// Pin parser_version so doc_id / block_ids are reproducible.
|
||||
let pv = ParserVersion("code-js-v1".into());
|
||||
let doc_id = id_for_doc(&wp, &aid, &pv);
|
||||
|
||||
// Build a >200-line function body to force split_oversize.
|
||||
let big_body: String = {
|
||||
let header = "function bigTransform(items) {\n";
|
||||
let body: String = (0..210u32)
|
||||
.map(|i| format!(" const v{i} = items[{i}] !== undefined ? items[{i}] : null;\n"))
|
||||
.collect();
|
||||
let footer = " return items;\n}";
|
||||
format!("{header}{body}{footer}")
|
||||
};
|
||||
let big_line_count = big_body.lines().count() as u32;
|
||||
let big_line_end = 48 + big_line_count - 1;
|
||||
|
||||
// Representative units:
|
||||
// 0. require/import block (lines 1–5, ≤200)
|
||||
// 1. free fn `add` (lines 7–12, ≤200)
|
||||
// 2. class `EventBus` (lines 14–20, ≤200)
|
||||
// 3. class `BaseHandler` (lines 22–30, ≤200)
|
||||
// 4. method `EventBus.emit` (lines 32–38, ≤200)
|
||||
// 5. method `EventBus.on` (lines 40–46, ≤200)
|
||||
// 6. bigTransform (>200 lines) to force split_oversize
|
||||
let raw_units: Vec<(&str, u32, u32, String)> = vec![
|
||||
(
|
||||
"requires",
|
||||
1,
|
||||
5,
|
||||
"const fs = require('fs');\nconst path = require('path');\nconst { EventEmitter } = require('events');\nconst assert = require('assert');\nconst crypto = require('crypto');".to_string(),
|
||||
),
|
||||
(
|
||||
"add",
|
||||
7,
|
||||
12,
|
||||
"export function add(a, b) {\n if (typeof a !== 'number') throw new TypeError('a');\n if (typeof b !== 'number') throw new TypeError('b');\n const result = a + b;\n assert(isFinite(result));\n return result;\n}".to_string(),
|
||||
),
|
||||
(
|
||||
"EventBus",
|
||||
14,
|
||||
20,
|
||||
"class EventBus {\n constructor() {\n this._handlers = new Map();\n this._history = [];\n this._maxHistory = 100;\n this._seq = 0;\n }\n}".to_string(),
|
||||
),
|
||||
(
|
||||
"BaseHandler",
|
||||
22,
|
||||
30,
|
||||
"class BaseHandler {\n handle(event) {\n throw new Error('not implemented');\n }\n batchHandle(events) {\n const results = [];\n for (const ev of events) {\n results.push(this.handle(ev));\n }\n return results;\n }\n}".to_string(),
|
||||
),
|
||||
(
|
||||
"EventBus.emit",
|
||||
32,
|
||||
38,
|
||||
"class EventBus {\n emit(name, payload) {\n const handlers = this._handlers.get(name) ?? [];\n for (const h of handlers) {\n h(payload);\n }\n return this;\n }\n}".to_string(),
|
||||
),
|
||||
(
|
||||
"EventBus.on",
|
||||
40,
|
||||
46,
|
||||
"class EventBus {\n on(name, handler) {\n if (!this._handlers.has(name)) {\n this._handlers.set(name, []);\n }\n this._handlers.get(name).push(handler);\n return this;\n }\n}".to_string(),
|
||||
),
|
||||
("bigTransform", 48, big_line_end, big_body),
|
||||
];
|
||||
|
||||
let blocks: Vec<Block> = raw_units
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, (sym, ls, le, code))| {
|
||||
let span = SourceSpan::Code {
|
||||
line_start: *ls,
|
||||
line_end: *le,
|
||||
symbol: Some((*sym).to_string()),
|
||||
lang: Some("javascript".into()),
|
||||
};
|
||||
let bid = id_for_block(&doc_id, "code", &[], i as u32, &span);
|
||||
Block::Code(CodeBlock {
|
||||
common: CommonBlock {
|
||||
block_id: bid,
|
||||
heading_path: vec![],
|
||||
source_span: span,
|
||||
},
|
||||
lang: Some("javascript".into()),
|
||||
code: code.clone(),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
CanonicalDocument {
|
||||
doc_id,
|
||||
source_asset_id: aid,
|
||||
workspace_path: wp,
|
||||
title: "bar.js".into(),
|
||||
lang: Lang("und".into()),
|
||||
blocks,
|
||||
metadata: Metadata {
|
||||
aliases: vec![],
|
||||
tags: vec![],
|
||||
created_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
updated_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
source_type: SourceType::Note,
|
||||
trust_level: TrustLevel::Primary,
|
||||
user_id_alias: None,
|
||||
user: Default::default(),
|
||||
repo: Some("kebab".into()),
|
||||
git_branch: Some("main".into()),
|
||||
git_commit: Some("0".repeat(40)),
|
||||
code_lang: Some("javascript".into()),
|
||||
},
|
||||
provenance: Provenance { events: vec![] },
|
||||
parser_version: pv,
|
||||
schema_version: 1,
|
||||
doc_version: 1,
|
||||
last_chunker_version: None,
|
||||
last_embedding_version: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn fixed_policy() -> ChunkPolicy {
|
||||
ChunkPolicy {
|
||||
target_tokens: 500,
|
||||
overlap_tokens: 80,
|
||||
respect_markdown_headings: false,
|
||||
chunker_version: ChunkerVersion("code-js-ast-v1".into()),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn code_js_ast_chunks_snapshot() {
|
||||
let doc = fixed_doc();
|
||||
let policy = fixed_policy();
|
||||
|
||||
let chunks = CodeJsAstV1Chunker.chunk(&doc, &policy).expect("chunk");
|
||||
let actual = serde_json::to_value(&chunks).unwrap();
|
||||
|
||||
let dir = fixtures_dir();
|
||||
let baseline_path = dir.join("code-sample.js.chunks.snapshot.json");
|
||||
let baseline_text = match std::fs::read_to_string(&baseline_path) {
|
||||
Ok(s) => s,
|
||||
Err(_) if std::env::var("UPDATE_SNAPSHOTS").is_ok() => {
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let pretty = serde_json::to_string_pretty(&actual).unwrap();
|
||||
std::fs::write(&baseline_path, format!("{pretty}\n")).unwrap();
|
||||
return;
|
||||
}
|
||||
Err(e) => panic!(
|
||||
"missing baseline {}; run with UPDATE_SNAPSHOTS=1 to create: {e}",
|
||||
baseline_path.display()
|
||||
),
|
||||
};
|
||||
let expected: Value = serde_json::from_str(&baseline_text).expect("baseline parses as json");
|
||||
|
||||
if actual != expected {
|
||||
if std::env::var("UPDATE_SNAPSHOTS").is_ok() {
|
||||
let pretty = serde_json::to_string_pretty(&actual).unwrap();
|
||||
std::fs::write(&baseline_path, format!("{pretty}\n")).unwrap();
|
||||
eprintln!("updated baseline {}", baseline_path.display());
|
||||
return;
|
||||
}
|
||||
let pretty = serde_json::to_string_pretty(&actual).unwrap();
|
||||
panic!(
|
||||
"code-js-ast-v1 chunks snapshot drift\n\
|
||||
--- expected ({}) ---\n{baseline_text}\n\
|
||||
--- actual ---\n{pretty}\n\
|
||||
If intentional, re-run with UPDATE_SNAPSHOTS=1.",
|
||||
baseline_path.display()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Determinism cross-check: re-running the same pipeline yields the same
|
||||
/// chunk_ids byte-for-byte.
|
||||
#[test]
|
||||
fn code_js_ast_chunks_are_deterministic() {
|
||||
let policy = fixed_policy();
|
||||
let baseline: Vec<String> = CodeJsAstV1Chunker
|
||||
.chunk(&fixed_doc(), &policy)
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|c| c.chunk_id.0)
|
||||
.collect();
|
||||
for _ in 0..5 {
|
||||
let again: Vec<String> = CodeJsAstV1Chunker
|
||||
.chunk(&fixed_doc(), &policy)
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|c| c.chunk_id.0)
|
||||
.collect();
|
||||
assert_eq!(again, baseline);
|
||||
}
|
||||
}
|
||||
221
crates/kebab-chunk/tests/code_python_ast_snapshot.rs
Normal file
221
crates/kebab-chunk/tests/code_python_ast_snapshot.rs
Normal file
@@ -0,0 +1,221 @@
|
||||
//! Snapshot test pinning the `Vec<Chunk>` JSON for a
|
||||
//! representative Python code `CanonicalDocument`.
|
||||
//!
|
||||
//! This is an integration test. `kebab-parse-code` is intentionally NOT
|
||||
//! a dev-dep (design §6.3 / §8 boundary: AST extraction is parser-side).
|
||||
//! The `CanonicalDocument` is built inline from hand-crafted `Block::Code`
|
||||
//! units, which is the same pattern used in `code_rust_ast_v1.rs`'s
|
||||
//! internal `code_doc` test helper.
|
||||
//!
|
||||
//! Set `UPDATE_SNAPSHOTS=1` to re-bake the baseline.
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use kebab_chunk::CodePythonAstV1Chunker;
|
||||
use kebab_core::{
|
||||
AssetId, Block, CanonicalDocument, ChunkPolicy, Chunker, ChunkerVersion, CodeBlock, CommonBlock,
|
||||
Lang, Metadata, ParserVersion, Provenance, SourceSpan, SourceType, TrustLevel, WorkspacePath,
|
||||
id_for_block, id_for_doc,
|
||||
};
|
||||
use serde_json::Value;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
fn fixtures_dir() -> PathBuf {
|
||||
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("tests")
|
||||
.join("fixtures")
|
||||
}
|
||||
|
||||
fn fixed_doc() -> CanonicalDocument {
|
||||
let wp = WorkspacePath("kebab_eval/metrics.py".into());
|
||||
let aid = AssetId("b".repeat(64));
|
||||
// Pin parser_version so doc_id / block_ids are reproducible.
|
||||
let pv = ParserVersion("code-python-v1".into());
|
||||
let doc_id = id_for_doc(&wp, &aid, &pv);
|
||||
|
||||
// Build a >200-line function body to force split_oversize.
|
||||
let big_body: String = {
|
||||
let header = "def big_compute(data):\n";
|
||||
let body: String = (0..210u32)
|
||||
.map(|i| format!(" v{i} = data[{i}] if {i} < len(data) else 0\n"))
|
||||
.collect();
|
||||
let footer = " return sum(data)";
|
||||
format!("{header}{body}{footer}")
|
||||
};
|
||||
let big_line_count = big_body.lines().count() as u32;
|
||||
let big_line_end = 48 + big_line_count - 1;
|
||||
|
||||
// Representative units:
|
||||
// 0. import block (lines 1–5, ≤200)
|
||||
// 1. free fn `compute_mrr` (lines 7–12, ≤200)
|
||||
// 2. class `MetricsCollector` (lines 14–20, ≤200)
|
||||
// 3. class `BaseEvaluator` (lines 22–30, ≤200)
|
||||
// 4. method `run` (lines 32–38, ≤200)
|
||||
// 5. method `report` (lines 40–46, ≤200)
|
||||
// 6. big_compute (>200 lines) to force split_oversize
|
||||
let raw_units: Vec<(&str, u32, u32, String)> = vec![
|
||||
(
|
||||
"imports",
|
||||
1,
|
||||
5,
|
||||
"import os\nimport sys\nfrom typing import List\nfrom pathlib import Path\nfrom collections import defaultdict".to_string(),
|
||||
),
|
||||
(
|
||||
"compute_mrr",
|
||||
7,
|
||||
12,
|
||||
"def compute_mrr(scores):\n if not scores:\n return 0.0\n return sum(\n 1.0 / r for r in scores\n ) / len(scores)".to_string(),
|
||||
),
|
||||
(
|
||||
"MetricsCollector",
|
||||
14,
|
||||
20,
|
||||
"class MetricsCollector:\n def __init__(self):\n self.scores = []\n self.labels = []\n self.counts = defaultdict(int)\n self.totals = defaultdict(float)\n self.tags = []".to_string(),
|
||||
),
|
||||
(
|
||||
"BaseEvaluator",
|
||||
22,
|
||||
30,
|
||||
"class BaseEvaluator:\n def evaluate(self, data):\n raise NotImplementedError\n def batch_evaluate(self, items):\n results = []\n for item in items:\n results.append(self.evaluate(item))\n return results\n def name(self):\n return type(self).__name__".to_string(),
|
||||
),
|
||||
(
|
||||
"MetricsCollector.run",
|
||||
32,
|
||||
38,
|
||||
"class MetricsCollector:\n def run(self, inputs):\n for inp in inputs:\n score = self._score(inp)\n self.scores.append(\n score\n )".to_string(),
|
||||
),
|
||||
(
|
||||
"MetricsCollector.report",
|
||||
40,
|
||||
46,
|
||||
"class MetricsCollector:\n def report(self):\n return {\n 'mean': sum(self.scores) / max(len(self.scores), 1),\n 'count': len(self.scores),\n 'tags': self.tags,\n }".to_string(),
|
||||
),
|
||||
("big_compute", 48, big_line_end, big_body),
|
||||
];
|
||||
|
||||
let blocks: Vec<Block> = raw_units
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, (sym, ls, le, code))| {
|
||||
let span = SourceSpan::Code {
|
||||
line_start: *ls,
|
||||
line_end: *le,
|
||||
symbol: Some((*sym).to_string()),
|
||||
lang: Some("python".into()),
|
||||
};
|
||||
let bid = id_for_block(&doc_id, "code", &[], i as u32, &span);
|
||||
Block::Code(CodeBlock {
|
||||
common: CommonBlock {
|
||||
block_id: bid,
|
||||
heading_path: vec![],
|
||||
source_span: span,
|
||||
},
|
||||
lang: Some("python".into()),
|
||||
code: code.clone(),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
CanonicalDocument {
|
||||
doc_id,
|
||||
source_asset_id: aid,
|
||||
workspace_path: wp,
|
||||
title: "metrics.py".into(),
|
||||
lang: Lang("und".into()),
|
||||
blocks,
|
||||
metadata: Metadata {
|
||||
aliases: vec![],
|
||||
tags: vec![],
|
||||
created_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
updated_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
source_type: SourceType::Note,
|
||||
trust_level: TrustLevel::Primary,
|
||||
user_id_alias: None,
|
||||
user: Default::default(),
|
||||
repo: Some("kebab".into()),
|
||||
git_branch: Some("main".into()),
|
||||
git_commit: Some("0".repeat(40)),
|
||||
code_lang: Some("python".into()),
|
||||
},
|
||||
provenance: Provenance { events: vec![] },
|
||||
parser_version: pv,
|
||||
schema_version: 1,
|
||||
doc_version: 1,
|
||||
last_chunker_version: None,
|
||||
last_embedding_version: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn fixed_policy() -> ChunkPolicy {
|
||||
ChunkPolicy {
|
||||
target_tokens: 500,
|
||||
overlap_tokens: 80,
|
||||
respect_markdown_headings: false,
|
||||
chunker_version: ChunkerVersion("code-python-ast-v1".into()),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn code_python_ast_chunks_snapshot() {
|
||||
let doc = fixed_doc();
|
||||
let policy = fixed_policy();
|
||||
|
||||
let chunks = CodePythonAstV1Chunker.chunk(&doc, &policy).expect("chunk");
|
||||
let actual = serde_json::to_value(&chunks).unwrap();
|
||||
|
||||
let dir = fixtures_dir();
|
||||
let baseline_path = dir.join("code-sample.py.chunks.snapshot.json");
|
||||
let baseline_text = match std::fs::read_to_string(&baseline_path) {
|
||||
Ok(s) => s,
|
||||
Err(_) if std::env::var("UPDATE_SNAPSHOTS").is_ok() => {
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let pretty = serde_json::to_string_pretty(&actual).unwrap();
|
||||
std::fs::write(&baseline_path, format!("{pretty}\n")).unwrap();
|
||||
return;
|
||||
}
|
||||
Err(e) => panic!(
|
||||
"missing baseline {}; run with UPDATE_SNAPSHOTS=1 to create: {e}",
|
||||
baseline_path.display()
|
||||
),
|
||||
};
|
||||
let expected: Value = serde_json::from_str(&baseline_text).expect("baseline parses as json");
|
||||
|
||||
if actual != expected {
|
||||
if std::env::var("UPDATE_SNAPSHOTS").is_ok() {
|
||||
let pretty = serde_json::to_string_pretty(&actual).unwrap();
|
||||
std::fs::write(&baseline_path, format!("{pretty}\n")).unwrap();
|
||||
eprintln!("updated baseline {}", baseline_path.display());
|
||||
return;
|
||||
}
|
||||
let pretty = serde_json::to_string_pretty(&actual).unwrap();
|
||||
panic!(
|
||||
"code-python-ast-v1 chunks snapshot drift\n\
|
||||
--- expected ({}) ---\n{baseline_text}\n\
|
||||
--- actual ---\n{pretty}\n\
|
||||
If intentional, re-run with UPDATE_SNAPSHOTS=1.",
|
||||
baseline_path.display()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Determinism cross-check: re-running the same pipeline yields the same
|
||||
/// chunk_ids byte-for-byte.
|
||||
#[test]
|
||||
fn code_python_ast_chunks_are_deterministic() {
|
||||
let policy = fixed_policy();
|
||||
let baseline: Vec<String> = CodePythonAstV1Chunker
|
||||
.chunk(&fixed_doc(), &policy)
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|c| c.chunk_id.0)
|
||||
.collect();
|
||||
for _ in 0..5 {
|
||||
let again: Vec<String> = CodePythonAstV1Chunker
|
||||
.chunk(&fixed_doc(), &policy)
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|c| c.chunk_id.0)
|
||||
.collect();
|
||||
assert_eq!(again, baseline);
|
||||
}
|
||||
}
|
||||
221
crates/kebab-chunk/tests/code_rust_ast_snapshot.rs
Normal file
221
crates/kebab-chunk/tests/code_rust_ast_snapshot.rs
Normal file
@@ -0,0 +1,221 @@
|
||||
//! Snapshot test pinning the `Vec<Chunk>` JSON for a
|
||||
//! representative Rust code `CanonicalDocument`.
|
||||
//!
|
||||
//! This is an integration test. `kebab-parse-code` is intentionally NOT
|
||||
//! a dev-dep (design §6.3 / §8 boundary: AST extraction is parser-side).
|
||||
//! The `CanonicalDocument` is built inline from hand-crafted `Block::Code`
|
||||
//! units, which is the same pattern used in `code_rust_ast_v1.rs`'s
|
||||
//! internal `code_doc` test helper.
|
||||
//!
|
||||
//! Set `UPDATE_SNAPSHOTS=1` to re-bake the baseline.
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use kebab_chunk::CodeRustAstV1Chunker;
|
||||
use kebab_core::{
|
||||
AssetId, Block, CanonicalDocument, ChunkPolicy, Chunker, ChunkerVersion, CodeBlock, CommonBlock,
|
||||
Lang, Metadata, ParserVersion, Provenance, SourceSpan, SourceType, TrustLevel, WorkspacePath,
|
||||
id_for_block, id_for_doc,
|
||||
};
|
||||
use serde_json::Value;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
fn fixtures_dir() -> PathBuf {
|
||||
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("tests")
|
||||
.join("fixtures")
|
||||
}
|
||||
|
||||
fn fixed_doc() -> CanonicalDocument {
|
||||
let wp = WorkspacePath("crates/kebab-chunk/src/code_rust_ast_v1.rs".into());
|
||||
let aid = AssetId("b".repeat(64));
|
||||
// Pin parser_version so doc_id / block_ids are reproducible.
|
||||
let pv = ParserVersion("code-rust-v1".into());
|
||||
let doc_id = id_for_doc(&wp, &aid, &pv);
|
||||
|
||||
// Build a >200-line function body to force split_oversize.
|
||||
let big_body: String = {
|
||||
let header = "pub fn big_fn(input: &[u8]) -> Vec<u8> {\n";
|
||||
let body: String = (0..210u32)
|
||||
.map(|i| format!(" let v{i} = input.get({i} as usize).copied().unwrap_or(0);\n"))
|
||||
.collect();
|
||||
let footer = " vec![0u8]\n}";
|
||||
format!("{header}{body}{footer}")
|
||||
};
|
||||
let big_line_count = big_body.lines().count() as u32;
|
||||
let big_line_end = 48 + big_line_count - 1;
|
||||
|
||||
// Representative units:
|
||||
// 0. top-level use+const block (lines 1–5, ≤200)
|
||||
// 1. free fn `parse` (lines 7–12, ≤200)
|
||||
// 2. struct `Foo` (lines 14–20, ≤200)
|
||||
// 3. trait `Frobable` (lines 22–30, ≤200)
|
||||
// 4. impl Foo::double (lines 32–38, ≤200)
|
||||
// 5. impl Foo::triple (lines 40–46, ≤200)
|
||||
// 6. big_fn (>200 lines) to force split_oversize
|
||||
let raw_units: Vec<(&str, u32, u32, String)> = vec![
|
||||
(
|
||||
"use+const",
|
||||
1,
|
||||
5,
|
||||
"use std::collections::HashMap;\nuse std::fmt;\n\nconst MAX: usize = 1024;\nconst MIN: usize = 0;".to_string(),
|
||||
),
|
||||
(
|
||||
"parse",
|
||||
7,
|
||||
12,
|
||||
"pub fn parse(input: &str) -> Option<u32> {\n input\n .trim()\n .parse()\n .ok()\n}".to_string(),
|
||||
),
|
||||
(
|
||||
"Foo",
|
||||
14,
|
||||
20,
|
||||
"pub struct Foo {\n pub name: String,\n pub value: u32,\n pub tags: Vec<String>,\n pub meta: Option<String>,\n pub count: usize,\n}".to_string(),
|
||||
),
|
||||
(
|
||||
"Frobable",
|
||||
22,
|
||||
30,
|
||||
"pub trait Frobable {\n fn frob(&self) -> String;\n fn frob_twice(&self) -> String {\n let a = self.frob();\n let b = self.frob();\n format!(\"{a}{b}\")\n }\n fn name(&self) -> &str;\n}".to_string(),
|
||||
),
|
||||
(
|
||||
"Foo::double",
|
||||
32,
|
||||
38,
|
||||
"impl Foo {\n pub fn double(&self) -> u32 {\n self.value\n .checked_mul(2)\n .unwrap_or(u32::MAX)\n }\n}".to_string(),
|
||||
),
|
||||
(
|
||||
"Foo::triple",
|
||||
40,
|
||||
46,
|
||||
"impl Foo {\n pub fn triple(&self) -> u32 {\n self.value\n .checked_mul(3)\n .unwrap_or(u32::MAX)\n }\n}".to_string(),
|
||||
),
|
||||
("big_fn", 48, big_line_end, big_body),
|
||||
];
|
||||
|
||||
let blocks: Vec<Block> = raw_units
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, (sym, ls, le, code))| {
|
||||
let span = SourceSpan::Code {
|
||||
line_start: *ls,
|
||||
line_end: *le,
|
||||
symbol: Some((*sym).to_string()),
|
||||
lang: Some("rust".into()),
|
||||
};
|
||||
let bid = id_for_block(&doc_id, "code", &[], i as u32, &span);
|
||||
Block::Code(CodeBlock {
|
||||
common: CommonBlock {
|
||||
block_id: bid,
|
||||
heading_path: vec![],
|
||||
source_span: span,
|
||||
},
|
||||
lang: Some("rust".into()),
|
||||
code: code.clone(),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
CanonicalDocument {
|
||||
doc_id,
|
||||
source_asset_id: aid,
|
||||
workspace_path: wp,
|
||||
title: "code_rust_ast_v1.rs".into(),
|
||||
lang: Lang("und".into()),
|
||||
blocks,
|
||||
metadata: Metadata {
|
||||
aliases: vec![],
|
||||
tags: vec![],
|
||||
created_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
updated_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
source_type: SourceType::Note,
|
||||
trust_level: TrustLevel::Primary,
|
||||
user_id_alias: None,
|
||||
user: Default::default(),
|
||||
repo: Some("kebab".into()),
|
||||
git_branch: Some("main".into()),
|
||||
git_commit: Some("0".repeat(40)),
|
||||
code_lang: Some("rust".into()),
|
||||
},
|
||||
provenance: Provenance { events: vec![] },
|
||||
parser_version: pv,
|
||||
schema_version: 1,
|
||||
doc_version: 1,
|
||||
last_chunker_version: None,
|
||||
last_embedding_version: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn fixed_policy() -> ChunkPolicy {
|
||||
ChunkPolicy {
|
||||
target_tokens: 500,
|
||||
overlap_tokens: 80,
|
||||
respect_markdown_headings: false,
|
||||
chunker_version: ChunkerVersion("code-rust-ast-v1".into()),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn code_rust_ast_chunks_snapshot() {
|
||||
let doc = fixed_doc();
|
||||
let policy = fixed_policy();
|
||||
|
||||
let chunks = CodeRustAstV1Chunker.chunk(&doc, &policy).expect("chunk");
|
||||
let actual = serde_json::to_value(&chunks).unwrap();
|
||||
|
||||
let dir = fixtures_dir();
|
||||
let baseline_path = dir.join("code-sample.chunks.snapshot.json");
|
||||
let baseline_text = match std::fs::read_to_string(&baseline_path) {
|
||||
Ok(s) => s,
|
||||
Err(_) if std::env::var("UPDATE_SNAPSHOTS").is_ok() => {
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let pretty = serde_json::to_string_pretty(&actual).unwrap();
|
||||
std::fs::write(&baseline_path, format!("{pretty}\n")).unwrap();
|
||||
return;
|
||||
}
|
||||
Err(e) => panic!(
|
||||
"missing baseline {}; run with UPDATE_SNAPSHOTS=1 to create: {e}",
|
||||
baseline_path.display()
|
||||
),
|
||||
};
|
||||
let expected: Value = serde_json::from_str(&baseline_text).expect("baseline parses as json");
|
||||
|
||||
if actual != expected {
|
||||
if std::env::var("UPDATE_SNAPSHOTS").is_ok() {
|
||||
let pretty = serde_json::to_string_pretty(&actual).unwrap();
|
||||
std::fs::write(&baseline_path, format!("{pretty}\n")).unwrap();
|
||||
eprintln!("updated baseline {}", baseline_path.display());
|
||||
return;
|
||||
}
|
||||
let pretty = serde_json::to_string_pretty(&actual).unwrap();
|
||||
panic!(
|
||||
"code-rust-ast-v1 chunks snapshot drift\n\
|
||||
--- expected ({}) ---\n{baseline_text}\n\
|
||||
--- actual ---\n{pretty}\n\
|
||||
If intentional, re-run with UPDATE_SNAPSHOTS=1.",
|
||||
baseline_path.display()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Determinism cross-check: re-running the same pipeline yields the same
|
||||
/// chunk_ids byte-for-byte.
|
||||
#[test]
|
||||
fn code_rust_ast_chunks_are_deterministic() {
|
||||
let policy = fixed_policy();
|
||||
let baseline: Vec<String> = CodeRustAstV1Chunker
|
||||
.chunk(&fixed_doc(), &policy)
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|c| c.chunk_id.0)
|
||||
.collect();
|
||||
for _ in 0..5 {
|
||||
let again: Vec<String> = CodeRustAstV1Chunker
|
||||
.chunk(&fixed_doc(), &policy)
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|c| c.chunk_id.0)
|
||||
.collect();
|
||||
assert_eq!(again, baseline);
|
||||
}
|
||||
}
|
||||
221
crates/kebab-chunk/tests/code_ts_ast_snapshot.rs
Normal file
221
crates/kebab-chunk/tests/code_ts_ast_snapshot.rs
Normal file
@@ -0,0 +1,221 @@
|
||||
//! Snapshot test pinning the `Vec<Chunk>` JSON for a
|
||||
//! representative TypeScript code `CanonicalDocument`.
|
||||
//!
|
||||
//! This is an integration test. `kebab-parse-code` is intentionally NOT
|
||||
//! a dev-dep (design §6.3 / §8 boundary: AST extraction is parser-side).
|
||||
//! The `CanonicalDocument` is built inline from hand-crafted `Block::Code`
|
||||
//! units, which is the same pattern used in `code_rust_ast_v1.rs`'s
|
||||
//! internal `code_doc` test helper.
|
||||
//!
|
||||
//! Set `UPDATE_SNAPSHOTS=1` to re-bake the baseline.
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use kebab_chunk::CodeTsAstV1Chunker;
|
||||
use kebab_core::{
|
||||
AssetId, Block, CanonicalDocument, ChunkPolicy, Chunker, ChunkerVersion, CodeBlock, CommonBlock,
|
||||
Lang, Metadata, ParserVersion, Provenance, SourceSpan, SourceType, TrustLevel, WorkspacePath,
|
||||
id_for_block, id_for_doc,
|
||||
};
|
||||
use serde_json::Value;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
fn fixtures_dir() -> PathBuf {
|
||||
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("tests")
|
||||
.join("fixtures")
|
||||
}
|
||||
|
||||
fn fixed_doc() -> CanonicalDocument {
|
||||
let wp = WorkspacePath("src/Foo.ts".into());
|
||||
let aid = AssetId("b".repeat(64));
|
||||
// Pin parser_version so doc_id / block_ids are reproducible.
|
||||
let pv = ParserVersion("code-ts-v1".into());
|
||||
let doc_id = id_for_doc(&wp, &aid, &pv);
|
||||
|
||||
// Build a >200-line method body to force split_oversize.
|
||||
let big_body: String = {
|
||||
let header = "export class BigProcessor {\n process(items: string[]): string[] {\n";
|
||||
let body: String = (0..210u32)
|
||||
.map(|i| format!(" const v{i} = items[{i}] ?? '';\n"))
|
||||
.collect();
|
||||
let footer = " return items;\n }\n}";
|
||||
format!("{header}{body}{footer}")
|
||||
};
|
||||
let big_line_count = big_body.lines().count() as u32;
|
||||
let big_line_end = 48 + big_line_count - 1;
|
||||
|
||||
// Representative units:
|
||||
// 0. import block (lines 1–5, ≤200)
|
||||
// 1. free fn `parseInput` (lines 7–12, ≤200)
|
||||
// 2. interface `Frobable` (lines 14–20, ≤200)
|
||||
// 3. class `Foo` (lines 22–30, ≤200)
|
||||
// 4. method `Foo.double` (lines 32–38, ≤200)
|
||||
// 5. method `Foo.triple` (lines 40–46, ≤200)
|
||||
// 6. BigProcessor (>200 lines) to force split_oversize
|
||||
let raw_units: Vec<(&str, u32, u32, String)> = vec![
|
||||
(
|
||||
"imports",
|
||||
1,
|
||||
5,
|
||||
"import { readFileSync } from 'fs';\nimport { join } from 'path';\nimport type { Config } from './config';\nimport { Logger } from './logger';\nimport { EventEmitter } from 'events';".to_string(),
|
||||
),
|
||||
(
|
||||
"parseInput",
|
||||
7,
|
||||
12,
|
||||
"export function parseInput(raw: string): number | null {\n const trimmed = raw.trim();\n const n = Number(trimmed);\n if (isNaN(n)) return null;\n return n;\n}".to_string(),
|
||||
),
|
||||
(
|
||||
"Frobable",
|
||||
14,
|
||||
20,
|
||||
"export interface Frobable {\n frob(): string;\n frobTwice(): string;\n readonly name: string;\n readonly tags: string[];\n count: number;\n reset(): void;\n}".to_string(),
|
||||
),
|
||||
(
|
||||
"Foo",
|
||||
22,
|
||||
30,
|
||||
"export class Foo implements Frobable {\n constructor(\n public readonly name: string,\n public value: number,\n public tags: string[] = [],\n ) {}\n frob(): string { return this.name; }\n frobTwice(): string { return this.name.repeat(2); }\n reset(): void { this.value = 0; }\n}".to_string(),
|
||||
),
|
||||
(
|
||||
"Foo.double",
|
||||
32,
|
||||
38,
|
||||
"export class Foo {\n double(): number {\n const result = this.value * 2;\n if (result > Number.MAX_SAFE_INTEGER) {\n return Number.MAX_SAFE_INTEGER;\n }\n return result;\n }\n}".to_string(),
|
||||
),
|
||||
(
|
||||
"Foo.triple",
|
||||
40,
|
||||
46,
|
||||
"export class Foo {\n triple(): number {\n const result = this.value * 3;\n if (result > Number.MAX_SAFE_INTEGER) {\n return Number.MAX_SAFE_INTEGER;\n }\n return result;\n }\n}".to_string(),
|
||||
),
|
||||
("BigProcessor", 48, big_line_end, big_body),
|
||||
];
|
||||
|
||||
let blocks: Vec<Block> = raw_units
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, (sym, ls, le, code))| {
|
||||
let span = SourceSpan::Code {
|
||||
line_start: *ls,
|
||||
line_end: *le,
|
||||
symbol: Some((*sym).to_string()),
|
||||
lang: Some("typescript".into()),
|
||||
};
|
||||
let bid = id_for_block(&doc_id, "code", &[], i as u32, &span);
|
||||
Block::Code(CodeBlock {
|
||||
common: CommonBlock {
|
||||
block_id: bid,
|
||||
heading_path: vec![],
|
||||
source_span: span,
|
||||
},
|
||||
lang: Some("typescript".into()),
|
||||
code: code.clone(),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
CanonicalDocument {
|
||||
doc_id,
|
||||
source_asset_id: aid,
|
||||
workspace_path: wp,
|
||||
title: "Foo.ts".into(),
|
||||
lang: Lang("und".into()),
|
||||
blocks,
|
||||
metadata: Metadata {
|
||||
aliases: vec![],
|
||||
tags: vec![],
|
||||
created_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
updated_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
source_type: SourceType::Note,
|
||||
trust_level: TrustLevel::Primary,
|
||||
user_id_alias: None,
|
||||
user: Default::default(),
|
||||
repo: Some("kebab".into()),
|
||||
git_branch: Some("main".into()),
|
||||
git_commit: Some("0".repeat(40)),
|
||||
code_lang: Some("typescript".into()),
|
||||
},
|
||||
provenance: Provenance { events: vec![] },
|
||||
parser_version: pv,
|
||||
schema_version: 1,
|
||||
doc_version: 1,
|
||||
last_chunker_version: None,
|
||||
last_embedding_version: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn fixed_policy() -> ChunkPolicy {
|
||||
ChunkPolicy {
|
||||
target_tokens: 500,
|
||||
overlap_tokens: 80,
|
||||
respect_markdown_headings: false,
|
||||
chunker_version: ChunkerVersion("code-ts-ast-v1".into()),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn code_ts_ast_chunks_snapshot() {
|
||||
let doc = fixed_doc();
|
||||
let policy = fixed_policy();
|
||||
|
||||
let chunks = CodeTsAstV1Chunker.chunk(&doc, &policy).expect("chunk");
|
||||
let actual = serde_json::to_value(&chunks).unwrap();
|
||||
|
||||
let dir = fixtures_dir();
|
||||
let baseline_path = dir.join("code-sample.ts.chunks.snapshot.json");
|
||||
let baseline_text = match std::fs::read_to_string(&baseline_path) {
|
||||
Ok(s) => s,
|
||||
Err(_) if std::env::var("UPDATE_SNAPSHOTS").is_ok() => {
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let pretty = serde_json::to_string_pretty(&actual).unwrap();
|
||||
std::fs::write(&baseline_path, format!("{pretty}\n")).unwrap();
|
||||
return;
|
||||
}
|
||||
Err(e) => panic!(
|
||||
"missing baseline {}; run with UPDATE_SNAPSHOTS=1 to create: {e}",
|
||||
baseline_path.display()
|
||||
),
|
||||
};
|
||||
let expected: Value = serde_json::from_str(&baseline_text).expect("baseline parses as json");
|
||||
|
||||
if actual != expected {
|
||||
if std::env::var("UPDATE_SNAPSHOTS").is_ok() {
|
||||
let pretty = serde_json::to_string_pretty(&actual).unwrap();
|
||||
std::fs::write(&baseline_path, format!("{pretty}\n")).unwrap();
|
||||
eprintln!("updated baseline {}", baseline_path.display());
|
||||
return;
|
||||
}
|
||||
let pretty = serde_json::to_string_pretty(&actual).unwrap();
|
||||
panic!(
|
||||
"code-ts-ast-v1 chunks snapshot drift\n\
|
||||
--- expected ({}) ---\n{baseline_text}\n\
|
||||
--- actual ---\n{pretty}\n\
|
||||
If intentional, re-run with UPDATE_SNAPSHOTS=1.",
|
||||
baseline_path.display()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Determinism cross-check: re-running the same pipeline yields the same
|
||||
/// chunk_ids byte-for-byte.
|
||||
#[test]
|
||||
fn code_ts_ast_chunks_are_deterministic() {
|
||||
let policy = fixed_policy();
|
||||
let baseline: Vec<String> = CodeTsAstV1Chunker
|
||||
.chunk(&fixed_doc(), &policy)
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|c| c.chunk_id.0)
|
||||
.collect();
|
||||
for _ in 0..5 {
|
||||
let again: Vec<String> = CodeTsAstV1Chunker
|
||||
.chunk(&fixed_doc(), &policy)
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|c| c.chunk_id.0)
|
||||
.collect();
|
||||
assert_eq!(again, baseline);
|
||||
}
|
||||
}
|
||||
170
crates/kebab-chunk/tests/fixtures/code-sample.chunks.snapshot.json
vendored
Normal file
170
crates/kebab-chunk/tests/fixtures/code-sample.chunks.snapshot.json
vendored
Normal file
File diff suppressed because one or more lines are too long
170
crates/kebab-chunk/tests/fixtures/code-sample.js.chunks.snapshot.json
vendored
Normal file
170
crates/kebab-chunk/tests/fixtures/code-sample.js.chunks.snapshot.json
vendored
Normal file
File diff suppressed because one or more lines are too long
170
crates/kebab-chunk/tests/fixtures/code-sample.py.chunks.snapshot.json
vendored
Normal file
170
crates/kebab-chunk/tests/fixtures/code-sample.py.chunks.snapshot.json
vendored
Normal file
File diff suppressed because one or more lines are too long
170
crates/kebab-chunk/tests/fixtures/code-sample.ts.chunks.snapshot.json
vendored
Normal file
170
crates/kebab-chunk/tests/fixtures/code-sample.ts.chunks.snapshot.json
vendored
Normal file
File diff suppressed because one or more lines are too long
@@ -46,3 +46,7 @@ ctrlc = "3"
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = { workspace = true }
|
||||
# p9-fb-32: backdate `documents.updated_at` in CLI integration tests
|
||||
# to simulate stale docs. `time` is the formatter used by the helper.
|
||||
rusqlite = { workspace = true }
|
||||
time = { workspace = true }
|
||||
|
||||
@@ -46,6 +46,11 @@ struct Cli {
|
||||
command: Cmd,
|
||||
}
|
||||
|
||||
// p10-1A-1: adding `repo` and `code_lang` Vec<String> fields pushed `Cmd`
|
||||
// over clippy's large_enum_variant threshold. The enum is short-lived
|
||||
// (parsed once at startup, never cloned in a hot path) — boxing would add
|
||||
// noise with no real benefit.
|
||||
#[allow(clippy::large_enum_variant)]
|
||||
#[derive(Subcommand, Debug)]
|
||||
enum Cmd {
|
||||
/// Initialise XDG dirs + workspace + `config.toml`.
|
||||
@@ -86,9 +91,16 @@ enum Cmd {
|
||||
what: InspectWhat,
|
||||
},
|
||||
|
||||
/// p9-fb-35: verbatim chunk / doc / span fetch.
|
||||
Fetch {
|
||||
#[command(subcommand)]
|
||||
what: FetchWhat,
|
||||
},
|
||||
|
||||
/// Lexical / vector / hybrid search over chunks.
|
||||
Search {
|
||||
query: String,
|
||||
/// Query text. Not required when `--bulk` is set (queries from stdin).
|
||||
query: Option<String>,
|
||||
|
||||
#[arg(long, default_value_t = 10)]
|
||||
k: usize,
|
||||
@@ -108,6 +120,85 @@ enum Cmd {
|
||||
/// future TUI cache-aware search and for explicit intent.
|
||||
#[arg(long)]
|
||||
no_cache: bool,
|
||||
|
||||
/// p9-fb-34: cap result wire JSON size at approximately N tokens
|
||||
/// (chars/4 estimate). When set, smaller snippets and fewer hits
|
||||
/// may be returned; check `truncated` in the JSON wire.
|
||||
#[arg(long)]
|
||||
max_tokens: Option<usize>,
|
||||
|
||||
/// p9-fb-34: per-hit snippet character cap, overrides
|
||||
/// `config.search.snippet_chars` for this call only.
|
||||
#[arg(long)]
|
||||
snippet_chars: Option<usize>,
|
||||
|
||||
/// p9-fb-34: opaque cursor from a previous response's
|
||||
/// `next_cursor` to fetch the next page. Mismatched
|
||||
/// `corpus_revision` returns `error.v1.code = stale_cursor`.
|
||||
#[arg(long)]
|
||||
cursor: Option<String>,
|
||||
|
||||
/// p9-fb-36: filter by `metadata.tags`. Repeatable; OR-within (any tag).
|
||||
#[arg(long)]
|
||||
tag: Vec<String>,
|
||||
|
||||
/// p9-fb-36: filter by `documents.lang` (ISO code).
|
||||
#[arg(long)]
|
||||
lang: Option<String>,
|
||||
|
||||
/// p9-fb-36: filter by `documents.workspace_path` glob.
|
||||
#[arg(long)]
|
||||
path_glob: Option<String>,
|
||||
|
||||
/// p9-fb-36: filter by minimum `documents.trust_level`.
|
||||
#[arg(long, value_enum)]
|
||||
trust_min: Option<TrustLevelFlag>,
|
||||
|
||||
/// p9-fb-36: filter by `assets.media_type` kind. Comma-separated.
|
||||
/// Aliases: `md` → `markdown`. Other accepted: `markdown`, `pdf`,
|
||||
/// `image`, `audio`, `other`. Unknown values match nothing.
|
||||
#[arg(long, value_delimiter = ',')]
|
||||
media: Vec<String>,
|
||||
|
||||
/// p9-fb-36: filter to docs whose `updated_at` is >= this RFC3339
|
||||
/// timestamp (UTC). Invalid format → exit 2 with error.v1
|
||||
/// code = config_invalid.
|
||||
#[arg(long)]
|
||||
ingested_after: Option<String>,
|
||||
|
||||
/// p9-fb-36: filter to a single doc by id.
|
||||
#[arg(long)]
|
||||
doc_id: Option<String>,
|
||||
|
||||
/// p10-1A-1: filter by repo name (`metadata.repo`). Repeatable;
|
||||
/// multi-value = OR. Empty = no filter (all repos returned).
|
||||
#[arg(long = "repo", value_name = "NAME", num_args = 1)]
|
||||
repo: Vec<String>,
|
||||
|
||||
/// p10-1A-1: filter by code language identifier (lowercase
|
||||
/// canonical). Repeatable or comma-separated.
|
||||
/// Examples: `rust`, `python`, `typescript`.
|
||||
/// Unknown values produce empty hits.
|
||||
#[arg(long = "code-lang", value_name = "LANG", num_args = 1, value_delimiter = ',')]
|
||||
code_lang: Vec<String>,
|
||||
|
||||
/// p9-fb-37: emit pre-fusion lexical / vector / RRF candidate
|
||||
/// lists + per-stage timing in the response. Bypasses cache
|
||||
/// (debug intent — fresh run guaranteed). Requires embeddings
|
||||
/// when `--mode hybrid` or `--mode vector`; lexical mode runs
|
||||
/// without embeddings via a no-op vector stub.
|
||||
#[arg(long)]
|
||||
trace: bool,
|
||||
|
||||
/// p9-fb-42: bulk multi-query mode. Reads ndjson from stdin —
|
||||
/// one JSON object per line, each item shape mirrors the
|
||||
/// single-query input. Output is per-query ndjson on stdout
|
||||
/// (one `bulk_search_item.v1` per line) plus a summary line on
|
||||
/// stderr. Single-query flags (`--mode`, `--k`, `--tag`, etc.)
|
||||
/// are ignored when `--bulk` is set; pass them per-item in the
|
||||
/// stdin JSON instead. Caps at 100 queries per call.
|
||||
#[arg(long)]
|
||||
bulk: bool,
|
||||
},
|
||||
|
||||
/// Retrieval-augmented question answering.
|
||||
@@ -153,6 +244,12 @@ enum Cmd {
|
||||
/// (e.g. `kebab-rust-async-2026-05`).
|
||||
#[arg(long, value_name = "ID")]
|
||||
session: Option<String>,
|
||||
|
||||
/// p9-fb-33: emit ndjson `answer_event.v1` events on stderr
|
||||
/// while streaming. Final stdout line is the existing
|
||||
/// `answer.v1`. Off by default to preserve final-only behavior.
|
||||
#[arg(long)]
|
||||
stream: bool,
|
||||
},
|
||||
|
||||
/// Wipe XDG data dirs (and optionally the Lance vector store) so the
|
||||
@@ -238,6 +335,33 @@ enum InspectWhat {
|
||||
Chunk { id: String },
|
||||
}
|
||||
|
||||
#[derive(Subcommand, Debug)]
|
||||
enum FetchWhat {
|
||||
/// Fetch a single chunk verbatim, optionally with surrounding context.
|
||||
Chunk {
|
||||
id: String,
|
||||
/// p9-fb-35: include ±N chunks before and after the target.
|
||||
#[arg(long)]
|
||||
context: Option<u32>,
|
||||
},
|
||||
/// Fetch the entire normalized markdown text of a document.
|
||||
Doc {
|
||||
id: String,
|
||||
/// p9-fb-35: chars/4 budget cap.
|
||||
#[arg(long)]
|
||||
max_tokens: Option<usize>,
|
||||
},
|
||||
/// Fetch a 1-based line range of a document. PDF / audio rejected.
|
||||
Span {
|
||||
doc_id: String,
|
||||
line_start: u32,
|
||||
line_end: u32,
|
||||
/// p9-fb-35: chars/4 budget cap.
|
||||
#[arg(long)]
|
||||
max_tokens: Option<usize>,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Subcommand, Debug)]
|
||||
enum EvalWhat {
|
||||
/// Run the golden suite end-to-end and persist `eval_runs` +
|
||||
@@ -295,6 +419,25 @@ impl From<ModeFlag> for kebab_core::SearchMode {
|
||||
}
|
||||
}
|
||||
|
||||
/// p9-fb-36: clap value enum for `--trust-min`. Maps to
|
||||
/// `kebab_core::TrustLevel` via `From`.
|
||||
#[derive(clap::ValueEnum, Clone, Debug)]
|
||||
enum TrustLevelFlag {
|
||||
Primary,
|
||||
Secondary,
|
||||
Generated,
|
||||
}
|
||||
|
||||
impl From<TrustLevelFlag> for kebab_core::TrustLevel {
|
||||
fn from(f: TrustLevelFlag) -> Self {
|
||||
match f {
|
||||
TrustLevelFlag::Primary => kebab_core::TrustLevel::Primary,
|
||||
TrustLevelFlag::Secondary => kebab_core::TrustLevel::Secondary,
|
||||
TrustLevelFlag::Generated => kebab_core::TrustLevel::Generated,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse boolean env var accepting "1", "true", "yes", "on" (case-insensitive)
|
||||
/// as truthy; "0", "false", "no", "off" as falsy. Used for `KEBAB_READONLY`.
|
||||
fn parse_bool_env(s: &str) -> Result<bool, String> {
|
||||
@@ -503,32 +646,246 @@ fn run(cli: &Cli) -> anyhow::Result<()> {
|
||||
}
|
||||
},
|
||||
|
||||
Cmd::Fetch { what } => {
|
||||
let cfg = kebab_config::Config::load(cli.config.as_deref())?;
|
||||
let (query, opts) = match what {
|
||||
FetchWhat::Chunk { id, context } => (
|
||||
kebab_core::FetchQuery::Chunk(kebab_core::ChunkId(id.clone())),
|
||||
kebab_core::FetchOpts {
|
||||
context: *context,
|
||||
max_tokens: None,
|
||||
},
|
||||
),
|
||||
FetchWhat::Doc { id, max_tokens } => (
|
||||
kebab_core::FetchQuery::Doc(kebab_core::DocumentId(id.clone())),
|
||||
kebab_core::FetchOpts {
|
||||
context: None,
|
||||
max_tokens: *max_tokens,
|
||||
},
|
||||
),
|
||||
FetchWhat::Span {
|
||||
doc_id,
|
||||
line_start,
|
||||
line_end,
|
||||
max_tokens,
|
||||
} => (
|
||||
kebab_core::FetchQuery::Span {
|
||||
doc_id: kebab_core::DocumentId(doc_id.clone()),
|
||||
line_start: *line_start,
|
||||
line_end: *line_end,
|
||||
},
|
||||
kebab_core::FetchOpts {
|
||||
context: None,
|
||||
max_tokens: *max_tokens,
|
||||
},
|
||||
),
|
||||
};
|
||||
let result = kebab_app::fetch_with_config(cfg, query, opts)?;
|
||||
if cli.json {
|
||||
println!("{}", serde_json::to_string(&wire::wire_fetch_result(&result))?);
|
||||
} else {
|
||||
render_fetch_plain(&result);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
Cmd::Search {
|
||||
query,
|
||||
k,
|
||||
mode,
|
||||
explain: _,
|
||||
no_cache,
|
||||
max_tokens,
|
||||
snippet_chars,
|
||||
cursor,
|
||||
tag,
|
||||
lang,
|
||||
path_glob,
|
||||
trust_min,
|
||||
media,
|
||||
ingested_after,
|
||||
doc_id,
|
||||
repo,
|
||||
code_lang,
|
||||
trace,
|
||||
bulk,
|
||||
} => {
|
||||
// p9-fb-42: bulk mode — stdin ndjson → bulk_search_with_config
|
||||
// → stdout ndjson per query + stderr summary. Single-query
|
||||
// flags are ignored (each item supplies its own).
|
||||
if *bulk {
|
||||
use std::io::{BufRead, Write};
|
||||
|
||||
let cfg = kebab_config::Config::load(cli.config.as_deref())?;
|
||||
|
||||
let stdin = std::io::stdin();
|
||||
let stdin_locked = stdin.lock();
|
||||
let mut raw_items: Vec<serde_json::Value> = Vec::new();
|
||||
for (lineno, line) in stdin_locked.lines().enumerate() {
|
||||
let line = line?;
|
||||
if line.trim().is_empty() {
|
||||
continue;
|
||||
}
|
||||
let v: serde_json::Value =
|
||||
serde_json::from_str(&line).map_err(|e| {
|
||||
anyhow::Error::new(kebab_app::StructuredError(
|
||||
kebab_app::ErrorV1 {
|
||||
schema_version: kebab_app::ERROR_V1_ID
|
||||
.to_string(),
|
||||
code: "config_invalid".to_string(),
|
||||
message: format!(
|
||||
"stdin ndjson line {} parse error: {e}",
|
||||
lineno + 1
|
||||
),
|
||||
details: serde_json::Value::Null,
|
||||
hint: Some(
|
||||
"each line must be a JSON object with at least `query`"
|
||||
.to_string(),
|
||||
),
|
||||
},
|
||||
))
|
||||
})?;
|
||||
raw_items.push(v);
|
||||
}
|
||||
|
||||
let (items, summary) =
|
||||
kebab_app::bulk_search_with_config(cfg, raw_items)?;
|
||||
|
||||
if cli.json {
|
||||
let mut stdout = std::io::stdout().lock();
|
||||
for item in &items {
|
||||
let v = wire::wire_bulk_search_item(item);
|
||||
writeln!(stdout, "{}", serde_json::to_string(&v)?)?;
|
||||
}
|
||||
eprintln!(
|
||||
"bulk_summary: total={} succeeded={} failed={}",
|
||||
summary.total, summary.succeeded, summary.failed,
|
||||
);
|
||||
} else {
|
||||
let mut stdout = std::io::stdout().lock();
|
||||
for (idx, item) in items.iter().enumerate() {
|
||||
writeln!(
|
||||
stdout,
|
||||
"# Query {}: {}",
|
||||
idx + 1,
|
||||
serde_json::to_string(&item.query)?,
|
||||
)?;
|
||||
if let Some(err) = &item.error {
|
||||
writeln!(stdout, "error: {}", err)?;
|
||||
} else if let Some(resp) = &item.response {
|
||||
writeln!(
|
||||
stdout,
|
||||
"{}",
|
||||
serde_json::to_string_pretty(resp)?
|
||||
)?;
|
||||
}
|
||||
writeln!(stdout)?;
|
||||
}
|
||||
eprintln!(
|
||||
"bulk_summary: total={} succeeded={} failed={}",
|
||||
summary.total, summary.succeeded, summary.failed,
|
||||
);
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let cfg = kebab_config::Config::load(cli.config.as_deref())?;
|
||||
|
||||
// p9-fb-42: bulk mode requires no query; single-query mode requires query.
|
||||
let query_text = match query.as_ref() {
|
||||
Some(q) => q.clone(),
|
||||
None => {
|
||||
return Err(anyhow::anyhow!("query is required unless --bulk is set"));
|
||||
}
|
||||
};
|
||||
|
||||
// p9-fb-36: normalize --media aliases (md → markdown).
|
||||
fn normalize_media_alias(s: &str) -> String {
|
||||
match s.to_ascii_lowercase().as_str() {
|
||||
"md" => "markdown".to_string(),
|
||||
other => other.to_string(),
|
||||
}
|
||||
}
|
||||
let media_norm: Vec<String> =
|
||||
media.iter().map(|s| normalize_media_alias(s)).collect();
|
||||
|
||||
// p9-fb-36: parse --ingested-after as RFC3339; structured error on failure.
|
||||
let ingested_after_parsed: Option<time::OffsetDateTime> =
|
||||
match ingested_after.as_deref() {
|
||||
Some(s) => {
|
||||
match time::OffsetDateTime::parse(
|
||||
s,
|
||||
&time::format_description::well_known::Rfc3339,
|
||||
) {
|
||||
Ok(ts) => Some(ts),
|
||||
Err(e) => {
|
||||
return Err(anyhow::Error::new(
|
||||
kebab_app::StructuredError(kebab_app::ErrorV1 {
|
||||
schema_version: kebab_app::ERROR_V1_ID.to_string(),
|
||||
code: "config_invalid".to_string(),
|
||||
message: format!(
|
||||
"--ingested-after: invalid RFC3339 timestamp '{s}': {e}"
|
||||
),
|
||||
details: serde_json::Value::Null,
|
||||
hint: Some(
|
||||
"expected format like 2026-04-01T00:00:00Z".to_string(),
|
||||
),
|
||||
}),
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
|
||||
// p9-fb-36 + p10-1A-1: build SearchFilters from CLI flags.
|
||||
let filters = kebab_core::SearchFilters {
|
||||
tags_any: tag.clone(),
|
||||
lang: lang.as_ref().map(|s| kebab_core::Lang(s.clone())),
|
||||
path_glob: path_glob.clone(),
|
||||
trust_min: trust_min.clone().map(Into::into),
|
||||
media: media_norm,
|
||||
ingested_after: ingested_after_parsed,
|
||||
doc_id: doc_id.as_ref().map(|s| kebab_core::DocumentId(s.clone())),
|
||||
repo: repo.clone(),
|
||||
code_lang: code_lang.clone(),
|
||||
};
|
||||
|
||||
let q = kebab_core::SearchQuery {
|
||||
text: query.clone(),
|
||||
text: query_text,
|
||||
mode: (*mode).into(),
|
||||
k: *k,
|
||||
filters: kebab_core::SearchFilters::default(),
|
||||
filters,
|
||||
};
|
||||
// p9-fb-19: --no-cache routes to the uncached facade.
|
||||
// Both calls go through the same App; only the cache
|
||||
// lookup/insert is skipped.
|
||||
let hits = if *no_cache {
|
||||
kebab_app::search_uncached_with_config(cfg, q)?
|
||||
} else {
|
||||
kebab_app::search_with_config(cfg, q)?
|
||||
let opts = kebab_core::SearchOpts {
|
||||
max_tokens: *max_tokens,
|
||||
snippet_chars: *snippet_chars,
|
||||
cursor: cursor.clone(),
|
||||
trace: *trace,
|
||||
};
|
||||
// p9-fb-34: budget-aware path. --no-cache still bypasses the
|
||||
// App-level LRU; wire wrapper applies regardless.
|
||||
let app = kebab_app::App::open_with_config(cfg)?;
|
||||
if *no_cache {
|
||||
app.clear_search_cache();
|
||||
}
|
||||
let resp = app.search_with_opts(q, opts)?;
|
||||
|
||||
if cli.json {
|
||||
println!("{}", serde_json::to_string(&wire::wire_search_hits(&hits))?);
|
||||
println!(
|
||||
"{}",
|
||||
serde_json::to_string(&wire::wire_search_response(&resp))?
|
||||
);
|
||||
} else {
|
||||
for h in &hits {
|
||||
// p9-fb-32: prefix `[stale]` on the doc_path for hits
|
||||
// whose `stale: true`. Yellow on TTY, plain otherwise —
|
||||
// mirrors the warning convention used by the progress
|
||||
// renderer (`progress.rs`). Detection uses stdlib
|
||||
// `IsTerminal` against stdout (the surface this print
|
||||
// lands on); no new dep.
|
||||
use std::io::IsTerminal;
|
||||
let color = std::io::stdout().is_terminal();
|
||||
for h in &resp.hits {
|
||||
// Show 4-digit score so RRF fused scores (bounded
|
||||
// ~0–0.033 for k_rrf=60) don't all collapse to "0.02".
|
||||
// Append heading_path so multiple chunks from the same
|
||||
@@ -538,14 +895,46 @@ fn run(cli: &Cli) -> anyhow::Result<()> {
|
||||
} else {
|
||||
format!(" > {}", h.heading_path.join(" / "))
|
||||
};
|
||||
let stale_tag = if h.stale {
|
||||
if color {
|
||||
"\x1b[33m[stale]\x1b[0m "
|
||||
} else {
|
||||
"[stale] "
|
||||
}
|
||||
} else {
|
||||
""
|
||||
};
|
||||
println!(
|
||||
"{:>2}. {:.4} {}{}",
|
||||
"{:>2}. {:.4} {}{}{}",
|
||||
h.rank,
|
||||
h.retrieval.fusion_score,
|
||||
stale_tag,
|
||||
h.doc_path.0,
|
||||
heading,
|
||||
);
|
||||
}
|
||||
// p9-fb-34: truncation hint goes to stderr so it
|
||||
// doesn't pollute the stdout hit list.
|
||||
if resp.truncated {
|
||||
let next = resp.next_cursor.as_deref().unwrap_or("(none)");
|
||||
eprintln!("[truncated; use --cursor {next} for the next page]");
|
||||
}
|
||||
if *trace {
|
||||
if let Some(t) = &resp.trace {
|
||||
eprintln!();
|
||||
eprintln!("Trace:");
|
||||
eprintln!(" lexical ({} hits, {}ms):", t.lexical.len(), t.timing.lexical_ms);
|
||||
for c in t.lexical.iter().take(3) {
|
||||
eprintln!(" rank={} score={:.4} chunk={}", c.rank, c.score, c.chunk_id.0);
|
||||
}
|
||||
eprintln!(" vector ({} hits, {}ms):", t.vector.len(), t.timing.vector_ms);
|
||||
for c in t.vector.iter().take(3) {
|
||||
eprintln!(" rank={} score={:.4} chunk={}", c.rank, c.score, c.chunk_id.0);
|
||||
}
|
||||
eprintln!(" fusion ({} inputs, {}ms)", t.rrf_inputs.len(), t.timing.fusion_ms);
|
||||
eprintln!(" total: {}ms", t.timing.total_ms);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -560,69 +949,138 @@ fn run(cli: &Cli) -> anyhow::Result<()> {
|
||||
show_citations,
|
||||
hide_citations,
|
||||
session,
|
||||
stream,
|
||||
} => {
|
||||
let cfg = kebab_config::Config::load(cli.config.as_deref())?;
|
||||
let opts = kebab_app::AskOpts {
|
||||
k: *k,
|
||||
explain: *explain,
|
||||
mode: (*mode).into(),
|
||||
temperature: *temperature,
|
||||
seed: *seed,
|
||||
// CLI ask is non-streaming today (the answer prints all at
|
||||
// once on completion). The TUI ask pane (P9-3) is what
|
||||
// wires up a real `mpsc::Sender` here.
|
||||
stream_sink: None,
|
||||
// p9-fb-18: when `--session` is set, the facade
|
||||
// (`ask_with_session_with_config`) loads prior turns
|
||||
// from SQLite and stuffs them into AskOpts.history
|
||||
// before calling `ask_with_history`. Single-shot path
|
||||
// (no `--session`) keeps the empty defaults.
|
||||
history: Vec::new(),
|
||||
conversation_id: None,
|
||||
turn_index: None,
|
||||
};
|
||||
let ans = match session.as_deref() {
|
||||
Some(sid) => kebab_app::ask_with_session_with_config(cfg, sid, query, opts)?,
|
||||
None => kebab_app::ask_with_config(cfg, query, opts)?,
|
||||
};
|
||||
if cli.json {
|
||||
println!("{}", serde_json::to_string(&wire::wire_answer(&ans))?);
|
||||
} else {
|
||||
println!("{}", ans.answer);
|
||||
// p9-fb-20: print the citation block after the
|
||||
// answer body when --hide-citations is not set
|
||||
// (--show-citations is the default). Skipped on
|
||||
// refusal-with-zero-citations to avoid an empty
|
||||
// `근거:` header.
|
||||
let print_citations = *show_citations && !*hide_citations;
|
||||
if print_citations && !ans.citations.is_empty() {
|
||||
println!();
|
||||
println!("근거:");
|
||||
for (idx, c) in ans.citations.iter().enumerate() {
|
||||
let marker = c
|
||||
.marker
|
||||
.clone()
|
||||
.unwrap_or_else(|| format!("{}", idx + 1));
|
||||
println!(" [{}] {}", marker, c.citation.to_uri());
|
||||
if *stream {
|
||||
// p9-fb-33: streaming branch. Background thread runs
|
||||
// ask_with_config (which calls into the rag pipeline);
|
||||
// main thread drains the receiver and writes
|
||||
// `answer_event.v1` ndjson to stderr. On BrokenPipe
|
||||
// (downstream consumer closed), drop the receiver so
|
||||
// the worker's next `send` returns SendError →
|
||||
// pipeline cancels with LlmStreamAborted. Final stdout
|
||||
// line is the existing `answer.v1` (mirrors
|
||||
// ingest_progress.v1 + ingest_report.v1 split).
|
||||
use std::io::Write;
|
||||
use std::sync::mpsc;
|
||||
|
||||
let (tx, rx) = mpsc::channel::<kebab_app::StreamEvent>();
|
||||
let opts = kebab_app::AskOpts {
|
||||
k: *k,
|
||||
explain: *explain,
|
||||
mode: (*mode).into(),
|
||||
temperature: *temperature,
|
||||
seed: *seed,
|
||||
stream_sink: Some(tx),
|
||||
history: Vec::new(),
|
||||
conversation_id: None,
|
||||
turn_index: None,
|
||||
};
|
||||
let cfg2 = cfg.clone();
|
||||
let q = query.clone();
|
||||
let session2 = session.clone();
|
||||
let handle = std::thread::spawn(
|
||||
move || -> anyhow::Result<kebab_core::Answer> {
|
||||
match session2.as_deref() {
|
||||
Some(sid) => kebab_app::ask_with_session_with_config(
|
||||
cfg2, sid, &q, opts,
|
||||
),
|
||||
None => kebab_app::ask_with_config(cfg2, &q, opts),
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
// Drain receiver, write ndjson to stderr until
|
||||
// completion or BrokenPipe.
|
||||
let mut cancelled_pipe = false;
|
||||
{
|
||||
let mut stderr = std::io::stderr().lock();
|
||||
for ev in &rx {
|
||||
let now = time::OffsetDateTime::now_utc();
|
||||
let v = wire::wire_answer_event(&ev, now);
|
||||
let line = serde_json::to_string(&v)?;
|
||||
if let Err(e) = writeln!(stderr, "{line}") {
|
||||
if e.kind() == std::io::ErrorKind::BrokenPipe {
|
||||
cancelled_pipe = true;
|
||||
break;
|
||||
}
|
||||
return Err(e.into());
|
||||
}
|
||||
}
|
||||
// p9-fb-20: retrieval 메타는 citation 별 점수가
|
||||
// AnswerCitation 에 없는 (`top_score` 만 retrieval-
|
||||
// 전체 max) 한계상 한 줄로 분리. per-citation score
|
||||
// 노출은 facade + AnswerCitation 의 미래 확장 후.
|
||||
println!(
|
||||
"(retrieval: top_score={:.2}, k={}, used={}/{})",
|
||||
ans.retrieval.top_score,
|
||||
ans.retrieval.k,
|
||||
ans.retrieval.chunks_used,
|
||||
ans.retrieval.chunks_returned,
|
||||
);
|
||||
}
|
||||
if cancelled_pipe {
|
||||
// Dropping the receiver signals to the worker —
|
||||
// the next `send` returns SendError, which the
|
||||
// pipeline interprets as a cancel.
|
||||
drop(rx);
|
||||
}
|
||||
|
||||
let result = handle
|
||||
.join()
|
||||
.map_err(|_| anyhow::anyhow!("ask worker panicked"))?;
|
||||
let ans = result?;
|
||||
|
||||
// Final stdout line — answer.v1 for backwards
|
||||
// compat. BrokenPipe on stdout is silent (caller
|
||||
// already gone).
|
||||
let final_json = serde_json::to_string(&wire::wire_answer(&ans))?;
|
||||
let _ = writeln!(std::io::stdout().lock(), "{final_json}");
|
||||
|
||||
if !ans.grounded {
|
||||
return Err(RefusalSignal.into());
|
||||
}
|
||||
Ok(())
|
||||
} else {
|
||||
let opts = kebab_app::AskOpts {
|
||||
k: *k,
|
||||
explain: *explain,
|
||||
mode: (*mode).into(),
|
||||
temperature: *temperature,
|
||||
seed: *seed,
|
||||
// CLI ask is non-streaming by default (the answer
|
||||
// prints all at once on completion). `--stream`
|
||||
// takes the branch above; the TUI ask pane (P9-3)
|
||||
// wires up its own `mpsc::Sender`.
|
||||
stream_sink: None,
|
||||
// p9-fb-18: when `--session` is set, the facade
|
||||
// (`ask_with_session_with_config`) loads prior turns
|
||||
// from SQLite and stuffs them into AskOpts.history
|
||||
// before calling `ask_with_history`. Single-shot path
|
||||
// (no `--session`) keeps the empty defaults.
|
||||
history: Vec::new(),
|
||||
conversation_id: None,
|
||||
turn_index: None,
|
||||
};
|
||||
let ans = match session.as_deref() {
|
||||
Some(sid) => kebab_app::ask_with_session_with_config(cfg, sid, query, opts)?,
|
||||
None => kebab_app::ask_with_config(cfg, query, opts)?,
|
||||
};
|
||||
if cli.json {
|
||||
println!("{}", serde_json::to_string(&wire::wire_answer(&ans))?);
|
||||
} else {
|
||||
println!("{}", ans.answer);
|
||||
// p9-fb-20: print the citation block after the
|
||||
// answer body when --hide-citations is not set
|
||||
// (--show-citations is the default). Skipped on
|
||||
// refusal-with-zero-citations to avoid an empty
|
||||
// `근거:` header.
|
||||
let print_citations = *show_citations && !*hide_citations;
|
||||
if print_citations && !ans.citations.is_empty() {
|
||||
// p9-fb-32: yellow `[stale]` prefix on TTY (mirrors
|
||||
// the search renderer's pattern in `Cmd::Search`).
|
||||
use std::io::IsTerminal;
|
||||
let color = std::io::stdout().is_terminal();
|
||||
let mut out = std::io::stdout().lock();
|
||||
render_ask_plain_citations(&mut out, &ans, color)?;
|
||||
}
|
||||
}
|
||||
// Refusal → exit 1.
|
||||
if !ans.grounded {
|
||||
return Err(RefusalSignal.into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
// Refusal → exit 1.
|
||||
if !ans.grounded {
|
||||
return Err(RefusalSignal.into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
Cmd::Reset {
|
||||
@@ -860,6 +1318,54 @@ fn run(cli: &Cli) -> anyhow::Result<()> {
|
||||
}
|
||||
}
|
||||
|
||||
/// p9-fb-32: render the plain (non-JSON) citation block for `kebab ask`.
|
||||
/// Mirrors the `Cmd::Search` plain renderer's `[stale]` convention —
|
||||
/// yellow ANSI on TTY, plain text otherwise. Detection uses stdlib
|
||||
/// `IsTerminal` at the call site; this function takes the resolved
|
||||
/// `color` boolean so tests can pin both branches deterministically.
|
||||
///
|
||||
/// Skipping the empty / no-citation path is the caller's responsibility
|
||||
/// (matches the original inline guard at the call site).
|
||||
fn render_ask_plain_citations(
|
||||
w: &mut impl std::io::Write,
|
||||
ans: &kebab_core::Answer,
|
||||
color: bool,
|
||||
) -> std::io::Result<()> {
|
||||
writeln!(w)?;
|
||||
writeln!(w, "근거:")?;
|
||||
for (idx, c) in ans.citations.iter().enumerate() {
|
||||
let marker = c
|
||||
.marker
|
||||
.clone()
|
||||
.unwrap_or_else(|| format!("{}", idx + 1));
|
||||
// p9-fb-32: `[stale]` prefix on the URI for citations whose
|
||||
// `stale: true`. Yellow on TTY, plain otherwise — mirrors the
|
||||
// search-plain renderer in `Cmd::Search`.
|
||||
let stale_tag = if c.stale {
|
||||
if color {
|
||||
"\x1b[33m[stale]\x1b[0m "
|
||||
} else {
|
||||
"[stale] "
|
||||
}
|
||||
} else {
|
||||
""
|
||||
};
|
||||
writeln!(w, " [{}] {}{}", marker, stale_tag, c.citation.to_uri())?;
|
||||
}
|
||||
// p9-fb-20: retrieval 메타는 citation 별 점수가 AnswerCitation 에
|
||||
// 없는 (`top_score` 만 retrieval-전체 max) 한계상 한 줄로 분리.
|
||||
// per-citation score 노출은 facade + AnswerCitation 의 미래 확장 후.
|
||||
writeln!(
|
||||
w,
|
||||
"(retrieval: top_score={:.2}, k={}, used={}/{})",
|
||||
ans.retrieval.top_score,
|
||||
ans.retrieval.k,
|
||||
ans.retrieval.chunks_used,
|
||||
ans.retrieval.chunks_returned,
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn print_schema_text(s: &kebab_app::SchemaV1) {
|
||||
println!("kebab v{}", s.kebab_version);
|
||||
println!();
|
||||
@@ -880,6 +1386,7 @@ fn print_schema_text(s: &kebab_app::SchemaV1) {
|
||||
("http_daemon", s.capabilities.http_daemon),
|
||||
("mcp_server", s.capabilities.mcp_server),
|
||||
("single_file_ingest", s.capabilities.single_file_ingest),
|
||||
("bulk_search", s.capabilities.bulk_search),
|
||||
];
|
||||
for (name, on) in caps {
|
||||
let mark = if on { "✓" } else { "✗" };
|
||||
@@ -937,3 +1444,154 @@ fn confirm_destructive(
|
||||
Ok(matches!(s.as_str(), "y" | "yes"))
|
||||
}
|
||||
|
||||
/// p9-fb-35: human-friendly plain output for `kebab fetch`.
|
||||
fn render_fetch_plain(r: &kebab_core::FetchResult) {
|
||||
println!("# {} ({})", r.doc_path.0, format_kind(r.kind));
|
||||
if r.stale {
|
||||
println!("[stale; indexed_at = {}]", r.indexed_at);
|
||||
}
|
||||
match r.kind {
|
||||
kebab_core::FetchKind::Chunk => {
|
||||
if !r.context_before.is_empty() {
|
||||
println!("\n=== before ===");
|
||||
for c in &r.context_before {
|
||||
let heading = c.heading_path.last().map(|s| s.as_str()).unwrap_or("");
|
||||
println!("[{} § {}]\n{}\n", c.chunk_id.0, heading, c.text);
|
||||
}
|
||||
}
|
||||
if let Some(c) = &r.chunk {
|
||||
println!("\n=== target ===");
|
||||
let heading = c.heading_path.last().map(|s| s.as_str()).unwrap_or("");
|
||||
println!("[{} § {}]\n{}\n", c.chunk_id.0, heading, c.text);
|
||||
}
|
||||
if !r.context_after.is_empty() {
|
||||
println!("\n=== after ===");
|
||||
for c in &r.context_after {
|
||||
let heading = c.heading_path.last().map(|s| s.as_str()).unwrap_or("");
|
||||
println!("[{} § {}]\n{}\n", c.chunk_id.0, heading, c.text);
|
||||
}
|
||||
}
|
||||
}
|
||||
kebab_core::FetchKind::Doc | kebab_core::FetchKind::Span => {
|
||||
if let Some(text) = &r.text {
|
||||
println!("\n{text}");
|
||||
}
|
||||
if r.truncated {
|
||||
eprintln!("[truncated; widen --max-tokens for fuller text]");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn format_kind(k: kebab_core::FetchKind) -> &'static str {
|
||||
match k {
|
||||
kebab_core::FetchKind::Chunk => "chunk",
|
||||
kebab_core::FetchKind::Doc => "doc",
|
||||
kebab_core::FetchKind::Span => "span",
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
//! p9-fb-32: unit tests for `render_ask_plain_citations`. The
|
||||
//! integration end-to-end (`tests/wire_ask_stale.rs`) is gated on
|
||||
//! a real Ollama, so we cover the renderer's `[stale]` logic here
|
||||
//! against a synthetic `Answer` instead.
|
||||
use super::*;
|
||||
use kebab_core::{
|
||||
Answer, AnswerCitation, AnswerRetrievalSummary, Citation, ModelRef,
|
||||
PromptTemplateVersion, SearchMode, TokenUsage, TraceId, WorkspacePath,
|
||||
};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
fn mk_answer(citations: Vec<AnswerCitation>) -> Answer {
|
||||
Answer {
|
||||
answer: "ans".into(),
|
||||
citations,
|
||||
grounded: true,
|
||||
refusal_reason: None,
|
||||
model: ModelRef {
|
||||
id: "test".into(),
|
||||
provider: "test".into(),
|
||||
dimensions: None,
|
||||
},
|
||||
embedding: None,
|
||||
prompt_template_version: PromptTemplateVersion("rag-v2".into()),
|
||||
retrieval: AnswerRetrievalSummary {
|
||||
trace_id: TraceId("ret_test".into()),
|
||||
mode: SearchMode::Lexical,
|
||||
k: 5,
|
||||
score_gate: 0.30,
|
||||
top_score: 0.80,
|
||||
chunks_returned: 1,
|
||||
chunks_used: 1,
|
||||
},
|
||||
usage: TokenUsage {
|
||||
prompt_tokens: 0,
|
||||
completion_tokens: 0,
|
||||
latency_ms: 0,
|
||||
},
|
||||
created_at: OffsetDateTime::now_utc(),
|
||||
conversation_id: None,
|
||||
turn_index: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn mk_citation(path: &str, stale: bool) -> AnswerCitation {
|
||||
AnswerCitation {
|
||||
marker: Some("1".into()),
|
||||
citation: Citation::Line {
|
||||
path: WorkspacePath::new(path.into()).unwrap(),
|
||||
start: 1,
|
||||
end: 1,
|
||||
section: None,
|
||||
},
|
||||
indexed_at: OffsetDateTime::now_utc(),
|
||||
stale,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn plain_marks_stale_citation_no_color() {
|
||||
let ans = mk_answer(vec![mk_citation("a.md", true)]);
|
||||
let mut buf = Vec::new();
|
||||
render_ask_plain_citations(&mut buf, &ans, false).unwrap();
|
||||
let out = String::from_utf8(buf).unwrap();
|
||||
assert!(
|
||||
out.contains("[stale]"),
|
||||
"expected `[stale]` marker in plain output, got:\n{out}"
|
||||
);
|
||||
// No ANSI when color = false.
|
||||
assert!(
|
||||
!out.contains("\x1b["),
|
||||
"unexpected ANSI escape in non-color output:\n{out}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn plain_marks_stale_citation_color_uses_yellow_ansi() {
|
||||
let ans = mk_answer(vec![mk_citation("a.md", true)]);
|
||||
let mut buf = Vec::new();
|
||||
render_ask_plain_citations(&mut buf, &ans, true).unwrap();
|
||||
let out = String::from_utf8(buf).unwrap();
|
||||
// Yellow ANSI + reset around the `[stale]` token, mirroring the
|
||||
// search-plain renderer in `Cmd::Search`.
|
||||
assert!(
|
||||
out.contains("\x1b[33m[stale]\x1b[0m"),
|
||||
"expected yellow [stale] ANSI sequence in color output, got:\n{out:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn plain_no_stale_tag_for_fresh_citation() {
|
||||
let ans = mk_answer(vec![mk_citation("a.md", false)]);
|
||||
let mut buf = Vec::new();
|
||||
render_ask_plain_citations(&mut buf, &ans, true).unwrap();
|
||||
let out = String::from_utf8(buf).unwrap();
|
||||
assert!(
|
||||
!out.contains("[stale]"),
|
||||
"unexpected `[stale]` marker for fresh citation:\n{out}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -75,10 +75,24 @@ pub fn wire_search_hit(h: &SearchHit) -> Value {
|
||||
tag_object(v, "search_hit.v1")
|
||||
}
|
||||
|
||||
/// Wrap a list of [`SearchHit`] values as a JSON array of `search_hit.v1`
|
||||
/// objects (one tag per element, per design §2.2).
|
||||
pub fn wire_search_hits(hits: &[SearchHit]) -> Value {
|
||||
Value::Array(hits.iter().map(wire_search_hit).collect())
|
||||
/// p9-fb-34: tag a `SearchResponse` as `search_response.v1`. Wraps
|
||||
/// the existing `search_hit.v1[]` array with pagination + truncation
|
||||
/// metadata. Replaces the previous bare `search_hit.v1[]` top-level
|
||||
/// array (`wire_search_hits`) — see HOTFIXES / fb-34 for the
|
||||
/// breaking shape change.
|
||||
pub fn wire_search_response(r: &kebab_app::SearchResponse) -> Value {
|
||||
let mut v = serde_json::json!({
|
||||
"hits": r.hits.iter().map(wire_search_hit).collect::<Vec<_>>(),
|
||||
"next_cursor": r.next_cursor,
|
||||
"truncated": r.truncated,
|
||||
});
|
||||
if let Some(trace) = &r.trace {
|
||||
let trace_v = serde_json::to_value(trace).expect("SearchTrace serializes");
|
||||
if let Value::Object(ref mut map) = v {
|
||||
map.insert("trace".to_string(), trace_v);
|
||||
}
|
||||
}
|
||||
tag_object(v, "search_response.v1")
|
||||
}
|
||||
|
||||
/// Wrap an [`Answer`] as `answer.v1`.
|
||||
@@ -87,6 +101,25 @@ pub fn wire_answer(a: &Answer) -> Value {
|
||||
tag_object(v, "answer.v1")
|
||||
}
|
||||
|
||||
/// p9-fb-33: tag a [`StreamEvent`] as `answer_event.v1` ndjson.
|
||||
///
|
||||
/// The timestamp is added at emit time (caller fills `ts`), since the
|
||||
/// pipeline doesn't carry one in the in-process enum — mirrors the
|
||||
/// `wire_ingest_progress` pattern (§2 ingest_progress.v1).
|
||||
pub fn wire_answer_event(
|
||||
ev: &kebab_app::StreamEvent,
|
||||
ts: time::OffsetDateTime,
|
||||
) -> Value {
|
||||
let mut v = serde_json::to_value(ev).expect("StreamEvent serializes");
|
||||
let ts_str = ts
|
||||
.format(&time::format_description::well_known::Rfc3339)
|
||||
.expect("OffsetDateTime formats as RFC3339");
|
||||
if let Value::Object(ref mut map) = v {
|
||||
map.insert("ts".to_string(), Value::String(ts_str));
|
||||
}
|
||||
tag_object(v, "answer_event.v1")
|
||||
}
|
||||
|
||||
/// Idempotent pass-through for [`DoctorReport`] — the type already carries
|
||||
/// `schema_version: "doctor.v1"` (struct-field convention, the one
|
||||
/// exception called out in the module doc above). This helper exists so
|
||||
@@ -162,6 +195,26 @@ pub fn wire_error_v1(e: &kebab_app::ErrorV1) -> Value {
|
||||
tag_object(v, "error.v1")
|
||||
}
|
||||
|
||||
/// p9-fb-35: tag a [`kebab_core::FetchResult`] as `fetch_result.v1`.
|
||||
pub fn wire_fetch_result(r: &kebab_core::FetchResult) -> Value {
|
||||
let v = serde_json::to_value(r).expect("FetchResult serializes");
|
||||
tag_object(v, "fetch_result.v1")
|
||||
}
|
||||
|
||||
/// p9-fb-42: tag a `BulkSearchItem` (already serialized as a Value)
|
||||
/// as `bulk_search_item.v1`. The inner `query` / `response` / `error`
|
||||
/// fields stay verbatim — only the envelope gets the schema_version stamp.
|
||||
pub fn wire_bulk_search_item(item: &kebab_core::BulkSearchItem) -> Value {
|
||||
let mut v = serde_json::to_value(item).expect("BulkSearchItem serializes");
|
||||
if let Value::Object(ref mut map) = v {
|
||||
map.insert(
|
||||
"schema_version".to_string(),
|
||||
Value::String("bulk_search_item.v1".to_string()),
|
||||
);
|
||||
}
|
||||
v
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -186,7 +239,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn ingest_wrapper_tags_schema_version() {
|
||||
use kebab_core::SourceScope;
|
||||
use kebab_core::{SkipExamples, SourceScope};
|
||||
let r = IngestReport {
|
||||
scope: SourceScope {
|
||||
root: std::path::PathBuf::from("/tmp"),
|
||||
@@ -201,6 +254,12 @@ mod tests {
|
||||
errors: 0,
|
||||
duration_ms: 0,
|
||||
skipped_by_extension: std::collections::BTreeMap::new(),
|
||||
skipped_gitignore: 0,
|
||||
skipped_kebabignore: 0,
|
||||
skipped_builtin_blacklist: 0,
|
||||
skipped_generated: 0,
|
||||
skipped_size_exceeded: 0,
|
||||
skip_examples: SkipExamples::default(),
|
||||
items: None,
|
||||
};
|
||||
let v = wire_ingest(&r);
|
||||
@@ -215,13 +274,6 @@ mod tests {
|
||||
assert_eq!(v.as_array().unwrap().len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn search_hits_wraps_each_element() {
|
||||
let v = wire_search_hits(&[]);
|
||||
assert!(v.is_array());
|
||||
assert_eq!(v.as_array().unwrap().len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tag_object_inserts_into_object() {
|
||||
let v = Value::Object(serde_json::Map::new());
|
||||
@@ -229,6 +281,31 @@ mod tests {
|
||||
assert_eq!(schema_of(&tagged), Some("x.v1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn search_response_carries_pagination_metadata() {
|
||||
// p9-fb-34: empty-hits SearchResponse round-trips through the
|
||||
// wrapper with its `next_cursor` + `truncated` fields preserved
|
||||
// and the top-level `schema_version` set to `search_response.v1`.
|
||||
let r = kebab_app::SearchResponse {
|
||||
hits: vec![],
|
||||
next_cursor: Some("opaque-cursor-abc".to_string()),
|
||||
truncated: true,
|
||||
trace: None,
|
||||
};
|
||||
let v = wire_search_response(&r);
|
||||
assert_eq!(schema_of(&v), Some("search_response.v1"));
|
||||
assert!(v.get("hits").and_then(|h| h.as_array()).is_some());
|
||||
assert_eq!(
|
||||
v.get("hits").and_then(|h| h.as_array()).unwrap().len(),
|
||||
0
|
||||
);
|
||||
assert_eq!(
|
||||
v.get("next_cursor").and_then(|c| c.as_str()),
|
||||
Some("opaque-cursor-abc")
|
||||
);
|
||||
assert_eq!(v.get("truncated").and_then(|t| t.as_bool()), Some(true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn schema_wrapper_tags_schema_version() {
|
||||
use kebab_app::{Capabilities, Models, SchemaV1, Stats, WireBlock};
|
||||
@@ -240,7 +317,7 @@ mod tests {
|
||||
json_mode: true, ingest_progress: true, ingest_cancellation: true,
|
||||
rag_multi_turn: true, search_cache: true, incremental_ingest: true,
|
||||
streaming_ask: false, http_daemon: false, mcp_server: false,
|
||||
single_file_ingest: false,
|
||||
single_file_ingest: false, bulk_search: true,
|
||||
},
|
||||
models: Models {
|
||||
parser_version: "x".to_string(),
|
||||
@@ -253,6 +330,12 @@ mod tests {
|
||||
stats: Stats {
|
||||
doc_count: 1, chunk_count: 2, asset_count: 1,
|
||||
last_ingest_at: None,
|
||||
media_breakdown: Default::default(),
|
||||
lang_breakdown: Default::default(),
|
||||
index_bytes: Default::default(),
|
||||
stale_doc_count: 0,
|
||||
// p10-1A-1: new fields added to Stats; use Default for the test fixture.
|
||||
..Default::default()
|
||||
},
|
||||
};
|
||||
let v = wire_schema(&schema);
|
||||
@@ -293,4 +376,49 @@ mod tests {
|
||||
assert_eq!(paths.len(), 1);
|
||||
assert_eq!(paths[0].as_str(), Some("/tmp/x"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn search_response_with_trace_serializes_trace_field() {
|
||||
use kebab_core::{SearchTrace, TraceCandidate, TraceFusionInput,
|
||||
TraceTiming, ChunkId, DocumentId, WorkspacePath};
|
||||
let r = kebab_app::SearchResponse {
|
||||
hits: vec![],
|
||||
next_cursor: None,
|
||||
truncated: false,
|
||||
trace: Some(SearchTrace {
|
||||
lexical: vec![TraceCandidate {
|
||||
chunk_id: ChunkId("c1".into()),
|
||||
doc_id: DocumentId("d1".into()),
|
||||
doc_path: WorkspacePath::new("a.md".into()).unwrap(),
|
||||
rank: 1,
|
||||
score: 0.42,
|
||||
}],
|
||||
vector: vec![],
|
||||
rrf_inputs: vec![TraceFusionInput {
|
||||
chunk_id: ChunkId("c1".into()),
|
||||
lexical_rank: Some(1),
|
||||
vector_rank: None,
|
||||
fusion_score: 0.0,
|
||||
}],
|
||||
timing: TraceTiming { lexical_ms: 5, vector_ms: 0, fusion_ms: 1, total_ms: 7 },
|
||||
}),
|
||||
};
|
||||
let v = wire_search_response(&r);
|
||||
assert_eq!(schema_of(&v), Some("search_response.v1"));
|
||||
assert!(v["trace"].is_object());
|
||||
assert_eq!(v["trace"]["timing"]["lexical_ms"], 5);
|
||||
assert_eq!(v["trace"]["lexical"][0]["chunk_id"], "c1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn search_response_without_trace_omits_field() {
|
||||
let r = kebab_app::SearchResponse {
|
||||
hits: vec![],
|
||||
next_cursor: None,
|
||||
truncated: false,
|
||||
trace: None,
|
||||
};
|
||||
let v = wire_search_response(&r);
|
||||
assert!(v.get("trace").is_none(), "trace field absent when None");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -66,8 +66,8 @@ fn cli_mcp_initialize_then_tools_list() {
|
||||
.expect("tools/list result.tools must be an array");
|
||||
assert_eq!(
|
||||
tools.len(),
|
||||
6,
|
||||
"expected 6 tools (schema, doctor, search, ask, ingest_file, ingest_stdin), got {}: {list}",
|
||||
8,
|
||||
"expected 8 tools (schema, doctor, search, bulk_search, ask, fetch, ingest_file, ingest_stdin), got {}: {list}",
|
||||
tools.len()
|
||||
);
|
||||
|
||||
|
||||
243
crates/kebab-cli/tests/common/mod.rs
Normal file
243
crates/kebab-cli/tests/common/mod.rs
Normal file
@@ -0,0 +1,243 @@
|
||||
//! Shared CLI integration-test helpers.
|
||||
//!
|
||||
//! Each consumer (`tests/wire_search_stale.rs`, `tests/wire_ask_stale.rs`)
|
||||
//! does `mod common;` and calls these via `common::write_config(...)`,
|
||||
//! `common::ingest(...)`, `common::backdate_updated_at(...)`.
|
||||
//!
|
||||
//! `#![allow(dead_code)]` because each consumer typically uses only a
|
||||
//! subset of the helpers; rustc would otherwise warn about the unused
|
||||
//! ones in any single consumer's compilation.
|
||||
|
||||
#![allow(dead_code)]
|
||||
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
|
||||
/// Build a `config.toml` text under `dir`. `workspace_root` and
|
||||
/// `data_dir` live inside `dir`. `stale_threshold_days` is plumbed
|
||||
/// into `[search]` so the staleness post-process can fire.
|
||||
///
|
||||
/// Returns `(cfg_path, workspace_dir, data_dir)`.
|
||||
pub fn write_config(dir: &Path, stale_threshold_days: u32) -> (PathBuf, PathBuf, PathBuf) {
|
||||
write_config_with_llm_model(dir, stale_threshold_days, "none")
|
||||
}
|
||||
|
||||
/// Like [`write_config`] but lets the caller pin a specific
|
||||
/// `[models.llm].model` value — needed by `wire_ask_stale.rs` which
|
||||
/// hits a real Ollama and wants `gemma4:e4b` instead of `none`.
|
||||
pub fn write_config_with_llm_model(
|
||||
dir: &Path,
|
||||
stale_threshold_days: u32,
|
||||
llm_model: &str,
|
||||
) -> (PathBuf, PathBuf, PathBuf) {
|
||||
let workspace = dir.join("workspace");
|
||||
let data = dir.join("data");
|
||||
fs::create_dir_all(&workspace).unwrap();
|
||||
fs::create_dir_all(&data).unwrap();
|
||||
|
||||
let cfg_path = dir.join("config.toml");
|
||||
fs::write(
|
||||
&cfg_path,
|
||||
format!(
|
||||
r#"schema_version = 1
|
||||
|
||||
[workspace]
|
||||
root = "{workspace}"
|
||||
exclude = [".git/**"]
|
||||
|
||||
[storage]
|
||||
data_dir = "{data}"
|
||||
sqlite = "{{data_dir}}/kebab.sqlite"
|
||||
vector_dir = "{{data_dir}}/lancedb"
|
||||
asset_dir = "{{data_dir}}/assets"
|
||||
artifact_dir = "{{data_dir}}/artifacts"
|
||||
model_dir = "{{data_dir}}/models"
|
||||
runs_dir = "{{data_dir}}/runs"
|
||||
copy_threshold_mb = 100
|
||||
|
||||
[indexing]
|
||||
max_parallel_extractors = 2
|
||||
max_parallel_embeddings = 1
|
||||
watch_filesystem = false
|
||||
|
||||
[chunking]
|
||||
target_tokens = 80
|
||||
overlap_tokens = 20
|
||||
respect_markdown_headings = true
|
||||
chunker_version = "md-heading-v1"
|
||||
|
||||
[models.embedding]
|
||||
provider = "none"
|
||||
model = "none"
|
||||
version = "v0"
|
||||
dimensions = 0
|
||||
batch_size = 1
|
||||
|
||||
[models.llm]
|
||||
provider = "ollama"
|
||||
model = "{llm_model}"
|
||||
context_tokens = 4096
|
||||
endpoint = "http://127.0.0.1:11434"
|
||||
temperature = 0.0
|
||||
seed = 0
|
||||
|
||||
[search]
|
||||
default_k = 10
|
||||
hybrid_fusion = "rrf"
|
||||
rrf_k = 60
|
||||
snippet_chars = 220
|
||||
stale_threshold_days = {stale_threshold_days}
|
||||
|
||||
[rag]
|
||||
prompt_template_version = "rag-v1"
|
||||
score_gate = 0.30
|
||||
explain_default = false
|
||||
max_context_tokens = 8000
|
||||
"#,
|
||||
workspace = workspace.display(),
|
||||
data = data.display(),
|
||||
llm_model = llm_model,
|
||||
stale_threshold_days = stale_threshold_days,
|
||||
),
|
||||
)
|
||||
.unwrap();
|
||||
(cfg_path, workspace, data)
|
||||
}
|
||||
|
||||
/// Run `kebab ingest --root <workspace>` against the given config.
|
||||
/// Asserts success — failures abort the calling test.
|
||||
pub fn ingest(cfg: &Path, workspace: &Path) {
|
||||
let bin = env!("CARGO_BIN_EXE_kebab");
|
||||
let out = Command::new(bin)
|
||||
.args([
|
||||
"--config",
|
||||
cfg.to_str().unwrap(),
|
||||
"ingest",
|
||||
"--root",
|
||||
workspace.to_str().unwrap(),
|
||||
])
|
||||
.output()
|
||||
.unwrap();
|
||||
assert!(
|
||||
out.status.success(),
|
||||
"ingest failed: stderr={}",
|
||||
String::from_utf8_lossy(&out.stderr)
|
||||
);
|
||||
}
|
||||
|
||||
/// p9-fb-34: invoke `kebab search` with arbitrary trailing flags +
|
||||
/// query, capture stdout + stderr. Caller is responsible for
|
||||
/// supplying `--mode lexical` / `--json` etc. as needed; this helper
|
||||
/// stays unopinionated so a single test can exercise both wire shapes
|
||||
/// (JSON wrapper + plain stderr hint). Asserts the binary exited 0;
|
||||
/// non-zero exits fail the test with stderr included.
|
||||
pub fn run_search_with_args(cfg: &Path, args: &[&str]) -> (String, String) {
|
||||
let bin = env!("CARGO_BIN_EXE_kebab");
|
||||
let mut cmd = Command::new(bin);
|
||||
cmd.arg("--config").arg(cfg).arg("search");
|
||||
cmd.args(args);
|
||||
let out = cmd.output().expect("kebab search");
|
||||
assert!(
|
||||
out.status.success(),
|
||||
"search failed: args={args:?} stderr={}",
|
||||
String::from_utf8_lossy(&out.stderr)
|
||||
);
|
||||
(
|
||||
String::from_utf8_lossy(&out.stdout).to_string(),
|
||||
String::from_utf8_lossy(&out.stderr).to_string(),
|
||||
)
|
||||
}
|
||||
|
||||
/// p9-fb-33: invoke `kebab ask --stream --mode lexical <query>` and
|
||||
/// capture stdout + stderr. Lexical mode skips embeddings (matches
|
||||
/// `wire_ask_stale.rs::run_ask_lexical`). Caller asserts on the
|
||||
/// resulting (stdout, stderr) pair.
|
||||
pub fn run_ask_stream(cfg: &Path, query: &str) -> (String, String) {
|
||||
let bin = env!("CARGO_BIN_EXE_kebab");
|
||||
let out = Command::new(bin)
|
||||
.args([
|
||||
"--config",
|
||||
cfg.to_str().unwrap(),
|
||||
"ask",
|
||||
"--stream",
|
||||
"--mode",
|
||||
"lexical",
|
||||
query,
|
||||
])
|
||||
.output()
|
||||
.expect("kebab ask --stream");
|
||||
(
|
||||
String::from_utf8_lossy(&out.stdout).to_string(),
|
||||
String::from_utf8_lossy(&out.stderr).to_string(),
|
||||
)
|
||||
}
|
||||
|
||||
/// p9-fb-33: invoke `kebab --json ask --mode lexical <query>` (no
|
||||
/// `--stream`) — used by `wire_ask_stream::non_stream_path_unchanged`
|
||||
/// to confirm the non-streaming JSON path still emits a single
|
||||
/// `answer.v1` line on stdout. Returns stdout only (mirrors
|
||||
/// `wire_ask_stale.rs::run_ask_lexical(json=true)` minus the
|
||||
/// `Output` indirection).
|
||||
pub fn run_ask_json(cfg: &Path, query: &str) -> String {
|
||||
let bin = env!("CARGO_BIN_EXE_kebab");
|
||||
let out = Command::new(bin)
|
||||
.args([
|
||||
"--config",
|
||||
cfg.to_str().unwrap(),
|
||||
"--json",
|
||||
"ask",
|
||||
"--mode",
|
||||
"lexical",
|
||||
query,
|
||||
])
|
||||
.output()
|
||||
.expect("kebab ask --json");
|
||||
String::from_utf8_lossy(&out.stdout).to_string()
|
||||
}
|
||||
|
||||
/// p9-fb-35: invoke `kebab fetch` with arbitrary trailing flags,
|
||||
/// capture stdout + stderr. Caller is responsible for supplying
|
||||
/// `--json` (global flag) before the subcommand position via the
|
||||
/// `args` slice (e.g. `&["--json", "chunk", &id]`). Asserts the
|
||||
/// binary exited 0; non-zero exits fail the test with stderr
|
||||
/// included — for negative-path tests (unknown chunk_id etc.) drive
|
||||
/// the binary directly via `std::process::Command`.
|
||||
pub fn run_fetch_with_args(cfg: &Path, args: &[&str]) -> (String, String) {
|
||||
let bin = env!("CARGO_BIN_EXE_kebab");
|
||||
let mut cmd = Command::new(bin);
|
||||
cmd.arg("--config").arg(cfg).arg("fetch");
|
||||
cmd.args(args);
|
||||
let out = cmd.output().expect("kebab fetch");
|
||||
assert!(
|
||||
out.status.success(),
|
||||
"fetch failed: args={args:?} stderr={}",
|
||||
String::from_utf8_lossy(&out.stderr)
|
||||
);
|
||||
(
|
||||
String::from_utf8_lossy(&out.stdout).to_string(),
|
||||
String::from_utf8_lossy(&out.stderr).to_string(),
|
||||
)
|
||||
}
|
||||
|
||||
/// Rewrite `documents.updated_at` for one workspace path to
|
||||
/// `now - days_ago` (RFC3339 UTC). Mirrors
|
||||
/// `kebab-app/tests/common/mod.rs::backdate_document_updated_at`.
|
||||
/// Asserts exactly one row is updated — typo-proofs the workspace path.
|
||||
pub fn backdate_updated_at(data_dir: &Path, workspace_path: &str, days_ago: i64) {
|
||||
let backdated = (time::OffsetDateTime::now_utc() - time::Duration::days(days_ago))
|
||||
.format(&time::format_description::well_known::Rfc3339)
|
||||
.expect("format backdated updated_at");
|
||||
let db_path = data_dir.join("kebab.sqlite");
|
||||
let conn = rusqlite::Connection::open(&db_path).expect("open kebab.sqlite");
|
||||
let updated = conn
|
||||
.execute(
|
||||
"UPDATE documents SET updated_at = ?1 WHERE workspace_path = ?2",
|
||||
rusqlite::params![backdated, workspace_path],
|
||||
)
|
||||
.expect("UPDATE documents.updated_at");
|
||||
assert_eq!(
|
||||
updated, 1,
|
||||
"backdate_updated_at: expected to update exactly 1 row for {workspace_path}, got {updated}"
|
||||
);
|
||||
}
|
||||
102
crates/kebab-cli/tests/wire_ask_stale.rs
Normal file
102
crates/kebab-cli/tests/wire_ask_stale.rs
Normal file
@@ -0,0 +1,102 @@
|
||||
//! p9-fb-32: CLI ask output — JSON path emits `indexed_at` + `stale`
|
||||
//! on each citation; plain output prefixes stale citations with
|
||||
//! `[stale]` (yellow on TTY).
|
||||
//!
|
||||
//! These end-to-end checks exercise `kebab ask`, which requires a real
|
||||
//! Ollama on `127.0.0.1:11434` (same constraint as
|
||||
//! `kebab-app/tests/ask_smoke.rs`). Both tests are therefore
|
||||
//! `#[ignore]` by default — run with
|
||||
//! `cargo test -p kebab-cli --test wire_ask_stale -- --ignored`
|
||||
//! against a live Ollama.
|
||||
//!
|
||||
//! The `[stale]` rendering logic itself is also covered by a unit test
|
||||
//! in `kebab-cli/src/main.rs` (`tests::plain_marks_stale_citation_*`)
|
||||
//! that constructs a synthetic `Answer` and pipes it through
|
||||
//! `render_ask_plain_citations` — that path is the always-on guard.
|
||||
//!
|
||||
//! Shared TempDir / ingest / backdate helpers live in
|
||||
//! `tests/common/mod.rs`; see also `wire_search_stale.rs`.
|
||||
|
||||
mod common;
|
||||
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
use std::process::Command;
|
||||
|
||||
/// Run `kebab ask` in lexical mode (no embedding required). `json`
|
||||
/// toggles `--json`. The caller asserts on the resulting stdout.
|
||||
fn run_ask_lexical(cfg: &Path, query: &str, json: bool) -> std::process::Output {
|
||||
let bin = env!("CARGO_BIN_EXE_kebab");
|
||||
let mut cmd = Command::new(bin);
|
||||
cmd.arg("--config").arg(cfg);
|
||||
if json {
|
||||
cmd.arg("--json");
|
||||
}
|
||||
cmd.args(["ask", "--mode", "lexical", query]);
|
||||
cmd.output().unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires real Ollama on 127.0.0.1:11434"]
|
||||
fn ask_json_citations_include_indexed_at_and_stale() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (cfg, workspace, data) = common::write_config_with_llm_model(dir.path(), 30, "gemma4:e4b");
|
||||
fs::write(workspace.join("a.md"), "# T\n\napples are fruit\n").unwrap();
|
||||
common::ingest(&cfg, &workspace);
|
||||
common::backdate_updated_at(&data, "a.md", 60);
|
||||
|
||||
// ask returns exit 1 on refusal; the JSON envelope still goes to
|
||||
// stdout. Don't assert on `status.success()` — accept either path
|
||||
// and require the citations array to be present + structurally valid.
|
||||
let out = run_ask_lexical(&cfg, "what about apples", true);
|
||||
let stdout = String::from_utf8_lossy(&out.stdout);
|
||||
let answer: serde_json::Value = serde_json::from_str(stdout.trim())
|
||||
.unwrap_or_else(|e| panic!("expected JSON answer, got {stdout:?}: {e}"));
|
||||
let cits = answer["citations"]
|
||||
.as_array()
|
||||
.unwrap_or_else(|| panic!("expected citations array, got {answer}"));
|
||||
if let Some(cit) = cits.first() {
|
||||
// Schema fields are always present on a structurally-valid
|
||||
// AnswerCitation (serde-derived per Task 2 + Task 8).
|
||||
assert!(
|
||||
cit.get("indexed_at").is_some(),
|
||||
"missing indexed_at on citation: {cit}"
|
||||
);
|
||||
assert!(
|
||||
cit.get("stale").is_some(),
|
||||
"missing stale on citation: {cit}"
|
||||
);
|
||||
assert_eq!(
|
||||
cit["stale"], true,
|
||||
"doc backdated 60d at threshold 30d must be stale: {cit}"
|
||||
);
|
||||
}
|
||||
// If the model refused with zero citations the schema-shape claim
|
||||
// is vacuously true; the unit-test path
|
||||
// (`tests::plain_marks_stale_citation_*` in main.rs) is the
|
||||
// always-on guard.
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires real Ollama on 127.0.0.1:11434"]
|
||||
fn ask_plain_marks_stale_citation() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (cfg, workspace, data) = common::write_config_with_llm_model(dir.path(), 30, "gemma4:e4b");
|
||||
fs::write(workspace.join("a.md"), "# T\n\napples are fruit\n").unwrap();
|
||||
common::ingest(&cfg, &workspace);
|
||||
common::backdate_updated_at(&data, "a.md", 60);
|
||||
|
||||
// Refusal exits 1 — that's still fine here, the renderer prints
|
||||
// the citation block before the refusal exit when citations exist.
|
||||
// If the model refused with zero citations, this test is
|
||||
// best-effort (skip the assert): the unit-test path in main.rs
|
||||
// (`tests::plain_marks_stale_citation_*`) is the always-on guard.
|
||||
let out = run_ask_lexical(&cfg, "what about apples", false);
|
||||
let stdout = String::from_utf8_lossy(&out.stdout);
|
||||
if stdout.contains("근거:") {
|
||||
assert!(
|
||||
stdout.contains("[stale]"),
|
||||
"stale tag missing in plain ask output:\n{stdout}"
|
||||
);
|
||||
}
|
||||
}
|
||||
241
crates/kebab-cli/tests/wire_ask_stream.rs
Normal file
241
crates/kebab-cli/tests/wire_ask_stream.rs
Normal file
@@ -0,0 +1,241 @@
|
||||
//! p9-fb-33: CLI streaming surface — stderr ndjson `answer_event.v1`
|
||||
//! events while the answer streams; final stdout line is the existing
|
||||
//! `answer.v1` (backwards compat with the non-`--stream` path).
|
||||
//!
|
||||
//! These end-to-end checks exercise `kebab ask --stream`, which
|
||||
//! requires a real Ollama on `127.0.0.1:11434` (same constraint as
|
||||
//! `wire_ask_stale.rs` + `kebab-app/tests/ask_smoke.rs`). All three
|
||||
//! tests are therefore `#[ignore]` by default — run with
|
||||
//! `cargo test -p kebab-cli --test wire_ask_stream -- --ignored`
|
||||
//! against a live Ollama with `gemma4:e4b` pulled.
|
||||
//!
|
||||
//! The `BrokenPipe → cancel` test (Task 7 of the fb-33 plan) verifies
|
||||
//! that closing the stderr reader propagates SendError through the
|
||||
//! pipeline so the child terminates instead of hanging. That's the
|
||||
//! main thing the integration test layer can prove that unit tests
|
||||
//! can't — pipeline cancel is a cross-process concern.
|
||||
//!
|
||||
//! Shared TempDir / ingest helpers live in `tests/common/mod.rs`.
|
||||
|
||||
mod common;
|
||||
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
/// Drop `[rag].score_gate` to ~0 in the test config so the
|
||||
/// score-gate refusal path doesn't short-circuit the LLM call.
|
||||
/// Lexical retrieval against a one-doc corpus produces tiny fusion
|
||||
/// scores (well below the default 0.30 gate); the pipeline would
|
||||
/// take the `refuse_score_gate` early-return — which does not emit
|
||||
/// a `Final` event — making the streaming-event ordering assertion
|
||||
/// vacuous. Lower the gate so the LLM actually runs.
|
||||
fn relax_score_gate(cfg: &Path) {
|
||||
let body = fs::read_to_string(cfg).expect("read config.toml");
|
||||
let body = body.replace("score_gate = 0.30", "score_gate = 0.0");
|
||||
fs::write(cfg, body).expect("write relaxed config.toml");
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires real Ollama on 127.0.0.1:11434"]
|
||||
fn stream_emits_ndjson_events_on_stderr() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (cfg, workspace, _data) =
|
||||
common::write_config_with_llm_model(dir.path(), 30, "gemma4:e4b");
|
||||
relax_score_gate(&cfg);
|
||||
fs::write(
|
||||
workspace.join("a.md"),
|
||||
"# T\n\nrust ownership is a memory model.\n",
|
||||
)
|
||||
.unwrap();
|
||||
common::ingest(&cfg, &workspace);
|
||||
|
||||
let (stdout, stderr) = common::run_ask_stream(&cfg, "ownership");
|
||||
|
||||
// stderr: every non-empty line should parse as JSON with
|
||||
// schema_version == "answer_event.v1" and a recognized kind.
|
||||
let mut kinds: Vec<String> = vec![];
|
||||
for line in stderr.lines() {
|
||||
if line.trim().is_empty() {
|
||||
continue;
|
||||
}
|
||||
let v: Value = serde_json::from_str(line)
|
||||
.unwrap_or_else(|e| panic!("non-JSON stderr line: {line:?}: {e}"));
|
||||
assert_eq!(v["schema_version"], "answer_event.v1");
|
||||
let kind = v["kind"].as_str().expect("kind").to_string();
|
||||
assert!(
|
||||
matches!(kind.as_str(), "retrieval_done" | "token" | "final"),
|
||||
"unexpected kind: {kind}"
|
||||
);
|
||||
assert!(v["ts"].is_string(), "ts must be RFC3339 string");
|
||||
kinds.push(kind);
|
||||
}
|
||||
|
||||
// First event must be retrieval_done. Last must be final.
|
||||
// Note: this test only exercises the LLM-running path which always
|
||||
// closes with `final`. score-gate / no-chunks refusal paths emit
|
||||
// only `retrieval_done` and skip `final` — that's why the test uses
|
||||
// `relax_score_gate()` above to force the LLM path. See
|
||||
// `stream_score_gate_refusal_emits_only_retrieval_done` for the
|
||||
// refusal-path coverage.
|
||||
assert_eq!(
|
||||
kinds.first().map(String::as_str),
|
||||
Some("retrieval_done"),
|
||||
"first event must be retrieval_done, all kinds: {kinds:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
kinds.last().map(String::as_str),
|
||||
Some("final"),
|
||||
"last event must be final, all kinds: {kinds:?}"
|
||||
);
|
||||
|
||||
// stdout: last line is answer.v1 (backwards compat with the
|
||||
// non-streaming path — same wire shape, just emitted after the
|
||||
// ndjson event stream rather than instead of it).
|
||||
let final_line = stdout
|
||||
.lines()
|
||||
.last()
|
||||
.expect("stdout has at least one line");
|
||||
let answer: Value =
|
||||
serde_json::from_str(final_line).expect("stdout final line = answer.v1");
|
||||
assert_eq!(answer["schema_version"], "answer.v1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires real Ollama on 127.0.0.1:11434"]
|
||||
fn non_stream_path_unchanged() {
|
||||
// Verify that the non-streaming JSON path (no `--stream`) still
|
||||
// emits a single `answer.v1` line on stdout — fb-33 must not
|
||||
// perturb the existing wire surface.
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (cfg, workspace, _data) =
|
||||
common::write_config_with_llm_model(dir.path(), 30, "gemma4:e4b");
|
||||
relax_score_gate(&cfg);
|
||||
fs::write(
|
||||
workspace.join("a.md"),
|
||||
"# T\n\nrust ownership is a memory model.\n",
|
||||
)
|
||||
.unwrap();
|
||||
common::ingest(&cfg, &workspace);
|
||||
|
||||
let stdout = common::run_ask_json(&cfg, "ownership");
|
||||
let v: Value = serde_json::from_str(stdout.trim())
|
||||
.unwrap_or_else(|e| panic!("expected answer.v1, got {stdout:?}: {e}"));
|
||||
assert_eq!(v["schema_version"], "answer.v1");
|
||||
}
|
||||
|
||||
// p9-fb-33 (Task 7): BrokenPipe → cancel propagation. Spawn the
|
||||
// binary, read the first stderr line (retrieval_done), drop the
|
||||
// reader. The pipeline's next `Token` send returns SendError, the
|
||||
// cancel branch fires, child.wait() returns instead of blocking
|
||||
// forever. The key invariant is *liveness* — that `wait()` returns
|
||||
// in bounded time. Don't assert exit code: refusal is exit 1, but
|
||||
// the child may also exit 0 if the LLM happened to finish before
|
||||
// cancel propagated.
|
||||
#[test]
|
||||
#[ignore = "requires real Ollama on 127.0.0.1:11434 + writes to a closed pipe"]
|
||||
fn stream_cancels_when_stderr_closes() {
|
||||
use std::io::{BufRead, BufReader};
|
||||
use std::process::{Command, Stdio};
|
||||
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (cfg, workspace, _data) =
|
||||
common::write_config_with_llm_model(dir.path(), 30, "gemma4:e4b");
|
||||
relax_score_gate(&cfg);
|
||||
fs::write(
|
||||
workspace.join("a.md"),
|
||||
"# T\n\nrust ownership is a memory model. it tracks lifetimes.\n",
|
||||
)
|
||||
.unwrap();
|
||||
common::ingest(&cfg, &workspace);
|
||||
|
||||
let bin = env!("CARGO_BIN_EXE_kebab");
|
||||
let mut child = Command::new(bin)
|
||||
.args([
|
||||
"--config",
|
||||
cfg.to_str().unwrap(),
|
||||
"ask",
|
||||
"--stream",
|
||||
"--mode",
|
||||
"lexical",
|
||||
"ownership",
|
||||
])
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()
|
||||
.expect("spawn kebab");
|
||||
|
||||
{
|
||||
let stderr = child.stderr.take().expect("stderr piped");
|
||||
let mut reader = BufReader::new(stderr);
|
||||
let mut first = String::new();
|
||||
reader
|
||||
.read_line(&mut first)
|
||||
.expect("read first stderr line");
|
||||
assert!(
|
||||
first.contains("\"kind\":\"retrieval_done\""),
|
||||
"first event must be retrieval_done, got {first:?}"
|
||||
);
|
||||
// Drop the reader → child's stderr write end will see
|
||||
// BrokenPipe on the next write → main thread drops rx →
|
||||
// worker's pipeline.send returns SendError → cancel.
|
||||
}
|
||||
|
||||
let status = child.wait().expect("child completes after cancel");
|
||||
// Don't assert specific exit code — refusal is exit 1, but child
|
||||
// may also exit 0 if the LLM finished before cancel propagated.
|
||||
// The load-bearing assertion is that wait() returned at all.
|
||||
let _ = status;
|
||||
}
|
||||
|
||||
// p9-fb-33 (PR #124 round 1, item 4): score-gate refusal path —
|
||||
// thin doc + unrelated query trips the default 0.30 score gate
|
||||
// before the LLM runs. The pipeline emits only `retrieval_done`
|
||||
// on stderr (no `token`, no `final`); stdout still carries the
|
||||
// canonical `answer.v1` with `grounded=false`.
|
||||
#[test]
|
||||
#[ignore = "requires real Ollama on 127.0.0.1:11434"]
|
||||
fn stream_score_gate_refusal_emits_only_retrieval_done() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (cfg, workspace, _data) =
|
||||
common::write_config_with_llm_model(dir.path(), 30, "gemma4:e4b");
|
||||
// Intentionally NO relax_score_gate — keep the default 0.30
|
||||
// so the thin-doc + unrelated-query combo trips refusal.
|
||||
fs::write(
|
||||
workspace.join("a.md"),
|
||||
"# Title\n\nrust is a language.\n",
|
||||
)
|
||||
.unwrap();
|
||||
common::ingest(&cfg, &workspace);
|
||||
|
||||
let (stdout, stderr) =
|
||||
common::run_ask_stream(&cfg, "completely unrelated topic about cooking pasta");
|
||||
|
||||
let kinds: Vec<String> = stderr
|
||||
.lines()
|
||||
.filter(|l| !l.trim().is_empty())
|
||||
.filter_map(|l| serde_json::from_str::<Value>(l).ok())
|
||||
.filter_map(|v| v["kind"].as_str().map(String::from))
|
||||
.collect();
|
||||
|
||||
// Refusal path: only retrieval_done, no token, no final.
|
||||
assert!(
|
||||
kinds.iter().all(|k| k == "retrieval_done"),
|
||||
"refusal path must emit only retrieval_done, got {kinds:?}"
|
||||
);
|
||||
assert!(
|
||||
!kinds.is_empty(),
|
||||
"expected at least one retrieval_done event, got empty stderr"
|
||||
);
|
||||
|
||||
// Stdout still has answer.v1 with grounded=false.
|
||||
let final_line = stdout
|
||||
.lines()
|
||||
.last()
|
||||
.expect("stdout has at least one line");
|
||||
let answer: Value =
|
||||
serde_json::from_str(final_line).expect("answer.v1");
|
||||
assert_eq!(answer["schema_version"], "answer.v1");
|
||||
assert_eq!(answer["grounded"], false);
|
||||
}
|
||||
174
crates/kebab-cli/tests/wire_bulk_search.rs
Normal file
174
crates/kebab-cli/tests/wire_bulk_search.rs
Normal file
@@ -0,0 +1,174 @@
|
||||
//! p9-fb-42: integration tests for `kebab search --bulk`.
|
||||
//!
|
||||
//! Lexical-only — no fastembed / no Ollama. Each test builds its own
|
||||
//! TempDir KB via `common::write_config` + `common::ingest` and drives
|
||||
//! `kebab search --bulk` through stdin. Verifies:
|
||||
//!
|
||||
//! - Two queries over stdin emit per-query ndjson `bulk_search_item.v1` lines.
|
||||
//! - Empty stdin returns empty results with zero summary.
|
||||
//! - Malformed ndjson exits with code 2 (config_invalid).
|
||||
//! - Input over the 100-item cap fails with "max 100" error message.
|
||||
//! - Invalid item field (e.g. bad `mode`) emits per-item error and continues.
|
||||
|
||||
mod common;
|
||||
|
||||
use serde_json::Value;
|
||||
use std::fs;
|
||||
use std::io::Write;
|
||||
use std::process::{Command, Stdio};
|
||||
|
||||
fn cargo_bin() -> &'static str {
|
||||
env!("CARGO_BIN_EXE_kebab")
|
||||
}
|
||||
|
||||
fn run_bulk_with_stdin(cfg: &std::path::Path, stdin_body: &str, json: bool) -> std::process::Output {
|
||||
let mut cmd = Command::new(cargo_bin());
|
||||
cmd.arg("--config").arg(cfg).arg("search").arg("--bulk");
|
||||
if json {
|
||||
cmd.arg("--json");
|
||||
}
|
||||
cmd.stdin(Stdio::piped())
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped());
|
||||
let mut child = cmd.spawn().expect("spawn kebab");
|
||||
{
|
||||
let mut sin = child.stdin.take().expect("stdin");
|
||||
sin.write_all(stdin_body.as_bytes()).expect("write stdin");
|
||||
}
|
||||
child.wait_with_output().expect("wait")
|
||||
}
|
||||
|
||||
fn seed_workspace(workspace: &std::path::Path) {
|
||||
fs::write(workspace.join("a.md"), "# Alpha\n\nrust async hello").unwrap();
|
||||
fs::write(workspace.join("b.md"), "# Bravo\n\nbread and kebab").unwrap();
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test 1: Two queries over stdin emit per-query ndjson
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn two_query_bulk_emits_per_query_ndjson() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (cfg, workspace, _data) = common::write_config(dir.path(), 0);
|
||||
seed_workspace(&workspace);
|
||||
common::ingest(&cfg, &workspace);
|
||||
|
||||
let out = run_bulk_with_stdin(
|
||||
&cfg,
|
||||
"{\"query\":\"rust\",\"mode\":\"lexical\"}\n{\"query\":\"kebab\",\"mode\":\"lexical\"}\n",
|
||||
true,
|
||||
);
|
||||
assert!(
|
||||
out.status.success(),
|
||||
"stderr: {}",
|
||||
String::from_utf8_lossy(&out.stderr)
|
||||
);
|
||||
let stdout = String::from_utf8_lossy(&out.stdout);
|
||||
let lines: Vec<&str> = stdout.lines().filter(|l| !l.trim().is_empty()).collect();
|
||||
assert_eq!(lines.len(), 2, "expected 2 ndjson lines, got {lines:?}");
|
||||
for line in &lines {
|
||||
let v: Value = serde_json::from_str(line).expect("valid JSON line");
|
||||
assert_eq!(v["schema_version"], "bulk_search_item.v1");
|
||||
assert!(v["response"].is_object());
|
||||
assert!(v["error"].is_null());
|
||||
}
|
||||
let stderr = String::from_utf8_lossy(&out.stderr);
|
||||
assert!(
|
||||
stderr.contains("bulk_summary: total=2 succeeded=2 failed=0"),
|
||||
"stderr summary missing: {stderr}"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test 2: Empty stdin returns empty results with zero summary
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn empty_stdin_returns_empty_results_with_zero_summary() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (cfg, workspace, _data) = common::write_config(dir.path(), 0);
|
||||
seed_workspace(&workspace);
|
||||
common::ingest(&cfg, &workspace);
|
||||
|
||||
let out = run_bulk_with_stdin(&cfg, "", true);
|
||||
assert!(out.status.success());
|
||||
let stdout = String::from_utf8_lossy(&out.stdout);
|
||||
assert!(stdout.trim().is_empty(), "expected empty stdout, got: {stdout}");
|
||||
let stderr = String::from_utf8_lossy(&out.stderr);
|
||||
assert!(stderr.contains("bulk_summary: total=0 succeeded=0 failed=0"));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test 3: Malformed ndjson line emits config_invalid exit 2
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn malformed_ndjson_line_emits_config_invalid_exit_2() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (cfg, workspace, _data) = common::write_config(dir.path(), 0);
|
||||
seed_workspace(&workspace);
|
||||
common::ingest(&cfg, &workspace);
|
||||
|
||||
let out = run_bulk_with_stdin(&cfg, "not json\n", true);
|
||||
assert_eq!(out.status.code(), Some(2), "expected exit 2");
|
||||
let stderr = String::from_utf8_lossy(&out.stderr);
|
||||
assert!(
|
||||
stderr.contains("config_invalid") || stderr.contains("parse error"),
|
||||
"expected config_invalid or parse error in stderr: {stderr}"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test 4: Over cap input (>100) emits error
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn over_cap_input_emits_error() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (cfg, workspace, _data) = common::write_config(dir.path(), 0);
|
||||
seed_workspace(&workspace);
|
||||
common::ingest(&cfg, &workspace);
|
||||
|
||||
let body: String = (0..101)
|
||||
.map(|_| "{\"query\":\"x\",\"mode\":\"lexical\"}\n")
|
||||
.collect();
|
||||
let out = run_bulk_with_stdin(&cfg, &body, true);
|
||||
// bulk_search_with_config returns Err — surfaces as exit 1 (anyhow chain)
|
||||
// or 2 if classified by error_wire. Accept either, but message must mention `max 100`.
|
||||
assert!(out.status.code().is_some());
|
||||
let stderr = String::from_utf8_lossy(&out.stderr);
|
||||
assert!(
|
||||
stderr.contains("max 100"),
|
||||
"expected 'max 100' in stderr: {stderr}"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test 5: Invalid item field (bad mode) emits per-item error and continues
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn invalid_item_field_emits_per_item_error_continues() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (cfg, workspace, _data) = common::write_config(dir.path(), 0);
|
||||
seed_workspace(&workspace);
|
||||
common::ingest(&cfg, &workspace);
|
||||
|
||||
let out = run_bulk_with_stdin(
|
||||
&cfg,
|
||||
"{\"query\":\"rust\",\"mode\":\"lexical\"}\n{\"query\":\"x\",\"mode\":\"bogus\"}\n",
|
||||
true,
|
||||
);
|
||||
assert!(out.status.success());
|
||||
let stdout = String::from_utf8_lossy(&out.stdout);
|
||||
let lines: Vec<&str> = stdout.lines().filter(|l| !l.trim().is_empty()).collect();
|
||||
assert_eq!(lines.len(), 2);
|
||||
let v0: Value = serde_json::from_str(lines[0]).unwrap();
|
||||
let v1: Value = serde_json::from_str(lines[1]).unwrap();
|
||||
assert!(v0["error"].is_null());
|
||||
assert!(v1["error"].is_object());
|
||||
assert_eq!(v1["error"]["code"], "invalid_input");
|
||||
let stderr = String::from_utf8_lossy(&out.stderr);
|
||||
assert!(stderr.contains("succeeded=1 failed=1"));
|
||||
}
|
||||
100
crates/kebab-cli/tests/wire_citation_5_variants_unchanged.rs
Normal file
100
crates/kebab-cli/tests/wire_citation_5_variants_unchanged.rs
Normal file
@@ -0,0 +1,100 @@
|
||||
//! p10-1A-1 Task 13: regression — the 5 original Citation variants
|
||||
//! (Line, Page, Region, Caption, Time) serialize byte-identically to
|
||||
//! pre-Task-1 form. No spurious `code`, `line_start`, or `symbol` keys
|
||||
//! must leak into these variants.
|
||||
|
||||
use kebab_core::{Citation, WorkspacePath};
|
||||
|
||||
#[test]
|
||||
fn line_variant_serialization_unchanged() {
|
||||
let c = Citation::Line {
|
||||
path: WorkspacePath::new("a.md".into()).unwrap(),
|
||||
start: 1,
|
||||
end: 2,
|
||||
section: Some("§14".into()),
|
||||
};
|
||||
let v = serde_json::to_value(&c).unwrap();
|
||||
assert_eq!(v["kind"], "line");
|
||||
assert_eq!(v["start"], 1);
|
||||
assert_eq!(v["end"], 2);
|
||||
assert_eq!(v["section"], "§14");
|
||||
// Must not bleed Code-variant keys.
|
||||
assert!(v.get("line_start").is_none(), "line_start must be absent: {v}");
|
||||
assert!(v.get("symbol").is_none(), "symbol must be absent: {v}");
|
||||
assert!(v.get("code").is_none(), "code must be absent: {v}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn line_variant_null_section_omitted() {
|
||||
let c = Citation::Line {
|
||||
path: WorkspacePath::new("b.md".into()).unwrap(),
|
||||
start: 5,
|
||||
end: 10,
|
||||
section: None,
|
||||
};
|
||||
let v = serde_json::to_value(&c).unwrap();
|
||||
assert_eq!(v["kind"], "line");
|
||||
// `section` with None should be omitted (skip_serializing_if = is_none).
|
||||
assert!(v.get("section").is_none() || v["section"].is_null());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn page_variant_serialization_unchanged() {
|
||||
let c = Citation::Page {
|
||||
path: WorkspacePath::new("a.pdf".into()).unwrap(),
|
||||
page: 13,
|
||||
section: None,
|
||||
};
|
||||
let v = serde_json::to_value(&c).unwrap();
|
||||
assert_eq!(v["kind"], "page");
|
||||
assert_eq!(v["page"], 13);
|
||||
assert!(v.get("line_start").is_none(), "line_start must be absent: {v}");
|
||||
assert!(v.get("symbol").is_none(), "symbol must be absent: {v}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn region_variant_serialization_unchanged() {
|
||||
let c = Citation::Region {
|
||||
path: WorkspacePath::new("img.png".into()).unwrap(),
|
||||
x: 10,
|
||||
y: 20,
|
||||
w: 100,
|
||||
h: 200,
|
||||
};
|
||||
let v = serde_json::to_value(&c).unwrap();
|
||||
assert_eq!(v["kind"], "region");
|
||||
assert_eq!(v["x"], 10);
|
||||
assert_eq!(v["y"], 20);
|
||||
assert_eq!(v["w"], 100);
|
||||
assert_eq!(v["h"], 200);
|
||||
assert!(v.get("line_start").is_none(), "line_start must be absent: {v}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn caption_variant_serialization_unchanged() {
|
||||
let c = Citation::Caption {
|
||||
path: WorkspacePath::new("a.png".into()).unwrap(),
|
||||
model: "qwen2.5-vl:7b".into(),
|
||||
};
|
||||
let v = serde_json::to_value(&c).unwrap();
|
||||
assert_eq!(v["kind"], "caption");
|
||||
assert_eq!(v["model"], "qwen2.5-vl:7b");
|
||||
assert!(v.get("line_start").is_none(), "line_start must be absent: {v}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn time_variant_serialization_unchanged() {
|
||||
let c = Citation::Time {
|
||||
path: WorkspacePath::new("audio.mp3".into()).unwrap(),
|
||||
start_ms: 1000,
|
||||
end_ms: 5000,
|
||||
speaker: Some("Alice".into()),
|
||||
};
|
||||
let v = serde_json::to_value(&c).unwrap();
|
||||
assert_eq!(v["kind"], "time");
|
||||
assert_eq!(v["start_ms"], 1000);
|
||||
assert_eq!(v["end_ms"], 5000);
|
||||
assert_eq!(v["speaker"], "Alice");
|
||||
assert!(v.get("line_start").is_none(), "line_start must be absent: {v}");
|
||||
assert!(v.get("symbol").is_none(), "symbol must be absent: {v}");
|
||||
}
|
||||
130
crates/kebab-cli/tests/wire_fetch.rs
Normal file
130
crates/kebab-cli/tests/wire_fetch.rs
Normal file
@@ -0,0 +1,130 @@
|
||||
//! p9-fb-35: CLI fetch wire shape + plain output + exit codes.
|
||||
//!
|
||||
//! Lexical-only — no fastembed / no Ollama. Each test builds its own
|
||||
//! TempDir KB via `common::write_config` + `common::ingest` and drives
|
||||
//! `kebab fetch` through `common::run_fetch_with_args`. Verifies:
|
||||
//!
|
||||
//! - `--json fetch chunk <id>` emits the `fetch_result.v1` wrapper
|
||||
//! with `kind = "chunk"` and a populated `chunk` object.
|
||||
//! - `--json fetch doc <id> --max-tokens N` flips `truncated: true`
|
||||
//! once the budget binds.
|
||||
//! - Unknown `chunk_id` exits non-zero and emits an `error.v1`
|
||||
//! ndjson line on stderr with `code = "chunk_not_found"`.
|
||||
|
||||
mod common;
|
||||
|
||||
use serde_json::Value;
|
||||
use std::fs;
|
||||
|
||||
#[test]
|
||||
fn fetch_chunk_json_emits_fetch_result_v1() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (cfg, workspace, _data) = common::write_config(dir.path(), 30);
|
||||
fs::write(workspace.join("a.md"), "# T\n\napples are red.\n").unwrap();
|
||||
common::ingest(&cfg, &workspace);
|
||||
|
||||
// Find chunk_id via search.
|
||||
let (search_stdout, _) = common::run_search_with_args(
|
||||
&cfg,
|
||||
&["--json", "--mode", "lexical", "--k", "1", "apples"],
|
||||
);
|
||||
let search: Value = serde_json::from_str(search_stdout.trim())
|
||||
.unwrap_or_else(|e| panic!("search not JSON: {search_stdout:?}: {e}"));
|
||||
let chunk_id = search["hits"][0]["chunk_id"]
|
||||
.as_str()
|
||||
.expect("chunk_id on first hit")
|
||||
.to_string();
|
||||
|
||||
let (stdout, _) = common::run_fetch_with_args(
|
||||
&cfg,
|
||||
&["--json", "chunk", &chunk_id],
|
||||
);
|
||||
let v: Value = serde_json::from_str(stdout.trim())
|
||||
.unwrap_or_else(|e| panic!("fetch not JSON: {stdout:?}: {e}"));
|
||||
assert_eq!(v["schema_version"], "fetch_result.v1");
|
||||
assert_eq!(v["kind"], "chunk");
|
||||
assert!(
|
||||
v["chunk"].is_object(),
|
||||
"target chunk must be populated: {v}"
|
||||
);
|
||||
assert_eq!(v["truncated"], false);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fetch_doc_json_with_max_tokens_truncates() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (cfg, workspace, _data) = common::write_config(dir.path(), 30);
|
||||
let body: String = "Lorem ipsum dolor sit amet. ".repeat(20);
|
||||
fs::write(workspace.join("big.md"), format!("# Big\n\n{body}\n")).unwrap();
|
||||
common::ingest(&cfg, &workspace);
|
||||
|
||||
// Find doc_id via search.
|
||||
let (search_stdout, _) = common::run_search_with_args(
|
||||
&cfg,
|
||||
&["--json", "--mode", "lexical", "--k", "1", "Lorem"],
|
||||
);
|
||||
let search: Value = serde_json::from_str(search_stdout.trim())
|
||||
.unwrap_or_else(|e| panic!("search not JSON: {search_stdout:?}: {e}"));
|
||||
let doc_id = search["hits"][0]["doc_id"]
|
||||
.as_str()
|
||||
.expect("doc_id on first hit")
|
||||
.to_string();
|
||||
|
||||
let (stdout, _) = common::run_fetch_with_args(
|
||||
&cfg,
|
||||
&["--json", "doc", &doc_id, "--max-tokens", "20"],
|
||||
);
|
||||
let v: Value = serde_json::from_str(stdout.trim())
|
||||
.unwrap_or_else(|e| panic!("fetch not JSON: {stdout:?}: {e}"));
|
||||
assert_eq!(v["kind"], "doc");
|
||||
assert_eq!(
|
||||
v["truncated"], true,
|
||||
"20-token cap must trip truncation: {v}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fetch_chunk_unknown_id_exits_with_error_v1() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (cfg, _workspace, _data) = common::write_config(dir.path(), 30);
|
||||
|
||||
// Direct invocation (not via the success-asserting helper) so we
|
||||
// can read stderr on failure — mirrors the stale_cursor test in
|
||||
// `wire_search_response.rs`.
|
||||
let exe = env!("CARGO_BIN_EXE_kebab");
|
||||
let cfg_str = cfg.to_str().expect("utf8");
|
||||
let out = std::process::Command::new(exe)
|
||||
.args([
|
||||
"--config",
|
||||
cfg_str,
|
||||
"--json",
|
||||
"fetch",
|
||||
"chunk",
|
||||
"nonexistent",
|
||||
])
|
||||
.output()
|
||||
.expect("kebab fetch");
|
||||
|
||||
assert_ne!(out.status.code(), Some(0), "must exit non-zero");
|
||||
let stderr = String::from_utf8_lossy(&out.stderr);
|
||||
let err_line = stderr
|
||||
.lines()
|
||||
.find(|l| {
|
||||
serde_json::from_str::<Value>(l)
|
||||
.ok()
|
||||
.and_then(|v| {
|
||||
v.get("schema_version")
|
||||
.and_then(|s| s.as_str())
|
||||
.map(String::from)
|
||||
})
|
||||
.as_deref()
|
||||
== Some("error.v1")
|
||||
})
|
||||
.unwrap_or_else(|| panic!("no error.v1 line on stderr: {stderr:?}"));
|
||||
|
||||
let v: Value = serde_json::from_str(err_line).expect("error.v1 json");
|
||||
assert_eq!(
|
||||
v["code"], "chunk_not_found",
|
||||
"code must be chunk_not_found: {err_line}"
|
||||
);
|
||||
}
|
||||
57
crates/kebab-cli/tests/wire_schema_breakdowns.rs
Normal file
57
crates/kebab-cli/tests/wire_schema_breakdowns.rs
Normal file
@@ -0,0 +1,57 @@
|
||||
//! p9-fb-37: integration tests for `kebab schema --json` extended stats.
|
||||
|
||||
mod common;
|
||||
|
||||
use serde_json::Value;
|
||||
use std::fs;
|
||||
use std::process::Command;
|
||||
|
||||
fn run_schema(cfg: &std::path::Path) -> Value {
|
||||
let bin = env!("CARGO_BIN_EXE_kebab");
|
||||
let out = Command::new(bin)
|
||||
.args(["--config", cfg.to_str().unwrap(), "schema", "--json"])
|
||||
.output()
|
||||
.expect("run kebab schema");
|
||||
assert!(
|
||||
out.status.success(),
|
||||
"schema failed: stderr={}",
|
||||
String::from_utf8_lossy(&out.stderr)
|
||||
);
|
||||
serde_json::from_slice(&out.stdout).expect("valid JSON")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn schema_stats_includes_breakdowns_on_fresh_corpus() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (cfg, workspace, _data) = common::write_config(dir.path(), 0);
|
||||
// Run a no-op ingest to bring up migrations + create the SQLite file.
|
||||
fs::write(workspace.join("placeholder.md"), "# placeholder\n").unwrap();
|
||||
common::ingest(&cfg, &workspace);
|
||||
|
||||
let v = run_schema(&cfg);
|
||||
let stats = &v["stats"];
|
||||
let m = stats["media_breakdown"].as_object().unwrap();
|
||||
assert_eq!(m.len(), 5, "5 media keys padded");
|
||||
for k in &["markdown", "pdf", "image", "audio", "other"] {
|
||||
assert!(m[*k].is_number(), "media[{k}] is integer");
|
||||
}
|
||||
assert!(stats["lang_breakdown"].is_object());
|
||||
assert!(stats["index_bytes"]["sqlite"].is_number());
|
||||
assert!(stats["index_bytes"]["lancedb"].is_number());
|
||||
assert!(stats["stale_doc_count"].is_number());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn schema_stats_breakdowns_after_ingest() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (cfg, workspace, _data) = common::write_config(dir.path(), 0);
|
||||
fs::write(workspace.join("a.md"), "---\nlang: en\n---\nhello\n").unwrap();
|
||||
fs::write(workspace.join("b.md"), "---\nlang: ko\n---\n안녕\n").unwrap();
|
||||
common::ingest(&cfg, &workspace);
|
||||
|
||||
let v = run_schema(&cfg);
|
||||
let stats = &v["stats"];
|
||||
assert_eq!(stats["media_breakdown"]["markdown"], 2);
|
||||
assert!(stats["lang_breakdown"].is_object());
|
||||
assert!(stats["index_bytes"]["sqlite"].as_u64().unwrap() > 0);
|
||||
}
|
||||
306
crates/kebab-cli/tests/wire_search_filters.rs
Normal file
306
crates/kebab-cli/tests/wire_search_filters.rs
Normal file
@@ -0,0 +1,306 @@
|
||||
//! p9-fb-36: CLI integration tests for search filter flags.
|
||||
//!
|
||||
//! Lexical-only — no fastembed / no Ollama. Each test builds its own
|
||||
//! TempDir KB via `common::write_config` + `common::ingest` and drives
|
||||
//! `kebab search` through `common::run_search_with_args` or direct
|
||||
//! `Command` invocations. Verifies:
|
||||
//!
|
||||
//! - `--doc-id <id>` restricts all returned hits to the target document.
|
||||
//! - `--ingested-after <bad>` exits non-zero and emits `error.v1` on
|
||||
//! stderr with `code = "config_invalid"`.
|
||||
//! - `--media md` (alias) normalises to `markdown` and matches `.md` docs.
|
||||
//! - `--tag <tag>` (repeatable, OR-within) filters by frontmatter tags.
|
||||
|
||||
mod common;
|
||||
|
||||
use serde_json::Value;
|
||||
use std::fs;
|
||||
use std::process::Command;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test 1: --doc-id restricts hits to a single document
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn search_with_doc_id_filter_returns_only_target_doc() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (cfg, workspace, _data) = common::write_config(dir.path(), 30);
|
||||
|
||||
// Two docs that both contain the search term.
|
||||
fs::write(workspace.join("a.md"), "# Alpha\n\nrust ownership rules\n").unwrap();
|
||||
fs::write(workspace.join("b.md"), "# Beta\n\nrust borrow checker\n").unwrap();
|
||||
common::ingest(&cfg, &workspace);
|
||||
|
||||
// First, search without a doc-id filter to find what doc_ids exist.
|
||||
let (stdout, _) = common::run_search_with_args(
|
||||
&cfg,
|
||||
&["--json", "--mode", "lexical", "rust"],
|
||||
);
|
||||
let resp: Value = serde_json::from_str(stdout.trim())
|
||||
.unwrap_or_else(|e| panic!("not JSON: {stdout:?}: {e}"));
|
||||
let hits = resp["hits"].as_array().expect("hits array");
|
||||
assert!(
|
||||
hits.len() >= 2,
|
||||
"expected ≥2 hits from two docs before filter: {resp}"
|
||||
);
|
||||
|
||||
// Grab one doc_id from the results.
|
||||
let target_doc_id = hits[0]["doc_id"]
|
||||
.as_str()
|
||||
.expect("doc_id string")
|
||||
.to_string();
|
||||
|
||||
// Re-search with --doc-id set to the first hit's doc_id.
|
||||
let (stdout2, _) = common::run_search_with_args(
|
||||
&cfg,
|
||||
&[
|
||||
"--json",
|
||||
"--mode",
|
||||
"lexical",
|
||||
"--doc-id",
|
||||
&target_doc_id,
|
||||
"rust",
|
||||
],
|
||||
);
|
||||
let resp2: Value = serde_json::from_str(stdout2.trim())
|
||||
.unwrap_or_else(|e| panic!("not JSON after filter: {stdout2:?}: {e}"));
|
||||
let filtered_hits = resp2["hits"].as_array().expect("hits array (filtered)");
|
||||
|
||||
assert!(
|
||||
!filtered_hits.is_empty(),
|
||||
"expected at least one hit for the target doc"
|
||||
);
|
||||
for hit in filtered_hits {
|
||||
let got = hit["doc_id"].as_str().expect("doc_id string in hit");
|
||||
assert_eq!(
|
||||
got, target_doc_id,
|
||||
"--doc-id filter must restrict all hits to target doc, got {got}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test 2: --ingested-after with bad RFC3339 → exit non-zero + error.v1
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn search_with_invalid_ingested_after_emits_config_invalid() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (cfg, workspace, _data) = common::write_config(dir.path(), 30);
|
||||
fs::write(workspace.join("a.md"), "# T\n\nrust stuff\n").unwrap();
|
||||
common::ingest(&cfg, &workspace);
|
||||
|
||||
let bin = env!("CARGO_BIN_EXE_kebab");
|
||||
let out = Command::new(bin)
|
||||
.args([
|
||||
"--config",
|
||||
cfg.to_str().unwrap(),
|
||||
"--json",
|
||||
"search",
|
||||
"--mode",
|
||||
"lexical",
|
||||
"--ingested-after",
|
||||
"not-a-date",
|
||||
"rust",
|
||||
])
|
||||
.output()
|
||||
.expect("kebab search --ingested-after bad");
|
||||
|
||||
assert!(
|
||||
!out.status.success(),
|
||||
"expected non-zero exit for invalid --ingested-after, got: status={} stderr={}",
|
||||
out.status,
|
||||
String::from_utf8_lossy(&out.stderr)
|
||||
);
|
||||
|
||||
let stderr = String::from_utf8_lossy(&out.stderr);
|
||||
// Find the error.v1 ndjson line on stderr (one JSON event per line).
|
||||
let err_line = stderr
|
||||
.lines()
|
||||
.find(|l| {
|
||||
serde_json::from_str::<Value>(l)
|
||||
.ok()
|
||||
.and_then(|v| {
|
||||
v.get("schema_version")
|
||||
.and_then(|s| s.as_str())
|
||||
.map(String::from)
|
||||
})
|
||||
.as_deref()
|
||||
== Some("error.v1")
|
||||
})
|
||||
.unwrap_or_else(|| panic!("no error.v1 line on stderr: {stderr:?}"));
|
||||
|
||||
let v: Value = serde_json::from_str(err_line).expect("error.v1 json");
|
||||
assert_eq!(
|
||||
v["code"], "config_invalid",
|
||||
"code must be config_invalid for bad RFC3339: {err_line}"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test 3: --media md (alias) normalises to markdown and matches .md docs
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn search_with_media_filter_md_alias_normalizes_to_markdown() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (cfg, workspace, _data) = common::write_config(dir.path(), 30);
|
||||
|
||||
// Only a markdown file — the `md` alias should match it.
|
||||
fs::write(workspace.join("notes.md"), "# Notes\n\nrust async programming\n").unwrap();
|
||||
common::ingest(&cfg, &workspace);
|
||||
|
||||
let (stdout, _) = common::run_search_with_args(
|
||||
&cfg,
|
||||
&["--json", "--mode", "lexical", "--media", "md", "rust"],
|
||||
);
|
||||
let resp: Value = serde_json::from_str(stdout.trim())
|
||||
.unwrap_or_else(|e| panic!("not JSON: {stdout:?}: {e}"));
|
||||
let hits = resp["hits"].as_array().expect("hits array");
|
||||
|
||||
assert!(
|
||||
!hits.is_empty(),
|
||||
"--media md must match the markdown doc; got 0 hits: {resp}"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test 4: --tag (repeatable, OR-within) filters by frontmatter tags
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn search_with_tag_filter_matches_frontmatter_tags() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (cfg, workspace, _data) = common::write_config(dir.path(), 30);
|
||||
|
||||
// Doc with `rust` tag.
|
||||
fs::write(
|
||||
workspace.join("rust_doc.md"),
|
||||
"---\ntags: [rust, systems]\n---\n# Rust\n\nrust ownership\n",
|
||||
)
|
||||
.unwrap();
|
||||
// Doc without the tag (but same keyword in body so it appears in
|
||||
// unfiltered results — the tag filter must exclude it).
|
||||
fs::write(
|
||||
workspace.join("other_doc.md"),
|
||||
"# Other\n\nrust programming\n",
|
||||
)
|
||||
.unwrap();
|
||||
common::ingest(&cfg, &workspace);
|
||||
|
||||
// Without filter — both docs must produce hits.
|
||||
let (unfiltered, _) = common::run_search_with_args(
|
||||
&cfg,
|
||||
&["--json", "--mode", "lexical", "rust"],
|
||||
);
|
||||
let uresp: Value = serde_json::from_str(unfiltered.trim())
|
||||
.unwrap_or_else(|e| panic!("not JSON (unfiltered): {unfiltered:?}: {e}"));
|
||||
let uhits = uresp["hits"].as_array().expect("unfiltered hits array");
|
||||
assert!(
|
||||
uhits.len() >= 2,
|
||||
"expected ≥2 hits before tag filter: {uresp}"
|
||||
);
|
||||
|
||||
// With --tag rust — only the tagged doc's hits should appear.
|
||||
let (filtered, _) = common::run_search_with_args(
|
||||
&cfg,
|
||||
&["--json", "--mode", "lexical", "--tag", "rust", "rust"],
|
||||
);
|
||||
let fresp: Value = serde_json::from_str(filtered.trim())
|
||||
.unwrap_or_else(|e| panic!("not JSON (tag-filtered): {filtered:?}: {e}"));
|
||||
let fhits = fresp["hits"].as_array().expect("filtered hits array");
|
||||
|
||||
assert!(
|
||||
!fhits.is_empty(),
|
||||
"--tag rust must match the tagged doc; got 0 hits: {fresp}"
|
||||
);
|
||||
|
||||
// Every returned hit must come from rust_doc.md (the tagged file).
|
||||
for hit in fhits {
|
||||
let path = hit["doc_path"].as_str().unwrap_or("");
|
||||
assert!(
|
||||
path.ends_with("rust_doc.md"),
|
||||
"--tag rust must only return hits from the tagged doc, got path={path}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test 5: --tag is repeatable (OR-within); two --tag values form an IN-list
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn search_with_two_tag_filters_returns_or_within_tags() {
|
||||
// Two docs with different tag sets:
|
||||
// a.md → tags: [rust]
|
||||
// b.md → tags: [async]
|
||||
// c.md → no tags (but same keyword in body)
|
||||
// Search with --tag rust --tag async (OR within --tag).
|
||||
// Expect a.md and b.md, not c.md.
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (cfg, workspace, _data) = common::write_config(dir.path(), 30);
|
||||
|
||||
fs::write(
|
||||
workspace.join("a.md"),
|
||||
"---\ntags: [rust]\n---\n# A\n\nrust systems programming\n",
|
||||
)
|
||||
.unwrap();
|
||||
fs::write(
|
||||
workspace.join("b.md"),
|
||||
"---\ntags: [async]\n---\n# B\n\nrust async programming\n",
|
||||
)
|
||||
.unwrap();
|
||||
fs::write(workspace.join("c.md"), "# C\n\nrust programming\n").unwrap();
|
||||
common::ingest(&cfg, &workspace);
|
||||
|
||||
// Without filter: all three docs produce hits.
|
||||
let (unfiltered, _) = common::run_search_with_args(
|
||||
&cfg,
|
||||
&["--json", "--mode", "lexical", "rust"],
|
||||
);
|
||||
let uresp: Value = serde_json::from_str(unfiltered.trim())
|
||||
.unwrap_or_else(|e| panic!("not JSON (unfiltered): {unfiltered:?}: {e}"));
|
||||
let uhits = uresp["hits"].as_array().expect("unfiltered hits array");
|
||||
assert!(
|
||||
uhits.len() >= 3,
|
||||
"expected ≥3 hits before tag filter: {uresp}"
|
||||
);
|
||||
|
||||
// With --tag rust --tag async: only a.md and b.md should appear.
|
||||
let (filtered, _) = common::run_search_with_args(
|
||||
&cfg,
|
||||
&[
|
||||
"--json", "--mode", "lexical",
|
||||
"--tag", "rust",
|
||||
"--tag", "async",
|
||||
"rust",
|
||||
],
|
||||
);
|
||||
let fresp: Value = serde_json::from_str(filtered.trim())
|
||||
.unwrap_or_else(|e| panic!("not JSON (two-tag-filtered): {filtered:?}: {e}"));
|
||||
let fhits = fresp["hits"].as_array().expect("filtered hits array");
|
||||
|
||||
assert!(
|
||||
!fhits.is_empty(),
|
||||
"--tag rust --tag async must return hits from tagged docs; got 0: {fresp}"
|
||||
);
|
||||
|
||||
// c.md must not appear — it has no tags.
|
||||
for hit in fhits {
|
||||
let path = hit["doc_path"].as_str().unwrap_or("");
|
||||
assert!(
|
||||
path.ends_with("a.md") || path.ends_with("b.md"),
|
||||
"--tag rust --tag async must only return a.md or b.md, got path={path}"
|
||||
);
|
||||
}
|
||||
|
||||
// Both a.md and b.md must appear (OR, not AND).
|
||||
let paths: Vec<&str> = fhits
|
||||
.iter()
|
||||
.filter_map(|h| h["doc_path"].as_str())
|
||||
.collect();
|
||||
let has_a = paths.iter().any(|p| p.ends_with("a.md"));
|
||||
let has_b = paths.iter().any(|p| p.ends_with("b.md"));
|
||||
assert!(has_a, "--tag rust must include a.md (rust-tagged): paths={paths:?}");
|
||||
assert!(has_b, "--tag async must include b.md (async-tagged): paths={paths:?}");
|
||||
}
|
||||
72
crates/kebab-cli/tests/wire_search_filters_code.rs
Normal file
72
crates/kebab-cli/tests/wire_search_filters_code.rs
Normal file
@@ -0,0 +1,72 @@
|
||||
//! p10-1A-1 Task 15: CLI accepts --repo and --code-lang flags.
|
||||
//!
|
||||
//! These tests verify that clap parses the new flags without error.
|
||||
//! They drive `kebab search --help` (which exercises flag parsing
|
||||
//! via clap's help generation path, exiting 0) or use a minimal
|
||||
//! config + `--json` round-trip to verify the flags reach the wire.
|
||||
|
||||
use std::process::Command;
|
||||
|
||||
fn kebab() -> Command {
|
||||
Command::new(env!("CARGO_BIN_EXE_kebab"))
|
||||
}
|
||||
|
||||
/// `kebab search --help` must exit 0 and mention `--repo`.
|
||||
#[test]
|
||||
fn cli_search_help_mentions_repo_flag() {
|
||||
let out = kebab()
|
||||
.args(["search", "--help"])
|
||||
.output()
|
||||
.expect("failed to run kebab");
|
||||
// clap help exits 0.
|
||||
assert!(
|
||||
out.status.success(),
|
||||
"kebab search --help exited non-zero: {:?}",
|
||||
out.status
|
||||
);
|
||||
let stdout = String::from_utf8_lossy(&out.stdout);
|
||||
assert!(
|
||||
stdout.contains("--repo"),
|
||||
"--repo flag must appear in search help output:\n{stdout}"
|
||||
);
|
||||
}
|
||||
|
||||
/// `kebab search --help` must exit 0 and mention `--code-lang`.
|
||||
#[test]
|
||||
fn cli_search_help_mentions_code_lang_flag() {
|
||||
let out = kebab()
|
||||
.args(["search", "--help"])
|
||||
.output()
|
||||
.expect("failed to run kebab");
|
||||
assert!(
|
||||
out.status.success(),
|
||||
"kebab search --help exited non-zero: {:?}",
|
||||
out.status
|
||||
);
|
||||
let stdout = String::from_utf8_lossy(&out.stdout);
|
||||
assert!(
|
||||
stdout.contains("--code-lang"),
|
||||
"--code-lang flag must appear in search help output:\n{stdout}"
|
||||
);
|
||||
}
|
||||
|
||||
/// `kebab search --help` must exit 0 and mention `--media`.
|
||||
/// Confirms `--media code` value pathway is available (media is
|
||||
/// a free-form Vec<String> that already accepted arbitrary values).
|
||||
#[test]
|
||||
fn cli_search_help_mentions_media_flag() {
|
||||
let out = kebab()
|
||||
.args(["search", "--help"])
|
||||
.output()
|
||||
.expect("failed to run kebab");
|
||||
assert!(
|
||||
out.status.success(),
|
||||
"kebab search --help exited non-zero: {:?}",
|
||||
out.status
|
||||
);
|
||||
let stdout = String::from_utf8_lossy(&out.stdout);
|
||||
assert!(
|
||||
stdout.contains("--media"),
|
||||
"--media flag must appear in search help output:\n{stdout}"
|
||||
);
|
||||
}
|
||||
47
crates/kebab-cli/tests/wire_search_hit_no_code_fields.rs
Normal file
47
crates/kebab-cli/tests/wire_search_hit_no_code_fields.rs
Normal file
@@ -0,0 +1,47 @@
|
||||
//! p10-1A-1 Task 13: regression — markdown SearchHit omits `repo` and
|
||||
//! `code_lang` from JSON when both are `None`.
|
||||
//!
|
||||
//! Proves that adding optional fields to SearchHit does not silently
|
||||
//! inject spurious keys into the existing markdown corpus wire shape.
|
||||
|
||||
use kebab_core::{
|
||||
Citation, ChunkId, ChunkerVersion, DocumentId, IndexVersion, RetrievalDetail, ScoreKind,
|
||||
SearchHit, WorkspacePath,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn markdown_hit_omits_repo_and_code_lang() {
|
||||
let hit = SearchHit {
|
||||
rank: 1,
|
||||
chunk_id: ChunkId("c1".into()),
|
||||
doc_id: DocumentId("d1".into()),
|
||||
doc_path: WorkspacePath::new("notes/foo.md".into()).unwrap(),
|
||||
heading_path: vec!["A".into(), "B".into()],
|
||||
section_label: Some("B".into()),
|
||||
snippet: "hi".into(),
|
||||
citation: Citation::Line {
|
||||
path: WorkspacePath::new("notes/foo.md".into()).unwrap(),
|
||||
start: 1,
|
||||
end: 2,
|
||||
section: None,
|
||||
},
|
||||
retrieval: RetrievalDetail::default(),
|
||||
index_version: IndexVersion("v1".into()),
|
||||
embedding_model: None,
|
||||
chunker_version: ChunkerVersion("md-heading-v1".into()),
|
||||
indexed_at: time::OffsetDateTime::UNIX_EPOCH,
|
||||
stale: false,
|
||||
score_kind: ScoreKind::Rrf,
|
||||
repo: None,
|
||||
code_lang: None,
|
||||
};
|
||||
let s = serde_json::to_string(&hit).unwrap();
|
||||
assert!(
|
||||
!s.contains("\"repo\""),
|
||||
"repo should be absent from markdown hit JSON: {s}"
|
||||
);
|
||||
assert!(
|
||||
!s.contains("\"code_lang\""),
|
||||
"code_lang should be absent from markdown hit JSON: {s}"
|
||||
);
|
||||
}
|
||||
226
crates/kebab-cli/tests/wire_search_response.rs
Normal file
226
crates/kebab-cli/tests/wire_search_response.rs
Normal file
@@ -0,0 +1,226 @@
|
||||
//! p9-fb-34: CLI search wire wrapper + budget controls.
|
||||
//!
|
||||
//! Lexical-only — no fastembed / no Ollama. Each test builds its own
|
||||
//! TempDir KB via `common::write_config` + `common::ingest` and drives
|
||||
//! `kebab search` through `common::run_search_with_args`. Verifies:
|
||||
//!
|
||||
//! - `--json` emits the `search_response.v1` wrapper (hits + cursor +
|
||||
//! truncated).
|
||||
//! - `--max-tokens` flips `truncated: true` once the budget binds.
|
||||
//! - `--cursor` advances paging (page 2 chunk_ids disjoint from page 1).
|
||||
//! - Plain (non-JSON) output prints the `[truncated; ...]` hint to
|
||||
//! stderr (stdout stays the hit list).
|
||||
|
||||
mod common;
|
||||
|
||||
use serde_json::Value;
|
||||
use std::fs;
|
||||
|
||||
#[test]
|
||||
fn search_json_emits_search_response_v1_wrapper() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (cfg, workspace, _data) = common::write_config(dir.path(), 30);
|
||||
fs::write(workspace.join("a.md"), "# T\n\napples are red.\n").unwrap();
|
||||
common::ingest(&cfg, &workspace);
|
||||
|
||||
let (stdout, _stderr) = common::run_search_with_args(
|
||||
&cfg,
|
||||
&["--json", "--mode", "lexical", "apples"],
|
||||
);
|
||||
let v: Value = serde_json::from_str(stdout.trim())
|
||||
.unwrap_or_else(|e| panic!("not JSON: {stdout:?}: {e}"));
|
||||
assert_eq!(v["schema_version"], "search_response.v1");
|
||||
assert!(v["hits"].is_array(), "hits must be array, got {v}");
|
||||
assert!(
|
||||
v["next_cursor"].is_null() || v["next_cursor"].is_string(),
|
||||
"next_cursor must be null or string, got {}",
|
||||
v["next_cursor"]
|
||||
);
|
||||
assert!(
|
||||
v["truncated"].is_boolean(),
|
||||
"truncated must be bool, got {}",
|
||||
v["truncated"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn search_json_truncates_with_max_tokens() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (cfg, workspace, _data) = common::write_config(dir.path(), 30);
|
||||
let body: String = "rust ownership is a memory model. ".repeat(10);
|
||||
fs::write(workspace.join("a.md"), format!("# T\n\n{body}\n")).unwrap();
|
||||
common::ingest(&cfg, &workspace);
|
||||
|
||||
let (stdout, _stderr) = common::run_search_with_args(
|
||||
&cfg,
|
||||
&["--json", "--mode", "lexical", "--max-tokens", "30", "rust"],
|
||||
);
|
||||
let v: Value = serde_json::from_str(stdout.trim())
|
||||
.unwrap_or_else(|e| panic!("not JSON: {stdout:?}: {e}"));
|
||||
assert_eq!(
|
||||
v["truncated"], true,
|
||||
"30-token cap must trip truncation: {v}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn search_json_cursor_paginates() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (cfg, workspace, _data) = common::write_config(dir.path(), 30);
|
||||
for i in 0..6 {
|
||||
fs::write(
|
||||
workspace.join(format!("d{i}.md")),
|
||||
format!("# T{i}\n\nrust topic {i}\n"),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
common::ingest(&cfg, &workspace);
|
||||
|
||||
let (page1, _) = common::run_search_with_args(
|
||||
&cfg,
|
||||
&["--json", "--mode", "lexical", "--k", "2", "rust"],
|
||||
);
|
||||
let v1: Value = serde_json::from_str(page1.trim())
|
||||
.unwrap_or_else(|e| panic!("page1 not JSON: {page1:?}: {e}"));
|
||||
let cursor = v1["next_cursor"]
|
||||
.as_str()
|
||||
.unwrap_or_else(|| panic!("next_cursor missing on page1: {v1}"));
|
||||
|
||||
let (page2, _) = common::run_search_with_args(
|
||||
&cfg,
|
||||
&[
|
||||
"--json",
|
||||
"--mode",
|
||||
"lexical",
|
||||
"--k",
|
||||
"2",
|
||||
"--cursor",
|
||||
cursor,
|
||||
"rust",
|
||||
],
|
||||
);
|
||||
let v2: Value = serde_json::from_str(page2.trim())
|
||||
.unwrap_or_else(|e| panic!("page2 not JSON: {page2:?}: {e}"));
|
||||
|
||||
let p1_ids: Vec<String> = v1["hits"]
|
||||
.as_array()
|
||||
.expect("page1 hits array")
|
||||
.iter()
|
||||
.map(|h| {
|
||||
h["chunk_id"]
|
||||
.as_str()
|
||||
.expect("chunk_id string")
|
||||
.to_string()
|
||||
})
|
||||
.collect();
|
||||
let p2_ids: Vec<String> = v2["hits"]
|
||||
.as_array()
|
||||
.expect("page2 hits array")
|
||||
.iter()
|
||||
.map(|h| {
|
||||
h["chunk_id"]
|
||||
.as_str()
|
||||
.expect("chunk_id string")
|
||||
.to_string()
|
||||
})
|
||||
.collect();
|
||||
assert!(
|
||||
!p2_ids.is_empty(),
|
||||
"page2 must return at least one hit (cursor advanced past page1)"
|
||||
);
|
||||
assert!(
|
||||
p2_ids.iter().all(|id| !p1_ids.contains(id)),
|
||||
"page2 must not repeat page1 chunk_ids: page1={p1_ids:?} page2={p2_ids:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn search_stale_cursor_returns_error_v1_with_stale_cursor_code() {
|
||||
// p9-fb-34 round-1 review: end-to-end wire contract — when the
|
||||
// corpus_revision bumps between cursor issuance and the cursored
|
||||
// search, `kebab --json search --cursor <stale>` must emit an
|
||||
// `error.v1` ndjson line on stderr with `code = "stale_cursor"`.
|
||||
// Pre-fix this returned `code = "generic"` because
|
||||
// `App::search_with_opts` string-formatted the typed payload into
|
||||
// anyhow, losing the structured wrapper.
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (cfg, workspace, _data) = common::write_config(dir.path(), 30);
|
||||
fs::write(workspace.join("a.md"), "# T\n\napples\n").unwrap();
|
||||
common::ingest(&cfg, &workspace);
|
||||
|
||||
// Get a valid cursor first.
|
||||
let (page1_stdout, _) = common::run_search_with_args(
|
||||
&cfg,
|
||||
&["--mode", "lexical", "--json", "--k", "1", "apples"],
|
||||
);
|
||||
let v1: Value = serde_json::from_str(page1_stdout.trim()).expect("json");
|
||||
let cursor = v1["next_cursor"]
|
||||
.as_str()
|
||||
.expect("k=1 page must emit next_cursor — fixture too small if this fails")
|
||||
.to_string();
|
||||
|
||||
// Bump corpus_revision by ingesting a second doc.
|
||||
fs::write(workspace.join("b.md"), "# B\n\nbananas\n").unwrap();
|
||||
common::ingest(&cfg, &workspace);
|
||||
|
||||
// Use the now-stale cursor. Direct invocation (not via the
|
||||
// success-asserting helper) so we can read stderr on failure.
|
||||
let exe = env!("CARGO_BIN_EXE_kebab");
|
||||
let cfg_str = cfg.to_str().expect("utf8");
|
||||
let out = std::process::Command::new(exe)
|
||||
.args([
|
||||
"--config",
|
||||
cfg_str,
|
||||
"--json",
|
||||
"search",
|
||||
"--mode",
|
||||
"lexical",
|
||||
"--json",
|
||||
"--cursor",
|
||||
&cursor,
|
||||
"apples",
|
||||
])
|
||||
.output()
|
||||
.expect("kebab search --cursor");
|
||||
|
||||
let stderr = String::from_utf8_lossy(&out.stderr);
|
||||
// Find the error.v1 ndjson line on stderr (one event per line).
|
||||
let err_line = stderr
|
||||
.lines()
|
||||
.find(|l| {
|
||||
serde_json::from_str::<Value>(l)
|
||||
.ok()
|
||||
.and_then(|v| {
|
||||
v.get("schema_version")
|
||||
.and_then(|s| s.as_str())
|
||||
.map(String::from)
|
||||
})
|
||||
.as_deref()
|
||||
== Some("error.v1")
|
||||
})
|
||||
.unwrap_or_else(|| panic!("no error.v1 line on stderr: {stderr:?}"));
|
||||
|
||||
let v: Value = serde_json::from_str(err_line).expect("error.v1 json");
|
||||
assert_eq!(
|
||||
v["code"], "stale_cursor",
|
||||
"code must be stale_cursor: {err_line}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn search_plain_emits_truncated_hint_to_stderr() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (cfg, workspace, _data) = common::write_config(dir.path(), 30);
|
||||
let body: String = "rust ownership is a memory model. ".repeat(10);
|
||||
fs::write(workspace.join("a.md"), format!("# T\n\n{body}\n")).unwrap();
|
||||
common::ingest(&cfg, &workspace);
|
||||
|
||||
let (_stdout, stderr) = common::run_search_with_args(
|
||||
&cfg,
|
||||
&["--mode", "lexical", "--max-tokens", "30", "rust"],
|
||||
);
|
||||
assert!(
|
||||
stderr.contains("[truncated;"),
|
||||
"stderr must carry truncated hint: {stderr:?}"
|
||||
);
|
||||
}
|
||||
50
crates/kebab-cli/tests/wire_search_score_kind.rs
Normal file
50
crates/kebab-cli/tests/wire_search_score_kind.rs
Normal file
@@ -0,0 +1,50 @@
|
||||
//! p9-fb-38: integration tests for `search_hit.v1.score_kind`.
|
||||
|
||||
mod common;
|
||||
|
||||
use serde_json::Value;
|
||||
use std::fs;
|
||||
|
||||
fn doc_with_term(workspace: &std::path::Path) {
|
||||
fs::write(workspace.join("doc1.md"), "# Title\n\nrust async hello\n").unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lexical_mode_hits_carry_bm25_score_kind() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (cfg, workspace, _data) = common::write_config(dir.path(), 0);
|
||||
doc_with_term(&workspace);
|
||||
common::ingest(&cfg, &workspace);
|
||||
|
||||
let (stdout, _stderr) = common::run_search_with_args(
|
||||
&cfg,
|
||||
&["--mode", "lexical", "--json", "rust"],
|
||||
);
|
||||
let v: Value = serde_json::from_str(stdout.trim()).expect("valid JSON");
|
||||
let hits = v["hits"].as_array().expect("hits array");
|
||||
assert!(!hits.is_empty(), "expected at least 1 hit");
|
||||
for h in hits {
|
||||
assert_eq!(h["score_kind"], "bm25");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn old_wire_reader_compat_score_kind_optional_field() {
|
||||
// The wire schema marks `score_kind` as additive (not required).
|
||||
// We can't easily simulate an old reader from inside Rust, but we
|
||||
// can confirm the JSON includes the field — old readers that
|
||||
// ignore unknown fields are unaffected. This test just ensures
|
||||
// the field is always present in fb-38+ output.
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (cfg, workspace, _data) = common::write_config(dir.path(), 0);
|
||||
doc_with_term(&workspace);
|
||||
common::ingest(&cfg, &workspace);
|
||||
|
||||
let (stdout, _stderr) = common::run_search_with_args(
|
||||
&cfg,
|
||||
&["--mode", "lexical", "--json", "rust"],
|
||||
);
|
||||
let v: Value = serde_json::from_str(stdout.trim()).unwrap();
|
||||
let hit = &v["hits"][0];
|
||||
assert!(hit.get("score_kind").is_some(), "score_kind always emitted");
|
||||
}
|
||||
106
crates/kebab-cli/tests/wire_search_stale.rs
Normal file
106
crates/kebab-cli/tests/wire_search_stale.rs
Normal file
@@ -0,0 +1,106 @@
|
||||
//! p9-fb-32: CLI emits `indexed_at` + `stale` on JSON; plain output
|
||||
//! gains a `[stale]` tag prefix on stale hits.
|
||||
//!
|
||||
//! Self-contained: each test builds a TempDir workspace + config,
|
||||
//! invokes the `kebab` binary via `CARGO_BIN_EXE_kebab`, and (for the
|
||||
//! plain-output stale path) backdates `documents.updated_at` directly
|
||||
//! via `rusqlite` to simulate an aged-out doc without faking system
|
||||
//! time. Mirrors the helper pattern in
|
||||
//! `crates/kebab-app/tests/common/mod.rs::backdate_document_updated_at`.
|
||||
//!
|
||||
//! Shared TempDir / ingest / backdate helpers live in
|
||||
//! `tests/common/mod.rs`; see also `wire_ask_stale.rs`.
|
||||
|
||||
mod common;
|
||||
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
use std::process::Command;
|
||||
|
||||
fn run_search_lexical(cfg: &Path, query: &str, json: bool) -> std::process::Output {
|
||||
let bin = env!("CARGO_BIN_EXE_kebab");
|
||||
let mut cmd = Command::new(bin);
|
||||
cmd.arg("--config").arg(cfg);
|
||||
if json {
|
||||
cmd.arg("--json");
|
||||
}
|
||||
// Force lexical so the test doesn't need fastembed / AVX. Hybrid
|
||||
// is the CLI default which would try the vector path.
|
||||
cmd.args(["search", "--mode", "lexical", query]);
|
||||
let out = cmd.output().unwrap();
|
||||
assert!(
|
||||
out.status.success(),
|
||||
"search failed: stderr={}",
|
||||
String::from_utf8_lossy(&out.stderr)
|
||||
);
|
||||
out
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn search_json_includes_indexed_at_and_stale() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (cfg, workspace, _data) = common::write_config(dir.path(), 30);
|
||||
fs::write(workspace.join("a.md"), "# Title\n\napples are fruit\n").unwrap();
|
||||
common::ingest(&cfg, &workspace);
|
||||
|
||||
let out = run_search_lexical(&cfg, "apples", true);
|
||||
let stdout = String::from_utf8_lossy(&out.stdout);
|
||||
// p9-fb-34: top-level wire is now `search_response.v1` wrapping the
|
||||
// legacy `search_hit.v1[]` under a `hits` field (with pagination +
|
||||
// truncation metadata). Hit shape inside `hits` is unchanged.
|
||||
let resp: serde_json::Value = serde_json::from_str(stdout.trim())
|
||||
.unwrap_or_else(|e| panic!("expected JSON object, got {stdout:?}: {e}"));
|
||||
assert_eq!(
|
||||
resp.get("schema_version").and_then(|v| v.as_str()),
|
||||
Some("search_response.v1"),
|
||||
"expected search_response.v1 wrapper, got {resp}"
|
||||
);
|
||||
let arr = resp
|
||||
.get("hits")
|
||||
.and_then(|h| h.as_array())
|
||||
.unwrap_or_else(|| panic!("expected hits array, got {stdout}"));
|
||||
let first = arr.first().unwrap_or_else(|| panic!("expected ≥1 hit, got empty hits: {stdout}"));
|
||||
assert!(
|
||||
first.get("indexed_at").is_some(),
|
||||
"missing indexed_at in {first}"
|
||||
);
|
||||
assert!(
|
||||
first.get("stale").is_some(),
|
||||
"missing stale in {first}"
|
||||
);
|
||||
assert_eq!(
|
||||
first["stale"], false,
|
||||
"freshly ingested doc must not be stale at default 30d threshold"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn search_plain_marks_stale_doc() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (cfg, workspace, data) = common::write_config(dir.path(), 30);
|
||||
fs::write(workspace.join("a.md"), "# Title\n\napples are fruit\n").unwrap();
|
||||
common::ingest(&cfg, &workspace);
|
||||
common::backdate_updated_at(&data, "a.md", 60);
|
||||
|
||||
let out = run_search_lexical(&cfg, "apples", false);
|
||||
let stdout = String::from_utf8_lossy(&out.stdout);
|
||||
assert!(
|
||||
stdout.contains("[stale]"),
|
||||
"stale tag missing in plain output:\n{stdout}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn search_plain_no_stale_tag_for_fresh_doc() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (cfg, workspace, _data) = common::write_config(dir.path(), 30);
|
||||
fs::write(workspace.join("a.md"), "# Title\n\napples are fruit\n").unwrap();
|
||||
common::ingest(&cfg, &workspace);
|
||||
|
||||
let out = run_search_lexical(&cfg, "apples", false);
|
||||
let stdout = String::from_utf8_lossy(&out.stdout);
|
||||
assert!(
|
||||
!stdout.contains("[stale]"),
|
||||
"unexpected stale tag in plain output for fresh doc:\n{stdout}"
|
||||
);
|
||||
}
|
||||
58
crates/kebab-cli/tests/wire_search_trace.rs
Normal file
58
crates/kebab-cli/tests/wire_search_trace.rs
Normal file
@@ -0,0 +1,58 @@
|
||||
//! p9-fb-37: integration tests for `kebab search --trace --json`.
|
||||
|
||||
mod common;
|
||||
|
||||
use serde_json::Value;
|
||||
use std::fs;
|
||||
|
||||
#[test]
|
||||
fn search_trace_json_includes_trace_block() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (cfg, workspace, _data) = common::write_config(dir.path(), 0);
|
||||
fs::write(workspace.join("doc1.md"), "# Title\n\nrust async hello\n").unwrap();
|
||||
common::ingest(&cfg, &workspace);
|
||||
|
||||
let (stdout, _stderr) = common::run_search_with_args(
|
||||
&cfg,
|
||||
&["--mode", "lexical", "--trace", "--json", "rust"],
|
||||
);
|
||||
let v: Value = serde_json::from_str(stdout.trim()).expect("valid JSON");
|
||||
assert_eq!(v["schema_version"], "search_response.v1");
|
||||
assert!(v["trace"].is_object(), "trace block present");
|
||||
assert!(v["trace"]["timing"].is_object());
|
||||
assert!(v["trace"]["timing"]["total_ms"].is_number());
|
||||
assert!(v["trace"]["lexical"].is_array());
|
||||
assert!(v["trace"]["vector"].is_array());
|
||||
assert!(v["trace"]["rrf_inputs"].is_array());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn search_without_trace_omits_trace_field() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (cfg, workspace, _data) = common::write_config(dir.path(), 0);
|
||||
fs::write(workspace.join("doc1.md"), "# Title\n\nrust async hello\n").unwrap();
|
||||
common::ingest(&cfg, &workspace);
|
||||
|
||||
let (stdout, _stderr) = common::run_search_with_args(
|
||||
&cfg,
|
||||
&["--mode", "lexical", "--json", "rust"],
|
||||
);
|
||||
let v: Value = serde_json::from_str(stdout.trim()).expect("valid JSON");
|
||||
assert!(v.get("trace").is_none(), "trace field absent without --trace");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn search_trace_lexical_mode_vector_list_empty() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (cfg, workspace, _data) = common::write_config(dir.path(), 0);
|
||||
fs::write(workspace.join("doc1.md"), "# Title\n\nrust async hello\n").unwrap();
|
||||
common::ingest(&cfg, &workspace);
|
||||
|
||||
let (stdout, _stderr) = common::run_search_with_args(
|
||||
&cfg,
|
||||
&["--mode", "lexical", "--trace", "--json", "rust"],
|
||||
);
|
||||
let v: Value = serde_json::from_str(stdout.trim()).expect("valid JSON");
|
||||
assert_eq!(v["trace"]["vector"].as_array().unwrap().len(), 0);
|
||||
assert_eq!(v["trace"]["timing"]["vector_ms"], 0);
|
||||
}
|
||||
@@ -45,6 +45,11 @@ pub struct Config {
|
||||
/// `dark`).
|
||||
#[serde(default = "UiCfg::defaults")]
|
||||
pub ui: UiCfg,
|
||||
/// p10-1A-1: code ingest settings. `#[serde(default)]` so existing
|
||||
/// config files without an `[ingest]` / `[ingest.code]` section
|
||||
/// load cleanly with built-in defaults.
|
||||
#[serde(default)]
|
||||
pub ingest: IngestCfg,
|
||||
/// p9-fb-05: directory of the on-disk config file this `Config`
|
||||
/// was loaded from, if any. Populated by `Config::from_file` /
|
||||
/// `Config::load` — never serialized (`#[serde(skip)]`). Used by
|
||||
@@ -131,12 +136,21 @@ pub struct SearchCfg {
|
||||
/// (corpus_revision mismatch) are evicted on next access.
|
||||
#[serde(default = "default_cache_capacity")]
|
||||
pub cache_capacity: usize,
|
||||
/// p9-fb-32: hits and citations whose source doc was last
|
||||
/// re-processed more than this many days ago are marked
|
||||
/// `stale: true` in wire / TUI / CLI surfaces. `0` disables.
|
||||
#[serde(default = "default_stale_threshold_days")]
|
||||
pub stale_threshold_days: u32,
|
||||
}
|
||||
|
||||
fn default_cache_capacity() -> usize {
|
||||
256
|
||||
}
|
||||
|
||||
fn default_stale_threshold_days() -> u32 {
|
||||
30
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
pub struct RagCfg {
|
||||
pub prompt_template_version: String,
|
||||
@@ -256,6 +270,52 @@ impl UiCfg {
|
||||
}
|
||||
}
|
||||
|
||||
/// p10-1A-1: top-level ingest configuration wrapper. Contains per-media-type
|
||||
/// sub-sections; currently only `code` is defined.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct IngestCfg {
|
||||
pub code: IngestCodeCfg,
|
||||
}
|
||||
|
||||
/// p10-1A-1: settings for the code ingest pipeline. All fields have
|
||||
/// reasonable defaults so the user need not set anything in `config.toml`
|
||||
/// to get working code ingest.
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct IngestCodeCfg {
|
||||
/// Generated header sniff. Reads first ~512 bytes, checks 7 markers.
|
||||
pub skip_generated_header: bool,
|
||||
/// Max byte size per file. Bigger files skipped.
|
||||
pub max_file_bytes: u64,
|
||||
/// Max line count per file. Bigger files skipped (byte cap checked first).
|
||||
pub max_file_lines: u32,
|
||||
/// User extra skip globs (gitignore syntax). Applied on top of built-in
|
||||
/// + `.gitignore` + `.kebabignore`.
|
||||
pub extra_skip_globs: Vec<String>,
|
||||
/// AST chunk size cap. Functions/classes longer than this fall back to
|
||||
/// paragraph-based split (1A-2 and later).
|
||||
pub ast_chunk_max_lines: u32,
|
||||
/// Tier 3 fallback chunker: lines per chunk.
|
||||
pub fallback_lines_per_chunk: u32,
|
||||
/// Tier 3 fallback chunker: line overlap between adjacent chunks.
|
||||
pub fallback_lines_overlap: u32,
|
||||
}
|
||||
|
||||
impl Default for IngestCodeCfg {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
skip_generated_header: true,
|
||||
max_file_bytes: 262_144,
|
||||
max_file_lines: 5_000,
|
||||
extra_skip_globs: vec![],
|
||||
ast_chunk_max_lines: 200,
|
||||
fallback_lines_per_chunk: 80,
|
||||
fallback_lines_overlap: 20,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Config {
|
||||
/// Defaults per design §6.4.
|
||||
pub fn defaults() -> Self {
|
||||
@@ -293,9 +353,9 @@ impl Config {
|
||||
models: ModelsCfg {
|
||||
embedding: EmbeddingModelCfg {
|
||||
provider: "fastembed".to_string(),
|
||||
model: "multilingual-e5-small".to_string(),
|
||||
model: "multilingual-e5-large".to_string(),
|
||||
version: "v1".to_string(),
|
||||
dimensions: 384,
|
||||
dimensions: 1024,
|
||||
batch_size: 64,
|
||||
},
|
||||
llm: LlmCfg {
|
||||
@@ -317,15 +377,17 @@ impl Config {
|
||||
rrf_k: 60,
|
||||
snippet_chars: 220,
|
||||
cache_capacity: default_cache_capacity(),
|
||||
stale_threshold_days: 30,
|
||||
},
|
||||
rag: RagCfg {
|
||||
prompt_template_version: "rag-v1".to_string(),
|
||||
prompt_template_version: "rag-v2".to_string(),
|
||||
score_gate: 0.30,
|
||||
explain_default: false,
|
||||
max_context_tokens: 8000,
|
||||
},
|
||||
image: ImageCfg::defaults(),
|
||||
ui: UiCfg::defaults(),
|
||||
ingest: IngestCfg::default(),
|
||||
// p9-fb-05: defaults are not loaded from disk, so no
|
||||
// source_dir. Relative `workspace.root` (rare with
|
||||
// defaults) falls back to caller `cwd` via the
|
||||
@@ -577,6 +639,11 @@ impl Config {
|
||||
self.search.snippet_chars = n;
|
||||
}
|
||||
}
|
||||
"KEBAB_SEARCH_STALE_THRESHOLD_DAYS" => {
|
||||
if let Ok(n) = v.parse::<u32>() {
|
||||
self.search.stale_threshold_days = n;
|
||||
}
|
||||
}
|
||||
|
||||
// rag
|
||||
"KEBAB_RAG_PROMPT_TEMPLATE_VERSION" => {
|
||||
@@ -749,10 +816,17 @@ mod tests {
|
||||
let c = Config::defaults();
|
||||
assert_eq!(c.rag.score_gate, 0.30);
|
||||
assert_eq!(c.chunking.target_tokens, 500);
|
||||
assert_eq!(c.models.embedding.dimensions, 384);
|
||||
assert_eq!(c.models.embedding.model, "multilingual-e5-large");
|
||||
assert_eq!(c.models.embedding.dimensions, 1024);
|
||||
assert_eq!(c.search.rrf_k, 60);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn defaults_rag_prompt_template_version_is_rag_v2() {
|
||||
let c = Config::defaults();
|
||||
assert_eq!(c.rag.prompt_template_version, "rag-v2");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn env_override_score_gate() {
|
||||
let mut env = HashMap::new();
|
||||
@@ -926,9 +1000,9 @@ chunker_version = "md-heading-v1"
|
||||
|
||||
[models.embedding]
|
||||
provider = "fastembed"
|
||||
model = "multilingual-e5-small"
|
||||
model = "multilingual-e5-large"
|
||||
version = "v1"
|
||||
dimensions = 384
|
||||
dimensions = 1024
|
||||
batch_size = 64
|
||||
|
||||
[models.llm]
|
||||
@@ -944,9 +1018,10 @@ default_k = 10
|
||||
hybrid_fusion = "rrf"
|
||||
rrf_k = 60
|
||||
snippet_chars = 220
|
||||
stale_threshold_days = 30
|
||||
|
||||
[rag]
|
||||
prompt_template_version = "rag-v1"
|
||||
prompt_template_version = "rag-v2"
|
||||
score_gate = 0.30
|
||||
explain_default = false
|
||||
max_context_tokens = 8000
|
||||
@@ -981,6 +1056,44 @@ max_context_tokens = 8000
|
||||
let WorkspaceCfg { root: _, exclude: _ } = &ws;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn default_stale_threshold_is_30() {
|
||||
let c = Config::defaults();
|
||||
assert_eq!(c.search.stale_threshold_days, 30);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn env_override_stale_threshold() {
|
||||
let c = Config::defaults();
|
||||
let env: HashMap<String, String> = [
|
||||
("KEBAB_SEARCH_STALE_THRESHOLD_DAYS".to_string(), "7".to_string()),
|
||||
]
|
||||
.into_iter()
|
||||
.collect();
|
||||
let c = c.apply_env(&env);
|
||||
assert_eq!(c.search.stale_threshold_days, 7);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn env_negative_threshold_silently_ignored() {
|
||||
// Env path: malformed numeric values (including negatives that
|
||||
// can't fit `u32`) are silently ignored — same pattern as
|
||||
// `KEBAB_SEARCH_DEFAULT_K`. The TOML file-load path (covered in
|
||||
// `fb27_tests::file_negative_stale_threshold_returns_config_invalid`)
|
||||
// is the spec-required hard error surface.
|
||||
let c = Config::defaults();
|
||||
let env: HashMap<String, String> = [
|
||||
("KEBAB_SEARCH_STALE_THRESHOLD_DAYS".to_string(), "-5".to_string()),
|
||||
]
|
||||
.into_iter()
|
||||
.collect();
|
||||
let c = c.apply_env(&env);
|
||||
assert_eq!(
|
||||
c.search.stale_threshold_days, 30,
|
||||
"env path: malformed value must leave the default unchanged"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn xdg_paths_honor_env() {
|
||||
// Must restore env after the test to avoid polluting other tests.
|
||||
@@ -999,6 +1112,49 @@ max_context_tokens = 8000
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ingest_code_cfg_defaults() {
|
||||
let cfg: IngestCodeCfg = toml::from_str("").unwrap();
|
||||
assert_eq!(cfg.max_file_bytes, 262_144);
|
||||
assert_eq!(cfg.max_file_lines, 5_000);
|
||||
assert!(cfg.skip_generated_header);
|
||||
assert!(cfg.extra_skip_globs.is_empty());
|
||||
assert_eq!(cfg.ast_chunk_max_lines, 200);
|
||||
assert_eq!(cfg.fallback_lines_per_chunk, 80);
|
||||
assert_eq!(cfg.fallback_lines_overlap, 20);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ingest_code_cfg_user_override() {
|
||||
let toml = r#"
|
||||
max_file_bytes = 1048576
|
||||
max_file_lines = 20000
|
||||
skip_generated_header = false
|
||||
extra_skip_globs = ["**/fixtures/**", "**/snapshots/**"]
|
||||
"#;
|
||||
let cfg: IngestCodeCfg = toml::from_str(toml).unwrap();
|
||||
assert_eq!(cfg.max_file_bytes, 1_048_576);
|
||||
assert_eq!(cfg.max_file_lines, 20_000);
|
||||
assert!(!cfg.skip_generated_header);
|
||||
assert_eq!(cfg.extra_skip_globs.len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_with_ingest_code_section() {
|
||||
// Build a full valid Config serialization and patch only the
|
||||
// [ingest.code] field we care about — avoids having to enumerate
|
||||
// every required Config field in the test fixture.
|
||||
let base = Config::defaults();
|
||||
let mut toml_text = toml::to_string(&base).unwrap();
|
||||
// Inject max_file_bytes override into the [ingest.code] table.
|
||||
toml_text = toml_text.replace(
|
||||
"max_file_bytes = 262144",
|
||||
"max_file_bytes = 524288",
|
||||
);
|
||||
let cfg: Config = toml::from_str(&toml_text).unwrap();
|
||||
assert_eq!(cfg.ingest.code.max_file_bytes, 524_288);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -1027,4 +1183,38 @@ mod fb27_tests {
|
||||
assert_eq!(signal.path, p);
|
||||
assert!(!signal.cause.is_empty(), "cause should be non-empty");
|
||||
}
|
||||
|
||||
/// Spec §Config: a negative `stale_threshold_days` in TOML must be
|
||||
/// rejected at load time (not silently coerced or ignored). serde's
|
||||
/// `u32` type-check surfaces the failure as a parse error, which
|
||||
/// `from_file` wraps into `ConfigInvalid`. CLI's `error_classify`
|
||||
/// downcasts this and emits `error.v1.code = "config_invalid"`.
|
||||
#[test]
|
||||
fn file_negative_stale_threshold_returns_config_invalid() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("neg.toml");
|
||||
// Build a minimally valid TOML and override only the field
|
||||
// under test — this isolates the failure to the negative
|
||||
// value rather than missing required sections.
|
||||
let cfg = Config::defaults();
|
||||
let mut toml_text = toml::to_string(&cfg).expect("default round-trips");
|
||||
assert!(
|
||||
toml_text.contains("stale_threshold_days = 30"),
|
||||
"default value drifted; update test fixture"
|
||||
);
|
||||
toml_text = toml_text.replace(
|
||||
"stale_threshold_days = 30",
|
||||
"stale_threshold_days = -5",
|
||||
);
|
||||
std::fs::write(&p, &toml_text).unwrap();
|
||||
let err = Config::from_file(&p).unwrap_err();
|
||||
let signal = err.downcast_ref::<ConfigInvalid>()
|
||||
.expect("negative stale_threshold_days should downcast to ConfigInvalid");
|
||||
assert_eq!(signal.path, p);
|
||||
assert!(
|
||||
signal.cause.contains("parse_failed"),
|
||||
"expected parse_failed cause, got: {}",
|
||||
signal.cause
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -35,6 +35,11 @@ pub struct Answer {
|
||||
pub struct AnswerCitation {
|
||||
pub marker: Option<String>,
|
||||
pub citation: Citation,
|
||||
/// p9-fb-32: cited doc's `documents.updated_at`.
|
||||
#[serde(with = "time::serde::rfc3339")]
|
||||
pub indexed_at: OffsetDateTime,
|
||||
/// p9-fb-32: server-computed staleness flag per config threshold.
|
||||
pub stale: bool,
|
||||
}
|
||||
|
||||
/// p9-fb-15: history 가 prompt 에 들어갈 때의 한 turn. RAG facade 가
|
||||
@@ -90,3 +95,29 @@ pub struct TokenUsage {
|
||||
|
||||
#[derive(Clone, Debug, Eq, Hash, PartialEq, Serialize, Deserialize)]
|
||||
pub struct TraceId(pub String);
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::asset::WorkspacePath;
|
||||
use crate::citation::Citation;
|
||||
use time::macros::datetime;
|
||||
|
||||
#[test]
|
||||
fn answer_citation_serializes_indexed_at_and_stale() {
|
||||
let ac = AnswerCitation {
|
||||
marker: Some("[1]".to_string()),
|
||||
citation: Citation::Line {
|
||||
path: WorkspacePath::new("a.md".to_string()).unwrap(),
|
||||
start: 1,
|
||||
end: 1,
|
||||
section: None,
|
||||
},
|
||||
indexed_at: datetime!(2026-05-09 12:00:00 UTC),
|
||||
stale: false,
|
||||
};
|
||||
let v = serde_json::to_value(&ac).unwrap();
|
||||
assert_eq!(v["indexed_at"], "2026-05-09T12:00:00Z");
|
||||
assert_eq!(v["stale"], false);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -37,6 +37,13 @@ pub enum Citation {
|
||||
end_ms: u64,
|
||||
speaker: Option<String>,
|
||||
},
|
||||
Code {
|
||||
path: WorkspacePath,
|
||||
line_start: u32,
|
||||
line_end: u32,
|
||||
symbol: Option<String>,
|
||||
lang: Option<String>,
|
||||
},
|
||||
}
|
||||
|
||||
impl Citation {
|
||||
@@ -46,7 +53,8 @@ impl Citation {
|
||||
| Citation::Page { path, .. }
|
||||
| Citation::Region { path, .. }
|
||||
| Citation::Caption { path, .. }
|
||||
| Citation::Time { path, .. } => path,
|
||||
| Citation::Time { path, .. }
|
||||
| Citation::Code { path, .. } => path,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -80,6 +88,18 @@ impl Citation {
|
||||
None => format!("{}#t={},{}", path.0, s, e),
|
||||
}
|
||||
}
|
||||
Citation::Code {
|
||||
path,
|
||||
line_start,
|
||||
line_end,
|
||||
..
|
||||
} => {
|
||||
if line_start == line_end {
|
||||
format!("{}#L{}", path.0, line_start)
|
||||
} else {
|
||||
format!("{}#L{}-L{}", path.0, line_start, line_end)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -354,4 +374,64 @@ mod tests {
|
||||
let r = Citation::parse("notes/x#evil.md#L7");
|
||||
assert!(r.is_err(), "path with embedded '#' must be rejected");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn citation_code_variant_serializes_with_kind_tag() {
|
||||
let c = Citation::Code {
|
||||
path: WorkspacePath("crates/kebab-chunk/src/md_heading_v1.rs".into()),
|
||||
line_start: 142,
|
||||
line_end: 168,
|
||||
symbol: Some("MdHeadingV1Chunker::chunk_doc".into()),
|
||||
lang: Some("rust".into()),
|
||||
};
|
||||
let v = serde_json::to_value(&c).unwrap();
|
||||
assert_eq!(v["kind"], "code");
|
||||
assert_eq!(v["line_start"], 142);
|
||||
assert_eq!(v["line_end"], 168);
|
||||
assert_eq!(v["symbol"], "MdHeadingV1Chunker::chunk_doc");
|
||||
assert_eq!(v["lang"], "rust");
|
||||
// Existing 5 variants must NOT pick up these fields.
|
||||
let line = Citation::Line {
|
||||
path: WorkspacePath("notes/foo.md".into()),
|
||||
start: 1,
|
||||
end: 10,
|
||||
section: None,
|
||||
};
|
||||
let lv = serde_json::to_value(&line).unwrap();
|
||||
assert!(lv.get("line_start").is_none());
|
||||
assert!(lv.get("symbol").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn citation_code_uri_format() {
|
||||
let c = Citation::Code {
|
||||
path: WorkspacePath("a/b.rs".into()),
|
||||
line_start: 10,
|
||||
line_end: 20,
|
||||
symbol: None,
|
||||
lang: Some("rust".into()),
|
||||
};
|
||||
assert_eq!(c.to_uri(), "a/b.rs#L10-L20");
|
||||
// Single-line uses `#L10`.
|
||||
let single = Citation::Code {
|
||||
path: WorkspacePath("a/b.rs".into()),
|
||||
line_start: 5,
|
||||
line_end: 5,
|
||||
symbol: None,
|
||||
lang: None,
|
||||
};
|
||||
assert_eq!(single.to_uri(), "a/b.rs#L5");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn citation_code_path_accessor() {
|
||||
let c = Citation::Code {
|
||||
path: WorkspacePath("x.rs".into()),
|
||||
line_start: 1,
|
||||
line_end: 1,
|
||||
symbol: None,
|
||||
lang: None,
|
||||
};
|
||||
assert_eq!(c.path().0, "x.rs");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -142,6 +142,18 @@ pub enum SourceSpan {
|
||||
start_ms: u64,
|
||||
end_ms: u64,
|
||||
},
|
||||
/// p10-1A-2: AST-unit span for code ingest. Internal storage shape
|
||||
/// (chunks.source_spans_json) — `citation_helper` maps this to the
|
||||
/// wire `Citation::Code` (added 1A-1). `symbol` is the per-language
|
||||
/// self-reference path (design §3.4); `<top-level>` / `<module>` for
|
||||
/// glue regions, never null for an identified unit. `lang` is the
|
||||
/// canonical code_lang.
|
||||
Code {
|
||||
line_start: u32,
|
||||
line_end: u32,
|
||||
symbol: Option<String>,
|
||||
lang: Option<String>,
|
||||
},
|
||||
}
|
||||
|
||||
// ── Forward-declared stubs (§3.7a). Bodies are final per design. ────────
|
||||
@@ -195,6 +207,24 @@ mod tests {
|
||||
/// previously failed at serde runtime because `tag = "kind"` cannot
|
||||
/// describe a newtype carrying a non-struct value. The struct-variant
|
||||
/// shape used here is the §9 schema migration.
|
||||
#[test]
|
||||
fn source_span_code_round_trips_and_tags_lowercase() {
|
||||
let s = SourceSpan::Code {
|
||||
line_start: 10,
|
||||
line_end: 42,
|
||||
symbol: Some("foo::Bar::baz".to_string()),
|
||||
lang: Some("rust".to_string()),
|
||||
};
|
||||
let v = serde_json::to_value(&s).unwrap();
|
||||
assert_eq!(v["kind"], "code");
|
||||
assert_eq!(v["line_start"], 10);
|
||||
assert_eq!(v["line_end"], 42);
|
||||
assert_eq!(v["symbol"], "foo::Bar::baz");
|
||||
assert_eq!(v["lang"], "rust");
|
||||
let back: SourceSpan = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(back, s);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inline_serde_round_trip() {
|
||||
let cases = vec![
|
||||
|
||||
87
crates/kebab-core/src/fetch.rs
Normal file
87
crates/kebab-core/src/fetch.rs
Normal file
@@ -0,0 +1,87 @@
|
||||
//! p9-fb-35 verbatim fetch domain types.
|
||||
//!
|
||||
//! Three modes (chunk / doc / span) carried by [`FetchQuery`]; one
|
||||
//! response shape ([`FetchResult`]) discriminated by [`FetchKind`].
|
||||
//! All types are `Serialize` so the CLI / MCP wire layers can hand
|
||||
//! them straight through `serde_json::to_value`.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
use crate::asset::WorkspacePath;
|
||||
use crate::chunk::Chunk;
|
||||
use crate::ids::{ChunkId, DocumentId};
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub enum FetchQuery {
|
||||
Chunk(ChunkId),
|
||||
Doc(DocumentId),
|
||||
Span {
|
||||
doc_id: DocumentId,
|
||||
line_start: u32,
|
||||
line_end: u32,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct FetchOpts {
|
||||
/// chunk mode only: ±N chunks. None = no surrounding context.
|
||||
pub context: Option<u32>,
|
||||
/// doc / span mode only: chars/4 budget. None = no cap.
|
||||
pub max_tokens: Option<usize>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum FetchKind {
|
||||
Chunk,
|
||||
Doc,
|
||||
Span,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct FetchResult {
|
||||
pub kind: FetchKind,
|
||||
pub doc_id: DocumentId,
|
||||
pub doc_path: WorkspacePath,
|
||||
#[serde(with = "time::serde::rfc3339")]
|
||||
pub indexed_at: OffsetDateTime,
|
||||
pub stale: bool,
|
||||
// chunk mode payloads
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub chunk: Option<Chunk>,
|
||||
#[serde(skip_serializing_if = "Vec::is_empty", default)]
|
||||
pub context_before: Vec<Chunk>,
|
||||
#[serde(skip_serializing_if = "Vec::is_empty", default)]
|
||||
pub context_after: Vec<Chunk>,
|
||||
// doc / span payloads
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub text: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub line_start: Option<u32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub line_end: Option<u32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub effective_end: Option<u32>,
|
||||
pub truncated: bool,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn fetch_opts_default_is_all_none() {
|
||||
let o = FetchOpts::default();
|
||||
assert!(o.context.is_none());
|
||||
assert!(o.max_tokens.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fetch_kind_serializes_snake_case() {
|
||||
let v = serde_json::to_value(FetchKind::Chunk).unwrap();
|
||||
assert_eq!(v, serde_json::json!("chunk"));
|
||||
let v = serde_json::to_value(FetchKind::Span).unwrap();
|
||||
assert_eq!(v, serde_json::json!("span"));
|
||||
}
|
||||
}
|
||||
@@ -25,10 +25,46 @@ pub struct IngestReport {
|
||||
/// extension key under "<no-ext>". `BTreeMap` so the wire JSON
|
||||
/// has stable key order across runs.
|
||||
pub skipped_by_extension: std::collections::BTreeMap<String, u32>,
|
||||
/// p10-1A-1: files skipped because they matched a repo-local `.gitignore`.
|
||||
#[serde(default)]
|
||||
pub skipped_gitignore: u32,
|
||||
/// p10-1A-1: files skipped because they matched a `.kebabignore` entry.
|
||||
#[serde(default)]
|
||||
pub skipped_kebabignore: u32,
|
||||
/// p10-1A-1: files skipped because they matched the built-in safety-net
|
||||
/// blacklist (`node_modules/`, `target/`, `__pycache__/`, `.venv/`,
|
||||
/// `venv/`, `env/`).
|
||||
#[serde(default)]
|
||||
pub skipped_builtin_blacklist: u32,
|
||||
/// p10-1A-1: files skipped because their first ~512 bytes contained a
|
||||
/// generated-file marker (`@generated`, `do not edit`, …).
|
||||
#[serde(default)]
|
||||
pub skipped_generated: u32,
|
||||
/// p10-1A-1: files skipped because they exceeded `max_file_bytes` or
|
||||
/// `max_file_lines` in `[ingest.code]`.
|
||||
#[serde(default)]
|
||||
pub skipped_size_exceeded: u32,
|
||||
/// p10-1A-1: sample file paths per skip category (≤ 5 each).
|
||||
#[serde(default)]
|
||||
pub skip_examples: SkipExamples,
|
||||
/// `None` ↔ wire `items: null` (`--summary-only`).
|
||||
pub items: Option<Vec<IngestItem>>,
|
||||
}
|
||||
|
||||
/// p10-1A-1: per-category sample of skipped file paths. Each category caps at
|
||||
/// 5 entries (oldest-first). Used for debugging "why was X not indexed?"
|
||||
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
|
||||
pub struct SkipExamples {
|
||||
#[serde(default)]
|
||||
pub generated: Vec<String>,
|
||||
#[serde(default)]
|
||||
pub size_exceeded: Vec<String>,
|
||||
#[serde(default)]
|
||||
pub builtin_blacklist: Vec<String>,
|
||||
#[serde(default)]
|
||||
pub gitignore: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
pub struct IngestItem {
|
||||
pub kind: IngestItemKind,
|
||||
@@ -58,3 +94,55 @@ pub enum IngestItemKind {
|
||||
Unchanged,
|
||||
Error,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::traits::SourceScope;
|
||||
|
||||
#[test]
|
||||
fn skip_examples_default_is_empty() {
|
||||
let s = SkipExamples::default();
|
||||
assert!(s.generated.is_empty());
|
||||
assert!(s.size_exceeded.is_empty());
|
||||
assert!(s.builtin_blacklist.is_empty());
|
||||
assert!(s.gitignore.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ingest_report_skip_counters_serialize() {
|
||||
let r = IngestReport {
|
||||
scope: SourceScope {
|
||||
root: std::path::PathBuf::from("/tmp"),
|
||||
include: vec![],
|
||||
exclude: vec![],
|
||||
},
|
||||
scanned: 100,
|
||||
new: 50,
|
||||
updated: 0,
|
||||
skipped: 0,
|
||||
unchanged: 0,
|
||||
errors: 0,
|
||||
duration_ms: 1234,
|
||||
skipped_by_extension: Default::default(),
|
||||
skipped_gitignore: 30,
|
||||
skipped_kebabignore: 5,
|
||||
skipped_builtin_blacklist: 10,
|
||||
skipped_generated: 3,
|
||||
skipped_size_exceeded: 2,
|
||||
skip_examples: SkipExamples {
|
||||
generated: vec!["a/b.pb.rs".into()],
|
||||
size_exceeded: vec![],
|
||||
builtin_blacklist: vec!["node_modules/x.js".into()],
|
||||
gitignore: vec![],
|
||||
},
|
||||
items: None,
|
||||
};
|
||||
let v = serde_json::to_value(&r).unwrap();
|
||||
assert_eq!(v["skipped_gitignore"], 30);
|
||||
assert_eq!(v["skipped_builtin_blacklist"], 10);
|
||||
assert_eq!(v["skipped_generated"], 3);
|
||||
assert_eq!(v["skipped_size_exceeded"], 2);
|
||||
assert_eq!(v["skip_examples"]["generated"][0], "a/b.pb.rs");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,6 +23,7 @@ pub mod vector;
|
||||
pub mod errors;
|
||||
pub mod traits;
|
||||
pub mod normalize;
|
||||
pub mod fetch;
|
||||
|
||||
// Re-export the most commonly used items at the crate root, mirroring the
|
||||
// public surface listed in the task spec.
|
||||
@@ -50,14 +51,15 @@ pub use metadata::{
|
||||
TrustLevel,
|
||||
};
|
||||
pub use search::{
|
||||
DocFilter, DocSummary, RetrievalDetail, SearchFilters, SearchHit,
|
||||
SearchMode, SearchQuery,
|
||||
BulkSearchItem, BulkSearchResponse, BulkSearchSummary, DocFilter, DocSummary, IndexBytes, MEDIA_KINDS,
|
||||
RetrievalDetail, ScoreKind, SearchFilters, SearchHit, SearchMode, SearchOpts, SearchQuery, SearchTrace,
|
||||
TraceCandidate, TraceFusionInput, TraceTiming,
|
||||
};
|
||||
pub use answer::{
|
||||
Answer, AnswerCitation, AnswerRetrievalSummary, ModelRef, RefusalReason, TokenUsage,
|
||||
TraceId, Turn,
|
||||
};
|
||||
pub use ingest::{IngestItem, IngestItemKind, IngestReport};
|
||||
pub use ingest::{IngestItem, IngestItemKind, IngestReport, SkipExamples};
|
||||
pub use jobs::{JobFilter, JobId, JobKind, JobRow, JobStatus};
|
||||
pub use vector::{VectorHit, VectorRecord};
|
||||
pub use errors::CoreError;
|
||||
@@ -68,3 +70,4 @@ pub use traits::{
|
||||
SourceScope, TokenChunk, VectorStore,
|
||||
};
|
||||
pub use normalize::{nfc, to_posix};
|
||||
pub use fetch::{FetchKind, FetchOpts, FetchQuery, FetchResult};
|
||||
|
||||
@@ -40,5 +40,23 @@ pub enum MediaType {
|
||||
Pdf,
|
||||
Image(ImageType),
|
||||
Audio(AudioType),
|
||||
/// p10-1A-2: a source-code file. Inner string is the canonical
|
||||
/// code_lang (design §3.5). 1A activates `"rust"` only; other
|
||||
/// recognized code langs are still routed `Other` until their phase.
|
||||
Code(String),
|
||||
Other(String),
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn media_type_code_serializes_lowercase_tagged() {
|
||||
let m = MediaType::Code("rust".to_string());
|
||||
let v = serde_json::to_value(&m).unwrap();
|
||||
assert_eq!(v, serde_json::json!({ "code": "rust" }));
|
||||
let back: MediaType = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(back, m);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,6 +17,25 @@ pub struct Metadata {
|
||||
pub user_id_alias: Option<String>,
|
||||
/// Frontmatter keys we don't recognise are preserved here per §0 Q9.
|
||||
pub user: Map<String, Value>,
|
||||
|
||||
/// p10-1A-1: name of the source repo if the file lives inside a git
|
||||
/// working tree (`.git/` walk-up). null otherwise.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub repo: Option<String>,
|
||||
|
||||
/// p10-1A-1: HEAD branch at ingest time. null when no repo or detached HEAD.
|
||||
/// Informational only — current-state observability, not a partition key.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub git_branch: Option<String>,
|
||||
|
||||
/// p10-1A-1: HEAD commit (40-hex) at ingest time. null when no repo.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub git_commit: Option<String>,
|
||||
|
||||
/// p10-1A-1: programming language identifier (lowercase canonical). null
|
||||
/// for markdown / pdf / image. Set by `kebab_parse_code::lang::code_lang_for_path`.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub code_lang: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, Serialize, Deserialize)]
|
||||
@@ -66,3 +85,54 @@ pub enum ProvenanceKind {
|
||||
Warning,
|
||||
Error,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn metadata_repo_fields_default_to_none_and_omit_when_serialized() {
|
||||
let m = Metadata {
|
||||
aliases: vec![],
|
||||
tags: vec![],
|
||||
created_at: time::OffsetDateTime::UNIX_EPOCH,
|
||||
updated_at: time::OffsetDateTime::UNIX_EPOCH,
|
||||
source_type: SourceType::Markdown,
|
||||
trust_level: TrustLevel::Primary,
|
||||
user_id_alias: None,
|
||||
user: Default::default(),
|
||||
repo: None,
|
||||
git_branch: None,
|
||||
git_commit: None,
|
||||
code_lang: None,
|
||||
};
|
||||
let v = serde_json::to_value(&m).unwrap();
|
||||
assert!(v.get("repo").is_none());
|
||||
assert!(v.get("git_branch").is_none());
|
||||
assert!(v.get("git_commit").is_none());
|
||||
assert!(v.get("code_lang").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_repo_fields_present_when_some() {
|
||||
let m = Metadata {
|
||||
aliases: vec![],
|
||||
tags: vec![],
|
||||
created_at: time::OffsetDateTime::UNIX_EPOCH,
|
||||
updated_at: time::OffsetDateTime::UNIX_EPOCH,
|
||||
source_type: SourceType::Markdown,
|
||||
trust_level: TrustLevel::Primary,
|
||||
user_id_alias: None,
|
||||
user: Default::default(),
|
||||
repo: Some("kebab".into()),
|
||||
git_branch: Some("main".into()),
|
||||
git_commit: Some("a".repeat(40)),
|
||||
code_lang: Some("rust".into()),
|
||||
};
|
||||
let v = serde_json::to_value(&m).unwrap();
|
||||
assert_eq!(v["repo"], "kebab");
|
||||
assert_eq!(v["git_branch"], "main");
|
||||
assert_eq!(v["git_commit"].as_str().unwrap().len(), 40);
|
||||
assert_eq!(v["code_lang"], "rust");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,12 +26,49 @@ pub struct SearchQuery {
|
||||
pub filters: SearchFilters,
|
||||
}
|
||||
|
||||
/// p9-fb-36: canonical kind labels for `SearchFilters.media`. Mirrors
|
||||
/// `MediaType` variant tags; CLI / MCP normalize aliases (`md` → `markdown`)
|
||||
/// before populating this Vec.
|
||||
pub const MEDIA_KINDS: &[&str] = &["markdown", "pdf", "image", "audio", "other"];
|
||||
|
||||
/// p9-fb-38: top-level `SearchHit.score` declaration.
|
||||
/// `Rrf` (hybrid) / `Bm25` (lexical-only) / `Cosine` (vector-only).
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum ScoreKind {
|
||||
#[default]
|
||||
Rrf,
|
||||
Bm25,
|
||||
Cosine,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
|
||||
pub struct SearchFilters {
|
||||
pub tags_any: Vec<String>,
|
||||
pub lang: Option<Lang>,
|
||||
pub path_glob: Option<String>,
|
||||
pub trust_min: Option<TrustLevel>,
|
||||
/// p9-fb-36: media_type filter — IN-list of `MediaType.kind`
|
||||
/// strings (`"markdown"`, `"pdf"`, `"image"`, `"audio"`, `"other"`).
|
||||
/// Empty Vec = no filter. Match is on the variant tag only;
|
||||
/// e.g. `["image"]` matches `Image(Png)` and `Image(Jpeg)`.
|
||||
#[serde(default)]
|
||||
pub media: Vec<String>,
|
||||
/// p9-fb-36: hits whose source doc's `documents.updated_at` is at
|
||||
/// or after this timestamp. None = no filter. RFC3339 / UTC.
|
||||
#[serde(default, with = "time::serde::rfc3339::option")]
|
||||
pub ingested_after: Option<OffsetDateTime>,
|
||||
/// p9-fb-36: restrict hits to a single document. None = no filter.
|
||||
#[serde(default)]
|
||||
pub doc_id: Option<DocumentId>,
|
||||
/// p10-1A-1: filter by `metadata.repo`. Empty = no filter; multi-value = OR.
|
||||
#[serde(default)]
|
||||
pub repo: Vec<String>,
|
||||
/// p10-1A-1: filter by `metadata.code_lang`. Empty = no filter; multi-value = OR.
|
||||
/// Identifiers are lowercase canonical names (`rust`, `python`, `typescript`, ...).
|
||||
/// Unknown values produce empty hits (consistent with `media` policy).
|
||||
#[serde(default)]
|
||||
pub code_lang: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
@@ -48,6 +85,27 @@ pub struct SearchHit {
|
||||
pub index_version: IndexVersion,
|
||||
pub embedding_model: Option<EmbeddingModelId>,
|
||||
pub chunker_version: ChunkerVersion,
|
||||
/// p9-fb-32: source doc's `documents.updated_at` (last actual re-process).
|
||||
/// fb-23 incremental ingest skip path leaves this unchanged.
|
||||
#[serde(with = "time::serde::rfc3339")]
|
||||
pub indexed_at: OffsetDateTime,
|
||||
/// p9-fb-32: server-computed `now - indexed_at > threshold` per
|
||||
/// `config.search.stale_threshold_days`. `false` when threshold = 0.
|
||||
pub stale: bool,
|
||||
/// p9-fb-38: declares the meaning of the top-level `score`.
|
||||
/// `Rrf` (hybrid mode), `Bm25` (lexical-only), `Cosine` (vector-only).
|
||||
/// 옛 wire (fb-38 미만) 부재 시 `Rrf` default — hybrid 가 기본 mode.
|
||||
#[serde(default)]
|
||||
pub score_kind: ScoreKind,
|
||||
/// p10-1A-1: optional. Filled when the source file lives in a git repo
|
||||
/// (`.git/` walk-up). null for markdown / pdf / image hits and for code
|
||||
/// hits ingested via `kebab ingest-file` outside a repo boundary.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub repo: Option<String>,
|
||||
/// p10-1A-1: optional. Programming language identifier (lowercase). Set for
|
||||
/// every code/manifest/k8s chunk; null for markdown / pdf / image hits.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub code_lang: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
@@ -60,6 +118,19 @@ pub struct RetrievalDetail {
|
||||
pub vector_rank: Option<u32>,
|
||||
}
|
||||
|
||||
impl Default for RetrievalDetail {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
method: SearchMode::Hybrid,
|
||||
fusion_score: 0.0,
|
||||
lexical_score: None,
|
||||
vector_score: None,
|
||||
lexical_rank: None,
|
||||
vector_rank: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Filter for `kb-app::list_docs` (§7.2 DocumentStore::list_documents).
|
||||
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
|
||||
pub struct DocFilter {
|
||||
@@ -88,3 +159,376 @@ pub struct DocSummary {
|
||||
pub parser_version: ParserVersion,
|
||||
pub chunker_version: ChunkerVersion,
|
||||
}
|
||||
|
||||
/// p9-fb-34: caller-supplied output budget knobs for `App::search_with_opts`.
|
||||
/// All `None` = no enforcement (existing behavior).
|
||||
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
|
||||
pub struct SearchOpts {
|
||||
/// chars/4 approximation of wire JSON token cost. None = no cap.
|
||||
pub max_tokens: Option<usize>,
|
||||
/// Per-hit snippet character cap. None = use config default.
|
||||
pub snippet_chars: Option<usize>,
|
||||
/// Opaque base64 cursor from a previous response. None = first page.
|
||||
pub cursor: Option<String>,
|
||||
/// p9-fb-37: when true, capture pipeline trace (cache bypassed,
|
||||
/// lex / vec pre-fusion lists + timing populated on the response).
|
||||
#[serde(default)]
|
||||
pub trace: bool,
|
||||
}
|
||||
|
||||
/// p9-fb-37: search retrieval pipeline trace. Populated only when
|
||||
/// `SearchOpts.trace = true`; `None` on the wrapping `SearchResponse`
|
||||
/// otherwise. `lexical` / `vector` are pre-fusion candidate lists
|
||||
/// (each retriever's full output for the fanout query). `rrf_inputs`
|
||||
/// is the union (chunk_id) used by RRF, with each side's rank
|
||||
/// captured. `timing` is wall-clock per stage.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
|
||||
pub struct SearchTrace {
|
||||
pub lexical: Vec<TraceCandidate>,
|
||||
pub vector: Vec<TraceCandidate>,
|
||||
pub rrf_inputs: Vec<TraceFusionInput>,
|
||||
pub timing: TraceTiming,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
pub struct TraceCandidate {
|
||||
pub chunk_id: ChunkId,
|
||||
pub doc_id: DocumentId,
|
||||
pub doc_path: WorkspacePath,
|
||||
pub rank: u32,
|
||||
pub score: f32,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
pub struct TraceFusionInput {
|
||||
pub chunk_id: ChunkId,
|
||||
pub lexical_rank: Option<u32>,
|
||||
pub vector_rank: Option<u32>,
|
||||
/// Hybrid mode: normalized RRF score in `[0, 1]`.
|
||||
/// Lexical / Vector mode: equals the underlying retriever's score
|
||||
/// (no fusion ran). 0.0 for chunks dropped past `target_k`.
|
||||
pub fusion_score: f32,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct TraceTiming {
|
||||
pub lexical_ms: u64,
|
||||
pub vector_ms: u64,
|
||||
pub fusion_ms: u64,
|
||||
pub total_ms: u64,
|
||||
}
|
||||
|
||||
/// p9-fb-37: on-disk index size breakdown. Mirrored on the
|
||||
/// wire `schema.v1.stats.index_bytes` block.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct IndexBytes {
|
||||
pub sqlite: u64,
|
||||
pub lancedb: u64,
|
||||
}
|
||||
|
||||
/// p9-fb-42: per-query result in bulk search. `response` XOR `error` —
|
||||
/// exactly one is `Some`. `query` is the input echo (raw JSON value)
|
||||
/// so consumers can correlate input to output without index tracking.
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
pub struct BulkSearchItem {
|
||||
pub query: serde_json::Value,
|
||||
pub response: Option<serde_json::Value>,
|
||||
pub error: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
/// p9-fb-42: bulk summary counts. Invariant: total == succeeded + failed.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct BulkSearchSummary {
|
||||
pub total: u32,
|
||||
pub succeeded: u32,
|
||||
pub failed: u32,
|
||||
}
|
||||
|
||||
/// p9-fb-42: MCP-only envelope. CLI emits raw ndjson without envelope.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct BulkSearchResponse {
|
||||
pub schema_version: String,
|
||||
pub results: Vec<BulkSearchItem>,
|
||||
pub summary: BulkSearchSummary,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use time::macros::datetime;
|
||||
|
||||
#[test]
|
||||
fn search_hit_serializes_indexed_at_and_stale() {
|
||||
let hit = SearchHit {
|
||||
rank: 1,
|
||||
chunk_id: ChunkId("c".to_string()),
|
||||
doc_id: DocumentId("d".to_string()),
|
||||
doc_path: WorkspacePath::new("a/b.md".to_string()).unwrap(),
|
||||
heading_path: vec!["H".to_string()],
|
||||
section_label: None,
|
||||
snippet: "s".to_string(),
|
||||
citation: Citation::Line {
|
||||
path: WorkspacePath::new("a/b.md".to_string()).unwrap(),
|
||||
start: 1,
|
||||
end: 1,
|
||||
section: None,
|
||||
},
|
||||
retrieval: RetrievalDetail {
|
||||
method: SearchMode::Lexical,
|
||||
fusion_score: 0.5,
|
||||
lexical_score: Some(0.5),
|
||||
vector_score: None,
|
||||
lexical_rank: Some(1),
|
||||
vector_rank: None,
|
||||
},
|
||||
index_version: IndexVersion("v1".to_string()),
|
||||
embedding_model: None,
|
||||
chunker_version: ChunkerVersion("c1".to_string()),
|
||||
indexed_at: datetime!(2026-05-09 12:00:00 UTC),
|
||||
stale: true,
|
||||
score_kind: ScoreKind::Rrf,
|
||||
repo: None,
|
||||
code_lang: None,
|
||||
};
|
||||
let v = serde_json::to_value(&hit).unwrap();
|
||||
assert_eq!(v["indexed_at"], "2026-05-09T12:00:00Z");
|
||||
assert_eq!(v["stale"], true);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn search_opts_default_is_all_none() {
|
||||
let opts = SearchOpts::default();
|
||||
assert!(opts.max_tokens.is_none());
|
||||
assert!(opts.snippet_chars.is_none());
|
||||
assert!(opts.cursor.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn search_filters_default_includes_new_fb36_fields() {
|
||||
let f = SearchFilters::default();
|
||||
assert!(f.media.is_empty(), "media default empty");
|
||||
assert!(f.ingested_after.is_none(), "ingested_after default None");
|
||||
assert!(f.doc_id.is_none(), "doc_id default None");
|
||||
assert!(f.tags_any.is_empty());
|
||||
assert!(f.lang.is_none());
|
||||
assert!(f.path_glob.is_none());
|
||||
assert!(f.trust_min.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn search_filters_serialize_with_serde_default_compat() {
|
||||
let old: SearchFilters = serde_json::from_str(r#"{"tags_any":[],"lang":null,"path_glob":null,"trust_min":null}"#).unwrap();
|
||||
assert!(old.media.is_empty());
|
||||
assert!(old.ingested_after.is_none());
|
||||
assert!(old.doc_id.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn search_trace_serde_roundtrip() {
|
||||
let t = SearchTrace {
|
||||
lexical: vec![TraceCandidate {
|
||||
chunk_id: ChunkId("c1".into()),
|
||||
doc_id: DocumentId("d1".into()),
|
||||
doc_path: WorkspacePath::new("a.md".into()).unwrap(),
|
||||
rank: 1,
|
||||
score: 0.42,
|
||||
}],
|
||||
vector: vec![],
|
||||
rrf_inputs: vec![TraceFusionInput {
|
||||
chunk_id: ChunkId("c1".into()),
|
||||
lexical_rank: Some(1),
|
||||
vector_rank: None,
|
||||
fusion_score: 0.0234,
|
||||
}],
|
||||
timing: TraceTiming {
|
||||
lexical_ms: 12,
|
||||
vector_ms: 0,
|
||||
fusion_ms: 1,
|
||||
total_ms: 14,
|
||||
},
|
||||
};
|
||||
let v = serde_json::to_value(&t).unwrap();
|
||||
assert_eq!(v["timing"]["lexical_ms"], 12);
|
||||
assert_eq!(
|
||||
v["lexical"][0]["score"].as_f64().unwrap() as f32,
|
||||
0.42_f32
|
||||
);
|
||||
let back: SearchTrace = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(back, t);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn index_bytes_default_is_zero() {
|
||||
let b = IndexBytes::default();
|
||||
assert_eq!(b.sqlite, 0);
|
||||
assert_eq!(b.lancedb, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn search_opts_trace_default_false() {
|
||||
let opts = SearchOpts::default();
|
||||
assert!(!opts.trace);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn score_kind_serde_roundtrip() {
|
||||
use ScoreKind::*;
|
||||
for (kind, expected) in [(Rrf, "rrf"), (Bm25, "bm25"), (Cosine, "cosine")] {
|
||||
let v = serde_json::to_value(kind).unwrap();
|
||||
assert_eq!(v.as_str(), Some(expected));
|
||||
let back: ScoreKind = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(back, kind);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn score_kind_default_is_rrf() {
|
||||
assert_eq!(ScoreKind::default(), ScoreKind::Rrf);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn search_hit_deserialize_without_score_kind_defaults_to_rrf() {
|
||||
let json = serde_json::json!({
|
||||
"rank": 1,
|
||||
"chunk_id": "c1",
|
||||
"doc_id": "d1",
|
||||
"doc_path": "a.md",
|
||||
"heading_path": [],
|
||||
"section_label": null,
|
||||
"snippet": "x",
|
||||
"citation": { "kind": "line", "path": "a.md", "start": 1, "end": 1, "section": null },
|
||||
"retrieval": {
|
||||
"method": "lexical",
|
||||
"fusion_score": 0.5,
|
||||
"lexical_score": 0.5,
|
||||
"vector_score": null,
|
||||
"lexical_rank": 1,
|
||||
"vector_rank": null
|
||||
},
|
||||
"index_version": "v1",
|
||||
"embedding_model": null,
|
||||
"chunker_version": "c1",
|
||||
"indexed_at": "2026-05-10T12:00:00Z",
|
||||
"stale": false
|
||||
});
|
||||
let hit: SearchHit = serde_json::from_value(json).unwrap();
|
||||
assert_eq!(hit.score_kind, ScoreKind::Rrf);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bulk_search_summary_serde_roundtrip() {
|
||||
let s = BulkSearchSummary {
|
||||
total: 5,
|
||||
succeeded: 4,
|
||||
failed: 1,
|
||||
};
|
||||
let v = serde_json::to_value(s).unwrap();
|
||||
assert_eq!(v["total"], 5);
|
||||
assert_eq!(v["succeeded"], 4);
|
||||
assert_eq!(v["failed"], 1);
|
||||
let back: BulkSearchSummary = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(back, s);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bulk_search_summary_default_is_zeros() {
|
||||
let s = BulkSearchSummary::default();
|
||||
assert_eq!(s.total, 0);
|
||||
assert_eq!(s.succeeded, 0);
|
||||
assert_eq!(s.failed, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bulk_search_item_serde_response_variant() {
|
||||
let item = BulkSearchItem {
|
||||
query: serde_json::json!({"query": "rust"}),
|
||||
response: Some(serde_json::json!({"hits": []})),
|
||||
error: None,
|
||||
};
|
||||
let v = serde_json::to_value(&item).unwrap();
|
||||
assert!(v["response"].is_object());
|
||||
assert!(v["error"].is_null());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bulk_search_item_serde_error_variant() {
|
||||
let item = BulkSearchItem {
|
||||
query: serde_json::json!({"query": "rust"}),
|
||||
response: None,
|
||||
error: Some(serde_json::json!({"code": "config_invalid", "message": "bad"})),
|
||||
};
|
||||
let v = serde_json::to_value(&item).unwrap();
|
||||
assert!(v["response"].is_null());
|
||||
assert_eq!(v["error"]["code"], "config_invalid");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn search_hit_repo_and_code_lang_are_optional_and_omit_when_none() {
|
||||
let hit = SearchHit {
|
||||
rank: 1,
|
||||
chunk_id: ChunkId("c1".into()),
|
||||
doc_id: DocumentId("d1".into()),
|
||||
doc_path: WorkspacePath("a.md".into()),
|
||||
heading_path: vec![],
|
||||
section_label: None,
|
||||
snippet: "".into(),
|
||||
citation: Citation::Line {
|
||||
path: WorkspacePath("a.md".into()),
|
||||
start: 1,
|
||||
end: 2,
|
||||
section: None,
|
||||
},
|
||||
retrieval: RetrievalDetail::default(),
|
||||
index_version: IndexVersion("v1".into()),
|
||||
embedding_model: None,
|
||||
chunker_version: ChunkerVersion("md-heading-v1".into()),
|
||||
indexed_at: time::OffsetDateTime::UNIX_EPOCH,
|
||||
stale: false,
|
||||
score_kind: ScoreKind::Rrf,
|
||||
repo: None,
|
||||
code_lang: None,
|
||||
};
|
||||
let v = serde_json::to_value(&hit).unwrap();
|
||||
assert!(v.get("repo").is_none(), "repo should be omitted when None");
|
||||
assert!(v.get("code_lang").is_none(), "code_lang should be omitted when None");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn search_hit_repo_and_code_lang_present_when_some() {
|
||||
let hit = SearchHit {
|
||||
rank: 1,
|
||||
chunk_id: ChunkId("c1".into()),
|
||||
doc_id: DocumentId("d1".into()),
|
||||
doc_path: WorkspacePath("a.rs".into()),
|
||||
heading_path: vec![],
|
||||
section_label: None,
|
||||
snippet: "".into(),
|
||||
citation: Citation::Code {
|
||||
path: WorkspacePath("a.rs".into()),
|
||||
line_start: 1,
|
||||
line_end: 2,
|
||||
symbol: None,
|
||||
lang: Some("rust".into()),
|
||||
},
|
||||
retrieval: RetrievalDetail::default(),
|
||||
index_version: IndexVersion("v1".into()),
|
||||
embedding_model: None,
|
||||
chunker_version: ChunkerVersion("code-rust-ast-v1".into()),
|
||||
indexed_at: time::OffsetDateTime::UNIX_EPOCH,
|
||||
stale: false,
|
||||
score_kind: ScoreKind::Rrf,
|
||||
repo: Some("kebab".into()),
|
||||
code_lang: Some("rust".into()),
|
||||
};
|
||||
let v = serde_json::to_value(&hit).unwrap();
|
||||
assert_eq!(v["repo"], "kebab");
|
||||
assert_eq!(v["code_lang"], "rust");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn search_filters_repo_and_code_lang_default_to_empty_vec() {
|
||||
let f = SearchFilters::default();
|
||||
assert!(f.repo.is_empty());
|
||||
assert!(f.code_lang.is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -98,6 +98,11 @@ pub enum FinishReason {
|
||||
Stop,
|
||||
Length,
|
||||
Aborted,
|
||||
/// p9-fb-33: caller-side cancel. The pipeline breaks the LM loop
|
||||
/// when a `Token` send into `AskOpts.stream_sink` returns
|
||||
/// `SendError` (receiver dropped). The persisted answer is
|
||||
/// flagged with `RefusalReason::LlmStreamAborted`.
|
||||
Cancelled,
|
||||
Error(String),
|
||||
}
|
||||
|
||||
@@ -164,6 +169,20 @@ pub trait DocumentStore {
|
||||
&self,
|
||||
path: &WorkspacePath,
|
||||
) -> anyhow::Result<Option<RawAsset>>;
|
||||
|
||||
/// Look up a document row by its workspace path. Used by the
|
||||
/// document-centric skip path in `try_skip_unchanged` to avoid the
|
||||
/// twin-file flip-flop that the asset-side lookup suffers from
|
||||
/// (multiple files with identical content share one `assets` row
|
||||
/// whose `workspace_path` is overwritten on every UPSERT, so
|
||||
/// `get_asset_by_workspace_path` returns the wrong twin's path).
|
||||
///
|
||||
/// `documents.workspace_path` is UNIQUE (V001), so each twin has
|
||||
/// its own stable document row regardless of the asset de-dup.
|
||||
fn get_document_by_workspace_path(
|
||||
&self,
|
||||
path: &WorkspacePath,
|
||||
) -> anyhow::Result<Option<CanonicalDocument>>;
|
||||
}
|
||||
|
||||
pub trait VectorStore {
|
||||
|
||||
@@ -5,14 +5,14 @@ edition = { workspace = true }
|
||||
rust-version = { workspace = true }
|
||||
license = { workspace = true }
|
||||
repository = { workspace = true }
|
||||
description = "Local fastembed-rs adapter implementing kb_core::Embedder (multilingual-e5-small default)"
|
||||
description = "Local fastembed-rs adapter implementing kb_core::Embedder (multilingual-e5-large default, e5-small backwards-compat)"
|
||||
|
||||
[dependencies]
|
||||
kebab-config = { path = "../kebab-config" }
|
||||
kebab-embed = { path = "../kebab-embed" }
|
||||
# Default features bring `ort-download-binaries` (bundled ONNX runtime)
|
||||
# and `hf-hub-native-tls` (first-run model download). No extra features
|
||||
# needed for the multilingual-e5-small path.
|
||||
# needed for the multilingual-e5-{small,large} paths.
|
||||
fastembed = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
anyhow = { workspace = true }
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
//! `kb-embed-local` — `FastembedEmbedder`, a local ONNX-backed
|
||||
//! [`Embedder`](kebab_embed::Embedder) implementation.
|
||||
//!
|
||||
//! Wraps [`fastembed::TextEmbedding`] for the default `multilingual-e5-small`
|
||||
//! (384-dim) model. Honors `config.models.embedding.batch_size` and applies
|
||||
//! Wraps [`fastembed::TextEmbedding`]. Default is `multilingual-e5-large`
|
||||
//! (1024-dim, p9-fb-39b); `multilingual-e5-small` (384-dim) is also supported
|
||||
//! for backwards-compat. Honors `config.models.embedding.batch_size` and applies
|
||||
//! the e5 prefix convention (§11.3 of the design report):
|
||||
//!
|
||||
//! * `EmbeddingKind::Document` → `"passage: "` prefix
|
||||
@@ -69,9 +70,9 @@ impl FastembedEmbedder {
|
||||
.with_context(|| format!("create fastembed cache dir {}", cache_dir.display()))?;
|
||||
|
||||
// 2. Resolve the fastembed enum variant from
|
||||
// `config.models.embedding.model`. Currently only the default
|
||||
// `multilingual-e5-small` is wired; other model names error
|
||||
// out with a clear message rather than silently misconfiguring.
|
||||
// `config.models.embedding.model`. Currently `multilingual-e5-large`
|
||||
// (default) and `multilingual-e5-small` are wired; other model names
|
||||
// error out with a clear message rather than silently misconfiguring.
|
||||
let model_name = resolve_model(&config.models.embedding.model)?;
|
||||
|
||||
// 3. Verify dim match BEFORE loading the model — if the config
|
||||
@@ -100,7 +101,7 @@ impl FastembedEmbedder {
|
||||
target: "kebab-embed-local",
|
||||
model = %config.models.embedding.model,
|
||||
cache_dir = %cache_dir.display(),
|
||||
"loading embedding model (first run will download ~470MB)"
|
||||
"loading embedding model (first run downloads model weights — ~470MB for e5-small, ~1.3GB for e5-large)"
|
||||
);
|
||||
let inner = TextEmbedding::try_new(opts)
|
||||
.context("fastembed: TextEmbedding::try_new")?;
|
||||
@@ -193,17 +194,18 @@ fn prefix_input(input: &EmbeddingInput<'_>) -> String {
|
||||
}
|
||||
|
||||
/// Resolve a `config.models.embedding.model` string to a fastembed
|
||||
/// `EmbeddingModel` enum variant. Only `multilingual-e5-small` is wired
|
||||
/// for p3-2; additional model names should be added (and their dims
|
||||
/// pinned in tests) as needed.
|
||||
/// `EmbeddingModel` enum variant. Currently supports `multilingual-e5-small`
|
||||
/// (384-dim) and `multilingual-e5-large` (1024-dim); additional model names
|
||||
/// should be added (and their dims pinned in tests) as needed.
|
||||
fn resolve_model(name: &str) -> Result<EmbeddingModel> {
|
||||
match name {
|
||||
"multilingual-e5-small" => Ok(EmbeddingModel::MultilingualE5Small),
|
||||
"multilingual-e5-large" => Ok(EmbeddingModel::MultilingualE5Large),
|
||||
other => anyhow::bail!(
|
||||
"kb-embed-local: unsupported embedding model {other:?}; \
|
||||
this adapter currently only ships `multilingual-e5-small`. \
|
||||
Add a new arm to `resolve_model` (and a fastembed feature \
|
||||
flag if needed) to support more."
|
||||
this adapter currently ships `multilingual-e5-small` and \
|
||||
`multilingual-e5-large`. Add a new arm to `resolve_model` \
|
||||
(and a fastembed feature flag if needed) to support more."
|
||||
),
|
||||
}
|
||||
}
|
||||
@@ -294,6 +296,12 @@ mod tests {
|
||||
resolve_model("multilingual-e5-small").expect("default model resolves");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_model_supports_e5_large() {
|
||||
let m = resolve_model("multilingual-e5-large").expect("e5-large should resolve");
|
||||
let _ = m;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_unknown_model_errors() {
|
||||
let err = resolve_model("not-a-real-model").expect_err("unknown model errors");
|
||||
@@ -301,6 +309,21 @@ mod tests {
|
||||
assert!(msg.contains("unsupported embedding model"), "msg={msg}");
|
||||
}
|
||||
|
||||
// ── check_dim ────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn check_dim_passes_for_1024() {
|
||||
check_dim(1024, 1024).expect("matching dims must pass");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn check_dim_rejects_384_vs_1024() {
|
||||
let err = check_dim(384, 1024).expect_err("dim mismatch must error");
|
||||
let msg = format!("{err}");
|
||||
assert!(msg.contains("384") && msg.contains("1024"),
|
||||
"error must mention both dims, got: {msg}");
|
||||
}
|
||||
|
||||
// expand_path tests live in `kb-config::paths`. The adapter imports
|
||||
// it and trusts the upstream coverage rather than duplicating it.
|
||||
}
|
||||
|
||||
@@ -3,10 +3,11 @@
|
||||
//!
|
||||
//! ## Why every test in this file is `#[ignore]`
|
||||
//!
|
||||
//! The first call to `FastembedEmbedder::new` downloads ~470 MB of
|
||||
//! weights from Hugging Face into `data_dir/models/fastembed/`. Doing
|
||||
//! that on every `cargo test` invocation is wasteful, so the bare
|
||||
//! invocation skips this file entirely.
|
||||
//! The first call to `FastembedEmbedder::new` downloads ~1.3 GB of
|
||||
//! weights (multilingual-e5-large per p9-fb-39b default) from Hugging
|
||||
//! Face into `data_dir/models/fastembed/`. Doing that on every
|
||||
//! `cargo test` invocation is wasteful, so the bare invocation skips
|
||||
//! this file entirely.
|
||||
//!
|
||||
//! Run the full suite with:
|
||||
//! ```text
|
||||
@@ -58,19 +59,20 @@ fn shared_embedder() -> &'static FastembedEmbedder {
|
||||
// ─── construction ─────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
#[ignore = "downloads ~470MB ONNX model on first run; CI-only"]
|
||||
fn default_config_constructs_with_dims_384() {
|
||||
#[ignore = "downloads ~1.3GB ONNX model on first run; CI-only"]
|
||||
fn default_config_constructs_with_dims_1024() {
|
||||
// p9-fb-39b: default flipped to multilingual-e5-large (1024 dim).
|
||||
let emb = shared_embedder();
|
||||
assert_eq!(emb.dimensions(), 384);
|
||||
assert_eq!(emb.model_id().0, "multilingual-e5-small");
|
||||
assert_eq!(emb.dimensions(), 1024);
|
||||
assert_eq!(emb.model_id().0, "multilingual-e5-large");
|
||||
assert_eq!(emb.model_version().0, "v1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "downloads ~470MB ONNX model on first run; CI-only"]
|
||||
#[ignore = "downloads ~1.3GB ONNX model on first run; CI-only"]
|
||||
fn mismatched_dims_in_config_errors_at_construction() {
|
||||
let (mut cfg, _tmp) = test_config();
|
||||
cfg.models.embedding.dimensions = 512; // model is 384
|
||||
cfg.models.embedding.dimensions = 512; // model is 1024 (e5-large default)
|
||||
// `FastembedEmbedder` deliberately does not implement `Debug`
|
||||
// (its inner ONNX session has no useful debug shape), so we
|
||||
// can't use `expect_err`; match the Result manually.
|
||||
@@ -80,7 +82,7 @@ fn mismatched_dims_in_config_errors_at_construction() {
|
||||
};
|
||||
let msg = format!("{err}");
|
||||
assert!(msg.contains("dimension mismatch"), "msg={msg}");
|
||||
assert!(msg.contains("384"), "msg={msg}");
|
||||
assert!(msg.contains("1024"), "msg={msg}");
|
||||
assert!(msg.contains("512"), "msg={msg}");
|
||||
}
|
||||
|
||||
@@ -104,8 +106,8 @@ fn document_and_query_yield_different_vectors() {
|
||||
])
|
||||
.expect("embed two inputs");
|
||||
assert_eq!(out.len(), 2);
|
||||
assert_eq!(out[0].len(), 384);
|
||||
assert_eq!(out[1].len(), 384);
|
||||
assert_eq!(out[0].len(), 1024);
|
||||
assert_eq!(out[1].len(), 1024);
|
||||
|
||||
// Both vectors are L2-normalized → cosine similarity == dot product.
|
||||
let cos: f32 = out[0]
|
||||
@@ -142,11 +144,11 @@ fn output_vectors_are_l2_normalized() {
|
||||
];
|
||||
let out = emb.embed(&inputs).expect("embed");
|
||||
// Per `kebab_embed::assert_unit_norm` docs: `5e-4` is the safe bound at
|
||||
// 384 dims (f32::EPSILON × √384 ≈ 2.3e-6, but ONNX kernels add
|
||||
// 1024 dims (f32::EPSILON × √1024 ≈ 2.3e-6, but ONNX kernels add
|
||||
// their own per-component noise; 1e-3 is very generous and matches
|
||||
// the spec's `± 1e-3`).
|
||||
kebab_embed::assert_unit_norm(&out, 1e-3);
|
||||
kebab_embed::assert_vector_shape(&out, 384);
|
||||
kebab_embed::assert_vector_shape(&out, 1024);
|
||||
}
|
||||
|
||||
// ─── determinism ──────────────────────────────────────────────────────
|
||||
@@ -254,7 +256,7 @@ fn snapshot_aggregate_hash_is_stable() {
|
||||
// Round every component to 4 decimal places, hash deterministically.
|
||||
let mut hasher = DefaultHasher::new();
|
||||
for (i, v) in out.iter().enumerate() {
|
||||
assert_eq!(v.len(), 384, "row {i} dim mismatch");
|
||||
assert_eq!(v.len(), 1024, "row {i} dim mismatch");
|
||||
for x in v {
|
||||
let rounded: i32 = (*x * 1.0e4).round() as i32;
|
||||
rounded.hash(&mut hasher);
|
||||
|
||||
@@ -184,6 +184,18 @@ pub fn render_report_md(report: &CompareReport) -> String {
|
||||
),
|
||||
);
|
||||
}
|
||||
for k in crate::metrics::TOP_K_VARIANTS {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"| precision@{k}_chunk | {} | {} | {} |",
|
||||
fmt(a.precision_at_k_chunk.get(k).copied().unwrap_or(f32::NAN)),
|
||||
fmt(b.precision_at_k_chunk.get(k).copied().unwrap_or(f32::NAN)),
|
||||
fmt_delta(
|
||||
a.precision_at_k_chunk.get(k).copied().unwrap_or(f32::NAN),
|
||||
b.precision_at_k_chunk.get(k).copied().unwrap_or(f32::NAN),
|
||||
),
|
||||
);
|
||||
}
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"| citation_coverage | {} | {} | {} |",
|
||||
@@ -419,6 +431,7 @@ fn build_deltas(
|
||||
}
|
||||
let mut hit = serde_json::Map::new();
|
||||
let mut recall = serde_json::Map::new();
|
||||
let mut precision = serde_json::Map::new();
|
||||
for k in crate::metrics::TOP_K_VARIANTS {
|
||||
hit.insert(
|
||||
k.to_string(),
|
||||
@@ -434,11 +447,19 @@ fn build_deltas(
|
||||
b.recall_at_k_doc.get(k).copied().unwrap_or(f32::NAN),
|
||||
),
|
||||
);
|
||||
precision.insert(
|
||||
k.to_string(),
|
||||
d(
|
||||
a.precision_at_k_chunk.get(k).copied().unwrap_or(f32::NAN),
|
||||
b.precision_at_k_chunk.get(k).copied().unwrap_or(f32::NAN),
|
||||
),
|
||||
);
|
||||
}
|
||||
serde_json::json!({
|
||||
"hit_at_k": hit,
|
||||
"mrr": d(a.mrr, b.mrr),
|
||||
"recall_at_k_doc": recall,
|
||||
"precision_at_k_chunk": precision,
|
||||
"citation_coverage": d(a.citation_coverage, b.citation_coverage),
|
||||
"groundedness": d(a.groundedness, b.groundedness),
|
||||
"empty_result_rate": d(a.empty_result_rate, b.empty_result_rate),
|
||||
@@ -484,6 +505,7 @@ mod tests {
|
||||
hit_at_k: Default::default(),
|
||||
mrr: 0.5,
|
||||
recall_at_k_doc: Default::default(),
|
||||
precision_at_k_chunk: Default::default(),
|
||||
citation_coverage: f32::NAN,
|
||||
groundedness: 0.0,
|
||||
empty_result_rate: 0.0,
|
||||
|
||||
@@ -58,6 +58,14 @@ pub struct AggregateMetrics {
|
||||
pub hit_at_k: BTreeMap<u32, f32>,
|
||||
pub mrr: f32,
|
||||
pub recall_at_k_doc: BTreeMap<u32, f32>,
|
||||
/// p9-fb-39: chunk-level precision at k. Binary relevance via
|
||||
/// `expected_chunk_ids` (a hit is "relevant" if its chunk_id is
|
||||
/// in the golden's `expected_chunk_ids`). Denominator is k (fixed)
|
||||
/// — `hits.len() < k` still divides by k, treating shortfall as
|
||||
/// precision loss (mirrors `hit_at_k`). Queries with empty
|
||||
/// `expected_chunk_ids` are skipped (mirrors `hit_at_k_chunk`).
|
||||
#[serde(default)]
|
||||
pub precision_at_k_chunk: BTreeMap<u32, f32>,
|
||||
#[serde(
|
||||
serialize_with = "serialize_f32_nan_as_null",
|
||||
deserialize_with = "deserialize_f32_or_nan"
|
||||
@@ -187,6 +195,8 @@ pub(crate) fn aggregate_from_rows(
|
||||
TOP_K_VARIANTS.iter().map(|k| (*k, (0_u32, 0_u32))).collect();
|
||||
let mut recall_at_k_doc: BTreeMap<u32, (f64, u32)> =
|
||||
TOP_K_VARIANTS.iter().map(|k| (*k, (0.0_f64, 0_u32))).collect();
|
||||
let mut precision_at_k_chunk: BTreeMap<u32, (f64, u32)> =
|
||||
TOP_K_VARIANTS.iter().map(|k| (*k, (0.0_f64, 0_u32))).collect();
|
||||
|
||||
let mut mrr_sum: f64 = 0.0;
|
||||
let mut mrr_denom: u32 = 0;
|
||||
@@ -243,6 +253,18 @@ pub(crate) fn aggregate_from_rows(
|
||||
{
|
||||
mrr_sum += 1.0 / f64::from(rank);
|
||||
}
|
||||
// p9-fb-39: precision@k_chunk — count of top-k hits whose
|
||||
// chunk_id is in `expected`, divided by k (fixed denominator).
|
||||
for k in TOP_K_VARIANTS {
|
||||
let hits_in_topk_relevant = qr
|
||||
.hits_top_k
|
||||
.iter()
|
||||
.filter(|h| h.rank <= *k && expected.contains(&h.chunk_id))
|
||||
.count();
|
||||
let entry = precision_at_k_chunk.get_mut(k).expect("init");
|
||||
entry.0 += hits_in_topk_relevant as f64 / f64::from(*k);
|
||||
entry.1 += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// recall@k_doc (doc-level, requires non-empty expected_doc_ids
|
||||
@@ -316,7 +338,8 @@ pub(crate) fn aggregate_from_rows(
|
||||
| Citation::Page { path, .. }
|
||||
| Citation::Region { path, .. }
|
||||
| Citation::Caption { path, .. }
|
||||
| Citation::Time { path, .. } => !path.0.is_empty(),
|
||||
| Citation::Time { path, .. }
|
||||
| Citation::Code { path, .. } => !path.0.is_empty(),
|
||||
});
|
||||
if covered {
|
||||
citation_num += 1;
|
||||
@@ -333,6 +356,7 @@ pub(crate) fn aggregate_from_rows(
|
||||
mrr_sum / f64::from(mrr_denom)
|
||||
}),
|
||||
recall_at_k_doc: round_recall_map(&recall_at_k_doc),
|
||||
precision_at_k_chunk: round_recall_map(&precision_at_k_chunk),
|
||||
citation_coverage: ratio_or_nan(citation_num, citation_denom),
|
||||
groundedness: ratio_or_zero(groundedness_num, groundedness_denom),
|
||||
empty_result_rate: ratio_or_zero(empty_result_count, total_queries),
|
||||
@@ -444,6 +468,13 @@ mod tests {
|
||||
index_version: IndexVersion(format!("idx@{rank}")),
|
||||
embedding_model: None,
|
||||
chunker_version: ChunkerVersion("test@1".into()),
|
||||
// fb-32: synthetic eval fixtures don't exercise staleness;
|
||||
// pin UNIX_EPOCH + stale=false so hits stay deterministic.
|
||||
indexed_at: OffsetDateTime::UNIX_EPOCH,
|
||||
stale: false,
|
||||
score_kind: kebab_core::ScoreKind::Rrf,
|
||||
repo: None,
|
||||
code_lang: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -479,6 +510,9 @@ mod tests {
|
||||
end: 1,
|
||||
section: None,
|
||||
},
|
||||
// fb-32: synthetic eval citations don't exercise staleness.
|
||||
indexed_at: OffsetDateTime::UNIX_EPOCH,
|
||||
stale: false,
|
||||
}).collect(),
|
||||
grounded,
|
||||
refusal_reason: None,
|
||||
@@ -666,4 +700,114 @@ mod tests {
|
||||
assert_eq!(agg.failed_queries, 1);
|
||||
assert_eq!(agg.total_queries, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn precision_at_k_chunk_field_default_empty_on_old_json() {
|
||||
// Old eval_runs.metrics_json predates fb-39 — no precision_at_k_chunk field.
|
||||
// serde(default) yields empty BTreeMap.
|
||||
let old = serde_json::json!({
|
||||
"hit_at_k": {"1": 0.5, "3": 0.5, "5": 0.5, "10": 0.5},
|
||||
"mrr": 0.5,
|
||||
"recall_at_k_doc": {"1": 0.0, "3": 0.0, "5": 0.0, "10": 0.0},
|
||||
"citation_coverage": null,
|
||||
"groundedness": 0.0,
|
||||
"empty_result_rate": 0.0,
|
||||
"refusal_correctness": null,
|
||||
"total_queries": 1,
|
||||
"failed_queries": 0
|
||||
});
|
||||
let parsed: AggregateMetrics =
|
||||
serde_json::from_value(old).expect("backwards-compat deserialize");
|
||||
assert!(parsed.precision_at_k_chunk.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn precision_at_k_chunk_exact_match() {
|
||||
// expected = [c1, c2, c3]. Top-5 hits: [c1@1, c2@2, c3@3, x@4, y@5].
|
||||
// P@5 = 3/5 = 0.6. P@10 = 3/10 = 0.3.
|
||||
let queries = vec![gq("q1", &["c1", "c2", "c3"], &["d1"])];
|
||||
let rows = vec![record(
|
||||
"q1",
|
||||
vec![
|
||||
hit(1, "c1", "d1"),
|
||||
hit(2, "c2", "d1"),
|
||||
hit(3, "c3", "d1"),
|
||||
hit(4, "x", "d1"),
|
||||
hit(5, "y", "d1"),
|
||||
],
|
||||
None,
|
||||
None,
|
||||
)];
|
||||
let agg = aggregate_from_rows(&queries, &rows).unwrap();
|
||||
assert_eq!(agg.precision_at_k_chunk[&5], 0.6);
|
||||
assert_eq!(agg.precision_at_k_chunk[&10], 0.3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn precision_at_k_chunk_partial_topk_divides_by_k() {
|
||||
// expected = [c1, c2]. Hits: only [c1@1, c2@2, x@3] (3 results).
|
||||
// P@5 = 2/5 = 0.4 (denominator is k, not hits.len()).
|
||||
let queries = vec![gq("q1", &["c1", "c2"], &["d1"])];
|
||||
let rows = vec![record(
|
||||
"q1",
|
||||
vec![hit(1, "c1", "d1"), hit(2, "c2", "d1"), hit(3, "x", "d1")],
|
||||
None,
|
||||
None,
|
||||
)];
|
||||
let agg = aggregate_from_rows(&queries, &rows).unwrap();
|
||||
assert_eq!(agg.precision_at_k_chunk[&5], 0.4);
|
||||
assert_eq!(agg.precision_at_k_chunk[&10], 0.2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn precision_at_k_chunk_zero_relevant_in_topk() {
|
||||
// expected = [c1]. Hits: [x@1, y@2, z@3] (none relevant).
|
||||
// P@5 = 0/5 = 0.0.
|
||||
let queries = vec![gq("q1", &["c1"], &["d1"])];
|
||||
let rows = vec![record(
|
||||
"q1",
|
||||
vec![hit(1, "x", "d1"), hit(2, "y", "d1"), hit(3, "z", "d1")],
|
||||
None,
|
||||
None,
|
||||
)];
|
||||
let agg = aggregate_from_rows(&queries, &rows).unwrap();
|
||||
assert_eq!(agg.precision_at_k_chunk[&5], 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn precision_at_k_chunk_empty_expected_skipped() {
|
||||
// expected_chunk_ids = []. Skipped → final BTreeMap entry value = 0.0
|
||||
// (zero-denom path in round_recall_map). Mirrors recall_at_k_doc behavior.
|
||||
let queries = vec![gq("q1", &[], &["d1"])];
|
||||
let rows = vec![record("q1", vec![hit(1, "c1", "d1")], None, None)];
|
||||
let agg = aggregate_from_rows(&queries, &rows).unwrap();
|
||||
assert_eq!(agg.precision_at_k_chunk[&5], 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn precision_at_k_chunk_two_queries_averaged() {
|
||||
// q1: expected=[c1], hits=[c1@1, x@2, y@3] → P@5 = 1/5 = 0.2
|
||||
// q2: expected=[c1, c2], hits=[c1@1, c2@2] → P@5 = 2/5 = 0.4
|
||||
// Avg P@5 = 0.3.
|
||||
let queries = vec![
|
||||
gq("q1", &["c1"], &["d1"]),
|
||||
gq("q2", &["c1", "c2"], &["d2"]),
|
||||
];
|
||||
let rows = vec![
|
||||
record(
|
||||
"q1",
|
||||
vec![hit(1, "c1", "d1"), hit(2, "x", "d1"), hit(3, "y", "d1")],
|
||||
None,
|
||||
None,
|
||||
),
|
||||
record(
|
||||
"q2",
|
||||
vec![hit(1, "c1", "d2"), hit(2, "c2", "d2")],
|
||||
None,
|
||||
None,
|
||||
),
|
||||
];
|
||||
let agg = aggregate_from_rows(&queries, &rows).unwrap();
|
||||
assert_eq!(agg.precision_at_k_chunk[&5], 0.3);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,6 +11,12 @@
|
||||
"5": 0.666700005531311
|
||||
},
|
||||
"mrr": 0.41670000553131104,
|
||||
"precision_at_k_chunk": {
|
||||
"1": 0.33329999446868896,
|
||||
"10": 0.06669999659061432,
|
||||
"3": 0.11110000312328339,
|
||||
"5": 0.13330000638961792
|
||||
},
|
||||
"recall_at_k_doc": {
|
||||
"1": 0.33329999446868896,
|
||||
"10": 0.666700005531311,
|
||||
@@ -32,6 +38,12 @@
|
||||
"5": 1.0
|
||||
},
|
||||
"mrr": 0.833299994468689,
|
||||
"precision_at_k_chunk": {
|
||||
"1": 0.666700005531311,
|
||||
"10": 0.10000000149011612,
|
||||
"3": 0.33329999446868896,
|
||||
"5": 0.20000000298023224
|
||||
},
|
||||
"recall_at_k_doc": {
|
||||
"1": 0.666700005531311,
|
||||
"10": 1.0,
|
||||
@@ -53,6 +65,12 @@
|
||||
"5": 0.33329999446868896
|
||||
},
|
||||
"mrr": 0.41659998893737793,
|
||||
"precision_at_k_chunk": {
|
||||
"1": 0.33340001106262207,
|
||||
"10": 0.0333000048995018,
|
||||
"3": 0.22219999134540558,
|
||||
"5": 0.06669999659061432
|
||||
},
|
||||
"recall_at_k_doc": {
|
||||
"1": 0.33340001106262207,
|
||||
"10": 0.33329999446868896,
|
||||
|
||||
@@ -82,6 +82,13 @@ fn hit(rank: u32, chunk_id: &str, doc_id: &str) -> SearchHit {
|
||||
index_version: IndexVersion("idx@1".into()),
|
||||
embedding_model: None,
|
||||
chunker_version: ChunkerVersion("test@1".into()),
|
||||
// fb-32: synthetic eval fixtures don't exercise staleness;
|
||||
// pin UNIX_EPOCH + stale=false so hits stay deterministic.
|
||||
indexed_at: OffsetDateTime::UNIX_EPOCH,
|
||||
stale: false,
|
||||
score_kind: kebab_core::ScoreKind::Rrf,
|
||||
repo: None,
|
||||
code_lang: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -198,6 +205,7 @@ fn store_aggregate_rejects_missing_run() {
|
||||
hit_at_k: Default::default(),
|
||||
mrr: 0.0,
|
||||
recall_at_k_doc: Default::default(),
|
||||
precision_at_k_chunk: Default::default(),
|
||||
citation_coverage: f32::NAN,
|
||||
groundedness: 0.0,
|
||||
empty_result_rate: 0.0,
|
||||
|
||||
@@ -213,7 +213,7 @@ fn runner_records_config_snapshot_with_versions() {
|
||||
assert!(snap.pointer("/llm/model_id").is_some());
|
||||
assert_eq!(
|
||||
snap.pointer("/prompt_template_version"),
|
||||
Some(&serde_json::Value::String("rag-v1".to_string())),
|
||||
Some(&serde_json::Value::String("rag-v2".to_string())),
|
||||
);
|
||||
assert!(snap.pointer("/score_gate").is_some());
|
||||
assert!(snap.pointer("/rrf_k").is_some());
|
||||
@@ -336,21 +336,29 @@ fn runner_lexical_is_deterministic_per_query_payload() {
|
||||
"- id: q1\n query: ownership\n- id: q2\n query: heading\n",
|
||||
);
|
||||
|
||||
let run_a = run_with_golden(&yaml, || {
|
||||
let mut run_a = run_with_golden(&yaml, || {
|
||||
run_eval_with_config(&env.config, &lexical_opts()).unwrap()
|
||||
});
|
||||
let run_b = run_with_golden(&yaml, || {
|
||||
let mut run_b = run_with_golden(&yaml, || {
|
||||
run_eval_with_config(&env.config, &lexical_opts()).unwrap()
|
||||
});
|
||||
|
||||
// Run-level fields (`run_id`, `created_at`) intentionally diverge;
|
||||
// the per-query payload (which is what the snapshot fixture pins)
|
||||
// must be byte-identical.
|
||||
// must be byte-identical EXCEPT for `elapsed_ms`. Timing-sensitive
|
||||
// fields aren't determinism signals — they're µs-scale wall-clock
|
||||
// jitter and would otherwise make this assertion a flaky one (a 0
|
||||
// vs 1 ms divergence was observed under contended-CI load). Normalize
|
||||
// before comparing; see test #7 for the same exclusion done via a
|
||||
// projection.
|
||||
for qr in run_a.per_query.iter_mut().chain(run_b.per_query.iter_mut()) {
|
||||
qr.elapsed_ms = 0;
|
||||
}
|
||||
let a_json = serde_json::to_string(&run_a.per_query).unwrap();
|
||||
let b_json = serde_json::to_string(&run_b.per_query).unwrap();
|
||||
assert_eq!(
|
||||
a_json, b_json,
|
||||
"lexical-only per_query payload must be byte-identical across runs"
|
||||
"lexical-only per_query payload must be byte-identical across runs (timing normalized)"
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -19,6 +19,8 @@ tracing = { workspace = true }
|
||||
# /dependencies endpoint — rmcp declares optional schemars = "^1.0").
|
||||
schemars = "1"
|
||||
|
||||
time = { workspace = true }
|
||||
|
||||
kebab-app = { path = "../kebab-app" }
|
||||
kebab-config = { path = "../kebab-config" }
|
||||
kebab-core = { path = "../kebab-core" }
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
//! MCP (Model Context Protocol) server over stdio. Exposes 6 tools
|
||||
//! (`search` / `ask` / `schema` / `doctor` / `ingest_file` / `ingest_stdin`)
|
||||
//! backed by `kebab-app` facade methods. Used by `kebab-cli`'s `Cmd::Mcp` arm.
|
||||
//! MCP (Model Context Protocol) server over stdio. Exposes 8 tools
|
||||
//! (`search` / `ask` / `schema` / `doctor` / `ingest_file` / `ingest_stdin`
|
||||
//! / `fetch` / `bulk_search`) backed by `kebab-app` facade methods. Used by
|
||||
//! `kebab-cli`'s `Cmd::Mcp` arm.
|
||||
//!
|
||||
//! See spec `docs/superpowers/specs/2026-05-07-p9-fb-30-mcp-server-design.md`.
|
||||
|
||||
@@ -61,6 +62,16 @@ pub fn build_tools_vec() -> Vec<Tool> {
|
||||
"Ingest markdown content into the knowledge base. v1 markdown only. Frontmatter (title + source_uri) auto-injected.",
|
||||
schema_for_type::<tools::ingest_stdin::IngestStdinInput>(),
|
||||
),
|
||||
Tool::new(
|
||||
"fetch",
|
||||
"Verbatim fetch — chunk / doc / span modes. Returns fetch_result.v1 with the indexed text (no LLM rewrite).",
|
||||
schema_for_type::<tools::fetch::FetchInput>(),
|
||||
),
|
||||
Tool::new(
|
||||
"bulk_search",
|
||||
"Bulk multi-query search — N queries per call (cap 100). Each query mirrors the `search` input shape; returns `bulk_search_response.v1` with per-query results + summary. Sequential execution reuses one App instance so cache / embedder cold-start cost amortizes.",
|
||||
schema_for_type::<tools::bulk_search::BulkSearchInput>(),
|
||||
),
|
||||
]
|
||||
}
|
||||
|
||||
@@ -157,6 +168,20 @@ impl ServerHandler for KebabHandler {
|
||||
})
|
||||
.await
|
||||
}
|
||||
"fetch" => {
|
||||
let args = request.arguments.unwrap_or_default();
|
||||
self.spawn_tool(args, |state, input| {
|
||||
tools::fetch::handle(&state, input)
|
||||
})
|
||||
.await
|
||||
}
|
||||
"bulk_search" => {
|
||||
let args = request.arguments.unwrap_or_default();
|
||||
self.spawn_tool(args, |state, input| {
|
||||
tools::bulk_search::handle(&state, input)
|
||||
})
|
||||
.await
|
||||
}
|
||||
_other => Err(ErrorData::method_not_found::<
|
||||
rmcp::model::CallToolRequestMethod,
|
||||
>()),
|
||||
|
||||
55
crates/kebab-mcp/src/tools/bulk_search.rs
Normal file
55
crates/kebab-mcp/src/tools/bulk_search.rs
Normal file
@@ -0,0 +1,55 @@
|
||||
//! `bulk_search` tool — wraps `kebab_app::bulk_search_with_config`.
|
||||
//! Input: `{ queries: [<SearchInput shape>, ...] }`.
|
||||
//! Output: `bulk_search_response.v1` envelope (results + summary).
|
||||
|
||||
use rmcp::model::CallToolResult;
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::error::{to_tool_error, to_tool_success};
|
||||
use crate::state::KebabAppState;
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize, JsonSchema)]
|
||||
pub struct BulkSearchInput {
|
||||
/// Per-query inputs. Each item mirrors the single-query `search`
|
||||
/// tool's input shape — `query` is required, all other fields are
|
||||
/// optional and default to single-search defaults. Capped at 100
|
||||
/// items; exceeding returns an `invalid_input` tool error without
|
||||
/// running any query.
|
||||
pub queries: Vec<serde_json::Value>,
|
||||
}
|
||||
|
||||
pub fn handle(state: &KebabAppState, input: BulkSearchInput) -> CallToolResult {
|
||||
let cfg_clone = (*state.config).clone();
|
||||
match kebab_app::bulk_search_with_config(cfg_clone, input.queries) {
|
||||
Ok((items, summary)) => {
|
||||
let tagged_items: Vec<serde_json::Value> = items
|
||||
.iter()
|
||||
.map(|it| {
|
||||
let mut v = serde_json::to_value(it).unwrap_or(serde_json::Value::Null);
|
||||
if let serde_json::Value::Object(ref mut map) = v {
|
||||
map.insert(
|
||||
"schema_version".to_string(),
|
||||
serde_json::Value::String("bulk_search_item.v1".to_string()),
|
||||
);
|
||||
}
|
||||
v
|
||||
})
|
||||
.collect();
|
||||
let envelope = serde_json::json!({
|
||||
"schema_version": "bulk_search_response.v1",
|
||||
"results": tagged_items,
|
||||
"summary": {
|
||||
"total": summary.total,
|
||||
"succeeded": summary.succeeded,
|
||||
"failed": summary.failed,
|
||||
},
|
||||
});
|
||||
match serde_json::to_string(&envelope) {
|
||||
Ok(json) => to_tool_success(json),
|
||||
Err(e) => to_tool_error(&anyhow::anyhow!(e)),
|
||||
}
|
||||
}
|
||||
Err(e) => to_tool_error(&e),
|
||||
}
|
||||
}
|
||||
99
crates/kebab-mcp/src/tools/fetch.rs
Normal file
99
crates/kebab-mcp/src/tools/fetch.rs
Normal file
@@ -0,0 +1,99 @@
|
||||
//! p9-fb-35 `fetch` tool — wraps `kebab_app::fetch_with_config`.
|
||||
//!
|
||||
//! Three modes (chunk / doc / span). Output is `fetch_result.v1`.
|
||||
//!
|
||||
//! Mirrors the CLI surface (`kebab fetch <kind> ...`): same input shape,
|
||||
//! same wire envelope. Missing kind-specific fields produce an `error.v1`
|
||||
//! with `code = "invalid_input"`.
|
||||
|
||||
use rmcp::model::CallToolResult;
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::error::{to_tool_error, to_tool_success};
|
||||
use crate::state::KebabAppState;
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize, JsonSchema)]
|
||||
pub struct FetchInput {
|
||||
/// "chunk" | "doc" | "span"
|
||||
pub kind: String,
|
||||
/// Required when kind = "chunk".
|
||||
pub chunk_id: Option<String>,
|
||||
/// Required when kind = "doc" or "span".
|
||||
pub doc_id: Option<String>,
|
||||
/// Required when kind = "span" (1-based, inclusive).
|
||||
pub line_start: Option<u32>,
|
||||
pub line_end: Option<u32>,
|
||||
/// chunk only: ±N surrounding chunks.
|
||||
pub context: Option<u32>,
|
||||
/// doc/span only: chars/4 budget.
|
||||
pub max_tokens: Option<usize>,
|
||||
}
|
||||
|
||||
pub fn handle(state: &KebabAppState, input: FetchInput) -> CallToolResult {
|
||||
let query = match input.kind.as_str() {
|
||||
"chunk" => match input.chunk_id {
|
||||
Some(id) => kebab_core::FetchQuery::Chunk(kebab_core::ChunkId(id)),
|
||||
None => return invalid_input("kind=chunk requires chunk_id"),
|
||||
},
|
||||
"doc" => match input.doc_id {
|
||||
Some(id) => kebab_core::FetchQuery::Doc(kebab_core::DocumentId(id)),
|
||||
None => return invalid_input("kind=doc requires doc_id"),
|
||||
},
|
||||
"span" => match (input.doc_id, input.line_start, input.line_end) {
|
||||
(Some(id), Some(start), Some(end)) => kebab_core::FetchQuery::Span {
|
||||
doc_id: kebab_core::DocumentId(id),
|
||||
line_start: start,
|
||||
line_end: end,
|
||||
},
|
||||
_ => return invalid_input("kind=span requires doc_id, line_start, line_end"),
|
||||
},
|
||||
other => {
|
||||
return invalid_input(&format!(
|
||||
"unknown kind '{other}'; expected chunk|doc|span"
|
||||
));
|
||||
}
|
||||
};
|
||||
|
||||
let opts = kebab_core::FetchOpts {
|
||||
context: input.context,
|
||||
max_tokens: input.max_tokens,
|
||||
};
|
||||
|
||||
let cfg_clone = (*state.config).clone();
|
||||
match kebab_app::fetch_with_config(cfg_clone, query, opts) {
|
||||
Ok(r) => {
|
||||
// FetchResult does not carry a `schema_version` field, so we
|
||||
// tag the envelope inline (mirrors search.rs's pattern).
|
||||
let mut v = match serde_json::to_value(&r) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
return to_tool_error(&anyhow::anyhow!("FetchResult serialize: {e}"));
|
||||
}
|
||||
};
|
||||
if let serde_json::Value::Object(ref mut map) = v {
|
||||
map.insert(
|
||||
"schema_version".to_string(),
|
||||
serde_json::Value::String("fetch_result.v1".to_string()),
|
||||
);
|
||||
}
|
||||
match serde_json::to_string(&v) {
|
||||
Ok(json) => to_tool_success(json),
|
||||
Err(e) => to_tool_error(&anyhow::anyhow!(e)),
|
||||
}
|
||||
}
|
||||
Err(e) => to_tool_error(&e),
|
||||
}
|
||||
}
|
||||
|
||||
fn invalid_input(msg: &str) -> CallToolResult {
|
||||
use kebab_app::{ErrorV1, StructuredError};
|
||||
let err = anyhow::Error::new(StructuredError(ErrorV1 {
|
||||
schema_version: "error.v1".to_string(),
|
||||
code: "invalid_input".to_string(),
|
||||
message: msg.to_string(),
|
||||
details: serde_json::Value::Null,
|
||||
hint: None,
|
||||
}));
|
||||
to_tool_error(&err)
|
||||
}
|
||||
@@ -6,3 +6,5 @@ pub mod search;
|
||||
pub mod ask;
|
||||
pub mod ingest_file;
|
||||
pub mod ingest_stdin;
|
||||
pub mod fetch;
|
||||
pub mod bulk_search;
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
//! `search` tool — wraps `kebab_app::search_with_config`.
|
||||
//! Input: { query, mode?, k? }. Output: search_hit.v1 array JSON.
|
||||
//! `search` tool — wraps `kebab_app::search_with_opts_with_config`.
|
||||
//! Input: { query, mode?, k?, max_tokens?, snippet_chars?, cursor?,
|
||||
//! tags?, lang?, path_glob?, trust_min?, media?,
|
||||
//! ingested_after?, doc_id? }.
|
||||
//! Output: search_response.v1 envelope (hits + next_cursor + truncated).
|
||||
//!
|
||||
//! First tool with a non-empty `inputSchema`: `SearchInput` derives
|
||||
//! `JsonSchema` and `Tool::new` uses
|
||||
@@ -9,6 +12,8 @@ use rmcp::model::CallToolResult;
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use kebab_app::ERROR_V1_ID;
|
||||
|
||||
use crate::error::{to_tool_error, to_tool_success};
|
||||
use crate::state::KebabAppState;
|
||||
|
||||
@@ -17,38 +22,117 @@ pub struct SearchInput {
|
||||
/// User query (free text).
|
||||
pub query: String,
|
||||
/// Retrieval mode: "hybrid" (default), "lexical", or "vector".
|
||||
#[serde(default = "default_mode")]
|
||||
pub mode: String,
|
||||
pub mode: Option<String>,
|
||||
/// Top-K results. Defaults to 10. Clamped to 1–100.
|
||||
#[serde(default = "default_k")]
|
||||
pub k: usize,
|
||||
}
|
||||
|
||||
fn default_mode() -> String {
|
||||
"hybrid".to_string()
|
||||
}
|
||||
fn default_k() -> usize {
|
||||
10
|
||||
pub k: Option<usize>,
|
||||
/// p9-fb-34: cap result wire size at ~N tokens (chars/4 estimate).
|
||||
pub max_tokens: Option<usize>,
|
||||
/// p9-fb-34: per-hit snippet character cap.
|
||||
pub snippet_chars: Option<usize>,
|
||||
/// p9-fb-34: opaque cursor from a previous response.
|
||||
pub cursor: Option<String>,
|
||||
/// p9-fb-36: filter by `metadata.tags` (OR-within).
|
||||
pub tags: Option<Vec<String>>,
|
||||
/// p9-fb-36: filter by `documents.lang` (ISO code).
|
||||
pub lang: Option<String>,
|
||||
/// p9-fb-36: filter by `documents.workspace_path` glob.
|
||||
pub path_glob: Option<String>,
|
||||
/// p9-fb-36: filter by minimum `documents.trust_level`.
|
||||
/// Accepts: `"primary"`, `"secondary"`, `"generated"`.
|
||||
pub trust_min: Option<String>,
|
||||
/// p9-fb-36: filter by `assets.media_type` kind. IN-list. Accepts:
|
||||
/// `"markdown"`, `"pdf"`, `"image"`, `"audio"`, `"other"`. Aliases: `md` → `markdown`.
|
||||
pub media: Option<Vec<String>>,
|
||||
/// p9-fb-36: RFC3339 UTC timestamp. Invalid format → invalid_input.
|
||||
pub ingested_after: Option<String>,
|
||||
/// p9-fb-36: filter to a single doc.
|
||||
pub doc_id: Option<String>,
|
||||
/// p9-fb-37: when true, include a `trace` field on the response
|
||||
/// with pre-fusion lexical/vector candidate lists + per-stage timing.
|
||||
/// Bypasses cache (debug intent — fresh run guaranteed). Default false.
|
||||
pub trace: Option<bool>,
|
||||
}
|
||||
|
||||
pub fn handle(state: &KebabAppState, input: SearchInput) -> CallToolResult {
|
||||
let k = input.k.clamp(1, 100);
|
||||
let mode = match input.mode.as_str() {
|
||||
let k = input.k.unwrap_or(10).clamp(1, 100);
|
||||
let mode_str = input.mode.as_deref().unwrap_or("hybrid");
|
||||
let mode = match mode_str {
|
||||
"lexical" => kebab_core::SearchMode::Lexical,
|
||||
"vector" => kebab_core::SearchMode::Vector,
|
||||
_ => kebab_core::SearchMode::Hybrid,
|
||||
};
|
||||
|
||||
// p9-fb-36: parse filter inputs, returning invalid_input on bad values.
|
||||
let trust_min = match input.trust_min.as_deref() {
|
||||
Some(s) => match s.to_ascii_lowercase().as_str() {
|
||||
"primary" => Some(kebab_core::TrustLevel::Primary),
|
||||
"secondary" => Some(kebab_core::TrustLevel::Secondary),
|
||||
"generated" => Some(kebab_core::TrustLevel::Generated),
|
||||
other => {
|
||||
return invalid_input(&format!(
|
||||
"trust_min: unknown level '{other}'; expected primary|secondary|generated"
|
||||
));
|
||||
}
|
||||
},
|
||||
None => None,
|
||||
};
|
||||
|
||||
let ingested_after = match input.ingested_after.as_deref() {
|
||||
Some(s) => {
|
||||
match time::OffsetDateTime::parse(
|
||||
s,
|
||||
&time::format_description::well_known::Rfc3339,
|
||||
) {
|
||||
Ok(ts) => Some(ts),
|
||||
Err(e) => {
|
||||
return invalid_input(&format!(
|
||||
"ingested_after: invalid RFC3339 '{s}': {e}"
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
|
||||
let media: Vec<String> = input
|
||||
.media
|
||||
.clone()
|
||||
.unwrap_or_default()
|
||||
.iter()
|
||||
.map(|s| normalize_media_alias(s))
|
||||
.collect();
|
||||
|
||||
let filters = kebab_core::SearchFilters {
|
||||
tags_any: input.tags.clone().unwrap_or_default(),
|
||||
lang: input.lang.clone().map(kebab_core::Lang),
|
||||
path_glob: input.path_glob.clone(),
|
||||
trust_min,
|
||||
media,
|
||||
ingested_after,
|
||||
doc_id: input.doc_id.clone().map(kebab_core::DocumentId),
|
||||
repo: vec![],
|
||||
code_lang: vec![],
|
||||
};
|
||||
|
||||
let query = kebab_core::SearchQuery {
|
||||
text: input.query,
|
||||
mode,
|
||||
k,
|
||||
filters: kebab_core::SearchFilters::default(),
|
||||
filters,
|
||||
};
|
||||
match kebab_app::search_with_config((*state.config).clone(), query) {
|
||||
Ok(hits) => {
|
||||
let opts = kebab_core::SearchOpts {
|
||||
max_tokens: input.max_tokens,
|
||||
snippet_chars: input.snippet_chars,
|
||||
cursor: input.cursor,
|
||||
trace: input.trace.unwrap_or(false),
|
||||
};
|
||||
let cfg_clone = (*state.config).clone();
|
||||
match kebab_app::search_with_opts_with_config(cfg_clone, query, opts) {
|
||||
Ok(resp) => {
|
||||
// SearchHit (kebab-core) does not carry a `schema_version` field,
|
||||
// so we tag each element inline before serialising.
|
||||
let tagged: Vec<serde_json::Value> = hits
|
||||
let tagged: Vec<serde_json::Value> = resp
|
||||
.hits
|
||||
.iter()
|
||||
.map(|h| {
|
||||
let mut v = serde_json::to_value(h).unwrap_or_default();
|
||||
@@ -61,7 +145,20 @@ pub fn handle(state: &KebabAppState, input: SearchInput) -> CallToolResult {
|
||||
v
|
||||
})
|
||||
.collect();
|
||||
match serde_json::to_string(&serde_json::Value::Array(tagged)) {
|
||||
let mut envelope = serde_json::json!({
|
||||
"schema_version": "search_response.v1",
|
||||
"hits": tagged,
|
||||
"next_cursor": resp.next_cursor,
|
||||
"truncated": resp.truncated,
|
||||
});
|
||||
if let Some(trace) = &resp.trace {
|
||||
let trace_v =
|
||||
serde_json::to_value(trace).unwrap_or(serde_json::Value::Null);
|
||||
if let serde_json::Value::Object(ref mut map) = envelope {
|
||||
map.insert("trace".to_string(), trace_v);
|
||||
}
|
||||
}
|
||||
match serde_json::to_string(&envelope) {
|
||||
Ok(json) => to_tool_success(json),
|
||||
Err(e) => to_tool_error(&anyhow::anyhow!(e)),
|
||||
}
|
||||
@@ -69,3 +166,22 @@ pub fn handle(state: &KebabAppState, input: SearchInput) -> CallToolResult {
|
||||
Err(e) => to_tool_error(&e),
|
||||
}
|
||||
}
|
||||
|
||||
fn normalize_media_alias(s: &str) -> String {
|
||||
match s.to_ascii_lowercase().as_str() {
|
||||
"md" => "markdown".to_string(),
|
||||
other => other.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn invalid_input(msg: &str) -> CallToolResult {
|
||||
use kebab_app::{ErrorV1, StructuredError};
|
||||
let err = anyhow::Error::new(StructuredError(ErrorV1 {
|
||||
schema_version: ERROR_V1_ID.to_string(),
|
||||
code: "invalid_input".to_string(),
|
||||
message: msg.to_string(),
|
||||
details: serde_json::Value::Null,
|
||||
hint: None,
|
||||
}));
|
||||
to_tool_error(&err)
|
||||
}
|
||||
|
||||
121
crates/kebab-mcp/tests/tools_call_bulk_search.rs
Normal file
121
crates/kebab-mcp/tests/tools_call_bulk_search.rs
Normal file
@@ -0,0 +1,121 @@
|
||||
//! p9-fb-42: integration tests for `mcp__kebab__bulk_search`.
|
||||
|
||||
use std::fs;
|
||||
|
||||
use kebab_config::Config;
|
||||
use kebab_core::SourceScope;
|
||||
use kebab_mcp::{KebabAppState, KebabHandler};
|
||||
use rmcp::model::RawContent;
|
||||
use serde_json::json;
|
||||
|
||||
fn minimal_config(data_dir: &std::path::Path, workspace_root: &std::path::Path) -> Config {
|
||||
let mut cfg = Config::defaults();
|
||||
cfg.storage.data_dir = data_dir.to_string_lossy().into_owned();
|
||||
cfg.storage.model_dir = data_dir.join("models").to_string_lossy().into_owned();
|
||||
cfg.workspace.root = workspace_root.to_string_lossy().into_owned();
|
||||
cfg.workspace.exclude.clear();
|
||||
cfg.models.embedding.provider = "none".to_string();
|
||||
cfg.models.embedding.dimensions = 0;
|
||||
cfg
|
||||
}
|
||||
|
||||
fn setup() -> (tempfile::TempDir, KebabHandler) {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let data_dir = dir.path().join("data");
|
||||
let workspace_root = dir.path().join("notes");
|
||||
fs::create_dir_all(&data_dir).unwrap();
|
||||
fs::create_dir_all(&workspace_root).unwrap();
|
||||
let config = minimal_config(&data_dir, &workspace_root);
|
||||
fs::write(
|
||||
workspace_root.join("a.md"),
|
||||
"# Alpha\n\nThis document mentions kebab and bread.",
|
||||
)
|
||||
.unwrap();
|
||||
let scope = SourceScope { root: workspace_root.clone(), include: vec![], exclude: vec![] };
|
||||
let _ = kebab_app::ingest_with_config(config.clone(), scope, false).unwrap();
|
||||
let state = KebabAppState::new(config, None);
|
||||
let handler = KebabHandler::new(state);
|
||||
(dir, handler)
|
||||
}
|
||||
|
||||
fn extract_json(result: &rmcp::model::CallToolResult) -> serde_json::Value {
|
||||
assert!(!result.is_error.unwrap_or(false), "expected isError=false, got {result:?}");
|
||||
let content = result.content.first().expect("at least one content item");
|
||||
let text = match &content.raw {
|
||||
RawContent::Text(t) => &t.text,
|
||||
other => panic!("expected Text content, got {other:?}"),
|
||||
};
|
||||
serde_json::from_str(text).expect("valid JSON")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bulk_search_two_queries_returns_envelope() {
|
||||
let (_dir, handler) = setup();
|
||||
let input = kebab_mcp::tools::bulk_search::BulkSearchInput {
|
||||
queries: vec![
|
||||
json!({"query": "kebab", "mode": "lexical", "k": 5}),
|
||||
json!({"query": "bread", "mode": "lexical", "k": 5}),
|
||||
],
|
||||
};
|
||||
let result = kebab_mcp::tools::bulk_search::handle(handler.state(), input);
|
||||
let v = extract_json(&result);
|
||||
assert_eq!(v["schema_version"], "bulk_search_response.v1");
|
||||
let results = v["results"].as_array().expect("results array");
|
||||
assert_eq!(results.len(), 2);
|
||||
for r in results {
|
||||
assert_eq!(r["schema_version"], "bulk_search_item.v1");
|
||||
assert!(r["response"].is_object());
|
||||
assert!(r["error"].is_null());
|
||||
}
|
||||
assert_eq!(v["summary"]["total"], 2);
|
||||
assert_eq!(v["summary"]["succeeded"], 2);
|
||||
assert_eq!(v["summary"]["failed"], 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bulk_search_empty_queries_returns_empty_envelope() {
|
||||
let (_dir, handler) = setup();
|
||||
let input = kebab_mcp::tools::bulk_search::BulkSearchInput { queries: vec![] };
|
||||
let result = kebab_mcp::tools::bulk_search::handle(handler.state(), input);
|
||||
let v = extract_json(&result);
|
||||
assert_eq!(v["schema_version"], "bulk_search_response.v1");
|
||||
assert_eq!(v["results"].as_array().unwrap().len(), 0);
|
||||
assert_eq!(v["summary"]["total"], 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bulk_search_invalid_item_field_continues_with_per_item_error() {
|
||||
let (_dir, handler) = setup();
|
||||
let input = kebab_mcp::tools::bulk_search::BulkSearchInput {
|
||||
queries: vec![
|
||||
json!({"query": "kebab", "mode": "lexical"}),
|
||||
json!({"query": "bread", "mode": "bogus"}), // invalid mode
|
||||
],
|
||||
};
|
||||
let result = kebab_mcp::tools::bulk_search::handle(handler.state(), input);
|
||||
let v = extract_json(&result);
|
||||
let results = v["results"].as_array().unwrap();
|
||||
assert_eq!(results.len(), 2);
|
||||
assert!(results[0]["error"].is_null());
|
||||
assert!(results[1]["error"].is_object());
|
||||
assert_eq!(results[1]["error"]["code"], "invalid_input");
|
||||
assert_eq!(v["summary"]["succeeded"], 1);
|
||||
assert_eq!(v["summary"]["failed"], 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bulk_search_over_cap_returns_tool_error() {
|
||||
let (_dir, handler) = setup();
|
||||
let queries: Vec<serde_json::Value> = (0..101)
|
||||
.map(|_| json!({"query": "x", "mode": "lexical"}))
|
||||
.collect();
|
||||
let input = kebab_mcp::tools::bulk_search::BulkSearchInput { queries };
|
||||
let result = kebab_mcp::tools::bulk_search::handle(handler.state(), input);
|
||||
assert!(result.is_error.unwrap_or(false), "expected isError=true");
|
||||
let content = result.content.first().expect("error content");
|
||||
let text = match &content.raw {
|
||||
RawContent::Text(t) => &t.text,
|
||||
other => panic!("expected Text content, got {other:?}"),
|
||||
};
|
||||
assert!(text.contains("max 100"), "expected 'max 100' in error: {text}");
|
||||
}
|
||||
223
crates/kebab-mcp/tests/tools_call_fetch.rs
Normal file
223
crates/kebab-mcp/tests/tools_call_fetch.rs
Normal file
@@ -0,0 +1,223 @@
|
||||
//! p9-fb-35: tools/call name=fetch — chunk happy path + invalid_input.
|
||||
//!
|
||||
//! Mirrors `tools_call_search.rs` setup: a TempDir KB with embedding
|
||||
//! provider = "none" (no Ollama / fastembed) and a single ingested
|
||||
//! markdown doc. We discover a `chunk_id` via the search tool, call
|
||||
//! `fetch` with it, then exercise the missing-arg branch separately.
|
||||
|
||||
use std::fs;
|
||||
|
||||
use kebab_config::Config;
|
||||
use kebab_core::SourceScope;
|
||||
use kebab_mcp::{KebabAppState, KebabHandler};
|
||||
use rmcp::model::RawContent;
|
||||
|
||||
fn minimal_config(data_dir: &std::path::Path, workspace_root: &std::path::Path) -> Config {
|
||||
let mut cfg = Config::defaults();
|
||||
cfg.storage.data_dir = data_dir.to_string_lossy().into_owned();
|
||||
cfg.storage.model_dir = data_dir
|
||||
.join("models")
|
||||
.to_string_lossy()
|
||||
.into_owned();
|
||||
cfg.workspace.root = workspace_root.to_string_lossy().into_owned();
|
||||
cfg.workspace.exclude.clear();
|
||||
cfg.models.embedding.provider = "none".to_string();
|
||||
cfg.models.embedding.dimensions = 0;
|
||||
cfg
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fetch_tool_chunk_returns_fetch_result_v1() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let data_dir = dir.path().join("data");
|
||||
let workspace_root = dir.path().join("notes");
|
||||
fs::create_dir_all(&data_dir).unwrap();
|
||||
fs::create_dir_all(&workspace_root).unwrap();
|
||||
|
||||
let config = minimal_config(&data_dir, &workspace_root);
|
||||
|
||||
fs::write(
|
||||
workspace_root.join("a.md"),
|
||||
"# Alpha\n\nThis document mentions kebab and bread.",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let scope = SourceScope {
|
||||
root: workspace_root.clone(),
|
||||
include: vec![],
|
||||
exclude: vec![],
|
||||
};
|
||||
let _ = kebab_app::ingest_with_config(config.clone(), scope, false).unwrap();
|
||||
|
||||
let state = KebabAppState::new(config, None);
|
||||
let handler = KebabHandler::new(state);
|
||||
|
||||
// Discover a chunk_id via the search tool.
|
||||
let search_result = kebab_mcp::tools::search::handle(
|
||||
handler.state(),
|
||||
kebab_mcp::tools::search::SearchInput {
|
||||
query: "kebab".to_string(),
|
||||
mode: Some("lexical".to_string()),
|
||||
k: Some(1),
|
||||
max_tokens: None,
|
||||
snippet_chars: None,
|
||||
cursor: None,
|
||||
tags: None,
|
||||
lang: None,
|
||||
path_glob: None,
|
||||
trust_min: None,
|
||||
media: None,
|
||||
ingested_after: None,
|
||||
doc_id: None,
|
||||
trace: None,
|
||||
},
|
||||
);
|
||||
let search_text = match &search_result.content.first().unwrap().raw {
|
||||
RawContent::Text(t) => t.text.clone(),
|
||||
other => panic!("expected text content, got {other:?}"),
|
||||
};
|
||||
let search_v: serde_json::Value = serde_json::from_str(&search_text).unwrap();
|
||||
let chunk_id = search_v["hits"][0]["chunk_id"]
|
||||
.as_str()
|
||||
.expect("chunk_id on first hit")
|
||||
.to_string();
|
||||
|
||||
// Call fetch with kind=chunk.
|
||||
let result = kebab_mcp::tools::fetch::handle(
|
||||
handler.state(),
|
||||
kebab_mcp::tools::fetch::FetchInput {
|
||||
kind: "chunk".to_string(),
|
||||
chunk_id: Some(chunk_id),
|
||||
doc_id: None,
|
||||
line_start: None,
|
||||
line_end: None,
|
||||
context: None,
|
||||
max_tokens: None,
|
||||
},
|
||||
);
|
||||
|
||||
assert!(
|
||||
!result.is_error.unwrap_or(false),
|
||||
"expected isError=false, got {:?}",
|
||||
result
|
||||
);
|
||||
|
||||
let content = result
|
||||
.content
|
||||
.first()
|
||||
.expect("expected at least one content item");
|
||||
let text = match &content.raw {
|
||||
RawContent::Text(t) => &t.text,
|
||||
other => panic!("expected text content, got {other:?}"),
|
||||
};
|
||||
|
||||
let v: serde_json::Value = serde_json::from_str(text).unwrap();
|
||||
assert_eq!(
|
||||
v.get("schema_version").and_then(|s| s.as_str()),
|
||||
Some("fetch_result.v1"),
|
||||
"envelope must carry schema_version=fetch_result.v1"
|
||||
);
|
||||
assert_eq!(
|
||||
v.get("kind").and_then(|s| s.as_str()),
|
||||
Some("chunk"),
|
||||
"kind must be 'chunk'"
|
||||
);
|
||||
assert!(
|
||||
v.get("chunk").is_some_and(|c| c.is_object()),
|
||||
"chunk payload must be populated for kind=chunk"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fetch_tool_invalid_kind_returns_invalid_input() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let data_dir = dir.path().join("data");
|
||||
let workspace_root = dir.path().join("notes");
|
||||
fs::create_dir_all(&data_dir).unwrap();
|
||||
fs::create_dir_all(&workspace_root).unwrap();
|
||||
|
||||
let config = minimal_config(&data_dir, &workspace_root);
|
||||
|
||||
let state = KebabAppState::new(config, None);
|
||||
let handler = KebabHandler::new(state);
|
||||
|
||||
let result = kebab_mcp::tools::fetch::handle(
|
||||
handler.state(),
|
||||
kebab_mcp::tools::fetch::FetchInput {
|
||||
kind: "garbage".to_string(),
|
||||
chunk_id: None,
|
||||
doc_id: None,
|
||||
line_start: None,
|
||||
line_end: None,
|
||||
context: None,
|
||||
max_tokens: None,
|
||||
},
|
||||
);
|
||||
|
||||
assert!(
|
||||
result.is_error.unwrap_or(false),
|
||||
"expected isError=true for unknown kind"
|
||||
);
|
||||
let content = result
|
||||
.content
|
||||
.first()
|
||||
.expect("expected at least one content item");
|
||||
let text = match &content.raw {
|
||||
RawContent::Text(t) => &t.text,
|
||||
other => panic!("expected text content, got {other:?}"),
|
||||
};
|
||||
let v: serde_json::Value = serde_json::from_str(text).unwrap();
|
||||
assert_eq!(
|
||||
v.get("schema_version").and_then(|s| s.as_str()),
|
||||
Some("error.v1"),
|
||||
"must carry error.v1 envelope"
|
||||
);
|
||||
assert_eq!(
|
||||
v.get("code").and_then(|s| s.as_str()),
|
||||
Some("invalid_input"),
|
||||
"code must be invalid_input for unknown kind"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fetch_tool_chunk_missing_id_returns_invalid_input() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let data_dir = dir.path().join("data");
|
||||
let workspace_root = dir.path().join("notes");
|
||||
fs::create_dir_all(&data_dir).unwrap();
|
||||
fs::create_dir_all(&workspace_root).unwrap();
|
||||
|
||||
let config = minimal_config(&data_dir, &workspace_root);
|
||||
|
||||
let state = KebabAppState::new(config, None);
|
||||
let handler = KebabHandler::new(state);
|
||||
|
||||
// kind=chunk but no chunk_id — invalid_input.
|
||||
let result = kebab_mcp::tools::fetch::handle(
|
||||
handler.state(),
|
||||
kebab_mcp::tools::fetch::FetchInput {
|
||||
kind: "chunk".to_string(),
|
||||
chunk_id: None,
|
||||
doc_id: None,
|
||||
line_start: None,
|
||||
line_end: None,
|
||||
context: None,
|
||||
max_tokens: None,
|
||||
},
|
||||
);
|
||||
|
||||
assert!(
|
||||
result.is_error.unwrap_or(false),
|
||||
"expected isError=true when chunk_id is missing"
|
||||
);
|
||||
let content = result.content.first().unwrap();
|
||||
let text = match &content.raw {
|
||||
RawContent::Text(t) => &t.text,
|
||||
other => panic!("expected text content, got {other:?}"),
|
||||
};
|
||||
let v: serde_json::Value = serde_json::from_str(text).unwrap();
|
||||
assert_eq!(
|
||||
v.get("code").and_then(|s| s.as_str()),
|
||||
Some("invalid_input")
|
||||
);
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
//! Integration: tools/call name=search — verify response is search_hit.v1 array.
|
||||
//! Integration: tools/call name=search — verify response is search_response.v1.
|
||||
|
||||
use std::fs;
|
||||
|
||||
@@ -22,7 +22,7 @@ fn minimal_config(data_dir: &std::path::Path, workspace_root: &std::path::Path)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn search_tool_returns_search_hits_array() {
|
||||
async fn search_tool_returns_search_response_v1() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let data_dir = dir.path().join("data");
|
||||
let workspace_root = dir.path().join("notes");
|
||||
@@ -53,8 +53,19 @@ async fn search_tool_returns_search_hits_array() {
|
||||
handler.state(),
|
||||
kebab_mcp::tools::search::SearchInput {
|
||||
query: "kebab".to_string(),
|
||||
mode: "lexical".to_string(),
|
||||
k: 5,
|
||||
mode: Some("lexical".to_string()),
|
||||
k: Some(5),
|
||||
max_tokens: None,
|
||||
snippet_chars: None,
|
||||
cursor: None,
|
||||
tags: None,
|
||||
lang: None,
|
||||
path_glob: None,
|
||||
trust_min: None,
|
||||
media: None,
|
||||
ingested_after: None,
|
||||
doc_id: None,
|
||||
trace: None,
|
||||
},
|
||||
);
|
||||
|
||||
@@ -75,16 +86,208 @@ async fn search_tool_returns_search_hits_array() {
|
||||
};
|
||||
|
||||
let v: serde_json::Value = serde_json::from_str(text).unwrap();
|
||||
let arr = v.as_array().expect("search returns a JSON array");
|
||||
assert_eq!(
|
||||
v.get("schema_version").and_then(|s| s.as_str()),
|
||||
Some("search_response.v1"),
|
||||
"envelope should carry schema_version=search_response.v1"
|
||||
);
|
||||
let hits = v
|
||||
.get("hits")
|
||||
.and_then(|h| h.as_array())
|
||||
.expect("hits must be a JSON array");
|
||||
assert!(
|
||||
!arr.is_empty(),
|
||||
!hits.is_empty(),
|
||||
"expected at least one hit for 'kebab' in 'a.md'"
|
||||
);
|
||||
assert_eq!(
|
||||
arr[0]
|
||||
hits[0]
|
||||
.get("schema_version")
|
||||
.and_then(|s| s.as_str()),
|
||||
Some("search_hit.v1"),
|
||||
"first hit should carry schema_version=search_hit.v1"
|
||||
);
|
||||
// truncated must be present (bool); next_cursor may be null on last page.
|
||||
assert!(
|
||||
v.get("truncated").and_then(|t| t.as_bool()).is_some(),
|
||||
"envelope should carry truncated:bool"
|
||||
);
|
||||
assert!(
|
||||
v.get("next_cursor").is_some(),
|
||||
"envelope should carry next_cursor (possibly null)"
|
||||
);
|
||||
}
|
||||
|
||||
/// p9-fb-36: search with doc_id filter — only hits from the target doc.
|
||||
#[tokio::test]
|
||||
async fn search_with_doc_id_filter_returns_only_target() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let data_dir = dir.path().join("data");
|
||||
let workspace_root = dir.path().join("notes");
|
||||
fs::create_dir_all(&data_dir).unwrap();
|
||||
fs::create_dir_all(&workspace_root).unwrap();
|
||||
|
||||
let config = minimal_config(&data_dir, &workspace_root);
|
||||
|
||||
// Write two markdown documents, both containing the query term.
|
||||
fs::write(
|
||||
workspace_root.join("a.md"),
|
||||
"# Alpha\n\nThis document mentions kebab and flatbread.",
|
||||
)
|
||||
.unwrap();
|
||||
fs::write(
|
||||
workspace_root.join("b.md"),
|
||||
"# Beta\n\nAnother document about kebab wraps and fillings.",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let scope = SourceScope {
|
||||
root: workspace_root.clone(),
|
||||
include: vec![],
|
||||
exclude: vec![],
|
||||
};
|
||||
let _ = kebab_app::ingest_with_config(config.clone(), scope, false).unwrap();
|
||||
|
||||
let state = KebabAppState::new(config, None);
|
||||
let handler = KebabHandler::new(state);
|
||||
|
||||
// First: unfiltered search to discover a doc_id from one of the docs.
|
||||
let unfiltered = kebab_mcp::tools::search::handle(
|
||||
handler.state(),
|
||||
kebab_mcp::tools::search::SearchInput {
|
||||
query: "kebab".to_string(),
|
||||
mode: Some("lexical".to_string()),
|
||||
k: Some(10),
|
||||
max_tokens: None,
|
||||
snippet_chars: None,
|
||||
cursor: None,
|
||||
tags: None,
|
||||
lang: None,
|
||||
path_glob: None,
|
||||
trust_min: None,
|
||||
media: None,
|
||||
ingested_after: None,
|
||||
doc_id: None,
|
||||
trace: None,
|
||||
},
|
||||
);
|
||||
assert!(
|
||||
!unfiltered.is_error.unwrap_or(false),
|
||||
"unfiltered search failed: {:?}",
|
||||
unfiltered
|
||||
);
|
||||
let unfiltered_text = match &unfiltered.content.first().unwrap().raw {
|
||||
RawContent::Text(t) => t.text.clone(),
|
||||
other => panic!("expected text content, got {other:?}"),
|
||||
};
|
||||
let unfiltered_v: serde_json::Value = serde_json::from_str(&unfiltered_text).unwrap();
|
||||
let hits = unfiltered_v["hits"].as_array().expect("hits must be array");
|
||||
assert!(hits.len() >= 2, "expected hits from both docs");
|
||||
|
||||
// Pick the doc_id of the first hit.
|
||||
let target_doc_id = hits[0]["doc_id"]
|
||||
.as_str()
|
||||
.expect("doc_id on first hit")
|
||||
.to_string();
|
||||
|
||||
// Now search with doc_id filter — all results must belong to that doc.
|
||||
let filtered = kebab_mcp::tools::search::handle(
|
||||
handler.state(),
|
||||
kebab_mcp::tools::search::SearchInput {
|
||||
query: "kebab".to_string(),
|
||||
mode: Some("lexical".to_string()),
|
||||
k: Some(10),
|
||||
max_tokens: None,
|
||||
snippet_chars: None,
|
||||
cursor: None,
|
||||
tags: None,
|
||||
lang: None,
|
||||
path_glob: None,
|
||||
trust_min: None,
|
||||
media: None,
|
||||
ingested_after: None,
|
||||
doc_id: Some(target_doc_id.clone()),
|
||||
trace: None,
|
||||
},
|
||||
);
|
||||
assert!(
|
||||
!filtered.is_error.unwrap_or(false),
|
||||
"filtered search failed: {:?}",
|
||||
filtered
|
||||
);
|
||||
let filtered_text = match &filtered.content.first().unwrap().raw {
|
||||
RawContent::Text(t) => t.text.clone(),
|
||||
other => panic!("expected text content, got {other:?}"),
|
||||
};
|
||||
let filtered_v: serde_json::Value = serde_json::from_str(&filtered_text).unwrap();
|
||||
let filtered_hits = filtered_v["hits"].as_array().expect("hits must be array");
|
||||
|
||||
assert!(
|
||||
!filtered_hits.is_empty(),
|
||||
"expected at least one hit for target doc"
|
||||
);
|
||||
for hit in filtered_hits {
|
||||
assert_eq!(
|
||||
hit["doc_id"].as_str(),
|
||||
Some(target_doc_id.as_str()),
|
||||
"all filtered hits must belong to the target doc"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// p9-fb-36: invalid RFC3339 for ingested_after → invalid_input error.v1.
|
||||
#[tokio::test]
|
||||
async fn search_with_invalid_ingested_after_returns_invalid_input() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let data_dir = dir.path().join("data");
|
||||
let workspace_root = dir.path().join("notes");
|
||||
fs::create_dir_all(&data_dir).unwrap();
|
||||
fs::create_dir_all(&workspace_root).unwrap();
|
||||
|
||||
let config = minimal_config(&data_dir, &workspace_root);
|
||||
let state = KebabAppState::new(config, None);
|
||||
let handler = KebabHandler::new(state);
|
||||
|
||||
let result = kebab_mcp::tools::search::handle(
|
||||
handler.state(),
|
||||
kebab_mcp::tools::search::SearchInput {
|
||||
query: "kebab".to_string(),
|
||||
mode: None,
|
||||
k: None,
|
||||
max_tokens: None,
|
||||
snippet_chars: None,
|
||||
cursor: None,
|
||||
tags: None,
|
||||
lang: None,
|
||||
path_glob: None,
|
||||
trust_min: None,
|
||||
media: None,
|
||||
ingested_after: Some("garbage".to_string()),
|
||||
doc_id: None,
|
||||
trace: None,
|
||||
},
|
||||
);
|
||||
|
||||
assert!(
|
||||
result.is_error.unwrap_or(false),
|
||||
"expected isError=true for invalid ingested_after"
|
||||
);
|
||||
let content = result
|
||||
.content
|
||||
.first()
|
||||
.expect("expected at least one content item");
|
||||
let text = match &content.raw {
|
||||
RawContent::Text(t) => &t.text,
|
||||
other => panic!("expected text content, got {other:?}"),
|
||||
};
|
||||
let v: serde_json::Value = serde_json::from_str(text).unwrap();
|
||||
assert_eq!(
|
||||
v.get("schema_version").and_then(|s| s.as_str()),
|
||||
Some("error.v1"),
|
||||
"must carry error.v1 envelope"
|
||||
);
|
||||
assert_eq!(
|
||||
v.get("code").and_then(|s| s.as_str()),
|
||||
Some("invalid_input"),
|
||||
"code must be invalid_input for bad RFC3339"
|
||||
);
|
||||
}
|
||||
|
||||
104
crates/kebab-mcp/tests/tools_call_search_trace.rs
Normal file
104
crates/kebab-mcp/tests/tools_call_search_trace.rs
Normal file
@@ -0,0 +1,104 @@
|
||||
//! p9-fb-37: integration test for `mcp__kebab__search` trace input/output.
|
||||
|
||||
use std::fs;
|
||||
|
||||
use kebab_config::Config;
|
||||
use kebab_core::SourceScope;
|
||||
use kebab_mcp::{KebabAppState, KebabHandler};
|
||||
use rmcp::model::RawContent;
|
||||
|
||||
fn minimal_config(data_dir: &std::path::Path, workspace_root: &std::path::Path) -> Config {
|
||||
let mut cfg = Config::defaults();
|
||||
cfg.storage.data_dir = data_dir.to_string_lossy().into_owned();
|
||||
cfg.storage.model_dir = data_dir.join("models").to_string_lossy().into_owned();
|
||||
cfg.workspace.root = workspace_root.to_string_lossy().into_owned();
|
||||
cfg.workspace.exclude.clear();
|
||||
cfg.models.embedding.provider = "none".to_string();
|
||||
cfg.models.embedding.dimensions = 0;
|
||||
cfg
|
||||
}
|
||||
|
||||
fn setup() -> (tempfile::TempDir, KebabHandler) {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let data_dir = dir.path().join("data");
|
||||
let workspace_root = dir.path().join("notes");
|
||||
fs::create_dir_all(&data_dir).unwrap();
|
||||
fs::create_dir_all(&workspace_root).unwrap();
|
||||
let config = minimal_config(&data_dir, &workspace_root);
|
||||
fs::write(
|
||||
workspace_root.join("a.md"),
|
||||
"# Alpha\n\nThis document mentions kebab and bread.",
|
||||
)
|
||||
.unwrap();
|
||||
let scope = SourceScope {
|
||||
root: workspace_root.clone(),
|
||||
include: vec![],
|
||||
exclude: vec![],
|
||||
};
|
||||
let _ = kebab_app::ingest_with_config(config.clone(), scope, false).unwrap();
|
||||
let state = KebabAppState::new(config, None);
|
||||
let handler = KebabHandler::new(state);
|
||||
(dir, handler)
|
||||
}
|
||||
|
||||
fn make_input(trace: Option<bool>) -> kebab_mcp::tools::search::SearchInput {
|
||||
kebab_mcp::tools::search::SearchInput {
|
||||
query: "kebab".to_string(),
|
||||
mode: Some("lexical".to_string()),
|
||||
k: Some(5),
|
||||
max_tokens: None,
|
||||
snippet_chars: None,
|
||||
cursor: None,
|
||||
tags: None,
|
||||
lang: None,
|
||||
path_glob: None,
|
||||
trust_min: None,
|
||||
media: None,
|
||||
ingested_after: None,
|
||||
doc_id: None,
|
||||
trace,
|
||||
}
|
||||
}
|
||||
|
||||
fn extract_json(result: &rmcp::model::CallToolResult) -> serde_json::Value {
|
||||
assert!(
|
||||
!result.is_error.unwrap_or(false),
|
||||
"expected isError=false, got {result:?}"
|
||||
);
|
||||
let content = result.content.first().expect("at least one content item");
|
||||
let text = match &content.raw {
|
||||
RawContent::Text(t) => &t.text,
|
||||
other => panic!("expected Text content, got {other:?}"),
|
||||
};
|
||||
serde_json::from_str(text).expect("valid JSON")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn search_with_trace_true_returns_trace_field() {
|
||||
let (_dir, handler) = setup();
|
||||
let result = kebab_mcp::tools::search::handle(handler.state(), make_input(Some(true)));
|
||||
let v = extract_json(&result);
|
||||
assert_eq!(v["schema_version"], "search_response.v1");
|
||||
assert!(v["trace"].is_object(), "trace field present when trace:true");
|
||||
assert!(v["trace"]["timing"]["total_ms"].is_number());
|
||||
assert!(v["trace"]["lexical"].is_array());
|
||||
assert!(v["trace"]["vector"].is_array());
|
||||
assert!(v["trace"]["rrf_inputs"].is_array());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn search_without_trace_omits_trace_field() {
|
||||
let (_dir, handler) = setup();
|
||||
let result = kebab_mcp::tools::search::handle(handler.state(), make_input(None));
|
||||
let v = extract_json(&result);
|
||||
assert_eq!(v["schema_version"], "search_response.v1");
|
||||
assert!(v.get("trace").is_none(), "trace absent when None");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn search_with_trace_false_omits_trace_field() {
|
||||
let (_dir, handler) = setup();
|
||||
let result = kebab_mcp::tools::search::handle(handler.state(), make_input(Some(false)));
|
||||
let v = extract_json(&result);
|
||||
assert!(v.get("trace").is_none(), "trace absent when false");
|
||||
}
|
||||
@@ -1,13 +1,13 @@
|
||||
//! Integration: `build_tools_vec` returns 6 tools with correct names and
|
||||
//! Integration: `build_tools_vec` returns 8 tools with correct names and
|
||||
//! inputSchema. Uses the extracted `pub fn build_tools_vec()` helper — no
|
||||
//! transport or RequestContext needed.
|
||||
|
||||
use kebab_mcp::build_tools_vec;
|
||||
|
||||
#[test]
|
||||
fn tools_list_returns_six_tools() {
|
||||
fn tools_list_returns_eight_tools() {
|
||||
let tools = build_tools_vec();
|
||||
assert_eq!(tools.len(), 6, "expected exactly 6 tools, got {}", tools.len());
|
||||
assert_eq!(tools.len(), 8, "expected exactly 8 tools, got {}", tools.len());
|
||||
|
||||
let names: Vec<&str> = tools.iter().map(|t| t.name.as_ref()).collect();
|
||||
assert!(names.contains(&"schema"), "missing 'schema' tool");
|
||||
@@ -16,6 +16,8 @@ fn tools_list_returns_six_tools() {
|
||||
assert!(names.contains(&"ask"), "missing 'ask' tool");
|
||||
assert!(names.contains(&"ingest_file"), "missing 'ingest_file' tool");
|
||||
assert!(names.contains(&"ingest_stdin"), "missing 'ingest_stdin' tool");
|
||||
assert!(names.contains(&"fetch"), "missing 'fetch' tool");
|
||||
assert!(names.contains(&"bulk_search"), "missing 'bulk_search' tool");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -467,6 +467,10 @@ mod tests {
|
||||
trust_level: TrustLevel::Primary,
|
||||
user_id_alias: None,
|
||||
user,
|
||||
repo: None,
|
||||
git_branch: None,
|
||||
git_commit: None,
|
||||
code_lang: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
24
crates/kebab-parse-code/Cargo.toml
Normal file
24
crates/kebab-parse-code/Cargo.toml
Normal file
@@ -0,0 +1,24 @@
|
||||
[package]
|
||||
name = "kebab-parse-code"
|
||||
version = { workspace = true }
|
||||
edition = { workspace = true }
|
||||
rust-version = { workspace = true }
|
||||
license = { workspace = true }
|
||||
repository = { workspace = true }
|
||||
description = "Language-aware code parsing for the kebab pipeline: lang dispatch / .git detect / skip helpers (P10-1A-1) + tree-sitter Rust AST extractor (P10-1A-2)"
|
||||
|
||||
[dependencies]
|
||||
kebab-core = { path = "../kebab-core" }
|
||||
anyhow = { workspace = true }
|
||||
gix = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
time = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
tree-sitter = { workspace = true }
|
||||
tree-sitter-rust = { workspace = true }
|
||||
tree-sitter-python = { workspace = true }
|
||||
tree-sitter-typescript = { workspace = true }
|
||||
tree-sitter-javascript = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = { workspace = true }
|
||||
574
crates/kebab-parse-code/src/javascript.rs
Normal file
574
crates/kebab-parse-code/src/javascript.rs
Normal file
@@ -0,0 +1,574 @@
|
||||
//! `kebab-parse-code::javascript` — tree-sitter JavaScript / JSX AST
|
||||
//! extractor (P10-1B Task K).
|
||||
//!
|
||||
//! Implements [`kebab_core::Extractor`] for [`MediaType::Code("javascript")`].
|
||||
//! Walks the tree-sitter parse tree (single grammar
|
||||
//! [`tree_sitter_javascript::LANGUAGE`] — the JS grammar handles `.jsx`
|
||||
//! as well, no second grammar needed) and emits one [`Block::Code`] per
|
||||
//! top-level AST semantic unit (free fn, class, each method,
|
||||
//! recursively per nested class), each carrying [`SourceSpan::Code`]
|
||||
//! with the unit's dotted symbol path prefixed by
|
||||
//! [`module_path_for_tsjs`].
|
||||
//!
|
||||
//! Glue declarations (`import_statement`, bare `export_statement`
|
||||
//! re-exports, `lexical_declaration` / `variable_declaration` at the
|
||||
//! module level, etc.) collapse into one grouped `<top-level>` (or
|
||||
//! `<module>`) unit.
|
||||
//!
|
||||
//! `export_statement` is unwrapped: an `export function|class` is
|
||||
//! treated as the inner declaration arm but the unit's line range
|
||||
//! comes from the OUTER `export_statement` so the `export ` prefix is
|
||||
//! folded in. `export default function () {}` / `export default class
|
||||
//! {}` (no `name` field) emits `default` as the symbol name.
|
||||
//!
|
||||
//! Differs from `typescript.rs` only by: single-grammar (no
|
||||
//! TS/TSX selection) and no `interface_declaration` /
|
||||
//! `type_alias_declaration` / `enum_declaration` arms (TS-only). All
|
||||
//! other walker behavior (export unwrap with `value`-field quirk for
|
||||
//! default-exported anonymous function/class, class-body method walk,
|
||||
//! glue flush, post-pass `<module>` → `<top-level>` rewrite) is
|
||||
//! identical.
|
||||
//!
|
||||
//! Scope follows 1A-2 / 1B Task K: AST unit extraction + dotted symbol
|
||||
//! paths + line ranges. Per design §3.4 / §9.1 / §9 versioning.
|
||||
|
||||
use anyhow::Result;
|
||||
use kebab_core::{
|
||||
Block, CanonicalDocument, CodeBlock, CommonBlock, Extractor, Lang, MediaType, Metadata,
|
||||
ParserVersion, Provenance, ProvenanceEvent, ProvenanceKind, SourceSpan, SourceType, TrustLevel,
|
||||
id_for_block, id_for_doc,
|
||||
};
|
||||
use serde_json::Map;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
use crate::scaffold::{filename_from_workspace_path, join_symbol, strip_extension};
|
||||
|
||||
pub const PARSER_VERSION: &str = "code-js-v1";
|
||||
|
||||
/// JavaScript / JSX AST extractor. Per-unit blocks via
|
||||
/// tree-sitter-javascript 0.25 (single `LANGUAGE` `LanguageFn` — the
|
||||
/// JS grammar covers `.jsx` natively, no second grammar) parsed by
|
||||
/// tree-sitter 0.26.
|
||||
pub struct JavascriptAstExtractor;
|
||||
|
||||
impl JavascriptAstExtractor {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for JavascriptAstExtractor {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl Extractor for JavascriptAstExtractor {
|
||||
fn supports(&self, m: &MediaType) -> bool {
|
||||
matches!(m, MediaType::Code(l) if l == "javascript")
|
||||
}
|
||||
|
||||
fn parser_version(&self) -> ParserVersion {
|
||||
ParserVersion(PARSER_VERSION.to_string())
|
||||
}
|
||||
|
||||
fn extract(
|
||||
&self,
|
||||
ctx: &kebab_core::ExtractContext<'_>,
|
||||
bytes: &[u8],
|
||||
) -> Result<CanonicalDocument> {
|
||||
let asset = ctx.asset;
|
||||
if !self.supports(&asset.media_type) {
|
||||
anyhow::bail!(
|
||||
"kebab-parse-code: unsupported media_type for JavascriptAstExtractor: {:?}",
|
||||
asset.media_type
|
||||
);
|
||||
}
|
||||
|
||||
let parser_version = self.parser_version();
|
||||
let doc_id = id_for_doc(&asset.workspace_path, &asset.asset_id, &parser_version);
|
||||
|
||||
let source = String::from_utf8(bytes.to_vec()).map_err(|e| {
|
||||
anyhow::anyhow!("kebab-parse-code: JavaScript source is not valid UTF-8: {e}")
|
||||
})?;
|
||||
|
||||
let mod_prefix = crate::lang::module_path_for_tsjs(&asset.workspace_path.0);
|
||||
let language: tree_sitter::Language = tree_sitter_javascript::LANGUAGE.into();
|
||||
let blocks = build_blocks(&source, &doc_id, &mod_prefix, language)?;
|
||||
let unit_count = blocks.len() as u32;
|
||||
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let mut events: Vec<ProvenanceEvent> = Vec::with_capacity(2);
|
||||
events.push(ProvenanceEvent {
|
||||
at: asset.discovered_at,
|
||||
agent: "kb-source-fs".to_string(),
|
||||
kind: ProvenanceKind::Discovered,
|
||||
note: None,
|
||||
});
|
||||
events.push(ProvenanceEvent {
|
||||
at: now,
|
||||
agent: "kb-parse-code".to_string(),
|
||||
kind: ProvenanceKind::Parsed,
|
||||
note: Some(format!(
|
||||
"parser_version={}; unit_count={}",
|
||||
parser_version.0, unit_count
|
||||
)),
|
||||
});
|
||||
|
||||
let title = {
|
||||
let fname = filename_from_workspace_path(&asset.workspace_path.0);
|
||||
strip_extension(&fname)
|
||||
};
|
||||
|
||||
// Resolve the file's absolute path for repo detection. If the
|
||||
// source URI carries a relative path, anchor it at the workspace
|
||||
// root so the `.git/` walk-up starts from the right place.
|
||||
let abs_path = match &asset.source_uri {
|
||||
kebab_core::SourceUri::File(p) => {
|
||||
if p.is_absolute() {
|
||||
p.clone()
|
||||
} else {
|
||||
ctx.workspace_root.join(p)
|
||||
}
|
||||
}
|
||||
kebab_core::SourceUri::Kb(_) => ctx.workspace_root.to_path_buf(),
|
||||
};
|
||||
let (repo, git_branch, git_commit) = match crate::repo::detect_repo(&abs_path) {
|
||||
Some(r) => (Some(r.name), r.branch, r.commit),
|
||||
None => (None, None, None),
|
||||
};
|
||||
|
||||
let metadata = Metadata {
|
||||
aliases: Vec::new(),
|
||||
tags: Vec::new(),
|
||||
created_at: asset.discovered_at,
|
||||
updated_at: asset.discovered_at,
|
||||
source_type: SourceType::Note,
|
||||
trust_level: TrustLevel::Primary,
|
||||
user_id_alias: None,
|
||||
user: Map::new(),
|
||||
repo,
|
||||
git_branch,
|
||||
git_commit,
|
||||
code_lang: Some("javascript".to_string()),
|
||||
};
|
||||
|
||||
tracing::debug!(
|
||||
target: "kebab-parse-code",
|
||||
"extracted JavaScript doc_id={} workspace_path={} units={}",
|
||||
doc_id.0,
|
||||
asset.workspace_path.0,
|
||||
unit_count
|
||||
);
|
||||
|
||||
Ok(CanonicalDocument {
|
||||
doc_id,
|
||||
source_asset_id: asset.asset_id.clone(),
|
||||
workspace_path: asset.workspace_path.clone(),
|
||||
title,
|
||||
lang: Lang("und".to_string()),
|
||||
blocks,
|
||||
metadata,
|
||||
provenance: Provenance { events },
|
||||
parser_version,
|
||||
schema_version: 1,
|
||||
doc_version: 1,
|
||||
last_chunker_version: None,
|
||||
last_embedding_version: None,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn build_blocks(
|
||||
source: &str,
|
||||
doc_id: &kebab_core::DocumentId,
|
||||
mod_prefix: &str,
|
||||
language: tree_sitter::Language,
|
||||
) -> anyhow::Result<Vec<kebab_core::Block>> {
|
||||
let mut parser = tree_sitter::Parser::new();
|
||||
parser
|
||||
.set_language(&language)
|
||||
.map_err(|e| anyhow::anyhow!("set tree-sitter-javascript language: {e}"))?;
|
||||
let tree = parser
|
||||
.parse(source.as_bytes(), None)
|
||||
.ok_or_else(|| anyhow::anyhow!("tree-sitter failed to parse JavaScript source"))?;
|
||||
let lines: Vec<&str> = source.split('\n').collect();
|
||||
|
||||
// units: (symbol, line_start, line_end, is_real_semantic_unit).
|
||||
// Glue groups are pushed with a sentinel symbol + is_real=false so a
|
||||
// post-pass can decide `<module>` vs `<top-level>` (same algorithm
|
||||
// as 1A Gap 1 / 1B Python / 1B TS).
|
||||
let mut units: Vec<(String, u32, u32, bool)> = Vec::new();
|
||||
// (is_module_only_kind 0/1, s, e). `is_module_only_kind` flags
|
||||
// `import_statement` and bare re-export `export_statement`s — used by
|
||||
// the glue flush to pick `<module>` vs `<top-level>` provisional
|
||||
// label (1A's `is_mod_decl` analog).
|
||||
let mut glue: Vec<(usize, u32, u32)> = Vec::new();
|
||||
|
||||
/// Walk preceding `comment` siblings to extend the unit's line range
|
||||
/// upward, folding leading doc / line comments into the unit.
|
||||
fn unit_start(n: &tree_sitter::Node) -> u32 {
|
||||
let mut start = n.start_position().row as u32 + 1;
|
||||
let mut prev = n.prev_sibling();
|
||||
while let Some(p) = prev {
|
||||
if p.kind() == "comment" {
|
||||
start = p.start_position().row as u32 + 1;
|
||||
prev = p.prev_sibling();
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
start
|
||||
}
|
||||
fn name_text<'a>(n: &tree_sitter::Node, src: &'a str) -> Option<&'a str> {
|
||||
n.child_by_field_name("name")
|
||||
.map(|c| &src[c.start_byte()..c.end_byte()])
|
||||
}
|
||||
/// Walk a class body, emitting one unit per `method_definition`.
|
||||
/// Class names already pushed onto `mod_path` by the caller, so
|
||||
/// method symbols come out as `<mod_prefix>.<Class>.<method>`.
|
||||
fn walk_class_body(
|
||||
body: tree_sitter::Node,
|
||||
src: &str,
|
||||
mod_prefix: &str,
|
||||
mod_path: &[String],
|
||||
units: &mut Vec<(String, u32, u32, bool)>,
|
||||
) {
|
||||
let mut cur = body.walk();
|
||||
for child in body.named_children(&mut cur) {
|
||||
if child.kind() == "method_definition" {
|
||||
if let Some(name) = name_text(&child, src) {
|
||||
let s = unit_start(&child);
|
||||
let e = child.end_position().row as u32 + 1;
|
||||
let sym = join_symbol(mod_prefix, mod_path, name);
|
||||
units.push((sym, s, e, true));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
fn walk(
|
||||
node: tree_sitter::Node,
|
||||
src: &str,
|
||||
mod_prefix: &str,
|
||||
mod_path: &[String],
|
||||
units: &mut Vec<(String, u32, u32, bool)>,
|
||||
glue: &mut Vec<(usize, u32, u32)>,
|
||||
) {
|
||||
let mut cur = node.walk();
|
||||
for child in node.named_children(&mut cur) {
|
||||
let s = unit_start(&child);
|
||||
let e = child.end_position().row as u32 + 1;
|
||||
match child.kind() {
|
||||
"function_declaration" => {
|
||||
if let Some(name) = name_text(&child, src) {
|
||||
glue.retain(|(_, gs, _)| *gs < s);
|
||||
flush_glue(glue, units, mod_prefix, mod_path);
|
||||
let sym = join_symbol(mod_prefix, mod_path, name);
|
||||
units.push((sym, s, e, true));
|
||||
}
|
||||
}
|
||||
"class_declaration" => {
|
||||
if let Some(name) = name_text(&child, src) {
|
||||
glue.retain(|(_, gs, _)| *gs < s);
|
||||
flush_glue(glue, units, mod_prefix, mod_path);
|
||||
let sym = join_symbol(mod_prefix, mod_path, name);
|
||||
units.push((sym, s, e, true));
|
||||
if let Some(body) = child.child_by_field_name("body") {
|
||||
let mut np = mod_path.to_vec();
|
||||
np.push(name.to_string());
|
||||
walk_class_body(body, src, mod_prefix, &np, units);
|
||||
}
|
||||
}
|
||||
}
|
||||
"export_statement" => {
|
||||
// Try field "declaration" first (export class /
|
||||
// function). If absent, fall back to "value" —
|
||||
// `export default function () {}` / `export default
|
||||
// class {}` expose the anonymous function_expression
|
||||
// / class under the `value` field (same grammar
|
||||
// quirk as TS 0.23).
|
||||
let outer_s = s; // includes `export ` prefix line
|
||||
let outer_e = e;
|
||||
if let Some(inner) = child.child_by_field_name("declaration") {
|
||||
let inner_kind = inner.kind();
|
||||
match inner_kind {
|
||||
"function_declaration" | "class_declaration" => {
|
||||
let name_opt = name_text(&inner, src).map(|s| s.to_string());
|
||||
if let Some(name) = name_opt {
|
||||
glue.retain(|(_, gs, _)| *gs < outer_s);
|
||||
flush_glue(glue, units, mod_prefix, mod_path);
|
||||
let sym = join_symbol(mod_prefix, mod_path, &name);
|
||||
units.push((sym, outer_s, outer_e, true));
|
||||
if inner_kind == "class_declaration" {
|
||||
if let Some(body) = inner.child_by_field_name("body") {
|
||||
let mut np = mod_path.to_vec();
|
||||
np.push(name);
|
||||
walk_class_body(body, src, mod_prefix, &np, units);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Defensive: `export default` with a
|
||||
// function_declaration that somehow
|
||||
// lacks `name`. Emit `default`.
|
||||
glue.retain(|(_, gs, _)| *gs < outer_s);
|
||||
flush_glue(glue, units, mod_prefix, mod_path);
|
||||
let sym = join_symbol(mod_prefix, mod_path, "default");
|
||||
units.push((sym, outer_s, outer_e, true));
|
||||
}
|
||||
}
|
||||
// `lexical_declaration` etc. wrapped in
|
||||
// export: treat as glue (assigned arrow
|
||||
// fns / consts don't get their own unit).
|
||||
_ => {
|
||||
glue.push((0, s, e));
|
||||
}
|
||||
}
|
||||
} else if let Some(value) = child.child_by_field_name("value") {
|
||||
// `export default <expr>`. We emit a unit only
|
||||
// for the function / class shapes (named or
|
||||
// anonymous); other value shapes are glue.
|
||||
match value.kind() {
|
||||
"function_expression"
|
||||
| "function_declaration"
|
||||
| "class"
|
||||
| "class_declaration" => {
|
||||
let name_opt = name_text(&value, src).map(|s| s.to_string());
|
||||
let leaf =
|
||||
name_opt.as_deref().unwrap_or("default").to_string();
|
||||
glue.retain(|(_, gs, _)| *gs < outer_s);
|
||||
flush_glue(glue, units, mod_prefix, mod_path);
|
||||
let sym = join_symbol(mod_prefix, mod_path, &leaf);
|
||||
units.push((sym, outer_s, outer_e, true));
|
||||
// Recurse into class body if we have one.
|
||||
if matches!(value.kind(), "class" | "class_declaration") {
|
||||
if let Some(body) = value.child_by_field_name("body") {
|
||||
let mut np = mod_path.to_vec();
|
||||
np.push(leaf);
|
||||
walk_class_body(body, src, mod_prefix, &np, units);
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
glue.push((0, s, e));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Bare `export { x };` / `export * from "..."` —
|
||||
// a re-export, glue with module-only flag set
|
||||
// (we have no `declaration` / `value` field for
|
||||
// it).
|
||||
glue.push((1, s, e));
|
||||
}
|
||||
}
|
||||
"import_statement" => {
|
||||
glue.push((1, s, e));
|
||||
}
|
||||
"lexical_declaration" | "variable_declaration" => {
|
||||
glue.push((0, s, e));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
flush_glue(glue, units, mod_prefix, mod_path);
|
||||
}
|
||||
fn flush_glue(
|
||||
glue: &mut Vec<(usize, u32, u32)>,
|
||||
units: &mut Vec<(String, u32, u32, bool)>,
|
||||
mod_prefix: &str,
|
||||
mod_path: &[String],
|
||||
) {
|
||||
if glue.is_empty() {
|
||||
return;
|
||||
}
|
||||
let s = glue.iter().map(|(_, a, _)| *a).min().unwrap();
|
||||
let e = glue.iter().map(|(_, _, b)| *b).max().unwrap();
|
||||
let only_module = glue.iter().all(|(is_mod, _, _)| *is_mod == 1);
|
||||
let label = if only_module { "<module>" } else { "<top-level>" };
|
||||
units.push((join_symbol(mod_prefix, mod_path, label), s, e, false));
|
||||
glue.clear();
|
||||
}
|
||||
|
||||
walk(
|
||||
tree.root_node(),
|
||||
source,
|
||||
mod_prefix,
|
||||
&[],
|
||||
&mut units,
|
||||
&mut glue,
|
||||
);
|
||||
|
||||
// `<module>` is correct only when the file produced no real unit.
|
||||
// Otherwise the import-only group becomes `<top-level>` (same
|
||||
// post-pass as 1A Gap 1 / Python / TS).
|
||||
let has_real_unit = units.iter().any(|(_, _, _, is_real)| *is_real);
|
||||
if has_real_unit {
|
||||
for (sym, _, _, is_real) in units.iter_mut() {
|
||||
if !*is_real && sym.ends_with("<module>") {
|
||||
let pre = &sym[..sym.len() - "<module>".len()];
|
||||
*sym = format!("{pre}<top-level>");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let total_lines = lines.len() as u32;
|
||||
let mut blocks = Vec::with_capacity(units.len());
|
||||
for (ordinal, (symbol, ls, le, _is_real)) in units.into_iter().enumerate() {
|
||||
let line_start = ls.max(1);
|
||||
let line_end = le.min(total_lines.max(1));
|
||||
let span = SourceSpan::Code {
|
||||
line_start,
|
||||
line_end,
|
||||
symbol: Some(symbol),
|
||||
lang: Some("javascript".to_string()),
|
||||
};
|
||||
let block_id = id_for_block(doc_id, "code", &[], ordinal as u32, &span);
|
||||
let code = lines[(line_start as usize - 1)..=(line_end as usize - 1)].join("\n");
|
||||
blocks.push(Block::Code(CodeBlock {
|
||||
common: CommonBlock {
|
||||
block_id,
|
||||
heading_path: Vec::new(),
|
||||
source_span: span,
|
||||
},
|
||||
lang: Some("javascript".to_string()),
|
||||
code,
|
||||
}));
|
||||
}
|
||||
Ok(blocks)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use kebab_core::{Block, MediaType, SourceSpan};
|
||||
|
||||
fn extract_fixture(workspace_path: &str) -> kebab_core::CanonicalDocument {
|
||||
let bytes = std::fs::read(
|
||||
concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/sample.js"),
|
||||
)
|
||||
.unwrap();
|
||||
let asset = crate::rust::tests_support::fixed_code_asset(workspace_path, "javascript");
|
||||
let cfg = kebab_core::ExtractConfig::default();
|
||||
let root = std::path::PathBuf::from("/tmp");
|
||||
let ctx = kebab_core::ExtractContext {
|
||||
asset: &asset,
|
||||
workspace_root: &root,
|
||||
config: &cfg,
|
||||
};
|
||||
JavascriptAstExtractor::new().extract(&ctx, &bytes).unwrap()
|
||||
}
|
||||
fn symbols(doc: &kebab_core::CanonicalDocument) -> Vec<String> {
|
||||
let mut s: Vec<String> = doc
|
||||
.blocks
|
||||
.iter()
|
||||
.filter_map(|b| match b {
|
||||
Block::Code(c) => match &c.common.source_span {
|
||||
SourceSpan::Code { symbol, lang, .. } => {
|
||||
assert_eq!(lang.as_deref(), Some("javascript"));
|
||||
symbol.clone()
|
||||
}
|
||||
_ => None,
|
||||
},
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
s.sort();
|
||||
s
|
||||
}
|
||||
#[test]
|
||||
fn extractor_supports_only_media_code_javascript() {
|
||||
let e = JavascriptAstExtractor::new();
|
||||
assert!(e.supports(&MediaType::Code("javascript".into())));
|
||||
assert!(!e.supports(&MediaType::Code("typescript".into())));
|
||||
assert!(!e.supports(&MediaType::Markdown));
|
||||
}
|
||||
#[test]
|
||||
fn js_units_match_design_3_4_symbols() {
|
||||
let doc = extract_fixture("src/sample.js");
|
||||
let syms = symbols(&doc);
|
||||
assert!(syms.iter().any(|s| s == "src/sample.add"), "got {syms:?}");
|
||||
assert!(syms.iter().any(|s| s == "src/sample.Retriever"));
|
||||
assert!(syms.iter().any(|s| s == "src/sample.Retriever.search"));
|
||||
assert!(syms.iter().any(|s| s == "src/sample.Retriever.create"));
|
||||
assert!(syms.iter().any(|s| s == "src/sample.default"));
|
||||
assert!(syms.iter().any(|s| s == "src/sample.<top-level>"));
|
||||
}
|
||||
#[test]
|
||||
fn jsx_via_js_grammar() {
|
||||
// tree-sitter-javascript handles .jsx via the same single grammar.
|
||||
let bytes = b"export function App() { return null; }\n";
|
||||
let asset = crate::rust::tests_support::fixed_code_asset("src/App.jsx", "javascript");
|
||||
let cfg = kebab_core::ExtractConfig::default();
|
||||
let root = std::path::PathBuf::from("/tmp");
|
||||
let ctx = kebab_core::ExtractContext {
|
||||
asset: &asset,
|
||||
workspace_root: &root,
|
||||
config: &cfg,
|
||||
};
|
||||
let doc = JavascriptAstExtractor::new().extract(&ctx, bytes).unwrap();
|
||||
let syms = symbols(&doc);
|
||||
assert!(syms.iter().any(|s| s == "src/App.App"), "got {syms:?}");
|
||||
}
|
||||
#[test]
|
||||
fn deterministic_across_runs() {
|
||||
let a = extract_fixture("src/sample.js");
|
||||
for _ in 0..30 {
|
||||
assert_eq!(extract_fixture("src/sample.js").blocks, a.blocks);
|
||||
}
|
||||
}
|
||||
|
||||
/// In tree-sitter-javascript, `decorator` is a CHILD of
|
||||
/// `method_definition` (stored in the `decorator` field), so
|
||||
/// `method_definition.start_row` already covers the decorator line
|
||||
/// without any sibling walk. Verify that the emitted unit already
|
||||
/// includes the decorator line and line_start is 2 (the @Log() line).
|
||||
#[test]
|
||||
fn js_class_method_decorator_already_folded_by_grammar() {
|
||||
// Line 1 (1-indexed): "class Foo {"
|
||||
// Line 2: " @Log()" <- decorator (child of method_definition in JS grammar)
|
||||
// Line 3: " bar() { return 1; }"
|
||||
// Line 4: "}"
|
||||
let bytes = b"class Foo {\n @Log()\n bar() { return 1; }\n}\n";
|
||||
let asset = crate::rust::tests_support::fixed_code_asset("src/foo.js", "javascript");
|
||||
let cfg = kebab_core::ExtractConfig::default();
|
||||
let root = std::path::PathBuf::from("/tmp");
|
||||
let ctx = kebab_core::ExtractContext {
|
||||
asset: &asset,
|
||||
workspace_root: &root,
|
||||
config: &cfg,
|
||||
};
|
||||
let doc = JavascriptAstExtractor::new().extract(&ctx, bytes).unwrap();
|
||||
|
||||
let bar_block = doc
|
||||
.blocks
|
||||
.iter()
|
||||
.find_map(|b| match b {
|
||||
Block::Code(c) => match &c.common.source_span {
|
||||
SourceSpan::Code { symbol, .. }
|
||||
if symbol.as_deref() == Some("src/foo.Foo.bar") =>
|
||||
{
|
||||
Some(c)
|
||||
}
|
||||
_ => None,
|
||||
},
|
||||
_ => None,
|
||||
})
|
||||
.expect("src/foo.Foo.bar block should be present");
|
||||
|
||||
// JS grammar: method_definition.start_row == decorator row, so
|
||||
// no sibling walk change needed -- decorator is already included.
|
||||
assert!(
|
||||
bar_block.code.contains("@Log()"),
|
||||
"JS method unit must include decorator (grammar folds it natively); got: {:?}",
|
||||
bar_block.code
|
||||
);
|
||||
match &bar_block.common.source_span {
|
||||
SourceSpan::Code { line_start, .. } => {
|
||||
assert_eq!(
|
||||
*line_start, 2,
|
||||
"JS line_start must cover the @Log() decorator line (got {line_start})"
|
||||
);
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
}
|
||||
121
crates/kebab-parse-code/src/lang.rs
Normal file
121
crates/kebab-parse-code/src/lang.rs
Normal file
@@ -0,0 +1,121 @@
|
||||
//! Canonical extension → language identifier mapping (spec §3.5).
|
||||
//!
|
||||
//! Lowercase canonical identifiers, matching tree-sitter parser conventions:
|
||||
//! `rust`, `python`, `typescript`, `javascript`, `go`, `java`, `kotlin`, `c`,
|
||||
//! `cpp`, `yaml`, `toml`, `json`, `shell`, `make`, `dockerfile`.
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
/// Returns the canonical language identifier for a given file path, or
|
||||
/// `None` if the extension / filename is not recognized.
|
||||
///
|
||||
/// Matching priority:
|
||||
/// 1. exact filename match (e.g. `Dockerfile`, `Makefile`)
|
||||
/// 2. lowercase extension match
|
||||
pub fn code_lang_for_path(path: &Path) -> Option<&'static str> {
|
||||
if let Some(name) = path.file_name().and_then(|n| n.to_str()) {
|
||||
match name {
|
||||
"Dockerfile" => return Some("dockerfile"),
|
||||
"Makefile" | "GNUmakefile" => return Some("make"),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
let ext = path.extension()?.to_str()?.to_ascii_lowercase();
|
||||
match ext.as_str() {
|
||||
"rs" => Some("rust"),
|
||||
"py" | "pyi" => Some("python"),
|
||||
"ts" | "tsx" => Some("typescript"),
|
||||
"js" | "mjs" | "cjs" | "jsx" => Some("javascript"),
|
||||
"go" => Some("go"),
|
||||
"java" => Some("java"),
|
||||
"kt" | "kts" => Some("kotlin"),
|
||||
"c" | "h" => Some("c"),
|
||||
"cpp" | "cc" | "cxx" | "hpp" | "hh" | "hxx" => Some("cpp"),
|
||||
"yaml" | "yml" => Some("yaml"),
|
||||
"toml" => Some("toml"),
|
||||
"json" => Some("json"),
|
||||
"sh" | "bash" | "zsh" => Some("shell"),
|
||||
"mk" => Some("make"),
|
||||
"dockerfile" => Some("dockerfile"),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// p10-1B: workspace-relative Python file path → dotted module-path prefix.
|
||||
/// See plan §Task C for the exact rules + tasks/p10/p10-1b for the §3.4
|
||||
/// design contract.
|
||||
///
|
||||
/// Stripped source-roots: `src/`, `lib/`, and `crates/<crate>/src/`.
|
||||
/// `tests/`, `examples/`, and `benches/` are intentionally NOT stripped —
|
||||
/// they appear in test/example/bench namespaces and dropping them would
|
||||
/// conflate identical symbol names across conventional Python directories
|
||||
/// (e.g. `tests/test_foo.py` → `tests.test_foo`, not `test_foo`).
|
||||
pub fn module_path_for_python(workspace_path: &str) -> String {
|
||||
let mut p: &str = workspace_path;
|
||||
if let Some(rest) = p.strip_prefix("crates/") {
|
||||
if let Some(slash) = rest.find('/') {
|
||||
let after = &rest[slash + 1..];
|
||||
if let Some(stripped) = after.strip_prefix("src/") {
|
||||
p = stripped;
|
||||
}
|
||||
}
|
||||
} else if let Some(stripped) = p.strip_prefix("src/") {
|
||||
p = stripped;
|
||||
} else if let Some(stripped) = p.strip_prefix("lib/") {
|
||||
p = stripped;
|
||||
}
|
||||
let p = match p.strip_suffix(".py") {
|
||||
Some(s) => s,
|
||||
None => p.strip_suffix(".pyi").unwrap_or(p),
|
||||
};
|
||||
let p = if let Some(parent) = p.strip_suffix("/__init__") {
|
||||
parent
|
||||
} else if p == "__init__" {
|
||||
""
|
||||
} else {
|
||||
p
|
||||
};
|
||||
p.replace('/', ".")
|
||||
}
|
||||
|
||||
/// p10-1B: workspace-relative TS/JS file path → path-style prefix
|
||||
/// (no slash replacement, no source-root strip). See plan §Task C.
|
||||
pub fn module_path_for_tsjs(workspace_path: &str) -> String {
|
||||
let p = workspace_path;
|
||||
for ext in [".tsx", ".ts", ".jsx", ".mjs", ".cjs", ".js"] {
|
||||
if let Some(stripped) = p.strip_suffix(ext) {
|
||||
return stripped.to_string();
|
||||
}
|
||||
}
|
||||
p.to_string()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn module_path_for_python_strips_src_roots_and_extensions() {
|
||||
assert_eq!(module_path_for_python("kebab_eval/metrics.py"), "kebab_eval.metrics");
|
||||
assert_eq!(module_path_for_python("kebab_eval/__init__.py"), "kebab_eval");
|
||||
assert_eq!(module_path_for_python("src/foo/bar.py"), "foo.bar");
|
||||
assert_eq!(module_path_for_python("crates/x/src/foo/bar.py"), "foo.bar");
|
||||
assert_eq!(module_path_for_python("a/b/c.pyi"), "a.b.c");
|
||||
assert_eq!(module_path_for_python("standalone.py"), "standalone");
|
||||
assert_eq!(module_path_for_python("src/__init__.py"), "");
|
||||
// `tests/` is NOT a stripped source-root — it is preserved as
|
||||
// part of the module path so test symbols stay namespaced.
|
||||
assert_eq!(module_path_for_python("tests/test_foo.py"), "tests.test_foo");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn module_path_for_tsjs_keeps_slashes_and_strips_ext() {
|
||||
for ext in ["ts", "tsx", "js", "jsx", "mjs", "cjs"] {
|
||||
let p = format!("src/search/retriever/Retriever.{ext}");
|
||||
assert_eq!(module_path_for_tsjs(&p), "src/search/retriever/Retriever");
|
||||
}
|
||||
assert_eq!(module_path_for_tsjs("foo.ts"), "foo");
|
||||
assert_eq!(module_path_for_tsjs("a/b/c.ts"), "a/b/c");
|
||||
assert_eq!(module_path_for_tsjs("packages/x/src/Foo.ts"), "packages/x/src/Foo");
|
||||
}
|
||||
}
|
||||
31
crates/kebab-parse-code/src/lib.rs
Normal file
31
crates/kebab-parse-code/src/lib.rs
Normal file
@@ -0,0 +1,31 @@
|
||||
//! `kebab-parse-code` — language-aware parsing for code corpora.
|
||||
//!
|
||||
//! Phase 1A-1 ships infrastructure only:
|
||||
//!
|
||||
//! - [`lang::code_lang_for_path`] — extension → language identifier.
|
||||
//! - [`repo::detect_repo`] — `.git/` walk-up → repo / branch / commit metadata.
|
||||
//! - [`skip::is_generated_file`] / [`skip::is_oversized`] — pre-ingest skip
|
||||
//! helpers consulted by `kebab-source-fs`.
|
||||
//! - [`skip::BUILTIN_BLACKLIST`] — 6-entry safety-net pattern list.
|
||||
//!
|
||||
//! Per-language parser modules (`rust`, `python`, `typescript`, …) land in
|
||||
//! later phases (1A-2 onwards). The crate boundary follows other
|
||||
//! `kebab-parse-*` crates per design §8: must NOT depend on store / embed
|
||||
//! / llm / rag.
|
||||
|
||||
pub mod javascript;
|
||||
pub mod lang;
|
||||
pub mod python;
|
||||
pub mod repo;
|
||||
pub mod rust;
|
||||
pub(crate) mod scaffold;
|
||||
pub mod skip;
|
||||
pub mod typescript;
|
||||
|
||||
pub use javascript::{PARSER_VERSION as JS_PARSER_VERSION, JavascriptAstExtractor};
|
||||
pub use lang::{code_lang_for_path, module_path_for_python, module_path_for_tsjs};
|
||||
pub use python::{PARSER_VERSION as PYTHON_PARSER_VERSION, PythonAstExtractor};
|
||||
pub use repo::{RepoMeta, detect_repo};
|
||||
pub use rust::{PARSER_VERSION as RUST_PARSER_VERSION, RustAstExtractor};
|
||||
pub use skip::{BUILTIN_BLACKLIST, is_generated_file, is_oversized};
|
||||
pub use typescript::{PARSER_VERSION as TS_PARSER_VERSION, TypescriptAstExtractor};
|
||||
437
crates/kebab-parse-code/src/python.rs
Normal file
437
crates/kebab-parse-code/src/python.rs
Normal file
@@ -0,0 +1,437 @@
|
||||
//! `kebab-parse-code::python` — tree-sitter Python AST extractor (P10-1B Task E).
|
||||
//!
|
||||
//! Implements [`kebab_core::Extractor`] for [`MediaType::Code("python")`].
|
||||
//! Walks the tree-sitter parse tree and emits one [`Block::Code`] per
|
||||
//! top-level AST semantic unit (free fn, class, each method, recursively
|
||||
//! per nested class), each carrying [`SourceSpan::Code`] with the unit's
|
||||
//! dotted self-reference symbol path prefixed by `module_path_for_python`
|
||||
//! (design §3.4). Glue declarations (`import` / `import from` /
|
||||
//! `expression_statement` / `assignment` / `global_statement` /
|
||||
//! `future_import_statement`) collapse into one grouped `<top-level>`
|
||||
//! (or `<module>`) unit.
|
||||
//!
|
||||
//! Decorators are folded into the decorated unit's line range via the
|
||||
//! `decorated_definition` unwrap arm (analog of the Rust `attribute_item`
|
||||
//! re-absorption in 1A — see §9.1).
|
||||
//!
|
||||
//! Scope follows 1A: AST unit extraction + dotted symbol paths + line
|
||||
//! ranges. Per design §3.4 / §9.1 / §9 versioning.
|
||||
|
||||
use anyhow::Result;
|
||||
use kebab_core::{
|
||||
Block, CanonicalDocument, CodeBlock, CommonBlock, Extractor, Lang, MediaType, Metadata,
|
||||
ParserVersion, Provenance, ProvenanceEvent, ProvenanceKind, SourceSpan, SourceType, TrustLevel,
|
||||
id_for_block, id_for_doc,
|
||||
};
|
||||
use serde_json::Map;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
use crate::scaffold::{filename_from_workspace_path, join_symbol, strip_extension};
|
||||
|
||||
pub const PARSER_VERSION: &str = "code-python-v1";
|
||||
|
||||
/// Python AST extractor. Per-unit blocks via tree-sitter-python 0.25
|
||||
/// (`LANGUAGE: LanguageFn`) parsed by tree-sitter 0.26.
|
||||
pub struct PythonAstExtractor;
|
||||
|
||||
impl PythonAstExtractor {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for PythonAstExtractor {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl Extractor for PythonAstExtractor {
|
||||
fn supports(&self, m: &MediaType) -> bool {
|
||||
matches!(m, MediaType::Code(l) if l == "python")
|
||||
}
|
||||
|
||||
fn parser_version(&self) -> ParserVersion {
|
||||
ParserVersion(PARSER_VERSION.to_string())
|
||||
}
|
||||
|
||||
fn extract(
|
||||
&self,
|
||||
ctx: &kebab_core::ExtractContext<'_>,
|
||||
bytes: &[u8],
|
||||
) -> Result<CanonicalDocument> {
|
||||
let asset = ctx.asset;
|
||||
if !self.supports(&asset.media_type) {
|
||||
anyhow::bail!(
|
||||
"kebab-parse-code: unsupported media_type for PythonAstExtractor: {:?}",
|
||||
asset.media_type
|
||||
);
|
||||
}
|
||||
|
||||
let parser_version = self.parser_version();
|
||||
let doc_id = id_for_doc(&asset.workspace_path, &asset.asset_id, &parser_version);
|
||||
|
||||
let source = String::from_utf8(bytes.to_vec()).map_err(|e| {
|
||||
anyhow::anyhow!("kebab-parse-code: Python source is not valid UTF-8: {e}")
|
||||
})?;
|
||||
|
||||
let mod_prefix = crate::lang::module_path_for_python(&asset.workspace_path.0);
|
||||
let blocks = build_blocks(&source, &doc_id, &mod_prefix)?;
|
||||
let unit_count = blocks.len() as u32;
|
||||
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let mut events: Vec<ProvenanceEvent> = Vec::with_capacity(2);
|
||||
events.push(ProvenanceEvent {
|
||||
at: asset.discovered_at,
|
||||
agent: "kb-source-fs".to_string(),
|
||||
kind: ProvenanceKind::Discovered,
|
||||
note: None,
|
||||
});
|
||||
events.push(ProvenanceEvent {
|
||||
at: now,
|
||||
agent: "kb-parse-code".to_string(),
|
||||
kind: ProvenanceKind::Parsed,
|
||||
note: Some(format!(
|
||||
"parser_version={}; unit_count={}",
|
||||
parser_version.0, unit_count
|
||||
)),
|
||||
});
|
||||
|
||||
let title = {
|
||||
let fname = filename_from_workspace_path(&asset.workspace_path.0);
|
||||
strip_extension(&fname)
|
||||
};
|
||||
|
||||
// Resolve the file's absolute path for repo detection. If the
|
||||
// source URI carries a relative path, anchor it at the workspace
|
||||
// root so the `.git/` walk-up starts from the right place.
|
||||
let abs_path = match &asset.source_uri {
|
||||
kebab_core::SourceUri::File(p) => {
|
||||
if p.is_absolute() {
|
||||
p.clone()
|
||||
} else {
|
||||
ctx.workspace_root.join(p)
|
||||
}
|
||||
}
|
||||
kebab_core::SourceUri::Kb(_) => ctx.workspace_root.to_path_buf(),
|
||||
};
|
||||
let (repo, git_branch, git_commit) = match crate::repo::detect_repo(&abs_path) {
|
||||
Some(r) => (Some(r.name), r.branch, r.commit),
|
||||
None => (None, None, None),
|
||||
};
|
||||
|
||||
let metadata = Metadata {
|
||||
aliases: Vec::new(),
|
||||
tags: Vec::new(),
|
||||
created_at: asset.discovered_at,
|
||||
updated_at: asset.discovered_at,
|
||||
source_type: SourceType::Note,
|
||||
trust_level: TrustLevel::Primary,
|
||||
user_id_alias: None,
|
||||
user: Map::new(),
|
||||
repo,
|
||||
git_branch,
|
||||
git_commit,
|
||||
code_lang: Some("python".to_string()),
|
||||
};
|
||||
|
||||
tracing::debug!(
|
||||
target: "kebab-parse-code",
|
||||
"extracted Python doc_id={} workspace_path={} units={}",
|
||||
doc_id.0,
|
||||
asset.workspace_path.0,
|
||||
unit_count
|
||||
);
|
||||
|
||||
Ok(CanonicalDocument {
|
||||
doc_id,
|
||||
source_asset_id: asset.asset_id.clone(),
|
||||
workspace_path: asset.workspace_path.clone(),
|
||||
title,
|
||||
lang: Lang("und".to_string()),
|
||||
blocks,
|
||||
metadata,
|
||||
provenance: Provenance { events },
|
||||
parser_version,
|
||||
schema_version: 1,
|
||||
doc_version: 1,
|
||||
last_chunker_version: None,
|
||||
last_embedding_version: None,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn build_blocks(
|
||||
source: &str,
|
||||
doc_id: &kebab_core::DocumentId,
|
||||
mod_prefix: &str,
|
||||
) -> anyhow::Result<Vec<kebab_core::Block>> {
|
||||
let mut parser = tree_sitter::Parser::new();
|
||||
parser
|
||||
.set_language(&tree_sitter_python::LANGUAGE.into())
|
||||
.map_err(|e| anyhow::anyhow!("set tree-sitter-python language: {e}"))?;
|
||||
let tree = parser
|
||||
.parse(source.as_bytes(), None)
|
||||
.ok_or_else(|| anyhow::anyhow!("tree-sitter failed to parse Python source"))?;
|
||||
let lines: Vec<&str> = source.split('\n').collect();
|
||||
|
||||
// units: (symbol, line_start, line_end, is_real_semantic_unit).
|
||||
// Glue groups are pushed with a sentinel symbol + is_real=false so a
|
||||
// post-pass can decide `<module>` vs `<top-level>` (same algorithm
|
||||
// as 1A Gap 1).
|
||||
let mut units: Vec<(String, u32, u32, bool)> = Vec::new();
|
||||
// (is_import 0/1, s, e). `is_import` flags `import_statement` /
|
||||
// `import_from_statement` / `future_import_statement` — used by the
|
||||
// glue flush to pick `<module>` vs `<top-level>` provisional label
|
||||
// (1A's `is_mod_decl` analog).
|
||||
let mut glue: Vec<(usize, u32, u32)> = Vec::new();
|
||||
|
||||
fn node_name<'a>(n: &tree_sitter::Node, src: &'a str) -> Option<&'a str> {
|
||||
n.child_by_field_name("name")
|
||||
.map(|c| &src[c.start_byte()..c.end_byte()])
|
||||
}
|
||||
/// Walk preceding `comment` siblings to extend the unit's line range
|
||||
/// upward, folding leading doc / line comments into the unit. Note
|
||||
/// that Python decorators are NOT preceding siblings — they live
|
||||
/// INSIDE a `decorated_definition` parent — so they are handled by
|
||||
/// the unwrap arm below, not here.
|
||||
fn unit_start(n: &tree_sitter::Node) -> u32 {
|
||||
let mut start = n.start_position().row as u32 + 1;
|
||||
let mut prev = n.prev_sibling();
|
||||
while let Some(p) = prev {
|
||||
if p.kind() == "comment" {
|
||||
start = p.start_position().row as u32 + 1;
|
||||
prev = p.prev_sibling();
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
start
|
||||
}
|
||||
fn walk(
|
||||
node: tree_sitter::Node,
|
||||
src: &str,
|
||||
mod_prefix: &str,
|
||||
mod_path: &[String],
|
||||
units: &mut Vec<(String, u32, u32, bool)>,
|
||||
glue: &mut Vec<(usize, u32, u32)>,
|
||||
) {
|
||||
let mut cur = node.walk();
|
||||
for child in node.named_children(&mut cur) {
|
||||
// Default unit line range — overridden by the
|
||||
// `decorated_definition` unwrap arm so decorator lines are
|
||||
// included.
|
||||
let s = unit_start(&child);
|
||||
let e = child.end_position().row as u32 + 1;
|
||||
match child.kind() {
|
||||
"function_definition" => {
|
||||
if let Some(name) = node_name(&child, src) {
|
||||
glue.retain(|(_, gs, _)| *gs < s);
|
||||
flush_glue(glue, units, mod_prefix, mod_path);
|
||||
let sym = join_symbol(mod_prefix, mod_path, name);
|
||||
units.push((sym, s, e, true));
|
||||
}
|
||||
}
|
||||
"class_definition" => {
|
||||
if let Some(name) = node_name(&child, src) {
|
||||
glue.retain(|(_, gs, _)| *gs < s);
|
||||
flush_glue(glue, units, mod_prefix, mod_path);
|
||||
let sym = join_symbol(mod_prefix, mod_path, name);
|
||||
units.push((sym, s, e, true));
|
||||
// Recurse into the class body with the class
|
||||
// name pushed onto mod_path; methods become
|
||||
// `<...>.<ClassName>.<method>` and nested
|
||||
// classes recurse further with both names.
|
||||
if let Some(body) = child.child_by_field_name("body") {
|
||||
let mut np = mod_path.to_vec();
|
||||
np.push(name.to_string());
|
||||
walk(body, src, mod_prefix, &np, units, glue);
|
||||
debug_assert!(
|
||||
glue.is_empty(),
|
||||
"inner walk must flush its glue before returning"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
"decorated_definition" => {
|
||||
// Unwrap: the inner definition supplies the symbol
|
||||
// name, but the unit's line range comes from the
|
||||
// OUTER `decorated_definition` so decorator lines
|
||||
// are folded in (analog of `attribute_item`
|
||||
// re-absorption in 1A — see plan §Task E note (b)).
|
||||
if let Some(inner) = child.child_by_field_name("definition") {
|
||||
let outer_s = s; // already includes decorators
|
||||
let outer_e = e;
|
||||
match inner.kind() {
|
||||
"function_definition" => {
|
||||
if let Some(name) = node_name(&inner, src) {
|
||||
glue.retain(|(_, gs, _)| *gs < outer_s);
|
||||
flush_glue(glue, units, mod_prefix, mod_path);
|
||||
let sym = join_symbol(mod_prefix, mod_path, name);
|
||||
units.push((sym, outer_s, outer_e, true));
|
||||
}
|
||||
}
|
||||
"class_definition" => {
|
||||
if let Some(name) = node_name(&inner, src) {
|
||||
glue.retain(|(_, gs, _)| *gs < outer_s);
|
||||
flush_glue(glue, units, mod_prefix, mod_path);
|
||||
let sym = join_symbol(mod_prefix, mod_path, name);
|
||||
units.push((sym, outer_s, outer_e, true));
|
||||
if let Some(body) = inner.child_by_field_name("body") {
|
||||
let mut np = mod_path.to_vec();
|
||||
np.push(name.to_string());
|
||||
walk(body, src, mod_prefix, &np, units, glue);
|
||||
debug_assert!(
|
||||
glue.is_empty(),
|
||||
"inner walk must flush its glue before returning"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
"import_statement" | "import_from_statement" | "future_import_statement" => {
|
||||
glue.push((1, s, e));
|
||||
}
|
||||
"expression_statement" | "assignment" | "global_statement" => {
|
||||
glue.push((0, s, e));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
flush_glue(glue, units, mod_prefix, mod_path);
|
||||
}
|
||||
fn flush_glue(
|
||||
glue: &mut Vec<(usize, u32, u32)>,
|
||||
units: &mut Vec<(String, u32, u32, bool)>,
|
||||
mod_prefix: &str,
|
||||
mod_path: &[String],
|
||||
) {
|
||||
if glue.is_empty() {
|
||||
return;
|
||||
}
|
||||
let s = glue.iter().map(|(_, a, _)| *a).min().unwrap();
|
||||
let e = glue.iter().map(|(_, _, b)| *b).max().unwrap();
|
||||
// Provisional label: `<module>` only if the group is exclusively
|
||||
// imports (1A's `only_mod_decls` analog). The post-pass below
|
||||
// demotes any `<module>` to `<top-level>` if the file produced
|
||||
// any real unit.
|
||||
let only_imports = glue.iter().all(|(is_import, _, _)| *is_import == 1);
|
||||
let label = if only_imports { "<module>" } else { "<top-level>" };
|
||||
units.push((join_symbol(mod_prefix, mod_path, label), s, e, false));
|
||||
glue.clear();
|
||||
}
|
||||
|
||||
walk(tree.root_node(), source, mod_prefix, &[], &mut units, &mut glue);
|
||||
|
||||
// `<module>` is correct only when the file produced no real unit.
|
||||
// Otherwise the import-only group becomes `<top-level>` (same
|
||||
// algorithm as 1A Gap 1). Match on the suffix so a class-nested
|
||||
// glue group (which doesn't exist in current Python AST but is
|
||||
// future-proofed) still demotes correctly.
|
||||
let has_real_unit = units.iter().any(|(_, _, _, is_real)| *is_real);
|
||||
if has_real_unit {
|
||||
for (sym, _, _, is_real) in units.iter_mut() {
|
||||
if !*is_real && sym.ends_with("<module>") {
|
||||
let pre = &sym[..sym.len() - "<module>".len()];
|
||||
*sym = format!("{pre}<top-level>");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let total_lines = lines.len() as u32;
|
||||
let mut blocks = Vec::with_capacity(units.len());
|
||||
for (ordinal, (symbol, ls, le, _is_real)) in units.into_iter().enumerate() {
|
||||
let line_start = ls.max(1);
|
||||
let line_end = le.min(total_lines.max(1));
|
||||
let span = SourceSpan::Code {
|
||||
line_start,
|
||||
line_end,
|
||||
symbol: Some(symbol),
|
||||
lang: Some("python".to_string()),
|
||||
};
|
||||
let block_id = id_for_block(doc_id, "code", &[], ordinal as u32, &span);
|
||||
let code = lines[(line_start as usize - 1)..=(line_end as usize - 1)].join("\n");
|
||||
blocks.push(Block::Code(CodeBlock {
|
||||
common: CommonBlock {
|
||||
block_id,
|
||||
heading_path: Vec::new(),
|
||||
source_span: span,
|
||||
},
|
||||
lang: Some("python".to_string()),
|
||||
code,
|
||||
}));
|
||||
}
|
||||
Ok(blocks)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use kebab_core::{Block, MediaType, SourceSpan};
|
||||
|
||||
fn extract_fixture() -> kebab_core::CanonicalDocument {
|
||||
let bytes = std::fs::read(
|
||||
concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/sample.py"),
|
||||
)
|
||||
.unwrap();
|
||||
let asset = crate::rust::tests_support::fixed_code_asset(
|
||||
"kebab_eval/metrics.py", "python",
|
||||
);
|
||||
let cfg = kebab_core::ExtractConfig::default();
|
||||
let root = std::path::PathBuf::from("/tmp");
|
||||
let ctx = kebab_core::ExtractContext {
|
||||
asset: &asset, workspace_root: &root, config: &cfg,
|
||||
};
|
||||
PythonAstExtractor::new().extract(&ctx, &bytes).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extractor_supports_only_media_code_python() {
|
||||
let e = PythonAstExtractor::new();
|
||||
assert!(e.supports(&MediaType::Code("python".into())));
|
||||
assert!(!e.supports(&MediaType::Code("rust".into())));
|
||||
assert!(!e.supports(&MediaType::Markdown));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn python_units_carry_module_prefixed_symbols() {
|
||||
let doc = extract_fixture();
|
||||
let mut syms: Vec<String> = doc.blocks.iter().map(|b| match b {
|
||||
Block::Code(c) => match &c.common.source_span {
|
||||
SourceSpan::Code { symbol, lang, .. } => {
|
||||
assert_eq!(lang.as_deref(), Some("python"));
|
||||
symbol.clone().unwrap()
|
||||
}
|
||||
_ => panic!("expected SourceSpan::Code"),
|
||||
},
|
||||
other => panic!("expected Block::Code, got {other:?}"),
|
||||
}).collect();
|
||||
syms.sort();
|
||||
assert!(syms.iter().any(|s| s == "kebab_eval.metrics.free"));
|
||||
assert!(syms.iter().any(|s| s == "kebab_eval.metrics.Foo"));
|
||||
assert!(syms.iter().any(|s| s == "kebab_eval.metrics.Foo.double"));
|
||||
assert!(syms.iter().any(|s| s == "kebab_eval.metrics.Foo.name"));
|
||||
assert!(syms.iter().any(|s| s == "kebab_eval.metrics.Outer"));
|
||||
assert!(syms.iter().any(|s| s == "kebab_eval.metrics.Outer.Inner"));
|
||||
assert!(syms.iter().any(|s| s == "kebab_eval.metrics.Outer.Inner.helper"));
|
||||
assert!(syms.iter().any(|s| s == "kebab_eval.metrics.with_decorator"));
|
||||
assert!(syms.iter().any(|s| s == "kebab_eval.metrics.<top-level>"));
|
||||
// The `@no_type_check` decorator on `free` is folded into its
|
||||
// unit's line range (decorated_definition unwrap).
|
||||
let free_src = doc.blocks.iter().find_map(|b| match b {
|
||||
Block::Code(c) if matches!(&c.common.source_span,
|
||||
SourceSpan::Code{symbol,..} if symbol.as_deref()==Some("kebab_eval.metrics.free")) => Some(c.code.clone()),
|
||||
_ => None,
|
||||
}).unwrap();
|
||||
assert!(free_src.contains("@no_type_check"), "decorator folded in: {free_src}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deterministic_across_runs() {
|
||||
let a = extract_fixture();
|
||||
for _ in 0..50 { assert_eq!(extract_fixture().blocks, a.blocks); }
|
||||
}
|
||||
}
|
||||
61
crates/kebab-parse-code/src/repo.rs
Normal file
61
crates/kebab-parse-code/src/repo.rs
Normal file
@@ -0,0 +1,61 @@
|
||||
//! Git repo auto-detection (spec §5.1).
|
||||
//!
|
||||
//! Walks up from `path` looking for a `.git/` directory. If found, reads
|
||||
//! repo dir name, current branch, and HEAD commit using `gix` (pure Rust;
|
||||
//! no `git` binary on PATH required).
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct RepoMeta {
|
||||
pub name: String,
|
||||
pub branch: Option<String>,
|
||||
pub commit: Option<String>,
|
||||
}
|
||||
|
||||
/// Walk up from `path` until a `.git/` directory is found. Returns repo
|
||||
/// metadata, or `None` if no repo boundary is reached before the filesystem
|
||||
/// root.
|
||||
///
|
||||
/// - `name`: directory name containing `.git/`.
|
||||
/// - `branch`: current HEAD branch, or `"detached"` if detached HEAD, or
|
||||
/// `None` if branch can't be read.
|
||||
/// - `commit`: 40-hex commit SHA at HEAD, or `None` if empty repo / read
|
||||
/// failure.
|
||||
///
|
||||
/// `.git/` as a file (worktree marker / submodule) returns `None` for
|
||||
/// `branch` and `commit` and falls back to the parent dir name for `name`.
|
||||
pub fn detect_repo(path: &Path) -> Option<RepoMeta> {
|
||||
let mut cur = if path.is_dir() { path } else { path.parent()? };
|
||||
loop {
|
||||
let dotgit = cur.join(".git");
|
||||
if dotgit.is_dir() {
|
||||
let name = cur.file_name()?.to_string_lossy().into_owned();
|
||||
let (branch, commit) = read_head(cur);
|
||||
return Some(RepoMeta { name, branch, commit });
|
||||
} else if dotgit.is_file() {
|
||||
let name = cur.file_name()?.to_string_lossy().into_owned();
|
||||
return Some(RepoMeta { name, branch: None, commit: None });
|
||||
}
|
||||
cur = cur.parent()?;
|
||||
}
|
||||
}
|
||||
|
||||
fn read_head(repo_dir: &Path) -> (Option<String>, Option<String>) {
|
||||
match gix::open(repo_dir) {
|
||||
Ok(repo) => {
|
||||
let branch = repo
|
||||
.head_name()
|
||||
.ok()
|
||||
.flatten()
|
||||
.map(|n| n.shorten().to_string())
|
||||
.or_else(|| Some("detached".to_string()));
|
||||
let commit = repo
|
||||
.head_id()
|
||||
.ok()
|
||||
.map(|id| id.to_string());
|
||||
(branch, commit)
|
||||
}
|
||||
Err(_) => (None, None),
|
||||
}
|
||||
}
|
||||
545
crates/kebab-parse-code/src/rust.rs
Normal file
545
crates/kebab-parse-code/src/rust.rs
Normal file
@@ -0,0 +1,545 @@
|
||||
//! `kebab-parse-code::rust` — tree-sitter Rust AST extractor (P10-1A-2).
|
||||
//!
|
||||
//! Implements [`kebab_core::Extractor`] for [`MediaType::Code("rust")`].
|
||||
//! Walks the tree-sitter parse tree and emits one [`Block::Code`] per
|
||||
//! top-level AST semantic unit (free fn, type, trait, macro, each impl
|
||||
//! method, recursively per module), each carrying [`SourceSpan::Code`]
|
||||
//! with the unit's self-reference symbol path (design §3.4). Glue
|
||||
//! declarations (`use` / `const` / `static` / bodyless `mod` / top-level
|
||||
//! attributes / macro invocations) collapse into one grouped
|
||||
//! `<top-level>` (or `<module>`) unit.
|
||||
//!
|
||||
//! Doc comments and attributes immediately preceding an item are folded
|
||||
//! into that item's line range (design §9.1 "선언 + doc comment").
|
||||
//!
|
||||
//! Scope is intentionally narrow: AST unit extraction + symbol paths +
|
||||
//! line ranges for Rust. The `CanonicalDocument` scaffold mirrors
|
||||
//! `kebab-parse-pdf`. Per design §3.4 / §9.1 / §9 versioning.
|
||||
//!
|
||||
//! Edge cases: a Rust file consisting solely of comments / whitespace
|
||||
//! (no fn / type / impl / mod / glue items) yields zero blocks → zero
|
||||
//! chunks → not surfaced in search. Safe (no panic) and consistent with
|
||||
//! "an empty page produces no chunks" in `pdf-page-v1`.
|
||||
|
||||
use anyhow::Result;
|
||||
use kebab_core::{
|
||||
Block, CanonicalDocument, CodeBlock, CommonBlock, Extractor, Lang, MediaType, Metadata,
|
||||
ParserVersion, Provenance, ProvenanceEvent, ProvenanceKind, SourceSpan, SourceType, TrustLevel,
|
||||
id_for_block, id_for_doc,
|
||||
};
|
||||
use serde_json::Map;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
use crate::scaffold::{filename_from_workspace_path, strip_extension};
|
||||
|
||||
pub const PARSER_VERSION: &str = "code-rust-v1";
|
||||
|
||||
/// Rust AST extractor. Per-unit blocks via tree-sitter-rust 0.24
|
||||
/// (`LANGUAGE: LanguageFn`) parsed by tree-sitter 0.26.
|
||||
pub struct RustAstExtractor;
|
||||
|
||||
impl RustAstExtractor {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for RustAstExtractor {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl Extractor for RustAstExtractor {
|
||||
fn supports(&self, m: &MediaType) -> bool {
|
||||
matches!(m, MediaType::Code(l) if l == "rust")
|
||||
}
|
||||
|
||||
fn parser_version(&self) -> ParserVersion {
|
||||
ParserVersion(PARSER_VERSION.to_string())
|
||||
}
|
||||
|
||||
fn extract(
|
||||
&self,
|
||||
ctx: &kebab_core::ExtractContext<'_>,
|
||||
bytes: &[u8],
|
||||
) -> Result<CanonicalDocument> {
|
||||
let asset = ctx.asset;
|
||||
if !self.supports(&asset.media_type) {
|
||||
anyhow::bail!(
|
||||
"kebab-parse-code: unsupported media_type for RustAstExtractor: {:?}",
|
||||
asset.media_type
|
||||
);
|
||||
}
|
||||
|
||||
let parser_version = self.parser_version();
|
||||
let doc_id = id_for_doc(&asset.workspace_path, &asset.asset_id, &parser_version);
|
||||
|
||||
let source = String::from_utf8(bytes.to_vec()).map_err(|e| {
|
||||
anyhow::anyhow!("kebab-parse-code: Rust source is not valid UTF-8: {e}")
|
||||
})?;
|
||||
|
||||
let blocks = build_blocks(&source, &doc_id)?;
|
||||
let unit_count = blocks.len() as u32;
|
||||
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let mut events: Vec<ProvenanceEvent> = Vec::with_capacity(2);
|
||||
events.push(ProvenanceEvent {
|
||||
at: asset.discovered_at,
|
||||
agent: "kb-source-fs".to_string(),
|
||||
kind: ProvenanceKind::Discovered,
|
||||
note: None,
|
||||
});
|
||||
events.push(ProvenanceEvent {
|
||||
at: now,
|
||||
agent: "kb-parse-code".to_string(),
|
||||
kind: ProvenanceKind::Parsed,
|
||||
note: Some(format!(
|
||||
"parser_version={}; unit_count={}",
|
||||
parser_version.0, unit_count
|
||||
)),
|
||||
});
|
||||
|
||||
let title = {
|
||||
let fname = filename_from_workspace_path(&asset.workspace_path.0);
|
||||
strip_extension(&fname)
|
||||
};
|
||||
|
||||
// Resolve the file's absolute path for repo detection. If the
|
||||
// source URI carries a relative path, anchor it at the workspace
|
||||
// root so the `.git/` walk-up starts from the right place.
|
||||
let abs_path = match &asset.source_uri {
|
||||
kebab_core::SourceUri::File(p) => {
|
||||
if p.is_absolute() {
|
||||
p.clone()
|
||||
} else {
|
||||
ctx.workspace_root.join(p)
|
||||
}
|
||||
}
|
||||
kebab_core::SourceUri::Kb(_) => ctx.workspace_root.to_path_buf(),
|
||||
};
|
||||
let (repo, git_branch, git_commit) = match crate::repo::detect_repo(&abs_path) {
|
||||
Some(r) => (Some(r.name), r.branch, r.commit),
|
||||
None => (None, None, None),
|
||||
};
|
||||
|
||||
let metadata = Metadata {
|
||||
aliases: Vec::new(),
|
||||
tags: Vec::new(),
|
||||
created_at: asset.discovered_at,
|
||||
updated_at: asset.discovered_at,
|
||||
source_type: SourceType::Note,
|
||||
trust_level: TrustLevel::Primary,
|
||||
user_id_alias: None,
|
||||
user: Map::new(),
|
||||
repo,
|
||||
git_branch,
|
||||
git_commit,
|
||||
code_lang: Some("rust".to_string()),
|
||||
};
|
||||
|
||||
tracing::debug!(
|
||||
target: "kebab-parse-code",
|
||||
"extracted Rust doc_id={} workspace_path={} units={}",
|
||||
doc_id.0,
|
||||
asset.workspace_path.0,
|
||||
unit_count
|
||||
);
|
||||
|
||||
Ok(CanonicalDocument {
|
||||
doc_id,
|
||||
source_asset_id: asset.asset_id.clone(),
|
||||
workspace_path: asset.workspace_path.clone(),
|
||||
title,
|
||||
lang: Lang("und".to_string()),
|
||||
blocks,
|
||||
metadata,
|
||||
provenance: Provenance { events },
|
||||
parser_version,
|
||||
schema_version: 1,
|
||||
doc_version: 1,
|
||||
last_chunker_version: None,
|
||||
last_embedding_version: None,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn build_blocks(
|
||||
source: &str,
|
||||
doc_id: &kebab_core::DocumentId,
|
||||
) -> anyhow::Result<Vec<kebab_core::Block>> {
|
||||
let mut parser = tree_sitter::Parser::new();
|
||||
parser
|
||||
.set_language(&tree_sitter_rust::LANGUAGE.into())
|
||||
.map_err(|e| anyhow::anyhow!("set tree-sitter-rust language: {e}"))?;
|
||||
let tree = parser
|
||||
.parse(source.as_bytes(), None)
|
||||
.ok_or_else(|| anyhow::anyhow!("tree-sitter failed to parse Rust source"))?;
|
||||
let lines: Vec<&str> = source.split('\n').collect();
|
||||
|
||||
// units: (symbol, line_start, line_end, is_real_semantic_unit).
|
||||
// Glue groups are pushed with a sentinel symbol + is_real=false so a
|
||||
// post-pass can decide `<module>` vs `<top-level>` (Gap 1).
|
||||
let mut units: Vec<(String, u32, u32, bool)> = Vec::new();
|
||||
let mut glue: Vec<(usize, u32, u32)> = Vec::new(); // (is_mod_decl 0/1, s, e)
|
||||
|
||||
fn node_name<'a>(n: &tree_sitter::Node, src: &'a str) -> Option<&'a str> {
|
||||
n.child_by_field_name("name")
|
||||
.map(|c| &src[c.start_byte()..c.end_byte()])
|
||||
}
|
||||
fn unit_start(n: &tree_sitter::Node) -> u32 {
|
||||
let mut start = n.start_position().row as u32 + 1;
|
||||
let mut prev = n.prev_sibling();
|
||||
while let Some(p) = prev {
|
||||
let k = p.kind();
|
||||
if k == "line_comment" || k == "block_comment" || k == "attribute_item" {
|
||||
start = p.start_position().row as u32 + 1;
|
||||
prev = p.prev_sibling();
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
start
|
||||
}
|
||||
fn walk(
|
||||
node: tree_sitter::Node,
|
||||
src: &str,
|
||||
mod_path: &[String],
|
||||
units: &mut Vec<(String, u32, u32, bool)>,
|
||||
glue: &mut Vec<(usize, u32, u32)>,
|
||||
) {
|
||||
// Module-path prefix for this scope. Used for both real units
|
||||
// (`format!("{prefix}{name}")`) and glue group labels
|
||||
// (`format!("{prefix}<top-level>")`) so glue from `mod inner`
|
||||
// doesn't collide on symbol with file-top-level glue and keeps
|
||||
// module context downstream. Empty at file top level -> glue
|
||||
// stays exactly `<top-level>` / `<module>`.
|
||||
let prefix = if mod_path.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!("{}::", mod_path.join("::"))
|
||||
};
|
||||
let mut cur = node.walk();
|
||||
for child in node.named_children(&mut cur) {
|
||||
let s = unit_start(&child);
|
||||
let e = child.end_position().row as u32 + 1;
|
||||
match child.kind() {
|
||||
"function_item" | "struct_item" | "enum_item" | "union_item"
|
||||
| "trait_item" | "type_item" => {
|
||||
if let Some(name) = node_name(&child, src) {
|
||||
// Gap 2: a leading attribute/comment that this unit
|
||||
// re-absorbs (via `unit_start`'s upward extension to
|
||||
// `s`) must not also remain in the glue group, or it
|
||||
// would be emitted in both chunks. Drop glue entries
|
||||
// at/after the unit's extended start.
|
||||
glue.retain(|(_, gs, _)| *gs < s);
|
||||
flush_glue(glue, units, &prefix);
|
||||
units.push((format!("{prefix}{name}"), s, e, true));
|
||||
}
|
||||
}
|
||||
"macro_definition" => {
|
||||
if let Some(name) = node_name(&child, src) {
|
||||
glue.retain(|(_, gs, _)| *gs < s);
|
||||
flush_glue(glue, units, &prefix);
|
||||
units.push((format!("{prefix}{name}!"), s, e, true));
|
||||
}
|
||||
}
|
||||
// `impl` blocks: emit one unit per inner `function_item`.
|
||||
// Associated consts / types / non-fn members do not become
|
||||
// their own units in 1A (plan §1A scope; HOTFIXES will log
|
||||
// if a future need arises). See inner comment below.
|
||||
"impl_item" => {
|
||||
glue.retain(|(_, gs, _)| *gs < s);
|
||||
flush_glue(glue, units, &prefix);
|
||||
let ty = child
|
||||
.child_by_field_name("type")
|
||||
.map(|c| src[c.start_byte()..c.end_byte()].trim().to_string());
|
||||
let tr = child
|
||||
.child_by_field_name("trait")
|
||||
.map(|c| src[c.start_byte()..c.end_byte()].trim().to_string());
|
||||
let owner = tr.or(ty).unwrap_or_else(|| "<impl>".to_string());
|
||||
if let Some(body) = child.child_by_field_name("body") {
|
||||
let mut bc = body.walk();
|
||||
// 1A scope: only inner `function_item` children
|
||||
// become units. Associated consts / types and other
|
||||
// non-fn impl members are intentionally NOT emitted
|
||||
// as separate units in 1A (plan spec: "1 per inner
|
||||
// function_item").
|
||||
for m in body.named_children(&mut bc) {
|
||||
if m.kind() == "function_item" {
|
||||
if let Some(mn) = node_name(&m, src) {
|
||||
let ms = unit_start(&m);
|
||||
let me = m.end_position().row as u32 + 1;
|
||||
units.push((format!("{prefix}{owner}::{mn}"), ms, me, true));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
"mod_item" => {
|
||||
if let Some(body) = child.child_by_field_name("body") {
|
||||
flush_glue(glue, units, &prefix);
|
||||
let name = node_name(&child, src).unwrap_or("mod").to_string();
|
||||
let mut np = mod_path.to_vec();
|
||||
np.push(name);
|
||||
walk(body, src, &np, units, glue);
|
||||
// Invariant: `glue` is shared by `&mut` across
|
||||
// recursive `walk` calls; every `walk` path ends with
|
||||
// a `flush_glue`, so inner-scope glue can never leak
|
||||
// into this outer scope's group. Assert it structurally
|
||||
// rather than relying on that being incidental.
|
||||
debug_assert!(
|
||||
glue.is_empty(),
|
||||
"inner walk must flush its glue before returning"
|
||||
);
|
||||
} else {
|
||||
glue.push((1, s, e));
|
||||
}
|
||||
}
|
||||
"use_declaration" | "extern_crate_declaration" | "const_item"
|
||||
| "static_item" | "attribute_item" | "macro_invocation" => {
|
||||
glue.push((0, s, e));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
flush_glue(glue, units, &prefix);
|
||||
}
|
||||
fn flush_glue(
|
||||
glue: &mut Vec<(usize, u32, u32)>,
|
||||
units: &mut Vec<(String, u32, u32, bool)>,
|
||||
prefix: &str,
|
||||
) {
|
||||
if glue.is_empty() {
|
||||
return;
|
||||
}
|
||||
let s = glue.iter().map(|(_, a, _)| *a).min().unwrap();
|
||||
let e = glue.iter().map(|(_, _, b)| *b).max().unwrap();
|
||||
// Provisional label: `<module>` only if this group is exclusively
|
||||
// bodyless `mod foo;` declarations. The final decision (Gap 1) also
|
||||
// requires the *whole file* to have produced zero real units; that
|
||||
// demotion to `<top-level>` happens in the post-pass below.
|
||||
let only_mod_decls = glue.iter().all(|(is_mod, _, _)| *is_mod == 1);
|
||||
let label = if only_mod_decls { "<module>" } else { "<top-level>" };
|
||||
// Module-path-prefix the label so glue from `mod inner` carries
|
||||
// module context (`inner::<top-level>`) and doesn't collide with
|
||||
// file-top-level glue. `prefix` is empty at file top level, so the
|
||||
// symbol stays exactly `<top-level>` / `<module>` there.
|
||||
units.push((format!("{prefix}{label}"), s, e, false));
|
||||
glue.clear();
|
||||
}
|
||||
|
||||
walk(tree.root_node(), source, &[], &mut units, &mut glue);
|
||||
|
||||
// Gap 1: `<module>` is correct only when the file produced no real
|
||||
// (non-glue) semantic unit at all. If any real unit exists, every glue
|
||||
// group is `<top-level>`, even a pure mod-decl group.
|
||||
let has_real_unit = units.iter().any(|(_, _, _, is_real)| *is_real);
|
||||
if has_real_unit {
|
||||
for (sym, _, _, is_real) in units.iter_mut() {
|
||||
// Match on the *suffix*: a glue group may now carry a module
|
||||
// prefix (`inner::<module>`), so demote any `…<module>` to the
|
||||
// same-prefixed `…<top-level>` rather than only the bare form.
|
||||
if !*is_real && sym.ends_with("<module>") {
|
||||
let pre = &sym[..sym.len() - "<module>".len()];
|
||||
*sym = format!("{pre}<top-level>");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let total_lines = lines.len() as u32;
|
||||
let mut blocks = Vec::with_capacity(units.len());
|
||||
for (ordinal, (symbol, ls, le, _is_real)) in units.into_iter().enumerate() {
|
||||
let line_start = ls.max(1);
|
||||
let line_end = le.min(total_lines.max(1));
|
||||
let span = SourceSpan::Code {
|
||||
line_start,
|
||||
line_end,
|
||||
symbol: Some(symbol),
|
||||
lang: Some("rust".to_string()),
|
||||
};
|
||||
let block_id = id_for_block(doc_id, "code", &[], ordinal as u32, &span);
|
||||
let code = lines[(line_start as usize - 1)..=(line_end as usize - 1)].join("\n");
|
||||
blocks.push(Block::Code(CodeBlock {
|
||||
common: CommonBlock {
|
||||
block_id,
|
||||
heading_path: Vec::new(),
|
||||
source_span: span,
|
||||
},
|
||||
lang: Some("rust".to_string()),
|
||||
code,
|
||||
}));
|
||||
}
|
||||
Ok(blocks)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use kebab_core::{Block, MediaType, SourceSpan};
|
||||
|
||||
fn extract_fixture() -> kebab_core::CanonicalDocument {
|
||||
let bytes = std::fs::read(
|
||||
concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/sample.rs"),
|
||||
)
|
||||
.unwrap();
|
||||
let asset = tests_support::fixed_code_asset("crates/x/src/sample.rs", "rust");
|
||||
let cfg = kebab_core::ExtractConfig::default();
|
||||
let root = std::path::PathBuf::from("/tmp");
|
||||
let ctx = kebab_core::ExtractContext { asset: &asset, workspace_root: &root, config: &cfg };
|
||||
RustAstExtractor::new().extract(&ctx, &bytes).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extractor_supports_only_media_code_rust() {
|
||||
let e = RustAstExtractor::new();
|
||||
assert!(e.supports(&MediaType::Code("rust".into())));
|
||||
assert!(!e.supports(&MediaType::Code("python".into())));
|
||||
assert!(!e.supports(&MediaType::Markdown));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn emits_one_block_per_semantic_unit_with_symbols() {
|
||||
let doc = extract_fixture();
|
||||
let mut syms: Vec<(String, u32, u32)> = doc
|
||||
.blocks
|
||||
.iter()
|
||||
.map(|b| match b {
|
||||
Block::Code(c) => match &c.common.source_span {
|
||||
SourceSpan::Code { symbol, line_start, line_end, lang } => {
|
||||
assert_eq!(lang.as_deref(), Some("rust"));
|
||||
(symbol.clone().unwrap(), *line_start, *line_end)
|
||||
}
|
||||
_ => panic!("code block must carry SourceSpan::Code"),
|
||||
},
|
||||
other => panic!("expected Block::Code, got {other:?}"),
|
||||
})
|
||||
.collect();
|
||||
syms.sort();
|
||||
let names: Vec<&str> = syms.iter().map(|(s, _, _)| s.as_str()).collect();
|
||||
assert!(names.contains(&"parse"));
|
||||
assert!(names.contains(&"Foo"));
|
||||
assert!(names.contains(&"Foo::double"));
|
||||
assert!(names.contains(&"Foo::name"));
|
||||
assert!(names.contains(&"Greet"));
|
||||
assert!(names.contains(&"inner::helper"));
|
||||
assert!(names.contains(&"<top-level>")); // use + const grouped
|
||||
let parse_src = doc.blocks.iter().find_map(|b| match b {
|
||||
Block::Code(c) if matches!(&c.common.source_span, SourceSpan::Code{symbol,..} if symbol.as_deref()==Some("parse")) => Some(c.code.clone()),
|
||||
_ => None,
|
||||
}).unwrap();
|
||||
assert!(parse_src.contains("/// Doc comment on a free fn."), "doc comment folded in: {parse_src}");
|
||||
}
|
||||
|
||||
/// Run the extractor on an in-memory Rust source string (no fixture
|
||||
/// file) and return (symbol, code) for every emitted block.
|
||||
fn extract_inline(source: &str) -> Vec<(String, String)> {
|
||||
let asset = tests_support::fixed_code_asset("crates/x/src/inline.rs", "rust");
|
||||
let cfg = kebab_core::ExtractConfig::default();
|
||||
let root = std::path::PathBuf::from("/tmp");
|
||||
let ctx = kebab_core::ExtractContext { asset: &asset, workspace_root: &root, config: &cfg };
|
||||
let doc = RustAstExtractor::new()
|
||||
.extract(&ctx, source.as_bytes())
|
||||
.unwrap();
|
||||
doc.blocks
|
||||
.iter()
|
||||
.map(|b| match b {
|
||||
Block::Code(c) => match &c.common.source_span {
|
||||
SourceSpan::Code { symbol, .. } => {
|
||||
(symbol.clone().unwrap(), c.code.clone())
|
||||
}
|
||||
_ => panic!("code block must carry SourceSpan::Code"),
|
||||
},
|
||||
other => panic!("expected Block::Code, got {other:?}"),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn module_label_scope_and_attribute_dedup() {
|
||||
// Source A (Gap 2): leading attribute is re-absorbed into the unit
|
||||
// and must NOT also form a separate <top-level> glue chunk.
|
||||
let a = extract_inline("#[derive(Debug)]\npub struct Tagged { x: u32 }\n");
|
||||
assert_eq!(a.len(), 1, "Gap 2: exactly one block, got {a:?}");
|
||||
assert_eq!(a[0].0, "Tagged");
|
||||
assert!(
|
||||
a[0].1.contains("#[derive(Debug)]"),
|
||||
"attribute folded into unit: {:?}",
|
||||
a[0].1
|
||||
);
|
||||
assert!(
|
||||
!a.iter().any(|(s, _)| s == "<top-level>"),
|
||||
"attribute must not also form a glue chunk: {a:?}"
|
||||
);
|
||||
|
||||
// Source B (Gap 1): file has no real units, only bodyless mod
|
||||
// decls -> the glue group is <module>.
|
||||
let b = extract_inline("mod a;\nmod b;\n");
|
||||
assert_eq!(b.len(), 1, "one glue block, got {b:?}");
|
||||
assert_eq!(b[0].0, "<module>");
|
||||
|
||||
// Source C (Gap 1): mod decls + a real unit -> the glue group is
|
||||
// <top-level>, NOT <module>, because the file has a real unit.
|
||||
let c = extract_inline("mod a;\nmod b;\npub fn f() {}\n");
|
||||
let syms: Vec<&str> = c.iter().map(|(s, _)| s.as_str()).collect();
|
||||
assert!(syms.contains(&"f"), "real unit present: {c:?}");
|
||||
assert!(
|
||||
syms.contains(&"<top-level>"),
|
||||
"mod-decl glue demoted to <top-level>: {c:?}"
|
||||
);
|
||||
assert!(
|
||||
!syms.contains(&"<module>"),
|
||||
"must not be <module> when file has a real unit: {c:?}"
|
||||
);
|
||||
|
||||
// Source D (Fix 1): glue inside a bodied `mod inner` must carry the
|
||||
// module-path prefix so it doesn't collide with file-top-level glue
|
||||
// and keeps module context downstream.
|
||||
let d = extract_inline("mod inner {\n use std::fmt;\n pub fn helper() {}\n}\n");
|
||||
let dsyms: Vec<&str> = d.iter().map(|(s, _)| s.as_str()).collect();
|
||||
assert!(
|
||||
dsyms.contains(&"inner::helper"),
|
||||
"real unit inside mod is prefixed: {d:?}"
|
||||
);
|
||||
assert!(
|
||||
dsyms.contains(&"inner::<top-level>"),
|
||||
"glue inside mod inner is module-prefixed, not bare: {d:?}"
|
||||
);
|
||||
assert!(
|
||||
!dsyms.contains(&"<top-level>"),
|
||||
"glue inside mod inner must NOT be the bare top-level symbol: {d:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deterministic_across_runs() {
|
||||
let a = extract_fixture();
|
||||
for _ in 0..50 {
|
||||
assert_eq!(extract_fixture().blocks, a.blocks);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod tests_support {
|
||||
use kebab_core::*;
|
||||
use time::OffsetDateTime;
|
||||
/// Test-only `RawAsset` builder for any tree-sitter language. Shared
|
||||
/// across `rust.rs` / `python.rs` / future TS+JS extractor tests so all
|
||||
/// in-crate code-extractor tests use a single canonical fixture shape.
|
||||
pub fn fixed_code_asset(workspace_path: &str, code_lang: &str) -> RawAsset {
|
||||
RawAsset {
|
||||
asset_id: AssetId("a".repeat(64)),
|
||||
source_uri: SourceUri::File(std::path::PathBuf::from(workspace_path)),
|
||||
workspace_path: WorkspacePath(workspace_path.to_string()),
|
||||
media_type: MediaType::Code(code_lang.to_string()),
|
||||
byte_len: 0,
|
||||
checksum: Checksum("b".repeat(64)),
|
||||
discovered_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
stored: AssetStorage::Reference {
|
||||
path: std::path::PathBuf::from(workspace_path),
|
||||
sha: Checksum("b".repeat(64)),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
45
crates/kebab-parse-code/src/scaffold.rs
Normal file
45
crates/kebab-parse-code/src/scaffold.rs
Normal file
@@ -0,0 +1,45 @@
|
||||
//! `kebab-parse-code::scaffold` — shared pure helpers used by all
|
||||
//! per-language extractor modules.
|
||||
//!
|
||||
//! These are `pub(crate)` utilities extracted from the four extractor
|
||||
//! modules (rust / python / typescript / javascript) where identical
|
||||
//! copies existed. Keeping them here is the single source of truth.
|
||||
|
||||
/// Extract the last path component (filename) from a `/`-separated
|
||||
/// workspace path string.
|
||||
/// For a path like `crates/x/src/foo.rs` this returns `foo.rs`.
|
||||
pub(crate) fn filename_from_workspace_path(p: &str) -> String {
|
||||
p.rsplit('/').next().unwrap_or(p).to_string()
|
||||
}
|
||||
|
||||
/// Strip the last dot-extension from a filename string.
|
||||
/// A leading dot (hidden-file convention) is preserved as-is.
|
||||
/// `foo.rs` → `foo`, `.hidden` → `.hidden`, `noext` → `noext`.
|
||||
pub(crate) fn strip_extension(filename: &str) -> String {
|
||||
match filename.rfind('.') {
|
||||
Some(0) => filename.to_string(),
|
||||
Some(idx) => filename[..idx].to_string(),
|
||||
None => filename.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Join `(mod_prefix, mod_path, name)` into a dotted symbol string.
|
||||
///
|
||||
/// Used by Python / TypeScript / JavaScript extractors. Rust uses
|
||||
/// `::` separators instead and builds symbols inline; this helper
|
||||
/// covers the `.`-joined languages.
|
||||
///
|
||||
/// Empty `mod_prefix` (e.g. file is `__init__.py` at workspace root)
|
||||
/// drops the leading prefix segment; empty `mod_path` (file top-level)
|
||||
/// drops the class-nesting middle segment.
|
||||
pub(crate) fn join_symbol(mod_prefix: &str, mod_path: &[String], name: &str) -> String {
|
||||
let mut parts: Vec<&str> = Vec::with_capacity(mod_path.len() + 2);
|
||||
if !mod_prefix.is_empty() {
|
||||
parts.push(mod_prefix);
|
||||
}
|
||||
for p in mod_path {
|
||||
parts.push(p.as_str());
|
||||
}
|
||||
parts.push(name);
|
||||
parts.join(".")
|
||||
}
|
||||
65
crates/kebab-parse-code/src/skip.rs
Normal file
65
crates/kebab-parse-code/src/skip.rs
Normal file
@@ -0,0 +1,65 @@
|
||||
//! Pre-ingest skip helpers (spec §5.2 + §5.3 + §5.4).
|
||||
//!
|
||||
//! - [`BUILTIN_BLACKLIST`] — 6 gitignore-style patterns universal across
|
||||
//! ecosystems. Source of truth: spec §5.2.
|
||||
//! - [`is_generated_file`] — reads first ~512 bytes, checks for 7
|
||||
//! case-insensitive markers.
|
||||
//! - [`is_oversized`] — byte cap then line cap.
|
||||
|
||||
use anyhow::Result;
|
||||
use std::fs::File;
|
||||
use std::io::{BufRead, BufReader, Read};
|
||||
use std::path::Path;
|
||||
|
||||
/// 6 built-in gitignore-style patterns. Applied in addition to `.gitignore`
|
||||
/// + `.kebabignore`. User can override via `.kebabignore` negation
|
||||
/// (`!pattern`).
|
||||
pub const BUILTIN_BLACKLIST: &[&str] = &[
|
||||
"**/node_modules/**",
|
||||
"**/target/**",
|
||||
"**/__pycache__/**",
|
||||
"**/.venv/**",
|
||||
"**/venv/**",
|
||||
"**/env/**",
|
||||
];
|
||||
|
||||
/// Read first 512 bytes, check for any of 7 case-insensitive generated-file
|
||||
/// markers. Returns Ok(true) on match, Ok(false) otherwise.
|
||||
pub fn is_generated_file(path: &Path) -> Result<bool> {
|
||||
let mut buf = [0u8; 512];
|
||||
let mut f = File::open(path)?;
|
||||
let n = f.read(&mut buf)?;
|
||||
if n == 0 {
|
||||
return Ok(false);
|
||||
}
|
||||
let head = std::str::from_utf8(&buf[..n]).unwrap_or("");
|
||||
let lower: String = head.lines().take(10).collect::<Vec<_>>().join("\n").to_ascii_lowercase();
|
||||
Ok(
|
||||
lower.contains("@generated")
|
||||
|| lower.contains("code generated by")
|
||||
|| lower.contains("do not edit")
|
||||
|| lower.contains("do not modify")
|
||||
|| lower.contains("automatically generated")
|
||||
|| lower.contains("auto-generated")
|
||||
|| lower.contains("autogenerated"),
|
||||
)
|
||||
}
|
||||
|
||||
/// Check if `path` exceeds `max_bytes` or `max_lines`. Byte cap first
|
||||
/// (cheap), then line cap (streaming with early exit).
|
||||
pub fn is_oversized(path: &Path, max_bytes: u64, max_lines: u32) -> Result<bool> {
|
||||
let meta = std::fs::metadata(path)?;
|
||||
if meta.len() > max_bytes {
|
||||
return Ok(true);
|
||||
}
|
||||
let reader = BufReader::new(File::open(path)?);
|
||||
let mut count: u32 = 0;
|
||||
for line in reader.lines() {
|
||||
let _ = line?;
|
||||
count = count.saturating_add(1);
|
||||
if count > max_lines {
|
||||
return Ok(true);
|
||||
}
|
||||
}
|
||||
Ok(false)
|
||||
}
|
||||
690
crates/kebab-parse-code/src/typescript.rs
Normal file
690
crates/kebab-parse-code/src/typescript.rs
Normal file
@@ -0,0 +1,690 @@
|
||||
//! `kebab-parse-code::typescript` — tree-sitter TypeScript / TSX AST
|
||||
//! extractor (P10-1B Task H).
|
||||
//!
|
||||
//! Implements [`kebab_core::Extractor`] for [`MediaType::Code("typescript")`].
|
||||
//! Walks the tree-sitter parse tree (one of two grammars selected by the
|
||||
//! workspace path's extension — `.tsx` uses [`tree_sitter_typescript::LANGUAGE_TSX`],
|
||||
//! everything else uses [`tree_sitter_typescript::LANGUAGE_TYPESCRIPT`]) and
|
||||
//! emits one [`Block::Code`] per top-level AST semantic unit (free fn,
|
||||
//! class, each method, interface, type alias, enum, recursively per
|
||||
//! nested class), each carrying [`SourceSpan::Code`] with the unit's
|
||||
//! dotted symbol path prefixed by [`module_path_for_tsjs`].
|
||||
//!
|
||||
//! Glue declarations (`import_statement`, bare `export_statement`
|
||||
//! re-exports, `lexical_declaration` / `variable_declaration` at the
|
||||
//! module level, namespace / module declarations, etc.) collapse into
|
||||
//! one grouped `<top-level>` (or `<module>`) unit.
|
||||
//!
|
||||
//! `export_statement` is unwrapped: an `export function|class|interface
|
||||
//! |type|enum` is treated as the inner declaration arm but the unit's
|
||||
//! line range comes from the OUTER `export_statement` so the `export `
|
||||
//! prefix is folded in. `export default function () {}` / `export
|
||||
//! default class {}` (no `name` field) emits `default` as the symbol
|
||||
//! name.
|
||||
//!
|
||||
//! Scope follows 1A-2 / 1B Task E: AST unit extraction + dotted symbol
|
||||
//! paths + line ranges. Per design §3.4 / §9.1 / §9 versioning.
|
||||
|
||||
use anyhow::Result;
|
||||
use kebab_core::{
|
||||
Block, CanonicalDocument, CodeBlock, CommonBlock, Extractor, Lang, MediaType, Metadata,
|
||||
ParserVersion, Provenance, ProvenanceEvent, ProvenanceKind, SourceSpan, SourceType, TrustLevel,
|
||||
id_for_block, id_for_doc,
|
||||
};
|
||||
use serde_json::Map;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
use crate::scaffold::{filename_from_workspace_path, join_symbol, strip_extension};
|
||||
|
||||
pub const PARSER_VERSION: &str = "code-ts-v1";
|
||||
|
||||
/// TypeScript / TSX AST extractor. Per-unit blocks via
|
||||
/// tree-sitter-typescript 0.23 (`LANGUAGE_TYPESCRIPT` / `LANGUAGE_TSX`
|
||||
/// — two `LanguageFn`s, selected by extension) parsed by tree-sitter
|
||||
/// 0.26.
|
||||
pub struct TypescriptAstExtractor;
|
||||
|
||||
impl TypescriptAstExtractor {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for TypescriptAstExtractor {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl Extractor for TypescriptAstExtractor {
|
||||
fn supports(&self, m: &MediaType) -> bool {
|
||||
matches!(m, MediaType::Code(l) if l == "typescript")
|
||||
}
|
||||
|
||||
fn parser_version(&self) -> ParserVersion {
|
||||
ParserVersion(PARSER_VERSION.to_string())
|
||||
}
|
||||
|
||||
fn extract(
|
||||
&self,
|
||||
ctx: &kebab_core::ExtractContext<'_>,
|
||||
bytes: &[u8],
|
||||
) -> Result<CanonicalDocument> {
|
||||
let asset = ctx.asset;
|
||||
if !self.supports(&asset.media_type) {
|
||||
anyhow::bail!(
|
||||
"kebab-parse-code: unsupported media_type for TypescriptAstExtractor: {:?}",
|
||||
asset.media_type
|
||||
);
|
||||
}
|
||||
|
||||
let parser_version = self.parser_version();
|
||||
let doc_id = id_for_doc(&asset.workspace_path, &asset.asset_id, &parser_version);
|
||||
|
||||
let source = String::from_utf8(bytes.to_vec()).map_err(|e| {
|
||||
anyhow::anyhow!("kebab-parse-code: TypeScript source is not valid UTF-8: {e}")
|
||||
})?;
|
||||
|
||||
let mod_prefix = crate::lang::module_path_for_tsjs(&asset.workspace_path.0);
|
||||
let language = select_grammar(&asset.workspace_path.0);
|
||||
let blocks = build_blocks(&source, &doc_id, &mod_prefix, language)?;
|
||||
let unit_count = blocks.len() as u32;
|
||||
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let mut events: Vec<ProvenanceEvent> = Vec::with_capacity(2);
|
||||
events.push(ProvenanceEvent {
|
||||
at: asset.discovered_at,
|
||||
agent: "kb-source-fs".to_string(),
|
||||
kind: ProvenanceKind::Discovered,
|
||||
note: None,
|
||||
});
|
||||
events.push(ProvenanceEvent {
|
||||
at: now,
|
||||
agent: "kb-parse-code".to_string(),
|
||||
kind: ProvenanceKind::Parsed,
|
||||
note: Some(format!(
|
||||
"parser_version={}; unit_count={}",
|
||||
parser_version.0, unit_count
|
||||
)),
|
||||
});
|
||||
|
||||
let title = {
|
||||
let fname = filename_from_workspace_path(&asset.workspace_path.0);
|
||||
strip_extension(&fname)
|
||||
};
|
||||
|
||||
// Resolve the file's absolute path for repo detection. If the
|
||||
// source URI carries a relative path, anchor it at the workspace
|
||||
// root so the `.git/` walk-up starts from the right place.
|
||||
let abs_path = match &asset.source_uri {
|
||||
kebab_core::SourceUri::File(p) => {
|
||||
if p.is_absolute() {
|
||||
p.clone()
|
||||
} else {
|
||||
ctx.workspace_root.join(p)
|
||||
}
|
||||
}
|
||||
kebab_core::SourceUri::Kb(_) => ctx.workspace_root.to_path_buf(),
|
||||
};
|
||||
let (repo, git_branch, git_commit) = match crate::repo::detect_repo(&abs_path) {
|
||||
Some(r) => (Some(r.name), r.branch, r.commit),
|
||||
None => (None, None, None),
|
||||
};
|
||||
|
||||
let metadata = Metadata {
|
||||
aliases: Vec::new(),
|
||||
tags: Vec::new(),
|
||||
created_at: asset.discovered_at,
|
||||
updated_at: asset.discovered_at,
|
||||
source_type: SourceType::Note,
|
||||
trust_level: TrustLevel::Primary,
|
||||
user_id_alias: None,
|
||||
user: Map::new(),
|
||||
repo,
|
||||
git_branch,
|
||||
git_commit,
|
||||
code_lang: Some("typescript".to_string()),
|
||||
};
|
||||
|
||||
tracing::debug!(
|
||||
target: "kebab-parse-code",
|
||||
"extracted TypeScript doc_id={} workspace_path={} units={}",
|
||||
doc_id.0,
|
||||
asset.workspace_path.0,
|
||||
unit_count
|
||||
);
|
||||
|
||||
Ok(CanonicalDocument {
|
||||
doc_id,
|
||||
source_asset_id: asset.asset_id.clone(),
|
||||
workspace_path: asset.workspace_path.clone(),
|
||||
title,
|
||||
lang: Lang("und".to_string()),
|
||||
blocks,
|
||||
metadata,
|
||||
provenance: Provenance { events },
|
||||
parser_version,
|
||||
schema_version: 1,
|
||||
doc_version: 1,
|
||||
last_chunker_version: None,
|
||||
last_embedding_version: None,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Select the tree-sitter grammar based on the workspace path's
|
||||
/// extension. `.tsx` → TSX grammar; everything else (`.ts`, `.d.ts`,
|
||||
/// missing extension) → TypeScript grammar.
|
||||
fn select_grammar(workspace_path: &str) -> tree_sitter::Language {
|
||||
if workspace_path.ends_with(".tsx") {
|
||||
tree_sitter_typescript::LANGUAGE_TSX.into()
|
||||
} else {
|
||||
tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into()
|
||||
}
|
||||
}
|
||||
|
||||
fn build_blocks(
|
||||
source: &str,
|
||||
doc_id: &kebab_core::DocumentId,
|
||||
mod_prefix: &str,
|
||||
language: tree_sitter::Language,
|
||||
) -> anyhow::Result<Vec<kebab_core::Block>> {
|
||||
let mut parser = tree_sitter::Parser::new();
|
||||
parser
|
||||
.set_language(&language)
|
||||
.map_err(|e| anyhow::anyhow!("set tree-sitter-typescript language: {e}"))?;
|
||||
let tree = parser
|
||||
.parse(source.as_bytes(), None)
|
||||
.ok_or_else(|| anyhow::anyhow!("tree-sitter failed to parse TypeScript source"))?;
|
||||
let lines: Vec<&str> = source.split('\n').collect();
|
||||
|
||||
// units: (symbol, line_start, line_end, is_real_semantic_unit).
|
||||
// Glue groups are pushed with a sentinel symbol + is_real=false so a
|
||||
// post-pass can decide `<module>` vs `<top-level>` (same algorithm
|
||||
// as 1A Gap 1 / 1B Python).
|
||||
let mut units: Vec<(String, u32, u32, bool)> = Vec::new();
|
||||
// (is_module_only_kind 0/1, s, e). `is_module_only_kind` flags
|
||||
// `import_statement` and bare re-export `export_statement`s — used by
|
||||
// the glue flush to pick `<module>` vs `<top-level>` provisional
|
||||
// label (1A's `is_mod_decl` analog).
|
||||
let mut glue: Vec<(usize, u32, u32)> = Vec::new();
|
||||
|
||||
/// Walk preceding `comment` and `decorator` siblings to extend the
|
||||
/// unit's line range upward, folding leading doc/line comments and
|
||||
/// decorators into the unit.
|
||||
///
|
||||
/// In tree-sitter-typescript 0.23, TS class-method decorators (and
|
||||
/// class-level decorators) are **`class_body` siblings** that
|
||||
/// immediately precede the `method_definition` node — they are NOT
|
||||
/// children of `method_definition`. (Contrast with
|
||||
/// tree-sitter-javascript, where the `decorator` IS stored inside
|
||||
/// `method_definition` as a named child via the `decorator` field, so
|
||||
/// `method_definition.start_row` already covers the decorator line
|
||||
/// there — no sibling walk needed in `javascript.rs`.)
|
||||
///
|
||||
/// Extending backward over `decorator` siblings here matches Python's
|
||||
/// `decorated_definition` arm behavior: the decorator line is folded
|
||||
/// into the emitted unit's line range.
|
||||
fn unit_start(n: &tree_sitter::Node) -> u32 {
|
||||
let mut start = n.start_position().row as u32 + 1;
|
||||
let mut prev = n.prev_sibling();
|
||||
while let Some(p) = prev {
|
||||
if p.kind() == "comment" || p.kind() == "decorator" {
|
||||
start = p.start_position().row as u32 + 1;
|
||||
prev = p.prev_sibling();
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
start
|
||||
}
|
||||
fn name_text<'a>(n: &tree_sitter::Node, src: &'a str) -> Option<&'a str> {
|
||||
n.child_by_field_name("name")
|
||||
.map(|c| &src[c.start_byte()..c.end_byte()])
|
||||
}
|
||||
/// Walk a class body, emitting one unit per `method_definition`.
|
||||
/// Class names already pushed onto `mod_path` by the caller, so
|
||||
/// method symbols come out as `<mod_prefix>.<Class>.<method>`.
|
||||
fn walk_class_body(
|
||||
body: tree_sitter::Node,
|
||||
src: &str,
|
||||
mod_prefix: &str,
|
||||
mod_path: &[String],
|
||||
units: &mut Vec<(String, u32, u32, bool)>,
|
||||
) {
|
||||
let mut cur = body.walk();
|
||||
for child in body.named_children(&mut cur) {
|
||||
if child.kind() == "method_definition" {
|
||||
if let Some(name) = name_text(&child, src) {
|
||||
let s = unit_start(&child);
|
||||
let e = child.end_position().row as u32 + 1;
|
||||
let sym = join_symbol(mod_prefix, mod_path, name);
|
||||
units.push((sym, s, e, true));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
fn walk(
|
||||
node: tree_sitter::Node,
|
||||
src: &str,
|
||||
mod_prefix: &str,
|
||||
mod_path: &[String],
|
||||
units: &mut Vec<(String, u32, u32, bool)>,
|
||||
glue: &mut Vec<(usize, u32, u32)>,
|
||||
) {
|
||||
let mut cur = node.walk();
|
||||
for child in node.named_children(&mut cur) {
|
||||
let s = unit_start(&child);
|
||||
let e = child.end_position().row as u32 + 1;
|
||||
match child.kind() {
|
||||
"function_declaration" => {
|
||||
if let Some(name) = name_text(&child, src) {
|
||||
glue.retain(|(_, gs, _)| *gs < s);
|
||||
flush_glue(glue, units, mod_prefix, mod_path);
|
||||
let sym = join_symbol(mod_prefix, mod_path, name);
|
||||
units.push((sym, s, e, true));
|
||||
}
|
||||
}
|
||||
"class_declaration" => {
|
||||
if let Some(name) = name_text(&child, src) {
|
||||
glue.retain(|(_, gs, _)| *gs < s);
|
||||
flush_glue(glue, units, mod_prefix, mod_path);
|
||||
let sym = join_symbol(mod_prefix, mod_path, name);
|
||||
units.push((sym, s, e, true));
|
||||
if let Some(body) = child.child_by_field_name("body") {
|
||||
let mut np = mod_path.to_vec();
|
||||
np.push(name.to_string());
|
||||
walk_class_body(body, src, mod_prefix, &np, units);
|
||||
}
|
||||
}
|
||||
}
|
||||
"interface_declaration"
|
||||
| "type_alias_declaration"
|
||||
| "enum_declaration" => {
|
||||
if let Some(name) = name_text(&child, src) {
|
||||
glue.retain(|(_, gs, _)| *gs < s);
|
||||
flush_glue(glue, units, mod_prefix, mod_path);
|
||||
let sym = join_symbol(mod_prefix, mod_path, name);
|
||||
units.push((sym, s, e, true));
|
||||
}
|
||||
}
|
||||
"export_statement" => {
|
||||
// Try field "declaration" first (export class /
|
||||
// function / interface / type / enum). If absent,
|
||||
// fall back to "value" — `export default function
|
||||
// () {}` / `export default class {}` expose the
|
||||
// anonymous function_expression / class under the
|
||||
// `value` field (TS grammar 0.23).
|
||||
let outer_s = s; // includes `export ` prefix line
|
||||
let outer_e = e;
|
||||
if let Some(inner) = child.child_by_field_name("declaration") {
|
||||
let inner_kind = inner.kind();
|
||||
match inner_kind {
|
||||
"function_declaration"
|
||||
| "class_declaration"
|
||||
| "interface_declaration"
|
||||
| "type_alias_declaration"
|
||||
| "enum_declaration" => {
|
||||
let name_opt = name_text(&inner, src).map(|s| s.to_string());
|
||||
if let Some(name) = name_opt {
|
||||
glue.retain(|(_, gs, _)| *gs < outer_s);
|
||||
flush_glue(glue, units, mod_prefix, mod_path);
|
||||
let sym =
|
||||
join_symbol(mod_prefix, mod_path, &name);
|
||||
units.push((sym, outer_s, outer_e, true));
|
||||
if inner_kind == "class_declaration" {
|
||||
if let Some(body) =
|
||||
inner.child_by_field_name("body")
|
||||
{
|
||||
let mut np = mod_path.to_vec();
|
||||
np.push(name);
|
||||
walk_class_body(
|
||||
body, src, mod_prefix, &np, units,
|
||||
);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// `export default function foo() {}`
|
||||
// path is covered by name_opt =
|
||||
// Some(_) above; the no-name path
|
||||
// here is `export default` with a
|
||||
// function_declaration that
|
||||
// somehow lacks `name`. Emit
|
||||
// `default` defensively.
|
||||
glue.retain(|(_, gs, _)| *gs < outer_s);
|
||||
flush_glue(glue, units, mod_prefix, mod_path);
|
||||
let sym =
|
||||
join_symbol(mod_prefix, mod_path, "default");
|
||||
units.push((sym, outer_s, outer_e, true));
|
||||
}
|
||||
}
|
||||
// `lexical_declaration` etc. wrapped in
|
||||
// export: treat as glue (assigned arrow
|
||||
// fns / consts don't get their own unit).
|
||||
_ => {
|
||||
glue.push((0, s, e));
|
||||
}
|
||||
}
|
||||
} else if let Some(value) = child.child_by_field_name("value") {
|
||||
// `export default <expr>`. We emit a unit only
|
||||
// for the function / class shapes (named or
|
||||
// anonymous); other value shapes are glue.
|
||||
match value.kind() {
|
||||
"function_expression"
|
||||
| "function_declaration"
|
||||
| "class"
|
||||
| "class_declaration" => {
|
||||
let name_opt =
|
||||
name_text(&value, src).map(|s| s.to_string());
|
||||
let leaf = name_opt
|
||||
.as_deref()
|
||||
.unwrap_or("default")
|
||||
.to_string();
|
||||
glue.retain(|(_, gs, _)| *gs < outer_s);
|
||||
flush_glue(glue, units, mod_prefix, mod_path);
|
||||
let sym = join_symbol(mod_prefix, mod_path, &leaf);
|
||||
units.push((sym, outer_s, outer_e, true));
|
||||
// Recurse into class body if we have one.
|
||||
if matches!(
|
||||
value.kind(),
|
||||
"class" | "class_declaration"
|
||||
) {
|
||||
if let Some(body) =
|
||||
value.child_by_field_name("body")
|
||||
{
|
||||
let mut np = mod_path.to_vec();
|
||||
np.push(leaf);
|
||||
walk_class_body(
|
||||
body, src, mod_prefix, &np, units,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
glue.push((0, s, e));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Bare `export { x };` / `export * from "..."` —
|
||||
// a re-export, glue with module-only flag set
|
||||
// (we have no `declaration` / `value` field for
|
||||
// it).
|
||||
glue.push((1, s, e));
|
||||
}
|
||||
}
|
||||
"import_statement" => {
|
||||
glue.push((1, s, e));
|
||||
}
|
||||
"lexical_declaration" | "variable_declaration" => {
|
||||
glue.push((0, s, e));
|
||||
}
|
||||
// Namespace / module declarations (rare in app code,
|
||||
// common in `.d.ts`): treat as glue per plan §Task H
|
||||
// (1B 1차 scope; documented under spec Risks).
|
||||
"internal_module" | "module" | "ambient_declaration" => {
|
||||
glue.push((0, s, e));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
flush_glue(glue, units, mod_prefix, mod_path);
|
||||
}
|
||||
fn flush_glue(
|
||||
glue: &mut Vec<(usize, u32, u32)>,
|
||||
units: &mut Vec<(String, u32, u32, bool)>,
|
||||
mod_prefix: &str,
|
||||
mod_path: &[String],
|
||||
) {
|
||||
if glue.is_empty() {
|
||||
return;
|
||||
}
|
||||
let s = glue.iter().map(|(_, a, _)| *a).min().unwrap();
|
||||
let e = glue.iter().map(|(_, _, b)| *b).max().unwrap();
|
||||
let only_module = glue.iter().all(|(is_mod, _, _)| *is_mod == 1);
|
||||
let label = if only_module { "<module>" } else { "<top-level>" };
|
||||
units.push((join_symbol(mod_prefix, mod_path, label), s, e, false));
|
||||
glue.clear();
|
||||
}
|
||||
|
||||
walk(
|
||||
tree.root_node(),
|
||||
source,
|
||||
mod_prefix,
|
||||
&[],
|
||||
&mut units,
|
||||
&mut glue,
|
||||
);
|
||||
|
||||
// `<module>` is correct only when the file produced no real unit.
|
||||
// Otherwise the import-only group becomes `<top-level>` (same
|
||||
// post-pass as 1A Gap 1 / Python).
|
||||
let has_real_unit = units.iter().any(|(_, _, _, is_real)| *is_real);
|
||||
if has_real_unit {
|
||||
for (sym, _, _, is_real) in units.iter_mut() {
|
||||
if !*is_real && sym.ends_with("<module>") {
|
||||
let pre = &sym[..sym.len() - "<module>".len()];
|
||||
*sym = format!("{pre}<top-level>");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let total_lines = lines.len() as u32;
|
||||
let mut blocks = Vec::with_capacity(units.len());
|
||||
for (ordinal, (symbol, ls, le, _is_real)) in units.into_iter().enumerate() {
|
||||
let line_start = ls.max(1);
|
||||
let line_end = le.min(total_lines.max(1));
|
||||
let span = SourceSpan::Code {
|
||||
line_start,
|
||||
line_end,
|
||||
symbol: Some(symbol),
|
||||
lang: Some("typescript".to_string()),
|
||||
};
|
||||
let block_id = id_for_block(doc_id, "code", &[], ordinal as u32, &span);
|
||||
let code = lines[(line_start as usize - 1)..=(line_end as usize - 1)].join("\n");
|
||||
blocks.push(Block::Code(CodeBlock {
|
||||
common: CommonBlock {
|
||||
block_id,
|
||||
heading_path: Vec::new(),
|
||||
source_span: span,
|
||||
},
|
||||
lang: Some("typescript".to_string()),
|
||||
code,
|
||||
}));
|
||||
}
|
||||
Ok(blocks)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use kebab_core::{Block, MediaType, SourceSpan};
|
||||
|
||||
fn extract_fixture(name: &str, workspace_path: &str) -> kebab_core::CanonicalDocument {
|
||||
let bytes = std::fs::read(format!(
|
||||
concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/{}"),
|
||||
name
|
||||
))
|
||||
.unwrap();
|
||||
let asset = crate::rust::tests_support::fixed_code_asset(workspace_path, "typescript");
|
||||
let cfg = kebab_core::ExtractConfig::default();
|
||||
let root = std::path::PathBuf::from("/tmp");
|
||||
let ctx = kebab_core::ExtractContext {
|
||||
asset: &asset,
|
||||
workspace_root: &root,
|
||||
config: &cfg,
|
||||
};
|
||||
TypescriptAstExtractor::new()
|
||||
.extract(&ctx, &bytes)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn symbols(doc: &kebab_core::CanonicalDocument) -> Vec<String> {
|
||||
let mut s: Vec<String> = doc
|
||||
.blocks
|
||||
.iter()
|
||||
.filter_map(|b| match b {
|
||||
Block::Code(c) => match &c.common.source_span {
|
||||
SourceSpan::Code { symbol, lang, .. } => {
|
||||
assert_eq!(lang.as_deref(), Some("typescript"));
|
||||
symbol.clone()
|
||||
}
|
||||
_ => None,
|
||||
},
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
s.sort();
|
||||
s
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extractor_supports_only_media_code_typescript() {
|
||||
let e = TypescriptAstExtractor::new();
|
||||
assert!(e.supports(&MediaType::Code("typescript".into())));
|
||||
assert!(!e.supports(&MediaType::Code("rust".into())));
|
||||
assert!(!e.supports(&MediaType::Markdown));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ts_units_match_design_3_4_symbols() {
|
||||
// workspace_path `src/sample.ts` → mod_prefix `src/sample`
|
||||
let doc = extract_fixture("sample.ts", "src/sample.ts");
|
||||
let syms = symbols(&doc);
|
||||
assert!(syms.iter().any(|s| s == "src/sample.add"), "got {syms:?}");
|
||||
assert!(syms.iter().any(|s| s == "src/sample.Greet"));
|
||||
assert!(syms.iter().any(|s| s == "src/sample.Maybe"));
|
||||
assert!(syms.iter().any(|s| s == "src/sample.Retriever"));
|
||||
assert!(syms.iter().any(|s| s == "src/sample.Retriever.search"));
|
||||
assert!(syms.iter().any(|s| s == "src/sample.Retriever.create"));
|
||||
assert!(syms.iter().any(|s| s == "src/sample.default"));
|
||||
assert!(syms.iter().any(|s| s == "src/sample.<top-level>"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tsx_uses_tsx_grammar_and_emits_units() {
|
||||
let doc = extract_fixture("sample.tsx", "src/sample.tsx");
|
||||
let syms = symbols(&doc);
|
||||
assert!(
|
||||
syms.iter().any(|s| s == "src/sample.Hello"),
|
||||
"got {syms:?}"
|
||||
);
|
||||
assert!(
|
||||
syms.iter().any(|s| s == "src/sample.<top-level>"),
|
||||
"arrow fn + import should roll into top-level glue"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deterministic_across_runs() {
|
||||
let a = extract_fixture("sample.ts", "src/sample.ts");
|
||||
for _ in 0..30 {
|
||||
assert_eq!(extract_fixture("sample.ts", "src/sample.ts").blocks, a.blocks);
|
||||
}
|
||||
}
|
||||
|
||||
/// Regression: TS class-method decorators are `class_body` preceding
|
||||
/// siblings (not children of `method_definition`). The `unit_start`
|
||||
/// backward walk must fold the decorator line into the emitted unit's
|
||||
/// line range, matching Python's `decorated_definition` behavior.
|
||||
#[test]
|
||||
fn class_method_decorator_folded_into_method_unit() {
|
||||
// Line 1 (1-indexed): "class Foo {"
|
||||
// Line 2: " @Log()" <- decorator
|
||||
// Line 3: " bar() { return 1; }"
|
||||
// Line 4: "}"
|
||||
let bytes = b"class Foo {\n @Log()\n bar() { return 1; }\n}\n";
|
||||
let asset = crate::rust::tests_support::fixed_code_asset("src/foo.ts", "typescript");
|
||||
let cfg = kebab_core::ExtractConfig::default();
|
||||
let root = std::path::PathBuf::from("/tmp");
|
||||
let ctx = kebab_core::ExtractContext {
|
||||
asset: &asset,
|
||||
workspace_root: &root,
|
||||
config: &cfg,
|
||||
};
|
||||
let doc = TypescriptAstExtractor::new().extract(&ctx, bytes).unwrap();
|
||||
|
||||
let bar_block = doc
|
||||
.blocks
|
||||
.iter()
|
||||
.find_map(|b| match b {
|
||||
Block::Code(c) => match &c.common.source_span {
|
||||
SourceSpan::Code { symbol, .. }
|
||||
if symbol.as_deref() == Some("src/foo.Foo.bar") =>
|
||||
{
|
||||
Some(c)
|
||||
}
|
||||
_ => None,
|
||||
},
|
||||
_ => None,
|
||||
})
|
||||
.expect("src/foo.Foo.bar block should be present");
|
||||
|
||||
// After the fix, the unit MUST include the @Log() decorator line.
|
||||
assert!(
|
||||
bar_block.code.contains("@Log()"),
|
||||
"decorator must be folded into class-method unit (Python parity); got code: {:?}",
|
||||
bar_block.code
|
||||
);
|
||||
|
||||
// line_start must be 2 (the @Log() line), NOT 3 (the bar() line).
|
||||
match &bar_block.common.source_span {
|
||||
SourceSpan::Code { line_start, .. } => {
|
||||
assert_eq!(
|
||||
*line_start, 2,
|
||||
"line_start must cover the @Log() decorator line (got {line_start})"
|
||||
);
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Class-level decorator (preceding sibling of `class_declaration` in
|
||||
/// the module root): same `unit_start` backward walk folds it in.
|
||||
/// Line 1: "@Injectable()"
|
||||
/// Line 2: "class Service {"
|
||||
/// Line 3: "}"
|
||||
#[test]
|
||||
fn ts_class_decorator_folded_into_class_unit() {
|
||||
let bytes = b"@Injectable()\nclass Service {\n}\n";
|
||||
let asset = crate::rust::tests_support::fixed_code_asset("src/svc.ts", "typescript");
|
||||
let cfg = kebab_core::ExtractConfig::default();
|
||||
let root = std::path::PathBuf::from("/tmp");
|
||||
let ctx = kebab_core::ExtractContext {
|
||||
asset: &asset,
|
||||
workspace_root: &root,
|
||||
config: &cfg,
|
||||
};
|
||||
let doc = TypescriptAstExtractor::new().extract(&ctx, bytes).unwrap();
|
||||
|
||||
let svc_block = doc
|
||||
.blocks
|
||||
.iter()
|
||||
.find_map(|b| match b {
|
||||
Block::Code(c) => match &c.common.source_span {
|
||||
SourceSpan::Code { symbol, .. }
|
||||
if symbol.as_deref() == Some("src/svc.Service") =>
|
||||
{
|
||||
Some(c)
|
||||
}
|
||||
_ => None,
|
||||
},
|
||||
_ => None,
|
||||
})
|
||||
.expect("src/svc.Service block should be present");
|
||||
|
||||
assert!(
|
||||
svc_block.code.contains("@Injectable()"),
|
||||
"class-level decorator must be folded into the class unit; got code: {:?}",
|
||||
svc_block.code
|
||||
);
|
||||
match &svc_block.common.source_span {
|
||||
SourceSpan::Code { line_start, .. } => {
|
||||
assert_eq!(
|
||||
*line_start, 1,
|
||||
"line_start must cover the @Injectable() line (got {line_start})"
|
||||
);
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
}
|
||||
9
crates/kebab-parse-code/tests/fixtures/sample.js
vendored
Normal file
9
crates/kebab-parse-code/tests/fixtures/sample.js
vendored
Normal file
@@ -0,0 +1,9 @@
|
||||
// sample.js
|
||||
import { x } from "./other";
|
||||
const ANSWER = 42;
|
||||
export function add(a, b) { return a + b; }
|
||||
export class Retriever {
|
||||
search(q) { return []; }
|
||||
static create() { return new Retriever(); }
|
||||
}
|
||||
export default function () { return 1; }
|
||||
26
crates/kebab-parse-code/tests/fixtures/sample.py
vendored
Normal file
26
crates/kebab-parse-code/tests/fixtures/sample.py
vendored
Normal file
@@ -0,0 +1,26 @@
|
||||
"""sample fixture."""
|
||||
import os
|
||||
|
||||
ANSWER = 42
|
||||
|
||||
@no_type_check
|
||||
def free(x):
|
||||
"""free fn."""
|
||||
return x + 1
|
||||
|
||||
class Foo:
|
||||
"""doc."""
|
||||
def double(self, n):
|
||||
return n * 2
|
||||
|
||||
@classmethod
|
||||
def name(cls):
|
||||
return "foo"
|
||||
|
||||
class Outer:
|
||||
class Inner:
|
||||
def helper(self):
|
||||
return True
|
||||
|
||||
def with_decorator():
|
||||
pass
|
||||
35
crates/kebab-parse-code/tests/fixtures/sample.rs
vendored
Normal file
35
crates/kebab-parse-code/tests/fixtures/sample.rs
vendored
Normal file
@@ -0,0 +1,35 @@
|
||||
//! sample fixture
|
||||
|
||||
use std::fmt;
|
||||
|
||||
const ANSWER: u32 = 42;
|
||||
|
||||
/// Doc comment on a free fn.
|
||||
pub fn parse(input: &str) -> usize {
|
||||
input.len()
|
||||
}
|
||||
|
||||
pub struct Foo {
|
||||
pub n: u32,
|
||||
}
|
||||
|
||||
impl Foo {
|
||||
/// method doc
|
||||
pub fn double(&self) -> u32 {
|
||||
self.n * 2
|
||||
}
|
||||
|
||||
fn name() -> &'static str {
|
||||
"foo"
|
||||
}
|
||||
}
|
||||
|
||||
pub trait Greet {
|
||||
fn hello(&self) -> String;
|
||||
}
|
||||
|
||||
mod inner {
|
||||
pub fn helper() -> bool {
|
||||
true
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user