Compare commits
29 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 7a90df1485 | |||
| 46f408dc0f | |||
| 49e60fb314 | |||
| 6bc7a83d3c | |||
| df3c5b8caf | |||
| 5051ea7534 | |||
| 88d7fbc182 | |||
| 0b7d8af759 | |||
| 9342b9543f | |||
| a8aa03042f | |||
| 9d4a60aac5 | |||
| 8ce7a911ee | |||
| 75c1c7b911 | |||
| b5c12ecb6f | |||
| a1192ce3b2 | |||
| 17ee400fd5 | |||
| 217dddb4ba | |||
| 308666dbd5 | |||
| 522ae7b8bc | |||
| 166e1ddfaf | |||
| 226ce8b744 | |||
| 22d4161728 | |||
| 51004ac593 | |||
| 8996e73282 | |||
| 22dba09857 | |||
| aaa90b1754 | |||
| 077f92f41e | |||
| 5ce7f60932 | |||
| 47857b2622 |
47
Cargo.lock
generated
47
Cargo.lock
generated
@@ -4127,7 +4127,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-app"
|
||||
version = "0.13.0"
|
||||
version = "0.15.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"base64 0.22.1",
|
||||
@@ -4172,7 +4172,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-chunk"
|
||||
version = "0.13.0"
|
||||
version = "0.15.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"blake3",
|
||||
@@ -4181,13 +4181,14 @@ dependencies = [
|
||||
"kebab-parse-md",
|
||||
"serde_json",
|
||||
"serde_json_canonicalizer",
|
||||
"serde_yaml",
|
||||
"time",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "kebab-cli"
|
||||
version = "0.13.0"
|
||||
version = "0.15.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"clap",
|
||||
@@ -4208,7 +4209,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-config"
|
||||
version = "0.13.0"
|
||||
version = "0.15.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"dirs 5.0.1",
|
||||
@@ -4223,7 +4224,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-core"
|
||||
version = "0.13.0"
|
||||
version = "0.15.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"blake3",
|
||||
@@ -4237,7 +4238,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-embed"
|
||||
version = "0.13.0"
|
||||
version = "0.15.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"blake3",
|
||||
@@ -4251,7 +4252,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-embed-local"
|
||||
version = "0.13.0"
|
||||
version = "0.15.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"fastembed",
|
||||
@@ -4264,7 +4265,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-eval"
|
||||
version = "0.13.0"
|
||||
version = "0.15.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"kebab-app",
|
||||
@@ -4283,7 +4284,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-llm"
|
||||
version = "0.13.0"
|
||||
version = "0.15.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"kebab-core",
|
||||
@@ -4292,7 +4293,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-llm-local"
|
||||
version = "0.13.0"
|
||||
version = "0.15.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"kebab-config",
|
||||
@@ -4309,7 +4310,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-mcp"
|
||||
version = "0.13.0"
|
||||
version = "0.15.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"kebab-app",
|
||||
@@ -4327,7 +4328,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-normalize"
|
||||
version = "0.13.0"
|
||||
version = "0.15.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"kebab-core",
|
||||
@@ -4342,7 +4343,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-parse-code"
|
||||
version = "0.13.0"
|
||||
version = "0.15.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"gix",
|
||||
@@ -4363,7 +4364,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-parse-image"
|
||||
version = "0.13.0"
|
||||
version = "0.15.0"
|
||||
dependencies = [
|
||||
"ab_glyph",
|
||||
"anyhow",
|
||||
@@ -4387,7 +4388,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-parse-md"
|
||||
version = "0.13.0"
|
||||
version = "0.15.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"kebab-core",
|
||||
@@ -4404,7 +4405,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-parse-pdf"
|
||||
version = "0.13.0"
|
||||
version = "0.15.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"blake3",
|
||||
@@ -4417,7 +4418,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-parse-types"
|
||||
version = "0.13.0"
|
||||
version = "0.15.0"
|
||||
dependencies = [
|
||||
"kebab-core",
|
||||
"serde",
|
||||
@@ -4425,7 +4426,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-rag"
|
||||
version = "0.13.0"
|
||||
version = "0.15.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"blake3",
|
||||
@@ -4446,7 +4447,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-search"
|
||||
version = "0.13.0"
|
||||
version = "0.15.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"globset",
|
||||
@@ -4465,7 +4466,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-source-fs"
|
||||
version = "0.13.0"
|
||||
version = "0.15.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"blake3",
|
||||
@@ -4484,7 +4485,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-store-sqlite"
|
||||
version = "0.13.0"
|
||||
version = "0.15.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"blake3",
|
||||
@@ -4505,7 +4506,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-store-vector"
|
||||
version = "0.13.0"
|
||||
version = "0.15.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"arrow",
|
||||
@@ -4529,7 +4530,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "kebab-tui"
|
||||
version = "0.13.0"
|
||||
version = "0.15.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"crossterm",
|
||||
|
||||
@@ -31,7 +31,7 @@ edition = "2024"
|
||||
rust-version = "1.85"
|
||||
license = "MIT OR Apache-2.0"
|
||||
repository = "https://github.com/altair823/kebab"
|
||||
version = "0.13.0"
|
||||
version = "0.15.0"
|
||||
|
||||
[workspace.dependencies]
|
||||
anyhow = "1"
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
## 한 줄 요약
|
||||
|
||||
P0–P5 + P6 + P7 + P9-1/2/3/4 (Library / Search / Ask / Inspect) 머지 완료. `kebab ingest` 가 markdown / image / PDF / 소스코드 (Rust / Python / TS / JS / Go / Java / Kotlin) 처리. `kebab search` / `kebab ask` 가 매체 가로질러 결과 + page / code citation 반환. `kebab tui` 가 4 패널 (Library + Search + Ask + Inspect) 제공. P10-1C (Go + Java + Kotlin) 완료 — 다음 후보 = P10-1D (C/C++) 또는 P9-5 (desktop tauri) 또는 보류 중인 P8 (audio).
|
||||
P0–P5 + P6 + P7 + P9-1/2/3/4 (Library / Search / Ask / Inspect) 머지 완료. `kebab ingest` 가 markdown / image / PDF / 소스코드 (Rust / Python / TS / JS / Go / Java / Kotlin) / Tier 2 리소스 파일 (yaml/k8s / dockerfile / toml / json / xml / groovy / go-mod) + Tier 3 paragraph fallback (shell / 비-k8s YAML / AST 실패 케이스) 처리. `kebab search` / `kebab ask` 가 매체 가로질러 결과 + page / code citation 반환. `kebab tui` 가 4 패널 (Library + Search + Ask + Inspect) 제공. P10-3 (Tier 3 paragraph fallback) 완료 — 다음 후보 = P10-1D (C/C++) 또는 P9-5 (desktop tauri) 또는 보류 중인 P8 (audio).
|
||||
|
||||
## Phase 로드맵
|
||||
|
||||
@@ -20,7 +20,7 @@ P0–P5 + P6 + P7 + P9-1/2/3/4 (Library / Search / Ask / Inspect) 머지 완료.
|
||||
| **P7** | PDF text + page citation | `kebab-parse-pdf` | P5 | ✅ 완료 (3/3 component, page-level chunker + ingest wiring) |
|
||||
| **P8** | 음성 transcription + timestamp citation | `kebab-parse-audio` | P5 | ⏸ 보류 (whisper-rs 시스템 dep brainstorm 필요) |
|
||||
| **P9** | TUI + desktop app | `kebab-tui`, `kebab-desktop` | P5 | 🟡 진행 (4/5 component — P9-1/2/3/4 완료 [Library / Search / Ask / Inspect], P9-5 desktop 예정 · 도그푸딩 피드백 **20/20 ✅**) |
|
||||
| **P10** | code ingest framework | `kebab-parse-code` | P5 | 🟡 진행 중 — 1A-1 ✅ (wire schema + parse-code skeleton + filter flags), 1A-2 ✅ (Rust AST chunker, `code-rust-ast-v1` — v0.7.0), 1B ✅ (Python/TS/JS AST chunkers — v0.8.0 이후), **1C-Go ✅ (Go AST chunker, `code-go-ast-v1` — v0.12.0)**, **1C-JavaKotlin ✅ (Java + Kotlin AST chunkers, `code-java-ast-v1` / `code-kotlin-ast-v1` — v0.13.0)** |
|
||||
| **P10** | code ingest framework | `kebab-parse-code` | P5 | 🟡 진행 중 — 1A-1 ✅ (wire schema + parse-code skeleton + filter flags), 1A-2 ✅ (Rust AST chunker, `code-rust-ast-v1` — v0.7.0), 1B ✅ (Python/TS/JS AST chunkers — v0.8.0 이후), **1C-Go ✅ (Go AST chunker, `code-go-ast-v1` — v0.12.0)**, **1C-JavaKotlin ✅ (Java + Kotlin AST chunkers, `code-java-ast-v1` / `code-kotlin-ast-v1` — v0.13.0)**, **2 ✅ (Tier 2 resource-aware: yaml/k8s + dockerfile + manifest, `k8s-manifest-resource-v1` / `dockerfile-file-v1` / `manifest-file-v1` — v0.14.0)**, **3 ✅ (Tier 3 paragraph fallback: code-text-paragraph-v1 — v0.15.0)** |
|
||||
|
||||
P0~P5 직렬. P6~P9 P5 이후 병렬 가능.
|
||||
|
||||
|
||||
@@ -70,7 +70,7 @@ kebab doctor
|
||||
| 명령 | 동작 |
|
||||
|------|------|
|
||||
| `kebab init` | XDG 경로에 데이터 디렉토리 + config.toml 생성 |
|
||||
| `kebab ingest [<path>]` | Markdown / 이미지 / PDF / Rust 소스코드 색인 (idempotent). TTY 에서는 stderr 진행 바, non-TTY (CI / pipe) 는 stderr 한 줄씩, `--json` 은 stdout 에 `ingest_progress.v1` 라인 streaming 후 마지막에 `ingest_report.v1`. Ctrl-C 한 번이면 현재 asset 마무리 후 abort (부분 commit 보존, idempotent re-run), 두 번째 Ctrl-C 는 hard exit. Markdown title 이 frontmatter 에 없어도 첫 H1 → H2 → 첫 paragraph 80 자 → 파일명 순으로 자동 채움 (parser_version `md-frontmatter-v2`) — 기존 색인된 doc 도 다음 ingest 에서 새 title 로 갱신. **Incremental** (p9-fb-23): 두 번째 이후의 ingest 는 변하지 않은 doc (blake3 + parser/chunker/embedder version 모두 동일) 의 parse/chunk/embed/vector upsert 를 자동 스킵. final summary 에 `N unchanged` 카운트 표시. `--force-reingest` 로 skip 무시 강제 재처리. **지원 형식** (extractor 자동 결정 — config 에 명시 불가): Markdown (`.md`), 이미지 (`.png` / `.jpg` / `.jpeg`, OCR + caption), PDF (`.pdf`), **소스코드** (`.rs` → `code-rust-ast-v1`, `.py` → `code-python-ast-v1`, `.ts`/`.tsx` → `code-ts-ast-v1`, `.js`/`.mjs`/`.cjs`/`.jsx` → `code-js-ast-v1`, `.go` → `code-go-ast-v1`, `.java` → `code-java-ast-v1`, `.kt`/`.kts` → `code-kotlin-ast-v1` — 모두 tree-sitter AST chunker). 다른 확장자는 자동 skip — `IngestItem.warnings` 에 사유 (`"unsupported media type: .docx"` 등), `IngestReport.skipped_by_extension` 에 카운트 분류, CLI / TUI summary 에 breakdown 표시. 코드 chunk 는 `citation.kind = "code"` 에 `citation.lang = "<lang>"` + `symbol` + line range 를 담고, SearchHit top-level 에 `code_lang` + `repo` (`.git/` walk-up 의 디렉토리 이름) 가 backfill 됨. `--code-lang rust` / `--code-lang python` / `--code-lang typescript` / `--code-lang javascript` / `--code-lang go` / `--code-lang java` / `--code-lang kotlin` / `--media code` filter 로 언어별·코드 전용 검색 가능 (p10-1A-1 filter flags). Python symbol 은 workspace 경로 → dotted module path prefix (예: `kebab_eval.metrics.compute_mrr`), TS/JS symbol 은 slash-style module path prefix (예: `src/Foo.Foo.search`), Go symbol 은 `package.Func` / `package.(*Receiver).Method` 형식, Java / Kotlin symbol 은 `com.foo.Foo.bar` 형식 (패키지 + 클래스 + 메서드/필드). |
|
||||
| `kebab ingest [<path>]` | Markdown / 이미지 / PDF / Rust 소스코드 색인 (idempotent). TTY 에서는 stderr 진행 바, non-TTY (CI / pipe) 는 stderr 한 줄씩, `--json` 은 stdout 에 `ingest_progress.v1` 라인 streaming 후 마지막에 `ingest_report.v1`. Ctrl-C 한 번이면 현재 asset 마무리 후 abort (부분 commit 보존, idempotent re-run), 두 번째 Ctrl-C 는 hard exit. Markdown title 이 frontmatter 에 없어도 첫 H1 → H2 → 첫 paragraph 80 자 → 파일명 순으로 자동 채움 (parser_version `md-frontmatter-v2`) — 기존 색인된 doc 도 다음 ingest 에서 새 title 로 갱신. **Incremental** (p9-fb-23): 두 번째 이후의 ingest 는 변하지 않은 doc (blake3 + parser/chunker/embedder version 모두 동일) 의 parse/chunk/embed/vector upsert 를 자동 스킵. final summary 에 `N unchanged` 카운트 표시. `--force-reingest` 로 skip 무시 강제 재처리. **지원 형식** (extractor 자동 결정 — config 에 명시 불가): Markdown (`.md`), 이미지 (`.png` / `.jpg` / `.jpeg`, OCR + caption), PDF (`.pdf`), **소스코드** (`.rs` → `code-rust-ast-v1`, `.py` → `code-python-ast-v1`, `.ts`/`.tsx` → `code-ts-ast-v1`, `.js`/`.mjs`/`.cjs`/`.jsx` → `code-js-ast-v1`, `.go` → `code-go-ast-v1`, `.java` → `code-java-ast-v1`, `.kt`/`.kts` → `code-kotlin-ast-v1` — 모두 tree-sitter AST chunker; **Tier 2 리소스 파일**: `.yaml`/`.yml` → `k8s-manifest-resource-v1` (apiVersion+kind 파싱), `Dockerfile`/`Dockerfile.*`/`*.dockerfile` → `dockerfile-file-v1` (전체 파일), `Cargo.toml`/`pyproject.toml`/`.toml`/`package.json`/`tsconfig.json`/`.json`/`pom.xml`/`.xml`/`build.gradle`/`.gradle`/`go.mod` → `manifest-file-v1` (전체 파일) — yaml (k8s) / dockerfile / toml / json / xml / groovy / go-mod 지원); **Tier 3 paragraph fallback** (`.sh`/`.bash`/`.zsh` → `code-text-paragraph-v1`, blank-line paragraph split + 80-line/20-overlap line-window. Tier 1/2 가 0 chunk 또는 Err 시 자동 fallback — 비-k8s YAML 같은 케이스 picked up. symbol = None, lang 은 원본 보존.). 다른 확장자는 자동 skip — `IngestItem.warnings` 에 사유 (`"unsupported media type: .docx"` 등), `IngestReport.skipped_by_extension` 에 카운트 분류, CLI / TUI summary 에 breakdown 표시. 코드 chunk 는 `citation.kind = "code"` 에 `citation.lang = "<lang>"` + `symbol` + line range 를 담고, SearchHit top-level 에 `code_lang` + `repo` (`.git/` walk-up 의 디렉토리 이름) 가 backfill 됨. `--code-lang rust` / `--code-lang python` / `--code-lang typescript` / `--code-lang javascript` / `--code-lang go` / `--code-lang java` / `--code-lang kotlin` / `--code-lang yaml` / `--code-lang dockerfile` / `--code-lang toml` / `--code-lang json` / `--code-lang xml` / `--code-lang groovy` / `--code-lang go-mod` / `--code-lang shell` / `--media code` filter 로 언어별·코드 전용 검색 가능 (p10-1A-1 filter flags). Python symbol 은 workspace 경로 → dotted module path prefix (예: `kebab_eval.metrics.compute_mrr`), TS/JS symbol 은 slash-style module path prefix (예: `src/Foo.Foo.search`), Go symbol 은 `package.Func` / `package.(*Receiver).Method` 형식, Java / Kotlin symbol 은 `com.foo.Foo.bar` 형식 (패키지 + 클래스 + 메서드/필드). |
|
||||
| `kebab search --mode {lexical,vector,hybrid} "<query>" [--no-cache] [--max-tokens N] [--snippet-chars N] [--cursor <opaque>] [--tag T] [--lang L] [--path-glob G] [--trust-min LEVEL] [--media TYPE] [--ingested-after RFC3339] [--doc-id ID] [--trace] [--bulk] [--repo NAME ...] [--code-lang LIST]` | 검색. hybrid는 RRF fusion, citation 포함. 같은 process 안에서 동일 query (NFKC + trim + lowercase 정규화) 반복 시 in-process LRU 캐시 hit (capacity = `[search] cache_capacity`, default 256). `--no-cache` 로 강제 bypass — 디버깅용. ingest commit 발생 시 `kv['corpus_revision']` bump 으로 모든 entry 자동 stale. **`--max-tokens` / `--snippet-chars` / `--cursor` (p9-fb-34)** — agent budget controls. `--json` 출력은 `search_response.v1` wrapper (`{hits, next_cursor, truncated}`) — pre-fb-34 의 bare array 와 호환 안 됨. mismatched cursor → `error.v1.code = stale_cursor`. **filter flags (p9-fb-36):** `--tag` 는 반복 가능 flag (`--tag rust --tag async`) 로 OR 매칭, `--media` 는 `,` 구분 다중 값 OR 매칭, 나머지 flags 간은 AND 조합. `--trust-min` 은 `primary\|secondary\|generated` 중 하나 (해당 level 이상 포함). `--ingested-after` 는 RFC3339 UTC — 파싱 실패 시 `error.v1.code = config_invalid` (exit 2). `--media md` 는 `markdown` alias 로 정규화. 알 수 없는 `--media` 값은 무조건 empty hits (오류 아님). **`--trace` (p9-fb-37)** — `search_response.v1.trace` 에 lexical / vector pre-fusion 후보 + RRF union + per-stage timing (`lexical_ms` / `vector_ms` / `fusion_ms` / `total_ms`) 노출. trace 요청은 캐시 우회 (`--no-cache` 없이도 항상 cold). **`--bulk` (p9-fb-42)** — stdin ndjson 으로 N query 한 번에 실행. `--json` 면 stdout per-query ndjson (`bulk_search_item.v1`) + stderr summary (`bulk_summary: total=N succeeded=S failed=F`). Cap 100. agent 가 query decomposition 후 sub-query 일괄 실행 시 single round-trip — App instance 재사용으로 캐시 / embedder cold-start 비용 한 번만. Per-query failure 는 item 의 `error` (error.v1) 에 격리, 다른 query 계속 진행. **code corpus filters (p10-1A-1):** `--repo` 는 반복 가능 (`--repo kebab --repo other`) OR 매칭. `--code-lang` 는 반복 또는 comma 다중 값 (`--code-lang rust,python`), 알 수 없는 값은 빈 hits. `--media code` 는 Tier 1/2/3 모든 code chunk 포함. 1A-1 시점에서는 indexed 된 code chunk 가 없어 filter 가 항상 빈 결과 — 1A-2 (Rust AST chunker) 머지 이후 실효. |
|
||||
| `kebab list docs` | 색인된 문서 목록 |
|
||||
| `kebab inspect doc <id>` / `kebab inspect chunk <id>` | raw record 보기 |
|
||||
@@ -132,7 +132,7 @@ flowchart TB
|
||||
|
||||
subgraph Pipeline["도메인 + 파이프라인"]
|
||||
parse["parse-md / parse-pdf / parse-image / parse-code"]
|
||||
chunker["chunker (md-heading-v1, pdf-page-v1, code-rust-ast-v1, code-python-ast-v1, code-ts-ast-v1, code-js-ast-v1, code-go-ast-v1)"]
|
||||
chunker["chunker (md-heading-v1, pdf-page-v1, code-{rust,python,ts,js,go,java,kotlin}-ast-v1, k8s-manifest-resource-v1, dockerfile-file-v1, manifest-file-v1, code-text-paragraph-v1)"]
|
||||
embedder["embedder (fastembed multilingual-e5-large)"]
|
||||
retriever["retriever (lexical / vector / hybrid RRF)"]
|
||||
rag["RAG pipeline"]
|
||||
|
||||
@@ -39,7 +39,7 @@ use std::sync::Arc;
|
||||
use anyhow::{Context, anyhow};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use kebab_chunk::{CodeGoAstV1Chunker, CodeJavaAstV1Chunker, CodeJsAstV1Chunker, CodeKotlinAstV1Chunker, CodePythonAstV1Chunker, CodeRustAstV1Chunker, CodeTsAstV1Chunker, MdHeadingV1Chunker, PdfPageV1Chunker};
|
||||
use kebab_chunk::{CodeGoAstV1Chunker, CodeJavaAstV1Chunker, CodeJsAstV1Chunker, CodeKotlinAstV1Chunker, CodePythonAstV1Chunker, CodeRustAstV1Chunker, CodeTextParagraphV1Chunker, CodeTsAstV1Chunker, DockerfileFileV1Chunker, K8sManifestResourceV1Chunker, ManifestFileV1Chunker, MdHeadingV1Chunker, PdfPageV1Chunker};
|
||||
use kebab_core::{
|
||||
Answer, Block, CanonicalDocument, Chunk, ChunkId, ChunkPolicy, ChunkerVersion, Chunker,
|
||||
DocFilter, DocSummary, DocumentId, DocumentStore, Embedder, EmbeddingInput,
|
||||
@@ -948,10 +948,12 @@ fn ingest_one_asset(
|
||||
force_reingest,
|
||||
);
|
||||
}
|
||||
// p10-1A-2 / 1B: code ingest dispatch.
|
||||
// p10-1A-2 / 1B: code ingest dispatch. p10-2: Tier 2 langs added. p10-3: shell added.
|
||||
MediaType::Code(lang)
|
||||
if matches!(lang.as_str(),
|
||||
"rust" | "python" | "typescript" | "javascript" | "go" | "java" | "kotlin") =>
|
||||
"rust" | "python" | "typescript" | "javascript" | "go" | "java" | "kotlin"
|
||||
| "yaml" | "dockerfile" | "toml" | "json" | "xml" | "groovy" | "go-mod"
|
||||
| "shell") =>
|
||||
{
|
||||
return ingest_one_code_asset(
|
||||
app,
|
||||
@@ -1831,18 +1833,30 @@ fn ingest_one_code_asset(
|
||||
"go" => ParserVersion(kebab_parse_code::GO_PARSER_VERSION.to_string()),
|
||||
"java" => ParserVersion(kebab_parse_code::JAVA_PARSER_VERSION.to_string()),
|
||||
"kotlin" => ParserVersion(kebab_parse_code::KOTLIN_PARSER_VERSION.to_string()),
|
||||
// p10-2: Tier 2 has no parse step — sentinel "none-v1".
|
||||
"yaml" | "dockerfile" | "toml" | "json" | "xml" | "groovy" | "go-mod"
|
||||
=> ParserVersion("none-v1".to_string()),
|
||||
// p10-3: shell direct routes to Tier 3 (no parse step).
|
||||
"shell" => ParserVersion("none-v1".to_string()),
|
||||
other => anyhow::bail!("unsupported code_lang: {other}"),
|
||||
};
|
||||
|
||||
// p10-1b Task D/G/J/L: chunker_version per-lang.
|
||||
let chunker_version = match code_lang {
|
||||
let mut chunker_version = match code_lang {
|
||||
"rust" => CodeRustAstV1Chunker.chunker_version(),
|
||||
"python" => CodePythonAstV1Chunker.chunker_version(),
|
||||
"typescript" => CodeTsAstV1Chunker.chunker_version(),
|
||||
"javascript" => CodeJsAstV1Chunker.chunker_version(),
|
||||
"go" => CodeGoAstV1Chunker.chunker_version(),
|
||||
"java" => CodeJavaAstV1Chunker.chunker_version(),
|
||||
"kotlin" => CodeKotlinAstV1Chunker.chunker_version(),
|
||||
"kotlin" => CodeKotlinAstV1Chunker.chunker_version(),
|
||||
// p10-2 Tier 2:
|
||||
"yaml" => K8sManifestResourceV1Chunker.chunker_version(),
|
||||
"dockerfile" => DockerfileFileV1Chunker.chunker_version(),
|
||||
"toml" | "json" | "xml" | "groovy" | "go-mod"
|
||||
=> ManifestFileV1Chunker.chunker_version(),
|
||||
// p10-3:
|
||||
"shell" => CodeTextParagraphV1Chunker.chunker_version(),
|
||||
other => anyhow::bail!("unreachable chunker_version: {other}"),
|
||||
};
|
||||
|
||||
@@ -1868,55 +1882,145 @@ fn ingest_one_code_asset(
|
||||
};
|
||||
|
||||
// p10-1b Task D/G/J/L: extractor per-lang.
|
||||
let mut canonical = match code_lang {
|
||||
// p10-3: capture Result so Tier 1 extractor errors can fall back to Tier 3.
|
||||
let canonical_result: anyhow::Result<kebab_core::CanonicalDocument> = match code_lang {
|
||||
"rust" => RustAstExtractor::new()
|
||||
.extract(&ctx, &bytes)
|
||||
.context("kb-parse-code::RustAstExtractor::extract (code:rust)")?,
|
||||
.context("kb-parse-code::RustAstExtractor::extract (code:rust)"),
|
||||
"python" => PythonAstExtractor::new()
|
||||
.extract(&ctx, &bytes)
|
||||
.context("kb-parse-code::PythonAstExtractor::extract (code:python)")?,
|
||||
.context("kb-parse-code::PythonAstExtractor::extract (code:python)"),
|
||||
"typescript" => TypescriptAstExtractor::new()
|
||||
.extract(&ctx, &bytes)
|
||||
.context("kb-parse-code::TypescriptAstExtractor::extract (code:typescript)")?,
|
||||
.context("kb-parse-code::TypescriptAstExtractor::extract (code:typescript)"),
|
||||
"javascript" => JavascriptAstExtractor::new()
|
||||
.extract(&ctx, &bytes)
|
||||
.context("kb-parse-code::JavascriptAstExtractor::extract (code:javascript)")?,
|
||||
.context("kb-parse-code::JavascriptAstExtractor::extract (code:javascript)"),
|
||||
"go" => GoAstExtractor::new()
|
||||
.extract(&ctx, &bytes)
|
||||
.context("kb-parse-code::GoAstExtractor::extract (code:go)")?,
|
||||
.context("kb-parse-code::GoAstExtractor::extract (code:go)"),
|
||||
"java" => JavaAstExtractor::new()
|
||||
.extract(&ctx, &bytes)
|
||||
.context("kb-parse-code::JavaAstExtractor::extract (code:java)")?,
|
||||
.context("kb-parse-code::JavaAstExtractor::extract (code:java)"),
|
||||
"kotlin" => KotlinAstExtractor::new()
|
||||
.extract(&ctx, &bytes)
|
||||
.context("kb-parse-code::KotlinAstExtractor::extract (code:kotlin)")?,
|
||||
.context("kb-parse-code::KotlinAstExtractor::extract (code:kotlin)"),
|
||||
// p10-2 Tier 2: no extractor — synthesize Document directly from raw bytes.
|
||||
"yaml" | "dockerfile" | "toml" | "json" | "xml" | "groovy" | "go-mod" => {
|
||||
synthesize_tier2_document(asset, &bytes, code_lang, &parser_version)
|
||||
}
|
||||
// p10-3: shell reuses the same synthesizer.
|
||||
"shell" => synthesize_tier2_document(asset, &bytes, "shell", &parser_version),
|
||||
other => anyhow::bail!("unreachable (extract): {other}"),
|
||||
};
|
||||
|
||||
// p10-3: Tier 1 extractor failure → fall back to Tier 3 synthesized doc.
|
||||
// Tier 2 (yaml/dockerfile/…) and shell errors are real (e.g. non-UTF-8) — propagate.
|
||||
let mut canonical = match canonical_result {
|
||||
Ok(d) => d,
|
||||
Err(e) if code_lang == "shell"
|
||||
|| matches!(code_lang, "yaml" | "dockerfile" | "toml" | "json" | "xml" | "groovy" | "go-mod") =>
|
||||
{
|
||||
return Err(e).context("synthesize_tier2_document failed for tier 2/3 lang");
|
||||
}
|
||||
Err(e) => {
|
||||
// Tier 1 extractor errored — fall back to Tier 3 synthesized doc.
|
||||
tracing::warn!(
|
||||
workspace_path = %asset.workspace_path.0,
|
||||
code_lang = code_lang,
|
||||
error = %e,
|
||||
"tier1 extract errored; falling back to tier 3 synthesized doc"
|
||||
);
|
||||
chunker_version = CodeTextParagraphV1Chunker.chunker_version();
|
||||
let tier3_parser_version = ParserVersion("none-v1".to_string());
|
||||
synthesize_tier2_document(asset, &bytes, code_lang, &tier3_parser_version)
|
||||
.context("synthesize_tier2_document for tier 3 fallback after extract error")?
|
||||
}
|
||||
};
|
||||
|
||||
// p10-1b Task D/G/J/L: chunker per-lang.
|
||||
let chunks = match code_lang {
|
||||
"rust" => CodeRustAstV1Chunker
|
||||
// p10-3: track whether the extract stage already fell back to Tier 3.
|
||||
// Tier 2 langs already have "none-v1" parser_version normally, so exclude them
|
||||
// from the extract_fell_back guard with the !matches! exclusion.
|
||||
let extract_fell_back = canonical.parser_version.0 == "none-v1"
|
||||
&& !matches!(code_lang, "yaml" | "dockerfile" | "toml" | "json" | "xml" | "groovy" | "go-mod" | "shell");
|
||||
|
||||
let chunks_result: anyhow::Result<Vec<Chunk>> = if extract_fell_back {
|
||||
// Tier 1 lang whose extractor errored — go straight to Tier 3 chunker.
|
||||
CodeTextParagraphV1Chunker
|
||||
.chunk(&canonical, chunk_policy)
|
||||
.context("kb-chunk::CodeRustAstV1Chunker::chunk (code:rust)")?,
|
||||
"python" => CodePythonAstV1Chunker
|
||||
.chunk(&canonical, chunk_policy)
|
||||
.context("kb-chunk::CodePythonAstV1Chunker::chunk (code:python)")?,
|
||||
"typescript" => CodeTsAstV1Chunker
|
||||
.chunk(&canonical, chunk_policy)
|
||||
.context("kb-chunk::CodeTsAstV1Chunker::chunk (code:typescript)")?,
|
||||
"javascript" => CodeJsAstV1Chunker
|
||||
.chunk(&canonical, chunk_policy)
|
||||
.context("kb-chunk::CodeJsAstV1Chunker::chunk (code:javascript)")?,
|
||||
"go" => CodeGoAstV1Chunker
|
||||
.chunk(&canonical, chunk_policy)
|
||||
.context("kb-chunk::CodeGoAstV1Chunker::chunk (code:go)")?,
|
||||
"java" => CodeJavaAstV1Chunker
|
||||
.chunk(&canonical, chunk_policy)
|
||||
.context("kb-chunk::CodeJavaAstV1Chunker::chunk (code:java)")?,
|
||||
"kotlin" => CodeKotlinAstV1Chunker
|
||||
.chunk(&canonical, chunk_policy)
|
||||
.context("kb-chunk::CodeKotlinAstV1Chunker::chunk (code:kotlin)")?,
|
||||
other => anyhow::bail!("unreachable (chunk): {other}"),
|
||||
.context("kb-chunk::CodeTextParagraphV1Chunker::chunk (tier 3 after extract fallback)")
|
||||
} else {
|
||||
match code_lang {
|
||||
"rust" => CodeRustAstV1Chunker
|
||||
.chunk(&canonical, chunk_policy)
|
||||
.context("kb-chunk::CodeRustAstV1Chunker::chunk (code:rust)"),
|
||||
"python" => CodePythonAstV1Chunker
|
||||
.chunk(&canonical, chunk_policy)
|
||||
.context("kb-chunk::CodePythonAstV1Chunker::chunk (code:python)"),
|
||||
"typescript" => CodeTsAstV1Chunker
|
||||
.chunk(&canonical, chunk_policy)
|
||||
.context("kb-chunk::CodeTsAstV1Chunker::chunk (code:typescript)"),
|
||||
"javascript" => CodeJsAstV1Chunker
|
||||
.chunk(&canonical, chunk_policy)
|
||||
.context("kb-chunk::CodeJsAstV1Chunker::chunk (code:javascript)"),
|
||||
"go" => CodeGoAstV1Chunker
|
||||
.chunk(&canonical, chunk_policy)
|
||||
.context("kb-chunk::CodeGoAstV1Chunker::chunk (code:go)"),
|
||||
"java" => CodeJavaAstV1Chunker
|
||||
.chunk(&canonical, chunk_policy)
|
||||
.context("kb-chunk::CodeJavaAstV1Chunker::chunk (code:java)"),
|
||||
"kotlin" => CodeKotlinAstV1Chunker
|
||||
.chunk(&canonical, chunk_policy)
|
||||
.context("kb-chunk::CodeKotlinAstV1Chunker::chunk (code:kotlin)"),
|
||||
// p10-2 Tier 2:
|
||||
"yaml" => K8sManifestResourceV1Chunker
|
||||
.chunk(&canonical, chunk_policy)
|
||||
.context("kb-chunk::K8sManifestResourceV1Chunker::chunk"),
|
||||
"dockerfile" => DockerfileFileV1Chunker
|
||||
.chunk(&canonical, chunk_policy)
|
||||
.context("kb-chunk::DockerfileFileV1Chunker::chunk"),
|
||||
"toml" | "json" | "xml" | "groovy" | "go-mod" => ManifestFileV1Chunker
|
||||
.chunk(&canonical, chunk_policy)
|
||||
.context("kb-chunk::ManifestFileV1Chunker::chunk"),
|
||||
// p10-3:
|
||||
"shell" => CodeTextParagraphV1Chunker
|
||||
.chunk(&canonical, chunk_policy)
|
||||
.context("kb-chunk::CodeTextParagraphV1Chunker::chunk (code:shell)"),
|
||||
other => anyhow::bail!("unreachable (chunk): {other}"),
|
||||
}
|
||||
};
|
||||
|
||||
// p10-3: Tier 1/2 0-chunk OR error → Tier 3 fallback retry.
|
||||
// "shell" direct path is already Tier 3 — don't retry-double-up.
|
||||
let chunks: Vec<Chunk> = match chunks_result {
|
||||
Ok(v) if !v.is_empty() => v,
|
||||
other if code_lang == "shell" => other?, // shell propagates directly
|
||||
Ok(_empty) => {
|
||||
tracing::warn!(
|
||||
workspace_path = %asset.workspace_path.0,
|
||||
code_lang = code_lang,
|
||||
"tier1/2 emitted 0 chunks; falling back to tier 3 (code-text-paragraph-v1)"
|
||||
);
|
||||
chunker_version = CodeTextParagraphV1Chunker.chunker_version();
|
||||
canonical.parser_version = ParserVersion("none-v1".to_string());
|
||||
CodeTextParagraphV1Chunker
|
||||
.chunk(&canonical, chunk_policy)
|
||||
.context("kb-chunk::CodeTextParagraphV1Chunker::chunk (tier 3 fallback)")?
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
workspace_path = %asset.workspace_path.0,
|
||||
code_lang = code_lang,
|
||||
error = %e,
|
||||
"tier1/2 chunker errored; falling back to tier 3 (code-text-paragraph-v1)"
|
||||
);
|
||||
chunker_version = CodeTextParagraphV1Chunker.chunker_version();
|
||||
canonical.parser_version = ParserVersion("none-v1".to_string());
|
||||
CodeTextParagraphV1Chunker
|
||||
.chunk(&canonical, chunk_policy)
|
||||
.context("kb-chunk::CodeTextParagraphV1Chunker::chunk (tier 3 fallback after error)")?
|
||||
}
|
||||
};
|
||||
|
||||
// Stamp chunker + embedding versions so incremental skip detection has
|
||||
@@ -2011,6 +2115,139 @@ fn ingest_one_code_asset(
|
||||
})
|
||||
}
|
||||
|
||||
/// p10-2: Build a minimal [`CanonicalDocument`] for Tier 2 code assets
|
||||
/// (yaml / dockerfile / toml / json / xml / groovy / go-mod) that have
|
||||
/// no AST extractor. Produces a single `Block::Code` whose source span
|
||||
/// covers the entire file, mirroring the shape the Tier 1 extractors
|
||||
/// produce for glue / top-level regions.
|
||||
fn synthesize_tier2_document(
|
||||
asset: &RawAsset,
|
||||
bytes: &[u8],
|
||||
code_lang: &str,
|
||||
parser_version: &ParserVersion,
|
||||
) -> anyhow::Result<kebab_core::CanonicalDocument> {
|
||||
use anyhow::Context as _;
|
||||
use kebab_core::{
|
||||
BlockId, CodeBlock, CommonBlock, Lang, Metadata, Provenance, ProvenanceEvent,
|
||||
ProvenanceKind, SourceSpan, SourceType, TrustLevel, id_for_block, id_for_doc,
|
||||
};
|
||||
|
||||
let text = std::str::from_utf8(bytes)
|
||||
.with_context(|| format!("tier2 doc not utf-8: {}", asset.workspace_path.0))?
|
||||
.to_string();
|
||||
|
||||
let doc_id = id_for_doc(&asset.workspace_path, &asset.asset_id, parser_version);
|
||||
|
||||
let n_lines = text.lines().count().max(1) as u32;
|
||||
let span = SourceSpan::Code {
|
||||
line_start: 1,
|
||||
line_end: n_lines,
|
||||
symbol: Some("<file>".to_string()),
|
||||
lang: Some(code_lang.to_string()),
|
||||
};
|
||||
let block_id: BlockId = id_for_block(
|
||||
&doc_id,
|
||||
"code",
|
||||
&[],
|
||||
0,
|
||||
&span,
|
||||
);
|
||||
let block = kebab_core::Block::Code(CodeBlock {
|
||||
common: CommonBlock {
|
||||
block_id,
|
||||
heading_path: vec![],
|
||||
source_span: span,
|
||||
},
|
||||
lang: Some(code_lang.to_string()),
|
||||
code: text,
|
||||
});
|
||||
|
||||
let now = time::OffsetDateTime::now_utc();
|
||||
let events = vec![
|
||||
ProvenanceEvent {
|
||||
at: asset.discovered_at,
|
||||
agent: "kb-source-fs".to_string(),
|
||||
kind: ProvenanceKind::Discovered,
|
||||
note: None,
|
||||
},
|
||||
ProvenanceEvent {
|
||||
at: now,
|
||||
agent: "kb-app".to_string(),
|
||||
kind: ProvenanceKind::Parsed,
|
||||
note: Some(format!(
|
||||
"parser_version={}; tier2_synthesized; lang={}",
|
||||
parser_version.0, code_lang
|
||||
)),
|
||||
},
|
||||
];
|
||||
|
||||
// Resolve absolute path for repo detection. FsSourceConnector always
|
||||
// emits absolute paths in SourceUri::File (verified in connector.rs); Kb
|
||||
// URIs were rejected earlier in ingest_one_code_asset (returns Skipped),
|
||||
// so the fallback below is purely defensive. This does NOT mirror
|
||||
// RustAstExtractor — that extractor joins ctx.workspace_root for relative
|
||||
// paths, but Tier 2 trusts the connector invariant.
|
||||
let abs_path = match &asset.source_uri {
|
||||
kebab_core::SourceUri::File(p) => p.clone(),
|
||||
kebab_core::SourceUri::Kb(_) => std::path::PathBuf::new(),
|
||||
};
|
||||
let (repo, git_branch, git_commit) = match kebab_parse_code::detect_repo(&abs_path) {
|
||||
Some(r) => (Some(r.name), r.branch, r.commit),
|
||||
None => (None, None, None),
|
||||
};
|
||||
|
||||
let title = {
|
||||
let fname = asset.workspace_path.0
|
||||
.rsplit('/')
|
||||
.next()
|
||||
.unwrap_or(&asset.workspace_path.0);
|
||||
// strip extension
|
||||
match fname.rfind('.') {
|
||||
Some(i) => fname[..i].to_string(),
|
||||
None => fname.to_string(),
|
||||
}
|
||||
};
|
||||
|
||||
let metadata = Metadata {
|
||||
aliases: vec![],
|
||||
tags: vec![],
|
||||
created_at: asset.discovered_at,
|
||||
updated_at: asset.discovered_at,
|
||||
source_type: SourceType::Note,
|
||||
trust_level: TrustLevel::Primary,
|
||||
user_id_alias: None,
|
||||
user: serde_json::Map::new(),
|
||||
repo,
|
||||
git_branch,
|
||||
git_commit,
|
||||
code_lang: Some(code_lang.to_string()),
|
||||
};
|
||||
|
||||
tracing::debug!(
|
||||
target: "kebab-app",
|
||||
"synthesized tier2 doc_id={} workspace_path={} lang={}",
|
||||
doc_id.0,
|
||||
asset.workspace_path.0,
|
||||
code_lang,
|
||||
);
|
||||
|
||||
Ok(kebab_core::CanonicalDocument {
|
||||
doc_id,
|
||||
source_asset_id: asset.asset_id.clone(),
|
||||
workspace_path: asset.workspace_path.clone(),
|
||||
title,
|
||||
lang: Lang("und".to_string()),
|
||||
blocks: vec![block],
|
||||
metadata,
|
||||
provenance: Provenance { events },
|
||||
parser_version: parser_version.clone(),
|
||||
schema_version: 1,
|
||||
doc_version: 1,
|
||||
last_chunker_version: None,
|
||||
last_embedding_version: None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Pull the BCP-47 language hint from the canonical document. P6-1
|
||||
/// stamps `Lang("und")` by default; image-pipeline OCR / caption
|
||||
/// adapters special-case "und" so the hint is intentionally dropped
|
||||
|
||||
@@ -603,6 +603,428 @@ fn kotlin_file_ingests_and_searches_as_code_citation() {
|
||||
);
|
||||
}
|
||||
|
||||
/// p10-2 Task H: a `k8s/deploy.yaml` file with a Deployment resource is
|
||||
/// ingested and the resulting `Citation::Code` hit must carry
|
||||
/// `lang="yaml"`, `symbol="Deployment/prod/api"`, and `line_start >= 1`.
|
||||
/// Exercises the k8s-manifest-resource-v1 chunker end-to-end.
|
||||
#[test]
|
||||
fn tier2_k8s_yaml_ingest_searchable() {
|
||||
let env = TestEnv::lexical_only();
|
||||
|
||||
let k8s_dir = env.workspace_root.join("k8s");
|
||||
std::fs::create_dir_all(&k8s_dir).unwrap();
|
||||
std::fs::write(
|
||||
k8s_dir.join("deploy.yaml"),
|
||||
"apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: api\n namespace: prod\nspec:\n replicas: 1\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let report = kebab_app::ingest_with_config(env.config.clone(), env.scope(), false)
|
||||
.expect("ingest must succeed");
|
||||
assert_eq!(report.errors, 0, "no ingest errors: {report:?}");
|
||||
assert!(report.new >= 1, "yaml file ingested: {report:?}");
|
||||
|
||||
let yaml_item = report
|
||||
.items
|
||||
.as_ref()
|
||||
.expect("items present")
|
||||
.iter()
|
||||
.find(|i| i.doc_path.0.ends_with("deploy.yaml"))
|
||||
.expect("deploy.yaml item present");
|
||||
assert_eq!(
|
||||
yaml_item.parser_version.as_ref().map(|p| p.0.as_str()),
|
||||
Some("none-v1"),
|
||||
"parser_version must be none-v1"
|
||||
);
|
||||
assert_eq!(
|
||||
yaml_item.chunker_version.as_ref().map(|c| c.0.as_str()),
|
||||
Some("k8s-manifest-resource-v1"),
|
||||
"chunker_version must be k8s-manifest-resource-v1"
|
||||
);
|
||||
|
||||
let query = kebab_core::SearchQuery {
|
||||
text: "api".to_string(),
|
||||
mode: kebab_core::SearchMode::Lexical,
|
||||
k: 10,
|
||||
filters: kebab_core::SearchFilters {
|
||||
code_lang: vec!["yaml".to_string()],
|
||||
..Default::default()
|
||||
},
|
||||
};
|
||||
let hits = kebab_app::search_with_config(env.config.clone(), query)
|
||||
.expect("search must succeed");
|
||||
|
||||
let h = hits
|
||||
.iter()
|
||||
.find(|h| matches!(&h.citation, Citation::Code { .. }))
|
||||
.expect("at least one Citation::Code hit for 'api'");
|
||||
|
||||
match &h.citation {
|
||||
Citation::Code {
|
||||
lang,
|
||||
symbol,
|
||||
line_start,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(lang.as_deref(), Some("yaml"), "citation.lang must be 'yaml'");
|
||||
assert_eq!(
|
||||
symbol.as_deref(),
|
||||
Some("Deployment/prod/api"),
|
||||
"citation.symbol must be 'Deployment/prod/api'"
|
||||
);
|
||||
assert!(*line_start >= 1, "line_start must be >=1");
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
h.code_lang.as_deref(),
|
||||
Some("yaml"),
|
||||
"SearchHit.code_lang must be 'yaml'"
|
||||
);
|
||||
}
|
||||
|
||||
/// p10-2 Task H: a `Dockerfile` is ingested and the resulting
|
||||
/// `Citation::Code` hit must carry `lang="dockerfile"`,
|
||||
/// `symbol="<dockerfile>"`, and `line_start >= 1`.
|
||||
/// Exercises the dockerfile-file-v1 chunker end-to-end.
|
||||
#[test]
|
||||
fn tier2_dockerfile_ingest_searchable() {
|
||||
let env = TestEnv::lexical_only();
|
||||
|
||||
std::fs::write(
|
||||
env.workspace_root.join("Dockerfile"),
|
||||
"FROM rust:1.94\nRUN cargo install foo\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let report = kebab_app::ingest_with_config(env.config.clone(), env.scope(), false)
|
||||
.expect("ingest must succeed");
|
||||
assert_eq!(report.errors, 0, "no ingest errors: {report:?}");
|
||||
assert!(report.new >= 1, "Dockerfile ingested: {report:?}");
|
||||
|
||||
let df_item = report
|
||||
.items
|
||||
.as_ref()
|
||||
.expect("items present")
|
||||
.iter()
|
||||
.find(|i| i.doc_path.0.ends_with("Dockerfile"))
|
||||
.expect("Dockerfile item present");
|
||||
assert_eq!(
|
||||
df_item.parser_version.as_ref().map(|p| p.0.as_str()),
|
||||
Some("none-v1"),
|
||||
"parser_version must be none-v1"
|
||||
);
|
||||
assert_eq!(
|
||||
df_item.chunker_version.as_ref().map(|c| c.0.as_str()),
|
||||
Some("dockerfile-file-v1"),
|
||||
"chunker_version must be dockerfile-file-v1"
|
||||
);
|
||||
|
||||
let query = kebab_core::SearchQuery {
|
||||
text: "cargo".to_string(),
|
||||
mode: kebab_core::SearchMode::Lexical,
|
||||
k: 10,
|
||||
filters: kebab_core::SearchFilters {
|
||||
code_lang: vec!["dockerfile".to_string()],
|
||||
..Default::default()
|
||||
},
|
||||
};
|
||||
let hits = kebab_app::search_with_config(env.config.clone(), query)
|
||||
.expect("search must succeed");
|
||||
|
||||
let h = hits
|
||||
.iter()
|
||||
.find(|h| matches!(&h.citation, Citation::Code { .. }))
|
||||
.expect("at least one Citation::Code hit for 'cargo'");
|
||||
|
||||
match &h.citation {
|
||||
Citation::Code {
|
||||
lang,
|
||||
symbol,
|
||||
line_start,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(
|
||||
lang.as_deref(),
|
||||
Some("dockerfile"),
|
||||
"citation.lang must be 'dockerfile'"
|
||||
);
|
||||
assert_eq!(
|
||||
symbol.as_deref(),
|
||||
Some("<dockerfile>"),
|
||||
"citation.symbol must be '<dockerfile>'"
|
||||
);
|
||||
assert!(*line_start >= 1, "line_start must be >=1");
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
h.code_lang.as_deref(),
|
||||
Some("dockerfile"),
|
||||
"SearchHit.code_lang must be 'dockerfile'"
|
||||
);
|
||||
}
|
||||
|
||||
/// p10-2 Task H: a `Cargo.toml` manifest is ingested and the resulting
|
||||
/// `Citation::Code` hit must carry `lang="toml"`, `symbol="<manifest>"`,
|
||||
/// and `line_start >= 1`.
|
||||
/// Exercises the manifest-file-v1 chunker end-to-end.
|
||||
#[test]
|
||||
fn tier2_cargo_toml_ingest_searchable() {
|
||||
let env = TestEnv::lexical_only();
|
||||
|
||||
std::fs::write(
|
||||
env.workspace_root.join("Cargo.toml"),
|
||||
"[package]\nname = \"demo\"\nversion = \"0.1.0\"\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let report = kebab_app::ingest_with_config(env.config.clone(), env.scope(), false)
|
||||
.expect("ingest must succeed");
|
||||
assert_eq!(report.errors, 0, "no ingest errors: {report:?}");
|
||||
assert!(report.new >= 1, "Cargo.toml ingested: {report:?}");
|
||||
|
||||
let toml_item = report
|
||||
.items
|
||||
.as_ref()
|
||||
.expect("items present")
|
||||
.iter()
|
||||
.find(|i| i.doc_path.0.ends_with("Cargo.toml"))
|
||||
.expect("Cargo.toml item present");
|
||||
assert_eq!(
|
||||
toml_item.parser_version.as_ref().map(|p| p.0.as_str()),
|
||||
Some("none-v1"),
|
||||
"parser_version must be none-v1"
|
||||
);
|
||||
assert_eq!(
|
||||
toml_item.chunker_version.as_ref().map(|c| c.0.as_str()),
|
||||
Some("manifest-file-v1"),
|
||||
"chunker_version must be manifest-file-v1"
|
||||
);
|
||||
|
||||
let query = kebab_core::SearchQuery {
|
||||
text: "demo".to_string(),
|
||||
mode: kebab_core::SearchMode::Lexical,
|
||||
k: 10,
|
||||
filters: kebab_core::SearchFilters {
|
||||
code_lang: vec!["toml".to_string()],
|
||||
..Default::default()
|
||||
},
|
||||
};
|
||||
let hits = kebab_app::search_with_config(env.config.clone(), query)
|
||||
.expect("search must succeed");
|
||||
|
||||
let h = hits
|
||||
.iter()
|
||||
.find(|h| matches!(&h.citation, Citation::Code { .. }))
|
||||
.expect("at least one Citation::Code hit for 'demo'");
|
||||
|
||||
match &h.citation {
|
||||
Citation::Code {
|
||||
lang,
|
||||
symbol,
|
||||
line_start,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(
|
||||
lang.as_deref(),
|
||||
Some("toml"),
|
||||
"citation.lang must be 'toml'"
|
||||
);
|
||||
assert_eq!(
|
||||
symbol.as_deref(),
|
||||
Some("<manifest>"),
|
||||
"citation.symbol must be '<manifest>'"
|
||||
);
|
||||
assert!(*line_start >= 1, "line_start must be >=1");
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
h.code_lang.as_deref(),
|
||||
Some("toml"),
|
||||
"SearchHit.code_lang must be 'toml'"
|
||||
);
|
||||
}
|
||||
|
||||
/// p10-3 Task E: a `.sh` file is ingested via the shell direct-Tier-3 path
|
||||
/// and the resulting `Citation::Code` hit must carry `lang="shell"`,
|
||||
/// `symbol=None`, `line_start >= 1`, and
|
||||
/// `chunker_version = "code-text-paragraph-v1"`.
|
||||
#[test]
|
||||
fn tier3_shell_ingest_searchable() {
|
||||
let env = TestEnv::lexical_only();
|
||||
|
||||
std::fs::write(
|
||||
env.workspace_root.join("deploy.sh"),
|
||||
"#!/usr/bin/env bash\nset -e\necho hello\n\nkebab ingest --json\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let report = kebab_app::ingest_with_config(env.config.clone(), env.scope(), false)
|
||||
.expect("ingest must succeed");
|
||||
assert_eq!(report.errors, 0, "no ingest errors: {report:?}");
|
||||
assert!(report.new >= 1, "shell file ingested: {report:?}");
|
||||
|
||||
let sh_item = report
|
||||
.items
|
||||
.as_ref()
|
||||
.expect("items present")
|
||||
.iter()
|
||||
.find(|i| i.doc_path.0.ends_with("deploy.sh"))
|
||||
.expect("deploy.sh item present");
|
||||
assert_eq!(
|
||||
sh_item.parser_version.as_ref().map(|p| p.0.as_str()),
|
||||
Some("none-v1"),
|
||||
"parser_version must be none-v1 for shell (Tier 3 direct)"
|
||||
);
|
||||
assert_eq!(
|
||||
sh_item.chunker_version.as_ref().map(|c| c.0.as_str()),
|
||||
Some("code-text-paragraph-v1"),
|
||||
"chunker_version must be code-text-paragraph-v1 for shell"
|
||||
);
|
||||
|
||||
let query = kebab_core::SearchQuery {
|
||||
text: "kebab".to_string(),
|
||||
mode: kebab_core::SearchMode::Lexical,
|
||||
k: 10,
|
||||
filters: kebab_core::SearchFilters {
|
||||
code_lang: vec!["shell".to_string()],
|
||||
..Default::default()
|
||||
},
|
||||
};
|
||||
let hits = kebab_app::search_with_config(env.config.clone(), query)
|
||||
.expect("search must succeed");
|
||||
|
||||
let h = hits
|
||||
.iter()
|
||||
.find(|h| matches!(&h.citation, Citation::Code { .. }))
|
||||
.expect("at least one Citation::Code hit for 'kebab'");
|
||||
|
||||
match &h.citation {
|
||||
Citation::Code {
|
||||
lang,
|
||||
symbol,
|
||||
line_start,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(
|
||||
lang.as_deref(),
|
||||
Some("shell"),
|
||||
"citation.lang must be 'shell'"
|
||||
);
|
||||
assert_eq!(*symbol, None, "Tier 3 symbol must be None");
|
||||
assert!(*line_start >= 1, "line_start must be >=1");
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
h.code_lang.as_deref(),
|
||||
Some("shell"),
|
||||
"SearchHit.code_lang must be 'shell'"
|
||||
);
|
||||
assert_eq!(
|
||||
h.chunker_version.0.as_str(),
|
||||
"code-text-paragraph-v1",
|
||||
"shell chunks must be stamped with the Tier 3 chunker_version"
|
||||
);
|
||||
}
|
||||
|
||||
/// p10-3 Task E: a docker-compose-shaped YAML file (no `apiVersion`/`kind`)
|
||||
/// is ingested; the k8s chunker returns `Ok(vec![])` and the Tier 3 fallback
|
||||
/// wrapper retries with `CodeTextParagraphV1Chunker`. The resulting
|
||||
/// `Citation::Code` hit must carry `lang="yaml"`, `symbol=None`,
|
||||
/// `line_start >= 1`, and `chunker_version = "code-text-paragraph-v1"`.
|
||||
#[test]
|
||||
fn tier3_yaml_fallback_picks_up_non_k8s_yaml() {
|
||||
let env = TestEnv::lexical_only();
|
||||
|
||||
// docker-compose-shaped YAML — version + services but no apiVersion/kind.
|
||||
// The k8s chunker returns Ok(vec![]); Tier 3 fallback should pick this up.
|
||||
std::fs::write(
|
||||
env.workspace_root.join("docker-compose.yml"),
|
||||
"version: '3'\nservices:\n api:\n image: nginx:latest\n ports:\n - 8080:80\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let report = kebab_app::ingest_with_config(env.config.clone(), env.scope(), false)
|
||||
.expect("ingest must succeed");
|
||||
assert_eq!(report.errors, 0, "no ingest errors: {report:?}");
|
||||
assert!(
|
||||
report.new >= 1,
|
||||
"expected non-k8s yaml ingested via Tier 3, got {} new docs",
|
||||
report.new
|
||||
);
|
||||
|
||||
let yaml_item = report
|
||||
.items
|
||||
.as_ref()
|
||||
.expect("items present")
|
||||
.iter()
|
||||
.find(|i| i.doc_path.0.ends_with("docker-compose.yml"))
|
||||
.expect("docker-compose.yml item present");
|
||||
assert_eq!(
|
||||
yaml_item.parser_version.as_ref().map(|p| p.0.as_str()),
|
||||
Some("none-v1"),
|
||||
"parser_version must be none-v1 after Tier 3 fallback"
|
||||
);
|
||||
assert_eq!(
|
||||
yaml_item.chunker_version.as_ref().map(|c| c.0.as_str()),
|
||||
Some("code-text-paragraph-v1"),
|
||||
"chunker_version must be code-text-paragraph-v1 after Tier 3 fallback"
|
||||
);
|
||||
|
||||
let query = kebab_core::SearchQuery {
|
||||
text: "nginx".to_string(),
|
||||
mode: kebab_core::SearchMode::Lexical,
|
||||
k: 10,
|
||||
filters: kebab_core::SearchFilters {
|
||||
code_lang: vec!["yaml".to_string()],
|
||||
..Default::default()
|
||||
},
|
||||
};
|
||||
let hits = kebab_app::search_with_config(env.config.clone(), query)
|
||||
.expect("search must succeed");
|
||||
|
||||
let h = hits
|
||||
.iter()
|
||||
.find(|h| matches!(&h.citation, Citation::Code { .. }))
|
||||
.expect("at least one Citation::Code hit for 'nginx'");
|
||||
|
||||
match &h.citation {
|
||||
Citation::Code {
|
||||
lang,
|
||||
symbol,
|
||||
line_start,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(
|
||||
lang.as_deref(),
|
||||
Some("yaml"),
|
||||
"citation.lang must be 'yaml'"
|
||||
);
|
||||
assert_eq!(*symbol, None, "Tier 3 fallback symbol must be None");
|
||||
assert!(*line_start >= 1, "line_start must be >=1");
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
h.code_lang.as_deref(),
|
||||
Some("yaml"),
|
||||
"SearchHit.code_lang must be 'yaml'"
|
||||
);
|
||||
assert_eq!(
|
||||
h.chunker_version.0.as_str(),
|
||||
"code-text-paragraph-v1",
|
||||
"non-k8s yaml fallback must be stamped code-text-paragraph-v1"
|
||||
);
|
||||
}
|
||||
|
||||
/// Re-ingesting the same `.rs` file without changes must report
|
||||
/// `Unchanged` (incremental-skip path exercised).
|
||||
#[test]
|
||||
|
||||
@@ -13,6 +13,7 @@ serde_json_canonicalizer = "0.3"
|
||||
blake3 = { workspace = true }
|
||||
anyhow = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
serde_yaml = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
# kb-parse-md / kb-normalize are dev-only — used by the snapshot integration
|
||||
|
||||
170
crates/kebab-chunk/src/code_text_paragraph_v1.rs
Normal file
170
crates/kebab-chunk/src/code_text_paragraph_v1.rs
Normal file
@@ -0,0 +1,170 @@
|
||||
//! p10-3: Tier 3 paragraph + line-window fallback chunker.
|
||||
//!
|
||||
//! Splits code/text files on blank-line paragraph boundaries. Paragraphs
|
||||
//! with more than 80 lines are further split into 80-line windows with a
|
||||
//! 20-line overlap (stride 60) — the same oversize pattern used by Tier 1/2
|
||||
//! chunkers but without AST structure, hence no symbol.
|
||||
//!
|
||||
//! Per spec §9.3: all emitted chunks carry `symbol: None`.
|
||||
|
||||
use crate::tier2_shared::{build_chunk_no_symbol, policy_hash};
|
||||
use anyhow::Result;
|
||||
use kebab_core::{Block, CanonicalDocument, Chunk, ChunkPolicy, ChunkerVersion, Chunker};
|
||||
|
||||
pub const VERSION_LABEL: &str = "code-text-paragraph-v1";
|
||||
|
||||
/// Lines-per-window for the oversize fallback (Tier 3).
|
||||
const FALLBACK_LINES_PER_CHUNK: usize = 80;
|
||||
/// Overlap between consecutive windows.
|
||||
const FALLBACK_LINES_OVERLAP: usize = 20;
|
||||
// stride = FALLBACK_LINES_PER_CHUNK - FALLBACK_LINES_OVERLAP = 60.
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
pub struct CodeTextParagraphV1Chunker;
|
||||
|
||||
impl Chunker for CodeTextParagraphV1Chunker {
|
||||
fn chunker_version(&self) -> ChunkerVersion {
|
||||
ChunkerVersion(VERSION_LABEL.to_string())
|
||||
}
|
||||
|
||||
fn policy_hash(&self, policy: &ChunkPolicy) -> String {
|
||||
policy_hash(policy)
|
||||
}
|
||||
|
||||
fn chunk(&self, doc: &CanonicalDocument, policy: &ChunkPolicy) -> Result<Vec<Chunk>> {
|
||||
// Expect a single Block::Code carrying the full source text.
|
||||
let (text, lang_str) = match doc.blocks.first() {
|
||||
Some(Block::Code(cb)) => (cb.code.as_str(), cb.lang.as_deref().unwrap_or("")),
|
||||
_ => return Ok(vec![]),
|
||||
};
|
||||
|
||||
let mut chunks = Vec::new();
|
||||
for para in split_paragraphs(text) {
|
||||
push_paragraph(&mut chunks, doc, policy, ¶, lang_str)?;
|
||||
}
|
||||
|
||||
tracing::debug!(
|
||||
target: "kebab-chunk",
|
||||
doc_id = %doc.doc_id,
|
||||
chunks = chunks.len(),
|
||||
"code-text-paragraph-v1 chunked",
|
||||
);
|
||||
|
||||
Ok(chunks)
|
||||
}
|
||||
}
|
||||
|
||||
/// A contiguous run of non-blank lines from the source text.
|
||||
struct Paragraph {
|
||||
/// Lines joined with `\n` (no trailing newline).
|
||||
text: String,
|
||||
/// 1-indexed line number of the first line in the source file.
|
||||
line_start: u32,
|
||||
/// 1-indexed line number of the last line in the source file.
|
||||
line_end: u32,
|
||||
}
|
||||
|
||||
/// Split `text` into `Paragraph`s separated by blank (all-whitespace) lines.
|
||||
///
|
||||
/// Blank lines are treated as boundaries and are NOT included in any
|
||||
/// paragraph's line range. Paragraphs that would consist entirely of blank
|
||||
/// lines are skipped.
|
||||
fn split_paragraphs(text: &str) -> Vec<Paragraph> {
|
||||
let mut paragraphs = Vec::new();
|
||||
let mut current: Vec<&str> = Vec::new();
|
||||
let mut current_start: Option<u32> = None;
|
||||
|
||||
for (idx, line) in text.lines().enumerate() {
|
||||
let line_no = (idx + 1) as u32;
|
||||
let is_blank = line.trim().is_empty();
|
||||
if is_blank {
|
||||
if let Some(start) = current_start.take() {
|
||||
let end = start + current.len() as u32 - 1;
|
||||
paragraphs.push(Paragraph {
|
||||
text: current.join("\n"),
|
||||
line_start: start,
|
||||
line_end: end,
|
||||
});
|
||||
current.clear();
|
||||
}
|
||||
} else {
|
||||
if current_start.is_none() {
|
||||
current_start = Some(line_no);
|
||||
}
|
||||
current.push(line);
|
||||
}
|
||||
}
|
||||
// Flush any trailing paragraph not terminated by a blank line.
|
||||
if let Some(start) = current_start {
|
||||
let end = start + current.len() as u32 - 1;
|
||||
paragraphs.push(Paragraph {
|
||||
text: current.join("\n"),
|
||||
line_start: start,
|
||||
line_end: end,
|
||||
});
|
||||
}
|
||||
paragraphs
|
||||
}
|
||||
|
||||
/// Emit one or more chunks for a single paragraph.
|
||||
///
|
||||
/// Paragraphs with ≤ `FALLBACK_LINES_PER_CHUNK` lines become a single chunk.
|
||||
/// Larger paragraphs are split into overlapping windows of
|
||||
/// `FALLBACK_LINES_PER_CHUNK` lines with stride `FALLBACK_LINES_PER_CHUNK -
|
||||
/// FALLBACK_LINES_OVERLAP`. The last window may be shorter. Window starts
|
||||
/// are passed as `split_key` so `id_for_chunk` can produce distinct ids
|
||||
/// across windows.
|
||||
fn push_paragraph(
|
||||
out: &mut Vec<Chunk>,
|
||||
doc: &CanonicalDocument,
|
||||
policy: &ChunkPolicy,
|
||||
para: &Paragraph,
|
||||
lang: &str,
|
||||
) -> Result<()> {
|
||||
let n_lines = (para.line_end - para.line_start + 1) as usize;
|
||||
|
||||
if n_lines <= FALLBACK_LINES_PER_CHUNK {
|
||||
// Use line_start as split_key so each paragraph gets a distinct
|
||||
// chunk_id even when block_ids is empty (no symbol, no AST structure).
|
||||
// Without this, all short paragraphs from the same doc share the same
|
||||
// base_policy_hash and therefore the same id_for_chunk result.
|
||||
out.push(build_chunk_no_symbol(
|
||||
doc,
|
||||
policy,
|
||||
¶.text,
|
||||
para.line_start,
|
||||
para.line_end,
|
||||
lang,
|
||||
VERSION_LABEL,
|
||||
Some(para.line_start),
|
||||
));
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Oversize: line-window split with overlap.
|
||||
let stride = FALLBACK_LINES_PER_CHUNK - FALLBACK_LINES_OVERLAP;
|
||||
let lines: Vec<&str> = para.text.lines().collect();
|
||||
let mut i = 0usize;
|
||||
loop {
|
||||
let end = (i + FALLBACK_LINES_PER_CHUNK).min(lines.len());
|
||||
let window_text = lines[i..end].join("\n");
|
||||
let window_start = para.line_start + i as u32;
|
||||
let window_end = para.line_start + (end as u32) - 1;
|
||||
// Use window_start as split_key so chunk_ids are unique across windows.
|
||||
out.push(build_chunk_no_symbol(
|
||||
doc,
|
||||
policy,
|
||||
&window_text,
|
||||
window_start,
|
||||
window_end,
|
||||
lang,
|
||||
VERSION_LABEL,
|
||||
Some(window_start),
|
||||
));
|
||||
if end == lines.len() {
|
||||
break;
|
||||
}
|
||||
i += stride;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
57
crates/kebab-chunk/src/dockerfile_file_v1.rs
Normal file
57
crates/kebab-chunk/src/dockerfile_file_v1.rs
Normal file
@@ -0,0 +1,57 @@
|
||||
//! p10-2: dockerfile whole-file chunker (Tier 2).
|
||||
//!
|
||||
//! Reads entire Dockerfile content and emits a single Chunk with symbol
|
||||
//! "<dockerfile>", code_lang "dockerfile", line range 1..EOF.
|
||||
//! Oversize >200 lines splits into line-windows sharing the symbol via
|
||||
//! tier2_shared::push_chunks_with_oversize.
|
||||
|
||||
use crate::tier2_shared::{policy_hash, push_chunks_with_oversize};
|
||||
use anyhow::Result;
|
||||
use kebab_core::{Block, CanonicalDocument, Chunk, ChunkPolicy, ChunkerVersion, Chunker};
|
||||
|
||||
pub const VERSION_LABEL: &str = "dockerfile-file-v1";
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
pub struct DockerfileFileV1Chunker;
|
||||
|
||||
impl Chunker for DockerfileFileV1Chunker {
|
||||
fn chunker_version(&self) -> ChunkerVersion {
|
||||
ChunkerVersion(VERSION_LABEL.to_string())
|
||||
}
|
||||
|
||||
fn policy_hash(&self, policy: &ChunkPolicy) -> String {
|
||||
policy_hash(policy)
|
||||
}
|
||||
|
||||
fn chunk(&self, doc: &CanonicalDocument, policy: &ChunkPolicy) -> Result<Vec<Chunk>> {
|
||||
// Expect a single Block::Code carrying the full Dockerfile text.
|
||||
let text = match doc.blocks.first() {
|
||||
Some(Block::Code(cb)) => cb.code.as_str(),
|
||||
_ => return Ok(vec![]),
|
||||
};
|
||||
|
||||
let total_lines = text.lines().count().max(1) as u32;
|
||||
let mut chunks = Vec::new();
|
||||
|
||||
push_chunks_with_oversize(
|
||||
&mut chunks,
|
||||
doc,
|
||||
policy,
|
||||
text,
|
||||
1,
|
||||
total_lines,
|
||||
"<dockerfile>",
|
||||
"dockerfile",
|
||||
VERSION_LABEL,
|
||||
)?;
|
||||
|
||||
tracing::debug!(
|
||||
target: "kebab-chunk",
|
||||
doc_id = %doc.doc_id,
|
||||
chunks = chunks.len(),
|
||||
"dockerfile-file-v1 chunked",
|
||||
);
|
||||
|
||||
Ok(chunks)
|
||||
}
|
||||
}
|
||||
169
crates/kebab-chunk/src/k8s_manifest_resource_v1.rs
Normal file
169
crates/kebab-chunk/src/k8s_manifest_resource_v1.rs
Normal file
@@ -0,0 +1,169 @@
|
||||
//! p10-2: k8s manifest resource-aware chunker.
|
||||
//!
|
||||
//! Splits a multi-document YAML file on `^---\s*$` boundaries, recognises
|
||||
//! documents that have both `apiVersion` and `kind` string fields as k8s
|
||||
//! resources, and emits one `Chunk` per resource (with oversize >200-line
|
||||
//! fallback). Non-k8s documents are skipped; invalid YAML yields 0 chunks
|
||||
//! for the entire file.
|
||||
|
||||
use crate::tier2_shared::{policy_hash, push_chunks_with_oversize};
|
||||
use anyhow::Result;
|
||||
use kebab_core::{Block, CanonicalDocument, Chunk, ChunkPolicy, ChunkerVersion, Chunker};
|
||||
|
||||
pub const VERSION_LABEL: &str = "k8s-manifest-resource-v1";
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
pub struct K8sManifestResourceV1Chunker;
|
||||
|
||||
impl Chunker for K8sManifestResourceV1Chunker {
|
||||
fn chunker_version(&self) -> ChunkerVersion {
|
||||
ChunkerVersion(VERSION_LABEL.to_string())
|
||||
}
|
||||
|
||||
fn policy_hash(&self, policy: &ChunkPolicy) -> String {
|
||||
policy_hash(policy)
|
||||
}
|
||||
|
||||
fn chunk(&self, doc: &CanonicalDocument, policy: &ChunkPolicy) -> Result<Vec<Chunk>> {
|
||||
// Expect a single Block::Code carrying the full YAML text.
|
||||
let text = match doc.blocks.first() {
|
||||
Some(Block::Code(cb)) => cb.code.as_str(),
|
||||
_ => return Ok(vec![]),
|
||||
};
|
||||
|
||||
let slices = split_yaml_documents(text);
|
||||
let mut chunks: Vec<Chunk> = Vec::new();
|
||||
|
||||
for slice in slices {
|
||||
// Invalid YAML in any document → return 0 chunks for the file.
|
||||
let value: serde_yaml::Value = match serde_yaml::from_str(slice.text) {
|
||||
Ok(v) => v,
|
||||
Err(_) => return Ok(vec![]),
|
||||
};
|
||||
|
||||
let Some(mapping) = value.as_mapping() else {
|
||||
continue;
|
||||
};
|
||||
|
||||
let api = mapping
|
||||
.get("apiVersion")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("");
|
||||
let kind = mapping
|
||||
.get("kind")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("");
|
||||
|
||||
// Skip non-k8s documents.
|
||||
if api.is_empty() || kind.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let metadata = mapping
|
||||
.get("metadata")
|
||||
.and_then(|v| v.as_mapping());
|
||||
let name = metadata
|
||||
.and_then(|m| m.get("name"))
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("<unnamed>");
|
||||
let namespace = metadata
|
||||
.and_then(|m| m.get("namespace"))
|
||||
.and_then(|v| v.as_str());
|
||||
|
||||
let symbol = match namespace {
|
||||
Some(ns) if !ns.is_empty() => format!("{kind}/{ns}/{name}"),
|
||||
_ => format!("{kind}/{name}"),
|
||||
};
|
||||
|
||||
push_chunks_with_oversize(
|
||||
&mut chunks,
|
||||
doc,
|
||||
policy,
|
||||
slice.text,
|
||||
slice.line_start,
|
||||
slice.line_end,
|
||||
&symbol,
|
||||
"yaml",
|
||||
VERSION_LABEL,
|
||||
)?;
|
||||
}
|
||||
|
||||
tracing::debug!(
|
||||
target: "kebab-chunk",
|
||||
doc_id = %doc.doc_id,
|
||||
chunks = chunks.len(),
|
||||
"k8s-manifest-resource-v1 chunked",
|
||||
);
|
||||
|
||||
Ok(chunks)
|
||||
}
|
||||
}
|
||||
|
||||
struct YamlSlice<'a> {
|
||||
text: &'a str,
|
||||
line_start: u32,
|
||||
line_end: u32,
|
||||
}
|
||||
|
||||
/// Split raw YAML text into per-document slices on `---` separator lines.
|
||||
/// Line numbers are 1-indexed.
|
||||
fn split_yaml_documents(text: &str) -> Vec<YamlSlice<'_>> {
|
||||
let lines: Vec<&str> = text.lines().collect();
|
||||
|
||||
// Collect indices of separator lines (0-based), then append a sentinel at
|
||||
// the end so the last slice is always terminated.
|
||||
let mut separators: Vec<usize> = lines
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(i, l)| {
|
||||
let trimmed = l.trim_end();
|
||||
if trimmed == "---"
|
||||
|| trimmed.starts_with("--- ")
|
||||
|| trimmed.starts_with("---\t")
|
||||
{
|
||||
Some(i)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
separators.push(lines.len());
|
||||
|
||||
let mut slices: Vec<YamlSlice<'_>> = Vec::new();
|
||||
let mut doc_start_line: usize = 0; // 0-based index of current doc start
|
||||
|
||||
for sep_line in separators {
|
||||
if sep_line > doc_start_line {
|
||||
let start_byte = byte_offset_of_line(text, doc_start_line);
|
||||
let end_byte = byte_offset_of_line(text, sep_line);
|
||||
let slice_text = &text[start_byte..end_byte];
|
||||
if !slice_text.trim().is_empty() {
|
||||
slices.push(YamlSlice {
|
||||
text: slice_text,
|
||||
line_start: (doc_start_line + 1) as u32,
|
||||
line_end: sep_line as u32,
|
||||
});
|
||||
}
|
||||
}
|
||||
doc_start_line = sep_line + 1;
|
||||
}
|
||||
|
||||
slices
|
||||
}
|
||||
|
||||
/// Return the byte offset of the start of `line_idx` (0-based line index).
|
||||
fn byte_offset_of_line(text: &str, line_idx: usize) -> usize {
|
||||
if line_idx == 0 {
|
||||
return 0;
|
||||
}
|
||||
let mut count = 0usize;
|
||||
for (i, c) in text.char_indices() {
|
||||
if c == '\n' {
|
||||
count += 1;
|
||||
if count == line_idx {
|
||||
return i + 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
text.len()
|
||||
}
|
||||
@@ -24,6 +24,11 @@ mod code_rust_ast_v1;
|
||||
mod code_ts_ast_v1;
|
||||
mod md_heading_v1;
|
||||
mod pdf_page_v1;
|
||||
mod tier2_shared;
|
||||
pub mod k8s_manifest_resource_v1;
|
||||
pub mod dockerfile_file_v1;
|
||||
pub mod manifest_file_v1;
|
||||
pub mod code_text_paragraph_v1;
|
||||
|
||||
pub use code_go_ast_v1::CodeGoAstV1Chunker;
|
||||
pub use code_java_ast_v1::CodeJavaAstV1Chunker;
|
||||
@@ -34,3 +39,7 @@ pub use code_rust_ast_v1::CodeRustAstV1Chunker;
|
||||
pub use code_ts_ast_v1::CodeTsAstV1Chunker;
|
||||
pub use md_heading_v1::MdHeadingV1Chunker;
|
||||
pub use pdf_page_v1::PdfPageV1Chunker;
|
||||
pub use k8s_manifest_resource_v1::K8sManifestResourceV1Chunker;
|
||||
pub use dockerfile_file_v1::DockerfileFileV1Chunker;
|
||||
pub use manifest_file_v1::ManifestFileV1Chunker;
|
||||
pub use code_text_paragraph_v1::CodeTextParagraphV1Chunker;
|
||||
|
||||
58
crates/kebab-chunk/src/manifest_file_v1.rs
Normal file
58
crates/kebab-chunk/src/manifest_file_v1.rs
Normal file
@@ -0,0 +1,58 @@
|
||||
//! p10-2: manifest whole-file chunker (Tier 2).
|
||||
//!
|
||||
//! Reads entire manifest file (Cargo.toml / package.json / pom.xml / go.mod /
|
||||
//! build.gradle / pyproject.toml / tsconfig.json) and emits a single Chunk
|
||||
//! with symbol "<manifest>", code_lang read from Block::Code.lang, line range
|
||||
//! 1..EOF. Oversize >200 lines splits into line-windows sharing the symbol via
|
||||
//! tier2_shared::push_chunks_with_oversize.
|
||||
|
||||
use crate::tier2_shared::{policy_hash, push_chunks_with_oversize};
|
||||
use anyhow::Result;
|
||||
use kebab_core::{Block, CanonicalDocument, Chunk, ChunkPolicy, ChunkerVersion, Chunker};
|
||||
|
||||
pub const VERSION_LABEL: &str = "manifest-file-v1";
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
pub struct ManifestFileV1Chunker;
|
||||
|
||||
impl Chunker for ManifestFileV1Chunker {
|
||||
fn chunker_version(&self) -> ChunkerVersion {
|
||||
ChunkerVersion(VERSION_LABEL.to_string())
|
||||
}
|
||||
|
||||
fn policy_hash(&self, policy: &ChunkPolicy) -> String {
|
||||
policy_hash(policy)
|
||||
}
|
||||
|
||||
fn chunk(&self, doc: &CanonicalDocument, policy: &ChunkPolicy) -> Result<Vec<Chunk>> {
|
||||
// Expect a single Block::Code carrying the full manifest text.
|
||||
let (text, lang) = match doc.blocks.first() {
|
||||
Some(Block::Code(cb)) => (cb.code.as_str(), cb.lang.as_deref().unwrap_or("")),
|
||||
_ => return Ok(vec![]),
|
||||
};
|
||||
|
||||
let total_lines = text.lines().count().max(1) as u32;
|
||||
let mut chunks = Vec::new();
|
||||
|
||||
push_chunks_with_oversize(
|
||||
&mut chunks,
|
||||
doc,
|
||||
policy,
|
||||
text,
|
||||
1,
|
||||
total_lines,
|
||||
"<manifest>",
|
||||
lang,
|
||||
VERSION_LABEL,
|
||||
)?;
|
||||
|
||||
tracing::debug!(
|
||||
target: "kebab-chunk",
|
||||
doc_id = %doc.doc_id,
|
||||
chunks = chunks.len(),
|
||||
"manifest-file-v1 chunked",
|
||||
);
|
||||
|
||||
Ok(chunks)
|
||||
}
|
||||
}
|
||||
184
crates/kebab-chunk/src/tier2_shared.rs
Normal file
184
crates/kebab-chunk/src/tier2_shared.rs
Normal file
@@ -0,0 +1,184 @@
|
||||
//! p10-2: Tier 2 chunker shared helpers (oversize fallback + Chunk build).
|
||||
//!
|
||||
//! Mirrors `code_rust_ast_v1`'s Chunk-construction pattern exactly so that
|
||||
//! id / hashes / token-count / ChunkPolicy semantics stay identical across
|
||||
//! Tier 1 (AST) and Tier 2 (resource-aware) chunkers.
|
||||
|
||||
use anyhow::Result;
|
||||
use kebab_core::{
|
||||
BlockId, CanonicalDocument, Chunk, ChunkPolicy, ChunkerVersion, DocumentId, SourceSpan,
|
||||
id_for_chunk,
|
||||
};
|
||||
|
||||
pub(crate) const AST_CHUNK_MAX_LINES: u32 = 200;
|
||||
const BYTES_PER_TOKEN: usize = 3;
|
||||
const POLICY_HASH_HEX_LEN: usize = 16;
|
||||
|
||||
/// Compute the policy hash the same way `code_rust_ast_v1` does.
|
||||
pub(crate) fn policy_hash(policy: &ChunkPolicy) -> String {
|
||||
let bytes = serde_json_canonicalizer::to_vec(policy)
|
||||
.expect("canonical JSON serialization of ChunkPolicy must not fail");
|
||||
let hex = blake3::hash(&bytes).to_hex().to_string();
|
||||
hex[..POLICY_HASH_HEX_LEN].to_string()
|
||||
}
|
||||
|
||||
/// Emit one chunk for `(text, line_start..=line_end, symbol, lang)`, splitting
|
||||
/// into line-windows of at most `AST_CHUNK_MAX_LINES` if the slice is oversize.
|
||||
/// Mirrors the oversize path in `code_rust_ast_v1`'s `chunk` impl.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) fn push_chunks_with_oversize(
|
||||
out: &mut Vec<Chunk>,
|
||||
doc: &CanonicalDocument,
|
||||
policy: &ChunkPolicy,
|
||||
text: &str,
|
||||
line_start: u32,
|
||||
line_end: u32,
|
||||
symbol: &str,
|
||||
lang: &str,
|
||||
chunker_version: &str,
|
||||
) -> Result<()> {
|
||||
let n_lines = (line_end - line_start + 1).max(1);
|
||||
let cv = ChunkerVersion(chunker_version.to_string());
|
||||
let base_policy_hash = policy_hash(policy);
|
||||
|
||||
if n_lines <= AST_CHUNK_MAX_LINES {
|
||||
out.push(build_chunk(
|
||||
doc,
|
||||
&cv,
|
||||
&base_policy_hash,
|
||||
text,
|
||||
line_start,
|
||||
line_end,
|
||||
symbol,
|
||||
lang,
|
||||
None,
|
||||
));
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let lines: Vec<&str> = text.lines().collect();
|
||||
let total = lines.len();
|
||||
let mut window_start = line_start;
|
||||
let mut i = 0usize;
|
||||
while i < total {
|
||||
let take = (AST_CHUNK_MAX_LINES as usize).min(total - i);
|
||||
let window_text = lines[i..i + take].join("\n");
|
||||
let window_end = window_start + take as u32 - 1;
|
||||
out.push(build_chunk(
|
||||
doc,
|
||||
&cv,
|
||||
&base_policy_hash,
|
||||
&window_text,
|
||||
window_start,
|
||||
window_end,
|
||||
symbol,
|
||||
lang,
|
||||
Some(window_start),
|
||||
));
|
||||
i += take;
|
||||
window_start = window_end + 1;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Build a single `Chunk`, mirroring `make_chunk` in `code_rust_ast_v1.rs`
|
||||
/// exactly (same id recipe, same token estimate, same field set).
|
||||
///
|
||||
/// `split_key` is `Some(line_start_of_window)` for oversize splits, `None`
|
||||
/// for normal single-chunk emission. Mirrors the `Some(part_ls)` / `None`
|
||||
/// split_key pattern in 1A-2.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) fn build_chunk(
|
||||
doc: &CanonicalDocument,
|
||||
chunker_version: &ChunkerVersion,
|
||||
base_policy_hash: &str,
|
||||
text: &str,
|
||||
line_start: u32,
|
||||
line_end: u32,
|
||||
symbol: &str,
|
||||
lang: &str,
|
||||
split_key: Option<u32>,
|
||||
) -> Chunk {
|
||||
let span = SourceSpan::Code {
|
||||
line_start,
|
||||
line_end,
|
||||
symbol: Some(symbol.to_string()),
|
||||
lang: Some(lang.to_string()),
|
||||
};
|
||||
build_chunk_from_span(doc, chunker_version, base_policy_hash, text, span, split_key)
|
||||
}
|
||||
|
||||
/// Like `build_chunk` but emits `symbol: None`. Used by Tier 3 (per spec §9.3).
|
||||
///
|
||||
/// Accepts `policy: &ChunkPolicy` and `chunker_version: &str` (string slice)
|
||||
/// so callers don't need to pre-compute the hash and version wrapper.
|
||||
/// `split_key` is `Some(window_start)` for oversize line-window splits.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) fn build_chunk_no_symbol(
|
||||
doc: &CanonicalDocument,
|
||||
policy: &ChunkPolicy,
|
||||
text: &str,
|
||||
line_start: u32,
|
||||
line_end: u32,
|
||||
lang: &str,
|
||||
chunker_version: &str,
|
||||
split_key: Option<u32>,
|
||||
) -> Chunk {
|
||||
let cv = ChunkerVersion(chunker_version.to_string());
|
||||
let base_policy_hash = policy_hash(policy);
|
||||
let span = SourceSpan::Code {
|
||||
line_start,
|
||||
line_end,
|
||||
symbol: None,
|
||||
lang: Some(lang.to_string()),
|
||||
};
|
||||
build_chunk_from_span(doc, &cv, &base_policy_hash, text, span, split_key)
|
||||
}
|
||||
|
||||
/// Core chunk-building logic shared by `build_chunk` and `build_chunk_no_symbol`.
|
||||
///
|
||||
/// Takes a pre-built `SourceSpan` so the only difference between the two
|
||||
/// public helpers is whether `symbol` is `Some` or `None`. All id/hash/
|
||||
/// token mechanics are identical.
|
||||
fn build_chunk_from_span(
|
||||
doc: &CanonicalDocument,
|
||||
chunker_version: &ChunkerVersion,
|
||||
base_policy_hash: &str,
|
||||
text: &str,
|
||||
span: SourceSpan,
|
||||
split_key: Option<u32>,
|
||||
) -> Chunk {
|
||||
// id_hash mirrors code_rust_ast_v1's make_chunk logic:
|
||||
// split_key Some(k) => "{base_policy_hash}#L{k}"
|
||||
// split_key None => base_policy_hash
|
||||
let id_hash = match split_key {
|
||||
Some(k) => format!("{base_policy_hash}#L{k}"),
|
||||
None => base_policy_hash.to_string(),
|
||||
};
|
||||
|
||||
// block_ids: Tier 2/3 chunkers have no per-block structure (the whole file
|
||||
// is one Block::Code), so we pass an empty slice — same as using the doc-
|
||||
// level slice without explicit block granularity.
|
||||
let block_ids: Vec<BlockId> = vec![];
|
||||
|
||||
let chunk_id = id_for_chunk(
|
||||
&DocumentId(doc.doc_id.0.clone()),
|
||||
chunker_version,
|
||||
&block_ids,
|
||||
&id_hash,
|
||||
);
|
||||
|
||||
let token_estimate = text.len().div_ceil(BYTES_PER_TOKEN);
|
||||
|
||||
Chunk {
|
||||
chunk_id,
|
||||
doc_id: DocumentId(doc.doc_id.0.clone()),
|
||||
block_ids,
|
||||
text: text.to_string(),
|
||||
heading_path: Vec::new(),
|
||||
source_spans: vec![span],
|
||||
token_estimate,
|
||||
chunker_version: chunker_version.clone(),
|
||||
policy_hash: base_policy_hash.to_string(),
|
||||
}
|
||||
}
|
||||
270
crates/kebab-chunk/tests/code_text_paragraph_v1.rs
Normal file
270
crates/kebab-chunk/tests/code_text_paragraph_v1.rs
Normal file
@@ -0,0 +1,270 @@
|
||||
//! Behavioural tests for `CodeTextParagraphV1Chunker`.
|
||||
//!
|
||||
//! Documents are constructed manually (no kebab-parse-code dependency) by
|
||||
//! placing raw text into a single `Block::Code`, mirroring the pattern used
|
||||
//! in `k8s_manifest_resource_v1.rs`.
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use kebab_chunk::CodeTextParagraphV1Chunker;
|
||||
use kebab_core::{
|
||||
AssetId, Block, CanonicalDocument, ChunkPolicy, Chunker, ChunkerVersion, CodeBlock,
|
||||
CommonBlock, Lang, Metadata, ParserVersion, Provenance, SourceSpan, SourceType, TrustLevel,
|
||||
WorkspacePath, id_for_block, id_for_doc,
|
||||
};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
// ── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
fn fixtures_dir() -> PathBuf {
|
||||
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("tests")
|
||||
.join("fixtures")
|
||||
}
|
||||
|
||||
/// Build a `CanonicalDocument` with a single `Block::Code` containing `text`
|
||||
/// and the supplied `lang` label.
|
||||
fn text_doc(lang: &str, text: &str) -> CanonicalDocument {
|
||||
let wp = WorkspacePath("scripts/sample.sh".into());
|
||||
let aid = AssetId("d".repeat(64));
|
||||
let pv = ParserVersion("code-text-paragraph-v1".into());
|
||||
let doc_id = id_for_doc(&wp, &aid, &pv);
|
||||
|
||||
let line_count = text.lines().count() as u32;
|
||||
let span = SourceSpan::Code {
|
||||
line_start: 1,
|
||||
line_end: line_count.max(1),
|
||||
symbol: None,
|
||||
lang: Some(lang.into()),
|
||||
};
|
||||
let bid = id_for_block(&doc_id, "code", &[], 0, &span);
|
||||
let block = Block::Code(CodeBlock {
|
||||
common: CommonBlock {
|
||||
block_id: bid,
|
||||
heading_path: vec![],
|
||||
source_span: span,
|
||||
},
|
||||
lang: Some(lang.into()),
|
||||
code: text.to_string(),
|
||||
});
|
||||
|
||||
CanonicalDocument {
|
||||
doc_id,
|
||||
source_asset_id: aid,
|
||||
workspace_path: wp,
|
||||
title: "sample.sh".into(),
|
||||
lang: Lang("und".into()),
|
||||
blocks: vec![block],
|
||||
metadata: Metadata {
|
||||
aliases: vec![],
|
||||
tags: vec![],
|
||||
created_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
updated_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
source_type: SourceType::Note,
|
||||
trust_level: TrustLevel::Primary,
|
||||
user_id_alias: None,
|
||||
user: Default::default(),
|
||||
repo: Some("kebab".into()),
|
||||
git_branch: Some("main".into()),
|
||||
git_commit: Some("0".repeat(40)),
|
||||
code_lang: Some(lang.into()),
|
||||
},
|
||||
provenance: Provenance { events: vec![] },
|
||||
parser_version: pv,
|
||||
schema_version: 1,
|
||||
doc_version: 1,
|
||||
last_chunker_version: None,
|
||||
last_embedding_version: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn policy() -> ChunkPolicy {
|
||||
ChunkPolicy {
|
||||
target_tokens: 500,
|
||||
overlap_tokens: 80,
|
||||
respect_markdown_headings: false,
|
||||
chunker_version: ChunkerVersion("code-text-paragraph-v1".into()),
|
||||
}
|
||||
}
|
||||
|
||||
// ── tests ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// `sample_shell.sh` has 4 paragraphs separated by 3 blank lines:
|
||||
/// - paragraph 1: lines 1-2 (shebang + set -euo pipefail)
|
||||
/// - paragraph 2: lines 4-7 (env setup block)
|
||||
/// - paragraph 3: lines 9-11 (ingest block)
|
||||
/// - paragraph 4: lines 13-15 (report block)
|
||||
///
|
||||
/// We assert:
|
||||
/// - exactly 4 chunks (one per paragraph)
|
||||
/// - all symbols are None (Tier 3 spec §9.3)
|
||||
/// - all langs are "shell"
|
||||
/// - line ranges are strictly ascending and do NOT include the blank lines
|
||||
/// (lines 3, 8, 12 must not appear in any range)
|
||||
#[test]
|
||||
fn shell_multi_paragraph_splits_on_blank_lines() {
|
||||
let fixture_path = fixtures_dir().join("sample_shell.sh");
|
||||
let text = std::fs::read_to_string(&fixture_path)
|
||||
.unwrap_or_else(|e| panic!("cannot read fixture {}: {e}", fixture_path.display()));
|
||||
|
||||
let doc = text_doc("shell", &text);
|
||||
let chunks = CodeTextParagraphV1Chunker
|
||||
.chunk(&doc, &policy())
|
||||
.expect("chunk");
|
||||
|
||||
assert_eq!(
|
||||
chunks.len(),
|
||||
4,
|
||||
"expected 4 chunks (one per paragraph), got {}: {chunks:#?}",
|
||||
chunks.len()
|
||||
);
|
||||
|
||||
// All symbols must be None (Tier 3 requirement).
|
||||
for (i, chunk) in chunks.iter().enumerate() {
|
||||
match &chunk.source_spans[0] {
|
||||
SourceSpan::Code { symbol, .. } => {
|
||||
assert!(
|
||||
symbol.is_none(),
|
||||
"chunk[{i}] symbol must be None for Tier 3 chunker, got {symbol:?}"
|
||||
);
|
||||
}
|
||||
other => panic!("chunk[{i}]: expected Code span, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
// All langs must be "shell".
|
||||
for (i, chunk) in chunks.iter().enumerate() {
|
||||
match &chunk.source_spans[0] {
|
||||
SourceSpan::Code { lang, .. } => {
|
||||
assert_eq!(
|
||||
lang.as_deref(),
|
||||
Some("shell"),
|
||||
"chunk[{i}] lang must be 'shell', got {lang:?}"
|
||||
);
|
||||
}
|
||||
other => panic!("chunk[{i}]: expected Code span, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
// Line ranges must be strictly ascending with no overlap,
|
||||
// and blank lines (3, 8, 12) must not be included in any range.
|
||||
let expected_ranges: &[(u32, u32)] = &[(1, 2), (4, 7), (9, 11), (13, 15)];
|
||||
let actual_ranges: Vec<(u32, u32)> = chunks
|
||||
.iter()
|
||||
.map(|c| match &c.source_spans[0] {
|
||||
SourceSpan::Code {
|
||||
line_start,
|
||||
line_end,
|
||||
..
|
||||
} => (*line_start, *line_end),
|
||||
other => panic!("expected Code span, got {other:?}"),
|
||||
})
|
||||
.collect();
|
||||
|
||||
assert_eq!(
|
||||
actual_ranges, expected_ranges,
|
||||
"line ranges mismatch: got {actual_ranges:?}, expected {expected_ranges:?}"
|
||||
);
|
||||
}
|
||||
|
||||
/// `sample_long_paragraph.txt` has exactly 200 non-blank lines and no blank
|
||||
/// lines, so the entire file is one paragraph. 200 > 80 (FALLBACK_LINES_PER_CHUNK),
|
||||
/// so the oversize window split fires with stride 60:
|
||||
/// - window 1: lines 1-80
|
||||
/// - window 2: lines 61-140
|
||||
/// - window 3: lines 121-200
|
||||
///
|
||||
/// All chunk_ids must be distinct (the #L{window_start} split_key suffix).
|
||||
#[test]
|
||||
fn single_long_paragraph_line_window_split() {
|
||||
let fixture_path = fixtures_dir().join("sample_long_paragraph.txt");
|
||||
let text = std::fs::read_to_string(&fixture_path)
|
||||
.unwrap_or_else(|e| panic!("cannot read fixture {}: {e}", fixture_path.display()));
|
||||
|
||||
assert_eq!(
|
||||
text.lines().count(),
|
||||
200,
|
||||
"fixture must have exactly 200 lines"
|
||||
);
|
||||
|
||||
let doc = text_doc("shell", &text);
|
||||
let chunks = CodeTextParagraphV1Chunker
|
||||
.chunk(&doc, &policy())
|
||||
.expect("chunk");
|
||||
|
||||
assert_eq!(
|
||||
chunks.len(),
|
||||
3,
|
||||
"expected 3 window chunks for 200-line paragraph, got {}: {chunks:#?}",
|
||||
chunks.len()
|
||||
);
|
||||
|
||||
let expected_ranges: &[(u32, u32)] = &[(1, 80), (61, 140), (121, 200)];
|
||||
let actual_ranges: Vec<(u32, u32)> = chunks
|
||||
.iter()
|
||||
.map(|c| match &c.source_spans[0] {
|
||||
SourceSpan::Code {
|
||||
line_start,
|
||||
line_end,
|
||||
..
|
||||
} => (*line_start, *line_end),
|
||||
other => panic!("expected Code span, got {other:?}"),
|
||||
})
|
||||
.collect();
|
||||
|
||||
assert_eq!(
|
||||
actual_ranges, expected_ranges,
|
||||
"window ranges mismatch: got {actual_ranges:?}, expected {expected_ranges:?}"
|
||||
);
|
||||
|
||||
// All chunk_ids must be distinct (#L{window_start} suffix differentiates them).
|
||||
let ids: std::collections::HashSet<_> = chunks.iter().map(|c| c.chunk_id.clone()).collect();
|
||||
assert_eq!(
|
||||
ids.len(),
|
||||
chunks.len(),
|
||||
"oversize window chunks must have distinct chunk_ids"
|
||||
);
|
||||
}
|
||||
|
||||
/// An empty source file (no non-blank lines) must yield zero chunks.
|
||||
#[test]
|
||||
fn empty_file_emits_zero_chunks() {
|
||||
let doc = text_doc("shell", "");
|
||||
let chunks = CodeTextParagraphV1Chunker
|
||||
.chunk(&doc, &policy())
|
||||
.expect("chunk");
|
||||
|
||||
assert_eq!(
|
||||
chunks.len(),
|
||||
0,
|
||||
"empty file must yield 0 chunks, got {}: {chunks:#?}",
|
||||
chunks.len()
|
||||
);
|
||||
}
|
||||
|
||||
/// The `lang` field on each emitted chunk must match the `lang` passed to
|
||||
/// `text_doc`, regardless of content. `symbol` must be `None` (Tier 3 spec).
|
||||
#[test]
|
||||
fn lang_field_preserved_from_input_doc() {
|
||||
let doc = text_doc("yaml", "key1: value1\nkey2: value2\n");
|
||||
let chunks = CodeTextParagraphV1Chunker
|
||||
.chunk(&doc, &policy())
|
||||
.expect("chunk");
|
||||
|
||||
assert!(!chunks.is_empty(), "expected at least one chunk");
|
||||
|
||||
match &chunks[0].source_spans[0] {
|
||||
SourceSpan::Code { lang, symbol, .. } => {
|
||||
assert_eq!(
|
||||
lang.as_deref(),
|
||||
Some("yaml"),
|
||||
"lang must be 'yaml', got {lang:?}"
|
||||
);
|
||||
assert!(
|
||||
symbol.is_none(),
|
||||
"symbol must be None for Tier 3 chunker, got {symbol:?}"
|
||||
);
|
||||
}
|
||||
other => panic!("expected Code span, got {other:?}"),
|
||||
}
|
||||
}
|
||||
134
crates/kebab-chunk/tests/dockerfile_file_v1.rs
Normal file
134
crates/kebab-chunk/tests/dockerfile_file_v1.rs
Normal file
@@ -0,0 +1,134 @@
|
||||
//! Behavioural tests for `DockerfileFileV1Chunker`.
|
||||
//!
|
||||
//! Documents are constructed manually (no kebab-parse-code dependency) by
|
||||
//! placing the raw Dockerfile text into a single `Block::Code`, mirroring the
|
||||
//! pattern used in `k8s_manifest_resource_v1.rs`.
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use kebab_chunk::DockerfileFileV1Chunker;
|
||||
use kebab_core::{
|
||||
AssetId, Block, CanonicalDocument, ChunkPolicy, Chunker, ChunkerVersion, CodeBlock,
|
||||
CommonBlock, Lang, Metadata, ParserVersion, Provenance, SourceSpan, SourceType, TrustLevel,
|
||||
WorkspacePath, id_for_block, id_for_doc,
|
||||
};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
// ── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
fn fixtures_dir() -> PathBuf {
|
||||
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("tests")
|
||||
.join("fixtures")
|
||||
}
|
||||
|
||||
/// Build a `CanonicalDocument` with a single `Block::Code` containing `dockerfile_text`.
|
||||
fn dockerfile_doc(dockerfile_text: &str) -> CanonicalDocument {
|
||||
let wp = WorkspacePath("build/Dockerfile".into());
|
||||
let aid = AssetId("d".repeat(64));
|
||||
let pv = ParserVersion("code-dockerfile-v1".into());
|
||||
let doc_id = id_for_doc(&wp, &aid, &pv);
|
||||
|
||||
let line_count = dockerfile_text.lines().count() as u32;
|
||||
let span = SourceSpan::Code {
|
||||
line_start: 1,
|
||||
line_end: line_count.max(1),
|
||||
symbol: None,
|
||||
lang: Some("dockerfile".into()),
|
||||
};
|
||||
let bid = id_for_block(&doc_id, "code", &[], 0, &span);
|
||||
let block = Block::Code(CodeBlock {
|
||||
common: CommonBlock {
|
||||
block_id: bid,
|
||||
heading_path: vec![],
|
||||
source_span: span,
|
||||
},
|
||||
lang: Some("dockerfile".into()),
|
||||
code: dockerfile_text.to_string(),
|
||||
});
|
||||
|
||||
CanonicalDocument {
|
||||
doc_id,
|
||||
source_asset_id: aid,
|
||||
workspace_path: wp,
|
||||
title: "Dockerfile".into(),
|
||||
lang: Lang("und".into()),
|
||||
blocks: vec![block],
|
||||
metadata: Metadata {
|
||||
aliases: vec![],
|
||||
tags: vec![],
|
||||
created_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
updated_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
source_type: SourceType::Note,
|
||||
trust_level: TrustLevel::Primary,
|
||||
user_id_alias: None,
|
||||
user: Default::default(),
|
||||
repo: Some("kebab".into()),
|
||||
git_branch: Some("main".into()),
|
||||
git_commit: Some("0".repeat(40)),
|
||||
code_lang: Some("dockerfile".into()),
|
||||
},
|
||||
provenance: Provenance { events: vec![] },
|
||||
parser_version: pv,
|
||||
schema_version: 1,
|
||||
doc_version: 1,
|
||||
last_chunker_version: None,
|
||||
last_embedding_version: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn policy() -> ChunkPolicy {
|
||||
ChunkPolicy {
|
||||
target_tokens: 500,
|
||||
overlap_tokens: 80,
|
||||
respect_markdown_headings: false,
|
||||
chunker_version: ChunkerVersion("dockerfile-file-v1".into()),
|
||||
}
|
||||
}
|
||||
|
||||
// ── tests ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// A simple 5-line Dockerfile fixture must emit exactly 1 chunk with the
|
||||
/// correct symbol, lang, and line range.
|
||||
#[test]
|
||||
fn dockerfile_emits_single_chunk() {
|
||||
let fixture_path = fixtures_dir().join("sample.dockerfile");
|
||||
let text = std::fs::read_to_string(&fixture_path)
|
||||
.unwrap_or_else(|e| panic!("cannot read fixture {}: {e}", fixture_path.display()));
|
||||
|
||||
let doc = dockerfile_doc(&text);
|
||||
let chunks = DockerfileFileV1Chunker
|
||||
.chunk(&doc, &policy())
|
||||
.expect("chunk");
|
||||
|
||||
assert_eq!(
|
||||
chunks.len(),
|
||||
1,
|
||||
"expected 1 chunk, got {}: {chunks:#?}",
|
||||
chunks.len()
|
||||
);
|
||||
|
||||
// Inspect the Chunk's source_spans for symbol / lang / line range.
|
||||
let span = chunks[0].source_spans.first().expect("at least one span");
|
||||
match span {
|
||||
SourceSpan::Code {
|
||||
line_start,
|
||||
line_end,
|
||||
symbol,
|
||||
lang,
|
||||
} => {
|
||||
assert_eq!(*line_start, 1, "line_start must be 1");
|
||||
assert_eq!(*line_end, 5, "line_end must be 5 (5-line fixture)");
|
||||
assert_eq!(
|
||||
symbol.as_deref(),
|
||||
Some("<dockerfile>"),
|
||||
"symbol must be '<dockerfile>'"
|
||||
);
|
||||
assert_eq!(lang.as_deref(), Some("dockerfile"), "lang must be 'dockerfile'");
|
||||
}
|
||||
other => panic!("expected SourceSpan::Code, got {other:?}"),
|
||||
}
|
||||
|
||||
// Verify chunker_version label.
|
||||
assert_eq!(chunks[0].chunker_version.0, "dockerfile-file-v1");
|
||||
}
|
||||
5
crates/kebab-chunk/tests/fixtures/sample.dockerfile
vendored
Normal file
5
crates/kebab-chunk/tests/fixtures/sample.dockerfile
vendored
Normal file
@@ -0,0 +1,5 @@
|
||||
FROM rust:1.94-slim AS builder
|
||||
WORKDIR /app
|
||||
COPY . .
|
||||
RUN cargo build --release
|
||||
CMD ["/app/target/release/kebab"]
|
||||
7
crates/kebab-chunk/tests/fixtures/sample_cargo.toml
vendored
Normal file
7
crates/kebab-chunk/tests/fixtures/sample_cargo.toml
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
[package]
|
||||
name = "demo"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
[dependencies]
|
||||
serde = "1"
|
||||
5
crates/kebab-chunk/tests/fixtures/sample_go.mod
vendored
Normal file
5
crates/kebab-chunk/tests/fixtures/sample_go.mod
vendored
Normal file
@@ -0,0 +1,5 @@
|
||||
module example.com/demo
|
||||
|
||||
go 1.22
|
||||
|
||||
require github.com/spf13/cobra v1.8.0
|
||||
34
crates/kebab-chunk/tests/fixtures/sample_k8s.yaml
vendored
Normal file
34
crates/kebab-chunk/tests/fixtures/sample_k8s.yaml
vendored
Normal file
@@ -0,0 +1,34 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: api-server
|
||||
namespace: prod
|
||||
spec:
|
||||
replicas: 3
|
||||
selector:
|
||||
matchLabels:
|
||||
app: api-server
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: api-server
|
||||
spec:
|
||||
containers:
|
||||
- name: api
|
||||
image: example/api:1.2.3
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: api-server
|
||||
namespace: prod
|
||||
spec:
|
||||
selector:
|
||||
app: api-server
|
||||
ports:
|
||||
- port: 80
|
||||
targetPort: 8080
|
||||
---
|
||||
# Non-k8s document — apiVersion missing
|
||||
kind: ClusterIP
|
||||
foo: bar
|
||||
200
crates/kebab-chunk/tests/fixtures/sample_long_paragraph.txt
vendored
Normal file
200
crates/kebab-chunk/tests/fixtures/sample_long_paragraph.txt
vendored
Normal file
@@ -0,0 +1,200 @@
|
||||
line 001
|
||||
line 002
|
||||
line 003
|
||||
line 004
|
||||
line 005
|
||||
line 006
|
||||
line 007
|
||||
line 008
|
||||
line 009
|
||||
line 010
|
||||
line 011
|
||||
line 012
|
||||
line 013
|
||||
line 014
|
||||
line 015
|
||||
line 016
|
||||
line 017
|
||||
line 018
|
||||
line 019
|
||||
line 020
|
||||
line 021
|
||||
line 022
|
||||
line 023
|
||||
line 024
|
||||
line 025
|
||||
line 026
|
||||
line 027
|
||||
line 028
|
||||
line 029
|
||||
line 030
|
||||
line 031
|
||||
line 032
|
||||
line 033
|
||||
line 034
|
||||
line 035
|
||||
line 036
|
||||
line 037
|
||||
line 038
|
||||
line 039
|
||||
line 040
|
||||
line 041
|
||||
line 042
|
||||
line 043
|
||||
line 044
|
||||
line 045
|
||||
line 046
|
||||
line 047
|
||||
line 048
|
||||
line 049
|
||||
line 050
|
||||
line 051
|
||||
line 052
|
||||
line 053
|
||||
line 054
|
||||
line 055
|
||||
line 056
|
||||
line 057
|
||||
line 058
|
||||
line 059
|
||||
line 060
|
||||
line 061
|
||||
line 062
|
||||
line 063
|
||||
line 064
|
||||
line 065
|
||||
line 066
|
||||
line 067
|
||||
line 068
|
||||
line 069
|
||||
line 070
|
||||
line 071
|
||||
line 072
|
||||
line 073
|
||||
line 074
|
||||
line 075
|
||||
line 076
|
||||
line 077
|
||||
line 078
|
||||
line 079
|
||||
line 080
|
||||
line 081
|
||||
line 082
|
||||
line 083
|
||||
line 084
|
||||
line 085
|
||||
line 086
|
||||
line 087
|
||||
line 088
|
||||
line 089
|
||||
line 090
|
||||
line 091
|
||||
line 092
|
||||
line 093
|
||||
line 094
|
||||
line 095
|
||||
line 096
|
||||
line 097
|
||||
line 098
|
||||
line 099
|
||||
line 100
|
||||
line 101
|
||||
line 102
|
||||
line 103
|
||||
line 104
|
||||
line 105
|
||||
line 106
|
||||
line 107
|
||||
line 108
|
||||
line 109
|
||||
line 110
|
||||
line 111
|
||||
line 112
|
||||
line 113
|
||||
line 114
|
||||
line 115
|
||||
line 116
|
||||
line 117
|
||||
line 118
|
||||
line 119
|
||||
line 120
|
||||
line 121
|
||||
line 122
|
||||
line 123
|
||||
line 124
|
||||
line 125
|
||||
line 126
|
||||
line 127
|
||||
line 128
|
||||
line 129
|
||||
line 130
|
||||
line 131
|
||||
line 132
|
||||
line 133
|
||||
line 134
|
||||
line 135
|
||||
line 136
|
||||
line 137
|
||||
line 138
|
||||
line 139
|
||||
line 140
|
||||
line 141
|
||||
line 142
|
||||
line 143
|
||||
line 144
|
||||
line 145
|
||||
line 146
|
||||
line 147
|
||||
line 148
|
||||
line 149
|
||||
line 150
|
||||
line 151
|
||||
line 152
|
||||
line 153
|
||||
line 154
|
||||
line 155
|
||||
line 156
|
||||
line 157
|
||||
line 158
|
||||
line 159
|
||||
line 160
|
||||
line 161
|
||||
line 162
|
||||
line 163
|
||||
line 164
|
||||
line 165
|
||||
line 166
|
||||
line 167
|
||||
line 168
|
||||
line 169
|
||||
line 170
|
||||
line 171
|
||||
line 172
|
||||
line 173
|
||||
line 174
|
||||
line 175
|
||||
line 176
|
||||
line 177
|
||||
line 178
|
||||
line 179
|
||||
line 180
|
||||
line 181
|
||||
line 182
|
||||
line 183
|
||||
line 184
|
||||
line 185
|
||||
line 186
|
||||
line 187
|
||||
line 188
|
||||
line 189
|
||||
line 190
|
||||
line 191
|
||||
line 192
|
||||
line 193
|
||||
line 194
|
||||
line 195
|
||||
line 196
|
||||
line 197
|
||||
line 198
|
||||
line 199
|
||||
line 200
|
||||
7
crates/kebab-chunk/tests/fixtures/sample_package.json
vendored
Normal file
7
crates/kebab-chunk/tests/fixtures/sample_package.json
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"name": "demo",
|
||||
"version": "0.1.0",
|
||||
"dependencies": {
|
||||
"react": "^18.0.0"
|
||||
}
|
||||
}
|
||||
7
crates/kebab-chunk/tests/fixtures/sample_pom.xml
vendored
Normal file
7
crates/kebab-chunk/tests/fixtures/sample_pom.xml
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>com.demo</groupId>
|
||||
<artifactId>demo</artifactId>
|
||||
<version>0.1.0</version>
|
||||
</project>
|
||||
15
crates/kebab-chunk/tests/fixtures/sample_shell.sh
vendored
Normal file
15
crates/kebab-chunk/tests/fixtures/sample_shell.sh
vendored
Normal file
@@ -0,0 +1,15 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# First paragraph: env setup
|
||||
export KEBAB_HOME="${KEBAB_HOME:-$HOME/.local/share/kebab}"
|
||||
mkdir -p "$KEBAB_HOME"
|
||||
cd "$KEBAB_HOME"
|
||||
|
||||
# Second paragraph: ingest
|
||||
echo "ingesting workspace..."
|
||||
kebab ingest --config /etc/kebab/config.toml
|
||||
|
||||
# Third paragraph: report
|
||||
echo "done"
|
||||
kebab schema --json | jq '.stats'
|
||||
277
crates/kebab-chunk/tests/k8s_manifest_resource_v1.rs
Normal file
277
crates/kebab-chunk/tests/k8s_manifest_resource_v1.rs
Normal file
@@ -0,0 +1,277 @@
|
||||
//! Behavioural tests for `K8sManifestResourceV1Chunker`.
|
||||
//!
|
||||
//! Documents are constructed manually (no kebab-parse-code dependency) by
|
||||
//! placing the raw YAML text into a single `Block::Code`, mirroring the
|
||||
//! pattern used in `code_rust_ast_snapshot.rs`.
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use kebab_chunk::K8sManifestResourceV1Chunker;
|
||||
use kebab_core::{
|
||||
AssetId, Block, CanonicalDocument, ChunkPolicy, Chunker, ChunkerVersion, CodeBlock,
|
||||
CommonBlock, Lang, Metadata, ParserVersion, Provenance, SourceSpan, SourceType, TrustLevel,
|
||||
WorkspacePath, id_for_block, id_for_doc,
|
||||
};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
// ── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
fn fixtures_dir() -> PathBuf {
|
||||
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("tests")
|
||||
.join("fixtures")
|
||||
}
|
||||
|
||||
/// Build a `CanonicalDocument` with a single `Block::Code` containing `yaml_text`.
|
||||
fn yaml_doc(yaml_text: &str) -> CanonicalDocument {
|
||||
let wp = WorkspacePath("manifests/deploy.yaml".into());
|
||||
let aid = AssetId("c".repeat(64));
|
||||
let pv = ParserVersion("code-yaml-v1".into());
|
||||
let doc_id = id_for_doc(&wp, &aid, &pv);
|
||||
|
||||
let line_count = yaml_text.lines().count() as u32;
|
||||
let span = SourceSpan::Code {
|
||||
line_start: 1,
|
||||
line_end: line_count.max(1),
|
||||
symbol: None,
|
||||
lang: Some("yaml".into()),
|
||||
};
|
||||
let bid = id_for_block(&doc_id, "code", &[], 0, &span);
|
||||
let block = Block::Code(CodeBlock {
|
||||
common: CommonBlock {
|
||||
block_id: bid,
|
||||
heading_path: vec![],
|
||||
source_span: span,
|
||||
},
|
||||
lang: Some("yaml".into()),
|
||||
code: yaml_text.to_string(),
|
||||
});
|
||||
|
||||
CanonicalDocument {
|
||||
doc_id,
|
||||
source_asset_id: aid,
|
||||
workspace_path: wp,
|
||||
title: "deploy.yaml".into(),
|
||||
lang: Lang("und".into()),
|
||||
blocks: vec![block],
|
||||
metadata: Metadata {
|
||||
aliases: vec![],
|
||||
tags: vec![],
|
||||
created_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
updated_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
source_type: SourceType::Note,
|
||||
trust_level: TrustLevel::Primary,
|
||||
user_id_alias: None,
|
||||
user: Default::default(),
|
||||
repo: Some("kebab".into()),
|
||||
git_branch: Some("main".into()),
|
||||
git_commit: Some("0".repeat(40)),
|
||||
code_lang: Some("yaml".into()),
|
||||
},
|
||||
provenance: Provenance { events: vec![] },
|
||||
parser_version: pv,
|
||||
schema_version: 1,
|
||||
doc_version: 1,
|
||||
last_chunker_version: None,
|
||||
last_embedding_version: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn policy() -> ChunkPolicy {
|
||||
ChunkPolicy {
|
||||
target_tokens: 500,
|
||||
overlap_tokens: 80,
|
||||
respect_markdown_headings: false,
|
||||
chunker_version: ChunkerVersion("k8s-manifest-resource-v1".into()),
|
||||
}
|
||||
}
|
||||
|
||||
// ── tests ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Three YAML documents: 2 valid k8s resources + 1 non-k8s (no apiVersion).
|
||||
/// The chunker must emit exactly 2 chunks with the correct symbols and lang.
|
||||
#[test]
|
||||
fn k8s_multi_doc_emits_one_chunk_per_resource() {
|
||||
let fixture_path = fixtures_dir().join("sample_k8s.yaml");
|
||||
let text = std::fs::read_to_string(&fixture_path)
|
||||
.unwrap_or_else(|e| panic!("cannot read fixture {}: {e}", fixture_path.display()));
|
||||
|
||||
let doc = yaml_doc(&text);
|
||||
let chunks = K8sManifestResourceV1Chunker
|
||||
.chunk(&doc, &policy())
|
||||
.expect("chunk");
|
||||
|
||||
assert_eq!(
|
||||
chunks.len(),
|
||||
2,
|
||||
"expected 2 k8s chunks, got {}: {chunks:#?}",
|
||||
chunks.len()
|
||||
);
|
||||
|
||||
let symbols: Vec<&str> = chunks
|
||||
.iter()
|
||||
.map(|c| {
|
||||
match &c.source_spans[0] {
|
||||
SourceSpan::Code { symbol, .. } => {
|
||||
symbol.as_deref().expect("symbol must be Some for k8s chunks")
|
||||
}
|
||||
other => panic!("expected Code span, got {other:?}"),
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
assert_eq!(
|
||||
symbols,
|
||||
vec!["Deployment/prod/api-server", "Service/prod/api-server"],
|
||||
"symbols mismatch: {symbols:?}"
|
||||
);
|
||||
|
||||
// Verify lang = "yaml" on every chunk.
|
||||
for chunk in &chunks {
|
||||
match &chunk.source_spans[0] {
|
||||
SourceSpan::Code { lang, .. } => {
|
||||
assert_eq!(lang.as_deref(), Some("yaml"), "lang must be 'yaml'");
|
||||
}
|
||||
other => panic!("expected Code span, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
// Verify chunker_version label.
|
||||
for chunk in &chunks {
|
||||
assert_eq!(chunk.chunker_version.0, "k8s-manifest-resource-v1");
|
||||
}
|
||||
}
|
||||
|
||||
/// A YAML document with an indentation error (tab in a space-indented context)
|
||||
/// must cause the chunker to return 0 chunks for the entire file.
|
||||
#[test]
|
||||
fn k8s_invalid_yaml_emits_zero_chunks() {
|
||||
// serde_yaml 0.9 is lenient about duplicate keys (last wins), so use a
|
||||
// genuine YAML structural error (unclosed flow sequence) to force a parse
|
||||
// failure.
|
||||
let actually_bad = "apiVersion: v1\nkind: Service\nfoo: [\nbar\n";
|
||||
|
||||
let doc = yaml_doc(actually_bad);
|
||||
let chunks = K8sManifestResourceV1Chunker
|
||||
.chunk(&doc, &policy())
|
||||
.expect("chunk should not error — return Ok(vec![]) for invalid yaml");
|
||||
|
||||
assert_eq!(
|
||||
chunks.len(),
|
||||
0,
|
||||
"invalid YAML must yield 0 chunks, got {}: {chunks:#?}",
|
||||
chunks.len()
|
||||
);
|
||||
}
|
||||
|
||||
/// A cluster-scoped resource (no `metadata.namespace`) must produce a symbol
|
||||
/// of the form `<Kind>/<name>` (two components, no namespace segment).
|
||||
#[test]
|
||||
fn k8s_cluster_scoped_resource_symbol() {
|
||||
let yaml = "\
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: cluster-admin
|
||||
rules:
|
||||
- apiGroups: [\"*\"]
|
||||
resources: [\"*\"]
|
||||
verbs: [\"*\"]
|
||||
";
|
||||
|
||||
let doc = yaml_doc(yaml);
|
||||
let chunks = K8sManifestResourceV1Chunker
|
||||
.chunk(&doc, &policy())
|
||||
.expect("chunk");
|
||||
|
||||
assert_eq!(
|
||||
chunks.len(),
|
||||
1,
|
||||
"expected 1 chunk for cluster-scoped resource, got {}: {chunks:#?}",
|
||||
chunks.len()
|
||||
);
|
||||
|
||||
match &chunks[0].source_spans[0] {
|
||||
SourceSpan::Code { symbol, lang, .. } => {
|
||||
assert_eq!(
|
||||
symbol.as_deref(),
|
||||
Some("ClusterRole/cluster-admin"),
|
||||
"cluster-scoped symbol must be <Kind>/<name>"
|
||||
);
|
||||
assert_eq!(lang.as_deref(), Some("yaml"));
|
||||
}
|
||||
other => panic!("expected Code span, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// 200+ line resource exercises `tier2_shared::push_chunks_with_oversize`'s
|
||||
/// line-window split branch. All chunks must share the same symbol
|
||||
/// (`<Kind>/<ns>/<name>`); their line ranges must form a contiguous
|
||||
/// partition; chunk_ids must all differ (the `#L{k}` suffix on `id_for_chunk`
|
||||
/// ensures uniqueness across windows). Spec p10-2 risks section explicitly
|
||||
/// flags "거대 ConfigMap" — this test covers that path.
|
||||
#[test]
|
||||
fn k8s_oversize_splits_into_line_windows_sharing_symbol() {
|
||||
// ConfigMap with 250 data keys → ~256 total lines, > AST_CHUNK_MAX_LINES (200).
|
||||
let mut yaml = String::from(
|
||||
"apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: big\n namespace: prod\ndata:\n",
|
||||
);
|
||||
for i in 0..250 {
|
||||
yaml.push_str(&format!(" key{i}: value{i}\n"));
|
||||
}
|
||||
|
||||
let doc = yaml_doc(&yaml);
|
||||
let chunks = K8sManifestResourceV1Chunker
|
||||
.chunk(&doc, &policy())
|
||||
.expect("chunk");
|
||||
|
||||
assert!(
|
||||
chunks.len() >= 2,
|
||||
"expected ≥2 chunks for oversize resource, got {}",
|
||||
chunks.len()
|
||||
);
|
||||
|
||||
// Every chunk must share the same symbol + lang.
|
||||
let expected_symbol = "ConfigMap/prod/big";
|
||||
for (i, c) in chunks.iter().enumerate() {
|
||||
match &c.source_spans[0] {
|
||||
SourceSpan::Code { symbol, lang, .. } => {
|
||||
assert_eq!(
|
||||
symbol.as_deref(),
|
||||
Some(expected_symbol),
|
||||
"chunk[{i}] symbol must equal `{expected_symbol}`"
|
||||
);
|
||||
assert_eq!(lang.as_deref(), Some("yaml"));
|
||||
}
|
||||
other => panic!("chunk[{i}]: expected Code span, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
// chunk_ids must all be distinct (oversize fallback's #L{k} suffix).
|
||||
let ids: std::collections::HashSet<_> = chunks.iter().map(|c| c.chunk_id.clone()).collect();
|
||||
assert_eq!(
|
||||
ids.len(),
|
||||
chunks.len(),
|
||||
"oversize chunks must have distinct chunk_ids (the #L{{k}} suffix should disambiguate)"
|
||||
);
|
||||
|
||||
// Line ranges must form a contiguous partition: chunk[i].line_end + 1 == chunk[i+1].line_start.
|
||||
let ranges: Vec<(u32, u32)> = chunks
|
||||
.iter()
|
||||
.map(|c| match &c.source_spans[0] {
|
||||
SourceSpan::Code { line_start, line_end, .. } => (*line_start, *line_end),
|
||||
other => panic!("expected Code span, got {other:?}"),
|
||||
})
|
||||
.collect();
|
||||
for w in ranges.windows(2) {
|
||||
let (_, prev_end) = w[0];
|
||||
let (next_start, _) = w[1];
|
||||
assert_eq!(
|
||||
prev_end + 1,
|
||||
next_start,
|
||||
"line ranges must be contiguous: {} → {} (got gap or overlap)",
|
||||
prev_end,
|
||||
next_start
|
||||
);
|
||||
}
|
||||
}
|
||||
267
crates/kebab-chunk/tests/manifest_file_v1.rs
Normal file
267
crates/kebab-chunk/tests/manifest_file_v1.rs
Normal file
@@ -0,0 +1,267 @@
|
||||
//! Behavioural tests for `ManifestFileV1Chunker`.
|
||||
//!
|
||||
//! Documents are constructed manually (no kebab-parse-code dependency) by
|
||||
//! placing the raw manifest text into a single `Block::Code`, mirroring the
|
||||
//! pattern used in `dockerfile_file_v1.rs`.
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use kebab_chunk::ManifestFileV1Chunker;
|
||||
use kebab_core::{
|
||||
AssetId, Block, CanonicalDocument, ChunkPolicy, Chunker, ChunkerVersion, CodeBlock,
|
||||
CommonBlock, Lang, Metadata, ParserVersion, Provenance, SourceSpan, SourceType, TrustLevel,
|
||||
WorkspacePath, id_for_block, id_for_doc,
|
||||
};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
// ── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
fn fixtures_dir() -> PathBuf {
|
||||
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("tests")
|
||||
.join("fixtures")
|
||||
}
|
||||
|
||||
/// Build a `CanonicalDocument` with a single `Block::Code` containing manifest text.
|
||||
fn manifest_doc(lang: &str, manifest_text: &str) -> CanonicalDocument {
|
||||
let wp = WorkspacePath(format!("build/{}", manifest_filename(lang)));
|
||||
let aid = AssetId("m".repeat(64));
|
||||
let pv = ParserVersion("code-manifest-v1".into());
|
||||
let doc_id = id_for_doc(&wp, &aid, &pv);
|
||||
|
||||
let line_count = manifest_text.lines().count() as u32;
|
||||
let span = SourceSpan::Code {
|
||||
line_start: 1,
|
||||
line_end: line_count.max(1),
|
||||
symbol: None,
|
||||
lang: Some(lang.into()),
|
||||
};
|
||||
let bid = id_for_block(&doc_id, "code", &[], 0, &span);
|
||||
let block = Block::Code(CodeBlock {
|
||||
common: CommonBlock {
|
||||
block_id: bid,
|
||||
heading_path: vec![],
|
||||
source_span: span,
|
||||
},
|
||||
lang: Some(lang.into()),
|
||||
code: manifest_text.to_string(),
|
||||
});
|
||||
|
||||
CanonicalDocument {
|
||||
doc_id,
|
||||
source_asset_id: aid,
|
||||
workspace_path: wp,
|
||||
title: format!("Manifest ({})", lang),
|
||||
lang: Lang("und".into()),
|
||||
blocks: vec![block],
|
||||
metadata: Metadata {
|
||||
aliases: vec![],
|
||||
tags: vec![],
|
||||
created_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
updated_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
|
||||
source_type: SourceType::Note,
|
||||
trust_level: TrustLevel::Primary,
|
||||
user_id_alias: None,
|
||||
user: Default::default(),
|
||||
repo: Some("kebab".into()),
|
||||
git_branch: Some("main".into()),
|
||||
git_commit: Some("0".repeat(40)),
|
||||
code_lang: Some(lang.into()),
|
||||
},
|
||||
provenance: Provenance { events: vec![] },
|
||||
parser_version: pv,
|
||||
schema_version: 1,
|
||||
doc_version: 1,
|
||||
last_chunker_version: None,
|
||||
last_embedding_version: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn manifest_filename(lang: &str) -> &'static str {
|
||||
match lang {
|
||||
"toml" => "Cargo.toml",
|
||||
"json" => "package.json",
|
||||
"xml" => "pom.xml",
|
||||
"go-mod" => "go.mod",
|
||||
_ => "manifest",
|
||||
}
|
||||
}
|
||||
|
||||
fn policy() -> ChunkPolicy {
|
||||
ChunkPolicy {
|
||||
target_tokens: 500,
|
||||
overlap_tokens: 80,
|
||||
respect_markdown_headings: false,
|
||||
chunker_version: ChunkerVersion("manifest-file-v1".into()),
|
||||
}
|
||||
}
|
||||
|
||||
// ── tests ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// A Cargo.toml fixture must emit exactly 1 chunk with the correct symbol,
|
||||
/// lang, and line range.
|
||||
#[test]
|
||||
fn cargo_toml_single_chunk_with_toml_lang() {
|
||||
let fixture_path = fixtures_dir().join("sample_cargo.toml");
|
||||
let text = std::fs::read_to_string(&fixture_path)
|
||||
.unwrap_or_else(|e| panic!("cannot read fixture {}: {e}", fixture_path.display()));
|
||||
|
||||
let doc = manifest_doc("toml", &text);
|
||||
let chunks = ManifestFileV1Chunker
|
||||
.chunk(&doc, &policy())
|
||||
.expect("chunk");
|
||||
|
||||
assert_eq!(
|
||||
chunks.len(),
|
||||
1,
|
||||
"expected 1 chunk, got {}: {chunks:#?}",
|
||||
chunks.len()
|
||||
);
|
||||
|
||||
let span = chunks[0].source_spans.first().expect("at least one span");
|
||||
match span {
|
||||
SourceSpan::Code {
|
||||
line_start,
|
||||
line_end: _,
|
||||
symbol,
|
||||
lang,
|
||||
} => {
|
||||
assert_eq!(*line_start, 1, "line_start must be 1");
|
||||
assert_eq!(
|
||||
symbol.as_deref(),
|
||||
Some("<manifest>"),
|
||||
"symbol must be '<manifest>'"
|
||||
);
|
||||
assert_eq!(lang.as_deref(), Some("toml"), "lang must be 'toml'");
|
||||
}
|
||||
other => panic!("expected SourceSpan::Code, got {other:?}"),
|
||||
}
|
||||
|
||||
assert_eq!(chunks[0].chunker_version.0, "manifest-file-v1");
|
||||
}
|
||||
|
||||
/// A package.json fixture must emit exactly 1 chunk with the correct symbol,
|
||||
/// lang, and line range.
|
||||
#[test]
|
||||
fn package_json_single_chunk_with_json_lang() {
|
||||
let fixture_path = fixtures_dir().join("sample_package.json");
|
||||
let text = std::fs::read_to_string(&fixture_path)
|
||||
.unwrap_or_else(|e| panic!("cannot read fixture {}: {e}", fixture_path.display()));
|
||||
|
||||
let doc = manifest_doc("json", &text);
|
||||
let chunks = ManifestFileV1Chunker
|
||||
.chunk(&doc, &policy())
|
||||
.expect("chunk");
|
||||
|
||||
assert_eq!(
|
||||
chunks.len(),
|
||||
1,
|
||||
"expected 1 chunk, got {}: {chunks:#?}",
|
||||
chunks.len()
|
||||
);
|
||||
|
||||
let span = chunks[0].source_spans.first().expect("at least one span");
|
||||
match span {
|
||||
SourceSpan::Code {
|
||||
line_start,
|
||||
line_end: _,
|
||||
symbol,
|
||||
lang,
|
||||
} => {
|
||||
assert_eq!(*line_start, 1, "line_start must be 1");
|
||||
assert_eq!(
|
||||
symbol.as_deref(),
|
||||
Some("<manifest>"),
|
||||
"symbol must be '<manifest>'"
|
||||
);
|
||||
assert_eq!(lang.as_deref(), Some("json"), "lang must be 'json'");
|
||||
}
|
||||
other => panic!("expected SourceSpan::Code, got {other:?}"),
|
||||
}
|
||||
|
||||
assert_eq!(chunks[0].chunker_version.0, "manifest-file-v1");
|
||||
}
|
||||
|
||||
/// A pom.xml fixture must emit exactly 1 chunk with the correct symbol,
|
||||
/// lang, and line range.
|
||||
#[test]
|
||||
fn pom_xml_single_chunk_with_xml_lang() {
|
||||
let fixture_path = fixtures_dir().join("sample_pom.xml");
|
||||
let text = std::fs::read_to_string(&fixture_path)
|
||||
.unwrap_or_else(|e| panic!("cannot read fixture {}: {e}", fixture_path.display()));
|
||||
|
||||
let doc = manifest_doc("xml", &text);
|
||||
let chunks = ManifestFileV1Chunker
|
||||
.chunk(&doc, &policy())
|
||||
.expect("chunk");
|
||||
|
||||
assert_eq!(
|
||||
chunks.len(),
|
||||
1,
|
||||
"expected 1 chunk, got {}: {chunks:#?}",
|
||||
chunks.len()
|
||||
);
|
||||
|
||||
let span = chunks[0].source_spans.first().expect("at least one span");
|
||||
match span {
|
||||
SourceSpan::Code {
|
||||
line_start,
|
||||
line_end: _,
|
||||
symbol,
|
||||
lang,
|
||||
} => {
|
||||
assert_eq!(*line_start, 1, "line_start must be 1");
|
||||
assert_eq!(
|
||||
symbol.as_deref(),
|
||||
Some("<manifest>"),
|
||||
"symbol must be '<manifest>'"
|
||||
);
|
||||
assert_eq!(lang.as_deref(), Some("xml"), "lang must be 'xml'");
|
||||
}
|
||||
other => panic!("expected SourceSpan::Code, got {other:?}"),
|
||||
}
|
||||
|
||||
assert_eq!(chunks[0].chunker_version.0, "manifest-file-v1");
|
||||
}
|
||||
|
||||
/// A go.mod fixture must emit exactly 1 chunk with the correct symbol,
|
||||
/// lang, and line range.
|
||||
#[test]
|
||||
fn go_mod_single_chunk_with_go_mod_lang() {
|
||||
let fixture_path = fixtures_dir().join("sample_go.mod");
|
||||
let text = std::fs::read_to_string(&fixture_path)
|
||||
.unwrap_or_else(|e| panic!("cannot read fixture {}: {e}", fixture_path.display()));
|
||||
|
||||
let doc = manifest_doc("go-mod", &text);
|
||||
let chunks = ManifestFileV1Chunker
|
||||
.chunk(&doc, &policy())
|
||||
.expect("chunk");
|
||||
|
||||
assert_eq!(
|
||||
chunks.len(),
|
||||
1,
|
||||
"expected 1 chunk, got {}: {chunks:#?}",
|
||||
chunks.len()
|
||||
);
|
||||
|
||||
let span = chunks[0].source_spans.first().expect("at least one span");
|
||||
match span {
|
||||
SourceSpan::Code {
|
||||
line_start,
|
||||
line_end: _,
|
||||
symbol,
|
||||
lang,
|
||||
} => {
|
||||
assert_eq!(*line_start, 1, "line_start must be 1");
|
||||
assert_eq!(
|
||||
symbol.as_deref(),
|
||||
Some("<manifest>"),
|
||||
"symbol must be '<manifest>'"
|
||||
);
|
||||
assert_eq!(lang.as_deref(), Some("go-mod"), "lang must be 'go-mod'");
|
||||
}
|
||||
other => panic!("expected SourceSpan::Code, got {other:?}"),
|
||||
}
|
||||
|
||||
assert_eq!(chunks[0].chunker_version.0, "manifest-file-v1");
|
||||
}
|
||||
@@ -10,18 +10,39 @@ use std::path::Path;
|
||||
/// `None` if the extension / filename is not recognized.
|
||||
///
|
||||
/// Matching priority:
|
||||
/// 1. exact filename match (e.g. `Dockerfile`, `Makefile`)
|
||||
/// 2. lowercase extension match
|
||||
/// 1. Tier 1 basename exact match (e.g. `Dockerfile`, `Makefile`)
|
||||
/// 2. Tier 2 basename match (e.g. `Cargo.toml`, `package.json`, `build.gradle`)
|
||||
/// 3. Tier 2 `Dockerfile.*` prefix variant
|
||||
/// 4. Tier 1 + Tier 2 extension fallback (lowercase)
|
||||
pub fn code_lang_for_path(path: &Path) -> Option<&'static str> {
|
||||
if let Some(name) = path.file_name().and_then(|n| n.to_str()) {
|
||||
// Tier 1 basename exact match
|
||||
match name {
|
||||
"Dockerfile" => return Some("dockerfile"),
|
||||
"Makefile" | "GNUmakefile" => return Some("make"),
|
||||
_ => {}
|
||||
}
|
||||
|
||||
// Tier 2 basename match (configuration / manifest files)
|
||||
match name {
|
||||
"Cargo.toml" | "pyproject.toml" => return Some("toml"),
|
||||
"package.json" | "tsconfig.json" => return Some("json"),
|
||||
"go.mod" => return Some("go-mod"),
|
||||
"pom.xml" => return Some("xml"),
|
||||
"build.gradle" => return Some("groovy"),
|
||||
_ => {}
|
||||
}
|
||||
|
||||
// Tier 2: `Dockerfile.*` prefix variant (e.g. `Dockerfile.dev`, `Dockerfile.prod`)
|
||||
if name.starts_with("Dockerfile.") && name.len() > "Dockerfile.".len() {
|
||||
return Some("dockerfile");
|
||||
}
|
||||
}
|
||||
|
||||
// Extension fallback (Tier 1 + Tier 2)
|
||||
let ext = path.extension()?.to_str()?.to_ascii_lowercase();
|
||||
match ext.as_str() {
|
||||
// Tier 1 extensions
|
||||
"rs" => Some("rust"),
|
||||
"py" | "pyi" => Some("python"),
|
||||
"ts" | "tsx" | "mts" | "cts" => Some("typescript"),
|
||||
@@ -31,12 +52,15 @@ pub fn code_lang_for_path(path: &Path) -> Option<&'static str> {
|
||||
"kt" | "kts" => Some("kotlin"),
|
||||
"c" | "h" => Some("c"),
|
||||
"cpp" | "cc" | "cxx" | "hpp" | "hh" | "hxx" => Some("cpp"),
|
||||
"sh" | "bash" | "zsh" => Some("shell"),
|
||||
"mk" => Some("make"),
|
||||
// Tier 2 extensions
|
||||
"yaml" | "yml" => Some("yaml"),
|
||||
"toml" => Some("toml"),
|
||||
"json" => Some("json"),
|
||||
"sh" | "bash" | "zsh" => Some("shell"),
|
||||
"mk" => Some("make"),
|
||||
"xml" => Some("xml"),
|
||||
"dockerfile" => Some("dockerfile"),
|
||||
"gradle" => Some("groovy"),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
@@ -118,4 +142,28 @@ mod tests {
|
||||
assert_eq!(module_path_for_tsjs("a/b/c.ts"), "a/b/c");
|
||||
assert_eq!(module_path_for_tsjs("packages/x/src/Foo.ts"), "packages/x/src/Foo");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier2_basename_takes_precedence_over_extension() {
|
||||
assert_eq!(code_lang_for_path(Path::new("Dockerfile")), Some("dockerfile"));
|
||||
assert_eq!(code_lang_for_path(Path::new("foo/Dockerfile.dev")), Some("dockerfile"));
|
||||
assert_eq!(code_lang_for_path(Path::new("myapp.dockerfile")), Some("dockerfile"));
|
||||
assert_eq!(code_lang_for_path(Path::new("repo/Cargo.toml")), Some("toml"));
|
||||
assert_eq!(code_lang_for_path(Path::new("pyproject.toml")), Some("toml"));
|
||||
assert_eq!(code_lang_for_path(Path::new("repo/package.json")), Some("json"));
|
||||
assert_eq!(code_lang_for_path(Path::new("tsconfig.json")), Some("json"));
|
||||
assert_eq!(code_lang_for_path(Path::new("go.mod")), Some("go-mod"));
|
||||
assert_eq!(code_lang_for_path(Path::new("pom.xml")), Some("xml"));
|
||||
assert_eq!(code_lang_for_path(Path::new("build.gradle")), Some("groovy"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier2_extension_fallback() {
|
||||
assert_eq!(code_lang_for_path(Path::new("k8s/deploy.yaml")), Some("yaml"));
|
||||
assert_eq!(code_lang_for_path(Path::new("k8s/deploy.yml")), Some("yaml"));
|
||||
assert_eq!(code_lang_for_path(Path::new("foo/bar.toml")), Some("toml"));
|
||||
assert_eq!(code_lang_for_path(Path::new("foo/bar.json")), Some("json"));
|
||||
assert_eq!(code_lang_for_path(Path::new("foo/bar.xml")), Some("xml"));
|
||||
assert_eq!(code_lang_for_path(Path::new("foo/bar.gradle")), Some("groovy"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,6 +12,12 @@ use kebab_core::{AudioType, ImageType, MediaType};
|
||||
/// `MediaType::Image(_)` / `MediaType::Audio(_)`. Anything else (including
|
||||
/// missing extension) → `MediaType::Other(ext)`.
|
||||
pub(crate) fn media_type_for(path: &Path) -> MediaType {
|
||||
// p10-2: code_lang_for_path is the single source of truth for code lang
|
||||
// (design §3.5). Delegate before falling back to extension branches.
|
||||
if let Some(lang) = kebab_parse_code::code_lang_for_path(path) {
|
||||
return MediaType::Code(lang.to_string());
|
||||
}
|
||||
|
||||
let ext = path
|
||||
.extension()
|
||||
.and_then(|s| s.to_str())
|
||||
@@ -36,23 +42,6 @@ pub(crate) fn media_type_for(path: &Path) -> MediaType {
|
||||
"flac" => MediaType::Audio(AudioType::Flac),
|
||||
"ogg" => MediaType::Audio(AudioType::Ogg),
|
||||
|
||||
// p10-1A-2: Rust is the only code lang activated in 1A. Other
|
||||
// recognized code langs stay Other until their phase (1B+).
|
||||
"rs" => MediaType::Code("rust".to_string()),
|
||||
|
||||
// p10-1B: Python / TS / JS AST chunkers active.
|
||||
"py" | "pyi" => MediaType::Code("python".into()),
|
||||
// .mts / .cts are TypeScript ESM / CommonJS variants — same grammar.
|
||||
"ts" | "tsx" | "mts" | "cts" => MediaType::Code("typescript".into()),
|
||||
"js" | "mjs" | "cjs" | "jsx" => MediaType::Code("javascript".into()),
|
||||
|
||||
// p10-1C-Go: Go ingest activated.
|
||||
"go" => MediaType::Code("go".into()),
|
||||
|
||||
// p10-1C-JK: JVM family (Java + Kotlin) ingest activated.
|
||||
"java" => MediaType::Code("java".into()),
|
||||
"kt" | "kts" => MediaType::Code("kotlin".into()),
|
||||
|
||||
// Empty string (no extension) and any other extension: bucket as
|
||||
// Other and let downstream extractors decide if they support it.
|
||||
_ => MediaType::Other(ext),
|
||||
@@ -96,7 +85,8 @@ mod tests {
|
||||
media_type_for(Path::new("crates/kebab-core/src/lib.rs")),
|
||||
MediaType::Code("rust".to_string())
|
||||
);
|
||||
assert_eq!(media_type_for(Path::new("Cargo.toml")), MediaType::Other("toml".to_string()));
|
||||
// Cargo.toml is a Tier 2 code manifest (p10-2), handled by code_lang_for_path
|
||||
assert_eq!(media_type_for(Path::new("Cargo.toml")), MediaType::Code("toml".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -149,4 +139,14 @@ mod tests {
|
||||
MediaType::Other(String::new())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier2_files_map_to_media_code() {
|
||||
assert_eq!(media_type_for(Path::new("a/deploy.yaml")), MediaType::Code("yaml".into()));
|
||||
assert_eq!(media_type_for(Path::new("a/Dockerfile")), MediaType::Code("dockerfile".into()));
|
||||
assert_eq!(media_type_for(Path::new("a/Cargo.toml")), MediaType::Code("toml".into()));
|
||||
assert_eq!(media_type_for(Path::new("a/pom.xml")), MediaType::Code("xml".into()));
|
||||
assert_eq!(media_type_for(Path::new("a/build.gradle")), MediaType::Code("groovy".into()));
|
||||
assert_eq!(media_type_for(Path::new("a/go.mod")), MediaType::Code("go-mod".into()));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,7 +22,7 @@ Cargo workspace, 함수 호출 기반 모듈러 모놀리스. UI binary (`kebab-
|
||||
| OCR | Ollama vision LM (default `gemma4:e4b`) — `OcrEngine` trait 으로 Tesseract / Apple Vision 등 future swap (HOTFIXES P6-2) |
|
||||
| Image caption | Ollama vision LM, runtime gate `image.caption.enabled` (default OFF) |
|
||||
| PDF parser | `lopdf` per-page 텍스트, `chunker_version = "pdf-page-v1"` 가 PDF 자산에 하드코딩 (HOTFIXES P7-3) |
|
||||
| code parser | `tree-sitter` + `tree-sitter-rust` / `tree-sitter-python` / `tree-sitter-typescript` / `tree-sitter-javascript` / `tree-sitter-go` / `tree-sitter-java` / `tree-sitter-kotlin-ng` — **parser-side** (`kebab-parse-code`), chunker-side 아님 (design §6.3). chunker versions: Rust = `code-rust-ast-v1`, Python = `code-python-ast-v1`, TypeScript = `code-ts-ast-v1`, JavaScript = `code-js-ast-v1`, Go = `code-go-ast-v1`, Java = `code-java-ast-v1`, Kotlin = `code-kotlin-ast-v1`. `ast_chunk_max_lines = 200` 상수 고정 (HOTFIXES 2026-05-19 — Chunker trait 이 per-medium config 미노출). Kotlin grammar 은 `tree-sitter-kotlin-ng` 사용 — bare `tree-sitter-kotlin` 은 tree-sitter 0.21–0.23 에 고착되어 있어 사용 불가. |
|
||||
| code parser | `tree-sitter` + `tree-sitter-rust` / `tree-sitter-python` / `tree-sitter-typescript` / `tree-sitter-javascript` / `tree-sitter-go` / `tree-sitter-java` / `tree-sitter-kotlin-ng` — **parser-side** (`kebab-parse-code`), chunker-side 아님 (design §6.3). chunker versions: Rust = `code-rust-ast-v1`, Python = `code-python-ast-v1`, TypeScript = `code-ts-ast-v1`, JavaScript = `code-js-ast-v1`, Go = `code-go-ast-v1`, Java = `code-java-ast-v1`, Kotlin = `code-kotlin-ast-v1`. `ast_chunk_max_lines = 200` 상수 고정 (HOTFIXES 2026-05-19 — Chunker trait 이 per-medium config 미노출). Kotlin grammar 은 `tree-sitter-kotlin-ng` 사용 — bare `tree-sitter-kotlin` 은 tree-sitter 0.21–0.23 에 고착되어 있어 사용 불가. **Tier 2 (p10-2)**: YAML/k8s → `serde_yaml` + `k8s-manifest-resource-v1` (apiVersion+kind per resource), Dockerfile → `dockerfile-file-v1` (whole-file), Cargo.toml/go.mod/.json/.xml/.groovy → `manifest-file-v1` (whole-file). Tier 2 chunkers live in `kebab-chunk`; no tree-sitter grammar needed (structure from file type, not AST). **Tier 3 (p10-3)**: shell scripts (`.sh`/`.bash`/`.zsh`) direct → `code-text-paragraph-v1` (blank-line paragraph segmentation + 80-line / 20-overlap line-window for oversize). Same chunker also serves as fallback when Tier 1/2 emit 0 chunks or Err — non-k8s YAML / invalid YAML / AST extractor failures all picked up. symbol = None; lang preserved from input doc. |
|
||||
| 1B symbol path | workspace path → module path: Python = dotted prefix (`kebab_eval.metrics.compute_mrr`), TypeScript/JavaScript = slash-style prefix (`src/Foo.Foo.search`). Rust 1A-2 는 file-scope nesting 만 (workspace prefix 없음, 비일관 수용 — HOTFIXES 2026-05-20). |
|
||||
| TUI | Ratatui + crossterm — P9-1 Library 패널, P9-2/3/4 진행 예정 |
|
||||
| Desktop | Tauri 2 + `pdfjs-dist` (native PDF render backend 금지) — P9-5 |
|
||||
@@ -52,7 +52,7 @@ flowchart TB
|
||||
ppdf["kebab-parse-pdf"]
|
||||
pimg["kebab-parse-image"]
|
||||
paud["kebab-parse-audio<br/>(P8 보류)"]
|
||||
pcode["kebab-parse-code<br/>(P10-1A-2 + P10-1B + P10-1C-Go + P10-1C-JK)"]
|
||||
pcode["kebab-parse-code<br/>(P10-1A-2 + P10-1B + P10-1C-Go + P10-1C-JK + P10-2 + P10-3)"]
|
||||
ptypes["kebab-parse-types"]
|
||||
norm["kebab-normalize"]
|
||||
chunk["kebab-chunk"]
|
||||
@@ -165,7 +165,14 @@ kebab/
|
||||
│ ├── kebab-source-fs/ # 워크스페이스 walk + checksum (P1-1)
|
||||
│ ├── kebab-parse-md/ # Markdown frontmatter + blocks (P1-2/3)
|
||||
│ ├── kebab-normalize/ # ParsedBlock → CanonicalDocument (P1-4)
|
||||
│ ├── kebab-chunk/ # heading-aware + pdf-page-v1 + code-rust-ast-v1 + code-python-ast-v1 + code-ts-ast-v1 + code-js-ast-v1 + code-go-ast-v1 + code-java-ast-v1 + code-kotlin-ast-v1 chunker (P1-5, P7-2, P10-1A-2, P10-1B, P10-1C-Go, P10-1C-JK)
|
||||
│ ├── kebab-chunk/ # heading-aware + pdf-page-v1 + code-*-ast-v1 (Tier 1) + k8s-manifest-resource-v1 + dockerfile-file-v1 + manifest-file-v1 + tier2_shared (P10-2) + code-text-paragraph-v1 (P10-3) chunker (P1-5, P7-2, P10-1A-2, P10-1B, P10-1C-Go, P10-1C-JK, P10-2, P10-3)
|
||||
│ │ └── src/
|
||||
│ │ ├── code_*_ast_v1.rs # Tier 1 AST chunkers (rust/python/ts/js/go/java/kotlin)
|
||||
│ │ ├── k8s_manifest_resource_v1.rs # Tier 2 (p10-2): YAML multi-doc, apiVersion+kind per resource
|
||||
│ │ ├── dockerfile_file_v1.rs # Tier 2 (p10-2): whole-file Dockerfile
|
||||
│ │ ├── manifest_file_v1.rs # Tier 2 (p10-2): whole-file Cargo.toml / go.mod / .json / .xml / .groovy
|
||||
│ │ ├── code_text_paragraph_v1.rs # Tier 3 (p10-3): blank-line paragraph + 80/20 line-window fallback
|
||||
│ │ └── tier2_shared.rs # Tier 2 (p10-2): shared oversize fallback + Chunk builder helpers
|
||||
│ ├── kebab-store-sqlite/ # SQLite + FTS5 (V001/V002/V003) (P1-6, P2-1, P3-3)
|
||||
│ ├── kebab-search/ # Lexical + Vector + Hybrid retriever (P2-2, P3-4)
|
||||
│ ├── kebab-embed/ kebab-embed-local/ # Embedder trait + fastembed adapter (P3-1, P3-2)
|
||||
|
||||
128
docs/SMOKE.md
128
docs/SMOKE.md
@@ -422,6 +422,132 @@ KB search --mode hybrid "Hello" --code-lang go --json | \
|
||||
# 기대: symbol = "main.Hello", lang = "go"
|
||||
```
|
||||
|
||||
## P10-2 Tier 2 리소스 파일 색인
|
||||
|
||||
P10-1C-Go 와 동일한 격리 KB 설정. `.yaml` / `Dockerfile` / `.toml` 등 Tier 2 리소스 파일을 워크스페이스에 두고 ingest 하면 각 확장자에 맞는 chunker 로 처리된다.
|
||||
|
||||
```bash
|
||||
# 1) Kubernetes manifest (YAML multi-doc)
|
||||
cat > /tmp/kebab-smoke/workspace/deploy.yaml <<'EOF'
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: my-app
|
||||
namespace: default
|
||||
spec:
|
||||
replicas: 2
|
||||
selector:
|
||||
matchLabels:
|
||||
app: my-app
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: my-app
|
||||
spec:
|
||||
containers:
|
||||
- name: app
|
||||
image: my-app:latest
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: my-app-svc
|
||||
namespace: default
|
||||
spec:
|
||||
selector:
|
||||
app: my-app
|
||||
ports:
|
||||
- port: 80
|
||||
EOF
|
||||
|
||||
# 2) Dockerfile (전체 파일 단일 chunk)
|
||||
cat > /tmp/kebab-smoke/workspace/Dockerfile <<'EOF'
|
||||
FROM rust:1.85 AS builder
|
||||
WORKDIR /app
|
||||
COPY . .
|
||||
RUN cargo build --release
|
||||
|
||||
FROM debian:bookworm-slim
|
||||
COPY --from=builder /app/target/release/kebab /usr/local/bin/kebab
|
||||
ENTRYPOINT ["kebab"]
|
||||
EOF
|
||||
|
||||
# 3) Cargo.toml (manifest — 전체 파일 단일 chunk)
|
||||
cp Cargo.toml /tmp/kebab-smoke/workspace/Cargo.toml
|
||||
|
||||
# 4) ingest
|
||||
KB ingest
|
||||
|
||||
# 5) 언어별 검색 (citation.symbol 확인)
|
||||
KB search --mode hybrid "Deployment" --code-lang yaml --json | \
|
||||
jq '{hits: [.hits[] | {symbol: .citation.symbol, lang: .citation.lang}]}'
|
||||
# 기대: symbol = "Deployment/default/my-app" (kind/namespace/name), lang = "yaml"
|
||||
|
||||
KB search --mode hybrid "rust:1.85" --code-lang dockerfile --json | \
|
||||
jq '{hits: [.hits[] | {symbol: .citation.symbol, lang: .citation.lang}]}'
|
||||
# 기대: symbol = "<dockerfile>", lang = "dockerfile"
|
||||
|
||||
KB search --mode hybrid "kebab-cli" --code-lang toml --json | \
|
||||
jq '{hits: [.hits[] | {symbol: .citation.symbol, lang: .citation.lang}]}'
|
||||
# 기대: symbol = "<manifest>", lang = "toml"
|
||||
|
||||
# 6) schema stats 에 Tier 2 언어 카운트 확인
|
||||
KB --json schema | jq '.stats.code_lang_breakdown'
|
||||
# 기대: {"yaml": N, "dockerfile": N, "toml": N, ...}
|
||||
```
|
||||
|
||||
**Tier 2 citation.symbol 컨벤션**:
|
||||
|
||||
- **YAML k8s 리소스**: `<kind>/<namespace>/<name>` (예: `Deployment/default/my-app`). `namespace` 없으면 `<kind>/<name>`. multi-doc YAML 은 `---` 구분자 기준으로 resource 별 chunk.
|
||||
- **Dockerfile**: `<dockerfile>` (고정 심볼, 전체 파일이 단일 chunk).
|
||||
- **TOML / JSON / XML / Groovy / go.mod**: `<manifest>` (고정 심볼, 전체 파일이 단일 chunk). 단, 파일이 `tier2_shared` 의 oversize threshold 초과 시 줄 단위 fallback chunk.
|
||||
|
||||
## P10-3 Tier 3 paragraph fallback
|
||||
|
||||
P10-2 와 동일한 격리 KB 설정. `.sh` 파일은 direct, 비-k8s YAML 은 fallback 으로 들어간다.
|
||||
|
||||
```bash
|
||||
# 1) shell script (direct Tier 3)
|
||||
cat > /tmp/kebab-smoke/workspace/deploy.sh <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
set -e
|
||||
|
||||
echo "ingesting..."
|
||||
kebab ingest
|
||||
|
||||
echo "done"
|
||||
kebab schema --json | jq '.stats'
|
||||
EOF
|
||||
|
||||
# 2) 비-k8s YAML (Tier 2 가 0 chunk → Tier 3 fallback)
|
||||
cat > /tmp/kebab-smoke/workspace/docker-compose.yml <<'EOF'
|
||||
version: '3'
|
||||
services:
|
||||
api:
|
||||
image: nginx:latest
|
||||
ports:
|
||||
- 8080:80
|
||||
EOF
|
||||
|
||||
# 3) ingest
|
||||
KB ingest
|
||||
|
||||
# 4) 언어별 검색 (citation.symbol = None 확인)
|
||||
KB search --mode hybrid "ingest" --code-lang shell --json | \
|
||||
jq '{hits: [.hits[] | {symbol: .citation.symbol, lang: .citation.lang, chunker: .chunker_version}]}'
|
||||
# 기대: symbol = null, lang = "shell", chunker_version = "code-text-paragraph-v1"
|
||||
|
||||
KB search --mode hybrid "nginx" --code-lang yaml --json | \
|
||||
jq '{hits: [.hits[] | {symbol: .citation.symbol, lang: .citation.lang, chunker: .chunker_version}]}'
|
||||
# 기대: symbol = null, lang = "yaml", chunker_version = "code-text-paragraph-v1"
|
||||
|
||||
# 5) schema stats 에 shell 카운트 확인
|
||||
KB --json schema | jq '.stats.code_lang_breakdown'
|
||||
# 기대: {"shell": N, "yaml": M, ...}
|
||||
```
|
||||
|
||||
**Tier 3 citation.symbol 컨벤션**: 항상 `null`. 의미 단위 식별 안 함. `lang` 은 원본 lang 보존 (shell → `"shell"`, yaml → `"yaml"` 등).
|
||||
|
||||
## 검증 체크리스트
|
||||
|
||||
- `kebab doctor` 가 `--config` path 를 honor 하고 그 안의 `storage.data_dir` 를 출력 (XDG default 가 아님).
|
||||
@@ -456,6 +582,8 @@ rm -rf /tmp/kebab-smoke # 통째로 정리
|
||||
- (P10-1B) `.py` / `.ts` / `.tsx` / `.js` / `.mjs` / `.cjs` / `.jsx` 파일을 워크스페이스에 두면 `kebab ingest` 결과에 `new` 카운터에 포함. `--code-lang python` / `--code-lang typescript` / `--code-lang javascript` 검색이 `citation.symbol` 에 module path prefix 를 포함한 결과를 반환하면 wiring 정상. `kebab schema --json | jq .stats.code_lang_breakdown` 에 해당 언어 카운트 등장 확인.
|
||||
- (P10-1C-Go) `.go` 파일을 워크스페이스에 두면 `kebab ingest` 가 `code-go-ast-v1` 로 처리. `--code-lang go` 검색이 `citation.symbol` 에 `<package>.<Func>` / `<package>.(*Receiver).<Method>` 형식 결과를 반환하면 wiring 정상. `kebab schema --json | jq .stats.code_lang_breakdown` 에 `"go": N` 등장 확인.
|
||||
- (P10-1C-JK) `.java` 파일은 `code-java-ast-v1`, `.kt`/`.kts` 파일은 `code-kotlin-ast-v1` 로 처리. `--code-lang java` / `--code-lang kotlin` 검색이 `citation.symbol` 에 `com.foo.Foo.bar` 형식 결과를 반환하면 wiring 정상. `kebab schema --json | jq .stats.code_lang_breakdown` 에 `"java": N` / `"kotlin": N` 등장 확인.
|
||||
- (P10-2) `.yaml`/`.yml` 파일은 apiVersion+kind 파싱으로 k8s resource 별 chunk 생성 (`k8s-manifest-resource-v1`). `Dockerfile`/`Dockerfile.*` 는 전체 파일 단일 chunk (`dockerfile-file-v1`). `.toml`/`.json`/`.xml`/`.groovy`/`go.mod` 는 전체 파일 단일 chunk (`manifest-file-v1`). `--code-lang yaml` / `--code-lang dockerfile` / `--code-lang toml` 검색이 `citation.symbol` 에 각각 `Deployment/default/my-app` / `<dockerfile>` / `<manifest>` 형식 결과를 반환하면 wiring 정상. `kebab schema --json | jq .stats.code_lang_breakdown` 에 `"yaml": N` / `"dockerfile": N` / `"toml": N` 등장 확인.
|
||||
- (P10-3) `.sh`/`.bash`/`.zsh` 파일은 direct Tier 3 (`code-text-paragraph-v1`). 비-k8s YAML (apiVersion+kind 없는 yaml) 은 k8s chunker 가 0 chunk → Tier 3 fallback 으로 picked up. `--code-lang shell` / `--code-lang yaml` 검색이 `citation.symbol = null`, `chunker_version = "code-text-paragraph-v1"` 결과를 반환하면 wiring 정상. `kebab schema --json | jq .stats.code_lang_breakdown` 에 `"shell": N` 등장 확인.
|
||||
- (P7-3 + follow-up) 동일 path 에 byte 가 다른 PDF 를 두 번째 ingest 하면 `purge_vector_orphans_for_workspace_path` 가 옛 chunk_id 를 LanceDB 에서 먼저 삭제, 이어서 `purge_orphan_at_workspace_path` 가 옛 doc / chunks / embedding_records 를 SQLite 에서 sweep. 새 byte 가 새 `doc_id` 로 색인됨. `IngestReport` 에 그 자산만 `new+=1` (다른 자산은 `updated`). 두 store 모두 정합 — 옛 본문 검색 시 옛 chunks 가 더 이상 surface 되지 않음.
|
||||
|
||||
### Embedding upgrade (fb-39b)
|
||||
|
||||
1343
docs/superpowers/plans/2026-05-20-p10-2-tier2-resource-aware.md
Normal file
1343
docs/superpowers/plans/2026-05-20-p10-2-tier2-resource-aware.md
Normal file
File diff suppressed because it is too large
Load Diff
1296
docs/superpowers/plans/2026-05-21-p10-3-tier3-paragraph-fallback.md
Normal file
1296
docs/superpowers/plans/2026-05-21-p10-3-tier3-paragraph-fallback.md
Normal file
File diff suppressed because it is too large
Load Diff
@@ -1549,6 +1549,10 @@ transitional 형태) 의 source of truth.
|
||||
|
||||
**p10-1C-JavaKotlin 활성화 (Java + Kotlin) (2026-05-20)**: Java (`code-java-ast-v1`, `.java`) + Kotlin (`code-kotlin-ast-v1`, `.kt`/`.kts`) AST chunker 활성화. symbol = `com.foo.Foo.bar` 형식 (패키지 + 클래스 + 메서드/필드). Kotlin grammar 은 `tree-sitter-kotlin-ng` 사용 (bare `tree-sitter-kotlin` 은 tree-sitter 0.21–0.23 고착으로 사용 불가).
|
||||
|
||||
**p10-2 활성화 (Tier 2 chunker) (2026-05-20)**: Tier 2 resource-aware chunker 3종 활성화 — k8s-manifest-resource-v1 (`.yaml`/`.yml`), dockerfile-file-v1 (`Dockerfile`), manifest-file-v1 (`Cargo.toml` 등 설정 파일). 추가 code_lang 매핑: XML (`.xml`, `pom.xml`), Groovy (`build.gradle`, `.gradle`), Go module (`go.mod`).
|
||||
|
||||
**p10-3 활성화 (Tier 3 paragraph fallback) (2026-05-21)**: Tier 3 chunker `code-text-paragraph-v1` 활성화. shell script (`.sh`/`.bash`/`.zsh`) direct routing + Tier 1/2 가 0 chunk 또는 Err 시 자동 fallback 으로 retry. 비-k8s YAML / invalid YAML / AST 실패 케이스 모두 picked up. lang 은 입력 보존 (shell → "shell", yaml → "yaml" 등), symbol 은 항상 None.
|
||||
|
||||
### 10.2 MCP server transport (fb-30)
|
||||
|
||||
`kebab mcp` 가 stdio JSON-RPC server. Rust SDK = `rmcp 1.6`. Tool surface
|
||||
|
||||
@@ -237,6 +237,9 @@ pub struct Metadata {
|
||||
- Dockerfile (`Dockerfile`, `*.dockerfile`) → `dockerfile`
|
||||
- TOML (`.toml`) → `toml`
|
||||
- JSON (`.json`) → `json`
|
||||
- XML (`.xml`, `pom.xml`) → `xml`
|
||||
- Groovy (`build.gradle`, `.gradle`) → `groovy`
|
||||
- Go module (`go.mod`) → `go-mod`
|
||||
- Shell (`.sh`, `.bash`, `.zsh`) → `shell`
|
||||
- Make (`Makefile`, `*.mk`) → `make`
|
||||
- 미지원 / Tier 3 fallback → null
|
||||
|
||||
@@ -145,8 +145,8 @@ P0~P5 는 직렬. P6~P9 는 P5 이후 병렬 가능.
|
||||
- p10-1C-Go Go AST chunker — 🟡 PR 오픈 (v0.12.0, `code-go-ast-v1`)
|
||||
- p10-1C-JavaKotlin Java + Kotlin AST chunkers — 🟢 PR 오픈 (v0.13.0, `code-java-ast-v1` / `code-kotlin-ast-v1`)
|
||||
- p10-1D C + C++ AST chunkers — ⏳
|
||||
- p10-2 Tier 2 resource-aware — ⏳
|
||||
- p10-3 Tier 3 paragraph + line-window fallback — ⏳
|
||||
- p10-2 Tier 2 resource-aware — ✅ 머지 (v0.14.0, `k8s-manifest-resource-v1` / `dockerfile-file-v1` / `manifest-file-v1`)
|
||||
- p10-3 Tier 3 paragraph + line-window fallback — ✅ 머지 (v0.15.0, `code-text-paragraph-v1`)
|
||||
|
||||
## Post-merge 핫픽스
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
| 1C-Go | Go AST chunker (`code-go-ast-v1`) | 🟡 PR 오픈 (v0.12.0) |
|
||||
| 1C-JavaKotlin | Java + Kotlin AST chunkers (`code-java-ast-v1` / `code-kotlin-ast-v1`) | 🟢 PR 오픈 (v0.13.0) |
|
||||
| 1D | C + C++ AST chunkers | ⏳ |
|
||||
| 2 | Tier 2 resource-aware (k8s / Dockerfile / manifest) | ⏳ |
|
||||
| 3 | Tier 3 paragraph + line-window fallback | ⏳ |
|
||||
| 2 | Tier 2 resource-aware (k8s / Dockerfile / manifest) | ✅ 머지 (v0.14.0) |
|
||||
| 3 | Tier 3 paragraph + line-window fallback | ✅ 머지 (v0.15.0) |
|
||||
|
||||
Design: [2026-05-15-kebab-code-ingest-design.md](../../docs/superpowers/specs/2026-05-15-kebab-code-ingest-design.md)
|
||||
|
||||
120
tasks/p10/p10-2-tier2-resource-aware.md
Normal file
120
tasks/p10/p10-2-tier2-resource-aware.md
Normal file
@@ -0,0 +1,120 @@
|
||||
# p10-2 — Tier 2 resource-aware chunkers (k8s + Dockerfile + manifest)
|
||||
|
||||
**Status:** 🟡 진행 중
|
||||
**Contract sections:** §3.3 (chunker_version `k8s-manifest-resource-v1` + `dockerfile-file-v1` + `manifest-file-v1`), §3.4 (citation symbol — `<kind>/<namespace>/<name>` / `<dockerfile>` / `<manifest>`), §3.5 (code_lang 추가 매핑 `xml` / `groovy` / `go-mod`), §6.1 (`kebab-parse-code/src/lang.rs` 갱신 + `kebab-source-fs/src/media.rs` 의 inline duplication 정리), §6.2 (`kebab-chunk/src/{k8s_manifest_resource_v1,dockerfile_file_v1,manifest_file_v1}.rs`), §9.2 (Tier 2 정의), §10.1 (deactivation log 한 줄).
|
||||
**Design:** [2026-05-15-kebab-code-ingest-design.md](../../docs/superpowers/specs/2026-05-15-kebab-code-ingest-design.md) §1.2 (Phase 2) + §9.2.
|
||||
**Plan:** [2026-05-20-p10-2-tier2-resource-aware.md](../../docs/superpowers/plans/2026-05-20-p10-2-tier2-resource-aware.md).
|
||||
|
||||
## Goal
|
||||
|
||||
p10-1A-2 / 1B / 1C 인프라 위에 Tier 2 resource-aware chunker 3종을 단일 PR 로 활성화. AST 가 아닌 file/document-level chunking — 1B (Python+TS+JS) 의 묶음 패턴 따름. 머지 시점부터 `.yaml` / `.yml` / `Dockerfile` / 매니페스트 7종 dogfooding 가능.
|
||||
|
||||
비-k8s YAML (Helm values, CI yml, docker-compose 등) 및 invalid YAML 은 본 phase 에선 skip — p10-3 의 paragraph fallback 이 머지되면 자동으로 wire 됨.
|
||||
|
||||
## 동결된 설계 결정 (이 task 로 확정)
|
||||
|
||||
### 공통
|
||||
|
||||
- **3 chunker = self-contained**. `kebab-parse-code` 에 Tier 2 용 extractor 모듈 추가 없음. lang.rs 의 `code_lang_for_path` 갱신만. AST 가 아니라 추상화 비용이 코드 보상보다 큼.
|
||||
- **`code_lang_for_path` = single source of truth** (design §3.5). `kebab-source-fs/src/media.rs` 의 inline 확장자 match 는 이 함수 호출로 통일 (1A-1 부터 누적된 duplication 정리, 작은 리팩토링).
|
||||
- **parser_version** = `"none-v1"` 통일. Tier 2 는 parse 단계가 없음을 명시하는 sentinel. chunker_version cascade 만 의미 있음.
|
||||
- **oversize fallback** = AST chunker 와 동일 정책 (`AST_CHUNK_MAX_LINES = 200` 초과 시 line-window split). 거대 ConfigMap / multi-stage Dockerfile / aggregate POM 대비. split chunk 는 같은 symbol 공유 (line range 만 다름).
|
||||
- **frozen design 갱신** (본 PR 안에서):
|
||||
- §3.5 `code_lang` 매핑 표에 3 줄 추가:
|
||||
- XML (`.xml`, `pom.xml`) → `xml`
|
||||
- Groovy (`build.gradle`, `.gradle`) → `groovy`
|
||||
- Go module (`go.mod`) → `go-mod`
|
||||
- §10.1 deactivation log 한 줄 추가: "p10-2 활성화 — Tier 2 chunker 3종 active."
|
||||
|
||||
### k8s-manifest-resource-v1
|
||||
|
||||
- **Trigger**: `MediaType::Code("yaml")` (= `.yaml` / `.yml`).
|
||||
- **k8s 식별**: YAML document 의 top-level mapping 에 `apiVersion: <string>` + `kind: <string>` 둘 다 있어야 인정. 하나라도 없거나 string 타입이 아니면 그 document skip (전체 파일 skip 아님 — 다른 document 는 정상 처리).
|
||||
- **Multi-document split 구현**: `serde_yaml::Deserializer::from_str` 의 multi-document iterator 가 line offset 을 안 줘서, 원본 텍스트의 `^---\s*$` 줄 정규식 기준으로 pre-split 후 각 슬라이스를 deserialize. line_start/line_end 는 pre-split 단계에서 추적. trailing `---` 의 빈 슬라이스는 skip.
|
||||
- **Symbol**: `<kind>/<metadata.namespace>/<metadata.name>` (namespace 있으면) 또는 `<kind>/<metadata.name>` (cluster-scoped) 또는 `<kind>/<unnamed>` (name 누락). 예: `Deployment/prod/api-server`, `ClusterRole/cluster-admin`, `ConfigMap/<unnamed>`.
|
||||
- **Chunk text**: pre-split 슬라이스의 원본 텍스트 그대로 (deserialized form 아님 — 원본 보존).
|
||||
- **Citation**: `Citation::Code { path, line_start, line_end, symbol: Some(<위>), lang: Some("yaml") }`.
|
||||
- **Failure modes**:
|
||||
- Invalid YAML (어떤 document 라도 deserialize 실패) → 파일 전체 emit 0 chunk + warning log `invalid yaml: {path}`. p10-3 의 paragraph fallback 이 picked up.
|
||||
- 인정된 document 0개 (모두 비-k8s) → 파일 전체 emit 0 chunk. 동일 fallback.
|
||||
|
||||
### dockerfile-file-v1
|
||||
|
||||
- **Trigger**: `MediaType::Code("dockerfile")` — 파일명이 정확히 `Dockerfile`, 또는 prefix `Dockerfile.` (e.g. `Dockerfile.dev`), 또는 확장자 `.dockerfile` (e.g. `myapp.dockerfile`).
|
||||
- **Algorithm**: 파일 전체 텍스트 → 1 chunk emit.
|
||||
- **Symbol**: 통일 `<dockerfile>`.
|
||||
- **Citation**: `Citation::Code { path, line_start: 1, line_end: <EOF>, symbol: Some("<dockerfile>"), lang: Some("dockerfile") }`.
|
||||
|
||||
### manifest-file-v1
|
||||
|
||||
- **Trigger**: 파일명이 design §9.2 의 7종 중 하나:
|
||||
| basename | code_lang |
|
||||
|----------------|-----------|
|
||||
| `Cargo.toml` | `toml` |
|
||||
| `pyproject.toml` | `toml` |
|
||||
| `package.json` | `json` |
|
||||
| `tsconfig.json`| `json` |
|
||||
| `go.mod` | `go-mod` |
|
||||
| `pom.xml` | `xml` |
|
||||
| `build.gradle` | `groovy` |
|
||||
- **제외**: `build.gradle.kts` 는 1C-JK 의 Kotlin AST chunker (code-kotlin-ast-v1) 가 잡으므로 본 chunker 의 대상 아님.
|
||||
- **Algorithm**: 파일 전체 텍스트 → 1 chunk emit.
|
||||
- **Symbol**: 통일 `<manifest>` (7종 모두). manifest 종류 구분은 `code_lang` 으로 — 예: `--code-lang go-mod` 는 go.mod 만, `--code-lang toml` 은 Cargo.toml + pyproject.toml.
|
||||
- **Citation**: `Citation::Code { path, line_start: 1, line_end: <EOF>, symbol: Some("<manifest>"), lang: Some(<위 매핑>) }`.
|
||||
|
||||
### Routing (kebab-app::ingest_one_code_asset)
|
||||
|
||||
기존 7-arm AST match 옆에 Tier 2 분기 추가:
|
||||
|
||||
```text
|
||||
"rust" | "python" | "typescript" | "javascript"
|
||||
| "go" | "java" | "kotlin" → 기존 AST chunker (1A-2 / 1B / 1C)
|
||||
"yaml" → k8s_manifest_resource_v1
|
||||
"dockerfile" → dockerfile_file_v1
|
||||
"toml" | "json" | "xml"
|
||||
| "groovy" | "go-mod" → manifest_file_v1
|
||||
_ → skip (p10-3 fallback 의 자리)
|
||||
```
|
||||
|
||||
`code_lang_for_path` 의 lookup 순서: basename 우선 매칭 (`Cargo.toml` / `Dockerfile.*` / etc.) → 확장자 fallback (`.yaml` / `.toml` / etc.).
|
||||
|
||||
## Acceptance criteria
|
||||
|
||||
- `cargo test --workspace --no-fail-fast -j 1` passes (memory-conscious: per-crate 위주, full-suite gate 는 docs task 직전 1회).
|
||||
- `cargo clippy --workspace --all-targets -- -D warnings` passes.
|
||||
- 각 chunker 의 snapshot test 안정:
|
||||
- `crates/kebab-chunk/tests/fixtures/sample.yaml` — 2 k8s doc (Deployment + Service) + 1 비-k8s doc (apiVersion 빠짐) → 2 chunk emit, 비-k8s doc skip.
|
||||
- `crates/kebab-chunk/tests/fixtures/sample.dockerfile` → 1 chunk, symbol `<dockerfile>`.
|
||||
- `crates/kebab-chunk/tests/fixtures/sample.Cargo.toml` + `sample.package.json` + `sample.pom.xml` + `sample.go.mod` (4종) → 각 1 chunk, symbol `<manifest>`, 매핑된 code_lang.
|
||||
- `code_lang_for_path` 의 basename 우선 매칭 + 확장자 fallback unit test.
|
||||
- 격리 TempDir KB 에 yaml + Dockerfile + Cargo.toml 두고 `kebab search --code-lang yaml --json` / `--code-lang dockerfile --json` / `--code-lang toml --json` 각각 `Citation::Code` 반환 (기존 `code_ingest_smoke.rs` 에 3 테스트 추가, 총 12 테스트).
|
||||
- `kebab schema --json | jq .stats.code_lang_breakdown` 에 `yaml` / `dockerfile` / `toml` / `json` / `xml` / `groovy` / `go-mod` 카운트 (사용된 것만 등장).
|
||||
- README + HANDOFF + docs/ARCHITECTURE + docs/SMOKE + tasks/INDEX + tasks/p10/INDEX 갱신.
|
||||
- frozen design §3.5 매핑 3 줄 + §10.1 활성화 한 줄.
|
||||
- workspace `Cargo.toml` minor bump (0.13.0 → 0.14.0), gitea-release v0.14.0.
|
||||
|
||||
## Allowed dependencies
|
||||
|
||||
- `kebab-chunk` 에 새 모듈 3개 (`k8s_manifest_resource_v1.rs` / `dockerfile_file_v1.rs` / `manifest_file_v1.rs`) 및 dep entry `serde_yaml = { workspace = true }` (workspace 에 이미 존재). 기존 deps (kebab-core / serde_json_canonicalizer / blake3 / anyhow / tracing) 유지.
|
||||
- `kebab-parse-code` 의 `lang.rs` 갱신만. extractor 모듈 추가 없음, 새 crate dep 없음.
|
||||
- `kebab-source-fs/src/media.rs` — `code_lang_for_path` 호출로 inline match 정리. 기존 dep 유지 (kebab-parse-code 는 이미 의존).
|
||||
- `kebab-app::ingest_one_code_asset` — match 분기 확장. 새 crate dep 없음.
|
||||
|
||||
## Forbidden dependencies
|
||||
|
||||
- `kebab-chunk` 가 store / embed / llm / rag / tree-sitter 직접 import 금지 (boundary §6.3 유지).
|
||||
- `kebab-parse-code` 가 store / embed / llm / rag 직접 import 금지.
|
||||
- UI crate (`kebab-cli` / `kebab-mcp` / `kebab-tui` / `kebab-desktop`) 가 `kebab-parse-code` / `kebab-chunk` 직접 import 금지 — `kebab-app` facade 만.
|
||||
|
||||
## Risks / notes
|
||||
|
||||
- **serde_yaml line offset 없음** → 원본 텍스트의 `^---\s*$` 정규식 split 으로 line 추적. trailing `---` 의 빈 슬라이스 / 첫 슬라이스에 `---` prefix 없음 / 비-표준 separator (예: `--- # comment`) 모두 fixture 로 검증.
|
||||
- **apiVersion / kind 가 string 이 아닌 경우** (예: `kind: 42`) — `serde_yaml::Value::as_str()` 으로 string 체크 후 인정. 비-string 이면 비-k8s 취급.
|
||||
- **cluster-scoped resource** (Namespace, ClusterRole, ClusterRoleBinding, …) — metadata.namespace 없음이 정상. symbol = `<kind>/<name>` 형태.
|
||||
- **metadata.name 누락** — 비정상이지만 panic 금지. `<kind>/<unnamed>` fallback + warning log.
|
||||
- **거대 ConfigMap / Helm-rendered manifest** — `AST_CHUNK_MAX_LINES = 200` oversize fallback. split chunk 가 같은 symbol 공유 → search 시 dedupe 또는 user-visible 두 hit 으로 보임 (1A-2 의 oversize 와 동일 동작).
|
||||
- **YAML anchor / merge keys (`&`, `<<`, `*`)** — serde_yaml 가 자동 resolve. 원본 텍스트 보존 정책상 chunk text 는 원본 (resolve 전) 유지, 파싱은 resolve 후 값으로.
|
||||
- **`Dockerfile.example` 같은 doc-purpose 파일** — 확장자/접두사 매칭에 잡힘. user intent 와 어긋날 수 있으나 본 phase 의 scope 밖 (skip 정책은 1A-1 의 size/built-in/generated 정책으로 통제). dogfood 후 false positive 빈도 보고 HOTFIXES 결정.
|
||||
- **`pom.xml` aggregate parent POM** — 매우 큼 (수백~수천 줄). oversize fallback 으로 split. 거대 fixture 로 한 번 검증.
|
||||
- **`media.rs` 정리** — 1A-1 부터 누적된 inline `match extension` duplication 을 `code_lang_for_path` 호출로 교체. 기존 단위 테스트 동작 보존 (테스트는 결과 값만 보므로 통과해야 함).
|
||||
- **머지 후 deviation** 은 `tasks/HOTFIXES.md` dated 로그 + 본 spec `Risks / notes` 에 one-line cross-link.
|
||||
116
tasks/p10/p10-3-tier3-paragraph-fallback.md
Normal file
116
tasks/p10/p10-3-tier3-paragraph-fallback.md
Normal file
@@ -0,0 +1,116 @@
|
||||
# p10-3 — Tier 3 paragraph + line-window fallback chunker
|
||||
|
||||
**Status:** 🟡 진행 중
|
||||
**Contract sections:** §3.3 (chunker_version `code-text-paragraph-v1`), §3.5 (code_lang routing — `shell` 활성화 + "미지원 / Tier 3 fallback" 명확화), §6.2 (`kebab-chunk/src/code_text_paragraph_v1.rs`), §6.3 (`tier2_shared::build_chunk` 의 `pub(crate)` 노출), §9.3 (Tier 3 정의), §10.1 (deactivation log 한 줄).
|
||||
**Design:** [2026-05-15-kebab-code-ingest-design.md](../../docs/superpowers/specs/2026-05-15-kebab-code-ingest-design.md) §1.3 (Phase 3) + §9.3.
|
||||
**Plan:** [2026-05-20-p10-3-tier3-paragraph-fallback.md](../../docs/superpowers/plans/2026-05-20-p10-3-tier3-paragraph-fallback.md).
|
||||
|
||||
## Goal
|
||||
|
||||
p10-1A-2 / 1B / 1C / 1A-1 의 framework + p10-2 Tier 2 인프라 위에 Tier 3 paragraph fallback chunker 활성화. 단일 PR. 머지 시점부터:
|
||||
|
||||
- `.sh` / `.bash` / `.zsh` 파일이 paragraph 단위로 색인.
|
||||
- p10-2 의 비-k8s YAML / invalid YAML / Tier 1 AST extractor 실패 등 0-chunk 결과가 자동으로 Tier 3 로 fallback 되어 색인 — 이전에 skip 되던 파일이 search 가능.
|
||||
|
||||
## 동결된 설계 결정 (이 task 로 확정)
|
||||
|
||||
### chunker (`code-text-paragraph-v1`)
|
||||
|
||||
- **Input**: `Document` with single `Block::Code { text, lang, ... }`. Tier 2 의 `synthesize_tier2_document` 와 동일한 모양 — fallback wrapper 가 같은 doc 재사용.
|
||||
- **VERSION_LABEL**: `"code-text-paragraph-v1"`.
|
||||
- **Paragraph 분할**: `text.lines()` 순회. 빈 줄 (정확히 빈 줄 또는 only-whitespace) 을 paragraph boundary 로. 빈 줄 자체는 어느 paragraph 에도 포함되지 않음 (chunk 의 line range 에 미포함). 빈 paragraph (전부 whitespace) skip.
|
||||
- **Paragraph 크기 룰** (design §9.3 default 그대로, hardcoded):
|
||||
- paragraph line count ≤ 80 → 1 chunk emit.
|
||||
- paragraph line count > 80 → line-window split with window size 80 / overlap 20 (stride 60). 즉 line 1-80, 61-140, 121-200, … 마지막 window 는 EOF 까지 (≤ 80 lines).
|
||||
- `FALLBACK_LINES_PER_CHUNK = 80`, `FALLBACK_LINES_OVERLAP = 20` 둘 다 hardcoded constants (1A-2 의 `AST_CHUNK_MAX_LINES = 200` 패턴 그대로 — 사용자 config 노출 안 함, 미래 HOTFIXES 시 노출 검토).
|
||||
- **Citation**: `SourceSpan::Code { line_start, line_end, symbol: None, lang: <input lang> }`. `symbol = None` 통일 (Tier 3 는 의미 단위 식별 안 함). `lang` 은 입력 Document 의 `Block::Code.lang` 그대로 보존 — shell → `"shell"`, k8s skip → `"yaml"`, Rust extractor 실패 → `"rust"` 등.
|
||||
- **chunk_id 충돌 방지**: 동일 paragraph 의 line-window split 시 `id_for_chunk` 의 `split_key` 에 `window_start` 전달 (Tier 2 `#L{k}` 패턴 동일).
|
||||
- **Edge cases**:
|
||||
- 전체 파일이 빈 줄만 → 0 chunk emit (fallback 의 fallback 없음). `tracing::warn!`.
|
||||
- 단일 paragraph + ≤ 80 lines → 1 chunk, line range 1..N.
|
||||
- 빈 줄 없는 거대 파일 (한 paragraph 전체) → line-window split.
|
||||
|
||||
### Routing / fallback wrapper
|
||||
|
||||
- **`code_lang_for_path`** 변경 없음 (shell 매핑은 1A-1 시점부터 이미 존재).
|
||||
- **`ingest_one_code_asset` allowlist** (`crates/kebab-app/src/lib.rs:953`) 에 `"shell"` 추가.
|
||||
- **4-arm match (parser_version / chunker_version / extract / chunks)** 에 `"shell"` arm 추가:
|
||||
- parser_version = `"none-v1"` (Tier 2 sentinel 재사용).
|
||||
- chunker_version = `CodeTextParagraphV1Chunker.chunker_version()`.
|
||||
- extract = `synthesize_tier2_document(asset, &bytes, "shell", &parser_version)?` (재사용).
|
||||
- chunks = `CodeTextParagraphV1Chunker.chunk(&canonical, chunk_policy)?`.
|
||||
- **Fallback wrapper** (핵심 신규 로직) — chunks match 직후 후처리:
|
||||
- Tier 1/2 lang 의 결과가 `Err(_)` 또는 `Ok(empty_vec)` 이면 Tier 3 retry.
|
||||
- retry 시:
|
||||
- `chunker_version` 를 `code-text-paragraph-v1` 로 swap (downstream stamping 정확성).
|
||||
- `canonical.parser_version` 도 `"none-v1"` 로 swap (Tier 1 의 `RUST_PARSER_VERSION` 등이 misleading 하므로).
|
||||
- `CodeTextParagraphV1Chunker.chunk(&canonical, chunk_policy)` 실행.
|
||||
- 실패 사유는 `tracing::warn!("tier1/2 emitted 0 chunks or errored for {workspace_path} ({code_lang}); falling back to tier 3")`.
|
||||
- **Tier 3 자체가 0 chunk 또는 Err** 인 경우는 그대로 fail/skip (fallback 의 fallback 없음).
|
||||
|
||||
### `tier2_shared::build_chunk` 노출
|
||||
|
||||
- 현재 module-private `fn build_chunk`. Tier 3 가 동일 Chunk 생성 (hash / token / policy_hash 일관) 을 위해 호출 — `pub(crate) fn build_chunk(...)` 으로 visibility 만 변경. signature 동일.
|
||||
|
||||
### Lang 보존 정책
|
||||
|
||||
- Tier 3 chunk 의 `Citation::Code.lang` = 입력 Document 의 `Block::Code.lang` 그대로. 명시적으로 표:
|
||||
| Source | input lang | Tier 3 output lang |
|
||||
|--------|-----------|----------|
|
||||
| shell direct | `"shell"` | `"shell"` |
|
||||
| k8s 0-chunk fallback | `"yaml"` | `"yaml"` |
|
||||
| Rust AST 실패 fallback | `"rust"` | `"rust"` |
|
||||
| manifest 0-chunk (이론상, 거의 발생 안 함) | `"toml"` 등 | 유지 |
|
||||
- 검색 시 `--code-lang shell` / `--code-lang yaml` 등이 fallback chunk 도 매칭 — search filter 동작 자연.
|
||||
|
||||
### Non-scope
|
||||
|
||||
- **미지원 확장자 wiring**: `.txt` / `.log` / `.scala` / `.rb` 등은 본 PR scope 밖. `code_lang_for_path` 의 매핑은 unchanged. Tier 3 chunker 자체는 만들어두고, 미래에 `code_lang_for_path` 에 새 lang 추가 시 자동 picked up (1A-2 패턴).
|
||||
- **config 노출**: `FALLBACK_LINES_PER_CHUNK` / `FALLBACK_LINES_OVERLAP` hardcoded. config.toml 노출 없음.
|
||||
|
||||
### Frozen design 갱신
|
||||
|
||||
- `docs/superpowers/specs/2026-05-15-kebab-code-ingest-design.md` §10.1 활성화 로그 한 줄.
|
||||
- `docs/superpowers/specs/2026-04-27-kebab-final-form-design.md` §10 activation log 한 줄.
|
||||
- §3.5 의 "미지원 / Tier 3 fallback → null" 표현은 그대로 유지 (해당 표현이 본 phase 의 정확한 의미 — Tier 3 chunk 의 lang 은 입력 lang 보존이므로 "null" 은 미지원 확장자 wire 시 적용).
|
||||
|
||||
## Acceptance criteria
|
||||
|
||||
- `cargo test --workspace --no-fail-fast -j 1` PASS (memory-conscious `-j 1`).
|
||||
- `cargo clippy --workspace --all-targets -- -D warnings` clean.
|
||||
- 4 신규 unit test in `crates/kebab-chunk/tests/code_text_paragraph_v1.rs`:
|
||||
- `shell_multi_paragraph_splits_on_blank_lines` — 3-paragraph fixture → 3 chunk, symbol=None, lang=shell, contiguous (exclusive of blank lines).
|
||||
- `single_long_paragraph_line_window_split` — 200+ line single paragraph → window split, distinct chunk_ids, expected line ranges (1-80, 61-140, 121-200, …).
|
||||
- `empty_file_emits_zero_chunks` — 빈 텍스트 → `Ok(vec![])`.
|
||||
- `lang_field_preserved_from_input_doc` — lang=yaml 입력 → emit chunk lang=yaml.
|
||||
- 2 신규 integration test in `crates/kebab-app/tests/code_ingest_smoke.rs`:
|
||||
- `tier3_shell_ingest_searchable` — `.sh` 파일 ingest → `--code-lang shell` 검색 → `Citation::Code { symbol: None, lang: "shell" }`, `chunker_version: "code-text-paragraph-v1"`.
|
||||
- `tier3_yaml_fallback_picks_up_non_k8s_yaml` — apiVersion+kind 없는 yaml ingest → fallback 발동 → `Citation::Code { symbol: None, lang: "yaml" }`, chunker_version `code-text-paragraph-v1`.
|
||||
- 기존 12 smoke test + 2 신규 = 14 testing surface. (Tier 1 9 + Tier 2 3 + Tier 3 2.)
|
||||
- `kebab schema --json | jq .stats.code_lang_breakdown` 에 `"shell"` 카운트 등장 (.sh 파일 ingest 후). 비-k8s YAML 도 `"yaml"` 카운트에 누적 (Tier 2 와 Tier 3 가 같은 lang).
|
||||
- README + HANDOFF + docs/ARCHITECTURE + docs/SMOKE + tasks/INDEX + tasks/p10/INDEX 갱신.
|
||||
- frozen design §10.1 + §10 activation log 한 줄씩.
|
||||
- workspace `Cargo.toml` minor bump (0.14.0 → 0.15.0), gitea-release v0.15.0.
|
||||
|
||||
## Allowed dependencies
|
||||
|
||||
- `kebab-chunk` 의 새 모듈 `code_text_paragraph_v1.rs` — kebab-core + anyhow + tracing. tier2_shared 의 `build_chunk` 호출 (visibility `pub(crate)` 로 노출). tree-sitter / serde_yaml 비사용.
|
||||
- `kebab-app::ingest_one_code_asset` — 4-arm match + allowlist + fallback wrapper 확장. 새 crate dep 없음.
|
||||
- `kebab-parse-code` — 변경 없음 (lang.rs 의 shell 매핑은 1A-1 부터 존재).
|
||||
- `kebab-source-fs` — 변경 없음 (media.rs 이미 `code_lang_for_path` 위임).
|
||||
|
||||
## Forbidden dependencies
|
||||
|
||||
- `kebab-chunk` 가 store / embed / llm / rag / tree-sitter 직접 import 금지 (boundary §6.3 유지).
|
||||
- UI crate (`kebab-cli` / `kebab-mcp` / `kebab-tui` / `kebab-desktop`) 가 `kebab-parse-code` / `kebab-chunk` 직접 import 금지 — `kebab-app` facade 만.
|
||||
|
||||
## Risks / notes
|
||||
|
||||
- **Fallback infinite loop 방지**: Tier 3 자체가 0 chunk 또는 Err 인 경우는 그대로 fail/skip — fallback 의 fallback 없음. 명시 spec.
|
||||
- **chunker_version swap 시 `try_skip_unchanged` 일관성**: fallback 발동 후 stored chunker_version = `code-text-paragraph-v1`. 다음 ingest 에 동일 파일 → 동일 chunker_version 으로 lookup 매칭 (skip 동작 OK). Tier 1 chunker 가 미래에 작동하기 시작하면 (예: tree-sitter grammar fix) cascade rule 로 incremental cache miss → 자동 reprocess 가 정상 동작.
|
||||
- **lang 보존 vs fallback 의미**: fallback chunk 의 lang 이 원본 lang 유지라 search filter `--code-lang yaml` 가 Tier 2 와 Tier 3 chunk 둘 다 매칭. 의도된 동작 — 사용자가 "yaml 파일 검색" 했을 때 모든 yaml 결과 표시.
|
||||
- **line-window overlap 의미**: 80/20 (stride 60) 은 design §9.3 default. 거대 paragraph (예: minified JSON 한 줄) 의 경우에도 동일 알고리즘 — 단 한 줄 = 한 line 이라 split 발생 안 함 (length 80 lines 기준). minified 의 경우 chunk 한 개에 매우 긴 텍스트가 들어가는데 이는 paragraph 분할 정책의 inherent limitation. 미래 HOTFIXES 검토.
|
||||
- **빈 줄 처리**: `^\s*$` 매칭 (whitespace-only) 줄을 paragraph boundary 로. 탭만 있는 줄 / CR-only 줄 등 edge case fixture 로 검증.
|
||||
- **shell line-comment 처리**: shell script 의 `# comment` 줄은 일반 line. paragraph 분할에 영향 없음 (빈 줄 아님). chunk 안에 그대로 보존.
|
||||
- **fallback wrapper 의 `canonical.parser_version` mutation**: Document 의 parser_version 을 Tier 3 fallback 시 `"none-v1"` 로 swap. CanonicalDocument 가 `mut` 로 받아져야 함. 이미 `let mut canonical = match ...` 이라 mut 가능. plan 단계 검증.
|
||||
- **머지 후 deviation** 은 `tasks/HOTFIXES.md` dated 로그 + 본 spec `Risks / notes` cross-link.
|
||||
Reference in New Issue
Block a user