diff --git a/Cargo.lock b/Cargo.lock index 4f98525..311a7be 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4127,7 +4127,7 @@ dependencies = [ [[package]] name = "kebab-app" -version = "0.12.0" +version = "0.13.0" dependencies = [ "anyhow", "base64 0.22.1", @@ -4172,7 +4172,7 @@ dependencies = [ [[package]] name = "kebab-chunk" -version = "0.12.0" +version = "0.13.0" dependencies = [ "anyhow", "blake3", @@ -4187,7 +4187,7 @@ dependencies = [ [[package]] name = "kebab-cli" -version = "0.12.0" +version = "0.13.0" dependencies = [ "anyhow", "clap", @@ -4208,7 +4208,7 @@ dependencies = [ [[package]] name = "kebab-config" -version = "0.12.0" +version = "0.13.0" dependencies = [ "anyhow", "dirs 5.0.1", @@ -4223,7 +4223,7 @@ dependencies = [ [[package]] name = "kebab-core" -version = "0.12.0" +version = "0.13.0" dependencies = [ "anyhow", "blake3", @@ -4237,7 +4237,7 @@ dependencies = [ [[package]] name = "kebab-embed" -version = "0.12.0" +version = "0.13.0" dependencies = [ "anyhow", "blake3", @@ -4251,7 +4251,7 @@ dependencies = [ [[package]] name = "kebab-embed-local" -version = "0.12.0" +version = "0.13.0" dependencies = [ "anyhow", "fastembed", @@ -4264,7 +4264,7 @@ dependencies = [ [[package]] name = "kebab-eval" -version = "0.12.0" +version = "0.13.0" dependencies = [ "anyhow", "kebab-app", @@ -4283,7 +4283,7 @@ dependencies = [ [[package]] name = "kebab-llm" -version = "0.12.0" +version = "0.13.0" dependencies = [ "anyhow", "kebab-core", @@ -4292,7 +4292,7 @@ dependencies = [ [[package]] name = "kebab-llm-local" -version = "0.12.0" +version = "0.13.0" dependencies = [ "anyhow", "kebab-config", @@ -4309,7 +4309,7 @@ dependencies = [ [[package]] name = "kebab-mcp" -version = "0.12.0" +version = "0.13.0" dependencies = [ "anyhow", "kebab-app", @@ -4327,7 +4327,7 @@ dependencies = [ [[package]] name = "kebab-normalize" -version = "0.12.0" +version = "0.13.0" dependencies = [ "anyhow", "kebab-core", @@ -4342,7 +4342,7 @@ dependencies = [ [[package]] name = "kebab-parse-code" -version = "0.12.0" +version = "0.13.0" dependencies = [ "anyhow", "gix", @@ -4353,7 +4353,9 @@ dependencies = [ "tracing", "tree-sitter", "tree-sitter-go", + "tree-sitter-java", "tree-sitter-javascript", + "tree-sitter-kotlin-ng", "tree-sitter-python", "tree-sitter-rust", "tree-sitter-typescript", @@ -4361,7 +4363,7 @@ dependencies = [ [[package]] name = "kebab-parse-image" -version = "0.12.0" +version = "0.13.0" dependencies = [ "ab_glyph", "anyhow", @@ -4385,7 +4387,7 @@ dependencies = [ [[package]] name = "kebab-parse-md" -version = "0.12.0" +version = "0.13.0" dependencies = [ "anyhow", "kebab-core", @@ -4402,7 +4404,7 @@ dependencies = [ [[package]] name = "kebab-parse-pdf" -version = "0.12.0" +version = "0.13.0" dependencies = [ "anyhow", "blake3", @@ -4415,7 +4417,7 @@ dependencies = [ [[package]] name = "kebab-parse-types" -version = "0.12.0" +version = "0.13.0" dependencies = [ "kebab-core", "serde", @@ -4423,7 +4425,7 @@ dependencies = [ [[package]] name = "kebab-rag" -version = "0.12.0" +version = "0.13.0" dependencies = [ "anyhow", "blake3", @@ -4444,7 +4446,7 @@ dependencies = [ [[package]] name = "kebab-search" -version = "0.12.0" +version = "0.13.0" dependencies = [ "anyhow", "globset", @@ -4463,7 +4465,7 @@ dependencies = [ [[package]] name = "kebab-source-fs" -version = "0.12.0" +version = "0.13.0" dependencies = [ "anyhow", "blake3", @@ -4482,7 +4484,7 @@ dependencies = [ [[package]] name = "kebab-store-sqlite" -version = "0.12.0" +version = "0.13.0" dependencies = [ "anyhow", "blake3", @@ -4503,7 +4505,7 @@ dependencies = [ [[package]] name = "kebab-store-vector" -version = "0.12.0" +version = "0.13.0" dependencies = [ "anyhow", "arrow", @@ -4527,7 +4529,7 @@ dependencies = [ [[package]] name = "kebab-tui" -version = "0.12.0" +version = "0.13.0" dependencies = [ "anyhow", "crossterm", @@ -8538,6 +8540,16 @@ dependencies = [ "tree-sitter-language", ] +[[package]] +name = "tree-sitter-java" +version = "0.23.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aa6cbcdc8c679b214e616fd3300da67da0e492e066df01bcf5a5921a71e90d6" +dependencies = [ + "cc", + "tree-sitter-language", +] + [[package]] name = "tree-sitter-javascript" version = "0.25.0" @@ -8548,6 +8560,16 @@ dependencies = [ "tree-sitter-language", ] +[[package]] +name = "tree-sitter-kotlin-ng" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e800ebbda938acfbf224f4d2c34947a31994b1295ee6e819b65226c7b51b4450" +dependencies = [ + "cc", + "tree-sitter-language", +] + [[package]] name = "tree-sitter-language" version = "0.1.7" diff --git a/Cargo.toml b/Cargo.toml index 2f1aae2..6d3c9f9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,7 +31,7 @@ edition = "2024" rust-version = "1.85" license = "MIT OR Apache-2.0" repository = "https://github.com/altair823/kebab" -version = "0.12.0" +version = "0.13.0" [workspace.dependencies] anyhow = "1" @@ -96,6 +96,9 @@ tree-sitter-typescript = "0.23.2" tree-sitter-javascript = "0.25.0" # Go grammar for code ingest (kebab-parse-code, p10-1C-Go). tree-sitter-go = "0.25.0" +# JVM family grammars for code ingest (kebab-parse-code, p10-1C-JK). +tree-sitter-java = "0.23.5" +tree-sitter-kotlin-ng = "1.1.0" # bare tree-sitter-kotlin requires ts <0.23; -ng uses tree-sitter-language 0.1 (ts 0.26 compat) # Disk-footprint trim for dev / test builds. Codegen, opt-level, and # behavior are unchanged — only DWARF debug info is reduced (line diff --git a/HANDOFF.md b/HANDOFF.md index c01bfe9..3554fac 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -4,7 +4,7 @@ ## 한 줄 요약 -P0–P5 + P6 + P7 + P9-1/2/3/4 (Library / Search / Ask / Inspect) 머지 완료. `kebab ingest` 가 markdown / image / PDF / 소스코드 (Rust / Python / TS / JS / Go) 처리. `kebab search` / `kebab ask` 가 매체 가로질러 결과 + page / code citation 반환. `kebab tui` 가 4 패널 (Library + Search + Ask + Inspect) 제공. 다음 후보 = P10-1C-JavaKotlin 또는 P9-5 (desktop tauri) 또는 보류 중인 P8 (audio). +P0–P5 + P6 + P7 + P9-1/2/3/4 (Library / Search / Ask / Inspect) 머지 완료. `kebab ingest` 가 markdown / image / PDF / 소스코드 (Rust / Python / TS / JS / Go / Java / Kotlin) 처리. `kebab search` / `kebab ask` 가 매체 가로질러 결과 + page / code citation 반환. `kebab tui` 가 4 패널 (Library + Search + Ask + Inspect) 제공. P10-1C (Go + Java + Kotlin) 완료 — 다음 후보 = P10-1D (C/C++) 또는 P9-5 (desktop tauri) 또는 보류 중인 P8 (audio). ## Phase 로드맵 @@ -20,7 +20,7 @@ P0–P5 + P6 + P7 + P9-1/2/3/4 (Library / Search / Ask / Inspect) 머지 완료. | **P7** | PDF text + page citation | `kebab-parse-pdf` | P5 | ✅ 완료 (3/3 component, page-level chunker + ingest wiring) | | **P8** | 음성 transcription + timestamp citation | `kebab-parse-audio` | P5 | ⏸ 보류 (whisper-rs 시스템 dep brainstorm 필요) | | **P9** | TUI + desktop app | `kebab-tui`, `kebab-desktop` | P5 | 🟡 진행 (4/5 component — P9-1/2/3/4 완료 [Library / Search / Ask / Inspect], P9-5 desktop 예정 · 도그푸딩 피드백 **20/20 ✅**) | -| **P10** | code ingest framework | `kebab-parse-code` | P5 | 🟡 진행 중 — 1A-1 ✅ (wire schema + parse-code skeleton + filter flags), 1A-2 ✅ (Rust AST chunker, `code-rust-ast-v1` — v0.7.0), 1B ✅ (Python/TS/JS AST chunkers — v0.8.0 이후), **1C-Go ✅ (Go AST chunker, `code-go-ast-v1` — v0.12.0)**, 1C-JavaKotlin ⏳ (후속 PR) | +| **P10** | code ingest framework | `kebab-parse-code` | P5 | 🟡 진행 중 — 1A-1 ✅ (wire schema + parse-code skeleton + filter flags), 1A-2 ✅ (Rust AST chunker, `code-rust-ast-v1` — v0.7.0), 1B ✅ (Python/TS/JS AST chunkers — v0.8.0 이후), **1C-Go ✅ (Go AST chunker, `code-go-ast-v1` — v0.12.0)**, **1C-JavaKotlin ✅ (Java + Kotlin AST chunkers, `code-java-ast-v1` / `code-kotlin-ast-v1` — v0.13.0)** | P0~P5 직렬. P6~P9 P5 이후 병렬 가능. diff --git a/README.md b/README.md index 9a025b6..fddb742 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ kebab doctor | 명령 | 동작 | |------|------| | `kebab init` | XDG 경로에 데이터 디렉토리 + config.toml 생성 | -| `kebab ingest []` | Markdown / 이미지 / PDF / Rust 소스코드 색인 (idempotent). TTY 에서는 stderr 진행 바, non-TTY (CI / pipe) 는 stderr 한 줄씩, `--json` 은 stdout 에 `ingest_progress.v1` 라인 streaming 후 마지막에 `ingest_report.v1`. Ctrl-C 한 번이면 현재 asset 마무리 후 abort (부분 commit 보존, idempotent re-run), 두 번째 Ctrl-C 는 hard exit. Markdown title 이 frontmatter 에 없어도 첫 H1 → H2 → 첫 paragraph 80 자 → 파일명 순으로 자동 채움 (parser_version `md-frontmatter-v2`) — 기존 색인된 doc 도 다음 ingest 에서 새 title 로 갱신. **Incremental** (p9-fb-23): 두 번째 이후의 ingest 는 변하지 않은 doc (blake3 + parser/chunker/embedder version 모두 동일) 의 parse/chunk/embed/vector upsert 를 자동 스킵. final summary 에 `N unchanged` 카운트 표시. `--force-reingest` 로 skip 무시 강제 재처리. **지원 형식** (extractor 자동 결정 — config 에 명시 불가): Markdown (`.md`), 이미지 (`.png` / `.jpg` / `.jpeg`, OCR + caption), PDF (`.pdf`), **소스코드** (`.rs` → `code-rust-ast-v1`, `.py` → `code-python-ast-v1`, `.ts`/`.tsx` → `code-ts-ast-v1`, `.js`/`.mjs`/`.cjs`/`.jsx` → `code-js-ast-v1`, `.go` → `code-go-ast-v1` — 모두 tree-sitter AST chunker). 다른 확장자는 자동 skip — `IngestItem.warnings` 에 사유 (`"unsupported media type: .docx"` 등), `IngestReport.skipped_by_extension` 에 카운트 분류, CLI / TUI summary 에 breakdown 표시. 코드 chunk 는 `citation.kind = "code"` 에 `citation.lang = ""` + `symbol` + line range 를 담고, SearchHit top-level 에 `code_lang` + `repo` (`.git/` walk-up 의 디렉토리 이름) 가 backfill 됨. `--code-lang rust` / `--code-lang python` / `--code-lang typescript` / `--code-lang javascript` / `--code-lang go` / `--media code` filter 로 언어별·코드 전용 검색 가능 (p10-1A-1 filter flags). Python symbol 은 workspace 경로 → dotted module path prefix (예: `kebab_eval.metrics.compute_mrr`), TS/JS symbol 은 slash-style module path prefix (예: `src/Foo.Foo.search`), Go symbol 은 `package.Func` / `package.(*Receiver).Method` 형식. | +| `kebab ingest []` | Markdown / 이미지 / PDF / Rust 소스코드 색인 (idempotent). TTY 에서는 stderr 진행 바, non-TTY (CI / pipe) 는 stderr 한 줄씩, `--json` 은 stdout 에 `ingest_progress.v1` 라인 streaming 후 마지막에 `ingest_report.v1`. Ctrl-C 한 번이면 현재 asset 마무리 후 abort (부분 commit 보존, idempotent re-run), 두 번째 Ctrl-C 는 hard exit. Markdown title 이 frontmatter 에 없어도 첫 H1 → H2 → 첫 paragraph 80 자 → 파일명 순으로 자동 채움 (parser_version `md-frontmatter-v2`) — 기존 색인된 doc 도 다음 ingest 에서 새 title 로 갱신. **Incremental** (p9-fb-23): 두 번째 이후의 ingest 는 변하지 않은 doc (blake3 + parser/chunker/embedder version 모두 동일) 의 parse/chunk/embed/vector upsert 를 자동 스킵. final summary 에 `N unchanged` 카운트 표시. `--force-reingest` 로 skip 무시 강제 재처리. **지원 형식** (extractor 자동 결정 — config 에 명시 불가): Markdown (`.md`), 이미지 (`.png` / `.jpg` / `.jpeg`, OCR + caption), PDF (`.pdf`), **소스코드** (`.rs` → `code-rust-ast-v1`, `.py` → `code-python-ast-v1`, `.ts`/`.tsx` → `code-ts-ast-v1`, `.js`/`.mjs`/`.cjs`/`.jsx` → `code-js-ast-v1`, `.go` → `code-go-ast-v1`, `.java` → `code-java-ast-v1`, `.kt`/`.kts` → `code-kotlin-ast-v1` — 모두 tree-sitter AST chunker). 다른 확장자는 자동 skip — `IngestItem.warnings` 에 사유 (`"unsupported media type: .docx"` 등), `IngestReport.skipped_by_extension` 에 카운트 분류, CLI / TUI summary 에 breakdown 표시. 코드 chunk 는 `citation.kind = "code"` 에 `citation.lang = ""` + `symbol` + line range 를 담고, SearchHit top-level 에 `code_lang` + `repo` (`.git/` walk-up 의 디렉토리 이름) 가 backfill 됨. `--code-lang rust` / `--code-lang python` / `--code-lang typescript` / `--code-lang javascript` / `--code-lang go` / `--code-lang java` / `--code-lang kotlin` / `--media code` filter 로 언어별·코드 전용 검색 가능 (p10-1A-1 filter flags). Python symbol 은 workspace 경로 → dotted module path prefix (예: `kebab_eval.metrics.compute_mrr`), TS/JS symbol 은 slash-style module path prefix (예: `src/Foo.Foo.search`), Go symbol 은 `package.Func` / `package.(*Receiver).Method` 형식, Java / Kotlin symbol 은 `com.foo.Foo.bar` 형식 (패키지 + 클래스 + 메서드/필드). | | `kebab search --mode {lexical,vector,hybrid} "" [--no-cache] [--max-tokens N] [--snippet-chars N] [--cursor ] [--tag T] [--lang L] [--path-glob G] [--trust-min LEVEL] [--media TYPE] [--ingested-after RFC3339] [--doc-id ID] [--trace] [--bulk] [--repo NAME ...] [--code-lang LIST]` | 검색. hybrid는 RRF fusion, citation 포함. 같은 process 안에서 동일 query (NFKC + trim + lowercase 정규화) 반복 시 in-process LRU 캐시 hit (capacity = `[search] cache_capacity`, default 256). `--no-cache` 로 강제 bypass — 디버깅용. ingest commit 발생 시 `kv['corpus_revision']` bump 으로 모든 entry 자동 stale. **`--max-tokens` / `--snippet-chars` / `--cursor` (p9-fb-34)** — agent budget controls. `--json` 출력은 `search_response.v1` wrapper (`{hits, next_cursor, truncated}`) — pre-fb-34 의 bare array 와 호환 안 됨. mismatched cursor → `error.v1.code = stale_cursor`. **filter flags (p9-fb-36):** `--tag` 는 반복 가능 flag (`--tag rust --tag async`) 로 OR 매칭, `--media` 는 `,` 구분 다중 값 OR 매칭, 나머지 flags 간은 AND 조합. `--trust-min` 은 `primary\|secondary\|generated` 중 하나 (해당 level 이상 포함). `--ingested-after` 는 RFC3339 UTC — 파싱 실패 시 `error.v1.code = config_invalid` (exit 2). `--media md` 는 `markdown` alias 로 정규화. 알 수 없는 `--media` 값은 무조건 empty hits (오류 아님). **`--trace` (p9-fb-37)** — `search_response.v1.trace` 에 lexical / vector pre-fusion 후보 + RRF union + per-stage timing (`lexical_ms` / `vector_ms` / `fusion_ms` / `total_ms`) 노출. trace 요청은 캐시 우회 (`--no-cache` 없이도 항상 cold). **`--bulk` (p9-fb-42)** — stdin ndjson 으로 N query 한 번에 실행. `--json` 면 stdout per-query ndjson (`bulk_search_item.v1`) + stderr summary (`bulk_summary: total=N succeeded=S failed=F`). Cap 100. agent 가 query decomposition 후 sub-query 일괄 실행 시 single round-trip — App instance 재사용으로 캐시 / embedder cold-start 비용 한 번만. Per-query failure 는 item 의 `error` (error.v1) 에 격리, 다른 query 계속 진행. **code corpus filters (p10-1A-1):** `--repo` 는 반복 가능 (`--repo kebab --repo other`) OR 매칭. `--code-lang` 는 반복 또는 comma 다중 값 (`--code-lang rust,python`), 알 수 없는 값은 빈 hits. `--media code` 는 Tier 1/2/3 모든 code chunk 포함. 1A-1 시점에서는 indexed 된 code chunk 가 없어 filter 가 항상 빈 결과 — 1A-2 (Rust AST chunker) 머지 이후 실효. | | `kebab list docs` | 색인된 문서 목록 | | `kebab inspect doc ` / `kebab inspect chunk ` | raw record 보기 | diff --git a/crates/kebab-app/src/lib.rs b/crates/kebab-app/src/lib.rs index ea7f97e..08c89cc 100644 --- a/crates/kebab-app/src/lib.rs +++ b/crates/kebab-app/src/lib.rs @@ -39,7 +39,7 @@ use std::sync::Arc; use anyhow::{Context, anyhow}; use serde::{Deserialize, Serialize}; -use kebab_chunk::{CodeGoAstV1Chunker, CodeJsAstV1Chunker, CodePythonAstV1Chunker, CodeRustAstV1Chunker, CodeTsAstV1Chunker, MdHeadingV1Chunker, PdfPageV1Chunker}; +use kebab_chunk::{CodeGoAstV1Chunker, CodeJavaAstV1Chunker, CodeJsAstV1Chunker, CodeKotlinAstV1Chunker, CodePythonAstV1Chunker, CodeRustAstV1Chunker, CodeTsAstV1Chunker, MdHeadingV1Chunker, PdfPageV1Chunker}; use kebab_core::{ Answer, Block, CanonicalDocument, Chunk, ChunkId, ChunkPolicy, ChunkerVersion, Chunker, DocFilter, DocSummary, DocumentId, DocumentStore, Embedder, EmbeddingInput, @@ -50,7 +50,7 @@ use kebab_core::{ use kebab_llm_local::OllamaLanguageModel; use kebab_normalize::build_canonical_document; use kebab_parse_image::{ImageExtractor, OllamaVisionOcr, apply_caption, apply_ocr}; -use kebab_parse_code::{GoAstExtractor, JavascriptAstExtractor, PythonAstExtractor, RustAstExtractor, TypescriptAstExtractor}; +use kebab_parse_code::{GoAstExtractor, JavaAstExtractor, JavascriptAstExtractor, KotlinAstExtractor, PythonAstExtractor, RustAstExtractor, TypescriptAstExtractor}; use kebab_parse_pdf::PdfTextExtractor; use kebab_parse_md::{BodyHints, parse_blocks, parse_frontmatter}; use kebab_source_fs::FsSourceConnector; @@ -950,7 +950,8 @@ fn ingest_one_asset( } // p10-1A-2 / 1B: code ingest dispatch. MediaType::Code(lang) - if matches!(lang.as_str(), "rust" | "python" | "typescript" | "javascript" | "go") => + if matches!(lang.as_str(), + "rust" | "python" | "typescript" | "javascript" | "go" | "java" | "kotlin") => { return ingest_one_code_asset( app, @@ -1828,6 +1829,8 @@ fn ingest_one_code_asset( "typescript" => ParserVersion(kebab_parse_code::TS_PARSER_VERSION.to_string()), "javascript" => ParserVersion(kebab_parse_code::JS_PARSER_VERSION.to_string()), "go" => ParserVersion(kebab_parse_code::GO_PARSER_VERSION.to_string()), + "java" => ParserVersion(kebab_parse_code::JAVA_PARSER_VERSION.to_string()), + "kotlin" => ParserVersion(kebab_parse_code::KOTLIN_PARSER_VERSION.to_string()), other => anyhow::bail!("unsupported code_lang: {other}"), }; @@ -1838,6 +1841,8 @@ fn ingest_one_code_asset( "typescript" => CodeTsAstV1Chunker.chunker_version(), "javascript" => CodeJsAstV1Chunker.chunker_version(), "go" => CodeGoAstV1Chunker.chunker_version(), + "java" => CodeJavaAstV1Chunker.chunker_version(), + "kotlin" => CodeKotlinAstV1Chunker.chunker_version(), other => anyhow::bail!("unreachable chunker_version: {other}"), }; @@ -1879,6 +1884,12 @@ fn ingest_one_code_asset( "go" => GoAstExtractor::new() .extract(&ctx, &bytes) .context("kb-parse-code::GoAstExtractor::extract (code:go)")?, + "java" => JavaAstExtractor::new() + .extract(&ctx, &bytes) + .context("kb-parse-code::JavaAstExtractor::extract (code:java)")?, + "kotlin" => KotlinAstExtractor::new() + .extract(&ctx, &bytes) + .context("kb-parse-code::KotlinAstExtractor::extract (code:kotlin)")?, other => anyhow::bail!("unreachable (extract): {other}"), }; @@ -1899,6 +1910,12 @@ fn ingest_one_code_asset( "go" => CodeGoAstV1Chunker .chunk(&canonical, chunk_policy) .context("kb-chunk::CodeGoAstV1Chunker::chunk (code:go)")?, + "java" => CodeJavaAstV1Chunker + .chunk(&canonical, chunk_policy) + .context("kb-chunk::CodeJavaAstV1Chunker::chunk (code:java)")?, + "kotlin" => CodeKotlinAstV1Chunker + .chunk(&canonical, chunk_policy) + .context("kb-chunk::CodeKotlinAstV1Chunker::chunk (code:kotlin)")?, other => anyhow::bail!("unreachable (chunk): {other}"), }; diff --git a/crates/kebab-app/tests/code_ingest_smoke.rs b/crates/kebab-app/tests/code_ingest_smoke.rs index ee852f2..6cffb66 100644 --- a/crates/kebab-app/tests/code_ingest_smoke.rs +++ b/crates/kebab-app/tests/code_ingest_smoke.rs @@ -461,6 +461,148 @@ fn go_file_ingests_and_searches_as_code_citation() { ); } +/// p10-1c-jk Task F: a `.java` file in a package directory is ingested and the +/// resulting `Citation::Code` hit must carry `lang="java"`, +/// `symbol="com.foo.Foo.bar"`, and `line_start >= 1`. +/// The sub-directory (`com/foo/`) ensures the Java package-prefix wiring +/// produces a non-empty module prefix so the fully-qualified symbol assertion +/// exercises that path end-to-end. +#[test] +fn java_file_ingests_and_searches_as_code_citation() { + let env = TestEnv::lexical_only(); + + let pkg_dir = env.workspace_root.join("com").join("foo"); + std::fs::create_dir_all(&pkg_dir).unwrap(); + std::fs::write( + pkg_dir.join("Foo.java"), + "package com.foo;\n\npublic class Foo {\n public String bar() { return \"x\"; }\n}\n", + ) + .unwrap(); + + let report = kebab_app::ingest_with_config(env.config.clone(), env.scope(), false) + .expect("ingest must succeed"); + assert_eq!(report.errors, 0); + assert!(report.new >= 1); + + let java_item = report + .items + .as_ref() + .expect("items present") + .iter() + .find(|i| i.doc_path.0.ends_with("Foo.java")) + .expect("Foo.java item present"); + assert_eq!( + java_item.parser_version.as_ref().map(|p| p.0.as_str()), + Some("code-java-v1"), + "parser_version must be code-java-v1" + ); + assert_eq!( + java_item.chunker_version.as_ref().map(|c| c.0.as_str()), + Some("code-java-ast-v1"), + "chunker_version must be code-java-ast-v1" + ); + + let hits = kebab_app::search_with_config(env.config.clone(), lexical_query("bar")) + .expect("search must succeed"); + let h = hits + .iter() + .find(|h| matches!(&h.citation, kebab_core::Citation::Code { .. })) + .expect("Citation::Code hit"); + match &h.citation { + kebab_core::Citation::Code { + lang, + symbol, + line_start, + .. + } => { + assert_eq!(lang.as_deref(), Some("java"), "citation.lang must be 'java'"); + assert_eq!( + symbol.as_deref(), + Some("com.foo.Foo.bar"), + "citation.symbol must be 'com.foo.Foo.bar'" + ); + assert!(*line_start >= 1, "line_start must be >=1"); + } + _ => unreachable!(), + } + assert_eq!( + h.code_lang.as_deref(), + Some("java"), + "SearchHit.code_lang must be 'java'" + ); +} + +/// p10-1c-jk Task I: a `.kt` file in a package directory is ingested and the +/// resulting `Citation::Code` hit must carry `lang="kotlin"`, +/// `symbol="com.foo.Foo.bar"`, and `line_start >= 1`. +/// The sub-directory (`com/foo/`) ensures the Kotlin package-prefix wiring +/// produces a non-empty module prefix so the fully-qualified symbol assertion +/// exercises that path end-to-end. +#[test] +fn kotlin_file_ingests_and_searches_as_code_citation() { + let env = TestEnv::lexical_only(); + + let pkg_dir = env.workspace_root.join("com").join("foo"); + std::fs::create_dir_all(&pkg_dir).unwrap(); + std::fs::write( + pkg_dir.join("Foo.kt"), + "package com.foo\n\nclass Foo {\n fun bar(): String = \"x\"\n}\n", + ) + .unwrap(); + + let report = kebab_app::ingest_with_config(env.config.clone(), env.scope(), false) + .expect("ingest must succeed"); + assert_eq!(report.errors, 0); + assert!(report.new >= 1); + + let kt_item = report + .items + .as_ref() + .expect("items present") + .iter() + .find(|i| i.doc_path.0.ends_with("Foo.kt")) + .expect("Foo.kt item present"); + assert_eq!( + kt_item.parser_version.as_ref().map(|p| p.0.as_str()), + Some("code-kotlin-v1"), + "parser_version must be code-kotlin-v1" + ); + assert_eq!( + kt_item.chunker_version.as_ref().map(|c| c.0.as_str()), + Some("code-kotlin-ast-v1"), + "chunker_version must be code-kotlin-ast-v1" + ); + + let hits = kebab_app::search_with_config(env.config.clone(), lexical_query("bar")) + .expect("search must succeed"); + let h = hits + .iter() + .find(|h| matches!(&h.citation, kebab_core::Citation::Code { .. })) + .expect("Citation::Code hit"); + match &h.citation { + kebab_core::Citation::Code { + lang, + symbol, + line_start, + .. + } => { + assert_eq!(lang.as_deref(), Some("kotlin"), "citation.lang must be 'kotlin'"); + assert_eq!( + symbol.as_deref(), + Some("com.foo.Foo.bar"), + "citation.symbol must be 'com.foo.Foo.bar'" + ); + assert!(*line_start >= 1, "line_start must be >=1"); + } + _ => unreachable!(), + } + assert_eq!( + h.code_lang.as_deref(), + Some("kotlin"), + "SearchHit.code_lang must be 'kotlin'" + ); +} + /// Re-ingesting the same `.rs` file without changes must report /// `Unchanged` (incremental-skip path exercised). #[test] diff --git a/crates/kebab-chunk/src/code_java_ast_v1.rs b/crates/kebab-chunk/src/code_java_ast_v1.rs new file mode 100644 index 0000000..4f39658 --- /dev/null +++ b/crates/kebab-chunk/src/code_java_ast_v1.rs @@ -0,0 +1,322 @@ +//! `code-java-ast-v1` — maps a tree-sitter-derived Java AST +//! `CanonicalDocument` (one `Block::Code` per semantic unit, each with +//! `SourceSpan::Code`) to chunks 1:1. A unit longer than +//! `AST_CHUNK_MAX_LINES` is split into ` [part i/N]` sub-chunks +//! at blank-line paragraph boundaries (design §9.1 oversize fallback). +//! +//! tree-sitter is intentionally NOT a dependency here: AST work is +//! parser-side (`kebab-parse-code`, design §6.3). This chunker only +//! consumes the `CanonicalDocument`. +//! +//! `AST_CHUNK_MAX_LINES` is a constant matching +//! `IngestCodeCfg::default().ast_chunk_max_lines` (200). Per-medium +//! config threading needs a chunker registry (P+); same deviation +//! pattern as `pdf-page-v1`'s pinned `chunker_version` +//! (`tasks/HOTFIXES.md`). + +use kebab_core::{ + Block, BlockId, CanonicalDocument, Chunk, ChunkPolicy, Chunker, ChunkerVersion, DocumentId, + SourceSpan, id_for_chunk, +}; + +const VERSION_LABEL: &str = "code-java-ast-v1"; +const BYTES_PER_TOKEN: usize = 3; +const POLICY_HASH_HEX_LEN: usize = 16; +const AST_CHUNK_MAX_LINES: u32 = 200; + +#[derive(Clone, Copy, Debug, Default)] +pub struct CodeJavaAstV1Chunker; + +impl Chunker for CodeJavaAstV1Chunker { + fn chunker_version(&self) -> ChunkerVersion { + ChunkerVersion(VERSION_LABEL.to_string()) + } + + fn policy_hash(&self, policy: &ChunkPolicy) -> String { + let bytes = serde_json_canonicalizer::to_vec(policy) + .expect("canonical JSON serialization of ChunkPolicy must not fail"); + let hex = blake3::hash(&bytes).to_hex().to_string(); + hex[..POLICY_HASH_HEX_LEN].to_string() + } + + fn chunk( + &self, + doc: &CanonicalDocument, + policy: &ChunkPolicy, + ) -> anyhow::Result> { + for b in &doc.blocks { + let c = match b { + Block::Code(c) => c, + _ => anyhow::bail!( + "CodeJavaAstV1Chunker only handles code docs (got non-Code block)" + ), + }; + if !matches!(c.common.source_span, SourceSpan::Code { .. }) { + anyhow::bail!( + "CodeJavaAstV1Chunker only handles code docs (got non-Code source_span)" + ); + } + } + + let base_policy_hash = self.policy_hash(policy); + let chunker_version = self.chunker_version(); + let mut out: Vec = Vec::new(); + + for b in &doc.blocks { + let cb = match b { + Block::Code(c) => c, + _ => unreachable!("validated above"), + }; + let (ls, le, symbol, lang) = match &cb.common.source_span { + SourceSpan::Code { line_start, line_end, symbol, lang } => { + (*line_start, *line_end, symbol.clone(), lang.clone()) + } + _ => unreachable!("validated above"), + }; + let block_ids: Vec = vec![cb.common.block_id.clone()]; + let span_lines = le.saturating_sub(ls) + 1; + + if span_lines <= AST_CHUNK_MAX_LINES { + let span = SourceSpan::Code { + line_start: ls, + line_end: le, + symbol: symbol.clone(), + lang: lang.clone(), + }; + out.push(make_chunk( + doc, &chunker_version, &block_ids, &base_policy_hash, + None, span, cb.code.clone(), + )); + } else { + let parts = split_oversize(&cb.code); + let n = parts.len(); + for (i, (off_start, off_end, text)) in parts.into_iter().enumerate() { + let part_ls = ls + off_start; + let part_le = ls + off_end; + let part_sym = symbol + .as_ref() + .map(|s| format!("{s} [part {}/{n}]", i + 1)); + let span = SourceSpan::Code { + line_start: part_ls, + line_end: part_le, + symbol: part_sym, + lang: lang.clone(), + }; + out.push(make_chunk( + doc, &chunker_version, &block_ids, &base_policy_hash, + Some(part_ls), span, text, + )); + } + } + } + + tracing::debug!( + target: "kebab-chunk", + doc_id = %doc.doc_id, + chunks = out.len(), + "code-java-ast-v1 chunked", + ); + Ok(out) + } +} + +#[allow(clippy::too_many_arguments)] +fn make_chunk( + doc: &CanonicalDocument, + chunker_version: &ChunkerVersion, + block_ids: &[BlockId], + base_policy_hash: &str, + split_key: Option, + span: SourceSpan, + text: String, +) -> Chunk { + let id_hash = match split_key { + Some(k) => format!("{base_policy_hash}#L{k}"), + None => base_policy_hash.to_string(), + }; + let chunk_id = id_for_chunk(&doc.doc_id, chunker_version, block_ids, &id_hash); + let token_estimate = text.len().div_ceil(BYTES_PER_TOKEN); + Chunk { + chunk_id, + doc_id: DocumentId(doc.doc_id.0.clone()), + block_ids: block_ids.to_vec(), + text, + heading_path: Vec::new(), + source_spans: vec![span], + token_estimate, + chunker_version: chunker_version.clone(), + policy_hash: base_policy_hash.to_string(), + } +} + +/// Split an oversize unit at blank-line paragraph boundaries, greedily +/// gluing paragraphs until ~`AST_CHUNK_MAX_LINES` lines accumulate. +/// Returns `(line_offset_start, line_offset_end, text)` where offsets are +/// 0-based within the unit (caller adds the unit's absolute `line_start`). +fn split_oversize(code: &str) -> Vec<(u32, u32, String)> { + let lines: Vec<&str> = code.split('\n').collect(); + let total = lines.len() as u32; + let mut out: Vec<(u32, u32, String)> = Vec::new(); + let mut start: u32 = 0; + while start < total { + let mut end = (start + AST_CHUNK_MAX_LINES).min(total); + let floor = start + (AST_CHUNK_MAX_LINES * 4 / 5); + if end < total { + if let Some(b) = (floor.min(end)..end) + .rev() + .find(|&i| lines[i as usize].trim().is_empty()) + { + end = b + 1; + } + } + let text = lines[start as usize..end as usize].join("\n"); + out.push((start, end.saturating_sub(1), text)); + start = end; + } + if out.is_empty() { + out.push((0, total.saturating_sub(1), code.to_string())); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use kebab_core::{ + Block, CanonicalDocument, ChunkPolicy, Chunker, ChunkerVersion, CodeBlock, CommonBlock, + SourceSpan, id_for_block, id_for_doc, AssetId, Lang, Metadata, ParserVersion, Provenance, + SourceType, TrustLevel, WorkspacePath, + }; + use time::OffsetDateTime; + + fn code_doc(units: &[(&str, u32, u32, &str)]) -> CanonicalDocument { + let wp = WorkspacePath("crates/x/src/Main.java".into()); + let aid = AssetId("a".repeat(64)); + let pv = ParserVersion("code-java-v1".into()); + let doc_id = id_for_doc(&wp, &aid, &pv); + let blocks = units + .iter() + .enumerate() + .map(|(i, (sym, ls, le, code))| { + let span = SourceSpan::Code { + line_start: *ls, + line_end: *le, + symbol: Some((*sym).to_string()), + lang: Some("java".into()), + }; + let bid = id_for_block(&doc_id, "code", &[], i as u32, &span); + Block::Code(CodeBlock { + common: CommonBlock { block_id: bid, heading_path: vec![], source_span: span }, + lang: Some("java".into()), + code: (*code).to_string(), + }) + }) + .collect(); + CanonicalDocument { + doc_id, source_asset_id: aid, workspace_path: wp, title: "a".into(), + lang: Lang("und".into()), blocks, + metadata: Metadata { + aliases: vec![], tags: vec![], + created_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(), + updated_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(), + source_type: SourceType::Note, trust_level: TrustLevel::Primary, + user_id_alias: None, user: Default::default(), + repo: Some("kebab".into()), git_branch: Some("main".into()), + git_commit: Some("0".repeat(40)), code_lang: Some("java".into()), + }, + provenance: Provenance { events: vec![] }, + parser_version: pv, schema_version: 1, doc_version: 1, + last_chunker_version: None, last_embedding_version: None, + } + } + fn policy() -> ChunkPolicy { + ChunkPolicy { target_tokens: 500, overlap_tokens: 80, + respect_markdown_headings: false, + chunker_version: ChunkerVersion(VERSION_LABEL.into()) } + } + + #[test] + fn chunker_version_is_code_java_ast_v1() { + assert_eq!(CodeJavaAstV1Chunker.chunker_version(), + ChunkerVersion("code-java-ast-v1".into())); + } + + #[test] + fn one_chunk_per_unit_preserves_code_span() { + let doc = code_doc(&[ + ("parse", 1, 3, "void parse() {\n\t// x\n}"), + ("Foo.double", 5, 7, "int double() {\n\t//\n\treturn 0;\n}"), + ]); + let chunks = CodeJavaAstV1Chunker.chunk(&doc, &policy()).unwrap(); + assert_eq!(chunks.len(), 2); + for c in &chunks { + assert_eq!(c.source_spans.len(), 1); + assert!(matches!(c.source_spans[0], SourceSpan::Code { .. })); + assert_eq!(c.heading_path, Vec::::new()); + assert_eq!(c.chunker_version.0, "code-java-ast-v1"); + } + match &chunks[0].source_spans[0] { + SourceSpan::Code { symbol, line_start, line_end, .. } => { + assert_eq!(symbol.as_deref(), Some("parse")); + assert_eq!((*line_start, *line_end), (1, 3)); + } + _ => unreachable!(), + } + } + + #[test] + fn oversize_unit_splits_into_parts_with_unique_ids() { + let body = (0..500).map(|i| format!("\tint x{i} = {i};")).collect::>().join("\n"); + let code = format!("void big() {{\n{body}\n}}"); + let doc = code_doc(&[("big", 1, 502, &code)]); + let chunks = CodeJavaAstV1Chunker.chunk(&doc, &policy()).unwrap(); + assert!(chunks.len() >= 2, "oversize unit must split, got {}", chunks.len()); + for c in &chunks { + match &c.source_spans[0] { + SourceSpan::Code { symbol, .. } => { + assert!(symbol.as_deref().unwrap().starts_with("big [part "), + "part-numbered symbol, got {symbol:?}"); + } + _ => unreachable!(), + } + } + let mut ids: Vec<&str> = chunks.iter().map(|c| c.chunk_id.0.as_str()).collect(); + let n = ids.len(); ids.sort(); ids.dedup(); + assert_eq!(ids.len(), n, "chunk_ids unique across split parts"); + } + + #[test] + fn non_code_doc_errors() { + use kebab_core::TextBlock; + let mut doc = code_doc(&[("parse", 1, 1, "void parse() {}")]); + doc.blocks = vec![Block::Paragraph(TextBlock { + common: CommonBlock { + block_id: kebab_core::BlockId("b".into()), + heading_path: vec![], + source_span: SourceSpan::Line { start: 1, end: 1 }, + }, + text: "x".into(), inlines: vec![], + })]; + let err = CodeJavaAstV1Chunker.chunk(&doc, &policy()).unwrap_err(); + assert!(err.to_string().contains("CodeJavaAstV1Chunker")); + } + + #[test] + fn deterministic_chunk_ids_1000() { + let doc = code_doc(&[("parse", 1, 2, "void parse() {}\n")]); + let base: Vec = CodeJavaAstV1Chunker.chunk(&doc, &policy()) + .unwrap().into_iter().map(|c| c.chunk_id.0).collect(); + for _ in 0..1000 { + let again: Vec = CodeJavaAstV1Chunker.chunk(&doc, &policy()) + .unwrap().into_iter().map(|c| c.chunk_id.0).collect(); + assert_eq!(again, base); + } + } + + #[test] + fn policy_hash_matches_md_heading_v1() { + let p = policy(); + assert_eq!(CodeJavaAstV1Chunker.policy_hash(&p), + crate::MdHeadingV1Chunker.policy_hash(&p)); + } +} diff --git a/crates/kebab-chunk/src/code_kotlin_ast_v1.rs b/crates/kebab-chunk/src/code_kotlin_ast_v1.rs new file mode 100644 index 0000000..e1c2983 --- /dev/null +++ b/crates/kebab-chunk/src/code_kotlin_ast_v1.rs @@ -0,0 +1,322 @@ +//! `code-kotlin-ast-v1` — maps a tree-sitter-derived Kotlin AST +//! `CanonicalDocument` (one `Block::Code` per semantic unit, each with +//! `SourceSpan::Code`) to chunks 1:1. A unit longer than +//! `AST_CHUNK_MAX_LINES` is split into ` [part i/N]` sub-chunks +//! at blank-line paragraph boundaries (design §9.1 oversize fallback). +//! +//! tree-sitter is intentionally NOT a dependency here: AST work is +//! parser-side (`kebab-parse-code`, design §6.3). This chunker only +//! consumes the `CanonicalDocument`. +//! +//! `AST_CHUNK_MAX_LINES` is a constant matching +//! `IngestCodeCfg::default().ast_chunk_max_lines` (200). Per-medium +//! config threading needs a chunker registry (P+); same deviation +//! pattern as `pdf-page-v1`'s pinned `chunker_version` +//! (`tasks/HOTFIXES.md`). + +use kebab_core::{ + Block, BlockId, CanonicalDocument, Chunk, ChunkPolicy, Chunker, ChunkerVersion, DocumentId, + SourceSpan, id_for_chunk, +}; + +const VERSION_LABEL: &str = "code-kotlin-ast-v1"; +const BYTES_PER_TOKEN: usize = 3; +const POLICY_HASH_HEX_LEN: usize = 16; +const AST_CHUNK_MAX_LINES: u32 = 200; + +#[derive(Clone, Copy, Debug, Default)] +pub struct CodeKotlinAstV1Chunker; + +impl Chunker for CodeKotlinAstV1Chunker { + fn chunker_version(&self) -> ChunkerVersion { + ChunkerVersion(VERSION_LABEL.to_string()) + } + + fn policy_hash(&self, policy: &ChunkPolicy) -> String { + let bytes = serde_json_canonicalizer::to_vec(policy) + .expect("canonical JSON serialization of ChunkPolicy must not fail"); + let hex = blake3::hash(&bytes).to_hex().to_string(); + hex[..POLICY_HASH_HEX_LEN].to_string() + } + + fn chunk( + &self, + doc: &CanonicalDocument, + policy: &ChunkPolicy, + ) -> anyhow::Result> { + for b in &doc.blocks { + let c = match b { + Block::Code(c) => c, + _ => anyhow::bail!( + "CodeKotlinAstV1Chunker only handles code docs (got non-Code block)" + ), + }; + if !matches!(c.common.source_span, SourceSpan::Code { .. }) { + anyhow::bail!( + "CodeKotlinAstV1Chunker only handles code docs (got non-Code source_span)" + ); + } + } + + let base_policy_hash = self.policy_hash(policy); + let chunker_version = self.chunker_version(); + let mut out: Vec = Vec::new(); + + for b in &doc.blocks { + let cb = match b { + Block::Code(c) => c, + _ => unreachable!("validated above"), + }; + let (ls, le, symbol, lang) = match &cb.common.source_span { + SourceSpan::Code { line_start, line_end, symbol, lang } => { + (*line_start, *line_end, symbol.clone(), lang.clone()) + } + _ => unreachable!("validated above"), + }; + let block_ids: Vec = vec![cb.common.block_id.clone()]; + let span_lines = le.saturating_sub(ls) + 1; + + if span_lines <= AST_CHUNK_MAX_LINES { + let span = SourceSpan::Code { + line_start: ls, + line_end: le, + symbol: symbol.clone(), + lang: lang.clone(), + }; + out.push(make_chunk( + doc, &chunker_version, &block_ids, &base_policy_hash, + None, span, cb.code.clone(), + )); + } else { + let parts = split_oversize(&cb.code); + let n = parts.len(); + for (i, (off_start, off_end, text)) in parts.into_iter().enumerate() { + let part_ls = ls + off_start; + let part_le = ls + off_end; + let part_sym = symbol + .as_ref() + .map(|s| format!("{s} [part {}/{n}]", i + 1)); + let span = SourceSpan::Code { + line_start: part_ls, + line_end: part_le, + symbol: part_sym, + lang: lang.clone(), + }; + out.push(make_chunk( + doc, &chunker_version, &block_ids, &base_policy_hash, + Some(part_ls), span, text, + )); + } + } + } + + tracing::debug!( + target: "kebab-chunk", + doc_id = %doc.doc_id, + chunks = out.len(), + "code-kotlin-ast-v1 chunked", + ); + Ok(out) + } +} + +#[allow(clippy::too_many_arguments)] +fn make_chunk( + doc: &CanonicalDocument, + chunker_version: &ChunkerVersion, + block_ids: &[BlockId], + base_policy_hash: &str, + split_key: Option, + span: SourceSpan, + text: String, +) -> Chunk { + let id_hash = match split_key { + Some(k) => format!("{base_policy_hash}#L{k}"), + None => base_policy_hash.to_string(), + }; + let chunk_id = id_for_chunk(&doc.doc_id, chunker_version, block_ids, &id_hash); + let token_estimate = text.len().div_ceil(BYTES_PER_TOKEN); + Chunk { + chunk_id, + doc_id: DocumentId(doc.doc_id.0.clone()), + block_ids: block_ids.to_vec(), + text, + heading_path: Vec::new(), + source_spans: vec![span], + token_estimate, + chunker_version: chunker_version.clone(), + policy_hash: base_policy_hash.to_string(), + } +} + +/// Split an oversize unit at blank-line paragraph boundaries, greedily +/// gluing paragraphs until ~`AST_CHUNK_MAX_LINES` lines accumulate. +/// Returns `(line_offset_start, line_offset_end, text)` where offsets are +/// 0-based within the unit (caller adds the unit's absolute `line_start`). +fn split_oversize(code: &str) -> Vec<(u32, u32, String)> { + let lines: Vec<&str> = code.split('\n').collect(); + let total = lines.len() as u32; + let mut out: Vec<(u32, u32, String)> = Vec::new(); + let mut start: u32 = 0; + while start < total { + let mut end = (start + AST_CHUNK_MAX_LINES).min(total); + let floor = start + (AST_CHUNK_MAX_LINES * 4 / 5); + if end < total { + if let Some(b) = (floor.min(end)..end) + .rev() + .find(|&i| lines[i as usize].trim().is_empty()) + { + end = b + 1; + } + } + let text = lines[start as usize..end as usize].join("\n"); + out.push((start, end.saturating_sub(1), text)); + start = end; + } + if out.is_empty() { + out.push((0, total.saturating_sub(1), code.to_string())); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use kebab_core::{ + Block, CanonicalDocument, ChunkPolicy, Chunker, ChunkerVersion, CodeBlock, CommonBlock, + SourceSpan, id_for_block, id_for_doc, AssetId, Lang, Metadata, ParserVersion, Provenance, + SourceType, TrustLevel, WorkspacePath, + }; + use time::OffsetDateTime; + + fn code_doc(units: &[(&str, u32, u32, &str)]) -> CanonicalDocument { + let wp = WorkspacePath("crates/x/src/Main.kt".into()); + let aid = AssetId("a".repeat(64)); + let pv = ParserVersion("code-kotlin-v1".into()); + let doc_id = id_for_doc(&wp, &aid, &pv); + let blocks = units + .iter() + .enumerate() + .map(|(i, (sym, ls, le, code))| { + let span = SourceSpan::Code { + line_start: *ls, + line_end: *le, + symbol: Some((*sym).to_string()), + lang: Some("kotlin".into()), + }; + let bid = id_for_block(&doc_id, "code", &[], i as u32, &span); + Block::Code(CodeBlock { + common: CommonBlock { block_id: bid, heading_path: vec![], source_span: span }, + lang: Some("kotlin".into()), + code: (*code).to_string(), + }) + }) + .collect(); + CanonicalDocument { + doc_id, source_asset_id: aid, workspace_path: wp, title: "a".into(), + lang: Lang("und".into()), blocks, + metadata: Metadata { + aliases: vec![], tags: vec![], + created_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(), + updated_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(), + source_type: SourceType::Note, trust_level: TrustLevel::Primary, + user_id_alias: None, user: Default::default(), + repo: Some("kebab".into()), git_branch: Some("main".into()), + git_commit: Some("0".repeat(40)), code_lang: Some("kotlin".into()), + }, + provenance: Provenance { events: vec![] }, + parser_version: pv, schema_version: 1, doc_version: 1, + last_chunker_version: None, last_embedding_version: None, + } + } + fn policy() -> ChunkPolicy { + ChunkPolicy { target_tokens: 500, overlap_tokens: 80, + respect_markdown_headings: false, + chunker_version: ChunkerVersion(VERSION_LABEL.into()) } + } + + #[test] + fn chunker_version_is_code_kotlin_ast_v1() { + assert_eq!(CodeKotlinAstV1Chunker.chunker_version(), + ChunkerVersion("code-kotlin-ast-v1".into())); + } + + #[test] + fn one_chunk_per_unit_preserves_code_span() { + let doc = code_doc(&[ + ("parse", 1, 3, "fun parse() {\n\t// x\n}"), + ("Foo.double", 5, 7, "fun double(): Int {\n\t//\n\treturn 0\n}"), + ]); + let chunks = CodeKotlinAstV1Chunker.chunk(&doc, &policy()).unwrap(); + assert_eq!(chunks.len(), 2); + for c in &chunks { + assert_eq!(c.source_spans.len(), 1); + assert!(matches!(c.source_spans[0], SourceSpan::Code { .. })); + assert_eq!(c.heading_path, Vec::::new()); + assert_eq!(c.chunker_version.0, "code-kotlin-ast-v1"); + } + match &chunks[0].source_spans[0] { + SourceSpan::Code { symbol, line_start, line_end, .. } => { + assert_eq!(symbol.as_deref(), Some("parse")); + assert_eq!((*line_start, *line_end), (1, 3)); + } + _ => unreachable!(), + } + } + + #[test] + fn oversize_unit_splits_into_parts_with_unique_ids() { + let body = (0..500).map(|i| format!("\tval x{i} = {i}")).collect::>().join("\n"); + let code = format!("fun big() {{\n{body}\n}}"); + let doc = code_doc(&[("big", 1, 502, &code)]); + let chunks = CodeKotlinAstV1Chunker.chunk(&doc, &policy()).unwrap(); + assert!(chunks.len() >= 2, "oversize unit must split, got {}", chunks.len()); + for c in &chunks { + match &c.source_spans[0] { + SourceSpan::Code { symbol, .. } => { + assert!(symbol.as_deref().unwrap().starts_with("big [part "), + "part-numbered symbol, got {symbol:?}"); + } + _ => unreachable!(), + } + } + let mut ids: Vec<&str> = chunks.iter().map(|c| c.chunk_id.0.as_str()).collect(); + let n = ids.len(); ids.sort(); ids.dedup(); + assert_eq!(ids.len(), n, "chunk_ids unique across split parts"); + } + + #[test] + fn non_code_doc_errors() { + use kebab_core::TextBlock; + let mut doc = code_doc(&[("parse", 1, 1, "fun parse() {}")]); + doc.blocks = vec![Block::Paragraph(TextBlock { + common: CommonBlock { + block_id: kebab_core::BlockId("b".into()), + heading_path: vec![], + source_span: SourceSpan::Line { start: 1, end: 1 }, + }, + text: "x".into(), inlines: vec![], + })]; + let err = CodeKotlinAstV1Chunker.chunk(&doc, &policy()).unwrap_err(); + assert!(err.to_string().contains("CodeKotlinAstV1Chunker")); + } + + #[test] + fn deterministic_chunk_ids_1000() { + let doc = code_doc(&[("parse", 1, 2, "fun parse() {}\n")]); + let base: Vec = CodeKotlinAstV1Chunker.chunk(&doc, &policy()) + .unwrap().into_iter().map(|c| c.chunk_id.0).collect(); + for _ in 0..1000 { + let again: Vec = CodeKotlinAstV1Chunker.chunk(&doc, &policy()) + .unwrap().into_iter().map(|c| c.chunk_id.0).collect(); + assert_eq!(again, base); + } + } + + #[test] + fn policy_hash_matches_md_heading_v1() { + let p = policy(); + assert_eq!(CodeKotlinAstV1Chunker.policy_hash(&p), + crate::MdHeadingV1Chunker.policy_hash(&p)); + } +} diff --git a/crates/kebab-chunk/src/lib.rs b/crates/kebab-chunk/src/lib.rs index cc50571..750d18e 100644 --- a/crates/kebab-chunk/src/lib.rs +++ b/crates/kebab-chunk/src/lib.rs @@ -16,7 +16,9 @@ //! It consumes `CanonicalDocument` purely through `kb-core` types. mod code_go_ast_v1; +mod code_java_ast_v1; mod code_js_ast_v1; +mod code_kotlin_ast_v1; mod code_python_ast_v1; mod code_rust_ast_v1; mod code_ts_ast_v1; @@ -24,7 +26,9 @@ mod md_heading_v1; mod pdf_page_v1; pub use code_go_ast_v1::CodeGoAstV1Chunker; +pub use code_java_ast_v1::CodeJavaAstV1Chunker; pub use code_js_ast_v1::CodeJsAstV1Chunker; +pub use code_kotlin_ast_v1::CodeKotlinAstV1Chunker; pub use code_python_ast_v1::CodePythonAstV1Chunker; pub use code_rust_ast_v1::CodeRustAstV1Chunker; pub use code_ts_ast_v1::CodeTsAstV1Chunker; diff --git a/crates/kebab-chunk/tests/code_java_ast_snapshot.rs b/crates/kebab-chunk/tests/code_java_ast_snapshot.rs new file mode 100644 index 0000000..75473d6 --- /dev/null +++ b/crates/kebab-chunk/tests/code_java_ast_snapshot.rs @@ -0,0 +1,221 @@ +//! Snapshot test pinning the `Vec` JSON for a +//! representative Java code `CanonicalDocument`. +//! +//! This is an integration test. `kebab-parse-code` is intentionally NOT +//! a dev-dep (design §6.3 / §8 boundary: AST extraction is parser-side). +//! The `CanonicalDocument` is built inline from hand-crafted `Block::Code` +//! units, which is the same pattern used in `code_rust_ast_v1.rs`'s +//! internal `code_doc` test helper. +//! +//! Set `UPDATE_SNAPSHOTS=1` to re-bake the baseline. + +use std::path::PathBuf; + +use kebab_chunk::CodeJavaAstV1Chunker; +use kebab_core::{ + AssetId, Block, CanonicalDocument, ChunkPolicy, Chunker, ChunkerVersion, CodeBlock, CommonBlock, + Lang, Metadata, ParserVersion, Provenance, SourceSpan, SourceType, TrustLevel, WorkspacePath, + id_for_block, id_for_doc, +}; +use serde_json::Value; +use time::OffsetDateTime; + +fn fixtures_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests") + .join("fixtures") +} + +fn fixed_doc() -> CanonicalDocument { + let wp = WorkspacePath("src/main/java/com/example/Metrics.java".into()); + let aid = AssetId("b".repeat(64)); + // Pin parser_version so doc_id / block_ids are reproducible. + let pv = ParserVersion("code-java-v1".into()); + let doc_id = id_for_doc(&wp, &aid, &pv); + + // Build a >200-line method body to force split_oversize. + let big_body: String = { + let header = "public class BigCompute {\n public int compute(int[] data) {\n"; + let body: String = (0..210u32) + .map(|i| format!(" int v{i} = {i} < data.length ? data[{i}] : 0;\n")) + .collect(); + let footer = " return data.length;\n }\n}"; + format!("{header}{body}{footer}") + }; + let big_line_count = big_body.lines().count() as u32; + let big_line_end = 48 + big_line_count - 1; + + // Representative units: + // 0. import block (lines 1–5, ≤200) + // 1. free method `computeMRR` (lines 7–12, ≤200) + // 2. class `MetricsCollector` (lines 14–20, ≤200) + // 3. class `BaseEvaluator` (lines 22–30, ≤200) + // 4. method `MetricsCollector.run` (lines 32–38, ≤200) + // 5. method `MetricsCollector.report` (lines 40–46, ≤200) + // 6. BigCompute (>200 lines) to force split_oversize + let raw_units: Vec<(&str, u32, u32, String)> = vec![ + ( + "imports", + 1, + 5, + "import java.util.List;\nimport java.util.Map;\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.stream.Collectors;".to_string(), + ), + ( + "computeMRR", + 7, + 12, + "public static double computeMRR(List scores) {\n if (scores.isEmpty()) {\n return 0.0;\n }\n return 1.0 / scores.size();\n}".to_string(), + ), + ( + "MetricsCollector", + 14, + 20, + "public class MetricsCollector {\n private List scores;\n private List labels;\n private Map counts;\n private Map totals;\n private List tags;\n}".to_string(), + ), + ( + "BaseEvaluator", + 22, + 30, + "public class BaseEvaluator {\n private String name;\n\n public BaseEvaluator(String name) {\n this.name = name;\n }\n\n public void evaluate(List data) throws Exception {\n String joined = String.join(\",\", data);\n }\n}".to_string(), + ), + ( + "MetricsCollector.run", + 32, + 38, + "public void run(List inputs) {\n for (Double inp : inputs) {\n scores.add(\n inp\n );\n }\n}".to_string(), + ), + ( + "MetricsCollector.report", + 40, + 46, + "public Map report() {\n Map result = new HashMap<>();\n result.put(\"mean\", 0.0);\n result.put(\"count\", scores.size());\n result.put(\"tags\", tags);\n return result;\n}".to_string(), + ), + ("BigCompute", 48, big_line_end, big_body), + ]; + + let blocks: Vec = raw_units + .iter() + .enumerate() + .map(|(i, (sym, ls, le, code))| { + let span = SourceSpan::Code { + line_start: *ls, + line_end: *le, + symbol: Some((*sym).to_string()), + lang: Some("java".into()), + }; + let bid = id_for_block(&doc_id, "code", &[], i as u32, &span); + Block::Code(CodeBlock { + common: CommonBlock { + block_id: bid, + heading_path: vec![], + source_span: span, + }, + lang: Some("java".into()), + code: code.clone(), + }) + }) + .collect(); + + CanonicalDocument { + doc_id, + source_asset_id: aid, + workspace_path: wp, + title: "Metrics.java".into(), + lang: Lang("und".into()), + blocks, + metadata: Metadata { + aliases: vec![], + tags: vec![], + created_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(), + updated_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(), + source_type: SourceType::Note, + trust_level: TrustLevel::Primary, + user_id_alias: None, + user: Default::default(), + repo: Some("kebab".into()), + git_branch: Some("main".into()), + git_commit: Some("0".repeat(40)), + code_lang: Some("java".into()), + }, + provenance: Provenance { events: vec![] }, + parser_version: pv, + schema_version: 1, + doc_version: 1, + last_chunker_version: None, + last_embedding_version: None, + } +} + +fn fixed_policy() -> ChunkPolicy { + ChunkPolicy { + target_tokens: 500, + overlap_tokens: 80, + respect_markdown_headings: false, + chunker_version: ChunkerVersion("code-java-ast-v1".into()), + } +} + +#[test] +fn code_java_ast_chunks_snapshot() { + let doc = fixed_doc(); + let policy = fixed_policy(); + + let chunks = CodeJavaAstV1Chunker.chunk(&doc, &policy).expect("chunk"); + let actual = serde_json::to_value(&chunks).unwrap(); + + let dir = fixtures_dir(); + let baseline_path = dir.join("code-sample.java.chunks.snapshot.json"); + let baseline_text = match std::fs::read_to_string(&baseline_path) { + Ok(s) => s, + Err(_) if std::env::var("UPDATE_SNAPSHOTS").is_ok() => { + std::fs::create_dir_all(&dir).unwrap(); + let pretty = serde_json::to_string_pretty(&actual).unwrap(); + std::fs::write(&baseline_path, format!("{pretty}\n")).unwrap(); + return; + } + Err(e) => panic!( + "missing baseline {}; run with UPDATE_SNAPSHOTS=1 to create: {e}", + baseline_path.display() + ), + }; + let expected: Value = serde_json::from_str(&baseline_text).expect("baseline parses as json"); + + if actual != expected { + if std::env::var("UPDATE_SNAPSHOTS").is_ok() { + let pretty = serde_json::to_string_pretty(&actual).unwrap(); + std::fs::write(&baseline_path, format!("{pretty}\n")).unwrap(); + eprintln!("updated baseline {}", baseline_path.display()); + return; + } + let pretty = serde_json::to_string_pretty(&actual).unwrap(); + panic!( + "code-java-ast-v1 chunks snapshot drift\n\ + --- expected ({}) ---\n{baseline_text}\n\ + --- actual ---\n{pretty}\n\ + If intentional, re-run with UPDATE_SNAPSHOTS=1.", + baseline_path.display() + ); + } +} + +/// Determinism cross-check: re-running the same pipeline yields the same +/// chunk_ids byte-for-byte. +#[test] +fn code_java_ast_chunks_are_deterministic() { + let policy = fixed_policy(); + let baseline: Vec = CodeJavaAstV1Chunker + .chunk(&fixed_doc(), &policy) + .unwrap() + .into_iter() + .map(|c| c.chunk_id.0) + .collect(); + for _ in 0..5 { + let again: Vec = CodeJavaAstV1Chunker + .chunk(&fixed_doc(), &policy) + .unwrap() + .into_iter() + .map(|c| c.chunk_id.0) + .collect(); + assert_eq!(again, baseline); + } +} diff --git a/crates/kebab-chunk/tests/code_kotlin_ast_snapshot.rs b/crates/kebab-chunk/tests/code_kotlin_ast_snapshot.rs new file mode 100644 index 0000000..a1eafa6 --- /dev/null +++ b/crates/kebab-chunk/tests/code_kotlin_ast_snapshot.rs @@ -0,0 +1,221 @@ +//! Snapshot test pinning the `Vec` JSON for a +//! representative Kotlin code `CanonicalDocument`. +//! +//! This is an integration test. `kebab-parse-code` is intentionally NOT +//! a dev-dep (design §6.3 / §8 boundary: AST extraction is parser-side). +//! The `CanonicalDocument` is built inline from hand-crafted `Block::Code` +//! units, which is the same pattern used in `code_rust_ast_v1.rs`'s +//! internal `code_doc` test helper. +//! +//! Set `UPDATE_SNAPSHOTS=1` to re-bake the baseline. + +use std::path::PathBuf; + +use kebab_chunk::CodeKotlinAstV1Chunker; +use kebab_core::{ + AssetId, Block, CanonicalDocument, ChunkPolicy, Chunker, ChunkerVersion, CodeBlock, CommonBlock, + Lang, Metadata, ParserVersion, Provenance, SourceSpan, SourceType, TrustLevel, WorkspacePath, + id_for_block, id_for_doc, +}; +use serde_json::Value; +use time::OffsetDateTime; + +fn fixtures_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests") + .join("fixtures") +} + +fn fixed_doc() -> CanonicalDocument { + let wp = WorkspacePath("src/main/kotlin/com/example/Metrics.kt".into()); + let aid = AssetId("b".repeat(64)); + // Pin parser_version so doc_id / block_ids are reproducible. + let pv = ParserVersion("code-kotlin-v1".into()); + let doc_id = id_for_doc(&wp, &aid, &pv); + + // Build a >200-line function body to force split_oversize. + let big_body: String = { + let header = "class BigCompute {\n fun compute(data: IntArray): Int {\n"; + let body: String = (0..210u32) + .map(|i| format!(" val v{i} = if ({i} < data.size) data[{i}] else 0\n")) + .collect(); + let footer = " return data.size\n }\n}"; + format!("{header}{body}{footer}") + }; + let big_line_count = big_body.lines().count() as u32; + let big_line_end = 48 + big_line_count - 1; + + // Representative units: + // 0. import block (lines 1–5, ≤200) + // 1. top-level fn `computeMRR` (lines 7–12, ≤200) + // 2. data class `MetricsCollector` (lines 14–20, ≤200) + // 3. class `BaseEvaluator` (lines 22–30, ≤200) + // 4. method `MetricsCollector.run` (lines 32–38, ≤200) + // 5. method `MetricsCollector.report` (lines 40–46, ≤200) + // 6. BigCompute (>200 lines) to force split_oversize + let raw_units: Vec<(&str, u32, u32, String)> = vec![ + ( + "imports", + 1, + 5, + "import kotlin.collections.List\nimport kotlin.collections.Map\nimport kotlin.collections.MutableList\nimport kotlin.collections.MutableMap\nimport kotlin.collections.mutableListOf".to_string(), + ), + ( + "computeMRR", + 7, + 12, + "fun computeMRR(scores: List): Double {\n if (scores.isEmpty()) {\n return 0.0\n }\n return 1.0 / scores.size\n}".to_string(), + ), + ( + "MetricsCollector", + 14, + 20, + "data class MetricsCollector(\n val scores: MutableList = mutableListOf(),\n val labels: MutableList = mutableListOf(),\n val counts: MutableMap = mutableMapOf(),\n val totals: MutableMap = mutableMapOf(),\n val tags: MutableList = mutableListOf(),\n)".to_string(), + ), + ( + "BaseEvaluator", + 22, + 30, + "open class BaseEvaluator(val name: String) {\n\n fun evaluate(data: List) {\n val joined = data.joinToString(\",\")\n println(joined)\n }\n\n open fun describe(): String = name\n}".to_string(), + ), + ( + "MetricsCollector.run", + 32, + 38, + "fun MetricsCollector.run(inputs: List) {\n for (inp in inputs) {\n scores.add(\n inp\n )\n }\n}".to_string(), + ), + ( + "MetricsCollector.report", + 40, + 46, + "fun MetricsCollector.report(): Map {\n return mapOf(\n \"mean\" to 0.0,\n \"count\" to scores.size,\n \"tags\" to tags,\n )\n}".to_string(), + ), + ("BigCompute", 48, big_line_end, big_body), + ]; + + let blocks: Vec = raw_units + .iter() + .enumerate() + .map(|(i, (sym, ls, le, code))| { + let span = SourceSpan::Code { + line_start: *ls, + line_end: *le, + symbol: Some((*sym).to_string()), + lang: Some("kotlin".into()), + }; + let bid = id_for_block(&doc_id, "code", &[], i as u32, &span); + Block::Code(CodeBlock { + common: CommonBlock { + block_id: bid, + heading_path: vec![], + source_span: span, + }, + lang: Some("kotlin".into()), + code: code.clone(), + }) + }) + .collect(); + + CanonicalDocument { + doc_id, + source_asset_id: aid, + workspace_path: wp, + title: "Metrics.kt".into(), + lang: Lang("und".into()), + blocks, + metadata: Metadata { + aliases: vec![], + tags: vec![], + created_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(), + updated_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(), + source_type: SourceType::Note, + trust_level: TrustLevel::Primary, + user_id_alias: None, + user: Default::default(), + repo: Some("kebab".into()), + git_branch: Some("main".into()), + git_commit: Some("0".repeat(40)), + code_lang: Some("kotlin".into()), + }, + provenance: Provenance { events: vec![] }, + parser_version: pv, + schema_version: 1, + doc_version: 1, + last_chunker_version: None, + last_embedding_version: None, + } +} + +fn fixed_policy() -> ChunkPolicy { + ChunkPolicy { + target_tokens: 500, + overlap_tokens: 80, + respect_markdown_headings: false, + chunker_version: ChunkerVersion("code-kotlin-ast-v1".into()), + } +} + +#[test] +fn code_kotlin_ast_chunks_snapshot() { + let doc = fixed_doc(); + let policy = fixed_policy(); + + let chunks = CodeKotlinAstV1Chunker.chunk(&doc, &policy).expect("chunk"); + let actual = serde_json::to_value(&chunks).unwrap(); + + let dir = fixtures_dir(); + let baseline_path = dir.join("code-sample.kt.chunks.snapshot.json"); + let baseline_text = match std::fs::read_to_string(&baseline_path) { + Ok(s) => s, + Err(_) if std::env::var("UPDATE_SNAPSHOTS").is_ok() => { + std::fs::create_dir_all(&dir).unwrap(); + let pretty = serde_json::to_string_pretty(&actual).unwrap(); + std::fs::write(&baseline_path, format!("{pretty}\n")).unwrap(); + return; + } + Err(e) => panic!( + "missing baseline {}; run with UPDATE_SNAPSHOTS=1 to create: {e}", + baseline_path.display() + ), + }; + let expected: Value = serde_json::from_str(&baseline_text).expect("baseline parses as json"); + + if actual != expected { + if std::env::var("UPDATE_SNAPSHOTS").is_ok() { + let pretty = serde_json::to_string_pretty(&actual).unwrap(); + std::fs::write(&baseline_path, format!("{pretty}\n")).unwrap(); + eprintln!("updated baseline {}", baseline_path.display()); + return; + } + let pretty = serde_json::to_string_pretty(&actual).unwrap(); + panic!( + "code-kotlin-ast-v1 chunks snapshot drift\n\ + --- expected ({}) ---\n{baseline_text}\n\ + --- actual ---\n{pretty}\n\ + If intentional, re-run with UPDATE_SNAPSHOTS=1.", + baseline_path.display() + ); + } +} + +/// Determinism cross-check: re-running the same pipeline yields the same +/// chunk_ids byte-for-byte. +#[test] +fn code_kotlin_ast_chunks_are_deterministic() { + let policy = fixed_policy(); + let baseline: Vec = CodeKotlinAstV1Chunker + .chunk(&fixed_doc(), &policy) + .unwrap() + .into_iter() + .map(|c| c.chunk_id.0) + .collect(); + for _ in 0..5 { + let again: Vec = CodeKotlinAstV1Chunker + .chunk(&fixed_doc(), &policy) + .unwrap() + .into_iter() + .map(|c| c.chunk_id.0) + .collect(); + assert_eq!(again, baseline); + } +} diff --git a/crates/kebab-chunk/tests/fixtures/code-sample.java.chunks.snapshot.json b/crates/kebab-chunk/tests/fixtures/code-sample.java.chunks.snapshot.json new file mode 100644 index 0000000..e42d8d0 --- /dev/null +++ b/crates/kebab-chunk/tests/fixtures/code-sample.java.chunks.snapshot.json @@ -0,0 +1,170 @@ +[ + { + "block_ids": [ + "03d62d5e6fe70e2b05546e2e65001238" + ], + "chunk_id": "0ed8c548c0ce05f9efab29fe0eaf3d8a", + "chunker_version": "code-java-ast-v1", + "doc_id": "8f387bbe64ae860e73eef14ea38e65b4", + "heading_path": [], + "policy_hash": "78baaa9b8319bd24", + "source_spans": [ + { + "kind": "code", + "lang": "java", + "line_end": 5, + "line_start": 1, + "symbol": "imports" + } + ], + "text": "import java.util.List;\nimport java.util.Map;\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.stream.Collectors;", + "token_estimate": 45 + }, + { + "block_ids": [ + "bbd220978bbe6cf920664d0d4007a1eb" + ], + "chunk_id": "743256121a51a58a1e8006cda750d245", + "chunker_version": "code-java-ast-v1", + "doc_id": "8f387bbe64ae860e73eef14ea38e65b4", + "heading_path": [], + "policy_hash": "78baaa9b8319bd24", + "source_spans": [ + { + "kind": "code", + "lang": "java", + "line_end": 12, + "line_start": 7, + "symbol": "computeMRR" + } + ], + "text": "public static double computeMRR(List scores) {\n if (scores.isEmpty()) {\n return 0.0;\n }\n return 1.0 / scores.size();\n}", + "token_estimate": 48 + }, + { + "block_ids": [ + "484d1abcad06dabb45c81284ad887a25" + ], + "chunk_id": "025d2a911be6362013e15f2f3d52a58f", + "chunker_version": "code-java-ast-v1", + "doc_id": "8f387bbe64ae860e73eef14ea38e65b4", + "heading_path": [], + "policy_hash": "78baaa9b8319bd24", + "source_spans": [ + { + "kind": "code", + "lang": "java", + "line_end": 20, + "line_start": 14, + "symbol": "MetricsCollector" + } + ], + "text": "public class MetricsCollector {\n private List scores;\n private List labels;\n private Map counts;\n private Map totals;\n private List tags;\n}", + "token_estimate": 71 + }, + { + "block_ids": [ + "8b1cb841f509de0ce14425e09bd959ad" + ], + "chunk_id": "38f4f1fc7e6304414005c705829690e0", + "chunker_version": "code-java-ast-v1", + "doc_id": "8f387bbe64ae860e73eef14ea38e65b4", + "heading_path": [], + "policy_hash": "78baaa9b8319bd24", + "source_spans": [ + { + "kind": "code", + "lang": "java", + "line_end": 30, + "line_start": 22, + "symbol": "BaseEvaluator" + } + ], + "text": "public class BaseEvaluator {\n private String name;\n\n public BaseEvaluator(String name) {\n this.name = name;\n }\n\n public void evaluate(List data) throws Exception {\n String joined = String.join(\",\", data);\n }\n}", + "token_estimate": 82 + }, + { + "block_ids": [ + "761545865918f8f5e94aa346c9bb2012" + ], + "chunk_id": "54b178f6df31abbb14ac3975f2cdb649", + "chunker_version": "code-java-ast-v1", + "doc_id": "8f387bbe64ae860e73eef14ea38e65b4", + "heading_path": [], + "policy_hash": "78baaa9b8319bd24", + "source_spans": [ + { + "kind": "code", + "lang": "java", + "line_end": 38, + "line_start": 32, + "symbol": "MetricsCollector.run" + } + ], + "text": "public void run(List inputs) {\n for (Double inp : inputs) {\n scores.add(\n inp\n );\n }\n}", + "token_estimate": 42 + }, + { + "block_ids": [ + "bc2f4d65b9b2d43adbfb352e6e0d4d3c" + ], + "chunk_id": "cb90222c8fc4b1046f6488874335d9e6", + "chunker_version": "code-java-ast-v1", + "doc_id": "8f387bbe64ae860e73eef14ea38e65b4", + "heading_path": [], + "policy_hash": "78baaa9b8319bd24", + "source_spans": [ + { + "kind": "code", + "lang": "java", + "line_end": 46, + "line_start": 40, + "symbol": "MetricsCollector.report" + } + ], + "text": "public Map report() {\n Map result = new HashMap<>();\n result.put(\"mean\", 0.0);\n result.put(\"count\", scores.size());\n result.put(\"tags\", tags);\n return result;\n}", + "token_estimate": 69 + }, + { + "block_ids": [ + "f22ed2da0b1c3d24cc606aebd24bf6d1" + ], + "chunk_id": "b776c1307469ae299694c68dfcc22771", + "chunker_version": "code-java-ast-v1", + "doc_id": "8f387bbe64ae860e73eef14ea38e65b4", + "heading_path": [], + "policy_hash": "78baaa9b8319bd24", + "source_spans": [ + { + "kind": "code", + "lang": "java", + "line_end": 247, + "line_start": 48, + "symbol": "BigCompute [part 1/2]" + } + ], + "text": "public class BigCompute {\n public int compute(int[] data) {\n int v0 = 0 < data.length ? data[0] : 0;\n int v1 = 1 < data.length ? data[1] : 0;\n int v2 = 2 < data.length ? data[2] : 0;\n int v3 = 3 < data.length ? data[3] : 0;\n int v4 = 4 < data.length ? data[4] : 0;\n int v5 = 5 < data.length ? data[5] : 0;\n int v6 = 6 < data.length ? data[6] : 0;\n int v7 = 7 < data.length ? data[7] : 0;\n int v8 = 8 < data.length ? data[8] : 0;\n int v9 = 9 < data.length ? data[9] : 0;\n int v10 = 10 < data.length ? data[10] : 0;\n int v11 = 11 < data.length ? data[11] : 0;\n int v12 = 12 < data.length ? data[12] : 0;\n int v13 = 13 < data.length ? data[13] : 0;\n int v14 = 14 < data.length ? data[14] : 0;\n int v15 = 15 < data.length ? data[15] : 0;\n int v16 = 16 < data.length ? data[16] : 0;\n int v17 = 17 < data.length ? data[17] : 0;\n int v18 = 18 < data.length ? data[18] : 0;\n int v19 = 19 < data.length ? data[19] : 0;\n int v20 = 20 < data.length ? data[20] : 0;\n int v21 = 21 < data.length ? data[21] : 0;\n int v22 = 22 < data.length ? data[22] : 0;\n int v23 = 23 < data.length ? data[23] : 0;\n int v24 = 24 < data.length ? data[24] : 0;\n int v25 = 25 < data.length ? data[25] : 0;\n int v26 = 26 < data.length ? data[26] : 0;\n int v27 = 27 < data.length ? data[27] : 0;\n int v28 = 28 < data.length ? data[28] : 0;\n int v29 = 29 < data.length ? data[29] : 0;\n int v30 = 30 < data.length ? data[30] : 0;\n int v31 = 31 < data.length ? data[31] : 0;\n int v32 = 32 < data.length ? data[32] : 0;\n int v33 = 33 < data.length ? data[33] : 0;\n int v34 = 34 < data.length ? data[34] : 0;\n int v35 = 35 < data.length ? data[35] : 0;\n int v36 = 36 < data.length ? data[36] : 0;\n int v37 = 37 < data.length ? data[37] : 0;\n int v38 = 38 < data.length ? data[38] : 0;\n int v39 = 39 < data.length ? data[39] : 0;\n int v40 = 40 < data.length ? data[40] : 0;\n int v41 = 41 < data.length ? data[41] : 0;\n int v42 = 42 < data.length ? data[42] : 0;\n int v43 = 43 < data.length ? data[43] : 0;\n int v44 = 44 < data.length ? data[44] : 0;\n int v45 = 45 < data.length ? data[45] : 0;\n int v46 = 46 < data.length ? data[46] : 0;\n int v47 = 47 < data.length ? data[47] : 0;\n int v48 = 48 < data.length ? data[48] : 0;\n int v49 = 49 < data.length ? data[49] : 0;\n int v50 = 50 < data.length ? data[50] : 0;\n int v51 = 51 < data.length ? data[51] : 0;\n int v52 = 52 < data.length ? data[52] : 0;\n int v53 = 53 < data.length ? data[53] : 0;\n int v54 = 54 < data.length ? data[54] : 0;\n int v55 = 55 < data.length ? data[55] : 0;\n int v56 = 56 < data.length ? data[56] : 0;\n int v57 = 57 < data.length ? data[57] : 0;\n int v58 = 58 < data.length ? data[58] : 0;\n int v59 = 59 < data.length ? data[59] : 0;\n int v60 = 60 < data.length ? data[60] : 0;\n int v61 = 61 < data.length ? data[61] : 0;\n int v62 = 62 < data.length ? data[62] : 0;\n int v63 = 63 < data.length ? data[63] : 0;\n int v64 = 64 < data.length ? data[64] : 0;\n int v65 = 65 < data.length ? data[65] : 0;\n int v66 = 66 < data.length ? data[66] : 0;\n int v67 = 67 < data.length ? data[67] : 0;\n int v68 = 68 < data.length ? data[68] : 0;\n int v69 = 69 < data.length ? data[69] : 0;\n int v70 = 70 < data.length ? data[70] : 0;\n int v71 = 71 < data.length ? data[71] : 0;\n int v72 = 72 < data.length ? data[72] : 0;\n int v73 = 73 < data.length ? data[73] : 0;\n int v74 = 74 < data.length ? data[74] : 0;\n int v75 = 75 < data.length ? data[75] : 0;\n int v76 = 76 < data.length ? data[76] : 0;\n int v77 = 77 < data.length ? data[77] : 0;\n int v78 = 78 < data.length ? data[78] : 0;\n int v79 = 79 < data.length ? data[79] : 0;\n int v80 = 80 < data.length ? data[80] : 0;\n int v81 = 81 < data.length ? data[81] : 0;\n int v82 = 82 < data.length ? data[82] : 0;\n int v83 = 83 < data.length ? data[83] : 0;\n int v84 = 84 < data.length ? data[84] : 0;\n int v85 = 85 < data.length ? data[85] : 0;\n int v86 = 86 < data.length ? data[86] : 0;\n int v87 = 87 < data.length ? data[87] : 0;\n int v88 = 88 < data.length ? data[88] : 0;\n int v89 = 89 < data.length ? data[89] : 0;\n int v90 = 90 < data.length ? data[90] : 0;\n int v91 = 91 < data.length ? data[91] : 0;\n int v92 = 92 < data.length ? data[92] : 0;\n int v93 = 93 < data.length ? data[93] : 0;\n int v94 = 94 < data.length ? data[94] : 0;\n int v95 = 95 < data.length ? data[95] : 0;\n int v96 = 96 < data.length ? data[96] : 0;\n int v97 = 97 < data.length ? data[97] : 0;\n int v98 = 98 < data.length ? data[98] : 0;\n int v99 = 99 < data.length ? data[99] : 0;\n int v100 = 100 < data.length ? data[100] : 0;\n int v101 = 101 < data.length ? data[101] : 0;\n int v102 = 102 < data.length ? data[102] : 0;\n int v103 = 103 < data.length ? data[103] : 0;\n int v104 = 104 < data.length ? data[104] : 0;\n int v105 = 105 < data.length ? data[105] : 0;\n int v106 = 106 < data.length ? data[106] : 0;\n int v107 = 107 < data.length ? data[107] : 0;\n int v108 = 108 < data.length ? data[108] : 0;\n int v109 = 109 < data.length ? data[109] : 0;\n int v110 = 110 < data.length ? data[110] : 0;\n int v111 = 111 < data.length ? data[111] : 0;\n int v112 = 112 < data.length ? data[112] : 0;\n int v113 = 113 < data.length ? data[113] : 0;\n int v114 = 114 < data.length ? data[114] : 0;\n int v115 = 115 < data.length ? data[115] : 0;\n int v116 = 116 < data.length ? data[116] : 0;\n int v117 = 117 < data.length ? data[117] : 0;\n int v118 = 118 < data.length ? data[118] : 0;\n int v119 = 119 < data.length ? data[119] : 0;\n int v120 = 120 < data.length ? data[120] : 0;\n int v121 = 121 < data.length ? data[121] : 0;\n int v122 = 122 < data.length ? data[122] : 0;\n int v123 = 123 < data.length ? data[123] : 0;\n int v124 = 124 < data.length ? data[124] : 0;\n int v125 = 125 < data.length ? data[125] : 0;\n int v126 = 126 < data.length ? data[126] : 0;\n int v127 = 127 < data.length ? data[127] : 0;\n int v128 = 128 < data.length ? data[128] : 0;\n int v129 = 129 < data.length ? data[129] : 0;\n int v130 = 130 < data.length ? data[130] : 0;\n int v131 = 131 < data.length ? data[131] : 0;\n int v132 = 132 < data.length ? data[132] : 0;\n int v133 = 133 < data.length ? data[133] : 0;\n int v134 = 134 < data.length ? data[134] : 0;\n int v135 = 135 < data.length ? data[135] : 0;\n int v136 = 136 < data.length ? data[136] : 0;\n int v137 = 137 < data.length ? data[137] : 0;\n int v138 = 138 < data.length ? data[138] : 0;\n int v139 = 139 < data.length ? data[139] : 0;\n int v140 = 140 < data.length ? data[140] : 0;\n int v141 = 141 < data.length ? data[141] : 0;\n int v142 = 142 < data.length ? data[142] : 0;\n int v143 = 143 < data.length ? data[143] : 0;\n int v144 = 144 < data.length ? data[144] : 0;\n int v145 = 145 < data.length ? data[145] : 0;\n int v146 = 146 < data.length ? data[146] : 0;\n int v147 = 147 < data.length ? data[147] : 0;\n int v148 = 148 < data.length ? data[148] : 0;\n int v149 = 149 < data.length ? data[149] : 0;\n int v150 = 150 < data.length ? data[150] : 0;\n int v151 = 151 < data.length ? data[151] : 0;\n int v152 = 152 < data.length ? data[152] : 0;\n int v153 = 153 < data.length ? data[153] : 0;\n int v154 = 154 < data.length ? data[154] : 0;\n int v155 = 155 < data.length ? data[155] : 0;\n int v156 = 156 < data.length ? data[156] : 0;\n int v157 = 157 < data.length ? data[157] : 0;\n int v158 = 158 < data.length ? data[158] : 0;\n int v159 = 159 < data.length ? data[159] : 0;\n int v160 = 160 < data.length ? data[160] : 0;\n int v161 = 161 < data.length ? data[161] : 0;\n int v162 = 162 < data.length ? data[162] : 0;\n int v163 = 163 < data.length ? data[163] : 0;\n int v164 = 164 < data.length ? data[164] : 0;\n int v165 = 165 < data.length ? data[165] : 0;\n int v166 = 166 < data.length ? data[166] : 0;\n int v167 = 167 < data.length ? data[167] : 0;\n int v168 = 168 < data.length ? data[168] : 0;\n int v169 = 169 < data.length ? data[169] : 0;\n int v170 = 170 < data.length ? data[170] : 0;\n int v171 = 171 < data.length ? data[171] : 0;\n int v172 = 172 < data.length ? data[172] : 0;\n int v173 = 173 < data.length ? data[173] : 0;\n int v174 = 174 < data.length ? data[174] : 0;\n int v175 = 175 < data.length ? data[175] : 0;\n int v176 = 176 < data.length ? data[176] : 0;\n int v177 = 177 < data.length ? data[177] : 0;\n int v178 = 178 < data.length ? data[178] : 0;\n int v179 = 179 < data.length ? data[179] : 0;\n int v180 = 180 < data.length ? data[180] : 0;\n int v181 = 181 < data.length ? data[181] : 0;\n int v182 = 182 < data.length ? data[182] : 0;\n int v183 = 183 < data.length ? data[183] : 0;\n int v184 = 184 < data.length ? data[184] : 0;\n int v185 = 185 < data.length ? data[185] : 0;\n int v186 = 186 < data.length ? data[186] : 0;\n int v187 = 187 < data.length ? data[187] : 0;\n int v188 = 188 < data.length ? data[188] : 0;\n int v189 = 189 < data.length ? data[189] : 0;\n int v190 = 190 < data.length ? data[190] : 0;\n int v191 = 191 < data.length ? data[191] : 0;\n int v192 = 192 < data.length ? data[192] : 0;\n int v193 = 193 < data.length ? data[193] : 0;\n int v194 = 194 < data.length ? data[194] : 0;\n int v195 = 195 < data.length ? data[195] : 0;\n int v196 = 196 < data.length ? data[196] : 0;\n int v197 = 197 < data.length ? data[197] : 0;", + "token_estimate": 3475 + }, + { + "block_ids": [ + "f22ed2da0b1c3d24cc606aebd24bf6d1" + ], + "chunk_id": "d84568139b72017d6f95c23bf8ba6d81", + "chunker_version": "code-java-ast-v1", + "doc_id": "8f387bbe64ae860e73eef14ea38e65b4", + "heading_path": [], + "policy_hash": "78baaa9b8319bd24", + "source_spans": [ + { + "kind": "code", + "lang": "java", + "line_end": 262, + "line_start": 248, + "symbol": "BigCompute [part 2/2]" + } + ], + "text": " int v198 = 198 < data.length ? data[198] : 0;\n int v199 = 199 < data.length ? data[199] : 0;\n int v200 = 200 < data.length ? data[200] : 0;\n int v201 = 201 < data.length ? data[201] : 0;\n int v202 = 202 < data.length ? data[202] : 0;\n int v203 = 203 < data.length ? data[203] : 0;\n int v204 = 204 < data.length ? data[204] : 0;\n int v205 = 205 < data.length ? data[205] : 0;\n int v206 = 206 < data.length ? data[206] : 0;\n int v207 = 207 < data.length ? data[207] : 0;\n int v208 = 208 < data.length ? data[208] : 0;\n int v209 = 209 < data.length ? data[209] : 0;\n return data.length;\n }\n}", + "token_estimate": 228 + } +] diff --git a/crates/kebab-chunk/tests/fixtures/code-sample.kt.chunks.snapshot.json b/crates/kebab-chunk/tests/fixtures/code-sample.kt.chunks.snapshot.json new file mode 100644 index 0000000..3e046ff --- /dev/null +++ b/crates/kebab-chunk/tests/fixtures/code-sample.kt.chunks.snapshot.json @@ -0,0 +1,170 @@ +[ + { + "block_ids": [ + "d11c97fb8204b59f00fccc5c8b64492e" + ], + "chunk_id": "0dab69cdf08ed0d1028e8067909a056c", + "chunker_version": "code-kotlin-ast-v1", + "doc_id": "e9dfb08b45fd884ec31b25f9dfac6b8f", + "heading_path": [], + "policy_hash": "a8236b5d66bd59a2", + "source_spans": [ + { + "kind": "code", + "lang": "kotlin", + "line_end": 5, + "line_start": 1, + "symbol": "imports" + } + ], + "text": "import kotlin.collections.List\nimport kotlin.collections.Map\nimport kotlin.collections.MutableList\nimport kotlin.collections.MutableMap\nimport kotlin.collections.mutableListOf", + "token_estimate": 59 + }, + { + "block_ids": [ + "8cd5b3ab9657de15405ee3ac5fcf75c1" + ], + "chunk_id": "caaed42871995181992721ec2b5a8c5c", + "chunker_version": "code-kotlin-ast-v1", + "doc_id": "e9dfb08b45fd884ec31b25f9dfac6b8f", + "heading_path": [], + "policy_hash": "a8236b5d66bd59a2", + "source_spans": [ + { + "kind": "code", + "lang": "kotlin", + "line_end": 12, + "line_start": 7, + "symbol": "computeMRR" + } + ], + "text": "fun computeMRR(scores: List): Double {\n if (scores.isEmpty()) {\n return 0.0\n }\n return 1.0 / scores.size\n}", + "token_estimate": 44 + }, + { + "block_ids": [ + "378cc4eede82b166c9b35b0e85b8c62f" + ], + "chunk_id": "7ac5cecfe91de50bdd314391c495f1d1", + "chunker_version": "code-kotlin-ast-v1", + "doc_id": "e9dfb08b45fd884ec31b25f9dfac6b8f", + "heading_path": [], + "policy_hash": "a8236b5d66bd59a2", + "source_spans": [ + { + "kind": "code", + "lang": "kotlin", + "line_end": 20, + "line_start": 14, + "symbol": "MetricsCollector" + } + ], + "text": "data class MetricsCollector(\n val scores: MutableList = mutableListOf(),\n val labels: MutableList = mutableListOf(),\n val counts: MutableMap = mutableMapOf(),\n val totals: MutableMap = mutableMapOf(),\n val tags: MutableList = mutableListOf(),\n)", + "token_estimate": 104 + }, + { + "block_ids": [ + "6f2dd4880b621f9340771a9dccb3b5f1" + ], + "chunk_id": "c6473dc1d96c9f0958632595d1186ef5", + "chunker_version": "code-kotlin-ast-v1", + "doc_id": "e9dfb08b45fd884ec31b25f9dfac6b8f", + "heading_path": [], + "policy_hash": "a8236b5d66bd59a2", + "source_spans": [ + { + "kind": "code", + "lang": "kotlin", + "line_end": 30, + "line_start": 22, + "symbol": "BaseEvaluator" + } + ], + "text": "open class BaseEvaluator(val name: String) {\n\n fun evaluate(data: List) {\n val joined = data.joinToString(\",\")\n println(joined)\n }\n\n open fun describe(): String = name\n}", + "token_estimate": 67 + }, + { + "block_ids": [ + "f6f2b4549b29dc68836950d508a00207" + ], + "chunk_id": "3910815f53944fb3b6cdf307e96c825c", + "chunker_version": "code-kotlin-ast-v1", + "doc_id": "e9dfb08b45fd884ec31b25f9dfac6b8f", + "heading_path": [], + "policy_hash": "a8236b5d66bd59a2", + "source_spans": [ + { + "kind": "code", + "lang": "kotlin", + "line_end": 38, + "line_start": 32, + "symbol": "MetricsCollector.run" + } + ], + "text": "fun MetricsCollector.run(inputs: List) {\n for (inp in inputs) {\n scores.add(\n inp\n )\n }\n}", + "token_estimate": 43 + }, + { + "block_ids": [ + "32386dba97278b43f9c892c8c4e78e9d" + ], + "chunk_id": "fdba90c38bf2ee4bdfe7ad6edbd5c07f", + "chunker_version": "code-kotlin-ast-v1", + "doc_id": "e9dfb08b45fd884ec31b25f9dfac6b8f", + "heading_path": [], + "policy_hash": "a8236b5d66bd59a2", + "source_spans": [ + { + "kind": "code", + "lang": "kotlin", + "line_end": 46, + "line_start": 40, + "symbol": "MetricsCollector.report" + } + ], + "text": "fun MetricsCollector.report(): Map {\n return mapOf(\n \"mean\" to 0.0,\n \"count\" to scores.size,\n \"tags\" to tags,\n )\n}", + "token_estimate": 52 + }, + { + "block_ids": [ + "fbefaf4289794148afd38f22b2e1bd1d" + ], + "chunk_id": "fd98b08ca67b30033c4cc8b0ce8615fa", + "chunker_version": "code-kotlin-ast-v1", + "doc_id": "e9dfb08b45fd884ec31b25f9dfac6b8f", + "heading_path": [], + "policy_hash": "a8236b5d66bd59a2", + "source_spans": [ + { + "kind": "code", + "lang": "kotlin", + "line_end": 247, + "line_start": 48, + "symbol": "BigCompute [part 1/2]" + } + ], + "text": "class BigCompute {\n fun compute(data: IntArray): Int {\n val v0 = if (0 < data.size) data[0] else 0\n val v1 = if (1 < data.size) data[1] else 0\n val v2 = if (2 < data.size) data[2] else 0\n val v3 = if (3 < data.size) data[3] else 0\n val v4 = if (4 < data.size) data[4] else 0\n val v5 = if (5 < data.size) data[5] else 0\n val v6 = if (6 < data.size) data[6] else 0\n val v7 = if (7 < data.size) data[7] else 0\n val v8 = if (8 < data.size) data[8] else 0\n val v9 = if (9 < data.size) data[9] else 0\n val v10 = if (10 < data.size) data[10] else 0\n val v11 = if (11 < data.size) data[11] else 0\n val v12 = if (12 < data.size) data[12] else 0\n val v13 = if (13 < data.size) data[13] else 0\n val v14 = if (14 < data.size) data[14] else 0\n val v15 = if (15 < data.size) data[15] else 0\n val v16 = if (16 < data.size) data[16] else 0\n val v17 = if (17 < data.size) data[17] else 0\n val v18 = if (18 < data.size) data[18] else 0\n val v19 = if (19 < data.size) data[19] else 0\n val v20 = if (20 < data.size) data[20] else 0\n val v21 = if (21 < data.size) data[21] else 0\n val v22 = if (22 < data.size) data[22] else 0\n val v23 = if (23 < data.size) data[23] else 0\n val v24 = if (24 < data.size) data[24] else 0\n val v25 = if (25 < data.size) data[25] else 0\n val v26 = if (26 < data.size) data[26] else 0\n val v27 = if (27 < data.size) data[27] else 0\n val v28 = if (28 < data.size) data[28] else 0\n val v29 = if (29 < data.size) data[29] else 0\n val v30 = if (30 < data.size) data[30] else 0\n val v31 = if (31 < data.size) data[31] else 0\n val v32 = if (32 < data.size) data[32] else 0\n val v33 = if (33 < data.size) data[33] else 0\n val v34 = if (34 < data.size) data[34] else 0\n val v35 = if (35 < data.size) data[35] else 0\n val v36 = if (36 < data.size) data[36] else 0\n val v37 = if (37 < data.size) data[37] else 0\n val v38 = if (38 < data.size) data[38] else 0\n val v39 = if (39 < data.size) data[39] else 0\n val v40 = if (40 < data.size) data[40] else 0\n val v41 = if (41 < data.size) data[41] else 0\n val v42 = if (42 < data.size) data[42] else 0\n val v43 = if (43 < data.size) data[43] else 0\n val v44 = if (44 < data.size) data[44] else 0\n val v45 = if (45 < data.size) data[45] else 0\n val v46 = if (46 < data.size) data[46] else 0\n val v47 = if (47 < data.size) data[47] else 0\n val v48 = if (48 < data.size) data[48] else 0\n val v49 = if (49 < data.size) data[49] else 0\n val v50 = if (50 < data.size) data[50] else 0\n val v51 = if (51 < data.size) data[51] else 0\n val v52 = if (52 < data.size) data[52] else 0\n val v53 = if (53 < data.size) data[53] else 0\n val v54 = if (54 < data.size) data[54] else 0\n val v55 = if (55 < data.size) data[55] else 0\n val v56 = if (56 < data.size) data[56] else 0\n val v57 = if (57 < data.size) data[57] else 0\n val v58 = if (58 < data.size) data[58] else 0\n val v59 = if (59 < data.size) data[59] else 0\n val v60 = if (60 < data.size) data[60] else 0\n val v61 = if (61 < data.size) data[61] else 0\n val v62 = if (62 < data.size) data[62] else 0\n val v63 = if (63 < data.size) data[63] else 0\n val v64 = if (64 < data.size) data[64] else 0\n val v65 = if (65 < data.size) data[65] else 0\n val v66 = if (66 < data.size) data[66] else 0\n val v67 = if (67 < data.size) data[67] else 0\n val v68 = if (68 < data.size) data[68] else 0\n val v69 = if (69 < data.size) data[69] else 0\n val v70 = if (70 < data.size) data[70] else 0\n val v71 = if (71 < data.size) data[71] else 0\n val v72 = if (72 < data.size) data[72] else 0\n val v73 = if (73 < data.size) data[73] else 0\n val v74 = if (74 < data.size) data[74] else 0\n val v75 = if (75 < data.size) data[75] else 0\n val v76 = if (76 < data.size) data[76] else 0\n val v77 = if (77 < data.size) data[77] else 0\n val v78 = if (78 < data.size) data[78] else 0\n val v79 = if (79 < data.size) data[79] else 0\n val v80 = if (80 < data.size) data[80] else 0\n val v81 = if (81 < data.size) data[81] else 0\n val v82 = if (82 < data.size) data[82] else 0\n val v83 = if (83 < data.size) data[83] else 0\n val v84 = if (84 < data.size) data[84] else 0\n val v85 = if (85 < data.size) data[85] else 0\n val v86 = if (86 < data.size) data[86] else 0\n val v87 = if (87 < data.size) data[87] else 0\n val v88 = if (88 < data.size) data[88] else 0\n val v89 = if (89 < data.size) data[89] else 0\n val v90 = if (90 < data.size) data[90] else 0\n val v91 = if (91 < data.size) data[91] else 0\n val v92 = if (92 < data.size) data[92] else 0\n val v93 = if (93 < data.size) data[93] else 0\n val v94 = if (94 < data.size) data[94] else 0\n val v95 = if (95 < data.size) data[95] else 0\n val v96 = if (96 < data.size) data[96] else 0\n val v97 = if (97 < data.size) data[97] else 0\n val v98 = if (98 < data.size) data[98] else 0\n val v99 = if (99 < data.size) data[99] else 0\n val v100 = if (100 < data.size) data[100] else 0\n val v101 = if (101 < data.size) data[101] else 0\n val v102 = if (102 < data.size) data[102] else 0\n val v103 = if (103 < data.size) data[103] else 0\n val v104 = if (104 < data.size) data[104] else 0\n val v105 = if (105 < data.size) data[105] else 0\n val v106 = if (106 < data.size) data[106] else 0\n val v107 = if (107 < data.size) data[107] else 0\n val v108 = if (108 < data.size) data[108] else 0\n val v109 = if (109 < data.size) data[109] else 0\n val v110 = if (110 < data.size) data[110] else 0\n val v111 = if (111 < data.size) data[111] else 0\n val v112 = if (112 < data.size) data[112] else 0\n val v113 = if (113 < data.size) data[113] else 0\n val v114 = if (114 < data.size) data[114] else 0\n val v115 = if (115 < data.size) data[115] else 0\n val v116 = if (116 < data.size) data[116] else 0\n val v117 = if (117 < data.size) data[117] else 0\n val v118 = if (118 < data.size) data[118] else 0\n val v119 = if (119 < data.size) data[119] else 0\n val v120 = if (120 < data.size) data[120] else 0\n val v121 = if (121 < data.size) data[121] else 0\n val v122 = if (122 < data.size) data[122] else 0\n val v123 = if (123 < data.size) data[123] else 0\n val v124 = if (124 < data.size) data[124] else 0\n val v125 = if (125 < data.size) data[125] else 0\n val v126 = if (126 < data.size) data[126] else 0\n val v127 = if (127 < data.size) data[127] else 0\n val v128 = if (128 < data.size) data[128] else 0\n val v129 = if (129 < data.size) data[129] else 0\n val v130 = if (130 < data.size) data[130] else 0\n val v131 = if (131 < data.size) data[131] else 0\n val v132 = if (132 < data.size) data[132] else 0\n val v133 = if (133 < data.size) data[133] else 0\n val v134 = if (134 < data.size) data[134] else 0\n val v135 = if (135 < data.size) data[135] else 0\n val v136 = if (136 < data.size) data[136] else 0\n val v137 = if (137 < data.size) data[137] else 0\n val v138 = if (138 < data.size) data[138] else 0\n val v139 = if (139 < data.size) data[139] else 0\n val v140 = if (140 < data.size) data[140] else 0\n val v141 = if (141 < data.size) data[141] else 0\n val v142 = if (142 < data.size) data[142] else 0\n val v143 = if (143 < data.size) data[143] else 0\n val v144 = if (144 < data.size) data[144] else 0\n val v145 = if (145 < data.size) data[145] else 0\n val v146 = if (146 < data.size) data[146] else 0\n val v147 = if (147 < data.size) data[147] else 0\n val v148 = if (148 < data.size) data[148] else 0\n val v149 = if (149 < data.size) data[149] else 0\n val v150 = if (150 < data.size) data[150] else 0\n val v151 = if (151 < data.size) data[151] else 0\n val v152 = if (152 < data.size) data[152] else 0\n val v153 = if (153 < data.size) data[153] else 0\n val v154 = if (154 < data.size) data[154] else 0\n val v155 = if (155 < data.size) data[155] else 0\n val v156 = if (156 < data.size) data[156] else 0\n val v157 = if (157 < data.size) data[157] else 0\n val v158 = if (158 < data.size) data[158] else 0\n val v159 = if (159 < data.size) data[159] else 0\n val v160 = if (160 < data.size) data[160] else 0\n val v161 = if (161 < data.size) data[161] else 0\n val v162 = if (162 < data.size) data[162] else 0\n val v163 = if (163 < data.size) data[163] else 0\n val v164 = if (164 < data.size) data[164] else 0\n val v165 = if (165 < data.size) data[165] else 0\n val v166 = if (166 < data.size) data[166] else 0\n val v167 = if (167 < data.size) data[167] else 0\n val v168 = if (168 < data.size) data[168] else 0\n val v169 = if (169 < data.size) data[169] else 0\n val v170 = if (170 < data.size) data[170] else 0\n val v171 = if (171 < data.size) data[171] else 0\n val v172 = if (172 < data.size) data[172] else 0\n val v173 = if (173 < data.size) data[173] else 0\n val v174 = if (174 < data.size) data[174] else 0\n val v175 = if (175 < data.size) data[175] else 0\n val v176 = if (176 < data.size) data[176] else 0\n val v177 = if (177 < data.size) data[177] else 0\n val v178 = if (178 < data.size) data[178] else 0\n val v179 = if (179 < data.size) data[179] else 0\n val v180 = if (180 < data.size) data[180] else 0\n val v181 = if (181 < data.size) data[181] else 0\n val v182 = if (182 < data.size) data[182] else 0\n val v183 = if (183 < data.size) data[183] else 0\n val v184 = if (184 < data.size) data[184] else 0\n val v185 = if (185 < data.size) data[185] else 0\n val v186 = if (186 < data.size) data[186] else 0\n val v187 = if (187 < data.size) data[187] else 0\n val v188 = if (188 < data.size) data[188] else 0\n val v189 = if (189 < data.size) data[189] else 0\n val v190 = if (190 < data.size) data[190] else 0\n val v191 = if (191 < data.size) data[191] else 0\n val v192 = if (192 < data.size) data[192] else 0\n val v193 = if (193 < data.size) data[193] else 0\n val v194 = if (194 < data.size) data[194] else 0\n val v195 = if (195 < data.size) data[195] else 0\n val v196 = if (196 < data.size) data[196] else 0\n val v197 = if (197 < data.size) data[197] else 0", + "token_estimate": 3671 + }, + { + "block_ids": [ + "fbefaf4289794148afd38f22b2e1bd1d" + ], + "chunk_id": "e6355c8ab1f97526f0d238f61ba9f25c", + "chunker_version": "code-kotlin-ast-v1", + "doc_id": "e9dfb08b45fd884ec31b25f9dfac6b8f", + "heading_path": [], + "policy_hash": "a8236b5d66bd59a2", + "source_spans": [ + { + "kind": "code", + "lang": "kotlin", + "line_end": 262, + "line_start": 248, + "symbol": "BigCompute [part 2/2]" + } + ], + "text": " val v198 = if (198 < data.size) data[198] else 0\n val v199 = if (199 < data.size) data[199] else 0\n val v200 = if (200 < data.size) data[200] else 0\n val v201 = if (201 < data.size) data[201] else 0\n val v202 = if (202 < data.size) data[202] else 0\n val v203 = if (203 < data.size) data[203] else 0\n val v204 = if (204 < data.size) data[204] else 0\n val v205 = if (205 < data.size) data[205] else 0\n val v206 = if (206 < data.size) data[206] else 0\n val v207 = if (207 < data.size) data[207] else 0\n val v208 = if (208 < data.size) data[208] else 0\n val v209 = if (209 < data.size) data[209] else 0\n return data.size\n }\n}", + "token_estimate": 239 + } +] diff --git a/crates/kebab-parse-code/Cargo.toml b/crates/kebab-parse-code/Cargo.toml index 4698357..caaceaf 100644 --- a/crates/kebab-parse-code/Cargo.toml +++ b/crates/kebab-parse-code/Cargo.toml @@ -20,6 +20,8 @@ tree-sitter-python = { workspace = true } tree-sitter-typescript = { workspace = true } tree-sitter-javascript = { workspace = true } tree-sitter-go = { workspace = true } +tree-sitter-java = { workspace = true } +tree-sitter-kotlin-ng = { workspace = true } [dev-dependencies] tempfile = { workspace = true } diff --git a/crates/kebab-parse-code/src/java.rs b/crates/kebab-parse-code/src/java.rs new file mode 100644 index 0000000..c9eaeb7 --- /dev/null +++ b/crates/kebab-parse-code/src/java.rs @@ -0,0 +1,543 @@ +//! `kebab-parse-code::java` — tree-sitter Java AST extractor (P10-1C-JK Task D). +//! +//! Implements [`kebab_core::Extractor`] for [`MediaType::Code("java")`]. +//! Walks the tree-sitter parse tree and emits one [`Block::Code`] per +//! top-level AST semantic unit (class / interface / enum / record / +//! annotation-type at any nesting level, plus methods + constructors +//! inside class / interface / record bodies), each carrying +//! [`SourceSpan::Code`] with the unit's dotted self-reference symbol +//! path (design §3.4 Java row). Glue declarations (`import`) collapse +//! into one grouped `` (or ``) unit. +//! +//! Like the Go extractor, Java's package identity comes from the +//! source itself (the `package_declaration` clause), not from the +//! workspace file path — `extract_package` reads it from the AST. If +//! the clause is missing the prefix falls back to `""`. +//! +//! Class/interface/record bodies are recursed (1B Python pattern): +//! the type name is pushed onto `mod_path` so methods and nested +//! types become `...`. Constructors use +//! the Java convention `.<...>..` (name +//! duplicated, per design §3.4). Enum bodies are not recursed for +//! the 1차 cut — enum constants are not emitted as units. +//! +//! Javadoc (`/** ... */` → `block_comment`) and line comments +//! immediately preceding an item are folded into that item's line +//! range via `unit_start` (1B pattern). Annotations are children of +//! the declaration node itself (inside `modifiers`), so they are +//! already part of the declaration's span — no separate unwrap arm. +//! +//! Per design §3.4 / §9.1 / §9 versioning. + +use anyhow::Result; +use kebab_core::{ + Block, CanonicalDocument, CodeBlock, CommonBlock, Extractor, Lang, MediaType, Metadata, + ParserVersion, Provenance, ProvenanceEvent, ProvenanceKind, SourceSpan, SourceType, TrustLevel, + id_for_block, id_for_doc, +}; +use serde_json::Map; +use time::OffsetDateTime; + +use crate::scaffold::{filename_from_workspace_path, join_symbol, strip_extension}; + +pub const PARSER_VERSION: &str = "code-java-v1"; + +/// Java AST extractor. Per-unit blocks via tree-sitter-java 0.23 +/// (`LANGUAGE: LanguageFn`) parsed by tree-sitter 0.26. +pub struct JavaAstExtractor; + +impl JavaAstExtractor { + pub fn new() -> Self { + Self + } +} + +impl Default for JavaAstExtractor { + fn default() -> Self { + Self::new() + } +} + +impl Extractor for JavaAstExtractor { + fn supports(&self, m: &MediaType) -> bool { + matches!(m, MediaType::Code(l) if l == "java") + } + + fn parser_version(&self) -> ParserVersion { + ParserVersion(PARSER_VERSION.to_string()) + } + + fn extract( + &self, + ctx: &kebab_core::ExtractContext<'_>, + bytes: &[u8], + ) -> Result { + let asset = ctx.asset; + if !self.supports(&asset.media_type) { + anyhow::bail!( + "kebab-parse-code: unsupported media_type for JavaAstExtractor: {:?}", + asset.media_type + ); + } + + let parser_version = self.parser_version(); + let doc_id = id_for_doc(&asset.workspace_path, &asset.asset_id, &parser_version); + + let source = String::from_utf8(bytes.to_vec()) + .map_err(|e| anyhow::anyhow!("kebab-parse-code: Java source is not valid UTF-8: {e}"))?; + + let blocks = build_blocks(&source, &doc_id)?; + let unit_count = blocks.len() as u32; + + let now = OffsetDateTime::now_utc(); + let mut events: Vec = Vec::with_capacity(2); + events.push(ProvenanceEvent { + at: asset.discovered_at, + agent: "kb-source-fs".to_string(), + kind: ProvenanceKind::Discovered, + note: None, + }); + events.push(ProvenanceEvent { + at: now, + agent: "kb-parse-code".to_string(), + kind: ProvenanceKind::Parsed, + note: Some(format!( + "parser_version={}; unit_count={}", + parser_version.0, unit_count + )), + }); + + let title = { + let fname = filename_from_workspace_path(&asset.workspace_path.0); + strip_extension(&fname) + }; + + // Resolve the file's absolute path for repo detection. If the + // source URI carries a relative path, anchor it at the workspace + // root so the `.git/` walk-up starts from the right place. + let abs_path = match &asset.source_uri { + kebab_core::SourceUri::File(p) => { + if p.is_absolute() { + p.clone() + } else { + ctx.workspace_root.join(p) + } + } + kebab_core::SourceUri::Kb(_) => ctx.workspace_root.to_path_buf(), + }; + let (repo, git_branch, git_commit) = match crate::repo::detect_repo(&abs_path) { + Some(r) => (Some(r.name), r.branch, r.commit), + None => (None, None, None), + }; + + let metadata = Metadata { + aliases: Vec::new(), + tags: Vec::new(), + created_at: asset.discovered_at, + updated_at: asset.discovered_at, + source_type: SourceType::Note, + trust_level: TrustLevel::Primary, + user_id_alias: None, + user: Map::new(), + repo, + git_branch, + git_commit, + code_lang: Some("java".to_string()), + }; + + tracing::debug!( + target: "kebab-parse-code", + "extracted Java doc_id={} workspace_path={} units={}", + doc_id.0, + asset.workspace_path.0, + unit_count + ); + + Ok(CanonicalDocument { + doc_id, + source_asset_id: asset.asset_id.clone(), + workspace_path: asset.workspace_path.clone(), + title, + lang: Lang("und".to_string()), + blocks, + metadata, + provenance: Provenance { events }, + parser_version, + schema_version: 1, + doc_version: 1, + last_chunker_version: None, + last_embedding_version: None, + }) + } +} + +/// p10-1C-JK: extract `package` declaration text from a tree-sitter-java +/// `program`. Returns `None` if no `package_declaration` (default-package +/// Java file). The package_declaration's named children are either a +/// single `identifier` (single-segment package, rare) or a +/// `scoped_identifier` (dotted, common). Per design §3.4 Java row. +fn extract_package(root: tree_sitter::Node, src: &str) -> Option { + let mut cur = root.walk(); + for child in root.named_children(&mut cur) { + if child.kind() == "package_declaration" { + let mut c2 = child.walk(); + for sub in child.named_children(&mut c2) { + if sub.kind() == "scoped_identifier" || sub.kind() == "identifier" { + return Some(src[sub.start_byte()..sub.end_byte()].to_string()); + } + } + } + } + None +} + +/// Walk preceding `line_comment` / `block_comment` siblings to extend +/// the unit's line range upward, folding leading Javadoc / line +/// comments into the unit. Annotations live INSIDE `modifiers` on the +/// declaration node itself, so their lines are already inside +/// `n.start_position()` — no separate unwrap arm is needed for them. +fn unit_start(n: &tree_sitter::Node) -> u32 { + let mut start = n.start_position().row as u32 + 1; + let mut prev = n.prev_sibling(); + while let Some(p) = prev { + let k = p.kind(); + if k == "line_comment" || k == "block_comment" { + start = p.start_position().row as u32 + 1; + prev = p.prev_sibling(); + } else { + break; + } + } + start +} + +fn node_name_text<'a>(n: &tree_sitter::Node, src: &'a str) -> Option<&'a str> { + n.child_by_field_name("name") + .map(|c| &src[c.start_byte()..c.end_byte()]) +} + +fn build_blocks( + source: &str, + doc_id: &kebab_core::DocumentId, +) -> anyhow::Result> { + let mut parser = tree_sitter::Parser::new(); + parser + .set_language(&tree_sitter_java::LANGUAGE.into()) + .map_err(|e| anyhow::anyhow!("set tree-sitter-java language: {e}"))?; + let tree = parser + .parse(source.as_bytes(), None) + .ok_or_else(|| anyhow::anyhow!("tree-sitter failed to parse Java source"))?; + let lines: Vec<&str> = source.split('\n').collect(); + + let root = tree.root_node(); + let mod_prefix = extract_package(root, source).unwrap_or_else(|| "".to_string()); + + // units: (symbol, line_start, line_end, is_real_semantic_unit). + // Glue groups are pushed with a sentinel symbol + is_real=false so a + // post-pass can decide `` vs `` (1B/1C-Go pattern). + let mut units: Vec<(String, u32, u32, bool)> = Vec::new(); + // (is_import 0/1, s, e). `is_import` flags `import_declaration` — + // used by the glue flush to pick `` vs `` + // provisional label. + let mut glue: Vec<(usize, u32, u32)> = Vec::new(); + + walk_top(root, source, &mod_prefix, &mut units, &mut glue); + + // `` is correct only when the file produced no real unit. + // Otherwise the import-only group becomes `` (same + // post-pass as 1B / 1C-Go). + let has_real_unit = units.iter().any(|(_, _, _, is_real)| *is_real); + if has_real_unit { + for (sym, _, _, is_real) in units.iter_mut() { + if !*is_real && sym.ends_with("") { + let pre = &sym[..sym.len() - "".len()]; + *sym = format!("{pre}"); + } + } + } + + let total_lines = lines.len() as u32; + let mut blocks = Vec::with_capacity(units.len()); + for (ordinal, (symbol, ls, le, _is_real)) in units.into_iter().enumerate() { + let line_start = ls.max(1); + let line_end = le.min(total_lines.max(1)); + let span = SourceSpan::Code { + line_start, + line_end, + symbol: Some(symbol), + lang: Some("java".to_string()), + }; + let block_id = id_for_block(doc_id, "code", &[], ordinal as u32, &span); + let code = lines[(line_start as usize - 1)..=(line_end as usize - 1)].join("\n"); + blocks.push(Block::Code(CodeBlock { + common: CommonBlock { + block_id, + heading_path: Vec::new(), + source_span: span, + }, + lang: Some("java".to_string()), + code, + })); + } + Ok(blocks) +} + +/// Walk the file's top-level children — `program` named children: +/// `package_declaration` (handled by `extract_package`), `import_declaration` +/// (glue), and the five type declarations (`class` / `interface` / +/// `enum` / `record` / `annotation_type`). Type-declaration bodies +/// are recursed via [`walk_body`] with the type name pushed onto +/// `mod_path` (1B Python pattern). Enum bodies are NOT recursed +/// (1차 cut — see module-level doc). +fn walk_top( + node: tree_sitter::Node, + src: &str, + mod_prefix: &str, + units: &mut Vec<(String, u32, u32, bool)>, + glue: &mut Vec<(usize, u32, u32)>, +) { + let mod_path: &[String] = &[]; + let mut cur = node.walk(); + for child in node.named_children(&mut cur) { + let s = unit_start(&child); + let e = child.end_position().row as u32 + 1; + match child.kind() { + "class_declaration" + | "interface_declaration" + | "record_declaration" => { + if let Some(name) = node_name_text(&child, src) { + glue.retain(|(_, gs, _)| *gs < s); + flush_glue(glue, units, mod_prefix, mod_path); + let sym = join_symbol(mod_prefix, mod_path, name); + units.push((sym, s, e, true)); + if let Some(body) = child.child_by_field_name("body") { + let np: Vec = vec![name.to_string()]; + walk_body(body, src, mod_prefix, &np, units); + } + } + } + "enum_declaration" => { + if let Some(name) = node_name_text(&child, src) { + glue.retain(|(_, gs, _)| *gs < s); + flush_glue(glue, units, mod_prefix, mod_path); + let sym = join_symbol(mod_prefix, mod_path, name); + units.push((sym, s, e, true)); + // Enum body NOT recursed for 1차 — enum constants are + // not emitted as units, and method declarations inside + // enum bodies (rare) live under `enum_body_declarations` + // not `class_body`. Skip per design §3.4 1차 scope. + } + } + "annotation_type_declaration" => { + if let Some(name) = node_name_text(&child, src) { + glue.retain(|(_, gs, _)| *gs < s); + flush_glue(glue, units, mod_prefix, mod_path); + let sym = join_symbol(mod_prefix, mod_path, name); + units.push((sym, s, e, true)); + } + } + "import_declaration" => { + glue.push((1, s, e)); + } + // package_declaration is handled by `extract_package`; no + // glue entry — it's structural metadata, not a unit. + _ => {} + } + } + flush_glue(glue, units, mod_prefix, mod_path); +} + +/// Walk a `class_body` / `interface_body` (or record's `class_body`). +/// Emits one unit per method / constructor, and recurses into nested +/// type declarations. Field declarations are NOT emitted (would +/// explode unit count). `compact_constructor_declaration` (records) +/// is handled the same as `constructor_declaration`. +/// +/// No `glue` parameter: Java does not have imports inside type +/// bodies — they only appear at file top level, handled by +/// [`walk_top`]. +fn walk_body( + body: tree_sitter::Node, + src: &str, + mod_prefix: &str, + mod_path: &[String], + units: &mut Vec<(String, u32, u32, bool)>, +) { + let mut cur = body.walk(); + for child in body.named_children(&mut cur) { + let s = unit_start(&child); + let e = child.end_position().row as u32 + 1; + match child.kind() { + "method_declaration" + | "constructor_declaration" + | "compact_constructor_declaration" => { + // Constructor: name field equals the class name. Per + // design §3.4 Java convention, symbol is + // `..` with the constructor + // name (== class name) as the trailing segment. This + // means the symbol duplicates the class name (e.g. + // `com.x.Foo.Foo`), which is the documented convention. + if let Some(name) = node_name_text(&child, src) { + let sym = join_symbol(mod_prefix, mod_path, name); + units.push((sym, s, e, true)); + } + } + "class_declaration" + | "interface_declaration" + | "record_declaration" + | "enum_declaration" + | "annotation_type_declaration" => { + // Nested type — emit unit, then recurse into its body + // (skipped for enum + annotation_type per 1차 scope). + let name = match node_name_text(&child, src) { + Some(n) => n, + None => continue, + }; + let sym = join_symbol(mod_prefix, mod_path, name); + units.push((sym, s, e, true)); + if child.kind() != "enum_declaration" + && child.kind() != "annotation_type_declaration" + { + if let Some(inner_body) = child.child_by_field_name("body") { + let mut np = mod_path.to_vec(); + np.push(name.to_string()); + walk_body(inner_body, src, mod_prefix, &np, units); + } + } + } + // field_declaration, static_initializer, block: NOT emitted. + _ => {} + } + } +} + +fn flush_glue( + glue: &mut Vec<(usize, u32, u32)>, + units: &mut Vec<(String, u32, u32, bool)>, + mod_prefix: &str, + mod_path: &[String], +) { + if glue.is_empty() { + return; + } + let s = glue.iter().map(|(_, a, _)| *a).min().unwrap(); + let e = glue.iter().map(|(_, _, b)| *b).max().unwrap(); + // Provisional label: `` only if the group is exclusively + // imports (1A's `only_mod_decls` analog). The post-pass demotes any + // `` to `` if the file produced any real unit. + let only_imports = glue.iter().all(|(is_import, _, _)| *is_import == 1); + let label = if only_imports { "" } else { "" }; + units.push((join_symbol(mod_prefix, mod_path, label), s, e, false)); + glue.clear(); +} + +#[cfg(test)] +mod tests { + use super::*; + use kebab_core::{Block, MediaType, SourceSpan}; + + fn extract_fixture() -> kebab_core::CanonicalDocument { + let bytes = std::fs::read(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/sample.java" + )) + .unwrap(); + let asset = + crate::rust::tests_support::fixed_code_asset("crates/x/src/sample.java", "java"); + let cfg = kebab_core::ExtractConfig::default(); + let root = std::path::PathBuf::from("/tmp"); + let ctx = kebab_core::ExtractContext { + asset: &asset, + workspace_root: &root, + config: &cfg, + }; + JavaAstExtractor::new().extract(&ctx, &bytes).unwrap() + } + + #[test] + fn extractor_supports_only_media_code_java() { + let e = JavaAstExtractor::new(); + assert!(e.supports(&MediaType::Code("java".into()))); + assert!(!e.supports(&MediaType::Code("rust".into()))); + assert!(!e.supports(&MediaType::Markdown)); + } + + #[test] + fn java_units_match_design_3_4_symbols() { + let doc = extract_fixture(); + let mut syms: Vec = doc + .blocks + .iter() + .filter_map(|b| match b { + Block::Code(c) => match &c.common.source_span { + SourceSpan::Code { symbol, lang, .. } => { + assert_eq!(lang.as_deref(), Some("java")); + symbol.clone() + } + _ => None, + }, + _ => None, + }) + .collect(); + syms.sort(); + // package extracted from source = com.kebab.chunk + assert!( + syms.iter().any(|s| s == "com.kebab.chunk.MdHeadingV1Chunker"), + "got {syms:?}" + ); + // constructor — Java convention is class-name-as-method-name + assert!( + syms.iter() + .any(|s| s == "com.kebab.chunk.MdHeadingV1Chunker.MdHeadingV1Chunker"), + "got {syms:?}" + ); + assert!( + syms.iter() + .any(|s| s == "com.kebab.chunk.MdHeadingV1Chunker.chunkDoc"), + "got {syms:?}" + ); + assert!( + syms.iter() + .any(|s| s == "com.kebab.chunk.MdHeadingV1Chunker.getName"), + "got {syms:?}" + ); + // static nested class + assert!( + syms.iter() + .any(|s| s == "com.kebab.chunk.MdHeadingV1Chunker.Builder"), + "got {syms:?}" + ); + assert!( + syms.iter() + .any(|s| s == "com.kebab.chunk.MdHeadingV1Chunker.Builder.withName"), + "got {syms:?}" + ); + assert!( + syms.iter() + .any(|s| s == "com.kebab.chunk.MdHeadingV1Chunker.Builder.build"), + "got {syms:?}" + ); + // package-private interface + enum + assert!( + syms.iter().any(|s| s == "com.kebab.chunk.Stringer"), + "got {syms:?}" + ); + assert!( + syms.iter().any(|s| s == "com.kebab.chunk.Mode"), + "got {syms:?}" + ); + // import grouped as + assert!( + syms.iter().any(|s| s == "com.kebab.chunk."), + "got {syms:?}" + ); + } + + #[test] + fn deterministic_across_runs() { + let a = extract_fixture(); + for _ in 0..50 { + assert_eq!(extract_fixture().blocks, a.blocks); + } + } +} diff --git a/crates/kebab-parse-code/src/kotlin.rs b/crates/kebab-parse-code/src/kotlin.rs new file mode 100644 index 0000000..0e6d5d3 --- /dev/null +++ b/crates/kebab-parse-code/src/kotlin.rs @@ -0,0 +1,627 @@ +//! `kebab-parse-code::kotlin` — tree-sitter Kotlin AST extractor (P10-1C-JK Task G). +//! +//! Implements [`kebab_core::Extractor`] for [`MediaType::Code("kotlin")`]. +//! Mirrors the Java extractor (JVM family, source-side `package` extraction + +//! class-nesting) with Kotlin-specific adjustments: +//! +//! * Root is `source_file` (not `program`). +//! * `package_header` carries a single `qualified_identifier` child whose +//! slice text IS the dotted package path — never a bare `identifier` +//! sub-form for the package (the grammar always wraps a single segment +//! in `qualified_identifier` too). +//! * `class_declaration` covers `class`, `data class`, `sealed class`, +//! `enum class`, AND `interface` — Kotlin uses ONE node kind with a +//! `modifiers` child rather than separate `interface_declaration` / +//! `enum_declaration` nodes (verified via tree-sitter-kotlin-ng +//! `node-types.json`). +//! * The body child of `class_declaration` is either `class_body` (normal +//! classes / interfaces) OR `enum_class_body` (enum class). Neither +//! carries a `body` field name, so it is matched by kind, not by +//! `child_by_field_name("body")`. +//! * `companion_object` is a SEPARATE node kind (not `object_declaration` +//! with a modifier). Its `name` field is OPTIONAL — when omitted (the +//! common case `companion object { ... }`) the symbol uses the +//! implicit Kotlin convention name `Companion`. +//! * `object_declaration` (named singleton) carries a `name` field and a +//! `class_body` child. +//! * `function_declaration` may appear at top level (Kotlin top-level +//! function) AND inside `class_body` — same node kind, the +//! `mod_path` state distinguishes the two emit forms. +//! +//! Enum bodies (`enum_class_body`) are NOT recursed for the 1차 cut — +//! `enum_entry` declarations are not emitted as units, matching the +//! Java extractor's enum policy (design §3.4 1차 scope). +//! +//! Per design §3.4 / §9.1 / §9 versioning. + +use anyhow::Result; +use kebab_core::{ + Block, CanonicalDocument, CodeBlock, CommonBlock, Extractor, Lang, MediaType, Metadata, + ParserVersion, Provenance, ProvenanceEvent, ProvenanceKind, SourceSpan, SourceType, TrustLevel, + id_for_block, id_for_doc, +}; +use serde_json::Map; +use time::OffsetDateTime; + +use crate::scaffold::{filename_from_workspace_path, join_symbol, strip_extension}; + +pub const PARSER_VERSION: &str = "code-kotlin-v1"; + +/// Kotlin AST extractor. Per-unit blocks via tree-sitter-kotlin-ng 1.1 +/// (`LANGUAGE: LanguageFn`) parsed by tree-sitter 0.26. +pub struct KotlinAstExtractor; + +impl KotlinAstExtractor { + pub fn new() -> Self { + Self + } +} + +impl Default for KotlinAstExtractor { + fn default() -> Self { + Self::new() + } +} + +impl Extractor for KotlinAstExtractor { + fn supports(&self, m: &MediaType) -> bool { + matches!(m, MediaType::Code(l) if l == "kotlin") + } + + fn parser_version(&self) -> ParserVersion { + ParserVersion(PARSER_VERSION.to_string()) + } + + fn extract( + &self, + ctx: &kebab_core::ExtractContext<'_>, + bytes: &[u8], + ) -> Result { + let asset = ctx.asset; + if !self.supports(&asset.media_type) { + anyhow::bail!( + "kebab-parse-code: unsupported media_type for KotlinAstExtractor: {:?}", + asset.media_type + ); + } + + let parser_version = self.parser_version(); + let doc_id = id_for_doc(&asset.workspace_path, &asset.asset_id, &parser_version); + + let source = String::from_utf8(bytes.to_vec()).map_err(|e| { + anyhow::anyhow!("kebab-parse-code: Kotlin source is not valid UTF-8: {e}") + })?; + + let blocks = build_blocks(&source, &doc_id)?; + let unit_count = blocks.len() as u32; + + let now = OffsetDateTime::now_utc(); + let mut events: Vec = Vec::with_capacity(2); + events.push(ProvenanceEvent { + at: asset.discovered_at, + agent: "kb-source-fs".to_string(), + kind: ProvenanceKind::Discovered, + note: None, + }); + events.push(ProvenanceEvent { + at: now, + agent: "kb-parse-code".to_string(), + kind: ProvenanceKind::Parsed, + note: Some(format!( + "parser_version={}; unit_count={}", + parser_version.0, unit_count + )), + }); + + let title = { + let fname = filename_from_workspace_path(&asset.workspace_path.0); + strip_extension(&fname) + }; + + // Resolve the file's absolute path for repo detection. If the + // source URI carries a relative path, anchor it at the workspace + // root so the `.git/` walk-up starts from the right place. + let abs_path = match &asset.source_uri { + kebab_core::SourceUri::File(p) => { + if p.is_absolute() { + p.clone() + } else { + ctx.workspace_root.join(p) + } + } + kebab_core::SourceUri::Kb(_) => ctx.workspace_root.to_path_buf(), + }; + let (repo, git_branch, git_commit) = match crate::repo::detect_repo(&abs_path) { + Some(r) => (Some(r.name), r.branch, r.commit), + None => (None, None, None), + }; + + let metadata = Metadata { + aliases: Vec::new(), + tags: Vec::new(), + created_at: asset.discovered_at, + updated_at: asset.discovered_at, + source_type: SourceType::Note, + trust_level: TrustLevel::Primary, + user_id_alias: None, + user: Map::new(), + repo, + git_branch, + git_commit, + code_lang: Some("kotlin".to_string()), + }; + + tracing::debug!( + target: "kebab-parse-code", + "extracted Kotlin doc_id={} workspace_path={} units={}", + doc_id.0, + asset.workspace_path.0, + unit_count + ); + + Ok(CanonicalDocument { + doc_id, + source_asset_id: asset.asset_id.clone(), + workspace_path: asset.workspace_path.clone(), + title, + lang: Lang("und".to_string()), + blocks, + metadata, + provenance: Provenance { events }, + parser_version, + schema_version: 1, + doc_version: 1, + last_chunker_version: None, + last_embedding_version: None, + }) + } +} + +/// p10-1C-JK: extract `package` declaration text from a tree-sitter-kotlin +/// `source_file`. Returns `None` if no `package_header` (default-package +/// Kotlin file). The package_header's single named child is a +/// `qualified_identifier`; its slice text is the dotted path. Per design +/// §3.4 Kotlin row. +fn extract_package(root: tree_sitter::Node, src: &str) -> Option { + let mut cur = root.walk(); + for child in root.named_children(&mut cur) { + if child.kind() == "package_header" { + let mut c2 = child.walk(); + for sub in child.named_children(&mut c2) { + let k = sub.kind(); + if k == "qualified_identifier" || k == "identifier" { + return Some(src[sub.start_byte()..sub.end_byte()].to_string()); + } + } + } + } + None +} + +/// Walk preceding `line_comment` / `block_comment` siblings to extend +/// the unit's line range upward, folding leading KDoc / line comments +/// into the unit. Modifiers / annotations live INSIDE the declaration +/// node itself, so their lines are already inside `n.start_position()`. +fn unit_start(n: &tree_sitter::Node) -> u32 { + let mut start = n.start_position().row as u32 + 1; + let mut prev = n.prev_sibling(); + while let Some(p) = prev { + let k = p.kind(); + if k == "line_comment" || k == "block_comment" { + start = p.start_position().row as u32 + 1; + prev = p.prev_sibling(); + } else { + break; + } + } + start +} + +fn node_name_text<'a>(n: &tree_sitter::Node, src: &'a str) -> Option<&'a str> { + n.child_by_field_name("name") + .map(|c| &src[c.start_byte()..c.end_byte()]) +} + +/// Find the first child of a node with one of the given kinds. Used to +/// locate `class_body` / `enum_class_body` on `class_declaration` since +/// the kotlin grammar attaches them without a `body` field name. +fn first_child_of_kinds<'a>( + n: &tree_sitter::Node<'a>, + kinds: &[&str], +) -> Option> { + let mut cur = n.walk(); + n.named_children(&mut cur) + .find(|child| kinds.contains(&child.kind())) +} + +/// `true` iff a `class_declaration` carries the `enum` class modifier. +/// Detected by walking `modifiers` → `class_modifier` and checking the +/// child text. The grammar exposes "enum" / "sealed" / "data" / +/// "annotation" / "inner" as named `class_modifier` children of +/// `modifiers`. We only need to know about "enum" to decide whether to +/// look for `class_body` or `enum_class_body` and whether to skip body +/// recursion. +fn class_decl_is_enum(n: &tree_sitter::Node, src: &str) -> bool { + let mut cur = n.walk(); + for child in n.named_children(&mut cur) { + if child.kind() == "modifiers" { + let mut c2 = child.walk(); + for sub in child.named_children(&mut c2) { + if sub.kind() == "class_modifier" { + let text = &src[sub.start_byte()..sub.end_byte()]; + if text == "enum" { + return true; + } + } + } + } + } + false +} + +fn build_blocks( + source: &str, + doc_id: &kebab_core::DocumentId, +) -> anyhow::Result> { + let mut parser = tree_sitter::Parser::new(); + parser + .set_language(&tree_sitter_kotlin_ng::LANGUAGE.into()) + .map_err(|e| anyhow::anyhow!("set tree-sitter-kotlin-ng language: {e}"))?; + let tree = parser + .parse(source.as_bytes(), None) + .ok_or_else(|| anyhow::anyhow!("tree-sitter failed to parse Kotlin source"))?; + let lines: Vec<&str> = source.split('\n').collect(); + + let root = tree.root_node(); + let mod_prefix = extract_package(root, source).unwrap_or_else(|| "".to_string()); + + // units: (symbol, line_start, line_end, is_real_semantic_unit). + // Glue groups are pushed with a sentinel symbol + is_real=false so a + // post-pass can decide `` vs `` (JVM family pattern). + let mut units: Vec<(String, u32, u32, bool)> = Vec::new(); + // (is_import 0/1, s, e). `is_import` flags `import` — used by the + // glue flush to pick `` vs `` provisional label. + let mut glue: Vec<(usize, u32, u32)> = Vec::new(); + + walk_top(root, source, &mod_prefix, &mut units, &mut glue); + + // `` is correct only when the file produced no real unit. + // Otherwise the import-only group becomes `` (same + // post-pass as 1B / 1C-Go / Java). + let has_real_unit = units.iter().any(|(_, _, _, is_real)| *is_real); + if has_real_unit { + for (sym, _, _, is_real) in units.iter_mut() { + if !*is_real && sym.ends_with("") { + let pre = &sym[..sym.len() - "".len()]; + *sym = format!("{pre}"); + } + } + } + + let total_lines = lines.len() as u32; + let mut blocks = Vec::with_capacity(units.len()); + for (ordinal, (symbol, ls, le, _is_real)) in units.into_iter().enumerate() { + let line_start = ls.max(1); + let line_end = le.min(total_lines.max(1)); + let span = SourceSpan::Code { + line_start, + line_end, + symbol: Some(symbol), + lang: Some("kotlin".to_string()), + }; + let block_id = id_for_block(doc_id, "code", &[], ordinal as u32, &span); + let code = lines[(line_start as usize - 1)..=(line_end as usize - 1)].join("\n"); + blocks.push(Block::Code(CodeBlock { + common: CommonBlock { + block_id, + heading_path: Vec::new(), + source_span: span, + }, + lang: Some("kotlin".to_string()), + code, + })); + } + Ok(blocks) +} + +/// Walk the file's top-level children — `source_file` named children: +/// `package_header` (handled by `extract_package`), `import` (glue), +/// `class_declaration` (class / interface / enum class), `object_declaration`, +/// `function_declaration` (top-level), `property_declaration` (top-level), +/// `type_alias` (currently treated as glue). Class / object bodies are +/// recursed via [`walk_body`] with the type name pushed onto `mod_path` +/// (JVM family pattern). Enum bodies are NOT recursed (1차 cut). +fn walk_top( + node: tree_sitter::Node, + src: &str, + mod_prefix: &str, + units: &mut Vec<(String, u32, u32, bool)>, + glue: &mut Vec<(usize, u32, u32)>, +) { + let mod_path: &[String] = &[]; + let mut cur = node.walk(); + for child in node.named_children(&mut cur) { + let s = unit_start(&child); + let e = child.end_position().row as u32 + 1; + match child.kind() { + "class_declaration" => { + // Covers class / data class / sealed class / interface / + // enum class — single grammar node, the modifiers child + // distinguishes them. The body is `class_body` for + // non-enum and `enum_class_body` for enum class; both + // attach without a `body` field name. + if let Some(name) = node_name_text(&child, src) { + glue.retain(|(_, gs, _)| *gs < s); + flush_glue(glue, units, mod_prefix, mod_path); + let sym = join_symbol(mod_prefix, mod_path, name); + units.push((sym, s, e, true)); + let is_enum = class_decl_is_enum(&child, src); + if !is_enum { + if let Some(body) = first_child_of_kinds(&child, &["class_body"]) { + let np: Vec = vec![name.to_string()]; + walk_body(body, src, mod_prefix, &np, units); + } + } + // enum_class_body NOT recursed — enum constants are + // not emitted as units (1차 scope, matches Java). + } + } + "object_declaration" => { + // Singleton object — name field is required by the grammar. + if let Some(name) = node_name_text(&child, src) { + glue.retain(|(_, gs, _)| *gs < s); + flush_glue(glue, units, mod_prefix, mod_path); + let sym = join_symbol(mod_prefix, mod_path, name); + units.push((sym, s, e, true)); + if let Some(body) = first_child_of_kinds(&child, &["class_body"]) { + let np: Vec = vec![name.to_string()]; + walk_body(body, src, mod_prefix, &np, units); + } + } + } + "function_declaration" => { + // Top-level Kotlin function (unlike Java). + if let Some(name) = node_name_text(&child, src) { + glue.retain(|(_, gs, _)| *gs < s); + flush_glue(glue, units, mod_prefix, mod_path); + let sym = join_symbol(mod_prefix, mod_path, name); + units.push((sym, s, e, true)); + } + } + "import" => { + glue.push((1, s, e)); + } + // `property_declaration` (top-level val/var) and `type_alias` + // are not emitted as standalone units in the 1차 cut — they + // glue into the import group instead. `package_header` is + // handled by `extract_package` (structural metadata, not a + // unit). + _ => {} + } + } + flush_glue(glue, units, mod_prefix, mod_path); +} + +/// Walk a `class_body` (or object's `class_body`). Emits one unit per +/// method / secondary constructor and recurses into nested type +/// declarations + companion objects. Property declarations are NOT +/// emitted (would explode unit count, parallel to Java field policy). +/// +/// `companion_object` carries an optional `name` field — when omitted +/// (the common case `companion object { ... }`) the implicit Kotlin +/// convention name `Companion` is used. +/// +/// No `glue` parameter: Kotlin imports are file-level only. +fn walk_body( + body: tree_sitter::Node, + src: &str, + mod_prefix: &str, + mod_path: &[String], + units: &mut Vec<(String, u32, u32, bool)>, +) { + let mut cur = body.walk(); + for child in body.named_children(&mut cur) { + let s = unit_start(&child); + let e = child.end_position().row as u32 + 1; + match child.kind() { + "function_declaration" => { + if let Some(name) = node_name_text(&child, src) { + let sym = join_symbol(mod_prefix, mod_path, name); + units.push((sym, s, e, true)); + } + } + "secondary_constructor" => { + // Kotlin secondary constructor — no `name` field on the + // grammar node. Per design §3.4 (Java JVM convention) the + // symbol uses the enclosing class name as the trailing + // segment (matches the Java `.<...>..` + // duplication for constructors). + if let Some(class_name) = mod_path.last() { + let sym = join_symbol(mod_prefix, mod_path, class_name); + units.push((sym, s, e, true)); + } + } + "companion_object" => { + // Companion's name field is OPTIONAL — fall back to the + // Kotlin implicit name `Companion`. + let name: &str = node_name_text(&child, src).unwrap_or("Companion"); + let sym = join_symbol(mod_prefix, mod_path, name); + units.push((sym, s, e, true)); + if let Some(inner_body) = first_child_of_kinds(&child, &["class_body"]) { + let mut np = mod_path.to_vec(); + np.push(name.to_string()); + walk_body(inner_body, src, mod_prefix, &np, units); + } + } + "class_declaration" => { + let name = match node_name_text(&child, src) { + Some(n) => n, + None => continue, + }; + let sym = join_symbol(mod_prefix, mod_path, name); + units.push((sym, s, e, true)); + let is_enum = class_decl_is_enum(&child, src); + if !is_enum { + if let Some(inner_body) = first_child_of_kinds(&child, &["class_body"]) { + let mut np = mod_path.to_vec(); + np.push(name.to_string()); + walk_body(inner_body, src, mod_prefix, &np, units); + } + } + } + "object_declaration" => { + let name = match node_name_text(&child, src) { + Some(n) => n, + None => continue, + }; + let sym = join_symbol(mod_prefix, mod_path, name); + units.push((sym, s, e, true)); + if let Some(inner_body) = first_child_of_kinds(&child, &["class_body"]) { + let mut np = mod_path.to_vec(); + np.push(name.to_string()); + walk_body(inner_body, src, mod_prefix, &np, units); + } + } + // property_declaration, anonymous_initializer: NOT emitted. + _ => {} + } + } +} + +fn flush_glue( + glue: &mut Vec<(usize, u32, u32)>, + units: &mut Vec<(String, u32, u32, bool)>, + mod_prefix: &str, + mod_path: &[String], +) { + if glue.is_empty() { + return; + } + let s = glue.iter().map(|(_, a, _)| *a).min().unwrap(); + let e = glue.iter().map(|(_, _, b)| *b).max().unwrap(); + // Provisional label: `` only if the group is exclusively + // imports. The post-pass demotes any `` to `` if + // the file produced any real unit. + let only_imports = glue.iter().all(|(is_import, _, _)| *is_import == 1); + let label = if only_imports { "" } else { "" }; + units.push((join_symbol(mod_prefix, mod_path, label), s, e, false)); + glue.clear(); +} + +#[cfg(test)] +mod tests { + use super::*; + use kebab_core::{Block, MediaType, SourceSpan}; + + fn extract_fixture() -> kebab_core::CanonicalDocument { + let bytes = std::fs::read(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/sample.kt" + )) + .unwrap(); + let asset = + crate::rust::tests_support::fixed_code_asset("crates/x/src/sample.kt", "kotlin"); + let cfg = kebab_core::ExtractConfig::default(); + let root = std::path::PathBuf::from("/tmp"); + let ctx = kebab_core::ExtractContext { + asset: &asset, + workspace_root: &root, + config: &cfg, + }; + KotlinAstExtractor::new().extract(&ctx, &bytes).unwrap() + } + + #[test] + fn extractor_supports_only_media_code_kotlin() { + let e = KotlinAstExtractor::new(); + assert!(e.supports(&MediaType::Code("kotlin".into()))); + assert!(!e.supports(&MediaType::Code("java".into()))); + assert!(!e.supports(&MediaType::Code("rust".into()))); + assert!(!e.supports(&MediaType::Markdown)); + } + + #[test] + fn kotlin_units_match_design_3_4_symbols() { + let doc = extract_fixture(); + let mut syms: Vec = doc + .blocks + .iter() + .filter_map(|b| match b { + Block::Code(c) => match &c.common.source_span { + SourceSpan::Code { symbol, lang, .. } => { + assert_eq!(lang.as_deref(), Some("kotlin")); + symbol.clone() + } + _ => None, + }, + _ => None, + }) + .collect(); + syms.sort(); + // package extracted from source = com.kebab.chunk + assert!( + syms.iter() + .any(|s| s == "com.kebab.chunk.MdHeadingV1Chunker"), + "got {syms:?}" + ); + assert!( + syms.iter() + .any(|s| s == "com.kebab.chunk.MdHeadingV1Chunker.chunkDoc"), + "got {syms:?}" + ); + assert!( + syms.iter() + .any(|s| s == "com.kebab.chunk.MdHeadingV1Chunker.getName"), + "got {syms:?}" + ); + // Implicit companion object name = Companion (grammar leaves the + // name field unset; the extractor fills it in). + assert!( + syms.iter() + .any(|s| s == "com.kebab.chunk.MdHeadingV1Chunker.Companion"), + "got {syms:?}" + ); + assert!( + syms.iter() + .any(|s| s == "com.kebab.chunk.MdHeadingV1Chunker.Companion.withName"), + "got {syms:?}" + ); + // interface — also via class_declaration in the grammar + assert!( + syms.iter().any(|s| s == "com.kebab.chunk.Stringer"), + "got {syms:?}" + ); + // enum class — also via class_declaration; body NOT recursed + assert!( + syms.iter().any(|s| s == "com.kebab.chunk.Mode"), + "got {syms:?}" + ); + // Kotlin top-level fn — unlike Java + assert!( + syms.iter().any(|s| s == "com.kebab.chunk.freeFunction"), + "got {syms:?}" + ); + // Singleton object + its method + assert!( + syms.iter().any(|s| s == "com.kebab.chunk.Singleton"), + "got {syms:?}" + ); + assert!( + syms.iter().any(|s| s == "com.kebab.chunk.Singleton.ping"), + "got {syms:?}" + ); + // import grouped as + assert!( + syms.iter().any(|s| s == "com.kebab.chunk."), + "got {syms:?}" + ); + } + + #[test] + fn deterministic_across_runs() { + let a = extract_fixture(); + for _ in 0..50 { + assert_eq!(extract_fixture().blocks, a.blocks); + } + } +} diff --git a/crates/kebab-parse-code/src/lib.rs b/crates/kebab-parse-code/src/lib.rs index d6ff0d3..854ba27 100644 --- a/crates/kebab-parse-code/src/lib.rs +++ b/crates/kebab-parse-code/src/lib.rs @@ -14,7 +14,9 @@ //! / llm / rag. pub mod go; +pub mod java; pub mod javascript; +pub mod kotlin; pub mod lang; pub mod python; pub mod repo; @@ -24,7 +26,9 @@ pub mod skip; pub mod typescript; pub use go::{PARSER_VERSION as GO_PARSER_VERSION, GoAstExtractor}; +pub use java::{PARSER_VERSION as JAVA_PARSER_VERSION, JavaAstExtractor}; pub use javascript::{PARSER_VERSION as JS_PARSER_VERSION, JavascriptAstExtractor}; +pub use kotlin::{PARSER_VERSION as KOTLIN_PARSER_VERSION, KotlinAstExtractor}; pub use lang::{code_lang_for_path, module_path_for_python, module_path_for_tsjs}; pub use python::{PARSER_VERSION as PYTHON_PARSER_VERSION, PythonAstExtractor}; pub use repo::{RepoMeta, detect_repo}; diff --git a/crates/kebab-parse-code/tests/fixtures/sample.java b/crates/kebab-parse-code/tests/fixtures/sample.java new file mode 100644 index 0000000..228d3f1 --- /dev/null +++ b/crates/kebab-parse-code/tests/fixtures/sample.java @@ -0,0 +1,36 @@ +// sample.java +package com.kebab.chunk; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * Heading-aware Markdown chunker. + */ +public class MdHeadingV1Chunker { + private final String name; + + public MdHeadingV1Chunker(String name) { + this.name = name; + } + + public List chunkDoc(String input) { + return List.of(name, input); + } + + public String getName() { + return name; + } + + public static class Builder { + private String name; + public Builder withName(String n) { this.name = n; return this; } + public MdHeadingV1Chunker build() { return new MdHeadingV1Chunker(name); } + } +} + +interface Stringer { + String asString(); +} + +enum Mode { DEFAULT, FAST } diff --git a/crates/kebab-parse-code/tests/fixtures/sample.kt b/crates/kebab-parse-code/tests/fixtures/sample.kt new file mode 100644 index 0000000..200cff7 --- /dev/null +++ b/crates/kebab-parse-code/tests/fixtures/sample.kt @@ -0,0 +1,29 @@ +// sample.kt +package com.kebab.chunk + +import java.util.List + +/** + * Heading-aware Markdown chunker. + */ +class MdHeadingV1Chunker(val name: String) { + fun chunkDoc(input: String): List = listOf(name, input) + + fun getName(): String = name + + companion object { + fun withName(n: String): MdHeadingV1Chunker = MdHeadingV1Chunker(n) + } +} + +interface Stringer { + fun asString(): String +} + +enum class Mode { DEFAULT, FAST } + +fun freeFunction(x: Int): Int = x + 1 + +object Singleton { + fun ping(): String = "pong" +} diff --git a/crates/kebab-source-fs/src/media.rs b/crates/kebab-source-fs/src/media.rs index 4e17a2d..3f9e0c6 100644 --- a/crates/kebab-source-fs/src/media.rs +++ b/crates/kebab-source-fs/src/media.rs @@ -49,6 +49,10 @@ pub(crate) fn media_type_for(path: &Path) -> MediaType { // p10-1C-Go: Go ingest activated. "go" => MediaType::Code("go".into()), + // p10-1C-JK: JVM family (Java + Kotlin) ingest activated. + "java" => MediaType::Code("java".into()), + "kt" | "kts" => MediaType::Code("kotlin".into()), + // Empty string (no extension) and any other extension: bucket as // Other and let downstream extractors decide if they support it. _ => MediaType::Other(ext), @@ -127,6 +131,13 @@ mod tests { assert_eq!(media_type_for(Path::new("a/b.go")), MediaType::Code("go".into())); } + #[test] + fn java_kotlin_files_map_to_media_code() { + assert_eq!(media_type_for(Path::new("a/b.java")), MediaType::Code("java".into())); + assert_eq!(media_type_for(Path::new("a/b.kt")), MediaType::Code("kotlin".into())); + assert_eq!(media_type_for(Path::new("a/b.kts")), MediaType::Code("kotlin".into())); + } + #[test] fn unknown_and_missing_extension() { assert_eq!( diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index efbacb7..beafdec 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -22,7 +22,7 @@ Cargo workspace, 함수 호출 기반 모듈러 모놀리스. UI binary (`kebab- | OCR | Ollama vision LM (default `gemma4:e4b`) — `OcrEngine` trait 으로 Tesseract / Apple Vision 등 future swap (HOTFIXES P6-2) | | Image caption | Ollama vision LM, runtime gate `image.caption.enabled` (default OFF) | | PDF parser | `lopdf` per-page 텍스트, `chunker_version = "pdf-page-v1"` 가 PDF 자산에 하드코딩 (HOTFIXES P7-3) | -| code parser | `tree-sitter` + `tree-sitter-rust` / `tree-sitter-python` / `tree-sitter-typescript` / `tree-sitter-javascript` / `tree-sitter-go` — **parser-side** (`kebab-parse-code`), chunker-side 아님 (design §6.3). chunker versions: Rust = `code-rust-ast-v1`, Python = `code-python-ast-v1`, TypeScript = `code-ts-ast-v1`, JavaScript = `code-js-ast-v1`, Go = `code-go-ast-v1`. `ast_chunk_max_lines = 200` 상수 고정 (HOTFIXES 2026-05-19 — Chunker trait 이 per-medium config 미노출). | +| code parser | `tree-sitter` + `tree-sitter-rust` / `tree-sitter-python` / `tree-sitter-typescript` / `tree-sitter-javascript` / `tree-sitter-go` / `tree-sitter-java` / `tree-sitter-kotlin-ng` — **parser-side** (`kebab-parse-code`), chunker-side 아님 (design §6.3). chunker versions: Rust = `code-rust-ast-v1`, Python = `code-python-ast-v1`, TypeScript = `code-ts-ast-v1`, JavaScript = `code-js-ast-v1`, Go = `code-go-ast-v1`, Java = `code-java-ast-v1`, Kotlin = `code-kotlin-ast-v1`. `ast_chunk_max_lines = 200` 상수 고정 (HOTFIXES 2026-05-19 — Chunker trait 이 per-medium config 미노출). Kotlin grammar 은 `tree-sitter-kotlin-ng` 사용 — bare `tree-sitter-kotlin` 은 tree-sitter 0.21–0.23 에 고착되어 있어 사용 불가. | | 1B symbol path | workspace path → module path: Python = dotted prefix (`kebab_eval.metrics.compute_mrr`), TypeScript/JavaScript = slash-style prefix (`src/Foo.Foo.search`). Rust 1A-2 는 file-scope nesting 만 (workspace prefix 없음, 비일관 수용 — HOTFIXES 2026-05-20). | | TUI | Ratatui + crossterm — P9-1 Library 패널, P9-2/3/4 진행 예정 | | Desktop | Tauri 2 + `pdfjs-dist` (native PDF render backend 금지) — P9-5 | @@ -52,7 +52,7 @@ flowchart TB ppdf["kebab-parse-pdf"] pimg["kebab-parse-image"] paud["kebab-parse-audio
(P8 보류)"] - pcode["kebab-parse-code
(P10-1A-2 + P10-1B + P10-1C-Go)"] + pcode["kebab-parse-code
(P10-1A-2 + P10-1B + P10-1C-Go + P10-1C-JK)"] ptypes["kebab-parse-types"] norm["kebab-normalize"] chunk["kebab-chunk"] @@ -127,7 +127,7 @@ flowchart TB UI → store/llm/parse 직접 의존 금지. 모든 user-facing 진입은 `kebab-app` facade 만 통한다 (frozen 설계 §8). `kebab-cli` 가 `--config ` flag 를 honor 하려면 `kebab_app::*_with_config(cfg, …)` companion 을 통해 Config 을 명시적으로 thread 하는 패턴 — 자세한 이유는 [tasks/HOTFIXES.md](../tasks/HOTFIXES.md) 의 `--config` 항목. -`kebab-parse-code` 의 외부 tree-sitter grammar crate 의존: P10-1A-2 에서 `tree-sitter-rust` 추가, P10-1B 에서 `tree-sitter-python` / `tree-sitter-typescript` / `tree-sitter-javascript` 추가, P10-1C-Go 에서 `tree-sitter-go` 추가. 모두 `kebab-parse-code` 에만 격리 (facade 룰 — UI crate / chunker 가 직접 import 금지). +`kebab-parse-code` 의 외부 tree-sitter grammar crate 의존: P10-1A-2 에서 `tree-sitter-rust` 추가, P10-1B 에서 `tree-sitter-python` / `tree-sitter-typescript` / `tree-sitter-javascript` 추가, P10-1C-Go 에서 `tree-sitter-go` 추가, P10-1C-JK 에서 `tree-sitter-java` / `tree-sitter-kotlin-ng` 추가. 모두 `kebab-parse-code` 에만 격리 (facade 룰 — UI crate / chunker 가 직접 import 금지). Kotlin 은 `tree-sitter-kotlin-ng` 사용 (bare `tree-sitter-kotlin` 은 tree-sitter 0.21–0.23 에 고착 — 사용 불가). ## 디렉토리 구조 @@ -165,7 +165,7 @@ kebab/ │ ├── kebab-source-fs/ # 워크스페이스 walk + checksum (P1-1) │ ├── kebab-parse-md/ # Markdown frontmatter + blocks (P1-2/3) │ ├── kebab-normalize/ # ParsedBlock → CanonicalDocument (P1-4) -│ ├── kebab-chunk/ # heading-aware + pdf-page-v1 + code-rust-ast-v1 + code-python-ast-v1 + code-ts-ast-v1 + code-js-ast-v1 + code-go-ast-v1 chunker (P1-5, P7-2, P10-1A-2, P10-1B, P10-1C-Go) +│ ├── kebab-chunk/ # heading-aware + pdf-page-v1 + code-rust-ast-v1 + code-python-ast-v1 + code-ts-ast-v1 + code-js-ast-v1 + code-go-ast-v1 + code-java-ast-v1 + code-kotlin-ast-v1 chunker (P1-5, P7-2, P10-1A-2, P10-1B, P10-1C-Go, P10-1C-JK) │ ├── kebab-store-sqlite/ # SQLite + FTS5 (V001/V002/V003) (P1-6, P2-1, P3-3) │ ├── kebab-search/ # Lexical + Vector + Hybrid retriever (P2-2, P3-4) │ ├── kebab-embed/ kebab-embed-local/ # Embedder trait + fastembed adapter (P3-1, P3-2) @@ -175,7 +175,7 @@ kebab/ │ ├── kebab-eval/ # golden query runner + metrics (P5-1, P5-2) │ ├── kebab-parse-image/ # ImageExtractor + Ollama OCR + caption (P6) │ ├── kebab-parse-pdf/ # lopdf per-page text extractor (P7-1) -│ ├── kebab-parse-code/ # tree-sitter AST extractors: Rust (P10-1A-2), Python + TypeScript + JavaScript (P10-1B), Go (P10-1C-Go); chunker lives in kebab-chunk +│ ├── kebab-parse-code/ # tree-sitter AST extractors: Rust (P10-1A-2), Python + TypeScript + JavaScript (P10-1B), Go (P10-1C-Go), Java + Kotlin (P10-1C-JK — java.rs + kotlin.rs); chunker lives in kebab-chunk │ ├── kebab-app/ # facade (P0 시그니처 + P3-5/P6-4/P7-3 본체) │ ├── kebab-tui/ # Ratatui shell + Library 패널 (P9-1) │ ├── kebab-mcp/ # stdio MCP server — tools: schema, doctor, search, ask (P9-FB-30) diff --git a/docs/SMOKE.md b/docs/SMOKE.md index 7831024..b609a2f 100644 --- a/docs/SMOKE.md +++ b/docs/SMOKE.md @@ -455,6 +455,7 @@ rm -rf /tmp/kebab-smoke # 통째로 정리 - (P10-1A-2) `.rs` 파일을 워크스페이스에 두면 `kebab ingest` 결과에 `new` 카운터에 포함. `kebab search --mode hybrid "<함수명>" --code-lang rust --json` 가 `citation.kind = "code"`, `citation.lang = "rust"` (SearchHit top-level `code_lang` 도 동일), `citation.symbol` (함수/타입 이름), `citation.line_start` / `citation.line_end` 를 반환하면 wiring 정상. `kebab schema --json | jq .stats.code_lang_breakdown` 에 `"rust": N` 이 나오면 chunk 가 색인됨. - (P10-1B) `.py` / `.ts` / `.tsx` / `.js` / `.mjs` / `.cjs` / `.jsx` 파일을 워크스페이스에 두면 `kebab ingest` 결과에 `new` 카운터에 포함. `--code-lang python` / `--code-lang typescript` / `--code-lang javascript` 검색이 `citation.symbol` 에 module path prefix 를 포함한 결과를 반환하면 wiring 정상. `kebab schema --json | jq .stats.code_lang_breakdown` 에 해당 언어 카운트 등장 확인. - (P10-1C-Go) `.go` 파일을 워크스페이스에 두면 `kebab ingest` 가 `code-go-ast-v1` 로 처리. `--code-lang go` 검색이 `citation.symbol` 에 `.` / `.(*Receiver).` 형식 결과를 반환하면 wiring 정상. `kebab schema --json | jq .stats.code_lang_breakdown` 에 `"go": N` 등장 확인. +- (P10-1C-JK) `.java` 파일은 `code-java-ast-v1`, `.kt`/`.kts` 파일은 `code-kotlin-ast-v1` 로 처리. `--code-lang java` / `--code-lang kotlin` 검색이 `citation.symbol` 에 `com.foo.Foo.bar` 형식 결과를 반환하면 wiring 정상. `kebab schema --json | jq .stats.code_lang_breakdown` 에 `"java": N` / `"kotlin": N` 등장 확인. - (P7-3 + follow-up) 동일 path 에 byte 가 다른 PDF 를 두 번째 ingest 하면 `purge_vector_orphans_for_workspace_path` 가 옛 chunk_id 를 LanceDB 에서 먼저 삭제, 이어서 `purge_orphan_at_workspace_path` 가 옛 doc / chunks / embedding_records 를 SQLite 에서 sweep. 새 byte 가 새 `doc_id` 로 색인됨. `IngestReport` 에 그 자산만 `new+=1` (다른 자산은 `updated`). 두 store 모두 정합 — 옛 본문 검색 시 옛 chunks 가 더 이상 surface 되지 않음. ### Embedding upgrade (fb-39b) diff --git a/docs/superpowers/plans/2026-05-20-p10-1c-jk-ast-chunker.md b/docs/superpowers/plans/2026-05-20-p10-1c-jk-ast-chunker.md new file mode 100644 index 0000000..11f6b86 --- /dev/null +++ b/docs/superpowers/plans/2026-05-20-p10-1c-jk-ast-chunker.md @@ -0,0 +1,494 @@ +# p10-1C-JavaKotlin Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development to implement this plan task-by-task. + +**Goal:** Activate Java + Kotlin code ingest end-to-end. Mirror 1C-Go (PR #151 / v0.12.0) for Java (single-language scaffold) and Kotlin (additional top-level fn variant). Both use source-side `package` extraction (design §3.4 JVM convention). + +**Architecture:** Same shape as 1B (multi-language single PR). 2 new tree-sitter grammars + 2 extractors + 2 chunkers + media routing + app dispatch arms. 1C-Go pattern is the closest template for source-side `package` extraction. + +**Tech Stack:** Rust 2024 workspace, `tree-sitter` 0.26 (already), `tree-sitter-java` + `tree-sitter-kotlin` (NEW). 1A-2/1B/1C-Go infrastructure unchanged. + +**Memory note:** Host has been OOM'd previously. Per-crate cargo only. ONE full-suite + clippy invocation in Task J. + +--- + +## Pre-flight + +Branch `feat/p10-1c-jk` already exists. + +- [ ] **Disk hygiene**: `cargo clean` if heavy (last cleanup recovered 34 GB). + +Reference files: +- 1C-Go extractor: `crates/kebab-parse-code/src/go.rs` — closest template for source-side package extraction. +- 1B Python extractor: `crates/kebab-parse-code/src/python.rs` — class-nesting recursion model (relevant for Java/Kotlin). +- 1A-2 chunker: `crates/kebab-chunk/src/code_rust_ast_v1.rs` — duplicate-with-substitution. +- 1B dispatch generalization: `crates/kebab-app/src/lib.rs::ingest_one_code_asset` 4-arm match (~L1645). 1C-Go already added `"go"`; this PR adds `"java"` + `"kotlin"`. + +--- + +## Task A: Workspace deps (tree-sitter-java + tree-sitter-kotlin) + +**Files:** +- Modify: `Cargo.toml` (workspace `[workspace.dependencies]`, after `tree-sitter-go` line) +- Modify: `crates/kebab-parse-code/Cargo.toml` + +- [ ] **Step 1**: `cargo add tree-sitter-java tree-sitter-kotlin -p kebab-parse-code`. If `tree-sitter-kotlin` resolves to a fork name, verify the actively-maintained crate (e.g. check crates.io page / GitHub stars / last update). Likely `tree-sitter-kotlin` (without fork suffix) is the default. + +- [ ] **Step 2**: Lift the two resolved versions into `[workspace.dependencies]` after `tree-sitter-go`: + +```toml +# JVM family grammars for code ingest (kebab-parse-code, p10-1C-JK). +tree-sitter-java = "" +tree-sitter-kotlin = "" +``` + +Switch crate's entries to `{ workspace = true }`. + +- [ ] **Step 3**: `cargo build -p kebab-parse-code` → clean. Unused dep warning is fine. + +- [ ] **Step 4**: Commit: + +```bash +git add Cargo.toml Cargo.lock crates/kebab-parse-code/Cargo.toml +git commit -m "build(p10-1c-jk): add tree-sitter-java + tree-sitter-kotlin workspace deps + +Co-Authored-By: Claude Opus 4.7 (1M context) " +``` + +If the kotlin crate has a different actual name (e.g. `tree-sitter-kotlin-ng` or fork suffix), document the choice in the commit body briefly. + +--- + +## Task B: source-fs routing `.java` / `.kt` / `.kts` + +**Files:** +- Modify: `crates/kebab-source-fs/src/media.rs` (add arm after the existing `.go` arm) +- Test: same file's test module + +- [ ] **Step 1 (failing test)** — add near `go_files_map_to_media_code_go`: + +```rust +#[test] +fn java_kotlin_files_map_to_media_code() { + assert_eq!(media_type_for(Path::new("a/b.java")), MediaType::Code("java".into())); + assert_eq!(media_type_for(Path::new("a/b.kt")), MediaType::Code("kotlin".into())); + assert_eq!(media_type_for(Path::new("a/b.kts")), MediaType::Code("kotlin".into())); +} +``` + +- [ ] **Step 2**: Run → FAIL. + +- [ ] **Step 3**: Add the arms before the `_ => MediaType::Other(ext)` fallback (after `"go" => ...`): + +```rust + // p10-1C-JK: JVM family (Java + Kotlin) ingest activated. + "java" => MediaType::Code("java".into()), + "kt" | "kts" => MediaType::Code("kotlin".into()), +``` + +- [ ] **Step 4**: Run → PASS. `cargo test -p kebab-source-fs` → no regression. + +- [ ] **Step 5**: clippy clean, commit. + +```bash +cargo clippy -p kebab-source-fs --all-targets -- -D warnings +git add crates/kebab-source-fs/ +git commit -m "feat(p10-1c-jk): route .java/.kt/.kts to MediaType::Code + +Co-Authored-By: Claude Opus 4.7 (1M context) " +``` + +--- + +## Task C: App dispatch + bail arms for "java" + "kotlin" + +**Files:** +- Modify: `crates/kebab-app/src/lib.rs` + +- [ ] **Step 1**: Find the dispatch arm guard (currently `matches!(lang.as_str(), "rust" | "python" | "typescript" | "javascript" | "go")`). Add `"java"` + `"kotlin"`: + +```rust +MediaType::Code(lang) + if matches!(lang.as_str(), + "rust" | "python" | "typescript" | "javascript" | "go" | "java" | "kotlin") => +``` + +- [ ] **Step 2**: In `ingest_one_code_asset` the 4 `match code_lang` blocks add `"java"` and `"kotlin"` arms that `bail!()` for now: + +```rust +"java" => anyhow::bail!("java ingest not yet wired (p10-1c-jk Task F)"), +"kotlin" => anyhow::bail!("kotlin ingest not yet wired (p10-1c-jk Task I)"), +``` + +(in each of the 4 blocks before the `other =>` catch-all). + +- [ ] **Step 3**: Verify per-crate: +- `cargo test -p kebab-app --lib` → 52 stay green +- `cargo test -p kebab-app --test code_ingest_smoke` → 7 stay green +- `cargo clippy -p kebab-app --all-targets -- -D warnings` clean + +- [ ] **Step 4**: Commit: + +```bash +git add crates/kebab-app/ +git commit -m "refactor(p10-1c-jk): add java + kotlin to ingest dispatch allowlist (bail until Tasks F/I) + +Co-Authored-By: Claude Opus 4.7 (1M context) " +``` + +--- + +## Task D: `JavaAstExtractor` + +**Files:** +- Create: `crates/kebab-parse-code/src/java.rs` +- Modify: `crates/kebab-parse-code/src/lib.rs` (`pub mod java;` + re-exports `JAVA_PARSER_VERSION`, `JavaAstExtractor`) +- Create: `crates/kebab-parse-code/tests/fixtures/sample.java` + +Scaffold mirrors `crates/kebab-parse-code/src/go.rs` (1C-Go) — single-language with source-side `package` extraction. Differences: + +### Constants + +```rust +pub const PARSER_VERSION: &str = "code-java-v1"; +pub struct JavaAstExtractor; +// supports: matches!(m, MediaType::Code(l) if l == "java") +// code_lang = Some("java"), SourceType::Note, repo via detect_repo +``` + +### Package extraction (Java) + +tree-sitter-java grammar: +- Root: `program` +- `package_declaration` (top-level child) → contains `scoped_identifier` (dotted) OR `identifier` (single-segment) + +```rust +fn extract_package(root: tree_sitter::Node, src: &str) -> Option { + let mut cur = root.walk(); + for child in root.named_children(&mut cur) { + if child.kind() == "package_declaration" { + // package_declaration has scoped_identifier OR identifier as first named child + let mut c2 = child.walk(); + for sub in child.named_children(&mut c2) { + if sub.kind() == "scoped_identifier" || sub.kind() == "identifier" { + return Some(src[sub.start_byte()..sub.end_byte()].to_string()); + } + } + } + } + None +} +``` + +(Verify field names against tree-sitter-java's node-types.json if any field differs.) + +### AST mapping + +| node kind | unit | symbol | +|-----------|------|--------| +| `class_declaration` (name field) | 1 + recurse body | `.` | +| `interface_declaration` (name) | 1 + recurse body | `.` | +| `enum_declaration` (name) | 1 | `.` | +| `record_declaration` (name, Java 14+) | 1 | `.` | +| `annotation_type_declaration` (name) | 1 | `.` | +| Inside class body: `method_declaration` (name) | 1 | `..` | +| Inside class body: `constructor_declaration` (name = class name) | 1 | `..` (matches Java convention) | +| Nested classes recurse with class name pushed onto mod_path | as above | `..` etc. | +| `import_declaration`, `package_declaration` | glue | `.` | +| `field_declaration` at top of class | NOT a unit in 1C-JK (would explode unit count for value-only fields) | n/a | + +`unit_start` walks `comment` siblings; Java has `@interface` annotations but those are part of `annotation_type_declaration` itself, not separate sibling nodes. + +`mod_path` = class nesting (like 1B Python). Empty at file top level. + +### Fixture `tests/fixtures/sample.java`: + +```java +// sample.java +package com.kebab.chunk; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * Heading-aware Markdown chunker. + */ +public class MdHeadingV1Chunker { + private final String name; + + public MdHeadingV1Chunker(String name) { + this.name = name; + } + + public List chunkDoc(String input) { + return List.of(name, input); + } + + public String getName() { + return name; + } + + public static class Builder { + private String name; + public Builder withName(String n) { this.name = n; return this; } + public MdHeadingV1Chunker build() { return new MdHeadingV1Chunker(name); } + } +} + +interface Stringer { + String asString(); +} + +enum Mode { DEFAULT, FAST } +``` + +### Test module (inline `#[cfg(test)] mod tests`) + +Mirror 1C-Go shape: + +```rust +#[cfg(test)] +mod tests { + use super::*; + use kebab_core::{Block, MediaType, SourceSpan}; + + fn extract_fixture() -> kebab_core::CanonicalDocument { + let bytes = std::fs::read( + concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/sample.java"), + ).unwrap(); + let asset = crate::rust::tests_support::fixed_code_asset( + "crates/x/src/sample.java", "java", + ); + let cfg = kebab_core::ExtractConfig::default(); + let root = std::path::PathBuf::from("/tmp"); + let ctx = kebab_core::ExtractContext { asset: &asset, workspace_root: &root, config: &cfg }; + JavaAstExtractor::new().extract(&ctx, &bytes).unwrap() + } + + #[test] + fn extractor_supports_only_media_code_java() { /* ... */ } + + #[test] + fn java_units_match_design_3_4_symbols() { + let doc = extract_fixture(); + let mut syms: Vec = doc.blocks.iter().filter_map(|b| match b { + Block::Code(c) => match &c.common.source_span { + SourceSpan::Code { symbol, lang, .. } => { + assert_eq!(lang.as_deref(), Some("java")); + symbol.clone() + } + _ => None, + }, + _ => None, + }).collect(); + syms.sort(); + // workspace path → package extracted from source = com.kebab.chunk + assert!(syms.iter().any(|s| s == "com.kebab.chunk.MdHeadingV1Chunker"), "got {syms:?}"); + assert!(syms.iter().any(|s| s == "com.kebab.chunk.MdHeadingV1Chunker.MdHeadingV1Chunker")); // constructor + assert!(syms.iter().any(|s| s == "com.kebab.chunk.MdHeadingV1Chunker.chunkDoc")); + assert!(syms.iter().any(|s| s == "com.kebab.chunk.MdHeadingV1Chunker.getName")); + assert!(syms.iter().any(|s| s == "com.kebab.chunk.MdHeadingV1Chunker.Builder")); + assert!(syms.iter().any(|s| s == "com.kebab.chunk.MdHeadingV1Chunker.Builder.withName")); + assert!(syms.iter().any(|s| s == "com.kebab.chunk.MdHeadingV1Chunker.Builder.build")); + assert!(syms.iter().any(|s| s == "com.kebab.chunk.Stringer")); + assert!(syms.iter().any(|s| s == "com.kebab.chunk.Mode")); + assert!(syms.iter().any(|s| s == "com.kebab.chunk.")); + } + + #[test] + fn deterministic_across_runs() { + let a = extract_fixture(); + for _ in 0..50 { assert_eq!(extract_fixture().blocks, a.blocks); } + } +} +``` + +### Wire into lib.rs + +```rust +pub mod java; +pub use java::{PARSER_VERSION as JAVA_PARSER_VERSION, JavaAstExtractor}; +``` + +### Verify + commit + +- `cargo test -p kebab-parse-code` → all pass +- `cargo clippy -p kebab-parse-code --all-targets -- -D warnings` clean +- commit `feat(p10-1c-jk): tree-sitter-java AST extractor (JavaAstExtractor)` + +--- + +## Task E: `code-java-ast-v1` chunker + +Identical pattern to 1C-Go Task E. Duplicate `code_rust_ast_v1.rs` with substitutions: +- `VERSION_LABEL = "code-java-ast-v1"`, struct `CodeJavaAstV1Chunker` +- error message + module doc-comment prose +- Test module: parser_version `"code-java-v1"`, code_lang `"java"` +- Keep cross-chunker `policy_hash_matches_md_heading_v1` + +Wire into `crates/kebab-chunk/src/lib.rs` (alphabetical). Verify + commit. + +--- + +## Task F: Activate Java in app dispatch + +Replace the `"java"` `bail!()` arms in `ingest_one_code_asset` with real calls (`JavaAstExtractor` + `CodeJavaAstV1Chunker`). Add integration test `java_file_ingests_and_searches_as_code_citation` (mirror 1C-Go test, fixture `pkg_dir/Foo.java` with `package com.foo;` and `public class Foo { public String bar() { ... } }`, assert symbol `com.foo.Foo.bar`). + +Verify + commit. + +--- + +## Task G: `KotlinAstExtractor` + +**Files:** +- Create: `crates/kebab-parse-code/src/kotlin.rs` +- Modify: `crates/kebab-parse-code/src/lib.rs` +- Create: `crates/kebab-parse-code/tests/fixtures/sample.kt` + +Constants: `PARSER_VERSION = "code-kotlin-v1"`, `KotlinAstExtractor`, `code_lang = "kotlin"`. + +### Package extraction (Kotlin) + +tree-sitter-kotlin grammar: +- Root: `source_file` +- `package_header` (top-level) → contains `identifier` (dotted is single `identifier` node text; verify against node-types.json) + +```rust +fn extract_package(root: tree_sitter::Node, src: &str) -> Option { + let mut cur = root.walk(); + for child in root.named_children(&mut cur) { + if child.kind() == "package_header" { + let mut c2 = child.walk(); + for sub in child.named_children(&mut c2) { + if sub.kind() == "identifier" { + return Some(src[sub.start_byte()..sub.end_byte()].to_string()); + } + } + } + } + None +} +``` + +(Verify against tree-sitter-kotlin's node-types.json — Kotlin grammar varies more than Java's.) + +### AST mapping (Kotlin) + +| node kind | unit | symbol | +|-----------|------|--------| +| `class_declaration` (name field) — covers `class`, `data class`, `sealed class`, `enum class`, `interface` (Kotlin's interface is a class_declaration variant) | 1 + recurse body | `.` | +| `object_declaration` (name) — singleton | 1 + recurse | `.` | +| `function_declaration` (name) | 1 | `.` (top-level) or `..` (inside class) | +| Inside class body: `function_declaration` → method | 1 | `..` | +| `property_declaration` at top-level (`val` / `var`) | glue | `` (Kotlin top-level properties are common — keep as glue not unit) | +| `import_header`, `package_header` | glue | `` | + +(Detect class-vs-interface via modifier; for 1C 1차 treat both as `class_declaration` arm — symbol differs only via name. If tree-sitter-kotlin exposes `interface` keyword via modifier list, mention in HOTFIXES if special handling needed.) + +### Fixture `sample.kt`: + +```kotlin +// sample.kt +package com.kebab.chunk + +import java.util.List + +/** + * Heading-aware Markdown chunker. + */ +class MdHeadingV1Chunker(val name: String) { + fun chunkDoc(input: String): List = listOf(name, input) + + fun getName(): String = name + + companion object { + fun withName(n: String): MdHeadingV1Chunker = MdHeadingV1Chunker(n) + } +} + +interface Stringer { + fun asString(): String +} + +enum class Mode { DEFAULT, FAST } + +fun freeFunction(x: Int): Int = x + 1 + +object Singleton { + fun ping(): String = "pong" +} +``` + +### Test module — assert symbols + +```rust +// Asserted symbols: +"com.kebab.chunk.MdHeadingV1Chunker" +"com.kebab.chunk.MdHeadingV1Chunker.chunkDoc" +"com.kebab.chunk.MdHeadingV1Chunker.getName" +"com.kebab.chunk.MdHeadingV1Chunker.Companion" // companion object (verify name) +"com.kebab.chunk.MdHeadingV1Chunker.Companion.withName" // method on companion +"com.kebab.chunk.Stringer" +"com.kebab.chunk.Mode" +"com.kebab.chunk.freeFunction" // top-level fn (Kotlin-specific!) +"com.kebab.chunk.Singleton" +"com.kebab.chunk.Singleton.ping" +"com.kebab.chunk." // import + property glue +``` + +(Companion object: tree-sitter-kotlin may use `companion_object` or `object_declaration` with `companion` modifier — verify and adjust the symbol if `Companion` isn't the right name.) + +### Wire into lib.rs + +```rust +pub mod kotlin; +pub use kotlin::{PARSER_VERSION as KOTLIN_PARSER_VERSION, KotlinAstExtractor}; +``` + +Verify + commit. + +--- + +## Task H: `code-kotlin-ast-v1` chunker + +Same pattern as Task E. Substitute kotlin labels. Verify + commit. + +--- + +## Task I: Activate Kotlin in app dispatch + +Replace `"kotlin"` bail arms with real calls. Add integration test `kotlin_file_ingests_and_searches_as_code_citation`. Verify + commit. + +--- + +## Task J: Snapshots + full-suite + SMOKE + +- Create 2 snapshot tests (`code_java_ast_snapshot.rs`, `code_kotlin_ast_snapshot.rs`) + baselines. Mirror 1C-Go Task G snapshot test. +- ONE workspace test + clippy invocation. +- Manual SMOKE: write a `.java` and `.kt` file in TempDir, ingest, search. + +Verify + commit (snapshot only). + +--- + +## Task K: Docs + version bump + +- README + HANDOFF + ARCHITECTURE + SMOKE + 2 INDEX updates + design §10.1. +- `Cargo.toml` version `0.12.0 → 0.13.0` (minor, surface 확장). + +Commit `docs(p10-1c-jk): ... + chore: bump 0.12.0 → 0.13.0`. + +--- + +## Finalize + +`gitea-pr` → review loop → merge → main pull → branch cleanup → `cargo clean` → `gitea-release v0.13.0`. + +--- + +## Self-Review (filled by plan author) + +- **Spec coverage**: design §1C Java + Kotlin → Tasks D-I; §3.4 symbol path → extractor (Java D, Kotlin G); §6.1/§6.2 module structure → Tasks D/E/G/H; §6.3 dep graph → Task A; §9.1 Tier-1 + oversize fallback → chunkers E/H. +- **No placeholders**: novel logic (Java `extract_package`, Kotlin `extract_package`, AST walk arm tables) given concretely. Chunkers (E, H) are explicit "duplicate code_rust_ast_v1.rs with substitution X/Y/Z". +- **Type consistency**: `JavaAstExtractor` / `JAVA_PARSER_VERSION` / `CodeJavaAstV1Chunker` + `KotlinAstExtractor` / `KOTLIN_PARSER_VERSION` / `CodeKotlinAstV1Chunker` used consistently. `MediaType::Code("java")` / `("kotlin")` in routing + dispatch. +- **Kotlin grammar risk**: noted — tree-sitter-kotlin's exact node kinds (`class_declaration` vs `object_declaration`, `companion_object` vs companion modifier, `package_header` vs `package_directive`) should be verified against the resolved crate's node-types.json. Pin contract via test fixture; HOTFIXES any deviation found during implementation. diff --git a/docs/superpowers/specs/2026-04-27-kebab-final-form-design.md b/docs/superpowers/specs/2026-04-27-kebab-final-form-design.md index 05d4276..c216ed6 100644 --- a/docs/superpowers/specs/2026-04-27-kebab-final-form-design.md +++ b/docs/superpowers/specs/2026-04-27-kebab-final-form-design.md @@ -1545,7 +1545,9 @@ transitional 형태) 의 source of truth. **p10-1B 활성화 (Python / TypeScript / JavaScript) (2026-05-20)**: Python (`code-python-ast-v1`, `.py`), TypeScript (`code-ts-ast-v1`, `.ts`/`.tsx`), JavaScript (`code-js-ast-v1`, `.js`/`.mjs`/`.cjs`/`.jsx`) AST chunker 활성화. symbol path 는 workspace 경로 → module path prefix: Python = dotted (예: `kebab_eval.metrics.compute_mrr`), TypeScript/JavaScript = slash-style (예: `src/Foo.Foo.search`). Rust 1A-2 의 file-scope-only symbol 과 비일관 수용 (HOTFIXES 2026-05-20). expression-level 함수 (`const foo = () => {}`) 는 glue 처리 (HOTFIXES 2026-05-20). -**p10-1C-Go 활성화 (Go) (2026-05-20)**: Go (`code-go-ast-v1`, `.go`) AST chunker 활성화. symbol = `.` / `.(*Receiver).` 형식. Java / Kotlin 은 후속 PR (p10-1C-JavaKotlin) 에서 별도 활성화. +**p10-1C-Go 활성화 (Go) (2026-05-20)**: Go (`code-go-ast-v1`, `.go`) AST chunker 활성화. symbol = `.` / `.(*Receiver).` 형식. + +**p10-1C-JavaKotlin 활성화 (Java + Kotlin) (2026-05-20)**: Java (`code-java-ast-v1`, `.java`) + Kotlin (`code-kotlin-ast-v1`, `.kt`/`.kts`) AST chunker 활성화. symbol = `com.foo.Foo.bar` 형식 (패키지 + 클래스 + 메서드/필드). Kotlin grammar 은 `tree-sitter-kotlin-ng` 사용 (bare `tree-sitter-kotlin` 은 tree-sitter 0.21–0.23 고착으로 사용 불가). ### 10.2 MCP server transport (fb-30) diff --git a/tasks/INDEX.md b/tasks/INDEX.md index 929bafc..c9496fe 100644 --- a/tasks/INDEX.md +++ b/tasks/INDEX.md @@ -143,7 +143,7 @@ P0~P5 는 직렬. P6~P9 는 P5 이후 병렬 가능. - [p10-1A-2 Rust AST chunker](p10/p10-1a-2-rust-ast-chunker.md) — ✅ 머지 - [p10-1B Python + TS/JS AST chunkers](p10/p10-1b-py-ts-js-ast-chunkers.md) — 🟡 PR 오픈 (코드 완성, 머지 대기) - p10-1C-Go Go AST chunker — 🟡 PR 오픈 (v0.12.0, `code-go-ast-v1`) - - p10-1C-JavaKotlin Java + Kotlin AST chunkers — ⏳ + - p10-1C-JavaKotlin Java + Kotlin AST chunkers — 🟢 PR 오픈 (v0.13.0, `code-java-ast-v1` / `code-kotlin-ast-v1`) - p10-1D C + C++ AST chunkers — ⏳ - p10-2 Tier 2 resource-aware — ⏳ - p10-3 Tier 3 paragraph + line-window fallback — ⏳ diff --git a/tasks/p10/INDEX.md b/tasks/p10/INDEX.md index 93dad3c..2ec3bd9 100644 --- a/tasks/p10/INDEX.md +++ b/tasks/p10/INDEX.md @@ -6,7 +6,7 @@ | 1A-2 | Rust AST chunker | ✅ 머지 | | 1B | Python + TS/JS AST chunkers | 🟡 PR 오픈 (코드 완성, 머지 대기) | | 1C-Go | Go AST chunker (`code-go-ast-v1`) | 🟡 PR 오픈 (v0.12.0) | -| 1C-JavaKotlin | Java + Kotlin AST chunkers | ⏳ | +| 1C-JavaKotlin | Java + Kotlin AST chunkers (`code-java-ast-v1` / `code-kotlin-ast-v1`) | 🟢 PR 오픈 (v0.13.0) | | 1D | C + C++ AST chunkers | ⏳ | | 2 | Tier 2 resource-aware (k8s / Dockerfile / manifest) | ⏳ | | 3 | Tier 3 paragraph + line-window fallback | ⏳ | diff --git a/tasks/p10/p10-1c-jk-ast-chunker.md b/tasks/p10/p10-1c-jk-ast-chunker.md new file mode 100644 index 0000000..26a8cb2 --- /dev/null +++ b/tasks/p10/p10-1c-jk-ast-chunker.md @@ -0,0 +1,69 @@ +# p10-1C-JavaKotlin — Java + Kotlin AST chunkers + +**Status:** 🟡 진행 중 +**Contract sections:** §3.3 (chunker_version `code-java-ast-v1` + `code-kotlin-ast-v1`), §3.4 (symbol path — Java/Kotlin `package.Class.method`), §3.5 (code_lang `java` + `kotlin`, ext `.java` / `.kt` / `.kts`), §6.1 (`kebab-parse-code/src/{java,kotlin}.rs`), §6.2 (`kebab-chunk/src/code_{java,kotlin}_ast_v1.rs`), §9.1 (Tier 1 AST per-language + oversize fallback). +**Design:** [2026-05-15-kebab-code-ingest-design.md](../../docs/superpowers/specs/2026-05-15-kebab-code-ingest-design.md) §1C (Java + Kotlin 부분 — Go 는 PR #151 / v0.12.0 별 PR 완료). +**Plan:** [2026-05-20-p10-1c-jk-ast-chunker.md](../../docs/superpowers/plans/2026-05-20-p10-1c-jk-ast-chunker.md). + +## Goal + +1C-Go (PR #151 / v0.12.0) 의 자매 PR. 같은 1C phase 의 JVM family (Java + Kotlin) 묶음. 머지 시점부터 `.java` / `.kt` / `.kts` 파일 dogfooding 가능. + +## 동결된 설계 결정 (이 task 로 확정) + +- **Symbol prefix = 소스 코드의 `package` 선언에서 추출** (design §3.4 그대로, 1C-Go 모델과 동일). 1B 의 workspace-path 변환과 다름. + - **Java**: tree-sitter-java 의 `package_declaration` → 안의 `scoped_identifier` 또는 `identifier` 텍스트 (e.g. `com.kebab.chunk`). 없으면 ``. + - **Kotlin**: tree-sitter-kotlin 의 `package_header` → `identifier` 텍스트. 없으면 (default package) ``. +- **Symbol 형식** (design §3.4): `package.Class.method`. 예시: `com.kebab.chunk.MdHeadingV1Chunker.chunkDoc`. +- **Java AST mapping**: + - `class_declaration` (name) → 1 unit + recurse body + - `interface_declaration` (name) → 1 unit + recurse + - `enum_declaration` (name) → 1 unit + - `record_declaration` (Java 14+) (name) → 1 unit + - `annotation_type_declaration` → 1 unit + - Inside class/interface/enum: `method_declaration` (name) → unit `package.Class.method` (class nesting like 1B Python) + - `import_declaration`, `package_declaration` 자체 → glue `` + - Top-level fn 없음 (Java 자체에 없음) +- **Kotlin AST mapping**: + - `class_declaration` (name) → 1 unit + recurse class_body. `data class` / `sealed class` / `enum class` 도 같은 노드. + - `object_declaration` (name) → 1 unit + recurse class_body (singleton) + - `function_declaration` (name) — **top-level 가능** → unit `package.fnName`. Class 내부면 `package.Class.method`. + - `property_declaration` at top-level → glue + - `interface` (in tree-sitter-kotlin 보통 `class_declaration` with `interface` modifier 또는 별 노드) → 1 unit + - `import_header`, `package_header` 자체 → glue `` +- **Glue grouping**: 1B Python / 1C-Go 패턴 동일 — imports + 기타 → 하나의 `` (또는 `` post-pass if file has zero real units). +- **Tree-sitter Kotlin crate 선택**: tree-sitter-kotlin 의 가장 잘 유지되는 crate 사용 (`tree-sitter-kotlin` 또는 fork). resolve 시 active maintainer 확인. +- frozen design 자체 변경 없음 — §10.1 에 1C-JK 활성화 한 줄. + +## Acceptance criteria + +- `cargo test --workspace --no-fail-fast -j 1` passes. +- `cargo clippy --workspace --all-targets -- -D warnings` passes. +- Java/Kotlin fixture 각각 (`tests/fixtures/sample.java`, `tests/fixtures/sample.kt`) ingest → chunk snapshot 안정 + symbol 이 §3.4 컨벤션 일치. +- 격리 TempDir KB 에 `.java` / `.kt` 파일 두고 `kebab search --code-lang java --json` / `--code-lang kotlin --json` 가 `Citation::Code` 반환. +- `kebab schema --json | jq .stats.code_lang_breakdown` 에 `"java"` + `"kotlin"` 카운트. +- README + HANDOFF + ARCHITECTURE + SMOKE + tasks/INDEX + tasks/p10/INDEX 갱신. +- frozen design §10.1 한 줄. +- workspace `Cargo.toml` minor bump (0.12.0 → 0.13.0). + +## Allowed dependencies + +- `kebab-parse-code` 에 `tree-sitter-java` + `tree-sitter-kotlin` 추가. 기존 deps 유지. +- `kebab-chunk` 의 새 모듈 2개 (`code_java_ast_v1.rs`, `code_kotlin_ast_v1.rs`) — language-agnostic body. tree-sitter import 금지. +- `kebab-app`, `kebab-source-fs` — 새 crate dep 없음. + +## Forbidden dependencies + +- `kebab-chunk` 가 tree-sitter-java / tree-sitter-kotlin import 금지 (boundary §6.3). +- UI crate 가 `kebab-parse-code` 직접 import 금지. +- `kebab-parse-code` 가 store / embed / llm / rag 직접 import 금지. + +## Risks / notes + +- tree-sitter-kotlin: 공식 또는 가장 활발히 유지되는 crate (`tree-sitter-kotlin` 또는 fork) 선택 필요. resolve 시 metadata 확인. +- Kotlin 의 grammar 가 다른 tree-sitter-* 보다 update 빈도 낮을 수 있어 grammar field 명 변동 가능 — 테스트 fixture 로 contract 고정. +- Java record (Java 14+) — tree-sitter-java 에서 `record_declaration` 노드 (확인 필요). +- Kotlin sealed class / data class / object declaration 등 변종 노드 — tree-sitter-kotlin 의 정확한 node kind 명 확인 필요 (grammar.json / node-types.json). +- Java class 안의 inner class — Python 패턴 (recursion with class name pushed) 동일 처리. +- Kotlin top-level fn 은 1B Python 의 top-level fn 패턴 + 1C-Go 의 package-prefix 패턴 hybrid — `package.fnName`. +- 머지 후 deviation 은 `tasks/HOTFIXES.md` dated 로그 + 본 spec `Risks / notes` cross-link.