From 2d68827cf5685fe78da45c39d57315321e782fb2 Mon Sep 17 00:00:00 2001 From: altair823 Date: Sat, 27 Jun 2026 01:06:14 +0000 Subject: [PATCH] =?UTF-8?q?refactor(core):=20=EB=B9=88=20re-export=20shim?= =?UTF-8?q?=20crate=20kebab-embed/kebab-llm=20=E2=86=92=20kebab-core=20?= =?UTF-8?q?=ED=9D=A1=EC=88=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit kebab-embed/kebab-llm 은 "새 type 없음"을 자처한 순수 re-export 셸이었다 (trait 은 이미 kebab-core 소유, mock + test helper 만 보유). "kebab-core 재구성 시 안정 surface" 라는 명분은 1인 RAG 엔 speculative YAGNI. 흡수: - kebab-core 에 default-OFF `mock` feature + src/mock.rs (MockEmbedder, MockLanguageModel, assert_vector_shape/assert_unit_norm/assert_finish_chunk 을 kebab_core:: → crate:: import 만 바꿔 verbatim 이동). - production import 2곳(kebab-embed-local, kebab-llm-local) + test import 다수(search/rag/parse-image/embed-local) 를 kebab_core 로 repoint. mock 쓰는 crate 는 dev-dep 에 features=["mock"] (default 빌드 무영향). - shim 자체 테스트: mock 동작 테스트는 kebab-core/tests/ 로 이동, reexports.rs(셸 재수출 테스트)는 폐기. - crates/kebab-embed, crates/kebab-llm 삭제 + workspace member/deps 정리. - ARCHITECTURE/HANDOFF/component README 의 crate 그래프·표·rationale 갱신 (22 → 20 crates). llm-local 의 broken intra-doc link 2건도 정리. trait surface·동작 불변 (test-only + import-rename). workspace build 는 mock default-OFF 라 mock 코드 미컴파일. 적대적 검증 3렌즈(build-test-integrity + behavior-identity[mock byte-identical] + dead-crate-completeness) 통과, clippy --workspace -D warnings 클린. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_012Mc6W1fgsrbFKTsqA6P8La --- CLAUDE.md | 2 +- Cargo.lock | 32 +- Cargo.toml | 2 - HANDOFF.md | 1 + crates/kebab-app/Cargo.toml | 6 - crates/kebab-core/Cargo.toml | 13 + crates/kebab-core/src/lib.rs | 6 + crates/kebab-core/src/mock.rs | 319 ++++++++++++++++++ .../tests/mock_embedder.rs} | 7 +- .../tests/mock_language_model.rs} | 7 +- crates/kebab-embed-local/Cargo.toml | 6 +- crates/kebab-embed-local/src/lib.rs | 6 +- crates/kebab-embed-local/tests/embed_model.rs | 8 +- crates/kebab-embed/Cargo.toml | 33 -- crates/kebab-embed/src/lib.rs | 76 ----- crates/kebab-embed/src/mock.rs | 142 -------- crates/kebab-embed/tests/reexports.rs | 61 ---- crates/kebab-llm-local/Cargo.toml | 1 - crates/kebab-llm-local/src/lib.rs | 19 +- crates/kebab-llm-local/src/ollama.rs | 2 +- crates/kebab-llm/Cargo.toml | 24 -- crates/kebab-llm/src/lib.rs | 49 --- crates/kebab-llm/src/mock.rs | 115 ------- crates/kebab-llm/tests/reexports.rs | 75 ---- crates/kebab-parse-image/Cargo.toml | 15 +- crates/kebab-parse-image/src/caption.rs | 2 +- crates/kebab-parse-image/tests/caption.rs | 4 +- crates/kebab-rag/Cargo.toml | 3 +- crates/kebab-rag/tests/pipeline.rs | 2 +- .../tests/prompt_template_dispatch.rs | 2 +- crates/kebab-rag/tests/streaming_events.rs | 2 +- crates/kebab-search/Cargo.toml | 5 +- crates/kebab-search/tests/common/mod.rs | 2 +- docs/ARCHITECTURE.md | 16 +- docs/components/README.md | 2 +- docs/components/embed/README.md | 12 +- docs/components/llm/README.md | 12 +- docs/components/rag/README.md | 2 +- docs/components/search/README.md | 4 +- 39 files changed, 409 insertions(+), 688 deletions(-) create mode 100644 crates/kebab-core/src/mock.rs rename crates/{kebab-embed/tests/mock.rs => kebab-core/tests/mock_embedder.rs} (94%) rename crates/{kebab-llm/tests/mock.rs => kebab-core/tests/mock_language_model.rs} (95%) delete mode 100644 crates/kebab-embed/Cargo.toml delete mode 100644 crates/kebab-embed/src/lib.rs delete mode 100644 crates/kebab-embed/src/mock.rs delete mode 100644 crates/kebab-embed/tests/reexports.rs delete mode 100644 crates/kebab-llm/Cargo.toml delete mode 100644 crates/kebab-llm/src/lib.rs delete mode 100644 crates/kebab-llm/src/mock.rs delete mode 100644 crates/kebab-llm/tests/reexports.rs diff --git a/CLAUDE.md b/CLAUDE.md index 906b15f..c548638 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,7 +4,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## Project -Single-user local-first knowledge base + RAG. Rust 2024 workspace, 22 crates, single binary (`kebab`). All inference is local (Ollama + fastembed + whisper.cpp). +Single-user local-first knowledge base + RAG. Rust 2024 workspace, 20 crates, single binary (`kebab`). All inference is local (Ollama + fastembed + whisper.cpp). The repo's documentation is split by audience — don't duplicate across them: diff --git a/Cargo.lock b/Cargo.lock index 42ae9af..ab9bbc7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4248,10 +4248,8 @@ dependencies = [ "kebab-chunk", "kebab-config", "kebab-core", - "kebab-embed", "kebab-embed-local", "kebab-embed-ollama", - "kebab-llm", "kebab-llm-local", "kebab-nli", "kebab-parse-code", @@ -4340,6 +4338,7 @@ version = "0.31.0" dependencies = [ "anyhow", "blake3", + "proptest", "serde", "serde_json", "serde_json_canonicalizer", @@ -4348,20 +4347,6 @@ dependencies = [ "unicode-normalization", ] -[[package]] -name = "kebab-embed" -version = "0.31.0" -dependencies = [ - "anyhow", - "blake3", - "kebab-config", - "kebab-core", - "proptest", - "serde", - "thiserror 2.0.18", - "tracing", -] - [[package]] name = "kebab-embed-local" version = "0.31.0" @@ -4369,7 +4354,7 @@ dependencies = [ "anyhow", "fastembed", "kebab-config", - "kebab-embed", + "kebab-core", "serde_json", "tempfile", "tracing", @@ -4409,15 +4394,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "kebab-llm" -version = "0.31.0" -dependencies = [ - "anyhow", - "kebab-core", - "proptest", -] - [[package]] name = "kebab-llm-local" version = "0.31.0" @@ -4425,7 +4401,6 @@ dependencies = [ "anyhow", "kebab-config", "kebab-core", - "kebab-llm", "reqwest 0.12.28", "serde", "serde_json", @@ -4504,7 +4479,6 @@ dependencies = [ "kamadak-exif", "kebab-config", "kebab-core", - "kebab-llm", "kebab-llm-local", "ndarray", "ort", @@ -4559,7 +4533,6 @@ dependencies = [ "blake3", "kebab-config", "kebab-core", - "kebab-llm", "kebab-nli", "kebab-search", "kebab-store-sqlite", @@ -4581,7 +4554,6 @@ dependencies = [ "globset", "kebab-config", "kebab-core", - "kebab-embed", "kebab-store-sqlite", "kebab-store-vector", "rusqlite", diff --git a/Cargo.toml b/Cargo.toml index e2d04e1..7a21f34 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,10 +9,8 @@ members = [ "crates/kebab-store-sqlite", "crates/kebab-store-vector", "crates/kebab-search", - "crates/kebab-embed", "crates/kebab-embed-local", "crates/kebab-embed-ollama", - "crates/kebab-llm", "crates/kebab-llm-local", "crates/kebab-rag", "crates/kebab-app", diff --git a/HANDOFF.md b/HANDOFF.md index 6f290a2..e57a9ab 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -35,6 +35,7 @@ P0~P5 직렬. P6~P9 P5 이후 병렬 가능. 머지 후 발견된 모든 deviation / hotfix 의 dated 로그는 [tasks/HOTFIXES.md](tasks/HOTFIXES.md). 본 요약은 \"누군가가 인수받을 때 알아두면 시간을 많이 절약하는\" 항목만: +- **2026-06-27 ponytail-audit 정리 arc (over-engineering 제거)** — 1인 RAG 가 과복잡해진 표면·구조를 감사 후 정리(능력 불변). (1) #219 죽은 search-cache scaffold 제거 — #214 spine 에서 LRU 캐시를 없앤 뒤 남은 `App::search_uncached`/`search_uncached_with_config` facade/`search --no-cache`·`--explain` 플래그/`explain_default` config/관련 주석. `search()` 는 byte-identical(본문 verbatim 이동), `search_cache: false` wire capability 만 유지. (2) #220 9개 동일 code AST chunker(`code_*_ast_v1.rs`) → 단일 `CodeAstV1Chunker { version_label }` + `for_lang(lang)` 통합 (**−3030줄**). chunker 는 tree-sitter 미사용·lang 은 SourceSpan 데이터 → struct 차이는 VERSION_LABEL 문자열뿐. 라벨 verbatim 유지 → chunk_id byte-identical → **재인덱싱 0**; 9개 골든 스냅샷이 expected 무수정 통과로 증명. (3) pure re-export shim 이던 `kebab-embed`/`kebab-llm`(trait 은 이미 kebab-core 소유, mock+test helper 만 보유)를 kebab-core 의 default-OFF `mock` feature 로 흡수 — **22 → 20 crates**, trait surface·동작 불변(test-only + import-rename churn, dev-dep `features=["mock"]`). 후속(작은 tail): FusionPolicy 1-arm enum→inline, NliVerifier default-0 shim 제거, dual-YAML 통일. 자세한 내용: 각 PR(#219/#220/…) + `docs/ARCHITECTURE.md`. - **2026-06-24 md-heading-v2: 예산 초과 청크 일반 분할** — v0.30.0. markdown 청커가 v1 의 "블록 미분할" 한계를 일반화 — 거대 list/code/table/paragraph 가 한 청크로 임베더 ctx 를 초과하던 문제를, `token_estimate > max_chunk_tokens`(신규 config, byte/3, default 4000)인 청크만 줄(→UTF-8 char) 경계로 분할해 해소. 미분할 청크는 v1 과 byte-identical. 분할 조각 chunk_id 는 `#seg{i}` 접미사로 충돌 회피, `max_chunk_tokens` 는 v2 policy_hash 에 fold(공유 ChunkPolicy 미변경). `chunker_version` v1→v2 라 다음 plain ingest 에서 markdown 1회 자동 재청크(코드/PDF 무영향). **동기**: strict 임베더(AMD Lemonade `/api/embed`)는 oversize 입력을 truncate 아닌 거부(`500 too large`) — ollama 가 조용히 truncate 하던 걸 청커가 애초에 안 만들도록. **known limitation**: 분할 조각 citation 은 블록 단위(sub-line 정밀 아님). 도그푸딩(실험 KB, arctic@Lemonade): v2 전 620 중 2 doc 임베드 실패 → v2 후 620/620·7114 청크 전부 ≤4000, "WiredTiger excessive memory" 질의에 거대 doc SERVER-22906 가 1위(0.977). 자세한 내용: `tasks/HOTFIXES.md` (2026-06-24), 설계 `docs/superpowers/plans/2026-06-24-md-heading-v2-oversize-split.md`. - **2026-06-21 provenance 출처 필터: `[[workspace.sources]]` 멀티소스 + `--source`/`--source-type`** — v0.29.0. 혼합 출처 KB(위키+jira 등)에서 색인은 전부 하되 질의 시 출처로 좁히는 레버. config `[[workspace.sources]]`(각 id/root/trust_level/source_type) + `documents.source_id` 컬럼(V014, additive, 재색인 0) + config v3→v4 migration(`step_3_to_4`, 단일 root→implicit `default` source, 멱등) + 검색 `--source ` / `--source-type `(lexical+vector 두 site, OR). trust precedence = frontmatter > per-source 기본값 > Primary. **설계 근거**: 전역 trust 곱셈가중(weighted-RRF)은 A/B 에서 반증(θ=0.85 만으로 incident MRR 0.918→0.340 절벽) — 필터가 see-saw 없는 올바른 레버. 도그푸딩(620 doc, jira400+wiki220): `--source wiki` concept 0.780→0.810, `--source jira` incident 0.918→0.975. **follow-up**: MCP search 필터 미노출 · `kebab list` source_id 미표시 · RAG provenance 라벨 미구현. 자세한 내용: `tasks/HOTFIXES.md` (2026-06-21). - **2026-06-04 PP-OCRv5 ONNX Rust 네이티브 OCR** — v0.27.0. `[image.ocr] engine = "paddle-onnx"` 로 PP-OCRv5(검출+인식) ONNX 를 in-process(`ort` =2.0.0-rc.9) 실행 — Python 런타임/원격 호출 없이 큰 페이지 CPU <4초(Ollama vision ~50초 대비). default 는 여전히 `"ollama-vision"`. 후처리(min-area rect/unclip)는 pure-Rust. **함정**: unclip 은 corner 를 centroid 에서 방사 확장하면 안 되고 edge 별 polygon offset 이어야 함(방사 확장 시 wide/short 텍스트 박스 높이가 안 커져 글자 윗부분 잘림 → ㄷ→ㄴ, e2e CER 0.26). 수정 후 CER 0.005. 모델 ONNX 는 `crates/kebab-parse-image/assets/paddleocr-onnx/`(LFS). 자세한 내용: `tasks/HOTFIXES.md` (2026-06-04 PP-OCRv5 ONNX), spec/plan `docs/superpowers/{specs,plans}/2026-06-04-rust-native-ocr-*.md`. diff --git a/crates/kebab-app/Cargo.toml b/crates/kebab-app/Cargo.toml index dcff92c..c540097 100644 --- a/crates/kebab-app/Cargo.toml +++ b/crates/kebab-app/Cargo.toml @@ -16,10 +16,8 @@ kebab-chunk = { path = "../kebab-chunk" } kebab-store-sqlite = { path = "../kebab-store-sqlite" } kebab-store-vector = { path = "../kebab-store-vector" } kebab-search = { path = "../kebab-search" } -kebab-embed = { path = "../kebab-embed" } kebab-embed-local = { path = "../kebab-embed-local" } kebab-embed-ollama = { path = "../kebab-embed-ollama" } -kebab-llm = { path = "../kebab-llm" } kebab-llm-local = { path = "../kebab-llm-local" } kebab-rag = { path = "../kebab-rag" } # p9-fb-41 PR-9c-2: facade construction of OnnxNliVerifier when @@ -67,10 +65,6 @@ rusqlite = { workspace = true } [dev-dependencies] kebab-config = { path = "../kebab-config" } -# doc-side expansion (Phase 2) Task 4: ExpansionGenerator unit tests build -# MockLanguageModel (gated behind kebab-llm's `mock` feature, default OFF in -# [dependencies]). Enabling it here turns it on for the test build only. -kebab-llm = { path = "../kebab-llm", features = ["mock"] } rusqlite = { workspace = true } filetime = "0.2" tempfile = { workspace = true } diff --git a/crates/kebab-core/Cargo.toml b/crates/kebab-core/Cargo.toml index 5b9c603..792e5d6 100644 --- a/crates/kebab-core/Cargo.toml +++ b/crates/kebab-core/Cargo.toml @@ -17,5 +17,18 @@ blake3 = { workspace = true } serde_json_canonicalizer = "0.3" unicode-normalization = "0.1" +[features] +default = [] +# Opt-in `MockEmbedder` / `MockLanguageModel` + test helpers (src/mock.rs). +# Default OFF so release builds (no `--features mock`) compile the symbols out +# entirely (verifiable via `nm`/`cargo bloat`). Moved here from the deleted +# `kebab-embed` / `kebab-llm` re-export shim crates. +mock = [] + +[dev-dependencies] +# For the mock tests (tests/mock_embedder.rs, tests/mock_language_model.rs), +# which run only under `--features mock`. +proptest = { workspace = true } + [lints] workspace = true diff --git a/crates/kebab-core/src/lib.rs b/crates/kebab-core/src/lib.rs index 9cbc845..78ef561 100644 --- a/crates/kebab-core/src/lib.rs +++ b/crates/kebab-core/src/lib.rs @@ -20,6 +20,8 @@ pub mod ingest; pub mod jobs; pub mod media; pub mod metadata; +#[cfg(feature = "mock")] +mod mock; pub mod normalize; pub mod search; pub mod traits; @@ -53,6 +55,10 @@ pub use ingest::{IngestItem, IngestItemKind, IngestReport, SkipExamples}; pub use jobs::{JobFilter, JobId, JobKind, JobRow, JobStatus}; pub use media::{AudioType, Checksum, ImageType, Lang, MediaType}; pub use metadata::{Metadata, Provenance, ProvenanceEvent, ProvenanceKind, SourceType, TrustLevel}; +#[cfg(feature = "mock")] +pub use mock::{ + MockEmbedder, MockLanguageModel, assert_finish_chunk, assert_unit_norm, assert_vector_shape, +}; pub use normalize::{nfc, to_posix}; pub use search::{ BulkSearchItem, BulkSearchResponse, BulkSearchSummary, DocFilter, DocSummary, IndexBytes, diff --git a/crates/kebab-core/src/mock.rs b/crates/kebab-core/src/mock.rs new file mode 100644 index 0000000..a245b6a --- /dev/null +++ b/crates/kebab-core/src/mock.rs @@ -0,0 +1,319 @@ +//! Deterministic mock `Embedder` / `LanguageModel` + test helpers. +//! +//! Compiled only when the `mock` feature is enabled. Default builds +//! (`cargo build`, no `--features mock`) MUST NOT contain the `MockEmbedder` / +//! `MockLanguageModel` symbols — verifiable by symbol scan (`nm`/`cargo bloat`). +//! +//! Moved here verbatim from the former `kebab-embed` / `kebab-llm` re-export +//! shim crates (folded into `kebab-core`); those crates defined no new types. +//! +//! # `MockEmbedder` determinism contract +//! +//! For every call to [`MockEmbedder::embed`], component `i` of the output +//! vector for input `(text, kind)` is computed as: +//! +//! ```text +//! h = blake3(seed_le8 || kind_byte || text_len_le8 || text_utf8 || i_le8) +//! raw_i64 = i64::from_le_bytes(h[0..8]) +//! comp = (raw_i64 as f64 / i64::MAX as f64) as f32 // ∈ [-1.0, 1.0] +//! ``` +//! +//! `kind_byte` is `0u8` for [`EmbeddingKind::Document`] and `1u8` for +//! [`EmbeddingKind::Query`] — mirrors the e5-style prefix behavior (the same +//! text in different roles produces different vectors). `text_len_le8` is the +//! length of `text_utf8` (in bytes) as a little-endian `u64`; it provides +//! domain separation so the boundary between `text` and the trailing `i_le8` +//! cannot be ambiguous (without it, e.g. `("ABCDEFGH", 0)` and +//! `("", u64::from_le_bytes(*b"ABCDEFGH"))` would hash identically). +//! +//! After the per-component pass each vector is **L2-normalized to unit +//! length** so downstream cosine-similarity tests can rely on a unit-norm +//! input (‖v‖ ≈ 1.0 within f32 epsilon × √dims — the per-component f32 +//! truncation is bounded by `f32::EPSILON`, summed in quadrature gives +//! roughly `√dims · EPSILON` in the L2 norm). If a vector ends up all-zeros +//! (vanishingly unlikely from BLAKE3), it is left untouched rather than +//! dividing by zero. +//! +//! Invariants the contract guarantees: +//! +//! * Identical `(seed, kind, text, dimensions)` → byte-identical output. +//! * Different `kind` for the same text → different output (kind_byte differs). +//! * Different `text` → different output with overwhelming probability. +//! * All output components are finite (`is_finite()`). +//! +//! # `MockLanguageModel` streaming contract +//! +//! For every call to [`MockLanguageModel::generate_stream`]: +//! +//! 1. The configured `canned_response` is examined for any of `req.stop`. If +//! one or more stop strings are substrings of the response, the response +//! is truncated at the **earliest byte position** of any match (i.e., the +//! first stop string to land — ties broken by the order entries appear in +//! `req.stop`, since `Iterator::min` returns the first equal element on +//! ties, breaking by `req.stop` declaration order). +//! 2. The (possibly truncated) string is iterated by Unicode scalar +//! (`str::chars()`) and each character is yielded as +//! [`TokenChunk::Token`]`(c.to_string())`. This makes streaming UTF-8 safe +//! by construction (no character is split across chunks). Emits one +//! `TokenChunk` per Unicode scalar value (`char`), not per grapheme +//! cluster — Hangul jamo, emoji ZWJ sequences, and combining marks split +//! into multiple chunks. Acceptable for trait-shape testing; real adapters +//! MAY combine. +//! 3. After all tokens, a single terminal [`TokenChunk::Done`] is yielded +//! with: +//! * `finish_reason = FinishReason::Stop` if a stop string truncated the +//! canned text — mirroring real LLM behavior, which reports Stop on +//! stop-sequence termination regardless of the configured finish. +//! * `finish_reason = canned_finish.clone()` otherwise. +//! * `usage = canned_usage.clone()` always. +//! +//! No network. No filesystem. No async runtime. No tokenizer — `usage` fields +//! are whatever the constructor was given. + +use crate::{ + Embedder, EmbeddingInput, EmbeddingKind, EmbeddingModelId, EmbeddingVersion, FinishReason, + GenerateRequest, LanguageModel, ModelRef, TokenChunk, TokenUsage, +}; + +// ── Embed test helpers ──────────────────────────────────────────────────── + +/// Assert every vector has length `expected_dims` and contains only finite +/// floats. Intended for downstream test crates so they don't each rewrite the +/// shape check. +/// +/// Panics on mismatch (test-only helper — callers are tests). +pub fn assert_vector_shape(vecs: &[Vec], expected_dims: usize) { + for (i, v) in vecs.iter().enumerate() { + assert_eq!( + v.len(), + expected_dims, + "vector {i}: dims {} != expected {expected_dims}", + v.len(), + ); + for (j, x) in v.iter().enumerate() { + assert!(x.is_finite(), "vector {i}[{j}] = {x} is not finite"); + } + } +} + +/// Assert every vector has L2 norm within `tolerance` of `1.0`. +/// +/// L2 norm is computed in `f64` (per-component square accumulation in `f64` +/// then `sqrt`) before truncating back to `f32`, so the comparison is not +/// dominated by accumulation error in the check itself — only the f32 +/// truncation of the input vector's components contributes. +/// +/// Tolerance guidance: callers pass their own. For `dims = 384` and +/// f32-truncated unit vectors, `5e-4` is a safe upper bound under quadratic +/// accumulation of per-component f32 truncation (`f32::EPSILON × √dims`). +/// Smaller dims tolerate tighter bounds; larger dims need looser ones. +/// +/// Panics on mismatch (test-only helper — callers are tests). +pub fn assert_unit_norm(vecs: &[Vec], tolerance: f32) { + for (i, v) in vecs.iter().enumerate() { + let norm_sq: f64 = v.iter().map(|&x| f64::from(x) * f64::from(x)).sum(); + let norm = norm_sq.sqrt() as f32; + assert!( + (norm - 1.0).abs() <= tolerance, + "vector {i}: ‖v‖ = {norm} (off from 1.0 by {})", + (norm - 1.0).abs(), + ); + } +} + +// ── LLM test helper ─────────────────────────────────────────────────────── + +/// Assert the streamed `TokenChunk` sequence ends with a [`TokenChunk::Done`] +/// frame. Per spec §7.2 / §0 Q5 every stream — even an erroring one — must +/// terminate with a `Done` chunk; this helper centralizes that contract check +/// so downstream test crates don't each rewrite it. +/// +/// Panics on mismatch (test-only helper — callers are tests). +pub fn assert_finish_chunk(chunks: &[TokenChunk]) { + assert!( + matches!(chunks.last(), Some(TokenChunk::Done { .. })), + "stream must end with TokenChunk::Done; got {:?}", + chunks.last(), + ); +} + +// ── MockEmbedder ────────────────────────────────────────────────────────── + +/// Deterministic test double. See module docs for the hashing recipe. +pub struct MockEmbedder { + model_id: EmbeddingModelId, + version: EmbeddingVersion, + dimensions: usize, + seed: u64, +} + +impl MockEmbedder { + /// Construct with `seed = 0`. Use [`Self::with_seed`] to pick a different + /// seed (e.g., to verify two embedders with the same identity but + /// different seeds yield different vectors). + pub fn new(model_id: EmbeddingModelId, version: EmbeddingVersion, dimensions: usize) -> Self { + Self { + model_id, + version, + dimensions, + seed: 0, + } + } + + /// Construct with an explicit seed. Useful for differential tests. + pub fn with_seed( + model_id: EmbeddingModelId, + version: EmbeddingVersion, + dimensions: usize, + seed: u64, + ) -> Self { + Self { + model_id, + version, + dimensions, + seed, + } + } + + fn kind_byte(kind: EmbeddingKind) -> u8 { + match kind { + EmbeddingKind::Document => 0, + EmbeddingKind::Query => 1, + } + } + + fn component(&self, kind: EmbeddingKind, text: &str, i: usize) -> f32 { + let mut hasher = blake3::Hasher::new(); + hasher.update(&self.seed.to_le_bytes()); + hasher.update(&[Self::kind_byte(kind)]); + // Length-prefix `text` (LE u64) so the boundary between `text` and the + // trailing `i` field is unambiguous — without this, `("ABCDEFGH", 0)` + // and `("", u64::from_le_bytes(*b"ABCDEFGH"))` would feed identical + // bytes into the hasher. + hasher.update(&(text.len() as u64).to_le_bytes()); + hasher.update(text.as_bytes()); + hasher.update(&(i as u64).to_le_bytes()); + let digest = hasher.finalize(); + let bytes = digest.as_bytes(); + let mut head = [0u8; 8]; + head.copy_from_slice(&bytes[..8]); + let raw = i64::from_le_bytes(head); + // Map to [-1.0, 1.0]. `i64::MAX` is finite in f64 so the ratio is + // always finite. Casting back to f32 cannot produce a NaN/Inf for + // values in this range. + // Note: i64::MIN/i64::MAX gives -1.0000000000000002 → f32 cast rounds to -1.0; range [-1, 1] holds in f32 even with this asymmetry. + ((raw as f64) / (i64::MAX as f64)) as f32 + } +} + +impl Embedder for MockEmbedder { + fn model_id(&self) -> EmbeddingModelId { + self.model_id.clone() + } + + fn model_version(&self) -> EmbeddingVersion { + self.version.clone() + } + + fn dimensions(&self) -> usize { + self.dimensions + } + + fn embed(&self, inputs: &[EmbeddingInput<'_>]) -> anyhow::Result>> { + let mut out = Vec::with_capacity(inputs.len()); + for input in inputs { + let mut v: Vec = (0..self.dimensions) + .map(|i| self.component(input.kind, input.text, i)) + .collect(); + + // L2-normalize. Skip the rare all-zero case to avoid 0/0 = NaN. + let norm_sq: f64 = v.iter().map(|&x| f64::from(x) * f64::from(x)).sum(); + if norm_sq > 0.0 { + let inv = (1.0 / norm_sq.sqrt()) as f32; + for x in &mut v { + *x *= inv; + } + } + out.push(v); + } + Ok(out) + } +} + +// ── MockLanguageModel ───────────────────────────────────────────────────── + +/// Deterministic test double. See module docs for the streaming recipe. +pub struct MockLanguageModel { + pub model_id: String, + pub provider: String, + pub context_tokens: usize, + pub canned_response: String, + pub canned_finish: FinishReason, + pub canned_usage: TokenUsage, +} + +impl MockLanguageModel { + /// Apply `req.stop` to `canned_response`. Returns `(truncated_text, + /// stop_hit)` where `stop_hit` is true iff any stop string was found. + fn apply_stop<'a>(canned: &'a str, stop: &[String]) -> (&'a str, bool) { + // Earliest byte position wins. Ties break by first occurrence in + // `stop` (Iterator::min returns the first equal element, and we + // iterate `stop` in its declared order). Empty stop strings are + // ignored — they would otherwise match at position 0 and silently + // eat the entire response. + let earliest = stop + .iter() + .filter(|s| !s.is_empty()) + .filter_map(|s| canned.find(s.as_str())) + .min(); + match earliest { + // `str::find` returns a UTF-8 char boundary by contract, so direct byte-slice is sound. + Some(idx) => (&canned[..idx], true), + None => (canned, false), + } + } +} + +impl LanguageModel for MockLanguageModel { + fn model_ref(&self) -> ModelRef { + ModelRef { + id: self.model_id.clone(), + provider: self.provider.clone(), + // Per §3.8: `dimensions` carries the embedder's output dim and is + // intentionally None for chat models. + dimensions: None, + } + } + + fn context_tokens(&self) -> usize { + self.context_tokens + } + + fn generate_stream( + &self, + req: GenerateRequest, + ) -> anyhow::Result> + Send>> { + let (truncated, stop_hit) = Self::apply_stop(&self.canned_response, &req.stop); + + // Pre-materialize the full chunk sequence into an owned Vec. This + // sidesteps lifetime juggling around `&self.canned_response` inside + // a `'static` iterator and trivially gives `Send` (Vec + // is Send because TokenChunk is Send). + let mut chunks: Vec = truncated + .chars() + .map(|c| TokenChunk::Token(c.to_string())) + .collect(); + + let finish_reason = if stop_hit { + FinishReason::Stop + } else { + self.canned_finish.clone() + }; + chunks.push(TokenChunk::Done { + finish_reason, + usage: self.canned_usage.clone(), + }); + + Ok(Box::new(chunks.into_iter().map(Ok))) + } +} diff --git a/crates/kebab-embed/tests/mock.rs b/crates/kebab-core/tests/mock_embedder.rs similarity index 94% rename from crates/kebab-embed/tests/mock.rs rename to crates/kebab-core/tests/mock_embedder.rs index d3ac109..796c43c 100644 --- a/crates/kebab-embed/tests/mock.rs +++ b/crates/kebab-core/tests/mock_embedder.rs @@ -1,10 +1,13 @@ //! Integration tests for `MockEmbedder`. Gated behind the `mock` feature. //! -//! Canonical invocation: `cargo test -p kb-embed --features mock`. +//! Canonical invocation: `cargo test -p kebab-core --features mock`. +//! (Without `--features mock` this file compiles to nothing — the `cfg` gate +//! below short-circuits, since the mock lives in `kebab-core`'s own optional +//! `mock` module and cannot be enabled via a self dev-dependency.) #![cfg(feature = "mock")] -use kebab_embed::{ +use kebab_core::{ Embedder, EmbeddingInput, EmbeddingKind, EmbeddingModelId, EmbeddingVersion, MockEmbedder, assert_unit_norm, assert_vector_shape, }; diff --git a/crates/kebab-llm/tests/mock.rs b/crates/kebab-core/tests/mock_language_model.rs similarity index 95% rename from crates/kebab-llm/tests/mock.rs rename to crates/kebab-core/tests/mock_language_model.rs index 97fe2e0..1effb59 100644 --- a/crates/kebab-llm/tests/mock.rs +++ b/crates/kebab-core/tests/mock_language_model.rs @@ -1,10 +1,13 @@ //! Integration tests for `MockLanguageModel`. Gated behind the `mock` feature. //! -//! Canonical invocation: `cargo test -p kb-llm --features mock`. +//! Canonical invocation: `cargo test -p kebab-core --features mock`. +//! (Without `--features mock` this file compiles to nothing — the `cfg` gate +//! below short-circuits, since the mock lives in `kebab-core`'s own optional +//! `mock` module and cannot be enabled via a self dev-dependency.) #![cfg(feature = "mock")] -use kebab_llm::{ +use kebab_core::{ FinishReason, GenerateRequest, LanguageModel, MockLanguageModel, TokenChunk, TokenUsage, assert_finish_chunk, }; diff --git a/crates/kebab-embed-local/Cargo.toml b/crates/kebab-embed-local/Cargo.toml index b0b2085..44aa918 100644 --- a/crates/kebab-embed-local/Cargo.toml +++ b/crates/kebab-embed-local/Cargo.toml @@ -8,8 +8,8 @@ repository = { workspace = true } description = "Local fastembed-rs adapter implementing kb_core::Embedder (multilingual-e5-large default, e5-small backwards-compat)" [dependencies] +kebab-core = { path = "../kebab-core" } kebab-config = { path = "../kebab-config" } -kebab-embed = { path = "../kebab-embed" } # Default features bring `ort-download-binaries` (bundled ONNX runtime) # and `hf-hub-native-tls` (first-run model download). No extra features # needed for the multilingual-e5-{small,large} paths. @@ -18,6 +18,10 @@ tracing = { workspace = true } anyhow = { workspace = true } [dev-dependencies] +# `tests/embed_model.rs` uses `kebab_core::assert_unit_norm` / +# `assert_vector_shape` (the `mock` module's test helpers, gated OFF by +# default). Enable the feature for the test build only. +kebab-core = { path = "../kebab-core", features = ["mock"] } tempfile = { workspace = true } serde_json = { workspace = true } diff --git a/crates/kebab-embed-local/src/lib.rs b/crates/kebab-embed-local/src/lib.rs index 3cf170f..939093c 100644 --- a/crates/kebab-embed-local/src/lib.rs +++ b/crates/kebab-embed-local/src/lib.rs @@ -1,5 +1,5 @@ //! `kb-embed-local` — `FastembedEmbedder`, a local ONNX-backed -//! [`Embedder`](kebab_embed::Embedder) implementation. +//! [`Embedder`](kebab_core::Embedder) implementation. //! //! Wraps [`fastembed::TextEmbedding`]. Default is `multilingual-e5-large` //! (1024-dim, p9-fb-39b); `multilingual-e5-small` (384-dim) is also supported @@ -29,7 +29,7 @@ use std::sync::Mutex; use anyhow::{Context, Result}; use fastembed::{EmbeddingModel, InitOptions, TextEmbedding}; use kebab_config::EmbeddingModelCfg; -use kebab_embed::{Embedder, EmbeddingInput, EmbeddingKind, EmbeddingModelId, EmbeddingVersion}; +use kebab_core::{Embedder, EmbeddingInput, EmbeddingKind, EmbeddingModelId, EmbeddingVersion}; /// Subdirectory under `config.storage.model_dir` where the fastembed /// adapter writes / reads ONNX + tokenizer files. Hard-coded per task @@ -224,7 +224,7 @@ pub(crate) fn check_dim(model_dim: usize, cfg_dim: usize) -> Result<()> { #[cfg(test)] mod tests { use super::*; - use kebab_embed::EmbeddingInput; + use kebab_core::EmbeddingInput; // ── check_dim ──────────────────────────────────────────────────── // diff --git a/crates/kebab-embed-local/tests/embed_model.rs b/crates/kebab-embed-local/tests/embed_model.rs index 82dd0ad..9a2adcb 100644 --- a/crates/kebab-embed-local/tests/embed_model.rs +++ b/crates/kebab-embed-local/tests/embed_model.rs @@ -23,7 +23,7 @@ use std::hash::{Hash, Hasher}; use std::sync::OnceLock; use std::time::Instant; -use kebab_embed::{Embedder, EmbeddingInput, EmbeddingKind}; +use kebab_core::{Embedder, EmbeddingInput, EmbeddingKind}; use kebab_embed_local::{FASTEMBED_CACHE_SUBDIR, FastembedEmbedder}; /// Resolve the fastembed cache dir from a `Config`'s storage paths, @@ -149,12 +149,12 @@ fn output_vectors_are_l2_normalized() { }, ]; let out = emb.embed(&inputs).expect("embed"); - // Per `kebab_embed::assert_unit_norm` docs: `5e-4` is the safe bound at + // Per `kebab_core::assert_unit_norm` docs: `5e-4` is the safe bound at // 1024 dims (f32::EPSILON × √1024 ≈ 2.3e-6, but ONNX kernels add // their own per-component noise; 1e-3 is very generous and matches // the spec's `± 1e-3`). - kebab_embed::assert_unit_norm(&out, 1e-3); - kebab_embed::assert_vector_shape(&out, 1024); + kebab_core::assert_unit_norm(&out, 1e-3); + kebab_core::assert_vector_shape(&out, 1024); } // ─── determinism ────────────────────────────────────────────────────── diff --git a/crates/kebab-embed/Cargo.toml b/crates/kebab-embed/Cargo.toml deleted file mode 100644 index e16c144..0000000 --- a/crates/kebab-embed/Cargo.toml +++ /dev/null @@ -1,33 +0,0 @@ -[package] -name = "kebab-embed" -version = { workspace = true } -edition = { workspace = true } -rust-version = { workspace = true } -license = { workspace = true } -repository = { workspace = true } -description = "Embedder trait re-exports + opt-in deterministic MockEmbedder for downstream tests" - -[dependencies] -kebab-core = { path = "../kebab-core" } -kebab-config = { path = "../kebab-config" } -serde = { workspace = true } -thiserror = { workspace = true } -tracing = { workspace = true } -anyhow = { workspace = true } -# Used only by `MockEmbedder` (feature = "mock") for deterministic per-component -# hashing. Kept as an unconditional dep because `blake3` is already in the -# workspace lockfile (transitively via kb-core); pulling it in here adds zero -# build cost and keeps Cargo.toml simple. -blake3 = { workspace = true } - -[features] -default = [] -# Opt-in `MockEmbedder`. Default OFF so release builds (no `--features mock`) -# compile the symbol out entirely (verifiable via `nm`/`cargo bloat`). -mock = [] - -[dev-dependencies] -proptest = { workspace = true } - -[lints] -workspace = true diff --git a/crates/kebab-embed/src/lib.rs b/crates/kebab-embed/src/lib.rs deleted file mode 100644 index 000a556..0000000 --- a/crates/kebab-embed/src/lib.rs +++ /dev/null @@ -1,76 +0,0 @@ -//! `kb-embed` — thin re-export crate for the [`Embedder`] trait surface. -//! -//! This crate exists so downstream code (`kb-store-vector`, `kb-search`, -//! adapters in p3-2) can `use kebab_embed::Embedder` and stay stable across -//! kb-core reorganizations. It defines **no new types**; everything is a -//! re-export of [`kebab_core`]. -//! -//! ## Mock implementation -//! -//! [`MockEmbedder`] (gated behind the `mock` feature, default **OFF**) is a -//! deterministic test double. Real adapters (fastembed, candle, ollama-embed) -//! live in p3-2 and MUST NOT be implemented here. -//! -//! See `docs/superpowers/specs/2026-04-27-kebab-final-form-design.md` §7.1, §7.2, -//! §11 for the contract. - -// ── Trait re-exports ────────────────────────────────────────────────────── -// -// Per spec §7.2 — these are the only public-surface types this crate offers. -// Adding new types is forbidden by the task contract. - -pub use kebab_core::{Embedder, EmbeddingInput, EmbeddingKind, EmbeddingModelId, EmbeddingVersion}; - -// ── Test helper ─────────────────────────────────────────────────────────── - -/// Assert every vector has length `expected_dims` and contains only finite -/// floats. Intended for downstream test crates so they don't each rewrite the -/// shape check. -/// -/// Panics on mismatch (test-only helper — callers are tests). -pub fn assert_vector_shape(vecs: &[Vec], expected_dims: usize) { - for (i, v) in vecs.iter().enumerate() { - assert_eq!( - v.len(), - expected_dims, - "vector {i}: dims {} != expected {expected_dims}", - v.len(), - ); - for (j, x) in v.iter().enumerate() { - assert!(x.is_finite(), "vector {i}[{j}] = {x} is not finite"); - } - } -} - -/// Assert every vector has L2 norm within `tolerance` of `1.0`. -/// -/// L2 norm is computed in `f64` (per-component square accumulation in `f64` -/// then `sqrt`) before truncating back to `f32`, so the comparison is not -/// dominated by accumulation error in the check itself — only the f32 -/// truncation of the input vector's components contributes. -/// -/// Tolerance guidance: callers pass their own. For `dims = 384` and -/// f32-truncated unit vectors, `5e-4` is a safe upper bound under quadratic -/// accumulation of per-component f32 truncation (`f32::EPSILON × √dims`). -/// Smaller dims tolerate tighter bounds; larger dims need looser ones. -/// -/// Panics on mismatch (test-only helper — callers are tests). -pub fn assert_unit_norm(vecs: &[Vec], tolerance: f32) { - for (i, v) in vecs.iter().enumerate() { - let norm_sq: f64 = v.iter().map(|&x| f64::from(x) * f64::from(x)).sum(); - let norm = norm_sq.sqrt() as f32; - assert!( - (norm - 1.0).abs() <= tolerance, - "vector {i}: ‖v‖ = {norm} (off from 1.0 by {})", - (norm - 1.0).abs(), - ); - } -} - -// ── MockEmbedder (feature = "mock") ─────────────────────────────────────── - -#[cfg(feature = "mock")] -mod mock; - -#[cfg(feature = "mock")] -pub use mock::MockEmbedder; diff --git a/crates/kebab-embed/src/mock.rs b/crates/kebab-embed/src/mock.rs deleted file mode 100644 index fe1131b..0000000 --- a/crates/kebab-embed/src/mock.rs +++ /dev/null @@ -1,142 +0,0 @@ -//! Deterministic mock embedder for downstream tests. -//! -//! Compiled only when the `mock` feature is enabled. Default builds -//! (`cargo build --release -p kb-embed`) MUST NOT contain the `MockEmbedder` -//! symbol — verifiable by symbol scan (`nm`, `cargo bloat`). -//! -//! ## Determinism contract -//! -//! For every call to [`MockEmbedder::embed`], component `i` of the output -//! vector for input `(text, kind)` is computed as: -//! -//! ```text -//! h = blake3(seed_le8 || kind_byte || text_len_le8 || text_utf8 || i_le8) -//! raw_i64 = i64::from_le_bytes(h[0..8]) -//! comp = (raw_i64 as f64 / i64::MAX as f64) as f32 // ∈ [-1.0, 1.0] -//! ``` -//! -//! `kind_byte` is `0u8` for [`EmbeddingKind::Document`] and `1u8` for -//! [`EmbeddingKind::Query`] — mirrors the e5-style prefix behavior (the same -//! text in different roles produces different vectors). `text_len_le8` is the -//! length of `text_utf8` (in bytes) as a little-endian `u64`; it provides -//! domain separation so the boundary between `text` and the trailing `i_le8` -//! cannot be ambiguous (without it, e.g. `("ABCDEFGH", 0)` and -//! `("", u64::from_le_bytes(*b"ABCDEFGH"))` would hash identically). -//! -//! After the per-component pass each vector is **L2-normalized to unit -//! length** so downstream cosine-similarity tests can rely on a unit-norm -//! input (‖v‖ ≈ 1.0 within f32 epsilon × √dims — the per-component f32 -//! truncation is bounded by `f32::EPSILON`, summed in quadrature gives -//! roughly `√dims · EPSILON` in the L2 norm). If a vector ends up all-zeros -//! (vanishingly unlikely from BLAKE3), it is left untouched rather than -//! dividing by zero. -//! -//! Invariants the contract guarantees: -//! -//! * Identical `(seed, kind, text, dimensions)` → byte-identical output. -//! * Different `kind` for the same text → different output (kind_byte differs). -//! * Different `text` → different output with overwhelming probability. -//! * All output components are finite (`is_finite()`). - -use kebab_core::{Embedder, EmbeddingInput, EmbeddingKind, EmbeddingModelId, EmbeddingVersion}; - -/// Deterministic test double. See module docs for the hashing recipe. -pub struct MockEmbedder { - model_id: EmbeddingModelId, - version: EmbeddingVersion, - dimensions: usize, - seed: u64, -} - -impl MockEmbedder { - /// Construct with `seed = 0`. Use [`Self::with_seed`] to pick a different - /// seed (e.g., to verify two embedders with the same identity but - /// different seeds yield different vectors). - pub fn new(model_id: EmbeddingModelId, version: EmbeddingVersion, dimensions: usize) -> Self { - Self { - model_id, - version, - dimensions, - seed: 0, - } - } - - /// Construct with an explicit seed. Useful for differential tests. - pub fn with_seed( - model_id: EmbeddingModelId, - version: EmbeddingVersion, - dimensions: usize, - seed: u64, - ) -> Self { - Self { - model_id, - version, - dimensions, - seed, - } - } - - fn kind_byte(kind: EmbeddingKind) -> u8 { - match kind { - EmbeddingKind::Document => 0, - EmbeddingKind::Query => 1, - } - } - - fn component(&self, kind: EmbeddingKind, text: &str, i: usize) -> f32 { - let mut hasher = blake3::Hasher::new(); - hasher.update(&self.seed.to_le_bytes()); - hasher.update(&[Self::kind_byte(kind)]); - // Length-prefix `text` (LE u64) so the boundary between `text` and the - // trailing `i` field is unambiguous — without this, `("ABCDEFGH", 0)` - // and `("", u64::from_le_bytes(*b"ABCDEFGH"))` would feed identical - // bytes into the hasher. - hasher.update(&(text.len() as u64).to_le_bytes()); - hasher.update(text.as_bytes()); - hasher.update(&(i as u64).to_le_bytes()); - let digest = hasher.finalize(); - let bytes = digest.as_bytes(); - let mut head = [0u8; 8]; - head.copy_from_slice(&bytes[..8]); - let raw = i64::from_le_bytes(head); - // Map to [-1.0, 1.0]. `i64::MAX` is finite in f64 so the ratio is - // always finite. Casting back to f32 cannot produce a NaN/Inf for - // values in this range. - // Note: i64::MIN/i64::MAX gives -1.0000000000000002 → f32 cast rounds to -1.0; range [-1, 1] holds in f32 even with this asymmetry. - ((raw as f64) / (i64::MAX as f64)) as f32 - } -} - -impl Embedder for MockEmbedder { - fn model_id(&self) -> EmbeddingModelId { - self.model_id.clone() - } - - fn model_version(&self) -> EmbeddingVersion { - self.version.clone() - } - - fn dimensions(&self) -> usize { - self.dimensions - } - - fn embed(&self, inputs: &[EmbeddingInput<'_>]) -> anyhow::Result>> { - let mut out = Vec::with_capacity(inputs.len()); - for input in inputs { - let mut v: Vec = (0..self.dimensions) - .map(|i| self.component(input.kind, input.text, i)) - .collect(); - - // L2-normalize. Skip the rare all-zero case to avoid 0/0 = NaN. - let norm_sq: f64 = v.iter().map(|&x| f64::from(x) * f64::from(x)).sum(); - if norm_sq > 0.0 { - let inv = (1.0 / norm_sq.sqrt()) as f32; - for x in &mut v { - *x *= inv; - } - } - out.push(v); - } - Ok(out) - } -} diff --git a/crates/kebab-embed/tests/reexports.rs b/crates/kebab-embed/tests/reexports.rs deleted file mode 100644 index 72203ab..0000000 --- a/crates/kebab-embed/tests/reexports.rs +++ /dev/null @@ -1,61 +0,0 @@ -//! Compile-only test: verifies the crate's public surface (trait re-exports -//! and the `assert_vector_shape` helper) is reachable without the `mock` -//! feature. -//! -//! Runs under both `cargo test -p kb-embed` and -//! `cargo test -p kb-embed --features mock`. - -use kebab_embed::{ - Embedder, EmbeddingInput, EmbeddingKind, EmbeddingModelId, EmbeddingVersion, - assert_vector_shape, -}; - -/// A trivial in-test impl that does NOT rely on the `mock` feature — proves -/// the trait surface alone is enough to write an `Embedder`. -struct ZeroEmbedder { - dims: usize, -} - -impl Embedder for ZeroEmbedder { - fn model_id(&self) -> EmbeddingModelId { - EmbeddingModelId("zero".into()) - } - fn model_version(&self) -> EmbeddingVersion { - EmbeddingVersion("0".into()) - } - fn dimensions(&self) -> usize { - self.dims - } - fn embed(&self, inputs: &[EmbeddingInput<'_>]) -> anyhow::Result>> { - Ok(inputs.iter().map(|_| vec![0.0; self.dims]).collect()) - } -} - -#[test] -fn reexports_compile_without_mock_feature() { - let e: Box = Box::new(ZeroEmbedder { dims: 4 }); - let inputs = [ - EmbeddingInput { - text: "hello", - kind: EmbeddingKind::Document, - }, - EmbeddingInput { - text: "world", - kind: EmbeddingKind::Query, - }, - ]; - let v = e.embed(&inputs).expect("zero embed"); - assert_eq!(v.len(), 2); - assert_vector_shape(&v, 4); -} - -/// Sanity: when built WITHOUT `--features mock`, the `MockEmbedder` symbol -/// is absent. We can't usefully test `nm` from inside a unit test, but we -/// can at least confirm the cfg gate parses both ways. See PR notes for the -/// CI-side `nm`/`cargo bloat` symbol scan. -#[cfg(not(feature = "mock"))] -#[test] -fn mock_feature_off_compiles() { - // No-op — the test's existence proves the `not(feature = "mock")` gate - // compiles and the crate is usable without `MockEmbedder`. -} diff --git a/crates/kebab-llm-local/Cargo.toml b/crates/kebab-llm-local/Cargo.toml index 1db5299..111de90 100644 --- a/crates/kebab-llm-local/Cargo.toml +++ b/crates/kebab-llm-local/Cargo.toml @@ -10,7 +10,6 @@ description = "Ollama HTTP adapter implementing kb_core::LanguageModel via req [dependencies] kebab-core = { path = "../kebab-core" } kebab-config = { path = "../kebab-config" } -kebab-llm = { path = "../kebab-llm" } # `default-features = false` drops the `default-tls` (native-tls / openssl) # feature so we don't pull in a system OpenSSL; we explicitly pin rustls. # Note: `default-features = false` does NOT drop tokio — reqwest 0.12's diff --git a/crates/kebab-llm-local/src/lib.rs b/crates/kebab-llm-local/src/lib.rs index d97e521..6cfcb0b 100644 --- a/crates/kebab-llm-local/src/lib.rs +++ b/crates/kebab-llm-local/src/lib.rs @@ -3,12 +3,12 @@ //! //! ## Why a separate crate //! -//! `kb-llm` re-exports the trait + [`MockLanguageModel`] for downstream tests. -//! Real adapters (Ollama, llama.cpp, candle) live outside `kb-llm` so swapping -//! providers stays config-only and so the trait crate has no heavy -//! dependencies. p4-2 ("first real LM") is the home of [`OllamaLanguageModel`] -//! and the [`LlmError`] enum the rest of the workspace will pattern-match -//! against. +//! `kebab-core` exposes the [`LanguageModel`] trait + a feature-gated +//! `MockLanguageModel` for downstream tests. Real adapters (Ollama, llama.cpp, +//! candle) live outside `kebab-core` so swapping providers stays config-only +//! and so the core crate stays free of heavy adapter dependencies. p4-2 +//! ("first real LM") is the home of [`OllamaLanguageModel`] and the +//! [`LlmError`] enum the rest of the workspace will pattern-match against. //! //! ## Runtime contract //! @@ -40,10 +40,9 @@ pub use error::LlmError; pub use ollama::OllamaLanguageModel; // Re-export the trait surface so adapter consumers can `use kebab_llm_local::*` -// without also depending on `kb-llm` directly. These are the same symbols -// `kb-llm` re-exports from `kb-core`; this crate adds **no new types** to -// the trait surface (`LlmError` and `OllamaLanguageModel` are +// without also depending on `kb-core` directly. This crate adds **no new +// types** to the trait surface (`LlmError` and `OllamaLanguageModel` are // implementation-side only). -pub use kebab_llm::{ +pub use kebab_core::{ FinishReason, GenerateRequest, LanguageModel, ModelRef, TokenChunk, TokenUsage, }; diff --git a/crates/kebab-llm-local/src/ollama.rs b/crates/kebab-llm-local/src/ollama.rs index e42fe73..f13178d 100644 --- a/crates/kebab-llm-local/src/ollama.rs +++ b/crates/kebab-llm-local/src/ollama.rs @@ -60,7 +60,7 @@ use crate::error::LlmError; /// `reqwest::blocking` adapter implementing [`LanguageModel`] over Ollama's /// local HTTP API. Construction is cheap and offline; the first network -/// call happens inside [`generate_stream`]. +/// call happens inside [`LanguageModel::generate_stream`]. pub struct OllamaLanguageModel { client: reqwest::blocking::Client, /// Already-validated endpoint URL string (e.g. `"http://127.0.0.1:11434"`). diff --git a/crates/kebab-llm/Cargo.toml b/crates/kebab-llm/Cargo.toml deleted file mode 100644 index 2a0252a..0000000 --- a/crates/kebab-llm/Cargo.toml +++ /dev/null @@ -1,24 +0,0 @@ -[package] -name = "kebab-llm" -version = { workspace = true } -edition = { workspace = true } -rust-version = { workspace = true } -license = { workspace = true } -repository = { workspace = true } -description = "LanguageModel trait re-export + feature-gated MockLanguageModel for downstream tests" - -[dependencies] -kebab-core = { path = "../kebab-core" } -anyhow = { workspace = true } - -[features] -default = [] -# Opt-in `MockLanguageModel`. Default OFF so release builds (no `--features mock`) -# compile the symbol out entirely (verifiable via `nm`/`cargo bloat`). -mock = [] - -[dev-dependencies] -proptest = { workspace = true } - -[lints] -workspace = true diff --git a/crates/kebab-llm/src/lib.rs b/crates/kebab-llm/src/lib.rs deleted file mode 100644 index c536ae8..0000000 --- a/crates/kebab-llm/src/lib.rs +++ /dev/null @@ -1,49 +0,0 @@ -//! `kb-llm` — thin re-export crate for the [`LanguageModel`] trait surface. -//! -//! This crate exists so downstream code (`kb-rag`, adapters in p4-2) can -//! `use kebab_llm::LanguageModel` and stay stable across kb-core reorganizations. -//! It defines **no new types**; everything is a re-export of [`kebab_core`]. -//! -//! ## Mock implementation -//! -//! [`MockLanguageModel`] (gated behind the `mock` feature, default **OFF**) is -//! a deterministic test double. Real adapters (Ollama, llama.cpp, candle) live -//! in p4-2 and MUST NOT be implemented here. Real adapters MAY return `Err` -//! from `generate_stream` itself (e.g., connection refused) before any chunk -//! is yielded; the mock never does. -//! -//! See `docs/superpowers/specs/2026-04-27-kebab-final-form-design.md` §7.1, §7.2, -//! §0 Q5 (streaming), §3.8 (`ModelRef`) for the contract. - -// ── Trait re-exports ────────────────────────────────────────────────────── -// -// Per spec §7.2 — these are the only public-surface types this crate offers. -// Adding new types is forbidden by the task contract. - -pub use kebab_core::{ - FinishReason, GenerateRequest, LanguageModel, ModelRef, TokenChunk, TokenUsage, -}; - -// ── Test helper ─────────────────────────────────────────────────────────── - -/// Assert the streamed `TokenChunk` sequence ends with a [`TokenChunk::Done`] -/// frame. Per spec §7.2 / §0 Q5 every stream — even an erroring one — must -/// terminate with a `Done` chunk; this helper centralizes that contract check -/// so downstream test crates don't each rewrite it. -/// -/// Panics on mismatch (test-only helper — callers are tests). -pub fn assert_finish_chunk(chunks: &[TokenChunk]) { - assert!( - matches!(chunks.last(), Some(TokenChunk::Done { .. })), - "stream must end with TokenChunk::Done; got {:?}", - chunks.last(), - ); -} - -// ── MockLanguageModel (feature = "mock") ────────────────────────────────── - -#[cfg(feature = "mock")] -mod mock; - -#[cfg(feature = "mock")] -pub use mock::MockLanguageModel; diff --git a/crates/kebab-llm/src/mock.rs b/crates/kebab-llm/src/mock.rs deleted file mode 100644 index 3d8c4ed..0000000 --- a/crates/kebab-llm/src/mock.rs +++ /dev/null @@ -1,115 +0,0 @@ -//! Deterministic mock language model for downstream tests. -//! -//! Compiled only when the `mock` feature is enabled. Default builds -//! (`cargo build --release -p kb-llm`) MUST NOT contain the `MockLanguageModel` -//! symbol — verifiable by symbol scan (`nm`/`cargo bloat`). -//! -//! ## Streaming contract -//! -//! For every call to [`MockLanguageModel::generate_stream`]: -//! -//! 1. The configured `canned_response` is examined for any of `req.stop`. If -//! one or more stop strings are substrings of the response, the response -//! is truncated at the **earliest byte position** of any match (i.e., the -//! first stop string to land — ties broken by the order entries appear in -//! `req.stop`, since `Iterator::min` returns the first equal element on -//! ties, breaking by `req.stop` declaration order). -//! 2. The (possibly truncated) string is iterated by Unicode scalar -//! (`str::chars()`) and each character is yielded as -//! [`TokenChunk::Token`]`(c.to_string())`. This makes streaming UTF-8 safe -//! by construction (no character is split across chunks). Emits one -//! `TokenChunk` per Unicode scalar value (`char`), not per grapheme -//! cluster — Hangul jamo, emoji ZWJ sequences, and combining marks split -//! into multiple chunks. Acceptable for trait-shape testing; real adapters -//! MAY combine. -//! 3. After all tokens, a single terminal [`TokenChunk::Done`] is yielded -//! with: -//! * `finish_reason = FinishReason::Stop` if a stop string truncated the -//! canned text — mirroring real LLM behavior, which reports Stop on -//! stop-sequence termination regardless of the configured finish. -//! * `finish_reason = canned_finish.clone()` otherwise. -//! * `usage = canned_usage.clone()` always. -//! -//! ## Non-effects -//! -//! - No network. No filesystem. No async runtime. -//! - No tokenizer. `usage.prompt_tokens` / `completion_tokens` are whatever -//! the constructor was given — the mock does not count. - -use kebab_core::{FinishReason, GenerateRequest, LanguageModel, ModelRef, TokenChunk, TokenUsage}; - -/// Deterministic test double. See module docs for the streaming recipe. -pub struct MockLanguageModel { - pub model_id: String, - pub provider: String, - pub context_tokens: usize, - pub canned_response: String, - pub canned_finish: FinishReason, - pub canned_usage: TokenUsage, -} - -impl MockLanguageModel { - /// Apply `req.stop` to `canned_response`. Returns `(truncated_text, - /// stop_hit)` where `stop_hit` is true iff any stop string was found. - fn apply_stop<'a>(canned: &'a str, stop: &[String]) -> (&'a str, bool) { - // Earliest byte position wins. Ties break by first occurrence in - // `stop` (Iterator::min returns the first equal element, and we - // iterate `stop` in its declared order). Empty stop strings are - // ignored — they would otherwise match at position 0 and silently - // eat the entire response. - let earliest = stop - .iter() - .filter(|s| !s.is_empty()) - .filter_map(|s| canned.find(s.as_str())) - .min(); - match earliest { - // `str::find` returns a UTF-8 char boundary by contract, so direct byte-slice is sound. - Some(idx) => (&canned[..idx], true), - None => (canned, false), - } - } -} - -impl LanguageModel for MockLanguageModel { - fn model_ref(&self) -> ModelRef { - ModelRef { - id: self.model_id.clone(), - provider: self.provider.clone(), - // Per §3.8: `dimensions` carries the embedder's output dim and is - // intentionally None for chat models. - dimensions: None, - } - } - - fn context_tokens(&self) -> usize { - self.context_tokens - } - - fn generate_stream( - &self, - req: GenerateRequest, - ) -> anyhow::Result> + Send>> { - let (truncated, stop_hit) = Self::apply_stop(&self.canned_response, &req.stop); - - // Pre-materialize the full chunk sequence into an owned Vec. This - // sidesteps lifetime juggling around `&self.canned_response` inside - // a `'static` iterator and trivially gives `Send` (Vec - // is Send because TokenChunk is Send). - let mut chunks: Vec = truncated - .chars() - .map(|c| TokenChunk::Token(c.to_string())) - .collect(); - - let finish_reason = if stop_hit { - FinishReason::Stop - } else { - self.canned_finish.clone() - }; - chunks.push(TokenChunk::Done { - finish_reason, - usage: self.canned_usage.clone(), - }); - - Ok(Box::new(chunks.into_iter().map(Ok))) - } -} diff --git a/crates/kebab-llm/tests/reexports.rs b/crates/kebab-llm/tests/reexports.rs deleted file mode 100644 index 0e479de..0000000 --- a/crates/kebab-llm/tests/reexports.rs +++ /dev/null @@ -1,75 +0,0 @@ -//! Compile-only test: verifies the crate's public surface (trait re-exports -//! and the `assert_finish_chunk` helper) is reachable without the `mock` -//! feature. -//! -//! Runs under both `cargo test -p kb-llm` and -//! `cargo test -p kb-llm --features mock`. - -use kebab_llm::{ - FinishReason, GenerateRequest, LanguageModel, ModelRef, TokenChunk, TokenUsage, - assert_finish_chunk, -}; - -/// A trivial in-test impl that does NOT rely on the `mock` feature — proves -/// the trait surface alone is enough to write a `LanguageModel`. It returns a -/// stream that terminates immediately with `Done`. -struct ZeroLanguageModel; - -impl LanguageModel for ZeroLanguageModel { - fn model_ref(&self) -> ModelRef { - ModelRef { - id: "zero".into(), - provider: "zero".into(), - dimensions: None, - } - } - fn context_tokens(&self) -> usize { - 0 - } - fn generate_stream( - &self, - _req: GenerateRequest, - ) -> anyhow::Result> + Send>> { - let chunks = vec![TokenChunk::Done { - finish_reason: FinishReason::Stop, - usage: TokenUsage { - prompt_tokens: 0, - completion_tokens: 0, - latency_ms: 0, - }, - }]; - Ok(Box::new(chunks.into_iter().map(Ok))) - } -} - -#[test] -fn dyn_dispatch_via_box_works() { - let m: Box = Box::new(ZeroLanguageModel); - assert_eq!(m.model_ref().id, "zero"); - assert_eq!(m.context_tokens(), 0); - - let req = GenerateRequest { - system: "sys".into(), - user: "usr".into(), - stop: vec![], - max_tokens: 16, - temperature: 0.0, - seed: None, - images: Vec::new(), - }; - let stream = m.generate_stream(req).expect("stream"); - let chunks: Vec = stream.map(|r| r.expect("ok chunk")).collect(); - assert_eq!(chunks.len(), 1); - assert_finish_chunk(&chunks); -} - -/// Sanity: when built WITHOUT `--features mock`, the `MockLanguageModel` -/// symbol is absent. We can't usefully test `nm` from inside a unit test, but -/// we can at least confirm the cfg gate parses both ways. See PR notes for -/// the CI-side `nm`/`cargo bloat` symbol scan. -#[cfg(not(feature = "mock"))] -#[test] -fn mock_feature_off_compiles() { - // No-op — the test's existence proves the `not(feature = "mock")` gate - // compiles and the crate is usable without `MockLanguageModel`. -} diff --git a/crates/kebab-parse-image/Cargo.toml b/crates/kebab-parse-image/Cargo.toml index 7fd2590..767fbfa 100644 --- a/crates/kebab-parse-image/Cargo.toml +++ b/crates/kebab-parse-image/Cargo.toml @@ -10,12 +10,9 @@ description = "Image extractor + EXIF + OCR (Ollama-vision) for the kebab pipe [dependencies] kebab-core = { path = "../kebab-core" } kebab-config = { path = "../kebab-config" } -# `kebab-llm` re-exports the trait crate (`kebab-core::LanguageModel`) -# under a stable surface; the caption adapter consumes any -# `dyn LanguageModel`. We do NOT depend on `kebab-llm-local` (forbidden -# by p6-3 design §8) — the trait abstraction is exactly what spec -# requires. -kebab-llm = { path = "../kebab-llm" } +# The caption adapter consumes any `dyn LanguageModel` (the trait lives in +# `kebab-core`). We do NOT depend on `kebab-llm-local` (forbidden by p6-3 +# design §8) — the trait abstraction is exactly what spec requires. anyhow = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } @@ -66,12 +63,12 @@ tokio = { workspace = true, features = ["rt-multi-thread"] } # font rendering. ab_glyph = "0.2" base64 = { workspace = true } -# `kebab-llm/mock` exposes `MockLanguageModel` for hermetic caption -# tests. Real adapters (Ollama) live in `kebab-llm-local`, which is +# `kebab-core`'s `mock` feature exposes `MockLanguageModel` for hermetic +# caption tests. Real adapters (Ollama) live in `kebab-llm-local`, which is # only allowed at the dev-dep level here — the runtime crate stays # trait-only, so the §8 forbidden-deps rule (no `kebab-llm-local` # at runtime) is preserved. -kebab-llm = { path = "../kebab-llm", features = ["mock"] } +kebab-core = { path = "../kebab-core", features = ["mock"] } kebab-llm-local = { path = "../kebab-llm-local" } [lints] diff --git a/crates/kebab-parse-image/src/caption.rs b/crates/kebab-parse-image/src/caption.rs index 12795d6..1b056ea 100644 --- a/crates/kebab-parse-image/src/caption.rs +++ b/crates/kebab-parse-image/src/caption.rs @@ -20,7 +20,7 @@ //! OFF at compile time). We collapse this into a single runtime gate //! (`config.ingest.image.caption.enabled = false`, default OFF). Reasoning: //! the captioning module's only extra deps are `base64` + `image` + -//! `kebab-llm` trait — all already pulled in by the rest of the +//! the `kebab-core` `LanguageModel` trait — all already pulled in by the rest of the //! crate. A cargo feature would only complicate the build matrix //! without saving meaningful binary weight. See `tasks/HOTFIXES.md` //! (2026-05-02) for the deviation log. diff --git a/crates/kebab-parse-image/tests/caption.rs b/crates/kebab-parse-image/tests/caption.rs index e7d700b..cbc6567 100644 --- a/crates/kebab-parse-image/tests/caption.rs +++ b/crates/kebab-parse-image/tests/caption.rs @@ -1,6 +1,6 @@ //! Integration tests for the caption adapter (P6-3). //! -//! All hermetic tests use `MockLanguageModel` from `kebab-llm/mock` +//! All hermetic tests use `MockLanguageModel` from `kebab-core` (`mock` feature) //! which captures `req.images` indirectly via the canned response. A //! single opt-in test (`#[ignore]`) wires the real //! `kebab-llm-local::OllamaLanguageModel` against the workspace's @@ -15,7 +15,7 @@ use kebab_core::{ AssetId, BlockId, CommonBlock, FinishReason, GenerateRequest, ImageRefBlock, Lang, LanguageModel, ModelRef, ProvenanceEvent, ProvenanceKind, SourceSpan, TokenChunk, TokenUsage, }; -use kebab_llm::MockLanguageModel; +use kebab_core::MockLanguageModel; use kebab_parse_image::{apply_caption, caption_image}; use crate::common::red_100x50_png; diff --git a/crates/kebab-rag/Cargo.toml b/crates/kebab-rag/Cargo.toml index 5064fdf..206cf83 100644 --- a/crates/kebab-rag/Cargo.toml +++ b/crates/kebab-rag/Cargo.toml @@ -11,7 +11,6 @@ description = "RAG pipeline: retrieve → gate → pack → generate → cite- kebab-core = { path = "../kebab-core" } kebab-config = { path = "../kebab-config" } kebab-search = { path = "../kebab-search" } -kebab-llm = { path = "../kebab-llm" } kebab-nli = { path = "../kebab-nli" } kebab-store-sqlite = { path = "../kebab-store-sqlite" } serde = { workspace = true } @@ -24,7 +23,7 @@ anyhow = { workspace = true } blake3 = { workspace = true } [dev-dependencies] -kebab-llm = { path = "../kebab-llm", features = ["mock"] } +kebab-core = { path = "../kebab-core", features = ["mock"] } tempfile = { workspace = true } rusqlite = { workspace = true } serde_json = { workspace = true } diff --git a/crates/kebab-rag/tests/pipeline.rs b/crates/kebab-rag/tests/pipeline.rs index 9e297d6..397da9c 100644 --- a/crates/kebab-rag/tests/pipeline.rs +++ b/crates/kebab-rag/tests/pipeline.rs @@ -11,7 +11,7 @@ use std::sync::atomic::Ordering; use common::{MockRetriever, RagEnv, id32, mk_hit, mk_hit_with_indexed_at}; use kebab_core::{FinishReason, LanguageModel, Retriever, SearchMode, TokenChunk, TokenUsage}; -use kebab_llm::MockLanguageModel; +use kebab_core::MockLanguageModel; use kebab_rag::{AskOpts, RagPipeline, RefusalReason, StreamEvent}; /// LM ID used everywhere — kept short so snapshots stay stable. diff --git a/crates/kebab-rag/tests/prompt_template_dispatch.rs b/crates/kebab-rag/tests/prompt_template_dispatch.rs index eb04a8f..9c88c5c 100644 --- a/crates/kebab-rag/tests/prompt_template_dispatch.rs +++ b/crates/kebab-rag/tests/prompt_template_dispatch.rs @@ -12,7 +12,7 @@ use common::{MockRetriever, RagEnv, id32, mk_hit}; use kebab_core::{ FinishReason, LanguageModel, Retriever, SearchMode, TokenChunk, TokenUsage, TrustLevel, }; -use kebab_llm::MockLanguageModel; +use kebab_core::MockLanguageModel; use kebab_rag::{AskOpts, RagPipeline}; const TEST_LM_ID: &str = "mock-lm"; diff --git a/crates/kebab-rag/tests/streaming_events.rs b/crates/kebab-rag/tests/streaming_events.rs index b99c59a..5238f8b 100644 --- a/crates/kebab-rag/tests/streaming_events.rs +++ b/crates/kebab-rag/tests/streaming_events.rs @@ -11,7 +11,7 @@ use common::{MockRetriever, RagEnv, id32, mk_hit}; use kebab_core::{ FinishReason, LanguageModel, RefusalReason, Retriever, SearchMode, TokenChunk, TokenUsage, }; -use kebab_llm::MockLanguageModel; +use kebab_core::MockLanguageModel; use kebab_rag::{AskOpts, RagPipeline, StreamEvent}; const TEST_LM_ID: &str = "mock-lm"; diff --git a/crates/kebab-search/Cargo.toml b/crates/kebab-search/Cargo.toml index adc8e0e..b16c5f2 100644 --- a/crates/kebab-search/Cargo.toml +++ b/crates/kebab-search/Cargo.toml @@ -18,7 +18,6 @@ kebab-store-sqlite = { path = "../kebab-store-sqlite" } # adapter — the concrete adapter (`kb-embed-local`) stays out of this # crate per the spec's Forbidden deps list. kebab-store-vector = { path = "../kebab-store-vector" } -kebab-embed = { path = "../kebab-embed" } rusqlite = { workspace = true } globset = { workspace = true } serde_json = { workspace = true } @@ -31,11 +30,11 @@ time = { workspace = true } [dev-dependencies] tempfile = { workspace = true } -# Hybrid integration tests inject a `MockEmbedder` (kb-embed `mock` +# Hybrid integration tests inject a `MockEmbedder` (kebab-core `mock` # feature) and stand up a real `LanceVectorStore` on a tmp directory. # The mock-retriever unit tests (the bulk of the hybrid suite) do not # need either, but the integration / snapshot lane does. -kebab-embed = { path = "../kebab-embed", features = ["mock"] } +kebab-core = { path = "../kebab-core", features = ["mock"] } [lints] workspace = true diff --git a/crates/kebab-search/tests/common/mod.rs b/crates/kebab-search/tests/common/mod.rs index eb072a5..d667ef7 100644 --- a/crates/kebab-search/tests/common/mod.rs +++ b/crates/kebab-search/tests/common/mod.rs @@ -22,7 +22,7 @@ use kebab_core::{ EmbeddingVersion, IndexVersion, MediaType, Retriever, SearchFilters, SearchHit, SearchMode, SearchQuery, VectorRecord, VectorStore, }; -use kebab_embed::{Embedder, MockEmbedder}; +use kebab_core::{Embedder, MockEmbedder}; use kebab_search::{LexicalRetriever, VectorRetriever}; use kebab_store_sqlite::SqliteStore; use kebab_store_vector::LanceVectorStore; diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 30a875d..9ef88cb 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -64,10 +64,8 @@ flowchart TB vector["kebab-store-vector"] end subgraph Adapters ["traits + adapters"] - embed["kebab-embed
(trait)"] embedlocal["kebab-embed-local
(fastembed, default)"] embedollama["kebab-embed-ollama
(Ollama /api/embed, opt-in)"] - llm["kebab-llm
(trait)"] llmlocal["kebab-llm-local
(Ollama)"] search["kebab-search"] rag["kebab-rag"] @@ -103,24 +101,20 @@ flowchart TB pimg --> core paud --> core pcode --> core - embedlocal --> embed + embedlocal --> core embedollama --> core embedollama --> config - llmlocal --> llm + llmlocal --> core rag --> search - rag --> llm rag --> sqlite rag --> nli app --> nli nli --> config search --> sqlite search --> vector - search --> embed eval --> app config --> core - embed --> core - llm --> core sqlite --> core vector --> core chunk --> core @@ -132,7 +126,7 @@ flowchart TB UI → store/llm/parse 직접 의존 금지. 모든 user-facing 진입은 `kebab-app` facade 만 통한다 (frozen 설계 §8). `kebab-cli` 가 `--config ` flag 를 honor 하려면 `kebab_app::*_with_config(cfg, …)` companion 을 통해 Config 을 명시적으로 thread 하는 패턴 — 자세한 이유는 [tasks/HOTFIXES.md](../tasks/HOTFIXES.md) 의 `--config` 항목. -`kebab-parse-code` 의 외부 tree-sitter grammar crate 의존: P10-1A-2 에서 `tree-sitter-rust` 추가, P10-1B 에서 `tree-sitter-python` / `tree-sitter-typescript` / `tree-sitter-javascript` 추가, P10-1C-Go 에서 `tree-sitter-go` 추가, P10-1C-JK 에서 `tree-sitter-java` / `tree-sitter-kotlin-ng` 추가, P10-1D 에서 `tree-sitter-c` / `tree-sitter-cpp` 추가. 모두 `kebab-parse-code` 에만 격리 (facade 룰 — UI crate / chunker 가 직접 import 금지). Kotlin 은 `tree-sitter-kotlin-ng` 사용 (bare `tree-sitter-kotlin` 은 tree-sitter 0.21–0.23 에 고착 — 사용 불가). v0.18.0+ 부터 `kebab-source-fs` 는 자체 `code_meta` 모듈 (lang detect + skip helpers + BUILTIN_BLACKLIST) 을 보유, kebab-parse-code 와 분리 (refactor 2026-05-26). v0.19.0 부터 `kebab-parse-md` 가 `kebab-parse-types` (parser intermediate types) + `kebab-normalize` (CanonicalDocument lift) 두 crate 를 흡수 — 24 → 22 crates, design §3.7b 재작성 (HOTFIXES 2026-05-26). v0.20.1 부터 `kebab-search` 가 `lindera-ko-dic` 를 의존해 한국어 FTS5 형태소 tokenizer 지원 — V009 migration 으로 2자 이상 한국어 query 매칭 (Bug #8 closure). +`kebab-parse-code` 의 외부 tree-sitter grammar crate 의존: P10-1A-2 에서 `tree-sitter-rust` 추가, P10-1B 에서 `tree-sitter-python` / `tree-sitter-typescript` / `tree-sitter-javascript` 추가, P10-1C-Go 에서 `tree-sitter-go` 추가, P10-1C-JK 에서 `tree-sitter-java` / `tree-sitter-kotlin-ng` 추가, P10-1D 에서 `tree-sitter-c` / `tree-sitter-cpp` 추가. 모두 `kebab-parse-code` 에만 격리 (facade 룰 — UI crate / chunker 가 직접 import 금지). Kotlin 은 `tree-sitter-kotlin-ng` 사용 (bare `tree-sitter-kotlin` 은 tree-sitter 0.21–0.23 에 고착 — 사용 불가). v0.18.0+ 부터 `kebab-source-fs` 는 자체 `code_meta` 모듈 (lang detect + skip helpers + BUILTIN_BLACKLIST) 을 보유, kebab-parse-code 와 분리 (refactor 2026-05-26). v0.19.0 부터 `kebab-parse-md` 가 `kebab-parse-types` (parser intermediate types) + `kebab-normalize` (CanonicalDocument lift) 두 crate 를 흡수 — 24 → 22 crates, design §3.7b 재작성 (HOTFIXES 2026-05-26). v0.20.1 부터 `kebab-search` 가 `lindera-ko-dic` 를 의존해 한국어 FTS5 형태소 tokenizer 지원 — V009 migration 으로 2자 이상 한국어 query 매칭 (Bug #8 closure). pure re-export shim 이던 `kebab-embed` / `kebab-llm` (trait 은 이미 `kebab-core` 소유, mock + test helper 만 보유) 를 `kebab-core` 의 default-OFF `mock` feature 로 흡수 — 22 → 20 crates, trait surface · 동작 불변 (test-only + import-rename churn). ### 임베딩 백엔드 결정표 (v0.26.0) @@ -195,10 +189,10 @@ kebab/ │ │ └── tier2_shared.rs # Tier 2 (p10-2): shared oversize fallback + Chunk builder helpers │ ├── kebab-store-sqlite/ # SQLite + FTS5 (V001/V002/V003) (P1-6, P2-1, P3-3). src/derivation_cache.rs = derivation_cache 테이블 저장소 (V012, v0.21.0) │ ├── kebab-search/ # Lexical + Vector + Hybrid retriever (P2-2, P3-4) -│ ├── kebab-embed/ kebab-embed-local/ # Embedder trait + fastembed adapter (P3-1, P3-2) +│ ├── kebab-embed-local/ # fastembed Embedder adapter (P3-2; trait lives in kebab-core) │ ├── kebab-embed-ollama/ # Ollama /api/embed Embedder, opt-in provider=ollama (arctic 경로, v0.26.0) │ ├── kebab-store-vector/ # LanceDB VectorStore (P3-3, P7-3 follow-up) -│ ├── kebab-llm/ kebab-llm-local/ # LanguageModel trait + Ollama adapter (P4-1, P4-2) +│ ├── kebab-llm-local/ # Ollama LanguageModel adapter (P4-2; trait lives in kebab-core) │ ├── kebab-rag/ # RAG pipeline (P4-3) │ ├── kebab-nli/ # NLI verifier (mDeBERTa-v3 XNLI, fb-41 PR-9a/9b/9c-1) │ ├── kebab-eval/ # golden query runner + metrics (P5-1, P5-2) diff --git a/docs/components/README.md b/docs/components/README.md index cceb30d..a3abb7a 100644 --- a/docs/components/README.md +++ b/docs/components/README.md @@ -95,7 +95,7 @@ flowchart TB - **새 미디어 타입 추가** (예: epub) — Parse → Normalize+Chunk → Store (chunker_version) → App facade (라우팅). - **새 retrieval 모드** — Search → App facade (mode dispatch) → UI (--mode flag). -- **새 LLM 어댑터** — LLM (trait crate, 새 type 금지) + 새 `kebab-llm-` crate → App facade (config provider switch). +- **새 LLM 어댑터** — `kebab-core` 의 `LanguageModel` trait 구현 + 새 `kebab-llm-` crate → App facade (config provider switch). - **TUI 신규 pane** — UI 만. Mode + Theme + InputBuffer 재사용. ## 다이어그램 제약 diff --git a/docs/components/embed/README.md b/docs/components/embed/README.md index 15975ec..441370a 100644 --- a/docs/components/embed/README.md +++ b/docs/components/embed/README.md @@ -6,7 +6,6 @@ | Crate | 역할 | |-------|------| -| `kebab-embed` | `Embedder` trait re-export + 테스트 도구 (`assert_vector_shape`, `assert_unit_norm`) + optional `MockEmbedder` (feature gated). 새 type 추가 **금지** — 순수 facade. | | `kebab-embed-local` | `FastembedEmbedder` — fastembed-rs 위 ONNX-backed local 임베더. default `multilingual-e5-small` 384d. | ## 구조 @@ -64,7 +63,7 @@ flowchart LR ## 주요 type / trait / 함수 -**Trait** (`kebab-core`, re-export `kebab-embed`): +**Trait** (`kebab-core`): - `Embedder::embed(&self, inputs: &[EmbeddingInput<'_>]) -> Result>>` — 출력 shape `inputs.len()` × `dimensions()`. 결과 벡터 모두 L2 = 1 + finite. - `EmbeddingInput { text: &str, kind: EmbeddingKind }` — kind = `Document` / `Query` (E5 prefix 분기). - `EmbeddingModelId(String)`, `EmbeddingVersion(String)` — `model_id × version × dim` 으로 vector store 테이블 분리. @@ -75,22 +74,21 @@ flowchart LR - E5 prefix 자동 적용: `Document` → `"passage: "`, `Query` → `"query: "` (§11.3). - L2 정규화 = fastembed 내장 (`transformer_with_precedence`). 별도 정규화 안 함, 단 `assert_unit_norm` 테스트로 invariant pin. -**테스트 도구** (`kebab-embed`): +**테스트 도구** (`kebab-core`, feature `mock`): - `assert_vector_shape(&[Vec], expected_dims)` — 길이 + finite 검증. - `assert_unit_norm(&[Vec], tolerance)` — L2 norm 이 `1.0 ± tolerance`. f32 384d 권장 tol = `5e-4`. - `MockEmbedder` (feature `mock`, default OFF) — 테스트용 deterministic double. 실 어댑터는 `kebab-embed-local` 또는 future P+ adapter 가 담당. ## 외부 의존 -- `kebab-embed` → `kebab-core` 만 (re-export crate). -- `kebab-embed-local` → `kebab-embed` + `kebab-config`, `fastembed`, `anyhow`. +- `kebab-embed-local` → `kebab-core` + `kebab-config`, `fastembed`, `anyhow`. - 외부 lib: `fastembed-rs` (ONNX wrapper, Hugging Face 모델 다운로드 포함). 로컬 ORT runtime. - 외부 서비스: 첫 호출 시 모델 다운로드 (Hugging Face). 그 후 오프라인. ## 핵심 결정 -- **`kebab-embed` = trait re-export only, **새 type 금지****. - **왜**: `kebab-store-vector`, `kebab-search` 등 downstream 이 `use kebab_embed::Embedder` 안정 surface 의존. `kebab-core` 재구성 시 trait 이동해도 downstream 안 깨짐. spec 가 명시 — 어댑터 코드는 `kebab-embed-local` 또는 future `kebab-embed-` 로. +- **`Embedder` trait + 테스트 도구가 `kebab-core` 에 직접 거주 (`mock` feature)**. + **왜**: `kebab-store-vector`, `kebab-search` 등 downstream 은 `use kebab_core::Embedder` 로 의존 — 별도 re-export shim 불필요. `MockEmbedder` / `assert_vector_shape` / `assert_unit_norm` 은 default-OFF `mock` feature 뒤에 둠. 과거의 순수 facade `kebab-embed` 는 `kebab-core` 로 fold-in 되어 삭제됨 (crate 그래프는 [`docs/ARCHITECTURE.md`](../../ARCHITECTURE.md) 참조). 어댑터 코드는 `kebab-embed-local` 또는 future `kebab-embed-` 로. - **`multilingual-e5-small` 384d default**. **왜**: 한국어 + 영어 동시 강함, ONNX 작음 (~120MB), 384d 가 retrieval 정확도/저장 비용 균형 좋음. e5 prefix 컨벤션 (`"passage: "` / `"query: "`) 으로 같은 모델이 doc + query 두 모드 cover. diff --git a/docs/components/llm/README.md b/docs/components/llm/README.md index 5c4a27f..866568f 100644 --- a/docs/components/llm/README.md +++ b/docs/components/llm/README.md @@ -6,7 +6,6 @@ | Crate | 역할 | |-------|------| -| `kebab-llm` | `LanguageModel` trait re-export + `MockLanguageModel` (feature `mock`, default OFF). 새 type 추가 **금지** — 순수 facade. | | `kebab-llm-local` | `OllamaLanguageModel` — `reqwest::blocking` 기반 Ollama `POST /api/generate` 어댑터. line-delimited JSON streaming 디코드. | ## 구조 @@ -86,7 +85,7 @@ flowchart LR ## 주요 type / trait / 함수 -**Trait** (`kebab-core`, re-export `kebab-llm`): +**Trait** (`kebab-core`): - `LanguageModel::model_ref() -> ModelRef` — provider/model/version 식별. `Answer.model_ref` 으로 흘려서 wire payload 가 자가 식별. - `LanguageModel::context_tokens() -> usize` — 모델 별 max prompt+completion 합. RAG 가 budget 계산에 사용. - `LanguageModel::generate_stream(req: GenerateRequest) -> Result> + Send>>` — async 안 됨, 매 next() 가 blocking. 모든 stream 이 마지막에 `TokenChunk::Done` 으로 끝남 (error 케이스 포함, §0 Q5). @@ -106,20 +105,19 @@ flowchart LR **`LlmError`** (`kebab-llm-local::error`): - ConnectionRefused / HttpStatus(code) / Decode(json error) / Timeout / Aborted / 그 외 — `Err` 로 first chunk 전 surface 가능. -**테스트 도구** (`kebab-llm`): +**테스트 도구** (`kebab-core`, feature `mock`): - `assert_finish_chunk(chunks: &[TokenChunk])` — 마지막이 `Done` 이어야 — 모든 stream contract pin. - `MockLanguageModel` (feature `mock`, default OFF) — deterministic test double. 실 adapter 만 `Err` 가능, mock 은 항상 stream 시작 후 yield. ## 외부 의존 -- `kebab-llm` → `kebab-core` 만 (re-export crate). -- `kebab-llm-local` → `kebab-llm` + `kebab-config`, `reqwest` (`blocking` feature, JSON), `serde` + `serde_json`, `thiserror`, `anyhow`. +- `kebab-llm-local` → `kebab-core` + `kebab-config`, `reqwest` (`blocking` feature, JSON), `serde` + `serde_json`, `thiserror`, `anyhow`. - 외부 서비스: **Ollama HTTP** (default `http://127.0.0.1:11434`). default 모델 `gemma4:e4b` (OCR / caption / RAG 모두 같은 family — 단일 모델 다운로드면 전 시스템 동작). ## 핵심 결정 -- **`kebab-llm` = trait re-export only, **새 type 금지****. - **왜**: `kebab-rag` 등 downstream 이 `use kebab_llm::LanguageModel` 안정 surface 의존. 어댑터 (Ollama/llama.cpp/candle) 는 별 crate. swap config-only. +- **`LanguageModel` trait + `MockLanguageModel` 가 `kebab-core` 에 직접 거주 (`mock` feature)**. + **왜**: `kebab-rag` 등 downstream 은 `use kebab_core::LanguageModel` 로 의존 — 별도 re-export shim 불필요. 어댑터 (Ollama/llama.cpp/candle) 는 여전히 별 crate 라 swap config-only. 과거의 순수 facade `kebab-llm` 은 `kebab-core` 로 fold-in 되어 삭제됨 (crate 그래프는 [`docs/ARCHITECTURE.md`](../../ARCHITECTURE.md) 참조). - **synchronous + blocking + stream iterator**. **왜**: §0 Q5 가 streaming 명시. `async` 가 trait object 와 잘 안 맞음 (Rust async-in-trait 안정성 + Send bound 복잡). `reqwest::blocking` + line-delimited frame 의 `Iterator` 가 caller 코드 단순. RAG 가 동기 소비 + UI thread 가 별도 worker 로 spawn. diff --git a/docs/components/rag/README.md b/docs/components/rag/README.md index c3ffc4f..71e2d47 100644 --- a/docs/components/rag/README.md +++ b/docs/components/rag/README.md @@ -106,7 +106,7 @@ flowchart LR ## 외부 의존 -- crate dep: `kebab-core` + `kebab-config` + `kebab-search` (`Retriever` trait 만) + `kebab-llm` (trait 만) + `kebab-store-sqlite` (`DocumentStore` + `put_answer` helper). +- crate dep: `kebab-core` (`LanguageModel` trait 포함) + `kebab-config` + `kebab-search` (`Retriever` trait 만) + `kebab-store-sqlite` (`DocumentStore` + `put_answer` helper). - 외부 lib: `serde`/`serde_json`, `regex` (citation marker `[N]` 매칭), `time` (timestamps), `blake3` (`TraceId` 채굴), `thiserror`, `anyhow`. - 외부 서비스: 없음 (concrete adapter 가 가져옴). diff --git a/docs/components/search/README.md b/docs/components/search/README.md index fbf8244..59d9b62 100644 --- a/docs/components/search/README.md +++ b/docs/components/search/README.md @@ -101,7 +101,7 @@ flowchart LR ## 외부 의존 -- crate dep: `kebab-core` + `kebab-config` + `kebab-store-sqlite` + `kebab-store-vector` + `kebab-embed` (trait re-export). `kebab-embed-local` 은 caller 가 inject (forbidden direct dep). +- crate dep: `kebab-core` (`Embedder` trait 포함) + `kebab-config` + `kebab-store-sqlite` + `kebab-store-vector`. `kebab-embed-local` 은 caller 가 inject (forbidden direct dep). - 외부 lib: `rusqlite` (FTS5 쿼리), `globset` (filter 매칭), `serde_json`, `tracing`. - 외부 서비스: 없음. @@ -125,7 +125,7 @@ flowchart LR - **두 측 `index_version` mismatch = warn (not error)**. **왜**: lexical 이 v2, vector 가 v1 (re-embed 안 했음) 같은 stale state 가 운영 시 일어남. 즉시 fail = ingest 끝나기 전 search 막힘. warning 만 띄우고 계속 동작 = 사용자가 인지하고 re-index 결정. -- **`kebab-embed` (trait crate) 만 의존, `kebab-embed-local` (concrete) **금지****. +- **`Embedder` trait (`kebab-core`) 만 의존, `kebab-embed-local` (concrete) **금지****. **왜**: future MVP 의 swap 가능성 (candle, ollama-embed 등). `kebab-search` 가 concrete 어댑터 import 하면 `kebab-embed-local` 의 fastembed dep (큰 ONNX runtime) 이 search 에 강제 → unrelated build 비용. caller 가 runtime inject. ## 관련 spec / HOTFIXES