altair823
8f7b6ee538
duo-socket NUMA 서버에서 fastembed(onnxruntime)가 intra-op 스레드를 48개로 하드코딩해 NUMA 힙 손상 → double-free 로 ingest 가 죽는 문제를 회피하기 위해, 같은 multilingual-e5-large 모델을 순수 Rust(candle)로 돌리는 opt-in 임베딩 provider 를 추가한다. - 신규 crate kebab-embed-candle: CandleEmbedder (kebab_core::Embedder). hf-hub safetensors → XLMRobertaModel forward → mask mean-pool → L2 → e5 prefix. candle 의존성 트리를 이 crate 에 격리 (core/config 외 kebab-* 의존 0). - 스레드 캡: [models.embedding].num_threads + env KEBAB_EMBED_THREADS → 글로벌 rayon 풀 1회 캡 (NUMA-안전 레버). - kebab-app::embedder() 가 provider 분기 (fastembed/onnx/"" → 기존 경로 불변, candle → CandleEmbedder, 미지값 → 에러). - Phase 0 스파이크 crate 제거 (production 흡수). - 버전 0.21.1 → 0.22.0 (신규 config surface, pre-1.0 minor bump). 패리티: cosine_min=1.000000, max abs diff=2.01e-7 (< 1e-5) → embedding_version 유지, 재색인 0. fastembed default 동작/벡터 불변. wire schema 변경 없음. 검증(파일+exit code): clippy -D warnings EXIT=0(warning 0), test EXIT=0 (candle unit 5 + thread_cap rayon=4 + config 68), parity #[ignore] EXIT=0. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
106 lines
5.2 KiB
TOML
106 lines
5.2 KiB
TOML
[package]
|
|
name = "kebab-app"
|
|
version = { workspace = true }
|
|
edition = { workspace = true }
|
|
rust-version = { workspace = true }
|
|
license = { workspace = true }
|
|
repository = { workspace = true }
|
|
description = "Facade — orchestrates components for kb-cli/tui/desktop"
|
|
|
|
[dependencies]
|
|
kebab-core = { path = "../kebab-core" }
|
|
kebab-config = { path = "../kebab-config" }
|
|
kebab-source-fs = { path = "../kebab-source-fs" }
|
|
kebab-parse-md = { path = "../kebab-parse-md" }
|
|
kebab-chunk = { path = "../kebab-chunk" }
|
|
kebab-store-sqlite = { path = "../kebab-store-sqlite" }
|
|
kebab-store-vector = { path = "../kebab-store-vector" }
|
|
kebab-search = { path = "../kebab-search" }
|
|
kebab-embed = { path = "../kebab-embed" }
|
|
kebab-embed-local = { path = "../kebab-embed-local" }
|
|
kebab-embed-candle = { path = "../kebab-embed-candle" }
|
|
kebab-llm = { path = "../kebab-llm" }
|
|
kebab-llm-local = { path = "../kebab-llm-local" }
|
|
kebab-rag = { path = "../kebab-rag" }
|
|
# p9-fb-41 PR-9c-2: facade construction of OnnxNliVerifier when
|
|
# `[rag] nli_threshold > 0`. Trait-only consumption via kebab-rag's
|
|
# `with_verifier`; no kebab-nli internals leak into kebab-app code
|
|
# beyond the construction site in `open_with_config`.
|
|
kebab-nli = { path = "../kebab-nli" }
|
|
# P6-4: image extractor + OCR + caption adapters live here. App
|
|
# threads them into the per-asset dispatch (see `ingest_one_asset`
|
|
# image branch). Trait-only consumption — no `kebab-parse-image`
|
|
# internals leak into kb-app code.
|
|
kebab-parse-image = { path = "../kebab-parse-image" }
|
|
# P7-3: PDF text extractor lives here. App threads it into the
|
|
# per-asset dispatch (see `ingest_one_asset` PDF branch) and runs the
|
|
# resulting `CanonicalDocument` through `kebab-chunk::PdfPageV1Chunker`.
|
|
kebab-parse-pdf = { path = "../kebab-parse-pdf" }
|
|
lopdf = { workspace = true }
|
|
# Enhancement 1 (v0.20.x r2): JPEG dimension decode in pdf_ocr_apply.rs.
|
|
# jpeg feature added explicitly (F3 closure-r1) rather than relying on
|
|
# feature unification via kebab-parse-image.
|
|
image = { version = "0.25", default-features = false, features = ["png", "jpeg"] }
|
|
# p10-1A-2: Rust AST extractor lives here. App threads it into the
|
|
# per-asset dispatch (see `ingest_one_asset` Code branch) and runs the
|
|
# resulting `CanonicalDocument` through `kebab-chunk::CodeRustAstV1Chunker`.
|
|
kebab-parse-code = { path = "../kebab-parse-code" }
|
|
anyhow = { workspace = true }
|
|
blake3 = { workspace = true }
|
|
serde = { workspace = true }
|
|
serde_json = { workspace = true }
|
|
time = { workspace = true }
|
|
uuid = { workspace = true }
|
|
tracing = { workspace = true }
|
|
tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt", "json"] }
|
|
tracing-appender = "0.2"
|
|
toml = "0.8"
|
|
dirs = "5"
|
|
# p9-fb-19: in-process LRU cache for `App::search`. Capacity from
|
|
# `config.search.cache_capacity` (default 256, ~1.3 MB cap).
|
|
lru = { workspace = true }
|
|
# p9-fb-19: NFKC-normalize cache-key queries so `"Foo"` / `"FOO"` /
|
|
# `" foo "` collapse to one entry. Same crate kebab-normalize +
|
|
# kebab-core already use, no version drift.
|
|
unicode-normalization = "0.1"
|
|
# p9-fb-31: GitignoreBuilder for .kebabignore matching in ingest_file_with_config.
|
|
# Same version as kebab-source-fs (0.4) to avoid duplicate dep versions.
|
|
ignore = "0.4"
|
|
# p9-fb-34: opaque pagination cursor encodes payload as base64.
|
|
base64 = { workspace = true }
|
|
# Enhancement 3 (v0.20.x r2): direct SQL queries for inspect_ocr_stats/failures.
|
|
rusqlite = { workspace = true }
|
|
|
|
[dev-dependencies]
|
|
kebab-config = { path = "../kebab-config" }
|
|
# doc-side expansion (Phase 2) Task 4: ExpansionGenerator unit tests build
|
|
# MockLanguageModel (gated behind kebab-llm's `mock` feature, default OFF in
|
|
# [dependencies]). Enabling it here turns it on for the test build only.
|
|
kebab-llm = { path = "../kebab-llm", features = ["mock"] }
|
|
rusqlite = { workspace = true }
|
|
filetime = "0.2"
|
|
tempfile = { workspace = true }
|
|
# Image-pipeline integration tests use wiremock to stub Ollama for OCR
|
|
# / caption HTTP calls. Async runtime to host the mock server only;
|
|
# the kb-app code under test stays sync.
|
|
wiremock = { workspace = true }
|
|
tokio = { workspace = true, features = ["rt-multi-thread"] }
|
|
image = { version = "0.25", default-features = false, features = ["png", "jpeg"] }
|
|
# P7-3 PDF integration tests build in-memory PDF fixtures via the same
|
|
# lopdf builder pattern `kebab-parse-pdf::tests::common` uses; pinned
|
|
# to the same major (0.32) so byte output is identical between the two
|
|
# fixture surfaces.
|
|
lopdf = { workspace = true }
|
|
# error_wire::tests::llm_unreachable_classifies_to_model_unreachable needs a real
|
|
# reqwest::Error (private constructor) — built from a connect-refused call.
|
|
reqwest = { version = "0.12", default-features = false, features = ["blocking", "rustls-tls"] }
|
|
|
|
[features]
|
|
# Marker feature — spec §6.3 Option A (단순): lindera 는 kebab-chunk 가 default dep 으로 소유.
|
|
# disable path 없음; 이 feature 는 spec §6.3 명시를 honor 하는 role 만.
|
|
default = ["fts_korean_morphological"]
|
|
fts_korean_morphological = []
|
|
|
|
[lints]
|
|
workspace = true
|