refactor(rename): kb crates → kebab — Cargo packages, folders, Rust modules
프로젝트 이름 `kb` → `kebab` rename 의 첫 단계. - workspace `Cargo.toml`: members `crates/kb-*` → `crates/kebab-*`, repository URL `altair823/kb` → `altair823/kebab`. - 18 crate 폴더 rename via `git mv` (history 보존). - 각 crate `Cargo.toml`: `name = "kb-*"` → `"kebab-*"`, path deps `../kb-*` → `../kebab-*`. - 모든 `.rs`: `kb_<id>` snake-case 모듈 path 18 개 (`kb_core`, `kb_config`, `kb_app`, `kb_cli`, `kb_eval`, `kb_search`, `kb_chunk`, `kb_normalize`, `kb_source_fs`, `kb_parse_md`, `kb_parse_types`, `kb_store_sqlite`, `kb_store_vector`, `kb_embed`, `kb_embed_local`, `kb_llm`, `kb_llm_local`, `kb_rag`) → `kebab_<id>` 일괄 sed (단어 경계 \\b 사용해 영어 문장 안의 "kb" 약어 미오염). CLI binary 이름 (`[[bin]] name = "kb"`), 환경변수 `KB_*`, XDG paths, tracing target, 그리고 docs sweep 은 다음 commit 에서. ## 검증 - `cargo check --workspace` clean — 모든 crate 빌드 통과 후 commit. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
187
crates/kebab-rag/tests/common/mod.rs
Normal file
187
crates/kebab-rag/tests/common/mod.rs
Normal file
@@ -0,0 +1,187 @@
|
||||
//! Shared scaffolding for kb-rag tests.
|
||||
//!
|
||||
//! Provides:
|
||||
//! - [`RagEnv`] — a tempdir-backed `SqliteStore` with helpers to seed
|
||||
//! asset/document/chunk rows directly via SQL (so the test crate's
|
||||
//! deps stay inside the allowed list).
|
||||
//! - [`MockRetriever`] — returns canned `Vec<SearchHit>` regardless of
|
||||
//! the query, so the pipeline exercise is independent of any real
|
||||
//! indexer.
|
||||
//! - small helpers to build `Citation` / `SearchHit` / canned LM
|
||||
//! responses without rewriting boilerplate in every test.
|
||||
|
||||
#![allow(dead_code)]
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use kebab_config::Config;
|
||||
use kebab_core::{
|
||||
ChunkerVersion, ChunkId, Citation, DocumentId, IndexVersion, RetrievalDetail,
|
||||
Retriever, SearchHit, SearchMode, SearchQuery, WorkspacePath,
|
||||
};
|
||||
use kebab_store_sqlite::SqliteStore;
|
||||
use rusqlite::params;
|
||||
use tempfile::TempDir;
|
||||
|
||||
/// Tempdir-backed test environment. Holds an open `SqliteStore` with
|
||||
/// V001 + V002 + V003 migrations applied so chunk reads work end-to-end.
|
||||
pub struct RagEnv {
|
||||
pub temp: TempDir,
|
||||
pub config: Config,
|
||||
pub sqlite: Arc<SqliteStore>,
|
||||
}
|
||||
|
||||
impl RagEnv {
|
||||
pub fn new() -> Self {
|
||||
let temp = tempfile::tempdir().expect("tempdir");
|
||||
let mut config = Config::defaults();
|
||||
config.storage.data_dir = temp.path().to_string_lossy().into_owned();
|
||||
let sqlite = SqliteStore::open(&config).unwrap();
|
||||
sqlite.run_migrations().unwrap();
|
||||
Self {
|
||||
temp,
|
||||
config,
|
||||
sqlite: Arc::new(sqlite),
|
||||
}
|
||||
}
|
||||
|
||||
/// Seed the minimal (assets, documents, chunks) row triple needed
|
||||
/// for `DocumentStore::get_chunk` to round-trip in tests.
|
||||
/// `chunk_id` / `doc_id` must already be 32-hex-char shaped (use
|
||||
/// [`id32`] to pad short prefixes).
|
||||
pub fn seed_chunk(
|
||||
&self,
|
||||
chunk_id: &str,
|
||||
doc_id: &str,
|
||||
workspace_path: &str,
|
||||
text: &str,
|
||||
heading_path: &[&str],
|
||||
) {
|
||||
let asset_id = format!("a{}", &doc_id[..31]);
|
||||
let conn = self.sqlite.read_conn();
|
||||
conn.execute(
|
||||
"INSERT OR IGNORE INTO assets (
|
||||
asset_id, source_uri, workspace_path, media_type, byte_len,
|
||||
checksum, storage_kind, storage_path, discovered_at
|
||||
) VALUES (?, ?, ?, '\"markdown\"', 0,
|
||||
'deadbeefdeadbeefdeadbeefdeadbeef',
|
||||
'reference', ?, '1970-01-01T00:00:00Z')",
|
||||
params![
|
||||
asset_id,
|
||||
format!("file://{workspace_path}"),
|
||||
workspace_path,
|
||||
workspace_path,
|
||||
],
|
||||
)
|
||||
.unwrap();
|
||||
conn.execute(
|
||||
"INSERT OR IGNORE INTO documents (
|
||||
doc_id, asset_id, workspace_path, title, lang, source_type,
|
||||
trust_level, parser_version, doc_version, schema_version,
|
||||
metadata_json, provenance_json, created_at, updated_at
|
||||
) VALUES (?, ?, ?, NULL, 'en', 'markdown', 'primary', 'v1', 1, 1,
|
||||
'{}', '{}', '1970-01-01T00:00:00Z', '1970-01-01T00:00:00Z')",
|
||||
params![doc_id, asset_id, workspace_path],
|
||||
)
|
||||
.unwrap();
|
||||
let heading_json = serde_json::to_string(heading_path).unwrap();
|
||||
conn.execute(
|
||||
"INSERT OR IGNORE INTO chunks (
|
||||
chunk_id, doc_id, text, heading_path_json, section_label,
|
||||
source_spans_json, token_estimate, chunker_version,
|
||||
policy_hash, block_ids_json, created_at
|
||||
) VALUES (?, ?, ?, ?, NULL,
|
||||
'[{\"kind\":\"line\",\"start\":1,\"end\":3}]',
|
||||
1, 'v1', 'h', '[]', '1970-01-01T00:00:00Z')",
|
||||
params![chunk_id, doc_id, text, heading_json],
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
/// Count rows in `answers`. Tests use this to assert that every
|
||||
/// `ask` (incl. refusals) writes exactly one row.
|
||||
pub fn count_answers(&self) -> i64 {
|
||||
let conn = self.sqlite.read_conn();
|
||||
conn.query_row("SELECT COUNT(*) FROM answers", [], |r| r.get(0))
|
||||
.unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a `SearchHit` with canned scores. Citation defaults to a
|
||||
/// `Line { 1..=3 }` over `workspace_path`.
|
||||
pub fn mk_hit(
|
||||
rank: u32,
|
||||
chunk_id: &str,
|
||||
doc_id: &str,
|
||||
workspace_path: &str,
|
||||
fusion_score: f32,
|
||||
heading: &[&str],
|
||||
) -> SearchHit {
|
||||
let p = WorkspacePath::new(workspace_path.to_string()).expect("workspace path valid");
|
||||
SearchHit {
|
||||
rank,
|
||||
chunk_id: ChunkId(chunk_id.to_string()),
|
||||
doc_id: DocumentId(doc_id.to_string()),
|
||||
doc_path: p.clone(),
|
||||
heading_path: heading.iter().map(|s| s.to_string()).collect(),
|
||||
section_label: None,
|
||||
snippet: "snippet".to_string(),
|
||||
citation: Citation::Line {
|
||||
path: p,
|
||||
start: 1,
|
||||
end: 3,
|
||||
section: None,
|
||||
},
|
||||
retrieval: RetrievalDetail {
|
||||
method: SearchMode::Lexical,
|
||||
fusion_score,
|
||||
lexical_score: Some(fusion_score),
|
||||
vector_score: None,
|
||||
lexical_rank: Some(rank),
|
||||
vector_rank: None,
|
||||
},
|
||||
index_version: IndexVersion("test-iv".to_string()),
|
||||
embedding_model: None,
|
||||
chunker_version: ChunkerVersion("v1".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Mock retriever that returns a fixed `Vec<SearchHit>` regardless of
|
||||
/// the query / k / filters. Captures the invocation count for assertions.
|
||||
pub struct MockRetriever {
|
||||
pub hits: Vec<SearchHit>,
|
||||
pub calls: std::sync::atomic::AtomicUsize,
|
||||
}
|
||||
|
||||
impl MockRetriever {
|
||||
pub fn new(hits: Vec<SearchHit>) -> Self {
|
||||
Self {
|
||||
hits,
|
||||
calls: std::sync::atomic::AtomicUsize::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn calls(&self) -> usize {
|
||||
self.calls.load(std::sync::atomic::Ordering::SeqCst)
|
||||
}
|
||||
}
|
||||
|
||||
impl Retriever for MockRetriever {
|
||||
fn search(&self, _q: &SearchQuery) -> anyhow::Result<Vec<SearchHit>> {
|
||||
self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
Ok(self.hits.clone())
|
||||
}
|
||||
fn index_version(&self) -> IndexVersion {
|
||||
IndexVersion("test-iv".to_string())
|
||||
}
|
||||
}
|
||||
|
||||
/// Pad a short prefix to the 32-hex shape `kebab_core` newtypes expect.
|
||||
pub fn id32(prefix: &str) -> String {
|
||||
let mut s = prefix.to_string();
|
||||
while s.len() < 32 {
|
||||
s.push('0');
|
||||
}
|
||||
s.truncate(32);
|
||||
s
|
||||
}
|
||||
456
crates/kebab-rag/tests/pipeline.rs
Normal file
456
crates/kebab-rag/tests/pipeline.rs
Normal file
@@ -0,0 +1,456 @@
|
||||
//! Integration tests for `RagPipeline` (P4-3 spec test plan).
|
||||
//!
|
||||
//! Real adapters (Ollama, fastembed, LanceDB) are NOT used. Every test
|
||||
//! injects a `MockLanguageModel` and a `MockRetriever` so the pipeline's
|
||||
//! behavior is exercised in isolation from network / heavy IO.
|
||||
|
||||
mod common;
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
use common::{MockRetriever, RagEnv, id32, mk_hit};
|
||||
use kebab_core::{
|
||||
FinishReason, LanguageModel, Retriever, SearchMode, TokenChunk, TokenUsage,
|
||||
};
|
||||
use kebab_llm::MockLanguageModel;
|
||||
use kebab_rag::{AskOpts, RagPipeline, RefusalReason};
|
||||
|
||||
/// LM ID used everywhere — kept short so snapshots stay stable.
|
||||
const TEST_LM_ID: &str = "mock-lm";
|
||||
|
||||
/// Counter wrapper so tests can assert "no LLM call happened".
|
||||
struct CountingLm {
|
||||
inner: MockLanguageModel,
|
||||
calls: std::sync::atomic::AtomicUsize,
|
||||
}
|
||||
|
||||
impl CountingLm {
|
||||
fn new(canned: &str) -> Self {
|
||||
Self {
|
||||
inner: MockLanguageModel {
|
||||
model_id: TEST_LM_ID.to_string(),
|
||||
provider: "mock".to_string(),
|
||||
context_tokens: 32_768,
|
||||
canned_response: canned.to_string(),
|
||||
canned_finish: FinishReason::Stop,
|
||||
canned_usage: TokenUsage {
|
||||
prompt_tokens: 10,
|
||||
completion_tokens: 5,
|
||||
latency_ms: 7,
|
||||
},
|
||||
},
|
||||
calls: std::sync::atomic::AtomicUsize::new(0),
|
||||
}
|
||||
}
|
||||
fn calls(&self) -> usize {
|
||||
self.calls.load(Ordering::SeqCst)
|
||||
}
|
||||
}
|
||||
|
||||
impl LanguageModel for CountingLm {
|
||||
fn model_ref(&self) -> kebab_core::ModelRef {
|
||||
self.inner.model_ref()
|
||||
}
|
||||
fn context_tokens(&self) -> usize {
|
||||
self.inner.context_tokens()
|
||||
}
|
||||
fn generate_stream(
|
||||
&self,
|
||||
req: kebab_core::GenerateRequest,
|
||||
) -> anyhow::Result<Box<dyn Iterator<Item = anyhow::Result<TokenChunk>> + Send>> {
|
||||
self.calls.fetch_add(1, Ordering::SeqCst);
|
||||
self.inner.generate_stream(req)
|
||||
}
|
||||
}
|
||||
|
||||
fn default_opts() -> AskOpts {
|
||||
AskOpts {
|
||||
k: 5,
|
||||
explain: false,
|
||||
mode: SearchMode::Lexical,
|
||||
temperature: Some(0.0),
|
||||
seed: Some(0),
|
||||
stream_sink: None,
|
||||
}
|
||||
}
|
||||
|
||||
// ── 1. empty hits → NoChunks, no LLM call ────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn empty_hits_refuses_no_chunks_without_llm_call() {
|
||||
let env = RagEnv::new();
|
||||
let retriever: Arc<dyn Retriever> = Arc::new(MockRetriever::new(Vec::new()));
|
||||
let lm = Arc::new(CountingLm::new("(unused)"));
|
||||
let lm_dyn: Arc<dyn LanguageModel> = lm.clone();
|
||||
let pipeline = RagPipeline::new(env.config.clone(), retriever, lm_dyn, env.sqlite.clone());
|
||||
|
||||
let answer = pipeline.ask("anything", default_opts()).unwrap();
|
||||
assert_eq!(answer.refusal_reason, Some(RefusalReason::NoChunks));
|
||||
assert!(!answer.grounded);
|
||||
assert!(answer.citations.is_empty());
|
||||
assert_eq!(lm.calls(), 0, "LM must NOT be called on empty hits");
|
||||
assert_eq!(env.count_answers(), 1, "answers row written for refusal");
|
||||
}
|
||||
|
||||
// ── 2. score gate refuses without LLM call ────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn top_below_gate_refuses_score_gate_without_llm_call() {
|
||||
let env = RagEnv::new();
|
||||
// top score 0.10 below default gate 0.30
|
||||
let hits = vec![
|
||||
mk_hit(1, &id32("c1"), &id32("d1"), "notes/a.md", 0.10, &["A"]),
|
||||
mk_hit(2, &id32("c2"), &id32("d2"), "notes/b.md", 0.05, &["B"]),
|
||||
];
|
||||
let retriever: Arc<dyn Retriever> = Arc::new(MockRetriever::new(hits));
|
||||
let lm = Arc::new(CountingLm::new("(unused)"));
|
||||
let lm_dyn: Arc<dyn LanguageModel> = lm.clone();
|
||||
let pipeline = RagPipeline::new(env.config.clone(), retriever, lm_dyn, env.sqlite.clone());
|
||||
|
||||
let answer = pipeline.ask("q", default_opts()).unwrap();
|
||||
assert_eq!(answer.refusal_reason, Some(RefusalReason::ScoreGate));
|
||||
assert!(!answer.grounded);
|
||||
assert_eq!(answer.citations.len(), 2, "all near-miss candidates surfaced");
|
||||
for c in &answer.citations {
|
||||
assert!(c.marker.is_none(), "ScoreGate citations have no marker");
|
||||
}
|
||||
assert_eq!(lm.calls(), 0, "LM must NOT be called when gate refuses");
|
||||
assert_eq!(env.count_answers(), 1);
|
||||
assert!(answer.answer.contains("근거 부족"));
|
||||
assert!(answer.answer.contains("notes/a.md"));
|
||||
}
|
||||
|
||||
// ── 3. grounded happy path with [#1] ──────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn grounded_happy_path_marker_one() {
|
||||
let env = RagEnv::new();
|
||||
let cid = id32("c1");
|
||||
let did = id32("d1");
|
||||
env.seed_chunk(&cid, &did, "notes/a.md", "Rust is a systems language.", &["Intro"]);
|
||||
let hits = vec![mk_hit(1, &cid, &did, "notes/a.md", 0.85, &["Intro"])];
|
||||
let retriever: Arc<dyn Retriever> = Arc::new(MockRetriever::new(hits));
|
||||
let canned = "Rust is a systems language. [#1]";
|
||||
let lm: Arc<dyn LanguageModel> = Arc::new(CountingLm::new(canned));
|
||||
let pipeline = RagPipeline::new(env.config.clone(), retriever, lm, env.sqlite.clone());
|
||||
|
||||
let answer = pipeline.ask("what is rust", default_opts()).unwrap();
|
||||
assert!(answer.grounded);
|
||||
assert_eq!(answer.refusal_reason, None);
|
||||
assert_eq!(answer.citations.len(), 1);
|
||||
assert_eq!(answer.citations[0].marker.as_deref(), Some("[1]"));
|
||||
assert_eq!(answer.retrieval.chunks_used, 1);
|
||||
assert_eq!(env.count_answers(), 1);
|
||||
}
|
||||
|
||||
// ── 4. unknown marker [#7] → LlmSelfJudge ─────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn unknown_marker_refuses_llm_self_judge() {
|
||||
let env = RagEnv::new();
|
||||
let cid = id32("c1");
|
||||
let did = id32("d1");
|
||||
env.seed_chunk(&cid, &did, "notes/a.md", "doc text", &["Intro"]);
|
||||
let hits = vec![mk_hit(1, &cid, &did, "notes/a.md", 0.85, &["Intro"])];
|
||||
let retriever: Arc<dyn Retriever> = Arc::new(MockRetriever::new(hits));
|
||||
// Marker 7 is NOT in the packed set (only #1 is).
|
||||
let lm: Arc<dyn LanguageModel> = Arc::new(CountingLm::new("answer text [#7]"));
|
||||
let pipeline = RagPipeline::new(env.config.clone(), retriever, lm, env.sqlite.clone());
|
||||
|
||||
let answer = pipeline.ask("q", default_opts()).unwrap();
|
||||
assert_eq!(answer.refusal_reason, Some(RefusalReason::LlmSelfJudge));
|
||||
assert!(!answer.grounded);
|
||||
// Even unknown markers are NOT included in citations (we only report
|
||||
// markers that map to the packed set).
|
||||
assert!(answer.citations.is_empty());
|
||||
}
|
||||
|
||||
// ── 5. [1] (no #) → LlmSelfJudge (regex strictness) ───────────────────────
|
||||
|
||||
#[test]
|
||||
fn marker_without_hash_is_no_marker() {
|
||||
let env = RagEnv::new();
|
||||
let cid = id32("c1");
|
||||
let did = id32("d1");
|
||||
env.seed_chunk(&cid, &did, "notes/a.md", "doc text", &["Intro"]);
|
||||
let hits = vec![mk_hit(1, &cid, &did, "notes/a.md", 0.85, &["Intro"])];
|
||||
let retriever: Arc<dyn Retriever> = Arc::new(MockRetriever::new(hits));
|
||||
// `[1]` is NOT a valid marker — strict regex requires `[#1]`.
|
||||
let lm: Arc<dyn LanguageModel> = Arc::new(CountingLm::new("the answer [1]"));
|
||||
let pipeline = RagPipeline::new(env.config.clone(), retriever, lm, env.sqlite.clone());
|
||||
|
||||
let answer = pipeline.ask("q", default_opts()).unwrap();
|
||||
assert_eq!(answer.refusal_reason, Some(RefusalReason::LlmSelfJudge));
|
||||
assert!(!answer.grounded);
|
||||
}
|
||||
|
||||
// ── 6. vec![1] no real citation → LlmSelfJudge (no false positive) ────────
|
||||
|
||||
#[test]
|
||||
fn vec_bracket_one_is_no_false_positive() {
|
||||
let env = RagEnv::new();
|
||||
let cid = id32("c1");
|
||||
let did = id32("d1");
|
||||
env.seed_chunk(&cid, &did, "notes/a.md", "doc", &["Intro"]);
|
||||
let hits = vec![mk_hit(1, &cid, &did, "notes/a.md", 0.85, &["Intro"])];
|
||||
let retriever: Arc<dyn Retriever> = Arc::new(MockRetriever::new(hits));
|
||||
// `vec![1]` MUST NOT be misread as a citation marker.
|
||||
let lm: Arc<dyn LanguageModel> = Arc::new(CountingLm::new("see vec![1] in code"));
|
||||
let pipeline = RagPipeline::new(env.config.clone(), retriever, lm, env.sqlite.clone());
|
||||
|
||||
let answer = pipeline.ask("q", default_opts()).unwrap();
|
||||
assert_eq!(answer.refusal_reason, Some(RefusalReason::LlmSelfJudge));
|
||||
assert!(!answer.grounded);
|
||||
}
|
||||
|
||||
// ── 7. "근거가 부족합니다" → LlmSelfJudge ────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn explicit_korean_refusal_is_self_judge() {
|
||||
let env = RagEnv::new();
|
||||
let cid = id32("c1");
|
||||
let did = id32("d1");
|
||||
env.seed_chunk(&cid, &did, "notes/a.md", "doc", &["Intro"]);
|
||||
let hits = vec![mk_hit(1, &cid, &did, "notes/a.md", 0.85, &["Intro"])];
|
||||
let retriever: Arc<dyn Retriever> = Arc::new(MockRetriever::new(hits));
|
||||
let lm: Arc<dyn LanguageModel> = Arc::new(CountingLm::new("근거가 부족합니다."));
|
||||
let pipeline = RagPipeline::new(env.config.clone(), retriever, lm, env.sqlite.clone());
|
||||
|
||||
let answer = pipeline.ask("q", default_opts()).unwrap();
|
||||
assert_eq!(answer.refusal_reason, Some(RefusalReason::LlmSelfJudge));
|
||||
assert!(!answer.grounded);
|
||||
}
|
||||
|
||||
// ── 8. context packing budget overflow ────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn packing_stops_before_budget_overflow() {
|
||||
let env = RagEnv::new();
|
||||
// Squeeze the budget so only one chunk fits.
|
||||
let mut cfg = env.config.clone();
|
||||
cfg.rag.max_context_tokens = 50; // very small budget
|
||||
// Three giant chunks
|
||||
let huge_text: String = "X".repeat(2_000); // ~500 tokens each
|
||||
let mut hits = Vec::new();
|
||||
for i in 0..3_u32 {
|
||||
let cid = id32(&format!("c{i}"));
|
||||
let did = id32(&format!("d{i}"));
|
||||
env.seed_chunk(&cid, &did, &format!("notes/a{i}.md"), &huge_text, &["Intro"]);
|
||||
hits.push(mk_hit(i + 1, &cid, &did, &format!("notes/a{i}.md"), 0.9, &["Intro"]));
|
||||
}
|
||||
let retriever: Arc<dyn Retriever> = Arc::new(MockRetriever::new(hits));
|
||||
let lm: Arc<dyn LanguageModel> = Arc::new(CountingLm::new("ok [#1]"));
|
||||
let pipeline = RagPipeline::new(cfg, retriever, lm, env.sqlite.clone());
|
||||
|
||||
let answer = pipeline.ask("q", default_opts()).unwrap();
|
||||
// At least one chunk was packed; the budget cap should keep it to <= 1.
|
||||
assert_eq!(
|
||||
answer.retrieval.chunks_used, 1,
|
||||
"exactly one chunk fits when budget is tiny"
|
||||
);
|
||||
assert_eq!(answer.retrieval.chunks_returned, 3);
|
||||
assert!(answer.grounded);
|
||||
}
|
||||
|
||||
// ── 9. streaming forwards tokens to mpsc ──────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn streaming_forwards_tokens_to_sink() {
|
||||
let env = RagEnv::new();
|
||||
let cid = id32("c1");
|
||||
let did = id32("d1");
|
||||
env.seed_chunk(&cid, &did, "notes/a.md", "doc", &["Intro"]);
|
||||
let hits = vec![mk_hit(1, &cid, &did, "notes/a.md", 0.85, &["Intro"])];
|
||||
let retriever: Arc<dyn Retriever> = Arc::new(MockRetriever::new(hits));
|
||||
let canned = "ok [#1]";
|
||||
let lm: Arc<dyn LanguageModel> = Arc::new(CountingLm::new(canned));
|
||||
let pipeline = RagPipeline::new(env.config.clone(), retriever, lm, env.sqlite.clone());
|
||||
|
||||
let (tx, rx) = std::sync::mpsc::channel::<String>();
|
||||
let mut opts = default_opts();
|
||||
opts.stream_sink = Some(tx);
|
||||
let _ = pipeline.ask("q", opts).unwrap();
|
||||
let collected: String = rx.into_iter().collect::<Vec<_>>().join("");
|
||||
assert_eq!(collected, canned);
|
||||
}
|
||||
|
||||
// ── 10. dropped receiver does NOT abort generation ────────────────────────
|
||||
|
||||
#[test]
|
||||
fn dropped_receiver_does_not_abort_generation() {
|
||||
let env = RagEnv::new();
|
||||
let cid = id32("c1");
|
||||
let did = id32("d1");
|
||||
env.seed_chunk(&cid, &did, "notes/a.md", "doc", &["Intro"]);
|
||||
let hits = vec![mk_hit(1, &cid, &did, "notes/a.md", 0.85, &["Intro"])];
|
||||
let retriever: Arc<dyn Retriever> = Arc::new(MockRetriever::new(hits));
|
||||
let canned = "ok [#1]";
|
||||
let lm: Arc<dyn LanguageModel> = Arc::new(CountingLm::new(canned));
|
||||
let pipeline = RagPipeline::new(env.config.clone(), retriever, lm, env.sqlite.clone());
|
||||
|
||||
let (tx, rx) = std::sync::mpsc::channel::<String>();
|
||||
drop(rx); // receiver gone — every send fails silently
|
||||
let mut opts = default_opts();
|
||||
opts.stream_sink = Some(tx);
|
||||
let answer = pipeline.ask("q", opts).unwrap();
|
||||
assert_eq!(answer.answer, canned, "generation completes despite dead sink");
|
||||
assert!(answer.grounded);
|
||||
assert_eq!(env.count_answers(), 1, "answers row still persisted");
|
||||
}
|
||||
|
||||
// ── 11. Send + Sync compile check ─────────────────────────────────────────
|
||||
// Implemented inside `kb-rag::pipeline::tests::rag_pipeline_is_send_sync`.
|
||||
|
||||
// ── 12. usage from final Done chunk ───────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn usage_populated_from_done_chunk() {
|
||||
let env = RagEnv::new();
|
||||
let cid = id32("c1");
|
||||
let did = id32("d1");
|
||||
env.seed_chunk(&cid, &did, "notes/a.md", "doc", &["Intro"]);
|
||||
let hits = vec![mk_hit(1, &cid, &did, "notes/a.md", 0.85, &["Intro"])];
|
||||
let retriever: Arc<dyn Retriever> = Arc::new(MockRetriever::new(hits));
|
||||
let lm: Arc<dyn LanguageModel> = Arc::new(CountingLm::new("ok [#1]"));
|
||||
let pipeline = RagPipeline::new(env.config.clone(), retriever, lm, env.sqlite.clone());
|
||||
|
||||
let answer = pipeline.ask("q", default_opts()).unwrap();
|
||||
assert_eq!(answer.usage.prompt_tokens, 10, "from canned_usage");
|
||||
assert_eq!(answer.usage.completion_tokens, 5);
|
||||
}
|
||||
|
||||
// ── 13. answers row inserted in all paths (incl. refusals) ────────────────
|
||||
|
||||
#[test]
|
||||
fn answers_row_inserted_for_each_refusal_kind() {
|
||||
// NoChunks
|
||||
{
|
||||
let env = RagEnv::new();
|
||||
let retriever: Arc<dyn Retriever> = Arc::new(MockRetriever::new(Vec::new()));
|
||||
let lm: Arc<dyn LanguageModel> = Arc::new(CountingLm::new(""));
|
||||
let p = RagPipeline::new(env.config.clone(), retriever, lm, env.sqlite.clone());
|
||||
p.ask("q", default_opts()).unwrap();
|
||||
assert_eq!(env.count_answers(), 1);
|
||||
}
|
||||
// ScoreGate
|
||||
{
|
||||
let env = RagEnv::new();
|
||||
let cid = id32("c1");
|
||||
let did = id32("d1");
|
||||
env.seed_chunk(&cid, &did, "notes/a.md", "doc", &["Intro"]);
|
||||
let hits = vec![mk_hit(1, &cid, &did, "notes/a.md", 0.05, &["Intro"])];
|
||||
let retriever: Arc<dyn Retriever> = Arc::new(MockRetriever::new(hits));
|
||||
let lm: Arc<dyn LanguageModel> = Arc::new(CountingLm::new(""));
|
||||
let p = RagPipeline::new(env.config.clone(), retriever, lm, env.sqlite.clone());
|
||||
p.ask("q", default_opts()).unwrap();
|
||||
assert_eq!(env.count_answers(), 1);
|
||||
}
|
||||
// LlmSelfJudge (silent ungrounded)
|
||||
{
|
||||
let env = RagEnv::new();
|
||||
let cid = id32("c1");
|
||||
let did = id32("d1");
|
||||
env.seed_chunk(&cid, &did, "notes/a.md", "doc", &["Intro"]);
|
||||
let hits = vec![mk_hit(1, &cid, &did, "notes/a.md", 0.85, &["Intro"])];
|
||||
let retriever: Arc<dyn Retriever> = Arc::new(MockRetriever::new(hits));
|
||||
let lm: Arc<dyn LanguageModel> = Arc::new(CountingLm::new("answer with no marker"));
|
||||
let p = RagPipeline::new(env.config.clone(), retriever, lm, env.sqlite.clone());
|
||||
p.ask("q", default_opts()).unwrap();
|
||||
assert_eq!(env.count_answers(), 1);
|
||||
}
|
||||
}
|
||||
|
||||
// ── 14. determinism: temp=0 + seed=0 → identical Answer (mock) ────────────
|
||||
|
||||
#[test]
|
||||
fn determinism_temperature_zero_seed_zero() {
|
||||
let env = RagEnv::new();
|
||||
let cid = id32("c1");
|
||||
let did = id32("d1");
|
||||
env.seed_chunk(&cid, &did, "notes/a.md", "doc", &["Intro"]);
|
||||
let hits = vec![mk_hit(1, &cid, &did, "notes/a.md", 0.85, &["Intro"])];
|
||||
// Two pipelines, two retrievers, two LMs — but identical canned configs.
|
||||
let mk_pipeline = || {
|
||||
let r: Arc<dyn Retriever> = Arc::new(MockRetriever::new(hits.clone()));
|
||||
let lm: Arc<dyn LanguageModel> = Arc::new(CountingLm::new("Rust is. [#1]"));
|
||||
RagPipeline::new(env.config.clone(), r, lm, env.sqlite.clone())
|
||||
};
|
||||
let a1 = mk_pipeline().ask("q", default_opts()).unwrap();
|
||||
let a2 = mk_pipeline().ask("q", default_opts()).unwrap();
|
||||
assert_eq!(a1.answer, a2.answer);
|
||||
assert_eq!(a1.grounded, a2.grounded);
|
||||
assert_eq!(a1.citations, a2.citations);
|
||||
assert_eq!(a1.retrieval.chunks_used, a2.retrieval.chunks_used);
|
||||
assert_eq!(a1.retrieval.k, a2.retrieval.k);
|
||||
// trace_id and created_at and latency_ms WILL differ — they include
|
||||
// wall-clock — so we don't compare them.
|
||||
}
|
||||
|
||||
// ── 15a. all chunks unfetchable from store → NoChunks fallback ───────────
|
||||
|
||||
#[test]
|
||||
fn unfetchable_chunks_fall_back_to_no_chunks() {
|
||||
// Hits exist (so the score gate passes) but their chunk_id rows are
|
||||
// never seeded into the store — `DocumentStore::get_chunk` returns
|
||||
// None for every one. Pipeline should detect the empty packed list
|
||||
// and refuse with NoChunks rather than letting the LLM run with an
|
||||
// empty `[근거]` block (which would self-refuse → LlmSelfJudge).
|
||||
let env = RagEnv::new();
|
||||
let cid = id32("missing");
|
||||
let did = id32("d_missing");
|
||||
// NOTE: no `env.seed_chunk(...)` call — chunk row absent from store.
|
||||
let hits = vec![mk_hit(1, &cid, &did, "notes/missing.md", 0.85, &["X"])];
|
||||
let retriever: Arc<dyn Retriever> = Arc::new(MockRetriever::new(hits));
|
||||
let lm = Arc::new(CountingLm::new("(should never run)"));
|
||||
let lm_dyn: Arc<dyn LanguageModel> = lm.clone();
|
||||
let pipeline = RagPipeline::new(env.config.clone(), retriever, lm_dyn, env.sqlite.clone());
|
||||
|
||||
let answer = pipeline.ask("q", default_opts()).unwrap();
|
||||
assert_eq!(answer.refusal_reason, Some(RefusalReason::NoChunks));
|
||||
assert!(!answer.grounded);
|
||||
assert!(answer.citations.is_empty());
|
||||
assert_eq!(
|
||||
lm.calls(),
|
||||
0,
|
||||
"LM must NOT be called when every retrieved chunk is unfetchable"
|
||||
);
|
||||
assert_eq!(env.count_answers(), 1, "answers row written for refusal");
|
||||
}
|
||||
|
||||
// ── 15. snapshot Answer JSON stable ───────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn answer_json_serializes_with_expected_keys() {
|
||||
let env = RagEnv::new();
|
||||
let cid = id32("c1");
|
||||
let did = id32("d1");
|
||||
env.seed_chunk(&cid, &did, "notes/a.md", "Rust is a systems language.", &["Intro"]);
|
||||
let hits = vec![mk_hit(1, &cid, &did, "notes/a.md", 0.85, &["Intro"])];
|
||||
let retriever: Arc<dyn Retriever> = Arc::new(MockRetriever::new(hits));
|
||||
let lm: Arc<dyn LanguageModel> = Arc::new(CountingLm::new("Rust is. [#1]"));
|
||||
let pipeline = RagPipeline::new(env.config.clone(), retriever, lm, env.sqlite.clone());
|
||||
let answer = pipeline.ask("what", default_opts()).unwrap();
|
||||
let v: serde_json::Value = serde_json::to_value(&answer).unwrap();
|
||||
// Stable top-level key set per `answer.v1` (§2.3).
|
||||
let keys: Vec<&str> = v.as_object().unwrap().keys().map(|s| s.as_str()).collect();
|
||||
for needed in [
|
||||
"answer",
|
||||
"citations",
|
||||
"grounded",
|
||||
"refusal_reason",
|
||||
"model",
|
||||
"embedding",
|
||||
"prompt_template_version",
|
||||
"retrieval",
|
||||
"usage",
|
||||
"created_at",
|
||||
] {
|
||||
assert!(keys.contains(&needed), "missing top-level key {needed}");
|
||||
}
|
||||
// citations is a JSON array
|
||||
assert!(v["citations"].is_array());
|
||||
// retrieval.trace_id starts with `ret_`
|
||||
let trace_id = v["retrieval"]["trace_id"].as_str().unwrap();
|
||||
assert!(trace_id.starts_with("ret_"), "got trace_id {trace_id:?}");
|
||||
}
|
||||
Reference in New Issue
Block a user