Merge pull request 'fix: 28.4k 문서 도그푸딩이 잡은 결함 3건 + eval 회귀 게이트' (#233) from fix/lance-compaction into main
This commit was merged in pull request #233.
This commit is contained in:
@@ -89,7 +89,7 @@ Markdown · PDF · 이미지(OCR + caption) · 소스코드(Rust/Python/TS/JS/Go
|
||||
| `kebab list docs` | 색인된 문서 목록 |
|
||||
| `kebab inspect doc <id>` / `inspect chunk <id>` | raw record 보기 |
|
||||
| `kebab fetch chunk\|doc\|span <id> [flags]` | indexed corpus 에서 verbatim text fetch |
|
||||
| `kebab eval run \| aggregate \| compare \| variants` | golden query 회귀 측정 + 변형 일관성 진단 |
|
||||
| `kebab eval run \| aggregate \| compare \| variants` | golden query 회귀 측정 + 변형 일관성 진단. `compare --max-drop <낙폭>` 은 어떤 지표든 그 이상 **떨어지면** exit 1 — 절대 하한이 아니라 델타 예산이다 (없으면 delta 만 출력하고 항상 exit 0) |
|
||||
| `kebab schema [--json]` | introspection — wire schemas / capabilities / models / stats |
|
||||
| `kebab doctor` | 설정 / 모델 / DB 헬스 체크 |
|
||||
| `kebab mcp` | MCP stdio server (`search` / `bulk_search` / `ask` / `fetch` / `schema` / `doctor` / `ingest_file` / `ingest_stdin`) |
|
||||
|
||||
@@ -37,3 +37,18 @@ impl fmt::Display for NoHitSignal {
|
||||
}
|
||||
|
||||
impl std::error::Error for NoHitSignal {}
|
||||
|
||||
/// `kebab eval compare --max-drop` found a metric that regressed past
|
||||
/// the tolerance. Like the others here this is normal output, not a
|
||||
/// failure — it just has to reach the shell as a non-zero status so a
|
||||
/// golden set can actually gate something.
|
||||
#[derive(Debug)]
|
||||
pub struct EvalRegression;
|
||||
|
||||
impl fmt::Display for EvalRegression {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str("eval regression")
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for EvalRegression {}
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
//!
|
||||
//! See `docs/superpowers/specs/2026-05-07-p9-fb-27-introspection-and-error-wire-design.md`.
|
||||
|
||||
pub use crate::doctor_signal::{DoctorUnhealthy, NoHitSignal, RefusalSignal};
|
||||
pub use crate::doctor_signal::{DoctorUnhealthy, EvalRegression, NoHitSignal, RefusalSignal};
|
||||
|
||||
pub use kebab_config::{ConfigInvalid, ConfigNotFound};
|
||||
pub use kebab_llm_local::LlmError;
|
||||
|
||||
@@ -7,7 +7,7 @@ use std::process::ExitCode;
|
||||
use anyhow::Context;
|
||||
use clap::{Parser, Subcommand};
|
||||
|
||||
use kebab_app::doctor_signal::{DoctorUnhealthy, NoHitSignal, RefusalSignal};
|
||||
use kebab_app::doctor_signal::{DoctorUnhealthy, EvalRegression, NoHitSignal, RefusalSignal};
|
||||
|
||||
mod cancel;
|
||||
mod progress;
|
||||
@@ -457,6 +457,14 @@ enum EvalWhat {
|
||||
/// `runs_dir/<run_b>/report.md`.
|
||||
#[arg(long)]
|
||||
write_report: bool,
|
||||
/// Exit 1 when any metric dropped by more than this much
|
||||
/// (e.g. `--max-drop 0.03` tolerates a 3-point fall). This is a
|
||||
/// **delta** budget, not an absolute floor. Without it `compare`
|
||||
/// prints deltas and always exits 0, so a golden set cannot gate
|
||||
/// anything. `empty_result_rate` is checked in the opposite
|
||||
/// direction — a rise there is the regression.
|
||||
#[arg(long, value_name = "DELTA")]
|
||||
max_drop: Option<f64>,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -576,6 +584,9 @@ fn exit_code(err: &anyhow::Error) -> u8 {
|
||||
if err.downcast_ref::<NoHitSignal>().is_some() {
|
||||
return 1;
|
||||
}
|
||||
if err.downcast_ref::<EvalRegression>().is_some() {
|
||||
return 1;
|
||||
}
|
||||
if err.downcast_ref::<DoctorUnhealthy>().is_some() {
|
||||
return 3;
|
||||
}
|
||||
@@ -1451,6 +1462,7 @@ fn run(cli: &Cli) -> anyhow::Result<()> {
|
||||
run_b,
|
||||
strict_chunker_version,
|
||||
write_report,
|
||||
max_drop,
|
||||
} => {
|
||||
let opts = kebab_eval::CompareOpts {
|
||||
strict_chunker_version: *strict_chunker_version,
|
||||
@@ -1477,6 +1489,25 @@ fn run(cli: &Cli) -> anyhow::Result<()> {
|
||||
eprintln!("wrote {}", path.display());
|
||||
}
|
||||
}
|
||||
if let Some(tol) = *max_drop {
|
||||
if !(tol.is_finite() && tol >= 0.0) {
|
||||
anyhow::bail!(
|
||||
"--max-drop 은 0 이상의 유한한 수여야 한다 (받은 값: {tol}). \
|
||||
음수나 nan 은 게이트를 무력화한다"
|
||||
);
|
||||
}
|
||||
let bad = kebab_eval::regressions(&report, tol);
|
||||
if !bad.is_empty() {
|
||||
eprintln!("eval regression (허용 낙폭 {tol}):");
|
||||
for line in &bad {
|
||||
eprintln!(" {line}");
|
||||
}
|
||||
return Err(EvalRegression.into());
|
||||
}
|
||||
if !cli.json {
|
||||
eprintln!("eval: 허용 낙폭 {tol} 안 — 회귀 없음");
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -488,6 +488,91 @@ fn build_deltas(
|
||||
})
|
||||
}
|
||||
|
||||
/// Metrics that got worse by more than `tolerance` from run A to run B.
|
||||
///
|
||||
/// Returns one line per violation, empty when B is within tolerance of A.
|
||||
/// This is what turns a golden set into a gate: `compare` on its own only
|
||||
/// prints deltas and always exits 0, so "did this change make retrieval
|
||||
/// worse" had no machine-checkable answer.
|
||||
///
|
||||
/// Direction matters per metric. Everything here is higher-is-better
|
||||
/// except `empty_result_rate`, where a rise means more queries came back
|
||||
/// with nothing.
|
||||
///
|
||||
/// A metric that was measurable in A but is `NaN` in B counts as a
|
||||
/// violation rather than a skip. That case means the measurement itself
|
||||
/// disappeared — usually a golden set that lost its ground truth — and
|
||||
/// silently reporting "no regression" for it would be the worst possible
|
||||
/// answer.
|
||||
#[must_use]
|
||||
pub fn regressions(report: &CompareReport, tolerance: f64) -> Vec<String> {
|
||||
let mut out = Vec::new();
|
||||
let (a, b) = (&report.aggregate_a, &report.aggregate_b);
|
||||
|
||||
let mut check = |name: String, av: f32, bv: f32, lower_is_better: bool| {
|
||||
if av.is_nan() {
|
||||
// Never measured on the baseline side — nothing to regress from.
|
||||
return;
|
||||
}
|
||||
if bv.is_nan() {
|
||||
out.push(format!(
|
||||
"{name}: {av:.4} -> 측정 불가 (B 에서 지표가 사라짐)"
|
||||
));
|
||||
return;
|
||||
}
|
||||
let delta = f64::from(bv - av);
|
||||
let worse = if lower_is_better { delta } else { -delta };
|
||||
if worse > tolerance {
|
||||
out.push(format!(
|
||||
"{name}: {av:.4} -> {bv:.4} ({delta:+.4}, 허용치 {tolerance:.4})"
|
||||
));
|
||||
}
|
||||
};
|
||||
|
||||
for k in crate::metrics::TOP_K_VARIANTS {
|
||||
let nan = f32::NAN;
|
||||
check(
|
||||
format!("hit_at_k[{k}]"),
|
||||
a.hit_at_k.get(k).copied().unwrap_or(nan),
|
||||
b.hit_at_k.get(k).copied().unwrap_or(nan),
|
||||
false,
|
||||
);
|
||||
check(
|
||||
format!("recall_at_k_doc[{k}]"),
|
||||
a.recall_at_k_doc.get(k).copied().unwrap_or(nan),
|
||||
b.recall_at_k_doc.get(k).copied().unwrap_or(nan),
|
||||
false,
|
||||
);
|
||||
check(
|
||||
format!("precision_at_k_chunk[{k}]"),
|
||||
a.precision_at_k_chunk.get(k).copied().unwrap_or(nan),
|
||||
b.precision_at_k_chunk.get(k).copied().unwrap_or(nan),
|
||||
false,
|
||||
);
|
||||
}
|
||||
check("mrr".into(), a.mrr, b.mrr, false);
|
||||
check(
|
||||
"citation_coverage".into(),
|
||||
a.citation_coverage,
|
||||
b.citation_coverage,
|
||||
false,
|
||||
);
|
||||
check("groundedness".into(), a.groundedness, b.groundedness, false);
|
||||
check(
|
||||
"refusal_correctness".into(),
|
||||
a.refusal_correctness,
|
||||
b.refusal_correctness,
|
||||
false,
|
||||
);
|
||||
check(
|
||||
"empty_result_rate".into(),
|
||||
a.empty_result_rate,
|
||||
b.empty_result_rate,
|
||||
true,
|
||||
);
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -545,6 +630,84 @@ mod tests {
|
||||
assert_eq!(d["chunker_version_match"], "exact");
|
||||
}
|
||||
|
||||
fn agg(mrr: f32, recall10: f32, empty: f32) -> AggregateMetrics {
|
||||
AggregateMetrics {
|
||||
hit_at_k: Default::default(),
|
||||
mrr,
|
||||
recall_at_k_doc: [(10u32, recall10)].into_iter().collect(),
|
||||
precision_at_k_chunk: Default::default(),
|
||||
citation_coverage: f32::NAN,
|
||||
groundedness: 1.0,
|
||||
empty_result_rate: empty,
|
||||
refusal_correctness: f32::NAN,
|
||||
total_queries: 10,
|
||||
failed_queries: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn report(a: AggregateMetrics, b: AggregateMetrics) -> CompareReport {
|
||||
CompareReport {
|
||||
run_a: "a".into(),
|
||||
run_b: "b".into(),
|
||||
deltas: build_deltas(&a, &b, "exact"),
|
||||
aggregate_a: a,
|
||||
aggregate_b: b,
|
||||
per_query: vec![],
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn regressions_empty_when_within_tolerance() {
|
||||
// 0.01 하락은 허용치 0.03 안이다.
|
||||
let r = report(agg(0.50, 0.95, 0.0), agg(0.49, 0.94, 0.0));
|
||||
assert!(regressions(&r, 0.03).is_empty());
|
||||
// 개선은 당연히 위반이 아니다.
|
||||
let up = report(agg(0.50, 0.95, 0.0), agg(0.80, 0.99, 0.0));
|
||||
assert!(regressions(&up, 0.03).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn regressions_flag_metric_drops_beyond_tolerance() {
|
||||
let r = report(agg(0.50, 0.95, 0.0), agg(0.40, 0.80, 0.0));
|
||||
let v = regressions(&r, 0.03);
|
||||
assert_eq!(v.len(), 2, "mrr + recall@10 두 개여야 한다: {v:?}");
|
||||
assert!(v.iter().any(|s| s.starts_with("mrr:")), "{v:?}");
|
||||
assert!(
|
||||
v.iter().any(|s| s.starts_with("recall_at_k_doc[10]:")),
|
||||
"{v:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn regressions_treat_empty_result_rate_as_lower_is_better() {
|
||||
// 빈 결과 비율은 오르는 쪽이 나빠지는 것이다.
|
||||
let worse = report(agg(0.5, 0.95, 0.10), agg(0.5, 0.95, 0.30));
|
||||
assert_eq!(regressions(&worse, 0.03).len(), 1);
|
||||
// 내려가는 것은 개선이므로 위반이 아니다.
|
||||
let better = report(agg(0.5, 0.95, 0.30), agg(0.5, 0.95, 0.10));
|
||||
assert!(regressions(&better, 0.03).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn regressions_flag_metric_that_stopped_being_measurable() {
|
||||
// A 에서는 재던 지표가 B 에서 NaN 이면 "회귀 없음"이 아니라 위반이다.
|
||||
// 골든셋이 ground truth 를 잃은 경우가 정확히 이 모양으로 나타난다.
|
||||
let a = agg(0.5, 0.95, 0.0);
|
||||
let mut b = agg(0.5, 0.95, 0.0);
|
||||
b.recall_at_k_doc.clear();
|
||||
let v = regressions(&report(a, b), 0.03);
|
||||
assert_eq!(v.len(), 1, "{v:?}");
|
||||
assert!(v[0].contains("측정 불가"), "{v:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn regressions_skip_metrics_never_measured_on_either_side() {
|
||||
// citation_coverage / refusal_correctness 는 --with-rag 없이는 양쪽 다 NaN 이다.
|
||||
// 잰 적이 없는 지표로 게이트를 실패시키면 안 된다.
|
||||
let r = report(agg(0.5, 0.95, 0.0), agg(0.5, 0.95, 0.0));
|
||||
assert!(regressions(&r, 0.0).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_chunker_version_from_snapshot() {
|
||||
let s = r#"{"config":{},"chunker_version":"slot@1"}"#;
|
||||
|
||||
@@ -29,7 +29,7 @@ mod variant;
|
||||
|
||||
pub use compare::{
|
||||
CompareOpts, CompareReport, ComparisonKind, QueryComparison, compare_runs,
|
||||
compare_runs_with_config, render_report_md,
|
||||
compare_runs_with_config, regressions, render_report_md,
|
||||
};
|
||||
pub use loader::load_golden_set;
|
||||
pub use metrics::{
|
||||
|
||||
@@ -127,7 +127,8 @@ pub(crate) fn validate_against_db(
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let store = SqliteStore::open(&cfg.storage).context("open SqliteStore for golden validation")?;
|
||||
let store =
|
||||
SqliteStore::open(&cfg.storage).context("open SqliteStore for golden validation")?;
|
||||
store
|
||||
.run_migrations()
|
||||
.context("run migrations for golden validation")?;
|
||||
@@ -205,8 +206,14 @@ mod tests {
|
||||
.unwrap();
|
||||
let err = load_golden_set(&yaml_path).unwrap_err();
|
||||
let msg = format!("{err:#}");
|
||||
assert!(msg.contains("'g2'"), "should name the divergent query g2: {msg}");
|
||||
assert!(!msg.contains("'g3'"), "g3 matches canonical, must not be flagged: {msg}");
|
||||
assert!(
|
||||
msg.contains("'g2'"),
|
||||
"should name the divergent query g2: {msg}"
|
||||
);
|
||||
assert!(
|
||||
!msg.contains("'g3'"),
|
||||
"g3 matches canonical, must not be flagged: {msg}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -114,7 +114,8 @@ pub fn compute_aggregate(run_id: &str) -> Result<AggregateMetrics> {
|
||||
/// Compute aggregate metrics for `run_id` against an explicit
|
||||
/// [`Config`] (used by tests with a TempDir-backed `data_dir`).
|
||||
pub fn compute_aggregate_with_config(cfg: &Config, run_id: &str) -> Result<AggregateMetrics> {
|
||||
let store = SqliteStore::open(&cfg.storage).context("open SqliteStore for compute_aggregate")?;
|
||||
let store =
|
||||
SqliteStore::open(&cfg.storage).context("open SqliteStore for compute_aggregate")?;
|
||||
store
|
||||
.run_migrations()
|
||||
.context("run migrations for compute_aggregate")?;
|
||||
|
||||
@@ -108,16 +108,15 @@ pub fn compute_variant_consistency(
|
||||
let (recall_narrow, recall_pool) = recall_narrow_pool(&qr, &gq.expected_doc_ids);
|
||||
// Mirrors metrics.rs groundedness guards: skip errored rows and
|
||||
// vacuous-true (no must_contain/forbidden configured).
|
||||
let answer_ok = if qr.error.is_some()
|
||||
|| (gq.must_contain.is_empty() && gq.forbidden.is_empty())
|
||||
{
|
||||
None
|
||||
} else {
|
||||
qr.answer.as_ref().map(|a| {
|
||||
gq.must_contain.iter().all(|s| a.answer.contains(s))
|
||||
&& !gq.forbidden.iter().any(|s| a.answer.contains(s))
|
||||
})
|
||||
};
|
||||
let answer_ok =
|
||||
if qr.error.is_some() || (gq.must_contain.is_empty() && gq.forbidden.is_empty()) {
|
||||
None
|
||||
} else {
|
||||
qr.answer.as_ref().map(|a| {
|
||||
gq.must_contain.iter().all(|s| a.answer.contains(s))
|
||||
&& !gq.forbidden.iter().any(|s| a.answer.contains(s))
|
||||
})
|
||||
};
|
||||
let class = classify(&gq.expected_doc_ids, recall_narrow, recall_pool);
|
||||
grouped.entry(group).or_default().push(VariantResult {
|
||||
query_id: qr.query_id.clone(),
|
||||
@@ -220,8 +219,14 @@ fn rollup_group(group: String, variants: Vec<VariantResult>) -> VariantGroupRepo
|
||||
} else {
|
||||
Some(answer_flags.iter().all(|&ok| ok))
|
||||
};
|
||||
let mis_ranked = variants.iter().filter(|v| v.class == VariantClass::MisRanked).count() as u32;
|
||||
let missing = variants.iter().filter(|v| v.class == VariantClass::Missing).count() as u32;
|
||||
let mis_ranked = variants
|
||||
.iter()
|
||||
.filter(|v| v.class == VariantClass::MisRanked)
|
||||
.count() as u32;
|
||||
let missing = variants
|
||||
.iter()
|
||||
.filter(|v| v.class == VariantClass::Missing)
|
||||
.count() as u32;
|
||||
VariantGroupReport {
|
||||
group,
|
||||
variants,
|
||||
@@ -239,7 +244,8 @@ pub fn compute_variant_consistency_with_config(
|
||||
cfg: &Config,
|
||||
run_id: &str,
|
||||
) -> Result<VariantConsistencyReport> {
|
||||
let store = SqliteStore::open(&cfg.storage).context("open SqliteStore for variant consistency")?;
|
||||
let store =
|
||||
SqliteStore::open(&cfg.storage).context("open SqliteStore for variant consistency")?;
|
||||
store.run_migrations().context("run migrations")?;
|
||||
let run_record = store
|
||||
.load_eval_run(run_id)
|
||||
@@ -298,9 +304,18 @@ pub fn render_variants_md(rep: &VariantConsistencyReport) -> String {
|
||||
let _ = writeln!(
|
||||
s,
|
||||
"## {} — spread@{}={:.2} worst={:.2} A={} B={} answers={}",
|
||||
g.group, NARROW_K, g.recall_spread_narrow, g.worst_recall_narrow, g.mis_ranked, g.missing, ac
|
||||
g.group,
|
||||
NARROW_K,
|
||||
g.recall_spread_narrow,
|
||||
g.worst_recall_narrow,
|
||||
g.mis_ranked,
|
||||
g.missing,
|
||||
ac
|
||||
);
|
||||
let _ = writeln!(
|
||||
s,
|
||||
"| variant | recall@{NARROW_K} | recall@{POOL_K} | class | answer |"
|
||||
);
|
||||
let _ = writeln!(s, "| variant | recall@{NARROW_K} | recall@{POOL_K} | class | answer |");
|
||||
let _ = writeln!(s, "|---|---|---|---|---|");
|
||||
for v in &g.variants {
|
||||
let ans = match v.answer_ok {
|
||||
@@ -338,7 +353,12 @@ mod tests {
|
||||
heading_path: vec![],
|
||||
section_label: None,
|
||||
snippet: String::new(),
|
||||
citation: Citation::Line { path, start: 1, end: 1, section: None },
|
||||
citation: Citation::Line {
|
||||
path,
|
||||
start: 1,
|
||||
end: 1,
|
||||
section: None,
|
||||
},
|
||||
retrieval: RetrievalDetail {
|
||||
method: SearchMode::Vector,
|
||||
fusion_score: 1.0 / rank as f32,
|
||||
@@ -396,7 +416,11 @@ mod tests {
|
||||
// v1: docX at rank 3 → narrow=1.0 → Ok
|
||||
// v2: docX at rank 25 → narrow=0.0, pool=1.0 → MisRanked (A)
|
||||
// v3: docX 없음 → narrow=0.0, pool=0.0 → Missing (B)
|
||||
let queries = vec![gq("v1", "g", "docX"), gq("v2", "g", "docX"), gq("v3", "g", "docX")];
|
||||
let queries = vec![
|
||||
gq("v1", "g", "docX"),
|
||||
gq("v2", "g", "docX"),
|
||||
gq("v3", "g", "docX"),
|
||||
];
|
||||
let rows = vec![
|
||||
row("v1", vec![hit("docX", 3)]),
|
||||
row("v2", vec![hit("docX", 25)]),
|
||||
@@ -422,7 +446,10 @@ mod tests {
|
||||
#[test]
|
||||
fn fully_consistent_group_when_all_ok() {
|
||||
let queries = vec![gq("v1", "g", "docX"), gq("v2", "g", "docX")];
|
||||
let rows = vec![row("v1", vec![hit("docX", 1)]), row("v2", vec![hit("docX", 2)])];
|
||||
let rows = vec![
|
||||
row("v1", vec![hit("docX", 1)]),
|
||||
row("v2", vec![hit("docX", 2)]),
|
||||
];
|
||||
let rep = compute_variant_consistency(&queries, &rows).unwrap();
|
||||
assert_eq!(rep.fully_consistent_groups, 1);
|
||||
assert!((rep.groups[0].recall_spread_narrow - 0.0).abs() < 1e-6);
|
||||
@@ -443,8 +470,9 @@ mod tests {
|
||||
error: Option<&str>,
|
||||
) -> EvalQueryResultRecord {
|
||||
let hits_json = serde_json::to_value(&hits).unwrap();
|
||||
let error_json =
|
||||
error.map_or(serde_json::Value::Null, |e| serde_json::Value::String(e.into()));
|
||||
let error_json = error.map_or(serde_json::Value::Null, |e| {
|
||||
serde_json::Value::String(e.into())
|
||||
});
|
||||
let qr_json = serde_json::json!({
|
||||
"query_id": query_id,
|
||||
"query": query_id,
|
||||
@@ -489,7 +517,10 @@ mod tests {
|
||||
row("v2", vec![hit("other", 7)]), // rank 7 ≤ NARROW_K=10
|
||||
];
|
||||
let rep = compute_variant_consistency(&queries, &rows).unwrap();
|
||||
assert!(rep.pool_possibly_truncated, "all ranks ≤ NARROW_K must set pool_possibly_truncated");
|
||||
assert!(
|
||||
rep.pool_possibly_truncated,
|
||||
"all ranks ≤ NARROW_K must set pool_possibly_truncated"
|
||||
);
|
||||
// v2 misses docX, pool also has no rank>10 → classified Missing, not MisRanked
|
||||
assert_eq!(rep.a_dominant_groups, 0);
|
||||
assert_eq!(rep.b_dominant_groups, 1);
|
||||
@@ -505,7 +536,10 @@ mod tests {
|
||||
let row_v1 = row_with_answer("v1", vec![], "any text", None);
|
||||
let rep = compute_variant_consistency(&[gq_no_check], &[row_v1]).unwrap();
|
||||
let v = &rep.groups[0].variants[0];
|
||||
assert_eq!(v.answer_ok, None, "vacuous-true guard: no checks → answer_ok = None");
|
||||
assert_eq!(
|
||||
v.answer_ok, None,
|
||||
"vacuous-true guard: no checks → answer_ok = None"
|
||||
);
|
||||
assert_eq!(rep.groups[0].answer_consistency, None);
|
||||
|
||||
// M1b: must_contain present but error is also set
|
||||
@@ -514,7 +548,10 @@ mod tests {
|
||||
let row_v2 = row_with_answer("v2", vec![], "expected text", Some("llm error"));
|
||||
let rep2 = compute_variant_consistency(&[gq_check], &[row_v2]).unwrap();
|
||||
let v2 = &rep2.groups[0].variants[0];
|
||||
assert_eq!(v2.answer_ok, None, "error guard: qr.error present → answer_ok = None");
|
||||
assert_eq!(
|
||||
v2.answer_ok, None,
|
||||
"error guard: qr.error present → answer_ok = None"
|
||||
);
|
||||
}
|
||||
|
||||
/// N1 순수 B: 두 변형 모두 pool 에서도 정답 없음 → b_dominant=1, a_dominant=0.
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
//! "temperature": <float>,
|
||||
//! "seed": <u64>,
|
||||
//! "num_ctx": <usize>,
|
||||
//! "num_predict": <usize>,
|
||||
//! "stop": ["<str>", ...]
|
||||
//! }
|
||||
//! }
|
||||
@@ -177,6 +178,7 @@ impl LanguageModel for OllamaLanguageModel {
|
||||
temperature: effective_temperature,
|
||||
seed: effective_seed,
|
||||
num_ctx: self.context_tokens,
|
||||
num_predict: req.max_tokens,
|
||||
stop: &req.stop,
|
||||
},
|
||||
};
|
||||
@@ -233,6 +235,16 @@ struct OllamaOptions<'a> {
|
||||
temperature: f32,
|
||||
seed: u64,
|
||||
num_ctx: usize,
|
||||
/// Generation cap, from `GenerateRequest::max_tokens`.
|
||||
///
|
||||
/// Ollama defaults `num_predict` to -1 (unbounded) and shifts the
|
||||
/// context window when it fills, so a model that falls into a
|
||||
/// repetition loop streams forever. `request_timeout_secs` cannot
|
||||
/// save the caller: the connection keeps producing bytes, so no read
|
||||
/// ever times out. Seen in the v0.32 dogfood — one `eval` query held
|
||||
/// the run for 30+ minutes and had received 13 MB of token frames
|
||||
/// when it was killed.
|
||||
num_predict: usize,
|
||||
stop: &'a [String],
|
||||
}
|
||||
|
||||
@@ -513,6 +525,31 @@ fn truncate_body(s: &str, n: usize) -> String {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// v0.32 dogfood: `GenerateRequest::max_tokens` was computed by the RAG
|
||||
/// pipeline and then dropped on the floor — `OllamaOptions` never carried
|
||||
/// it. Ollama's default `num_predict` is -1 (unbounded), so a degenerate
|
||||
/// generation ran until something else killed it.
|
||||
#[test]
|
||||
fn request_body_carries_max_tokens_as_num_predict() {
|
||||
let stop: Vec<String> = vec!["</end>".into()];
|
||||
let body = OllamaRequest {
|
||||
model: "m",
|
||||
prompt: "p".into(),
|
||||
images: &[],
|
||||
stream: true,
|
||||
options: OllamaOptions {
|
||||
temperature: 0.0,
|
||||
seed: 7,
|
||||
num_ctx: 8192,
|
||||
num_predict: 1234,
|
||||
stop: &stop,
|
||||
},
|
||||
};
|
||||
let v = serde_json::to_value(&body).unwrap();
|
||||
assert_eq!(v["options"]["num_predict"], 1234);
|
||||
assert_eq!(v["options"]["num_ctx"], 8192);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trim_trailing_newline_removes_lf_and_crlf() {
|
||||
assert_eq!(trim_trailing_newline(b"hi\n"), b"hi");
|
||||
|
||||
@@ -1868,17 +1868,38 @@ fn est_tokens(s: &str) -> usize {
|
||||
s.chars().count().div_ceil(4)
|
||||
}
|
||||
|
||||
/// Strict marker regex per design §1 / spec line 107: `[#1]` … `[#999]`.
|
||||
/// Matches without `#`, with whitespace, or with non-digit content are
|
||||
/// intentionally ignored (see test plan rows 5–6).
|
||||
/// Strict marker regex per design §1 / spec line 107: `[#1]` … `[#999]`,
|
||||
/// plus the grouped form `[#2, #10]` the model emits when one claim rests
|
||||
/// on several sources. Matches without `#`, with leading/trailing
|
||||
/// whitespace inside the brackets, or with non-digit content are still
|
||||
/// intentionally ignored (see test plan rows 5–6) — the strictness exists
|
||||
/// so Rust snippets like `vec![1]` in an answer are not read as citations.
|
||||
static MARKER_REGEX: OnceLock<Regex> = OnceLock::new();
|
||||
/// Pulls the individual `#N` out of a marker group matched by
|
||||
/// [`MARKER_REGEX`]. Only ever run on an already-validated span.
|
||||
static MARKER_NUM_REGEX: OnceLock<Regex> = OnceLock::new();
|
||||
static REFUSAL_PHRASE: OnceLock<Regex> = OnceLock::new();
|
||||
|
||||
/// Markers cited by the answer, in the order they appear.
|
||||
///
|
||||
/// Grouped markers matter beyond bookkeeping: `citations` is the
|
||||
/// intersection of this set with the packed entries, so a marker missed
|
||||
/// here is a source silently dropped from `answer.v1` while its `[#N]`
|
||||
/// stays visible in the answer text. Found in the v0.32 dogfood — the
|
||||
/// model wrote `[#2, #10]` and the reader had no way to resolve `[#2]`.
|
||||
fn extract_markers(s: &str) -> Vec<u32> {
|
||||
let re =
|
||||
MARKER_REGEX.get_or_init(|| Regex::new(r"\[#(\d{1,3})\]").expect("static regex compiles"));
|
||||
re.captures_iter(s)
|
||||
.filter_map(|c| c.get(1).and_then(|m| m.as_str().parse::<u32>().ok()))
|
||||
let group = MARKER_REGEX.get_or_init(|| {
|
||||
Regex::new(r"\[#\d{1,3}(?:\s*,\s*#\d{1,3})*\]").expect("static regex compiles")
|
||||
});
|
||||
let num =
|
||||
MARKER_NUM_REGEX.get_or_init(|| Regex::new(r"#(\d{1,3})").expect("static regex compiles"));
|
||||
group
|
||||
.find_iter(s)
|
||||
.flat_map(|g| {
|
||||
num.captures_iter(g.as_str())
|
||||
.filter_map(|c| c.get(1).and_then(|m| m.as_str().parse::<u32>().ok()))
|
||||
.collect::<Vec<_>>()
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
@@ -1961,6 +1982,29 @@ mod tests {
|
||||
assert_send_sync::<RagPipeline>();
|
||||
}
|
||||
|
||||
/// v0.32 dogfood (28.4k-document KB): the model groups markers when a
|
||||
/// claim rests on several sources — `근거입니다 [#2, #10]`. The prompt
|
||||
/// only says "attribute with [#번호]", so grouping is not a violation.
|
||||
/// The single-marker-only regex dropped those, and since `citations`
|
||||
/// is built by intersecting the extracted set with the packed entries
|
||||
/// (`cited_set` below), the grouped sources vanished from `answer.v1`
|
||||
/// while their markers stayed visible in the answer text — a reader
|
||||
/// or agent sees `[#2]` with nothing to resolve it to.
|
||||
#[test]
|
||||
fn extract_markers_accepts_grouped_markers() {
|
||||
assert_eq!(extract_markers("근거입니다 [#2, #10]."), vec![2, 10]);
|
||||
assert_eq!(extract_markers("no spaces [#6,#7]"), vec![6, 7]);
|
||||
assert_eq!(extract_markers("three [#1, #2, #3]"), vec![1, 2, 3]);
|
||||
// Mixed with the single form, in document order.
|
||||
assert_eq!(
|
||||
extract_markers("a [#1]. b [#2, #10]. c [#9]."),
|
||||
vec![1, 2, 10, 9]
|
||||
);
|
||||
// Grouping must not loosen the strictness the single form has:
|
||||
// a bare number after the comma is not a marker.
|
||||
assert!(extract_markers("see [#1, 2]").is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_markers_strict_regex() {
|
||||
// Valid markers.
|
||||
|
||||
@@ -18,6 +18,7 @@ use kebab_core::{
|
||||
use kebab_store_sqlite::{EmbeddingRecordRow, SqliteStore};
|
||||
use lancedb::Connection;
|
||||
use lancedb::query::{ExecutableQuery, QueryBase};
|
||||
use lancedb::table::{CompactionOptions, Duration, OptimizeAction};
|
||||
use serde_json::json;
|
||||
use time::OffsetDateTime;
|
||||
use tokio::runtime::{Builder as RuntimeBuilder, Runtime};
|
||||
@@ -50,6 +51,31 @@ const INDEX_VERSION: &str = "v1";
|
||||
/// reaching into a private constant.
|
||||
pub const INDEX_VERSION_STR: &str = INDEX_VERSION;
|
||||
|
||||
/// Lance table commits between compactions.
|
||||
///
|
||||
/// Every `upsert` is one `merge_insert`, which creates a new table
|
||||
/// version whose manifest lists **every** fragment in the table. kebab
|
||||
/// upserts once per document, so an N-document ingest ends up with N
|
||||
/// fragments and an N-entry manifest rewritten on every write: per-write
|
||||
/// cost grows linearly with corpus size, manifest bytes quadratically.
|
||||
///
|
||||
/// Measured on a 16.8k-document dogfood KB with this compaction absent:
|
||||
/// ingest decayed 30.7 → 4.3 documents/min, and `_versions/` held 12.2 GB
|
||||
/// of manifests against 1.7 GB of vectors. A single full compaction of
|
||||
/// that table took 153 s and returned it to 1.70 GB / 1 version.
|
||||
///
|
||||
/// The interval trades compaction work (each pass rewrites the table)
|
||||
/// against manifest size. At 512 the rewrite cost over a 30k-document
|
||||
/// ingest is roughly an hour, well under what the un-compacted manifests
|
||||
/// cost — and the tail of that ingest stays at full speed instead of
|
||||
/// running seven times slower.
|
||||
///
|
||||
/// Counted against the table's own version rather than a field on this
|
||||
/// struct, so single-document paths (`kebab ingest-file`, MCP
|
||||
/// `ingest_file`/`ingest_stdin`) still reach it: each opens a fresh
|
||||
/// store, and an in-memory counter would restart at zero every call.
|
||||
const COMPACT_EVERY_N_UPSERTS: u64 = 512;
|
||||
|
||||
/// Lance VectorStore.
|
||||
///
|
||||
/// Holds a single `lancedb::Connection` opened against
|
||||
@@ -79,6 +105,8 @@ pub struct LanceVectorStore {
|
||||
/// only — the `Connection` already knows it.
|
||||
#[allow(dead_code)]
|
||||
vector_dir: PathBuf,
|
||||
/// Commits between compactions. See [`COMPACT_EVERY_N_UPSERTS`].
|
||||
compact_every: u64,
|
||||
}
|
||||
|
||||
impl LanceVectorStore {
|
||||
@@ -94,6 +122,21 @@ impl LanceVectorStore {
|
||||
/// within a runtime"`. See the struct-level `# Async context`
|
||||
/// section.
|
||||
pub fn new(storage: &kebab_config::StorageCfg, sqlite: Arc<SqliteStore>) -> Result<Self> {
|
||||
Self::new_with_compact_interval(storage, sqlite, COMPACT_EVERY_N_UPSERTS)
|
||||
}
|
||||
|
||||
/// [`LanceVectorStore::new`] with an explicit compaction interval.
|
||||
///
|
||||
/// `#[doc(hidden)]` but public: the integration test drives the
|
||||
/// compaction path with a small interval instead of issuing
|
||||
/// [`COMPACT_EVERY_N_UPSERTS`] upserts. Not a config surface —
|
||||
/// production always goes through `new`.
|
||||
#[doc(hidden)]
|
||||
pub fn new_with_compact_interval(
|
||||
storage: &kebab_config::StorageCfg,
|
||||
sqlite: Arc<SqliteStore>,
|
||||
compact_every: u64,
|
||||
) -> Result<Self> {
|
||||
let data_dir = expand_path(&storage.data_dir, "");
|
||||
let vector_dir = expand_path(&storage.vector_dir, &data_dir.to_string_lossy());
|
||||
std::fs::create_dir_all(&vector_dir)
|
||||
@@ -125,6 +168,45 @@ impl LanceVectorStore {
|
||||
connection,
|
||||
sqlite,
|
||||
vector_dir,
|
||||
compact_every,
|
||||
})
|
||||
}
|
||||
|
||||
/// Compact the table's fragments and drop superseded versions.
|
||||
///
|
||||
/// `delete_unverified` tells Lance to reclaim files no manifest
|
||||
/// references yet, regardless of age. That is only sound when nothing
|
||||
/// else is writing this table concurrently — an in-flight commit's
|
||||
/// staged files look identical to garbage.
|
||||
///
|
||||
/// kebab does not enforce that: there is no lock file anywhere in the
|
||||
/// workspace, so a second `kebab ingest` or a long-lived `kebab mcp`
|
||||
/// server calling `ingest_file` can overlap. Concurrent ingest is
|
||||
/// already unsound for other reasons (`documents.workspace_path` is
|
||||
/// UNIQUE and racing writers collide there first), so this does not
|
||||
/// add a failure mode that a single-writer workspace was free of —
|
||||
/// but it does raise the cost of violating that assumption. Without
|
||||
/// this flag nothing younger than seven days is ever reclaimed, which
|
||||
/// on a fresh index means nothing at all, so the disk half of the fix
|
||||
/// would not happen.
|
||||
fn compact_table(&self, table: &lancedb::Table) -> Result<()> {
|
||||
self.runtime.block_on(async {
|
||||
table
|
||||
.optimize(OptimizeAction::Compact {
|
||||
options: CompactionOptions::default(),
|
||||
remap_options: None,
|
||||
})
|
||||
.await
|
||||
.context("Lance compact")?;
|
||||
table
|
||||
.optimize(OptimizeAction::Prune {
|
||||
older_than: Some(Duration::zero()),
|
||||
delete_unverified: Some(true),
|
||||
error_if_tagged_old_versions: None,
|
||||
})
|
||||
.await
|
||||
.context("Lance prune")?;
|
||||
Result::<()>::Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
@@ -270,6 +352,40 @@ impl VectorStore for LanceVectorStore {
|
||||
rows = recs.len(),
|
||||
"upsert committed"
|
||||
);
|
||||
|
||||
// Fold the per-document fragments back together every so often.
|
||||
//
|
||||
// The trigger is Lance's own table version, not a counter on this
|
||||
// struct. Version is monotonic and lives in the table, so it keeps
|
||||
// counting across processes — an in-memory counter would restart at
|
||||
// zero on every `kebab ingest-file` and every MCP `ingest_file`
|
||||
// call, which is exactly the workload that accumulates fragments
|
||||
// one at a time and would therefore never compact.
|
||||
//
|
||||
// Failure here is not fatal — the data is already committed and a
|
||||
// later commit crosses the next multiple — but it is logged at warn
|
||||
// because the symptom of silently skipping it is a slow ingest much
|
||||
// later.
|
||||
let version = self
|
||||
.runtime
|
||||
.block_on(async { table.version().await })
|
||||
.unwrap_or(0);
|
||||
if version > 0 && self.compact_every > 0 && version % self.compact_every == 0 {
|
||||
if let Err(e) = self.compact_table(&table) {
|
||||
tracing::warn!(
|
||||
target: "kebab-store-vector",
|
||||
table = %table_name,
|
||||
error = %e,
|
||||
"Lance compaction failed; ingest continues but writes will slow down"
|
||||
);
|
||||
} else {
|
||||
tracing::info!(
|
||||
target: "kebab-store-vector",
|
||||
table = %table_name,
|
||||
"Lance compaction done"
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -91,6 +91,28 @@ impl TestEnv {
|
||||
}
|
||||
}
|
||||
|
||||
/// Same as [`TestEnv::new`] but with an explicit Lance compaction
|
||||
/// interval, so `tests/compaction.rs` can exercise the compaction
|
||||
/// path without driving the production `COMPACT_EVERY_N_UPSERTS`
|
||||
/// number of upserts.
|
||||
pub fn with_compact_interval(every: u64) -> Self {
|
||||
let temp = tempfile::tempdir().expect("tempdir");
|
||||
let mut config = Config::defaults();
|
||||
config.storage.data_dir = temp.path().to_string_lossy().into_owned();
|
||||
let sqlite = SqliteStore::open(&config.storage).unwrap();
|
||||
sqlite.run_migrations().unwrap();
|
||||
let sqlite = Arc::new(sqlite);
|
||||
let vector =
|
||||
LanceVectorStore::new_with_compact_interval(&config.storage, sqlite.clone(), every)
|
||||
.unwrap();
|
||||
Self {
|
||||
temp,
|
||||
config,
|
||||
sqlite,
|
||||
vector,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn data_dir(&self) -> PathBuf {
|
||||
self.temp.path().to_path_buf()
|
||||
}
|
||||
|
||||
90
crates/kebab-store-vector/tests/compaction.rs
Normal file
90
crates/kebab-store-vector/tests/compaction.rs
Normal file
@@ -0,0 +1,90 @@
|
||||
//! Regression test for Lance version/fragment accumulation.
|
||||
//!
|
||||
//! Every `upsert` is one Lance `merge_insert`, and every `merge_insert`
|
||||
//! creates a new table version whose manifest lists **all** fragments in
|
||||
//! the table. kebab upserts once per ingested document, so without
|
||||
//! periodic compaction an N-document corpus leaves N fragments and the
|
||||
//! N-th write rewrites a manifest with N entries — write cost grows
|
||||
//! linearly with corpus size and manifest bytes grow quadratically.
|
||||
//!
|
||||
//! Measured on a 16.8k-document dogfood KB before the fix: ingest decayed
|
||||
//! from 30.7 to 4.3 documents/min, and `_versions/` held 12.2 GB of
|
||||
//! manifests against 1.7 GB of actual vectors. One full compaction took
|
||||
//! 153 s and brought the table back to 1.70 GB / 1 version.
|
||||
//!
|
||||
//! This test drives many more upserts than the compaction interval and
|
||||
//! asserts the on-disk version count stays bounded.
|
||||
//!
|
||||
//! `#[ignore]` + AVX gate per `tests/common/mod.rs` policy.
|
||||
|
||||
use kebab_core::VectorStore;
|
||||
|
||||
mod common;
|
||||
use common::{TestEnv, make_record, require_avx_or_panic};
|
||||
|
||||
const MODEL: &str = "compaction-model";
|
||||
|
||||
/// Count `*.manifest` files under the single Lance table directory.
|
||||
/// Lance keeps one manifest per surviving version, so this is a direct
|
||||
/// proxy for "how much history is still on disk".
|
||||
fn manifest_count(data_dir: &std::path::Path) -> usize {
|
||||
let lance_root = data_dir.join("lancedb");
|
||||
let mut n = 0;
|
||||
let mut stack = vec![lance_root];
|
||||
while let Some(dir) = stack.pop() {
|
||||
let Ok(entries) = std::fs::read_dir(&dir) else {
|
||||
continue;
|
||||
};
|
||||
for e in entries.flatten() {
|
||||
let p = e.path();
|
||||
if p.is_dir() {
|
||||
stack.push(p);
|
||||
} else if p.extension().is_some_and(|x| x == "manifest") {
|
||||
n += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
n
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires AVX-capable hardware (LanceDB)"]
|
||||
fn repeated_upserts_do_not_accumulate_lance_versions() {
|
||||
require_avx_or_panic();
|
||||
|
||||
// Compact every 8 upserts so the test stays quick; production uses
|
||||
// `COMPACT_EVERY_N_UPSERTS`.
|
||||
let env = TestEnv::with_compact_interval(8);
|
||||
env.seed_chunk(
|
||||
&format!("{:032x}", 0x1100u32),
|
||||
&format!("{:032x}", 0xd0c0u32),
|
||||
"note.md",
|
||||
"en",
|
||||
&[],
|
||||
"primary",
|
||||
);
|
||||
|
||||
let rec = make_record(0, 0, vec![1.0, 0.0, 0.0, 0.0], "hi", &[], MODEL);
|
||||
const UPSERTS: usize = 40;
|
||||
for _ in 0..UPSERTS {
|
||||
env.vector.upsert(std::slice::from_ref(&rec)).unwrap();
|
||||
}
|
||||
|
||||
let manifests = manifest_count(&env.data_dir());
|
||||
assert!(
|
||||
manifests <= 12,
|
||||
"Lance versions accumulated: {manifests} manifests after {UPSERTS} upserts \
|
||||
(compaction interval 8). Without compaction this grows one-per-upsert."
|
||||
);
|
||||
|
||||
// Compaction must not lose rows: the record is still searchable.
|
||||
let hits = env
|
||||
.vector
|
||||
.search(
|
||||
&[1.0, 0.0, 0.0, 0.0],
|
||||
5,
|
||||
&kebab_core::SearchFilters::default(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(hits.len(), 1, "compaction dropped the upserted row");
|
||||
}
|
||||
@@ -14,6 +14,67 @@ historical contract that was implemented; this file accumulates the
|
||||
deltas so phase 5+ readers can find the live behavior without diffing
|
||||
git history.
|
||||
|
||||
## 2026-08-16 — 28.4k 문서 도그푸딩: Lance 압축 부재 · 인용 마커 누락 · ollama 생성 무제한
|
||||
|
||||
나무위키 18,282 + Apache Jira 10,145 = **28,427 문서 / 600,808 청크** 코퍼스로 종단 도그푸딩. 기존 최대 규모(620 문서)의 46배라 그 규모에서는 보이지 않던 결함 3개가 드러났다. 셋 다 이 PR 에서 수정.
|
||||
|
||||
### 1. Lance fragment 무한 누적 — ingest 가 코퍼스 크기에 비례해 느려진다 (이슈 #230 제안 2)
|
||||
|
||||
`upsert` 가 asset 1건당 `merge_insert` 를 1회 호출하고 Lance 는 그때마다 새 테이블 버전 + fragment 를 만든다. manifest 는 **그 시점의 전 fragment 를 나열**하므로 N번째 쓰기가 N줄짜리 manifest 를 새로 쓴다 → 쓰기 비용이 문서 수에 비례, manifest 총량은 제곱. 코드베이스 전체에 `optimize` / `compact_files` / `cleanup_old_versions` 호출이 **0건**이었다.
|
||||
|
||||
| | 16,828 문서 시점 | 압축 후 (28,427 문서) |
|
||||
|---|---|---|
|
||||
| fragment / manifest | 16,814 | 336 |
|
||||
| `_versions/` | 12.2 GB | 6.6 MB |
|
||||
| 실제 벡터 데이터 | 1.7 GB | 1.7 GB |
|
||||
| ingest 속도 | 30.7 → **4.3 문서/분** (단조 하락) | **32~36 문서/분** (평평) |
|
||||
|
||||
이슈 #230 이 11,229 문서 시점에 잰 메타/실데이터 비율은 2.5배였는데, 16,828 문서에서는 **7.2배**였다 — 제곱 증가가 두 지점으로 확인됨.
|
||||
|
||||
수정: `COMPACT_EVERY_N_UPSERTS = 512` 마다 Compact + Prune. 기존 테이블 1회 압축은 **153초에 15 GB → 1.7 GB**(행 365,991 보존)로 끝났다. 트리거는 이 struct 의 카운터가 아니라 **Lance 테이블 자신의 version** 이다 — 인메모리 카운터는 `kebab ingest-file` 이나 MCP `ingest_file` 처럼 호출마다 store 를 새로 여는 경로에서 매번 0 으로 되돌아가 영영 512 에 못 닿는다. 즉 #230 본문의 "회복 수단이 `reset --vector-only`(전량 재임베딩) 밖에 없다" 는 사실이 아니다.
|
||||
|
||||
미해결(같은 이슈의 나머지): 삭제 경로 배치화(제안 1), geodatafusion(제안 3), doctor 지표(제안 4).
|
||||
|
||||
### 2. 묶음 인용 마커가 `answer.v1` 에서 조용히 사라짐
|
||||
|
||||
마커 추출 정규식이 `\[#(\d{1,3})\]` 라 **괄호 하나에 마커 하나**만 인정했다. 모델은 한 주장이 여러 근거에 걸리면 `[#2, #10]` 으로 묶는데, `SYSTEM_PROMPT_RAG_V4` 는 "`[#번호]` 로 귀속하라"고만 하고 한 괄호에 하나씩 쓰라고 지시하지 않으므로 지시 위반이 아니다.
|
||||
|
||||
`citations` 는 추출 마커와 packed entry 의 교집합이라(`cited_set`), 못 잡힌 마커는 근거가 멀쩡히 있는데도 배열에서 빠졌다. 본문에는 `[#2]` 가 남아 사용자도 agent 도 해소할 수 없는 인용이 된다. 부수적으로 `unknown_markers` / `grounded` 판정도 반쪽 마커 집합 위에서 계산됐다.
|
||||
|
||||
실측(28.4k KB, "Spark shuffle OOM" 질의): 본문 인용 `1,2,6,7,8,9,10` vs `citations` `1,8,9,10` → 3건 끊김. 수정 후 7/7 해소, `citation_coverage` 1.0.
|
||||
|
||||
기존 엄격함(`vec![1]` · `[1]` · `[ #1 ]` · `[#foo]` · `[#1234]` 불인정)은 유지 — 답변에 Rust 코드가 섞일 때의 오탐 방지가 원래 목적이고 기존 테스트가 그걸 고정한다.
|
||||
|
||||
### 3. ollama 요청이 `max_tokens` 를 안 보냄 — 반복 루프가 무한 스트림이 된다
|
||||
|
||||
`GenerateRequest::max_tokens` 를 RAG 파이프라인이 계산해 넘기는데 `OllamaOptions` 가 `temperature`/`seed`/`num_ctx`/`stop` 만 직렬화하고 **그 값을 버렸다**. ollama 기본 `num_predict` 는 -1(무제한)이고 컨텍스트가 차면 창을 밀어 계속 생성한다.
|
||||
|
||||
`request_timeout_secs` 로는 못 막는다 — 연결에 바이트가 계속 흐르므로 읽기 타임아웃이 안 터진다. `eval run --with-rag` 216 질의가 **1시간 45분에 21개**만 끝냈고, 붙잡고 있던 질의 하나가 **13 MB** 를 받은 상태였다(15초에 142 KB 유입 확인, ollama runner 198% CPU).
|
||||
|
||||
수정: `num_predict: req.max_tokens`. 같은 216 질의가 **31분**에 완료.
|
||||
|
||||
### 골든셋 + 회귀 게이트 (신규)
|
||||
|
||||
이 KB 전용 골든셋을 도그푸딩 store 에 만들었다 — 문서 42건 × 표현 5변형(abbr/ko/en/syn/para) + 거절 6 = **216 질의**. 정답(`expected_doc_ids`)은 검색이 아니라 SQLite 에서 `workspace_path → doc_id` 로 직접 뽑아 순환을 피했고, chunk 단위 대신 **문서 단위**로 잡았다(chunk_id 가 blake3(doc_id, chunker_version, block_ids, policy_hash) 파생이라 파일 1바이트 수정으로 전부 무효화됨).
|
||||
|
||||
베이스라인:
|
||||
|
||||
```
|
||||
recall@1 0.8381 · recall@5 0.9429 · recall@10 0.9524
|
||||
변형 일관성 42그룹 중 33개 완전 일치 · mean spread@10 0.214
|
||||
citation_coverage 1.0 · groundedness 0.3905 · refusal_correctness 0.1667
|
||||
```
|
||||
|
||||
실패한 9개 그룹은 **전부 para(내용 서술) 변형**이다. 7개는 top-50 에는 있고 top-10 밖(MisRanked, 랭킹 문제), 2개는 top-50 에도 없다(Missing, 어휘 격차).
|
||||
|
||||
`kebab eval compare --max-drop <낙폭>` 을 추가했다. 이름은 절대 하한이 아니라 **델타 예산**임을 드러내려고 고른 것이다 — `--fail-under 0.9` 를 "recall 0.9 밑이면 실패"로 읽으면 recall 이 0.95→0.10 으로 무너져도 낙폭 0.85 < 0.9 라 통과하는 함정이 있었다. 음수·nan 은 시작 시점에 거부한다. 이전에는 delta 만 출력하고 exit code 가 항상 0 이라 "회귀했는가"를 기계가 판정할 수 없었다. `empty_result_rate` 는 반대 방향으로 검사하고, **A 에서 재던 지표가 B 에서 NaN 이 되면 위반**으로 잡는다(골든셋이 ground truth 를 잃은 경우가 정확히 그 모양이다).
|
||||
|
||||
### 남은 문제 — `refusal_correctness 0.1667`
|
||||
|
||||
코퍼스에 없는 가짜 질의 6개 중 **1개만** 제대로 거절했다. "Who won the 2044 Interplanetary Curling Championship?" 에 **"T1이 우승했습니다 [#3]"** 라고 답하고 LoL 문서를 인용하면서 `grounded=True` 로 표시됐다.
|
||||
|
||||
원인은 `grounded` 가 "모델이 유효한 마커를 달았는가" 만 보기 때문이다(`grounded_unaware`). 지어낸 답도 마커만 달면 grounded 다. 이걸 막으라고 있는 `rag.nli_threshold` 는 기본값이 0(꺼짐)이라, **기본 설정에는 환각 방어가 사실상 없다.** NLI 를 켜고 같은 6개를 재측정하는 것이 다음 과제.
|
||||
|
||||
## 2026-06-27 — v0.32.0 ponytail-audit 정리 arc 도그푸딩 (chunker A/B byte-identical + GPU r9700)
|
||||
|
||||
over-engineering 감사(ponytail-audit) 후 4 PR(#219–#222) 로 표면·구조·crate 수 단순화. **능력 불변, 이 arc(#219–#222) 누적 −3400줄 / crate 22→20 / dep −1**(serde_yaml 제거; unsafe-libyaml 은 serde_yaml_ng 가 여전히 transitive 로 가져옴). 실엔진 종단 검증:
|
||||
|
||||
Reference in New Issue
Block a user