altair823-org/kebab
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 43 Wiki Activity

fix(cli): thread --config through kebab eval run/aggregate/compare (facade-rule)

Browse Source 
Cmd::Eval now loads Config via cli.config (same pattern as all other
subcommands) before dispatching to the inner match.  Each arm now calls
the *_with_config variant:

  run_eval(&opts)             → run_eval_with_config(&cfg, &opts)
  compute_aggregate(run_id)   → compute_aggregate_with_config(&cfg, run_id)
  store_aggregate(run_id, ..) → store_aggregate_with_config(&cfg, run_id, ..)
  Compare already called compare_runs_with_config but sourced cfg from
  Config::load(None) — that redundant load is removed; cfg comes from
  the shared binding above.

Fixes the same facade-rule regression pattern as P3-5 / P4-3: previously
`kebab --config /build/dogfood/config.toml eval run` silently evaluated
the XDG-default (empty) KB instead of the dogfood KB.

Also fixes runner.rs test that hardcoded rag-v2 after commit 5719969
bumped the default prompt_template_version to rag-v3.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
altair823
2026-05-29 03:42:40 +00:00
parent 571996938c
commit d93b757cf1
2 changed files with 88 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
crates/kebab-cli/src/main.rs
View File

@@ -1337,7 +1337,9 @@ fn run(cli: &Cli) -> anyhow::Result<()> {
app.run()
}
Cmd::Eval { what } => match what {
Cmd::Eval { what } => {
let cfg = kebab_config::Config::load(cli.config.as_deref())?;
match what {
EvalWhat::Run {
suite,
mode,
@@ -1354,7 +1356,7 @@ fn run(cli: &Cli) -> anyhow::Result<()> {
temperature: *temperature,
seed: *seed,
};
let run = kebab_eval::run_eval(&opts)?;
let run = kebab_eval::run_eval_with_config(&cfg, &opts)?;
if cli.json {
println!("{}", serde_json::to_string_pretty(&run)?);
} else {
@@ -1367,8 +1369,8 @@ fn run(cli: &Cli) -> anyhow::Result<()> {
}
EvalWhat::Aggregate { run_id } => {
let agg = kebab_eval::compute_aggregate(run_id)?;
kebab_eval::store_aggregate(run_id, &agg)?;
let agg = kebab_eval::compute_aggregate_with_config(&cfg, run_id)?;
kebab_eval::store_aggregate_with_config(&cfg, run_id, &agg)?;
if cli.json {
println!("{}", serde_json::to_string_pretty(&agg)?);
} else {
@@ -1396,7 +1398,6 @@ fn run(cli: &Cli) -> anyhow::Result<()> {
strict_chunker_version,
write_report,
} => {
let cfg = kebab_config::Config::load(None)?;
let opts = kebab_eval::CompareOpts {
strict_chunker_version: *strict_chunker_version,
};
@@ -1408,7 +1409,8 @@ fn run(cli: &Cli) -> anyhow::Result<()> {
print!("{md}");
}
if *write_report {
let resolved_data_dir = kebab_config::expand_path(&cfg.storage.data_dir, "");
let resolved_data_dir =
kebab_config::expand_path(&cfg.storage.data_dir, "");
let runs_dir = kebab_config::expand_path(
&cfg.storage.runs_dir,
&resolved_data_dir.to_string_lossy(),
@@ -1423,7 +1425,8 @@ fn run(cli: &Cli) -> anyhow::Result<()> {
}
Ok(())
}
},
}
}
Cmd::IngestFile { path } => {
let cfg = kebab_config::Config::load(cli.config.as_deref())?;

2
crates/kebab-eval/tests/runner.rs
View File

@@ -215,7 +215,7 @@ fn runner_records_config_snapshot_with_versions() {
assert!(snap.pointer("/llm/model_id").is_some());
assert_eq!(
snap.pointer("/prompt_template_version"),
Some(&serde_json::Value::String("rag-v2".to_string())),
Some(&serde_json::Value::String("rag-v3".to_string())),
);
assert!(snap.pointer("/score_gate").is_some());
assert!(snap.pointer("/rrf_k").is_some());