altair823-org/kebab
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 43 Wiki Activity

fix(cli): thread --config through kebab eval run/aggregate/compare (facade-rule)

Browse Source 
Cmd::Eval now loads Config via cli.config (same pattern as all other
subcommands) before dispatching to the inner match.  Each arm now calls
the *_with_config variant:

  run_eval(&opts)             → run_eval_with_config(&cfg, &opts)
  compute_aggregate(run_id)   → compute_aggregate_with_config(&cfg, run_id)
  store_aggregate(run_id, ..) → store_aggregate_with_config(&cfg, run_id, ..)
  Compare already called compare_runs_with_config but sourced cfg from
  Config::load(None) — that redundant load is removed; cfg comes from
  the shared binding above.

Fixes the same facade-rule regression pattern as P3-5 / P4-3: previously
`kebab --config /build/dogfood/config.toml eval run` silently evaluated
the XDG-default (empty) KB instead of the dogfood KB.

Also fixes runner.rs test that hardcoded rag-v2 after commit 5719969
bumped the default prompt_template_version to rag-v3.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
altair823
2026-05-29 03:42:40 +00:00
parent 571996938c
commit d93b757cf1
2 changed files with 88 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
crates/kebab-cli/src/main.rs
View File

@@ -1337,7 +1337,9 @@ fn run(cli: &Cli) -> anyhow::Result<()> {
app.run() app.run()
} }
Cmd::Eval { what } => match what { Cmd::Eval { what } => {
let cfg = kebab_config::Config::load(cli.config.as_deref())?;
match what {
EvalWhat::Run { EvalWhat::Run {
suite, suite,
mode, mode,
@@ -1354,7 +1356,7 @@ fn run(cli: &Cli) -> anyhow::Result<()> {
temperature: *temperature, temperature: *temperature,
seed: *seed, seed: *seed,
}; };
let run = kebab_eval::run_eval(&opts)?; let run = kebab_eval::run_eval_with_config(&cfg, &opts)?;
if cli.json { if cli.json {
println!("{}", serde_json::to_string_pretty(&run)?); println!("{}", serde_json::to_string_pretty(&run)?);
} else { } else {
@@ -1367,8 +1369,8 @@ fn run(cli: &Cli) -> anyhow::Result<()> {
} }
EvalWhat::Aggregate { run_id } => { EvalWhat::Aggregate { run_id } => {
let agg = kebab_eval::compute_aggregate(run_id)?; let agg = kebab_eval::compute_aggregate_with_config(&cfg, run_id)?;
kebab_eval::store_aggregate(run_id, &agg)?; kebab_eval::store_aggregate_with_config(&cfg, run_id, &agg)?;
if cli.json { if cli.json {
println!("{}", serde_json::to_string_pretty(&agg)?); println!("{}", serde_json::to_string_pretty(&agg)?);
} else { } else {
@@ -1396,7 +1398,6 @@ fn run(cli: &Cli) -> anyhow::Result<()> {
strict_chunker_version, strict_chunker_version,
write_report, write_report,
} => { } => {
let cfg = kebab_config::Config::load(None)?;
let opts = kebab_eval::CompareOpts { let opts = kebab_eval::CompareOpts {
strict_chunker_version: *strict_chunker_version, strict_chunker_version: *strict_chunker_version,
}; };
@@ -1408,7 +1409,8 @@ fn run(cli: &Cli) -> anyhow::Result<()> {
print!("{md}"); print!("{md}");
} }
if *write_report { if *write_report {
let resolved_data_dir = kebab_config::expand_path(&cfg.storage.data_dir, ""); let resolved_data_dir =
kebab_config::expand_path(&cfg.storage.data_dir, "");
let runs_dir = kebab_config::expand_path( let runs_dir = kebab_config::expand_path(
&cfg.storage.runs_dir, &cfg.storage.runs_dir,
&resolved_data_dir.to_string_lossy(), &resolved_data_dir.to_string_lossy(),
@@ -1423,7 +1425,8 @@ fn run(cli: &Cli) -> anyhow::Result<()> {
} }
Ok(()) Ok(())
} }
}, }
}
Cmd::IngestFile { path } => { Cmd::IngestFile { path } => {
let cfg = kebab_config::Config::load(cli.config.as_deref())?; let cfg = kebab_config::Config::load(cli.config.as_deref())?;

2
crates/kebab-eval/tests/runner.rs
View File

@@ -215,7 +215,7 @@ fn runner_records_config_snapshot_with_versions() {
assert!(snap.pointer("/llm/model_id").is_some()); assert!(snap.pointer("/llm/model_id").is_some());
assert_eq!( assert_eq!(
snap.pointer("/prompt_template_version"), snap.pointer("/prompt_template_version"),
Some(&serde_json::Value::String("rag-v2".to_string())), Some(&serde_json::Value::String("rag-v3".to_string())),
); );
assert!(snap.pointer("/score_gate").is_some()); assert!(snap.pointer("/score_gate").is_some());
assert!(snap.pointer("/rrf_k").is_some()); assert!(snap.pointer("/rrf_k").is_some());