fix(rag): S3 NLI unavailable — hypothesis char budget + token-count fallback retry
S3 dogfood query 의 `nli_model_unavailable` consistent fail root cause = mDeBERTa-v3 tokenizer 의 `OnlyFirst` strategy + 949-token hypothesis. 기존 char-budget 단독 fix 의 KR-extreme density 미해결 → token-count fallback retry + RC1-residual trait dispatch 정합. 핵심 변경: - kebab-nli::NliVerifier: `hypothesis_token_count(&str) -> Result<usize>` trait method 추가 (default `Ok(0)` backward-compat). `OnnxNliVerifier` 가 *trait impl block* 안에서 real mDeBERTa tokenize override — vtable 등록 보장 (round-3 critic RC1-residual closure). - kebab-rag::pipeline: `MAX_NLI_HYPOTHESIS_CHARS_INITIAL = 1200` + `MAX_NLI_HYPOTHESIS_CHARS_MIN = 150` const + `pub(crate) fn truncate_chars` pure-fn + `pub fn truncate_hypothesis_for_nli_with_budget` retry helper (char budget 반감 retry, min floor 시 graceful unavailable). step 8.5 hook 의 callsite explicit `match` + `return self.refuse_nli_model_unavailable` 패턴 (`?` 금지 — round-2 plan critic CRITICAL #1 closure). - SpyNliVerifier 신규 helper (closure score_fn + hypothesis_token_count_fn, 2-arg constructor). - §5.1 의 2 ignored test (EN-long err + vtable dispatch RC1-residual pin) + §5.2 의 4 boundary test (truncate_chars) + §5.3 의 3 mock multi-hop test (long_en_grounded / long_kr_retries / unrelenting_fallback). +7 new tests (2 ignored default skip). - tasks/HOTFIXES.md 신규 dated entry `## 2026-05-26 — S3 NLI unavailable ...` — Symptom / Root cause / Action / Amends 4-block. - spec + plan (`docs/superpowers/{specs,plans}/2026-05-26-s3-nli-model-unavailable-diagnose-*.md`) — 4 round spec + 3 round plan OMC reviewer ACCEPT 산출물. 검증: - cargo test -p kebab-nli -j 1 → 11/11 pass + 7 ignored default skip. - cargo test -p kebab-rag -j 1 → 19+3+3+... 전체 pass + 3 new mock + 4 new boundary. - cargo test --workspace --no-fail-fast -j 1 → **1313 pass (+7 new)**, 0 failed. 회귀 0 (HOTFIX #15 이미 fixed, no remaining flaky). - cargo clippy --workspace --all-targets -j 1 -- -D warnings clean (type_complexity allow on Arc<dyn Fn> type aliases). KR safe (token-count retry path) + graceful fallback (min floor 시 기존 unavailable wire 유지, regression 0). Wire 영향 없음 (additive trait method). Cargo bump 불필요. Refs: - spec: docs/superpowers/specs/2026-05-26-s3-nli-model-unavailable-diagnose-spec.md (4 round APPROVE — analyst → critic + verifier × 4 rounds) - plan: docs/superpowers/plans/2026-05-26-s3-nli-model-unavailable-diagnose-plan.md (3 round ACCEPT — planner → critic-plan + verifier-plan × 3 rounds) Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -47,6 +47,21 @@ impl NliScores {
|
||||
/// entails hypothesis ⇒ answer is grounded in retrieved evidence).
|
||||
pub trait NliVerifier: Send + Sync {
|
||||
fn score(&self, premise: &str, hypothesis: &str) -> anyhow::Result<NliScores>;
|
||||
|
||||
/// Probe-only tokenize for caller-side budget verification. S3
|
||||
/// follow-up (2026-05-26) — pipeline 의 char-budget retry loop 가
|
||||
/// 이 API 로 mDeBERTa-v3 의 `OnlyFirst` dead-end (hypothesis 단독이
|
||||
/// 512-token cap 초과 시 truncate 불가) 를 회피.
|
||||
///
|
||||
/// **Default impl 반환 `Ok(0)`** — 기존 mock implementations
|
||||
/// (`MockNliVerifier` 등) 가 trait 확장 후에도 backward-compat
|
||||
/// (compile fail 회피, retry loop immediate 통과). `OnnxNliVerifier`
|
||||
/// 는 real tokenizer 로 *trait impl 블록 안에서* override 해야 함
|
||||
/// — inherent method 는 vtable 미등록 → trait dispatch 시 default
|
||||
/// 호출 → production silent NO-OP.
|
||||
fn hypothesis_token_count(&self, _hypothesis: &str) -> anyhow::Result<usize> {
|
||||
Ok(0)
|
||||
}
|
||||
}
|
||||
|
||||
/// Numerically stable 3-way softmax (subtract max for log-sum-exp safety).
|
||||
|
||||
@@ -66,6 +66,13 @@ pub struct OnnxNliVerifier {
|
||||
}
|
||||
|
||||
impl OnnxNliVerifier {
|
||||
/// Hypothesis-side budget. Pipeline 의
|
||||
/// `truncate_hypothesis_for_nli_with_budget` retry loop 가 char-truncate
|
||||
/// 후 token-count 재검증 시 이 값을 cap 으로 사용. = `MAX_TOKENS`
|
||||
/// (512) - 3 special tokens reserved (CLS, SEP, SEP) - 253 premise
|
||||
/// room (caller decides). 안전 마진 (S3 follow-up 2026-05-26).
|
||||
pub const HYPOTHESIS_TOKEN_BUDGET: usize = 256;
|
||||
|
||||
/// Construct a verifier from the user's `Config`. Eagerly resolves
|
||||
/// `cache_dir = config.storage.model_dir/nli/<sanitized-model-id>/`
|
||||
/// and runs `create_dir_all` so the first `score` call can drop
|
||||
@@ -278,6 +285,24 @@ impl NliVerifier for OnnxNliVerifier {
|
||||
let l = [logits[[0, 0]], logits[[0, 1]], logits[[0, 2]]];
|
||||
Ok(NliScores::from_xnli_logits(l))
|
||||
}
|
||||
|
||||
/// **Override** the trait default `Ok(0)` with a real mDeBERTa
|
||||
/// tokenize. Pipeline 의 `truncate_hypothesis_for_nli_with_budget`
|
||||
/// retry loop 가 이 method 를 vtable 통해 호출 — production code
|
||||
/// path 에서 실 token count 측정.
|
||||
///
|
||||
/// **CRITICAL placement**: 이 method 는 *trait impl block 안* 에
|
||||
/// 위치해야 vtable 에 등록 — inherent `impl OnnxNliVerifier {}` 안에
|
||||
/// 두면 dispatch 시 trait default (`Ok(0)`) 호출 → retry loop
|
||||
/// 즉시 통과 → production silent NO-OP (S3 follow-up 2026-05-26
|
||||
/// RC1-residual closure).
|
||||
fn hypothesis_token_count(&self, hypothesis: &str) -> Result<usize> {
|
||||
let (_session, tokenizer) = self.ensure_loaded()?;
|
||||
let enc = tokenizer
|
||||
.encode(hypothesis, /*add_special_tokens=*/ false)
|
||||
.map_err(|e| anyhow!("kebab-nli: tokenizer.encode (probe) failed: {e}"))?;
|
||||
Ok(enc.get_ids().len())
|
||||
}
|
||||
}
|
||||
|
||||
/// Make a HuggingFace model id (`"owner/repo"`) into a single
|
||||
|
||||
@@ -139,3 +139,59 @@ fn empty_hypothesis_returns_err() {
|
||||
"expected 'empty hypothesis' in error, got: {msg}"
|
||||
);
|
||||
}
|
||||
|
||||
/// Test 6 (S3 follow-up 2026-05-26): EN-long hypothesis alone exceeds
|
||||
/// max_length. Without pipeline-side truncation, `OnlyFirst` strategy
|
||||
/// dead-ends. Pin raw nli crate behavior so any future regression in
|
||||
/// the pipeline-side budget surfaces as a clear nli-level err.
|
||||
#[test]
|
||||
#[ignore]
|
||||
fn score_long_en_hypothesis_returns_err_without_pipeline_truncation() {
|
||||
let cfg = Config::defaults();
|
||||
let v = OnnxNliVerifier::new(&cfg).expect("verifier construction");
|
||||
let premise = "short premise";
|
||||
let hypothesis = "lorem ipsum ".repeat(500); // ~6 000 chars / >>512 tokens
|
||||
let result = v.score(premise, &hypothesis);
|
||||
assert!(result.is_err(), "long hypothesis should err under OnlyFirst");
|
||||
let msg = result.err().unwrap().to_string();
|
||||
assert!(
|
||||
msg.contains("Truncation error") || msg.contains("too short to respect"),
|
||||
"expected tokenizer truncation err, got: {msg}"
|
||||
);
|
||||
}
|
||||
|
||||
/// Test 7 (S3 follow-up 2026-05-26): `hypothesis_token_count` helper —
|
||||
/// pure tokenizer probe. **vtable dispatch 검증** (RC1-residual pin) —
|
||||
/// concrete type 호출은 inherent method 우선이라 RC1-residual 버그
|
||||
/// 잡지 못함; `&dyn NliVerifier` 통해 dispatch 해야 vtable 등록 검증.
|
||||
/// inherent-only 배치 시 default `Ok(0)` 반환 → `assert!(count > 0)`
|
||||
/// 실패. trait impl block 배치 시 real tokenizer → PASS. Pipeline 이
|
||||
/// retry budget 결정에 사용하는 API 의 정확성 pin.
|
||||
#[test]
|
||||
#[ignore]
|
||||
fn hypothesis_token_count_dispatches_correctly_via_dyn_trait() {
|
||||
let cfg = Config::defaults();
|
||||
let v = OnnxNliVerifier::new(&cfg).expect("verifier construction");
|
||||
// ★ vtable dispatch — &dyn NliVerifier 통해 호출. inherent-only
|
||||
// 배치 시 default `Ok(0)` 반환 → assert!(count > 0) 실패.
|
||||
// trait impl block 배치 시 real tokenizer → PASS. RC1-residual
|
||||
// 의 코드-수준 regression pin.
|
||||
let v_dyn: &dyn NliVerifier = &v;
|
||||
// 짧은 EN — 4 chars/token 추정 (27 chars / 4 = ~6 tokens)
|
||||
let en_count = v_dyn
|
||||
.hypothesis_token_count("short english test sentence")
|
||||
.expect("EN dyn dispatch must reach real tokenizer (vtable check)");
|
||||
assert!(
|
||||
en_count > 0 && en_count < 20,
|
||||
"EN ~6 tokens expected via vtable dispatch, got {en_count} \
|
||||
(Ok(0) signals inherent-only placement bug — RC1-residual)"
|
||||
);
|
||||
// 짧은 KR — 1-2 chars/token (15 chars / 1.5 = ~10 tokens)
|
||||
let kr_count = v_dyn
|
||||
.hypothesis_token_count("짧은 한국어 테스트 문장입니다")
|
||||
.expect("KR dyn dispatch must reach real tokenizer");
|
||||
assert!(
|
||||
kr_count > 0 && kr_count < 30,
|
||||
"KR ~10 tokens expected, got {kr_count}"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -22,4 +22,8 @@ pub use kebab_core::{Answer, AnswerCitation, AnswerRetrievalSummary, RefusalReas
|
||||
|
||||
mod pipeline;
|
||||
|
||||
pub use pipeline::{AskOpts, MAX_NLI_PREMISE_CHARS, RagPipeline, StreamEvent, truncate_for_nli};
|
||||
pub use pipeline::{
|
||||
AskOpts, MAX_NLI_HYPOTHESIS_CHARS_INITIAL, MAX_NLI_HYPOTHESIS_CHARS_MIN,
|
||||
MAX_NLI_PREMISE_CHARS, RagPipeline, StreamEvent, truncate_for_nli,
|
||||
truncate_hypothesis_for_nli_with_budget,
|
||||
};
|
||||
|
||||
@@ -1038,14 +1038,37 @@ impl RagPipeline {
|
||||
"verifier must be Some when nli_threshold > 0.0 \
|
||||
(kebab-app's open_with_config enforces this invariant)",
|
||||
);
|
||||
let (truncated_premise, was_truncated) = truncate_for_nli(&packed_text);
|
||||
if was_truncated {
|
||||
let (truncated_premise, premise_was_truncated) = truncate_for_nli(&packed_text);
|
||||
if premise_was_truncated {
|
||||
tracing::debug!(
|
||||
target: "kebab-rag",
|
||||
"NLI premise truncated to MAX_NLI_PREMISE_CHARS for entailment check"
|
||||
);
|
||||
}
|
||||
match v.score(&truncated_premise, &acc) {
|
||||
// S3 follow-up (2026-05-26): hypothesis-side budget + token-count
|
||||
// fallback retry. `?` 사용 금지 — wire `answer.v1 + NliModelUnavailable
|
||||
// refusal` 유지 (graceful fallback, regression 0). v.score() Err
|
||||
// 분기와 *대칭* explicit match + return refuse.
|
||||
let (truncated_hypothesis, hypothesis_was_truncated) =
|
||||
match truncate_hypothesis_for_nli_with_budget(v.as_ref(), &acc) {
|
||||
Ok(x) => x,
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
target: "kebab-rag",
|
||||
error = %e,
|
||||
"NLI hypothesis budget retry exhausted; refusing with NliModelUnavailable"
|
||||
);
|
||||
return self.refuse_nli_model_unavailable(query, &opts, hops, started);
|
||||
}
|
||||
};
|
||||
if hypothesis_was_truncated {
|
||||
tracing::debug!(
|
||||
target: "kebab-rag",
|
||||
original_chars = acc.chars().count(),
|
||||
"NLI hypothesis truncated to MAX_NLI_HYPOTHESIS_CHARS"
|
||||
);
|
||||
}
|
||||
match v.score(&truncated_premise, &truncated_hypothesis) {
|
||||
Ok(scores) => {
|
||||
let passed = scores.entailment >= self.config.rag.nli_threshold;
|
||||
Some(VerificationSummary {
|
||||
@@ -1816,6 +1839,82 @@ pub fn truncate_for_nli(premise: &str) -> (String, bool) {
|
||||
}
|
||||
}
|
||||
|
||||
/// S3 follow-up (2026-05-26): NLI hypothesis (= synthesized answer)
|
||||
/// 가 mDeBERTa-v3 의 512-token cap 을 단독 초과하면 `OnlyFirst`
|
||||
/// truncation 이 premise 를 0 까지 잘라도 fit 시킬 수 없어 tokenizer
|
||||
/// `SequenceTooShortToTruncate` err. char-budget 으로 자른 후 *실
|
||||
/// mDeBERTa tokenizer* 로 token count 재검증 → 초과 시 char budget
|
||||
/// 절반으로 retry. KR-heavy hypothesis (1-2 chars/token) safe.
|
||||
pub const MAX_NLI_HYPOTHESIS_CHARS_INITIAL: usize = 1200;
|
||||
|
||||
/// S3 follow-up (2026-05-26): retry budget 의 최소 floor. budget 이
|
||||
/// 이 값 미만으로 절반화되면 graceful `nli_model_unavailable` fallback
|
||||
/// 으로 빠짐 (regression 0). KR-extreme density (한자/CJK 의 1 char
|
||||
/// = 2-3 tokens) 케이스 보호.
|
||||
pub const MAX_NLI_HYPOTHESIS_CHARS_MIN: usize = 150;
|
||||
|
||||
/// S3 follow-up (2026-05-26): chars-only truncation arithmetic. Pure
|
||||
/// fn: input → output, no side effect. Codepoint-aware (chars().count()
|
||||
/// + chars().take()) — KR / emoji / multi-byte 안전.
|
||||
///
|
||||
/// Used internally by `truncate_hypothesis_for_nli_with_budget` 의
|
||||
/// retry loop 의 각 step. Pure-fn unit tests (in this file's
|
||||
/// `#[cfg(test)] mod tests`) pin the arithmetic 회귀.
|
||||
pub(crate) fn truncate_chars(s: &str, budget: usize) -> (String, bool) {
|
||||
if s.chars().count() <= budget {
|
||||
(s.to_string(), false)
|
||||
} else {
|
||||
let truncated: String = s.chars().take(budget).collect();
|
||||
(truncated, true)
|
||||
}
|
||||
}
|
||||
|
||||
/// S3 follow-up (2026-05-26): hypothesis-side budget + token-count
|
||||
/// fallback retry. Char-truncate (`Right` direction = front preserved
|
||||
/// — LLM 답변의 도입부에 핵심 claim 이 있음) → real mDeBERTa tokenizer
|
||||
/// 로 token count 재검증 → 초과 시 char budget 절반화 retry (1200 →
|
||||
/// 600 → 300 → 150). Min floor 미달 시 `anyhow::Err` — caller (step
|
||||
/// 8.5 hook) 가 graceful `nli_model_unavailable` refusal 로 fallback
|
||||
/// (regression 0).
|
||||
///
|
||||
/// Returns `(truncated_hypothesis, was_truncated)`. `was_truncated`
|
||||
/// 은 logging 용 — wire 영향 0.
|
||||
pub fn truncate_hypothesis_for_nli_with_budget(
|
||||
verifier: &(dyn kebab_nli::NliVerifier + 'static),
|
||||
hypothesis: &str,
|
||||
) -> anyhow::Result<(String, bool)> {
|
||||
let original_chars = hypothesis.chars().count();
|
||||
let mut budget = MAX_NLI_HYPOTHESIS_CHARS_INITIAL;
|
||||
let mut was_truncated = false;
|
||||
|
||||
loop {
|
||||
let (candidate, this_truncated) = truncate_chars(hypothesis, budget);
|
||||
if this_truncated {
|
||||
was_truncated = true;
|
||||
}
|
||||
|
||||
// verifier 의 internal tokenizer 로 token count 재검증.
|
||||
// trait method (vtable dispatch) — `OnnxNliVerifier` 는
|
||||
// trait impl block 안에서 override (RC1-residual).
|
||||
let token_count = verifier
|
||||
.hypothesis_token_count(&candidate)
|
||||
.with_context(|| "kebab-rag: hypothesis token-count probe failed")?;
|
||||
if token_count <= kebab_nli::OnnxNliVerifier::HYPOTHESIS_TOKEN_BUDGET {
|
||||
return Ok((candidate, was_truncated));
|
||||
}
|
||||
|
||||
// 초과 — char budget 절반화 retry.
|
||||
budget /= 2;
|
||||
if budget < MAX_NLI_HYPOTHESIS_CHARS_MIN {
|
||||
anyhow::bail!(
|
||||
"kebab-rag: hypothesis remains over token budget after retry (original {original_chars} chars, last budget {} chars, tokens {token_count} > {})",
|
||||
budget * 2,
|
||||
kebab_nli::OnnxNliVerifier::HYPOTHESIS_TOKEN_BUDGET,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const MULTI_HOP_DECOMPOSE_SYSTEM_PROMPT: &str = "당신은 사용자의 질문을 다단계 검색에 필요한 sub-question 들로 분해하는 도구다.\n- multi-hop 정보가 필요한 경우 독립적으로 검색 가능한 sub-question 들로 분해한다.\n- 각 sub-question 은 자기 자신만으로 의미가 통해야 한다 (대명사 / \"위 답변\" 같은 reference 금지).\n- 원본이 이미 단순하면 원본 그대로 1 개만 반환한다.\n- 응답은 JSON array of strings 만 출력한다. 다른 prose / markdown fence / 설명 금지.";
|
||||
|
||||
const MULTI_HOP_DECIDE_SYSTEM_PROMPT: &str = "당신은 multi-hop 검색의 매 iter 에서 \"추가 retrieval 이 필요한가?\" 를 판단하는 도구다.\n- 지금까지 모은 [근거] 가 [원본 질문] 의 모든 측면을 cover 하는지 평가한다.\n- 추가가 필요하면 새 sub-question 들 (이미 모은 정보로 답할 수 없는 부분만, 독립적으로 검색 가능한 형태로) 을 JSON array of strings 로 반환한다.\n- 충분하면 빈 array `[]` 를 반환한다.\n- 응답은 JSON array of strings 만 출력한다. 다른 prose / markdown fence / 설명 금지.\n- 각 sub-question 은 자기 자신만으로 의미가 통해야 한다 (대명사 / \"위 답변\" 같은 reference 금지).";
|
||||
@@ -2097,6 +2196,52 @@ mod tests {
|
||||
assert_eq!(out, vec!["valid q"]);
|
||||
}
|
||||
|
||||
// ── S3 follow-up (2026-05-26): truncate_chars boundary tests ─────────
|
||||
//
|
||||
// Pure-fn arithmetic 회귀 핀. `truncate_chars` 가 `pub(crate)` 라
|
||||
// integration test 파일에서 접근 불가 — 동일 crate 의 `#[cfg(test)]
|
||||
// mod tests` 안에서 직접 호출.
|
||||
|
||||
#[test]
|
||||
fn truncate_chars_identity_when_under_budget() {
|
||||
let s = "short";
|
||||
let (out, was_truncated) = truncate_chars(s, 100);
|
||||
assert_eq!(out, s);
|
||||
assert!(!was_truncated);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncate_chars_truncates_when_over_budget() {
|
||||
let s = "abcdefghij"; // 10 chars
|
||||
let (out, was_truncated) = truncate_chars(s, 3);
|
||||
assert_eq!(out, "abc");
|
||||
assert_eq!(out.chars().count(), 3);
|
||||
assert!(was_truncated);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncate_chars_empty_input_is_identity() {
|
||||
let (out, was_truncated) = truncate_chars("", 100);
|
||||
assert_eq!(out, "");
|
||||
assert!(!was_truncated);
|
||||
// budget = 0 도 empty 입력에서는 identity.
|
||||
let (out2, was_truncated2) = truncate_chars("", 0);
|
||||
assert_eq!(out2, "");
|
||||
assert!(!was_truncated2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncate_chars_counts_codepoints_not_bytes() {
|
||||
// "가나다라마" = 5 chars, 각 char 는 3 bytes (UTF-8) → 15 bytes.
|
||||
// budget = 3 chars 일 때 "가나다" 3 chars / 9 bytes 가 정확.
|
||||
let kr = "가나다라마";
|
||||
let (out, was_truncated) = truncate_chars(kr, 3);
|
||||
assert_eq!(out, "가나다");
|
||||
assert_eq!(out.chars().count(), 3);
|
||||
assert_eq!(out.len(), 9, "3 KR codepoints × 3 bytes/char = 9 bytes");
|
||||
assert!(was_truncated);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn est_tokens_approx_quarters() {
|
||||
assert_eq!(est_tokens(""), 0);
|
||||
|
||||
@@ -455,3 +455,52 @@ impl NliVerifier for MockNliVerifier {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// S3 follow-up (2026-05-26): closure type aliases for `SpyNliVerifier`
|
||||
/// fields — clippy `type_complexity` 회피.
|
||||
#[allow(clippy::type_complexity)]
|
||||
pub type ScoreFn = Arc<dyn Fn(&str, &str) -> anyhow::Result<NliScores> + Send + Sync>;
|
||||
#[allow(clippy::type_complexity)]
|
||||
pub type HypothesisTokenCountFn = Arc<dyn Fn(&str) -> anyhow::Result<usize> + Send + Sync>;
|
||||
|
||||
/// S3 follow-up (2026-05-26): closure-based NLI verifier — caller 가
|
||||
/// `(premise, hypothesis) -> Result<NliScores>` + `(hypothesis) ->
|
||||
/// Result<usize>` 두 closures 정의 가능 + spy 로 입력 capture. 기존
|
||||
/// `MockNliVerifier` (고정 mode) 와 sibling. truncate_hypothesis_for_nli_with_budget
|
||||
/// retry loop 의 token-count 시뮬레이션 + final score 동작을 한 verifier
|
||||
/// 안에서 inject 하기 위함.
|
||||
pub struct SpyNliVerifier {
|
||||
pub score_fn: ScoreFn,
|
||||
pub hypothesis_token_count_fn: HypothesisTokenCountFn,
|
||||
pub received_premises: Mutex<Vec<String>>,
|
||||
pub received_hypotheses: Mutex<Vec<String>>,
|
||||
}
|
||||
|
||||
impl SpyNliVerifier {
|
||||
/// 2-arg constructor — score + hypothesis_token_count 둘 다 closure
|
||||
/// 로 주입. `Arc<T>` field mutation 불가 회피를 위해 한 번에 전달.
|
||||
pub fn new<F, G>(score_fn: F, token_count_fn: G) -> Arc<Self>
|
||||
where
|
||||
F: Fn(&str, &str) -> anyhow::Result<NliScores> + Send + Sync + 'static,
|
||||
G: Fn(&str) -> anyhow::Result<usize> + Send + Sync + 'static,
|
||||
{
|
||||
Arc::new(Self {
|
||||
score_fn: Arc::new(score_fn),
|
||||
hypothesis_token_count_fn: Arc::new(token_count_fn),
|
||||
received_premises: Mutex::new(Vec::new()),
|
||||
received_hypotheses: Mutex::new(Vec::new()),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl NliVerifier for SpyNliVerifier {
|
||||
fn score(&self, premise: &str, hypothesis: &str) -> anyhow::Result<NliScores> {
|
||||
self.received_premises.lock().unwrap().push(premise.to_string());
|
||||
self.received_hypotheses.lock().unwrap().push(hypothesis.to_string());
|
||||
(self.score_fn)(premise, hypothesis)
|
||||
}
|
||||
|
||||
fn hypothesis_token_count(&self, hypothesis: &str) -> anyhow::Result<usize> {
|
||||
(self.hypothesis_token_count_fn)(hypothesis)
|
||||
}
|
||||
}
|
||||
|
||||
245
crates/kebab-rag/tests/multi_hop_nli_truncate.rs
Normal file
245
crates/kebab-rag/tests/multi_hop_nli_truncate.rs
Normal file
@@ -0,0 +1,245 @@
|
||||
//! S3 follow-up (2026-05-26): hypothesis-side char budget + token-count
|
||||
//! fallback retry — multi-hop integration tests via `SpyNliVerifier`.
|
||||
//!
|
||||
//! Coverage:
|
||||
//!
|
||||
//! 1. `long_en_synth_answer_truncated_before_nli_call` — EN long answer
|
||||
//! → char-budget 만으로 충분 (token_count Ok(100)) → retry 0 회 →
|
||||
//! hypothesis 가 정확히 1200 chars 로 truncate + Right direction pin
|
||||
//! (앞부분 보존).
|
||||
//! 2. `long_kr_synth_answer_retries_with_smaller_budget` — KR-sim long
|
||||
//! answer → token_count fn 이 chars > 1000 → 900 / > 500 → 450 /
|
||||
//! else → 220 시뮬레이션 → retry >= 3 회 + 최종 hypothesis ≤ 300
|
||||
//! chars + happy path. KR safety pin.
|
||||
//! 3. `unrelenting_token_overflow_falls_through_to_unavailable` —
|
||||
//! token_count fn 이 무조건 Ok(9_999) → retry 소진 → graceful
|
||||
//! `NliModelUnavailable` refusal (regression 0).
|
||||
//!
|
||||
//! Pipeline construction pattern (Option B inline — plan §2 step 8):
|
||||
//! 각 test 안에서 `RagEnv::new()` + `ScriptedRetriever::new(...)` +
|
||||
//! `ScriptedLm::new(vec![decompose, decide, synth])` +
|
||||
//! `RagPipeline::new(...).with_verifier(verifier)` inline. helper
|
||||
//! `build_test_pipeline_with_long_answer` 미작성 (1회용 + 매 test 마다
|
||||
//! long-answer 길이/언어 다름).
|
||||
|
||||
mod common;
|
||||
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use common::{RagEnv, ScriptedLm, ScriptedRetriever, SpyNliVerifier, id32, mk_hit};
|
||||
use kebab_core::{LanguageModel, RefusalReason, Retriever, SearchMode};
|
||||
use kebab_nli::{NliScores, NliVerifier};
|
||||
use kebab_rag::{AskOpts, RagPipeline};
|
||||
|
||||
/// Default `AskOpts` for multi-hop tests: deterministic seed, lexical
|
||||
/// mode (so the test crate doesn't need to wire up an embedder), and
|
||||
/// `multi_hop: true` to route through `ask_multi_hop`.
|
||||
fn multi_hop_opts() -> AskOpts {
|
||||
AskOpts {
|
||||
k: 5,
|
||||
explain: false,
|
||||
mode: SearchMode::Lexical,
|
||||
temperature: Some(0.0),
|
||||
seed: Some(0),
|
||||
stream_sink: None,
|
||||
history: Vec::new(),
|
||||
conversation_id: None,
|
||||
turn_index: None,
|
||||
multi_hop: true,
|
||||
}
|
||||
}
|
||||
|
||||
/// EN long answer (5 000 chars) → char-budget 만으로 충분 → retry 0회 →
|
||||
/// grounded. Right direction pin: hypothesis 의 첫 1200 chars 가 input 의
|
||||
/// 첫 1200 chars 와 일치 (= Right direction = 앞부분 보존).
|
||||
#[test]
|
||||
fn long_en_synth_answer_truncated_before_nli_call() {
|
||||
let env = RagEnv::new();
|
||||
let cid = id32("c1");
|
||||
let did = id32("d1");
|
||||
env.seed_chunk(&cid, &did, "notes/a.md", "Body text.", &["Intro"]);
|
||||
let hits = vec![mk_hit(1, &cid, &did, "notes/a.md", 0.85, &["Intro"])];
|
||||
|
||||
let mut cfg = env.config.clone();
|
||||
cfg.rag.nli_threshold = 0.5;
|
||||
|
||||
// ScriptedRetriever: entry 0 = probe, entry 1 = q1 sub-query.
|
||||
let retriever = Arc::new(ScriptedRetriever::new(vec![hits.clone(), hits]));
|
||||
let retriever_dyn: Arc<dyn Retriever> = retriever;
|
||||
|
||||
// Synthesize answer ~6000 chars (lorem ipsum, 12 chars × 500 reps),
|
||||
// citation marker appended after so `grounded_unaware` 통과.
|
||||
let long_answer = format!("{} [#1]", "lorem ipsum ".repeat(500));
|
||||
let lm = Arc::new(ScriptedLm::new(vec![
|
||||
r#"["q1"]"#,
|
||||
r"[]",
|
||||
&long_answer,
|
||||
]));
|
||||
let lm_dyn: Arc<dyn LanguageModel> = lm;
|
||||
|
||||
let verifier = SpyNliVerifier::new(
|
||||
|_premise, _hypothesis| {
|
||||
Ok(NliScores {
|
||||
entailment: 0.9,
|
||||
neutral: 0.05,
|
||||
contradiction: 0.05,
|
||||
})
|
||||
},
|
||||
|_h| Ok(100), // budget 안 — retry 0
|
||||
);
|
||||
let verifier_handle = verifier.clone();
|
||||
let verifier_dyn: Arc<dyn NliVerifier> = verifier;
|
||||
|
||||
let pipeline = RagPipeline::new(cfg, retriever_dyn, lm_dyn, env.sqlite.clone())
|
||||
.with_verifier(verifier_dyn);
|
||||
|
||||
let answer = pipeline.ask("compound", multi_hop_opts()).unwrap();
|
||||
|
||||
let received = verifier_handle.received_hypotheses.lock().unwrap();
|
||||
assert_eq!(received.len(), 1, "verifier called exactly once");
|
||||
|
||||
let hyp = &received[0];
|
||||
assert_eq!(
|
||||
hyp.chars().count(),
|
||||
1200,
|
||||
"hypothesis truncated to MAX_NLI_HYPOTHESIS_CHARS_INITIAL"
|
||||
);
|
||||
|
||||
// Right direction pin — hypothesis 의 첫 1200 chars 가 input 의
|
||||
// 첫 1200 chars 와 일치 (= Right direction = 앞부분 보존). Left/Middle
|
||||
// direction 으로 regress 시 본 test 가 즉시 fail.
|
||||
let input_first_1200: String = long_answer.chars().take(1200).collect();
|
||||
assert_eq!(
|
||||
hyp.as_str(),
|
||||
input_first_1200.as_str(),
|
||||
"Right direction = front preserved"
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
answer.refusal_reason, None,
|
||||
"long answer must reach happy path"
|
||||
);
|
||||
}
|
||||
|
||||
/// KR long answer → token count > budget → char budget 절반화 retry →
|
||||
/// eventual fit. KR safety pin (1-2 chars/token density 시뮬레이션).
|
||||
#[test]
|
||||
fn long_kr_synth_answer_retries_with_smaller_budget() {
|
||||
let env = RagEnv::new();
|
||||
let cid = id32("c1");
|
||||
let did = id32("d1");
|
||||
env.seed_chunk(&cid, &did, "notes/a.md", "Body text.", &["Intro"]);
|
||||
let hits = vec![mk_hit(1, &cid, &did, "notes/a.md", 0.85, &["Intro"])];
|
||||
|
||||
let mut cfg = env.config.clone();
|
||||
cfg.rag.nli_threshold = 0.5;
|
||||
|
||||
let retriever = Arc::new(ScriptedRetriever::new(vec![hits.clone(), hits]));
|
||||
let retriever_dyn: Arc<dyn Retriever> = retriever;
|
||||
|
||||
// ~2500-char KR-sim answer (한국어 6 chars × 416 reps ≈ 2496 chars)
|
||||
// + citation marker.
|
||||
let kr_long_answer = format!("{} [#1]", "한국어 본문 ".repeat(416));
|
||||
let lm = Arc::new(ScriptedLm::new(vec![
|
||||
r#"["q1"]"#,
|
||||
r"[]",
|
||||
&kr_long_answer,
|
||||
]));
|
||||
let lm_dyn: Arc<dyn LanguageModel> = lm;
|
||||
|
||||
let token_count_call_count = Arc::new(Mutex::new(0_usize));
|
||||
let tcc = token_count_call_count.clone();
|
||||
// 시뮬레이션: 1200 chars → 900 tokens (cap 초과), 600 chars → 450
|
||||
// tokens (cap 초과), 300 chars → 220 tokens (cap 안). retry 3 회.
|
||||
let verifier = SpyNliVerifier::new(
|
||||
|_premise, _hypothesis| {
|
||||
Ok(NliScores {
|
||||
entailment: 0.85,
|
||||
neutral: 0.10,
|
||||
contradiction: 0.05,
|
||||
})
|
||||
},
|
||||
move |h| {
|
||||
*tcc.lock().unwrap() += 1;
|
||||
let count = h.chars().count();
|
||||
if count > 1000 {
|
||||
Ok(900)
|
||||
} else if count > 500 {
|
||||
Ok(450)
|
||||
} else {
|
||||
Ok(220)
|
||||
}
|
||||
},
|
||||
);
|
||||
let verifier_handle = verifier.clone();
|
||||
let verifier_dyn: Arc<dyn NliVerifier> = verifier;
|
||||
|
||||
let pipeline = RagPipeline::new(cfg, retriever_dyn, lm_dyn, env.sqlite.clone())
|
||||
.with_verifier(verifier_dyn);
|
||||
|
||||
let answer = pipeline.ask("compound", multi_hop_opts()).unwrap();
|
||||
|
||||
assert!(
|
||||
*token_count_call_count.lock().unwrap() >= 3,
|
||||
"retry loop must call token_count >= 3 (1200, 600, 300 candidates)"
|
||||
);
|
||||
let received = verifier_handle.received_hypotheses.lock().unwrap();
|
||||
assert!(
|
||||
received[0].chars().count() <= 300,
|
||||
"final hypothesis <= 300 chars after retry, got {}",
|
||||
received[0].chars().count()
|
||||
);
|
||||
assert_eq!(
|
||||
answer.refusal_reason, None,
|
||||
"KR long answer reaches happy path after retry"
|
||||
);
|
||||
}
|
||||
|
||||
/// Retry budget 소진 시 graceful unavailable — fix 의 fallback path 가
|
||||
/// 기존 unavailable wire shape 유지 (regression 0).
|
||||
#[test]
|
||||
fn unrelenting_token_overflow_falls_through_to_unavailable() {
|
||||
let env = RagEnv::new();
|
||||
let cid = id32("c1");
|
||||
let did = id32("d1");
|
||||
env.seed_chunk(&cid, &did, "notes/a.md", "Body text.", &["Intro"]);
|
||||
let hits = vec![mk_hit(1, &cid, &did, "notes/a.md", 0.85, &["Intro"])];
|
||||
|
||||
let mut cfg = env.config.clone();
|
||||
cfg.rag.nli_threshold = 0.5;
|
||||
|
||||
let retriever = Arc::new(ScriptedRetriever::new(vec![hits.clone(), hits]));
|
||||
let retriever_dyn: Arc<dyn Retriever> = retriever;
|
||||
|
||||
// ~3000-char answer + marker — over budget regardless.
|
||||
let unrelenting_answer = format!("{} [#1]", "단단한 압축 ".repeat(500));
|
||||
let lm = Arc::new(ScriptedLm::new(vec![
|
||||
r#"["q1"]"#,
|
||||
r"[]",
|
||||
&unrelenting_answer,
|
||||
]));
|
||||
let lm_dyn: Arc<dyn LanguageModel> = lm;
|
||||
|
||||
let verifier = SpyNliVerifier::new(
|
||||
|_premise, _hypothesis| {
|
||||
unreachable!("score not reached when token-count check fails");
|
||||
},
|
||||
|_h| Ok(9_999), // 모든 budget 에서 token count 초과 — retry 소진
|
||||
);
|
||||
let verifier_dyn: Arc<dyn NliVerifier> = verifier;
|
||||
|
||||
let pipeline = RagPipeline::new(cfg, retriever_dyn, lm_dyn, env.sqlite.clone())
|
||||
.with_verifier(verifier_dyn);
|
||||
|
||||
let answer = pipeline.ask("compound", multi_hop_opts()).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
answer.refusal_reason,
|
||||
Some(RefusalReason::NliModelUnavailable),
|
||||
"graceful fallback to unavailable when retry exhausted"
|
||||
);
|
||||
assert!(
|
||||
answer.verification.is_none(),
|
||||
"NliModelUnavailable: verification stays None"
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user