feat(rag): pipeline emits StreamEvent + cancel on SendError (fb-33)

RetrievalDone after retrieve+stale-stamp, Token per LM chunk
(SendError → break, FinishReason::Cancelled, RefusalReason::
LlmStreamAborted), Final on success. answers row still persists
on cancel for audit. Adds FinishReason::Cancelled, re-exports
StreamEvent from kebab_rag, migrates two pre-fb-33 sink tests
in tests/pipeline.rs to the new StreamEvent type (the
"dropped receiver does not abort" test inverts to record cancel).

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

crates/kebab-rag/tests/pipeline.rs

@@ -14,7 +14,7 @@ use kebab_core::{
     FinishReason, LanguageModel, Retriever, SearchMode, TokenChunk, TokenUsage,
 };
 use kebab_llm::MockLanguageModel;
-use kebab_rag::{AskOpts, RagPipeline, RefusalReason};
+use kebab_rag::{AskOpts, RagPipeline, RefusalReason, StreamEvent};
 
 /// LM ID used everywhere — kept short so snapshots stay stable.
 const TEST_LM_ID: &str = "mock-lm";
@@ -270,18 +270,32 @@ fn streaming_forwards_tokens_to_sink() {
     let lm: Arc<dyn LanguageModel> = Arc::new(CountingLm::new(canned));
     let pipeline = RagPipeline::new(env.config.clone(), retriever, lm, env.sqlite.clone());
 
-    let (tx, rx) = std::sync::mpsc::channel::<String>();
+    let (tx, rx) = std::sync::mpsc::channel::<StreamEvent>();
     let mut opts = default_opts();
     opts.stream_sink = Some(tx);
     let _ = pipeline.ask("q", opts).unwrap();
-    let collected: String = rx.into_iter().collect::<Vec<_>>().join("");
+    // p9-fb-33: extract Token deltas from the staged event stream.
+    let collected: String = rx
+        .into_iter()
+        .filter_map(|ev| match ev {
+            StreamEvent::Token { delta, .. } => Some(delta),
+            _ => None,
+        })
+        .collect::<Vec<_>>()
+        .join("");
     assert_eq!(collected, canned);
 }
 
-// ── 10. dropped receiver does NOT abort generation ────────────────────────
+// ── 10. dropped receiver aborts generation, records LlmStreamAborted ──────
+//
+// p9-fb-33: cancel semantics changed. Pre-fb-33 the pipeline drove
+// the LM loop to completion and silently dropped sends. Now a
+// SendError breaks the loop and stamps `RefusalReason::LlmStreamAborted`
+// onto the persisted row — the partial answer (whatever was buffered
+// before the cancel) still gets written for audit.
 
 #[test]
-fn dropped_receiver_does_not_abort_generation() {
+fn dropped_receiver_aborts_with_llm_stream_aborted() {
     let env = RagEnv::new();
     let cid = id32("c1");
     let did = id32("d1");
@@ -292,13 +306,17 @@ fn dropped_receiver_does_not_abort_generation() {
     let lm: Arc<dyn LanguageModel> = Arc::new(CountingLm::new(canned));
     let pipeline = RagPipeline::new(env.config.clone(), retriever, lm, env.sqlite.clone());
 
-    let (tx, rx) = std::sync::mpsc::channel::<String>();
-    drop(rx); // receiver gone — every send fails silently
+    let (tx, rx) = std::sync::mpsc::channel::<StreamEvent>();
+    drop(rx); // receiver gone — first Token send fails, loop breaks
     let mut opts = default_opts();
     opts.stream_sink = Some(tx);
     let answer = pipeline.ask("q", opts).unwrap();
-    assert_eq!(answer.answer, canned, "generation completes despite dead sink");
-    assert!(answer.grounded);
+    assert!(!answer.grounded, "cancel takes priority over grounded");
+    assert_eq!(
+        answer.refusal_reason,
+        Some(RefusalReason::LlmStreamAborted),
+        "cancel records LlmStreamAborted",
+    );
     assert_eq!(env.count_answers(), 1, "answers row still persisted");
 }
 

crates/kebab-rag/tests/streaming_events.rs

@@ -0,0 +1,131 @@
+//! p9-fb-33: pipeline-level streaming behavior — order invariants,
+//! cancel propagation, refusal flagging.
+
+mod common;
+
+use std::sync::Arc;
+use std::sync::atomic::Ordering;
+use std::sync::mpsc;
+
+use common::{MockRetriever, RagEnv, id32, mk_hit};
+use kebab_core::{
+    FinishReason, LanguageModel, RefusalReason, Retriever, SearchMode, TokenChunk, TokenUsage,
+};
+use kebab_llm::MockLanguageModel;
+use kebab_rag::{AskOpts, RagPipeline, StreamEvent};
+
+const TEST_LM_ID: &str = "mock-lm";
+
+/// Minimal LM mirroring `tests/pipeline.rs::CountingLm` so the
+/// streaming-events suite stays self-contained.
+struct CountingLm {
+    inner: MockLanguageModel,
+    calls: std::sync::atomic::AtomicUsize,
+}
+
+impl CountingLm {
+    fn new(canned: &str) -> Self {
+        Self {
+            inner: MockLanguageModel {
+                model_id: TEST_LM_ID.to_string(),
+                provider: "mock".to_string(),
+                context_tokens: 32_768,
+                canned_response: canned.to_string(),
+                canned_finish: FinishReason::Stop,
+                canned_usage: TokenUsage {
+                    prompt_tokens: 10,
+                    completion_tokens: 5,
+                    latency_ms: 7,
+                },
+            },
+            calls: std::sync::atomic::AtomicUsize::new(0),
+        }
+    }
+}
+
+impl LanguageModel for CountingLm {
+    fn model_ref(&self) -> kebab_core::ModelRef {
+        self.inner.model_ref()
+    }
+    fn context_tokens(&self) -> usize {
+        self.inner.context_tokens()
+    }
+    fn generate_stream(
+        &self,
+        req: kebab_core::GenerateRequest,
+    ) -> anyhow::Result<Box<dyn Iterator<Item = anyhow::Result<TokenChunk>> + Send>> {
+        self.calls.fetch_add(1, Ordering::SeqCst);
+        self.inner.generate_stream(req)
+    }
+}
+
+fn opts_with_sink(tx: mpsc::Sender<StreamEvent>) -> AskOpts {
+    AskOpts {
+        k: 3,
+        explain: false,
+        mode: SearchMode::Lexical,
+        temperature: Some(0.0),
+        seed: Some(0),
+        stream_sink: Some(tx),
+        history: Vec::new(),
+        conversation_id: None,
+        turn_index: None,
+    }
+}
+
+/// Build a pipeline with one seeded chunk + canned LM response so
+/// retrieval lands a single hit and the LM emits at least one token.
+fn env_with_one_hit(canned: &str) -> (RagEnv, RagPipeline) {
+    let env = RagEnv::new();
+    let cid = id32("c1");
+    let did = id32("d1");
+    env.seed_chunk(&cid, &did, "notes/a.md", "apples are red.", &["Intro"]);
+    let hits = vec![mk_hit(1, &cid, &did, "notes/a.md", 0.85, &["Intro"])];
+    let retriever: Arc<dyn Retriever> = Arc::new(MockRetriever::new(hits));
+    let lm: Arc<dyn LanguageModel> = Arc::new(CountingLm::new(canned));
+    let pipeline = RagPipeline::new(env.config.clone(), retriever, lm, env.sqlite.clone());
+    (env, pipeline)
+}
+
+#[test]
+fn ask_emits_retrieval_then_tokens_then_final() {
+    let (_env, pipeline) = env_with_one_hit("apples are red. [#1]");
+    let (tx, rx) = mpsc::channel::<StreamEvent>();
+    let _ans = pipeline.ask("apples", opts_with_sink(tx)).unwrap();
+    let events: Vec<StreamEvent> = rx.iter().collect();
+
+    // First event must be RetrievalDone.
+    assert!(
+        matches!(events.first(), Some(StreamEvent::RetrievalDone { .. })),
+        "first event must be RetrievalDone, got {:?}",
+        events.first()
+    );
+
+    // Last event must be Final.
+    assert!(
+        matches!(events.last(), Some(StreamEvent::Final { .. })),
+        "last event must be Final, got {:?}",
+        events.last()
+    );
+
+    // Everything in between is Token.
+    for ev in &events[1..events.len() - 1] {
+        assert!(
+            matches!(ev, StreamEvent::Token { .. }),
+            "middle events must be Token, got {ev:?}"
+        );
+    }
+}
+
+#[test]
+fn ask_records_llm_stream_aborted_when_receiver_drops() {
+    let (env, pipeline) = env_with_one_hit("apples are red. [#1]");
+    let (tx, rx) = mpsc::channel::<StreamEvent>();
+    // Drop the receiver immediately so the first Token send fails.
+    drop(rx);
+    let ans = pipeline.ask("apples", opts_with_sink(tx)).unwrap();
+    assert!(!ans.grounded);
+    assert_eq!(ans.refusal_reason, Some(RefusalReason::LlmStreamAborted));
+    // Persistence still happens on cancel — the row is the audit trail.
+    assert_eq!(env.count_answers(), 1, "answers row written on cancel");
+}