feat(expansion): doc-side expansion 별칭 개별 dense 벡터 + 파생물 캐시(V012)

별칭을 줄별 개별 dense 벡터(sentinel `{chunk}#alias#N`)로 색인하고
boilerplate 청크는 별칭 생성을 skip. 묶음 1벡터 방식은 평균화로 특정
표현이 희석돼 오히려 회귀(13/18)했던 것을 폐기. 변형 일관성 14/18 →
16/18, mean_spread@10 0.222 → 0.111 (나무위키 ~1000 문서 CS corpus).
`kebab-core::strip_alias_suffix` 가 suffix 형과 per-alias 형 둘 다 처리.

파생물 캐시(V012): embedding 벡터 + 별칭 LLM 결과를 청크 내용 해시
키로 캐싱해 재색인 시 내용 불변 청크의 재계산을 skip. cache_key =
blake3(kind ‖ text_blake3 ‖ version_key)[:32], version_key 에
model/prompt/dimensions 포함 → §9 cascade 와 정합(버전 bump 시 자동
miss). 측정: 정답 3개 cold 1879s → warm 13s ≈ 145배. 순수 가산이라
corpus_revision bump 없음. search/ask 는 kebab.sqlite+lancedb 만으로
동작 → 외부 서버 색인 후 DB 만 복사하는 이식 워크플로 가능.

V012 schema migration + 신규 surface 로 workspace version 0.20.2 →
0.21.0 (minor) bump. README/HANDOFF/ARCHITECTURE/HOTFIXES sync.
known limitation: stack·svm 설명형 2개 잔존 + grounded 판정이 부분
인용을 grounded 로 오분류(후속 후보).

측정 상세: docs/superpowers/handoffs/2026-05-31-namu-wiki-alias-cache-study.md

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-05-31 08:24:04 +00:00
parent 0282a81c67
commit a8fd76499c
18 changed files with 1000 additions and 71 deletions

View File

@@ -0,0 +1,61 @@
//! Derivation-cache payload encoding helpers (design 2026-05-31 §3.3).
//!
//! - embedding: `dimensions × f32` little-endian bytes (1024×4 = 4096 B/chunk).
//! - alias / korean_tokens: UTF-8 as-is (handled inline by the caller — no
//! helper needed, `String::as_bytes` / `String::from_utf8`).
/// Encode an embedding vector as a little-endian `f32` byte string (§3.3).
pub fn encode_embedding(vector: &[f32]) -> Vec<u8> {
let mut out = Vec::with_capacity(vector.len() * 4);
for &v in vector {
out.extend_from_slice(&v.to_le_bytes());
}
out
}
/// Decode a little-endian `f32` byte string back into a vector (§3.3).
///
/// Returns `None` if the payload length is not a multiple of 4 (corrupt
/// entry) — the caller treats this as a cache miss and recomputes, so a bad
/// payload never produces a wrong vector.
pub fn decode_embedding(payload: &[u8]) -> Option<Vec<f32>> {
if payload.len() % 4 != 0 {
return None;
}
Some(
payload
.chunks_exact(4)
.map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]]))
.collect(),
)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn roundtrips_vector() {
let v = vec![0.0_f32, 1.5, -2.25, 3.125e10, f32::MIN, f32::MAX];
let bytes = encode_embedding(&v);
assert_eq!(bytes.len(), v.len() * 4);
assert_eq!(decode_embedding(&bytes), Some(v));
}
#[test]
fn empty_vector_roundtrips() {
assert_eq!(encode_embedding(&[]), Vec::<u8>::new());
assert_eq!(decode_embedding(&[]), Some(vec![]));
}
#[test]
fn misaligned_payload_is_none() {
assert_eq!(decode_embedding(&[1, 2, 3]), None);
}
#[test]
fn little_endian_layout_is_fixed() {
// 1.0_f32 == 0x3F800000, little-endian bytes [0x00,0x00,0x80,0x3F].
assert_eq!(encode_embedding(&[1.0]), vec![0x00, 0x00, 0x80, 0x3F]);
}
}

View File

@@ -7,6 +7,11 @@ use kebab_core::{Chunk, GenerateRequest, LanguageModel};
/// 별칭 1줄의 최대 글자 수(이 이상은 문장형/환각으로 보고 drop).
const MAX_ALIAS_CHARS: usize = 120;
/// 별칭 프롬프트 템플릿 버전. derivation cache 의 alias version_key 에 포함되어
/// (§3.1), 프롬프트를 바꾸면 bump 해 캐시를 무효화한다(전부 miss → 재생성).
/// `build_request` 의 gemma 프롬프트와 한 쌍 — 프롬프트 수정 시 함께 bump.
pub const PROMPT_VERSION: &str = "expansion-v1";
/// 청크당 검색용 별칭을 생성한다.
///
/// 반환: 검증·상한 적용된 별칭들을 개행 join 한 문자열. 생성 0개 / LLM
@@ -45,6 +50,11 @@ impl<'a> ExpansionGenerator<'a> {
}
pub fn generate(&self, chunk: &Chunk) -> Option<String> {
// 나무위키 네비게이션 boilerplate 청크는 LLM 호출 없이 skip — 별칭
// 생성 가치가 없고 노이즈 sentinel 벡터만 만든다.
if is_nav_boilerplate(chunk) {
return None;
}
let req = Self::build_request(chunk);
let raw = match self.llm.generate_stream(req) {
Ok(iter) => {
@@ -69,6 +79,26 @@ impl<'a> ExpansionGenerator<'a> {
}
}
/// 나무위키 네비게이션 boilerplate 청크 판정.
///
/// heading_path 가 비어 있고(문서 본문 섹션이 아닌 머리/꼬리 nav), text 앞부분에
/// nav 키워드("최근 변경" 등)가 하나라도 있으면 boilerplate 로 본다. 둘 다
/// 만족할 때만 true — 정상 본문(heading 있음, 또는 nav 키워드 없음)은 false.
pub fn is_nav_boilerplate(chunk: &Chunk) -> bool {
const NAV_KEYWORDS: [&str; 5] = [
"최근 변경",
"Recent changes",
"최근 토론",
"특수 기능",
"편집 토론 역사",
];
if !chunk.heading_path.is_empty() {
return false;
}
let head: String = chunk.text.chars().take(200).collect();
NAV_KEYWORDS.iter().any(|kw| head.contains(kw))
}
/// 줄 선두의 목록 마커만 1회 제거한다. **마커 뒤 공백이 필수** — 별칭 내용이
/// 숫자/하이픈/별표로 시작하는 경우(예: "3D 렌더링", "-fast", "2단계")는 보존한다.
/// (Task 4 리뷰 MAJOR-1: 탐욕적 `trim_start_matches` 가 정당한 별칭을 손상시키던 버그 수정.)
@@ -185,6 +215,50 @@ mod tests {
assert_eq!(out, "3D 렌더링\n2단계 커밋\n-fast 플래그\n메모리 안전성\n첫 항목");
}
fn mk_chunk_nav(text: &str, heading: Vec<String>) -> Chunk {
let mut c = mk_chunk(text);
c.heading_path = heading;
c
}
#[test]
fn nav_boilerplate_skips_alias_generation() {
// heading 없음 + nav 키워드 → boilerplate → LLM 호출 전에 None.
let llm = mock("별칭1\n별칭2");
let generator = ExpansionGenerator::new(&llm, 8);
let chunk = mk_chunk_nav("최근 변경 최근 토론 특수 기능", vec![]);
assert_eq!(generator.generate(&chunk), None);
}
#[test]
fn normal_body_chunk_generates_aliases() {
// heading 없지만 nav 키워드도 없음 → 정상 본문 → 별칭 생성.
let llm = mock("별칭1\n별칭2");
let generator = ExpansionGenerator::new(&llm, 8);
let chunk = mk_chunk_nav("러스트의 소유권과 빌림 검사기 개요", vec![]);
assert_eq!(generator.generate(&chunk).unwrap(), "별칭1\n별칭2");
}
#[test]
fn nav_keyword_with_heading_is_not_boilerplate() {
// nav 키워드가 있어도 heading 이 있으면 본문 섹션 → 생성.
let llm = mock("별칭1");
let generator = ExpansionGenerator::new(&llm, 8);
let chunk = mk_chunk_nav("최근 변경 내역 설명", vec!["문서 변경사항".into()]);
assert_eq!(generator.generate(&chunk).unwrap(), "별칭1");
}
#[test]
fn is_nav_boilerplate_unit() {
assert!(is_nav_boilerplate(&mk_chunk_nav("Recent changes list", vec![])));
assert!(is_nav_boilerplate(&mk_chunk_nav("편집 토론 역사", vec![])));
assert!(!is_nav_boilerplate(&mk_chunk_nav("일반 본문 텍스트", vec![])));
assert!(!is_nav_boilerplate(&mk_chunk_nav(
"최근 변경",
vec!["섹션".into()]
)));
}
#[test]
fn strip_list_marker_unit() {
assert_eq!(strip_list_marker("- 메모리"), "메모리");

View File

@@ -59,6 +59,7 @@ use kebab_source_fs::FsSourceConnector;
mod app;
mod bulk;
pub mod cursor;
pub mod derivation_payload;
pub mod doctor_signal;
pub mod error_signal;
pub mod error_wire;
@@ -1057,6 +1058,70 @@ fn unsupported_media_warning(path: &str) -> String {
}
}
/// Embed `texts` with the derivation cache (design 2026-05-31 §3.4).
///
/// 1) 각 text 의 embedding cache_key 계산 → 히트/미스 분리.
/// 2) 미스 text 만 `emb.embed`(축소 배치) 호출.
/// 3) 미스 결과를 `Vec<f32>` little-endian 으로 캐시 put.
/// 4) 히트(bytes→Vec<f32>) + 미스 벡터를 **원래 순서대로** 합쳐 반환.
///
/// 손상된 payload(길이 misalign)는 미스로 강등 → 재계산(정확성 우선, §3.5).
/// 히트 키는 `touch_keys` 에 누적(호출측이 배치로 last_used_at 갱신).
fn embed_with_cache(
emb: &dyn Embedder,
sqlite: &kebab_store_sqlite::SqliteStore,
texts: &[&str],
version_key: &str,
hit: &mut usize,
miss: &mut usize,
touch_keys: &mut Vec<String>,
) -> anyhow::Result<Vec<Vec<f32>>> {
let mut out: Vec<Option<Vec<f32>>> = Vec::with_capacity(texts.len());
let mut miss_indices: Vec<usize> = Vec::new();
let mut miss_inputs: Vec<EmbeddingInput<'_>> = Vec::new();
let mut keys: Vec<String> = Vec::with_capacity(texts.len());
for (i, text) in texts.iter().enumerate() {
let key = kebab_core::derivation_cache_key("embedding", text, version_key);
// 히트 = 캐시에 있고 payload 가 정상 디코드되는 경우. 손상 payload 는
// 미스로 강등(재계산, 정확성 우선 §3.5).
let cached = sqlite
.derivation_cache_get(&key)?
.and_then(|p| crate::derivation_payload::decode_embedding(&p));
if let Some(v) = cached {
*hit += 1;
touch_keys.push(key.clone());
out.push(Some(v));
} else {
*miss += 1;
miss_indices.push(i);
miss_inputs.push(EmbeddingInput {
text,
kind: EmbeddingKind::Document,
});
out.push(None);
}
keys.push(key);
}
if !miss_inputs.is_empty() {
let miss_vectors = emb.embed(&miss_inputs)?;
for (slot, v) in miss_indices.iter().zip(miss_vectors) {
sqlite.derivation_cache_put(
&keys[*slot],
"embedding",
&crate::derivation_payload::encode_embedding(&v),
)?;
out[*slot] = Some(v);
}
}
Ok(out
.into_iter()
.map(|v| v.expect("every slot filled by hit or miss"))
.collect())
}
/// Process a single asset: read bytes, parse, normalize, chunk,
/// persist, embed. Per-asset failures bubble up to the caller for
/// labelling as `IngestItemKind::Error` — they do NOT abort the
@@ -1256,8 +1321,19 @@ fn ingest_one_asset(
.context("kb-chunk::MdHeadingV1Chunker::chunk")?;
// Phase 2 doc-side expansion: flag on 이면 청크당 별칭 생성 (fail-soft).
// derivation cache(§3.4): 같은 청크 text + 같은 alias version_key 면 LLM
// 호출 없이 캐시된 별칭 재사용. version_key = {prompt_version}|{max}|{model}.
let mut alias_cache_hit = 0_usize;
let mut alias_cache_miss = 0_usize;
let mut alias_touch_keys: Vec<String> = Vec::new();
if app.config.ingest.expansion.enabled {
let exp = &app.config.ingest.expansion;
let alias_version_key = format!(
"{}|{}|{}",
crate::expansion::PROMPT_VERSION,
exp.max_aliases_per_chunk,
exp.model
);
let llm_built = if exp.model.is_empty() {
OllamaLanguageModel::new(&app.config)
} else {
@@ -1268,7 +1344,29 @@ fn ingest_one_asset(
let generator =
crate::expansion::ExpansionGenerator::new(&llm, exp.max_aliases_per_chunk);
for chunk in &mut chunks {
chunk.aliases = generator.generate(chunk);
let key = kebab_core::derivation_cache_key(
"alias",
&chunk.text,
&alias_version_key,
);
if let Some(payload) = app.sqlite.derivation_cache_get(&key)? {
// 히트: 저장된 별칭(UTF-8) 재사용. LLM 호출 없음.
chunk.aliases = String::from_utf8(payload).ok();
alias_cache_hit += 1;
alias_touch_keys.push(key);
} else if crate::expansion::is_nav_boilerplate(chunk) {
// 미스지만 nav boilerplate → 생성 가치 없음(기존 skip 규칙).
// 캐시에 넣지 않음(None 은 payload 로 표현 불가, 다음 run 도 동일 판정).
chunk.aliases = None;
} else {
// 미스 → LLM 생성 후 캐시 저장.
chunk.aliases = generator.generate(chunk);
alias_cache_miss += 1;
if let Some(a) = &chunk.aliases {
app.sqlite
.derivation_cache_put(&key, "alias", a.as_bytes())?;
}
}
}
}
Err(e) => {
@@ -1306,21 +1404,30 @@ fn ingest_one_asset(
.context("DocumentStore::put_chunks")?;
// Embed + vector upsert (only when both sides are configured).
let mut emb_cache_hit = 0_usize;
let mut emb_cache_miss = 0_usize;
if let (Some(emb), Some(vec_store)) = (embedder, vector_store) {
if !chunks.is_empty() {
let inputs: Vec<EmbeddingInput<'_>> = chunks
.iter()
.map(|c| EmbeddingInput {
text: c.text.as_str(),
kind: EmbeddingKind::Document,
})
.collect();
let vectors = emb
.embed(&inputs)
.context("Embedder::embed (document chunks)")?;
let model_id = emb.model_id();
let model_version = emb.model_version();
let dimensions = emb.dimensions();
// derivation cache(§3.4): embedding version_key = {model_id}|{model_version}|{dimensions}.
// 본문 청크 + 별칭 문자열 양쪽이 같은 메커니즘(같은 text → 같은 캐시).
let emb_version_key =
format!("{}|{}|{}", model_id.0, model_version.0, dimensions);
let mut emb_touch_keys: Vec<String> = Vec::new();
// 본문 청크 text 로 캐시 조회 → 미스만 embed → 원래 순서로 합침.
let body_texts: Vec<&str> = chunks.iter().map(|c| c.text.as_str()).collect();
let vectors = embed_with_cache(
&**emb,
&app.sqlite,
&body_texts,
&emb_version_key,
&mut emb_cache_hit,
&mut emb_cache_miss,
&mut emb_touch_keys,
)
.context("Embedder::embed (document chunks)")?;
let records: Vec<VectorRecord> = chunks
.iter()
.zip(vectors)
@@ -1350,47 +1457,91 @@ fn ingest_one_asset(
.filter(|c| c.aliases.as_deref().is_some_and(|a| !a.is_empty()))
.collect();
if !alias_chunks.is_empty() {
let alias_inputs: Vec<EmbeddingInput<'_>> = alias_chunks
// 각 별칭을 줄 단위로 분리해 개별 sentinel 벡터로 임베딩한다.
// 묶음 1벡터는 벡터를 희석시켜 효과가 없으므로(측정), 별칭 i
// 마다 chunk_id `{orig}#alias#{i}` 의 VectorRecord 를 만든다.
// `(청크 참조, 별칭 문자열)` 쌍을 평탄화한 뒤 한 번에 임베딩.
let alias_lines: Vec<(&kebab_core::Chunk, &str)> = alias_chunks
.iter()
.map(|c| EmbeddingInput {
text: c.aliases.as_deref().unwrap(),
kind: EmbeddingKind::Document,
.flat_map(|c| {
c.aliases
.as_deref()
.unwrap()
.split('\n')
.map(str::trim)
.filter(|line| !line.is_empty())
.map(move |line| (*c, line))
})
.collect();
let alias_vectors = emb
.embed(&alias_inputs)
if !alias_lines.is_empty() {
// 별칭 dense 벡터도 본문과 동일한 embedding 캐시 재사용:
// 같은 별칭 문자열이면 본문 embedding 캐시와 같은 키로 적중(§3.4).
let alias_texts: Vec<&str> =
alias_lines.iter().map(|(_, line)| *line).collect();
let alias_vectors = embed_with_cache(
&**emb,
&app.sqlite,
&alias_texts,
&emb_version_key,
&mut emb_cache_hit,
&mut emb_cache_miss,
&mut emb_touch_keys,
)
.context("Embedder::embed (alias vectors)")?;
for (c, v) in alias_chunks.iter().zip(alias_vectors) {
let alias_chunk_id = kebab_core::ChunkId(format!(
"{}{}",
c.chunk_id.0,
kebab_core::ALIAS_SUFFIX
));
all_records.push(VectorRecord {
embedding_id: kebab_core::id_for_embedding(
&alias_chunk_id,
&model_id,
&model_version,
// 같은 청크 안에서 별칭 인덱스를 0부터 매긴다.
let mut per_chunk_idx: std::collections::HashMap<String, usize> =
std::collections::HashMap::new();
for ((c, line), v) in alias_lines.iter().zip(alias_vectors) {
let i = per_chunk_idx.entry(c.chunk_id.0.clone()).or_insert(0);
let alias_chunk_id = kebab_core::ChunkId(format!(
"{}{}#{}",
c.chunk_id.0,
kebab_core::ALIAS_SUFFIX,
*i
));
*i += 1;
all_records.push(VectorRecord {
embedding_id: kebab_core::id_for_embedding(
&alias_chunk_id,
&model_id,
&model_version,
dimensions,
),
chunk_id: alias_chunk_id,
vector: v,
doc_id: canonical.doc_id.clone(),
text: (*line).to_string(),
heading_path: c.heading_path.clone(),
model_id: model_id.clone(),
model_version: model_version.clone(),
dimensions,
),
chunk_id: alias_chunk_id,
vector: v,
doc_id: canonical.doc_id.clone(),
text: c.aliases.clone().unwrap_or_default(),
heading_path: c.heading_path.clone(),
model_id: model_id.clone(),
model_version: model_version.clone(),
dimensions,
});
});
}
}
}
}
vec_store
.upsert(&all_records)
.context("VectorStore::upsert")?;
// 히트한 embedding 키들의 last_used_at 갱신(LRU 보존, §3.5).
app.sqlite.derivation_cache_touch(&emb_touch_keys)?;
}
}
// 히트한 alias 키들의 last_used_at 갱신(LRU 보존, §3.5).
app.sqlite.derivation_cache_touch(&alias_touch_keys)?;
// 검증용 hit/miss 카운트 노출(§3.4 / §6): warm 재색인이 LLM·embed 0회임을
// 로그로 확인. tracing target 은 stderr 로 흐른다.
if alias_cache_hit + alias_cache_miss + emb_cache_hit + emb_cache_miss > 0 {
tracing::info!(
target: "kebab-app",
doc = %canonical.doc_id.0,
"derivation cache: embedding hit={emb_cache_hit} miss={emb_cache_miss}, \
alias hit={alias_cache_hit} miss={alias_cache_miss}"
);
}
let kind = if existing_doc_ids.contains(&canonical.doc_id.0) {
kebab_core::IngestItemKind::Updated
} else {

View File

@@ -109,10 +109,11 @@ fn first_ingest_bumps_corpus_revision() {
let env = TestEnv::lexical_only();
let store_before = kebab_store_sqlite::SqliteStore::open(&env.config).unwrap();
store_before.run_migrations().unwrap();
// V004 seeds 0; V009 + V010 migrations each bump by 1 to invalidate
// stale LRU caches (spec §5.2). Baseline before ingest = 2.
// V004 seeds 0; V009 + V010 + V011 migrations each bump by 1 to
// invalidate stale LRU caches (spec §5.2). Baseline before ingest = 3.
// (V012 derivation_cache is purely additive — does NOT bump.)
let baseline = store_before.corpus_revision();
assert_eq!(baseline, 2, "fresh store post-V010 baseline = 2");
assert_eq!(baseline, 3, "fresh store post-V011 baseline = 3");
let report = kebab_app::ingest_with_config(env.config.clone(), env.scope(), true).unwrap();
assert!(

View File

@@ -0,0 +1,110 @@
//! Content-hash derivation cache key (design 2026-05-31 §3.1).
//!
//! Expensive ingest derivations (embedding vectors, LLM aliases, optional
//! Korean morphological tokens) are cached by the *content hash* of the chunk
//! text so that re-indexing an updated document skips recomputation for any
//! chunk whose text is unchanged — independent of position / `chunk_id`
//! (which is position-based, see `ids::id_for_block`).
//!
//! ```text
//! cache_key = blake3_hex( kind || 0x00 || text_blake3 || 0x00 || version_key )[:32]
//! ```
//! - `text_blake3` = blake3(NFC-normalized UTF-8 bytes of the chunk text).
//! - `kind` ∈ { "embedding", "alias", "korean_tokens" }.
//! - `version_key` folds every §9 version-cascade input for that kind
//! (model / prompt / tokenizer version). A version bump changes the key →
//! automatic cache miss → recompute, keeping the cache consistent with the
//! cascade contract (§3.5 / §3.6).
//!
//! Pure: depends only on `blake3` + `unicode-normalization`. No other
//! `kebab-*` crate is referenced (deps boundary §5).
use crate::normalize::nfc;
/// Derivation-cache key per design §3.1.
///
/// `text` is NFC-normalized before hashing so the same logical content always
/// maps to the same key regardless of Unicode encoding form. `kind` and
/// `version_key` are folded in with `0x00` separators (which cannot occur in
/// hex digests) so distinct kinds / versions never collide.
pub fn derivation_cache_key(kind: &str, text: &str, version_key: &str) -> String {
let text_blake3 = blake3::hash(nfc(text).as_bytes()).to_hex().to_string();
let mut hasher = blake3::Hasher::new();
hasher.update(kind.as_bytes());
hasher.update(&[0x00]);
hasher.update(text_blake3.as_bytes());
hasher.update(&[0x00]);
hasher.update(version_key.as_bytes());
hasher.finalize().to_hex().to_string()[..32].to_string()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn key_is_32_hex_chars() {
let k = derivation_cache_key("embedding", "hello world", "v1");
assert_eq!(k.len(), 32);
assert!(k.bytes().all(|b| b.is_ascii_hexdigit()));
}
#[test]
fn same_inputs_same_key() {
let a = derivation_cache_key("embedding", "러스트 소유권", "model|1|1024");
let b = derivation_cache_key("embedding", "러스트 소유권", "model|1|1024");
assert_eq!(a, b);
}
#[test]
fn nfc_normalization_collapses_encoding_forms() {
// "가" as a precomposed syllable (NFC) vs decomposed jamo (NFD) must
// hash to the same key after NFC normalization.
let precomposed = "\u{AC00}"; // 가
let decomposed = "\u{1100}\u{1161}"; // ᄀ + ᅡ
assert_ne!(precomposed, decomposed);
let a = derivation_cache_key("embedding", precomposed, "v1");
let b = derivation_cache_key("embedding", decomposed, "v1");
assert_eq!(a, b);
}
#[test]
fn different_kind_different_key() {
let e = derivation_cache_key("embedding", "same text", "v1");
let a = derivation_cache_key("alias", "same text", "v1");
assert_ne!(e, a);
}
#[test]
fn different_version_key_different_key_miss() {
// §3.6 correctness guard: a version_key change MUST produce a different
// cache_key (so a stale derivation never gets reused after a cascade
// bump). This is the most safety-critical invariant of the cache.
let v1 = derivation_cache_key("embedding", "same text", "modelA|1|1024");
let v2 = derivation_cache_key("embedding", "same text", "modelA|2|1024");
assert_ne!(v1, v2);
// alias prompt_version bump → miss.
let p1 = derivation_cache_key("alias", "문단", "expansion-v1|8|");
let p2 = derivation_cache_key("alias", "문단", "expansion-v2|8|");
assert_ne!(p1, p2);
}
#[test]
fn different_text_different_key() {
let a = derivation_cache_key("embedding", "text one", "v1");
let b = derivation_cache_key("embedding", "text two", "v1");
assert_ne!(a, b);
}
#[test]
fn separator_prevents_field_smearing() {
// Without the 0x00 separators, ("ab","","c") and ("a","b","c") shaped
// inputs could collide. The kind/version boundaries must be distinct.
let a = derivation_cache_key("ab", "x", "c");
let b = derivation_cache_key("a", "x", "bc");
assert_ne!(a, b);
}
}

View File

@@ -61,10 +61,18 @@ fn validate_hex32(s: &str) -> Result<(), CoreError> {
/// Suffix appended to a chunk's vector ID to mark an alias embedding row.
pub const ALIAS_SUFFIX: &str = "#alias";
/// Strip `#alias` suffix from `id`, returning the bare chunk ID.
/// If `id` does not end with `ALIAS_SUFFIX`, returns `id` unchanged.
/// Strip the alias marker from `id`, returning the bare chunk ID.
///
/// Returns everything before the first occurrence of `ALIAS_SUFFIX`. This
/// handles both the suffix form `{orig}#alias` and the per-alias form
/// `{orig}#alias#N`. A bare chunk ID is blake3 hex (32 chars, no `#`), so the
/// first `#alias` always marks the boundary. If `id` contains no `ALIAS_SUFFIX`,
/// returns `id` unchanged.
pub fn strip_alias_suffix(id: &str) -> &str {
id.strip_suffix(ALIAS_SUFFIX).unwrap_or(id)
match id.find(ALIAS_SUFFIX) {
Some(pos) => &id[..pos],
None => id,
}
}
/// Canonical-JSON + blake3 + hex prefix 32. Per design §4.2.
@@ -447,6 +455,10 @@ mod tests {
assert_eq!(strip_alias_suffix(bare), bare);
assert_eq!(strip_alias_suffix(""), "");
assert_eq!(strip_alias_suffix("#alias"), "");
// Per-alias form `{orig}#alias#N` strips to the bare chunk ID.
assert_eq!(strip_alias_suffix(&format!("{bare}{ALIAS_SUFFIX}#3")), bare);
assert_eq!(strip_alias_suffix(&format!("{bare}{ALIAS_SUFFIX}#0")), bare);
assert_eq!(strip_alias_suffix("#alias#3"), "");
}
/// Independent pin for id_for_index.

View File

@@ -11,6 +11,7 @@ pub mod answer;
pub mod asset;
pub mod chunk;
pub mod citation;
pub mod derivation;
pub mod document;
pub mod errors;
pub mod fetch;
@@ -35,6 +36,7 @@ pub use answer::{
pub use asset::{AssetStorage, RawAsset, SourceUri, WorkspacePath};
pub use chunk::Chunk;
pub use citation::Citation;
pub use derivation::derivation_cache_key;
pub use document::{
AudioRefBlock, Block, CanonicalDocument, CodeBlock, CommonBlock, HeadingBlock, ImageRefBlock,
Inline, ListBlock, ModelCaption, OcrRegion, OcrText, SourceSpan, TableBlock, TextBlock,

View File

@@ -0,0 +1,192 @@
//! Content-hash derivation cache store (design 2026-05-31 §3.2 / §3.5).
//!
//! Backs the `derivation_cache` table (`V012`). The cache stores expensive
//! ingest derivations (embedding vectors, LLM aliases, optional Korean
//! tokens) keyed by `derivation_cache_key` (§3.1). It is a pure performance
//! layer: corruption / deletion only forces recomputation, never wrong
//! results (§3.5). Timestamps follow the same RFC3339 `OffsetDateTime`
//! formatting the asset / document / embedding writers use.
use anyhow::{Context, Result};
use rusqlite::{OptionalExtension, params};
use time::OffsetDateTime;
use time::format_description::well_known::Rfc3339;
use crate::error::StoreError;
use crate::store::SqliteStore;
impl SqliteStore {
/// Look up a cached derivation payload by its content-hash key.
///
/// Pure read — does **not** bump `last_used_at`. Callers that want LRU
/// freshness on a hit collect the hit keys and call [`Self::touch`] once
/// per batch (cheaper than a write per `get`).
pub fn derivation_cache_get(&self, cache_key: &str) -> Result<Option<Vec<u8>>> {
let conn = self.lock_conn();
let payload: Option<Vec<u8>> = conn
.query_row(
"SELECT payload FROM derivation_cache WHERE cache_key = ?",
params![cache_key],
|row| row.get::<_, Vec<u8>>(0),
)
.optional()
.map_err(StoreError::from)
.context("derivation_cache_get")?;
Ok(payload)
}
/// Insert (or overwrite) a cached derivation payload.
///
/// `INSERT OR REPLACE` so a re-computation of the same key (e.g. after a
/// manual cache clear, or a non-deterministic LLM regenerating) refreshes
/// `created_at` / `last_used_at` to the new attempt. The key already folds
/// every version-cascade input (§3.1), so an overwrite is always the same
/// logical derivation.
pub fn derivation_cache_put(&self, cache_key: &str, kind: &str, payload: &[u8]) -> Result<()> {
let now = OffsetDateTime::now_utc()
.format(&Rfc3339)
.context("format derivation_cache.created_at")?;
let conn = self.lock_conn();
conn.execute(
"INSERT OR REPLACE INTO derivation_cache
(cache_key, kind, payload, created_at, last_used_at)
VALUES (?, ?, ?, ?, ?)",
params![cache_key, kind, payload, now, now],
)
.map_err(StoreError::from)
.context("derivation_cache_put")?;
Ok(())
}
/// Bump `last_used_at` for the given hit keys (LRU freshness, §3.5).
///
/// Run in a single transaction. Missing keys are a no-op. Called once per
/// ingest batch with the keys that hit, so the GC pass keeps live chunks.
pub fn derivation_cache_touch(&self, keys: &[String]) -> Result<()> {
if keys.is_empty() {
return Ok(());
}
let now = OffsetDateTime::now_utc()
.format(&Rfc3339)
.context("format derivation_cache.last_used_at")?;
let mut conn = self.lock_conn();
let tx = conn.transaction().map_err(StoreError::from)?;
{
let mut stmt = tx
.prepare("UPDATE derivation_cache SET last_used_at = ? WHERE cache_key = ?")
.map_err(StoreError::from)?;
for key in keys {
stmt.execute(params![now, key])
.map_err(StoreError::from)
.context("derivation_cache_touch")?;
}
}
tx.commit().map_err(StoreError::from)?;
Ok(())
}
/// Delete cache entries whose `last_used_at` is older than `ttl_days`
/// (§3.5 lightweight GC). Returns the number of rows removed.
///
/// `ttl_days <= 0` is a no-op guard (never wipe the whole cache by an
/// accidental zero TTL).
pub fn derivation_cache_gc(&self, ttl_days: i64) -> Result<usize> {
if ttl_days <= 0 {
return Ok(0);
}
let cutoff = (OffsetDateTime::now_utc() - time::Duration::days(ttl_days))
.format(&Rfc3339)
.context("format derivation_cache gc cutoff")?;
let conn = self.lock_conn();
let removed = conn
.execute(
"DELETE FROM derivation_cache WHERE last_used_at < ?",
params![cutoff],
)
.map_err(StoreError::from)
.context("derivation_cache_gc")?;
Ok(removed)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::store::SqliteStore;
fn open_store() -> (tempfile::TempDir, SqliteStore) {
let dir = tempfile::tempdir().unwrap();
let mut cfg = kebab_config::Config::defaults();
cfg.storage.data_dir = dir.path().to_string_lossy().into_owned();
let store = SqliteStore::open(&cfg).unwrap();
store.run_migrations().unwrap();
(dir, store)
}
#[test]
fn put_then_get_roundtrips() {
let (_d, store) = open_store();
store
.derivation_cache_put("key1", "embedding", &[1, 2, 3, 4])
.unwrap();
let got = store.derivation_cache_get("key1").unwrap();
assert_eq!(got, Some(vec![1, 2, 3, 4]));
}
#[test]
fn get_miss_returns_none() {
let (_d, store) = open_store();
assert_eq!(store.derivation_cache_get("absent").unwrap(), None);
}
#[test]
fn put_replaces_existing() {
let (_d, store) = open_store();
store.derivation_cache_put("k", "alias", b"old").unwrap();
store.derivation_cache_put("k", "alias", b"new").unwrap();
assert_eq!(
store.derivation_cache_get("k").unwrap(),
Some(b"new".to_vec())
);
}
#[test]
fn touch_missing_keys_is_noop() {
let (_d, store) = open_store();
store
.derivation_cache_touch(&["nope".to_string()])
.unwrap();
assert_eq!(store.derivation_cache_get("nope").unwrap(), None);
}
#[test]
fn gc_zero_ttl_is_noop() {
let (_d, store) = open_store();
store.derivation_cache_put("k", "embedding", b"x").unwrap();
assert_eq!(store.derivation_cache_gc(0).unwrap(), 0);
assert!(store.derivation_cache_get("k").unwrap().is_some());
}
#[test]
fn gc_removes_stale_entries() {
let (_d, store) = open_store();
store.derivation_cache_put("fresh", "embedding", b"x").unwrap();
// Backdate one row by 100 days via a direct UPDATE.
let old = (OffsetDateTime::now_utc() - time::Duration::days(100))
.format(&Rfc3339)
.unwrap();
{
let conn = store.lock_conn();
conn.execute(
"INSERT INTO derivation_cache (cache_key, kind, payload, created_at, last_used_at)
VALUES ('stale', 'embedding', ?, ?, ?)",
params![&b"y"[..], &old, &old],
)
.unwrap();
}
let removed = store.derivation_cache_gc(30).unwrap();
assert_eq!(removed, 1);
assert!(store.derivation_cache_get("stale").unwrap().is_none());
assert!(store.derivation_cache_get("fresh").unwrap().is_some());
}
}

View File

@@ -19,6 +19,7 @@
mod answers;
mod chat_sessions;
mod derivation_cache;
mod documents;
mod embeddings;
mod error;