altair823-org/kebab
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 40 Wiki Activity

feat(p10-1d): code-c-ast-v1 chunker + snapshot test

Browse Source 
Mirrors code-go-ast-v1's chunker pattern. Snapshot test against
tests/fixtures/sample.c (function + typedef struct + typedef enum +
preprocessor) verifies symbol list + lang=c stamping.

Chunks produced (4 total):
- <top-level> glue: includes, defines, static vars, typedefs (lines 1-18)
- parse_record function (lines 20-23)
- print_record function (lines 25-27)
- main function (lines 29-33)

All chunks stamped with lang=c and chunker_version=code-c-ast-v1.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
altair823
2026-05-21 13:41:19 +00:00
parent 926042049c
commit 03cd41c48f
5 changed files with 639 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

196
crates/kebab-chunk/tests/code_c_ast_snapshot.rs Normal file
View File

@@ -0,0 +1,196 @@
//! Snapshot test pinning the `Vec<Chunk>` JSON for a
//! representative C code `CanonicalDocument`.
//!
//! This is an integration test. `kebab-parse-code` is intentionally NOT
//! a dev-dep (design §6.3 / §8 boundary: AST extraction is parser-side).
//! The `CanonicalDocument` is built inline from hand-crafted `Block::Code`
//! units, which is the same pattern used in `code_go_ast_v1.rs`'s
//! internal `code_doc` test helper.
//!
//! Set `UPDATE_SNAPSHOTS=1` to re-bake the baseline.
use std::path::PathBuf;
use kebab_chunk::CodeCAstV1Chunker;
use kebab_core::{
AssetId, Block, CanonicalDocument, ChunkPolicy, Chunker, ChunkerVersion, CodeBlock, CommonBlock,
Lang, Metadata, ParserVersion, Provenance, SourceSpan, SourceType, TrustLevel, WorkspacePath,
id_for_block, id_for_doc,
};
use serde_json::Value;
use time::OffsetDateTime;
fn fixtures_dir() -> PathBuf {
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("tests")
.join("fixtures")
}
fn fixed_doc() -> CanonicalDocument {
let wp = WorkspacePath("projects/record.c".into());
let aid = AssetId("c".repeat(64));
// Pin parser_version so doc_id / block_ids are reproducible.
let pv = ParserVersion("code-c-v1".into());
let doc_id = id_for_doc(&wp, &aid, &pv);
// Representative units:
// 0. imports + defines (lines 1–4, ≤200)
// 1. status_t enum typedef (lines 6–9, ≤200)
// 2. record_t struct typedef (lines 11–16, ≤200)
// 3. static counter decl glue (line 18, ≤200)
// 4. parse_record fn (lines 20–23, ≤200)
// 5. print_record fn (lines 25–27, ≤200)
// 6. main fn (lines 29–33, ≤200)
let raw_units: Vec<(&str, u32, u32, String)> = vec![
(
"<top-level>",
1,
18,
"#include <stdio.h>\n#include <stdlib.h>\n\n#define MAX_BUF 4096\n\ntypedef enum {\n OK = 0,\n ERR_PARSE,\n ERR_IO,\n} status_t;\n\ntypedef struct {\n int id;\n char name[64];\n status_t status;\n} record_t;\n\nstatic int counter = 0;".to_string(),
),
(
"parse_record",
20,
23,
"int parse_record(const char *line, record_t *out) {\n if (line == NULL || out == NULL) return ERR_PARSE;\n return OK;\n}".to_string(),
),
(
"print_record",
25,
27,
"void print_record(const record_t *r) {\n printf(\"[%d] %s (status=%d)\\n\", r->id, r->name, r->status);\n}".to_string(),
),
(
"main",
29,
33,
"int main(void) {\n record_t r = { .id = 1, .name = \"foo\", .status = OK };\n print_record(&r);\n return 0;\n}".to_string(),
),
];
let blocks: Vec<Block> = raw_units
.iter()
.enumerate()
.map(|(i, (sym, ls, le, code))| {
let span = SourceSpan::Code {
line_start: *ls,
line_end: *le,
symbol: Some((*sym).to_string()),
lang: Some("c".into()),
};
let bid = id_for_block(&doc_id, "code", &[], i as u32, &span);
Block::Code(CodeBlock {
common: CommonBlock {
block_id: bid,
heading_path: vec![],
source_span: span,
},
lang: Some("c".into()),
code: code.clone(),
})
})
.collect();
CanonicalDocument {
doc_id,
source_asset_id: aid,
workspace_path: wp,
title: "record.c".into(),
lang: Lang("und".into()),
blocks,
metadata: Metadata {
aliases: vec![],
tags: vec![],
created_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
updated_at: OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap(),
source_type: SourceType::Note,
trust_level: TrustLevel::Primary,
user_id_alias: None,
user: Default::default(),
repo: Some("kebab".into()),
git_branch: Some("main".into()),
git_commit: Some("0".repeat(40)),
code_lang: Some("c".into()),
},
provenance: Provenance { events: vec![] },
parser_version: pv,
schema_version: 1,
doc_version: 1,
last_chunker_version: None,
last_embedding_version: None,
}
}
fn fixed_policy() -> ChunkPolicy {
ChunkPolicy {
target_tokens: 500,
overlap_tokens: 80,
respect_markdown_headings: false,
chunker_version: ChunkerVersion("code-c-ast-v1".into()),
}
}
#[test]
fn code_c_ast_chunks_snapshot() {
let doc = fixed_doc();
let policy = fixed_policy();
let chunks = CodeCAstV1Chunker.chunk(&doc, &policy).expect("chunk");
let actual = serde_json::to_value(&chunks).unwrap();
let dir = fixtures_dir();
let baseline_path = dir.join("code-sample.c.chunks.snapshot.json");
let baseline_text = match std::fs::read_to_string(&baseline_path) {
Ok(s) => s,
Err(_) if std::env::var("UPDATE_SNAPSHOTS").is_ok() => {
std::fs::create_dir_all(&dir).unwrap();
let pretty = serde_json::to_string_pretty(&actual).unwrap();
std::fs::write(&baseline_path, format!("{pretty}\n")).unwrap();
return;
}
Err(e) => panic!(
"missing baseline {}; run with UPDATE_SNAPSHOTS=1 to create: {e}",
baseline_path.display()
),
};
let expected: Value = serde_json::from_str(&baseline_text).expect("baseline parses as json");
if actual != expected {
if std::env::var("UPDATE_SNAPSHOTS").is_ok() {
let pretty = serde_json::to_string_pretty(&actual).unwrap();
std::fs::write(&baseline_path, format!("{pretty}\n")).unwrap();
eprintln!("updated baseline {}", baseline_path.display());
return;
}
let pretty = serde_json::to_string_pretty(&actual).unwrap();
panic!(
"code-c-ast-v1 chunks snapshot drift\n\
--- expected ({}) ---\n{baseline_text}\n\
--- actual ---\n{pretty}\n\
If intentional, re-run with UPDATE_SNAPSHOTS=1.",
baseline_path.display()
);
}
}
/// Determinism cross-check: re-running the same pipeline yields the same
/// chunk_ids byte-for-byte.
#[test]
fn code_c_ast_chunks_are_deterministic() {
let policy = fixed_policy();
let baseline: Vec<String> = CodeCAstV1Chunker
.chunk(&fixed_doc(), &policy)
.unwrap()
.into_iter()
.map(|c| c.chunk_id.0)
.collect();
for _ in 0..5 {
let again: Vec<String> = CodeCAstV1Chunker
.chunk(&fixed_doc(), &policy)
.unwrap()
.into_iter()
.map(|c| c.chunk_id.0)
.collect();
assert_eq!(again, baseline);
}
}

86
crates/kebab-chunk/tests/fixtures/code-sample.c.chunks.snapshot.json vendored Normal file
View File

@@ -0,0 +1,86 @@
[
{
"block_ids": [
"8149e12ca002489acb4a0f74c97a061a"
],
"chunk_id": "ec3cf06ae56c8e9796bbc9196438b7c5",
"chunker_version": "code-c-ast-v1",
"doc_id": "6bec42dd593920a060541db16c4e8e45",
"heading_path": [],
"policy_hash": "ecfad2ec1223662d",
"source_spans": [
{
"kind": "code",
"lang": "c",
"line_end": 18,
"line_start": 1,
"symbol": "<top-level>"
}
],
"text": "#include <stdio.h>\n#include <stdlib.h>\n\n#define MAX_BUF 4096\n\ntypedef enum {\n OK = 0,\n ERR_PARSE,\n ERR_IO,\n} status_t;\n\ntypedef struct {\n int id;\n char name[64];\n status_t status;\n} record_t;\n\nstatic int counter = 0;",
"token_estimate": 78
},
{
"block_ids": [
"1baaa89f21a47b2f32d6396a24a85454"
],
"chunk_id": "c2d7a81c898106733ef2e703774a6a4a",
"chunker_version": "code-c-ast-v1",
"doc_id": "6bec42dd593920a060541db16c4e8e45",
"heading_path": [],
"policy_hash": "ecfad2ec1223662d",
"source_spans": [
{
"kind": "code",
"lang": "c",
"line_end": 23,
"line_start": 20,
"symbol": "parse_record"
}
],
"text": "int parse_record(const char *line, record_t *out) {\n if (line == NULL || out == NULL) return ERR_PARSE;\n return OK;\n}",
"token_estimate": 41
},
{
"block_ids": [
"8d0e14cbcc6d1e92d7878ab796ea68b8"
],
"chunk_id": "0e4d7b131ab64eba03b51903b5d8f96d",
"chunker_version": "code-c-ast-v1",
"doc_id": "6bec42dd593920a060541db16c4e8e45",
"heading_path": [],
"policy_hash": "ecfad2ec1223662d",
"source_spans": [
{
"kind": "code",
"lang": "c",
"line_end": 27,
"line_start": 25,
"symbol": "print_record"
}
],
"text": "void print_record(const record_t *r) {\n printf(\"[%d] %s (status=%d)\\n\", r->id, r->name, r->status);\n}",
"token_estimate": 35
},
{
"block_ids": [
"9c2ede84423871b615d48c38fefb1853"
],
"chunk_id": "e076f8edb2ff141d7e99b4106bb95157",
"chunker_version": "code-c-ast-v1",
"doc_id": "6bec42dd593920a060541db16c4e8e45",
"heading_path": [],
"policy_hash": "ecfad2ec1223662d",
"source_spans": [
{
"kind": "code",
"lang": "c",
"line_end": 33,
"line_start": 29,
"symbol": "main"
}
],
"text": "int main(void) {\n record_t r = { .id = 1, .name = \"foo\", .status = OK };\n print_record(&r);\n return 0;\n}",
"token_estimate": 38
}
]

33
crates/kebab-chunk/tests/fixtures/sample.c vendored Normal file
View File

@@ -0,0 +1,33 @@
#include <stdio.h>
#include <stdlib.h>
#define MAX_BUF 4096
typedef enum {
OK = 0,
ERR_PARSE,
ERR_IO,
} status_t;
typedef struct {
int id;
char name[64];
status_t status;
} record_t;
static int counter = 0;
int parse_record(const char *line, record_t *out) {
if (line == NULL || out == NULL) return ERR_PARSE;
return OK;
}
void print_record(const record_t *r) {
printf("[%d] %s (status=%d)\n", r->id, r->name, r->status);
}
int main(void) {
record_t r = { .id = 1, .name = "foo", .status = OK };
print_record(&r);
return 0;
}