feat(p10-1a-1): add IngestReport skip counters + SkipExamples

Adds five new u32 counters (skipped_gitignore, skipped_kebabignore,
skipped_builtin_blacklist, skipped_generated, skipped_size_exceeded)
and a SkipExamples struct (≤5 sample paths per category) to
IngestReport. All new fields are #[serde(default)] for backward-compat
deserialization. Downstream literal construction sites patched with
zeros/empty; snapshot re-baked.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
th-kim0823
2026-05-15 15:28:19 +09:00
parent 7329ba96ee
commit 351c7a0826
6 changed files with 120 additions and 3 deletions

View File

@@ -44,7 +44,7 @@ use kebab_core::{
Answer, Block, CanonicalDocument, Chunk, ChunkId, ChunkPolicy, ChunkerVersion, Chunker,
DocFilter, DocSummary, DocumentId, DocumentStore, Embedder, EmbeddingInput,
EmbeddingKind, ExtractContext, Extractor, IngestReport, Lang, LanguageModel, MediaType,
ParserVersion, RawAsset, SearchHit, SearchQuery, SourceConnector, SourceScope,
ParserVersion, RawAsset, SearchHit, SearchQuery, SkipExamples, SourceConnector, SourceScope,
SourceUri, VectorRecord, VectorStore,
};
use kebab_llm_local::OllamaLanguageModel;
@@ -675,6 +675,12 @@ pub fn ingest_with_config_opts(
errors: error_count,
duration_ms,
skipped_by_extension,
skipped_gitignore: 0,
skipped_kebabignore: 0,
skipped_builtin_blacklist: 0,
skipped_generated: 0,
skipped_size_exceeded: 0,
skip_examples: SkipExamples::default(),
items: if summary_only { None } else { Some(items) },
})
}

View File

@@ -239,7 +239,7 @@ mod tests {
#[test]
fn ingest_wrapper_tags_schema_version() {
use kebab_core::SourceScope;
use kebab_core::{SkipExamples, SourceScope};
let r = IngestReport {
scope: SourceScope {
root: std::path::PathBuf::from("/tmp"),
@@ -254,6 +254,12 @@ mod tests {
errors: 0,
duration_ms: 0,
skipped_by_extension: std::collections::BTreeMap::new(),
skipped_gitignore: 0,
skipped_kebabignore: 0,
skipped_builtin_blacklist: 0,
skipped_generated: 0,
skipped_size_exceeded: 0,
skip_examples: SkipExamples::default(),
items: None,
};
let v = wire_ingest(&r);

View File

@@ -25,10 +25,46 @@ pub struct IngestReport {
/// extension key under "<no-ext>". `BTreeMap` so the wire JSON
/// has stable key order across runs.
pub skipped_by_extension: std::collections::BTreeMap<String, u32>,
/// p10-1A-1: files skipped because they matched a repo-local `.gitignore`.
#[serde(default)]
pub skipped_gitignore: u32,
/// p10-1A-1: files skipped because they matched a `.kebabignore` entry.
#[serde(default)]
pub skipped_kebabignore: u32,
/// p10-1A-1: files skipped because they matched the built-in safety-net
/// blacklist (`node_modules/`, `target/`, `__pycache__/`, `.venv/`,
/// `venv/`, `env/`).
#[serde(default)]
pub skipped_builtin_blacklist: u32,
/// p10-1A-1: files skipped because their first ~512 bytes contained a
/// generated-file marker (`@generated`, `do not edit`, …).
#[serde(default)]
pub skipped_generated: u32,
/// p10-1A-1: files skipped because they exceeded `max_file_bytes` or
/// `max_file_lines` in `[ingest.code]`.
#[serde(default)]
pub skipped_size_exceeded: u32,
/// p10-1A-1: sample file paths per skip category (≤ 5 each).
#[serde(default)]
pub skip_examples: SkipExamples,
/// `None` ↔ wire `items: null` (`--summary-only`).
pub items: Option<Vec<IngestItem>>,
}
/// p10-1A-1: per-category sample of skipped file paths. Each category caps at
/// 5 entries (oldest-first). Used for debugging "why was X not indexed?"
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
pub struct SkipExamples {
#[serde(default)]
pub generated: Vec<String>,
#[serde(default)]
pub size_exceeded: Vec<String>,
#[serde(default)]
pub builtin_blacklist: Vec<String>,
#[serde(default)]
pub gitignore: Vec<String>,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct IngestItem {
pub kind: IngestItemKind,
@@ -58,3 +94,55 @@ pub enum IngestItemKind {
Unchanged,
Error,
}
#[cfg(test)]
mod tests {
use super::*;
use crate::traits::SourceScope;
#[test]
fn skip_examples_default_is_empty() {
let s = SkipExamples::default();
assert!(s.generated.is_empty());
assert!(s.size_exceeded.is_empty());
assert!(s.builtin_blacklist.is_empty());
assert!(s.gitignore.is_empty());
}
#[test]
fn ingest_report_skip_counters_serialize() {
let r = IngestReport {
scope: SourceScope {
root: std::path::PathBuf::from("/tmp"),
include: vec![],
exclude: vec![],
},
scanned: 100,
new: 50,
updated: 0,
skipped: 0,
unchanged: 0,
errors: 0,
duration_ms: 1234,
skipped_by_extension: Default::default(),
skipped_gitignore: 30,
skipped_kebabignore: 5,
skipped_builtin_blacklist: 10,
skipped_generated: 3,
skipped_size_exceeded: 2,
skip_examples: SkipExamples {
generated: vec!["a/b.pb.rs".into()],
size_exceeded: vec![],
builtin_blacklist: vec!["node_modules/x.js".into()],
gitignore: vec![],
},
items: None,
};
let v = serde_json::to_value(&r).unwrap();
assert_eq!(v["skipped_gitignore"], 30);
assert_eq!(v["skipped_builtin_blacklist"], 10);
assert_eq!(v["skipped_generated"], 3);
assert_eq!(v["skipped_size_exceeded"], 2);
assert_eq!(v["skip_examples"]["generated"][0], "a/b.pb.rs");
}
}

View File

@@ -59,7 +59,7 @@ pub use answer::{
Answer, AnswerCitation, AnswerRetrievalSummary, ModelRef, RefusalReason, TokenUsage,
TraceId, Turn,
};
pub use ingest::{IngestItem, IngestItemKind, IngestReport};
pub use ingest::{IngestItem, IngestItemKind, IngestReport, SkipExamples};
pub use jobs::{JobFilter, JobId, JobKind, JobRow, JobStatus};
pub use vector::{VectorHit, VectorRecord};
pub use errors::CoreError;

View File

@@ -42,8 +42,19 @@
],
"root": "/home/u/KB"
},
"skip_examples": {
"builtin_blacklist": [],
"generated": [],
"gitignore": [],
"size_exceeded": []
},
"skipped": 0,
"skipped_builtin_blacklist": 0,
"skipped_by_extension": {},
"skipped_generated": 0,
"skipped_gitignore": 0,
"skipped_kebabignore": 0,
"skipped_size_exceeded": 0,
"unchanged": 0,
"updated": 1
}

View File

@@ -35,6 +35,12 @@ fn fixture_report() -> IngestReport {
errors: 0,
duration_ms: 187,
skipped_by_extension: std::collections::BTreeMap::new(),
skipped_gitignore: 0,
skipped_kebabignore: 0,
skipped_builtin_blacklist: 0,
skipped_generated: 0,
skipped_size_exceeded: 0,
skip_examples: kebab_core::SkipExamples::default(),
items: Some(vec![
IngestItem {
kind: IngestItemKind::New,