From 351c7a0826930fc863deb58ec15489ed870595f6 Mon Sep 17 00:00:00 2001 From: th-kim0823 Date: Fri, 15 May 2026 15:28:19 +0900 Subject: [PATCH] feat(p10-1a-1): add IngestReport skip counters + SkipExamples MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds five new u32 counters (skipped_gitignore, skipped_kebabignore, skipped_builtin_blacklist, skipped_generated, skipped_size_exceeded) and a SkipExamples struct (≤5 sample paths per category) to IngestReport. All new fields are #[serde(default)] for backward-compat deserialization. Downstream literal construction sites patched with zeros/empty; snapshot re-baked. Co-Authored-By: Claude Sonnet 4.6 --- crates/kebab-app/src/lib.rs | 8 +- crates/kebab-cli/src/wire.rs | 8 +- crates/kebab-core/src/ingest.rs | 88 +++++++++++++++++++ crates/kebab-core/src/lib.rs | 2 +- .../snapshots/ingest_report.snapshot.json | 11 +++ .../tests/ingest_report_snapshot.rs | 6 ++ 6 files changed, 120 insertions(+), 3 deletions(-) diff --git a/crates/kebab-app/src/lib.rs b/crates/kebab-app/src/lib.rs index ebb538d..3c8e03c 100644 --- a/crates/kebab-app/src/lib.rs +++ b/crates/kebab-app/src/lib.rs @@ -44,7 +44,7 @@ use kebab_core::{ Answer, Block, CanonicalDocument, Chunk, ChunkId, ChunkPolicy, ChunkerVersion, Chunker, DocFilter, DocSummary, DocumentId, DocumentStore, Embedder, EmbeddingInput, EmbeddingKind, ExtractContext, Extractor, IngestReport, Lang, LanguageModel, MediaType, - ParserVersion, RawAsset, SearchHit, SearchQuery, SourceConnector, SourceScope, + ParserVersion, RawAsset, SearchHit, SearchQuery, SkipExamples, SourceConnector, SourceScope, SourceUri, VectorRecord, VectorStore, }; use kebab_llm_local::OllamaLanguageModel; @@ -675,6 +675,12 @@ pub fn ingest_with_config_opts( errors: error_count, duration_ms, skipped_by_extension, + skipped_gitignore: 0, + skipped_kebabignore: 0, + skipped_builtin_blacklist: 0, + skipped_generated: 0, + skipped_size_exceeded: 0, + skip_examples: SkipExamples::default(), items: if summary_only { None } else { Some(items) }, }) } diff --git a/crates/kebab-cli/src/wire.rs b/crates/kebab-cli/src/wire.rs index a71397a..3fab435 100644 --- a/crates/kebab-cli/src/wire.rs +++ b/crates/kebab-cli/src/wire.rs @@ -239,7 +239,7 @@ mod tests { #[test] fn ingest_wrapper_tags_schema_version() { - use kebab_core::SourceScope; + use kebab_core::{SkipExamples, SourceScope}; let r = IngestReport { scope: SourceScope { root: std::path::PathBuf::from("/tmp"), @@ -254,6 +254,12 @@ mod tests { errors: 0, duration_ms: 0, skipped_by_extension: std::collections::BTreeMap::new(), + skipped_gitignore: 0, + skipped_kebabignore: 0, + skipped_builtin_blacklist: 0, + skipped_generated: 0, + skipped_size_exceeded: 0, + skip_examples: SkipExamples::default(), items: None, }; let v = wire_ingest(&r); diff --git a/crates/kebab-core/src/ingest.rs b/crates/kebab-core/src/ingest.rs index 8ada477..a3a9916 100644 --- a/crates/kebab-core/src/ingest.rs +++ b/crates/kebab-core/src/ingest.rs @@ -25,10 +25,46 @@ pub struct IngestReport { /// extension key under "". `BTreeMap` so the wire JSON /// has stable key order across runs. pub skipped_by_extension: std::collections::BTreeMap, + /// p10-1A-1: files skipped because they matched a repo-local `.gitignore`. + #[serde(default)] + pub skipped_gitignore: u32, + /// p10-1A-1: files skipped because they matched a `.kebabignore` entry. + #[serde(default)] + pub skipped_kebabignore: u32, + /// p10-1A-1: files skipped because they matched the built-in safety-net + /// blacklist (`node_modules/`, `target/`, `__pycache__/`, `.venv/`, + /// `venv/`, `env/`). + #[serde(default)] + pub skipped_builtin_blacklist: u32, + /// p10-1A-1: files skipped because their first ~512 bytes contained a + /// generated-file marker (`@generated`, `do not edit`, …). + #[serde(default)] + pub skipped_generated: u32, + /// p10-1A-1: files skipped because they exceeded `max_file_bytes` or + /// `max_file_lines` in `[ingest.code]`. + #[serde(default)] + pub skipped_size_exceeded: u32, + /// p10-1A-1: sample file paths per skip category (≤ 5 each). + #[serde(default)] + pub skip_examples: SkipExamples, /// `None` ↔ wire `items: null` (`--summary-only`). pub items: Option>, } +/// p10-1A-1: per-category sample of skipped file paths. Each category caps at +/// 5 entries (oldest-first). Used for debugging "why was X not indexed?" +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct SkipExamples { + #[serde(default)] + pub generated: Vec, + #[serde(default)] + pub size_exceeded: Vec, + #[serde(default)] + pub builtin_blacklist: Vec, + #[serde(default)] + pub gitignore: Vec, +} + #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub struct IngestItem { pub kind: IngestItemKind, @@ -58,3 +94,55 @@ pub enum IngestItemKind { Unchanged, Error, } + +#[cfg(test)] +mod tests { + use super::*; + use crate::traits::SourceScope; + + #[test] + fn skip_examples_default_is_empty() { + let s = SkipExamples::default(); + assert!(s.generated.is_empty()); + assert!(s.size_exceeded.is_empty()); + assert!(s.builtin_blacklist.is_empty()); + assert!(s.gitignore.is_empty()); + } + + #[test] + fn ingest_report_skip_counters_serialize() { + let r = IngestReport { + scope: SourceScope { + root: std::path::PathBuf::from("/tmp"), + include: vec![], + exclude: vec![], + }, + scanned: 100, + new: 50, + updated: 0, + skipped: 0, + unchanged: 0, + errors: 0, + duration_ms: 1234, + skipped_by_extension: Default::default(), + skipped_gitignore: 30, + skipped_kebabignore: 5, + skipped_builtin_blacklist: 10, + skipped_generated: 3, + skipped_size_exceeded: 2, + skip_examples: SkipExamples { + generated: vec!["a/b.pb.rs".into()], + size_exceeded: vec![], + builtin_blacklist: vec!["node_modules/x.js".into()], + gitignore: vec![], + }, + items: None, + }; + let v = serde_json::to_value(&r).unwrap(); + assert_eq!(v["skipped_gitignore"], 30); + assert_eq!(v["skipped_builtin_blacklist"], 10); + assert_eq!(v["skipped_generated"], 3); + assert_eq!(v["skipped_size_exceeded"], 2); + assert_eq!(v["skip_examples"]["generated"][0], "a/b.pb.rs"); + } +} diff --git a/crates/kebab-core/src/lib.rs b/crates/kebab-core/src/lib.rs index ba0ceb0..3b3b285 100644 --- a/crates/kebab-core/src/lib.rs +++ b/crates/kebab-core/src/lib.rs @@ -59,7 +59,7 @@ pub use answer::{ Answer, AnswerCitation, AnswerRetrievalSummary, ModelRef, RefusalReason, TokenUsage, TraceId, Turn, }; -pub use ingest::{IngestItem, IngestItemKind, IngestReport}; +pub use ingest::{IngestItem, IngestItemKind, IngestReport, SkipExamples}; pub use jobs::{JobFilter, JobId, JobKind, JobRow, JobStatus}; pub use vector::{VectorHit, VectorRecord}; pub use errors::CoreError; diff --git a/crates/kebab-store-sqlite/snapshots/ingest_report.snapshot.json b/crates/kebab-store-sqlite/snapshots/ingest_report.snapshot.json index 133aad3..f637f94 100644 --- a/crates/kebab-store-sqlite/snapshots/ingest_report.snapshot.json +++ b/crates/kebab-store-sqlite/snapshots/ingest_report.snapshot.json @@ -42,8 +42,19 @@ ], "root": "/home/u/KB" }, + "skip_examples": { + "builtin_blacklist": [], + "generated": [], + "gitignore": [], + "size_exceeded": [] + }, "skipped": 0, + "skipped_builtin_blacklist": 0, "skipped_by_extension": {}, + "skipped_generated": 0, + "skipped_gitignore": 0, + "skipped_kebabignore": 0, + "skipped_size_exceeded": 0, "unchanged": 0, "updated": 1 } diff --git a/crates/kebab-store-sqlite/tests/ingest_report_snapshot.rs b/crates/kebab-store-sqlite/tests/ingest_report_snapshot.rs index 458c6c6..5caf9dc 100644 --- a/crates/kebab-store-sqlite/tests/ingest_report_snapshot.rs +++ b/crates/kebab-store-sqlite/tests/ingest_report_snapshot.rs @@ -35,6 +35,12 @@ fn fixture_report() -> IngestReport { errors: 0, duration_ms: 187, skipped_by_extension: std::collections::BTreeMap::new(), + skipped_gitignore: 0, + skipped_kebabignore: 0, + skipped_builtin_blacklist: 0, + skipped_generated: 0, + skipped_size_exceeded: 0, + skip_examples: kebab_core::SkipExamples::default(), items: Some(vec![ IngestItem { kind: IngestItemKind::New,