cc-transcript 14.33.1__tar.gz → 14.35.0__tar.gz

{cc_transcript-14.33.1 → cc_transcript-14.35.0}/Cargo.lock

@@ -226,6 +226,7 @@ dependencies = [
  "once_cell",
  "rable",
  "regex",
+ "regex-syntax",
  "rusqlite",
  "ryu-js",
  "serde",

{cc_transcript-14.33.1 → cc_transcript-14.35.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cc-transcript
-Version: 14.33.1
+Version: 14.35.0
 Classifier: Development Status :: 3 - Alpha
 Classifier: Environment :: Console
 Classifier: Intended Audience :: Developers

{cc_transcript-14.33.1 → cc_transcript-14.35.0}/cc_transcript/snapshot_schema/config.schema.json

@@ -86,6 +86,41 @@
       "title": "Max Retained Bytes",
       "type": "integer"
     },
+    "max_scan_discovery_entries": {
+      "default": 4096,
+      "exclusiveMinimum": 0,
+      "maximum": 9007199254740991,
+      "title": "Max Scan Discovery Entries",
+      "type": "integer"
+    },
+    "max_scan_events": {
+      "default": 4096,
+      "exclusiveMinimum": 0,
+      "maximum": 9007199254740991,
+      "title": "Max Scan Events",
+      "type": "integer"
+    },
+    "max_scan_read_bytes": {
+      "default": 8388608,
+      "exclusiveMinimum": 0,
+      "maximum": 9007199254740991,
+      "title": "Max Scan Read Bytes",
+      "type": "integer"
+    },
+    "max_scan_sources": {
+      "default": 256,
+      "exclusiveMinimum": 0,
+      "maximum": 9007199254740991,
+      "title": "Max Scan Sources",
+      "type": "integer"
+    },
+    "max_scan_validate_bytes": {
+      "default": 1073741824,
+      "exclusiveMinimum": 0,
+      "maximum": 9007199254740991,
+      "title": "Max Scan Validate Bytes",
+      "type": "integer"
+    },
     "max_source_bytes": {
       "default": 536870912,
       "exclusiveMinimum": 0,

{cc_transcript-14.33.1 → cc_transcript-14.35.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cc-transcript"
-version = "14.33.1" # informational — releases stamp the version from the git tag
+version = "14.35.0" # informational — releases stamp the version from the git tag
 description = "Grep every Claude Code session you've ever run."
 readme = "README.md"
 license = "PolyForm-Noncommercial-1.0.0"

{cc_transcript-14.33.1 → cc_transcript-14.35.0}/rust/crates/cli/src/commands/grep.rs

@@ -1,10 +1,15 @@
+use std::collections::HashMap;
+use std::os::unix::fs::MetadataExt;
 use std::path::{Path, PathBuf};
 
 use cc_transcript_core::render::{compact_line, display_path, event_json, transcript_header, Json};
 use cc_transcript_core::scan::{
     scan_corpus, ScanBudget, ScanControl, ScanOutcome, ScanPlan, ScanSession,
 };
-use cc_transcript_core::scan_grep::{GrepOptions, GrepPatternSpec, GrepReducer, GrepSourceResult};
+use cc_transcript_core::scan_checkpoint::GrepCheckpoints;
+use cc_transcript_core::scan_grep::{
+    GrepEvent, GrepOptions, GrepPatternSpec, GrepReducer, GrepResultMetadata,
+};
 use cc_transcript_core::snapshot::{Cancellation, NativeStore, SnapshotError, Status, WorkLimits};
 use cc_transcript_core::types::{ContentBlock, Entry};
 use regex::{Regex, RegexBuilder};
@@ -212,8 +217,11 @@ impl Emitter {
         }
         self.out.finish()?;
         if let Some(reason) = &outcome.reason {
-            if reason == "result_limit" {
+            if let Some(scope) = reason.strip_prefix("result_limit") {
                 eline(&format!("warning: stopped at --max-matches {}; more matches may exist — raise it, or pass --max-matches 0 for no cap",counts.first().copied().unwrap_or(0)));
+                if let Some(scope) = scope.strip_prefix("; ") {
+                    eline(&format!("warning: partial view: {scope}"));
+                }
             } else {
                 eline(&format!("warning: scan incomplete: {reason}"));
                 return Err(CliExit(3));
@@ -226,7 +234,10 @@ impl Emitter {
     }
 }
 
-fn result_fields(event: &Entry, result: &GrepSourceResult<'_, '_>) -> Vec<(String, Json)> {
+fn result_fields(
+    event: &Entry,
+    results: &HashMap<&str, GrepResultMetadata<'_>>,
+) -> Vec<(String, Json)> {
     if !matches!(event, Entry::Assistant(_)) {
         return Vec::new();
     }
@@ -237,7 +248,7 @@ fn result_fields(event: &Entry, result: &GrepSourceResult<'_, '_>) -> Vec<(Strin
             let ContentBlock::ToolUse(tool) = block else {
                 return None;
             };
-            let value = result.results.get(tool.id.as_str())?;
+            let value = results.get(tool.id.as_str())?;
             Some((
                 tool.id.clone(),
                 Json::Obj(vec![
@@ -253,7 +264,7 @@ fn result_fields(event: &Entry, result: &GrepSourceResult<'_, '_>) -> Vec<(Strin
         .collect()
 }
 
-fn result_suffix(event: &Entry, result: &GrepSourceResult<'_, '_>) -> String {
+fn result_suffix(event: &Entry, results: &HashMap<&str, GrepResultMetadata<'_>>) -> String {
     if !matches!(event, Entry::Assistant(_)) {
         return String::new();
     }
@@ -264,7 +275,7 @@ fn result_suffix(event: &Entry, result: &GrepSourceResult<'_, '_>) -> String {
             let ContentBlock::ToolUse(tool) = block else {
                 return None;
             };
-            let value = result.results.get(tool.id.as_str())?;
+            let value = results.get(tool.id.as_str())?;
             let status = if value.denied {
                 "[denied]"
             } else if value.is_error {
@@ -329,84 +340,39 @@ pub fn run(args: GrepArgs) -> Result<(), CliExit> {
         contains: args.discovery.contains.clone(),
         source_limit: args.discovery.effective_limit(),
     };
-    let mut emitter = Emitter::new(args.scan_json, &mut session.budget, &cancel)?;
-    let mut files = 0usize;
-    let mut outcome = session.run(&plan, |path, snapshot, budget, cancel| {
-        let result = reducer.scan_source(snapshot, budget, cancel)?;
-        if result.hits.is_empty() {
-            return Ok(if result.quota_reached {
+    let structured = args.json || args.scan_json;
+    let checkpoints = checkpoints();
+    let mut render = Render {
+        args: &args,
+        emitter: Emitter::new(args.scan_json, &mut session.budget, &cancel)?,
+        files: 0,
+    };
+    let mut outcome = session.each_source(&plan, |session, path| {
+        if reducer.streams() {
+            if let Some(control) = reducer.scan_stream(
+                path,
+                !structured,
+                Some(&checkpoints),
+                &mut session.budget,
+                &cancel,
+                |event, budget, cancel| render.event(path, event, budget, cancel),
+            )? {
+                return Ok(control);
+            }
+        }
+        session.visit_snapshot(path, &mut |path, snapshot, budget, cancel| {
+            let result = reducer.scan_source(snapshot, budget, cancel)?;
+            result.emit(snapshot, budget, cancel, |event, budget, cancel| {
+                render.event(path, event, budget, cancel)
+            })?;
+            Ok(if result.quota_reached {
                 ScanControl::Stop {
                     source_complete: result.source_complete,
+                    names_through: None,
                 }
             } else {
                 ScanControl::Continue
-            });
-        }
-        files += 1;
-        if !args.json && !args.scan_json {
-            emitter.emit(transcript_header(&path.to_string_lossy()), budget, cancel)?;
-        }
-        for (window_index, window) in result.windows.iter().enumerate() {
-            if !args.json && !args.scan_json && args.context > 0 && window_index > 0 {
-                emitter.emit("--".to_owned(), budget, cancel)?;
-            }
-            for index in window.clone() {
-                let _render_staging = result.preflight_render(snapshot, index, budget, cancel)?;
-                let event = snapshot.entry(index);
-                let hit = result.hits.iter().find(|hit| hit.event_index == index);
-                let line = if args.json || args.scan_json {
-                    let Json::Obj(mut fields) = event_json(index, event) else {
-                        unreachable!("event_json object")
-                    };
-                    fields.insert(
-                        0,
-                        (
-                            "path".into(),
-                            Json::Str(path.to_string_lossy().into_owned()),
-                        ),
-                    );
-                    if hit.is_none() {
-                        fields.push(("context".into(), Json::Bool(true)));
-                    }
-                    if args.with_result {
-                        let results = result_fields(event, &result);
-                        if !results.is_empty() {
-                            fields.push(("results".into(), Json::Obj(results)));
-                        }
-                    }
-                    if args.scan_json {
-                        fields.push(("generation".into(), Json::Str(snapshot.id.clone())));
-                        fields.push((
-                            "pattern_ids".into(),
-                            Json::Arr(
-                                hit.map(|hit| {
-                                    hit.pattern_ids
-                                        .iter()
-                                        .map(|id| Json::Int(*id as i64))
-                                        .collect()
-                                })
-                                .unwrap_or_default(),
-                            ),
-                        ));
-                    }
-                    Json::Obj(fields).dumps()
-                } else {
-                    let mut line =
-                        compact_line(index, event, &result.names, args.width, false, args.uuids);
-                    if args.with_result {
-                        line.push_str(&result_suffix(event, &result));
-                    }
-                    line
-                };
-                emitter.emit(line, budget, cancel)?;
-            }
-        }
-        Ok(if result.quota_reached {
-            ScanControl::Stop {
-                source_complete: result.source_complete,
-            }
-        } else {
-            ScanControl::Continue
+            })
         })
     });
     if outcome.complete && !reducer.complete() {
@@ -423,9 +389,18 @@ pub fn run(args: GrepArgs) -> Result<(), CliExit> {
         } else {
             String::new()
         };
-        if let Err(error) = emitter.emit(
+        let validated = if session.budget.progress.validated_bytes > 0 {
+            format!(
+                " · re-read {} bytes for validation",
+                session.budget.progress.validated_bytes
+            )
+        } else {
+            String::new()
+        };
+        if let Err(error) = render.emitter.emit(
             format!(
-                "{files} files, {} matches{note}",
+                "{} files, {} matches{note}{validated}",
+                render.files,
                 counts.iter().sum::<usize>()
             ),
             &mut session.budget,
@@ -436,7 +411,85 @@ pub fn run(args: GrepArgs) -> Result<(), CliExit> {
         }
         outcome.progress = session.budget.progress.clone();
     }
-    emitter.finish(&outcome, counts)
+    render.emitter.finish(&outcome, counts)
+}
+
+struct Render<'a> {
+    args: &'a GrepArgs,
+    emitter: Emitter,
+    files: usize,
+}
+
+impl Render<'_> {
+    fn event(
+        &mut self,
+        path: &Path,
+        event: GrepEvent<'_>,
+        budget: &mut ScanBudget,
+        cancel: &Cancellation,
+    ) -> Result<(), SnapshotError> {
+        let args = self.args;
+        let structured = args.json || args.scan_json;
+        if event.opens_source {
+            self.files += 1;
+            if !structured {
+                self.emitter
+                    .emit(transcript_header(&path.to_string_lossy()), budget, cancel)?;
+            }
+        }
+        if !structured && args.context > 0 && event.opens_window && !event.opens_source {
+            self.emitter.emit("--".to_owned(), budget, cancel)?;
+        }
+        let _render_staging = event.preflight_render(budget, cancel)?;
+        let line = if structured {
+            let Json::Obj(mut fields) = event_json(event.index, event.entry) else {
+                unreachable!("event_json object")
+            };
+            fields.insert(
+                0,
+                (
+                    "path".into(),
+                    Json::Str(path.to_string_lossy().into_owned()),
+                ),
+            );
+            if event.pattern_ids.is_none() {
+                fields.push(("context".into(), Json::Bool(true)));
+            }
+            if args.with_result {
+                let results = result_fields(event.entry, event.results);
+                if !results.is_empty() {
+                    fields.push(("results".into(), Json::Obj(results)));
+                }
+            }
+            if args.scan_json {
+                fields.push(("generation".into(), Json::Str(event.generation.to_owned())));
+                fields.push((
+                    "pattern_ids".into(),
+                    Json::Arr(
+                        event
+                            .pattern_ids
+                            .map(|ids| ids.iter().map(|id| Json::Int(*id as i64)).collect())
+                            .unwrap_or_default(),
+                    ),
+                ));
+            }
+            Json::Obj(fields).dumps()
+        } else {
+            let mut line = compact_line(
+                event.index,
+                event.entry,
+                event.names,
+                args.width,
+                false,
+                args.uuids,
+            );
+            if args.with_result {
+                line.push_str(&result_suffix(event.entry, event.results));
+            }
+            line
+        };
+        self.emitter.emit(line, budget, cancel)
+    }
 }
 
 fn run_over_corpus(
@@ -521,6 +574,25 @@ fn run_over_corpus(
     emitter.finish(&outcome, counts)
 }
 
+fn checkpoints() -> GrepCheckpoints {
+    let exe = std::env::current_exe()
+        .and_then(std::fs::metadata)
+        .expect("current executable metadata");
+    GrepCheckpoints::new(
+        crate::target::home_dir()
+            .join(".claude")
+            .join("cc-transcript")
+            .join("scan"),
+        format!(
+            "{} {} {}.{}",
+            crate::pkg_version(),
+            exe.len(),
+            exe.mtime(),
+            exe.mtime_nsec()
+        ),
+    )
+}
+
 fn failed_outcome(error: SnapshotError, budget: &ScanBudget) -> ScanOutcome {
     ScanOutcome {
         complete: false,

{cc_transcript-14.33.1 → cc_transcript-14.35.0}/rust/crates/core/Cargo.toml

@@ -15,6 +15,7 @@ chrono = { version = "0.4", default-features = false, features = ["std", "serde"
 memchr = "2.7"
 once_cell = "1"
 regex = "1"
+regex-syntax = "0.8"
 # Gated behind `command`: a real bash AST (rable) backs the command parser. The Swift/CI build
 # takes core with default-features off, so this stays out of the python-free path.
 rable = { version = "0.2.1", optional = true }

{cc_transcript-14.33.1 → cc_transcript-14.35.0}/rust/crates/core/src/lib.rs

@@ -29,7 +29,11 @@ pub mod query;
 pub mod render;
 pub mod rng;
 pub mod scan;
+pub mod scan_checkpoint;
 pub mod scan_grep;
+pub mod scan_index;
+pub mod scan_projection;
+pub mod scan_stream;
 #[cfg(feature = "sqlite")]
 pub mod schema;
 pub mod snapshot;

{cc_transcript-14.33.1 → cc_transcript-14.35.0}/rust/crates/core/src/render.rs

@@ -834,7 +834,7 @@ fn search_limit() -> crate::snapshot::SnapshotError {
     )
 }
 
-fn search_add(
+pub(crate) fn search_add(
     total: &mut usize,
     bytes: usize,
     limit: usize,

{cc_transcript-14.33.1 → cc_transcript-14.35.0}/rust/crates/core/src/scan.rs

@@ -23,6 +23,8 @@ pub struct ScanProgress {
     pub output_bytes: usize,
     pub source_opens: usize,
     pub cache_hits: usize,
+    pub cache_invalidations: usize,
+    pub validated_bytes: usize,
 }
 
 pub struct ScanBudget<'store> {
@@ -31,6 +33,7 @@ pub struct ScanBudget<'store> {
     store: &'store NativeStore,
     context: Value,
     staging: ProjectionArena<'store>,
+    validation: usize,
 }
 
 pub type StagingReservation<'store> = crate::snapshot::ProjectionAllocation<'store>;
@@ -45,6 +48,7 @@ impl<'store> ScanBudget<'store> {
             store,
             staging: ProjectionArena::new(store, context.clone()),
             context,
+            validation: store.scan_validate_bytes(),
         }
     }
 
@@ -148,6 +152,40 @@ impl<'store> ScanBudget<'store> {
         Ok(())
     }
 
+    pub fn validation_remaining(&self) -> usize {
+        self.validation
+            .saturating_sub(self.progress.validated_bytes)
+    }
+
+    pub fn charge_validation(&mut self, bytes: usize) {
+        self.progress.validated_bytes += bytes;
+    }
+
+    pub fn charge_source(&mut self, bytes: usize) -> Result<(), SnapshotError> {
+        if bytes > self.remaining().max_source_read_bytes {
+            return Err(self.read_exhausted());
+        }
+        self.progress.source_bytes += bytes;
+        Ok(())
+    }
+
+    pub fn read_exhausted(&self) -> SnapshotError {
+        if self.progress.source_bytes >= self.limits.max_source_read_bytes {
+            crate::snapshot::source_read_limit()
+        } else {
+            incomplete("cumulative scan work budget exhausted")
+        }
+    }
+
+    pub fn charge_parse(&mut self, cancel: &Cancellation) -> Result<(), SnapshotError> {
+        self.checkpoint(cancel)?;
+        if self.remaining().max_events == 0 {
+            return Err(incomplete("cumulative scan work budget exhausted"));
+        }
+        self.progress.parsed_events += 1;
+        Ok(())
+    }
+
     pub fn charge_output(
         &mut self,
         bytes: usize,
@@ -227,7 +265,10 @@ pub struct ScanOutcome {
 
 pub enum ScanControl {
     Continue,
-    Stop { source_complete: bool },
+    Stop {
+        source_complete: bool,
+        names_through: Option<u64>,
+    },
 }
 
 pub struct ScanSession<'a> {
@@ -401,6 +442,15 @@ impl<'a> ScanSession<'a> {
             &mut ScanBudget<'a>,
             &Cancellation,
         ) -> Result<ScanControl, SnapshotError>,
+    {
+        self.each_source(plan, |session, path| {
+            session.visit_snapshot(path, &mut visit)
+        })
+    }
+
+    pub fn each_source<F>(&mut self, plan: &ScanPlan, mut source: F) -> ScanOutcome
+    where
+        F: FnMut(&mut Self, &Path) -> Result<ScanControl, SnapshotError>,
     {
         let mut selected_sources = 0;
         let mut available_sources = 0;
@@ -413,54 +463,28 @@ impl<'a> ScanSession<'a> {
                     return Err(incomplete("source budget exhausted"));
                 }
                 self.budget.progress.sources += 1;
-                let mut request = self.bounded_request("acquire");
-                request.insert("path", json!(path.to_string_lossy().as_ref()));
-                request.insert("classifier", json!({"id":"native","version":"1"}));
-                loop {
-                    let response = self.request(request)?;
-                    if let Some(cursor) = response["cursor"].as_str() {
-                        request = json!({"operation":"resume","cursor":cursor});
-                        if let Err(error) = self.budget.checkpoint(&self.cancel) {
-                            self.store.discard_response(&response, &self.context)?;
-                            return Err(error);
-                        }
-                        continue;
-                    }
-                    let guarded = ResponseGuard {
-                        store: self.store,
-                        context: &self.context,
-                        response,
-                        keep: false,
-                    };
-                    if guarded.response["status"].as_str() != Some("ok") {
-                        return Err(response_error(&guarded.response));
-                    }
-                    let snapshot = self
-                        .store
-                        .pin_scope_for_work(
-                            &guarded.response["data"]["description"]["handle"],
-                            &self.context,
-                            self.budget.limits.deadline_unix_ms,
-                        )?
-                        .0;
-                    if snapshot.event_count > self.budget.remaining().max_events {
-                        return Err(incomplete("preparation event budget exhausted"));
-                    }
-                    self.budget.progress.preparation_reserved_events += snapshot.event_count;
-                    match visit(&path, &snapshot, &mut self.budget, &self.cancel)? {
-                        ScanControl::Continue => {}
-                        ScanControl::Stop { source_complete } => {
-                            return Ok(source_complete && source_index + 1 == selected_sources)
-                        }
-                    }
-                    break;
+                if let ScanControl::Stop {
+                    source_complete,
+                    names_through,
+                } = source(self, &path)?
+                {
+                    return Ok((
+                        source_complete && source_index + 1 == selected_sources,
+                        names_through,
+                    ));
                 }
             }
-            Ok(true)
+            Ok((true, None))
         })();
         let (complete, reason) = match result {
-            Ok(true) => (true, None),
-            Ok(false) => (false, Some("result_limit".to_owned())),
+            Ok((true, _)) => (true, None),
+            Ok((false, None)) => (false, Some("result_limit".to_owned())),
+            Ok((false, Some(through))) => (
+                false,
+                Some(format!(
+                    "result_limit; tool names resolved through byte {through}"
+                )),
+            ),
             Err(error) => (
                 false,
                 Some(format!("{}: {}", error.status.as_str(), error.reason)),
@@ -474,6 +498,57 @@ impl<'a> ScanSession<'a> {
             progress: self.budget.progress.clone(),
         }
     }
+
+    pub fn visit_snapshot<F>(
+        &mut self,
+        path: &Path,
+        visit: &mut F,
+    ) -> Result<ScanControl, SnapshotError>
+    where
+        F: FnMut(
+            &Path,
+            &TranscriptSnapshot,
+            &mut ScanBudget<'a>,
+            &Cancellation,
+        ) -> Result<ScanControl, SnapshotError>,
+    {
+        let mut request = self.bounded_request("acquire");
+        request.insert("path", json!(path.to_string_lossy().as_ref()));
+        request.insert("classifier", json!({"id":"native","version":"1"}));
+        loop {
+            let response = self.request(request)?;
+            if let Some(cursor) = response["cursor"].as_str() {
+                request = json!({"operation":"resume","cursor":cursor});
+                if let Err(error) = self.budget.checkpoint(&self.cancel) {
+                    self.store.discard_response(&response, &self.context)?;
+                    return Err(error);
+                }
+                continue;
+            }
+            let guarded = ResponseGuard {
+                store: self.store,
+                context: &self.context,
+                response,
+                keep: false,
+            };
+            if guarded.response["status"].as_str() != Some("ok") {
+                return Err(response_error(&guarded.response));
+            }
+            let snapshot = self
+                .store
+                .pin_scope_for_work(
+                    &guarded.response["data"]["description"]["handle"],
+                    &self.context,
+                    self.budget.limits.deadline_unix_ms,
+                )?
+                .0;
+            if snapshot.event_count > self.budget.remaining().max_events {
+                return Err(incomplete("preparation event budget exhausted"));
+            }
+            self.budget.progress.preparation_reserved_events += snapshot.event_count;
+            return visit(path, &snapshot, &mut self.budget, &self.cancel);
+        }
+    }
 }
 
 impl Drop for ScanSession<'_> {