cli-consumption 0.4.2__tar.gz → 0.4.4__tar.gz

{cli_consumption-0.4.2 → cli_consumption-0.4.4}/CHANGELOG.md

@@ -6,6 +6,22 @@ use [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
 
+## [0.4.4] - 2026-09-08
+
+### Fixed
+
+- Mistral Vibe now counts each persisted assistant message as a model call while
+  retaining cumulative session tokens as one unsplit aggregate.
+
+## [0.4.3] - 2026-09-05
+
+### Added
+
+- Added opt-in `collect --incremental` automatic bounded batching for large Codex
+  stores, with restart-safe ingestion, strict metadata-only preflight staging,
+  deterministic duplicate convergence, privacy-safe partial results, and unchanged
+  per-file and per-line safety limits; authoritative subagent graphs remain untouched.
+
 ## [0.4.2] - 2026-09-05
 
 ### Added
@@ -185,7 +201,9 @@ use [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 - Refreshed the provider guide for the first minor release ([#26]).
 
-[Unreleased]: https://github.com/Guillaume-Lombardo/cli-consumption/compare/v0.4.2...HEAD
+[Unreleased]: https://github.com/Guillaume-Lombardo/cli-consumption/compare/v0.4.4...HEAD
+[0.4.4]: https://github.com/Guillaume-Lombardo/cli-consumption/compare/v0.4.3...v0.4.4
+[0.4.3]: https://github.com/Guillaume-Lombardo/cli-consumption/compare/v0.4.2...v0.4.3
 [0.4.2]: https://github.com/Guillaume-Lombardo/cli-consumption/compare/v0.4.1...v0.4.2
 [0.4.1]: https://github.com/Guillaume-Lombardo/cli-consumption/compare/v0.4.0...v0.4.1
 [0.4.0]: https://github.com/Guillaume-Lombardo/cli-consumption/compare/v0.3.3...v0.4.0

{cli_consumption-0.4.2 → cli_consumption-0.4.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: cli-consumption
-Version: 0.4.2
+Version: 0.4.4
 Summary: Analyze and consolidate AI coding CLI consumption across machines.
 Project-URL: Homepage, https://github.com/Guillaume-Lombardo/cli-consumption
 Project-URL: Documentation, https://github.com/Guillaume-Lombardo/cli-consumption#readme
@@ -117,6 +117,10 @@ To collect one provider or select another database:
 uv run cli-consumption collect --provider codex --database usage.sqlite
 ```
 
+For a Codex store that exceeds aggregate collection limits, add `--incremental`.
+The command automatically writes bounded, restart-safe batches to the same database;
+individual file and line safety limits remain enforced.
+
 Use `--source [LABEL=]PATH` for trusted offline copies and repeated
 `--project NAME=PATH_PREFIX` mappings for stable project labels. See the
 [usage and operations guide](https://github.com/Guillaume-Lombardo/cli-consumption/blob/main/docs/usage.md)
@@ -145,7 +149,8 @@ uv run cli-consumption providers --json
 
 - Provider files are untrusted. Collection enforces discovery, file, line, SQLite row,
   structured-field, and total normalized-record limits; direct provider-file symlinks
-  are refused.
+  are refused. Opt-in Codex incremental collection resets only aggregate per-batch
+  limits and keeps a separate hard batch-count ceiling.
 - `collect --strict` and `sync --strict` refuse a batch when any provider skipped
   malformed records. Collection distinguishes `provider_limit_exceeded`,
   `provider_format_incompatible`, `invalid_snapshot`, and the unexpected-failure

{cli_consumption-0.4.2 → cli_consumption-0.4.4}/README.md

@@ -78,6 +78,10 @@ To collect one provider or select another database:
 uv run cli-consumption collect --provider codex --database usage.sqlite
 ```
 
+For a Codex store that exceeds aggregate collection limits, add `--incremental`.
+The command automatically writes bounded, restart-safe batches to the same database;
+individual file and line safety limits remain enforced.
+
 Use `--source [LABEL=]PATH` for trusted offline copies and repeated
 `--project NAME=PATH_PREFIX` mappings for stable project labels. See the
 [usage and operations guide](https://github.com/Guillaume-Lombardo/cli-consumption/blob/main/docs/usage.md)
@@ -106,7 +110,8 @@ uv run cli-consumption providers --json
 
 - Provider files are untrusted. Collection enforces discovery, file, line, SQLite row,
   structured-field, and total normalized-record limits; direct provider-file symlinks
-  are refused.
+  are refused. Opt-in Codex incremental collection resets only aggregate per-batch
+  limits and keeps a separate hard batch-count ceiling.
 - `collect --strict` and `sync --strict` refuse a batch when any provider skipped
   malformed records. Collection distinguishes `provider_limit_exceeded`,
   `provider_format_incompatible`, `invalid_snapshot`, and the unexpected-failure

{cli_consumption-0.4.2 → cli_consumption-0.4.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "cli-consumption"
-version = "0.4.2"
+version = "0.4.4"
 description = "Analyze and consolidate AI coding CLI consumption across machines."
 readme = "README.md"
 requires-python = ">=3.11"

cli_consumption-0.4.4/src/cli_consumption/adapters/base.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+from collections.abc import Iterator
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Protocol, runtime_checkable
+
+from cli_consumption.models import Snapshot
+
+
+@dataclass(frozen=True, slots=True)
+class CollectionBatch:
+    """One snapshot plus an optional subagent-scope authority override."""
+
+    snapshot: Snapshot
+    authoritative_subagent_scopes: frozenset[tuple[str, str]] | None = None
+
+
+class UnsupportedProviderFormat(ValueError):
+    """Raised when a detected provider store has an incompatible schema."""
+
+
+class Adapter(Protocol):
+    """Contract implemented by every supported AI CLI."""
+
+    name: str
+
+    def collect(
+        self,
+        sources: list[tuple[str, Path]],
+        project_mappings: list[tuple[str, str]],
+    ) -> Snapshot: ...
+
+
+@runtime_checkable
+class IncrementalAdapter(Adapter, Protocol):
+    """Optional adapter contract for bounded, independently ingestible batches."""
+
+    def collect_incrementally(
+        self,
+        sources: list[tuple[str, Path]],
+        project_mappings: list[tuple[str, str]],
+    ) -> Iterator[CollectionBatch]: ...

{cli_consumption-0.4.2 → cli_consumption-0.4.4}/src/cli_consumption/adapters/codex.py

@@ -3,8 +3,10 @@ from __future__ import annotations
 import hashlib
 import json
 import math
+import os
 import re
 import sqlite3
+from collections.abc import Iterator
 from datetime import UTC, datetime
 from pathlib import Path
 from typing import Any
@@ -13,11 +15,18 @@ from cli_consumption.adapters._shared import (
     MAX_BIGINT as MAX_BIGINT,
 )
 from cli_consumption.adapters._shared import (
+    ProviderDataLimitError,
     ProviderInputBudget,
     iter_bounded_jsonl_bytes,
     open_provider_sqlite,
 )
-from cli_consumption.models import TOKEN_FIELDS, Snapshot, empty_tokens
+from cli_consumption.adapters.base import CollectionBatch
+from cli_consumption.models import (
+    TOKEN_FIELDS,
+    Snapshot,
+    SnapshotValidationError,
+    empty_tokens,
+)
 
 OUTSIDE_PROJECT = "outside-project"
 TOOL_PATTERN = re.compile(r"(?:tools|collaboration)\.([A-Za-z][A-Za-z0-9_]*)\s*\(")
@@ -85,6 +94,24 @@ AGENT_ROLE_ALIASES = {
     "worker": "worker",
 }
 SAFE_DIMENSION = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.:/+-]*")
+INCREMENTAL_CANDIDATES_PER_BATCH = 1_000
+
+
+def _iter_session_files(root: Path) -> Iterator[Path]:
+    """Walk a provider tree deterministically without materializing every path."""
+    pending = [root]
+    while pending:
+        directory = pending.pop()
+        with os.scandir(directory) as entries:
+            ordered = sorted(entries, key=lambda entry: entry.name)
+        child_directories: list[Path] = []
+        for entry in ordered:
+            path = Path(entry.path)
+            if entry.is_dir(follow_symlinks=False):
+                child_directories.append(path)
+            elif entry.name.endswith(".jsonl"):
+                yield path
+        pending.extend(reversed(child_directories))
 
 
 def parse_timestamp(value: object) -> datetime | None:
@@ -166,6 +193,73 @@ class CodexAdapter:
             )
         return snapshot
 
+    def collect_incrementally(
+        self,
+        sources: list[tuple[str, Path]],
+        project_mappings: list[tuple[str, str]] | None = None,
+    ) -> Iterator[CollectionBatch]:
+        """Yield deterministic, bounded snapshots from arbitrarily many rollouts."""
+        mappings = project_mappings or []
+        for machine, codex_home in sources:
+            sessions = codex_home / "sessions"
+            if not sessions.is_dir():
+                raise ValueError("Missing Codex sessions directory")
+
+            yielded_sessions = False
+            batch: list[tuple[str, Path]] = []
+            for path in _iter_session_files(sessions):
+                batch.append((machine, path))
+                if len(batch) == INCREMENTAL_CANDIDATES_PER_BATCH:
+                    yielded_sessions = True
+                    yield from self._collect_incremental_batch(batch, mappings)
+                    batch = []
+            if batch:
+                yielded_sessions = True
+                yield from self._collect_incremental_batch(batch, mappings)
+
+            if not yielded_sessions:
+                yield CollectionBatch(Snapshot(provider=self.name), frozenset())
+
+    def _collect_incremental_batch(
+        self,
+        candidates: list[tuple[str, Path]],
+        mappings: list[tuple[str, str]],
+    ) -> Iterator[CollectionBatch]:
+        try:
+            yield CollectionBatch(
+                self._collect_candidates(candidates, mappings), frozenset()
+            )
+        except ProviderDataLimitError as error:
+            if str(error) != "provider_read_limit_exceeded" or len(candidates) == 1:
+                raise
+            midpoint = len(candidates) // 2
+            yield from self._collect_incremental_batch(candidates[:midpoint], mappings)
+            yield from self._collect_incremental_batch(candidates[midpoint:], mappings)
+        except SnapshotValidationError as error:
+            if error.code != "snapshot_too_large" or len(candidates) == 1:
+                raise
+            midpoint = len(candidates) // 2
+            yield from self._collect_incremental_batch(candidates[:midpoint], mappings)
+            yield from self._collect_incremental_batch(candidates[midpoint:], mappings)
+
+    def _collect_candidates(
+        self,
+        candidates: list[tuple[str, Path]],
+        mappings: list[tuple[str, str]],
+    ) -> Snapshot:
+        budget = ProviderInputBudget()
+        selected, duplicates, malformed = self._discover_candidates(candidates, budget)
+        snapshot = Snapshot(
+            provider=self.name,
+            duplicate_conversations=duplicates,
+            malformed_records=malformed,
+        )
+        for machine, path, event_count, digest in selected:
+            self._read_rollout(
+                snapshot, machine, path, event_count, digest, mappings, budget
+            )
+        return snapshot
+
     def _read_subagents(
         self,
         state_path: Path,
@@ -215,46 +309,62 @@ class CodexAdapter:
     def _discover(
         self, sources: list[tuple[str, Path]], budget: ProviderInputBudget
     ) -> tuple[list[tuple[str, Path, int, str]], int, int]:
-        selected: dict[str, tuple[str, Path, int, str]] = {}
-        duplicates = 0
-        malformed = 0
+        candidates: list[tuple[str, Path]] = []
         for machine, codex_home in sources:
             sessions = codex_home / "sessions"
             if not sessions.is_dir():
                 raise ValueError(f"Missing Codex sessions directory: {sessions}")
-            for path in budget.sorted_paths(sessions.rglob("*.jsonl")):
-                event_count = 0
-                conversation_id = ""
-                digest = hashlib.sha256()
-                for raw_line in iter_bounded_jsonl_bytes(path, budget):
-                    digest.update(raw_line)
-                    try:
-                        event = json.loads(raw_line)
-                    except (json.JSONDecodeError, UnicodeDecodeError):
-                        malformed += 1
-                        continue
-                    if not isinstance(event, dict):
+            candidates.extend(
+                (machine, path)
+                for path in budget.sorted_paths(sessions.rglob("*.jsonl"))
+            )
+        return self._discover_candidates(candidates, budget, charge_candidates=False)
+
+    def _discover_candidates(
+        self,
+        candidates: list[tuple[str, Path]],
+        budget: ProviderInputBudget,
+        *,
+        charge_candidates: bool = True,
+    ) -> tuple[list[tuple[str, Path, int, str]], int, int]:
+        selected: dict[str, tuple[str, Path, int, str]] = {}
+        duplicates = 0
+        malformed = 0
+        for machine, path in candidates:
+            if charge_candidates:
+                budget.item()
+            event_count = 0
+            conversation_id = ""
+            digest = hashlib.sha256()
+            for raw_line in iter_bounded_jsonl_bytes(path, budget):
+                digest.update(raw_line)
+                try:
+                    event = json.loads(raw_line)
+                except (json.JSONDecodeError, UnicodeDecodeError):
+                    malformed += 1
+                    continue
+                if not isinstance(event, dict):
+                    malformed += 1
+                    continue
+                event_count += 1
+                if event.get("type") == "session_meta":
+                    payload = event.get("payload")
+                    if not isinstance(payload, dict):
                         malformed += 1
                         continue
-                    event_count += 1
-                    if event.get("type") == "session_meta":
-                        payload = event.get("payload")
-                        if not isinstance(payload, dict):
-                            malformed += 1
-                            continue
-                        candidate_id = _safe_dimension(payload.get("id"), 512)
-                        if candidate_id and not conversation_id:
-                            conversation_id = candidate_id
-                content_hash = digest.hexdigest()
-                conversation_id = conversation_id or f"content-{content_hash}"
-                candidate = (machine, path, event_count, content_hash)
-                previous = selected.get(conversation_id)
-                if previous is None:
+                    candidate_id = _safe_dimension(payload.get("id"), 512)
+                    if candidate_id and not conversation_id:
+                        conversation_id = candidate_id
+            content_hash = digest.hexdigest()
+            conversation_id = conversation_id or f"content-{content_hash}"
+            candidate = (machine, path, event_count, content_hash)
+            previous = selected.get(conversation_id)
+            if previous is None:
+                selected[conversation_id] = candidate
+            else:
+                duplicates += 1
+                if candidate[2:] > previous[2:]:
                     selected[conversation_id] = candidate
-                else:
-                    duplicates += 1
-                    if candidate[2:] > previous[2:]:
-                        selected[conversation_id] = candidate
         return list(selected.values()), duplicates, malformed
 
     def _read_rollout(

{cli_consumption-0.4.2 → cli_consumption-0.4.4}/src/cli_consumption/adapters/mistral_vibe.py

@@ -84,6 +84,7 @@ class MistralVibeAdapter:
         started_at = timestamp(source.metadata.get("start_time"))
         ended_at = timestamp(source.metadata.get("end_time"))
         active_turn: dict[str, Any] | None = None
+        assistant_turn_ids: list[str | None] = []
         turn_number = 0
         tool_number = 0
         compaction_number = 0
@@ -125,6 +126,8 @@ class MistralVibeAdapter:
                 continue
             if active_turn is not None:
                 active_turn["status"] = "completed"
+                active_turn["model_calls"] += 1
+            assistant_turn_ids.append(active_turn["id"] if active_turn else None)
             raw_tool_calls = message.get("tool_calls")
             if not isinstance(raw_tool_calls, list):
                 continue
@@ -160,20 +163,25 @@ class MistralVibeAdapter:
         )
         config = mapping(source.metadata.get("config"))
         model = label(config.get("active_model"), 255) or "unknown"
-        model_calls = 0
         totals = empty_tokens()
-        if usage["total_tokens"] > 0 or counter(stats.get("steps")) > 0:
-            model_calls = 1
+        if usage["total_tokens"] > 0:
             add_tokens(totals, usage)
+
+        for sequence, turn_id in enumerate(assistant_turn_ids, 1):
+            call_usage = (
+                usage if sequence == len(assistant_turn_ids) else empty_tokens()
+            )
             snapshot.model_calls.append(
                 {
-                    "id": f"{conversation_id}:model:aggregate",
+                    "id": f"{conversation_id}:model:{sequence}",
                     "conversation_id": conversation_id,
-                    "turn_id": None,
-                    "sequence": 1,
-                    "timestamp": iso(ended_at),
+                    "turn_id": turn_id,
+                    "sequence": sequence,
+                    "timestamp": iso(ended_at)
+                    if sequence == len(assistant_turn_ids)
+                    else None,
                     "model": model,
-                    **usage,
+                    **call_usage,
                 }
             )
 
@@ -193,9 +201,9 @@ class MistralVibeAdapter:
                 "ended_at": iso(ended_at),
                 "duration_seconds": _duration(started_at, ended_at),
                 "source": "local-session-json",
-                "models": [model] if model_calls else [],
+                "models": [model] if assistant_turn_ids else [],
                 "iterations": turn_number,
-                "model_calls": model_calls,
+                "model_calls": len(assistant_turn_ids),
                 "tool_calls": tool_number,
                 "compactions": compaction_number,
                 "event_count": source.event_count,

{cli_consumption-0.4.2 → cli_consumption-0.4.4}/src/cli_consumption/adapters/registry.py

@@ -346,10 +346,12 @@ ADAPTER_SPECS = (
         token_semantics="conversation-aggregate",
         qualification=_qualification(
             "mistral-vibe",
-            "CLI 2.24.5",
+            "CLI 2.25.0",
             "session meta JSON and messages JSONL",
-            "https://github.com/mistralai/mistral-vibe/tree/v2.24.5",
-            "Session aggregates only; no timing or historical model attribution.",
+            "https://github.com/mistralai/mistral-vibe/tree/v2.25.0",
+            "Assistant-message call counts with session-aggregate tokens; no timing "
+            "or historical model attribution.",
+            qualified_on="2026-09-08",
         ),
     ),
     AdapterSpec(

{cli_consumption-0.4.2 → cli_consumption-0.4.4}/src/cli_consumption/cli.py

@@ -3,16 +3,22 @@ from __future__ import annotations
 import json
 import os
 import platform
+import tempfile
+from collections.abc import Iterator
 from datetime import UTC, datetime, timedelta
 from pathlib import Path
-from typing import Annotated, Never
+from typing import Annotated, Never, TextIO, TypedDict
 
 import typer
 from sqlalchemy.engine import Engine
 
 from cli_consumption import __version__
 from cli_consumption.adapters._shared import ProviderDataLimitError
-from cli_consumption.adapters.base import UnsupportedProviderFormat
+from cli_consumption.adapters.base import (
+    CollectionBatch,
+    IncrementalAdapter,
+    UnsupportedProviderFormat,
+)
 from cli_consumption.adapters.registry import (
     ADAPTER_SPECS,
     AdapterSpec,
@@ -35,8 +41,12 @@ from cli_consumption.storage import (
     create_database_engine,
     ingest_snapshot,
     initialize_database,
+    validate_snapshot,
 )
 
+MAX_INCREMENTAL_BATCHES = 10_000
+MAX_INCREMENTAL_STAGING_BYTES = 4 * 1024 * 1024 * 1024
+
 app = typer.Typer(
     name="cli-consumption",
     help="Analyze AI coding CLI consumption without exporting conversation content.",
@@ -59,6 +69,31 @@ class CollectionFailure(RuntimeError):
         super().__init__(code)
 
 
+class IncrementalIngestion(TypedDict):
+    provider: str
+    batches: int
+    received: int
+    written: int
+    skipped: int
+    malformed: int
+    batch_duplicates: int
+
+
+class _BoundedStagingWriter:
+    """Count UTF-8 metadata bytes before writing them to strict staging."""
+
+    def __init__(self, handle: TextIO, consumed: int) -> None:
+        self.handle = handle
+        self.consumed = consumed
+
+    def write(self, value: str) -> int:
+        size = len(value.encode("utf-8"))
+        if self.consumed + size > MAX_INCREMENTAL_STAGING_BYTES:
+            raise ProviderDataLimitError("provider_incremental_staging_limit_exceeded")
+        self.consumed += size
+        return self.handle.write(value)
+
+
 def version_callback(value: bool) -> None:
     if value:
         typer.echo(__version__)
@@ -145,11 +180,31 @@ def collect(
             help="Refuse ingestion when any malformed provider record was skipped.",
         ),
     ] = False,
+    incremental: Annotated[
+        bool,
+        typer.Option(
+            "--incremental",
+            help=(
+                "Automatically ingest supported large provider stores in bounded, "
+                "restart-safe batches."
+            ),
+        ),
+    ] = False,
     json_output: Annotated[
         bool, typer.Option("--json", help="Emit a deterministic JSON result.")
     ] = False,
 ) -> None:
     """Collect one or more local/copied CLI data directories into SQL storage."""
+    if incremental:
+        _collect_incrementally(
+            provider,
+            source,
+            project,
+            database,
+            strict=strict,
+            json_output=json_output,
+        )
+        return
     try:
         snapshots = _collect_snapshots(provider, source, project)
     except CollectionFailure as error:
@@ -742,25 +797,45 @@ def _emit_database_upload_json(
     typer.echo(json.dumps(payload, sort_keys=True, separators=(",", ":")))
 
 
-def _emit_collection_json(error: CollectionFailure) -> None:
+def _emit_collection_json(
+    error: CollectionFailure,
+    *,
+    ingestions: list[IncrementalIngestion] | None = None,
+    incremental: bool = False,
+) -> None:
     """Emit one deterministic collection failure without provider error details."""
+    payload: dict[str, object] = {
+        "error": {"code": error.code, "provider": error.provider},
+        "ingestions": ingestions or [],
+    }
+    if incremental:
+        payload["incremental"] = True
     typer.echo(
         json.dumps(
-            {
-                "error": {"code": error.code, "provider": error.provider},
-                "ingestions": [],
-            },
+            payload,
             sort_keys=True,
             separators=(",", ":"),
         )
     )
 
 
-def _abort_collection(error: CollectionFailure, *, json_output: bool) -> Never:
+def _abort_collection(
+    error: CollectionFailure,
+    *,
+    json_output: bool,
+    ingestions: list[IncrementalIngestion] | None = None,
+    incremental: bool = False,
+) -> Never:
     if json_output:
-        _emit_collection_json(error)
+        _emit_collection_json(error, ingestions=ingestions, incremental=incremental)
     else:
-        typer.echo(error.message, err=True)
+        completed = sum(item["batches"] for item in ingestions or [])
+        suffix = (
+            f" {completed} earlier incremental batch(es) were committed; rerun is safe."
+            if incremental and completed
+            else ""
+        )
+        typer.echo(error.message + suffix, err=True)
     raise typer.Exit(code=2) from None
 
 
@@ -1020,11 +1095,176 @@ def serve(
         engine.dispose()
 
 
-def _collect_snapshots(
+def _collect_incrementally(
     provider: str,
     source_values: list[str] | None,
     project_values: list[str] | None,
-) -> list[Snapshot]:
+    database: str,
+    *,
+    strict: bool,
+    json_output: bool,
+) -> None:
+    summaries: dict[str, IncrementalIngestion] = {}
+
+    def ingest(engine: Engine, batch: CollectionBatch) -> None:
+        snapshot = batch.snapshot
+        try:
+            result = ingest_snapshot(
+                engine,
+                snapshot,
+                authoritative_subagent_scopes=batch.authoritative_subagent_scopes,
+            )
+        except SnapshotValidationError as error:
+            raise _snapshot_failure(snapshot.provider, error) from None
+        summary = summaries.setdefault(
+            snapshot.provider,
+            {
+                "provider": snapshot.provider,
+                "batches": 0,
+                "received": 0,
+                "written": 0,
+                "skipped": 0,
+                "malformed": 0,
+                "batch_duplicates": 0,
+            },
+        )
+        summary["batches"] += 1
+        summary["received"] += result.received
+        summary["written"] += result.written
+        summary["skipped"] += result.skipped
+        summary["malformed"] += snapshot.malformed_records
+        summary["batch_duplicates"] += snapshot.duplicate_conversations
+
+    failure: CollectionFailure | None = None
+    if strict:
+        with tempfile.TemporaryDirectory(prefix="cli-consumption-") as staging:
+            staged: list[Path] = []
+            staged_bytes = 0
+            active_provider = provider
+            try:
+                for index, batch in enumerate(
+                    _iter_incremental_batches(provider, source_values, project_values)
+                ):
+                    snapshot = batch.snapshot
+                    active_provider = snapshot.provider
+                    if snapshot.malformed_records:
+                        raise CollectionFailure(
+                            snapshot.provider,
+                            "malformed_records",
+                            "--strict refused incremental snapshots containing "
+                            "malformed provider records.",
+                        )
+                    validated = validate_snapshot(snapshot)
+                    path = Path(staging) / f"{index:08d}.json"
+                    descriptor = os.open(
+                        path,
+                        os.O_WRONLY | os.O_CREAT | os.O_EXCL,
+                        0o600,
+                    )
+                    with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
+                        writer = _BoundedStagingWriter(handle, staged_bytes)
+                        json.dump(
+                            {
+                                "snapshot": validated.to_dict(),
+                                "authoritative_subagent_scopes": (
+                                    sorted(batch.authoritative_subagent_scopes)
+                                    if batch.authoritative_subagent_scopes is not None
+                                    else None
+                                ),
+                            },
+                            writer,
+                            sort_keys=True,
+                            separators=(",", ":"),
+                        )
+                        staged_bytes = writer.consumed
+                    staged.append(path)
+            except CollectionFailure as error:
+                failure = error
+            except SnapshotValidationError as error:
+                failure = _snapshot_failure(active_provider, error)
+            except ProviderDataLimitError:
+                failure = CollectionFailure(
+                    active_provider,
+                    "provider_limit_exceeded",
+                    f"Provider {active_provider!r} data exceeds collection "
+                    "safety limits.",
+                )
+            except OSError:
+                failure = CollectionFailure(
+                    active_provider,
+                    "provider_collection_failed",
+                    f"Provider {active_provider!r} collection failed.",
+                )
+
+            if failure is None:
+                engine = _open_database(database)
+                try:
+                    for path in staged:
+                        with path.open(encoding="utf-8") as handle:
+                            payload = json.load(handle)
+                        raw_scopes = payload["authoritative_subagent_scopes"]
+                        ingest(
+                            engine,
+                            CollectionBatch(
+                                Snapshot.from_dict(payload["snapshot"]),
+                                (
+                                    frozenset(
+                                        (str(item[0]), str(item[1]))
+                                        for item in raw_scopes
+                                    )
+                                    if raw_scopes is not None
+                                    else None
+                                ),
+                            ),
+                        )
+                except CollectionFailure as error:
+                    failure = error
+                finally:
+                    engine.dispose()
+    else:
+        engine = _open_database(database)
+        try:
+            try:
+                for batch in _iter_incremental_batches(
+                    provider, source_values, project_values
+                ):
+                    ingest(engine, batch)
+            except CollectionFailure as error:
+                failure = error
+        finally:
+            engine.dispose()
+
+    outcomes = list(summaries.values())
+    if failure is not None:
+        _abort_collection(
+            failure,
+            json_output=json_output,
+            ingestions=outcomes,
+            incremental=True,
+        )
+    if json_output:
+        typer.echo(
+            json.dumps(
+                {"incremental": True, "ingestions": outcomes},
+                sort_keys=True,
+                separators=(",", ":"),
+            )
+        )
+        return
+    for summary in outcomes:
+        typer.echo(
+            f"Incremental ingestion {summary['provider']}: "
+            f"{summary['batches']} batches, {summary['written']} written, "
+            f"{summary['skipped']} unchanged, "
+            f"{summary['malformed']} malformed skipped."
+        )
+
+
+def _collection_inputs(
+    provider: str,
+    source_values: list[str] | None,
+    project_values: list[str] | None,
+) -> tuple[list[tuple[AdapterSpec, list[tuple[str, Path]]]], list[tuple[str, str]]]:
     spec = resolve_adapter_spec(provider) if provider != "all" else None
     if provider != "all" and spec is None:
         raise typer.BadParameter(
@@ -1032,11 +1272,9 @@ def _collect_snapshots(
         )
     mappings = _parse_project_mappings(project_values or [])
     if spec is not None:
-        return [
-            _collect_adapter(spec, _parse_sources(source_values or [], spec), mappings)
-        ]
+        return [(spec, _parse_sources(source_values or [], spec))], mappings
 
-    snapshots: list[Snapshot] = []
+    inputs: list[tuple[AdapterSpec, list[tuple[str, Path]]]] = []
     if source_values:
         sources = _parse_source_values(source_values)
         matched_labels: set[str] = set()
@@ -1046,7 +1284,7 @@ def _collect_snapshots(
             ]
             if matched:
                 matched_labels.update(label for label, _ in matched)
-                snapshots.append(_collect_adapter(candidate, matched, mappings))
+                inputs.append((candidate, matched))
         unmatched = [label for label, _ in sources if label not in matched_labels]
         if unmatched:
             raise typer.BadParameter(
@@ -1058,12 +1296,63 @@ def _collect_snapshots(
         for candidate in ADAPTER_SPECS:
             path = default_source_path(candidate)
             if has_provider_data(candidate, path):
-                snapshots.append(
-                    _collect_adapter(candidate, [(machine, path)], mappings)
-                )
-    if not snapshots:
+                inputs.append((candidate, [(machine, path)]))
+    if not inputs:
         raise typer.BadParameter("No supported provider data detected.")
-    return snapshots
+    return inputs, mappings
+
+
+def _collect_snapshots(
+    provider: str,
+    source_values: list[str] | None,
+    project_values: list[str] | None,
+) -> list[Snapshot]:
+    inputs, mappings = _collection_inputs(provider, source_values, project_values)
+    return [_collect_adapter(spec, sources, mappings) for spec, sources in inputs]
+
+
+def _iter_incremental_batches(
+    provider: str,
+    source_values: list[str] | None,
+    project_values: list[str] | None,
+) -> Iterator[CollectionBatch]:
+    inputs, mappings = _collection_inputs(provider, source_values, project_values)
+    batches = 0
+    for spec, sources in inputs:
+        adapter = spec.adapter_type()
+        try:
+            batches_for_adapter = (
+                adapter.collect_incrementally(sources, mappings)
+                if isinstance(adapter, IncrementalAdapter)
+                else iter((CollectionBatch(adapter.collect(sources, mappings)),))
+            )
+            for batch in batches_for_adapter:
+                batches += 1
+                if batches > MAX_INCREMENTAL_BATCHES:
+                    raise ProviderDataLimitError(
+                        "provider_incremental_batch_limit_exceeded"
+                    )
+                yield batch
+        except ProviderDataLimitError:
+            raise CollectionFailure(
+                spec.name,
+                "provider_limit_exceeded",
+                f"Provider {spec.name!r} data exceeds collection safety limits.",
+            ) from None
+        except UnsupportedProviderFormat:
+            raise CollectionFailure(
+                spec.name,
+                "provider_format_incompatible",
+                f"Provider {spec.name!r} data format is incompatible.",
+            ) from None
+        except SnapshotValidationError as error:
+            raise _snapshot_failure(spec.name, error) from None
+        except Exception:
+            raise CollectionFailure(
+                spec.name,
+                "provider_collection_failed",
+                f"Provider {spec.name!r} collection failed.",
+            ) from None
 
 
 def _collect_adapter(

{cli_consumption-0.4.2 → cli_consumption-0.4.4}/src/cli_consumption/storage.py

@@ -364,8 +364,16 @@ def ingest_snapshot(
     snapshot: Snapshot,
     *,
     idempotency_key: str | None = None,
+    authoritative_subagent_scopes: frozenset[tuple[str, str]] | None = None,
 ) -> IngestionResult:
     snapshot = validate_snapshot(snapshot)
+    if authoritative_subagent_scopes is not None and any(
+        provider != snapshot.provider
+        or not isinstance(source_machine, str)
+        or not 1 <= len(source_machine) <= 255
+        for provider, source_machine in authoritative_subagent_scopes
+    ):
+        raise SnapshotValidationError()
     initialize_database(engine)
     if idempotency_key is not None:
         idempotency_key = _canonical_idempotency_key(idempotency_key)
@@ -381,10 +389,15 @@ def ingest_snapshot(
     context_by_conversation = _group(snapshot.context_samples)
     settings_by_conversation = _group(snapshot.turn_settings)
     compactions_by_conversation = _group(snapshot.compaction_events)
-    subagent_scopes = {
+    inferred_subagent_scopes = {
         (snapshot.provider, str(record["source_machine"]))
         for record in (*snapshot.conversations, *snapshot.subagents)
     }
+    subagent_scopes = (
+        inferred_subagent_scopes
+        if authoritative_subagent_scopes is None
+        else set(authoritative_subagent_scopes)
+    )
     stale_subagent_scopes: set[tuple[str, str]] = set()
     richer_subagent_scopes: set[tuple[str, str]] = set()
     try:
@@ -398,21 +411,20 @@ def ingest_snapshot(
                 conversation_id = str(record["id"])
                 existing = session.get(Conversation, conversation_id)
                 scope = (snapshot.provider, str(record["source_machine"]))
-                if existing is not None and existing.event_count > int(
-                    record["event_count"]
-                ):
+                incoming_rank = (
+                    int(record["event_count"]),
+                    str(record["content_hash"]),
+                )
+                existing_rank = (
+                    (existing.event_count, existing.content_hash)
+                    if existing is not None
+                    else None
+                )
+                if existing_rank is not None and existing_rank > incoming_rank:
                     stale_subagent_scopes.add(scope)
-                elif existing is not None and existing.event_count < int(
-                    record["event_count"]
-                ):
+                elif existing_rank is not None and existing_rank < incoming_rank:
                     richer_subagent_scopes.add(scope)
-                if existing is not None and (
-                    existing.event_count > int(record["event_count"])
-                    or (
-                        existing.event_count == int(record["event_count"])
-                        and existing.content_hash == record["content_hash"]
-                    )
-                ):
+                if existing_rank is not None and existing_rank >= incoming_rank:
                     skipped += 1
                     continue
                 session.execute(
@@ -461,9 +473,14 @@ def ingest_snapshot(
                 for compaction in compactions_by_conversation.get(conversation_id, []):
                     session.add(CompactionEvent(**compaction))
                 written += 1
-            authoritative_scopes = initial_subagent_scopes | (
+            inferred_authoritative_scopes = initial_subagent_scopes | (
                 richer_subagent_scopes - stale_subagent_scopes
             )
+            authoritative_scopes = (
+                inferred_authoritative_scopes
+                if authoritative_subagent_scopes is None
+                else authoritative_subagent_scopes
+            )
             for provider, source_machine in authoritative_scopes:
                 session.execute(
                     delete(Subagent).where(

cli_consumption-0.4.2/src/cli_consumption/adapters/base.py

@@ -1,22 +0,0 @@
-from __future__ import annotations
-
-from pathlib import Path
-from typing import Protocol
-
-from cli_consumption.models import Snapshot
-
-
-class UnsupportedProviderFormat(ValueError):
-    """Raised when a detected provider store has an incompatible schema."""
-
-
-class Adapter(Protocol):
-    """Contract implemented by every supported AI CLI."""
-
-    name: str
-
-    def collect(
-        self,
-        sources: list[tuple[str, Path]],
-        project_mappings: list[tuple[str, str]],
-    ) -> Snapshot: ...