driftcut 0.8.0__tar.gz

driftcut-0.8.0/.dockerignore

@@ -0,0 +1,14 @@
+.git
+.github
+.mypy_cache
+.pytest_cache
+.ruff_cache
+.tmp
+.venv
+driftcut-results
+site
+docs-live.html
+pytest-cache-files-*
+pytest-temp-*
+__pycache__
+*.pyc

driftcut-0.8.0/.github/workflows/ci.yml

@@ -0,0 +1,48 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+      - run: pip install -e ".[dev]"
+      - run: python scripts/check_release_alignment.py
+      - run: ruff check src tests
+      - run: ruff format --check src tests
+      - run: mypy src tests
+
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v5
+      - uses: actions/setup-python@v6
+        with:
+          python-version: ${{ matrix.python-version }}
+      - run: pip install -e ".[dev]"
+      - run: pytest -v
+
+  package:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.13"
+      - run: pip install -e ".[dev]"
+      - run: python -m build
+      - run: python -m twine check dist/*
+      - run: python -m venv .pkg-venv
+      - run: .pkg-venv/bin/pip install dist/*.whl
+      - run: .pkg-venv/bin/driftcut --version

driftcut-0.8.0/.github/workflows/pages.yml

@@ -0,0 +1,35 @@
+name: Deploy landing page
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'site/**'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+concurrency:
+  group: pages
+  cancel-in-progress: true
+
+env:
+  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
+
+jobs:
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+      - uses: actions/configure-pages@v5
+      - uses: actions/upload-pages-artifact@v4
+        with:
+          path: site
+      - id: deployment
+        uses: actions/deploy-pages@v4

driftcut-0.8.0/.github/workflows/publish.yml

@@ -0,0 +1,75 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types:
+      - published
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: Git ref to build and publish, for example main or v0.8.0
+        required: true
+        default: main
+
+concurrency:
+  group: pypi-publish-${{ github.event.release.tag_name || github.event.inputs.ref || github.ref_name }}
+  cancel-in-progress: false
+
+jobs:
+  build-distributions:
+    name: Build distributions
+    if: github.event_name != 'release' || github.event.release.prerelease == false
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v5
+        with:
+          ref: ${{ github.event.inputs.ref || github.ref }}
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.13"
+
+      - name: Install build tooling
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install build twine
+
+      - name: Build distributions
+        run: python -m build
+
+      - name: Validate distribution metadata
+        run: python -m twine check dist/*
+
+      - name: Upload built distributions
+        uses: actions/upload-artifact@v4
+        with:
+          name: python-package-distributions
+          path: dist/
+          if-no-files-found: error
+
+  publish-to-pypi:
+    name: Publish to PyPI
+    if: github.event_name != 'release' || github.event.release.prerelease == false
+    needs:
+      - build-distributions
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/project/driftcut/
+    permissions:
+      id-token: write
+
+    steps:
+      - name: Download distributions
+        uses: actions/download-artifact@v4
+        with:
+          name: python-package-distributions
+          path: dist/
+
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

driftcut-0.8.0/.github/workflows/release-audit.yml

@@ -0,0 +1,21 @@
+name: Release Audit
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  audit:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+
+      - run: python scripts/check_release_alignment.py
+
+      - run: python scripts/check_release_state.py --retries 6 --delay-seconds 10

driftcut-0.8.0/.gitignore

@@ -0,0 +1,43 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.egg-info/
+dist/
+build/
+*.egg
+.eggs/
+
+# Virtual environments
+.venv/
+venv/
+env/
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+.mypy_cache/
+.ruff_cache/
+
+# Environment
+.env
+.env.local
+
+# SQLite
+*.db
+*.sqlite3
+
+# Driftcut run outputs
+driftcut-results/

driftcut-0.8.0/CHANGELOG.md

@@ -0,0 +1,179 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [0.8.0] - 2026-04-02
+
+### Added
+
+- Richer prompt-level failure archetypes, including semantic regressions such as `refusal_regression`, `instruction_miss`, `incomplete_answer`, and `format_drift`
+- Per-category scorecards in decision metrics, JSON output, and HTML reports
+- Category-aware decision reasoning and console summaries that highlight the highest-risk category
+- 4 new tests covering richer archetypes, category scorecards, and clearer run-level reasoning (110 total)
+
+### Changed
+
+- Prompt evaluations now retain multiple failure archetypes instead of collapsing to a single coarse label
+- Judge-driven regressions can now classify into more actionable semantic buckets instead of only `judge_worse`
+- HTML examples now surface archetype summaries alongside deterministic and judge rationale
+
+## [0.7.0] - 2026-04-02
+
+### Added
+
+- Optional Redis memory layer for baseline response caching and searchable run-history persistence
+- `cache_hit`, cache summary, and saved baseline cost metrics in JSON and HTML outputs
+- `driftcut[redis]` extra plus a Redis-enabled sample config for local testing
+- `Dockerfile`, `docker-compose.yml`, and `.dockerignore` for reproducible local Redis-backed runs
+- 9 new tests covering Redis config, caching behavior, reporting, and store adapters (106 total)
+
+### Changed
+
+- Cached baseline responses are excluded from live latency comparisons so reuse does not distort candidate latency decisions
+- Memory-backed runs now persist canonical run payloads through the same reporting shape used for file exports
+- README and docs now document local Docker + Redis workflows alongside the normal Python path
+
+## [0.6.0] - 2026-04-01
+
+### Added
+
+- Real tiered judging with light-to-heavy escalation when light judge confidence is below threshold
+- Configurable `tiered_escalation_threshold` in evaluation config (default 0.6)
+- `tier` and `escalated` fields on judge results for tracking which judge tier produced the verdict
+- `escalated_prompts` metric in decision metrics and JSON/HTML reports
+- Split judge cost tracking: `judge_light_usd` and `judge_heavy_usd` in cost summaries
+- Escalation threshold shown in HTML thresholds table when strategy is tiered
+- 8 new tests for tiered escalation, config validation, cost splitting, and reporting (97 total)
+
+### Changed
+
+- `judge_strategy: tiered` now performs actual light-then-heavy escalation instead of aliasing to light
+- Judge cost breakdown (light vs heavy) shown in HTML report when both tiers are used
+- Console run summary includes escalated count when escalation occurs
+
+## [0.5.1] - 2026-03-30
+
+### Added
+
+- CLI reference documentation page for `validate`, `run`, `replay`, and global flags
+- Visible quickstart output examples across the README and docs site, including terminal and `results.json` excerpts
+
+### Changed
+
+- README, landing page, and docs now show the produced artifacts more concretely instead of only describing the workflow
+- Concept documentation now reflects the shipped three-way decision engine and no longer implies category-scoped proceed decisions
+
+### Fixed
+
+- Live model calls now retry transient rate-limit, timeout, connection, and 5xx failures before counting them as API errors
+- JSON exports now include per-response `retry_count` so retry behavior is auditable in saved artifacts
+
+## [0.5.0] - 2026-03-30
+
+### Added
+
+- `driftcut replay --config ... --input ...` for historical paired-output backtesting
+- Canonical replay JSON loader with versioned schema validation
+- Replay-aware JSON and HTML reports with explicit `mode: replay` labeling
+- Replay example assets and an external converter template under `scripts/`
+- 8 new tests covering replay loading, CLI behavior, reporting, and live/replay parity
+
+### Changed
+
+- Refactored the runner so live execution and replay share the same post-processing path
+- Replay now reuses stratified sampling, deterministic checks, judge flow, decision logic, and early-stop behavior
+- Replay configs can omit `corpus.file`; the canonical replay input provides prompt metadata directly
+
+## [0.4.0] - 2026-03-29
+
+### Added
+
+- Judge layer for ambiguous prompts with semantic verdicts, confidence, and rationale
+- Judge-aware decision metrics, confidence scoring, and cost tracking
+- Judge details in JSON output and HTML reports
+- 6 new tests for judge helpers and runtime integration
+
+### Changed
+
+- `judge_strategy: light` is now the active default for the alpha runtime
+- Ambiguous prompts now lower confidence until they are judged or the strategy is disabled
+- `tiered` remains a compatibility alias for light judging until heavy escalation lands
+
+## [0.3.0] - 2026-03-29
+
+### Added
+
+- Deterministic quality checks for expected output formats, required content, JSON keys, and length limits
+- Early-stop decision engine with `STOP`, `CONTINUE`, and `PROCEED` outcomes
+- HTML report generation alongside richer JSON output
+- Run-level risk metrics, confidence, and failure archetype summaries
+- 6 new tests for quality checks, reporting, and decision behavior
+
+### Changed
+
+- `min_batches` now actively gates early `PROCEED` decisions
+- `risk` and most `output` settings are now active runtime behavior instead of validation-only config
+- Public repo messaging and examples now reflect the real runtime feature set
+
+## [0.2.2] - 2026-03-29
+
+### Changed
+
+- Raised the repository quality bar with enforced `mypy` checks in CI and a clean strict-typing pass
+- Stabilized sampled batch result ordering and refreshed the public alpha messaging across the app site
+
+### Fixed
+
+- Preserved successful model responses when pricing metadata is unavailable
+- Aligned the shipped app version across package metadata, CLI output, changelog, and landing page badge
+
+## [0.2.1] - 2026-03-29
+
+### Added
+
+- `api_base` field on model config for custom endpoints (Azure, proxies, self-hosted)
+- OpenRouter support documented and tested
+- Multi-provider config examples (same-provider, OpenRouter, custom endpoint)
+- 5 new tests (66 total)
+
+## [0.2.0] - 2026-03-29
+
+### Added
+
+- Async model execution via LiteLLM (`executor.py`)
+- Latency tracker with p50/p95 per category (`trackers.py`)
+- Cost tracker with per-prompt and cumulative spend (`trackers.py`)
+- Migration runner with concurrent batch execution (`runner.py`)
+- `driftcut run --config` command — fully wired end-to-end
+- JSON results export to `driftcut-results/results.json`
+- Result data models: `ModelResponse`, `PromptResult`, `BatchResult` (`models.py`)
+- Rich progress bars during batch execution
+- 26 new tests (61 total)
+
+## [0.1.0] - 2026-03-28
+
+### Added
+
+- YAML config loading and validation with Pydantic models
+- Corpus loading from CSV and JSON with full validation
+- Stratified batch sampler (high-criticality prioritized in early batches)
+- `driftcut validate --config` command with Rich terminal output
+- CI pipeline (ruff lint + format + pytest on Python 3.12 & 3.13)
+- Pre-launch landing page at driftcut.dev
+- Documentation site at docs.driftcut.dev
+- 35 tests covering config, corpus, sampler, and CLI
+
+[0.8.0]: https://github.com/riccardomerenda/driftcut/compare/v0.7.0...v0.8.0
+[0.7.0]: https://github.com/riccardomerenda/driftcut/compare/v0.6.0...v0.7.0
+[0.6.0]: https://github.com/riccardomerenda/driftcut/compare/v0.5.1...v0.6.0
+[0.5.1]: https://github.com/riccardomerenda/driftcut/compare/v0.5.0...v0.5.1
+[0.5.0]: https://github.com/riccardomerenda/driftcut/compare/v0.4.0...v0.5.0
+[0.4.0]: https://github.com/riccardomerenda/driftcut/compare/v0.3.0...v0.4.0
+[0.3.0]: https://github.com/riccardomerenda/driftcut/compare/v0.2.2...v0.3.0
+[0.2.2]: https://github.com/riccardomerenda/driftcut/compare/v0.2.1...v0.2.2
+[0.2.1]: https://github.com/riccardomerenda/driftcut/compare/v0.2.0...v0.2.1
+[0.2.0]: https://github.com/riccardomerenda/driftcut/compare/v0.1.0...v0.2.0
+[0.1.0]: https://github.com/riccardomerenda/driftcut/releases/tag/v0.1.0

driftcut-0.8.0/CLAUDE.md

@@ -0,0 +1,88 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## What is this?
+
+Driftcut is an early-stop canary testing tool for LLM model migrations. It samples strategically from a prompt corpus, compares baseline and candidate models, runs deterministic checks first, escalates only ambiguous prompts to a judge model, and outputs a STOP/CONTINUE/PROCEED decision — all before committing to a full evaluation run.
+
+## Commands
+
+```bash
+pip install -e ".[dev]"              # Install in dev mode
+pytest                                # Run all tests
+pytest tests/test_runner.py           # Run a single test file
+pytest tests/test_runner.py -k "test_name"  # Run a single test
+ruff check src tests                  # Lint
+ruff format src tests                 # Format
+mypy src                              # Type check
+driftcut validate --config migration.yaml   # Validate config + corpus
+driftcut run --config migration.yaml        # Run a migration test
+driftcut replay --config replay.yaml --input replay.json  # Replay historical outputs
+```
+
+## Architecture
+
+### Data flow
+
+```
+CLI (cli.py) loads config + corpus
+    → StratifiedSampler yields Batch objects (criticality-first)
+    → runner.py orchestrates batch-by-batch execution
+        → executor.py runs baseline + candidate concurrently per prompt (asyncio.gather)
+        → quality.py runs deterministic checks on every response (free)
+        → judge.py called only for ambiguous prompts (both pass deterministic but differ semantically)
+        → decision.py evaluates STOP/CONTINUE/PROCEED after each batch
+    → reporting.py writes JSON + HTML to driftcut-results/
+```
+
+Replay mode substitutes pre-recorded responses for live execution but uses the same evaluation pipeline.
+
+### Module roles
+
+| Module | Role |
+|---|---|
+| `cli.py` | Typer commands: `run`, `validate`, `replay` |
+| `config.py` | Pydantic models for YAML config with constrained fields |
+| `corpus.py` | CSV/JSON loader → `PromptRecord` list with flexible delimiter parsing (`\|` or `;`) |
+| `sampler.py` | `StratifiedSampler` iterator — yields equal-sized batches, pre-sorted by criticality within category |
+| `runner.py` | Async orchestrator — batch loop, wires executor → quality → judge → decision |
+| `executor.py` | Async LiteLLM wrapper with retry (3 attempts, exponential backoff for rate limits/5xx/timeouts) |
+| `quality.py` | Deterministic checks driven by `expected_output_type`: JSON validity, required keys, substrings, length |
+| `judge.py` | Sends ambiguous prompt pairs to judge model, extracts JSON verdict from freeform responses |
+| `decision.py` | Decision engine: hard stops on schema breaks/high-crit failures, proceed gate on overall risk + latency |
+| `trackers.py` | Cost and latency accumulators (cost gracefully handles missing LiteLLM pricing) |
+| `models.py` | Frozen dataclasses: `ModelResponse`, `PromptEvaluation`, `JudgeResult`, `RunDecision`, `RunResult` |
+| `reporting.py` | JSON serialization + HTML report with failure archetypes |
+| `replay.py` | Replay-specific data models for loading historical paired outputs |
+
+### Decision engine (`decision.py`)
+
+Evaluated after every batch:
+
+1. **Hard stops** (checked first): `schema_break_rate >= 0.25` or `high_criticality_failure_rate >= 0.20`
+2. **Proceed gate**: `overall_risk < 0.08` AND latency ratios within bounds AND `min_batches` reached
+3. **Continue**: neither triggered and batches remain
+4. **Final stop**: budget exhausted without proceeding
+
+Overall risk is a weighted average of regression rate, failure rate, schema breaks, high-criticality failures (weighted 2x), and latency penalty.
+
+### Concurrency model
+
+- Prompts within a batch run concurrently via `asyncio.as_completed()`
+- Baseline and candidate for each prompt run in parallel via `asyncio.gather()`
+- Results are re-sorted by original index after completion (preserves deterministic order)
+- Judge calls are sequential per prompt
+
+## Key conventions
+
+- **src layout**: imports are `from driftcut.xxx import yyy`
+- **Config validation at load time**: Pydantic fields have `ge`/`le` constraints; invalid configs fail immediately
+- **`expected_output_type` drives evaluation**: `"json"` checks validity + keys, `"labels"` parses as list, `"free_text"`/`"markdown"` check substrings + length only
+- **Judge strategy enum**: `"none"` (deterministic only), `"light"` (cheap model), `"heavy"` (expensive model), `"tiered"` (light first, escalates to heavy when confidence < `tiered_escalation_threshold`)
+- **Cost error tolerance**: if LiteLLM can't price a model, run continues with `cost_usd=0.0` and `cost_error` stores the message
+- **Conservative defaults**: decision engine favors false negatives (saying STOP) over false positives (saying PROCEED)
+- **B008 suppressed in cli.py**: `typer.Option()` in function defaults is idiomatic Typer
+- **ruff line length**: 100 chars, target Python 3.12
+- **mypy strict mode** enabled
+- **pytest-asyncio auto mode**: async test functions are auto-detected

driftcut-0.8.0/Dockerfile

@@ -0,0 +1,14 @@
+FROM python:3.13-slim
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PIP_NO_CACHE_DIR=1
+
+WORKDIR /workspace
+
+COPY . /workspace
+
+RUN python -m pip install --upgrade pip \
+    && python -m pip install -e ".[dev,redis]"
+
+CMD ["driftcut", "--help"]

driftcut-0.8.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Riccardo Merenda
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.