driftless 0.2.6__tar.gz → 0.2.7__tar.gz

{driftless-0.2.6 → driftless-0.2.7}/CHANGELOG.md

@@ -17,6 +17,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ---
 
+## [0.2.7] - 2026-07-01
+
+### Added
+
+- **P0.3 per-class support floors** — warn when any class has fewer than five gold
+  examples on a split (`assess_class_support`); surfaced on `migrate` (tuning +
+  holdout), `compare` (baseline + target), CLI "Confidence caveats", and saved
+  compare JSON.
+
+---
+
 ## [0.2.6] - 2026-07-01
 
 ### Added
@@ -142,9 +153,9 @@ First public release on [PyPI](https://pypi.org/project/driftless/0.1.0/).
 - **Docs** — project overview, repair algorithm spec, 2×2 migration methodology,
   Poetry + Dependabot product framing.
 
-[Unreleased]: https://github.com/driftless-dev/driftless/compare/v0.2.6...HEAD
-[0.2.6]: https://github.com/driftless-dev/driftless/releases/tag/v0.2.6
-[0.2.5]: https://github.com/driftless-dev/driftless/compare/v0.2.5...v0.2.6
+[Unreleased]: https://github.com/driftless-dev/driftless/compare/v0.2.7...HEAD
+[0.2.7]: https://github.com/driftless-dev/driftless/releases/tag/v0.2.7
+[0.2.6]: https://github.com/driftless-dev/driftless/compare/v0.2.6...v0.2.7
 [0.2.4]: https://github.com/driftless-dev/driftless/compare/v0.2.4...v0.2.5
 [0.2.3]: https://github.com/driftless-dev/driftless/compare/v0.2.3...v0.2.4
 [0.2.2]: https://github.com/driftless-dev/driftless/compare/v0.2.2...v0.2.3

{driftless-0.2.6 → driftless-0.2.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: driftless
-Version: 0.2.6
+Version: 0.2.7
 Summary: Keep prompts in sync when model or eval data changes — Poetry-style lock regeneration, Dependabot-style PRs.
 Project-URL: Homepage, https://github.com/driftless-dev/driftless
 Project-URL: Repository, https://github.com/driftless-dev/driftless
@@ -133,7 +133,7 @@ can run in CI. See `.github/workflows/` for a scheduled deprecation scan, weekly
 `plan --act` triage, and manually-triggered migration workflows.
 
 ```yaml
-- uses: driftless-dev/driftless@v0.2.6
+- uses: driftless-dev/driftless@v0.2.7
   with:
     command: scan
 ```

{driftless-0.2.6 → driftless-0.2.7}/README.md

@@ -94,7 +94,7 @@ can run in CI. See `.github/workflows/` for a scheduled deprecation scan, weekly
 `plan --act` triage, and manually-triggered migration workflows.
 
 ```yaml
-- uses: driftless-dev/driftless@v0.2.6
+- uses: driftless-dev/driftless@v0.2.7
   with:
     command: scan
 ```

{driftless-0.2.6 → driftless-0.2.7}/docs/RELEASE.md

@@ -153,7 +153,7 @@ After a release, users can pin the composite Action by release tag
 (`action.yml` lives at the repo root — no `/action` path segment):
 
 ```yaml
-- uses: driftless-dev/driftless@v0.2.6
+- uses: driftless-dev/driftless@v0.2.7
   with:
     command: scan
 ```
@@ -161,9 +161,9 @@ After a release, users can pin the composite Action by release tag
 Or pin the PyPI package in the Action input:
 
 ```yaml
-- uses: driftless-dev/driftless@v0.2.6
+- uses: driftless-dev/driftless@v0.2.7
   with:
-    version: "==0.2.6"
+    version: "==0.2.7"
     command: migrate
 ```
 
@@ -171,7 +171,7 @@ Optionally maintain a floating **`v1`** tag on the latest stable minor release
 (point it at the current release tag after each publish):
 
 ```bash
-git tag -f v1 v0.2.6 && git push origin v1 --force
+git tag -f v1 v0.2.7 && git push origin v1 --force
 ```
 
 Update [`action.yml`](../action.yml) default `version` input when cutting releases.

{driftless-0.2.6 → driftless-0.2.7}/site/docs.html

@@ -428,7 +428,7 @@ driftless view -w support_classifier</code></pre>
     <span class="tok-k">runs-on</span>: ubuntu-latest
     <span class="tok-k">steps</span>:
       - <span class="tok-k">uses</span>: actions/checkout@v4
-      - <span class="tok-k">uses</span>: driftless-dev/driftless@v0.2.6
+      - <span class="tok-k">uses</span>: driftless-dev/driftless@v0.2.7
         <span class="tok-k">with</span>:
           <span class="tok-k">command</span>: <span class="tok-s">plan</span></code></pre>
         <p>A scheduled <code class="inline">plan</code> gates CI when a deprecated model needs attention; a manually-triggered <code class="inline">migrate</code> opens a PR (or an issue when blocked) with the evidence attached.</p>

{driftless-0.2.6 → driftless-0.2.7}/src/driftless/__init__.py

@@ -1,3 +1,3 @@
 """driftless: Dependabot for LLM models."""
 
-__version__ = "0.2.6"
+__version__ = "0.2.7"

{driftless-0.2.6 → driftless-0.2.7}/src/driftless/cli.py

@@ -446,6 +446,11 @@ def compare(
 
     console.print(_scorecard(comparison))
 
+    if comparison.warnings:
+        console.print("\n[bold yellow]Confidence caveats[/]:")
+        for w in comparison.warnings:
+            console.print(f"  • {w}")
+
     console.print("\n[bold]Thresholds[/] (target vs contract):")
     if not comparison.checks:
         console.print("  [dim]no thresholds configured[/]")

{driftless-0.2.6 → driftless-0.2.7}/src/driftless/compare.py

@@ -15,7 +15,7 @@ from typing import cast
 
 from .contract import ThresholdsSpec, Workflow
 from .errors import DriftlessError
-from .evaluation import Metrics, evaluate
+from .evaluation import Metrics, assess_class_support, evaluate
 from .harness import run_workflow
 from .progress import log as progress_log
 
@@ -35,6 +35,7 @@ class Comparison:
     baseline: Metrics
     target: Metrics
     checks: list[ThresholdCheck] = field(default_factory=list)
+    warnings: list[str] = field(default_factory=list)
 
     @property
     def passed(self) -> bool:
@@ -218,6 +219,14 @@ def compare_models(
     )
 
     checks = check_thresholds(workflow.thresholds, baseline_metrics, target_metrics)
+    warnings: list[str] = []
+    for metrics, label in (
+        (baseline_metrics, "baseline"),
+        (target_metrics, "target"),
+    ):
+        for w in assess_class_support(metrics, context=f"{label} eval"):
+            if w not in warnings:
+                warnings.append(w)
 
     return Comparison(
         workflow=workflow_name,
@@ -226,6 +235,7 @@ def compare_models(
         baseline=baseline_metrics,
         target=target_metrics,
         checks=checks,
+        warnings=warnings,
     )
 
 
@@ -241,6 +251,7 @@ def save_comparison(comparison: Comparison, cwd: Path | None = None) -> Path:
         "baseline": asdict(comparison.baseline),
         "target": asdict(comparison.target),
         "checks": [asdict(c) for c in comparison.checks],
+        "warnings": comparison.warnings,
         "passed": comparison.passed,
     }
     out_path.write_text(json.dumps(payload, indent=2), encoding="utf-8")

{driftless-0.2.6 → driftless-0.2.7}/src/driftless/engine.py

@@ -30,7 +30,7 @@ from .calibrate import suggest_thresholds
 from .compare import ThresholdCheck, check_thresholds
 from .contract import ThresholdsSpec, Workflow
 from .errors import DriftlessError
-from .evaluation import Metrics, RecordRow, RunAnalysis, analyze, average_metrics
+from .evaluation import Metrics, RecordRow, RunAnalysis, analyze, average_metrics, assess_class_support
 from .harness import run_workflow
 from .progress import log as progress_log
 from .splits import Split, make_splits, materialize_inputs
@@ -593,6 +593,7 @@ def run_migration(
     )
     progress_log("migration: phase 1/3 — baseline prompt on tuning split...")
     baseline_tuning = evaluate_on(current, split.tuning_idx).metrics
+    size_warnings.extend(assess_class_support(baseline_tuning, context="tuning split"))
     progress_log(f"migration: phase 1/3 — baseline F1={_fmt_f1(baseline_tuning.f1)}")
     progress_log("migration: phase 1/3 — current prompt on tuning split...")
     naive_analysis = evaluate_on(target_model, split.tuning_idx)
@@ -605,8 +606,15 @@ def run_migration(
         baseline_holdout = evaluate_on(current, split.holdout_idx).metrics
         holdout_metrics = evaluate_on(target_model, split.holdout_idx, files=files).metrics
         checks = check_thresholds(thresholds, baseline_holdout, holdout_metrics)
+        append_holdout_class_warnings(holdout_metrics)
         return all(c.passed for c in checks), holdout_metrics, checks
 
+    def append_holdout_class_warnings(holdout_metrics: Metrics | None) -> None:
+        if holdout_metrics is not None:
+            size_warnings.extend(
+                assess_class_support(holdout_metrics, context="holdout split")
+            )
+
     # Step: naive target already good? (migrate only -- in refine the model is
     # pinned, so the "naive target" is just the current prompt and there's no
     # model-only change to short-circuit on.)
@@ -858,6 +866,7 @@ def run_migration(
             refine_holdout_checks = check_thresholds(
                 ThresholdsSpec(), baseline_holdout, refine_holdout_metrics
             )
+            append_holdout_class_warnings(refine_holdout_metrics)
         basis = refine_holdout_metrics if refine_holdout_metrics is not None else best_metrics
         suggested = suggest_thresholds(basis)
 

{driftless-0.2.6 → driftless-0.2.7}/src/driftless/evaluation.py

@@ -74,6 +74,33 @@ class ClassMetrics:
     f1: float
 
 
+# Warn when macro-F1 aggregates classes with very few gold examples on a split.
+MIN_CLASS_SUPPORT = 5
+
+
+def assess_class_support(
+    metrics: Metrics,
+    *,
+    context: str,
+    min_support: int = MIN_CLASS_SUPPORT,
+) -> list[str]:
+    """Low-confidence warnings for rare classes in classification metrics."""
+    if metrics.f1 is None or not metrics.per_class or min_support <= 0:
+        return []
+    low = [
+        (name, cm.support)
+        for name, cm in sorted(metrics.per_class.items())
+        if 0 < cm.support < min_support
+    ]
+    if not low:
+        return []
+    bits = ", ".join(f"{name} ({n})" for name, n in low)
+    return [
+        f"Low per-class support on {context}: {bits} — each below {min_support} gold "
+        "examples. Macro-F1 may not reflect rare-class performance."
+    ]
+
+
 @dataclass
 class Metrics:
     n: int

{driftless-0.2.6 → driftless-0.2.7}/tests/test_engine.py

@@ -191,6 +191,7 @@ def test_small_dataset_run_carries_warning(tmp_path: Path):
     wf = _make_workflow(tmp_path)  # 6 examples -> below the min thresholds
     result = run_migration("demo", wf, "weak", generator=StrictGen(), cwd=tmp_path, seed=1)
     assert any("Small dataset" in w for w in result.warnings)
+    assert any("Low per-class support" in w for w in result.warnings)
 
 
 def test_cluster_failures():

{driftless-0.2.6 → driftless-0.2.7}/tests/test_evaluation.py

@@ -309,6 +309,25 @@ def test_id_alignment_duplicate_output_id_raises(tmp_path: Path):
         evaluate(wf, run, cwd=tmp_path)
 
 
+def test_assess_class_support_flags_rare_classes():
+    from driftless.evaluation import ClassMetrics, Metrics, assess_class_support
+
+    metrics = Metrics(
+        n=12,
+        schema_error_rate=0.0,
+        refusal_rate=0.0,
+        f1=0.9,
+        per_class={
+            "billing": ClassMetrics(4, 1.0, 1.0, 1.0),
+            "technical": ClassMetrics(8, 0.9, 0.9, 0.9),
+        },
+    )
+    warnings = assess_class_support(metrics, context="tuning split")
+    assert len(warnings) == 1
+    assert "billing (4)" in warnings[0]
+    assert "tuning split" in warnings[0]
+
+
 def test_load_labels_by_id_rejects_duplicates(tmp_path: Path):
     from driftless.evaluation import load_labels_by_id