driftless 0.2.6__tar.gz → 0.2.8__tar.gz

{driftless-0.2.6 → driftless-0.2.8}/CHANGELOG.md

@@ -17,6 +17,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ---
 
+## [0.2.8] - 2026-07-01
+
+### Added
+
+- **P1.1 provider model discovery** — `tools/fetch_provider_models.py` queries
+  OpenAI and Anthropic `/models` APIs and emits new catalog entries only (never
+  overwrites lifecycle on existing ids). The scheduled `refresh-catalog.yml`
+  job merges discoveries when API keys are configured.
+
+---
+
+## [0.2.7] - 2026-07-01
+
+### Added
+
+- **P0.3 per-class support floors** — warn when any class has fewer than five gold
+  examples on a split (`assess_class_support`); surfaced on `migrate` (tuning +
+  holdout), `compare` (baseline + target), CLI "Confidence caveats", and saved
+  compare JSON.
+
+---
+
 ## [0.2.6] - 2026-07-01
 
 ### Added
@@ -142,9 +164,9 @@ First public release on [PyPI](https://pypi.org/project/driftless/0.1.0/).
 - **Docs** — project overview, repair algorithm spec, 2×2 migration methodology,
   Poetry + Dependabot product framing.
 
-[Unreleased]: https://github.com/driftless-dev/driftless/compare/v0.2.6...HEAD
-[0.2.6]: https://github.com/driftless-dev/driftless/releases/tag/v0.2.6
-[0.2.5]: https://github.com/driftless-dev/driftless/compare/v0.2.5...v0.2.6
+[Unreleased]: https://github.com/driftless-dev/driftless/compare/v0.2.8...HEAD
+[0.2.8]: https://github.com/driftless-dev/driftless/releases/tag/v0.2.8
+[0.2.7]: https://github.com/driftless-dev/driftless/compare/v0.2.7...v0.2.8
 [0.2.4]: https://github.com/driftless-dev/driftless/compare/v0.2.4...v0.2.5
 [0.2.3]: https://github.com/driftless-dev/driftless/compare/v0.2.3...v0.2.4
 [0.2.2]: https://github.com/driftless-dev/driftless/compare/v0.2.2...v0.2.3

{driftless-0.2.6 → driftless-0.2.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: driftless
-Version: 0.2.6
+Version: 0.2.8
 Summary: Keep prompts in sync when model or eval data changes — Poetry-style lock regeneration, Dependabot-style PRs.
 Project-URL: Homepage, https://github.com/driftless-dev/driftless
 Project-URL: Repository, https://github.com/driftless-dev/driftless
@@ -133,7 +133,7 @@ can run in CI. See `.github/workflows/` for a scheduled deprecation scan, weekly
 `plan --act` triage, and manually-triggered migration workflows.
 
 ```yaml
-- uses: driftless-dev/driftless@v0.2.6
+- uses: driftless-dev/driftless@v0.2.8
   with:
     command: scan
 ```

{driftless-0.2.6 → driftless-0.2.8}/README.md

@@ -94,7 +94,7 @@ can run in CI. See `.github/workflows/` for a scheduled deprecation scan, weekly
 `plan --act` triage, and manually-triggered migration workflows.
 
 ```yaml
-- uses: driftless-dev/driftless@v0.2.6
+- uses: driftless-dev/driftless@v0.2.8
   with:
     command: scan
 ```

{driftless-0.2.6 → driftless-0.2.8}/docs/RELEASE.md

@@ -153,7 +153,7 @@ After a release, users can pin the composite Action by release tag
 (`action.yml` lives at the repo root — no `/action` path segment):
 
 ```yaml
-- uses: driftless-dev/driftless@v0.2.6
+- uses: driftless-dev/driftless@v0.2.8
   with:
     command: scan
 ```
@@ -161,9 +161,9 @@ After a release, users can pin the composite Action by release tag
 Or pin the PyPI package in the Action input:
 
 ```yaml
-- uses: driftless-dev/driftless@v0.2.6
+- uses: driftless-dev/driftless@v0.2.8
   with:
-    version: "==0.2.6"
+    version: "==0.2.8"
     command: migrate
 ```
 
@@ -171,7 +171,7 @@ Optionally maintain a floating **`v1`** tag on the latest stable minor release
 (point it at the current release tag after each publish):
 
 ```bash
-git tag -f v1 v0.2.6 && git push origin v1 --force
+git tag -f v1 v0.2.8 && git push origin v1 --force
 ```
 
 Update [`action.yml`](../action.yml) default `version` input when cutting releases.

{driftless-0.2.6 → driftless-0.2.8}/site/docs.html

@@ -428,7 +428,7 @@ driftless view -w support_classifier</code></pre>
     <span class="tok-k">runs-on</span>: ubuntu-latest
     <span class="tok-k">steps</span>:
       - <span class="tok-k">uses</span>: actions/checkout@v4
-      - <span class="tok-k">uses</span>: driftless-dev/driftless@v0.2.6
+      - <span class="tok-k">uses</span>: driftless-dev/driftless@v0.2.8
         <span class="tok-k">with</span>:
           <span class="tok-k">command</span>: <span class="tok-s">plan</span></code></pre>
         <p>A scheduled <code class="inline">plan</code> gates CI when a deprecated model needs attention; a manually-triggered <code class="inline">migrate</code> opens a PR (or an issue when blocked) with the evidence attached.</p>

{driftless-0.2.6 → driftless-0.2.8}/src/driftless/__init__.py

@@ -1,3 +1,3 @@
 """driftless: Dependabot for LLM models."""
 
-__version__ = "0.2.6"
+__version__ = "0.2.8"

{driftless-0.2.6 → driftless-0.2.8}/src/driftless/cli.py

@@ -446,6 +446,11 @@ def compare(
 
     console.print(_scorecard(comparison))
 
+    if comparison.warnings:
+        console.print("\n[bold yellow]Confidence caveats[/]:")
+        for w in comparison.warnings:
+            console.print(f"  • {w}")
+
     console.print("\n[bold]Thresholds[/] (target vs contract):")
     if not comparison.checks:
         console.print("  [dim]no thresholds configured[/]")

{driftless-0.2.6 → driftless-0.2.8}/src/driftless/compare.py

@@ -15,7 +15,7 @@ from typing import cast
 
 from .contract import ThresholdsSpec, Workflow
 from .errors import DriftlessError
-from .evaluation import Metrics, evaluate
+from .evaluation import Metrics, assess_class_support, evaluate
 from .harness import run_workflow
 from .progress import log as progress_log
 
@@ -35,6 +35,7 @@ class Comparison:
     baseline: Metrics
     target: Metrics
     checks: list[ThresholdCheck] = field(default_factory=list)
+    warnings: list[str] = field(default_factory=list)
 
     @property
     def passed(self) -> bool:
@@ -218,6 +219,14 @@ def compare_models(
     )
 
     checks = check_thresholds(workflow.thresholds, baseline_metrics, target_metrics)
+    warnings: list[str] = []
+    for metrics, label in (
+        (baseline_metrics, "baseline"),
+        (target_metrics, "target"),
+    ):
+        for w in assess_class_support(metrics, context=f"{label} eval"):
+            if w not in warnings:
+                warnings.append(w)
 
     return Comparison(
         workflow=workflow_name,
@@ -226,6 +235,7 @@ def compare_models(
         baseline=baseline_metrics,
         target=target_metrics,
         checks=checks,
+        warnings=warnings,
     )
 
 
@@ -241,6 +251,7 @@ def save_comparison(comparison: Comparison, cwd: Path | None = None) -> Path:
         "baseline": asdict(comparison.baseline),
         "target": asdict(comparison.target),
         "checks": [asdict(c) for c in comparison.checks],
+        "warnings": comparison.warnings,
         "passed": comparison.passed,
     }
     out_path.write_text(json.dumps(payload, indent=2), encoding="utf-8")

{driftless-0.2.6 → driftless-0.2.8}/src/driftless/engine.py

@@ -30,7 +30,7 @@ from .calibrate import suggest_thresholds
 from .compare import ThresholdCheck, check_thresholds
 from .contract import ThresholdsSpec, Workflow
 from .errors import DriftlessError
-from .evaluation import Metrics, RecordRow, RunAnalysis, analyze, average_metrics
+from .evaluation import Metrics, RecordRow, RunAnalysis, analyze, average_metrics, assess_class_support
 from .harness import run_workflow
 from .progress import log as progress_log
 from .splits import Split, make_splits, materialize_inputs
@@ -593,6 +593,7 @@ def run_migration(
     )
     progress_log("migration: phase 1/3 — baseline prompt on tuning split...")
     baseline_tuning = evaluate_on(current, split.tuning_idx).metrics
+    size_warnings.extend(assess_class_support(baseline_tuning, context="tuning split"))
     progress_log(f"migration: phase 1/3 — baseline F1={_fmt_f1(baseline_tuning.f1)}")
     progress_log("migration: phase 1/3 — current prompt on tuning split...")
     naive_analysis = evaluate_on(target_model, split.tuning_idx)
@@ -605,8 +606,15 @@ def run_migration(
         baseline_holdout = evaluate_on(current, split.holdout_idx).metrics
         holdout_metrics = evaluate_on(target_model, split.holdout_idx, files=files).metrics
         checks = check_thresholds(thresholds, baseline_holdout, holdout_metrics)
+        append_holdout_class_warnings(holdout_metrics)
         return all(c.passed for c in checks), holdout_metrics, checks
 
+    def append_holdout_class_warnings(holdout_metrics: Metrics | None) -> None:
+        if holdout_metrics is not None:
+            size_warnings.extend(
+                assess_class_support(holdout_metrics, context="holdout split")
+            )
+
     # Step: naive target already good? (migrate only -- in refine the model is
     # pinned, so the "naive target" is just the current prompt and there's no
     # model-only change to short-circuit on.)
@@ -858,6 +866,7 @@ def run_migration(
             refine_holdout_checks = check_thresholds(
                 ThresholdsSpec(), baseline_holdout, refine_holdout_metrics
             )
+            append_holdout_class_warnings(refine_holdout_metrics)
         basis = refine_holdout_metrics if refine_holdout_metrics is not None else best_metrics
         suggested = suggest_thresholds(basis)
 

{driftless-0.2.6 → driftless-0.2.8}/src/driftless/evaluation.py

@@ -74,6 +74,33 @@ class ClassMetrics:
     f1: float
 
 
+# Warn when macro-F1 aggregates classes with very few gold examples on a split.
+MIN_CLASS_SUPPORT = 5
+
+
+def assess_class_support(
+    metrics: Metrics,
+    *,
+    context: str,
+    min_support: int = MIN_CLASS_SUPPORT,
+) -> list[str]:
+    """Low-confidence warnings for rare classes in classification metrics."""
+    if metrics.f1 is None or not metrics.per_class or min_support <= 0:
+        return []
+    low = [
+        (name, cm.support)
+        for name, cm in sorted(metrics.per_class.items())
+        if 0 < cm.support < min_support
+    ]
+    if not low:
+        return []
+    bits = ", ".join(f"{name} ({n})" for name, n in low)
+    return [
+        f"Low per-class support on {context}: {bits} — each below {min_support} gold "
+        "examples. Macro-F1 may not reflect rare-class performance."
+    ]
+
+
 @dataclass
 class Metrics:
     n: int

{driftless-0.2.6 → driftless-0.2.8}/tests/test_engine.py

@@ -191,6 +191,7 @@ def test_small_dataset_run_carries_warning(tmp_path: Path):
     wf = _make_workflow(tmp_path)  # 6 examples -> below the min thresholds
     result = run_migration("demo", wf, "weak", generator=StrictGen(), cwd=tmp_path, seed=1)
     assert any("Small dataset" in w for w in result.warnings)
+    assert any("Low per-class support" in w for w in result.warnings)
 
 
 def test_cluster_failures():

{driftless-0.2.6 → driftless-0.2.8}/tests/test_evaluation.py

@@ -309,6 +309,25 @@ def test_id_alignment_duplicate_output_id_raises(tmp_path: Path):
         evaluate(wf, run, cwd=tmp_path)
 
 
+def test_assess_class_support_flags_rare_classes():
+    from driftless.evaluation import ClassMetrics, Metrics, assess_class_support
+
+    metrics = Metrics(
+        n=12,
+        schema_error_rate=0.0,
+        refusal_rate=0.0,
+        f1=0.9,
+        per_class={
+            "billing": ClassMetrics(4, 1.0, 1.0, 1.0),
+            "technical": ClassMetrics(8, 0.9, 0.9, 0.9),
+        },
+    )
+    warnings = assess_class_support(metrics, context="tuning split")
+    assert len(warnings) == 1
+    assert "billing (4)" in warnings[0]
+    assert "tuning split" in warnings[0]
+
+
 def test_load_labels_by_id_rejects_duplicates(tmp_path: Path):
     from driftless.evaluation import load_labels_by_id
 

driftless-0.2.8/tests/test_fetch_provider_models.py

@@ -0,0 +1,111 @@
+import json
+import sys
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "tools"))
+
+import fetch_provider_models as fpm  # noqa: E402
+
+
+def _catalog(models) -> Path:
+    import tempfile
+
+    p = Path(tempfile.mkdtemp()) / "cat.json"
+    p.write_text(json.dumps({"models": models}), encoding="utf-8")
+    return p
+
+
+def test_discover_new_models_skips_known_and_filters_openai(tmp_path):
+    cat = _catalog(
+        [
+            {"model": "gpt-4o", "provider": "openai"},
+            {"model": "claude-3-5-sonnet", "provider": "anthropic"},
+        ]
+    )
+
+    def fake_fetch(_key):
+        return [
+            "gpt-4o",  # known
+            "gpt-5-mini",  # new
+            "ft:gpt-4o:org:123",  # fine-tune — skip
+            "tts-1",  # infra — skip
+            "whisper-1",
+        ]
+
+    updates = fpm.discover_new_models(
+        provider="openai",
+        catalog_path=cat,
+        fetch_ids=fake_fetch,
+        keep=fpm._keep_openai,
+        api_key="k",
+    )
+    assert [u["model"] for u in updates] == ["gpt-5-mini"]
+    assert updates[0]["status"] == "active"
+
+
+def test_discover_new_models_anthropic_claude_only(tmp_path):
+    cat = _catalog([{"model": "claude-3-5-sonnet", "provider": "anthropic"}])
+
+    updates = fpm.discover_new_models(
+        provider="anthropic",
+        catalog_path=cat,
+        fetch_ids=lambda _k: ["claude-3-5-sonnet", "claude-3-7-sonnet", "not-a-model"],
+        keep=fpm._keep_anthropic,
+        api_key="k",
+    )
+    assert [u["model"] for u in updates] == ["claude-3-7-sonnet"]
+
+
+def test_fetch_updates_merges_providers_and_skips_missing_keys(tmp_path, monkeypatch):
+    cat = _catalog([{"model": "gpt-4o", "provider": "openai"}])
+    monkeypatch.delenv("OPENAI_API_KEY", raising=False)
+    monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
+
+    updates = fpm.fetch_updates(["openai", "anthropic"], catalog_path=cat)
+    assert updates == []
+
+
+def test_fetch_updates_openai(monkeypatch, tmp_path):
+    cat = _catalog([{"model": "gpt-4o", "provider": "openai"}])
+    monkeypatch.setenv("OPENAI_API_KEY", "sekret")
+    monkeypatch.setattr(
+        fpm,
+        "_openai_model_ids",
+        lambda key: (["gpt-4o", "o3-mini"] if key == "sekret" else []),
+    )
+
+    updates = fpm.fetch_updates(["openai"], catalog_path=cat)
+    assert [u["model"] for u in updates] == ["o3-mini"]
+
+
+def test_cli_writes_output(tmp_path, monkeypatch):
+    cat = tmp_path / "cat.json"
+    cat.write_text(json.dumps({"models": []}), encoding="utf-8")
+    out = tmp_path / "updates.json"
+    monkeypatch.setattr(
+        fpm,
+        "fetch_updates",
+        lambda providers, catalog_path: [
+            {"model": "gpt-5", "provider": "openai", "status": "active"}
+        ],
+    )
+    assert fpm.main(["--provider", "openai", "--catalog", str(cat), "-o", str(out)]) == 0
+    data = json.loads(out.read_text(encoding="utf-8"))
+    assert data[0]["model"] == "gpt-5"
+
+
+def test_http_get_json_raises_on_http_error(monkeypatch):
+    import urllib.error
+
+    class FakeHTTPError(urllib.error.HTTPError):
+        def __init__(self):
+            super().__init__(url="http://x", code=401, msg="nope", hdrs={}, fp=None)
+
+    def boom(*a, **k):
+        raise FakeHTTPError()
+
+    monkeypatch.setattr(fpm.urllib.request, "urlopen", boom)
+    with pytest.raises(RuntimeError, match="HTTP 401"):
+        fpm._http_get_json("http://x", {})