modelrot 0.1.0__py3-none-any.whl

modelrot/__init__.py

@@ -0,0 +1,5 @@
+"""modelrot — a static audit of the LLM surface of a codebase."""
+
+from .cli import __version__
+
+__all__ = ["__version__"]

modelrot/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())

modelrot/catalog.py

@@ -0,0 +1,75 @@
+"""The snapshot of what each provider says about its own models.
+
+The catalog is data, not opinion: every entry comes from a published
+deprecation page, and every finding modelrot reports carries the URL it came
+from. It is also *dated*. A model retired after ``captured_on`` is not in here,
+and the report says so rather than implying the silence means "fine".
+"""
+
+from __future__ import annotations
+
+import json
+from datetime import date
+from pathlib import Path
+from typing import Dict, Iterable, List, Optional
+
+DATA = Path(__file__).parent / "data"
+
+
+def _load(name: str) -> dict:
+    return json.loads((DATA / name).read_text(encoding="utf-8"))
+
+
+class Catalog:
+    def __init__(self) -> None:
+        self._models: Dict[str, dict] = {m["id"]: m for m in _load("models.json")["models"]}
+        self.sources: dict = _load("sources.json")
+        self.captured_on: date = date.fromisoformat(self.sources["captured_on"])
+
+    def __len__(self) -> int:
+        return len(self._models)
+
+    def get(self, model_id: str) -> Optional[dict]:
+        return self._models.get(model_id)
+
+    @staticmethod
+    def state_of(entry: dict, today: Optional[date] = None) -> str:
+        """What state this model is in, and how that was decided.
+
+        Anthropic publishes a state column, and a published state is taken as
+        given — they are the authority on their own models. OpenAI publishes
+        none, only a shutdown date, so the state is derived from that date at
+        read time. Deriving rather than freezing means a catalog built in March
+        is still correct in October, when the date has passed.
+        """
+        published = entry.get("published_state")
+        if published:
+            return published
+        retirement = entry.get("retirement_date")
+        if not retirement:
+            return "deprecated"
+        return "retired" if date.fromisoformat(retirement) <= (today or date.today()) else "deprecated"
+
+    @staticmethod
+    def state_is_published(entry: dict) -> bool:
+        return bool(entry.get("published_state"))
+
+    def ids(self) -> Iterable[str]:
+        return self._models.keys()
+
+    def source_for(self, provider: str) -> dict:
+        for s in self.sources["sources"]:
+            if s["provider"] == provider:
+                return s
+        return {"url": "", "covers": ""}
+
+    @property
+    def parameters(self) -> List[dict]:
+        return self.sources.get("parameters", [])
+
+    def age_days(self, today: Optional[date] = None) -> int:
+        return ((today or date.today()) - self.captured_on).days
+
+    def sorted_ids_by_length(self) -> List[str]:
+        """Longest first, so ``gpt-4-turbo-2024-04-09`` wins over ``gpt-4``."""
+        return sorted(self._models, key=len, reverse=True)

modelrot/checks.py

@@ -0,0 +1,170 @@
+"""Turning what was found into findings, each one carrying its source.
+
+Severity here means consequence, not confidence:
+
+* ``high``    this is already broken, or will break on a known date soon
+* ``medium``  this has an announced end, far enough away to plan for
+* ``low``     worth knowing, nothing is failing
+* ``note``    inventory
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import date
+from pathlib import Path
+from typing import List, Optional
+
+from .catalog import Catalog
+from .scan import CallSite, Hit, ScanResult
+
+SOON_DAYS = 90
+
+
+@dataclass
+class Finding:
+    severity: str
+    title: str
+    where: str
+    detail: str
+    url: str
+    quote: str = ""
+    inferred: bool = False
+
+
+def _rel(path: Path, root: Path) -> str:
+    try:
+        return str(path.relative_to(root))
+    except ValueError:
+        return str(path)
+
+
+def _days_until(day: Optional[str], today: date) -> Optional[int]:
+    if not day:
+        return None
+    try:
+        return (date.fromisoformat(day) - today).days
+    except ValueError:
+        return None
+
+
+def check_models(result: ScanResult, catalog: Catalog, root: Path, today: date) -> List[Finding]:
+    findings: List[Finding] = []
+    grouped = {}
+    for hit in result.hits:
+        grouped.setdefault(hit.model, []).append(hit)
+
+    for model, hits in sorted(grouped.items()):
+        entry = catalog.get(model)
+        if entry is None:
+            continue
+        state = catalog.state_of(entry, today)
+        if state in ("active", "legacy"):
+            continue
+        # Per-row provenance: the URL and document hash this row was read from,
+        # not a provider-level guess at where it probably came from.
+        url = entry.get("source_url") or catalog.source_for(entry["provider"])["url"]
+        derived = "" if catalog.state_is_published(entry) else (
+            f' {entry["provider"]} publishes the shutdown date rather than a state; this is derived from it.')
+        where = ", ".join(f"{_rel(h.path, root)}:{h.line}" for h in hits[:6])
+        if len(hits) > 6:
+            where += f" (+{len(hits) - 6} more)"
+
+        replacement = entry.get("replacement")
+        swap = f" The provider recommends {replacement}." if replacement else ""
+        left = _days_until(entry.get("retirement_date"), today)
+
+        if state == "retired":
+            findings.append(Finding(
+                severity="high",
+                title=f'"{model}" was retired on {entry["retirement_date"]}',
+                where=where,
+                detail=f"Calls naming this model do not work any more.{swap}{derived}",
+                url=url,
+                quote=f'{entry["provider"]} lists {model} as retired.',
+            ))
+        elif left is not None and left <= SOON_DAYS:
+            findings.append(Finding(
+                severity="high",
+                title=f'"{model}" is switched off in {left} days, on {entry["retirement_date"]}',
+                where=where,
+                detail=f"It still works today. On that date it stops, with no further notice.{swap}{derived}",
+                url=url,
+                quote=f'{entry["provider"]} lists {model} as deprecated, shutting down {entry["retirement_date"]}.',
+            ))
+        else:
+            when = entry.get("retirement_date") or "a date yet to be announced"
+            findings.append(Finding(
+                severity="medium",
+                title=f'"{model}" is deprecated, ending {when}',
+                where=where,
+                detail=f"Nothing is failing yet. This is the window to move.{swap}{derived}",
+                url=url,
+                quote=f'{entry["provider"]} lists {model} as deprecated.',
+            ))
+    return findings
+
+
+def check_parameters(result: ScanResult, catalog: Catalog, root: Path) -> List[Finding]:
+    """Parameters passed to a model that does not accept them.
+
+    Both rules here are the providers' own, and both live in data rather than
+    in code: which models a rule applies to is a matcher name plus a value, so
+    adding a provider is an edit to sources.json.
+    """
+    findings: List[Finding] = []
+    for rule in catalog.parameters:
+        names = set(rule["names"])
+        for call in result.calls:
+            offending = sorted(names & set(call.params))
+            if not offending or not _applies(rule["applies_to"], call.model):
+                continue
+            listed = ", ".join(offending)
+            findings.append(Finding(
+                severity="high",
+                title=f"{listed} passed to {call.model}, which rejects it",
+                where=f"{_rel(call.path, root)}:{min(call.params[n] for n in offending)}",
+                detail=f"{rule['consequence']} {rule['replacement']}",
+                url=rule["url"],
+                quote=rule["quote"],
+            ))
+    return findings
+
+
+def _applies(matcher: dict, model: str) -> bool:
+    kind = matcher.get("kind")
+    if kind == "exact":
+        return model in matcher["value"]
+    if kind == "claude_min_version":
+        return _is_affected_claude(model, tuple(matcher["value"]))
+    return False
+
+
+def _is_affected_claude(model: str, minimum: tuple = (4, 7)) -> bool:
+    """Claude at or past a given version, plus Mythos. Conservative by design."""
+    if model.startswith("claude-mythos"):
+        return True
+    if not model.startswith("claude-"):
+        return False
+    parts = model.split("-")
+    for i, part in enumerate(parts):
+        if part.isdigit() and i + 1 < len(parts) and parts[i + 1].isdigit():
+            major, minor = int(part), int(parts[i + 1])
+            return (major, minor) >= minimum
+        if part.isdigit() and int(part) >= minimum[0] + 1:
+            return True
+    return False
+
+
+def inventory(result: ScanResult, catalog: Catalog, root: Path) -> List[dict]:
+    seen = {}
+    for hit in result.hits:
+        entry = catalog.get(hit.model)
+        row = seen.setdefault(hit.model, {
+            "model": hit.model,
+            "state": catalog.state_of(entry) if entry else "unknown",
+            "provider": entry["provider"] if entry else "unknown",
+            "places": [],
+        })
+        row["places"].append(f"{_rel(hit.path, root)}:{hit.line}")
+    return sorted(seen.values(), key=lambda r: (r["state"] != "retired", r["model"]))

modelrot/cli.py

@@ -0,0 +1,79 @@
+"""modelrot — the model names in your code that the provider already retired."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from datetime import date
+from pathlib import Path
+
+from .catalog import Catalog
+from .checks import check_models, check_parameters, inventory
+from .report import print_catalog_note, print_coverage, print_findings, print_inventory, bold, dim, green
+from .scan import scan
+
+__version__ = "0.1.0"
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="modelrot",
+        description="Find the model names in your code that the provider has already retired, "
+                    "and the parameters that now return a 400.",
+    )
+    parser.add_argument("path", nargs="?", default=".", help="directory to scan (default: the current one)")
+    parser.add_argument("--json", action="store_true", help="machine-readable output")
+    parser.add_argument("--inventory", action="store_true", help="also list every model found, including the healthy ones")
+    parser.add_argument("--prose", action="store_true", help="also search markdown, rst and txt files, where most mentions are discussion rather than use")
+    parser.add_argument("--no-sources", action="store_true", help="omit the quoted provider documentation")
+    parser.add_argument("--fail-on", choices=["high", "medium", "none"], default="high",
+                        help="exit non-zero at this severity or worse (default: high)")
+    parser.add_argument("--version", action="version", version=__version__)
+    return parser
+
+
+def main(argv=None) -> int:
+    args = build_parser().parse_args(argv)
+    root = Path(args.path).resolve()
+    if not root.is_dir():
+        print(f"not a directory: {root}", file=sys.stderr)
+        return 2
+
+    catalog = Catalog()
+    today = date.today()
+    result = scan(root, set(catalog.ids()), catalog.sorted_ids_by_length(), prose=args.prose)
+
+    findings = check_models(result, catalog, root, today) + check_parameters(result, catalog, root)
+    rows = inventory(result, catalog, root)
+
+    if args.json:
+        print(json.dumps({
+            "root": str(root),
+            "catalog": {"captured_on": str(catalog.captured_on), "models": len(catalog), "age_days": catalog.age_days(today)},
+            "findings": [f.__dict__ for f in findings],
+            "inventory": rows,
+        }, indent=2))
+    else:
+        print()
+        print_catalog_note(catalog, today)
+        if findings:
+            print_findings(findings, show_sources=not args.no_sources)
+        else:
+            print(f"{green('No findings.')} {dim('That is not the same as being up to date — here is what was looked at.')}\n")
+        if args.inventory or not findings:
+            print_inventory(rows)
+        print_coverage()
+        counts = {}
+        for f in findings:
+            counts[f.severity] = counts.get(f.severity, 0) + 1
+        summary = ", ".join(f"{counts[s]} {s}" for s in ("high", "medium", "low", "note") if s in counts)
+        if summary:
+            print(f"{bold(summary)}\n")
+
+    threshold = {"high": ["high"], "medium": ["high", "medium"], "none": []}[args.fail_on]
+    return 1 if any(f.severity in threshold for f in findings) else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())