modelbump 0.1.0__tar.gz → 0.2.0__tar.gz

{modelbump-0.1.0 → modelbump-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: modelbump
-Version: 0.1.0
+Version: 0.2.0
 Summary: Find the LLM model IDs in your code and config, and warn before providers retire them.
 Keywords: llm,model deprecation,model retirement,static analysis,github action,openai,anthropic,gemini
 Author: Hammad Hassan
@@ -15,6 +15,7 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Software Development :: Quality Assurance
 Classifier: Typing :: Typed
+Requires-Dist: httpx>=0.28.1
 Requires-Dist: pydantic>=2.13
 Requires-Dist: pyyaml>=6.0.3
 Requires-Dist: tree-sitter>=0.26

{modelbump-0.1.0 → modelbump-0.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "modelbump"
-version = "0.1.0"
+version = "0.2.0"
 description = "Find the LLM model IDs in your code and config, and warn before providers retire them."
 readme = "README.md"
 license = "Apache-2.0"
@@ -28,6 +28,7 @@ classifiers = [
     "Typing :: Typed",
 ]
 dependencies = [
+    "httpx>=0.28.1",
     "pydantic>=2.13",
     "pyyaml>=6.0.3",
     "tree-sitter>=0.26",

{modelbump-0.1.0 → modelbump-0.2.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "modelbump"
-version = "0.1.0"
+version = "0.2.0"
 description = "Find the LLM model IDs in your code and config, and warn before providers retire them."
 readme = "README.md"
 authors = [
@@ -22,6 +22,7 @@ classifiers = [
     "Typing :: Typed",
 ]
 dependencies = [
+    "httpx>=0.28.1",
     "pydantic>=2.13",
     "pyyaml>=6.0.3",
     "tree-sitter>=0.26",

modelbump-0.2.0/src/modelbump/cli.py

@@ -0,0 +1,338 @@
+"""Command-line interface for ModelBump."""
+
+import asyncio
+from datetime import date, datetime
+from pathlib import Path
+from typing import Annotated
+
+import typer
+
+from modelbump import __version__
+from modelbump.registry import Status, load_registry
+from modelbump.runner import (
+    Budget,
+    BudgetExceeded,
+    ChatRequest,
+    ChatResult,
+    ResponseCache,
+    Runner,
+    RunnerError,
+    list_local_models,
+    pin_versions,
+    resolve_model,
+)
+from modelbump.scanner import DEFAULT_SOON_DAYS, scan_path
+from modelbump.scanner.report import (
+    RENDERERS,
+    OutputFormat,
+    build_report,
+    failure_line,
+    summary_line,
+)
+
+app = typer.Typer(
+    name="modelbump",
+    help="Dependabot for LLM models.",
+    no_args_is_help=True,
+)
+
+
+def _version_callback(value: bool) -> None:
+    if value:
+        typer.echo(f"modelbump {__version__}")
+        raise typer.Exit()
+
+
+@app.callback()
+def main(
+    version: Annotated[
+        bool,
+        typer.Option(
+            "--version",
+            "-V",
+            help="Show the version and exit.",
+            callback=_version_callback,
+            is_eager=True,
+        ),
+    ] = False,
+) -> None:
+    """Dependabot for LLM models."""
+
+
+@app.command()
+def scan(
+    path: Annotated[
+        Path,
+        typer.Argument(help="Folder or file to scan.", exists=True, resolve_path=True),
+    ] = Path("."),
+    soon_days: Annotated[
+        int,
+        typer.Option(
+            min=0, help="Mark a model RETIRING_SOON when it shuts down within this many days."
+        ),
+    ] = DEFAULT_SOON_DAYS,
+    fail_within: Annotated[
+        int | None,
+        typer.Option(
+            min=0,
+            help="Exit with code 1 if a model is retired or shuts down within this many days.",
+        ),
+    ] = None,
+    today: Annotated[
+        datetime | None,
+        typer.Option(
+            formats=["%Y-%m-%d"],
+            help="Check as if today were this day (YYYY-MM-DD), to replay the past.",
+        ),
+    ] = None,
+    output_format: Annotated[
+        OutputFormat,
+        typer.Option("--format", help="Report format: text, json or sarif."),
+    ] = OutputFormat.TEXT,
+    exclude: Annotated[
+        list[str] | None,
+        typer.Option(
+            help="Skip paths matching this pattern, e.g. 'tests' or '*.ipynb'. Repeat for more.",
+        ),
+    ] = None,
+    skip_comments: Annotated[
+        bool,
+        typer.Option(
+            help="Leave out IDs that only describe a model: comments, docstrings, "
+            "sentences and price tables.",
+        ),
+    ] = False,
+    skip_tests: Annotated[
+        bool,
+        typer.Option(help="Leave out test files and test folders."),
+    ] = False,
+    output: Annotated[
+        Path | None,
+        typer.Option(help="Write the report to this file instead of the screen.", dir_okay=False),
+    ] = None,
+) -> None:
+    """Find the LLM model IDs in a repository's code and config, and how urgent each is."""
+    registry = load_registry()
+    day = today.date() if today is not None else date.today()
+    result = scan_path(path, registry, exclude or (), skip_comments, skip_tests)
+    report = build_report(result, registry, day, soon_days, fail_within, _root_uri(path))
+    rendered = RENDERERS[output_format](report)
+    if output is None:
+        typer.echo(rendered, nl=False)
+    else:
+        output.write_text(rendered, encoding="utf-8")
+        typer.echo(f"Wrote {output_format.value} report to {output}.")
+        typer.echo(summary_line(report))
+    failure = failure_line(report)
+    if failure is not None:
+        if output is not None or output_format is not OutputFormat.TEXT:
+            # Keep the screen output a valid JSON or SARIF document.
+            typer.echo(failure, err=True)
+        raise typer.Exit(code=1)
+
+
+def _root_uri(path: Path) -> str:
+    """The scan root relative to the working directory, for SARIF paths."""
+    base = path.parent if path.is_file() else path
+    try:
+        relative = base.relative_to(Path.cwd()).as_posix()
+    except ValueError:
+        return ""
+    return "" if relative == "." else relative + "/"
+
+
+@app.command()
+def check(
+    model_id: Annotated[str, typer.Argument(help="Model ID, e.g. claude-3-haiku-20240307.")],
+) -> None:
+    """Look up one model ID in the retirement registry."""
+    found = load_registry().get(model_id)
+    if found is None:
+        typer.echo(f"{model_id}: no retirement announced (not in the registry).")
+        return
+    provider, model = found
+    status = model.status_on(date.today())
+    if status is Status.RETIRED:
+        line = f"RETIRED since {model.shutdown}"
+        if model.shutdown_is_earliest:
+            line += " (the earliest possible date the provider gave)"
+    elif model.shutdown_is_earliest:
+        line = f"{status.value.upper()}, shuts down no earlier than {model.shutdown}"
+    else:
+        line = f"{status.value.upper()}, shuts down on {model.shutdown}"
+    typer.echo(f"{model_id} ({provider.value}): {line}.")
+    if model.superseded:
+        typer.echo(f"Rescheduled: {len(model.superseded)} earlier plan(s) were replaced.")
+    if model.replacement:
+        typer.echo(f"Suggested replacement: {model.replacement}")
+
+
+@app.command()
+def ask(
+    prompts: Annotated[
+        list[str], typer.Argument(help="What to ask the model. Give several to ask them all.")
+    ],
+    model: Annotated[
+        str,
+        typer.Option("--model", "-m", help="endpoint/model, e.g. ollama/qwen2.5:7b."),
+    ] = "ollama/qwen2.5:7b",
+    system: Annotated[str | None, typer.Option(help="A system message.")] = None,
+    temperature: Annotated[float, typer.Option(min=0.0, help="Sampling temperature.")] = 0.0,
+    max_tokens: Annotated[
+        int | None, typer.Option(min=1, help="Longest answer, in tokens.")
+    ] = None,
+    seed: Annotated[
+        int | None, typer.Option(help="Sampling seed, if the model supports it.")
+    ] = None,
+    sample: Annotated[
+        int, typer.Option(min=0, help="Which repeated draw this is (0, 1, 2, ...).")
+    ] = 0,
+    timeout: Annotated[float, typer.Option(min=1.0, help="Seconds each attempt may take.")] = 120.0,
+    attempts: Annotated[int, typer.Option(min=1, help="Tries before giving up.")] = 4,
+    use_cache: Annotated[
+        bool,
+        typer.Option("--cache/--no-cache", help="Reuse a stored answer to the same request."),
+    ] = True,
+    budget_tokens: Annotated[
+        int | None,
+        typer.Option(min=1, help="Stop sending requests after this many tokens (in + out)."),
+    ] = None,
+    budget_requests: Annotated[
+        int | None,
+        typer.Option(min=1, help="Stop after this many requests to the model, retries included."),
+    ] = None,
+    think: Annotated[
+        str | None,
+        typer.Option(
+            help="Reasoning effort for thinking models: none, low, medium or high. "
+            "Leave it out for the model's default."
+        ),
+    ] = None,
+) -> None:
+    """Send prompts to a model and print the answers (to try the runner)."""
+    requests = [
+        ChatRequest.ask(
+            model,
+            prompt,
+            system,
+            temperature=temperature,
+            max_tokens=max_tokens,
+            seed=seed,
+            sample=sample,
+            reasoning_effort=think,
+        )
+        for prompt in prompts
+    ]
+    try:
+        endpoint, _ = resolve_model(model)
+        requests = pin_versions(requests)
+    except ValueError as error:
+        typer.echo(f"Error: {error}", err=True)
+        raise typer.Exit(code=1) from error
+    except RunnerError as error:
+        _print_error(error, model)
+        raise typer.Exit(code=1) from error
+    if not endpoint.local:
+        typer.echo(
+            f"Note: {endpoint.name} is a free cloud tier. Your prompts leave this machine "
+            "and may be kept or read by the provider. Never send private data.",
+            err=True,
+        )
+    cache = ResponseCache() if use_cache else None
+    budget = Budget(max_tokens=budget_tokens, max_requests=budget_requests)
+    runner = Runner(timeout=timeout, max_attempts=attempts, cache=cache, budget=budget)
+    try:
+        outcomes = asyncio.run(runner.run_each(requests))
+    finally:
+        if cache is not None:
+            cache.close()
+    failed = False
+    for prompt, outcome in zip(prompts, outcomes, strict=True):
+        if len(prompts) > 1:
+            typer.echo(f"## {prompt}")
+        if isinstance(outcome, RunnerError):
+            failed = True
+            _print_error(outcome, model)
+            continue
+        typer.echo(outcome.text)
+        typer.echo(_result_line(outcome), err=True)
+    if len(prompts) > 1 or budget_tokens is not None or budget_requests is not None:
+        typer.echo(budget.summary(), err=True)
+    if any(isinstance(o, BudgetExceeded) for o in outcomes):
+        typer.echo(
+            "Answers so far are cached: run the same command with a higher limit to continue.",
+            err=True,
+        )
+    if failed:
+        raise typer.Exit(code=1)
+
+
+def _print_error(error: RunnerError, model: str) -> None:
+    typer.echo(f"Error: {error}", err=True)
+    if model.startswith("ollama/") and "could not reach" in str(error):
+        typer.echo("Is Ollama running? Start it with: open -a Ollama", err=True)
+
+
+@app.command()
+def models() -> None:
+    """List the models pulled into Ollama, with the digest that pins each one."""
+    try:
+        found = list_local_models()
+    except RunnerError as error:
+        _print_error(error, "ollama/")
+        raise typer.Exit(code=1) from error
+    if not found:
+        typer.echo("No models yet. Pull one with: ollama pull qwen2.5:7b")
+        return
+    width = max(len(m.name) for m in found)
+    typer.echo(f"{'MODEL':<{width + 7}}  PARAMS  QUANT    SIZE     DIGEST")
+    for m in found:
+        size = f"{m.size_bytes / 1e9:.1f} GB"
+        typer.echo(
+            f"{'ollama/' + m.name:<{width + 7}}  {m.parameter_size:<6}  {m.quantization:<7}  "
+            f"{size:<7}  {m.digest[:12]}"
+        )
+
+
+def _result_line(result: ChatResult) -> str:
+    if result.cached:
+        how = "from cache"
+    else:
+        tries = "attempt" if result.attempts == 1 else "attempts"
+        how = f"{result.attempts} {tries}, {result.seconds:.1f}s"
+    return (
+        f"[{result.model}: {result.input_tokens} tokens in, {result.output_tokens} out, "
+        f"{how}, finish: {result.finish_reason}]"
+    )
+
+
+cache_app = typer.Typer(help="Look at or empty the cache of model answers.", no_args_is_help=True)
+app.add_typer(cache_app, name="cache")
+
+
+@cache_app.command("info")
+def cache_info() -> None:
+    """Show where the cache is and how much it holds."""
+    cache = ResponseCache()
+    try:
+        stats = cache.stats()
+    finally:
+        cache.close()
+    typer.echo(f"Cache: {stats.path}")
+    typer.echo(f"Answers stored: {stats.entries} ({stats.output_tokens} output tokens)")
+    typer.echo(f"Size on disk: {stats.size_bytes / 1024:.1f} KiB")
+
+
+@cache_app.command("clear")
+def cache_clear(
+    yes: Annotated[bool, typer.Option("--yes", "-y", help="Do not ask for confirmation.")] = False,
+) -> None:
+    """Delete every stored answer."""
+    cache = ResponseCache()
+    try:
+        if not yes:
+            typer.confirm(f"Delete every answer stored in {cache.path}?", abort=True)
+        deleted = cache.clear()
+    finally:
+        cache.close()
+    typer.echo(f"Deleted {deleted} stored answers.")

modelbump-0.2.0/src/modelbump/runner/__init__.py

@@ -0,0 +1,30 @@
+"""Runner: call LLMs with retries, a cache and a budget (ADRs 0036-0038)."""
+
+from modelbump.runner.backends import ENDPOINTS, Endpoint, resolve_model
+from modelbump.runner.budget import Budget, BudgetExceeded, Spend
+from modelbump.runner.cache import CacheStats, ResponseCache, cache_key, default_cache_path
+from modelbump.runner.ollama import LocalModel, find_model, list_local_models, pin_versions
+from modelbump.runner.run import Runner
+from modelbump.runner.types import ChatRequest, ChatResult, Message, RunnerError
+
+__all__ = [
+    "ENDPOINTS",
+    "Budget",
+    "BudgetExceeded",
+    "CacheStats",
+    "ChatRequest",
+    "ChatResult",
+    "Endpoint",
+    "LocalModel",
+    "Message",
+    "ResponseCache",
+    "Runner",
+    "RunnerError",
+    "Spend",
+    "cache_key",
+    "default_cache_path",
+    "find_model",
+    "list_local_models",
+    "pin_versions",
+    "resolve_model",
+]

modelbump-0.2.0/src/modelbump/runner/backends.py

@@ -0,0 +1,92 @@
+"""Where each model is served.
+
+Every endpoint speaks the OpenAI-compatible chat completions API, so one
+small HTTP client covers all of them (ADR 0036). Only free endpoints are
+listed: local models through Ollama, and free cloud tiers (ADR 0040).
+"""
+
+import os
+from dataclasses import dataclass
+from typing import Any
+
+from modelbump.runner.types import ChatRequest, RunnerError
+
+
+@dataclass(frozen=True)
+class Endpoint:
+    """An OpenAI-compatible server.
+
+    - `api_key_env`: the environment variable that holds the key, if one is needed.
+    - `rpm`: requests per minute to stay under; the runner spaces requests to
+      this endpoint so a free tier's per-minute limit is never hit.
+    - `local`: True when the model runs on this machine. Prompts sent to a
+      cloud endpoint leave the machine, and free tiers may keep and read them.
+    """
+
+    name: str
+    base_url: str
+    api_key_env: str | None = None
+    rpm: float | None = None
+    local: bool = False
+
+    def headers(self) -> dict[str, str]:
+        """HTTP headers, with the API key read from the environment if one is needed."""
+        if self.api_key_env is None:
+            return {}
+        key = os.environ.get(self.api_key_env)
+        if not key:
+            raise RunnerError(f"Set {self.api_key_env} to use the {self.name} endpoint.")
+        return {"Authorization": f"Bearer {key}"}
+
+
+# Ollama listens on port 11434 of this machine unless MODELBUMP_OLLAMA_URL says otherwise.
+OLLAMA_URL = os.environ.get("MODELBUMP_OLLAMA_URL", "http://localhost:11434").rstrip("/")
+
+# Free tiers only. Limits change often: the rpm values sit at or below the
+# free limits checked on 2026-10-27 (ADR 0040). Their keys are free and need no card.
+ENDPOINTS: dict[str, Endpoint] = {
+    "ollama": Endpoint("ollama", f"{OLLAMA_URL}/v1", local=True),
+    "groq": Endpoint("groq", "https://api.groq.com/openai/v1", "GROQ_API_KEY", rpm=30),
+    "gemini": Endpoint(
+        "gemini",
+        "https://generativelanguage.googleapis.com/v1beta/openai",
+        "GEMINI_API_KEY",
+        rpm=10,
+    ),
+    "openrouter": Endpoint(
+        "openrouter", "https://openrouter.ai/api/v1", "OPENROUTER_API_KEY", rpm=20
+    ),
+}
+
+
+def resolve_model(model: str) -> tuple[Endpoint, str]:
+    """Split `endpoint/model` into the endpoint and the model name it expects.
+
+    Only the first `/` splits, so model names may contain `/` themselves.
+    """
+    name, sep, rest = model.partition("/")
+    if not sep or not rest:
+        raise ValueError(
+            f"Write the model as endpoint/model, e.g. ollama/qwen2.5:7b (got {model!r})."
+        )
+    if name not in ENDPOINTS:
+        known = ", ".join(sorted(ENDPOINTS))
+        raise ValueError(f"Unknown endpoint {name!r} in {model!r}. Known: {known}.")
+    return ENDPOINTS[name], rest
+
+
+def request_body(request: ChatRequest, model_name: str) -> dict[str, Any]:
+    """The JSON body for POST /chat/completions."""
+    body: dict[str, Any] = {
+        "model": model_name,
+        "messages": [{"role": m.role, "content": m.content} for m in request.messages],
+        "temperature": request.temperature,
+        "stream": False,
+    }
+    if request.max_tokens is not None:
+        body["max_tokens"] = request.max_tokens
+    if request.seed is not None:
+        body["seed"] = request.seed
+    if request.reasoning_effort is not None:
+        body["reasoning_effort"] = request.reasoning_effort
+    return body

modelbump-0.2.0/src/modelbump/runner/budget.py

@@ -0,0 +1,109 @@
+"""A spending limit for model calls, counted in tokens and requests (ADR 0038).
+
+ModelBump spends no money: local models cost time and free API tiers allow
+a fixed number of requests and tokens a day. So the budget counts tokens and
+HTTP requests, not dollars. Answers read from the cache cost nothing and are
+counted separately, as savings.
+"""
+
+from dataclasses import dataclass
+
+from modelbump.runner.types import ChatResult, RunnerError
+
+
+class BudgetExceeded(RunnerError):
+    """The budget ran out before this request could be sent."""
+
+
+@dataclass(frozen=True)
+class Spend:
+    """What a run used so far."""
+
+    requests: int
+    answers: int
+    input_tokens: int
+    output_tokens: int
+    cached_answers: int
+    cached_tokens: int
+
+    @property
+    def tokens(self) -> int:
+        return self.input_tokens + self.output_tokens
+
+
+class Budget:
+    """Stops sending requests once a limit is reached.
+
+    - `max_tokens`: input plus output tokens of real (not cached) answers.
+    - `max_requests`: HTTP requests sent, retries included, because free
+      tiers count every request.
+
+    A limit is checked before each request is sent. Requests already in
+    flight are allowed to finish, so the token total can end slightly above
+    `max_tokens` (by at most what the in-flight answers use).
+    """
+
+    def __init__(self, max_tokens: int | None = None, max_requests: int | None = None) -> None:
+        if max_tokens is not None and max_tokens < 1:
+            raise ValueError("max_tokens must be at least 1.")
+        if max_requests is not None and max_requests < 1:
+            raise ValueError("max_requests must be at least 1.")
+        self.max_tokens = max_tokens
+        self.max_requests = max_requests
+        self._requests = 0
+        self._answers = 0
+        self._input = 0
+        self._output = 0
+        self._cached_answers = 0
+        self._cached_tokens = 0
+
+    def before_request(self) -> None:
+        """Count one request about to be sent, or raise BudgetExceeded."""
+        if self.max_requests is not None and self._requests >= self.max_requests:
+            raise BudgetExceeded(
+                f"Budget reached: {self._requests:,} of {self.max_requests:,} requests sent."
+            )
+        if self.max_tokens is not None and self._input + self._output >= self.max_tokens:
+            raise BudgetExceeded(
+                f"Budget reached: {self._input + self._output:,} of "
+                f"{self.max_tokens:,} tokens used."
+            )
+        self._requests += 1
+
+    def record(self, result: ChatResult) -> None:
+        """Count an answer: real ones use the budget, cached ones are savings."""
+        if result.cached:
+            self._cached_answers += 1
+            self._cached_tokens += result.input_tokens + result.output_tokens
+            return
+        self._answers += 1
+        self._input += result.input_tokens
+        self._output += result.output_tokens
+
+    @property
+    def spend(self) -> Spend:
+        return Spend(
+            requests=self._requests,
+            answers=self._answers,
+            input_tokens=self._input,
+            output_tokens=self._output,
+            cached_answers=self._cached_answers,
+            cached_tokens=self._cached_tokens,
+        )
+
+    def summary(self) -> str:
+        """One line for people, e.g. at the end of a run."""
+        s = self.spend
+        line = (
+            f"Spent: {s.requests:,} requests, {s.tokens:,} tokens "
+            f"({s.input_tokens:,} in, {s.output_tokens:,} out). "
+            f"From cache: {s.cached_answers:,} answers ({s.cached_tokens:,} tokens saved)."
+        )
+        limits = []
+        if self.max_tokens is not None:
+            limits.append(f"{s.tokens / self.max_tokens:.0%} of {self.max_tokens:,} tokens")
+        if self.max_requests is not None:
+            limits.append(f"{s.requests / self.max_requests:.0%} of {self.max_requests:,} requests")
+        if limits:
+            line += " Budget used: " + ", ".join(limits) + "."
+        return line

modelbump-0.2.0/src/modelbump/runner/cache.py

@@ -0,0 +1,136 @@
+"""A content-addressed cache of model answers, in one SQLite file (ADR 0037).
+
+The key is a SHA-256 hash of everything that can change an answer: the model,
+the full message list, the sampling options and the sample number. Asking the
+same thing twice reads the first answer back instead of calling the model, so
+experiments can be re-run for free and give the same numbers.
+"""
+
+import hashlib
+import json
+import os
+import sqlite3
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+
+from modelbump.runner.types import ChatRequest, ChatResult
+
+# Bump this when the key recipe changes, so old entries are never misread.
+# 2: adds reasoning_effort and model_version (ADR 0039).
+KEY_VERSION = 2
+
+_SCHEMA = """
+CREATE TABLE IF NOT EXISTS responses (
+    key TEXT PRIMARY KEY,
+    model TEXT NOT NULL,
+    text TEXT NOT NULL,
+    input_tokens INTEGER NOT NULL,
+    output_tokens INTEGER NOT NULL,
+    finish_reason TEXT NOT NULL,
+    created_at TEXT NOT NULL
+)
+"""
+
+
+def default_cache_path() -> Path:
+    """`$MODELBUMP_CACHE_DIR/responses.sqlite`, else the user's cache folder."""
+    folder = os.environ.get("MODELBUMP_CACHE_DIR")
+    if folder:
+        return Path(folder) / "responses.sqlite"
+    base = os.environ.get("XDG_CACHE_HOME") or Path.home() / ".cache"
+    return Path(base) / "modelbump" / "responses.sqlite"
+
+
+def cache_key(request: ChatRequest) -> str:
+    """The SHA-256 hex digest of the request's canonical JSON form."""
+    payload = {
+        "v": KEY_VERSION,
+        "model": request.model,
+        "messages": [[m.role, m.content] for m in request.messages],
+        "temperature": request.temperature,
+        "max_tokens": request.max_tokens,
+        "seed": request.seed,
+        "sample": request.sample,
+        "reasoning_effort": request.reasoning_effort,
+        "model_version": request.model_version,
+    }
+    canonical = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
+    return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
+
+
+@dataclass(frozen=True)
+class CacheStats:
+    """What the cache holds."""
+
+    path: Path
+    entries: int
+    output_tokens: int
+    size_bytes: int
+
+
+class ResponseCache:
+    """Stores successful answers by request hash. Errors are never cached."""
+
+    def __init__(self, path: Path | None = None) -> None:
+        self.path = path if path is not None else default_cache_path()
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self._db = sqlite3.connect(self.path)
+        # WAL lets two processes read and write the same cache safely.
+        self._db.execute("PRAGMA journal_mode=WAL")
+        self._db.execute(_SCHEMA)
+        self._db.commit()
+
+    def get(self, request: ChatRequest) -> ChatResult | None:
+        """The stored answer for this request, or None."""
+        row = self._db.execute(
+            "SELECT text, input_tokens, output_tokens, finish_reason FROM responses WHERE key = ?",
+            (cache_key(request),),
+        ).fetchone()
+        if row is None:
+            return None
+        text, input_tokens, output_tokens, finish_reason = row
+        return ChatResult(
+            text=text,
+            model=request.model,
+            input_tokens=input_tokens,
+            output_tokens=output_tokens,
+            finish_reason=finish_reason,
+            attempts=0,
+            seconds=0.0,
+            cached=True,
+        )
+
+    def put(self, request: ChatRequest, result: ChatResult) -> None:
+        """Store an answer. A second answer for the same request replaces the first."""
+        self._db.execute(
+            "INSERT OR REPLACE INTO responses VALUES (?, ?, ?, ?, ?, ?, ?)",
+            (
+                cache_key(request),
+                request.model,
+                result.text,
+                result.input_tokens,
+                result.output_tokens,
+                result.finish_reason,
+                datetime.now(UTC).isoformat(timespec="seconds"),
+            ),
+        )
+        self._db.commit()
+
+    def stats(self) -> CacheStats:
+        """How many answers are stored, and how big the file is."""
+        entries, tokens = self._db.execute(
+            "SELECT COUNT(*), COALESCE(SUM(output_tokens), 0) FROM responses"
+        ).fetchone()
+        size = sum(p.stat().st_size for p in (self.path, Path(f"{self.path}-wal")) if p.exists())
+        return CacheStats(self.path, entries, tokens, size)
+
+    def clear(self) -> int:
+        """Delete every stored answer. Returns how many were deleted."""
+        deleted = self._db.execute("DELETE FROM responses").rowcount
+        self._db.commit()
+        self._db.execute("VACUUM")
+        return deleted
+
+    def close(self) -> None:
+        self._db.close()

modelbump-0.2.0/src/modelbump/runner/ollama.py

@@ -0,0 +1,84 @@
+"""Ask the local Ollama server which models it has, and their exact versions (ADR 0039)."""
+
+from dataclasses import dataclass, replace
+
+import httpx
+
+from modelbump.runner.backends import OLLAMA_URL
+from modelbump.runner.types import ChatRequest, RunnerError
+
+
+@dataclass(frozen=True)
+class LocalModel:
+    """A model pulled into Ollama."""
+
+    name: str
+    digest: str
+    size_bytes: int
+    family: str
+    parameter_size: str
+    quantization: str
+
+
+def list_local_models(
+    base_url: str = OLLAMA_URL, transport: httpx.BaseTransport | None = None
+) -> list[LocalModel]:
+    """Every model Ollama has, sorted by name (GET /api/tags)."""
+    try:
+        with httpx.Client(timeout=10.0, transport=transport) as client:
+            response = client.get(f"{base_url}/api/tags")
+    except httpx.HTTPError as error:
+        raise RunnerError(f"could not reach {base_url} ({type(error).__name__})") from error
+    if response.status_code != 200:
+        raise RunnerError(f"{base_url}/api/tags answered HTTP {response.status_code}")
+    models = []
+    for item in response.json().get("models") or []:
+        details = item.get("details") or {}
+        models.append(
+            LocalModel(
+                name=str(item.get("name") or item.get("model") or ""),
+                digest=str(item.get("digest") or ""),
+                size_bytes=int(item.get("size") or 0),
+                family=str(details.get("family") or ""),
+                parameter_size=str(details.get("parameter_size") or ""),
+                quantization=str(details.get("quantization_level") or ""),
+            )
+        )
+    return sorted(models, key=lambda m: m.name)
+
+
+def find_model(name: str, models: list[LocalModel]) -> LocalModel | None:
+    """The local model called `name`. A name without a tag means `:latest`, as in Ollama."""
+    wanted = name if ":" in name else f"{name}:latest"
+    for model in models:
+        if model.name == wanted:
+            return model
+    return None
+
+
+def pin_versions(
+    requests: list[ChatRequest],
+    base_url: str = OLLAMA_URL,
+    transport: httpx.BaseTransport | None = None,
+) -> list[ChatRequest]:
+    """Fill in `model_version` with the Ollama digest of every `ollama/...` request.
+
+    Then a model re-pulled with new weights under the same name gets new cache
+    entries instead of old answers. A model that is not pulled is an error that
+    says how to pull it. Other endpoints are left as they are.
+    """
+    if not any(r.model.startswith("ollama/") for r in requests):
+        return list(requests)
+    models = list_local_models(base_url, transport)
+    pinned = []
+    for request in requests:
+        if request.model.startswith("ollama/"):
+            name = request.model.removeprefix("ollama/")
+            found = find_model(name, models)
+            if found is None:
+                raise RunnerError(
+                    f"{request.model}: this model is not pulled. Run: ollama pull {name}"
+                )
+            request = replace(request, model_version=found.digest)
+        pinned.append(request)
+    return pinned

modelbump-0.2.0/src/modelbump/runner/run.py

@@ -0,0 +1,211 @@
+"""Send chat requests with a timeout, retries with backoff, and a concurrency limit."""
+
+import asyncio
+import random
+import re
+import time
+from collections.abc import Awaitable, Callable, Sequence
+from typing import Any
+
+import httpx
+
+from modelbump.runner.backends import Endpoint, request_body, resolve_model
+from modelbump.runner.budget import Budget
+from modelbump.runner.cache import ResponseCache
+from modelbump.runner.types import ChatRequest, ChatResult, RunnerError
+
+# Statuses worth another try: rate limits and server-side failures.
+RETRY_STATUSES = frozenset({408, 429, 500, 502, 503, 504})
+
+Sleep = Callable[[float], Awaitable[None]]
+Clock = Callable[[], float]
+
+# Some thinking models put their reasoning at the start of the answer.
+_THINK = re.compile(r"^\s*<think>.*?</think>\s*", re.DOTALL)
+
+
+class Runner:
+    """Calls models. One Runner can be reused for many requests.
+
+    - `timeout`: seconds each attempt may take.
+    - `max_attempts`: tries per request, counting the first one.
+    - `base_delay`, `max_delay`: exponential backoff with full jitter, in seconds.
+      A `Retry-After` header from the server wins when it asks for longer.
+    - `concurrency`: requests in flight at once in `run_many`.
+    - `cache`: answers are read from it first and written to it after a
+      successful call (ADR 0037). None means every request calls the model.
+    - `budget`: counts tokens and requests, and stops sending requests once
+      a limit is reached (ADR 0038). None means no limit.
+
+    Requests to an endpoint with an `rpm` are spaced at least 60/rpm seconds
+    apart, retries included (ADR 0040).
+    """
+
+    def __init__(
+        self,
+        *,
+        timeout: float = 120.0,
+        max_attempts: int = 4,
+        base_delay: float = 1.0,
+        max_delay: float = 30.0,
+        concurrency: int = 2,
+        cache: ResponseCache | None = None,
+        budget: Budget | None = None,
+        transport: httpx.AsyncBaseTransport | None = None,
+        sleep: Sleep = asyncio.sleep,
+        clock: Clock = time.monotonic,
+        rng: random.Random | None = None,
+    ) -> None:
+        if max_attempts < 1:
+            raise ValueError("max_attempts must be at least 1.")
+        if concurrency < 1:
+            raise ValueError("concurrency must be at least 1.")
+        self.timeout = timeout
+        self.max_attempts = max_attempts
+        self.base_delay = base_delay
+        self.max_delay = max_delay
+        self.concurrency = concurrency
+        self.cache = cache
+        self.budget = budget
+        self._transport = transport
+        self._sleep = sleep
+        self._clock = clock
+        self._next_slot: dict[str, float] = {}
+        self._rng = rng or random.Random()
+
+    async def run(self, request: ChatRequest) -> ChatResult:
+        """Send one request."""
+        async with self._client() as client:
+            return await self._run(client, request)
+
+    async def run_many(self, requests: Sequence[ChatRequest]) -> list[ChatResult]:
+        """Send many requests, at most `concurrency` at a time. Results keep the input order.
+
+        Every request is tried, so all answers that succeed are cached. Then
+        the first error, if any, is raised.
+        """
+        outcomes = await self.run_each(requests)
+        for outcome in outcomes:
+            if isinstance(outcome, RunnerError):
+                raise outcome
+        return [o for o in outcomes if isinstance(o, ChatResult)]
+
+    async def run_each(self, requests: Sequence[ChatRequest]) -> list[ChatResult | RunnerError]:
+        """Like `run_many`, but each failed request gives its error in place of a result."""
+        gate = asyncio.Semaphore(self.concurrency)
+        async with self._client() as client:
+
+            async def one(request: ChatRequest) -> ChatResult | RunnerError:
+                async with gate:
+                    try:
+                        return await self._run(client, request)
+                    except RunnerError as error:
+                        return error
+
+            return list(await asyncio.gather(*(one(r) for r in requests)))
+
+    def _client(self) -> httpx.AsyncClient:
+        return httpx.AsyncClient(timeout=self.timeout, transport=self._transport)
+
+    async def _run(self, client: httpx.AsyncClient, request: ChatRequest) -> ChatResult:
+        if self.cache is not None:
+            cached = self.cache.get(request)
+            if cached is not None:
+                if self.budget is not None:
+                    self.budget.record(cached)
+                return cached
+        result = await self._call(client, request)
+        if self.budget is not None:
+            self.budget.record(result)
+        if self.cache is not None:
+            self.cache.put(request, result)
+        return result
+
+    async def _call(self, client: httpx.AsyncClient, request: ChatRequest) -> ChatResult:
+        endpoint, model_name = resolve_model(request.model)
+        url = f"{endpoint.base_url}/chat/completions"
+        body = request_body(request, model_name)
+        headers = endpoint.headers()
+        start = time.monotonic()
+        for attempt in range(1, self.max_attempts + 1):
+            wait: float | None = None
+            if self.budget is not None:
+                self.budget.before_request()
+            await self._pace(endpoint)
+            try:
+                response = await client.post(url, json=body, headers=headers)
+            except httpx.TimeoutException:
+                reason = f"timed out after {self.timeout:g}s"
+            except httpx.TransportError as error:
+                reason = f"could not reach {endpoint.base_url} ({type(error).__name__})"
+            else:
+                if response.status_code == 200:
+                    return _parse(response.json(), request.model, attempt, time.monotonic() - start)
+                reason = f"HTTP {response.status_code}: {_error_text(response)}"
+                if response.status_code not in RETRY_STATUSES:
+                    raise RunnerError(f"{request.model}: {reason}")
+                wait = _retry_after(response)
+            if attempt == self.max_attempts:
+                raise RunnerError(f"{request.model}: {reason} (gave up after {attempt} attempts)")
+            await self._sleep(self._backoff(attempt, wait))
+        raise AssertionError("unreachable")  # pragma: no cover
+
+    async def _pace(self, endpoint: Endpoint) -> None:
+        """Wait for this endpoint's next free slot, so requests stay under its rpm."""
+        if endpoint.rpm is None:
+            return
+        now = self._clock()
+        slot = max(now, self._next_slot.get(endpoint.name, now))
+        # Book the slot before waiting, so concurrent requests queue up behind it.
+        self._next_slot[endpoint.name] = slot + 60.0 / endpoint.rpm
+        if slot > now:
+            await self._sleep(slot - now)
+
+    def _backoff(self, attempt: int, retry_after: float | None) -> float:
+        """Full jitter: a random wait up to base * 2^(attempt-1), capped at max_delay."""
+        ceiling = min(self.max_delay, self.base_delay * 2 ** (attempt - 1))
+        delay = self._rng.uniform(0, ceiling)
+        if retry_after is not None:
+            delay = max(delay, min(retry_after, self.max_delay))
+        return delay
+
+
+def _parse(data: dict[str, Any], model: str, attempts: int, seconds: float) -> ChatResult:
+    try:
+        choice = data["choices"][0]
+        text = _THINK.sub("", choice["message"]["content"] or "", count=1)
+    except (KeyError, IndexError, TypeError) as error:
+        raise RunnerError(f"{model}: unexpected response shape: {data!r:.200}") from error
+    usage = data.get("usage") or {}
+    return ChatResult(
+        text=text,
+        model=model,
+        input_tokens=int(usage.get("prompt_tokens", 0)),
+        output_tokens=int(usage.get("completion_tokens", 0)),
+        finish_reason=str(choice.get("finish_reason") or "unknown"),
+        attempts=attempts,
+        seconds=seconds,
+    )
+
+
+def _retry_after(response: httpx.Response) -> float | None:
+    value = response.headers.get("retry-after")
+    if value is None:
+        return None
+    try:
+        return max(0.0, float(value))
+    except ValueError:
+        return None  # an HTTP date; fall back to our own backoff
+
+
+def _error_text(response: httpx.Response) -> str:
+    try:
+        data = response.json()
+    except ValueError:
+        return response.text[:200] or "no body"
+    error = data.get("error") if isinstance(data, dict) else None
+    if isinstance(error, dict):
+        return str(error.get("message", error))[:200]
+    if error is not None:
+        return str(error)[:200]
+    return str(data)[:200]

modelbump-0.2.0/src/modelbump/runner/types.py

@@ -0,0 +1,78 @@
+"""What goes into a model call and what comes back."""
+
+from dataclasses import dataclass
+from typing import Literal
+
+Role = Literal["system", "user", "assistant"]
+
+
+class RunnerError(Exception):
+    """A call failed for good: a non-retryable error, or every attempt failed."""
+
+
+@dataclass(frozen=True)
+class Message:
+    """One chat message."""
+
+    role: Role
+    content: str
+
+
+@dataclass(frozen=True)
+class ChatRequest:
+    """One model call.
+
+    `model` is written as `endpoint/model`, for example `ollama/qwen2.5:7b`.
+    Everything that can change the answer lives here, so the cache can hash a
+    request into its key (ADR 0037). `sample` numbers repeated draws of the
+    same request (0, 1, 2, ...): it is part of the cache key but is not sent
+    to the model. `reasoning_effort` turns a thinking model's reasoning down
+    or off (`none`, `low`, `medium`, `high`). `model_version` names the exact
+    weights, such as an Ollama digest (ADR 0039); it is part of the cache key
+    but is not sent either.
+    """
+
+    model: str
+    messages: tuple[Message, ...]
+    temperature: float = 0.0
+    max_tokens: int | None = None
+    seed: int | None = None
+    sample: int = 0
+    reasoning_effort: str | None = None
+    model_version: str = ""
+
+    @classmethod
+    def ask(
+        cls,
+        model: str,
+        prompt: str,
+        system: str | None = None,
+        *,
+        temperature: float = 0.0,
+        max_tokens: int | None = None,
+        seed: int | None = None,
+        sample: int = 0,
+        reasoning_effort: str | None = None,
+    ) -> "ChatRequest":
+        """A request with one user message and an optional system message."""
+        messages = [Message("user", prompt)]
+        if system is not None:
+            messages.insert(0, Message("system", system))
+        return cls(model, tuple(messages), temperature, max_tokens, seed, sample, reasoning_effort)
+
+
+@dataclass(frozen=True)
+class ChatResult:
+    """The model's answer and what it took to get it.
+
+    A result read from the cache has `cached=True`, `attempts=0` and `seconds=0`.
+    """
+
+    text: str
+    model: str
+    input_tokens: int
+    output_tokens: int
+    finish_reason: str
+    attempts: int
+    seconds: float
+    cached: bool = False

modelbump-0.1.0/src/modelbump/cli.py

@@ -1,154 +0,0 @@
-"""Command-line interface for ModelBump."""
-
-from datetime import date, datetime
-from pathlib import Path
-from typing import Annotated
-
-import typer
-
-from modelbump import __version__
-from modelbump.registry import Status, load_registry
-from modelbump.scanner import DEFAULT_SOON_DAYS, scan_path
-from modelbump.scanner.report import (
-    RENDERERS,
-    OutputFormat,
-    build_report,
-    failure_line,
-    summary_line,
-)
-
-app = typer.Typer(
-    name="modelbump",
-    help="Dependabot for LLM models.",
-    no_args_is_help=True,
-)
-
-
-def _version_callback(value: bool) -> None:
-    if value:
-        typer.echo(f"modelbump {__version__}")
-        raise typer.Exit()
-
-
-@app.callback()
-def main(
-    version: Annotated[
-        bool,
-        typer.Option(
-            "--version",
-            "-V",
-            help="Show the version and exit.",
-            callback=_version_callback,
-            is_eager=True,
-        ),
-    ] = False,
-) -> None:
-    """Dependabot for LLM models."""
-
-
-@app.command()
-def scan(
-    path: Annotated[
-        Path,
-        typer.Argument(help="Folder or file to scan.", exists=True, resolve_path=True),
-    ] = Path("."),
-    soon_days: Annotated[
-        int,
-        typer.Option(
-            min=0, help="Mark a model RETIRING_SOON when it shuts down within this many days."
-        ),
-    ] = DEFAULT_SOON_DAYS,
-    fail_within: Annotated[
-        int | None,
-        typer.Option(
-            min=0,
-            help="Exit with code 1 if a model is retired or shuts down within this many days.",
-        ),
-    ] = None,
-    today: Annotated[
-        datetime | None,
-        typer.Option(
-            formats=["%Y-%m-%d"],
-            help="Check as if today were this day (YYYY-MM-DD), to replay the past.",
-        ),
-    ] = None,
-    output_format: Annotated[
-        OutputFormat,
-        typer.Option("--format", help="Report format: text, json or sarif."),
-    ] = OutputFormat.TEXT,
-    exclude: Annotated[
-        list[str] | None,
-        typer.Option(
-            help="Skip paths matching this pattern, e.g. 'tests' or '*.ipynb'. Repeat for more.",
-        ),
-    ] = None,
-    skip_comments: Annotated[
-        bool,
-        typer.Option(
-            help="Leave out IDs that only describe a model: comments, docstrings, "
-            "sentences and price tables.",
-        ),
-    ] = False,
-    skip_tests: Annotated[
-        bool,
-        typer.Option(help="Leave out test files and test folders."),
-    ] = False,
-    output: Annotated[
-        Path | None,
-        typer.Option(help="Write the report to this file instead of the screen.", dir_okay=False),
-    ] = None,
-) -> None:
-    """Find the LLM model IDs in a repository's code and config, and how urgent each is."""
-    registry = load_registry()
-    day = today.date() if today is not None else date.today()
-    result = scan_path(path, registry, exclude or (), skip_comments, skip_tests)
-    report = build_report(result, registry, day, soon_days, fail_within, _root_uri(path))
-    rendered = RENDERERS[output_format](report)
-    if output is None:
-        typer.echo(rendered, nl=False)
-    else:
-        output.write_text(rendered, encoding="utf-8")
-        typer.echo(f"Wrote {output_format.value} report to {output}.")
-        typer.echo(summary_line(report))
-    failure = failure_line(report)
-    if failure is not None:
-        if output is not None or output_format is not OutputFormat.TEXT:
-            # Keep the screen output a valid JSON or SARIF document.
-            typer.echo(failure, err=True)
-        raise typer.Exit(code=1)
-
-
-def _root_uri(path: Path) -> str:
-    """The scan root relative to the working directory, for SARIF paths."""
-    base = path.parent if path.is_file() else path
-    try:
-        relative = base.relative_to(Path.cwd()).as_posix()
-    except ValueError:
-        return ""
-    return "" if relative == "." else relative + "/"
-
-
-@app.command()
-def check(
-    model_id: Annotated[str, typer.Argument(help="Model ID, e.g. claude-3-haiku-20240307.")],
-) -> None:
-    """Look up one model ID in the retirement registry."""
-    found = load_registry().get(model_id)
-    if found is None:
-        typer.echo(f"{model_id}: no retirement announced (not in the registry).")
-        return
-    provider, model = found
-    status = model.status_on(date.today())
-    if status is Status.RETIRED:
-        line = f"RETIRED since {model.shutdown}"
-        if model.shutdown_is_earliest:
-            line += " (the earliest possible date the provider gave)"
-    elif model.shutdown_is_earliest:
-        line = f"{status.value.upper()}, shuts down no earlier than {model.shutdown}"
-    else:
-        line = f"{status.value.upper()}, shuts down on {model.shutdown}"
-    typer.echo(f"{model_id} ({provider.value}): {line}.")
-    if model.superseded:
-        typer.echo(f"Rescheduled: {len(model.superseded)} earlier plan(s) were replaced.")
-    if model.replacement:
-        typer.echo(f"Suggested replacement: {model.replacement}")