misterdev 0.2.0__py3-none-any.whl

misterdev/core/economics/free_models.py

@@ -0,0 +1,108 @@
+"""Harvest OpenRouter's rotating free models.
+
+OpenRouter publishes a public model list whose ``:free`` entries (zero prompt
+and completion price) rotate over time. This fetches and day-caches that list
+so the selection policy can offer free models as cheap-tier candidates.
+
+Reliability is preserved upstream of this module: a free model only ever
+appears on non-final attempts, is subject to the same proven-first-try gate as
+any cheap model, and (when a paid failover is configured) degrades to the paid
+provider on rate-limit/outage. So harvesting can only reduce cost, never lower
+the quality floor enforced by the validation gates.
+"""
+
+import json
+from pathlib import Path
+from typing import Callable, Dict, List, Optional
+
+from misterdev.logging_setup import setup_logger
+
+logger = setup_logger(__name__)
+
+_MODELS_URL = "https://openrouter.ai/api/v1/models"
+_DAY_SECONDS = 24 * 3600
+
+# Per-process cache of fetched list endpoints, keyed by URL. Free-model
+# harvesting, the model catalog, and each failover client's catalog all hit the
+# same /models endpoint; this collapses those into one network round-trip per
+# URL per run. Successful results only (failures re-raise without caching).
+_FETCH_CACHE: Dict[str, list] = {}
+
+
+def _http_fetch(url: str = _MODELS_URL) -> list:
+    """Fetch and return the ``data`` array from an OpenRouter list endpoint.
+
+    Shared by free-model harvesting, the model catalog, and embedding-model
+    discovery — the network boundary lives in exactly one place.
+    """
+    if url in _FETCH_CACHE:
+        return _FETCH_CACHE[url]
+    import urllib.request
+
+    req = urllib.request.Request(url, headers={"User-Agent": "misterdev"})
+    with urllib.request.urlopen(req, timeout=10) as resp:
+        payload = json.loads(resp.read().decode("utf-8"))
+    data = payload.get("data", []) if isinstance(payload, dict) else []
+    _FETCH_CACHE[url] = data
+    return data
+
+
+def _is_free(model: dict) -> bool:
+    pricing = model.get("pricing") or {}
+
+    def _zero(value) -> bool:
+        try:
+            return float(value) == 0.0
+        except (TypeError, ValueError):
+            return False
+
+    # A model is free only when both prompt and completion are explicitly zero.
+    return (
+        "prompt" in pricing
+        and "completion" in pricing
+        and _zero(pricing.get("prompt"))
+        and _zero(pricing.get("completion"))
+    )
+
+
+class FreeModelCache:
+    """Day-cached list of free OpenRouter model ids, backed by a JSON file."""
+
+    def __init__(self, path: Path, fetcher: Optional[Callable[[], list]] = None):
+        self.path = Path(path)
+        self._fetcher = fetcher or _http_fetch
+
+    def _load(self) -> Optional[dict]:
+        if not self.path.exists():
+            return None
+        try:
+            data = json.loads(self.path.read_text(encoding="utf-8"))
+            return data if isinstance(data, dict) else None
+        except (json.JSONDecodeError, OSError):
+            return None
+
+    def _save(self, data: dict) -> None:
+        from misterdev.utils.file_utils import atomic_write_json
+
+        atomic_write_json(self.path, data, indent=2, sort_keys=True)
+
+    def get(self, now: float, max_age_seconds: int = _DAY_SECONDS) -> List[str]:
+        """Return current free model ids, refetching when the cache is stale.
+
+        On a fetch failure, falls back to the last cached list (possibly empty)
+        so a transient network problem never blocks a build.
+        """
+        cached = self._load()
+        if cached and (now - cached.get("fetched_at", 0)) < max_age_seconds:
+            return list(cached.get("models", []))
+        try:
+            raw = self._fetcher()
+        except Exception as e:
+            logger.warning(f"Free-model fetch failed, using cached list: {e}")
+            return list(cached.get("models", [])) if cached else []
+        free = sorted(
+            m["id"] for m in raw if isinstance(m, dict) and m.get("id") and _is_free(m)
+        )
+        self._save({"fetched_at": now, "models": free})
+        logger.info(f"Harvested {len(free)} free OpenRouter models")
+        return free

misterdev/core/economics/llm_cache.py

@@ -0,0 +1,105 @@
+"""Content-addressed memoization of gate-passing LLM outputs.
+
+Only outputs that cleared the validation gates are stored, keyed by a hash of
+the full prompt (system + user). Because the executor's prompt already embeds
+the current file contents, interface contracts, and strategy, the key changes
+whenever any input that affects the correct output changes — so a stale edit
+auto-invalidates. A cache hit is still re-applied through the gates, so even a
+collision or staleness can only cost a wasted gate run, never ship bad code.
+
+The store is model-agnostic on purpose: a cheap (or free) model's successful
+result is reusable regardless of which model would be picked next time.
+"""
+
+import hashlib
+import json
+import os
+from pathlib import Path
+from typing import Optional
+
+from misterdev.logging_setup import setup_logger
+
+logger = setup_logger(__name__)
+
+# Cap on stored entries. One file per entry would otherwise grow without bound
+# across many builds; past the cap the oldest entries (by mtime) are evicted.
+# A re-applied hit only ever saves a gate run, so dropping cold entries is safe.
+DEFAULT_MAX_ENTRIES = 2000
+
+
+class LLMCache:
+    """One JSON file per cache entry under a directory, bounded by mtime LRU."""
+
+    def __init__(self, dir_path: Path, max_entries: int = DEFAULT_MAX_ENTRIES):
+        self.dir = Path(dir_path)
+        self.max_entries = max_entries
+
+    @staticmethod
+    def _key(system_prompt: str, prompt: str) -> str:
+        h = hashlib.sha256()
+        h.update((system_prompt or "").encode("utf-8"))
+        h.update(b"\x00")
+        h.update((prompt or "").encode("utf-8"))
+        return h.hexdigest()
+
+    def _path(self, key: str) -> Path:
+        return self.dir / f"{key}.json"
+
+    def get(self, system_prompt: str, prompt: str) -> Optional[str]:
+        """Return a cached output for this prompt, or None on miss."""
+        path = self._path(self._key(system_prompt, prompt))
+        if not path.exists():
+            return None
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+        except (json.JSONDecodeError, OSError) as e:
+            logger.warning(f"Ignoring unreadable cache entry {path.name}: {e}")
+            return None
+        output = data.get("output")
+        return output if isinstance(output, str) else None
+
+    def put(
+        self,
+        system_prompt: str,
+        prompt: str,
+        output: str,
+        model: str = "",
+        timestamp: float = 0.0,
+    ) -> None:
+        """Store a gate-passing output (atomic write)."""
+        from misterdev.utils.file_utils import atomic_write_json
+
+        key = self._key(system_prompt, prompt)
+        path = self._path(key)
+        atomic_write_json(
+            path,
+            {"output": output, "model": model, "created_at": timestamp, "key": key},
+            indent=2,
+        )
+        self._evict_if_needed()
+
+    def _evict_if_needed(self) -> None:
+        """Drop the oldest entries (by mtime) once the cap is exceeded.
+
+        Best-effort: any filesystem error degrades to leaving the cache as-is
+        rather than raising into the build, like the rest of this module.
+        """
+        if self.max_entries <= 0:
+            return
+        try:
+            entries = [
+                (e.stat().st_mtime, e.path)
+                for e in os.scandir(self.dir)
+                if e.name.endswith(".json") and e.is_file()
+            ]
+        except OSError:
+            return
+        excess = len(entries) - self.max_entries
+        if excess <= 0:
+            return
+        entries.sort(key=lambda t: t[0])
+        for _mtime, victim in entries[:excess]:
+            try:
+                os.remove(victim)
+            except OSError:
+                continue

misterdev/core/economics/model_catalog.py

@@ -0,0 +1,79 @@
+"""Per-model capability profiles from OpenRouter's model list.
+
+Different models accept different parameters: a reasoning model rejects
+``temperature``, an o-series model wants ``max_completion_tokens`` instead of
+``max_tokens``, and only some models support a ``reasoning`` effort budget.
+OpenRouter's ``/api/v1/models`` response carries a ``supported_parameters``
+array, ``context_length``, and ``top_provider.max_completion_tokens`` per model
+— this reads them so a request only ever sends parameters a model accepts.
+
+Fetched once per process and cached in memory; failures degrade to an empty
+catalog (callers then fall back to their existing default parameters).
+"""
+
+from dataclasses import dataclass
+from typing import Callable, Dict, FrozenSet, Optional
+
+from misterdev.core.economics.free_models import _http_fetch
+from misterdev.logging_setup import setup_logger
+
+logger = setup_logger(__name__)
+
+
+@dataclass(frozen=True)
+class ModelProfile:
+    id: str
+    supported_parameters: FrozenSet[str]
+    max_completion_tokens: Optional[int] = None
+    context_length: Optional[int] = None
+
+    def supports(self, param: str) -> bool:
+        return param in self.supported_parameters
+
+    @property
+    def supports_reasoning(self) -> bool:
+        return (
+            "reasoning" in self.supported_parameters
+            or "reasoning_effort" in self.supported_parameters
+        )
+
+
+def _parse(raw: list) -> Dict[str, ModelProfile]:
+    profiles: Dict[str, ModelProfile] = {}
+    for m in raw:
+        if not isinstance(m, dict):
+            continue
+        model_id = m.get("id")
+        if not model_id:
+            continue
+        params = m.get("supported_parameters") or []
+        top = m.get("top_provider")
+        max_out = top.get("max_completion_tokens") if isinstance(top, dict) else None
+        profiles[model_id] = ModelProfile(
+            id=model_id,
+            supported_parameters=frozenset(p for p in params if isinstance(p, str)),
+            max_completion_tokens=max_out,
+            context_length=m.get("context_length"),
+        )
+    return profiles
+
+
+class ModelCatalog:
+    """In-memory, lazily-fetched map of model id -> ModelProfile."""
+
+    def __init__(self, fetcher: Optional[Callable[[], list]] = None):
+        self._fetcher = fetcher or _http_fetch
+        self._profiles: Optional[Dict[str, ModelProfile]] = None
+
+    def _ensure(self) -> Dict[str, ModelProfile]:
+        if self._profiles is None:
+            try:
+                self._profiles = _parse(self._fetcher())
+            except Exception as e:
+                logger.warning(f"Model catalog fetch failed; no profiles: {e}")
+                self._profiles = {}
+        return self._profiles
+
+    def profile(self, model_id: str) -> Optional[ModelProfile]:
+        """Profile for a model, or None when unknown (caller uses its defaults)."""
+        return self._ensure().get(model_id)

misterdev/core/economics/model_ledger.py

@@ -0,0 +1,331 @@
+"""Persistent model-performance ledger.
+
+Records per-(model, category, complexity) outcomes across runs so model
+selection can be data-driven instead of static or random. Stored as JSON under
+``.orchestrator/model_stats.json`` alongside the other run artifacts.
+
+The ledger only stores and scores; the selection policy lives in
+``core.model_selector``. Quality is scored by the Wilson lower confidence bound
+on the gate-pass rate (a conservative estimate that does not let a 1/1 model
+outrank a 95/100 one), and exploration of under-sampled models is handled by an
+optimistic UCB term so new or freshly-rotated models still get tried.
+"""
+
+import json
+import math
+import threading
+import time
+from dataclasses import asdict, dataclass, fields, replace
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
+
+from misterdev.logging_setup import setup_logger
+
+logger = setup_logger(__name__)
+
+# z for a ~95% confidence interval, used by both the Wilson lower bound
+# (conservative quality) and the UCB exploration bonus (optimistic).
+_Z = 1.96
+
+# Half-life for time-decaying accumulated outcomes (seconds). Models drift —
+# yesterday's strong model can regress after a provider update — so older
+# observations are exponentially down-weighted toward zero, letting recent
+# outcomes dominate quality/selection. 30 days balances stability against
+# responsiveness; <= 0 disables decay (all history weighed equally).
+_DEFAULT_HALF_LIFE_SECONDS = 30 * 86400.0
+
+# Dead-band below which no decay is applied. Decay is meant to fade data ACROSS
+# runs (hours/days apart), not within a single build where the same model is
+# scored several times seconds apart. Holding the factor at exactly 1.0 below an
+# hour keeps the within-build effective counts whole integers, so the selector's
+# integer sample thresholds (e.g. "proven after >= 3 first tries") aren't tripped
+# by sub-second drift. At a 30-day half-life an hour's decay is negligible anyway.
+_MIN_DECAY_ELAPSED_SECONDS = 3600.0
+
+
+def _decay_factor(elapsed: float, half_life: float) -> float:
+    """Exponential decay multiplier for ``elapsed`` seconds at ``half_life``.
+
+    1.0 for non-positive elapsed (no time passed, or a clock that went backward),
+    for elapsed within the sub-hour dead-band, or when decay is disabled
+    (``half_life`` <= 0); else ``0.5 ** (elapsed / half_life)`` in (0, 1).
+    """
+    if half_life <= 0 or elapsed < _MIN_DECAY_ELAPSED_SECONDS:
+        return 1.0
+    return 0.5 ** (elapsed / half_life)
+
+
+# Field separator for the composite stat key. Unit-separator char: never
+# appears in a model id, category, or complexity.
+_SEP = "␟"
+
+
+@dataclass
+class ModelStat:
+    """Aggregated outcomes for one (model, category, complexity) cell."""
+
+    model: str
+    category: str = ""
+    complexity: str = ""
+    # Effective (time-decayed) counts: floats, not raw integers, because older
+    # observations are exponentially down-weighted on each new record.
+    attempts: float = 0.0
+    successes: float = 0.0  # attempts that passed the validation gates
+    first_try_attempts: float = 0.0
+    first_try_successes: float = 0.0
+    aborts: float = 0.0
+    total_cost: float = 0.0
+    total_latency: float = 0.0
+    last_seen: float = 0.0  # epoch seconds of the most recent record
+
+    @property
+    def success_rate(self) -> float:
+        return self.successes / self.attempts if self.attempts else 0.0
+
+    @property
+    def first_try_rate(self) -> float:
+        return (
+            self.first_try_successes / self.first_try_attempts
+            if self.first_try_attempts
+            else 0.0
+        )
+
+    @property
+    def abort_rate(self) -> float:
+        return self.aborts / self.attempts if self.attempts else 0.0
+
+    @property
+    def avg_cost(self) -> float:
+        """Mean dollar cost of a successful attempt (0.0 with no successes)."""
+        return self.total_cost / self.successes if self.successes else 0.0
+
+    @property
+    def avg_latency(self) -> float:
+        return self.total_latency / self.attempts if self.attempts else 0.0
+
+    def quality_score(self) -> float:
+        """Wilson lower bound on the gate-pass rate.
+
+        A conservative point estimate of quality: under-sampling widens the
+        interval and pulls the score down, so a model must earn its ranking
+        with volume, not a lucky single success.
+        """
+        n = self.attempts
+        if n == 0:
+            return 0.0
+        # Clamp to [0,1]: a torn read (or float drift from decay scaling) could
+        # make successes/attempts exceed 1, and p*(1-p) < 0 would raise ValueError
+        # from math.sqrt below.
+        p = min(1.0, max(0.0, self.successes / n))
+        z2 = _Z * _Z
+        denom = 1 + z2 / n
+        center = p + z2 / (2 * n)
+        margin = _Z * math.sqrt((p * (1 - p) + z2 / (4 * n)) / n)
+        return max(0.0, (center - margin) / denom)
+
+
+class ModelLedger:
+    """Thread-safe, file-backed store of per-model outcome statistics."""
+
+    def __init__(
+        self, path: Path, half_life_seconds: float = _DEFAULT_HALF_LIFE_SECONDS
+    ):
+        self.path = Path(path)
+        self.half_life_seconds = half_life_seconds
+        self._lock = threading.Lock()
+        self._stats: Dict[str, ModelStat] = {}
+        self.load()
+
+    @staticmethod
+    def _key(model: str, category: str, complexity: str) -> str:
+        return f"{model}{_SEP}{category}{_SEP}{complexity}"
+
+    def load(self) -> None:
+        """Load stats from disk, tolerating a missing or corrupt file."""
+        if not self.path.exists():
+            return
+        try:
+            raw = json.loads(self.path.read_text(encoding="utf-8"))
+        except (json.JSONDecodeError, OSError) as e:
+            logger.warning(
+                f"Model ledger at {self.path} unreadable, starting fresh: {e}"
+            )
+            return
+        valid = {f.name for f in fields(ModelStat)}
+        with self._lock:
+            self._stats = {}
+            for key, data in (raw or {}).items():
+                if not isinstance(data, dict):
+                    continue
+                # Drop unknown keys so an older/newer schema still loads.
+                clean = {k: v for k, v in data.items() if k in valid}
+                try:
+                    self._stats[key] = ModelStat(**clean)
+                except TypeError:
+                    continue
+
+    def save(self) -> None:
+        """Persist stats atomically (write-temp-then-rename).
+
+        Best-effort: the ledger is telemetry that steers model selection, not
+        build state, so a write failure (full/read-only disk, missing artifact
+        dir) is logged and swallowed rather than allowed to crash an otherwise
+        healthy build. The in-memory stats stay authoritative for the rest of
+        the run; only cross-run persistence is lost.
+        """
+        with self._lock:
+            snapshot = {k: asdict(v) for k, v in self._stats.items()}
+        from misterdev.utils.file_utils import atomic_write_json
+
+        try:
+            atomic_write_json(self.path, snapshot, indent=2, sort_keys=True)
+        except OSError as e:
+            logger.warning(f"Model ledger at {self.path} could not be saved: {e}")
+
+    def stat(self, model: str, category: str = "", complexity: str = "") -> ModelStat:
+        """Return a snapshot of the stat cell for a key (empty if absent).
+
+        Returns a COPY taken under the lock, not the live object: the selector
+        reads several fields of the result without the lock, so handing out the
+        live cell would let it observe a half-applied ``record()``/``_decay_stat``
+        update (e.g. attempts scaled before successes -> a torn read).
+        """
+        key = self._key(model, category, complexity)
+        with self._lock:
+            if key not in self._stats:
+                self._stats[key] = ModelStat(
+                    model=model, category=category, complexity=complexity
+                )
+            return replace(self._stats[key])
+
+    def record(
+        self,
+        model: str,
+        category: str = "",
+        complexity: str = "",
+        *,
+        success: bool,
+        first_try: bool = False,
+        aborted: bool = False,
+        cost: float = 0.0,
+        latency: float = 0.0,
+        timestamp: Optional[float] = None,
+    ) -> ModelStat:
+        """Record one attempt's outcome and persist the ledger.
+
+        ``success`` means the attempt passed the validation gates. ``first_try``
+        marks attempt 0 so first-try success can be tracked separately (it is the
+        signal that decides whether a cheap model is worth trying first).
+        """
+        ts = time.time() if timestamp is None else timestamp
+        key = self._key(model, category, complexity)
+        with self._lock:
+            s = self._stats.get(key)
+            if s is None:
+                s = ModelStat(model=model, category=category, complexity=complexity)
+                self._stats[key] = s
+            else:
+                # Down-weight everything accumulated before this record so older
+                # outcomes fade. Same-instant records (elapsed 0) don't decay, so
+                # a tight burst of attempts behaves exactly like raw counting.
+                self._decay_stat(s, ts)
+            s.attempts += 1
+            if success:
+                s.successes += 1
+                s.total_cost += cost
+            if first_try:
+                s.first_try_attempts += 1
+                if success:
+                    s.first_try_successes += 1
+            if aborted:
+                s.aborts += 1
+            s.total_latency += latency
+            s.last_seen = ts
+        self.save()
+        return s
+
+    def _decay_stat(self, s: ModelStat, now: float) -> None:
+        """Scale a cell's accumulated counts by the time-decay since last_seen.
+
+        Decays the effective counts and the cost/latency sums together, so every
+        rate (success_rate, avg_cost, ...) is preserved while the *confidence*
+        (effective sample size) shrinks for a model not seen recently — which
+        correctly widens its Wilson interval and lowers its quality score.
+        """
+        if s.last_seen <= 0:
+            return
+        factor = _decay_factor(now - s.last_seen, self.half_life_seconds)
+        if factor >= 1.0:
+            return
+        s.attempts *= factor
+        s.successes *= factor
+        s.first_try_attempts *= factor
+        s.first_try_successes *= factor
+        s.aborts *= factor
+        s.total_cost *= factor
+        s.total_latency *= factor
+
+    def selection_score(
+        self, model: str, category: str, complexity: str, total_observations: int
+    ) -> float:
+        """Optimistic quality estimate for selection (UCB).
+
+        Wilson-lower-bound quality plus an exploration bonus that shrinks as a
+        model accrues attempts. Under-sampled or freshly-rotated models score
+        high enough to be tried without being chosen blindly forever. Returns
+        +inf for a never-seen model (textbook UCB) so it is always explored at
+        least once before the policy can settle on a proven model.
+        """
+        s = self.stat(model, category, complexity)
+        if s.attempts == 0:
+            return float("inf")
+        bonus = _Z * math.sqrt(math.log(max(total_observations, 2)) / s.attempts)
+        return s.quality_score() + bonus
+
+    def total_observations(self, category: str = "", complexity: str = "") -> int:
+        """Total recorded attempts, optionally scoped to a cell's context."""
+        with self._lock:
+            return sum(
+                s.attempts
+                for s in self._stats.values()
+                if (not category or s.category == category)
+                and (not complexity or s.complexity == complexity)
+            )
+
+    def global_first_try(
+        self,
+        model: str,
+        exclude_category: str = "",
+        exclude_complexity: str = "",
+    ) -> Tuple[float, float]:
+        """A model's first-try (attempts, success_rate) aggregated across its
+        (category, complexity) cells, optionally EXCLUDING one cell.
+
+        Used as an empirical-Bayes prior to warm-start a cell the model has little
+        data on: performance elsewhere is evidence — imperfect, since competence
+        varies by task kind, so callers blend it with a small weight that cell
+        data quickly overrides. The target cell is excluded so its own data isn't
+        double-counted into its prior. Returns (0.0, 0.0) for an unseen model.
+        """
+        att = succ = 0.0
+        with self._lock:
+            for s in self._stats.values():
+                if s.model != model:
+                    continue
+                if (
+                    s.category == exclude_category
+                    and s.complexity == exclude_complexity
+                ):
+                    continue
+                att += s.first_try_attempts
+                succ += s.first_try_successes
+        return att, (succ / att if att > 0 else 0.0)
+
+    def known_models(self) -> List[str]:
+        with self._lock:
+            return sorted({s.model for s in self._stats.values()})
+
+    def all_stats(self) -> List[ModelStat]:
+        """A snapshot of every stat cell (for reporting/inspection)."""
+        with self._lock:
+            return list(self._stats.values())