misterdev 0.2.0__py3-none-any.whl

misterdev/core/economics/model_selector.py

@@ -0,0 +1,281 @@
+"""Cost-aware, data-driven model selection.
+
+Turns the static complexity->tier->model routing into a policy that picks a
+model to maximize quality-per-dollar under an unconditional quality floor: the
+validation gates. A cheaper model that writes worse code simply fails the gate
+and the policy escalates, so model choice can only change cost/latency, never
+shipped quality.
+
+The policy is deterministic (UCB scoring from the ledger, not random draws) and
+disabled by default; with it off, ``select`` returns None and the executor
+keeps its existing static routing.
+"""
+
+from typing import List, Optional
+
+from misterdev.config import get_setting
+from misterdev.core.economics.model_ledger import ModelLedger
+from misterdev.logging_setup import setup_logger
+
+logger = setup_logger(__name__)
+
+
+class ModelSelector:
+    """Picks a model id per attempt from the escalation ladder and ledger."""
+
+    def __init__(
+        self, config: dict, ledger: ModelLedger, free_models: Optional[List[str]] = None
+    ):
+        self.ledger = ledger
+        # dynamic_selection is False (off), True (on), or "auto" (self-
+        # activating). A non-"auto" string is treated as off, never truthy.
+        raw = get_setting(config, "llm", "dynamic_selection")
+        if isinstance(raw, str):
+            self.auto = raw.strip().lower() == "auto"
+            self.enabled = self.auto
+        else:
+            self.auto = False
+            self.enabled = bool(raw)
+        self.escalation: List[str] = list(
+            get_setting(config, "llm", "escalation") or []
+        )
+        self.models = dict(get_setting(config, "llm", "models") or {})
+        self.min_obs = get_setting(config, "llm", "min_observations")
+        self.first_try_floor = get_setting(config, "llm", "first_try_floor")
+        self.incompetence_floor = get_setting(config, "llm", "incompetence_floor")
+        self.max_latency = get_setting(config, "llm", "max_attempt_latency_seconds")
+        self.posture = get_setting(config, "llm", "selection_posture")
+        self.maturity_threshold = get_setting(config, "llm", "maturity_threshold")
+        # Cells already announced as graduated to cheap-first (one log per cell).
+        self._graduated: set = set()
+        free = list(free_models or [])
+        if self.escalation and free:
+            # Harvested free models join the configured cheapest tier as extra
+            # candidates, so they are only ever tried on early (non-final)
+            # attempts and under the same proven-first-try gate as any cheap
+            # model. They go first so an exploration tie among unseen candidates
+            # (equal +inf UCB) breaks toward the zero-cost option.
+            cheapest = self.escalation[0]
+            existing = self._tier_models(cheapest)
+            self.models[cheapest] = [m for m in free if m not in existing] + existing
+        elif not self.escalation and free:
+            # No ladder configured: self-assemble one from the harvested free
+            # models (cheap) and the default model (strong). This is what makes
+            # free models work out of the box with no escalation/models config.
+            default_model = get_setting(config, "llm", "model")
+            self.models = {**self.models, "_free": free}
+            if default_model:
+                self.models["_default"] = [default_model]
+                self.escalation = ["_free", "_default"]
+
+    def _tier_models(self, tier: str) -> List[str]:
+        v = self.models.get(tier)
+        if v is None:
+            return []
+        if isinstance(v, str):
+            return [v]
+        return [m for m in v if isinstance(m, str)]
+
+    def _rungs(self) -> List[str]:
+        """Escalation tiers that actually resolve to at least one model."""
+        return [t for t in self.escalation if self._tier_models(t)]
+
+    def _cost(self, model: str, category: str, complexity: str) -> float:
+        """Mean successful-attempt cost, or a neutral 1.0 when unknown.
+
+        Unknown cost stays neutral so an unseen model is ranked purely on its
+        (optimistic) quality score rather than being favored for a $0 estimate.
+        """
+        avg = self.ledger.stat(model, category, complexity).avg_cost
+        return avg if avg > 0 else 1.0
+
+    # Empirical-Bayes prior weight: how many pseudo-observations of the model's
+    # GLOBAL first-try rate to fold into a cold cell. Small, so cell-specific data
+    # overrides it within a few real attempts — the prior only warm-starts.
+    _PRIOR_WEIGHT = 3.0
+
+    def _proven(self, model: str, category: str, complexity: str) -> bool:
+        """True when a model has earned trust for a first attempt on this cell.
+
+        A cold cell borrows strength from the model's GLOBAL first-try rate
+        (shrinkage): a model reliable everywhere else clears the floor here with
+        fewer local observations, while a globally-weak model does not. As the
+        cell accumulates real attempts, the blended rate converges to the cell's
+        own — the prior fades. A model with no history anywhere stays unproven.
+        """
+        s = self.ledger.stat(model, category, complexity)
+        g_att, g_rate = self.ledger.global_first_try(model, category, complexity)
+        prior = self._PRIOR_WEIGHT if g_att >= self.min_obs else 0.0
+        effective_obs = s.first_try_attempts + prior
+        if effective_obs < self.min_obs:
+            return False
+        blended = (s.first_try_successes + prior * g_rate) / effective_obs
+        return blended >= self.first_try_floor
+
+    def _incompetent(self, model: str, category: str, complexity: str) -> bool:
+        """Proven UNABLE to do this kind of task: enough attempts to judge and a
+        success rate below the floor. Distinct from unproven (too few attempts).
+        Trying such a model only burns a failed attempt and forces escalation."""
+        if not self.incompetence_floor:
+            return False
+        s = self.ledger.stat(model, category, complexity)
+        return s.attempts >= self.min_obs and s.success_rate < self.incompetence_floor
+
+    def _too_slow(self, model: str, category: str, complexity: str) -> bool:
+        """Proven too slow for the per-task wall-clock budget (ledger avg latency
+        above the configured ceiling). Unseen models (avg 0) are never flagged."""
+        if not self.max_latency:
+            return False
+        return (
+            self.ledger.stat(model, category, complexity).avg_latency > self.max_latency
+        )
+
+    # Free (`:free`) endpoints are reserved for the easiest tasks: they are slow
+    # (2.5-5 min/call observed) and unreliable, so on anything heavier than a
+    # small task they cost more wall-clock and failed attempts than they save.
+    _FREE_OK_COMPLEXITY = ("trivial", "small")
+
+    def _pick(
+        self,
+        models: List[str],
+        category: str,
+        complexity: str,
+        explore: bool,
+        final: bool = False,
+    ) -> Optional[str]:
+        """Best model by quality-per-dollar.
+
+        explore=True uses the optimistic UCB score (unseen models score +inf, so
+        they get tried); explore=False uses the conservative Wilson lower bound
+        (unseen models score 0, so only proven models win). Free endpoints are
+        dropped for non-easy tasks (see ``_FREE_OK_COMPLEXITY``), and models the
+        ledger has proven incompetent (or, on non-final attempts, too slow) are
+        dropped; if that empties the tier, returns None so the caller climbs to
+        the next (paid) rung. Incompetence still filters on the final attempt — a
+        model that cannot do the task is never the right last resort.
+        """
+        if complexity not in self._FREE_OK_COMPLEXITY:
+            models = [m for m in models if ":free" not in m]
+        models = [m for m in models if not self._incompetent(m, category, complexity)]
+        if not final:
+            models = [m for m in models if not self._too_slow(m, category, complexity)]
+        if not models:
+            return None
+        total = self.ledger.total_observations(category, complexity)
+
+        def value(model: str) -> float:
+            if explore:
+                score = self.ledger.selection_score(model, category, complexity, total)
+            else:
+                score = self.ledger.stat(model, category, complexity).quality_score()
+            return score / self._cost(model, category, complexity)
+
+        return max(models, key=value)
+
+    def _mature(self, category: str, complexity: str) -> bool:
+        """True once a cell has enough data to stop exploring (auto mode)."""
+        return (
+            self.ledger.total_observations(category, complexity)
+            >= self.maturity_threshold
+        )
+
+    def _explore_on_first(self, category: str, complexity: str) -> bool:
+        if self.auto:
+            # Self-regulating: explore cheap/free models on easy tasks while the
+            # cell is immature, then settle into conservative cheap-first.
+            # Only trivial/small tasks are explored on the FIRST attempt: on a
+            # large repo, unproven free models returned no usable edits on medium
+            # refactor tasks (the emathy run burned first attempts on null
+            # responses). A cheap model can still earn medium first-attempt use
+            # once PROVEN (the conservative branch picks a proven cheap model).
+            if self._mature(category, complexity):
+                return False
+            return complexity in ("trivial", "small")
+        if self.posture == "aggressive":
+            return True
+        if self.posture == "balanced":
+            return complexity in ("trivial", "small", "medium")
+        return False  # conservative
+
+    def select(
+        self, category: str, complexity: str, attempt: int, max_attempts: int
+    ) -> Optional[str]:
+        """Choose a model id for this attempt, or None to use the default.
+
+        Returns None when disabled or when no escalation ladder is configured,
+        leaving the executor's existing static routing in charge.
+        """
+        if not self.enabled:
+            return None
+        rungs = self._rungs()
+        if not rungs:
+            return None
+        last = len(rungs) - 1
+
+        # The final attempt always uses the strongest tier: the quality floor's
+        # safety net, so every prior attempt can afford to try something cheaper.
+        # final=True lifts the latency guard here (a slow-but-capable last resort
+        # beats giving up), but incompetent models are still excluded.
+        if attempt >= max_attempts - 1:
+            return self._pick(
+                self._tier_models(rungs[last]), category, complexity, True, final=True
+            )
+
+        if attempt == 0 and not self._explore_on_first(category, complexity):
+            # Conservative first attempt: cheapest rung with a proven model, else
+            # the strongest tier. Never gamble the first impression on an
+            # unproven cheap model.
+            for tier in rungs:
+                proven = [
+                    m
+                    for m in self._tier_models(tier)
+                    if self._proven(m, category, complexity)
+                ]
+                if proven:
+                    chosen = self._pick(proven, category, complexity, False)
+                    self._announce_graduation(category, complexity, chosen)
+                    return chosen
+            return self._pick(
+                self._tier_models(rungs[last]), category, complexity, True
+            )
+
+        # Climbing path: one rung up per attempt, capped at the strongest tier.
+        # If the target rung is emptied by the free/incompetence/latency filters,
+        # climb to the next (stronger) rung rather than fall back to the client
+        # default — a filtered tier should escalate, not disable selection.
+        target = min(attempt, last)
+        return self._pick_climbing(rungs, target, category, complexity)
+
+    def _pick_climbing(
+        self, rungs: List[str], start: int, category: str, complexity: str
+    ) -> Optional[str]:
+        """Pick from rung ``start``, climbing to each stronger rung in turn until
+        one yields a model the filters didn't drop; None only when every rung
+        from ``start`` up is empty."""
+        for i in range(start, len(rungs)):
+            chosen = self._pick(self._tier_models(rungs[i]), category, complexity, True)
+            if chosen:
+                return chosen
+        return None
+
+    def _announce_graduation(self, category: str, complexity: str, model: str) -> None:
+        """Log the first time a cell settles on a proven cheap model (auto)."""
+        if not self.auto:
+            return
+        cell = (category, complexity)
+        if cell not in self._graduated:
+            self._graduated.add(cell)
+            logger.info(
+                f"[auto] {category}/{complexity} now using proven cheap model "
+                f"{model!r} on first attempt"
+            )
+
+    def is_ready(self, category: str, complexity: str) -> bool:
+        """Whether this cell would use a proven cheap model on a first attempt."""
+        if not self.enabled:
+            return False
+        return any(
+            self._proven(m, category, complexity)
+            for tier in self._rungs()[:-1]
+            for m in self._tier_models(tier)
+        )

misterdev/core/execution/bounded.py

@@ -0,0 +1,50 @@
+"""Shared bounded best-effort runner for the optional gates.
+
+Several optional gates — goal check, adversarial critic, runtime/web/vision/
+mutation smoke, LSP diagnostics, and the MCP substrate — must be strictly
+non-blocking: a slow or hung model, server, or subprocess can NEVER stall a
+build. They all shared one shape: run the work in a daemon thread, join with a
+hard timeout, and on timeout abandon the thread (it dies with the process) and
+return a SKIP sentinel. This centralizes that shape so the call sites don't each
+re-implement it.
+"""
+
+import threading
+from typing import Callable, TypeVar
+
+from misterdev.logging_setup import setup_logger
+
+logger = setup_logger(__name__)
+
+T = TypeVar("T")
+
+
+def run_bounded(
+    fn: Callable[[], T], timeout: float, default: T, what: str = "operation"
+) -> T:
+    """Run ``fn()`` in a daemon thread, returning its result or ``default``.
+
+    Returns ``default`` when ``fn`` does not finish within ``timeout`` seconds
+    (the thread is abandoned and dies with the process) or when it raises (logged
+    at debug — a backstop, since ``fn`` is expected to handle its own expected
+    failures and return an appropriate value itself). ``what`` names the
+    operation for log context. The caller is guaranteed control back within
+    ``timeout`` seconds and is never handed an exception.
+    """
+    box = {"result": default}
+
+    def _worker() -> None:
+        try:
+            box["result"] = fn()
+        except Exception as e:  # backstop: fn should catch its own failures
+            logger.debug(f"{what} failed in bounded runner: {e}")
+            box["result"] = default
+
+    worker = threading.Thread(target=_worker, daemon=True)
+    worker.start()
+    worker.join(timeout)
+    if worker.is_alive():
+        logger.warning(f"{what} timed out after {timeout}s; skipping (never blocks).")
+        return default
+
+    return box["result"]

misterdev/core/execution/container.py

@@ -0,0 +1,221 @@
+"""Engine-agnostic OCI container execution substrate (opt-in).
+
+Runs gate commands (build/lint/test/golden) inside a throwaway container so a
+build is exercised against a pinned, reproducible toolchain instead of whatever
+happens to be installed on the host. Strictly best-effort and opt-in: it is used
+only when ``environment.type`` is ``docker``/``container`` AND a usable OCI
+engine is actually present. With no engine reachable (no daemon, CLI missing)
+the orchestrator falls back to executing commands locally exactly as before, so
+the feature can never hard-fail a build or require a daemon to exist.
+
+Design mirrors :mod:`misterdev.core.context.lsp`: detection probes are
+bounded by a short timeout (a hung/absent engine can never block), every failure
+is logged at debug and degrades to a safe no-op, and the runtime knob is off by
+default.
+
+Git operations (branch/commit/revert) always stay on the host; only the gate
+commands are routed through the container, executed against the repository
+bind-mounted at the same path so produced paths line up with the host tree.
+"""
+
+import os
+import shlex
+import subprocess
+from pathlib import Path
+from typing import List, Optional, Tuple
+
+from misterdev.logging_setup import setup_logger
+
+logger = setup_logger(__name__)
+
+# Engines we know how to drive, in preference order: rootless/daemonless first
+# (podman, nerdctl), then docker, then the colima-managed docker socket. All
+# four speak the same ``<engine> run`` CLI surface we rely on.
+_PREFERRED_ENGINES: Tuple[str, ...] = ("podman", "docker", "nerdctl", "colima")
+
+# How long an engine availability probe may take before we treat the engine as
+# unavailable. A short bound so a wedged daemon can never stall detection.
+_PROBE_TIMEOUT = 8
+
+# Default image per detected project language. Conservative, widely-mirrored
+# tags; a project may override with ``environment.image``.
+_LANGUAGE_IMAGES = {
+    "python": "python:3.12-slim",
+    "rust": "rust:slim",
+    "node": "node:20-slim",
+    "javascript": "node:20-slim",
+    "typescript": "node:20-slim",
+    "go": "golang:1.22",
+    "ruby": "ruby:3.3-slim",
+    "java": "eclipse-temurin:21",
+}
+
+_DEFAULT_IMAGE = "debian:stable-slim"
+
+
+def _probe_engine(engine: str) -> bool:
+    """True if ``engine`` is installed and its backend responds within the probe
+    timeout. Never raises: any failure (missing binary, unreachable daemon,
+    timeout) is reported as unavailable."""
+    if engine == "colima":
+        # colima is a VM manager, not a runtime CLI; a running colima exposes a
+        # docker engine. Treat it as available only when its status is running.
+        cmd = ["colima", "status"]
+    else:
+        # `info` requires a working backend (daemon for docker; none for
+        # podman/nerdctl rootless), so it distinguishes "CLI present" from
+        # "engine actually usable".
+        cmd = [engine, "info"]
+    try:
+        proc = subprocess.run(
+            cmd,
+            capture_output=True,
+            text=True,
+            timeout=_PROBE_TIMEOUT,
+        )
+        return proc.returncode == 0
+    except (OSError, subprocess.SubprocessError) as e:
+        logger.debug(f"Container engine probe failed for {engine}: {e}")
+        return False
+
+
+def detect_engine(preferred: Optional[str] = None) -> Optional[str]:
+    """Return the name of the first usable OCI engine, or ``None`` if none.
+
+    ``preferred`` (from ``environment.engine``) is tried first when given; the
+    standard rootless-first order follows. ``colima`` resolves to ``docker`` as
+    the run CLI since colima manages a docker-compatible daemon.
+    """
+    order: List[str] = []
+    if preferred:
+        order.append(preferred)
+    order.extend(e for e in _PREFERRED_ENGINES if e not in order)
+    for engine in order:
+        if _probe_engine(engine):
+            # colima provides a docker daemon; the actual run CLI is docker.
+            return "docker" if engine == "colima" else engine
+    return None
+
+
+def image_for_language(language: Optional[str]) -> str:
+    """Pick a default base image for ``language``; falls back to a generic image
+    when the language is unknown so the container can still run shell gates."""
+    return _LANGUAGE_IMAGES.get((language or "").lower(), _DEFAULT_IMAGE)
+
+
+class ContainerEngine:
+    """Thin wrapper over a detected OCI engine for running gate commands.
+
+    One instance per build. Commands run in a fresh ``--rm`` container with the
+    repository bind-mounted at ``mount_path`` and the process mapped to the
+    host uid/gid so artifacts are not left root-owned. The container is not kept
+    running between commands (each gate is a self-contained ``run``), which
+    keeps lifecycle trivial and leak-free while still pinning the toolchain.
+    """
+
+    def __init__(
+        self,
+        engine: str,
+        image: str,
+        host_path: Path,
+        mount_path: str = "/workspace",
+        network: Optional[str] = None,
+        memory: Optional[str] = None,
+        cpus: Optional[str] = None,
+        pids_limit: Optional[int] = None,
+        cap_drop: Optional[List[str]] = None,
+        security_opt: Optional[List[str]] = None,
+    ):
+        self.engine = engine
+        self.image = image
+        self.host_path = Path(host_path).resolve()
+        self.mount_path = mount_path
+        # Egress control: "none" runs the container with no network (governance.
+        # network); any other value (or None) leaves the engine default, so the
+        # off path is byte-identical to before. This constrains CONTAINERIZED
+        # execution only — host execution and git stay on the host network.
+        self.network = network
+        # Optional resource caps (environment.memory / .cpus / .pids_limit). Each
+        # is emitted only when set, so an unconfigured engine produces the exact
+        # same argv as before. They bound a runaway gate (fork bomb, memory hog)
+        # in the throwaway container without affecting host execution.
+        self.memory = memory
+        self.cpus = cpus
+        self.pids_limit = pids_limit
+        # Optional sandbox hardening for running model-generated code with less
+        # trust: cap_drop (e.g. ["ALL"]) drops Linux capabilities; security_opt
+        # (e.g. ["no-new-privileges", "seccomp=/path/profile.json"]) passes
+        # --security-opt. Both emitted only when set; a bare string is accepted
+        # and wrapped, so ``cap_drop: ALL`` in YAML works too.
+        self.cap_drop = [cap_drop] if isinstance(cap_drop, str) else (cap_drop or [])
+        self.security_opt = (
+            [security_opt] if isinstance(security_opt, str) else (security_opt or [])
+        )
+
+    def is_available(self) -> bool:
+        return bool(self.engine)
+
+    def _user_args(self) -> List[str]:
+        """``--user uid:gid`` so host-bind-mounted writes are owned by the
+        invoking user, not root. Skipped on platforms without uid/gid
+        (Windows), where the flag is meaningless."""
+        getuid = getattr(os, "getuid", None)
+        getgid = getattr(os, "getgid", None)
+        if getuid is None or getgid is None:
+            return []
+        return ["--user", f"{getuid()}:{getgid()}"]
+
+    def wrap_command(self, command: str, timeout: int) -> List[str]:
+        """Build the argv that runs ``command`` inside a throwaway container.
+
+        The user command is passed verbatim to ``sh -c`` inside the container,
+        so shell features (``&&``, pipes, globs) behave as on the host. The repo
+        is the working directory via the bind mount.
+        """
+        argv = [
+            self.engine,
+            "run",
+            "--rm",
+            "-v",
+            f"{self.host_path}:{self.mount_path}",
+            "-w",
+            self.mount_path,
+        ]
+        if self.network == "none":
+            argv.extend(["--network", "none"])
+        if self.memory:
+            argv.extend(["--memory", str(self.memory)])
+        if self.cpus:
+            argv.extend(["--cpus", str(self.cpus)])
+        if self.pids_limit:
+            argv.extend(["--pids-limit", str(self.pids_limit)])
+        for cap in self.cap_drop:
+            argv.extend(["--cap-drop", str(cap)])
+        for opt in self.security_opt:
+            argv.extend(["--security-opt", str(opt)])
+        argv.extend(self._user_args())
+        argv.extend([self.image, "sh", "-c", command])
+        return argv
+
+    def run(self, command: str, timeout: int = 180) -> Tuple[bool, str]:
+        """Execute ``command`` inside the container. Returns ``(ok, output)``
+        with the same contract as the local runner, so callers are agnostic to
+        where the command ran. Never raises: engine/timeout failures return
+        ``(False, message)``."""
+        argv = self.wrap_command(command, timeout)
+        logger.debug(f"Container exec ({self.engine}): {shlex.join(argv)}")
+        try:
+            proc = subprocess.run(
+                argv,
+                capture_output=True,
+                text=True,
+                timeout=timeout,
+            )
+            output = proc.stdout
+            if proc.stderr:
+                output += "\n" + proc.stderr
+            return proc.returncode == 0, output
+        except subprocess.TimeoutExpired:
+            return False, f"Container command timed out after {timeout}s: {command}"
+        except (OSError, subprocess.SubprocessError) as e:
+            return False, f"Container command failed: {e}"