misterdev 0.2.0__py3-none-any.whl

misterdev/llm/client/__init__.py

@@ -0,0 +1,152 @@
+import time  # noqa: F401 — kept so tests can patch `client.time.sleep` on this package
+from typing import Optional
+
+from misterdev.config import get_setting
+from misterdev.logging_setup import setup_logger
+
+from .base import BaseLLMClient
+from .edits import APPLY_EDITS_TOOL, _edits_to_markdown, code_gen_abort_check
+from .embeddings import (
+    LocalEmbeddingClient,
+    OpenRouterEmbeddingClient,
+    _create_local_embedding_client,
+    _create_openrouter_embedding_client,
+    create_embedding_client,
+)
+from .errors import (
+    RETRYABLE_ERROR_MARKERS,
+    RETRYABLE_EXCEPTION_NAMES,
+    RETRYABLE_STATUS_CODES,
+    BudgetExceededError,
+    LLMCallError,
+    _api_error,
+    _error_status_code,
+    _is_retryable_error,
+)
+from .providers import (
+    CACHE_BREAKPOINT,
+    AnthropicLLMClient,
+    OpenRouterLLMClient,
+    _deny_unless_training_allowed,
+    _openrouter_sdk,
+)
+from .response import LLMResponse, LLMUsage
+
+logger = setup_logger(__name__)
+
+
+class FailoverLLMClient(BaseLLMClient):
+    """Wraps a primary client plus ordered fallbacks for provider outages.
+
+    On a retryable error from one provider it advances to the next; a
+    non-retryable error (bad request) propagates immediately. Usage from
+    whichever provider served the request is tracked on this wrapper.
+    """
+
+    def __init__(self, config: dict):
+        super().__init__(config)
+        self.primary = _create_single_client(config)
+        self.failover_clients: list[BaseLLMClient] = []
+        for fc in get_setting(config, "llm", "failover"):
+            try:
+                merged = {**config, "llm": {**config["llm"], **fc, "failover": []}}
+                self.failover_clients.append(_create_single_client(merged))
+            except (ValueError, ImportError) as e:
+                logger.warning(f"Failover provider unavailable, skipping: {e}")
+
+    @property
+    def model(self) -> str:
+        return self.primary.model
+
+    def with_model(self, model: str):
+        # Route the primary path; fallbacks keep their configured models.
+        return self.primary.with_model(model)
+
+    def _call(self, prompt: str, system_prompt: str) -> LLMResponse:
+        clients = [self.primary] + self.failover_clients
+        last_error: Optional[LLMCallError] = None
+        for i, client in enumerate(clients):
+            try:
+                response = client._call(prompt, system_prompt)
+                if i > 0:
+                    logger.info(
+                        f"Failover successful via {client.__class__.__name__} ({client.model})."
+                    )
+                return response
+            except LLMCallError as e:
+                last_error = e
+                if not e.retryable:
+                    raise
+                logger.warning(
+                    f"Provider {client.__class__.__name__} failed ({e}); trying next provider..."
+                )
+        raise last_error or LLMCallError("All LLM providers failed", retryable=False)
+
+    def _call_stream(self, prompt: str, system_prompt: str):
+        clients = [self.primary] + self.failover_clients
+        last_error: Optional[LLMCallError] = None
+        for i, client in enumerate(clients):
+            try:
+                usage = yield from client._call_stream(prompt, system_prompt)
+                if i > 0:
+                    logger.info(
+                        f"Failover stream via {client.__class__.__name__} ({client.model})."
+                    )
+                return usage
+            except LLMCallError as e:
+                last_error = e
+                if not e.retryable:
+                    raise
+                logger.warning(
+                    f"Provider {client.__class__.__name__} stream failed ({e}); "
+                    "trying next provider..."
+                )
+        raise last_error or LLMCallError("All LLM providers failed", retryable=False)
+
+
+def _create_single_client(config: dict) -> BaseLLMClient:
+    provider = get_setting(config, "llm", "provider")
+    if provider == "openrouter":
+        return OpenRouterLLMClient(config)
+    elif provider == "anthropic":
+        return AnthropicLLMClient(config)
+    else:
+        raise ValueError(f"Unsupported LLM provider: {provider}")
+
+
+def create_llm_client(config: dict) -> BaseLLMClient:
+    """Factory: returns a failover-wrapped client when failover config exists."""
+    if get_setting(config, "llm", "failover"):
+        return FailoverLLMClient(config)
+    return _create_single_client(config)
+
+
+__all__ = [
+    "APPLY_EDITS_TOOL",
+    "CACHE_BREAKPOINT",
+    "AnthropicLLMClient",
+    "BaseLLMClient",
+    "BudgetExceededError",
+    "FailoverLLMClient",
+    "LLMCallError",
+    "LLMResponse",
+    "LLMUsage",
+    "LocalEmbeddingClient",
+    "OpenRouterEmbeddingClient",
+    "OpenRouterLLMClient",
+    "RETRYABLE_ERROR_MARKERS",
+    "RETRYABLE_EXCEPTION_NAMES",
+    "RETRYABLE_STATUS_CODES",
+    "_api_error",
+    "_create_local_embedding_client",
+    "_create_openrouter_embedding_client",
+    "_create_single_client",
+    "_deny_unless_training_allowed",
+    "_edits_to_markdown",
+    "_error_status_code",
+    "_is_retryable_error",
+    "_openrouter_sdk",
+    "code_gen_abort_check",
+    "create_embedding_client",
+    "create_llm_client",
+]

misterdev/llm/client/base.py

@@ -0,0 +1,382 @@
+import threading
+import time
+from abc import ABC, abstractmethod
+from contextlib import contextmanager
+from typing import Dict, Optional, Tuple
+
+from misterdev.config import get_setting
+from misterdev.logging_setup import setup_logger
+
+from .errors import BudgetExceededError, LLMCallError
+from .response import LLMResponse, LLMUsage
+
+logger = setup_logger(__name__)
+
+
+class BaseLLMClient(ABC):
+    """Abstract LLM client with token tracking and budget enforcement."""
+
+    # Fraction of budget-remaining a single task may spend when its per-task
+    # cap is set to "auto".
+    AUTO_TASK_CAP_FRACTION = 0.5
+
+    def __init__(self, config: dict):
+        self.config = config
+        self.cumulative_usage = LLMUsage()
+        # Guards the usage accumulators: one client instance is hit by several
+        # worker threads at once (parallel analyzers, executor waves), so the
+        # read-modify-write of cost/token counters must be serialized or
+        # concurrent calls lose updates and under-count spend.
+        self._usage_lock = threading.Lock()
+        # Per-task ROUTING/ATTRIBUTION state (active model, current task, reasoning
+        # effort, edit-tool mode) is thread-local: parallel executor workers share
+        # ONE client instance (see agent._WorktreeProjectView), so a plain instance
+        # attribute would let one worker's with_model()/track_task() clobber
+        # another's mid-call — issuing a request to the wrong model or attributing
+        # its cost to the wrong task. The shared accounting below stays shared.
+        self._tls = threading.local()
+        self._default_model: Optional[str] = None
+        self.cost_by_task: Dict[str, float] = {}
+        self._budget = get_setting(config, "build", "budget")
+        # Optional per-task cost ceiling. None disables it; a number is an
+        # absolute cap; "auto" makes it a fraction of the budget remaining when
+        # the task starts (snapshotted in track_task), so it shrinks as the run
+        # spends and no single task can drain the global budget.
+        self._max_cost_per_task = get_setting(
+            config, "orchestrator", "max_cost_per_task"
+        )
+        self._task_caps: Dict[str, Optional[float]] = {}
+
+    @property
+    def model(self) -> Optional[str]:
+        """Active model: a thread-local ``with_model`` override, else the
+        configured default (set once at construction, shared across threads).
+
+        Tolerates a partially-constructed instance (``_tls``/``_default_model``
+        absent) so unit tests that build a client via ``__new__`` to exercise a
+        pure method still read ``model``."""
+        override = getattr(getattr(self, "_tls", None), "model", None)
+        return override or getattr(self, "_default_model", None)
+
+    @model.setter
+    def model(self, value: Optional[str]) -> None:
+        # Construction sets the shared default; per-call overrides go through
+        # with_model() (thread-local), never this setter.
+        self._default_model = value
+
+    @property
+    def _current_task(self) -> Optional[str]:
+        return getattr(self._tls, "current_task", None)
+
+    @_current_task.setter
+    def _current_task(self, value: Optional[str]) -> None:
+        self._tls.current_task = value
+
+    @property
+    def _reasoning_effort(self) -> Optional[str]:
+        return getattr(self._tls, "reasoning_effort", None)
+
+    @_reasoning_effort.setter
+    def _reasoning_effort(self, value: Optional[str]) -> None:
+        self._tls.reasoning_effort = value
+
+    @property
+    def _edit_tool_mode(self) -> bool:
+        return getattr(self._tls, "edit_tool_mode", False)
+
+    @_edit_tool_mode.setter
+    def _edit_tool_mode(self, value: bool) -> None:
+        self._tls.edit_tool_mode = value
+
+    def generate(self, prompt: str, system_prompt: str = "") -> LLMResponse:
+        """Generate a response with retry and budget enforcement.
+
+        This is the primary public interface. Subclasses implement _call().
+        """
+        self._enforce_budget()
+
+        # Free models (`:free`) are frequently rate-limited upstream and slow to
+        # even return the 429. Since routed calls fall back to the reliable paid
+        # model on failure, a free model should FAIL FAST (one shot) rather than
+        # burn ~minutes on slow retries before that fallback kicks in. Paid models
+        # keep full retry resilience for genuine transient errors.
+        is_free = ":free" in (getattr(self, "model", "") or "")
+        max_retries = 1 if is_free else 3
+        base_delay = 1.0
+
+        last_error = None
+        for attempt in range(max_retries):
+            try:
+                response = self._call(prompt, system_prompt)
+                self._track_usage(response.usage)
+                return response
+            except LLMCallError as e:
+                last_error = e
+                if not e.retryable or attempt == max_retries - 1:
+                    raise
+                delay = base_delay * (2**attempt)
+                logger.warning(
+                    f"LLM call failed (attempt {attempt + 1}/{max_retries}), "
+                    f"retrying in {delay:.1f}s: {e}"
+                )
+                time.sleep(delay)
+
+        raise last_error
+
+    def generate_code(self, prompt: str, system_prompt: str = "") -> str:
+        """Convenience wrapper returning just the text content.
+
+        Maintains backward compatibility with existing callers.
+        """
+        return self.generate(prompt, system_prompt).content
+
+    def generate_edits(self, prompt: str, system_prompt: str = "") -> LLMResponse:
+        """Generate file edits, preferring a structured tool call when supported.
+
+        The base implementation just calls :meth:`generate` (markdown edits);
+        clients that support function-calling override to force the apply_edits
+        tool and return content rendered into the canonical fence format.
+        """
+        return self.generate(prompt, system_prompt)
+
+    def chat_multimodal(
+        self, prompt: str, image_b64: str, model: Optional[str] = None
+    ) -> str:
+        """Send a text+image message and return the model's text reply.
+
+        First-class multimodal entry point used by the vision gate. The base
+        raises so providers without a multimodal path degrade cleanly (the
+        vision gate then falls back / SKIPs rather than crashing).
+        """
+        raise NotImplementedError("multimodal chat not supported by this client")
+
+    def health_check(self) -> Tuple[bool, str]:
+        """Verify the configured model actually resolves before a real run.
+
+        A retired/misrouted model id (e.g. an OpenRouter 404) otherwise only
+        surfaces on the first analysis call, after preflight has already
+        spent setup time. This makes one minimal request and reports failure
+        with the model id so the build can abort early with a clear message.
+        """
+        model = getattr(self, "model", "<unknown>")
+        prior = self.cumulative_usage.estimated_cost
+        try:
+            self.generate("ping", "Reply with the single word OK.")
+            return True, model
+        except Exception as e:
+            return False, f"model {model!r} unavailable: {e}"
+        finally:
+            self.cumulative_usage.estimated_cost = prior
+
+    @contextmanager
+    def with_model(self, model: str):
+        """Temporarily override the active model for per-task routing.
+
+        The override is thread-local (a shared client is used by parallel
+        workers), so it must set/restore ``_tls.model`` directly rather than the
+        ``model`` setter, which writes the shared default.
+        """
+        had_override = hasattr(self._tls, "model")
+        previous = getattr(self._tls, "model", None)
+        self._tls.model = model
+        try:
+            yield
+        finally:
+            if had_override:
+                self._tls.model = previous
+            else:
+                try:
+                    del self._tls.model
+                except AttributeError:
+                    pass
+
+    @contextmanager
+    def with_reasoning_effort(self, effort: Optional[str]):
+        """Temporarily request a reasoning effort for the enclosed call(s).
+
+        Honored only by clients/models that support a reasoning budget; ignored
+        elsewhere. ``None`` is a no-op.
+        """
+        original = getattr(self, "_reasoning_effort", None)
+        self._reasoning_effort = effort
+        try:
+            yield
+        finally:
+            self._reasoning_effort = original
+
+    def _enforce_budget(self) -> None:
+        """Raise BudgetExceededError if the global or per-task cap is spent.
+
+        Shared by generate() and generate_stream() so both paths honor the
+        same budget gates before issuing a call.
+        """
+        # Read the cumulative cost under the same lock that _track_usage writes
+        # it, so a parallel worker mid-update can't be observed as a torn/stale
+        # value that lets a call slip past the ceiling.
+        with self._usage_lock:
+            spent = self.cumulative_usage.estimated_cost
+        if spent >= self._budget:
+            raise BudgetExceededError(
+                f"Budget of ${self._budget:.2f} exceeded (spent ${spent:.2f})"
+            )
+        if self.task_cost_exceeded(getattr(self, "_current_task", None)):
+            task_id = getattr(self, "_current_task", None)
+            cap = self.effective_task_cap(task_id)
+            raise BudgetExceededError(
+                f"Per-task budget of ${cap:.2f} exceeded "
+                f"for task {task_id!r} "
+                f"(spent ${self.task_cost(task_id):.2f})"
+            )
+
+    @abstractmethod
+    def _call(self, prompt: str, system_prompt: str) -> LLMResponse:
+        """Execute a single LLM API call. Subclasses implement this."""
+
+    def generate_stream(
+        self, prompt: str, system_prompt: str = "", abort_check=None
+    ) -> LLMResponse:
+        """Stream a response, aborting early if abort_check flags the output.
+
+        Returns finish_reason="aborted" with the partial content when the check
+        trips, so a caller can retry with a stricter prompt instead of waiting
+        for a full bad response. Usage is captured from the provider stream when
+        available (and estimated otherwise) so streaming honors the same budget
+        accounting as generate().
+        """
+        self._enforce_budget()
+
+        chunks: list[str] = []
+        usage: Optional[LLMUsage] = None
+        finish_reason = "stop"
+        stream = self._call_stream(prompt, system_prompt)
+        try:
+            while True:
+                chunks.append(next(stream))
+                if abort_check is not None and abort_check("".join(chunks)):
+                    logger.warning("Aborting LLM stream: bad output pattern detected")
+                    stream.close()
+                    finish_reason = "aborted"
+                    break
+        except StopIteration as stop:
+            usage = stop.value
+
+        content = "".join(chunks)
+        if usage is None:
+            usage = self._estimate_usage(prompt, system_prompt, content)
+        self._track_usage(usage)
+        return LLMResponse(
+            content=content,
+            usage=usage,
+            model=getattr(self, "model", ""),
+            finish_reason=finish_reason,
+        )
+
+    def _call_stream(self, prompt: str, system_prompt: str):
+        raise NotImplementedError("streaming not supported by this client")
+        yield  # pragma: no cover - marks this a generator for callers
+
+    def _estimate_usage(
+        self, prompt: str, system_prompt: str, content: str
+    ) -> LLMUsage:
+        """Approximate usage when a stream yields no API usage data.
+
+        Used on early abort (the stream is abandoned before its usage chunk)
+        or with providers that omit usage. Tokens are estimated at ~4 chars
+        each; cost defers to the provider's _estimate_cost.
+        """
+        prompt_tokens = (len(prompt) + len(system_prompt)) // 4
+        completion_tokens = len(content) // 4
+        return LLMUsage(
+            prompt_tokens=prompt_tokens,
+            completion_tokens=completion_tokens,
+            total_tokens=prompt_tokens + completion_tokens,
+            estimated_cost=self._estimate_cost(prompt_tokens, completion_tokens),
+        )
+
+    def _estimate_cost(
+        self, prompt_tokens: int, completion_tokens: int, *args
+    ) -> float:
+        """Dollar cost for a token count. Overridden per provider."""
+        return 0.0
+
+    @contextmanager
+    def track_task(self, task_id: str):
+        """Attribute LLM cost/calls made in this block to a task id."""
+        previous = getattr(self, "_current_task", None)
+        self._current_task = task_id
+        # Snapshot the per-task cost cap on first entry so an "auto" cap is
+        # fixed to the budget available when the task started, not recomputed
+        # (and shrinking) on every call as the task spends.
+        if task_id is not None and task_id not in self._task_caps:
+            self._task_caps[task_id] = self._resolve_task_cap()
+        try:
+            yield
+        finally:
+            self._current_task = previous
+
+    def _resolve_task_cap(self) -> Optional[float]:
+        """Resolve the configured per-task cap to a dollar figure (or None)."""
+        raw = self._max_cost_per_task
+        if isinstance(raw, bool) or raw is None:
+            return None
+        if isinstance(raw, (int, float)):
+            return float(raw)
+        if isinstance(raw, str) and raw.strip().lower() == "auto":
+            return self.budget_remaining * self.AUTO_TASK_CAP_FRACTION
+        return None
+
+    def effective_task_cap(self, task_id: Optional[str]) -> Optional[float]:
+        """The dollar cap for a task, or None when uncapped.
+
+        Prefers the value snapshotted by ``track_task``; falls back to a live
+        resolve so the cap is meaningful even when queried before the task
+        block is entered.
+        """
+        if task_id is None:
+            return None
+        if task_id in self._task_caps:
+            return self._task_caps[task_id]
+        return self._resolve_task_cap()
+
+    def _track_usage(self, usage: LLMUsage) -> None:
+        bucket = getattr(self, "_current_task", None) or "overhead"
+        with self._usage_lock:
+            self.cumulative_usage.prompt_tokens += usage.prompt_tokens
+            self.cumulative_usage.completion_tokens += usage.completion_tokens
+            self.cumulative_usage.total_tokens += usage.total_tokens
+            self.cumulative_usage.estimated_cost += usage.estimated_cost
+            self.cumulative_usage.call_count += 1
+            self.cumulative_usage.cache_creation_tokens += usage.cache_creation_tokens
+            self.cumulative_usage.cache_read_tokens += usage.cache_read_tokens
+            self.cost_by_task[bucket] = (
+                self.cost_by_task.get(bucket, 0.0) + usage.estimated_cost
+            )
+            running = self.cumulative_usage.estimated_cost
+        logger.debug(
+            "llm call model=%s bucket=%s tokens=%d/%d cost=$%.4f cumulative=$%.4f",
+            getattr(self, "model", ""),
+            bucket,
+            usage.prompt_tokens,
+            usage.completion_tokens,
+            usage.estimated_cost,
+            running,
+        )
+
+    @property
+    def budget_remaining(self) -> float:
+        return max(0.0, self._budget - self.cumulative_usage.estimated_cost)
+
+    def task_cost(self, task_id: Optional[str]) -> float:
+        """Accumulated cost attributed to a task id (0.0 if untracked)."""
+        if task_id is None:
+            return 0.0
+        return getattr(self, "cost_by_task", {}).get(task_id, 0.0)
+
+    def task_cost_exceeded(self, task_id: Optional[str]) -> bool:
+        """True when task_id has crossed its snapshotted per-task cost cap."""
+        if task_id is None:
+            return False
+        cap = self.effective_task_cap(task_id)
+        if cap is None:
+            return False
+        return self.task_cost(task_id) >= cap

misterdev/llm/client/edits.py

@@ -0,0 +1,70 @@
+# Structured tool for edit extraction. Forcing this (when a model supports
+# `tools`) replaces brittle markdown-fence parsing: the model returns
+# well-formed JSON we render back into the canonical fence format the executor
+# already consumes, so nothing downstream changes.
+APPLY_EDITS_TOOL = {
+    "type": "function",
+    "function": {
+        "name": "apply_edits",
+        "description": (
+            "Write the complete final content of each file to create or modify "
+            "to satisfy the task."
+        ),
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "edits": {
+                    "type": "array",
+                    "items": {
+                        "type": "object",
+                        "properties": {
+                            "path": {
+                                "type": "string",
+                                "description": "Project-relative file path.",
+                            },
+                            "content": {
+                                "type": "string",
+                                "description": "Full final content of the file.",
+                            },
+                        },
+                        "required": ["path", "content"],
+                    },
+                }
+            },
+            "required": ["edits"],
+        },
+    },
+}
+
+
+def _edits_to_markdown(edits: list) -> str:
+    """Render structured edits into the canonical ```lang:path fence format.
+
+    The executor's parser keys on the ``path`` after the colon; the language
+    token is re-derived from the path downstream, so a placeholder is fine.
+    """
+    blocks = []
+    for edit in edits:
+        if not isinstance(edit, dict):
+            continue
+        path = edit.get("path")
+        content = edit.get("content", "")
+        if path:
+            blocks.append(f"```text:{path}\n{content}\n```")
+    return "\n\n".join(blocks)
+
+
+def code_gen_abort_check(accumulated: str) -> bool:
+    """Heuristic: True when a code-gen stream is clearly going wrong.
+
+    Trips when a lot of text arrives with no code fence or file marker, or when
+    the model opens with conversational filler instead of code.
+    """
+    if (
+        len(accumulated) > 2000
+        and "```" not in accumulated
+        and "# File:" not in accumulated
+    ):
+        return True
+    head = accumulated[:200]
+    return ("I'll help you" in head) or ("Sure, here" in head)