core-runtime 0.1.1__py3-none-any.whl

shared/infrastructure/llm/fallback_client.py

@@ -0,0 +1,144 @@
+# src/shared/infrastructure/llm/fallback_client.py
+
+"""
+FallbackAwareLLMClient — wraps an ordered chain of LLM resources with
+lazy client construction and transparent fallback.
+
+Each resource in the chain is held as a factory callable. The wrapper
+builds the underlying ``LLMClient`` only when a call actually reaches
+that position. Two failure modes are skipped and recorded as "tried":
+
+1. ``httpx.HTTPStatusError`` with status 402 (Payment Required) or 429
+   (Too Many Requests) — the resource is reachable but out of quota.
+2. ``ValueError`` raised by the factory itself — typically a provisioning
+   failure (missing ``api_url`` / ``model_name`` in ``runtime_settings``,
+   missing ``env_prefix``). The resource cannot be reached at all.
+
+When every resource has been disqualified, ``ProviderQuotaExhausted`` is
+raised with the per-resource reason tags. Other exceptions propagate
+unchanged so genuine bugs and 5xx failures are not masked.
+
+Lazy construction matters: prior to this design, the orchestrator built
+clients eagerly via a list comprehension, so a single misconfigured
+resource anywhere in the qualified set raised ``ValueError`` during
+chain construction and poisoned the entire role — even when an earlier,
+healthy resource would have served the call. Surfaced as a regression
+in the Architect role (#293 follow-up).
+"""
+
+from __future__ import annotations
+
+from collections.abc import Awaitable, Callable
+from typing import Any
+
+import httpx
+
+from shared.exceptions import ProviderQuotaExhausted
+from shared.infrastructure.llm.client import LLMClient
+from shared.logger import getLogger
+
+
+logger = getLogger(__name__)
+
+
+_RETRYABLE_STATUSES: frozenset[int] = frozenset({402, 429})
+
+ClientFactory = Callable[[], Awaitable[LLMClient]]
+
+
+# ID: 4e7b9d2a-3c1f-4a85-b6d9-2e8c5f7a1b3d
+class FallbackAwareLLMClient:
+    """
+    Duck-compatible wrapper around an ordered chain of LLM resources.
+    Exposes the same public surface as ``LLMClient``
+    (``make_request_async``, ``make_request_with_system_async``,
+    ``get_embedding``) so existing callers of
+    ``CognitiveService.aget_client_for_role`` need no change.
+    """
+
+    def __init__(
+        self,
+        client_factories: list[ClientFactory],
+        resource_names: list[str],
+    ) -> None:
+        if not client_factories:
+            raise ValueError(
+                "FallbackAwareLLMClient requires at least one client factory."
+            )
+        if len(client_factories) != len(resource_names):
+            raise ValueError(
+                "client_factories and resource_names must have the same length "
+                f"(got {len(client_factories)} factories, "
+                f"{len(resource_names)} names)."
+            )
+        self._client_factories = client_factories
+        self._resource_names = resource_names
+        self._clients: dict[int, LLMClient] = {}
+        self._unavailable: dict[int, str] = {}
+
+    # ID: 9c2e4d6f-5a8b-4d72-9e1c-3f8b6a2d5c4e
+    async def make_request_async(self, *args: Any, **kwargs: Any) -> str:
+        return await self._call_with_fallback("make_request_async", *args, **kwargs)
+
+    # ID: b7d3a5c9-2e4f-4861-9c8b-5d7a3e1f9c4b
+    async def make_request_with_system_async(self, *args: Any, **kwargs: Any) -> str:
+        return await self._call_with_fallback(
+            "make_request_with_system_async", *args, **kwargs
+        )
+
+    # ID: 5f8a2c4d-9e3b-4751-8c2d-6a4b9e3f1c7d
+    async def get_embedding(self, *args: Any, **kwargs: Any) -> list[float]:
+        return await self._call_with_fallback("get_embedding", *args, **kwargs)
+
+    async def _ensure_client(self, idx: int) -> LLMClient:
+        if idx not in self._clients:
+            self._clients[idx] = await self._client_factories[idx]()
+        return self._clients[idx]
+
+    async def _call_with_fallback(
+        self, method_name: str, *args: Any, **kwargs: Any
+    ) -> Any:
+        tried: list[tuple[str, str]] = []
+        for idx in range(len(self._client_factories)):
+            name = self._resource_names[idx]
+
+            if idx in self._unavailable:
+                tried.append((name, self._unavailable[idx]))
+                continue
+
+            try:
+                client = await self._ensure_client(idx)
+            except ValueError as e:
+                tag = "provisioning_error"
+                self._unavailable[idx] = tag
+                tried.append((name, tag))
+                logger.warning(
+                    "LLM resource '%s' failed to provision (%s) — marking "
+                    "unavailable, trying next fallback (%d/%d)",
+                    name,
+                    e,
+                    idx + 1,
+                    len(self._client_factories),
+                )
+                continue
+
+            try:
+                return await getattr(client, method_name)(*args, **kwargs)
+            except httpx.HTTPStatusError as e:
+                status = e.response.status_code
+                if status not in _RETRYABLE_STATUSES:
+                    raise
+                tag = str(status)
+                self._unavailable[idx] = tag
+                tried.append((name, tag))
+                logger.warning(
+                    "LLM resource '%s' returned %d on %s — marking "
+                    "unavailable, trying next fallback (%d/%d)",
+                    name,
+                    status,
+                    method_name,
+                    idx + 1,
+                    len(self._client_factories),
+                )
+
+        raise ProviderQuotaExhausted(tried_resources=tried)

shared/infrastructure/llm/providers/anthropic.py

@@ -0,0 +1,113 @@
+# src/shared/infrastructure/llm/providers/anthropic.py
+
+"""
+Provides an AIProvider implementation for the Anthropic Messages API.
+Uses httpx directly — no Anthropic SDK dependency required.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import httpx
+
+from shared.infrastructure.intent.operational_config import load_operational_config
+from shared.logger import getLogger
+
+from .base import AIProvider
+
+
+logger = getLogger(__name__)
+
+_DEFAULT_API_URL = "https://api.anthropic.com"
+_ANTHROPIC_VERSION = "2023-06-01"
+_DEFAULT_SYSTEM = "You are a helpful assistant."
+
+
+# ID: 3a1b2c4d-5e6f-7a8b-9c0d-1e2f3a4b5c6d
+class AnthropicProvider(AIProvider):
+    """Provider for the Anthropic Messages API."""
+
+    def _prepare_headers(self) -> dict[str, str]:
+        headers = {
+            "Content-Type": "application/json",
+            "anthropic-version": _ANTHROPIC_VERSION,
+        }
+        if self.api_key:
+            headers["x-api-key"] = self.api_key
+        return headers
+
+    # ID: 7b8c9d0e-1f2a-3b4c-5d6e-7f8a9b0c1d2e
+    async def chat_completion(
+        self,
+        prompt: str,
+        user_id: str,
+        system_prompt: str = "",
+        max_tokens: int | None = None,
+        response_format: dict[str, Any] | None = None,
+        usage_sink: dict[str, int] | None = None,
+    ) -> str:
+        """
+        Generates a chat completion using the Anthropic Messages API.
+
+        Args:
+            prompt: User-turn content.
+            user_id: Audit identifier (not forwarded — Anthropic API has no user field).
+            system_prompt: Constitutional system prompt. Falls back to a neutral
+                           default when empty.
+            max_tokens: Maximum tokens to generate. When None, falls back to the
+                        operational default.
+            response_format: Optional provider-agnostic structured-output contract.
+
+                Currently ignored for Anthropic in this implementation.
+                Anthropic structured output requires a different mapping model
+                (typically tool use / constrained interfaces), so unsupported
+                formats safely fall back to standard text generation.
+        """
+        if max_tokens is None:
+            max_tokens = load_operational_config().llm.default_max_tokens
+        if response_format:
+            logger.debug(
+                "AnthropicProvider received response_format type '%s' but will ignore it "
+                "and fall back to standard text generation.",
+                response_format.get("type"),
+            )
+
+        api_url = self.api_url if self.api_url else _DEFAULT_API_URL
+        endpoint = f"{api_url}/v1/messages"
+        effective_system = (
+            system_prompt.strip() if system_prompt.strip() else _DEFAULT_SYSTEM
+        )
+
+        payload = {
+            "model": self.model_name,
+            "max_tokens": max_tokens,
+            "system": effective_system,
+            "messages": [
+                {"role": "user", "content": prompt},
+            ],
+        }
+
+        async with httpx.AsyncClient(timeout=self.timeout) as client:
+            response = await client.post(endpoint, headers=self.headers, json=payload)
+            response.raise_for_status()
+            data = response.json()
+            if usage_sink is not None:
+                usage = data.get("usage") or {}
+                if "input_tokens" in usage:
+                    usage_sink["prompt_tokens"] = int(usage["input_tokens"])
+                if "output_tokens" in usage:
+                    usage_sink["completion_tokens"] = int(usage["output_tokens"])
+            return data["content"][0]["text"]
+
+    # ID: 4e5f6a7b-8c9d-0e1f-2a3b-4c5d6e7f8a9b
+    async def get_embedding(
+        self,
+        text: str,
+        usage_sink: dict[str, int] | None = None,
+    ) -> list[float]:
+        """Not supported — Anthropic does not offer an embeddings endpoint."""
+        raise NotImplementedError(
+            "AnthropicProvider does not support embeddings. "
+            "Use OllamaProvider or OpenAIProvider for embedding generation."
+        )

shared/infrastructure/llm/providers/base.py

@@ -0,0 +1,130 @@
+# src/shared/infrastructure/llm/providers/base.py
+
+"""
+Defines the abstract base class for all AI provider strategies.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+import httpx
+
+from shared.infrastructure.intent.operational_config import load_operational_config
+
+
+# ID: 32b9740b-010f-4fd0-8886-f17093aa855f
+class AIProvider(ABC):
+    """
+    Abstract base class defining the interface for an AI service provider.
+    """
+
+    def __init__(
+        self,
+        api_url: str,
+        model_name: str,
+        api_key: str | None = None,
+        timeout: int | None = None,
+    ):
+        if timeout is None:
+            timeout = load_operational_config().llm.provider_timeout_sec
+        self.api_url = api_url.rstrip("/")
+        self.model_name = model_name
+        self.api_key = api_key
+        self.timeout = httpx.Timeout(timeout)
+        self.headers = self._prepare_headers()
+
+    @abstractmethod
+    def _prepare_headers(self) -> dict[str, str]:
+        """Prepare the specific headers for this provider."""
+        pass
+
+    @abstractmethod
+    # ID: af87b72f-3b74-419d-b6c1-635c4185c033
+    async def chat_completion(
+        self,
+        prompt: str,
+        user_id: str,
+        system_prompt: str = "",
+        max_tokens: int | None = None,
+        response_format: dict[str, Any] | None = None,
+        usage_sink: dict[str, int] | None = None,
+    ) -> str:
+        """
+        Generates a text completion for a given prompt.
+
+        Args:
+            prompt: User-turn content.
+            user_id: Audit identifier.
+            system_prompt: Constitutional system prompt. Empty string uses
+                           the provider's built-in default.
+            max_tokens: Maximum tokens to generate. Controls response length;
+                        passed directly to the underlying API. When None,
+                        concrete implementations fall back to the operational
+                        default.
+            response_format: Optional provider-agnostic structured-output
+                             request contract.
+
+                             Supported shapes from upper layers:
+                                 {"type": "json_object"}
+                                 {"type": "json_schema", "schema": {...}}
+
+                             Providers may:
+                                 - support both shapes natively,
+                                 - support only one of them,
+                                 - ignore unsupported variants and fall back
+                                   to normal text generation.
+            usage_sink: Optional dict mutated in place after a successful
+                        HTTP response to record token usage from the provider
+                        envelope. Populated keys: 'prompt_tokens' and
+                        'completion_tokens' when the provider exposes them.
+                        Allocated per-call by the caller; never shared across
+                        concurrent invocations.
+
+        Returns:
+            Raw provider response content as a string.
+        """
+        pass
+
+    @abstractmethod
+    # ID: bf6da823-1185-4a93-98bb-da095eb92f4f
+    async def get_embedding(
+        self,
+        text: str,
+        usage_sink: dict[str, int] | None = None,
+    ) -> list[float]:
+        """Generate an embedding vector for a given text.
+
+        Args:
+            text: Source text to embed.
+            usage_sink: Optional dict mutated in place with 'prompt_tokens'
+                        when the provider exposes embedding token counts.
+        """
+        pass
+
+    # ID: 6f5d4b78-0a92-4a3f-9c50-bb3a4f0e7b21
+    async def get_embeddings_batch(
+        self,
+        texts: list[str],
+        usage_sink: dict[str, int] | None = None,
+    ) -> list[list[float]]:
+        """Generate embedding vectors for a list of texts.
+
+        Default implementation loops `get_embedding` per text. Providers
+        whose backend natively supports list input (e.g., Ollama's
+        `/api/embed`) MUST override with a single round-trip variant to
+        realise the throughput win (#461). Providers without batch support
+        inherit this default unchanged — the call returns identical
+        vectors to N serial single-input calls, just without speedup.
+
+        Args:
+            texts: Source texts to embed.
+            usage_sink: Optional dict mutated in place with summed
+                        `prompt_tokens` across the batch. Per-text token
+                        attribution is not preserved (#461 D2 decision).
+        """
+        results: list[list[float]] = []
+        for text in texts:
+            results.append(await self.get_embedding(text, usage_sink=usage_sink))
+        return results

shared/infrastructure/llm/providers/ollama.py

@@ -0,0 +1,265 @@
+# src/shared/infrastructure/llm/providers/ollama.py
+
+"""
+Provides an AIProvider implementation for Ollama APIs.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import httpx
+
+from shared.infrastructure.intent.operational_config import load_operational_config
+from shared.logger import getLogger
+
+from .base import AIProvider
+
+
+logger = getLogger(__name__)
+GHOST_VECTOR_START = [0.63719, 0.45393, -4.16063]
+_DEFAULT_SYSTEM = "You are a helpful assistant."
+
+# nomic-embed-text supports up to 8192 tokens. At ~3 chars/token for source
+# code, the configured cap (default 20000) sits safely under the limit for
+# the vast majority of files.
+_CFG_EMB = load_operational_config().embedding
+
+
+# ID: 3f78f7ca-33b1-4ac3-a701-30885722e7b1
+class OllamaProvider(AIProvider):
+    """Provider for Ollama-compatible chat and embedding APIs."""
+
+    def _prepare_headers(self) -> dict[str, str]:
+        return {"Content-Type": "application/json"}
+
+    # ID: b4ddef76-9da6-4b19-ad12-8f92eac28f86
+    async def chat_completion(
+        self,
+        prompt: str,
+        user_id: str,
+        system_prompt: str = "",
+        max_tokens: int | None = None,
+        response_format: dict[str, Any] | None = None,
+        usage_sink: dict[str, int] | None = None,
+    ) -> str:
+        """
+        Generates a chat completion using the Ollama /api/chat format.
+
+        Args:
+            prompt: User-turn content.
+            user_id: Audit identifier (unused by Ollama but kept for interface parity).
+            system_prompt: Constitutional system prompt sent as the first
+                           system-role message. Falls back to a neutral default
+                           when empty so the model always receives a system turn.
+            max_tokens: Mapped to Ollama's options.num_predict to cap output length.
+                        When None, falls back to the operational default.
+            response_format: Optional provider-agnostic structured-output contract.
+
+                Supported:
+                    {"type": "json_object"}
+                    {"type": "json_schema", "schema": {...}}
+
+                Ollama mapping:
+                    - json_object -> payload["format"] = "json"
+                    - json_schema -> payload["format"] = <raw schema dict>
+
+        Returns:
+            Raw text content from Ollama's assistant message.
+        """
+        if max_tokens is None:
+            max_tokens = load_operational_config().llm.default_max_tokens
+        endpoint = f"{self.api_url}/api/chat"
+        effective_system = (
+            system_prompt.strip() if system_prompt.strip() else _DEFAULT_SYSTEM
+        )
+
+        payload: dict[str, Any] = {
+            "model": self.model_name,
+            "messages": [
+                {"role": "system", "content": effective_system},
+                {"role": "user", "content": prompt},
+            ],
+            "stream": False,
+            "options": {
+                "num_predict": max_tokens,
+            },
+        }
+
+        if response_format:
+            format_type = response_format.get("type")
+
+            if format_type == "json_schema":
+                schema = response_format.get("schema")
+                if isinstance(schema, dict) and schema:
+                    payload["format"] = schema
+                    logger.debug(
+                        "OllamaProvider using native JSON Schema structured output "
+                        "for model '%s'.",
+                        self.model_name,
+                    )
+                else:
+                    logger.warning(
+                        "OllamaProvider received invalid json_schema response_format; "
+                        "falling back to normal text generation."
+                    )
+            elif format_type == "json_object":
+                payload["format"] = "json"
+                logger.debug(
+                    "OllamaProvider using JSON object mode for model '%s'.",
+                    self.model_name,
+                )
+            else:
+                logger.warning(
+                    "OllamaProvider received unsupported response_format type '%s'; "
+                    "falling back to normal text generation.",
+                    format_type,
+                )
+
+        async with httpx.AsyncClient(timeout=self.timeout) as client:
+            response = await client.post(endpoint, headers=self.headers, json=payload)
+            response.raise_for_status()
+            data = response.json()
+            if usage_sink is not None:
+                if "prompt_eval_count" in data:
+                    usage_sink["prompt_tokens"] = int(data["prompt_eval_count"])
+                if "eval_count" in data:
+                    usage_sink["completion_tokens"] = int(data["eval_count"])
+            return data["message"]["content"]
+
+    # ID: fcc3342d-746d-4bb4-b153-8eef9465c0f0
+    async def get_embedding(
+        self,
+        text: str,
+        usage_sink: dict[str, int] | None = None,
+    ) -> list[float]:
+        """
+        Generates an embedding using the Ollama /api/embed format (Ollama 0.4+).
+
+        Uses the current Ollama embedding API:
+            POST /api/embed
+            payload: {"model": ..., "input": ..., "options": {"num_ctx": ...}}
+            response: {"embeddings": [[...]]}
+
+        num_ctx is passed explicitly per-request because Ollama ignores
+        num_ctx in the Modelfile for embedding models.
+
+        Input is truncated to _CFG_EMB.max_chars before sending as a
+        last-resort safety net.
+
+        Raises:
+            RuntimeError: If the model returns a Ghost Vector, indicating
+                          an Ollama model failure rather than a real embedding.
+        """
+        endpoint = f"{self.api_url}/api/embed"
+
+        if len(text) > _CFG_EMB.max_chars:
+            logger.warning(
+                "Embedding input truncated from %d to %d chars (model context limit).",
+                len(text),
+                _CFG_EMB.max_chars,
+            )
+            text = text[: _CFG_EMB.max_chars]
+
+        payload = {
+            "model": self.model_name,
+            "input": text,
+            "options": {"num_ctx": 8192},
+        }
+
+        async with httpx.AsyncClient(timeout=self.timeout) as client:
+            response = await client.post(endpoint, headers=self.headers, json=payload)
+            response.raise_for_status()
+            data = response.json()
+            vec = data["embeddings"][0]
+
+            if len(vec) > 3:
+                is_ghost = all(
+                    abs(a - b) < 0.001 for a, b in zip(vec[:3], GHOST_VECTOR_START)
+                )
+                if is_ghost:
+                    logger.error(
+                        "Ollama returned Ghost Vector (Model Failure) for input length %s",
+                        len(text),
+                    )
+                    raise RuntimeError("Embedding model failed (Ghost Vector returned)")
+
+            return vec
+
+    # ID: 9e2c4f81-3a6d-4b85-b4f7-c8d3a1e09f72
+    async def get_embeddings_batch(
+        self,
+        texts: list[str],
+        usage_sink: dict[str, int] | None = None,
+    ) -> list[list[float]]:
+        """Generate embeddings for a list of texts in a single round-trip.
+
+        Uses Ollama's `/api/embed` list-input form (Ollama 0.4+):
+            payload: {"model": ..., "input": [t1, t2, ...], "options": {...}}
+            response: {"embeddings": [[...], [...], ...]} aligned to input order
+
+        Empty input list returns `[]` without an HTTP call. Each text is
+        truncated to `_CFG_EMB.max_chars` before sending, matching the
+        single-input path. Ghost-vector check applied per-element; one bad
+        vector raises and aborts the whole batch (consumer handles the
+        retry/skip — same posture as the single-input path).
+
+        Per #461 D2: token usage from the batch response is summed into
+        `usage_sink`; per-text attribution is lost.
+        """
+        if not texts:
+            return []
+
+        endpoint = f"{self.api_url}/api/embed"
+
+        prepared: list[str] = []
+        for text in texts:
+            if len(text) > _CFG_EMB.max_chars:
+                logger.warning(
+                    "Embedding input truncated from %d to %d chars (model context limit).",
+                    len(text),
+                    _CFG_EMB.max_chars,
+                )
+                prepared.append(text[: _CFG_EMB.max_chars])
+            else:
+                prepared.append(text)
+
+        payload = {
+            "model": self.model_name,
+            "input": prepared,
+            "options": {"num_ctx": 8192},
+        }
+
+        async with httpx.AsyncClient(timeout=self.timeout) as client:
+            response = await client.post(endpoint, headers=self.headers, json=payload)
+            response.raise_for_status()
+            data = response.json()
+            vectors = data["embeddings"]
+
+            if len(vectors) != len(prepared):
+                raise RuntimeError(
+                    f"Ollama returned {len(vectors)} embeddings for {len(prepared)} "
+                    "inputs — batch response misalignment"
+                )
+
+            for i, vec in enumerate(vectors):
+                if len(vec) > 3:
+                    is_ghost = all(
+                        abs(a - b) < 0.001 for a, b in zip(vec[:3], GHOST_VECTOR_START)
+                    )
+                    if is_ghost:
+                        logger.error(
+                            "Ollama returned Ghost Vector (Model Failure) for batch "
+                            "input %d/%d (length %s)",
+                            i,
+                            len(prepared),
+                            len(prepared[i]),
+                        )
+                        raise RuntimeError(
+                            "Embedding model failed (Ghost Vector returned in batch)"
+                        )
+
+            if usage_sink is not None and "prompt_eval_count" in data:
+                usage_sink["prompt_tokens"] = int(data["prompt_eval_count"])
+
+            return vectors