crprotocol 2.0.0__py3-none-any.whl

crp/providers/diagnostic.py

@@ -0,0 +1,251 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""ProviderDiagnostic — 6-step health check (§05, §06 §26.3).
+
+Executed at init() and on demand via diagnose(). Each step produces
+a diagnostic code string indicating pass/fail/warning.
+"""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any
+
+from crp.providers.base import LLMProvider
+
+
+class DiagnosticCode(str, Enum):
+    """All provider diagnostic result codes."""
+
+    # Step 1: Endpoint
+    ENDPOINT_REACHABLE = "endpoint_reachable"
+    ENDPOINT_UNREACHABLE = "endpoint_unreachable"
+    ENDPOINT_TIMEOUT = "endpoint_timeout"
+
+    # Step 2: Auth
+    AUTH_VALID = "auth_valid"
+    AUTH_INVALID = "auth_invalid"
+    AUTH_EXPIRED = "auth_expired"
+
+    # Step 3: Capabilities
+    CAPABILITIES_DETERMINED = "capabilities_determined"
+    CAPABILITIES_UNAVAILABLE = "capabilities_unavailable"
+
+    # Step 4: Tokenizer
+    TOKENIZER_ACCURATE = "tokenizer_accurate"
+    TOKENIZER_MISMATCH = "tokenizer_mismatch"
+    TOKENIZER_UNAVAILABLE = "tokenizer_unavailable"
+
+    # Step 5: Inference
+    INFERENCE_WORKING = "inference_working"
+    INFERENCE_SLOW = "inference_slow"
+    RATE_LIMITED = "rate_limited"
+
+    # Step 6: Context window
+    CONTEXT_WINDOW_VALID = "context_window_valid"
+    CONTEXT_WINDOW_MISMATCH = "context_window_mismatch"
+
+
+@dataclass
+class DiagnosticResult:
+    """Result of a single diagnostic step."""
+
+    step: int
+    name: str
+    code: DiagnosticCode
+    latency_ms: float = 0.0
+    details: dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def passed(self) -> bool:
+        return self.code in _PASSING_CODES
+
+
+_PASSING_CODES = frozenset({
+    DiagnosticCode.ENDPOINT_REACHABLE,
+    DiagnosticCode.AUTH_VALID,
+    DiagnosticCode.CAPABILITIES_DETERMINED,
+    DiagnosticCode.TOKENIZER_ACCURATE,
+    DiagnosticCode.INFERENCE_WORKING,
+    DiagnosticCode.CONTEXT_WINDOW_VALID,
+})
+
+
+@dataclass
+class DiagnosticReport:
+    """Full 6-step diagnostic report."""
+
+    provider_name: str
+    results: list[DiagnosticResult] = field(default_factory=list)
+    total_latency_ms: float = 0.0
+
+    @property
+    def all_passed(self) -> bool:
+        return all(r.passed for r in self.results)
+
+    @property
+    def summary(self) -> dict[str, str]:
+        return {r.name: r.code.value for r in self.results}
+
+
+class ProviderDiagnostic:
+    """Execute 6-step diagnostic sequence on an LLM provider."""
+
+    # Sample texts for tokenizer validation (Step 4)
+    _SAMPLE_SHORT = "Hello world"
+    _SAMPLE_MEDIUM = "The quick brown fox jumps over the lazy dog. " * 10
+    _SAMPLE_LONG = "CRP context relay protocol " * 200
+
+    def diagnose(self, provider: LLMProvider) -> DiagnosticReport:
+        """Run all 6 diagnostic steps and return a report."""
+        report = DiagnosticReport(provider_name=provider.model_name)
+        start = time.monotonic()
+
+        report.results.append(self._step1_endpoint(provider))
+        report.results.append(self._step2_auth(provider))
+        report.results.append(self._step3_capabilities(provider))
+        report.results.append(self._step4_tokenizer(provider))
+        report.results.append(self._step5_inference(provider))
+        report.results.append(self._step6_context_window(provider))
+
+        report.total_latency_ms = (time.monotonic() - start) * 1000
+        return report
+
+    # ------------------------------------------------------------------
+    # Individual diagnostic steps
+    # ------------------------------------------------------------------
+
+    def _step1_endpoint(self, provider: LLMProvider) -> DiagnosticResult:
+        """Step 1: endpoint availability — verify provider is reachable."""
+        t0 = time.monotonic()
+        try:
+            # For CustomProvider this will always pass; cloud providers
+            # will override with actual HTTP ping in their adapters.
+            _ = provider.context_window_size()
+            return DiagnosticResult(
+                step=1,
+                name="endpoint",
+                code=DiagnosticCode.ENDPOINT_REACHABLE,
+                latency_ms=(time.monotonic() - t0) * 1000,
+            )
+        except TimeoutError:
+            return DiagnosticResult(
+                step=1, name="endpoint", code=DiagnosticCode.ENDPOINT_TIMEOUT,
+                latency_ms=(time.monotonic() - t0) * 1000,
+            )
+        except Exception as exc:
+            return DiagnosticResult(
+                step=1, name="endpoint", code=DiagnosticCode.ENDPOINT_UNREACHABLE,
+                latency_ms=(time.monotonic() - t0) * 1000,
+                details={"error": str(exc)},
+            )
+
+    def _step2_auth(self, provider: LLMProvider) -> DiagnosticResult:
+        """Step 2: authentication — verify API key/token accepted."""
+        # Phase 1: CustomProvider has no auth; cloud adapters will override.
+        return DiagnosticResult(step=2, name="auth", code=DiagnosticCode.AUTH_VALID)
+
+    def _step3_capabilities(self, provider: LLMProvider) -> DiagnosticResult:
+        """Step 3: capability probing — query context windows, token limits."""
+        t0 = time.monotonic()
+        try:
+            ctx = provider.context_window_size()
+            return DiagnosticResult(
+                step=3,
+                name="capabilities",
+                code=DiagnosticCode.CAPABILITIES_DETERMINED,
+                latency_ms=(time.monotonic() - t0) * 1000,
+                details={"context_window": ctx, "max_output": provider.max_output_tokens},
+            )
+        except Exception as exc:
+            return DiagnosticResult(
+                step=3, name="capabilities", code=DiagnosticCode.CAPABILITIES_UNAVAILABLE,
+                latency_ms=(time.monotonic() - t0) * 1000,
+                details={"error": str(exc)},
+            )
+
+    def _step4_tokenizer(self, provider: LLMProvider) -> DiagnosticResult:
+        """Step 4: tokenizer validation — test on 3 sample texts."""
+        t0 = time.monotonic()
+        try:
+            counts = [
+                provider.count_tokens(self._SAMPLE_SHORT),
+                provider.count_tokens(self._SAMPLE_MEDIUM),
+                provider.count_tokens(self._SAMPLE_LONG),
+            ]
+            # Sanity: all counts should be positive and roughly proportional
+            if all(c > 0 for c in counts) and counts[2] > counts[1] > counts[0]:
+                return DiagnosticResult(
+                    step=4, name="tokenizer", code=DiagnosticCode.TOKENIZER_ACCURATE,
+                    latency_ms=(time.monotonic() - t0) * 1000,
+                    details={"sample_counts": counts},
+                )
+            return DiagnosticResult(
+                step=4, name="tokenizer", code=DiagnosticCode.TOKENIZER_MISMATCH,
+                latency_ms=(time.monotonic() - t0) * 1000,
+                details={"sample_counts": counts},
+            )
+        except Exception as exc:
+            return DiagnosticResult(
+                step=4, name="tokenizer", code=DiagnosticCode.TOKENIZER_UNAVAILABLE,
+                latency_ms=(time.monotonic() - t0) * 1000,
+                details={"error": str(exc)},
+            )
+
+    def _step5_inference(self, provider: LLMProvider) -> DiagnosticResult:
+        """Step 5: inference test — minimal generation request."""
+        t0 = time.monotonic()
+        try:
+            output, finish_reason = provider.generate_chat(
+                [{"role": "user", "content": "count: 1"}]
+            )
+            latency = (time.monotonic() - t0) * 1000
+            if latency > 30_000:  # 30s threshold
+                code = DiagnosticCode.INFERENCE_SLOW
+            else:
+                code = DiagnosticCode.INFERENCE_WORKING
+            return DiagnosticResult(
+                step=5, name="inference", code=code,
+                latency_ms=latency,
+                details={"output_length": len(output), "finish_reason": finish_reason},
+            )
+        except Exception as exc:
+            err_str = str(exc).lower()
+            if "rate" in err_str or "429" in err_str:
+                code = DiagnosticCode.RATE_LIMITED
+            else:
+                code = DiagnosticCode.INFERENCE_WORKING  # Can't test, assume working
+            return DiagnosticResult(
+                step=5, name="inference", code=code,
+                latency_ms=(time.monotonic() - t0) * 1000,
+                details={"error": str(exc)},
+            )
+
+    def _step6_context_window(self, provider: LLMProvider) -> DiagnosticResult:
+        """Step 6: context window test — verify reported size is accurate."""
+        t0 = time.monotonic()
+        try:
+            ctx = provider.context_window_size()
+            # Verify count_tokens is consistent with context_window_size
+            # by checking a text that we know the approximate token count of
+            test_text = "word " * (ctx // 5)  # ~80% context fill
+            token_count = provider.count_tokens(test_text)
+            ratio = token_count / ctx if ctx > 0 else 0
+            # Token count should be roughly 80% of context (±20%)
+            if 0.5 < ratio < 1.0:
+                code = DiagnosticCode.CONTEXT_WINDOW_VALID
+            else:
+                code = DiagnosticCode.CONTEXT_WINDOW_MISMATCH
+            return DiagnosticResult(
+                step=6, name="context_window", code=code,
+                latency_ms=(time.monotonic() - t0) * 1000,
+                details={"context_window": ctx, "test_tokens": token_count, "ratio": ratio},
+            )
+        except Exception as exc:
+            return DiagnosticResult(
+                step=6, name="context_window", code=DiagnosticCode.CONTEXT_WINDOW_MISMATCH,
+                latency_ms=(time.monotonic() - t0) * 1000,
+                details={"error": str(exc)},
+            )

crp/providers/llamacpp.py

@@ -0,0 +1,224 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""llama.cpp adapter — local model inference via llama-cpp-python or HTTP (§6.1).
+
+Supports two modes:
+  1. **Python binding**: ``pip install llama-cpp-python`` (in-process inference).
+  2. **HTTP server**: ``llama-server --model model.gguf --port 8080``
+     (talks to llama.cpp's OpenAI-compatible endpoint).
+
+Usage (Python binding)::
+
+    from crp.providers.llamacpp import LlamaCppAdapter
+
+    provider = LlamaCppAdapter(model_path="/models/llama3-8b.gguf")
+    output, reason = provider.generate_chat([
+        {"role": "user", "content": "Hello!"},
+    ])
+
+Usage (HTTP server)::
+
+    provider = LlamaCppAdapter(server_url="http://localhost:8080")
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import random
+import time
+import urllib.request
+import urllib.error
+from typing import Any
+
+from crp.providers.base import LLMProvider
+
+logger = logging.getLogger("crp.providers.llamacpp")
+
+
+class LlamaCppAdapter(LLMProvider):
+    """llama.cpp adapter — local inference or HTTP server.
+
+    Args:
+        model_path:     Path to a GGUF model file (Python binding mode).
+        server_url:     Base URL for llama.cpp's HTTP server (e.g. "http://localhost:8080").
+                        If provided, *model_path* is ignored.
+        context_size:   Context window size in tokens (default: 4096).
+        max_tokens:     Max output tokens per generation (default: 2048).
+        n_gpu_layers:   GPU layers for Python binding (default: -1 = all).
+        n_threads:      CPU threads for Python binding (default: os.cpu_count()).
+    """
+
+    def __init__(
+        self,
+        *,
+        model_path: str | None = None,
+        server_url: str | None = None,
+        context_size: int = 4_096,
+        max_tokens: int = 2_048,
+        n_gpu_layers: int = -1,
+        n_threads: int | None = None,
+    ) -> None:
+        if not model_path and not server_url:
+            raise ValueError(
+                "LlamaCppAdapter requires either model_path (Python binding) "
+                "or server_url (HTTP mode)."
+            )
+
+        self._context_size = context_size
+        self._max_tokens = max_tokens
+        self._server_url = server_url
+        self._llama = None  # Lazy-loaded Python binding
+        self._model_path = model_path
+        self._n_gpu_layers = n_gpu_layers
+        self._n_threads = n_threads or (os.cpu_count() or 4)
+
+        if server_url:
+            logger.debug("LlamaCppAdapter: HTTP mode → %s", server_url)
+        else:
+            logger.debug("LlamaCppAdapter: Python binding → %s", model_path)
+
+    # -- lazy init --------------------------------------------------------
+
+    def _ensure_model(self) -> Any:
+        """Lazy-load the llama-cpp-python model on first use."""
+        if self._llama is not None:
+            return self._llama
+
+        if self._server_url:
+            return None  # HTTP mode — no local model
+
+        try:
+            from llama_cpp import Llama
+        except ImportError:
+            raise ImportError(
+                "Python binding mode requires 'llama-cpp-python'. "
+                "Install with: pip install llama-cpp-python"
+            ) from None
+
+        self._llama = Llama(  # type: ignore[assignment]
+            model_path=self._model_path,  # type: ignore[arg-type]
+            n_ctx=self._context_size,
+            n_gpu_layers=self._n_gpu_layers,
+            n_threads=self._n_threads,
+            verbose=False,
+        )
+        return self._llama
+
+    # -- LLMProvider interface --------------------------------------------
+
+    def generate_chat(
+        self, messages: list[dict[str, str]], **kwargs: Any
+    ) -> tuple[str, str]:
+        """Generate via Python binding or HTTP server."""
+        max_tokens = kwargs.pop("max_tokens", self._max_tokens)
+
+        if self._server_url:
+            return self._generate_http(messages, max_tokens, **kwargs)
+        return self._generate_binding(messages, max_tokens, **kwargs)
+
+    def count_tokens(self, text: str) -> int:
+        """Count tokens via llama.cpp's tokenizer or heuristic fallback."""
+        if self._server_url:
+            # HTTP mode: heuristic (no local tokenizer available)
+            return max(1, len(text) // 4)
+
+        model = self._ensure_model()
+        try:
+            tokens = model.tokenize(text.encode("utf-8"))
+            return len(tokens)
+        except Exception:
+            return max(1, len(text) // 4)
+
+    def context_window_size(self) -> int:
+        return self._context_size
+
+    @property
+    def max_output_tokens(self) -> int | None:
+        return self._max_tokens
+
+    @property
+    def model_name(self) -> str:
+        if self._model_path:
+            return os.path.basename(self._model_path)
+        return f"llamacpp@{self._server_url}"
+
+    # -- private ----------------------------------------------------------
+
+    def _generate_binding(
+        self, messages: list[dict[str, str]], max_tokens: int, **kwargs: Any
+    ) -> tuple[str, str]:
+        """Generate using llama-cpp-python's create_chat_completion."""
+        model = self._ensure_model()
+        try:
+            result = model.create_chat_completion(
+                messages=messages,
+                max_tokens=max_tokens,
+                **kwargs,
+            )
+            choice = result["choices"][0]
+            text = choice["message"]["content"] or ""
+            reason = choice.get("finish_reason", "stop")
+            if reason == "length":
+                pass  # Physical wall
+            else:
+                reason = "stop"
+            return (text, reason)
+        except Exception as exc:
+            logger.error("llama.cpp binding error: %s", exc)
+            return ("", "error")
+
+    def _generate_http(
+        self, messages: list[dict[str, str]], max_tokens: int, **kwargs: Any
+    ) -> tuple[str, str]:
+        """Generate via llama.cpp's OpenAI-compatible HTTP endpoint with retry (§audit H5)."""
+        url = f"{self._server_url.rstrip('/')}/v1/chat/completions"  # type: ignore[union-attr]
+        payload = json.dumps({
+            "messages": messages,
+            "max_tokens": max_tokens,
+            **kwargs,
+        }).encode("utf-8")
+
+        max_retries = 4
+        base_delay = 2.0
+        last_exc: Exception | None = None
+
+        for attempt in range(max_retries):
+            req = urllib.request.Request(
+                url,
+                data=payload,
+                headers={
+                    "Content-Type": "application/json",
+                    "Connection": "keep-alive",
+                },
+                method="POST",
+            )
+
+            try:
+                with urllib.request.urlopen(req, timeout=120) as resp:
+                    data = json.loads(resp.read())
+                choice = data["choices"][0]
+                text = choice["message"]["content"] or ""
+                reason = choice.get("finish_reason", "stop")
+                if reason != "length":
+                    reason = "stop"
+                return (text, reason)
+            except (urllib.error.URLError, ConnectionError, TimeoutError, OSError) as exc:
+                last_exc = exc
+                if attempt < max_retries - 1:
+                    delay = base_delay * (2 ** attempt) + random.uniform(0, 0.5)
+                    logger.warning(
+                        "llama.cpp HTTP request failed (attempt %d/%d), retrying in %.1fs: %s",
+                        attempt + 1, max_retries, delay, exc,
+                    )
+                    time.sleep(delay)
+                    continue
+                logger.error("llama.cpp HTTP error after %d retries: %s", max_retries, exc)
+                return ("", "error")
+            except Exception as exc:
+                logger.error("llama.cpp HTTP error (non-retryable): %s", exc)
+                return ("", "error")
+
+        logger.error("llama.cpp HTTP failed after %d retries: %s", max_retries, last_exc)
+        return ("", "error")

crp/providers/manager.py

@@ -0,0 +1,139 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""LLMProviderManager — multi-provider routing with fallback chain (§05).
+
+Supports:
+  - Primary provider selection
+  - Fallback chain: if primary fails, try registered providers in order
+  - Provider registration and retrieval by name
+"""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Generator
+
+from crp.core.errors import ProviderError
+from crp.providers.base import LLMProvider
+
+logger = logging.getLogger("crp.providers.manager")
+
+
+class LLMProviderManager:
+    """Routes requests to one or more LLM providers with fallback.
+
+    Usage::
+
+        mgr = LLMProviderManager(primary_provider)
+        mgr.register(fallback_provider)
+
+        # Generate with automatic fallback
+        output, reason = mgr.generate_with_fallback(messages)
+    """
+
+    def __init__(self, provider: LLMProvider) -> None:
+        self._primary = provider
+        self._providers: dict[str, LLMProvider] = {provider.model_name: provider}
+        self._fallback_order: list[str] = []  # model names in fallback priority
+
+    @property
+    def primary(self) -> LLMProvider:
+        return self._primary
+
+    @property
+    def provider_count(self) -> int:
+        return len(self._providers)
+
+    @property
+    def provider_names(self) -> list[str]:
+        return list(self._providers.keys())
+
+    def register(self, provider: LLMProvider) -> None:
+        """Register an additional provider for fallback routing."""
+        self._providers[provider.model_name] = provider
+        if provider.model_name != self._primary.model_name:
+            self._fallback_order.append(provider.model_name)
+        logger.info("Provider registered: %s (total: %d)", provider.model_name, len(self._providers))
+
+    def get(self, name: str | None = None) -> LLMProvider:
+        """Get a provider by name, or the primary if *name* is None."""
+        if name is None:
+            return self._primary
+        provider = self._providers.get(name)
+        if provider is None:
+            raise ProviderError(
+                f"Provider '{name}' not registered",
+                available=list(self._providers.keys()),
+            )
+        return provider
+
+    def generate_with_fallback(
+        self,
+        messages: list[dict[str, str]],
+        **kwargs: object,
+    ) -> tuple[str, str, str]:
+        """Generate using primary, fall back to registered providers on failure.
+
+        Returns:
+            (output_text, finish_reason, provider_name) — the provider that
+            successfully generated is identified in provider_name.
+
+        Raises:
+            ProviderError: If ALL providers fail.
+        """
+        providers_to_try = [self._primary.model_name] + self._fallback_order
+        last_error: Exception | None = None
+
+        for name in providers_to_try:
+            provider = self._providers.get(name)
+            if provider is None:
+                continue
+            try:
+                output, finish_reason = provider.generate_chat(messages, **kwargs)
+                if name != self._primary.model_name:
+                    logger.warning(
+                        "Primary provider failed, fell back to '%s'", name,
+                    )
+                return output, finish_reason, name
+            except Exception as exc:
+                logger.warning("Provider '%s' failed: %s", name, exc)
+                last_error = exc
+                continue
+
+        raise ProviderError(
+            f"All {len(providers_to_try)} providers failed",
+            available=list(self._providers.keys()),
+        ) from last_error
+
+    def generate_with_tools_fallback(
+        self,
+        messages: list[dict[str, object]],
+        tools: list[dict[str, object]],
+        **kwargs: object,
+    ) -> tuple[str, str, list[dict[str, object]] | None, dict[str, object] | None, str]:
+        """Generate with tools using primary, fall back on failure.
+
+        Returns:
+            (output_text, finish_reason, tool_calls, raw_message, provider_name)
+        """
+        providers_to_try = [self._primary.model_name] + self._fallback_order
+        last_error: Exception | None = None
+
+        for name in providers_to_try:
+            provider = self._providers.get(name)
+            if provider is None or not provider.supports_tools():
+                continue
+            try:
+                result = provider.generate_chat_with_tools(messages, tools, **kwargs)
+                if name != self._primary.model_name:
+                    logger.warning("Tools fallback to '%s'", name)
+                return (*result, name)
+            except Exception as exc:
+                logger.warning("Provider '%s' tools failed: %s", name, exc)
+                last_error = exc
+                continue
+
+        raise ProviderError(
+            "All providers failed for tool-mediated dispatch",
+            available=list(self._providers.keys()),
+        ) from last_error