cct-cli 0.7.9.0__py3-none-any.whl

calc_terminal/resilience/capability_matcher.py

@@ -0,0 +1,174 @@
+"""
+CAT — Backup Provider & Resilience System: Capability Matcher
+Author: Kazi Zillani
+
+Ensures backup selection is intelligent rather than sequential:
+Compares Task Requirements + Provider Capabilities + Privacy Policy + Cost Policy
+to select only backups that can actually perform the user's task.
+"""
+
+from typing import Any, Dict, List, Optional, Tuple
+
+from ..model_router import (
+    POLICY_BEST_AVAILABLE,
+    POLICY_CLOUD_FIRST,
+    POLICY_LOCAL_FIRST,
+    POLICY_NEVER_CLOUD,
+    ModelCapabilities,
+    _capabilities_for,
+)
+from .health_monitor import get_health_monitor
+from .types import CandidateTarget, CostPolicy, SecurityLevel, TaskRequirements
+
+
+class CapabilityMatcher:
+    """Matches task requirements against candidate provider/model endpoints."""
+
+    def __init__(self):
+        self.health_monitor = get_health_monitor()
+
+    def is_local_provider(self, provider: str) -> bool:
+        return str(provider or "").strip().lower() in ("ollama", "local", "vllm", "llamacpp")
+
+    def is_free_provider(self, provider: str, model: str = "") -> bool:
+        p_low = str(provider or "").strip().lower()
+        if self.is_local_provider(p_low):
+            return True
+        # Free cloud providers/tiers
+        if p_low in ("groq", "cerebras", "gemini"):
+            return True
+        return False
+
+    def evaluate_target(
+        self,
+        config: dict,
+        requirements: TaskRequirements,
+        privacy_policy: str = POLICY_LOCAL_FIRST,
+        cost_policy: CostPolicy = CostPolicy.ANY,
+        security_level: SecurityLevel = SecurityLevel.BALANCED,
+        entry: Optional[dict] = None,
+        is_backup: bool = True,
+        tier: int = 1,
+    ) -> Optional[CandidateTarget]:
+        """Verify eligibility and score a candidate target. Returns CandidateTarget or None."""
+        provider = str(config.get("provider") or "").strip().lower()
+        model = str(config.get("model") or "").strip()
+        if not provider or not model:
+            return None
+
+        is_local = self.is_local_provider(provider)
+
+        # 1. Privacy Policy Enforcement (ABSOLUTE RULE: User privacy overrides failover)
+        if privacy_policy == POLICY_NEVER_CLOUD and not is_local:
+            return None
+
+        # 2. Cost Policy Enforcement
+        if cost_policy == CostPolicy.LOCAL_ONLY and not is_local:
+            return None
+        if cost_policy == CostPolicy.FREE_ONLY and not self.is_free_provider(provider, model):
+            return None
+
+        # 3. Credential Check (Ollama needs no key; cloud needs API key)
+        needs_key = config.get("needs_key", not is_local)
+        if is_local:
+            needs_key = False
+        api_key = str(config.get("api_key") or "").strip()
+        if needs_key and not api_key:
+            return None
+
+        # 4. Capability Check
+        caps = _capabilities_for(config, is_backup=is_backup)
+
+        # Vision requirement
+        if requirements.vision and not caps.vision:
+            return None
+
+        # Tools requirement (e.g. coding agent requires tool loop compatibility)
+        if requirements.tools:
+            # Base models that cannot follow instructions should be skipped
+            m_low = model.lower()
+            if any(b in m_low for b in ("-base", "_base", "base-q", ":base")):
+                return None
+            if not caps.tools:
+                return None
+
+        # Context Window requirement
+        if requirements.min_context > 0 and caps.context_window < requirements.min_context:
+            return None
+        if requirements.long_context and caps.context_window < 60000:
+            return None
+
+        # 5. Circuit Breaker & Health Check
+        health = self.health_monitor.get_health(provider, model)
+        if health.circuit_open and not self.health_monitor.is_healthy(provider, model):
+            # Barred by active cooldown
+            return None
+
+        # 6. Scoring
+        score = self.calculate_score(caps, health, requirements, privacy_policy, is_local)
+
+        return CandidateTarget(
+            provider=provider,
+            model=model,
+            config=dict(config),
+            entry=entry,
+            is_backup=is_backup,
+            is_local=is_local,
+            score=score,
+            tier=tier,
+        )
+
+    def calculate_score(
+        self,
+        caps: ModelCapabilities,
+        health: Any,
+        requirements: TaskRequirements,
+        privacy_policy: str,
+        is_local: bool,
+    ) -> float:
+        """Dynamic routing score (0 to 100) based on observable CAT state."""
+        score = 50.0
+
+        # Capability fitness (up to +25)
+        if requirements.coding:
+            m_low = caps.model.lower()
+            if any(k in m_low for k in ("coder", "code", "qwen", "deepseek", "sonnet")):
+                score += 15.0
+        if requirements.reasoning and caps.reasoning:
+            score += 10.0
+        if requirements.vision and caps.vision:
+            score += 10.0
+
+        # Privacy fitness (up to +20)
+        if privacy_policy == POLICY_LOCAL_FIRST:
+            score += 20.0 if is_local else 0.0
+        elif privacy_policy == POLICY_CLOUD_FIRST:
+            score += 20.0 if not is_local else 5.0
+        elif privacy_policy == POLICY_BEST_AVAILABLE:
+            score += 10.0
+
+        # Health & Latency fitness (up to +20)
+        if health.status in ("healthy", "local"):
+            score += 15.0
+        elif health.status == "slow":
+            score += 5.0
+        if health.latency_ms:
+            if health.latency_ms < 1000:
+                score += 5.0
+            elif health.latency_ms > 4000:
+                score -= 10.0
+
+        # Consecutive failures penalty
+        score -= min(30.0, health.consecutive_failures * 10.0)
+
+        return max(0.0, min(100.0, score))
+
+
+_GLOBAL_MATCHER: Optional[CapabilityMatcher] = None
+
+
+def get_capability_matcher() -> CapabilityMatcher:
+    global _GLOBAL_MATCHER
+    if _GLOBAL_MATCHER is None:
+        _GLOBAL_MATCHER = CapabilityMatcher()
+    return _GLOBAL_MATCHER

calc_terminal/resilience/circuit_breaker.py

@@ -0,0 +1,158 @@
+"""
+CAT — Backup Provider & Resilience System: Circuit Breaker
+Author: Kazi Zillani
+
+Prevents CAT from wasting precious seconds or burning rate limits on broken,
+unreachable, or rate-limited AI endpoints.
+"""
+
+import threading
+import time
+from typing import Dict, Optional, Tuple
+
+
+class CircuitBreakerState:
+    CLOSED = "closed"        # Normal operation: requests pass through
+    OPEN = "open"            # Tripped: requests fail immediately to prevent stalls
+    HALF_OPEN = "half_open"  # Cooldown expired: single probe attempt allowed
+
+
+class CircuitBreaker:
+    """Per-endpoint circuit breaker guarding providers and models."""
+
+    def __init__(
+        self,
+        failure_threshold: int = 3,
+        default_cooldown: float = 30.0,
+        rate_limit_cooldown: float = 60.0,
+    ):
+        self.failure_threshold = max(1, failure_threshold)
+        self.default_cooldown = max(0.01, default_cooldown)
+        self.rate_limit_cooldown = max(0.01, rate_limit_cooldown)
+        self._lock = threading.Lock()
+        # (provider, model) -> dict
+        self._endpoints: Dict[Tuple[str, str], dict] = {}
+
+    def _key(self, provider: str, model: str) -> Tuple[str, str]:
+        return (str(provider or "").strip().lower(), str(model or "").strip().lower())
+
+    def _get_record(self, provider: str, model: str) -> dict:
+        k = self._key(provider, model)
+        if k not in self._endpoints:
+            self._endpoints[k] = {
+                "state": CircuitBreakerState.CLOSED,
+                "consecutive_failures": 0,
+                "cooldown_until": 0.0,
+                "last_failure_ts": 0.0,
+                "last_success_ts": 0.0,
+                "last_error": "",
+                "probe_in_flight": False,
+            }
+        return self._endpoints[k]
+
+    def can_attempt(self, provider: str, model: str) -> bool:
+        """True if the endpoint is allowed to receive a request."""
+        with self._lock:
+            rec = self._get_record(provider, model)
+            now = time.time()
+            state = rec["state"]
+
+            if state == CircuitBreakerState.CLOSED:
+                return True
+
+            if state == CircuitBreakerState.OPEN:
+                if now >= rec["cooldown_until"]:
+                    # Cooldown elapsed -> transition to half_open for a probe
+                    rec["state"] = CircuitBreakerState.HALF_OPEN
+                    rec["probe_in_flight"] = True
+                    return True
+                return False
+
+            if state == CircuitBreakerState.HALF_OPEN:
+                # Only allow one probe at a time
+                if not rec["probe_in_flight"]:
+                    rec["probe_in_flight"] = True
+                    return True
+                return False
+
+            return True
+
+    def record_success(self, provider: str, model: str, latency_ms: Optional[float] = None) -> None:
+        """Reset breaker to CLOSED upon a successful turn."""
+        with self._lock:
+            rec = self._get_record(provider, model)
+            rec["state"] = CircuitBreakerState.CLOSED
+            rec["consecutive_failures"] = 0
+            rec["cooldown_until"] = 0.0
+            rec["last_success_ts"] = time.time()
+            rec["probe_in_flight"] = False
+
+    def record_failure(
+        self,
+        provider: str,
+        model: str,
+        error: str = "",
+        is_rate_limit: bool = False,
+        cooldown_seconds: Optional[float] = None,
+    ) -> None:
+        """Increment failure counter and trip breaker if threshold exceeded."""
+        with self._lock:
+            rec = self._get_record(provider, model)
+            rec["consecutive_failures"] += 1
+            now = time.time()
+            rec["last_failure_ts"] = now
+            rec["last_error"] = str(error)[:160]
+            rec["probe_in_flight"] = False
+
+            cooldown = cooldown_seconds or (
+                self.rate_limit_cooldown if is_rate_limit else self.default_cooldown
+            )
+
+            # In HALF_OPEN, any failure immediately trips back to OPEN with full cooldown
+            if rec["state"] == CircuitBreakerState.HALF_OPEN:
+                rec["state"] = CircuitBreakerState.OPEN
+                rec["cooldown_until"] = now + cooldown
+            elif is_rate_limit or rec["consecutive_failures"] >= self.failure_threshold:
+                rec["state"] = CircuitBreakerState.OPEN
+                rec["cooldown_until"] = now + cooldown
+
+    def get_state(self, provider: str, model: str) -> str:
+        """Return current breaker state string."""
+        with self._lock:
+            rec = self._get_record(provider, model)
+            now = time.time()
+            if rec["state"] == CircuitBreakerState.OPEN and now >= rec["cooldown_until"]:
+                return CircuitBreakerState.HALF_OPEN
+            return rec["state"]
+
+    def is_in_cooldown(self, provider: str, model: str) -> bool:
+        """True if the endpoint is strictly barred by an active cooldown."""
+        with self._lock:
+            rec = self._get_record(provider, model)
+            return rec["state"] == CircuitBreakerState.OPEN and time.time() < rec["cooldown_until"]
+
+    def reset(self, provider: str, model: str) -> None:
+        with self._lock:
+            rec = self._get_record(provider, model)
+            rec["state"] = CircuitBreakerState.CLOSED
+            rec["consecutive_failures"] = 0
+            rec["cooldown_until"] = 0.0
+            rec["probe_in_flight"] = False
+
+    def reset_all(self) -> None:
+        with self._lock:
+            for rec in self._endpoints.values():
+                rec["state"] = CircuitBreakerState.CLOSED
+                rec["consecutive_failures"] = 0
+                rec["cooldown_until"] = 0.0
+                rec["probe_in_flight"] = False
+
+
+_GLOBAL_BREAKER: Optional[CircuitBreaker] = None
+
+
+def get_circuit_breaker() -> CircuitBreaker:
+    global _GLOBAL_BREAKER
+    if _GLOBAL_BREAKER is None:
+        _GLOBAL_BREAKER = CircuitBreaker()
+    return _GLOBAL_BREAKER

calc_terminal/resilience/failover_engine.py

@@ -0,0 +1,230 @@
+"""
+CAT — Backup Provider & Resilience System: Failover Engine
+Author: Kazi Zillani
+
+Handles intelligent error classification and two-tier failover orchestration:
+- Tier 1: Alternate fallback models within the SAME provider.
+- Tier 2: Priority-ordered backup providers from the dynamic backup pool.
+"""
+
+import logging
+import re
+import time
+from typing import Any, Dict, List, Optional, Tuple
+
+from .capability_matcher import get_capability_matcher
+from .health_monitor import get_health_monitor, sanitize_error
+from .types import CandidateTarget, CostPolicy, FailureType, SecurityLevel, TaskRequirements
+
+_LOG = logging.getLogger(__name__)
+
+# Error pattern classifiers
+_RATE_LIMIT_HINTS = (
+    "rate limit", "rate_limit", "429", "too many requests", "resource has been exhausted",
+    "insufficient_quota", "quota exceeded", "exceeded your current quota", "tokens per minute",
+    "requests per minute", "tpm limit", "rpm limit",
+)
+
+_AUTH_HINTS = (
+    "401", "403", "unauthorized", "api key", "invalid api key", "authentication failed",
+    "forbidden", "permission denied", "invalid token", "bad api key",
+)
+
+_MODEL_UNAVAILABLE_HINTS = (
+    "model not found", "does not exist", "model unavailable", "not loaded", "no such model",
+    "404", "model_not_found", "unknown model", "deprecated",
+)
+
+_CONTEXT_HINTS = (
+    "maximum context length", "context length exceeded", "too many tokens",
+    "token limit", "prompt is too long", "exceeds maximum context",
+)
+
+_TRANSIENT_HINTS = (
+    "timed out", "timeout", "connection reset", "connection refused", "connection aborted",
+    "broken pipe", "server disconnected", "502", "503", "504", "bad gateway",
+    "service unavailable", "remote end closed", "incomplete read",
+)
+
+_SERVER_ERROR_HINTS = (
+    "500", "internal server error", "server error", "backend error",
+)
+
+
+class FailoverEngine:
+    """Classifies provider failures and generates optimal two-tier candidate targets."""
+
+    def __init__(self):
+        self.matcher = get_capability_matcher()
+        self.health_monitor = get_health_monitor()
+
+    def classify_failure(self, error_text: str, status_code: Optional[int] = None) -> FailureType:
+        """Classify raw error message or HTTP response into standardized FailureType."""
+        if not error_text and not status_code:
+            return FailureType.UNKNOWN
+
+        code = status_code or 0
+        if code == 429:
+            return FailureType.RATE_LIMIT
+        if code in (401, 403):
+            return FailureType.AUTH_FAILURE
+        if code == 404:
+            return FailureType.MODEL_UNAVAILABLE
+        if code in (502, 503, 504):
+            return FailureType.TRANSIENT
+        if code == 500:
+            return FailureType.SERVER_ERROR
+
+        low = (error_text or "").lower()
+
+        if any(h in low for h in _RATE_LIMIT_HINTS):
+            return FailureType.RATE_LIMIT
+        if any(h in low for h in _AUTH_HINTS):
+            return FailureType.AUTH_FAILURE
+        if any(h in low for h in _MODEL_UNAVAILABLE_HINTS):
+            return FailureType.MODEL_UNAVAILABLE
+        if any(h in low for h in _CONTEXT_HINTS):
+            return FailureType.CONTEXT_TOO_LARGE
+        if any(h in low for h in _TRANSIENT_HINTS):
+            return FailureType.TRANSIENT
+        if any(h in low for h in _SERVER_ERROR_HINTS):
+            return FailureType.SERVER_ERROR
+
+        return FailureType.UNKNOWN
+
+    def is_retryable(self, failure_type: FailureType) -> bool:
+        """True if failure is transient and warrants an immediate backoff retry on same model."""
+        return failure_type in (FailureType.TRANSIENT, FailureType.TIMEOUT)
+
+    def should_failover(self, result_text: str) -> bool:
+        """True if the assistant response represents an error needing failover."""
+        if not result_text or not isinstance(result_text, str):
+            return True
+        if not result_text.strip():
+            return True
+
+        from ..aicore import is_error_response
+        if is_error_response(result_text):
+            return True
+
+        ftype = self.classify_failure(result_text)
+        return ftype != FailureType.UNKNOWN
+
+    def get_provider_fallback_models(self, provider_id: str) -> List[str]:
+        """Fetch alternative models configured for the given provider.
+        For local providers like Ollama, dynamically checks actually-installed
+        models so non-existent catalog entries never cause failover loops."""
+        try:
+            pid = str(provider_id or "").strip().lower()
+            if pid == "ollama":
+                try:
+                    from ..providers.adapters.ollama_adapter import OllamaAdapter
+                    adapter = OllamaAdapter()
+                    live = adapter.discover_models(timeout=1.5)
+                    if live:
+                        installed = []
+                        for m in live:
+                            mid = str(getattr(m, "model_id", "") or "").strip()
+                            if mid and mid not in installed:
+                                installed.append(mid)
+                        if installed:
+                            return installed
+                except Exception:
+                    pass
+            from ..models.manager import get_provider
+            prov = get_provider(provider_id)
+            if prov and prov.get("fallback_models"):
+                return [str(m).strip() for m in prov["fallback_models"] if str(m).strip()]
+        except Exception:
+            pass
+        return []
+
+    def build_failover_targets(
+        self,
+        primary_config: dict,
+        requirements: Optional[TaskRequirements] = None,
+        privacy_policy: str = "local_first",
+        cost_policy: CostPolicy = CostPolicy.ANY,
+        security_level: SecurityLevel = SecurityLevel.BALANCED,
+    ) -> List[CandidateTarget]:
+        """Build the complete, two-tier failover walk:
+        Tier 1: Primary provider (active model, then alternative models on same provider).
+        Tier 2: Backup pool providers in configured priority order.
+        """
+        req = requirements or TaskRequirements()
+        targets: List[CandidateTarget] = []
+        seen_endpoints = set()
+
+        def _add_target(cfg: dict, entry: Optional[dict], is_backup: bool, tier: int):
+            prov = str(cfg.get("provider") or "").strip().lower()
+            model = str(cfg.get("model") or "").strip()
+            if not prov or not model:
+                return
+            key = (prov, model)
+            if key in seen_endpoints:
+                return
+
+            candidate = self.matcher.evaluate_target(
+                config=cfg,
+                requirements=req,
+                privacy_policy=privacy_policy,
+                cost_policy=cost_policy,
+                security_level=security_level,
+                entry=entry,
+                is_backup=is_backup,
+                tier=tier,
+            )
+            if candidate:
+                targets.append(candidate)
+                seen_endpoints.add(key)
+
+        # 1. Primary endpoint
+        prim = dict(primary_config or {})
+        prim_prov = str(prim.get("provider") or "").strip().lower()
+        if prim_prov:
+            # Ensure API key resolved
+            if not (prim.get("api_key") or "").strip():
+                try:
+                    from ..providers.provider_manager import get_env_api_key
+                    k = get_env_api_key(prim_prov)
+                    if k:
+                        prim["api_key"] = k
+                except Exception:
+                    pass
+            _add_target(prim, entry=None, is_backup=False, tier=1)
+
+            # Tier 1 Fallback Models on the SAME provider (capped to at most 2 to avoid cascading hangs)
+            fb_models = self.get_provider_fallback_models(prim_prov)
+            for fb_model in fb_models[:2]:
+                if fb_model.lower() != str(prim.get("model", "")).lower():
+                    tier1_cfg = dict(prim)
+                    tier1_cfg["model"] = fb_model
+                    _add_target(tier1_cfg, entry=None, is_backup=False, tier=1)
+
+        # 2. Tier 2: Dynamic Backup Pool
+        try:
+            from ..providers.provider_manager import backup_configs
+            for bcfg, entry in backup_configs():
+                _add_target(bcfg, entry=entry, is_backup=True, tier=2)
+        except Exception as e:
+            _LOG.debug("Error loading backup configs: %s", e)
+
+        # If strict security, only allow explicitly configured backup providers
+        if security_level == SecurityLevel.STRICT:
+            targets = [t for t in targets if not t.is_backup or (t.entry and t.entry.get("enabled"))]
+
+        # Prioritize fresh endpoints over endpoints recently in cooldown
+        fresh = [t for t in targets if self.health_monitor.is_healthy(t.provider, t.model)]
+        demoted = [t for t in targets if t not in fresh]
+
+        return fresh + demoted
+
+
+_GLOBAL_ENGINE: Optional[FailoverEngine] = None
+
+
+def get_failover_engine() -> FailoverEngine:
+    global _GLOBAL_ENGINE
+    if _GLOBAL_ENGINE is None:
+        _GLOBAL_ENGINE = FailoverEngine()
+    return _GLOBAL_ENGINE