cct-cli 0.7.9.0__py3-none-any.whl

calc_terminal/resilience/health_monitor.py

@@ -0,0 +1,192 @@
+"""
+CAT — Backup Provider & Resilience System: Health Monitor
+Author: Kazi Zillani
+
+Lightweight, real-time provider and model health tracking based on actual
+CAT observations. Strictly guarantees ZERO credential exposure.
+"""
+
+import re
+import threading
+import time
+from typing import Dict, List, Optional, Tuple
+
+from .circuit_breaker import CircuitBreakerState, get_circuit_breaker
+from .types import ProviderHealthState
+
+_KEY_CLEAN_RE = re.compile(r"(sk-[a-zA-Z0-9_\-]{8,}|gsk_[a-zA-Z0-9_\-]{8,}|AIzaSy[a-zA-Z0-9_\-]{8,}|[a-f0-9]{32,})", re.I)
+
+
+def sanitize_error(text: str) -> str:
+    """Mask any API keys, credentials, or auth tokens from error strings."""
+    if not text or not isinstance(text, str):
+        return ""
+    cleaned = _KEY_CLEAN_RE.sub("[CREDENTIAL_REDACTED]", text)
+    # Remove authorization headers or token params if present
+    cleaned = re.sub(
+        r"((?:authorization[:=]\s*)?bearer\s+|api[_\-\s]?key[:=\s]+|authorization[:=]\s*)['\"]?[a-zA-Z0-9_\.\-]+['\"]?",
+        r"\1[REDACTED]",
+        cleaned,
+        flags=re.I,
+    )
+    return cleaned[:200]
+
+
+class HealthMonitor:
+    """Central registry of observable AI health state across CAT."""
+
+    def __init__(self):
+        self._lock = threading.Lock()
+        # (provider, model) -> ProviderHealthState
+        self._states: Dict[Tuple[str, str], ProviderHealthState] = {}
+        self._breaker = get_circuit_breaker()
+
+    def _key(self, provider: str, model: str) -> Tuple[str, str]:
+        return (str(provider or "").strip().lower(), str(model or "").strip().lower())
+
+    def record_turn_start(self, provider: str, model: str) -> None:
+        """Mark turn attempt initiation."""
+        with self._lock:
+            k = self._key(provider, model)
+            if k not in self._states:
+                self._states[k] = ProviderHealthState(provider=provider, model=model)
+            self._states[k].last_checked = time.time()
+
+    def record_success(self, provider: str, model: str, latency_ms: Optional[float] = None) -> None:
+        """Record successful inference turn."""
+        p_low = str(provider or "").strip().lower()
+        is_local = p_low in ("ollama", "local", "vllm")
+        now = time.time()
+
+        with self._lock:
+            k = self._key(provider, model)
+            state = self._states.get(k)
+            if not state:
+                state = ProviderHealthState(provider=provider, model=model)
+                self._states[k] = state
+
+            state.last_success = now
+            state.last_checked = now
+            state.consecutive_failures = 0
+            state.success_count += 1
+            state.circuit_open = False
+            state.cooldown_until = 0.0
+
+            if latency_ms is not None:
+                if state.latency_ms is None:
+                    state.latency_ms = latency_ms
+                else:
+                    # Exponential smoothing
+                    state.latency_ms = round(0.7 * latency_ms + 0.3 * state.latency_ms, 1)
+
+            if is_local:
+                state.status = "local"
+            elif state.latency_ms and state.latency_ms > 3500.0:
+                state.status = "slow"
+            else:
+                state.status = "healthy"
+
+        self._breaker.record_success(provider, model, latency_ms)
+
+    def record_failure(
+        self,
+        provider: str,
+        model: str,
+        error_message: str = "",
+        is_rate_limit: bool = False,
+        cooldown_seconds: Optional[float] = None,
+    ) -> None:
+        """Record a failure without leaking any credential details."""
+        now = time.time()
+        safe_msg = sanitize_error(error_message)
+
+        with self._lock:
+            k = self._key(provider, model)
+            state = self._states.get(k)
+            if not state:
+                state = ProviderHealthState(provider=provider, model=model)
+                self._states[k] = state
+
+            state.last_checked = now
+            state.consecutive_failures += 1
+            state.failure_count += 1
+            state.last_error = safe_msg
+
+            if is_rate_limit:
+                state.status = "rate_limited"
+                cd = cooldown_seconds or 60.0
+                state.cooldown_until = now + cd
+                state.circuit_open = True
+            else:
+                if state.consecutive_failures >= self._breaker.failure_threshold:
+                    state.status = "offline"
+                    cd = cooldown_seconds or self._breaker.default_cooldown
+                    state.cooldown_until = now + cd
+                    state.circuit_open = True
+                else:
+                    state.status = "slow"
+
+        self._breaker.record_failure(
+            provider,
+            model,
+            error=safe_msg,
+            is_rate_limit=is_rate_limit,
+            cooldown_seconds=cooldown_seconds,
+        )
+
+    def get_health(self, provider: str, model: str) -> ProviderHealthState:
+        """Retrieve the health record for a provider/model."""
+        with self._lock:
+            k = self._key(provider, model)
+            state = self._states.get(k)
+            if not state:
+                p_low = str(provider or "").strip().lower()
+                is_local = p_low in ("ollama", "local", "vllm")
+                state = ProviderHealthState(
+                    provider=provider,
+                    model=model,
+                    status="local" if is_local else "disconnected",
+                )
+                self._states[k] = state
+            # Synchronize breaker status
+            breaker_state = self._breaker.get_state(provider, model)
+            state.circuit_open = breaker_state in (CircuitBreakerState.OPEN,)
+            return state
+
+    def is_healthy(self, provider: str, model: str) -> bool:
+        """Check if an endpoint is usable."""
+        if not self._breaker.can_attempt(provider, model):
+            return False
+        st = self.get_health(provider, model)
+        return st.status not in ("offline", "rate_limited") or not st.circuit_open
+
+    def all_states(self) -> List[ProviderHealthState]:
+        """Return a snapshot list of all tracked health records."""
+        with self._lock:
+            return list(self._states.values())
+
+    def get_status_badge(self, provider: str, model: str) -> Tuple[str, str]:
+        """Return (glyph, label) representation (e.g. ('🟢', 'Healthy'))."""
+        st = self.get_health(provider, model)
+        status = st.status
+        if status == "local":
+            return ("🟢", "Local")
+        elif status == "healthy":
+            return ("🟢", "Healthy")
+        elif status == "slow":
+            return ("🟡", "Slow")
+        elif status == "rate_limited":
+            return ("🟠", "Rate Limited")
+        elif status == "offline":
+            return ("🔴", "In Cooldown")
+        return ("⚫", "Disconnected")
+
+
+_GLOBAL_MONITOR: Optional[HealthMonitor] = None
+
+
+def get_health_monitor() -> HealthMonitor:
+    global _GLOBAL_MONITOR
+    if _GLOBAL_MONITOR is None:
+        _GLOBAL_MONITOR = HealthMonitor()
+    return _GLOBAL_MONITOR

calc_terminal/resilience/ollama_adapter.py

@@ -0,0 +1,125 @@
+"""
+CAT — Backup Provider & Resilience System: Native Ollama Local Adapter
+Author: Kazi Zillani
+
+First-class native integration for local Ollama fallback.
+Requires:
+- Zero API keys
+- Zero cloud endpoints
+- Automatic discovery of local models via /api/tags
+"""
+
+import json
+import logging
+import os
+import time
+from typing import Any, Dict, List, Optional
+
+_LOG = logging.getLogger(__name__)
+
+DEFAULT_OLLAMA_URL = "http://localhost:11434"
+
+# Quality instruct model heuristics
+INSTRUCT_PREFERENCES = [
+    "qwen2.5-coder", "qwen3", "qwen2.5", "llama3.3", "llama3.2",
+    "llama3.1", "deepseek-r1", "deepseek-coder", "codellama",
+    "mistral", "gemma2", "phi3", "phi4", "starcoder",
+]
+
+
+class OllamaAdapter:
+    """Zero-config discovery and management for local Ollama backup."""
+
+    def __init__(self, base_url: str = DEFAULT_OLLAMA_URL):
+        self.base_url = (base_url or DEFAULT_OLLAMA_URL).rstrip("/")
+        self._cached_models: Optional[List[dict]] = None
+        self._last_checked = 0.0
+
+    def is_available(self, timeout: float = 1.5) -> bool:
+        """Check if local Ollama daemon is reachable."""
+        try:
+            import requests
+            resp = requests.get(f"{self.base_url}/api/tags", timeout=timeout)
+            return resp.status_code == 200
+        except Exception:
+            return False
+
+    def list_installed_models(self, force_refresh: bool = False) -> List[dict]:
+        """Fetch models currently pulled on the local machine."""
+        now = time.time()
+        if not force_refresh and self._cached_models is not None and (now - self._last_checked) < 30.0:
+            return self._cached_models
+
+        try:
+            import requests
+            resp = requests.get(f"{self.base_url}/api/tags", timeout=2.5)
+            if resp.status_code == 200:
+                data = resp.json()
+                models = data.get("models", [])
+                self._cached_models = models
+                self._last_checked = now
+                return models
+        except Exception as e:
+            _LOG.debug("Ollama discovery failed: %s", e)
+
+        return []
+
+    def get_model_names(self) -> List[str]:
+        return [str(m.get("name", "")).strip() for m in self.list_installed_models() if m.get("name")]
+
+    def select_best_model(self, coding: bool = False, reasoning: bool = False) -> Optional[str]:
+        """Pick the best available instruct model installed locally."""
+        names = self.get_model_names()
+        if not names:
+            return None
+
+        # Filter out raw non-instruct base models
+        non_base = [
+            n for n in names
+            if not any(b in n.lower() for b in ("-base", "_base", "base-q", ":base"))
+        ]
+        candidates = non_base if non_base else names
+
+        if coding:
+            for pref in ("coder", "code", "qwen", "deepseek"):
+                for m in candidates:
+                    if pref in m.lower():
+                        return m
+
+        if reasoning:
+            for m in candidates:
+                if "r1" in m.lower() or "reasoning" in m.lower():
+                    return m
+
+        for pref in INSTRUCT_PREFERENCES:
+            for m in candidates:
+                if pref in m.lower():
+                    return m
+
+        return candidates[0]
+
+    def build_backup_entry(self, model: Optional[str] = None) -> dict:
+        """Create a standard backup configuration entry for Ollama."""
+        selected_model = model or self.select_best_model() or "llama3.3"
+        return {
+            "provider": "ollama",
+            "model": selected_model,
+            "api_key": "",
+            "base_url": self.base_url,
+            "api_style": "ollama",
+            "name": "Ollama (Local Backup)",
+            "enabled": True,
+            "status": "connected" if self.is_available() else "disconnected",
+            "last_used": 0,
+            "priority": 999,  # Default to final local safety net
+        }
+
+
+_GLOBAL_OLLAMA: Optional[OllamaAdapter] = None
+
+
+def get_ollama_adapter() -> OllamaAdapter:
+    global _GLOBAL_OLLAMA
+    if _GLOBAL_OLLAMA is None:
+        _GLOBAL_OLLAMA = OllamaAdapter()
+    return _GLOBAL_OLLAMA

calc_terminal/resilience/orchestrator.py

@@ -0,0 +1,312 @@
+"""
+CAT — Backup Provider & Resilience System: Unified AI Orchestrator
+Author: Kazi Zillani
+
+The central orchestrator coordinating AI provider access across CAT's entire
+architecture: Chat, Coding Agent, IDE Agent, Tools, Browser Agent, and Live Preview.
+"""
+
+import logging
+import time
+from typing import Any, Callable, Dict, Generator, List, Optional, Tuple
+
+from .agent_state import AgentTaskState
+from .circuit_breaker import get_circuit_breaker
+from .failover_engine import get_failover_engine
+from .health_monitor import get_health_monitor, sanitize_error
+from .ollama_adapter import get_ollama_adapter
+from .types import CandidateTarget, CostPolicy, FailureType, SecurityLevel, TaskRequirements
+
+_LOG = logging.getLogger(__name__)
+
+
+class AIOrchestrator:
+    """Master AI Orchestration & Resilience Core for CAT."""
+
+    def __init__(self):
+        self.engine = get_failover_engine()
+        self.health_monitor = get_health_monitor()
+        self.circuit_breaker = get_circuit_breaker()
+        self.ollama = get_ollama_adapter()
+        self.security_level = SecurityLevel.BALANCED
+        self.cost_policy = CostPolicy.ANY
+        self._failover_callbacks: List[Callable[[str], None]] = []
+
+    def add_failover_listener(self, callback: Callable[[str], None]) -> None:
+        """Register a callback to receive user-facing failover announcements."""
+        if callback not in self._failover_callbacks:
+            self._failover_callbacks.append(callback)
+
+    def remove_failover_listener(self, callback: Callable[[str], None]) -> None:
+        if callback in self._failover_callbacks:
+            self._failover_callbacks.remove(callback)
+
+    def notify_failover(self, message: str) -> None:
+        for cb in self._failover_callbacks:
+            try:
+                cb(message)
+            except Exception:
+                pass
+
+    def get_privacy_policy(self) -> str:
+        try:
+            from ..model_router import get_privacy_policy
+            return get_privacy_policy()
+        except Exception:
+            return "local_first"
+
+    def set_privacy_policy(self, policy: str) -> None:
+        try:
+            from ..model_router import set_privacy_policy
+            set_privacy_policy(policy)
+        except Exception:
+            pass
+
+    def enable_emergency_offline_mode(self) -> Tuple[bool, str]:
+        """Switch CAT to local-only emergency mode: zero cloud, Ollama primary."""
+        self.set_privacy_policy("never_cloud")
+        self.cost_policy = CostPolicy.LOCAL_ONLY
+        is_ollama_ready = self.ollama.is_available()
+        models = self.ollama.get_model_names() if is_ollama_ready else []
+
+        if is_ollama_ready and models:
+            msg = f"Offline Local Mode active. Local Ollama available ({len(models)} models: {', '.join(models[:3])})."
+            return True, msg
+        elif is_ollama_ready:
+            msg = "Offline Local Mode active. Local Ollama daemon running, but no models found (run `cat models pull <name>`)."
+            return True, msg
+        else:
+            msg = "Offline Local Mode active. Cloud disabled. Note: Local Ollama is not currently detected at localhost:11434."
+            return False, msg
+
+    def build_targets(
+        self,
+        primary_config: Optional[dict] = None,
+        requirements: Optional[TaskRequirements] = None,
+    ) -> List[CandidateTarget]:
+        """Resolve ordered list of capable, allowed candidate targets."""
+        try:
+            from .. import aicore
+            prim = dict(primary_config or aicore.load_config() or {})
+        except Exception:
+            prim = dict(primary_config or {})
+
+        return self.engine.build_failover_targets(
+            primary_config=prim,
+            requirements=requirements or TaskRequirements(),
+            privacy_policy=self.get_privacy_policy(),
+            cost_policy=self.cost_policy,
+            security_level=self.security_level,
+        )
+
+    def execute_query(
+        self,
+        prompt: str,
+        system_prompt: str = "",
+        history: Optional[List] = None,
+        attachments: Optional[List] = None,
+        requirements: Optional[TaskRequirements] = None,
+        config: Optional[dict] = None,
+        on_failover: Optional[Callable[[str], None]] = None,
+        size_class: str = "normal",
+    ) -> str:
+        """Execute chat/generation turn with resilient automatic failover."""
+        targets = self.build_targets(primary_config=config, requirements=requirements)
+        if not targets:
+            return "No eligible AI providers available for this task under current privacy and capability settings."
+
+        from ..aicore import (
+            _attempts_per_provider,
+            _backoff_sleep,
+            _mark_backup_success,
+            _query_ai_once,
+            _short_id,
+            log_request_event,
+        )
+
+        last_result = ""
+        max_attempts = _attempts_per_provider()
+        fallback_used = False
+
+        for t_idx, target in enumerate(targets):
+            cfg = target.config
+            provider = target.provider
+            model = target.model
+
+            self.health_monitor.record_turn_start(provider, model)
+            t_start = time.time()
+
+            for attempt in range(max_attempts):
+                if attempt > 0:
+                    log_request_event(
+                        _short_id(), "retry_same_provider",
+                        provider=provider, model=model,
+                        size_class=size_class, retry_count=attempt
+                    )
+
+                last_result = _query_ai_once(
+                    prompt,
+                    system_prompt=system_prompt,
+                    history=history,
+                    config=cfg,
+                    attachments=attachments,
+                    size_class=size_class,
+                )
+
+                if not self.engine.should_failover(last_result):
+                    elapsed_ms = (time.time() - t_start) * 1000.0
+                    self.health_monitor.record_success(provider, model, latency_ms=elapsed_ms)
+                    if target.is_backup and target.entry:
+                        _mark_backup_success(target.entry)
+                        self.notify_failover(f"✓ Connected to backup ({provider}/{model}).")
+                        if on_failover:
+                            on_failover(f"✓ Connected to backup ({provider}/{model}).")
+
+                    log_request_event(
+                        _short_id(), "finish",
+                        provider=provider, model=model,
+                        size_class=size_class, chars=len(last_result),
+                        fallback=fallback_used
+                    )
+                    return last_result
+
+                # Classify failure
+                ftype = self.engine.classify_failure(last_result)
+                is_rate_limit = ftype == FailureType.RATE_LIMIT
+
+                if attempt + 1 < max_attempts and self.engine.is_retryable(ftype):
+                    log_request_event(
+                        _short_id(), "attempt_failed_retryable",
+                        provider=provider, detail=last_result[:120],
+                        size_class=size_class
+                    )
+                    _backoff_sleep(attempt)
+                    continue
+
+                # Non-retryable or retries exhausted
+                self.health_monitor.record_failure(
+                    provider, model, error_message=last_result, is_rate_limit=is_rate_limit
+                )
+                break
+
+            # Failover to next target
+            if t_idx + 1 < len(targets):
+                fallback_used = True
+                nxt = targets[t_idx + 1]
+                cur_desc = f"{provider} ({model})"
+                nxt_desc = f"{nxt.provider} ({nxt.model})"
+                msg = f"⚠ Provider issue with {cur_desc}. Switching to {'Model Fallback' if nxt.tier == 1 else 'Backup Provider'} ({nxt_desc})..."
+                self.notify_failover(msg)
+                if on_failover:
+                    on_failover(msg)
+
+        return last_result or "All configured AI providers failed to respond."
+
+    def execute_stream(
+        self,
+        prompt: str,
+        system_prompt: str = "",
+        history: Optional[List] = None,
+        attachments: Optional[List] = None,
+        requirements: Optional[TaskRequirements] = None,
+        config: Optional[dict] = None,
+        on_failover: Optional[Callable[[str], None]] = None,
+        size_class: str = "normal",
+    ) -> Generator[str, None, None]:
+        """Stream response fragments with automatic failover if first chunk fails."""
+        targets = self.build_targets(primary_config=config, requirements=requirements)
+        if not targets:
+            yield "No eligible AI providers available for this task."
+            return
+
+        from ..aicore import (
+            _attempts_per_provider,
+            _backoff_sleep,
+            _mark_backup_success,
+            _peek_first,
+            _stream_ai_once,
+        )
+
+        last_err = ""
+        max_attempts = _attempts_per_provider()
+
+        for t_idx, target in enumerate(targets):
+            cfg = target.config
+            provider = target.provider
+            model = target.model
+
+            self.health_monitor.record_turn_start(provider, model)
+            t_start = time.time()
+
+            for attempt in range(max_attempts):
+                gen = _stream_ai_once(
+                    prompt,
+                    system_prompt=system_prompt,
+                    history=history,
+                    config=cfg,
+                    attachments=attachments,
+                    size_class=size_class,
+                )
+                first, rest = _peek_first(gen)
+
+                if first is None:
+                    last_err = "No output produced by AI provider."
+                    continue
+
+                if not self.engine.should_failover(first):
+                    elapsed_ms = (time.time() - t_start) * 1000.0
+                    self.health_monitor.record_success(provider, model, latency_ms=elapsed_ms)
+                    if target.is_backup and target.entry:
+                        _mark_backup_success(target.entry)
+                    yield first
+                    yield from rest
+                    return
+
+                # First chunk indicates failure
+                last_err = first
+                ftype = self.engine.classify_failure(first)
+                is_rate_limit = ftype == FailureType.RATE_LIMIT
+
+                if attempt + 1 < max_attempts and self.engine.is_retryable(ftype):
+                    _backoff_sleep(attempt)
+                    continue
+
+                self.health_monitor.record_failure(
+                    provider, model, error_message=first, is_rate_limit=is_rate_limit
+                )
+                break
+
+            if t_idx + 1 < len(targets):
+                nxt = targets[t_idx + 1]
+                msg = f"⚠ Switching provider from {provider}/{model} to {nxt.provider}/{nxt.model}..."
+                self.notify_failover(msg)
+                if on_failover:
+                    on_failover(msg)
+
+        yield last_err or "All configured AI providers failed."
+
+    def test_provider(self, config: dict) -> Tuple[bool, str, List[str]]:
+        """Verify connectivity, model availability, and response latency."""
+        from ..providers import provider_manager as pm
+        start = time.time()
+        ok, msg, models = pm.connect_provider(config)
+        elapsed_ms = (time.time() - start) * 1000.0
+
+        prov = config.get("provider", "?")
+        model = config.get("model", "")
+        if ok:
+            self.health_monitor.record_success(prov, model, latency_ms=elapsed_ms)
+        else:
+            self.health_monitor.record_failure(prov, model, error_message=msg)
+
+        return ok, sanitize_error(msg), models
+
+
+_GLOBAL_ORCHESTRATOR: Optional[AIOrchestrator] = None
+
+
+def get_orchestrator() -> AIOrchestrator:
+    global _GLOBAL_ORCHESTRATOR
+    if _GLOBAL_ORCHESTRATOR is None:
+        _GLOBAL_ORCHESTRATOR = AIOrchestrator()
+    return _GLOBAL_ORCHESTRATOR