velune-cli 0.9.0__py3-none-any.whl

velune/providers/discovery/scanner.py

@@ -0,0 +1,115 @@
+"""Unified model discovery coordinator."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+
+from velune.core.types.model import ModelDescriptor
+from velune.providers.discovery.anthropic import AnthropicDiscovery
+from velune.providers.discovery.fireworks import FireworksDiscovery
+from velune.providers.discovery.gguf import GGUFDiscovery
+from velune.providers.discovery.google import GoogleDiscovery
+from velune.providers.discovery.groq import GroqDiscovery
+from velune.providers.discovery.huggingface import HuggingFaceDiscovery
+from velune.providers.discovery.lmstudio import LMStudioDiscovery
+from velune.providers.discovery.ollama import OllamaDiscovery
+from velune.providers.discovery.openai import OpenAIDiscovery
+from velune.providers.discovery.openrouter import OpenRouterDiscovery
+from velune.providers.discovery.together import TogetherDiscovery
+from velune.providers.discovery.xai import XAIDiscovery
+
+logger = logging.getLogger("velune.providers.discovery.scanner")
+
+# Providers that run locally and need no API key
+_LOCAL_PROVIDERS: frozenset[str] = frozenset({"ollama", "lmstudio", "gguf", "llamacpp"})
+
+
+class ModelDiscoveryScanner:
+    """Coordinates model discovery across all configured providers.
+
+    Cloud discoverers are only invoked when a key is available for that
+    provider.  Local discoverers always run; Ollama and LM Studio are
+    additionally gated on their daemon being reachable.
+    """
+
+    def __init__(self) -> None:
+        self.discoverers = [
+            OllamaDiscovery(),
+            LMStudioDiscovery(),
+            GGUFDiscovery(),
+            HuggingFaceDiscovery(),
+            OpenAIDiscovery(),
+            AnthropicDiscovery(),
+            XAIDiscovery(),
+            GoogleDiscovery(),
+            GroqDiscovery(),
+            OpenRouterDiscovery(),
+            TogetherDiscovery(),
+            FireworksDiscovery(),
+        ]
+
+    def _should_run(self, discoverer) -> bool:
+        """Return True if this discoverer should be queried."""
+        if discoverer.provider_id in _LOCAL_PROVIDERS:
+            return True
+        from velune.providers.keystore import has_key
+
+        return has_key(discoverer.provider_id)
+
+    async def _collect(self, discoverer) -> list[ModelDescriptor]:
+        """Run a single discoverer, returning [] on any failure."""
+        try:
+            return await discoverer.discover()
+        except Exception as e:
+            logger.debug("Discovery failed for %s: %s", discoverer.provider_id, e)
+            return []
+
+    async def scan_all(self) -> list[ModelDescriptor]:
+        """Scan all providers for models in parallel."""
+        # Gate server-dependent local providers on reachability
+        ollama_ok, lmstudio_ok = await asyncio.gather(
+            OllamaDiscovery.is_running(),
+            LMStudioDiscovery.is_running(),
+        )
+
+        tasks = []
+        for d in self.discoverers:
+            if d.provider_id == "ollama" and not ollama_ok:
+                continue
+            if d.provider_id == "lmstudio" and not lmstudio_ok:
+                continue
+            if self._should_run(d):
+                tasks.append(self._collect(d))
+
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+
+        all_models: list[ModelDescriptor] = []
+        for result in results:
+            if isinstance(result, list):
+                all_models.extend(result)
+
+        # Summary log
+        counts: dict[str, int] = {"gguf": 0, "ollama": 0, "lmstudio": 0, "cloud": 0}
+        for m in all_models:
+            if m.provider_id in counts:
+                counts[m.provider_id] += 1
+            elif m.provider_id not in _LOCAL_PROVIDERS:
+                counts["cloud"] += 1
+
+        logger.info(
+            "Local: %d GGUF, %d Ollama, %d LM Studio | Cloud: %d models",
+            counts["gguf"],
+            counts["ollama"],
+            counts["lmstudio"],
+            counts["cloud"],
+        )
+
+        return all_models
+
+    async def scan_provider(self, provider_id: str) -> list[ModelDescriptor]:
+        """Scan a specific provider by ID."""
+        for discoverer in self.discoverers:
+            if discoverer.provider_id == provider_id:
+                return await self._collect(discoverer)
+        return []

velune/providers/discovery/together.py

@@ -0,0 +1,114 @@
+"""Together.AI model discovery."""
+
+from __future__ import annotations
+
+from velune.core.types.model import CapabilityLevel, ModelCapabilityProfile, ModelDescriptor
+from velune.providers.keystore import get_key
+
+
+class TogetherDiscovery:
+    """Returns the hardcoded Together.AI model list when a key is configured."""
+
+    provider_id = "together"
+
+    async def discover(self) -> list[ModelDescriptor]:
+        if not get_key("together"):
+            return []
+
+        return [
+            ModelDescriptor(
+                model_id="meta-llama/Llama-3.3-70B-Instruct-Turbo",
+                display_name="Llama 3.3 70B Instruct Turbo",
+                provider_id="together",
+                context_length=131072,
+                capabilities=ModelCapabilityProfile(
+                    coding=CapabilityLevel.ADVANCED,
+                    reasoning=CapabilityLevel.ADVANCED,
+                    planning=CapabilityLevel.ADVANCED,
+                    summarization=CapabilityLevel.EXPERT,
+                    instruction_following=CapabilityLevel.EXPERT,
+                    tool_use=CapabilityLevel.ADVANCED,
+                    long_context=CapabilityLevel.ADVANCED,
+                ),
+                speed_tier="fast",
+                cost_per_1k_tokens=0.00088,
+                tags=["cloud", "together", "llama", "turbo"],
+                metadata={},
+            ),
+            ModelDescriptor(
+                model_id="meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo",
+                display_name="Llama 3.2 11B Vision Instruct",
+                provider_id="together",
+                context_length=131072,
+                capabilities=ModelCapabilityProfile(
+                    coding=CapabilityLevel.INTERMEDIATE,
+                    reasoning=CapabilityLevel.INTERMEDIATE,
+                    planning=CapabilityLevel.INTERMEDIATE,
+                    summarization=CapabilityLevel.ADVANCED,
+                    instruction_following=CapabilityLevel.ADVANCED,
+                    tool_use=CapabilityLevel.INTERMEDIATE,
+                    long_context=CapabilityLevel.INTERMEDIATE,
+                ),
+                speed_tier="fast",
+                cost_per_1k_tokens=0.00018,
+                tags=["cloud", "together", "llama", "vision", "cheap"],
+                metadata={},
+            ),
+            ModelDescriptor(
+                model_id="Qwen/Qwen2.5-Coder-32B-Instruct",
+                display_name="Qwen 2.5 Coder 32B Instruct",
+                provider_id="together",
+                context_length=131072,
+                capabilities=ModelCapabilityProfile(
+                    coding=CapabilityLevel.EXPERT,
+                    reasoning=CapabilityLevel.ADVANCED,
+                    planning=CapabilityLevel.ADVANCED,
+                    summarization=CapabilityLevel.ADVANCED,
+                    instruction_following=CapabilityLevel.EXPERT,
+                    tool_use=CapabilityLevel.ADVANCED,
+                    long_context=CapabilityLevel.ADVANCED,
+                ),
+                speed_tier="medium",
+                cost_per_1k_tokens=0.0008,
+                tags=["cloud", "together", "qwen", "coding"],
+                metadata={},
+            ),
+            ModelDescriptor(
+                model_id="deepseek-ai/DeepSeek-R1",
+                display_name="DeepSeek R1",
+                provider_id="together",
+                context_length=163840,
+                capabilities=ModelCapabilityProfile(
+                    coding=CapabilityLevel.EXPERT,
+                    reasoning=CapabilityLevel.EXPERT,
+                    planning=CapabilityLevel.EXPERT,
+                    summarization=CapabilityLevel.ADVANCED,
+                    instruction_following=CapabilityLevel.EXPERT,
+                    tool_use=CapabilityLevel.ADVANCED,
+                    long_context=CapabilityLevel.ADVANCED,
+                ),
+                speed_tier="slow",
+                cost_per_1k_tokens=0.003,
+                tags=["cloud", "together", "deepseek", "reasoning"],
+                metadata={},
+            ),
+            ModelDescriptor(
+                model_id="mistralai/Mistral-7B-Instruct-v0.3",
+                display_name="Mistral 7B Instruct v0.3",
+                provider_id="together",
+                context_length=32768,
+                capabilities=ModelCapabilityProfile(
+                    coding=CapabilityLevel.INTERMEDIATE,
+                    reasoning=CapabilityLevel.INTERMEDIATE,
+                    planning=CapabilityLevel.INTERMEDIATE,
+                    summarization=CapabilityLevel.ADVANCED,
+                    instruction_following=CapabilityLevel.ADVANCED,
+                    tool_use=CapabilityLevel.INTERMEDIATE,
+                    long_context=CapabilityLevel.BASIC,
+                ),
+                speed_tier="fast",
+                cost_per_1k_tokens=0.0002,
+                tags=["cloud", "together", "mistral", "cheap"],
+                metadata={},
+            ),
+        ]

velune/providers/discovery/xai.py

@@ -0,0 +1,57 @@
+"""xAI (Grok) model discovery."""
+
+from __future__ import annotations
+
+from velune.core.types.model import CapabilityLevel, ModelCapabilityProfile, ModelDescriptor
+from velune.providers.keystore import get_key
+
+
+class XAIDiscovery:
+    """Returns the hardcoded xAI Grok model list when a key is configured."""
+
+    provider_id = "xai"
+
+    async def discover(self) -> list[ModelDescriptor]:
+        if not get_key("xai"):
+            return []
+
+        return [
+            ModelDescriptor(
+                model_id="grok-2",
+                provider_id="xai",
+                display_name="Grok 2",
+                context_length=131072,
+                capabilities=ModelCapabilityProfile(
+                    coding=CapabilityLevel.ADVANCED,
+                    reasoning=CapabilityLevel.ADVANCED,
+                    planning=CapabilityLevel.ADVANCED,
+                    summarization=CapabilityLevel.ADVANCED,
+                    instruction_following=CapabilityLevel.EXPERT,
+                    tool_use=CapabilityLevel.ADVANCED,
+                    long_context=CapabilityLevel.ADVANCED,
+                ),
+                speed_tier="medium",
+                cost_per_1k_tokens=0.005,
+                tags=["cloud", "xai"],
+                metadata={},
+            ),
+            ModelDescriptor(
+                model_id="grok-2-mini",
+                provider_id="xai",
+                display_name="Grok 2 Mini",
+                context_length=131072,
+                capabilities=ModelCapabilityProfile(
+                    coding=CapabilityLevel.INTERMEDIATE,
+                    reasoning=CapabilityLevel.INTERMEDIATE,
+                    planning=CapabilityLevel.INTERMEDIATE,
+                    summarization=CapabilityLevel.ADVANCED,
+                    instruction_following=CapabilityLevel.ADVANCED,
+                    tool_use=CapabilityLevel.INTERMEDIATE,
+                    long_context=CapabilityLevel.ADVANCED,
+                ),
+                speed_tier="fast",
+                cost_per_1k_tokens=0.0005,
+                tags=["cloud", "xai", "mini"],
+                metadata={},
+            ),
+        ]

velune/providers/health.py

@@ -0,0 +1,67 @@
+"""Internet connectivity detection with TTL-based caching."""
+
+from __future__ import annotations
+
+import logging
+import socket
+import time
+
+logger = logging.getLogger("velune.providers.health")
+
+_DEFAULT_TTL = 30.0  # seconds
+_CHECK_HOST = "1.1.1.1"
+_CHECK_PORT = 80
+_CHECK_TIMEOUT = 2.0  # seconds
+
+
+class InternetConnectivityChecker:
+    """Checks internet connectivity via a TCP probe to 1.1.1.1:80.
+
+    Results are cached for *ttl* seconds to avoid hammering the network on
+    every router decision.
+    """
+
+    def __init__(self, ttl: float = _DEFAULT_TTL) -> None:
+        self._ttl = ttl
+        self._cached: bool | None = None
+        self._cached_at: float = 0.0
+
+    def _probe(self) -> bool:
+        """Attempt a TCP connection. Returns True if reachable within timeout."""
+        try:
+            with socket.create_connection((_CHECK_HOST, _CHECK_PORT), timeout=_CHECK_TIMEOUT):
+                return True
+        except OSError:
+            return False
+
+    @property
+    def is_online(self) -> bool:
+        """True if the internet is reachable; cached for *ttl* seconds."""
+        now = time.monotonic()
+        if self._cached is None or (now - self._cached_at) >= self._ttl:
+            self._cached = self._probe()
+            self._cached_at = now
+            if not self._cached:
+                logger.debug("Internet connectivity check: OFFLINE")
+        return self._cached
+
+    def invalidate(self) -> None:
+        """Force the next call to is_online to re-probe."""
+        self._cached = None
+
+
+# ---------------------------------------------------------------------------
+# Module-level singleton — shared across the process
+# ---------------------------------------------------------------------------
+
+_checker = InternetConnectivityChecker()
+
+
+def is_online() -> bool:
+    """Return True if the internet appears reachable (cached 30 s)."""
+    return _checker.is_online
+
+
+def get_checker() -> InternetConnectivityChecker:
+    """Return the shared InternetConnectivityChecker instance."""
+    return _checker

velune/providers/health_monitor.py

@@ -0,0 +1,169 @@
+"""Provider health monitoring with real-time capability tracking."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import time
+from collections import defaultdict, deque
+
+from velune.core.types.provider import CapabilityManifest, ProviderHealth
+from velune.providers.base import ModelProvider
+from velune.providers.registry import ProviderRegistry
+
+logger = logging.getLogger("velune.providers.health_monitor")
+
+
+class ProviderHealthMonitor:
+    """Continuously polls registered providers and maintains real-time capability manifests."""
+
+    def __init__(self, registry: ProviderRegistry) -> None:
+        self._registry = registry
+        self._manifests: dict[str, CapabilityManifest] = {}
+        self._latency_windows: dict[str, deque[int]] = defaultdict(lambda: deque(maxlen=5))
+        self._health_history: dict[str, deque[ProviderHealth]] = defaultdict(
+            lambda: deque(maxlen=3)
+        )
+        self._polling_task: asyncio.Task | None = None
+        self._poll_interval = 30.0  # seconds
+        self._health_check_timeout = 2.0  # seconds
+        self._running = False
+
+    async def start(self) -> None:
+        """Start the background polling task."""
+        if self._running:
+            logger.warning("ProviderHealthMonitor already running")
+            return
+
+        self._running = True
+        self._polling_task = asyncio.create_task(self._polling_loop())
+        logger.info("ProviderHealthMonitor started")
+
+    async def stop(self) -> None:
+        """Stop the background polling task."""
+        self._running = False
+        if self._polling_task:
+            self._polling_task.cancel()
+            try:
+                await self._polling_task
+            except asyncio.CancelledError:
+                pass
+        logger.info("ProviderHealthMonitor stopped")
+
+    def get_manifest(self, provider_id: str) -> CapabilityManifest | None:
+        """Get the latest manifest for a provider."""
+        return self._manifests.get(provider_id)
+
+    def get_all_manifests(self) -> dict[str, CapabilityManifest]:
+        """Get all provider manifests."""
+        return self._manifests.copy()
+
+    def record_latency(self, provider_id: str, latency_ms: int) -> None:
+        """Record a call latency for rolling average calculation."""
+        self._latency_windows[provider_id].append(latency_ms)
+        if manifest := self._manifests.get(provider_id):
+            avg_latency = int(
+                sum(self._latency_windows[provider_id]) / len(self._latency_windows[provider_id])
+            )
+            manifest.estimated_latency_ms = avg_latency
+
+    async def _polling_loop(self) -> None:
+        """Background task that polls all providers every 30 seconds."""
+        # Standard provider IDs that may be registered
+        standard_providers = [
+            "ollama",
+            "openai",
+            "anthropic",
+            "google",
+            "groq",
+            "xai",
+            "openrouter",
+            "together",
+            "fireworks",
+            "huggingface",
+            "lmstudio",
+            "llamacpp",
+        ]
+
+        while self._running:
+            try:
+                # Get all registered provider IDs
+                providers_to_check = []
+                for provider_id in standard_providers:
+                    if provider := self._registry.get(provider_id):
+                        providers_to_check.append((provider_id, provider))
+
+                # Poll all providers in parallel
+                tasks = [
+                    self._health_check_provider(provider_id, provider)
+                    for provider_id, provider in providers_to_check
+                ]
+                await asyncio.gather(*tasks, return_exceptions=True)
+
+            except Exception as e:
+                logger.error(f"Error in health monitoring loop: {e}")
+
+            # Sleep before next poll
+            try:
+                await asyncio.sleep(self._poll_interval)
+            except asyncio.CancelledError:
+                break
+
+    async def _health_check_provider(self, provider_id: str, provider: ModelProvider) -> None:
+        """Check health of a single provider and update manifest."""
+        try:
+            # Call health_check with timeout
+            health = await asyncio.wait_for(
+                provider.health_check(), timeout=self._health_check_timeout
+            )
+        except TimeoutError:
+            health = ProviderHealth.DEGRADED
+            logger.debug(f"Provider {provider_id} health check timed out")
+        except Exception as e:
+            health = ProviderHealth.UNAVAILABLE
+            logger.debug(f"Provider {provider_id} health check failed: {e}")
+
+        try:
+            # Get list of available models
+            available_models = await asyncio.wait_for(
+                provider.list_models(), timeout=self._health_check_timeout
+            )
+        except (TimeoutError, Exception):
+            available_models = []
+
+        # Get provider capabilities
+        capabilities = provider.get_capabilities()
+
+        # Track health history for consecutive unavailability detection
+        self._health_history[provider_id].append(health)
+
+        # Create or update manifest
+        manifest = CapabilityManifest(
+            provider_id=provider_id,
+            health=health,
+            available_models=available_models,
+            rate_limit_remaining=None,  # Would be populated from response headers
+            rate_limit_reset_at=None,
+            estimated_latency_ms=int(
+                sum(self._latency_windows[provider_id]) / len(self._latency_windows[provider_id])
+            )
+            if self._latency_windows[provider_id]
+            else 0,
+            supports_streaming=capabilities.supports_streaming,
+            supports_tools=capabilities.supports_function_calling,
+            is_online=True,  # Connected check would be done here
+            refreshed_at=time.time(),
+        )
+
+        # Detect status changes
+        old_manifest = self._manifests.get(provider_id)
+        if old_manifest and old_manifest.health != health:
+            logger.info(f"Provider {provider_id} health changed: {old_manifest.health} → {health}")
+
+        # Check for 3 consecutive unavailable polls
+        if len(self._health_history[provider_id]) >= 3:
+            recent = list(self._health_history[provider_id])
+            if all(h == ProviderHealth.UNAVAILABLE for h in recent[-3:]):
+                logger.warning(f"Provider {provider_id} unavailable for 3 consecutive polls")
+
+        self._manifests[provider_id] = manifest

velune/providers/keystore.py

@@ -0,0 +1,142 @@
+"""BYOK key storage backed by the OS keyring with env-var fallback.
+
+Performance notes
+-----------------
+On Windows the OS keyring is the Windows Credential Manager (DPAPI). A single
+``keyring.get_password`` call can block for *seconds* the first time it is hit
+after a cold boot or screen unlock, and Velune historically issued one such
+call per provider (9+) sequentially during startup — tens of seconds of pure
+blocking. This module now:
+
+* imports ``keyring`` lazily (env-only users never pay the import cost);
+* checks the free, instant environment variables before ever touching the
+  keyring;
+* parallelizes the remaining keyring probes across a thread pool.
+
+We deliberately do *not* cache resolved keys process-wide: a stale cache would
+mask a key the user just saved (or deleted) mid-session, and correctness here
+matters more than shaving a second off repeated lookups.
+"""
+
+from __future__ import annotations
+
+import os
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+_SERVICE = "velune/{}"
+_USERNAME = "api_key"
+
+PROVIDER_ENV_VARS: dict[str, str] = {
+    "anthropic": "ANTHROPIC_API_KEY",
+    "openai": "OPENAI_API_KEY",
+    "xai": "XAI_API_KEY",
+    "google": "GOOGLE_API_KEY",
+    "groq": "GROQ_API_KEY",
+    "openrouter": "OPENROUTER_API_KEY",
+    "huggingface": "HF_TOKEN",
+    "together": "TOGETHER_API_KEY",
+    "fireworks": "FIREWORKS_API_KEY",
+}
+
+_ENV_VARS = PROVIDER_ENV_VARS
+
+
+def _keyring_get(provider_id: str) -> str | None:
+    """Raw, uncached OS-keyring lookup. Lazily imports keyring."""
+    try:
+        import keyring
+
+        return keyring.get_password(_SERVICE.format(provider_id), _USERNAME)
+    except Exception:
+        return None
+
+
+def _resolve_uncached(provider_id: str) -> str | None:
+    """Resolve a key without consulting the cache (keyring then env)."""
+    key = _keyring_get(provider_id)
+    if key:
+        return key
+    env_var = _ENV_VARS.get(provider_id)
+    if env_var:
+        return os.getenv(env_var) or None
+    return None
+
+
+def save_key(provider_id: str, api_key: str) -> None:
+    """Persist *api_key* for *provider_id* in the OS keyring."""
+    import keyring
+
+    keyring.set_password(_SERVICE.format(provider_id), _USERNAME, api_key)
+
+
+def get_key(provider_id: str) -> str | None:
+    """Return the API key for *provider_id* (OS keyring first, then env var)."""
+    return _resolve_uncached(provider_id)
+
+
+def delete_key(provider_id: str) -> None:
+    """Remove the stored key for *provider_id* from the OS keyring."""
+    import keyring
+    import keyring.errors
+
+    try:
+        keyring.delete_password(_SERVICE.format(provider_id), _USERNAME)
+    except keyring.errors.PasswordDeleteError:
+        pass
+
+
+def has_key(provider_id: str) -> bool:
+    """Return True if a key is available for *provider_id*."""
+    return get_key(provider_id) is not None
+
+
+def is_ollama_live(timeout: float = 1.0) -> bool:
+    """Return True if a local Ollama server is reachable.
+
+    Kept deliberately short (1s) — the startup path should not block multiple
+    seconds probing an optional local service.
+    """
+    try:
+        import httpx
+
+        r = httpx.get("http://localhost:11434/api/tags", timeout=timeout)
+        return r.status_code == 200
+    except Exception:
+        return False
+
+
+def list_configured_providers(include_ollama: bool = True) -> list[str]:
+    """Return provider IDs that have a usable key, resolved in parallel.
+
+    Environment variables are checked first (free, instant). Only providers
+    not already satisfied by an env var incur a (parallelized, cached) keyring
+    probe. Ollama is local and keyless — it counts as configured when its
+    server is reachable.
+    """
+    configured: set[str] = set()
+    needs_keyring: list[str] = []
+
+    for pid, env_var in _ENV_VARS.items():
+        if os.getenv(env_var):
+            configured.add(pid)
+        else:
+            needs_keyring.append(pid)
+
+    if needs_keyring:
+        with ThreadPoolExecutor(max_workers=min(8, len(needs_keyring))) as pool:
+            futures = {pool.submit(_keyring_get, pid): pid for pid in needs_keyring}
+            for fut in as_completed(futures):
+                pid = futures[fut]
+                try:
+                    key = fut.result()
+                except Exception:
+                    key = None
+                if key:
+                    configured.add(pid)
+
+    # Preserve a stable, declaration-order result.
+    ordered = [pid for pid in _ENV_VARS if pid in configured]
+
+    if include_ollama and is_ollama_live():
+        ordered.insert(0, "ollama")
+    return ordered