velune-cli 1.0.0__py3-none-any.whl

velune/providers/base.py

@@ -0,0 +1,86 @@
+"""Provider abstraction base interfaces and capabilities."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from collections.abc import AsyncIterator
+from typing import Protocol, runtime_checkable
+
+from velune.core.types.inference import InferenceRequest, InferenceResponse, StreamChunk
+from velune.core.types.model import ModelDescriptor
+from velune.core.types.provider import ProviderCapabilities, ProviderHealth
+
+
+@runtime_checkable
+class ModelProvider(Protocol):
+    """Core provider contract. All LLM and Embedding adapters implement this."""
+
+    @property
+    def provider_id(self) -> str:
+        """The distinct ID slug of this provider (e.g., 'ollama', 'openai')."""
+        ...
+
+    async def list_models(self) -> list[ModelDescriptor]:
+        """Query and list all active/available models for this provider."""
+        ...
+
+    async def infer(self, request: InferenceRequest) -> InferenceResponse:
+        """Single-turn, non-streaming model completion."""
+        ...
+
+    async def stream(self, request: InferenceRequest) -> AsyncIterator[StreamChunk]:
+        """Multi-turn, token-streaming model completion."""
+        ...
+
+    async def embed(self, texts: list[str], model_id: str) -> list[list[float]]:
+        """Generate vector embeddings. Raises NotImplementedError if unsupported."""
+        ...
+
+    async def health_check(self) -> ProviderHealth:
+        """Query host or ping API to verify provider state."""
+        ...
+
+    def get_capabilities(self) -> ProviderCapabilities:
+        """Query static/dynamic capabilities of the provider."""
+        ...
+
+    async def initialize(self) -> None:
+        """Perform provider connection setup and warmups."""
+        ...
+
+    async def shutdown(self) -> None:
+        """Gracefully release provider connection resource pools."""
+        ...
+
+
+class InferenceEngine(ABC):
+    """Abstract inference engine for unified task executions."""
+
+    @abstractmethod
+    async def infer(self, request: InferenceRequest) -> InferenceResponse:
+        """Perform non-streaming inference."""
+        pass
+
+    @abstractmethod
+    async def infer_stream(self, request: InferenceRequest) -> AsyncIterator[StreamChunk]:
+        """Perform streaming inference."""
+        pass
+
+
+class EmbeddingProvider(ABC):
+    """Abstract embedding provider interface."""
+
+    @abstractmethod
+    async def embed(self, text: str) -> list[float]:
+        """Generate embedding for a single text."""
+        pass
+
+    @abstractmethod
+    async def embed_batch(self, texts: list[str]) -> list[list[float]]:
+        """Generate embeddings for a batch of texts."""
+        pass
+
+    @abstractmethod
+    def get_dimension(self) -> int:
+        """Get the embedding dimension vector width."""
+        pass

velune/providers/benchmarker.py

@@ -0,0 +1,135 @@
+"""Provider and model benchmarking engine."""
+
+from __future__ import annotations
+
+import time
+
+from velune.core.types.inference import InferenceRequest
+from velune.providers.base import ModelProvider
+
+
+class ModelBenchmarkMetrics:
+    """Live performance metrics recorded during model benchmarks."""
+
+    def __init__(self) -> None:
+        self.ttft_ms: float = 0.0          # Time to first token
+        self.tps: float = 0.0               # Tokens per second
+        self.total_latency_ms: float = 0.0 # Total latency in ms
+        self.tokens_generated: int = 0
+        self.tool_accuracy: float = 0.0     # Percentage correctness for tools
+        self.json_validity: float = 0.0     # Percentage valid JSON formats
+
+
+class ProviderBenchmarker:
+    """Evaluates provider latency, generation throughput, and structured accuracy."""
+
+    def __init__(self, provider: ModelProvider, model_id: str) -> None:
+        self.provider = provider
+        self.model_id = model_id
+
+    async def run_latency_probe(self, prompt: str = "Hello, respond with exactly 'pong'.") -> ModelBenchmarkMetrics:
+        """Probe time-to-first-token and total generation latency."""
+        metrics = ModelBenchmarkMetrics()
+        request = InferenceRequest(
+            model_id=self.model_id,
+            messages=[{"role": "user", "content": prompt}],
+            temperature=0.0,
+            max_tokens=10,
+        )
+
+        start_time = time.perf_counter()
+        first_token_time: float | None = None
+        chars_received = 0
+
+        try:
+            # We use streaming to calculate TTFT and TPS
+            async for chunk in self.provider.stream(request):
+                if first_token_time is None and chunk.content:
+                    first_token_time = time.perf_counter()
+                    metrics.ttft_ms = (first_token_time - start_time) * 1000.0
+
+                chars_received += len(chunk.content)
+                # Estimating tokens: 1 token ≈ 4 characters
+                metrics.tokens_generated = int(chars_received / 4.0)
+
+            end_time = time.perf_counter()
+            metrics.total_latency_ms = (end_time - start_time) * 1000.0
+
+            if first_token_time is not None:
+                gen_duration = end_time - first_token_time
+                if gen_duration > 0 and metrics.tokens_generated > 0:
+                    metrics.tps = metrics.tokens_generated / gen_duration
+            else:
+                metrics.ttft_ms = metrics.total_latency_ms
+                metrics.tps = 0.0
+
+        except Exception:
+            # Handle non-streaming fallback
+            try:
+                start_time = time.perf_counter()
+                response = await self.provider.infer(request)
+                end_time = time.perf_counter()
+
+                metrics.total_latency_ms = (end_time - start_time) * 1000.0
+                metrics.ttft_ms = metrics.total_latency_ms  # No stream TTFT estimation
+                metrics.tokens_generated = response.tokens_used or int(len(response.content) / 4.0)
+
+                duration = (end_time - start_time)
+                if duration > 0:
+                    metrics.tps = metrics.tokens_generated / duration
+            except Exception:
+                # Failed probe
+                metrics.ttft_ms = -1.0
+                metrics.tps = 0.0
+
+        return metrics
+
+    async def run_structured_probe(self) -> dict[str, float]:
+        """Probe model capabilities in generating valid JSON format."""
+        prompt = (
+            "Return a JSON object representing a file list with keys 'files' (list of strings) and 'count' (integer). "
+            "Return ONLY raw valid JSON code. No markdown wrapper, no extra text."
+        )
+        request = InferenceRequest(
+            model_id=self.model_id,
+            messages=[{"role": "user", "content": prompt}],
+            temperature=0.0,
+            max_tokens=100,
+        )
+
+        json_valid = 0.0
+        structure_valid = 0.0
+
+        try:
+            response = await self.provider.infer(request)
+            content = response.content.strip()
+
+            # Clean possible markdown wrappers
+            if content.startswith("```"):
+                lines = content.splitlines()
+                if len(lines) > 2:
+                    content = "\n".join(lines[1:-1])
+                    if content.startswith("json"):
+                        content = content[4:].strip()
+
+            import json
+            parsed = json.loads(content)
+            json_valid = 1.0
+
+            if isinstance(parsed, dict) and "files" in parsed and "count" in parsed:
+                if isinstance(parsed["files"], list) and isinstance(parsed["count"], int):
+                    structure_valid = 1.0
+        except Exception:
+            pass
+
+        return {"json_validity": json_valid, "structure_validity": structure_valid}
+
+    async def evaluate(self) -> ModelBenchmarkMetrics:
+        """Run all benchmark probes and aggregate metrics."""
+        metrics = await self.run_latency_probe()
+        struct_res = await self.run_structured_probe()
+
+        metrics.json_validity = struct_res["json_validity"]
+        metrics.tool_accuracy = struct_res["structure_validity"]
+
+        return metrics

velune/providers/discovery/__init__.py

@@ -0,0 +1,33 @@
+"""Model discovery components."""
+
+from velune.providers.discovery.anthropic import AnthropicDiscovery
+from velune.providers.discovery.benchmarks import CapabilityBenchmark
+from velune.providers.discovery.classifier import CapabilityClassifier
+from velune.providers.discovery.gguf import GGUFDiscovery
+from velune.providers.discovery.google import GoogleDiscovery
+from velune.providers.discovery.gpu import GPUDetector
+from velune.providers.discovery.groq import GroqDiscovery
+from velune.providers.discovery.huggingface import HuggingFaceDiscovery
+from velune.providers.discovery.lmstudio import LMStudioDiscovery
+from velune.providers.discovery.ollama import OllamaDiscovery
+from velune.providers.discovery.openai import OpenAIDiscovery
+from velune.providers.discovery.openrouter import OpenRouterDiscovery
+from velune.providers.discovery.scanner import ModelDiscoveryScanner
+from velune.providers.discovery.xai import XAIDiscovery
+
+__all__ = [
+    "ModelDiscoveryScanner",
+    "OllamaDiscovery",
+    "LMStudioDiscovery",
+    "GGUFDiscovery",
+    "HuggingFaceDiscovery",
+    "OpenAIDiscovery",
+    "AnthropicDiscovery",
+    "XAIDiscovery",
+    "GoogleDiscovery",
+    "GroqDiscovery",
+    "OpenRouterDiscovery",
+    "GPUDetector",
+    "CapabilityClassifier",
+    "CapabilityBenchmark",
+]

velune/providers/discovery/anthropic.py

@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+from velune.core.types.model import CapabilityLevel, ModelCapabilityProfile, ModelDescriptor
+from velune.providers.keystore import get_key
+
+
+class AnthropicDiscovery:
+    """Discovers models from Anthropic."""
+
+    def __init__(self):
+        self.provider_id = "anthropic"
+        self.api_key = get_key("anthropic")
+        self.base_url = "https://api.anthropic.com"
+
+    async def discover(self) -> list[ModelDescriptor]:
+        """Discover models from Anthropic."""
+        if not self.api_key:
+            return []
+
+        # Anthropic has a fixed set of models
+        models = [
+            self._create_model_descriptor("claude-opus-4-5", 200000, 0.015),
+            self._create_model_descriptor("claude-sonnet-4-5", 200000, 0.003),
+            self._create_model_descriptor("claude-haiku-4-5", 200000, 0.00025),
+        ]
+
+        return models
+
+    def _create_model_descriptor(self, model_id: str, context_length: int, cost_per_1k: float) -> ModelDescriptor:
+        """Create a model descriptor."""
+        capabilities = self._classify_capabilities(model_id)
+
+        return ModelDescriptor(
+            model_id=model_id,
+            provider_id=self.provider_id,
+            display_name=model_id,
+            context_length=context_length,
+            capabilities=capabilities,
+            quantization=None,
+            vram_required_gb=None,
+            parameter_count_b=None,
+            speed_tier="medium",
+            cost_per_1k_tokens=cost_per_1k,
+            tags=["cloud", "anthropic"],
+            metadata={},
+        )
+
+    def _classify_capabilities(self, model_id: str) -> ModelCapabilityProfile:
+        """Classify capabilities for Anthropic models."""
+        profile = ModelCapabilityProfile()
+
+        if "opus" in model_id:
+            profile.coding = CapabilityLevel.EXPERT
+            profile.reasoning = CapabilityLevel.EXPERT
+            profile.planning = CapabilityLevel.EXPERT
+            profile.summarization = CapabilityLevel.EXPERT
+            profile.instruction_following = CapabilityLevel.EXPERT
+            profile.tool_use = CapabilityLevel.EXPERT
+            profile.long_context = CapabilityLevel.EXPERT
+        elif "sonnet" in model_id:
+            profile.coding = CapabilityLevel.ADVANCED
+            profile.reasoning = CapabilityLevel.ADVANCED
+            profile.planning = CapabilityLevel.ADVANCED
+            profile.summarization = CapabilityLevel.ADVANCED
+            profile.instruction_following = CapabilityLevel.ADVANCED
+            profile.tool_use = CapabilityLevel.EXPERT
+            profile.long_context = CapabilityLevel.ADVANCED
+        elif "haiku" in model_id:
+            profile.coding = CapabilityLevel.INTERMEDIATE
+            profile.reasoning = CapabilityLevel.INTERMEDIATE
+            profile.planning = CapabilityLevel.INTERMEDIATE
+            profile.summarization = CapabilityLevel.INTERMEDIATE
+            profile.instruction_following = CapabilityLevel.ADVANCED
+            profile.tool_use = CapabilityLevel.ADVANCED
+            profile.long_context = CapabilityLevel.INTERMEDIATE
+
+        return profile

velune/providers/discovery/benchmarks.py

@@ -0,0 +1,44 @@
+"""Lightweight capability benchmarking."""
+
+from velune.core.types.model import CapabilityLevel, ModelCapabilityProfile
+
+
+class CapabilityBenchmark:
+    """Lightweight benchmark probes for capability detection."""
+
+    async def benchmark_coding(self, model_id: str) -> CapabilityLevel:
+        """Benchmark coding capability."""
+        # In production, this would run actual coding tasks
+        # For now, return based on heuristics
+        model_lower = model_id.lower()
+
+        if any(name in model_lower for name in ["coder", "deepseek-coder"]):
+            return CapabilityLevel.ADVANCED
+        elif any(name in model_lower for name in ["llama", "mistral"]):
+            return CapabilityLevel.INTERMEDIATE
+
+        return CapabilityLevel.BASIC
+
+    async def benchmark_reasoning(self, model_id: str) -> CapabilityLevel:
+        """Benchmark reasoning capability."""
+        model_lower = model_id.lower()
+
+        if any(name in model_lower for name in ["r1", "qwq", "deepseek-r1"]):
+            return CapabilityLevel.EXPERT
+        elif any(name in model_lower for name in ["qwen"]):
+            return CapabilityLevel.INTERMEDIATE
+
+        return CapabilityLevel.BASIC
+
+    async def run_full_benchmark(self, model_id: str) -> ModelCapabilityProfile:
+        """Run full capability benchmark."""
+        profile = ModelCapabilityProfile()
+
+        profile.coding = await self.benchmark_coding(model_id)
+        profile.reasoning = await self.benchmark_reasoning(model_id)
+
+        # Infer other capabilities
+        if profile.reasoning >= CapabilityLevel.INTERMEDIATE:
+            profile.planning = CapabilityLevel.INTERMEDIATE
+
+        return profile

velune/providers/discovery/classifier.py

@@ -0,0 +1,69 @@
+"""Capability classification engine."""
+
+from velune.core.types.model import CapabilityLevel, ModelCapabilityProfile, ModelDescriptor
+
+
+class CapabilityClassifier:
+    """Classifies model capabilities using multiple strategies."""
+
+    def __init__(self):
+        self.name_patterns = {
+            "coding": ["coder", "code", "deepseek-coder", "qwen-coder", "starcoder"],
+            "reasoning": ["r1", "reason", "deepseek-r1", "qwq", "qwen"],
+            "planning": ["qwq", "qwen", "r1"],
+            "summarization": ["llama", "mistral"],
+            "instruction_following": ["instruct", "chat"],
+            "long_context": ["long", "32k", "128k", "200k"],
+        }
+
+    def classify(self, model: ModelDescriptor) -> ModelCapabilityProfile:
+        """Classify capabilities for a model."""
+        profile = ModelCapabilityProfile()
+        model_lower = model.model_id.lower()
+
+        # Name-based classification
+        for capability, patterns in self.name_patterns.items():
+            if any(pattern in model_lower for pattern in patterns):
+                setattr(profile, capability, self._infer_level(model_lower, capability))
+
+        # Architecture-based inference
+        if "moe" in model_lower or "mixture-of-experts" in model_lower:
+            profile.reasoning = max(profile.reasoning, CapabilityLevel.INTERMEDIATE)
+
+        # GGUF metadata parsing
+        if model.metadata.get("gguf_metadata"):
+            self._parse_gguf_metadata(model.metadata["gguf_metadata"], profile)
+
+        return profile
+
+    def _infer_level(self, model_id: str, capability: str) -> CapabilityLevel:
+        """Infer capability level from model name."""
+        model_lower = model_id.lower()
+
+        # Strong indicators
+        if any(indicator in model_lower for indicator in ["v2", "latest", "pro"]):
+            return CapabilityLevel.ADVANCED
+
+        # Capable indicators
+        if any(indicator in model_lower for indicator in ["coder", "instruct"]):
+            return CapabilityLevel.INTERMEDIATE
+
+        # Basic indicators
+        if capability == "reasoning" and "r1" in model_lower:
+            return CapabilityLevel.EXPERT
+
+        return CapabilityLevel.BASIC
+
+    def _parse_gguf_metadata(self, metadata: dict, profile: ModelCapabilityProfile) -> None:
+        """Parse GGUF metadata for capability hints."""
+        # Check for long context
+        context_length = metadata.get("context_length", 0)
+        if context_length >= 32000:
+            profile.long_context = CapabilityLevel.INTERMEDIATE
+        elif context_length >= 8000:
+            profile.long_context = CapabilityLevel.BASIC
+
+        # Check parameter count for capability inference
+        param_count = metadata.get("parameter_count", 0)
+        if param_count >= 70e9:  # 70B+
+            profile.reasoning = max(profile.reasoning, CapabilityLevel.INTERMEDIATE)

velune/providers/discovery/fireworks.py

@@ -0,0 +1,95 @@
+"""Fireworks.AI model discovery."""
+
+from __future__ import annotations
+
+from velune.core.types.model import CapabilityLevel, ModelCapabilityProfile, ModelDescriptor
+from velune.providers.keystore import get_key
+
+
+class FireworksDiscovery:
+    """Returns the hardcoded Fireworks.AI model list when a key is configured."""
+
+    provider_id = "fireworks"
+
+    async def discover(self) -> list[ModelDescriptor]:
+        if not get_key("fireworks"):
+            return []
+
+        return [
+            ModelDescriptor(
+                model_id="accounts/fireworks/models/llama-v3p3-70b-instruct",
+                display_name="Llama 3.3 70B Instruct",
+                provider_id="fireworks",
+                context_length=131072,
+                capabilities=ModelCapabilityProfile(
+                    coding=CapabilityLevel.ADVANCED,
+                    reasoning=CapabilityLevel.ADVANCED,
+                    planning=CapabilityLevel.ADVANCED,
+                    summarization=CapabilityLevel.EXPERT,
+                    instruction_following=CapabilityLevel.EXPERT,
+                    tool_use=CapabilityLevel.ADVANCED,
+                    long_context=CapabilityLevel.ADVANCED,
+                ),
+                speed_tier="fast",
+                cost_per_1k_tokens=0.0009,
+                tags=["cloud", "fireworks", "llama"],
+                metadata={},
+            ),
+            ModelDescriptor(
+                model_id="accounts/fireworks/models/deepseek-r1",
+                display_name="DeepSeek R1",
+                provider_id="fireworks",
+                context_length=163840,
+                capabilities=ModelCapabilityProfile(
+                    coding=CapabilityLevel.EXPERT,
+                    reasoning=CapabilityLevel.EXPERT,
+                    planning=CapabilityLevel.EXPERT,
+                    summarization=CapabilityLevel.ADVANCED,
+                    instruction_following=CapabilityLevel.EXPERT,
+                    tool_use=CapabilityLevel.ADVANCED,
+                    long_context=CapabilityLevel.ADVANCED,
+                ),
+                speed_tier="slow",
+                cost_per_1k_tokens=0.003,
+                tags=["cloud", "fireworks", "deepseek", "reasoning"],
+                metadata={},
+            ),
+            ModelDescriptor(
+                model_id="accounts/fireworks/models/qwen2p5-coder-32b-instruct",
+                display_name="Qwen 2.5 Coder 32B Instruct",
+                provider_id="fireworks",
+                context_length=131072,
+                capabilities=ModelCapabilityProfile(
+                    coding=CapabilityLevel.EXPERT,
+                    reasoning=CapabilityLevel.ADVANCED,
+                    planning=CapabilityLevel.ADVANCED,
+                    summarization=CapabilityLevel.ADVANCED,
+                    instruction_following=CapabilityLevel.EXPERT,
+                    tool_use=CapabilityLevel.ADVANCED,
+                    long_context=CapabilityLevel.ADVANCED,
+                ),
+                speed_tier="medium",
+                cost_per_1k_tokens=0.0009,
+                tags=["cloud", "fireworks", "qwen", "coding"],
+                metadata={},
+            ),
+            ModelDescriptor(
+                model_id="accounts/fireworks/models/mixtral-8x22b-instruct",
+                display_name="Mixtral 8x22B Instruct",
+                provider_id="fireworks",
+                context_length=65536,
+                capabilities=ModelCapabilityProfile(
+                    coding=CapabilityLevel.ADVANCED,
+                    reasoning=CapabilityLevel.ADVANCED,
+                    planning=CapabilityLevel.ADVANCED,
+                    summarization=CapabilityLevel.EXPERT,
+                    instruction_following=CapabilityLevel.ADVANCED,
+                    tool_use=CapabilityLevel.ADVANCED,
+                    long_context=CapabilityLevel.ADVANCED,
+                ),
+                speed_tier="medium",
+                cost_per_1k_tokens=0.0009,
+                tags=["cloud", "fireworks", "mixtral", "moe"],
+                metadata={},
+            ),
+        ]

velune/providers/discovery/gguf.py

@@ -0,0 +1,87 @@
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+
+from velune.core.types.model import CapabilityLevel, ModelCapabilityProfile, ModelDescriptor
+
+logger = logging.getLogger("velune.providers.discovery.gguf")
+
+
+class GGUFDiscovery:
+    """Discovers GGUF models from filesystem using LocalModelResolver."""
+
+    def __init__(self):
+        self.provider_id = "gguf"
+
+    async def discover(self) -> list[ModelDescriptor]:
+        """Discover GGUF models across all well-known paths."""
+        from velune.providers.local_resolver import LocalModelResolver
+        resolver = LocalModelResolver()
+        files = resolver.scan_gguf_files()
+
+        models: list[ModelDescriptor] = []
+        for gguf_path in files:
+            descriptor = self._build_descriptor(gguf_path, resolver)
+            if descriptor is not None:
+                models.append(descriptor)
+        return models
+
+    def _build_descriptor(self, gguf_path: Path, resolver) -> ModelDescriptor | None:
+        try:
+            meta = resolver.get_model_metadata(gguf_path)
+
+            try:
+                model_id = str(gguf_path.relative_to(Path.home()))
+            except ValueError:
+                model_id = str(gguf_path)
+
+            display_name = gguf_path.stem
+            param_count_b = meta.get("param_count_b")
+            quantization = meta.get("quantization")
+            context_length = meta.get("context_length") or 4096
+            capabilities = self._classify_capabilities(display_name)
+            vram_gb = self._estimate_vram(param_count_b, quantization)
+
+            return ModelDescriptor(
+                model_id=model_id,
+                provider_id=self.provider_id,
+                display_name=display_name,
+                context_length=context_length,
+                capabilities=capabilities,
+                quantization=quantization,
+                vram_required_gb=vram_gb,
+                parameter_count_b=param_count_b,
+                speed_tier="medium",
+                cost_per_1k_tokens=None,
+                tags=["local", "gguf"],
+                metadata={"gguf_path": str(gguf_path), "family": meta.get("family")},
+            )
+        except Exception:
+            logger.debug("Failed to build descriptor for %s", gguf_path, exc_info=True)
+            return None
+
+    def _estimate_vram(self, param_count_b: float | None, quantization: str | None) -> float | None:
+        if not param_count_b:
+            return None
+        quant_lower = (quantization or "").lower()
+        if "q4" in quant_lower:
+            bpp = 0.55
+        elif "q8" in quant_lower:
+            bpp = 1.0
+        elif "fp16" in quant_lower or "f16" in quant_lower:
+            bpp = 2.0
+        else:
+            bpp = 0.55
+        return param_count_b * bpp + 0.5
+
+    def _classify_capabilities(self, filename: str) -> ModelCapabilityProfile:
+        lower = filename.lower()
+        profile = ModelCapabilityProfile()
+        if any(kw in lower for kw in ("coder", "code")):
+            profile.coding = CapabilityLevel.INTERMEDIATE
+        else:
+            profile.coding = CapabilityLevel.BASIC
+        profile.reasoning = CapabilityLevel.BASIC
+        profile.instruction_following = CapabilityLevel.BASIC
+        return profile