velune-cli 0.9.0__py3-none-any.whl

velune/providers/adapters/google.py

@@ -0,0 +1,234 @@
+"""Google Gemini provider adapter — Generative Language REST API."""
+
+from __future__ import annotations
+
+import json
+import time
+from collections.abc import AsyncIterator
+
+import httpx
+
+from velune.core.errors.provider import InferenceError, ProviderAuthenticationError
+from velune.core.types.inference import InferenceRequest, InferenceResponse, StreamChunk
+from velune.core.types.model import CapabilityLevel, ModelDescriptor
+from velune.core.types.provider import ProviderCapabilities, ProviderHealth
+from velune.providers.base import ModelProvider
+from velune.providers.keystore import get_key
+
+_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"
+
+_MODELS = [
+    ModelDescriptor(
+        model_id="gemini-2.0-flash",
+        display_name="Gemini 2.0 Flash",
+        provider_id="google",
+        context_length=1048576,
+        capabilities={
+            "coding": CapabilityLevel.ADVANCED,
+            "reasoning": CapabilityLevel.ADVANCED,
+            "planning": CapabilityLevel.ADVANCED,
+            "summarization": CapabilityLevel.EXPERT,
+            "instruction_following": CapabilityLevel.EXPERT,
+            "tool_use": CapabilityLevel.ADVANCED,
+            "long_context": CapabilityLevel.EXPERT,
+        },
+        is_local=False,
+        speed_tier="fast",
+        cost_per_1k_tokens=0.000075,
+        tags=["cloud", "google", "flash", "free"],
+    ),
+    ModelDescriptor(
+        model_id="gemini-1.5-pro",
+        display_name="Gemini 1.5 Pro",
+        provider_id="google",
+        context_length=2097152,
+        capabilities={
+            "coding": CapabilityLevel.EXPERT,
+            "reasoning": CapabilityLevel.EXPERT,
+            "planning": CapabilityLevel.EXPERT,
+            "summarization": CapabilityLevel.EXPERT,
+            "instruction_following": CapabilityLevel.EXPERT,
+            "tool_use": CapabilityLevel.EXPERT,
+            "long_context": CapabilityLevel.EXPERT,
+        },
+        is_local=False,
+        speed_tier="medium",
+        cost_per_1k_tokens=0.00125,
+        tags=["cloud", "google", "pro"],
+    ),
+    ModelDescriptor(
+        model_id="gemini-1.5-flash",
+        display_name="Gemini 1.5 Flash",
+        provider_id="google",
+        context_length=1048576,
+        capabilities={
+            "coding": CapabilityLevel.ADVANCED,
+            "reasoning": CapabilityLevel.ADVANCED,
+            "planning": CapabilityLevel.INTERMEDIATE,
+            "summarization": CapabilityLevel.ADVANCED,
+            "instruction_following": CapabilityLevel.EXPERT,
+            "tool_use": CapabilityLevel.ADVANCED,
+            "long_context": CapabilityLevel.EXPERT,
+        },
+        is_local=False,
+        speed_tier="fast",
+        cost_per_1k_tokens=0.000075,
+        tags=["cloud", "google", "flash", "free"],
+    ),
+    ModelDescriptor(
+        model_id="gemini-2.0-flash-thinking-exp",
+        display_name="Gemini 2.0 Flash Thinking",
+        provider_id="google",
+        context_length=32767,
+        capabilities={
+            "coding": CapabilityLevel.EXPERT,
+            "reasoning": CapabilityLevel.EXPERT,
+            "planning": CapabilityLevel.EXPERT,
+            "summarization": CapabilityLevel.ADVANCED,
+            "instruction_following": CapabilityLevel.EXPERT,
+            "tool_use": CapabilityLevel.ADVANCED,
+            "long_context": CapabilityLevel.INTERMEDIATE,
+        },
+        is_local=False,
+        speed_tier="medium",
+        cost_per_1k_tokens=0.0,
+        tags=["cloud", "google", "thinking", "free"],
+    ),
+]
+
+# Public alias used by tests and tooling
+GEMINI_MODELS = _MODELS
+
+
+def _build_contents(messages: list[dict]) -> tuple[list[dict], str]:
+    """Split messages into Gemini *contents* + system instruction text."""
+    system_parts: list[str] = []
+    contents: list[dict] = []
+    for msg in messages:
+        role = msg.get("role", "user")
+        text = msg.get("content", "")
+        if role == "system":
+            system_parts.append(text)
+        else:
+            gemini_role = "model" if role == "assistant" else "user"
+            contents.append({"role": gemini_role, "parts": [{"text": text}]})
+    return contents, "\n".join(system_parts)
+
+
+class GoogleProvider(ModelProvider):
+    """Google Gemini provider using the Generative Language REST API."""
+
+    def __init__(self, api_key: str | None = None) -> None:
+        self._api_key = api_key or get_key("google")
+        self.client: httpx.AsyncClient | None = None
+        self._capabilities = ProviderCapabilities(
+            supports_streaming=True,
+            supports_function_calling=True,
+            supports_embeddings=False,
+            max_context_window=2097152,
+        )
+
+    def _convert_messages(self, request: InferenceRequest) -> dict:
+        """Build the full Gemini REST payload from an InferenceRequest."""
+        contents, system_text = _build_contents(request.messages)
+        payload: dict = {
+            "contents": contents,
+            "generationConfig": {
+                "temperature": request.temperature,
+                "topP": request.top_p,
+                **({"maxOutputTokens": request.max_tokens} if request.max_tokens else {}),
+                **({"stopSequences": request.stop_sequences} if request.stop_sequences else {}),
+            },
+        }
+        if system_text:
+            payload["systemInstruction"] = {"parts": [{"text": system_text}]}
+        return payload
+
+    @property
+    def provider_id(self) -> str:
+        return "google"
+
+    async def initialize(self) -> None:
+        if not self._api_key:
+            raise ProviderAuthenticationError(
+                "Google API key not found — set GOOGLE_API_KEY or run: velune config set-key google"
+            )
+        if not self.client:
+            self.client = httpx.AsyncClient(base_url=_BASE_URL, timeout=300.0)
+
+    async def list_models(self) -> list[ModelDescriptor]:
+        return list(_MODELS)
+
+    async def infer(self, request: InferenceRequest) -> InferenceResponse:
+        await self.initialize()
+        assert self.client is not None
+        start = time.perf_counter()
+        payload = self._convert_messages(request)
+
+        try:
+            url = f"/models/{request.model_id}:generateContent"
+            resp = await self.client.post(url, json=payload, params={"key": self._api_key})
+            resp.raise_for_status()
+            data = resp.json()
+            latency = (time.perf_counter() - start) * 1000.0
+
+            candidate = data.get("candidates", [{}])[0]
+            text = "".join(p.get("text", "") for p in candidate.get("content", {}).get("parts", []))
+            usage = data.get("usageMetadata", {})
+            return InferenceResponse(
+                content=text,
+                model_id=request.model_id,
+                finish_reason=(candidate.get("finishReason") or "STOP").lower(),
+                tokens_used=usage.get("totalTokenCount", 0),
+                latency_ms=latency,
+            )
+        except httpx.HTTPError as e:
+            raise InferenceError(f"Google Gemini inference failed: {e}")
+
+    async def stream(self, request: InferenceRequest) -> AsyncIterator[StreamChunk]:
+        await self.initialize()
+        assert self.client is not None
+        payload = self._convert_messages(request)
+
+        try:
+            url = f"/models/{request.model_id}:streamGenerateContent"
+            params = {"key": self._api_key, "alt": "sse"}
+            async with self.client.stream("POST", url, json=payload, params=params) as resp:
+                resp.raise_for_status()
+                async for line in resp.aiter_lines():
+                    if not line.startswith("data: "):
+                        continue
+                    try:
+                        data = json.loads(line[6:])
+                        candidate = data.get("candidates", [{}])[0]
+                        parts = candidate.get("content", {}).get("parts", [])
+                        text = "".join(p.get("text", "") for p in parts)
+                        finish = candidate.get("finishReason")
+                        yield StreamChunk(
+                            content=text,
+                            finish_reason=finish.lower() if finish else None,
+                        )
+                    except (json.JSONDecodeError, IndexError):
+                        continue
+        except httpx.HTTPError as e:
+            raise InferenceError(f"Google Gemini stream failed: {e}")
+
+    async def embed(self, texts: list[str], model_id: str) -> list[list[float]]:
+        raise NotImplementedError("GoogleProvider does not support embeddings via this adapter.")
+
+    async def health_check(self) -> ProviderHealth:
+        try:
+            await self.initialize()
+            assert self.client is not None
+            resp = await self.client.get("/models", params={"key": self._api_key})
+            return ProviderHealth.HEALTHY if resp.status_code == 200 else ProviderHealth.DEGRADED
+        except Exception:
+            return ProviderHealth.UNAVAILABLE
+
+    def get_capabilities(self) -> ProviderCapabilities:
+        return self._capabilities
+
+    async def shutdown(self) -> None:
+        if self.client:
+            await self.client.aclose()
+            self.client = None

velune/providers/adapters/groq.py

@@ -0,0 +1,151 @@
+"""Groq provider adapter — OpenAI-compatible endpoint, free tier."""
+
+from __future__ import annotations
+
+from velune.core.types.model import CapabilityLevel, ModelCapabilityProfile, ModelDescriptor
+from velune.core.types.provider import ProviderHealth
+from velune.providers.adapters.openai import OpenAIProvider
+from velune.providers.keystore import get_key, has_key
+
+GROQ_MODELS: list[ModelDescriptor] = [
+    ModelDescriptor(
+        model_id="llama-3.3-70b-versatile",
+        provider_id="groq",
+        display_name="Llama 3.3 70B Versatile",
+        context_length=131072,
+        is_local=False,
+        free_tier=True,
+        cost_per_1k_tokens=0.0,
+        speed_tier="fast",
+        capabilities=ModelCapabilityProfile(
+            coding=CapabilityLevel.ADVANCED,
+            reasoning=CapabilityLevel.ADVANCED,
+            planning=CapabilityLevel.ADVANCED,
+            summarization=CapabilityLevel.EXPERT,
+            instruction_following=CapabilityLevel.EXPERT,
+            tool_use=CapabilityLevel.ADVANCED,
+            long_context=CapabilityLevel.EXPERT,
+        ),
+        tags=["cloud", "groq", "free", "llama"],
+        metadata={"free_tier": True},
+    ),
+    ModelDescriptor(
+        model_id="llama-3.1-8b-instant",
+        provider_id="groq",
+        display_name="Llama 3.1 8B Instant",
+        context_length=131072,
+        is_local=False,
+        free_tier=True,
+        cost_per_1k_tokens=0.0,
+        speed_tier="fast",
+        capabilities=ModelCapabilityProfile(
+            coding=CapabilityLevel.INTERMEDIATE,
+            reasoning=CapabilityLevel.INTERMEDIATE,
+            planning=CapabilityLevel.INTERMEDIATE,
+            summarization=CapabilityLevel.ADVANCED,
+            instruction_following=CapabilityLevel.ADVANCED,
+            tool_use=CapabilityLevel.INTERMEDIATE,
+            long_context=CapabilityLevel.ADVANCED,
+        ),
+        tags=["cloud", "groq", "free", "llama", "instant"],
+        metadata={"free_tier": True},
+    ),
+    ModelDescriptor(
+        model_id="mixtral-8x7b-32768",
+        provider_id="groq",
+        display_name="Mixtral 8x7B",
+        context_length=32768,
+        is_local=False,
+        free_tier=True,
+        cost_per_1k_tokens=0.0,
+        speed_tier="fast",
+        capabilities=ModelCapabilityProfile(
+            coding=CapabilityLevel.ADVANCED,
+            reasoning=CapabilityLevel.ADVANCED,
+            planning=CapabilityLevel.INTERMEDIATE,
+            summarization=CapabilityLevel.ADVANCED,
+            instruction_following=CapabilityLevel.ADVANCED,
+            tool_use=CapabilityLevel.INTERMEDIATE,
+            long_context=CapabilityLevel.INTERMEDIATE,
+        ),
+        tags=["cloud", "groq", "free", "mixtral"],
+        metadata={"free_tier": True},
+    ),
+    ModelDescriptor(
+        model_id="gemma2-9b-it",
+        provider_id="groq",
+        display_name="Gemma 2 9B Instruct",
+        context_length=8192,
+        is_local=False,
+        free_tier=True,
+        cost_per_1k_tokens=0.0,
+        speed_tier="fast",
+        capabilities=ModelCapabilityProfile(
+            coding=CapabilityLevel.INTERMEDIATE,
+            reasoning=CapabilityLevel.INTERMEDIATE,
+            planning=CapabilityLevel.BASIC,
+            summarization=CapabilityLevel.ADVANCED,
+            instruction_following=CapabilityLevel.ADVANCED,
+            tool_use=CapabilityLevel.BASIC,
+            long_context=CapabilityLevel.BASIC,
+        ),
+        tags=["cloud", "groq", "free", "gemma"],
+        metadata={"free_tier": True},
+    ),
+    ModelDescriptor(
+        model_id="llama-3.2-11b-vision-preview",
+        provider_id="groq",
+        display_name="Llama 3.2 11B Vision",
+        context_length=8192,
+        is_local=False,
+        free_tier=True,
+        cost_per_1k_tokens=0.0,
+        speed_tier="fast",
+        capabilities=ModelCapabilityProfile(
+            coding=CapabilityLevel.INTERMEDIATE,
+            reasoning=CapabilityLevel.INTERMEDIATE,
+            planning=CapabilityLevel.BASIC,
+            summarization=CapabilityLevel.INTERMEDIATE,
+            instruction_following=CapabilityLevel.ADVANCED,
+            tool_use=CapabilityLevel.BASIC,
+            long_context=CapabilityLevel.BASIC,
+        ),
+        tags=["cloud", "groq", "free", "llama", "vision"],
+        metadata={"free_tier": True},
+    ),
+]
+
+
+class GroqProvider(OpenAIProvider):
+    """Groq Cloud provider — wire-compatible with the OpenAI chat API.
+
+    Uses Groq's custom LPU hardware for extremely fast free-tier inference.
+    """
+
+    def __init__(
+        self,
+        api_key: str | None = None,
+        base_url: str = "https://api.groq.com/openai/v1",
+    ) -> None:
+        super().__init__(api_key=api_key or get_key("groq"), base_url=base_url)
+
+    @property
+    def provider_id(self) -> str:
+        return "groq"
+
+    async def list_models(self) -> list[ModelDescriptor]:
+        return GROQ_MODELS
+
+    async def health_check(self) -> ProviderHealth:
+        if not has_key("groq"):
+            return ProviderHealth.UNAVAILABLE
+        return await super().health_check()
+
+    def get_provider_info(self) -> dict:
+        return {
+            "provider_id": "groq",
+            "display_name": "Groq",
+            "is_free_tier": True,
+            "base_url": "https://api.groq.com/openai/v1",
+            "note": "Free tier — extremely fast inference via custom LPU hardware",
+        }

velune/providers/adapters/huggingface.py

@@ -0,0 +1,210 @@
+"""Hugging Face provider adapter implementation."""
+
+from __future__ import annotations
+
+import json
+import time
+from collections.abc import AsyncIterator
+
+import httpx
+from pydantic import SecretStr
+
+from velune.core.errors.provider import (
+    InferenceError,
+    ProviderAuthenticationError,
+)
+from velune.core.types.inference import InferenceRequest, InferenceResponse, StreamChunk
+from velune.core.types.model import ModelDescriptor
+from velune.core.types.provider import ProviderCapabilities, ProviderHealth
+from velune.providers.base import ModelProvider
+from velune.providers.keystore import get_key
+
+
+class HuggingFaceProvider(ModelProvider):
+    """Hugging Face provider for serverless Inference API."""
+
+    def __init__(
+        self,
+        api_key: str | SecretStr | None = None,
+        base_url: str = "https://api-inference.huggingface.co",
+    ) -> None:
+        self._api_key = api_key or get_key("huggingface")
+        if hasattr(self._api_key, "get_secret_value"):
+            self._api_key = self._api_key.get_secret_value()
+        self._base_url = base_url
+        self.client: httpx.AsyncClient | None = None
+        self._capabilities = ProviderCapabilities(
+            supports_streaming=True,
+            supports_function_calling=False,
+            supports_embeddings=True,
+            max_context_window=32768,
+        )
+
+    @property
+    def provider_id(self) -> str:
+        return "huggingface"
+
+    async def initialize(self) -> None:
+        """Initialize client headers."""
+        if not self._api_key:
+            raise ProviderAuthenticationError(
+                "Hugging Face API token (HF_TOKEN) not found in environment or config"
+            )
+        if not self.client:
+            headers = {"Authorization": f"Bearer {self._api_key}"}
+            self.client = httpx.AsyncClient(base_url=self._base_url, headers=headers, timeout=300.0)
+
+    async def list_models(self) -> list[ModelDescriptor]:
+        """Fetch list of local cached Hugging Face models."""
+        from velune.providers.discovery.huggingface import HuggingFaceDiscovery
+
+        discovery = HuggingFaceDiscovery()
+        return await discovery.discover()
+
+    async def infer(self, request: InferenceRequest) -> InferenceResponse:
+        """Query Hugging Face serverless chat completion API."""
+        await self.initialize()
+        assert self.client is not None
+        start = time.perf_counter()
+        try:
+            # Map standard messages to conversational prompt
+            prompt = self._format_messages_to_prompt(request.messages)
+
+            payload = {
+                "inputs": prompt,
+                "parameters": {
+                    "temperature": request.temperature,
+                    "max_new_tokens": request.max_tokens or 1024,
+                    "top_p": request.top_p,
+                },
+                "options": {"wait_for_model": True},
+            }
+
+            model_path = f"/models/{request.model_id}"
+            response = await self.client.post(model_path, json=payload)
+            response.raise_for_status()
+            data = response.json()
+            latency = (time.perf_counter() - start) * 1000.0
+
+            # HF Serverless response formatting varies by model/pipeline type
+            content = ""
+            if isinstance(data, list) and len(data) > 0:
+                content = data[0].get("generated_text", "")
+                # Strip the prompt from generation if the model prepends it
+                if content.startswith(prompt):
+                    content = content[len(prompt) :]
+            elif isinstance(data, dict):
+                content = data.get("generated_text", "")
+
+            return InferenceResponse(
+                content=content.strip(),
+                model_id=request.model_id,
+                finish_reason="stop",
+                tokens_used=0,  # HF serverless doesn't return exact token metrics consistently
+                latency_ms=latency,
+            )
+        except httpx.HTTPError as e:
+            raise InferenceError(f"Hugging Face Inference completion failed: {e}")
+
+    async def stream(self, request: InferenceRequest) -> AsyncIterator[StreamChunk]:
+        """Stream conversational replies from Serverless Inference API."""
+        await self.initialize()
+        assert self.client is not None
+        try:
+            prompt = self._format_messages_to_prompt(request.messages)
+            payload = {
+                "inputs": prompt,
+                "parameters": {
+                    "temperature": request.temperature,
+                    "max_new_tokens": request.max_tokens or 1024,
+                    "top_p": request.top_p,
+                },
+                "options": {"wait_for_model": True},
+                "stream": True,
+            }
+
+            model_path = f"/models/{request.model_id}"
+            async with self.client.stream("POST", model_path, json=payload) as response:
+                response.raise_for_status()
+                # Serverless stream format is line-delimited SSE chunks
+                async for line in response.aiter_lines():
+                    if line.startswith("data:"):
+                        try:
+                            chunk_data = json.loads(line[5:])
+                            token_text = chunk_data.get("token", {}).get("text", "")
+                            yield StreamChunk(
+                                content=token_text,
+                                finish_reason="stop"
+                                if chunk_data.get("token", {}).get("special", False)
+                                else None,
+                            )
+                        except Exception:
+                            continue
+        except httpx.HTTPError as e:
+            raise InferenceError(f"Hugging Face Inference streaming failed: {e}")
+
+    async def embed(self, texts: list[str], model_id: str) -> list[list[float]]:
+        """Batch embeddings generation using HF feature-extraction pipeline."""
+        await self.initialize()
+        assert self.client is not None
+        try:
+            model_path = f"/models/{model_id}"
+            response = await self.client.post(
+                model_path, json={"inputs": texts, "options": {"wait_for_model": True}}
+            )
+            response.raise_for_status()
+            embeddings = response.json()
+
+            # Embeddings could be 1D or 2D/3D depending on token poolings. Ensure we return 2D floats.
+            if isinstance(embeddings, list) and len(embeddings) > 0:
+                if isinstance(embeddings[0], list):
+                    # Check if it has token-level embeddings or pooled
+                    if isinstance(embeddings[0][0], list):
+                        # Simple average pooling for token embeddings
+                        pooled = []
+                        for seq in embeddings:
+                            avg = [sum(col) / len(seq) for col in zip(*seq, strict=False)]
+                            pooled.append(avg)
+                        return pooled
+                    return embeddings
+                # Single sequence 1D returned, wrap in list
+                return [embeddings]
+            raise ValueError("Invalid embedding response structure from HF Inference API")
+        except httpx.HTTPError as e:
+            raise InferenceError(f"Hugging Face embedding failed: {e}")
+
+    def _format_messages_to_prompt(self, messages: list[dict]) -> str:
+        """Utility to stitch general messages into standard chat-template prompt representation."""
+        prompt = ""
+        for msg in messages:
+            role = msg.get("role", "user")
+            content = msg.get("content", "")
+            if role == "system":
+                prompt += f"<|system|>\n{content}</s>\n"
+            elif role == "user":
+                prompt += f"<|user|>\n{content}</s>\n"
+            else:
+                prompt += f"<|assistant|>\n{content}</s>\n"
+        prompt += "<|assistant|>\n"
+        return prompt
+
+    async def health_check(self) -> ProviderHealth:
+        """Query HF API viability."""
+        try:
+            await self.initialize()
+            assert self.client is not None
+            # Fetch meta details for standard model to verify connection
+            resp = await self.client.get("/models/gpt2")
+            if resp.status_code == 200:
+                return ProviderHealth.HEALTHY
+            return ProviderHealth.DEGRADED
+        except Exception:
+            return ProviderHealth.UNAVAILABLE
+
+    def get_capabilities(self) -> ProviderCapabilities:
+        return self._capabilities
+
+    async def shutdown(self) -> None:
+        if self.client:
+            await self.client.aclose()
+            self.client = None