velune-cli 0.9.0__py3-none-any.whl

velune/providers/adapters/llamacpp.py

@@ -0,0 +1,208 @@
+"""Llama.cpp local GGUF model provider adapter implementation."""
+
+from __future__ import annotations
+
+import asyncio
+import time
+from collections.abc import AsyncIterator
+from pathlib import Path
+from typing import Any
+
+from velune.core.errors.provider import InferenceError, ProviderConnectionError
+from velune.core.types.inference import InferenceRequest, InferenceResponse, StreamChunk
+from velune.core.types.model import ModelDescriptor
+from velune.core.types.provider import ProviderCapabilities, ProviderHealth
+from velune.providers.base import ModelProvider
+
+
+class LlamaCppProvider(ModelProvider):
+    """Llama.cpp provider for running in-process GGUF models."""
+
+    def __init__(self, models_dir: str | None = None) -> None:
+        self._models_dir = Path(models_dir) if models_dir else Path.home() / "models"
+        self._loaded_models = {}
+        self._capabilities = ProviderCapabilities(
+            supports_streaming=True,
+            supports_function_calling=False,
+            supports_embeddings=True,
+            max_context_window=32768,
+        )
+
+    @property
+    def provider_id(self) -> str:
+        return "llamacpp"
+
+    async def initialize(self) -> None:
+        """Verify llama-cpp-python library is available."""
+        try:
+            import llama_cpp  # noqa: F401
+        except ImportError:
+            raise ProviderConnectionError(
+                "llama-cpp-python dependency is missing. Install with: pip install llama-cpp-python"
+            )
+
+    def _resolve_model_path(self, model_id: str) -> Path:
+        """Resolve the GGUF model path from model ID."""
+        from velune.providers.local_paths import get_model_path, save_model_path
+        from velune.providers.local_resolver import LocalModelResolver
+
+        # 1. Check persistent cache first
+        cached = get_model_path(model_id)
+        if cached:
+            return cached
+
+        # 2. Ask LocalModelResolver (absolute, relative, stem scan)
+        resolver = LocalModelResolver()
+        found = resolver.resolve_model_path(model_id)
+        if found:
+            save_model_path(model_id, found)
+            return found
+
+        # 3. Interactive prompt — only in a real terminal
+        prompted = resolver.prompt_for_path(model_id)
+        if prompted:
+            save_model_path(model_id, prompted)
+            return prompted
+
+        raise FileNotFoundError(f"GGUF model file not found for ID: {model_id}")
+
+    def _get_model(self, model_id: str, context_window: int = 4096) -> Any:
+        """Synchronously get or load the llama_cpp Llama instance."""
+        if model_id in self._loaded_models:
+            return self._loaded_models[model_id]
+
+        from llama_cpp import Llama
+
+        model_path = self._resolve_model_path(model_id)
+
+        # Load the model in-memory.
+        # Using typical defaults, letting it use GPU if compiled with CUDA/metal.
+        llm = Llama(
+            model_path=str(model_path),
+            n_ctx=context_window,
+            n_gpu_layers=-1,  # Load as many layers as possible to GPU if available
+            verbose=False,
+        )
+        self._loaded_models[model_id] = llm
+        return llm
+
+    async def list_models(self) -> list[ModelDescriptor]:
+        """List local GGUF models via filesystem discovery."""
+        await self.initialize()
+        from velune.providers.discovery.gguf import GGUFDiscovery
+
+        return await GGUFDiscovery().discover()
+
+    async def infer(self, request: InferenceRequest) -> InferenceResponse:
+        """Non-blocking in-process inference using asyncio thread offloading."""
+        await self.initialize()
+        start = time.perf_counter()
+
+        try:
+            # Resolve model context window size
+            ctx_len = request.max_tokens or 4096
+            llm = await asyncio.to_thread(self._get_model, request.model_id, ctx_len)
+
+            # Map standard messages to llama_cpp chat completions format
+            messages = [
+                {"role": msg.get("role"), "content": msg.get("content")} for msg in request.messages
+            ]
+
+            completion = await asyncio.to_thread(
+                llm.create_chat_completion,
+                messages=messages,
+                temperature=request.temperature,
+                max_tokens=request.max_tokens,
+                top_p=request.top_p,
+                stream=False,
+            )
+
+            latency = (time.perf_counter() - start) * 1000.0
+            choice = completion["choices"][0]
+
+            return InferenceResponse(
+                content=choice["message"]["content"] or "",
+                model_id=request.model_id,
+                finish_reason=choice.get("finish_reason") or "stop",
+                tokens_used=completion.get("usage", {}).get("total_tokens", 0),
+                latency_ms=latency,
+            )
+        except Exception as e:
+            raise InferenceError(f"Local llama.cpp inference failed: {e}")
+
+    async def stream(self, request: InferenceRequest) -> AsyncIterator[StreamChunk]:
+        """Streaming chat completions in non-blocking fashion."""
+        await self.initialize()
+
+        try:
+            ctx_len = request.max_tokens or 4096
+            llm = await asyncio.to_thread(self._get_model, request.model_id, ctx_len)
+            messages = [
+                {"role": msg.get("role"), "content": msg.get("content")} for msg in request.messages
+            ]
+
+            # Run the generator in a thread pool and yield chunks back to async loop
+            def run_stream():
+                return llm.create_chat_completion(
+                    messages=messages,
+                    temperature=request.temperature,
+                    max_tokens=request.max_tokens,
+                    top_p=request.top_p,
+                    stream=True,
+                )
+
+            stream_gen = await asyncio.to_thread(run_stream)
+
+            # Helper to fetch next item synchronously in thread
+            def next_chunk(iterator):
+                try:
+                    return next(iterator)
+                except StopIteration:
+                    return None
+
+            while True:
+                chunk = await asyncio.to_thread(next_chunk, stream_gen)
+                if chunk is None:
+                    break
+
+                choice = chunk["choices"][0]
+                delta = choice.get("delta", {})
+                content = delta.get("content", "")
+                finish = choice.get("finish_reason")
+
+                yield StreamChunk(
+                    content=content,
+                    finish_reason=finish,
+                )
+
+        except Exception as e:
+            raise InferenceError(f"Local llama.cpp streaming failed: {e}")
+
+    async def embed(self, texts: list[str], model_id: str) -> list[list[float]]:
+        """Generate batch embeddings in-process."""
+        await self.initialize()
+
+        try:
+            llm = await asyncio.to_thread(self._get_model, model_id)
+            embeddings = []
+            for text in texts:
+                res = await asyncio.to_thread(llm.create_embedding, input=text)
+                embeddings.append(res["data"][0]["embedding"])
+            return embeddings
+        except Exception as e:
+            raise InferenceError(f"Local llama.cpp embedding failed: {e}")
+
+    async def health_check(self) -> ProviderHealth:
+        """Pings provider availability."""
+        try:
+            await self.initialize()
+            return ProviderHealth.HEALTHY
+        except Exception:
+            return ProviderHealth.UNAVAILABLE
+
+    def get_capabilities(self) -> ProviderCapabilities:
+        return self._capabilities
+
+    async def shutdown(self) -> None:
+        """Release loaded model states."""
+        self._loaded_models.clear()

velune/providers/adapters/lmstudio.py

@@ -0,0 +1,175 @@
+"""LM Studio provider adapter implementation."""
+
+from __future__ import annotations
+
+import json
+import time
+from collections.abc import AsyncIterator
+
+import httpx
+
+from velune.core.errors.provider import InferenceError, ProviderConnectionError
+from velune.core.types.inference import InferenceRequest, InferenceResponse, StreamChunk
+from velune.core.types.model import CapabilityLevel, ModelDescriptor
+from velune.core.types.provider import ProviderCapabilities, ProviderHealth
+from velune.providers.base import ModelProvider
+
+
+class LMStudioProvider(ModelProvider):
+    """LM Studio provider for local OpenAI-compatible endpoints."""
+
+    def __init__(self, base_url: str = "http://localhost:1234/v1") -> None:
+        self._base_url = base_url
+        self.client: httpx.AsyncClient | None = None
+        self._capabilities = ProviderCapabilities(
+            supports_streaming=True,
+            supports_function_calling=True,
+            supports_embeddings=True,
+            max_context_window=32768,
+        )
+
+    @property
+    def provider_id(self) -> str:
+        return "lmstudio"
+
+    async def initialize(self) -> None:
+        """Initialize headers and async client connection."""
+        if not self.client:
+            self.client = httpx.AsyncClient(base_url=self._base_url, timeout=300.0)
+
+    async def list_models(self) -> list[ModelDescriptor]:
+        """Fetch list of active models loaded in LM Studio."""
+        await self.initialize()
+        assert self.client is not None
+        try:
+            response = await self.client.get("/models")
+            response.raise_for_status()
+            data = response.json()
+
+            descriptors: list[ModelDescriptor] = []
+            for item in data.get("data", []):
+                m_id = item["id"]
+                descriptors.append(
+                    ModelDescriptor(
+                        model_id=m_id,
+                        display_name=m_id,
+                        provider_id="lmstudio",
+                        context_length=32768,
+                        capabilities={
+                            "coding": CapabilityLevel.INTERMEDIATE,
+                            "reasoning": CapabilityLevel.INTERMEDIATE,
+                            "planning": CapabilityLevel.BASIC,
+                            "summarization": CapabilityLevel.INTERMEDIATE,
+                            "embedding": CapabilityLevel.INTERMEDIATE,
+                            "instruction_following": CapabilityLevel.INTERMEDIATE,
+                            "multimodal": CapabilityLevel.NONE,
+                            "tool_use": CapabilityLevel.INTERMEDIATE,
+                            "long_context": CapabilityLevel.BASIC,
+                        },
+                        is_local=True,
+                    )
+                )
+            return descriptors
+        except httpx.HTTPError as e:
+            raise ProviderConnectionError(f"LM Studio connection error: {e}")
+
+    async def infer(self, request: InferenceRequest) -> InferenceResponse:
+        """Standard chat inference."""
+        await self.initialize()
+        assert self.client is not None
+        start = time.perf_counter()
+        try:
+            payload = {
+                "model": request.model_id,
+                "messages": request.messages,
+                "temperature": request.temperature,
+                "max_tokens": request.max_tokens,
+                "top_p": request.top_p,
+            }
+            if request.stop_sequences:
+                payload["stop"] = request.stop_sequences
+
+            response = await self.client.post("/chat/completions", json=payload)
+            response.raise_for_status()
+            data = response.json()
+            latency = (time.perf_counter() - start) * 1000.0
+
+            return InferenceResponse(
+                content=data["choices"][0]["message"]["content"],
+                model_id=request.model_id,
+                finish_reason=data["choices"][0]["finish_reason"] or "stop",
+                tokens_used=data.get("usage", {}).get("total_tokens", 0),
+                latency_ms=latency,
+            )
+        except httpx.HTTPError as e:
+            raise InferenceError(f"LM Studio completion failed: {e}")
+
+    async def stream(self, request: InferenceRequest) -> AsyncIterator[StreamChunk]:
+        """Streaming chat completions."""
+        await self.initialize()
+        assert self.client is not None
+        try:
+            payload = {
+                "model": request.model_id,
+                "messages": request.messages,
+                "temperature": request.temperature,
+                "max_tokens": request.max_tokens,
+                "top_p": request.top_p,
+                "stream": True,
+            }
+            if request.stop_sequences:
+                payload["stop"] = request.stop_sequences
+
+            async with self.client.stream("POST", "/chat/completions", json=payload) as response:
+                response.raise_for_status()
+                async for line in response.aiter_lines():
+                    if line.startswith("data: "):
+                        data_str = line[6:]
+                        if data_str == "[DONE]":
+                            break
+                        try:
+                            data = json.loads(data_str)
+                            delta = data["choices"][0]["delta"]
+                            yield StreamChunk(
+                                content=delta.get("content", ""),
+                                finish_reason=data["choices"][0].get("finish_reason"),
+                            )
+                        except (json.JSONDecodeError, KeyError):
+                            continue
+        except httpx.HTTPError as e:
+            raise InferenceError(f"LM Studio stream failed: {e}")
+
+    async def embed(self, texts: list[str], model_id: str) -> list[list[float]]:
+        """Generate batch embeddings."""
+        await self.initialize()
+        assert self.client is not None
+        try:
+            response = await self.client.post(
+                "/embeddings", json={"model": model_id, "input": texts}
+            )
+            response.raise_for_status()
+            data = response.json()
+            sorted_data = sorted(data["data"], key=lambda x: x["index"])
+            return [item["embedding"] for item in sorted_data]
+        except httpx.HTTPError as e:
+            raise InferenceError(f"LM Studio embedding failed: {e}")
+
+    async def health_check(self) -> ProviderHealth:
+        """Verifies connectivity."""
+        try:
+            await self.initialize()
+            assert self.client is not None
+            resp = await self.client.get("/models")
+            if resp.status_code == 200:
+                return ProviderHealth.HEALTHY
+            return ProviderHealth.DEGRADED
+        except Exception:
+            return ProviderHealth.UNAVAILABLE
+
+    def get_capabilities(self) -> ProviderCapabilities:
+        return self._capabilities
+
+    async def shutdown(self) -> None:
+        if self.client:
+            await self.client.aclose()
+            self.client = None

velune/providers/adapters/ollama.py

@@ -0,0 +1,233 @@
+"""Ollama provider adapter implementation."""
+
+from __future__ import annotations
+
+import json
+import logging
+import time
+from collections.abc import AsyncIterator
+
+import httpx
+
+from velune.core.errors.provider import InferenceError, ProviderConnectionError
+from velune.core.types.inference import InferenceRequest, InferenceResponse, StreamChunk
+from velune.core.types.model import CapabilityLevel, ModelDescriptor
+from velune.core.types.provider import ProviderCapabilities, ProviderHealth
+from velune.providers.base import ModelProvider
+
+logger = logging.getLogger("velune.providers.adapters.ollama")
+
+
+class OllamaProvider(ModelProvider):
+    """Ollama provider for local models."""
+
+    def __init__(self, base_url: str = "http://localhost:11434") -> None:
+        self._base_url = base_url
+        self.client: httpx.AsyncClient | None = None
+        self._capabilities = ProviderCapabilities(
+            supports_streaming=True,
+            supports_function_calling=False,
+            supports_embeddings=True,
+            max_context_window=8192,
+        )
+
+    @property
+    def provider_id(self) -> str:
+        return "ollama"
+
+    async def initialize(self) -> None:
+        """Initialize the async client."""
+        if not self.client:
+            self.client = httpx.AsyncClient(base_url=self._base_url, timeout=300.0)
+
+    async def _get_model_context_length(self, model_name: str) -> int:
+        """Query /api/show for the model's actual context window size.
+
+        Ollama's ``/api/show`` returns a ``parameters`` string that may contain
+        a line like ``num_ctx                        131072``.  We parse that to
+        get the real context length instead of hard-coding 8192.
+
+        Falls back to 8 192 if the endpoint is unreachable, the model is not
+        loaded, or the field is absent.
+        """
+        assert self.client is not None
+        try:
+            resp = await self.client.post("/api/show", json={"name": model_name})
+            if resp.status_code != 200:
+                return 8192
+            data = resp.json()
+            # ``parameters`` is a newline-delimited string of key-value pairs
+            params_str: str = data.get("parameters", "")
+            for line in params_str.splitlines():
+                parts = line.split()
+                if len(parts) >= 2 and parts[0].lower() == "num_ctx":
+                    try:
+                        return int(parts[1])
+                    except ValueError:
+                        pass
+            # Fallback: check model_info dict returned by newer Ollama builds
+            model_info: dict = data.get("model_info", {})
+            for key, value in model_info.items():
+                if "context" in key.lower() and isinstance(value, int) and value > 0:
+                    return value
+        except Exception as exc:
+            logger.debug("Could not fetch context length for %s: %s", model_name, exc)
+        return 8192
+
+    async def list_models(self) -> list[ModelDescriptor]:
+        """Fetch models from active Ollama endpoint with accurate context lengths.
+
+        Queries ``/api/show`` for each model to populate the real ``num_ctx``
+        value instead of defaulting every model to 8 192 tokens.
+        """
+        await self.initialize()
+        assert self.client is not None
+        try:
+            response = await self.client.get("/api/tags")
+            response.raise_for_status()
+            data = response.json()
+
+            descriptors: list[ModelDescriptor] = []
+            for item in data.get("models", []):
+                model_name = item["name"]
+                ctx_len = await self._get_model_context_length(model_name)
+                descriptors.append(
+                    ModelDescriptor(
+                        model_id=model_name,
+                        display_name=model_name,
+                        provider_id="ollama",
+                        context_length=ctx_len,
+                        capabilities={
+                            "coding": CapabilityLevel.INTERMEDIATE,
+                            "reasoning": CapabilityLevel.INTERMEDIATE,
+                            "planning": CapabilityLevel.BASIC,
+                            "summarization": CapabilityLevel.INTERMEDIATE,
+                            "embedding": CapabilityLevel.INTERMEDIATE,
+                            "instruction_following": CapabilityLevel.INTERMEDIATE,
+                            "multimodal": CapabilityLevel.NONE,
+                            "tool_use": CapabilityLevel.NONE,
+                            "long_context": (
+                                CapabilityLevel.INTERMEDIATE
+                                if ctx_len > 32768
+                                else CapabilityLevel.NONE
+                            ),
+                        },
+                        is_local=True,
+                    )
+                )
+            return descriptors
+        except httpx.HTTPError as e:
+            raise ProviderConnectionError(f"Failed to fetch models from Ollama: {e}")
+
+    async def infer(self, request: InferenceRequest) -> InferenceResponse:
+        """Synchronous chat inference."""
+        await self.initialize()
+        assert self.client is not None
+        start = time.perf_counter()
+        try:
+            payload = {
+                "model": request.model_id,
+                "messages": request.messages,
+                "stream": False,
+                "options": {
+                    "temperature": request.temperature,
+                    "num_predict": request.max_tokens,
+                    "top_p": request.top_p,
+                },
+            }
+            if request.stop_sequences:
+                payload["options"]["stop"] = request.stop_sequences
+
+            response = await self.client.post("/api/chat", json=payload)
+            response.raise_for_status()
+            data = response.json()
+            latency = (time.perf_counter() - start) * 1000.0
+
+            if latency > 30000.0:
+                logger.warning(
+                    "Slow inference on %s (%.1fs). Consider a smaller model for your hardware.",
+                    request.model_id,
+                    latency / 1000.0,
+                )
+
+            return InferenceResponse(
+                content=data["message"]["content"],
+                model_id=request.model_id,
+                finish_reason=data.get("done_reason", "stop"),
+                tokens_used=data.get("eval_count", 0) + data.get("prompt_eval_count", 0),
+                latency_ms=latency,
+            )
+        except httpx.HTTPError as e:
+            raise InferenceError(f"Ollama inference failed: {e}")
+
+    async def stream(self, request: InferenceRequest) -> AsyncIterator[StreamChunk]:
+        """Streaming chat completion."""
+        await self.initialize()
+        assert self.client is not None
+        try:
+            payload = {
+                "model": request.model_id,
+                "messages": request.messages,
+                "stream": True,
+                "options": {
+                    "temperature": request.temperature,
+                    "num_predict": request.max_tokens,
+                    "top_p": request.top_p,
+                },
+            }
+            if request.stop_sequences:
+                payload["options"]["stop"] = request.stop_sequences
+
+            async with self.client.stream("POST", "/api/chat", json=payload) as response:
+                response.raise_for_status()
+                async for line in response.aiter_lines():
+                    if not line:
+                        continue
+                    try:
+                        data = json.loads(line)
+                        if "message" in data:
+                            yield StreamChunk(
+                                content=data["message"].get("content", ""),
+                                finish_reason=data.get("done_reason"),
+                            )
+                    except json.JSONDecodeError:
+                        continue
+        except httpx.HTTPError as e:
+            raise InferenceError(f"Ollama streaming failed: {e}")
+
+    async def embed(self, texts: list[str], model_id: str) -> list[list[float]]:
+        """Batch embedding generation."""
+        await self.initialize()
+        assert self.client is not None
+        embeddings: list[list[float]] = []
+        try:
+            for text in texts:
+                resp = await self.client.post(
+                    "/api/embeddings", json={"model": model_id, "prompt": text}
+                )
+                resp.raise_for_status()
+                embeddings.append(resp.json()["embedding"])
+            return embeddings
+        except httpx.HTTPError as e:
+            raise InferenceError(f"Ollama embedding failed: {e}")
+
+    async def health_check(self) -> ProviderHealth:
+        """Pings Ollama core endpoint."""
+        await self.initialize()
+        assert self.client is not None
+        try:
+            resp = await self.client.get("/")
+            if resp.status_code == 200:
+                return ProviderHealth.HEALTHY
+            return ProviderHealth.DEGRADED
+        except Exception:
+            return ProviderHealth.UNAVAILABLE
+
+    def get_capabilities(self) -> ProviderCapabilities:
+        return self._capabilities
+
+    async def shutdown(self) -> None:
+        """Close connection pools."""
+        if self.client:
+            await self.client.aclose()
+            self.client = None