siftcode 0.3.0__py3-none-any.whl

siftcode/core/llm.py

@@ -0,0 +1,295 @@
+"""
+Ollama LLM client for Siftcode.
+
+Provides a clean interface for chat-based interaction with locally-running
+LLMs via Ollama's HTTP API. Handles retries, timeouts, and per-call model
+overrides.
+
+No GTK dependencies — this module is testable in isolation.
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from dataclasses import dataclass
+from typing import Any
+
+import requests
+
+from siftcode.core.config import ModelConfig
+
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Exceptions
+# ---------------------------------------------------------------------------
+
+class LLMError(Exception):
+    """Base class for all LLM-related errors."""
+
+
+class LLMConnectionError(LLMError):
+    """Raised when the Ollama server cannot be reached."""
+
+
+class LLMTimeoutError(LLMError):
+    """Raised when a request times out."""
+
+
+class LLMResponseError(LLMError):
+    """Raised when the server responds with an error or malformed payload."""
+
+
+class LLMModelNotFoundError(LLMError):
+    """Raised when the requested model is not available in Ollama."""
+
+
+# ---------------------------------------------------------------------------
+# Response dataclass
+# ---------------------------------------------------------------------------
+
+@dataclass
+class ChatResponse:
+    """Result of a single chat completion."""
+
+    text: str
+    model: str
+    duration_seconds: float
+    prompt_tokens: int = 0
+    completion_tokens: int = 0
+
+    @property
+    def total_tokens(self) -> int:
+        return self.prompt_tokens + self.completion_tokens
+
+
+# ---------------------------------------------------------------------------
+# Client
+# ---------------------------------------------------------------------------
+
+class OllamaClient:
+    """
+    Client for Ollama's /api/chat endpoint.
+
+    Example:
+        config = ModelConfig(name="qwen2.5-coder:14b", num_ctx=8192)
+        client = OllamaClient(config)
+
+        response = client.chat(
+            system="You are a code analyzer.",
+            user="Analyze this file: ...",
+        )
+        print(response.text)
+
+        # Override model for a single call
+        response = client.chat(
+            system="You are a technical writer.",
+            user="Write a guide...",
+            model="qwen2.5-coder:32b",
+            max_tokens=4000,
+        )
+    """
+
+    # Retry configuration
+    MAX_RETRIES = 3
+    RETRY_BACKOFF = 2.0  # seconds, multiplied by attempt number
+
+    def __init__(self, config: ModelConfig) -> None:
+        self.config = config
+        self._session = requests.Session()
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    def chat(
+        self,
+        system: str,
+        user: str,
+        *,
+        model: str | None = None,
+        max_tokens: int | None = None,
+        temperature: float | None = None,
+        num_ctx: int | None = None,
+        timeout: int | None = None,
+    ) -> ChatResponse:
+        """
+        Send a chat request and return the response.
+
+        Args:
+            system: System prompt.
+            user: User message.
+            model: Override the model from config for this call.
+            max_tokens: Override max_tokens_file from config.
+            temperature: Override temperature from config.
+            num_ctx: Override num_ctx from config.
+            timeout: Override timeout from config.
+
+        Returns:
+            ChatResponse with the generated text.
+
+        Raises:
+            LLMConnectionError: Cannot reach Ollama.
+            LLMTimeoutError: Request timed out.
+            LLMResponseError: Server returned an error or bad payload.
+            LLMModelNotFoundError: Requested model is not installed.
+        """
+        model_name = model or self.config.name
+        max_tokens_val = max_tokens if max_tokens is not None else self.config.max_tokens_file
+        temperature_val = temperature if temperature is not None else self.config.temperature
+        num_ctx_val = num_ctx if num_ctx is not None else self.config.num_ctx
+        timeout_val = timeout if timeout is not None else self.config.timeout
+
+        payload = {
+            "model": model_name,
+            "stream": False,
+            "options": {
+                "temperature": temperature_val,
+                "num_predict": max_tokens_val,
+                "num_ctx": num_ctx_val,
+            },
+            "messages": [
+                {"role": "system", "content": system},
+                {"role": "user", "content": user},
+            ],
+        }
+
+        url = f"{self.config.ollama_url.rstrip('/')}/api/chat"
+
+        start = time.monotonic()
+        data = self._post_with_retry(url, payload, timeout_val, model_name)
+        duration = time.monotonic() - start
+
+        # Validate response shape
+        if "message" not in data or "content" not in data.get("message", {}):
+            raise LLMResponseError(
+                f"Malformed response from Ollama: missing 'message.content'. "
+                f"Got keys: {list(data.keys())}"
+            )
+
+        text = data["message"]["content"]
+
+        return ChatResponse(
+            text=text,
+            model=model_name,
+            duration_seconds=duration,
+            prompt_tokens=data.get("prompt_eval_count", 0),
+            completion_tokens=data.get("eval_count", 0),
+        )
+
+    def is_available(self) -> bool:
+        """Return True if the Ollama server responds to /api/tags."""
+        url = f"{self.config.ollama_url.rstrip('/')}/api/tags"
+        try:
+            r = self._session.get(url, timeout=5)
+            return r.status_code == 200
+        except requests.RequestException:
+            return False
+
+    def list_models(self) -> list[str]:
+        """Return the names of all models installed in Ollama."""
+        url = f"{self.config.ollama_url.rstrip('/')}/api/tags"
+        try:
+            r = self._session.get(url, timeout=10)
+            r.raise_for_status()
+        except requests.RequestException as e:
+            raise LLMConnectionError(
+                f"Cannot list models from {url}: {e}"
+            ) from e
+
+        try:
+            data = r.json()
+        except ValueError as e:
+            raise LLMResponseError(
+                f"Ollama returned invalid JSON for /api/tags: {e}"
+            ) from e
+
+        return [m["name"] for m in data.get("models", [])]
+
+    def has_model(self, name: str) -> bool:
+        """Return True if a model with the given name is installed."""
+        # Ollama model names include tags like "qwen2.5-coder:14b"
+        # Allow prefix matching so "qwen2.5-coder" matches "qwen2.5-coder:14b"
+        try:
+            available = self.list_models()
+        except LLMError:
+            return False
+        return any(name == m or m.startswith(name + ":") for m in available)
+
+    # ------------------------------------------------------------------
+    # Internal
+    # ------------------------------------------------------------------
+
+    def _post_with_retry(
+        self,
+        url: str,
+        payload: dict[str, Any],
+        timeout: int,
+        model_name: str,
+    ) -> dict[str, Any]:
+        """POST with retry on connection errors. Does not retry on 4xx."""
+        last_error: Exception | None = None
+
+        for attempt in range(1, self.MAX_RETRIES + 1):
+            try:
+                r = self._session.post(url, json=payload, timeout=timeout)
+            except requests.Timeout as e:
+                last_error = LLMTimeoutError(
+                    f"Request to {url} timed out after {timeout}s "
+                    f"(attempt {attempt}/{self.MAX_RETRIES})"
+                )
+                logger.warning(str(last_error))
+                if attempt < self.MAX_RETRIES:
+                    time.sleep(self.RETRY_BACKOFF * attempt)
+                    continue
+                raise last_error from e
+
+            except requests.ConnectionError as e:
+                last_error = LLMConnectionError(
+                    f"Cannot connect to Ollama at {url}: {e}"
+                )
+                logger.warning(str(last_error))
+                if attempt < self.MAX_RETRIES:
+                    time.sleep(self.RETRY_BACKOFF * attempt)
+                    continue
+                raise last_error from e
+
+            except requests.RequestException as e:
+                raise LLMConnectionError(
+                    f"Unexpected request error to {url}: {e}"
+                ) from e
+
+            # HTTP-level error handling (no retry)
+            if r.status_code == 404:
+                raise LLMModelNotFoundError(
+                    f"Model '{model_name}' not found in Ollama. "
+                    f"Run: ollama pull {model_name}"
+                )
+            if r.status_code >= 400:
+                raise LLMResponseError(
+                    f"Ollama returned HTTP {r.status_code}: {r.text[:500]}"
+                )
+
+            # Parse JSON
+            try:
+                return r.json()
+            except ValueError as e:
+                raise LLMResponseError(
+                    f"Ollama returned invalid JSON: {e}"
+                ) from e
+
+        # Only reached if loop never returns/raises
+        raise LLMConnectionError(f"All {self.MAX_RETRIES} retries failed: {last_error}")
+
+    def close(self) -> None:
+        """Close the underlying HTTP session."""
+        self._session.close()
+
+    def __enter__(self) -> "OllamaClient":
+        return self
+
+    def __exit__(self, *exc_info: Any) -> None:
+        self.close()

siftcode/core/manual.py

@@ -0,0 +1,208 @@
+"""
+core/manual.py — Advanced Technical Analysis & Specification Generator.
+
+Performs deep per-file analysis on source code using Ollama via OllamaClient, 
+leveraging AnalysisCache for zero redundant work and zero external UI dependencies.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from pathlib import Path
+from typing import Callable, Dict, List, Optional
+import tomllib
+
+from siftcode.core.cache import AnalysisCache, file_hash
+from siftcode.core.config import ProjectConfig
+from siftcode.core.llm import LLMError, OllamaClient
+
+logger = logging.getLogger(__name__)
+
+ADVANCED_SYSTEM_PROMPT = """You are an expert software architect and technical writer.
+Your task is to analyze the provided source code file and generate a strict, highly detailed Technical Specification in Markdown format.
+
+You MUST follow this EXACT structure without deviating. Do NOT wrap the entire output in a global triple-backtick block. 
+
+# {filename} — Technical Documentation
+
+| Field | Value |
+|---|---|
+| File | {relative_path} |
+| Author | {author} |
+| License | {license} |
+| Version | {version} |
+
+## Overview
+<Provide a clear, high-level summary of the file's primary responsibilities, architecture, and logic flow.>
+
+## Class & Function Reference
+<For every class and stand-alone function in the file, provide its description and method table.>
+
+### ClassOrFunctionName
+<Concise description of purpose>
+
+Python Signature:
+[Insert Python signature or class definition here]
+
+| Method / Attribute | Signature / Type | Purpose |
+|---|---|---|
+| name | signature_or_type | description |
+
+## Test Cases
+<Provide a comprehensive test case matrix describing expected behaviors and expected outcomes.>
+
+| # | Method / Scenario | Input | Expected Result |
+|---|---|---|---|
+| 1 | method_name | input_description | expected_outcome |
+
+## Error Handling
+<List all explicitly raised or caught exceptions, failure modes, and fallback logic.>
+
+| Condition | Exception / Behavior | Raised / Handled By |
+|---|---|---|
+| trigger_condition | exception_type | function_or_method_name |
+
+## Summary
+<Provide a brief 2-3 sentence summary concluding the technical design of this file.>
+"""
+
+
+class SpecGenerator:
+    """Core generator for per-file technical specifications using Siftcode infrastructure."""
+
+    def __init__(
+        self,
+        config: ProjectConfig,
+        client: OllamaClient,
+        cache: AnalysisCache | None = None,
+        author: str = "",
+        license_name: str = "",
+        version: str = "",
+    ) -> None:
+        self.config = config
+        self.client = client
+        self.cache = cache if cache is not None else AnalysisCache(enabled=config.analysis.use_cache)
+        
+        # Haetaan metatiedot dynaamisesti projektista, jos niitä ei ole erikseen annettu
+        project_meta = self._detect_project_metadata()
+        self.author = author or project_meta.get("author", "Not specified")
+        self.license_name = license_name or project_meta.get("license", "Not specified")
+        self.version = version or project_meta.get("version", "0.1.0")
+
+    def _detect_project_metadata(self) -> Dict[str, str]:
+        """Attempts to parse project author, license, and version from standard manifest files."""
+        meta: Dict[str, str] = {}
+        root = self.config.path
+
+        # 1. Tarkistetaan pyproject.toml
+        pyproject_path = root / "pyproject.toml"
+        if pyproject_path.exists():
+            try:
+                with open(pyproject_path, "rb") as f:
+                    data = tomllib.load(f)
+                
+                # Poetry / PEP 621 rakenteet
+                project_section = data.get("project", {})
+                tool_poetry = data.get("tool", {}).get("poetry", {})
+
+                meta["version"] = project_section.get("version") or tool_poetry.get("version")
+                
+                # Lisenssi
+                license_field = project_section.get("license") or tool_poetry.get("license")
+                if isinstance(license_field, dict):
+                    meta["license"] = license_field.get("text")
+                elif isinstance(license_field, str):
+                    meta["license"] = license_field
+
+                # Tekijä(t)
+                authors = project_section.get("authors") or tool_poetry.get("authors")
+                if authors and isinstance(authors, list):
+                    first_author = authors[0]
+                    if isinstance(first_author, dict):
+                        meta["author"] = first_author.get("name")
+                    elif isinstance(first_author, str):
+                        meta["author"] = first_author
+            except Exception as e:
+                logger.debug(f"Could not parse pyproject.toml for metadata: {e}")
+
+        # 2. Tarkistetaan package.json (Node.js)
+        package_json = root / "package.json"
+        if not meta and package_json.exists():
+            try:
+                import json
+                with open(package_json, "r", encoding="utf-8") as f:
+                    data = json.load(f)
+                meta["version"] = data.get("version")
+                meta["license"] = data.get("license")
+                author_field = data.get("author")
+                if isinstance(author_field, dict):
+                    meta["author"] = author_field.get("name")
+                elif isinstance(author_field, str):
+                    meta["author"] = author_field
+            except Exception as e:
+                logger.debug(f"Could not parse package.json for metadata: {e}")
+
+        return meta
+
+    def generate_file_spec(
+        self,
+        file_path: Path,
+    ) -> str:
+        """Analyzes a single source file and returns its technical documentation in Markdown, with caching."""
+        if not file_path.exists() or not file_path.is_file():
+            raise FileNotFoundError(f"Target file does not exist: {file_path}")
+
+        try:
+            source_code = file_path.read_text(encoding="utf-8")
+        except Exception as e:
+            raise IOError(f"Could not read source file {file_path}: {e}") from e
+
+        rel_path = file_path.relative_to(self.config.path).as_posix()
+        
+        key = file_hash(file_path)
+        model = self.config.model.name
+        cached = self.cache.get_file(key, model=model)
+        if cached is not None:
+            logger.debug(f"Spec generator cache hit: {rel_path}")
+            return cached
+
+        system_prompt = ADVANCED_SYSTEM_PROMPT.format(
+            filename=file_path.name,
+            relative_path=rel_path,
+            author=self.author,
+            license=self.license_name,
+            version=self.version,
+        )
+
+        try:
+            response = self.client.chat(
+                system=system_prompt,
+                user=f"Source code to analyze:\n\n```python\n{source_code}\n```",
+            )
+            spec_content = response.text.strip()
+            self.cache.put_file(key, path=rel_path, model=model, summary=spec_content)
+            return spec_content
+        except LLMError as e:
+            raise RuntimeError(f"LLM analysis failed for {rel_path}: {e}") from e
+
+    def generate_project_specs(
+        self,
+        file_list: List[Path],
+        progress_callback: Optional[Callable[[int, int, Path], None]] = None,
+    ) -> Dict[str, str]:
+        """
+        Iterates over a list of source files, generating a specification for each.
+        Returns a dictionary mapping relative file paths to their Markdown analysis.
+        """
+        results: Dict[str, str] = {}
+        total = len(file_list)
+
+        for idx, path in enumerate(file_list, start=1):
+            if progress_callback:
+                progress_callback(idx, total, path)
+
+            rel_key = path.relative_to(self.config.path).as_posix()
+            results[rel_key] = self.generate_file_spec(file_path=path)
+
+        return results