siftcode 0.3.0__py3-none-any.whl

siftcode/core/analyzer.py

@@ -0,0 +1,575 @@
+"""
+Core analyzer module for Siftcode.
+
+Reads a project's files, sends them to a local LLM for analysis,
+and produces a developer guide. Yields progress events so callers
+(GTK UI, CLI) can update their display in real time.
+
+No GTK dependencies — testable with a mock LLM client.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+from dataclasses import dataclass, field
+from datetime import datetime
+from pathlib import Path
+from typing import Iterator
+
+
+from siftcode.core.cache import AnalysisCache, file_hash, string_hash
+from siftcode.core.config import AnalysisConfig, ModelConfig, ProjectConfig
+from siftcode.core.llm import LLMError, OllamaClient
+from siftcode.core import prompts
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Progress event
+# ---------------------------------------------------------------------------
+
+@dataclass
+class Progress:
+    """A single progress event emitted by the analyzer."""
+
+    stage: str          # "root", "file", "directory", "guide", "done"
+    current: str        # what's being worked on (path or label)
+    completed: int      # how many items done in this stage
+    total: int          # how many items total in this stage
+    fraction: float     # overall fraction 0.0 - 1.0
+    message: str = ""   # optional human-readable status
+
+    def __str__(self) -> str:
+        return f"[{self.stage}] {self.current} ({self.completed}/{self.total}) {self.fraction:.0%}"
+
+
+# ---------------------------------------------------------------------------
+# Findings — the accumulated result
+# ---------------------------------------------------------------------------
+
+@dataclass
+class Findings:
+    """Everything the analyzer has collected about a project."""
+
+    root_summary: str = ""
+    directories: dict[str, str] = field(default_factory=dict)
+    files: dict[str, str] = field(default_factory=dict)
+    errors: list[dict[str, str]] = field(default_factory=list)
+    started: str = ""
+    finished: str = ""
+    guide: str = ""
+
+    def to_dict(self) -> dict:
+        return {
+            "root_summary": self.root_summary,
+            "directories": self.directories,
+            "files": self.files,
+            "errors": self.errors,
+            "started": self.started,
+            "finished": self.finished,
+            "guide": self.guide,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "Findings":
+        return cls(
+            root_summary=data.get("root_summary", ""),
+            directories=data.get("directories", {}),
+            files=data.get("files", {}),
+            errors=data.get("errors", []),
+            started=data.get("started", ""),
+            finished=data.get("finished", ""),
+            guide=data.get("guide", ""),
+        )
+
+    def save(self, path: Path) -> None:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        with open(path, "w", encoding="utf-8") as f:
+            json.dump(self.to_dict(), f, indent=2, ensure_ascii=False)
+
+
+# ---------------------------------------------------------------------------
+# Analyzer
+# ---------------------------------------------------------------------------
+
+class ProjectAnalyzer:
+    """
+    Analyzes a software project using a local LLM.
+
+    Usage:
+        config = ProjectConfig(name="My Project", path=Path("/path/to/proj"))
+        client = OllamaClient(config.model)
+        analyzer = ProjectAnalyzer(config, client)
+
+        for progress in analyzer.analyze_project():
+            print(progress)
+
+        findings = analyzer.findings
+        findings.save(Path("findings.json"))
+    """
+
+    # Fraction of total work allocated to each phase
+    ROOT_WEIGHT = 0.05
+    FILES_WEIGHT = 0.70
+    DIRS_WEIGHT = 0.15
+    GUIDE_WEIGHT = 0.10
+
+    def __init__(
+        self,
+        config: ProjectConfig,
+        client: OllamaClient,
+        cache: AnalysisCache | None = None,
+    ) -> None:
+        self.config = config
+        self.client = client
+        # Create a default cache unless caller provides one
+        self.cache = cache if cache is not None else AnalysisCache(
+        enabled=config.analysis.use_cache,
+        )
+        self.findings = Findings()
+
+        # Log-related paths (set on analyze_project())
+        self._log_dir: Path | None = None
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    def analyze_project(self) -> Iterator[Progress]:
+        """
+        Run the full analysis pipeline, yielding Progress events.
+
+        Each yield happens AFTER the corresponding work is done, so
+        callers can trust that when they see an event, the work it
+        describes has completed. This is important for the GTK UI,
+        which shows 'Analysis complete' when the generator ends.
+        """
+        self.findings = Findings(
+            started=datetime.now().isoformat(timespec="seconds")
+        )
+
+        # 1. Root
+        yield Progress("root", str(self.config.path), 0, 1, 0.0,
+                       "Analyzing project root...")
+        self._analyze_root()
+        yield Progress("root", str(self.config.path), 1, 1, self.ROOT_WEIGHT,
+                       "Root analysis complete")
+
+        # 2. Files
+        files = self._collect_files()
+        if files:
+            for i, file_path in enumerate(files):
+                rel = self._rel(file_path)
+                # Work first, then yield progress
+                self._analyze_file(file_path)
+                yield Progress("file", rel, i + 1, len(files),
+                               self.ROOT_WEIGHT + ((i + 1) / len(files)) * self.FILES_WEIGHT,
+                               f"Analyzed {rel}")
+        total_after_files = self.ROOT_WEIGHT + self.FILES_WEIGHT
+        yield Progress("file", "", len(files), len(files),
+                       total_after_files,
+                       f"Analyzed {len(files)} files")
+
+        # 3. Directories
+        directories = self._collect_directories(files)
+        if directories:
+            for i, dir_path in enumerate(directories):
+                rel = self._rel(dir_path)
+                # Work first, then yield progress
+                self._analyze_directory(dir_path)
+                yield Progress("directory", rel, i + 1, len(directories),
+                               total_after_files + ((i + 1) / len(directories)) * self.DIRS_WEIGHT,
+                               f"Summarized {rel}")
+        total_after_dirs = total_after_files + self.DIRS_WEIGHT
+        yield Progress("directory", "", len(directories), len(directories),
+                       total_after_dirs,
+                       f"Summarized {len(directories)} directories")
+
+        # 4. Guide
+        yield Progress("guide", "", 0, 1, total_after_dirs,
+                       "Generating developer guide...")
+        self._generate_guide()
+        self.findings.finished = datetime.now().isoformat(timespec="seconds")
+
+        yield Progress("done", "", 1, 1, 1.0, "Analysis complete")
+
+    # ------------------------------------------------------------------
+    # Stage 1: root
+    # ------------------------------------------------------------------
+
+    def _analyze_root(self) -> None:
+        """Ask the LLM to summarize the project from its file tree."""
+        contents = self._root_tree_preview()
+
+        # Cache key: project path + file listing + manifest content.
+        # The path is essential so different projects don't collide.
+        # The manifest content catches changes to dependencies or
+        # build tooling that don't affect the file listing.
+        manifest = self._read_project_manifest()
+        key_input = f"{self.config.path.resolve()}\n{contents}\n{manifest}"
+        key = string_hash(key_input)
+        model = self.config.model.name
+
+        cached = self.cache.get_root(key, model=model)
+        if cached is not None:
+            logger.debug("Root cache hit")
+            self.findings.root_summary = cached
+            return
+
+        user_prompt = prompts.ROOT_SUMMARY.format(
+            project_dir=str(self.config.path),
+            contents=contents,
+        )
+
+        try:
+            response = self.client.chat(
+                system=prompts.SYSTEM_ROOT,
+                user=user_prompt,
+            )
+            summary = response.text.strip()
+            self.findings.root_summary = summary
+            self.cache.put_root(key, path=".", model=model, summary=summary)
+        except LLMError as e:
+            logger.error(f"Root analysis failed: {e}")
+            self.findings.errors.append({
+                "stage": "root",
+                "path": str(self.config.path),
+                "error": str(e),
+            })
+            self.findings.root_summary = "(root analysis failed)"
+
+    def _root_tree_preview(self) -> str:
+        """Return a newline-separated list of project-relative paths."""
+        lines: list[str] = []
+        for root, dirs, files in os.walk(self.config.path):
+            # Filter out excluded directories in-place so os.walk skips them
+            dirs[:] = [
+                d for d in dirs
+                if not self._is_excluded(self._rel(Path(root) / d))
+            ]
+            for name in files:
+                full = Path(root) / name
+                if not self._is_excluded(self._rel(full)):
+                    lines.append(self._rel(full))
+
+        lines.sort()
+        return "\n".join(lines)
+
+    # ------------------------------------------------------------------
+    # Stage 2: files
+    # ------------------------------------------------------------------
+
+    ''' def _collect_files(self) -> list[Path]:
+        """Return a sorted list of all non-excluded files in the project."""
+        result: list[Path] = []
+        for root, dirs, files in os.walk(self.config.path):
+            dirs[:] = [
+                d for d in dirs
+                if not self._is_excluded(self._rel(Path(root) / d))
+            ]
+            for name in files:
+                full = Path(root) / name
+                if not self._is_excluded(self._rel(full)):
+                    result.append(full)
+        result.sort()
+        return result '''
+
+    def _collect_files(self) -> list[Path]:
+        """Return a sorted list of non-excluded source and configuration files."""
+        root = Path(self.config.path).resolve()
+        if not root.is_dir():
+            return []
+
+        result: list[Path] = []
+        for path in root.rglob("*"):
+            if path.is_file():
+                rel = self._rel(path)
+                if not self._is_excluded(rel):
+                    # Otetaan mukaan .py-tiedostot TAI juuritason tärkeät tiedostot (kuten README.md, pyproject.toml)
+                    if path.suffix.lower() == ".py" or path.parent == root:
+                        result.append(path)
+        result.sort()
+        return result
+
+    def _is_excluded(self, rel_path: str | Path) -> bool:
+        """Check whether a project-relative path should be skipped."""
+        try:
+            parts = Path(rel_path).parts
+            name = Path(rel_path).name
+        except Exception:
+            parts = Path(str(rel_path)).parts
+            name = Path(str(rel_path)).name
+
+        exclude = self.config.analysis.exclude
+
+        # Tarkistetaan osat tai päätetyyppiset metadatakansiot
+        for part in parts:
+            if part in exclude:
+                return True
+            if part.endswith((".egg-info", ".dist-info", "__pycache__", ".mypy_cache", ".pytest_cache")):
+                return True
+
+        # Editor backup files: end with ~ or .bak or .swp or .swo
+        for backup_suffix in ("~", ".bak", ".swp", ".swo"):
+            if name.endswith(backup_suffix):
+                return True
+
+        return False
+        
+
+    def _analyze_file(self, file_path: Path) -> None:
+        rel = self._rel(file_path)
+        logger.info(f"Analyzing file: {rel}")
+
+        content = self._read_text_file(file_path)
+        if content is None:
+            self.findings.errors.append({
+                "stage": "file",
+                "path": rel,
+                "error": "could not be read as UTF-8 text (likely binary)",
+            })
+            return
+
+        content = self._truncate(content, self.config.analysis.max_file_chars)
+
+        key = file_hash(file_path)
+        model = self.config.model.name
+
+        cached = self.cache.get_file(key, model=model)
+        if cached is not None:
+            logger.debug(f"File cache hit: {rel}")
+            self.findings.files[rel] = cached
+            return
+
+        context = self._file_context(rel)
+        user_prompt = prompts.FILE_ANALYSIS.format(
+            context=context,
+            path=rel,
+            content=content,
+        )
+
+        try:
+            response = self.client.chat(
+                system=prompts.SYSTEM_FILE,
+                user=user_prompt,
+            )
+            summary = response.text.strip()
+            self.findings.files[rel] = summary
+            self.cache.put_file(key, path=rel, model=model, summary=summary)
+        except LLMError as e:
+            logger.error(f"Error analyzing file {rel}: {e}")
+            self.findings.errors.append({
+                "stage": "file",
+                "path": rel,
+                "error": str(e),
+            })
+
+    def _read_project_manifest(self) -> str:
+        """
+        Read the project's manifest file for global context.
+
+        Manifest files describe dependencies, build tooling, and
+        project metadata. They are valuable context for every file
+        analysis and help the model avoid inventing typical files
+        that don't exist.
+        """
+        manifests = [
+            "pyproject.toml",
+            "package.json",
+            "Cargo.toml",
+            "go.mod",
+            "setup.py",
+            "setup.cfg",
+            "requirements.txt",
+            "Gemfile",
+            "pom.xml",
+        ]
+        for name in manifests:
+            path = self.config.path / name
+            if not path.exists() or not path.is_file():
+                continue
+            if self._is_excluded(name):
+                continue
+            content = self._read_text_file(path)
+            if content:
+                return f"### Project manifest: {name}\n{content[:3000]}\n"
+        return ""
+
+    def _file_context(self, rel: str) -> str:
+        """Return context string prepended to a file analysis prompt."""
+        parts: list[str] = []
+
+        # Project manifest — the strongest signal for tech stack.
+        manifest = self._read_project_manifest()
+        if manifest:
+            parts.append(manifest)
+
+        if self.findings.root_summary:
+            parts.append(
+                "Project overview:\n"
+                f"{self.findings.root_summary}\n"
+            )
+
+        # Sibling files already analyzed in the same directory
+        parent = str(Path(rel).parent)
+        siblings = [
+            p for p in self.findings.files
+            if str(Path(p).parent) == parent and p != rel
+        ]
+        if siblings:
+            listing = "\n".join(f"- {s}" for s in sorted(siblings))
+            parts.append(f"Other files in this directory:\n{listing}\n")
+
+        return "\n".join(parts)
+
+    # ------------------------------------------------------------------
+    # Stage 3: directories
+    # ------------------------------------------------------------------
+
+    def _collect_directories(self, files: list[Path]) -> list[Path]:
+        """Return directories that contain at least one analyzed file."""
+        dirs: set[Path] = set()
+        for f in files:
+            rel = self._rel(f)
+            if rel in self.findings.files:
+                dirs.add(f.parent)
+        return sorted(dirs)
+
+    def _analyze_directory(self, dir_path: Path) -> None:
+        """Summarize a directory based on the file summaries inside it."""
+        rel = self._rel(dir_path)
+        logger.info(f"Analyzing directory: {rel}")
+
+        # Gather summaries of files directly in this directory
+        parts: list[str] = []
+        for file_rel, summary in self.findings.files.items():
+            if str(Path(file_rel).parent) == rel:
+                parts.append(f"### {Path(file_rel).name}\n{summary}\n")
+
+        if not parts:
+            return  # no summaries, skip
+
+        user_prompt = prompts.DIRECTORY_SUMMARY.format(
+            path=rel if rel else ".",
+            file_summaries="\n".join(parts),
+        )
+
+        try:
+            response = self.client.chat(
+                system=prompts.SYSTEM_DIRECTORY,
+                user=user_prompt,
+            )
+            self.findings.directories[rel] = response.text.strip()
+        except LLMError as e:
+            logger.error(f"Error analyzing directory {rel}: {e}")
+            self.findings.errors.append({
+                "stage": "directory",
+                "path": rel,
+                "error": str(e),
+            })
+
+    # ------------------------------------------------------------------
+    # Stage 4: guide
+    # ------------------------------------------------------------------
+
+    def _generate_guide(self) -> None:
+        """Ask the LLM to compose the final developer guide."""
+        user_prompt = prompts.DEVELOPER_GUIDE.format(
+            context="",
+            root_summary=self.findings.root_summary or "(not available)",
+            directory_summaries=self._format_directory_summaries(),
+            file_summaries=self._format_file_summaries(),
+        )
+
+        try:
+            response = self.client.chat(
+                system=prompts.SYSTEM_GUIDE,
+                user=user_prompt,
+                max_tokens=self.config.model.max_tokens_guide,
+            )
+            self.findings.guide = response.text.strip()
+        except LLMError as e:
+            logger.error(f"Guide generation failed: {e}")
+            self.findings.errors.append({
+                "stage": "guide",
+                "path": "",
+                "error": str(e),
+            })
+            self.findings.guide = "(guide generation failed)"
+
+    def _format_directory_summaries(self) -> str:
+        if not self.findings.directories:
+            return "(no directories analyzed)"
+        parts = [
+            f"### {path or '.'}\n{summary}\n"
+            for path, summary in sorted(self.findings.directories.items())
+        ]
+        return "\n".join(parts)
+
+    def _format_file_summaries(self) -> str:
+        if not self.findings.files:
+            return "(no files analyzed)"
+        parts = [
+            f"### {path}\n{summary}\n"
+            for path, summary in sorted(self.findings.files.items())
+        ]
+        return "\n".join(parts)
+
+    # ------------------------------------------------------------------
+    # Helpers
+    # ------------------------------------------------------------------
+
+    def _rel(self, path: Path) -> str:
+        """Return path relative to the project root as a POSIX string."""
+        try:
+            return path.relative_to(self.config.path).as_posix()
+        except ValueError:
+            # Path is outside project root — shouldn't happen but be safe
+            return str(path)
+
+    def _is_excluded(self, rel_path: str) -> bool:
+        """Check whether a project-relative path should be skipped."""
+        parts = Path(rel_path).parts
+        name = Path(rel_path).name
+        suffix = Path(rel_path).suffix.lower()
+        exclude = self.config.analysis.exclude
+
+        # Directory/file name match (exact)
+        if any(part in exclude for part in parts):
+            return True
+
+        # Extension match
+        if suffix and suffix in exclude:
+            return True
+
+        # Editor backup files: end with ~ or .bak or .swp or .swo
+        for backup_suffix in ("~", ".bak", ".swp", ".swo"):
+            if name.endswith(backup_suffix):
+                return True
+
+        return False
+
+    def _read_text_file(self, path: Path) -> str | None:
+        """Read a file as UTF-8. Return None for binary or unreadable files."""
+        try:
+            with open(path, "r", encoding="utf-8") as f:
+                return f.read()
+        except UnicodeDecodeError:
+            return None
+        except OSError as e:
+            logger.warning(f"Cannot read {path}: {e}")
+            return None
+
+    @staticmethod
+    def _truncate(text: str, max_chars: int) -> str:
+        """Truncate long text with a clear marker."""
+        if len(text) <= max_chars:
+            return text
+        head = text[:max_chars]
+        return (
+            f"{head}\n\n"
+            f"[... truncated, original length {len(text)} characters ...]"
+        )