siftcode 0.3.0__py3-none-any.whl

siftcode/core/cache.py

@@ -0,0 +1,300 @@
+"""
+Hash-based cache for Siftcode analysis results.
+
+Caches LLM responses keyed by file content hash and model name.
+This means:
+- Unchanged files are never re-analyzed
+- Switching models invalidates the cache (different model = different result)
+- Cache entries never expire on their own — run `siftcode cache gc` to prune
+
+No GTK dependencies.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+import time
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from typing import Any
+
+from platformdirs import user_cache_dir
+
+from siftcode.core.config import APP_AUTHOR, APP_NAME
+
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Default location
+# ---------------------------------------------------------------------------
+
+CACHE_DIR = Path(user_cache_dir(APP_NAME, APP_AUTHOR))
+FILES_CACHE_DIR = CACHE_DIR / "files"
+DIRS_CACHE_DIR = CACHE_DIR / "dirs"
+ROOT_CACHE_DIR = CACHE_DIR / "roots"
+
+
+# ---------------------------------------------------------------------------
+# Hash helpers
+# ---------------------------------------------------------------------------
+
+def file_hash(path: Path) -> str:
+    """
+    Compute a stable SHA-256 hash of a file's content.
+
+    Reads in 8 KB chunks so large files don't blow up memory.
+    Returns the first 16 hex characters — collision-free in practice.
+    """
+    h = hashlib.sha256()
+    with open(path, "rb") as f:
+        for chunk in iter(lambda: f.read(8192), b""):
+            h.update(chunk)
+    return h.hexdigest()[:16]
+
+
+def string_hash(text: str) -> str:
+    """Hash a string (used for directory and root keys)."""
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()[:16]
+
+
+# ---------------------------------------------------------------------------
+# Cache entry
+# ---------------------------------------------------------------------------
+
+@dataclass
+class CacheEntry:
+    """A single cached analysis result."""
+
+    key: str            # hash of the input (file content, dir listing, etc.)
+    path: str           # project-relative path or label
+    model: str          # model name — cache is model-specific
+    summary: str        # the LLM's response
+    analyzed_at: str    # ISO timestamp
+    last_used: str      # ISO timestamp — updated on every hit
+
+    @property
+    def age_days(self) -> float:
+        """Days since this entry was last used."""
+        last = time.mktime(time.strptime(self.last_used[:19], "%Y-%m-%dT%H:%M:%S"))
+        return (time.time() - last) / 86400.0
+
+
+# ---------------------------------------------------------------------------
+# Cache
+# ---------------------------------------------------------------------------
+
+class AnalysisCache:
+    """
+    File-backed cache for LLM analysis results.
+
+    Layout:
+        ~/.cache/siftcode/files/<hash>.json
+        ~/.cache/siftcode/dirs/<hash>.json
+        ~/.cache/siftcode/roots/<hash>.json
+    """
+
+    def __init__(
+        self,
+        base_dir: Path | None = None,
+        enabled: bool = True,
+    ) -> None:
+        self.base_dir = base_dir or CACHE_DIR
+        self.enabled = enabled
+
+        self.files_dir = self.base_dir / "files"
+        self.dirs_dir = self.base_dir / "dirs"
+        self.roots_dir = self.base_dir / "roots"
+
+        for d in (self.files_dir, self.dirs_dir, self.roots_dir):
+            d.mkdir(parents=True, exist_ok=True)
+
+    # ------------------------------------------------------------------
+    # Public API — file-level
+    # ------------------------------------------------------------------
+
+    def get_file(self, key: str, model: str) -> str | None:
+        """Return cached file summary, or None on miss."""
+        return self._get(self.files_dir, key, model)
+
+    def put_file(self, key: str, path: str, model: str, summary: str) -> None:
+        """Store a file summary in the cache."""
+        self._put(self.files_dir, key, path, model, summary)
+
+    # ------------------------------------------------------------------
+    # Public API — directory-level
+    # ------------------------------------------------------------------
+
+    def get_directory(self, key: str, model: str) -> str | None:
+        return self._get(self.dirs_dir, key, model)
+
+    def put_directory(self, key: str, path: str, model: str, summary: str) -> None:
+        self._put(self.dirs_dir, key, path, model, summary)
+
+    # ------------------------------------------------------------------
+    # Public API — root-level
+    # ------------------------------------------------------------------
+
+    def get_root(self, key: str, model: str) -> str | None:
+        return self._get(self.roots_dir, key, model)
+
+    def put_root(self, key: str, path: str, model: str, summary: str) -> None:
+        self._put(self.roots_dir, key, path, model, summary)
+
+    # ------------------------------------------------------------------
+    # Public API — stats / maintenance
+    # ------------------------------------------------------------------
+
+    def stats(self) -> dict[str, Any]:
+        """Return counts and total size per cache category."""
+        result: dict[str, Any] = {}
+        total_entries = 0
+        total_bytes = 0
+
+        for name, d in (
+            ("files", self.files_dir),
+            ("directories", self.dirs_dir),
+            ("roots", self.roots_dir),
+        ):
+            count = 0
+            size = 0
+            for f in d.glob("*.json"):
+                count += 1
+                size += f.stat().st_size
+            result[name] = {"entries": count, "bytes": size}
+            total_entries += count
+            total_bytes += size
+
+        result["total"] = {
+            "entries": total_entries,
+            "bytes": total_bytes,
+            "human_size": _human_size(total_bytes),
+        }
+        result["base_dir"] = str(self.base_dir)
+        return result
+
+    def clear(self, model: str | None = None) -> int:
+        """
+        Remove cache entries.
+
+        If model is None, clears everything. Otherwise only entries for
+        that model. Returns the number of entries removed.
+        """
+        removed = 0
+        for d in (self.files_dir, self.dirs_dir, self.roots_dir):
+            for f in list(d.glob("*.json")):
+                if model is None:
+                    f.unlink()
+                    removed += 1
+                else:
+                    try:
+                        data = json.loads(f.read_text())
+                        if data.get("model") == model:
+                            f.unlink()
+                            removed += 1
+                    except (OSError, json.JSONDecodeError):
+                        # Corrupt file — remove it anyway
+                        f.unlink()
+                        removed += 1
+        return removed
+
+    def gc(self, max_age_days: float = 30.0) -> int:
+        """
+        Delete cache entries not used within max_age_days.
+
+        Returns the number of entries removed.
+        """
+        removed = 0
+        for d in (self.files_dir, self.dirs_dir, self.roots_dir):
+            for f in d.glob("*.json"):
+                try:
+                    data = json.loads(f.read_text())
+                    entry = CacheEntry(**data)
+                    if entry.age_days > max_age_days:
+                        f.unlink()
+                        removed += 1
+                except (OSError, json.JSONDecodeError, TypeError, ValueError):
+                    # Corrupt or missing fields — treat as garbage
+                    f.unlink()
+                    removed += 1
+        return removed
+
+    # ------------------------------------------------------------------
+    # Internals
+    # ------------------------------------------------------------------
+
+    def _get(self, directory: Path, key: str, model: str) -> str | None:
+        if not self.enabled:
+            return None
+
+        path = directory / f"{key}.json"
+        if not path.exists():
+            return None
+
+        try:
+            data = json.loads(path.read_text())
+        except (OSError, json.JSONDecodeError) as e:
+            logger.warning(f"Corrupt cache entry {path}: {e}")
+            return None
+
+        # Model must match
+        if data.get("model") != model:
+            return None
+
+        # Update last_used timestamp (fire-and-forget)
+        try:
+            data["last_used"] = _now_iso()
+            path.write_text(json.dumps(data, indent=2, ensure_ascii=False))
+        except OSError:
+            pass
+
+        return data.get("summary")
+
+    def _put(
+        self,
+        directory: Path,
+        key: str,
+        path_str: str,
+        model: str,
+        summary: str,
+    ) -> None:
+        if not self.enabled:
+            return
+
+        entry = CacheEntry(
+            key=key,
+            path=path_str,
+            model=model,
+            summary=summary,
+            analyzed_at=_now_iso(),
+            last_used=_now_iso(),
+        )
+
+        target = directory / f"{key}.json"
+        try:
+            target.write_text(
+                json.dumps(asdict(entry), indent=2, ensure_ascii=False),
+                encoding="utf-8",
+            )
+        except OSError as e:
+            logger.warning(f"Cannot write cache entry {target}: {e}")
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+def _now_iso() -> str:
+    from datetime import datetime
+    return datetime.now().isoformat(timespec="seconds")
+
+
+def _human_size(num_bytes: int) -> str:
+    for unit in ("B", "KB", "MB", "GB"):
+        if num_bytes < 1024:
+            return f"{num_bytes:.1f} {unit}"
+        num_bytes /= 1024
+    return f"{num_bytes:.1f} TB"

siftcode/core/config.py

@@ -0,0 +1,258 @@
+"""
+Configuration management for Siftcode.
+
+Handles loading, saving and validating project configuration files.
+Uses XDG base directory specification via platformdirs.
+"""
+
+from __future__ import annotations
+
+import re
+import tomllib
+from dataclasses import dataclass, field, asdict
+from pathlib import Path
+from typing import Any
+
+import tomli_w
+from platformdirs import user_config_dir, user_cache_dir
+
+
+# ---------------------------------------------------------------------------
+# XDG-compliant directories
+# ---------------------------------------------------------------------------
+
+APP_NAME = "siftcode"
+APP_AUTHOR = "siftcode"
+
+CONFIG_DIR = Path(user_config_dir(APP_NAME, APP_AUTHOR))
+PROJECTS_DIR = CONFIG_DIR / "projects"
+CACHE_DIR = Path(user_cache_dir(APP_NAME, APP_AUTHOR))
+
+
+# ---------------------------------------------------------------------------
+# Default exclusion list — used if user hasn't customized it
+# ---------------------------------------------------------------------------
+
+DEFAULT_EXCLUSIONS = [
+    # Version control / environment
+    ".git", ".venv", "venv", "env", "node_modules", "__pycache__",
+    # Version control metadata (not source code)
+    ".gitignore", ".gitattributes", ".gitmodules", ".gitkeep",
+    # Build artifacts
+    "build", "dist", ".eggs", ".tox", ".pytest_cache", ".mypy_cache",
+    ".ruff_cache", ".coverage", ".htmlcov", "htmlcov",
+    # Documentation
+    "docs", "doc", "documentation", "README.md~", "docs/specs/", "doc/specs/",
+    # Editor backup files
+    "*~", "*.bak", "*.swp", "*.swo",
+    # Binary / media files
+    ".png", ".jpg", ".jpeg", ".gif", ".svg", ".ico", ".webp",
+    ".pdf", ".zip", ".tar", ".gz", ".whl", ".so", ".dll", ".exe",
+    ".ttf", ".otf", ".woff", ".woff2",
+    ".pyc", ".pyo",
+    # Lock files
+    ".lock",
+    # Coverage data
+    ".coverage", ".coverage.*",
+    # Editor / IDE configs (low architectural value)
+    ".editorconfig", ".pre-commit-config.yaml",
+    ".env", ".env.example", ".envrc",
+    # OS noise
+    ".DS_Store", "Thumbs.db",
+    # Common metadata files that are just lists
+    "MANIFEST.in", ".dockerignore", ".npmignore",
+]
+
+
+# ---------------------------------------------------------------------------
+# Dataclasses
+# ---------------------------------------------------------------------------
+
+@dataclass
+class ModelConfig:
+    """LLM settings for a single analysis run."""
+
+    name: str = "qwen2.5-coder:14b"
+    ollama_url: str = "http://localhost:11434"
+    num_ctx: int = 32768        # ← 8192 → 16384 → 32768
+    temperature: float = 0.0
+    timeout: int = 1200
+    max_tokens_file: int = 1000
+    max_tokens_guide: int = 8000
+
+    def validate(self) -> None:
+        """Raise ValueError if any field is out of acceptable range."""
+        if self.num_ctx < 512 or self.num_ctx > 131072:
+            raise ValueError(
+                f"num_ctx must be between 512 and 131072, got {self.num_ctx}"
+            )
+        if not 0.0 <= self.temperature <= 2.0:
+            raise ValueError(
+                f"temperature must be between 0.0 and 2.0, got {self.temperature}"
+            )
+        if self.timeout < 1:
+            raise ValueError(f"timeout must be positive, got {self.timeout}")
+        if not self.ollama_url.startswith(("http://", "https://")):
+            raise ValueError(
+                f"ollama_url must start with http:// or https://, "
+                f"got {self.ollama_url}"
+            )
+
+
+@dataclass
+class AnalysisConfig:
+    """Settings controlling what gets analyzed and how."""
+
+    exclude: list[str] = field(default_factory=lambda: list(DEFAULT_EXCLUSIONS))
+    priority_files: list[str] = field(default_factory=list)
+    use_cache: bool = True
+    max_file_chars: int = 30000
+
+    def validate(self) -> None:
+        if self.max_file_chars < 1000:
+            raise ValueError(
+                f"max_file_chars must be at least 1000, "
+                f"got {self.max_file_chars}"
+            )
+
+
+@dataclass
+class ProjectConfig:
+    """Complete configuration for a single project."""
+
+    name: str
+    path: Path
+    model: ModelConfig = field(default_factory=ModelConfig)
+    analysis: AnalysisConfig = field(default_factory=AnalysisConfig)
+    created: str = ""
+    last_run: str = ""
+    notes: str = ""
+
+    def validate(self) -> None:
+        """Validate project config and all nested configs."""
+        if not self.name:
+            raise ValueError("Project name cannot be empty")
+        if not self.path:
+            raise ValueError("Project path cannot be empty")
+        self.model.validate()
+        self.analysis.validate()
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to a dictionary suitable for TOML serialization."""
+        data = asdict(self)
+        # Path → str for TOML
+        data["path"] = str(self.path)
+        return data
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> ProjectConfig:
+        """Build a ProjectConfig from a dictionary (loaded from TOML)."""
+        data = dict(data)  # copy — do not mutate caller's data
+
+        model_data = data.pop("model", {})
+        analysis_data = data.pop("analysis", {})
+
+        return cls(
+            name=data["name"],
+            path=Path(data["path"]),
+            model=ModelConfig(**model_data),
+            analysis=AnalysisConfig(**analysis_data),
+            created=data.get("created", ""),
+            last_run=data.get("last_run", ""),
+            notes=data.get("notes", ""),
+        )
+
+
+# ---------------------------------------------------------------------------
+# Slug helper
+# ---------------------------------------------------------------------------
+
+def project_slug(name: str) -> str:
+    """
+    Convert a project name to a safe filename slug.
+
+    >>> project_slug("My Project")
+    'my-project'
+    >>> project_slug("SHL Policy Editor v2")
+    'shl-policy-editor-v2'
+    """
+    slug = name.lower().strip()
+    slug = re.sub(r"[^\w\s-]", "", slug)
+    slug = re.sub(r"[-\s]+", "-", slug)
+    return slug.strip("-") or "unnamed"
+
+
+# ---------------------------------------------------------------------------
+# Load / Save
+# ---------------------------------------------------------------------------
+
+def project_path(name: str, projects_dir: Path | None = None) -> Path:
+    """Return the TOML file path for a given project name."""
+    base = projects_dir if projects_dir is not None else PROJECTS_DIR
+    return base / f"{project_slug(name)}.toml"
+
+
+def save_project(
+    config: ProjectConfig,
+    projects_dir: Path | None = None,
+) -> Path:
+    """
+    Save a project configuration to disk.
+
+    Returns the path where it was written.
+    """
+    config.validate()
+    path = project_path(config.name, projects_dir)
+    path.parent.mkdir(parents=True, exist_ok=True)
+
+    with open(path, "wb") as f:
+        tomli_w.dump(config.to_dict(), f)
+
+    return path
+
+
+def load_project(
+    name: str,
+    projects_dir: Path | None = None,
+) -> ProjectConfig:
+    """Load a project configuration by name."""
+    path = project_path(name, projects_dir)
+    if not path.exists():
+        raise FileNotFoundError(f"Project '{name}' not found at {path}")
+
+    with open(path, "rb") as f:
+        data = tomllib.load(f)
+
+    return ProjectConfig.from_dict(data)
+
+
+def list_projects(projects_dir: Path | None = None) -> list[ProjectConfig]:
+    """List all saved projects, sorted by last run (newest first)."""
+    base = projects_dir if projects_dir is not None else PROJECTS_DIR
+    if not base.exists():
+        return []
+
+    projects: list[ProjectConfig] = []
+    for toml_file in sorted(base.glob("*.toml")):
+        try:
+            with open(toml_file, "rb") as f:
+                data = tomllib.load(f)
+            projects.append(ProjectConfig.from_dict(data))
+        except (OSError, tomllib.TOMLDecodeError, KeyError, ValueError):
+            # Skip malformed files silently — they don't break the listing
+            continue
+
+    projects.sort(key=lambda p: p.last_run or p.created or "", reverse=True)
+    return projects
+
+
+def delete_project(
+    name: str,
+    projects_dir: Path | None = None,
+) -> bool:
+    """Delete a project configuration. Returns True if deleted."""
+    path = project_path(name, projects_dir)
+    if path.exists():
+        path.unlink()
+        return True
+    return False