code-context-engine 0.4.0__py3-none-any.whl

context_engine/indexer/git_indexer.py

@@ -0,0 +1,136 @@
+"""Parse git log into searchable chunks."""
+import asyncio
+import logging
+import re
+import subprocess
+from pathlib import Path
+
+from context_engine.models import (
+    Chunk, ChunkType, GraphNode, GraphEdge, NodeType, EdgeType,
+)
+
+log = logging.getLogger(__name__)
+
+_SHA_RE = re.compile(r"^[0-9a-f]{40}$")
+
+# Delimiter placed at the START of each commit record so we can split cleanly.
+_RECORD_START = "---CCE_START---"
+
+
+async def index_commits(
+    project_dir: Path,
+    since_sha: str | None = None,
+    max_commits: int = 200,
+) -> tuple[list[Chunk], list[GraphNode], list[GraphEdge]]:
+    """Parse recent git history into searchable chunks."""
+    # Use two separate git calls:
+    # 1. git log --format=... to get commit metadata in order
+    # 2. git log --name-only to get changed files per commit
+    range_arg = f"{since_sha}..HEAD" if since_sha else f"-{max_commits}"
+
+    meta_result = await asyncio.to_thread(
+        subprocess.run,
+        ["git", "log", range_arg, "--format=%H%n%an%n%ai%n%s%n%b%x00"],
+        cwd=project_dir, capture_output=True, text=True, check=False,
+    )
+
+    if meta_result.returncode != 0:
+        log.debug("git log skipped: %s", meta_result.stderr.strip())
+        return [], [], []
+
+    files_result = await asyncio.to_thread(
+        subprocess.run,
+        ["git", "log", range_arg, "--name-only", "--format=%H"],
+        cwd=project_dir, capture_output=True, text=True, check=False,
+    )
+
+    changed_files_by_hash: dict[str, list[str]] = {}
+    if files_result.returncode == 0:
+        changed_files_by_hash = _parse_name_only(files_result.stdout)
+
+    return _parse_meta(meta_result.stdout, changed_files_by_hash)
+
+
+def _parse_name_only(output: str) -> dict[str, list[str]]:
+    """Parse `git log --name-only --format=%H` output into {hash: [files]}."""
+    result: dict[str, list[str]] = {}
+    current_hash: str | None = None
+    for line in output.splitlines():
+        stripped = line.strip()
+        if not stripped:
+            continue
+        if _SHA_RE.match(stripped):
+            current_hash = stripped
+            result.setdefault(current_hash, [])
+        elif current_hash is not None:
+            result[current_hash].append(stripped)
+    return result
+
+
+def _parse_meta(
+    output: str,
+    changed_files_by_hash: dict[str, list[str]],
+) -> tuple[list[Chunk], list[GraphNode], list[GraphEdge]]:
+    """Parse commit metadata output and build chunks/nodes/edges."""
+    chunks: list[Chunk] = []
+    nodes: list[GraphNode] = []
+    edges: list[GraphEdge] = []
+
+    # Records are separated by NUL bytes (\x00)
+    records = output.split("\x00")
+    for record in records:
+        record = record.strip()
+        if not record:
+            continue
+
+        lines = record.splitlines()
+        if len(lines) < 4:
+            continue
+
+        commit_hash = lines[0].strip()
+        if not _SHA_RE.match(commit_hash):
+            continue
+
+        author = lines[1].strip()
+        date = lines[2].strip()
+        subject = lines[3].strip()
+        body = "\n".join(lines[4:]).strip()
+
+        content = f"{subject}\n\n{body}".strip()
+        short_hash = commit_hash[:7]
+
+        chunk = Chunk(
+            id=f"commit_{short_hash}",
+            content=content,
+            chunk_type=ChunkType.COMMIT,
+            file_path=f"git:{short_hash}",
+            start_line=0,
+            end_line=0,
+            language="git",
+            metadata={
+                "author": author,
+                "date": date,
+                "hash": commit_hash,
+                "chunk_kind": "commit",
+            },
+        )
+        chunks.append(chunk)
+
+        node = GraphNode(
+            id=f"commit_{short_hash}",
+            node_type=NodeType.COMMIT,
+            name=subject,
+            file_path=f"git:{short_hash}",
+        )
+        nodes.append(node)
+
+        for fname in changed_files_by_hash.get(commit_hash, []):
+            edges.append(
+                GraphEdge(
+                    source_id=f"commit_{short_hash}",
+                    target_id=f"file_{fname}",
+                    edge_type=EdgeType.MODIFIES,
+                )
+            )
+
+    return chunks, nodes, edges

context_engine/indexer/ignorefile.py

@@ -0,0 +1,96 @@
+"""`.cceignore` parser — gitignore-style patterns for the indexer.
+
+Supports the practical subset of `.gitignore` syntax that covers ~95% of
+real-world use:
+
+  · Glob patterns: `*.log`, `temp/*`, `**/build/`
+  · Directory matches: `node_modules/` (trailing slash)
+  · Comments: lines starting with `#`
+  · Blank lines: ignored
+
+Deliberate deviation from strict gitignore: `*` here matches across
+path separators (fnmatch behaviour), so `temp/*` excludes everything
+under `temp/`, not just direct children. In our experience that's what
+users actually want from an indexer ignore file.
+
+NOT supported (intentionally — adds dependency and complexity for
+diminishing returns):
+
+  · Negation patterns (`!keep.log`)
+  · Anchored patterns (leading `/`) — all patterns match anywhere in the tree
+  · Character classes beyond what `fnmatch` provides
+
+Users who need full gitignore semantics can add `pathspec` to their
+project and wire a custom matcher; this module covers the common case
+without a third-party dependency.
+"""
+from __future__ import annotations
+
+import fnmatch
+from pathlib import Path
+
+CCEIGNORE_FILENAME = ".cceignore"
+
+
+def load_ignore_patterns(project_dir: Path) -> list[str]:
+    """Read `.cceignore` from `project_dir` and return its non-comment,
+    non-blank lines. Returns an empty list if the file doesn't exist.
+
+    Patterns are returned verbatim (whitespace stripped); matching is
+    delegated to `matches_any`.
+    """
+    path = project_dir / CCEIGNORE_FILENAME
+    if not path.is_file():
+        return []
+    try:
+        raw = path.read_text(encoding="utf-8", errors="strict")
+    except OSError:
+        return []
+    out: list[str] = []
+    for line in raw.splitlines():
+        s = line.strip()
+        if not s or s.startswith("#"):
+            continue
+        out.append(s)
+    return out
+
+
+def matches_any(rel_path: str, is_dir: bool, patterns: list[str]) -> bool:
+    """True if `rel_path` matches any of the given patterns.
+
+    `rel_path` is the path relative to the project root, using forward
+    slashes regardless of platform. `is_dir` distinguishes directories
+    so trailing-slash patterns (e.g. `build/`) only match directories.
+    """
+    if not patterns:
+        return False
+    # Normalise: forward slashes, no leading "./"
+    rel = rel_path.replace("\\", "/").lstrip("./")
+    name = rel.rsplit("/", 1)[-1]
+    for pat in patterns:
+        # Trailing slash → directory-only pattern.
+        is_dir_pat = pat.endswith("/")
+        p = pat[:-1] if is_dir_pat else pat
+        if is_dir_pat and not is_dir:
+            continue
+        # Pattern with no slash → match against basename anywhere in tree.
+        # Pattern with a slash → match against the relative path from root.
+        if "/" not in p:
+            if fnmatch.fnmatchcase(name, p):
+                return True
+        else:
+            # Strip a leading `/` if user wrote it (anchored), our matcher
+            # is implicitly anchored against the project root anyway.
+            anchored = p.lstrip("/")
+            if fnmatch.fnmatchcase(rel, anchored):
+                return True
+            # `**` support — fnmatch treats it as `*`. We extend by also
+            # trying the pattern with `**/` stripped from the front, so
+            # `**/build/foo` matches `build/foo` and `src/build/foo`.
+            if anchored.startswith("**/"):
+                tail = anchored[3:]
+                if fnmatch.fnmatchcase(rel, tail):
+                    return True
+                if fnmatch.fnmatchcase(rel, f"*/{tail}"):
+                    return True
+    return False

context_engine/indexer/manifest.py

@@ -0,0 +1,78 @@
+"""Content hash manifest for incremental indexing."""
+import json
+import logging
+from pathlib import Path
+
+from context_engine.utils import atomic_write_text
+
+log = logging.getLogger(__name__)
+
+CURRENT_SCHEMA_VERSION = 2
+
+
+class Manifest:
+    def __init__(self, manifest_path: Path) -> None:
+        self._path = manifest_path
+        self._entries: dict[str, str] = {}
+        self._schema_version: int = CURRENT_SCHEMA_VERSION
+        self._last_git_sha: str | None = None
+
+        if self._path.exists():
+            try:
+                with open(self._path) as f:
+                    loaded = json.load(f)
+                if isinstance(loaded, dict):
+                    if "__schema_version" in loaded:
+                        # New versioned format
+                        self._schema_version = loaded["__schema_version"]
+                        self._entries = loaded.get("files", {})
+                        self._last_git_sha = loaded.get("last_git_sha")
+                    else:
+                        # Old plain-dict format (pre-v0.2) — treat as version 1
+                        self._schema_version = 1
+                        self._entries = loaded
+                else:
+                    log.warning(
+                        "Manifest at %s was not a dict (got %s); starting empty.",
+                        self._path,
+                        type(loaded).__name__,
+                    )
+            except (json.JSONDecodeError, OSError) as exc:
+                log.warning("Manifest at %s unreadable (%s); starting empty.", self._path, exc)
+                self._entries = {}
+
+    @property
+    def schema_version(self) -> int:
+        return self._schema_version
+
+    @property
+    def needs_reindex(self) -> bool:
+        return self._schema_version != CURRENT_SCHEMA_VERSION
+
+    @property
+    def last_git_sha(self) -> str | None:
+        return self._last_git_sha
+
+    @last_git_sha.setter
+    def last_git_sha(self, value: str | None) -> None:
+        self._last_git_sha = value
+
+    def get_hash(self, file_path: str) -> str | None:
+        return self._entries.get(file_path)
+
+    def update(self, file_path: str, content_hash: str) -> None:
+        self._entries[file_path] = content_hash
+
+    def remove(self, file_path: str) -> None:
+        self._entries.pop(file_path, None)
+
+    def has_changed(self, file_path: str, content_hash: str) -> bool:
+        return self._entries.get(file_path) != content_hash
+
+    def save(self) -> None:
+        payload = {
+            "__schema_version": CURRENT_SCHEMA_VERSION,
+            "files": self._entries,
+            "last_git_sha": self._last_git_sha,
+        }
+        atomic_write_text(self._path, json.dumps(payload))