attocode-code-intel 0.1.0__py3-none-any.whl

attocode_intel/indexing/incremental.py

@@ -0,0 +1,240 @@
+"""Content-hash-gated incremental update pipeline.
+
+This is the core pipeline that processes file changes from hook notifications.
+It reads file content, computes SHA-256, compares to the current branch manifest,
+and only re-parses/re-indexes files whose content has actually changed.
+
+Flow:
+    paths → read content → hash → compare to manifest → skip unchanged
+    → store new content → update overlay → parse symbols → queue embeddings
+    → publish event
+"""
+
+from __future__ import annotations
+
+import logging
+import uuid
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from attocode_intel.indexing.parser import detect_language, extract_imports, extract_symbols
+
+if TYPE_CHECKING:
+    from sqlalchemy.ext.asyncio import AsyncSession
+
+logger = logging.getLogger(__name__)
+
+_MAX_FILE_SIZE = 1_000_000  # 1MB
+
+
+class IncrementalPipeline:
+    """Content-hash-gated incremental indexing pipeline.
+
+    Given a list of changed file paths and a branch, this pipeline:
+    1. Reads each file from disk (local mode) or git (service mode)
+    2. Computes SHA-256 of the content
+    3. Compares to current manifest — skips if SHA matches (no-op)
+    4. Stores new content in content-addressed store
+    5. Updates branch overlay (path → new SHA)
+    6. Extracts symbols (skips if SHA already has symbols — dedup)
+    7. Queues embedding generation for new content
+    8. Returns stats about what changed
+    """
+
+    def __init__(self, session: AsyncSession) -> None:
+        from attocode_intel.storage.branch_overlay import BranchOverlay
+        from attocode_intel.storage.content_store import ContentStore
+        from attocode_intel.storage.embedding_store import EmbeddingStore
+        from attocode_intel.storage.symbol_store import SymbolStore
+
+        self._session = session
+        from attocode_intel.storage.dependency_store import DependencyStore
+
+        self._content_store = ContentStore(session)
+        self._symbol_store = SymbolStore(session)
+        self._embedding_store = EmbeddingStore(session)
+        self._overlay = BranchOverlay(session)
+        self._dep_store = DependencyStore(session)
+
+    @staticmethod
+    async def acquire_branch_lock(session: AsyncSession, branch_id: uuid.UUID) -> None:
+        """Acquire a PostgreSQL advisory lock for a branch to prevent concurrent indexing."""
+        from sqlalchemy import text
+
+        lock_key = int.from_bytes(branch_id.bytes[:8], "big") & 0x7FFFFFFFFFFFFFFF
+        await session.execute(
+            text("SELECT pg_advisory_xact_lock(:key)").bindparams(key=lock_key)
+        )
+
+    async def process_file_changes(
+        self,
+        branch_id: uuid.UUID,
+        paths: list[str],
+        base_dir: str | None = None,
+        git_reader: object | None = None,
+        repo_id: str | None = None,
+        ref: str | None = None,
+        file_contents: dict[str, bytes] | None = None,
+    ) -> dict:
+        """Process a batch of file changes for a branch.
+
+        Content sources (checked in order):
+        1. file_contents — inline content sent by the client (remote mode)
+        2. base_dir — read from local filesystem
+        3. git_reader+repo_id+ref — read from git
+
+        Returns stats: {processed, skipped_unchanged, skipped_missing,
+                        skipped_too_large, symbols_updated, errors}
+        """
+        stats = {
+            "processed": 0,
+            "skipped_unchanged": 0,
+            "skipped_missing": 0,
+            "skipped_too_large": 0,
+            "skipped_binary": 0,
+            "symbols_updated": 0,
+            "errors": 0,
+        }
+
+        # Acquire advisory lock to prevent concurrent indexing on same branch
+        try:
+            await self.acquire_branch_lock(self._session, branch_id)
+        except Exception:
+            pass  # Non-Postgres backends (e.g. tests) — skip locking
+
+        # Get current manifest for comparison
+        manifest = await self._overlay.resolve_manifest(branch_id)
+
+        # Batch: collect files to update
+        overlay_updates: list[tuple[str, str, str]] = []  # (path, sha, status)
+
+        for path in paths:
+            try:
+                content = await self._read_file(path, base_dir, git_reader, repo_id, ref, file_contents=file_contents)
+
+                if content is None:
+                    # File was deleted
+                    if path in manifest:
+                        await self._overlay.delete_file(branch_id, path)
+                        stats["processed"] += 1
+                    else:
+                        stats["skipped_missing"] += 1
+                    continue
+
+                if len(content) > _MAX_FILE_SIZE:
+                    stats["skipped_too_large"] += 1
+                    continue
+
+                # Content-hash comparison — the key optimization
+                sha = self._content_store.hash_content(content)
+                current_sha = manifest.get(path)
+
+                if sha == current_sha:
+                    stats["skipped_unchanged"] += 1
+                    continue
+
+                # New or changed content
+                language = detect_language(path)
+                await self._content_store.store(content, language)
+
+                # Determine status
+                status = "added" if path not in manifest else "modified"
+                overlay_updates.append((path, sha, status))
+
+                # Extract symbols (skip if SHA already indexed — content dedup)
+                symbols = extract_symbols(content, path)
+                if symbols:
+                    count = await self._symbol_store.upsert_symbols(
+                        sha, symbols, skip_if_exists=True
+                    )
+                    stats["symbols_updated"] += count or len(symbols)
+
+                # Extract imports and resolve to dependencies
+                imports = extract_imports(content, path, language)
+                if imports and not await self._dep_store.has_dependencies(sha):
+                    from attocode_intel.indexing.full_indexer import _resolve_import_path
+
+                    known_paths = set(manifest.keys()) | {u[0] for u in overlay_updates}
+                    path_to_sha = dict(manifest)
+                    for u_path, u_sha, _ in overlay_updates:
+                        path_to_sha[u_path] = u_sha
+
+                    deps = []
+                    for imp in imports:
+                        target = _resolve_import_path(imp, path, known_paths, language)
+                        if target and target in path_to_sha:
+                            deps.append({
+                                "target_sha": path_to_sha[target],
+                                "dep_type": "import",
+                                "weight": 1.0,
+                            })
+                    if deps:
+                        await self._dep_store.upsert_dependencies(sha, deps)
+                        stats["dependencies_extracted"] = stats.get("dependencies_extracted", 0) + len(deps)
+
+                # Queue embedding generation (dedup: skip if already embedded)
+                if not await self._embedding_store.has_embeddings(sha):
+                    if repo_id:
+                        from attocode_intel.workers.job_utils import enqueue_embedding_job
+                        await enqueue_embedding_job(repo_id, ref or "main")
+
+                stats["processed"] += 1
+
+            except Exception as e:
+                logger.warning("Error processing %s: %s", path, e)
+                stats["errors"] += 1
+
+        # Batch update overlay (single version bump)
+        if overlay_updates:
+            await self._overlay.set_files_batch(branch_id, overlay_updates)
+
+        return stats
+
+    async def _read_file(
+        self,
+        path: str,
+        base_dir: str | None,
+        git_reader: object | None,
+        repo_id: str | None,
+        ref: str | None,
+        file_contents: dict[str, bytes] | None = None,
+    ) -> bytes | None:
+        """Read file content from inline content, disk, or git.
+
+        Content sources (checked in order):
+        1. file_contents — inline content sent by the client
+        2. base_dir — local filesystem
+        3. git_reader — git object reader
+
+        Returns None if file doesn't exist (deleted) or no content source available.
+        C1 fix: Validates resolved path is within base_dir to prevent path traversal.
+        """
+        # Priority 1: inline content (sent by client for remote servers)
+        if file_contents is not None and path in file_contents:
+            return file_contents[path]
+
+        # Priority 2: local filesystem
+        if base_dir:
+            base = Path(base_dir).resolve()
+            full_path = (base / path).resolve()
+            # C1: prevent path traversal (e.g. ../../etc/passwd)
+            if not full_path.is_relative_to(base):
+                raise ValueError(
+                    f"Path traversal detected: '{path}' resolves outside base directory"
+                )
+            if not full_path.is_file():
+                return None
+            try:
+                return full_path.read_bytes()
+            except (OSError, PermissionError):
+                return None
+
+        # Priority 3: git reader
+        if git_reader and repo_id and ref:
+            try:
+                return git_reader.read_file(repo_id, ref, path)  # type: ignore[union-attr]
+            except Exception:
+                return None
+
+        # No content source — file is missing or deleted
+        return None

attocode_intel/indexing/parser.py

@@ -0,0 +1,403 @@
+"""Parser bridge — connects indexing pipeline with existing AST/CodeAnalyzer."""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    pass
+
+logger = logging.getLogger(__name__)
+
+# Language detection by extension
+_EXT_TO_LANG: dict[str, str] = {
+    ".py": "python",
+    ".js": "javascript",
+    ".ts": "typescript",
+    ".tsx": "typescript",
+    ".jsx": "javascript",
+    ".go": "go",
+    ".rs": "rust",
+    ".java": "java",
+    ".rb": "ruby",
+    ".c": "c",
+    ".cpp": "cpp",
+    ".h": "c",
+    ".hpp": "cpp",
+    ".metal": "cpp",
+    ".cs": "csharp",
+    ".php": "php",
+    ".swift": "swift",
+    ".kt": "kotlin",
+    ".scala": "scala",
+    ".lua": "lua",
+    ".ex": "elixir",
+    ".exs": "elixir",
+    ".sh": "bash",
+    ".bash": "bash",
+    ".zig": "zig",
+    ".nim": "nim",
+    ".fs": "fsharp",
+    ".fsi": "fsharp",
+    ".cr": "crystal",
+    ".m": "objc",
+    ".mm": "objc",
+    ".yaml": "yaml",
+    ".yml": "yaml",
+    ".toml": "toml",
+    ".json": "json",
+    ".html": "html",
+    ".css": "css",
+}
+
+
+def detect_language(path: str) -> str | None:
+    """Detect programming language from file extension."""
+    ext = Path(path).suffix.lower()
+    return _EXT_TO_LANG.get(ext)
+
+
+def parse_content(content_sha: str, content: bytes, filename: str) -> dict:
+    """Parse file content and return structured result keyed by content SHA.
+
+    Bridge function for the content-hash-gated incremental pipeline.
+    Returns: {content_sha, language, symbols, imports}
+    """
+    language = detect_language(filename)
+    symbols = extract_symbols(content, filename)
+    imports = extract_imports(content, filename, language)
+
+    return {
+        "content_sha": content_sha,
+        "language": language,
+        "symbols": symbols,
+        "imports": imports,
+    }
+
+
+def extract_imports(content: bytes, path: str, language: str | None) -> list[str]:
+    """Extract import statements from file content. Returns list of module names."""
+    import re
+
+    if language is None:
+        return []
+
+    try:
+        text = content.decode("utf-8", errors="replace")
+    except Exception:
+        return []
+
+    imports = []
+    if language == "python":
+        for m in re.finditer(r"^from\s+([\w.]+)\s+import\s+", text, re.MULTILINE):
+            imports.append(m.group(1))
+        for m in re.finditer(r"^import\s+([\w.]+(?:\s*,\s*[\w.]+)*)", text, re.MULTILINE):
+            for name in m.group(1).split(","):
+                name = name.strip().split(" as ")[0].strip()
+                if name:
+                    imports.append(name)
+    elif language in ("javascript", "typescript"):
+        # import X from 'module'; import { X } from 'module'; import 'module'
+        for m in re.finditer(r"""import\s+(?:.*?\s+from\s+)?['"]([\w@/.\\-]+)['"]""", text):
+            imports.append(m.group(1))
+        # require('module')
+        for m in re.finditer(r"""require\s*\(\s*['"]([\w@/.\\-]+)['"]""", text):
+            imports.append(m.group(1))
+        # Re-exports: export { X } from 'module'; export * from 'module'
+        for m in re.finditer(r"""export\s+(?:\*|(?:\{[^}]*\}|\w+)\s+)from\s+['"]([\w@/.\\-]+)['"]""", text):
+            imports.append(m.group(1))
+    elif language == "go":
+        # Restrict to the Go import block, not all quoted strings.
+        import_block = re.search(r'import\s*\((.*?)\)', text, re.DOTALL)
+        if import_block:
+            for m in re.finditer(r'"([\w/.\\-]+)"', import_block.group(1)):
+                imports.append(m.group(1))
+        # Single import: import "pkg"
+        for m in re.finditer(r'^import\s+"([\w/.\\-]+)"', text, re.MULTILINE):
+            imports.append(m.group(1))
+    elif language == "rust":
+        for m in re.finditer(r"^use\s+([\w:]+)", text, re.MULTILINE):
+            imports.append(m.group(1))
+    elif language == "java":
+        for m in re.finditer(r"^import\s+([\w.]+);", text, re.MULTILINE):
+            imports.append(m.group(1))
+
+    return imports
+
+
+def _enclosing_caller(symbols: list[dict], line: int) -> str:
+    """Return the qualified name of the function/method whose line range
+    contains ``line``. When several symbols nest, the innermost (smallest
+    span) wins. Returns an empty string when no enclosing function exists
+    (top-level call) or when symbols extraction failed.
+    """
+    best_qname = ""
+    best_span = -1
+    for sym in symbols:
+        kind = sym.get("kind", "")
+        if kind not in ("function", "method"):
+            continue
+        start = sym.get("line_start", 0)
+        end = sym.get("line_end", start)
+        if start <= line <= end:
+            span = end - start
+            if best_span < 0 or span < best_span:
+                qname = sym.get("qualified_name") or sym.get("name") or ""
+                if qname:
+                    best_qname = qname
+                    best_span = span
+    return best_qname
+
+
+def extract_references(content: bytes, path: str) -> list[dict]:
+    """Extract symbol references (call sites, imports) from file content.
+
+    Returns list of dicts: {symbol_name, ref_kind, line, caller_qualified_name}.
+    ``caller_qualified_name`` is the enclosing function/method (best-effort,
+    inferred by line range against extracted symbols); empty for top-level
+    calls or when symbol extraction is unavailable.
+    """
+    import re
+
+    language = detect_language(path)
+    if not language or language not in ("python", "javascript", "typescript"):
+        return []
+
+    try:
+        text = content.decode("utf-8", errors="replace")
+    except Exception:
+        return []
+
+    # Symbols are needed to attribute callers; failure here is non-fatal,
+    # we just emit refs without caller_qualified_name.
+    try:
+        symbols = extract_symbols(content, path)
+    except Exception:
+        symbols = []
+
+    refs: list[dict] = []
+    seen: set[tuple[str, int]] = set()  # dedupe (name, line)
+
+    for i, line_text in enumerate(text.split("\n"), 1):
+        stripped = line_text.strip()
+        # Skip comment lines
+        if stripped.startswith("#") or stripped.startswith("//"):
+            continue
+        # Skip definition lines (def/class/function/interface/type)
+        if re.match(r"^\s*(?:async\s+)?(?:def|class|function|interface|type)\s", line_text):
+            continue
+        # Skip import lines — those are tracked as dependencies
+        if re.match(r"^\s*(?:from\s|import\s|export\s.*from)", line_text):
+            continue
+
+        caller = _enclosing_caller(symbols, i)
+
+        # Function/method calls: name(
+        for m in re.finditer(r"\b(\w+)\s*\(", stripped):
+            name = m.group(1)
+            # Skip language keywords and builtins
+            if name in _SKIP_NAMES:
+                continue
+            key = (name, i)
+            if key not in seen:
+                seen.add(key)
+                refs.append({
+                    "symbol_name": name,
+                    "ref_kind": "call",
+                    "line": i,
+                    "caller_qualified_name": caller,
+                })
+
+        # Method calls: obj.method(
+        for m in re.finditer(r"\b\w+\.(\w+)\s*\(", stripped):
+            name = m.group(1)
+            if name in _SKIP_NAMES:
+                continue
+            key = (name, i)
+            if key not in seen:
+                seen.add(key)
+                refs.append({
+                    "symbol_name": name,
+                    "ref_kind": "call",
+                    "line": i,
+                    "caller_qualified_name": caller,
+                })
+
+    return refs
+
+
+_SKIP_NAMES = frozenset({
+    # Python keywords/builtins
+    "if", "else", "elif", "for", "while", "try", "except", "finally",
+    "with", "as", "return", "yield", "raise", "pass", "break", "continue",
+    "print", "len", "range", "list", "dict", "set", "tuple", "str", "int",
+    "float", "bool", "type", "super", "isinstance", "issubclass", "hasattr",
+    "getattr", "setattr", "delattr", "property", "staticmethod", "classmethod",
+    "enumerate", "zip", "map", "filter", "sorted", "reversed", "min", "max",
+    "sum", "abs", "any", "all", "next", "iter", "open", "round", "repr",
+    # JS/TS keywords/builtins
+    "require", "import", "export", "new", "typeof", "void", "delete",
+    "Array", "Object", "String", "Number", "Boolean", "Promise", "Date",
+    "Math", "JSON", "console", "setTimeout", "setInterval", "clearTimeout",
+    "clearInterval", "parseInt", "parseFloat", "Error", "RegExp", "Map", "Set",
+})
+
+
+def extract_symbols(content: bytes, path: str) -> list[dict]:
+    """Extract symbols from file content using tree-sitter if available.
+
+    Falls back to regex-based extraction for common patterns.
+    Returns list of symbol dicts with keys: name, kind, line_start, line_end,
+    signature, exported.
+    """
+    language = detect_language(path)
+    if not language:
+        return []
+
+    try:
+        text = content.decode("utf-8", errors="replace")
+    except Exception:
+        return []
+
+    # Try tree-sitter first
+    try:
+        return _extract_with_tree_sitter(text, language)
+    except Exception:
+        pass
+
+    # Fallback to regex
+    return _extract_with_regex(text, language)
+
+
+def _extract_with_tree_sitter(text: str, language: str) -> list[dict]:
+    """Extract symbols using tree-sitter."""
+
+    # This will raise if tree-sitter is not available
+    from attocode_intel._internal.integrations.context.ts_parser import _get_parser as get_parser
+
+    parser = get_parser(language)
+    if parser is None:
+        raise RuntimeError("No parser available")
+
+    tree = parser.parse(bytes(text, "utf-8"))
+    symbols = []
+
+    # Walk tree for function/class definitions
+    _walk_tree(tree.root_node, symbols, language)
+    return symbols
+
+
+def _walk_tree(node, symbols: list[dict], language: str) -> None:
+    """Recursively walk AST nodes to extract symbol definitions."""
+    kind_map = {
+        "function_definition": "function",
+        "function_declaration": "function",
+        "method_definition": "method",
+        "method_declaration": "method",
+        "class_definition": "class",
+        "class_declaration": "class",
+        "interface_declaration": "interface",
+        "type_alias_declaration": "type",
+        "variable_declaration": "variable",
+        "const_declaration": "constant",
+        "struct_item": "class",
+        "enum_item": "class",
+        "trait_item": "interface",
+        "type_declaration": "type",
+        "enum_declaration": "class",
+        "struct_declaration": "class",
+        "object_declaration": "class",
+        "protocol_declaration": "interface",
+    }
+
+    node_type = node.type
+    if node_type in kind_map:
+        name_node = node.child_by_field_name("name")
+        if name_node:
+            is_async = node.parent and node.parent.type in (
+                "async_function_definition", "decorated_definition",
+            ) or node_type.startswith("async_")
+            # Build signature from parameters node if available
+            sig = None
+            params_node = node.child_by_field_name("parameters")
+            if params_node:
+                sig = params_node.text.decode("utf-8", errors="replace")
+                ret_node = node.child_by_field_name("return_type")
+                if ret_node:
+                    sig += f" -> {ret_node.text.decode('utf-8', errors='replace')}"
+            exported = False
+            if node.parent and node.parent.type == "export_statement":
+                exported = True
+            symbols.append({
+                "name": name_node.text.decode("utf-8"),
+                "kind": kind_map[node_type],
+                "line_start": node.start_point[0] + 1,
+                "line_end": node.end_point[0] + 1,
+                "signature": sig,
+                "exported": exported,
+                "metadata": {"async": True} if is_async else {},
+            })
+
+    for child in node.children:
+        _walk_tree(child, symbols, language)
+
+
+def _extract_with_regex(text: str, language: str) -> list[dict]:
+    """Regex-based symbol extraction fallback."""
+    import re
+
+    symbols = []
+    lines = text.split("\n")
+
+    patterns = {
+        "python": [
+            (r"^(?:async\s+)?def\s+(\w+)", "function"),
+            (r"^class\s+(\w+)", "class"),
+        ],
+        "javascript": [
+            (r"(?:export\s+)?(?:async\s+)?function\s+(\w+)", "function"),
+            (r"(?:export\s+)?class\s+(\w+)", "class"),
+            (r"(?:export\s+)?const\s+(\w+)\s*=", "constant"),
+        ],
+        "typescript": [
+            (r"(?:export\s+)?(?:async\s+)?function\s+(\w+)", "function"),
+            (r"(?:export\s+)?class\s+(\w+)", "class"),
+            (r"(?:export\s+)?interface\s+(\w+)", "interface"),
+            (r"(?:export\s+)?type\s+(\w+)", "type"),
+        ],
+    }
+
+    lang_patterns = patterns.get(language, patterns.get("javascript", []))
+
+    for i, line in enumerate(lines, 1):
+        stripped = line.strip()
+        for pattern, kind in lang_patterns:
+            match = re.match(pattern, stripped)
+            if match:
+                is_async = "async" in stripped.split("def ")[0] if "def " in stripped else "async" in stripped.split("function")[0] if "function" in stripped else False
+                # Extract signature from the line
+                sig = None
+                paren_start = stripped.find("(")
+                if paren_start != -1:
+                    # Find matching close paren (simple single-line)
+                    paren_end = stripped.rfind(")")
+                    if paren_end > paren_start:
+                        sig = stripped[paren_start:paren_end + 1]
+                        # Check for return type annotation
+                        rest = stripped[paren_end + 1:].strip()
+                        if rest.startswith("->"):
+                            sig += f" {rest.split(':')[0].strip()}"
+                symbols.append({
+                    "name": match.group(1),
+                    "kind": kind,
+                    "line_start": i,
+                    "line_end": i,
+                    "signature": sig,
+                    "exported": "export" in line,
+                    "metadata": {"async": True} if is_async else {},
+                })
+                break
+
+    return symbols