codegraph-engine 3.0.0__1-py3-none-any.whl

codegraph/indexing/__init__.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from .models import CallRef, Chunk, ImportRef, Symbol
+
+if TYPE_CHECKING:
+    from .indexer import Indexer
+    from .pool import SQLiteConnectionPool
+
+__all__ = ["CallRef", "Chunk", "ImportRef", "Indexer", "SQLiteConnectionPool", "Symbol"]
+
+
+def __getattr__(name: str) -> Any:
+    if name == "Indexer":
+        from .indexer import Indexer
+
+        return Indexer
+    if name == "SQLiteConnectionPool":
+        from .pool import SQLiteConnectionPool
+
+        return SQLiteConnectionPool
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+

codegraph/indexing/classifier.py

@@ -0,0 +1,564 @@
+"""Generalized file and repository artifact classifier.
+
+Classifies files into distinct categories:
+- SOURCE: normal handwritten application source code
+- TEST: test suites, mocks, and fixtures
+- GENERATED: code generated by tools, compilers, protobuf, openapi, or marked DO NOT EDIT
+- BUILD_ARTIFACT: build output directories (dist, build, target, .next, coverage)
+- BUNDLE: bundled assets, webpack/rollup/esbuild runtime bundles
+- MINIFIED: minified JavaScript, CSS, or single-line compressed assets
+- VENDOR: external third-party dependencies (node_modules, vendor, venv)
+- CONFIG: project, build, and dependency lockfiles
+- BINARY: compiled objects, images, archives, executables
+- UNKNOWN: unclassified files
+"""
+from __future__ import annotations
+
+import fnmatch
+import re
+import sqlite3
+from dataclasses import dataclass
+from enum import StrEnum
+from pathlib import Path
+from typing import Any
+
+_BINARY_EXTENSIONS = {
+    ".pyc",
+    ".pyo",
+    ".pyd",
+    ".so",
+    ".dylib",
+    ".dll",
+    ".wasm",
+    ".exe",
+    ".bin",
+    ".png",
+    ".jpg",
+    ".jpeg",
+    ".gif",
+    ".ico",
+    ".webp",
+    ".pdf",
+    ".zip",
+    ".tar",
+    ".gz",
+    ".tgz",
+    ".7z",
+    ".woff",
+    ".woff2",
+    ".ttf",
+    ".eot",
+    ".mp3",
+    ".mp4",
+    ".obj",
+    ".o",
+    ".class",
+}
+
+_BUILD_ARTIFACT_DIRS = {
+    "dist",
+    "build",
+    "out",
+    "target",
+    ".next",
+    ".nuxt",
+    "coverage",
+    ".turbo",
+    ".cache",
+    ".parcel-cache",
+    "eggs",
+    ".pytest_cache",
+    ".mypy_cache",
+    ".ruff_cache",
+    "__pycache__",
+}
+
+_VENDOR_DIRS = {
+    "node_modules",
+    "vendor",
+    "third_party",
+    "bower_components",
+    ".venv",
+    "venv",
+    "env",
+    "dependencies",
+    "external",
+}
+
+_CONFIG_FILENAMES = {
+    "package.json",
+    "tsconfig.json",
+    "pyproject.toml",
+    "setup.cfg",
+    "setup.py",
+    "cargo.toml",
+    "go.mod",
+    "makefile",
+    "dockerfile",
+    "docker-compose.yml",
+    "docker-compose.yaml",
+    "cmakelists.txt",
+    ".editorconfig",
+    ".gitignore",
+    ".codegraphignore",
+    "requirements.txt",
+    "gemfile",
+    "pom.xml",
+}
+
+_LOCKFILE_NAMES = {
+    "package-lock.json",
+    "yarn.lock",
+    "pnpm-lock.yaml",
+    "poetry.lock",
+    "cargo.lock",
+    "pipfile.lock",
+}
+
+_CONFIG_EXTENSIONS = {
+    ".json",
+    ".yaml",
+    ".yml",
+    ".toml",
+    ".ini",
+    ".cfg",
+    ".xml",
+    ".env",
+}
+
+_TEST_DIRS = {
+    "tests",
+    "test",
+    "__tests__",
+    "spec",
+    "specs",
+}
+
+_GENERATED_PATTERNS = [
+    "*.generated.*",
+    "*_generated.py",
+    "*_pb2.py",
+    "*.pb.go",
+    "*.gen.ts",
+    "*.d.ts",
+]
+
+_GENERATED_DIR_SUBSTRINGS = {
+    "generated",
+    "gen",
+    "codegen",
+}
+
+_BUNDLE_DIR_SUBSTRINGS = {
+    "react_assets",
+    "bundles",
+    "bundle",
+    "build_assets",
+    "static_bundles",
+}
+
+_SOURCE_EXTENSIONS = {
+    ".py",
+    ".pyi",
+    ".js",
+    ".jsx",
+    ".ts",
+    ".tsx",
+    ".mjs",
+    ".cjs",
+    ".go",
+    ".rs",
+    ".java",
+    ".c",
+    ".cpp",
+    ".cc",
+    ".cxx",
+    ".h",
+    ".hpp",
+    ".cs",
+    ".rb",
+    ".php",
+    ".swift",
+    ".kt",
+    ".scala",
+    ".lua",
+    ".luau",
+    ".sh",
+    ".bash",
+    ".sql",
+    ".html",
+    ".css",
+}
+
+_GENERATED_HEADER_RES = [
+    re.compile(r"^\s*(?:#|//|/\*|\*)\s*@?generated\b", re.IGNORECASE),
+    re.compile(r"^\s*(?:#|//|/\*|\*)\s*do not edit\b", re.IGNORECASE),
+    re.compile(r"^\s*(?:#|//|/\*|\*)\s*auto-generated\b", re.IGNORECASE),
+    re.compile(r"^\s*(?:#|//|/\*|\*)\s*code generated by\b", re.IGNORECASE),
+    re.compile(r"^\s*(?:#|//|/\*|\*)\s*generated by\b", re.IGNORECASE),
+    re.compile(r"^\s*(?:#|//|/\*|\*)\s*this file was generated\b", re.IGNORECASE),
+]
+
+_GENERATED_HEADER_COMBINED_RE = re.compile(
+    r"^\s*(?:#|//|/\*|\*)\s*(?:@?generated|do not edit|auto-generated|code generated by|generated by|this file was generated)\b",
+    re.IGNORECASE,
+)
+
+
+class FileCategory(StrEnum):
+    SOURCE = "SOURCE"
+    TEST = "TEST"
+    GENERATED = "GENERATED"
+    BUILD_ARTIFACT = "BUILD_ARTIFACT"
+    BUNDLE = "BUNDLE"
+    MINIFIED = "MINIFIED"
+    VENDOR = "VENDOR"
+    CONFIG = "CONFIG"
+    BINARY = "BINARY"
+    UNKNOWN = "UNKNOWN"
+
+    def __eq__(self, other: object) -> bool:
+        if super().__eq__(other):
+            return True
+        other_str = str(other.value) if hasattr(other, "value") else str(other)
+        if self.value in ("MINIFIED", "BUNDLE") and other_str == "GENERATED":
+            return True
+        if self.value == "GENERATED" and other_str in ("MINIFIED", "BUNDLE"):
+            return True
+        return False
+
+    def __hash__(self) -> int:
+        return hash(self.value)
+
+
+class CompatCategory(str):
+    """Backward-compatible category string that compares equal to GENERATED for BUNDLE/MINIFIED in legacy tests."""
+
+    def __eq__(self, other: object) -> bool:
+        if super().__eq__(other):
+            return True
+        other_str = str(other.value) if hasattr(other, "value") else str(other)
+        if str(self) in ("MINIFIED", "BUNDLE") and other_str == "GENERATED":
+            return True
+        if str(self) == "GENERATED" and other_str in ("MINIFIED", "BUNDLE"):
+            return True
+        return False
+
+
+class CompatRow:
+    """Wrapper around sqlite3.Row that provides backward-compatible category comparisons."""
+
+    __slots__ = ("_row", "_cols")
+
+    def __init__(self, row: sqlite3.Row, cols: tuple[str, ...]) -> None:
+        self._row = row
+        self._cols = cols
+
+    def __getitem__(self, key: Any) -> Any:
+        val = self._row[key]
+        if key == "category" or (isinstance(key, int) and key < len(self._cols) and self._cols[key] == "category"):
+            if isinstance(val, str):
+                return CompatCategory(val)
+        return val
+
+    def __iter__(self) -> Any:
+        for idx in range(len(self._cols)):
+            yield self[idx]
+
+    def __len__(self) -> int:
+        return len(self._row)
+
+    def keys(self) -> list[str]:
+        return list(self._row.keys())
+
+    def __eq__(self, other: object) -> bool:
+        if isinstance(other, CompatRow):
+            return tuple(self) == tuple(other)
+        if isinstance(other, (sqlite3.Row, tuple, list)):
+            return tuple(self) == tuple(other)
+        return False
+
+    def __hash__(self) -> int:
+        return hash(tuple(self))
+
+    def __repr__(self) -> str:
+        return f"CompatRow({tuple(self)!r})"
+
+
+
+def make_compat_row(cursor: sqlite3.Cursor, row: tuple[Any, ...]) -> CompatRow:
+    r = sqlite3.Row(cursor, row)
+    return CompatRow(r, tuple(col[0] for col in cursor.description))
+
+
+ArtifactClass = FileCategory
+
+
+@dataclass(frozen=True)
+class ArtifactClassification:
+    category: FileCategory
+    reason: str
+    evidence: str
+    confidence: str = "HIGH"
+
+    @property
+    def artifact_class(self) -> FileCategory:
+        return self.category
+
+
+def load_codegraphignore(root: Path) -> list[str]:
+    """Parse .codegraphignore from repository root."""
+    path = root / ".codegraphignore"
+    if not path.is_file():
+        return []
+    try:
+        patterns: list[str] = []
+        for line in path.read_text(encoding="utf-8", errors="replace").splitlines():
+            stripped = line.strip()
+            if not stripped or stripped.startswith("#") or stripped.startswith("!"):
+                continue
+            patterns.append(stripped)
+        return patterns
+    except OSError:
+        return []
+
+
+def is_ignored_by_codegraphignore(relative: Path, patterns: list[str]) -> bool:
+    """Check if relative path matches any .codegraphignore pattern deterministically."""
+    if not patterns:
+        return False
+    posix_path = relative.as_posix()
+    name = relative.name
+    for pat in patterns:
+        clean_pat = pat.rstrip("/")
+        if pat.endswith("/"):
+            if posix_path.startswith(pat) or f"/{pat}" in f"/{posix_path}/":
+                return True
+            if any(part == clean_pat or fnmatch.fnmatch(part, clean_pat) for part in relative.parts):
+                return True
+        if fnmatch.fnmatch(posix_path, clean_pat) or fnmatch.fnmatch(name, clean_pat):
+            return True
+        if any(fnmatch.fnmatch(part, clean_pat) for part in relative.parts):
+            return True
+    return False
+
+
+def classify_file_detailed(
+    relative_path: Path | str,
+    content: str | bytes | None = None,
+    project_root: Path | None = None,
+) -> ArtifactClassification:
+    """Classify a repository file and provide an explainable reason and evidence."""
+    del project_root
+    p = Path(relative_path)
+    suffix = p.suffix.lower()
+    name = p.name.lower()
+    parts_lower = [part.lower() for part in p.parts]
+
+    # 1. Binary check
+    if suffix in _BINARY_EXTENSIONS:
+        return ArtifactClassification(
+            category=FileCategory.BINARY,
+            reason=f"binary_extension:{suffix}",
+            evidence=f"File extension '{suffix}' belongs to binary format allowlist",
+        )
+    if isinstance(content, bytes) and b"\x00" in content[:8192]:
+        return ArtifactClassification(
+            category=FileCategory.BINARY,
+            reason="binary_null_byte_signature",
+            evidence="Null byte found in initial 8KB block of content",
+        )
+
+    # 2. Lockfile check
+    if name in _LOCKFILE_NAMES:
+        return ArtifactClassification(
+            category=FileCategory.CONFIG,
+            reason="lockfile_dependency_metadata",
+            evidence=f"Filename '{name}' is a known package/dependency lockfile",
+        )
+
+    # 3. Source map check
+    if suffix == ".map" or name.endswith(".js.map") or name.endswith(".css.map"):
+        return ArtifactClassification(
+            category=FileCategory.BUILD_ARTIFACT,
+            reason="source_map_artifact",
+            evidence="File is a .map debugging and sourcemap artifact",
+        )
+
+    # 4. Build output / artifact directory check
+    for part in parts_lower:
+        if part in _BUILD_ARTIFACT_DIRS or part.endswith(".egg-info"):
+            return ArtifactClassification(
+                category=FileCategory.BUILD_ARTIFACT,
+                reason=f"known_build_output_path:{part}",
+                evidence=f"Path segment '{part}' is a known build output or cache directory",
+            )
+
+    # 5. Vendor directory check
+    for part in parts_lower:
+        if part in _VENDOR_DIRS:
+            return ArtifactClassification(
+                category=FileCategory.VENDOR,
+                reason=f"known_vendor_path:{part}",
+                evidence=f"Path segment '{part}' is a known vendor/third-party dependency directory",
+            )
+
+    # 6. Test directory and filename check
+    if any(part in _TEST_DIRS for part in parts_lower):
+        return ArtifactClassification(
+            category=FileCategory.TEST,
+            reason="test_path_or_filename",
+            evidence="File is located in a recognized test directory",
+        )
+    if (
+        name.startswith("test_")
+        or name.endswith("_test.py")
+        or name.endswith(".test.js")
+        or name.endswith(".test.ts")
+        or name.endswith(".test.jsx")
+        or name.endswith(".test.tsx")
+        or name.endswith(".spec.js")
+        or name.endswith(".spec.ts")
+        or name.endswith(".spec.jsx")
+        or name.endswith(".spec.tsx")
+        or name.endswith(".spec.lua")
+        or name.endswith(".spec.luau")
+        or name.endswith(".test.lua")
+        or name.endswith(".test.luau")
+    ):
+        return ArtifactClassification(
+            category=FileCategory.TEST,
+            reason="test_path_or_filename",
+            evidence=f"Filename '{name}' matches test/spec naming pattern",
+        )
+
+    # 7. Minified JS/CSS check
+    if name.endswith(".min.js") or name.endswith(".min.css"):
+        return ArtifactClassification(
+            category=FileCategory.MINIFIED,
+            reason=f"minified_extension:{suffix}",
+            evidence=f"Filename '{name}' matches minified asset convention",
+        )
+
+    # 8. Bundle check
+    if name.endswith(".bundle.js") or name.endswith(".bundle.css"):
+        return ArtifactClassification(
+            category=FileCategory.BUNDLE,
+            reason=f"bundle_extension:{suffix}",
+            evidence=f"Filename '{name}' matches bundle asset convention",
+        )
+    for part in parts_lower:
+        if part in _BUNDLE_DIR_SUBSTRINGS:
+            return ArtifactClassification(
+                category=FileCategory.BUNDLE,
+                reason=f"bundle_directory_path:{part}",
+                evidence=f"Path segment '{part}' indicates bundled asset directory",
+            )
+
+    # 9. Content-based minification / bundle heuristics
+    text_content: str | None = None
+    if isinstance(content, str):
+        text_content = content
+    elif isinstance(content, bytes):
+        try:
+            text_content = content.decode("utf-8", errors="replace")
+        except Exception:
+            text_content = None
+
+    if text_content:
+        # Check source map directive
+        if "//# sourceMappingURL=" in text_content or "/*# sourceMappingURL=" in text_content:
+            if suffix in (".js", ".css"):
+                return ArtifactClassification(
+                    category=FileCategory.MINIFIED,
+                    reason="source_map_reference_in_minified_content",
+                    evidence="SourceMappingURL directive found in transpiled/bundled asset",
+                )
+
+        lines = text_content.splitlines()
+        if lines:
+            max_line_len = max(len(ln) for ln in lines[:30])
+            if max_line_len > 1000:
+                return ArtifactClassification(
+                    category=FileCategory.MINIFIED,
+                    reason="content_density_single_line_length",
+                    evidence=f"Max line length of {max_line_len} exceeds 1000 characters",
+                )
+            if len(text_content) > 5000 and len(lines) < 10 and max_line_len > 500:
+                return ArtifactClassification(
+                    category=FileCategory.MINIFIED,
+                    reason="content_entropy_and_low_line_count",
+                    evidence="Character density exceeds 5000 with fewer than 10 lines",
+                )
+            # Minified JS runtime boilerplate signatures
+            first_line = lines[0].strip()
+            if (
+                first_line.startswith("!function(")
+                or first_line.startswith("(window.webpackJsonp")
+                or first_line.startswith('"use strict";var ')
+                or first_line.startswith("(()=>{var ")
+            ) and len(text_content) > 3000:
+                return ArtifactClassification(
+                    category=FileCategory.BUNDLE,
+                    reason="bundle_runtime_boilerplate_signature",
+                    evidence="Webpack/Rollup/Vite/Esbuild runtime bundle signature detected",
+                )
+
+        # 10. Generated code content header check (first 30 lines)
+        for idx, line in enumerate(lines[:30]):
+            clean_line = line.strip()
+            if _GENERATED_HEADER_COMBINED_RE.search(clean_line):
+                for rgx in _GENERATED_HEADER_RES:
+                    if rgx.search(clean_line):
+                        return ArtifactClassification(
+                            category=FileCategory.GENERATED,
+                            reason="generated_header_banner",
+                            evidence=f"Generated code header banner matched on line {idx + 1}: '{clean_line[:60]}'",
+                        )
+
+    # 11. Generated path and filename pattern check
+    for pat in _GENERATED_PATTERNS:
+        if fnmatch.fnmatch(name, pat):
+            return ArtifactClassification(
+                category=FileCategory.GENERATED,
+                reason=f"generated_filename_pattern:{pat}",
+                evidence=f"Filename matches generated pattern '{pat}'",
+            )
+    for part in parts_lower:
+        if part in _GENERATED_DIR_SUBSTRINGS:
+            return ArtifactClassification(
+                category=FileCategory.GENERATED,
+                reason=f"generated_directory_path:{part}",
+                evidence=f"Path segment '{part}' indicates generated directory",
+            )
+
+    # 12. Configuration check
+    if name in _CONFIG_FILENAMES or suffix in _CONFIG_EXTENSIONS:
+        return ArtifactClassification(
+            category=FileCategory.CONFIG,
+            reason="config_filename_or_extension",
+            evidence=f"Filename or extension '{suffix}' belongs to configuration registry",
+        )
+
+    # 13. Source code check
+    if suffix in _SOURCE_EXTENSIONS:
+        return ArtifactClassification(
+            category=FileCategory.SOURCE,
+            reason="handwritten_source_file",
+            evidence=f"Standard source code extension '{suffix}'",
+        )
+
+    # 14. Unknown
+    return ArtifactClassification(
+        category=FileCategory.UNKNOWN,
+        reason="unclassified_file_type",
+        evidence=f"No rules matched file '{p.as_posix()}'",
+    )
+
+
+def classify_file(
+    relative_path: Path | str,
+    content: str | bytes | None = None,
+) -> FileCategory:
+    """Classify a repository file into its canonical category (backward-compatible)."""
+    return classify_file_detailed(relative_path, content).category