codegraph-engine 2.1.1__py3-none-any.whl

codegraph/indexing/scanner.py

@@ -0,0 +1,200 @@
+"""Secure, deterministic repository scanner.
+
+Architectural Invariants:
+1. Low-level primitive `safe_path()` is strict and fail-closed (raises SecurityError on escape).
+2. Repository scanner catches SecurityError on individual unsafe symlink/traversal entries,
+   records the security skip, and continues scanning without aborting the entire repository.
+3. Unsafe/escaped paths are NEVER indexed or exposed.
+4. Scan statistics are tracked explicitly.
+"""
+from __future__ import annotations
+
+import fnmatch
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from codegraph.errors import SecurityError
+from codegraph.indexing.classifier import (
+    FileCategory,
+    classify_file,
+    is_ignored_by_codegraphignore,
+    load_codegraphignore,
+)
+from codegraph.security import is_sensitive, safe_path
+
+IGNORED_DIRS = {
+    ".git",
+    ".venv",
+    "venv",
+    "node_modules",
+    "__pycache__",
+    "dist",
+    "build",
+    ".mypy_cache",
+    ".pytest_cache",
+}
+
+LANGUAGES = {
+    ".py": "python",
+    ".js": "javascript",
+    ".jsx": "javascript",
+    ".ts": "typescript",
+    ".tsx": "typescript",
+}
+
+
+@dataclass(frozen=True)
+class ScanResult:
+    path: Path
+    relative_path: Path
+    language: str
+    category: FileCategory = FileCategory.SOURCE
+
+
+@dataclass
+class ScanStats:
+    scanned: int = 0
+    accepted: int = 0
+    skipped: int = 0
+    skipped_security: int = 0
+    skipped_binary: int = 0
+    skipped_secret: int = 0
+    skipped_large: int = 0
+    skipped_ignored: int = 0
+    skipped_details: list[str] = field(default_factory=list)
+
+    def as_dict(self) -> dict[str, object]:
+        return {
+            "scanned": self.scanned,
+            "accepted": self.accepted,
+            "skipped": self.skipped,
+            "skipped_security": self.skipped_security,
+            "skipped_binary": self.skipped_binary,
+            "skipped_secret": self.skipped_secret,
+            "skipped_large": self.skipped_large,
+            "skipped_ignored": self.skipped_ignored,
+        }
+
+
+def is_binary(path: Path) -> bool:
+    try:
+        return b"\x00" in path.read_bytes()[:8192]
+    except OSError:
+        return True
+
+
+def scan_with_stats(
+    repository: Path,
+    max_file_size: int,
+    exclusions: tuple[str, ...] = (),
+) -> tuple[list[ScanResult], ScanStats]:
+    """Scan repository for supported source files, catching unsafe symlinks and collecting stats."""
+    root = repository.resolve(strict=True)
+    gitignore = _gitignore_patterns(root)
+    codegraphignore = load_codegraphignore(root)
+    results: list[ScanResult] = []
+    stats = ScanStats()
+
+    for directory, dirs, files in os.walk(root, followlinks=False):
+        dirs[:] = [d for d in dirs if d not in IGNORED_DIRS]
+        for filename in files:
+            stats.scanned += 1
+            absolute = Path(directory) / filename
+            try:
+                rel_unresolved = absolute.relative_to(root)
+            except ValueError:
+                stats.skipped += 1
+                stats.skipped_security += 1
+                continue
+
+            try:
+                # safe_path raises SecurityError if symlink resolves outside root or uses traversal escapes
+                resolved = safe_path(root, rel_unresolved)
+            except SecurityError:
+                stats.skipped += 1
+                stats.skipped_security += 1
+                stats.skipped_details.append(f"Security blocked: {rel_unresolved}")
+                continue
+
+            relative = resolved.relative_to(root)
+
+            try:
+                if not resolved.is_file():
+                    stats.skipped += 1
+                    continue
+                if resolved.stat().st_size > max_file_size:
+                    stats.skipped += 1
+                    stats.skipped_large += 1
+                    continue
+                if is_sensitive(relative):
+                    stats.skipped += 1
+                    stats.skipped_secret += 1
+                    continue
+                if (
+                    _ignored_by_gitignore(relative, gitignore)
+                    or is_ignored_by_codegraphignore(relative, codegraphignore)
+                    or any(relative.match(p) for p in exclusions)
+                ):
+                    stats.skipped += 1
+                    stats.skipped_ignored += 1
+                    continue
+                if is_binary(resolved):
+                    stats.skipped += 1
+                    stats.skipped_binary += 1
+                    continue
+
+                category = classify_file(relative)
+                if category in (FileCategory.BUILD_ARTIFACT, FileCategory.VENDOR, FileCategory.BINARY):
+                    stats.skipped += 1
+                    if category == FileCategory.BINARY:
+                        stats.skipped_binary += 1
+                    else:
+                        stats.skipped_ignored += 1
+                    continue
+            except OSError:
+                stats.skipped += 1
+                continue
+
+            language = LANGUAGES.get(relative.suffix.lower())
+            if language:
+                stats.accepted += 1
+                results.append(ScanResult(resolved, relative, language, category=category))
+            else:
+                stats.skipped += 1
+
+    results.sort(key=lambda item: item.relative_path.as_posix())
+    return results, stats
+
+
+def scan(
+    repository: Path,
+    max_file_size: int,
+    exclusions: tuple[str, ...] = (),
+) -> list[ScanResult]:
+    """Securely scan repository for supported source files."""
+    results, _ = scan_with_stats(repository, max_file_size, exclusions)
+    return results
+
+
+def _gitignore_patterns(root: Path) -> list[str]:
+    path = root / ".gitignore"
+    try:
+        return [
+            line.strip()
+            for line in path.read_text().splitlines()
+            if line.strip() and not line.lstrip().startswith("#") and not line.startswith("!")
+        ]
+    except OSError:
+        return []
+
+
+def _ignored_by_gitignore(relative: Path, patterns: list[str]) -> bool:
+    """Small, deterministic subset: ordinary glob and directory patterns; negation is unsupported."""
+    value = relative.as_posix()
+    return any(
+        fnmatch.fnmatch(value, pattern.rstrip("/"))
+        or (pattern.endswith("/") and value.startswith(pattern))
+        or fnmatch.fnmatch(relative.name, pattern)
+        for pattern in patterns
+    )

codegraph/indexing/test_framework.py

@@ -0,0 +1,116 @@
+"""Source-backed test framework detection.
+
+Detects test frameworks (PYTEST, UNITTEST, JEST, VITEST, MOCHA, OTHER, UNKNOWN)
+using concrete repository facts (imports, class inheritance, configuration files).
+Does not guess or assume tests exist merely because 'test' appears in a path.
+"""
+from __future__ import annotations
+
+import json
+import sqlite3
+from enum import StrEnum
+from pathlib import Path
+
+
+class TestFramework(StrEnum):
+    PYTEST = "PYTEST"
+    UNITTEST = "UNITTEST"
+    JEST = "JEST"
+    VITEST = "VITEST"
+    MOCHA = "MOCHA"
+    OTHER = "OTHER"
+    UNKNOWN = "UNKNOWN"
+
+
+def detect_test_framework(
+    con: sqlite3.Connection,
+    repository: Path,
+) -> tuple[TestFramework, bool, str]:
+    """Detect test framework and whether tests are present in the repository.
+
+    Returns:
+        (framework, tests_present, evidence)
+    """
+    root = repository.resolve()
+
+    # 1. Query files for test files
+    test_rows = []
+    try:
+        test_rows = con.execute(
+            "SELECT path FROM files WHERE (category='TEST' OR path LIKE 'tests/%' "
+            "OR path LIKE 'test/%' OR path LIKE '%/test_%.py' OR path LIKE '%_test.py' "
+            "OR path LIKE '%.test.%' OR path LIKE '%.spec.%') AND status='ok'"
+        ).fetchall()
+    except sqlite3.OperationalError:
+        pass
+
+    test_paths = [r[0] for r in test_rows]
+
+    # If no test files in index, check filesystem as fallback
+    if not test_paths:
+        for pat in ("tests", "test"):
+            d = root / pat
+            if d.is_dir() and any(d.iterdir()):
+                test_paths.append(pat)
+                break
+
+    if not test_paths:
+        return (TestFramework.UNKNOWN, False, "No test files or directories detected")
+
+    # 2. Check Python imports from test files
+    try:
+        imp_rows = con.execute(
+            "SELECT DISTINCT module FROM imports WHERE module IN ('pytest', 'unittest') "
+            "OR module LIKE 'pytest.%' OR module LIKE 'unittest.%'"
+        ).fetchall()
+        imp_modules = {r[0] for r in imp_rows}
+        if any(m.startswith("pytest") for m in imp_modules):
+            return (TestFramework.PYTEST, True, "Source evidence: imports pytest")
+        if any(m.startswith("unittest") for m in imp_modules):
+            return (TestFramework.UNITTEST, True, "Source evidence: imports unittest")
+    except sqlite3.OperationalError:
+        pass
+
+    # 3. Check Python configuration files
+    if (root / "pytest.ini").is_file() or (root / "conftest.py").is_file():
+        return (TestFramework.PYTEST, True, "Configuration evidence: pytest.ini or conftest.py present")
+
+    pyproject = root / "pyproject.toml"
+    if pyproject.is_file():
+        try:
+            content = pyproject.read_text(encoding="utf-8", errors="replace")
+            if "tool.pytest" in content or "pytest" in content:
+                return (TestFramework.PYTEST, True, "Configuration evidence: pyproject.toml defines pytest")
+        except OSError:
+            pass
+
+    # 4. Check JS/TS package.json
+    pkg_json = root / "package.json"
+    if pkg_json.is_file():
+        try:
+            data = json.loads(pkg_json.read_text(encoding="utf-8", errors="replace"))
+            dev_deps = data.get("devDependencies", {})
+            deps = data.get("dependencies", {})
+            scripts = data.get("scripts", {})
+            all_deps = {**deps, **dev_deps}
+            scripts_str = " ".join(str(v) for v in scripts.values())
+
+            if "vitest" in all_deps or "vitest" in scripts_str:
+                return (TestFramework.VITEST, True, "Package evidence: vitest declared in package.json")
+            if "jest" in all_deps or "jest" in scripts_str:
+                return (TestFramework.JEST, True, "Package evidence: jest declared in package.json")
+            if "mocha" in all_deps or "mocha" in scripts_str:
+                return (TestFramework.MOCHA, True, "Package evidence: mocha declared in package.json")
+        except (OSError, json.JSONDecodeError):
+            pass
+
+    # 5. Check config files for JS test runners
+    for f in root.glob("jest.config.*"):
+        if f.is_file():
+            return (TestFramework.JEST, True, f"Configuration evidence: {f.name} present")
+    for f in root.glob("vitest.config.*"):
+        if f.is_file():
+            return (TestFramework.VITEST, True, f"Configuration evidence: {f.name} present")
+
+    # 6. Fallback if test files exist but specific runner not identified
+    return (TestFramework.OTHER, True, f"Test files present ({len(test_paths)} test files) with unspecified runner")