coltrace 0.1.0.dev4__py3-none-any.whl

coltrace/models.py

@@ -0,0 +1,92 @@
+"""Serializable discovery records, independent of AST and graph libraries."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import asdict, dataclass
+from typing import Any, Literal
+
+Evidence = Literal["confirmed", "inferred", "unresolved", "rejected"]
+
+
+@dataclass(frozen=True)
+class Location:
+    path: str
+    line: int
+    column: int
+    end_line: int | None = None
+    end_column: int | None = None
+
+
+@dataclass(frozen=True)
+class Diagnostic:
+    code: str
+    severity: Literal["warning", "error"]
+    message: str
+    location: Location
+    scope: str = "<module>"
+    hint: str = ""
+
+
+@dataclass(frozen=True)
+class FileInfo:
+    path: str
+    module: str
+    status: Literal["parsed", "invalid", "unreadable"]
+    sha256: str | None
+
+
+@dataclass(frozen=True)
+class FunctionInfo:
+    name: str
+    scope: str
+    module: str
+    parameters: tuple[str, ...]
+    is_async: bool
+    location: Location
+
+
+@dataclass(frozen=True)
+class DataFrameInfo:
+    id: str
+    scope: str
+    origin: str
+    receiver_evidence: Evidence
+    reason: str
+    location: Location
+
+
+@dataclass(frozen=True)
+class ColumnOccurrence:
+    column: str | None
+    receiver: str
+    dataframe_id: str | None
+    receiver_evidence: Evidence
+    reason: str
+    access: Literal["read", "write", "delete", "read_write"]
+    syntax: Literal["subscript", "loc"]
+    supported: bool
+    scope: str
+    module: str
+    location: Location
+
+
+@dataclass(frozen=True)
+class ScanReport:
+    files: tuple[FileInfo, ...]
+    functions: tuple[FunctionInfo, ...]
+    dataframes: tuple[DataFrameInfo, ...]
+    occurrences: tuple[ColumnOccurrence, ...]
+    diagnostics: tuple[Diagnostic, ...]
+    exclude: tuple[str, ...] = ()
+    schema_version: str = "0.1"
+    report_kind: str = "column_occurrences"
+    tool_version: str = "0.1.0.dev4"
+
+    def to_dict(self) -> dict[str, Any]:
+        """Return JSON-compatible containers, with no AST or runtime objects."""
+        return json.loads(self.to_json())  # type: ignore[no-any-return]
+
+    def to_json(self, *, indent: int | None = 2) -> str:
+        """Deterministic for the same source, config and Python interpreter."""
+        return json.dumps(asdict(self), ensure_ascii=False, sort_keys=True, indent=indent)

coltrace/scanner.py

@@ -0,0 +1,236 @@
+"""Deterministic discovery, encoding-aware parsing, and report assembly."""
+
+from __future__ import annotations
+
+import ast
+import fnmatch
+import hashlib
+import io
+import os
+import stat
+import tokenize
+import tomllib
+from dataclasses import dataclass
+from pathlib import Path
+
+from .collector import OccurrenceCollector
+from .models import Diagnostic, FileInfo, Location, ScanReport
+
+DEFAULT_IGNORES = frozenset(
+    {
+        ".venv",
+        "venv",
+        "__pycache__",
+        ".git",
+        ".coltrace",
+        ".pytest_cache",
+        ".mypy_cache",
+        ".ruff_cache",
+        "build",
+        "dist",
+    }
+)
+
+
+@dataclass(frozen=True)
+class ScanConfig:
+    """Excludes extend built-in ignores; explicit config overrides TOML discovery."""
+
+    exclude: tuple[str, ...] = ()
+
+    def __post_init__(self) -> None:
+        if not isinstance(self.exclude, tuple) or not all(
+            isinstance(pattern, str) and pattern for pattern in self.exclude
+        ):
+            raise ValueError("exclude must be a tuple of non-empty glob strings")
+
+
+def _load_config(root: Path) -> ScanConfig:
+    path = root / "pyproject.toml"
+    if not path.is_file():
+        return ScanConfig()
+    try:
+        with path.open("rb") as stream:
+            document = tomllib.load(stream)
+    except (OSError, ValueError) as exc:
+        raise ValueError(f"Cannot read configuration {path}: {exc}") from exc
+    tool = document.get("tool", {})
+    if not isinstance(tool, dict):
+        raise ValueError("pyproject.toml 'tool' must be a table")
+    section = tool.get("coltrace", {})
+    if not isinstance(section, dict) or set(section) - {"exclude"}:
+        raise ValueError("[tool.coltrace] supports only the 'exclude' option in Phase 1")
+    exclude = section.get("exclude", [])
+    if not isinstance(exclude, list) or not all(isinstance(item, str) for item in exclude):
+        raise ValueError("[tool.coltrace].exclude must be an array of strings")
+    return ScanConfig(tuple(exclude))
+
+
+def _ignored(relative: Path, config: ScanConfig) -> bool:
+    if any(part in DEFAULT_IGNORES for part in relative.parts):
+        return True
+    value = relative.as_posix()
+    return any(
+        fnmatch.fnmatchcase(value, pattern.rstrip("/"))
+        or ("/" not in pattern and any(fnmatch.fnmatchcase(p, pattern) for p in relative.parts))
+        for pattern in config.exclude
+    )
+
+
+def _is_link(path: Path) -> bool:
+    if path.is_symlink():
+        return True
+    # Covers junctions on Python 3.11 too; conservatively skip Windows reparse points.
+    return os.name == "nt" and bool(
+        path.lstat().st_file_attributes & stat.FILE_ATTRIBUTE_REPARSE_POINT
+    )
+
+
+def _discover(root: Path, config: ScanConfig, diagnostics: list[Diagnostic]) -> list[Path]:
+    files: list[Path] = []
+
+    def on_error(error: OSError) -> None:
+        path = Path(error.filename) if error.filename else root
+        relative = path.relative_to(root).as_posix()
+        diagnostics.append(
+            Diagnostic(
+                "CT_DISCOVERY_ERROR",
+                "error",
+                str(error),
+                Location(relative, 1, 0),
+                hint="Check directory permissions; this part of the scan is incomplete.",
+            )
+        )
+
+    for directory, subdirs, names in os.walk(root, followlinks=False, onerror=on_error):
+        current = Path(directory)
+        subdirs[:] = sorted(
+            name
+            for name in subdirs
+            if not _is_link(current / name)
+            and not _ignored((current / name).relative_to(root), config)
+        )
+        files.extend(
+            current / name
+            for name in sorted(names)
+            if name.endswith(".py")
+            and not _is_link(current / name)
+            and not _ignored((current / name).relative_to(root), config)
+        )
+    return sorted(files, key=lambda path: path.relative_to(root).as_posix())
+
+
+def _module_name(relative: Path) -> str:
+    parts = relative.with_suffix("").parts
+    return ".".join(parts[:-1] if parts[-1] == "__init__" else parts) or "__init__"
+
+
+def scan_project(path: str | Path, *, config: ScanConfig | None = None) -> ScanReport:
+    """Scan a directory without importing it, executing it, or reading its data sources.
+
+    Relative paths in the report use this directory as their root. Invalid files
+    produce diagnostics and do not prevent valid files from being scanned.
+    """
+    root = Path(path).resolve()
+    if not root.is_dir():
+        raise ValueError(f"Scan root must be an existing directory: {root}")
+    settings = config if config is not None else _load_config(root)
+    diagnostics: list[Diagnostic] = []
+    files: list[FileInfo] = []
+    collectors: list[OccurrenceCollector] = []
+    for source_path in _discover(root, settings, diagnostics):
+        relative = source_path.relative_to(root)
+        filename = relative.as_posix()
+        module = _module_name(relative)
+        digest = None
+        try:
+            raw = source_path.read_bytes()
+            digest = hashlib.sha256(raw).hexdigest()
+            encoding, _ = tokenize.detect_encoding(io.BytesIO(raw).readline)
+            source = raw.decode(encoding).replace("\r\n", "\n").replace("\r", "\n")
+        except (UnicodeError, LookupError, SyntaxError) as exc:
+            files.append(FileInfo(filename, module, "invalid", digest))
+            diagnostics.append(
+                Diagnostic(
+                    "CT_ENCODING_ERROR",
+                    "error",
+                    str(exc),
+                    Location(filename, 1, 0),
+                    hint="Use a valid Python encoding declaration or UTF-8.",
+                )
+            )
+            continue
+        except OSError as exc:
+            files.append(FileInfo(filename, module, "unreadable", None))
+            diagnostics.append(
+                Diagnostic(
+                    "CT_READ_ERROR",
+                    "error",
+                    str(exc),
+                    Location(filename, 1, 0),
+                    hint="Check file permissions and scan again.",
+                )
+            )
+            continue
+        try:
+            tree = ast.parse(source, filename=filename)
+        except (SyntaxError, RecursionError) as exc:
+            location = Location(filename, 1, 0)
+            if isinstance(exc, SyntaxError):
+                location = Location(filename, exc.lineno or 1, max((exc.offset or 1) - 1, 0))
+            files.append(FileInfo(filename, module, "invalid", digest))
+            diagnostics.append(
+                Diagnostic(
+                    "CT_SYNTAX_ERROR",
+                    "error",
+                    str(exc),
+                    location,
+                    hint="Check syntax and use an interpreter supporting the source grammar.",
+                )
+            )
+            continue
+        files.append(FileInfo(filename, module, "parsed", digest))
+        collector = OccurrenceCollector(filename, module, source)
+        try:
+            collector.visit(tree)
+        except RecursionError:
+            diagnostics.append(
+                Diagnostic(
+                    "CT_ANALYSIS_LIMIT",
+                    "error",
+                    "The source exceeds AST traversal depth limits.",
+                    Location(filename, 1, 0),
+                    hint="Simplify nested expressions; this file's occurrences are omitted.",
+                )
+            )
+            continue
+        collectors.append(collector)
+        diagnostics.extend(collector.diagnostics)
+    return ScanReport(
+        files=tuple(files),
+        functions=tuple(item for c in collectors for item in c.functions),
+        dataframes=tuple(item for c in collectors for item in c.dataframes),
+        occurrences=tuple(
+            sorted(
+                (item for c in collectors for item in c.occurrences),
+                key=lambda item: (
+                    item.location.path,
+                    item.location.line,
+                    item.location.column,
+                    item.access,
+                ),
+            )
+        ),
+        diagnostics=tuple(
+            sorted(
+                diagnostics,
+                key=lambda d: (
+                    d.location.path,
+                    d.location.line,
+                    d.location.column,
+                    d.code,
+                ),
+            )
+        ),
+        exclude=settings.exclude,
+    )