openworkproof 1.0.0__py3-none-any.whl

openworkproof/repo_pipeline/output.py

@@ -0,0 +1,68 @@
+"""Structured output: render a ``RepositoryAnalysis`` to canonical JSON.
+
+The output is a single self-describing object so downstream tools can consume
+it without extra context:
+
+.. code-block:: json
+
+    {
+      "schema_version": "openworkproof/repo-analysis/0.1",
+      "root": "/abs/path",
+      "totals": {"files": 3, "bytes": 42},
+      "files": [{"path": "a.py", "language": "Python", "size_bytes": 42,
+                 "sha256": "...", "lines": 2}],
+      "results": [{"analyzer": "language_stats", "data": {...}}]
+    }
+"""
+
+from __future__ import annotations
+
+import json
+
+from .errors import OutputError, get_logger
+from .models import RepositoryAnalysis
+
+_logger = get_logger()
+
+SCHEMA_VERSION = "openworkproof/repo-analysis/0.1"
+
+
+def _file_record(entry) -> dict[str, object]:
+    return {
+        "path": entry.relative_path,
+        "language": entry.language,
+        "size_bytes": entry.size_bytes,
+        "sha256": entry.sha256,
+        "lines": entry.content.count("\n"),
+    }
+
+
+def to_dict(analysis: RepositoryAnalysis) -> dict[str, object]:
+    return {
+        "schema_version": SCHEMA_VERSION,
+        "root": analysis.scan.root,
+        "totals": {
+            "files": analysis.scan.total_files,
+            "bytes": analysis.scan.total_bytes,
+            "skipped_directories": len(analysis.scan.skipped),
+        },
+        "files": [
+            _file_record(entry) for entry in analysis.scan.entries
+        ],
+        "results": [
+            {"analyzer": result.analyzer, "data": dict(result.data)}
+            for result in analysis.results
+        ],
+    }
+
+
+def render_json(analysis: RepositoryAnalysis, *, indent: int | None = 2) -> str:
+    try:
+        return json.dumps(
+            to_dict(analysis),
+            ensure_ascii=False,
+            sort_keys=True,
+            indent=indent,
+        )
+    except (TypeError, ValueError) as error:
+        raise OutputError("repository analysis cannot be serialized") from error

openworkproof/repo_pipeline/reader.py

@@ -0,0 +1,47 @@
+"""File reading: decode a text file with size limits and permission guards.
+
+The reader is intentionally a small dependency-free seam so traversal and
+analysis stages can be tested with injected readers.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+from pathlib import Path
+
+from .errors import DecodeError, PermissionDeniedError, get_logger
+
+MAX_TEXT_BYTES = 1_048_576  # 1 MiB: larger files are skipped by the reader
+_logger = get_logger()
+
+
+def _decode_utf8(raw: bytes) -> str:
+    try:
+        return raw.decode("utf-8")
+    except UnicodeDecodeError as error:
+        raise DecodeError("file is not valid UTF-8 text") from error
+
+
+def read_text_file(path: Path, *, max_bytes: int = MAX_TEXT_BYTES) -> str:
+    """Read one file as UTF-8 text with a size cap and permission guard."""
+    try:
+        size = path.stat().st_size
+    except OSError as error:
+        raise PermissionDeniedError(f"cannot stat file: {path}") from error
+    if size > max_bytes:
+        _logger.warning("skipping oversized file (%.1f KiB): %s", size / 1024, path)
+        raise DecodeError(f"file exceeds the {max_bytes}-byte text limit: {path}")
+    try:
+        raw = path.read_bytes()
+    except OSError as error:
+        raise PermissionDeniedError(f"cannot read file: {path}") from error
+    return _decode_utf8(raw)
+
+
+def sha256_bytes(raw: bytes) -> str:
+    return hashlib.sha256(raw).hexdigest()
+
+
+def file_sha256(path: Path) -> str:
+    return sha256_bytes(path.read_bytes())

openworkproof/repo_pipeline/sources.py

@@ -0,0 +1,120 @@
+"""Input resolution: turn a local path or a remote repository URL into a
+concrete readable repository root.
+
+``resolve_repository_source`` accepts:
+
+- a local absolute or relative path to an existing directory;
+- a GitHub URL such as ``https://github.com/owner/repo`` or
+  ``https://github.com/owner/repo/tree/<ref>``.
+
+Remote sources are cloned into a caller-managed temporary root when
+``clone_enabled`` is true and the ``git`` executable is available; otherwise a
+``RemoteCloneError`` is raised so callers can degrade gracefully.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+import re
+import shutil
+import subprocess
+import tempfile
+from urllib.parse import urlparse
+
+from .errors import (
+    PathNotFoundError,
+    PermissionDeniedError,
+    RemoteCloneError,
+    UnsupportedSourceError,
+    get_logger,
+)
+
+_GITHUB_URL = re.compile(
+    r"^https?://(?:www\.)?github\.com/(?P<owner>[A-Za-z0-9_.-]+)/"
+    r"(?P<repo>[A-Za-z0-9_.-]+)(?:/tree/(?P<ref>.+))?$"
+)
+_KNOWN_REMOTE_HOSTS = frozenset({"github.com"})
+_logger = get_logger()
+
+
+def _looks_like_remote_url(value: str) -> bool:
+    parsed = urlparse(value)
+    return parsed.scheme in {"http", "https"} and parsed.netloc != ""
+
+
+def _classify(value: str) -> str:
+    if _looks_like_remote_url(value):
+        match = _GITHUB_URL.match(value)
+        if match is not None:
+            return "github"
+        host = urlparse(value).netloc
+        if host in _KNOWN_REMOTE_HOSTS:
+            return "github"
+        raise UnsupportedSourceError(f"unsupported remote host: {host}")
+    return "local"
+
+
+def resolve_local_path(value: str | Path) -> Path:
+    path = Path(value).expanduser()
+    if not path.exists():
+        raise PathNotFoundError(f"repository path does not exist: {path}")
+    if not path.is_dir():
+        raise UnsupportedSourceError(f"repository source is not a directory: {path}")
+    if not os.access(path, os.R_OK):
+        raise PermissionDeniedError(f"repository path is not readable: {path}")
+    return path.resolve()
+
+
+def _clone_github(owner: str, repo: str, ref: str | None) -> Path:
+    git = shutil.which("git")
+    if git is None:
+        raise RemoteCloneError("git executable is unavailable")
+    root = Path(tempfile.mkdtemp(prefix="owp-repo-"))
+    url = f"https://github.com/{owner}/{repo}.git"
+    try:
+        subprocess.run(
+            (git, "clone", "--depth", "1", url, str(root)),
+            check=True,
+            capture_output=True,
+            timeout=300,
+        )
+        if ref is not None and ref != "HEAD":
+            subprocess.run(
+                (git, "-C", str(root), "checkout", ref),
+                check=True,
+                capture_output=True,
+                timeout=120,
+            )
+    except (subprocess.CalledProcessError, OSError) as error:
+        shutil.rmtree(root, ignore_errors=True)
+        raise RemoteCloneError(f"remote clone failed for {owner}/{repo}") from error
+    return root
+
+
+def resolve_repository_source(
+    source: str | Path,
+    *,
+    clone_enabled: bool = True,
+) -> Path:
+    """Resolve a local path or remote URL to a readable repository root.
+
+    Returns the local repository root. Remote sources are cloned into a
+    temporary directory that the caller must clean up.
+    """
+    if isinstance(source, Path):
+        return resolve_local_path(source)
+    if not isinstance(source, str) or not source:
+        raise UnsupportedSourceError("repository source must be a path or URL")
+    kind = _classify(source)
+    if kind == "local":
+        _logger.debug("resolved local repository source: %s", source)
+        return resolve_local_path(source)
+    match = _GITHUB_URL.match(source)
+    if match is None:
+        raise UnsupportedSourceError(f"unsupported repository URL: {source}")
+    if not clone_enabled:
+        raise RemoteCloneError("remote clone is disabled for this pipeline")
+    owner, repo, ref = match.group("owner"), match.group("repo"), match.group("ref")
+    _logger.info("cloning remote repository %s/%s", owner, repo)
+    return _clone_github(owner, repo, ref)

openworkproof/repo_pipeline/traversal.py

@@ -0,0 +1,253 @@
+"""Repository traversal: recursively walk a repository, apply directory
+filters, recognise code file types, and read each supported file.
+
+The traversal stage is a pure walker over the local repository root. It
+accepts an injected ``read_file`` callable so tests and callers can substitute
+the reader without touching the walk logic.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Callable
+
+from .errors import ReadError, get_logger
+from .models import RepoFileEntry, RepositoryScan, SkippedDirectory
+from .reader import read_text_file, sha256_bytes
+
+_logger = get_logger()
+
+MAX_FILES = 5_000
+MAX_DIRECTORY_DEPTH = 32
+
+# Directories never scanned: VCS, dependency installs, build output, caches.
+FILTERED_DIRECTORIES = frozenset(
+    {
+        ".git",
+        ".hg",
+        ".svn",
+        "node_modules",
+        "bower_components",
+        "__pycache__",
+        ".venv",
+        "venv",
+        ".tox",
+        ".mypy_cache",
+        ".pytest_cache",
+        ".ruff_cache",
+        ".idea",
+        ".vscode",
+        "dist",
+        "build",
+        "out",
+        "target",
+        ".next",
+        ".nuxt",
+        ".cache",
+        "coverage",
+        ".terraform",
+        "Pods",
+        "vendor",
+    }
+)
+
+# Extension -> language. Unknown text files still load with language None.
+LANGUAGE_BY_EXTENSION: dict[str, str] = {
+    ".py": "Python",
+    ".pyi": "Python",
+    ".js": "JavaScript",
+    ".mjs": "JavaScript",
+    ".cjs": "JavaScript",
+    ".ts": "TypeScript",
+    ".tsx": "TypeScript",
+    ".jsx": "JavaScript",
+    ".go": "Go",
+    ".rs": "Rust",
+    ".java": "Java",
+    ".kt": "Kotlin",
+    ".swift": "Swift",
+    ".c": "C",
+    ".h": "C",
+    ".cpp": "C++",
+    ".cc": "C++",
+    ".hpp": "C++",
+    ".cs": "C#",
+    ".rb": "Ruby",
+    ".php": "PHP",
+    ".sh": "Shell",
+    ".bash": "Shell",
+    ".zsh": "Shell",
+    ".fish": "Shell",
+    ".ps1": "PowerShell",
+    ".sql": "SQL",
+    ".html": "HTML",
+    ".htm": "HTML",
+    ".css": "CSS",
+    ".scss": "SCSS",
+    ".less": "Less",
+    ".json": "JSON",
+    ".yaml": "YAML",
+    ".yml": "YAML",
+    ".toml": "TOML",
+    ".xml": "XML",
+    ".md": "Markdown",
+    ".rst": "RST",
+    ".tex": "LaTeX",
+    ".proto": "Protocol Buffers",
+    ".graphql": "GraphQL",
+    ".vue": "Vue",
+    ".svelte": "Svelte",
+    ".dockerfile": "Dockerfile",
+    ".tf": "Terraform",
+    ".lua": "Lua",
+    ".pl": "Perl",
+    ".r": "R",
+    ".m": "Objective-C",
+    ".dart": "Dart",
+    ".ex": "Elixir",
+    ".exs": "Elixir",
+    ".erl": "Erlang",
+    ".hs": "Haskell",
+    ".clj": "Clojure",
+    ".scala": "Scala",
+    ".groovy": "Groovy",
+    ".fs": "F#",
+}
+
+# File names treated as code regardless of extension.
+NAME_LANGUAGES: dict[str, str] = {
+    "Dockerfile": "Dockerfile",
+    "Makefile": "Makefile",
+    "CMakeLists.txt": "CMake",
+    "Justfile": "Justfile",
+    "Jenkinsfile": "Jenkinsfile",
+    "Rakefile": "Ruby",
+    "Gemfile": "Ruby",
+    "Cargo.toml": "TOML",
+    "go.mod": "Go",
+}
+
+
+def language_for(path: Path) -> str | None:
+    name = path.name
+    if name in NAME_LANGUAGES:
+        return NAME_LANGUAGES[name]
+    return LANGUAGE_BY_EXTENSION.get(path.suffix.lower())
+
+
+def _walk_relative(
+    top: Path,
+    current: Path,
+    *,
+    depth: int,
+    filters: frozenset[str],
+    max_files: int,
+    max_depth: int,
+) -> tuple[list[RepoFileEntry], list[SkippedDirectory]]:
+    entries: list[RepoFileEntry] = []
+    skipped: list[SkippedDirectory] = []
+    if depth > max_depth:
+        _logger.warning("traversal depth limit reached at %s", current)
+        return entries, skipped
+    try:
+        children = sorted(current.iterdir(), key=lambda item: item.name)
+    except OSError as error:
+        _logger.error("cannot list directory %s: %s", current, error)
+        return entries, skipped
+    for child in children:
+        if child.is_symlink():
+            _logger.debug("skipping symlink: %s", child)
+            continue
+        if child.is_dir():
+            if child.name in filters:
+                skipped.append(
+                    SkippedDirectory(
+                        relative_path=child.relative_to(top).as_posix(),
+                        rule="filtered-directory",
+                    )
+                )
+                _logger.debug("filtering directory: %s", child.name)
+                continue
+            sub_entries, sub_skipped = _walk_relative(
+                top,
+                child,
+                depth=depth + 1,
+                filters=filters,
+                max_files=max_files,
+                max_depth=max_depth,
+            )
+            entries.extend(sub_entries)
+            skipped.extend(sub_skipped)
+            continue
+        if not child.is_file():
+            continue
+        if len(entries) >= max_files:
+            _logger.warning("file count limit reached (%d files)", max_files)
+            break
+        language = language_for(child)
+        if language is None:
+            _logger.debug("skipping unrecognised file type: %s", child.name)
+            continue
+        entries.append(
+            RepoFileEntry(
+                relative_path=child.relative_to(top).as_posix(),
+                language=language,
+                size_bytes=0,
+                content="",
+                sha256="",
+            )
+        )
+    return entries, skipped
+
+
+def scan_repository(
+    root: Path,
+    *,
+    filters: frozenset[str] = FILTERED_DIRECTORIES,
+    max_files: int = MAX_FILES,
+    max_depth: int = MAX_DIRECTORY_DEPTH,
+    read_file: Callable[[Path], str] = read_text_file,
+) -> RepositoryScan:
+    """Walk ``root`` and build a ``RepositoryScan`` of supported text files."""
+    placeholder_entries, skipped = _walk_relative(
+        root,
+        root,
+        depth=0,
+        filters=filters,
+        max_files=max_files,
+        max_depth=max_depth,
+    )
+    entries: list[RepoFileEntry] = []
+    total_bytes = 0
+    for placeholder in placeholder_entries:
+        path = root / placeholder.relative_path
+        try:
+            content = read_file(path)
+        except ReadError as error:
+            _logger.warning("skipping unreadable file %s: %s", path, error)
+            continue
+        raw = content.encode("utf-8")
+        total_bytes += len(raw)
+        entries.append(
+            RepoFileEntry(
+                relative_path=placeholder.relative_path,
+                language=placeholder.language,
+                size_bytes=len(raw),
+                content=content,
+                sha256=sha256_bytes(raw),
+            )
+        )
+    entries.sort(key=lambda entry: entry.relative_path)
+    languages: dict[str, int] = {}
+    for entry in entries:
+        if entry.language is not None:
+            languages[entry.language] = languages.get(entry.language, 0) + 1
+    return RepositoryScan(
+        root=str(root),
+        entries=tuple(entries),
+        skipped=tuple(skipped),
+        total_files=len(entries),
+        total_bytes=total_bytes,
+        languages=languages,
+    )