sourcebound 1.2.1__py3-none-any.whl

clean_docs/inventory.py

@@ -0,0 +1,474 @@
+from __future__ import annotations
+
+import ast
+import hashlib
+import json
+import re
+import subprocess
+import sys
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from clean_docs.errors import ConfigurationError
+
+if sys.version_info >= (3, 11):
+    import tomllib
+else:
+    import tomli as tomllib
+
+
+SKIP_PARTS = {".git", ".venv", "build", "dist", "node_modules", "__pycache__"}
+SKIP_PATHS = {".sourcebound.yml", ".sourcebound-residue.yml", "llms.txt"}
+LANGUAGES = {
+    ".py": "Python",
+    ".ts": "TypeScript",
+    ".tsx": "TypeScript",
+    ".js": "JavaScript",
+    ".jsx": "JavaScript",
+    ".go": "Go",
+    ".rs": "Rust",
+}
+LINK = re.compile(r"(?<!!)\[[^\]]+\]\(([^)]+)\)")
+TS_EXPORT = re.compile(
+    r"^\s*export\s+(?:default\s+)?(?:async\s+)?(?:abstract\s+)?"
+    r"(?:function|class|const|interface|type|enum)\s+([A-Za-z_$][\w$]*)",
+    re.M,
+)
+TS_CLI_COMMAND = re.compile(r"\.command\(\s*['\"]([^'\"]+)['\"]")
+TS_CLI_OPTION = re.compile(r"\.option\(\s*['\"]([^'\"]+)['\"]")
+TS_MCP_TOOL = re.compile(r"\.(?:tool|registerTool)\(\s*['\"]([^'\"]+)['\"]")
+HTTP_METHODS = {"get", "put", "post", "delete", "patch", "head", "options", "trace"}
+PYTHON_TOOLING_MODULES = {"conftest.py", "noxfile.py", "setup.py"}
+PUBLIC_SURFACE_KINDS = frozenset(
+    {
+        "api-endpoint",
+        "api-symbol",
+        "cli-command",
+        "cli-option",
+        "config-key",
+        "mcp-tool",
+        "package",
+        "package-script",
+        "runtime-constraint",
+        "schema",
+    }
+)
+
+
+@dataclass(frozen=True)
+class InventoryItem:
+    id: str
+    kind: str
+    name: str
+    source: str
+    locator: str
+    adapter: str
+    digest: str
+    coverage: str
+    coverage_reason: str | None = None
+
+
+@dataclass(frozen=True)
+class InventoryReport:
+    languages: tuple[str, ...]
+    items: tuple[InventoryItem, ...]
+
+    def as_dict(self) -> dict[str, object]:
+        return {
+            "schema": "sourcebound.inventory.v1",
+            "languages": list(self.languages),
+            "items": [asdict(item) for item in self.items],
+            "counts": {
+                state: sum(item.coverage == state for item in self.items)
+                for state in ("bound", "cataloged", "ignored", "standard-gap")
+            },
+        }
+
+
+def _repository_files(root: Path) -> list[Path]:
+    proc = subprocess.run(
+        ["git", "-C", str(root), "ls-files", "--cached", "--others", "--exclude-standard", "-z"],
+        capture_output=True,
+        timeout=30,
+        check=False,
+    )
+    if proc.returncode == 0:
+        candidates = [root / item for item in proc.stdout.decode().split("\0") if item]
+    else:
+        candidates = list(root.rglob("*"))
+    return sorted(
+        path
+        for path in candidates
+        if path.is_file()
+        and not set(path.relative_to(root).parts) & SKIP_PARTS
+        and "docs/archive" not in path.relative_to(root).as_posix()
+        and path.relative_to(root).as_posix() not in SKIP_PATHS
+    )
+
+
+def _digest(value: str) -> str:
+    return hashlib.sha256(value.encode("utf-8")).hexdigest()
+
+
+def _identifier(kind: str, source: str, locator: str) -> str:
+    return f"{kind}:{source}:{locator}"
+
+
+def _item(
+    kind: str,
+    name: str,
+    source: str,
+    locator: str,
+    adapter: str,
+    evidence: str,
+) -> dict[str, str]:
+    return {
+        "id": _identifier(kind, source, locator),
+        "kind": kind,
+        "name": name,
+        "source": source,
+        "locator": locator,
+        "adapter": adapter,
+        "digest": _digest(evidence),
+    }
+
+
+def _python_evidence(text: str, node: ast.AST) -> str:
+    """Return source-bound evidence that is stable across CPython AST versions."""
+    parts = []
+    decorators = getattr(node, "decorator_list", ())
+    for decorator in decorators:
+        segment = ast.get_source_segment(text, decorator)
+        if segment is not None:
+            parts.append(segment)
+    segment = ast.get_source_segment(text, node)
+    if segment is not None:
+        parts.append(segment)
+    return "\n".join(parts)
+
+
+def _python_items(path: str, text: str) -> list[dict[str, str]]:
+    try:
+        tree = ast.parse(text, filename=path)
+    except SyntaxError:
+        return []
+    items: list[dict[str, str]] = []
+    is_test = Path(path).name.startswith("test_") or "/tests/" in f"/{path}"
+    if not is_test and Path(path).name not in PYTHON_TOOLING_MODULES:
+        for node in tree.body:
+            if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) and not node.name.startswith("_"):
+                items.append(_item("api-symbol", node.name, path, node.name, "python-ast", _python_evidence(text, node)))
+            if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+                decorators = [ast.unparse(item).lower() for item in node.decorator_list]
+                if any("tool" in item for item in decorators):
+                    items.append(_item("mcp-tool", node.name, path, node.name, "python-ast", _python_evidence(text, node)))
+                for decorator in node.decorator_list:
+                    if not isinstance(decorator, ast.Call) or not isinstance(
+                        decorator.func, ast.Attribute
+                    ):
+                        continue
+                    if decorator.func.attr != "command":
+                        continue
+                    command_name = node.name
+                    if (
+                        decorator.args
+                        and isinstance(decorator.args[0], ast.Constant)
+                        and isinstance(decorator.args[0].value, str)
+                    ):
+                        command_name = decorator.args[0].value
+                    items.append(
+                        _item(
+                            "cli-command",
+                            command_name,
+                            path,
+                            command_name,
+                            "python-cli-framework",
+                            _python_evidence(text, node),
+                        )
+                    )
+            if isinstance(node, ast.ClassDef) and any(
+                "basesettings" in ast.unparse(base).lower() for base in node.bases
+            ):
+                for field in node.body:
+                    if isinstance(field, (ast.Assign, ast.AnnAssign)):
+                        target = field.target if isinstance(field, ast.AnnAssign) else field.targets[0]
+                        if isinstance(target, ast.Name) and not target.id.startswith("_"):
+                            items.append(
+                                _item(
+                                    "config-key",
+                                    target.id,
+                                    path,
+                                    f"{node.name}.{target.id}",
+                                    "python-settings-ast",
+                                    _python_evidence(text, field),
+                                )
+                            )
+    for candidate in ast.walk(tree):
+        if not isinstance(candidate, ast.Call) or not isinstance(candidate.func, ast.Attribute):
+            continue
+        if not candidate.args or not isinstance(candidate.args[0], ast.Constant) or not isinstance(candidate.args[0].value, str):
+            continue
+        value = candidate.args[0].value
+        if candidate.func.attr == "add_parser":
+            items.append(_item("cli-command", value, path, value, "argparse-ast", _python_evidence(text, candidate)))
+        elif candidate.func.attr == "add_argument" and value.startswith("-"):
+            items.append(_item("cli-option", value, path, value, "argparse-ast", _python_evidence(text, candidate)))
+    return items
+
+
+def _structured(path: Path, text: str) -> Any:
+    try:
+        if path.suffix.lower() == ".json":
+            return json.loads(text)
+        if path.suffix.lower() in {".yaml", ".yml"}:
+            return yaml.safe_load(text)
+        if path.name == "pyproject.toml":
+            return tomllib.loads(text)
+    except (json.JSONDecodeError, yaml.YAMLError, tomllib.TOMLDecodeError):
+        return None
+    return None
+
+
+def _structured_items(path: str, data: Any) -> list[dict[str, str]]:
+    if not isinstance(data, dict):
+        return []
+    items: list[dict[str, str]] = []
+    if path == "pyproject.toml" and isinstance(data.get("project"), dict):
+        project = data["project"]
+        name = str(project.get("name", "Python package"))
+        version = str(project.get("version", "unknown"))
+        items.append(_item("package", name, path, "project", "python-package", f"{name}:{version}"))
+        scripts = project.get("scripts")
+        if isinstance(scripts, dict):
+            for command, target in sorted(scripts.items()):
+                items.append(
+                    _item(
+                        "cli-command",
+                        str(command),
+                        path,
+                        f"project.scripts.{command}",
+                        "python-package",
+                        str(target),
+                    )
+                )
+        requires_python = project.get("requires-python")
+        if isinstance(requires_python, str):
+            items.append(
+                _item(
+                    "runtime-constraint",
+                    f"Python {requires_python}",
+                    path,
+                    "project.requires-python",
+                    "python-package",
+                    requires_python,
+                )
+            )
+    if Path(path).name == "package.json":
+        name = str(data.get("name", "JavaScript package"))
+        version = str(data.get("version", "unknown"))
+        items.append(_item("package", name, path, "package", "node-package", f"{name}:{version}"))
+        if data.get("type") == "module":
+            items.append(
+                _item(
+                    "runtime-constraint",
+                    "ES modules",
+                    path,
+                    "type",
+                    "node-package",
+                    "module",
+                )
+            )
+        engines = data.get("engines")
+        if isinstance(engines, dict):
+            for runtime, constraint in sorted(engines.items()):
+                items.append(
+                    _item(
+                        "runtime-constraint",
+                        f"{runtime} {constraint}",
+                        path,
+                        f"engines.{runtime}",
+                        "node-package",
+                        str(constraint),
+                    )
+                )
+        scripts = data.get("scripts")
+        if isinstance(scripts, dict):
+            for script, command in sorted(scripts.items()):
+                if str(script).startswith("//"):
+                    continue
+                kind = "test-runner" if script == "test" or script.startswith("test:") else "package-script"
+                items.append(_item(kind, str(script), path, f"scripts.{script}", "node-package", str(command)))
+        binaries = data.get("bin")
+        if isinstance(binaries, str):
+            items.append(_item("cli-command", name, path, "bin", "node-package", binaries))
+        elif isinstance(binaries, dict):
+            for command, target in sorted(binaries.items()):
+                items.append(_item("cli-command", str(command), path, f"bin.{command}", "node-package", str(target)))
+    if "openapi" in data and isinstance(data.get("paths"), dict):
+        for route, operations in sorted(data["paths"].items()):
+            if not isinstance(operations, dict):
+                continue
+            for method in sorted(set(operations) & HTTP_METHODS):
+                locator = f"{method.upper()} {route}"
+                items.append(_item("api-endpoint", locator, path, locator, "openapi", json.dumps(operations[method], sort_keys=True)))
+    if "$schema" in data or path.endswith(".schema.json"):
+        name = str(data.get("title") or data.get("$id") or Path(path).stem)
+        items.append(_item("schema", name, path, name, "json-schema", json.dumps(data, sort_keys=True)))
+        properties = data.get("properties")
+        if isinstance(properties, dict):
+            for key, value in sorted(properties.items()):
+                items.append(
+                    _item(
+                        "config-key",
+                        str(key),
+                        path,
+                        f"properties.{key}",
+                        "json-schema",
+                        json.dumps(value, sort_keys=True),
+                    )
+                )
+    return items
+
+
+def _coverage_ignores(root: Path, identifiers: set[str]) -> dict[str, str]:
+    ignore_path = root / ".sourcebound-ignore.yml"
+    if not ignore_path.exists():
+        return {}
+    try:
+        raw = yaml.safe_load(ignore_path.read_text(encoding="utf-8"))
+    except OSError as exc:
+        raise ConfigurationError(f"cannot read coverage policy {ignore_path}") from exc
+    except yaml.YAMLError as exc:
+        raise ConfigurationError(f"invalid coverage policy {ignore_path}") from exc
+    if not isinstance(raw, dict) or set(raw) != {"version", "ignore"}:
+        raise ConfigurationError("coverage policy must contain only version and ignore")
+    if raw["version"] != 1 or not isinstance(raw["ignore"], list):
+        raise ConfigurationError("coverage policy must use version 1 and an ignore list")
+    ignored: dict[str, str] = {}
+    for index, record in enumerate(raw["ignore"]):
+        if not isinstance(record, dict) or set(record) != {"id", "reason"}:
+            raise ConfigurationError(f"coverage ignore {index} must contain id and reason")
+        identifier = record["id"]
+        reason = record["reason"]
+        if not isinstance(identifier, str) or identifier not in identifiers:
+            raise ConfigurationError(f"coverage ignore {index} names an unknown inventory id")
+        if not isinstance(reason, str) or len(reason.strip()) < 12:
+            raise ConfigurationError(f"coverage ignore {index} needs a specific reason")
+        if identifier in ignored:
+            raise ConfigurationError(f"coverage ignore {index} duplicates an inventory id")
+        ignored[identifier] = reason.strip()
+    return ignored
+
+
+def _coverage(root: Path, identifiers: set[str]) -> tuple[set[tuple[str, str]], bool, dict[str, str]]:
+    direct_locators: set[tuple[str, str]] = set()
+    cataloged = False
+    manifest = root / ".sourcebound.yml"
+    if manifest.exists():
+        try:
+            data = yaml.safe_load(manifest.read_text(encoding="utf-8"))
+            bindings = data.get("bindings", []) if isinstance(data, dict) else []
+            for binding in bindings if isinstance(bindings, list) else []:
+                if not isinstance(binding, dict):
+                    continue
+                source = binding.get("source")
+                source_data = source if isinstance(source, dict) else {}
+                path = source_data.get("path")
+                if binding.get("extractor") in {
+                    "repository-inventory",
+                    "repository-overview",
+                } and path == ".":
+                    cataloged = True
+                elif isinstance(path, str):
+                    locator = source_data.get("symbol") or source_data.get("pointer")
+                    if isinstance(locator, str):
+                        direct_locators.add((Path(path).as_posix(), locator))
+        except (OSError, yaml.YAMLError):
+            pass
+    return direct_locators, cataloged, _coverage_ignores(root, identifiers)
+
+
+def scan_inventory(root: Path) -> InventoryReport:
+    root = root.resolve()
+    raw_items: list[dict[str, str]] = []
+    languages: set[str] = set()
+    for file_path in _repository_files(root):
+        relative = file_path.relative_to(root).as_posix()
+        language = LANGUAGES.get(file_path.suffix.lower())
+        if language:
+            languages.add(language)
+        try:
+            text = file_path.read_text(encoding="utf-8")
+        except (OSError, UnicodeDecodeError):
+            continue
+        if file_path.suffix.lower() == ".py":
+            raw_items.extend(_python_items(relative, text))
+        if file_path.suffix.lower() in {".ts", ".tsx", ".js", ".jsx"}:
+            adapter = (
+                "typescript-static"
+                if file_path.suffix.lower() in {".ts", ".tsx"}
+                else "javascript-static"
+            )
+            for name in TS_EXPORT.findall(text):
+                raw_items.append(_item("api-symbol", name, relative, name, adapter, name))
+            for name in TS_CLI_COMMAND.findall(text):
+                raw_items.append(
+                    _item("cli-command", name, relative, name, "typescript-cli-framework", name)
+                )
+            for name in TS_CLI_OPTION.findall(text):
+                raw_items.append(
+                    _item("cli-option", name, relative, name, "typescript-cli-framework", name)
+                )
+            for name in TS_MCP_TOOL.findall(text):
+                raw_items.append(_item("mcp-tool", name, relative, name, "typescript-mcp", name))
+        data = _structured(file_path, text)
+        if data is not None:
+            raw_items.extend(_structured_items(relative, data))
+        if file_path.suffix.lower() == ".md":
+            raw_items.append(_item("document", relative, relative, "document", "markdown", text))
+            for line_number, line in enumerate(text.splitlines(), start=1):
+                for target in LINK.findall(line):
+                    raw_items.append(_item("doc-link", target, relative, f"line {line_number}", "markdown-links", target))
+        if (
+            file_path.name.startswith("test_") and file_path.suffix == ".py"
+        ) or file_path.name.endswith((".test.ts", ".spec.ts", ".test.js", ".spec.js")):
+            raw_items.append(_item("test-suite", relative, relative, "file", "test-files", text))
+    declaration_symbols = {
+        (item["name"], item["source"][:-5])
+        for item in raw_items
+        if item["kind"] == "api-symbol" and item["source"].endswith(".d.ts")
+    }
+    raw_items = [
+        item
+        for item in raw_items
+        if not (
+            item["kind"] == "api-symbol"
+            and not item["source"].endswith(".d.ts")
+            and (item["name"], str(Path(item["source"]).with_suffix("")))
+            in declaration_symbols
+        )
+    ]
+    unique = {item["id"]: item for item in raw_items}
+    direct_locators, cataloged, ignored = _coverage(root, set(unique))
+    items = []
+    for identifier in sorted(unique):
+        item = unique[identifier]
+        if identifier in ignored:
+            coverage = "ignored"
+        elif (item["source"], item["locator"]) in direct_locators:
+            coverage = "bound"
+        elif cataloged:
+            coverage = "cataloged"
+        else:
+            coverage = "standard-gap"
+        items.append(
+            InventoryItem(
+                **item,
+                coverage=coverage,
+                coverage_reason=ignored.get(identifier),
+            )
+        )
+    return InventoryReport(tuple(sorted(languages)), tuple(items))

clean_docs/isolation.py

@@ -0,0 +1,157 @@
+"""Run declared processes in disposable repository copies with bounded I/O."""
+
+from __future__ import annotations
+
+import os
+import shutil
+import subprocess
+import tempfile
+import threading
+from dataclasses import dataclass
+from pathlib import Path
+
+from clean_docs.errors import ExtractionError
+from clean_docs.snapshot import RepositorySnapshot
+from clean_docs.write_gate import redact_secrets
+
+
+MAX_PROCESS_IO_BYTES = 1_000_000
+COPY_IGNORES = (".git", ".venv", "node_modules", "__pycache__")
+
+
+@dataclass(frozen=True)
+class IsolatedProcessResult:
+    returncode: int
+    stdout: str
+    stderr: str
+
+
+def _isolated_copy(source: Path, destination: Path) -> None:
+    shutil.copytree(
+        source,
+        destination,
+        symlinks=True,
+        ignore=shutil.ignore_patterns(*COPY_IGNORES),
+    )
+    for path in destination.rglob("*"):
+        # Known limit by design: reject every symlink rather than preserving one
+        # that could resolve outside this disposable snapshot. Repositories that
+        # require symlinks must use --no-exec or static paths instead.
+        if path.is_symlink():
+            raise ExtractionError(
+                "process snapshot contains a symbolic link: "
+                f"{path.relative_to(destination)}"
+            )
+
+
+def _safe_output(label: str, stdout: bytes, stderr: bytes) -> IsolatedProcessResult:
+    rendered_stdout = stdout.decode("utf-8", errors="replace")
+    rendered_stderr = stderr.decode("utf-8", errors="replace")
+    _, stdout_flags = redact_secrets(rendered_stdout)
+    _, stderr_flags = redact_secrets(rendered_stderr)
+    flags = (*stdout_flags, *stderr_flags)
+    if flags:
+        raise ExtractionError(
+            f"{label} output contains secret-like data ({flags[0]})"
+        )
+    return IsolatedProcessResult(0, rendered_stdout, rendered_stderr)
+
+
+def _sandbox_environment(home: Path, temp: Path) -> dict[str, str]:
+    return {
+        "HOME": str(home),
+        "TMPDIR": str(temp),
+        "PATH": os.environ.get("PATH", ""),
+        "NO_COLOR": "1",
+    }
+
+
+def run_isolated_process(
+    snapshot: RepositorySnapshot,
+    argv: tuple[str, ...],
+    *,
+    label: str,
+    timeout_seconds: int,
+    input_text: str = "",
+) -> IsolatedProcessResult:
+    request = input_text.encode("utf-8")
+    if len(request) > MAX_PROCESS_IO_BYTES:
+        raise ExtractionError(
+            f"{label} input exceeds {MAX_PROCESS_IO_BYTES} bytes"
+        )
+    with snapshot.materialized_root() as source, tempfile.TemporaryDirectory(
+        prefix="sourcebound-process-"
+    ) as temporary:
+        sandbox = Path(temporary)
+        worktree = sandbox / "repository"
+        _isolated_copy(source, worktree)
+        home = sandbox / "home"
+        temp = sandbox / "tmp"
+        home.mkdir()
+        temp.mkdir()
+        environment = _sandbox_environment(home, temp)
+        try:
+            process = subprocess.Popen(
+                argv,
+                cwd=worktree,
+                env=environment,
+                stdin=subprocess.PIPE,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+            )
+        except OSError as exc:
+            raise ExtractionError(f"{label} failed to run: {exc}") from exc
+
+        stdout = bytearray()
+        stderr = bytearray()
+        total = 0
+        lock = threading.Lock()
+        exceeded = threading.Event()
+
+        def drain(stream: object, destination: bytearray) -> None:
+            nonlocal total
+            reader = stream
+            while True:
+                chunk = reader.read(64 * 1024)  # type: ignore[attr-defined]
+                if not chunk:
+                    return
+                with lock:
+                    total += len(chunk)
+                    if total > MAX_PROCESS_IO_BYTES:
+                        exceeded.set()
+                        process.kill()
+                        return
+                    destination.extend(chunk)
+
+        assert process.stdout is not None
+        assert process.stderr is not None
+        readers = (
+            threading.Thread(target=drain, args=(process.stdout, stdout), daemon=True),
+            threading.Thread(target=drain, args=(process.stderr, stderr), daemon=True),
+        )
+        for reader in readers:
+            reader.start()
+        try:
+            assert process.stdin is not None
+            process.stdin.write(request)
+            process.stdin.close()
+            process.wait(timeout=timeout_seconds)
+        except subprocess.TimeoutExpired as exc:
+            process.kill()
+            process.wait()
+            raise ExtractionError(
+                f"{label} timed out after {timeout_seconds} seconds"
+            ) from exc
+        except OSError as exc:
+            process.kill()
+            process.wait()
+            raise ExtractionError(f"{label} failed during execution: {exc}") from exc
+        finally:
+            for reader in readers:
+                reader.join(timeout=5)
+        if exceeded.is_set():
+            raise ExtractionError(
+                f"{label} output exceeds {MAX_PROCESS_IO_BYTES} bytes"
+            )
+        result = _safe_output(label, bytes(stdout), bytes(stderr))
+        return IsolatedProcessResult(process.returncode, result.stdout, result.stderr)