sourcebound 1.2.1__py3-none-any.whl

clean_docs/execution.py

@@ -0,0 +1,138 @@
+"""Resolve explicit runtime tokens in allowlisted command arrays."""
+
+from __future__ import annotations
+
+import os
+import signal
+import subprocess
+import sys
+import tempfile
+import threading
+import time
+from dataclasses import dataclass
+from enum import Enum
+from pathlib import Path
+
+from clean_docs.errors import ConfigurationError
+
+
+PYTHON_EXECUTABLE_TOKEN = "{python}"
+
+
+class ExecutionPolicy(str, Enum):
+    TRUSTED = "trusted"
+    STATIC_ONLY = "static-only"
+
+
+def resolve_argv(argv: tuple[str, ...]) -> tuple[str, ...]:
+    if argv and argv[0] == PYTHON_EXECUTABLE_TOKEN:
+        return (sys.executable, *argv[1:])
+    return argv
+
+
+@dataclass(frozen=True)
+class BoundedCommandResult:
+    stdout: bytes
+    stderr: bytes
+    duration_seconds: float
+
+
+def run_bounded_command(
+    argv: tuple[str, ...],
+    *,
+    environment: dict[str, str],
+    input_bytes: bytes,
+    timeout_seconds: int,
+    max_input_bytes: int,
+    max_output_bytes: int,
+    prefix: str,
+) -> BoundedCommandResult:
+    """Run an operator-selected command in a disposable directory with bounded I/O."""
+    if len(input_bytes) > max_input_bytes:
+        raise ConfigurationError(f"{prefix} input exceeds {max_input_bytes} bytes")
+    with tempfile.TemporaryDirectory(prefix=f"{prefix}-") as temporary:
+        start = time.monotonic()
+        try:
+            process = subprocess.Popen(
+                resolve_argv(argv),
+                cwd=Path(temporary),
+                env=environment,
+                stdin=subprocess.PIPE,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                start_new_session=True,
+            )
+        except OSError as exc:
+            raise ConfigurationError(f"{prefix} failed to start: {exc}") from exc
+        stdout = bytearray()
+        stderr = bytearray()
+        total = 0
+        lock = threading.Lock()
+        exceeded = threading.Event()
+
+        def stop_process() -> None:
+            if process.poll() is not None:
+                return
+            try:
+                # The configured command can start children. Killing its process group
+                # keeps an inherited pipe from outliving the deadline and temp directory.
+                os.killpg(process.pid, signal.SIGKILL)
+            except (AttributeError, OSError):
+                process.kill()
+
+        def drain(stream: object, destination: bytearray) -> None:
+            nonlocal total
+            while True:
+                chunk = stream.read(64 * 1024)  # type: ignore[attr-defined]
+                if not chunk:
+                    return
+                with lock:
+                    total += len(chunk)
+                    if total > max_output_bytes:
+                        exceeded.set()
+                        stop_process()
+                        return
+                    destination.extend(chunk)
+
+        def supply_input() -> None:
+            try:
+                assert process.stdin is not None
+                process.stdin.write(input_bytes)
+                process.stdin.close()
+            except (BrokenPipeError, OSError):
+                # A nonzero exit is reported from the process status below. The writer
+                # must not hide that status or block the timeout path.
+                return
+
+        assert process.stdin is not None and process.stdout is not None and process.stderr is not None
+        readers = (
+            threading.Thread(target=drain, args=(process.stdout, stdout), daemon=True),
+            threading.Thread(target=drain, args=(process.stderr, stderr), daemon=True),
+        )
+        for reader in readers:
+            reader.start()
+        writer = threading.Thread(target=supply_input, daemon=True)
+        writer.start()
+        try:
+            process.wait(timeout=timeout_seconds)
+        except subprocess.TimeoutExpired as exc:
+            stop_process()
+            process.wait()
+            raise ConfigurationError(f"{prefix} timed out after {timeout_seconds} seconds") from exc
+        except OSError as exc:
+            stop_process()
+            process.wait()
+            raise ConfigurationError(f"{prefix} failed during execution: {exc}") from exc
+        finally:
+            writer.join(timeout=5)
+            for reader in readers:
+                reader.join(timeout=5)
+        if exceeded.is_set():
+            raise ConfigurationError(f"{prefix} output exceeds {max_output_bytes} bytes")
+        if process.returncode != 0:
+            raise ConfigurationError(f"{prefix} exited {process.returncode}")
+        return BoundedCommandResult(
+            stdout=bytes(stdout),
+            stderr=bytes(stderr),
+            duration_seconds=time.monotonic() - start,
+        )

clean_docs/explain.py

@@ -0,0 +1,123 @@
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass
+from pathlib import Path
+
+from clean_docs.errors import ConfigurationError
+from clean_docs.inventory import scan_inventory
+
+
+RULES = {
+    "audience": (
+        "The document addresses a future executor instead of its product reader.",
+        "Rewrite it for the reader's current task or move process state under docs/archive.",
+    ),
+    "broken-local-link": (
+        "A relative Markdown link does not resolve from its document.",
+        "Fix the target path or remove the link.",
+    ),
+    "cross-project-residue": (
+        "Tracked content contains a token reserved for another repository.",
+        "Replace the token with this repository's term or remove the copied residue.",
+    ),
+    "doc-length": (
+        "A document exceeds the packaged line budget.",
+        "Split distinct reader tasks or add a reasoned canonical-reference allowance.",
+    ),
+    "generated-artifact": (
+        "Generated runtime residue is tracked as product source.",
+        "Remove the artifact and add its pattern to the repository ignore file.",
+    ),
+    "local-path-residue": (
+        "Tracked content contains a machine-specific home path.",
+        "Replace it with a repository-relative or portable path.",
+    ),
+    "near-duplicate": (
+        "Two reader-facing documents carry substantially the same content.",
+        "Keep one canonical explanation and link to it from the other task surface.",
+    ),
+    "process-artifact": (
+        "A status, handoff, plan, or report remains on the reader-facing surface.",
+        "Move process history under docs/archive and link only if readers need it.",
+    ),
+    "purpose-contract": (
+        "The document does not open with one marked BLUF purpose contract.",
+        "Put a plain prose block after the H1 that names who should read, the problem, and the resulting capability.",
+    ),
+    "prohibited-booster": (
+        "The prose uses a prohibited booster instead of a verifiable claim.",
+        "Remove the booster and state the measured property directly.",
+    ),
+    "provenance": (
+        "Reader-facing reference text contains authoring receipts or run provenance.",
+        "Move the receipt to history and leave current truth in the reference.",
+    ),
+    "restatement": (
+        "A fact is restated across reader-facing documents without one canonical home.",
+        "Keep the fact in one document and replace sibling copies with links.",
+    ),
+    "section-length": (
+        "A section exceeds the packaged line budget.",
+        "Split the section by reader task or add a reasoned canonical-reference allowance.",
+    ),
+    "unreadable-document": (
+        "A tracked Markdown path cannot be read from the current worktree.",
+        "Restore the document, remove the stale tracked path, or fix its permissions.",
+    ),
+}
+
+
+@dataclass(frozen=True)
+class Explanation:
+    id: str
+    kind: str
+    state: str
+    summary: str
+    evidence: dict[str, str]
+    repair: str
+    required: bool
+
+    def as_dict(self) -> dict[str, object]:
+        return asdict(self)
+
+
+def explain(root: Path, identifier: str) -> Explanation:
+    if identifier in RULES:
+        summary, repair = RULES[identifier]
+        return Explanation(identifier, "policy-rule", "required", summary, {}, repair, True)
+    item = next(
+        (candidate for candidate in scan_inventory(root).items if candidate.id == identifier),
+        None,
+    )
+    if item is None:
+        raise ConfigurationError(f"unknown finding or inventory id: {identifier}")
+    evidence = {
+        "source": item.source,
+        "locator": item.locator,
+        "adapter": item.adapter,
+        "sha256": item.digest,
+    }
+    if item.coverage == "bound":
+        summary = "The detected surface has a source-specific documentation binding."
+        repair = "No repair is required while the binding remains current."
+    elif item.coverage == "cataloged":
+        summary = (
+            "The detected surface is tracked by a repository catalog, but has no "
+            "source-specific documentation binding."
+        )
+        repair = "Add a source-specific binding if readers depend on this surface."
+    elif item.coverage == "ignored":
+        summary = f"The detected surface is ignored by policy: {item.coverage_reason}"
+        repair = "Remove the reasoned ignore when this surface becomes reader-facing."
+    else:
+        summary = "The detected surface has evidence but no binding or reasoned ignore."
+        repair = "Add a source binding or a specific record in .sourcebound-ignore.yml."
+    return Explanation(
+        item.id,
+        "inventory-surface",
+        item.coverage,
+        summary,
+        evidence,
+        repair,
+        False,
+    )

clean_docs/extractors/__init__.py

@@ -0,0 +1,19 @@
+from clean_docs.extractors.json_pointer import extract_json_pointer
+from clean_docs.extractors.inventory import (
+    extract_repository_inventory,
+    extract_repository_overview,
+)
+from clean_docs.extractors.python_literal import extract_python_literal
+from clean_docs.extractors.static import extract_file, extract_paths, extract_structured
+
+__all__ = [
+    "extract_command",
+    "extract_file",
+    "extract_json_pointer",
+    "extract_paths",
+    "extract_python_literal",
+    "extract_repository_inventory",
+    "extract_repository_overview",
+    "extract_structured",
+]
+from clean_docs.extractors.command import extract_command

clean_docs/extractors/command.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from typing import Any
+
+from clean_docs.errors import ExtractionError
+from clean_docs.execution import resolve_argv
+from clean_docs.isolation import run_isolated_process
+from clean_docs.models import CommandSpec, EvidenceValue, Provenance
+from clean_docs.snapshot import RepositorySnapshot
+
+
+def _select(value: Any, path: str) -> Any:
+    current = value
+    for token in path.removeprefix("$.").split("."):
+        if not isinstance(current, dict) or token not in current:
+            raise ExtractionError(f"command JSON path does not resolve: {path}")
+        current = current[token]
+    return current
+
+
+def extract_command(
+    snapshot: RepositorySnapshot, command: CommandSpec, json_path: str
+) -> EvidenceValue:
+    proc = run_isolated_process(
+        snapshot,
+        resolve_argv(command.argv),
+        label=f"command {command.id}",
+        timeout_seconds=command.timeout_seconds,
+    )
+    if proc.returncode != 0:
+        detail = proc.stderr.strip() or proc.stdout.strip()
+        raise ExtractionError(f"command {command.id} exited {proc.returncode}: {detail[:500]}")
+    try:
+        payload = json.loads(proc.stdout)
+    except json.JSONDecodeError as exc:
+        raise ExtractionError(f"command {command.id} did not return JSON: {exc}") from exc
+    value = _select(payload, json_path)
+    normalized = json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
+    return EvidenceValue(
+        kind="scalar",
+        value=value,
+        provenance=Provenance(
+            ref=snapshot.label,
+            path="<command>",
+            locator=command.id,
+            extractor="command@1",
+            digest=hashlib.sha256(normalized.encode()).hexdigest(),
+        ),
+    )

clean_docs/extractors/inventory.py

@@ -0,0 +1,176 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from pathlib import Path
+
+from clean_docs.inventory import InventoryItem, scan_inventory
+from clean_docs.models import EvidenceValue, Provenance, RegionBinding
+from clean_docs.snapshot import RepositorySnapshot
+
+
+INCLUDED_KINDS = {
+    "api-endpoint",
+    "api-symbol",
+    "cli-command",
+    "cli-option",
+    "mcp-tool",
+    "package",
+    "package-script",
+    "runtime-constraint",
+    "schema",
+    "test-runner",
+    "test-suite",
+}
+
+
+def _inventory_rows_from_items(
+    items: tuple[InventoryItem, ...],
+) -> list[dict[str, str]]:
+    rows = [
+        {
+            "kind": item.kind,
+            "name": item.name,
+            "source": item.source,
+            "locator": item.locator,
+            "adapter": item.adapter,
+            "digest": item.digest,
+        }
+        for item in items
+        if item.kind in INCLUDED_KINDS and not item.adapter.startswith("plugin:")
+    ]
+    return sorted(
+        rows,
+        key=lambda item: (
+            item["kind"],
+            item["name"],
+            item["source"],
+            item["locator"],
+            item["adapter"],
+            item["digest"],
+        ),
+    )
+
+
+def _inventory_rows(root: Path) -> list[dict[str, str]]:
+    return _inventory_rows_from_items(scan_inventory(root).items)
+
+
+def extract_repository_inventory(
+    snapshot: RepositorySnapshot, binding: RegionBinding
+) -> EvidenceValue:
+    with snapshot.materialized_root() as root:
+        inventory_rows = _inventory_rows(root)
+    rows = [
+        {
+            "kind": item["kind"],
+            "name": item["name"],
+            "source": item["source"],
+            "locator": item["locator"],
+        }
+        for item in inventory_rows
+    ]
+    normalized = json.dumps(rows, sort_keys=True, separators=(",", ":"))
+    return EvidenceValue(
+        kind="table",
+        value=rows,
+        provenance=Provenance(
+            ref=snapshot.label,
+            path=".",
+            locator="public-surface",
+            extractor="repository-inventory@1",
+            digest=hashlib.sha256(normalized.encode("utf-8")).hexdigest(),
+        ),
+    )
+
+
+def _inline(value: str) -> str:
+    return (
+        value.replace("\\", "\\\\")
+        .replace("|", "\\|")
+        .replace("`", "'")
+        .replace("\n", " ")
+    )
+
+
+def _extract_repository_overview(
+    snapshot: RepositorySnapshot,
+    *,
+    include_item_digests: bool,
+    extractor: str,
+    inventory_items: tuple[InventoryItem, ...] | None = None,
+) -> EvidenceValue:
+    if inventory_items is None:
+        with snapshot.materialized_root() as root:
+            inventory_rows = _inventory_rows(root)
+    else:
+        inventory_rows = _inventory_rows_from_items(inventory_items)
+    by_kind: dict[str, list[str]] = {}
+    for item in inventory_rows:
+        by_kind.setdefault(item["kind"], []).append(item["name"])
+    lines = [
+        "| surface | discovered | examples |",
+        "| --- | ---: | --- |",
+    ]
+    for kind in sorted(by_kind):
+        names = sorted(set(by_kind[kind]))
+        examples = ", ".join(f"`{_inline(name)}`" for name in names[:3])
+        if len(by_kind[kind]) > 3:
+            examples += f", and {len(by_kind[kind]) - 3} more"
+        lines.append(f"| {_inline(kind)} | {len(by_kind[kind])} | {examples} |")
+    receipt_rows = (
+        inventory_rows
+        if include_item_digests
+        else [
+            {
+                "kind": item["kind"],
+                "name": item["name"],
+                "source": item["source"],
+                "locator": item["locator"],
+                "adapter": item["adapter"],
+            }
+            for item in inventory_rows
+        ]
+    )
+    normalized = json.dumps(receipt_rows, sort_keys=True, separators=(",", ":"))
+    digest = hashlib.sha256(normalized.encode("utf-8")).hexdigest()
+    lines.extend(("", f"<!-- sourcebound:inventory-sha256 {digest} -->"))
+    return EvidenceValue(
+        kind="markdown",
+        value="\n".join(lines),
+        provenance=Provenance(
+            ref=snapshot.label,
+            path=".",
+            locator="public-surface-overview",
+            extractor=extractor,
+            digest=digest,
+        ),
+    )
+
+
+def extract_repository_overview(
+    snapshot: RepositorySnapshot,
+    binding: RegionBinding,
+    *,
+    inventory_items: tuple[InventoryItem, ...] | None = None,
+) -> EvidenceValue:
+    return _extract_repository_overview(
+        snapshot,
+        include_item_digests=False,
+        extractor="repository-overview@2",
+        inventory_items=inventory_items,
+    )
+
+
+def _extract_repository_overview_legacy(
+    snapshot: RepositorySnapshot,
+    binding: RegionBinding,
+    *,
+    inventory_items: tuple[InventoryItem, ...] | None = None,
+) -> EvidenceValue:
+    return _extract_repository_overview(
+        snapshot,
+        include_item_digests=True,
+        extractor="repository-overview@1",
+        inventory_items=inventory_items,
+    )

clean_docs/extractors/json_pointer.py

@@ -0,0 +1,84 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from typing import Any
+
+from clean_docs.errors import ExtractionError
+from clean_docs.models import EvidenceValue, Provenance, RegionBinding
+from clean_docs.snapshot import RepositorySnapshot
+
+
+def _decode(token: str) -> str:
+    decoded = []
+    index = 0
+    while index < len(token):
+        if token[index] != "~":
+            decoded.append(token[index])
+            index += 1
+            continue
+        if index + 1 >= len(token) or token[index + 1] not in {"0", "1"}:
+            raise ExtractionError(f"invalid JSON Pointer token: {token!r}")
+        decoded.append("~" if token[index + 1] == "0" else "/")
+        index += 2
+    return "".join(decoded)
+
+
+def _resolve(value: Any, pointer: str) -> Any:
+    current = value
+    for raw_token in pointer.removeprefix("/").split("/"):
+        token = _decode(raw_token)
+        if isinstance(current, dict):
+            if token not in current:
+                raise ExtractionError(f"JSON Pointer {pointer!r} does not resolve")
+            current = current[token]
+        elif isinstance(current, list):
+            if not token.isdigit() or (len(token) > 1 and token.startswith("0")):
+                raise ExtractionError(f"JSON Pointer {pointer!r} does not resolve")
+            try:
+                index = int(token)
+                current = current[index]
+            except (ValueError, IndexError) as exc:
+                raise ExtractionError(f"JSON Pointer {pointer!r} does not resolve") from exc
+        else:
+            raise ExtractionError(f"JSON Pointer {pointer!r} traverses a scalar value")
+    return current
+
+
+def _rows(value: Any) -> list[dict[str, Any]]:
+    if isinstance(value, list) and all(isinstance(item, dict) for item in value):
+        return value
+    if isinstance(value, dict) and all(isinstance(item, dict) for item in value.values()):
+        rows = []
+        for key, item in value.items():
+            row = dict(item)
+            row.setdefault("key", key)
+            rows.append(row)
+        return rows
+    raise ExtractionError("markdown-table evidence must be a list or mapping of records")
+
+
+def extract_json_pointer(
+    snapshot: RepositorySnapshot, binding: RegionBinding
+) -> EvidenceValue:
+    if binding.source.pointer is None:
+        raise ExtractionError("json source requires a pointer")
+    text = snapshot.read_text(binding.source.path)
+    try:
+        document = json.loads(text)
+    except json.JSONDecodeError as exc:
+        raise ExtractionError(f"cannot parse {binding.source.path}: {exc}") from exc
+    value = _rows(_resolve(document, binding.source.pointer))
+    normalized = json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
+    digest = hashlib.sha256(normalized.encode("utf-8")).hexdigest()
+    return EvidenceValue(
+        kind="table",
+        value=value,
+        provenance=Provenance(
+            ref=snapshot.label,
+            path=binding.source.path.as_posix(),
+            locator=binding.source.pointer,
+            extractor="json@1",
+            digest=digest,
+        ),
+    )

clean_docs/extractors/python_literal.py

@@ -0,0 +1,104 @@
+from __future__ import annotations
+
+import ast
+import hashlib
+import json
+from typing import Any
+
+from clean_docs.errors import ExtractionError
+from clean_docs.models import EvidenceValue, Provenance, RegionBinding
+from clean_docs.snapshot import RepositorySnapshot
+
+
+def _evaluate(node: ast.AST) -> Any:
+    if isinstance(node, ast.Constant):
+        return node.value
+    if isinstance(node, ast.Dict):
+        keys = []
+        for key in node.keys:
+            if key is None:
+                raise ExtractionError("python-literal mappings cannot contain unpacking")
+            keys.append(key)
+        return {_evaluate(key): _evaluate(value) for key, value in zip(keys, node.values)}
+    if isinstance(node, (ast.List, ast.Tuple, ast.Set)):
+        return [_evaluate(item) for item in node.elts]
+    if isinstance(node, ast.Call):
+        if node.args:
+            raise ExtractionError("python-literal constructor calls may use keyword arguments only")
+        if any(keyword.arg is None for keyword in node.keywords):
+            raise ExtractionError("python-literal constructor calls cannot contain unpacking")
+        return {keyword.arg: _evaluate(keyword.value) for keyword in node.keywords}
+    if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add):
+        left, right = _evaluate(node.left), _evaluate(node.right)
+        if isinstance(left, str) and isinstance(right, str):
+            return left + right
+    if isinstance(node, ast.UnaryOp) and isinstance(node.op, (ast.USub, ast.UAdd)):
+        value = _evaluate(node.operand)
+        if isinstance(value, (int, float)):
+            return -value if isinstance(node.op, ast.USub) else value
+    raise ExtractionError(f"unsupported Python syntax in bound literal: {type(node).__name__}")
+
+
+def _find_assignment(tree: ast.Module, symbol: str) -> ast.AST:
+    for node in tree.body:
+        if isinstance(node, ast.Assign):
+            if any(isinstance(target, ast.Name) and target.id == symbol for target in node.targets):
+                return node.value
+        if isinstance(node, ast.AnnAssign):
+            if isinstance(node.target, ast.Name) and node.target.id == symbol and node.value:
+                return node.value
+    raise ExtractionError(f"Python symbol not found: {symbol}")
+
+
+def _rows(value: Any) -> list[dict[str, Any]]:
+    if isinstance(value, list) and all(isinstance(item, dict) for item in value):
+        return value
+    if isinstance(value, dict) and all(isinstance(item, dict) for item in value.values()):
+        rows = []
+        for key, item in value.items():
+            row = dict(item)
+            row.setdefault("key", key)
+            rows.append(row)
+        return rows
+    raise ExtractionError("markdown-table evidence must be a list or mapping of records")
+
+
+def extract_python_literal(
+    snapshot: RepositorySnapshot, binding: RegionBinding
+) -> EvidenceValue:
+    if binding.source.symbol is None:
+        raise ExtractionError("python-literal source requires a symbol")
+    text = snapshot.read_text(binding.source.path)
+    try:
+        tree = ast.parse(text, filename=binding.source.path.as_posix())
+    except SyntaxError as exc:
+        raise ExtractionError(f"cannot parse {binding.source.path}: {exc}") from exc
+    extracted = _evaluate(_find_assignment(tree, binding.source.symbol))
+    value: Any
+    kind: str
+    if binding.renderer == "markdown-table":
+        value = _rows(extracted)
+        kind = "table"
+    elif binding.renderer == "markdown-fragment" and isinstance(extracted, str):
+        value = extracted
+        kind = "markdown"
+    elif binding.renderer == "scalar" and not isinstance(extracted, (dict, list)):
+        value = extracted
+        kind = "scalar"
+    else:
+        raise ExtractionError(
+            f"{binding.renderer} cannot render the bound Python literal"
+        )
+    normalized = json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
+    digest = hashlib.sha256(normalized.encode("utf-8")).hexdigest()
+    return EvidenceValue(
+        kind=kind,
+        value=value,
+        provenance=Provenance(
+            ref=snapshot.label,
+            path=binding.source.path.as_posix(),
+            locator=binding.source.symbol,
+            extractor="python-literal@1",
+            digest=digest,
+        ),
+    )