sourcebound 1.2.1__py3-none-any.whl

clean_docs/projections.py

@@ -0,0 +1,255 @@
+"""Render and verify projections of the canonical documentation graph."""
+
+from __future__ import annotations
+
+import difflib
+import hashlib
+import os
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from urllib.parse import unquote
+
+from clean_docs.demo import load_demo_evidence, render_static_demo
+from clean_docs.emit import render_llms_txt
+from clean_docs.errors import ConfigurationError
+from clean_docs.models import BindingResult, ContextBundleProjection, Manifest, Provenance
+from clean_docs.regions import atomic_write
+from clean_docs.visuals import (
+    load_visual_record,
+    render_agent_visual,
+    render_human_visual,
+)
+
+
+LINK = re.compile(r"\[[^\]]+\]\(([^)\s]+)(?:\s+[^)]*)?\)")
+HTML_LINK = re.compile(r'href="([^"]+)"')
+HEADING = re.compile(r"^#{1,6}\s+(.+?)\s*$")
+CANONICAL_BLOCK = re.compile(
+    r"<!-- sourcebound:canonical .+? begin -->.*?"
+    r"<!-- sourcebound:canonical .+? end -->",
+    re.DOTALL,
+)
+
+
+@dataclass(frozen=True)
+class ProjectionSet:
+    source_ref: str
+    corpus_digest: str
+    files: dict[Path, str]
+    digests: dict[Path, str]
+
+
+def _read_documents(root: Path, manifest: Manifest) -> dict[str, bytes]:
+    documents: dict[str, bytes] = {}
+    for document in sorted({binding.doc.as_posix() for binding in manifest.bindings}):
+        try:
+            documents[document] = (root / document).read_bytes()
+        except OSError as exc:
+            raise ConfigurationError(
+                f"cannot project bound document {document}: {exc}"
+            ) from exc
+    return documents
+
+
+def _corpus_digest(documents: dict[str, bytes]) -> str:
+    digest = hashlib.sha256()
+    for path, content in sorted(documents.items()):
+        digest.update(path.encode("utf-8"))
+        digest.update(b"\0")
+        digest.update(hashlib.sha256(content).digest())
+    return digest.hexdigest()
+
+
+def _source_ref() -> str:
+    # A working-tree projection cannot embed HEAD without becoming stale when its own commit
+    # changes HEAD. The corpus digest identifies the exact bytes; immutable refs use a separate
+    # snapshot path when supported.
+    return "WORKTREE"
+
+
+def _relative_link(root: Path, output: Path, document: Path) -> str:
+    link = os.path.relpath(root / document, (root / output).parent).replace(os.sep, "/")
+    return link
+
+
+def _render_bundle(
+    root: Path,
+    bundle: ContextBundleProjection,
+    documents: dict[str, bytes],
+    source_ref: str,
+    corpus_digest: str,
+) -> str:
+    lines = [
+        f"# Context bundle: {bundle.id}",
+        "",
+        f"- Source ref: `{source_ref}`",
+        f"- Corpus sha256: `{corpus_digest}`",
+        "- Content: exact canonical document bytes",
+    ]
+    for document in bundle.include:
+        key = document.as_posix()
+        content = documents[key].decode("utf-8")
+        link = _relative_link(root, bundle.output, document)
+        digest = hashlib.sha256(documents[key]).hexdigest()
+        lines.extend([
+            "",
+            f"## Canonical document: {key}",
+            "",
+            f"- Source: [{key}]({link})",
+            f"- Content sha256: `{digest}`",
+            "",
+            f"<!-- sourcebound:canonical {key} begin -->",
+            content.rstrip(),
+            f"<!-- sourcebound:canonical {key} end -->",
+        ])
+    return "\n".join(lines) + "\n"
+
+
+def _slug(title: str) -> str:
+    value = re.sub(r"<[^>]+>", "", title).strip().lower()
+    value = re.sub(r"[^a-z0-9 _-]", "", value)
+    return re.sub(r"[ _]+", "-", value).strip("-")
+
+
+def _anchors(content: str) -> set[str]:
+    counts: dict[str, int] = {}
+    anchors: set[str] = set()
+    for line in content.splitlines():
+        match = HEADING.match(line)
+        if not match:
+            continue
+        base = _slug(match.group(1))
+        count = counts.get(base, 0)
+        counts[base] = count + 1
+        anchors.add(base if count == 0 else f"{base}-{count}")
+    anchors.update(re.findall(r'\bid="([^"]+)"', content))
+    return anchors
+
+
+def _verify_links(root: Path, files: dict[Path, str]) -> None:
+    source_paths = set(files)
+    for relative, content in files.items():
+        # Embedded canonical bytes retain links relative to their original page. The original
+        # page is checked separately; generated bundle metadata is checked at the bundle path.
+        checked_content = CANONICAL_BLOCK.sub("", content)
+        matches = [*LINK.finditer(checked_content), *HTML_LINK.finditer(checked_content)]
+        for match in matches:
+            raw = unquote(match.group(1))
+            if raw.startswith(("http://", "https://", "mailto:")):
+                continue
+            target_text, _, fragment = raw.partition("#")
+            target = (
+                relative
+                if not target_text
+                else Path(os.path.normpath((relative.parent / target_text).as_posix()))
+            )
+            if target.is_absolute() or ".." in target.parts:
+                resolved = (root / relative.parent / target_text).resolve()
+                try:
+                    target = resolved.relative_to(root)
+                except ValueError as exc:
+                    raise ConfigurationError(
+                        f"projection link escapes repository: {relative} -> {raw}"
+                    ) from exc
+            if target in source_paths:
+                target_content = files[target]
+            else:
+                path = root / target
+                if path.exists() and not fragment:
+                    continue
+                try:
+                    target_content = path.read_text(encoding="utf-8")
+                except OSError as exc:
+                    raise ConfigurationError(
+                        f"broken projection link: {relative} -> {raw}"
+                    ) from exc
+            if fragment and unquote(fragment) not in _anchors(target_content):
+                raise ConfigurationError(
+                    f"broken projection anchor: {relative} -> {raw}"
+                )
+
+
+def render_projections(root: Path, manifest: Manifest) -> ProjectionSet:
+    root = root.resolve()
+    if manifest.projections is None:
+        raise ConfigurationError("manifest does not configure projections")
+    documents = _read_documents(root, manifest)
+    corpus_digest = _corpus_digest(documents)
+    source_ref = _source_ref()
+    files: dict[Path, str] = {}
+    digests: dict[Path, str] = {}
+    llms = manifest.projections.llms_txt
+    if llms:
+        files[llms.output] = render_llms_txt(
+            manifest,
+            title=llms.title,
+            summary=llms.summary,
+            documents=documents,
+            output_path=root / llms.output,
+        )
+        digests[llms.output] = corpus_digest
+    for bundle in manifest.projections.bundles:
+        files[bundle.output] = _render_bundle(
+            root, bundle, documents, source_ref, corpus_digest
+        )
+        digests[bundle.output] = corpus_digest
+    demo = manifest.projections.demo
+    if demo:
+        evidence = load_demo_evidence(root / demo.evidence)
+        files[demo.output] = render_static_demo(evidence, demo.output)
+        digests[demo.output] = evidence.digest
+    for visual in manifest.projections.visuals:
+        record = load_visual_record(root / visual.source, visual.id)
+        files[visual.human_output] = render_human_visual(root, visual, record)
+        files[visual.agent_output] = render_agent_visual(root, visual, record)
+        digests[visual.human_output] = record.digest
+        digests[visual.agent_output] = record.digest
+    combined = {Path(path): content.decode("utf-8") for path, content in documents.items()}
+    combined.update(files)
+    _verify_links(root, combined)
+    return ProjectionSet(source_ref, corpus_digest, files, digests)
+
+
+def evaluate_projections(root: Path, manifest: Manifest) -> list[BindingResult]:
+    projection_set = render_projections(root, manifest)
+    results = []
+    for path, expected in sorted(projection_set.files.items(), key=lambda item: item[0].as_posix()):
+        try:
+            observed = (root / path).read_text(encoding="utf-8")
+        except FileNotFoundError:
+            observed = ""
+        except OSError as exc:
+            raise ConfigurationError(f"cannot read projection {path}: {exc}") from exc
+        diff = "".join(difflib.unified_diff(
+            observed.splitlines(keepends=True),
+            expected.splitlines(keepends=True),
+            fromfile=path.as_posix(),
+            tofile=f"{path.as_posix()} (projected)",
+        ))
+        results.append(BindingResult(
+            binding_id=f"projection:{path.as_posix()}",
+            doc=path.as_posix(),
+            changed=observed != expected,
+            expected=expected,
+            observed=observed,
+            diff=diff,
+            provenance=Provenance(
+                ref=projection_set.source_ref,
+                path=path.as_posix(),
+                locator="documentation-graph",
+                extractor="projection",
+                digest=projection_set.digests[path],
+            ),
+            binding_type="projection",
+        ))
+    return results
+
+
+def write_projections(root: Path, manifest: Manifest) -> tuple[Path, ...]:
+    projection_set = render_projections(root, manifest)
+    written = []
+    for path, content in sorted(projection_set.files.items(), key=lambda item: item[0].as_posix()):
+        atomic_write(root / path, content)
+        written.append(path)
+    return tuple(written)

clean_docs/regions.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+
+import os
+import stat
+import tempfile
+from pathlib import Path
+
+from clean_docs.errors import RegionError
+
+
+def markers(region: str) -> tuple[str, str]:
+    return (
+        f"<!-- sourcebound:begin {region} -->",
+        f"<!-- sourcebound:end {region} -->",
+    )
+
+
+def mdx_markers(region: str) -> tuple[str, str]:
+    return (
+        f"{{/* sourcebound:begin {region} */}}",
+        f"{{/* sourcebound:end {region} */}}",
+    )
+
+
+def replace_region(document: str, region: str, generated: str) -> str:
+    forms = [
+        candidate
+        for candidate in (markers(region), mdx_markers(region))
+        if candidate[0] in document or candidate[1] in document
+    ]
+    if len(forms) != 1:
+        raise RegionError(
+            f"region {region!r} must use exactly one Markdown or MDX marker form"
+        )
+    begin, end = forms[0]
+    if document.count(begin) != 1 or document.count(end) != 1:
+        raise RegionError(f"region {region!r} must have exactly one begin and one end marker")
+    start = document.index(begin) + len(begin)
+    finish = document.index(end)
+    if finish < start:
+        raise RegionError(f"region {region!r} end marker precedes its begin marker")
+    between = document[start:finish]
+    if any(
+        marker in between
+        for marker in (
+            "<!-- sourcebound:begin ",
+            "<!-- sourcebound:end ",
+            "{/* sourcebound:begin ",
+            "{/* sourcebound:end ",
+        )
+    ):
+        raise RegionError(f"region {region!r} contains nested sourcebound markers")
+    return document[:start] + "\n" + generated.rstrip() + "\n" + document[finish:]
+
+
+def atomic_write(path: Path, content: str) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    try:
+        mode = stat.S_IMODE(path.stat().st_mode)
+    except OSError:
+        mode = 0o644
+    descriptor, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent)
+    try:
+        os.fchmod(descriptor, mode)
+        with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
+            handle.write(content)
+            handle.flush()
+            os.fsync(handle.fileno())
+        os.replace(temporary, path)
+    except BaseException:
+        try:
+            os.unlink(temporary)
+        except OSError:
+            pass
+        raise

clean_docs/release.py

@@ -0,0 +1,232 @@
+"""Build deterministic release facts from normalized evidence at two git refs."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from typing import Any
+
+from clean_docs.errors import ConfigurationError
+from clean_docs.inventory import InventoryItem, scan_inventory
+from clean_docs.policy import check_prose
+from clean_docs.plugins import discover_plugin_items, merge_plugin_inventory
+from clean_docs.manifest import load_manifest
+from clean_docs.snapshot import RepositorySnapshot
+from clean_docs.standard import load_default_pack
+
+
+DELTA_KINDS = ("added", "removed", "changed")
+
+
+@dataclass(frozen=True)
+class EvidenceDelta:
+    id: str
+    change: str
+    kind: str
+    name: str
+    source: str
+    locator: str
+    adapter: str
+    before_sha256: str | None
+    after_sha256: str | None
+
+
+@dataclass(frozen=True)
+class ReleaseReport:
+    from_ref: str
+    to_ref: str
+    deltas: tuple[EvidenceDelta, ...]
+
+    def as_dict(self) -> dict[str, object]:
+        return {
+            "schema": "sourcebound.release-delta.v1",
+            "from": self.from_ref,
+            "to": self.to_ref,
+            "deltas": [asdict(delta) for delta in self.deltas],
+            "counts": {
+                change: sum(delta.change == change for delta in self.deltas)
+                for change in DELTA_KINDS
+            },
+        }
+
+
+@dataclass(frozen=True)
+class NarrativeDraft:
+    delta_id: str
+    text: str
+    citation: str
+
+
+@dataclass(frozen=True)
+class NarrativeResult:
+    drafts: tuple[NarrativeDraft, ...]
+    findings: tuple[str, ...]
+
+    @property
+    def ok(self) -> bool:
+        return not self.findings
+
+    def as_dict(self) -> dict[str, object]:
+        return {
+            "ok": self.ok,
+            "drafts": [asdict(draft) for draft in self.drafts],
+            "findings": list(self.findings),
+        }
+
+
+def _items(root: Path, ref: str) -> tuple[str, dict[str, InventoryItem]]:
+    snapshot = RepositorySnapshot(root, ref)
+    label = snapshot.label
+    with snapshot.materialized_root() as materialized:
+        items = list(scan_inventory(materialized).items)
+        manifest_path = materialized / ".sourcebound.yml"
+        plugins = load_manifest(manifest_path).plugins if manifest_path.is_file() else ()
+    merged = merge_plugin_inventory(
+        tuple(items), discover_plugin_items(snapshot, plugins)
+    )
+    return label, {item.id: item for item in merged}
+
+
+def _delta(
+    identifier: str,
+    change: str,
+    before: InventoryItem | None,
+    after: InventoryItem | None,
+) -> EvidenceDelta:
+    item = after or before
+    if item is None:  # pragma: no cover - protected by the caller
+        raise AssertionError("a release delta needs evidence")
+    return EvidenceDelta(
+        identifier,
+        change,
+        item.kind,
+        item.name,
+        item.source,
+        item.locator,
+        item.adapter,
+        before.digest if before is not None else None,
+        after.digest if after is not None else None,
+    )
+
+
+def build_release_report(root: Path, from_ref: str, to_ref: str) -> ReleaseReport:
+    if not from_ref or not to_ref:
+        raise ConfigurationError("release requires --from and --to refs")
+    root = root.resolve()
+    from_label, before = _items(root, from_ref)
+    to_label, after = _items(root, to_ref)
+    deltas: list[EvidenceDelta] = []
+    for identifier in sorted(set(before) | set(after)):
+        old = before.get(identifier)
+        new = after.get(identifier)
+        if old is None:
+            deltas.append(_delta(identifier, "added", None, new))
+        elif new is None:
+            deltas.append(_delta(identifier, "removed", old, None))
+        elif old != new:
+            deltas.append(_delta(identifier, "changed", old, new))
+    return ReleaseReport(from_label, to_label, tuple(deltas))
+
+
+def _mapping(value: Any, where: str) -> dict[str, Any]:
+    if not isinstance(value, dict):
+        raise ConfigurationError(f"{where} must be an object")
+    return value
+
+
+def validate_release_narrative(report: ReleaseReport, response: str) -> NarrativeResult:
+    try:
+        raw = json.loads(response)
+    except json.JSONDecodeError as exc:
+        raise ConfigurationError(f"release narrative is not valid JSON: {exc}") from exc
+    root = _mapping(raw, "release narrative")
+    if set(root) != {"schema", "drafts"} or root.get("schema") != "sourcebound.release-narrative.v1":
+        raise ConfigurationError(
+            "release narrative must contain schema sourcebound.release-narrative.v1 and drafts"
+        )
+    if not isinstance(root["drafts"], list):
+        raise ConfigurationError("release narrative drafts must be a list")
+    expected = {delta.id: delta for delta in report.deltas}
+    seen: set[str] = set()
+    accepted: list[NarrativeDraft] = []
+    findings: list[str] = []
+    for index, candidate in enumerate(root["drafts"]):
+        draft = _mapping(candidate, f"release narrative draft {index}")
+        required = {"delta_id", "change", "kind", "name", "citation", "text"}
+        if set(draft) != required:
+            raise ConfigurationError(
+                f"release narrative draft {index} must contain exactly: "
+                + ", ".join(sorted(required))
+            )
+        delta_id = draft["delta_id"]
+        if not isinstance(delta_id, str) or delta_id not in expected:
+            findings.append(f"draft {index} names an unknown delta")
+            continue
+        if delta_id in seen:
+            findings.append(f"delta {delta_id} has duplicate drafts")
+            continue
+        seen.add(delta_id)
+        delta = expected[delta_id]
+        mirrors = {
+            "change": delta.change,
+            "kind": delta.kind,
+            "name": delta.name,
+            "citation": f"{delta.source}#{delta.locator}",
+        }
+        mismatched = [key for key, value in mirrors.items() if draft[key] != value]
+        text = draft["text"]
+        if not isinstance(text, str) or not text.strip() or len(text) > 500:
+            findings.append(f"delta {delta_id} has invalid narrative text")
+            continue
+        if mismatched:
+            findings.append(
+                f"delta {delta_id} contradicts deterministic fields: {', '.join(mismatched)}"
+            )
+            continue
+        policy = check_prose("<release-narrative>", text, load_default_pack())
+        if policy:
+            findings.extend(
+                f"delta {delta_id} violates {finding.rule}: {finding.detail}"
+                for finding in policy
+            )
+            continue
+        accepted.append(NarrativeDraft(delta_id, text.strip(), mirrors["citation"]))
+    for delta_id in sorted(set(expected) - seen):
+        findings.append(f"delta {delta_id} is omitted from the narrative")
+    if findings:
+        accepted = []
+    return NarrativeResult(tuple(accepted), tuple(findings))
+
+
+def render_release_markdown(
+    report: ReleaseReport, narrative: NarrativeResult | None = None
+) -> str:
+    lines = [
+        "# Release facts",
+        "",
+        f"Evidence compared from `{report.from_ref}` to `{report.to_ref}`.",
+    ]
+    for change in DELTA_KINDS:
+        lines.extend(("", f"## {change.title()}"))
+        selected = [delta for delta in report.deltas if delta.change == change]
+        if not selected:
+            lines.extend(("", "None."))
+            continue
+        lines.append("")
+        for delta in selected:
+            digest = delta.after_sha256 or delta.before_sha256
+            lines.append(
+                f"- `{delta.kind}` `{delta.name}` at "
+                f"[{delta.source}]({delta.source}) locator `{delta.locator}` via "
+                f"`{delta.adapter}`; evidence sha256 `{digest}`."
+            )
+    if narrative is not None:
+        lines.extend(("", "## Narrative draft", ""))
+        if narrative.ok:
+            for draft in narrative.drafts:
+                lines.append(f"- {draft.text} (`{draft.citation}`)")
+        else:
+            lines.append("Withheld because the draft did not preserve every release fact.")
+            lines.extend(f"- {finding}" for finding in narrative.findings)
+    return "\n".join(lines) + "\n"

clean_docs/renderers.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+from typing import Any
+
+from clean_docs.errors import ExtractionError
+from clean_docs.models import EvidenceValue, RegionBinding
+
+
+def _cell(value: Any) -> str:
+    if value is None:
+        return ""
+    if isinstance(value, bool):
+        text = "true" if value else "false"
+    elif isinstance(value, (dict, list)):
+        raise ExtractionError("nested values cannot be rendered as Markdown table cells")
+    else:
+        text = str(value)
+    return text.replace("|", "\\|").replace("\n", " ")
+
+
+def render_markdown_table(evidence: EvidenceValue, binding: RegionBinding) -> str:
+    rows = evidence.value
+    missing = sorted({column for column in binding.columns for row in rows if column not in row})
+    if missing:
+        raise ExtractionError(f"binding {binding.id} is missing column(s): {', '.join(missing)}")
+    header = "| " + " | ".join(binding.columns) + " |"
+    divider = "| " + " | ".join("---" for _ in binding.columns) + " |"
+    body = [
+        "| " + " | ".join(_cell(row[column]) for column in binding.columns) + " |"
+        for row in rows
+    ]
+    return "\n".join([header, divider, *body])
+
+
+def render(evidence: EvidenceValue, binding: RegionBinding) -> str:
+    if binding.renderer == "markdown-table":
+        return render_markdown_table(evidence, binding)
+    if binding.renderer == "markdown-list":
+        if not isinstance(evidence.value, list) or any(
+            isinstance(item, (dict, list)) for item in evidence.value
+        ):
+            raise ExtractionError("markdown-list evidence must contain scalar items")
+        return "\n".join(f"- {_cell(item)}" for item in evidence.value)
+    if binding.renderer == "scalar":
+        if isinstance(evidence.value, (dict, list)):
+            raise ExtractionError("scalar renderer requires a scalar value")
+        return _cell(evidence.value)
+    if binding.renderer == "markdown-fragment":
+        if not isinstance(evidence.value, str) or evidence.kind != "markdown":
+            raise ExtractionError("markdown-fragment requires generated Markdown evidence")
+        return evidence.value.rstrip()
+    if binding.renderer == "fenced-text":
+        if not isinstance(evidence.value, str):
+            raise ExtractionError("fenced-text renderer requires text evidence")
+        fence = "```" + (binding.language or "")
+        return f"{fence}\n{evidence.value.rstrip()}\n```"
+    raise ExtractionError(f"unsupported renderer: {binding.renderer}")