sourcebound 1.2.1__py3-none-any.whl

clean_docs/snapshot.py

@@ -0,0 +1,212 @@
+from __future__ import annotations
+
+import ntpath
+import posixpath
+import subprocess
+import tarfile
+import tempfile
+from contextlib import contextmanager
+from dataclasses import dataclass
+from collections.abc import Iterator
+from fnmatch import fnmatch
+from pathlib import Path, PurePosixPath, PureWindowsPath
+
+from clean_docs.errors import ExtractionError
+
+
+def _archive_path_is_safe(value: str) -> bool:
+    posix_path = PurePosixPath(value)
+    windows_path = PureWindowsPath(value)
+    return not (
+        posix_path.is_absolute()
+        or windows_path.drive
+        or windows_path.root
+        or ".." in posix_path.parts
+        or ".." in windows_path.parts
+    )
+
+
+def _symlink_target_is_safe(member_name: str, link_name: str) -> bool:
+    posix_link = PurePosixPath(link_name)
+    windows_link = PureWindowsPath(link_name)
+    if (
+        posix_link.is_absolute()
+        or windows_link.drive
+        or windows_link.root
+        or ("\\" in link_name and ".." in windows_link.parts)
+    ):
+        return False
+
+    posix_target = posixpath.normpath(
+        (PurePosixPath(member_name).parent / posix_link).as_posix()
+    )
+    windows_target = ntpath.normpath(
+        str(PureWindowsPath(member_name).parent / windows_link)
+    )
+    return _archive_path_is_safe(posix_target) and _archive_path_is_safe(
+        windows_target
+    )
+
+
+def _member_is_safe(member: tarfile.TarInfo) -> bool:
+    if not _archive_path_is_safe(member.name) or member.islnk():
+        return False
+    if not member.issym():
+        return True
+    return _symlink_target_is_safe(member.name, member.linkname)
+
+
+@dataclass(frozen=True)
+class RepositorySnapshot:
+    root: Path
+    ref: str | None = None
+
+    @property
+    def label(self) -> str:
+        if self.ref is None:
+            return "WORKTREE"
+        proc = self._git("rev-parse", "--verify", f"{self.ref}^{{commit}}")
+        return proc.stdout.strip()
+
+    def read_text(self, path: Path) -> str:
+        if path.is_absolute() or ".." in path.parts:
+            raise ExtractionError(f"source path escapes repository: {path}")
+        if self.ref is None:
+            target = self.root / path
+            try:
+                return target.read_text(encoding="utf-8")
+            except OSError as exc:
+                raise ExtractionError(f"cannot read source {path}: {exc}") from exc
+        proc = self._git("show", f"{self.ref}:{path.as_posix()}")
+        return proc.stdout
+
+    def matching_files(self, pattern: str) -> list[Path]:
+        candidate = Path(pattern)
+        if candidate.is_absolute() or ".." in candidate.parts:
+            raise ExtractionError(f"source glob escapes repository: {pattern}")
+        if self.ref is None:
+            return sorted(
+                path.relative_to(self.root)
+                for path in self.root.glob(pattern)
+                if path.is_file() and self.root in path.resolve().parents
+            )
+        proc = self._git("ls-tree", "-r", "--name-only", self.ref)
+        return [Path(path) for path in proc.stdout.splitlines() if fnmatch(path, pattern)]
+
+    @contextmanager
+    def materialized_root(
+        self,
+        *,
+        paths: tuple[Path, ...] = (),
+    ) -> Iterator[Path]:
+        if self.ref is None:
+            yield self.root
+            return
+        archive_paths: list[str] = []
+        for path in paths:
+            if path == Path("."):
+                continue
+            if path.is_absolute() or ".." in path.parts:
+                raise ExtractionError(
+                    f"snapshot path escapes repository: {path}"
+                )
+            archive_paths.append(path.as_posix().strip("/"))
+        archive_paths = self._include_symlink_targets(archive_paths)
+        with tempfile.TemporaryDirectory(prefix="sourcebound-snapshot-") as temporary:
+            destination = Path(temporary)
+            archive = destination / "snapshot.tar"
+            arguments = [
+                "archive",
+                "--format=tar",
+                f"--output={archive}",
+                self.label,
+            ]
+            if archive_paths:
+                arguments.extend(("--", *archive_paths))
+            self._git(*arguments)
+            with tarfile.open(archive) as handle:
+                for member in handle.getmembers():
+                    if not _member_is_safe(member):
+                        raise ExtractionError(f"unsafe path in repository snapshot: {member.name}")
+                handle.extractall(destination)
+            archive.unlink()
+            yield destination
+
+    def _include_symlink_targets(self, paths: list[str]) -> list[str]:
+        if not paths:
+            return paths
+        selected = set(paths)
+        pending = list(paths)
+        while pending:
+            candidate = pending.pop()
+            entries = self._git(
+                "ls-tree",
+                "-r",
+                "-z",
+                self.label,
+                "--",
+                candidate,
+            ).stdout
+            for entry in entries.split("\0"):
+                if not entry:
+                    continue
+                header, separator, name = entry.partition("\t")
+                if not separator or not header.startswith("120000 "):
+                    continue
+                link = self._git(
+                    "show",
+                    f"{self.label}:{name}",
+                ).stdout.rstrip("\n")
+                link_path = PurePosixPath(link)
+                if link_path.is_absolute():
+                    raise ExtractionError(
+                        f"selected snapshot contains an absolute symlink: {name}"
+                    )
+                target = PurePosixPath(
+                    posixpath.normpath(
+                        (PurePosixPath(name).parent / link_path).as_posix()
+                    )
+                )
+                if target.is_absolute() or ".." in target.parts:
+                    raise ExtractionError(
+                        f"selected snapshot symlink escapes repository: {name}"
+                    )
+                target_name = target.as_posix()
+                if target_name == ".":
+                    raise ExtractionError(
+                        "selected snapshot symlink expands scope to "
+                        f"repository root: {name}"
+                    )
+                if any(
+                    target_name == selected_path
+                    or target_name.startswith(selected_path + "/")
+                    for selected_path in selected
+                ):
+                    continue
+                target_entry = self._git(
+                    "ls-tree",
+                    self.label,
+                    "--",
+                    target_name,
+                ).stdout
+                if not target_entry:
+                    continue
+                selected.add(target_name)
+                pending.append(target_name)
+        return sorted(selected)
+
+    def _git(self, *args: str) -> subprocess.CompletedProcess[str]:
+        try:
+            proc = subprocess.run(
+                ["git", "-C", str(self.root), *args],
+                text=True,
+                capture_output=True,
+                timeout=30,
+                check=False,
+            )
+        except (OSError, subprocess.SubprocessError) as exc:
+            raise ExtractionError(f"git {' '.join(args)} failed: {exc}") from exc
+        if proc.returncode != 0:
+            message = proc.stderr.strip() or proc.stdout.strip() or "unknown git error"
+            raise ExtractionError(f"git {' '.join(args)} failed: {message}")
+        return proc

clean_docs/standard.py

@@ -0,0 +1,281 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from importlib import resources
+from pathlib import Path
+from typing import Any
+
+from clean_docs.errors import ConfigurationError
+
+PACK_VERSION = 1
+DEFAULT_PROFILE = "sourcebound-default"
+HEADING_RE = re.compile(r"^(#{2,4})\s+(.+?)\s*$")
+CHECK_RE = re.compile(r"^- \[ \]\s+(.+)$")
+
+
+def _digest(text: str) -> str:
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+def _pack_digest(pack: dict[str, Any]) -> str:
+    content = {key: value for key, value in pack.items() if key != "pack_sha256"}
+    canonical = json.dumps(content, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
+    return _digest(canonical)
+
+
+def _exemplar_asset(standard: Path) -> tuple[str, str]:
+    local = standard.parent / "src/clean_docs/standards/exemplars.md"
+    if local.exists():
+        text = local.read_text(encoding="utf-8")
+    else:
+        resource = resources.files("clean_docs").joinpath("standards/exemplars.md")
+        with resources.as_file(resource) as path:
+            text = path.read_text(encoding="utf-8")
+    return text, _digest(text)
+
+
+def _headings(lines: list[str]) -> list[dict[str, Any]]:
+    result = []
+    for line_number, line in enumerate(lines, start=1):
+        match = HEADING_RE.match(line)
+        if match:
+            result.append({
+                "level": len(match.group(1)),
+                "title": match.group(2),
+                "line": line_number,
+            })
+    return result
+
+
+def _checklist(lines: list[str]) -> list[str]:
+    start = next(
+        (index for index, line in enumerate(lines) if line.startswith("## 8. Pre-publish checklist")),
+        None,
+    )
+    if start is None:
+        raise ConfigurationError("standard is missing the pre-publish checklist")
+    checks: list[str] = []
+    current: list[str] = []
+    for line in lines[start + 1:]:
+        match = CHECK_RE.match(line)
+        if match:
+            if current:
+                checks.append(" ".join(current))
+            current = [match.group(1).strip()]
+        elif current and line.startswith("      "):
+            current.append(line.strip())
+        elif current and not line.strip():
+            checks.append(" ".join(current))
+            current = []
+    if current:
+        checks.append(" ".join(current))
+    if not checks:
+        raise ConfigurationError("standard pre-publish checklist is empty")
+    return checks
+
+
+def _mechanical_policy(text: str, checks: list[str]) -> dict[str, Any]:
+    length = re.search(
+        r"README pages over (?P<readme>\d+) lines and guides over (?P<guide>\d+) lines",
+        text,
+    )
+    section = re.search(r"A section over (?P<section>\d+) lines", text)
+    preamble = re.search(r"first (?P<preamble>\d+) lines", text)
+    nominalizations = re.search(
+        r"sentence with (?P<nominalizations>\w+) or more abstraction-suffix", text
+    )
+    qualifiers = re.search(r"gets at most (?P<qualifiers>\w+) `may`", text)
+    if not all((length, section, preamble, nominalizations, qualifiers)):
+        raise ConfigurationError(
+            "standard is missing page budgets, preamble window, or register thresholds"
+        )
+    number_words = {
+        "zero": 0,
+        "one": 1,
+        "two": 2,
+        "three": 3,
+        "four": 4,
+        "five": 5,
+    }
+    assert length is not None
+    assert section is not None
+    assert preamble is not None
+    assert nominalizations is not None
+    assert qualifiers is not None
+    booster_check = next((check for check in checks if "booster adjectives" in check), None)
+    if booster_check is None:
+        raise ConfigurationError("standard is missing the booster-adjective check")
+    boosters = re.findall(r"`([^`]+)`", booster_check)
+    if len(boosters) < 3:
+        raise ConfigurationError("standard booster-adjective check has no usable registry")
+    return {
+        "readme_max_lines": int(length.group("readme")),
+        "guide_max_lines": int(length.group("guide")),
+        "section_max_lines": int(section.group("section")),
+        "preamble_window_lines": int(preamble.group("preamble")),
+        "nominalization_threshold": number_words[nominalizations.group("nominalizations")],
+        "nominalization_allowlist": [
+            "documentation",
+            "application",
+            "section",
+            "configuration",
+        ],
+        "sentence_variance_min_words": 15,
+        "sentence_variance_max_words": 35,
+        "qualifier_threshold": number_words[qualifiers.group("qualifiers")],
+        "significance_phrases": [
+            "exactly the",
+            "the very",
+            "this demonstrates",
+            "deliberately",
+            "is itself",
+            "which is precisely",
+        ],
+        "prohibited_boosters": boosters,
+        "require_grounded_facts": any("factual claim" in check for check in checks),
+        "require_definition_first": any("first screen defines" in check for check in checks),
+        "require_one_job": any("one job" in check for check in checks),
+        "require_purpose_contract": any("BLUF purpose contract" in check for check in checks),
+        "require_preamble_contract": any(
+            "first 15 lines contain the purpose" in check for check in checks
+        ),
+        "require_readme_routes": any("README routes decisions" in check for check in checks),
+        "require_depth_links": any(
+            "explanatory section over 80 words" in check for check in checks
+        ),
+    }
+
+
+def _style_contract(text: str, checks: list[str]) -> dict[str, Any]:
+    normalized = " ".join(text.split())
+    required_phrases = {
+        "second_person": "Second person + imperative",
+        "system_actor": "Name the system as an actor",
+        "informative_clauses": "Every clause adds information",
+        "concrete_verbs": "Plain, concrete verbs",
+        "direct_facts": "State facts without hedging",
+        "senior_colleague": "helpful senior colleague",
+        "bounded_personality": "Personality has a budget",
+        "bluf": "BLUF purpose contract",
+    }
+    missing = [name for name, phrase in required_phrases.items() if phrase not in normalized]
+    if missing:
+        raise ConfigurationError(
+            "standard is missing required style trait(s): " + ", ".join(missing)
+        )
+    if not any("BLUF purpose contract" in check for check in checks):
+        raise ConfigurationError("standard checklist is missing the BLUF purpose contract")
+    if not any("subject-derived memorable element" in check for check in checks):
+        raise ConfigurationError("standard checklist is missing bounded personality review")
+    return {
+        "precedence": [
+            "truth and honesty",
+            "grounding",
+            "reader budget",
+            "register",
+            "warmth",
+        ],
+        "voice": {
+            "register": "helpful senior colleague",
+            "reader_actions": "second person and imperative",
+            "system_behavior": "name the system as an actor and state behavior as fact",
+            "sentence_shape": "split claims that need separate evidence or differ in scope",
+            "verbs": "plain and concrete",
+            "certainty": "direct facts; mark genuine uncertainty explicitly",
+            "contractions": "allowed",
+        },
+        "purpose_contract": {
+            "begin_marker": "<!-- sourcebound:purpose -->",
+            "end_marker": "<!-- sourcebound:end purpose -->",
+            "position": "first meaningful block after the H1",
+            "mechanical": [
+                "exactly one marked purpose block",
+                "purpose block precedes body content",
+                "purpose prose does not restate the H1",
+            ],
+            "judgment": [
+                "defines the project-specific subject and intended operator",
+                "names the consequential failure or decision the page addresses",
+                "states the authority boundary and a falsifiable resulting capability",
+                "uses authored language grounded in the implementation and cited sources",
+            ],
+        },
+    }
+
+
+def compile_standard(path: Path) -> dict[str, Any]:
+    try:
+        text = path.read_text(encoding="utf-8")
+    except OSError as exc:
+        raise ConfigurationError(f"cannot read standard {path}: {exc}") from exc
+    lines = text.splitlines()
+    headings = _headings(lines)
+    titles = [heading["title"] for heading in headings]
+    required = {
+        "voice": "2. Voice at the sentence level",
+        "document": "3. How to explain something technical simply (the actual techniques)",
+        "corpus": "6. Beyond the single doc: does it earn its existence?",
+        "grounding": "7. Grounding: the doc must be true to the code (and stay true)",
+    }
+    missing = [title for title in required.values() if title not in titles]
+    if missing:
+        raise ConfigurationError(f"standard is missing required tier heading(s): {', '.join(missing)}")
+    checks = _checklist(lines)
+    style = _style_contract(text, checks)
+    exemplars, exemplar_sha256 = _exemplar_asset(path)
+    pack: dict[str, Any] = {
+        "pack_version": PACK_VERSION,
+        "profile": DEFAULT_PROFILE,
+        "source": {
+            "name": path.name,
+            "sha256": _digest(text),
+        },
+        "tiers": required,
+        "headings": headings,
+        "checklist": checks,
+        "policy": _mechanical_policy(text, checks),
+        "style": style,
+        "generation": {
+            "instructions": text,
+            "constraint": "Phrase only the supplied evidence and preserve its scope.",
+            "voice": style["voice"],
+            "purpose_contract": style["purpose_contract"],
+            "precedence": style["precedence"],
+            "exemplars": exemplars,
+            "exemplars_sha256": exemplar_sha256,
+        },
+    }
+    pack["pack_sha256"] = _pack_digest(pack)
+    return pack
+
+
+def write_pack(pack: dict[str, Any], path: Path) -> None:
+    from clean_docs.regions import atomic_write
+
+    atomic_write(path, json.dumps(pack, indent=2, ensure_ascii=False) + "\n")
+
+
+def load_pack(path: Path) -> dict[str, Any]:
+    try:
+        raw = json.loads(path.read_text(encoding="utf-8"))
+    except OSError as exc:
+        raise ConfigurationError(f"cannot read policy pack {path}: {exc}") from exc
+    except json.JSONDecodeError as exc:
+        raise ConfigurationError(f"invalid policy pack {path}: {exc}") from exc
+    if not isinstance(raw, dict) or raw.get("pack_version") != PACK_VERSION:
+        raise ConfigurationError(f"unsupported policy pack: {path}")
+    if raw.get("pack_sha256") != _pack_digest(raw):
+        raise ConfigurationError(f"policy pack integrity check failed: {path}")
+    return raw
+
+
+def load_default_pack() -> dict[str, Any]:
+    resource = resources.files("clean_docs").joinpath("standards/default.json")
+    with resources.as_file(resource) as path:
+        return load_pack(path)
+
+
+def pack_matches_standard(standard: Path, pack_path: Path) -> bool:
+    return compile_standard(standard) == load_pack(pack_path)