sourcebound 1.2.1__py3-none-any.whl

clean_docs/context.py

@@ -0,0 +1,307 @@
+"""Compile bounded, source-addressed context without a model or retrieval index."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+import subprocess
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from typing import Any
+
+from clean_docs.errors import ConfigurationError
+
+
+REQUEST_SCHEMA = "sourcebound.context-request.v1"
+BUNDLE_SCHEMA = "sourcebound.context-bundle.v1"
+KINDS = {
+    "example",
+    "fact",
+    "history",
+    "hypothesis",
+    "instruction",
+    "policy",
+    "projection",
+}
+AUTHORITIES = {
+    "accepted-policy": 50,
+    "direct-evidence": 40,
+    "generated": 30,
+    "repository-doc": 20,
+    "hypothesis": 10,
+}
+SHA = re.compile(r"^[0-9a-f]{40}$")
+
+
+@dataclass(frozen=True)
+class ContextItem:
+    id: str
+    kind: str
+    path: str
+    locator: str
+    source_commit: str
+    source_sha256: str
+    authority: str
+    relationship: str
+    inclusion_reason: str
+    rank: int
+    instruction_allowed: bool
+    bytes: int
+    content: str
+
+
+@dataclass(frozen=True)
+class ExcludedContext:
+    id: str
+    path: str
+    reason: str
+    required: bool
+    source_sha256: str
+    bytes: int
+
+
+@dataclass(frozen=True)
+class ContextBundle:
+    repository_commit: str
+    budget_bytes: int
+    used_bytes: int
+    rejected_bytes: int
+    status: str
+    items: tuple[ContextItem, ...]
+    excluded: tuple[ExcludedContext, ...]
+    digest: str
+
+    @property
+    def ok(self) -> bool:
+        return self.status == "current"
+
+    def as_dict(self) -> dict[str, object]:
+        return {
+            "schema": BUNDLE_SCHEMA,
+            "repository_commit": self.repository_commit,
+            "budget": {
+                "bytes": self.budget_bytes,
+                "used": self.used_bytes,
+                "rejected": self.rejected_bytes,
+            },
+            "status": self.status,
+            "items": [asdict(item) for item in self.items],
+            "excluded": [asdict(item) for item in self.excluded],
+            "digest": self.digest,
+        }
+
+
+def _mapping(value: Any, where: str) -> dict[str, Any]:
+    if not isinstance(value, dict):
+        raise ConfigurationError(f"{where} must be an object")
+    return value
+
+
+def _git_commit(root: Path) -> str:
+    process = subprocess.run(
+        ["git", "-C", str(root), "rev-parse", "HEAD"],
+        text=True,
+        capture_output=True,
+        timeout=30,
+        check=False,
+    )
+    if process.returncode != 0:
+        raise ConfigurationError("context compilation requires a git commit")
+    return process.stdout.strip()
+
+
+def _source_text(
+    root: Path,
+    *,
+    commit: str,
+    path: str,
+    start_line: int,
+    end_line: int,
+) -> str:
+    relative = Path(path)
+    if relative.is_absolute() or ".." in relative.parts:
+        raise ConfigurationError(f"context path must stay inside the repository: {path}")
+    process = subprocess.run(
+        ["git", "-C", str(root), "show", f"{commit}:{relative.as_posix()}"],
+        capture_output=True,
+        timeout=30,
+        check=False,
+    )
+    if process.returncode != 0:
+        raise ConfigurationError(
+            f"context path does not exist at {commit}: {path}"
+        )
+    try:
+        lines = process.stdout.decode("utf-8").splitlines()
+    except UnicodeDecodeError as exc:
+        raise ConfigurationError(f"context path is not UTF-8: {path}") from exc
+    if start_line < 1 or end_line < start_line or end_line > len(lines):
+        raise ConfigurationError(
+            f"context locator is outside {path}: L{start_line}-L{end_line}"
+        )
+    return "\n".join(lines[start_line - 1:end_line]) + "\n"
+
+
+def _canonical_digest(payload: dict[str, object]) -> str:
+    return hashlib.sha256(
+        json.dumps(
+            payload,
+            sort_keys=True,
+            separators=(",", ":"),
+            ensure_ascii=False,
+        ).encode()
+    ).hexdigest()
+
+
+def compile_context(root: Path, request_path: Path) -> ContextBundle:
+    root = root.resolve()
+    try:
+        raw = json.loads(request_path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as exc:
+        raise ConfigurationError(f"cannot read context request {request_path}: {exc}") from exc
+    request = _mapping(raw, "context request")
+    if set(request) != {"schema", "repository_commit", "budget_bytes", "items"}:
+        raise ConfigurationError("context request has unsupported fields")
+    if request.get("schema") != REQUEST_SCHEMA:
+        raise ConfigurationError("context request has an unsupported schema")
+    commit = request.get("repository_commit")
+    if not isinstance(commit, str) or not SHA.fullmatch(commit):
+        raise ConfigurationError("context request repository_commit must be a full SHA")
+    observed_commit = _git_commit(root)
+    if commit != observed_commit:
+        raise ConfigurationError(
+            f"context request commit is {commit}, but repository is {observed_commit}"
+        )
+    budget = request.get("budget_bytes")
+    if not isinstance(budget, int) or isinstance(budget, bool) or budget < 1:
+        raise ConfigurationError("context request budget_bytes must be positive")
+    raw_items = request.get("items")
+    if not isinstance(raw_items, list) or not raw_items:
+        raise ConfigurationError("context request items must be non-empty")
+
+    prepared: list[tuple[dict[str, Any], str, int]] = []
+    seen: set[str] = set()
+    for index, value in enumerate(raw_items):
+        item = _mapping(value, f"context request item {index}")
+        expected = {
+            "id",
+            "kind",
+            "path",
+            "start_line",
+            "end_line",
+            "authority",
+            "relationship",
+            "reason",
+            "rank",
+            "required",
+            "instruction",
+        }
+        if set(item) != expected:
+            raise ConfigurationError(f"context request item {index} has unsupported fields")
+        identifier = item.get("id")
+        if not isinstance(identifier, str) or not identifier:
+            raise ConfigurationError(f"context request item {index} needs an id")
+        if identifier in seen:
+            raise ConfigurationError(f"duplicate context item id: {identifier}")
+        seen.add(identifier)
+        if item.get("kind") not in KINDS:
+            raise ConfigurationError(f"context request item {identifier} has an invalid kind")
+        authority = item.get("authority")
+        if authority not in AUTHORITIES:
+            raise ConfigurationError(
+                f"context request item {identifier} has an invalid authority"
+            )
+        for field in ("path", "relationship", "reason"):
+            if not isinstance(item.get(field), str) or not item[field]:
+                raise ConfigurationError(
+                    f"context request item {identifier}.{field} must be non-empty"
+                )
+        for field in ("start_line", "end_line", "rank"):
+            if not isinstance(item.get(field), int) or isinstance(item[field], bool):
+                raise ConfigurationError(
+                    f"context request item {identifier}.{field} must be an integer"
+                )
+        for field in ("required", "instruction"):
+            if not isinstance(item.get(field), bool):
+                raise ConfigurationError(
+                    f"context request item {identifier}.{field} must be boolean"
+                )
+        content = _source_text(
+            root,
+            commit=commit,
+            path=item["path"],
+            start_line=item["start_line"],
+            end_line=item["end_line"],
+        )
+        prepared.append((item, content, len(content.encode())))
+
+    prepared.sort(
+        key=lambda row: (
+            not row[0]["required"],
+            -AUTHORITIES[row[0]["authority"]],
+            -row[0]["rank"],
+            row[0]["id"],
+        )
+    )
+    included: list[ContextItem] = []
+    excluded: list[ExcludedContext] = []
+    used = 0
+    unknown = False
+    for item, content, byte_count in prepared:
+        if used + byte_count > budget:
+            reason = "required-over-budget" if item["required"] else "budget-exhausted"
+            excluded.append(
+                ExcludedContext(
+                    item["id"],
+                    item["path"],
+                    reason,
+                    item["required"],
+                    hashlib.sha256(content.encode()).hexdigest(),
+                    byte_count,
+                )
+            )
+            unknown = unknown or item["required"]
+            continue
+        instruction_allowed = (
+            item["instruction"]
+            and item["kind"] in {"instruction", "policy"}
+            and item["authority"] == "accepted-policy"
+        )
+        included.append(
+            ContextItem(
+                item["id"],
+                item["kind"],
+                item["path"],
+                f"L{item['start_line']}-L{item['end_line']}",
+                commit,
+                hashlib.sha256(content.encode()).hexdigest(),
+                item["authority"],
+                item["relationship"],
+                item["reason"],
+                item["rank"],
+                instruction_allowed,
+                byte_count,
+                content,
+            )
+        )
+        used += byte_count
+    status = "unknown" if unknown or not included else "current"
+    unsigned: dict[str, object] = {
+        "schema": BUNDLE_SCHEMA,
+        "repository_commit": commit,
+        "budget": {"bytes": budget, "used": used},
+        "status": status,
+        "items": [asdict(item) for item in included],
+        "excluded": [asdict(item) for item in excluded],
+    }
+    return ContextBundle(
+        commit,
+        budget,
+        used,
+        sum(item.bytes for item in excluded),
+        status,
+        tuple(included),
+        tuple(excluded),
+        _canonical_digest(unsigned),
+    )

clean_docs/corpus.py

@@ -0,0 +1,377 @@
+from __future__ import annotations
+
+import os
+import json
+import re
+import subprocess
+import sys
+from collections.abc import Iterable
+from collections.abc import Mapping
+from pathlib import Path
+
+from clean_docs.mdx import MdxParserError, parse_mdx
+from clean_docs.policy import PolicyFinding
+
+
+DOC_MAX_LINES = 120
+SECTION_MAX_LINES = 40
+MIN_PARA_TOKENS = 8
+NEAR_DUP = 0.80
+RESTATEMENT = 0.60
+POSTINGS_CAP = 200
+
+PROCESS_RE = re.compile(
+    r"(REPORT|HANDOFF|DISPATCH|BLOCKED|STATUS|PROGRESS|RECEIPT|FINDINGS"
+    r"|WEEK\d|WAVE\d|EXECUTION_PLAN|EXECUTOR|RETRO|_AUDIT)",
+    re.IGNORECASE,
+)
+CHANGELOG_RE = re.compile(r"(CHANGELOG|DECISION_LOG|PROGRAM_REPORT|RETRO)", re.IGNORECASE)
+HARNESS_RE = re.compile(
+    r"\b(next executor|pick up this branch|worktree|STOP condition|tripwire"
+    r"|do not edit|mid-edit|reconciles exactly|diff budget|skeptical reviewer"
+    r"|origin/main|git log|killed PID|baseline SHA|DoD table)\b",
+    re.IGNORECASE,
+)
+PROVENANCE_RE = re.compile(
+    r"(\((?:Program|Wave|Harvest|Dispatch|WS-[A-Za-z]+)\s*\d*\)"
+    r"|verified after authoring|\$\d+\.\d{6})"
+)
+HARNESS_HITS = 3
+STOPWORDS = frozenset(
+    "the a an and or but of to in on at for with without from into is are was were be been "
+    "being it its this that these those as by not no also than then so such can may must will "
+    "which when where what who how why each any all one two per via if do does done only same "
+    "here there they them their you your we our us".split()
+)
+WORD_RE = re.compile(r"[a-z][a-z0-9-]{2,}")
+FENCE_RE = re.compile(r"^```")
+HTML_COMMENT_RE = re.compile(r"^\s*<!--.*?-->\s*$")
+FALLBACK_SKIP_PARTS = frozenset({
+    ".git",
+    ".nox",
+    ".pytest_cache",
+    ".tox",
+    ".venv",
+    "__pycache__",
+    "build",
+    "dist",
+    "node_modules",
+})
+
+
+def _is_document_candidate(relative: Path, *, fallback: bool) -> bool:
+    """Return whether a Markdown path can belong to the reader-facing corpus."""
+    parts = relative.parts
+    packaged_standard = any(
+        parts[index:index + 2] == ("clean_docs", "standards")
+        for index in range(len(parts) - 1)
+    )
+    return not (
+        parts[:2] == ("tests", "fixtures")
+        or packaged_standard
+        or ".fixture." in relative.name.lower()
+        or (fallback and bool(set(parts) & FALLBACK_SKIP_PARTS))
+    )
+
+
+def _git_visible_markdown(root: Path) -> list[Path] | None:
+    try:
+        proc = subprocess.run(
+            [
+                "git",
+                "-C",
+                str(root),
+                "ls-files",
+                "--cached",
+                "--others",
+                "--exclude-standard",
+                "--",
+                "*.md",
+                "*.mdx",
+            ],
+            capture_output=True,
+            text=True,
+            timeout=20,
+            check=False,
+        )
+    except (subprocess.SubprocessError, OSError):
+        return None
+    if proc.returncode != 0:
+        return None
+    candidates = sorted(
+        {
+            root / line
+            for line in proc.stdout.splitlines()
+            if line.strip() and (root / line).is_file()
+        },
+        key=lambda path: (path.is_symlink(), path.as_posix()),
+    )
+    canonical: dict[Path, Path] = {}
+    for path in candidates:
+        try:
+            identity = path.resolve(strict=True)
+        except OSError:
+            identity = path.absolute()
+        canonical.setdefault(identity, path)
+    return sorted(canonical.values())
+
+
+def _hidden_document(relative: Path) -> bool:
+    hidden = [part for part in relative.parts if part.startswith(".")]
+    return bool(hidden) and relative.parts[0] != ".agents"
+
+
+def list_documents(root: Path) -> list[Path]:
+    """Return the reader-facing Markdown surface used by the Version 0 linter."""
+    if root.is_file():
+        return [root]
+    visible = _git_visible_markdown(root)
+    if visible is not None:
+        return [
+            path
+            for path in visible
+            if "archive" not in path.relative_to(root).parts
+            and _is_document_candidate(path.relative_to(root), fallback=False)
+            and not _hidden_document(path.relative_to(root))
+        ]
+    return sorted(
+        path
+        for pattern in ("*.md", "*.mdx")
+        for path in root.rglob(pattern)
+        if _is_document_candidate(path.relative_to(root), fallback=True)
+        and "archive" not in path.relative_to(root).parts
+        and ".sourcebound" not in path.relative_to(root).parts
+        and not _hidden_document(path.relative_to(root))
+    )
+
+
+def _read(path: Path) -> str:
+    try:
+        text = path.read_text(encoding="utf-8")
+    except OSError:
+        return ""
+    if path.suffix.lower() == ".mdx":
+        try:
+            return parse_mdx(text).policy_text(text)
+        except MdxParserError:
+            return ""
+    return text
+
+
+def _content_tokens(text: str) -> frozenset[str]:
+    return frozenset(word for word in WORD_RE.findall(text.lower()) if word not in STOPWORDS)
+
+
+def _paragraphs(text: str) -> list[tuple[int, str]]:
+    result: list[tuple[int, str]] = []
+    buffer: list[str] = []
+    in_fence = False
+    start = 1
+    for line_number, raw in enumerate(text.splitlines(), start=1):
+        if FENCE_RE.match(raw):
+            in_fence = not in_fence
+            continue
+        if in_fence:
+            continue
+        if not raw.strip():
+            if buffer:
+                result.append((start, " ".join(buffer)))
+                buffer = []
+            continue
+        if HTML_COMMENT_RE.match(raw):
+            if buffer:
+                result.append((start, " ".join(buffer)))
+                buffer = []
+            continue
+        if not buffer:
+            start = line_number
+        stripped = raw.lstrip()
+        if stripped.startswith(("#", "|", ">")):
+            if buffer:
+                result.append((start, " ".join(buffer)))
+                buffer = []
+            continue
+        buffer.append(stripped)
+    if buffer:
+        result.append((start, " ".join(buffer)))
+    return result
+
+
+def _sections(text: str) -> list[tuple[str, int, int]]:
+    lines = text.splitlines()
+    headings = [
+        (index + 1, line)
+        for index, line in enumerate(lines)
+        if re.match(r"^#{2,} ", line)
+    ]
+    result = []
+    for position, (line_number, title) in enumerate(headings):
+        end = headings[position + 1][0] if position + 1 < len(headings) else len(lines) + 1
+        result.append((title.strip("# ").strip(), line_number, end - line_number))
+    return result
+
+
+def _duplicate_findings(
+    paragraphs: list[tuple[str, int, frozenset[str], str]],
+    index: dict[str, list[int]],
+) -> list[PolicyFinding]:
+    seen_pairs: set[tuple[int, int]] = set()
+    findings = []
+    for postings in index.values():
+        if len(postings) > POSTINGS_CAP or len(postings) < 2:
+            continue
+        for left_position, left_id in enumerate(postings):
+            for right_id in postings[left_position + 1:]:
+                pair = (left_id, right_id)
+                if pair in seen_pairs:
+                    continue
+                seen_pairs.add(pair)
+                left_tokens = paragraphs[left_id][2]
+                right_tokens = paragraphs[right_id][2]
+                intersection = len(left_tokens & right_tokens)
+                if intersection < 4:
+                    continue
+                overlap = intersection / len(left_tokens | right_tokens)
+                if overlap < RESTATEMENT:
+                    continue
+                left = paragraphs[left_id]
+                right = paragraphs[right_id]
+                same_document = left[0] == right[0]
+                rule = "near-dup" if overlap >= NEAR_DUP or not same_document else "restatement"
+                location = f"{right[0]}:{right[1]}"
+                detail = f"~{overlap:.0%} overlap with {location}"
+                if not same_document:
+                    detail += " (different doc -- pick one canonical home)"
+                findings.append(PolicyFinding(left[0], left[1], rule, detail))
+    return findings
+
+
+def scan_corpus(
+    root: Path,
+    *,
+    include_lengths: bool = True,
+    prepared_documents: Mapping[str, str] | None = None,
+) -> list[PolicyFinding]:
+    """Run the tuned Version 0 corpus rules with stable finding identifiers."""
+    root = root.resolve()
+    base = root if root.is_dir() else root.parent
+    findings: list[PolicyFinding] = []
+    paragraph_index: dict[str, list[int]] = {}
+    paragraphs: list[tuple[str, int, frozenset[str], str]] = []
+
+    document_rows = (
+        (
+            os.path.relpath(path, base),
+            path.name,
+            _read(path),
+        )
+        for path in list_documents(root)
+    ) if prepared_documents is None else (
+        (relative, Path(relative).name, text)
+        for relative, text in sorted(prepared_documents.items())
+    )
+    for relative, name, text in document_rows:
+        if PROCESS_RE.search(name):
+            findings.append(PolicyFinding(
+                relative,
+                1,
+                "surface",
+                "process-artifact name on the reader-facing doc surface",
+            ))
+        if not CHANGELOG_RE.search(name):
+            harness_hits = len(HARNESS_RE.findall(text))
+            if harness_hits >= HARNESS_HITS:
+                findings.append(PolicyFinding(
+                    relative,
+                    1,
+                    "audience",
+                    f"{harness_hits} harness/agent-address terms -- audience reads as an agent",
+                ))
+            provenance = PROVENANCE_RE.findall(text)
+            if provenance:
+                match = PROVENANCE_RE.search(text)
+                assert match is not None
+                line = text.count("\n", 0, match.start()) + 1
+                findings.append(PolicyFinding(
+                    relative,
+                    line,
+                    "provenance",
+                    f"{len(provenance)} provenance/receipt marks in a reference doc "
+                    f"(e.g. {match.group(0)!r})",
+                ))
+        if include_lengths:
+            line_count = text.count("\n") + 1
+            if line_count > DOC_MAX_LINES and not PROCESS_RE.search(name):
+                findings.append(PolicyFinding(
+                    relative,
+                    1,
+                    "doc-length",
+                    f"{line_count} lines (> {DOC_MAX_LINES}) -- justify one file or split",
+                ))
+            for title, line_number, count in _sections(text):
+                if count > SECTION_MAX_LINES:
+                    findings.append(PolicyFinding(
+                        relative,
+                        line_number,
+                        "section-length",
+                        f"section {title!r} is {count} lines (> {SECTION_MAX_LINES})",
+                    ))
+        for start, paragraph in _paragraphs(text):
+            tokens = _content_tokens(paragraph)
+            if len(tokens) < MIN_PARA_TOKENS:
+                continue
+            paragraph_id = len(paragraphs)
+            paragraphs.append((relative, start, tokens, paragraph))
+            for token in tokens:
+                paragraph_index.setdefault(token, []).append(paragraph_id)
+
+    findings.extend(_duplicate_findings(paragraphs, paragraph_index))
+    order = {
+        "surface": 0,
+        "audience": 1,
+        "provenance": 2,
+        "near-dup": 3,
+        "doc-length": 4,
+        "section-length": 5,
+        "restatement": 6,
+    }
+    findings.sort(key=lambda finding: (
+        order.get(finding.rule, 9), finding.doc, finding.line
+    ))
+    return findings
+
+
+def findings_as_json(findings: Iterable[PolicyFinding]) -> list[dict[str, object]]:
+    return [
+        {
+            "check": finding.rule,
+            "file": finding.doc,
+            "line": finding.line,
+            "detail": finding.detail,
+        }
+        for finding in findings
+    ]
+
+
+def main(argv: list[str] | None = None) -> int:
+    """Run the Version 0 command-line contract through the packaged engine."""
+    arguments = list(sys.argv[1:] if argv is None else argv)
+    as_json = "--json" in arguments
+    positional = [argument for argument in arguments if argument != "--json"]
+    root = Path(positional[0] if positional else os.getcwd()).resolve()
+    findings = scan_corpus(root)
+    if as_json:
+        print(json.dumps(findings_as_json(findings), indent=2))
+        return 1 if findings else 0
+    if not findings:
+        print(f"doc-hygiene: clean ({root})")
+        return 0
+    counts: dict[str, int] = {}
+    for finding in findings:
+        counts[finding.rule] = counts.get(finding.rule, 0) + 1
+    print(f"doc-hygiene: {len(findings)} finding(s) in {root}")
+    print("  " + ", ".join(f"{key}={value}" for key, value in sorted(counts.items())))
+    print()
+    for finding in findings:
+        print(f"  [{finding.rule}] {finding.doc}:{finding.line}  {finding.detail}")
+    return 1