psyclaw 0.27.22 → 0.28.0

package/vendor/ars/scripts/check_revision_token_conservation.py

@@ -0,0 +1,263 @@
+#!/usr/bin/env python3
+"""Deterministic token-conservation audit between revision rounds (#570).
+
+Compares the multisets of numeric tokens, citation tokens
+(`<!--ref:-->` / `<!--anchor:-->` markers, bracketed groups, author-year
+parentheticals), and optional protected terms between a source text and
+its revision — either as a plain pair of drafts or per-op against a #390
+revision patch document, attributing every delta to the op's
+`roadmap_item_ids` and rendering `ADV-REV-<n>` advisory rows.
+
+Contract boundaries (issue #570):
+
+  - **Advisory, not a gate.** A delta is not a violation — the Revision
+    Roadmap may authorize it. The checker surfaces every delta with its
+    claimed roadmap items so the round checkpoint can judge; default exit
+    is 0 regardless of deltas (`--strict` exits 1 on any delta, for
+    fixtures and tests).
+  - **Necessary but not sufficient.** Exact-token conservation cannot see
+    negation, comparison direction, modality, causal strength, citation
+    attachment, or paraphrase-level meaning change. Those stay with the
+    LLM/human audit layers; a conserved result must never be presented as
+    a semantic-fidelity guarantee.
+  - **Fold before split.** The whole text is Unicode-normalized (NFKC +
+    dash/minus folding) BEFORE any tokenization, so a fullwidth or
+    homoglyph respelling can neither smuggle a changed value past the
+    comparison nor false-flag an unchanged one (the #524 ordering
+    lesson).
+  - Tokens are compared verbatim after folding: `612,418` → `612418` is
+    a reported delta, not a silent reformat — formatting changes are for
+    the checkpoint to wave through, not for the checker to forgive.
+
+Mechanism shape borrowed from Yila-AI/sci-ssci-skills
+(`sci-ssci-polishing`, `scripts/check_invariants.py`), extended with
+marker-aware citation tokens, patch-mode attribution, and fold-before-
+split normalization.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+import unicodedata
+from collections import Counter
+from pathlib import Path
+
+if __package__ in (None, ""):  # pragma: no cover - direct CLI invocation
+    sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+COMMENT_RE = re.compile(r"<!--.*?-->", re.DOTALL)
+# Canonical ref-marker grammar (scripts/verify_submission_package.py,
+# pipeline_orchestrator_agent.md L3-1): the finalizer resolves a bare
+# <!--ref:slug--> into 0/1/2-status-token + optional policy_hash forms, and
+# revision reruns run on already-resolved drafts. Key the citation multiset on
+# the SLUG alone — status/suffix tokens are derived metadata, not writer
+# content — so bare↔resolved churn is not a spurious delta and a DROPPED
+# resolved marker still surfaces (matching only the bare shape would let it
+# vanish under COMMENT_RE, the exact miss this checker exists to prevent).
+REF_MARKER_RE = re.compile(r"<!--ref:([A-Za-z][A-Za-z0-9_:-]*)(?:\s[^>]*)?-->")
+ANCHOR_MARKER_RE = re.compile(r"<!--(anchor:[^>]+?)-->")
+# Sign attaches only when preceded by start/whitespace/([/=/:/,/; — never
+# between digits, so `0.58-0.92` stays two unsigned range endpoints while
+# `β = -0.45`, `CI [-0.45, ...]`, and a comma-led `-1.3` keep their sign.
+# Letter-adjacent embedded digits (H2, Model-A7, T0, v2.7.10a) are identifier
+# fragments — entity-layer tokens for `--protected-terms`, not prose numbers —
+# so the unsigned branch excludes them with an explicit [A-Za-z] guard (NOT
+# \w: CJK-adjacent numbers like 共612名 must still count). A scientific
+# exponent (`6.02e23`, `1.2E-5`) is part of the token, so `6.02e23`→`6.02e24`
+# is a delta, not a truncation to a conserved `6.02`.
+_MANTISSA = r"(?:\d+(?:,\d{3})*(?:\.\d+)?|\.\d+)"
+_EXPONENT = r"(?:[eE][-+]?\d+)?"
+NUMBER_RE = re.compile(
+    rf"(?:(?<=[\s(\[=:,;])|^)-?{_MANTISSA}{_EXPONENT}%?"
+    rf"|(?<![A-Za-z\d.]){_MANTISSA}{_EXPONENT}%?"
+)
+SQUARE_CITATION_RE = re.compile(r"\[(?:\d+[-,;\s]*)+\]")
+# Parenthetical author-year: `(Li & Chen, 2023)`. Also capture an optional
+# NARRATIVE author phrase immediately preceding a bare `(year)` —
+# `Smith (2020)`, `Okonkwo and Vidal (2021)`, `Al-Masri et al. (2019)` — so a
+# `Smith`→`Jones` swap that keeps the same year and the same ref marker still
+# surfaces as a citation delta. The author phrase is capitalized-word runs
+# joined by `&` / `and` / `et al.`; the year alone is the parenthetical form.
+_NARRATIVE_AUTHOR = (
+    r"(?:[A-Z][\w'’-]+(?:\s+(?:&|and|et\s+al\.?)\s*)?)+"
+    r"(?:\s+et\s+al\.?)?\s+"
+)
+AUTHOR_YEAR_RE = re.compile(
+    rf"(?:{_NARRATIVE_AUTHOR})?\((?:[^()]*\b(?:19|20)\d{{2}}[a-z]?\b[^()]*|(?:19|20)\d{{2}}[a-z]?)\)"
+)
+
+_DASH_FOLD = str.maketrans({"–": "-", "—": "-", "−": "-"})
+
+
+def normalize_text(text: str) -> str:
+    """NFKC + dash/minus folding over the WHOLE text, before any split."""
+    return unicodedata.normalize("NFKC", text).translate(_DASH_FOLD)
+
+
+def _prose(normalized: str) -> str:
+    return COMMENT_RE.sub(" ", normalized)
+
+
+def extract_numbers(text: str) -> Counter:
+    return Counter(m.group(0) for m in NUMBER_RE.finditer(_prose(normalize_text(text))))
+
+
+def extract_citation_tokens(text: str) -> Counter:
+    normalized = normalize_text(text)
+    tokens: Counter = Counter()
+    for m in REF_MARKER_RE.finditer(normalized):
+        tokens[f"ref:{m.group(1)}"] += 1  # keyed on slug, status tokens ignored
+    for m in ANCHOR_MARKER_RE.finditer(normalized):
+        tokens[m.group(1)] += 1
+    prose = _prose(normalized)
+    for pattern in (SQUARE_CITATION_RE, AUTHOR_YEAR_RE):
+        for m in pattern.finditer(prose):
+            tokens[re.sub(r"\s+", " ", m.group(0))] += 1
+    return tokens
+
+
+def protected_term_counts(text: str, protected_terms) -> Counter:
+    """Case-sensitive counts with word boundaries on alphanumeric edges."""
+    normalized = normalize_text(text)
+    counts: Counter = Counter()
+    for term in protected_terms:
+        pattern = re.escape(normalize_text(term))
+        if term and term[0].isalnum():
+            pattern = r"(?<!\w)" + pattern
+        if term and term[-1].isalnum():
+            pattern = pattern + r"(?!\w)"
+        counts[term] = len(re.findall(pattern, normalized))
+    return counts
+
+
+def _counter_delta(source: Counter, revision: Counter) -> dict:
+    return {"removed": dict(source - revision), "added": dict(revision - source)}
+
+
+def audit_pair(source: str, revision: str, protected_terms=()) -> dict:
+    result = {
+        "numbers_delta": _counter_delta(extract_numbers(source), extract_numbers(revision)),
+        "citations_delta": _counter_delta(
+            extract_citation_tokens(source), extract_citation_tokens(revision)
+        ),
+        "protected_terms_delta": _counter_delta(
+            protected_term_counts(source, protected_terms),
+            protected_term_counts(revision, protected_terms),
+        ),
+    }
+    result["conserved"] = all(
+        not delta[side]
+        for delta in result.values()
+        for side in ("removed", "added")
+    )
+    return result
+
+
+def _op_texts(op: dict, blocks: dict) -> tuple[str, str]:
+    kind = op["op"]
+    block_id = op["block_id"]
+    if kind == "insert_after":
+        return "", op["new_text"]
+    if block_id not in blocks:
+        raise ValueError(f"op targets unknown block {block_id}")
+    old = blocks[block_id].normalized_text
+    if kind == "replace_block":
+        return old, op["new_text"]
+    if kind == "delete_block":
+        return old, ""
+    raise ValueError(f"unknown op kind {kind!r}")
+
+
+def _summarize_delta(delta: dict) -> str:
+    parts = []
+    for label, key in (
+        ("numbers", "numbers_delta"),
+        ("citations", "citations_delta"),
+        ("protected terms", "protected_terms_delta"),
+    ):
+        for side in ("removed", "added"):
+            tokens = delta[key][side]
+            if tokens:
+                listed = ", ".join(sorted(tokens))
+                parts.append(f"{label} {side} [{listed}]")
+    return "; ".join(parts)
+
+
+def audit_patch(patch: dict, base_text: str, protected_terms=()) -> dict:
+    from scripts._block_parser import parse_document
+
+    blocks = parse_document(base_text).block_by_id()
+    op_reports = []
+    advisory_rows = []
+    for op_index, op in enumerate(patch["ops"]):
+        old, new = _op_texts(op, blocks)
+        delta = audit_pair(old, new, protected_terms)
+        roadmap_ids = op.get("roadmap_item_ids", [])
+        op_reports.append(
+            {
+                "op_index": op_index,
+                "op": op["op"],
+                "block_id": op["block_id"],
+                "roadmap_item_ids": roadmap_ids,
+                "delta": delta,
+            }
+        )
+        if not delta["conserved"]:
+            roadmap = ", ".join(roadmap_ids) or "(none)"
+            advisory_rows.append(
+                f"ADV-REV-{len(advisory_rows) + 1}: op {op_index} {op['op']} "
+                f"{op['block_id']} (roadmap: {roadmap}) — {_summarize_delta(delta)}"
+            )
+    return {
+        "conserved": not advisory_rows,
+        "op_reports": op_reports,
+        "advisory_rows": advisory_rows,
+    }
+
+
+def _split_terms(raw: str | None) -> list[str]:
+    return [t.strip() for t in raw.split(",") if t.strip()] if raw else []
+
+
+def main(argv=None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    sub = parser.add_subparsers(dest="mode", required=True)
+
+    pair = sub.add_parser("pair", help="audit two full drafts")
+    pair.add_argument("--source", type=Path, required=True)
+    pair.add_argument("--revision", type=Path, required=True)
+
+    patch = sub.add_parser("patch", help="audit a #390 patch against its base")
+    patch.add_argument("--patch", type=Path, required=True)
+    patch.add_argument("--base", type=Path, required=True)
+
+    for p in (pair, patch):
+        p.add_argument("--protected-terms", help="comma-separated protected terms")
+        p.add_argument(
+            "--strict",
+            action="store_true",
+            help="exit 1 on any delta (advisory default: always exit 0)",
+        )
+
+    args = parser.parse_args(argv)
+    terms = _split_terms(args.protected_terms)
+    if args.mode == "pair":
+        report = audit_pair(
+            args.source.read_text(encoding="utf-8"),
+            args.revision.read_text(encoding="utf-8"),
+            terms,
+        )
+    else:
+        report = audit_patch(
+            json.loads(args.patch.read_text(encoding="utf-8")),
+            args.base.read_text(encoding="utf-8"),
+            terms,
+        )
+    print(json.dumps(report, ensure_ascii=False, indent=2))
+    return 1 if (args.strict and not report["conserved"]) else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/vendor/ars/scripts/check_risk_register.py

@@ -0,0 +1,280 @@
+#!/usr/bin/env python3
+"""Defrift lint for docs/RISK_REGISTER.md (#759).
+
+The risk register is a hand-maintained index linking each standing risk to
+its existing controls, its evidence status, and its residual gap. Its whole
+value is its pointers, so this lint pins exactly three drift classes and
+deliberately nothing else — risk wording, control selection, and residual-gap
+semantics stay owned by code review, mirroring check_degradation_registry.py's
+index-not-author posture:
+
+  RR-1  Pointer integrity: every relative markdown link in the doc resolves
+        inside the repo to an existing file (anchor fragments must match a
+        GitHub-slugified heading in the target), and every backtick-quoted
+        repo path (a code span containing "/" that does not start with "/")
+        names an existing file or directory. A control a row cites must
+        exist on disk.
+  RR-2  Status mirroring: every "- **Evidence status**:" bullet carries at
+        least one recognized citation. A citation of the form
+        `STATUS` (capability matrix row `row_id`) must name an existing row
+        in shared/contracts/capability/stage_capability_matrix.json and
+        STATUS must equal that row's behavioral_evidence.status verbatim —
+        the register mirrors the matrix, it can never upgrade it. A citation
+        of the form `STATUS` (asserted here; no capability-matrix row) is a
+        declared maintainer assertion and may only carry a no-measurement
+        status (DESIGNED / NOT_RUN / OUT_OF_SCOPE): the matrix stays the
+        sole authority for MEASURED/MIXED claims. Every occurrence of a
+        citation phrase must belong to a well-formed citation (reported per
+        segment, so the offending text is locatable), and the shipped
+        matrix-row citations are inventory-locked to Evidence-status bullets
+        (house pattern: the degradation registry's D5, the matrix lint's
+        M13) so a lint-pinned citation can be neither demoted to an asserted
+        one nor relocated out of the bullet the lock is about.
+  RR-3  Discoverability: README.md carries at least one rendered link that
+        resolves to docs/RISK_REGISTER.md (the #759 acceptance criterion,
+        pinned so a refactor cannot orphan the page). Links inside code
+        spans render literally and do not count.
+
+Exit 0 when all invariants hold; exit 1 with one line per violation.
+"""
+from __future__ import annotations
+
+import re
+import sys
+from pathlib import Path
+
+_SCRIPTS_DIR = Path(__file__).resolve().parent
+if str(_SCRIPTS_DIR) not in sys.path:
+    sys.path.insert(0, str(_SCRIPTS_DIR))
+
+# Shared markdown machinery (#771); the matrix vocabulary comes from the
+# matrix lint, a genuinely data-owning dependency.
+from _markdown_lint_util import (  # noqa: E402
+    NON_RELATIVE_LINK_PREFIXES,
+    RelativeLinkFailure,
+    code_spans,
+    link_targets,
+    links_to,
+    relative_link_failure,
+    strip_non_rendering,
+)
+from check_stage_capability_matrix import (  # noqa: E402
+    _BEHAVIORAL_STATUSES,
+    _load,
+    _status,
+)
+
+DOC_RELPATH = Path("docs/RISK_REGISTER.md")
+MATRIX_RELPATH = Path("shared/contracts/capability/stage_capability_matrix.json")
+README_RELPATH = Path("README.md")
+
+_CITE_RE = re.compile(
+    r"`([A-Za-z_]+)`\s*\((?:capability matrix row\s+`([A-Za-z0-9_.]+)`"
+    r"|asserted here; no capability-matrix row)\)"
+)
+# Substrings whose every occurrence must be consumed by a well-formed
+# citation, so a typo cannot demote a pinned status to unchecked prose.
+_CITE_PHRASES = ("capability matrix row", "asserted here")
+_EVIDENCE_BULLET_PREFIX = "- **Evidence status**:"
+# Segment boundaries for locatable per-segment reporting: top-level bullets
+# and headings.
+_SEGMENT_RE = re.compile(r"\n(?=- \*\*|#)")
+
+# An asserted-here citation may only describe absence of evidence; any
+# measurement claim (MEASURED / MIXED) must come from the matrix.
+_ASSERTED_ALLOWED = frozenset({"DESIGNED", "NOT_RUN", "OUT_OF_SCOPE"})
+
+# Shipped-inventory lock: every row_id here must stay cited as a matrix-tied
+# citation in the register. Dropping one (or demoting it to asserted-here)
+# fails CI until this constant is updated in the same commit.
+_EXPECTED_MATRIX_ROW_CITATIONS = frozenset(
+    {
+        "retrieval.citation_existence_gate",
+        "integrity_check.claim_verification",
+        "revision.claim_drift_guard",
+        "review.calibration",
+    }
+)
+
+
+def _doc_text(root: Path) -> str:
+    return strip_non_rendering(
+        (root / DOC_RELPATH).read_text(encoding="utf-8")
+    )
+
+
+def _looks_like_repo_path(token: str) -> bool:
+    """A code span is treated as a repo path when it contains a "/" but is
+    not an absolute path or slash command (leading "/") or a URL. Spaces do
+    NOT opt a span out — a typo'd `scripts/no such file.py` must fail RR-1,
+    not silently skip it. Everything else (env vars, row_ids, JSON keys) is
+    opted out by construction."""
+    if "/" not in token or token.startswith("/"):
+        return False
+    if "://" in token:
+        return False
+    return True
+
+
+def referenced_paths(doc_text: str) -> tuple[list[str], list[str]]:
+    """(relative link targets, backtick repo paths) named by the doc.
+    Link targets keep their #fragment; callers split as needed."""
+    links = []
+    for target in link_targets(doc_text):
+        if target.startswith(NON_RELATIVE_LINK_PREFIXES):
+            continue
+        links.append(target)
+    spans = [
+        token
+        for token in code_spans(doc_text)
+        if _looks_like_repo_path(token)
+    ]
+    return links, spans
+
+
+def check_pointer_integrity(root: Path) -> list[str]:
+    errors: list[str] = []
+    root_abs = root.resolve()
+    doc_dir = (root / DOC_RELPATH).parent
+    links, spans = referenced_paths(_doc_text(root))
+    for target in links:
+        path_part, _, fragment = target.partition("#")
+        failure = relative_link_failure(root, root / DOC_RELPATH, target)
+        if failure is RelativeLinkFailure.ESCAPES_REPOSITORY:
+            errors.append(
+                f"RR-1: {DOC_RELPATH} links to {target!r}, which escapes "
+                "the repository root"
+            )
+        elif failure is RelativeLinkFailure.DOES_NOT_EXIST:
+            errors.append(
+                f"RR-1: {DOC_RELPATH} links to {target!r}, which does not "
+                f"exist relative to {doc_dir.name}/"
+            )
+        elif failure is RelativeLinkFailure.ANCHOR_ON_NON_MARKDOWN:
+            errors.append(
+                f"RR-1: {DOC_RELPATH} links to {target!r}, an anchor "
+                "on a non-markdown target"
+            )
+        elif failure is RelativeLinkFailure.ANCHOR_NOT_FOUND:
+            errors.append(
+                f"RR-1: {DOC_RELPATH} links to {target!r}, but no "
+                f"heading in {path_part!r} has id {fragment!r}"
+            )
+    for token in spans:
+        resolved = (root / token).resolve()
+        if not resolved.is_relative_to(root_abs) or not resolved.exists():
+            errors.append(
+                f"RR-1: {DOC_RELPATH} cites repo path `{token}`, which does "
+                "not exist"
+            )
+    return errors
+
+
+def check_status_mirroring(root: Path) -> list[str]:
+    errors: list[str] = []
+    doc_text = _doc_text(root)
+    try:
+        matrix = _load(root / MATRIX_RELPATH)
+    except ValueError as exc:
+        return [f"RR-2: cannot load {MATRIX_RELPATH}: {exc}"]
+    rows = {row.get("row_id"): _status(row) for row in matrix.get("rows", [])}
+
+    cited_matrix_rows: set[str] = set()
+    for segment in _SEGMENT_RE.split(doc_text):
+        first_line = segment.strip().split("\n", 1)[0]
+        is_evidence_bullet = segment.strip().startswith(_EVIDENCE_BULLET_PREFIX)
+        cites = _CITE_RE.findall(segment)
+
+        # Malformed-citation guard, reported against the segment so the
+        # offending text is locatable.
+        consumed = _CITE_RE.sub("", segment)
+        for phrase in _CITE_PHRASES:
+            if phrase in consumed:
+                errors.append(
+                    f"RR-2: {phrase!r} appears outside a well-formed "
+                    f"citation in segment starting {first_line!r}"
+                )
+
+        if is_evidence_bullet and not cites:
+            errors.append(
+                f"RR-2: evidence-status bullet carries no recognized "
+                f"citation: {first_line!r}"
+            )
+
+        for status, row_id in cites:
+            if status not in _BEHAVIORAL_STATUSES:
+                errors.append(
+                    f"RR-2: status {status!r} not in matrix vocabulary "
+                    f"{_BEHAVIORAL_STATUSES}"
+                )
+            if row_id:
+                # The inventory lock counts only citations sitting in an
+                # Evidence-status bullet: relocating a matrix citation into
+                # another bullet must not satisfy the lock.
+                if is_evidence_bullet:
+                    cited_matrix_rows.add(row_id)
+                if row_id not in rows:
+                    errors.append(
+                        f"RR-2: capability matrix row {row_id!r} does not "
+                        f"exist in {MATRIX_RELPATH}"
+                    )
+                elif rows[row_id] != status:
+                    errors.append(
+                        f"RR-2: register states {status!r} for matrix row "
+                        f"{row_id!r} but the matrix records {rows[row_id]!r}"
+                    )
+            elif status in _BEHAVIORAL_STATUSES and status not in _ASSERTED_ALLOWED:
+                errors.append(
+                    f"RR-2: asserted-here citation may not claim {status!r}; "
+                    "the matrix is the sole authority for measurement "
+                    "statuses"
+                )
+
+    for row_id in sorted(_EXPECTED_MATRIX_ROW_CITATIONS - cited_matrix_rows):
+        errors.append(
+            f"RR-2: expected matrix-row citation {row_id!r} is no longer "
+            "present in an Evidence-status bullet; update "
+            "_EXPECTED_MATRIX_ROW_CITATIONS in the same commit if intentional"
+        )
+    return errors
+
+
+def check_inbound_link(root: Path) -> list[str]:
+    """RR-3, same grammar as DF-3/CA-3: resolved-path compare, code spans
+    stripped so a backticked pseudo-link cannot satisfy the invariant."""
+    doc_abs = (root / DOC_RELPATH).resolve()
+    surface = root / README_RELPATH
+    if links_to(surface.read_text(encoding="utf-8"), surface.parent, doc_abs):
+        return []
+    return [
+        f"RR-3: {README_RELPATH} carries no rendered link to {DOC_RELPATH}"
+    ]
+
+
+def run_all_checks(root: Path) -> list[str]:
+    if not (root / DOC_RELPATH).exists():
+        return [f"RR-1: {DOC_RELPATH} is missing"]
+    errors: list[str] = []
+    errors.extend(check_pointer_integrity(root))
+    errors.extend(check_status_mirroring(root))
+    errors.extend(check_inbound_link(root))
+    return errors
+
+
+def main() -> int:
+    root = _SCRIPTS_DIR.parent
+    errors = run_all_checks(root)
+    if errors:
+        for line in errors:
+            print(f"ERROR: {line}", file=sys.stderr)
+        print(
+            f"check_risk_register: {len(errors)} violation(s)",
+            file=sys.stderr,
+        )
+        return 1
+    print("check_risk_register: OK (RR-1..RR-3)")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())