contextcompass 2.15.0__py3-none-any.whl

contextcompass_cli/_package/tools/system_documents/python/assemble_graph.py

@@ -0,0 +1,387 @@
+#!/usr/bin/env python3
+"""Assemble per-file graph descriptors into one Markdown document plus an index.
+
+Stage 2 of the system-documents pipeline. Stage 1 (`extract_graph.py`) crawls a
+source tree and writes one descriptor per source file. This script reads that
+tree and emits:
+
+    src_graph.md          one document, one section per source file
+    src_graph_index.md    the line range of every section, plus staleness proof
+
+Why Markdown rather than the previous `readable_src_graph.json` +
+`src_graph.json` pair: the JSON was 768 KB on a 574-file codebase and had to be
+read whole or not at all, because a JSON object has no addressable interior. A
+Markdown document with recorded line ranges is sliceable - an agent reads the
+index, finds the range for the file it cares about, and reads only those lines.
+Same content, and the index costs about 5% of the document.
+
+THE INDEX IS A BYPRODUCT, NOT A SECOND PASS
+This script knows every section's exact start and end line because it emitted
+them. It does not re-parse the document to find headings. That removes an entire
+class of drift: an index generated by re-walking a document can disagree with
+the document, and the only defence is a hash. Here they are produced together
+and cannot disagree. The hash is still recorded, but it guards against something
+else - a human editing the assembled file by hand, which they should not do.
+
+Line numbers are 1-based and inclusive on both ends, matching `sed -n 'S,Ep'`,
+the Read tool's offset/limit, and the `path:start_line-end_line` evidence
+convention used throughout this repository.
+
+Usage:
+    python assemble_graph.py --descriptors system_docs/graph --out system_docs
+    python assemble_graph.py --descriptors ... --out ... --check
+"""
+
+from __future__ import annotations
+
+import argparse
+import datetime
+import hashlib
+import json
+import pathlib
+import sys
+from typing import Any
+
+SCHEMA_VERSION = 1
+DOC_NAME = "src_graph.md"
+INDEX_NAME = "src_graph_index.md"
+
+# Section delimiters. Chosen to be greppable and unambiguous: a reader scanning
+# the assembled document can find any file's boundaries without the index, and a
+# validator can confirm the index agrees with the text.
+def fmt_cell(value: object) -> str:
+    """Render an authored edge attribute for a table cell.
+
+    A list becomes comma-joined; absent or empty becomes `-`. Derived edges
+    always render `-`, because these attributes are design facts and the
+    extractor is explicit that it cannot produce them.
+    """
+    if value is None or value == "" or value == []:
+        return "-"
+    if isinstance(value, (list, tuple)):
+        return ",".join(str(v) for v in value) or "-"
+    return str(value)
+
+
+HEADER = "<!-- BEGIN FILE: {source} -->"
+FOOTER = "<!-- END FILE: {source} -->"
+
+
+# Pipeline manifests that live beside descriptors but are not descriptors.
+# Named explicitly: an earlier version skipped anything starting with `_`, which
+# silently swallowed 14 real descriptors - `__init__.py`, every dunder metadata
+# module (`__version__.py`, `__architecture__.py`), and the `_builder.py` files.
+# A prefix is a naming convention, not a type. Identify by shape instead.
+MANIFEST_NAMES = frozenset({"_package_candidates.json"})
+
+
+def write_if_changed(path: pathlib.Path, text: str, volatile: str = "generated_at") -> bool:
+    """Write only when content differs ignoring the timestamp line. Returns wrote?
+
+    Regeneration must be idempotent. If the source did not change, the index
+    should not change either - otherwise every CI run or verification pass
+    rewrites the file with a fresh `generated_at` and produces a diff that says
+    nothing happened. A tool that churns its own output trains people to ignore
+    its diffs, which is precisely when a real change slips past.
+    """
+    if path.exists():
+        old = path.read_text(encoding="utf-8")
+        strip = lambda s: "\n".join(l for l in s.split("\n") if volatile not in l)
+        if strip(old) == strip(text):
+            return False
+    path.write_text(text, encoding="utf-8", newline="\n")
+    return True
+
+
+def load_descriptors(root: pathlib.Path) -> list[dict[str, Any]]:
+    """Read every descriptor under root, sorted by source path."""
+    out: list[dict[str, Any]] = []
+    for path in sorted(root.rglob("*.json")):
+        if path.name in MANIFEST_NAMES:
+            continue
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+        except json.JSONDecodeError as exc:
+            print(f"  SKIP (bad json) {path}: {exc}", file=sys.stderr)
+            continue
+        if "source" not in data or "nodes" not in data:
+            print(f"  SKIP (not a descriptor) {path}", file=sys.stderr)
+            continue
+        out.append(data)
+    return sorted(out, key=lambda d: d["source"])
+
+
+def render_node(node: dict[str, Any]) -> list[str]:
+    """One node as Markdown. Authored fields render only when present."""
+    lines = [f"#### `{node['label']}` ({node['kind']})", ""]
+    lines.append(f"- id: `{node['id']}`")
+    if node.get("lineno"):
+        lines.append(f"- defined at: `{node['file']}:{node['lineno']}`")
+    if node.get("bases"):
+        lines.append(f"- extends: {', '.join(f'`{b}`' for b in node['bases'])}")
+    if node.get("markers"):
+        lines.append(f"- markers: {', '.join(f'`{m}`' for m in node['markers'])}")
+    if node.get("role"):
+        lines.append(f"- role: {node['role']}")
+    for field in ("responsibilities", "owns_state", "phases"):
+        if node.get(field):
+            lines.append(f"- {field}: {', '.join(f'`{v}`' for v in node[field])}")
+    if node.get("public_methods"):
+        methods = node["public_methods"]
+        shown = ", ".join(f"`{m}`" for m in methods[:12])
+        more = f" (+{len(methods) - 12} more)" if len(methods) > 12 else ""
+        lines.append(f"- public methods: {shown}{more}")
+    if not any(node.get(f) for f in ("role", "responsibilities", "owns_state")):
+        lines.append("- **UNSEMANTIC** - mechanical scaffold only, not yet authored")
+    lines.append("")
+    return lines
+
+
+def render_descriptor(desc: dict[str, Any]) -> list[str]:
+    """One source file's whole section, between its header and footer."""
+    src = desc["source"]
+    lines = [HEADER.format(source=src), "", f"## {src}", ""]
+    lines.append(f"- source_sha256: `{desc.get('source_sha256', 'UNKNOWN')}`")
+    nodes = desc.get("nodes", {})
+    lines.append(f"- nodes: {len(nodes)}")
+    lines.append("")
+
+    if nodes:
+        lines.append("### Nodes")
+        lines.append("")
+        for node in nodes.values():
+            lines.extend(render_node(node))
+
+    edges = desc.get("edges_out", [])
+    authored = desc.get("edges_authored", [])
+    if edges or authored:
+        lines.append("### Edges out")
+        lines.append("")
+        lines.append("| from | relation | to | cardinality | phase | origin |")
+        lines.append("| --- | --- | --- | --- | --- | --- |")
+        for e in edges:
+            target = e.get("to") or f"{e.get('to_label', '?')} (unresolved)"
+            lines.append(f"| `{e['from']}` | {e['relation']} | `{target}` | - | - | derived |")
+        for e in authored:
+            lines.append(f"| `{e.get('from', '?')}` | {e.get('relation', '?')} | "
+                         f"`{e.get('to', '?')}` | {fmt_cell(e.get('cardinality'))} | "
+                         f"{fmt_cell(e.get('phase'))} | authored |")
+        lines.append("")
+
+        # `why` is the justification for a claim the extractor states it cannot
+        # derive. It goes beneath the table, not in it: the table stays scannable
+        # and the prose stays readable. Only authored edges carry one.
+        # Do not name these `src` - that is the source path this section is for,
+        # and the footer still needs it. Shadowing it emits a footer carrying a
+        # node id, which the range check catches but which should never be
+        # written in the first place.
+        whys = [(e.get("from", "?"), e.get("to", "?"), e["why"])
+                for e in authored if e.get("why")]
+        if whys:
+            for edge_from, edge_to, why in whys:
+                lines.append(f"- `{edge_from}` -> `{edge_to}`: {why}")
+            lines.append("")
+
+    candidates = desc.get("edge_candidates", [])
+    if candidates:
+        lines.append(f"### Edge candidates ({len(candidates)}, unconfirmed)")
+        lines.append("")
+        lines.append("Instantiation guesses from the AST. Over-generated roughly 8x "
+                     "against the reference graph; confirm or drop before relying on them.")
+        lines.append("")
+        for c in candidates[:20]:
+            lines.append(f"- `{c['from']}` creates `{c['to_label']}`")
+        if len(candidates) > 20:
+            lines.append(f"- ... {len(candidates) - 20} more")
+        lines.append("")
+
+    if desc.get("published_aliases"):
+        lines.append("### Published aliases")
+        lines.append("")
+        for a in desc["published_aliases"]:
+            lines.append(f"- `{a['alias']}` = `{a['target']}` (line {a['lineno']})")
+        lines.append("")
+
+    lines.append(FOOTER.format(source=src))
+    return lines
+
+
+def assemble(descriptors: list[dict[str, Any]]) -> tuple[str, list[dict[str, Any]]]:
+    """Build the document and the section table in one pass.
+
+    Returns (document_text, sections). Each section records the 1-based inclusive
+    line range of its file's block, including the header and footer comments.
+    """
+    # The preamble deliberately does NOT spell the delimiter literally. An
+    # earlier version documented the format inline, and the description of the
+    # marker was then indistinguishable from an instance of it - a validator
+    # grepping the document counted 576 sections where 575 exist. A file's own
+    # documentation must not be parseable as its own data.
+    lines: list[str] = [
+        "# src_graph",
+        "",
+        "Assembled system graph. One section per source file. Each section opens",
+        "with an HTML comment reading BEGIN FILE followed by the source path, and",
+        "closes with the matching END FILE comment. Exact line ranges are in",
+        f"`{INDEX_NAME}`; prefer the index over scanning for delimiters.",
+        "",
+        # Deliberately no tool path here. This document ships with the codebase
+        # it describes - committed, reviewed, and in a packaged project carried
+        # inside the wheel - and its readers have the source but not the tooling
+        # that assembled it. The old line named
+        # `tools/system_documents/python/assemble_graph.py`, which is dead text
+        # for every one of them, and which a check looking for `context_compass/`
+        # never caught because the prefix was never there.
+        "Generated, not authored. Do not hand-edit: edit the per-file",
+        "descriptors and reassemble.",
+        "",
+    ]
+    sections: list[dict[str, Any]] = []
+    for desc in descriptors:
+        start = len(lines) + 1                      # 1-based, header line
+        block = render_descriptor(desc)
+        lines.extend(block)
+        end = len(lines)                            # 1-based, footer line
+        sections.append({
+            "source": desc["source"],
+            "start": start,
+            "end": end,
+            "lines": end - start + 1,
+            "nodes": len(desc.get("nodes", {})),
+            # Must match what "### Edges out" actually renders, which is derived
+            # plus authored. Counting only edges_out undercounts exactly the
+            # edges that carry design meaning: 68% of relationships are not
+            # derivable, so the authored ones are the point of the layer. An
+            # index column that disagrees with the section it indexes is the
+            # failure this whole system exists to prevent.
+            "edges": len(desc.get("edges_out", [])) + len(desc.get("edges_authored", [])),
+            "source_sha256": desc.get("source_sha256", ""),
+        })
+        lines.append("")
+    return "\n".join(lines) + "\n", sections
+
+
+def render_index(doc_text: str, sections: list[dict[str, Any]]) -> str:
+    """The index. Ranges come from the assembly, never from re-parsing."""
+    line_count = len(doc_text.split("\n"))
+    if doc_text.endswith("\n"):
+        line_count -= 1
+    digest = hashlib.sha256(doc_text.encode("utf-8")).hexdigest()
+    stamp = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    out = [
+        "# src_graph_index",
+        "",
+        f"Line ranges into `{DOC_NAME}`. Emitted by the same pass that wrote the",
+        "document, so the ranges cannot have drifted from it.",
+        "",
+        "Line numbers are 1-based and inclusive on both ends.",
+        "",
+        "## Staleness proof",
+        "",
+        "| field | value |",
+        "| --- | --- |",
+        f"| document | `{DOC_NAME}` |",
+        f"| index_version | {SCHEMA_VERSION}.0.0 |",
+        f"| generated_at | {stamp} |",
+        f"| line_count | {line_count} |",
+        "| line_ending | lf |",
+        f"| content_sha256 | `{digest}` |",
+        f"| sections | {len(sections)} |",
+        "",
+        "Recompute `line_count` and `content_sha256` before slicing. On any",
+        "mismatch the document was hand-edited: STOP, do not slice, reassemble.",
+        "",
+        "## Sections",
+        "",
+        "| lines | source | nodes | edges |",
+        "| --- | --- | --- | --- |",
+    ]
+    for s in sections:
+        out.append(f"| {s['start']}-{s['end']} | `{s['source']}` | {s['nodes']} | {s['edges']} |")
+    out.append("")
+    return "\n".join(out)
+
+
+def verify(doc_text: str, sections: list[dict[str, Any]]) -> list[str]:
+    """Prove every recorded range actually brackets its own file's block.
+
+    An off-by-one here silently corrupts every downstream read, and it is the
+    one failure this whole design exists to prevent - so it is checked on every
+    run rather than trusted.
+    """
+    lines = doc_text.split("\n")
+    problems: list[str] = []
+    for s in sections:
+        head = lines[s["start"] - 1]
+        foot = lines[s["end"] - 1]
+        if head != HEADER.format(source=s["source"]):
+            problems.append(f"{s['source']}: start {s['start']} is {head!r}")
+        if foot != FOOTER.format(source=s["source"]):
+            problems.append(f"{s['source']}: end {s['end']} is {foot!r}")
+    for a, b in zip(sections, sections[1:]):
+        if a["end"] >= b["start"]:
+            problems.append(f"overlap: {a['source']} ends {a['end']}, "
+                            f"{b['source']} starts {b['start']}")
+    return problems
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description=__doc__,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--descriptors", required=True, type=pathlib.Path,
+                    help="descriptor tree written by extract_graph.py")
+    ap.add_argument("--out", required=True, type=pathlib.Path,
+                    help="directory to write src_graph.md and src_graph_index.md")
+    ap.add_argument("--check", action="store_true", help="verify only, write nothing")
+    args = ap.parse_args()
+
+    root: pathlib.Path = args.descriptors.resolve()
+    out_root: pathlib.Path = args.out.resolve()
+    if not root.is_dir():
+        print(f"descriptor root not found: {root}", file=sys.stderr)
+        return 2
+
+    descriptors = load_descriptors(root)
+    if not descriptors:
+        print(f"no descriptors under {root}", file=sys.stderr)
+        return 2
+
+    doc_text, sections = assemble(descriptors)
+    problems = verify(doc_text, sections)
+    index_text = render_index(doc_text, sections)
+
+    total_nodes = sum(s["nodes"] for s in sections)
+    total_edges = sum(s["edges"] for s in sections)
+    doc_lines = len(doc_text.split("\n")) - 1
+    idx_lines = len(index_text.split("\n"))
+
+    if problems:
+        print(f"VERIFY FAILED: {len(problems)} problem(s)", file=sys.stderr)
+        for p in problems[:20]:
+            print(f"  {p}", file=sys.stderr)
+        return 1
+
+    if not args.check:
+        out_root.mkdir(parents=True, exist_ok=True)
+        wrote_doc = write_if_changed(out_root / DOC_NAME, doc_text)
+        wrote_idx = write_if_changed(out_root / INDEX_NAME, index_text)
+    else:
+        wrote_doc = wrote_idx = False
+
+    if args.check:
+        mode = "CHECK (no writes)"
+    elif wrote_doc or wrote_idx:
+        mode = "WROTE"
+    else:
+        mode = "UNCHANGED"
+    print(f"{mode}: {DOC_NAME} + {INDEX_NAME} in {out_root}")
+    print(f"  sections={len(sections)}  nodes={total_nodes}  edges={total_edges}")
+    print(f"  document={doc_lines} lines  index={idx_lines} lines "
+          f"({100 * idx_lines // max(doc_lines, 1)}% of document)")
+    print(f"  all {len(sections)} ranges verified against their own headers")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())