contextcompass 2.15.0__py3-none-any.whl

contextcompass_cli/_package/tools/build_llm_full.py

@@ -0,0 +1,307 @@
+#!/usr/bin/env python3
+"""Concatenate the whole package into one file, with a line-range index.
+
+`llm_full.md` is every shipped file end to end, each bracketed by markers naming
+its path. `llm_full_index.md` says which lines each file occupies. Together they
+turn a million-character blob into something addressable: read the index, slice
+the two or three files you actually need, and never load the rest.
+
+    python build_llm_full.py --root . --out ../../llm_full.md
+    python build_llm_full.py --root . --out ../../llm_full.md --check
+    python build_llm_full.py --root . --out ../../llm_full.md --slice AGENTS.MD
+
+WHY THE INDEX IS A BYPRODUCT, NOT A SECOND PASS
+
+The ranges are recorded by the same loop that emits the lines, so the index
+cannot disagree with the document - there is no re-parse to get wrong. This is
+the same guarantee `assemble_graph.py` makes, and it is the reason both tools
+verify their own ranges before writing: an off-by-one here silently corrupts
+every downstream read, which is the one failure the whole design exists to
+prevent.
+
+The index also carries a staleness proof (`line_count`, `line_ending`,
+`content_sha256`). A line-offset index is more fragile than the text it
+indexes: insert one line near the top and every range below it is wrong while
+still parsing and still returning content - the WRONG content, confidently. So
+a consumer recomputes the proof and refuses to slice on any mismatch.
+
+DETERMINISM
+
+Paths are emitted with forward slashes in sorted order, so the same tree
+produces the same bytes on every platform. The artifact this replaced carried
+Windows separators because of where it happened to be generated, which meant it
+changed whenever the operating system did.
+"""
+
+from __future__ import annotations
+
+import argparse
+import datetime
+import hashlib
+import pathlib
+import sys
+
+INDEX_VERSION = "1.0.0"
+
+SKIP_DIRS = {"__pycache__", ".git", ".venv", ".pytest_cache", ".ruff_cache"}
+SKIP_SUFFIXES = {".pyc", ".pyo", ".so", ".dylib"}
+
+# Assembled from parts so this source file does not CONTAIN a complete marker.
+# It ships inside the package, so it gets inlined into its own output - and a
+# literal here is indistinguishable from a real delimiter to anything counting
+# them. The first build of this script emitted 443 markers for 442 files for
+# exactly that reason. Same failure `assemble_graph.py` records: a file's own
+# definition of its format must not be parseable as an instance of that format.
+_MARKER = "--- {kind} OF FILE: {path} ---"
+HEADER = _MARKER.replace("{kind}", "START")
+FOOTER = _MARKER.replace("{kind}", "END")
+
+BINARY_NOTE = "[binary file, {size} bytes - contents omitted]"
+
+
+def index_path_for(out: pathlib.Path) -> pathlib.Path:
+    return out.with_name(f"{out.stem}_index.md")
+
+
+def iter_files(root: pathlib.Path):
+    for p in sorted(root.rglob("*"), key=lambda x: x.relative_to(root).as_posix()):
+        if not p.is_file():
+            continue
+        if any(part in SKIP_DIRS for part in p.parts):
+            continue
+        if p.suffix.lower() in SKIP_SUFFIXES:
+            continue
+        yield p
+
+
+def build(root: pathlib.Path, label: str) -> tuple[str, list[dict]]:
+    """Return (document_text, sections). Ranges are 1-based, inclusive."""
+    lines: list[str] = [
+        f"# {label} - full package",
+        "",
+        "Every shipped file, concatenated. Each file opens with an HTML-style",
+        "comment reading START OF FILE followed by its path, and closes with the",
+        "matching END OF FILE comment.",
+        "",
+        "Exact line ranges are in the index beside this file. Read the index and",
+        "slice; scanning this document for delimiters defeats the point of having",
+        "one. Generated by `tools/build_llm_full.py` - do not hand-edit.",
+        "",
+    ]
+    sections: list[dict] = []
+    for p in iter_files(root):
+        rel = f"{label}/{p.relative_to(root).as_posix()}"
+        try:
+            body = p.read_text(encoding="utf-8")
+        except (UnicodeDecodeError, ValueError):
+            body = BINARY_NOTE.format(size=p.stat().st_size)
+
+        start = len(lines) + 1
+        lines.append(HEADER.format(path=rel))
+        lines.append("")
+        lines.extend(body.rstrip("\n").split("\n"))
+        lines.append("")
+        lines.append(FOOTER.format(path=rel))
+        end = len(lines)
+
+        sections.append({
+            "path": rel,
+            "start": start,
+            "end": end,
+            "lines": end - start + 1,
+            "bytes": p.stat().st_size,
+            "kind": p.suffix.lower().lstrip(".") or "none",
+        })
+        lines.append("")
+    return "\n".join(lines) + "\n", sections
+
+
+def verify(doc_text: str, sections: list[dict]) -> list[str]:
+    """Prove every recorded range brackets its own file. Checked, never assumed."""
+    lines = doc_text.split("\n")
+    problems: list[str] = []
+    for s in sections:
+        head, foot = lines[s["start"] - 1], lines[s["end"] - 1]
+        if head != HEADER.format(path=s["path"]):
+            problems.append(f"{s['path']}: start {s['start']} is {head!r}")
+        if foot != FOOTER.format(path=s["path"]):
+            problems.append(f"{s['path']}: end {s['end']} is {foot!r}")
+    for a, b in zip(sections, sections[1:]):
+        if a["end"] >= b["start"]:
+            problems.append(f"overlap: {a['path']} ends {a['end']}, "
+                            f"{b['path']} starts {b['start']}")
+    return problems
+
+
+def render_index(doc: pathlib.Path, doc_text: str, sections: list[dict]) -> str:
+    line_count = len(doc_text.split("\n"))
+    if doc_text.endswith("\n"):
+        line_count -= 1
+    digest = hashlib.sha256(doc_text.encode("utf-8")).hexdigest()
+    stamp = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    out = [
+        f"# {doc.stem}_index",
+        "",
+        f"Line ranges into `{doc.name}`. Emitted by the same pass that wrote the",
+        "document, so the ranges cannot have drifted from it.",
+        "",
+        "Line numbers are 1-based and inclusive on both ends, matching",
+        "`sed -n 'S,Ep'`, a Read tool's offset/limit, and this repository's",
+        "`path:start_line-end_line` evidence convention.",
+        "",
+        "## Staleness proof",
+        "",
+        "| field | value |",
+        "| --- | --- |",
+        f"| document | `{doc.name}` |",
+        f"| index_version | {INDEX_VERSION} |",
+        f"| generated_at | {stamp} |",
+        f"| line_count | {line_count} |",
+        "| line_ending | lf |",
+        f"| content_sha256 | `{digest}` |",
+        f"| files | {len(sections)} |",
+        "",
+        "Recompute `line_count` and `content_sha256` before slicing. On any",
+        "mismatch the document was hand-edited or rebuilt from a different tree:",
+        "STOP, do not slice, regenerate.",
+        "",
+        "## How to read one file",
+        "",
+        "Find its row, then read exactly those lines. Do not scan the document.",
+        "",
+        "```bash",
+        f"sed -n '11,58p' {doc.name}          # a range from the table below",
+        f"python tools/build_llm_full.py --root . --out {doc.name} --slice AGENTS.MD",
+        "```",
+        "",
+        "## Files",
+        "",
+        "| lines | path | span | bytes |",
+        "| --- | --- | --- | --- |",
+    ]
+    for s in sections:
+        out.append(f"| {s['start']}-{s['end']} | `{s['path']}` | {s['lines']} | {s['bytes']} |")
+    out.append("")
+    return "\n".join(out)
+
+
+def write_if_changed(path: pathlib.Path, text: str, volatile: str = "generated_at") -> bool:
+    """Write only when content differs ignoring the timestamp line.
+
+    Regeneration must be idempotent, or every verification pass rewrites the
+    file with a fresh stamp and produces a diff that says nothing happened. A
+    tool that churns its own output trains people to ignore its diffs, which is
+    precisely when a real change slips past.
+    """
+    if path.exists():
+        old = path.read_text(encoding="utf-8", errors="replace")
+        strip = lambda s: "\n".join(l for l in s.split("\n") if volatile not in l)
+        if strip(old) == strip(text):
+            return False
+    path.write_bytes(text.encode("utf-8"))
+    return True
+
+
+def do_slice(doc: pathlib.Path, needle: str, doc_text: str, sections: list[dict]) -> int:
+    """Print one file's lines, after proving the index on disk is current."""
+    index = index_path_for(doc)
+    if not index.is_file():
+        print(f"NO INDEX: {index.name}. Generate it before slicing.", file=sys.stderr)
+        return 1
+    import re
+    on_disk = index.read_text(encoding="utf-8")
+    want = re.search(r"content_sha256 \| `([0-9a-f]{64})`", on_disk)
+    live = hashlib.sha256(doc_text.encode("utf-8")).hexdigest()
+    if not want or want.group(1) != live:
+        print(f"INDEX STALE - refusing to slice. Regenerate {index.name}.", file=sys.stderr)
+        return 1
+
+    hits = [s for s in sections if needle.lower() in s["path"].lower()]
+    if not hits:
+        print(f"no file matching {needle!r}. Read {index.name} for the list.",
+              file=sys.stderr)
+        return 1
+    if len(hits) > 1:
+        print(f"{len(hits)} files match {needle!r} - narrow it:", file=sys.stderr)
+        for h in hits[:15]:
+            print(f"  {h['start']}-{h['end']}  {h['path']}", file=sys.stderr)
+        return 1
+    s = hits[0]
+    print(f"<!-- {doc.name}:{s['start']}-{s['end']}  {s['path']} -->")
+    print("\n".join(doc_text.split("\n")[s["start"] - 1:s["end"]]))
+    return 0
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0],
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--root", required=True, type=pathlib.Path,
+                    help="package root, the directory holding AGENTS.MD")
+    ap.add_argument("--out", required=True, type=pathlib.Path,
+                    help="file to write; the index goes beside it as <stem>_index.md")
+    ap.add_argument("--label", default="context_compass",
+                    help="prefix for each emitted path (default: context_compass)")
+    ap.add_argument("--check", action="store_true",
+                    help="compare against what is on disk, write nothing")
+    ap.add_argument("--slice", metavar="PATH",
+                    help="print one file's lines (verifies the index first)")
+    args = ap.parse_args()
+
+    if not (args.root / "AGENTS.MD").is_file():
+        print(f"ERROR: {args.root} is not a package root (no AGENTS.MD)")
+        return 2
+
+    text, sections = build(args.root, args.label)
+
+    problems = verify(text, sections)
+    if problems:
+        print(f"VERIFICATION FAILED: {len(problems)} problem(s). Nothing written.",
+              file=sys.stderr)
+        for p in problems[:15]:
+            print(f"  {p}", file=sys.stderr)
+        return 1
+
+    index_path = index_path_for(args.out)
+    index_text = render_index(args.out, text, sections)
+
+    if args.slice:
+        if not args.out.is_file():
+            print(f"MISSING: {args.out}", file=sys.stderr)
+            return 1
+        return do_slice(args.out, args.slice, args.out.read_text(encoding="utf-8"),
+                        sections)
+
+    if args.check:
+        missing = [p.name for p in (args.out, index_path) if not p.is_file()]
+        if missing:
+            print(f"MISSING: {', '.join(missing)}")
+            return 1
+        doc_same = args.out.read_text(encoding="utf-8", errors="replace") == text
+        idx_same = "\n".join(
+            l for l in args.out.with_name(index_path.name)
+            .read_text(encoding="utf-8").split("\n") if "generated_at" not in l
+        ) == "\n".join(l for l in index_text.split("\n") if "generated_at" not in l)
+        if doc_same and idx_same:
+            print(f"OK: {args.out.name} and {index_path.name} are current "
+                  f"({len(sections)} files)")
+            return 0
+        print(f"STALE: {'document' if not doc_same else ''}"
+              f"{' and ' if not doc_same and not idx_same else ''}"
+              f"{'index' if not idx_same else ''} differs "
+              f"({len(sections)} files rebuilt)")
+        return 1
+
+    wrote_doc = write_if_changed(args.out, text, volatile="\x00")
+    wrote_idx = write_if_changed(index_path, index_text)
+
+    print(f"{'WROTE' if wrote_doc else 'UNCHANGED'}: {args.out}")
+    print(f"{'WROTE' if wrote_idx else 'UNCHANGED'}: {index_path}")
+    print(f"  {len(sections)} files, {len(text.split(chr(10))):,} lines, {len(text):,} chars")
+    print(f"  all {len(sections)} ranges verified against their own markers")
+    print("  sorted order and forward slashes, so this is byte-stable")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())