codebase-explorer 2.2.0__py3-none-any.whl

cbe/batching.py

@@ -0,0 +1,283 @@
+"""Source scan and reading batches: which files are read together by one detail author.
+
+A batch (``Module``) is a reading unit, not a semantic module. Batches keep a
+subsystem and its tests apart, keep fitting directory subtrees whole, and join
+files linked by imports first, within a token target per batch.
+"""
+
+from __future__ import annotations
+
+import ast
+import hashlib
+import os
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from cbe import grammars
+from cbe.token_budget import count_text_tokens
+
+
+class BatchingError(ValueError):
+    """The repository cannot be split into reading batches."""
+
+
+TARGET_TOKENS = 20_000
+MAX_TOKENS = 25_000
+MAX_SINGLE_FILE_TOKENS = 60_000
+
+
+def is_test_path(path: str) -> bool:
+    return path.startswith(("tests/", "test/", "t/")) or "/test_" in path
+
+
+@dataclass
+class SourceFile:
+    path: str
+    source: str
+    tokens: int
+    digest: str
+    imports: set[str] = field(default_factory=set)
+
+    @property
+    def is_test(self) -> bool:
+        return is_test_path(self.path)
+
+
+@dataclass
+class Module:
+    number: int
+    files: list[SourceFile]
+    bucket: str
+    title: str = ""
+
+    @property
+    def tokens(self) -> int:
+        return sum(item.tokens for item in self.files)
+
+    @property
+    def is_test(self) -> bool:
+        return all(item.is_test for item in self.files)
+
+
+def _imports(path: str, source: str) -> set[str]:
+    names: set[str] = set()
+    if path.endswith(".py"):
+        try:
+            tree = ast.parse(source)
+        except SyntaxError:
+            return names
+        for node in ast.walk(tree):
+            if isinstance(node, ast.Import):
+                names.update(alias.name for alias in node.names)
+            elif isinstance(node, ast.ImportFrom) and node.module:
+                names.add(node.module)
+                names.update(node.module + "." + alias.name for alias in node.names)
+    else:
+        names.update(re.findall(r"#include\s*[\"<]([^\">]+)[\">]", source))
+        names.update(re.findall(r"\bfrom\s*[\"']([^\"']+)[\"']", source))
+        names.update(re.findall(r"\brequire\s*\(\s*[\"']([^\"']+)[\"']", source))
+    return names
+
+
+def scan(repo: Path) -> list[SourceFile]:
+    """Every regular source file outside the excluded directories; symlinks are never followed."""
+    suffixes = grammars.registered_suffixes() | grammars.FALLBACK_SUFFIXES
+    files: list[SourceFile] = []
+    for path in sorted(repo.rglob("*")):
+        if not path.is_file() or path.is_symlink() or path.suffix.lower() not in suffixes:
+            continue
+        if any(part in grammars.DEFAULT_EXCLUDE_DIR_NAMES for part in path.relative_to(repo).parts):
+            continue
+        raw = path.read_bytes()
+        try:
+            source = raw.decode("utf-8")
+        except UnicodeDecodeError as exc:
+            raise BatchingError(f"source is not UTF-8: {path.relative_to(repo)}") from exc
+        rel = path.relative_to(repo).as_posix()
+        files.append(SourceFile(rel, source, count_text_tokens(source), hashlib.sha256(raw).hexdigest(),
+                                _imports(rel, source)))
+    if not files:
+        raise BatchingError("no supported source files")
+    return files
+
+
+def _subsystem(item: SourceFile, production_areas: set[str],
+               production_files: list[SourceFile], primary_root: str) -> str:
+    parts = Path(item.path).parts
+    if not item.is_test:
+        if parts[0] != primary_root:
+            return parts[0]
+        return parts[1] if len(parts) > 2 else "core"
+
+    # A test's physical directory remains its packing boundary. The matching
+    # source subsystem only decides where its page appears in the index.
+    ignored = {"t", "test", "tests", "unit", "integration", "smoke", "benchmarks"}
+    for part in parts[1:-1]:
+        if part in ignored:
+            continue
+        if part in production_areas:
+            return part
+        for area in sorted(production_areas):
+            if part.rstrip("s") == area.rstrip("s"):
+                return area
+    local_area = next((part for part in parts[1:-1] if part not in ignored), "core")
+    stem = Path(item.path).stem.removeprefix("test_").removesuffix("_test")
+    if stem in {"__init__", "conftest"}:
+        return local_area
+    words = set(stem.split("_"))
+    for area in sorted(production_areas):
+        if area.rstrip("s") in {word.rstrip("s") for word in words}:
+            return area
+    matches = [(len(Path(source.path).parts),
+                _subsystem(source, production_areas, [], primary_root))
+               for source in production_files
+               if Path(source.path).stem == stem or
+               Path(source.path).stem.rstrip("s") == stem.rstrip("s")]
+    if matches:
+        # A top-level implementation is a better default than an equally
+        # named CLI adapter or example nested several directories below it.
+        return min(matches)[1]
+    return local_area
+
+
+def _pack_directory(files: list[SourceFile], *, strict: bool = False,
+                    target_tokens: int = TARGET_TOKENS, max_tokens: int = MAX_TOKENS) -> list[list[SourceFile]]:
+    """Join imports first, then fill only within a specific source directory."""
+    groups = {i: [item] for i, item in enumerate(sorted(files, key=lambda item: item.path))}
+    tokens = {i: group[0].tokens for i, group in groups.items()}
+    candidates: dict[str, set[int]] = {}
+    for i, group in groups.items():
+        path = Path(group[0].path)
+        for name in {group[0].path, path.name, path.stem, ".".join(path.with_suffix("").parts),
+                     ".".join(path.with_suffix("").parts[1:])}:
+            candidates.setdefault(name, set()).add(i)
+    links: dict[int, dict[int, int]] = {i: {} for i in groups}
+    for i, group in groups.items():
+        targets: set[int] = set()
+        for ref in group[0].imports:
+            # Exact names and nonempty dotted suffixes implement the original
+            # candidate match; a set counts each directed file pair just once.
+            keys = {ref} | {ref[pos + 1:] for pos, char in enumerate(ref)
+                            if char == "." and ref[pos + 1:]}
+            for key in keys:
+                targets.update(candidates.get(key, ()))
+        for j in targets - {i}:
+            links[i][j] = links[i].get(j, 0) + 1
+            links[j][i] = links[j].get(i, 0) + 1
+    while True:
+        pairs = []
+        ids = list(groups)
+        for offset, left in enumerate(ids):
+            for right in ids[offset + 1:]:
+                combined_tokens = tokens[left] + tokens[right]
+                if combined_tokens > max_tokens:
+                    continue
+                weight = links[left].get(right, 0)
+                if weight:
+                    # Stable ids retain group order after removals, hence the
+                    # same left/right tie break as the original list indices.
+                    pairs.append((weight, -abs(target_tokens - combined_tokens),
+                                  -left, -right, left, right))
+        if not pairs:
+            break
+        *_, left, right = max(pairs)
+        groups[left].extend(groups.pop(right))
+        tokens[left] += tokens.pop(right)
+        for other in groups:
+            if other == left:
+                continue
+            weight = links[left].get(other, 0) + links[right].get(other, 0)
+            if weight:
+                links[left][other] = links[other][left] = weight
+            links[other].pop(right, None)
+        links[left].pop(right, None)
+        links.pop(right)
+    if strict:
+        return list(groups.values())
+    packed: list[list[SourceFile]] = []
+    for group in sorted(groups.values(), key=lambda group: (-sum(item.tokens for item in group), group[0].path)):
+        candidates = [existing for existing in packed
+                      if sum(item.tokens for item in existing + group) <= max_tokens]
+        if candidates:
+            chosen = max(candidates, key=lambda existing: (
+                min(sum(item.tokens for item in existing + group), target_tokens),
+                -packed.index(existing)))
+            chosen.extend(group)
+        else:
+            packed.append(group)
+    return packed
+
+
+def _directory_groups(files: list[SourceFile], *, strict_root: Path | None = None,
+                      target_tokens: int = TARGET_TOKENS, max_tokens: int = MAX_TOKENS) -> list[list[SourceFile]]:
+    """Keep fitting subtrees whole; recurse only when a subtree is oversized."""
+    if sum(item.tokens for item in files) <= max_tokens:
+        return [files]
+    parents = {Path(item.path).parent for item in files}
+    common = Path(os.path.commonpath([str(parent) for parent in parents]))
+    direct = [item for item in files if Path(item.path).parent == common]
+    children: dict[str, list[SourceFile]] = {}
+    for item in files:
+        if item in direct:
+            continue
+        relative = Path(item.path).relative_to(common)
+        children.setdefault(relative.parts[0], []).append(item)
+    groups = _pack_directory(direct, strict=common == strict_root,
+                             target_tokens=target_tokens, max_tokens=max_tokens)
+    child_groups = [_directory_groups(children[name], strict_root=strict_root,
+                                     target_tokens=target_tokens, max_tokens=max_tokens)
+                    for name in sorted(children)]
+    # Small sibling directories may share one module, but never cross an area
+    # or a test/production boundary (the caller already partitioned both).
+    small = [group for sibling in child_groups for group in sibling
+             if sum(item.tokens for item in group) < 4_000]
+    groups.extend(group for sibling in child_groups for group in sibling
+                  if sum(item.tokens for item in group) >= 4_000)
+    if small:
+        groups.extend(_pack_directory([item for group in small for item in group],
+                                      target_tokens=target_tokens, max_tokens=max_tokens))
+    return groups
+
+
+def plan(files: list[SourceFile], *, target_tokens: int = TARGET_TOKENS,
+         max_tokens: int = MAX_TOKENS) -> list[Module]:
+    if target_tokens < 1 or max_tokens < target_tokens:
+        raise BatchingError("batch limits require 0 < target_tokens <= max_tokens")
+    for item in files:
+        if item.tokens > MAX_SINGLE_FILE_TOKENS:
+            raise BatchingError(f"one whole source file exceeds {MAX_SINGLE_FILE_TOKENS} tokens: {item.path}")
+    production = [item for item in files if not item.is_test]
+    root_tokens: dict[str, int] = {}
+    for item in production:
+        root = Path(item.path).parts[0]
+        root_tokens[root] = root_tokens.get(root, 0) + item.tokens
+    primary_root = max(root_tokens, key=root_tokens.get) if root_tokens else ""
+    production_areas = {(_subsystem(item, set(), [], primary_root)) for item in production}
+    partitions: dict[tuple[str, bool], list[SourceFile]] = {}
+    for item in files:
+        area = _subsystem(item, production_areas, production, primary_root)
+        partitions.setdefault((area, item.is_test), []).append(item)
+    bins: list[Module] = []
+    for (area, is_test), members in sorted(partitions.items()):
+        strict_root = Path(primary_root) if area == "core" and not is_test else None
+        for group in _directory_groups(members, strict_root=strict_root,
+                                       target_tokens=target_tokens, max_tokens=max_tokens):
+            bins.append(Module(len(bins), sorted(group, key=lambda item: item.path), area))
+    return bins
+
+
+def fallback_title(module: Module) -> str:
+    """A batch label a person can recognise in the progress log: directory plus its first files."""
+    directory = primary_directory(module)
+    names = sorted(Path(item.path).name for item in module.files)
+    more = f" (+{len(names) - 2})" if len(names) > 2 else ""
+    return f"{directory or '.'}: {', '.join(names[:2])}{more}"[:80]
+
+
+def primary_directory(module: Module) -> str:
+    counts: dict[str, int] = {}
+    for item in module.files:
+        directory = Path(item.path).parent.as_posix()
+        counts[directory] = counts.get(directory, 0) + 1
+    return min(counts, key=lambda path: (-counts[path], path))

cbe/catalog_query.py

@@ -0,0 +1,376 @@
+"""Read-side queries over a rendered run's ``catalog.json``.
+
+``find``, ``query`` and ``source`` back the CLI lookup commands;
+``layer_metrics`` measures the storage layers of a run.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+from typing import Any
+
+from cbe.symbols import SemanticError
+
+RUN_VERSION_ERROR = "this run directory was made by a different version of cbe; start a new plan"
+
+
+def catalog_symbols(catalog: dict) -> dict[str, dict]:
+    """Resolve the current catalog's public search view.
+
+    Inventory IDs retain their names/facts. The semantic index supplies their
+    disposition and primary page; synthetic units are also first-class query
+    targets.
+    """
+    records = {key: dict(row) for key, row in (catalog.get("symbols") or {}).items()}
+    for uid, unit in (catalog.get("units") or {}).items():
+        record = {**records.get(uid, {}), **unit}
+        record.setdefault("name", (catalog.get("short_names") or {}).get(uid, uid))
+        paths = sorted({row["path"] for row in unit.get("ranges") or [] if row.get("path")})
+        record.setdefault("files", paths)
+        if len(paths) == 1:
+            record.setdefault("file", paths[0])
+        records[uid] = record
+    for sid, resolution in catalog["symbol_index"].items():
+        records[sid] = {**records.get(sid, {}),
+                        **(catalog.get("dispositions") or {}).get(sid, {}),
+                        **resolution}
+    for sid, record in records.items():
+        target = record.get("unit") or (sid if sid in (catalog.get("units") or {}) else None)
+        detail = (catalog.get("units") or {}).get(target)
+        if detail is not None:
+            records[sid] = {**detail, **record, "id": sid,
+                            "unit_id": target, "unit_ranges": detail.get("ranges", [])}
+        else:
+            record.setdefault("explain", record.get("reason", ""))
+            for field in ("signatures", "relations", "reverse_relations", "outgoing", "incoming"):
+                record.setdefault(field, [])
+    edge_table = catalog.get("edges") or {}
+    for sid, record in records.items():
+        record["display_name"] = (catalog.get("short_names") or {}).get(sid, record.get("name", sid))
+        for direction in ("outgoing", "incoming"):
+            record[direction] = [edge_table[e] if isinstance(e, str) else e
+                                 for e in record.get(direction) or []]
+    return records
+
+
+def query_catalog(run_dir: Path) -> dict[str, Any]:
+    catalog = json.loads((run_dir / "catalog.json").read_text(encoding="utf-8"))
+    if "symbol_index" not in catalog:
+        raise SemanticError(RUN_VERSION_ERROR)
+    catalog["symbols"] = catalog_symbols(catalog)
+    return catalog
+
+
+def format_ranges(ranges: list[dict]) -> str:
+    """Compact, unambiguous source coordinates for agent-facing text."""
+    return ", ".join(f"{row['path']}:L{start}-L{end}"
+                     for row in ranges for start, end in row.get("lines", []))
+
+
+def find(run_dir: Path, term: str, limit: int = 25) -> list[dict[str, Any]]:
+    """Locate matches; detailed evidence stays behind query."""
+    from cbe.records import summary_sentence
+
+    catalog = query_catalog(run_dir)
+    inventory = {s["id"]: (path, s) for path, entry in catalog.get("inventory", {}).items()
+                 for s in entry.get("symbols", [])}
+    needle = term.casefold()
+    results = []
+    candidates = [(sid, record, False) for sid, record in catalog["symbols"].items()]
+    candidates += [(path, record, True) for path, record in catalog["files"].items() if record.get("facts")]
+    for sid, record, is_file in candidates:
+        searchable = {"facts": record["facts"]} if is_file else {"id": sid, **record}
+        matched = [field for field, value in searchable.items()
+                   if needle in json.dumps(value, ensure_ascii=False).casefold()]
+        if not matched:
+            continue
+        path, symbol = inventory.get(sid, (record.get("file"), {}))
+        ranges = ([{"path": path, "lines": symbol["lines"]}] if symbol.get("lines") else
+                  record.get("ranges") or [])
+        if is_file and not ranges:
+            ranges = [{"path": sid, "lines": [[1, max(1, catalog.get("inventory", {}).get(sid, {}).get("nlines", 1))]]}]
+        name = record.get("display_name") or record.get("name") or sid
+        if "::" in name:
+            name = symbol.get("name") or name
+        summary = summary_sentence(record) or summary_sentence({"explain": record.get("reason", "")})
+        words = summary.split()
+        summary = " ".join(words[:25]) + ("…" if len(words) > 25 else "")
+        results.append({"id": sid, "name": name, "kind": symbol.get("kind") or record.get("kind") or
+                        ("file" if is_file else "symbol"), "location": format_ranges(ranges),
+                        "summary": summary, "matched_fields": matched})
+    results.sort(key=lambda row: (0 if needle in {row["id"].casefold(), row["name"].casefold(),
+                                                  str(catalog["symbols"].get(row["id"], {}).get("display_name", "")).casefold()}
+                                 else 1))
+    return results[:limit]
+
+
+def find_text(results: list[dict]) -> str:
+    return "".join(f"{r['name']} | {r['id']} | {r['kind']} | {r['location']} | "
+                   f"{r['summary']} | match: {','.join(r['matched_fields'])}\n"
+                   for r in results)
+
+
+def resolve_name(catalog: dict[str, Any], name: str) -> str | None:
+    """The one ID a name copied from a page stands for; None when it is no page name.
+
+    Tried in order: a group unit's name without its ``grp:`` prefix; a unit heading or
+    anchor as the page prints it; a symbol name. A step that matches several IDs raises
+    with all of them: the caller picks, the program does not guess."""
+    term = name.strip().strip("`").strip()
+    if not term:
+        return None
+    units, names = catalog.get("units", {}), catalog.get("short_names", {})
+    anchor = term.lstrip("#")
+    steps = [
+        [f"grp:{term}"] if f"grp:{term}" in catalog["symbols"] else [],
+        sorted(uid for uid, unit in units.items()
+               if names.get(uid) == term or (unit.get("anchor") or "").lstrip("#") == anchor),
+        sorted(sid for sid, record in catalog["symbols"].items()
+               if term in {record.get("name"), record.get("display_name"),
+                           sid.split("::")[2] if sid.count("::") >= 2 else None}),
+    ]
+    for ids in steps:
+        if len(ids) == 1:
+            return ids[0]
+        if ids:
+            raise SemanticError(f"{name} matches {len(ids)} IDs; query one of them:\n" + "\n".join(ids))
+    return None
+
+
+def query(run_dir: Path, symbol_id: str, edges: str | None = None) -> dict[str, Any]:
+    """Compact object view; frozen graph is read only for explicit expansion."""
+    if edges is not None and edges != "all" and (
+            not edges or set(edges.split(",")) - {"import", "call", "cfg", "dataflow"}):
+        raise SemanticError("--edges must be all or comma-separated import,call,cfg,dataflow")
+    catalog = query_catalog(run_dir)
+    asked = symbol_id
+    if not any(symbol_id in catalog.get(table, {}) for table in ("nodes", "symbols", "files")):
+        symbol_id = resolve_name(catalog, symbol_id) or symbol_id
+    table = catalog.get("edges") or {}
+    if symbol_id in catalog.get("nodes", {}):
+        return {"id": symbol_id, "name": catalog["nodes"][symbol_id]["title"],
+                **catalog["nodes"][symbol_id],
+                "dependencies": [e for e in catalog.get("node_dependencies", []) if e["from"] == symbol_id],
+                "depended_on_by": [e for e in catalog.get("node_dependencies", []) if e["to"] == symbol_id]}
+    if symbol_id in catalog["symbols"]:
+        record = {"id": symbol_id, **catalog["symbols"][symbol_id]}
+        for direction in ("outgoing", "incoming"):
+            record[direction] = [table[e] if isinstance(e, str) else e
+                                 for e in record.get(direction) or []]
+    elif symbol_id in catalog["files"]:
+        def listing(uid, row):
+            return {"id": uid, "name": catalog.get("short_names", {}).get(uid, row.get("name", uid)),
+                    "ranges": row.get("ranges", []), "summary": row.get("summary", row.get("reason", ""))}
+        record = {"id": symbol_id, "ranges": catalog["files"][symbol_id].get("ranges", []),
+                  **{key: catalog["files"][symbol_id][key] for key in ("tokens", "language", "known_empty")
+                     if key in catalog["files"][symbol_id]},
+                  "units": [listing(uid, u) for uid, u in catalog.get("units", {}).items()
+                            if any(r["path"] == symbol_id for r in u.get("ranges") or [])],
+                  "symbols": [listing(sid, s) for sid, s in catalog["symbols"].items()
+                              if s.get("file") == symbol_id and sid not in catalog.get("units", {})],
+                  "facts": catalog["files"][symbol_id].get("facts", {}),
+                  "imports": [e for e in table.values() if e.get("kind") == "import"
+                              and e.get("path") == symbol_id]}
+    else:
+        raise SemanticError(f"unknown unit, symbol or file: {symbol_id}; "
+                            f"first use cbe find --run-dir {run_dir} --term {symbol_id}")
+    record.setdefault("name", symbol_id)
+    record["display_name"] = catalog.get("short_names", {}).get(symbol_id, record["name"])
+    if asked != symbol_id:
+        record["resolved_from"] = asked
+    if edges:
+        from cbe.records import SourceOwnership, frozen_sources
+
+        context = json.loads((run_dir / "context.json").read_text(encoding="utf-8"))
+        graph = context.get("graph") or catalog.get("graph") or {}
+        nodes = {n["id"]: n for n in graph.get("nodes") or []}
+        ownership = SourceOwnership(frozen_sources(context), context.get("chunks") or [],
+                                    catalog["symbols"])
+        members = ({symbol_id, *record.get("folded_members", [])}
+                   if symbol_id in catalog.get("units", {}) else {symbol_id})
+        def matches(edge):
+            if symbol_id in catalog["files"]:
+                node = nodes.get(edge.get("source"), {})
+                owner = nodes.get(node.get("owner"), node)
+                return (edge.get("path") or owner.get("path")) == symbol_id
+            return ownership.edge_source(edge, nodes) in members
+        from cbe.graph import NO_DATAFLOW_MESSAGE, built_without_dataflow
+
+        if built_without_dataflow(graph) and "dataflow" in edges.split(","):
+            if edges == "dataflow":
+                raise SemanticError(NO_DATAFLOW_MESSAGE)
+            record["notice"] = NO_DATAFLOW_MESSAGE
+        record["raw_edges"] = [e for e in graph.get("edges") or []
+                               if (edges == "all" or e.get("kind") in edges.split(",")) and matches(e)]
+    return record
+
+
+def query_text(record: dict) -> str:
+    """Agent view: prose once, one line per relation or graph edge."""
+    from cbe.records import summary_and_remainder
+
+    summary, remainder = summary_and_remainder(record)
+    lines = [f"resolved: {record['resolved_from']} -> {record['id']}"] if record.get("resolved_from") else []
+    lines.append(f"{record.get('display_name', record['name'])}  {record['id']}")
+    if record.get("ungrounded"):
+        lines.append(f"check: names {', '.join(record['ungrounded'])}, not found in this unit's source")
+    for field in ("dependencies", "depended_on_by", "signature", "signatures", "ranges", "summary", "explain", "extras",
+                  "uncertainties", "relations", "reverse_relations", "folded_members",
+                  "internal_edge_counts", "facts", "units", "symbols", "notice"):
+        value = record.get(field)
+        if not value or (field == "summary" and remainder != (record.get("explain") or "").strip()):
+            continue
+        if field == "signatures" and len(value) == 1 and value[0]["signature"] == record.get("signature"):
+            continue
+        if field == "ranges":
+            lines.append(f"ranges: {format_ranges(value)}")
+        elif field in {"units", "symbols"}:
+            lines.extend(f"{field}: {row['name']} | {row['id']} | {format_ranges(row['ranges'])} | {row['summary']}"
+                         for row in value)
+        elif isinstance(value, list):
+            lines.extend(f"{field}: {json.dumps(row, ensure_ascii=False)}" for row in value)
+        else:
+            lines.append(f"{field}: {value}")
+    for direction in ("outgoing", "incoming", "imports", "raw_edges"):
+        for edge in record.get(direction) or []:
+            endpoint = "source" if direction == "incoming" else "target"
+            lines.append(f"{direction}: {edge.get('kind')} {edge.get(endpoint + '_name', edge.get(endpoint))} "
+                         f"{edge.get(endpoint + '_symbol') or edge.get(endpoint)} [{edge.get('status', 'structural')}] "
+                         f"{edge.get('path', '')}:L{edge.get('line', '')} id={edge['id']}")
+    return "\n".join(lines) + "\n"
+
+
+def source(run_dir: Path, object_id: str, context: int = 0) -> str:
+    """Return numbered frozen source, checking all selected files before output."""
+    from cbe.records import frozen_sources
+
+    if context < 0:
+        raise SemanticError("--context must be nonnegative")
+    record = query(run_dir, object_id)
+    ranges = record.get("ranges") or []
+    if not ranges:
+        raise SemanticError(f"no source ranges for: {object_id}")
+    frozen = json.loads((run_dir / "context.json").read_text(encoding="utf-8"))
+    texts = frozen_sources(frozen, [row["path"] for row in ranges])
+    selected: dict[str, list[list[int]]] = {}
+    for row in ranges:
+        nlines = len(texts[row["path"]].splitlines())
+        for start, end in row["lines"]:
+            if nlines == 0:
+                selected.setdefault(row["path"], [])
+                continue
+            if not 1 <= start <= end <= nlines:
+                raise SemanticError(f"source range out of bounds: {row['path']}:{start}-{end}")
+            selected.setdefault(row["path"], []).append([max(1, start - context), min(nlines, end + context)])
+    output = [f"# resolved: {record['resolved_from']} -> {record['id']}"] if record.get("resolved_from") else []
+    for path, spans in sorted(selected.items()):
+        merged: list[list[int]] = []
+        for start, end in sorted(spans):
+            if merged and start <= merged[-1][1] + 1:
+                merged[-1][1] = max(merged[-1][1], end)
+            else:
+                merged.append([start, end])
+        lines = texts[path].splitlines()
+        if not merged:
+            output.append(f"# {path}: empty file")
+        for start, end in merged:
+            output.append(f"# {path}:L{start}-L{end}")
+            output.extend(f"{number}: {lines[number - 1]}" for number in range(start, end + 1))
+    return "\n".join(output) + "\n"
+
+
+def layer_metrics(run_dir: Path) -> dict:
+    """One storage-based token measure.
+
+    Markdown includes page metadata, navigation, signatures and usage examples.
+    Catalog JSON includes its preserved copies of prose. Frozen graph evidence
+    is reported as UTF-8 JSON bytes, excluded from token ratios.
+    These are storage layers, not disjoint semantic content classifications.
+    The denominator counts each complete frozen source file exactly once.
+    """
+    from cbe.records import frozen_sources, summary_and_remainder
+    from cbe.token_budget import count_text_tokens, count_section_tokens
+
+    context = json.loads((run_dir / "context.json").read_text(encoding="utf-8"))
+    source_tokens = sum(count_text_tokens(text) for text in frozen_sources(context).values())
+    semantic = sum(count_text_tokens(path.read_text(encoding="utf-8"))
+                   for path in sorted((run_dir / "docs").rglob("*.md")))
+    catalog_text = (run_dir / "catalog.json").read_text(encoding="utf-8")
+    catalog = json.loads(catalog_text)
+    catalog_tokens = count_text_tokens(catalog_text)
+    body_lines = set()
+    for unit in catalog.get("units", {}).values():
+        for field in ("explain", "summary"):
+            body_lines.update(line.strip() for line in (unit.get(field) or "").splitlines() if line.strip())
+        _, remainder = summary_and_remainder(unit)
+        body_lines.update(line.strip() for line in remainder.splitlines() if line.strip())
+        for extra in unit.get("extras") or []:
+            if isinstance(extra, dict):
+                body_lines.update(line.strip() for line in
+                                  f"- {extra.get('topic')}: {str(extra.get('note', '')).strip()}".splitlines()
+                                  if line.strip())
+        for item in unit.get("uncertainties") or []:
+            body_lines.update(line.strip() for line in f"- {item}".splitlines() if line.strip())
+    breakdown = dict.fromkeys(("model_body", "signature_location", "heading_anchor", "relations",
+                               "footer", "index", "node_text"), 0)
+    for path in sorted((run_dir / "docs").rglob("*.md")):
+        sections = []
+        in_units = False
+        in_footer = False
+        for line in path.read_text(encoding="utf-8").splitlines(keepends=True):
+            stripped = line.strip()
+            if stripped.startswith("### ") or '<a id="u-' in stripped:
+                in_units = True
+            if stripped.startswith("Look up code:"):
+                in_footer = True
+            if path.parent == run_dir / "docs":
+                category = "index"
+            elif re.match(r"\[f\d+\]: ", stripped):
+                category = "signature_location"
+            elif in_footer:
+                category = "footer"
+            elif in_units and stripped in body_lines:
+                category = "model_body"
+            elif stripped.startswith(("### ", '<a id="u-')):
+                category = "heading_anchor"
+            elif in_units and re.match(r"`.*` · \[", stripped):
+                category = "signature_location"
+            elif in_units and re.search(r": \[.*\](?:\(|\[)", stripped):
+                category = "relations"
+            else:
+                category = "node_text"
+            sections.append((category, line))
+        for category, count in count_section_tokens(sections).items():
+            breakdown[category] += count
+    graph = context.get("graph") or catalog.get("graph")
+    graph_bytes = (len(json.dumps(graph, ensure_ascii=False, separators=(",", ":")).encode("utf-8"))
+                   if graph else 0)
+    if catalog.get("graph"):
+        catalog_tokens = count_text_tokens(json.dumps({**catalog, "graph": None}, ensure_ascii=False, indent=2) + "\n")
+    graph_tokens = 0
+    deterministic = catalog_tokens
+    return {"tokenizer": "o200k_base", "source_tokens": source_tokens,
+            "semantic_tokens": semantic, "semantic_breakdown": breakdown,
+            "model_body_tokens": breakdown["model_body"],
+            "model_body_source_ratio": breakdown["model_body"] / source_tokens if source_tokens else None,
+            "published_markdown_source_ratio": semantic / source_tokens if source_tokens else None,
+            "deterministic_tokens": deterministic,
+            "catalog_tokens": catalog_tokens, "frozen_graph_tokens": graph_tokens,
+            "frozen_graph_bytes": graph_bytes,
+            "semantic_source_ratio": semantic / source_tokens if source_tokens else None,
+            "deterministic_source_ratio": deterministic / source_tokens if source_tokens else None,
+            "scope": "Breakdown partitions full Markdown BPE tokens by the first byte's line: model_body "
+                     "is published unit explain, summary, extras and uncertainties; signature_location includes "
+                     "shared source-file reference definitions (including ones used by relations); "
+                     "heading_anchor, relations, footer, INDEX and node_text (including labels/whitespace) "
+                     "partition the rest. model_body/source and published_markdown/source use complete frozen files. "
+                     "semantic: all published Markdown (node pages + INDEX, including navigation, "
+                     "signatures and commands); deterministic: catalog.json as serialized, including "
+                     "preserved prose copies and edge references, excluding the graph field; frozen graph "
+                     "is excluded from token ratios (frozen_graph_tokens=0), reported separately as frozen_graph_bytes "
+                     "using UTF-8 JSON with ensure_ascii=False and compact separators; complete "
+                     "graph evidence is expanded only with query --edges; source: complete frozen files counted once; "
+                     "context source chunks, host prompts, raw answers and summary.json excluded; layers are "
+                     "storage measures, not disjoint content classes"}