leyline-code 0.8.1__py3-none-any.whl

leyline/packing.py

@@ -0,0 +1,86 @@
+"""Keep a large repository's parse output small while names are resolved.
+
+A file's calls and field uses (and the variable declarations the parse read for the generic resolver) are most of
+what an adapter returns: on a repository of millions of lines they were gigabytes as objects, and every one of them
+was held from the parse to the end of resolving. They are only read a file at a time, in a few passes. So the parse
+worker packs them, pickled and compressed, beside the rest of the file's result, and on a large repository they stay
+packed: the indexer unpacks one file's calls when a pass reaches that file, and lets them go when it moves on.
+
+A small repository unpacks everything at once, as before: its output is small and the passes then pay nothing.
+LEYLINE_PACKED=1 packs every repository and LEYLINE_PACKED=0 none (the tests use both).
+"""
+
+from __future__ import annotations
+
+import os
+import pickle
+import zlib
+
+from .model import CallSite
+
+PACKED_MIN_FILES = 2000   # files to parse in a repository before its calls are kept packed
+_P = pickle.HIGHEST_PROTOCOL
+
+
+def packs(files: int) -> bool:
+    env = os.environ.get("LEYLINE_PACKED", "")
+    if env in ("0", "1"):
+        return env == "1"
+    return files >= PACKED_MIN_FILES
+
+
+def pack(res, decls) -> bytes:
+    """One file's parse output as the parse worker hands it back (and the parse cache keeps it): the result without
+    its calls and field uses, then those two packed together (a call can be the receiver of a field use, and the
+    two must stay one object), then the declarations packed."""
+    calls, uses = res.calls, res.field_uses
+    res.calls = res.field_uses = None
+    try:
+        return pickle.dumps((res, pack_heavy(calls, uses), None if decls is None else zlib.compress(pickle.dumps(decls, _P), 1)), _P)
+    finally:
+        res.calls, res.field_uses = calls, uses
+
+
+def unpack(blob: bytes) -> tuple:
+    """(result without calls and field uses, the packed calls and field uses, the packed declarations or None)."""
+    return pickle.loads(blob)
+
+
+def pack_heavy(calls, uses) -> bytes:
+    return zlib.compress(pickle.dumps((calls, uses), _P), 1)
+
+
+def unpack_heavy(packed: bytes) -> tuple:
+    return pickle.loads(zlib.decompress(packed))
+
+
+def unpack_decls(packed):
+    return None if packed is None else pickle.loads(zlib.decompress(packed))
+
+
+def call_keys(res, index: int) -> dict:
+    """id of every call site in a file's unpacked calls and field uses -> a key that names the same call each time
+    the file is unpacked: its place in a walk of the calls (receivers and argument calls first), counted down from
+    a range of negative numbers kept for the file. Results remembered about a call (Indexer._chain_memo) are kept
+    under this key, so they outlive one unpacking as they outlived nothing before; a negative key is never an id."""
+    out: dict = {}
+    base = -1 - (index << 24)
+    stack: list = []
+    for c in reversed(res.calls):
+        stack.append(c)
+    roots = [u.chain for u in res.field_uses if u.chain is not None]
+    n = 0
+    for group in (None, roots):
+        if group is not None:
+            stack.extend(reversed(group))
+        while stack:
+            c = stack.pop()
+            if id(c) in out:
+                continue
+            out[id(c)] = base - n
+            n += 1
+            more = [a for a in c.args if type(a) is CallSite]
+            if c.chain is not None:
+                more.insert(0, c.chain)
+            stack.extend(reversed(more))
+    return out

leyline/patterns.py

@@ -0,0 +1,478 @@
+"""Label design patterns by their shape in the graph.
+
+Each matcher looks for a structure, not a name: a strategy is an abstraction with several
+implementations that some other type holds and calls. A match says the code has that shape; it does
+not say the author meant the pattern. Matches are written to the inferred layer with the nodes that
+play each role, a sentence saying why, and a fixed confidence per matcher. An agent can add labels the
+matchers cannot see through `label`.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from collections import defaultdict
+from typing import Optional
+
+from . import store
+
+MATCHER = "leyline-patterns/0.1"
+SIDE_DIRS = ("sample", "samples", "example", "examples", "bench", "benchmarks", "docs", "doc", "demo", "demos", "snippets")
+
+# How far each shape is from proof of intent. Shapes that also occur by accident score lower.
+CONFIDENCE = {"strategy": 0.8, "decorator": 0.75, "composite": 0.75, "template method": 0.7, "observer": 0.85,
+              "factory": 0.6, "builder": 0.6, "singleton": 0.7, "process boundary": 0.9}
+
+ABOUT = {
+    "strategy": "An abstraction with interchangeable implementations, chosen and held by another type.",
+    "decorator": "A type that implements an abstraction and wraps another object of the same abstraction.",
+    "composite": "A type that implements an abstraction and holds a collection of that same abstraction.",
+    "template method": "A base method that fixes the steps and leaves some of them to subclasses.",
+    "observer": "A type raises an event; other code subscribes and is called back.",
+    "factory": "One function decides which of several related types to create.",
+    "builder": "A type that collects settings step by step and then produces another object.",
+    "singleton": "A type that keeps one shared instance of itself.",
+    "process boundary": "Two programs that share no code and talk over a pipe.",
+}
+
+
+class _Parsed(dict):
+    """key -> attrs, read from their JSON text the first time they are asked for. The matchers look at the attrs
+    of a few thousand nodes and edges; parsing all of them up front was most of the time on a large repository."""
+
+    def __init__(self, text: dict):
+        super().__init__()
+        self.text = text
+
+    def __missing__(self, key):
+        got = self[key] = json.loads(self.text[key] or "{}")   # KeyError when the key is not there at all
+        return got
+
+    def get(self, key, default=None):
+        return self[key] if key in self.text else default
+
+
+# A node as _Graph keeps it: a tuple, these fields.
+_ID, _KIND, _NAME, _PARENT, _LANG, _PATH = range(6)
+_ATTR_EDGES = ("uses_type", "communicates")   # the edge kinds whose attrs a matcher reads
+
+
+class _Graph:
+    def __init__(self, con):
+        # Each id is one string, used everywhere the node is named, and the few kinds, languages, paths and modules
+        # are one string each: every row read gives its own copies, and on a large repository the copies were
+        # most of what this held (GBs).
+        share = {}
+        share = share.setdefault
+        nodes = self.nodes = {}
+        text = {}
+        for r in con.execute("SELECT id, kind, name, parent_id, language, path, attrs FROM nodes WHERE layer = 'fact'"):
+            i = r[0]
+            nodes[i] = (i, share(r[1], r[1]), r[2], r[3], share(r[4], r[4]), share(r[5], r[5]))
+            text[i] = r[6]
+        for i, n in nodes.items():
+            p = nodes.get(n[_PARENT])
+            if p is not None and p[_ID] is not n[_PARENT]:
+                nodes[i] = (n[0], n[1], n[2], p[_ID], n[4], n[5])
+        self.attrs = _Parsed(text)
+        self.children = defaultdict(list)
+        for i, n in nodes.items():
+            self.children[n[_PARENT]].append(i)
+        self.out = defaultdict(lambda: defaultdict(set))
+        self.inn = defaultdict(lambda: defaultdict(set))
+        edge_attrs = {}
+        for e in con.execute("SELECT kind, src_id, dst_id, attrs FROM edges WHERE kind != 'contains'"):
+            kind, src, dst = share(e[0], e[0]), e[1], e[2]
+            n = nodes.get(src)
+            src = n[_ID] if n is not None else src
+            n = nodes.get(dst)
+            dst = n[_ID] if n is not None else dst
+            self.out[kind][src].add(dst)
+            self.inn[kind][dst].add(src)
+            if kind in _ATTR_EDGES:
+                edge_attrs[(kind, src, dst)] = e[3]
+        self.edge_attrs = _Parsed(edge_attrs)
+        self.calls = defaultdict(set)
+        for c in con.execute("SELECT DISTINCT src_id, dst_id FROM calls"):
+            src, dst = c[0], c[1]
+            n = nodes.get(src)
+            src = n[_ID] if n is not None else src
+            n = nodes.get(dst)
+            self.calls[src].add(n[_ID] if n is not None else dst)
+        self.module = {}
+        for r in con.execute("SELECT node_id, module_id FROM ancestry"):
+            n = nodes.get(r[0])
+            self.module[n[_ID] if n is not None else r[0]] = share(r[1], r[1])
+        self.test_files, self.test_modules = store.test_places(con)
+
+    def kind(self, i):
+        return self.nodes[i][_KIND] if i in self.nodes else None
+
+    def name(self, i):
+        if i not in self.nodes:
+            return i
+        n = self.nodes[i]
+        if n[_KIND] == "type" and "`" in i.rsplit(".", 1)[-1]:
+            return n[_NAME] + "<T>"  # tell Foo<T> from Foo
+        return n[_NAME]
+
+    def qual(self, i):
+        """A member with its owner: Simulation.Step."""
+        owner = self.owner(i) if self.kind(i) in ("callable", "field") else None
+        return f"{self.name(owner)}.{self.name(i)}" if owner else self.name(i)
+
+    def subtypes(self, t) -> set:
+        return self.inn["implements"][t] | self.inn["extends"][t]
+
+    def supertypes(self, t, seen=None) -> set:
+        seen = seen if seen is not None else set()
+        for b in self.out["implements"][t] | self.out["extends"][t]:
+            if b not in seen:
+                seen.add(b)
+                self.supertypes(b, seen)
+        return seen
+
+    def owner(self, i) -> Optional[str]:
+        """The type a member belongs to."""
+        cur = self.nodes[i][_PARENT] if i in self.nodes else None
+        while cur in self.nodes and self.nodes[cur][_KIND] != "type":
+            cur = self.nodes[cur][_PARENT]
+        return cur if cur in self.nodes else None
+
+    def fields(self, t):
+        return [c for c in self.children[t] if self.kind(c) == "field"]
+
+    def methods(self, t):
+        return [c for c in self.children[t] if self.kind(c) == "callable"]
+
+    def is_abstraction(self, t) -> bool:
+        a = self.attrs.get(t, {})
+        return a.get("native_kind") == "interface" or bool(a.get("is_abstract")) or (
+            self.nodes[t][_LANG] == "python" and len(self.subtypes(t)) >= 2)
+
+    def in_tests(self, i) -> bool:
+        """Test code, and code beside the product: samples, benchmarks, docs."""
+        path = (self.nodes[i][_PATH] or "") if i in self.nodes else ""
+        parts = [p.lower() for p in path.split("/")[:-1]]
+        return (self.module.get(i) in self.test_modules or path in self.test_files
+                or any(p.startswith("test") or p in SIDE_DIRS for p in parts))
+
+
+def _held(g: _Graph, abstraction: str):
+    """Fields whose type is the abstraction itself (not a collection of it), by owning type."""
+    out = defaultdict(list)
+    for f in sorted(g.inn["uses_type"][abstraction]):   # sorted: the rationale names them in this order
+        if g.kind(f) == "field" and g.edge_attrs.get(("uses_type", f, abstraction), {}).get("role") == "field_type":
+            owner = g.owner(f)
+            if owner:
+                out[owner].append(f)
+    return out
+
+
+def _collections(g: _Graph, abstraction: str):
+    """Fields that hold many of the abstraction (List<I>, I[], dict of I), by owning type."""
+    out = defaultdict(list)
+    name = g.name(abstraction)
+    for f in sorted(g.inn["uses_type"][abstraction]):
+        if g.kind(f) != "field":
+            continue
+        role = g.edge_attrs.get(("uses_type", f, abstraction), {}).get("role")
+        declared = g.attrs[f].get("declared_type") or ""
+        if role == "generic_arg" or declared.replace(" ", "").endswith(name + "[]"):
+            owner = g.owner(f)
+            if owner:
+                out[owner].append(f)
+    return out
+
+
+# What makes a function choose between the types it creates: a branch, a ternary, or a lookup by key.
+_BRANCH = re.compile(r"\b(?:if|elif|else|switch|case|match|when|default)\b|\?(?![?.\[])|=>\s*$")
+_LOOKUP = re.compile(r"[\]})]\s*\[\s*[A-Za-z_][\w.]*\s*\]|\bTryGetValue\b|\.get\(|\[\s*[A-Za-z_][\w.]*\s*\]\s*\(")
+
+
+def _builds_all(con, f: str, names: list[str], sources: dict) -> bool:
+    """Whether a function creates all of these types together (`new IRule[] { new A(), new B() }`, `[Box(), Disc()]`)
+    rather than one of them by a branch, a ternary or a lookup by key. False when its source cannot be read."""
+    r = con.execute("SELECT repo_id, path, span_start, span_end FROM nodes WHERE id = ?", (f,)).fetchone()
+    if r is None or not r[1] or not r[2]:
+        return False
+    key = (r[0], r[1])
+    if key not in sources:
+        from . import diff
+        try:
+            data = diff.source(con, *key)
+        except Exception:
+            data = None
+        sources[key] = data.decode("utf-8", errors="replace").split("\n") if data else None
+    lines = sources[key]
+    if lines is None:
+        return False
+    body = lines[r[2] - 1:(r[3] or r[2])]
+    made = re.compile(r"(?<![\w.])(?:new\s+)?(?:" + "|".join(re.escape(n) for n in names) + r")\s*[({]")
+    at = [k for k, ln in enumerate(body) if made.search(ln)]
+    if len(at) < 2 and not (at and len(made.findall(body[at[0]])) >= 2):
+        return False
+    span = body[max(at[0] - 1, 0):at[-1] + 1]   # the line before the first: `if kind == "box":` above `return Box()`
+    code = [re.sub(r"(\"[^\"]*\"|'[^']*')", '""', ln.split("//")[0].split(" #")[0]) for ln in span]
+    if any(_BRANCH.search(ln) or re.match(r'\s*(?:""|[\w.]+)\s*=>', ln) for ln in code) or sum(bool(re.search(r"\breturn\b", ln)) for ln in code) >= 2:
+        return False
+    after = " ".join(body[at[-1]:at[-1] + 4])
+    return not _LOOKUP.search(after) and not any(_LOOKUP.search(ln) for ln in body[at[-1] + 1:])
+
+
+def detect(con) -> list[dict]:
+    g = _Graph(con)
+    found: list[dict] = []
+
+    def add(pattern, anchor, roles, rationale, confidence=None, peripheral=None):
+        roles = {k: sorted(set(v)) for k, v in roles.items() if v}
+        found.append({"id": "pat:" + hashlib.sha1(f"{pattern}|{anchor}".encode()).hexdigest()[:10],
+                      "pattern": pattern, "anchor": anchor, "roles": roles, "rationale": rationale,
+                      "confidence": confidence or CONFIDENCE[pattern],
+                      "in_tests": g.in_tests(anchor.split("|")[0]) if peripheral is None else peripheral})
+
+    types = [i for i, r in g.nodes.items() if r[_KIND] == "type"]
+    wrappers: dict[str, set] = defaultdict(set)   # abstraction -> types already explained as decorator/composite
+
+    # Decorator and composite first: a wrapper also holds the abstraction, and should not count as a strategy's context.
+    for t in types:
+        supers = g.supertypes(t)
+        for a in supers:
+            if not g.is_abstraction(a):
+                continue
+            many = _collections(g, a).get(t)
+            one = _held(g, a).get(t)
+            if many:
+                wrappers[a].add(t)
+                add("composite", f"{t}|{a}", {"component": [a], "composite": [t], "children": many},
+                    f"{g.name(t)} implements {g.name(a)} and holds a collection of {g.name(a)} in "
+                    f"{', '.join(g.name(f) for f in many)}, so one object can stand for many.")
+            elif one:
+                wrappers[a].add(t)
+                add("decorator", f"{t}|{a}", {"component": [a], "decorator": [t], "wrapped": one},
+                    f"{g.name(t)} implements {g.name(a)} and keeps another {g.name(a)} in "
+                    f"{', '.join(g.name(f) for f in one)}, so it can add behavior around the one it wraps.")
+
+    for a in types:
+        if not g.is_abstraction(a):
+            continue
+        impls = sorted(s for s in g.subtypes(a) if g.kind(s) == "type" and not g.attrs[s].get("native_kind") == "interface")
+        real = [s for s in impls if not g.in_tests(s)]
+        if len(real) < 2:
+            continue
+        holders = {t: fs for t, fs in _held(g, a).items() if t not in wrappers[a] and t not in impls and t != a}
+        abstract_methods = set(g.methods(a))
+        using = {}
+        for t, fs in holders.items():
+            callers = [m for m in g.methods(t) if g.calls[m] & abstract_methods]
+            if callers:
+                using[t] = (fs, callers)
+        if any(not g.in_tests(t) for t in using):
+            using = {t: v for t, v in using.items() if not g.in_tests(t)}
+        if using:
+            contexts = sorted(using)
+            add("strategy", a, {"strategy": [a], "implementation": real, "context": contexts,
+                                "held in": [f for t in contexts for f in using[t][0]]},
+                f"{g.name(a)} has {len(real)} implementations ({', '.join(g.name(s) for s in real[:4])}"
+                f"{', ...' if len(real) > 4 else ''}). {', '.join(g.name(t) for t in contexts[:3])} "
+                f"{'holds' if len(contexts) == 1 else 'each hold'} one and "
+                f"{'calls' if len(contexts) == 1 else 'call'} it without knowing which.",
+                peripheral=all(g.in_tests(t) for t in contexts))
+
+    # Template method: a concrete method on a base type calls that type's own abstract or virtual methods,
+    # and at least one subclass overrides one of them.
+    for t in types:
+        subs = g.subtypes(t)
+        if not subs or g.attrs[t].get("native_kind") == "interface":
+            continue
+        own = g.methods(t)
+        overridden = {m for m in own if g.inn["overrides"][m]}
+        hooks_all = {m for m in overridden if g.attrs[m].get("is_virtual") or g.nodes[m][_LANG] == "python"}
+        if not hooks_all:
+            continue
+        for m in own:
+            if m in hooks_all or g.name(m) in (".ctor", "__init__"):
+                continue
+            hooks = g.calls[m] & hooks_all
+            sig = g.attrs[m].get("signature") or ""
+            if hooks and " abstract " not in f" {sig} ":
+                overriders = {g.owner(o) for h in hooks for o in g.inn["overrides"][h]} - {None}
+                add("template method", m, {"template": [m], "step": list(hooks), "base": [t], "subclass": list(overriders)},
+                    f"{g.name(t)}.{g.name(m)} is written once on the base and calls {', '.join(g.name(h) for h in sorted(hooks)[:4])}, "
+                    f"which {len(overriders)} subclass{'es supply' if len(overriders) != 1 else ' supplies'}.")
+
+    # Observer: event channels found by the indexer.
+    by_subject = defaultdict(lambda: {"raiser": set(), "observer": set(), "event": set()})
+    for src, dsts in g.out["communicates"].items():
+        for dst in dsts:
+            a = g.edge_attrs.get(("communicates", src, dst), {})
+            if a.get("channel") != "event":
+                continue
+            subject = g.owner(src) or src
+            key = (subject, a.get("address") or "")
+            by_subject[key]["raiser"].add(src)
+            by_subject[key]["observer"].add(dst)
+    for (subject, address), r in by_subject.items():
+        add("observer", f"{subject}|{address}", {"subject": [subject], "raised in": list(r["raiser"]), "observer": list(r["observer"])},
+            f"{g.name(subject)} raises {address.split('.')[-1] or 'an event'}; {len(r['observer'])} handler"
+            f"{'s elsewhere subscribe' if len(r['observer']) != 1 else ' elsewhere subscribes'} to it, so the raiser does not know who listens.")
+
+    # Process boundary.
+    for src, dsts in g.out["communicates"].items():
+        for dst in dsts:
+            a = g.edge_attrs.get(("communicates", src, dst), {})
+            if a.get("channel") == "process":
+                add("process boundary", f"{src}|{dst}", {"launcher": [src], "program": [dst]},
+                    f"{g.qual(src)} starts {g.name(g.module.get(dst) or dst)} as a separate program"
+                    f"{' and talks to it over its standard input and output' if a.get('pipes') else ''}. "
+                    f"Nothing in the type system ties the two sides together.")
+
+    # Factory: one function creates two or more types that share a supertype.
+    sources: dict = {}
+    for f, made in g.out["instantiates"].items():
+        if g.kind(f) != "callable" or g.name(f) in (".ctor", "__init__") or g.in_tests(f):
+            continue
+        made = {m for m in made if g.kind(m) == "type"}
+        if len(made) < 2:
+            continue
+        common = defaultdict(set)
+        for m in made:
+            for s in g.supertypes(m):
+                common[s].add(m)
+        best = max(common.items(), key=lambda kv: (len(kv[1]), kv[0]), default=None)
+        if best and len(best[1]) >= 2:
+            base, products = best
+            if not g.methods(base):
+                continue  # a marker type with no behavior: creating several of them is not a choice between implementations
+            a = g.attrs[f]
+            rets = [n.split("`")[0] for n in (a.get("returns_names") or ([a["returns"]] if a.get("returns") else []))]
+            returns_base = bool(rets) and rets[0] == g.nodes[base][_NAME]
+            kinds = f"{len(products)} kinds of {g.name(base)} ({', '.join(g.name(p) for p in sorted(products)[:4])})"
+            if _builds_all(con, f, [g.name(p).split("<")[0] for p in products], sources):
+                add("factory", f, {"factory": [f], "product": list(products), "product type": [base]},
+                    f"{g.qual(f)} creates every one of {kinds} together, as a list or a registry: it does not choose "
+                    f"between them.", 0.4)
+            elif returns_base:
+                add("factory", f, {"factory": [f], "product": list(products), "product type": [base]},
+                    f"{g.qual(f)} creates {kinds} and returns it as {g.name(base)}, so callers do not name the concrete type.", 0.8)
+            else:
+                add("factory", f, {"factory": [f], "product": list(products), "product type": [base]},
+                    f"{g.qual(f)} decides which of {kinds} to create. It does not return one, so this is where the "
+                    f"choice is wired in, not a factory others call.", 0.5)
+
+    # Builder: several methods return the type itself (chaining) and one produces a different type.
+    for t in types:
+        if g.in_tests(t):
+            continue
+        tname = g.name(t)
+        fluent, produce = [], []
+        for m in g.methods(t):
+            a = g.attrs[m]
+            rets = [n.split("`")[0] for n in (a.get("returns_names") or ([a["returns"]] if a.get("returns") else []))]
+            if g.name(m) in (".ctor", "__init__") or a.get("visibility") == "private":
+                continue
+            if rets and rets[0] == tname:
+                fluent.append(m)
+            elif g.name(m).lower().startswith(("build", "create")) and rets:
+                produce.append((m, rets[0]))
+        if len(fluent) >= 2 and produce:
+            add("builder", t, {"builder": [t], "step": fluent, "build": [m for m, _ in produce]},
+                f"{tname} has {len(fluent)} methods that return the {tname} itself, so calls chain, and "
+                f"{g.name(produce[0][0])} turns what was collected into a {produce[0][1]}.")
+
+    # Singleton: a static field of the type's own type, and no public constructor.
+    for t in types:
+        own_static = [f for f in g.fields(t) if g.attrs[f].get("is_static") and (g.attrs[f].get("type_name") or "").split("`")[0] == g.name(t)]
+        ctors = [m for m in g.methods(t) if g.name(m) == ".ctor"]
+        if own_static and ctors and all(g.attrs[c].get("visibility") in ("private", "protected") for c in ctors):
+            add("singleton", t, {"singleton": [t], "instance": own_static},
+                f"{g.name(t)} keeps an instance of itself in the static {g.name(own_static[0])} and its constructors "
+                f"are not public, so every caller shares that one object.")
+    return found
+
+
+def run(con, repo_id: Optional[str] = None) -> dict:
+    """Replace the matcher's labels with a fresh pass. Labels written by others are kept and checked for staleness."""
+    found = detect(con)
+    with con:
+        old = [r[0] for r in con.execute("SELECT id FROM pattern_instances WHERE matcher = ?", (MATCHER,))]
+        con.executemany("DELETE FROM pattern_roles WHERE instance_id = ?", [(i,) for i in old])
+        con.execute("DELETE FROM pattern_instances WHERE matcher = ?", (MATCHER,))
+        for p in found:
+            evidence = [i for ids in p["roles"].values() for i in ids]
+            con.execute("INSERT OR REPLACE INTO pattern_instances (id, pattern, matcher, rationale, confidence, evidence_hash, stale, attrs)"
+                        " VALUES (?,?,?,?,?,?,0,?)",
+                        (p["id"], p["pattern"], MATCHER, p["rationale"], p["confidence"], store.evidence_hash(con, evidence),
+                         json.dumps({"peripheral": bool(p["in_tests"])})))
+            con.executemany("INSERT INTO pattern_roles VALUES (?,?,?)",
+                            [(p["id"], role, i) for role, ids in p["roles"].items() for i in ids])
+        # Labels from agents: stale when the code behind them changed, dropped roles when a node is gone.
+        for row in con.execute("SELECT id, evidence_hash FROM pattern_instances WHERE matcher != ?", (MATCHER,)).fetchall():
+            ids = [r[0] for r in con.execute("SELECT node_id FROM pattern_roles WHERE instance_id = ?", (row["id"],))]
+            con.execute("UPDATE pattern_instances SET stale = ? WHERE id = ?",
+                        (int(store.evidence_hash(con, ids) != row["evidence_hash"]), row["id"]))
+        counts = defaultdict(int)
+        for p in found:
+            counts[p["pattern"]] += 1
+        if repo_id:
+            con.execute("INSERT OR REPLACE INTO extractor_coverage VALUES (?,?,?,?,?,?)",
+                        (repo_id, "patterns:structural", "0.1", "ok", None, json.dumps(dict(counts))))
+    return dict(counts)
+
+
+def label(con, pattern: str, roles: dict, rationale: str, confidence: float = 0.6, source: str = "mcp") -> dict:
+    """Record a pattern the matchers did not find. `roles` maps a role name to node ids."""
+    if not rationale.strip():
+        return {"error": "a pattern label needs a rationale: say what in the code makes it this pattern"}
+    known = {r[0] for r in con.execute("SELECT id FROM nodes")}
+    clean = {role: [i for i in ids if i in known] for role, ids in roles.items()}
+    missing = [i for ids in roles.values() for i in ids if i not in known]
+    clean = {k: v for k, v in clean.items() if v}
+    if not clean:
+        return {"error": "none of the node ids exist in the store", "missing": missing[:5]}
+    evidence = [i for ids in clean.values() for i in ids]
+    pid = "pat:" + hashlib.sha1(f"{pattern}|{sorted(evidence)}|{source}".encode()).hexdigest()[:10]
+    with con:
+        con.execute("DELETE FROM pattern_roles WHERE instance_id = ?", (pid,))
+        con.execute("INSERT OR REPLACE INTO pattern_instances (id, pattern, matcher, rationale, confidence, evidence_hash, stale)"
+                    " VALUES (?,?,?,?,?,?,0)",
+                    (pid, pattern.strip().lower(), source, rationale, max(0.0, min(1.0, confidence)), store.evidence_hash(con, evidence)))
+        con.executemany("INSERT INTO pattern_roles VALUES (?,?,?)", [(pid, role, i) for role, ids in clean.items() for i in ids])
+    return {"id": pid, "pattern": pattern.strip().lower(), "missing": missing[:10]}
+
+
+def listing(con, pattern: Optional[str] = None, node_id: Optional[str] = None, include_tests: bool = False,
+            limit: int = 100) -> dict:
+    """Pattern labels, optionally of one kind or touching one node (or anything inside it)."""
+    share = {}   # one string for each kind, path and module, not one per row (hundreds of MB on a large repository)
+    share = share.setdefault
+    names = {r[0]: (r[1], share(r[2], r[2]), share(r[3], r[3])) for r in con.execute("SELECT id, name, kind, path FROM nodes")}
+    test_mods = {r[0] for r in con.execute(
+        "SELECT DISTINCT a.module_id FROM nodes n JOIN ancestry a ON a.node_id = n.id WHERE n.kind = 'test'")}
+    module = {r[0]: share(r[1], r[1]) for r in con.execute("SELECT node_id, module_id FROM ancestry")}
+    out, counts = [], defaultdict(int)
+    for p in con.execute("SELECT * FROM pattern_instances ORDER BY confidence DESC, pattern, id"):
+        roles = defaultdict(list)
+        for r in con.execute("SELECT role, node_id FROM pattern_roles WHERE instance_id = ? ORDER BY role, node_id", (p["id"],)):
+            roles[r["role"]].append(r["node_id"])
+        ids = [i for v in roles.values() for i in v]
+        if not ids:
+            continue
+        stored = json.loads(p["attrs"] or "{}").get("peripheral")
+        in_tests = stored if stored is not None else sum(1 for i in ids if module.get(i) in test_mods) * 2 > len(ids)
+        if in_tests and not include_tests and not node_id:
+            continue
+        if pattern and p["pattern"] != pattern.strip().lower():
+            continue
+        if node_id and not any(i == node_id or i.startswith(node_id + ".") or i.startswith(node_id + "/") for i in ids):
+            continue
+        counts[p["pattern"]] += 1
+        if len(out) < limit:
+            out.append({"id": p["id"], "pattern": p["pattern"], "about": ABOUT.get(p["pattern"], ""),
+                        "rationale": p["rationale"], "confidence": p["confidence"],
+                        "source": "structural matcher" if p["matcher"] == MATCHER else p["matcher"],
+                        "stale": bool(p["stale"]), "in_tests": in_tests,
+                        "roles": {role: [{"id": i, "name": names.get(i, (i,))[0]} for i in v] for role, v in roles.items()}})
+    return {"total": sum(counts.values()), "by_pattern": dict(counts), "patterns": out,
+            "note": "A label says the code has this shape. It does not say the author intended the pattern."}