system-diff 0.1.0__py3-none-any.whl

system_diff/diff.py

@@ -0,0 +1,171 @@
+"""The typed difference between two blueprints, in one sentence (headline) and exactly (plan, `terraform plan`-style)."""
+from __future__ import annotations
+
+import re
+from collections import Counter, defaultdict
+from pathlib import Path
+from typing import Any, Dict, Iterable, List, Optional
+
+from .extract import Code, matches
+from .sources import git
+
+IMPACT_WORDS = {
+    "new-component": "adds a part", "removed-component": "removes a part",
+    "new-interface": "adds an endpoint", "breaking-interface": "removes an endpoint callers may rely on",
+    "new-external": "talks to an outside system it did not before", "removed-external": "stops using an outside system",
+    "data-schema-change": "changes stored data", "deployment-change": "changes what is deployed",
+    "new-flow": "connects parts that were not connected", "removed-flow": "disconnects parts",
+    "new-config": "reads new settings", "test-gap": "changes code without changing tests",
+}
+
+
+def files_word(n: int) -> str:
+    return f"{n} file" + ("" if n == 1 else "s")
+
+
+def is_test_path(path: str, code: Optional[Code] = None) -> bool:
+    """A test file: one the config's `tests` globs name, under tests/, or named like `x.test.ts`."""
+    return path.startswith("tests/") or ".test." in path or bool(code and code.is_test(path))
+
+
+def numstat(root: Path, base: str, head: Optional[str], cfg: Dict[str, Any]) -> Dict[str, tuple]:
+    """{path: (lines added, lines removed, owning part or None)} for every file the change touched; `head` None is
+    the working tree."""
+    code = Code(cfg)
+    out = {}
+    for line in git("diff", "--numstat", "-M", base, *([head] if head else []), cwd=root).splitlines():
+        parts = line.split("\t")
+        if len(parts) != 3:
+            continue
+        plus, minus, path = parts
+        path = re.sub(r"\{([^}]*) => ([^}]*)\}", r"\2", path).split(" => ")[-1]
+        comp = code.part_of(path) if code.is_code(path) and not is_test_path(path, code) else None
+        out[path] = (int(plus) if plus.isdigit() else 0, int(minus) if minus.isdigit() else 0, comp)
+    return out
+
+
+def diff(a: dict, b: dict, numstat: Optional[Dict[str, tuple]] = None, deploy_globs: Iterable[str] = (),
+         code: Optional[Code] = None) -> dict:
+    changes: List[dict] = []
+
+    def add(op, entity, key, label="", detail=(), impact=()):
+        changes.append({"op": op, "entity": entity, "id": key, "label": label or key,
+                        "detail": list(detail), "impact": list(impact)})
+
+    ca, cb = a["components"], b["components"]
+    lines = defaultdict(lambda: [0, 0, 0])
+    code_changed = tests_changed = False
+    for path, (plus, minus, comp) in (numstat or {}).items():
+        if comp:
+            lines[comp][0] += plus
+            lines[comp][1] += minus
+            lines[comp][2] += 1
+            code_changed = True
+        elif is_test_path(path, code):
+            tests_changed = True
+    for cid in sorted(set(ca) | set(cb)):
+        if cid not in ca:
+            add("add", "component", cid, cb[cid]["label"], [files_word(cb[cid]["files"])], ["new-component"])
+        elif cid not in cb:
+            add("remove", "component", cid, ca[cid]["label"], [], ["removed-component"])
+        elif cid in lines:
+            p, m, n = lines[cid]
+            add("change", "component", cid, cb[cid]["label"], [f"+{p} −{m} lines in {files_word(n)}"])
+    for kind, impact_add, impact_remove in (("flows", "new-flow", "removed-flow"), ("api", "new-flow", "removed-flow"),
+                                            ("internal", "new-flow", "removed-flow")):
+        for key in sorted(set(a[kind]) ^ set(b[kind])):
+            src, dst = key.split(">")
+            label = f"{name_of(b, a, src)} → {name_of(b, a, dst)}"
+            add("add" if key in b[kind] else "remove", "flow", key, label, [],
+                [impact_add if key in b[kind] else impact_remove])
+    for route in sorted(set(a["routes"]) ^ set(b["routes"])):
+        if route in b["routes"]:
+            add("add", "route", route, route, [name_of(b, a, b["routes"][route])], ["new-interface"])
+        else:
+            add("remove", "route", route, route, [name_of(b, a, a["routes"][route])], ["breaking-interface"])
+    pairs = lambda bp: {(e, u) for e, v in bp["externals"].items() for u in v["used_by"]}  # noqa: E731
+    for ext, user in sorted(pairs(a) ^ pairs(b)):
+        new = (ext, user) in pairs(b)
+        meta = (b if new else a)["externals"][ext]
+        add("add" if new else "remove", "external", f"{user}>{ext}", f"{name_of(b, a, user)} → {meta['label']}", [],
+            ["new-external" if new else "removed-external"])
+    for name in sorted(set(a["containers"]) | set(b["containers"])):
+        x, y = a["containers"].get(name), b["containers"].get(name)
+        if x != y:
+            op = "add" if x is None else "remove" if y is None else "change"
+            detail = [] if op != "change" else [f"{k}: {x.get(k)!r} → {y.get(k)!r}" for k in y if x.get(k) != y.get(k)]
+            add(op, "container", name, name, detail, ["deployment-change"])
+    for table in sorted(set(a["tables"]) | set(b["tables"])):
+        if table not in a["tables"]:
+            add("add", "table", table, table, [], ["data-schema-change"])
+        elif table not in b["tables"]:
+            add("remove", "table", table, table, [], ["data-schema-change"])
+        else:
+            cols = sorted(set(b["tables"][table]) - set(a["tables"][table]))
+            if cols:
+                add("change", "table", table, table, [f"+ column {c}" for c in cols], ["data-schema-change"])
+    for m in sorted(set(b["migrations"]) - set(a["migrations"])):
+        add("add", "migration", m, m, [], ["data-schema-change"])
+    for path, (plus, minus, _comp) in sorted((numstat or {}).items()):
+        if matches(path, deploy_globs):
+            add("change", "deploy", path, path, [f"+{plus} −{minus} lines"], ["deployment-change"])
+    for comp in sorted(set(b["env"]) & set(a["env"])):
+        new_vars = sorted(set(b["env"][comp]) - set(a["env"][comp]))
+        if new_vars:
+            add("change", "settings", comp, name_of(b, a, comp), [", ".join(new_vars)], ["new-config"])
+
+    impacts = Counter(i for c in changes for i in c["impact"])
+    if code_changed and not tests_changed:
+        impacts["test-gap"] += 1
+    return {"from": a["ref"], "to": b["ref"], "changes": changes, "impacts": dict(impacts),
+            "headline": headline(changes), "plan": plan(changes, impacts),
+            "dormant": sorted(b["dormant"]), "undeclared": sorted(c for c, v in cb.items() if not v["declared"])}
+
+
+def name_of(b: dict, a: dict, ref: str) -> str:
+    for bp in (b, a):
+        if ref in bp["components"]:
+            return bp["components"][ref]["label"]
+        if ref in bp.get("container_labels", {}):
+            return bp["container_labels"][ref]
+    return ref
+
+
+NOUNS = {"deploy": ("deployment file", "deployment files"), "component": ("part", "parts"), "route": ("endpoint", "endpoints"),
+         "external": ("outside connection", "outside connections"), "container": ("container", "containers"),
+         "table": ("table", "tables"), "migration": ("migration", "migrations"), "flow": ("connection", "connections"),
+         "settings": ("set of settings", "sets of settings")}
+
+
+def headline(changes: List[dict]) -> str:
+    """One sentence: what the change does to the system's structure."""
+    parts = []
+    for op, verb in (("add", "adds"), ("change", "changes"), ("remove", "removes")):
+        phrases = []
+        for entity in ("component", "route", "table", "migration", "external", "container", "deploy", "flow", "settings"):
+            items = [c for c in changes if c["op"] == op and c["entity"] == entity]
+            if items:
+                one, many = NOUNS[entity]
+                names = [c["label"] for c in items]
+                shown = "" if entity == "flow" else " (" + ", ".join(names[:3]) + (f" +{len(names) - 3} more" if len(names) > 3 else "") + ")"
+                phrases.append(f"{len(items)} {one if len(items) == 1 else many}{shown}")
+        if phrases:
+            parts.append(verb + " " + (", ".join(phrases[:-1]) + " and " + phrases[-1] if len(phrases) > 1 else phrases[0]))
+    if not parts:
+        return "No change to the system's structure."
+    s = "; ".join(parts)
+    return s[0].upper() + s[1:] + "."
+
+
+def plan(changes: List[dict], impacts: Counter) -> str:
+    """The same diff as `terraform plan`-style text: exact, and readable anywhere."""
+    sym = {"add": "+", "change": "~", "remove": "-"}
+    out = []
+    for c in changes:
+        tag = f"   [{', '.join(c['impact'])}]" if c["impact"] else ""
+        label = f" — {c['label']}" if c["label"] != c["id"] else ""
+        out.append(f"{sym[c['op']]} {c['entity']} {c['id']}{label}{tag}")
+        out += [f"    {x}" for x in c["detail"]]
+    if impacts.get("test-gap"):
+        out.append("! test-gap: code changed and no test file did")
+    return "\n".join(out) or "no change to the system's structure"

system_diff/extract.py

@@ -0,0 +1,279 @@
+"""The system's structure as one JSON-able dict (a "blueprint"), read from the code of one commit or directory.
+
+Observed, not declared: the imports between parts, FastAPI-style routes, the API paths the front end calls, the settings
+each part reads (which name the outside systems it talks to), the deployed containers (a compose file) and the tables the
+migrations create. Declared in the config: what the parts are called, which column each belongs in, and which settings
+mean which outside system. An outside system is drawn only when some deploy configuration sets one of its settings;
+read by the code but configured nowhere, it is DORMANT: listed, not drawn.
+"""
+from __future__ import annotations
+
+import os
+import re
+from collections import Counter, defaultdict
+from typing import Any, Dict, Iterable, List, Optional
+
+from .config import segment_rx
+
+
+def glob_rx(pattern: str) -> re.Pattern:
+    """* stays within one directory, ** crosses them, ? is one character."""
+    out, i = "", 0
+    while i < len(pattern):
+        if pattern.startswith("**/", i):
+            out, i = out + "(?:.*/)?", i + 3
+        elif pattern.startswith("**", i):
+            out, i = out + ".*", i + 2
+        elif pattern[i] == "*":
+            out, i = out + "[^/]*", i + 1
+        elif pattern[i] == "?":
+            out, i = out + "[^/]", i + 1
+        else:
+            out, i = out + re.escape(pattern[i]), i + 1
+    return re.compile(out + r"\Z")
+
+
+def matches(path: str, patterns: Iterable[str]) -> bool:
+    return any(glob_rx(p).match(path) for p in patterns)
+
+
+PY_FROM = re.compile(r"^[ \t]*from[ \t]+(\.*)([\w.]*)[ \t]+import[ \t]+\(?([^)#\n]+)", re.M)
+PY_IMPORT = re.compile(r"^[ \t]*import[ \t]+([\w.]+(?:[ \t]*,[ \t]*[\w.]+)*)", re.M)
+TS_IMPORT = re.compile(r"""(?:from\s+|import\s*\(\s*|require\(\s*)["']([^"']+)["']""")
+ROUTE = re.compile(r"""^[ \t]*@(?:router|app)\.(get|post|put|patch|delete)\(\s*[rfb]?["']([^"']*)["']""", re.M)
+PREFIX = re.compile(r"""APIRouter\([^)]*prefix\s*=\s*["']([^"']+)["']""")
+ENV_PY = re.compile(r"""(?:getenv|environ\.get|environ\.setdefault)\(\s*["']([A-Z][A-Z0-9_]+)["']|environ\[\s*["']([A-Z][A-Z0-9_]+)["']\s*\]""")
+ENV_TS = re.compile(r"(?:import\.meta\.env|process\.env)\.([A-Z][A-Z0-9_]+)")
+API_STR = re.compile(r"""["'`](/api/[^"'`\s]*)""")
+CREATE_TABLE = re.compile(r"CREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?([\w.\"]+)", re.I)
+ADD_COLUMN = re.compile(r"ALTER\s+TABLE\s+(?:IF\s+EXISTS\s+)?(?:ONLY\s+)?([\w.\"]+)\s+ADD\s+COLUMN\s+(?:IF\s+NOT\s+EXISTS\s+)?\"?(\w+)", re.I)
+SQL_COMMENT = re.compile(r"--[^\n]*|/\*.*?\*/", re.S)
+CONFIG_KEY = re.compile(r"^\s*(?:-\s*)?\"?(?:export\s+)?([A-Z][A-Z0-9_]+)\s*[=:]", re.M)
+TS_EXT = (".ts", ".tsx", ".js", ".jsx")
+
+
+def py_targets(path: str, text: str, files: set, python_paths: Iterable[str] = ("",)) -> List[str]:
+    """Files a Python file imports, among `files`; imports of anything else (the standard library,
+    third-party packages) resolve to nothing and are ignored."""
+    package = path.split("/")[:-1]
+
+    def module_file(dotted: str) -> Optional[str]:
+        stem = dotted.replace(".", "/")
+        return next((c for base in python_paths for c in (base + stem + ".py", base + stem + "/__init__.py")
+                     if c in files), None)
+
+    def relative_file(dotted: str) -> Optional[str]:
+        stem = dotted.replace(".", "/")
+        return next((c for c in (stem + ".py", stem + "/__init__.py") if c in files), None)
+
+    def nearest(dotted: str, find=module_file) -> Optional[str]:  # `import a.b.c` where only a/b.py exists
+        parts = dotted.split(".")
+        while parts:
+            hit = find(".".join(parts))
+            if hit:
+                return hit
+            parts.pop()
+        return None
+
+    found: List[Optional[str]] = []
+    for dots, mod, names in PY_FROM.findall(text):
+        base = package[: len(package) - (len(dots) - 1)] if dots else []
+        stem = ".".join(base + ([mod] if mod else []))
+        find = relative_file if dots else module_file
+        modules = [find(f"{stem}.{n}") for n in
+                   (x.strip().split(" as ")[0].strip() for x in names.split(",")) if n and n != "*"]
+        modules = [m for m in modules if m]
+        found += modules or [nearest(stem, find)]  # `from x import name`: x itself, unless the names are modules
+    for mods in PY_IMPORT.findall(text):
+        found += [nearest(m.strip()) for m in mods.split(",")]
+    return [f for f in found if f]
+
+
+def ts_targets(path: str, text: str, files: set, aliases: Optional[Dict[str, str]] = None) -> List[str]:
+    base = path.rsplit("/", 1)[0]
+    out = []
+    for spec in TS_IMPORT.findall(text):
+        alias = next((a for a in (aliases or {}) if spec.startswith(a)), None)
+        if alias:
+            stem = aliases[alias] + spec[len(alias):]
+        elif spec.startswith("."):
+            stem = os.path.normpath(os.path.join(base, spec)).replace(os.sep, "/")
+        else:
+            continue
+        for cand in (stem, *(stem + e for e in TS_EXT), *(stem + "/index" + e for e in TS_EXT)):
+            if cand in files:
+                out.append(cand)
+                break
+    return out
+
+
+def compose_services(text: str) -> Dict[str, dict]:
+    """The services of a compose file: just enough YAML to see what changed in the deployment."""
+    services: Dict[str, dict] = {}
+    current, in_services, key = None, False, None
+    for line in text.splitlines():
+        if not line.strip() or line.lstrip().startswith("#"):
+            continue
+        if re.match(r"^\S", line):
+            in_services, current = line.startswith("services:"), None
+            continue
+        if not in_services:
+            continue
+        m = re.match(r"^  ([A-Za-z0-9_.-]+):\s*$", line)
+        if m:
+            current = services.setdefault(m.group(1), {"image": "", "command": "", "ports": [], "replicas": ""})
+            continue
+        if current is None:
+            continue
+        m = re.match(r"^\s{4}(image|command|ports):\s*(.*)$", line)
+        if m:
+            key = m.group(1)
+            if key != "ports" and m.group(2):
+                current[key] = m.group(2).strip().strip("\"'")
+            continue
+        m = re.match(r"^\s+replicas:\s*(\S+)", line)
+        if m:
+            current["replicas"] = m.group(1)
+            continue
+        m = re.match(r"^\s{4,}-\s*\"?([^\"#]+?)\"?\s*$", line)
+        if m and key == "ports":
+            current["ports"].append(m.group(1))
+        elif re.match(r"^\s{4}\w", line):
+            key = None
+    return services
+
+
+class Code:
+    """Which files are code, which are tests, and which part owns each: one answer for the map and the diff."""
+
+    def __init__(self, cfg: Dict[str, Any]):
+        c = cfg["code"]
+        self.skip = segment_rx(c["skip"])
+        self.roots = tuple(c["roots"])
+        self.ext = tuple(c["extensions"])
+        self.group = c["group"]
+        self.strip = tuple(c["strip"] if c.get("strip") is not None else c["extensions"])
+        self.tests = [glob_rx(p) for p in cfg.get("tests", [])]
+        self.declared = [(d, [glob_rx(p) for p in d.get("paths", [])]) for d in cfg.get("components", [])]
+
+    def is_code(self, path: str) -> bool:
+        return path.endswith(self.ext) and path.startswith(self.roots) and not self.skip.search(path)
+
+    def is_test(self, path: str) -> bool:
+        return any(r.match(path) for r in self.tests)
+
+    def part_of(self, path: str) -> str:
+        """The declared part claiming `path`; else one named by its directory."""
+        hit = next((d["id"] for d, rs in self.declared if any(r.match(path) for r in rs)), None)
+        if hit is not None:
+            return hit
+        parts = path.split("/")
+        name = "/".join(parts[: int(self.group.get(parts[0], self.group.get("default", 2)))])
+        return next((name[: -len(s)] for s in self.strip if name.endswith(s)), name)
+
+
+def extract(source, cfg: Dict[str, Any]) -> dict:
+    """The blueprint of one source."""
+    code_of = Code(cfg)
+    comps = cfg.get("components", [])
+    files = [f for f in source.files if not code_of.skip.search(f)]
+    fileset = set(files)
+    code = [f for f in files if code_of.is_code(f) and not code_of.is_test(f)]
+    aliases, python_paths = cfg["code"]["aliases"], cfg["code"]["python_paths"]
+
+    owner: Dict[str, str] = {}
+    components: Dict[str, dict] = {}
+    for f in code:
+        cid = code_of.part_of(f)
+        owner[f] = cid
+        meta = next((c for c in comps if c["id"] == cid), None)
+        entry = components.setdefault(cid, {
+            "label": (meta or {}).get("label", cid), "layer": meta.get("layer", "") if meta else "undeclared",
+            "kind": (meta or {}).get("kind", ""), "order": comps.index(meta) if meta else len(comps),
+            "container": (meta or {}).get("container", ""), "declared": meta is not None, "files": 0, "lines": 0,
+            "hub": bool((meta or {}).get("hub")), "quiet": bool((meta or {}).get("quiet"))})
+        entry["files"] += 1
+
+    texts = source.read_many(code)
+    flows, api, env_by_comp, routes = Counter(), Counter(), defaultdict(Counter), {}
+    for f in code:
+        text, cid = texts[f], owner[f]
+        components[cid]["lines"] += text.count("\n")
+        python = f.endswith(".py")
+        targets = py_targets(f, text, fileset, python_paths) if python else ts_targets(f, text, fileset, aliases)
+        for t in targets:
+            if t in owner and owner[t] != cid:
+                flows[f"{cid}>{owner[t]}"] += 1
+        if python:
+            prefix = (PREFIX.search(text) or [None, ""])[1]
+            for method, path in ROUTE.findall(text):
+                routes[f"{method.upper()} {prefix}{path}"] = cid
+            env_by_comp[cid].update(a or b for a, b in ENV_PY.findall(text))
+        else:
+            env_by_comp[cid].update(ENV_TS.findall(text))
+            for s in API_STR.findall(text):
+                target = next((a["component"] for a in cfg.get("api", []) if s.startswith(a["prefix"])), None)
+                if target and target != cid:
+                    api[f"{cid}>{target}"] += 1
+
+    config_rx = [glob_rx(p) for p in cfg.get("config_files", [])]
+    configured = set()
+    for f in files:
+        if any(r.match(f) for r in config_rx):
+            configured |= set(CONFIG_KEY.findall(source.read(f)))
+    externals, dormant = {}, {}
+    for ext in cfg.get("externals", []):
+        reads = {c: sum(n for v, n in vs.items() if matches(v, ext["env"])) for c, vs in env_by_comp.items()}
+        users = sorted((c for c, n in reads.items() if n), key=lambda c: (-reads[c], c))  # heaviest reader first
+        if not users:
+            continue
+        live = any(matches(v, ext["env"]) for v in configured)
+        (externals if live else dormant)[ext["id"]] = {"label": ext["label"], "shape": ext.get("shape", ""), "used_by": users}
+    internal = Counter()
+    for var, container in cfg.get("internal_env", {}).items():
+        for c, vs in env_by_comp.items():
+            if var in vs:
+                internal[f"{c}>{container}"] += 1
+
+    compose = cfg.get("compose", "")
+    containers = compose_services(source.read(compose)) if compose in fileset else {}
+
+    tables, migrations = defaultdict(set), []
+    mig = cfg.get("migrations", {})
+    mig_files = sorted(f for f in files if mig and glob_rx(mig.get("glob", "")).match(f))
+    mig_texts = source.read_many(mig_files)
+    for f in mig_files:
+        name = f.rsplit("/", 1)[1]
+        migrations.append(name)
+        num = int(re.match(r"\d+", name).group(0)) if re.match(r"\d+", name) else 0
+        schema = next((s for lo, hi, s in mig.get("schemas", []) if lo <= num <= hi), "public")
+        text = SQL_COMMENT.sub("", mig_texts[f])  # "CREATE TABLE IF NOT EXISTS only." in a comment is not a table
+        for table in CREATE_TABLE.findall(text):
+            tables[qualify(table, schema)] |= set()
+        for table, col in ADD_COLUMN.findall(text):
+            tables[qualify(table, schema)].add(col)
+
+    test_files = Counter(("/".join(f.split("/")[:2]) if f.startswith("tests/") else f.split("/")[0])
+                         for f in files if code_of.is_test(f) and f.endswith(code_of.ext))
+    return {
+        "ref": source.label,
+        "components": components,
+        "flows": dict(sorted(flows.items())),
+        "api": dict(sorted(api.items())),
+        "routes": dict(sorted(routes.items())),
+        "externals": externals,
+        "dormant": dormant,
+        "internal": dict(sorted(internal.items())),
+        "env": {c: sorted(v) for c, v in sorted(env_by_comp.items()) if v},
+        "containers": containers,
+        "container_labels": cfg.get("containers", {}),
+        "tables": {t: sorted(c) for t, c in sorted(tables.items())},
+        "store": mig.get("store", "postgres"),   # the outside system the migrations' tables live in
+        "migrations": migrations,
+        "tests": dict(sorted(test_files.items())),
+    }
+
+
+def qualify(name: str, schema: str) -> str:
+    name = name.replace('"', "")
+    return name if "." in name else f"{schema}.{name}"

system_diff/graph.py

@@ -0,0 +1,209 @@
+"""From a blueprint (or a diff) to the graph a picture draws: a column per role, each part where the config declares it,
+so a part that did not change keeps its place from one version to the next. Drawn by `pictures.map_svg`."""
+from __future__ import annotations
+
+from collections import Counter, defaultdict
+from typing import Any, Dict, List, Optional
+
+from . import pictures
+from .diff import files_word
+
+PART_KINDS = ("ui", "interface", "logic", "data")
+LAYER_RANK = {"new": 0, "shared": 1, "legacy": 2}
+def drawn_flows(bp: dict, cfg: Dict[str, Any]) -> List[tuple]:
+    """The connections worth drawing: not into a hub, not from or to a quiet part, not between two parts of the
+    front end (that is page composition, not architecture), strong enough (`min_flow` import sites), and each part's
+    strongest few (`top_flows`). Weaker links stay in system.json and out of the picture."""
+    comps = bp["components"]
+    ui = set(cfg["picture"]["ui_containers"])
+    min_flow, top_flows = cfg["picture"]["min_flow"], cfg["picture"]["top_flows"]
+    outgoing = defaultdict(list)
+    for key, n in bp["flows"].items():
+        src, dst = key.split(">")
+        a, b = comps.get(src, {}), comps.get(dst, {})
+        if n < min_flow or b.get("hub") or a.get("quiet") or b.get("quiet"):
+            continue
+        if a.get("container") in ui and b.get("container") in ui:
+            continue
+        outgoing[src].append((-n, dst))
+    return [(src, dst) for src in sorted(outgoing) for _, dst in sorted(outgoing[src])[:top_flows]]
+
+
+def drawn_api(bp: dict) -> List[tuple]:
+    """Each front-end part's main API target, plus any other it calls often."""
+    by_src = defaultdict(list)
+    for key, n in bp["api"].items():
+        src, dst = key.split(">")
+        by_src[src].append((-n, dst))
+    out = []
+    for src in sorted(by_src):
+        ranked = sorted(by_src[src])
+        out += [(src, dst) for i, (neg, dst) in enumerate(ranked) if i == 0 or -neg >= 10]
+    return out
+
+
+def part_kind(cid: str, c: dict, bp: dict, cfg: Dict[str, Any]) -> str:
+    """The column: declared in the config, else the front end is what people use and a part with routes an entry point."""
+    if c.get("kind") in PART_KINDS:
+        return c["kind"]
+    if c.get("container") and c.get("container") in cfg["picture"]["ui_containers"]:
+        return "ui"
+    return "interface" if cid in set(bp["routes"].values()) else "logic"
+
+
+def route_lines(bp: dict, cid: str) -> List[dict]:
+    """A part's endpoints grouped by their first three path segments, the largest group first."""
+    groups: Counter = Counter()
+    for route, owner in bp["routes"].items():
+        if owner == cid:
+            segments = [s for s in route.split(" ", 1)[1].split("/") if s][:3]
+            groups["/" + "/".join(segments)] += 1
+    return [{"text": f"{g} ({n})", "code": True} for g, n in sorted(groups.items(), key=lambda kv: (-kv[1], kv[0]))]
+
+
+def part_order(c: dict, cid: str) -> tuple:
+    return LAYER_RANK.get(c.get("layer"), 3), c.get("order", 10 ** 6), cid
+
+
+def part_node(cid: str, c: dict, bp: dict, cfg: Dict[str, Any]) -> dict:
+    sub = cfg["picture"]["hub_note"] if c.get("hub") else files_word(c["files"])
+    container = c.get("container", "")
+    if container and container not in cfg["picture"]["main_containers"]:
+        sub = f"{bp.get('container_labels', {}).get(container, container)} · {sub}"
+    kind = part_kind(cid, c, bp, cfg)
+    return {"id": cid, "col": kind, "title": c["label"], "sub": sub, "type": "component", "style": c.get("layer", ""),
+            "lines": route_lines(bp, cid) if kind == "interface" else []}
+
+
+def outside_node(ext_id: str, e: dict, bp: dict) -> dict:
+    n = len(e["used_by"])
+    db = e.get("shape") == "db"
+    node = {"id": "x:" + ext_id, "col": "store" if db else "external", "type": "store" if db else "host",
+            "title": e["label"], "sub": f"used by {n} part{'' if n == 1 else 's'}", "lines": []}
+    if ext_id == bp.get("store", "postgres") and bp["tables"]:
+        schemas = Counter(t.split(".", 1)[0] for t in bp["tables"])
+        node["sub"] = f"{len(bp['tables'])} tables · {node['sub']}"
+        node["lines"] = [{"text": f"{s}: {k} table{'' if k == 1 else 's'}"} for s, k in schemas.most_common()]
+    return node
+
+
+def edges(bp: dict, cfg: Dict[str, Any]) -> List[dict]:
+    """The connections a picture draws: each part's strongest calls, the API, and the heaviest users of each outside
+    system (`max_ext_edges`; the node counts the rest, and system.json lists them)."""
+    comps = bp["components"]
+    out_edges = [{"from": s, "to": t, "kind": "call"} for s, t in drawn_flows(bp, cfg) + drawn_api(bp)]
+    for key in sorted(bp["internal"]):
+        src, container = key.split(">")
+        out_edges += [{"from": src, "to": cid, "kind": "call"} for cid, c in sorted(comps.items())
+                      if c.get("container") == container and not c.get("quiet")]
+    for ext_id, e in bp["externals"].items():
+        users = [u for u in e["used_by"] if not comps.get(u, {}).get("quiet")]
+        kind = "data" if e.get("shape") == "db" else "network"
+        out_edges += [{"from": u, "to": "x:" + ext_id, "kind": kind} for u in users[: cfg["picture"]["max_ext_edges"]]]
+    seen, out = set(), []
+    for e in out_edges:
+        if (e["from"], e["to"]) not in seen:
+            seen.add((e["from"], e["to"]))
+            out.append(e)
+    return out
+
+
+def map_legend(nodes: List[dict], drawn: List[dict], cfg: Dict[str, Any]) -> List[tuple]:
+    """The legend of a system map: a mark for each kind of part the map has, then each kind of line it draws."""
+    words = cfg["picture"]["legend"]
+    styles = {n.get("style") for n in nodes}
+    kinds = {e["kind"] for e in drawn}
+    marks = [("band", "new", words.get("new", "new")), ("dashed", "", words.get("legacy", "legacy")),
+             ("band", "undeclared", words.get("undeclared", "undeclared"))]
+    present = [m for m, s in zip(marks, ("new", "legacy", "undeclared")) if s in styles]
+    return present + [m for m in pictures.MAP_LEGEND if m[0] == "line" and m[1] in kinds]
+
+
+def system_graph(bp: dict, cfg: Dict[str, Any]) -> dict:
+    comps = bp["components"]
+    nodes = [part_node(cid, comps[cid], bp, cfg) for cid in sorted(comps, key=lambda k: part_order(comps[k], k))]
+    nodes += [outside_node(x, e, bp) for x, e in bp["externals"].items()]
+    drawn = edges(bp, cfg)
+    return {"columns": cfg["picture"]["columns"], "nodes": nodes, "edges": drawn, "legend": map_legend(nodes, drawn, cfg)}
+
+
+def system_svg(bp: dict, cfg: Dict[str, Any]) -> str:
+    return pictures.map_svg(system_graph(bp, cfg), label="System map")
+
+
+def diff_graph(d: dict, a: dict, b: dict, cfg: Dict[str, Any]) -> Optional[dict]:
+    """Only what changed, lit, with the parts it touches beside it for context; None when nothing structural changed."""
+    status: Dict[str, str] = {}
+    lines: Dict[str, List[dict]] = defaultdict(list)
+    subs: Dict[str, str] = {}
+    changed: Dict[tuple, tuple] = {}  # (from, to) -> (kind, status)
+    comps = {**a["components"], **b["components"]}
+    containers = set(a.get("container_labels", {})) | set(b.get("container_labels", {}))
+    store = "x:" + b.get("store", "postgres")
+    for c in d["changes"]:
+        op, entity = {"add": "added", "remove": "removed", "change": "changed"}[c["op"]], c["entity"]
+        if entity == "component":
+            status[c["id"]] = op
+            if c["detail"]:
+                subs[c["id"]] = c["detail"][0]
+        elif entity == "route":
+            owner = b["routes"].get(c["id"]) or a["routes"].get(c["id"])
+            lines[owner].append({"text": c["id"], "code": True, "status": op})
+            status.setdefault(owner, "changed")
+        elif entity == "settings":
+            lines[c["id"]].append({"text": "reads " + c["detail"][0], "code": True, "status": "added"})
+            status.setdefault(c["id"], "changed")
+        elif entity == "flow":
+            src, dst = c["id"].split(">")
+            targets = [k for k, v in sorted(comps.items()) if v.get("container") == dst] if dst in containers else [dst]
+            for t in targets:
+                changed[(src, t)] = ("call", op)
+        elif entity == "external":
+            user, ext = c["id"].split(">")
+            meta = b["externals"].get(ext) or a["externals"].get(ext) or {}
+            changed[(user, "x:" + ext)] = ("data" if meta.get("shape") == "db" else "network", op)
+        elif entity in ("table", "migration"):
+            detail = ", ".join(x.replace("+ column ", "+ ") for x in c["detail"])
+            lines[store].append({"text": c["id"] + (f": {detail}" if detail else ""), "code": True, "status": op})
+            status[store] = "changed"
+    for ext in set(a["externals"]) ^ set(b["externals"]):
+        status["x:" + ext] = "added" if ext in b["externals"] else "removed"
+    hot = set(status) | {end for pair in changed for end in pair}
+    if not hot:
+        return None
+    context = [e for e in edges(b, cfg) if (e["from"], e["to"]) not in changed]
+    context += [e for e in edges(a, cfg) if status.get(e["from"]) == "removed" or status.get(e["to"]) == "removed"]
+    shown = set(hot)
+    for e in context:
+        if e["from"] in hot or e["to"] in hot:
+            shown |= {e["from"], e["to"]}
+    nodes = []
+    for cid in sorted(comps, key=lambda k: part_order(comps[k], k)):
+        if cid in shown:
+            n = part_node(cid, comps[cid], b if cid in b["components"] else a, cfg)
+            n.update(status=status.get(cid, "same"), lines=lines.get(cid, []), sub=subs.get(cid, n["sub"]))
+            nodes.append(n)
+    outside = {**{x: (e, a) for x, e in a["externals"].items()}, **{x: (e, b) for x, e in b["externals"].items()}}
+    for ext, (e, side) in outside.items():
+        if "x:" + ext in shown:
+            n = outside_node(ext, e, side)
+            n.update(status=status.get(n["id"], "same"), lines=lines.get(n["id"], []))
+            nodes.append(n)
+    if store in shown and store not in {n["id"] for n in nodes}:
+        nodes.append({"id": store, "col": "store", "type": "store", "title": "database schema", "sub": "",
+                      "status": "changed", "lines": lines[store]})
+    out_edges = [{"from": s, "to": t, "kind": k, "status": st} for (s, t), (k, st) in changed.items()]
+    seen = set(changed)
+    for e in context:
+        if (e["from"], e["to"]) not in seen and e["from"] in shown and e["to"] in shown:
+            seen.add((e["from"], e["to"]))
+            out_edges.append({**e, "status": "same"})
+    deploy = [f"the {c['id']} container" if c["entity"] == "container" else c["id"]
+              for c in d["changes"] if c["entity"] in ("deploy", "container")]
+    return {"columns": cfg["picture"]["columns"], "nodes": nodes, "edges": out_edges,
+            "footer": ["Also changed, not drawn: " + ", ".join(deploy)] if deploy else []}
+
+
+def diff_svg(d: dict, a: dict, b: dict, cfg: Dict[str, Any]) -> Optional[str]:
+    graph = diff_graph(d, a, b, cfg)
+    return pictures.map_svg(graph, diff=True, label="System diff") if graph else None