keygraph 0.2.0__py3-none-any.whl

keygraph/context.py

@@ -0,0 +1,387 @@
+"""Business context stored relationally in SQLite (with FTS5 search): concepts, links, relations,
+column semantics, metrics and table notes. Every row records its source, a verified flag and a timestamp."""
+
+from __future__ import annotations
+
+import json
+import re
+import sqlite3
+import time
+from pathlib import Path
+
+_DIGITS = {**{0x0660 + i: 0x30 + i for i in range(10)}, **{0x06F0 + i: 0x30 + i for i in range(10)}}
+_MAP = {0x064A: 0x06CC, 0x0649: 0x06CC, 0x0643: 0x06A9, 0x200C: 0x20, 0x0640: None, 0x06C0: 0x0647, 0x0629: 0x0647}
+for _c in range(0x064B, 0x0660):
+    _MAP[_c] = None
+
+
+def normalize(text: str) -> str:
+    """Lowercase, unify Persian/Arabic digits, ye/kaf variants, drop diacritics, split snake/camel/dotted names."""
+    text = str(text or "")
+    text = re.sub(r"(?<=[a-z0-9])(?=[A-Z])", " ", text)
+    text = text.translate(_DIGITS).translate(_MAP).lower()
+    return re.sub(r"[\s_./\\:;,|()\[\]{}\-]+", " ", text).strip()
+
+
+STOPWORDS = frozenset("a an and are as at be by for from in into is of on or per the to with".split())
+PHRASE_BOOST = 2.0
+
+
+def tokens(text: str) -> list[str]:
+    return re.findall(r"\w+", normalize(text))
+
+
+SCHEMA = """
+CREATE TABLE IF NOT EXISTS concepts(name TEXT PRIMARY KEY, definition TEXT, domain TEXT, owner TEXT, synonyms TEXT,
+  source TEXT, verified INTEGER, updated_at TEXT);
+CREATE TABLE IF NOT EXISTS concept_links(concept TEXT, table_id TEXT, col TEXT, role TEXT, note TEXT, source TEXT,
+  verified INTEGER, updated_at TEXT, PRIMARY KEY(concept, table_id, col, role));
+CREATE TABLE IF NOT EXISTS concept_relations(from_concept TEXT, rel TEXT, to_concept TEXT, edge_id TEXT, note TEXT,
+  source TEXT, verified INTEGER, updated_at TEXT, PRIMARY KEY(from_concept, rel, to_concept));
+CREATE TABLE IF NOT EXISTS column_semantics(table_id TEXT, col TEXT, meaning TEXT, unit TEXT, codes TEXT, null_meaning TEXT,
+  pii INTEGER, typical_filters TEXT, index_hint TEXT, source TEXT, verified INTEGER, updated_at TEXT,
+  PRIMARY KEY(table_id, col));
+CREATE TABLE IF NOT EXISTS metrics(name TEXT PRIMARY KEY, definition TEXT, expression TEXT, grain TEXT, tables TEXT,
+  filters TEXT, caveats TEXT, source TEXT, verified INTEGER, updated_at TEXT);
+CREATE TABLE IF NOT EXISTS table_notes(table_id TEXT PRIMARY KEY, grain TEXT, freshness TEXT, soft_delete TEXT,
+  date_column TEXT, recommended_indexes TEXT, pitfalls TEXT, summary TEXT, source TEXT, verified INTEGER, updated_at TEXT);
+CREATE TABLE IF NOT EXISTS column_profile(table_id TEXT, col TEXT, profile TEXT, profiled_at TEXT, sample_size INTEGER, method TEXT,
+  PRIMARY KEY(table_id, col));
+CREATE TABLE IF NOT EXISTS column_details(table_id TEXT, col TEXT, detail TEXT, PRIMARY KEY(table_id, col));
+CREATE TABLE IF NOT EXISTS table_info(table_id TEXT PRIMARY KEY, info TEXT);
+CREATE TABLE IF NOT EXISTS curation(kind TEXT, body JSON, source TEXT);
+CREATE VIRTUAL TABLE IF NOT EXISTS search_fts USING fts5(kind, ref, title, body, tokenize='unicode61');
+"""
+
+KINDS = {"concept", "relation", "link", "column", "metric", "table"}
+ROLES = ("primary-record", "attribute", "foreign-reference", "measure", "filter")
+
+
+def _j(v) -> str:
+    return json.dumps(v, ensure_ascii=False) if v not in (None, "") else ""
+
+
+def _l(v) -> list:
+    if v in (None, ""):
+        return []
+    return v if isinstance(v, list) else [x.strip() for x in str(v).split(",") if x.strip()]
+
+
+def safe_source(src) -> str:
+    """A context source as it may appear in outputs: never an absolute local path (keep the bare file name)."""
+    s = str(src or "").strip()
+    if s.startswith(("/", "~", "\\")) or re.match(r"^[A-Za-z]:[\\/]", s):
+        return re.split(r"[\\/]", s.rstrip("/\\"))[-1]
+    return s
+
+
+class ContextDB:
+    def __init__(self, path: str | Path, graph=None, readonly: bool = False):
+        """readonly: open an existing file without writing to it; a missing file gives an empty in-memory context."""
+        self.path = Path(path)
+        if readonly and self.path.exists():
+            self.db = sqlite3.connect(f"{self.path.resolve().as_uri()}?mode=ro", uri=True)
+        elif readonly:
+            self.db = sqlite3.connect(":memory:")
+        else:
+            self.path.parent.mkdir(parents=True, exist_ok=True)
+            self.db = sqlite3.connect(self.path)
+        self.db.row_factory = sqlite3.Row
+        if not (readonly and self.path.exists()):
+            self.db.executescript(SCHEMA)
+        else:  # a file from an older keygraph may lack newer tables: read them as empty (temp tables, file untouched)
+            have = {r[0] for r in self.db.execute("SELECT name FROM sqlite_master")}
+            for stmt in SCHEMA.split(";"):
+                m = re.search(r"CREATE (?:VIRTUAL )?TABLE IF NOT EXISTS (\w+)", stmt)
+                if m and m.group(1) not in have:
+                    self.db.execute(stmt.replace("IF NOT EXISTS ", "IF NOT EXISTS temp.", 1))
+        self.graph = graph
+        self.warnings: list[str] = []
+
+    def close(self) -> None:
+        try:
+            self.db.commit()
+        finally:
+            self.db.close()
+
+    def __enter__(self) -> "ContextDB":
+        return self
+
+    def __exit__(self, exc_type, *_) -> None:
+        if exc_type is None:
+            self.close()
+        else:  # leave half-done writes uncommitted, but always release the handle
+            self.db.close()
+
+    # --- reference checks --------------------------------------------------------
+    def _tid(self, ref: str | None) -> str:
+        if not ref or not self.graph:
+            return ref or ""
+        try:
+            return self.graph.resolve(ref)
+        except ValueError as exc:
+            self.warnings.append(f"unknown table '{ref}': {exc}")
+            return ref
+
+    def _check_col(self, tid: str, col: str | None) -> None:
+        if col and self.graph and tid in self.graph.nodes and col not in {c["name"] for c in self.graph.nodes[tid]["columns"]}:
+            self.warnings.append(f"unknown column {tid}.{col}")
+
+    # --- writes ------------------------------------------------------------------
+    def _meta(self, d: dict, default_source: str, default_verified) -> tuple:
+        v = d.get("verified", default_verified)
+        return (safe_source(d.get("source") or default_source) or "unknown", 1 if v in (True, 1, "true", "yes") else 0,
+                time.strftime("%Y-%m-%dT%H:%M:%S"))
+
+    def upsert(self, kind: str, d: dict, source: str = "", verified=False) -> None:
+        if kind not in KINDS:
+            raise ValueError(f"unknown kind '{kind}'; use one of {sorted(KINDS)}")
+        src = self._meta(d, source, verified)
+        x = self.db
+        if kind == "concept":
+            name = str(d["name"]).strip()
+            x.execute("INSERT OR REPLACE INTO concepts VALUES(?,?,?,?,?,?,?,?)",
+                      (name, d.get("definition", ""), d.get("domain", ""), d.get("owner", ""), _j(_l(d.get("synonyms"))), *src))
+            for ln in d.get("links") or []:
+                self.upsert("link", {**ln, "concept": name, "source": ln.get("source") or d.get("source"),
+                                     "verified": ln.get("verified", d.get("verified"))}, source, verified)
+        elif kind == "link":
+            tid = self._tid(d["table"])
+            col = d.get("column") or ""
+            self._check_col(tid, col)
+            role = d.get("role") or ("primary-record" if not col else "attribute")
+            if role not in ROLES:
+                self.warnings.append(f"link role '{role}' is not one of {ROLES}")
+            x.execute("INSERT OR REPLACE INTO concept_links VALUES(?,?,?,?,?,?,?,?)",
+                      (d["concept"], tid, col, role, d.get("note", ""), *src))
+        elif kind == "relation":
+            x.execute("INSERT OR REPLACE INTO concept_relations VALUES(?,?,?,?,?,?,?,?)",
+                      (d["from"], d.get("rel", "related-to"), d["to"], d.get("edge", ""), d.get("note", ""), *src))
+        elif kind == "column":
+            tid = self._tid(d["table"])
+            self._check_col(tid, d["column"])
+            x.execute("INSERT OR REPLACE INTO column_semantics VALUES(?,?,?,?,?,?,?,?,?,?,?,?)",
+                      (tid, d["column"], d.get("meaning", ""), d.get("unit", ""), _j(d.get("codes") or {}),
+                       d.get("null_meaning", ""), 1 if d.get("pii") in (True, 1, "true", "yes") else 0,
+                       _j(d.get("typical_filters", "")), d.get("index_hint", ""), *src))
+        elif kind == "metric":
+            tbls = [self._tid(t) for t in _l(d.get("tables"))]
+            x.execute("INSERT OR REPLACE INTO metrics VALUES(?,?,?,?,?,?,?,?,?,?)",
+                      (d["name"], d.get("definition", ""), d.get("expression", ""), d.get("grain", ""), _j(tbls),
+                       d.get("filters", ""), d.get("caveats", ""), *src))
+        elif kind == "table":
+            tid = self._tid(d["table"])
+            self._check_col(tid, d.get("date_column"))
+            x.execute("INSERT OR REPLACE INTO table_notes VALUES(?,?,?,?,?,?,?,?,?,?,?)",
+                      (tid, d.get("grain", ""), d.get("freshness", ""), d.get("soft_delete", ""), d.get("date_column", ""),
+                       _j(_l(d.get("recommended_indexes"))), d.get("pitfalls", ""), d.get("summary", ""), *src))
+        x.commit()
+
+    def import_path(self, path: str | Path, default_verified=False) -> dict:
+        p = Path(path)
+        files = sorted(f for f in ([p] if p.is_file() else p.rglob("*")) if f.suffix.lower() in (".yaml", ".yml", ".json"))
+        counts = {k: 0 for k in ("concepts", "relations", "columns", "metrics", "tables")}
+        for f in files:
+            text = f.read_text(encoding="utf-8")
+            if f.suffix.lower() == ".json":
+                raw = json.loads(text)
+            else:
+                import yaml
+                raw = yaml.safe_load(text) or {}
+            rel = f.relative_to(p).as_posix() if p.is_dir() else f.name
+            if not isinstance(raw, dict):
+                raise ValueError(f"context file {rel}: expected a mapping at the top level, got {type(raw).__name__}")
+            self._save_curation(rel, raw)
+            src, ver = safe_source(raw.get("source")) or rel, raw.get("verified", default_verified)
+            for key, kind in (("concepts", "concept"), ("relations", "relation"), ("columns", "column"),
+                              ("metrics", "metric"), ("tables", "table")):
+                for item in raw.get(key) or []:
+                    self.upsert(kind, item, src, ver)
+                    counts[key] += 1
+        self.reindex()
+        return counts
+
+    def clear_curation(self) -> None:
+        """Forget all curation rules (a build that imports context starts from the files it imports now)."""
+        self.db.execute("DELETE FROM curation")
+        self.db.commit()
+
+    def _save_curation(self, source: str, raw: dict) -> None:
+        """Replace this file's curation rows, so importing the same file again never duplicates rules."""
+        self.db.execute("DELETE FROM curation WHERE source=?", (source,))
+        for rule in raw.get("areas") or []:
+            self.db.execute("INSERT INTO curation VALUES('area',?,?)", (json.dumps(rule, ensure_ascii=False), source))
+        if raw.get("mirrors"):
+            self.db.execute("INSERT INTO curation VALUES('mirrors',?,?)", (json.dumps(raw["mirrors"], ensure_ascii=False), source))
+        if raw.get("hubs"):
+            self.db.execute("INSERT INTO curation VALUES('hubs',?,?)", (json.dumps(raw["hubs"], ensure_ascii=False), source))
+        self.db.commit()
+
+    # --- reads -------------------------------------------------------------------
+    def rows(self, table: str, where: str = "", args: tuple = ()) -> list[dict]:
+        out = [dict(r) for r in self.db.execute(f"SELECT * FROM {table} {where}", args)]
+        for r in out:  # rows written before sources were made relative may still hold absolute paths
+            if r.get("source"):
+                r["source"] = safe_source(r["source"])
+        return out
+
+    def curation(self) -> dict:
+        """Area rules, hub and mirror overrides from the imported files. Malformed entries (user-written YAML)
+        are skipped with a line in out["warnings"], never raised."""
+        out: dict = {"areas": [], "hubs": {"include": [], "exclude": []}, "mirrors": {}, "warnings": []}
+        warn = out["warnings"].append
+        for r in self.db.execute("SELECT * FROM curation ORDER BY source, rowid"):
+            body, src = json.loads(r["body"]), safe_source(r["source"])
+            if r["kind"] == "area":
+                if not isinstance(body, dict):
+                    warn(f"{src}: area rule {body!r} ignored: expected a mapping with name, match or merge")
+                elif not any(body.get(k) for k in ("name", "match", "merge")):
+                    warn(f"{src}: area rule {body!r} ignored: it needs a name, a match or a merge")
+                else:
+                    rule = dict(body)
+                    for k in ("match", "pin", "exclude", "merge"):
+                        if k in rule:
+                            rule[k] = [str(x) for x in _l(rule[k]) if x is not None]
+                    if "name" in rule:
+                        rule["name"] = str(rule["name"])
+                    out["areas"].append(rule)
+            elif r["kind"] == "mirrors":  # {mirror schema: original schema, or false for "never a mirror"}
+                if isinstance(body, dict):
+                    out["mirrors"].update({str(k): v for k, v in body.items()})
+                else:
+                    warn(f"{src}: mirrors ignored: expected a mapping of mirror schema to original schema")
+            elif isinstance(body, dict):
+                for k in ("include", "exclude"):
+                    out["hubs"][k] += [str(t) for t in _l(body.get(k)) if str(t) not in out["hubs"][k]]
+            else:
+                warn(f"{src}: hubs ignored: expected a mapping with include and/or exclude lists")
+        return out
+
+    def counts(self) -> dict:
+        return {t: self.db.execute(f"SELECT COUNT(*) FROM {t}").fetchone()[0]
+                for t in ("concepts", "concept_links", "concept_relations", "column_semantics", "metrics", "table_notes")}
+
+    def table_context(self, tid: str) -> dict:
+        n = self.rows("table_notes", "WHERE table_id=?", (tid,))
+        links = self.rows("concept_links", "WHERE table_id=?", (tid,))
+        cols = self.rows("column_semantics", "WHERE table_id=? ORDER BY col", (tid,))
+        for c in cols:
+            c["codes"] = json.loads(c["codes"]) if c["codes"] else {}
+        metrics = [m for m in self.rows("metrics") if tid in json.loads(m["tables"] or "[]")]
+        return {"notes": n[0] if n else None, "concepts": links, "columns": cols, "metrics": metrics}
+
+    def dump(self) -> dict:
+        out = {"concepts": [], "relations": [], "columns": [], "metrics": [], "tables": []}
+        for c in self.rows("concepts", "ORDER BY name"):
+            c["synonyms"] = json.loads(c["synonyms"] or "[]")
+            c["links"] = [{"table": l["table_id"], "column": l["col"], "role": l["role"], "note": l["note"],
+                           "source": safe_source(l["source"]), "verified": bool(l["verified"])}
+                          for l in self.rows("concept_links", "WHERE concept=?", (c["name"],))]
+            c["verified"] = bool(c["verified"])
+            out["concepts"].append(c)
+        for r in self.rows("concept_relations", "ORDER BY from_concept,rel,to_concept"):
+            out["relations"].append({"from": r["from_concept"], "rel": r["rel"], "to": r["to_concept"], "edge": r["edge_id"],
+                                     "note": r["note"], "source": safe_source(r["source"]), "verified": bool(r["verified"])})
+        for r in self.rows("column_semantics", "ORDER BY table_id,col"):
+            r["codes"] = json.loads(r["codes"]) if r["codes"] else {}
+            r["verified"], r["pii"] = bool(r["verified"]), bool(r["pii"])
+            r["table"], r["column"] = r.pop("table_id"), r.pop("col")
+            out["columns"].append(r)
+        for r in self.rows("metrics", "ORDER BY name"):
+            r["tables"] = json.loads(r["tables"] or "[]")
+            r["verified"] = bool(r["verified"])
+            out["metrics"].append(r)
+        for r in self.rows("table_notes", "ORDER BY table_id"):
+            r["recommended_indexes"] = json.loads(r["recommended_indexes"] or "[]")
+            r["verified"] = bool(r["verified"])
+            r["table"] = r.pop("table_id")
+            out["tables"].append(r)
+        return out
+
+    # --- full-text index ---------------------------------------------------------
+    def index_graph(self, graph) -> None:
+        self.graph = graph
+        self.db.execute("DELETE FROM search_fts WHERE kind IN ('table','column')")
+        rows = []
+        self.db.execute("DELETE FROM column_details")
+        self.db.execute("DELETE FROM table_info")
+        det, tinfo = [], []
+        for n in graph.nodes.values():
+            rows.append(("table", n["id"], n["id"], normalize(n["id"] + " " + str((n.get("info") or {}).get("comment", "")))))
+            if n.get("info"):
+                tinfo.append((n["id"], json.dumps(n["info"], ensure_ascii=False, default=str)))
+            for c in n["columns"]:
+                extra = " ".join([c.get("comment", ""), " ".join(c.get("values", []))])
+                rows.append(("column", f"{n['id']}.{c['name']}", c["name"], normalize(n["table"] + " " + c["name"] + " " + extra)))
+                if len(c) > 5:
+                    det.append((n["id"], c["name"], json.dumps({k: v for k, v in c.items() if k not in ("name", "family")}, ensure_ascii=False)))
+        self.db.executemany("INSERT INTO column_details VALUES(?,?,?)", det)
+        self.db.executemany("INSERT OR REPLACE INTO table_info VALUES(?,?)", tinfo)
+        self.db.executemany("INSERT INTO search_fts(kind,ref,title,body) VALUES(?,?,?,?)", rows)
+        self.db.commit()
+
+    def save_profiles(self, profiles: dict) -> None:
+        """profiles: {"schema.table.col": profile dict}. PII profiles never carry raw values (see profile.py)."""
+        for key, p in profiles.items():
+            tid, col = key.rsplit(".", 1)
+            self.db.execute("INSERT OR REPLACE INTO column_profile VALUES(?,?,?,?,?,?)",
+                            (tid, col, json.dumps(p, ensure_ascii=False, default=str), p.get("profiled_at"), p.get("sample_size"), p.get("method")))
+        self.db.commit()
+
+    def load_profiles(self) -> dict:
+        return {f"{r['table_id']}.{r['col']}": json.loads(r["profile"]) for r in self.db.execute("SELECT * FROM column_profile")}
+
+    def reindex(self) -> None:
+        self.db.execute("DELETE FROM search_fts WHERE kind NOT IN ('table','column')")
+        rows = []
+        for r in self.db.execute("SELECT * FROM column_profile"):
+            p = json.loads(r["profile"])
+            if p.get("pii") or not p.get("top_values"):
+                continue
+            rows.append(("column_value", f"{r['table_id']}.{r['col']}", r["col"], normalize(r["col"] + " " + " ".join(str(k) for k, _ in p["top_values"]))))
+        for c in self.rows("concepts"):
+            syn = " ".join(json.loads(c["synonyms"] or "[]"))
+            rows.append(("concept", c["name"], c["name"], normalize(" ".join([c["name"], syn, c["definition"] or "", c["domain"] or ""]))))
+        for m in self.rows("metrics"):
+            rows.append(("metric", m["name"], m["name"], normalize(" ".join([m["name"], m["definition"] or "", m["caveats"] or ""]))))
+        for t in self.rows("table_notes"):
+            rows.append(("table_note", t["table_id"], t["table_id"], normalize(" ".join(
+                [t["grain"] or "", t["summary"] or "", t["pitfalls"] or "", t["freshness"] or ""]))))
+        for c in self.rows("column_semantics"):
+            codes = " ".join(f"{k} {v}" for k, v in json.loads(c["codes"] or "{}").items())
+            rows.append(("column_ctx", f"{c['table_id']}.{c['col']}", c["col"], normalize(" ".join(
+                [c["col"], c["meaning"] or "", c["unit"] or "", codes, c["null_meaning"] or ""]))))
+        self.db.executemany("INSERT INTO search_fts(kind,ref,title,body) VALUES(?,?,?,?)", rows)
+        self.db.commit()
+
+    def fts(self, query: str, limit: int = 300) -> list[tuple[str, str, float]]:
+        """Prefix OR over the meaningful words, plus a boost for rows holding the whole query as a phrase.
+
+        Words shorter than 3 characters and common stopwords are left out of the prefix OR ("in"* would match
+        inventory, invoices and every int column), unless the query has nothing else.
+        """
+        toks = tokens(query)
+        if not toks:
+            return []
+        words = [t for t in toks if len(t) >= 3 and t not in STOPWORDS]
+        q = " OR ".join(f'"{t}"*' for t in words) if words else " OR ".join(f'"{t}"' for t in toks)
+        scores: dict[tuple[str, str], float] = {}
+        order: list[tuple[str, str]] = []
+
+        def run(match: str, boost: float) -> None:
+            try:
+                cur = self.db.execute(
+                    "SELECT kind, ref, bm25(search_fts, 0.0, 0.0, 3.0, 1.0) AS s FROM search_fts WHERE search_fts MATCH ? "
+                    "ORDER BY s LIMIT ?", (match, limit))
+            except sqlite3.OperationalError:
+                return
+            for r in cur:
+                k = (r["kind"], r["ref"])
+                if k not in scores:
+                    order.append(k)
+                scores[k] = scores.get(k, 0.0) + boost * -r["s"]
+
+        run(q, 1.0)
+        if len(toks) > 1:
+            run('"' + " ".join(toks) + '"', PHRASE_BOOST)
+        return sorted(((k, r, scores[(k, r)]) for k, r in order), key=lambda x: -x[2])[:limit]

keygraph/docs_out.py

@@ -0,0 +1,199 @@
+"""Markdown + Mermaid pages per area and per table, for agents and docs."""
+
+from __future__ import annotations
+
+import re
+from itertools import combinations
+from pathlib import Path
+
+from keygraph.areas import area_of
+from keygraph.graph import Graph
+from keygraph.query import find_paths
+from keygraph.summary import relation_summary
+
+MERMAID_LIMIT = 25  # an area's Mermaid diagram draws its first (most connected) 25 tables
+
+
+def slug(s: str) -> str:
+    return re.sub(r"[^a-z0-9]+", "-", s.lower()).strip("-")
+
+
+def table_file(t: str) -> str:
+    return re.sub(r"[^\w.\-]", "_", t) + ".md"
+
+
+def _ident(text: str, fallback: str = "t") -> str:
+    """A Mermaid-safe identifier: ASCII letters, digits and underscores, never starting with a digit."""
+    n = re.sub(r"[^A-Za-z0-9_]", "_", str(text or ""))
+    if not re.search(r"[A-Za-z0-9]", n):
+        n = fallback
+    return "t_" + n if n[0].isdigit() else n
+
+
+def _names(graph: Graph, tables: list[str]) -> dict[str, str]:
+    """Unique Mermaid entity names: bare table name, or schema_table when names collide."""
+    bare = {t: _ident(graph.nodes[t]["table"]) for t in tables}
+    count: dict[str, int] = {}
+    for n in bare.values():
+        count[n] = count.get(n, 0) + 1
+    names: dict[str, str] = {}
+    used: set[str] = set()
+    for t in tables:
+        n = bare[t]
+        if count[n] > 1:
+            n = _ident(graph.nodes[t]["schema"] + "_" + n)
+        base, i = n, 2
+        while n in used:
+            n, i = f"{base}_{i}", i + 1
+        used.add(n)
+        names[t] = n
+    return names
+
+
+def _edge_index(graph: Graph) -> dict[str, list[dict]]:
+    """table id -> the edges that start at it (built once per docs run)."""
+    idx: dict[str, list[dict]] = {}
+    for e in graph.edges:
+        idx.setdefault(e["from"], []).append(e)
+    return idx
+
+
+def mermaid_area(graph: Graph, area: dict, limit: int = MERMAID_LIMIT, out_edges: dict[str, list[dict]] | None = None) -> str:
+    tables = area["tables"][:limit]
+    inside = set(tables)
+    ent = _names(graph, tables)
+    out_edges = out_edges if out_edges is not None else _edge_index(graph)
+    lines = ["erDiagram"]
+    for t in tables:
+        n = graph.nodes[t]
+        mine = out_edges.get(t, [])
+        rel = [e for e in mine if not (e.get("hub") or e.get("role"))]
+        keys = set(n.get("primary_key", [])) | {c for e in rel for c in e["from_columns"]}
+        if ent[t] != n["table"]:
+            lines.append(f"    %% {ent[t]} is {t}")
+        lines.append(f"    {ent[t]} {{")
+        for c in n.get("columns", []):
+            if c["name"] in keys:
+                typ = _ident((c["type"] or "col").split("(")[0], "col")
+                tag = " PK" if c["name"] in n.get("primary_key", []) else " FK"
+                lines.append(f"        {typ} {_ident(c['name'], 'col')}{tag}")
+        lines.append("    }")
+        for e in mine:
+            if e.get("hub") or e.get("role"):
+                kind = "hub" if e.get("hub") else "role"
+                lines.append(f"    %% {ent[t]} {kind} ref to {e['to']} via {', '.join(e['from_columns'])}")
+    for t in tables:
+        for e in out_edges.get(t, []):
+            if e.get("hub") or e.get("role") or e["to"] not in inside or e["from"] == e["to"]:
+                continue
+            left = "||" if e.get("cardinality") == "one-to-one" else "}o"
+            col = ", ".join(e["from_columns"]).replace('"', "'")
+            lines.append(f'    {ent[e["from"]]} {left}--|| {ent[e["to"]]} : "{col}"')
+    return "\n".join(lines)
+
+
+def _plural(n: int, word: str) -> str:
+    return f"{n} {word}{'' if n == 1 else 's'}"
+
+
+def _table_md(graph: Graph, t: str, areas: list[dict], name_of: dict, ctx_notes: dict,
+              touching: dict[str, list[dict]] | None = None, of: dict[str, str] | None = None) -> str:
+    n, s = graph.nodes[t], relation_summary(graph, t, areas, edges=None if touching is None else touching.get(t, []), of=of)
+    note = ctx_notes.get(t, {})
+    out = [f"# {t}", "", f"Area: {name_of.get(s['area'], '-')}", ""]
+    if note.get("grain"):
+        out += [f"One row: {note['grain']}", ""]
+    out += ["## Keys and indexes", "", f"- primary key: {', '.join(n['primary_key']) or '(none)'}"]
+    out += [f"- {ix['name']}{' (unique)' if ix.get('unique') else ''}: {', '.join(ix['columns'])}" for ix in n["indexes"]]
+    out += ["", "## Columns", "", "| column | type | nullable |", "|---|---|---|"]
+    out += [f"| {c['name']} | {c['type']} | {'yes' if c.get('nullable', True) else 'no'} |" for c in n["columns"]]
+    for title, groups in (("Points to", s["out"]), ("Pointed to by", s["in"])):
+        out += ["", f"## {title}", ""]
+        for g in groups:
+            more = f" (+{g['count'] - 8} more)" if g["count"] > 8 else ""
+            out.append(f"- via `{g['column']}`: {', '.join(g['tables'][:8])}{more}")
+        if not groups:
+            out.append("- (none)")
+    if s["hub_labels"]:
+        out += ["", "Hub references: " + ", ".join(f"{h['to']} via {', '.join(h['columns'])}" for h in s["hub_labels"])]
+    return "\n".join(out) + "\n"
+
+
+def _degrees(graph: Graph) -> dict[str, int]:
+    d: dict[str, int] = {}
+    for e in graph.edges:
+        if e.get("hub") or e.get("role") or e["confidence"] < 0.6 or e["from"] == e["to"]:
+            continue
+        d[e["from"]] = d.get(e["from"], 0) + 1
+        d[e["to"]] = d.get(e["to"], 0) + 1
+    return d
+
+
+def _join_paths(graph: Graph, area: dict, deg: dict[str, int]) -> list[str]:
+    lines = []
+    top = sorted(area["tables"], key=lambda t: (-deg.get(t, 0), t))[:5]
+    for a, b in combinations(top, 2):
+        paths = find_paths(graph, a, b, 3, 0.6, 1, max_pops=20000)
+        if paths:
+            lines.append(f"- {' -> '.join(paths[0]['tables'])}")
+    return lines
+
+
+def _neighbours(graph: Graph, areas: list[dict]) -> dict[str, dict[str, int]]:
+    """Per area id: other area id -> count of non-hub, non-role edges (confidence >= 0.6) crossing in either direction."""
+    of = {t: a["id"] for a in areas for t in a["tables"]}
+    res: dict[str, dict[str, int]] = {}
+    for e in graph.edges:
+        if e.get("hub") or e.get("role") or e["confidence"] < 0.6:
+            continue
+        a, b = of.get(e["from"]), of.get(e["to"])
+        if a and b and a != b:
+            res.setdefault(a, {})[b] = res.setdefault(a, {}).get(b, 0) + 1
+            res.setdefault(b, {})[a] = res.setdefault(b, {}).get(a, 0) + 1
+    return res
+
+
+def write_docs(out: Path, graph: Graph, areas: list[dict], ctx_dump: dict | None) -> dict:
+    out = Path(out)
+    (out / "areas").mkdir(parents=True, exist_ok=True)
+    (out / "tables").mkdir(parents=True, exist_ok=True)
+    name_of = {a["id"]: a["name"] for a in areas}
+    ctx_notes = {x["table"]: x for x in (ctx_dump or {}).get("tables", [])}
+    written: list[str] = ["areas/index.md"]
+    idx = ["# Areas", "", "| area | tables | top tables |", "|---|---|---|"]
+    used: set[str] = set()
+    fname: dict[str, str] = {}
+    for a in areas:
+        fn = slug(a["name"]) or "area"
+        if fn in used or fn == "index":
+            fn = f"{fn}-{a['id']}"
+        used.add(fn)
+        fname[a["id"]] = fn
+    nb, deg = _neighbours(graph, areas), _degrees(graph)
+    out_edges, of = _edge_index(graph), area_of(areas)
+    touching: dict[str, list[dict]] = {}
+    for e in graph.edges:
+        touching.setdefault(e["from"], []).append(e)
+        if e["to"] != e["from"]:
+            touching.setdefault(e["to"], []).append(e)
+    for a in areas:
+        fn = fname[a["id"]]
+        idx.append(f"| [{a['name']}]({fn}.md) | {a['size']} | {', '.join(a['top'])} |")
+        shown = f" The diagram shows the {MERMAID_LIMIT} most connected." if a["size"] > MERMAID_LIMIT else ""
+        body = [f"# {a['name']}", "", f"{_plural(a['size'], 'table')}{' (curated)' if not a.get('auto', True) else ''}.{shown}", "",
+                "```mermaid", mermaid_area(graph, a, out_edges=out_edges), "```", ""]
+        paths = _join_paths(graph, a, deg)
+        if paths:
+            body += ["## Main join paths", ""] + paths + [""]
+        links = sorted(nb.get(a["id"], {}).items(), key=lambda kv: (-kv[1], name_of[kv[0]]))
+        body += ["## Neighbouring areas", ""]
+        body += [f"- [{name_of[o]}]({fname[o]}.md): {_plural(n, 'relation')}" for o, n in links] or ["- (none)"]
+        body += ["", "## Tables", ""]
+        body += [f"- {t}: {ctx_notes.get(t, {}).get('grain', '')}".rstrip(": ") for t in a["tables"]]
+        (out / "areas" / f"{fn}.md").write_text("\n".join(body) + "\n", encoding="utf-8")
+        written.append(f"areas/{fn}.md")
+    (out / "areas" / "index.md").write_text("\n".join(idx) + "\n", encoding="utf-8")
+    for t in graph.nodes:
+        (out / "tables" / table_file(t)).write_text(_table_md(graph, t, areas, name_of, ctx_notes, touching, of), encoding="utf-8")
+        written.append(f"tables/{table_file(t)}")
+    return {"areas": len(areas), "tables": len(graph.nodes), "files": written}