keygraph 0.2.0__py3-none-any.whl

keygraph/explorer.py

@@ -0,0 +1,128 @@
+"""Build the self-contained keygraph.html explorer.
+
+The page is one file: the keygraph design system CSS (fonts embedded), dagre, the theme helpers, the diagram
+renderer and the app script are inlined from static/ (copied there from design/ by `design/build.py --export-static`),
+and the graph plus context is embedded as compact JSON (gzip + base64 when large). Nothing is fetched at runtime.
+"""
+
+from __future__ import annotations
+
+import base64
+import gzip
+import json
+import re
+from pathlib import Path
+
+from .context import safe_source
+from .graph import Graph
+from .query import _is_log, table_info_line
+
+STATIC = Path(__file__).parent / "static"
+STATUS = {"declared": 0, "inferred": 1, "confirmed": 2, "added": 3, "probed": 4}
+CARD = {"many-to-one": 0, "one-to-one": 1, "many-to-many": 2, "one-to-many": 3}
+NOTE_KEYS = ("extra", "generated", "charset", "collation", "comment")
+
+
+def _col(c: dict) -> list:
+    """[name, type, nullable, default|None, enum/set members|None, other details]."""
+    note = "; ".join(f"{k}: {c[k]}" for k in NOTE_KEYS if c.get(k))
+    return [c["name"], c["type"], int(c.get("nullable", True)), c.get("default"), c.get("values") or None, note]
+
+
+def compact(graph: Graph, ctx_dump: dict | None = None) -> dict:
+    schemas = sorted({n["schema"] for n in graph.nodes.values()})
+    sidx = {s: i for i, s in enumerate(schemas)}
+    ids = sorted(graph.nodes)
+    tidx = {t: i for i, t in enumerate(ids)}
+    hubs = set(graph.data.get("hubs") or [])
+    tables = []
+    for t in ids:
+        n = graph.nodes[t]
+        ci = {c["name"]: i for i, c in enumerate(n["columns"])}
+        info = n.get("info") or {}
+        tables.append([
+            n["table"], sidx[n["schema"]], [_col(c) for c in n["columns"]],
+            [ci[c] for c in n["primary_key"] if c in ci],
+            [[i["name"], int(i["unique"]), int(i["primary"]), [ci[c] for c in i["columns"] if c in ci]] for i in n["indexes"]],
+            [[ci.get(p["id_column"], -1), ci.get(p["type_column"], -1)] for p in n["polymorphic"]],
+            table_info_line(n),
+            int(_is_log(graph, t)) | (2 if t in hubs else 0) | (4 if n.get("mirror_of") else 0),
+            info.get("rows_estimate"),
+        ])
+    edges = []
+    for e in graph.edges:
+        fi, ti = tidx[e["from"]], tidx[e["to"]]
+        fci = {c["name"]: i for i, c in enumerate(graph.nodes[e["from"]]["columns"])}
+        tci = {c["name"]: i for i, c in enumerate(graph.nodes[e["to"]]["columns"])}
+        edges.append([fi, ti, [fci[c] for c in e["from_columns"] if c in fci], [tci[c] for c in e["to_columns"] if c in tci],
+                      e["confidence"], STATUS.get(e["status"], 1), CARD.get(e["cardinality"], 0),
+                      int(e["ambiguous"]), e["evidence"], (1 if e.get("hub") else 0) | (2 if e.get("role") else 0)])
+    # area flags: 1 unconnected (no relations left after hubs), 2 mirror schema
+    areas = [[a["id"], a["name"], [tidx[t] for t in a["tables"] if t in tidx], int(bool(a.get("auto", True))),
+              (1 if a.get("unconnected") else 0) | (2 if a.get("mirror") else 0)]
+             for a in graph.data.get("areas") or []]
+    aidx = {a[0]: i for i, a in enumerate(areas)}
+    links = [[aidx[l["a"]], aidx[l["b"]], l["edges"]] for l in graph.data.get("area_links") or []
+             if l["a"] in aidx and l["b"] in aidx]
+    c = {"concepts": [], "rels": [], "cols": [], "metrics": [], "tn": []}
+    if ctx_dump:
+        for x in ctx_dump["concepts"]:
+            c["concepts"].append({"n": x["name"], "d": x["definition"], "dm": x["domain"], "sy": x["synonyms"], "v": int(x["verified"]),
+                                  "l": [[tidx[l["table"]], l["column"], l["role"], l["note"]] for l in x["links"] if l["table"] in tidx]})
+        c["rels"] = [[r["from"], r["rel"], r["to"], r["note"], int(r["verified"])] for r in ctx_dump["relations"]]
+        c["cols"] = [{"t": tidx[x["table"]], "c": x["column"], "m": x["meaning"], "u": x["unit"], "k": x["codes"], "nm": x["null_meaning"],
+                      "pii": int(x["pii"]), "f": x["typical_filters"], "ih": x["index_hint"], "v": int(x["verified"]), "s": safe_source(x["source"])}
+                     for x in ctx_dump["columns"] if x["table"] in tidx]
+        c["metrics"] = [{"n": x["name"], "d": x["definition"], "e": x["expression"], "g": x["grain"], "t": [tidx[t] for t in x["tables"] if t in tidx],
+                         "f": x["filters"], "cv": x["caveats"], "v": int(x["verified"])} for x in ctx_dump["metrics"]]
+        c["tn"] = [{"t": tidx[x["table"]], "g": x["grain"], "fr": x["freshness"], "sd": x["soft_delete"], "dc": x["date_column"],
+                    "ri": x["recommended_indexes"], "p": x["pitfalls"], "sm": x["summary"], "v": int(x["verified"]), "s": safe_source(x["source"])}
+                   for x in ctx_dump["tables"] if x["table"] in tidx]
+    prof = {}
+    for key, p in (graph.data.get("profiles") or {}).items():
+        tid, col = key.rsplit(".", 1)
+        if tid in tidx:
+            names = [c["name"] for c in graph.nodes[tid]["columns"]]
+            if col in names:
+                if p.get("pii"):  # PII: never embed raw values, whatever an older profile stored
+                    p = {k: v for k, v in p.items() if k not in ("top_values", "numeric", "dates", "length")}
+                prof[f"{tidx[tid]}:{names.index(col)}"] = p
+    c["p"] = prof
+    return {"v": 2, "s": schemas, "t": tables, "e": edges, "a": areas, "al": links, "mi": graph.data.get("mirrors") or {},
+            "c": c, "st": graph.data["stats"]}
+
+
+def _svg(name: str) -> str:
+    s = (STATIC / name).read_text(encoding="utf-8")
+    s = re.sub(r"<title>.*?</title>", "", s, flags=re.S)
+    return s.replace("#17181A", "var(--kg-text)").replace("#E2A317", "var(--kg-accent)")
+
+
+def _js(name: str) -> str:
+    return (STATIC / name).read_text(encoding="utf-8").replace("</script", "<\\/script")
+
+
+def build_html(graph: Graph, ctx_dump: dict | None = None, gzip_over: int = 300_000) -> str:
+    payload = json.dumps(compact(graph, ctx_dump), ensure_ascii=False, separators=(",", ":"))
+    if len(payload) > gzip_over:
+        data = "GZ:" + base64.b64encode(gzip.compress(payload.encode("utf-8"), 9)).decode("ascii")
+    else:
+        data = payload.replace("<", "\\u003c")  # JSON-safe; no "</script" or "<!--" can end or alter the block
+    lic = "\n\n".join((STATIC / f).read_text(encoding="utf-8") for f in ("DAGRE_LICENSE", "OFL-Inter.txt", "OFL-JetBrainsMono.txt"))
+    fav = (STATIC / "favicon.svg").read_text(encoding="utf-8").replace("#", "%23").replace('"', "'").replace("\n", "")
+    logo = _svg("keygraph-logo.svg").replace("<svg ", '<svg style="fill:var(--kg-text)" ', 1)
+    mark = _svg("keygraph-mark.svg").replace("<svg ", '<svg width="24" height="24" ', 1)
+    parts = {
+        "LICENSES": lic.replace("--", "- -"),
+        "FAVICON": fav,
+        "CSS": (STATIC / "keygraph.inline.css").read_text(encoding="utf-8").replace("</style", "<\\/style"),
+        "LOGOS": json.dumps({"logo": logo, "mark": mark}).replace("</", "<\\/"),
+        "DAGRE": _js("dagre-1.1.4.min.js"),
+        "THEME": _js("kg-theme.js"),
+        "DIAGRAM": _js("kg-diagram.js"),
+        "APP": _js("explorer.js"),
+        "DATA": data,
+    }
+    tpl = (STATIC / "explorer.html").read_text(encoding="utf-8")
+    # one pass over the template only, so text inside the data or the scripts is never treated as a marker
+    return re.sub(r"/\*__([A-Z]+)__\*/", lambda m: parts[m.group(1)], tpl)

keygraph/graph.py

@@ -0,0 +1,163 @@
+"""Graph assembly, persistence and table-name resolution."""
+
+from __future__ import annotations
+
+import difflib
+import json
+from pathlib import Path
+
+from .infer import Inferer, Options, declared_edges, redundant_indexes
+from .model import Schema, type_family
+from .overrides import Overrides, apply_overrides
+
+VERSION = 1
+
+
+class AmbiguousTable(ValueError):
+    def __init__(self, name: str, candidates: list[str]):
+        super().__init__(f"'{name}' is ambiguous; use schema.table: " + ", ".join(candidates))
+        self.candidates = candidates
+
+
+class UnknownTable(ValueError):
+    def __init__(self, name: str, suggestions: list[str]):
+        msg = f"unknown table '{name}'" + (f"; did you mean: {', '.join(suggestions)}" if suggestions else "")
+        super().__init__(msg)
+        self.suggestions = suggestions
+
+
+def edge_id(e: dict) -> str:
+    return f"{e['from']}({','.join(e['from_columns'])})->{e['to']}({','.join(e['to_columns'])})"
+
+
+def build_graph(schema: Schema, overrides: Overrides | None = None, options: Options | None = None) -> "Graph":
+    opts = options or Options()
+    if overrides:
+        overrides.configure(opts)
+    warnings: list[str] = []
+    inf = Inferer(schema, opts)
+    declared = declared_edges(schema, warnings)
+    inferred = inf.run()
+    taken = {(e["from"], tuple(e["from_columns"]), e["to"]) for e in declared}
+    edges = declared + [e for e in inferred if (e["from"], tuple(e["from_columns"]), e["to"]) not in taken]
+    nodes = []
+    for t in sorted(schema.tables.values(), key=lambda x: x.id):
+        nodes.append({
+            "id": t.id, "schema": t.schema, "table": t.name,
+            "info": t.info,
+            "columns": [{"name": c.name, "type": c.type, "family": type_family(c.type), "nullable": c.nullable,
+                         **({"key": c.key} if c.key else {}),
+                         **{k: v for k, v in (("default", c.default), ("extra", c.extra), ("charset", c.charset),
+                                              ("collation", c.collation), ("comment", c.comment), ("values", c.values),
+                                              ("generated", c.generated)) if v not in (None, "", [])}}
+                        for c in sorted(t.columns, key=lambda c: (c.position, c.name))],
+            "primary_key": t.primary_key,
+            "unique_keys": [{"name": ix.name, "columns": ix.columns, **({"approximate": True} if ix.approximate else {})}
+                            for ix in t.indexes if ix.unique and not ix.primary],
+            "indexes": [{"name": ix.name, "columns": ix.columns, "unique": ix.unique, "primary": ix.primary,
+                         **({"approximate": True} if ix.approximate else {})}
+                        for ix in sorted(t.indexes, key=lambda i: (not i.primary, i.name))],
+            "polymorphic": inf.polymorphic.get(t.id, []),
+            "unresolved_id_columns": sorted(inf.unresolved.get(t.id, [])),
+            "redundant_indexes": redundant_indexes(t),
+        })
+    data = {"version": VERSION, "nodes": nodes, "edges": edges, "warnings": warnings, "overrides": {}}
+    if overrides:
+        apply_overrides(data, overrides)
+    for e in data["edges"]:
+        e["id"] = edge_id(e)
+    data["edges"].sort(key=lambda e: e["id"])
+    data["warnings"] = sorted(set(data["warnings"]))
+    g = Graph(data)
+    g.refresh_stats()
+    return g
+
+
+class Graph:
+    def __init__(self, data: dict):
+        self.data = data
+        self.nodes: dict[str, dict] = {n["id"]: n for n in data["nodes"]}
+        self.edges: list[dict] = data["edges"]
+        self._by_name: dict[str, list[str]] = {}
+        for nid, n in self.nodes.items():
+            self._by_name.setdefault(n["table"].lower(), []).append(nid)
+        self._adj: dict[str, list[dict]] | None = None
+
+    @classmethod
+    def load(cls, path: str | Path) -> "Graph":
+        return cls(json.loads(Path(path).read_text(encoding="utf-8")))
+
+    def save(self, path: str | Path) -> None:
+        Path(path).parent.mkdir(parents=True, exist_ok=True)
+        Path(path).write_text(json.dumps(self.data, ensure_ascii=False, indent=1) + "\n", encoding="utf-8")
+
+    def refresh_stats(self) -> None:
+        by_kind: dict[str, int] = {}
+        bands = {"1.0": 0, "0.8-0.99": 0, "0.6-0.79": 0, "0.4-0.59": 0, "<0.4": 0}
+        for e in self.edges:
+            by_kind[e["kind"]] = by_kind.get(e["kind"], 0) + 1
+            c = e["confidence"]
+            bands["1.0" if c >= 1 else "0.8-0.99" if c >= .8 else "0.6-0.79" if c >= .6 else "0.4-0.59" if c >= .4 else "<0.4"] += 1
+        self.data["stats"] = {
+            "tables": len(self.nodes), "columns": sum(len(n["columns"]) for n in self.nodes.values()),
+            "schemas": len({n["schema"] for n in self.nodes.values()}),
+            "indexes": sum(len(n["indexes"]) for n in self.nodes.values()),
+            "edges": len(self.edges), "edges_by_kind": by_kind, "edges_by_confidence": bands,
+            "polymorphic_columns": sum(len(n["polymorphic"]) for n in self.nodes.values()),
+        }
+        self._adj = None
+
+    # --- name resolution -------------------------------------------------------
+    def resolve(self, name: str) -> str:
+        if name in self.nodes:
+            return name
+        low = name.lower()
+        for nid in self.nodes:
+            if nid.lower() == low:
+                return nid
+        if "." not in name:
+            hits = self._by_name.get(low, [])
+            if len(hits) == 1:
+                return hits[0]
+            if len(hits) > 1:
+                raise AmbiguousTable(name, sorted(hits))
+        sug = difflib.get_close_matches(name, list(self.nodes), n=4, cutoff=0.6)
+        sug += [h for k in difflib.get_close_matches(low, list(self._by_name), n=3, cutoff=0.7) for h in self._by_name[k]]
+        raise UnknownTable(name, sorted(set(sug)))
+
+    def split_column(self, ref: str) -> tuple[str, str | None]:
+        """Parse 'table', 'table.col' or 'schema.table.col' into (table_id, column or None)."""
+        try:
+            return self.resolve(ref), None
+        except (UnknownTable, AmbiguousTable) as first:
+            if "." not in ref:
+                raise
+            head, col = ref.rsplit(".", 1)
+            tid = self.resolve(head)
+            if col not in {c["name"] for c in self.nodes[tid]["columns"]}:
+                raise ValueError(f"table {tid} has no column '{col}'") from first
+            return tid, col
+
+    # --- adjacency -------------------------------------------------------------
+    def adjacency(self) -> dict[str, list[dict]]:
+        if self._adj is None:
+            adj: dict[str, list[dict]] = {n: [] for n in self.nodes}
+            for e in self.edges:
+                adj.setdefault(e["from"], []).append(oriented(e, False))
+                if e["to"] != e["from"]:
+                    adj.setdefault(e["to"], []).append(oriented(e, True))
+            self._adj = adj
+        return self._adj
+
+
+def flip_card(c: str) -> str:
+    return {"many-to-one": "one-to-many", "one-to-many": "many-to-one"}.get(c, c)
+
+
+def oriented(e: dict, reverse: bool) -> dict:
+    """View an edge from one end: near/far table and columns as walked."""
+    if not reverse:
+        return {"edge": e, "near": e["from"], "near_cols": e["from_columns"], "far": e["to"],
+                "far_cols": e["to_columns"], "cardinality": e["cardinality"], "reverse": False}
+    return {"edge": e, "near": e["to"], "near_cols": e["to_columns"], "far": e["from"],
+            "far_cols": e["from_columns"], "cardinality": flip_card(e["cardinality"]), "reverse": True}

keygraph/hubs.py

@@ -0,0 +1,48 @@
+"""Hub tables: tables so many others point to that drawing their edges would swamp every view."""
+
+from __future__ import annotations
+
+import re
+
+from keygraph.graph import Graph
+
+ROLE_PATTERNS = [r"(^|_)(created|updated|deleted|modified|approved|rejected|changed|edited)_by(_id)?$",
+                 r"^(owner|author|creator|editor|operator|admin)_id$", r"_by_id$", r"_by$"]
+
+
+def is_role_column(name: str, patterns=ROLE_PATTERNS) -> bool:
+    low = name.lower()
+    return any(re.search(p, low) for p in patterns)
+
+
+def _edge_is_role(e: dict, patterns) -> bool:
+    return any(is_role_column(c, patterns) for c in e["from_columns"])
+
+
+def detect_hubs(graph: Graph, percentile: float = 99, role_share: float = 0.6, include=(), exclude=(),
+                patterns=ROLE_PATTERNS) -> set[str]:
+    incoming: dict[str, int] = {}
+    role_in: dict[str, int] = {}
+    mirrors = {t for t, n in graph.nodes.items() if n.get("mirror_of")}  # a copy's edges would double the counts
+    for e in graph.edges:
+        if e["from"] == e["to"] or e["from"] in mirrors or e["to"] in mirrors:
+            continue
+        incoming[e["to"]] = incoming.get(e["to"], 0) + 1
+        if _edge_is_role(e, patterns):
+            role_in[e["to"]] = role_in.get(e["to"], 0) + 1
+    counts = sorted(incoming.values())
+    hubs: set[str] = set()
+    if counts:
+        cut = counts[min(len(counts) - 1, int(len(counts) * percentile / 100))]
+        hubs |= {t for t, n in incoming.items() if n >= max(cut, 5)}
+        hubs |= {t for t, n in incoming.items() if n >= 5 and role_in.get(t, 0) / n >= role_share}
+    hubs |= {graph.resolve(t) for t in include}
+    hubs -= {graph.resolve(t) for t in exclude}
+    return hubs
+
+
+def mark_hubs(graph: Graph, hubs: set[str], patterns=ROLE_PATTERNS) -> None:
+    for e in graph.edges:
+        e["role"] = _edge_is_role(e, patterns)
+        e["hub"] = e["to"] in hubs and e["from"] != e["to"]
+    graph.data["hubs"] = sorted(hubs)