keygraph 0.2.0__py3-none-any.whl

keygraph/adapters/mysql.py

@@ -0,0 +1,30 @@
+"""MySQL / MariaDB through information_schema. Works with any run(sql) callable: a DB-API or SQLAlchemy
+wrapper, or an external read-only command (see fetch.command_runner). Read-only: SELECTs on information_schema
+plus bounded sample SELECTs, all through assert_select."""
+
+from __future__ import annotations
+
+from ..model import Schema
+from .base import Source, guarded
+from .dump import load_dump
+from .fetch import fetch_mysql_all
+
+
+class MySQLSource(Source):
+    kind, dialect, quote = "mysql", "mysql", "`"
+
+    def __init__(self, run, schema_expr: str = "DATABASE()", schema_name: str | None = None, limit: int = 10000):
+        self.run, self.schema_expr, self.schema_name, self.limit = guarded(run), schema_expr, schema_name, limit
+
+    def schema(self) -> Schema:
+        d = fetch_mysql_all(self.run, self.schema_expr, self.limit)
+        if self.schema_name:
+            for rows in d.values():
+                for r in rows:
+                    r["TABLE_SCHEMA"] = self.schema_name
+        sch = load_dump(d["columns"], d["statistics"] or None, d["key_columns"] or None, tables=d["tables"])
+        for r in d["checks"]:
+            t = sch.tables.get(f"{r.get('TABLE_SCHEMA')}.{r.get('TABLE_NAME')}")
+            if t:
+                t.info.setdefault("checks", []).append({"name": r.get("CONSTRAINT_NAME"), "expr": r.get("CHECK_CLAUSE")})
+        return sch

keygraph/adapters/postgres.py

@@ -0,0 +1,68 @@
+"""PostgreSQL through information_schema and pg_catalog via a run(sql) callable. Read-only (single SELECTs)."""
+
+from __future__ import annotations
+
+from ..model import Column, ForeignKey, Index, Schema
+from .base import Source, dicts, guarded
+
+SYS = "('pg_catalog','information_schema','pg_toast')"
+COLS = ("SELECT c.table_schema, c.table_name, c.column_name, c.ordinal_position, CASE WHEN c.data_type='USER-DEFINED' THEN c.udt_name "
+        "WHEN c.data_type='character varying' THEN 'varchar('||COALESCE(c.character_maximum_length::text,'')||')' ELSE c.data_type END, "
+        "c.is_nullable, c.column_default, c.is_generated, c.generation_expression, "
+        "pg_catalog.col_description((quote_ident(c.table_schema)||'.'||quote_ident(c.table_name))::regclass::oid, c.ordinal_position) "
+        "FROM information_schema.columns c JOIN information_schema.tables t ON t.table_schema=c.table_schema AND t.table_name=c.table_name "
+        f"WHERE t.table_type='BASE TABLE' AND c.table_schema NOT IN {SYS} ORDER BY 1,2,4 LIMIT 200000")
+COLN = ["schema", "table", "column", "pos", "type", "nullable", "default", "generated", "genexpr", "comment"]
+IDX = ("SELECT n.nspname, c.relname, i.relname, ix.indisunique, ix.indisprimary, array_to_string(ARRAY(SELECT a.attname FROM "
+       "unnest(ix.indkey) WITH ORDINALITY k(attnum,ord) JOIN pg_attribute a ON a.attrelid=c.oid AND a.attnum=k.attnum ORDER BY k.ord),',') "
+       "FROM pg_index ix JOIN pg_class c ON c.oid=ix.indrelid JOIN pg_class i ON i.oid=ix.indexrelid JOIN pg_namespace n ON n.oid=c.relnamespace "
+       f"WHERE n.nspname NOT IN {SYS} LIMIT 200000")
+IDXN = ["schema", "table", "index", "unique", "primary", "cols"]
+FK = ("SELECT n.nspname, c.relname, con.conname, array_to_string(ARRAY(SELECT a.attname FROM unnest(con.conkey) WITH ORDINALITY k(attnum,ord) "
+      "JOIN pg_attribute a ON a.attrelid=con.conrelid AND a.attnum=k.attnum ORDER BY k.ord),','), rn.nspname, rc.relname, "
+      "array_to_string(ARRAY(SELECT a.attname FROM unnest(con.confkey) WITH ORDINALITY k(attnum,ord) JOIN pg_attribute a ON a.attrelid=con.confrelid "
+      "AND a.attnum=k.attnum ORDER BY k.ord),',') FROM pg_constraint con JOIN pg_class c ON c.oid=con.conrelid JOIN pg_namespace n ON n.oid=c.relnamespace "
+      f"JOIN pg_class rc ON rc.oid=con.confrelid JOIN pg_namespace rn ON rn.oid=rc.relnamespace WHERE con.contype='f' AND n.nspname NOT IN {SYS} LIMIT 100000")
+FKN = ["schema", "table", "name", "cols", "ref_schema", "ref_table", "ref_cols"]
+ENUMS = "SELECT t.typname, string_agg(e.enumlabel, ',' ORDER BY e.enumsortorder) FROM pg_type t JOIN pg_enum e ON e.enumtypid=t.oid GROUP BY t.typname LIMIT 10000"
+ROWS = ("SELECT n.nspname, c.relname, c.reltuples::bigint, pg_total_relation_size(c.oid), obj_description(c.oid) FROM pg_class c "
+        f"JOIN pg_namespace n ON n.oid=c.relnamespace WHERE c.relkind IN ('r','p') AND n.nspname NOT IN {SYS} LIMIT 200000")
+CHECKS = ("SELECT n.nspname, c.relname, con.conname, pg_get_constraintdef(con.oid) FROM pg_constraint con JOIN pg_class c ON c.oid=con.conrelid "
+          f"JOIN pg_namespace n ON n.oid=c.relnamespace WHERE con.contype='c' AND n.nspname NOT IN {SYS} LIMIT 100000")
+
+
+class PostgresSource(Source):
+    kind, dialect, quote = "postgres", "postgres", '"'
+
+    def __init__(self, run):
+        self.run = guarded(run)
+
+    def schema(self) -> Schema:
+        sch = Schema()
+        for r in dicts(self.run(COLS), COLN):
+            t = sch.get_or_create(r["schema"], r["table"])
+            gen = r["genexpr"] if str(r["generated"]).upper() == "ALWAYS" else ""
+            t.columns.append(Column(r["column"], str(r["type"]), str(r["nullable"]).upper() != "NO", int(r["pos"]),
+                                    default=None if r["default"] is None else str(r["default"]), comment=r["comment"] or "",
+                                    generated=gen or ""))
+        enums = {n: v.split(",") for n, v in (tuple(x.values()) if isinstance(x, dict) else x for x in self.run(ENUMS))}
+        for t in sch.tables.values():
+            for c in t.columns:
+                c.values = enums.get(c.type, [])
+        for r in dicts(self.run(IDX), IDXN):
+            t = sch.tables.get(f"{r['schema']}.{r['table']}")
+            if t:
+                t.indexes.append(Index(r["index"], r["cols"].split(","), bool(r["unique"]), bool(r["primary"])))
+        for r in dicts(self.run(FK), FKN):
+            t = sch.tables.get(f"{r['schema']}.{r['table']}")
+            if t:
+                t.foreign_keys.append(ForeignKey(r["name"], r["cols"].split(","), r["ref_schema"], r["ref_table"], r["ref_cols"].split(",")))
+        for r in dicts(self.run(ROWS), ["schema", "table", "rows", "bytes", "comment"]):
+            t = sch.tables.get(f"{r['schema']}.{r['table']}")
+            if t:
+                t.info.update({k: v for k, v in (("rows_estimate", r["rows"]), ("data_bytes", r["bytes"]), ("comment", r["comment"])) if v not in (None, "")})
+        for r in dicts(self.run(CHECKS), ["schema", "table", "name", "expr"]):
+            t = sch.tables.get(f"{r['schema']}.{r['table']}")
+            if t:
+                t.info.setdefault("checks", []).append({"name": r["name"], "expr": r["expr"]})
+        return sch

keygraph/adapters/sqlite.py

@@ -0,0 +1,24 @@
+"""SQLite (and any other SQLAlchemy dialect) through the inspector. Metadata only; samples are bounded SELECTs."""
+
+from __future__ import annotations
+
+from ..probe import SelectOnlyRunner
+from .base import Source
+from .live import scan_url
+
+
+class SQLAlchemySource(Source):
+    kind = "sqlalchemy"
+
+    def __init__(self, url: str, schemas=None, all_schemas: bool = False):
+        self.url, self.schemas, self.all_schemas = url, schemas, all_schemas
+        self.dialect = "mysql" if url.startswith("mysql") else "postgres" if url.startswith("postgres") else "sqlite"
+        self.quote = "`" if self.dialect == "mysql" else '"'
+        self._runner = SelectOnlyRunner.from_url(url)
+        self.run = self._runner.run
+
+    def schema(self):
+        return scan_url(self.url, self.schemas, self.all_schemas)
+
+
+SQLiteSource = SQLAlchemySource

keygraph/adapters/sqlserver.py

@@ -0,0 +1,65 @@
+"""SQL Server through INFORMATION_SCHEMA and sys.* via a run(sql) callable. Read-only (single SELECTs).
+Samples use SELECT TOP n."""
+
+from __future__ import annotations
+
+from ..model import Column, ForeignKey, Index, Schema
+from .base import Source, dicts, guarded
+
+COLS = ("SELECT TOP 200000 c.TABLE_SCHEMA, c.TABLE_NAME, c.COLUMN_NAME, c.ORDINAL_POSITION, c.DATA_TYPE + CASE WHEN c.CHARACTER_MAXIMUM_LENGTH IS NOT NULL "
+        "THEN '(' + CAST(c.CHARACTER_MAXIMUM_LENGTH AS varchar(10)) + ')' ELSE '' END, c.IS_NULLABLE, c.COLUMN_DEFAULT, c.COLLATION_NAME, c.CHARACTER_SET_NAME "
+        "FROM INFORMATION_SCHEMA.COLUMNS c JOIN INFORMATION_SCHEMA.TABLES t ON t.TABLE_SCHEMA=c.TABLE_SCHEMA AND t.TABLE_NAME=c.TABLE_NAME "
+        "WHERE t.TABLE_TYPE='BASE TABLE' ORDER BY 1,2,4")
+COLN = ["schema", "table", "column", "pos", "type", "nullable", "default", "collation", "charset"]
+IDX = ("SELECT TOP 200000 s.name, t.name, i.name, i.is_unique, i.is_primary_key, c.name, ic.key_ordinal FROM sys.indexes i "
+       "JOIN sys.tables t ON t.object_id=i.object_id JOIN sys.schemas s ON s.schema_id=t.schema_id "
+       "JOIN sys.index_columns ic ON ic.object_id=i.object_id AND ic.index_id=i.index_id AND ic.is_included_column=0 "
+       "JOIN sys.columns c ON c.object_id=ic.object_id AND c.column_id=ic.column_id WHERE i.name IS NOT NULL ORDER BY 1,2,3,7")
+IDXN = ["schema", "table", "index", "unique", "primary", "col", "ord"]
+FK = ("SELECT TOP 100000 s.name, t.name, fk.name, c.name, rs.name, rt.name, rc.name, fkc.constraint_column_id FROM sys.foreign_key_columns fkc "
+      "JOIN sys.foreign_keys fk ON fk.object_id=fkc.constraint_object_id JOIN sys.tables t ON t.object_id=fkc.parent_object_id "
+      "JOIN sys.schemas s ON s.schema_id=t.schema_id JOIN sys.columns c ON c.object_id=fkc.parent_object_id AND c.column_id=fkc.parent_column_id "
+      "JOIN sys.tables rt ON rt.object_id=fkc.referenced_object_id JOIN sys.schemas rs ON rs.schema_id=rt.schema_id "
+      "JOIN sys.columns rc ON rc.object_id=fkc.referenced_object_id AND rc.column_id=fkc.referenced_column_id ORDER BY 1,2,3,8")
+FKN = ["schema", "table", "name", "col", "ref_schema", "ref_table", "ref_col", "ord"]
+ROWS = ("SELECT TOP 200000 s.name, t.name, SUM(p.rows) FROM sys.tables t JOIN sys.schemas s ON s.schema_id=t.schema_id "
+        "JOIN sys.partitions p ON p.object_id=t.object_id AND p.index_id IN (0,1) GROUP BY s.name, t.name")
+CHECKS = "SELECT TOP 100000 s.name, t.name, k.name, k.definition FROM sys.check_constraints k JOIN sys.tables t ON t.object_id=k.parent_object_id JOIN sys.schemas s ON s.schema_id=t.schema_id"
+
+
+class SQLServerSource(Source):
+    kind, dialect, quote = "sqlserver", "sqlserver", '"'
+
+    def __init__(self, run):
+        self.run = guarded(run)
+
+    def schema(self) -> Schema:
+        sch = Schema()
+        for r in dicts(self.run(COLS), COLN):
+            t = sch.get_or_create(r["schema"], r["table"])
+            t.columns.append(Column(r["column"], str(r["type"]), str(r["nullable"]).upper() != "NO", int(r["pos"]),
+                                    default=None if r["default"] is None else str(r["default"]),
+                                    charset=r["charset"] or "", collation=r["collation"] or ""))
+        groups: dict = {}
+        for r in dicts(self.run(IDX), IDXN):
+            groups.setdefault((r["schema"], r["table"], r["index"]), {"u": bool(r["unique"]), "p": bool(r["primary"]), "c": []})["c"].append(r["col"])
+        for (s, tn, name), g in groups.items():
+            if f"{s}.{tn}" in sch.tables:
+                sch.tables[f"{s}.{tn}"].indexes.append(Index(name, g["c"], g["u"], g["p"]))
+        fks: dict = {}
+        for r in dicts(self.run(FK), FKN):
+            g = fks.setdefault((r["schema"], r["table"], r["name"], r["ref_schema"], r["ref_table"]), ([], []))
+            g[0].append(r["col"])
+            g[1].append(r["ref_col"])
+        for (s, tn, name, rs, rt), (cols, rcols) in fks.items():
+            if f"{s}.{tn}" in sch.tables:
+                sch.tables[f"{s}.{tn}"].foreign_keys.append(ForeignKey(name, cols, rs, rt, rcols))
+        for r in dicts(self.run(ROWS), ["schema", "table", "rows"]):
+            t = sch.tables.get(f"{r['schema']}.{r['table']}")
+            if t and r["rows"] is not None:
+                t.info["rows_estimate"] = int(r["rows"])
+        for r in dicts(self.run(CHECKS), ["schema", "table", "name", "expr"]):
+            t = sch.tables.get(f"{r['schema']}.{r['table']}")
+            if t:
+                t.info.setdefault("checks", []).append({"name": r["name"], "expr": r["expr"]})
+        return sch

keygraph/areas.py

@@ -0,0 +1,225 @@
+"""Business areas: deterministic clusters of tables, ignoring hub and role edges."""
+
+from __future__ import annotations
+
+import hashlib
+import random
+from collections import Counter
+
+from keygraph.graph import Graph
+
+
+def _weights(graph: Graph, skip: set[str] = frozenset()) -> dict[str, dict[str, float]]:
+    w: dict[str, dict[str, float]] = {t: {} for t in graph.nodes if t not in skip}
+    for e in graph.edges:
+        if e.get("hub") or e.get("role") or e["from"] == e["to"] or e["from"] in skip or e["to"] in skip:
+            continue
+        a, b, c = e["from"], e["to"], float(e["confidence"])
+        w[a][b] = w[a].get(b, 0) + c
+        w[b][a] = w[b].get(a, 0) + c
+    return w
+
+
+def _propagate(w, seed: int, rounds: int) -> dict[str, str]:
+    rng = random.Random(seed)
+    label = {n: n for n in w}
+    order = sorted(w)
+    for _ in range(rounds):
+        rng.shuffle(order)
+        changed = 0
+        for n in order:
+            score: dict[str, float] = {}
+            for m, x in w[n].items():
+                score[label[m]] = score.get(label[m], 0) + x
+            if not score:
+                continue
+            best = max(sorted(score), key=lambda l: score[l])
+            if best != label[n]:
+                label[n], changed = best, changed + 1
+        if not changed:
+            break
+    return label
+
+
+def _stem(word: str) -> str:
+    """Drop a plural 's', but keep words such as status, address and analysis whole."""
+    return word if word.endswith(("ss", "us", "is")) else word.removesuffix("s")
+
+
+PREFIX_SHARE = 0.6
+
+
+def _name(tables: list[str], graph: Graph) -> str:
+    """The shared table-name prefix when it covers at least PREFIX_SHARE of the area ("Order" for orders,
+    order_items, order_notes), else the most connected table that is not a hub (tables arrive ordered by
+    connections). A minority prefix or a hub's name would mislead."""
+    names = [graph.nodes[t]["table"] for t in tables]
+    heads = Counter(_stem(n.split("_")[0]) for n in names)
+    head, n = heads.most_common(1)[0]
+    if n >= 2 and n / len(names) >= PREFIX_SHARE:
+        return head.capitalize()
+    hubs = set(graph.data.get("hubs") or [])
+    return next((graph.nodes[t]["table"] for t in tables if t not in hubs), names[0])
+
+
+def _id(top: list[str]) -> str:
+    return "a_" + hashlib.sha1("|".join(sorted(top)).encode()).hexdigest()[:8]
+
+
+def cluster_areas(graph: Graph, min_size: int = 3, seed: int = 7, rounds: int = 30) -> list[dict]:
+    """Areas over the non-mirror tables, plus one area per mirror schema (flagged `mirror`), and one
+    "Unconnected (<schema>)" area per schema for tables with no remaining edges (flagged `unconnected`)."""
+    mirrors = {t for t, n in graph.nodes.items() if n.get("mirror_of")}
+    w = _weights(graph, mirrors)
+    degree = {t: len(v) for t, v in w.items()}
+    label = _propagate(w, seed, rounds)
+    groups: dict[str, set[str]] = {}
+    for t, l in label.items():
+        groups.setdefault(l, set()).add(t)
+    merged = True
+    while merged:  # repeat passes: a target that grew may now be small enough to merge again
+        merged = False
+        for l in sorted(groups, key=lambda l: (len(groups[l]), l)):  # merge small groups into their heaviest neighbour
+            g = groups.get(l)
+            if g is None or len(g) >= min_size:
+                continue
+            pull: dict[str, float] = {}
+            for t in sorted(g):
+                for m, x in sorted(w[t].items()):
+                    if label[m] != l:
+                        pull[label[m]] = pull.get(label[m], 0) + x
+            if pull:
+                target = max(sorted(pull), key=lambda k: pull[k])
+                groups[target] |= g
+                for t in sorted(g):
+                    label[t] = target
+                del groups[l]
+                merged = True
+    areas, loose = [], {}
+    for g in groups.values():
+        if len(g) < min_size and all(not w[t] for t in g):
+            for t in g:
+                loose.setdefault(graph.nodes[t]["schema"], []).append(t)
+            continue
+        tables = sorted(g, key=lambda t: (-degree[t], t))
+        areas.append({"id": _id(tables[:3]), "name": _name(tables, graph), "tables": tables, "top": tables[:3],
+                      "size": len(tables), "auto": True})
+    for schema, ts in sorted(loose.items()):
+        ts = sorted(ts)
+        areas.append({"id": _id([f"unconnected:{schema}"]), "name": f"Unconnected ({schema})", "tables": ts,
+                      "top": ts[:3], "size": len(ts), "auto": True, "unconnected": True})
+    seen: Counter = Counter(a["name"] for a in areas)
+    for a in areas:
+        if seen[a["name"]] > 1 and not a.get("unconnected"):
+            top = graph.nodes[a["top"][0]]
+            a["name"] = (f"{a['name']} ({top['schema']})" if a["name"] == top["table"]
+                         else f"{a['name']} ({top['schema']}: {top['table']})")
+    by_schema: dict[str, list[str]] = {}
+    for t in mirrors:
+        by_schema.setdefault(graph.nodes[t]["schema"], []).append(t)
+    deg: Counter = Counter()
+    for e in graph.edges:  # one pass, not one per table
+        deg[e["from"]] += 1
+        if e["to"] != e["from"]:
+            deg[e["to"]] += 1
+    for schema, ts in sorted(by_schema.items()):
+        ts = sorted(ts, key=lambda t: (-deg[t], t))
+        of = graph.nodes[ts[0]]["mirror_of"]
+        areas.append({"id": _id([f"mirror:{schema}"]), "name": f"Mirror of {of} ({schema})", "tables": ts, "top": ts[:3],
+                      "size": len(ts), "auto": True, "mirror": schema, "mirror_of": of})
+    return sorted(areas, key=lambda a: (-a["size"], a["name"]))
+
+
+def area_of(areas: list[dict]) -> dict[str, str]:
+    return {t: a["id"] for a in areas for t in a["tables"]}
+
+
+def area_links(graph: Graph, areas: list[dict]) -> list[dict]:
+    of = area_of(areas)
+    c: Counter = Counter()
+    mirrors = {t for t, n in graph.nodes.items() if n.get("mirror_of")}
+    for e in graph.edges:
+        if e.get("hub") or e.get("role") or e["from"] in mirrors or e["to"] in mirrors:
+            continue
+        a, b = of.get(e["from"]), of.get(e["to"])
+        if a and b and a != b:
+            c[tuple(sorted((a, b)))] += 1
+    return [{"a": a, "b": b, "edges": n} for (a, b), n in c.most_common()]
+
+
+def apply_curation(graph: Graph, areas: list[dict], rules: list[dict]) -> tuple[list[dict], list[str]]:
+    areas = [dict(a, tables=list(a["tables"])) for a in areas]
+    warns: list[str] = []
+
+    def find(key: str):
+        key = str(key)
+        return next((a for a in areas if a["id"] == key or a["name"].lower() == key.lower()
+                     or key in a["tables"]), None)
+
+    def move(t: str, dest: dict):
+        for a in areas:
+            if t in a["tables"]:
+                a["tables"].remove(t)
+        dest["tables"].append(t)
+
+    for r in rules:
+        if not isinstance(r, dict) or not (r.get("name") or r.get("match") or r.get("merge")):
+            warns.append(f"area rule {r!r} ignored: it needs a name, a match or a merge")
+            continue
+        if r.get("merge"):
+            hits = [find(k) for k in r["merge"]]
+            if any(h is None for h in hits):
+                warns.append(f"merge {r['merge']}: an area was not found")
+                continue
+            dest = hits[0]
+            for h in hits[1:]:
+                if h is not dest:
+                    dest["tables"] += h["tables"]
+                    h["tables"] = []
+            dest["auto"] = False
+            continue
+        dest = next((find(k) for k in r.get("match", []) if find(k)), None)
+        if dest is None:
+            if r.get("match"):
+                warns.append(f"area rule '{r.get('name')}': no area matches {r['match']}")
+                continue
+            if not r.get("name"):
+                warns.append(f"area rule {r!r} ignored: it needs a name, a match or a merge")
+                continue
+            dest = {"id": _id([f"curated:{r['name']}"]), "name": r["name"], "tables": [], "auto": False}
+            areas.append(dest)
+        dest["name"], dest["auto"] = r.get("name", dest["name"]), False
+        for t in r.get("pin", []):
+            try:
+                move(graph.resolve(t), dest)
+            except ValueError:  # UnknownTable and AmbiguousTable both subclass ValueError
+                warns.append(f"area '{dest['name']}': unknown table {t}")
+        for t in r.get("exclude", []):
+            try:
+                tid = graph.resolve(t)
+            except ValueError:
+                warns.append(f"area '{dest['name']}': unknown table {t}")
+                continue
+            if tid in dest["tables"]:
+                dest["tables"].remove(tid)
+                schema = graph.nodes[tid]["schema"]
+                loose = find(f"Unconnected ({schema})") or {"id": _id([f"unconnected:{schema}"]),
+                                                            "name": f"Unconnected ({schema})", "tables": [], "auto": True,
+                                                            "unconnected": True}
+                if loose not in areas:
+                    areas.append(loose)
+                loose["tables"].append(tid)
+    deg: Counter = Counter()
+    for e in graph.edges:  # one pass over the edges
+        if not e.get("hub"):
+            deg[e["from"]] += 1
+            if e["to"] != e["from"]:
+                deg[e["to"]] += 1
+    out = []
+    for a in areas:
+        if not a["tables"]:
+            continue
+        a["tables"] = sorted(set(a["tables"]), key=lambda t: (-deg[t], t))
+        a["top"], a["size"] = a["tables"][:3], len(a["tables"])
+        out.append(a)
+    return sorted(out, key=lambda a: (-a["size"], a["name"])), warns