keygraph 0.2.0__py3-none-any.whl

keygraph/__init__.py

@@ -0,0 +1,8 @@
+"""keygraph: map keys and relations between database tables."""
+
+from .graph import Graph, build_graph
+from .model import Column, ForeignKey, Index, Schema, Table
+from .infer import Options
+
+__all__ = ["Graph", "build_graph", "Column", "ForeignKey", "Index", "Schema", "Table", "Options"]
+__version__ = "0.2.0"

keygraph/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())

keygraph/adapters/__init__.py

@@ -0,0 +1,33 @@
+"""Pluggable sources: mysql, postgres, sqlite, sqlserver, clickhouse, metabase, mongodb, plus the dump and command adapters.
+
+    from keygraph.adapters import open_source
+    src = open_source("clickhouse", run=my_select_runner)
+    schema = src.schema()
+"""
+
+from __future__ import annotations
+
+
+def open_source(kind: str, url: str | None = None, run=None, **kw):
+    kind = kind.lower()
+    if kind in ("sqlite", "sqlalchemy"):
+        from .sqlite import SQLAlchemySource
+        return SQLAlchemySource(url, **kw)
+    if run is None and url:
+        from ..probe import SelectOnlyRunner
+        r = SelectOnlyRunner.from_url(url)
+        run = r.run
+    table = {"mysql": ("mysql", "MySQLSource"), "mariadb": ("mysql", "MySQLSource"), "postgres": ("postgres", "PostgresSource"),
+             "postgresql": ("postgres", "PostgresSource"), "sqlserver": ("sqlserver", "SQLServerSource"),
+             "mssql": ("sqlserver", "SQLServerSource"), "clickhouse": ("clickhouse", "ClickHouseSource")}
+    if kind in table:
+        import importlib
+        mod, cls = table[kind]
+        return getattr(importlib.import_module(f"keygraph.adapters.{mod}"), cls)(run, **kw)
+    if kind == "metabase":
+        from .metabase import MetabaseSource
+        return MetabaseSource(**kw)
+    if kind in ("mongo", "mongodb"):
+        from .mongo import MongoSource
+        return MongoSource(**kw)
+    raise ValueError(f"unknown source kind '{kind}'")

keygraph/adapters/base.py

@@ -0,0 +1,57 @@
+"""Pluggable source interface.
+
+A source turns one kind of system into a normalised Schema (metadata) and can sample rows for profiling.
+Every source is read-only by construction: SQL goes through `guard` (a single SELECT, no write keywords),
+HTTP sources only GET metadata and POST native SELECTs, Mongo sources only call find/list operations.
+"""
+
+from __future__ import annotations
+
+from typing import Callable
+
+from ..model import Schema
+from ..probe import assert_select
+
+
+def guarded(run: Callable[[str], list]) -> Callable[[str], list]:
+    def safe(sql: str) -> list:
+        return run(assert_select(sql))
+    return safe
+
+
+def dicts(rows: list, names: list[str]) -> list[dict]:
+    """Rows may be dicts (command tools) or tuples (DB-API / SQLAlchemy); return dicts keyed by `names`."""
+    out = []
+    for r in rows:
+        if isinstance(r, dict):
+            low = {k.lower(): v for k, v in r.items()}
+            out.append({n: r[n] if n in r else low.get(n.lower()) for n in names})
+        else:
+            out.append(dict(zip(names, r)))
+    return out
+
+
+class Source:
+    kind = "generic"
+    dialect = "generic"
+    quote = '"'
+    read_only = "all SQL passes keygraph.probe.assert_select (single SELECT only)"
+
+    def schema(self) -> Schema:  # pragma: no cover - interface
+        raise NotImplementedError
+
+    bare = False  # use bare table names (graph schema is a label, not a real schema)
+
+    def table_ref(self, table_id: str) -> str:
+        sch, name = table_id.split(".", 1)
+        if self.bare:
+            sch = ""
+        q = self.quote
+        return f"{q}{sch}{q}.{q}{name}{q}" if sch not in ("", "main", "default") else f"{q}{name}{q}"
+
+    def sample(self, table_id: str, cols: list[str], n: int, strategy: str = "limit", pk: str | None = None):
+        """Return (cols, rows) for a bounded sample. SQL sources override `run`."""
+        from ..profile import build_sample_sql
+        sql = build_sample_sql(self.dialect, self.table_ref(table_id), cols, n, strategy, pk, self.quote)
+        rows = self.run(sql)  # type: ignore[attr-defined]
+        return cols, [tuple(r[c] for c in cols) if isinstance(r, dict) else tuple(r) for r in rows]

keygraph/adapters/clickhouse.py

@@ -0,0 +1,67 @@
+"""ClickHouse through system.tables / system.columns / system.data_skipping_indices via a run(sql) callable.
+
+"Keys": the primary key columns become the PRIMARY index (ClickHouse primary keys are sorting prefixes, not unique),
+the ORDER BY key becomes an index named 'ORDER BY', data-skipping indices become indexes, and the partition key is
+kept in table info. Enum8/Enum16 members and LowCardinality/Nullable wrappers are understood. Read-only (SELECT only)."""
+
+from __future__ import annotations
+
+import re
+
+from ..model import Column, Index, Schema, parse_enum_members
+from .base import Source, dicts, guarded
+
+TABLES = ("SELECT database, name, engine, total_rows, total_bytes, partition_key, sorting_key, primary_key, comment FROM system.tables "
+          "WHERE database IN ({dbs}) AND NOT is_temporary AND engine NOT LIKE '%View' LIMIT 100000")
+TABN = ["database", "name", "engine", "total_rows", "total_bytes", "partition_key", "sorting_key", "primary_key", "comment"]
+COLUMNS = ("SELECT database, table, name, position, type, default_kind, default_expression, comment FROM system.columns "
+           "WHERE database IN ({dbs}) ORDER BY database, table, position LIMIT 500000")
+COLN = ["database", "table", "name", "position", "type", "default_kind", "default_expression", "comment"]
+SKIP = "SELECT database, table, name, type, expr, granularity FROM system.data_skipping_indices WHERE database IN ({dbs}) LIMIT 100000"
+SKIPN = ["database", "table", "name", "type", "expr", "granularity"]
+
+
+def key_columns(expr: str | None, known: set[str]) -> list[str]:
+    """Columns named in a key expression such as 'tuple(a, toDate(b))' or 'a, b' (functions are looked through)."""
+    return [t for t in dict.fromkeys(re.findall(r"[A-Za-z_][A-Za-z0-9_]*", expr or "")) if t in known]
+
+
+class ClickHouseSource(Source):
+    kind, dialect, quote = "clickhouse", "clickhouse", '"'
+
+    def __init__(self, run, databases: list[str] | None = None):
+        self.run = guarded(run)
+        self.dbs = ", ".join("'" + d.replace("'", "") + "'" for d in databases) if databases else "currentDatabase()"
+
+    def schema(self) -> Schema:
+        sch = Schema()
+        info = {(r["database"], r["name"]): r for r in dicts(self.run(TABLES.format(dbs=self.dbs)), TABN)}
+        for r in dicts(self.run(COLUMNS.format(dbs=self.dbs)), COLN):
+            if (r["database"], r["table"]) not in info:
+                continue
+            t = sch.get_or_create(r["database"], r["table"])
+            ty = str(r["type"])
+            t.columns.append(Column(r["name"], ty, ty.startswith("Nullable(") or "Nullable(" in ty, int(r["position"]),
+                                    default=r["default_expression"] or None, extra=r["default_kind"] or "", comment=r["comment"] or "",
+                                    values=parse_enum_members(ty), generated=r["default_expression"] if r["default_kind"] in ("MATERIALIZED", "ALIAS") else ""))
+        for (db, name), r in info.items():
+            t = sch.tables.get(f"{db}.{name}")
+            if not t:
+                continue
+            known = {c.name for c in t.columns}
+            pk = key_columns(r["primary_key"], known)
+            if pk:
+                t.indexes.append(Index("PRIMARY", pk, False, True))
+            sk = key_columns(r["sorting_key"], known)
+            if sk and sk != pk:
+                t.indexes.append(Index("ORDER BY", sk))
+            t.info.update({k: v for k, v in (("engine", r["engine"]), ("rows_estimate", r["total_rows"]), ("data_bytes", r["total_bytes"]),
+                                             ("partition_key", r["partition_key"]), ("sorting_key", r["sorting_key"]), ("comment", r["comment"]))
+                           if v not in (None, "")})
+        for r in dicts(self.run(SKIP.format(dbs=self.dbs)), SKIPN):
+            t = sch.tables.get(f"{r['database']}.{r['table']}")
+            if t:
+                cols = key_columns(r["expr"], {c.name for c in t.columns})
+                if cols:
+                    t.indexes.append(Index(f"{r['name']} ({r['type']})", cols))
+        return sch

keygraph/adapters/dump.py

@@ -0,0 +1,169 @@
+"""Load information_schema-style dumps (JSON, JSONL, CSV) with configurable field mapping."""
+
+from __future__ import annotations
+
+import csv
+import json
+from pathlib import Path
+from typing import Callable, Iterable
+
+from ..model import Column, ForeignKey, Index, Schema, parse_enum_members
+
+# Canonical field names are the MySQL information_schema ones. A mapping renames YOUR fields to these:
+# {"schema": "database", "table": "table", "column": "column", "raw": "raw"} style is expressed as
+# {"TABLE_SCHEMA": "database", "TABLE_NAME": "table", "COLUMN_NAME": "column"}: canonical -> source field.
+# A field mapped to "a.b" reads the nested key b of key a. "raw" flattening: set "_flatten": ["raw"].
+
+
+def read_rows(path: str | Path) -> list[dict]:
+    p = Path(path)
+    suffix = p.suffix.lower()
+    if suffix == ".csv":
+        with p.open(newline="", encoding="utf-8-sig") as fh:
+            return list(csv.DictReader(fh))
+    text = p.read_text(encoding="utf-8")
+    if suffix in (".jsonl", ".ndjson"):
+        return [json.loads(line) for line in text.splitlines() if line.strip()]
+    data = json.loads(text)
+    if isinstance(data, dict):
+        for key in ("rows", "items", "data", "results"):
+            if isinstance(data.get(key), list):
+                return data[key]
+        raise ValueError(f"{p}: JSON object without a rows/items/data/results list")
+    return data
+
+
+def _get(row: dict, key: str, mapping: dict, default=None):
+    src = mapping.get(key, key)
+    for cand in (src, src.lower(), src.upper()):
+        if "." in cand:
+            cur = row
+            for part in cand.split("."):
+                cur = cur.get(part) if isinstance(cur, dict) else None
+            if cur is not None:
+                return cur
+        elif cand in row and row[cand] is not None:
+            return row[cand]
+    return default
+
+
+def _flat(row: dict, flatten: Iterable[str]) -> dict:
+    out = dict(row)
+    for k in flatten:
+        if isinstance(out.get(k), dict):
+            out.update(out[k])
+    return out
+
+
+def _bool_yes(v) -> bool:
+    return str(v).strip().lower() in ("yes", "y", "true", "1", "t")
+
+
+def load_dump(columns: str | Path | list | None = None, statistics: str | Path | list | None = None,
+              key_columns: str | Path | list | None = None, default_schema: str = "main",
+              columns_map: dict | None = None, statistics_map: dict | None = None,
+              kcu_map: dict | None = None, row_filter: Callable[[dict], bool] | None = None,
+              schema_map: Callable[[dict], str] | None = None, tables: str | Path | list | None = None,
+              tables_map: dict | None = None) -> Schema:
+    """Build a Schema. Each argument is a path or an already-loaded list of row dicts.
+
+    Missing STATISTICS: indexes fall back to COLUMN_KEY (PRI/UNI/MUL), marked approximate.
+    Missing KEY_COLUMN_USAGE: no declared foreign keys (everything is inferred).
+    """
+    cm, sm, km = columns_map or {}, statistics_map or {}, kcu_map or {}
+    sch = Schema()
+
+    def rows_of(src, mp):
+        if src is None:
+            return None
+        rows = src if isinstance(src, list) else read_rows(src)
+        return [_flat(r, mp.get("_flatten", [])) for r in rows]
+
+    def tschema(row, mp):
+        if schema_map:
+            return schema_map(row)
+        v = _get(row, "TABLE_SCHEMA", mp)
+        return str(v) if v not in (None, "") else default_schema
+
+    col_rows = rows_of(columns, cm) or []
+    for r in col_rows:
+        if row_filter and not row_filter(r):
+            continue
+        name = _get(r, "TABLE_NAME", cm)
+        cname = _get(r, "COLUMN_NAME", cm)
+        if not name or not cname:
+            continue
+        t = sch.get_or_create(tschema(r, cm), str(name))
+        ctype = _get(r, "COLUMN_TYPE", cm) or _get(r, "DATA_TYPE", cm) or ""
+        t.columns.append(Column(str(cname), str(ctype), not (str(_get(r, "IS_NULLABLE", cm, "YES")).upper() == "NO"),
+                                int(_get(r, "ORDINAL_POSITION", cm, len(t.columns) + 1) or 0),
+                                str(_get(r, "COLUMN_KEY", cm, "") or "")))
+        c = t.columns[-1]
+        dv = _get(r, "COLUMN_DEFAULT", cm)
+        c.default = None if dv is None else str(dv)
+        c.extra = str(_get(r, "EXTRA", cm, "") or "")
+        c.charset = str(_get(r, "CHARACTER_SET_NAME", cm, "") or "")
+        c.collation = str(_get(r, "COLLATION_NAME", cm, "") or "")
+        c.comment = str(_get(r, "COLUMN_COMMENT", cm, "") or "")
+        c.generated = str(_get(r, "GENERATION_EXPRESSION", cm, "") or "")
+        c.values = parse_enum_members(str(ctype))
+    stat_rows = rows_of(statistics, sm)
+    if stat_rows is not None:
+        groups: dict[tuple, dict] = {}
+        for r in stat_rows:
+            key = (tschema(r, sm), str(_get(r, "TABLE_NAME", sm)), str(_get(r, "INDEX_NAME", sm)))
+            g = groups.setdefault(key, {"unique": str(_get(r, "NON_UNIQUE", sm, 1)) in ("0", "False", "false"), "cols": {}})
+            col = _get(r, "COLUMN_NAME", sm)
+            if col is not None:
+                g["cols"][int(_get(r, "SEQ_IN_INDEX", sm, len(g["cols"]) + 1))] = str(col)
+        for (s, tn, ixn), g in sorted(groups.items()):
+            tid = f"{s}.{tn}"
+            if tid not in sch.tables:
+                continue
+            cols = [g["cols"][k] for k in sorted(g["cols"])]
+            sch.tables[tid].indexes.append(Index(ixn, cols, g["unique"], ixn.upper() == "PRIMARY"))
+    else:
+        for t in sch.tables.values():
+            cs = sorted(t.columns, key=lambda c: c.position)
+            pk = [c.name for c in cs if c.key == "PRI"]
+            if pk:
+                t.indexes.append(Index("PRIMARY", pk, True, True))
+            for c in cs:
+                if c.key == "UNI":
+                    t.indexes.append(Index(f"(unique:{c.name})", [c.name], True, False, True))
+                elif c.key == "MUL":
+                    t.indexes.append(Index(f"(index:{c.name})", [c.name], False, False, True))
+    kcu = rows_of(key_columns, km)
+    if kcu is not None:
+        fks: dict[tuple, dict] = {}
+        for r in kcu:
+            rt = _get(r, "REFERENCED_TABLE_NAME", km)
+            tn = _get(r, "TABLE_NAME", km)
+            if not rt:
+                if stat_rows is None and str(_get(r, "CONSTRAINT_NAME", km, "")).upper() == "PRIMARY" and tn:
+                    t = sch.tables.get(f"{tschema(r, km)}.{tn}")
+                    if t and not t.primary_key:
+                        t.indexes.append(Index("PRIMARY", [str(_get(r, "COLUMN_NAME", km))], True, True))
+                continue
+            s = tschema(r, km)
+            g = fks.setdefault((s, str(tn), str(_get(r, "CONSTRAINT_NAME", km, "")),
+                                str(_get(r, "REFERENCED_TABLE_SCHEMA", km) or s), str(rt)), {})
+            g[int(_get(r, "ORDINAL_POSITION", km, len(g) + 1))] = (str(_get(r, "COLUMN_NAME", km)),
+                                                                   str(_get(r, "REFERENCED_COLUMN_NAME", km)))
+        for (s, tn, cn, rs, rt), g in sorted(fks.items()):
+            t = sch.tables.get(f"{s}.{tn}")
+            if t is not None:
+                pairs = [g[k] for k in sorted(g)]
+                t.foreign_keys.append(ForeignKey(cn, [p[0] for p in pairs], rs, rt, [p[1] for p in pairs]))
+    trows = rows_of(tables, tables_map or {})
+    for r in trows or []:
+        t = sch.tables.get(f"{tschema(r, tables_map or {})}.{_get(r, 'TABLE_NAME', tables_map or {})}")
+        if t is None:
+            continue
+        g = lambda k: _get(r, k, tables_map or {})  # noqa: E731
+        info = {"engine": g("ENGINE"), "rows_estimate": g("TABLE_ROWS"), "data_bytes": g("DATA_LENGTH"),
+                "index_bytes": g("INDEX_LENGTH"), "comment": g("TABLE_COMMENT"), "partitioning": g("PARTITIONING"),
+                "create_options": g("CREATE_OPTIONS")}
+        t.info.update({k: (int(v) if k.endswith(("estimate", "bytes")) and str(v).isdigit() else v)
+                       for k, v in info.items() if v not in (None, "")})
+    return sch

keygraph/adapters/fetch.py

@@ -0,0 +1,93 @@
+"""Fetch information_schema STATISTICS and KEY_COLUMN_USAGE through any command or callable.
+
+Read-only: only SELECT statements on information_schema are issued. `run_sql(sql) -> list[dict]` can be a
+DB-API wrapper, a SQLAlchemy wrapper, or `command_runner("some-cli --sql {sql}")` for an external tool that
+prints JSON (a list of rows, or an object with a "rows" list; leading log lines are skipped).
+"""
+
+from __future__ import annotations
+
+import json
+import shlex
+import string
+import subprocess
+from typing import Callable
+
+STATS_SQL = ("SELECT TABLE_SCHEMA,TABLE_NAME,INDEX_NAME,NON_UNIQUE,SEQ_IN_INDEX,COLUMN_NAME "
+             "FROM information_schema.STATISTICS WHERE TABLE_SCHEMA={schema} {extra} "
+             "ORDER BY TABLE_NAME,INDEX_NAME,SEQ_IN_INDEX LIMIT {limit}")
+KCU_SQL = ("SELECT TABLE_SCHEMA,TABLE_NAME,CONSTRAINT_NAME,ORDINAL_POSITION,COLUMN_NAME,"
+           "REFERENCED_TABLE_SCHEMA,REFERENCED_TABLE_NAME,REFERENCED_COLUMN_NAME "
+           "FROM information_schema.KEY_COLUMN_USAGE WHERE TABLE_SCHEMA={schema} "
+           "AND REFERENCED_TABLE_NAME IS NOT NULL {extra} ORDER BY TABLE_NAME,CONSTRAINT_NAME,ORDINAL_POSITION LIMIT {limit}")
+
+Rows = list[dict]
+
+
+def parse_rows(stdout: str) -> Rows:
+    text = stdout.strip()
+    starts = [i for i in (text.find("{"), text.find("[")) if i >= 0]
+    if not starts:
+        raise ValueError("no JSON in command output: " + text[:200])
+    data = json.loads(text[min(starts):])
+    if isinstance(data, dict):
+        data = data.get("rows", [])
+    return data
+
+
+def command_runner(template: str, timeout: int = 120) -> Callable[[str], Rows]:
+    """template like 'mytool --engine prod query --sql {sql}'; {sql} is passed as one argument."""
+    parts = shlex.split(template)
+
+    def run(sql: str) -> Rows:
+        argv = [p.replace("{sql}", sql) if "{sql}" in p else p for p in parts]
+        done = subprocess.run(argv, capture_output=True, text=True, timeout=timeout)
+        if done.returncode != 0:
+            raise RuntimeError(f"command failed ({done.returncode}): {done.stderr.strip()[-300:]}")
+        return parse_rows(done.stdout)
+
+    return run
+
+
+def _chunked(run_sql, template: str, schema_expr: str, limit: int, name_col: str = "TABLE_NAME") -> Rows:
+    """Run the query; when a result hits the row limit, split by table-name prefix until each part fits."""
+    out: Rows = []
+
+    def go(prefix: str):
+        extra = f"AND {name_col} LIKE '{prefix}%'" if prefix else ""
+        rows = run_sql(template.format(schema=schema_expr, extra=extra, limit=limit))
+        if len(rows) < limit or len(prefix) >= 6:
+            out.extend(rows)
+            return
+        alphabet = string.ascii_lowercase + string.digits + "_"
+        for ch in alphabet:
+            go(prefix + ch.replace("_", "\\_"))
+        # names starting with an uppercase letter match LIKE case-insensitively on most collations
+
+    go("")
+    return out
+
+
+def fetch_metadata(run_sql: Callable[[str], Rows], schema_expr: str = "DATABASE()", limit: int = 10000) -> tuple[Rows, Rows]:
+    return (_chunked(run_sql, STATS_SQL, schema_expr, limit), _chunked(run_sql, KCU_SQL, schema_expr, limit))
+
+
+COLUMNS_SQL = ("SELECT TABLE_SCHEMA,TABLE_NAME,COLUMN_NAME,ORDINAL_POSITION,COLUMN_TYPE,DATA_TYPE,IS_NULLABLE,COLUMN_KEY,"
+               "COLUMN_DEFAULT,EXTRA,CHARACTER_SET_NAME,COLLATION_NAME,COLUMN_COMMENT,GENERATION_EXPRESSION "
+               "FROM information_schema.COLUMNS WHERE TABLE_SCHEMA={schema} {extra} ORDER BY TABLE_NAME,ORDINAL_POSITION LIMIT {limit}")
+TABLES_SQL = ("SELECT TABLE_SCHEMA,TABLE_NAME,ENGINE,TABLE_ROWS,DATA_LENGTH,INDEX_LENGTH,TABLE_COMMENT,CREATE_OPTIONS "
+              "FROM information_schema.TABLES WHERE TABLE_SCHEMA={schema} AND TABLE_TYPE='BASE TABLE' {extra} ORDER BY TABLE_NAME LIMIT {limit}")
+CHECKS_SQL = ("SELECT tc.TABLE_SCHEMA,tc.TABLE_NAME,cc.CONSTRAINT_NAME,cc.CHECK_CLAUSE FROM information_schema.CHECK_CONSTRAINTS cc "
+              "JOIN information_schema.TABLE_CONSTRAINTS tc ON tc.CONSTRAINT_SCHEMA=cc.CONSTRAINT_SCHEMA AND "
+              "tc.CONSTRAINT_NAME=cc.CONSTRAINT_NAME WHERE tc.TABLE_SCHEMA={schema} AND tc.CONSTRAINT_TYPE='CHECK' LIMIT {limit}")
+
+
+def fetch_mysql_all(run_sql, schema_expr: str = "DATABASE()", limit: int = 10000) -> dict:
+    """COLUMNS, TABLES, STATISTICS, KEY_COLUMN_USAGE (chunked) and CHECK constraints (best effort)."""
+    out = {"columns": _chunked(run_sql, COLUMNS_SQL, schema_expr, limit), "tables": _chunked(run_sql, TABLES_SQL, schema_expr, limit)}
+    out["statistics"], out["key_columns"] = fetch_metadata(run_sql, schema_expr, limit)
+    try:
+        out["checks"] = run_sql(CHECKS_SQL.format(schema=schema_expr, limit=limit))
+    except Exception:  # noqa: BLE001  (MariaDB / old MySQL / tool restrictions)
+        out["checks"] = []
+    return out

keygraph/adapters/live.py

@@ -0,0 +1,50 @@
+"""Live scan through the SQLAlchemy inspector. Metadata only; never reads table rows."""
+
+from __future__ import annotations
+
+from ..model import Column, ForeignKey, Index, Schema, Table
+
+SYSTEM_SCHEMAS = {"information_schema", "mysql", "performance_schema", "sys", "pg_catalog", "pg_toast"}
+
+
+def scan_url(url: str, schemas: list[str] | None = None, all_schemas: bool = False) -> Schema:
+    try:
+        from sqlalchemy import create_engine, inspect
+    except ImportError as exc:  # pragma: no cover
+        raise SystemExit("live scanning needs SQLAlchemy: pip install 'keygraph[live]'") from exc
+    engine = create_engine(url)
+    insp = inspect(engine)
+    default = insp.default_schema_name or "main"
+    if all_schemas:
+        names = [s for s in insp.get_schema_names() if s.lower() not in SYSTEM_SCHEMAS and not s.startswith("pg_")]
+    else:
+        names = schemas or [default]
+    sch = Schema()
+    for s in names:
+        qual = None if s == default else s
+        for tn in insp.get_table_names(schema=qual):
+            t = Table(s, tn)
+            pk = insp.get_pk_constraint(tn, schema=qual) or {}
+            pkcols = list(pk.get("constrained_columns") or [])
+            for i, c in enumerate(insp.get_columns(tn, schema=qual), 1):
+                t.columns.append(Column(c["name"], str(c["type"]), bool(c.get("nullable", True)), i,
+                                        "PRI" if c["name"] in pkcols else ""))
+            if pkcols:
+                t.indexes.append(Index("PRIMARY", pkcols, True, True))
+            seen = {tuple(pkcols)} if pkcols else set()
+            for u in insp.get_unique_constraints(tn, schema=qual):
+                cols = [c for c in u.get("column_names", []) if c]
+                if cols and tuple(cols) not in seen:
+                    seen.add(tuple(cols))
+                    t.indexes.append(Index(u.get("name") or f"uq_{'_'.join(cols)}", cols, True))
+            for ix in insp.get_indexes(tn, schema=qual):
+                cols = [c for c in ix.get("column_names", []) if c]
+                if not cols or (ix.get("unique") and tuple(cols) in seen):
+                    continue
+                t.indexes.append(Index(ix.get("name") or f"ix_{'_'.join(cols)}", cols, bool(ix.get("unique"))))
+            for fk in insp.get_foreign_keys(tn, schema=qual):
+                t.foreign_keys.append(ForeignKey(fk.get("name") or "", list(fk["constrained_columns"]),
+                                                 fk.get("referred_schema") or s, fk["referred_table"],
+                                                 list(fk["referred_columns"])))
+            sch.add(t)
+    return sch

keygraph/adapters/metabase.py

@@ -0,0 +1,78 @@
+"""Metabase REST API: metadata (tables, fields, semantic types, foreign-key targets Metabase knows), free profile
+statistics from field fingerprints, and native read-only queries.
+
+`http(method, path, body=None) -> parsed JSON` is injected, so authentication (session token or API key) stays
+outside this module. Read-only: only GET requests and POST /api/dataset with a native SELECT (assert_select)."""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Callable
+
+from ..model import Column, ForeignKey, Index, Schema
+from .base import Source, guarded
+
+
+class MetabaseSource(Source):
+    kind, dialect, quote = "metabase", "generic", '"'
+
+    def __init__(self, http: Callable, database_id: int, schema_name: str | None = None):
+        self.http, self.db_id, self.schema_name = http, database_id, schema_name
+        self._fp: dict = {}
+        self.run = guarded(self._native)
+
+    def _native(self, sql: str) -> list:
+        res = self.http("POST", "/api/dataset", {"database": self.db_id, "type": "native", "native": {"query": sql}})
+        data = res.get("data", res)
+        names = [c["name"] for c in data.get("cols", [])]
+        return [dict(zip(names, row)) for row in data.get("rows", [])]
+
+    def schema(self) -> Schema:
+        meta = self.http("GET", f"/api/database/{self.db_id}/metadata?include_hidden=false")
+        sch = Schema()
+        fields_by_id: dict = {}
+        for tb in meta.get("tables", []):
+            sub = tb.get("schema") if tb.get("schema") not in (None, "", "public", "default") else ""
+            s = (f"{self.schema_name}_{sub}" if sub else self.schema_name) if self.schema_name else (tb.get("schema") or "default")
+            t = sch.get_or_create(s, tb["name"])
+            if tb.get("description"):
+                t.info["comment"] = tb["description"]
+            for f in sorted(tb.get("fields", []), key=lambda f: f.get("database_position", 0)):
+                sem = f.get("semantic_type") or ""
+                c = Column(f["name"], f.get("database_type") or f.get("base_type", ""), not f.get("database_required", False),
+                           f.get("database_position", 0) + 1, key="PRI" if sem == "type/PK" else "",
+                           comment=(f.get("description") or "") + (f" [semantic: {sem}]" if sem and sem != "type/PK" else ""))
+                t.columns.append(c)
+                fields_by_id[f["id"]] = (t, c)
+                if f.get("fingerprint"):
+                    self._fp[f"{t.id}.{c.name}"] = (f["fingerprint"], c.type)
+            pk = [c.name for c in t.columns if c.key == "PRI"]
+            if pk:
+                t.indexes.append(Index("PRIMARY", pk, True, True))
+        for tb in meta.get("tables", []):
+            for f in tb.get("fields", []):
+                tgt = fields_by_id.get(f.get("fk_target_field_id"))
+                src = fields_by_id.get(f["id"])
+                if tgt and src:
+                    src[0].foreign_keys.append(ForeignKey(f"metabase_fk_{f['name']}", [f["name"]], tgt[0].schema, tgt[0].name, [tgt[1].name]))
+        return sch
+
+    def fingerprints(self) -> dict:
+        """Profiles derived from Metabase field fingerprints (approximate, no query needed)."""
+        out = {}
+        now = datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
+        for key, (fp, _ty) in self._fp.items():
+            g = fp.get("global", {})
+            p = {"method": "metabase fingerprint (sampled by Metabase)", "profiled_at": now, "sample_size": None,
+                 "null_rate": g.get("nil%"), "distinct_in_sample": g.get("distinct-count")}
+            ty = fp.get("type", {})
+            if "type/Number" in ty:
+                n = ty["type/Number"]
+                p["numeric"] = {"min": n.get("min"), "max": n.get("max"), "p25": n.get("q1"), "p75": n.get("q3"), "mean": n.get("avg")}
+            if "type/Text" in ty:
+                p["length"] = {"avg": ty["type/Text"].get("average-length")}
+            if "type/DateTime" in ty:
+                d = ty["type/DateTime"]
+                p["dates"] = {"min": d.get("earliest"), "max": d.get("latest"), "recency_days": None}
+            out[key] = p
+        return out

keygraph/adapters/mongo.py

@@ -0,0 +1,93 @@
+"""MongoDB: schema inferred from sampled documents (nested paths become columns), indexes from the collection's
+index information, `_id` as primary key, DBRef values as declared references. The database object is injected
+(pymongo style) and wrapped so only read operations can be called."""
+
+from __future__ import annotations
+
+from ..model import Column, ForeignKey, Index, Schema, Table
+from .base import Source
+
+READ_DB = {"list_collection_names", "__getitem__", "name"}
+READ_COLL = {"find", "index_information", "estimated_document_count", "name"}
+
+
+class ReadOnlyProxy:
+    def __init__(self, obj, allowed):
+        self._o, self._a = obj, allowed
+
+    def __getattr__(self, k):
+        if k not in self._a:
+            raise PermissionError(f"keygraph is read-only: '{k}' is not allowed")
+        return getattr(self._o, k)
+
+    def __getitem__(self, k):
+        if "__getitem__" not in self._a:
+            raise PermissionError("not allowed")
+        return ReadOnlyProxy(self._o[k], READ_COLL)
+
+
+def _tname(v) -> str:
+    n = type(v).__name__
+    return {"str": "string", "int": "int", "float": "double", "bool": "bool", "NoneType": "null", "datetime": "date", "dict": "object",
+            "list": "array", "Int64": "long", "Decimal128": "decimal"}.get(n, n)
+
+
+def flatten(doc: dict, prefix: str = "") -> dict:
+    out = {}
+    for k, v in doc.items():
+        path = f"{prefix}{k}"
+        if isinstance(v, dict) and not ("$ref" in v and "$id" in v) and v:
+            out.update(flatten(v, path + "."))
+        else:
+            out[path] = v
+    return out
+
+
+class MongoSource(Source):
+    kind, dialect = "mongodb", "mongo"
+    read_only = "only list_collection_names, find, index_information and estimated_document_count are reachable"
+
+    def __init__(self, db, schema_name: str = "default", sample: int = 200):
+        self.db, self.schema_name, self.n = ReadOnlyProxy(db, READ_DB), schema_name, sample
+
+    def schema(self) -> Schema:
+        sch = Schema()
+        for name in sorted(self.db.list_collection_names()):
+            coll = self.db[name]
+            t = Table(self.schema_name, name)
+            types: dict[str, dict] = {}
+            refs: dict[str, str] = {}
+            docs = list(coll.find({}).limit(self.n))
+            for d in docs:
+                for path, v in flatten(d).items():
+                    types.setdefault(path, {}).setdefault(_tname(v), 0)
+                    types[path][_tname(v)] += 1
+                    if isinstance(v, dict) and "$ref" in v and "$id" in v:
+                        refs[path] = v["$ref"]
+            for i, (path, tc) in enumerate(sorted(types.items(), key=lambda x: (x[0] != "_id", x[0])), 1):
+                main = max(tc, key=tc.get)
+                present = sum(tc.values())
+                t.columns.append(Column(path, main if len(tc) == 1 else "/".join(sorted(tc)), present < len(docs), i,
+                                        key="PRI" if path == "_id" else "", comment="inferred from sampled documents"))
+            if "_id" in types:
+                t.indexes.append(Index("_id_", ["_id"], True, True))
+            try:
+                for iname, spec in coll.index_information().items():
+                    cols = [k for k, _ in spec.get("key", [])]
+                    if iname != "_id_" and cols:
+                        t.indexes.append(Index(iname, cols, bool(spec.get("unique"))))
+            except Exception:  # noqa: BLE001
+                pass
+            for path, ref in refs.items():
+                t.foreign_keys.append(ForeignKey(f"dbref_{path}", [path], self.schema_name, ref, ["_id"]))
+            try:
+                t.info["rows_estimate"] = coll.estimated_document_count()
+            except Exception:  # noqa: BLE001
+                pass
+            sch.add(t)
+        return sch
+
+    def sample(self, table_id: str, cols: list[str], n: int, strategy: str = "limit", pk=None):
+        name = table_id.split(".", 1)[1]
+        docs = [flatten(d) for d in self.db[name].find({}).limit(n)]
+        return cols, [tuple(d.get(c) for c in cols) for d in docs]