keygraph 0.2.0__py3-none-any.whl

keygraph/report.py

@@ -0,0 +1,92 @@
+"""RELATIONS_REPORT.md: plain-language review of the graph."""
+
+from __future__ import annotations
+
+from .graph import Graph
+
+
+def _leading(node: dict, cols: list[str]) -> bool:
+    n = len(cols)
+    return any(len(i["columns"]) >= n and set(i["columns"][:n]) == set(cols) for i in node["indexes"])
+
+
+def write_report(graph: Graph, ctx=None, cap: int = 40) -> str:
+    st = graph.data["stats"]
+    nodes = graph.nodes
+    deg: dict[str, int] = {n: 0 for n in nodes}
+    for e in graph.edges:
+        deg[e["from"]] += 1
+        if e["to"] != e["from"]:
+            deg[e["to"]] += 1
+    out = ["# Relations report", ""]
+    out.append(f"{st['tables']} tables in {st['schemas']} schema(s), {st['columns']} columns, {st['indexes']} indexes and "
+               f"{st['edges']} relations. Declared foreign keys: {st['edges_by_kind'].get('declared', 0)}; inferred: "
+               f"{st['edges_by_kind'].get('inferred', 0)}. Inferred relations are guesses from names and key shapes: "
+               "treat them by their confidence.")
+    out += ["", "Confidence bands: " + ", ".join(f"{k}: {v}" for k, v in st["edges_by_confidence"].items()), ""]
+    areas = graph.data.get("areas") or []
+    if areas:
+        big = max(areas, key=lambda a: a["size"])
+        out += [f"Areas: {len(areas)}; largest is {big['name']} with {big['size']} tables. Hub tables: {len(graph.data.get('hubs', []))}.", ""]
+
+    def section(title, rows, note=""):
+        out.append(f"## {title} ({len(rows)})")
+        if note:
+            out.append(note)
+        out.append("")
+        if not rows:
+            out.append("None.")
+        for r in rows[:cap]:
+            out.append("- " + r)
+        if len(rows) > cap:
+            out.append(f"- ... and {len(rows) - cap} more (see graph.json)")
+        out.append("")
+
+    hubs = sorted(nodes, key=lambda n: (-deg[n], n))[:10]
+    section("Hub tables", [f"{n}: {deg[n]} relations" for n in hubs if deg[n]],
+            "Tables with the most relations; joins usually route through them.")
+    section("Tables with no relations", [n for n in sorted(nodes) if deg[n] == 0],
+            "No declared or inferred relation found. They may be logs, config tables, or use keys with unusual names.")
+    weak = [e for e in graph.edges if e["kind"] == "inferred" and (e["confidence"] < 0.6 or e["ambiguous"])]
+    weak.sort(key=lambda e: (e["confidence"], e["id"]))
+    section("Ambiguous or low-confidence relations to review", [
+        f"{e['id']} confidence {e['confidence']:.2f}{' ambiguous' if e['ambiguous'] else ''}: {'; '.join(e['evidence'][:2])}" for e in weak],
+        "Confirm or reject these in an overrides file.")
+    poly = [f"{n['id']}.{p['id_column']} (type in {p['type_column']})" for n in nodes.values() for p in n["polymorphic"]]
+    section("Polymorphic columns", poly, "The target table depends on a type column; there is no single join target.")
+    section("Tables without a primary key", [n for n in sorted(nodes) if not nodes[n]["primary_key"]],
+            "Row identity is unclear; joins into these tables may duplicate rows.")
+    red = [f"{n['id']}: {r['index']}({', '.join(r['columns'])}) is covered by {r['covered_by']} ({r['reason']})"
+           for n in nodes.values() for r in n["redundant_indexes"]]
+    section("Redundant indexes", red)
+    unidx = sorted({f"{e['from']}({', '.join(e['from_columns'])}) -> {e['to']}" for e in graph.edges
+                    if e["confidence"] >= 0.5 and not _leading(nodes[e["from"]], e["from_columns"])
+                    and not any(set(nodes[e['from']]['primary_key']) >= set(e['from_columns']) for _ in [0])})
+    section("Foreign-key-like columns without an index", unidx,
+            "Joining or filtering on these columns scans the whole table.")
+    unres = [f"{n['id']}: {', '.join(n['unresolved_id_columns'])}" for n in nodes.values() if n["unresolved_id_columns"]]
+    section("Id-like columns with no matching table", unres)
+    if graph.data["warnings"]:
+        section("Warnings", graph.data["warnings"])
+    if ctx is not None:
+        notes = {r["table_id"]: r for r in ctx.rows("table_notes")}
+        sem = {r["table_id"] for r in ctx.rows("column_semantics")}
+        linked = {r["table_id"] for r in ctx.rows("concept_links")}
+        covered = set(notes) | sem | linked
+        c = ctx.counts()
+        out.append("## Context coverage")
+        out.append("")
+        out.append(f"{len(covered & set(nodes))} of {len(nodes)} tables have any business context. Entries: " +
+                   ", ".join(f"{v} {k.replace('_', ' ')}" for k, v in c.items()) + ".")
+        out.append("")
+        nohub = [f"{n}: {deg[n]} relations" for n in sorted(nodes, key=lambda n: (-deg[n], n))
+                 if deg[n] >= 3 and not (notes.get(n) or {}).get("grain")][:cap]
+        section("Hub tables with no grain note", nohub)
+        unv = [f"table note {k}" for k, v in notes.items() if not v["verified"]]
+        unv += [f"column {r['table_id']}.{r['col']}" for r in ctx.rows("column_semantics") if not r["verified"]]
+        unv += [f"concept {r['name']}" for r in ctx.rows("concepts") if not r["verified"]]
+        section("Unverified context entries", sorted(unv))
+        section("Tables with no context (first by relations)",
+                [n for n in sorted(nodes, key=lambda n: (-deg[n], n)) if n not in covered][:cap],
+                f"{len([n for n in nodes if n not in covered])} tables in total.")
+    return "\n".join(out).rstrip() + "\n"

keygraph/search.py

@@ -0,0 +1,175 @@
+"""Business-question search over names + context, describe, and context-to-graph evidence."""
+
+from __future__ import annotations
+
+import json
+
+from .context import ContextDB, safe_source
+from .graph import Graph
+from .query import column_detail, filter_columns, find_paths, table_info_line
+
+WEIGHT = {"concept": 1.0, "table_note": 1.2, "metric": 1.0, "column_ctx": 0.8, "column_value": 0.7, "table": 1.0, "column": 0.5}
+
+
+def apply_context_to_graph(graph: Graph, ctx: ContextDB) -> int:
+    """Concept relations that map onto a graph edge add evidence and raise confidence (idempotent)."""
+    prim: dict[str, set[str]] = {}
+    for l in ctx.rows("concept_links"):
+        if l["role"] == "primary-record":
+            prim.setdefault(l["concept"], set()).add(l["table_id"])
+    by_id = {e["id"]: e for e in graph.edges}
+    pair: dict[tuple[str, str], list[dict]] = {}
+    for e in graph.edges:
+        pair.setdefault((e["from"], e["to"]), []).append(e)
+        pair.setdefault((e["to"], e["from"]), []).append(e)
+    n = 0
+    for r in ctx.rows("concept_relations"):
+        targets = [by_id[r["edge_id"]]] if r["edge_id"] in by_id else [
+            e for a in prim.get(r["from_concept"], ()) for b in prim.get(r["to_concept"], ()) for e in pair.get((a, b), [])]
+        for e in targets:
+            msg = f"business relation: {r['from_concept']} {r['rel']} {r['to_concept']} (source: {safe_source(r['source'])})"
+            if msg in e["evidence"]:
+                continue
+            e["evidence"].append(msg)
+            if e["kind"] == "inferred":
+                e["confidence"] = round(min(0.95, e["confidence"] + (0.2 if r["verified"] else 0.1)), 2)
+            n += 1
+    graph.refresh_stats()
+    return n
+
+
+def search(graph: Graph, ctx: ContextDB, query: str, limit: int = 8, with_paths: bool = True) -> dict:
+    hits = ctx.fts(query)
+    score: dict[str, float] = {}
+    why: dict[str, list[str]] = {}
+    concepts, metrics = [], []
+
+    def add(tid, s, reason):
+        if tid in graph.nodes:
+            score[tid] = score.get(tid, 0) + s
+            if reason not in why.setdefault(tid, []):
+                why[tid].append(reason)
+
+    links = ctx.rows("concept_links")
+    by_concept: dict[str, list[dict]] = {}
+    for l in links:
+        by_concept.setdefault(l["concept"], []).append(l)
+    rels = ctx.rows("concept_relations")
+    for kind, ref, s in hits:
+        w = WEIGHT.get(kind, 0.5) * s
+        if kind == "table":
+            add(ref, w, "table name matches")
+        elif kind == "column":
+            add(ref.rsplit(".", 1)[0], w, f"column {ref.rsplit('.', 1)[1]} matches")
+        elif kind == "column_value":
+            add(ref.rsplit(".", 1)[0], w, f"observed values of {ref.rsplit('.', 1)[1]} match")
+        elif kind == "column_ctx":
+            add(ref.rsplit(".", 1)[0], w, f"column context: {ref.rsplit('.', 1)[1]}")
+        elif kind == "table_note":
+            add(ref, w, "table notes match")
+        elif kind == "concept":
+            concepts.append(ref)
+            for l in by_concept.get(ref, []):
+                add(l["table_id"], w * (1.0 if l["role"] == "primary-record" else 0.5), f"concept '{ref}' ({l['role']})")
+            for r in rels:
+                other = r["to_concept"] if r["from_concept"] == ref else r["from_concept"] if r["to_concept"] == ref else None
+                if other:
+                    for l in by_concept.get(other, []):
+                        if l["role"] == "primary-record":
+                            add(l["table_id"], w * 0.4, f"related concept '{other}' ({r['from_concept']} {r['rel']} {r['to_concept']})")
+        elif kind == "metric":
+            m = ctx.rows("metrics", "WHERE name=?", (ref,))[0]
+            metrics.append(ref)
+            for t in json.loads(m["tables"] or "[]"):
+                add(t, w, f"metric '{ref}'")
+    top = sorted(score, key=lambda t: (-score[t], t))[:limit]
+    out_tables = []
+    for t in top:
+        c = ctx.table_context(t)
+        notes = c["notes"] or {}
+        node = graph.nodes[t]
+        out_tables.append({
+            "table": t, "score": round(score[t], 2), "why": why[t][:5], "grain": notes.get("grain", ""),
+            "summary": notes.get("summary", ""), "primary_key": node["primary_key"],
+            "filter_columns": filter_columns(node, notes), "concepts": sorted({l["concept"] for l in c["concepts"]}),
+            "pitfalls": notes.get("pitfalls", ""), "verified": bool(notes.get("verified"))})
+    paths = []
+    if with_paths and len(top) >= 2:
+        for other in top[1:3]:
+            ps = find_paths(graph, top[0], other, 4, 0.6, 1)
+            if ps:
+                paths.append({"from": top[0], "to": other, **ps[0]})
+    return {"query": query, "tables": out_tables, "concepts": sorted(set(concepts)), "metrics": sorted(set(metrics)),
+            "join_paths": paths}
+
+
+def format_search(r: dict) -> str:
+    out = [f"Search: {r['query']}"]
+    if r["concepts"]:
+        out.append("  concepts: " + ", ".join(r["concepts"]))
+    if r["metrics"]:
+        out.append("  metrics: " + ", ".join(r["metrics"]))
+    if not r["tables"]:
+        out.append("  no matching tables")
+    for i, t in enumerate(r["tables"], 1):
+        out.append(f"{i}. {t['table']}  (score {t['score']}{', verified' if t['verified'] else ''})")
+        if t["grain"] or t["summary"]:
+            out.append(f"     grain/summary: {t['grain'] or t['summary']}"[:260])
+        out.append(f"     pk: {', '.join(t['primary_key']) or 'none'}   filter on: {', '.join(t['filter_columns']) or 'n/a'}")
+        out.append("     why: " + "; ".join(t["why"]))
+        if t["pitfalls"]:
+            out.append(f"     pitfalls: {t['pitfalls']}"[:260])
+    for p in r["join_paths"]:
+        out.append(f"join {p['from']} .. {p['to']} (confidence {p['confidence']}):")
+        out.append("   " + p["sql"].replace("\n", "\n   "))
+    return "\n".join(out)
+
+
+def describe(graph: Graph, ctx: ContextDB, ref: str) -> dict:
+    from .query import keys, neighbors
+    t = graph.resolve(ref)
+    c = ctx.table_context(t)
+    nb = neighbors(graph, t, 1)
+    return {"table": t, "keys": keys(graph, t), "filter_columns": filter_columns(graph.nodes[t], c["notes"]),
+            "context": c, "relations": nb["levels"].get("1", []),
+            "info_line": table_info_line(graph.nodes[t]),
+            "columns": [{**x, "detail": column_detail(x), "profile": (graph.data.get("profiles") or {}).get(f"{t}.{x['name']}")}
+                        for x in graph.nodes[t]["columns"]]}
+
+
+def format_describe(d: dict) -> str:
+    from .query import format_keys
+    out = [format_keys(d["keys"])]
+    n = d["context"]["notes"]
+    if n:
+        out.append(f"  grain: {n['grain']}")
+        for k in ("summary", "freshness", "soft_delete", "date_column", "pitfalls"):
+            if n.get(k):
+                out.append(f"  {k}: {n[k]}")
+        out.append(f"  context source: {safe_source(n['source'])}  verified: {bool(n['verified'])}")
+    out.append("  efficient filters: " + (", ".join(d["filter_columns"]) or "n/a"))
+    if d["context"]["concepts"]:
+        out.append("  concepts: " + ", ".join(f"{l['concept']} ({l['role']})" for l in d["context"]["concepts"]))
+    for m in d["context"]["metrics"]:
+        out.append(f"  metric {m['name']}: {m['definition']}  [{m['expression']}]")
+    sem = {c["col"]: c for c in d["context"]["columns"]}
+    out.append("  columns:")
+    for c in d["columns"]:
+        s = sem.get(c["name"])
+        extra = f" - {s['meaning']}" if s and s["meaning"] else ""
+        if s and s["unit"]:
+            extra += f" [{s['unit']}]"
+        if s and s["codes"]:
+            extra += " codes: " + ", ".join(f"{k}={v}" for k, v in s["codes"].items())
+        if s and s["pii"]:
+            extra += " [PII]"
+        pf = c.get("profile")
+        if pf:
+            from .profile import summarize
+            extra += "  [" + summarize(pf) + "]"
+        out.append(f"    {c['name']} {c['detail']}{extra}")
+    if d["relations"]:
+        out.append("  relations:")
+        for x in d["relations"]:
+            out.append(f"    {x['from']}({','.join(x['from_columns'])}) -> {x['to']}({','.join(x['to_columns'])}) [{x['kind']} {x['confidence']:.2f}]")
+    return "\n".join(out)

keygraph/static/DAGRE_LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2012-2014 Chris Pettitt
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

keygraph/static/OFL-Inter.txt

@@ -0,0 +1,93 @@
+Copyright 2016 The Inter Project Authors (https://github.com/rsms/inter)
+
+This Font Software is licensed under the SIL Open Font License, Version 1.1.
+This license is copied below, and is also available with a FAQ at:
+http://scripts.sil.org/OFL
+
+
+-----------------------------------------------------------
+SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
+-----------------------------------------------------------
+
+PREAMBLE
+The goals of the Open Font License (OFL) are to stimulate worldwide
+development of collaborative font projects, to support the font creation
+efforts of academic and linguistic communities, and to provide a free and
+open framework in which fonts may be shared and improved in partnership
+with others.
+
+The OFL allows the licensed fonts to be used, studied, modified and
+redistributed freely as long as they are not sold by themselves. The
+fonts, including any derivative works, can be bundled, embedded, 
+redistributed and/or sold with any software provided that any reserved
+names are not used by derivative works. The fonts and derivatives,
+however, cannot be released under any other type of license. The
+requirement for fonts to remain under this license does not apply
+to any document created using the fonts or their derivatives.
+
+DEFINITIONS
+"Font Software" refers to the set of files released by the Copyright
+Holder(s) under this license and clearly marked as such. This may
+include source files, build scripts and documentation.
+
+"Reserved Font Name" refers to any names specified as such after the
+copyright statement(s).
+
+"Original Version" refers to the collection of Font Software components as
+distributed by the Copyright Holder(s).
+
+"Modified Version" refers to any derivative made by adding to, deleting,
+or substituting -- in part or in whole -- any of the components of the
+Original Version, by changing formats or by porting the Font Software to a
+new environment.
+
+"Author" refers to any designer, engineer, programmer, technical
+writer or other person who contributed to the Font Software.
+
+PERMISSION & CONDITIONS
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of the Font Software, to use, study, copy, merge, embed, modify,
+redistribute, and sell modified and unmodified copies of the Font
+Software, subject to the following conditions:
+
+1) Neither the Font Software nor any of its individual components,
+in Original or Modified Versions, may be sold by itself.
+
+2) Original or Modified Versions of the Font Software may be bundled,
+redistributed and/or sold with any software, provided that each copy
+contains the above copyright notice and this license. These can be
+included either as stand-alone text files, human-readable headers or
+in the appropriate machine-readable metadata fields within text or
+binary files as long as those fields can be easily viewed by the user.
+
+3) No Modified Version of the Font Software may use the Reserved Font
+Name(s) unless explicit written permission is granted by the corresponding
+Copyright Holder. This restriction only applies to the primary font name as
+presented to the users.
+
+4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font
+Software shall not be used to promote, endorse or advertise any
+Modified Version, except to acknowledge the contribution(s) of the
+Copyright Holder(s) and the Author(s) or with their explicit written
+permission.
+
+5) The Font Software, modified or unmodified, in part or in whole,
+must be distributed entirely under this license, and must not be
+distributed under any other license. The requirement for fonts to
+remain under this license does not apply to any document created
+using the Font Software.
+
+TERMINATION
+This license becomes null and void if any of the above conditions are
+not met.
+
+DISCLAIMER
+THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
+OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE
+COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL
+DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM
+OTHER DEALINGS IN THE FONT SOFTWARE.

keygraph/static/OFL-JetBrainsMono.txt

@@ -0,0 +1,93 @@
+Copyright 2020 The JetBrains Mono Project Authors (https://github.com/JetBrains/JetBrainsMono)
+
+This Font Software is licensed under the SIL Open Font License, Version 1.1.
+This license is copied below, and is also available with a FAQ at:
+http://scripts.sil.org/OFL
+
+
+-----------------------------------------------------------
+SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
+-----------------------------------------------------------
+
+PREAMBLE
+The goals of the Open Font License (OFL) are to stimulate worldwide
+development of collaborative font projects, to support the font creation
+efforts of academic and linguistic communities, and to provide a free and
+open framework in which fonts may be shared and improved in partnership
+with others.
+
+The OFL allows the licensed fonts to be used, studied, modified and
+redistributed freely as long as they are not sold by themselves. The
+fonts, including any derivative works, can be bundled, embedded, 
+redistributed and/or sold with any software provided that any reserved
+names are not used by derivative works. The fonts and derivatives,
+however, cannot be released under any other type of license. The
+requirement for fonts to remain under this license does not apply
+to any document created using the fonts or their derivatives.
+
+DEFINITIONS
+"Font Software" refers to the set of files released by the Copyright
+Holder(s) under this license and clearly marked as such. This may
+include source files, build scripts and documentation.
+
+"Reserved Font Name" refers to any names specified as such after the
+copyright statement(s).
+
+"Original Version" refers to the collection of Font Software components as
+distributed by the Copyright Holder(s).
+
+"Modified Version" refers to any derivative made by adding to, deleting,
+or substituting -- in part or in whole -- any of the components of the
+Original Version, by changing formats or by porting the Font Software to a
+new environment.
+
+"Author" refers to any designer, engineer, programmer, technical
+writer or other person who contributed to the Font Software.
+
+PERMISSION & CONDITIONS
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of the Font Software, to use, study, copy, merge, embed, modify,
+redistribute, and sell modified and unmodified copies of the Font
+Software, subject to the following conditions:
+
+1) Neither the Font Software nor any of its individual components,
+in Original or Modified Versions, may be sold by itself.
+
+2) Original or Modified Versions of the Font Software may be bundled,
+redistributed and/or sold with any software, provided that each copy
+contains the above copyright notice and this license. These can be
+included either as stand-alone text files, human-readable headers or
+in the appropriate machine-readable metadata fields within text or
+binary files as long as those fields can be easily viewed by the user.
+
+3) No Modified Version of the Font Software may use the Reserved Font
+Name(s) unless explicit written permission is granted by the corresponding
+Copyright Holder. This restriction only applies to the primary font name as
+presented to the users.
+
+4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font
+Software shall not be used to promote, endorse or advertise any
+Modified Version, except to acknowledge the contribution(s) of the
+Copyright Holder(s) and the Author(s) or with their explicit written
+permission.
+
+5) The Font Software, modified or unmodified, in part or in whole,
+must be distributed entirely under this license, and must not be
+distributed under any other license. The requirement for fonts to
+remain under this license does not apply to any document created
+using the Font Software.
+
+TERMINATION
+This license becomes null and void if any of the above conditions are
+not met.
+
+DISCLAIMER
+THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
+OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE
+COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL
+DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM
+OTHER DEALINGS IN THE FONT SOFTWARE.