govgraph 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- govgraph/__init__.py +55 -0
- govgraph/__main__.py +3 -0
- govgraph/adapters/__init__.py +6 -0
- govgraph/adapters/base.py +49 -0
- govgraph/adapters/csv_json.py +162 -0
- govgraph/adapters/datahub.py +255 -0
- govgraph/adapters/dbt.py +264 -0
- govgraph/adapters/openmetadata.py +290 -0
- govgraph/adapters/registry.py +51 -0
- govgraph/cli.py +357 -0
- govgraph/engines/__init__.py +5 -0
- govgraph/engines/puppygraph.py +150 -0
- govgraph/errors.py +58 -0
- govgraph/export.py +67 -0
- govgraph/ids.py +63 -0
- govgraph/ingest.py +95 -0
- govgraph/mcp/__init__.py +5 -0
- govgraph/mcp/server.py +110 -0
- govgraph/py.typed +0 -0
- govgraph/records.py +97 -0
- govgraph/samples.py +337 -0
- govgraph/spec/__init__.py +6 -0
- govgraph/spec/docs.py +71 -0
- govgraph/spec/loader.py +301 -0
- govgraph/spec/model.py +177 -0
- govgraph/spec/ontologies/core-1.0.yaml +228 -0
- govgraph/spec/ontology.schema.json +91 -0
- govgraph/store/__init__.py +14 -0
- govgraph/store/base.py +104 -0
- govgraph/store/ddl.py +93 -0
- govgraph/store/memory.py +134 -0
- govgraph/store/postgres.py +373 -0
- govgraph/store/queries.py +168 -0
- govgraph/tools/__init__.py +18 -0
- govgraph/tools/context.py +203 -0
- govgraph/tools/limits.py +38 -0
- govgraph/tools/results.py +125 -0
- govgraph/tools/schemas.py +113 -0
- govgraph/tools/toolkit.py +603 -0
- govgraph/validation.py +157 -0
- govgraph/viewer/__init__.py +32 -0
- govgraph/viewer/viewer.html +84 -0
- govgraph/viewer/viewer.js +476 -0
- govgraph-0.1.0.dist-info/METADATA +265 -0
- govgraph-0.1.0.dist-info/RECORD +48 -0
- govgraph-0.1.0.dist-info/WHEEL +4 -0
- govgraph-0.1.0.dist-info/entry_points.txt +9 -0
- govgraph-0.1.0.dist-info/licenses/LICENSE +202 -0
govgraph/adapters/dbt.py
ADDED
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
"""Adapter for dbt artifacts.
|
|
2
|
+
|
|
3
|
+
Reads ``manifest.json`` and, optionally, ``run_results.json`` from a dbt project's
|
|
4
|
+
``target`` directory. Nothing is executed and no warehouse is contacted.
|
|
5
|
+
|
|
6
|
+
What becomes what:
|
|
7
|
+
|
|
8
|
+
========================== ===========================================================
|
|
9
|
+
dbt govgraph
|
|
10
|
+
========================== ===========================================================
|
|
11
|
+
model, seed, snapshot, ``DataAsset`` (table or view), named ``database.schema.name``
|
|
12
|
+
source
|
|
13
|
+
documented column ``DataAsset`` (column) whose parent is the relation
|
|
14
|
+
``depends_on`` ``DERIVES_FROM`` plus a ``LineageEdge`` naming the model
|
|
15
|
+
``meta.owner``, group owner ``Owner`` and ``OWNED_BY``
|
|
16
|
+
data test ``DQRule`` and ``VALIDATES``
|
|
17
|
+
exposure ``DataProduct``, ``DERIVES_FROM`` and ``OWNED_BY``
|
|
18
|
+
test result ``DQResult`` and ``RESULT_OF``
|
|
19
|
+
failed test result ``DQFailure``, ``RAISED_FROM`` and ``FAILED_ON``
|
|
20
|
+
========================== ===========================================================
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import os
|
|
26
|
+
from collections.abc import Iterator, Mapping
|
|
27
|
+
from typing import Any
|
|
28
|
+
|
|
29
|
+
from govgraph.adapters.base import asset_ref, column_record, read_json
|
|
30
|
+
from govgraph.errors import AdapterError
|
|
31
|
+
from govgraph.records import EdgeRecord, NodeRecord, NodeRef, Record
|
|
32
|
+
|
|
33
|
+
_RELATION_TYPES = {"model", "seed", "snapshot", "source"}
|
|
34
|
+
_DIMENSIONS = {
|
|
35
|
+
"not_null": "completeness",
|
|
36
|
+
"unique": "uniqueness",
|
|
37
|
+
"accepted_values": "validity",
|
|
38
|
+
"relationships": "consistency",
|
|
39
|
+
}
|
|
40
|
+
_SEVERITIES = {"error": "high", "warn": "medium"}
|
|
41
|
+
_STATUSES = {"pass": "pass", "fail": "fail", "warn": "warn", "error": "error"}
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _relation_name(node: Mapping[str, Any]) -> str:
|
|
45
|
+
name = node.get("alias") or node.get("identifier") or node.get("name")
|
|
46
|
+
parts = [node.get("database"), node.get("schema"), name]
|
|
47
|
+
return ".".join(str(part) for part in parts if part)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _owner(identifier: str) -> NodeRecord:
|
|
51
|
+
return NodeRecord("Owner", {"identifier": identifier}, {"name": identifier})
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class DbtAdapter:
|
|
55
|
+
"""Reads a dbt ``manifest.json`` and, optionally, a ``run_results.json``."""
|
|
56
|
+
|
|
57
|
+
name = "dbt"
|
|
58
|
+
|
|
59
|
+
def __init__(
|
|
60
|
+
self,
|
|
61
|
+
manifest_path: str | os.PathLike[str],
|
|
62
|
+
run_results_path: str | os.PathLike[str] | None = None,
|
|
63
|
+
) -> None:
|
|
64
|
+
self._manifest_path = manifest_path
|
|
65
|
+
self._run_results_path = run_results_path
|
|
66
|
+
|
|
67
|
+
def records(self) -> Iterator[Record]:
|
|
68
|
+
manifest = read_json(self._manifest_path, "dbt manifest")
|
|
69
|
+
if not isinstance(manifest, Mapping) or not isinstance(manifest.get("nodes"), Mapping):
|
|
70
|
+
raise AdapterError("the dbt manifest has no 'nodes' section")
|
|
71
|
+
nodes: dict[str, Mapping[str, Any]] = {**manifest["nodes"], **manifest.get("sources", {})}
|
|
72
|
+
relations = {
|
|
73
|
+
unique_id: _relation_name(node)
|
|
74
|
+
for unique_id, node in nodes.items()
|
|
75
|
+
if node.get("resource_type") in _RELATION_TYPES
|
|
76
|
+
}
|
|
77
|
+
groups = {
|
|
78
|
+
group.get("name"): (group.get("owner") or {}).get("name")
|
|
79
|
+
for group in (manifest.get("groups") or {}).values()
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
for unique_id, qualified_name in relations.items():
|
|
83
|
+
yield from self._relation(nodes[unique_id], qualified_name, relations, groups)
|
|
84
|
+
|
|
85
|
+
tests = {
|
|
86
|
+
unique_id: node
|
|
87
|
+
for unique_id, node in nodes.items()
|
|
88
|
+
if node.get("resource_type") == "test"
|
|
89
|
+
}
|
|
90
|
+
targets = {
|
|
91
|
+
unique_id: self._test_targets(node, relations) for unique_id, node in tests.items()
|
|
92
|
+
}
|
|
93
|
+
documented = {
|
|
94
|
+
f"{relations[unique_id]}.{column}"
|
|
95
|
+
for unique_id in relations
|
|
96
|
+
for column in nodes[unique_id].get("columns") or {}
|
|
97
|
+
}
|
|
98
|
+
for unique_id, node in tests.items():
|
|
99
|
+
yield from self._test(unique_id, node, targets[unique_id], documented)
|
|
100
|
+
|
|
101
|
+
for exposure in (manifest.get("exposures") or {}).values():
|
|
102
|
+
yield from self._exposure(exposure, relations)
|
|
103
|
+
|
|
104
|
+
if self._run_results_path is not None:
|
|
105
|
+
yield from self._results(self._run_results_path, tests, targets, documented)
|
|
106
|
+
|
|
107
|
+
def _relation(
|
|
108
|
+
self,
|
|
109
|
+
node: Mapping[str, Any],
|
|
110
|
+
qualified_name: str,
|
|
111
|
+
relations: Mapping[str, str],
|
|
112
|
+
groups: Mapping[Any, Any],
|
|
113
|
+
) -> Iterator[Record]:
|
|
114
|
+
materialized = (node.get("config") or {}).get("materialized")
|
|
115
|
+
yield NodeRecord(
|
|
116
|
+
"DataAsset",
|
|
117
|
+
{"qualified_name": qualified_name},
|
|
118
|
+
{
|
|
119
|
+
"name": node.get("name"),
|
|
120
|
+
"kind": "view" if materialized == "view" else "table",
|
|
121
|
+
"description": node.get("description") or None,
|
|
122
|
+
},
|
|
123
|
+
)
|
|
124
|
+
for column, details in (node.get("columns") or {}).items():
|
|
125
|
+
yield column_record(
|
|
126
|
+
f"{qualified_name}.{column}",
|
|
127
|
+
data_type=details.get("data_type"),
|
|
128
|
+
description=details.get("description") or None,
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
meta = {**((node.get("config") or {}).get("meta") or {}), **(node.get("meta") or {})}
|
|
132
|
+
owner = meta.get("owner") or groups.get(node.get("group"))
|
|
133
|
+
if isinstance(owner, str) and owner:
|
|
134
|
+
yield _owner(owner)
|
|
135
|
+
yield EdgeRecord("OWNED_BY", asset_ref(qualified_name), _owner(owner).ref())
|
|
136
|
+
|
|
137
|
+
for upstream_id in (node.get("depends_on") or {}).get("nodes") or []:
|
|
138
|
+
upstream = relations.get(upstream_id)
|
|
139
|
+
if upstream is None:
|
|
140
|
+
continue
|
|
141
|
+
lineage = NodeRecord(
|
|
142
|
+
"LineageEdge",
|
|
143
|
+
{
|
|
144
|
+
"source_qualified_name": upstream,
|
|
145
|
+
"target_qualified_name": qualified_name,
|
|
146
|
+
"process": node.get("unique_id") or qualified_name,
|
|
147
|
+
},
|
|
148
|
+
{"transformation": f"dbt {node.get('resource_type')}"},
|
|
149
|
+
)
|
|
150
|
+
yield lineage
|
|
151
|
+
yield EdgeRecord(
|
|
152
|
+
"DERIVES_FROM",
|
|
153
|
+
asset_ref(qualified_name),
|
|
154
|
+
asset_ref(upstream),
|
|
155
|
+
{"lineage_id": lineage.id},
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
@staticmethod
|
|
159
|
+
def _test_targets(node: Mapping[str, Any], relations: Mapping[str, str]) -> list[str]:
|
|
160
|
+
"""Qualified names of the relations a test checks."""
|
|
161
|
+
attached = node.get("attached_node")
|
|
162
|
+
if isinstance(attached, str) and attached in relations:
|
|
163
|
+
return [relations[attached]]
|
|
164
|
+
depends_on = (node.get("depends_on") or {}).get("nodes") or []
|
|
165
|
+
return [relations[unique_id] for unique_id in depends_on if unique_id in relations]
|
|
166
|
+
|
|
167
|
+
@staticmethod
|
|
168
|
+
def _checked_assets(
|
|
169
|
+
node: Mapping[str, Any], targets: list[str], documented: set[str]
|
|
170
|
+
) -> list[NodeRef]:
|
|
171
|
+
"""The relation under test and, when the test names one, its documented column."""
|
|
172
|
+
assets = [asset_ref(target) for target in targets]
|
|
173
|
+
column = node.get("column_name")
|
|
174
|
+
if column and targets and f"{targets[0]}.{column}" in documented:
|
|
175
|
+
assets.append(asset_ref(f"{targets[0]}.{column}"))
|
|
176
|
+
return assets
|
|
177
|
+
|
|
178
|
+
def _test(
|
|
179
|
+
self, unique_id: str, node: Mapping[str, Any], targets: list[str], documented: set[str]
|
|
180
|
+
) -> Iterator[Record]:
|
|
181
|
+
kind = (node.get("test_metadata") or {}).get("name")
|
|
182
|
+
column = node.get("column_name")
|
|
183
|
+
severity = str((node.get("config") or {}).get("severity") or "").lower()
|
|
184
|
+
rule = NodeRecord(
|
|
185
|
+
"DQRule",
|
|
186
|
+
{"rule_id": unique_id},
|
|
187
|
+
{
|
|
188
|
+
"name": node.get("name"),
|
|
189
|
+
"description": node.get("description") or None,
|
|
190
|
+
"dimension": _DIMENSIONS.get(str(kind)),
|
|
191
|
+
"severity": _SEVERITIES.get(severity),
|
|
192
|
+
"expression": f"{kind}({column})" if kind and column else kind,
|
|
193
|
+
},
|
|
194
|
+
)
|
|
195
|
+
yield rule
|
|
196
|
+
for checked in self._checked_assets(node, targets, documented):
|
|
197
|
+
yield EdgeRecord("VALIDATES", rule.ref(), checked)
|
|
198
|
+
|
|
199
|
+
@staticmethod
|
|
200
|
+
def _exposure(exposure: Mapping[str, Any], relations: Mapping[str, str]) -> Iterator[Record]:
|
|
201
|
+
product = NodeRecord(
|
|
202
|
+
"DataProduct",
|
|
203
|
+
{"name": exposure.get("label") or exposure["name"]},
|
|
204
|
+
{"description": exposure.get("description") or None},
|
|
205
|
+
)
|
|
206
|
+
yield product
|
|
207
|
+
for upstream_id in (exposure.get("depends_on") or {}).get("nodes") or []:
|
|
208
|
+
if upstream_id in relations:
|
|
209
|
+
yield EdgeRecord("DERIVES_FROM", product.ref(), asset_ref(relations[upstream_id]))
|
|
210
|
+
# Only the owner's name is used; contact details are left behind.
|
|
211
|
+
owner = (exposure.get("owner") or {}).get("name")
|
|
212
|
+
if owner:
|
|
213
|
+
yield _owner(owner)
|
|
214
|
+
yield EdgeRecord("OWNED_BY", product.ref(), _owner(owner).ref())
|
|
215
|
+
|
|
216
|
+
def _results(
|
|
217
|
+
self,
|
|
218
|
+
path: str | os.PathLike[str],
|
|
219
|
+
tests: Mapping[str, Mapping[str, Any]],
|
|
220
|
+
targets: Mapping[str, list[str]],
|
|
221
|
+
documented: set[str],
|
|
222
|
+
) -> Iterator[Record]:
|
|
223
|
+
run = read_json(path, "dbt run results")
|
|
224
|
+
if not isinstance(run, Mapping) or not isinstance(run.get("results"), list):
|
|
225
|
+
raise AdapterError("the dbt run results have no 'results' list")
|
|
226
|
+
metadata = run.get("metadata") or {}
|
|
227
|
+
run_id, finished_at = metadata.get("invocation_id"), metadata.get("generated_at")
|
|
228
|
+
if not run_id:
|
|
229
|
+
raise AdapterError("the dbt run results have no metadata.invocation_id")
|
|
230
|
+
for outcome in run["results"]:
|
|
231
|
+
unique_id = outcome.get("unique_id")
|
|
232
|
+
status = _STATUSES.get(str(outcome.get("status")))
|
|
233
|
+
if unique_id not in tests or status is None:
|
|
234
|
+
continue
|
|
235
|
+
failures = outcome.get("failures")
|
|
236
|
+
result = NodeRecord(
|
|
237
|
+
"DQResult",
|
|
238
|
+
{"rule_id": unique_id, "run_id": run_id},
|
|
239
|
+
{
|
|
240
|
+
"status": status,
|
|
241
|
+
"executed_at": finished_at,
|
|
242
|
+
"rows_failed": failures if isinstance(failures, int) else None,
|
|
243
|
+
},
|
|
244
|
+
)
|
|
245
|
+
yield result
|
|
246
|
+
yield EdgeRecord("RESULT_OF", result.ref(), NodeRef("DQRule", {"rule_id": unique_id}))
|
|
247
|
+
if status != "fail":
|
|
248
|
+
continue
|
|
249
|
+
severity = str((tests[unique_id].get("config") or {}).get("severity") or "").lower()
|
|
250
|
+
failure = NodeRecord(
|
|
251
|
+
"DQFailure",
|
|
252
|
+
{"failure_id": f"{run_id}:{unique_id}"},
|
|
253
|
+
{
|
|
254
|
+
"summary": outcome.get("message"),
|
|
255
|
+
"severity": _SEVERITIES.get(severity),
|
|
256
|
+
"status": "open",
|
|
257
|
+
"detected_at": finished_at,
|
|
258
|
+
"failed_count": failures if isinstance(failures, int) else None,
|
|
259
|
+
},
|
|
260
|
+
)
|
|
261
|
+
yield failure
|
|
262
|
+
yield EdgeRecord("RAISED_FROM", failure.ref(), result.ref())
|
|
263
|
+
for checked in self._checked_assets(tests[unique_id], targets[unique_id], documented):
|
|
264
|
+
yield EdgeRecord("FAILED_ON", failure.ref(), checked)
|
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
"""Adapter for metadata exported from OpenMetadata.
|
|
2
|
+
|
|
3
|
+
Reads one JSON file. It never calls the OpenMetadata API; export the entities first and
|
|
4
|
+
put them in a document shaped like this, each list holding entities exactly as the API
|
|
5
|
+
returns them::
|
|
6
|
+
|
|
7
|
+
{"tables": [...], "glossaryTerms": [...], "testCases": [...], "lineage": [...]}
|
|
8
|
+
|
|
9
|
+
``lineage`` holds entity-lineage responses (``entity``, ``nodes``, ``upstreamEdges``,
|
|
10
|
+
``downstreamEdges``). Every section is optional.
|
|
11
|
+
|
|
12
|
+
What becomes what:
|
|
13
|
+
|
|
14
|
+
========================== ===========================================================
|
|
15
|
+
OpenMetadata govgraph
|
|
16
|
+
========================== ===========================================================
|
|
17
|
+
table ``DataAsset`` (table or view), named by fullyQualifiedName
|
|
18
|
+
column ``DataAsset`` (column) whose parent is the table
|
|
19
|
+
owner ``Owner`` and ``OWNED_BY``
|
|
20
|
+
glossary term ``BusinessTerm``
|
|
21
|
+
glossary tag on an asset ``DEFINES``
|
|
22
|
+
data product on a table ``DataProduct`` and ``DERIVES_FROM``
|
|
23
|
+
lineage edge ``DERIVES_FROM`` plus a ``LineageEdge``
|
|
24
|
+
column lineage ``DERIVES_FROM`` between the column assets
|
|
25
|
+
test case ``DQRule`` and ``VALIDATES``
|
|
26
|
+
latest test result ``DQResult`` and ``RESULT_OF``
|
|
27
|
+
failed test result ``DQFailure``, ``RAISED_FROM`` and ``FAILED_ON``
|
|
28
|
+
========================== ===========================================================
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
from __future__ import annotations
|
|
32
|
+
|
|
33
|
+
import os
|
|
34
|
+
from collections.abc import Iterator, Mapping
|
|
35
|
+
from datetime import datetime, timezone
|
|
36
|
+
from typing import Any
|
|
37
|
+
|
|
38
|
+
from govgraph.adapters.base import asset_ref, column_record, read_json
|
|
39
|
+
from govgraph.errors import AdapterError
|
|
40
|
+
from govgraph.records import EdgeRecord, NodeRecord, NodeRef, Record
|
|
41
|
+
|
|
42
|
+
_VIEW_TYPES = {"View", "SecureView", "MaterializedView", "SemanticView"}
|
|
43
|
+
_DIMENSIONS = {"completeness", "validity", "uniqueness", "timeliness", "consistency", "accuracy"}
|
|
44
|
+
_STATUSES = {"Success": "pass", "Failed": "fail", "Aborted": "error"}
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _entity_link(link: Any) -> tuple[str, str | None] | None:
|
|
48
|
+
"""Split ``<#E::table::<fqn>>`` or ``<#E::table::<fqn>::columns::<column>>``."""
|
|
49
|
+
text = str(link or "")
|
|
50
|
+
if not (text.startswith("<#E::") and text.endswith(">")):
|
|
51
|
+
return None
|
|
52
|
+
parts = text[5:-1].split("::")
|
|
53
|
+
if len(parts) == 2 and parts[0] == "table" and parts[1]:
|
|
54
|
+
return parts[1], None
|
|
55
|
+
if len(parts) == 4 and parts[0] == "table" and parts[2] == "columns" and parts[1] and parts[3]:
|
|
56
|
+
return parts[1], parts[3]
|
|
57
|
+
return None
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _owner_records(owners: Any, owned: NodeRef) -> Iterator[Record]:
|
|
61
|
+
for owner in owners or []:
|
|
62
|
+
name = owner.get("name")
|
|
63
|
+
if not name:
|
|
64
|
+
continue
|
|
65
|
+
record = NodeRecord(
|
|
66
|
+
"Owner",
|
|
67
|
+
{"identifier": f"{owner.get('type') or 'owner'}:{name}"},
|
|
68
|
+
{
|
|
69
|
+
"name": owner.get("displayName") or name,
|
|
70
|
+
"kind": "person" if owner.get("type") == "user" else "team",
|
|
71
|
+
},
|
|
72
|
+
)
|
|
73
|
+
yield record
|
|
74
|
+
yield EdgeRecord("OWNED_BY", owned, record.ref())
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _term_edges(tags: Any, asset: NodeRef) -> Iterator[Record]:
|
|
78
|
+
for tag in tags or []:
|
|
79
|
+
if tag.get("source") == "Glossary" and tag.get("tagFQN"):
|
|
80
|
+
term = NodeRecord("BusinessTerm", {"name": tag["tagFQN"]})
|
|
81
|
+
yield term
|
|
82
|
+
yield EdgeRecord("DEFINES", term.ref(), asset)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _timestamp(millis: Any) -> datetime | None:
|
|
86
|
+
if isinstance(millis, bool) or not isinstance(millis, (int, float)):
|
|
87
|
+
return None
|
|
88
|
+
return datetime.fromtimestamp(millis / 1000, tz=timezone.utc)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class OpenMetadataAdapter:
|
|
92
|
+
"""Reads tables, glossary terms, test cases and lineage exported from OpenMetadata."""
|
|
93
|
+
|
|
94
|
+
name = "openmetadata"
|
|
95
|
+
|
|
96
|
+
def __init__(self, path: str | os.PathLike[str]) -> None:
|
|
97
|
+
self._path = path
|
|
98
|
+
|
|
99
|
+
def records(self) -> Iterator[Record]:
|
|
100
|
+
document = read_json(self._path, "OpenMetadata export")
|
|
101
|
+
if not isinstance(document, Mapping):
|
|
102
|
+
raise AdapterError("the OpenMetadata export must be a JSON object")
|
|
103
|
+
for table in document.get("tables") or []:
|
|
104
|
+
yield from self._table(table)
|
|
105
|
+
for term in document.get("glossaryTerms") or []:
|
|
106
|
+
yield from self._glossary_term(term)
|
|
107
|
+
for lineage in document.get("lineage") or []:
|
|
108
|
+
yield from self._lineage(lineage)
|
|
109
|
+
for test_case in document.get("testCases") or []:
|
|
110
|
+
yield from self._test_case(test_case)
|
|
111
|
+
|
|
112
|
+
def _table(self, table: Mapping[str, Any]) -> Iterator[Record]:
|
|
113
|
+
qualified_name = table.get("fullyQualifiedName")
|
|
114
|
+
if not qualified_name:
|
|
115
|
+
raise AdapterError("a table in the OpenMetadata export has no fullyQualifiedName")
|
|
116
|
+
asset = asset_ref(qualified_name)
|
|
117
|
+
yield NodeRecord(
|
|
118
|
+
"DataAsset",
|
|
119
|
+
asset.key,
|
|
120
|
+
{
|
|
121
|
+
"name": table.get("name"),
|
|
122
|
+
"kind": "view" if table.get("tableType") in _VIEW_TYPES else "table",
|
|
123
|
+
"description": table.get("description") or None,
|
|
124
|
+
},
|
|
125
|
+
)
|
|
126
|
+
yield from _owner_records(table.get("owners"), asset)
|
|
127
|
+
yield from _term_edges(table.get("tags"), asset)
|
|
128
|
+
for product in table.get("dataProducts") or []:
|
|
129
|
+
name = product.get("fullyQualifiedName") or product.get("name")
|
|
130
|
+
if name:
|
|
131
|
+
record = NodeRecord(
|
|
132
|
+
"DataProduct", {"name": name}, {"description": product.get("description")}
|
|
133
|
+
)
|
|
134
|
+
yield record
|
|
135
|
+
yield EdgeRecord("DERIVES_FROM", record.ref(), asset)
|
|
136
|
+
yield from self._columns(table.get("columns"), qualified_name)
|
|
137
|
+
|
|
138
|
+
def _columns(self, columns: Any, parent: str) -> Iterator[Record]:
|
|
139
|
+
for column in columns or []:
|
|
140
|
+
name = column.get("name")
|
|
141
|
+
if not name:
|
|
142
|
+
continue
|
|
143
|
+
# Built from the parent rather than taken from the export, so that the
|
|
144
|
+
# column's parent is always the asset it was listed under.
|
|
145
|
+
qualified_name = f"{parent}.{name}"
|
|
146
|
+
yield column_record(
|
|
147
|
+
qualified_name,
|
|
148
|
+
data_type=column.get("dataType"),
|
|
149
|
+
description=column.get("description") or None,
|
|
150
|
+
)
|
|
151
|
+
yield from _term_edges(column.get("tags"), asset_ref(qualified_name))
|
|
152
|
+
yield from self._columns(column.get("children"), qualified_name)
|
|
153
|
+
|
|
154
|
+
@staticmethod
|
|
155
|
+
def _glossary_term(term: Mapping[str, Any]) -> Iterator[Record]:
|
|
156
|
+
name = term.get("fullyQualifiedName") or term.get("name")
|
|
157
|
+
if not name:
|
|
158
|
+
raise AdapterError("a glossary term in the OpenMetadata export has no name")
|
|
159
|
+
record = NodeRecord(
|
|
160
|
+
"BusinessTerm",
|
|
161
|
+
{"name": name},
|
|
162
|
+
{
|
|
163
|
+
"definition": term.get("description") or None,
|
|
164
|
+
"domain": (term.get("glossary") or {}).get("name"),
|
|
165
|
+
},
|
|
166
|
+
)
|
|
167
|
+
yield record
|
|
168
|
+
yield from _owner_records(term.get("owners"), record.ref())
|
|
169
|
+
|
|
170
|
+
@staticmethod
|
|
171
|
+
def _lineage(lineage: Mapping[str, Any]) -> Iterator[Record]:
|
|
172
|
+
references = [lineage.get("entity") or {}, *(lineage.get("nodes") or [])]
|
|
173
|
+
tables = {
|
|
174
|
+
ref.get("id"): ref.get("fullyQualifiedName")
|
|
175
|
+
for ref in references
|
|
176
|
+
if ref.get("type") == "table" and ref.get("fullyQualifiedName")
|
|
177
|
+
}
|
|
178
|
+
edges = [*(lineage.get("upstreamEdges") or []), *(lineage.get("downstreamEdges") or [])]
|
|
179
|
+
for edge in edges:
|
|
180
|
+
upstream, downstream = (
|
|
181
|
+
tables.get(edge.get("fromEntity")),
|
|
182
|
+
tables.get(edge.get("toEntity")),
|
|
183
|
+
)
|
|
184
|
+
if not upstream or not downstream:
|
|
185
|
+
continue
|
|
186
|
+
details = edge.get("lineageDetails") or {}
|
|
187
|
+
pipeline = details.get("pipeline") or {}
|
|
188
|
+
record = NodeRecord(
|
|
189
|
+
"LineageEdge",
|
|
190
|
+
{
|
|
191
|
+
"source_qualified_name": upstream,
|
|
192
|
+
"target_qualified_name": downstream,
|
|
193
|
+
"process": pipeline.get("fullyQualifiedName")
|
|
194
|
+
or pipeline.get("name")
|
|
195
|
+
or details.get("source")
|
|
196
|
+
or "lineage",
|
|
197
|
+
},
|
|
198
|
+
{"transformation": details.get("description") or details.get("sqlQuery")},
|
|
199
|
+
)
|
|
200
|
+
# The assets may not be in this export, so make sure they exist.
|
|
201
|
+
yield NodeRecord("DataAsset", {"qualified_name": upstream})
|
|
202
|
+
yield NodeRecord("DataAsset", {"qualified_name": downstream})
|
|
203
|
+
yield record
|
|
204
|
+
yield EdgeRecord(
|
|
205
|
+
"DERIVES_FROM",
|
|
206
|
+
asset_ref(downstream),
|
|
207
|
+
asset_ref(upstream),
|
|
208
|
+
{"lineage_id": record.id},
|
|
209
|
+
)
|
|
210
|
+
for mapping in details.get("columnsLineage") or []:
|
|
211
|
+
target = mapping.get("toColumn")
|
|
212
|
+
for source in mapping.get("fromColumns") or []:
|
|
213
|
+
if not target or not source:
|
|
214
|
+
continue
|
|
215
|
+
yield column_record(source)
|
|
216
|
+
yield column_record(target)
|
|
217
|
+
yield EdgeRecord(
|
|
218
|
+
"DERIVES_FROM",
|
|
219
|
+
asset_ref(target),
|
|
220
|
+
asset_ref(source),
|
|
221
|
+
{"lineage_id": record.id},
|
|
222
|
+
)
|
|
223
|
+
|
|
224
|
+
@staticmethod
|
|
225
|
+
def _test_case(test_case: Mapping[str, Any]) -> Iterator[Record]:
|
|
226
|
+
rule_id = test_case.get("fullyQualifiedName") or test_case.get("name")
|
|
227
|
+
link = _entity_link(test_case.get("entityLink"))
|
|
228
|
+
if not rule_id or link is None:
|
|
229
|
+
raise AdapterError(
|
|
230
|
+
"a test case in the OpenMetadata export has no name or no table entityLink"
|
|
231
|
+
)
|
|
232
|
+
table, column = link
|
|
233
|
+
dimension = str(test_case.get("dataQualityDimension") or "").lower()
|
|
234
|
+
rule = NodeRecord(
|
|
235
|
+
"DQRule",
|
|
236
|
+
{"rule_id": rule_id},
|
|
237
|
+
{
|
|
238
|
+
"name": test_case.get("displayName") or test_case.get("name"),
|
|
239
|
+
"description": test_case.get("description") or None,
|
|
240
|
+
"dimension": dimension if dimension in _DIMENSIONS else None,
|
|
241
|
+
"expression": (test_case.get("testDefinition") or {}).get("name"),
|
|
242
|
+
},
|
|
243
|
+
)
|
|
244
|
+
yield rule
|
|
245
|
+
# A stub, so the rule can be loaded even when the table is not in this export.
|
|
246
|
+
if column:
|
|
247
|
+
checked = asset_ref(f"{table}.{column}")
|
|
248
|
+
yield column_record(f"{table}.{column}")
|
|
249
|
+
else:
|
|
250
|
+
checked = asset_ref(table)
|
|
251
|
+
yield NodeRecord("DataAsset", checked.key)
|
|
252
|
+
yield EdgeRecord("VALIDATES", rule.ref(), checked)
|
|
253
|
+
yield from _owner_records(test_case.get("owners"), rule.ref())
|
|
254
|
+
|
|
255
|
+
outcome = test_case.get("testCaseResult") or {}
|
|
256
|
+
status = _STATUSES.get(str(outcome.get("testCaseStatus")))
|
|
257
|
+
when = _timestamp(outcome.get("timestamp"))
|
|
258
|
+
if status is None or when is None:
|
|
259
|
+
return
|
|
260
|
+
failed, passed = outcome.get("failedRows"), outcome.get("passedRows")
|
|
261
|
+
checked_rows = (
|
|
262
|
+
failed + passed if isinstance(failed, int) and isinstance(passed, int) else None
|
|
263
|
+
)
|
|
264
|
+
result = NodeRecord(
|
|
265
|
+
"DQResult",
|
|
266
|
+
{"rule_id": rule_id, "run_id": str(outcome["timestamp"])},
|
|
267
|
+
{
|
|
268
|
+
"status": status,
|
|
269
|
+
"executed_at": when,
|
|
270
|
+
"rows_failed": failed if isinstance(failed, int) else None,
|
|
271
|
+
"rows_checked": checked_rows,
|
|
272
|
+
},
|
|
273
|
+
)
|
|
274
|
+
yield result
|
|
275
|
+
yield EdgeRecord("RESULT_OF", result.ref(), rule.ref())
|
|
276
|
+
if status != "fail":
|
|
277
|
+
return
|
|
278
|
+
failure = NodeRecord(
|
|
279
|
+
"DQFailure",
|
|
280
|
+
{"failure_id": f"{rule_id}@{outcome['timestamp']}"},
|
|
281
|
+
{
|
|
282
|
+
"summary": outcome.get("result"),
|
|
283
|
+
"status": "open",
|
|
284
|
+
"detected_at": when,
|
|
285
|
+
"failed_count": failed if isinstance(failed, int) else None,
|
|
286
|
+
},
|
|
287
|
+
)
|
|
288
|
+
yield failure
|
|
289
|
+
yield EdgeRecord("RAISED_FROM", failure.ref(), result.ref())
|
|
290
|
+
yield EdgeRecord("FAILED_ON", failure.ref(), checked)
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"""Discover adapters through the ``govgraph.adapters`` entry-point group."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import inspect
|
|
6
|
+
from importlib.metadata import entry_points
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from govgraph.adapters.base import Adapter
|
|
10
|
+
from govgraph.errors import AdapterError
|
|
11
|
+
from govgraph.spec.model import Ontology
|
|
12
|
+
|
|
13
|
+
ENTRY_POINT_GROUP = "govgraph.adapters"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def available() -> list[str]:
|
|
17
|
+
"""Names of the adapters that are installed."""
|
|
18
|
+
return sorted(entry.name for entry in entry_points(group=ENTRY_POINT_GROUP))
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _accepts_ontology(factory: Any) -> bool:
|
|
22
|
+
try:
|
|
23
|
+
return "ontology" in inspect.signature(factory).parameters
|
|
24
|
+
except (TypeError, ValueError):
|
|
25
|
+
return False
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def load(name: str, /, *, ontology: Ontology | None = None, **options: Any) -> Adapter:
|
|
29
|
+
"""Create the adapter registered as ``name``, passing ``options`` to its constructor.
|
|
30
|
+
|
|
31
|
+
``ontology`` is passed on only to adapters whose constructor has a parameter of that
|
|
32
|
+
name, which is how an adapter learns about types added by an extension.
|
|
33
|
+
"""
|
|
34
|
+
matches = [entry for entry in entry_points(group=ENTRY_POINT_GROUP) if entry.name == name]
|
|
35
|
+
if not matches:
|
|
36
|
+
raise AdapterError(
|
|
37
|
+
f"no adapter named {name!r} is installed. Available: {', '.join(available())}"
|
|
38
|
+
)
|
|
39
|
+
try:
|
|
40
|
+
factory = matches[0].load()
|
|
41
|
+
except Exception as exc:
|
|
42
|
+
raise AdapterError(f"adapter {name!r} could not be imported: {exc}") from exc
|
|
43
|
+
if ontology is not None and _accepts_ontology(factory):
|
|
44
|
+
options["ontology"] = ontology
|
|
45
|
+
try:
|
|
46
|
+
adapter = factory(**options)
|
|
47
|
+
except TypeError as exc:
|
|
48
|
+
raise AdapterError(f"adapter {name!r} rejected its options: {exc}") from exc
|
|
49
|
+
if not isinstance(adapter, Adapter):
|
|
50
|
+
raise AdapterError(f"adapter {name!r} does not have a name and a records() method")
|
|
51
|
+
return adapter
|