govgraph 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- govgraph/__init__.py +55 -0
- govgraph/__main__.py +3 -0
- govgraph/adapters/__init__.py +6 -0
- govgraph/adapters/base.py +49 -0
- govgraph/adapters/csv_json.py +162 -0
- govgraph/adapters/datahub.py +255 -0
- govgraph/adapters/dbt.py +264 -0
- govgraph/adapters/openmetadata.py +290 -0
- govgraph/adapters/registry.py +51 -0
- govgraph/cli.py +357 -0
- govgraph/engines/__init__.py +5 -0
- govgraph/engines/puppygraph.py +150 -0
- govgraph/errors.py +58 -0
- govgraph/export.py +67 -0
- govgraph/ids.py +63 -0
- govgraph/ingest.py +95 -0
- govgraph/mcp/__init__.py +5 -0
- govgraph/mcp/server.py +110 -0
- govgraph/py.typed +0 -0
- govgraph/records.py +97 -0
- govgraph/samples.py +337 -0
- govgraph/spec/__init__.py +6 -0
- govgraph/spec/docs.py +71 -0
- govgraph/spec/loader.py +301 -0
- govgraph/spec/model.py +177 -0
- govgraph/spec/ontologies/core-1.0.yaml +228 -0
- govgraph/spec/ontology.schema.json +91 -0
- govgraph/store/__init__.py +14 -0
- govgraph/store/base.py +104 -0
- govgraph/store/ddl.py +93 -0
- govgraph/store/memory.py +134 -0
- govgraph/store/postgres.py +373 -0
- govgraph/store/queries.py +168 -0
- govgraph/tools/__init__.py +18 -0
- govgraph/tools/context.py +203 -0
- govgraph/tools/limits.py +38 -0
- govgraph/tools/results.py +125 -0
- govgraph/tools/schemas.py +113 -0
- govgraph/tools/toolkit.py +603 -0
- govgraph/validation.py +157 -0
- govgraph/viewer/__init__.py +32 -0
- govgraph/viewer/viewer.html +84 -0
- govgraph/viewer/viewer.js +476 -0
- govgraph-0.1.0.dist-info/METADATA +265 -0
- govgraph-0.1.0.dist-info/RECORD +48 -0
- govgraph-0.1.0.dist-info/WHEEL +4 -0
- govgraph-0.1.0.dist-info/entry_points.txt +9 -0
- govgraph-0.1.0.dist-info/licenses/LICENSE +202 -0
govgraph/__init__.py
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
"""govgraph: data-governance metadata as a typed knowledge graph with bounded agent tools."""
|
|
2
|
+
|
|
3
|
+
from govgraph.errors import (
|
|
4
|
+
AdapterError,
|
|
5
|
+
GovGraphError,
|
|
6
|
+
IngestError,
|
|
7
|
+
OntologyError,
|
|
8
|
+
RecordError,
|
|
9
|
+
StoreError,
|
|
10
|
+
StoreTimeoutError,
|
|
11
|
+
ToolError,
|
|
12
|
+
UnsafeRoleError,
|
|
13
|
+
)
|
|
14
|
+
from govgraph.ids import edge_id, node_id
|
|
15
|
+
from govgraph.ingest import IngestReport, ingest
|
|
16
|
+
from govgraph.records import Edge, EdgeRecord, Node, NodeRecord, NodeRef, Record
|
|
17
|
+
from govgraph.spec import Ontology, load_ontology
|
|
18
|
+
from govgraph.store import GraphReader, GraphStore, MemoryStore, PostgresStore
|
|
19
|
+
from govgraph.tools import AuditEvent, Claim, Limits, Toolkit, ToolResult
|
|
20
|
+
|
|
21
|
+
__version__ = "0.1.0"
|
|
22
|
+
|
|
23
|
+
__all__ = [
|
|
24
|
+
"AdapterError",
|
|
25
|
+
"AuditEvent",
|
|
26
|
+
"Claim",
|
|
27
|
+
"Edge",
|
|
28
|
+
"EdgeRecord",
|
|
29
|
+
"GovGraphError",
|
|
30
|
+
"GraphReader",
|
|
31
|
+
"GraphStore",
|
|
32
|
+
"IngestError",
|
|
33
|
+
"IngestReport",
|
|
34
|
+
"Limits",
|
|
35
|
+
"MemoryStore",
|
|
36
|
+
"Node",
|
|
37
|
+
"NodeRecord",
|
|
38
|
+
"NodeRef",
|
|
39
|
+
"Ontology",
|
|
40
|
+
"OntologyError",
|
|
41
|
+
"PostgresStore",
|
|
42
|
+
"Record",
|
|
43
|
+
"RecordError",
|
|
44
|
+
"StoreError",
|
|
45
|
+
"StoreTimeoutError",
|
|
46
|
+
"ToolError",
|
|
47
|
+
"ToolResult",
|
|
48
|
+
"Toolkit",
|
|
49
|
+
"UnsafeRoleError",
|
|
50
|
+
"__version__",
|
|
51
|
+
"edge_id",
|
|
52
|
+
"ingest",
|
|
53
|
+
"load_ontology",
|
|
54
|
+
"node_id",
|
|
55
|
+
]
|
govgraph/__main__.py
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""The adapter interface."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
from collections.abc import Iterator
|
|
8
|
+
from typing import Any, Protocol, runtime_checkable
|
|
9
|
+
|
|
10
|
+
from govgraph.errors import AdapterError
|
|
11
|
+
from govgraph.records import NodeRecord, NodeRef, Record
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@runtime_checkable
|
|
15
|
+
class Adapter(Protocol):
|
|
16
|
+
"""Turns metadata from one source into normalized records.
|
|
17
|
+
|
|
18
|
+
An adapter only yields records. It is never given a store, so it cannot write to
|
|
19
|
+
one; :func:`govgraph.ingest` is the single path from an adapter to a store.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
name: str
|
|
23
|
+
|
|
24
|
+
def records(self) -> Iterator[Record]: ...
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def read_json(path: str | os.PathLike[str], what: str) -> Any:
|
|
28
|
+
"""Parse a JSON file, raising :class:`AdapterError` with a readable message."""
|
|
29
|
+
try:
|
|
30
|
+
with open(path, encoding="utf-8") as handle:
|
|
31
|
+
return json.load(handle)
|
|
32
|
+
except OSError as exc:
|
|
33
|
+
raise AdapterError(f"cannot read {what} {os.fspath(path)}: {exc.strerror}") from exc
|
|
34
|
+
except json.JSONDecodeError as exc:
|
|
35
|
+
raise AdapterError(f"{what} {os.fspath(path)} is not valid JSON: {exc}") from exc
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def asset_ref(qualified_name: str) -> NodeRef:
|
|
39
|
+
return NodeRef("DataAsset", {"qualified_name": qualified_name})
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def column_record(qualified_name: str, **properties: Any) -> NodeRecord:
|
|
43
|
+
"""A column asset whose parent is everything before the last dot of its name."""
|
|
44
|
+
parent, _, name = qualified_name.rpartition(".")
|
|
45
|
+
return NodeRecord(
|
|
46
|
+
"DataAsset",
|
|
47
|
+
{"qualified_name": qualified_name},
|
|
48
|
+
{"name": name, "kind": "column", "parent_qualified_name": parent or None, **properties},
|
|
49
|
+
)
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
"""Adapters for plain files: one JSON document, or a directory of CSV files.
|
|
2
|
+
|
|
3
|
+
JSON document::
|
|
4
|
+
|
|
5
|
+
{"nodes": [{"type": "Owner", "key": {"identifier": "a"}, "properties": {"name": "A"}}],
|
|
6
|
+
"edges": [{"type": "OWNED_BY",
|
|
7
|
+
"source": {"type": "DataAsset", "key": {"qualified_name": "shop.orders"}},
|
|
8
|
+
"target": {"type": "Owner", "key": {"identifier": "a"}}}]}
|
|
9
|
+
|
|
10
|
+
CSV directory: one file per type, named ``<Type>.csv``. A node file has one column per
|
|
11
|
+
property. An edge file has ``source.<key field>`` and ``target.<key field>`` columns, plus
|
|
12
|
+
``source_type`` and ``target_type`` when the edge type allows more than one pair of node
|
|
13
|
+
types, plus one column per edge property. Empty cells mean "not set".
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import csv
|
|
19
|
+
import os
|
|
20
|
+
from collections.abc import Iterator, Mapping
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
from typing import Any
|
|
23
|
+
|
|
24
|
+
from govgraph.adapters.base import read_json
|
|
25
|
+
from govgraph.errors import AdapterError
|
|
26
|
+
from govgraph.records import EdgeRecord, NodeRecord, NodeRef, Record
|
|
27
|
+
from govgraph.spec.loader import load_ontology
|
|
28
|
+
from govgraph.spec.model import EdgeType, NodeType, Ontology, Property
|
|
29
|
+
|
|
30
|
+
_TRUE, _FALSE = {"true", "t", "yes", "y", "1"}, {"false", "f", "no", "n", "0"}
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _mapping(value: Any, where: str) -> Mapping[str, Any]:
|
|
34
|
+
if not isinstance(value, Mapping):
|
|
35
|
+
raise AdapterError(f"{where} must be an object")
|
|
36
|
+
return value
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _ref(value: Any, where: str) -> NodeRef:
|
|
40
|
+
ref = _mapping(value, where)
|
|
41
|
+
if not isinstance(ref.get("type"), str):
|
|
42
|
+
raise AdapterError(f"{where} needs a 'type'")
|
|
43
|
+
return NodeRef(ref["type"], dict(_mapping(ref.get("key"), f"{where}.key")))
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class JsonAdapter:
|
|
47
|
+
"""Reads nodes and edges from one JSON document."""
|
|
48
|
+
|
|
49
|
+
name = "json"
|
|
50
|
+
|
|
51
|
+
def __init__(self, path: str | os.PathLike[str]) -> None:
|
|
52
|
+
self._path = path
|
|
53
|
+
|
|
54
|
+
def records(self) -> Iterator[Record]:
|
|
55
|
+
document = _mapping(read_json(self._path, "JSON file"), "the document")
|
|
56
|
+
for index, raw in enumerate(document.get("nodes", [])):
|
|
57
|
+
where = f"nodes[{index}]"
|
|
58
|
+
node = _ref(raw, where)
|
|
59
|
+
properties = _mapping(raw.get("properties", {}), f"{where}.properties")
|
|
60
|
+
yield NodeRecord(node.type, node.key, dict(properties))
|
|
61
|
+
for index, raw in enumerate(document.get("edges", [])):
|
|
62
|
+
where = f"edges[{index}]"
|
|
63
|
+
edge = _mapping(raw, where)
|
|
64
|
+
if not isinstance(edge.get("type"), str):
|
|
65
|
+
raise AdapterError(f"{where} needs a 'type'")
|
|
66
|
+
yield EdgeRecord(
|
|
67
|
+
edge["type"],
|
|
68
|
+
_ref(edge.get("source"), f"{where}.source"),
|
|
69
|
+
_ref(edge.get("target"), f"{where}.target"),
|
|
70
|
+
dict(_mapping(edge.get("properties", {}), f"{where}.properties")),
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _convert(prop: Property, text: str, where: str) -> Any:
|
|
75
|
+
"""Turn a CSV cell into the Python type the property declares."""
|
|
76
|
+
try:
|
|
77
|
+
if prop.type == "integer":
|
|
78
|
+
return int(text)
|
|
79
|
+
if prop.type == "number":
|
|
80
|
+
return float(text)
|
|
81
|
+
except ValueError:
|
|
82
|
+
raise AdapterError(f"{where}: {text!r} is not a valid {prop.type}") from None
|
|
83
|
+
if prop.type == "boolean":
|
|
84
|
+
lowered = text.strip().lower()
|
|
85
|
+
if lowered in _TRUE:
|
|
86
|
+
return True
|
|
87
|
+
if lowered in _FALSE:
|
|
88
|
+
return False
|
|
89
|
+
raise AdapterError(f"{where}: {text!r} is not a valid boolean")
|
|
90
|
+
return text
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class CsvAdapter:
|
|
94
|
+
"""Reads nodes and edges from a directory of ``<Type>.csv`` files."""
|
|
95
|
+
|
|
96
|
+
name = "csv"
|
|
97
|
+
|
|
98
|
+
def __init__(self, path: str | os.PathLike[str], ontology: Ontology | None = None) -> None:
|
|
99
|
+
self._path = Path(path)
|
|
100
|
+
self._ontology = ontology or load_ontology()
|
|
101
|
+
|
|
102
|
+
def records(self) -> Iterator[Record]:
|
|
103
|
+
if not self._path.is_dir():
|
|
104
|
+
raise AdapterError(f"{self._path} is not a directory")
|
|
105
|
+
files = sorted(self._path.glob("*.csv"))
|
|
106
|
+
unknown = [
|
|
107
|
+
file.name
|
|
108
|
+
for file in files
|
|
109
|
+
if file.stem not in self._ontology.node_types
|
|
110
|
+
and file.stem not in self._ontology.edge_types
|
|
111
|
+
]
|
|
112
|
+
if unknown:
|
|
113
|
+
raise AdapterError(f"no node or edge type is named after: {', '.join(unknown)}")
|
|
114
|
+
# Nodes first, so that edges in the same directory find their endpoints.
|
|
115
|
+
for file in files:
|
|
116
|
+
if file.stem in self._ontology.node_types:
|
|
117
|
+
yield from self._nodes(file, self._ontology.node_types[file.stem])
|
|
118
|
+
for file in files:
|
|
119
|
+
if file.stem in self._ontology.edge_types:
|
|
120
|
+
yield from self._edges(file, self._ontology.edge_types[file.stem])
|
|
121
|
+
|
|
122
|
+
@staticmethod
|
|
123
|
+
def _rows(file: Path) -> Iterator[tuple[str, dict[str, str]]]:
|
|
124
|
+
with file.open(newline="", encoding="utf-8-sig") as handle:
|
|
125
|
+
for number, row in enumerate(csv.DictReader(handle), start=2):
|
|
126
|
+
cells = {k: v for k, v in row.items() if k is not None and v not in (None, "")}
|
|
127
|
+
yield f"{file.name} line {number}", cells
|
|
128
|
+
|
|
129
|
+
def _values(
|
|
130
|
+
self, declared: Mapping[str, Property], cells: Mapping[str, str], where: str
|
|
131
|
+
) -> dict[str, Any]:
|
|
132
|
+
values = {}
|
|
133
|
+
for column, text in cells.items():
|
|
134
|
+
prop = declared.get(column)
|
|
135
|
+
if prop is None:
|
|
136
|
+
raise AdapterError(f"{where}: unknown column '{column}'")
|
|
137
|
+
values[column] = _convert(prop, text, f"{where}, column '{column}'")
|
|
138
|
+
return values
|
|
139
|
+
|
|
140
|
+
def _nodes(self, file: Path, node_type: NodeType) -> Iterator[Record]:
|
|
141
|
+
for where, cells in self._rows(file):
|
|
142
|
+
values = self._values(node_type.properties, cells, where)
|
|
143
|
+
key = {name: values.pop(name) for name in node_type.key if name in values}
|
|
144
|
+
yield NodeRecord(node_type.name, key, values)
|
|
145
|
+
|
|
146
|
+
def _edges(self, file: Path, edge_type: EdgeType) -> Iterator[Record]:
|
|
147
|
+
only = edge_type.endpoints[0] if len(edge_type.endpoints) == 1 else None
|
|
148
|
+
for where, cells in self._rows(file):
|
|
149
|
+
ends = []
|
|
150
|
+
for role, default in (
|
|
151
|
+
("source", only and only.source),
|
|
152
|
+
("target", only and only.target),
|
|
153
|
+
):
|
|
154
|
+
type_name = cells.pop(f"{role}_type", default)
|
|
155
|
+
declared = self._ontology.node_types.get(type_name or "")
|
|
156
|
+
if declared is None:
|
|
157
|
+
raise AdapterError(f"{where}: '{role}_type' must name a node type")
|
|
158
|
+
prefix = f"{role}."
|
|
159
|
+
raw = {c[len(prefix) :]: cells.pop(c) for c in list(cells) if c.startswith(prefix)}
|
|
160
|
+
ends.append(NodeRef(declared.name, self._values(declared.properties, raw, where)))
|
|
161
|
+
properties = self._values(edge_type.properties, cells, where)
|
|
162
|
+
yield EdgeRecord(edge_type.name, ends[0], ends[1], properties)
|
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
"""Adapter for metadata exported from DataHub.
|
|
2
|
+
|
|
3
|
+
Reads a JSON file holding a list of metadata change proposals, the format DataHub's file
|
|
4
|
+
sink writes. It never calls the DataHub API. Each entry names an entity and one aspect::
|
|
5
|
+
|
|
6
|
+
{"entityType": "dataset",
|
|
7
|
+
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,shop.orders,PROD)",
|
|
8
|
+
"aspectName": "datasetProperties",
|
|
9
|
+
"aspect": {"json": {"description": "All orders."}}}
|
|
10
|
+
|
|
11
|
+
Aspects that are read:
|
|
12
|
+
|
|
13
|
+
========================== ===========================================================
|
|
14
|
+
DataHub aspect govgraph
|
|
15
|
+
========================== ===========================================================
|
|
16
|
+
datasetProperties ``DataAsset`` (table), named by the name part of the URN
|
|
17
|
+
schemaMetadata ``DataAsset`` (column) per field
|
|
18
|
+
ownership ``Owner`` and ``OWNED_BY``
|
|
19
|
+
glossaryTermInfo ``BusinessTerm``
|
|
20
|
+
glossaryTerms ``DEFINES`` (on a dataset or on a schema field)
|
|
21
|
+
upstreamLineage ``DERIVES_FROM`` plus a ``LineageEdge``; fine-grained lineage
|
|
22
|
+
becomes ``DERIVES_FROM`` between column assets
|
|
23
|
+
dataProductProperties ``DataProduct`` and ``DERIVES_FROM``
|
|
24
|
+
========================== ===========================================================
|
|
25
|
+
|
|
26
|
+
Other aspects, and entries in the older snapshot format, are skipped.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
from __future__ import annotations
|
|
30
|
+
|
|
31
|
+
import json
|
|
32
|
+
import os
|
|
33
|
+
from collections.abc import Iterator, Mapping
|
|
34
|
+
from typing import Any
|
|
35
|
+
|
|
36
|
+
from govgraph.adapters.base import asset_ref, column_record, read_json
|
|
37
|
+
from govgraph.errors import AdapterError
|
|
38
|
+
from govgraph.records import EdgeRecord, NodeRecord, NodeRef, Record
|
|
39
|
+
|
|
40
|
+
_DATASET, _FIELD = "urn:li:dataset:(", "urn:li:schemaField:("
|
|
41
|
+
_TERM, _PRODUCT = "urn:li:glossaryTerm:", "urn:li:dataProduct:"
|
|
42
|
+
_OWNER_KINDS = {"corpuser": "person", "corpGroup": "team"}
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def dataset_name(urn: Any) -> str | None:
|
|
46
|
+
"""The name inside ``urn:li:dataset:(<platform urn>,<name>,<environment>)``."""
|
|
47
|
+
if not isinstance(urn, str) or not (urn.startswith(_DATASET) and urn.endswith(")")):
|
|
48
|
+
return None
|
|
49
|
+
parts = urn[len(_DATASET) : -1].split(",")
|
|
50
|
+
return ",".join(parts[1:-1]) or None if len(parts) >= 3 else None
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def field_name(urn: Any) -> str | None:
|
|
54
|
+
"""``<dataset name>.<field path>`` for ``urn:li:schemaField:(<dataset urn>,<path>)``."""
|
|
55
|
+
if not isinstance(urn, str) or not (urn.startswith(_FIELD) and urn.endswith(")")):
|
|
56
|
+
return None
|
|
57
|
+
dataset_urn, _, path = urn[len(_FIELD) : -1].rpartition(",")
|
|
58
|
+
dataset = dataset_name(dataset_urn)
|
|
59
|
+
return f"{dataset}.{path}" if dataset and path else None
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _aspect(entry: Mapping[str, Any]) -> Mapping[str, Any]:
|
|
63
|
+
aspect = entry.get("aspect") or {}
|
|
64
|
+
if isinstance(aspect.get("json"), Mapping):
|
|
65
|
+
body: Mapping[str, Any] = aspect["json"]
|
|
66
|
+
return body
|
|
67
|
+
if isinstance(aspect.get("value"), str):
|
|
68
|
+
try:
|
|
69
|
+
decoded = json.loads(aspect["value"])
|
|
70
|
+
except json.JSONDecodeError as exc:
|
|
71
|
+
raise AdapterError(f"aspect of {entry.get('entityUrn')} is not valid JSON") from exc
|
|
72
|
+
return decoded if isinstance(decoded, Mapping) else {}
|
|
73
|
+
return {}
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
class DataHubAdapter:
|
|
77
|
+
"""Reads metadata change proposals exported from DataHub."""
|
|
78
|
+
|
|
79
|
+
name = "datahub"
|
|
80
|
+
|
|
81
|
+
def __init__(self, path: str | os.PathLike[str]) -> None:
|
|
82
|
+
self._path = path
|
|
83
|
+
|
|
84
|
+
def records(self) -> Iterator[Record]:
|
|
85
|
+
entries = read_json(self._path, "DataHub export")
|
|
86
|
+
if not isinstance(entries, list):
|
|
87
|
+
raise AdapterError("the DataHub export must be a JSON list of change proposals")
|
|
88
|
+
proposals = [
|
|
89
|
+
(entry["entityUrn"], entry["aspectName"], _aspect(entry))
|
|
90
|
+
for entry in entries
|
|
91
|
+
if isinstance(entry, Mapping) and entry.get("entityUrn") and entry.get("aspectName")
|
|
92
|
+
]
|
|
93
|
+
# Terms are referred to by URN, so learn their names before anything points at them.
|
|
94
|
+
self._term_names = {
|
|
95
|
+
urn: aspect.get("name") or urn[len(_TERM) :]
|
|
96
|
+
for urn, name, aspect in proposals
|
|
97
|
+
if name == "glossaryTermInfo" and urn.startswith(_TERM)
|
|
98
|
+
}
|
|
99
|
+
handlers = {
|
|
100
|
+
"datasetProperties": self._dataset_properties,
|
|
101
|
+
"schemaMetadata": self._schema_metadata,
|
|
102
|
+
"ownership": self._ownership,
|
|
103
|
+
"glossaryTermInfo": self._glossary_term_info,
|
|
104
|
+
"glossaryTerms": self._glossary_terms,
|
|
105
|
+
"upstreamLineage": self._upstream_lineage,
|
|
106
|
+
"dataProductProperties": self._data_product,
|
|
107
|
+
}
|
|
108
|
+
for urn, name, aspect in proposals:
|
|
109
|
+
handler = handlers.get(name)
|
|
110
|
+
if handler is not None:
|
|
111
|
+
yield from handler(urn, aspect)
|
|
112
|
+
|
|
113
|
+
def _term(self, urn: Any) -> NodeRecord | None:
|
|
114
|
+
if not isinstance(urn, str) or not urn.startswith(_TERM):
|
|
115
|
+
return None
|
|
116
|
+
return NodeRecord("BusinessTerm", {"name": self._term_names.get(urn) or urn[len(_TERM) :]})
|
|
117
|
+
|
|
118
|
+
def _owned(self, urn: str) -> NodeRef | None:
|
|
119
|
+
"""The node an ownership aspect is attached to, for the entity types we map."""
|
|
120
|
+
dataset = dataset_name(urn)
|
|
121
|
+
if dataset:
|
|
122
|
+
return asset_ref(dataset)
|
|
123
|
+
term = self._term(urn)
|
|
124
|
+
if term is not None:
|
|
125
|
+
return term.ref()
|
|
126
|
+
if urn.startswith(_PRODUCT):
|
|
127
|
+
return NodeRef("DataProduct", {"name": urn[len(_PRODUCT) :]})
|
|
128
|
+
return None
|
|
129
|
+
|
|
130
|
+
@staticmethod
|
|
131
|
+
def _dataset_properties(urn: str, aspect: Mapping[str, Any]) -> Iterator[Record]:
|
|
132
|
+
dataset = dataset_name(urn)
|
|
133
|
+
if dataset:
|
|
134
|
+
yield NodeRecord(
|
|
135
|
+
"DataAsset",
|
|
136
|
+
{"qualified_name": dataset},
|
|
137
|
+
{
|
|
138
|
+
"name": aspect.get("name") or dataset.rsplit(".", 1)[-1],
|
|
139
|
+
"kind": "table",
|
|
140
|
+
"description": aspect.get("description") or None,
|
|
141
|
+
},
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
def _schema_metadata(self, urn: str, aspect: Mapping[str, Any]) -> Iterator[Record]:
|
|
145
|
+
dataset = dataset_name(urn)
|
|
146
|
+
if not dataset:
|
|
147
|
+
return
|
|
148
|
+
yield NodeRecord("DataAsset", {"qualified_name": dataset}, {"kind": "table"})
|
|
149
|
+
for field in aspect.get("fields") or []:
|
|
150
|
+
path = field.get("fieldPath")
|
|
151
|
+
if not path:
|
|
152
|
+
continue
|
|
153
|
+
qualified_name = f"{dataset}.{path}"
|
|
154
|
+
yield NodeRecord(
|
|
155
|
+
"DataAsset",
|
|
156
|
+
{"qualified_name": qualified_name},
|
|
157
|
+
{
|
|
158
|
+
"name": path,
|
|
159
|
+
"kind": "column",
|
|
160
|
+
"parent_qualified_name": dataset,
|
|
161
|
+
"data_type": field.get("nativeDataType"),
|
|
162
|
+
"description": field.get("description") or None,
|
|
163
|
+
},
|
|
164
|
+
)
|
|
165
|
+
yield from self._defines(field.get("glossaryTerms"), asset_ref(qualified_name))
|
|
166
|
+
|
|
167
|
+
def _defines(self, terms: Any, asset: NodeRef) -> Iterator[Record]:
|
|
168
|
+
for association in (terms or {}).get("terms") or []:
|
|
169
|
+
term = self._term(association.get("urn"))
|
|
170
|
+
if term is not None:
|
|
171
|
+
yield term
|
|
172
|
+
yield EdgeRecord("DEFINES", term.ref(), asset)
|
|
173
|
+
|
|
174
|
+
def _glossary_terms(self, urn: str, aspect: Mapping[str, Any]) -> Iterator[Record]:
|
|
175
|
+
dataset = dataset_name(urn)
|
|
176
|
+
if dataset:
|
|
177
|
+
yield NodeRecord("DataAsset", {"qualified_name": dataset})
|
|
178
|
+
yield from self._defines(aspect, asset_ref(dataset))
|
|
179
|
+
|
|
180
|
+
def _glossary_term_info(self, urn: str, aspect: Mapping[str, Any]) -> Iterator[Record]:
|
|
181
|
+
term = self._term(urn)
|
|
182
|
+
if term is not None:
|
|
183
|
+
yield NodeRecord(
|
|
184
|
+
"BusinessTerm", term.key, {"definition": aspect.get("definition") or None}
|
|
185
|
+
)
|
|
186
|
+
|
|
187
|
+
def _ownership(self, urn: str, aspect: Mapping[str, Any]) -> Iterator[Record]:
|
|
188
|
+
owned = self._owned(urn)
|
|
189
|
+
if owned is None:
|
|
190
|
+
return
|
|
191
|
+
yield NodeRecord(owned.type, owned.key)
|
|
192
|
+
for entry in aspect.get("owners") or []:
|
|
193
|
+
owner_urn = str(entry.get("owner") or "")
|
|
194
|
+
parts = owner_urn.split(":", 3)
|
|
195
|
+
if len(parts) != 4 or parts[:2] != ["urn", "li"] or not parts[3]:
|
|
196
|
+
continue
|
|
197
|
+
owner = NodeRecord(
|
|
198
|
+
"Owner",
|
|
199
|
+
{"identifier": owner_urn},
|
|
200
|
+
{"name": parts[3], "kind": _OWNER_KINDS.get(parts[2])},
|
|
201
|
+
)
|
|
202
|
+
yield owner
|
|
203
|
+
yield EdgeRecord("OWNED_BY", owned, owner.ref())
|
|
204
|
+
|
|
205
|
+
@staticmethod
|
|
206
|
+
def _upstream_lineage(urn: str, aspect: Mapping[str, Any]) -> Iterator[Record]:
|
|
207
|
+
downstream = dataset_name(urn)
|
|
208
|
+
if not downstream:
|
|
209
|
+
return
|
|
210
|
+
yield NodeRecord("DataAsset", {"qualified_name": downstream})
|
|
211
|
+
for entry in aspect.get("upstreams") or []:
|
|
212
|
+
upstream = dataset_name(entry.get("dataset"))
|
|
213
|
+
if not upstream:
|
|
214
|
+
continue
|
|
215
|
+
record = NodeRecord(
|
|
216
|
+
"LineageEdge",
|
|
217
|
+
{
|
|
218
|
+
"source_qualified_name": upstream,
|
|
219
|
+
"target_qualified_name": downstream,
|
|
220
|
+
"process": str(entry.get("query") or entry.get("type") or "lineage").lower(),
|
|
221
|
+
},
|
|
222
|
+
)
|
|
223
|
+
yield NodeRecord("DataAsset", {"qualified_name": upstream})
|
|
224
|
+
yield record
|
|
225
|
+
yield EdgeRecord(
|
|
226
|
+
"DERIVES_FROM",
|
|
227
|
+
asset_ref(downstream),
|
|
228
|
+
asset_ref(upstream),
|
|
229
|
+
{"lineage_id": record.id},
|
|
230
|
+
)
|
|
231
|
+
for fine in aspect.get("fineGrainedLineages") or []:
|
|
232
|
+
sources = [field_name(u) for u in fine.get("upstreams") or []]
|
|
233
|
+
targets = [field_name(u) for u in fine.get("downstreams") or []]
|
|
234
|
+
for target in filter(None, targets):
|
|
235
|
+
for source in filter(None, sources):
|
|
236
|
+
yield column_record(source)
|
|
237
|
+
yield column_record(target)
|
|
238
|
+
yield EdgeRecord("DERIVES_FROM", asset_ref(target), asset_ref(source))
|
|
239
|
+
|
|
240
|
+
@staticmethod
|
|
241
|
+
def _data_product(urn: str, aspect: Mapping[str, Any]) -> Iterator[Record]:
|
|
242
|
+
if not urn.startswith(_PRODUCT):
|
|
243
|
+
return
|
|
244
|
+
# Keyed by the URN's ID so that other aspects of the product find the same node.
|
|
245
|
+
product = NodeRecord(
|
|
246
|
+
"DataProduct",
|
|
247
|
+
{"name": urn[len(_PRODUCT) :]},
|
|
248
|
+
{"description": aspect.get("description") or aspect.get("name")},
|
|
249
|
+
)
|
|
250
|
+
yield product
|
|
251
|
+
for asset in aspect.get("assets") or []:
|
|
252
|
+
dataset = dataset_name(asset.get("destinationUrn"))
|
|
253
|
+
if dataset:
|
|
254
|
+
yield NodeRecord("DataAsset", {"qualified_name": dataset})
|
|
255
|
+
yield EdgeRecord("DERIVES_FROM", product.ref(), asset_ref(dataset))
|