govgraph 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. govgraph/__init__.py +55 -0
  2. govgraph/__main__.py +3 -0
  3. govgraph/adapters/__init__.py +6 -0
  4. govgraph/adapters/base.py +49 -0
  5. govgraph/adapters/csv_json.py +162 -0
  6. govgraph/adapters/datahub.py +255 -0
  7. govgraph/adapters/dbt.py +264 -0
  8. govgraph/adapters/openmetadata.py +290 -0
  9. govgraph/adapters/registry.py +51 -0
  10. govgraph/cli.py +357 -0
  11. govgraph/engines/__init__.py +5 -0
  12. govgraph/engines/puppygraph.py +150 -0
  13. govgraph/errors.py +58 -0
  14. govgraph/export.py +67 -0
  15. govgraph/ids.py +63 -0
  16. govgraph/ingest.py +95 -0
  17. govgraph/mcp/__init__.py +5 -0
  18. govgraph/mcp/server.py +110 -0
  19. govgraph/py.typed +0 -0
  20. govgraph/records.py +97 -0
  21. govgraph/samples.py +337 -0
  22. govgraph/spec/__init__.py +6 -0
  23. govgraph/spec/docs.py +71 -0
  24. govgraph/spec/loader.py +301 -0
  25. govgraph/spec/model.py +177 -0
  26. govgraph/spec/ontologies/core-1.0.yaml +228 -0
  27. govgraph/spec/ontology.schema.json +91 -0
  28. govgraph/store/__init__.py +14 -0
  29. govgraph/store/base.py +104 -0
  30. govgraph/store/ddl.py +93 -0
  31. govgraph/store/memory.py +134 -0
  32. govgraph/store/postgres.py +373 -0
  33. govgraph/store/queries.py +168 -0
  34. govgraph/tools/__init__.py +18 -0
  35. govgraph/tools/context.py +203 -0
  36. govgraph/tools/limits.py +38 -0
  37. govgraph/tools/results.py +125 -0
  38. govgraph/tools/schemas.py +113 -0
  39. govgraph/tools/toolkit.py +603 -0
  40. govgraph/validation.py +157 -0
  41. govgraph/viewer/__init__.py +32 -0
  42. govgraph/viewer/viewer.html +84 -0
  43. govgraph/viewer/viewer.js +476 -0
  44. govgraph-0.1.0.dist-info/METADATA +265 -0
  45. govgraph-0.1.0.dist-info/RECORD +48 -0
  46. govgraph-0.1.0.dist-info/WHEEL +4 -0
  47. govgraph-0.1.0.dist-info/entry_points.txt +9 -0
  48. govgraph-0.1.0.dist-info/licenses/LICENSE +202 -0
govgraph/__init__.py ADDED
@@ -0,0 +1,55 @@
1
+ """govgraph: data-governance metadata as a typed knowledge graph with bounded agent tools."""
2
+
3
+ from govgraph.errors import (
4
+ AdapterError,
5
+ GovGraphError,
6
+ IngestError,
7
+ OntologyError,
8
+ RecordError,
9
+ StoreError,
10
+ StoreTimeoutError,
11
+ ToolError,
12
+ UnsafeRoleError,
13
+ )
14
+ from govgraph.ids import edge_id, node_id
15
+ from govgraph.ingest import IngestReport, ingest
16
+ from govgraph.records import Edge, EdgeRecord, Node, NodeRecord, NodeRef, Record
17
+ from govgraph.spec import Ontology, load_ontology
18
+ from govgraph.store import GraphReader, GraphStore, MemoryStore, PostgresStore
19
+ from govgraph.tools import AuditEvent, Claim, Limits, Toolkit, ToolResult
20
+
21
+ __version__ = "0.1.0"
22
+
23
+ __all__ = [
24
+ "AdapterError",
25
+ "AuditEvent",
26
+ "Claim",
27
+ "Edge",
28
+ "EdgeRecord",
29
+ "GovGraphError",
30
+ "GraphReader",
31
+ "GraphStore",
32
+ "IngestError",
33
+ "IngestReport",
34
+ "Limits",
35
+ "MemoryStore",
36
+ "Node",
37
+ "NodeRecord",
38
+ "NodeRef",
39
+ "Ontology",
40
+ "OntologyError",
41
+ "PostgresStore",
42
+ "Record",
43
+ "RecordError",
44
+ "StoreError",
45
+ "StoreTimeoutError",
46
+ "ToolError",
47
+ "ToolResult",
48
+ "Toolkit",
49
+ "UnsafeRoleError",
50
+ "__version__",
51
+ "edge_id",
52
+ "ingest",
53
+ "load_ontology",
54
+ "node_id",
55
+ ]
govgraph/__main__.py ADDED
@@ -0,0 +1,3 @@
1
+ from govgraph.cli import main
2
+
3
+ raise SystemExit(main())
@@ -0,0 +1,6 @@
1
+ """Adapters turn metadata from a source system into normalized records."""
2
+
3
+ from govgraph.adapters.base import Adapter
4
+ from govgraph.adapters.registry import ENTRY_POINT_GROUP, available, load
5
+
6
+ __all__ = ["ENTRY_POINT_GROUP", "Adapter", "available", "load"]
@@ -0,0 +1,49 @@
1
+ """The adapter interface."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import os
7
+ from collections.abc import Iterator
8
+ from typing import Any, Protocol, runtime_checkable
9
+
10
+ from govgraph.errors import AdapterError
11
+ from govgraph.records import NodeRecord, NodeRef, Record
12
+
13
+
14
+ @runtime_checkable
15
+ class Adapter(Protocol):
16
+ """Turns metadata from one source into normalized records.
17
+
18
+ An adapter only yields records. It is never given a store, so it cannot write to
19
+ one; :func:`govgraph.ingest` is the single path from an adapter to a store.
20
+ """
21
+
22
+ name: str
23
+
24
+ def records(self) -> Iterator[Record]: ...
25
+
26
+
27
+ def read_json(path: str | os.PathLike[str], what: str) -> Any:
28
+ """Parse a JSON file, raising :class:`AdapterError` with a readable message."""
29
+ try:
30
+ with open(path, encoding="utf-8") as handle:
31
+ return json.load(handle)
32
+ except OSError as exc:
33
+ raise AdapterError(f"cannot read {what} {os.fspath(path)}: {exc.strerror}") from exc
34
+ except json.JSONDecodeError as exc:
35
+ raise AdapterError(f"{what} {os.fspath(path)} is not valid JSON: {exc}") from exc
36
+
37
+
38
+ def asset_ref(qualified_name: str) -> NodeRef:
39
+ return NodeRef("DataAsset", {"qualified_name": qualified_name})
40
+
41
+
42
+ def column_record(qualified_name: str, **properties: Any) -> NodeRecord:
43
+ """A column asset whose parent is everything before the last dot of its name."""
44
+ parent, _, name = qualified_name.rpartition(".")
45
+ return NodeRecord(
46
+ "DataAsset",
47
+ {"qualified_name": qualified_name},
48
+ {"name": name, "kind": "column", "parent_qualified_name": parent or None, **properties},
49
+ )
@@ -0,0 +1,162 @@
1
+ """Adapters for plain files: one JSON document, or a directory of CSV files.
2
+
3
+ JSON document::
4
+
5
+ {"nodes": [{"type": "Owner", "key": {"identifier": "a"}, "properties": {"name": "A"}}],
6
+ "edges": [{"type": "OWNED_BY",
7
+ "source": {"type": "DataAsset", "key": {"qualified_name": "shop.orders"}},
8
+ "target": {"type": "Owner", "key": {"identifier": "a"}}}]}
9
+
10
+ CSV directory: one file per type, named ``<Type>.csv``. A node file has one column per
11
+ property. An edge file has ``source.<key field>`` and ``target.<key field>`` columns, plus
12
+ ``source_type`` and ``target_type`` when the edge type allows more than one pair of node
13
+ types, plus one column per edge property. Empty cells mean "not set".
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import csv
19
+ import os
20
+ from collections.abc import Iterator, Mapping
21
+ from pathlib import Path
22
+ from typing import Any
23
+
24
+ from govgraph.adapters.base import read_json
25
+ from govgraph.errors import AdapterError
26
+ from govgraph.records import EdgeRecord, NodeRecord, NodeRef, Record
27
+ from govgraph.spec.loader import load_ontology
28
+ from govgraph.spec.model import EdgeType, NodeType, Ontology, Property
29
+
30
+ _TRUE, _FALSE = {"true", "t", "yes", "y", "1"}, {"false", "f", "no", "n", "0"}
31
+
32
+
33
+ def _mapping(value: Any, where: str) -> Mapping[str, Any]:
34
+ if not isinstance(value, Mapping):
35
+ raise AdapterError(f"{where} must be an object")
36
+ return value
37
+
38
+
39
+ def _ref(value: Any, where: str) -> NodeRef:
40
+ ref = _mapping(value, where)
41
+ if not isinstance(ref.get("type"), str):
42
+ raise AdapterError(f"{where} needs a 'type'")
43
+ return NodeRef(ref["type"], dict(_mapping(ref.get("key"), f"{where}.key")))
44
+
45
+
46
+ class JsonAdapter:
47
+ """Reads nodes and edges from one JSON document."""
48
+
49
+ name = "json"
50
+
51
+ def __init__(self, path: str | os.PathLike[str]) -> None:
52
+ self._path = path
53
+
54
+ def records(self) -> Iterator[Record]:
55
+ document = _mapping(read_json(self._path, "JSON file"), "the document")
56
+ for index, raw in enumerate(document.get("nodes", [])):
57
+ where = f"nodes[{index}]"
58
+ node = _ref(raw, where)
59
+ properties = _mapping(raw.get("properties", {}), f"{where}.properties")
60
+ yield NodeRecord(node.type, node.key, dict(properties))
61
+ for index, raw in enumerate(document.get("edges", [])):
62
+ where = f"edges[{index}]"
63
+ edge = _mapping(raw, where)
64
+ if not isinstance(edge.get("type"), str):
65
+ raise AdapterError(f"{where} needs a 'type'")
66
+ yield EdgeRecord(
67
+ edge["type"],
68
+ _ref(edge.get("source"), f"{where}.source"),
69
+ _ref(edge.get("target"), f"{where}.target"),
70
+ dict(_mapping(edge.get("properties", {}), f"{where}.properties")),
71
+ )
72
+
73
+
74
+ def _convert(prop: Property, text: str, where: str) -> Any:
75
+ """Turn a CSV cell into the Python type the property declares."""
76
+ try:
77
+ if prop.type == "integer":
78
+ return int(text)
79
+ if prop.type == "number":
80
+ return float(text)
81
+ except ValueError:
82
+ raise AdapterError(f"{where}: {text!r} is not a valid {prop.type}") from None
83
+ if prop.type == "boolean":
84
+ lowered = text.strip().lower()
85
+ if lowered in _TRUE:
86
+ return True
87
+ if lowered in _FALSE:
88
+ return False
89
+ raise AdapterError(f"{where}: {text!r} is not a valid boolean")
90
+ return text
91
+
92
+
93
+ class CsvAdapter:
94
+ """Reads nodes and edges from a directory of ``<Type>.csv`` files."""
95
+
96
+ name = "csv"
97
+
98
+ def __init__(self, path: str | os.PathLike[str], ontology: Ontology | None = None) -> None:
99
+ self._path = Path(path)
100
+ self._ontology = ontology or load_ontology()
101
+
102
+ def records(self) -> Iterator[Record]:
103
+ if not self._path.is_dir():
104
+ raise AdapterError(f"{self._path} is not a directory")
105
+ files = sorted(self._path.glob("*.csv"))
106
+ unknown = [
107
+ file.name
108
+ for file in files
109
+ if file.stem not in self._ontology.node_types
110
+ and file.stem not in self._ontology.edge_types
111
+ ]
112
+ if unknown:
113
+ raise AdapterError(f"no node or edge type is named after: {', '.join(unknown)}")
114
+ # Nodes first, so that edges in the same directory find their endpoints.
115
+ for file in files:
116
+ if file.stem in self._ontology.node_types:
117
+ yield from self._nodes(file, self._ontology.node_types[file.stem])
118
+ for file in files:
119
+ if file.stem in self._ontology.edge_types:
120
+ yield from self._edges(file, self._ontology.edge_types[file.stem])
121
+
122
+ @staticmethod
123
+ def _rows(file: Path) -> Iterator[tuple[str, dict[str, str]]]:
124
+ with file.open(newline="", encoding="utf-8-sig") as handle:
125
+ for number, row in enumerate(csv.DictReader(handle), start=2):
126
+ cells = {k: v for k, v in row.items() if k is not None and v not in (None, "")}
127
+ yield f"{file.name} line {number}", cells
128
+
129
+ def _values(
130
+ self, declared: Mapping[str, Property], cells: Mapping[str, str], where: str
131
+ ) -> dict[str, Any]:
132
+ values = {}
133
+ for column, text in cells.items():
134
+ prop = declared.get(column)
135
+ if prop is None:
136
+ raise AdapterError(f"{where}: unknown column '{column}'")
137
+ values[column] = _convert(prop, text, f"{where}, column '{column}'")
138
+ return values
139
+
140
+ def _nodes(self, file: Path, node_type: NodeType) -> Iterator[Record]:
141
+ for where, cells in self._rows(file):
142
+ values = self._values(node_type.properties, cells, where)
143
+ key = {name: values.pop(name) for name in node_type.key if name in values}
144
+ yield NodeRecord(node_type.name, key, values)
145
+
146
+ def _edges(self, file: Path, edge_type: EdgeType) -> Iterator[Record]:
147
+ only = edge_type.endpoints[0] if len(edge_type.endpoints) == 1 else None
148
+ for where, cells in self._rows(file):
149
+ ends = []
150
+ for role, default in (
151
+ ("source", only and only.source),
152
+ ("target", only and only.target),
153
+ ):
154
+ type_name = cells.pop(f"{role}_type", default)
155
+ declared = self._ontology.node_types.get(type_name or "")
156
+ if declared is None:
157
+ raise AdapterError(f"{where}: '{role}_type' must name a node type")
158
+ prefix = f"{role}."
159
+ raw = {c[len(prefix) :]: cells.pop(c) for c in list(cells) if c.startswith(prefix)}
160
+ ends.append(NodeRef(declared.name, self._values(declared.properties, raw, where)))
161
+ properties = self._values(edge_type.properties, cells, where)
162
+ yield EdgeRecord(edge_type.name, ends[0], ends[1], properties)
@@ -0,0 +1,255 @@
1
+ """Adapter for metadata exported from DataHub.
2
+
3
+ Reads a JSON file holding a list of metadata change proposals, the format DataHub's file
4
+ sink writes. It never calls the DataHub API. Each entry names an entity and one aspect::
5
+
6
+ {"entityType": "dataset",
7
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,shop.orders,PROD)",
8
+ "aspectName": "datasetProperties",
9
+ "aspect": {"json": {"description": "All orders."}}}
10
+
11
+ Aspects that are read:
12
+
13
+ ========================== ===========================================================
14
+ DataHub aspect govgraph
15
+ ========================== ===========================================================
16
+ datasetProperties ``DataAsset`` (table), named by the name part of the URN
17
+ schemaMetadata ``DataAsset`` (column) per field
18
+ ownership ``Owner`` and ``OWNED_BY``
19
+ glossaryTermInfo ``BusinessTerm``
20
+ glossaryTerms ``DEFINES`` (on a dataset or on a schema field)
21
+ upstreamLineage ``DERIVES_FROM`` plus a ``LineageEdge``; fine-grained lineage
22
+ becomes ``DERIVES_FROM`` between column assets
23
+ dataProductProperties ``DataProduct`` and ``DERIVES_FROM``
24
+ ========================== ===========================================================
25
+
26
+ Other aspects, and entries in the older snapshot format, are skipped.
27
+ """
28
+
29
+ from __future__ import annotations
30
+
31
+ import json
32
+ import os
33
+ from collections.abc import Iterator, Mapping
34
+ from typing import Any
35
+
36
+ from govgraph.adapters.base import asset_ref, column_record, read_json
37
+ from govgraph.errors import AdapterError
38
+ from govgraph.records import EdgeRecord, NodeRecord, NodeRef, Record
39
+
40
+ _DATASET, _FIELD = "urn:li:dataset:(", "urn:li:schemaField:("
41
+ _TERM, _PRODUCT = "urn:li:glossaryTerm:", "urn:li:dataProduct:"
42
+ _OWNER_KINDS = {"corpuser": "person", "corpGroup": "team"}
43
+
44
+
45
+ def dataset_name(urn: Any) -> str | None:
46
+ """The name inside ``urn:li:dataset:(<platform urn>,<name>,<environment>)``."""
47
+ if not isinstance(urn, str) or not (urn.startswith(_DATASET) and urn.endswith(")")):
48
+ return None
49
+ parts = urn[len(_DATASET) : -1].split(",")
50
+ return ",".join(parts[1:-1]) or None if len(parts) >= 3 else None
51
+
52
+
53
+ def field_name(urn: Any) -> str | None:
54
+ """``<dataset name>.<field path>`` for ``urn:li:schemaField:(<dataset urn>,<path>)``."""
55
+ if not isinstance(urn, str) or not (urn.startswith(_FIELD) and urn.endswith(")")):
56
+ return None
57
+ dataset_urn, _, path = urn[len(_FIELD) : -1].rpartition(",")
58
+ dataset = dataset_name(dataset_urn)
59
+ return f"{dataset}.{path}" if dataset and path else None
60
+
61
+
62
+ def _aspect(entry: Mapping[str, Any]) -> Mapping[str, Any]:
63
+ aspect = entry.get("aspect") or {}
64
+ if isinstance(aspect.get("json"), Mapping):
65
+ body: Mapping[str, Any] = aspect["json"]
66
+ return body
67
+ if isinstance(aspect.get("value"), str):
68
+ try:
69
+ decoded = json.loads(aspect["value"])
70
+ except json.JSONDecodeError as exc:
71
+ raise AdapterError(f"aspect of {entry.get('entityUrn')} is not valid JSON") from exc
72
+ return decoded if isinstance(decoded, Mapping) else {}
73
+ return {}
74
+
75
+
76
+ class DataHubAdapter:
77
+ """Reads metadata change proposals exported from DataHub."""
78
+
79
+ name = "datahub"
80
+
81
+ def __init__(self, path: str | os.PathLike[str]) -> None:
82
+ self._path = path
83
+
84
+ def records(self) -> Iterator[Record]:
85
+ entries = read_json(self._path, "DataHub export")
86
+ if not isinstance(entries, list):
87
+ raise AdapterError("the DataHub export must be a JSON list of change proposals")
88
+ proposals = [
89
+ (entry["entityUrn"], entry["aspectName"], _aspect(entry))
90
+ for entry in entries
91
+ if isinstance(entry, Mapping) and entry.get("entityUrn") and entry.get("aspectName")
92
+ ]
93
+ # Terms are referred to by URN, so learn their names before anything points at them.
94
+ self._term_names = {
95
+ urn: aspect.get("name") or urn[len(_TERM) :]
96
+ for urn, name, aspect in proposals
97
+ if name == "glossaryTermInfo" and urn.startswith(_TERM)
98
+ }
99
+ handlers = {
100
+ "datasetProperties": self._dataset_properties,
101
+ "schemaMetadata": self._schema_metadata,
102
+ "ownership": self._ownership,
103
+ "glossaryTermInfo": self._glossary_term_info,
104
+ "glossaryTerms": self._glossary_terms,
105
+ "upstreamLineage": self._upstream_lineage,
106
+ "dataProductProperties": self._data_product,
107
+ }
108
+ for urn, name, aspect in proposals:
109
+ handler = handlers.get(name)
110
+ if handler is not None:
111
+ yield from handler(urn, aspect)
112
+
113
+ def _term(self, urn: Any) -> NodeRecord | None:
114
+ if not isinstance(urn, str) or not urn.startswith(_TERM):
115
+ return None
116
+ return NodeRecord("BusinessTerm", {"name": self._term_names.get(urn) or urn[len(_TERM) :]})
117
+
118
+ def _owned(self, urn: str) -> NodeRef | None:
119
+ """The node an ownership aspect is attached to, for the entity types we map."""
120
+ dataset = dataset_name(urn)
121
+ if dataset:
122
+ return asset_ref(dataset)
123
+ term = self._term(urn)
124
+ if term is not None:
125
+ return term.ref()
126
+ if urn.startswith(_PRODUCT):
127
+ return NodeRef("DataProduct", {"name": urn[len(_PRODUCT) :]})
128
+ return None
129
+
130
+ @staticmethod
131
+ def _dataset_properties(urn: str, aspect: Mapping[str, Any]) -> Iterator[Record]:
132
+ dataset = dataset_name(urn)
133
+ if dataset:
134
+ yield NodeRecord(
135
+ "DataAsset",
136
+ {"qualified_name": dataset},
137
+ {
138
+ "name": aspect.get("name") or dataset.rsplit(".", 1)[-1],
139
+ "kind": "table",
140
+ "description": aspect.get("description") or None,
141
+ },
142
+ )
143
+
144
+ def _schema_metadata(self, urn: str, aspect: Mapping[str, Any]) -> Iterator[Record]:
145
+ dataset = dataset_name(urn)
146
+ if not dataset:
147
+ return
148
+ yield NodeRecord("DataAsset", {"qualified_name": dataset}, {"kind": "table"})
149
+ for field in aspect.get("fields") or []:
150
+ path = field.get("fieldPath")
151
+ if not path:
152
+ continue
153
+ qualified_name = f"{dataset}.{path}"
154
+ yield NodeRecord(
155
+ "DataAsset",
156
+ {"qualified_name": qualified_name},
157
+ {
158
+ "name": path,
159
+ "kind": "column",
160
+ "parent_qualified_name": dataset,
161
+ "data_type": field.get("nativeDataType"),
162
+ "description": field.get("description") or None,
163
+ },
164
+ )
165
+ yield from self._defines(field.get("glossaryTerms"), asset_ref(qualified_name))
166
+
167
+ def _defines(self, terms: Any, asset: NodeRef) -> Iterator[Record]:
168
+ for association in (terms or {}).get("terms") or []:
169
+ term = self._term(association.get("urn"))
170
+ if term is not None:
171
+ yield term
172
+ yield EdgeRecord("DEFINES", term.ref(), asset)
173
+
174
+ def _glossary_terms(self, urn: str, aspect: Mapping[str, Any]) -> Iterator[Record]:
175
+ dataset = dataset_name(urn)
176
+ if dataset:
177
+ yield NodeRecord("DataAsset", {"qualified_name": dataset})
178
+ yield from self._defines(aspect, asset_ref(dataset))
179
+
180
+ def _glossary_term_info(self, urn: str, aspect: Mapping[str, Any]) -> Iterator[Record]:
181
+ term = self._term(urn)
182
+ if term is not None:
183
+ yield NodeRecord(
184
+ "BusinessTerm", term.key, {"definition": aspect.get("definition") or None}
185
+ )
186
+
187
+ def _ownership(self, urn: str, aspect: Mapping[str, Any]) -> Iterator[Record]:
188
+ owned = self._owned(urn)
189
+ if owned is None:
190
+ return
191
+ yield NodeRecord(owned.type, owned.key)
192
+ for entry in aspect.get("owners") or []:
193
+ owner_urn = str(entry.get("owner") or "")
194
+ parts = owner_urn.split(":", 3)
195
+ if len(parts) != 4 or parts[:2] != ["urn", "li"] or not parts[3]:
196
+ continue
197
+ owner = NodeRecord(
198
+ "Owner",
199
+ {"identifier": owner_urn},
200
+ {"name": parts[3], "kind": _OWNER_KINDS.get(parts[2])},
201
+ )
202
+ yield owner
203
+ yield EdgeRecord("OWNED_BY", owned, owner.ref())
204
+
205
+ @staticmethod
206
+ def _upstream_lineage(urn: str, aspect: Mapping[str, Any]) -> Iterator[Record]:
207
+ downstream = dataset_name(urn)
208
+ if not downstream:
209
+ return
210
+ yield NodeRecord("DataAsset", {"qualified_name": downstream})
211
+ for entry in aspect.get("upstreams") or []:
212
+ upstream = dataset_name(entry.get("dataset"))
213
+ if not upstream:
214
+ continue
215
+ record = NodeRecord(
216
+ "LineageEdge",
217
+ {
218
+ "source_qualified_name": upstream,
219
+ "target_qualified_name": downstream,
220
+ "process": str(entry.get("query") or entry.get("type") or "lineage").lower(),
221
+ },
222
+ )
223
+ yield NodeRecord("DataAsset", {"qualified_name": upstream})
224
+ yield record
225
+ yield EdgeRecord(
226
+ "DERIVES_FROM",
227
+ asset_ref(downstream),
228
+ asset_ref(upstream),
229
+ {"lineage_id": record.id},
230
+ )
231
+ for fine in aspect.get("fineGrainedLineages") or []:
232
+ sources = [field_name(u) for u in fine.get("upstreams") or []]
233
+ targets = [field_name(u) for u in fine.get("downstreams") or []]
234
+ for target in filter(None, targets):
235
+ for source in filter(None, sources):
236
+ yield column_record(source)
237
+ yield column_record(target)
238
+ yield EdgeRecord("DERIVES_FROM", asset_ref(target), asset_ref(source))
239
+
240
+ @staticmethod
241
+ def _data_product(urn: str, aspect: Mapping[str, Any]) -> Iterator[Record]:
242
+ if not urn.startswith(_PRODUCT):
243
+ return
244
+ # Keyed by the URN's ID so that other aspects of the product find the same node.
245
+ product = NodeRecord(
246
+ "DataProduct",
247
+ {"name": urn[len(_PRODUCT) :]},
248
+ {"description": aspect.get("description") or aspect.get("name")},
249
+ )
250
+ yield product
251
+ for asset in aspect.get("assets") or []:
252
+ dataset = dataset_name(asset.get("destinationUrn"))
253
+ if dataset:
254
+ yield NodeRecord("DataAsset", {"qualified_name": dataset})
255
+ yield EdgeRecord("DERIVES_FROM", product.ref(), asset_ref(dataset))