pipeloom-openlineage 0.9.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pipeloom_openlineage-0.9.0/.gitignore +18 -0
- pipeloom_openlineage-0.9.0/LICENSE +202 -0
- pipeloom_openlineage-0.9.0/NOTICE +2 -0
- pipeloom_openlineage-0.9.0/PKG-INFO +65 -0
- pipeloom_openlineage-0.9.0/README.md +39 -0
- pipeloom_openlineage-0.9.0/pyproject.toml +44 -0
- pipeloom_openlineage-0.9.0/src/pipeloom_openlineage/__init__.py +1 -0
- pipeloom_openlineage-0.9.0/src/pipeloom_openlineage/py.typed +0 -0
- pipeloom_openlineage-0.9.0/src/pipeloom_openlineage/sink.py +527 -0
- pipeloom_openlineage-0.9.0/src/pipeloom_openlineage/spec.py +43 -0
- pipeloom_openlineage-0.9.0/tests/test_sink.py +317 -0
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
__pycache__/
|
|
2
|
+
*.py[cod]
|
|
3
|
+
.venv/
|
|
4
|
+
build/
|
|
5
|
+
dist/
|
|
6
|
+
*.egg-info/
|
|
7
|
+
.pytest_cache/
|
|
8
|
+
.mypy_cache/
|
|
9
|
+
.ruff_cache/
|
|
10
|
+
.coverage
|
|
11
|
+
coverage.xml
|
|
12
|
+
htmlcov/
|
|
13
|
+
site/
|
|
14
|
+
target/
|
|
15
|
+
.pipeloom/
|
|
16
|
+
.env
|
|
17
|
+
examples/data/
|
|
18
|
+
examples/airflow/include/wheels/
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
|
|
2
|
+
Apache License
|
|
3
|
+
Version 2.0, January 2004
|
|
4
|
+
http://www.apache.org/licenses/
|
|
5
|
+
|
|
6
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
7
|
+
|
|
8
|
+
1. Definitions.
|
|
9
|
+
|
|
10
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
11
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
12
|
+
|
|
13
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
14
|
+
the copyright owner that is granting the License.
|
|
15
|
+
|
|
16
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
17
|
+
other entities that control, are controlled by, or are under common
|
|
18
|
+
control with that entity. For the purposes of this definition,
|
|
19
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
20
|
+
direction or management of such entity, whether by contract or
|
|
21
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
22
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
23
|
+
|
|
24
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
25
|
+
exercising permissions granted by this License.
|
|
26
|
+
|
|
27
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
28
|
+
including but not limited to software source code, documentation
|
|
29
|
+
source, and configuration files.
|
|
30
|
+
|
|
31
|
+
"Object" form shall mean any form resulting from mechanical
|
|
32
|
+
transformation or translation of a Source form, including but
|
|
33
|
+
not limited to compiled object code, generated documentation,
|
|
34
|
+
and conversions to other media types.
|
|
35
|
+
|
|
36
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
37
|
+
Object form, made available under the License, as indicated by a
|
|
38
|
+
copyright notice that is included in or attached to the work
|
|
39
|
+
(an example is provided in the Appendix below).
|
|
40
|
+
|
|
41
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
42
|
+
form, that is based on (or derived from) the Work and for which the
|
|
43
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
44
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
45
|
+
of this License, Derivative Works shall not include works that remain
|
|
46
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
47
|
+
the Work and Derivative Works thereof.
|
|
48
|
+
|
|
49
|
+
"Contribution" shall mean any work of authorship, including
|
|
50
|
+
the original version of the Work and any modifications or additions
|
|
51
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
52
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
53
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
54
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
55
|
+
means any form of electronic, verbal, or written communication sent
|
|
56
|
+
to the Licensor or its representatives, including but not limited to
|
|
57
|
+
communication on electronic mailing lists, source code control systems,
|
|
58
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
59
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
60
|
+
excluding communication that is conspicuously marked or otherwise
|
|
61
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
62
|
+
|
|
63
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
64
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
65
|
+
subsequently incorporated within the Work.
|
|
66
|
+
|
|
67
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
68
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
69
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
70
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
71
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
72
|
+
Work and such Derivative Works in Source or Object form.
|
|
73
|
+
|
|
74
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
75
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
76
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
77
|
+
(except as stated in this section) patent license to make, have made,
|
|
78
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
79
|
+
where such license applies only to those patent claims licensable
|
|
80
|
+
by such Contributor that are necessarily infringed by their
|
|
81
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
82
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
83
|
+
institute patent litigation against any entity (including a
|
|
84
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
85
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
86
|
+
or contributory patent infringement, then any patent licenses
|
|
87
|
+
granted to You under this License for that Work shall terminate
|
|
88
|
+
as of the date such litigation is filed.
|
|
89
|
+
|
|
90
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
91
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
92
|
+
modifications, and in Source or Object form, provided that You
|
|
93
|
+
meet the following conditions:
|
|
94
|
+
|
|
95
|
+
(a) You must give any other recipients of the Work or
|
|
96
|
+
Derivative Works a copy of this License; and
|
|
97
|
+
|
|
98
|
+
(b) You must cause any modified files to carry prominent notices
|
|
99
|
+
stating that You changed the files; and
|
|
100
|
+
|
|
101
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
102
|
+
that You distribute, all copyright, patent, trademark, and
|
|
103
|
+
attribution notices from the Source form of the Work,
|
|
104
|
+
excluding those notices that do not pertain to any part of
|
|
105
|
+
the Derivative Works; and
|
|
106
|
+
|
|
107
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
108
|
+
distribution, then any Derivative Works that You distribute must
|
|
109
|
+
include a readable copy of the attribution notices contained
|
|
110
|
+
within such NOTICE file, excluding those notices that do not
|
|
111
|
+
pertain to any part of the Derivative Works, in at least one
|
|
112
|
+
of the following places: within a NOTICE text file distributed
|
|
113
|
+
as part of the Derivative Works; within the Source form or
|
|
114
|
+
documentation, if provided along with the Derivative Works; or,
|
|
115
|
+
within a display generated by the Derivative Works, if and
|
|
116
|
+
wherever such third-party notices normally appear. The contents
|
|
117
|
+
of the NOTICE file are for informational purposes only and
|
|
118
|
+
do not modify the License. You may add Your own attribution
|
|
119
|
+
notices within Derivative Works that You distribute, alongside
|
|
120
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
121
|
+
that such additional attribution notices cannot be construed
|
|
122
|
+
as modifying the License.
|
|
123
|
+
|
|
124
|
+
You may add Your own copyright statement to Your modifications and
|
|
125
|
+
may provide additional or different license terms and conditions
|
|
126
|
+
for use, reproduction, or distribution of Your modifications, or
|
|
127
|
+
for any such Derivative Works as a whole, provided Your use,
|
|
128
|
+
reproduction, and distribution of the Work otherwise complies with
|
|
129
|
+
the conditions stated in this License.
|
|
130
|
+
|
|
131
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
132
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
133
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
134
|
+
this License, without any additional terms or conditions.
|
|
135
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
136
|
+
the terms of any separate license agreement you may have executed
|
|
137
|
+
with Licensor regarding such Contributions.
|
|
138
|
+
|
|
139
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
140
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
141
|
+
except as required for reasonable and customary use in describing the
|
|
142
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
143
|
+
|
|
144
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
145
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
146
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
147
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
148
|
+
implied, including, without limitation, any warranties or conditions
|
|
149
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
150
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
151
|
+
appropriateness of using or redistributing the Work and assume any
|
|
152
|
+
risks associated with Your exercise of permissions under this License.
|
|
153
|
+
|
|
154
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
155
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
156
|
+
unless required by applicable law (such as deliberate and grossly
|
|
157
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
158
|
+
liable to You for damages, including any direct, indirect, special,
|
|
159
|
+
incidental, or consequential damages of any character arising as a
|
|
160
|
+
result of this License or out of the use or inability to use the
|
|
161
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
162
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
163
|
+
other commercial damages or losses), even if such Contributor
|
|
164
|
+
has been advised of the possibility of such damages.
|
|
165
|
+
|
|
166
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
167
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
168
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
169
|
+
or other liability obligations and/or rights consistent with this
|
|
170
|
+
License. However, in accepting such obligations, You may act only
|
|
171
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
172
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
173
|
+
defend, and hold each Contributor harmless for any liability
|
|
174
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
175
|
+
of your accepting any such warranty or additional liability.
|
|
176
|
+
|
|
177
|
+
END OF TERMS AND CONDITIONS
|
|
178
|
+
|
|
179
|
+
APPENDIX: How to apply the Apache License to your work.
|
|
180
|
+
|
|
181
|
+
To apply the Apache License to your work, attach the following
|
|
182
|
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
|
183
|
+
replaced with your own identifying information. (Don't include
|
|
184
|
+
the brackets!) The text should be enclosed in the appropriate
|
|
185
|
+
comment syntax for the file format. We also recommend that a
|
|
186
|
+
file or class name and description of purpose be included on the
|
|
187
|
+
same "printed page" as the copyright notice for easier
|
|
188
|
+
identification within third-party archives.
|
|
189
|
+
|
|
190
|
+
Copyright [yyyy] [name of copyright owner]
|
|
191
|
+
|
|
192
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
193
|
+
you may not use this file except in compliance with the License.
|
|
194
|
+
You may obtain a copy of the License at
|
|
195
|
+
|
|
196
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
197
|
+
|
|
198
|
+
Unless required by applicable law or agreed to in writing, software
|
|
199
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
200
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
201
|
+
See the License for the specific language governing permissions and
|
|
202
|
+
limitations under the License.
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: pipeloom-openlineage
|
|
3
|
+
Version: 0.9.0
|
|
4
|
+
Summary: OpenLineage event sink for pipeloom: runs, jobs, datasets and facets.
|
|
5
|
+
Project-URL: Repository, https://github.com/emanuel-luis/pipeloom
|
|
6
|
+
Project-URL: Documentation, https://emanuel-luis.github.io/pipeloom/
|
|
7
|
+
Project-URL: Changelog, https://github.com/emanuel-luis/pipeloom/blob/main/CHANGELOG.md
|
|
8
|
+
Author-email: Emanuel Luis <emanuel.filho08@gmail.com>
|
|
9
|
+
License-Expression: Apache-2.0
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
License-File: NOTICE
|
|
12
|
+
Keywords: data-engineering,lineage,openlineage,pipelines,pipeloom
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
21
|
+
Classifier: Typing :: Typed
|
|
22
|
+
Requires-Python: >=3.11
|
|
23
|
+
Requires-Dist: openlineage-python<2,>=1.52.0
|
|
24
|
+
Requires-Dist: pipeloom~=0.9.0
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
|
|
27
|
+
# pipeloom-openlineage
|
|
28
|
+
|
|
29
|
+
An [OpenLineage](https://openlineage.io) event sink for
|
|
30
|
+
[pipeloom](https://github.com/emanuel-luis/pipeloom). It turns run and unit events into
|
|
31
|
+
OpenLineage run events and sends them with the openlineage-python client, to Marquez or any other
|
|
32
|
+
OpenLineage backend.
|
|
33
|
+
|
|
34
|
+
```console
|
|
35
|
+
$ pip install "pipeloom[openlineage]"
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
Add it to the project's sinks:
|
|
39
|
+
|
|
40
|
+
```yaml
|
|
41
|
+
events:
|
|
42
|
+
sinks:
|
|
43
|
+
- kind: openlineage
|
|
44
|
+
buffered: true
|
|
45
|
+
transport:
|
|
46
|
+
type: http
|
|
47
|
+
url: http://localhost:5000
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
- The pipeloom run is the OpenLineage job `<project>`. Each persisted model is a child job
|
|
51
|
+
`<project>.<model>` with a parent run facet that points at the pipeline run.
|
|
52
|
+
- Run ids are uuid5 values derived from the project, the pipeloom run id, the unit id and the attempt, so every
|
|
53
|
+
process of one run (a unit task, `pipeloom finalize`) reports to the same OpenLineage runs.
|
|
54
|
+
- Inputs and outputs come from the manifest: catalog relations and source locations.
|
|
55
|
+
- Facets: `schema`, `outputStatistics`, `dataQualityAssertions`, `errorMessage`, `parent`,
|
|
56
|
+
`jobType` and `sourceCodeLocation`.
|
|
57
|
+
- Options: `namespace` (default `pipeloom.{target}`), `dataset_namespace`, `transport` (an
|
|
58
|
+
openlineage-python transport config; without it the client reads `OPENLINEAGE_URL` or
|
|
59
|
+
`openlineage.yml`) and `inside_airflow` (`skip`, the default, or `emit`). Inside an Airflow task
|
|
60
|
+
the Airflow OpenLineage provider reports the lineage, so the sink stays silent unless
|
|
61
|
+
`inside_airflow: emit`.
|
|
62
|
+
|
|
63
|
+
Documentation: https://emanuel-luis.github.io/pipeloom/latest/
|
|
64
|
+
|
|
65
|
+
License: Apache-2.0.
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# pipeloom-openlineage
|
|
2
|
+
|
|
3
|
+
An [OpenLineage](https://openlineage.io) event sink for
|
|
4
|
+
[pipeloom](https://github.com/emanuel-luis/pipeloom). It turns run and unit events into
|
|
5
|
+
OpenLineage run events and sends them with the openlineage-python client, to Marquez or any other
|
|
6
|
+
OpenLineage backend.
|
|
7
|
+
|
|
8
|
+
```console
|
|
9
|
+
$ pip install "pipeloom[openlineage]"
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
Add it to the project's sinks:
|
|
13
|
+
|
|
14
|
+
```yaml
|
|
15
|
+
events:
|
|
16
|
+
sinks:
|
|
17
|
+
- kind: openlineage
|
|
18
|
+
buffered: true
|
|
19
|
+
transport:
|
|
20
|
+
type: http
|
|
21
|
+
url: http://localhost:5000
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
- The pipeloom run is the OpenLineage job `<project>`. Each persisted model is a child job
|
|
25
|
+
`<project>.<model>` with a parent run facet that points at the pipeline run.
|
|
26
|
+
- Run ids are uuid5 values derived from the project, the pipeloom run id, the unit id and the attempt, so every
|
|
27
|
+
process of one run (a unit task, `pipeloom finalize`) reports to the same OpenLineage runs.
|
|
28
|
+
- Inputs and outputs come from the manifest: catalog relations and source locations.
|
|
29
|
+
- Facets: `schema`, `outputStatistics`, `dataQualityAssertions`, `errorMessage`, `parent`,
|
|
30
|
+
`jobType` and `sourceCodeLocation`.
|
|
31
|
+
- Options: `namespace` (default `pipeloom.{target}`), `dataset_namespace`, `transport` (an
|
|
32
|
+
openlineage-python transport config; without it the client reads `OPENLINEAGE_URL` or
|
|
33
|
+
`openlineage.yml`) and `inside_airflow` (`skip`, the default, or `emit`). Inside an Airflow task
|
|
34
|
+
the Airflow OpenLineage provider reports the lineage, so the sink stays silent unless
|
|
35
|
+
`inside_airflow: emit`.
|
|
36
|
+
|
|
37
|
+
Documentation: https://emanuel-luis.github.io/pipeloom/latest/
|
|
38
|
+
|
|
39
|
+
License: Apache-2.0.
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "pipeloom-openlineage"
|
|
7
|
+
version = "0.9.0"
|
|
8
|
+
description = "OpenLineage event sink for pipeloom: runs, jobs, datasets and facets."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
license = "Apache-2.0"
|
|
12
|
+
license-files = ["LICENSE", "NOTICE"]
|
|
13
|
+
authors = [{ name = "Emanuel Luis", email = "emanuel.filho08@gmail.com" }]
|
|
14
|
+
keywords = ["pipeloom", "openlineage", "lineage", "data-engineering", "pipelines"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 3 - Alpha",
|
|
17
|
+
"Intended Audience :: Developers",
|
|
18
|
+
"Programming Language :: Python :: 3",
|
|
19
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
20
|
+
"Programming Language :: Python :: 3.11",
|
|
21
|
+
"Programming Language :: Python :: 3.12",
|
|
22
|
+
"Programming Language :: Python :: 3.13",
|
|
23
|
+
"Programming Language :: Python :: 3.14",
|
|
24
|
+
"Typing :: Typed",
|
|
25
|
+
]
|
|
26
|
+
# The floor is what apache-airflow-providers-openlineage requires, so both share one client.
|
|
27
|
+
dependencies = ["pipeloom~=0.9.0", "openlineage-python>=1.52.0,<2"]
|
|
28
|
+
|
|
29
|
+
[project.urls]
|
|
30
|
+
Repository = "https://github.com/emanuel-luis/pipeloom"
|
|
31
|
+
Documentation = "https://emanuel-luis.github.io/pipeloom/"
|
|
32
|
+
Changelog = "https://github.com/emanuel-luis/pipeloom/blob/main/CHANGELOG.md"
|
|
33
|
+
|
|
34
|
+
[project.entry-points."pipeloom.sinks"]
|
|
35
|
+
openlineage = "pipeloom_openlineage.spec:SPEC"
|
|
36
|
+
|
|
37
|
+
[tool.uv.sources]
|
|
38
|
+
pipeloom = { workspace = true }
|
|
39
|
+
|
|
40
|
+
[tool.hatch.build.targets.wheel]
|
|
41
|
+
packages = ["src/pipeloom_openlineage"]
|
|
42
|
+
|
|
43
|
+
[tool.hatch.build.targets.sdist]
|
|
44
|
+
only-include = ["src", "tests", "LICENSE", "NOTICE", "README.md"]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""OpenLineage event sink for pipeloom. The client loads when a project names the sink."""
|
|
File without changes
|
|
@@ -0,0 +1,527 @@
|
|
|
1
|
+
"""Turn pipeloom run and unit events into OpenLineage run events (ADR-0046).
|
|
2
|
+
|
|
3
|
+
The pipeloom run is the job `<project>`; each persisted model is the job `<project>.<model>`,
|
|
4
|
+
whose runs point at the pipeline run through the parent facet. Run ids are uuid5 values, so every
|
|
5
|
+
process of one pipeloom run (unit tasks, finalize, an Airflow operator) derives the same ids.
|
|
6
|
+
Datasets and job facets come from the manifest that `plan_ready` hands over.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import uuid
|
|
13
|
+
from collections.abc import Mapping, Sequence
|
|
14
|
+
from datetime import UTC, datetime
|
|
15
|
+
from pathlib import Path, PurePosixPath
|
|
16
|
+
from typing import Any, Final, cast
|
|
17
|
+
from urllib.parse import urlsplit
|
|
18
|
+
|
|
19
|
+
from openlineage.client import OpenLineageClient
|
|
20
|
+
from openlineage.client.event_v2 import (
|
|
21
|
+
InputDataset,
|
|
22
|
+
Job,
|
|
23
|
+
OutputDataset,
|
|
24
|
+
Run,
|
|
25
|
+
RunEvent,
|
|
26
|
+
RunState,
|
|
27
|
+
)
|
|
28
|
+
from openlineage.client.facet_v2 import (
|
|
29
|
+
DatasetFacet,
|
|
30
|
+
JobFacet,
|
|
31
|
+
OutputDatasetFacet,
|
|
32
|
+
RunFacet,
|
|
33
|
+
data_quality_assertions_dataset,
|
|
34
|
+
error_message_run,
|
|
35
|
+
job_type_job,
|
|
36
|
+
output_statistics_output_dataset,
|
|
37
|
+
parent_run,
|
|
38
|
+
schema_dataset,
|
|
39
|
+
source_code_location_job,
|
|
40
|
+
)
|
|
41
|
+
from openlineage.client.transport import Transport
|
|
42
|
+
|
|
43
|
+
from pipeloom.events.bus import PlanInfo
|
|
44
|
+
from pipeloom.events.model import Event, Level
|
|
45
|
+
from pipeloom.secrets.resolve import contains_secret
|
|
46
|
+
|
|
47
|
+
PRODUCER: Final = "https://github.com/emanuel-luis/pipeloom/tree/main/plugins/pipeloom-openlineage"
|
|
48
|
+
AIRFLOW_KEY: Final = "orchestrator_id.airflow_dag_id"
|
|
49
|
+
_ID_SPACE: Final = uuid.uuid5(uuid.NAMESPACE_URL, "https://github.com/emanuel-luis/pipeloom")
|
|
50
|
+
_UNIT_END: Final = {
|
|
51
|
+
"unit.succeeded": RunState.COMPLETE,
|
|
52
|
+
"unit.failed": RunState.FAIL,
|
|
53
|
+
"unit.timed_out": RunState.FAIL,
|
|
54
|
+
"unit.cancelled": RunState.ABORT,
|
|
55
|
+
}
|
|
56
|
+
_RUN_END: Final = {
|
|
57
|
+
"succeeded": RunState.COMPLETE,
|
|
58
|
+
"failed": RunState.FAIL,
|
|
59
|
+
"cancelled": RunState.ABORT,
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def run_uuid(project: str, run_id: str) -> str:
|
|
64
|
+
"""The OpenLineage run id of a pipeloom run. The project scopes custom run ids."""
|
|
65
|
+
return str(uuid.uuid5(_ID_SPACE, f"run:{project}:{run_id}"))
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def unit_uuid(project: str, run_id: str, unit_id: str, attempt: int) -> str:
|
|
69
|
+
"""The OpenLineage run id of one attempt of one unit."""
|
|
70
|
+
return str(uuid.uuid5(_ID_SPACE, f"unit:{project}:{run_id}:{unit_id}:{attempt}"))
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
# -- manifest accessors: the only code that knows the manifest's field names ------------------
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _project(manifest: Mapping[str, Any]) -> str:
|
|
77
|
+
return str(manifest["metadata"]["project"])
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _target(manifest: Mapping[str, Any]) -> str:
|
|
81
|
+
return str(manifest["metadata"]["target"])
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _unit_nodes(manifest: Mapping[str, Any], unit_id: str) -> list[str]:
|
|
85
|
+
return list(manifest.get("units", {}).get(unit_id, ()))
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _node(manifest: Mapping[str, Any], unique_id: str) -> Mapping[str, Any] | None:
|
|
89
|
+
node: Mapping[str, Any] | None = manifest.get("nodes", {}).get(unique_id)
|
|
90
|
+
return node
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _source(manifest: Mapping[str, Any], unique_id: str) -> Mapping[str, Any] | None:
|
|
94
|
+
source: Mapping[str, Any] | None = manifest.get("sources", {}).get(unique_id)
|
|
95
|
+
return source
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _node_inputs(node: Mapping[str, Any]) -> list[str]:
|
|
99
|
+
return list(node.get("inputs", {}).values())
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _node_relation(node: Mapping[str, Any]) -> str | None:
|
|
103
|
+
rel = node.get("relation")
|
|
104
|
+
if not rel:
|
|
105
|
+
return None
|
|
106
|
+
return ".".join(p for p in (rel.get("catalog"), rel.get("schema"), rel.get("name")) if p)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _node_columns(node: Mapping[str, Any]) -> list[Mapping[str, Any]] | None:
|
|
110
|
+
columns: list[Mapping[str, Any]] | None = node.get("columns")
|
|
111
|
+
return columns
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _node_location(node: Mapping[str, Any]) -> tuple[str, int | None]:
|
|
115
|
+
return str(node["file"]), node.get("line")
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _source_location(source: Mapping[str, Any]) -> tuple[str | None, str | None]:
|
|
119
|
+
"""(location, table) of a source."""
|
|
120
|
+
return source.get("location"), source.get("table")
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
# -- the sink -----------------------------------------------------------------------------------
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
class OpenLineageSink:
|
|
127
|
+
"""Emits OpenLineage run events for runs and units. Silent inside Airflow unless told."""
|
|
128
|
+
|
|
129
|
+
name = "openlineage"
|
|
130
|
+
|
|
131
|
+
def __init__(
|
|
132
|
+
self,
|
|
133
|
+
*,
|
|
134
|
+
namespace: str = "pipeloom.{target}",
|
|
135
|
+
dataset_namespace: str | None = None,
|
|
136
|
+
transport: Mapping[str, Any] | None = None,
|
|
137
|
+
inside_airflow: str = "skip",
|
|
138
|
+
client: OpenLineageClient | None = None,
|
|
139
|
+
) -> None:
|
|
140
|
+
self._namespace = namespace
|
|
141
|
+
self._dataset_namespace = dataset_namespace
|
|
142
|
+
self._transport = None if transport is None else dict(transport)
|
|
143
|
+
self._inside_airflow = inside_airflow
|
|
144
|
+
self._client = client
|
|
145
|
+
self._manifest: Mapping[str, Any] | None = None
|
|
146
|
+
self._project_dir: Path | None = None
|
|
147
|
+
self._open: dict[str, int] = {}
|
|
148
|
+
"""Unit id to the attempt that started and has not ended."""
|
|
149
|
+
self._written: dict[str, Mapping[str, Any]] = {}
|
|
150
|
+
self._quality: dict[str, list[Mapping[str, Any]]] = {}
|
|
151
|
+
|
|
152
|
+
def __repr__(self) -> str: # a transport config may hold credentials
|
|
153
|
+
return "OpenLineageSink()"
|
|
154
|
+
|
|
155
|
+
# -- the sink protocol --------------------------------------------------------------------
|
|
156
|
+
|
|
157
|
+
def plan_ready(self, info: PlanInfo) -> None:
|
|
158
|
+
self._manifest = info.manifest
|
|
159
|
+
self._project_dir = info.project_dir
|
|
160
|
+
|
|
161
|
+
def emit(self, event: Event) -> None:
|
|
162
|
+
if self._inside_airflow == "skip" and AIRFLOW_KEY in event.context:
|
|
163
|
+
return # the Airflow provider reports this task
|
|
164
|
+
if event.run_id is None:
|
|
165
|
+
return
|
|
166
|
+
if event.type == "run.started":
|
|
167
|
+
self._run_started(event)
|
|
168
|
+
elif event.type == "run.completed":
|
|
169
|
+
self._run_completed(event)
|
|
170
|
+
elif event.type == "unit.started" and event.unit_id and event.attempt is not None:
|
|
171
|
+
self._open[event.unit_id] = event.attempt
|
|
172
|
+
self._written.pop(event.unit_id, None)
|
|
173
|
+
self._quality.pop(event.unit_id, None)
|
|
174
|
+
self._send(self._unit_event(event, RunState.START, event.attempt))
|
|
175
|
+
elif event.type == "model.written" and event.unit_id:
|
|
176
|
+
self._written[event.unit_id] = event.data
|
|
177
|
+
elif event.type == "model.quality_evaluated" and event.unit_id:
|
|
178
|
+
results = event.data.get("results")
|
|
179
|
+
self._quality[event.unit_id] = cast(list[Mapping[str, Any]], results or [])
|
|
180
|
+
elif event.type in _UNIT_END and event.unit_id:
|
|
181
|
+
attempt = self._open.pop(event.unit_id, None)
|
|
182
|
+
attempt = event.attempt if event.attempt is not None else attempt
|
|
183
|
+
if attempt is not None: # a unit cancelled before it started has no OpenLineage run
|
|
184
|
+
self._send(self._unit_event(event, _UNIT_END[event.type], attempt))
|
|
185
|
+
|
|
186
|
+
def flush(self) -> None:
|
|
187
|
+
pass
|
|
188
|
+
|
|
189
|
+
def close(self) -> None:
|
|
190
|
+
if self._client is not None:
|
|
191
|
+
self._client.close()
|
|
192
|
+
|
|
193
|
+
# -- events -------------------------------------------------------------------------------
|
|
194
|
+
|
|
195
|
+
def _send(self, ol_event: RunEvent) -> None:
|
|
196
|
+
client = self._get_client()
|
|
197
|
+
if client is not None:
|
|
198
|
+
client.emit(ol_event)
|
|
199
|
+
|
|
200
|
+
def _get_client(self) -> OpenLineageClient | None:
|
|
201
|
+
if self._client is None:
|
|
202
|
+
if self._transport is None:
|
|
203
|
+
self._client = OpenLineageClient()
|
|
204
|
+
elif contains_secret(json.dumps(self._transport)):
|
|
205
|
+
return None # built without secret resolution: a command that never runs
|
|
206
|
+
else:
|
|
207
|
+
self._client = OpenLineageClient(config={"transport": self._transport})
|
|
208
|
+
return self._client
|
|
209
|
+
|
|
210
|
+
def _names(self, event: Event) -> tuple[str, str]:
|
|
211
|
+
"""(project, target) from the manifest, else from a run event's data."""
|
|
212
|
+
if self._manifest is not None:
|
|
213
|
+
return _project(self._manifest), _target(self._manifest)
|
|
214
|
+
return str(event.data.get("project", "")), str(event.data.get("target", ""))
|
|
215
|
+
|
|
216
|
+
def _job_namespace(self, project: str, target: str) -> str:
|
|
217
|
+
return self._namespace.format(project=project, target=target)
|
|
218
|
+
|
|
219
|
+
def _run_started(self, event: Event) -> None:
|
|
220
|
+
project, target = self._names(event)
|
|
221
|
+
ns = self._job_namespace(project, target)
|
|
222
|
+
assert event.run_id is not None
|
|
223
|
+
self._send(
|
|
224
|
+
RunEvent(
|
|
225
|
+
eventType=RunState.START,
|
|
226
|
+
eventTime=_ts(event),
|
|
227
|
+
run=Run(runId=run_uuid(project, event.run_id)),
|
|
228
|
+
job=Job(namespace=ns, name=project, facets=_job_type("JOB")),
|
|
229
|
+
producer=PRODUCER,
|
|
230
|
+
)
|
|
231
|
+
)
|
|
232
|
+
|
|
233
|
+
def _run_completed(self, event: Event) -> None:
|
|
234
|
+
if event.data.get("replay") is True:
|
|
235
|
+
return # a retried finalize: the first one already reported the run
|
|
236
|
+
project, target = self._names(event)
|
|
237
|
+
state = _RUN_END.get(str(event.data.get("status")), RunState.FAIL)
|
|
238
|
+
facets: dict[str, RunFacet] = {}
|
|
239
|
+
failures = event.data.get("failures")
|
|
240
|
+
if state is RunState.FAIL and isinstance(failures, list) and failures:
|
|
241
|
+
lines = [
|
|
242
|
+
f"{f.get('unit_id')} {f.get('code')}: {f.get('message')}"
|
|
243
|
+
for f in failures
|
|
244
|
+
if isinstance(f, dict)
|
|
245
|
+
]
|
|
246
|
+
facets["errorMessage"] = error_message_run.ErrorMessageRunFacet(
|
|
247
|
+
message="\n".join(lines), programmingLanguage="python"
|
|
248
|
+
)
|
|
249
|
+
assert event.run_id is not None
|
|
250
|
+
self._send(
|
|
251
|
+
RunEvent(
|
|
252
|
+
eventType=state,
|
|
253
|
+
eventTime=_ts(event),
|
|
254
|
+
run=Run(runId=run_uuid(project, event.run_id), facets=facets),
|
|
255
|
+
job=Job(
|
|
256
|
+
namespace=self._job_namespace(project, target),
|
|
257
|
+
name=project,
|
|
258
|
+
facets=_job_type("JOB"),
|
|
259
|
+
),
|
|
260
|
+
producer=PRODUCER,
|
|
261
|
+
)
|
|
262
|
+
)
|
|
263
|
+
|
|
264
|
+
def _unit_event(self, event: Event, state: RunState, attempt: int) -> RunEvent:
|
|
265
|
+
assert event.run_id is not None and event.unit_id is not None
|
|
266
|
+
unit_id = event.unit_id
|
|
267
|
+
project, target = self._names(event)
|
|
268
|
+
ns = self._job_namespace(project, target)
|
|
269
|
+
parent_id = run_uuid(project, event.run_id)
|
|
270
|
+
run_facets: dict[str, RunFacet] = {
|
|
271
|
+
"parent": parent_run.ParentRunFacet(
|
|
272
|
+
run=parent_run.Run(runId=parent_id),
|
|
273
|
+
job=parent_run.Job(namespace=ns, name=project),
|
|
274
|
+
root=parent_run.Root(
|
|
275
|
+
run=parent_run.RootRun(runId=parent_id),
|
|
276
|
+
job=parent_run.RootJob(namespace=ns, name=project),
|
|
277
|
+
),
|
|
278
|
+
)
|
|
279
|
+
}
|
|
280
|
+
error = _error_facet(event)
|
|
281
|
+
if error is not None:
|
|
282
|
+
run_facets["errorMessage"] = error
|
|
283
|
+
job_facets = _job_type("MODEL")
|
|
284
|
+
inputs: list[InputDataset] = []
|
|
285
|
+
outputs: list[OutputDataset] = []
|
|
286
|
+
if self._manifest is not None:
|
|
287
|
+
nodes = _unit_nodes(self._manifest, unit_id)
|
|
288
|
+
if nodes:
|
|
289
|
+
node = _node(self._manifest, nodes[-1])
|
|
290
|
+
if node is not None:
|
|
291
|
+
job_facets["sourceCodeLocation"] = _code_location(node)
|
|
292
|
+
output = self._output(node, state, unit_id, event, ns)
|
|
293
|
+
if output is not None:
|
|
294
|
+
outputs.append(output)
|
|
295
|
+
inputs = self._inputs(nodes, ns)
|
|
296
|
+
return RunEvent(
|
|
297
|
+
eventType=state,
|
|
298
|
+
eventTime=_ts(event),
|
|
299
|
+
run=Run(runId=unit_uuid(project, event.run_id, unit_id, attempt), facets=run_facets),
|
|
300
|
+
job=Job(namespace=ns, name=f"{project}.{unit_id}", facets=job_facets),
|
|
301
|
+
inputs=inputs,
|
|
302
|
+
outputs=outputs,
|
|
303
|
+
producer=PRODUCER,
|
|
304
|
+
)
|
|
305
|
+
|
|
306
|
+
# -- datasets -----------------------------------------------------------------------------
|
|
307
|
+
|
|
308
|
+
def _relation_namespace(self, job_ns: str) -> str:
|
|
309
|
+
return self._dataset_namespace or job_ns
|
|
310
|
+
|
|
311
|
+
def _inputs(self, nodes: Sequence[str], job_ns: str) -> list[InputDataset]:
|
|
312
|
+
assert self._manifest is not None
|
|
313
|
+
inside = set(nodes)
|
|
314
|
+
seen: dict[tuple[str, str], InputDataset] = {}
|
|
315
|
+
for uid in nodes:
|
|
316
|
+
node = _node(self._manifest, uid)
|
|
317
|
+
for parent in [] if node is None else _node_inputs(node):
|
|
318
|
+
if parent in inside:
|
|
319
|
+
continue # an inlined ephemeral model
|
|
320
|
+
found = self._input(parent, job_ns)
|
|
321
|
+
if found is not None:
|
|
322
|
+
seen.setdefault((found.namespace, found.name), found)
|
|
323
|
+
return list(seen.values())
|
|
324
|
+
|
|
325
|
+
def _input(self, unique_id: str, job_ns: str) -> InputDataset | None:
|
|
326
|
+
assert self._manifest is not None
|
|
327
|
+
source = _source(self._manifest, unique_id)
|
|
328
|
+
if source is not None:
|
|
329
|
+
location, table = _source_location(source)
|
|
330
|
+
if table:
|
|
331
|
+
return InputDataset(namespace=self._relation_namespace(job_ns), name=table)
|
|
332
|
+
if location:
|
|
333
|
+
ns, name = self._location(location)
|
|
334
|
+
return InputDataset(namespace=ns, name=name)
|
|
335
|
+
return None
|
|
336
|
+
node = _node(self._manifest, unique_id)
|
|
337
|
+
relation = None if node is None else _node_relation(node)
|
|
338
|
+
if relation is None:
|
|
339
|
+
return None
|
|
340
|
+
return InputDataset(namespace=self._relation_namespace(job_ns), name=relation)
|
|
341
|
+
|
|
342
|
+
def _location(self, location: str) -> tuple[str, str]:
|
|
343
|
+
"""OpenLineage naming for a path (https://openlineage.io/docs/spec/naming): a URI with a
|
|
344
|
+
host is `scheme://host` and the path without its leading slash; `file:` is namespace
|
|
345
|
+
`file` and the absolute path; another scheme without a host is the scheme and the path
|
|
346
|
+
as reported; a relative path is joined to the project dir."""
|
|
347
|
+
parts = urlsplit(location)
|
|
348
|
+
# a Windows drive letter ("C:/x") parses as a one-letter scheme: treat it as a path
|
|
349
|
+
if len(parts.scheme) > 1:
|
|
350
|
+
if parts.scheme == "file":
|
|
351
|
+
return "file", parts.path
|
|
352
|
+
if parts.netloc:
|
|
353
|
+
return f"{parts.scheme}://{parts.netloc}", parts.path.lstrip("/")
|
|
354
|
+
return parts.scheme, parts.path
|
|
355
|
+
path = PurePosixPath(location)
|
|
356
|
+
if not path.is_absolute() and self._project_dir is not None:
|
|
357
|
+
return "file", (self._project_dir / location).as_posix()
|
|
358
|
+
return "file", path.as_posix()
|
|
359
|
+
|
|
360
|
+
def _output(
|
|
361
|
+
self, node: Mapping[str, Any], state: RunState, unit_id: str, event: Event, job_ns: str
|
|
362
|
+
) -> OutputDataset | None:
|
|
363
|
+
relation = _node_relation(node)
|
|
364
|
+
if relation is None:
|
|
365
|
+
return None
|
|
366
|
+
facets: dict[str, DatasetFacet] = {}
|
|
367
|
+
output_facets: dict[str, OutputDatasetFacet] = {}
|
|
368
|
+
written = self._written.get(unit_id, {})
|
|
369
|
+
columns = written.get("columns") or _node_columns(node)
|
|
370
|
+
if columns:
|
|
371
|
+
facets["schema"] = schema_dataset.SchemaDatasetFacet(
|
|
372
|
+
fields=[
|
|
373
|
+
schema_dataset.SchemaDatasetFacetFields(
|
|
374
|
+
name=str(c["name"]), type=str(c["dtype"]), ordinal_position=i
|
|
375
|
+
)
|
|
376
|
+
for i, c in enumerate(columns, start=1)
|
|
377
|
+
]
|
|
378
|
+
)
|
|
379
|
+
if state is RunState.COMPLETE:
|
|
380
|
+
rows, size = event.data.get("rows_written"), event.data.get("bytes_written")
|
|
381
|
+
if rows is not None or size is not None:
|
|
382
|
+
output_facets["outputStatistics"] = (
|
|
383
|
+
output_statistics_output_dataset.OutputStatisticsOutputDatasetFacet(
|
|
384
|
+
rowCount=_int(rows), size=_int(size)
|
|
385
|
+
)
|
|
386
|
+
)
|
|
387
|
+
assertions = _assertions(self._quality.get(unit_id, []))
|
|
388
|
+
if assertions:
|
|
389
|
+
# An input facet in the spec; on the output dataset it describes the rows the
|
|
390
|
+
# expectations checked, which is what Marquez shows on the dataset page.
|
|
391
|
+
facets["dataQualityAssertions"] = cast(
|
|
392
|
+
DatasetFacet,
|
|
393
|
+
data_quality_assertions_dataset.DataQualityAssertionsDatasetFacet(
|
|
394
|
+
assertions=assertions
|
|
395
|
+
),
|
|
396
|
+
)
|
|
397
|
+
return OutputDataset(
|
|
398
|
+
namespace=self._relation_namespace(job_ns),
|
|
399
|
+
name=relation,
|
|
400
|
+
facets=facets,
|
|
401
|
+
outputFacets=output_facets,
|
|
402
|
+
)
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
# -- facets -------------------------------------------------------------------------------------
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
def _ts(event: Event) -> str:
|
|
409
|
+
return str(event.to_json()["ts"])
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
def _int(value: Any) -> int | None:
|
|
413
|
+
return value if isinstance(value, int) and not isinstance(value, bool) else None
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
def _job_type(kind: str) -> dict[str, JobFacet]:
|
|
417
|
+
return {
|
|
418
|
+
"jobType": job_type_job.JobTypeJobFacet(
|
|
419
|
+
processingType="BATCH", integration="PIPELOOM", jobType=kind
|
|
420
|
+
)
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
|
|
424
|
+
def _code_location(node: Mapping[str, Any]) -> JobFacet:
|
|
425
|
+
"""The model's file, relative to the project dir, with its line when the manifest has it."""
|
|
426
|
+
file, line = _node_location(node)
|
|
427
|
+
return source_code_location_job.SourceCodeLocationJobFacet(
|
|
428
|
+
type="file", url=f"{file}#L{line}" if line else file, path=file
|
|
429
|
+
)
|
|
430
|
+
|
|
431
|
+
|
|
432
|
+
def _error_facet(event: Event) -> RunFacet | None:
|
|
433
|
+
if event.type == "unit.failed":
|
|
434
|
+
error = event.data.get("error")
|
|
435
|
+
err = error if isinstance(error, dict) else {}
|
|
436
|
+
trace = event.data.get("traceback")
|
|
437
|
+
return error_message_run.ErrorMessageRunFacet(
|
|
438
|
+
message=f"{err.get('code')}: {err.get('message')}",
|
|
439
|
+
programmingLanguage="python",
|
|
440
|
+
stackTrace=trace if isinstance(trace, str) else None,
|
|
441
|
+
)
|
|
442
|
+
if event.type == "unit.timed_out":
|
|
443
|
+
return error_message_run.ErrorMessageRunFacet(
|
|
444
|
+
message=f"timed out after {event.data.get('timeout_s')} s",
|
|
445
|
+
programmingLanguage="python",
|
|
446
|
+
)
|
|
447
|
+
return None
|
|
448
|
+
|
|
449
|
+
|
|
450
|
+
def _assertions(
|
|
451
|
+
results: Sequence[Mapping[str, Any]],
|
|
452
|
+
) -> list[data_quality_assertions_dataset.Assertion]:
|
|
453
|
+
return [
|
|
454
|
+
data_quality_assertions_dataset.Assertion(
|
|
455
|
+
assertion=str(r.get("kind")),
|
|
456
|
+
success=not r.get("failed"),
|
|
457
|
+
name=None if r.get("name") is None else str(r.get("name")),
|
|
458
|
+
severity=None if r.get("action") is None else str(r.get("action")),
|
|
459
|
+
actual=None if r.get("value") is None else str(r.get("value")),
|
|
460
|
+
)
|
|
461
|
+
for r in results
|
|
462
|
+
]
|
|
463
|
+
|
|
464
|
+
|
|
465
|
+
# -- one unit, for the Airflow operators ----------------------------------------------------------
|
|
466
|
+
|
|
467
|
+
|
|
468
|
+
class _Capture(Transport):
|
|
469
|
+
kind = "pipeloom-capture"
|
|
470
|
+
|
|
471
|
+
def __init__(self) -> None:
|
|
472
|
+
self.events: list[RunEvent] = []
|
|
473
|
+
|
|
474
|
+
def emit(self, event: Any) -> None:
|
|
475
|
+
self.events.append(event)
|
|
476
|
+
|
|
477
|
+
|
|
478
|
+
def unit_lineage(
|
|
479
|
+
manifest: Mapping[str, Any],
|
|
480
|
+
project_dir: Path,
|
|
481
|
+
unit_id: str,
|
|
482
|
+
events: Sequence[Event] = (),
|
|
483
|
+
*,
|
|
484
|
+
namespace: str = "pipeloom.{target}",
|
|
485
|
+
dataset_namespace: str | None = None,
|
|
486
|
+
) -> RunEvent:
|
|
487
|
+
"""The last OpenLineage event the sink sends for one unit, given that unit's pipeloom events.
|
|
488
|
+
Without a terminal event among them it is a START built from the manifest alone (the static
|
|
489
|
+
datasets a KUBERNETES task or a task that has not run yet reports)."""
|
|
490
|
+
transport = _Capture()
|
|
491
|
+
sink = OpenLineageSink(
|
|
492
|
+
namespace=namespace,
|
|
493
|
+
dataset_namespace=dataset_namespace,
|
|
494
|
+
inside_airflow="emit",
|
|
495
|
+
client=OpenLineageClient(transport=transport),
|
|
496
|
+
)
|
|
497
|
+
sink.plan_ready(PlanInfo(manifest, project_dir))
|
|
498
|
+
for event in events:
|
|
499
|
+
if event.unit_id == unit_id:
|
|
500
|
+
sink.emit(event)
|
|
501
|
+
if not transport.events or transport.events[-1].eventType is RunState.START:
|
|
502
|
+
transport.events.clear()
|
|
503
|
+
sink.emit(
|
|
504
|
+
Event(
|
|
505
|
+
id="static",
|
|
506
|
+
run_id="static",
|
|
507
|
+
invocation_id="static",
|
|
508
|
+
seq=0,
|
|
509
|
+
ts=datetime.now(UTC),
|
|
510
|
+
type="unit.started",
|
|
511
|
+
level=Level.INFO,
|
|
512
|
+
unit_id=unit_id,
|
|
513
|
+
attempt=1,
|
|
514
|
+
data={},
|
|
515
|
+
)
|
|
516
|
+
)
|
|
517
|
+
return transport.events[-1]
|
|
518
|
+
|
|
519
|
+
|
|
520
|
+
__all__ = [
|
|
521
|
+
"AIRFLOW_KEY",
|
|
522
|
+
"PRODUCER",
|
|
523
|
+
"OpenLineageSink",
|
|
524
|
+
"run_uuid",
|
|
525
|
+
"unit_lineage",
|
|
526
|
+
"unit_uuid",
|
|
527
|
+
]
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"""The plugin spec. It imports no OpenLineage code: the client loads when the sink is built."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any, Literal
|
|
6
|
+
|
|
7
|
+
from pipeloom.events.bus import EventSink
|
|
8
|
+
from pipeloom.plugins import API_VERSION, OptionsModel, PluginSpec
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class OpenLineageOptions(OptionsModel):
|
|
12
|
+
namespace: str = "pipeloom.{target}"
|
|
13
|
+
"""OpenLineage namespace of the jobs; {target} and {project} are filled in."""
|
|
14
|
+
dataset_namespace: str | None = None
|
|
15
|
+
"""Namespace of catalog relations; defaults to the job namespace. Sources with a location
|
|
16
|
+
use the location's scheme and authority, or `file` for a local path."""
|
|
17
|
+
transport: dict[str, Any] | None = None
|
|
18
|
+
"""An openlineage-python transport config, as `type: http` plus its keys. Without it the
|
|
19
|
+
client reads OPENLINEAGE_URL or openlineage.yml."""
|
|
20
|
+
inside_airflow: Literal["skip", "emit"] = "skip"
|
|
21
|
+
"""Inside an Airflow task the Airflow provider reports lineage; `emit` reports it here too."""
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _build(opts: Any) -> EventSink:
|
|
25
|
+
from pipeloom_openlineage.sink import OpenLineageSink
|
|
26
|
+
|
|
27
|
+
return OpenLineageSink(
|
|
28
|
+
namespace=opts.namespace,
|
|
29
|
+
dataset_namespace=opts.dataset_namespace,
|
|
30
|
+
transport=opts.transport,
|
|
31
|
+
inside_airflow=opts.inside_airflow,
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
SPEC: PluginSpec[EventSink] = PluginSpec(
|
|
36
|
+
"openlineage",
|
|
37
|
+
"openlineage",
|
|
38
|
+
"sinks",
|
|
39
|
+
API_VERSION,
|
|
40
|
+
_build,
|
|
41
|
+
OpenLineageOptions,
|
|
42
|
+
description="Sends runs, jobs, datasets and facets as OpenLineage run events.",
|
|
43
|
+
)
|
|
@@ -0,0 +1,317 @@
|
|
|
1
|
+
"""The OpenLineage sink, offline: a recording transport stands in for the backend."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import uuid
|
|
6
|
+
from collections.abc import Mapping
|
|
7
|
+
from datetime import UTC, datetime, timedelta
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
import pytest
|
|
12
|
+
from openlineage.client import OpenLineageClient
|
|
13
|
+
from openlineage.client.event_v2 import RunEvent, RunState
|
|
14
|
+
from openlineage.client.serde import Serde
|
|
15
|
+
from openlineage.client.transport import Transport
|
|
16
|
+
from tests.runtime.test_runner import shop
|
|
17
|
+
|
|
18
|
+
from pipeloom.api import Project, ProjectOptions
|
|
19
|
+
from pipeloom.events.bus import EventBus, PlanInfo
|
|
20
|
+
from pipeloom.events.model import (
|
|
21
|
+
ModelQualityEvaluated,
|
|
22
|
+
ModelWritten,
|
|
23
|
+
RunCompleted,
|
|
24
|
+
UnitStarted,
|
|
25
|
+
UnitSucceeded,
|
|
26
|
+
)
|
|
27
|
+
from pipeloom.graph.selectors import Selection
|
|
28
|
+
from pipeloom.runtime.run import RunOptions, RunOutcome
|
|
29
|
+
from pipeloom.testing.clock import FakeClock
|
|
30
|
+
from pipeloom.testing.recording import RecordingSink
|
|
31
|
+
from pipeloom_openlineage.sink import OpenLineageSink, run_uuid, unit_lineage, unit_uuid
|
|
32
|
+
from pipeloom_openlineage.spec import SPEC, OpenLineageOptions
|
|
33
|
+
|
|
34
|
+
CLOCK = FakeClock(datetime(2026, 1, 1, tzinfo=UTC), step=timedelta(0))
|
|
35
|
+
AIRFLOW = {"orchestrator_id.airflow_dag_id": "shop"}
|
|
36
|
+
UNITS = ("stg_customers", "stg_orders", "stg_payments", "dim_customers", "fct_orders")
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class Recording(Transport):
|
|
40
|
+
kind = "recording"
|
|
41
|
+
|
|
42
|
+
def __init__(self) -> None:
|
|
43
|
+
self.events: list[RunEvent] = []
|
|
44
|
+
|
|
45
|
+
def emit(self, event: Any) -> None:
|
|
46
|
+
assert isinstance(event, RunEvent)
|
|
47
|
+
Serde.to_json(event) # every event must serialize
|
|
48
|
+
self.events.append(event)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def make_sink(**options: Any) -> tuple[OpenLineageSink, Recording]:
|
|
52
|
+
transport = Recording()
|
|
53
|
+
return OpenLineageSink(client=OpenLineageClient(transport=transport), **options), transport
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def run(
|
|
57
|
+
root: Path, run_id: str = "run1", context: Mapping[str, str] | None = None, **options: Any
|
|
58
|
+
) -> tuple[RunOutcome, Recording]:
|
|
59
|
+
sink, transport = make_sink(**options)
|
|
60
|
+
project = Project(
|
|
61
|
+
ProjectOptions(project_dir=root, target="ci", env={"PIPELOOM_USER": "tester"}),
|
|
62
|
+
clock=CLOCK,
|
|
63
|
+
invocation_id="inv",
|
|
64
|
+
)
|
|
65
|
+
with EventBus([(sink, 1)], clock=CLOCK, invocation_id="inv", context=context) as bus:
|
|
66
|
+
outcome = project.run(RunOptions(Selection(()), run_id=run_id, workers=1), bus)
|
|
67
|
+
return outcome, transport
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def summary(events: list[RunEvent]) -> list[tuple[str, str]]:
|
|
71
|
+
return [(e.job.name, e.eventType.value if e.eventType else "") for e in events]
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def by_job(events: list[RunEvent], name: str, state: RunState) -> RunEvent:
|
|
75
|
+
return next(e for e in events if e.job.name == name and e.eventType is state)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def facet(facets: Mapping[str, Any] | None, key: str) -> Any:
|
|
79
|
+
"""A facet by key, typed loosely: the dicts are typed with the facet base classes."""
|
|
80
|
+
assert facets is not None
|
|
81
|
+
return facets[key]
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def test_a_successful_run(tmp_path: Path) -> None:
|
|
85
|
+
root = shop(tmp_path)
|
|
86
|
+
outcome, transport = run(root)
|
|
87
|
+
assert outcome.exit_code == 0
|
|
88
|
+
events = transport.events
|
|
89
|
+
got = summary(events)
|
|
90
|
+
assert got[0] == ("shop", "START") and got[-1] == ("shop", "COMPLETE")
|
|
91
|
+
for unit in (*UNITS, "rpt_revenue"):
|
|
92
|
+
assert got.index((f"shop.{unit}", "START")) < got.index((f"shop.{unit}", "COMPLETE"))
|
|
93
|
+
assert {e.job.namespace for e in events} == {"pipeloom.ci"}
|
|
94
|
+
|
|
95
|
+
parent = by_job(events, "shop", RunState.START)
|
|
96
|
+
assert parent.run.runId == run_uuid("shop", "run1")
|
|
97
|
+
assert facet(parent.job.facets, "jobType").jobType == "JOB"
|
|
98
|
+
|
|
99
|
+
done = by_job(events, "shop.stg_orders", RunState.COMPLETE)
|
|
100
|
+
assert done.run.runId == unit_uuid("shop", "run1", "stg_orders", 1)
|
|
101
|
+
assert facet(done.run.facets, "parent").run.runId == run_uuid("shop", "run1")
|
|
102
|
+
assert facet(done.run.facets, "parent").job.name == "shop"
|
|
103
|
+
assert done.inputs is not None and done.outputs is not None
|
|
104
|
+
[source] = done.inputs
|
|
105
|
+
assert (source.namespace, source.name) == ("file", f"{root.resolve().as_posix()}/raw/orders")
|
|
106
|
+
[out] = done.outputs
|
|
107
|
+
assert out.name.endswith("stg_orders") and out.namespace == "pipeloom.ci"
|
|
108
|
+
fields = facet(out.facets, "schema").fields
|
|
109
|
+
assert [f.name for f in fields] == ["id", "customer_id", "amount"]
|
|
110
|
+
assert facet(out.outputFacets, "outputStatistics").rowCount == 3
|
|
111
|
+
location = facet(done.job.facets, "sourceCodeLocation")
|
|
112
|
+
assert (location.type, location.path) == ("file", "models/staging/stg_orders.py")
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def test_an_ephemeral_model_passes_its_inputs_to_the_unit(tmp_path: Path) -> None:
|
|
116
|
+
_, transport = run(shop(tmp_path))
|
|
117
|
+
fct = by_job(transport.events, "shop.fct_orders", RunState.START)
|
|
118
|
+
assert fct.inputs is not None
|
|
119
|
+
names = sorted(i.name.rsplit(".", 1)[-1] for i in fct.inputs)
|
|
120
|
+
assert names == ["stg_customers", "stg_orders", "stg_payments"]
|
|
121
|
+
assert facet(fct.job.facets, "jobType").jobType == "MODEL"
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def test_a_failed_run(tmp_path: Path) -> None:
|
|
125
|
+
root = shop(tmp_path)
|
|
126
|
+
fct = root / "models" / "marts" / "fct_orders.py"
|
|
127
|
+
fct.write_text(
|
|
128
|
+
fct.read_text(encoding="utf-8").replace(
|
|
129
|
+
" return enriched\n", " raise ValueError('boom')\n"
|
|
130
|
+
),
|
|
131
|
+
encoding="utf-8",
|
|
132
|
+
)
|
|
133
|
+
outcome, transport = run(root)
|
|
134
|
+
assert outcome.exit_code == 1
|
|
135
|
+
events = transport.events
|
|
136
|
+
failed = by_job(events, "shop.fct_orders", RunState.FAIL)
|
|
137
|
+
error = facet(failed.run.facets, "errorMessage")
|
|
138
|
+
assert "boom" in error.message and error.programmingLanguage == "python"
|
|
139
|
+
assert "ValueError" in (error.stackTrace or "")
|
|
140
|
+
assert not [e for e in events if e.job.name == "shop.rpt_revenue"] # skipped: never started
|
|
141
|
+
end = events[-1]
|
|
142
|
+
assert (end.job.name, end.eventType) == ("shop", RunState.FAIL)
|
|
143
|
+
assert "fct_orders" in facet(end.run.facets, "errorMessage").message
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def test_ids_are_stable_across_invocations(tmp_path: Path) -> None:
|
|
147
|
+
_, first = run(shop(tmp_path / "a"), run_id="same")
|
|
148
|
+
_, second = run(shop(tmp_path / "b"), run_id="same")
|
|
149
|
+
ids = [(e.job.name, e.run.runId) for e in first.events]
|
|
150
|
+
assert ids == [(e.job.name, e.run.runId) for e in second.events]
|
|
151
|
+
assert all(uuid.UUID(i).version == 5 for _, i in ids)
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def test_the_project_scopes_custom_run_ids() -> None:
|
|
155
|
+
assert run_uuid("a", "nightly") != run_uuid("b", "nightly")
|
|
156
|
+
assert unit_uuid("a", "nightly", "m", 1) != unit_uuid("b", "nightly", "m", 1)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def test_silent_inside_airflow_unless_told(tmp_path: Path) -> None:
|
|
160
|
+
_, skipped = run(shop(tmp_path / "a"), context=AIRFLOW)
|
|
161
|
+
assert skipped.events == []
|
|
162
|
+
_, emitted = run(shop(tmp_path / "b"), context=AIRFLOW, inside_airflow="emit")
|
|
163
|
+
assert summary(emitted.events)[0] == ("shop", "START")
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
MANIFEST: dict[str, Any] = {
|
|
167
|
+
"metadata": {"project": "p", "target": "t"},
|
|
168
|
+
"nodes": {
|
|
169
|
+
"model.p.m": {
|
|
170
|
+
"file": "models/m.yml",
|
|
171
|
+
"line": 3,
|
|
172
|
+
"inputs": {"s": "source.p.s"},
|
|
173
|
+
"relation": {"catalog": None, "schema": "main", "name": "m"},
|
|
174
|
+
"columns": [{"name": "id", "dtype": "int64", "nullable": False}],
|
|
175
|
+
}
|
|
176
|
+
},
|
|
177
|
+
"sources": {"source.p.s": {"location": "s3://bucket/raw/s", "table": None}},
|
|
178
|
+
"units": {"m": ["model.p.m"]},
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def emit_unit(bus: EventBus) -> None:
|
|
183
|
+
bus.emit(
|
|
184
|
+
UnitStarted(materialization="table", executor="t", queue_wait_ms=0), unit_id="m", attempt=2
|
|
185
|
+
)
|
|
186
|
+
bus.emit(
|
|
187
|
+
ModelQualityEvaluated(
|
|
188
|
+
total_rows=2,
|
|
189
|
+
results=[
|
|
190
|
+
{"name": "ids", "kind": "not_null", "action": "fail", "passed": 2, "failed": 0},
|
|
191
|
+
{"name": "pos", "kind": "expr", "action": "warn", "passed": 1, "failed": 1},
|
|
192
|
+
],
|
|
193
|
+
rows_dropped=0,
|
|
194
|
+
rows_quarantined=0,
|
|
195
|
+
drop_ratio=0.0,
|
|
196
|
+
cached=False,
|
|
197
|
+
failures=[],
|
|
198
|
+
),
|
|
199
|
+
unit_id="m",
|
|
200
|
+
)
|
|
201
|
+
bus.emit(
|
|
202
|
+
ModelWritten(
|
|
203
|
+
relation="main.m",
|
|
204
|
+
materialization="table",
|
|
205
|
+
strategy="overwrite",
|
|
206
|
+
rows_written=2,
|
|
207
|
+
rows_inserted=2,
|
|
208
|
+
bytes_written=10,
|
|
209
|
+
duration_ms=0,
|
|
210
|
+
),
|
|
211
|
+
unit_id="m",
|
|
212
|
+
)
|
|
213
|
+
bus.emit(
|
|
214
|
+
UnitSucceeded(
|
|
215
|
+
materialization="table",
|
|
216
|
+
strategy="overwrite",
|
|
217
|
+
relation="main.m",
|
|
218
|
+
rows_written=2,
|
|
219
|
+
rows_inserted=2,
|
|
220
|
+
bytes_written=10,
|
|
221
|
+
duration_ms=0,
|
|
222
|
+
),
|
|
223
|
+
unit_id="m",
|
|
224
|
+
attempt=2,
|
|
225
|
+
)
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
@pytest.mark.parametrize(
|
|
229
|
+
("location", "expected"),
|
|
230
|
+
[
|
|
231
|
+
("s3://bucket/raw/s", ("s3://bucket", "raw/s")),
|
|
232
|
+
("gs://bucket/a/b", ("gs://bucket", "a/b")),
|
|
233
|
+
(
|
|
234
|
+
"abfss://box@acct.dfs.core.windows.net/p/q",
|
|
235
|
+
("abfss://box@acct.dfs.core.windows.net", "p/q"),
|
|
236
|
+
),
|
|
237
|
+
("file:///data/x", ("file", "/data/x")),
|
|
238
|
+
("file:/data/x", ("file", "/data/x")),
|
|
239
|
+
("dbfs:/Volumes/c/s/v/x", ("dbfs", "/Volumes/c/s/v/x")),
|
|
240
|
+
("/abs/x", ("file", "/abs/x")),
|
|
241
|
+
("rel/x", ("file", "/proj/rel/x")),
|
|
242
|
+
],
|
|
243
|
+
)
|
|
244
|
+
def test_location_follows_the_openlineage_naming(location: str, expected: tuple[str, str]) -> None:
|
|
245
|
+
sink, _ = make_sink()
|
|
246
|
+
sink._project_dir = Path("/proj")
|
|
247
|
+
assert sink._location(location) == expected
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def test_quality_contract_schema_and_naming_from_the_manifest() -> None:
|
|
251
|
+
sink, transport = make_sink(namespace="{project}-{target}", dataset_namespace="warehouse")
|
|
252
|
+
with EventBus([(sink, 1)], clock=CLOCK, invocation_id="i", run_id="r") as bus:
|
|
253
|
+
bus.plan_ready(PlanInfo(MANIFEST, Path("/p")))
|
|
254
|
+
emit_unit(bus)
|
|
255
|
+
start, done = transport.events
|
|
256
|
+
assert done.run.runId == unit_uuid("p", "r", "m", 2)
|
|
257
|
+
assert (done.job.namespace, done.job.name) == ("p-t", "p.m")
|
|
258
|
+
assert done.inputs is not None and done.outputs is not None
|
|
259
|
+
assert [(i.namespace, i.name) for i in done.inputs] == [("s3://bucket", "raw/s")]
|
|
260
|
+
[out] = done.outputs
|
|
261
|
+
assert (out.namespace, out.name) == ("warehouse", "main.m")
|
|
262
|
+
fields = facet(out.facets, "schema").fields
|
|
263
|
+
assert [f.name for f in fields] == ["id"] # from the contract: the event had no columns
|
|
264
|
+
assertions = facet(out.facets, "dataQualityAssertions").assertions
|
|
265
|
+
assert [(a.name, a.assertion, a.success) for a in assertions] == [
|
|
266
|
+
("ids", "not_null", True),
|
|
267
|
+
("pos", "expr", False),
|
|
268
|
+
]
|
|
269
|
+
assert facet(done.job.facets, "sourceCodeLocation").url == "models/m.yml#L3"
|
|
270
|
+
assert start.outputs is not None and "dataQualityAssertions" not in (
|
|
271
|
+
start.outputs[0].facets or {}
|
|
272
|
+
)
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def test_unit_lineage_replays_a_unit_or_falls_back_to_the_manifest() -> None:
|
|
276
|
+
recording = RecordingSink()
|
|
277
|
+
with EventBus([(recording, 1)], clock=CLOCK, invocation_id="i", run_id="r") as bus:
|
|
278
|
+
emit_unit(bus)
|
|
279
|
+
done = unit_lineage(MANIFEST, Path("/p"), "m", recording.events)
|
|
280
|
+
assert done.eventType is RunState.COMPLETE
|
|
281
|
+
assert done.outputs is not None
|
|
282
|
+
assert "dataQualityAssertions" in (done.outputs[0].facets or {})
|
|
283
|
+
|
|
284
|
+
static = unit_lineage(MANIFEST, Path("/p"), "m")
|
|
285
|
+
assert static.eventType is RunState.START and static.job.name == "p.m"
|
|
286
|
+
assert static.inputs is not None and static.outputs is not None
|
|
287
|
+
assert [i.name for i in static.inputs] == ["raw/s"]
|
|
288
|
+
assert [o.name for o in static.outputs] == ["main.m"]
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def test_a_replayed_finalize_reports_nothing() -> None:
|
|
292
|
+
sink, transport = make_sink()
|
|
293
|
+
with EventBus([(sink, 1)], clock=CLOCK, invocation_id="i", run_id="r") as bus:
|
|
294
|
+
bus.emit(
|
|
295
|
+
RunCompleted(
|
|
296
|
+
status="failed",
|
|
297
|
+
exit_code=1,
|
|
298
|
+
duration_ms=0,
|
|
299
|
+
counts={},
|
|
300
|
+
rows_written=0,
|
|
301
|
+
bytes_written=0,
|
|
302
|
+
failures=[],
|
|
303
|
+
artifacts=[],
|
|
304
|
+
replay=True,
|
|
305
|
+
)
|
|
306
|
+
)
|
|
307
|
+
assert transport.events == []
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def test_the_spec_builds_a_sink_that_waits_for_secrets() -> None:
|
|
311
|
+
opts = OpenLineageOptions(transport={"type": "http", "url": "${secret:env/OL_URL}"})
|
|
312
|
+
sink = SPEC.factory(opts)
|
|
313
|
+
assert isinstance(sink, OpenLineageSink)
|
|
314
|
+
with EventBus([(sink, 1)], clock=CLOCK, invocation_id="i", run_id="r") as bus:
|
|
315
|
+
bus.plan_ready(PlanInfo(MANIFEST, Path("/p")))
|
|
316
|
+
emit_unit(bus) # no client is built while the URL is a reference
|
|
317
|
+
assert repr(sink) == "OpenLineageSink()"
|