apache-airflow-providers-openlineage 1.11.0rc2__py3-none-any.whl → 1.12.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of apache-airflow-providers-openlineage might be problematic. Click here for more details.
- airflow/providers/openlineage/__init__.py +1 -1
- airflow/providers/openlineage/get_provider_info.py +4 -3
- airflow/providers/openlineage/plugins/adapter.py +49 -27
- airflow/providers/openlineage/plugins/listener.py +106 -49
- airflow/providers/openlineage/utils/utils.py +20 -9
- {apache_airflow_providers_openlineage-1.11.0rc2.dist-info → apache_airflow_providers_openlineage-1.12.0.dist-info}/METADATA +13 -13
- {apache_airflow_providers_openlineage-1.11.0rc2.dist-info → apache_airflow_providers_openlineage-1.12.0.dist-info}/RECORD +9 -9
- {apache_airflow_providers_openlineage-1.11.0rc2.dist-info → apache_airflow_providers_openlineage-1.12.0.dist-info}/WHEEL +0 -0
- {apache_airflow_providers_openlineage-1.11.0rc2.dist-info → apache_airflow_providers_openlineage-1.12.0.dist-info}/entry_points.txt +0 -0
|
@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
|
|
|
29
29
|
|
|
30
30
|
__all__ = ["__version__"]
|
|
31
31
|
|
|
32
|
-
__version__ = "1.
|
|
32
|
+
__version__ = "1.12.0"
|
|
33
33
|
|
|
34
34
|
if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
|
|
35
35
|
"2.8.0"
|
|
@@ -28,8 +28,9 @@ def get_provider_info():
|
|
|
28
28
|
"name": "OpenLineage Airflow",
|
|
29
29
|
"description": "`OpenLineage <https://openlineage.io/>`__\n",
|
|
30
30
|
"state": "ready",
|
|
31
|
-
"source-date-epoch":
|
|
31
|
+
"source-date-epoch": 1726861079,
|
|
32
32
|
"versions": [
|
|
33
|
+
"1.12.0",
|
|
33
34
|
"1.11.0",
|
|
34
35
|
"1.10.0",
|
|
35
36
|
"1.9.1",
|
|
@@ -55,8 +56,8 @@ def get_provider_info():
|
|
|
55
56
|
"apache-airflow-providers-common-sql>=1.6.0",
|
|
56
57
|
"apache-airflow-providers-common-compat>=1.2.0",
|
|
57
58
|
"attrs>=22.2",
|
|
58
|
-
"openlineage-integration-common>=1.
|
|
59
|
-
"openlineage-python>=1.
|
|
59
|
+
"openlineage-integration-common>=1.22.0",
|
|
60
|
+
"openlineage-python>=1.22.0",
|
|
60
61
|
],
|
|
61
62
|
"integrations": [
|
|
62
63
|
{
|
|
@@ -40,7 +40,6 @@ from openlineage.client.uuid import generate_static_uuid
|
|
|
40
40
|
from airflow.providers.openlineage import __version__ as OPENLINEAGE_PROVIDER_VERSION, conf
|
|
41
41
|
from airflow.providers.openlineage.utils.utils import (
|
|
42
42
|
OpenLineageRedactor,
|
|
43
|
-
get_airflow_dag_run_facet,
|
|
44
43
|
get_airflow_debug_facet,
|
|
45
44
|
get_airflow_state_run_facet,
|
|
46
45
|
)
|
|
@@ -50,9 +49,9 @@ from airflow.utils.log.logging_mixin import LoggingMixin
|
|
|
50
49
|
if TYPE_CHECKING:
|
|
51
50
|
from datetime import datetime
|
|
52
51
|
|
|
53
|
-
from airflow.models.dagrun import DagRun
|
|
54
52
|
from airflow.providers.openlineage.extractors import OperatorLineage
|
|
55
53
|
from airflow.utils.log.secrets_masker import SecretsMasker
|
|
54
|
+
from airflow.utils.state import DagRunState
|
|
56
55
|
|
|
57
56
|
_PRODUCER = f"https://github.com/apache/airflow/tree/providers-openlineage/{OPENLINEAGE_PROVIDER_VERSION}"
|
|
58
57
|
|
|
@@ -118,10 +117,10 @@ class OpenLineageAdapter(LoggingMixin):
|
|
|
118
117
|
return yaml.safe_load(config_file)
|
|
119
118
|
|
|
120
119
|
@staticmethod
|
|
121
|
-
def build_dag_run_id(dag_id: str,
|
|
120
|
+
def build_dag_run_id(dag_id: str, logical_date: datetime) -> str:
|
|
122
121
|
return str(
|
|
123
122
|
generate_static_uuid(
|
|
124
|
-
instant=
|
|
123
|
+
instant=logical_date,
|
|
125
124
|
data=f"{conf.namespace()}.{dag_id}".encode(),
|
|
126
125
|
)
|
|
127
126
|
)
|
|
@@ -336,33 +335,36 @@ class OpenLineageAdapter(LoggingMixin):
|
|
|
336
335
|
|
|
337
336
|
def dag_started(
|
|
338
337
|
self,
|
|
339
|
-
|
|
340
|
-
|
|
338
|
+
dag_id: str,
|
|
339
|
+
logical_date: datetime,
|
|
340
|
+
start_date: datetime,
|
|
341
341
|
nominal_start_time: str,
|
|
342
342
|
nominal_end_time: str,
|
|
343
|
+
owners: list[str],
|
|
344
|
+
run_facets: dict[str, RunFacet],
|
|
345
|
+
description: str | None = None,
|
|
343
346
|
job_facets: dict[str, JobFacet] | None = None, # Custom job facets
|
|
344
347
|
):
|
|
345
348
|
try:
|
|
346
|
-
owner = [x.strip() for x in dag_run.dag.owner.split(",")] if dag_run.dag else None
|
|
347
349
|
event = RunEvent(
|
|
348
350
|
eventType=RunState.START,
|
|
349
|
-
eventTime=
|
|
351
|
+
eventTime=start_date.isoformat(),
|
|
350
352
|
job=self._build_job(
|
|
351
|
-
job_name=
|
|
353
|
+
job_name=dag_id,
|
|
352
354
|
job_type=_JOB_TYPE_DAG,
|
|
353
|
-
job_description=
|
|
354
|
-
owners=
|
|
355
|
+
job_description=description,
|
|
356
|
+
owners=owners,
|
|
355
357
|
job_facets=job_facets,
|
|
356
358
|
),
|
|
357
359
|
run=self._build_run(
|
|
358
360
|
run_id=self.build_dag_run_id(
|
|
359
|
-
dag_id=
|
|
360
|
-
|
|
361
|
+
dag_id=dag_id,
|
|
362
|
+
logical_date=logical_date,
|
|
361
363
|
),
|
|
362
|
-
job_name=
|
|
364
|
+
job_name=dag_id,
|
|
363
365
|
nominal_start_time=nominal_start_time,
|
|
364
366
|
nominal_end_time=nominal_end_time,
|
|
365
|
-
run_facets={**
|
|
367
|
+
run_facets={**run_facets, **get_airflow_debug_facet()},
|
|
366
368
|
),
|
|
367
369
|
inputs=[],
|
|
368
370
|
outputs=[],
|
|
@@ -375,18 +377,29 @@ class OpenLineageAdapter(LoggingMixin):
|
|
|
375
377
|
# This part cannot be wrapped to deduplicate code, otherwise the method cannot be pickled in multiprocessing.
|
|
376
378
|
self.log.warning("Failed to emit DAG started event: \n %s", traceback.format_exc())
|
|
377
379
|
|
|
378
|
-
def dag_success(
|
|
380
|
+
def dag_success(
|
|
381
|
+
self,
|
|
382
|
+
dag_id: str,
|
|
383
|
+
run_id: str,
|
|
384
|
+
end_date: datetime,
|
|
385
|
+
logical_date: datetime,
|
|
386
|
+
dag_run_state: DagRunState,
|
|
387
|
+
task_ids: list[str],
|
|
388
|
+
):
|
|
379
389
|
try:
|
|
380
390
|
event = RunEvent(
|
|
381
391
|
eventType=RunState.COMPLETE,
|
|
382
|
-
eventTime=
|
|
383
|
-
job=self._build_job(job_name=
|
|
392
|
+
eventTime=end_date.isoformat(),
|
|
393
|
+
job=self._build_job(job_name=dag_id, job_type=_JOB_TYPE_DAG),
|
|
384
394
|
run=Run(
|
|
385
395
|
runId=self.build_dag_run_id(
|
|
386
|
-
dag_id=
|
|
387
|
-
|
|
396
|
+
dag_id=dag_id,
|
|
397
|
+
logical_date=logical_date,
|
|
388
398
|
),
|
|
389
|
-
facets={
|
|
399
|
+
facets={
|
|
400
|
+
**get_airflow_state_run_facet(dag_id, run_id, task_ids, dag_run_state),
|
|
401
|
+
**get_airflow_debug_facet(),
|
|
402
|
+
},
|
|
390
403
|
),
|
|
391
404
|
inputs=[],
|
|
392
405
|
outputs=[],
|
|
@@ -399,22 +412,31 @@ class OpenLineageAdapter(LoggingMixin):
|
|
|
399
412
|
# This part cannot be wrapped to deduplicate code, otherwise the method cannot be pickled in multiprocessing.
|
|
400
413
|
self.log.warning("Failed to emit DAG success event: \n %s", traceback.format_exc())
|
|
401
414
|
|
|
402
|
-
def dag_failed(
|
|
415
|
+
def dag_failed(
|
|
416
|
+
self,
|
|
417
|
+
dag_id: str,
|
|
418
|
+
run_id: str,
|
|
419
|
+
end_date: datetime,
|
|
420
|
+
logical_date: datetime,
|
|
421
|
+
dag_run_state: DagRunState,
|
|
422
|
+
task_ids: list[str],
|
|
423
|
+
msg: str,
|
|
424
|
+
):
|
|
403
425
|
try:
|
|
404
426
|
event = RunEvent(
|
|
405
427
|
eventType=RunState.FAIL,
|
|
406
|
-
eventTime=
|
|
407
|
-
job=self._build_job(job_name=
|
|
428
|
+
eventTime=end_date.isoformat(),
|
|
429
|
+
job=self._build_job(job_name=dag_id, job_type=_JOB_TYPE_DAG),
|
|
408
430
|
run=Run(
|
|
409
431
|
runId=self.build_dag_run_id(
|
|
410
|
-
dag_id=
|
|
411
|
-
|
|
432
|
+
dag_id=dag_id,
|
|
433
|
+
logical_date=logical_date,
|
|
412
434
|
),
|
|
413
435
|
facets={
|
|
414
436
|
"errorMessage": error_message_run.ErrorMessageRunFacet(
|
|
415
437
|
message=msg, programmingLanguage="python"
|
|
416
438
|
),
|
|
417
|
-
**get_airflow_state_run_facet(
|
|
439
|
+
**get_airflow_state_run_facet(dag_id, run_id, task_ids, dag_run_state),
|
|
418
440
|
**get_airflow_debug_facet(),
|
|
419
441
|
},
|
|
420
442
|
),
|
|
@@ -27,11 +27,13 @@ from setproctitle import getproctitle, setproctitle
|
|
|
27
27
|
|
|
28
28
|
from airflow import settings
|
|
29
29
|
from airflow.listeners import hookimpl
|
|
30
|
+
from airflow.models import DagRun
|
|
30
31
|
from airflow.providers.openlineage import conf
|
|
31
32
|
from airflow.providers.openlineage.extractors import ExtractorManager
|
|
32
33
|
from airflow.providers.openlineage.plugins.adapter import OpenLineageAdapter, RunState
|
|
33
34
|
from airflow.providers.openlineage.utils.utils import (
|
|
34
35
|
IS_AIRFLOW_2_10_OR_HIGHER,
|
|
36
|
+
get_airflow_dag_run_facet,
|
|
35
37
|
get_airflow_debug_facet,
|
|
36
38
|
get_airflow_job_facet,
|
|
37
39
|
get_airflow_mapped_task_facet,
|
|
@@ -51,7 +53,7 @@ from airflow.utils.timeout import timeout
|
|
|
51
53
|
if TYPE_CHECKING:
|
|
52
54
|
from sqlalchemy.orm import Session
|
|
53
55
|
|
|
54
|
-
from airflow.models import
|
|
56
|
+
from airflow.models import TaskInstance
|
|
55
57
|
|
|
56
58
|
_openlineage_listener: OpenLineageListener | None = None
|
|
57
59
|
|
|
@@ -134,7 +136,7 @@ class OpenLineageListener:
|
|
|
134
136
|
return
|
|
135
137
|
parent_run_id = self.adapter.build_dag_run_id(
|
|
136
138
|
dag_id=dag.dag_id,
|
|
137
|
-
|
|
139
|
+
logical_date=dagrun.logical_date,
|
|
138
140
|
)
|
|
139
141
|
|
|
140
142
|
task_uuid = self.adapter.build_task_instance_run_id(
|
|
@@ -213,7 +215,7 @@ class OpenLineageListener:
|
|
|
213
215
|
def on_success():
|
|
214
216
|
parent_run_id = OpenLineageAdapter.build_dag_run_id(
|
|
215
217
|
dag_id=dag.dag_id,
|
|
216
|
-
|
|
218
|
+
logical_date=dagrun.logical_date,
|
|
217
219
|
)
|
|
218
220
|
|
|
219
221
|
task_uuid = OpenLineageAdapter.build_task_instance_run_id(
|
|
@@ -312,7 +314,7 @@ class OpenLineageListener:
|
|
|
312
314
|
def on_failure():
|
|
313
315
|
parent_run_id = OpenLineageAdapter.build_dag_run_id(
|
|
314
316
|
dag_id=dag.dag_id,
|
|
315
|
-
|
|
317
|
+
logical_date=dagrun.logical_date,
|
|
316
318
|
)
|
|
317
319
|
|
|
318
320
|
task_uuid = OpenLineageAdapter.build_task_instance_run_id(
|
|
@@ -413,65 +415,120 @@ class OpenLineageListener:
|
|
|
413
415
|
|
|
414
416
|
@hookimpl
|
|
415
417
|
def on_dag_run_running(self, dag_run: DagRun, msg: str) -> None:
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
418
|
+
try:
|
|
419
|
+
if dag_run.dag and not is_selective_lineage_enabled(dag_run.dag):
|
|
420
|
+
self.log.debug(
|
|
421
|
+
"Skipping OpenLineage event emission for DAG `%s` "
|
|
422
|
+
"due to lack of explicit lineage enablement for DAG while "
|
|
423
|
+
"[openlineage] selective_enable is on.",
|
|
424
|
+
dag_run.dag_id,
|
|
425
|
+
)
|
|
426
|
+
return
|
|
427
|
+
|
|
428
|
+
if not self.executor:
|
|
429
|
+
self.log.debug("Executor have not started before `on_dag_run_running`")
|
|
430
|
+
return
|
|
431
|
+
|
|
432
|
+
data_interval_start = (
|
|
433
|
+
dag_run.data_interval_start.isoformat() if dag_run.data_interval_start else None
|
|
422
434
|
)
|
|
423
|
-
|
|
435
|
+
data_interval_end = dag_run.data_interval_end.isoformat() if dag_run.data_interval_end else None
|
|
424
436
|
|
|
425
|
-
|
|
426
|
-
self.log.debug("Executor have not started before `on_dag_run_running`")
|
|
427
|
-
return
|
|
437
|
+
run_facets = {**get_airflow_dag_run_facet(dag_run)}
|
|
428
438
|
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
439
|
+
self.submit_callable(
|
|
440
|
+
self.adapter.dag_started,
|
|
441
|
+
dag_id=dag_run.dag_id,
|
|
442
|
+
run_id=dag_run.run_id,
|
|
443
|
+
logical_date=dag_run.logical_date,
|
|
444
|
+
start_date=dag_run.start_date,
|
|
445
|
+
nominal_start_time=data_interval_start,
|
|
446
|
+
nominal_end_time=data_interval_end,
|
|
447
|
+
run_facets=run_facets,
|
|
448
|
+
owners=[x.strip() for x in dag_run.dag.owner.split(",")] if dag_run.dag else None,
|
|
449
|
+
description=dag_run.dag.description if dag_run.dag else None,
|
|
450
|
+
# AirflowJobFacet should be created outside ProcessPoolExecutor that pickles objects,
|
|
451
|
+
# as it causes lack of some TaskGroup attributes and crashes event emission.
|
|
452
|
+
job_facets=get_airflow_job_facet(dag_run=dag_run),
|
|
453
|
+
)
|
|
454
|
+
except BaseException as e:
|
|
455
|
+
self.log.warning("OpenLineage received exception in method on_dag_run_running", exc_info=e)
|
|
441
456
|
|
|
442
457
|
@hookimpl
|
|
443
458
|
def on_dag_run_success(self, dag_run: DagRun, msg: str) -> None:
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
459
|
+
try:
|
|
460
|
+
if dag_run.dag and not is_selective_lineage_enabled(dag_run.dag):
|
|
461
|
+
self.log.debug(
|
|
462
|
+
"Skipping OpenLineage event emission for DAG `%s` "
|
|
463
|
+
"due to lack of explicit lineage enablement for DAG while "
|
|
464
|
+
"[openlineage] selective_enable is on.",
|
|
465
|
+
dag_run.dag_id,
|
|
466
|
+
)
|
|
467
|
+
return
|
|
452
468
|
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
469
|
+
if not self.executor:
|
|
470
|
+
self.log.debug("Executor have not started before `on_dag_run_success`")
|
|
471
|
+
return
|
|
456
472
|
|
|
457
|
-
|
|
473
|
+
if IS_AIRFLOW_2_10_OR_HIGHER:
|
|
474
|
+
task_ids = DagRun._get_partial_task_ids(dag_run.dag)
|
|
475
|
+
else:
|
|
476
|
+
task_ids = dag_run.dag.task_ids if dag_run.dag and dag_run.dag.partial else None
|
|
477
|
+
self.submit_callable(
|
|
478
|
+
self.adapter.dag_success,
|
|
479
|
+
dag_id=dag_run.dag_id,
|
|
480
|
+
run_id=dag_run.run_id,
|
|
481
|
+
end_date=dag_run.end_date,
|
|
482
|
+
logical_date=dag_run.logical_date,
|
|
483
|
+
task_ids=task_ids,
|
|
484
|
+
dag_run_state=dag_run.get_state(),
|
|
485
|
+
)
|
|
486
|
+
except BaseException as e:
|
|
487
|
+
self.log.warning("OpenLineage received exception in method on_dag_run_success", exc_info=e)
|
|
458
488
|
|
|
459
489
|
@hookimpl
|
|
460
490
|
def on_dag_run_failed(self, dag_run: DagRun, msg: str) -> None:
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
491
|
+
try:
|
|
492
|
+
if dag_run.dag and not is_selective_lineage_enabled(dag_run.dag):
|
|
493
|
+
self.log.debug(
|
|
494
|
+
"Skipping OpenLineage event emission for DAG `%s` "
|
|
495
|
+
"due to lack of explicit lineage enablement for DAG while "
|
|
496
|
+
"[openlineage] selective_enable is on.",
|
|
497
|
+
dag_run.dag_id,
|
|
498
|
+
)
|
|
499
|
+
return
|
|
500
|
+
|
|
501
|
+
if not self.executor:
|
|
502
|
+
self.log.debug("Executor have not started before `on_dag_run_failed`")
|
|
503
|
+
return
|
|
504
|
+
|
|
505
|
+
if IS_AIRFLOW_2_10_OR_HIGHER:
|
|
506
|
+
task_ids = DagRun._get_partial_task_ids(dag_run.dag)
|
|
507
|
+
else:
|
|
508
|
+
task_ids = dag_run.dag.task_ids if dag_run.dag and dag_run.dag.partial else None
|
|
509
|
+
self.submit_callable(
|
|
510
|
+
self.adapter.dag_failed,
|
|
511
|
+
dag_id=dag_run.dag_id,
|
|
512
|
+
run_id=dag_run.run_id,
|
|
513
|
+
end_date=dag_run.end_date,
|
|
514
|
+
logical_date=dag_run.logical_date,
|
|
515
|
+
dag_run_state=dag_run.get_state(),
|
|
516
|
+
task_ids=task_ids,
|
|
517
|
+
msg=msg,
|
|
467
518
|
)
|
|
468
|
-
|
|
519
|
+
except BaseException as e:
|
|
520
|
+
self.log.warning("OpenLineage received exception in method on_dag_run_failed", exc_info=e)
|
|
469
521
|
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
522
|
+
def submit_callable(self, callable, *args, **kwargs):
|
|
523
|
+
fut = self.executor.submit(callable, *args, **kwargs)
|
|
524
|
+
fut.add_done_callback(self.log_submit_error)
|
|
525
|
+
return fut
|
|
473
526
|
|
|
474
|
-
|
|
527
|
+
def log_submit_error(self, fut):
|
|
528
|
+
if fut.exception():
|
|
529
|
+
self.log.warning("Failed to submit method to executor", exc_info=fut.exception())
|
|
530
|
+
else:
|
|
531
|
+
self.log.debug("Successfully submitted method to executor")
|
|
475
532
|
|
|
476
533
|
|
|
477
534
|
def get_openlineage_listener() -> OpenLineageListener:
|
|
@@ -33,7 +33,7 @@ from packaging.version import Version
|
|
|
33
33
|
from airflow import __version__ as AIRFLOW_VERSION
|
|
34
34
|
from airflow.datasets import Dataset
|
|
35
35
|
from airflow.exceptions import AirflowProviderDeprecationWarning # TODO: move this maybe to Airflow's logic?
|
|
36
|
-
from airflow.models import DAG, BaseOperator, MappedOperator
|
|
36
|
+
from airflow.models import DAG, BaseOperator, DagRun, MappedOperator
|
|
37
37
|
from airflow.providers.openlineage import conf
|
|
38
38
|
from airflow.providers.openlineage.plugins.facets import (
|
|
39
39
|
AirflowDagRunFacet,
|
|
@@ -58,9 +58,8 @@ if TYPE_CHECKING:
|
|
|
58
58
|
from openlineage.client.event_v2 import Dataset as OpenLineageDataset
|
|
59
59
|
from openlineage.client.facet_v2 import RunFacet
|
|
60
60
|
|
|
61
|
-
from airflow.models import
|
|
62
|
-
from airflow.utils.state import TaskInstanceState
|
|
63
|
-
|
|
61
|
+
from airflow.models import TaskInstance
|
|
62
|
+
from airflow.utils.state import DagRunState, TaskInstanceState
|
|
64
63
|
|
|
65
64
|
log = logging.getLogger(__name__)
|
|
66
65
|
_NOMINAL_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
|
|
@@ -199,7 +198,7 @@ class InfoJsonEncodable(dict):
|
|
|
199
198
|
return value.isoformat()
|
|
200
199
|
if isinstance(value, datetime.timedelta):
|
|
201
200
|
return f"{value.total_seconds()} seconds"
|
|
202
|
-
if isinstance(value, (set, tuple)):
|
|
201
|
+
if isinstance(value, (set, list, tuple)):
|
|
203
202
|
return str(list(value))
|
|
204
203
|
return value
|
|
205
204
|
|
|
@@ -244,7 +243,16 @@ class InfoJsonEncodable(dict):
|
|
|
244
243
|
class DagInfo(InfoJsonEncodable):
|
|
245
244
|
"""Defines encoding DAG object to JSON."""
|
|
246
245
|
|
|
247
|
-
includes = [
|
|
246
|
+
includes = [
|
|
247
|
+
"dag_id",
|
|
248
|
+
"description",
|
|
249
|
+
"fileloc",
|
|
250
|
+
"owner",
|
|
251
|
+
"schedule_interval", # For Airflow 2.
|
|
252
|
+
"timetable_summary", # For Airflow 3.
|
|
253
|
+
"start_date",
|
|
254
|
+
"tags",
|
|
255
|
+
]
|
|
248
256
|
casts = {"timetable": lambda dag: dag.timetable.serialize() if getattr(dag, "timetable", None) else None}
|
|
249
257
|
renames = {"_dag_id": "dag_id"}
|
|
250
258
|
|
|
@@ -430,11 +438,14 @@ def get_airflow_job_facet(dag_run: DagRun) -> dict[str, AirflowJobFacet]:
|
|
|
430
438
|
}
|
|
431
439
|
|
|
432
440
|
|
|
433
|
-
def get_airflow_state_run_facet(
|
|
441
|
+
def get_airflow_state_run_facet(
|
|
442
|
+
dag_id: str, run_id: str, task_ids: list[str], dag_run_state: DagRunState
|
|
443
|
+
) -> dict[str, AirflowStateRunFacet]:
|
|
444
|
+
tis = DagRun.fetch_task_instances(dag_id=dag_id, run_id=run_id, task_ids=task_ids)
|
|
434
445
|
return {
|
|
435
446
|
"airflowState": AirflowStateRunFacet(
|
|
436
|
-
dagRunState=
|
|
437
|
-
tasksState={ti.task_id: ti.state for ti in
|
|
447
|
+
dagRunState=dag_run_state,
|
|
448
|
+
tasksState={ti.task_id: ti.state for ti in tis},
|
|
438
449
|
)
|
|
439
450
|
}
|
|
440
451
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: apache-airflow-providers-openlineage
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.12.0
|
|
4
4
|
Summary: Provider package apache-airflow-providers-openlineage for Apache Airflow
|
|
5
5
|
Keywords: airflow-provider,openlineage,airflow,integration
|
|
6
6
|
Author-email: Apache Software Foundation <dev@airflow.apache.org>
|
|
@@ -21,16 +21,16 @@ Classifier: Programming Language :: Python :: 3.10
|
|
|
21
21
|
Classifier: Programming Language :: Python :: 3.11
|
|
22
22
|
Classifier: Programming Language :: Python :: 3.12
|
|
23
23
|
Classifier: Topic :: System :: Monitoring
|
|
24
|
-
Requires-Dist: apache-airflow-providers-common-compat>=1.2.
|
|
25
|
-
Requires-Dist: apache-airflow-providers-common-sql>=1.6.
|
|
26
|
-
Requires-Dist: apache-airflow>=2.8.
|
|
24
|
+
Requires-Dist: apache-airflow-providers-common-compat>=1.2.0
|
|
25
|
+
Requires-Dist: apache-airflow-providers-common-sql>=1.6.0
|
|
26
|
+
Requires-Dist: apache-airflow>=2.8.0
|
|
27
27
|
Requires-Dist: attrs>=22.2
|
|
28
|
-
Requires-Dist: openlineage-integration-common>=1.
|
|
29
|
-
Requires-Dist: openlineage-python>=1.
|
|
28
|
+
Requires-Dist: openlineage-integration-common>=1.22.0
|
|
29
|
+
Requires-Dist: openlineage-python>=1.22.0
|
|
30
30
|
Requires-Dist: apache-airflow-providers-common-sql ; extra == "common.sql"
|
|
31
31
|
Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
|
|
32
|
-
Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.
|
|
33
|
-
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.
|
|
32
|
+
Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.12.0/changelog.html
|
|
33
|
+
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.12.0
|
|
34
34
|
Project-URL: Slack Chat, https://s.apache.org/airflow-slack
|
|
35
35
|
Project-URL: Source Code, https://github.com/apache/airflow
|
|
36
36
|
Project-URL: Twitter, https://twitter.com/ApacheAirflow
|
|
@@ -81,7 +81,7 @@ Provides-Extra: common.sql
|
|
|
81
81
|
|
|
82
82
|
Package ``apache-airflow-providers-openlineage``
|
|
83
83
|
|
|
84
|
-
Release: ``1.
|
|
84
|
+
Release: ``1.12.0``
|
|
85
85
|
|
|
86
86
|
|
|
87
87
|
`OpenLineage <https://openlineage.io/>`__
|
|
@@ -94,7 +94,7 @@ This is a provider package for ``openlineage`` provider. All classes for this pr
|
|
|
94
94
|
are in ``airflow.providers.openlineage`` python package.
|
|
95
95
|
|
|
96
96
|
You can find package information and changelog for the provider
|
|
97
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.
|
|
97
|
+
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.12.0/>`_.
|
|
98
98
|
|
|
99
99
|
Installation
|
|
100
100
|
------------
|
|
@@ -115,8 +115,8 @@ PIP package Version required
|
|
|
115
115
|
``apache-airflow-providers-common-sql`` ``>=1.6.0``
|
|
116
116
|
``apache-airflow-providers-common-compat`` ``>=1.2.0``
|
|
117
117
|
``attrs`` ``>=22.2``
|
|
118
|
-
``openlineage-integration-common`` ``>=1.
|
|
119
|
-
``openlineage-python`` ``>=1.
|
|
118
|
+
``openlineage-integration-common`` ``>=1.22.0``
|
|
119
|
+
``openlineage-python`` ``>=1.22.0``
|
|
120
120
|
========================================== ==================
|
|
121
121
|
|
|
122
122
|
Cross provider package dependencies
|
|
@@ -139,4 +139,4 @@ Dependent package
|
|
|
139
139
|
============================================================================================================ ==============
|
|
140
140
|
|
|
141
141
|
The changelog for the provider package can be found in the
|
|
142
|
-
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.
|
|
142
|
+
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.12.0/changelog.html>`_.
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
airflow/providers/openlineage/LICENSE,sha256=FFb4jd2AXnOOf7XLP04pQW6jbdhG49TxlGY6fFpCV1Y,13609
|
|
2
|
-
airflow/providers/openlineage/__init__.py,sha256=
|
|
2
|
+
airflow/providers/openlineage/__init__.py,sha256=wVQWE9FroV_NOf4gaRxKTwClcRM3ZBe72ib47IlFqeU,1499
|
|
3
3
|
airflow/providers/openlineage/conf.py,sha256=paV6AHxPJPdQyVMoTGBRHymT-bpbs4Tovttqy9oicMs,5151
|
|
4
|
-
airflow/providers/openlineage/get_provider_info.py,sha256=
|
|
4
|
+
airflow/providers/openlineage/get_provider_info.py,sha256=YrvNXWZR6N66NzKrsqCTaNJXxula6wW0kg3DeY3n4Zg,8907
|
|
5
5
|
airflow/providers/openlineage/sqlparser.py,sha256=c7q3VVw41S87ZFozrkrEr2oZK79N12mC3KdDs9V1IuM,15581
|
|
6
6
|
airflow/providers/openlineage/extractors/__init__.py,sha256=I0X4f6zUniclyD9zT0DFHRImpCpJVP4MkPJT3cd7X5I,1081
|
|
7
7
|
airflow/providers/openlineage/extractors/base.py,sha256=olafVPBKxeGjqXerCYM0vj2q78Lm4ErWjex_R6nhjKY,6635
|
|
@@ -15,16 +15,16 @@ airflow/providers/openlineage/facets/AirflowRunFacet.json,sha256=70mEaZShgSJp-2x
|
|
|
15
15
|
airflow/providers/openlineage/facets/AirflowStateRunFacet.json,sha256=xhHQEKD9Jopw-oqbkCCrrwFjfXnxvuJAritsmegKjuQ,937
|
|
16
16
|
airflow/providers/openlineage/facets/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
|
17
17
|
airflow/providers/openlineage/plugins/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
|
18
|
-
airflow/providers/openlineage/plugins/adapter.py,sha256=
|
|
18
|
+
airflow/providers/openlineage/plugins/adapter.py,sha256=GmrGiU9uAgg_2dJfbM3y4QGGxrwAUNaHrg6s5i5Z3LA,19865
|
|
19
19
|
airflow/providers/openlineage/plugins/facets.py,sha256=VvyMYR6ONkC95q5FdNmohv0scbA1Ej_B5cQ97as5GvA,4161
|
|
20
|
-
airflow/providers/openlineage/plugins/listener.py,sha256=
|
|
20
|
+
airflow/providers/openlineage/plugins/listener.py,sha256=QYgYam7THCdWL5IxKkAmN7PcUX-UEIrQFgI1PF15gjM,21316
|
|
21
21
|
airflow/providers/openlineage/plugins/macros.py,sha256=hgFA3ZdQibyn4KXIOsKYBm4WRKDLA5q6Asscx5rvNfM,3076
|
|
22
22
|
airflow/providers/openlineage/plugins/openlineage.py,sha256=T0L5Yxpyq_wzs2_hltJCMY5NKzgsYp0vuEn8LppV5PU,1915
|
|
23
23
|
airflow/providers/openlineage/utils/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
|
24
24
|
airflow/providers/openlineage/utils/selective_enable.py,sha256=dFJ7wK7J_-BFwcOKp9tqFOSrASV3lmLv7HtRkEuMk3Q,3087
|
|
25
25
|
airflow/providers/openlineage/utils/sql.py,sha256=bnuU9WvjVKcWVMN3cUp0jaHtU5_ZRM5I1OP1WhIdztg,9583
|
|
26
|
-
airflow/providers/openlineage/utils/utils.py,sha256=
|
|
27
|
-
apache_airflow_providers_openlineage-1.
|
|
28
|
-
apache_airflow_providers_openlineage-1.
|
|
29
|
-
apache_airflow_providers_openlineage-1.
|
|
30
|
-
apache_airflow_providers_openlineage-1.
|
|
26
|
+
airflow/providers/openlineage/utils/utils.py,sha256=ps8hFtO6N8TSWdOzB0RAo9Y3lkcuGfoLS9EM7411JGI,23775
|
|
27
|
+
apache_airflow_providers_openlineage-1.12.0.dist-info/entry_points.txt,sha256=GAx0_i2OeZzqaiiiYuA-xchICDXiCT5kVqpKSxsOjt4,214
|
|
28
|
+
apache_airflow_providers_openlineage-1.12.0.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
|
|
29
|
+
apache_airflow_providers_openlineage-1.12.0.dist-info/METADATA,sha256=jk9CaYjA_3IXbn6IfMibHZP175chXzC04eP95s_CH5Q,6518
|
|
30
|
+
apache_airflow_providers_openlineage-1.12.0.dist-info/RECORD,,
|
|
File without changes
|