acryl-datahub-airflow-plugin 1.3.1.5__tar.gz → 1.3.1.5rc2__tar.gz

acryl_datahub_airflow_plugin-1.3.1.5rc2/PKG-INFO

@@ -0,0 +1,91 @@
+Metadata-Version: 2.4
+Name: acryl-datahub-airflow-plugin
+Version: 1.3.1.5rc2
+Summary: Datahub Airflow plugin to capture executions and send to Datahub
+Home-page: https://docs.datahub.com/
+License: Apache-2.0
+Project-URL: Documentation, https://docs.datahub.com/docs/
+Project-URL: Source, https://github.com/datahub-project/datahub
+Project-URL: Changelog, https://github.com/datahub-project/datahub/releases
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Information Technology
+Classifier: Intended Audience :: System Administrators
+Classifier: Operating System :: Unix
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Environment :: Console
+Classifier: Environment :: MacOS X
+Classifier: Topic :: Software Development
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: openlineage-airflow<=1.30.1,>=1.2.0
+Requires-Dist: apache-airflow<3,>=2.7.0
+Requires-Dist: pydantic>=2.4.0
+Requires-Dist: acryl-datahub[datahub-rest,sql-parser]==1.3.1.5rc2
+Requires-Dist: acryl-datahub[datahub-rest]==1.3.1.5rc2
+Provides-Extra: ignore
+Provides-Extra: datahub-rest
+Requires-Dist: acryl-datahub[datahub-rest]==1.3.1.5rc2; extra == "datahub-rest"
+Provides-Extra: datahub-kafka
+Requires-Dist: acryl-datahub[datahub-kafka]==1.3.1.5rc2; extra == "datahub-kafka"
+Provides-Extra: datahub-file
+Requires-Dist: acryl-datahub[sync-file-emitter]==1.3.1.5rc2; extra == "datahub-file"
+Provides-Extra: dev
+Requires-Dist: apache-airflow<3,>=2.7.0; extra == "dev"
+Requires-Dist: tenacity; extra == "dev"
+Requires-Dist: types-PyYAML; extra == "dev"
+Requires-Dist: pytest-cov>=2.8.1; extra == "dev"
+Requires-Dist: openlineage-airflow<=1.30.1,>=1.2.0; extra == "dev"
+Requires-Dist: pydantic>=2.4.0; extra == "dev"
+Requires-Dist: types-click==0.1.12; extra == "dev"
+Requires-Dist: tox; extra == "dev"
+Requires-Dist: types-toml; extra == "dev"
+Requires-Dist: deepdiff!=8.0.0; extra == "dev"
+Requires-Dist: build; extra == "dev"
+Requires-Dist: types-requests; extra == "dev"
+Requires-Dist: sqlalchemy-stubs; extra == "dev"
+Requires-Dist: types-dataclasses; extra == "dev"
+Requires-Dist: types-tabulate; extra == "dev"
+Requires-Dist: mypy==1.17.1; extra == "dev"
+Requires-Dist: types-python-dateutil; extra == "dev"
+Requires-Dist: types-setuptools; extra == "dev"
+Requires-Dist: pytest>=6.2.2; extra == "dev"
+Requires-Dist: packaging; extra == "dev"
+Requires-Dist: types-six; extra == "dev"
+Requires-Dist: acryl-datahub[datahub-rest]==1.3.1.5rc2; extra == "dev"
+Requires-Dist: ruff==0.11.7; extra == "dev"
+Requires-Dist: acryl-datahub[datahub-rest,sql-parser]==1.3.1.5rc2; extra == "dev"
+Requires-Dist: tox-uv; extra == "dev"
+Requires-Dist: twine; extra == "dev"
+Requires-Dist: types-cachetools; extra == "dev"
+Requires-Dist: coverage>=5.1; extra == "dev"
+Provides-Extra: integration-tests
+Requires-Dist: acryl-datahub[testing-utils]==1.3.1.5rc2; extra == "integration-tests"
+Requires-Dist: acryl-datahub[sync-file-emitter]==1.3.1.5rc2; extra == "integration-tests"
+Requires-Dist: virtualenv; extra == "integration-tests"
+Requires-Dist: apache-airflow[amazon,google,snowflake]>=2.0.2; extra == "integration-tests"
+Requires-Dist: snowflake-connector-python>=2.7.10; extra == "integration-tests"
+Requires-Dist: acryl-datahub[datahub-kafka]==1.3.1.5rc2; extra == "integration-tests"
+Requires-Dist: apache-airflow-providers-sqlite; extra == "integration-tests"
+Requires-Dist: apache-airflow-providers-teradata; extra == "integration-tests"
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license
+Dynamic: project-url
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# Datahub Airflow Plugin
+
+See [the DataHub Airflow docs](https://docs.datahub.com/docs/lineage/airflow) for details.
+
+## Developing
+
+See the [developing docs](../../metadata-ingestion/developing.md).

acryl_datahub_airflow_plugin-1.3.1.5rc2/README.md

@@ -0,0 +1,7 @@
+# Datahub Airflow Plugin
+
+See [the DataHub Airflow docs](https://docs.datahub.com/docs/lineage/airflow) for details.
+
+## Developing
+
+See the [developing docs](../../metadata-ingestion/developing.md).

{acryl_datahub_airflow_plugin-1.3.1.5 → acryl_datahub_airflow_plugin-1.3.1.5rc2}/setup.cfg

@@ -9,24 +9,6 @@ check_untyped_defs = yes
 disallow_incomplete_defs = yes
 disallow_untyped_decorators = yes
 warn_unused_configs = yes
-
-[mypy-datahub_airflow_plugin._extractors]
-ignore_errors = True
-
-[mypy-datahub_airflow_plugin.airflow2._extractors]
-ignore_errors = True
-
-[mypy-datahub_airflow_plugin.airflow2.datahub_listener]
-ignore_errors = True
-
-[mypy-tests.unit.test_teradata_extractor]
-ignore_errors = True
-
-[mypy-datahub_airflow_plugin.example_dags.*]
-ignore_errors = True
-
-[mypy-tests.integration.dags.*]
-ignore_errors = True
 disallow_untyped_defs = no
 
 [mypy-datahub.*]

{acryl_datahub_airflow_plugin-1.3.1.5 → acryl_datahub_airflow_plugin-1.3.1.5rc2}/setup.py

@@ -25,12 +25,12 @@ _self_pin = (
 base_requirements = {
     f"acryl-datahub[sql-parser,datahub-rest]{_self_pin}",
     "pydantic>=2.4.0",
-    # We require Airflow 2.5.x at minimum, since we need the new DAG listener API.
-    # We support both Airflow 2.x and 3.x with full backward compatibility.
-    "apache-airflow>=2.5.0,<4.0.0",
-    # Note: OpenLineage dependencies are version-specific and provided via extras:
-    # - airflow2: for Airflow 2.x (uses standalone openlineage-airflow package)
-    # - airflow3: for Airflow 3.x (uses native apache-airflow-providers-openlineage)
+    # We require Airflow 2.7.x at minimum, to be compatible with the native Airflow Openlineage provider.
+    "apache-airflow>=2.7.0,<3",
+    # We remain restrictive on the versions allowed here to prevent
+    # us from being broken by backwards-incompatible changes in the
+    # underlying package.
+    "openlineage-airflow>=1.2.0,<=1.30.1",
 }
 
 plugins: Dict[str, Set[str]] = {
@@ -43,14 +43,6 @@ plugins: Dict[str, Set[str]] = {
     "datahub-file": {
         f"acryl-datahub[sync-file-emitter]{_self_pin}",
     },
-    # airflow2: For Airflow 2.x, use standalone openlineage-airflow package
-    "airflow2": {
-        "openlineage-airflow>=1.2.0",
-    },
-    # airflow3: For Airflow 3.x, use native OpenLineage provider
-    "airflow3": {
-        "apache-airflow-providers-openlineage>=1.0.0",
-    },
 }
 
 # Require some plugins by default.

acryl_datahub_airflow_plugin-1.3.1.5rc2/src/acryl_datahub_airflow_plugin.egg-info/PKG-INFO

@@ -0,0 +1,91 @@
+Metadata-Version: 2.4
+Name: acryl-datahub-airflow-plugin
+Version: 1.3.1.5rc2
+Summary: Datahub Airflow plugin to capture executions and send to Datahub
+Home-page: https://docs.datahub.com/
+License: Apache-2.0
+Project-URL: Documentation, https://docs.datahub.com/docs/
+Project-URL: Source, https://github.com/datahub-project/datahub
+Project-URL: Changelog, https://github.com/datahub-project/datahub/releases
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Information Technology
+Classifier: Intended Audience :: System Administrators
+Classifier: Operating System :: Unix
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Environment :: Console
+Classifier: Environment :: MacOS X
+Classifier: Topic :: Software Development
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: openlineage-airflow<=1.30.1,>=1.2.0
+Requires-Dist: apache-airflow<3,>=2.7.0
+Requires-Dist: pydantic>=2.4.0
+Requires-Dist: acryl-datahub[datahub-rest,sql-parser]==1.3.1.5rc2
+Requires-Dist: acryl-datahub[datahub-rest]==1.3.1.5rc2
+Provides-Extra: ignore
+Provides-Extra: datahub-rest
+Requires-Dist: acryl-datahub[datahub-rest]==1.3.1.5rc2; extra == "datahub-rest"
+Provides-Extra: datahub-kafka
+Requires-Dist: acryl-datahub[datahub-kafka]==1.3.1.5rc2; extra == "datahub-kafka"
+Provides-Extra: datahub-file
+Requires-Dist: acryl-datahub[sync-file-emitter]==1.3.1.5rc2; extra == "datahub-file"
+Provides-Extra: dev
+Requires-Dist: apache-airflow<3,>=2.7.0; extra == "dev"
+Requires-Dist: tenacity; extra == "dev"
+Requires-Dist: types-PyYAML; extra == "dev"
+Requires-Dist: pytest-cov>=2.8.1; extra == "dev"
+Requires-Dist: openlineage-airflow<=1.30.1,>=1.2.0; extra == "dev"
+Requires-Dist: pydantic>=2.4.0; extra == "dev"
+Requires-Dist: types-click==0.1.12; extra == "dev"
+Requires-Dist: tox; extra == "dev"
+Requires-Dist: types-toml; extra == "dev"
+Requires-Dist: deepdiff!=8.0.0; extra == "dev"
+Requires-Dist: build; extra == "dev"
+Requires-Dist: types-requests; extra == "dev"
+Requires-Dist: sqlalchemy-stubs; extra == "dev"
+Requires-Dist: types-dataclasses; extra == "dev"
+Requires-Dist: types-tabulate; extra == "dev"
+Requires-Dist: mypy==1.17.1; extra == "dev"
+Requires-Dist: types-python-dateutil; extra == "dev"
+Requires-Dist: types-setuptools; extra == "dev"
+Requires-Dist: pytest>=6.2.2; extra == "dev"
+Requires-Dist: packaging; extra == "dev"
+Requires-Dist: types-six; extra == "dev"
+Requires-Dist: acryl-datahub[datahub-rest]==1.3.1.5rc2; extra == "dev"
+Requires-Dist: ruff==0.11.7; extra == "dev"
+Requires-Dist: acryl-datahub[datahub-rest,sql-parser]==1.3.1.5rc2; extra == "dev"
+Requires-Dist: tox-uv; extra == "dev"
+Requires-Dist: twine; extra == "dev"
+Requires-Dist: types-cachetools; extra == "dev"
+Requires-Dist: coverage>=5.1; extra == "dev"
+Provides-Extra: integration-tests
+Requires-Dist: acryl-datahub[testing-utils]==1.3.1.5rc2; extra == "integration-tests"
+Requires-Dist: acryl-datahub[sync-file-emitter]==1.3.1.5rc2; extra == "integration-tests"
+Requires-Dist: virtualenv; extra == "integration-tests"
+Requires-Dist: apache-airflow[amazon,google,snowflake]>=2.0.2; extra == "integration-tests"
+Requires-Dist: snowflake-connector-python>=2.7.10; extra == "integration-tests"
+Requires-Dist: acryl-datahub[datahub-kafka]==1.3.1.5rc2; extra == "integration-tests"
+Requires-Dist: apache-airflow-providers-sqlite; extra == "integration-tests"
+Requires-Dist: apache-airflow-providers-teradata; extra == "integration-tests"
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license
+Dynamic: project-url
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# Datahub Airflow Plugin
+
+See [the DataHub Airflow docs](https://docs.datahub.com/docs/lineage/airflow) for details.
+
+## Developing
+
+See the [developing docs](../../metadata-ingestion/developing.md).

acryl_datahub_airflow_plugin-1.3.1.5rc2/src/acryl_datahub_airflow_plugin.egg-info/SOURCES.txt

@@ -0,0 +1,39 @@
+README.md
+pyproject.toml
+setup.cfg
+setup.py
+src/acryl_datahub_airflow_plugin.egg-info/PKG-INFO
+src/acryl_datahub_airflow_plugin.egg-info/SOURCES.txt
+src/acryl_datahub_airflow_plugin.egg-info/dependency_links.txt
+src/acryl_datahub_airflow_plugin.egg-info/entry_points.txt
+src/acryl_datahub_airflow_plugin.egg-info/not-zip-safe
+src/acryl_datahub_airflow_plugin.egg-info/requires.txt
+src/acryl_datahub_airflow_plugin.egg-info/top_level.txt
+src/datahub_airflow_plugin/__init__.py
+src/datahub_airflow_plugin/_airflow_shims.py
+src/datahub_airflow_plugin/_config.py
+src/datahub_airflow_plugin/_datahub_ol_adapter.py
+src/datahub_airflow_plugin/_extractors.py
+src/datahub_airflow_plugin/_version.py
+src/datahub_airflow_plugin/datahub_listener.py
+src/datahub_airflow_plugin/datahub_plugin.py
+src/datahub_airflow_plugin/entities.py
+src/datahub_airflow_plugin/py.typed
+src/datahub_airflow_plugin/client/__init__.py
+src/datahub_airflow_plugin/client/airflow_generator.py
+src/datahub_airflow_plugin/example_dags/__init__.py
+src/datahub_airflow_plugin/example_dags/generic_recipe_sample_dag.py
+src/datahub_airflow_plugin/example_dags/graph_usage_sample_dag.py
+src/datahub_airflow_plugin/example_dags/lineage_backend_demo.py
+src/datahub_airflow_plugin/example_dags/lineage_backend_taskflow_demo.py
+src/datahub_airflow_plugin/example_dags/lineage_emission_dag.py
+src/datahub_airflow_plugin/example_dags/mysql_sample_dag.py
+src/datahub_airflow_plugin/example_dags/snowflake_sample_dag.py
+src/datahub_airflow_plugin/hooks/__init__.py
+src/datahub_airflow_plugin/hooks/datahub.py
+src/datahub_airflow_plugin/operators/__init__.py
+src/datahub_airflow_plugin/operators/datahub.py
+src/datahub_airflow_plugin/operators/datahub_assertion_operator.py
+src/datahub_airflow_plugin/operators/datahub_assertion_sensor.py
+src/datahub_airflow_plugin/operators/datahub_operation_operator.py
+src/datahub_airflow_plugin/operators/datahub_operation_sensor.py

{acryl_datahub_airflow_plugin-1.3.1.5 → acryl_datahub_airflow_plugin-1.3.1.5rc2}/src/acryl_datahub_airflow_plugin.egg-info/requires.txt

@@ -1,60 +1,56 @@
+openlineage-airflow<=1.30.1,>=1.2.0
+apache-airflow<3,>=2.7.0
 pydantic>=2.4.0
-acryl-datahub[datahub-rest,sql-parser]==1.3.1.5
-acryl-datahub[datahub-rest]==1.3.1.5
-apache-airflow<4.0.0,>=2.5.0
-
-[airflow2]
-openlineage-airflow>=1.2.0
-
-[airflow3]
-apache-airflow-providers-openlineage>=1.0.0
+acryl-datahub[datahub-rest,sql-parser]==1.3.1.5rc2
+acryl-datahub[datahub-rest]==1.3.1.5rc2
 
 [datahub-file]
-acryl-datahub[sync-file-emitter]==1.3.1.5
+acryl-datahub[sync-file-emitter]==1.3.1.5rc2
 
 [datahub-kafka]
-acryl-datahub[datahub-kafka]==1.3.1.5
+acryl-datahub[datahub-kafka]==1.3.1.5rc2
 
 [datahub-rest]
-acryl-datahub[datahub-rest]==1.3.1.5
+acryl-datahub[datahub-rest]==1.3.1.5rc2
 
 [dev]
-acryl-datahub[datahub-rest]==1.3.1.5
-types-setuptools
-deepdiff!=8.0.0
-pydantic>=2.4.0
-types-six
-types-toml
-pytest>=6.2.2
+apache-airflow<3,>=2.7.0
 tenacity
+types-PyYAML
 pytest-cov>=2.8.1
-sqlalchemy-stubs
-types-cachetools
-ruff==0.11.7
-coverage>=5.1
+openlineage-airflow<=1.30.1,>=1.2.0
+pydantic>=2.4.0
+types-click==0.1.12
 tox
-twine
-tox-uv
-types-PyYAML
-mypy==1.17.1
-packaging
-acryl-datahub[datahub-rest,sql-parser]==1.3.1.5
-types-python-dateutil
+types-toml
+deepdiff!=8.0.0
 build
-apache-airflow<4.0.0,>=2.5.0
-types-click==0.1.12
+types-requests
+sqlalchemy-stubs
 types-dataclasses
 types-tabulate
-types-requests
+mypy==1.17.1
+types-python-dateutil
+types-setuptools
+pytest>=6.2.2
+packaging
+types-six
+acryl-datahub[datahub-rest]==1.3.1.5rc2
+ruff==0.11.7
+acryl-datahub[datahub-rest,sql-parser]==1.3.1.5rc2
+tox-uv
+twine
+types-cachetools
+coverage>=5.1
 
 [ignore]
 
 [integration-tests]
+acryl-datahub[testing-utils]==1.3.1.5rc2
+acryl-datahub[sync-file-emitter]==1.3.1.5rc2
 virtualenv
 apache-airflow[amazon,google,snowflake]>=2.0.2
-acryl-datahub[sync-file-emitter]==1.3.1.5
-apache-airflow-providers-sqlite
 snowflake-connector-python>=2.7.10
+acryl-datahub[datahub-kafka]==1.3.1.5rc2
+apache-airflow-providers-sqlite
 apache-airflow-providers-teradata
-acryl-datahub[testing-utils]==1.3.1.5
-acryl-datahub[datahub-kafka]==1.3.1.5

acryl_datahub_airflow_plugin-1.3.1.5rc2/src/datahub_airflow_plugin/_airflow_shims.py

@@ -0,0 +1,42 @@
+from typing import List
+
+import airflow.version
+import packaging.version
+from airflow.models.operator import Operator
+
+try:
+    from airflow.sensors.external_task import ExternalTaskSensor
+except ImportError:
+    from airflow.sensors.external_task_sensor import ExternalTaskSensor  # type: ignore
+
+# Approach suggested by https://stackoverflow.com/a/11887885/5004662.
+AIRFLOW_VERSION = packaging.version.parse(airflow.version.version)
+HAS_AIRFLOW_DAG_LISTENER_API = True  # this is in Airflow 2.5+
+HAS_AIRFLOW_DATASET_LISTENER_API = AIRFLOW_VERSION >= packaging.version.parse(
+    "2.8.0.dev0"
+)
+
+
+def get_task_inlets(operator: "Operator") -> List:
+    # From Airflow 2.4 _inlets is dropped and inlets used consistently. Earlier it was not the case, so we have to stick there to _inlets
+    if hasattr(operator, "_inlets"):
+        return operator._inlets  # type: ignore[attr-defined, union-attr]
+    if hasattr(operator, "get_inlet_defs"):
+        return operator.get_inlet_defs()  # type: ignore[attr-defined]
+    return operator.inlets or []
+
+
+def get_task_outlets(operator: "Operator") -> List:
+    # From Airflow 2.4 _outlets is dropped and inlets used consistently. Earlier it was not the case, so we have to stick there to _outlets
+    # We have to use _outlets because outlets is empty in Airflow < 2.4.0
+    if hasattr(operator, "_outlets"):
+        return operator._outlets  # type: ignore[attr-defined, union-attr]
+    if hasattr(operator, "get_outlet_defs"):
+        return operator.get_outlet_defs()
+    return operator.outlets or []
+
+
+__all__ = [
+    "AIRFLOW_VERSION",
+    "ExternalTaskSensor",
+]

acryl_datahub_airflow_plugin-1.3.1.5rc2/src/datahub_airflow_plugin/_config.py

@@ -0,0 +1,142 @@
+from enum import Enum
+from typing import TYPE_CHECKING, Dict, List, Optional, Union
+
+from airflow.configuration import conf
+from pydantic import root_validator
+from pydantic.fields import Field
+
+import datahub.emitter.mce_builder as builder
+from datahub.configuration.common import AllowDenyPattern, ConfigModel
+
+if TYPE_CHECKING:
+    from datahub_airflow_plugin.hooks.datahub import (
+        DatahubCompositeHook,
+        DatahubGenericHook,
+    )
+
+
+class DatajobUrl(Enum):
+    GRID = "grid"
+    TASKINSTANCE = "taskinstance"
+
+
+class DatahubLineageConfig(ConfigModel):
+    enabled: bool
+
+    # DataHub hook connection ID.
+    datahub_conn_id: str
+
+    _datahub_connection_ids: List[str]
+
+    # Cluster to associate with the pipelines and tasks. Defaults to "prod".
+    cluster: str
+
+    # Platform instance to associate with the pipelines and tasks.
+    platform_instance: Optional[str]
+
+    # If true, the owners field of the DAG will be captured as a DataHub corpuser.
+    capture_ownership_info: bool
+
+    # If true, the owners field of the DAG will instead be captured as a DataHub corpgroup.
+    capture_ownership_as_group: bool
+
+    # If true, the tags field of the DAG will be captured as DataHub tags.
+    capture_tags_info: bool
+
+    # If true (default), we'll materialize and un-soft-delete any urns
+    # referenced by inlets or outlets.
+    materialize_iolets: bool
+
+    capture_executions: bool
+
+    datajob_url_link: DatajobUrl
+
+    enable_extractors: bool
+
+    # If true, ti.render_templates() will be called in the listener.
+    # Makes extraction of jinja-templated fields more accurate.
+    render_templates: bool
+
+    # Only if true, lineage will be emitted for the DataJobs.
+    enable_datajob_lineage: bool
+
+    dag_filter_pattern: AllowDenyPattern = Field(
+        description="regex patterns for DAGs to ingest",
+    )
+
+    log_level: Optional[str]
+    debug_emitter: bool
+
+    disable_openlineage_plugin: bool
+
+    def make_emitter_hook(self) -> Union["DatahubGenericHook", "DatahubCompositeHook"]:
+        # This is necessary to avoid issues with circular imports.
+        from datahub_airflow_plugin.hooks.datahub import (
+            DatahubCompositeHook,
+            DatahubGenericHook,
+        )
+
+        if len(self._datahub_connection_ids) == 1:
+            return DatahubGenericHook(self._datahub_connection_ids[0])
+        else:
+            return DatahubCompositeHook(self._datahub_connection_ids)
+
+    @root_validator(skip_on_failure=True)
+    def split_conn_ids(cls, values: Dict) -> Dict:
+        if not values.get("datahub_conn_id"):
+            raise ValueError("datahub_conn_id is required")
+        conn_ids = values.get("datahub_conn_id", "").split(",")
+        cls._datahub_connection_ids = [conn_id.strip() for conn_id in conn_ids]
+        return values
+
+
+def get_lineage_config() -> DatahubLineageConfig:
+    """Load the DataHub plugin config from airflow.cfg."""
+
+    enabled = conf.get("datahub", "enabled", fallback=True)
+    datahub_conn_id = conf.get("datahub", "conn_id", fallback="datahub_rest_default")
+    cluster = conf.get("datahub", "cluster", fallback=builder.DEFAULT_FLOW_CLUSTER)
+    platform_instance = conf.get("datahub", "platform_instance", fallback=None)
+    capture_tags_info = conf.get("datahub", "capture_tags_info", fallback=True)
+    capture_ownership_info = conf.get(
+        "datahub", "capture_ownership_info", fallback=True
+    )
+    capture_ownership_as_group = conf.get(
+        "datahub", "capture_ownership_as_group", fallback=False
+    )
+    capture_executions = conf.get("datahub", "capture_executions", fallback=True)
+    materialize_iolets = conf.get("datahub", "materialize_iolets", fallback=True)
+    enable_extractors = conf.get("datahub", "enable_extractors", fallback=True)
+    log_level = conf.get("datahub", "log_level", fallback=None)
+    debug_emitter = conf.get("datahub", "debug_emitter", fallback=False)
+    disable_openlineage_plugin = conf.get(
+        "datahub", "disable_openlineage_plugin", fallback=True
+    )
+    render_templates = conf.get("datahub", "render_templates", fallback=True)
+    datajob_url_link = conf.get(
+        "datahub", "datajob_url_link", fallback=DatajobUrl.TASKINSTANCE.value
+    )
+    dag_filter_pattern = AllowDenyPattern.model_validate_json(
+        conf.get("datahub", "dag_filter_str", fallback='{"allow": [".*"]}')
+    )
+    enable_lineage = conf.get("datahub", "enable_datajob_lineage", fallback=True)
+
+    return DatahubLineageConfig(
+        enabled=enabled,
+        datahub_conn_id=datahub_conn_id,
+        cluster=cluster,
+        platform_instance=platform_instance,
+        capture_ownership_info=capture_ownership_info,
+        capture_ownership_as_group=capture_ownership_as_group,
+        capture_tags_info=capture_tags_info,
+        capture_executions=capture_executions,
+        materialize_iolets=materialize_iolets,
+        enable_extractors=enable_extractors,
+        log_level=log_level,
+        debug_emitter=debug_emitter,
+        disable_openlineage_plugin=disable_openlineage_plugin,
+        datajob_url_link=datajob_url_link,
+        render_templates=render_templates,
+        dag_filter_pattern=dag_filter_pattern,
+        enable_datajob_lineage=enable_lineage,
+    )

acryl_datahub_airflow_plugin-1.3.1.5rc2/src/datahub_airflow_plugin/_datahub_ol_adapter.py

@@ -0,0 +1,23 @@
+import logging
+
+from openlineage.client.run import Dataset as OpenLineageDataset
+
+import datahub.emitter.mce_builder as builder
+
+logger = logging.getLogger(__name__)
+
+
+OL_SCHEME_TWEAKS = {
+    "sqlserver": "mssql",
+    "awsathena": "athena",
+}
+
+
+def translate_ol_to_datahub_urn(ol_uri: OpenLineageDataset) -> str:
+    namespace = ol_uri.namespace
+    name = ol_uri.name
+
+    scheme, *rest = namespace.split("://", maxsplit=1)
+
+    platform = OL_SCHEME_TWEAKS.get(scheme, scheme)
+    return builder.make_dataset_urn(platform=platform, name=name)