acryl-datahub-airflow-plugin 1.3.1.7rc1__tar.gz → 1.3.1.8rc1__tar.gz

{acryl_datahub_airflow_plugin-1.3.1.7rc1 → acryl_datahub_airflow_plugin-1.3.1.8rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: acryl-datahub-airflow-plugin
-Version: 1.3.1.7rc1
+Version: 1.3.1.8rc1
 Summary: Datahub Airflow plugin to capture executions and send to Datahub
 Home-page: https://docs.datahub.com/
 License: Apache-2.0
@@ -19,60 +19,60 @@ Classifier: Operating System :: POSIX :: Linux
 Classifier: Environment :: Console
 Classifier: Environment :: MacOS X
 Classifier: Topic :: Software Development
-Requires-Python: >=3.9
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
-Requires-Dist: pydantic>=2.4.0
-Requires-Dist: acryl-datahub[datahub-rest,sql-parser]==1.3.1.7rc1
+Requires-Dist: acryl-datahub[datahub-rest,sql-parser]==1.3.1.8rc1
 Requires-Dist: apache-airflow<4.0.0,>=2.5.0
-Requires-Dist: acryl-datahub[datahub-rest]==1.3.1.7rc1
+Requires-Dist: acryl-datahub[datahub-rest]==1.3.1.8rc1
+Requires-Dist: pydantic>=2.4.0
 Provides-Extra: ignore
 Provides-Extra: datahub-rest
-Requires-Dist: acryl-datahub[datahub-rest]==1.3.1.7rc1; extra == "datahub-rest"
+Requires-Dist: acryl-datahub[datahub-rest]==1.3.1.8rc1; extra == "datahub-rest"
 Provides-Extra: datahub-kafka
-Requires-Dist: acryl-datahub[datahub-kafka]==1.3.1.7rc1; extra == "datahub-kafka"
+Requires-Dist: acryl-datahub[datahub-kafka]==1.3.1.8rc1; extra == "datahub-kafka"
 Provides-Extra: datahub-file
-Requires-Dist: acryl-datahub[sync-file-emitter]==1.3.1.7rc1; extra == "datahub-file"
+Requires-Dist: acryl-datahub[sync-file-emitter]==1.3.1.8rc1; extra == "datahub-file"
 Provides-Extra: airflow2
 Requires-Dist: openlineage-airflow>=1.2.0; extra == "airflow2"
 Provides-Extra: airflow3
 Requires-Dist: apache-airflow-providers-openlineage>=1.0.0; extra == "airflow3"
 Provides-Extra: dev
+Requires-Dist: tox; extra == "dev"
 Requires-Dist: pydantic>=2.4.0; extra == "dev"
+Requires-Dist: packaging; extra == "dev"
 Requires-Dist: apache-airflow<4.0.0,>=2.5.0; extra == "dev"
-Requires-Dist: ruff==0.11.7; extra == "dev"
-Requires-Dist: tox; extra == "dev"
+Requires-Dist: build; extra == "dev"
+Requires-Dist: types-tabulate; extra == "dev"
+Requires-Dist: types-click==0.1.12; extra == "dev"
+Requires-Dist: types-cachetools; extra == "dev"
 Requires-Dist: types-six; extra == "dev"
-Requires-Dist: types-PyYAML; extra == "dev"
-Requires-Dist: coverage>=5.1; extra == "dev"
+Requires-Dist: tenacity; extra == "dev"
 Requires-Dist: deepdiff!=8.0.0; extra == "dev"
-Requires-Dist: packaging; extra == "dev"
+Requires-Dist: acryl-datahub[datahub-rest]==1.3.1.8rc1; extra == "dev"
 Requires-Dist: tox-uv; extra == "dev"
-Requires-Dist: acryl-datahub[datahub-rest]==1.3.1.7rc1; extra == "dev"
-Requires-Dist: types-cachetools; extra == "dev"
-Requires-Dist: mypy==1.17.1; extra == "dev"
-Requires-Dist: twine; extra == "dev"
-Requires-Dist: pytest>=6.2.2; extra == "dev"
-Requires-Dist: build; extra == "dev"
 Requires-Dist: types-dataclasses; extra == "dev"
-Requires-Dist: tenacity; extra == "dev"
+Requires-Dist: types-setuptools; extra == "dev"
 Requires-Dist: types-toml; extra == "dev"
-Requires-Dist: sqlalchemy-stubs; extra == "dev"
-Requires-Dist: types-tabulate; extra == "dev"
-Requires-Dist: acryl-datahub[datahub-rest,sql-parser]==1.3.1.7rc1; extra == "dev"
+Requires-Dist: types-PyYAML; extra == "dev"
+Requires-Dist: mypy==1.17.1; extra == "dev"
+Requires-Dist: pytest-cov>=2.8.1; extra == "dev"
 Requires-Dist: types-requests; extra == "dev"
+Requires-Dist: twine; extra == "dev"
+Requires-Dist: acryl-datahub[datahub-rest,sql-parser]==1.3.1.8rc1; extra == "dev"
+Requires-Dist: sqlalchemy-stubs; extra == "dev"
+Requires-Dist: coverage>=5.1; extra == "dev"
+Requires-Dist: ruff==0.11.7; extra == "dev"
 Requires-Dist: types-python-dateutil; extra == "dev"
-Requires-Dist: types-setuptools; extra == "dev"
-Requires-Dist: types-click==0.1.12; extra == "dev"
-Requires-Dist: pytest-cov>=2.8.1; extra == "dev"
+Requires-Dist: pytest>=6.2.2; extra == "dev"
 Provides-Extra: integration-tests
-Requires-Dist: acryl-datahub[sync-file-emitter]==1.3.1.7rc1; extra == "integration-tests"
-Requires-Dist: acryl-datahub[testing-utils]==1.3.1.7rc1; extra == "integration-tests"
-Requires-Dist: apache-airflow[amazon,google,snowflake]>=2.0.2; extra == "integration-tests"
-Requires-Dist: snowflake-connector-python>=2.7.10; extra == "integration-tests"
-Requires-Dist: apache-airflow-providers-sqlite; extra == "integration-tests"
+Requires-Dist: acryl-datahub[datahub-kafka]==1.3.1.8rc1; extra == "integration-tests"
 Requires-Dist: apache-airflow-providers-teradata; extra == "integration-tests"
+Requires-Dist: acryl-datahub[sync-file-emitter]==1.3.1.8rc1; extra == "integration-tests"
+Requires-Dist: snowflake-connector-python>=2.7.10; extra == "integration-tests"
+Requires-Dist: acryl-datahub[testing-utils]==1.3.1.8rc1; extra == "integration-tests"
+Requires-Dist: apache-airflow[amazon,google,snowflake]>=2.0.2; extra == "integration-tests"
 Requires-Dist: virtualenv; extra == "integration-tests"
-Requires-Dist: acryl-datahub[datahub-kafka]==1.3.1.7rc1; extra == "integration-tests"
+Requires-Dist: apache-airflow-providers-sqlite; extra == "integration-tests"
 Dynamic: classifier
 Dynamic: description
 Dynamic: description-content-type

{acryl_datahub_airflow_plugin-1.3.1.7rc1 → acryl_datahub_airflow_plugin-1.3.1.8rc1}/pyproject.toml

@@ -4,7 +4,7 @@ requires = ["setuptools>=78.1.1", "wheel", "pip>=21.0.0"]
 
 [tool.ruff]
 line-length = 88
-target-version = "py39"
+target-version = "py310"
 exclude = [
     ".git",
     "venv",

{acryl_datahub_airflow_plugin-1.3.1.7rc1 → acryl_datahub_airflow_plugin-1.3.1.8rc1}/setup.py

@@ -146,7 +146,7 @@ setuptools.setup(
     ],
     # Package info.
     zip_safe=False,
-    python_requires=">=3.9",
+    python_requires=">=3.10",
     package_data={
         "datahub_airflow_plugin": ["py.typed"],
     },

{acryl_datahub_airflow_plugin-1.3.1.7rc1 → acryl_datahub_airflow_plugin-1.3.1.8rc1}/src/acryl_datahub_airflow_plugin.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: acryl-datahub-airflow-plugin
-Version: 1.3.1.7rc1
+Version: 1.3.1.8rc1
 Summary: Datahub Airflow plugin to capture executions and send to Datahub
 Home-page: https://docs.datahub.com/
 License: Apache-2.0
@@ -19,60 +19,60 @@ Classifier: Operating System :: POSIX :: Linux
 Classifier: Environment :: Console
 Classifier: Environment :: MacOS X
 Classifier: Topic :: Software Development
-Requires-Python: >=3.9
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
-Requires-Dist: pydantic>=2.4.0
-Requires-Dist: acryl-datahub[datahub-rest,sql-parser]==1.3.1.7rc1
+Requires-Dist: acryl-datahub[datahub-rest,sql-parser]==1.3.1.8rc1
 Requires-Dist: apache-airflow<4.0.0,>=2.5.0
-Requires-Dist: acryl-datahub[datahub-rest]==1.3.1.7rc1
+Requires-Dist: acryl-datahub[datahub-rest]==1.3.1.8rc1
+Requires-Dist: pydantic>=2.4.0
 Provides-Extra: ignore
 Provides-Extra: datahub-rest
-Requires-Dist: acryl-datahub[datahub-rest]==1.3.1.7rc1; extra == "datahub-rest"
+Requires-Dist: acryl-datahub[datahub-rest]==1.3.1.8rc1; extra == "datahub-rest"
 Provides-Extra: datahub-kafka
-Requires-Dist: acryl-datahub[datahub-kafka]==1.3.1.7rc1; extra == "datahub-kafka"
+Requires-Dist: acryl-datahub[datahub-kafka]==1.3.1.8rc1; extra == "datahub-kafka"
 Provides-Extra: datahub-file
-Requires-Dist: acryl-datahub[sync-file-emitter]==1.3.1.7rc1; extra == "datahub-file"
+Requires-Dist: acryl-datahub[sync-file-emitter]==1.3.1.8rc1; extra == "datahub-file"
 Provides-Extra: airflow2
 Requires-Dist: openlineage-airflow>=1.2.0; extra == "airflow2"
 Provides-Extra: airflow3
 Requires-Dist: apache-airflow-providers-openlineage>=1.0.0; extra == "airflow3"
 Provides-Extra: dev
+Requires-Dist: tox; extra == "dev"
 Requires-Dist: pydantic>=2.4.0; extra == "dev"
+Requires-Dist: packaging; extra == "dev"
 Requires-Dist: apache-airflow<4.0.0,>=2.5.0; extra == "dev"
-Requires-Dist: ruff==0.11.7; extra == "dev"
-Requires-Dist: tox; extra == "dev"
+Requires-Dist: build; extra == "dev"
+Requires-Dist: types-tabulate; extra == "dev"
+Requires-Dist: types-click==0.1.12; extra == "dev"
+Requires-Dist: types-cachetools; extra == "dev"
 Requires-Dist: types-six; extra == "dev"
-Requires-Dist: types-PyYAML; extra == "dev"
-Requires-Dist: coverage>=5.1; extra == "dev"
+Requires-Dist: tenacity; extra == "dev"
 Requires-Dist: deepdiff!=8.0.0; extra == "dev"
-Requires-Dist: packaging; extra == "dev"
+Requires-Dist: acryl-datahub[datahub-rest]==1.3.1.8rc1; extra == "dev"
 Requires-Dist: tox-uv; extra == "dev"
-Requires-Dist: acryl-datahub[datahub-rest]==1.3.1.7rc1; extra == "dev"
-Requires-Dist: types-cachetools; extra == "dev"
-Requires-Dist: mypy==1.17.1; extra == "dev"
-Requires-Dist: twine; extra == "dev"
-Requires-Dist: pytest>=6.2.2; extra == "dev"
-Requires-Dist: build; extra == "dev"
 Requires-Dist: types-dataclasses; extra == "dev"
-Requires-Dist: tenacity; extra == "dev"
+Requires-Dist: types-setuptools; extra == "dev"
 Requires-Dist: types-toml; extra == "dev"
-Requires-Dist: sqlalchemy-stubs; extra == "dev"
-Requires-Dist: types-tabulate; extra == "dev"
-Requires-Dist: acryl-datahub[datahub-rest,sql-parser]==1.3.1.7rc1; extra == "dev"
+Requires-Dist: types-PyYAML; extra == "dev"
+Requires-Dist: mypy==1.17.1; extra == "dev"
+Requires-Dist: pytest-cov>=2.8.1; extra == "dev"
 Requires-Dist: types-requests; extra == "dev"
+Requires-Dist: twine; extra == "dev"
+Requires-Dist: acryl-datahub[datahub-rest,sql-parser]==1.3.1.8rc1; extra == "dev"
+Requires-Dist: sqlalchemy-stubs; extra == "dev"
+Requires-Dist: coverage>=5.1; extra == "dev"
+Requires-Dist: ruff==0.11.7; extra == "dev"
 Requires-Dist: types-python-dateutil; extra == "dev"
-Requires-Dist: types-setuptools; extra == "dev"
-Requires-Dist: types-click==0.1.12; extra == "dev"
-Requires-Dist: pytest-cov>=2.8.1; extra == "dev"
+Requires-Dist: pytest>=6.2.2; extra == "dev"
 Provides-Extra: integration-tests
-Requires-Dist: acryl-datahub[sync-file-emitter]==1.3.1.7rc1; extra == "integration-tests"
-Requires-Dist: acryl-datahub[testing-utils]==1.3.1.7rc1; extra == "integration-tests"
-Requires-Dist: apache-airflow[amazon,google,snowflake]>=2.0.2; extra == "integration-tests"
-Requires-Dist: snowflake-connector-python>=2.7.10; extra == "integration-tests"
-Requires-Dist: apache-airflow-providers-sqlite; extra == "integration-tests"
+Requires-Dist: acryl-datahub[datahub-kafka]==1.3.1.8rc1; extra == "integration-tests"
 Requires-Dist: apache-airflow-providers-teradata; extra == "integration-tests"
+Requires-Dist: acryl-datahub[sync-file-emitter]==1.3.1.8rc1; extra == "integration-tests"
+Requires-Dist: snowflake-connector-python>=2.7.10; extra == "integration-tests"
+Requires-Dist: acryl-datahub[testing-utils]==1.3.1.8rc1; extra == "integration-tests"
+Requires-Dist: apache-airflow[amazon,google,snowflake]>=2.0.2; extra == "integration-tests"
 Requires-Dist: virtualenv; extra == "integration-tests"
-Requires-Dist: acryl-datahub[datahub-kafka]==1.3.1.7rc1; extra == "integration-tests"
+Requires-Dist: apache-airflow-providers-sqlite; extra == "integration-tests"
 Dynamic: classifier
 Dynamic: description
 Dynamic: description-content-type

{acryl_datahub_airflow_plugin-1.3.1.7rc1 → acryl_datahub_airflow_plugin-1.3.1.8rc1}/src/acryl_datahub_airflow_plugin.egg-info/SOURCES.txt

@@ -10,6 +10,7 @@ src/acryl_datahub_airflow_plugin.egg-info/not-zip-safe
 src/acryl_datahub_airflow_plugin.egg-info/requires.txt
 src/acryl_datahub_airflow_plugin.egg-info/top_level.txt
 src/datahub_airflow_plugin/__init__.py
+src/datahub_airflow_plugin/_airflow_asset_adapter.py
 src/datahub_airflow_plugin/_airflow_compat.py
 src/datahub_airflow_plugin/_airflow_shims.py
 src/datahub_airflow_plugin/_airflow_version_specific.py

{acryl_datahub_airflow_plugin-1.3.1.7rc1 → acryl_datahub_airflow_plugin-1.3.1.8rc1}/src/acryl_datahub_airflow_plugin.egg-info/requires.txt

@@ -1,7 +1,7 @@
-pydantic>=2.4.0
-acryl-datahub[datahub-rest,sql-parser]==1.3.1.7rc1
+acryl-datahub[datahub-rest,sql-parser]==1.3.1.8rc1
 apache-airflow<4.0.0,>=2.5.0
-acryl-datahub[datahub-rest]==1.3.1.7rc1
+acryl-datahub[datahub-rest]==1.3.1.8rc1
+pydantic>=2.4.0
 
 [airflow2]
 openlineage-airflow>=1.2.0
@@ -10,51 +10,51 @@ openlineage-airflow>=1.2.0
 apache-airflow-providers-openlineage>=1.0.0
 
 [datahub-file]
-acryl-datahub[sync-file-emitter]==1.3.1.7rc1
+acryl-datahub[sync-file-emitter]==1.3.1.8rc1
 
 [datahub-kafka]
-acryl-datahub[datahub-kafka]==1.3.1.7rc1
+acryl-datahub[datahub-kafka]==1.3.1.8rc1
 
 [datahub-rest]
-acryl-datahub[datahub-rest]==1.3.1.7rc1
+acryl-datahub[datahub-rest]==1.3.1.8rc1
 
 [dev]
+tox
 pydantic>=2.4.0
+packaging
 apache-airflow<4.0.0,>=2.5.0
-ruff==0.11.7
-tox
+build
+types-tabulate
+types-click==0.1.12
+types-cachetools
 types-six
-types-PyYAML
-coverage>=5.1
+tenacity
 deepdiff!=8.0.0
-packaging
+acryl-datahub[datahub-rest]==1.3.1.8rc1
 tox-uv
-acryl-datahub[datahub-rest]==1.3.1.7rc1
-types-cachetools
-mypy==1.17.1
-twine
-pytest>=6.2.2
-build
 types-dataclasses
-tenacity
+types-setuptools
 types-toml
-sqlalchemy-stubs
-types-tabulate
-acryl-datahub[datahub-rest,sql-parser]==1.3.1.7rc1
+types-PyYAML
+mypy==1.17.1
+pytest-cov>=2.8.1
 types-requests
+twine
+acryl-datahub[datahub-rest,sql-parser]==1.3.1.8rc1
+sqlalchemy-stubs
+coverage>=5.1
+ruff==0.11.7
 types-python-dateutil
-types-setuptools
-types-click==0.1.12
-pytest-cov>=2.8.1
+pytest>=6.2.2
 
 [ignore]
 
 [integration-tests]
-acryl-datahub[sync-file-emitter]==1.3.1.7rc1
-acryl-datahub[testing-utils]==1.3.1.7rc1
-apache-airflow[amazon,google,snowflake]>=2.0.2
-snowflake-connector-python>=2.7.10
-apache-airflow-providers-sqlite
+acryl-datahub[datahub-kafka]==1.3.1.8rc1
 apache-airflow-providers-teradata
+acryl-datahub[sync-file-emitter]==1.3.1.8rc1
+snowflake-connector-python>=2.7.10
+acryl-datahub[testing-utils]==1.3.1.8rc1
+apache-airflow[amazon,google,snowflake]>=2.0.2
 virtualenv
-acryl-datahub[datahub-kafka]==1.3.1.7rc1
+apache-airflow-providers-sqlite

acryl_datahub_airflow_plugin-1.3.1.8rc1/src/datahub_airflow_plugin/_airflow_asset_adapter.py

@@ -0,0 +1,164 @@
+"""Adapter for converting Airflow Assets/Datasets to DataHub URNs.
+
+Airflow 2.4+ introduced airflow.datasets.Dataset for data-aware scheduling.
+Airflow 3.0+ renamed this to airflow.sdk.Asset (Dataset still works as alias).
+
+This module provides utilities to detect and convert these native Airflow
+Asset/Dataset objects to DataHub dataset URNs based on their URI.
+"""
+
+import logging
+from typing import Any, Iterable, List, Optional
+from urllib.parse import urlparse
+
+import datahub.emitter.mce_builder as builder
+from datahub_airflow_plugin.entities import _Entity
+
+logger = logging.getLogger(__name__)
+
+# URI scheme to DataHub platform mapping
+URI_SCHEME_TO_PLATFORM = {
+    "s3": "s3",
+    "s3a": "s3",
+    "gs": "gcs",
+    "gcs": "gcs",
+    "file": "file",
+    "hdfs": "hdfs",
+    "abfs": "adls",
+    "abfss": "adls",
+    "postgresql": "postgres",
+    "mysql": "mysql",
+    "bigquery": "bigquery",
+    "snowflake": "snowflake",
+}
+
+
+def is_airflow_asset(obj: Any) -> bool:
+    """Check if object is an Airflow Asset or Dataset.
+
+    Works with both Airflow 2.x Dataset and Airflow 3.x Asset classes,
+    including subclasses, by checking the class hierarchy (MRO) and
+    required 'uri' attribute.
+    """
+    if not hasattr(obj, "uri"):
+        return False
+
+    # Check class name and all base classes in the MRO
+    for cls in type(obj).__mro__:
+        if cls.__name__ in ("Asset", "Dataset"):
+            return True
+
+    return False
+
+
+def translate_airflow_asset_to_urn(
+    asset: Any, env: str = "PROD", platform_fallback: str = "airflow"
+) -> Optional[str]:
+    """Convert Airflow Asset URI to DataHub dataset URN.
+
+    Args:
+        asset: An Airflow Asset or Dataset object with a 'uri' attribute.
+        env: The DataHub environment (default: "PROD").
+        platform_fallback: Platform to use when URI has no scheme (default: "airflow").
+            This is used for @asset decorated functions that don't specify a URI.
+
+    Returns:
+        A DataHub dataset URN string, or None if the asset URI is invalid.
+
+    Examples:
+        >>> translate_airflow_asset_to_urn(mock_asset("s3://bucket/path"))
+        "urn:li:dataset:(urn:li:dataPlatform:s3,bucket/path,PROD)"
+
+        >>> translate_airflow_asset_to_urn(mock_asset("postgresql://host/db/table"))
+        "urn:li:dataset:(urn:li:dataPlatform:postgres,host/db/table,PROD)"
+
+        >>> translate_airflow_asset_to_urn(mock_asset("my_asset"))  # @asset decorator
+        "urn:li:dataset:(urn:li:dataPlatform:airflow,my_asset,PROD)"
+    """
+    uri = getattr(asset, "uri", None)
+    if not uri:
+        return None
+
+    try:
+        parsed = urlparse(uri)
+    except Exception as e:
+        logger.warning(
+            f"Failed to parse Airflow asset URI '{uri}': {e}. "
+            f"This asset will be excluded from lineage."
+        )
+        return None
+
+    scheme = parsed.scheme.lower() if parsed.scheme else ""
+
+    if scheme:
+        # URI has a scheme - map to DataHub platform
+        platform = URI_SCHEME_TO_PLATFORM.get(scheme, scheme)
+        # Build dataset name from URI components
+        if parsed.netloc:
+            name = f"{parsed.netloc}{parsed.path}".lstrip("/")
+        else:
+            name = parsed.path.lstrip("/")
+    else:
+        # No scheme - this is likely an @asset decorated function with just a name
+        # Use the fallback platform (default: "airflow") and the URI as the name
+        platform = platform_fallback
+        name = uri
+
+    # Ensure we have a valid name
+    if not name:
+        logger.debug(f"Airflow asset URI has empty name: '{uri}'. Skipping.")
+        return None
+
+    try:
+        return builder.make_dataset_urn(platform=platform, name=name, env=env)
+    except Exception as e:
+        logger.warning(
+            f"Failed to create DataHub URN for Airflow asset: {e}. "
+            f"URI: '{uri}', platform: '{platform}', name: '{name}'. "
+            f"This asset will be excluded from lineage."
+        )
+        return None
+
+
+def extract_urns_from_iolets(
+    iolets: Iterable[Any],
+    capture_airflow_assets: bool,
+    env: str = "PROD",
+) -> List[str]:
+    """Extract URNs from a list of inlets or outlets.
+
+    Processes both DataHub entity objects and native Airflow Assets/Datasets.
+
+    Args:
+        iolets: Iterable of inlet/outlet objects (from task.inlets or task.outlets).
+        capture_airflow_assets: Whether to capture native Airflow Assets.
+        env: The DataHub environment (default: "PROD").
+
+    Returns:
+        List of URN strings extracted from the iolets.
+    """
+    urns: List[str] = []
+    for iolet in iolets:
+        try:
+            if isinstance(iolet, _Entity):
+                urns.append(iolet.urn)
+            elif capture_airflow_assets and is_airflow_asset(iolet):
+                urn = translate_airflow_asset_to_urn(iolet, env=env)
+                if urn:
+                    urns.append(urn)
+                else:
+                    # translate_airflow_asset_to_urn already logs details
+                    uri = getattr(iolet, "uri", None)
+                    logger.warning(
+                        f"Skipping Airflow asset with URI '{uri}' - "
+                        f"could not convert to DataHub URN. "
+                        f"Check that the URI is valid and has a proper scheme."
+                    )
+        except Exception as e:
+            # Catch any unexpected errors to avoid breaking lineage extraction
+            logger.warning(
+                f"Failed to extract URN from iolet {type(iolet).__name__}: {e}. "
+                f"Continuing with remaining iolets."
+            )
+            continue
+    return urns

{acryl_datahub_airflow_plugin-1.3.1.7rc1 → acryl_datahub_airflow_plugin-1.3.1.8rc1}/src/datahub_airflow_plugin/_config.py

@@ -46,6 +46,11 @@ class DatahubLineageConfig(ConfigModel):
     # referenced by inlets or outlets.
     materialize_iolets: bool
 
+    # If true (default), capture native Airflow Assets/Datasets as DataHub lineage.
+    # Airflow 2.4+ Dataset and Airflow 3.x Asset objects in inlets/outlets
+    # will be converted to DataHub dataset URNs.
+    capture_airflow_assets: bool
+
     capture_executions: bool
 
     datajob_url_link: DatajobUrl
@@ -135,6 +140,9 @@ def get_lineage_config() -> DatahubLineageConfig:
     )
     capture_executions = conf.get("datahub", "capture_executions", fallback=True)
     materialize_iolets = conf.get("datahub", "materialize_iolets", fallback=True)
+    capture_airflow_assets = conf.get(
+        "datahub", "capture_airflow_assets", fallback=True
+    )
     enable_extractors = conf.get("datahub", "enable_extractors", fallback=True)
 
     # OpenLineage extractor patching/override configuration
@@ -204,6 +212,7 @@ def get_lineage_config() -> DatahubLineageConfig:
         capture_tags_info=capture_tags_info,
         capture_executions=capture_executions,
         materialize_iolets=materialize_iolets,
+        capture_airflow_assets=capture_airflow_assets,
         enable_extractors=enable_extractors,
         patch_sql_parser=patch_sql_parser,
         patch_snowflake_schema=patch_snowflake_schema,

{acryl_datahub_airflow_plugin-1.3.1.7rc1 → acryl_datahub_airflow_plugin-1.3.1.8rc1}/src/datahub_airflow_plugin/_version.py

@@ -1,3 +1,3 @@
 # Published at https://pypi.org/project/acryl-datahub-airflow-plugin/.
 __package_name__ = "acryl-datahub-airflow-plugin"
-__version__ = "1.3.1.7rc1"
+__version__ = "1.3.1.8rc1"

{acryl_datahub_airflow_plugin-1.3.1.7rc1 → acryl_datahub_airflow_plugin-1.3.1.8rc1}/src/datahub_airflow_plugin/airflow2/datahub_listener.py

@@ -22,7 +22,9 @@ import airflow
 # Import Airflow 2.x specific shims (clean, no cross-version complexity)
 import airflow.version
 import packaging.version
-from airflow.models import Variable
+
+# Note: We intentionally don't import Variable here to avoid DB access in listeners.
+# See check_kill_switch() for details on why we use os.getenv() instead.
 from airflow.models.serialized_dag import SerializedDagModel
 
 # Import Airflow 2.x compatibility and patches before any Airflow imports
@@ -74,6 +76,7 @@ from datahub.metadata.schema_classes import (
 )
 from datahub.sql_parsing.sqlglot_lineage import SqlParsingResult
 from datahub.telemetry import telemetry
+from datahub_airflow_plugin._airflow_asset_adapter import extract_urns_from_iolets
 from datahub_airflow_plugin._config import DatahubLineageConfig, get_lineage_config
 from datahub_airflow_plugin._constants import (
     DATAHUB_SQL_PARSING_RESULT_KEY,
@@ -94,7 +97,6 @@ from datahub_airflow_plugin.client.airflow_generator import (  # type: ignore[at
     AirflowGenerator,
 )
 from datahub_airflow_plugin.entities import (
-    _Entity,
     entities_to_datajob_urn_list,
     entities_to_dataset_urn_list,
 )
@@ -498,12 +500,20 @@ class DataHubListener:
         output_urns.extend(sql_output_urns)
         fine_grained_lineages.extend(sql_fine_grained_lineages)
 
-        # Add DataHub-native inlets/outlets
+        # Add DataHub-native inlets/outlets and Airflow Assets
         input_urns.extend(
-            iolet.urn for iolet in get_task_inlets(task) if isinstance(iolet, _Entity)
+            extract_urns_from_iolets(
+                get_task_inlets(task),
+                self.config.capture_airflow_assets,
+                env=self.config.cluster,
+            )
         )
         output_urns.extend(
-            iolet.urn for iolet in get_task_outlets(task) if isinstance(iolet, _Entity)
+            extract_urns_from_iolets(
+                get_task_outlets(task),
+                self.config.capture_airflow_assets,
+                env=self.config.cluster,
+            )
         )
 
         # Write the lineage to the datajob object
@@ -568,13 +578,29 @@ class DataHubListener:
                 )
 
     def check_kill_switch(self) -> bool:
-        # For Airflow 2.x, use Variable.get()
-        try:
-            if Variable.get(KILL_SWITCH_VARIABLE_NAME, "false").lower() == "true":
-                logger.debug("DataHub listener disabled by kill switch")
-                return True
-        except Exception as e:
-            logger.debug(f"Error checking kill switch variable: {e}")
+        """
+        Check kill switch using environment variable.
+
+        We use os.getenv() instead of Variable.get() because Variable.get()
+        creates a new database session and commits it. When called from listener
+        hooks (which execute during SQLAlchemy's after_flush event, before the
+        main transaction commits), this nested commit can corrupt the outer
+        transaction state and cause data loss.
+
+        Specifically, this was causing TaskInstanceHistory records to not be
+        persisted for retried tasks. See: https://github.com/apache/airflow/pull/48780
+
+        Users can set the kill switch via environment variable:
+            export AIRFLOW_VAR_DATAHUB_AIRFLOW_PLUGIN_DISABLE_LISTENER=true
+        """
+        if (
+            os.getenv(
+                f"AIRFLOW_VAR_{KILL_SWITCH_VARIABLE_NAME}".upper(), "false"
+            ).lower()
+            == "true"
+        ):
+            logger.debug("DataHub listener disabled by kill switch (env var)")
+            return True
         return False
 
     def _prepare_task_context(

{acryl_datahub_airflow_plugin-1.3.1.7rc1 → acryl_datahub_airflow_plugin-1.3.1.8rc1}/src/datahub_airflow_plugin/airflow3/datahub_listener.py

@@ -52,6 +52,7 @@ from datahub.sql_parsing.sqlglot_lineage import SqlParsingResult
 from datahub.telemetry import telemetry
 
 # Import Airflow 3.x specific shims (clean, no cross-version complexity)
+from datahub_airflow_plugin._airflow_asset_adapter import extract_urns_from_iolets
 from datahub_airflow_plugin._config import DatahubLineageConfig, get_lineage_config
 from datahub_airflow_plugin._constants import DATAHUB_SQL_PARSING_RESULT_KEY
 from datahub_airflow_plugin._version import __package_name__, __version__
@@ -69,7 +70,6 @@ from datahub_airflow_plugin.client.airflow_generator import (  # type: ignore[at
     AirflowGenerator,
 )
 from datahub_airflow_plugin.entities import (
-    _Entity,
     entities_to_datajob_urn_list,
     entities_to_dataset_urn_list,
 )
@@ -830,12 +830,20 @@ class DataHubListener:
 
         fine_grained_lineages.extend(sql_fine_grained_lineages)
 
-        # Add DataHub-native inlets/outlets
+        # Add DataHub-native inlets/outlets and Airflow Assets
         input_urns.extend(
-            iolet.urn for iolet in get_task_inlets(task) if isinstance(iolet, _Entity)
+            extract_urns_from_iolets(
+                get_task_inlets(task),
+                self.config.capture_airflow_assets,
+                env=self.config.cluster,
+            )
         )
         output_urns.extend(
-            iolet.urn for iolet in get_task_outlets(task) if isinstance(iolet, _Entity)
+            extract_urns_from_iolets(
+                get_task_outlets(task),
+                self.config.capture_airflow_assets,
+                env=self.config.cluster,
+            )
         )
 
         # Write the lineage to the datajob object

{acryl_datahub_airflow_plugin-1.3.1.7rc1 → acryl_datahub_airflow_plugin-1.3.1.8rc1}/src/datahub_airflow_plugin/client/airflow_generator.py

@@ -75,6 +75,35 @@ def _task_downstream_task_ids(operator: "Operator") -> Set[str]:
     return operator._downstream_task_id  # type: ignore[attr-defined,union-attr]
 
 
+def _serialize_iolets_for_properties(iolets: List[Any]) -> List[str]:
+    """Serialize inlets/outlets to a list of URIs for stable custom properties.
+
+    This function extracts the URI from various inlet/outlet object types:
+    - Airflow 2.4+ Dataset objects (have .uri attribute)
+    - Airflow 3.x Asset objects (have .uri attribute)
+    - Airflow 3.x AssetDefinition objects from @asset decorator (have .uri attribute)
+    - DataHub Dataset entities (have __repr__ that returns URN)
+    - Strings (used as-is)
+
+    This avoids using repr() on complex objects which would include memory addresses
+    and other unstable information.
+    """
+    result = []
+    for iolet in iolets:
+        if hasattr(iolet, "uri"):
+            # Airflow Dataset/Asset/AssetDefinition objects
+            result.append(str(iolet.uri))
+        elif hasattr(iolet, "urn"):
+            # DataHub entities with URN
+            result.append(str(iolet.urn))
+        elif isinstance(iolet, str):
+            result.append(iolet)
+        else:
+            # Fallback to string representation for unknown types
+            result.append(str(iolet))
+    return result
+
+
 class AirflowGenerator:
     @staticmethod
     def _get_dependencies(
@@ -362,6 +391,10 @@ class AirflowGenerator:
                     if out_key == "downstream_task_ids":
                         # Generate these in a consistent order.
                         v = list(sorted(v))
+                    if out_key in ("inlets", "outlets"):
+                        # Serialize inlets/outlets as list of URIs for stable representation.
+                        # This avoids including memory addresses from repr() of complex objects.
+                        v = _serialize_iolets_for_properties(v)
                     job_property_bag[out_key] = repr(v)
                     break