acryl-datahub-airflow-plugin 1.3.1.5rc6__py3-none-any.whl → 1.3.1.5rc9__py3-none-any.whl

acryl_datahub_airflow_plugin-1.3.1.5rc9.dist-info/METADATA

@@ -0,0 +1,303 @@
+Metadata-Version: 2.4
+Name: acryl-datahub-airflow-plugin
+Version: 1.3.1.5rc9
+Summary: Datahub Airflow plugin to capture executions and send to Datahub
+Home-page: https://docs.datahub.com/
+License: Apache-2.0
+Project-URL: Documentation, https://docs.datahub.com/docs/
+Project-URL: Source, https://github.com/datahub-project/datahub
+Project-URL: Changelog, https://github.com/datahub-project/datahub/releases
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Information Technology
+Classifier: Intended Audience :: System Administrators
+Classifier: Operating System :: Unix
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Environment :: Console
+Classifier: Environment :: MacOS X
+Classifier: Topic :: Software Development
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: acryl-datahub[datahub-rest]==1.3.1.5rc9
+Requires-Dist: pydantic>=2.4.0
+Requires-Dist: acryl-datahub[datahub-rest,sql-parser]==1.3.1.5rc9
+Requires-Dist: apache-airflow<4.0.0,>=2.5.0
+Provides-Extra: ignore
+Provides-Extra: datahub-rest
+Requires-Dist: acryl-datahub[datahub-rest]==1.3.1.5rc9; extra == "datahub-rest"
+Provides-Extra: datahub-kafka
+Requires-Dist: acryl-datahub[datahub-kafka]==1.3.1.5rc9; extra == "datahub-kafka"
+Provides-Extra: datahub-file
+Requires-Dist: acryl-datahub[sync-file-emitter]==1.3.1.5rc9; extra == "datahub-file"
+Provides-Extra: airflow2
+Requires-Dist: openlineage-airflow>=1.2.0; extra == "airflow2"
+Provides-Extra: airflow3
+Requires-Dist: apache-airflow-providers-openlineage>=1.0.0; extra == "airflow3"
+Provides-Extra: dev
+Requires-Dist: coverage>=5.1; extra == "dev"
+Requires-Dist: tox; extra == "dev"
+Requires-Dist: apache-airflow<4.0.0,>=2.5.0; extra == "dev"
+Requires-Dist: sqlalchemy-stubs; extra == "dev"
+Requires-Dist: types-tabulate; extra == "dev"
+Requires-Dist: acryl-datahub[datahub-rest]==1.3.1.5rc9; extra == "dev"
+Requires-Dist: types-setuptools; extra == "dev"
+Requires-Dist: packaging; extra == "dev"
+Requires-Dist: tox-uv; extra == "dev"
+Requires-Dist: acryl-datahub[datahub-rest,sql-parser]==1.3.1.5rc9; extra == "dev"
+Requires-Dist: ruff==0.11.7; extra == "dev"
+Requires-Dist: types-six; extra == "dev"
+Requires-Dist: tenacity; extra == "dev"
+Requires-Dist: twine; extra == "dev"
+Requires-Dist: pydantic>=2.4.0; extra == "dev"
+Requires-Dist: types-dataclasses; extra == "dev"
+Requires-Dist: pytest>=6.2.2; extra == "dev"
+Requires-Dist: types-python-dateutil; extra == "dev"
+Requires-Dist: types-toml; extra == "dev"
+Requires-Dist: mypy==1.17.1; extra == "dev"
+Requires-Dist: types-requests; extra == "dev"
+Requires-Dist: types-PyYAML; extra == "dev"
+Requires-Dist: types-cachetools; extra == "dev"
+Requires-Dist: deepdiff!=8.0.0; extra == "dev"
+Requires-Dist: types-click==0.1.12; extra == "dev"
+Requires-Dist: pytest-cov>=2.8.1; extra == "dev"
+Requires-Dist: build; extra == "dev"
+Provides-Extra: integration-tests
+Requires-Dist: acryl-datahub[datahub-kafka]==1.3.1.5rc9; extra == "integration-tests"
+Requires-Dist: acryl-datahub[testing-utils]==1.3.1.5rc9; extra == "integration-tests"
+Requires-Dist: snowflake-connector-python>=2.7.10; extra == "integration-tests"
+Requires-Dist: apache-airflow-providers-teradata; extra == "integration-tests"
+Requires-Dist: acryl-datahub[sync-file-emitter]==1.3.1.5rc9; extra == "integration-tests"
+Requires-Dist: virtualenv; extra == "integration-tests"
+Requires-Dist: apache-airflow-providers-sqlite; extra == "integration-tests"
+Requires-Dist: apache-airflow[amazon,google,snowflake]>=2.0.2; extra == "integration-tests"
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license
+Dynamic: project-url
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# Datahub Airflow Plugin
+
+See [the DataHub Airflow docs](https://docs.datahub.com/docs/lineage/airflow) for details.
+
+## Version Compatibility
+
+The plugin supports Apache Airflow versions 2.7+ and 3.1+.
+
+| Airflow Version | Extra to Install | Status                 | Notes                            |
+| --------------- | ---------------- | ---------------------- | -------------------------------- |
+| 2.7-2.10        | `[airflow2]`     | ✅ Fully Supported     |                                  |
+| 3.0.x           | `[airflow3]`     | ⚠️ Requires manual fix | Needs `pydantic>=2.11.8` upgrade |
+| 3.1+            | `[airflow3]`     | ✅ Fully Supported     |                                  |
+
+**Note on Airflow 3.0.x**: Airflow 3.0.6 pins pydantic==2.11.7, which contains a bug that prevents the DataHub plugin from importing correctly. This issue is resolved in Airflow 3.1.0+ which uses pydantic>=2.11.8. If you must use Airflow 3.0.6, you can manually upgrade pydantic to >=2.11.8, though this may conflict with Airflow's dependency constraints. We recommend upgrading to Airflow 3.1.0 or later.
+
+Related issue: https://github.com/pydantic/pydantic/issues/10963
+
+## Installation
+
+The installation command varies depending on your Airflow version due to different OpenLineage dependencies.
+
+### For Airflow 2.x (2.7+)
+
+```bash
+pip install 'acryl-datahub-airflow-plugin[airflow2]'
+```
+
+This installs the plugin with Legacy OpenLineage (`openlineage-airflow>=1.2.0`), which is required for Airflow 2.x lineage extraction.
+
+#### Alternative: Using Native OpenLineage Provider on Airflow 2.7+
+
+If your Airflow 2.7+ environment rejects the Legacy OpenLineage package (e.g., due to dependency conflicts), you can use the native OpenLineage provider instead:
+
+```bash
+# Install the native Airflow provider first
+pip install 'apache-airflow-providers-openlineage>=1.0.0'
+
+# Then install the DataHub plugin without OpenLineage extras
+pip install acryl-datahub-airflow-plugin
+```
+
+The plugin will automatically detect and use `apache-airflow-providers-openlineage` when available, providing the same functionality.
+
+### For Airflow 3.x (3.1+)
+
+```bash
+pip install 'acryl-datahub-airflow-plugin[airflow3]'
+```
+
+This installs the plugin with `apache-airflow-providers-openlineage>=1.0.0`, which is the native OpenLineage provider for Airflow 3.x.
+
+**Note**: If using Airflow 3.0.x (3.0.6 specifically), you'll need to manually upgrade pydantic:
+
+```bash
+pip install 'acryl-datahub-airflow-plugin[airflow3]' 'pydantic>=2.11.8'
+```
+
+We recommend using Airflow 3.1.0+ which resolves this issue. See the Version Compatibility section above for details.
+
+### What Gets Installed
+
+#### Base Installation (No Extras)
+
+When you install without any extras:
+
+```bash
+pip install acryl-datahub-airflow-plugin
+```
+
+You get:
+
+- `acryl-datahub[sql-parser,datahub-rest]` - DataHub SDK with SQL parsing and REST emitter
+- `pydantic>=2.4.0` - Required for data validation
+- `apache-airflow>=2.5.0,<4.0.0` - Airflow itself
+- **No OpenLineage package** - You'll need to provide your own or use one of the extras below
+
+#### With `[airflow2]` Extra
+
+```bash
+pip install 'acryl-datahub-airflow-plugin[airflow2]'
+```
+
+Adds:
+
+- `openlineage-airflow>=1.2.0` - Standalone OpenLineage package for Airflow 2.x
+
+#### With `[airflow3]` Extra
+
+```bash
+pip install 'acryl-datahub-airflow-plugin[airflow3]'
+```
+
+Adds:
+
+- `apache-airflow-providers-openlineage>=1.0.0` - Native OpenLineage provider for Airflow 3.x
+
+### Additional Extras
+
+You can combine multiple extras if needed:
+
+```bash
+# For Airflow 3.x with Kafka emitter support
+pip install 'acryl-datahub-airflow-plugin[airflow3,datahub-kafka]'
+
+# For Airflow 2.x with file emitter support
+pip install 'acryl-datahub-airflow-plugin[airflow2,datahub-file]'
+```
+
+Available extras:
+
+- `airflow2`: OpenLineage support for Airflow 2.x (adds `openlineage-airflow>=1.2.0`)
+- `airflow3`: OpenLineage support for Airflow 3.x (adds `apache-airflow-providers-openlineage>=1.0.0`)
+- `datahub-kafka`: Kafka-based metadata emission (adds `acryl-datahub[datahub-kafka]`)
+- `datahub-file`: File-based metadata emission (adds `acryl-datahub[sync-file-emitter]`) - useful for testing
+
+### Why Different Extras?
+
+Airflow 2.x and 3.x have different OpenLineage integrations:
+
+- **Airflow 2.x (2.5-2.6)** typically uses Legacy OpenLineage (`openlineage-airflow` package)
+- **Airflow 2.x (2.7+)** can use either Legacy OpenLineage or native OpenLineage Provider (`apache-airflow-providers-openlineage`)
+- **Airflow 3.x** uses native OpenLineage Provider (`apache-airflow-providers-openlineage`)
+
+The plugin automatically detects which OpenLineage variant is installed and uses it accordingly. This means:
+
+1. **With extras** (`[airflow2]` or `[airflow3]`): The appropriate OpenLineage dependency is installed automatically
+2. **Without extras**: You provide your own OpenLineage installation, and the plugin auto-detects it
+
+This flexibility allows you to adapt to different Airflow environments and dependency constraints.
+
+## Configuration
+
+The plugin can be configured via `airflow.cfg` under the `[datahub]` section. Below are the key configuration options:
+
+### Extractor Patching (OpenLineage Enhancements)
+
+When `enable_extractors=True` (default), the DataHub plugin enhances OpenLineage extractors to provide better lineage. You can fine-tune these enhancements:
+
+```ini
+[datahub]
+# Enable/disable all OpenLineage extractors
+enable_extractors = True  # Default: True
+
+# Fine-grained control over DataHub's OpenLineage enhancements
+
+# --- Patches (work with both Legacy OpenLineage and OpenLineage Provider) ---
+
+# Patch SqlExtractor to use DataHub's advanced SQL parser (enables column-level lineage)
+patch_sql_parser = True  # Default: True
+
+# Patch SnowflakeExtractor to fix default schema detection
+patch_snowflake_schema = True  # Default: True
+
+# --- Custom Extractors (only apply to Legacy OpenLineage) ---
+
+# Use DataHub's custom AthenaOperatorExtractor (better Athena lineage)
+extract_athena_operator = True  # Default: True
+
+# Use DataHub's custom BigQueryInsertJobOperatorExtractor (handles BQ job configuration)
+extract_bigquery_insert_job_operator = True  # Default: True
+```
+
+**How it works:**
+
+**Patches** (apply to both Legacy OpenLineage and OpenLineage Provider):
+
+- Apply **monkey-patching** to OpenLineage extractor/operator classes at runtime
+- Work on **both Airflow 2.x and Airflow 3.x**
+- When `patch_sql_parser=True`:
+  - **Airflow 2**: Patches `SqlExtractor.extract()` method
+  - **Airflow 3**: Patches `SQLParser.generate_openlineage_metadata_from_sql()` method
+  - Provides: More accurate lineage extraction, column-level lineage (CLL), better SQL dialect support
+- When `patch_snowflake_schema=True`:
+  - **Airflow 2**: Patches `SnowflakeExtractor.default_schema` property
+  - **Airflow 3**: Currently not needed (handled by Airflow's native support)
+  - Fixes Snowflake schema detection issues
+
+**Custom Extractors/Operator Patches**:
+
+- Register DataHub's custom implementations for specific operators
+- Work on **both Airflow 2.x and Airflow 3.x**
+- `extract_athena_operator`:
+  - **Airflow 2 (Legacy OpenLineage only)**: Registers `AthenaOperatorExtractor`
+  - **Airflow 3**: Patches `AthenaOperator.get_openlineage_facets_on_complete()`
+  - Uses DataHub's SQL parser for better Athena lineage
+- `extract_bigquery_insert_job_operator`:
+  - **Airflow 2 (Legacy OpenLineage only)**: Registers `BigQueryInsertJobOperatorExtractor`
+  - **Airflow 3**: Patches `BigQueryInsertJobOperator.get_openlineage_facets_on_complete()`
+  - Handles BigQuery job configuration and destination tables
+
+**Example use cases:**
+
+Disable DataHub's SQL parser to use OpenLineage's native parsing:
+
+```ini
+[datahub]
+enable_extractors = True
+patch_sql_parser = False  # Use OpenLineage's native SQL parser
+patch_snowflake_schema = True  # Still fix Snowflake schema detection
+```
+
+Disable custom Athena extractor (only relevant for Legacy OpenLineage):
+
+```ini
+[datahub]
+enable_extractors = True
+extract_athena_operator = False  # Use OpenLineage's default Athena extractor
+```
+
+### Other Configuration Options
+
+For a complete list of configuration options, see the [DataHub Airflow documentation](https://docs.datahub.com/docs/lineage/airflow#configuration).
+
+## Developing
+
+See the [developing docs](../../metadata-ingestion/developing.md).

acryl_datahub_airflow_plugin-1.3.1.5rc9.dist-info/RECORD

@@ -0,0 +1,65 @@
+datahub_airflow_plugin/__init__.py,sha256=NScUtA8N-m66Pyg0DO--YbPkrl48PK3UevpdQVW_y6E,1009
+datahub_airflow_plugin/_airflow_compat.py,sha256=VdlTeQ7XGU40uqeAfnJcDGLt8QD2lLpEhQw1ZEMTFvI,1345
+datahub_airflow_plugin/_airflow_shims.py,sha256=fe-BwJxdA56P6j_nD1uMwmNNCE0FnlbHsqQME30uQ_o,2243
+datahub_airflow_plugin/_airflow_version_specific.py,sha256=q4iNyScNQTNZnoC9n_8wYI99mhqiXbxUhjAv2x_sNks,5870
+datahub_airflow_plugin/_config.py,sha256=AAvxIGcG-wQWkFbpGCRurjnNXVCzqHDBmsFsSRHOEi4,9050
+datahub_airflow_plugin/_constants.py,sha256=HBrUztUwspK1mQ3iM9_Pdiu18pn-lPK3xhjmubqn_kI,627
+datahub_airflow_plugin/_datahub_ol_adapter.py,sha256=2YIhJDyLhzZGK3MMra4NGSIfG-az8sKt6ZXcbQGs2Yg,951
+datahub_airflow_plugin/_version.py,sha256=pYQwM4cdD49xNeRp2fSnBRQ33_fIW_4kuPQE_s4bJtQ,148
+datahub_airflow_plugin/datahub_listener.py,sha256=9g-MBf14zFs-cQP0Jr_9gnCbzQVqIe_x6ZjgQs0onzE,881
+datahub_airflow_plugin/datahub_plugin.py,sha256=rbZhs7s5O3_MlkQw5aZToC2W5mMic_EpI3oybHB0ofw,1224
+datahub_airflow_plugin/entities.py,sha256=xDZ-mZH7hjUkZbatWYUwI43_9B40wGiotlyQhiO8rEM,1987
+datahub_airflow_plugin/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datahub_airflow_plugin/airflow2/__init__.py,sha256=WdCqiDRuXMkPcLPb3D5PQYZJaRyDtuEIJ7ndHSThjkg,200
+datahub_airflow_plugin/airflow2/_airflow2_sql_parser_patch.py,sha256=nkt57jYixcocrfYpQ3bhAUDTeyblt8p99_QdNzqJl_o,16145
+datahub_airflow_plugin/airflow2/_airflow_compat.py,sha256=7GumOc-bOCySp739MEJmksgHsiElSNfm29hB3kHyJPY,4005
+datahub_airflow_plugin/airflow2/_extractors.py,sha256=S4LbR9_xDMl76YErWgS_FsPzFW1DUDfsOZ0A9tKrZJ4,17981
+datahub_airflow_plugin/airflow2/_legacy_shims.py,sha256=rToXtWz0p41_jIsjPk4ALVL0T8NcH_RXRNuW47uDHDU,527
+datahub_airflow_plugin/airflow2/_openlineage_compat.py,sha256=43Xgtl2LdEVsIxJUb-DFGYoP-kaEots599rTgNdCMD4,4517
+datahub_airflow_plugin/airflow2/_provider_shims.py,sha256=GD_r_RojjBWWFfxvakQWr4eTgPplfkRCMZMRDTaApM4,961
+datahub_airflow_plugin/airflow2/_shims.py,sha256=Ha4gLxDU_GVPj2T0dHTwru4Lm-kYb9rRsL7uhiba0Gs,2769
+datahub_airflow_plugin/airflow2/datahub_listener.py,sha256=IdvTYLY5TWbukJ44HmJI0GJnPHqu60wR8RQt10I3wss,42611
+datahub_airflow_plugin/airflow3/__init__.py,sha256=OIFuu2zwlzBMcB2FVlkM2aXsCVPW6Xoa82pEjPZK6g0,200
+datahub_airflow_plugin/airflow3/_airflow3_sql_parser_patch.py,sha256=7LZjG8I0Vl8sI4j7RZsoZt7IkG9RQ-tL8CxZcYIVqQE,16460
+datahub_airflow_plugin/airflow3/_airflow_compat.py,sha256=g5q6XfOZZQhhJwNHwYb0U_wML7_2K0BsGGnGzxEk22M,3857
+datahub_airflow_plugin/airflow3/_athena_openlineage_patch.py,sha256=BoyCFAZM7lB1lJqqu2XuLF_wgSMC4oUwRlu8e1E8ntU,6709
+datahub_airflow_plugin/airflow3/_bigquery_openlineage_patch.py,sha256=ph9ZePU8eff1ew-j5dvM-vBmp-Or9b1CsnOevAqoluM,10228
+datahub_airflow_plugin/airflow3/_shims.py,sha256=q2Nw5vS21R_y0cYg3aOm7AYylaNAXlN480ve5XItfNU,2553
+datahub_airflow_plugin/airflow3/_sqlite_openlineage_patch.py,sha256=QRgadtwS6a42k3J4zAGsIRjyExvRX7CvAM_U3DYivic,3552
+datahub_airflow_plugin/airflow3/_teradata_openlineage_patch.py,sha256=z-Udbz_7NahufipLOIgUlZGWRsIRF_qkV903rYxR_s4,12563
+datahub_airflow_plugin/airflow3/datahub_listener.py,sha256=Xonjn3SX7q9dmDEyW7uyiryvJ1AVJuQcByBbD6TFWTk,58791
+datahub_airflow_plugin/client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datahub_airflow_plugin/client/airflow_generator.py,sha256=7Cc9khsePHqjRAHZchaMKMKP0wCeT21wCtH8gUZeEEE,26135
+datahub_airflow_plugin/example_dags/__init__.py,sha256=nIAjilNA_uGtFYYmAakZOvJQjAeg8d7aR61MegkEjgU,1321
+datahub_airflow_plugin/example_dags/generic_recipe_sample_dag.py,sha256=BbrOErFboKMDFn712RHEKI9T4Vh0q6kYSVet56gPqVk,1319
+datahub_airflow_plugin/example_dags/graph_usage_sample_dag.py,sha256=-rWLI58fErRoTpr38RCJXE0_lSBNZp17U38hg9Ow83A,1295
+datahub_airflow_plugin/example_dags/lineage_backend_demo.py,sha256=Dy6MxwtX7G0mQeALqpLRu4F03IyU9fqIkr-CcKpo2JE,1625
+datahub_airflow_plugin/example_dags/lineage_backend_taskflow_demo.py,sha256=kW2rLFtOnoiMxBJ315GzlmR0Sz1cqQ_wwLbG9UC-u7Y,1499
+datahub_airflow_plugin/example_dags/lineage_emission_dag.py,sha256=LE29DzW51a4ZAl_zrcLrqSyzmy8qElcZagXsIMjaZLU,1946
+datahub_airflow_plugin/example_dags/mysql_sample_dag.py,sha256=Unx9Ger3R9ptEutfV-4NjjEaTIEYJ-tLrZr7OsK608k,1922
+datahub_airflow_plugin/example_dags/snowflake_sample_dag.py,sha256=b9iaE7zChQha9u57F84U6uqavGl7WrUnMNOzXEiZxjE,3234
+datahub_airflow_plugin/example_dags/airflow2/__init__.py,sha256=cAFFodve_HVEokz3ZVo9lWvpYZ9WeLgZ4d8lSypHINQ,230
+datahub_airflow_plugin/example_dags/airflow2/generic_recipe_sample_dag.py,sha256=zmrNm1L9q7hAksSYTcCNEFopy4Qy0sElM30JQZfan1U,1493
+datahub_airflow_plugin/example_dags/airflow2/graph_usage_sample_dag.py,sha256=-rWLI58fErRoTpr38RCJXE0_lSBNZp17U38hg9Ow83A,1295
+datahub_airflow_plugin/example_dags/airflow2/lineage_backend_demo.py,sha256=1sUWzCU4tCGOfMNFvV6QbP416Ld-Sc78O-9fNFv9zKo,2331
+datahub_airflow_plugin/example_dags/airflow2/lineage_backend_taskflow_demo.py,sha256=7dB0874wOOVXZwiEcd7uHCK_ngtTW6JOHtFmhoqzaXU,2286
+datahub_airflow_plugin/example_dags/airflow2/lineage_emission_dag.py,sha256=iZ2rN787gdiYvAfMD_ST-ZUijqXtH4UAQzoXgogSG9o,2649
+datahub_airflow_plugin/example_dags/airflow2/mysql_sample_dag.py,sha256=qeB16im8mlQ2U3is55Lxo45dbUXVB29uS_ixSV74dPg,2132
+datahub_airflow_plugin/example_dags/airflow2/snowflake_sample_dag.py,sha256=H1PMTRBBZcjGOiIKkMMJ57ujRFYPoxlc1sq7gMl-ahI,3619
+datahub_airflow_plugin/example_dags/airflow3/__init__.py,sha256=XXQzH-43L_WnjfF-ahoC0jUBltS5X-nZyDPZSbZSNfg,233
+datahub_airflow_plugin/example_dags/airflow3/lineage_backend_demo.py,sha256=U-xqMZtA1DN_WkdP6aE044efu6M0VGtBqMghwmBuDds,1635
+datahub_airflow_plugin/example_dags/airflow3/lineage_backend_taskflow_demo.py,sha256=oTKaIAvoU212ORg8SxfhKtS9ne2oflSyPPEQcu3m--Q,1533
+datahub_airflow_plugin/example_dags/airflow3/snowflake_sample_dag.py,sha256=6G8tJFSX9vFCdWn6wN1nvgZthuc3poBSBEeqylvQtzM,3265
+datahub_airflow_plugin/hooks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datahub_airflow_plugin/hooks/datahub.py,sha256=zFYFTm2HbkBXE5AKVDbUjMpBeRAdyZu0GM84eafqKWo,11550
+datahub_airflow_plugin/operators/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datahub_airflow_plugin/operators/datahub.py,sha256=vU-v2fuJUkm1gMZa6zJehaVJ6QCLwvQd4G_vJ37BBVA,4402
+datahub_airflow_plugin/operators/datahub_assertion_operator.py,sha256=j_P9M1a5qME55pKHAfTqZsVVtIslFBO59r8UQOOBvsk,2914
+datahub_airflow_plugin/operators/datahub_assertion_sensor.py,sha256=QJIZZYQhqscj3bhBN5Sei-ABMRRAl2KiQxXTXcZQ51Q,2917
+datahub_airflow_plugin/operators/datahub_operation_operator.py,sha256=KJ8M8jJ7UWW6kNbiS-rELc-kqCPkZ3ck7z51oAXGPSI,3351
+datahub_airflow_plugin/operators/datahub_operation_sensor.py,sha256=U19fi5DpjBRWm_1ljXcjnspUzfa3mqYfOQZHjLk-ufI,3618
+acryl_datahub_airflow_plugin-1.3.1.5rc9.dist-info/METADATA,sha256=aofkCQ5pCOBGXUCuNYj7NueCmbgxtfSSR7dQPGKGH7w,12333
+acryl_datahub_airflow_plugin-1.3.1.5rc9.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+acryl_datahub_airflow_plugin-1.3.1.5rc9.dist-info/entry_points.txt,sha256=HqmajDHtrsz0b5Lswe1-eeuObxdtucd9YoxH77jJBA8,179
+acryl_datahub_airflow_plugin-1.3.1.5rc9.dist-info/top_level.txt,sha256=VBzisOQfzqL1WRbNyItaruf3kTigXltjzgqzbheaFp0,23
+acryl_datahub_airflow_plugin-1.3.1.5rc9.dist-info/RECORD,,

datahub_airflow_plugin/_airflow_compat.py

@@ -0,0 +1,32 @@
+# This module must be imported before any Airflow imports in any of our files.
+# It dispatches to version-specific compatibility modules.
+
+from datahub.utilities._markupsafe_compat import MARKUPSAFE_PATCHED
+
+# Critical safety check: Ensure MarkupSafe compatibility patch is applied
+# This must happen before importing Airflow to prevent MarkupSafe version conflicts
+# Using explicit exception instead of assert to ensure it runs even with python -O
+if not MARKUPSAFE_PATCHED:
+    raise RuntimeError(
+        "MarkupSafe compatibility patch must be applied before importing Airflow modules. "
+        "This is a critical safety check that cannot be disabled. "
+        "The patch ensures compatibility between different MarkupSafe versions used by "
+        "Airflow and DataHub dependencies."
+    )
+
+# Detect Airflow version and dispatch to version-specific compat module
+# These imports must be after MARKUPSAFE_PATCHED assertion.
+import airflow
+import packaging.version
+
+AIRFLOW_VERSION = packaging.version.parse(airflow.__version__)
+IS_AIRFLOW_3_OR_HIGHER = AIRFLOW_VERSION >= packaging.version.parse("3.0.0")
+
+if IS_AIRFLOW_3_OR_HIGHER:
+    from datahub_airflow_plugin.airflow3._airflow_compat import AIRFLOW_PATCHED
+else:
+    from datahub_airflow_plugin.airflow2._airflow_compat import AIRFLOW_PATCHED
+
+__all__ = [
+    "AIRFLOW_PATCHED",
+]

datahub_airflow_plugin/_airflow_shims.py

@@ -1,42 +1,75 @@
-from typing import List
+"""
+Pure dispatcher for version-specific Airflow shims.
 
-import airflow.version
-import packaging.version
-from airflow.models.operator import Operator
-
-try:
-    from airflow.sensors.external_task import ExternalTaskSensor
-except ImportError:
-    from airflow.sensors.external_task_sensor import ExternalTaskSensor  # type: ignore
-
-# Approach suggested by https://stackoverflow.com/a/11887885/5004662.
-AIRFLOW_VERSION = packaging.version.parse(airflow.version.version)
-HAS_AIRFLOW_DAG_LISTENER_API = True  # this is in Airflow 2.5+
-HAS_AIRFLOW_DATASET_LISTENER_API = AIRFLOW_VERSION >= packaging.version.parse(
-    "2.8.0.dev0"
-)
+This module automatically imports the correct shims based on the installed
+Airflow version, dispatching to either airflow2 or airflow3 implementations.
 
+No logic lives here - just clean version detection and re-export.
+"""
 
-def get_task_inlets(operator: "Operator") -> List:
-    # From Airflow 2.4 _inlets is dropped and inlets used consistently. Earlier it was not the case, so we have to stick there to _inlets
-    if hasattr(operator, "_inlets"):
-        return operator._inlets  # type: ignore[attr-defined, union-attr]
-    if hasattr(operator, "get_inlet_defs"):
-        return operator.get_inlet_defs()  # type: ignore[attr-defined]
-    return operator.inlets or []
+import packaging.version
 
+from datahub_airflow_plugin._airflow_version_specific import (
+    AIRFLOW_VERSION,
+    IS_AIRFLOW_3_OR_HIGHER,
+)
 
-def get_task_outlets(operator: "Operator") -> List:
-    # From Airflow 2.4 _outlets is dropped and inlets used consistently. Earlier it was not the case, so we have to stick there to _outlets
-    # We have to use _outlets because outlets is empty in Airflow < 2.4.0
-    if hasattr(operator, "_outlets"):
-        return operator._outlets  # type: ignore[attr-defined, union-attr]
-    if hasattr(operator, "get_outlet_defs"):
-        return operator.get_outlet_defs()
-    return operator.outlets or []
+# Version feature flags - hardcode based on Airflow version
+# These were previously in the old _airflow_shims but are better kept simple
+HAS_AIRFLOW_STANDALONE_CMD = AIRFLOW_VERSION >= packaging.version.parse("2.2")
+HAS_AIRFLOW_LISTENER_API = AIRFLOW_VERSION >= packaging.version.parse("2.3")
+HAS_AIRFLOW_DAG_LISTENER_API = AIRFLOW_VERSION >= packaging.version.parse("2.5")
+HAS_AIRFLOW_DATASET_LISTENER_API = AIRFLOW_VERSION >= packaging.version.parse("2.5")
 
+if IS_AIRFLOW_3_OR_HIGHER:
+    # Airflow 3.x - use airflow3 shims
+    from datahub_airflow_plugin.airflow3._shims import (
+        BaseOperator,
+        ExternalTaskSensor,
+        MappedOperator,
+        OpenLineagePlugin,
+        Operator,
+        TaskHolder,
+        get_operator_class,
+        get_task_inlets,
+        get_task_outlets,
+        redact_with_exclusions,
+        try_import_from_string,
+    )
+else:
+    # Airflow 2.x - use airflow2 shims
+    from datahub_airflow_plugin.airflow2._shims import (  # type: ignore[assignment]
+        BaseOperator,
+        ExternalTaskSensor,
+        MappedOperator,
+        OpenLineagePlugin,
+        Operator,
+        TaskHolder,
+        get_operator_class,
+        get_task_inlets,
+        get_task_outlets,
+        redact_with_exclusions,
+        try_import_from_string,
+    )
 
 __all__ = [
+    # Airflow version and feature flags
     "AIRFLOW_VERSION",
+    "IS_AIRFLOW_3_OR_HIGHER",
+    "HAS_AIRFLOW_STANDALONE_CMD",
+    "HAS_AIRFLOW_LISTENER_API",
+    "HAS_AIRFLOW_DAG_LISTENER_API",
+    "HAS_AIRFLOW_DATASET_LISTENER_API",
+    # Airflow objects
+    "BaseOperator",
+    "Operator",
+    "MappedOperator",
     "ExternalTaskSensor",
+    "TaskHolder",
+    "OpenLineagePlugin",
+    "get_operator_class",
+    "try_import_from_string",
+    "redact_with_exclusions",
+    "get_task_inlets",
+    "get_task_outlets",
 ]