apache-airflow-providers-common-sql 2.4.0__tar.gz → 3.0.0rc1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/PKG-INFO +8 -8
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/README.rst +3 -3
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/docs/changelog.rst +32 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/docs/index.rst +3 -3
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/docs/operators.rst +47 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/provider.yaml +2 -1
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/pyproject.toml +6 -6
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/__init__.py +1 -1
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/decorators/analytics.py +3 -1
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/decorators/sql.py +3 -1
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/hooks/sql.py +6 -38
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/hooks/sql.pyi +0 -6
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/operators/sql.py +38 -17
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/sensors/sql.py +82 -4
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/sensors/sql.pyi +9 -1
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/triggers/sql.py +80 -5
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/triggers/sql.pyi +24 -2
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/version_compat.py +2 -0
- apache_airflow_providers_common_sql-3.0.0rc1/tests/system/common/sql/example_sql_sensor.py +50 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/decorators/test_analytics.py +20 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/decorators/test_sql.py +19 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/hooks/test_sql.py +33 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/operators/test_sql_execute.py +89 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/sensors/test_sql.py +198 -2
- apache_airflow_providers_common_sql-3.0.0rc1/tests/unit/common/sql/triggers/test_sql.py +146 -0
- apache_airflow_providers_common_sql-2.4.0/tests/unit/common/sql/triggers/test_sql.py +0 -54
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/.gitignore +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/LICENSE +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/NOTICE +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/docs/.latest-doc-only-change.txt +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/docs/commits.rst +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/docs/conf.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/docs/connections.rst +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/docs/dataframes.rst +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/docs/dialects.rst +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/docs/installing-providers-from-sources.rst +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/docs/integration-logos/sql.png +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/docs/security.rst +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/docs/supported-database-types.rst +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/README_API.md +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/config.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/datafusion/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/datafusion/base.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/datafusion/engine.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/datafusion/exceptions.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/datafusion/format_handlers.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/datafusion/object_storage_provider.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/decorators/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/dialects/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/dialects/dialect.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/dialects/dialect.pyi +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/doc/adr/0001-record-architecture-decisions.md +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/doc/adr/0002-return-common-data-structure-from-dbapihook-derived-hooks.md +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/doc/adr/0003-introduce-notion-of-dialects-in-dbapihook.md +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/example_dags/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/example_dags/example_analytics.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/get_provider_info.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/get_provider_info.pyi +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/hooks/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/hooks/handlers.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/hooks/handlers.pyi +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/hooks/lineage.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/hooks/lineage.pyi +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/operators/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/operators/analytics.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/operators/generic_transfer.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/operators/generic_transfer.pyi +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/sensors/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/triggers/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/conftest.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/system/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/system/common/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/system/common/sql/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/system/common/sql/example_generic_transfer.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/system/common/sql/example_sql_bulk_load.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/system/common/sql/example_sql_column_table_check.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/system/common/sql/example_sql_execute_query.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/system/common/sql/example_sql_insert_rows.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/system/common/sql/example_sql_threshold_check.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/system/common/sql/example_sql_value_check.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/datafusion/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/datafusion/test_base.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/datafusion/test_engine.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/datafusion/test_exceptions.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/datafusion/test_format_handlers.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/datafusion/test_object_storage_provider.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/decorators/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/dialects/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/dialects/test_dialect.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/hooks/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/hooks/test_dbapi.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/hooks/test_handlers.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/hooks/test_lineage.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/hooks/test_sqlparse.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/operators/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/operators/test_analytics.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/operators/test_generic_transfer.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/operators/test_sql.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/sensors/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/test_config.py +0 -0
- {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/triggers/__init__.py +0 -0
{apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/PKG-INFO
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: apache-airflow-providers-common-sql
|
|
3
|
-
Version:
|
|
3
|
+
Version: 3.0.0rc1
|
|
4
4
|
Summary: Provider package apache-airflow-providers-common-sql for Apache Airflow
|
|
5
5
|
Keywords: airflow-provider,common.sql,airflow,integration
|
|
6
6
|
Author-email: Apache Software Foundation <dev@airflow.apache.org>
|
|
@@ -22,8 +22,8 @@ Classifier: Programming Language :: Python :: 3.14
|
|
|
22
22
|
Classifier: Topic :: System :: Monitoring
|
|
23
23
|
License-File: LICENSE
|
|
24
24
|
License-File: NOTICE
|
|
25
|
-
Requires-Dist: apache-airflow>=2.11.
|
|
26
|
-
Requires-Dist: apache-airflow-providers-common-compat>=1.23.
|
|
25
|
+
Requires-Dist: apache-airflow>=2.11.0rc1
|
|
26
|
+
Requires-Dist: apache-airflow-providers-common-compat>=1.23.0rc1
|
|
27
27
|
Requires-Dist: sqlparse>=0.5.1
|
|
28
28
|
Requires-Dist: more-itertools>=9.0.0
|
|
29
29
|
Requires-Dist: methodtools>=0.4.7
|
|
@@ -38,8 +38,8 @@ Requires-Dist: polars>=1.26.0 ; extra == "polars"
|
|
|
38
38
|
Requires-Dist: pyiceberg-core>=0.8.0 ; extra == "pyiceberg-core"
|
|
39
39
|
Requires-Dist: sqlalchemy>=1.4.54,!=2.1.0 ; extra == "sqlalchemy"
|
|
40
40
|
Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
|
|
41
|
-
Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-common-sql/
|
|
42
|
-
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-common-sql/
|
|
41
|
+
Project-URL: Changelog, https://airflow.staged.apache.org/docs/apache-airflow-providers-common-sql/3.0.0/changelog.html
|
|
42
|
+
Project-URL: Documentation, https://airflow.staged.apache.org/docs/apache-airflow-providers-common-sql/3.0.0
|
|
43
43
|
Project-URL: Mastodon, https://fosstodon.org/@airflow
|
|
44
44
|
Project-URL: Slack Chat, https://s.apache.org/airflow-slack
|
|
45
45
|
Project-URL: Source Code, https://github.com/apache/airflow
|
|
@@ -82,7 +82,7 @@ Import-Namespace: airflow.providers.common
|
|
|
82
82
|
|
|
83
83
|
Package ``apache-airflow-providers-common-sql``
|
|
84
84
|
|
|
85
|
-
Release: ``
|
|
85
|
+
Release: ``3.0.0``
|
|
86
86
|
|
|
87
87
|
|
|
88
88
|
`Common SQL Provider <https://en.wikipedia.org/wiki/SQL>`__
|
|
@@ -95,7 +95,7 @@ This is a provider package for ``common.sql`` provider. All classes for this pro
|
|
|
95
95
|
are in ``airflow.providers.common.sql`` python package.
|
|
96
96
|
|
|
97
97
|
You can find package information and changelog for the provider
|
|
98
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/
|
|
98
|
+
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/3.0.0/>`_.
|
|
99
99
|
|
|
100
100
|
Installation
|
|
101
101
|
------------
|
|
@@ -157,5 +157,5 @@ Extra Dependencies
|
|
|
157
157
|
================== =======================================================================================================================================================================
|
|
158
158
|
|
|
159
159
|
The changelog for the provider package can be found in the
|
|
160
|
-
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/
|
|
160
|
+
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/3.0.0/changelog.html>`_.
|
|
161
161
|
|
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
|
|
24
24
|
Package ``apache-airflow-providers-common-sql``
|
|
25
25
|
|
|
26
|
-
Release: ``
|
|
26
|
+
Release: ``3.0.0``
|
|
27
27
|
|
|
28
28
|
|
|
29
29
|
`Common SQL Provider <https://en.wikipedia.org/wiki/SQL>`__
|
|
@@ -36,7 +36,7 @@ This is a provider package for ``common.sql`` provider. All classes for this pro
|
|
|
36
36
|
are in ``airflow.providers.common.sql`` python package.
|
|
37
37
|
|
|
38
38
|
You can find package information and changelog for the provider
|
|
39
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/
|
|
39
|
+
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/3.0.0/>`_.
|
|
40
40
|
|
|
41
41
|
Installation
|
|
42
42
|
------------
|
|
@@ -98,4 +98,4 @@ Extra Dependencies
|
|
|
98
98
|
================== =======================================================================================================================================================================
|
|
99
99
|
|
|
100
100
|
The changelog for the provider package can be found in the
|
|
101
|
-
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/
|
|
101
|
+
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/3.0.0/changelog.html>`_.
|
|
@@ -25,6 +25,38 @@
|
|
|
25
25
|
Changelog
|
|
26
26
|
---------
|
|
27
27
|
|
|
28
|
+
3.0.0
|
|
29
|
+
.....
|
|
30
|
+
|
|
31
|
+
.. note::
|
|
32
|
+
``@task.sql`` and ``@task.analytics`` now render only the SQL returned by the decorated function
|
|
33
|
+
inside ``execute()``. Their other template fields (``parameters``, ``op_args``, ``op_kwargs``, ...)
|
|
34
|
+
are rendered once before ``execute()`` and are no longer rendered a second time.
|
|
35
|
+
|
|
36
|
+
.. warning::
|
|
37
|
+
The deprecated ``DbApiHook.get_pandas_df`` and ``DbApiHook.get_pandas_df_by_chunks`` methods have been removed. Use ``DbApiHook.get_df(..., df_type="pandas")`` and ``DbApiHook.get_df_by_chunks(..., df_type="pandas")`` instead.
|
|
38
|
+
|
|
39
|
+
Breaking changes
|
|
40
|
+
~~~~~~~~~~~~~~~~
|
|
41
|
+
|
|
42
|
+
* ``Remove deprecated get_pandas_df methods from SQL hooks (#76130)``
|
|
43
|
+
|
|
44
|
+
Features
|
|
45
|
+
~~~~~~~~
|
|
46
|
+
|
|
47
|
+
* ``Add deferrable mode to 'SqlSensor' (#76270)``
|
|
48
|
+
|
|
49
|
+
Bug Fixes
|
|
50
|
+
~~~~~~~~~
|
|
51
|
+
|
|
52
|
+
* ``Render only the returned value in TaskFlow decorators that template it (#76244)``
|
|
53
|
+
* ``Surface errors from OpenLineage database info lookups without losing lineage (#74805)``
|
|
54
|
+
* ``Fix ''DbApiHook.inspector'' keeping a database connection open until process exit (#75327)``
|
|
55
|
+
|
|
56
|
+
.. Below changes are excluded from the changelog. Move them to
|
|
57
|
+
appropriate section above if needed. Do not delete the lines(!):
|
|
58
|
+
* ``[main] Upgrade important CI environment (#76278)``
|
|
59
|
+
|
|
28
60
|
2.4.0
|
|
29
61
|
.....
|
|
30
62
|
|
|
@@ -79,7 +79,7 @@ apache-airflow-providers-common-sql package
|
|
|
79
79
|
`Common SQL Provider <https://en.wikipedia.org/wiki/SQL>`__
|
|
80
80
|
|
|
81
81
|
|
|
82
|
-
Release:
|
|
82
|
+
Release: 3.0.0
|
|
83
83
|
|
|
84
84
|
Provider package
|
|
85
85
|
----------------
|
|
@@ -160,5 +160,5 @@ Downloading official packages
|
|
|
160
160
|
You can download officially released packages and verify their checksums and signatures from the
|
|
161
161
|
`Official Apache Download site <https://downloads.apache.org/airflow/providers/>`_
|
|
162
162
|
|
|
163
|
-
* `The apache-airflow-providers-common-sql
|
|
164
|
-
* `The apache-airflow-providers-common-sql
|
|
163
|
+
* `The apache-airflow-providers-common-sql 3.0.0 sdist package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-3.0.0.tar.gz>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-3.0.0.tar.gz.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-3.0.0.tar.gz.sha512>`__)
|
|
164
|
+
* `The apache-airflow-providers-common-sql 3.0.0 wheel package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-3.0.0-py3-none-any.whl>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-3.0.0-py3-none-any.whl.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-3.0.0-py3-none-any.whl.sha512>`__)
|
|
@@ -282,6 +282,53 @@ between two connections.
|
|
|
282
282
|
:start-after: [START howto_operator_generic_transfer]
|
|
283
283
|
:end-before: [END howto_operator_generic_transfer]
|
|
284
284
|
|
|
285
|
+
.. _howto/sensor:SqlSensor:
|
|
286
|
+
|
|
287
|
+
Wait for a SQL condition
|
|
288
|
+
~~~~~~~~~~~~~~~~~~~~~~~~
|
|
289
|
+
|
|
290
|
+
Use the :class:`~airflow.providers.common.sql.sensors.sql.SqlSensor` to wait until a query returns a row
|
|
291
|
+
whose first cell is truthy, for example until today's orders have landed. It works with any connection
|
|
292
|
+
whose hook is a :class:`~airflow.providers.common.sql.hooks.sql.DbApiHook`. The examples below use
|
|
293
|
+
PostgreSQL; write ``sql`` and ``parameters`` in your database's dialect and placeholder style.
|
|
294
|
+
|
|
295
|
+
.. exampleinclude:: /../tests/system/common/sql/example_sql_sensor.py
|
|
296
|
+
:language: python
|
|
297
|
+
:dedent: 4
|
|
298
|
+
:start-after: [START howto_sensor_sql]
|
|
299
|
+
:end-before: [END howto_sensor_sql]
|
|
300
|
+
|
|
301
|
+
For a condition you cannot write in SQL, pass ``success`` or ``failure`` callables. They receive the first
|
|
302
|
+
cell of the first row, or whatever ``selector`` returns for that row. ``fail_on_empty=True`` fails the task
|
|
303
|
+
when the query returns no rows instead of checking again.
|
|
304
|
+
|
|
305
|
+
Deferrable mode
|
|
306
|
+
---------------
|
|
307
|
+
|
|
308
|
+
With ``deferrable=True``, the sensor checks once on the worker. If the condition is not met, it defers and
|
|
309
|
+
the triggerer runs the query every ``poke_interval`` seconds, so the task holds no worker slot while it
|
|
310
|
+
waits. ``timeout`` caps the time spent deferred.
|
|
311
|
+
|
|
312
|
+
.. exampleinclude:: /../tests/system/common/sql/example_sql_sensor.py
|
|
313
|
+
:language: python
|
|
314
|
+
:dedent: 4
|
|
315
|
+
:start-after: [START howto_sensor_sql_deferrable]
|
|
316
|
+
:end-before: [END howto_sensor_sql_deferrable]
|
|
317
|
+
|
|
318
|
+
The triggerer only checks the default condition, that the first cell of the first row is truthy, so write
|
|
319
|
+
the condition in SQL as above. The sensor logs a warning and runs in its usual ``mode`` instead of
|
|
320
|
+
deferring when:
|
|
321
|
+
|
|
322
|
+
- ``success``, ``failure`` or ``selector`` is set, because these Python callables only run on a worker;
|
|
323
|
+
- ``silent_fail``, ``never_fail`` or ``exponential_backoff`` is set, because they only apply to pokes on a
|
|
324
|
+
worker;
|
|
325
|
+
- a subclass overrides ``poke`` or ``_get_hook``, because the triggerer would not run that code;
|
|
326
|
+
- it runs on Airflow 3 before 3.2.2, whose triggerer cannot safely look up connections from the thread
|
|
327
|
+
that runs the query. Deferring works on Airflow 2.11 and on 3.2.2 and later;
|
|
328
|
+
- on Airflow 2, ``parameters`` holds values other than strings, numbers, booleans and ``None``.
|
|
329
|
+
Airflow 2 stores trigger arguments as JSON, so dates, datetimes and ``Decimal`` values would reach
|
|
330
|
+
the triggerer as strings.
|
|
331
|
+
|
|
285
332
|
Analytics Operator
|
|
286
333
|
~~~~~~~~~~~~~~~~~~
|
|
287
334
|
|
|
@@ -23,12 +23,13 @@ description: |
|
|
|
23
23
|
|
|
24
24
|
state: ready
|
|
25
25
|
lifecycle: production
|
|
26
|
-
source-date-epoch:
|
|
26
|
+
source-date-epoch: 1793816745
|
|
27
27
|
# Note that those versions are maintained by release manager - do not update them manually
|
|
28
28
|
# with the exception of case where other provider in sources has >= new provider version.
|
|
29
29
|
# In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have
|
|
30
30
|
# to be done in the same PR
|
|
31
31
|
versions:
|
|
32
|
+
- 3.0.0
|
|
32
33
|
- 2.4.0
|
|
33
34
|
- 2.3.0
|
|
34
35
|
- 2.2.0
|
|
@@ -20,12 +20,12 @@
|
|
|
20
20
|
# IF YOU WANT TO MODIFY THIS FILE EXCEPT DEPENDENCIES, YOU SHOULD MODIFY THE TEMPLATE
|
|
21
21
|
# `pyproject_TEMPLATE.toml.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
|
|
22
22
|
[build-system]
|
|
23
|
-
requires = ["flit_core==4.
|
|
23
|
+
requires = ["flit_core==4.2.0"]
|
|
24
24
|
build-backend = "flit_core.buildapi"
|
|
25
25
|
|
|
26
26
|
[project]
|
|
27
27
|
name = "apache-airflow-providers-common-sql"
|
|
28
|
-
version = "
|
|
28
|
+
version = "3.0.0rc1"
|
|
29
29
|
description = "Provider package apache-airflow-providers-common-sql for Apache Airflow"
|
|
30
30
|
readme = "README.rst"
|
|
31
31
|
license = "Apache-2.0"
|
|
@@ -58,8 +58,8 @@ requires-python = ">=3.11"
|
|
|
58
58
|
# Make sure to run ``prek update-providers-dependencies --all-files``
|
|
59
59
|
# After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build``
|
|
60
60
|
dependencies = [
|
|
61
|
-
"apache-airflow>=2.11.
|
|
62
|
-
"apache-airflow-providers-common-compat>=1.23.
|
|
61
|
+
"apache-airflow>=2.11.0rc1",
|
|
62
|
+
"apache-airflow-providers-common-compat>=1.23.0rc1",
|
|
63
63
|
"sqlparse>=0.5.1",
|
|
64
64
|
"more-itertools>=9.0.0",
|
|
65
65
|
# The methodtools dependency is necessary since the introduction of dialects:
|
|
@@ -147,8 +147,8 @@ apache-airflow-providers-common-sql = {workspace = true}
|
|
|
147
147
|
apache-airflow-providers-standard = {workspace = true}
|
|
148
148
|
|
|
149
149
|
[project.urls]
|
|
150
|
-
"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-common-sql/
|
|
151
|
-
"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-common-sql/
|
|
150
|
+
"Documentation" = "https://airflow.staged.apache.org/docs/apache-airflow-providers-common-sql/3.0.0"
|
|
151
|
+
"Changelog" = "https://airflow.staged.apache.org/docs/apache-airflow-providers-common-sql/3.0.0/changelog.html"
|
|
152
152
|
"Bug Tracker" = "https://github.com/apache/airflow/issues"
|
|
153
153
|
"Source Code" = "https://github.com/apache/airflow"
|
|
154
154
|
"Slack Chat" = "https://s.apache.org/airflow-slack"
|
|
@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
|
|
|
29
29
|
|
|
30
30
|
__all__ = ["__version__"]
|
|
31
31
|
|
|
32
|
-
__version__ = "
|
|
32
|
+
__version__ = "3.0.0"
|
|
33
33
|
|
|
34
34
|
if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
|
|
35
35
|
"2.11.0"
|
|
@@ -117,7 +117,9 @@ class _AnalyticsDecoratedOperator(DecoratedOperator, AnalyticsOperator):
|
|
|
117
117
|
# AnalyticsOperator expects queries as a list of strings
|
|
118
118
|
self.queries = [result] if isinstance(result, str) else result
|
|
119
119
|
|
|
120
|
-
|
|
120
|
+
# Render only the queries the callable returned. The other template fields were already
|
|
121
|
+
# rendered before execute().
|
|
122
|
+
self.queries = self.render_template(self.queries, context)
|
|
121
123
|
|
|
122
124
|
return AnalyticsOperator.execute(self, context)
|
|
123
125
|
|
|
@@ -110,7 +110,9 @@ class _SQLDecoratedOperator(DecoratedOperator, SQLExecuteQueryOperator):
|
|
|
110
110
|
"string or a non-empty list of non-empty strings."
|
|
111
111
|
)
|
|
112
112
|
|
|
113
|
-
|
|
113
|
+
# Render only the SQL the callable returned. The other template fields were already
|
|
114
|
+
# rendered before execute().
|
|
115
|
+
self.sql = self.render_template(self.sql, context)
|
|
114
116
|
|
|
115
117
|
return super().execute(context)
|
|
116
118
|
|
|
@@ -27,7 +27,6 @@ from typing import TYPE_CHECKING, Any, Literal, Protocol, TypeVar, cast, overloa
|
|
|
27
27
|
from urllib.parse import urlparse
|
|
28
28
|
|
|
29
29
|
import sqlparse
|
|
30
|
-
from deprecated import deprecated
|
|
31
30
|
from methodtools import lru_cache
|
|
32
31
|
from more_itertools import chunked
|
|
33
32
|
|
|
@@ -35,15 +34,16 @@ try:
|
|
|
35
34
|
from sqlalchemy import create_engine, inspect
|
|
36
35
|
from sqlalchemy.engine import make_url
|
|
37
36
|
from sqlalchemy.exc import ArgumentError, NoSuchModuleError
|
|
37
|
+
from sqlalchemy.pool import NullPool
|
|
38
38
|
except ImportError:
|
|
39
39
|
create_engine = None # type: ignore[assignment]
|
|
40
40
|
inspect = None # type: ignore[assignment]
|
|
41
41
|
make_url = None # type: ignore[assignment]
|
|
42
|
+
NullPool = None # type: ignore[assignment,misc]
|
|
42
43
|
ArgumentError = Exception # type: ignore[misc,assignment]
|
|
43
44
|
NoSuchModuleError = Exception # type: ignore[misc,assignment]
|
|
44
45
|
|
|
45
46
|
|
|
46
|
-
from airflow.exceptions import AirflowProviderDeprecationWarning
|
|
47
47
|
from airflow.providers.common.compat.module_loading import import_string
|
|
48
48
|
from airflow.providers.common.compat.sdk import (
|
|
49
49
|
AirflowException,
|
|
@@ -378,7 +378,10 @@ class DbApiHook(BaseHook):
|
|
|
378
378
|
"SQLAlchemy is required for database inspection. "
|
|
379
379
|
"Install it with: pip install 'apache-airflow-providers-common-sql[sqlalchemy]'"
|
|
380
380
|
)
|
|
381
|
-
|
|
381
|
+
# The inspector is cached for the hook's lifetime, so a pooled engine would keep the
|
|
382
|
+
# connection it reflected with open until the process exits. With NullPool, no
|
|
383
|
+
# connection outlives the inspection that opened it.
|
|
384
|
+
return inspect(self.get_sqlalchemy_engine({"poolclass": NullPool}))
|
|
382
385
|
|
|
383
386
|
def get_table_schema(self, table_name: str, schema: str | None = None) -> list[dict[str, str]]:
|
|
384
387
|
"""
|
|
@@ -445,41 +448,6 @@ class DbApiHook(BaseHook):
|
|
|
445
448
|
self.log.debug("reserved words for '%s': %s", dialect_name, result)
|
|
446
449
|
return result
|
|
447
450
|
|
|
448
|
-
@deprecated(
|
|
449
|
-
reason="Replaced by function `get_df`.",
|
|
450
|
-
category=AirflowProviderDeprecationWarning,
|
|
451
|
-
action="ignore",
|
|
452
|
-
)
|
|
453
|
-
def get_pandas_df(
|
|
454
|
-
self,
|
|
455
|
-
sql,
|
|
456
|
-
parameters: list | tuple | Mapping[str, Any] | None = None,
|
|
457
|
-
**kwargs,
|
|
458
|
-
) -> PandasDataFrame:
|
|
459
|
-
"""
|
|
460
|
-
Execute the sql and returns a pandas dataframe.
|
|
461
|
-
|
|
462
|
-
:param sql: the sql statement to be executed (str) or a list of sql statements to execute
|
|
463
|
-
:param parameters: The parameters to render the SQL query with.
|
|
464
|
-
:param kwargs: (optional) passed into pandas.io.sql.read_sql method
|
|
465
|
-
"""
|
|
466
|
-
return self.get_df(sql, parameters, df_type="pandas", **kwargs)
|
|
467
|
-
|
|
468
|
-
@deprecated(
|
|
469
|
-
reason="Replaced by function `get_df_by_chunks`.",
|
|
470
|
-
category=AirflowProviderDeprecationWarning,
|
|
471
|
-
action="ignore",
|
|
472
|
-
)
|
|
473
|
-
def get_pandas_df_by_chunks(
|
|
474
|
-
self,
|
|
475
|
-
sql,
|
|
476
|
-
parameters: list | tuple | Mapping[str, Any] | None = None,
|
|
477
|
-
*,
|
|
478
|
-
chunksize: int,
|
|
479
|
-
**kwargs,
|
|
480
|
-
) -> Generator[PandasDataFrame, None, None]:
|
|
481
|
-
return self.get_df_by_chunks(sql, parameters, chunksize=chunksize, df_type="pandas", **kwargs)
|
|
482
|
-
|
|
483
451
|
@overload
|
|
484
452
|
def get_df(
|
|
485
453
|
self,
|
|
@@ -104,12 +104,6 @@ class DbApiHook(BaseHook):
|
|
|
104
104
|
@property
|
|
105
105
|
def reserved_words(self) -> set[str]: ...
|
|
106
106
|
def get_reserved_words(self, dialect_name: str) -> set[str]: ...
|
|
107
|
-
def get_pandas_df(
|
|
108
|
-
self, sql, parameters: list | tuple | Mapping[str, Any] | None = None, **kwargs
|
|
109
|
-
) -> PandasDataFrame: ...
|
|
110
|
-
def get_pandas_df_by_chunks(
|
|
111
|
-
self, sql, parameters: list | tuple | Mapping[str, Any] | None = None, *, chunksize: int, **kwargs
|
|
112
|
-
) -> Generator[PandasDataFrame]: ...
|
|
113
107
|
def get_records(
|
|
114
108
|
self, sql: str | list[str], parameters: Iterable | Mapping[str, Any] | None = None
|
|
115
109
|
) -> Any: ...
|
|
@@ -296,32 +296,47 @@ class BaseSQLOperator(BaseOperator):
|
|
|
296
296
|
# OpenLineage provider release < 1.8.0 - we always use connection
|
|
297
297
|
use_external_connection = True
|
|
298
298
|
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
299
|
+
conn_id = getattr(hook, hook.conn_name_attr)
|
|
300
|
+
connection = hook.get_connection(conn_id)
|
|
301
|
+
get_database_info = getattr(hook, "get_openlineage_database_info", None)
|
|
302
|
+
database_info = None
|
|
303
|
+
if get_database_info is not None:
|
|
304
|
+
try:
|
|
305
|
+
database_info = get_database_info(connection)
|
|
306
|
+
except Exception as e:
|
|
307
|
+
self.log.warning("Failed to get OpenLineage database info for %s: %s", conn_id, e)
|
|
308
|
+
self.log.debug("OpenLineage failure details:", exc_info=True)
|
|
305
309
|
|
|
306
310
|
if database_info is None:
|
|
307
311
|
self.log.debug("OpenLineage could not retrieve database information. Skipping.")
|
|
308
312
|
return OperatorLineage()
|
|
309
313
|
|
|
314
|
+
get_dialect = getattr(hook, "get_openlineage_database_dialect", None)
|
|
315
|
+
get_default_schema = getattr(hook, "get_openlineage_default_schema", None)
|
|
316
|
+
if get_dialect is None or get_default_schema is None:
|
|
317
|
+
self.log.debug("%s has no database dialect or default schema provided", conn_id)
|
|
318
|
+
return None
|
|
310
319
|
try:
|
|
311
|
-
sql_parser = SQLParser(
|
|
312
|
-
|
|
313
|
-
|
|
320
|
+
sql_parser = SQLParser(dialect=get_dialect(connection), default_schema=get_default_schema())
|
|
321
|
+
except Exception as e:
|
|
322
|
+
self.log.warning(
|
|
323
|
+
"Failed to get OpenLineage database dialect or default schema for %s: %s", conn_id, e
|
|
314
324
|
)
|
|
315
|
-
|
|
316
|
-
self.log.debug("%s failed to get database dialect", hook)
|
|
325
|
+
self.log.debug("OpenLineage failure details:", exc_info=True)
|
|
317
326
|
return None
|
|
318
327
|
|
|
328
|
+
sqlalchemy_engine = None
|
|
329
|
+
try:
|
|
330
|
+
sqlalchemy_engine = hook.get_sqlalchemy_engine()
|
|
331
|
+
except Exception as e:
|
|
332
|
+
self.log.debug("Failed to get sql alchemy engine: %s", e)
|
|
333
|
+
|
|
319
334
|
operator_lineage = sql_parser.generate_openlineage_metadata_from_sql(
|
|
320
335
|
sql=sql,
|
|
321
336
|
hook=hook,
|
|
322
337
|
database_info=database_info,
|
|
323
338
|
database=self.database,
|
|
324
|
-
sqlalchemy_engine=
|
|
339
|
+
sqlalchemy_engine=sqlalchemy_engine,
|
|
325
340
|
use_connection=use_external_connection,
|
|
326
341
|
)
|
|
327
342
|
|
|
@@ -337,11 +352,17 @@ class BaseSQLOperator(BaseOperator):
|
|
|
337
352
|
|
|
338
353
|
operator_lineage = self.get_openlineage_facets_on_start() or OperatorLineage()
|
|
339
354
|
hook = self.get_db_hook()
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
355
|
+
conn_id = getattr(hook, hook.conn_name_attr)
|
|
356
|
+
get_specific_lineage = getattr(hook, "get_openlineage_database_specific_lineage", None)
|
|
357
|
+
database_specific_lineage = None
|
|
358
|
+
if get_specific_lineage is None:
|
|
359
|
+
self.log.debug("%s has no database specific lineage provided", conn_id)
|
|
360
|
+
else:
|
|
361
|
+
try:
|
|
362
|
+
database_specific_lineage = get_specific_lineage(task_instance)
|
|
363
|
+
except Exception as e:
|
|
364
|
+
self.log.warning("Failed to get OpenLineage database specific lineage for %s: %s", conn_id, e)
|
|
365
|
+
self.log.debug("OpenLineage failure details:", exc_info=True)
|
|
345
366
|
|
|
346
367
|
if database_specific_lineage is None:
|
|
347
368
|
if not self.check_results:
|
|
@@ -17,15 +17,41 @@
|
|
|
17
17
|
from __future__ import annotations
|
|
18
18
|
|
|
19
19
|
from collections.abc import Callable, Mapping, Sequence
|
|
20
|
+
from datetime import timedelta
|
|
20
21
|
from operator import itemgetter
|
|
21
22
|
from typing import TYPE_CHECKING, Any
|
|
22
23
|
|
|
23
|
-
from airflow.providers.common.compat.sdk import AirflowException, BaseHook, BaseSensorOperator
|
|
24
|
+
from airflow.providers.common.compat.sdk import AirflowException, BaseHook, BaseSensorOperator, conf
|
|
24
25
|
from airflow.providers.common.sql.hooks.sql import DbApiHook
|
|
26
|
+
from airflow.providers.common.sql.triggers.sql import NO_ROWS_MESSAGE, SqlSensorTrigger
|
|
27
|
+
from airflow.providers.common.sql.version_compat import AIRFLOW_V_3_0_PLUS, AIRFLOW_V_3_2_2_PLUS
|
|
25
28
|
|
|
26
29
|
if TYPE_CHECKING:
|
|
27
30
|
from airflow.providers.common.compat.sdk import Context
|
|
28
31
|
|
|
32
|
+
_first_cell = itemgetter(0)
|
|
33
|
+
|
|
34
|
+
# Values that survive Airflow 2's JSON storage of trigger arguments unchanged.
|
|
35
|
+
_JSON_SCALARS = (str, int, float, bool, type(None))
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _is_first_cell(selector: Callable[[tuple[Any]], Any]) -> bool:
|
|
39
|
+
# Compare by value rather than identity: a deep copy of the sensor, or an explicit
|
|
40
|
+
# ``selector=itemgetter(0)``, must still count as the default condition.
|
|
41
|
+
return isinstance(selector, itemgetter) and selector.__reduce__() == _first_cell.__reduce__()
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _survives_json(value: Any) -> bool:
|
|
45
|
+
if isinstance(value, Mapping):
|
|
46
|
+
return all(isinstance(k, str) and _survives_json(v) for k, v in value.items())
|
|
47
|
+
if isinstance(value, (list, tuple)):
|
|
48
|
+
return all(_survives_json(v) for v in value)
|
|
49
|
+
return isinstance(value, _JSON_SCALARS)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class SqlSensorTriggerError(AirflowException):
|
|
53
|
+
"""Raised when the deferred ``SqlSensor`` trigger reports a failure."""
|
|
54
|
+
|
|
29
55
|
|
|
30
56
|
class SqlSensor(BaseSensorOperator):
|
|
31
57
|
"""
|
|
@@ -55,6 +81,12 @@ class SqlSensor(BaseSensorOperator):
|
|
|
55
81
|
:param fail_on_empty: Explicitly fail on no rows returned.
|
|
56
82
|
:param hook_params: Extra config params to be passed to the underlying hook.
|
|
57
83
|
Should match the desired hook constructor params.
|
|
84
|
+
:param deferrable: Wait in the triggerer instead of on a worker. The triggerer only
|
|
85
|
+
checks the default condition (the first cell is truthy), so the sensor logs a
|
|
86
|
+
warning and runs in its usual ``mode`` instead when ``success``, ``failure``,
|
|
87
|
+
``selector``, ``silent_fail``, ``never_fail`` or ``exponential_backoff`` is set,
|
|
88
|
+
when a subclass overrides ``poke`` or ``_get_hook``, on Airflow 3 before 3.2.2, or
|
|
89
|
+
on Airflow 2 when ``parameters`` holds values JSON cannot keep, such as dates.
|
|
58
90
|
"""
|
|
59
91
|
|
|
60
92
|
template_fields: Sequence[str] = ("sql", "hook_params", "parameters", "conn_id")
|
|
@@ -69,9 +101,10 @@ class SqlSensor(BaseSensorOperator):
|
|
|
69
101
|
parameters: Mapping[str, Any] | None = None,
|
|
70
102
|
success: Callable[[Any], bool] | None = None,
|
|
71
103
|
failure: Callable[[Any], bool] | None = None,
|
|
72
|
-
selector: Callable[[tuple[Any]], Any] =
|
|
104
|
+
selector: Callable[[tuple[Any]], Any] = _first_cell,
|
|
73
105
|
fail_on_empty: bool = False,
|
|
74
106
|
hook_params: Mapping[str, Any] | None = None,
|
|
107
|
+
deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
|
|
75
108
|
**kwargs,
|
|
76
109
|
) -> None:
|
|
77
110
|
self.conn_id = conn_id
|
|
@@ -82,6 +115,7 @@ class SqlSensor(BaseSensorOperator):
|
|
|
82
115
|
self.selector = selector
|
|
83
116
|
self.fail_on_empty = fail_on_empty
|
|
84
117
|
self.hook_params = hook_params
|
|
118
|
+
self.deferrable = deferrable
|
|
85
119
|
super().__init__(**kwargs)
|
|
86
120
|
|
|
87
121
|
def _get_hook(self) -> DbApiHook:
|
|
@@ -101,8 +135,7 @@ class SqlSensor(BaseSensorOperator):
|
|
|
101
135
|
records = hook.get_records(self.sql, self.parameters)
|
|
102
136
|
if not records:
|
|
103
137
|
if self.fail_on_empty:
|
|
104
|
-
|
|
105
|
-
raise AirflowException(message)
|
|
138
|
+
raise AirflowException(NO_ROWS_MESSAGE)
|
|
106
139
|
return False
|
|
107
140
|
|
|
108
141
|
condition = self.selector(records[0])
|
|
@@ -121,3 +154,48 @@ class SqlSensor(BaseSensorOperator):
|
|
|
121
154
|
message = f"self.success is present, but not callable -> {self.success}"
|
|
122
155
|
raise AirflowException(message)
|
|
123
156
|
return bool(condition)
|
|
157
|
+
|
|
158
|
+
def execute(self, context: Context) -> Any:
|
|
159
|
+
if not self.deferrable:
|
|
160
|
+
return super().execute(context)
|
|
161
|
+
if reason := self._reason_not_to_defer():
|
|
162
|
+
self.log.warning("Cannot defer because %s; running in %s mode instead.", reason, self.mode)
|
|
163
|
+
return super().execute(context)
|
|
164
|
+
# silent_fail and never_fail fall back above, so an error here propagates exactly as
|
|
165
|
+
# it would from the first poke of super().execute().
|
|
166
|
+
if self.poke(context):
|
|
167
|
+
self.log.info("Success criteria met. Exiting.")
|
|
168
|
+
return None
|
|
169
|
+
self.defer(
|
|
170
|
+
trigger=SqlSensorTrigger(
|
|
171
|
+
sql=self.sql,
|
|
172
|
+
conn_id=self.conn_id,
|
|
173
|
+
poke_interval=self.poke_interval,
|
|
174
|
+
parameters=self.parameters,
|
|
175
|
+
hook_params=dict(self.hook_params) if self.hook_params is not None else None,
|
|
176
|
+
fail_on_empty=self.fail_on_empty,
|
|
177
|
+
),
|
|
178
|
+
method_name=self.execute_complete.__name__,
|
|
179
|
+
timeout=timedelta(seconds=self.timeout),
|
|
180
|
+
)
|
|
181
|
+
|
|
182
|
+
def _reason_not_to_defer(self) -> str | None:
|
|
183
|
+
if AIRFLOW_V_3_0_PLUS and not AIRFLOW_V_3_2_2_PLUS:
|
|
184
|
+
return "the triggerer before Airflow 3.2.2 cannot look up connections from a thread safely"
|
|
185
|
+
if self.success is not None or self.failure is not None or not _is_first_cell(self.selector):
|
|
186
|
+
return "success, failure and selector only run on a worker"
|
|
187
|
+
if type(self).poke is not SqlSensor.poke or type(self)._get_hook is not SqlSensor._get_hook:
|
|
188
|
+
return f"{type(self).__name__} overrides poke or _get_hook, which the triggerer does not run"
|
|
189
|
+
if self.silent_fail or self.never_fail or self.exponential_backoff:
|
|
190
|
+
return "silent_fail, never_fail and exponential_backoff only apply when poking on a worker"
|
|
191
|
+
if not AIRFLOW_V_3_0_PLUS and not _survives_json(self.parameters):
|
|
192
|
+
return (
|
|
193
|
+
"Airflow 2 stores trigger arguments as JSON, which would hand the triggerer "
|
|
194
|
+
"date, datetime or Decimal parameters as strings"
|
|
195
|
+
)
|
|
196
|
+
return None
|
|
197
|
+
|
|
198
|
+
def execute_complete(self, context: Context, event: dict[str, Any]) -> None:
|
|
199
|
+
if event["status"] == "failure":
|
|
200
|
+
raise SqlSensorTriggerError(event["message"])
|
|
201
|
+
self.log.info("Success criteria met. Exiting.")
|
|
@@ -37,9 +37,11 @@ from typing import Any
|
|
|
37
37
|
|
|
38
38
|
from _typeshed import Incomplete as Incomplete
|
|
39
39
|
|
|
40
|
-
from airflow.providers.common.compat.sdk import BaseSensorOperator
|
|
40
|
+
from airflow.providers.common.compat.sdk import AirflowException, BaseSensorOperator
|
|
41
41
|
from airflow.sdk import Context
|
|
42
42
|
|
|
43
|
+
class SqlSensorTriggerError(AirflowException): ...
|
|
44
|
+
|
|
43
45
|
class SqlSensor(BaseSensorOperator):
|
|
44
46
|
template_fields: Sequence[str]
|
|
45
47
|
template_ext: Sequence[str]
|
|
@@ -49,8 +51,10 @@ class SqlSensor(BaseSensorOperator):
|
|
|
49
51
|
parameters: Incomplete
|
|
50
52
|
success: Incomplete
|
|
51
53
|
failure: Incomplete
|
|
54
|
+
selector: Incomplete
|
|
52
55
|
fail_on_empty: Incomplete
|
|
53
56
|
hook_params: Incomplete
|
|
57
|
+
deferrable: Incomplete
|
|
54
58
|
def __init__(
|
|
55
59
|
self,
|
|
56
60
|
*,
|
|
@@ -59,8 +63,12 @@ class SqlSensor(BaseSensorOperator):
|
|
|
59
63
|
parameters: Mapping[str, Any] | None = None,
|
|
60
64
|
success: Callable[[Any], bool] | None = None,
|
|
61
65
|
failure: Callable[[Any], bool] | None = None,
|
|
66
|
+
selector: Callable[[tuple[Any]], Any] = ...,
|
|
62
67
|
fail_on_empty: bool = False,
|
|
63
68
|
hook_params: Mapping[str, Any] | None = None,
|
|
69
|
+
deferrable: bool = ...,
|
|
64
70
|
**kwargs,
|
|
65
71
|
) -> None: ...
|
|
66
72
|
def poke(self, context: Context) -> bool: ...
|
|
73
|
+
def execute(self, context: Context) -> Any: ...
|
|
74
|
+
def execute_complete(self, context: Context, event: dict[str, Any]) -> None: ...
|