apache-airflow-providers-common-sql 2.4.0__tar.gz → 3.0.0rc1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/PKG-INFO +8 -8
  2. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/README.rst +3 -3
  3. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/docs/changelog.rst +32 -0
  4. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/docs/index.rst +3 -3
  5. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/docs/operators.rst +47 -0
  6. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/provider.yaml +2 -1
  7. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/pyproject.toml +6 -6
  8. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/__init__.py +1 -1
  9. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/decorators/analytics.py +3 -1
  10. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/decorators/sql.py +3 -1
  11. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/hooks/sql.py +6 -38
  12. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/hooks/sql.pyi +0 -6
  13. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/operators/sql.py +38 -17
  14. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/sensors/sql.py +82 -4
  15. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/sensors/sql.pyi +9 -1
  16. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/triggers/sql.py +80 -5
  17. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/triggers/sql.pyi +24 -2
  18. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/version_compat.py +2 -0
  19. apache_airflow_providers_common_sql-3.0.0rc1/tests/system/common/sql/example_sql_sensor.py +50 -0
  20. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/decorators/test_analytics.py +20 -0
  21. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/decorators/test_sql.py +19 -0
  22. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/hooks/test_sql.py +33 -0
  23. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/operators/test_sql_execute.py +89 -0
  24. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/sensors/test_sql.py +198 -2
  25. apache_airflow_providers_common_sql-3.0.0rc1/tests/unit/common/sql/triggers/test_sql.py +146 -0
  26. apache_airflow_providers_common_sql-2.4.0/tests/unit/common/sql/triggers/test_sql.py +0 -54
  27. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/.gitignore +0 -0
  28. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/LICENSE +0 -0
  29. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/NOTICE +0 -0
  30. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/docs/.latest-doc-only-change.txt +0 -0
  31. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/docs/commits.rst +0 -0
  32. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/docs/conf.py +0 -0
  33. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/docs/connections.rst +0 -0
  34. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/docs/dataframes.rst +0 -0
  35. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/docs/dialects.rst +0 -0
  36. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/docs/installing-providers-from-sources.rst +0 -0
  37. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/docs/integration-logos/sql.png +0 -0
  38. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/docs/security.rst +0 -0
  39. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/docs/supported-database-types.rst +0 -0
  40. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/__init__.py +0 -0
  41. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/__init__.py +0 -0
  42. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/__init__.py +0 -0
  43. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/README_API.md +0 -0
  44. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/config.py +0 -0
  45. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/datafusion/__init__.py +0 -0
  46. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/datafusion/base.py +0 -0
  47. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/datafusion/engine.py +0 -0
  48. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/datafusion/exceptions.py +0 -0
  49. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/datafusion/format_handlers.py +0 -0
  50. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/datafusion/object_storage_provider.py +0 -0
  51. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/decorators/__init__.py +0 -0
  52. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/dialects/__init__.py +0 -0
  53. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/dialects/dialect.py +0 -0
  54. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/dialects/dialect.pyi +0 -0
  55. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/doc/adr/0001-record-architecture-decisions.md +0 -0
  56. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/doc/adr/0002-return-common-data-structure-from-dbapihook-derived-hooks.md +0 -0
  57. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/doc/adr/0003-introduce-notion-of-dialects-in-dbapihook.md +0 -0
  58. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/example_dags/__init__.py +0 -0
  59. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/example_dags/example_analytics.py +0 -0
  60. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/get_provider_info.py +0 -0
  61. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/get_provider_info.pyi +0 -0
  62. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/hooks/__init__.py +0 -0
  63. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/hooks/handlers.py +0 -0
  64. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/hooks/handlers.pyi +0 -0
  65. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/hooks/lineage.py +0 -0
  66. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/hooks/lineage.pyi +0 -0
  67. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/operators/__init__.py +0 -0
  68. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/operators/analytics.py +0 -0
  69. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/operators/generic_transfer.py +0 -0
  70. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/operators/generic_transfer.pyi +0 -0
  71. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/sensors/__init__.py +0 -0
  72. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/src/airflow/providers/common/sql/triggers/__init__.py +0 -0
  73. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/conftest.py +0 -0
  74. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/system/__init__.py +0 -0
  75. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/system/common/__init__.py +0 -0
  76. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/system/common/sql/__init__.py +0 -0
  77. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/system/common/sql/example_generic_transfer.py +0 -0
  78. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/system/common/sql/example_sql_bulk_load.py +0 -0
  79. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/system/common/sql/example_sql_column_table_check.py +0 -0
  80. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/system/common/sql/example_sql_execute_query.py +0 -0
  81. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/system/common/sql/example_sql_insert_rows.py +0 -0
  82. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/system/common/sql/example_sql_threshold_check.py +0 -0
  83. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/system/common/sql/example_sql_value_check.py +0 -0
  84. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/__init__.py +0 -0
  85. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/__init__.py +0 -0
  86. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/__init__.py +0 -0
  87. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/datafusion/__init__.py +0 -0
  88. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/datafusion/test_base.py +0 -0
  89. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/datafusion/test_engine.py +0 -0
  90. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/datafusion/test_exceptions.py +0 -0
  91. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/datafusion/test_format_handlers.py +0 -0
  92. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/datafusion/test_object_storage_provider.py +0 -0
  93. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/decorators/__init__.py +0 -0
  94. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/dialects/__init__.py +0 -0
  95. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/dialects/test_dialect.py +0 -0
  96. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/hooks/__init__.py +0 -0
  97. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/hooks/test_dbapi.py +0 -0
  98. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/hooks/test_handlers.py +0 -0
  99. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/hooks/test_lineage.py +0 -0
  100. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/hooks/test_sqlparse.py +0 -0
  101. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/operators/__init__.py +0 -0
  102. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/operators/test_analytics.py +0 -0
  103. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/operators/test_generic_transfer.py +0 -0
  104. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/operators/test_sql.py +0 -0
  105. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/sensors/__init__.py +0 -0
  106. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/test_config.py +0 -0
  107. {apache_airflow_providers_common_sql-2.4.0 → apache_airflow_providers_common_sql-3.0.0rc1}/tests/unit/common/sql/triggers/__init__.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: apache-airflow-providers-common-sql
3
- Version: 2.4.0
3
+ Version: 3.0.0rc1
4
4
  Summary: Provider package apache-airflow-providers-common-sql for Apache Airflow
5
5
  Keywords: airflow-provider,common.sql,airflow,integration
6
6
  Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -22,8 +22,8 @@ Classifier: Programming Language :: Python :: 3.14
22
22
  Classifier: Topic :: System :: Monitoring
23
23
  License-File: LICENSE
24
24
  License-File: NOTICE
25
- Requires-Dist: apache-airflow>=2.11.0
26
- Requires-Dist: apache-airflow-providers-common-compat>=1.23.0
25
+ Requires-Dist: apache-airflow>=2.11.0rc1
26
+ Requires-Dist: apache-airflow-providers-common-compat>=1.23.0rc1
27
27
  Requires-Dist: sqlparse>=0.5.1
28
28
  Requires-Dist: more-itertools>=9.0.0
29
29
  Requires-Dist: methodtools>=0.4.7
@@ -38,8 +38,8 @@ Requires-Dist: polars>=1.26.0 ; extra == "polars"
38
38
  Requires-Dist: pyiceberg-core>=0.8.0 ; extra == "pyiceberg-core"
39
39
  Requires-Dist: sqlalchemy>=1.4.54,!=2.1.0 ; extra == "sqlalchemy"
40
40
  Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
41
- Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.4.0/changelog.html
42
- Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.4.0
41
+ Project-URL: Changelog, https://airflow.staged.apache.org/docs/apache-airflow-providers-common-sql/3.0.0/changelog.html
42
+ Project-URL: Documentation, https://airflow.staged.apache.org/docs/apache-airflow-providers-common-sql/3.0.0
43
43
  Project-URL: Mastodon, https://fosstodon.org/@airflow
44
44
  Project-URL: Slack Chat, https://s.apache.org/airflow-slack
45
45
  Project-URL: Source Code, https://github.com/apache/airflow
@@ -82,7 +82,7 @@ Import-Namespace: airflow.providers.common
82
82
 
83
83
  Package ``apache-airflow-providers-common-sql``
84
84
 
85
- Release: ``2.4.0``
85
+ Release: ``3.0.0``
86
86
 
87
87
 
88
88
  `Common SQL Provider <https://en.wikipedia.org/wiki/SQL>`__
@@ -95,7 +95,7 @@ This is a provider package for ``common.sql`` provider. All classes for this pro
95
95
  are in ``airflow.providers.common.sql`` python package.
96
96
 
97
97
  You can find package information and changelog for the provider
98
- in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.4.0/>`_.
98
+ in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/3.0.0/>`_.
99
99
 
100
100
  Installation
101
101
  ------------
@@ -157,5 +157,5 @@ Extra Dependencies
157
157
  ================== =======================================================================================================================================================================
158
158
 
159
159
  The changelog for the provider package can be found in the
160
- `changelog <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.4.0/changelog.html>`_.
160
+ `changelog <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/3.0.0/changelog.html>`_.
161
161
 
@@ -23,7 +23,7 @@
23
23
 
24
24
  Package ``apache-airflow-providers-common-sql``
25
25
 
26
- Release: ``2.4.0``
26
+ Release: ``3.0.0``
27
27
 
28
28
 
29
29
  `Common SQL Provider <https://en.wikipedia.org/wiki/SQL>`__
@@ -36,7 +36,7 @@ This is a provider package for ``common.sql`` provider. All classes for this pro
36
36
  are in ``airflow.providers.common.sql`` python package.
37
37
 
38
38
  You can find package information and changelog for the provider
39
- in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.4.0/>`_.
39
+ in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/3.0.0/>`_.
40
40
 
41
41
  Installation
42
42
  ------------
@@ -98,4 +98,4 @@ Extra Dependencies
98
98
  ================== =======================================================================================================================================================================
99
99
 
100
100
  The changelog for the provider package can be found in the
101
- `changelog <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.4.0/changelog.html>`_.
101
+ `changelog <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/3.0.0/changelog.html>`_.
@@ -25,6 +25,38 @@
25
25
  Changelog
26
26
  ---------
27
27
 
28
+ 3.0.0
29
+ .....
30
+
31
+ .. note::
32
+ ``@task.sql`` and ``@task.analytics`` now render only the SQL returned by the decorated function
33
+ inside ``execute()``. Their other template fields (``parameters``, ``op_args``, ``op_kwargs``, ...)
34
+ are rendered once before ``execute()`` and are no longer rendered a second time.
35
+
36
+ .. warning::
37
+ The deprecated ``DbApiHook.get_pandas_df`` and ``DbApiHook.get_pandas_df_by_chunks`` methods have been removed. Use ``DbApiHook.get_df(..., df_type="pandas")`` and ``DbApiHook.get_df_by_chunks(..., df_type="pandas")`` instead.
38
+
39
+ Breaking changes
40
+ ~~~~~~~~~~~~~~~~
41
+
42
+ * ``Remove deprecated get_pandas_df methods from SQL hooks (#76130)``
43
+
44
+ Features
45
+ ~~~~~~~~
46
+
47
+ * ``Add deferrable mode to 'SqlSensor' (#76270)``
48
+
49
+ Bug Fixes
50
+ ~~~~~~~~~
51
+
52
+ * ``Render only the returned value in TaskFlow decorators that template it (#76244)``
53
+ * ``Surface errors from OpenLineage database info lookups without losing lineage (#74805)``
54
+ * ``Fix ''DbApiHook.inspector'' keeping a database connection open until process exit (#75327)``
55
+
56
+ .. Below changes are excluded from the changelog. Move them to
57
+ appropriate section above if needed. Do not delete the lines(!):
58
+ * ``[main] Upgrade important CI environment (#76278)``
59
+
28
60
  2.4.0
29
61
  .....
30
62
 
@@ -79,7 +79,7 @@ apache-airflow-providers-common-sql package
79
79
  `Common SQL Provider <https://en.wikipedia.org/wiki/SQL>`__
80
80
 
81
81
 
82
- Release: 2.4.0
82
+ Release: 3.0.0
83
83
 
84
84
  Provider package
85
85
  ----------------
@@ -160,5 +160,5 @@ Downloading official packages
160
160
  You can download officially released packages and verify their checksums and signatures from the
161
161
  `Official Apache Download site <https://downloads.apache.org/airflow/providers/>`_
162
162
 
163
- * `The apache-airflow-providers-common-sql 2.4.0 sdist package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-2.4.0.tar.gz>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-2.4.0.tar.gz.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-2.4.0.tar.gz.sha512>`__)
164
- * `The apache-airflow-providers-common-sql 2.4.0 wheel package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-2.4.0-py3-none-any.whl>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-2.4.0-py3-none-any.whl.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-2.4.0-py3-none-any.whl.sha512>`__)
163
+ * `The apache-airflow-providers-common-sql 3.0.0 sdist package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-3.0.0.tar.gz>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-3.0.0.tar.gz.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-3.0.0.tar.gz.sha512>`__)
164
+ * `The apache-airflow-providers-common-sql 3.0.0 wheel package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-3.0.0-py3-none-any.whl>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-3.0.0-py3-none-any.whl.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-3.0.0-py3-none-any.whl.sha512>`__)
@@ -282,6 +282,53 @@ between two connections.
282
282
  :start-after: [START howto_operator_generic_transfer]
283
283
  :end-before: [END howto_operator_generic_transfer]
284
284
 
285
+ .. _howto/sensor:SqlSensor:
286
+
287
+ Wait for a SQL condition
288
+ ~~~~~~~~~~~~~~~~~~~~~~~~
289
+
290
+ Use the :class:`~airflow.providers.common.sql.sensors.sql.SqlSensor` to wait until a query returns a row
291
+ whose first cell is truthy, for example until today's orders have landed. It works with any connection
292
+ whose hook is a :class:`~airflow.providers.common.sql.hooks.sql.DbApiHook`. The examples below use
293
+ PostgreSQL; write ``sql`` and ``parameters`` in your database's dialect and placeholder style.
294
+
295
+ .. exampleinclude:: /../tests/system/common/sql/example_sql_sensor.py
296
+ :language: python
297
+ :dedent: 4
298
+ :start-after: [START howto_sensor_sql]
299
+ :end-before: [END howto_sensor_sql]
300
+
301
+ For a condition you cannot write in SQL, pass ``success`` or ``failure`` callables. They receive the first
302
+ cell of the first row, or whatever ``selector`` returns for that row. ``fail_on_empty=True`` fails the task
303
+ when the query returns no rows instead of checking again.
304
+
305
+ Deferrable mode
306
+ ---------------
307
+
308
+ With ``deferrable=True``, the sensor checks once on the worker. If the condition is not met, it defers and
309
+ the triggerer runs the query every ``poke_interval`` seconds, so the task holds no worker slot while it
310
+ waits. ``timeout`` caps the time spent deferred.
311
+
312
+ .. exampleinclude:: /../tests/system/common/sql/example_sql_sensor.py
313
+ :language: python
314
+ :dedent: 4
315
+ :start-after: [START howto_sensor_sql_deferrable]
316
+ :end-before: [END howto_sensor_sql_deferrable]
317
+
318
+ The triggerer only checks the default condition, that the first cell of the first row is truthy, so write
319
+ the condition in SQL as above. The sensor logs a warning and runs in its usual ``mode`` instead of
320
+ deferring when:
321
+
322
+ - ``success``, ``failure`` or ``selector`` is set, because these Python callables only run on a worker;
323
+ - ``silent_fail``, ``never_fail`` or ``exponential_backoff`` is set, because they only apply to pokes on a
324
+ worker;
325
+ - a subclass overrides ``poke`` or ``_get_hook``, because the triggerer would not run that code;
326
+ - it runs on Airflow 3 before 3.2.2, whose triggerer cannot safely look up connections from the thread
327
+ that runs the query. Deferring works on Airflow 2.11 and on 3.2.2 and later;
328
+ - on Airflow 2, ``parameters`` holds values other than strings, numbers, booleans and ``None``.
329
+ Airflow 2 stores trigger arguments as JSON, so dates, datetimes and ``Decimal`` values would reach
330
+ the triggerer as strings.
331
+
285
332
  Analytics Operator
286
333
  ~~~~~~~~~~~~~~~~~~
287
334
 
@@ -23,12 +23,13 @@ description: |
23
23
 
24
24
  state: ready
25
25
  lifecycle: production
26
- source-date-epoch: 1792489528
26
+ source-date-epoch: 1793816745
27
27
  # Note that those versions are maintained by release manager - do not update them manually
28
28
  # with the exception of case where other provider in sources has >= new provider version.
29
29
  # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have
30
30
  # to be done in the same PR
31
31
  versions:
32
+ - 3.0.0
32
33
  - 2.4.0
33
34
  - 2.3.0
34
35
  - 2.2.0
@@ -20,12 +20,12 @@
20
20
  # IF YOU WANT TO MODIFY THIS FILE EXCEPT DEPENDENCIES, YOU SHOULD MODIFY THE TEMPLATE
21
21
  # `pyproject_TEMPLATE.toml.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
22
22
  [build-system]
23
- requires = ["flit_core==4.1.0"]
23
+ requires = ["flit_core==4.2.0"]
24
24
  build-backend = "flit_core.buildapi"
25
25
 
26
26
  [project]
27
27
  name = "apache-airflow-providers-common-sql"
28
- version = "2.4.0"
28
+ version = "3.0.0rc1"
29
29
  description = "Provider package apache-airflow-providers-common-sql for Apache Airflow"
30
30
  readme = "README.rst"
31
31
  license = "Apache-2.0"
@@ -58,8 +58,8 @@ requires-python = ">=3.11"
58
58
  # Make sure to run ``prek update-providers-dependencies --all-files``
59
59
  # After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build``
60
60
  dependencies = [
61
- "apache-airflow>=2.11.0",
62
- "apache-airflow-providers-common-compat>=1.23.0",
61
+ "apache-airflow>=2.11.0rc1",
62
+ "apache-airflow-providers-common-compat>=1.23.0rc1",
63
63
  "sqlparse>=0.5.1",
64
64
  "more-itertools>=9.0.0",
65
65
  # The methodtools dependency is necessary since the introduction of dialects:
@@ -147,8 +147,8 @@ apache-airflow-providers-common-sql = {workspace = true}
147
147
  apache-airflow-providers-standard = {workspace = true}
148
148
 
149
149
  [project.urls]
150
- "Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.4.0"
151
- "Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.4.0/changelog.html"
150
+ "Documentation" = "https://airflow.staged.apache.org/docs/apache-airflow-providers-common-sql/3.0.0"
151
+ "Changelog" = "https://airflow.staged.apache.org/docs/apache-airflow-providers-common-sql/3.0.0/changelog.html"
152
152
  "Bug Tracker" = "https://github.com/apache/airflow/issues"
153
153
  "Source Code" = "https://github.com/apache/airflow"
154
154
  "Slack Chat" = "https://s.apache.org/airflow-slack"
@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
29
29
 
30
30
  __all__ = ["__version__"]
31
31
 
32
- __version__ = "2.4.0"
32
+ __version__ = "3.0.0"
33
33
 
34
34
  if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
35
35
  "2.11.0"
@@ -117,7 +117,9 @@ class _AnalyticsDecoratedOperator(DecoratedOperator, AnalyticsOperator):
117
117
  # AnalyticsOperator expects queries as a list of strings
118
118
  self.queries = [result] if isinstance(result, str) else result
119
119
 
120
- self.render_template_fields(context)
120
+ # Render only the queries the callable returned. The other template fields were already
121
+ # rendered before execute().
122
+ self.queries = self.render_template(self.queries, context)
121
123
 
122
124
  return AnalyticsOperator.execute(self, context)
123
125
 
@@ -110,7 +110,9 @@ class _SQLDecoratedOperator(DecoratedOperator, SQLExecuteQueryOperator):
110
110
  "string or a non-empty list of non-empty strings."
111
111
  )
112
112
 
113
- self.render_template_fields(context)
113
+ # Render only the SQL the callable returned. The other template fields were already
114
+ # rendered before execute().
115
+ self.sql = self.render_template(self.sql, context)
114
116
 
115
117
  return super().execute(context)
116
118
 
@@ -27,7 +27,6 @@ from typing import TYPE_CHECKING, Any, Literal, Protocol, TypeVar, cast, overloa
27
27
  from urllib.parse import urlparse
28
28
 
29
29
  import sqlparse
30
- from deprecated import deprecated
31
30
  from methodtools import lru_cache
32
31
  from more_itertools import chunked
33
32
 
@@ -35,15 +34,16 @@ try:
35
34
  from sqlalchemy import create_engine, inspect
36
35
  from sqlalchemy.engine import make_url
37
36
  from sqlalchemy.exc import ArgumentError, NoSuchModuleError
37
+ from sqlalchemy.pool import NullPool
38
38
  except ImportError:
39
39
  create_engine = None # type: ignore[assignment]
40
40
  inspect = None # type: ignore[assignment]
41
41
  make_url = None # type: ignore[assignment]
42
+ NullPool = None # type: ignore[assignment,misc]
42
43
  ArgumentError = Exception # type: ignore[misc,assignment]
43
44
  NoSuchModuleError = Exception # type: ignore[misc,assignment]
44
45
 
45
46
 
46
- from airflow.exceptions import AirflowProviderDeprecationWarning
47
47
  from airflow.providers.common.compat.module_loading import import_string
48
48
  from airflow.providers.common.compat.sdk import (
49
49
  AirflowException,
@@ -378,7 +378,10 @@ class DbApiHook(BaseHook):
378
378
  "SQLAlchemy is required for database inspection. "
379
379
  "Install it with: pip install 'apache-airflow-providers-common-sql[sqlalchemy]'"
380
380
  )
381
- return inspect(self.get_sqlalchemy_engine())
381
+ # The inspector is cached for the hook's lifetime, so a pooled engine would keep the
382
+ # connection it reflected with open until the process exits. With NullPool, no
383
+ # connection outlives the inspection that opened it.
384
+ return inspect(self.get_sqlalchemy_engine({"poolclass": NullPool}))
382
385
 
383
386
  def get_table_schema(self, table_name: str, schema: str | None = None) -> list[dict[str, str]]:
384
387
  """
@@ -445,41 +448,6 @@ class DbApiHook(BaseHook):
445
448
  self.log.debug("reserved words for '%s': %s", dialect_name, result)
446
449
  return result
447
450
 
448
- @deprecated(
449
- reason="Replaced by function `get_df`.",
450
- category=AirflowProviderDeprecationWarning,
451
- action="ignore",
452
- )
453
- def get_pandas_df(
454
- self,
455
- sql,
456
- parameters: list | tuple | Mapping[str, Any] | None = None,
457
- **kwargs,
458
- ) -> PandasDataFrame:
459
- """
460
- Execute the sql and returns a pandas dataframe.
461
-
462
- :param sql: the sql statement to be executed (str) or a list of sql statements to execute
463
- :param parameters: The parameters to render the SQL query with.
464
- :param kwargs: (optional) passed into pandas.io.sql.read_sql method
465
- """
466
- return self.get_df(sql, parameters, df_type="pandas", **kwargs)
467
-
468
- @deprecated(
469
- reason="Replaced by function `get_df_by_chunks`.",
470
- category=AirflowProviderDeprecationWarning,
471
- action="ignore",
472
- )
473
- def get_pandas_df_by_chunks(
474
- self,
475
- sql,
476
- parameters: list | tuple | Mapping[str, Any] | None = None,
477
- *,
478
- chunksize: int,
479
- **kwargs,
480
- ) -> Generator[PandasDataFrame, None, None]:
481
- return self.get_df_by_chunks(sql, parameters, chunksize=chunksize, df_type="pandas", **kwargs)
482
-
483
451
  @overload
484
452
  def get_df(
485
453
  self,
@@ -104,12 +104,6 @@ class DbApiHook(BaseHook):
104
104
  @property
105
105
  def reserved_words(self) -> set[str]: ...
106
106
  def get_reserved_words(self, dialect_name: str) -> set[str]: ...
107
- def get_pandas_df(
108
- self, sql, parameters: list | tuple | Mapping[str, Any] | None = None, **kwargs
109
- ) -> PandasDataFrame: ...
110
- def get_pandas_df_by_chunks(
111
- self, sql, parameters: list | tuple | Mapping[str, Any] | None = None, *, chunksize: int, **kwargs
112
- ) -> Generator[PandasDataFrame]: ...
113
107
  def get_records(
114
108
  self, sql: str | list[str], parameters: Iterable | Mapping[str, Any] | None = None
115
109
  ) -> Any: ...
@@ -296,32 +296,47 @@ class BaseSQLOperator(BaseOperator):
296
296
  # OpenLineage provider release < 1.8.0 - we always use connection
297
297
  use_external_connection = True
298
298
 
299
- connection = hook.get_connection(getattr(hook, hook.conn_name_attr))
300
- try:
301
- database_info = hook.get_openlineage_database_info(connection)
302
- except AttributeError:
303
- self.log.debug("%s has no database info provided", hook)
304
- database_info = None
299
+ conn_id = getattr(hook, hook.conn_name_attr)
300
+ connection = hook.get_connection(conn_id)
301
+ get_database_info = getattr(hook, "get_openlineage_database_info", None)
302
+ database_info = None
303
+ if get_database_info is not None:
304
+ try:
305
+ database_info = get_database_info(connection)
306
+ except Exception as e:
307
+ self.log.warning("Failed to get OpenLineage database info for %s: %s", conn_id, e)
308
+ self.log.debug("OpenLineage failure details:", exc_info=True)
305
309
 
306
310
  if database_info is None:
307
311
  self.log.debug("OpenLineage could not retrieve database information. Skipping.")
308
312
  return OperatorLineage()
309
313
 
314
+ get_dialect = getattr(hook, "get_openlineage_database_dialect", None)
315
+ get_default_schema = getattr(hook, "get_openlineage_default_schema", None)
316
+ if get_dialect is None or get_default_schema is None:
317
+ self.log.debug("%s has no database dialect or default schema provided", conn_id)
318
+ return None
310
319
  try:
311
- sql_parser = SQLParser(
312
- dialect=hook.get_openlineage_database_dialect(connection),
313
- default_schema=hook.get_openlineage_default_schema(),
320
+ sql_parser = SQLParser(dialect=get_dialect(connection), default_schema=get_default_schema())
321
+ except Exception as e:
322
+ self.log.warning(
323
+ "Failed to get OpenLineage database dialect or default schema for %s: %s", conn_id, e
314
324
  )
315
- except AttributeError:
316
- self.log.debug("%s failed to get database dialect", hook)
325
+ self.log.debug("OpenLineage failure details:", exc_info=True)
317
326
  return None
318
327
 
328
+ sqlalchemy_engine = None
329
+ try:
330
+ sqlalchemy_engine = hook.get_sqlalchemy_engine()
331
+ except Exception as e:
332
+ self.log.debug("Failed to get sql alchemy engine: %s", e)
333
+
319
334
  operator_lineage = sql_parser.generate_openlineage_metadata_from_sql(
320
335
  sql=sql,
321
336
  hook=hook,
322
337
  database_info=database_info,
323
338
  database=self.database,
324
- sqlalchemy_engine=hook.get_sqlalchemy_engine(),
339
+ sqlalchemy_engine=sqlalchemy_engine,
325
340
  use_connection=use_external_connection,
326
341
  )
327
342
 
@@ -337,11 +352,17 @@ class BaseSQLOperator(BaseOperator):
337
352
 
338
353
  operator_lineage = self.get_openlineage_facets_on_start() or OperatorLineage()
339
354
  hook = self.get_db_hook()
340
- try:
341
- database_specific_lineage = hook.get_openlineage_database_specific_lineage(task_instance)
342
- except AttributeError:
343
- self.log.debug("%s has no database specific lineage provided", hook)
344
- database_specific_lineage = None
355
+ conn_id = getattr(hook, hook.conn_name_attr)
356
+ get_specific_lineage = getattr(hook, "get_openlineage_database_specific_lineage", None)
357
+ database_specific_lineage = None
358
+ if get_specific_lineage is None:
359
+ self.log.debug("%s has no database specific lineage provided", conn_id)
360
+ else:
361
+ try:
362
+ database_specific_lineage = get_specific_lineage(task_instance)
363
+ except Exception as e:
364
+ self.log.warning("Failed to get OpenLineage database specific lineage for %s: %s", conn_id, e)
365
+ self.log.debug("OpenLineage failure details:", exc_info=True)
345
366
 
346
367
  if database_specific_lineage is None:
347
368
  if not self.check_results:
@@ -17,15 +17,41 @@
17
17
  from __future__ import annotations
18
18
 
19
19
  from collections.abc import Callable, Mapping, Sequence
20
+ from datetime import timedelta
20
21
  from operator import itemgetter
21
22
  from typing import TYPE_CHECKING, Any
22
23
 
23
- from airflow.providers.common.compat.sdk import AirflowException, BaseHook, BaseSensorOperator
24
+ from airflow.providers.common.compat.sdk import AirflowException, BaseHook, BaseSensorOperator, conf
24
25
  from airflow.providers.common.sql.hooks.sql import DbApiHook
26
+ from airflow.providers.common.sql.triggers.sql import NO_ROWS_MESSAGE, SqlSensorTrigger
27
+ from airflow.providers.common.sql.version_compat import AIRFLOW_V_3_0_PLUS, AIRFLOW_V_3_2_2_PLUS
25
28
 
26
29
  if TYPE_CHECKING:
27
30
  from airflow.providers.common.compat.sdk import Context
28
31
 
32
+ _first_cell = itemgetter(0)
33
+
34
+ # Values that survive Airflow 2's JSON storage of trigger arguments unchanged.
35
+ _JSON_SCALARS = (str, int, float, bool, type(None))
36
+
37
+
38
+ def _is_first_cell(selector: Callable[[tuple[Any]], Any]) -> bool:
39
+ # Compare by value rather than identity: a deep copy of the sensor, or an explicit
40
+ # ``selector=itemgetter(0)``, must still count as the default condition.
41
+ return isinstance(selector, itemgetter) and selector.__reduce__() == _first_cell.__reduce__()
42
+
43
+
44
+ def _survives_json(value: Any) -> bool:
45
+ if isinstance(value, Mapping):
46
+ return all(isinstance(k, str) and _survives_json(v) for k, v in value.items())
47
+ if isinstance(value, (list, tuple)):
48
+ return all(_survives_json(v) for v in value)
49
+ return isinstance(value, _JSON_SCALARS)
50
+
51
+
52
+ class SqlSensorTriggerError(AirflowException):
53
+ """Raised when the deferred ``SqlSensor`` trigger reports a failure."""
54
+
29
55
 
30
56
  class SqlSensor(BaseSensorOperator):
31
57
  """
@@ -55,6 +81,12 @@ class SqlSensor(BaseSensorOperator):
55
81
  :param fail_on_empty: Explicitly fail on no rows returned.
56
82
  :param hook_params: Extra config params to be passed to the underlying hook.
57
83
  Should match the desired hook constructor params.
84
+ :param deferrable: Wait in the triggerer instead of on a worker. The triggerer only
85
+ checks the default condition (the first cell is truthy), so the sensor logs a
86
+ warning and runs in its usual ``mode`` instead when ``success``, ``failure``,
87
+ ``selector``, ``silent_fail``, ``never_fail`` or ``exponential_backoff`` is set,
88
+ when a subclass overrides ``poke`` or ``_get_hook``, on Airflow 3 before 3.2.2, or
89
+ on Airflow 2 when ``parameters`` holds values JSON cannot keep, such as dates.
58
90
  """
59
91
 
60
92
  template_fields: Sequence[str] = ("sql", "hook_params", "parameters", "conn_id")
@@ -69,9 +101,10 @@ class SqlSensor(BaseSensorOperator):
69
101
  parameters: Mapping[str, Any] | None = None,
70
102
  success: Callable[[Any], bool] | None = None,
71
103
  failure: Callable[[Any], bool] | None = None,
72
- selector: Callable[[tuple[Any]], Any] = itemgetter(0),
104
+ selector: Callable[[tuple[Any]], Any] = _first_cell,
73
105
  fail_on_empty: bool = False,
74
106
  hook_params: Mapping[str, Any] | None = None,
107
+ deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
75
108
  **kwargs,
76
109
  ) -> None:
77
110
  self.conn_id = conn_id
@@ -82,6 +115,7 @@ class SqlSensor(BaseSensorOperator):
82
115
  self.selector = selector
83
116
  self.fail_on_empty = fail_on_empty
84
117
  self.hook_params = hook_params
118
+ self.deferrable = deferrable
85
119
  super().__init__(**kwargs)
86
120
 
87
121
  def _get_hook(self) -> DbApiHook:
@@ -101,8 +135,7 @@ class SqlSensor(BaseSensorOperator):
101
135
  records = hook.get_records(self.sql, self.parameters)
102
136
  if not records:
103
137
  if self.fail_on_empty:
104
- message = "No rows returned, raising as per fail_on_empty flag"
105
- raise AirflowException(message)
138
+ raise AirflowException(NO_ROWS_MESSAGE)
106
139
  return False
107
140
 
108
141
  condition = self.selector(records[0])
@@ -121,3 +154,48 @@ class SqlSensor(BaseSensorOperator):
121
154
  message = f"self.success is present, but not callable -> {self.success}"
122
155
  raise AirflowException(message)
123
156
  return bool(condition)
157
+
158
+ def execute(self, context: Context) -> Any:
159
+ if not self.deferrable:
160
+ return super().execute(context)
161
+ if reason := self._reason_not_to_defer():
162
+ self.log.warning("Cannot defer because %s; running in %s mode instead.", reason, self.mode)
163
+ return super().execute(context)
164
+ # silent_fail and never_fail fall back above, so an error here propagates exactly as
165
+ # it would from the first poke of super().execute().
166
+ if self.poke(context):
167
+ self.log.info("Success criteria met. Exiting.")
168
+ return None
169
+ self.defer(
170
+ trigger=SqlSensorTrigger(
171
+ sql=self.sql,
172
+ conn_id=self.conn_id,
173
+ poke_interval=self.poke_interval,
174
+ parameters=self.parameters,
175
+ hook_params=dict(self.hook_params) if self.hook_params is not None else None,
176
+ fail_on_empty=self.fail_on_empty,
177
+ ),
178
+ method_name=self.execute_complete.__name__,
179
+ timeout=timedelta(seconds=self.timeout),
180
+ )
181
+
182
+ def _reason_not_to_defer(self) -> str | None:
183
+ if AIRFLOW_V_3_0_PLUS and not AIRFLOW_V_3_2_2_PLUS:
184
+ return "the triggerer before Airflow 3.2.2 cannot look up connections from a thread safely"
185
+ if self.success is not None or self.failure is not None or not _is_first_cell(self.selector):
186
+ return "success, failure and selector only run on a worker"
187
+ if type(self).poke is not SqlSensor.poke or type(self)._get_hook is not SqlSensor._get_hook:
188
+ return f"{type(self).__name__} overrides poke or _get_hook, which the triggerer does not run"
189
+ if self.silent_fail or self.never_fail or self.exponential_backoff:
190
+ return "silent_fail, never_fail and exponential_backoff only apply when poking on a worker"
191
+ if not AIRFLOW_V_3_0_PLUS and not _survives_json(self.parameters):
192
+ return (
193
+ "Airflow 2 stores trigger arguments as JSON, which would hand the triggerer "
194
+ "date, datetime or Decimal parameters as strings"
195
+ )
196
+ return None
197
+
198
+ def execute_complete(self, context: Context, event: dict[str, Any]) -> None:
199
+ if event["status"] == "failure":
200
+ raise SqlSensorTriggerError(event["message"])
201
+ self.log.info("Success criteria met. Exiting.")
@@ -37,9 +37,11 @@ from typing import Any
37
37
 
38
38
  from _typeshed import Incomplete as Incomplete
39
39
 
40
- from airflow.providers.common.compat.sdk import BaseSensorOperator
40
+ from airflow.providers.common.compat.sdk import AirflowException, BaseSensorOperator
41
41
  from airflow.sdk import Context
42
42
 
43
+ class SqlSensorTriggerError(AirflowException): ...
44
+
43
45
  class SqlSensor(BaseSensorOperator):
44
46
  template_fields: Sequence[str]
45
47
  template_ext: Sequence[str]
@@ -49,8 +51,10 @@ class SqlSensor(BaseSensorOperator):
49
51
  parameters: Incomplete
50
52
  success: Incomplete
51
53
  failure: Incomplete
54
+ selector: Incomplete
52
55
  fail_on_empty: Incomplete
53
56
  hook_params: Incomplete
57
+ deferrable: Incomplete
54
58
  def __init__(
55
59
  self,
56
60
  *,
@@ -59,8 +63,12 @@ class SqlSensor(BaseSensorOperator):
59
63
  parameters: Mapping[str, Any] | None = None,
60
64
  success: Callable[[Any], bool] | None = None,
61
65
  failure: Callable[[Any], bool] | None = None,
66
+ selector: Callable[[tuple[Any]], Any] = ...,
62
67
  fail_on_empty: bool = False,
63
68
  hook_params: Mapping[str, Any] | None = None,
69
+ deferrable: bool = ...,
64
70
  **kwargs,
65
71
  ) -> None: ...
66
72
  def poke(self, context: Context) -> bool: ...
73
+ def execute(self, context: Context) -> Any: ...
74
+ def execute_complete(self, context: Context, event: dict[str, Any]) -> None: ...