apache-airflow-providers-common-sql 2.0.3__tar.gz → 2.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/PKG-INFO +6 -6
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/README.rst +3 -3
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/docs/changelog.rst +25 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/docs/index.rst +3 -3
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/docs/operators.rst +29 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/provider.yaml +2 -1
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/pyproject.toml +3 -3
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/__init__.py +1 -1
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/operators/sql.py +104 -28
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/system/common/sql/example_generic_transfer.py +1 -1
- apache_airflow_providers_common_sql-2.1.0/tests/system/common/sql/example_sql_bulk_load.py +101 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/system/common/sql/example_sql_column_table_check.py +1 -1
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/operators/test_generic_transfer.py +1 -1
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/operators/test_sql.py +160 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/.gitignore +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/LICENSE +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/NOTICE +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/docs/.latest-doc-only-change.txt +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/docs/commits.rst +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/docs/conf.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/docs/connections.rst +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/docs/dataframes.rst +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/docs/dialects.rst +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/docs/installing-providers-from-sources.rst +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/docs/integration-logos/sql.png +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/docs/security.rst +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/docs/supported-database-types.rst +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/README_API.md +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/config.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/datafusion/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/datafusion/base.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/datafusion/engine.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/datafusion/exceptions.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/datafusion/format_handlers.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/datafusion/object_storage_provider.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/decorators/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/decorators/analytics.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/decorators/sql.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/dialects/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/dialects/dialect.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/dialects/dialect.pyi +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/doc/adr/0001-record-architecture-decisions.md +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/doc/adr/0002-return-common-data-structure-from-dbapihook-derived-hooks.md +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/doc/adr/0003-introduce-notion-of-dialects-in-dbapihook.md +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/example_dags/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/example_dags/example_analytics.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/get_provider_info.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/get_provider_info.pyi +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/hooks/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/hooks/handlers.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/hooks/handlers.pyi +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/hooks/lineage.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/hooks/lineage.pyi +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/hooks/sql.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/hooks/sql.pyi +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/operators/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/operators/analytics.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/operators/generic_transfer.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/operators/generic_transfer.pyi +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/sensors/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/sensors/sql.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/sensors/sql.pyi +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/triggers/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/triggers/sql.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/triggers/sql.pyi +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/version_compat.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/conftest.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/system/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/system/common/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/system/common/sql/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/system/common/sql/example_sql_execute_query.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/system/common/sql/example_sql_insert_rows.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/system/common/sql/example_sql_threshold_check.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/system/common/sql/example_sql_value_check.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/datafusion/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/datafusion/test_engine.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/datafusion/test_format_handlers.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/datafusion/test_object_storage_provider.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/decorators/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/decorators/test_analytics.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/decorators/test_sql.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/dialects/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/dialects/test_dialect.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/hooks/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/hooks/test_dbapi.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/hooks/test_handlers.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/hooks/test_lineage.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/hooks/test_sql.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/hooks/test_sqlparse.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/operators/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/operators/test_analytics.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/operators/test_sql_execute.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/sensors/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/sensors/test_sql.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/test_config.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/triggers/__init__.py +0 -0
- {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/triggers/test_sql.py +0 -0
{apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/PKG-INFO
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: apache-airflow-providers-common-sql
|
|
3
|
-
Version: 2.0
|
|
3
|
+
Version: 2.1.0
|
|
4
4
|
Summary: Provider package apache-airflow-providers-common-sql for Apache Airflow
|
|
5
5
|
Keywords: airflow-provider,common.sql,airflow,integration
|
|
6
6
|
Author-email: Apache Software Foundation <dev@airflow.apache.org>
|
|
@@ -39,8 +39,8 @@ Requires-Dist: polars>=1.26.0 ; extra == "polars"
|
|
|
39
39
|
Requires-Dist: pyiceberg-core>=0.8.0 ; extra == "pyiceberg-core"
|
|
40
40
|
Requires-Dist: sqlalchemy>=1.4.54 ; extra == "sqlalchemy"
|
|
41
41
|
Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
|
|
42
|
-
Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.0
|
|
43
|
-
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.0
|
|
42
|
+
Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.1.0/changelog.html
|
|
43
|
+
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.1.0
|
|
44
44
|
Project-URL: Mastodon, https://fosstodon.org/@airflow
|
|
45
45
|
Project-URL: Slack Chat, https://s.apache.org/airflow-slack
|
|
46
46
|
Project-URL: Source Code, https://github.com/apache/airflow
|
|
@@ -79,7 +79,7 @@ Provides-Extra: sqlalchemy
|
|
|
79
79
|
|
|
80
80
|
Package ``apache-airflow-providers-common-sql``
|
|
81
81
|
|
|
82
|
-
Release: ``2.0
|
|
82
|
+
Release: ``2.1.0``
|
|
83
83
|
|
|
84
84
|
|
|
85
85
|
`Common SQL Provider <https://en.wikipedia.org/wiki/SQL>`__
|
|
@@ -92,7 +92,7 @@ This is a provider package for ``common.sql`` provider. All classes for this pro
|
|
|
92
92
|
are in ``airflow.providers.common.sql`` python package.
|
|
93
93
|
|
|
94
94
|
You can find package information and changelog for the provider
|
|
95
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.0
|
|
95
|
+
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.1.0/>`_.
|
|
96
96
|
|
|
97
97
|
Installation
|
|
98
98
|
------------
|
|
@@ -154,5 +154,5 @@ Extra Dependencies
|
|
|
154
154
|
================== =======================================================================================================================================================================
|
|
155
155
|
|
|
156
156
|
The changelog for the provider package can be found in the
|
|
157
|
-
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.0
|
|
157
|
+
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.1.0/changelog.html>`_.
|
|
158
158
|
|
{apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/README.rst
RENAMED
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
|
|
24
24
|
Package ``apache-airflow-providers-common-sql``
|
|
25
25
|
|
|
26
|
-
Release: ``2.0
|
|
26
|
+
Release: ``2.1.0``
|
|
27
27
|
|
|
28
28
|
|
|
29
29
|
`Common SQL Provider <https://en.wikipedia.org/wiki/SQL>`__
|
|
@@ -36,7 +36,7 @@ This is a provider package for ``common.sql`` provider. All classes for this pro
|
|
|
36
36
|
are in ``airflow.providers.common.sql`` python package.
|
|
37
37
|
|
|
38
38
|
You can find package information and changelog for the provider
|
|
39
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.0
|
|
39
|
+
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.1.0/>`_.
|
|
40
40
|
|
|
41
41
|
Installation
|
|
42
42
|
------------
|
|
@@ -98,4 +98,4 @@ Extra Dependencies
|
|
|
98
98
|
================== =======================================================================================================================================================================
|
|
99
99
|
|
|
100
100
|
The changelog for the provider package can be found in the
|
|
101
|
-
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.0
|
|
101
|
+
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.1.0/changelog.html>`_.
|
|
@@ -25,6 +25,31 @@
|
|
|
25
25
|
Changelog
|
|
26
26
|
---------
|
|
27
27
|
|
|
28
|
+
2.1.0
|
|
29
|
+
.....
|
|
30
|
+
|
|
31
|
+
Features
|
|
32
|
+
~~~~~~~~
|
|
33
|
+
|
|
34
|
+
* ``Add SQLBulkLoadOperator to Common SQL Provider (#69362)``
|
|
35
|
+
|
|
36
|
+
Bug Fixes
|
|
37
|
+
~~~~~~~~~
|
|
38
|
+
|
|
39
|
+
* ``Fix SQLColumnCheckOperator crash on non-numeric column bounds (#70895)``
|
|
40
|
+
* ``Fix SQL check tolerance for negative expected values (#69736)``
|
|
41
|
+
* ``Fix SQL value checks for negative expected values (#69893)``
|
|
42
|
+
|
|
43
|
+
Misc
|
|
44
|
+
~~~~
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
.. Below changes are excluded from the changelog. Move them to
|
|
48
|
+
appropriate section above if needed. Do not delete the lines(!):
|
|
49
|
+
* ``Use common.compat.sdk for timezone imports in providers (#70492)``
|
|
50
|
+
* ``Limit pandas to < 3 for DataFrame XComs (#70791)``
|
|
51
|
+
* ``Revert "Limit pandas to < 3 for DataFrame XComs (#70791)" (#71100)``
|
|
52
|
+
|
|
28
53
|
2.0.3
|
|
29
54
|
.....
|
|
30
55
|
|
|
@@ -79,7 +79,7 @@ apache-airflow-providers-common-sql package
|
|
|
79
79
|
`Common SQL Provider <https://en.wikipedia.org/wiki/SQL>`__
|
|
80
80
|
|
|
81
81
|
|
|
82
|
-
Release: 2.0
|
|
82
|
+
Release: 2.1.0
|
|
83
83
|
|
|
84
84
|
Provider package
|
|
85
85
|
----------------
|
|
@@ -160,5 +160,5 @@ Downloading official packages
|
|
|
160
160
|
You can download officially released packages and verify their checksums and signatures from the
|
|
161
161
|
`Official Apache Download site <https://downloads.apache.org/airflow/providers/>`_
|
|
162
162
|
|
|
163
|
-
* `The apache-airflow-providers-common-sql 2.0
|
|
164
|
-
* `The apache-airflow-providers-common-sql 2.0
|
|
163
|
+
* `The apache-airflow-providers-common-sql 2.1.0 sdist package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-2.1.0.tar.gz>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-2.1.0.tar.gz.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-2.1.0.tar.gz.sha512>`__)
|
|
164
|
+
* `The apache-airflow-providers-common-sql 2.1.0 wheel package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-2.1.0-py3-none-any.whl>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-2.1.0-py3-none-any.whl.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-2.1.0-py3-none-any.whl.sha512>`__)
|
|
@@ -239,6 +239,35 @@ The example below shows how to instantiate the SQLInsertRowsOperator task.
|
|
|
239
239
|
:start-after: [START howto_operator_sql_insert_rows]
|
|
240
240
|
:end-before: [END howto_operator_sql_insert_rows]
|
|
241
241
|
|
|
242
|
+
.. _howto/operator:SQLBulkLoadOperator:
|
|
243
|
+
|
|
244
|
+
Bulk load data into a table
|
|
245
|
+
~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
246
|
+
|
|
247
|
+
Use the :class:`~airflow.providers.common.sql.operators.sql.SQLBulkLoadOperator`
|
|
248
|
+
to bulk load a tab-delimited file into a database table using the database's
|
|
249
|
+
native bulk loading mechanism. Parameters of the operator are:
|
|
250
|
+
|
|
251
|
+
- ``table`` - name of the target table (templated).
|
|
252
|
+
- ``tmp_file`` - path to the tab-delimited file to load (templated).
|
|
253
|
+
- ``conn_id`` - the Airflow connection ID used to connect to the database.
|
|
254
|
+
- ``database`` (optional) - name of the database which overrides the one defined
|
|
255
|
+
in the connection.
|
|
256
|
+
- ``preoperator`` (optional) - SQL statement or list of statements to execute
|
|
257
|
+
before bulk loading (templated).
|
|
258
|
+
- ``postoperator`` (optional) - SQL statement or list of statements to execute
|
|
259
|
+
after bulk loading (templated).
|
|
260
|
+
- ``hook_params`` (optional) - dictionary of additional parameters passed to the
|
|
261
|
+
underlying hook.
|
|
262
|
+
|
|
263
|
+
The example below shows how to instantiate the SQLBulkLoadOperator task.
|
|
264
|
+
|
|
265
|
+
.. exampleinclude:: /../tests/system/common/sql/example_sql_bulk_load.py
|
|
266
|
+
:language: python
|
|
267
|
+
:dedent: 4
|
|
268
|
+
:start-after: [START howto_operator_sql_bulk_load]
|
|
269
|
+
:end-before: [END howto_operator_sql_bulk_load]
|
|
270
|
+
|
|
242
271
|
.. _howto/operator:GenericTransfer:
|
|
243
272
|
|
|
244
273
|
Generic Transfer
|
|
@@ -23,12 +23,13 @@ description: |
|
|
|
23
23
|
|
|
24
24
|
state: ready
|
|
25
25
|
lifecycle: production
|
|
26
|
-
source-date-epoch:
|
|
26
|
+
source-date-epoch: 1785633505
|
|
27
27
|
# Note that those versions are maintained by release manager - do not update them manually
|
|
28
28
|
# with the exception of case where other provider in sources has >= new provider version.
|
|
29
29
|
# In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have
|
|
30
30
|
# to be done in the same PR
|
|
31
31
|
versions:
|
|
32
|
+
- 2.1.0
|
|
32
33
|
- 2.0.3
|
|
33
34
|
- 2.0.2
|
|
34
35
|
- 2.0.1
|
|
@@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi"
|
|
|
25
25
|
|
|
26
26
|
[project]
|
|
27
27
|
name = "apache-airflow-providers-common-sql"
|
|
28
|
-
version = "2.0
|
|
28
|
+
version = "2.1.0"
|
|
29
29
|
description = "Provider package apache-airflow-providers-common-sql for Apache Airflow"
|
|
30
30
|
readme = "README.rst"
|
|
31
31
|
license = "Apache-2.0"
|
|
@@ -148,8 +148,8 @@ apache-airflow-providers-common-sql = {workspace = true}
|
|
|
148
148
|
apache-airflow-providers-standard = {workspace = true}
|
|
149
149
|
|
|
150
150
|
[project.urls]
|
|
151
|
-
"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.0
|
|
152
|
-
"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.0
|
|
151
|
+
"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.1.0"
|
|
152
|
+
"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.1.0/changelog.html"
|
|
153
153
|
"Bug Tracker" = "https://github.com/apache/airflow/issues"
|
|
154
154
|
"Source Code" = "https://github.com/apache/airflow"
|
|
155
155
|
"Slack Chat" = "https://s.apache.org/airflow-slack"
|
|
@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
|
|
|
29
29
|
|
|
30
30
|
__all__ = ["__version__"]
|
|
31
31
|
|
|
32
|
-
__version__ = "2.0
|
|
32
|
+
__version__ = "2.1.0"
|
|
33
33
|
|
|
34
34
|
if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
|
|
35
35
|
"2.11.0"
|
|
@@ -744,35 +744,32 @@ class SQLColumnCheckOperator(BaseSQLOperator):
|
|
|
744
744
|
if record is None and self.accept_none:
|
|
745
745
|
record = 0
|
|
746
746
|
match_boolean = True
|
|
747
|
+
|
|
748
|
+
# abs() so the tolerance margin widens the bound outward for negative expected values too.
|
|
749
|
+
# Without a tolerance the bound is compared as-is: a min/max check may be declared on a
|
|
750
|
+
# date or text column, where arithmetic on the bound raises TypeError.
|
|
751
|
+
def _lower(expected):
|
|
752
|
+
return expected - abs(expected) * tolerance if tolerance is not None else expected
|
|
753
|
+
|
|
754
|
+
def _upper(expected):
|
|
755
|
+
return expected + abs(expected) * tolerance if tolerance is not None else expected
|
|
756
|
+
|
|
747
757
|
if "geq_to" in check_values:
|
|
748
|
-
|
|
749
|
-
match_boolean = record >= check_values["geq_to"] * (1 - tolerance)
|
|
750
|
-
else:
|
|
751
|
-
match_boolean = record >= check_values["geq_to"]
|
|
758
|
+
match_boolean = record >= _lower(check_values["geq_to"])
|
|
752
759
|
elif "greater_than" in check_values:
|
|
753
|
-
|
|
754
|
-
match_boolean = record > check_values["greater_than"] * (1 - tolerance)
|
|
755
|
-
else:
|
|
756
|
-
match_boolean = record > check_values["greater_than"]
|
|
760
|
+
match_boolean = record > _lower(check_values["greater_than"])
|
|
757
761
|
if "leq_to" in check_values:
|
|
758
|
-
|
|
759
|
-
match_boolean = record <= check_values["leq_to"] * (1 + tolerance) and match_boolean
|
|
760
|
-
else:
|
|
761
|
-
match_boolean = record <= check_values["leq_to"] and match_boolean
|
|
762
|
+
match_boolean = record <= _upper(check_values["leq_to"]) and match_boolean
|
|
762
763
|
elif "less_than" in check_values:
|
|
763
|
-
|
|
764
|
-
match_boolean = record < check_values["less_than"] * (1 + tolerance) and match_boolean
|
|
765
|
-
else:
|
|
766
|
-
match_boolean = record < check_values["less_than"] and match_boolean
|
|
764
|
+
match_boolean = record < _upper(check_values["less_than"]) and match_boolean
|
|
767
765
|
if "equal_to" in check_values:
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
) and match_boolean
|
|
766
|
+
expected = check_values["equal_to"]
|
|
767
|
+
if tolerance is None:
|
|
768
|
+
# Equality, not a degenerate range: a NULL result must fail the check rather
|
|
769
|
+
# than raise on an ordering comparison against None.
|
|
770
|
+
match_boolean = record == expected and match_boolean
|
|
774
771
|
else:
|
|
775
|
-
match_boolean = record
|
|
772
|
+
match_boolean = (_lower(expected) <= record <= _upper(expected)) and match_boolean
|
|
776
773
|
return match_boolean
|
|
777
774
|
|
|
778
775
|
def _column_mapping_validation(self, check, check_values):
|
|
@@ -1279,12 +1276,14 @@ class SQLValueCheckOperator(BaseSQLOperator):
|
|
|
1279
1276
|
def _get_string_matches(self, records, pass_value_conv):
|
|
1280
1277
|
return [str(record) == pass_value_conv for record in records]
|
|
1281
1278
|
|
|
1279
|
+
def _get_tolerance_bounds(self, numeric_pass_value_conv):
|
|
1280
|
+
margin = abs(numeric_pass_value_conv) * self.tol
|
|
1281
|
+
return numeric_pass_value_conv - margin, numeric_pass_value_conv + margin
|
|
1282
|
+
|
|
1282
1283
|
def _get_numeric_matches(self, numeric_records, numeric_pass_value_conv):
|
|
1283
1284
|
if self.has_tolerance:
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
for record in numeric_records
|
|
1287
|
-
]
|
|
1285
|
+
lower_bound, upper_bound = self._get_tolerance_bounds(numeric_pass_value_conv)
|
|
1286
|
+
return [lower_bound <= record <= upper_bound for record in numeric_records]
|
|
1288
1287
|
|
|
1289
1288
|
return [record == numeric_pass_value_conv for record in numeric_records]
|
|
1290
1289
|
|
|
@@ -1308,7 +1307,8 @@ class SQLValueCheckOperator(BaseSQLOperator):
|
|
|
1308
1307
|
|
|
1309
1308
|
pass_value_conv = _convert_to_float_if_possible(self.pass_value)
|
|
1310
1309
|
if isinstance(pass_value_conv, float) and isinstance(self.tol, float):
|
|
1311
|
-
|
|
1310
|
+
lower_bound, upper_bound = self._get_tolerance_bounds(pass_value_conv)
|
|
1311
|
+
expected_str = f">= {lower_bound}, <= {upper_bound}"
|
|
1312
1312
|
check_type = "accepted_range"
|
|
1313
1313
|
|
|
1314
1314
|
return [
|
|
@@ -1916,3 +1916,79 @@ def _initialize_partition_clause(clause: str | None) -> str | None:
|
|
|
1916
1916
|
raise ValueError("Invalid partition_clause: semicolons (;) not allowed.")
|
|
1917
1917
|
|
|
1918
1918
|
return clause
|
|
1919
|
+
|
|
1920
|
+
|
|
1921
|
+
class SQLBulkLoadOperator(BaseSQLOperator):
|
|
1922
|
+
"""
|
|
1923
|
+
Bulk load a tab-delimited file into a database table.
|
|
1924
|
+
|
|
1925
|
+
This operator delegates to the underlying hook's ``bulk_load`` implementation,
|
|
1926
|
+
allowing each provider to use its native bulk loading mechanism.
|
|
1927
|
+
|
|
1928
|
+
.. seealso::
|
|
1929
|
+
For more information on how to use this operator, take a look at the guide:
|
|
1930
|
+
:ref:`howto/operator:SQLBulkLoadOperator`
|
|
1931
|
+
|
|
1932
|
+
:param table: Name of the target table (templated).
|
|
1933
|
+
:param tmp_file: Path to the tab-delimited file to load (templated).
|
|
1934
|
+
:param conn_id: The connection ID used to connect to the database. Defaults to ``None``.
|
|
1935
|
+
:param database: Name of the database which overrides the one defined in the connection.
|
|
1936
|
+
Defaults to ``None``.
|
|
1937
|
+
:param preoperator: SQL statement or list of statements to execute before bulk loading
|
|
1938
|
+
(templated). Defaults to ``None``.
|
|
1939
|
+
:param postoperator: SQL statement or list of statements to execute after bulk loading
|
|
1940
|
+
(templated). Defaults to ``None``.
|
|
1941
|
+
"""
|
|
1942
|
+
|
|
1943
|
+
template_fields: Sequence[str] = (
|
|
1944
|
+
"table",
|
|
1945
|
+
"tmp_file",
|
|
1946
|
+
"preoperator",
|
|
1947
|
+
"postoperator",
|
|
1948
|
+
*BaseSQLOperator.template_fields,
|
|
1949
|
+
)
|
|
1950
|
+
|
|
1951
|
+
def __init__(
|
|
1952
|
+
self,
|
|
1953
|
+
*,
|
|
1954
|
+
table: str,
|
|
1955
|
+
tmp_file: str,
|
|
1956
|
+
conn_id: str | None = None,
|
|
1957
|
+
database: str | None = None,
|
|
1958
|
+
preoperator: str | list[str] | None = None,
|
|
1959
|
+
postoperator: str | list[str] | None = None,
|
|
1960
|
+
**kwargs,
|
|
1961
|
+
) -> None:
|
|
1962
|
+
super().__init__(
|
|
1963
|
+
conn_id=conn_id,
|
|
1964
|
+
database=database,
|
|
1965
|
+
**kwargs,
|
|
1966
|
+
)
|
|
1967
|
+
self.table = table
|
|
1968
|
+
self.tmp_file = tmp_file
|
|
1969
|
+
self.preoperator = preoperator
|
|
1970
|
+
self.postoperator = postoperator
|
|
1971
|
+
|
|
1972
|
+
def execute(self, context: Context) -> None:
|
|
1973
|
+
hook = self.get_db_hook()
|
|
1974
|
+
|
|
1975
|
+
if self.preoperator:
|
|
1976
|
+
self.log.info("Executing preoperator.")
|
|
1977
|
+
hook.run(self.preoperator)
|
|
1978
|
+
|
|
1979
|
+
self.log.info(
|
|
1980
|
+
"Bulk loading '%s' into table '%s'.",
|
|
1981
|
+
self.tmp_file,
|
|
1982
|
+
self.table,
|
|
1983
|
+
)
|
|
1984
|
+
|
|
1985
|
+
hook.bulk_load(
|
|
1986
|
+
table=self.table,
|
|
1987
|
+
tmp_file=self.tmp_file,
|
|
1988
|
+
)
|
|
1989
|
+
|
|
1990
|
+
if self.postoperator:
|
|
1991
|
+
self.log.info("Executing postoperator.")
|
|
1992
|
+
hook.run(self.postoperator)
|
|
1993
|
+
|
|
1994
|
+
self.log.info("Bulk load completed.")
|
|
@@ -18,8 +18,8 @@
|
|
|
18
18
|
from __future__ import annotations
|
|
19
19
|
|
|
20
20
|
from airflow import DAG
|
|
21
|
+
from airflow.providers.common.compat.sdk import timezone
|
|
21
22
|
from airflow.providers.common.sql.operators.generic_transfer import GenericTransfer
|
|
22
|
-
from airflow.utils import timezone
|
|
23
23
|
|
|
24
24
|
connection_args = {
|
|
25
25
|
"conn_id": "airflow_db",
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
|
3
|
+
# or more contributor license agreements. See the NOTICE file
|
|
4
|
+
# distributed with this work for additional information
|
|
5
|
+
# regarding copyright ownership. The ASF licenses this file
|
|
6
|
+
# to you under the Apache License, Version 2.0 (the
|
|
7
|
+
# "License"); you may not use this file except in compliance
|
|
8
|
+
# with the License. You may obtain a copy of the License at
|
|
9
|
+
#
|
|
10
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
11
|
+
#
|
|
12
|
+
# Unless required by applicable law or agreed to in writing,
|
|
13
|
+
# software distributed under the License is distributed on an
|
|
14
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
15
|
+
# KIND, either express or implied. See the License for the
|
|
16
|
+
# specific language governing permissions and limitations
|
|
17
|
+
# under the License.
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
|
|
22
|
+
from airflow import DAG
|
|
23
|
+
from airflow.providers.common.sql.operators.sql import (
|
|
24
|
+
SQLBulkLoadOperator,
|
|
25
|
+
)
|
|
26
|
+
from airflow.providers.standard.operators.python import PythonOperator
|
|
27
|
+
from airflow.utils.timezone import datetime
|
|
28
|
+
|
|
29
|
+
TMP_FILE = "/tmp/actors.tsv"
|
|
30
|
+
|
|
31
|
+
connection_args = {
|
|
32
|
+
"conn_id": "airflow_db",
|
|
33
|
+
"conn_type": "Postgres",
|
|
34
|
+
"host": "postgres",
|
|
35
|
+
"schema": "postgres",
|
|
36
|
+
"login": "postgres",
|
|
37
|
+
"password": "postgres",
|
|
38
|
+
"port": 5432,
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def create_bulk_load_file():
|
|
43
|
+
Path(TMP_FILE).write_text(
|
|
44
|
+
"\n".join(
|
|
45
|
+
[
|
|
46
|
+
"Stallone\tSylvester\t78",
|
|
47
|
+
"Statham\tJason\t57",
|
|
48
|
+
"Li\tJet\t61",
|
|
49
|
+
"Lundgren\tDolph\t66",
|
|
50
|
+
"Norris\tChuck\t84",
|
|
51
|
+
]
|
|
52
|
+
),
|
|
53
|
+
encoding="utf-8",
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
with DAG(
|
|
58
|
+
"example_sql_bulk_load",
|
|
59
|
+
description="Example DAG for SQLBulkLoadOperator.",
|
|
60
|
+
default_args=connection_args,
|
|
61
|
+
start_date=datetime(2021, 1, 1),
|
|
62
|
+
schedule=None,
|
|
63
|
+
catchup=False,
|
|
64
|
+
) as dag:
|
|
65
|
+
"""
|
|
66
|
+
### Example SQL bulk load DAG
|
|
67
|
+
|
|
68
|
+
Runs the SQLBulkLoadOperator against the Airflow metadata DB.
|
|
69
|
+
"""
|
|
70
|
+
|
|
71
|
+
create_file = PythonOperator(
|
|
72
|
+
task_id="create_file",
|
|
73
|
+
python_callable=create_bulk_load_file,
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
# [START howto_operator_sql_bulk_load]
|
|
77
|
+
bulk_load = SQLBulkLoadOperator(
|
|
78
|
+
task_id="bulk_load",
|
|
79
|
+
table="actors",
|
|
80
|
+
tmp_file=TMP_FILE,
|
|
81
|
+
preoperator=[
|
|
82
|
+
"""
|
|
83
|
+
CREATE TABLE IF NOT EXISTS actors (
|
|
84
|
+
name TEXT NOT NULL,
|
|
85
|
+
firstname TEXT NOT NULL,
|
|
86
|
+
age BIGINT NOT NULL
|
|
87
|
+
);
|
|
88
|
+
""",
|
|
89
|
+
"TRUNCATE TABLE actors;",
|
|
90
|
+
],
|
|
91
|
+
postoperator="DROP TABLE IF EXISTS actors;",
|
|
92
|
+
)
|
|
93
|
+
# [END howto_operator_sql_bulk_load]
|
|
94
|
+
|
|
95
|
+
create_file >> bulk_load
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
from tests_common.test_utils.system_tests import get_test_run # noqa: E402
|
|
99
|
+
|
|
100
|
+
# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst)
|
|
101
|
+
test_run = get_test_run(dag)
|
|
@@ -18,8 +18,8 @@
|
|
|
18
18
|
from __future__ import annotations
|
|
19
19
|
|
|
20
20
|
from airflow import DAG
|
|
21
|
+
from airflow.providers.common.compat.sdk import timezone
|
|
21
22
|
from airflow.providers.common.sql.operators.sql import SQLColumnCheckOperator, SQLTableCheckOperator
|
|
22
|
-
from airflow.utils import timezone
|
|
23
23
|
|
|
24
24
|
AIRFLOW_DB_METADATA_TABLE = "ab_role"
|
|
25
25
|
connection_args = {
|
|
@@ -29,10 +29,10 @@ from more_itertools import flatten
|
|
|
29
29
|
from airflow.exceptions import AirflowProviderDeprecationWarning
|
|
30
30
|
from airflow.models.connection import Connection
|
|
31
31
|
from airflow.models.dag import DAG
|
|
32
|
+
from airflow.providers.common.compat.sdk import timezone
|
|
32
33
|
from airflow.providers.common.sql.hooks.sql import DbApiHook
|
|
33
34
|
from airflow.providers.mysql.hooks.mysql import MySqlHook
|
|
34
35
|
from airflow.providers.postgres.hooks.postgres import PostgresHook
|
|
35
|
-
from airflow.utils import timezone
|
|
36
36
|
|
|
37
37
|
from tests_common.test_utils.compat import GenericTransfer
|
|
38
38
|
from tests_common.test_utils.operators.run_deferrable import execute_operator, mock_context
|
|
@@ -34,6 +34,7 @@ from airflow.providers.common.sql.hooks.sql import DbApiHook
|
|
|
34
34
|
from airflow.providers.common.sql.operators.sql import (
|
|
35
35
|
BaseSQLOperator,
|
|
36
36
|
BranchSQLOperator,
|
|
37
|
+
SQLBulkLoadOperator,
|
|
37
38
|
SQLCheckOperator,
|
|
38
39
|
SQLCheckResult,
|
|
39
40
|
SQLColumnCheckOperator,
|
|
@@ -876,6 +877,12 @@ class TestValueCheckOperator:
|
|
|
876
877
|
with pytest.raises(AirflowException, match="Tolerance:100.0%"):
|
|
877
878
|
operator.execute(context=MagicMock())
|
|
878
879
|
|
|
880
|
+
@pytest.mark.parametrize("record", [-110, -100, -90])
|
|
881
|
+
def test_negative_pass_value_with_tolerance(self, record):
|
|
882
|
+
operator = self._construct_operator("select value from tab1 limit 1;", -100, 0.1)
|
|
883
|
+
|
|
884
|
+
operator.check_value([record])
|
|
885
|
+
|
|
879
886
|
|
|
880
887
|
class TestIntervalCheckOperator:
|
|
881
888
|
def _construct_operator(self, table, metric_thresholds, ratio_formula, ignore_zero):
|
|
@@ -1684,6 +1691,66 @@ class TestSQLColumnCheckOperatorBuildCheckResults:
|
|
|
1684
1691
|
assert r.expected == ">=1, <=100"
|
|
1685
1692
|
assert r.params == {"geq_to": 1, "leq_to": 100, "accept_none": True}
|
|
1686
1693
|
|
|
1694
|
+
@pytest.mark.parametrize(
|
|
1695
|
+
("check_values", "record", "tolerance", "expected"),
|
|
1696
|
+
[
|
|
1697
|
+
# Negative thresholds: tolerance widens the bound outward, so a record equal to
|
|
1698
|
+
# the threshold passes.
|
|
1699
|
+
({"geq_to": -1000}, -1000, 0.1, True),
|
|
1700
|
+
({"geq_to": -1000}, -1100, 0.1, True),
|
|
1701
|
+
({"geq_to": -1000}, -1101, 0.1, False),
|
|
1702
|
+
({"greater_than": -1000}, -1050, 0.1, True),
|
|
1703
|
+
({"greater_than": -1000}, -1100, 0.1, False), # strict: equal to the widened bound fails
|
|
1704
|
+
({"leq_to": -10}, -10, 0.1, True),
|
|
1705
|
+
({"less_than": -10}, -11, 0.1, True),
|
|
1706
|
+
({"equal_to": -100}, -100, 0.1, True),
|
|
1707
|
+
({"equal_to": -100}, -110, 0.1, True),
|
|
1708
|
+
({"equal_to": -100}, -111, 0.1, False),
|
|
1709
|
+
({"geq_to": -100, "leq_to": -50}, -75, 0.1, True),
|
|
1710
|
+
# Positive thresholds keep their existing bounds.
|
|
1711
|
+
({"geq_to": 1000}, 900, 0.1, True),
|
|
1712
|
+
({"geq_to": 1000}, 899, 0.1, False),
|
|
1713
|
+
({"equal_to": 100}, 110, 0.1, True),
|
|
1714
|
+
({"equal_to": 100}, 111, 0.1, False),
|
|
1715
|
+
# No tolerance: exact bounds.
|
|
1716
|
+
({"geq_to": 10}, 10, None, True),
|
|
1717
|
+
({"equal_to": 5}, 6, None, False),
|
|
1718
|
+
],
|
|
1719
|
+
)
|
|
1720
|
+
def test_get_match_tolerance_handles_negative_thresholds(self, check_values, record, tolerance, expected):
|
|
1721
|
+
op = self._make_operator({"col": {"min": {"geq_to": 1}}})
|
|
1722
|
+
assert op._get_match(check_values, record, tolerance) == expected
|
|
1723
|
+
|
|
1724
|
+
@pytest.mark.parametrize(
|
|
1725
|
+
("check_values", "record", "expected"),
|
|
1726
|
+
[
|
|
1727
|
+
({"geq_to": "2020-01-01"}, "2021-06-30", True),
|
|
1728
|
+
({"geq_to": "2020-01-01"}, "2019-12-31", False),
|
|
1729
|
+
({"greater_than": "2020-01-01"}, "2020-01-01", False),
|
|
1730
|
+
({"leq_to": "2020-01-01"}, "2019-12-31", True),
|
|
1731
|
+
({"less_than": "2020-01-01"}, "2020-01-01", False),
|
|
1732
|
+
({"equal_to": "abc"}, "abc", True),
|
|
1733
|
+
({"equal_to": "abc"}, "abcd", False),
|
|
1734
|
+
(
|
|
1735
|
+
{"geq_to": datetime.date(2020, 1, 1), "leq_to": datetime.date(2020, 12, 31)},
|
|
1736
|
+
datetime.date(2020, 6, 30),
|
|
1737
|
+
True,
|
|
1738
|
+
),
|
|
1739
|
+
(
|
|
1740
|
+
{"geq_to": datetime.date(2020, 1, 1), "leq_to": datetime.date(2020, 12, 31)},
|
|
1741
|
+
datetime.date(2021, 1, 1),
|
|
1742
|
+
False,
|
|
1743
|
+
),
|
|
1744
|
+
],
|
|
1745
|
+
)
|
|
1746
|
+
def test_get_match_compares_non_numeric_bounds_without_tolerance(self, check_values, record, expected):
|
|
1747
|
+
op = self._make_operator({"col": {"min": {"geq_to": 1}}})
|
|
1748
|
+
assert op._get_match(check_values, record) == expected
|
|
1749
|
+
|
|
1750
|
+
def test_get_match_equal_to_fails_cleanly_on_none_record(self):
|
|
1751
|
+
op = self._make_operator({"col": {"null_check": {"equal_to": 0}}}, accept_none=False)
|
|
1752
|
+
assert op._get_match({"equal_to": 0}, None) is False
|
|
1753
|
+
|
|
1687
1754
|
def test_multiple_checks_correct_names_and_order(self):
|
|
1688
1755
|
op = self._make_operator(
|
|
1689
1756
|
{
|
|
@@ -2044,6 +2111,16 @@ class TestSQLValueCheckOperatorBuildCheckResults:
|
|
|
2044
2111
|
assert r.expected == ">= 4.5, <= 5.5"
|
|
2045
2112
|
assert r.params == {"pass_value": "5", "tolerance": 0.1}
|
|
2046
2113
|
|
|
2114
|
+
def test_negative_numeric_tolerance_produces_accepted_range(self):
|
|
2115
|
+
op = self._make_operator(pass_value=-100, tolerance=0.1)
|
|
2116
|
+
results = op._build_check_results([-100])
|
|
2117
|
+
assert len(results) == 1
|
|
2118
|
+
r = results[0]
|
|
2119
|
+
assert r.success is True
|
|
2120
|
+
assert r.check_type == "accepted_range"
|
|
2121
|
+
assert r.expected == ">= -110.0, <= -90.0"
|
|
2122
|
+
assert r.params == {"pass_value": "-100", "tolerance": 0.1}
|
|
2123
|
+
|
|
2047
2124
|
def test_non_numeric_pass_value_is_accepted_values(self):
|
|
2048
2125
|
op = self._make_operator(pass_value="hello")
|
|
2049
2126
|
results = op._build_check_results(["hello"])
|
|
@@ -2957,3 +3034,86 @@ class TestSQLInsertRowsOperator:
|
|
|
2957
3034
|
(4, "Lundgren", "Dolph", 66),
|
|
2958
3035
|
(5, "Norris", "Chuck", 84),
|
|
2959
3036
|
]
|
|
3037
|
+
|
|
3038
|
+
|
|
3039
|
+
class TestSQLBulkLoadOperator:
|
|
3040
|
+
def _construct_operator(self, table, tmp_file, **kwargs):
|
|
3041
|
+
dag = DAG("test_dag", schedule=None, start_date=datetime.datetime(2017, 1, 1))
|
|
3042
|
+
return SQLBulkLoadOperator(
|
|
3043
|
+
task_id="test_task",
|
|
3044
|
+
conn_id="default_conn",
|
|
3045
|
+
table=table,
|
|
3046
|
+
tmp_file=tmp_file,
|
|
3047
|
+
dag=dag,
|
|
3048
|
+
**kwargs,
|
|
3049
|
+
)
|
|
3050
|
+
|
|
3051
|
+
@mock.patch.object(SQLBulkLoadOperator, "get_db_hook")
|
|
3052
|
+
def test_execute(self, mock_get_db_hook):
|
|
3053
|
+
operator = self._construct_operator(
|
|
3054
|
+
table="users",
|
|
3055
|
+
tmp_file="/tmp/users.tsv",
|
|
3056
|
+
preoperator="CREATE TABLE users (id INT);",
|
|
3057
|
+
postoperator="DROP TABLE users;",
|
|
3058
|
+
)
|
|
3059
|
+
|
|
3060
|
+
operator.execute(context=MagicMock())
|
|
3061
|
+
|
|
3062
|
+
hook = mock_get_db_hook.return_value
|
|
3063
|
+
|
|
3064
|
+
assert hook.mock_calls == [
|
|
3065
|
+
mock.call.run("CREATE TABLE users (id INT);"),
|
|
3066
|
+
mock.call.bulk_load(
|
|
3067
|
+
table="users",
|
|
3068
|
+
tmp_file="/tmp/users.tsv",
|
|
3069
|
+
),
|
|
3070
|
+
mock.call.run("DROP TABLE users;"),
|
|
3071
|
+
]
|
|
3072
|
+
|
|
3073
|
+
@mock.patch.object(SQLBulkLoadOperator, "get_db_hook")
|
|
3074
|
+
def test_execute_templated_fields(self, mock_get_db_hook):
|
|
3075
|
+
operator = self._construct_operator(
|
|
3076
|
+
table="{{ params.table }}",
|
|
3077
|
+
tmp_file="{{ params.file }}",
|
|
3078
|
+
preoperator="TRUNCATE TABLE {{ params.table }};",
|
|
3079
|
+
postoperator="DROP TABLE {{ params.table }};",
|
|
3080
|
+
)
|
|
3081
|
+
|
|
3082
|
+
operator.render_template_fields(
|
|
3083
|
+
{
|
|
3084
|
+
"params": {
|
|
3085
|
+
"table": "users",
|
|
3086
|
+
"file": "/tmp/users.tsv",
|
|
3087
|
+
}
|
|
3088
|
+
}
|
|
3089
|
+
)
|
|
3090
|
+
|
|
3091
|
+
operator.execute(context=MagicMock())
|
|
3092
|
+
|
|
3093
|
+
hook = mock_get_db_hook.return_value
|
|
3094
|
+
|
|
3095
|
+
assert hook.mock_calls == [
|
|
3096
|
+
mock.call.run("TRUNCATE TABLE users;"),
|
|
3097
|
+
mock.call.bulk_load(
|
|
3098
|
+
table="users",
|
|
3099
|
+
tmp_file="/tmp/users.tsv",
|
|
3100
|
+
),
|
|
3101
|
+
mock.call.run("DROP TABLE users;"),
|
|
3102
|
+
]
|
|
3103
|
+
|
|
3104
|
+
@mock.patch.object(SQLBulkLoadOperator, "get_db_hook")
|
|
3105
|
+
def test_execute_without_pre_or_post_operator(self, mock_get_db_hook):
|
|
3106
|
+
operator = self._construct_operator(
|
|
3107
|
+
table="users",
|
|
3108
|
+
tmp_file="/tmp/users.tsv",
|
|
3109
|
+
)
|
|
3110
|
+
|
|
3111
|
+
operator.execute(context=MagicMock())
|
|
3112
|
+
|
|
3113
|
+
hook = mock_get_db_hook.return_value
|
|
3114
|
+
|
|
3115
|
+
hook.run.assert_not_called()
|
|
3116
|
+
hook.bulk_load.assert_called_once_with(
|
|
3117
|
+
table="users",
|
|
3118
|
+
tmp_file="/tmp/users.tsv",
|
|
3119
|
+
)
|
{apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/.gitignore
RENAMED
|
File without changes
|
{apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/LICENSE
RENAMED
|
File without changes
|
{apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/NOTICE
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/docs/conf.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|