apache-airflow-providers-common-sql 2.0.3__tar.gz → 2.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/PKG-INFO +6 -6
  2. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/README.rst +3 -3
  3. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/docs/changelog.rst +25 -0
  4. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/docs/index.rst +3 -3
  5. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/docs/operators.rst +29 -0
  6. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/provider.yaml +2 -1
  7. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/pyproject.toml +3 -3
  8. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/__init__.py +1 -1
  9. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/operators/sql.py +104 -28
  10. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/system/common/sql/example_generic_transfer.py +1 -1
  11. apache_airflow_providers_common_sql-2.1.0/tests/system/common/sql/example_sql_bulk_load.py +101 -0
  12. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/system/common/sql/example_sql_column_table_check.py +1 -1
  13. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/operators/test_generic_transfer.py +1 -1
  14. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/operators/test_sql.py +160 -0
  15. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/.gitignore +0 -0
  16. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/LICENSE +0 -0
  17. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/NOTICE +0 -0
  18. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/docs/.latest-doc-only-change.txt +0 -0
  19. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/docs/commits.rst +0 -0
  20. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/docs/conf.py +0 -0
  21. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/docs/connections.rst +0 -0
  22. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/docs/dataframes.rst +0 -0
  23. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/docs/dialects.rst +0 -0
  24. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/docs/installing-providers-from-sources.rst +0 -0
  25. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/docs/integration-logos/sql.png +0 -0
  26. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/docs/security.rst +0 -0
  27. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/docs/supported-database-types.rst +0 -0
  28. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/__init__.py +0 -0
  29. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/__init__.py +0 -0
  30. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/__init__.py +0 -0
  31. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/README_API.md +0 -0
  32. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/config.py +0 -0
  33. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/datafusion/__init__.py +0 -0
  34. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/datafusion/base.py +0 -0
  35. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/datafusion/engine.py +0 -0
  36. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/datafusion/exceptions.py +0 -0
  37. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/datafusion/format_handlers.py +0 -0
  38. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/datafusion/object_storage_provider.py +0 -0
  39. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/decorators/__init__.py +0 -0
  40. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/decorators/analytics.py +0 -0
  41. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/decorators/sql.py +0 -0
  42. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/dialects/__init__.py +0 -0
  43. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/dialects/dialect.py +0 -0
  44. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/dialects/dialect.pyi +0 -0
  45. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/doc/adr/0001-record-architecture-decisions.md +0 -0
  46. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/doc/adr/0002-return-common-data-structure-from-dbapihook-derived-hooks.md +0 -0
  47. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/doc/adr/0003-introduce-notion-of-dialects-in-dbapihook.md +0 -0
  48. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/example_dags/__init__.py +0 -0
  49. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/example_dags/example_analytics.py +0 -0
  50. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/get_provider_info.py +0 -0
  51. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/get_provider_info.pyi +0 -0
  52. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/hooks/__init__.py +0 -0
  53. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/hooks/handlers.py +0 -0
  54. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/hooks/handlers.pyi +0 -0
  55. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/hooks/lineage.py +0 -0
  56. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/hooks/lineage.pyi +0 -0
  57. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/hooks/sql.py +0 -0
  58. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/hooks/sql.pyi +0 -0
  59. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/operators/__init__.py +0 -0
  60. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/operators/analytics.py +0 -0
  61. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/operators/generic_transfer.py +0 -0
  62. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/operators/generic_transfer.pyi +0 -0
  63. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/sensors/__init__.py +0 -0
  64. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/sensors/sql.py +0 -0
  65. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/sensors/sql.pyi +0 -0
  66. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/triggers/__init__.py +0 -0
  67. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/triggers/sql.py +0 -0
  68. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/triggers/sql.pyi +0 -0
  69. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/src/airflow/providers/common/sql/version_compat.py +0 -0
  70. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/conftest.py +0 -0
  71. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/system/__init__.py +0 -0
  72. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/system/common/__init__.py +0 -0
  73. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/system/common/sql/__init__.py +0 -0
  74. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/system/common/sql/example_sql_execute_query.py +0 -0
  75. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/system/common/sql/example_sql_insert_rows.py +0 -0
  76. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/system/common/sql/example_sql_threshold_check.py +0 -0
  77. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/system/common/sql/example_sql_value_check.py +0 -0
  78. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/__init__.py +0 -0
  79. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/__init__.py +0 -0
  80. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/__init__.py +0 -0
  81. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/datafusion/__init__.py +0 -0
  82. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/datafusion/test_engine.py +0 -0
  83. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/datafusion/test_format_handlers.py +0 -0
  84. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/datafusion/test_object_storage_provider.py +0 -0
  85. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/decorators/__init__.py +0 -0
  86. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/decorators/test_analytics.py +0 -0
  87. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/decorators/test_sql.py +0 -0
  88. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/dialects/__init__.py +0 -0
  89. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/dialects/test_dialect.py +0 -0
  90. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/hooks/__init__.py +0 -0
  91. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/hooks/test_dbapi.py +0 -0
  92. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/hooks/test_handlers.py +0 -0
  93. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/hooks/test_lineage.py +0 -0
  94. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/hooks/test_sql.py +0 -0
  95. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/hooks/test_sqlparse.py +0 -0
  96. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/operators/__init__.py +0 -0
  97. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/operators/test_analytics.py +0 -0
  98. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/operators/test_sql_execute.py +0 -0
  99. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/sensors/__init__.py +0 -0
  100. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/sensors/test_sql.py +0 -0
  101. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/test_config.py +0 -0
  102. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/triggers/__init__.py +0 -0
  103. {apache_airflow_providers_common_sql-2.0.3 → apache_airflow_providers_common_sql-2.1.0}/tests/unit/common/sql/triggers/test_sql.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: apache-airflow-providers-common-sql
3
- Version: 2.0.3
3
+ Version: 2.1.0
4
4
  Summary: Provider package apache-airflow-providers-common-sql for Apache Airflow
5
5
  Keywords: airflow-provider,common.sql,airflow,integration
6
6
  Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -39,8 +39,8 @@ Requires-Dist: polars>=1.26.0 ; extra == "polars"
39
39
  Requires-Dist: pyiceberg-core>=0.8.0 ; extra == "pyiceberg-core"
40
40
  Requires-Dist: sqlalchemy>=1.4.54 ; extra == "sqlalchemy"
41
41
  Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
42
- Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.0.3/changelog.html
43
- Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.0.3
42
+ Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.1.0/changelog.html
43
+ Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.1.0
44
44
  Project-URL: Mastodon, https://fosstodon.org/@airflow
45
45
  Project-URL: Slack Chat, https://s.apache.org/airflow-slack
46
46
  Project-URL: Source Code, https://github.com/apache/airflow
@@ -79,7 +79,7 @@ Provides-Extra: sqlalchemy
79
79
 
80
80
  Package ``apache-airflow-providers-common-sql``
81
81
 
82
- Release: ``2.0.3``
82
+ Release: ``2.1.0``
83
83
 
84
84
 
85
85
  `Common SQL Provider <https://en.wikipedia.org/wiki/SQL>`__
@@ -92,7 +92,7 @@ This is a provider package for ``common.sql`` provider. All classes for this pro
92
92
  are in ``airflow.providers.common.sql`` python package.
93
93
 
94
94
  You can find package information and changelog for the provider
95
- in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.0.3/>`_.
95
+ in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.1.0/>`_.
96
96
 
97
97
  Installation
98
98
  ------------
@@ -154,5 +154,5 @@ Extra Dependencies
154
154
  ================== =======================================================================================================================================================================
155
155
 
156
156
  The changelog for the provider package can be found in the
157
- `changelog <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.0.3/changelog.html>`_.
157
+ `changelog <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.1.0/changelog.html>`_.
158
158
 
@@ -23,7 +23,7 @@
23
23
 
24
24
  Package ``apache-airflow-providers-common-sql``
25
25
 
26
- Release: ``2.0.3``
26
+ Release: ``2.1.0``
27
27
 
28
28
 
29
29
  `Common SQL Provider <https://en.wikipedia.org/wiki/SQL>`__
@@ -36,7 +36,7 @@ This is a provider package for ``common.sql`` provider. All classes for this pro
36
36
  are in ``airflow.providers.common.sql`` python package.
37
37
 
38
38
  You can find package information and changelog for the provider
39
- in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.0.3/>`_.
39
+ in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.1.0/>`_.
40
40
 
41
41
  Installation
42
42
  ------------
@@ -98,4 +98,4 @@ Extra Dependencies
98
98
  ================== =======================================================================================================================================================================
99
99
 
100
100
  The changelog for the provider package can be found in the
101
- `changelog <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.0.3/changelog.html>`_.
101
+ `changelog <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.1.0/changelog.html>`_.
@@ -25,6 +25,31 @@
25
25
  Changelog
26
26
  ---------
27
27
 
28
+ 2.1.0
29
+ .....
30
+
31
+ Features
32
+ ~~~~~~~~
33
+
34
+ * ``Add SQLBulkLoadOperator to Common SQL Provider (#69362)``
35
+
36
+ Bug Fixes
37
+ ~~~~~~~~~
38
+
39
+ * ``Fix SQLColumnCheckOperator crash on non-numeric column bounds (#70895)``
40
+ * ``Fix SQL check tolerance for negative expected values (#69736)``
41
+ * ``Fix SQL value checks for negative expected values (#69893)``
42
+
43
+ Misc
44
+ ~~~~
45
+
46
+
47
+ .. Below changes are excluded from the changelog. Move them to
48
+ appropriate section above if needed. Do not delete the lines(!):
49
+ * ``Use common.compat.sdk for timezone imports in providers (#70492)``
50
+ * ``Limit pandas to < 3 for DataFrame XComs (#70791)``
51
+ * ``Revert "Limit pandas to < 3 for DataFrame XComs (#70791)" (#71100)``
52
+
28
53
  2.0.3
29
54
  .....
30
55
 
@@ -79,7 +79,7 @@ apache-airflow-providers-common-sql package
79
79
  `Common SQL Provider <https://en.wikipedia.org/wiki/SQL>`__
80
80
 
81
81
 
82
- Release: 2.0.3
82
+ Release: 2.1.0
83
83
 
84
84
  Provider package
85
85
  ----------------
@@ -160,5 +160,5 @@ Downloading official packages
160
160
  You can download officially released packages and verify their checksums and signatures from the
161
161
  `Official Apache Download site <https://downloads.apache.org/airflow/providers/>`_
162
162
 
163
- * `The apache-airflow-providers-common-sql 2.0.3 sdist package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-2.0.3.tar.gz>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-2.0.3.tar.gz.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-2.0.3.tar.gz.sha512>`__)
164
- * `The apache-airflow-providers-common-sql 2.0.3 wheel package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-2.0.3-py3-none-any.whl>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-2.0.3-py3-none-any.whl.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-2.0.3-py3-none-any.whl.sha512>`__)
163
+ * `The apache-airflow-providers-common-sql 2.1.0 sdist package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-2.1.0.tar.gz>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-2.1.0.tar.gz.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-2.1.0.tar.gz.sha512>`__)
164
+ * `The apache-airflow-providers-common-sql 2.1.0 wheel package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-2.1.0-py3-none-any.whl>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-2.1.0-py3-none-any.whl.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-2.1.0-py3-none-any.whl.sha512>`__)
@@ -239,6 +239,35 @@ The example below shows how to instantiate the SQLInsertRowsOperator task.
239
239
  :start-after: [START howto_operator_sql_insert_rows]
240
240
  :end-before: [END howto_operator_sql_insert_rows]
241
241
 
242
+ .. _howto/operator:SQLBulkLoadOperator:
243
+
244
+ Bulk load data into a table
245
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~
246
+
247
+ Use the :class:`~airflow.providers.common.sql.operators.sql.SQLBulkLoadOperator`
248
+ to bulk load a tab-delimited file into a database table using the database's
249
+ native bulk loading mechanism. Parameters of the operator are:
250
+
251
+ - ``table`` - name of the target table (templated).
252
+ - ``tmp_file`` - path to the tab-delimited file to load (templated).
253
+ - ``conn_id`` - the Airflow connection ID used to connect to the database.
254
+ - ``database`` (optional) - name of the database which overrides the one defined
255
+ in the connection.
256
+ - ``preoperator`` (optional) - SQL statement or list of statements to execute
257
+ before bulk loading (templated).
258
+ - ``postoperator`` (optional) - SQL statement or list of statements to execute
259
+ after bulk loading (templated).
260
+ - ``hook_params`` (optional) - dictionary of additional parameters passed to the
261
+ underlying hook.
262
+
263
+ The example below shows how to instantiate the SQLBulkLoadOperator task.
264
+
265
+ .. exampleinclude:: /../tests/system/common/sql/example_sql_bulk_load.py
266
+ :language: python
267
+ :dedent: 4
268
+ :start-after: [START howto_operator_sql_bulk_load]
269
+ :end-before: [END howto_operator_sql_bulk_load]
270
+
242
271
  .. _howto/operator:GenericTransfer:
243
272
 
244
273
  Generic Transfer
@@ -23,12 +23,13 @@ description: |
23
23
 
24
24
  state: ready
25
25
  lifecycle: production
26
- source-date-epoch: 1784746595
26
+ source-date-epoch: 1785633505
27
27
  # Note that those versions are maintained by release manager - do not update them manually
28
28
  # with the exception of case where other provider in sources has >= new provider version.
29
29
  # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have
30
30
  # to be done in the same PR
31
31
  versions:
32
+ - 2.1.0
32
33
  - 2.0.3
33
34
  - 2.0.2
34
35
  - 2.0.1
@@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi"
25
25
 
26
26
  [project]
27
27
  name = "apache-airflow-providers-common-sql"
28
- version = "2.0.3"
28
+ version = "2.1.0"
29
29
  description = "Provider package apache-airflow-providers-common-sql for Apache Airflow"
30
30
  readme = "README.rst"
31
31
  license = "Apache-2.0"
@@ -148,8 +148,8 @@ apache-airflow-providers-common-sql = {workspace = true}
148
148
  apache-airflow-providers-standard = {workspace = true}
149
149
 
150
150
  [project.urls]
151
- "Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.0.3"
152
- "Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.0.3/changelog.html"
151
+ "Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.1.0"
152
+ "Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.1.0/changelog.html"
153
153
  "Bug Tracker" = "https://github.com/apache/airflow/issues"
154
154
  "Source Code" = "https://github.com/apache/airflow"
155
155
  "Slack Chat" = "https://s.apache.org/airflow-slack"
@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
29
29
 
30
30
  __all__ = ["__version__"]
31
31
 
32
- __version__ = "2.0.3"
32
+ __version__ = "2.1.0"
33
33
 
34
34
  if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
35
35
  "2.11.0"
@@ -744,35 +744,32 @@ class SQLColumnCheckOperator(BaseSQLOperator):
744
744
  if record is None and self.accept_none:
745
745
  record = 0
746
746
  match_boolean = True
747
+
748
+ # abs() so the tolerance margin widens the bound outward for negative expected values too.
749
+ # Without a tolerance the bound is compared as-is: a min/max check may be declared on a
750
+ # date or text column, where arithmetic on the bound raises TypeError.
751
+ def _lower(expected):
752
+ return expected - abs(expected) * tolerance if tolerance is not None else expected
753
+
754
+ def _upper(expected):
755
+ return expected + abs(expected) * tolerance if tolerance is not None else expected
756
+
747
757
  if "geq_to" in check_values:
748
- if tolerance is not None:
749
- match_boolean = record >= check_values["geq_to"] * (1 - tolerance)
750
- else:
751
- match_boolean = record >= check_values["geq_to"]
758
+ match_boolean = record >= _lower(check_values["geq_to"])
752
759
  elif "greater_than" in check_values:
753
- if tolerance is not None:
754
- match_boolean = record > check_values["greater_than"] * (1 - tolerance)
755
- else:
756
- match_boolean = record > check_values["greater_than"]
760
+ match_boolean = record > _lower(check_values["greater_than"])
757
761
  if "leq_to" in check_values:
758
- if tolerance is not None:
759
- match_boolean = record <= check_values["leq_to"] * (1 + tolerance) and match_boolean
760
- else:
761
- match_boolean = record <= check_values["leq_to"] and match_boolean
762
+ match_boolean = record <= _upper(check_values["leq_to"]) and match_boolean
762
763
  elif "less_than" in check_values:
763
- if tolerance is not None:
764
- match_boolean = record < check_values["less_than"] * (1 + tolerance) and match_boolean
765
- else:
766
- match_boolean = record < check_values["less_than"] and match_boolean
764
+ match_boolean = record < _upper(check_values["less_than"]) and match_boolean
767
765
  if "equal_to" in check_values:
768
- if tolerance is not None:
769
- match_boolean = (
770
- check_values["equal_to"] * (1 - tolerance)
771
- <= record
772
- <= check_values["equal_to"] * (1 + tolerance)
773
- ) and match_boolean
766
+ expected = check_values["equal_to"]
767
+ if tolerance is None:
768
+ # Equality, not a degenerate range: a NULL result must fail the check rather
769
+ # than raise on an ordering comparison against None.
770
+ match_boolean = record == expected and match_boolean
774
771
  else:
775
- match_boolean = record == check_values["equal_to"] and match_boolean
772
+ match_boolean = (_lower(expected) <= record <= _upper(expected)) and match_boolean
776
773
  return match_boolean
777
774
 
778
775
  def _column_mapping_validation(self, check, check_values):
@@ -1279,12 +1276,14 @@ class SQLValueCheckOperator(BaseSQLOperator):
1279
1276
  def _get_string_matches(self, records, pass_value_conv):
1280
1277
  return [str(record) == pass_value_conv for record in records]
1281
1278
 
1279
+ def _get_tolerance_bounds(self, numeric_pass_value_conv):
1280
+ margin = abs(numeric_pass_value_conv) * self.tol
1281
+ return numeric_pass_value_conv - margin, numeric_pass_value_conv + margin
1282
+
1282
1283
  def _get_numeric_matches(self, numeric_records, numeric_pass_value_conv):
1283
1284
  if self.has_tolerance:
1284
- return [
1285
- numeric_pass_value_conv * (1 - self.tol) <= record <= numeric_pass_value_conv * (1 + self.tol)
1286
- for record in numeric_records
1287
- ]
1285
+ lower_bound, upper_bound = self._get_tolerance_bounds(numeric_pass_value_conv)
1286
+ return [lower_bound <= record <= upper_bound for record in numeric_records]
1288
1287
 
1289
1288
  return [record == numeric_pass_value_conv for record in numeric_records]
1290
1289
 
@@ -1308,7 +1307,8 @@ class SQLValueCheckOperator(BaseSQLOperator):
1308
1307
 
1309
1308
  pass_value_conv = _convert_to_float_if_possible(self.pass_value)
1310
1309
  if isinstance(pass_value_conv, float) and isinstance(self.tol, float):
1311
- expected_str = f">= {pass_value_conv * (1 - self.tol)}, <= {pass_value_conv * (1 + self.tol)}"
1310
+ lower_bound, upper_bound = self._get_tolerance_bounds(pass_value_conv)
1311
+ expected_str = f">= {lower_bound}, <= {upper_bound}"
1312
1312
  check_type = "accepted_range"
1313
1313
 
1314
1314
  return [
@@ -1916,3 +1916,79 @@ def _initialize_partition_clause(clause: str | None) -> str | None:
1916
1916
  raise ValueError("Invalid partition_clause: semicolons (;) not allowed.")
1917
1917
 
1918
1918
  return clause
1919
+
1920
+
1921
+ class SQLBulkLoadOperator(BaseSQLOperator):
1922
+ """
1923
+ Bulk load a tab-delimited file into a database table.
1924
+
1925
+ This operator delegates to the underlying hook's ``bulk_load`` implementation,
1926
+ allowing each provider to use its native bulk loading mechanism.
1927
+
1928
+ .. seealso::
1929
+ For more information on how to use this operator, take a look at the guide:
1930
+ :ref:`howto/operator:SQLBulkLoadOperator`
1931
+
1932
+ :param table: Name of the target table (templated).
1933
+ :param tmp_file: Path to the tab-delimited file to load (templated).
1934
+ :param conn_id: The connection ID used to connect to the database. Defaults to ``None``.
1935
+ :param database: Name of the database which overrides the one defined in the connection.
1936
+ Defaults to ``None``.
1937
+ :param preoperator: SQL statement or list of statements to execute before bulk loading
1938
+ (templated). Defaults to ``None``.
1939
+ :param postoperator: SQL statement or list of statements to execute after bulk loading
1940
+ (templated). Defaults to ``None``.
1941
+ """
1942
+
1943
+ template_fields: Sequence[str] = (
1944
+ "table",
1945
+ "tmp_file",
1946
+ "preoperator",
1947
+ "postoperator",
1948
+ *BaseSQLOperator.template_fields,
1949
+ )
1950
+
1951
+ def __init__(
1952
+ self,
1953
+ *,
1954
+ table: str,
1955
+ tmp_file: str,
1956
+ conn_id: str | None = None,
1957
+ database: str | None = None,
1958
+ preoperator: str | list[str] | None = None,
1959
+ postoperator: str | list[str] | None = None,
1960
+ **kwargs,
1961
+ ) -> None:
1962
+ super().__init__(
1963
+ conn_id=conn_id,
1964
+ database=database,
1965
+ **kwargs,
1966
+ )
1967
+ self.table = table
1968
+ self.tmp_file = tmp_file
1969
+ self.preoperator = preoperator
1970
+ self.postoperator = postoperator
1971
+
1972
+ def execute(self, context: Context) -> None:
1973
+ hook = self.get_db_hook()
1974
+
1975
+ if self.preoperator:
1976
+ self.log.info("Executing preoperator.")
1977
+ hook.run(self.preoperator)
1978
+
1979
+ self.log.info(
1980
+ "Bulk loading '%s' into table '%s'.",
1981
+ self.tmp_file,
1982
+ self.table,
1983
+ )
1984
+
1985
+ hook.bulk_load(
1986
+ table=self.table,
1987
+ tmp_file=self.tmp_file,
1988
+ )
1989
+
1990
+ if self.postoperator:
1991
+ self.log.info("Executing postoperator.")
1992
+ hook.run(self.postoperator)
1993
+
1994
+ self.log.info("Bulk load completed.")
@@ -18,8 +18,8 @@
18
18
  from __future__ import annotations
19
19
 
20
20
  from airflow import DAG
21
+ from airflow.providers.common.compat.sdk import timezone
21
22
  from airflow.providers.common.sql.operators.generic_transfer import GenericTransfer
22
- from airflow.utils import timezone
23
23
 
24
24
  connection_args = {
25
25
  "conn_id": "airflow_db",
@@ -0,0 +1,101 @@
1
+ #
2
+ # Licensed to the Apache Software Foundation (ASF) under one
3
+ # or more contributor license agreements. See the NOTICE file
4
+ # distributed with this work for additional information
5
+ # regarding copyright ownership. The ASF licenses this file
6
+ # to you under the Apache License, Version 2.0 (the
7
+ # "License"); you may not use this file except in compliance
8
+ # with the License. You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing,
13
+ # software distributed under the License is distributed on an
14
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15
+ # KIND, either express or implied. See the License for the
16
+ # specific language governing permissions and limitations
17
+ # under the License.
18
+ from __future__ import annotations
19
+
20
+ from pathlib import Path
21
+
22
+ from airflow import DAG
23
+ from airflow.providers.common.sql.operators.sql import (
24
+ SQLBulkLoadOperator,
25
+ )
26
+ from airflow.providers.standard.operators.python import PythonOperator
27
+ from airflow.utils.timezone import datetime
28
+
29
+ TMP_FILE = "/tmp/actors.tsv"
30
+
31
+ connection_args = {
32
+ "conn_id": "airflow_db",
33
+ "conn_type": "Postgres",
34
+ "host": "postgres",
35
+ "schema": "postgres",
36
+ "login": "postgres",
37
+ "password": "postgres",
38
+ "port": 5432,
39
+ }
40
+
41
+
42
+ def create_bulk_load_file():
43
+ Path(TMP_FILE).write_text(
44
+ "\n".join(
45
+ [
46
+ "Stallone\tSylvester\t78",
47
+ "Statham\tJason\t57",
48
+ "Li\tJet\t61",
49
+ "Lundgren\tDolph\t66",
50
+ "Norris\tChuck\t84",
51
+ ]
52
+ ),
53
+ encoding="utf-8",
54
+ )
55
+
56
+
57
+ with DAG(
58
+ "example_sql_bulk_load",
59
+ description="Example DAG for SQLBulkLoadOperator.",
60
+ default_args=connection_args,
61
+ start_date=datetime(2021, 1, 1),
62
+ schedule=None,
63
+ catchup=False,
64
+ ) as dag:
65
+ """
66
+ ### Example SQL bulk load DAG
67
+
68
+ Runs the SQLBulkLoadOperator against the Airflow metadata DB.
69
+ """
70
+
71
+ create_file = PythonOperator(
72
+ task_id="create_file",
73
+ python_callable=create_bulk_load_file,
74
+ )
75
+
76
+ # [START howto_operator_sql_bulk_load]
77
+ bulk_load = SQLBulkLoadOperator(
78
+ task_id="bulk_load",
79
+ table="actors",
80
+ tmp_file=TMP_FILE,
81
+ preoperator=[
82
+ """
83
+ CREATE TABLE IF NOT EXISTS actors (
84
+ name TEXT NOT NULL,
85
+ firstname TEXT NOT NULL,
86
+ age BIGINT NOT NULL
87
+ );
88
+ """,
89
+ "TRUNCATE TABLE actors;",
90
+ ],
91
+ postoperator="DROP TABLE IF EXISTS actors;",
92
+ )
93
+ # [END howto_operator_sql_bulk_load]
94
+
95
+ create_file >> bulk_load
96
+
97
+
98
+ from tests_common.test_utils.system_tests import get_test_run # noqa: E402
99
+
100
+ # Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst)
101
+ test_run = get_test_run(dag)
@@ -18,8 +18,8 @@
18
18
  from __future__ import annotations
19
19
 
20
20
  from airflow import DAG
21
+ from airflow.providers.common.compat.sdk import timezone
21
22
  from airflow.providers.common.sql.operators.sql import SQLColumnCheckOperator, SQLTableCheckOperator
22
- from airflow.utils import timezone
23
23
 
24
24
  AIRFLOW_DB_METADATA_TABLE = "ab_role"
25
25
  connection_args = {
@@ -29,10 +29,10 @@ from more_itertools import flatten
29
29
  from airflow.exceptions import AirflowProviderDeprecationWarning
30
30
  from airflow.models.connection import Connection
31
31
  from airflow.models.dag import DAG
32
+ from airflow.providers.common.compat.sdk import timezone
32
33
  from airflow.providers.common.sql.hooks.sql import DbApiHook
33
34
  from airflow.providers.mysql.hooks.mysql import MySqlHook
34
35
  from airflow.providers.postgres.hooks.postgres import PostgresHook
35
- from airflow.utils import timezone
36
36
 
37
37
  from tests_common.test_utils.compat import GenericTransfer
38
38
  from tests_common.test_utils.operators.run_deferrable import execute_operator, mock_context
@@ -34,6 +34,7 @@ from airflow.providers.common.sql.hooks.sql import DbApiHook
34
34
  from airflow.providers.common.sql.operators.sql import (
35
35
  BaseSQLOperator,
36
36
  BranchSQLOperator,
37
+ SQLBulkLoadOperator,
37
38
  SQLCheckOperator,
38
39
  SQLCheckResult,
39
40
  SQLColumnCheckOperator,
@@ -876,6 +877,12 @@ class TestValueCheckOperator:
876
877
  with pytest.raises(AirflowException, match="Tolerance:100.0%"):
877
878
  operator.execute(context=MagicMock())
878
879
 
880
+ @pytest.mark.parametrize("record", [-110, -100, -90])
881
+ def test_negative_pass_value_with_tolerance(self, record):
882
+ operator = self._construct_operator("select value from tab1 limit 1;", -100, 0.1)
883
+
884
+ operator.check_value([record])
885
+
879
886
 
880
887
  class TestIntervalCheckOperator:
881
888
  def _construct_operator(self, table, metric_thresholds, ratio_formula, ignore_zero):
@@ -1684,6 +1691,66 @@ class TestSQLColumnCheckOperatorBuildCheckResults:
1684
1691
  assert r.expected == ">=1, <=100"
1685
1692
  assert r.params == {"geq_to": 1, "leq_to": 100, "accept_none": True}
1686
1693
 
1694
+ @pytest.mark.parametrize(
1695
+ ("check_values", "record", "tolerance", "expected"),
1696
+ [
1697
+ # Negative thresholds: tolerance widens the bound outward, so a record equal to
1698
+ # the threshold passes.
1699
+ ({"geq_to": -1000}, -1000, 0.1, True),
1700
+ ({"geq_to": -1000}, -1100, 0.1, True),
1701
+ ({"geq_to": -1000}, -1101, 0.1, False),
1702
+ ({"greater_than": -1000}, -1050, 0.1, True),
1703
+ ({"greater_than": -1000}, -1100, 0.1, False), # strict: equal to the widened bound fails
1704
+ ({"leq_to": -10}, -10, 0.1, True),
1705
+ ({"less_than": -10}, -11, 0.1, True),
1706
+ ({"equal_to": -100}, -100, 0.1, True),
1707
+ ({"equal_to": -100}, -110, 0.1, True),
1708
+ ({"equal_to": -100}, -111, 0.1, False),
1709
+ ({"geq_to": -100, "leq_to": -50}, -75, 0.1, True),
1710
+ # Positive thresholds keep their existing bounds.
1711
+ ({"geq_to": 1000}, 900, 0.1, True),
1712
+ ({"geq_to": 1000}, 899, 0.1, False),
1713
+ ({"equal_to": 100}, 110, 0.1, True),
1714
+ ({"equal_to": 100}, 111, 0.1, False),
1715
+ # No tolerance: exact bounds.
1716
+ ({"geq_to": 10}, 10, None, True),
1717
+ ({"equal_to": 5}, 6, None, False),
1718
+ ],
1719
+ )
1720
+ def test_get_match_tolerance_handles_negative_thresholds(self, check_values, record, tolerance, expected):
1721
+ op = self._make_operator({"col": {"min": {"geq_to": 1}}})
1722
+ assert op._get_match(check_values, record, tolerance) == expected
1723
+
1724
+ @pytest.mark.parametrize(
1725
+ ("check_values", "record", "expected"),
1726
+ [
1727
+ ({"geq_to": "2020-01-01"}, "2021-06-30", True),
1728
+ ({"geq_to": "2020-01-01"}, "2019-12-31", False),
1729
+ ({"greater_than": "2020-01-01"}, "2020-01-01", False),
1730
+ ({"leq_to": "2020-01-01"}, "2019-12-31", True),
1731
+ ({"less_than": "2020-01-01"}, "2020-01-01", False),
1732
+ ({"equal_to": "abc"}, "abc", True),
1733
+ ({"equal_to": "abc"}, "abcd", False),
1734
+ (
1735
+ {"geq_to": datetime.date(2020, 1, 1), "leq_to": datetime.date(2020, 12, 31)},
1736
+ datetime.date(2020, 6, 30),
1737
+ True,
1738
+ ),
1739
+ (
1740
+ {"geq_to": datetime.date(2020, 1, 1), "leq_to": datetime.date(2020, 12, 31)},
1741
+ datetime.date(2021, 1, 1),
1742
+ False,
1743
+ ),
1744
+ ],
1745
+ )
1746
+ def test_get_match_compares_non_numeric_bounds_without_tolerance(self, check_values, record, expected):
1747
+ op = self._make_operator({"col": {"min": {"geq_to": 1}}})
1748
+ assert op._get_match(check_values, record) == expected
1749
+
1750
+ def test_get_match_equal_to_fails_cleanly_on_none_record(self):
1751
+ op = self._make_operator({"col": {"null_check": {"equal_to": 0}}}, accept_none=False)
1752
+ assert op._get_match({"equal_to": 0}, None) is False
1753
+
1687
1754
  def test_multiple_checks_correct_names_and_order(self):
1688
1755
  op = self._make_operator(
1689
1756
  {
@@ -2044,6 +2111,16 @@ class TestSQLValueCheckOperatorBuildCheckResults:
2044
2111
  assert r.expected == ">= 4.5, <= 5.5"
2045
2112
  assert r.params == {"pass_value": "5", "tolerance": 0.1}
2046
2113
 
2114
+ def test_negative_numeric_tolerance_produces_accepted_range(self):
2115
+ op = self._make_operator(pass_value=-100, tolerance=0.1)
2116
+ results = op._build_check_results([-100])
2117
+ assert len(results) == 1
2118
+ r = results[0]
2119
+ assert r.success is True
2120
+ assert r.check_type == "accepted_range"
2121
+ assert r.expected == ">= -110.0, <= -90.0"
2122
+ assert r.params == {"pass_value": "-100", "tolerance": 0.1}
2123
+
2047
2124
  def test_non_numeric_pass_value_is_accepted_values(self):
2048
2125
  op = self._make_operator(pass_value="hello")
2049
2126
  results = op._build_check_results(["hello"])
@@ -2957,3 +3034,86 @@ class TestSQLInsertRowsOperator:
2957
3034
  (4, "Lundgren", "Dolph", 66),
2958
3035
  (5, "Norris", "Chuck", 84),
2959
3036
  ]
3037
+
3038
+
3039
+ class TestSQLBulkLoadOperator:
3040
+ def _construct_operator(self, table, tmp_file, **kwargs):
3041
+ dag = DAG("test_dag", schedule=None, start_date=datetime.datetime(2017, 1, 1))
3042
+ return SQLBulkLoadOperator(
3043
+ task_id="test_task",
3044
+ conn_id="default_conn",
3045
+ table=table,
3046
+ tmp_file=tmp_file,
3047
+ dag=dag,
3048
+ **kwargs,
3049
+ )
3050
+
3051
+ @mock.patch.object(SQLBulkLoadOperator, "get_db_hook")
3052
+ def test_execute(self, mock_get_db_hook):
3053
+ operator = self._construct_operator(
3054
+ table="users",
3055
+ tmp_file="/tmp/users.tsv",
3056
+ preoperator="CREATE TABLE users (id INT);",
3057
+ postoperator="DROP TABLE users;",
3058
+ )
3059
+
3060
+ operator.execute(context=MagicMock())
3061
+
3062
+ hook = mock_get_db_hook.return_value
3063
+
3064
+ assert hook.mock_calls == [
3065
+ mock.call.run("CREATE TABLE users (id INT);"),
3066
+ mock.call.bulk_load(
3067
+ table="users",
3068
+ tmp_file="/tmp/users.tsv",
3069
+ ),
3070
+ mock.call.run("DROP TABLE users;"),
3071
+ ]
3072
+
3073
+ @mock.patch.object(SQLBulkLoadOperator, "get_db_hook")
3074
+ def test_execute_templated_fields(self, mock_get_db_hook):
3075
+ operator = self._construct_operator(
3076
+ table="{{ params.table }}",
3077
+ tmp_file="{{ params.file }}",
3078
+ preoperator="TRUNCATE TABLE {{ params.table }};",
3079
+ postoperator="DROP TABLE {{ params.table }};",
3080
+ )
3081
+
3082
+ operator.render_template_fields(
3083
+ {
3084
+ "params": {
3085
+ "table": "users",
3086
+ "file": "/tmp/users.tsv",
3087
+ }
3088
+ }
3089
+ )
3090
+
3091
+ operator.execute(context=MagicMock())
3092
+
3093
+ hook = mock_get_db_hook.return_value
3094
+
3095
+ assert hook.mock_calls == [
3096
+ mock.call.run("TRUNCATE TABLE users;"),
3097
+ mock.call.bulk_load(
3098
+ table="users",
3099
+ tmp_file="/tmp/users.tsv",
3100
+ ),
3101
+ mock.call.run("DROP TABLE users;"),
3102
+ ]
3103
+
3104
+ @mock.patch.object(SQLBulkLoadOperator, "get_db_hook")
3105
+ def test_execute_without_pre_or_post_operator(self, mock_get_db_hook):
3106
+ operator = self._construct_operator(
3107
+ table="users",
3108
+ tmp_file="/tmp/users.tsv",
3109
+ )
3110
+
3111
+ operator.execute(context=MagicMock())
3112
+
3113
+ hook = mock_get_db_hook.return_value
3114
+
3115
+ hook.run.assert_not_called()
3116
+ hook.bulk_load.assert_called_once_with(
3117
+ table="users",
3118
+ tmp_file="/tmp/users.tsv",
3119
+ )