apache-airflow-providers-common-sql 1.8.1rc1__py3-none-any.whl → 1.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of apache-airflow-providers-common-sql might be problematic. Click here for more details.
- airflow/providers/common/sql/__init__.py +3 -3
- airflow/providers/common/sql/doc/adr/0001-record-architecture-decisions.md +48 -0
- airflow/providers/common/sql/doc/adr/0002-return-common-data-structure-from-dbapihook-derived-hooks.md +139 -0
- airflow/providers/common/sql/get_provider_info.py +3 -2
- {apache_airflow_providers_common_sql-1.8.1rc1.dist-info → apache_airflow_providers_common_sql-1.9.0.dist-info}/METADATA +8 -8
- {apache_airflow_providers_common_sql-1.8.1rc1.dist-info → apache_airflow_providers_common_sql-1.9.0.dist-info}/RECORD +8 -6
- {apache_airflow_providers_common_sql-1.8.1rc1.dist-info → apache_airflow_providers_common_sql-1.9.0.dist-info}/WHEEL +0 -0
- {apache_airflow_providers_common_sql-1.8.1rc1.dist-info → apache_airflow_providers_common_sql-1.9.0.dist-info}/entry_points.txt +0 -0
|
@@ -27,7 +27,7 @@ import packaging.version
|
|
|
27
27
|
|
|
28
28
|
__all__ = ["__version__"]
|
|
29
29
|
|
|
30
|
-
__version__ = "1.
|
|
30
|
+
__version__ = "1.9.0"
|
|
31
31
|
|
|
32
32
|
try:
|
|
33
33
|
from airflow import __version__ as airflow_version
|
|
@@ -35,8 +35,8 @@ except ImportError:
|
|
|
35
35
|
from airflow.version import version as airflow_version
|
|
36
36
|
|
|
37
37
|
if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
|
|
38
|
-
"2.
|
|
38
|
+
"2.6.0"
|
|
39
39
|
):
|
|
40
40
|
raise RuntimeError(
|
|
41
|
-
f"The package `apache-airflow-providers-common-sql:{__version__}` needs Apache Airflow 2.
|
|
41
|
+
f"The package `apache-airflow-providers-common-sql:{__version__}` needs Apache Airflow 2.6.0+"
|
|
42
42
|
)
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
<!--
|
|
2
|
+
Licensed to the Apache Software Foundation (ASF) under one
|
|
3
|
+
or more contributor license agreements. See the NOTICE file
|
|
4
|
+
distributed with this work for additional information
|
|
5
|
+
regarding copyright ownership. The ASF licenses this file
|
|
6
|
+
to you under the Apache License, Version 2.0 (the
|
|
7
|
+
"License"); you may not use this file except in compliance
|
|
8
|
+
with the License. You may obtain a copy of the License at
|
|
9
|
+
|
|
10
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
11
|
+
|
|
12
|
+
Unless required by applicable law or agreed to in writing,
|
|
13
|
+
software distributed under the License is distributed on an
|
|
14
|
+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
15
|
+
KIND, either express or implied. See the License for the
|
|
16
|
+
specific language governing permissions and limitations
|
|
17
|
+
under the License.
|
|
18
|
+
-->
|
|
19
|
+
|
|
20
|
+
<!-- START doctoc generated TOC please keep comment here to allow auto update -->
|
|
21
|
+
<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
|
|
22
|
+
**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)*
|
|
23
|
+
|
|
24
|
+
- [1. Record architecture decisions](#1-record-architecture-decisions)
|
|
25
|
+
- [Status](#status)
|
|
26
|
+
- [Context](#context)
|
|
27
|
+
- [Decision](#decision)
|
|
28
|
+
- [Consequences](#consequences)
|
|
29
|
+
|
|
30
|
+
# 1. Record architecture decisions
|
|
31
|
+
|
|
32
|
+
Date: 2023-12-01
|
|
33
|
+
|
|
34
|
+
## Status
|
|
35
|
+
|
|
36
|
+
Accepted
|
|
37
|
+
|
|
38
|
+
## Context
|
|
39
|
+
|
|
40
|
+
We need to record the architectural decisions made on this project.
|
|
41
|
+
|
|
42
|
+
## Decision
|
|
43
|
+
|
|
44
|
+
We will use Architecture Decision Records, as [described by Michael Nygard](http://thinkrelevance.com/blog/2011/11/15/documenting-architecture-decisions).
|
|
45
|
+
|
|
46
|
+
## Consequences
|
|
47
|
+
|
|
48
|
+
See Michael Nygard's article, linked above. For a lightweight ADR toolset, see Nat Pryce's [adr-tools](https://github.com/npryce/adr-tools).
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
<!--
|
|
2
|
+
Licensed to the Apache Software Foundation (ASF) under one
|
|
3
|
+
or more contributor license agreements. See the NOTICE file
|
|
4
|
+
distributed with this work for additional information
|
|
5
|
+
regarding copyright ownership. The ASF licenses this file
|
|
6
|
+
to you under the Apache License, Version 2.0 (the
|
|
7
|
+
"License"); you may not use this file except in compliance
|
|
8
|
+
with the License. You may obtain a copy of the License at
|
|
9
|
+
|
|
10
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
11
|
+
|
|
12
|
+
Unless required by applicable law or agreed to in writing,
|
|
13
|
+
software distributed under the License is distributed on an
|
|
14
|
+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
15
|
+
KIND, either express or implied. See the License for the
|
|
16
|
+
specific language governing permissions and limitations
|
|
17
|
+
under the License.
|
|
18
|
+
-->
|
|
19
|
+
|
|
20
|
+
# 2. Return common data structure from DBApiHook derived hooks
|
|
21
|
+
|
|
22
|
+
Date: 2023-12-01
|
|
23
|
+
|
|
24
|
+
## Status
|
|
25
|
+
|
|
26
|
+
Accepted
|
|
27
|
+
|
|
28
|
+
## Context
|
|
29
|
+
|
|
30
|
+
Note: This ADR describes the decision made (but not recorded) when common.sql provider had been
|
|
31
|
+
introduced in July 2022, but this ADR is recorded in December 2023 to make sure the decision is
|
|
32
|
+
recorded.
|
|
33
|
+
|
|
34
|
+
Before common.sql provider, we had a number of DBAPI-derived Hooks which interfaced with Python
|
|
35
|
+
DBAPI-compliant database and the format of data returned from these hooks were different and
|
|
36
|
+
dependent on the implementation of the DBAPI interface as well as implementation of the Hooks and operators.
|
|
37
|
+
We also had a lot of very similar operators that performed similar tasks - like Querying, Sensing the
|
|
38
|
+
database, column check etc. This led to a lot of code duplication, and we decided that we need a common set
|
|
39
|
+
of operators that can be used across all the DBAPI-compliant databases.
|
|
40
|
+
|
|
41
|
+
Unfortunately there is no common standard for data returned by Python DBAPI interface. It's partially
|
|
42
|
+
standardized by [PEP-0249](https://peps.python.org/pep-0249/), but the specification is not very
|
|
43
|
+
strict and it allows for a lot of flexibility and interpretation. Consequently, the data returned
|
|
44
|
+
by the DBAPI interface can contain tuples, named tuples, lists, dictionaries, or even custom objects and
|
|
45
|
+
there are no guarantees that the data returned by the DBAPI interface is directly serializable.
|
|
46
|
+
|
|
47
|
+
Not having a common standard format made it difficult to implement planned open-lineage column-level
|
|
48
|
+
integration with the database Operators and - in later stage Hooks.
|
|
49
|
+
|
|
50
|
+
## Decision
|
|
51
|
+
|
|
52
|
+
We decided to introduce a common.sql provider that would contain a set of operators that can be used
|
|
53
|
+
across all the DBAPI-compliant databases. We also decided that the returned data structure from the
|
|
54
|
+
operators should be consistent across all the operators. For simplicity of transition we chose the format
|
|
55
|
+
returned by the `run` method of the ``DBApiHook`` class that was very close to what most of the
|
|
56
|
+
DBAPI-compliant Hooks already returned, even if it was not the most optimal format. In this case
|
|
57
|
+
backwards compatibility trumped the optimal format.
|
|
58
|
+
|
|
59
|
+
The decision has been made that more optimal formats (possibly some form of DataFrames) might be
|
|
60
|
+
introduced in the future if we find a need for that. However, this is likely not even needed in the
|
|
61
|
+
future because Pandas and similar libraries already have excellent support for converting many of
|
|
62
|
+
the formats returned by the DBAPI interface to DataFrames and we are already leveraging those
|
|
63
|
+
by directly using Pandas functionality via `get_pands_df` method of the ``DBApiHook`` class
|
|
64
|
+
|
|
65
|
+
The goal of the change was to standardize the format of the data returned that could be
|
|
66
|
+
directly used through existing DBAPI Airflow operators, with minimal backwards-compatibility problems,
|
|
67
|
+
and resulting in deprecating and redirecting all the existing DBAPI operators to the new operators
|
|
68
|
+
defined in the common.sql provider.
|
|
69
|
+
|
|
70
|
+
Therefore, the format of data returned by the Hook can be one of:
|
|
71
|
+
|
|
72
|
+
* base return value is a list of tuples where each tuple is a row returned by the query. The tuples
|
|
73
|
+
are not named tuples, they should be directly serializable and they should not contain
|
|
74
|
+
column metadata like column names.
|
|
75
|
+
* a tuple (same properties as above) - in case the query returns a single row -
|
|
76
|
+
based on parameters passed (e.g. `return_last=True`)
|
|
77
|
+
* it can also be a list of list of tuples in case the operator receives multiple queries to execute,
|
|
78
|
+
in which case the return value is a list of list of tuples, where each list of tuples is a result
|
|
79
|
+
of a single query. This is also the default format when ``split_statements`` parameter is set to
|
|
80
|
+
``True`` indicating the query passed contains multiple statements and it's up to the implementation
|
|
81
|
+
of the hook to split the statements and execute them separately.
|
|
82
|
+
* None - in case the run method is used to execute a query that does not return any rows (e.g. `INSERT`)
|
|
83
|
+
|
|
84
|
+
Additionally, it has been agreed to that description returned by the `run` method should be stored in
|
|
85
|
+
the ``descriptions`` field of the Hook - separately from the returned data. This is because the description
|
|
86
|
+
is not always available in the DBAPI interface, and it's not always available in the same format - the
|
|
87
|
+
format of ``descriptions`` field is that it is always an array of 7-element tuples described in PEP-0249 -
|
|
88
|
+
each element of the array correspond to a single query passed to the hook. There is also a separate
|
|
89
|
+
``last_description`` property that contains the description of the last query passed to the hook,
|
|
90
|
+
particularly, the only query if there was only one.
|
|
91
|
+
|
|
92
|
+
The DBApiHook implements the base `run` method that returns the list of tuples directly from the DBAPI
|
|
93
|
+
result, provide. The `run` method is extendable - it can receive handlers that can modify the data
|
|
94
|
+
returned by the direct DBAPI calls to the common format.
|
|
95
|
+
|
|
96
|
+
For backwards compatibility, the Hooks could implement their own parameters that should allow to return
|
|
97
|
+
the "old" format of the data. For example in SnowflakeHook we have `return_dict` parameter that allows
|
|
98
|
+
to return the data as a list of dictionaries instead of a list of tuples. This allowed to keep
|
|
99
|
+
easy backwards compatibility with the existing operators that our users already had, and allowed for
|
|
100
|
+
an easy, even gradual transition to new common format if the user would see the benefit of doing so
|
|
101
|
+
(for example to support column based open-lineage or simplify the usage of deprecated operators
|
|
102
|
+
and switch to the new ones from the old, deprecated specialized operators.
|
|
103
|
+
|
|
104
|
+
We also decided that the common.sql provider implements the operators that can instantiate the Hooks
|
|
105
|
+
based on the connection type/URI and execute the commands using that hook, while deprecating the use
|
|
106
|
+
of the specialized ones:
|
|
107
|
+
|
|
108
|
+
The new operators are:
|
|
109
|
+
|
|
110
|
+
* ``SQLExecuteQueryOperator`` - to execute SQL queries
|
|
111
|
+
* ``SQLColumnCheckOperator`` - to perform various checks (declarative) on columns in the database and
|
|
112
|
+
return matching rows
|
|
113
|
+
* ``SQLTableCheckOperator`` - to perform various checks (declarative) on tables in the database and
|
|
114
|
+
return calculated results (usually aggregated calculations)
|
|
115
|
+
* ``SQLCheckOperator`` - to perform various checks (declarative) on the database and return success/failure
|
|
116
|
+
based on the checks performed
|
|
117
|
+
* ``SQLValueCheckOperator`` - to perform various checks (declarative) on values in the database and
|
|
118
|
+
return matching rows
|
|
119
|
+
* ``SQLIntervalCheckOperator`` - to perform various checks (declarative) on intervals in the database and
|
|
120
|
+
return matching rows
|
|
121
|
+
* ``BranchSQLOperator``- branch operator where branching is based on sql and parameters provided
|
|
122
|
+
* ``SQLSensor`` - sensor that waits for a certain condition to be met in the database
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
## Consequences
|
|
126
|
+
|
|
127
|
+
The consequence of this decision is that the data returned by the operators in the common.sql provider
|
|
128
|
+
is consistent across all the operators and it's easy to use it for various databases in similar way
|
|
129
|
+
and easy to integrate with other components like open-lineage. The data returned by the operators
|
|
130
|
+
is not the most optimal format for all the databases, but it's a good compromise that allows for
|
|
131
|
+
easy transition and backwards compatibility with the existing operators.
|
|
132
|
+
|
|
133
|
+
Column-lineage integration is possible with the data returned by the operators in the common.sql
|
|
134
|
+
provider as well as returning the data directly by the hook to be stored in XCom or other ways SQL output
|
|
135
|
+
can be serialized to (for example CSV records).
|
|
136
|
+
|
|
137
|
+
A number of SQL/DBAPI operators implemented is therefore much smaller and the code to maintain is
|
|
138
|
+
only present in the common.sql provider. The code is also much more consistent and easier to maintain
|
|
139
|
+
and extend.
|
|
@@ -28,8 +28,9 @@ def get_provider_info():
|
|
|
28
28
|
"name": "Common SQL",
|
|
29
29
|
"description": "`Common SQL Provider <https://en.wikipedia.org/wiki/SQL>`__\n",
|
|
30
30
|
"suspended": False,
|
|
31
|
-
"source-date-epoch":
|
|
31
|
+
"source-date-epoch": 1701983370,
|
|
32
32
|
"versions": [
|
|
33
|
+
"1.9.0",
|
|
33
34
|
"1.8.1",
|
|
34
35
|
"1.8.0",
|
|
35
36
|
"1.7.2",
|
|
@@ -51,7 +52,7 @@ def get_provider_info():
|
|
|
51
52
|
"1.1.0",
|
|
52
53
|
"1.0.0",
|
|
53
54
|
],
|
|
54
|
-
"dependencies": ["apache-airflow>=2.
|
|
55
|
+
"dependencies": ["apache-airflow>=2.6.0", "sqlparse>=0.4.2"],
|
|
55
56
|
"additional-extras": [{"name": "pandas", "dependencies": ["pandas>=0.17.1"]}],
|
|
56
57
|
"integrations": [
|
|
57
58
|
{
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: apache-airflow-providers-common-sql
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.9.0
|
|
4
4
|
Summary: Provider package apache-airflow-providers-common-sql for Apache Airflow
|
|
5
5
|
Keywords: airflow-provider,common.sql,airflow,integration
|
|
6
6
|
Author-email: Apache Software Foundation <dev@airflow.apache.org>
|
|
@@ -20,13 +20,13 @@ Classifier: Programming Language :: Python :: 3.9
|
|
|
20
20
|
Classifier: Programming Language :: Python :: 3.10
|
|
21
21
|
Classifier: Programming Language :: Python :: 3.11
|
|
22
22
|
Classifier: Topic :: System :: Monitoring
|
|
23
|
-
Requires-Dist: apache-airflow>=2.
|
|
23
|
+
Requires-Dist: apache-airflow>=2.6.0
|
|
24
24
|
Requires-Dist: sqlparse>=0.4.2
|
|
25
25
|
Requires-Dist: apache-airflow-providers-openlineage ; extra == "openlineage"
|
|
26
26
|
Requires-Dist: pandas>=0.17.1 ; extra == "pandas"
|
|
27
27
|
Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
|
|
28
|
-
Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-common-sql/1.
|
|
29
|
-
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-common-sql/1.
|
|
28
|
+
Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-common-sql/1.9.0/changelog.html
|
|
29
|
+
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-common-sql/1.9.0
|
|
30
30
|
Project-URL: Slack Chat, https://s.apache.org/airflow-slack
|
|
31
31
|
Project-URL: Source Code, https://github.com/apache/airflow
|
|
32
32
|
Project-URL: Twitter, https://twitter.com/ApacheAirflow
|
|
@@ -78,7 +78,7 @@ Provides-Extra: pandas
|
|
|
78
78
|
|
|
79
79
|
Package ``apache-airflow-providers-common-sql``
|
|
80
80
|
|
|
81
|
-
Release: ``1.
|
|
81
|
+
Release: ``1.9.0``
|
|
82
82
|
|
|
83
83
|
|
|
84
84
|
`Common SQL Provider <https://en.wikipedia.org/wiki/SQL>`__
|
|
@@ -91,7 +91,7 @@ This is a provider package for ``common.sql`` provider. All classes for this pro
|
|
|
91
91
|
are in ``airflow.providers.common.sql`` python package.
|
|
92
92
|
|
|
93
93
|
You can find package information and changelog for the provider
|
|
94
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/1.
|
|
94
|
+
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/1.9.0/>`_.
|
|
95
95
|
|
|
96
96
|
Installation
|
|
97
97
|
------------
|
|
@@ -108,7 +108,7 @@ Requirements
|
|
|
108
108
|
================== ==================
|
|
109
109
|
PIP package Version required
|
|
110
110
|
================== ==================
|
|
111
|
-
``apache-airflow`` ``>=2.
|
|
111
|
+
``apache-airflow`` ``>=2.6.0``
|
|
112
112
|
``sqlparse`` ``>=0.4.2``
|
|
113
113
|
================== ==================
|
|
114
114
|
|
|
@@ -132,4 +132,4 @@ Dependent package
|
|
|
132
132
|
============================================================================================================== ===============
|
|
133
133
|
|
|
134
134
|
The changelog for the provider package can be found in the
|
|
135
|
-
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/1.
|
|
135
|
+
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/1.9.0/changelog.html>`_.
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
airflow/providers/common/sql/LICENSE,sha256=ywUBpKZc7Jb96rVt5I3IDbg7dIJAbUSHkuoDcF3jbH4,13569
|
|
2
2
|
airflow/providers/common/sql/README_API.md,sha256=CxhaS8EedZ4dcbLUPC4-GLCMaY3OH96oHxXttUGU06E,5932
|
|
3
|
-
airflow/providers/common/sql/__init__.py,sha256=
|
|
4
|
-
airflow/providers/common/sql/get_provider_info.py,sha256=
|
|
3
|
+
airflow/providers/common/sql/__init__.py,sha256=GUWCXqAy3zJ4t-dpvCANEfZ5wv88L3v34Me1V6RqWQg,1585
|
|
4
|
+
airflow/providers/common/sql/get_provider_info.py,sha256=Oq35Nk7qQdmhu3MLublr6PA5KibZ_JHSsaNIqkI9wXc,2837
|
|
5
|
+
airflow/providers/common/sql/doc/adr/0001-record-architecture-decisions.md,sha256=TfANqrzoFto9PMOMza3MitIkXHGLx2kY_BhhF-N0_ow,1675
|
|
6
|
+
airflow/providers/common/sql/doc/adr/0002-return-common-data-structure-from-dbapihook-derived-hooks.md,sha256=ze5w9IVS-HkUwdZvPW8_JaJaVwel7-N6XdEVN4pTuCE,8457
|
|
5
7
|
airflow/providers/common/sql/hooks/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
|
6
8
|
airflow/providers/common/sql/hooks/sql.py,sha256=-ZcSWjt1oBgDT3f1h81-nWE_GfRzhXQ8lxvU-zE7wwM,25069
|
|
7
9
|
airflow/providers/common/sql/hooks/sql.pyi,sha256=VWDJZEOnvW4si_ablFHXBmG9K6fQJoEKL4YA6U21Sbc,4072
|
|
@@ -11,7 +13,7 @@ airflow/providers/common/sql/operators/sql.pyi,sha256=-Wa-4uMtRPvUowYgSDnfH98Joe
|
|
|
11
13
|
airflow/providers/common/sql/sensors/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
|
12
14
|
airflow/providers/common/sql/sensors/sql.py,sha256=bqyaw3pQpvCRJ5h2R2eZiJJuGtW-6g4RcI-5WaREPRk,5669
|
|
13
15
|
airflow/providers/common/sql/sensors/sql.pyi,sha256=ZwVia3SUHrW7eB98r3vHYT_jhgkSWHRZqA2srYDHVbc,2295
|
|
14
|
-
apache_airflow_providers_common_sql-1.
|
|
15
|
-
apache_airflow_providers_common_sql-1.
|
|
16
|
-
apache_airflow_providers_common_sql-1.
|
|
17
|
-
apache_airflow_providers_common_sql-1.
|
|
16
|
+
apache_airflow_providers_common_sql-1.9.0.dist-info/entry_points.txt,sha256=h8UXRp2crPuGmYVYRM5oe168qIh7g-4t2QQbVMizKjI,106
|
|
17
|
+
apache_airflow_providers_common_sql-1.9.0.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
|
|
18
|
+
apache_airflow_providers_common_sql-1.9.0.dist-info/METADATA,sha256=u4T9w_MFetci3v2YJH24Z50yPrrMEN5-ECLbyNAJyNQ,5973
|
|
19
|
+
apache_airflow_providers_common_sql-1.9.0.dist-info/RECORD,,
|
|
File without changes
|