apache-airflow-providers-postgres 6.3.0__tar.gz → 6.4.0rc1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of apache-airflow-providers-postgres might be problematic. Click here for more details.
- {apache_airflow_providers_postgres-6.3.0 → apache_airflow_providers_postgres-6.4.0rc1}/PKG-INFO +45 -25
- {apache_airflow_providers_postgres-6.3.0 → apache_airflow_providers_postgres-6.4.0rc1}/README.rst +36 -19
- {apache_airflow_providers_postgres-6.3.0 → apache_airflow_providers_postgres-6.4.0rc1}/docs/changelog.rst +23 -2
- apache_airflow_providers_postgres-6.4.0rc1/docs/configurations-ref.rst +19 -0
- {apache_airflow_providers_postgres-6.3.0 → apache_airflow_providers_postgres-6.4.0rc1}/docs/connections/postgres.rst +14 -1
- {apache_airflow_providers_postgres-6.3.0 → apache_airflow_providers_postgres-6.4.0rc1}/docs/index.rst +23 -19
- {apache_airflow_providers_postgres-6.3.0 → apache_airflow_providers_postgres-6.4.0rc1}/provider.yaml +16 -1
- {apache_airflow_providers_postgres-6.3.0 → apache_airflow_providers_postgres-6.4.0rc1}/pyproject.toml +12 -6
- {apache_airflow_providers_postgres-6.3.0 → apache_airflow_providers_postgres-6.4.0rc1}/src/airflow/providers/postgres/__init__.py +1 -1
- {apache_airflow_providers_postgres-6.3.0 → apache_airflow_providers_postgres-6.4.0rc1}/src/airflow/providers/postgres/dialects/postgres.py +36 -43
- {apache_airflow_providers_postgres-6.3.0 → apache_airflow_providers_postgres-6.4.0rc1}/src/airflow/providers/postgres/get_provider_info.py +14 -0
- {apache_airflow_providers_postgres-6.3.0 → apache_airflow_providers_postgres-6.4.0rc1}/src/airflow/providers/postgres/hooks/postgres.py +43 -6
- {apache_airflow_providers_postgres-6.3.0 → apache_airflow_providers_postgres-6.4.0rc1}/tests/unit/postgres/assets/test_postgres.py +7 -2
- {apache_airflow_providers_postgres-6.3.0 → apache_airflow_providers_postgres-6.4.0rc1}/tests/unit/postgres/hooks/test_postgres.py +56 -13
- {apache_airflow_providers_postgres-6.3.0 → apache_airflow_providers_postgres-6.4.0rc1}/docs/.latest-doc-only-change.txt +0 -0
- {apache_airflow_providers_postgres-6.3.0 → apache_airflow_providers_postgres-6.4.0rc1}/docs/commits.rst +0 -0
- {apache_airflow_providers_postgres-6.3.0 → apache_airflow_providers_postgres-6.4.0rc1}/docs/conf.py +0 -0
- {apache_airflow_providers_postgres-6.3.0 → apache_airflow_providers_postgres-6.4.0rc1}/docs/dialects.rst +0 -0
- {apache_airflow_providers_postgres-6.3.0 → apache_airflow_providers_postgres-6.4.0rc1}/docs/installing-providers-from-sources.rst +0 -0
- {apache_airflow_providers_postgres-6.3.0 → apache_airflow_providers_postgres-6.4.0rc1}/docs/integration-logos/Postgres.png +0 -0
- {apache_airflow_providers_postgres-6.3.0 → apache_airflow_providers_postgres-6.4.0rc1}/docs/operators.rst +0 -0
- {apache_airflow_providers_postgres-6.3.0 → apache_airflow_providers_postgres-6.4.0rc1}/docs/redirects.txt +0 -0
- {apache_airflow_providers_postgres-6.3.0 → apache_airflow_providers_postgres-6.4.0rc1}/docs/security.rst +0 -0
- {apache_airflow_providers_postgres-6.3.0 → apache_airflow_providers_postgres-6.4.0rc1}/src/airflow/__init__.py +0 -0
- {apache_airflow_providers_postgres-6.3.0 → apache_airflow_providers_postgres-6.4.0rc1}/src/airflow/providers/__init__.py +0 -0
- {apache_airflow_providers_postgres-6.3.0 → apache_airflow_providers_postgres-6.4.0rc1}/src/airflow/providers/postgres/LICENSE +0 -0
- {apache_airflow_providers_postgres-6.3.0 → apache_airflow_providers_postgres-6.4.0rc1}/src/airflow/providers/postgres/assets/__init__.py +0 -0
- {apache_airflow_providers_postgres-6.3.0 → apache_airflow_providers_postgres-6.4.0rc1}/src/airflow/providers/postgres/assets/postgres.py +0 -0
- {apache_airflow_providers_postgres-6.3.0 → apache_airflow_providers_postgres-6.4.0rc1}/src/airflow/providers/postgres/dialects/__init__.py +0 -0
- {apache_airflow_providers_postgres-6.3.0 → apache_airflow_providers_postgres-6.4.0rc1}/src/airflow/providers/postgres/hooks/__init__.py +0 -0
- {apache_airflow_providers_postgres-6.3.0 → apache_airflow_providers_postgres-6.4.0rc1}/tests/conftest.py +0 -0
- {apache_airflow_providers_postgres-6.3.0 → apache_airflow_providers_postgres-6.4.0rc1}/tests/system/__init__.py +0 -0
- {apache_airflow_providers_postgres-6.3.0 → apache_airflow_providers_postgres-6.4.0rc1}/tests/system/postgres/__init__.py +0 -0
- {apache_airflow_providers_postgres-6.3.0 → apache_airflow_providers_postgres-6.4.0rc1}/tests/system/postgres/example_postgres.py +0 -0
- {apache_airflow_providers_postgres-6.3.0 → apache_airflow_providers_postgres-6.4.0rc1}/tests/unit/__init__.py +0 -0
- {apache_airflow_providers_postgres-6.3.0 → apache_airflow_providers_postgres-6.4.0rc1}/tests/unit/postgres/__init__.py +0 -0
- {apache_airflow_providers_postgres-6.3.0 → apache_airflow_providers_postgres-6.4.0rc1}/tests/unit/postgres/assets/__init__.py +0 -0
- {apache_airflow_providers_postgres-6.3.0 → apache_airflow_providers_postgres-6.4.0rc1}/tests/unit/postgres/dialects/__init__.py +0 -0
- {apache_airflow_providers_postgres-6.3.0 → apache_airflow_providers_postgres-6.4.0rc1}/tests/unit/postgres/dialects/test_postgres.py +0 -0
- {apache_airflow_providers_postgres-6.3.0 → apache_airflow_providers_postgres-6.4.0rc1}/tests/unit/postgres/hooks/__init__.py +0 -0
{apache_airflow_providers_postgres-6.3.0 → apache_airflow_providers_postgres-6.4.0rc1}/PKG-INFO
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: apache-airflow-providers-postgres
|
|
3
|
-
Version: 6.
|
|
3
|
+
Version: 6.4.0rc1
|
|
4
4
|
Summary: Provider package apache-airflow-providers-postgres for Apache Airflow
|
|
5
5
|
Keywords: airflow-provider,postgres,airflow,integration
|
|
6
6
|
Author-email: Apache Software Foundation <dev@airflow.apache.org>
|
|
@@ -20,25 +20,28 @@ Classifier: Programming Language :: Python :: 3.11
|
|
|
20
20
|
Classifier: Programming Language :: Python :: 3.12
|
|
21
21
|
Classifier: Programming Language :: Python :: 3.13
|
|
22
22
|
Classifier: Topic :: System :: Monitoring
|
|
23
|
-
Requires-Dist: apache-airflow>=2.10.
|
|
24
|
-
Requires-Dist: apache-airflow-providers-common-
|
|
23
|
+
Requires-Dist: apache-airflow>=2.10.0rc1
|
|
24
|
+
Requires-Dist: apache-airflow-providers-common-compat>=1.8.0rc1
|
|
25
|
+
Requires-Dist: apache-airflow-providers-common-sql>=1.23.0rc1
|
|
25
26
|
Requires-Dist: psycopg2-binary>=2.9.9; python_version < '3.13'
|
|
26
27
|
Requires-Dist: psycopg2-binary>=2.9.10; python_version >= '3.13'
|
|
27
28
|
Requires-Dist: asyncpg>=0.30.0
|
|
28
|
-
Requires-Dist: apache-airflow-providers-amazon>=2.6.
|
|
29
|
+
Requires-Dist: apache-airflow-providers-amazon>=2.6.0rc1 ; extra == "amazon"
|
|
30
|
+
Requires-Dist: apache-airflow-providers-microsoft-azure ; extra == "microsoft-azure"
|
|
29
31
|
Requires-Dist: apache-airflow-providers-openlineage ; extra == "openlineage"
|
|
30
32
|
Requires-Dist: pandas>=2.1.2 ; extra == "pandas" and ( python_version <"3.13")
|
|
31
33
|
Requires-Dist: pandas>=2.2.3 ; extra == "pandas" and ( python_version >="3.13")
|
|
32
34
|
Requires-Dist: polars>=1.26.0 ; extra == "polars"
|
|
33
35
|
Requires-Dist: psycopg[binary]>=3.2.9 ; extra == "psycopg"
|
|
34
36
|
Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
|
|
35
|
-
Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-postgres/6.
|
|
36
|
-
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-postgres/6.
|
|
37
|
+
Project-URL: Changelog, https://airflow.staged.apache.org/docs/apache-airflow-providers-postgres/6.4.0/changelog.html
|
|
38
|
+
Project-URL: Documentation, https://airflow.staged.apache.org/docs/apache-airflow-providers-postgres/6.4.0
|
|
37
39
|
Project-URL: Mastodon, https://fosstodon.org/@airflow
|
|
38
40
|
Project-URL: Slack Chat, https://s.apache.org/airflow-slack
|
|
39
41
|
Project-URL: Source Code, https://github.com/apache/airflow
|
|
40
42
|
Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
|
|
41
43
|
Provides-Extra: amazon
|
|
44
|
+
Provides-Extra: microsoft-azure
|
|
42
45
|
Provides-Extra: openlineage
|
|
43
46
|
Provides-Extra: pandas
|
|
44
47
|
Provides-Extra: polars
|
|
@@ -69,7 +72,7 @@ Provides-Extra: psycopg
|
|
|
69
72
|
|
|
70
73
|
Package ``apache-airflow-providers-postgres``
|
|
71
74
|
|
|
72
|
-
Release: ``6.
|
|
75
|
+
Release: ``6.4.0``
|
|
73
76
|
|
|
74
77
|
|
|
75
78
|
`PostgreSQL <https://www.postgresql.org/>`__
|
|
@@ -82,7 +85,7 @@ This is a provider package for ``postgres`` provider. All classes for this provi
|
|
|
82
85
|
are in ``airflow.providers.postgres`` python package.
|
|
83
86
|
|
|
84
87
|
You can find package information and changelog for the provider
|
|
85
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-postgres/6.
|
|
88
|
+
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-postgres/6.4.0/>`_.
|
|
86
89
|
|
|
87
90
|
Installation
|
|
88
91
|
------------
|
|
@@ -96,15 +99,16 @@ The package supports the following python versions: 3.10,3.11,3.12,3.13
|
|
|
96
99
|
Requirements
|
|
97
100
|
------------
|
|
98
101
|
|
|
99
|
-
|
|
100
|
-
PIP package
|
|
101
|
-
|
|
102
|
-
``apache-airflow``
|
|
103
|
-
``apache-airflow-providers-common-
|
|
104
|
-
``
|
|
105
|
-
``psycopg2-binary``
|
|
106
|
-
``
|
|
107
|
-
|
|
102
|
+
========================================== ======================================
|
|
103
|
+
PIP package Version required
|
|
104
|
+
========================================== ======================================
|
|
105
|
+
``apache-airflow`` ``>=2.10.0``
|
|
106
|
+
``apache-airflow-providers-common-compat`` ``>=1.8.0``
|
|
107
|
+
``apache-airflow-providers-common-sql`` ``>=1.23.0``
|
|
108
|
+
``psycopg2-binary`` ``>=2.9.9; python_version < "3.13"``
|
|
109
|
+
``psycopg2-binary`` ``>=2.9.10; python_version >= "3.13"``
|
|
110
|
+
``asyncpg`` ``>=0.30.0``
|
|
111
|
+
========================================== ======================================
|
|
108
112
|
|
|
109
113
|
Cross provider package dependencies
|
|
110
114
|
-----------------------------------
|
|
@@ -119,14 +123,30 @@ You can install such cross-provider dependencies when installing from PyPI. For
|
|
|
119
123
|
pip install apache-airflow-providers-postgres[amazon]
|
|
120
124
|
|
|
121
125
|
|
|
122
|
-
|
|
123
|
-
Dependent package
|
|
124
|
-
|
|
125
|
-
`apache-airflow-providers-amazon <https://airflow.apache.org/docs/apache-airflow-providers-amazon>`_
|
|
126
|
-
`apache-airflow-providers-common-
|
|
127
|
-
`apache-airflow-providers-
|
|
128
|
-
|
|
126
|
+
====================================================================================================================== ===================
|
|
127
|
+
Dependent package Extra
|
|
128
|
+
====================================================================================================================== ===================
|
|
129
|
+
`apache-airflow-providers-amazon <https://airflow.apache.org/docs/apache-airflow-providers-amazon>`_ ``amazon``
|
|
130
|
+
`apache-airflow-providers-common-compat <https://airflow.apache.org/docs/apache-airflow-providers-common-compat>`_ ``common.compat``
|
|
131
|
+
`apache-airflow-providers-common-sql <https://airflow.apache.org/docs/apache-airflow-providers-common-sql>`_ ``common.sql``
|
|
132
|
+
`apache-airflow-providers-microsoft-azure <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure>`_ ``microsoft.azure``
|
|
133
|
+
`apache-airflow-providers-openlineage <https://airflow.apache.org/docs/apache-airflow-providers-openlineage>`_ ``openlineage``
|
|
134
|
+
====================================================================================================================== ===================
|
|
135
|
+
|
|
136
|
+
Optional dependencies
|
|
137
|
+
----------------------
|
|
138
|
+
|
|
139
|
+
=================== =====================================================================================
|
|
140
|
+
Extra Dependencies
|
|
141
|
+
=================== =====================================================================================
|
|
142
|
+
``amazon`` ``apache-airflow-providers-amazon>=2.6.0``
|
|
143
|
+
``microsoft.azure`` ``apache-airflow-providers-microsoft-azure``
|
|
144
|
+
``openlineage`` ``apache-airflow-providers-openlineage``
|
|
145
|
+
``pandas`` ``pandas>=2.1.2; python_version <"3.13"``, ``pandas>=2.2.3; python_version >="3.13"``
|
|
146
|
+
``polars`` ``polars>=1.26.0``
|
|
147
|
+
``psycopg`` ``psycopg[binary]>=3.2.9``
|
|
148
|
+
=================== =====================================================================================
|
|
129
149
|
|
|
130
150
|
The changelog for the provider package can be found in the
|
|
131
|
-
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-postgres/6.
|
|
151
|
+
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-postgres/6.4.0/changelog.html>`_.
|
|
132
152
|
|
{apache_airflow_providers_postgres-6.3.0 → apache_airflow_providers_postgres-6.4.0rc1}/README.rst
RENAMED
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
|
|
24
24
|
Package ``apache-airflow-providers-postgres``
|
|
25
25
|
|
|
26
|
-
Release: ``6.
|
|
26
|
+
Release: ``6.4.0``
|
|
27
27
|
|
|
28
28
|
|
|
29
29
|
`PostgreSQL <https://www.postgresql.org/>`__
|
|
@@ -36,7 +36,7 @@ This is a provider package for ``postgres`` provider. All classes for this provi
|
|
|
36
36
|
are in ``airflow.providers.postgres`` python package.
|
|
37
37
|
|
|
38
38
|
You can find package information and changelog for the provider
|
|
39
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-postgres/6.
|
|
39
|
+
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-postgres/6.4.0/>`_.
|
|
40
40
|
|
|
41
41
|
Installation
|
|
42
42
|
------------
|
|
@@ -50,15 +50,16 @@ The package supports the following python versions: 3.10,3.11,3.12,3.13
|
|
|
50
50
|
Requirements
|
|
51
51
|
------------
|
|
52
52
|
|
|
53
|
-
|
|
54
|
-
PIP package
|
|
55
|
-
|
|
56
|
-
``apache-airflow``
|
|
57
|
-
``apache-airflow-providers-common-
|
|
58
|
-
``
|
|
59
|
-
``psycopg2-binary``
|
|
60
|
-
``
|
|
61
|
-
|
|
53
|
+
========================================== ======================================
|
|
54
|
+
PIP package Version required
|
|
55
|
+
========================================== ======================================
|
|
56
|
+
``apache-airflow`` ``>=2.10.0``
|
|
57
|
+
``apache-airflow-providers-common-compat`` ``>=1.8.0``
|
|
58
|
+
``apache-airflow-providers-common-sql`` ``>=1.23.0``
|
|
59
|
+
``psycopg2-binary`` ``>=2.9.9; python_version < "3.13"``
|
|
60
|
+
``psycopg2-binary`` ``>=2.9.10; python_version >= "3.13"``
|
|
61
|
+
``asyncpg`` ``>=0.30.0``
|
|
62
|
+
========================================== ======================================
|
|
62
63
|
|
|
63
64
|
Cross provider package dependencies
|
|
64
65
|
-----------------------------------
|
|
@@ -73,13 +74,29 @@ You can install such cross-provider dependencies when installing from PyPI. For
|
|
|
73
74
|
pip install apache-airflow-providers-postgres[amazon]
|
|
74
75
|
|
|
75
76
|
|
|
76
|
-
|
|
77
|
-
Dependent package
|
|
78
|
-
|
|
79
|
-
`apache-airflow-providers-amazon <https://airflow.apache.org/docs/apache-airflow-providers-amazon>`_
|
|
80
|
-
`apache-airflow-providers-common-
|
|
81
|
-
`apache-airflow-providers-
|
|
82
|
-
|
|
77
|
+
====================================================================================================================== ===================
|
|
78
|
+
Dependent package Extra
|
|
79
|
+
====================================================================================================================== ===================
|
|
80
|
+
`apache-airflow-providers-amazon <https://airflow.apache.org/docs/apache-airflow-providers-amazon>`_ ``amazon``
|
|
81
|
+
`apache-airflow-providers-common-compat <https://airflow.apache.org/docs/apache-airflow-providers-common-compat>`_ ``common.compat``
|
|
82
|
+
`apache-airflow-providers-common-sql <https://airflow.apache.org/docs/apache-airflow-providers-common-sql>`_ ``common.sql``
|
|
83
|
+
`apache-airflow-providers-microsoft-azure <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure>`_ ``microsoft.azure``
|
|
84
|
+
`apache-airflow-providers-openlineage <https://airflow.apache.org/docs/apache-airflow-providers-openlineage>`_ ``openlineage``
|
|
85
|
+
====================================================================================================================== ===================
|
|
86
|
+
|
|
87
|
+
Optional dependencies
|
|
88
|
+
----------------------
|
|
89
|
+
|
|
90
|
+
=================== =====================================================================================
|
|
91
|
+
Extra Dependencies
|
|
92
|
+
=================== =====================================================================================
|
|
93
|
+
``amazon`` ``apache-airflow-providers-amazon>=2.6.0``
|
|
94
|
+
``microsoft.azure`` ``apache-airflow-providers-microsoft-azure``
|
|
95
|
+
``openlineage`` ``apache-airflow-providers-openlineage``
|
|
96
|
+
``pandas`` ``pandas>=2.1.2; python_version <"3.13"``, ``pandas>=2.2.3; python_version >="3.13"``
|
|
97
|
+
``polars`` ``polars>=1.26.0``
|
|
98
|
+
``psycopg`` ``psycopg[binary]>=3.2.9``
|
|
99
|
+
=================== =====================================================================================
|
|
83
100
|
|
|
84
101
|
The changelog for the provider package can be found in the
|
|
85
|
-
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-postgres/6.
|
|
102
|
+
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-postgres/6.4.0/changelog.html>`_.
|
|
@@ -27,11 +27,32 @@
|
|
|
27
27
|
Changelog
|
|
28
28
|
---------
|
|
29
29
|
|
|
30
|
-
6.
|
|
30
|
+
6.4.0
|
|
31
31
|
.....
|
|
32
32
|
|
|
33
|
+
Features
|
|
34
|
+
~~~~~~~~
|
|
35
|
+
|
|
36
|
+
* ``Add Azure IAM/Entra ID support for PostgresHook (#55729)``
|
|
37
|
+
|
|
38
|
+
Misc
|
|
39
|
+
~~~~
|
|
40
|
+
|
|
41
|
+
* ``fix mypy type errors in common/sql provider for sqlalchemy 2 upgrade (#56824)``
|
|
42
|
+
* ``Migrate postgres provider to ''common.compat'' (#57022)``
|
|
43
|
+
|
|
44
|
+
Doc-only
|
|
45
|
+
~~~~~~~~
|
|
46
|
+
|
|
47
|
+
* ``Remove placeholder Release Date in changelog and index files (#56056)``
|
|
48
|
+
|
|
49
|
+
.. Below changes are excluded from the changelog. Move them to
|
|
50
|
+
appropriate section above if needed. Do not delete the lines(!):
|
|
51
|
+
* ``Enable PT011 rule to prvoider tests (#55980)``
|
|
52
|
+
|
|
53
|
+
6.3.0
|
|
54
|
+
.....
|
|
33
55
|
|
|
34
|
-
Release Date: ``|PypiReleaseDate|``
|
|
35
56
|
|
|
36
57
|
Features
|
|
37
58
|
~~~~~~~~
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
.. Licensed to the Apache Software Foundation (ASF) under one
|
|
2
|
+
or more contributor license agreements. See the NOTICE file
|
|
3
|
+
distributed with this work for additional information
|
|
4
|
+
regarding copyright ownership. The ASF licenses this file
|
|
5
|
+
to you under the Apache License, Version 2.0 (the
|
|
6
|
+
"License"); you may not use this file except in compliance
|
|
7
|
+
with the License. You may obtain a copy of the License at
|
|
8
|
+
|
|
9
|
+
.. http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
|
|
11
|
+
.. Unless required by applicable law or agreed to in writing,
|
|
12
|
+
software distributed under the License is distributed on an
|
|
13
|
+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
14
|
+
KIND, either express or implied. See the License for the
|
|
15
|
+
specific language governing permissions and limitations
|
|
16
|
+
under the License.
|
|
17
|
+
|
|
18
|
+
.. include:: /../../../devel-common/src/sphinx_exts/includes/providers-configurations-ref.rst
|
|
19
|
+
.. include:: /../../../devel-common/src/sphinx_exts/includes/sections-and-options.rst
|
|
@@ -96,7 +96,9 @@ Extra (optional)
|
|
|
96
96
|
* ``iam`` - If set to ``True`` than use AWS IAM database authentication for
|
|
97
97
|
`Amazon RDS <https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/UsingWithRDS.IAMDBAuth.html>`__,
|
|
98
98
|
`Amazon Aurora <https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/UsingWithRDS.IAMDBAuth.html>`__
|
|
99
|
-
|
|
99
|
+
`Amazon Redshift <https://docs.aws.amazon.com/redshift/latest/mgmt/generating-user-credentials.html>`__
|
|
100
|
+
or use Microsoft Entra Authentication for
|
|
101
|
+
`Azure Postgres Flexible Server <https://learn.microsoft.com/en-us/azure/postgresql/flexible-server/security-entra-concepts>`__.
|
|
100
102
|
* ``aws_conn_id`` - AWS Connection ID which use for authentication via AWS IAM,
|
|
101
103
|
if not specified then **aws_default** is used.
|
|
102
104
|
* ``redshift`` - Used when AWS IAM database authentication enabled.
|
|
@@ -104,6 +106,8 @@ Extra (optional)
|
|
|
104
106
|
* ``cluster-identifier`` - The unique identifier of the Amazon Redshift Cluster that contains the database
|
|
105
107
|
for which you are requesting credentials. This parameter is case sensitive.
|
|
106
108
|
If not specified than hostname from **Connection Host** is used.
|
|
109
|
+
* ``azure_conn_id`` - Azure Connection ID to be used for authentication via Azure Entra ID. Azure Oauth token
|
|
110
|
+
is retrieved from the azure connection which is used as password for PostgreSQL connection. Scope for the Azure OAuth token can be set in the config option ``azure_oauth_scope`` under the section ``[postgres]``. Requires `apache-airflow-providers-microsoft-azure>=12.8.0`.
|
|
107
111
|
|
|
108
112
|
Example "extras" field (Amazon RDS PostgreSQL or Amazon Aurora PostgreSQL):
|
|
109
113
|
|
|
@@ -125,6 +129,15 @@ Extra (optional)
|
|
|
125
129
|
"cluster-identifier": "awesome-redshift-identifier"
|
|
126
130
|
}
|
|
127
131
|
|
|
132
|
+
Example "extras" field (to use Azure Entra Authentication for Postgres Flexible Server):
|
|
133
|
+
|
|
134
|
+
.. code-block:: json
|
|
135
|
+
|
|
136
|
+
{
|
|
137
|
+
"iam": true,
|
|
138
|
+
"azure_conn_id": "azure_default_conn"
|
|
139
|
+
}
|
|
140
|
+
|
|
128
141
|
When specifying the connection as URI (in :envvar:`AIRFLOW_CONN_{CONN_ID}` variable) you should specify it
|
|
129
142
|
following the standard syntax of DB connections, where extras are passed as parameters
|
|
130
143
|
of the URI (note that all components of the URI should be URL-encoded).
|
|
@@ -41,6 +41,7 @@
|
|
|
41
41
|
:maxdepth: 1
|
|
42
42
|
:caption: References
|
|
43
43
|
|
|
44
|
+
Configuration <configurations-ref>
|
|
44
45
|
Python API <_api/airflow/providers/postgres/index>
|
|
45
46
|
Dialects <dialects>
|
|
46
47
|
|
|
@@ -77,7 +78,7 @@ apache-airflow-providers-postgres package
|
|
|
77
78
|
`PostgreSQL <https://www.postgresql.org/>`__
|
|
78
79
|
|
|
79
80
|
|
|
80
|
-
Release: 6.
|
|
81
|
+
Release: 6.4.0
|
|
81
82
|
|
|
82
83
|
Provider package
|
|
83
84
|
----------------
|
|
@@ -97,15 +98,16 @@ Requirements
|
|
|
97
98
|
|
|
98
99
|
The minimum Apache Airflow version supported by this provider distribution is ``2.10.0``.
|
|
99
100
|
|
|
100
|
-
|
|
101
|
-
PIP package
|
|
102
|
-
|
|
103
|
-
``apache-airflow``
|
|
104
|
-
``apache-airflow-providers-common-
|
|
105
|
-
``
|
|
106
|
-
``psycopg2-binary``
|
|
107
|
-
``
|
|
108
|
-
|
|
101
|
+
========================================== ======================================
|
|
102
|
+
PIP package Version required
|
|
103
|
+
========================================== ======================================
|
|
104
|
+
``apache-airflow`` ``>=2.10.0``
|
|
105
|
+
``apache-airflow-providers-common-compat`` ``>=1.8.0``
|
|
106
|
+
``apache-airflow-providers-common-sql`` ``>=1.23.0``
|
|
107
|
+
``psycopg2-binary`` ``>=2.9.9; python_version < "3.13"``
|
|
108
|
+
``psycopg2-binary`` ``>=2.9.10; python_version >= "3.13"``
|
|
109
|
+
``asyncpg`` ``>=0.30.0``
|
|
110
|
+
========================================== ======================================
|
|
109
111
|
|
|
110
112
|
Cross provider package dependencies
|
|
111
113
|
-----------------------------------
|
|
@@ -120,13 +122,15 @@ You can install such cross-provider dependencies when installing from PyPI. For
|
|
|
120
122
|
pip install apache-airflow-providers-postgres[amazon]
|
|
121
123
|
|
|
122
124
|
|
|
123
|
-
|
|
124
|
-
Dependent package
|
|
125
|
-
|
|
126
|
-
`apache-airflow-providers-amazon <https://airflow.apache.org/docs/apache-airflow-providers-amazon>`_
|
|
127
|
-
`apache-airflow-providers-common-
|
|
128
|
-
`apache-airflow-providers-
|
|
129
|
-
|
|
125
|
+
====================================================================================================================== ===================
|
|
126
|
+
Dependent package Extra
|
|
127
|
+
====================================================================================================================== ===================
|
|
128
|
+
`apache-airflow-providers-amazon <https://airflow.apache.org/docs/apache-airflow-providers-amazon>`_ ``amazon``
|
|
129
|
+
`apache-airflow-providers-common-compat <https://airflow.apache.org/docs/apache-airflow-providers-common-compat>`_ ``common.compat``
|
|
130
|
+
`apache-airflow-providers-common-sql <https://airflow.apache.org/docs/apache-airflow-providers-common-sql>`_ ``common.sql``
|
|
131
|
+
`apache-airflow-providers-microsoft-azure <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure>`_ ``microsoft.azure``
|
|
132
|
+
`apache-airflow-providers-openlineage <https://airflow.apache.org/docs/apache-airflow-providers-openlineage>`_ ``openlineage``
|
|
133
|
+
====================================================================================================================== ===================
|
|
130
134
|
|
|
131
135
|
Downloading official packages
|
|
132
136
|
-----------------------------
|
|
@@ -134,5 +138,5 @@ Downloading official packages
|
|
|
134
138
|
You can download officially released packages and verify their checksums and signatures from the
|
|
135
139
|
`Official Apache Download site <https://downloads.apache.org/airflow/providers/>`_
|
|
136
140
|
|
|
137
|
-
* `The apache-airflow-providers-postgres 6.
|
|
138
|
-
* `The apache-airflow-providers-postgres 6.
|
|
141
|
+
* `The apache-airflow-providers-postgres 6.4.0 sdist package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_postgres-6.4.0.tar.gz>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_postgres-6.4.0.tar.gz.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_postgres-6.4.0.tar.gz.sha512>`__)
|
|
142
|
+
* `The apache-airflow-providers-postgres 6.4.0 wheel package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_postgres-6.4.0-py3-none-any.whl>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_postgres-6.4.0-py3-none-any.whl.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_postgres-6.4.0-py3-none-any.whl.sha512>`__)
|
{apache_airflow_providers_postgres-6.3.0 → apache_airflow_providers_postgres-6.4.0rc1}/provider.yaml
RENAMED
|
@@ -22,12 +22,13 @@ description: |
|
|
|
22
22
|
`PostgreSQL <https://www.postgresql.org/>`__
|
|
23
23
|
|
|
24
24
|
state: ready
|
|
25
|
-
source-date-epoch:
|
|
25
|
+
source-date-epoch: 1761117431
|
|
26
26
|
# Note that those versions are maintained by release manager - do not update them manually
|
|
27
27
|
# with the exception of case where other provider in sources has >= new provider version.
|
|
28
28
|
# In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have
|
|
29
29
|
# to be done in the same PR
|
|
30
30
|
versions:
|
|
31
|
+
- 6.4.0
|
|
31
32
|
- 6.3.0
|
|
32
33
|
- 6.2.3
|
|
33
34
|
- 6.2.2
|
|
@@ -109,3 +110,17 @@ asset-uris:
|
|
|
109
110
|
dataset-uris:
|
|
110
111
|
- schemes: [postgres, postgresql]
|
|
111
112
|
handler: airflow.providers.postgres.assets.postgres.sanitize_uri
|
|
113
|
+
|
|
114
|
+
config:
|
|
115
|
+
postgres:
|
|
116
|
+
description: |
|
|
117
|
+
Configuration for Postgres hooks and operators.
|
|
118
|
+
options:
|
|
119
|
+
azure_oauth_scope:
|
|
120
|
+
description: |
|
|
121
|
+
The scope to use while retrieving Oauth token for Postgres Flexible Server
|
|
122
|
+
from Azure Entra authentication.
|
|
123
|
+
version_added: 6.4.0
|
|
124
|
+
type: string
|
|
125
|
+
example: ~
|
|
126
|
+
default: "https://ossrdbms-aad.database.windows.net/.default"
|
|
@@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi"
|
|
|
25
25
|
|
|
26
26
|
[project]
|
|
27
27
|
name = "apache-airflow-providers-postgres"
|
|
28
|
-
version = "6.
|
|
28
|
+
version = "6.4.0rc1"
|
|
29
29
|
description = "Provider package apache-airflow-providers-postgres for Apache Airflow"
|
|
30
30
|
readme = "README.rst"
|
|
31
31
|
authors = [
|
|
@@ -57,8 +57,9 @@ requires-python = ">=3.10"
|
|
|
57
57
|
# Make sure to run ``prek update-providers-dependencies --all-files``
|
|
58
58
|
# After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build``
|
|
59
59
|
dependencies = [
|
|
60
|
-
"apache-airflow>=2.10.
|
|
61
|
-
"apache-airflow-providers-common-
|
|
60
|
+
"apache-airflow>=2.10.0rc1",
|
|
61
|
+
"apache-airflow-providers-common-compat>=1.8.0rc1",
|
|
62
|
+
"apache-airflow-providers-common-sql>=1.23.0rc1",
|
|
62
63
|
"psycopg2-binary>=2.9.9; python_version < '3.13'",
|
|
63
64
|
"psycopg2-binary>=2.9.10; python_version >= '3.13'",
|
|
64
65
|
"asyncpg>=0.30.0",
|
|
@@ -68,7 +69,10 @@ dependencies = [
|
|
|
68
69
|
# Any change in the dependencies is preserved when the file is regenerated
|
|
69
70
|
[project.optional-dependencies]
|
|
70
71
|
"amazon" = [
|
|
71
|
-
"apache-airflow-providers-amazon>=2.6.
|
|
72
|
+
"apache-airflow-providers-amazon>=2.6.0rc1",
|
|
73
|
+
]
|
|
74
|
+
"microsoft.azure" = [
|
|
75
|
+
"apache-airflow-providers-microsoft-azure"
|
|
72
76
|
]
|
|
73
77
|
"openlineage" = [
|
|
74
78
|
"apache-airflow-providers-openlineage"
|
|
@@ -90,7 +94,9 @@ dev = [
|
|
|
90
94
|
"apache-airflow-task-sdk",
|
|
91
95
|
"apache-airflow-devel-common",
|
|
92
96
|
"apache-airflow-providers-amazon",
|
|
97
|
+
"apache-airflow-providers-common-compat",
|
|
93
98
|
"apache-airflow-providers-common-sql",
|
|
99
|
+
"apache-airflow-providers-microsoft-azure",
|
|
94
100
|
"apache-airflow-providers-openlineage",
|
|
95
101
|
# Additional devel dependencies (do not remove this line and add extra development dependencies)
|
|
96
102
|
"apache-airflow-providers-common-sql[pandas]",
|
|
@@ -124,8 +130,8 @@ apache-airflow-providers-common-sql = {workspace = true}
|
|
|
124
130
|
apache-airflow-providers-standard = {workspace = true}
|
|
125
131
|
|
|
126
132
|
[project.urls]
|
|
127
|
-
"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-postgres/6.
|
|
128
|
-
"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-postgres/6.
|
|
133
|
+
"Documentation" = "https://airflow.staged.apache.org/docs/apache-airflow-providers-postgres/6.4.0"
|
|
134
|
+
"Changelog" = "https://airflow.staged.apache.org/docs/apache-airflow-providers-postgres/6.4.0/changelog.html"
|
|
129
135
|
"Bug Tracker" = "https://github.com/apache/airflow/issues"
|
|
130
136
|
"Source Code" = "https://github.com/apache/airflow"
|
|
131
137
|
"Slack Chat" = "https://s.apache.org/airflow-slack"
|
|
@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
|
|
|
29
29
|
|
|
30
30
|
__all__ = ["__version__"]
|
|
31
31
|
|
|
32
|
-
__version__ = "6.
|
|
32
|
+
__version__ = "6.4.0"
|
|
33
33
|
|
|
34
34
|
if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
|
|
35
35
|
"2.10.0"
|
|
@@ -41,24 +41,21 @@ class PostgresDialect(Dialect):
|
|
|
41
41
|
"""
|
|
42
42
|
if schema is None:
|
|
43
43
|
table, schema = self.extract_schema_from_table(table)
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
(self.unescape_word(schema), self.unescape_word(table)),
|
|
60
|
-
)
|
|
61
|
-
]
|
|
44
|
+
table = self.unescape_word(table)
|
|
45
|
+
schema = self.unescape_word(schema) if schema else None
|
|
46
|
+
query = """
|
|
47
|
+
select kcu.column_name
|
|
48
|
+
from information_schema.table_constraints tco
|
|
49
|
+
join information_schema.key_column_usage kcu
|
|
50
|
+
on kcu.constraint_name = tco.constraint_name
|
|
51
|
+
and kcu.constraint_schema = tco.constraint_schema
|
|
52
|
+
and kcu.constraint_name = tco.constraint_name
|
|
53
|
+
where tco.constraint_type = 'PRIMARY KEY'
|
|
54
|
+
and kcu.table_schema = %s
|
|
55
|
+
and kcu.table_name = %s
|
|
56
|
+
order by kcu.ordinal_position
|
|
57
|
+
"""
|
|
58
|
+
pk_columns = [row[0] for row in self.get_records(query, (schema, table))]
|
|
62
59
|
return pk_columns or None
|
|
63
60
|
|
|
64
61
|
@staticmethod
|
|
@@ -78,31 +75,27 @@ class PostgresDialect(Dialect):
|
|
|
78
75
|
) -> list[str] | None:
|
|
79
76
|
if schema is None:
|
|
80
77
|
table, schema = self.extract_schema_from_table(table)
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
),
|
|
103
|
-
),
|
|
104
|
-
)
|
|
105
|
-
)
|
|
78
|
+
table = self.unescape_word(table)
|
|
79
|
+
schema = self.unescape_word(schema) if schema else None
|
|
80
|
+
query = """
|
|
81
|
+
select column_name,
|
|
82
|
+
data_type,
|
|
83
|
+
is_nullable,
|
|
84
|
+
column_default,
|
|
85
|
+
is_generated,
|
|
86
|
+
is_identity
|
|
87
|
+
from information_schema.columns
|
|
88
|
+
where table_schema = %s
|
|
89
|
+
and table_name = %s
|
|
90
|
+
order by ordinal_position
|
|
91
|
+
"""
|
|
92
|
+
column_names = []
|
|
93
|
+
for row in map(
|
|
94
|
+
self._to_row,
|
|
95
|
+
self.get_records(query, (schema, table)),
|
|
96
|
+
):
|
|
97
|
+
if predicate(row):
|
|
98
|
+
column_names.append(row["name"])
|
|
106
99
|
self.log.debug("Column names for table '%s': %s", table, column_names)
|
|
107
100
|
return column_names
|
|
108
101
|
|
|
@@ -65,4 +65,18 @@ def get_provider_info():
|
|
|
65
65
|
"handler": "airflow.providers.postgres.assets.postgres.sanitize_uri",
|
|
66
66
|
}
|
|
67
67
|
],
|
|
68
|
+
"config": {
|
|
69
|
+
"postgres": {
|
|
70
|
+
"description": "Configuration for Postgres hooks and operators.\n",
|
|
71
|
+
"options": {
|
|
72
|
+
"azure_oauth_scope": {
|
|
73
|
+
"description": "The scope to use while retrieving Oauth token for Postgres Flexible Server\nfrom Azure Entra authentication.\n",
|
|
74
|
+
"version_added": "6.4.0",
|
|
75
|
+
"type": "string",
|
|
76
|
+
"example": None,
|
|
77
|
+
"default": "https://ossrdbms-aad.database.windows.net/.default",
|
|
78
|
+
}
|
|
79
|
+
},
|
|
80
|
+
}
|
|
81
|
+
},
|
|
68
82
|
}
|
|
@@ -30,10 +30,12 @@ from more_itertools import chunked
|
|
|
30
30
|
from psycopg2.extras import DictCursor, NamedTupleCursor, RealDictCursor, execute_batch
|
|
31
31
|
from sqlalchemy.engine import URL
|
|
32
32
|
|
|
33
|
+
from airflow.configuration import conf
|
|
33
34
|
from airflow.exceptions import (
|
|
34
35
|
AirflowException,
|
|
35
36
|
AirflowOptionalProviderFeatureException,
|
|
36
37
|
)
|
|
38
|
+
from airflow.providers.common.compat.sdk import Connection
|
|
37
39
|
from airflow.providers.common.sql.hooks.sql import DbApiHook
|
|
38
40
|
from airflow.providers.postgres.dialects.postgres import PostgresDialect
|
|
39
41
|
|
|
@@ -64,11 +66,6 @@ if TYPE_CHECKING:
|
|
|
64
66
|
if USE_PSYCOPG3:
|
|
65
67
|
from psycopg.errors import Diagnostic
|
|
66
68
|
|
|
67
|
-
try:
|
|
68
|
-
from airflow.sdk import Connection
|
|
69
|
-
except ImportError:
|
|
70
|
-
from airflow.models.connection import Connection # type: ignore[assignment]
|
|
71
|
-
|
|
72
69
|
CursorType: TypeAlias = DictCursor | RealDictCursor | NamedTupleCursor
|
|
73
70
|
CursorRow: TypeAlias = dict[str, Any] | tuple[Any, ...]
|
|
74
71
|
|
|
@@ -156,7 +153,9 @@ class PostgresHook(DbApiHook):
|
|
|
156
153
|
"aws_conn_id",
|
|
157
154
|
"sqlalchemy_scheme",
|
|
158
155
|
"sqlalchemy_query",
|
|
156
|
+
"azure_conn_id",
|
|
159
157
|
}
|
|
158
|
+
default_azure_oauth_scope = "https://ossrdbms-aad.database.windows.net/.default"
|
|
160
159
|
|
|
161
160
|
def __init__(
|
|
162
161
|
self, *args, options: str | None = None, enable_log_db_messages: bool = False, **kwargs
|
|
@@ -177,6 +176,8 @@ class PostgresHook(DbApiHook):
|
|
|
177
176
|
query = conn.extra_dejson.get("sqlalchemy_query", {})
|
|
178
177
|
if not isinstance(query, dict):
|
|
179
178
|
raise AirflowException("The parameter 'sqlalchemy_query' must be of type dict!")
|
|
179
|
+
if conn.extra_dejson.get("iam", False):
|
|
180
|
+
conn.login, conn.password, conn.port = self.get_iam_token(conn)
|
|
180
181
|
return URL.create(
|
|
181
182
|
drivername="postgresql+psycopg" if USE_PSYCOPG3 else "postgresql",
|
|
182
183
|
username=self.__cast_nullable(conn.login, str),
|
|
@@ -441,8 +442,14 @@ class PostgresHook(DbApiHook):
|
|
|
441
442
|
return PostgresHook._serialize_cell_ppg2(cell, conn)
|
|
442
443
|
|
|
443
444
|
def get_iam_token(self, conn: Connection) -> tuple[str, str, int]:
|
|
445
|
+
"""Get the IAM token from different identity providers."""
|
|
446
|
+
if conn.extra_dejson.get("azure_conn_id"):
|
|
447
|
+
return self.get_azure_iam_token(conn)
|
|
448
|
+
return self.get_aws_iam_token(conn)
|
|
449
|
+
|
|
450
|
+
def get_aws_iam_token(self, conn: Connection) -> tuple[str, str, int]:
|
|
444
451
|
"""
|
|
445
|
-
Get the IAM token.
|
|
452
|
+
Get the AWS IAM token.
|
|
446
453
|
|
|
447
454
|
This uses AWSHook to retrieve a temporary password to connect to
|
|
448
455
|
Postgres or Redshift. Port is required. If none is provided, the default
|
|
@@ -500,6 +507,36 @@ class PostgresHook(DbApiHook):
|
|
|
500
507
|
token = rds_client.generate_db_auth_token(conn.host, port, conn.login)
|
|
501
508
|
return cast("str", login), cast("str", token), port
|
|
502
509
|
|
|
510
|
+
def get_azure_iam_token(self, conn: Connection) -> tuple[str, str, int]:
|
|
511
|
+
"""
|
|
512
|
+
Get the Azure IAM token.
|
|
513
|
+
|
|
514
|
+
This uses AzureBaseHook to retrieve an OAUTH token to connect to Postgres.
|
|
515
|
+
Scope for the OAuth token can be set in the config option ``azure_oauth_scope`` under the section ``[postgres]``.
|
|
516
|
+
"""
|
|
517
|
+
if TYPE_CHECKING:
|
|
518
|
+
from airflow.providers.microsoft.azure.hooks.base_azure import AzureBaseHook
|
|
519
|
+
|
|
520
|
+
azure_conn_id = conn.extra_dejson.get("azure_conn_id", "azure_default")
|
|
521
|
+
try:
|
|
522
|
+
azure_conn = Connection.get(azure_conn_id)
|
|
523
|
+
except AttributeError:
|
|
524
|
+
azure_conn = Connection.get_connection_from_secrets(azure_conn_id) # type: ignore[attr-defined]
|
|
525
|
+
azure_base_hook: AzureBaseHook = azure_conn.get_hook()
|
|
526
|
+
scope = conf.get("postgres", "azure_oauth_scope", fallback=self.default_azure_oauth_scope)
|
|
527
|
+
try:
|
|
528
|
+
token = azure_base_hook.get_token(scope).token
|
|
529
|
+
except AttributeError as e:
|
|
530
|
+
if e.name == "get_token" and e.obj == azure_base_hook:
|
|
531
|
+
raise AttributeError(
|
|
532
|
+
"'AzureBaseHook' object has no attribute 'get_token'. "
|
|
533
|
+
"Please upgrade apache-airflow-providers-microsoft-azure>=12.8.0",
|
|
534
|
+
name=e.name,
|
|
535
|
+
obj=e.obj,
|
|
536
|
+
) from e
|
|
537
|
+
raise
|
|
538
|
+
return cast("str", conn.login or azure_conn.login), token, conn.port or 5432
|
|
539
|
+
|
|
503
540
|
def get_table_primary_key(self, table: str, schema: str | None = "public") -> list[str] | None:
|
|
504
541
|
"""
|
|
505
542
|
Get the table's primary key.
|
|
@@ -56,11 +56,16 @@ def test_sanitize_uri_pass(original: str, normalized: str) -> None:
|
|
|
56
56
|
pytest.param("postgres://", id="blank"),
|
|
57
57
|
pytest.param("postgres:///database/schema/table", id="no-host"),
|
|
58
58
|
pytest.param("postgres://example.com/database/table", id="missing-component"),
|
|
59
|
-
pytest.param("postgres://example.com:abcd/database/schema/table", id="non-port"),
|
|
60
59
|
pytest.param("postgres://example.com/database/schema/table/column", id="extra-component"),
|
|
61
60
|
],
|
|
62
61
|
)
|
|
63
62
|
def test_sanitize_uri_fail(value: str) -> None:
|
|
64
63
|
uri_i = urllib.parse.urlsplit(value)
|
|
65
|
-
with pytest.raises(ValueError):
|
|
64
|
+
with pytest.raises(ValueError, match="URI format postgres:// must contain"):
|
|
65
|
+
sanitize_uri(uri_i)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def test_sanitize_uri_fail_non_port() -> None:
|
|
69
|
+
uri_i = urllib.parse.urlsplit("postgres://example.com:abcd/database/schema/table")
|
|
70
|
+
with pytest.raises(ValueError, match="Port could not be cast to integer value as 'abcd'"):
|
|
66
71
|
sanitize_uri(uri_i)
|
|
@@ -209,7 +209,7 @@ class TestPostgresHookConn:
|
|
|
209
209
|
@pytest.mark.usefixtures("mock_connect")
|
|
210
210
|
def test_get_conn_with_invalid_cursor(self):
|
|
211
211
|
self.connection.extra = '{"cursor": "mycursor"}'
|
|
212
|
-
with pytest.raises(ValueError):
|
|
212
|
+
with pytest.raises(ValueError, match="Invalid cursor passed mycursor."):
|
|
213
213
|
self.db_hook.get_conn()
|
|
214
214
|
|
|
215
215
|
def test_get_conn_from_connection(self, mock_connect):
|
|
@@ -444,6 +444,57 @@ class TestPostgresHookConn:
|
|
|
444
444
|
port=(port or 5439),
|
|
445
445
|
)
|
|
446
446
|
|
|
447
|
+
def test_get_conn_azure_iam(self, mocker, mock_connect):
|
|
448
|
+
mock_azure_conn_id = "azure_conn1"
|
|
449
|
+
mock_db_token = "azure_token1"
|
|
450
|
+
mock_conn_extra = {"iam": True, "azure_conn_id": mock_azure_conn_id}
|
|
451
|
+
self.connection.extra = json.dumps(mock_conn_extra)
|
|
452
|
+
|
|
453
|
+
mock_connection_class = mocker.patch("airflow.providers.postgres.hooks.postgres.Connection")
|
|
454
|
+
mock_azure_base_hook = mock_connection_class.get.return_value.get_hook.return_value
|
|
455
|
+
mock_azure_base_hook.get_token.return_value.token = mock_db_token
|
|
456
|
+
|
|
457
|
+
self.db_hook.get_conn()
|
|
458
|
+
|
|
459
|
+
# Check AzureBaseHook initialization and get_token call args
|
|
460
|
+
mock_connection_class.get.assert_called_once_with(mock_azure_conn_id)
|
|
461
|
+
mock_azure_base_hook.get_token.assert_called_once_with(PostgresHook.default_azure_oauth_scope)
|
|
462
|
+
|
|
463
|
+
# Check expected psycopg2 connection call args
|
|
464
|
+
mock_connect.assert_called_once_with(
|
|
465
|
+
user=self.connection.login,
|
|
466
|
+
password=mock_db_token,
|
|
467
|
+
host=self.connection.host,
|
|
468
|
+
dbname=self.connection.schema,
|
|
469
|
+
port=(self.connection.port or 5432),
|
|
470
|
+
)
|
|
471
|
+
|
|
472
|
+
assert mock_db_token in self.db_hook.sqlalchemy_url
|
|
473
|
+
|
|
474
|
+
def test_get_azure_iam_token_expect_failure_on_get_token(self, mocker):
|
|
475
|
+
"""Test get_azure_iam_token method gets token from provided connection id"""
|
|
476
|
+
|
|
477
|
+
class MockAzureBaseHookWithoutGetToken:
|
|
478
|
+
def __init__(self):
|
|
479
|
+
pass
|
|
480
|
+
|
|
481
|
+
azure_conn_id = "azure_test_conn"
|
|
482
|
+
mock_connection_class = mocker.patch("airflow.providers.postgres.hooks.postgres.Connection")
|
|
483
|
+
mock_connection_class.get.return_value.get_hook.return_value = MockAzureBaseHookWithoutGetToken()
|
|
484
|
+
|
|
485
|
+
self.connection.extra = json.dumps({"iam": True, "azure_conn_id": azure_conn_id})
|
|
486
|
+
with pytest.raises(
|
|
487
|
+
AttributeError,
|
|
488
|
+
match=(
|
|
489
|
+
"'AzureBaseHook' object has no attribute 'get_token'. "
|
|
490
|
+
"Please upgrade apache-airflow-providers-microsoft-azure>="
|
|
491
|
+
),
|
|
492
|
+
):
|
|
493
|
+
self.db_hook.get_azure_iam_token(self.connection)
|
|
494
|
+
|
|
495
|
+
# Check AzureBaseHook initialization
|
|
496
|
+
mock_connection_class.get.assert_called_once_with(azure_conn_id)
|
|
497
|
+
|
|
447
498
|
def test_get_uri_from_connection_without_database_override(self, mocker):
|
|
448
499
|
expected: str = f"postgresql{'+psycopg' if USE_PSYCOPG3 else ''}://login:password@host:1/database"
|
|
449
500
|
self.db_hook.get_connection = mocker.MagicMock(
|
|
@@ -879,11 +930,9 @@ class TestPostgresHookPPG2:
|
|
|
879
930
|
),
|
|
880
931
|
]
|
|
881
932
|
fields = ("id", "value")
|
|
882
|
-
with pytest.raises(ValueError
|
|
933
|
+
with pytest.raises(ValueError, match="PostgreSQL ON CONFLICT upsert syntax requires column names"):
|
|
883
934
|
setup.db_hook.insert_rows(table, rows, replace=True, replace_index=fields[0])
|
|
884
935
|
|
|
885
|
-
assert str(ctx.value) == "PostgreSQL ON CONFLICT upsert syntax requires column names"
|
|
886
|
-
|
|
887
936
|
def test_insert_rows_replace_missing_replace_index_arg(self, postgres_hook_setup):
|
|
888
937
|
setup = postgres_hook_setup
|
|
889
938
|
table = "table"
|
|
@@ -898,11 +947,9 @@ class TestPostgresHookPPG2:
|
|
|
898
947
|
),
|
|
899
948
|
]
|
|
900
949
|
fields = ("id", "value")
|
|
901
|
-
with pytest.raises(ValueError
|
|
950
|
+
with pytest.raises(ValueError, match="PostgreSQL ON CONFLICT upsert syntax requires an unique index"):
|
|
902
951
|
setup.db_hook.insert_rows(table, rows, fields, replace=True)
|
|
903
952
|
|
|
904
|
-
assert str(ctx.value) == "PostgreSQL ON CONFLICT upsert syntax requires an unique index"
|
|
905
|
-
|
|
906
953
|
def test_insert_rows_replace_all_index(self, postgres_hook_setup):
|
|
907
954
|
setup = postgres_hook_setup
|
|
908
955
|
table = "table"
|
|
@@ -1145,11 +1192,9 @@ class TestPostgresHookPPG3:
|
|
|
1145
1192
|
),
|
|
1146
1193
|
]
|
|
1147
1194
|
fields = ("id", "value")
|
|
1148
|
-
with pytest.raises(ValueError
|
|
1195
|
+
with pytest.raises(ValueError, match="PostgreSQL ON CONFLICT upsert syntax requires column names"):
|
|
1149
1196
|
self.db_hook.insert_rows(table, rows, replace=True, replace_index=fields[0])
|
|
1150
1197
|
|
|
1151
|
-
assert str(ctx.value) == "PostgreSQL ON CONFLICT upsert syntax requires column names"
|
|
1152
|
-
|
|
1153
1198
|
def test_insert_rows_replace_missing_replace_index_arg(self):
|
|
1154
1199
|
table = "table"
|
|
1155
1200
|
rows = [
|
|
@@ -1163,11 +1208,9 @@ class TestPostgresHookPPG3:
|
|
|
1163
1208
|
),
|
|
1164
1209
|
]
|
|
1165
1210
|
fields = ("id", "value")
|
|
1166
|
-
with pytest.raises(ValueError
|
|
1211
|
+
with pytest.raises(ValueError, match="PostgreSQL ON CONFLICT upsert syntax requires an unique index"):
|
|
1167
1212
|
self.db_hook.insert_rows(table, rows, fields, replace=True)
|
|
1168
1213
|
|
|
1169
|
-
assert str(ctx.value) == "PostgreSQL ON CONFLICT upsert syntax requires an unique index"
|
|
1170
|
-
|
|
1171
1214
|
def test_insert_rows_replace_all_index(self):
|
|
1172
1215
|
table = "table"
|
|
1173
1216
|
rows = [
|
|
File without changes
|
|
File without changes
|
{apache_airflow_providers_postgres-6.3.0 → apache_airflow_providers_postgres-6.4.0rc1}/docs/conf.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|