airflow-dbt-python 2.2.0__tar.gz → 3.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {airflow_dbt_python-2.2.0 → airflow_dbt_python-3.0.1}/PKG-INFO +19 -13
- {airflow_dbt_python-2.2.0 → airflow_dbt_python-3.0.1}/README.md +1 -1
- {airflow_dbt_python-2.2.0 → airflow_dbt_python-3.0.1}/airflow_dbt_python/__version__.py +1 -1
- airflow_dbt_python-3.0.1/airflow_dbt_python/hooks/__init__.py +1 -0
- {airflow_dbt_python-2.2.0 → airflow_dbt_python-3.0.1}/airflow_dbt_python/hooks/dbt.py +47 -250
- airflow_dbt_python-2.2.0/airflow_dbt_python/hooks/remote.py → airflow_dbt_python-3.0.1/airflow_dbt_python/hooks/fs/__init__.py +25 -19
- airflow_dbt_python-3.0.1/airflow_dbt_python/hooks/fs/gcs.py +228 -0
- {airflow_dbt_python-2.2.0/airflow_dbt_python/hooks → airflow_dbt_python-3.0.1/airflow_dbt_python/hooks/fs}/git.py +25 -14
- airflow_dbt_python-2.2.0/airflow_dbt_python/hooks/localfs.py → airflow_dbt_python-3.0.1/airflow_dbt_python/hooks/fs/local.py +10 -38
- {airflow_dbt_python-2.2.0/airflow_dbt_python/hooks → airflow_dbt_python-3.0.1/airflow_dbt_python/hooks/fs}/s3.py +8 -8
- airflow_dbt_python-3.0.1/airflow_dbt_python/hooks/target.py +464 -0
- {airflow_dbt_python-2.2.0 → airflow_dbt_python-3.0.1}/airflow_dbt_python/operators/dbt.py +120 -6
- {airflow_dbt_python-2.2.0 → airflow_dbt_python-3.0.1}/airflow_dbt_python/utils/configs.py +52 -9
- {airflow_dbt_python-2.2.0 → airflow_dbt_python-3.0.1}/pyproject.toml +59 -49
- airflow_dbt_python-2.2.0/airflow_dbt_python/hooks/__init__.py +0 -1
- {airflow_dbt_python-2.2.0 → airflow_dbt_python-3.0.1}/LICENSE +0 -0
- {airflow_dbt_python-2.2.0 → airflow_dbt_python-3.0.1}/airflow_dbt_python/__init__.py +0 -0
- {airflow_dbt_python-2.2.0 → airflow_dbt_python-3.0.1}/airflow_dbt_python/operators/__init__.py +0 -0
- {airflow_dbt_python-2.2.0 → airflow_dbt_python-3.0.1}/airflow_dbt_python/utils/__init__.py +0 -0
- {airflow_dbt_python-2.2.0 → airflow_dbt_python-3.0.1}/airflow_dbt_python/utils/enums.py +0 -0
- {airflow_dbt_python-2.2.0 → airflow_dbt_python-3.0.1}/airflow_dbt_python/utils/env.py +0 -0
- {airflow_dbt_python-2.2.0 → airflow_dbt_python-3.0.1}/airflow_dbt_python/utils/url.py +0 -0
- {airflow_dbt_python-2.2.0 → airflow_dbt_python-3.0.1}/airflow_dbt_python/utils/version.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: airflow-dbt-python
|
|
3
|
-
Version:
|
|
3
|
+
Version: 3.0.1
|
|
4
4
|
Summary: A collection of Airflow operators, hooks, and utilities to execute dbt commands
|
|
5
5
|
License: MIT
|
|
6
6
|
Author: Tomás Farías Santana
|
|
@@ -9,7 +9,6 @@ Requires-Python: >=3.9,<3.13
|
|
|
9
9
|
Classifier: Development Status :: 5 - Production/Stable
|
|
10
10
|
Classifier: Intended Audience :: Developers
|
|
11
11
|
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
-
Classifier: Programming Language :: Python :: 3
|
|
13
12
|
Classifier: Programming Language :: Python :: 3.9
|
|
14
13
|
Classifier: Programming Language :: Python :: 3.10
|
|
15
14
|
Classifier: Programming Language :: Python :: 3.11
|
|
@@ -17,7 +16,7 @@ Classifier: Programming Language :: Python :: 3.12
|
|
|
17
16
|
Provides-Extra: adapters
|
|
18
17
|
Provides-Extra: airflow-providers
|
|
19
18
|
Provides-Extra: bigquery
|
|
20
|
-
Provides-Extra:
|
|
19
|
+
Provides-Extra: gcs
|
|
21
20
|
Provides-Extra: git
|
|
22
21
|
Provides-Extra: postgres
|
|
23
22
|
Provides-Extra: redshift
|
|
@@ -25,18 +24,25 @@ Provides-Extra: s3
|
|
|
25
24
|
Provides-Extra: snowflake
|
|
26
25
|
Provides-Extra: spark
|
|
27
26
|
Requires-Dist: apache-airflow (>=2.8)
|
|
28
|
-
Requires-Dist: apache-airflow-providers-amazon (>=3.0.0) ; extra == "airflow-providers"
|
|
29
|
-
Requires-Dist: apache-airflow-providers-
|
|
30
|
-
Requires-Dist:
|
|
27
|
+
Requires-Dist: apache-airflow-providers-amazon (>=3.0.0) ; extra == "airflow-providers"
|
|
28
|
+
Requires-Dist: apache-airflow-providers-amazon (>=3.0.0) ; extra == "s3"
|
|
29
|
+
Requires-Dist: apache-airflow-providers-google (>=11.0.0) ; extra == "airflow-providers"
|
|
30
|
+
Requires-Dist: apache-airflow-providers-google (>=11.0.0) ; extra == "gcs"
|
|
31
|
+
Requires-Dist: apache-airflow-providers-ssh (>=3.0.0) ; extra == "airflow-providers"
|
|
32
|
+
Requires-Dist: apache-airflow-providers-ssh (>=3.0.0) ; extra == "git"
|
|
31
33
|
Requires-Dist: contextlib-chdir (==1.0.2) ; python_version < "3.11"
|
|
32
|
-
Requires-Dist: dbt-bigquery (>=1.8.0,<2.0.0) ; extra == "adapters"
|
|
34
|
+
Requires-Dist: dbt-bigquery (>=1.8.0,<2.0.0) ; extra == "adapters"
|
|
35
|
+
Requires-Dist: dbt-bigquery (>=1.8.0,<2.0.0) ; extra == "bigquery"
|
|
33
36
|
Requires-Dist: dbt-core (>=1.8.0,<2.0.0)
|
|
34
|
-
Requires-Dist: dbt-postgres (>=1.8.0,<2.0.0) ; extra == "adapters"
|
|
35
|
-
Requires-Dist: dbt-
|
|
36
|
-
Requires-Dist: dbt-
|
|
37
|
-
Requires-Dist: dbt-
|
|
37
|
+
Requires-Dist: dbt-postgres (>=1.8.0,<2.0.0) ; extra == "adapters"
|
|
38
|
+
Requires-Dist: dbt-postgres (>=1.8.0,<2.0.0) ; extra == "postgres"
|
|
39
|
+
Requires-Dist: dbt-redshift (>=1.8.0,<2.0.0) ; extra == "adapters"
|
|
40
|
+
Requires-Dist: dbt-redshift (>=1.8.0,<2.0.0) ; extra == "redshift"
|
|
41
|
+
Requires-Dist: dbt-snowflake (>=1.8.0,<2.0.0) ; extra == "adapters"
|
|
42
|
+
Requires-Dist: dbt-snowflake (>=1.8.0,<2.0.0) ; extra == "snowflake"
|
|
43
|
+
Requires-Dist: dbt-spark (>=1.8.0,<2.0.0) ; extra == "spark"
|
|
44
|
+
Requires-Dist: dbt-spark[all] (>=1.8.0,<2.0.0) ; extra == "adapters"
|
|
38
45
|
Requires-Dist: dulwich (>=0.21) ; extra == "git"
|
|
39
|
-
Project-URL: Repository, https://github.com/tomasfarias/airflow-dbt-python
|
|
40
46
|
Description-Content-Type: text/markdown
|
|
41
47
|
|
|
42
48
|
# airflow-dbt-python
|
|
@@ -62,7 +68,7 @@ Before using *airflow-dbt-python*, ensure you meet the following requirements:
|
|
|
62
68
|
* If using any managed service, like AWS MWAA or GCP Cloud Composer 2/3, ensure your environment is created with a supported version of Airflow.
|
|
63
69
|
* If self-hosting, Airflow installation instructions can be found in their [official documentation](https://airflow.apache.org/docs/apache-airflow/stable/installation/index.html).
|
|
64
70
|
|
|
65
|
-
* Running Python 3.
|
|
71
|
+
* Running Python 3.9 or later in your Airflow environment.
|
|
66
72
|
|
|
67
73
|
> **Warning**
|
|
68
74
|
>
|
|
@@ -21,7 +21,7 @@ Before using *airflow-dbt-python*, ensure you meet the following requirements:
|
|
|
21
21
|
* If using any managed service, like AWS MWAA or GCP Cloud Composer 2/3, ensure your environment is created with a supported version of Airflow.
|
|
22
22
|
* If self-hosting, Airflow installation instructions can be found in their [official documentation](https://airflow.apache.org/docs/apache-airflow/stable/installation/index.html).
|
|
23
23
|
|
|
24
|
-
* Running Python 3.
|
|
24
|
+
* Running Python 3.9 or later in your Airflow environment.
|
|
25
25
|
|
|
26
26
|
> **Warning**
|
|
27
27
|
>
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Hooks module provides DbtHooks and DbtFSHooks."""
|
|
@@ -5,6 +5,7 @@ from __future__ import annotations
|
|
|
5
5
|
import json
|
|
6
6
|
import logging
|
|
7
7
|
import sys
|
|
8
|
+
from abc import ABC
|
|
8
9
|
from contextlib import contextmanager
|
|
9
10
|
from pathlib import Path
|
|
10
11
|
from tempfile import TemporaryDirectory
|
|
@@ -17,13 +18,11 @@ from typing import (
|
|
|
17
18
|
NamedTuple,
|
|
18
19
|
Optional,
|
|
19
20
|
Tuple,
|
|
20
|
-
Union,
|
|
21
21
|
)
|
|
22
22
|
from urllib.parse import urlparse
|
|
23
23
|
|
|
24
24
|
from airflow.exceptions import AirflowException
|
|
25
|
-
from airflow.
|
|
26
|
-
from airflow.models.connection import Connection
|
|
25
|
+
from airflow.utils.log.logging_mixin import LoggingMixin
|
|
27
26
|
|
|
28
27
|
if sys.version_info >= (3, 11):
|
|
29
28
|
from contextlib import chdir as chdir_ctx
|
|
@@ -32,13 +31,13 @@ else:
|
|
|
32
31
|
|
|
33
32
|
if TYPE_CHECKING:
|
|
34
33
|
from dbt.contracts.results import RunResult
|
|
35
|
-
from dbt.task.base import BaseTask
|
|
36
34
|
|
|
37
|
-
from airflow_dbt_python.hooks.
|
|
35
|
+
from airflow_dbt_python.hooks.fs import DbtFSHook
|
|
36
|
+
from airflow_dbt_python.hooks.target import DbtConnectionHook
|
|
38
37
|
from airflow_dbt_python.utils.configs import BaseConfig
|
|
39
38
|
from airflow_dbt_python.utils.url import URLLike
|
|
40
39
|
|
|
41
|
-
|
|
40
|
+
DbtFSHooksDict = Dict[Tuple[str, Optional[str]], DbtFSHook]
|
|
42
41
|
|
|
43
42
|
|
|
44
43
|
class DbtTaskResult(NamedTuple):
|
|
@@ -55,35 +54,6 @@ class DbtTaskResult(NamedTuple):
|
|
|
55
54
|
artifacts: dict[str, Any]
|
|
56
55
|
|
|
57
56
|
|
|
58
|
-
class DbtConnectionParam(NamedTuple):
|
|
59
|
-
"""A tuple indicating connection parameters relevant to dbt.
|
|
60
|
-
|
|
61
|
-
Attributes:
|
|
62
|
-
name: The name of the connection parameter. This name will be used to get the
|
|
63
|
-
parameter from an Airflow Connection or its extras.
|
|
64
|
-
store_override_name: A new name for the connection parameter. If not None, this
|
|
65
|
-
is the name used in a dbt profiles.
|
|
66
|
-
default: A default value if the parameter is not found.
|
|
67
|
-
"""
|
|
68
|
-
|
|
69
|
-
name: str
|
|
70
|
-
store_override_name: Optional[str] = None
|
|
71
|
-
default: Optional[Any] = None
|
|
72
|
-
|
|
73
|
-
@property
|
|
74
|
-
def override_name(self):
|
|
75
|
-
"""Returns the override_name if defined, otherwise defaults to name.
|
|
76
|
-
|
|
77
|
-
>>> DbtConnectionParam("login", "user").override_name
|
|
78
|
-
'user'
|
|
79
|
-
>>> DbtConnectionParam("port").override_name
|
|
80
|
-
'port'
|
|
81
|
-
"""
|
|
82
|
-
if self.store_override_name is None:
|
|
83
|
-
return self.name
|
|
84
|
-
return self.store_override_name
|
|
85
|
-
|
|
86
|
-
|
|
87
57
|
class DbtTemporaryDirectory(TemporaryDirectory):
|
|
88
58
|
"""A wrapper on TemporaryDirectory for older versions of Python.
|
|
89
59
|
|
|
@@ -105,57 +75,43 @@ class DbtTemporaryDirectory(TemporaryDirectory):
|
|
|
105
75
|
)
|
|
106
76
|
|
|
107
77
|
|
|
108
|
-
class DbtHook(
|
|
78
|
+
class DbtHook(ABC, LoggingMixin):
|
|
109
79
|
"""A hook to interact with dbt.
|
|
110
80
|
|
|
111
81
|
Allows for running dbt tasks and provides required configurations for each task.
|
|
112
82
|
"""
|
|
113
83
|
|
|
114
|
-
conn_name_attr = "dbt_conn_id"
|
|
115
|
-
default_conn_name = "dbt_default"
|
|
116
|
-
conn_type = "dbt"
|
|
117
|
-
hook_name = "dbt Hook"
|
|
118
|
-
|
|
119
|
-
conn_params: list[Union[DbtConnectionParam, str]] = [
|
|
120
|
-
DbtConnectionParam("conn_type", "type"),
|
|
121
|
-
"host",
|
|
122
|
-
DbtConnectionParam("conn_id", "dbname"),
|
|
123
|
-
"schema",
|
|
124
|
-
DbtConnectionParam("login", "user"),
|
|
125
|
-
"password",
|
|
126
|
-
"port",
|
|
127
|
-
]
|
|
128
|
-
conn_extra_params: list[Union[DbtConnectionParam, str]] = []
|
|
129
|
-
|
|
130
84
|
def __init__(
|
|
131
85
|
self,
|
|
132
86
|
*args,
|
|
133
|
-
dbt_conn_id: Optional[str] =
|
|
87
|
+
dbt_conn_id: Optional[str] = None,
|
|
134
88
|
project_conn_id: Optional[str] = None,
|
|
135
89
|
profiles_conn_id: Optional[str] = None,
|
|
136
90
|
**kwargs,
|
|
137
91
|
):
|
|
138
|
-
self.remotes: DbtRemoteHooksDict = {}
|
|
139
92
|
self.dbt_conn_id = dbt_conn_id
|
|
140
93
|
self.project_conn_id = project_conn_id
|
|
141
94
|
self.profiles_conn_id = profiles_conn_id
|
|
142
95
|
super().__init__(*args, **kwargs)
|
|
143
96
|
|
|
144
|
-
|
|
145
|
-
|
|
97
|
+
@staticmethod
|
|
98
|
+
def get_dbt_target_hook(conn_id: str) -> DbtConnectionHook:
|
|
99
|
+
"""Get a hook to get a dbt profile based on the Airflow connection."""
|
|
100
|
+
from .target import DbtConnectionHook
|
|
146
101
|
|
|
147
|
-
|
|
102
|
+
return DbtConnectionHook.get_db_conn_hook(conn_id)
|
|
103
|
+
|
|
104
|
+
@staticmethod
|
|
105
|
+
def get_fs_hook(scheme: str, conn_id: Optional[str]) -> DbtFSHook:
|
|
106
|
+
"""Get a fs_hook to interact with dbt files.
|
|
107
|
+
|
|
108
|
+
FSHooks are defined by the scheme we are looking for and an optional
|
|
148
109
|
connection id if we are looking to interface with any Airflow hook that
|
|
149
110
|
uses a connection.
|
|
150
111
|
"""
|
|
151
|
-
from .
|
|
112
|
+
from .fs import get_fs_hook
|
|
152
113
|
|
|
153
|
-
|
|
154
|
-
return self.remotes[(scheme, conn_id)]
|
|
155
|
-
except KeyError:
|
|
156
|
-
remote = get_remote(scheme, conn_id)
|
|
157
|
-
self.remotes[(scheme, conn_id)] = remote
|
|
158
|
-
return remote
|
|
114
|
+
return get_fs_hook(scheme, conn_id)
|
|
159
115
|
|
|
160
116
|
def download_dbt_profiles(
|
|
161
117
|
self,
|
|
@@ -164,13 +120,13 @@ class DbtHook(BaseHook):
|
|
|
164
120
|
) -> Path:
|
|
165
121
|
"""Pull a dbt profiles.yml file from a given profiles_dir.
|
|
166
122
|
|
|
167
|
-
This operation is delegated to a
|
|
123
|
+
This operation is delegated to a DbtFSHook. An optional connection id is
|
|
168
124
|
supported for remotes that require it.
|
|
169
125
|
"""
|
|
170
126
|
scheme = urlparse(str(profiles_dir)).scheme
|
|
171
|
-
|
|
127
|
+
fs_hook = self.get_fs_hook(scheme, self.profiles_conn_id)
|
|
172
128
|
|
|
173
|
-
return
|
|
129
|
+
return fs_hook.download_dbt_profiles(profiles_dir, destination)
|
|
174
130
|
|
|
175
131
|
def download_dbt_project(
|
|
176
132
|
self,
|
|
@@ -179,13 +135,13 @@ class DbtHook(BaseHook):
|
|
|
179
135
|
) -> Path:
|
|
180
136
|
"""Pull a dbt project from a given project_dir.
|
|
181
137
|
|
|
182
|
-
This operation is delegated to a
|
|
138
|
+
This operation is delegated to a DbtFSHook. An optional connection id is
|
|
183
139
|
supported for remotes that require it.
|
|
184
140
|
"""
|
|
185
141
|
scheme = urlparse(str(project_dir)).scheme
|
|
186
|
-
|
|
142
|
+
fs_hook = self.get_fs_hook(scheme, self.project_conn_id)
|
|
187
143
|
|
|
188
|
-
return
|
|
144
|
+
return fs_hook.download_dbt_project(project_dir, destination)
|
|
189
145
|
|
|
190
146
|
def upload_dbt_project(
|
|
191
147
|
self,
|
|
@@ -196,13 +152,13 @@ class DbtHook(BaseHook):
|
|
|
196
152
|
) -> None:
|
|
197
153
|
"""Push a dbt project from a given project_dir.
|
|
198
154
|
|
|
199
|
-
This operation is delegated to a
|
|
155
|
+
This operation is delegated to a DbtFSHook. An optional connection id is
|
|
200
156
|
supported for remotes that require it.
|
|
201
157
|
"""
|
|
202
158
|
scheme = urlparse(str(destination)).scheme
|
|
203
|
-
|
|
159
|
+
fs_hook = self.get_fs_hook(scheme, self.project_conn_id)
|
|
204
160
|
|
|
205
|
-
return
|
|
161
|
+
return fs_hook.upload_dbt_project(
|
|
206
162
|
project_dir, destination, replace=replace, delete_before=delete_before
|
|
207
163
|
)
|
|
208
164
|
|
|
@@ -231,8 +187,14 @@ class DbtHook(BaseHook):
|
|
|
231
187
|
from dbt.task.deps import DepsTask
|
|
232
188
|
from dbt.tracking import track_run
|
|
233
189
|
|
|
190
|
+
if self.dbt_conn_id:
|
|
191
|
+
kwargs["target"] = self.dbt_conn_id
|
|
192
|
+
target_hook = self.get_dbt_target_hook(self.dbt_conn_id)
|
|
193
|
+
extra_target = target_hook.get_dbt_target_from_connection()
|
|
194
|
+
else:
|
|
195
|
+
extra_target = None
|
|
196
|
+
|
|
234
197
|
config = self.get_dbt_task_config(command, **kwargs)
|
|
235
|
-
extra_target = self.get_dbt_target_from_connection(config.target)
|
|
236
198
|
|
|
237
199
|
with self.dbt_directory(
|
|
238
200
|
config,
|
|
@@ -246,15 +208,13 @@ class DbtHook(BaseHook):
|
|
|
246
208
|
nearest_project_dir = get_nearest_project_dir(config.project_dir)
|
|
247
209
|
|
|
248
210
|
with chdir_ctx(nearest_project_dir):
|
|
249
|
-
self.ensure_profiles(config)
|
|
250
|
-
|
|
251
211
|
with adapter_management():
|
|
252
212
|
task, runtime_config = config.create_dbt_task(
|
|
253
213
|
extra_target, write_perf_info
|
|
254
214
|
)
|
|
255
215
|
requires_profile = isinstance(task, (CleanTask, DepsTask))
|
|
256
216
|
|
|
257
|
-
self.setup_dbt_logging(
|
|
217
|
+
self.setup_dbt_logging(config.debug)
|
|
258
218
|
|
|
259
219
|
if runtime_config is not None and not requires_profile:
|
|
260
220
|
# The deps command installs the dependencies, which means they
|
|
@@ -373,25 +333,24 @@ class DbtHook(BaseHook):
|
|
|
373
333
|
project_dir,
|
|
374
334
|
tmp_dir,
|
|
375
335
|
)
|
|
376
|
-
new_project_dir = str(project_dir_path) + "/"
|
|
377
|
-
|
|
378
|
-
if (project_dir_path / "profiles.yml").exists():
|
|
379
|
-
# We may have downloaded the profiles.yml file together
|
|
380
|
-
# with the project.
|
|
381
|
-
return new_project_dir, new_project_dir
|
|
382
336
|
|
|
383
337
|
if profiles_dir is not None:
|
|
384
338
|
profiles_file_path = self.download_dbt_profiles(
|
|
385
339
|
profiles_dir,
|
|
386
340
|
tmp_dir,
|
|
387
341
|
)
|
|
388
|
-
|
|
342
|
+
profiles_dir_path = profiles_file_path.parent
|
|
343
|
+
elif (project_dir_path / "profiles.yml").exists():
|
|
344
|
+
profiles_dir_path = project_dir_path
|
|
389
345
|
else:
|
|
390
|
-
|
|
346
|
+
profiles_dir_path = None
|
|
391
347
|
|
|
392
|
-
return
|
|
348
|
+
return (
|
|
349
|
+
str(project_dir_path),
|
|
350
|
+
str(profiles_dir_path) if profiles_dir_path is not None else None,
|
|
351
|
+
)
|
|
393
352
|
|
|
394
|
-
def setup_dbt_logging(self,
|
|
353
|
+
def setup_dbt_logging(self, debug: Optional[bool]):
|
|
395
354
|
"""Setup dbt logging.
|
|
396
355
|
|
|
397
356
|
Starting with dbt v1, dbt initializes two loggers: default_file and
|
|
@@ -407,6 +366,7 @@ class DbtHook(BaseHook):
|
|
|
407
366
|
configured_file = logging.getLogger("configured_file")
|
|
408
367
|
file_log = logging.getLogger("file_log")
|
|
409
368
|
stdout_log = logging.getLogger("stdout_log")
|
|
369
|
+
stdout_log.handlers.clear()
|
|
410
370
|
stdout_log.propagate = True
|
|
411
371
|
|
|
412
372
|
if not debug:
|
|
@@ -417,166 +377,3 @@ class DbtHook(BaseHook):
|
|
|
417
377
|
file_log.propagate = False
|
|
418
378
|
configured_file.setLevel("INFO")
|
|
419
379
|
configured_file.propagate = False
|
|
420
|
-
|
|
421
|
-
def ensure_profiles(self, config: BaseConfig):
|
|
422
|
-
"""Ensure a profiles file exists."""
|
|
423
|
-
if config.profiles_dir is not None:
|
|
424
|
-
# We expect one to exist given that we have passed a profiles_dir.
|
|
425
|
-
return
|
|
426
|
-
|
|
427
|
-
profiles_path = Path.home() / ".dbt/profiles.yml"
|
|
428
|
-
config.profiles_dir = str(profiles_path.parent)
|
|
429
|
-
if not profiles_path.exists():
|
|
430
|
-
profiles_path.parent.mkdir(exist_ok=True)
|
|
431
|
-
with profiles_path.open("w", encoding="utf-8") as f:
|
|
432
|
-
f.write("flags:\n send_anonymous_usage_stats: false\n")
|
|
433
|
-
|
|
434
|
-
def get_dbt_target_from_connection(
|
|
435
|
-
self, target: Optional[str]
|
|
436
|
-
) -> Optional[dict[str, Any]]:
|
|
437
|
-
"""Return a dictionary of connection details to use as a dbt target.
|
|
438
|
-
|
|
439
|
-
The connection details are fetched from an Airflow connection identified by
|
|
440
|
-
target or self.dbt_conn_id.
|
|
441
|
-
|
|
442
|
-
Args:
|
|
443
|
-
target: The target name to use as an Airflow connection ID. If ommitted, we
|
|
444
|
-
will use self.dbt_conn_id.
|
|
445
|
-
|
|
446
|
-
Returns:
|
|
447
|
-
A dictionary with a configuration for a dbt target, or None if a matching
|
|
448
|
-
Airflow connection is not found for given dbt target.
|
|
449
|
-
"""
|
|
450
|
-
conn_id = target or self.dbt_conn_id
|
|
451
|
-
|
|
452
|
-
if conn_id is None:
|
|
453
|
-
return None
|
|
454
|
-
|
|
455
|
-
try:
|
|
456
|
-
conn = self.get_connection(conn_id)
|
|
457
|
-
except AirflowException:
|
|
458
|
-
self.log.debug(
|
|
459
|
-
"No Airflow connection matching dbt target %s was found.", target
|
|
460
|
-
)
|
|
461
|
-
return None
|
|
462
|
-
|
|
463
|
-
details = self.get_dbt_details_from_connection(conn)
|
|
464
|
-
|
|
465
|
-
return {conn_id: details}
|
|
466
|
-
|
|
467
|
-
def get_dbt_details_from_connection(self, conn: Connection) -> dict[str, Any]:
|
|
468
|
-
"""Extract dbt connection details from Airflow Connection.
|
|
469
|
-
|
|
470
|
-
dbt connection details may be present as Airflow Connection attributes or in the
|
|
471
|
-
Connection's extras. This class' conn_params and conn_extra_params will be used
|
|
472
|
-
to fetch required attributes from attributes and extras respectively. If
|
|
473
|
-
conn_extra_params is empty, we merge parameters with all extras.
|
|
474
|
-
|
|
475
|
-
Subclasses may override this class attributes to narrow down the connection
|
|
476
|
-
details for a specific dbt target (like Postgres, or Redshift).
|
|
477
|
-
|
|
478
|
-
Args:
|
|
479
|
-
conn: The Airflow Connection to extract dbt connection details from.
|
|
480
|
-
|
|
481
|
-
Returns:
|
|
482
|
-
A dictionary of dbt connection details.
|
|
483
|
-
"""
|
|
484
|
-
dbt_details = {}
|
|
485
|
-
for param in self.conn_params:
|
|
486
|
-
if isinstance(param, DbtConnectionParam):
|
|
487
|
-
key = param.override_name
|
|
488
|
-
value = getattr(conn, param.name, param.default)
|
|
489
|
-
else:
|
|
490
|
-
key = param
|
|
491
|
-
value = getattr(conn, key, None)
|
|
492
|
-
|
|
493
|
-
if value is None:
|
|
494
|
-
continue
|
|
495
|
-
|
|
496
|
-
dbt_details[key] = value
|
|
497
|
-
|
|
498
|
-
extra = conn.extra_dejson
|
|
499
|
-
|
|
500
|
-
if not self.conn_extra_params:
|
|
501
|
-
return {**dbt_details, **extra}
|
|
502
|
-
|
|
503
|
-
for param in self.conn_extra_params:
|
|
504
|
-
if isinstance(param, DbtConnectionParam):
|
|
505
|
-
key = param.override_name
|
|
506
|
-
value = extra.get(param.name, param.default)
|
|
507
|
-
else:
|
|
508
|
-
key = param
|
|
509
|
-
value = extra.get(key, None)
|
|
510
|
-
|
|
511
|
-
if value is None:
|
|
512
|
-
continue
|
|
513
|
-
|
|
514
|
-
dbt_details[key] = value
|
|
515
|
-
|
|
516
|
-
return dbt_details
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
class DbtPostgresHook(DbtHook):
|
|
520
|
-
"""A hook to interact with dbt using a Postgres connection."""
|
|
521
|
-
|
|
522
|
-
conn_type = "postgres"
|
|
523
|
-
hook_name = "dbt Postgres Hook"
|
|
524
|
-
conn_params = [
|
|
525
|
-
DbtConnectionParam("conn_type", "type", "postgres"),
|
|
526
|
-
"host",
|
|
527
|
-
"schema",
|
|
528
|
-
DbtConnectionParam("login", "user"),
|
|
529
|
-
"password",
|
|
530
|
-
"port",
|
|
531
|
-
]
|
|
532
|
-
conn_extra_params = [
|
|
533
|
-
"dbname",
|
|
534
|
-
"threads",
|
|
535
|
-
"keepalives_idle",
|
|
536
|
-
"connect_timeout",
|
|
537
|
-
"retries",
|
|
538
|
-
"search_path",
|
|
539
|
-
"role",
|
|
540
|
-
"sslmode",
|
|
541
|
-
]
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
class DbtRedshiftHook(DbtPostgresHook):
|
|
545
|
-
"""A hook to interact with dbt using a Redshift connection."""
|
|
546
|
-
|
|
547
|
-
conn_type = "redshift"
|
|
548
|
-
hook_name = "dbt Redshift Hook"
|
|
549
|
-
conn_extra_params = DbtPostgresHook.conn_extra_params + [
|
|
550
|
-
"ra3_node",
|
|
551
|
-
"iam_profile",
|
|
552
|
-
"iam_duration_secons",
|
|
553
|
-
"autocreate",
|
|
554
|
-
"db_groups",
|
|
555
|
-
]
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
class DbtSnowflakeHook(DbtHook):
|
|
559
|
-
"""A hook to interact with dbt using a Snowflake connection."""
|
|
560
|
-
|
|
561
|
-
conn_type = "snowflake"
|
|
562
|
-
hook_name = "dbt Snowflake Hook"
|
|
563
|
-
conn_params = [
|
|
564
|
-
DbtConnectionParam("conn_type", "type", "postgres"),
|
|
565
|
-
"host",
|
|
566
|
-
"schema",
|
|
567
|
-
DbtConnectionParam("login", "user"),
|
|
568
|
-
"password",
|
|
569
|
-
]
|
|
570
|
-
conn_extra_params = [
|
|
571
|
-
"account",
|
|
572
|
-
"role",
|
|
573
|
-
"database",
|
|
574
|
-
"warehouse",
|
|
575
|
-
"threads",
|
|
576
|
-
"client_session_keep_alive",
|
|
577
|
-
"query_tag",
|
|
578
|
-
"connect_retries",
|
|
579
|
-
"connect_timeout",
|
|
580
|
-
"retry_on_database_errors",
|
|
581
|
-
"retry_all",
|
|
582
|
-
]
|
|
@@ -1,11 +1,12 @@
|
|
|
1
|
-
"""The
|
|
1
|
+
"""The DbtFSHook interface includes methods for downloading and uploading files.
|
|
2
2
|
|
|
3
|
-
Internally,
|
|
3
|
+
Internally, DbtFSHooks can use Airflow hooks to execute the actual operations.
|
|
4
4
|
|
|
5
5
|
Currently, only AWS S3 and the local filesystem are supported as remotes.
|
|
6
6
|
"""
|
|
7
7
|
|
|
8
8
|
from abc import ABC, abstractmethod
|
|
9
|
+
from functools import cache
|
|
9
10
|
from pathlib import Path
|
|
10
11
|
from typing import Optional, Type
|
|
11
12
|
|
|
@@ -16,7 +17,7 @@ from airflow_dbt_python.utils.url import URL, URLLike
|
|
|
16
17
|
StrPath = str
|
|
17
18
|
|
|
18
19
|
|
|
19
|
-
class
|
|
20
|
+
class DbtFSHook(ABC, LoggingMixin):
|
|
20
21
|
"""Represents a dbt project storing any dbt files.
|
|
21
22
|
|
|
22
23
|
A concrete backend class should implement the push and pull methods to fetch one
|
|
@@ -32,7 +33,7 @@ class DbtRemoteHook(ABC, LoggingMixin):
|
|
|
32
33
|
"""
|
|
33
34
|
|
|
34
35
|
@abstractmethod
|
|
35
|
-
def
|
|
36
|
+
def _download(
|
|
36
37
|
self,
|
|
37
38
|
source: URL,
|
|
38
39
|
destination: URL,
|
|
@@ -43,7 +44,7 @@ class DbtRemoteHook(ABC, LoggingMixin):
|
|
|
43
44
|
return NotImplemented
|
|
44
45
|
|
|
45
46
|
@abstractmethod
|
|
46
|
-
def
|
|
47
|
+
def _upload(
|
|
47
48
|
self,
|
|
48
49
|
source: URL,
|
|
49
50
|
destination: URL,
|
|
@@ -71,7 +72,7 @@ class DbtRemoteHook(ABC, LoggingMixin):
|
|
|
71
72
|
if source_url.is_archive():
|
|
72
73
|
destination_url = destination_url / source_url.name
|
|
73
74
|
|
|
74
|
-
self.
|
|
75
|
+
self._download(source_url, destination_url)
|
|
75
76
|
|
|
76
77
|
if destination_url.exists() and destination_url.is_archive():
|
|
77
78
|
destination_url.extract()
|
|
@@ -103,7 +104,7 @@ class DbtRemoteHook(ABC, LoggingMixin):
|
|
|
103
104
|
if destination_url.is_dir() or destination_url.name != "profiles.yml":
|
|
104
105
|
destination_url = destination_url / "profiles.yml"
|
|
105
106
|
|
|
106
|
-
self.
|
|
107
|
+
self._download(source_url, destination_url)
|
|
107
108
|
|
|
108
109
|
return destination_url.path
|
|
109
110
|
|
|
@@ -133,35 +134,40 @@ class DbtRemoteHook(ABC, LoggingMixin):
|
|
|
133
134
|
source_url.archive(zip_url)
|
|
134
135
|
source_url = zip_url
|
|
135
136
|
|
|
136
|
-
self.
|
|
137
|
+
self._upload(source_url, destination_url, replace, delete_before)
|
|
137
138
|
|
|
138
139
|
if destination_url.is_archive():
|
|
139
140
|
source_url.unlink()
|
|
140
141
|
|
|
141
142
|
|
|
142
|
-
|
|
143
|
-
|
|
143
|
+
@cache
|
|
144
|
+
def get_fs_hook(scheme: str, conn_id: Optional[str] = None) -> DbtFSHook:
|
|
145
|
+
"""Get a DbtFSHook as long as the scheme is supported.
|
|
144
146
|
|
|
145
147
|
In the future we should make our hooks discoverable and package ourselves as a
|
|
146
148
|
proper Airflow providers package.
|
|
147
149
|
"""
|
|
148
150
|
if scheme == "s3":
|
|
149
|
-
from .s3 import
|
|
151
|
+
from .s3 import DbtS3FSHook
|
|
150
152
|
|
|
151
|
-
|
|
153
|
+
fs_hook_cls: Type[DbtFSHook] = DbtS3FSHook
|
|
154
|
+
elif scheme == "gs":
|
|
155
|
+
from .gcs import DbtGCSFSHook
|
|
156
|
+
|
|
157
|
+
fs_hook_cls = DbtGCSFSHook
|
|
152
158
|
elif scheme in ("https", "git", "git+ssh", "ssh", "http"):
|
|
153
|
-
from .git import
|
|
159
|
+
from .git import DbtGitFSHook
|
|
154
160
|
|
|
155
|
-
|
|
161
|
+
fs_hook_cls = DbtGitFSHook
|
|
156
162
|
elif scheme == "":
|
|
157
|
-
from .
|
|
163
|
+
from .local import DbtLocalFsHook
|
|
158
164
|
|
|
159
|
-
|
|
165
|
+
fs_hook_cls = DbtLocalFsHook
|
|
160
166
|
else:
|
|
161
167
|
raise NotImplementedError(f"Backend {scheme} is not supported")
|
|
162
168
|
|
|
163
169
|
if conn_id is not None:
|
|
164
|
-
|
|
170
|
+
fs_hook = fs_hook_cls(conn_id)
|
|
165
171
|
else:
|
|
166
|
-
|
|
167
|
-
return
|
|
172
|
+
fs_hook = fs_hook_cls()
|
|
173
|
+
return fs_hook
|