airflow-dbt-python 2.2.0__tar.gz → 3.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (23) hide show
  1. {airflow_dbt_python-2.2.0 → airflow_dbt_python-3.0.0}/PKG-INFO +19 -13
  2. {airflow_dbt_python-2.2.0 → airflow_dbt_python-3.0.0}/README.md +1 -1
  3. {airflow_dbt_python-2.2.0 → airflow_dbt_python-3.0.0}/airflow_dbt_python/__version__.py +1 -1
  4. airflow_dbt_python-3.0.0/airflow_dbt_python/hooks/__init__.py +1 -0
  5. {airflow_dbt_python-2.2.0 → airflow_dbt_python-3.0.0}/airflow_dbt_python/hooks/dbt.py +47 -250
  6. airflow_dbt_python-2.2.0/airflow_dbt_python/hooks/remote.py → airflow_dbt_python-3.0.0/airflow_dbt_python/hooks/fs/__init__.py +25 -19
  7. airflow_dbt_python-3.0.0/airflow_dbt_python/hooks/fs/gcs.py +228 -0
  8. {airflow_dbt_python-2.2.0/airflow_dbt_python/hooks → airflow_dbt_python-3.0.0/airflow_dbt_python/hooks/fs}/git.py +21 -14
  9. airflow_dbt_python-2.2.0/airflow_dbt_python/hooks/localfs.py → airflow_dbt_python-3.0.0/airflow_dbt_python/hooks/fs/local.py +10 -38
  10. {airflow_dbt_python-2.2.0/airflow_dbt_python/hooks → airflow_dbt_python-3.0.0/airflow_dbt_python/hooks/fs}/s3.py +8 -8
  11. airflow_dbt_python-3.0.0/airflow_dbt_python/hooks/target.py +464 -0
  12. {airflow_dbt_python-2.2.0 → airflow_dbt_python-3.0.0}/airflow_dbt_python/operators/dbt.py +120 -6
  13. {airflow_dbt_python-2.2.0 → airflow_dbt_python-3.0.0}/airflow_dbt_python/utils/configs.py +52 -9
  14. {airflow_dbt_python-2.2.0 → airflow_dbt_python-3.0.0}/pyproject.toml +59 -49
  15. airflow_dbt_python-2.2.0/airflow_dbt_python/hooks/__init__.py +0 -1
  16. {airflow_dbt_python-2.2.0 → airflow_dbt_python-3.0.0}/LICENSE +0 -0
  17. {airflow_dbt_python-2.2.0 → airflow_dbt_python-3.0.0}/airflow_dbt_python/__init__.py +0 -0
  18. {airflow_dbt_python-2.2.0 → airflow_dbt_python-3.0.0}/airflow_dbt_python/operators/__init__.py +0 -0
  19. {airflow_dbt_python-2.2.0 → airflow_dbt_python-3.0.0}/airflow_dbt_python/utils/__init__.py +0 -0
  20. {airflow_dbt_python-2.2.0 → airflow_dbt_python-3.0.0}/airflow_dbt_python/utils/enums.py +0 -0
  21. {airflow_dbt_python-2.2.0 → airflow_dbt_python-3.0.0}/airflow_dbt_python/utils/env.py +0 -0
  22. {airflow_dbt_python-2.2.0 → airflow_dbt_python-3.0.0}/airflow_dbt_python/utils/url.py +0 -0
  23. {airflow_dbt_python-2.2.0 → airflow_dbt_python-3.0.0}/airflow_dbt_python/utils/version.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.3
2
2
  Name: airflow-dbt-python
3
- Version: 2.2.0
3
+ Version: 3.0.0
4
4
  Summary: A collection of Airflow operators, hooks, and utilities to execute dbt commands
5
5
  License: MIT
6
6
  Author: Tomás Farías Santana
@@ -9,7 +9,6 @@ Requires-Python: >=3.9,<3.13
9
9
  Classifier: Development Status :: 5 - Production/Stable
10
10
  Classifier: Intended Audience :: Developers
11
11
  Classifier: License :: OSI Approved :: MIT License
12
- Classifier: Programming Language :: Python :: 3
13
12
  Classifier: Programming Language :: Python :: 3.9
14
13
  Classifier: Programming Language :: Python :: 3.10
15
14
  Classifier: Programming Language :: Python :: 3.11
@@ -17,7 +16,7 @@ Classifier: Programming Language :: Python :: 3.12
17
16
  Provides-Extra: adapters
18
17
  Provides-Extra: airflow-providers
19
18
  Provides-Extra: bigquery
20
- Provides-Extra: docs
19
+ Provides-Extra: gcs
21
20
  Provides-Extra: git
22
21
  Provides-Extra: postgres
23
22
  Provides-Extra: redshift
@@ -25,18 +24,25 @@ Provides-Extra: s3
25
24
  Provides-Extra: snowflake
26
25
  Provides-Extra: spark
27
26
  Requires-Dist: apache-airflow (>=2.8)
28
- Requires-Dist: apache-airflow-providers-amazon (>=3.0.0) ; extra == "airflow-providers" or extra == "s3"
29
- Requires-Dist: apache-airflow-providers-ssh (>=3.0.0) ; extra == "airflow-providers" or extra == "git"
30
- Requires-Dist: connexion (<3.0)
27
+ Requires-Dist: apache-airflow-providers-amazon (>=3.0.0) ; extra == "airflow-providers"
28
+ Requires-Dist: apache-airflow-providers-amazon (>=3.0.0) ; extra == "s3"
29
+ Requires-Dist: apache-airflow-providers-google (>=11.0.0) ; extra == "airflow-providers"
30
+ Requires-Dist: apache-airflow-providers-google (>=11.0.0) ; extra == "gcs"
31
+ Requires-Dist: apache-airflow-providers-ssh (>=3.0.0) ; extra == "airflow-providers"
32
+ Requires-Dist: apache-airflow-providers-ssh (>=3.0.0) ; extra == "git"
31
33
  Requires-Dist: contextlib-chdir (==1.0.2) ; python_version < "3.11"
32
- Requires-Dist: dbt-bigquery (>=1.8.0,<2.0.0) ; extra == "adapters" or extra == "bigquery"
34
+ Requires-Dist: dbt-bigquery (>=1.8.0,<2.0.0) ; extra == "adapters"
35
+ Requires-Dist: dbt-bigquery (>=1.8.0,<2.0.0) ; extra == "bigquery"
33
36
  Requires-Dist: dbt-core (>=1.8.0,<2.0.0)
34
- Requires-Dist: dbt-postgres (>=1.8.0,<2.0.0) ; extra == "adapters" or extra == "postgres"
35
- Requires-Dist: dbt-redshift (>=1.8.0,<2.0.0) ; extra == "adapters" or extra == "redshift"
36
- Requires-Dist: dbt-snowflake (>=1.8.0,<2.0.0) ; extra == "adapters" or extra == "snowflake"
37
- Requires-Dist: dbt-spark (>=1.8.0,<2.0.0) ; extra == "adapters" or extra == "spark"
37
+ Requires-Dist: dbt-postgres (>=1.8.0,<2.0.0) ; extra == "adapters"
38
+ Requires-Dist: dbt-postgres (>=1.8.0,<2.0.0) ; extra == "postgres"
39
+ Requires-Dist: dbt-redshift (>=1.8.0,<2.0.0) ; extra == "adapters"
40
+ Requires-Dist: dbt-redshift (>=1.8.0,<2.0.0) ; extra == "redshift"
41
+ Requires-Dist: dbt-snowflake (>=1.8.0,<2.0.0) ; extra == "adapters"
42
+ Requires-Dist: dbt-snowflake (>=1.8.0,<2.0.0) ; extra == "snowflake"
43
+ Requires-Dist: dbt-spark (>=1.8.0,<2.0.0) ; extra == "spark"
44
+ Requires-Dist: dbt-spark[all] (>=1.8.0,<2.0.0) ; extra == "adapters"
38
45
  Requires-Dist: dulwich (>=0.21) ; extra == "git"
39
- Project-URL: Repository, https://github.com/tomasfarias/airflow-dbt-python
40
46
  Description-Content-Type: text/markdown
41
47
 
42
48
  # airflow-dbt-python
@@ -62,7 +68,7 @@ Before using *airflow-dbt-python*, ensure you meet the following requirements:
62
68
  * If using any managed service, like AWS MWAA or GCP Cloud Composer 2/3, ensure your environment is created with a supported version of Airflow.
63
69
  * If self-hosting, Airflow installation instructions can be found in their [official documentation](https://airflow.apache.org/docs/apache-airflow/stable/installation/index.html).
64
70
 
65
- * Running Python 3.8 or later in your Airflow environment.
71
+ * Running Python 3.9 or later in your Airflow environment.
66
72
 
67
73
  > **Warning**
68
74
  >
@@ -21,7 +21,7 @@ Before using *airflow-dbt-python*, ensure you meet the following requirements:
21
21
  * If using any managed service, like AWS MWAA or GCP Cloud Composer 2/3, ensure your environment is created with a supported version of Airflow.
22
22
  * If self-hosting, Airflow installation instructions can be found in their [official documentation](https://airflow.apache.org/docs/apache-airflow/stable/installation/index.html).
23
23
 
24
- * Running Python 3.8 or later in your Airflow environment.
24
+ * Running Python 3.9 or later in your Airflow environment.
25
25
 
26
26
  > **Warning**
27
27
  >
@@ -3,4 +3,4 @@
3
3
  __author__ = "Tomás Farías Santana"
4
4
  __copyright__ = "Copyright 2021 Tomás Farías Santana"
5
5
  __title__ = "airflow-dbt-python"
6
- __version__ = "2.2.0"
6
+ __version__ = "3.0.0"
@@ -0,0 +1 @@
1
+ """Hooks module provides DbtHooks and DbtFSHooks."""
@@ -5,6 +5,7 @@ from __future__ import annotations
5
5
  import json
6
6
  import logging
7
7
  import sys
8
+ from abc import ABC
8
9
  from contextlib import contextmanager
9
10
  from pathlib import Path
10
11
  from tempfile import TemporaryDirectory
@@ -17,13 +18,11 @@ from typing import (
17
18
  NamedTuple,
18
19
  Optional,
19
20
  Tuple,
20
- Union,
21
21
  )
22
22
  from urllib.parse import urlparse
23
23
 
24
24
  from airflow.exceptions import AirflowException
25
- from airflow.hooks.base import BaseHook
26
- from airflow.models.connection import Connection
25
+ from airflow.utils.log.logging_mixin import LoggingMixin
27
26
 
28
27
  if sys.version_info >= (3, 11):
29
28
  from contextlib import chdir as chdir_ctx
@@ -32,13 +31,13 @@ else:
32
31
 
33
32
  if TYPE_CHECKING:
34
33
  from dbt.contracts.results import RunResult
35
- from dbt.task.base import BaseTask
36
34
 
37
- from airflow_dbt_python.hooks.remote import DbtRemoteHook
35
+ from airflow_dbt_python.hooks.fs import DbtFSHook
36
+ from airflow_dbt_python.hooks.target import DbtConnectionHook
38
37
  from airflow_dbt_python.utils.configs import BaseConfig
39
38
  from airflow_dbt_python.utils.url import URLLike
40
39
 
41
- DbtRemoteHooksDict = Dict[Tuple[str, Optional[str]], DbtRemoteHook]
40
+ DbtFSHooksDict = Dict[Tuple[str, Optional[str]], DbtFSHook]
42
41
 
43
42
 
44
43
  class DbtTaskResult(NamedTuple):
@@ -55,35 +54,6 @@ class DbtTaskResult(NamedTuple):
55
54
  artifacts: dict[str, Any]
56
55
 
57
56
 
58
- class DbtConnectionParam(NamedTuple):
59
- """A tuple indicating connection parameters relevant to dbt.
60
-
61
- Attributes:
62
- name: The name of the connection parameter. This name will be used to get the
63
- parameter from an Airflow Connection or its extras.
64
- store_override_name: A new name for the connection parameter. If not None, this
65
- is the name used in a dbt profiles.
66
- default: A default value if the parameter is not found.
67
- """
68
-
69
- name: str
70
- store_override_name: Optional[str] = None
71
- default: Optional[Any] = None
72
-
73
- @property
74
- def override_name(self):
75
- """Returns the override_name if defined, otherwise defaults to name.
76
-
77
- >>> DbtConnectionParam("login", "user").override_name
78
- 'user'
79
- >>> DbtConnectionParam("port").override_name
80
- 'port'
81
- """
82
- if self.store_override_name is None:
83
- return self.name
84
- return self.store_override_name
85
-
86
-
87
57
  class DbtTemporaryDirectory(TemporaryDirectory):
88
58
  """A wrapper on TemporaryDirectory for older versions of Python.
89
59
 
@@ -105,57 +75,43 @@ class DbtTemporaryDirectory(TemporaryDirectory):
105
75
  )
106
76
 
107
77
 
108
- class DbtHook(BaseHook):
78
+ class DbtHook(ABC, LoggingMixin):
109
79
  """A hook to interact with dbt.
110
80
 
111
81
  Allows for running dbt tasks and provides required configurations for each task.
112
82
  """
113
83
 
114
- conn_name_attr = "dbt_conn_id"
115
- default_conn_name = "dbt_default"
116
- conn_type = "dbt"
117
- hook_name = "dbt Hook"
118
-
119
- conn_params: list[Union[DbtConnectionParam, str]] = [
120
- DbtConnectionParam("conn_type", "type"),
121
- "host",
122
- DbtConnectionParam("conn_id", "dbname"),
123
- "schema",
124
- DbtConnectionParam("login", "user"),
125
- "password",
126
- "port",
127
- ]
128
- conn_extra_params: list[Union[DbtConnectionParam, str]] = []
129
-
130
84
  def __init__(
131
85
  self,
132
86
  *args,
133
- dbt_conn_id: Optional[str] = default_conn_name,
87
+ dbt_conn_id: Optional[str] = None,
134
88
  project_conn_id: Optional[str] = None,
135
89
  profiles_conn_id: Optional[str] = None,
136
90
  **kwargs,
137
91
  ):
138
- self.remotes: DbtRemoteHooksDict = {}
139
92
  self.dbt_conn_id = dbt_conn_id
140
93
  self.project_conn_id = project_conn_id
141
94
  self.profiles_conn_id = profiles_conn_id
142
95
  super().__init__(*args, **kwargs)
143
96
 
144
- def get_remote(self, scheme: str, conn_id: Optional[str]) -> DbtRemoteHook:
145
- """Get a remote to interact with dbt files.
97
+ @staticmethod
98
+ def get_dbt_target_hook(conn_id: str) -> DbtConnectionHook:
99
+ """Get a hook to get a dbt profile based on the Airflow connection."""
100
+ from .target import DbtConnectionHook
146
101
 
147
- RemoteHooks are defined by the scheme we are looking for and an optional
102
+ return DbtConnectionHook.get_db_conn_hook(conn_id)
103
+
104
+ @staticmethod
105
+ def get_fs_hook(scheme: str, conn_id: Optional[str]) -> DbtFSHook:
106
+ """Get a fs_hook to interact with dbt files.
107
+
108
+ FSHooks are defined by the scheme we are looking for and an optional
148
109
  connection id if we are looking to interface with any Airflow hook that
149
110
  uses a connection.
150
111
  """
151
- from .remote import get_remote
112
+ from .fs import get_fs_hook
152
113
 
153
- try:
154
- return self.remotes[(scheme, conn_id)]
155
- except KeyError:
156
- remote = get_remote(scheme, conn_id)
157
- self.remotes[(scheme, conn_id)] = remote
158
- return remote
114
+ return get_fs_hook(scheme, conn_id)
159
115
 
160
116
  def download_dbt_profiles(
161
117
  self,
@@ -164,13 +120,13 @@ class DbtHook(BaseHook):
164
120
  ) -> Path:
165
121
  """Pull a dbt profiles.yml file from a given profiles_dir.
166
122
 
167
- This operation is delegated to a DbtRemoteHook. An optional connection id is
123
+ This operation is delegated to a DbtFSHook. An optional connection id is
168
124
  supported for remotes that require it.
169
125
  """
170
126
  scheme = urlparse(str(profiles_dir)).scheme
171
- remote = self.get_remote(scheme, self.project_conn_id)
127
+ fs_hook = self.get_fs_hook(scheme, self.profiles_conn_id)
172
128
 
173
- return remote.download_dbt_profiles(profiles_dir, destination)
129
+ return fs_hook.download_dbt_profiles(profiles_dir, destination)
174
130
 
175
131
  def download_dbt_project(
176
132
  self,
@@ -179,13 +135,13 @@ class DbtHook(BaseHook):
179
135
  ) -> Path:
180
136
  """Pull a dbt project from a given project_dir.
181
137
 
182
- This operation is delegated to a DbtRemoteHook. An optional connection id is
138
+ This operation is delegated to a DbtFSHook. An optional connection id is
183
139
  supported for remotes that require it.
184
140
  """
185
141
  scheme = urlparse(str(project_dir)).scheme
186
- remote = self.get_remote(scheme, self.project_conn_id)
142
+ fs_hook = self.get_fs_hook(scheme, self.project_conn_id)
187
143
 
188
- return remote.download_dbt_project(project_dir, destination)
144
+ return fs_hook.download_dbt_project(project_dir, destination)
189
145
 
190
146
  def upload_dbt_project(
191
147
  self,
@@ -196,13 +152,13 @@ class DbtHook(BaseHook):
196
152
  ) -> None:
197
153
  """Push a dbt project from a given project_dir.
198
154
 
199
- This operation is delegated to a DbtRemoteHook. An optional connection id is
155
+ This operation is delegated to a DbtFSHook. An optional connection id is
200
156
  supported for remotes that require it.
201
157
  """
202
158
  scheme = urlparse(str(destination)).scheme
203
- remote = self.get_remote(scheme, self.project_conn_id)
159
+ fs_hook = self.get_fs_hook(scheme, self.project_conn_id)
204
160
 
205
- return remote.upload_dbt_project(
161
+ return fs_hook.upload_dbt_project(
206
162
  project_dir, destination, replace=replace, delete_before=delete_before
207
163
  )
208
164
 
@@ -231,8 +187,14 @@ class DbtHook(BaseHook):
231
187
  from dbt.task.deps import DepsTask
232
188
  from dbt.tracking import track_run
233
189
 
190
+ if self.dbt_conn_id:
191
+ kwargs["target"] = self.dbt_conn_id
192
+ target_hook = self.get_dbt_target_hook(self.dbt_conn_id)
193
+ extra_target = target_hook.get_dbt_target_from_connection()
194
+ else:
195
+ extra_target = None
196
+
234
197
  config = self.get_dbt_task_config(command, **kwargs)
235
- extra_target = self.get_dbt_target_from_connection(config.target)
236
198
 
237
199
  with self.dbt_directory(
238
200
  config,
@@ -246,15 +208,13 @@ class DbtHook(BaseHook):
246
208
  nearest_project_dir = get_nearest_project_dir(config.project_dir)
247
209
 
248
210
  with chdir_ctx(nearest_project_dir):
249
- self.ensure_profiles(config)
250
-
251
211
  with adapter_management():
252
212
  task, runtime_config = config.create_dbt_task(
253
213
  extra_target, write_perf_info
254
214
  )
255
215
  requires_profile = isinstance(task, (CleanTask, DepsTask))
256
216
 
257
- self.setup_dbt_logging(task, config.debug)
217
+ self.setup_dbt_logging(config.debug)
258
218
 
259
219
  if runtime_config is not None and not requires_profile:
260
220
  # The deps command installs the dependencies, which means they
@@ -373,25 +333,24 @@ class DbtHook(BaseHook):
373
333
  project_dir,
374
334
  tmp_dir,
375
335
  )
376
- new_project_dir = str(project_dir_path) + "/"
377
-
378
- if (project_dir_path / "profiles.yml").exists():
379
- # We may have downloaded the profiles.yml file together
380
- # with the project.
381
- return new_project_dir, new_project_dir
382
336
 
383
337
  if profiles_dir is not None:
384
338
  profiles_file_path = self.download_dbt_profiles(
385
339
  profiles_dir,
386
340
  tmp_dir,
387
341
  )
388
- new_profiles_dir = str(profiles_file_path.parent) + "/"
342
+ profiles_dir_path = profiles_file_path.parent
343
+ elif (project_dir_path / "profiles.yml").exists():
344
+ profiles_dir_path = project_dir_path
389
345
  else:
390
- new_profiles_dir = None
346
+ profiles_dir_path = None
391
347
 
392
- return new_project_dir, new_profiles_dir
348
+ return (
349
+ str(project_dir_path),
350
+ str(profiles_dir_path) if profiles_dir_path is not None else None,
351
+ )
393
352
 
394
- def setup_dbt_logging(self, task: BaseTask, debug: Optional[bool]):
353
+ def setup_dbt_logging(self, debug: Optional[bool]):
395
354
  """Setup dbt logging.
396
355
 
397
356
  Starting with dbt v1, dbt initializes two loggers: default_file and
@@ -407,6 +366,7 @@ class DbtHook(BaseHook):
407
366
  configured_file = logging.getLogger("configured_file")
408
367
  file_log = logging.getLogger("file_log")
409
368
  stdout_log = logging.getLogger("stdout_log")
369
+ stdout_log.handlers.clear()
410
370
  stdout_log.propagate = True
411
371
 
412
372
  if not debug:
@@ -417,166 +377,3 @@ class DbtHook(BaseHook):
417
377
  file_log.propagate = False
418
378
  configured_file.setLevel("INFO")
419
379
  configured_file.propagate = False
420
-
421
- def ensure_profiles(self, config: BaseConfig):
422
- """Ensure a profiles file exists."""
423
- if config.profiles_dir is not None:
424
- # We expect one to exist given that we have passed a profiles_dir.
425
- return
426
-
427
- profiles_path = Path.home() / ".dbt/profiles.yml"
428
- config.profiles_dir = str(profiles_path.parent)
429
- if not profiles_path.exists():
430
- profiles_path.parent.mkdir(exist_ok=True)
431
- with profiles_path.open("w", encoding="utf-8") as f:
432
- f.write("flags:\n send_anonymous_usage_stats: false\n")
433
-
434
- def get_dbt_target_from_connection(
435
- self, target: Optional[str]
436
- ) -> Optional[dict[str, Any]]:
437
- """Return a dictionary of connection details to use as a dbt target.
438
-
439
- The connection details are fetched from an Airflow connection identified by
440
- target or self.dbt_conn_id.
441
-
442
- Args:
443
- target: The target name to use as an Airflow connection ID. If ommitted, we
444
- will use self.dbt_conn_id.
445
-
446
- Returns:
447
- A dictionary with a configuration for a dbt target, or None if a matching
448
- Airflow connection is not found for given dbt target.
449
- """
450
- conn_id = target or self.dbt_conn_id
451
-
452
- if conn_id is None:
453
- return None
454
-
455
- try:
456
- conn = self.get_connection(conn_id)
457
- except AirflowException:
458
- self.log.debug(
459
- "No Airflow connection matching dbt target %s was found.", target
460
- )
461
- return None
462
-
463
- details = self.get_dbt_details_from_connection(conn)
464
-
465
- return {conn_id: details}
466
-
467
- def get_dbt_details_from_connection(self, conn: Connection) -> dict[str, Any]:
468
- """Extract dbt connection details from Airflow Connection.
469
-
470
- dbt connection details may be present as Airflow Connection attributes or in the
471
- Connection's extras. This class' conn_params and conn_extra_params will be used
472
- to fetch required attributes from attributes and extras respectively. If
473
- conn_extra_params is empty, we merge parameters with all extras.
474
-
475
- Subclasses may override this class attributes to narrow down the connection
476
- details for a specific dbt target (like Postgres, or Redshift).
477
-
478
- Args:
479
- conn: The Airflow Connection to extract dbt connection details from.
480
-
481
- Returns:
482
- A dictionary of dbt connection details.
483
- """
484
- dbt_details = {}
485
- for param in self.conn_params:
486
- if isinstance(param, DbtConnectionParam):
487
- key = param.override_name
488
- value = getattr(conn, param.name, param.default)
489
- else:
490
- key = param
491
- value = getattr(conn, key, None)
492
-
493
- if value is None:
494
- continue
495
-
496
- dbt_details[key] = value
497
-
498
- extra = conn.extra_dejson
499
-
500
- if not self.conn_extra_params:
501
- return {**dbt_details, **extra}
502
-
503
- for param in self.conn_extra_params:
504
- if isinstance(param, DbtConnectionParam):
505
- key = param.override_name
506
- value = extra.get(param.name, param.default)
507
- else:
508
- key = param
509
- value = extra.get(key, None)
510
-
511
- if value is None:
512
- continue
513
-
514
- dbt_details[key] = value
515
-
516
- return dbt_details
517
-
518
-
519
- class DbtPostgresHook(DbtHook):
520
- """A hook to interact with dbt using a Postgres connection."""
521
-
522
- conn_type = "postgres"
523
- hook_name = "dbt Postgres Hook"
524
- conn_params = [
525
- DbtConnectionParam("conn_type", "type", "postgres"),
526
- "host",
527
- "schema",
528
- DbtConnectionParam("login", "user"),
529
- "password",
530
- "port",
531
- ]
532
- conn_extra_params = [
533
- "dbname",
534
- "threads",
535
- "keepalives_idle",
536
- "connect_timeout",
537
- "retries",
538
- "search_path",
539
- "role",
540
- "sslmode",
541
- ]
542
-
543
-
544
- class DbtRedshiftHook(DbtPostgresHook):
545
- """A hook to interact with dbt using a Redshift connection."""
546
-
547
- conn_type = "redshift"
548
- hook_name = "dbt Redshift Hook"
549
- conn_extra_params = DbtPostgresHook.conn_extra_params + [
550
- "ra3_node",
551
- "iam_profile",
552
- "iam_duration_secons",
553
- "autocreate",
554
- "db_groups",
555
- ]
556
-
557
-
558
- class DbtSnowflakeHook(DbtHook):
559
- """A hook to interact with dbt using a Snowflake connection."""
560
-
561
- conn_type = "snowflake"
562
- hook_name = "dbt Snowflake Hook"
563
- conn_params = [
564
- DbtConnectionParam("conn_type", "type", "postgres"),
565
- "host",
566
- "schema",
567
- DbtConnectionParam("login", "user"),
568
- "password",
569
- ]
570
- conn_extra_params = [
571
- "account",
572
- "role",
573
- "database",
574
- "warehouse",
575
- "threads",
576
- "client_session_keep_alive",
577
- "query_tag",
578
- "connect_retries",
579
- "connect_timeout",
580
- "retry_on_database_errors",
581
- "retry_all",
582
- ]
@@ -1,11 +1,12 @@
1
- """The DbtRemoteHook interface includes methods for downloading and uploading files.
1
+ """The DbtFSHook interface includes methods for downloading and uploading files.
2
2
 
3
- Internally, DbtRemoteHooks can use Airflow hooks to execute the actual operations.
3
+ Internally, DbtFSHooks can use Airflow hooks to execute the actual operations.
4
4
 
5
5
  Currently, only AWS S3 and the local filesystem are supported as remotes.
6
6
  """
7
7
 
8
8
  from abc import ABC, abstractmethod
9
+ from functools import cache
9
10
  from pathlib import Path
10
11
  from typing import Optional, Type
11
12
 
@@ -16,7 +17,7 @@ from airflow_dbt_python.utils.url import URL, URLLike
16
17
  StrPath = str
17
18
 
18
19
 
19
- class DbtRemoteHook(ABC, LoggingMixin):
20
+ class DbtFSHook(ABC, LoggingMixin):
20
21
  """Represents a dbt project storing any dbt files.
21
22
 
22
23
  A concrete backend class should implement the push and pull methods to fetch one
@@ -32,7 +33,7 @@ class DbtRemoteHook(ABC, LoggingMixin):
32
33
  """
33
34
 
34
35
  @abstractmethod
35
- def download(
36
+ def _download(
36
37
  self,
37
38
  source: URL,
38
39
  destination: URL,
@@ -43,7 +44,7 @@ class DbtRemoteHook(ABC, LoggingMixin):
43
44
  return NotImplemented
44
45
 
45
46
  @abstractmethod
46
- def upload(
47
+ def _upload(
47
48
  self,
48
49
  source: URL,
49
50
  destination: URL,
@@ -71,7 +72,7 @@ class DbtRemoteHook(ABC, LoggingMixin):
71
72
  if source_url.is_archive():
72
73
  destination_url = destination_url / source_url.name
73
74
 
74
- self.download(source_url, destination_url)
75
+ self._download(source_url, destination_url)
75
76
 
76
77
  if destination_url.exists() and destination_url.is_archive():
77
78
  destination_url.extract()
@@ -103,7 +104,7 @@ class DbtRemoteHook(ABC, LoggingMixin):
103
104
  if destination_url.is_dir() or destination_url.name != "profiles.yml":
104
105
  destination_url = destination_url / "profiles.yml"
105
106
 
106
- self.download(source_url, destination_url)
107
+ self._download(source_url, destination_url)
107
108
 
108
109
  return destination_url.path
109
110
 
@@ -133,35 +134,40 @@ class DbtRemoteHook(ABC, LoggingMixin):
133
134
  source_url.archive(zip_url)
134
135
  source_url = zip_url
135
136
 
136
- self.upload(source_url, destination_url, replace, delete_before)
137
+ self._upload(source_url, destination_url, replace, delete_before)
137
138
 
138
139
  if destination_url.is_archive():
139
140
  source_url.unlink()
140
141
 
141
142
 
142
- def get_remote(scheme: str, conn_id: Optional[str] = None) -> DbtRemoteHook:
143
- """Get a DbtRemoteHook as long as the scheme is supported.
143
+ @cache
144
+ def get_fs_hook(scheme: str, conn_id: Optional[str] = None) -> DbtFSHook:
145
+ """Get a DbtFSHook as long as the scheme is supported.
144
146
 
145
147
  In the future we should make our hooks discoverable and package ourselves as a
146
148
  proper Airflow providers package.
147
149
  """
148
150
  if scheme == "s3":
149
- from .s3 import DbtS3RemoteHook
151
+ from .s3 import DbtS3FSHook
150
152
 
151
- remote_cls: Type[DbtRemoteHook] = DbtS3RemoteHook
153
+ fs_hook_cls: Type[DbtFSHook] = DbtS3FSHook
154
+ elif scheme == "gs":
155
+ from .gcs import DbtGCSFSHook
156
+
157
+ fs_hook_cls = DbtGCSFSHook
152
158
  elif scheme in ("https", "git", "git+ssh", "ssh", "http"):
153
- from .git import DbtGitRemoteHook
159
+ from .git import DbtGitFSHook
154
160
 
155
- remote_cls = DbtGitRemoteHook
161
+ fs_hook_cls = DbtGitFSHook
156
162
  elif scheme == "":
157
- from .localfs import DbtLocalFsRemoteHook
163
+ from .local import DbtLocalFsHook
158
164
 
159
- remote_cls = DbtLocalFsRemoteHook
165
+ fs_hook_cls = DbtLocalFsHook
160
166
  else:
161
167
  raise NotImplementedError(f"Backend {scheme} is not supported")
162
168
 
163
169
  if conn_id is not None:
164
- remote = remote_cls(conn_id)
170
+ fs_hook = fs_hook_cls(conn_id)
165
171
  else:
166
- remote = remote_cls()
167
- return remote
172
+ fs_hook = fs_hook_cls()
173
+ return fs_hook