airflow-dbt-python 1.0.5__tar.gz → 2.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {airflow_dbt_python-1.0.5 → airflow_dbt_python-2.0.0}/PKG-INFO +19 -13
- {airflow_dbt_python-1.0.5 → airflow_dbt_python-2.0.0}/README.md +2 -2
- {airflow_dbt_python-1.0.5 → airflow_dbt_python-2.0.0}/airflow_dbt_python/__version__.py +1 -1
- {airflow_dbt_python-1.0.5 → airflow_dbt_python-2.0.0}/airflow_dbt_python/hooks/dbt.py +78 -42
- {airflow_dbt_python-1.0.5 → airflow_dbt_python-2.0.0}/airflow_dbt_python/hooks/git.py +1 -1
- {airflow_dbt_python-1.0.5 → airflow_dbt_python-2.0.0}/airflow_dbt_python/operators/dbt.py +23 -2
- {airflow_dbt_python-1.0.5 → airflow_dbt_python-2.0.0}/airflow_dbt_python/utils/configs.py +213 -46
- airflow_dbt_python-2.0.0/airflow_dbt_python/utils/version.py +30 -0
- {airflow_dbt_python-1.0.5 → airflow_dbt_python-2.0.0}/pyproject.toml +56 -19
- {airflow_dbt_python-1.0.5 → airflow_dbt_python-2.0.0}/setup.py +45 -15
- {airflow_dbt_python-1.0.5 → airflow_dbt_python-2.0.0}/LICENSE +0 -0
- {airflow_dbt_python-1.0.5 → airflow_dbt_python-2.0.0}/airflow_dbt_python/__init__.py +0 -0
- {airflow_dbt_python-1.0.5 → airflow_dbt_python-2.0.0}/airflow_dbt_python/hooks/__init__.py +0 -0
- {airflow_dbt_python-1.0.5 → airflow_dbt_python-2.0.0}/airflow_dbt_python/hooks/localfs.py +0 -0
- {airflow_dbt_python-1.0.5 → airflow_dbt_python-2.0.0}/airflow_dbt_python/hooks/remote.py +0 -0
- {airflow_dbt_python-1.0.5 → airflow_dbt_python-2.0.0}/airflow_dbt_python/hooks/s3.py +0 -0
- {airflow_dbt_python-1.0.5 → airflow_dbt_python-2.0.0}/airflow_dbt_python/operators/__init__.py +0 -0
- {airflow_dbt_python-1.0.5 → airflow_dbt_python-2.0.0}/airflow_dbt_python/utils/__init__.py +0 -0
- {airflow_dbt_python-1.0.5 → airflow_dbt_python-2.0.0}/airflow_dbt_python/utils/enums.py +0 -0
- {airflow_dbt_python-1.0.5 → airflow_dbt_python-2.0.0}/airflow_dbt_python/utils/env.py +0 -0
- {airflow_dbt_python-1.0.5 → airflow_dbt_python-2.0.0}/airflow_dbt_python/utils/url.py +0 -0
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: airflow-dbt-python
|
|
3
|
-
Version:
|
|
3
|
+
Version: 2.0.0
|
|
4
4
|
Summary: A collection of Airflow operators, hooks, and utilities to execute dbt commands
|
|
5
5
|
Home-page: https://github.com/tomasfarias/airflow-dbt-python
|
|
6
6
|
License: MIT
|
|
7
7
|
Author: Tomás Farías Santana
|
|
8
8
|
Author-email: tomas@tomasfarias.dev
|
|
9
|
-
Requires-Python: >=3.
|
|
9
|
+
Requires-Python: >=3.8,<3.12
|
|
10
10
|
Classifier: Development Status :: 5 - Production/Stable
|
|
11
11
|
Classifier: Intended Audience :: Developers
|
|
12
12
|
Classifier: License :: OSI Approved :: MIT License
|
|
@@ -17,11 +17,9 @@ Classifier: Programming Language :: Python :: 3.10
|
|
|
17
17
|
Classifier: Programming Language :: Python :: 3.11
|
|
18
18
|
Classifier: Programming Language :: Python :: 3.10
|
|
19
19
|
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
-
Classifier: Programming Language :: Python :: 3.7
|
|
21
20
|
Classifier: Programming Language :: Python :: 3.8
|
|
22
21
|
Classifier: Programming Language :: Python :: 3.9
|
|
23
22
|
Provides-Extra: adapters
|
|
24
|
-
Provides-Extra: airflow
|
|
25
23
|
Provides-Extra: airflow-providers
|
|
26
24
|
Provides-Extra: bigquery
|
|
27
25
|
Provides-Extra: docs
|
|
@@ -31,15 +29,23 @@ Provides-Extra: redshift
|
|
|
31
29
|
Provides-Extra: s3
|
|
32
30
|
Provides-Extra: snowflake
|
|
33
31
|
Provides-Extra: spark
|
|
34
|
-
Requires-Dist: apache-airflow (>=2.
|
|
32
|
+
Requires-Dist: apache-airflow (>=2.2)
|
|
35
33
|
Requires-Dist: apache-airflow-providers-amazon (>=3.0.0) ; extra == "airflow-providers" or extra == "s3"
|
|
36
34
|
Requires-Dist: apache-airflow-providers-ssh (>=3.0.0) ; extra == "airflow-providers" or extra == "git"
|
|
37
|
-
Requires-Dist:
|
|
38
|
-
Requires-Dist:
|
|
39
|
-
Requires-Dist: dbt-
|
|
40
|
-
Requires-Dist: dbt-
|
|
41
|
-
Requires-Dist: dbt-
|
|
42
|
-
Requires-Dist: dbt-
|
|
35
|
+
Requires-Dist: connexion (<3.0)
|
|
36
|
+
Requires-Dist: contextlib-chdir (==1.0.2) ; python_version < "3.11"
|
|
37
|
+
Requires-Dist: dbt-bigquery (>=1.4.0,<1.6.0) ; (python_version < "3.8") and (extra == "adapters" or extra == "bigquery")
|
|
38
|
+
Requires-Dist: dbt-bigquery (>=1.4.0,<1.8.0) ; (python_version >= "3.11") and (extra == "adapters" or extra == "bigquery")
|
|
39
|
+
Requires-Dist: dbt-core (>=1.4.0,<1.6.0) ; python_version < "3.8"
|
|
40
|
+
Requires-Dist: dbt-core (>=1.4.0,<1.8.0) ; python_version >= "3.11"
|
|
41
|
+
Requires-Dist: dbt-postgres (>=1.4.0,<1.6.0) ; (python_version < "3.8") and (extra == "adapters" or extra == "postgres")
|
|
42
|
+
Requires-Dist: dbt-postgres (>=1.4.0,<1.8.0) ; (python_version >= "3.11") and (extra == "adapters" or extra == "postgres")
|
|
43
|
+
Requires-Dist: dbt-redshift (>=1.4.0,<1.6.0) ; (python_version < "3.8") and (extra == "adapters" or extra == "redshift")
|
|
44
|
+
Requires-Dist: dbt-redshift (>=1.4.0,<1.8.0) ; (python_version >= "3.11") and (extra == "adapters" or extra == "redshift")
|
|
45
|
+
Requires-Dist: dbt-snowflake (>=1.4.0,<1.6.0) ; (python_version < "3.8") and (extra == "adapters" or extra == "snowflake")
|
|
46
|
+
Requires-Dist: dbt-snowflake (>=1.4.0,<1.8.0) ; (python_version >= "3.11") and (extra == "adapters" or extra == "snowflake")
|
|
47
|
+
Requires-Dist: dbt-spark (>=1.4.0,<1.6.0) ; (python_version < "3.8") and (extra == "adapters" or extra == "spark")
|
|
48
|
+
Requires-Dist: dbt-spark (>=1.4.0,<1.8.0) ; (python_version >= "3.11") and (extra == "adapters" or extra == "spark")
|
|
43
49
|
Requires-Dist: dulwich (>=0.21) ; extra == "git"
|
|
44
50
|
Project-URL: Repository, https://github.com/tomasfarias/airflow-dbt-python
|
|
45
51
|
Description-Content-Type: text/markdown
|
|
@@ -61,13 +67,13 @@ Read the [documentation](https://airflow-dbt-python.readthedocs.io) for examples
|
|
|
61
67
|
## Requirements
|
|
62
68
|
|
|
63
69
|
Before using *airflow-dbt-python*, ensure you meet the following requirements:
|
|
64
|
-
* A *dbt* project using [dbt-core](https://pypi.org/project/dbt-core/) version 1.
|
|
70
|
+
* A *dbt* project using [dbt-core](https://pypi.org/project/dbt-core/) version 1.4.0 or later.
|
|
65
71
|
* An Airflow environment using version 2.2 or later.
|
|
66
72
|
|
|
67
73
|
* If using any managed service, like AWS MWAA, ensure your environment is created with a supported version of Airflow.
|
|
68
74
|
* If self-hosting, Airflow installation instructions can be found in their [official documentation](https://airflow.apache.org/docs/apache-airflow/stable/installation/index.html).
|
|
69
75
|
|
|
70
|
-
* Running Python 3.
|
|
76
|
+
* Running Python 3.8 or later in your Airflow environment.
|
|
71
77
|
|
|
72
78
|
> **Warning**
|
|
73
79
|
>
|
|
@@ -15,13 +15,13 @@ Read the [documentation](https://airflow-dbt-python.readthedocs.io) for examples
|
|
|
15
15
|
## Requirements
|
|
16
16
|
|
|
17
17
|
Before using *airflow-dbt-python*, ensure you meet the following requirements:
|
|
18
|
-
* A *dbt* project using [dbt-core](https://pypi.org/project/dbt-core/) version 1.
|
|
18
|
+
* A *dbt* project using [dbt-core](https://pypi.org/project/dbt-core/) version 1.4.0 or later.
|
|
19
19
|
* An Airflow environment using version 2.2 or later.
|
|
20
20
|
|
|
21
21
|
* If using any managed service, like AWS MWAA, ensure your environment is created with a supported version of Airflow.
|
|
22
22
|
* If self-hosting, Airflow installation instructions can be found in their [official documentation](https://airflow.apache.org/docs/apache-airflow/stable/installation/index.html).
|
|
23
23
|
|
|
24
|
-
* Running Python 3.
|
|
24
|
+
* Running Python 3.8 or later in your Airflow environment.
|
|
25
25
|
|
|
26
26
|
> **Warning**
|
|
27
27
|
>
|
|
@@ -24,6 +24,14 @@ from airflow.exceptions import AirflowException
|
|
|
24
24
|
from airflow.hooks.base import BaseHook
|
|
25
25
|
from airflow.models.connection import Connection
|
|
26
26
|
|
|
27
|
+
from airflow_dbt_python.utils.version import DBT_INSTALLED_LESS_THAN_1_5
|
|
28
|
+
|
|
29
|
+
if sys.version_info >= (3, 11):
|
|
30
|
+
from contextlib import chdir as chdir_ctx
|
|
31
|
+
else:
|
|
32
|
+
from contextlib_chdir import chdir as chdir_ctx
|
|
33
|
+
|
|
34
|
+
|
|
27
35
|
if TYPE_CHECKING:
|
|
28
36
|
from dbt.contracts.results import RunResult
|
|
29
37
|
from dbt.task.base import BaseTask
|
|
@@ -208,6 +216,7 @@ class DbtHook(BaseHook):
|
|
|
208
216
|
replace_on_upload: bool = False,
|
|
209
217
|
artifacts: Optional[Iterable[str]] = None,
|
|
210
218
|
env_vars: Optional[Dict[str, Any]] = None,
|
|
219
|
+
write_perf_info: bool = False,
|
|
211
220
|
**kwargs,
|
|
212
221
|
) -> DbtTaskResult:
|
|
213
222
|
"""Run a dbt task with a given configuration and return the results.
|
|
@@ -219,9 +228,17 @@ class DbtHook(BaseHook):
|
|
|
219
228
|
of running the dbt command.
|
|
220
229
|
"""
|
|
221
230
|
from dbt.adapters.factory import register_adapter
|
|
222
|
-
from dbt.
|
|
223
|
-
from dbt.
|
|
224
|
-
from dbt.task.
|
|
231
|
+
from dbt.task.base import get_nearest_project_dir
|
|
232
|
+
from dbt.task.clean import CleanTask
|
|
233
|
+
from dbt.task.deps import DepsTask
|
|
234
|
+
|
|
235
|
+
from airflow_dbt_python.utils.version import DBT_INSTALLED_LESS_THAN_1_5
|
|
236
|
+
|
|
237
|
+
if DBT_INSTALLED_LESS_THAN_1_5:
|
|
238
|
+
from dbt.main import adapter_management, track_run # type: ignore
|
|
239
|
+
else:
|
|
240
|
+
from dbt.adapters.factory import adapter_management
|
|
241
|
+
from dbt.tracking import track_run
|
|
225
242
|
|
|
226
243
|
config = self.get_dbt_task_config(command, **kwargs)
|
|
227
244
|
extra_target = self.get_dbt_target_from_connection(config.target)
|
|
@@ -233,53 +250,62 @@ class DbtHook(BaseHook):
|
|
|
233
250
|
replace_on_upload=replace_on_upload,
|
|
234
251
|
env_vars=env_vars,
|
|
235
252
|
) as dbt_dir:
|
|
236
|
-
config.dbt_task.pre_init_hook(config)
|
|
237
|
-
self.ensure_profiles(config.profiles_dir)
|
|
238
|
-
|
|
239
|
-
task, runtime_config = config.create_dbt_task(extra_target)
|
|
240
|
-
|
|
241
253
|
# When creating tasks via from_args, dbt switches to the project directory.
|
|
242
254
|
# We have to do that here as we are not using from_args.
|
|
243
|
-
|
|
255
|
+
if DBT_INSTALLED_LESS_THAN_1_5:
|
|
256
|
+
# For compatibility with older versions of dbt, as the signature
|
|
257
|
+
# of move_to_nearest_project_dir changed in dbt-core 1.5 to take
|
|
258
|
+
# just the project_dir.
|
|
259
|
+
nearest_project_dir = get_nearest_project_dir(config) # type: ignore
|
|
260
|
+
else:
|
|
261
|
+
nearest_project_dir = get_nearest_project_dir(config.project_dir)
|
|
262
|
+
|
|
263
|
+
with chdir_ctx(nearest_project_dir):
|
|
264
|
+
config.dbt_task.pre_init_hook(config)
|
|
265
|
+
self.ensure_profiles(config)
|
|
244
266
|
|
|
245
|
-
|
|
267
|
+
task, runtime_config = config.create_dbt_task(
|
|
268
|
+
extra_target, write_perf_info
|
|
269
|
+
)
|
|
270
|
+
requires_profile = isinstance(task, (CleanTask, DepsTask))
|
|
246
271
|
|
|
247
|
-
|
|
248
|
-
|
|
272
|
+
self.setup_dbt_logging(task, config.debug)
|
|
273
|
+
|
|
274
|
+
if runtime_config is not None and not requires_profile:
|
|
249
275
|
# The deps command installs the dependencies, which means they may
|
|
250
276
|
# not exist before deps runs and the following would raise a
|
|
251
277
|
# CompilationError.
|
|
252
278
|
runtime_config.load_dependencies()
|
|
253
279
|
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
280
|
+
results = None
|
|
281
|
+
with adapter_management():
|
|
282
|
+
if not requires_profile:
|
|
283
|
+
if runtime_config is not None:
|
|
284
|
+
register_adapter(runtime_config)
|
|
259
285
|
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
286
|
+
with track_run(task):
|
|
287
|
+
results = task.run()
|
|
288
|
+
success = task.interpret_results(results)
|
|
263
289
|
|
|
264
|
-
|
|
265
|
-
|
|
290
|
+
if artifacts is None:
|
|
291
|
+
return DbtTaskResult(success, results, {})
|
|
266
292
|
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
293
|
+
saved_artifacts = {}
|
|
294
|
+
for artifact in artifacts:
|
|
295
|
+
artifact_path = Path(dbt_dir) / "target" / artifact
|
|
270
296
|
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
297
|
+
if not artifact_path.exists():
|
|
298
|
+
self.log.warning(
|
|
299
|
+
"Required dbt artifact %s was not found. "
|
|
300
|
+
"Perhaps dbt failed and couldn't generate it.",
|
|
301
|
+
artifact,
|
|
302
|
+
)
|
|
303
|
+
continue
|
|
278
304
|
|
|
279
|
-
|
|
280
|
-
|
|
305
|
+
with open(artifact_path) as artifact_file:
|
|
306
|
+
json_artifact = json.load(artifact_file)
|
|
281
307
|
|
|
282
|
-
|
|
308
|
+
saved_artifacts[artifact] = json_artifact
|
|
283
309
|
|
|
284
310
|
return DbtTaskResult(success, results, saved_artifacts)
|
|
285
311
|
|
|
@@ -373,7 +399,7 @@ class DbtHook(BaseHook):
|
|
|
373
399
|
if (project_dir_path / "profiles.yml").exists():
|
|
374
400
|
# We may have downloaded the profiles.yml file together
|
|
375
401
|
# with the project.
|
|
376
|
-
return
|
|
402
|
+
return new_project_dir, new_project_dir
|
|
377
403
|
|
|
378
404
|
if profiles_dir is not None:
|
|
379
405
|
profiles_file_path = self.download_dbt_profiles(
|
|
@@ -384,7 +410,7 @@ class DbtHook(BaseHook):
|
|
|
384
410
|
else:
|
|
385
411
|
new_profiles_dir = None
|
|
386
412
|
|
|
387
|
-
return
|
|
413
|
+
return new_project_dir, new_profiles_dir
|
|
388
414
|
|
|
389
415
|
def setup_dbt_logging(self, task: BaseTask, debug: Optional[bool]):
|
|
390
416
|
"""Setup dbt logging.
|
|
@@ -399,10 +425,18 @@ class DbtHook(BaseHook):
|
|
|
399
425
|
if task.config is not None:
|
|
400
426
|
log_path = getattr(task.config, "log_path", None)
|
|
401
427
|
|
|
402
|
-
|
|
428
|
+
if DBT_INSTALLED_LESS_THAN_1_5:
|
|
429
|
+
setup_event_logger(log_path or "logs")
|
|
430
|
+
else:
|
|
431
|
+
from dbt.flags import get_flags
|
|
432
|
+
|
|
433
|
+
flags = get_flags()
|
|
434
|
+
setup_event_logger(flags)
|
|
403
435
|
|
|
404
436
|
configured_file = logging.getLogger("configured_file")
|
|
405
437
|
file_log = logging.getLogger("file_log")
|
|
438
|
+
stdout_log = logging.getLogger("stdout_log")
|
|
439
|
+
stdout_log.propagate = True
|
|
406
440
|
|
|
407
441
|
if not debug:
|
|
408
442
|
# We have to do this after setting logs up as dbt hasn't
|
|
@@ -413,16 +447,18 @@ class DbtHook(BaseHook):
|
|
|
413
447
|
configured_file.setLevel("INFO")
|
|
414
448
|
configured_file.propagate = False
|
|
415
449
|
|
|
416
|
-
def ensure_profiles(self,
|
|
450
|
+
def ensure_profiles(self, config: BaseConfig):
|
|
417
451
|
"""Ensure a profiles file exists."""
|
|
418
|
-
if profiles_dir is not None:
|
|
419
|
-
# We expect one to exist given that we have
|
|
452
|
+
if config.profiles_dir is not None:
|
|
453
|
+
# We expect one to exist given that we have passed a profiles_dir.
|
|
420
454
|
return
|
|
421
455
|
|
|
422
456
|
profiles_path = Path.home() / ".dbt/profiles.yml"
|
|
457
|
+
config.profiles_dir = str(profiles_path.parent)
|
|
423
458
|
if not profiles_path.exists():
|
|
424
459
|
profiles_path.parent.mkdir(exist_ok=True)
|
|
425
|
-
profiles_path.
|
|
460
|
+
with profiles_path.open("w", encoding="utf-8") as f:
|
|
461
|
+
f.write("config:\n send_anonymous_usage_stats: false\n")
|
|
426
462
|
|
|
427
463
|
def get_dbt_target_from_connection(
|
|
428
464
|
self, target: Optional[str]
|
|
@@ -70,6 +70,9 @@ class DbtBaseOperator(BaseOperator):
|
|
|
70
70
|
warn_error: Optional[bool] = None,
|
|
71
71
|
# Logging
|
|
72
72
|
debug: Optional[bool] = None,
|
|
73
|
+
log_path: Optional[str] = None,
|
|
74
|
+
log_level: Optional[str] = None,
|
|
75
|
+
log_level_file: Optional[str] = None,
|
|
73
76
|
log_format: Optional[LogFormat] = None,
|
|
74
77
|
log_cache_events: Optional[bool] = False,
|
|
75
78
|
quiet: Optional[bool] = None,
|
|
@@ -80,12 +83,16 @@ class DbtBaseOperator(BaseOperator):
|
|
|
80
83
|
no_defer: Optional[bool] = None,
|
|
81
84
|
partial_parse: Optional[bool] = False,
|
|
82
85
|
no_partial_parse: Optional[bool] = None,
|
|
86
|
+
introspect: Optional[bool] = None,
|
|
87
|
+
no_introspect: Optional[bool] = None,
|
|
83
88
|
use_colors: Optional[bool] = None,
|
|
84
89
|
no_use_colors: Optional[bool] = None,
|
|
85
90
|
static_parser: Optional[bool] = None,
|
|
86
91
|
no_static_parser: Optional[bool] = None,
|
|
87
92
|
version_check: Optional[bool] = None,
|
|
88
93
|
no_version_check: Optional[bool] = None,
|
|
94
|
+
write_json: Optional[bool] = None,
|
|
95
|
+
write_perf_info: Optional[bool] = None,
|
|
89
96
|
anonymous_usage_stats: Optional[bool] = None,
|
|
90
97
|
no_anonymous_usage_stats: Optional[bool] = None,
|
|
91
98
|
# Extra features configuration
|
|
@@ -116,9 +123,12 @@ class DbtBaseOperator(BaseOperator):
|
|
|
116
123
|
self.warn_error = warn_error
|
|
117
124
|
|
|
118
125
|
self.debug = debug
|
|
126
|
+
self.log_path = log_path
|
|
119
127
|
self.log_cache_events = log_cache_events
|
|
120
128
|
self.quiet = quiet
|
|
121
129
|
self.no_print = no_print
|
|
130
|
+
self.log_level = log_level
|
|
131
|
+
self.log_level_file = log_level_file
|
|
122
132
|
self.log_format = log_format
|
|
123
133
|
self.record_timing_info = record_timing_info
|
|
124
134
|
|
|
@@ -137,9 +147,18 @@ class DbtBaseOperator(BaseOperator):
|
|
|
137
147
|
self.use_colors = use_colors
|
|
138
148
|
self.no_use_colors = no_use_colors
|
|
139
149
|
|
|
140
|
-
self.
|
|
150
|
+
self.introspect = introspect
|
|
151
|
+
self.no_introspect = no_introspect
|
|
152
|
+
|
|
153
|
+
self.version_check = version_check
|
|
141
154
|
self.no_version_check = no_version_check
|
|
142
155
|
|
|
156
|
+
self.write_json = write_json or (
|
|
157
|
+
do_xcom_push_artifacts and "run_results.json" in do_xcom_push_artifacts
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
self.write_perf_info = write_perf_info
|
|
161
|
+
|
|
143
162
|
self.dbt_conn_id = dbt_conn_id
|
|
144
163
|
self.profiles_conn_id = profiles_conn_id
|
|
145
164
|
self.project_conn_id = project_conn_id
|
|
@@ -174,7 +193,7 @@ class DbtBaseOperator(BaseOperator):
|
|
|
174
193
|
)
|
|
175
194
|
except Exception as e:
|
|
176
195
|
self.log.exception(
|
|
177
|
-
f"An error has
|
|
196
|
+
f"An error has occurred while executing dbt {self.command}", exc_info=e
|
|
178
197
|
)
|
|
179
198
|
raise AirflowException(
|
|
180
199
|
f"An error has occurred while executing dbt {self.command}"
|
|
@@ -428,11 +447,13 @@ class DbtCleanOperator(DbtBaseOperator):
|
|
|
428
447
|
self,
|
|
429
448
|
upload_dbt_project: bool = True,
|
|
430
449
|
delete_before_upload: bool = True,
|
|
450
|
+
clean_project_files_only: bool = True,
|
|
431
451
|
**kwargs,
|
|
432
452
|
) -> None:
|
|
433
453
|
super().__init__(**kwargs)
|
|
434
454
|
self.upload_dbt_project = upload_dbt_project
|
|
435
455
|
self.delete_before_upload = delete_before_upload
|
|
456
|
+
self.clean_project_files_only = clean_project_files_only
|
|
436
457
|
|
|
437
458
|
@property
|
|
438
459
|
def command(self) -> str:
|
|
@@ -1,15 +1,17 @@
|
|
|
1
1
|
"""A collection of configurations to interact with dbt-core."""
|
|
2
2
|
from __future__ import annotations
|
|
3
3
|
|
|
4
|
+
import argparse
|
|
4
5
|
import dataclasses
|
|
5
6
|
import json
|
|
6
7
|
import os
|
|
7
8
|
import pickle
|
|
8
9
|
from pathlib import Path
|
|
9
|
-
from typing import
|
|
10
|
+
from typing import Any, Optional, Type, Union
|
|
10
11
|
|
|
11
12
|
import dbt.flags as flags
|
|
12
13
|
import yaml
|
|
14
|
+
from dbt.adapters.factory import register_adapter
|
|
13
15
|
from dbt.clients import yaml_helper # type: ignore
|
|
14
16
|
from dbt.config.profile import Profile, read_profile
|
|
15
17
|
from dbt.config.project import PartialProject, Project
|
|
@@ -17,7 +19,8 @@ from dbt.config.renderer import DbtProjectYamlRenderer, ProfileRenderer
|
|
|
17
19
|
from dbt.config.runtime import RuntimeConfig
|
|
18
20
|
from dbt.contracts.graph.manifest import Manifest
|
|
19
21
|
from dbt.graph.graph import Graph
|
|
20
|
-
from dbt.
|
|
22
|
+
from dbt.helper_types import WarnErrorOptions
|
|
23
|
+
from dbt.task.base import BaseTask, ConfiguredTask
|
|
21
24
|
from dbt.task.build import BuildTask
|
|
22
25
|
from dbt.task.clean import CleanTask
|
|
23
26
|
from dbt.task.compile import CompileTask
|
|
@@ -26,18 +29,44 @@ from dbt.task.deps import DepsTask
|
|
|
26
29
|
from dbt.task.freshness import FreshnessTask
|
|
27
30
|
from dbt.task.generate import GenerateTask
|
|
28
31
|
from dbt.task.list import ListTask
|
|
29
|
-
from dbt.task.parse import ParseTask
|
|
30
32
|
from dbt.task.run import RunTask
|
|
31
33
|
from dbt.task.run_operation import RunOperationTask
|
|
34
|
+
from dbt.task.runnable import GraphRunnableTask
|
|
32
35
|
from dbt.task.seed import SeedTask
|
|
33
36
|
from dbt.task.snapshot import SnapshotTask
|
|
34
37
|
from dbt.task.test import TestTask
|
|
35
|
-
from dbt.tracking import initialize_from_flags
|
|
38
|
+
from dbt.tracking import initialize_from_flags as orig_initialize_from_flags
|
|
36
39
|
|
|
37
40
|
from airflow_dbt_python.utils.enums import FromStrEnum, LogFormat, Output
|
|
41
|
+
from airflow_dbt_python.utils.version import (
|
|
42
|
+
DBT_INSTALLED_GTE_1_5,
|
|
43
|
+
DBT_INSTALLED_GTE_1_7,
|
|
44
|
+
DBT_INSTALLED_LESS_THAN_1_5,
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
if DBT_INSTALLED_LESS_THAN_1_5:
|
|
48
|
+
# dbt 1.5 removed the ParseTask as it only generated a manifest.
|
|
49
|
+
# This task is now handled by a simple function.
|
|
50
|
+
# Airflow-dbt-python should still support a ParseTask that does the same,
|
|
51
|
+
# but this will require writing our own ParseTask class.
|
|
52
|
+
# In the meantime, we treat ParseTask as CompileTask (which also generates
|
|
53
|
+
# a manifest) if running with dbt-core >= 1.5.
|
|
54
|
+
from dbt.task.parse import ParseTask # type: ignore
|
|
55
|
+
|
|
56
|
+
# dbt 1.5 also changed the signature of initialize_from_flags, so we have
|
|
57
|
+
# this for backwards compatibility.
|
|
58
|
+
def initialize_from_flags(send_anonymous_usage_stats, profiles_dir):
|
|
59
|
+
"""Call initialize_from_flags without arguments.
|
|
60
|
+
|
|
61
|
+
Added for backwards compatibility with versions of dbt < 1.5.
|
|
62
|
+
"""
|
|
63
|
+
return orig_initialize_from_flags() # type: ignore
|
|
64
|
+
|
|
65
|
+
else:
|
|
66
|
+
from dbt.parser.manifest import ManifestLoader
|
|
38
67
|
|
|
39
|
-
|
|
40
|
-
|
|
68
|
+
ParseTask = CompileTask
|
|
69
|
+
initialize_from_flags = orig_initialize_from_flags
|
|
41
70
|
|
|
42
71
|
|
|
43
72
|
def parse_yaml_args(args: Optional[Union[str, dict[str, Any]]]) -> dict[str, Any]:
|
|
@@ -78,16 +107,24 @@ class BaseConfig:
|
|
|
78
107
|
single_threaded: Optional[bool] = None
|
|
79
108
|
threads: Optional[int] = None
|
|
80
109
|
use_experimental_parser: Optional[bool] = None
|
|
110
|
+
store_failures: Optional[bool] = None
|
|
81
111
|
vars: Optional[Union[str, dict[str, Any]]] = None
|
|
112
|
+
macro_debugging: Optional[bool] = None
|
|
82
113
|
warn_error: Optional[bool] = None
|
|
114
|
+
warn_error_options: dict[str, Any] = dataclasses.field(default_factory=dict)
|
|
115
|
+
indirect_selection: str = "eager"
|
|
83
116
|
|
|
84
117
|
# Logging
|
|
85
118
|
log_format: Optional[LogFormat] = None
|
|
86
|
-
|
|
119
|
+
log_format_file: Optional[LogFormat] = None
|
|
120
|
+
log_path: Optional[str] = None if DBT_INSTALLED_LESS_THAN_1_5 else "logs"
|
|
121
|
+
log_level: str = "info"
|
|
122
|
+
log_level_file: str = "none"
|
|
87
123
|
record_timing_info: Optional[str] = None
|
|
88
124
|
debug: Optional[bool] = None
|
|
89
125
|
quiet: Optional[bool] = None
|
|
90
126
|
no_print: Optional[bool] = None
|
|
127
|
+
printer_width: Optional[int] = None
|
|
91
128
|
|
|
92
129
|
# Mutually exclusive attributes
|
|
93
130
|
defer: Optional[bool] = None
|
|
@@ -96,23 +133,62 @@ class BaseConfig:
|
|
|
96
133
|
partial_parse: Optional[bool] = None
|
|
97
134
|
no_partial_parse: Optional[bool] = dataclasses.field(default=None, repr=False)
|
|
98
135
|
|
|
136
|
+
log_cache_events: Optional[bool] = None
|
|
137
|
+
no_log_cache_events: Optional[bool] = dataclasses.field(default=None, repr=False)
|
|
138
|
+
|
|
99
139
|
use_colors: Optional[bool] = None
|
|
100
140
|
no_use_colors: Optional[bool] = dataclasses.field(default=None, repr=False)
|
|
101
141
|
|
|
142
|
+
use_colors_file: Optional[bool] = None
|
|
143
|
+
no_use_colors_file: Optional[bool] = dataclasses.field(default=None, repr=False)
|
|
144
|
+
|
|
145
|
+
introspect: Optional[bool] = None
|
|
146
|
+
no_introspect: Optional[bool] = dataclasses.field(default=None, repr=False)
|
|
147
|
+
|
|
148
|
+
populate_cache: Optional[bool] = None
|
|
149
|
+
no_populate_cache: Optional[bool] = dataclasses.field(default=None, repr=False)
|
|
150
|
+
|
|
102
151
|
static_parser: Optional[bool] = None
|
|
103
152
|
no_static_parser: Optional[bool] = dataclasses.field(default=None, repr=False)
|
|
104
153
|
|
|
105
154
|
version_check: Optional[bool] = None
|
|
106
155
|
no_version_check: Optional[bool] = dataclasses.field(default=None, repr=False)
|
|
107
156
|
|
|
108
|
-
|
|
109
|
-
anonymous_usage_stats: Optional[bool] =
|
|
157
|
+
# dbt < 1.5
|
|
158
|
+
anonymous_usage_stats: Optional[bool] = None
|
|
110
159
|
no_anonymous_usage_stats: Optional[bool] = dataclasses.field(
|
|
111
160
|
default=None, repr=False
|
|
112
161
|
)
|
|
113
162
|
|
|
114
|
-
|
|
115
|
-
|
|
163
|
+
# dbt >= 1.5
|
|
164
|
+
if DBT_INSTALLED_GTE_1_5:
|
|
165
|
+
send_anonymous_usage_stats: Optional[bool] = None
|
|
166
|
+
no_send_anonymous_usage_stats: Optional[bool] = dataclasses.field(
|
|
167
|
+
default=None, repr=False
|
|
168
|
+
)
|
|
169
|
+
|
|
170
|
+
write_json: Optional[bool] = None
|
|
171
|
+
no_write_json: Optional[bool] = dataclasses.field(default=None, repr=False)
|
|
172
|
+
|
|
173
|
+
# dbt >= 1.7
|
|
174
|
+
if DBT_INSTALLED_GTE_1_7:
|
|
175
|
+
from dbt.cli.option_types import Package
|
|
176
|
+
|
|
177
|
+
include_saved_query: Optional[bool] = None
|
|
178
|
+
no_include_saved_query: Optional[bool] = dataclasses.field(
|
|
179
|
+
default=None, repr=False
|
|
180
|
+
)
|
|
181
|
+
|
|
182
|
+
clean_project_files_only: Optional[bool] = None
|
|
183
|
+
no_clean_project_files_only: Optional[bool] = dataclasses.field(
|
|
184
|
+
default=None, repr=False
|
|
185
|
+
)
|
|
186
|
+
|
|
187
|
+
add_package: Optional[Package] = None
|
|
188
|
+
dry_run: bool = False
|
|
189
|
+
lock: bool = False
|
|
190
|
+
static: bool = False
|
|
191
|
+
upgrade: bool = False
|
|
116
192
|
|
|
117
193
|
def __post_init__(self):
|
|
118
194
|
"""Post initialization actions for a dbt configuration."""
|
|
@@ -121,6 +197,7 @@ class BaseConfig:
|
|
|
121
197
|
self.set_mutually_exclusive_attributes()
|
|
122
198
|
|
|
123
199
|
self.send_anonymous_usage_stats = self.anonymous_usage_stats
|
|
200
|
+
self.no_send_anonymous_usage_stats = self.no_anonymous_usage_stats
|
|
124
201
|
|
|
125
202
|
def set_mutually_exclusive_attributes(self):
|
|
126
203
|
"""Support pairs of mutually exclusive parameters.
|
|
@@ -133,28 +210,82 @@ class BaseConfig:
|
|
|
133
210
|
to non-None values.
|
|
134
211
|
"""
|
|
135
212
|
mutually_exclusive_attrs = (
|
|
136
|
-
"defer",
|
|
137
|
-
"partial_parse",
|
|
138
|
-
"
|
|
139
|
-
"
|
|
140
|
-
"
|
|
141
|
-
"
|
|
142
|
-
"
|
|
213
|
+
("defer", False),
|
|
214
|
+
("partial_parse", True),
|
|
215
|
+
("log_cache_events", False),
|
|
216
|
+
("use_colors", True),
|
|
217
|
+
("use_colors_file", True),
|
|
218
|
+
("introspect", True),
|
|
219
|
+
("populate_cache", True),
|
|
220
|
+
("static_parser", True),
|
|
221
|
+
("version_check", True),
|
|
222
|
+
("anonymous_usage_stats", True),
|
|
223
|
+
("send_anonymous_usage_stats", True),
|
|
224
|
+
("write_json", True),
|
|
225
|
+
("include_saved_query", None),
|
|
226
|
+
("clean_project_files_only", True),
|
|
143
227
|
)
|
|
144
228
|
|
|
145
|
-
for
|
|
146
|
-
|
|
147
|
-
negative_value = getattr(self, f"no_{attr}", None)
|
|
229
|
+
for attrs, default_value in mutually_exclusive_attrs:
|
|
230
|
+
attr, negative_attr = attrs, f"no_{attrs}"
|
|
148
231
|
|
|
149
|
-
|
|
150
|
-
|
|
232
|
+
positive_value = getattr(self, attr, None)
|
|
233
|
+
negative_value = getattr(self, negative_attr, None)
|
|
234
|
+
|
|
235
|
+
if (
|
|
236
|
+
positive_value is None
|
|
237
|
+
and negative_value is None
|
|
238
|
+
and default_value is not None
|
|
239
|
+
):
|
|
240
|
+
setattr(self, attr, default_value)
|
|
241
|
+
setattr(self, negative_attr, not default_value)
|
|
151
242
|
elif positive_value is not None and negative_value is not None:
|
|
152
|
-
raise ValueError(f"{attr} and
|
|
243
|
+
raise ValueError(f"{attr} and {negative_attr} are mutually exclusive")
|
|
153
244
|
elif positive_value is not None:
|
|
154
|
-
setattr(self,
|
|
245
|
+
setattr(self, negative_attr, not positive_value)
|
|
155
246
|
else:
|
|
156
247
|
setattr(self, attr, not negative_value)
|
|
157
248
|
|
|
249
|
+
def __getattribute__(self, item: str):
|
|
250
|
+
"""Dbt 1.5+ uses uppercase attributes, let's handle this."""
|
|
251
|
+
if item.isupper():
|
|
252
|
+
item = item.lower()
|
|
253
|
+
return super().__getattribute__(item)
|
|
254
|
+
|
|
255
|
+
def to_flags(self) -> argparse.Namespace:
|
|
256
|
+
"""Return a Namespace to use as dbt flags."""
|
|
257
|
+
flags_dict = vars(flags.get_flags())
|
|
258
|
+
flags_dict["MACRO_DEBUGGING"] = self.macro_debugging
|
|
259
|
+
flags_dict["WARN_ERROR"] = self.warn_error
|
|
260
|
+
flags_dict["WARN_ERROR_OPTIONS"] = WarnErrorOptions(
|
|
261
|
+
include=self.warn_error_options.get("include", []),
|
|
262
|
+
exclude=self.warn_error_options.get("exclude", []),
|
|
263
|
+
)
|
|
264
|
+
|
|
265
|
+
flags_dict["PRINTER_WIDTH"] = self.printer_width
|
|
266
|
+
flags_dict["TARGET_PATH"] = None
|
|
267
|
+
flags_dict["LOG_PATH"] = self.log_path
|
|
268
|
+
flags_dict["LOG_FORMAT"] = self.log_format
|
|
269
|
+
flags_dict["LOG_FORMAT_FILE"] = self.log_format
|
|
270
|
+
flags_dict["SEND_ANONYMOUS_USAGE_STATS"] = self.send_anonymous_usage_stats
|
|
271
|
+
flags_dict["PARTIAL_PARSE"] = self.partial_parse
|
|
272
|
+
flags_dict["STATIC_PARSER"] = self.static_parser
|
|
273
|
+
flags_dict["PROFILES_DIR"] = self.profiles_dir
|
|
274
|
+
flags_dict["STORE_FAILURES"] = self.store_failures
|
|
275
|
+
flags_dict["LOG_LEVEL"] = self.log_level
|
|
276
|
+
flags_dict["LOG_LEVEL_FILE"] = self.log_level_file
|
|
277
|
+
flags_dict["LOG_CACHE_EVENTS"] = self.log_cache_events
|
|
278
|
+
flags_dict["QUIET"] = self.quiet
|
|
279
|
+
flags_dict["DEBUG"] = self.debug
|
|
280
|
+
flags_dict["USE_COLORS_FILE"] = self.use_colors_file
|
|
281
|
+
flags_dict["WRITE_JSON"] = self.write_json
|
|
282
|
+
flags_dict["INDIRECT_SELECTION"] = getattr(self, "indirect_selection", None)
|
|
283
|
+
flags_dict["FAIL_FAST"] = self.fail_fast
|
|
284
|
+
flags_dict["INTROSPECT"] = self.introspect
|
|
285
|
+
|
|
286
|
+
flags_ns = argparse.Namespace(**flags_dict)
|
|
287
|
+
return flags_ns
|
|
288
|
+
|
|
158
289
|
@property
|
|
159
290
|
def dbt_task(self) -> Type[BaseTask]:
|
|
160
291
|
"""Access to the underlying dbt task class."""
|
|
@@ -228,7 +359,9 @@ class BaseConfig:
|
|
|
228
359
|
task._runtime_initialize = _runtime_initialize # type: ignore
|
|
229
360
|
|
|
230
361
|
def create_dbt_task(
|
|
231
|
-
self,
|
|
362
|
+
self,
|
|
363
|
+
extra_targets: Optional[dict[str, Any]] = None,
|
|
364
|
+
write_perf_info: bool = False,
|
|
232
365
|
) -> tuple[BaseTask, Optional[RuntimeConfig]]:
|
|
233
366
|
"""Create a dbt task given with this configuration.
|
|
234
367
|
|
|
@@ -237,23 +370,46 @@ class BaseConfig:
|
|
|
237
370
|
|
|
238
371
|
Args:
|
|
239
372
|
extra_targets: Additional targets for a dbt project's Profile.
|
|
373
|
+
write_perf_info: Flag to indicate whether to write performance info.
|
|
240
374
|
|
|
241
375
|
Returns:
|
|
242
|
-
A tuple with the
|
|
376
|
+
A tuple with the corresponding subclass of a dbt BaseTask and the
|
|
243
377
|
RuntimeConfig for the task.
|
|
244
378
|
"""
|
|
245
|
-
|
|
246
|
-
|
|
379
|
+
if not DBT_INSTALLED_LESS_THAN_1_5:
|
|
380
|
+
local_flags = self.to_flags()
|
|
381
|
+
flags.set_flags(local_flags)
|
|
382
|
+
|
|
383
|
+
project, profile = self.create_dbt_project_and_profile(extra_targets)
|
|
384
|
+
runtime_config = self.create_runtime_config(project, profile)
|
|
385
|
+
|
|
386
|
+
if not DBT_INSTALLED_LESS_THAN_1_5:
|
|
387
|
+
if issubclass(self.dbt_task, ConfiguredTask) and runtime_config:
|
|
388
|
+
register_adapter(runtime_config)
|
|
389
|
+
manifest = ManifestLoader.get_full_manifest(
|
|
390
|
+
runtime_config, write_perf_info=write_perf_info
|
|
391
|
+
)
|
|
392
|
+
task: BaseTask = self.dbt_task(
|
|
393
|
+
args=self, config=runtime_config, manifest=manifest
|
|
394
|
+
)
|
|
395
|
+
elif issubclass(self.dbt_task, DepsTask):
|
|
396
|
+
task = self.dbt_task(args=self, project=project)
|
|
397
|
+
elif issubclass(self.dbt_task, DebugTask):
|
|
398
|
+
task = self.dbt_task(args=self, config=runtime_config)
|
|
399
|
+
else:
|
|
400
|
+
task = self.dbt_task(args=self, config=runtime_config, project=project)
|
|
401
|
+
else:
|
|
402
|
+
task = self.dbt_task(args=self, config=runtime_config)
|
|
247
403
|
|
|
248
404
|
if self.compiled_target is not None:
|
|
249
405
|
# Only supported by subclasses of dbt's GraphRunnableTask.
|
|
250
406
|
# Represented here by the presence of the compiled_target attribute.
|
|
251
407
|
self.patch_manifest_task(task) # type: ignore
|
|
252
408
|
|
|
253
|
-
return
|
|
409
|
+
return task, runtime_config
|
|
254
410
|
|
|
255
411
|
def create_runtime_config(
|
|
256
|
-
self,
|
|
412
|
+
self, project: Project, profile: Profile
|
|
257
413
|
) -> Optional[RuntimeConfig]:
|
|
258
414
|
"""Crete a dbt RuntimeConfig, if possible.
|
|
259
415
|
|
|
@@ -262,19 +418,21 @@ class BaseConfig:
|
|
|
262
418
|
Flags are also initialized as configuration is created.
|
|
263
419
|
|
|
264
420
|
Args:
|
|
265
|
-
|
|
421
|
+
project: dbt project.
|
|
422
|
+
profile: dbt profile.
|
|
266
423
|
|
|
267
424
|
Returns:
|
|
268
425
|
A RuntimeConfig instance.
|
|
269
426
|
"""
|
|
270
|
-
|
|
271
|
-
initialize_from_flags()
|
|
427
|
+
initialize_from_flags(self.send_anonymous_usage_stats, self.profiles_dir)
|
|
272
428
|
self.cls.set_log_format()
|
|
273
|
-
flags.set_from_args(self, profile.user_config)
|
|
429
|
+
flags.set_from_args(self, profile.user_config) # type: ignore
|
|
274
430
|
|
|
275
431
|
cfg_type = self.dbt_task.ConfigType
|
|
276
432
|
|
|
277
433
|
if issubclass(cfg_type, RuntimeConfig):
|
|
434
|
+
if not DBT_INSTALLED_LESS_THAN_1_5 and isinstance(self.vars, str):
|
|
435
|
+
self.vars = yaml.safe_load(self.vars)
|
|
278
436
|
return cfg_type.from_parts(project=project, profile=profile, args=self)
|
|
279
437
|
return None
|
|
280
438
|
|
|
@@ -295,11 +453,11 @@ class BaseConfig:
|
|
|
295
453
|
project = Project.from_project_root(
|
|
296
454
|
self.project_dir or ".",
|
|
297
455
|
project_renderer,
|
|
298
|
-
verify_version=bool(
|
|
456
|
+
verify_version=bool(getattr(self, "version_check", True)),
|
|
299
457
|
)
|
|
300
458
|
project.project_env_vars = project_renderer.ctx_obj.env_vars
|
|
301
459
|
|
|
302
|
-
return
|
|
460
|
+
return project, profile
|
|
303
461
|
|
|
304
462
|
@property
|
|
305
463
|
def profile_name(self) -> str:
|
|
@@ -336,8 +494,8 @@ class BaseConfig:
|
|
|
336
494
|
A PartialProject instance for this configuration.
|
|
337
495
|
"""
|
|
338
496
|
project_root = self.project_dir if self.project_dir else os.getcwd()
|
|
339
|
-
version_check = bool(
|
|
340
|
-
partial_project =
|
|
497
|
+
version_check = bool(getattr(self, "version_check", True))
|
|
498
|
+
partial_project = PartialProject.from_project_root(
|
|
341
499
|
project_root, verify_version=version_check
|
|
342
500
|
)
|
|
343
501
|
|
|
@@ -390,7 +548,9 @@ class SelectionConfig(BaseConfig):
|
|
|
390
548
|
exclude: Optional[list[str]] = None
|
|
391
549
|
select: Optional[list[str]] = None
|
|
392
550
|
selector_name: Optional[list[str]] = None
|
|
551
|
+
selector: Optional[str] = None
|
|
393
552
|
state: Optional[Union[Path, str]] = None
|
|
553
|
+
defer_state: Optional[Union[Path, str]] = None
|
|
394
554
|
|
|
395
555
|
# Kept for compatibility with dbt versions < 0.21
|
|
396
556
|
models: Optional[list[str]] = None
|
|
@@ -400,6 +560,8 @@ class SelectionConfig(BaseConfig):
|
|
|
400
560
|
super().__post_init__()
|
|
401
561
|
if isinstance(self.state, str):
|
|
402
562
|
self.state = Path(self.state)
|
|
563
|
+
if isinstance(self.defer_state, str):
|
|
564
|
+
self.defer_state = Path(self.defer_state)
|
|
403
565
|
|
|
404
566
|
|
|
405
567
|
@dataclasses.dataclass
|
|
@@ -419,7 +581,6 @@ class BuildTaskConfig(TableMutabilityConfig):
|
|
|
419
581
|
|
|
420
582
|
cls: Type[BaseTask] = dataclasses.field(default=BuildTask, init=False)
|
|
421
583
|
singular: Optional[bool] = None
|
|
422
|
-
indirect_selection: Optional[str] = None
|
|
423
584
|
resource_types: Optional[list[str]] = None
|
|
424
585
|
generic: Optional[bool] = None
|
|
425
586
|
show: Optional[bool] = None
|
|
@@ -466,6 +627,7 @@ class DebugTaskConfig(BaseConfig):
|
|
|
466
627
|
|
|
467
628
|
cls: Type[BaseTask] = dataclasses.field(default=DebugTask, init=False)
|
|
468
629
|
config_dir: Optional[bool] = None
|
|
630
|
+
connection: Optional[bool] = None
|
|
469
631
|
which: str = dataclasses.field(default="debug", init=False)
|
|
470
632
|
|
|
471
633
|
|
|
@@ -483,6 +645,7 @@ class GenerateTaskConfig(SelectionConfig):
|
|
|
483
645
|
|
|
484
646
|
cls: Type[BaseTask] = dataclasses.field(default=GenerateTask, init=False)
|
|
485
647
|
compile: bool = True
|
|
648
|
+
empty_catalog: bool = False
|
|
486
649
|
which: str = dataclasses.field(default="generate", init=False)
|
|
487
650
|
|
|
488
651
|
|
|
@@ -491,7 +654,7 @@ class ListTaskConfig(SelectionConfig):
|
|
|
491
654
|
"""Dbt list task arguments."""
|
|
492
655
|
|
|
493
656
|
cls: Type[BaseTask] = dataclasses.field(default=ListTask, init=False)
|
|
494
|
-
indirect_selection:
|
|
657
|
+
indirect_selection: str = "eager"
|
|
495
658
|
output: Output = Output.SELECTOR
|
|
496
659
|
output_keys: Optional[list[str]] = None
|
|
497
660
|
resource_types: Optional[list[str]] = None
|
|
@@ -499,8 +662,11 @@ class ListTaskConfig(SelectionConfig):
|
|
|
499
662
|
|
|
500
663
|
|
|
501
664
|
@dataclasses.dataclass
|
|
502
|
-
class ParseTaskConfig(
|
|
503
|
-
"""Dbt parse task arguments.
|
|
665
|
+
class ParseTaskConfig(SelectionConfig):
|
|
666
|
+
"""Dbt parse task arguments.
|
|
667
|
+
|
|
668
|
+
TODO: Temporarily, handle this as a CompileTask.
|
|
669
|
+
"""
|
|
504
670
|
|
|
505
671
|
cls: Type[BaseTask] = dataclasses.field(default=ParseTask, init=False)
|
|
506
672
|
compile: Optional[bool] = None
|
|
@@ -520,7 +686,7 @@ class RunTaskConfig(TableMutabilityConfig):
|
|
|
520
686
|
class RunOperationTaskConfig(BaseConfig):
|
|
521
687
|
"""Dbt run-operation task arguments."""
|
|
522
688
|
|
|
523
|
-
args: str =
|
|
689
|
+
args: dict[str, Any] | str = dataclasses.field(default_factory=dict)
|
|
524
690
|
cls: Type[BaseTask] = dataclasses.field(default=RunOperationTask, init=False)
|
|
525
691
|
macro: Optional[str] = None
|
|
526
692
|
which: str = dataclasses.field(default="run-operation", init=False)
|
|
@@ -528,7 +694,8 @@ class RunOperationTaskConfig(BaseConfig):
|
|
|
528
694
|
def __post_init__(self):
|
|
529
695
|
"""Support dictionary args by casting them to str after setting."""
|
|
530
696
|
super().__post_init__()
|
|
531
|
-
|
|
697
|
+
if DBT_INSTALLED_LESS_THAN_1_5:
|
|
698
|
+
self.args = str(self.args)
|
|
532
699
|
|
|
533
700
|
|
|
534
701
|
@dataclasses.dataclass
|
|
@@ -563,7 +730,7 @@ class TestTaskConfig(SelectionConfig):
|
|
|
563
730
|
|
|
564
731
|
cls: Type[BaseTask] = dataclasses.field(default=TestTask, init=False)
|
|
565
732
|
generic: Optional[bool] = None
|
|
566
|
-
indirect_selection:
|
|
733
|
+
indirect_selection: str = "eager"
|
|
567
734
|
singular: Optional[bool] = None
|
|
568
735
|
store_failures: Optional[bool] = None
|
|
569
736
|
which: str = dataclasses.field(default="test", init=False)
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""Provides utilities to differentiate between installed dbt versions.
|
|
2
|
+
|
|
3
|
+
These are only used to ensure backwards compatibility with older versions of dbt.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from dbt.semver import Matchers, VersionSpecifier
|
|
7
|
+
from dbt.version import installed
|
|
8
|
+
|
|
9
|
+
DBT_1_5 = VersionSpecifier(
|
|
10
|
+
major="1", minor="5", patch="0", matcher=Matchers.GREATER_THAN_OR_EQUAL
|
|
11
|
+
)
|
|
12
|
+
DBT_1_6 = VersionSpecifier(
|
|
13
|
+
major="1", minor="6", patch="0", matcher=Matchers.GREATER_THAN_OR_EQUAL
|
|
14
|
+
)
|
|
15
|
+
DBT_1_7 = VersionSpecifier(
|
|
16
|
+
major="1", minor="7", patch="0", matcher=Matchers.GREATER_THAN_OR_EQUAL
|
|
17
|
+
)
|
|
18
|
+
DBT_1_8 = VersionSpecifier(
|
|
19
|
+
major="1", minor="8", patch="0", matcher=Matchers.GREATER_THAN_OR_EQUAL
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
DBT_INSTALLED_LESS_THAN_1_5 = installed < DBT_1_5
|
|
23
|
+
|
|
24
|
+
DBT_INSTALLED_GTE_1_5 = installed.compare(DBT_1_5) == 1
|
|
25
|
+
DBT_INSTALLED_GTE_1_6 = installed.compare(DBT_1_6) == 1
|
|
26
|
+
DBT_INSTALLED_GTE_1_7 = installed.compare(DBT_1_7) == 1
|
|
27
|
+
|
|
28
|
+
DBT_INSTALLED_1_5 = DBT_1_5 < installed < DBT_1_6
|
|
29
|
+
DBT_INSTALLED_1_6 = DBT_1_6 < installed < DBT_1_7
|
|
30
|
+
DBT_INSTALLED_1_7 = DBT_1_7 < installed < DBT_1_8
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[tool.poetry]
|
|
2
2
|
name = "airflow-dbt-python"
|
|
3
|
-
version = "
|
|
3
|
+
version = "2.0.0"
|
|
4
4
|
description = "A collection of Airflow operators, hooks, and utilities to execute dbt commands"
|
|
5
5
|
authors = ["Tomás Farías Santana <tomas@tomasfarias.dev>"]
|
|
6
6
|
license = "MIT"
|
|
@@ -13,7 +13,6 @@ classifiers = [
|
|
|
13
13
|
|
|
14
14
|
"License :: OSI Approved :: MIT License",
|
|
15
15
|
|
|
16
|
-
"Programming Language :: Python :: 3.7",
|
|
17
16
|
"Programming Language :: Python :: 3.8",
|
|
18
17
|
"Programming Language :: Python :: 3.9",
|
|
19
18
|
"Programming Language :: Python :: 3.10",
|
|
@@ -21,19 +20,39 @@ classifiers = [
|
|
|
21
20
|
]
|
|
22
21
|
|
|
23
22
|
[tool.poetry.dependencies]
|
|
24
|
-
python = ">=3.
|
|
23
|
+
python = ">=3.8,<3.12" # Airflow providers require this upper boundary
|
|
25
24
|
|
|
26
|
-
apache-airflow = { version = ">=2.
|
|
25
|
+
apache-airflow = { version = ">=2.2" }
|
|
27
26
|
apache-airflow-providers-amazon = { version = ">=3.0.0", optional = true }
|
|
28
27
|
apache-airflow-providers-ssh = { version = ">=3.0.0", optional = true }
|
|
28
|
+
connexion = { version = "<3.0" } # https://github.com/apache/airflow/issues/35217
|
|
29
29
|
dulwich = { version = ">=0.21", optional = true }
|
|
30
|
+
contextlib-chdir = { version = "==1.0.2", python = "<3.11" }
|
|
30
31
|
|
|
31
|
-
dbt-core =
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
dbt-
|
|
36
|
-
|
|
32
|
+
dbt-core = [
|
|
33
|
+
{ python = "<3.8", version = ">=1.4.0,<1.6.0", optional = true },
|
|
34
|
+
{ python = ">=3.11", version = ">=1.4.0,<1.8.0", optional = true },
|
|
35
|
+
]
|
|
36
|
+
dbt-bigquery = [
|
|
37
|
+
{ python = "<3.8", version = ">=1.4.0,<1.6.0", optional = true },
|
|
38
|
+
{ python = ">=3.11", version = ">=1.4.0,<1.8.0", optional = true },
|
|
39
|
+
]
|
|
40
|
+
dbt-postgres = [
|
|
41
|
+
{ python = "<3.8", version = ">=1.4.0,<1.6.0", optional = true },
|
|
42
|
+
{ python = ">=3.11", version = ">=1.4.0,<1.8.0", optional = true },
|
|
43
|
+
]
|
|
44
|
+
dbt-redshift = [
|
|
45
|
+
{ python = "<3.8", version = ">=1.4.0,<1.6.0", optional = true },
|
|
46
|
+
{ python = ">=3.11", version = ">=1.4.0,<1.8.0", optional = true },
|
|
47
|
+
]
|
|
48
|
+
dbt-snowflake = [
|
|
49
|
+
{ python = "<3.8", version = ">=1.4.0,<1.6.0", optional = true },
|
|
50
|
+
{ python = ">=3.11", version = ">=1.4.0,<1.8.0", optional = true },
|
|
51
|
+
]
|
|
52
|
+
dbt-spark = [
|
|
53
|
+
{ python = "<3.8", version = ">=1.4.0,<1.6.0", optional = true },
|
|
54
|
+
{ python = ">=3.11", version = ">=1.4.0,<1.8.0", optional = true },
|
|
55
|
+
]
|
|
37
56
|
|
|
38
57
|
[tool.poetry.group.dev]
|
|
39
58
|
optional = true
|
|
@@ -44,8 +63,8 @@ apache-airflow-providers-ssh = ">=3.0.0"
|
|
|
44
63
|
black = ">=22"
|
|
45
64
|
boto3-stubs = { extras = ["s3"], version = ">=1.26.8" }
|
|
46
65
|
coverage = { extras = ["toml"], version = ">=7.2" }
|
|
47
|
-
dbt-postgres = ">=1.0.0"
|
|
48
|
-
dulwich = ">=0.21"
|
|
66
|
+
dbt-postgres = ">=1.0.0,<1.8.0"
|
|
67
|
+
dulwich = ">=0.21,!=0.21.6"
|
|
49
68
|
moto = ">=4.0.3"
|
|
50
69
|
mypy = ">=1"
|
|
51
70
|
freezegun = ">=1.1.0"
|
|
@@ -61,7 +80,11 @@ types-PyYAML = ">=6.0.7"
|
|
|
61
80
|
optional = true
|
|
62
81
|
|
|
63
82
|
[tool.poetry.group.docs.dependencies]
|
|
64
|
-
Sphinx =
|
|
83
|
+
Sphinx = [
|
|
84
|
+
{ python = "<3.8", version = "<6.0" },
|
|
85
|
+
{ python = ">=3.8,<3.9", version = "<7.2" },
|
|
86
|
+
{ python = ">=3.9", version = ">=7.2" },
|
|
87
|
+
]
|
|
65
88
|
sphinx-rtd-theme = ">=1.1"
|
|
66
89
|
sphinx-copybutton = ">=0.5"
|
|
67
90
|
attrs = ">=22"
|
|
@@ -70,14 +93,28 @@ attrs = ">=22"
|
|
|
70
93
|
optional = true
|
|
71
94
|
|
|
72
95
|
[tool.poetry.group.adapters.dependencies]
|
|
73
|
-
dbt-bigquery =
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
dbt-
|
|
96
|
+
dbt-bigquery = [
|
|
97
|
+
{ python = "<3.8", version = ">=1.4.0,<1.6.0", optional = true },
|
|
98
|
+
{ python = ">=3.11", version = ">=1.4.0,<1.8.0", optional = true },
|
|
99
|
+
]
|
|
100
|
+
dbt-postgres = [
|
|
101
|
+
{ python = "<3.8", version = ">=1.4.0,<1.6.0", optional = true },
|
|
102
|
+
{ python = ">=3.11", version = ">=1.4.0,<1.8.0", optional = true },
|
|
103
|
+
]
|
|
104
|
+
dbt-redshift = [
|
|
105
|
+
{ python = "<3.8", version = ">=1.4.0,<1.6.0", optional = true },
|
|
106
|
+
{ python = ">=3.11", version = ">=1.4.0,<1.8.0", optional = true },
|
|
107
|
+
]
|
|
108
|
+
dbt-snowflake = [
|
|
109
|
+
{ python = "<3.8", version = ">=1.4.0,<1.6.0", optional = true },
|
|
110
|
+
{ python = ">=3.11", version = ">=1.4.0,<1.8.0", optional = true },
|
|
111
|
+
]
|
|
112
|
+
dbt-spark = [
|
|
113
|
+
{ python = "<3.8", version = ">=1.4.0,<1.6.0", optional = true },
|
|
114
|
+
{ python = ">=3.11", version = ">=1.4.0,<1.8.0", optional = true },
|
|
115
|
+
]
|
|
78
116
|
|
|
79
117
|
[tool.poetry.extras]
|
|
80
|
-
airflow = ["apache-airflow"]
|
|
81
118
|
airflow-providers = ["apache-airflow-providers-amazon", "apache-airflow-providers-ssh"]
|
|
82
119
|
adapters = ["dbt-bigquery", "dbt-redshift", "dbt-postgres", "dbt-snowflake", "dbt-spark"]
|
|
83
120
|
bigquery = ["dbt-bigquery"]
|
|
@@ -11,30 +11,60 @@ package_data = \
|
|
|
11
11
|
{'': ['*']}
|
|
12
12
|
|
|
13
13
|
install_requires = \
|
|
14
|
-
['
|
|
14
|
+
['apache-airflow>=2.2', 'connexion<3.0']
|
|
15
15
|
|
|
16
16
|
extras_require = \
|
|
17
|
-
{'
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
17
|
+
{':python_version < "3.11"': ['contextlib-chdir==1.0.2'],
|
|
18
|
+
'adapters:python_version < "3.8"': ['dbt-bigquery>=1.4.0,<1.6.0',
|
|
19
|
+
'dbt-bigquery>=1.4.0,<1.6.0',
|
|
20
|
+
'dbt-postgres>=1.4.0,<1.6.0',
|
|
21
|
+
'dbt-postgres>=1.4.0,<1.6.0',
|
|
22
|
+
'dbt-redshift>=1.4.0,<1.6.0',
|
|
23
|
+
'dbt-redshift>=1.4.0,<1.6.0',
|
|
24
|
+
'dbt-snowflake>=1.4.0,<1.6.0',
|
|
25
|
+
'dbt-snowflake>=1.4.0,<1.6.0',
|
|
26
|
+
'dbt-spark>=1.4.0,<1.6.0',
|
|
27
|
+
'dbt-spark>=1.4.0,<1.6.0'],
|
|
28
|
+
'adapters:python_version >= "3.11"': ['dbt-bigquery>=1.4.0,<1.8.0',
|
|
29
|
+
'dbt-bigquery>=1.4.0,<1.8.0',
|
|
30
|
+
'dbt-postgres>=1.4.0,<1.8.0',
|
|
31
|
+
'dbt-postgres>=1.4.0,<1.8.0',
|
|
32
|
+
'dbt-redshift>=1.4.0,<1.8.0',
|
|
33
|
+
'dbt-redshift>=1.4.0,<1.8.0',
|
|
34
|
+
'dbt-snowflake>=1.4.0,<1.8.0',
|
|
35
|
+
'dbt-snowflake>=1.4.0,<1.8.0',
|
|
36
|
+
'dbt-spark>=1.4.0,<1.8.0',
|
|
37
|
+
'dbt-spark>=1.4.0,<1.8.0'],
|
|
23
38
|
'airflow-providers': ['apache-airflow-providers-amazon>=3.0.0',
|
|
24
39
|
'apache-airflow-providers-ssh>=3.0.0'],
|
|
25
|
-
'bigquery': ['dbt-bigquery>=1.0.0'
|
|
40
|
+
'bigquery:python_version < "3.8"': ['dbt-bigquery>=1.4.0,<1.6.0',
|
|
41
|
+
'dbt-bigquery>=1.4.0,<1.6.0'],
|
|
42
|
+
'bigquery:python_version >= "3.11"': ['dbt-bigquery>=1.4.0,<1.8.0',
|
|
43
|
+
'dbt-bigquery>=1.4.0,<1.8.0'],
|
|
26
44
|
'git': ['apache-airflow-providers-ssh>=3.0.0', 'dulwich>=0.21'],
|
|
27
|
-
'postgres': ['dbt-postgres>=1.0.0'
|
|
28
|
-
|
|
45
|
+
'postgres:python_version < "3.8"': ['dbt-postgres>=1.4.0,<1.6.0',
|
|
46
|
+
'dbt-postgres>=1.4.0,<1.6.0'],
|
|
47
|
+
'postgres:python_version >= "3.11"': ['dbt-postgres>=1.4.0,<1.8.0',
|
|
48
|
+
'dbt-postgres>=1.4.0,<1.8.0'],
|
|
49
|
+
'redshift:python_version < "3.8"': ['dbt-redshift>=1.4.0,<1.6.0',
|
|
50
|
+
'dbt-redshift>=1.4.0,<1.6.0'],
|
|
51
|
+
'redshift:python_version >= "3.11"': ['dbt-redshift>=1.4.0,<1.8.0',
|
|
52
|
+
'dbt-redshift>=1.4.0,<1.8.0'],
|
|
29
53
|
's3': ['apache-airflow-providers-amazon>=3.0.0'],
|
|
30
|
-
'snowflake': ['dbt-snowflake>=1.0.0'
|
|
31
|
-
|
|
54
|
+
'snowflake:python_version < "3.8"': ['dbt-snowflake>=1.4.0,<1.6.0',
|
|
55
|
+
'dbt-snowflake>=1.4.0,<1.6.0'],
|
|
56
|
+
'snowflake:python_version >= "3.11"': ['dbt-snowflake>=1.4.0,<1.8.0',
|
|
57
|
+
'dbt-snowflake>=1.4.0,<1.8.0'],
|
|
58
|
+
'spark:python_version < "3.8"': ['dbt-spark>=1.4.0,<1.6.0',
|
|
59
|
+
'dbt-spark>=1.4.0,<1.6.0'],
|
|
60
|
+
'spark:python_version >= "3.11"': ['dbt-spark>=1.4.0,<1.8.0',
|
|
61
|
+
'dbt-spark>=1.4.0,<1.8.0']}
|
|
32
62
|
|
|
33
63
|
setup_kwargs = {
|
|
34
64
|
'name': 'airflow-dbt-python',
|
|
35
|
-
'version': '
|
|
65
|
+
'version': '2.0.0',
|
|
36
66
|
'description': 'A collection of Airflow operators, hooks, and utilities to execute dbt commands',
|
|
37
|
-
'long_description': '# airflow-dbt-python\n\n[](https://pypi.org/project/airflow-dbt-python/)\n[](https://github.com/tomasfarias/airflow-dbt-python/actions)\n[](https://github.com/psf/black)\n[](https://github.com/tomasfarias/airflow-dbt-python/actions)\n[](https://airflow-dbt-python.readthedocs.io/en/latest/?badge=latest)\n\nA collection of [Airflow](https://airflow.apache.org/) operators, hooks, and utilities to execute [`dbt`](https://pypi.org/project/dbt-core/) commands.\n\nRead the [documentation](https://airflow-dbt-python.readthedocs.io) for examples, installation instructions, and more details.\n\n# Installation\n\n## Requirements\n\nBefore using *airflow-dbt-python*, ensure you meet the following requirements:\n* A *dbt* project using [dbt-core](https://pypi.org/project/dbt-core/) version 1.0.0 or later.\n* An Airflow environment using version 2.2 or later.\n\n * If using any managed service, like AWS MWAA, ensure your environment is created with a supported version of Airflow.\n * If self-hosting, Airflow installation instructions can be found in their [official documentation](https://airflow.apache.org/docs/apache-airflow/stable/installation/index.html).\n\n* Running Python 3.7 or later in your Airflow environment.\n\n> **Warning**\n>\n> Even though we don\'t impose any upper limits on versions of Airflow and *dbt*, it\'s possible that new versions are not supported immediately after release, particularly for *dbt*. We recommend testing the latest versions before upgrading and [reporting any issues](https://github.com/tomasfarias/airflow-dbt-python/issues/new/choose).\n\n> **Note**\n>\n> Older versions of Airflow and *dbt* may work with *airflow-dbt-python*, although we cannot guarantee this. Our testing pipeline runs the latest *dbt-core* with the latest Airflow release, and the latest version supported by [AWS MWAA](https://aws.amazon.com/managed-workflows-for-apache-airflow/).\n\n## From PyPI\n\n*airflow-dbt-python* is available in [PyPI](https://pypi.org/project/airflow-dbt-python/) and can be installed with *pip*:\n\n``` shell\npip install airflow-dbt-python\n```\n\nAs a convenience, some *dbt* adapters can be installed by specifying extras. For example, if requiring the *dbt-redshift* adapter:\n\n``` shell\npip install airflow-dbt-python[redshift]\n```\n\n## From this repo\n\n*airflow-dbt-python* can also be built from source by cloning this GitHub repository:\n\n``` shell\ngit clone https://github.com/tomasfarias/airflow-dbt-python.git\ncd airflow-dbt-python\n```\n\nAnd installing with *Poetry*:\n\n``` shell\npoetry install\n```\n\n## In AWS MWAA\n\nAdd *airflow-dbt-python* to your `requirements.txt` file and edit your Airflow environment to use this new `requirements.txt` file, or upload it as a plugin.\n\nRead the [documentation](https://airflow-dbt-python.readthedocs.io/en/latest/getting_started.html#installing-in-mwaa) for more a more detailed AWS MWAA installation breakdown.\n\n## In other managed services\n\n*airflow-dbt-python* should be compatible with most or all Airflow managed services. Consult the documentation specific to your provider.\n\nIf you notice an issue when installing *airflow-dbt-python* in a specific managed service, please open an [issue](https://github.com/tomasfarias/airflow-dbt-python/issues/new/choose).\n\n# Features\n\n*airflow-dbt-python* aims to make dbt a **first-class citizen** of Airflow by supporting additional features that integrate both tools. As you would expect, *airflow-dbt-python* can run all your dbt workflows in Airflow with the same interface you are used to from the CLI, but without being a mere wrapper: *airflow-dbt-python* directly communicates with internal *dbt-core* classes, bridging the gap between them and Airflow\'s operator interface. Essentially, we are attempting to use *dbt* **as a library**.\n\nAs this integration was completed, several features were developed to **extend the capabilities of dbt** to leverage Airflow as much as possible. Can you think of a way *dbt* could leverage Airflow that is not currently supported? Let us know in a [GitHub issue](https://github.com/tomasfarias/airflow-dbt-python/issues/new/choose)!\n\n## Independent task execution\n\nAirflow executes [Tasks](https://airflow.apache.org/docs/apache-airflow/stable/concepts/tasks.html) independent of one another: even though downstream and upstream dependencies between tasks exist, the execution of an individual task happens entirely independently of any other task execution (see: [Tasks Relationships](https://airflow.apache.org/docs/apache-airflow/stable/concepts/tasks.html#relationships)).\n\nIn order to work with this constraint, *airflow-dbt-python* runs each dbt command in a **temporary and isolated directory**. Before execution, all the relevant dbt files are copied from supported backends, and after executing the command any artifacts are exported. This ensures dbt can work with any Airflow deployment, including most production deployments as they are usually running [Remote Executors](https://airflow.apache.org/docs/apache-airflow/stable/executor/index.html#executor-types) and do not guarantee any files will be shared by default between tasks, since each task may run in a completely different environment.\n\n\n## Download dbt files from a remote storage\n\nThe dbt parameters `profiles_dir` and `project_dir` would normally point to a directory containing a `profiles.yml` file and a dbt project in the local environment respectively (defined by the presence of a *dbt_project.yml* file). *airflow-dbt-python* extends these parameters to also accept an URL pointing to a remote storage.\n\nCurrently, we support the following remote storages:\n\n* [AWS S3](https://aws.amazon.com/s3/) (identified by a *s3* scheme).\n* Remote git repositories, like those stored in GitHub (both *https* and *ssh* schemes are supported).\n\n* If a remote URL is used for `project_dir`, then this URL must point to a location in your remote storage containing a *dbt* project to run. A *dbt* project is identified by the prescence of a *dbt_project.yml*, and contains all your [resources](https://docs.getdbt.com/docs/build/projects). All of the contents of this remote location will be downloaded and made available for the operator. The URL may also point to an archived file containing all the files of a dbt project, which will be downloaded, uncompressed, and made available for the operator.\n* If a remote URL is used for `profiles_dir`, then this URL must point to a location in your remote storage that contains a *profiles.yml* file. The *profiles.yml* file will be downloaded and made available for the operator to use when running. The *profiles.yml* may be part of your *dbt* project, in which case this argument may be ommitted.\n\nThis feature is intended to work in line with Airflow\'s [description of the task concept](https://airflow.apache.org/docs/apache-airflow/stable/concepts/tasks.html#relationships):\n\n> Tasks don’t pass information to each other by default, and run entirely independently.\n\nWe interpret this as meaning a task should be responsible of fetching all the *dbt* related files it needs in order to run independently, as already described in [Independent Task Execution](#independent-task-execution).\n\n## Push dbt artifacts to XCom\n\nEach dbt execution produces one or more [JSON artifacts](https://docs.getdbt.com/reference/artifacts/dbt-artifacts/) that are valuable to produce meta-metrics, build conditional workflows, for reporting purposes, and other uses. *airflow-dbt-python* can push these artifacts to [XCom](https://airflow.apache.org/docs/apache-airflow/stable/concepts/xcoms.html) as requested via the `do_xcom_push_artifacts` parameter, which takes a list of artifacts to push.\n\n## Use Airflow connections as dbt targets (without a profiles.yml)\n\n[Airflow connections](https://airflow.apache.org/docs/apache-airflow/stable/howto/connection.html) allow users to manage and store connection information, such as hostname, port, username, and password, for operators to use when accessing certain applications, like databases. Similarly, a *dbt* `profiles.yml` file stores connection information under each target key. *airflow-dbt-python* bridges the gap between the two and allows you to use connection information stored as an Airflow connection by specifying the connection id as the `target` parameter of any of the *dbt* operators it provides. What\'s more, if using an Airflow connection, the `profiles.yml` file may be entirely omitted (although keep in mind a `profiles.yml` file contains a configuration block besides target connection information).\n\nSee an example DAG [here](examples/airflow_connection_target_dag.py).\n\n# Motivation\n\n## Airflow running in a managed environment\n\nAlthough [`dbt`](https://docs.getdbt.com/) is meant to be installed and used as a CLI, we may not have control of the environment where Airflow is running, disallowing us the option of using *dbt* as a CLI.\n\nThis is exactly what happens when using [Amazon\'s Managed Workflows for Apache Airflow](https://aws.amazon.com/managed-workflows-for-apache-airflow/) or MWAA: although a list of Python requirements can be passed, the CLI cannot be found in the worker\'s PATH.\n\nThere is a workaround which involves using Airflow\'s `BashOperator` and running Python from the command line:\n\n``` python\nfrom airflow.operators.bash import BashOperator\n\nBASH_COMMAND = "python -c \'from dbt.main import main; main()\' run"\noperator = BashOperator(\n task_id="dbt_run",\n bash_command=BASH_COMMAND,\n)\n```\n\nBut it can get cumbersome when appending all potential arguments a `dbt run` command (or other subcommand) can take.\n\nThat\'s where *airflow-dbt-python* comes in: it abstracts the complexity of interfacing with *dbt-core* and exposes one operator for each *dbt* subcommand that can be instantiated with all the corresponding arguments that the *dbt* CLI would take.\n\n## An alternative to *airflow-dbt* that works without the *dbt* CLI\n\nThe alternative [`airflow-dbt`](https://pypi.org/project/airflow-dbt/) package, by default, would not work if the *dbt* CLI is not in PATH, which means it would not be usable in MWAA. There is a workaround via the `dbt_bin` argument, which can be set to `"python -c \'from dbt.main import main; main()\' run"`, in similar fashion as the `BashOperator` example. Yet this approach is not without its limitations:\n* *airflow-dbt* works by wrapping the *dbt* CLI, which makes our code dependent on the environment in which it runs.\n* *airflow-dbt* does not support the full range of arguments a command can take. For example, `DbtRunOperator` does not have an attribute for `fail_fast`.\n* *airflow-dbt* does not offer access to *dbt* artifacts created during execution. *airflow-dbt-python* does so by pushing any artifacts to [XCom](https://airflow.apache.org/docs/apache-airflow/stable/concepts/xcoms.html).\n\n# Usage\n\nCurrently, the following *dbt* commands are supported:\n\n* `clean`\n* `compile`\n* `debug`\n* `deps`\n* `docs generate`\n* `ls`\n* `parse`\n* `run`\n* `run-operation`\n* `seed`\n* `snapshot`\n* `source`\n* `test`\n\n## Examples\n\nAll example DAGs are tested against the latest Airflow version. Some changes, like modifying `import` statements or changing types, may be required for them to work in other versions.\n\n``` python\nimport datetime as dt\n\nimport pendulum\nfrom airflow import DAG\n\nfrom airflow_dbt_python.operators.dbt import (\n DbtRunOperator,\n DbtSeedOperator,\n DbtTestOperator,\n)\n\nargs = {\n "owner": "airflow",\n}\n\nwith DAG(\n dag_id="example_dbt_operator",\n default_args=args,\n schedule="0 0 * * *",\n start_date=pendulum.today("UTC").add(days=-1),\n dagrun_timeout=dt.timedelta(minutes=60),\n tags=["example", "example2"],\n) as dag:\n dbt_test = DbtTestOperator(\n task_id="dbt_test",\n selector_name="pre-run-tests",\n )\n\n dbt_seed = DbtSeedOperator(\n task_id="dbt_seed",\n select=["/path/to/first.csv", "/path/to/second.csv"],\n full_refresh=True,\n )\n\n dbt_run = DbtRunOperator(\n task_id="dbt_run",\n select=["/path/to/models"],\n full_refresh=True,\n fail_fast=True,\n )\n\n dbt_test >> dbt_seed >> dbt_run\n```\n\nMore examples can be found in the [`examples/`](examples/) directory and the [documentation](https://airflow-dbt-python.readthedocs.io).\n\n# Development\n\nSee the [development documentation](https://airflow-dbt-python.readthedocs.io/en/latest/development.html) for a more in-depth dive into setting up a development environment, running the test-suite, and general commentary on working on *airflow-dbt-python*.\n\n## Testing\n\nTests are run with *pytest*, can be located in `tests/`. To run them locally, you may use *Poetry*:\n\n``` shell\npoetry run pytest tests/ -vv\n```\n\n# License\n\nThis project is licensed under the MIT license. See [LICENSE](LICENSE).\n',
|
|
67
|
+
'long_description': '# airflow-dbt-python\n\n[](https://pypi.org/project/airflow-dbt-python/)\n[](https://github.com/tomasfarias/airflow-dbt-python/actions)\n[](https://github.com/psf/black)\n[](https://github.com/tomasfarias/airflow-dbt-python/actions)\n[](https://airflow-dbt-python.readthedocs.io/en/latest/?badge=latest)\n\nA collection of [Airflow](https://airflow.apache.org/) operators, hooks, and utilities to execute [`dbt`](https://pypi.org/project/dbt-core/) commands.\n\nRead the [documentation](https://airflow-dbt-python.readthedocs.io) for examples, installation instructions, and more details.\n\n# Installation\n\n## Requirements\n\nBefore using *airflow-dbt-python*, ensure you meet the following requirements:\n* A *dbt* project using [dbt-core](https://pypi.org/project/dbt-core/) version 1.4.0 or later.\n* An Airflow environment using version 2.2 or later.\n\n * If using any managed service, like AWS MWAA, ensure your environment is created with a supported version of Airflow.\n * If self-hosting, Airflow installation instructions can be found in their [official documentation](https://airflow.apache.org/docs/apache-airflow/stable/installation/index.html).\n\n* Running Python 3.8 or later in your Airflow environment.\n\n> **Warning**\n>\n> Even though we don\'t impose any upper limits on versions of Airflow and *dbt*, it\'s possible that new versions are not supported immediately after release, particularly for *dbt*. We recommend testing the latest versions before upgrading and [reporting any issues](https://github.com/tomasfarias/airflow-dbt-python/issues/new/choose).\n\n> **Note**\n>\n> Older versions of Airflow and *dbt* may work with *airflow-dbt-python*, although we cannot guarantee this. Our testing pipeline runs the latest *dbt-core* with the latest Airflow release, and the latest version supported by [AWS MWAA](https://aws.amazon.com/managed-workflows-for-apache-airflow/).\n\n## From PyPI\n\n*airflow-dbt-python* is available in [PyPI](https://pypi.org/project/airflow-dbt-python/) and can be installed with *pip*:\n\n``` shell\npip install airflow-dbt-python\n```\n\nAs a convenience, some *dbt* adapters can be installed by specifying extras. For example, if requiring the *dbt-redshift* adapter:\n\n``` shell\npip install airflow-dbt-python[redshift]\n```\n\n## From this repo\n\n*airflow-dbt-python* can also be built from source by cloning this GitHub repository:\n\n``` shell\ngit clone https://github.com/tomasfarias/airflow-dbt-python.git\ncd airflow-dbt-python\n```\n\nAnd installing with *Poetry*:\n\n``` shell\npoetry install\n```\n\n## In AWS MWAA\n\nAdd *airflow-dbt-python* to your `requirements.txt` file and edit your Airflow environment to use this new `requirements.txt` file, or upload it as a plugin.\n\nRead the [documentation](https://airflow-dbt-python.readthedocs.io/en/latest/getting_started.html#installing-in-mwaa) for more a more detailed AWS MWAA installation breakdown.\n\n## In other managed services\n\n*airflow-dbt-python* should be compatible with most or all Airflow managed services. Consult the documentation specific to your provider.\n\nIf you notice an issue when installing *airflow-dbt-python* in a specific managed service, please open an [issue](https://github.com/tomasfarias/airflow-dbt-python/issues/new/choose).\n\n# Features\n\n*airflow-dbt-python* aims to make dbt a **first-class citizen** of Airflow by supporting additional features that integrate both tools. As you would expect, *airflow-dbt-python* can run all your dbt workflows in Airflow with the same interface you are used to from the CLI, but without being a mere wrapper: *airflow-dbt-python* directly communicates with internal *dbt-core* classes, bridging the gap between them and Airflow\'s operator interface. Essentially, we are attempting to use *dbt* **as a library**.\n\nAs this integration was completed, several features were developed to **extend the capabilities of dbt** to leverage Airflow as much as possible. Can you think of a way *dbt* could leverage Airflow that is not currently supported? Let us know in a [GitHub issue](https://github.com/tomasfarias/airflow-dbt-python/issues/new/choose)!\n\n## Independent task execution\n\nAirflow executes [Tasks](https://airflow.apache.org/docs/apache-airflow/stable/concepts/tasks.html) independent of one another: even though downstream and upstream dependencies between tasks exist, the execution of an individual task happens entirely independently of any other task execution (see: [Tasks Relationships](https://airflow.apache.org/docs/apache-airflow/stable/concepts/tasks.html#relationships)).\n\nIn order to work with this constraint, *airflow-dbt-python* runs each dbt command in a **temporary and isolated directory**. Before execution, all the relevant dbt files are copied from supported backends, and after executing the command any artifacts are exported. This ensures dbt can work with any Airflow deployment, including most production deployments as they are usually running [Remote Executors](https://airflow.apache.org/docs/apache-airflow/stable/executor/index.html#executor-types) and do not guarantee any files will be shared by default between tasks, since each task may run in a completely different environment.\n\n\n## Download dbt files from a remote storage\n\nThe dbt parameters `profiles_dir` and `project_dir` would normally point to a directory containing a `profiles.yml` file and a dbt project in the local environment respectively (defined by the presence of a *dbt_project.yml* file). *airflow-dbt-python* extends these parameters to also accept an URL pointing to a remote storage.\n\nCurrently, we support the following remote storages:\n\n* [AWS S3](https://aws.amazon.com/s3/) (identified by a *s3* scheme).\n* Remote git repositories, like those stored in GitHub (both *https* and *ssh* schemes are supported).\n\n* If a remote URL is used for `project_dir`, then this URL must point to a location in your remote storage containing a *dbt* project to run. A *dbt* project is identified by the prescence of a *dbt_project.yml*, and contains all your [resources](https://docs.getdbt.com/docs/build/projects). All of the contents of this remote location will be downloaded and made available for the operator. The URL may also point to an archived file containing all the files of a dbt project, which will be downloaded, uncompressed, and made available for the operator.\n* If a remote URL is used for `profiles_dir`, then this URL must point to a location in your remote storage that contains a *profiles.yml* file. The *profiles.yml* file will be downloaded and made available for the operator to use when running. The *profiles.yml* may be part of your *dbt* project, in which case this argument may be ommitted.\n\nThis feature is intended to work in line with Airflow\'s [description of the task concept](https://airflow.apache.org/docs/apache-airflow/stable/concepts/tasks.html#relationships):\n\n> Tasks don’t pass information to each other by default, and run entirely independently.\n\nWe interpret this as meaning a task should be responsible of fetching all the *dbt* related files it needs in order to run independently, as already described in [Independent Task Execution](#independent-task-execution).\n\n## Push dbt artifacts to XCom\n\nEach dbt execution produces one or more [JSON artifacts](https://docs.getdbt.com/reference/artifacts/dbt-artifacts/) that are valuable to produce meta-metrics, build conditional workflows, for reporting purposes, and other uses. *airflow-dbt-python* can push these artifacts to [XCom](https://airflow.apache.org/docs/apache-airflow/stable/concepts/xcoms.html) as requested via the `do_xcom_push_artifacts` parameter, which takes a list of artifacts to push.\n\n## Use Airflow connections as dbt targets (without a profiles.yml)\n\n[Airflow connections](https://airflow.apache.org/docs/apache-airflow/stable/howto/connection.html) allow users to manage and store connection information, such as hostname, port, username, and password, for operators to use when accessing certain applications, like databases. Similarly, a *dbt* `profiles.yml` file stores connection information under each target key. *airflow-dbt-python* bridges the gap between the two and allows you to use connection information stored as an Airflow connection by specifying the connection id as the `target` parameter of any of the *dbt* operators it provides. What\'s more, if using an Airflow connection, the `profiles.yml` file may be entirely omitted (although keep in mind a `profiles.yml` file contains a configuration block besides target connection information).\n\nSee an example DAG [here](examples/airflow_connection_target_dag.py).\n\n# Motivation\n\n## Airflow running in a managed environment\n\nAlthough [`dbt`](https://docs.getdbt.com/) is meant to be installed and used as a CLI, we may not have control of the environment where Airflow is running, disallowing us the option of using *dbt* as a CLI.\n\nThis is exactly what happens when using [Amazon\'s Managed Workflows for Apache Airflow](https://aws.amazon.com/managed-workflows-for-apache-airflow/) or MWAA: although a list of Python requirements can be passed, the CLI cannot be found in the worker\'s PATH.\n\nThere is a workaround which involves using Airflow\'s `BashOperator` and running Python from the command line:\n\n``` python\nfrom airflow.operators.bash import BashOperator\n\nBASH_COMMAND = "python -c \'from dbt.main import main; main()\' run"\noperator = BashOperator(\n task_id="dbt_run",\n bash_command=BASH_COMMAND,\n)\n```\n\nBut it can get cumbersome when appending all potential arguments a `dbt run` command (or other subcommand) can take.\n\nThat\'s where *airflow-dbt-python* comes in: it abstracts the complexity of interfacing with *dbt-core* and exposes one operator for each *dbt* subcommand that can be instantiated with all the corresponding arguments that the *dbt* CLI would take.\n\n## An alternative to *airflow-dbt* that works without the *dbt* CLI\n\nThe alternative [`airflow-dbt`](https://pypi.org/project/airflow-dbt/) package, by default, would not work if the *dbt* CLI is not in PATH, which means it would not be usable in MWAA. There is a workaround via the `dbt_bin` argument, which can be set to `"python -c \'from dbt.main import main; main()\' run"`, in similar fashion as the `BashOperator` example. Yet this approach is not without its limitations:\n* *airflow-dbt* works by wrapping the *dbt* CLI, which makes our code dependent on the environment in which it runs.\n* *airflow-dbt* does not support the full range of arguments a command can take. For example, `DbtRunOperator` does not have an attribute for `fail_fast`.\n* *airflow-dbt* does not offer access to *dbt* artifacts created during execution. *airflow-dbt-python* does so by pushing any artifacts to [XCom](https://airflow.apache.org/docs/apache-airflow/stable/concepts/xcoms.html).\n\n# Usage\n\nCurrently, the following *dbt* commands are supported:\n\n* `clean`\n* `compile`\n* `debug`\n* `deps`\n* `docs generate`\n* `ls`\n* `parse`\n* `run`\n* `run-operation`\n* `seed`\n* `snapshot`\n* `source`\n* `test`\n\n## Examples\n\nAll example DAGs are tested against the latest Airflow version. Some changes, like modifying `import` statements or changing types, may be required for them to work in other versions.\n\n``` python\nimport datetime as dt\n\nimport pendulum\nfrom airflow import DAG\n\nfrom airflow_dbt_python.operators.dbt import (\n DbtRunOperator,\n DbtSeedOperator,\n DbtTestOperator,\n)\n\nargs = {\n "owner": "airflow",\n}\n\nwith DAG(\n dag_id="example_dbt_operator",\n default_args=args,\n schedule="0 0 * * *",\n start_date=pendulum.today("UTC").add(days=-1),\n dagrun_timeout=dt.timedelta(minutes=60),\n tags=["example", "example2"],\n) as dag:\n dbt_test = DbtTestOperator(\n task_id="dbt_test",\n selector_name="pre-run-tests",\n )\n\n dbt_seed = DbtSeedOperator(\n task_id="dbt_seed",\n select=["/path/to/first.csv", "/path/to/second.csv"],\n full_refresh=True,\n )\n\n dbt_run = DbtRunOperator(\n task_id="dbt_run",\n select=["/path/to/models"],\n full_refresh=True,\n fail_fast=True,\n )\n\n dbt_test >> dbt_seed >> dbt_run\n```\n\nMore examples can be found in the [`examples/`](examples/) directory and the [documentation](https://airflow-dbt-python.readthedocs.io).\n\n# Development\n\nSee the [development documentation](https://airflow-dbt-python.readthedocs.io/en/latest/development.html) for a more in-depth dive into setting up a development environment, running the test-suite, and general commentary on working on *airflow-dbt-python*.\n\n## Testing\n\nTests are run with *pytest*, can be located in `tests/`. To run them locally, you may use *Poetry*:\n\n``` shell\npoetry run pytest tests/ -vv\n```\n\n# License\n\nThis project is licensed under the MIT license. See [LICENSE](LICENSE).\n',
|
|
38
68
|
'author': 'Tomás Farías Santana',
|
|
39
69
|
'author_email': 'tomas@tomasfarias.dev',
|
|
40
70
|
'maintainer': 'None',
|
|
@@ -44,7 +74,7 @@ setup_kwargs = {
|
|
|
44
74
|
'package_data': package_data,
|
|
45
75
|
'install_requires': install_requires,
|
|
46
76
|
'extras_require': extras_require,
|
|
47
|
-
'python_requires': '>=3.
|
|
77
|
+
'python_requires': '>=3.8,<3.12',
|
|
48
78
|
}
|
|
49
79
|
|
|
50
80
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{airflow_dbt_python-1.0.5 → airflow_dbt_python-2.0.0}/airflow_dbt_python/operators/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|