airflow-dbt-python 1.0.5__tar.gz → 2.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (21) hide show
  1. {airflow_dbt_python-1.0.5 → airflow_dbt_python-2.0.0}/PKG-INFO +19 -13
  2. {airflow_dbt_python-1.0.5 → airflow_dbt_python-2.0.0}/README.md +2 -2
  3. {airflow_dbt_python-1.0.5 → airflow_dbt_python-2.0.0}/airflow_dbt_python/__version__.py +1 -1
  4. {airflow_dbt_python-1.0.5 → airflow_dbt_python-2.0.0}/airflow_dbt_python/hooks/dbt.py +78 -42
  5. {airflow_dbt_python-1.0.5 → airflow_dbt_python-2.0.0}/airflow_dbt_python/hooks/git.py +1 -1
  6. {airflow_dbt_python-1.0.5 → airflow_dbt_python-2.0.0}/airflow_dbt_python/operators/dbt.py +23 -2
  7. {airflow_dbt_python-1.0.5 → airflow_dbt_python-2.0.0}/airflow_dbt_python/utils/configs.py +213 -46
  8. airflow_dbt_python-2.0.0/airflow_dbt_python/utils/version.py +30 -0
  9. {airflow_dbt_python-1.0.5 → airflow_dbt_python-2.0.0}/pyproject.toml +56 -19
  10. {airflow_dbt_python-1.0.5 → airflow_dbt_python-2.0.0}/setup.py +45 -15
  11. {airflow_dbt_python-1.0.5 → airflow_dbt_python-2.0.0}/LICENSE +0 -0
  12. {airflow_dbt_python-1.0.5 → airflow_dbt_python-2.0.0}/airflow_dbt_python/__init__.py +0 -0
  13. {airflow_dbt_python-1.0.5 → airflow_dbt_python-2.0.0}/airflow_dbt_python/hooks/__init__.py +0 -0
  14. {airflow_dbt_python-1.0.5 → airflow_dbt_python-2.0.0}/airflow_dbt_python/hooks/localfs.py +0 -0
  15. {airflow_dbt_python-1.0.5 → airflow_dbt_python-2.0.0}/airflow_dbt_python/hooks/remote.py +0 -0
  16. {airflow_dbt_python-1.0.5 → airflow_dbt_python-2.0.0}/airflow_dbt_python/hooks/s3.py +0 -0
  17. {airflow_dbt_python-1.0.5 → airflow_dbt_python-2.0.0}/airflow_dbt_python/operators/__init__.py +0 -0
  18. {airflow_dbt_python-1.0.5 → airflow_dbt_python-2.0.0}/airflow_dbt_python/utils/__init__.py +0 -0
  19. {airflow_dbt_python-1.0.5 → airflow_dbt_python-2.0.0}/airflow_dbt_python/utils/enums.py +0 -0
  20. {airflow_dbt_python-1.0.5 → airflow_dbt_python-2.0.0}/airflow_dbt_python/utils/env.py +0 -0
  21. {airflow_dbt_python-1.0.5 → airflow_dbt_python-2.0.0}/airflow_dbt_python/utils/url.py +0 -0
@@ -1,12 +1,12 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: airflow-dbt-python
3
- Version: 1.0.5
3
+ Version: 2.0.0
4
4
  Summary: A collection of Airflow operators, hooks, and utilities to execute dbt commands
5
5
  Home-page: https://github.com/tomasfarias/airflow-dbt-python
6
6
  License: MIT
7
7
  Author: Tomás Farías Santana
8
8
  Author-email: tomas@tomasfarias.dev
9
- Requires-Python: >=3.7.2,<4.0
9
+ Requires-Python: >=3.8,<3.12
10
10
  Classifier: Development Status :: 5 - Production/Stable
11
11
  Classifier: Intended Audience :: Developers
12
12
  Classifier: License :: OSI Approved :: MIT License
@@ -17,11 +17,9 @@ Classifier: Programming Language :: Python :: 3.10
17
17
  Classifier: Programming Language :: Python :: 3.11
18
18
  Classifier: Programming Language :: Python :: 3.10
19
19
  Classifier: Programming Language :: Python :: 3.11
20
- Classifier: Programming Language :: Python :: 3.7
21
20
  Classifier: Programming Language :: Python :: 3.8
22
21
  Classifier: Programming Language :: Python :: 3.9
23
22
  Provides-Extra: adapters
24
- Provides-Extra: airflow
25
23
  Provides-Extra: airflow-providers
26
24
  Provides-Extra: bigquery
27
25
  Provides-Extra: docs
@@ -31,15 +29,23 @@ Provides-Extra: redshift
31
29
  Provides-Extra: s3
32
30
  Provides-Extra: snowflake
33
31
  Provides-Extra: spark
34
- Requires-Dist: apache-airflow (>=2.0) ; extra == "airflow"
32
+ Requires-Dist: apache-airflow (>=2.2)
35
33
  Requires-Dist: apache-airflow-providers-amazon (>=3.0.0) ; extra == "airflow-providers" or extra == "s3"
36
34
  Requires-Dist: apache-airflow-providers-ssh (>=3.0.0) ; extra == "airflow-providers" or extra == "git"
37
- Requires-Dist: dbt-bigquery (>=1.0.0) ; extra == "adapters" or extra == "bigquery"
38
- Requires-Dist: dbt-core (>=1.0.0)
39
- Requires-Dist: dbt-postgres (>=1.0.0) ; extra == "adapters" or extra == "postgres"
40
- Requires-Dist: dbt-redshift (>=1.0.0) ; extra == "adapters" or extra == "redshift"
41
- Requires-Dist: dbt-snowflake (>=1.0.0) ; extra == "adapters" or extra == "snowflake"
42
- Requires-Dist: dbt-spark (>=1.0.0) ; extra == "adapters" or extra == "spark"
35
+ Requires-Dist: connexion (<3.0)
36
+ Requires-Dist: contextlib-chdir (==1.0.2) ; python_version < "3.11"
37
+ Requires-Dist: dbt-bigquery (>=1.4.0,<1.6.0) ; (python_version < "3.8") and (extra == "adapters" or extra == "bigquery")
38
+ Requires-Dist: dbt-bigquery (>=1.4.0,<1.8.0) ; (python_version >= "3.11") and (extra == "adapters" or extra == "bigquery")
39
+ Requires-Dist: dbt-core (>=1.4.0,<1.6.0) ; python_version < "3.8"
40
+ Requires-Dist: dbt-core (>=1.4.0,<1.8.0) ; python_version >= "3.11"
41
+ Requires-Dist: dbt-postgres (>=1.4.0,<1.6.0) ; (python_version < "3.8") and (extra == "adapters" or extra == "postgres")
42
+ Requires-Dist: dbt-postgres (>=1.4.0,<1.8.0) ; (python_version >= "3.11") and (extra == "adapters" or extra == "postgres")
43
+ Requires-Dist: dbt-redshift (>=1.4.0,<1.6.0) ; (python_version < "3.8") and (extra == "adapters" or extra == "redshift")
44
+ Requires-Dist: dbt-redshift (>=1.4.0,<1.8.0) ; (python_version >= "3.11") and (extra == "adapters" or extra == "redshift")
45
+ Requires-Dist: dbt-snowflake (>=1.4.0,<1.6.0) ; (python_version < "3.8") and (extra == "adapters" or extra == "snowflake")
46
+ Requires-Dist: dbt-snowflake (>=1.4.0,<1.8.0) ; (python_version >= "3.11") and (extra == "adapters" or extra == "snowflake")
47
+ Requires-Dist: dbt-spark (>=1.4.0,<1.6.0) ; (python_version < "3.8") and (extra == "adapters" or extra == "spark")
48
+ Requires-Dist: dbt-spark (>=1.4.0,<1.8.0) ; (python_version >= "3.11") and (extra == "adapters" or extra == "spark")
43
49
  Requires-Dist: dulwich (>=0.21) ; extra == "git"
44
50
  Project-URL: Repository, https://github.com/tomasfarias/airflow-dbt-python
45
51
  Description-Content-Type: text/markdown
@@ -61,13 +67,13 @@ Read the [documentation](https://airflow-dbt-python.readthedocs.io) for examples
61
67
  ## Requirements
62
68
 
63
69
  Before using *airflow-dbt-python*, ensure you meet the following requirements:
64
- * A *dbt* project using [dbt-core](https://pypi.org/project/dbt-core/) version 1.0.0 or later.
70
+ * A *dbt* project using [dbt-core](https://pypi.org/project/dbt-core/) version 1.4.0 or later.
65
71
  * An Airflow environment using version 2.2 or later.
66
72
 
67
73
  * If using any managed service, like AWS MWAA, ensure your environment is created with a supported version of Airflow.
68
74
  * If self-hosting, Airflow installation instructions can be found in their [official documentation](https://airflow.apache.org/docs/apache-airflow/stable/installation/index.html).
69
75
 
70
- * Running Python 3.7 or later in your Airflow environment.
76
+ * Running Python 3.8 or later in your Airflow environment.
71
77
 
72
78
  > **Warning**
73
79
  >
@@ -15,13 +15,13 @@ Read the [documentation](https://airflow-dbt-python.readthedocs.io) for examples
15
15
  ## Requirements
16
16
 
17
17
  Before using *airflow-dbt-python*, ensure you meet the following requirements:
18
- * A *dbt* project using [dbt-core](https://pypi.org/project/dbt-core/) version 1.0.0 or later.
18
+ * A *dbt* project using [dbt-core](https://pypi.org/project/dbt-core/) version 1.4.0 or later.
19
19
  * An Airflow environment using version 2.2 or later.
20
20
 
21
21
  * If using any managed service, like AWS MWAA, ensure your environment is created with a supported version of Airflow.
22
22
  * If self-hosting, Airflow installation instructions can be found in their [official documentation](https://airflow.apache.org/docs/apache-airflow/stable/installation/index.html).
23
23
 
24
- * Running Python 3.7 or later in your Airflow environment.
24
+ * Running Python 3.8 or later in your Airflow environment.
25
25
 
26
26
  > **Warning**
27
27
  >
@@ -2,4 +2,4 @@
2
2
  __author__ = "Tomás Farías Santana"
3
3
  __copyright__ = "Copyright 2021 Tomás Farías Santana"
4
4
  __title__ = "airflow-dbt-python"
5
- __version__ = "1.0.5"
5
+ __version__ = "2.0.0"
@@ -24,6 +24,14 @@ from airflow.exceptions import AirflowException
24
24
  from airflow.hooks.base import BaseHook
25
25
  from airflow.models.connection import Connection
26
26
 
27
+ from airflow_dbt_python.utils.version import DBT_INSTALLED_LESS_THAN_1_5
28
+
29
+ if sys.version_info >= (3, 11):
30
+ from contextlib import chdir as chdir_ctx
31
+ else:
32
+ from contextlib_chdir import chdir as chdir_ctx
33
+
34
+
27
35
  if TYPE_CHECKING:
28
36
  from dbt.contracts.results import RunResult
29
37
  from dbt.task.base import BaseTask
@@ -208,6 +216,7 @@ class DbtHook(BaseHook):
208
216
  replace_on_upload: bool = False,
209
217
  artifacts: Optional[Iterable[str]] = None,
210
218
  env_vars: Optional[Dict[str, Any]] = None,
219
+ write_perf_info: bool = False,
211
220
  **kwargs,
212
221
  ) -> DbtTaskResult:
213
222
  """Run a dbt task with a given configuration and return the results.
@@ -219,9 +228,17 @@ class DbtHook(BaseHook):
219
228
  of running the dbt command.
220
229
  """
221
230
  from dbt.adapters.factory import register_adapter
222
- from dbt.config.runtime import UnsetProfileConfig
223
- from dbt.main import adapter_management, track_run
224
- from dbt.task.base import move_to_nearest_project_dir
231
+ from dbt.task.base import get_nearest_project_dir
232
+ from dbt.task.clean import CleanTask
233
+ from dbt.task.deps import DepsTask
234
+
235
+ from airflow_dbt_python.utils.version import DBT_INSTALLED_LESS_THAN_1_5
236
+
237
+ if DBT_INSTALLED_LESS_THAN_1_5:
238
+ from dbt.main import adapter_management, track_run # type: ignore
239
+ else:
240
+ from dbt.adapters.factory import adapter_management
241
+ from dbt.tracking import track_run
225
242
 
226
243
  config = self.get_dbt_task_config(command, **kwargs)
227
244
  extra_target = self.get_dbt_target_from_connection(config.target)
@@ -233,53 +250,62 @@ class DbtHook(BaseHook):
233
250
  replace_on_upload=replace_on_upload,
234
251
  env_vars=env_vars,
235
252
  ) as dbt_dir:
236
- config.dbt_task.pre_init_hook(config)
237
- self.ensure_profiles(config.profiles_dir)
238
-
239
- task, runtime_config = config.create_dbt_task(extra_target)
240
-
241
253
  # When creating tasks via from_args, dbt switches to the project directory.
242
254
  # We have to do that here as we are not using from_args.
243
- move_to_nearest_project_dir(config)
255
+ if DBT_INSTALLED_LESS_THAN_1_5:
256
+ # For compatibility with older versions of dbt, as the signature
257
+ # of move_to_nearest_project_dir changed in dbt-core 1.5 to take
258
+ # just the project_dir.
259
+ nearest_project_dir = get_nearest_project_dir(config) # type: ignore
260
+ else:
261
+ nearest_project_dir = get_nearest_project_dir(config.project_dir)
262
+
263
+ with chdir_ctx(nearest_project_dir):
264
+ config.dbt_task.pre_init_hook(config)
265
+ self.ensure_profiles(config)
244
266
 
245
- self.setup_dbt_logging(task, config.debug)
267
+ task, runtime_config = config.create_dbt_task(
268
+ extra_target, write_perf_info
269
+ )
270
+ requires_profile = isinstance(task, (CleanTask, DepsTask))
246
271
 
247
- if not isinstance(runtime_config, UnsetProfileConfig):
248
- if runtime_config is not None:
272
+ self.setup_dbt_logging(task, config.debug)
273
+
274
+ if runtime_config is not None and not requires_profile:
249
275
  # The deps command installs the dependencies, which means they may
250
276
  # not exist before deps runs and the following would raise a
251
277
  # CompilationError.
252
278
  runtime_config.load_dependencies()
253
279
 
254
- results = None
255
- with adapter_management():
256
- if not isinstance(runtime_config, UnsetProfileConfig):
257
- if runtime_config is not None:
258
- register_adapter(runtime_config)
280
+ results = None
281
+ with adapter_management():
282
+ if not requires_profile:
283
+ if runtime_config is not None:
284
+ register_adapter(runtime_config)
259
285
 
260
- with track_run(task):
261
- results = task.run()
262
- success = task.interpret_results(results)
286
+ with track_run(task):
287
+ results = task.run()
288
+ success = task.interpret_results(results)
263
289
 
264
- if artifacts is None:
265
- return DbtTaskResult(success, results, {})
290
+ if artifacts is None:
291
+ return DbtTaskResult(success, results, {})
266
292
 
267
- saved_artifacts = {}
268
- for artifact in artifacts:
269
- artifact_path = Path(dbt_dir) / "target" / artifact
293
+ saved_artifacts = {}
294
+ for artifact in artifacts:
295
+ artifact_path = Path(dbt_dir) / "target" / artifact
270
296
 
271
- if not artifact_path.exists():
272
- self.log.warn(
273
- "Required dbt artifact %s was not found. "
274
- "Perhaps dbt failed and couldn't generate it.",
275
- artifact,
276
- )
277
- continue
297
+ if not artifact_path.exists():
298
+ self.log.warning(
299
+ "Required dbt artifact %s was not found. "
300
+ "Perhaps dbt failed and couldn't generate it.",
301
+ artifact,
302
+ )
303
+ continue
278
304
 
279
- with open(artifact_path) as artifact_file:
280
- json_artifact = json.load(artifact_file)
305
+ with open(artifact_path) as artifact_file:
306
+ json_artifact = json.load(artifact_file)
281
307
 
282
- saved_artifacts[artifact] = json_artifact
308
+ saved_artifacts[artifact] = json_artifact
283
309
 
284
310
  return DbtTaskResult(success, results, saved_artifacts)
285
311
 
@@ -373,7 +399,7 @@ class DbtHook(BaseHook):
373
399
  if (project_dir_path / "profiles.yml").exists():
374
400
  # We may have downloaded the profiles.yml file together
375
401
  # with the project.
376
- return (new_project_dir, new_project_dir)
402
+ return new_project_dir, new_project_dir
377
403
 
378
404
  if profiles_dir is not None:
379
405
  profiles_file_path = self.download_dbt_profiles(
@@ -384,7 +410,7 @@ class DbtHook(BaseHook):
384
410
  else:
385
411
  new_profiles_dir = None
386
412
 
387
- return (new_project_dir, new_profiles_dir)
413
+ return new_project_dir, new_profiles_dir
388
414
 
389
415
  def setup_dbt_logging(self, task: BaseTask, debug: Optional[bool]):
390
416
  """Setup dbt logging.
@@ -399,10 +425,18 @@ class DbtHook(BaseHook):
399
425
  if task.config is not None:
400
426
  log_path = getattr(task.config, "log_path", None)
401
427
 
402
- setup_event_logger(log_path or "logs")
428
+ if DBT_INSTALLED_LESS_THAN_1_5:
429
+ setup_event_logger(log_path or "logs")
430
+ else:
431
+ from dbt.flags import get_flags
432
+
433
+ flags = get_flags()
434
+ setup_event_logger(flags)
403
435
 
404
436
  configured_file = logging.getLogger("configured_file")
405
437
  file_log = logging.getLogger("file_log")
438
+ stdout_log = logging.getLogger("stdout_log")
439
+ stdout_log.propagate = True
406
440
 
407
441
  if not debug:
408
442
  # We have to do this after setting logs up as dbt hasn't
@@ -413,16 +447,18 @@ class DbtHook(BaseHook):
413
447
  configured_file.setLevel("INFO")
414
448
  configured_file.propagate = False
415
449
 
416
- def ensure_profiles(self, profiles_dir: Optional[str]):
450
+ def ensure_profiles(self, config: BaseConfig):
417
451
  """Ensure a profiles file exists."""
418
- if profiles_dir is not None:
419
- # We expect one to exist given that we have passsed a profiles_dir.
452
+ if config.profiles_dir is not None:
453
+ # We expect one to exist given that we have passed a profiles_dir.
420
454
  return
421
455
 
422
456
  profiles_path = Path.home() / ".dbt/profiles.yml"
457
+ config.profiles_dir = str(profiles_path.parent)
423
458
  if not profiles_path.exists():
424
459
  profiles_path.parent.mkdir(exist_ok=True)
425
- profiles_path.touch()
460
+ with profiles_path.open("w", encoding="utf-8") as f:
461
+ f.write("config:\n send_anonymous_usage_stats: false\n")
426
462
 
427
463
  def get_dbt_target_from_connection(
428
464
  self, target: Optional[str]
@@ -209,4 +209,4 @@ class DbtGitRemoteHook(SSHHook, DbtRemoteHook):
209
209
  else:
210
210
  raise ValueError(f"Unsupported scheme: {url.scheme}")
211
211
 
212
- return (client, path)
212
+ return client, path
@@ -70,6 +70,9 @@ class DbtBaseOperator(BaseOperator):
70
70
  warn_error: Optional[bool] = None,
71
71
  # Logging
72
72
  debug: Optional[bool] = None,
73
+ log_path: Optional[str] = None,
74
+ log_level: Optional[str] = None,
75
+ log_level_file: Optional[str] = None,
73
76
  log_format: Optional[LogFormat] = None,
74
77
  log_cache_events: Optional[bool] = False,
75
78
  quiet: Optional[bool] = None,
@@ -80,12 +83,16 @@ class DbtBaseOperator(BaseOperator):
80
83
  no_defer: Optional[bool] = None,
81
84
  partial_parse: Optional[bool] = False,
82
85
  no_partial_parse: Optional[bool] = None,
86
+ introspect: Optional[bool] = None,
87
+ no_introspect: Optional[bool] = None,
83
88
  use_colors: Optional[bool] = None,
84
89
  no_use_colors: Optional[bool] = None,
85
90
  static_parser: Optional[bool] = None,
86
91
  no_static_parser: Optional[bool] = None,
87
92
  version_check: Optional[bool] = None,
88
93
  no_version_check: Optional[bool] = None,
94
+ write_json: Optional[bool] = None,
95
+ write_perf_info: Optional[bool] = None,
89
96
  anonymous_usage_stats: Optional[bool] = None,
90
97
  no_anonymous_usage_stats: Optional[bool] = None,
91
98
  # Extra features configuration
@@ -116,9 +123,12 @@ class DbtBaseOperator(BaseOperator):
116
123
  self.warn_error = warn_error
117
124
 
118
125
  self.debug = debug
126
+ self.log_path = log_path
119
127
  self.log_cache_events = log_cache_events
120
128
  self.quiet = quiet
121
129
  self.no_print = no_print
130
+ self.log_level = log_level
131
+ self.log_level_file = log_level_file
122
132
  self.log_format = log_format
123
133
  self.record_timing_info = record_timing_info
124
134
 
@@ -137,9 +147,18 @@ class DbtBaseOperator(BaseOperator):
137
147
  self.use_colors = use_colors
138
148
  self.no_use_colors = no_use_colors
139
149
 
140
- self.version_check = no_version_check
150
+ self.introspect = introspect
151
+ self.no_introspect = no_introspect
152
+
153
+ self.version_check = version_check
141
154
  self.no_version_check = no_version_check
142
155
 
156
+ self.write_json = write_json or (
157
+ do_xcom_push_artifacts and "run_results.json" in do_xcom_push_artifacts
158
+ )
159
+
160
+ self.write_perf_info = write_perf_info
161
+
143
162
  self.dbt_conn_id = dbt_conn_id
144
163
  self.profiles_conn_id = profiles_conn_id
145
164
  self.project_conn_id = project_conn_id
@@ -174,7 +193,7 @@ class DbtBaseOperator(BaseOperator):
174
193
  )
175
194
  except Exception as e:
176
195
  self.log.exception(
177
- f"An error has ocurred while executing dbt {self.command}", exc_info=e
196
+ f"An error has occurred while executing dbt {self.command}", exc_info=e
178
197
  )
179
198
  raise AirflowException(
180
199
  f"An error has occurred while executing dbt {self.command}"
@@ -428,11 +447,13 @@ class DbtCleanOperator(DbtBaseOperator):
428
447
  self,
429
448
  upload_dbt_project: bool = True,
430
449
  delete_before_upload: bool = True,
450
+ clean_project_files_only: bool = True,
431
451
  **kwargs,
432
452
  ) -> None:
433
453
  super().__init__(**kwargs)
434
454
  self.upload_dbt_project = upload_dbt_project
435
455
  self.delete_before_upload = delete_before_upload
456
+ self.clean_project_files_only = clean_project_files_only
436
457
 
437
458
  @property
438
459
  def command(self) -> str:
@@ -1,15 +1,17 @@
1
1
  """A collection of configurations to interact with dbt-core."""
2
2
  from __future__ import annotations
3
3
 
4
+ import argparse
4
5
  import dataclasses
5
6
  import json
6
7
  import os
7
8
  import pickle
8
9
  from pathlib import Path
9
- from typing import TYPE_CHECKING, Any, Optional, Type, Union
10
+ from typing import Any, Optional, Type, Union
10
11
 
11
12
  import dbt.flags as flags
12
13
  import yaml
14
+ from dbt.adapters.factory import register_adapter
13
15
  from dbt.clients import yaml_helper # type: ignore
14
16
  from dbt.config.profile import Profile, read_profile
15
17
  from dbt.config.project import PartialProject, Project
@@ -17,7 +19,8 @@ from dbt.config.renderer import DbtProjectYamlRenderer, ProfileRenderer
17
19
  from dbt.config.runtime import RuntimeConfig
18
20
  from dbt.contracts.graph.manifest import Manifest
19
21
  from dbt.graph.graph import Graph
20
- from dbt.task.base import BaseTask
22
+ from dbt.helper_types import WarnErrorOptions
23
+ from dbt.task.base import BaseTask, ConfiguredTask
21
24
  from dbt.task.build import BuildTask
22
25
  from dbt.task.clean import CleanTask
23
26
  from dbt.task.compile import CompileTask
@@ -26,18 +29,44 @@ from dbt.task.deps import DepsTask
26
29
  from dbt.task.freshness import FreshnessTask
27
30
  from dbt.task.generate import GenerateTask
28
31
  from dbt.task.list import ListTask
29
- from dbt.task.parse import ParseTask
30
32
  from dbt.task.run import RunTask
31
33
  from dbt.task.run_operation import RunOperationTask
34
+ from dbt.task.runnable import GraphRunnableTask
32
35
  from dbt.task.seed import SeedTask
33
36
  from dbt.task.snapshot import SnapshotTask
34
37
  from dbt.task.test import TestTask
35
- from dbt.tracking import initialize_from_flags
38
+ from dbt.tracking import initialize_from_flags as orig_initialize_from_flags
36
39
 
37
40
  from airflow_dbt_python.utils.enums import FromStrEnum, LogFormat, Output
41
+ from airflow_dbt_python.utils.version import (
42
+ DBT_INSTALLED_GTE_1_5,
43
+ DBT_INSTALLED_GTE_1_7,
44
+ DBT_INSTALLED_LESS_THAN_1_5,
45
+ )
46
+
47
+ if DBT_INSTALLED_LESS_THAN_1_5:
48
+ # dbt 1.5 removed the ParseTask as it only generated a manifest.
49
+ # This task is now handled by a simple function.
50
+ # Airflow-dbt-python should still support a ParseTask that does the same,
51
+ # but this will require writing our own ParseTask class.
52
+ # In the meantime, we treat ParseTask as CompileTask (which also generates
53
+ # a manifest) if running with dbt-core >= 1.5.
54
+ from dbt.task.parse import ParseTask # type: ignore
55
+
56
+ # dbt 1.5 also changed the signature of initialize_from_flags, so we have
57
+ # this for backwards compatibility.
58
+ def initialize_from_flags(send_anonymous_usage_stats, profiles_dir):
59
+ """Call initialize_from_flags without arguments.
60
+
61
+ Added for backwards compatibility with versions of dbt < 1.5.
62
+ """
63
+ return orig_initialize_from_flags() # type: ignore
64
+
65
+ else:
66
+ from dbt.parser.manifest import ManifestLoader
38
67
 
39
- if TYPE_CHECKING:
40
- from dbt.task.runnable import GraphRunnableTask
68
+ ParseTask = CompileTask
69
+ initialize_from_flags = orig_initialize_from_flags
41
70
 
42
71
 
43
72
  def parse_yaml_args(args: Optional[Union[str, dict[str, Any]]]) -> dict[str, Any]:
@@ -78,16 +107,24 @@ class BaseConfig:
78
107
  single_threaded: Optional[bool] = None
79
108
  threads: Optional[int] = None
80
109
  use_experimental_parser: Optional[bool] = None
110
+ store_failures: Optional[bool] = None
81
111
  vars: Optional[Union[str, dict[str, Any]]] = None
112
+ macro_debugging: Optional[bool] = None
82
113
  warn_error: Optional[bool] = None
114
+ warn_error_options: dict[str, Any] = dataclasses.field(default_factory=dict)
115
+ indirect_selection: str = "eager"
83
116
 
84
117
  # Logging
85
118
  log_format: Optional[LogFormat] = None
86
- log_cache_events: Optional[bool] = None
119
+ log_format_file: Optional[LogFormat] = None
120
+ log_path: Optional[str] = None if DBT_INSTALLED_LESS_THAN_1_5 else "logs"
121
+ log_level: str = "info"
122
+ log_level_file: str = "none"
87
123
  record_timing_info: Optional[str] = None
88
124
  debug: Optional[bool] = None
89
125
  quiet: Optional[bool] = None
90
126
  no_print: Optional[bool] = None
127
+ printer_width: Optional[int] = None
91
128
 
92
129
  # Mutually exclusive attributes
93
130
  defer: Optional[bool] = None
@@ -96,23 +133,62 @@ class BaseConfig:
96
133
  partial_parse: Optional[bool] = None
97
134
  no_partial_parse: Optional[bool] = dataclasses.field(default=None, repr=False)
98
135
 
136
+ log_cache_events: Optional[bool] = None
137
+ no_log_cache_events: Optional[bool] = dataclasses.field(default=None, repr=False)
138
+
99
139
  use_colors: Optional[bool] = None
100
140
  no_use_colors: Optional[bool] = dataclasses.field(default=None, repr=False)
101
141
 
142
+ use_colors_file: Optional[bool] = None
143
+ no_use_colors_file: Optional[bool] = dataclasses.field(default=None, repr=False)
144
+
145
+ introspect: Optional[bool] = None
146
+ no_introspect: Optional[bool] = dataclasses.field(default=None, repr=False)
147
+
148
+ populate_cache: Optional[bool] = None
149
+ no_populate_cache: Optional[bool] = dataclasses.field(default=None, repr=False)
150
+
102
151
  static_parser: Optional[bool] = None
103
152
  no_static_parser: Optional[bool] = dataclasses.field(default=None, repr=False)
104
153
 
105
154
  version_check: Optional[bool] = None
106
155
  no_version_check: Optional[bool] = dataclasses.field(default=None, repr=False)
107
156
 
108
- send_anonymous_usage_stats: Optional[bool] = None
109
- anonymous_usage_stats: Optional[bool] = dataclasses.field(default=None, repr=False)
157
+ # dbt < 1.5
158
+ anonymous_usage_stats: Optional[bool] = None
110
159
  no_anonymous_usage_stats: Optional[bool] = dataclasses.field(
111
160
  default=None, repr=False
112
161
  )
113
162
 
114
- write_json: Optional[bool] = None
115
- no_write_json: Optional[bool] = dataclasses.field(default=None, repr=False)
163
+ # dbt >= 1.5
164
+ if DBT_INSTALLED_GTE_1_5:
165
+ send_anonymous_usage_stats: Optional[bool] = None
166
+ no_send_anonymous_usage_stats: Optional[bool] = dataclasses.field(
167
+ default=None, repr=False
168
+ )
169
+
170
+ write_json: Optional[bool] = None
171
+ no_write_json: Optional[bool] = dataclasses.field(default=None, repr=False)
172
+
173
+ # dbt >= 1.7
174
+ if DBT_INSTALLED_GTE_1_7:
175
+ from dbt.cli.option_types import Package
176
+
177
+ include_saved_query: Optional[bool] = None
178
+ no_include_saved_query: Optional[bool] = dataclasses.field(
179
+ default=None, repr=False
180
+ )
181
+
182
+ clean_project_files_only: Optional[bool] = None
183
+ no_clean_project_files_only: Optional[bool] = dataclasses.field(
184
+ default=None, repr=False
185
+ )
186
+
187
+ add_package: Optional[Package] = None
188
+ dry_run: bool = False
189
+ lock: bool = False
190
+ static: bool = False
191
+ upgrade: bool = False
116
192
 
117
193
  def __post_init__(self):
118
194
  """Post initialization actions for a dbt configuration."""
@@ -121,6 +197,7 @@ class BaseConfig:
121
197
  self.set_mutually_exclusive_attributes()
122
198
 
123
199
  self.send_anonymous_usage_stats = self.anonymous_usage_stats
200
+ self.no_send_anonymous_usage_stats = self.no_anonymous_usage_stats
124
201
 
125
202
  def set_mutually_exclusive_attributes(self):
126
203
  """Support pairs of mutually exclusive parameters.
@@ -133,28 +210,82 @@ class BaseConfig:
133
210
  to non-None values.
134
211
  """
135
212
  mutually_exclusive_attrs = (
136
- "defer",
137
- "partial_parse",
138
- "use_colors",
139
- "static_parser",
140
- "version_check",
141
- "anonymous_usage_stats",
142
- "write_json",
213
+ ("defer", False),
214
+ ("partial_parse", True),
215
+ ("log_cache_events", False),
216
+ ("use_colors", True),
217
+ ("use_colors_file", True),
218
+ ("introspect", True),
219
+ ("populate_cache", True),
220
+ ("static_parser", True),
221
+ ("version_check", True),
222
+ ("anonymous_usage_stats", True),
223
+ ("send_anonymous_usage_stats", True),
224
+ ("write_json", True),
225
+ ("include_saved_query", None),
226
+ ("clean_project_files_only", True),
143
227
  )
144
228
 
145
- for attr in mutually_exclusive_attrs:
146
- positive_value = getattr(self, attr, None)
147
- negative_value = getattr(self, f"no_{attr}", None)
229
+ for attrs, default_value in mutually_exclusive_attrs:
230
+ attr, negative_attr = attrs, f"no_{attrs}"
148
231
 
149
- if positive_value is None and negative_value is None:
150
- continue
232
+ positive_value = getattr(self, attr, None)
233
+ negative_value = getattr(self, negative_attr, None)
234
+
235
+ if (
236
+ positive_value is None
237
+ and negative_value is None
238
+ and default_value is not None
239
+ ):
240
+ setattr(self, attr, default_value)
241
+ setattr(self, negative_attr, not default_value)
151
242
  elif positive_value is not None and negative_value is not None:
152
- raise ValueError(f"{attr} and no_{attr} are mutually exclusive")
243
+ raise ValueError(f"{attr} and {negative_attr} are mutually exclusive")
153
244
  elif positive_value is not None:
154
- setattr(self, f"no_{attr}", not positive_value)
245
+ setattr(self, negative_attr, not positive_value)
155
246
  else:
156
247
  setattr(self, attr, not negative_value)
157
248
 
249
+ def __getattribute__(self, item: str):
250
+ """Dbt 1.5+ uses uppercase attributes, let's handle this."""
251
+ if item.isupper():
252
+ item = item.lower()
253
+ return super().__getattribute__(item)
254
+
255
+ def to_flags(self) -> argparse.Namespace:
256
+ """Return a Namespace to use as dbt flags."""
257
+ flags_dict = vars(flags.get_flags())
258
+ flags_dict["MACRO_DEBUGGING"] = self.macro_debugging
259
+ flags_dict["WARN_ERROR"] = self.warn_error
260
+ flags_dict["WARN_ERROR_OPTIONS"] = WarnErrorOptions(
261
+ include=self.warn_error_options.get("include", []),
262
+ exclude=self.warn_error_options.get("exclude", []),
263
+ )
264
+
265
+ flags_dict["PRINTER_WIDTH"] = self.printer_width
266
+ flags_dict["TARGET_PATH"] = None
267
+ flags_dict["LOG_PATH"] = self.log_path
268
+ flags_dict["LOG_FORMAT"] = self.log_format
269
+ flags_dict["LOG_FORMAT_FILE"] = self.log_format
270
+ flags_dict["SEND_ANONYMOUS_USAGE_STATS"] = self.send_anonymous_usage_stats
271
+ flags_dict["PARTIAL_PARSE"] = self.partial_parse
272
+ flags_dict["STATIC_PARSER"] = self.static_parser
273
+ flags_dict["PROFILES_DIR"] = self.profiles_dir
274
+ flags_dict["STORE_FAILURES"] = self.store_failures
275
+ flags_dict["LOG_LEVEL"] = self.log_level
276
+ flags_dict["LOG_LEVEL_FILE"] = self.log_level_file
277
+ flags_dict["LOG_CACHE_EVENTS"] = self.log_cache_events
278
+ flags_dict["QUIET"] = self.quiet
279
+ flags_dict["DEBUG"] = self.debug
280
+ flags_dict["USE_COLORS_FILE"] = self.use_colors_file
281
+ flags_dict["WRITE_JSON"] = self.write_json
282
+ flags_dict["INDIRECT_SELECTION"] = getattr(self, "indirect_selection", None)
283
+ flags_dict["FAIL_FAST"] = self.fail_fast
284
+ flags_dict["INTROSPECT"] = self.introspect
285
+
286
+ flags_ns = argparse.Namespace(**flags_dict)
287
+ return flags_ns
288
+
158
289
  @property
159
290
  def dbt_task(self) -> Type[BaseTask]:
160
291
  """Access to the underlying dbt task class."""
@@ -228,7 +359,9 @@ class BaseConfig:
228
359
  task._runtime_initialize = _runtime_initialize # type: ignore
229
360
 
230
361
  def create_dbt_task(
231
- self, extra_targets: Optional[dict[str, Any]] = None
362
+ self,
363
+ extra_targets: Optional[dict[str, Any]] = None,
364
+ write_perf_info: bool = False,
232
365
  ) -> tuple[BaseTask, Optional[RuntimeConfig]]:
233
366
  """Create a dbt task given with this configuration.
234
367
 
@@ -237,23 +370,46 @@ class BaseConfig:
237
370
 
238
371
  Args:
239
372
  extra_targets: Additional targets for a dbt project's Profile.
373
+ write_perf_info: Flag to indicate whether to write performance info.
240
374
 
241
375
  Returns:
242
- A tuple with the corresnponding subclass of a dbt BaseTask and the
376
+ A tuple with the corresponding subclass of a dbt BaseTask and the
243
377
  RuntimeConfig for the task.
244
378
  """
245
- runtime_config = self.create_runtime_config(extra_targets)
246
- task = self.dbt_task(args=self, config=runtime_config)
379
+ if not DBT_INSTALLED_LESS_THAN_1_5:
380
+ local_flags = self.to_flags()
381
+ flags.set_flags(local_flags)
382
+
383
+ project, profile = self.create_dbt_project_and_profile(extra_targets)
384
+ runtime_config = self.create_runtime_config(project, profile)
385
+
386
+ if not DBT_INSTALLED_LESS_THAN_1_5:
387
+ if issubclass(self.dbt_task, ConfiguredTask) and runtime_config:
388
+ register_adapter(runtime_config)
389
+ manifest = ManifestLoader.get_full_manifest(
390
+ runtime_config, write_perf_info=write_perf_info
391
+ )
392
+ task: BaseTask = self.dbt_task(
393
+ args=self, config=runtime_config, manifest=manifest
394
+ )
395
+ elif issubclass(self.dbt_task, DepsTask):
396
+ task = self.dbt_task(args=self, project=project)
397
+ elif issubclass(self.dbt_task, DebugTask):
398
+ task = self.dbt_task(args=self, config=runtime_config)
399
+ else:
400
+ task = self.dbt_task(args=self, config=runtime_config, project=project)
401
+ else:
402
+ task = self.dbt_task(args=self, config=runtime_config)
247
403
 
248
404
  if self.compiled_target is not None:
249
405
  # Only supported by subclasses of dbt's GraphRunnableTask.
250
406
  # Represented here by the presence of the compiled_target attribute.
251
407
  self.patch_manifest_task(task) # type: ignore
252
408
 
253
- return (task, runtime_config)
409
+ return task, runtime_config
254
410
 
255
411
  def create_runtime_config(
256
- self, extra_targets: Optional[dict[str, Any]]
412
+ self, project: Project, profile: Profile
257
413
  ) -> Optional[RuntimeConfig]:
258
414
  """Crete a dbt RuntimeConfig, if possible.
259
415
 
@@ -262,19 +418,21 @@ class BaseConfig:
262
418
  Flags are also initialized as configuration is created.
263
419
 
264
420
  Args:
265
- extra_targets: Additional targets for a dbt project's Profile.
421
+ project: dbt project.
422
+ profile: dbt profile.
266
423
 
267
424
  Returns:
268
425
  A RuntimeConfig instance.
269
426
  """
270
- project, profile = self.create_dbt_project_and_profile(extra_targets)
271
- initialize_from_flags()
427
+ initialize_from_flags(self.send_anonymous_usage_stats, self.profiles_dir)
272
428
  self.cls.set_log_format()
273
- flags.set_from_args(self, profile.user_config)
429
+ flags.set_from_args(self, profile.user_config) # type: ignore
274
430
 
275
431
  cfg_type = self.dbt_task.ConfigType
276
432
 
277
433
  if issubclass(cfg_type, RuntimeConfig):
434
+ if not DBT_INSTALLED_LESS_THAN_1_5 and isinstance(self.vars, str):
435
+ self.vars = yaml.safe_load(self.vars)
278
436
  return cfg_type.from_parts(project=project, profile=profile, args=self)
279
437
  return None
280
438
 
@@ -295,11 +453,11 @@ class BaseConfig:
295
453
  project = Project.from_project_root(
296
454
  self.project_dir or ".",
297
455
  project_renderer,
298
- verify_version=bool(flags.VERSION_CHECK),
456
+ verify_version=bool(getattr(self, "version_check", True)),
299
457
  )
300
458
  project.project_env_vars = project_renderer.ctx_obj.env_vars
301
459
 
302
- return (project, profile)
460
+ return project, profile
303
461
 
304
462
  @property
305
463
  def profile_name(self) -> str:
@@ -336,8 +494,8 @@ class BaseConfig:
336
494
  A PartialProject instance for this configuration.
337
495
  """
338
496
  project_root = self.project_dir if self.project_dir else os.getcwd()
339
- version_check = bool(flags.VERSION_CHECK)
340
- partial_project = Project.partial_load(
497
+ version_check = bool(getattr(self, "version_check", True))
498
+ partial_project = PartialProject.from_project_root(
341
499
  project_root, verify_version=version_check
342
500
  )
343
501
 
@@ -390,7 +548,9 @@ class SelectionConfig(BaseConfig):
390
548
  exclude: Optional[list[str]] = None
391
549
  select: Optional[list[str]] = None
392
550
  selector_name: Optional[list[str]] = None
551
+ selector: Optional[str] = None
393
552
  state: Optional[Union[Path, str]] = None
553
+ defer_state: Optional[Union[Path, str]] = None
394
554
 
395
555
  # Kept for compatibility with dbt versions < 0.21
396
556
  models: Optional[list[str]] = None
@@ -400,6 +560,8 @@ class SelectionConfig(BaseConfig):
400
560
  super().__post_init__()
401
561
  if isinstance(self.state, str):
402
562
  self.state = Path(self.state)
563
+ if isinstance(self.defer_state, str):
564
+ self.defer_state = Path(self.defer_state)
403
565
 
404
566
 
405
567
  @dataclasses.dataclass
@@ -419,7 +581,6 @@ class BuildTaskConfig(TableMutabilityConfig):
419
581
 
420
582
  cls: Type[BaseTask] = dataclasses.field(default=BuildTask, init=False)
421
583
  singular: Optional[bool] = None
422
- indirect_selection: Optional[str] = None
423
584
  resource_types: Optional[list[str]] = None
424
585
  generic: Optional[bool] = None
425
586
  show: Optional[bool] = None
@@ -466,6 +627,7 @@ class DebugTaskConfig(BaseConfig):
466
627
 
467
628
  cls: Type[BaseTask] = dataclasses.field(default=DebugTask, init=False)
468
629
  config_dir: Optional[bool] = None
630
+ connection: Optional[bool] = None
469
631
  which: str = dataclasses.field(default="debug", init=False)
470
632
 
471
633
 
@@ -483,6 +645,7 @@ class GenerateTaskConfig(SelectionConfig):
483
645
 
484
646
  cls: Type[BaseTask] = dataclasses.field(default=GenerateTask, init=False)
485
647
  compile: bool = True
648
+ empty_catalog: bool = False
486
649
  which: str = dataclasses.field(default="generate", init=False)
487
650
 
488
651
 
@@ -491,7 +654,7 @@ class ListTaskConfig(SelectionConfig):
491
654
  """Dbt list task arguments."""
492
655
 
493
656
  cls: Type[BaseTask] = dataclasses.field(default=ListTask, init=False)
494
- indirect_selection: Optional[str] = None
657
+ indirect_selection: str = "eager"
495
658
  output: Output = Output.SELECTOR
496
659
  output_keys: Optional[list[str]] = None
497
660
  resource_types: Optional[list[str]] = None
@@ -499,8 +662,11 @@ class ListTaskConfig(SelectionConfig):
499
662
 
500
663
 
501
664
  @dataclasses.dataclass
502
- class ParseTaskConfig(BaseConfig):
503
- """Dbt parse task arguments."""
665
+ class ParseTaskConfig(SelectionConfig):
666
+ """Dbt parse task arguments.
667
+
668
+ TODO: Temporarily, handle this as a CompileTask.
669
+ """
504
670
 
505
671
  cls: Type[BaseTask] = dataclasses.field(default=ParseTask, init=False)
506
672
  compile: Optional[bool] = None
@@ -520,7 +686,7 @@ class RunTaskConfig(TableMutabilityConfig):
520
686
  class RunOperationTaskConfig(BaseConfig):
521
687
  """Dbt run-operation task arguments."""
522
688
 
523
- args: str = "{}"
689
+ args: dict[str, Any] | str = dataclasses.field(default_factory=dict)
524
690
  cls: Type[BaseTask] = dataclasses.field(default=RunOperationTask, init=False)
525
691
  macro: Optional[str] = None
526
692
  which: str = dataclasses.field(default="run-operation", init=False)
@@ -528,7 +694,8 @@ class RunOperationTaskConfig(BaseConfig):
528
694
  def __post_init__(self):
529
695
  """Support dictionary args by casting them to str after setting."""
530
696
  super().__post_init__()
531
- self.args = str(self.args)
697
+ if DBT_INSTALLED_LESS_THAN_1_5:
698
+ self.args = str(self.args)
532
699
 
533
700
 
534
701
  @dataclasses.dataclass
@@ -563,7 +730,7 @@ class TestTaskConfig(SelectionConfig):
563
730
 
564
731
  cls: Type[BaseTask] = dataclasses.field(default=TestTask, init=False)
565
732
  generic: Optional[bool] = None
566
- indirect_selection: Optional[str] = None
733
+ indirect_selection: str = "eager"
567
734
  singular: Optional[bool] = None
568
735
  store_failures: Optional[bool] = None
569
736
  which: str = dataclasses.field(default="test", init=False)
@@ -0,0 +1,30 @@
1
+ """Provides utilities to differentiate between installed dbt versions.
2
+
3
+ These are only used to ensure backwards compatibility with older versions of dbt.
4
+ """
5
+
6
+ from dbt.semver import Matchers, VersionSpecifier
7
+ from dbt.version import installed
8
+
9
+ DBT_1_5 = VersionSpecifier(
10
+ major="1", minor="5", patch="0", matcher=Matchers.GREATER_THAN_OR_EQUAL
11
+ )
12
+ DBT_1_6 = VersionSpecifier(
13
+ major="1", minor="6", patch="0", matcher=Matchers.GREATER_THAN_OR_EQUAL
14
+ )
15
+ DBT_1_7 = VersionSpecifier(
16
+ major="1", minor="7", patch="0", matcher=Matchers.GREATER_THAN_OR_EQUAL
17
+ )
18
+ DBT_1_8 = VersionSpecifier(
19
+ major="1", minor="8", patch="0", matcher=Matchers.GREATER_THAN_OR_EQUAL
20
+ )
21
+
22
+ DBT_INSTALLED_LESS_THAN_1_5 = installed < DBT_1_5
23
+
24
+ DBT_INSTALLED_GTE_1_5 = installed.compare(DBT_1_5) == 1
25
+ DBT_INSTALLED_GTE_1_6 = installed.compare(DBT_1_6) == 1
26
+ DBT_INSTALLED_GTE_1_7 = installed.compare(DBT_1_7) == 1
27
+
28
+ DBT_INSTALLED_1_5 = DBT_1_5 < installed < DBT_1_6
29
+ DBT_INSTALLED_1_6 = DBT_1_6 < installed < DBT_1_7
30
+ DBT_INSTALLED_1_7 = DBT_1_7 < installed < DBT_1_8
@@ -1,6 +1,6 @@
1
1
  [tool.poetry]
2
2
  name = "airflow-dbt-python"
3
- version = "1.0.5"
3
+ version = "2.0.0"
4
4
  description = "A collection of Airflow operators, hooks, and utilities to execute dbt commands"
5
5
  authors = ["Tomás Farías Santana <tomas@tomasfarias.dev>"]
6
6
  license = "MIT"
@@ -13,7 +13,6 @@ classifiers = [
13
13
 
14
14
  "License :: OSI Approved :: MIT License",
15
15
 
16
- "Programming Language :: Python :: 3.7",
17
16
  "Programming Language :: Python :: 3.8",
18
17
  "Programming Language :: Python :: 3.9",
19
18
  "Programming Language :: Python :: 3.10",
@@ -21,19 +20,39 @@ classifiers = [
21
20
  ]
22
21
 
23
22
  [tool.poetry.dependencies]
24
- python = ">=3.7.2,<4.0" # Airflow providers require this upper boundary
23
+ python = ">=3.8,<3.12" # Airflow providers require this upper boundary
25
24
 
26
- apache-airflow = { version = ">=2.0", optional = true }
25
+ apache-airflow = { version = ">=2.2" }
27
26
  apache-airflow-providers-amazon = { version = ">=3.0.0", optional = true }
28
27
  apache-airflow-providers-ssh = { version = ">=3.0.0", optional = true }
28
+ connexion = { version = "<3.0" } # https://github.com/apache/airflow/issues/35217
29
29
  dulwich = { version = ">=0.21", optional = true }
30
+ contextlib-chdir = { version = "==1.0.2", python = "<3.11" }
30
31
 
31
- dbt-core = ">=1.0.0"
32
- dbt-bigquery = { version = ">=1.0.0", optional = true }
33
- dbt-postgres = { version = ">=1.0.0", optional = true }
34
- dbt-redshift = { version = ">=1.0.0", optional = true }
35
- dbt-snowflake = { version = ">=1.0.0", optional = true }
36
- dbt-spark = { version = ">=1.0.0", optional = true }
32
+ dbt-core = [
33
+ { python = "<3.8", version = ">=1.4.0,<1.6.0", optional = true },
34
+ { python = ">=3.11", version = ">=1.4.0,<1.8.0", optional = true },
35
+ ]
36
+ dbt-bigquery = [
37
+ { python = "<3.8", version = ">=1.4.0,<1.6.0", optional = true },
38
+ { python = ">=3.11", version = ">=1.4.0,<1.8.0", optional = true },
39
+ ]
40
+ dbt-postgres = [
41
+ { python = "<3.8", version = ">=1.4.0,<1.6.0", optional = true },
42
+ { python = ">=3.11", version = ">=1.4.0,<1.8.0", optional = true },
43
+ ]
44
+ dbt-redshift = [
45
+ { python = "<3.8", version = ">=1.4.0,<1.6.0", optional = true },
46
+ { python = ">=3.11", version = ">=1.4.0,<1.8.0", optional = true },
47
+ ]
48
+ dbt-snowflake = [
49
+ { python = "<3.8", version = ">=1.4.0,<1.6.0", optional = true },
50
+ { python = ">=3.11", version = ">=1.4.0,<1.8.0", optional = true },
51
+ ]
52
+ dbt-spark = [
53
+ { python = "<3.8", version = ">=1.4.0,<1.6.0", optional = true },
54
+ { python = ">=3.11", version = ">=1.4.0,<1.8.0", optional = true },
55
+ ]
37
56
 
38
57
  [tool.poetry.group.dev]
39
58
  optional = true
@@ -44,8 +63,8 @@ apache-airflow-providers-ssh = ">=3.0.0"
44
63
  black = ">=22"
45
64
  boto3-stubs = { extras = ["s3"], version = ">=1.26.8" }
46
65
  coverage = { extras = ["toml"], version = ">=7.2" }
47
- dbt-postgres = ">=1.0.0"
48
- dulwich = ">=0.21"
66
+ dbt-postgres = ">=1.0.0,<1.8.0"
67
+ dulwich = ">=0.21,!=0.21.6"
49
68
  moto = ">=4.0.3"
50
69
  mypy = ">=1"
51
70
  freezegun = ">=1.1.0"
@@ -61,7 +80,11 @@ types-PyYAML = ">=6.0.7"
61
80
  optional = true
62
81
 
63
82
  [tool.poetry.group.docs.dependencies]
64
- Sphinx = ">=5.3"
83
+ Sphinx = [
84
+ { python = "<3.8", version = "<6.0" },
85
+ { python = ">=3.8,<3.9", version = "<7.2" },
86
+ { python = ">=3.9", version = ">=7.2" },
87
+ ]
65
88
  sphinx-rtd-theme = ">=1.1"
66
89
  sphinx-copybutton = ">=0.5"
67
90
  attrs = ">=22"
@@ -70,14 +93,28 @@ attrs = ">=22"
70
93
  optional = true
71
94
 
72
95
  [tool.poetry.group.adapters.dependencies]
73
- dbt-bigquery = ">=1.0.0"
74
- dbt-postgres = ">=1.0.0"
75
- dbt-redshift = ">=1.0.0"
76
- dbt-snowflake = ">=1.0.0"
77
- dbt-spark = ">=1.0.0"
96
+ dbt-bigquery = [
97
+ { python = "<3.8", version = ">=1.4.0,<1.6.0", optional = true },
98
+ { python = ">=3.11", version = ">=1.4.0,<1.8.0", optional = true },
99
+ ]
100
+ dbt-postgres = [
101
+ { python = "<3.8", version = ">=1.4.0,<1.6.0", optional = true },
102
+ { python = ">=3.11", version = ">=1.4.0,<1.8.0", optional = true },
103
+ ]
104
+ dbt-redshift = [
105
+ { python = "<3.8", version = ">=1.4.0,<1.6.0", optional = true },
106
+ { python = ">=3.11", version = ">=1.4.0,<1.8.0", optional = true },
107
+ ]
108
+ dbt-snowflake = [
109
+ { python = "<3.8", version = ">=1.4.0,<1.6.0", optional = true },
110
+ { python = ">=3.11", version = ">=1.4.0,<1.8.0", optional = true },
111
+ ]
112
+ dbt-spark = [
113
+ { python = "<3.8", version = ">=1.4.0,<1.6.0", optional = true },
114
+ { python = ">=3.11", version = ">=1.4.0,<1.8.0", optional = true },
115
+ ]
78
116
 
79
117
  [tool.poetry.extras]
80
- airflow = ["apache-airflow"]
81
118
  airflow-providers = ["apache-airflow-providers-amazon", "apache-airflow-providers-ssh"]
82
119
  adapters = ["dbt-bigquery", "dbt-redshift", "dbt-postgres", "dbt-snowflake", "dbt-spark"]
83
120
  bigquery = ["dbt-bigquery"]
@@ -11,30 +11,60 @@ package_data = \
11
11
  {'': ['*']}
12
12
 
13
13
  install_requires = \
14
- ['dbt-core>=1.0.0']
14
+ ['apache-airflow>=2.2', 'connexion<3.0']
15
15
 
16
16
  extras_require = \
17
- {'adapters': ['dbt-bigquery>=1.0.0',
18
- 'dbt-postgres>=1.0.0',
19
- 'dbt-redshift>=1.0.0',
20
- 'dbt-snowflake>=1.0.0',
21
- 'dbt-spark>=1.0.0'],
22
- 'airflow': ['apache-airflow>=2.0'],
17
+ {':python_version < "3.11"': ['contextlib-chdir==1.0.2'],
18
+ 'adapters:python_version < "3.8"': ['dbt-bigquery>=1.4.0,<1.6.0',
19
+ 'dbt-bigquery>=1.4.0,<1.6.0',
20
+ 'dbt-postgres>=1.4.0,<1.6.0',
21
+ 'dbt-postgres>=1.4.0,<1.6.0',
22
+ 'dbt-redshift>=1.4.0,<1.6.0',
23
+ 'dbt-redshift>=1.4.0,<1.6.0',
24
+ 'dbt-snowflake>=1.4.0,<1.6.0',
25
+ 'dbt-snowflake>=1.4.0,<1.6.0',
26
+ 'dbt-spark>=1.4.0,<1.6.0',
27
+ 'dbt-spark>=1.4.0,<1.6.0'],
28
+ 'adapters:python_version >= "3.11"': ['dbt-bigquery>=1.4.0,<1.8.0',
29
+ 'dbt-bigquery>=1.4.0,<1.8.0',
30
+ 'dbt-postgres>=1.4.0,<1.8.0',
31
+ 'dbt-postgres>=1.4.0,<1.8.0',
32
+ 'dbt-redshift>=1.4.0,<1.8.0',
33
+ 'dbt-redshift>=1.4.0,<1.8.0',
34
+ 'dbt-snowflake>=1.4.0,<1.8.0',
35
+ 'dbt-snowflake>=1.4.0,<1.8.0',
36
+ 'dbt-spark>=1.4.0,<1.8.0',
37
+ 'dbt-spark>=1.4.0,<1.8.0'],
23
38
  'airflow-providers': ['apache-airflow-providers-amazon>=3.0.0',
24
39
  'apache-airflow-providers-ssh>=3.0.0'],
25
- 'bigquery': ['dbt-bigquery>=1.0.0'],
40
+ 'bigquery:python_version < "3.8"': ['dbt-bigquery>=1.4.0,<1.6.0',
41
+ 'dbt-bigquery>=1.4.0,<1.6.0'],
42
+ 'bigquery:python_version >= "3.11"': ['dbt-bigquery>=1.4.0,<1.8.0',
43
+ 'dbt-bigquery>=1.4.0,<1.8.0'],
26
44
  'git': ['apache-airflow-providers-ssh>=3.0.0', 'dulwich>=0.21'],
27
- 'postgres': ['dbt-postgres>=1.0.0'],
28
- 'redshift': ['dbt-redshift>=1.0.0'],
45
+ 'postgres:python_version < "3.8"': ['dbt-postgres>=1.4.0,<1.6.0',
46
+ 'dbt-postgres>=1.4.0,<1.6.0'],
47
+ 'postgres:python_version >= "3.11"': ['dbt-postgres>=1.4.0,<1.8.0',
48
+ 'dbt-postgres>=1.4.0,<1.8.0'],
49
+ 'redshift:python_version < "3.8"': ['dbt-redshift>=1.4.0,<1.6.0',
50
+ 'dbt-redshift>=1.4.0,<1.6.0'],
51
+ 'redshift:python_version >= "3.11"': ['dbt-redshift>=1.4.0,<1.8.0',
52
+ 'dbt-redshift>=1.4.0,<1.8.0'],
29
53
  's3': ['apache-airflow-providers-amazon>=3.0.0'],
30
- 'snowflake': ['dbt-snowflake>=1.0.0'],
31
- 'spark': ['dbt-spark>=1.0.0']}
54
+ 'snowflake:python_version < "3.8"': ['dbt-snowflake>=1.4.0,<1.6.0',
55
+ 'dbt-snowflake>=1.4.0,<1.6.0'],
56
+ 'snowflake:python_version >= "3.11"': ['dbt-snowflake>=1.4.0,<1.8.0',
57
+ 'dbt-snowflake>=1.4.0,<1.8.0'],
58
+ 'spark:python_version < "3.8"': ['dbt-spark>=1.4.0,<1.6.0',
59
+ 'dbt-spark>=1.4.0,<1.6.0'],
60
+ 'spark:python_version >= "3.11"': ['dbt-spark>=1.4.0,<1.8.0',
61
+ 'dbt-spark>=1.4.0,<1.8.0']}
32
62
 
33
63
  setup_kwargs = {
34
64
  'name': 'airflow-dbt-python',
35
- 'version': '1.0.5',
65
+ 'version': '2.0.0',
36
66
  'description': 'A collection of Airflow operators, hooks, and utilities to execute dbt commands',
37
- 'long_description': '# airflow-dbt-python\n\n[![PyPI version](https://img.shields.io/pypi/v/airflow-dbt-python?style=plastic)](https://pypi.org/project/airflow-dbt-python/)\n[![CI testing](https://github.com/tomasfarias/airflow-dbt-python/actions/workflows/ci.yaml/badge.svg)](https://github.com/tomasfarias/airflow-dbt-python/actions)\n[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)\n[![Test coverage](https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/tomasfarias/81ef37701aa088d18db8a58ce07c79c7/raw/covbadge.json)](https://github.com/tomasfarias/airflow-dbt-python/actions)\n[![Documentation](https://readthedocs.org/projects/airflow-dbt-python/badge/?version=latest)](https://airflow-dbt-python.readthedocs.io/en/latest/?badge=latest)\n\nA collection of [Airflow](https://airflow.apache.org/) operators, hooks, and utilities to execute [`dbt`](https://pypi.org/project/dbt-core/) commands.\n\nRead the [documentation](https://airflow-dbt-python.readthedocs.io) for examples, installation instructions, and more details.\n\n# Installation\n\n## Requirements\n\nBefore using *airflow-dbt-python*, ensure you meet the following requirements:\n* A *dbt* project using [dbt-core](https://pypi.org/project/dbt-core/) version 1.0.0 or later.\n* An Airflow environment using version 2.2 or later.\n\n * If using any managed service, like AWS MWAA, ensure your environment is created with a supported version of Airflow.\n * If self-hosting, Airflow installation instructions can be found in their [official documentation](https://airflow.apache.org/docs/apache-airflow/stable/installation/index.html).\n\n* Running Python 3.7 or later in your Airflow environment.\n\n> **Warning**\n>\n> Even though we don\'t impose any upper limits on versions of Airflow and *dbt*, it\'s possible that new versions are not supported immediately after release, particularly for *dbt*. We recommend testing the latest versions before upgrading and [reporting any issues](https://github.com/tomasfarias/airflow-dbt-python/issues/new/choose).\n\n> **Note**\n>\n> Older versions of Airflow and *dbt* may work with *airflow-dbt-python*, although we cannot guarantee this. Our testing pipeline runs the latest *dbt-core* with the latest Airflow release, and the latest version supported by [AWS MWAA](https://aws.amazon.com/managed-workflows-for-apache-airflow/).\n\n## From PyPI\n\n*airflow-dbt-python* is available in [PyPI](https://pypi.org/project/airflow-dbt-python/) and can be installed with *pip*:\n\n``` shell\npip install airflow-dbt-python\n```\n\nAs a convenience, some *dbt* adapters can be installed by specifying extras. For example, if requiring the *dbt-redshift* adapter:\n\n``` shell\npip install airflow-dbt-python[redshift]\n```\n\n## From this repo\n\n*airflow-dbt-python* can also be built from source by cloning this GitHub repository:\n\n``` shell\ngit clone https://github.com/tomasfarias/airflow-dbt-python.git\ncd airflow-dbt-python\n```\n\nAnd installing with *Poetry*:\n\n``` shell\npoetry install\n```\n\n## In AWS MWAA\n\nAdd *airflow-dbt-python* to your `requirements.txt` file and edit your Airflow environment to use this new `requirements.txt` file, or upload it as a plugin.\n\nRead the [documentation](https://airflow-dbt-python.readthedocs.io/en/latest/getting_started.html#installing-in-mwaa) for more a more detailed AWS MWAA installation breakdown.\n\n## In other managed services\n\n*airflow-dbt-python* should be compatible with most or all Airflow managed services. Consult the documentation specific to your provider.\n\nIf you notice an issue when installing *airflow-dbt-python* in a specific managed service, please open an [issue](https://github.com/tomasfarias/airflow-dbt-python/issues/new/choose).\n\n# Features\n\n*airflow-dbt-python* aims to make dbt a **first-class citizen** of Airflow by supporting additional features that integrate both tools. As you would expect, *airflow-dbt-python* can run all your dbt workflows in Airflow with the same interface you are used to from the CLI, but without being a mere wrapper: *airflow-dbt-python* directly communicates with internal *dbt-core* classes, bridging the gap between them and Airflow\'s operator interface. Essentially, we are attempting to use *dbt* **as a library**.\n\nAs this integration was completed, several features were developed to **extend the capabilities of dbt** to leverage Airflow as much as possible. Can you think of a way *dbt* could leverage Airflow that is not currently supported? Let us know in a [GitHub issue](https://github.com/tomasfarias/airflow-dbt-python/issues/new/choose)!\n\n## Independent task execution\n\nAirflow executes [Tasks](https://airflow.apache.org/docs/apache-airflow/stable/concepts/tasks.html) independent of one another: even though downstream and upstream dependencies between tasks exist, the execution of an individual task happens entirely independently of any other task execution (see: [Tasks Relationships](https://airflow.apache.org/docs/apache-airflow/stable/concepts/tasks.html#relationships)).\n\nIn order to work with this constraint, *airflow-dbt-python* runs each dbt command in a **temporary and isolated directory**. Before execution, all the relevant dbt files are copied from supported backends, and after executing the command any artifacts are exported. This ensures dbt can work with any Airflow deployment, including most production deployments as they are usually running [Remote Executors](https://airflow.apache.org/docs/apache-airflow/stable/executor/index.html#executor-types) and do not guarantee any files will be shared by default between tasks, since each task may run in a completely different environment.\n\n\n## Download dbt files from a remote storage\n\nThe dbt parameters `profiles_dir` and `project_dir` would normally point to a directory containing a `profiles.yml` file and a dbt project in the local environment respectively (defined by the presence of a *dbt_project.yml* file). *airflow-dbt-python* extends these parameters to also accept an URL pointing to a remote storage.\n\nCurrently, we support the following remote storages:\n\n* [AWS S3](https://aws.amazon.com/s3/) (identified by a *s3* scheme).\n* Remote git repositories, like those stored in GitHub (both *https* and *ssh* schemes are supported).\n\n* If a remote URL is used for `project_dir`, then this URL must point to a location in your remote storage containing a *dbt* project to run. A *dbt* project is identified by the prescence of a *dbt_project.yml*, and contains all your [resources](https://docs.getdbt.com/docs/build/projects). All of the contents of this remote location will be downloaded and made available for the operator. The URL may also point to an archived file containing all the files of a dbt project, which will be downloaded, uncompressed, and made available for the operator.\n* If a remote URL is used for `profiles_dir`, then this URL must point to a location in your remote storage that contains a *profiles.yml* file. The *profiles.yml* file will be downloaded and made available for the operator to use when running. The *profiles.yml* may be part of your *dbt* project, in which case this argument may be ommitted.\n\nThis feature is intended to work in line with Airflow\'s [description of the task concept](https://airflow.apache.org/docs/apache-airflow/stable/concepts/tasks.html#relationships):\n\n> Tasks don’t pass information to each other by default, and run entirely independently.\n\nWe interpret this as meaning a task should be responsible of fetching all the *dbt* related files it needs in order to run independently, as already described in [Independent Task Execution](#independent-task-execution).\n\n## Push dbt artifacts to XCom\n\nEach dbt execution produces one or more [JSON artifacts](https://docs.getdbt.com/reference/artifacts/dbt-artifacts/) that are valuable to produce meta-metrics, build conditional workflows, for reporting purposes, and other uses. *airflow-dbt-python* can push these artifacts to [XCom](https://airflow.apache.org/docs/apache-airflow/stable/concepts/xcoms.html) as requested via the `do_xcom_push_artifacts` parameter, which takes a list of artifacts to push.\n\n## Use Airflow connections as dbt targets (without a profiles.yml)\n\n[Airflow connections](https://airflow.apache.org/docs/apache-airflow/stable/howto/connection.html) allow users to manage and store connection information, such as hostname, port, username, and password, for operators to use when accessing certain applications, like databases. Similarly, a *dbt* `profiles.yml` file stores connection information under each target key. *airflow-dbt-python* bridges the gap between the two and allows you to use connection information stored as an Airflow connection by specifying the connection id as the `target` parameter of any of the *dbt* operators it provides. What\'s more, if using an Airflow connection, the `profiles.yml` file may be entirely omitted (although keep in mind a `profiles.yml` file contains a configuration block besides target connection information).\n\nSee an example DAG [here](examples/airflow_connection_target_dag.py).\n\n# Motivation\n\n## Airflow running in a managed environment\n\nAlthough [`dbt`](https://docs.getdbt.com/) is meant to be installed and used as a CLI, we may not have control of the environment where Airflow is running, disallowing us the option of using *dbt* as a CLI.\n\nThis is exactly what happens when using [Amazon\'s Managed Workflows for Apache Airflow](https://aws.amazon.com/managed-workflows-for-apache-airflow/) or MWAA: although a list of Python requirements can be passed, the CLI cannot be found in the worker\'s PATH.\n\nThere is a workaround which involves using Airflow\'s `BashOperator` and running Python from the command line:\n\n``` python\nfrom airflow.operators.bash import BashOperator\n\nBASH_COMMAND = "python -c \'from dbt.main import main; main()\' run"\noperator = BashOperator(\n task_id="dbt_run",\n bash_command=BASH_COMMAND,\n)\n```\n\nBut it can get cumbersome when appending all potential arguments a `dbt run` command (or other subcommand) can take.\n\nThat\'s where *airflow-dbt-python* comes in: it abstracts the complexity of interfacing with *dbt-core* and exposes one operator for each *dbt* subcommand that can be instantiated with all the corresponding arguments that the *dbt* CLI would take.\n\n## An alternative to *airflow-dbt* that works without the *dbt* CLI\n\nThe alternative [`airflow-dbt`](https://pypi.org/project/airflow-dbt/) package, by default, would not work if the *dbt* CLI is not in PATH, which means it would not be usable in MWAA. There is a workaround via the `dbt_bin` argument, which can be set to `"python -c \'from dbt.main import main; main()\' run"`, in similar fashion as the `BashOperator` example. Yet this approach is not without its limitations:\n* *airflow-dbt* works by wrapping the *dbt* CLI, which makes our code dependent on the environment in which it runs.\n* *airflow-dbt* does not support the full range of arguments a command can take. For example, `DbtRunOperator` does not have an attribute for `fail_fast`.\n* *airflow-dbt* does not offer access to *dbt* artifacts created during execution. *airflow-dbt-python* does so by pushing any artifacts to [XCom](https://airflow.apache.org/docs/apache-airflow/stable/concepts/xcoms.html).\n\n# Usage\n\nCurrently, the following *dbt* commands are supported:\n\n* `clean`\n* `compile`\n* `debug`\n* `deps`\n* `docs generate`\n* `ls`\n* `parse`\n* `run`\n* `run-operation`\n* `seed`\n* `snapshot`\n* `source`\n* `test`\n\n## Examples\n\nAll example DAGs are tested against the latest Airflow version. Some changes, like modifying `import` statements or changing types, may be required for them to work in other versions.\n\n``` python\nimport datetime as dt\n\nimport pendulum\nfrom airflow import DAG\n\nfrom airflow_dbt_python.operators.dbt import (\n DbtRunOperator,\n DbtSeedOperator,\n DbtTestOperator,\n)\n\nargs = {\n "owner": "airflow",\n}\n\nwith DAG(\n dag_id="example_dbt_operator",\n default_args=args,\n schedule="0 0 * * *",\n start_date=pendulum.today("UTC").add(days=-1),\n dagrun_timeout=dt.timedelta(minutes=60),\n tags=["example", "example2"],\n) as dag:\n dbt_test = DbtTestOperator(\n task_id="dbt_test",\n selector_name="pre-run-tests",\n )\n\n dbt_seed = DbtSeedOperator(\n task_id="dbt_seed",\n select=["/path/to/first.csv", "/path/to/second.csv"],\n full_refresh=True,\n )\n\n dbt_run = DbtRunOperator(\n task_id="dbt_run",\n select=["/path/to/models"],\n full_refresh=True,\n fail_fast=True,\n )\n\n dbt_test >> dbt_seed >> dbt_run\n```\n\nMore examples can be found in the [`examples/`](examples/) directory and the [documentation](https://airflow-dbt-python.readthedocs.io).\n\n# Development\n\nSee the [development documentation](https://airflow-dbt-python.readthedocs.io/en/latest/development.html) for a more in-depth dive into setting up a development environment, running the test-suite, and general commentary on working on *airflow-dbt-python*.\n\n## Testing\n\nTests are run with *pytest*, can be located in `tests/`. To run them locally, you may use *Poetry*:\n\n``` shell\npoetry run pytest tests/ -vv\n```\n\n# License\n\nThis project is licensed under the MIT license. See [LICENSE](LICENSE).\n',
67
+ 'long_description': '# airflow-dbt-python\n\n[![PyPI version](https://img.shields.io/pypi/v/airflow-dbt-python?style=plastic)](https://pypi.org/project/airflow-dbt-python/)\n[![CI testing](https://github.com/tomasfarias/airflow-dbt-python/actions/workflows/ci.yaml/badge.svg)](https://github.com/tomasfarias/airflow-dbt-python/actions)\n[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)\n[![Test coverage](https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/tomasfarias/81ef37701aa088d18db8a58ce07c79c7/raw/covbadge.json)](https://github.com/tomasfarias/airflow-dbt-python/actions)\n[![Documentation](https://readthedocs.org/projects/airflow-dbt-python/badge/?version=latest)](https://airflow-dbt-python.readthedocs.io/en/latest/?badge=latest)\n\nA collection of [Airflow](https://airflow.apache.org/) operators, hooks, and utilities to execute [`dbt`](https://pypi.org/project/dbt-core/) commands.\n\nRead the [documentation](https://airflow-dbt-python.readthedocs.io) for examples, installation instructions, and more details.\n\n# Installation\n\n## Requirements\n\nBefore using *airflow-dbt-python*, ensure you meet the following requirements:\n* A *dbt* project using [dbt-core](https://pypi.org/project/dbt-core/) version 1.4.0 or later.\n* An Airflow environment using version 2.2 or later.\n\n * If using any managed service, like AWS MWAA, ensure your environment is created with a supported version of Airflow.\n * If self-hosting, Airflow installation instructions can be found in their [official documentation](https://airflow.apache.org/docs/apache-airflow/stable/installation/index.html).\n\n* Running Python 3.8 or later in your Airflow environment.\n\n> **Warning**\n>\n> Even though we don\'t impose any upper limits on versions of Airflow and *dbt*, it\'s possible that new versions are not supported immediately after release, particularly for *dbt*. We recommend testing the latest versions before upgrading and [reporting any issues](https://github.com/tomasfarias/airflow-dbt-python/issues/new/choose).\n\n> **Note**\n>\n> Older versions of Airflow and *dbt* may work with *airflow-dbt-python*, although we cannot guarantee this. Our testing pipeline runs the latest *dbt-core* with the latest Airflow release, and the latest version supported by [AWS MWAA](https://aws.amazon.com/managed-workflows-for-apache-airflow/).\n\n## From PyPI\n\n*airflow-dbt-python* is available in [PyPI](https://pypi.org/project/airflow-dbt-python/) and can be installed with *pip*:\n\n``` shell\npip install airflow-dbt-python\n```\n\nAs a convenience, some *dbt* adapters can be installed by specifying extras. For example, if requiring the *dbt-redshift* adapter:\n\n``` shell\npip install airflow-dbt-python[redshift]\n```\n\n## From this repo\n\n*airflow-dbt-python* can also be built from source by cloning this GitHub repository:\n\n``` shell\ngit clone https://github.com/tomasfarias/airflow-dbt-python.git\ncd airflow-dbt-python\n```\n\nAnd installing with *Poetry*:\n\n``` shell\npoetry install\n```\n\n## In AWS MWAA\n\nAdd *airflow-dbt-python* to your `requirements.txt` file and edit your Airflow environment to use this new `requirements.txt` file, or upload it as a plugin.\n\nRead the [documentation](https://airflow-dbt-python.readthedocs.io/en/latest/getting_started.html#installing-in-mwaa) for more a more detailed AWS MWAA installation breakdown.\n\n## In other managed services\n\n*airflow-dbt-python* should be compatible with most or all Airflow managed services. Consult the documentation specific to your provider.\n\nIf you notice an issue when installing *airflow-dbt-python* in a specific managed service, please open an [issue](https://github.com/tomasfarias/airflow-dbt-python/issues/new/choose).\n\n# Features\n\n*airflow-dbt-python* aims to make dbt a **first-class citizen** of Airflow by supporting additional features that integrate both tools. As you would expect, *airflow-dbt-python* can run all your dbt workflows in Airflow with the same interface you are used to from the CLI, but without being a mere wrapper: *airflow-dbt-python* directly communicates with internal *dbt-core* classes, bridging the gap between them and Airflow\'s operator interface. Essentially, we are attempting to use *dbt* **as a library**.\n\nAs this integration was completed, several features were developed to **extend the capabilities of dbt** to leverage Airflow as much as possible. Can you think of a way *dbt* could leverage Airflow that is not currently supported? Let us know in a [GitHub issue](https://github.com/tomasfarias/airflow-dbt-python/issues/new/choose)!\n\n## Independent task execution\n\nAirflow executes [Tasks](https://airflow.apache.org/docs/apache-airflow/stable/concepts/tasks.html) independent of one another: even though downstream and upstream dependencies between tasks exist, the execution of an individual task happens entirely independently of any other task execution (see: [Tasks Relationships](https://airflow.apache.org/docs/apache-airflow/stable/concepts/tasks.html#relationships)).\n\nIn order to work with this constraint, *airflow-dbt-python* runs each dbt command in a **temporary and isolated directory**. Before execution, all the relevant dbt files are copied from supported backends, and after executing the command any artifacts are exported. This ensures dbt can work with any Airflow deployment, including most production deployments as they are usually running [Remote Executors](https://airflow.apache.org/docs/apache-airflow/stable/executor/index.html#executor-types) and do not guarantee any files will be shared by default between tasks, since each task may run in a completely different environment.\n\n\n## Download dbt files from a remote storage\n\nThe dbt parameters `profiles_dir` and `project_dir` would normally point to a directory containing a `profiles.yml` file and a dbt project in the local environment respectively (defined by the presence of a *dbt_project.yml* file). *airflow-dbt-python* extends these parameters to also accept an URL pointing to a remote storage.\n\nCurrently, we support the following remote storages:\n\n* [AWS S3](https://aws.amazon.com/s3/) (identified by a *s3* scheme).\n* Remote git repositories, like those stored in GitHub (both *https* and *ssh* schemes are supported).\n\n* If a remote URL is used for `project_dir`, then this URL must point to a location in your remote storage containing a *dbt* project to run. A *dbt* project is identified by the prescence of a *dbt_project.yml*, and contains all your [resources](https://docs.getdbt.com/docs/build/projects). All of the contents of this remote location will be downloaded and made available for the operator. The URL may also point to an archived file containing all the files of a dbt project, which will be downloaded, uncompressed, and made available for the operator.\n* If a remote URL is used for `profiles_dir`, then this URL must point to a location in your remote storage that contains a *profiles.yml* file. The *profiles.yml* file will be downloaded and made available for the operator to use when running. The *profiles.yml* may be part of your *dbt* project, in which case this argument may be ommitted.\n\nThis feature is intended to work in line with Airflow\'s [description of the task concept](https://airflow.apache.org/docs/apache-airflow/stable/concepts/tasks.html#relationships):\n\n> Tasks don’t pass information to each other by default, and run entirely independently.\n\nWe interpret this as meaning a task should be responsible of fetching all the *dbt* related files it needs in order to run independently, as already described in [Independent Task Execution](#independent-task-execution).\n\n## Push dbt artifacts to XCom\n\nEach dbt execution produces one or more [JSON artifacts](https://docs.getdbt.com/reference/artifacts/dbt-artifacts/) that are valuable to produce meta-metrics, build conditional workflows, for reporting purposes, and other uses. *airflow-dbt-python* can push these artifacts to [XCom](https://airflow.apache.org/docs/apache-airflow/stable/concepts/xcoms.html) as requested via the `do_xcom_push_artifacts` parameter, which takes a list of artifacts to push.\n\n## Use Airflow connections as dbt targets (without a profiles.yml)\n\n[Airflow connections](https://airflow.apache.org/docs/apache-airflow/stable/howto/connection.html) allow users to manage and store connection information, such as hostname, port, username, and password, for operators to use when accessing certain applications, like databases. Similarly, a *dbt* `profiles.yml` file stores connection information under each target key. *airflow-dbt-python* bridges the gap between the two and allows you to use connection information stored as an Airflow connection by specifying the connection id as the `target` parameter of any of the *dbt* operators it provides. What\'s more, if using an Airflow connection, the `profiles.yml` file may be entirely omitted (although keep in mind a `profiles.yml` file contains a configuration block besides target connection information).\n\nSee an example DAG [here](examples/airflow_connection_target_dag.py).\n\n# Motivation\n\n## Airflow running in a managed environment\n\nAlthough [`dbt`](https://docs.getdbt.com/) is meant to be installed and used as a CLI, we may not have control of the environment where Airflow is running, disallowing us the option of using *dbt* as a CLI.\n\nThis is exactly what happens when using [Amazon\'s Managed Workflows for Apache Airflow](https://aws.amazon.com/managed-workflows-for-apache-airflow/) or MWAA: although a list of Python requirements can be passed, the CLI cannot be found in the worker\'s PATH.\n\nThere is a workaround which involves using Airflow\'s `BashOperator` and running Python from the command line:\n\n``` python\nfrom airflow.operators.bash import BashOperator\n\nBASH_COMMAND = "python -c \'from dbt.main import main; main()\' run"\noperator = BashOperator(\n task_id="dbt_run",\n bash_command=BASH_COMMAND,\n)\n```\n\nBut it can get cumbersome when appending all potential arguments a `dbt run` command (or other subcommand) can take.\n\nThat\'s where *airflow-dbt-python* comes in: it abstracts the complexity of interfacing with *dbt-core* and exposes one operator for each *dbt* subcommand that can be instantiated with all the corresponding arguments that the *dbt* CLI would take.\n\n## An alternative to *airflow-dbt* that works without the *dbt* CLI\n\nThe alternative [`airflow-dbt`](https://pypi.org/project/airflow-dbt/) package, by default, would not work if the *dbt* CLI is not in PATH, which means it would not be usable in MWAA. There is a workaround via the `dbt_bin` argument, which can be set to `"python -c \'from dbt.main import main; main()\' run"`, in similar fashion as the `BashOperator` example. Yet this approach is not without its limitations:\n* *airflow-dbt* works by wrapping the *dbt* CLI, which makes our code dependent on the environment in which it runs.\n* *airflow-dbt* does not support the full range of arguments a command can take. For example, `DbtRunOperator` does not have an attribute for `fail_fast`.\n* *airflow-dbt* does not offer access to *dbt* artifacts created during execution. *airflow-dbt-python* does so by pushing any artifacts to [XCom](https://airflow.apache.org/docs/apache-airflow/stable/concepts/xcoms.html).\n\n# Usage\n\nCurrently, the following *dbt* commands are supported:\n\n* `clean`\n* `compile`\n* `debug`\n* `deps`\n* `docs generate`\n* `ls`\n* `parse`\n* `run`\n* `run-operation`\n* `seed`\n* `snapshot`\n* `source`\n* `test`\n\n## Examples\n\nAll example DAGs are tested against the latest Airflow version. Some changes, like modifying `import` statements or changing types, may be required for them to work in other versions.\n\n``` python\nimport datetime as dt\n\nimport pendulum\nfrom airflow import DAG\n\nfrom airflow_dbt_python.operators.dbt import (\n DbtRunOperator,\n DbtSeedOperator,\n DbtTestOperator,\n)\n\nargs = {\n "owner": "airflow",\n}\n\nwith DAG(\n dag_id="example_dbt_operator",\n default_args=args,\n schedule="0 0 * * *",\n start_date=pendulum.today("UTC").add(days=-1),\n dagrun_timeout=dt.timedelta(minutes=60),\n tags=["example", "example2"],\n) as dag:\n dbt_test = DbtTestOperator(\n task_id="dbt_test",\n selector_name="pre-run-tests",\n )\n\n dbt_seed = DbtSeedOperator(\n task_id="dbt_seed",\n select=["/path/to/first.csv", "/path/to/second.csv"],\n full_refresh=True,\n )\n\n dbt_run = DbtRunOperator(\n task_id="dbt_run",\n select=["/path/to/models"],\n full_refresh=True,\n fail_fast=True,\n )\n\n dbt_test >> dbt_seed >> dbt_run\n```\n\nMore examples can be found in the [`examples/`](examples/) directory and the [documentation](https://airflow-dbt-python.readthedocs.io).\n\n# Development\n\nSee the [development documentation](https://airflow-dbt-python.readthedocs.io/en/latest/development.html) for a more in-depth dive into setting up a development environment, running the test-suite, and general commentary on working on *airflow-dbt-python*.\n\n## Testing\n\nTests are run with *pytest*, can be located in `tests/`. To run them locally, you may use *Poetry*:\n\n``` shell\npoetry run pytest tests/ -vv\n```\n\n# License\n\nThis project is licensed under the MIT license. See [LICENSE](LICENSE).\n',
38
68
  'author': 'Tomás Farías Santana',
39
69
  'author_email': 'tomas@tomasfarias.dev',
40
70
  'maintainer': 'None',
@@ -44,7 +74,7 @@ setup_kwargs = {
44
74
  'package_data': package_data,
45
75
  'install_requires': install_requires,
46
76
  'extras_require': extras_require,
47
- 'python_requires': '>=3.7.2,<4.0',
77
+ 'python_requires': '>=3.8,<3.12',
48
78
  }
49
79
 
50
80