airflow-dbt-python 2.0.1__py3-none-any.whl → 2.2.0__py3-none-any.whl

airflow_dbt_python/__init__.py

@@ -1,2 +1,3 @@
 """Provides an Airflow operator and hooks to run all or most dbtcommands."""
+
 from .__version__ import __author__, __copyright__, __title__, __version__

airflow_dbt_python/__version__.py

@@ -1,5 +1,6 @@
 """The module's version information."""
+
 __author__ = "Tomás Farías Santana"
 __copyright__ = "Copyright 2021 Tomás Farías Santana"
 __title__ = "airflow-dbt-python"
-__version__ = "2.0.1"
+__version__ = "2.2.0"

airflow_dbt_python/hooks/dbt.py

@@ -1,4 +1,5 @@
 """Provides a hook to interact with a dbt project."""
+
 from __future__ import annotations
 
 import json
@@ -24,14 +25,11 @@ from airflow.exceptions import AirflowException
 from airflow.hooks.base import BaseHook
 from airflow.models.connection import Connection
 
-from airflow_dbt_python.utils.version import DBT_INSTALLED_LESS_THAN_1_5
-
 if sys.version_info >= (3, 11):
     from contextlib import chdir as chdir_ctx
 else:
     from contextlib_chdir import chdir as chdir_ctx
 
-
 if TYPE_CHECKING:
     from dbt.contracts.results import RunResult
     from dbt.task.base import BaseTask
@@ -227,18 +225,11 @@ class DbtHook(BaseHook):
             A tuple containing a boolean indicating success and optionally the results
                 of running the dbt command.
         """
-        from dbt.adapters.factory import register_adapter
+        from dbt.adapters.factory import adapter_management
         from dbt.task.base import get_nearest_project_dir
         from dbt.task.clean import CleanTask
         from dbt.task.deps import DepsTask
-
-        from airflow_dbt_python.utils.version import DBT_INSTALLED_LESS_THAN_1_5
-
-        if DBT_INSTALLED_LESS_THAN_1_5:
-            from dbt.main import adapter_management, track_run  # type: ignore
-        else:
-            from dbt.adapters.factory import adapter_management
-            from dbt.tracking import track_run
+        from dbt.tracking import track_run
 
         config = self.get_dbt_task_config(command, **kwargs)
         extra_target = self.get_dbt_target_from_connection(config.target)
@@ -252,36 +243,24 @@ class DbtHook(BaseHook):
         ) as dbt_dir:
             # When creating tasks via from_args, dbt switches to the project directory.
             # We have to do that here as we are not using from_args.
-            if DBT_INSTALLED_LESS_THAN_1_5:
-                # For compatibility with older versions of dbt, as the signature
-                # of move_to_nearest_project_dir changed in dbt-core 1.5 to take
-                # just the project_dir.
-                nearest_project_dir = get_nearest_project_dir(config)  # type: ignore
-            else:
-                nearest_project_dir = get_nearest_project_dir(config.project_dir)
+            nearest_project_dir = get_nearest_project_dir(config.project_dir)
 
             with chdir_ctx(nearest_project_dir):
-                config.dbt_task.pre_init_hook(config)
                 self.ensure_profiles(config)
 
-                task, runtime_config = config.create_dbt_task(
-                    extra_target, write_perf_info
-                )
-                requires_profile = isinstance(task, (CleanTask, DepsTask))
-
-                self.setup_dbt_logging(task, config.debug)
+                with adapter_management():
+                    task, runtime_config = config.create_dbt_task(
+                        extra_target, write_perf_info
+                    )
+                    requires_profile = isinstance(task, (CleanTask, DepsTask))
 
-                if runtime_config is not None and not requires_profile:
-                    # The deps command installs the dependencies, which means they may
-                    # not exist before deps runs and the following would raise a
-                    # CompilationError.
-                    runtime_config.load_dependencies()
+                    self.setup_dbt_logging(task, config.debug)
 
-                results = None
-                with adapter_management():
-                    if not requires_profile:
-                        if runtime_config is not None:
-                            register_adapter(runtime_config)
+                    if runtime_config is not None and not requires_profile:
+                        # The deps command installs the dependencies, which means they
+                        # may not exist before deps runs and the following would raise a
+                        # CompilationError.
+                        runtime_config.load_dependencies()
 
                     with track_run(task):
                         results = task.run()
@@ -419,19 +398,11 @@ class DbtHook(BaseHook):
         default_stdout. As these are initialized by the CLI app, we need to
         initialize them here.
         """
-        from dbt.events.functions import setup_event_logger
-
-        log_path = None
-        if task.config is not None:
-            log_path = getattr(task.config, "log_path", None)
-
-        if DBT_INSTALLED_LESS_THAN_1_5:
-            setup_event_logger(log_path or "logs")
-        else:
-            from dbt.flags import get_flags
+        from dbt.events.logging import setup_event_logger
+        from dbt.flags import get_flags
 
-            flags = get_flags()
-            setup_event_logger(flags)
+        flags = get_flags()
+        setup_event_logger(flags)
 
         configured_file = logging.getLogger("configured_file")
         file_log = logging.getLogger("file_log")
@@ -458,7 +429,7 @@ class DbtHook(BaseHook):
         if not profiles_path.exists():
             profiles_path.parent.mkdir(exist_ok=True)
             with profiles_path.open("w", encoding="utf-8") as f:
-                f.write("config:\n  send_anonymous_usage_stats: false\n")
+                f.write("flags:\n  send_anonymous_usage_stats: false\n")
 
     def get_dbt_target_from_connection(
         self, target: Optional[str]

airflow_dbt_python/hooks/git.py

@@ -1,4 +1,5 @@
 """A concrete DbtRemoteHook for git repositories with dulwich."""
+
 import datetime as dt
 from typing import Callable, Optional, Tuple, Union
 
@@ -90,7 +91,7 @@ class DbtGitRemoteHook(SSHHook, DbtRemoteHook):
 
             repo.stage(str(f.relative_to(source)))
 
-        ts = dt.datetime.utcnow()
+        ts = dt.datetime.now(dt.timezone.utc)
         repo.do_commit(
             self.commit_msg.format(ts=ts).encode(), self.commit_author.encode()
         )

airflow_dbt_python/hooks/localfs.py

@@ -2,6 +2,7 @@
 
 Intended to be used only when running Airflow with a LocalExceutor.
 """
+
 from __future__ import annotations
 
 import shutil

airflow_dbt_python/hooks/remote.py

@@ -4,6 +4,7 @@ Internally, DbtRemoteHooks can use Airflow hooks to execute the actual operation
 
 Currently, only AWS S3 and the local filesystem are supported as remotes.
 """
+
 from abc import ABC, abstractmethod
 from pathlib import Path
 from typing import Optional, Type

airflow_dbt_python/hooks/s3.py

@@ -1,4 +1,5 @@
 """An implementation for an S3 remote for dbt."""
+
 from __future__ import annotations
 
 from typing import Iterable, Optional

airflow_dbt_python/operators/dbt.py

@@ -1,4 +1,5 @@
 """Airflow operators for all dbt commands."""
+
 from __future__ import annotations
 
 import datetime as dt
@@ -11,11 +12,13 @@ from airflow.exceptions import AirflowException
 from airflow.models.baseoperator import BaseOperator
 from airflow.models.xcom import XCOM_RETURN_KEY
 
+from airflow_dbt_python.utils.enums import LogFormat
+
 if TYPE_CHECKING:
     from dbt.contracts.results import RunResult
 
     from airflow_dbt_python.hooks.dbt import DbtHook, DbtTaskResult
-    from airflow_dbt_python.utils.enums import LogFormat, Output
+    from airflow_dbt_python.utils.enums import Output
 
 base_template_fields = [
     "project_dir",
@@ -73,7 +76,7 @@ class DbtBaseOperator(BaseOperator):
         log_path: Optional[str] = None,
         log_level: Optional[str] = None,
         log_level_file: Optional[str] = None,
-        log_format: Optional[LogFormat] = None,
+        log_format: LogFormat = LogFormat.DEFAULT,
         log_cache_events: Optional[bool] = False,
         quiet: Optional[bool] = None,
         no_print: Optional[bool] = None,
@@ -93,8 +96,8 @@ class DbtBaseOperator(BaseOperator):
         no_version_check: Optional[bool] = None,
         write_json: Optional[bool] = None,
         write_perf_info: Optional[bool] = None,
-        anonymous_usage_stats: Optional[bool] = None,
-        no_anonymous_usage_stats: Optional[bool] = None,
+        send_anonymous_usage_stats: Optional[bool] = None,
+        no_send_anonymous_usage_stats: Optional[bool] = None,
         # Extra features configuration
         dbt_conn_id: Optional[str] = "dbt_conn_id",
         profiles_conn_id: Optional[str] = None,
@@ -138,8 +141,8 @@ class DbtBaseOperator(BaseOperator):
         self.static_parser = static_parser
         self.no_static_parser = no_static_parser
 
-        self.anonymous_usage_stats = anonymous_usage_stats
-        self.no_anonymous_usage_stats = no_anonymous_usage_stats
+        self.send_anonymous_usage_stats = send_anonymous_usage_stats
+        self.no_send_anonymous_usage_stats = no_send_anonymous_usage_stats
 
         self.partial_parse = partial_parse
         self.no_partial_parse = no_partial_parse
@@ -671,13 +674,13 @@ def run_result_factory(data: list[tuple[Any, Any]]):
     We need to handle dt.datetime and agate.table.Table.
     The rest of the types should already be JSON-serializable.
     """
-    from dbt.contracts.results import agate
+    from agate.table import Table
 
     d = {}
     for key, val in data:
         if isinstance(val, dt.datetime):
             val = val.isoformat()
-        elif isinstance(val, agate.table.Table):
+        elif isinstance(val, Table):
             # agate Tables have a few print methods but they offer plain
             # text representations of the table which are not very JSON
             # friendly. There is a to_json method, but I don't think

airflow_dbt_python/utils/configs.py

@@ -1,7 +1,7 @@
 """A collection of configurations to interact with dbt-core."""
+
 from __future__ import annotations
 
-import argparse
 import dataclasses
 import json
 import os
@@ -11,23 +11,24 @@ from typing import Any, Optional, Type, Union
 
 import dbt.flags as flags
 import yaml
-from dbt.adapters.factory import register_adapter
+from dbt.cli.option_types import Package
 from dbt.clients import yaml_helper  # type: ignore
 from dbt.config.profile import Profile, read_profile
 from dbt.config.project import PartialProject, Project
 from dbt.config.renderer import DbtProjectYamlRenderer, ProfileRenderer
 from dbt.config.runtime import RuntimeConfig
 from dbt.contracts.graph.manifest import Manifest
+from dbt.exceptions import DbtProjectError
 from dbt.graph.graph import Graph
-from dbt.helper_types import WarnErrorOptions
+from dbt.parser.manifest import parse_manifest
 from dbt.task.base import BaseTask, ConfiguredTask
 from dbt.task.build import BuildTask
 from dbt.task.clean import CleanTask
 from dbt.task.compile import CompileTask
 from dbt.task.debug import DebugTask
 from dbt.task.deps import DepsTask
+from dbt.task.docs.generate import GenerateTask
 from dbt.task.freshness import FreshnessTask
-from dbt.task.generate import GenerateTask
 from dbt.task.list import ListTask
 from dbt.task.run import RunTask
 from dbt.task.run_operation import RunOperationTask
@@ -35,38 +36,10 @@ from dbt.task.runnable import GraphRunnableTask
 from dbt.task.seed import SeedTask
 from dbt.task.snapshot import SnapshotTask
 from dbt.task.test import TestTask
-from dbt.tracking import initialize_from_flags as orig_initialize_from_flags
+from dbt.tracking import initialize_from_flags
 
 from airflow_dbt_python.utils.enums import FromStrEnum, LogFormat, Output
-from airflow_dbt_python.utils.version import (
-    DBT_INSTALLED_GTE_1_5,
-    DBT_INSTALLED_GTE_1_7,
-    DBT_INSTALLED_LESS_THAN_1_5,
-)
-
-if DBT_INSTALLED_LESS_THAN_1_5:
-    # dbt 1.5 removed the ParseTask as it only generated a manifest.
-    # This task is now handled by a simple function.
-    # Airflow-dbt-python should still support a ParseTask that does the same,
-    # but this will require writing our own ParseTask class.
-    # In the meantime, we treat ParseTask as CompileTask (which also generates
-    # a manifest) if running with dbt-core >= 1.5.
-    from dbt.task.parse import ParseTask  # type: ignore
-
-    # dbt 1.5 also changed the signature of initialize_from_flags, so we have
-    # this for backwards compatibility.
-    def initialize_from_flags(send_anonymous_usage_stats, profiles_dir):
-        """Call initialize_from_flags without arguments.
-
-        Added for backwards compatibility with versions of dbt < 1.5.
-        """
-        return orig_initialize_from_flags()  # type: ignore
-
-else:
-    from dbt.parser.manifest import ManifestLoader
-
-    ParseTask = CompileTask
-    initialize_from_flags = orig_initialize_from_flags
+from airflow_dbt_python.utils.version import DBT_INSTALLED_GTE_1_9
 
 
 def parse_yaml_args(args: Optional[Union[str, dict[str, Any]]]) -> dict[str, Any]:
@@ -108,16 +81,16 @@ class BaseConfig:
     threads: Optional[int] = None
     use_experimental_parser: Optional[bool] = None
     store_failures: Optional[bool] = None
-    vars: Optional[Union[str, dict[str, Any]]] = None
+    vars: Union[dict[str, Any]] = dataclasses.field(default_factory=dict)
     macro_debugging: Optional[bool] = None
     warn_error: Optional[bool] = None
     warn_error_options: dict[str, Any] = dataclasses.field(default_factory=dict)
     indirect_selection: str = "eager"
 
     # Logging
-    log_format: Optional[LogFormat] = None
-    log_format_file: Optional[LogFormat] = None
-    log_path: Optional[str] = None if DBT_INSTALLED_LESS_THAN_1_5 else "logs"
+    log_format: LogFormat = LogFormat.DEFAULT
+    log_format_file: LogFormat = LogFormat.DEFAULT
+    log_path: Optional[str] = "logs"
     log_level: str = "info"
     log_level_file: str = "none"
     record_timing_info: Optional[str] = None
@@ -154,51 +127,38 @@ class BaseConfig:
     version_check: Optional[bool] = None
     no_version_check: Optional[bool] = dataclasses.field(default=None, repr=False)
 
-    # dbt < 1.5
-    anonymous_usage_stats: Optional[bool] = None
-    no_anonymous_usage_stats: Optional[bool] = dataclasses.field(
+    send_anonymous_usage_stats: Optional[bool] = None
+    no_send_anonymous_usage_stats: Optional[bool] = dataclasses.field(
         default=None, repr=False
     )
 
-    # dbt >= 1.5
-    if DBT_INSTALLED_GTE_1_5:
-        send_anonymous_usage_stats: Optional[bool] = None
-        no_send_anonymous_usage_stats: Optional[bool] = dataclasses.field(
-            default=None, repr=False
-        )
-
-        write_json: Optional[bool] = None
-        no_write_json: Optional[bool] = dataclasses.field(default=None, repr=False)
+    include_saved_query: Optional[bool] = None
+    no_include_saved_query: Optional[bool] = dataclasses.field(default=None, repr=False)
 
-    # dbt >= 1.7
-    if DBT_INSTALLED_GTE_1_7:
-        from dbt.cli.option_types import Package
+    clean_project_files_only: Optional[bool] = None
+    no_clean_project_files_only: Optional[bool] = dataclasses.field(
+        default=None, repr=False
+    )
 
-        include_saved_query: Optional[bool] = None
-        no_include_saved_query: Optional[bool] = dataclasses.field(
-            default=None, repr=False
-        )
+    require_resource_names_without_spaces: bool = False
 
-        clean_project_files_only: Optional[bool] = None
-        no_clean_project_files_only: Optional[bool] = dataclasses.field(
-            default=None, repr=False
-        )
+    add_package: Optional[Package] = None
+    dry_run: bool = False
+    lock: bool = False
+    static: bool = False
+    upgrade: bool = False
 
-        add_package: Optional[Package] = None
-        dry_run: bool = False
-        lock: bool = False
-        static: bool = False
-        upgrade: bool = False
+    require_model_names_without_spaces: bool = False
+    source_freshness_run_project_hooks: bool = False
+    exclude_resource_types: list[str] = dataclasses.field(
+        default_factory=list, repr=False
+    )
 
     def __post_init__(self):
         """Post initialization actions for a dbt configuration."""
-        self.parsed_vars = parse_yaml_args(self.vars)
-        self.vars = yaml.dump(self.parsed_vars)
+        self.vars = parse_yaml_args(self.vars)
         self.set_mutually_exclusive_attributes()
 
-        self.send_anonymous_usage_stats = self.anonymous_usage_stats
-        self.no_send_anonymous_usage_stats = self.no_anonymous_usage_stats
-
     def set_mutually_exclusive_attributes(self):
         """Support pairs of mutually exclusive parameters.
 
@@ -219,7 +179,6 @@ class BaseConfig:
             ("populate_cache", True),
             ("static_parser", True),
             ("version_check", True),
-            ("anonymous_usage_stats", True),
             ("send_anonymous_usage_stats", True),
             ("write_json", True),
             ("include_saved_query", None),
@@ -252,40 +211,6 @@ class BaseConfig:
             item = item.lower()
         return super().__getattribute__(item)
 
-    def to_flags(self) -> argparse.Namespace:
-        """Return a Namespace to use as dbt flags."""
-        flags_dict = vars(flags.get_flags())
-        flags_dict["MACRO_DEBUGGING"] = self.macro_debugging
-        flags_dict["WARN_ERROR"] = self.warn_error
-        flags_dict["WARN_ERROR_OPTIONS"] = WarnErrorOptions(
-            include=self.warn_error_options.get("include", []),
-            exclude=self.warn_error_options.get("exclude", []),
-        )
-
-        flags_dict["PRINTER_WIDTH"] = self.printer_width
-        flags_dict["TARGET_PATH"] = None
-        flags_dict["LOG_PATH"] = self.log_path
-        flags_dict["LOG_FORMAT"] = self.log_format
-        flags_dict["LOG_FORMAT_FILE"] = self.log_format
-        flags_dict["SEND_ANONYMOUS_USAGE_STATS"] = self.send_anonymous_usage_stats
-        flags_dict["PARTIAL_PARSE"] = self.partial_parse
-        flags_dict["STATIC_PARSER"] = self.static_parser
-        flags_dict["PROFILES_DIR"] = self.profiles_dir
-        flags_dict["STORE_FAILURES"] = self.store_failures
-        flags_dict["LOG_LEVEL"] = self.log_level
-        flags_dict["LOG_LEVEL_FILE"] = self.log_level_file
-        flags_dict["LOG_CACHE_EVENTS"] = self.log_cache_events
-        flags_dict["QUIET"] = self.quiet
-        flags_dict["DEBUG"] = self.debug
-        flags_dict["USE_COLORS_FILE"] = self.use_colors_file
-        flags_dict["WRITE_JSON"] = self.write_json
-        flags_dict["INDIRECT_SELECTION"] = getattr(self, "indirect_selection", None)
-        flags_dict["FAIL_FAST"] = self.fail_fast
-        flags_dict["INTROSPECT"] = self.introspect
-
-        flags_ns = argparse.Namespace(**flags_dict)
-        return flags_ns
-
     @property
     def dbt_task(self) -> Type[BaseTask]:
         """Access to the underlying dbt task class."""
@@ -376,30 +301,36 @@ class BaseConfig:
             A tuple with the corresponding subclass of a dbt BaseTask and the
             RuntimeConfig for the task.
         """
-        if not DBT_INSTALLED_LESS_THAN_1_5:
-            local_flags = self.to_flags()
-            flags.set_flags(local_flags)
-
+        flags.set_from_args(self, {})  # type: ignore
         project, profile = self.create_dbt_project_and_profile(extra_targets)
+
+        if not DBT_INSTALLED_GTE_1_9:
+            self.cls.set_log_format()  # type: ignore[attr-defined]
+
+        initialize_from_flags(self.send_anonymous_usage_stats, self.profiles_dir)
+
+        local_flags = flags.get_flags()
+
         runtime_config = self.create_runtime_config(project, profile)
 
-        if not DBT_INSTALLED_LESS_THAN_1_5:
-            if issubclass(self.dbt_task, ConfiguredTask) and runtime_config:
-                register_adapter(runtime_config)
-                manifest = ManifestLoader.get_full_manifest(
-                    runtime_config, write_perf_info=write_perf_info
-                )
-                task: BaseTask = self.dbt_task(
-                    args=self, config=runtime_config, manifest=manifest
-                )
-            elif issubclass(self.dbt_task, DepsTask):
-                task = self.dbt_task(args=self, project=project)
-            elif issubclass(self.dbt_task, DebugTask):
-                task = self.dbt_task(args=self, config=runtime_config)
-            else:
-                task = self.dbt_task(args=self, config=runtime_config, project=project)
+        if issubclass(self.dbt_task, ConfiguredTask) and runtime_config:
+            manifest = parse_manifest(
+                runtime_config,
+                write_perf_info=write_perf_info,
+                write=False,
+                write_json=False,
+            )
+            task: BaseTask = self.dbt_task(
+                args=local_flags, config=runtime_config, manifest=manifest
+            )
+        elif issubclass(self.dbt_task, DepsTask):
+            task = self.dbt_task(args=local_flags, project=project)
+        elif issubclass(self.dbt_task, DebugTask):
+            task = self.dbt_task(args=local_flags)
+        elif issubclass(self.dbt_task, CleanTask):
+            task = self.dbt_task(args=local_flags, config=project)
         else:
-            task = self.dbt_task(args=self, config=runtime_config)
+            task = self.dbt_task(args=local_flags)
 
         if self.compiled_target is not None:
             # Only supported by subclasses of dbt's GraphRunnableTask.
@@ -415,7 +346,6 @@ class BaseConfig:
 
         If the given task's ConfigType is not RuntimeConfig, None will be returned
         instead.
-        Flags are also initialized as configuration is created.
 
         Args:
             project: dbt project.
@@ -424,17 +354,13 @@ class BaseConfig:
         Returns:
             A RuntimeConfig instance.
         """
-        initialize_from_flags(self.send_anonymous_usage_stats, self.profiles_dir)
-        self.cls.set_log_format()
-        flags.set_from_args(self, profile.user_config)  # type: ignore
-
-        cfg_type = self.dbt_task.ConfigType
+        if isinstance(self.vars, str):
+            self.vars = yaml.safe_load(self.vars)
 
-        if issubclass(cfg_type, RuntimeConfig):
-            if not DBT_INSTALLED_LESS_THAN_1_5 and isinstance(self.vars, str):
-                self.vars = yaml.safe_load(self.vars)
-            return cfg_type.from_parts(project=project, profile=profile, args=self)
-        return None
+        try:
+            return RuntimeConfig.from_parts(project=project, profile=profile, args=self)
+        except DbtProjectError:
+            return None
 
     def create_dbt_project_and_profile(
         self, extra_targets: Optional[dict[str, Any]] = None
@@ -449,7 +375,7 @@ class BaseConfig:
         """
         profile = self.create_dbt_profile(extra_targets)
 
-        project_renderer = DbtProjectYamlRenderer(profile, self.parsed_vars)
+        project_renderer = DbtProjectYamlRenderer(profile, self.vars)
         project = Project.from_project_root(
             self.project_dir or ".",
             project_renderer,
@@ -480,7 +406,7 @@ class BaseConfig:
     @property
     def profile_renderer(self) -> ProfileRenderer:
         """Return a ProfileRenderer with this config's parsed vars."""
-        profile_renderer = ProfileRenderer(self.parsed_vars)
+        profile_renderer = ProfileRenderer(self.vars)
         return profile_renderer
 
     @property
@@ -513,6 +439,11 @@ class BaseConfig:
         Returns:
             A dbt profile for the task represented by this configuration.
         """
+        from dbt_common.clients.system import get_env
+        from dbt_common.context import set_invocation_context
+
+        set_invocation_context(get_env())
+
         if self.profiles_dir is not None:
             raw_profiles = read_profile(self.profiles_dir)
         else:
@@ -524,15 +455,12 @@ class BaseConfig:
             outputs.setdefault("target", self.target)
             profile["outputs"] = {**outputs, **extra_targets}
 
-        user_config = raw_profiles.get("config", {})
-
         profile = Profile.from_raw_profile_info(
             raw_profile=raw_profiles.get(
                 self.profile_name, {}
             ),  # Let dbt handle missing profile errors.
             profile_name=self.profile_name,
             renderer=self.profile_renderer,
-            user_config=user_config,
             target_override=self.target,
             threads_override=self.threads,
         )
@@ -668,7 +596,7 @@ class ParseTaskConfig(SelectionConfig):
     TODO: Temporarily, handle this as a CompileTask.
     """
 
-    cls: Type[BaseTask] = dataclasses.field(default=ParseTask, init=False)
+    cls: Type[BaseTask] = dataclasses.field(default=CompileTask, init=False)
     compile: Optional[bool] = None
     which: str = dataclasses.field(default="parse", init=False)
     write_manifest: Optional[bool] = None
@@ -694,8 +622,6 @@ class RunOperationTaskConfig(BaseConfig):
     def __post_init__(self):
         """Support dictionary args by casting them to str after setting."""
         super().__post_init__()
-        if DBT_INSTALLED_LESS_THAN_1_5:
-            self.args = str(self.args)
 
 
 @dataclasses.dataclass
@@ -734,6 +660,7 @@ class TestTaskConfig(SelectionConfig):
     singular: Optional[bool] = None
     store_failures: Optional[bool] = None
     which: str = dataclasses.field(default="test", init=False)
+    resource_types: Optional[list[str]] = None
 
     def __post_init__(self):
         """Support for type casting arguments."""

airflow_dbt_python/utils/enums.py

@@ -1,4 +1,5 @@
 """Enumeration subclasses used in dbt configurations."""
+
 from enum import Enum
 from typing import Type, TypeVar
 
@@ -24,6 +25,7 @@ class LogFormat(str, Enum):
     """Allowed dbt log formats."""
 
     DEFAULT = "default"
+    DEBUG = "debug"
     JSON = "json"
     TEXT = "text"
 

airflow_dbt_python/utils/env.py

@@ -1,4 +1,5 @@
 """Provides utilities to interact with environment variables."""
+
 import copy
 import os
 from contextlib import contextmanager

airflow_dbt_python/utils/url.py

@@ -1,4 +1,5 @@
 """A URL class to locate dbt resources."""
+
 from __future__ import annotations
 
 import os
@@ -464,7 +465,7 @@ def tar_dir_from_url(url: URL, tar_url: URL) -> None:
     except IndexError:
         compression = ""
 
-    with tarfile.open(tar_url, f"w:{compression}") as tf:
+    with tarfile.open(tar_url, f"w:{compression}") as tf:  # type: ignore[call-overload]
         for _file in url:
             tf.add(_file)
 
@@ -472,6 +473,6 @@ def tar_dir_from_url(url: URL, tar_url: URL) -> None:
 if __name__ == "__main__":
     import doctest
 
-    import airflow_dbt_python.utils.url as url
+    import airflow_dbt_python.utils.url as url_utils
 
-    doctest.testmod(url)
+    doctest.testmod(url_utils)

airflow_dbt_python/utils/version.py

@@ -3,28 +3,24 @@
 These are only used to ensure backwards compatibility with older versions of dbt.
 """
 
-from dbt.semver import Matchers, VersionSpecifier
+try:
+    from dbt.semver import Matchers, VersionSpecifier
+except ImportError:
+    from dbt_common.semver import Matchers, VersionSpecifier
+
 from dbt.version import installed
 
-DBT_1_5 = VersionSpecifier(
-    major="1", minor="5", patch="0", matcher=Matchers.GREATER_THAN_OR_EQUAL
-)
-DBT_1_6 = VersionSpecifier(
-    major="1", minor="6", patch="0", matcher=Matchers.GREATER_THAN_OR_EQUAL
-)
-DBT_1_7 = VersionSpecifier(
-    major="1", minor="7", patch="0", matcher=Matchers.GREATER_THAN_OR_EQUAL
-)
 DBT_1_8 = VersionSpecifier(
     major="1", minor="8", patch="0", matcher=Matchers.GREATER_THAN_OR_EQUAL
 )
+DBT_1_9 = VersionSpecifier(
+    major="1", minor="9", patch="0", matcher=Matchers.GREATER_THAN_OR_EQUAL
+)
+DBT_2_0 = VersionSpecifier(
+    major="2", minor="0", patch="0", matcher=Matchers.GREATER_THAN_OR_EQUAL
+)
 
-DBT_INSTALLED_LESS_THAN_1_5 = installed < DBT_1_5
-
-DBT_INSTALLED_GTE_1_5 = installed.compare(DBT_1_5) == 1
-DBT_INSTALLED_GTE_1_6 = installed.compare(DBT_1_6) == 1
-DBT_INSTALLED_GTE_1_7 = installed.compare(DBT_1_7) == 1
+DBT_INSTALLED_GTE_1_9 = installed.compare(DBT_1_9) == 1
 
-DBT_INSTALLED_1_5 = DBT_1_5 < installed < DBT_1_6
-DBT_INSTALLED_1_6 = DBT_1_6 < installed < DBT_1_7
-DBT_INSTALLED_1_7 = DBT_1_7 < installed < DBT_1_8
+DBT_INSTALLED_1_8 = DBT_1_8 < installed < DBT_1_9
+DBT_INSTALLED_1_9 = DBT_1_9 < installed < DBT_2_0

{airflow_dbt_python-2.0.1.dist-info → airflow_dbt_python-2.2.0.dist-info}/METADATA

@@ -1,24 +1,19 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: airflow-dbt-python
-Version: 2.0.1
+Version: 2.2.0
 Summary: A collection of Airflow operators, hooks, and utilities to execute dbt commands
-Home-page: https://github.com/tomasfarias/airflow-dbt-python
 License: MIT
 Author: Tomás Farías Santana
 Author-email: tomas@tomasfarias.dev
-Requires-Python: >=3.8,<3.12
+Requires-Python: >=3.9,<3.13
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.8
-Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.12
 Provides-Extra: adapters
 Provides-Extra: airflow-providers
 Provides-Extra: bigquery
@@ -29,23 +24,17 @@ Provides-Extra: redshift
 Provides-Extra: s3
 Provides-Extra: snowflake
 Provides-Extra: spark
-Requires-Dist: apache-airflow (>=2.2)
+Requires-Dist: apache-airflow (>=2.8)
 Requires-Dist: apache-airflow-providers-amazon (>=3.0.0) ; extra == "airflow-providers" or extra == "s3"
 Requires-Dist: apache-airflow-providers-ssh (>=3.0.0) ; extra == "airflow-providers" or extra == "git"
 Requires-Dist: connexion (<3.0)
 Requires-Dist: contextlib-chdir (==1.0.2) ; python_version < "3.11"
-Requires-Dist: dbt-bigquery (>=1.4.0,<1.6.0) ; (python_version < "3.8") and (extra == "adapters" or extra == "bigquery")
-Requires-Dist: dbt-bigquery (>=1.4.0,<1.8.0) ; (python_version >= "3.11") and (extra == "adapters" or extra == "bigquery")
-Requires-Dist: dbt-core (>=1.4.0,<1.6.0) ; python_version < "3.8"
-Requires-Dist: dbt-core (>=1.4.0,<1.8.0) ; python_version >= "3.11"
-Requires-Dist: dbt-postgres (>=1.4.0,<1.6.0) ; (python_version < "3.8") and (extra == "adapters" or extra == "postgres")
-Requires-Dist: dbt-postgres (>=1.4.0,<1.8.0) ; (python_version >= "3.11") and (extra == "adapters" or extra == "postgres")
-Requires-Dist: dbt-redshift (>=1.4.0,<1.6.0) ; (python_version < "3.8") and (extra == "adapters" or extra == "redshift")
-Requires-Dist: dbt-redshift (>=1.4.0,<1.8.0) ; (python_version >= "3.11") and (extra == "adapters" or extra == "redshift")
-Requires-Dist: dbt-snowflake (>=1.4.0,<1.6.0) ; (python_version < "3.8") and (extra == "adapters" or extra == "snowflake")
-Requires-Dist: dbt-snowflake (>=1.4.0,<1.8.0) ; (python_version >= "3.11") and (extra == "adapters" or extra == "snowflake")
-Requires-Dist: dbt-spark (>=1.4.0,<1.6.0) ; (python_version < "3.8") and (extra == "adapters" or extra == "spark")
-Requires-Dist: dbt-spark (>=1.4.0,<1.8.0) ; (python_version >= "3.11") and (extra == "adapters" or extra == "spark")
+Requires-Dist: dbt-bigquery (>=1.8.0,<2.0.0) ; extra == "adapters" or extra == "bigquery"
+Requires-Dist: dbt-core (>=1.8.0,<2.0.0)
+Requires-Dist: dbt-postgres (>=1.8.0,<2.0.0) ; extra == "adapters" or extra == "postgres"
+Requires-Dist: dbt-redshift (>=1.8.0,<2.0.0) ; extra == "adapters" or extra == "redshift"
+Requires-Dist: dbt-snowflake (>=1.8.0,<2.0.0) ; extra == "adapters" or extra == "snowflake"
+Requires-Dist: dbt-spark (>=1.8.0,<2.0.0) ; extra == "adapters" or extra == "spark"
 Requires-Dist: dulwich (>=0.21) ; extra == "git"
 Project-URL: Repository, https://github.com/tomasfarias/airflow-dbt-python
 Description-Content-Type: text/markdown
@@ -67,10 +56,10 @@ Read the [documentation](https://airflow-dbt-python.readthedocs.io) for examples
 ## Requirements
 
 Before using *airflow-dbt-python*, ensure you meet the following requirements:
-* A *dbt* project using [dbt-core](https://pypi.org/project/dbt-core/) version 1.4.0 or later.
-* An Airflow environment using version 2.2 or later.
+* A *dbt* project using [dbt-core](https://pypi.org/project/dbt-core/) version 1.8 or later.
+* An Airflow environment using version 2.8 or later.
 
-  * If using any managed service, like AWS MWAA, ensure your environment is created with a supported version of Airflow.
+  * If using any managed service, like AWS MWAA or GCP Cloud Composer 2/3, ensure your environment is created with a supported version of Airflow.
   * If self-hosting, Airflow installation instructions can be found in their [official documentation](https://airflow.apache.org/docs/apache-airflow/stable/installation/index.html).
 
 * Running Python 3.8 or later in your Airflow environment.
@@ -81,7 +70,7 @@ Before using *airflow-dbt-python*, ensure you meet the following requirements:
 
 > **Note**
 >
-> Older versions of Airflow and *dbt* may work with *airflow-dbt-python*, although we cannot guarantee this. Our testing pipeline runs the latest *dbt-core* with the latest Airflow release, and the latest version supported by [AWS MWAA](https://aws.amazon.com/managed-workflows-for-apache-airflow/).
+> Older versions of Airflow and *dbt* may work with *airflow-dbt-python*, although we cannot guarantee this. Our testing pipeline runs the latest *dbt-core* with the latest Airflow release, and the latest version supported by [AWS MWAA](https://aws.amazon.com/managed-workflows-for-apache-airflow/) and [GCP Cloud Composer 2/3](https://aws.amazon.com/managed-workflows-for-apache-airflow/).
 
 ## From PyPI
 
@@ -118,6 +107,12 @@ Add *airflow-dbt-python* to your `requirements.txt` file and edit your Airflow e
 
 Read the [documentation](https://airflow-dbt-python.readthedocs.io/en/latest/getting_started.html#installing-in-mwaa) for more a more detailed AWS MWAA installation breakdown.
 
+## In GCP Cloud Composer
+
+Add *airflow-dbt-python* to your PyPI packages list.
+
+Read the [documentation](https://cloud.google.com/composer/docs/composer-2/install-python-dependencies#install-pypi) for more a more detailed GCP Cloud Composer 2 installation breakdown.
+
 ## In other managed services
 
 *airflow-dbt-python* should be compatible with most or all Airflow managed services. Consult the documentation specific to your provider.
@@ -171,7 +166,7 @@ See an example DAG [here](examples/airflow_connection_target_dag.py).
 
 Although [`dbt`](https://docs.getdbt.com/) is meant to be installed and used as a CLI, we may not have control of the environment where Airflow is running, disallowing us the option of using *dbt* as a CLI.
 
-This is exactly what happens when using [Amazon's Managed Workflows for Apache Airflow](https://aws.amazon.com/managed-workflows-for-apache-airflow/) or MWAA: although a list of Python requirements can be passed, the CLI cannot be found in the worker's PATH.
+This is exactly what happens when using [Amazon's Managed Workflows for Apache Airflow](https://aws.amazon.com/managed-workflows-for-apache-airflow/) (aka MWAA): although a list of Python requirements can be passed, the CLI cannot be found in the worker's PATH.
 
 There is a workaround which involves using Airflow's `BashOperator` and running Python from the command line:
 

airflow_dbt_python-2.2.0.dist-info/RECORD

@@ -0,0 +1,20 @@
+airflow_dbt_python/__init__.py,sha256=qYIsTHy4p3NX0QYIsLdatz6GlnaMwYsSoD3wtPD0BqY,153
+airflow_dbt_python/__version__.py,sha256=R3gvi55VePWbBsovUbmgfU1cEWnzuFVlZFXx74rtufo,190
+airflow_dbt_python/hooks/__init__.py,sha256=dcQQ5e8EpcQ_CzKVOeg_cRbqOEmloNLnWSp3B0bDSXs,57
+airflow_dbt_python/hooks/dbt.py,sha256=_M-WnwAgUtlpWP2hIa4g9ZlCy5QFXSYQOL_S5ARucYk,20001
+airflow_dbt_python/hooks/git.py,sha256=bxzyd1g9WPfZs8ukj-FSafrVki4hyw_GF06jF61ncxk,7126
+airflow_dbt_python/hooks/localfs.py,sha256=i49VldNRFyvsd8bMdpkWaaviP2VoqMnl-VmHYgrpfZM,5387
+airflow_dbt_python/hooks/remote.py,sha256=Wx6zP9DH6blIiremwicxgYFxssQ-5Na9UaHyZjcNJSI,5478
+airflow_dbt_python/hooks/s3.py,sha256=AUYtpI609fBzZXCY7esJbdFhijHoMVJWa6lERPFuAro,7205
+airflow_dbt_python/operators/__init__.py,sha256=zveHW17YWoCpdsQhCZtvdPoEiFiwWykqPnAnOAQ8vk0,46
+airflow_dbt_python/operators/dbt.py,sha256=6WF6yxjNJZI5HgVLA5eEnoBK7V8Jz2MZcfgEmAXRLg8,22098
+airflow_dbt_python/utils/__init__.py,sha256=J3_g-tV35XkrZSwYULsAq4B0TUa6rw0uxP-LFwl_k2Q,45
+airflow_dbt_python/utils/configs.py,sha256=BfYyws28MnMYvqgN9YcnrYWXa_x1cs9fmUknbmGFijI,25322
+airflow_dbt_python/utils/enums.py,sha256=rktUM8cTkEUsdNbzd_f6fxfgNiZGifJfOr0zvrsMyuQ,969
+airflow_dbt_python/utils/env.py,sha256=o2Y3RMyXADSpUCU-G0yRiqXB-yo3oPsXtCEI7H5GIyM,646
+airflow_dbt_python/utils/url.py,sha256=Ac7PJMl3AsrVMXGPgFdzOrAVsvCfS4hMWAtwgRMdbP0,16111
+airflow_dbt_python/utils/version.py,sha256=RAMagzZofIAiFUFc9g06yZXM81BaMI3hH58q4xjfE0M,811
+airflow_dbt_python-2.2.0.dist-info/LICENSE,sha256=8yx5JCe-4cW52JiqVrMlA990r7tuac2foiUryhV9dfY,1073
+airflow_dbt_python-2.2.0.dist-info/METADATA,sha256=Lfx7ptaVOY1ABFbIZe04Ru-ODdZMQf3ie8ct_Hx1LvA,15544
+airflow_dbt_python-2.2.0.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
+airflow_dbt_python-2.2.0.dist-info/RECORD,,

{airflow_dbt_python-2.0.1.dist-info → airflow_dbt_python-2.2.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 1.4.0
+Generator: poetry-core 2.0.1
 Root-Is-Purelib: true
 Tag: py3-none-any

airflow_dbt_python-2.0.1.dist-info/RECORD

@@ -1,20 +0,0 @@
-airflow_dbt_python/__init__.py,sha256=l_rEXiRH5IyzQ5xExdPwcR-HMqBJFqlnX0GSZpDpFJM,152
-airflow_dbt_python/__version__.py,sha256=afYljsyh04NUgREwaXVXz71v3JWcvWkGVewgoy4JW-0,189
-airflow_dbt_python/hooks/__init__.py,sha256=dcQQ5e8EpcQ_CzKVOeg_cRbqOEmloNLnWSp3B0bDSXs,57
-airflow_dbt_python/hooks/dbt.py,sha256=fu6T1CTN1Sur69BZbPBvvkxx3qsCJkm3vZD9C8LsAVk,21162
-airflow_dbt_python/hooks/git.py,sha256=u35yQM8FEfv0zfP37bPkN8rO2kp_GdCzQqBg9-pR0iY,7113
-airflow_dbt_python/hooks/localfs.py,sha256=TKVh6kuFXiFo-GCNNfZwwtQ8Te99l1hpOnEYStohtWc,5386
-airflow_dbt_python/hooks/remote.py,sha256=nDGeGXUl7vNs2uhulcrtxHXabWYzuzJVwOTp94-XsZ0,5477
-airflow_dbt_python/hooks/s3.py,sha256=LwLOOORg8-Ts3pert0MF6G8VFLkABsB_3m3AZNcVGrI,7204
-airflow_dbt_python/operators/__init__.py,sha256=zveHW17YWoCpdsQhCZtvdPoEiFiwWykqPnAnOAQ8vk0,46
-airflow_dbt_python/operators/dbt.py,sha256=c9U-lMwM44NeUZUzpdRXyN2pNWlx0A0DnjJBupXQ1yk,22043
-airflow_dbt_python/utils/__init__.py,sha256=J3_g-tV35XkrZSwYULsAq4B0TUa6rw0uxP-LFwl_k2Q,45
-airflow_dbt_python/utils/configs.py,sha256=leb7-feu3bGFDi4vS0eurgoAKoa_xOUVMTDxo-yuyEk,28805
-airflow_dbt_python/utils/enums.py,sha256=1pWwJe5MUI9eebbDufL44ILj_7sJ7hJA0v2MBCrnPok,948
-airflow_dbt_python/utils/env.py,sha256=4hY12-C44D5cq-6Vqxf68Cv9KERVRIloLgtSeN5WHw0,645
-airflow_dbt_python/utils/url.py,sha256=euuCJey7QoaBJATgB63uxGR5yGBbw5_eMnoqPRKvudA,16067
-airflow_dbt_python/utils/version.py,sha256=BO5jCI-aQb9UdZ6txUniawNu-ZUnj4F2DUO8oXBuu0M,1039
-airflow_dbt_python-2.0.1.dist-info/METADATA,sha256=fvAiknzyzp5rH3TAhLqOqtSnEuS0qxykyfT8vBHTSLU,16279
-airflow_dbt_python-2.0.1.dist-info/LICENSE,sha256=8yx5JCe-4cW52JiqVrMlA990r7tuac2foiUryhV9dfY,1073
-airflow_dbt_python-2.0.1.dist-info/WHEEL,sha256=vVCvjcmxuUltf8cYhJ0sJMRDLr1XsPuxEId8YDzbyCY,88
-airflow_dbt_python-2.0.1.dist-info/RECORD,,