airflow-dbt-python 1.0.4__tar.gz → 2.0.0__tar.gz

{airflow_dbt_python-1.0.4 → airflow_dbt_python-2.0.0}/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.1
 Name: airflow-dbt-python
-Version: 1.0.4
+Version: 2.0.0
 Summary: A collection of Airflow operators, hooks, and utilities to execute dbt commands
 Home-page: https://github.com/tomasfarias/airflow-dbt-python
 License: MIT
 Author: Tomás Farías Santana
 Author-email: tomas@tomasfarias.dev
-Requires-Python: >=3.7.2,<4.0
+Requires-Python: >=3.8,<3.12
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
@@ -17,11 +17,9 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Provides-Extra: adapters
-Provides-Extra: airflow
 Provides-Extra: airflow-providers
 Provides-Extra: bigquery
 Provides-Extra: docs
@@ -31,16 +29,24 @@ Provides-Extra: redshift
 Provides-Extra: s3
 Provides-Extra: snowflake
 Provides-Extra: spark
-Requires-Dist: apache-airflow (>=2.0) ; extra == "airflow"
+Requires-Dist: apache-airflow (>=2.2)
 Requires-Dist: apache-airflow-providers-amazon (>=3.0.0) ; extra == "airflow-providers" or extra == "s3"
 Requires-Dist: apache-airflow-providers-ssh (>=3.0.0) ; extra == "airflow-providers" or extra == "git"
-Requires-Dist: dbt-bigquery (>=1.0.0) ; extra == "adapters" or extra == "bigquery"
-Requires-Dist: dbt-core (>=1.0.0)
-Requires-Dist: dbt-postgres (>=1.0.0) ; extra == "adapters" or extra == "postgres"
-Requires-Dist: dbt-redshift (>=1.0.0) ; extra == "adapters" or extra == "redshift"
-Requires-Dist: dbt-snowflake (>=1.0.0) ; extra == "adapters" or extra == "snowflake"
-Requires-Dist: dbt-spark (>=1.0.0) ; extra == "adapters" or extra == "spark"
-Requires-Dist: dulwich (>=0.20) ; extra == "git"
+Requires-Dist: connexion (<3.0)
+Requires-Dist: contextlib-chdir (==1.0.2) ; python_version < "3.11"
+Requires-Dist: dbt-bigquery (>=1.4.0,<1.6.0) ; (python_version < "3.8") and (extra == "adapters" or extra == "bigquery")
+Requires-Dist: dbt-bigquery (>=1.4.0,<1.8.0) ; (python_version >= "3.11") and (extra == "adapters" or extra == "bigquery")
+Requires-Dist: dbt-core (>=1.4.0,<1.6.0) ; python_version < "3.8"
+Requires-Dist: dbt-core (>=1.4.0,<1.8.0) ; python_version >= "3.11"
+Requires-Dist: dbt-postgres (>=1.4.0,<1.6.0) ; (python_version < "3.8") and (extra == "adapters" or extra == "postgres")
+Requires-Dist: dbt-postgres (>=1.4.0,<1.8.0) ; (python_version >= "3.11") and (extra == "adapters" or extra == "postgres")
+Requires-Dist: dbt-redshift (>=1.4.0,<1.6.0) ; (python_version < "3.8") and (extra == "adapters" or extra == "redshift")
+Requires-Dist: dbt-redshift (>=1.4.0,<1.8.0) ; (python_version >= "3.11") and (extra == "adapters" or extra == "redshift")
+Requires-Dist: dbt-snowflake (>=1.4.0,<1.6.0) ; (python_version < "3.8") and (extra == "adapters" or extra == "snowflake")
+Requires-Dist: dbt-snowflake (>=1.4.0,<1.8.0) ; (python_version >= "3.11") and (extra == "adapters" or extra == "snowflake")
+Requires-Dist: dbt-spark (>=1.4.0,<1.6.0) ; (python_version < "3.8") and (extra == "adapters" or extra == "spark")
+Requires-Dist: dbt-spark (>=1.4.0,<1.8.0) ; (python_version >= "3.11") and (extra == "adapters" or extra == "spark")
+Requires-Dist: dulwich (>=0.21) ; extra == "git"
 Project-URL: Repository, https://github.com/tomasfarias/airflow-dbt-python
 Description-Content-Type: text/markdown
 
@@ -61,13 +67,13 @@ Read the [documentation](https://airflow-dbt-python.readthedocs.io) for examples
 ## Requirements
 
 Before using *airflow-dbt-python*, ensure you meet the following requirements:
-* A *dbt* project using [dbt-core](https://pypi.org/project/dbt-core/) version 1.0.0 or later.
+* A *dbt* project using [dbt-core](https://pypi.org/project/dbt-core/) version 1.4.0 or later.
 * An Airflow environment using version 2.2 or later.
 
   * If using any managed service, like AWS MWAA, ensure your environment is created with a supported version of Airflow.
   * If self-hosting, Airflow installation instructions can be found in their [official documentation](https://airflow.apache.org/docs/apache-airflow/stable/installation/index.html).
 
-* Running Python 3.7 or later in your Airflow environment.
+* Running Python 3.8 or later in your Airflow environment.
 
 > **Warning**
 >

{airflow_dbt_python-1.0.4 → airflow_dbt_python-2.0.0}/README.md

@@ -15,13 +15,13 @@ Read the [documentation](https://airflow-dbt-python.readthedocs.io) for examples
 ## Requirements
 
 Before using *airflow-dbt-python*, ensure you meet the following requirements:
-* A *dbt* project using [dbt-core](https://pypi.org/project/dbt-core/) version 1.0.0 or later.
+* A *dbt* project using [dbt-core](https://pypi.org/project/dbt-core/) version 1.4.0 or later.
 * An Airflow environment using version 2.2 or later.
 
   * If using any managed service, like AWS MWAA, ensure your environment is created with a supported version of Airflow.
   * If self-hosting, Airflow installation instructions can be found in their [official documentation](https://airflow.apache.org/docs/apache-airflow/stable/installation/index.html).
 
-* Running Python 3.7 or later in your Airflow environment.
+* Running Python 3.8 or later in your Airflow environment.
 
 > **Warning**
 >

{airflow_dbt_python-1.0.4 → airflow_dbt_python-2.0.0}/airflow_dbt_python/__version__.py

@@ -2,4 +2,4 @@
 __author__ = "Tomás Farías Santana"
 __copyright__ = "Copyright 2021 Tomás Farías Santana"
 __title__ = "airflow-dbt-python"
-__version__ = "1.0.4"
+__version__ = "2.0.0"

{airflow_dbt_python-1.0.4 → airflow_dbt_python-2.0.0}/airflow_dbt_python/hooks/dbt.py

@@ -24,6 +24,14 @@ from airflow.exceptions import AirflowException
 from airflow.hooks.base import BaseHook
 from airflow.models.connection import Connection
 
+from airflow_dbt_python.utils.version import DBT_INSTALLED_LESS_THAN_1_5
+
+if sys.version_info >= (3, 11):
+    from contextlib import chdir as chdir_ctx
+else:
+    from contextlib_chdir import chdir as chdir_ctx
+
+
 if TYPE_CHECKING:
     from dbt.contracts.results import RunResult
     from dbt.task.base import BaseTask
@@ -208,6 +216,7 @@ class DbtHook(BaseHook):
         replace_on_upload: bool = False,
         artifacts: Optional[Iterable[str]] = None,
         env_vars: Optional[Dict[str, Any]] = None,
+        write_perf_info: bool = False,
         **kwargs,
     ) -> DbtTaskResult:
         """Run a dbt task with a given configuration and return the results.
@@ -219,9 +228,17 @@ class DbtHook(BaseHook):
                 of running the dbt command.
         """
         from dbt.adapters.factory import register_adapter
-        from dbt.config.runtime import UnsetProfileConfig
-        from dbt.main import adapter_management, track_run
-        from dbt.task.base import move_to_nearest_project_dir
+        from dbt.task.base import get_nearest_project_dir
+        from dbt.task.clean import CleanTask
+        from dbt.task.deps import DepsTask
+
+        from airflow_dbt_python.utils.version import DBT_INSTALLED_LESS_THAN_1_5
+
+        if DBT_INSTALLED_LESS_THAN_1_5:
+            from dbt.main import adapter_management, track_run  # type: ignore
+        else:
+            from dbt.adapters.factory import adapter_management
+            from dbt.tracking import track_run
 
         config = self.get_dbt_task_config(command, **kwargs)
         extra_target = self.get_dbt_target_from_connection(config.target)
@@ -233,53 +250,62 @@ class DbtHook(BaseHook):
             replace_on_upload=replace_on_upload,
             env_vars=env_vars,
         ) as dbt_dir:
-            config.dbt_task.pre_init_hook(config)
-            self.ensure_profiles(config.profiles_dir)
-
-            task, runtime_config = config.create_dbt_task(extra_target)
-
             # When creating tasks via from_args, dbt switches to the project directory.
             # We have to do that here as we are not using from_args.
-            move_to_nearest_project_dir(config)
+            if DBT_INSTALLED_LESS_THAN_1_5:
+                # For compatibility with older versions of dbt, as the signature
+                # of move_to_nearest_project_dir changed in dbt-core 1.5 to take
+                # just the project_dir.
+                nearest_project_dir = get_nearest_project_dir(config)  # type: ignore
+            else:
+                nearest_project_dir = get_nearest_project_dir(config.project_dir)
+
+            with chdir_ctx(nearest_project_dir):
+                config.dbt_task.pre_init_hook(config)
+                self.ensure_profiles(config)
 
-            self.setup_dbt_logging(task, config.debug)
+                task, runtime_config = config.create_dbt_task(
+                    extra_target, write_perf_info
+                )
+                requires_profile = isinstance(task, (CleanTask, DepsTask))
 
-            if not isinstance(runtime_config, UnsetProfileConfig):
-                if runtime_config is not None:
+                self.setup_dbt_logging(task, config.debug)
+
+                if runtime_config is not None and not requires_profile:
                     # The deps command installs the dependencies, which means they may
                     # not exist before deps runs and the following would raise a
                     # CompilationError.
                     runtime_config.load_dependencies()
 
-            results = None
-            with adapter_management():
-                if not isinstance(runtime_config, UnsetProfileConfig):
-                    if runtime_config is not None:
-                        register_adapter(runtime_config)
+                results = None
+                with adapter_management():
+                    if not requires_profile:
+                        if runtime_config is not None:
+                            register_adapter(runtime_config)
 
-                with track_run(task):
-                    results = task.run()
-                success = task.interpret_results(results)
+                    with track_run(task):
+                        results = task.run()
+                    success = task.interpret_results(results)
 
-            if artifacts is None:
-                return DbtTaskResult(success, results, {})
+                if artifacts is None:
+                    return DbtTaskResult(success, results, {})
 
-            saved_artifacts = {}
-            for artifact in artifacts:
-                artifact_path = Path(dbt_dir) / "target" / artifact
+                saved_artifacts = {}
+                for artifact in artifacts:
+                    artifact_path = Path(dbt_dir) / "target" / artifact
 
-                if not artifact_path.exists():
-                    self.log.warn(
-                        "Required dbt artifact %s was not found. "
-                        "Perhaps dbt failed and couldn't generate it.",
-                        artifact,
-                    )
-                    continue
+                    if not artifact_path.exists():
+                        self.log.warning(
+                            "Required dbt artifact %s was not found. "
+                            "Perhaps dbt failed and couldn't generate it.",
+                            artifact,
+                        )
+                        continue
 
-                with open(artifact_path) as artifact_file:
-                    json_artifact = json.load(artifact_file)
+                    with open(artifact_path) as artifact_file:
+                        json_artifact = json.load(artifact_file)
 
-                saved_artifacts[artifact] = json_artifact
+                    saved_artifacts[artifact] = json_artifact
 
         return DbtTaskResult(success, results, saved_artifacts)
 
@@ -373,7 +399,7 @@ class DbtHook(BaseHook):
         if (project_dir_path / "profiles.yml").exists():
             # We may have downloaded the profiles.yml file together
             # with the project.
-            return (new_project_dir, new_project_dir)
+            return new_project_dir, new_project_dir
 
         if profiles_dir is not None:
             profiles_file_path = self.download_dbt_profiles(
@@ -384,7 +410,7 @@ class DbtHook(BaseHook):
         else:
             new_profiles_dir = None
 
-        return (new_project_dir, new_profiles_dir)
+        return new_project_dir, new_profiles_dir
 
     def setup_dbt_logging(self, task: BaseTask, debug: Optional[bool]):
         """Setup dbt logging.
@@ -399,10 +425,18 @@ class DbtHook(BaseHook):
         if task.config is not None:
             log_path = getattr(task.config, "log_path", None)
 
-        setup_event_logger(log_path or "logs")
+        if DBT_INSTALLED_LESS_THAN_1_5:
+            setup_event_logger(log_path or "logs")
+        else:
+            from dbt.flags import get_flags
+
+            flags = get_flags()
+            setup_event_logger(flags)
 
         configured_file = logging.getLogger("configured_file")
         file_log = logging.getLogger("file_log")
+        stdout_log = logging.getLogger("stdout_log")
+        stdout_log.propagate = True
 
         if not debug:
             # We have to do this after setting logs up as dbt hasn't
@@ -413,16 +447,18 @@ class DbtHook(BaseHook):
             configured_file.setLevel("INFO")
             configured_file.propagate = False
 
-    def ensure_profiles(self, profiles_dir: Optional[str]):
+    def ensure_profiles(self, config: BaseConfig):
         """Ensure a profiles file exists."""
-        if profiles_dir is not None:
-            # We expect one to exist given that we have passsed a profiles_dir.
+        if config.profiles_dir is not None:
+            # We expect one to exist given that we have passed a profiles_dir.
             return
 
         profiles_path = Path.home() / ".dbt/profiles.yml"
+        config.profiles_dir = str(profiles_path.parent)
         if not profiles_path.exists():
             profiles_path.parent.mkdir(exist_ok=True)
-            profiles_path.touch()
+            with profiles_path.open("w", encoding="utf-8") as f:
+                f.write("config:\n  send_anonymous_usage_stats: false\n")
 
     def get_dbt_target_from_connection(
         self, target: Optional[str]

{airflow_dbt_python-1.0.4 → airflow_dbt_python-2.0.0}/airflow_dbt_python/hooks/git.py

@@ -187,20 +187,26 @@ class DbtGitRemoteHook(SSHHook, DbtRemoteHook):
             path = f"{url.netloc.split(':')[1]}/{str(url.path)}"
 
         elif url.scheme in ("http", "https"):
-            parsedurl = url._parsed
-            netloc = parsedurl.hostname
+            base_url = url.hostname
 
-            if parsedurl.port:
-                netloc = "{}:{}".format(netloc, parsedurl.port)
-            if parsedurl.username:
-                netloc = "{}@{}".format(parsedurl.username, netloc)
+            if url.port:
+                base_url = f"{base_url}:{url.port}"
 
-            url._replace(netloc=netloc)
+            auth_params = {}
+            if url.authentication.username and url.authentication.password:
+                auth_params = {
+                    "username": url.authentication.username,
+                    "password": url.authentication.password,
+                }
+                base_url = f"{url.scheme}://{base_url}"
+            elif url.authentication.username:
+                base_url = f"{url.scheme}://{url.authentication.username}@{base_url}"
+
+            client = HttpGitClient(base_url, **auth_params)
 
-            client = HttpGitClient(url.netloc)
             path = str(url.path)
 
         else:
             raise ValueError(f"Unsupported scheme: {url.scheme}")
 
-        return (client, path)
+        return client, path

{airflow_dbt_python-1.0.4 → airflow_dbt_python-2.0.0}/airflow_dbt_python/operators/dbt.py

@@ -70,6 +70,9 @@ class DbtBaseOperator(BaseOperator):
         warn_error: Optional[bool] = None,
         # Logging
         debug: Optional[bool] = None,
+        log_path: Optional[str] = None,
+        log_level: Optional[str] = None,
+        log_level_file: Optional[str] = None,
         log_format: Optional[LogFormat] = None,
         log_cache_events: Optional[bool] = False,
         quiet: Optional[bool] = None,
@@ -80,12 +83,16 @@ class DbtBaseOperator(BaseOperator):
         no_defer: Optional[bool] = None,
         partial_parse: Optional[bool] = False,
         no_partial_parse: Optional[bool] = None,
+        introspect: Optional[bool] = None,
+        no_introspect: Optional[bool] = None,
         use_colors: Optional[bool] = None,
         no_use_colors: Optional[bool] = None,
         static_parser: Optional[bool] = None,
         no_static_parser: Optional[bool] = None,
         version_check: Optional[bool] = None,
         no_version_check: Optional[bool] = None,
+        write_json: Optional[bool] = None,
+        write_perf_info: Optional[bool] = None,
         anonymous_usage_stats: Optional[bool] = None,
         no_anonymous_usage_stats: Optional[bool] = None,
         # Extra features configuration
@@ -116,9 +123,12 @@ class DbtBaseOperator(BaseOperator):
         self.warn_error = warn_error
 
         self.debug = debug
+        self.log_path = log_path
         self.log_cache_events = log_cache_events
         self.quiet = quiet
         self.no_print = no_print
+        self.log_level = log_level
+        self.log_level_file = log_level_file
         self.log_format = log_format
         self.record_timing_info = record_timing_info
 
@@ -137,9 +147,18 @@ class DbtBaseOperator(BaseOperator):
         self.use_colors = use_colors
         self.no_use_colors = no_use_colors
 
-        self.version_check = no_version_check
+        self.introspect = introspect
+        self.no_introspect = no_introspect
+
+        self.version_check = version_check
         self.no_version_check = no_version_check
 
+        self.write_json = write_json or (
+            do_xcom_push_artifacts and "run_results.json" in do_xcom_push_artifacts
+        )
+
+        self.write_perf_info = write_perf_info
+
         self.dbt_conn_id = dbt_conn_id
         self.profiles_conn_id = profiles_conn_id
         self.project_conn_id = project_conn_id
@@ -174,7 +193,7 @@ class DbtBaseOperator(BaseOperator):
             )
         except Exception as e:
             self.log.exception(
-                f"An error has ocurred while executing dbt {self.command}", exc_info=e
+                f"An error has occurred while executing dbt {self.command}", exc_info=e
             )
             raise AirflowException(
                 f"An error has occurred while executing dbt {self.command}"
@@ -428,11 +447,13 @@ class DbtCleanOperator(DbtBaseOperator):
         self,
         upload_dbt_project: bool = True,
         delete_before_upload: bool = True,
+        clean_project_files_only: bool = True,
         **kwargs,
     ) -> None:
         super().__init__(**kwargs)
         self.upload_dbt_project = upload_dbt_project
         self.delete_before_upload = delete_before_upload
+        self.clean_project_files_only = clean_project_files_only
 
     @property
     def command(self) -> str: