dbt-bouncer 1.31.2rc3__py3-none-any.whl → 2.0.0rc1__py3-none-any.whl

dbt_bouncer/main.py

@@ -1,70 +1,43 @@
+import importlib
 import logging
+import sys
 from pathlib import Path, PurePath
-from typing import Union
 
 import click
 
+from dbt_bouncer.global_context import BouncerContext, set_context
 from dbt_bouncer.logger import configure_console_logging
 from dbt_bouncer.version import version
 
 
-@click.command()
-@click.option(
-    "--config-file",
-    default=Path("dbt-bouncer.yml"),
-    help="Location of the YML config file.",
-    required=False,
-    type=PurePath,
-)
-@click.option(
-    "--create-pr-comment-file",
-    default=False,
-    help="Create a `github-comment.md` file that will be sent to GitHub as a PR comment. Defaults to True when `dbt-bouncer` is run as a GitHub Action.",
-    hidden=True,
-    required=False,
-    type=click.BOOL,
-)
-@click.option(
-    "--only",
-    default="",
-    help="Limit the checks run to specific categories, comma-separated. Examples: 'manifest_checks', 'catalog_checks,manifest_checks'.",
-    required=False,
-    type=str,
-)
-@click.option(
-    "--output-file",
-    default=None,
-    help="Location of the json file where check metadata will be saved.",
-    required=False,
-    type=Path,
-)
-@click.option(
-    "--output-only-failures",
-    help="If passed then only failures will be included in the output file.",
-    is_flag=True,
-)
-@click.option(
-    "--show-all-failures",
-    help="If passed then all failures will be printed to the console.",
-    is_flag=True,
-)
-@click.option("-v", "--verbosity", help="Verbosity.", default=0, count=True)
-@click.pass_context
-@click.version_option()
-def cli(
-    ctx: click.Context,
-    config_file: PurePath,
-    create_pr_comment_file: bool,
-    only: str,
-    output_file: Union[Path, None],
-    output_only_failures: bool,
-    show_all_failures: bool,
-    verbosity: int,
-) -> None:
-    """Entrypoint for dbt-bouncer.
+def run_bouncer(
+    config_file: PurePath | None = None,
+    create_pr_comment_file: bool = False,
+    only: str = "",
+    output_file: Path | None = None,
+    output_only_failures: bool = False,
+    show_all_failures: bool = False,
+    verbosity: int = 0,
+    config_file_source: str | None = None,
+) -> int:
+    """Programmatic entrypoint for dbt-bouncer.
+
+    Args:
+        config_file: Location of the YML config file.
+        create_pr_comment_file: Create a `github-comment.md` file.
+        only: Limit the checks run to specific categories.
+        output_file: Location of the json file where check metadata will be saved.
+        output_only_failures: Only failures will be included in the output file.
+        show_all_failures: All failures will be printed to the console.
+        verbosity: Verbosity level.
+        config_file_source: Source of the config file ("COMMANDLINE", "DEFAULT", etc.).
+
+    Returns:
+        int: Exit code (0 for success, 1 for failure).
 
     Raises:
-        RuntimeError: If output file has an invalid extension.
+        AssertionError: If config_file_source is None.
+        RuntimeError: If output file has an invalid extension or other runtime errors.
 
     """
     configure_console_logging(verbosity)
@@ -94,11 +67,20 @@ def cli(
         load_config_file_contents,
     )
 
+    if config_file is None:
+        config_file = Path("dbt-bouncer.yml")
+        if config_file_source is None:
+            config_file_source = "DEFAULT"
+    else:
+        if config_file_source is None:
+            config_file_source = "COMMANDLINE"
+
+    if config_file_source is None:
+        raise AssertionError("config_file_source cannot be None")
+
     config_file_path = get_config_file_path(
         config_file=config_file,
-        config_file_source=click.get_current_context()
-        .get_parameter_source("config_file")
-        .name,  # type: ignore[union-attr]
+        config_file_source=config_file_source,
     )
     config_file_contents = load_config_file_contents(
         config_file_path, allow_default_config_file_creation=True
@@ -121,16 +103,23 @@ def cli(
     ]
     logging.debug(f"{check_categories=}")
 
-    # Set click context object for dbt_bouncer.utils.get_check_objects()
-    ctx.obj = {
-        "config_file_path": config_file_path,
-        "custom_checks_dir": config_file_contents.get("custom_checks_dir"),
-    }
+    # Set global context for dbt_bouncer.utils.get_check_objects() and config_file_parser
+    set_context(
+        BouncerContext(
+            config_file_path=config_file_path,
+            custom_checks_dir=config_file_contents.get("custom_checks_dir"),
+        )
+    )
+
+    # If config_file_parser is already loaded, reload it so that check types are updated (necessary for parallel tests)
+    if "dbt_bouncer.config_file_parser" in sys.modules:
+        importlib.reload(sys.modules["dbt_bouncer.config_file_parser"])
 
     from dbt_bouncer.config_file_validator import validate_conf
 
     bouncer_config = validate_conf(
-        check_categories=check_categories, config_file_contents=config_file_contents
+        check_categories=check_categories,
+        config_file_contents=dict(config_file_contents),
     )
     del config_file_contents
     logging.debug(f"{bouncer_config=}")
@@ -152,7 +141,9 @@ def cli(
 
     logging.debug(f"{bouncer_config=}")
 
-    dbt_artifacts_dir = config_file.parent / bouncer_config.dbt_artifacts_dir
+    dbt_artifacts_dir = Path(
+        config_file_path.parent / (bouncer_config.dbt_artifacts_dir or "target")
+    )
 
     from dbt_bouncer.artifact_parsers.parsers_common import parse_dbt_artifacts
 
@@ -161,6 +152,7 @@ def cli(
         project_exposures,
         project_macros,
         project_models,
+        project_seeds,
         project_semantic_models,
         project_snapshots,
         project_sources,
@@ -189,6 +181,7 @@ def cli(
         output_file=output_file,
         output_only_failures=output_only_failures,
         run_results=project_run_results,
+        seeds=project_seeds,
         semantic_models=project_semantic_models,
         show_all_failures=show_all_failures,
         snapshots=project_snapshots,
@@ -196,4 +189,72 @@ def cli(
         tests=project_tests,
         unit_tests=project_unit_tests,
     )
-    ctx.exit(results[0])
+    return results[0]
+
+
+@click.command()
+@click.option(
+    "--config-file",
+    default=Path("dbt-bouncer.yml"),
+    help="Location of the YML config file.",
+    required=False,
+    type=PurePath,
+)
+@click.option(
+    "--create-pr-comment-file",
+    default=False,
+    help="Create a `github-comment.md` file that will be sent to GitHub as a PR comment. Defaults to True when `dbt-bouncer` is run as a GitHub Action.",
+    hidden=True,
+    required=False,
+    type=click.BOOL,
+)
+@click.option(
+    "--only",
+    default="",
+    help="Limit the checks run to specific categories, comma-separated. Examples: 'manifest_checks', 'catalog_checks,manifest_checks'.",
+    required=False,
+    type=str,
+)
+@click.option(
+    "--output-file",
+    default=None,
+    help="Location of the json file where check metadata will be saved.",
+    required=False,
+    type=Path,
+)
+@click.option(
+    "--output-only-failures",
+    help="If passed then only failures will be included in the output file.",
+    is_flag=True,
+)
+@click.option(
+    "--show-all-failures",
+    help="If passed then all failures will be printed to the console.",
+    is_flag=True,
+)
+@click.option("-v", "--verbosity", help="Verbosity.", default=0, count=True)
+@click.pass_context
+@click.version_option()
+def cli(
+    ctx: click.Context,
+    config_file: PurePath,
+    create_pr_comment_file: bool,
+    only: str,
+    output_file: Path | None,
+    output_only_failures: bool,
+    show_all_failures: bool,
+    verbosity: int,
+) -> None:
+    """Entrypoint for dbt-bouncer."""
+    config_file_source = ctx.get_parameter_source("config_file").name  # type: ignore[union-attr]
+    exit_code = run_bouncer(
+        config_file=config_file,
+        create_pr_comment_file=create_pr_comment_file,
+        only=only,
+        output_file=output_file,
+        output_only_failures=output_only_failures,
+        show_all_failures=show_all_failures,
+        verbosity=verbosity,
+        config_file_source=config_file_source,
+    )
+    ctx.exit(exit_code)

dbt_bouncer/runner.py

@@ -1,16 +1,19 @@
 """Assemble and run all checks."""
 
 import copy
+import inspect
 import json
 import logging
 import operator
 import traceback
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Dict, List, Union
+from typing import TYPE_CHECKING, Any
 
+import click
 from progress.bar import Bar
 from tabulate import tabulate
 
+from dbt_bouncer.checks.common import DbtBouncerFailedCheckError
 from dbt_bouncer.utils import (
     create_github_comment_file,
     get_check_objects,
@@ -20,8 +23,6 @@ from dbt_bouncer.utils import (
 
 if TYPE_CHECKING:
     from dbt_bouncer.artifact_parsers.dbt_cloud.manifest_latest import (
-        Exposures,
-        Macros,
         UnitTests,
     )
     from dbt_bouncer.artifact_parsers.parsers_common import (
@@ -29,11 +30,16 @@ if TYPE_CHECKING:
         DbtBouncerManifest,
         DbtBouncerModel,
         DbtBouncerRunResult,
+        DbtBouncerSeed,
         DbtBouncerSemanticModel,
         DbtBouncerSnapshot,
         DbtBouncerSource,
         DbtBouncerTest,
     )
+    from dbt_bouncer.artifact_parsers.parsers_manifest import (
+        DbtBouncerExposureBase,
+        DbtBouncerMacroBase,
+    )
     from dbt_bouncer.config_file_parser import (
         DbtBouncerConfAllCategories as DbtBouncerConf,
     )
@@ -41,36 +47,46 @@ if TYPE_CHECKING:
 
 def runner(
     bouncer_config: "DbtBouncerConf",
-    catalog_nodes: List["DbtBouncerCatalogNode"],
-    catalog_sources: List["DbtBouncerCatalogNode"],
-    check_categories: List[str],
+    catalog_nodes: list["DbtBouncerCatalogNode"],
+    catalog_sources: list["DbtBouncerCatalogNode"],
+    check_categories: list[str],
     create_pr_comment_file: bool,
-    exposures: List["Exposures"],
-    macros: List["Macros"],
+    exposures: list["DbtBouncerExposureBase"],
+    macros: list["DbtBouncerMacroBase"],
     manifest_obj: "DbtBouncerManifest",
-    models: List["DbtBouncerModel"],
-    output_file: Union[Path, None],
-    run_results: List["DbtBouncerRunResult"],
-    semantic_models: List["DbtBouncerSemanticModel"],
+    models: list["DbtBouncerModel"],
+    output_file: Path | None,
+    run_results: list["DbtBouncerRunResult"],
+    seeds: list["DbtBouncerSeed"],
+    semantic_models: list["DbtBouncerSemanticModel"],
     output_only_failures: bool,
     show_all_failures: bool,
-    snapshots: List["DbtBouncerSnapshot"],
-    sources: List["DbtBouncerSource"],
-    tests: List["DbtBouncerTest"],
-    unit_tests: List["UnitTests"],
-) -> tuple[int, List[Any]]:
+    snapshots: list["DbtBouncerSnapshot"],
+    sources: list["DbtBouncerSource"],
+    tests: list["DbtBouncerTest"],
+    unit_tests: list["UnitTests"],
+) -> tuple[int, list[Any]]:
     """Run dbt-bouncer checks.
 
     Returns:
-        tuple[int, List[Any]]: A tuple containing the exit code and a list of failed checks.
+        tuple[int, list[Any]]: A tuple containing the exit code and a list of failed checks.
 
     Raises:
         RuntimeError: If more than one "iterate_over" argument is found.
 
     """
-    check_classes: List[Dict[str, Union[Any, str]]] = [
-        {"class": getattr(x, x.__name__), "source_file": x.__file__}
-        for x in get_check_objects()
+    try:
+        ctx = click.get_current_context()
+        config_file_path = ctx.obj.get("config_file_path")
+        custom_checks_dir = ctx.obj.get("custom_checks_dir")
+        if custom_checks_dir:
+            custom_checks_dir = config_file_path.parent / custom_checks_dir
+    except (RuntimeError, AttributeError, KeyError):
+        custom_checks_dir = None
+
+    check_classes: list[dict[str, Any | str]] = [
+        {"class": x, "source_file": inspect.getfile(x)}
+        for x in get_check_objects(custom_checks_dir)
     ]
     for c in check_classes:
         locals()[c["class"].__name__] = c["class"]  # type: ignore[union-attr]
@@ -83,6 +99,7 @@ def runner(
         "manifest_obj": manifest_obj,
         "models": [m.model for m in models],
         "run_results": [r.run_result for r in run_results],
+        "seeds": [s.seed for s in seeds],
         "semantic_models": [s.semantic_model for s in semantic_models],
         "snapshots": [s.snapshot for s in snapshots],
         "sources": sources,
@@ -103,19 +120,26 @@ def runner(
             "macro",
             "model",
             "run_result",
+            "seed",
             "semantic_model",
             "snapshot",
             "source",
             "unit_test",
         }
         iterate_over_value = valid_iterate_over_values.intersection(
-            set(check.__annotations__.keys()),
+            set(check.__class__.__annotations__.keys()),
         )
         if len(iterate_over_value) == 1:
             iterate_value = next(iter(iterate_over_value))
             for i in locals()[f"{iterate_value}s"]:
                 check_i = copy.deepcopy(check)
-                if iterate_value in ["model", "semantic_model", "snapshot", "source"]:
+                if iterate_value in [
+                    "model",
+                    "seed",
+                    "semantic_model",
+                    "snapshot",
+                    "source",
+                ]:
                     try:
                         d = getattr(i, iterate_value).config.meta
                     except Exception:
@@ -153,7 +177,9 @@ def runner(
                     )
                     setattr(check_i, iterate_value, getattr(i, iterate_value, i))
 
-                    for x in parsed_data.keys() & check_i.__annotations__.keys():
+                    for x in (
+                        parsed_data.keys() & check_i.__class__.__annotations__.keys()
+                    ):
                         setattr(check_i, x, parsed_data[x])
 
                     checks_to_run.append(
@@ -169,7 +195,7 @@ def runner(
             )
         else:
             check_run_id = f"{check.name}:{check.index}"
-            for x in parsed_data.keys() & check.__annotations__.keys():
+            for x in parsed_data.keys() & check.__class__.__annotations__.keys():
                 setattr(check, x, parsed_data[x])
             checks_to_run.append(
                 {
@@ -188,21 +214,25 @@ def runner(
             check["check"].execute()
             check["outcome"] = "success"
         except Exception as e:
-            failure_message_full = list(
-                traceback.TracebackException.from_exception(e).format(),
-            )
-            failure_message = failure_message_full[-1].strip()
+            if isinstance(e, DbtBouncerFailedCheckError):
+                failure_message = e.message
+            else:
+                failure_message_full = list(
+                    traceback.TracebackException.from_exception(e).format(),
+                )
+                failure_message = failure_message_full[-1].strip()
+
             if check["check"].description:
                 failure_message = f"{check['check'].description} - {failure_message}"
 
             logging.debug(
-                f"Check {check['check_run_id']} failed: {' '.join(failure_message_full)}"
+                f"Check {check['check_run_id']} failed: {' '.join(failure_message)}"
             )
             check["outcome"] = "failed"
             check["failure_message"] = failure_message
 
             # If a check encountered an issue, change severity to warn
-            if not isinstance(e, AssertionError):
+            if not isinstance(e, DbtBouncerFailedCheckError):
                 check["severity"] = "warn"
                 check["failure_message"] = (
                     f"`dbt-bouncer` encountered an error ({failure_message}), run with `-v` to see more details or report an issue at https://github.com/godatadriven/dbt-bouncer/issues."