dbt-bouncer 1.31.2rc2__py3-none-any.whl → 2.0.0__py3-none-any.whl

dbt_bouncer/checks/catalog/check_columns.py

@@ -1,10 +1,10 @@
-# mypy: disable-error-code="union-attr"
-
 import re
-from typing import TYPE_CHECKING, List, Literal, Optional
+from typing import TYPE_CHECKING, Literal
 
 from pydantic import ConfigDict, Field
 
+from dbt_bouncer.checks.common import DbtBouncerFailedCheckError
+
 if TYPE_CHECKING:
     import warnings
 
@@ -13,8 +13,8 @@ if TYPE_CHECKING:
         from dbt_artifacts_parser.parsers.catalog.catalog_v1 import (
             Nodes as CatalogNodes,
         )
-    from dbt_bouncer.artifact_parsers.parsers_common import DbtBouncerManifest
     from dbt_bouncer.artifact_parsers.parsers_manifest import (
+        DbtBouncerManifest,
         DbtBouncerModelBase,
         DbtBouncerTestBase,
     )
@@ -26,15 +26,18 @@ from dbt_bouncer.check_base import BaseCheck
 class CheckColumnDescriptionPopulated(BaseCheck):
     """Columns must have a populated description.
 
+    Parameters:
+        min_description_length (int | None): Minimum length required for the description to be considered populated.
+
     Receives:
         catalog_node (CatalogNodes): The CatalogNodes object to check.
-        models (List[DbtBouncerModelBase]): List of DbtBouncerModelBase objects parsed from `manifest.json`.
+        models (list[DbtBouncerModelBase]): List of DbtBouncerModelBase objects parsed from `manifest.json`.
 
     Other Parameters:
-        description (Optional[str]): Description of what the check does and why it is implemented.
-        exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.
-        include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.
-        severity (Optional[Literal["error", "warn"]]): Severity level of the check. Default: `error`.
+        description (str | None): Description of what the check does and why it is implemented.
+        exclude (str | None): Regex pattern to match the model path. Model paths that match the pattern will not be checked.
+        include (str | None): Regex pattern to match the model path. Only model paths that match the pattern will be checked.
+        severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
 
     Example(s):
         ```yaml
@@ -42,31 +45,44 @@ class CheckColumnDescriptionPopulated(BaseCheck):
             - name: check_column_description_populated
               include: ^models/marts
         ```
+        ```yaml
+        manifest_checks:
+            - name: check_column_description_populated
+              min_description_length: 25 # Setting a stricter requirement for description length
+        ```
 
     """
 
-    catalog_node: "CatalogNodes" = Field(default=None)
-    models: List["DbtBouncerModelBase"] = Field(default=[])
+    catalog_node: "CatalogNodes | None" = Field(default=None)
+    min_description_length: int | None = Field(default=None)
+    models: list["DbtBouncerModelBase"] = Field(default=[])
     name: Literal["check_column_description_populated"]
 
     def execute(self) -> None:
-        """Execute the check."""
+        """Execute the check.
+
+        Raises:
+            DbtBouncerFailedCheckError: If description is not populated.
+
+        """
+        if self.catalog_node is None:
+            raise DbtBouncerFailedCheckError("self.catalog_node is None")
         if self.is_catalog_node_a_model(self.catalog_node, self.models):
             model = next(
                 m for m in self.models if m.unique_id == self.catalog_node.unique_id
             )
             non_complying_columns = []
             for _, v in self.catalog_node.columns.items():
-                if model.columns.get(
-                    v.name
-                ) is None or not self.is_description_populated(
-                    model.columns[v.name].description
+                columns = model.columns or {}
+                if columns.get(v.name) is None or not self._is_description_populated(
+                    columns[v.name].description or "", self.min_description_length
                 ):
                     non_complying_columns.append(v.name)
 
-            assert not non_complying_columns, (
-                f"`{self.catalog_node.unique_id.split('.')[-1]}` has columns that do not have a populated description: {non_complying_columns}"
-            )
+            if non_complying_columns:
+                raise DbtBouncerFailedCheckError(
+                    f"`{str(self.catalog_node.unique_id).split('.')[-1]}` has columns that do not have a populated description: {non_complying_columns}"
+                )
 
 
 class CheckColumnHasSpecifiedTest(BaseCheck):
@@ -78,13 +94,13 @@ class CheckColumnHasSpecifiedTest(BaseCheck):
 
     Receives:
         catalog_node (CatalogNodes): The CatalogNodes object to check.
-        tests (List[DbtBouncerTestBase]): List of DbtBouncerTestBase objects parsed from `manifest.json`.
+        tests (list[DbtBouncerTestBase]): List of DbtBouncerTestBase objects parsed from `manifest.json`.
 
     Other Parameters:
-        description (Optional[str]): Description of what the check does and why it is implemented.
-        exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.
-        include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.
-        severity (Optional[Literal["error", "warn"]]): Severity level of the check. Default: `error`.
+        description (str | None): Description of what the check does and why it is implemented.
+        exclude (str | None): Regex pattern to match the model path. Model paths that match the pattern will not be checked.
+        include (str | None): Regex pattern to match the model path. Only model paths that match the pattern will be checked.
+        severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
 
     Example(s):
         ```yaml
@@ -96,37 +112,52 @@ class CheckColumnHasSpecifiedTest(BaseCheck):
 
     """
 
-    catalog_node: "CatalogNodes" = Field(default=None)
+    catalog_node: "CatalogNodes | None" = Field(default=None)
     column_name_pattern: str
     name: Literal["check_column_has_specified_test"]
     test_name: str
-    tests: List["DbtBouncerTestBase"] = Field(default=[])
+    tests: list["DbtBouncerTestBase"] = Field(default=[])
 
     def execute(self) -> None:
-        """Execute the check."""
+        """Execute the check.
+
+        Raises:
+            DbtBouncerFailedCheckError: If column does not have specified test.
+
+        """
+        if self.catalog_node is None:
+            raise DbtBouncerFailedCheckError("self.catalog_node is None")
         columns_to_check = [
             v.name
             for _, v in self.catalog_node.columns.items()
-            if re.compile(self.column_name_pattern.strip()).match(v.name) is not None
-        ]
-        relevant_tests = [
-            t
-            for t in self.tests
-            if hasattr(t, "test_metadata") is True
-            and hasattr(t, "attached_node") is True
-            and t.test_metadata.name == self.test_name
-            and t.attached_node == self.catalog_node.unique_id
+            if re.compile(self.column_name_pattern.strip()).match(str(v.name))
+            is not None
         ]
+        relevant_tests = []
+        for t in self.tests:
+            test_metadata = getattr(t, "test_metadata", None)
+            attached_node = getattr(t, "attached_node", None)
+            if (
+                test_metadata
+                and attached_node
+                and getattr(test_metadata, "name", None) == self.test_name
+                and attached_node == self.catalog_node.unique_id
+            ):
+                relevant_tests.append(t)
         non_complying_columns = [
             c
             for c in columns_to_check
             if f"{self.catalog_node.unique_id}.{c}"
-            not in [f"{t.attached_node}.{t.column_name}" for t in relevant_tests]
+            not in [
+                f"{getattr(t, 'attached_node', '')}.{getattr(t, 'column_name', '')}"
+                for t in relevant_tests
+            ]
         ]
 
-        assert not non_complying_columns, (
-            f"`{self.catalog_node.unique_id.split('.')[-1]}` has columns that should have a `{self.test_name}` test: {non_complying_columns}"
-        )
+        if non_complying_columns:
+            raise DbtBouncerFailedCheckError(
+                f"`{str(self.catalog_node.unique_id).split('.')[-1]}` has columns that should have a `{self.test_name}` test: {non_complying_columns}"
+            )
 
 
 class CheckColumnNameCompliesToColumnType(BaseCheck):
@@ -136,17 +167,17 @@ class CheckColumnNameCompliesToColumnType(BaseCheck):
 
     Parameters:
         column_name_pattern (str): Regex pattern to match the model name.
-        type_pattern (Optional[str]): Regex pattern to match the data types.
-        types (Optional[List[str]]): List of data types to check.
+        type_pattern (str | None): Regex pattern to match the data types.
+        types (list[str] | None): List of data types to check.
 
     Receives:
         catalog_node (CatalogNodes): The CatalogNodes object to check.
 
     Other Parameters:
-        description (Optional[str]): Description of what the check does and why it is implemented.
-        exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.
-        include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.
-        severity (Optional[Literal["error", "warn"]]): Severity level of the check. Default: `error`.
+        description (str | None): Description of what the check does and why it is implemented.
+        exclude (str | None): Regex pattern to match the model path. Model paths that match the pattern will not be checked.
+        include (str | None): Regex pattern to match the model path. Only model paths that match the pattern will be checked.
+        severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
 
     Example(s):
         ```yaml
@@ -188,38 +219,48 @@ class CheckColumnNameCompliesToColumnType(BaseCheck):
 
     """
 
-    catalog_node: "CatalogNodes" = Field(default=None)
+    catalog_node: "CatalogNodes | None" = Field(default=None)
     column_name_pattern: str
     name: Literal["check_column_name_complies_to_column_type"]
-    type_pattern: Optional[str] = None
-    types: Optional[List[str]] = None
+    type_pattern: str | None = None
+    types: list[str] | None = None
 
     def execute(self) -> None:
-        """Execute the check."""
+        """Execute the check.
+
+        Raises:
+            DbtBouncerFailedCheckError: If column name does not comply to column type.
+
+        """
+        if self.catalog_node is None:
+            raise DbtBouncerFailedCheckError("self.catalog_node is None")
         if self.type_pattern:
             non_complying_columns = [
                 v.name
                 for _, v in self.catalog_node.columns.items()
-                if re.compile(self.type_pattern.strip()).match(v.type) is None
-                and re.compile(self.column_name_pattern.strip()).match(v.name)
+                if re.compile(self.type_pattern.strip()).match(str(v.type)) is None
+                and re.compile(self.column_name_pattern.strip()).match(str(v.name))
                 is not None
             ]
 
-            assert not non_complying_columns, (
-                f"`{self.catalog_node.unique_id.split('.')[-1]}` has columns that don't comply with the specified data type regexp pattern (`{self.column_name_pattern}`): {non_complying_columns}"
-            )
+            if non_complying_columns:
+                raise DbtBouncerFailedCheckError(
+                    f"`{str(self.catalog_node.unique_id).split('.')[-1]}` has columns that don't comply with the specified data type regexp pattern (`{self.column_name_pattern}`): {non_complying_columns}"
+                )
 
         elif self.types:
             non_complying_columns = [
                 v.name
                 for _, v in self.catalog_node.columns.items()
                 if v.type in self.types
-                and re.compile(self.column_name_pattern.strip()).match(v.name) is None
+                and re.compile(self.column_name_pattern.strip()).match(str(v.name))
+                is None
             ]
 
-            assert not non_complying_columns, (
-                f"`{self.catalog_node.unique_id.split('.')[-1]}` has columns that don't comply with the specified regexp pattern (`{self.column_name_pattern}`): {non_complying_columns}"
-            )
+            if non_complying_columns:
+                raise DbtBouncerFailedCheckError(
+                    f"`{str(self.catalog_node.unique_id).split('.')[-1]}` has columns that don't comply with the specified regexp pattern (`{self.column_name_pattern}`): {non_complying_columns}"
+                )
 
     @model_validator(mode="after")
     def _check_type_pattern_or_types(self) -> "CheckColumnNameCompliesToColumnType":
@@ -238,14 +279,14 @@ class CheckColumnNames(BaseCheck):
 
     Receives:
         catalog_node (CatalogNodes): The CatalogNodes object to check.
-        models (List[DbtBouncerModelBase]): List of DbtBouncerModelBase objects parsed from `manifest.json`.
+        models (list[DbtBouncerModelBase]): List of DbtBouncerModelBase objects parsed from `manifest.json`.
 
     Other Parameters:
-        description (Optional[str]): Description of what the check does and why it is implemented.
-        exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.
-        include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.
-        materialization (Optional[Literal["ephemeral", "incremental", "table", "view"]]): Limit check to models with the specified materialization.
-        severity (Optional[Literal["error", "warn"]]): Severity level of the check. Default: `error`.
+        description (str | None): Description of what the check does and why it is implemented.
+        exclude (str | None): Regex pattern to match the model path. Model paths that match the pattern will not be checked.
+        include (str | None): Regex pattern to match the model path. Only model paths that match the pattern will be checked.
+        materialization (Literal["ephemeral", "incremental", "table", "view"] | None): Limit check to models with the specified materialization.
+        severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
 
     Example(s):
         ```yaml
@@ -258,40 +299,48 @@ class CheckColumnNames(BaseCheck):
 
     model_config = ConfigDict(extra="forbid", protected_namespaces=())
 
-    catalog_node: "CatalogNodes" = Field(default=None)
+    catalog_node: "CatalogNodes | None" = Field(default=None)
     column_name_pattern: str
-    models: List["DbtBouncerModelBase"] = Field(default=[])
+    models: list["DbtBouncerModelBase"] = Field(default=[])
     name: Literal["check_column_names"]
 
     def execute(self) -> None:
-        """Execute the check."""
+        """Execute the check.
+
+        Raises:
+            DbtBouncerFailedCheckError: If column name does not match regex.
+
+        """
+        if self.catalog_node is None:
+            raise DbtBouncerFailedCheckError("self.catalog_node is None")
         if self.is_catalog_node_a_model(self.catalog_node, self.models):
-            non_complying_columns: List[str] = []
+            non_complying_columns: list[str] = []
             non_complying_columns.extend(
                 v.name
                 for _, v in self.catalog_node.columns.items()
-                if re.fullmatch(self.column_name_pattern.strip(), v.name) is None
+                if re.fullmatch(self.column_name_pattern.strip(), str(v.name)) is None
             )
 
-            assert not non_complying_columns, (
-                f"`{self.catalog_node.unique_id.split('.')[-1]}` has columns ({non_complying_columns}) that do not match the supplied regex: `{self.column_name_pattern.strip()}`."
-            )
+            if non_complying_columns:
+                raise DbtBouncerFailedCheckError(
+                    f"`{str(self.catalog_node.unique_id).split('.')[-1]}` has columns ({non_complying_columns}) that do not match the supplied regex: `{self.column_name_pattern.strip()}`."
+                )
 
 
 class CheckColumnsAreAllDocumented(BaseCheck):
     """All columns in a model should be included in the model's properties file, i.e. `.yml` file.
 
     Receives:
-        case_sensitive (Optional[bool]): Whether the column names are case sensitive or not. Necessary for adapters like `dbt-snowflake` where the column in `catalog.json` is uppercase but the column in `manifest.json` can be lowercase. Defaults to `false` for `dbt-snowflake`, otherwise `true`.
+        case_sensitive (bool | None): Whether the column names are case sensitive or not. Necessary for adapters like `dbt-snowflake` where the column in `catalog.json` is uppercase but the column in `manifest.json` can be lowercase. Defaults to `false` for `dbt-snowflake`, otherwise `true`.
         catalog_node (CatalogNodes): The CatalogNodes object to check.
         manifest_obj (DbtBouncerManifest): The DbtBouncerManifest object parsed from `manifest.json`.
-        models (List[DbtBouncerModelBase]): List of DbtBouncerModelBase objects parsed from `manifest.json`.
+        models (list[DbtBouncerModelBase]): List of DbtBouncerModelBase objects parsed from `manifest.json`.
 
     Other Parameters:
-        description (Optional[str]): Description of what the check does and why it is implemented.
-        exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.
-        include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.
-        severity (Optional[Literal["error", "warn"]]): Severity level of the check. Default: `error`.
+        description (str | None): Description of what the check does and why it is implemented.
+        exclude (str | None): Regex pattern to match the model path. Model paths that match the pattern will not be checked.
+        include (str | None): Regex pattern to match the model path. Only model paths that match the pattern will be checked.
+        severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
 
     Example(s):
         ```yaml
@@ -301,14 +350,23 @@ class CheckColumnsAreAllDocumented(BaseCheck):
 
     """
 
-    case_sensitive: Optional[bool] = Field(default=True)
-    catalog_node: "CatalogNodes" = Field(default=None)
-    manifest_obj: "DbtBouncerManifest" = Field(default=None)
-    models: List["DbtBouncerModelBase"] = Field(default=[])
+    case_sensitive: bool | None = Field(default=True)
+    catalog_node: "CatalogNodes | None" = Field(default=None)
+    manifest_obj: "DbtBouncerManifest | None" = Field(default=None)
+    models: list["DbtBouncerModelBase"] = Field(default=[])
     name: Literal["check_columns_are_all_documented"]
 
     def execute(self) -> None:
-        """Execute the check."""
+        """Execute the check.
+
+        Raises:
+            DbtBouncerFailedCheckError: If columns are undocumented.
+
+        """
+        if self.catalog_node is None:
+            raise DbtBouncerFailedCheckError("self.catalog_node is None")
+        if self.manifest_obj is None:
+            raise DbtBouncerFailedCheckError("self.manifest_obj is None")
         if self.is_catalog_node_a_model(self.catalog_node, self.models):
             model = next(
                 m for m in self.models if m.unique_id == self.catalog_node.unique_id
@@ -317,22 +375,24 @@ class CheckColumnsAreAllDocumented(BaseCheck):
             if self.manifest_obj.manifest.metadata.adapter_type in ["snowflake"]:
                 self.case_sensitive = False
 
+            model_columns = model.columns or {}
             if self.case_sensitive:
                 undocumented_columns = [
                     v.name
                     for _, v in self.catalog_node.columns.items()
-                    if v.name not in model.columns
+                    if v.name not in model_columns
                 ]
             else:
                 undocumented_columns = [
                     v.name
                     for _, v in self.catalog_node.columns.items()
-                    if v.name.lower() not in [c.lower() for c in model.columns]
+                    if v.name.lower() not in [c.lower() for c in model_columns]
                 ]
 
-            assert not undocumented_columns, (
-                f"`{self.catalog_node.unique_id.split('.')[-1]}` has columns that are not included in the models properties file: {undocumented_columns}"
-            )
+            if undocumented_columns:
+                raise DbtBouncerFailedCheckError(
+                    f"`{str(self.catalog_node.unique_id).split('.')[-1]}` has columns that are not included in the models properties file: {undocumented_columns}"
+                )
 
 
 class CheckColumnsAreDocumentedInPublicModels(BaseCheck):
@@ -340,13 +400,14 @@ class CheckColumnsAreDocumentedInPublicModels(BaseCheck):
 
     Receives:
         catalog_node (CatalogNodes): The CatalogNodes object to check.
-        models (List[DbtBouncerModelBase]): List of DbtBouncerModelBase objects parsed from `manifest.json`.
+        min_description_length (int | None): Minimum length required for the description to be considered populated.
+        models (list[DbtBouncerModelBase]): List of DbtBouncerModelBase objects parsed from `manifest.json`.
 
     Other Parameters:
-        description (Optional[str]): Description of what the check does and why it is implemented.
-        exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.
-        include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.
-        severity (Optional[Literal["error", "warn"]]): Severity level of the check. Default: `error`.
+        description (str | None): Description of what the check does and why it is implemented.
+        exclude (str | None): Regex pattern to match the model path. Model paths that match the pattern will not be checked.
+        include (str | None): Regex pattern to match the model path. Only model paths that match the pattern will be checked.
+        severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
 
     Example(s):
         ```yaml
@@ -356,26 +417,35 @@ class CheckColumnsAreDocumentedInPublicModels(BaseCheck):
 
     """
 
-    catalog_node: "CatalogNodes" = Field(default=None)
-    models: List["DbtBouncerModelBase"] = Field(default=[])
+    catalog_node: "CatalogNodes | None" = Field(default=None)
+    min_description_length: int | None = Field(default=None)
+    models: list["DbtBouncerModelBase"] = Field(default=[])
     name: Literal["check_columns_are_documented_in_public_models"]
 
     def execute(self) -> None:
-        """Execute the check."""
+        """Execute the check.
+
+        Raises:
+            DbtBouncerFailedCheckError: If columns are undocumented in public model.
+
+        """
+        if self.catalog_node is None:
+            raise DbtBouncerFailedCheckError("self.catalog_node is None")
         if self.is_catalog_node_a_model(self.catalog_node, self.models):
             model = next(
                 m for m in self.models if m.unique_id == self.catalog_node.unique_id
             )
             non_complying_columns = []
             for _, v in self.catalog_node.columns.items():
-                if model.access.value == "public":
-                    column_config = model.columns.get(v.name)
-                    if (
-                        column_config is None
-                        or len(column_config.description.strip()) < 4
+                if model.access and model.access.value == "public":
+                    model_columns = model.columns or {}
+                    column_config = model_columns.get(v.name)
+                    if column_config is None or not self._is_description_populated(
+                        column_config.description or "", self.min_description_length
                     ):
                         non_complying_columns.append(v.name)
 
-            assert not non_complying_columns, (
-                f"`{self.catalog_node.unique_id.split('.')[-1]}` is a public model but has columns that don't have a populated description: {non_complying_columns}"
-            )
+            if non_complying_columns:
+                raise DbtBouncerFailedCheckError(
+                    f"`{str(self.catalog_node.unique_id).split('.')[-1]}` is a public model but has columns that don't have a populated description: {non_complying_columns}"
+                )

dbt_bouncer/checks/common.py

@@ -1,7 +1,28 @@
-from typing import Dict, List, Union
-
 from pydantic import RootModel
 
 
+class DbtBouncerFailedCheckError(Exception):
+    """A custom exception class for failing dbt-bouncer checks."""
+
+    def __init__(self, message: str):
+        """Initialize the DbtBouncerFailedCheck exception.
+
+        Args:
+            message (str): The exception message.
+
+        """
+        self.message = message
+        super().__init__(self.message)
+
+    def __str__(self) -> str:
+        """Return the string representation of the exception.
+
+        Returns:
+            str: The exception message.
+
+        """
+        return self.message
+
+
 class NestedDict(RootModel):  # type: ignore[type-arg]
-    root: Union[Dict[str, "NestedDict"], List["NestedDict"], str]
+    root: dict[str, "NestedDict"] | list["NestedDict"] | str