altimate-datapilot-cli 0.0.8__py3-none-any.whl

datapilot/core/platforms/dbt/insights/base.py

@@ -0,0 +1,133 @@
+from abc import abstractmethod
+from typing import ClassVar
+from typing import Dict
+from typing import List
+from typing import Union
+
+from datapilot.config.utils import get_insight_config
+from datapilot.core.insights.base.insight import Insight
+from datapilot.core.insights.schema import Severity
+from datapilot.core.platforms.dbt.constants import NON_MATERIALIZED
+from datapilot.core.platforms.dbt.schemas.manifest import AltimateManifestExposureNode
+from datapilot.core.platforms.dbt.schemas.manifest import AltimateManifestMacroNode
+from datapilot.core.platforms.dbt.schemas.manifest import AltimateManifestNode
+from datapilot.core.platforms.dbt.schemas.manifest import AltimateManifestSourceNode
+from datapilot.core.platforms.dbt.schemas.manifest import AltimateManifestTestNode
+from datapilot.core.platforms.dbt.schemas.manifest import AltimateResourceType
+from datapilot.core.platforms.dbt.schemas.manifest import AltimateSeedNode
+from datapilot.core.platforms.dbt.wrappers.manifest.wrapper import BaseManifestWrapper
+
+
+class DBTInsight(Insight):
+    DEFAULT_SEVERITY = Severity.ERROR
+    FILES_REQUIRED: ClassVar = ["Manifest"]
+
+    def __init__(
+        self,
+        manifest_wrapper: BaseManifestWrapper,
+        nodes: Dict[str, AltimateManifestNode],
+        sources: Dict[str, AltimateManifestSourceNode],
+        exposures: Dict[str, AltimateManifestExposureNode],
+        tests: Dict[str, AltimateManifestTestNode],
+        seeds: Dict[str, AltimateSeedNode],
+        macros: Dict[str, AltimateManifestMacroNode],
+        children_map: Dict[str, List[str]],
+        project_name: str,
+        selected_models: Union[List[str], None] = None,
+        excluded_models: Union[List[str], None] = None,
+        *args,
+        **kwargs,
+    ):
+        self.manifest = manifest_wrapper
+        self.nodes = nodes
+        self.macros = macros or {}
+        self.sources = sources
+        self.exposures = exposures
+        self.tests = tests
+        self.seeds = seeds
+        self.children_map = children_map
+        self.project_name = project_name
+        self.selected_models = selected_models
+        self.excluded_models = excluded_models
+        super().__init__(*args, **kwargs)
+
+    @abstractmethod
+    def generate(self, *args, **kwargs) -> Dict:
+        pass
+
+    def check_part_of_project(self, node_project_name: str) -> bool:
+        return node_project_name == self.project_name
+
+    def get_node(
+        self, node_id: str
+    ) -> Union[
+        AltimateManifestNode, AltimateManifestSourceNode, AltimateManifestExposureNode, AltimateManifestTestNode, AltimateManifestMacroNode
+    ]:
+        if node_id in self.nodes:
+            return self.nodes[node_id]
+        elif node_id in self.sources:
+            return self.sources[node_id]
+        elif node_id in self.exposures:
+            return self.exposures[node_id]
+        elif node_id in self.tests:
+            return self.tests[node_id]
+        elif node_id in self.macros:
+            return self.macros[node_id]
+        elif node_id in self.seeds:
+            return self.seeds[node_id]
+        else:
+            self.logger.debug(f"Model {node_id} not found in manifest")
+            return None
+
+    def find_long_chains(self, min_chain_length=4):
+        """
+        Find chains of nodes with 'materialized' set to 'view' or 'ephemeral' of a given minimum length.
+
+        :param nodes: Dictionary of nodes where key is node_id and value is a node with 'depends_on' and 'materialized'.
+        :param min_chain_length: Minimum length of the chain to be found.
+        :return: A list of chains, where each chain is a list of node IDs.
+        """
+
+        def is_not_materialized(node: Union[AltimateManifestNode, AltimateManifestSourceNode]) -> bool:
+            if node.resource_type == AltimateResourceType.source:
+                return False
+            return node.config.materialized in NON_MATERIALIZED
+
+        def build_chain(node_id, current_chain):
+            if len(current_chain) >= min_chain_length:
+                long_chains.append(current_chain)
+                return
+            for parent_id in self.get_node(node_id).depends_on.nodes:
+                if is_not_materialized(self.get_node(parent_id)):
+                    build_chain(parent_id, [*current_chain, parent_id])
+
+        long_chains = []
+        for node_id, node in self.nodes.items():
+            if is_not_materialized(node):
+                build_chain(node_id, [node_id])
+
+        return long_chains
+
+    def should_skip_model(self, model_unique_id):
+        """Check if a model is in the excluded models list."""
+        if self.selected_models:
+            return model_unique_id not in self.selected_models
+
+        return False
+
+    @classmethod
+    def get_config_schema(cls):
+        return {
+            "name": cls.NAME,
+            "alias": cls.ALIAS,
+            "type": cls.TYPE,
+            "files_required": cls.FILES_REQUIRED,
+            "description": cls.DESCRIPTION,
+            "config": {"$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": {}},
+        }
+
+    def requires_catalog(cls) -> bool:
+        return False
+
+    def get_check_config(self, key: str) -> any:
+        return get_insight_config(self.config, self.ALIAS, key)

datapilot/core/platforms/dbt/insights/checks/base.py

@@ -0,0 +1,26 @@
+from abc import abstractmethod
+from typing import ClassVar
+from typing import Tuple
+
+from datapilot.core.platforms.dbt.insights.base import DBTInsight
+
+
+class ChecksInsight(DBTInsight):
+    TYPE = "Checks"
+    FILES_REQUIRED: ClassVar = ["Manifest"]
+
+    @abstractmethod
+    def generate(self, *args, **kwargs) -> dict:
+        pass
+
+    @classmethod
+    def has_all_required_data(cls, has_manifest: bool, has_catalog: bool, **kwargs) -> Tuple[bool, str]:
+        """
+        Check if all required data is available for the insight to run.
+        :param has_manifest: A boolean indicating if manifest is available.
+        :return: A boolean indicating if all required data is available.
+        """
+        if not has_manifest:
+            return False, "Manifest is required for insight to run."
+
+        return True, ""

datapilot/core/platforms/dbt/insights/checks/check_column_desc_are_same.py

@@ -0,0 +1,105 @@
+from typing import ClassVar
+from typing import List
+
+from datapilot.core.insights.utils import get_severity
+from datapilot.core.platforms.dbt.insights.checks.base import ChecksInsight
+from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult
+from datapilot.core.platforms.dbt.insights.schema import DBTModelInsightResponse
+from datapilot.core.platforms.dbt.schemas.manifest import AltimateResourceType
+
+
+class CheckColumnDescAreSame(ChecksInsight):
+    NAME = "Column descriptions consistent for same column names"
+    ALIAS = "column_descriptions_are_same"
+    DESCRIPTION = "Column description for the same column name should be same "
+    REASON_TO_FLAG = (
+        "Different descriptions for the same column names can lead to confusion and hinder effective data "
+        "modeling and analysis. It's important to have consistent column descriptions."
+    )
+    FILES_REQUIRED: ClassVar = ["Manifest", "Catalog"]
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.columns_with_different_desc = []
+        self.columns = {}
+        self.column_node_map = {}
+
+    def _build_failure_result(
+        self,
+    ) -> DBTInsightResult:
+        """
+        Build failure result for the insight if a column has a different description in multiple models or sources.
+
+        :return: An instance of InsightResult containing failure message and recommendation.
+        """
+
+        failure_message = "The following models or sources have different descriptions for some columns:\n"
+        for col_name in self.columns_with_different_desc:
+            failure_message += f"- {self.column_node_map[col_name]} (column: {col_name})\n"
+
+        recommendation = "Ensure that the description for the columns is consistent across all instances."
+
+        return DBTInsightResult(
+            type=self.TYPE,
+            name=self.NAME,
+            message=failure_message,
+            recommendation=recommendation,
+            reason_to_flag=self.REASON_TO_FLAG,
+            metadata={
+                "columns_with_diff_desc": self.columns_with_different_desc,
+            },
+        )
+
+    def generate(self, *args, **kwargs) -> List[DBTModelInsightResponse]:
+        """
+        Generate a list of InsightResponse objects for each model in the DBT project,
+        identifying models with columns that have different descriptions for the same column name.
+        :return: A list of InsightResponse objects.
+        """
+
+        insights = []
+        for node_id, node in self.nodes.items():
+            if self.should_skip_model(node_id):
+                self.logger.debug(f"Skipping model {node_id} as it is not enabled for selected models")
+                continue
+            if node.resource_type == AltimateResourceType.model:
+                self._get_columns_with_different_desc(node_id)
+
+        for node_id, node in self.sources.items():
+            if self.should_skip_model(node_id):
+                self.logger.debug(f"Skipping model {node_id} as it is not enabled for selected models")
+                continue
+            elif node.resource_type == AltimateResourceType.source:
+                self._get_columns_with_different_desc(node_id)
+
+        if self.columns_with_different_desc:
+            insights.append(
+                DBTModelInsightResponse(
+                    unique_id=node_id,
+                    package_name=node.package_name,
+                    path=node.original_file_path,
+                    original_file_path=node.original_file_path,
+                    insight=self._build_failure_result(),
+                    severity=get_severity(self.config, self.ALIAS, self.DEFAULT_SEVERITY),
+                )
+            )
+
+        return insights
+
+    def _get_columns_with_different_desc(self, node_id):
+        """
+        Get the list of models or sources that have different descriptions for the same column name.
+        :param node_id: The unique ID of the node.
+        """
+        for column_name, column_node in self.get_node(node_id).columns.items():
+            if column_name in self.column_node_map:
+                self.column_node_map[column_name].append(node_id)
+            else:
+                self.column_node_map[column_name] = [node_id]
+
+            if column_name in self.columns:
+                if column_node.description != self.columns[column_name]:
+                    if column_name not in self.columns_with_different_desc:
+                        self.columns_with_different_desc.append(column_name)
+            else:
+                self.columns[column_name] = column_node.description

datapilot/core/platforms/dbt/insights/checks/check_column_name_contract.py

@@ -0,0 +1,154 @@
+import re
+from typing import ClassVar
+from typing import List
+from typing import Sequence
+from typing import Tuple
+
+from datapilot.core.insights.utils import get_severity
+from datapilot.core.platforms.dbt.insights.checks.base import ChecksInsight
+from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult
+from datapilot.core.platforms.dbt.insights.schema import DBTModelInsightResponse
+from datapilot.core.platforms.dbt.schemas.manifest import AltimateResourceType
+from datapilot.core.platforms.dbt.wrappers.catalog.wrapper import BaseCatalogWrapper
+from datapilot.utils.formatting.utils import numbered_list
+
+
+class CheckColumnNameContract(ChecksInsight):
+    NAME = "Column name follows contract pattern"
+    ALIAS = "column_name_contract"
+    DESCRIPTION = "Column names should adhere to the contract pattern defined for the data type. "
+    REASON_TO_FLAG = (
+        "Column names that do not adhere to the contract can lead to confusion and hinder effective data "
+        "modeling and analysis. It's important to maintain consistent column naming conventions."
+    )
+    FAILURE_MESSAGE = (
+        "The following columns in the model `{model_unique_id}` do not adhere to the contract:\n{columns}. "
+        "Inconsistent column naming conventions can impede understanding and usage of the model."
+    )
+    RECOMMENDATION = (
+        "Update the column names listed above in the model `{model_unique_id}` to adhere to the contract. "
+        "Consistent column naming conventions provide valuable context and aids in data understanding and collaboration."
+    )
+    PATTERN_STR = "pattern"
+    DATATYPE_STR = "dtype"
+    PATTERNS_LIST_STR = "patterns"
+    DEFAULT_PATTERN_STR = "default_pattern"
+    FILES_REQUIRED: ClassVar = ["Manifest", "Catalog"]
+
+    def __init__(self, catalog_wrapper: BaseCatalogWrapper, *args, **kwargs):
+        self.catalog = catalog_wrapper
+        super().__init__(*args, **kwargs)
+
+    def generate(self, *args, **kwargs) -> List[DBTModelInsightResponse]:
+        self.default_pattern = self.get_check_config(self.DEFAULT_PATTERN_STR)
+        datatype_configs = self.get_check_config(self.PATTERNS_LIST_STR)
+        # Patterns : [{"pattern": "^[a-z_]+$", "dtype": "string"}, {"pattern": "^[a-z_]+$", "dtype": "string"}]
+        if not datatype_configs:
+            self.logger.debug(f"Column name contract not found in insight config for {self.ALIAS}. Skipping insight.")
+            return []
+        self.patterns = {
+            pattern.get(self.DATATYPE_STR).lower(): pattern.get(self.PATTERN_STR)
+            for pattern in datatype_configs
+            if pattern.get(self.PATTERN_STR) and pattern.get(self.DATATYPE_STR)
+        }
+        if not self.patterns:
+            self.logger.debug(f"Column name contract not found in insight config for {self.ALIAS}")
+            return []
+
+        insights = []
+        for node_id, node in self.nodes.items():
+            if self.should_skip_model(node_id):
+                self.logger.debug(f"Skipping model {node_id} as it is not enabled for selected models")
+                continue
+            if node.resource_type == AltimateResourceType.model:
+                columns = self._get_columns_with_contract_violation(node_id)
+                if columns:
+                    insights.append(
+                        DBTModelInsightResponse(
+                            unique_id=node_id,
+                            package_name=node.package_name,
+                            path=node.original_file_path,
+                            original_file_path=node.original_file_path,
+                            insight=self._build_failure_result(node_id, columns),
+                            severity=get_severity(self.config, self.ALIAS, self.DEFAULT_SEVERITY),
+                        )
+                    )
+        return insights
+
+    def _build_failure_result(self, model_unique_id: str, columns: Sequence[str]) -> DBTInsightResult:
+        failure_message = self.FAILURE_MESSAGE.format(
+            columns=numbered_list(columns),
+            model_unique_id=model_unique_id,
+        )
+        recommendation = self.RECOMMENDATION.format(model_unique_id=model_unique_id)
+
+        return DBTInsightResult(
+            type=self.TYPE,
+            name=self.NAME,
+            message=failure_message,
+            recommendation=recommendation,
+            reason_to_flag=self.REASON_TO_FLAG,
+            metadata={"columns": columns, "model_unique_id": model_unique_id},
+        )
+
+    def _get_columns_in_model(self, node_id) -> List[str]:
+        if node_id not in self.catalog.get_schema():
+            return []
+        return self.catalog.get_schema()[node_id].keys()
+
+    def _get_columns_with_contract_violation(self, node_id) -> Sequence[str]:
+        columns = []
+        for col in self._get_columns_in_model(node_id):
+            schema = self.catalog.get_schema()[node_id]
+            col_name = col.lower()
+            col_type = schema[col]
+            if col_type.lower() in self.patterns:
+                if re.match(self.patterns[col_type.lower()], col_name, re.IGNORECASE) is None:
+                    columns.append(col)
+            if self.default_pattern and re.match(self.default_pattern, col_name, re.IGNORECASE) is None:
+                columns.append(col)
+        return columns
+
+    @classmethod
+    def has_all_required_data(cls, has_manifest: bool, has_catalog: bool, **kwargs) -> Tuple[bool, str]:
+        if not has_manifest:
+            return False, "Manifest is required for insight to run."
+
+        if not has_catalog:
+            return False, "Catalog is required for insight to run."
+
+        return True, ""
+
+    @classmethod
+    def get_config_schema(cls):
+        config_schema = super().get_config_schema()
+        config_schema["config"] = {
+            "$schema": "http://json-schema.org/draft-07/schema#",
+            "type": "object",
+            "properties": {
+                cls.DEFAULT_PATTERN_STR: {
+                    "type": "string",
+                    "description": "The regex pattern to check the column name against if no pattern is found for the data type",
+                    "default": "^[a-z_]+$",
+                },
+                cls.PATTERNS_LIST_STR: {
+                    "type": "array",
+                    "items": {
+                        "type": "object",
+                        "properties": {
+                            cls.PATTERN_STR: {"type": "string", "description": "The regex pattern to check the column name against"},
+                            cls.DATATYPE_STR: {
+                                "type": "string",
+                                "description": "The data type for which the pattern is defined",
+                            },
+                        },
+                        "required": [cls.PATTERN_STR, cls.DATATYPE_STR],
+                    },
+                    "description": "A list of patterns to check the column name against for different data types",
+                    "default": [],
+                },
+            },
+            "required": [cls.DEFAULT_PATTERN_STR, cls.PATTERNS_LIST_STR],
+        }
+        config_schema["files_required"] = cls.FILES_REQUIRED
+        return config_schema

datapilot/core/platforms/dbt/insights/checks/check_macro_args_have_desc.py

@@ -0,0 +1,75 @@
+from typing import List
+
+from datapilot.core.insights.utils import get_severity
+from datapilot.core.platforms.dbt.insights.checks.base import ChecksInsight
+from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult
+from datapilot.core.platforms.dbt.insights.schema import DBTModelInsightResponse
+from datapilot.core.platforms.dbt.schemas.manifest import AltimateResourceType
+
+
+class CheckMacroArgsHaveDesc(ChecksInsight):
+    NAME = "Check macro arguments has description"
+    ALIAS = "check_macro_args_have_desc"
+    DESCRIPTION = "Macro arguments should have a description. "
+    REASON_TO_FLAG = "Clear descriptions for macro arguments are crucial as they prevent misunderstandings, enhance user comprehension, and simplify maintenance. This leads to more accurate data analysis and efficient workflows."
+
+    def _build_failure_result(
+        self,
+        node_id: str,
+    ) -> DBTInsightResult:
+        """
+        Build failure result for the insight if a macro doesn't have a description.
+
+        :return: An instance of InsightResult containing failure message and recommendation.
+        """
+
+        failure_message = f"The macro `{node_id}` does not have a description."
+        recommendation = "Add a description to the macro to help in understanding the purpose of the macro."
+
+        return DBTInsightResult(
+            type=self.TYPE,
+            name=self.NAME,
+            message=failure_message,
+            recommendation=recommendation,
+            reason_to_flag=self.REASON_TO_FLAG,
+        )
+
+    def generate(self, *args, **kwargs) -> List[DBTModelInsightResponse]:
+        """
+        Generate a list of InsightResponse objects for each model in the DBT project,
+        identifying macros whose arguments don't have descriptions.
+        :return: A list of InsightResponse objects.
+        """
+
+        insights = []
+        for macro_id, macro in self.macros.items():
+            if self.should_skip_model(macro_id):
+                self.logger.debug(f"Skipping model {macro_id} as it is not enabled for selected models")
+                continue
+            if macro.resource_type == AltimateResourceType.macro:
+                if not self._check_macro_args_have_desc(macro_id):
+                    insights.append(
+                        DBTModelInsightResponse(
+                            unique_id=macro_id,
+                            package_name=macro.package_name,
+                            original_file_path=macro.original_file_path,
+                            path=macro.original_file_path,
+                            insight=self._build_failure_result(macro_id),
+                            severity=get_severity(self.config, self.ALIAS, self.DEFAULT_SEVERITY),
+                        )
+                    )
+
+        return insights
+
+    def _check_macro_args_have_desc(self, macro_id) -> bool:
+        """
+        Check if the macro has descriptions for its arguments.
+        """
+        macro = self.get_node(macro_id)
+        if not macro:
+            return True
+        args = macro.arguments or []
+        for arg in args:
+            if not arg.description:
+                return False
+        return True

datapilot/core/platforms/dbt/insights/checks/check_macro_has_desc.py

@@ -0,0 +1,63 @@
+from typing import List
+
+from datapilot.core.insights.utils import get_severity
+from datapilot.core.platforms.dbt.insights.checks.base import ChecksInsight
+from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult
+from datapilot.core.platforms.dbt.insights.schema import DBTModelInsightResponse
+from datapilot.core.platforms.dbt.schemas.manifest import AltimateResourceType
+
+
+class CheckMacroHasDesc(ChecksInsight):
+    NAME = "Macro has documentation"
+    ALIAS = "check_macro_has_desc"
+    DESCRIPTION = "Macros should be documented."
+    REASON_TO_FLAG = "Undocumented macros can cause misunderstandings and inefficiencies in data modeling and analysis, as they make it difficult to understand their purpose and usage. Clear descriptions are vital for accuracy and streamlined workflow."
+
+    def _build_failure_result(
+        self,
+        node_id: str,
+    ) -> DBTInsightResult:
+        """
+        Build failure result for the insight if a macro doesn't have a description.
+
+        :return: An instance of InsightResult containing failure message and recommendation.
+        """
+
+        failure_message = f"The macro `{node_id}` does not have a description."
+        recommendation = "Add a description to the macro to help in understanding the purpose of the macro."
+
+        return DBTInsightResult(
+            type=self.TYPE,
+            name=self.NAME,
+            message=failure_message,
+            recommendation=recommendation,
+            reason_to_flag=self.REASON_TO_FLAG,
+            metadata={"macro_unique_id": node_id},
+        )
+
+    def generate(self, *args, **kwargs) -> List[DBTModelInsightResponse]:
+        """
+        Generate a list of InsightResponse objects for each model in the DBT project,
+        identifying macros that don't have descriptions.
+        :return: A list of InsightResponse objects.
+        """
+
+        insights = []
+        for macro_id, macro in self.macros.items():
+            if self.should_skip_model(macro_id):
+                self.logger.debug(f"Skipping model {macro_id} as it is not enabled for selected models")
+                continue
+            if macro.resource_type == AltimateResourceType.macro:
+                if not macro.description:
+                    insights.append(
+                        DBTModelInsightResponse(
+                            unique_id=macro_id,
+                            package_name=macro.package_name,
+                            original_file_path=macro.original_file_path,
+                            path=macro.original_file_path,
+                            insight=self._build_failure_result(macro_id),
+                            severity=get_severity(self.config, self.ALIAS, self.DEFAULT_SEVERITY),
+                        )
+                    )
+
+        return insights

datapilot/core/platforms/dbt/insights/checks/check_model_has_all_columns.py

@@ -0,0 +1,96 @@
+from typing import ClassVar
+from typing import List
+from typing import Sequence
+from typing import Set
+from typing import Tuple
+
+from datapilot.core.insights.utils import get_severity
+from datapilot.core.platforms.dbt.insights.checks.base import ChecksInsight
+from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult
+from datapilot.core.platforms.dbt.insights.schema import DBTModelInsightResponse
+from datapilot.core.platforms.dbt.schemas.manifest import AltimateResourceType
+from datapilot.core.platforms.dbt.wrappers.catalog.wrapper import BaseCatalogWrapper
+from datapilot.utils.formatting.utils import numbered_list
+
+
+class CheckModelHasAllColumns(ChecksInsight):
+    NAME = "Model has all columns as per catalog"
+    ALIAS = "check_model_has_all_columns"
+    DESCRIPTION = "Models should have all the columns as per the catalog."
+    REASON_TO_FLAG = (
+        "Missing columns in the model can lead to data integrity issues and inconsistency in analysis. "
+        "It's important to ensure that the model has all the required columns as per the catalog definition."
+    )
+    FILES_REQUIRED: ClassVar = ["Manifest", "Catalog"]
+
+    def __init__(self, catalog_wrapper: BaseCatalogWrapper, *args, **kwargs):
+        self.catalog = catalog_wrapper
+        super().__init__(*args, **kwargs)
+
+    def generate(self, *args, **kwargs) -> List[DBTModelInsightResponse]:
+        insights = []
+        for node_id, node in self.nodes.items():
+            if self.should_skip_model(node_id):
+                self.logger.debug(f"Skipping model {node_id} as it is not enabled for selected models")
+                continue
+            if node.resource_type == AltimateResourceType.model:
+                missing_columns = self._check_model_columns(node_id)
+                if missing_columns:
+                    insights.append(
+                        DBTModelInsightResponse(
+                            unique_id=node_id,
+                            package_name=node.package_name,
+                            path=node.original_file_path,
+                            original_file_path=node.original_file_path,
+                            insight=self._build_failure_result(node_id, missing_columns),
+                            severity=get_severity(self.config, self.ALIAS, self.DEFAULT_SEVERITY),
+                        )
+                    )
+        return insights
+
+    def _build_failure_result(self, model_unique_id: str, columns: Sequence[str]) -> DBTInsightResult:
+        failure_message = (
+            "The following columns in the model `{model_unique_id}` are missing:\n{columns}. "
+            "Ensure that the model includes all the required columns."
+        )
+        recommendation = (
+            "Add the missing columns listed above in the model `{model_unique_id}`. "
+            "Ensuring that the model has all the required columns helps in maintaining data integrity and consistency."
+        )
+
+        return DBTInsightResult(
+            type=self.TYPE,
+            name=self.NAME,
+            message=failure_message.format(
+                columns=numbered_list(columns),
+                model_unique_id=model_unique_id,
+            ),
+            recommendation=recommendation.format(model_unique_id=model_unique_id),
+            reason_to_flag=self.REASON_TO_FLAG,
+            metadata={"columns": columns, "model_unique_id": model_unique_id},
+        )
+
+    def _check_model_columns(self, node_id) -> Tuple[int, Set[str]]:
+        missing_columns = set()
+        schema = self.catalog.get_schema()
+        if node_id not in schema:
+            return missing_columns
+        catalog_columns = schema[node_id].keys()
+        for col_name in self.get_node(node_id).columns.keys():
+            if col_name not in catalog_columns:
+                missing_columns.add(col_name)
+        return missing_columns
+
+    @classmethod
+    def has_all_required_data(cls, has_manifest: bool, has_catalog: bool, **kwargs) -> Tuple[bool, str]:
+        if not has_manifest:
+            return False, "Manifest is required for insight to run."
+
+        if not has_catalog:
+            return False, "Catalog is required for insight to run."
+
+        return True, ""
+
+    @classmethod
+    def requires_catalog(cls) -> bool:
+        return True