altimate-datapilot-cli 0.0.13__py3-none-any.whl → 0.0.15__py3-none-any.whl

datapilot/core/platforms/dbt/insights/sql/sql_check.py

@@ -0,0 +1,101 @@
+import inspect
+from typing import List
+
+from sqlglot import parse_one
+from sqlglot.optimizer.eliminate_ctes import eliminate_ctes
+from sqlglot.optimizer.eliminate_joins import eliminate_joins
+from sqlglot.optimizer.eliminate_subqueries import eliminate_subqueries
+from sqlglot.optimizer.normalize import normalize
+from sqlglot.optimizer.pushdown_projections import pushdown_projections
+from sqlglot.optimizer.qualify import qualify
+from sqlglot.optimizer.unnest_subqueries import unnest_subqueries
+
+from datapilot.core.insights.sql.base.insight import SqlInsight
+from datapilot.core.insights.utils import get_severity
+from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult
+from datapilot.core.platforms.dbt.insights.schema import DBTModelInsightResponse
+
+RULES = (
+    pushdown_projections,
+    normalize,
+    unnest_subqueries,
+    eliminate_subqueries,
+    eliminate_joins,
+    eliminate_ctes,
+)
+
+
+class SqlCheck(SqlInsight):
+    """
+    This class identifies DBT models with SQL optimization issues.
+    """
+
+    NAME = "sql optimization issues"
+    ALIAS = "check_sql_optimization"
+    DESCRIPTION = "Checks if the model has SQL optimization issues. "
+    REASON_TO_FLAG = "The query can be optimized."
+    FAILURE_MESSAGE = "The query for model `{model_unique_id}` has optimization opportunities:\n{rule_name}. "
+    RECOMMENDATION = "Please adapt the query of the model `{model_unique_id}` as in following example:\n{optimized_sql}"
+
+    def _build_failure_result(self, model_unique_id: str, rule_name: str, optimized_sql: str) -> DBTInsightResult:
+        """
+        Constructs a failure result for a given model with sql optimization issues.
+        :param model_unique_id: The unique id of the dbt model.
+        :param rule_name: The rule that generated this failure result.
+        :param optimized_sql: The optimized sql.
+        :return: An instance of DBTInsightResult containing failure details.
+        """
+        failure_message = self.FAILURE_MESSAGE.format(model_unique_id=model_unique_id, rule_name=rule_name)
+        recommendation = self.RECOMMENDATION.format(model_unique_id=model_unique_id, optimized_sql=optimized_sql)
+        return DBTInsightResult(
+            type=self.TYPE,
+            name=self.NAME,
+            message=failure_message,
+            recommendation=recommendation,
+            reason_to_flag=self.REASON_TO_FLAG,
+            metadata={"model_unique_id": model_unique_id, "rule_name": rule_name},
+        )
+
+    def generate(self, *args, **kwargs) -> List[DBTModelInsightResponse]:
+        """
+        Generates insights for each DBT model in the project, focusing on sql optimization issues.
+
+        :return: A list of DBTModelInsightResponse objects with insights for each model.
+        """
+        self.logger.debug("Generating sql insights for DBT models")
+        insights = []
+
+        possible_kwargs = {
+            "db": None,
+            "catalog": None,
+            "dialect": self.adapter_type,
+            "isolate_tables": True,  # needed for other optimizations to perform well
+            "quote_identifiers": False,
+            **kwargs,
+        }
+        for node_id, node in self.nodes.items():
+            try:
+                compiled_query = node.compiled_code
+                if compiled_query:
+                    parsed_query = parse_one(compiled_query, dialect=self.adapter_type)
+                    qualified = qualify(parsed_query, **possible_kwargs)
+                    changed = qualified.copy()
+                    for rule in RULES:
+                        original = changed.copy()
+                        rule_params = inspect.getfullargspec(rule).args
+                        rule_kwargs = {param: possible_kwargs[param] for param in rule_params if param in possible_kwargs}
+                        changed = rule(changed, **rule_kwargs)
+                        if changed.sql() != original.sql():
+                            insights.append(
+                                DBTModelInsightResponse(
+                                    unique_id=node_id,
+                                    package_name=node.package_name,
+                                    path=node.original_file_path,
+                                    original_file_path=node.original_file_path,
+                                    insight=self._build_failure_result(node_id, rule.__name__, changed.sql()),
+                                    severity=get_severity(self.config, self.ALIAS, self.DEFAULT_SEVERITY),
+                                )
+                            )
+            except Exception as e:
+                self.logger.error(e)
+        return insights

datapilot/core/platforms/dbt/schemas/catalog.py

@@ -5,13 +5,16 @@ from typing import List
 from typing import Optional
 from typing import Union
 
-from pydantic.config import Extra
+from pydantic import ConfigDict
 from pydantic.main import BaseModel
 
+from datapilot.constants import Extra
+
 
 class AltimateCatalogMetadata(BaseModel):
-    class Config:
-        extra = Extra.forbid
+    model_config = ConfigDict(
+        extra=Extra.forbid,
+    )
 
     dbt_schema_version: Optional[str] = "https://schemas.getdbt.com/dbt/catalog/v1.json"
     dbt_version: Optional[str] = "0.19.0"
@@ -21,8 +24,9 @@ class AltimateCatalogMetadata(BaseModel):
 
 
 class AltimateCatalogTableMetadata(BaseModel):
-    class Config:
-        extra = Extra.forbid
+    model_config = ConfigDict(
+        extra=Extra.forbid,
+    )
 
     type: str
     database: Optional[Optional[str]] = None
@@ -33,8 +37,9 @@ class AltimateCatalogTableMetadata(BaseModel):
 
 
 class AltimateCatalogColumnMetadata(BaseModel):
-    class Config:
-        extra = Extra.forbid
+    model_config = ConfigDict(
+        extra=Extra.forbid,
+    )
 
     type: str
     comment: Optional[Optional[str]] = None
@@ -43,8 +48,9 @@ class AltimateCatalogColumnMetadata(BaseModel):
 
 
 class AltimateCatalogStatsItem(BaseModel):
-    class Config:
-        extra = Extra.forbid
+    model_config = ConfigDict(
+        extra=Extra.forbid,
+    )
 
     id: str
     label: str
@@ -54,8 +60,9 @@ class AltimateCatalogStatsItem(BaseModel):
 
 
 class AltimateCatalogTable(BaseModel):
-    class Config:
-        extra = Extra.forbid
+    model_config = ConfigDict(
+        extra=Extra.forbid,
+    )
 
     metadata: AltimateCatalogTableMetadata
     columns: Dict[str, AltimateCatalogColumnMetadata]
@@ -64,8 +71,9 @@ class AltimateCatalogTable(BaseModel):
 
 
 class AltimateCatalogCatalogV1(BaseModel):
-    class Config:
-        extra = Extra.forbid
+    model_config = ConfigDict(
+        extra=Extra.forbid,
+    )
 
     metadata: AltimateCatalogMetadata
     nodes: Dict[str, AltimateCatalogTable]

datapilot/core/platforms/dbt/schemas/manifest.py

@@ -19,7 +19,9 @@ from dbt_artifacts_parser.parsers.manifest.manifest_v10 import ManifestV10
 from dbt_artifacts_parser.parsers.manifest.manifest_v11 import ManifestV11
 from dbt_artifacts_parser.parsers.manifest.manifest_v11 import SupportedLanguage
 from pydantic import BaseModel
-from pydantic import Extra
+from pydantic import ConfigDict
+
+from datapilot.constants import Extra
 
 
 class DBTVersion(BaseModel):
@@ -46,16 +48,17 @@ Catalog = CatalogV1
 
 
 class AltimateDocs(BaseModel):
-    class Config:
-        extra = Extra.forbid
+    model_config = ConfigDict(
+        extra=Extra.forbid,
+    )
 
     show: Optional[bool] = True
     node_color: Optional[Optional[str]] = None
 
 
 class AltimateDependsOn(BaseModel):
-    nodes: Optional[List[str]]
-    macros: Optional[List[str]]
+    nodes: Optional[List[str]] = None
+    macros: Optional[List[str]] = None
 
 
 class AltimateManifestColumnInfo(BaseModel):
@@ -98,8 +101,9 @@ class AltimateAccess(Enum):
 
 
 class AltimateDBTContract(BaseModel):
-    class Config:
-        extra = Extra.forbid
+    model_config = ConfigDict(
+        extra=Extra.forbid,
+    )
 
     enforced: Optional[bool] = False
     alias_types: Optional[bool] = True
@@ -107,8 +111,9 @@ class AltimateDBTContract(BaseModel):
 
 
 class AltimateHook(BaseModel):
-    class Config:
-        extra = Extra.forbid
+    model_config = ConfigDict(
+        extra=Extra.forbid,
+    )
 
     sql: str
     transaction: Optional[bool] = True
@@ -117,8 +122,9 @@ class AltimateHook(BaseModel):
 
 # TODO: Need to add the rest of the fields
 class AltimateNodeConfig(BaseModel):
-    class Config:
-        extra = Extra.allow
+    model_config = ConfigDict(
+        extra=Extra.allow,
+    )
 
     _extra: Optional[Dict[str, Any]] = None
     enabled: Optional[bool] = True
@@ -203,8 +209,9 @@ class AltimateSourceConfig(BaseModel):
 
 
 class AltimateDeferRelation(BaseModel):
-    class Config:
-        extra = Extra.forbid
+    model_config = ConfigDict(
+        extra=Extra.forbid,
+    )
 
     database: Optional[str]
     schema_name: str
@@ -213,8 +220,9 @@ class AltimateDeferRelation(BaseModel):
 
 
 class AltimateSeedConfig(BaseModel):
-    class Config:
-        extra = Extra.allow
+    model_config = ConfigDict(
+        extra=Extra.allow,
+    )
 
     _extra: Optional[Dict[str, Any]] = None
     enabled: Optional[bool] = True
@@ -314,8 +322,9 @@ class AltimateExposureType(Enum):
 
 
 class AltimateOwner(BaseModel):
-    class Config:
-        extra = Extra.allow
+    model_config = ConfigDict(
+        extra=Extra.allow,
+    )
 
     _extra: Optional[Dict[str, Any]] = None
     email: Optional[Optional[str]] = None
@@ -329,8 +338,9 @@ class AltimateMaturityEnum(Enum):
 
 
 class AltimateRefArgs(BaseModel):
-    class Config:
-        extra = Extra.forbid
+    model_config = ConfigDict(
+        extra=Extra.forbid,
+    )
 
     name: str
     package: Optional[Optional[str]] = None
@@ -338,8 +348,9 @@ class AltimateRefArgs(BaseModel):
 
 
 class AltimateExposureConfig(BaseModel):
-    class Config:
-        extra = Extra.allow
+    model_config = ConfigDict(
+        extra=Extra.allow,
+    )
 
     _extra: Optional[Dict[str, Any]] = None
     enabled: Optional[bool] = True
@@ -371,8 +382,9 @@ class AltimateManifestExposureNode(BaseModel):
 
 
 class AltimateTestMetadata(BaseModel):
-    class Config:
-        extra = Extra.forbid
+    model_config = ConfigDict(
+        extra=Extra.forbid,
+    )
 
     name: str
     kwargs: Optional[Dict[str, Any]] = None
@@ -380,8 +392,9 @@ class AltimateTestMetadata(BaseModel):
 
 
 class AltimateTestConfig(BaseModel):
-    class Config:
-        extra = Extra.allow
+    model_config = ConfigDict(
+        extra=Extra.allow,
+    )
 
     _extra: Optional[Dict[str, Any]] = None
     enabled: Optional[bool] = True
@@ -433,8 +446,9 @@ class AltimateManifestTestNode(BaseModel):
 
 
 class AltimateMacroArgument(BaseModel):
-    class Config:
-        extra = Extra.forbid
+    model_config = ConfigDict(
+        extra=Extra.forbid,
+    )
 
     name: str
     type: Optional[Optional[str]] = None

datapilot/core/platforms/dbt/wrappers/manifest/v10/wrapper.py

@@ -1,4 +1,5 @@
 from typing import Dict
+from typing import Optional
 from typing import Set
 
 from dbt_artifacts_parser.parsers.manifest.manifest_v10 import GenericTestNode
@@ -67,6 +68,7 @@ class ManifestV10Wrapper(BaseManifestWrapper):
             depends_on_macros = node.depends_on.macros if node.depends_on else None
             compiled_path = node.compiled_path
             compiled = node.compiled
+            compiled_code = node.compiled_code
             raw_code = node.raw_code
             language = node.language
             contract = AltimateDBTContract(**node.contract.__dict__) if node.contract else None
@@ -114,6 +116,7 @@ class ManifestV10Wrapper(BaseManifestWrapper):
             contract=contract,
             meta=node.meta,
             patch_path=node.patch_path,
+            access=node.access.value,
         )
 
     def _get_source(self, source: SourceNode) -> AltimateManifestSourceNode:
@@ -131,10 +134,10 @@ class ManifestV10Wrapper(BaseManifestWrapper):
             source_description=source.source_description,
             loader=source.loader,
             identifier=source.identifier,
-            quoting=AltimateQuoting(**source.quoting.dict()) if source.quoting else None,
+            quoting=AltimateQuoting(**source.quoting.model_dump()) if source.quoting else None,
             loaded_at_field=source.loaded_at_field,
-            freshness=AltimateFreshnessThreshold(**source.freshness.dict()) if source.freshness else None,
-            external=AltimateExternalTable(**source.external.dict()) if source.external else None,
+            freshness=AltimateFreshnessThreshold(**source.freshness.model_dump()) if source.freshness else None,
+            external=AltimateExternalTable(**source.external.model_dump()) if source.external else None,
             description=source.description,
             columns={
                 name: AltimateManifestColumnInfo(
@@ -151,7 +154,7 @@ class ManifestV10Wrapper(BaseManifestWrapper):
             relation_name=source.relation_name,
             source_meta=source.source_meta,
             tags=source.tags,
-            config=AltimateSourceConfig(**source.config.dict()) if source.config else None,
+            config=AltimateSourceConfig(**source.config.model_dump()) if source.config else None,
             patch_path=source.patch_path,
             unrendered_config=source.unrendered_config,
             created_at=source.created_at,
@@ -175,9 +178,9 @@ class ManifestV10Wrapper(BaseManifestWrapper):
             ),
             description=macro.description,
             meta=macro.meta,
-            docs=macro.docs,
+            docs=macro.docs.model_dump() if macro.docs else None,
             patch_path=macro.patch_path,
-            arguments=[AltimateMacroArgument(**arg.dict()) for arg in macro.arguments] if macro.arguments else None,
+            arguments=[AltimateMacroArgument(**arg.model_dump()) for arg in macro.arguments] if macro.arguments else None,
             created_at=macro.created_at,
             supported_languages=macro.supported_languages,
         )
@@ -192,22 +195,24 @@ class ManifestV10Wrapper(BaseManifestWrapper):
             unique_id=exposure.unique_id,
             fqn=exposure.fqn,
             type=AltimateExposureType(exposure.type.value) if exposure.type else None,
-            owner=AltimateOwner(**exposure.owner.dict()) if exposure.owner else None,
+            owner=AltimateOwner(**exposure.owner.model_dump()) if exposure.owner else None,
             description=exposure.description,
             label=exposure.label,
             maturity=AltimateMaturityEnum(exposure.maturity.value) if exposure.maturity else None,
             meta=exposure.meta,
             tags=exposure.tags,
-            config=AltimateSourceConfig(**exposure.config.dict()) if exposure.config else None,
+            config=AltimateSourceConfig(**exposure.config.model_dump()) if exposure.config else None,
             unrendered_config=exposure.unrendered_config,
             url=exposure.url,
-            depends_on=AltimateDependsOn(
-                nodes=exposure.depends_on.nodes,
-                macros=exposure.depends_on.macros,
-            )
-            if exposure.depends_on
-            else None,
-            refs=[AltimateRefArgs(**ref.dict()) for ref in exposure.refs] if exposure.refs else None,
+            depends_on=(
+                AltimateDependsOn(
+                    nodes=exposure.depends_on.nodes,
+                    macros=exposure.depends_on.macros,
+                )
+                if exposure.depends_on
+                else None
+            ),
+            refs=[AltimateRefArgs(**ref.model_dump()) for ref in exposure.refs] if exposure.refs else None,
             sources=exposure.sources,
             metrics=exposure.metrics,
             created_at=exposure.created_at,
@@ -217,7 +222,7 @@ class ManifestV10Wrapper(BaseManifestWrapper):
         test_metadata = None
         if isinstance(test, GenericTestNode):
             test_type = GENERIC
-            test_metadata = AltimateTestMetadata(**test.test_metadata.dict()) if test.test_metadata else None
+            test_metadata = AltimateTestMetadata(**test.test_metadata.model_dump()) if test.test_metadata else None
         elif isinstance(test, SingularTestNode):
             test_type = SINGULAR
         else:
@@ -233,42 +238,48 @@ class ManifestV10Wrapper(BaseManifestWrapper):
             unique_id=test.unique_id,
             fqn=test.fqn,
             alias=test.alias,
-            checksum=AltimateFileHash(
-                name=test.checksum.name,
-                checksum=test.checksum.checksum,
-            )
-            if test.checksum
-            else None,
-            config=AltimateTestConfig(**test.config.dict()) if test.config else None,
+            checksum=(
+                AltimateFileHash(
+                    name=test.checksum.name,
+                    checksum=test.checksum.checksum,
+                )
+                if test.checksum
+                else None
+            ),
+            config=AltimateTestConfig(**test.config.model_dump()) if test.config else None,
             description=test.description,
             tags=test.tags,
-            columns={
-                name: AltimateManifestColumnInfo(
-                    name=column.name,
-                    description=column.description,
-                    meta=column.meta,
-                    data_type=column.data_type,
-                    quote=column.quote,
-                    tags=column.tags,
-                )
-                for name, column in test.columns.items()
-            }
-            if test.columns
-            else None,
+            columns=(
+                {
+                    name: AltimateManifestColumnInfo(
+                        name=column.name,
+                        description=column.description,
+                        meta=column.meta,
+                        data_type=column.data_type,
+                        quote=column.quote,
+                        tags=column.tags,
+                    )
+                    for name, column in test.columns.items()
+                }
+                if test.columns
+                else None
+            ),
             meta=test.meta,
             relation_name=test.relation_name,
             group=test.group,
             raw_code=test.raw_code,
             language=test.language,
-            refs=[AltimateRefArgs(**ref.dict()) for ref in test.refs] if test.refs else None,
+            refs=[AltimateRefArgs(**ref.model_dump()) for ref in test.refs] if test.refs else None,
             sources=test.sources,
             metrics=test.metrics,
-            depends_on=AltimateDependsOn(
-                nodes=test.depends_on.nodes,
-                macros=test.depends_on.macros,
-            )
-            if test.depends_on
-            else None,
+            depends_on=(
+                AltimateDependsOn(
+                    nodes=test.depends_on.nodes,
+                    macros=test.depends_on.macros,
+                )
+                if test.depends_on
+                else None
+            ),
             compiled_path=test.compiled_path,
             compiled=test.compiled,
             compiled_code=test.compiled_code,
@@ -286,31 +297,35 @@ class ManifestV10Wrapper(BaseManifestWrapper):
             unique_id=seed.unique_id,
             fqn=seed.fqn,
             alias=seed.alias,
-            checksum=AltimateFileHash(
-                name=seed.checksum.name,
-                checksum=seed.checksum.checksum,
-            )
-            if seed.checksum
-            else None,
-            config=AltimateSeedConfig(**seed.config.dict()) if seed.config else None,
+            checksum=(
+                AltimateFileHash(
+                    name=seed.checksum.name,
+                    checksum=seed.checksum.checksum,
+                )
+                if seed.checksum
+                else None
+            ),
+            config=AltimateSeedConfig(**seed.config.model_dump()) if seed.config else None,
             description=seed.description,
             tags=seed.tags,
-            columns={
-                name: AltimateManifestColumnInfo(
-                    name=column.name,
-                    description=column.description,
-                    meta=column.meta,
-                    data_type=column.data_type,
-                    quote=column.quote,
-                    tags=column.tags,
-                )
-                for name, column in seed.columns.items()
-            }
-            if seed.columns
-            else None,
+            columns=(
+                {
+                    name: AltimateManifestColumnInfo(
+                        name=column.name,
+                        description=column.description,
+                        meta=column.meta,
+                        data_type=column.data_type,
+                        quote=column.quote,
+                        tags=column.tags,
+                    )
+                    for name, column in seed.columns.items()
+                }
+                if seed.columns
+                else None
+            ),
             meta=seed.meta,
             group=seed.group,
-            docs=seed.docs.dict() if seed.docs else None,
+            docs=seed.docs.model_dump() if seed.docs else None,
             patch_path=seed.patch_path,
             build_path=seed.build_path,
             deferred=seed.deferred,
@@ -381,6 +396,9 @@ class ManifestV10Wrapper(BaseManifestWrapper):
                 seeds[seed.unique_id] = self._get_seed(seed)
         return seeds
 
+    def get_adapter_type(self) -> Optional[str]:
+        return self.manifest.metadata.adapter_type
+
     def parent_to_child_map(self, nodes: Dict[str, AltimateManifestNode]) -> Dict[str, Set[str]]:
         """
         Current manifest contains information about parents