sqlmesh 0.227.2.dev5__py3-none-any.whl → 0.227.2.dev7__py3-none-any.whl

sqlmesh/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.227.2.dev5'
-__version_tuple__ = version_tuple = (0, 227, 2, 'dev5')
+__version__ = version = '0.227.2.dev7'
+__version_tuple__ = version_tuple = (0, 227, 2, 'dev7')
 
 __commit_id__ = commit_id = None

sqlmesh/core/context.py

@@ -115,6 +115,7 @@ from sqlmesh.core.test import (
     ModelTestMetadata,
     generate_test,
     run_tests,
+    filter_tests_by_patterns,
 )
 from sqlmesh.core.user import User
 from sqlmesh.utils import UniqueKeyDict, Verbosity
@@ -146,8 +147,8 @@ if t.TYPE_CHECKING:
     from typing_extensions import Literal
 
     from sqlmesh.core.engine_adapter._typing import (
-        BigframeSession,
         DF,
+        BigframeSession,
         PySparkDataFrame,
         PySparkSession,
         SnowparkSession,
@@ -398,6 +399,10 @@ class GenericContext(BaseContext, t.Generic[C]):
         self._standalone_audits: UniqueKeyDict[str, StandaloneAudit] = UniqueKeyDict(
             "standaloneaudits"
         )
+        self._model_test_metadata: t.List[ModelTestMetadata] = []
+        self._model_test_metadata_path_index: t.Dict[Path, t.List[ModelTestMetadata]] = {}
+        self._model_test_metadata_fully_qualified_name_index: t.Dict[str, ModelTestMetadata] = {}
+        self._models_with_tests: t.Set[str] = set()
         self._macros: UniqueKeyDict[str, ExecutableOrMacro] = UniqueKeyDict("macros")
         self._metrics: UniqueKeyDict[str, Metric] = UniqueKeyDict("metrics")
         self._jinja_macros = JinjaMacroRegistry()
@@ -636,6 +641,10 @@ class GenericContext(BaseContext, t.Generic[C]):
         self._excluded_requirements.clear()
         self._linters.clear()
         self._environment_statements = []
+        self._model_test_metadata.clear()
+        self._model_test_metadata_path_index.clear()
+        self._model_test_metadata_fully_qualified_name_index.clear()
+        self._models_with_tests.clear()
 
         for loader, project in zip(self._loaders, loaded_projects):
             self._jinja_macros = self._jinja_macros.merge(project.jinja_macros)
@@ -647,6 +656,15 @@ class GenericContext(BaseContext, t.Generic[C]):
             self._requirements.update(project.requirements)
             self._excluded_requirements.update(project.excluded_requirements)
             self._environment_statements.extend(project.environment_statements)
+            self._model_test_metadata.extend(project.model_test_metadata)
+            for metadata in project.model_test_metadata:
+                if metadata.path not in self._model_test_metadata_path_index:
+                    self._model_test_metadata_path_index[metadata.path] = []
+                self._model_test_metadata_path_index[metadata.path].append(metadata)
+                self._model_test_metadata_fully_qualified_name_index[
+                    metadata.fully_qualified_test_name
+                ] = metadata
+                self._models_with_tests.add(metadata.model_name)
 
             config = loader.config
             self._linters[config.project] = Linter.from_rules(
@@ -1049,6 +1067,11 @@ class GenericContext(BaseContext, t.Generic[C]):
         """Returns all registered standalone audits in this context."""
         return MappingProxyType(self._standalone_audits)
 
+    @property
+    def models_with_tests(self) -> t.Set[str]:
+        """Returns all models with tests in this context."""
+        return self._models_with_tests
+
     @property
     def snapshots(self) -> t.Dict[str, Snapshot]:
         """Generates and returns snapshots based on models registered in this context.
@@ -2220,7 +2243,9 @@ class GenericContext(BaseContext, t.Generic[C]):
 
             pd.set_option("display.max_columns", None)
 
-        test_meta = self.load_model_tests(tests=tests, patterns=match_patterns)
+        test_meta = self._select_tests(
+            test_meta=self._model_test_metadata, tests=tests, patterns=match_patterns
+        )
 
         result = run_tests(
             model_test_metadata=test_meta,
@@ -2782,6 +2807,33 @@ class GenericContext(BaseContext, t.Generic[C]):
             raise SQLMeshError(f"Gateway '{gateway}' not found in the available engine adapters.")
         return self.engine_adapter
 
+    def _select_tests(
+        self,
+        test_meta: t.List[ModelTestMetadata],
+        tests: t.Optional[t.List[str]] = None,
+        patterns: t.Optional[t.List[str]] = None,
+    ) -> t.List[ModelTestMetadata]:
+        """Filter pre-loaded test metadata based on tests and patterns."""
+
+        if tests:
+            filtered_tests = []
+            for test in tests:
+                if "::" in test:
+                    if test in self._model_test_metadata_fully_qualified_name_index:
+                        filtered_tests.append(
+                            self._model_test_metadata_fully_qualified_name_index[test]
+                        )
+                else:
+                    test_path = Path(test)
+                    if test_path in self._model_test_metadata_path_index:
+                        filtered_tests.extend(self._model_test_metadata_path_index[test_path])
+            test_meta = filtered_tests
+
+        if patterns:
+            test_meta = filter_tests_by_patterns(test_meta, patterns)
+
+        return test_meta
+
     def _snapshots(
         self, models_override: t.Optional[UniqueKeyDict[str, Model]] = None
     ) -> t.Dict[str, Snapshot]:

sqlmesh/core/engine_adapter/databricks.py

@@ -394,3 +394,17 @@ class DatabricksEngineAdapter(SparkEngineAdapter, GrantsFromInfoSchemaMixin):
             expressions.append(clustered_by_exp)
             properties = exp.Properties(expressions=expressions)
         return properties
+
+    def _build_column_defs(
+        self,
+        target_columns_to_types: t.Dict[str, exp.DataType],
+        column_descriptions: t.Optional[t.Dict[str, str]] = None,
+        is_view: bool = False,
+    ) -> t.List[exp.ColumnDef]:
+        # Databricks requires column types to be specified when adding column comments
+        # in CREATE MATERIALIZED VIEW statements. Override is_view to False to force
+        # column types to be included when comments are present.
+        if is_view and column_descriptions:
+            is_view = False
+
+        return super()._build_column_defs(target_columns_to_types, column_descriptions, is_view)

sqlmesh/core/linter/rules/builtin.py

@@ -129,6 +129,21 @@ class NoMissingAudits(Rule):
             return self.violation()
 
 
+class NoMissingUnitTest(Rule):
+    """All models must have a unit test found in the test/ directory yaml files"""
+
+    def check_model(self, model: Model) -> t.Optional[RuleViolation]:
+        #  External models cannot have unit tests
+        if isinstance(model, ExternalModel):
+            return None
+
+        if model.name not in self.context.models_with_tests:
+            return self.violation(
+                violation_msg=f"Model {model.name} is missing unit test(s). Please add in the tests/ directory."
+            )
+        return None
+
+
 class NoMissingExternalModels(Rule):
     """All external models must be registered in the external_models.yaml file"""
 

sqlmesh/core/loader.py

@@ -64,6 +64,7 @@ class LoadedProject:
     excluded_requirements: t.Set[str]
     environment_statements: t.List[EnvironmentStatements]
     user_rules: RuleSet
+    model_test_metadata: t.List[ModelTestMetadata]
 
 
 class CacheBase(abc.ABC):
@@ -243,6 +244,8 @@ class Loader(abc.ABC):
 
             user_rules = self._load_linting_rules()
 
+            model_test_metadata = self.load_model_tests()
+
             project = LoadedProject(
                 macros=macros,
                 jinja_macros=jinja_macros,
@@ -254,6 +257,7 @@ class Loader(abc.ABC):
                 excluded_requirements=excluded_requirements,
                 environment_statements=environment_statements,
                 user_rules=user_rules,
+                model_test_metadata=model_test_metadata,
             )
             return project
 

sqlmesh/core/test/discovery.py

@@ -20,6 +20,10 @@ class ModelTestMetadata(PydanticModel):
     def fully_qualified_test_name(self) -> str:
         return f"{self.path}::{self.test_name}"
 
+    @property
+    def model_name(self) -> str:
+        return self.body.get("model", "")
+
     def __hash__(self) -> int:
         return self.fully_qualified_test_name.__hash__()
 

{sqlmesh-0.227.2.dev5.dist-info → sqlmesh-0.227.2.dev7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.227.2.dev5
+Version: 0.227.2.dev7
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License

{sqlmesh-0.227.2.dev5.dist-info → sqlmesh-0.227.2.dev7.dist-info}/RECORD

@@ -1,5 +1,5 @@
 sqlmesh/__init__.py,sha256=v_spqQEhcnGaahp1yPvMqUIa6mhH3cs3Bc1CznxvCEA,7965
-sqlmesh/_version.py,sha256=Bz8PCaGCLCvM-UpGJrgN-1lIPfiCmdjQ8CSjFtdRJxc,721
+sqlmesh/_version.py,sha256=mpzvXCVjwx1_Z7KgS_YADLshcqrw9_uxYiLUBo2eWQ8,721
 sqlmesh/magics.py,sha256=xLh3u4eqpVrKRVN5KF3X84RPRqjygAB9AJP1TXwH8hg,42086
 sqlmesh/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/cicd/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -13,13 +13,13 @@ sqlmesh/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/core/_typing.py,sha256=PzXxMYnORq18JhblAOUttms3zPJZzZpIbfFA_jgKYPA,498
 sqlmesh/core/console.py,sha256=MYpVlciUY6rUuoqXyKfXTxD6a4-Bw4-ooATUTj_VHGg,172830
 sqlmesh/core/constants.py,sha256=BuQk43vluUm7LfP9nKp5o9qRhqIenWF_LiLXO_t_53c,2699
-sqlmesh/core/context.py,sha256=UxK9h9qvjwmnRTL7JLUEJFZvinWbJ_dDkXIKHq0s5ZU,131264
+sqlmesh/core/context.py,sha256=PZcI06NldePYMDkvsMuZf8vGreDk7dzjnzX5rZ422AM,133656
 sqlmesh/core/context_diff.py,sha256=mxkJu0IthFMOlaQ_kcq5C09mlgkq2RQb-pG2rd-x_nA,21648
 sqlmesh/core/dialect.py,sha256=CnKcPj6BnREfu9Zn1OyS7hZ3ktnaX03ygOg91nADlTU,53029
 sqlmesh/core/environment.py,sha256=Kgs_gUEUI072mh0JJFWNRynrCxp1TzRHZhX_NWJRfXc,13142
 sqlmesh/core/janitor.py,sha256=zJRN48ENjKexeiqa1Kmwyj_HsEEEIAa8hsFD8gTCmfg,7194
 sqlmesh/core/lineage.py,sha256=LtiOztX1xIbFfWz-eb5dPZW4B0o2sI942_IM4YDbsso,3163
-sqlmesh/core/loader.py,sha256=2u91WKnXWHbAmyo9mluXHhSNwhe4r35QHv031S4OXjU,37291
+sqlmesh/core/loader.py,sha256=sXGTeyDISd3Gtu9Ej9iEz1CMM4SXSwZMSBpbZNohE10,37458
 sqlmesh/core/macros.py,sha256=rkklwVnUEmEro4wpdel289mKhaS3x5_SPZrkYZt3Q9E,63173
 sqlmesh/core/node.py,sha256=2ejDwH1whl_ic1CRzX16Be-FQrosAf8pdyWb7oPzU6M,19895
 sqlmesh/core/notification_target.py,sha256=PPGoDrgbRKxr27vJEu03XqNTQLYTw0ZF_b0yAapxGeI,16158
@@ -64,7 +64,7 @@ sqlmesh/core/engine_adapter/base.py,sha256=t6c8HlWLmN_2GaWA3EV85Sq29al3fZpesE7Nr
 sqlmesh/core/engine_adapter/base_postgres.py,sha256=WTU0QingaTNM7n-mTVxS-sg4f6jFZGOSryK5IYacveY,7734
 sqlmesh/core/engine_adapter/bigquery.py,sha256=edBWbAbeXA4bOtVG-YNTQbt9qqwL9QFffZti8Ozv-Cw,60923
 sqlmesh/core/engine_adapter/clickhouse.py,sha256=GWGpwdxZd4RqLSAMlOHjtO8nPpSIo3zFeRWnj9eSOrM,36072
-sqlmesh/core/engine_adapter/databricks.py,sha256=452Og5LriNtvXk0DElUGmoR_pUFQvBgNZchpprTIJxA,15846
+sqlmesh/core/engine_adapter/databricks.py,sha256=no6X3E9T31gI-GO7wZaq2r2E7SgLROHLzmot8reVZNY,16485
 sqlmesh/core/engine_adapter/duckdb.py,sha256=9AXeRhaYXBcYSmIavyFY9LUzfgh94qkTO98v0-suQ8I,7993
 sqlmesh/core/engine_adapter/fabric.py,sha256=wky02p3UVu0FvEZwqqb5XBW--XCc1JLMLrvY6TVqCdM,14172
 sqlmesh/core/engine_adapter/mixins.py,sha256=3rB7B2PZSB920BODO7k_kKqu6z0N-zj1etiRCYzpUcQ,27096
@@ -82,7 +82,7 @@ sqlmesh/core/linter/definition.py,sha256=1EOhKdF16jmeqISfcrR-8fzMdgXuxpB7wb3Qaep
 sqlmesh/core/linter/helpers.py,sha256=cwKXP4sL6azRtNVGbMfJ5_6Hqq5Xx2M2rRLCgH3Y3ag,10743
 sqlmesh/core/linter/rule.py,sha256=nB3o1rHyN44ZOg5ImICP16SeUHimf-12ObdXJjkTGyM,3964
 sqlmesh/core/linter/rules/__init__.py,sha256=gevzfb67vFqckTCoVAe_TBGf6hQ-YtE1_YuGuXyh1L0,77
-sqlmesh/core/linter/rules/builtin.py,sha256=C6_cAeZEmZphl9SjGUFMsHGrDKNjxQ5Y6g66eO8HCZg,11177
+sqlmesh/core/linter/rules/builtin.py,sha256=mJjRrL97mTyqPgrUGk-1Ceml6ATCxElZVgiwi6WFqmU,11727
 sqlmesh/core/metric/__init__.py,sha256=H1HmoD5IwN4YWe9iJXyueLYNmTQFZwok5nSWNJcZIBQ,237
 sqlmesh/core/metric/definition.py,sha256=Yd5aVgsZCDPJ43aGP7WqtzZOuuSUtB8uJGVA6Jw9x9M,7201
 sqlmesh/core/metric/rewriter.py,sha256=GiSTHfn2kinqCfNPYgZPRk93JFLzVaaejHtHDQ0yXZI,7326
@@ -124,7 +124,7 @@ sqlmesh/core/state_sync/db/version.py,sha256=q5VDIIvY-585vTbvqPalU0N4qjG6RKs4gr8
 sqlmesh/core/test/__init__.py,sha256=e83TJPwPRR_rAG29Y0OVbZb-5oWVBzz-_wrcd22Qk10,418
 sqlmesh/core/test/context.py,sha256=-TjUrhM3WLtVPBgOMTkvRrnuZq7mT7BeIIyuCbrPePU,2332
 sqlmesh/core/test/definition.py,sha256=Lfflu-qgkqkI7T977F4h4X7c5Co7i3uBt5Efsi4XaZE,42219
-sqlmesh/core/test/discovery.py,sha256=WXIwPidLoacAGHoT_3TVFFEKamG89nHNtaMsZ01uGjs,1188
+sqlmesh/core/test/discovery.py,sha256=5duKXgH4Lms7rXhJ8tOLCmCtqHpv7c7a4VJf12VkGw8,1278
 sqlmesh/core/test/result.py,sha256=6gOKEsERciHhcrw9TedtNr7g1ynTO7UwA5-PPrzvYuM,4564
 sqlmesh/core/test/runner.py,sha256=8I-cL7Q9CggLvET_GPkrXB2YjlyCIHrvbFbbRDnSHRE,6169
 sqlmesh/dbt/__init__.py,sha256=KUv-lW5sG9D2ceXAIzA4MLcjyhzq3E-7qJP4P_PH2EU,144
@@ -238,7 +238,7 @@ sqlmesh/utils/pydantic.py,sha256=-yppkVlw6iSBaSiKjbe7OChxL-u3urOS4-KCjJEgsRU,120
 sqlmesh/utils/rich.py,sha256=cwQ5nJ6sgz64xHtoh6_ec7ReV5YpsOGhMtUJnwoRfEI,3549
 sqlmesh/utils/windows.py,sha256=0F9RdpuuCoG5NiEDXvWlAGCiJ-59OjSAmgFF5wW05aY,1133
 sqlmesh/utils/yaml.py,sha256=KFBd7hsKNRTtRudGR7d410qUYffQv0EWRcDM8hVNNZg,3025
-sqlmesh-0.227.2.dev5.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
+sqlmesh-0.227.2.dev7.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
 sqlmesh_dbt/__init__.py,sha256=awYS5y5mz-1NUmx6i5h5NSTJ7tidRl9NC0FAnFWSF6U,350
 sqlmesh_dbt/cli.py,sha256=p9foHjAW9ni7BTOJ2loynk47M0Sf43QIJZRggOzF5tc,6351
 sqlmesh_dbt/console.py,sha256=RwWLYnEZHzn9Xp-e2gbZvkdKbWbBLN146geI84mJitg,1132
@@ -363,8 +363,8 @@ web/server/api/endpoints/models.py,sha256=kwj0s7uve3iZSMfmjkoPVMFMeY1sD0peTeyrWf
 web/server/api/endpoints/modules.py,sha256=8hqqgonGay_mJmpCw0IdbjsPhWlQH2VLdKAqha-myac,468
 web/server/api/endpoints/plan.py,sha256=bbbY50W_2MsZSTxOHWMKz0tbIm75nsRSlPy8GI2fg9Q,9306
 web/server/api/endpoints/table_diff.py,sha256=8XTwgOh6QBbNy_hTM1JuHgRjbnie-pGPrphiW-FNLjQ,6058
-sqlmesh-0.227.2.dev5.dist-info/METADATA,sha256=IoFrCngJNn_KZY-PdY4qMp2EYD7cMuz9bxJh4UApwIM,26685
-sqlmesh-0.227.2.dev5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-sqlmesh-0.227.2.dev5.dist-info/entry_points.txt,sha256=sHAf6tQczIM8xZoduN4qaUjV7QEPVUUW_LCT8EDUMv4,155
-sqlmesh-0.227.2.dev5.dist-info/top_level.txt,sha256=RQ-33FPe2IgL0rgossAfJkCRtqslz9b7wFARqiWLC5Q,24
-sqlmesh-0.227.2.dev5.dist-info/RECORD,,
+sqlmesh-0.227.2.dev7.dist-info/METADATA,sha256=rH_cwyelAa0zkv2CW9gkxpX8wSQIZbKZPR8Vjic16TY,26685
+sqlmesh-0.227.2.dev7.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+sqlmesh-0.227.2.dev7.dist-info/entry_points.txt,sha256=sHAf6tQczIM8xZoduN4qaUjV7QEPVUUW_LCT8EDUMv4,155
+sqlmesh-0.227.2.dev7.dist-info/top_level.txt,sha256=RQ-33FPe2IgL0rgossAfJkCRtqslz9b7wFARqiWLC5Q,24
+sqlmesh-0.227.2.dev7.dist-info/RECORD,,