sqlmesh 0.213.2.dev2__tar.gz → 0.213.2.dev4__tar.gz

{sqlmesh-0.213.2.dev2 → sqlmesh-0.213.2.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.213.2.dev2
+Version: 0.213.2.dev4
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License

{sqlmesh-0.213.2.dev2 → sqlmesh-0.213.2.dev4}/sqlmesh/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.213.2.dev2'
-__version_tuple__ = version_tuple = (0, 213, 2, 'dev2')
+__version__ = version = '0.213.2.dev4'
+__version_tuple__ = version_tuple = (0, 213, 2, 'dev4')
 
-__commit_id__ = commit_id = 'g9e9159d69'
+__commit_id__ = commit_id = 'g7f5adcfcf'

{sqlmesh-0.213.2.dev2 → sqlmesh-0.213.2.dev4}/sqlmesh/core/model/meta.py

@@ -29,7 +29,6 @@ from sqlmesh.core.model.kind import (
     SCDType2ByTimeKind,
     TimeColumn,
     ViewKind,
-    _IncrementalBy,
     model_kind_validator,
     OnAdditiveChange,
 )
@@ -414,7 +413,7 @@ class ModelMeta(_Node):
     @property
     def lookback(self) -> int:
         """The incremental lookback window."""
-        return (self.kind.lookback if isinstance(self.kind, _IncrementalBy) else 0) or 0
+        return getattr(self.kind, "lookback", 0) or 0
 
     def lookback_start(self, start: TimeLike) -> TimeLike:
         if self.lookback == 0:

{sqlmesh-0.213.2.dev2 → sqlmesh-0.213.2.dev4}/sqlmesh/dbt/builtin.py

@@ -164,6 +164,58 @@ class Var:
         return name in self.variables
 
 
+class Config:
+    def __init__(self, config_dict: t.Dict[str, t.Any]) -> None:
+        self._config = config_dict
+
+    def __call__(self, **kwargs: t.Any) -> str:
+        self._config.update(**kwargs)
+        return ""
+
+    def set(self, name: str, value: t.Any) -> str:
+        self._config.update({name: value})
+        return ""
+
+    def _validate(self, name: str, validator: t.Callable, value: t.Optional[t.Any] = None) -> None:
+        try:
+            validator(value)
+        except Exception as e:
+            raise ConfigError(f"Config validation failed for '{name}': {e}")
+
+    def require(self, name: str, validator: t.Optional[t.Callable] = None) -> t.Any:
+        if name not in self._config:
+            raise ConfigError(f"Missing required config: {name}")
+
+        value = self._config[name]
+
+        if validator is not None:
+            self._validate(name, validator, value)
+
+        return value
+
+    def get(
+        self, name: str, default: t.Any = None, validator: t.Optional[t.Callable] = None
+    ) -> t.Any:
+        value = self._config.get(name, default)
+
+        if validator is not None and value is not None:
+            self._validate(name, validator, value)
+
+        return value
+
+    def persist_relation_docs(self) -> bool:
+        persist_docs = self.get("persist_docs", default={})
+        if not isinstance(persist_docs, dict):
+            return False
+        return persist_docs.get("relation", False)
+
+    def persist_column_docs(self) -> bool:
+        persist_docs = self.get("persist_docs", default={})
+        if not isinstance(persist_docs, dict):
+            return False
+        return persist_docs.get("columns", False)
+
+
 def env_var(name: str, default: t.Optional[str] = None) -> t.Optional[str]:
     if name not in os.environ and default is None:
         raise ConfigError(f"Missing environment variable '{name}'")
@@ -395,6 +447,8 @@ def create_builtin_globals(
     if variables is not None:
         builtin_globals["var"] = Var(variables)
 
+    builtin_globals["config"] = Config(jinja_globals.pop("config", {}))
+
     deployability_index = (
         jinja_globals.get("deployability_index") or DeployabilityIndex.all_deployable()
     )

{sqlmesh-0.213.2.dev2 → sqlmesh-0.213.2.dev4}/sqlmesh.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.213.2.dev2
+Version: 0.213.2.dev4
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License

{sqlmesh-0.213.2.dev2 → sqlmesh-0.213.2.dev4}/tests/core/test_model.py

@@ -7686,6 +7686,75 @@ lookback 3
     )
 
 
+def test_custom_kind_lookback_property():
+    """Test that CustomKind's lookback property is correctly accessed via ModelMeta.lookback.
+
+    This test verifies the fix for issue #5268 where CustomKind models were not respecting
+    the lookback parameter because the isinstance check for _IncrementalBy failed.
+    """
+
+    # Test 1: CustomKind with lookback = 3
+    class MyTestStrategy(CustomMaterialization):
+        pass
+
+    expressions = d.parse(
+        """
+        MODEL (
+            name db.custom_table,
+            kind CUSTOM (
+                materialization 'MyTestStrategy',
+                lookback 3
+            )
+        );
+        SELECT a, b FROM upstream
+        """
+    )
+
+    model = load_sql_based_model(expressions)
+    assert model.kind.is_custom
+
+    # Verify that the kind itself has lookback = 3
+    kind = t.cast(CustomKind, model.kind)
+    assert kind.lookback == 3
+
+    # The bug: model.lookback should return 3, but with the old implementation
+    # using isinstance(self.kind, _IncrementalBy), it would return 0
+    assert model.lookback == 3, "CustomKind lookback not accessible via model.lookback property"
+
+    # Test 2: CustomKind without lookback (should default to 0)
+    expressions_no_lookback = d.parse(
+        """
+        MODEL (
+            name db.custom_table_no_lookback,
+            kind CUSTOM (
+                materialization 'MyTestStrategy'
+            )
+        );
+        SELECT a, b FROM upstream
+        """
+    )
+
+    model_no_lookback = load_sql_based_model(expressions_no_lookback)
+    assert model_no_lookback.lookback == 0
+
+    # Test 3: Ensure IncrementalByTimeRangeKind still works correctly
+    incremental_expressions = d.parse(
+        """
+        MODEL (
+            name db.incremental_table,
+            kind INCREMENTAL_BY_TIME_RANGE (
+                time_column ds,
+                lookback 5
+            )
+        );
+        SELECT ds, a, b FROM upstream
+        """
+    )
+
+    incremental_model = load_sql_based_model(incremental_expressions)
+    assert incremental_model.lookback == 5
+
+
 def test_time_column_format_in_custom_kind():
     class TimeColumnCustomKind(CustomKind):  # type: ignore[no-untyped-def]
         _time_column: TimeColumn

{sqlmesh-0.213.2.dev2 → sqlmesh-0.213.2.dev4}/tests/core/test_snapshot.py

@@ -681,6 +681,43 @@ def test_lookback(make_snapshot):
     assert snapshot.missing_intervals("2023-01-28", "2023-01-30", "2023-01-31 04:00:00") == []
 
 
+def test_lookback_custom_materialization(make_snapshot):
+    from sqlmesh import CustomMaterialization
+
+    class MyTestStrategy(CustomMaterialization):
+        pass
+
+    expressions = parse(
+        """
+        MODEL (
+            name name,
+            kind CUSTOM (
+                materialization 'MyTestStrategy',
+                lookback 2
+            ),
+            start '2023-01-01',
+            cron '0 5 * * *',
+        );
+
+        SELECT ds FROM parent.tbl
+        """
+    )
+
+    snapshot = make_snapshot(load_sql_based_model(expressions))
+
+    assert snapshot.missing_intervals("2023-01-01", "2023-01-01") == [
+        (to_timestamp("2023-01-01"), to_timestamp("2023-01-02")),
+    ]
+
+    snapshot.add_interval("2023-01-01", "2023-01-04")
+    assert snapshot.missing_intervals("2023-01-01", "2023-01-04") == []
+    assert snapshot.missing_intervals("2023-01-01", "2023-01-05") == [
+        (to_timestamp("2023-01-03"), to_timestamp("2023-01-04")),
+        (to_timestamp("2023-01-04"), to_timestamp("2023-01-05")),
+        (to_timestamp("2023-01-05"), to_timestamp("2023-01-06")),
+    ]
+
+
 def test_seed_intervals(make_snapshot):
     snapshot_a = make_snapshot(
         SeedModel(

{sqlmesh-0.213.2.dev2 → sqlmesh-0.213.2.dev4}/tests/dbt/test_transformation.py

@@ -946,7 +946,7 @@ def test_schema_jinja(sushi_test_project: Project, assert_exp_eq):
 
 @pytest.mark.xdist_group("dbt_manifest")
 def test_config_jinja(sushi_test_project: Project):
-    hook = "{{ config(alias='bar') }} {{ config.alias }}"
+    hook = "{{ config(alias='bar') }} {{ config.get('alias') }}"
     model_config = ModelConfig(
         name="model",
         package_name="package",
@@ -961,6 +961,120 @@ def test_config_jinja(sushi_test_project: Project):
     assert model.render_pre_statements()[0].sql() == '"bar"'
 
 
+@pytest.mark.xdist_group("dbt_manifest")
+def test_config_jinja_get_methods(sushi_test_project: Project):
+    model_config = ModelConfig(
+        name="model_conf",
+        package_name="package",
+        schema="sushi",
+        sql="""SELECT 1 AS one FROM foo""",
+        alias="model_alias",
+        **{
+            "pre-hook": [
+                "{{ config(materialized='incremental', unique_key='id') }}"
+                "{{ config.get('missed', 'a') + config.get('missed', default='b')}}",
+                "{{ config.set('alias', 'new_alias')}}",
+                "{{ config.get('package_name') + '_' + config.require('unique_key')}}",
+                "{{ config.get('alias') or 'default'}}",
+            ]
+        },
+        **{"post-hook": "{{config.require('missing_key')}}"},
+    )
+    context = sushi_test_project.context
+    model = t.cast(SqlModel, model_config.to_sqlmesh(context))
+
+    assert model.render_pre_statements()[0].sql() == '"ab"'
+    assert model.render_pre_statements()[1].sql() == '"package_id"'
+    assert model.render_pre_statements()[2].sql() == '"new_alias"'
+
+    with pytest.raises(ConfigError, match="Missing required config: missing_key"):
+        model.render_post_statements()
+
+    # test get methods with operations
+    model_2_config = ModelConfig(
+        name="model_2",
+        package_name="package",
+        schema="sushi",
+        sql="""SELECT 1 AS one FROM foo""",
+        alias="mod",
+        materialized="table",
+        threads=8,
+        partition_by="date",
+        cluster_by=["user_id", "product_id"],
+        **{
+            "pre-hook": [
+                "{{ config.get('partition_by', default='none') }}",
+                "{{ config.get('cluster_by', default=[]) | length }}",
+                "{% if config.get('threads') > 4 %}high_threads{% else %}low_threads{% endif %}",
+            ]
+        },
+    )
+    model2 = t.cast(SqlModel, model_2_config.to_sqlmesh(context))
+
+    pre_statements2 = model2.render_pre_statements()
+    assert pre_statements2[0].sql() == "ARRAY('date')"
+    assert pre_statements2[1].sql() == "2"
+    assert pre_statements2[2].sql() == '"high_threads"'
+
+    # test seting variable and conditional
+    model_invalid_timeout = ModelConfig(
+        name="invalid_timeout_test",
+        package_name="package",
+        schema="sushi",
+        sql="""SELECT 1 AS one FROM foo""",
+        alias="invalid_timeout_alias",
+        connection_timeout=44,
+        **{
+            "pre-hook": [
+                """
+            {%- set value = config.require('connection_timeout') -%}
+            {%- set is_valid = value >= 10 and value <= 30 -%}
+            {%- if not is_valid -%}
+              {{ exceptions.raise_compiler_error("Validation failed for 'connection_timeout': Value must be between 10 and 30, got: " ~ value) }}
+            {%- endif -%}
+            {{ value }}
+            """,
+            ]
+        },
+    )
+
+    model_invalid = t.cast(SqlModel, model_invalid_timeout.to_sqlmesh(context))
+    with pytest.raises(
+        ConfigError,
+        match="Validation failed for 'connection_timeout': Value must be between 10 and 30, got: 44",
+    ):
+        model_invalid.render_pre_statements()
+
+    # test persist_docs methods
+    model_config_persist = ModelConfig(
+        name="persist_docs_model",
+        package_name="package",
+        schema="sushi",
+        sql="""SELECT 1 AS one FROM foo""",
+        alias="persist_alias",
+        **{
+            "pre-hook": [
+                "{{ config(persist_docs={'relation': true, 'columns': true}) }}",
+                "{{ config.persist_relation_docs() }}",
+                "{{ config.persist_column_docs() }}",
+                "{{ config(persist_docs={'relation': false, 'columns': true}) }}",
+                "{{ config.persist_relation_docs() }}",
+                "{{ config.persist_column_docs() }}",
+            ]
+        },
+    )
+    model3 = t.cast(SqlModel, model_config_persist.to_sqlmesh(context))
+
+    pre_statements3 = model3.render_pre_statements()
+
+    # it should filter out empty returns, so we get 4 statements
+    assert len(pre_statements3) == 4
+    assert pre_statements3[0].sql() == "TRUE"
+    assert pre_statements3[1].sql() == "TRUE"
+    assert pre_statements3[2].sql() == "FALSE"
+    assert pre_statements3[3].sql() == "TRUE"
+
+
 @pytest.mark.xdist_group("dbt_manifest")
 def test_model_this(assert_exp_eq, sushi_test_project: Project):
     model_config = ModelConfig(