sqlmesh 0.181.0__tar.gz → 0.181.1__tar.gz

{sqlmesh-0.181.0 → sqlmesh-0.181.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.181.0
+Version: 0.181.1
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License

{sqlmesh-0.181.0 → sqlmesh-0.181.1}/sqlmesh/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.181.0'
-__version_tuple__ = version_tuple = (0, 181, 0)
+__version__ = version = '0.181.1'
+__version_tuple__ = version_tuple = (0, 181, 1)

{sqlmesh-0.181.0 → sqlmesh-0.181.1}/sqlmesh/core/console.py

@@ -3472,6 +3472,53 @@ class DebuggerTerminalConsole(TerminalConsole):
     ) -> None:
         self._write(row_diff)
 
+    def show_table_diff(
+        self,
+        table_diffs: t.List[TableDiff],
+        show_sample: bool = True,
+        skip_grain_check: bool = False,
+        temp_schema: t.Optional[str] = None,
+    ) -> None:
+        for table_diff in table_diffs:
+            self.show_table_diff_summary(table_diff)
+            self.show_schema_diff(table_diff.schema_diff())
+            self.show_row_diff(
+                table_diff.row_diff(temp_schema=temp_schema, skip_grain_check=skip_grain_check),
+                show_sample=show_sample,
+                skip_grain_check=skip_grain_check,
+            )
+
+    def update_table_diff_progress(self, model: str) -> None:
+        self._write(f"Finished table diff for: {model}")
+
+    def start_table_diff_progress(self, models_to_diff: int) -> None:
+        self._write("Table diff started")
+
+    def start_table_diff_model_progress(self, model: str) -> None:
+        self._write(f"Calculating differences for: {model}")
+
+    def stop_table_diff_progress(self, success: bool) -> None:
+        self._write(f"Table diff finished with success={success}")
+
+    def show_table_diff_details(
+        self,
+        models_to_diff: t.List[str],
+    ) -> None:
+        if models_to_diff:
+            models = "\n".join(models_to_diff)
+            self._write(f"Models to compare: {models}")
+
+    def show_table_diff_summary(self, table_diff: TableDiff) -> None:
+        if table_diff.model_name:
+            self._write(f"Model: {table_diff.model_name}")
+            self._write(f"Source env: {table_diff.source_alias}")
+            self._write(f"Target env: {table_diff.target_alias}")
+        self._write(f"Source table: {table_diff.source}")
+        self._write(f"Target table: {table_diff.target}")
+        _, _, key_column_names = table_diff.key_columns
+        keys = ", ".join(key_column_names)
+        self._write(f"Join On: {keys}")
+
 
 _CONSOLE: Console = NoopConsole()
 

{sqlmesh-0.181.0 → sqlmesh-0.181.1}/sqlmesh/core/context.py

@@ -1677,9 +1677,9 @@ class GenericContext(BaseContext, t.Generic[C]):
                 target_snapshot = target_snapshots_to_name.get(model.fqn)
 
                 if target_snapshot and source_snapshot:
-                    if (
-                        source_snapshot.fingerprint.data_hash
-                        != target_snapshot.fingerprint.data_hash
+                    if (source_snapshot.fingerprint != target_snapshot.fingerprint) and (
+                        (source_snapshot.version != target_snapshot.version)
+                        or (source_snapshot.is_forward_only or target_snapshot.is_forward_only)
                     ):
                         # Compare the virtual layer instead of the physical layer because the virtual layer is guaranteed to point
                         # to the correct/active snapshot for the model in the specified environment, taking into account things like dev previews

{sqlmesh-0.181.0 → sqlmesh-0.181.1}/sqlmesh/core/engine_adapter/spark.py

@@ -105,6 +105,7 @@ class SparkEngineAdapter(
             exp.DataType.Type.BINARY: spark_types.BinaryType,
             exp.DataType.Type.BOOLEAN: spark_types.BooleanType,
             exp.DataType.Type.DATE: spark_types.DateType,
+            exp.DataType.Type.TIMESTAMPNTZ: spark_types.TimestampNTZType,
             exp.DataType.Type.DATETIME: spark_types.TimestampNTZType,
             exp.DataType.Type.TIMESTAMPLTZ: spark_types.TimestampType,
             exp.DataType.Type.TIMESTAMP: spark_types.TimestampType,

{sqlmesh-0.181.0 → sqlmesh-0.181.1}/sqlmesh/core/model/common.py

@@ -361,10 +361,16 @@ def sort_python_env(python_env: t.Dict[str, Executable]) -> t.List[t.Tuple[str,
 
 def sorted_python_env_payloads(python_env: t.Dict[str, Executable]) -> t.List[str]:
     """Returns the payloads of the sorted python env."""
-    return [
-        v.payload if v.is_import or v.is_definition else f"{k} = {v.payload}"
-        for k, v in sort_python_env(python_env)
-    ]
+
+    def _executable_to_str(k: str, v: Executable) -> str:
+        result = f"# {v.path}\n" if v.path is not None else ""
+        if v.is_import or v.is_definition:
+            result += v.payload
+        else:
+            result += f"{k} = {v.payload}"
+        return result
+
+    return [_executable_to_str(k, v) for k, v in sort_python_env(python_env)]
 
 
 expression_validator: t.Callable = field_validator(

{sqlmesh-0.181.0 → sqlmesh-0.181.1}/sqlmesh/lsp/main.py

@@ -14,7 +14,9 @@ from sqlmesh.core.linter.definition import AnnotatedRuleViolation
 from sqlmesh.lsp.completions import get_sql_completions
 from sqlmesh.lsp.context import LSPContext, ModelTarget
 from sqlmesh.lsp.custom import ALL_MODELS_FEATURE, AllModelsRequest, AllModelsResponse
-from sqlmesh.lsp.reference import get_model_definitions_for_a_path
+from sqlmesh.lsp.reference import (
+    get_references,
+)
 
 
 class SQLMeshLanguageServer:
@@ -175,6 +177,34 @@ class SQLMeshLanguageServer:
                 ls.show_message(f"Error formatting SQL: {e}", types.MessageType.Error)
                 return []
 
+        @self.server.feature(types.TEXT_DOCUMENT_HOVER)
+        def hover(ls: LanguageServer, params: types.HoverParams) -> t.Optional[types.Hover]:
+            """Provide hover information for an object."""
+            try:
+                self._ensure_context_for_document(params.text_document.uri)
+                document = ls.workspace.get_document(params.text_document.uri)
+                if self.lsp_context is None:
+                    raise RuntimeError(f"No context found for document: {document.path}")
+
+                references = get_references(
+                    self.lsp_context, params.text_document.uri, params.position
+                )
+                if not references:
+                    return None
+                reference = references[0]
+                if not reference.description:
+                    return None
+                return types.Hover(
+                    contents=types.MarkupContent(
+                        kind=types.MarkupKind.Markdown, value=reference.description
+                    ),
+                    range=reference.range,
+                )
+
+            except Exception as e:
+                ls.show_message(f"Error getting hover information: {e}", types.MessageType.Error)
+                return None
+
         @self.server.feature(types.TEXT_DOCUMENT_DEFINITION)
         def goto_definition(
             ls: LanguageServer, params: types.DefinitionParams
@@ -186,12 +216,9 @@ class SQLMeshLanguageServer:
                 if self.lsp_context is None:
                     raise RuntimeError(f"No context found for document: {document.path}")
 
-                references = get_model_definitions_for_a_path(
-                    self.lsp_context, params.text_document.uri
+                references = get_references(
+                    self.lsp_context, params.text_document.uri, params.position
                 )
-                if not references:
-                    return []
-
                 return [
                     types.LocationLink(
                         target_uri=reference.uri,
@@ -207,7 +234,6 @@ class SQLMeshLanguageServer:
                     )
                     for reference in references
                 ]
-
             except Exception as e:
                 ls.show_message(f"Error getting references: {e}", types.MessageType.Error)
                 return []

{sqlmesh-0.181.0 → sqlmesh-0.181.1}/sqlmesh/lsp/reference.py

@@ -10,8 +10,65 @@ from sqlmesh.utils.pydantic import PydanticModel
 
 
 class Reference(PydanticModel):
+    """
+    A reference to a model.
+
+    Attributes:
+        range: The range of the reference in the source file
+        uri: The uri of the referenced model
+        description: The description of the referenced model
+    """
+
     range: Range
     uri: str
+    description: t.Optional[str] = None
+
+
+def by_position(position: Position) -> t.Callable[[Reference], bool]:
+    """
+    Filter reference to only filter references that contain the given position.
+
+    Args:
+        position: The cursor position to check
+
+    Returns:
+        A function that returns True if the reference contains the position, False otherwise
+    """
+
+    def contains_position(r: Reference) -> bool:
+        return (
+            r.range.start.line < position.line
+            or (
+                r.range.start.line == position.line
+                and r.range.start.character <= position.character
+            )
+        ) and (
+            r.range.end.line > position.line
+            or (r.range.end.line == position.line and r.range.end.character >= position.character)
+        )
+
+    return contains_position
+
+
+def get_references(
+    lint_context: LSPContext, document_uri: str, position: Position
+) -> t.List[Reference]:
+    """
+    Get references at a specific position in a document.
+
+    Used for hover information.
+
+    Args:
+        lint_context: The LSP context
+        document_uri: The URI of the document
+        position: The position to check for references
+
+    Returns:
+        A list of references at the given position
+    """
+    references = get_model_definitions_for_a_path(lint_context, document_uri)
+    filtered_references = list(filter(by_position(position), references))
+    return filtered_references
 
 
 def get_model_definitions_for_a_path(
@@ -121,7 +178,11 @@ def get_model_definitions_for_a_path(
             start_pos = catalog_or_db_range.start
 
         references.append(
-            Reference(uri=referenced_model_uri, range=Range(start=start_pos, end=end_pos))
+            Reference(
+                uri=referenced_model_uri,
+                range=Range(start=start_pos, end=end_pos),
+                description=referenced_model.description,
+            )
         )
 
     return references

{sqlmesh-0.181.0 → sqlmesh-0.181.1}/sqlmesh.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.181.0
+Version: 0.181.1
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License

{sqlmesh-0.181.0 → sqlmesh-0.181.1}/tests/core/test_model.py

@@ -9702,3 +9702,26 @@ def test_scd_time_data_type_does_not_cause_diff_after_deserialization() -> None:
         deserialized_model = SqlModel.parse_raw(model.json())
 
         assert model.data_hash == deserialized_model.data_hash
+
+
+def test_python_env_includes_file_path_in_render_definition():
+    @model(
+        "db.test_model_path",
+        kind=dict(
+            name=ModelKindName.SCD_TYPE_2_BY_TIME,
+            unique_key=["id"],
+            disable_restatement=False,
+        ),
+        columns={"id": "string", "name": "string"},
+        optimize_query=False,
+    )
+    def test_model(context, **kwargs):
+        return pd.DataFrame([{"id": context.var("1")}])
+
+    python_model = model.get_registry()["db.test_model_path"].model(
+        module_path=Path("."), path=Path("."), dialect="duckdb"
+    )
+
+    model_executable_str = python_model.render_definition()[1].sql()
+    # Make sure the file path is included in the render definition
+    assert "# tests/core/test_model.py" in model_executable_str

{sqlmesh-0.181.0 → sqlmesh-0.181.1}/tests/core/test_table_diff.py

@@ -695,3 +695,96 @@ def test_data_diff_multiple_models(sushi_context_fixed_date, capsys, caplog):
         skip_grain_check=False,
     )
     assert len(diffs) == 0
+
+
+@pytest.mark.slow
+def test_data_diff_forward_only(sushi_context_fixed_date, capsys, caplog):
+    expressions = d.parse(
+        """
+        MODEL (name memory.sushi.full_1, kind full, grain(key),);
+        SELECT
+            key,
+            value,
+        FROM
+            (VALUES
+                (1, 3),
+                (2, 4),
+            ) AS t (key, value)
+    """
+    )
+    model_s = load_sql_based_model(expressions, dialect="snowflake")
+    sushi_context_fixed_date.upsert_model(model_s)
+
+    # Create second analytics model sourcing from first
+    expressions_2 = d.parse(
+        """
+        MODEL (name memory.sushi.full_2, kind full, grain(key),);
+        SELECT
+            key,
+            value as amount,
+        FROM
+            memory.sushi.full_1
+    """
+    )
+    model_s2 = load_sql_based_model(expressions_2, dialect="snowflake")
+    sushi_context_fixed_date.upsert_model(model_s2)
+
+    sushi_context_fixed_date.plan(
+        "target_dev",
+        no_prompts=True,
+        auto_apply=True,
+        skip_tests=True,
+        start="2023-01-31",
+        end="2023-01-31",
+    )
+
+    model = sushi_context_fixed_date.models['"MEMORY"."SUSHI"."FULL_1"']
+    modified_model = model.dict()
+    modified_model["query"] = exp.select("*").from_("(VALUES (12, 6),(5,3),) AS t (key, value)")
+    modified_sqlmodel = SqlModel(**modified_model)
+    sushi_context_fixed_date.upsert_model(modified_sqlmodel)
+
+    sushi_context_fixed_date.auto_categorize_changes = CategorizerConfig(
+        sql=AutoCategorizationMode.FULL
+    )
+
+    plan_builder = sushi_context_fixed_date.plan_builder(
+        "source_dev", skip_tests=True, forward_only=True
+    )
+    plan = plan_builder.build()
+
+    sushi_context_fixed_date.apply(plan)
+
+    # Get diffs for both models
+    selector = {"*full*"}
+    diffs = sushi_context_fixed_date.table_diff(
+        source="source_dev",
+        target="target_dev",
+        on=["key"],
+        select_models=selector,
+        skip_grain_check=False,
+    )
+
+    # Both models should be diffed
+    assert len(diffs) == 2
+
+    # Check full_1 diff
+    diff1 = next(d for d in diffs if "FULL_1" in d.source)
+    row_diff1 = diff1.row_diff()
+    diff2 = next(d for d in diffs if "FULL_2" in d.source)
+    row_diff2 = diff2.row_diff()
+
+    # Both diffs should show the same matches
+    for row_diff in [row_diff1, row_diff2]:
+        assert row_diff.full_match_count == 0
+        assert row_diff.full_match_pct == 0.0
+        assert row_diff.s_only_count == 2
+        assert row_diff.t_only_count == 2
+        assert row_diff.stats["join_count"] == 0
+        assert row_diff.stats["null_grain_count"] == 0
+        assert row_diff.stats["s_count"] == 2
+        assert row_diff.stats["distinct_count_s"] == 2
+        assert row_diff.stats["t_count"] == 2
+        assert row_diff.stats["distinct_count_t"] == 2
+        assert row_diff.s_sample.shape == (2, 2)
+        assert row_diff.t_sample.shape == (2, 2)

{sqlmesh-0.181.0 → sqlmesh-0.181.1}/tests/lsp/test_reference.py

@@ -1,7 +1,8 @@
 import pytest
+from lsprotocol.types import Position
 from sqlmesh.core.context import Context
 from sqlmesh.lsp.context import LSPContext, ModelTarget, AuditTarget
-from sqlmesh.lsp.reference import get_model_definitions_for_a_path
+from sqlmesh.lsp.reference import get_model_definitions_for_a_path, by_position
 
 
 @pytest.mark.fast
@@ -53,6 +54,7 @@ def test_reference_with_alias() -> None:
 
     assert references[0].uri.endswith("orders.py")
     assert get_string_from_range(read_file, references[0].range) == "sushi.orders"
+    assert references[0].description == "Table of sushi orders."
     assert references[1].uri.endswith("order_items.py")
     assert get_string_from_range(read_file, references[1].range) == "sushi.order_items"
     assert references[2].uri.endswith("items.py")
@@ -108,3 +110,70 @@ def get_string_from_range(file_lines, range_obj) -> str:
         result += file_lines[line_num]
     result += file_lines[end_line][:end_character]  # Last line up to end_character
     return result
+
+
+@pytest.mark.fast
+def test_filter_references_by_position() -> None:
+    """Test that we can filter references correctly based on cursor position."""
+    context = Context(paths=["examples/sushi"])
+    lsp_context = LSPContext(context)
+
+    # Use a file with multiple references (waiter_revenue_by_day)
+    waiter_revenue_by_day_uri = next(
+        uri
+        for uri, info in lsp_context.map.items()
+        if isinstance(info, ModelTarget) and "sushi.waiter_revenue_by_day" in info.names
+    )
+
+    # Get all references in the file
+    all_references = get_model_definitions_for_a_path(lsp_context, waiter_revenue_by_day_uri)
+    assert len(all_references) == 3
+
+    # Get file contents to locate positions for testing
+    path = waiter_revenue_by_day_uri.removeprefix("file://")
+    with open(path, "r") as file:
+        read_file = file.readlines()
+
+    # Test positions for each reference
+    for i, reference in enumerate(all_references):
+        # Position inside the reference - should return exactly one reference
+        middle_line = (reference.range.start.line + reference.range.end.line) // 2
+        middle_char = (reference.range.start.character + reference.range.end.character) // 2
+        position_inside = Position(line=middle_line, character=middle_char)
+        filtered = list(filter(by_position(position_inside), all_references))
+        assert len(filtered) == 1
+        assert filtered[0].uri == reference.uri
+        assert filtered[0].range == reference.range
+
+        # For testing outside position, use a position before the current reference
+        # or after the last reference for the last one
+        if i == 0:
+            outside_line = reference.range.start.line
+            outside_char = max(0, reference.range.start.character - 5)
+        else:
+            prev_ref = all_references[i - 1]
+            outside_line = prev_ref.range.end.line
+            outside_char = prev_ref.range.end.character + 5
+
+        position_outside = Position(line=outside_line, character=outside_char)
+        filtered_outside = list(filter(by_position(position_outside), all_references))
+        assert reference not in filtered_outside, (
+            f"Reference {i} should not match position outside its range"
+        )
+
+    # Test case: cursor at beginning of file - no references should match
+    position_start = Position(line=0, character=0)
+    filtered_start = list(filter(by_position(position_start), all_references))
+    assert len(filtered_start) == 0 or all(
+        ref.range.start.line == 0 and ref.range.start.character <= 0 for ref in filtered_start
+    )
+
+    # Test case: cursor at end of file - no references should match (unless there's a reference at the end)
+    last_line = len(read_file) - 1
+    last_char = len(read_file[last_line]) - 1
+    position_end = Position(line=last_line, character=last_char)
+    filtered_end = list(filter(by_position(position_end), all_references))
+    assert len(filtered_end) == 0 or all(
+        ref.range.end.line >= last_line and ref.range.end.character >= last_char
+        for ref in filtered_end
+    )