sqlmesh 0.189.2.dev5__tar.gz → 0.189.3.dev6__tar.gz

{sqlmesh-0.189.2.dev5 → sqlmesh-0.189.3.dev6}/.circleci/continue_config.yml

@@ -152,7 +152,6 @@ jobs:
       - image: cimg/node:20.19.0
     resource_class: small
     steps:
-      - halt_unless_client
       - checkout
       - restore_cache:
           name: Restore pnpm Package Cache

{sqlmesh-0.189.2.dev5 → sqlmesh-0.189.3.dev6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.189.2.dev5
+Version: 0.189.3.dev6
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License

{sqlmesh-0.189.2.dev5 → sqlmesh-0.189.3.dev6}/sqlmesh/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.189.2.dev5'
-__version_tuple__ = version_tuple = (0, 189, 2, 'dev5')
+__version__ = version = '0.189.3.dev6'
+__version_tuple__ = version_tuple = (0, 189, 3, 'dev6')

{sqlmesh-0.189.2.dev5 → sqlmesh-0.189.3.dev6}/sqlmesh/cli/main.py

@@ -938,6 +938,11 @@ def create_external_models(obj: Context, **kwargs: t.Any) -> None:
     multiple=True,
     help="Specify one or more models to data diff. Use wildcards to diff multiple models. Ex: '*' (all models with applied plan diffs), 'demo.model+' (this and downstream models), 'git:feature_branch' (models with direct modifications in this branch only)",
 )
+@click.option(
+    "--schema-diff-ignore-case",
+    is_flag=True,
+    help="If set, when performing a schema diff the case of column names is ignored when matching between the two schemas. For example, 'col_a' in the source schema and 'COL_A' in the target schema will be treated as the same column.",
+)
 @click.pass_obj
 @error_handler
 @cli_analytics

{sqlmesh-0.189.2.dev5 → sqlmesh-0.189.3.dev6}/sqlmesh/core/config/connection.py

@@ -43,7 +43,7 @@ if t.TYPE_CHECKING:
 
 logger = logging.getLogger(__name__)
 
-RECOMMENDED_STATE_SYNC_ENGINES = {"postgres", "gcp_postgres", "mysql", "mssql"}
+RECOMMENDED_STATE_SYNC_ENGINES = {"postgres", "gcp_postgres", "mysql", "mssql", "azuresql"}
 FORBIDDEN_STATE_SYNC_ENGINES = {
     # Do not support row-level operations
     "spark",

{sqlmesh-0.189.2.dev5 → sqlmesh-0.189.3.dev6}/sqlmesh/core/context.py

@@ -1403,7 +1403,7 @@ class GenericContext(BaseContext, t.Generic[C]):
             k: v for k, v in kwargs.items() if v is not None
         }
 
-        skip_tests = skip_tests or False
+        skip_tests = explain or skip_tests or False
         no_gaps = no_gaps or False
         skip_backfill = skip_backfill or False
         empty_backfill = empty_backfill or False
@@ -1673,6 +1673,7 @@ class GenericContext(BaseContext, t.Generic[C]):
         skip_grain_check: bool = False,
         warn_grain_check: bool = False,
         temp_schema: t.Optional[str] = None,
+        schema_diff_ignore_case: bool = False,
     ) -> t.List[TableDiff]:
         """Show a diff between two tables.
 
@@ -1796,6 +1797,7 @@ class GenericContext(BaseContext, t.Generic[C]):
                             show=show,
                             temp_schema=temp_schema,
                             skip_grain_check=skip_grain_check,
+                            schema_diff_ignore_case=schema_diff_ignore_case,
                         ),
                         tasks_num=tasks_num,
                     )
@@ -1821,6 +1823,7 @@ class GenericContext(BaseContext, t.Generic[C]):
                     on=on,
                     skip_columns=skip_columns,
                     where=where,
+                    schema_diff_ignore_case=schema_diff_ignore_case,
                 )
             ]
 
@@ -1845,6 +1848,7 @@ class GenericContext(BaseContext, t.Generic[C]):
         show: bool = True,
         temp_schema: t.Optional[str] = None,
         skip_grain_check: bool = False,
+        schema_diff_ignore_case: bool = False,
     ) -> TableDiff:
         self.console.start_table_diff_model_progress(model.name)
 
@@ -1860,6 +1864,7 @@ class GenericContext(BaseContext, t.Generic[C]):
             target=target,
             source_alias=source_alias,
             target_alias=target_alias,
+            schema_diff_ignore_case=schema_diff_ignore_case,
         )
 
         if show:
@@ -1883,6 +1888,7 @@ class GenericContext(BaseContext, t.Generic[C]):
         model: t.Optional[Model] = None,
         skip_columns: t.Optional[t.List[str]] = None,
         where: t.Optional[str | exp.Condition] = None,
+        schema_diff_ignore_case: bool = False,
     ) -> TableDiff:
         if not on:
             raise SQLMeshError(
@@ -1902,6 +1908,7 @@ class GenericContext(BaseContext, t.Generic[C]):
             decimals=decimals,
             model_name=model.name if model else None,
             model_dialect=model.dialect if model else None,
+            schema_diff_ignore_case=schema_diff_ignore_case,
         )
 
     @python_api_analytics

{sqlmesh-0.189.2.dev5 → sqlmesh-0.189.3.dev6}/sqlmesh/core/linter/definition.py

@@ -8,7 +8,7 @@ import operator as op
 from collections.abc import Iterator, Iterable, Set, Mapping, Callable
 from functools import reduce
 from sqlmesh.core.model import Model
-from sqlmesh.core.linter.rule import Rule, RuleViolation
+from sqlmesh.core.linter.rule import Rule, RuleViolation, Range
 from sqlmesh.core.console import LinterConsole, get_console
 
 if t.TYPE_CHECKING:
@@ -74,6 +74,7 @@ class Linter:
                 violation_msg=violation.violation_msg,
                 model=model,
                 violation_type="error",
+                violation_range=violation.violation_range,
             )
             for violation in error_violations
         ] + [
@@ -82,6 +83,7 @@ class Linter:
                 violation_msg=violation.violation_msg,
                 model=model,
                 violation_type="warning",
+                violation_range=violation.violation_range,
             )
             for violation in warn_violations
         ]
@@ -149,7 +151,8 @@ class AnnotatedRuleViolation(RuleViolation):
         violation_msg: str,
         model: Model,
         violation_type: t.Literal["error", "warning"],
+        violation_range: t.Optional[Range] = None,
     ) -> None:
-        super().__init__(rule, violation_msg)
+        super().__init__(rule, violation_msg, violation_range)
         self.model = model
         self.violation_type = violation_type

sqlmesh-0.189.3.dev6/sqlmesh/core/linter/helpers.py

@@ -0,0 +1,115 @@
+from pathlib import Path
+
+from sqlmesh.core.linter.rule import Position, Range
+from sqlmesh.utils.pydantic import PydanticModel
+import typing as t
+
+
+class TokenPositionDetails(PydanticModel):
+    """
+    Details about a token's position in the source code in the structure provided by SQLGlot.
+
+    Attributes:
+        line (int): The line that the token ends on.
+        col (int): The column that the token ends on.
+        start (int): The start index of the token.
+        end (int): The ending index of the token.
+    """
+
+    line: int
+    col: int
+    start: int
+    end: int
+
+    @staticmethod
+    def from_meta(meta: t.Dict[str, int]) -> "TokenPositionDetails":
+        return TokenPositionDetails(
+            line=meta["line"],
+            col=meta["col"],
+            start=meta["start"],
+            end=meta["end"],
+        )
+
+    def to_range(self, read_file: t.Optional[t.List[str]]) -> Range:
+        """
+        Convert a TokenPositionDetails object to a Range object.
+
+        In the circumstances where the token's start and end positions are the same,
+        there is no need for a read_file parameter, as the range can be derived from the token's
+        line and column. This is an optimization to avoid unnecessary file reads and should
+        only be used when the token represents a single character or position in the file.
+
+        If the token's start and end positions are different, the read_file parameter is required.
+
+        :param read_file: List of lines from the file. Optional
+        :return: A Range object representing the token's position
+        """
+        if self.start == self.end:
+            # If the start and end positions are the same, we can create a range directly
+            return Range(
+                start=Position(line=self.line - 1, character=self.col - 1),
+                end=Position(line=self.line - 1, character=self.col),
+            )
+
+        if read_file is None:
+            raise ValueError("read_file must be provided when start and end positions differ.")
+
+        # Convert from 1-indexed to 0-indexed for line only
+        end_line_0 = self.line - 1
+        end_col_0 = self.col
+
+        # Find the start line and column by counting backwards from the end position
+        start_pos = self.start
+        end_pos = self.end
+
+        # Initialize with the end position
+        start_line_0 = end_line_0
+        start_col_0 = end_col_0 - (end_pos - start_pos + 1)
+
+        # If start_col_0 is negative, we need to go back to previous lines
+        while start_col_0 < 0 and start_line_0 > 0:
+            start_line_0 -= 1
+            start_col_0 += len(read_file[start_line_0])
+            # Account for newline character
+            if start_col_0 >= 0:
+                break
+            start_col_0 += 1  # For the newline character
+
+        # Ensure we don't have negative values
+        start_col_0 = max(0, start_col_0)
+        return Range(
+            start=Position(line=start_line_0, character=start_col_0),
+            end=Position(line=end_line_0, character=end_col_0),
+        )
+
+
+def read_range_from_file(file: Path, text_range: Range) -> str:
+    """
+    Read the file and return the content within the specified range.
+
+    Args:
+        file: Path to the file to read
+        text_range: The range of text to extract
+
+    Returns:
+        The content within the specified range
+    """
+    with file.open("r", encoding="utf-8") as f:
+        lines = f.readlines()
+
+    # Ensure the range is within bounds
+    start_line = max(0, text_range.start.line)
+    end_line = min(len(lines), text_range.end.line + 1)
+
+    if start_line >= end_line:
+        return ""
+
+    # Extract the relevant portions of each line
+    result = []
+    for i in range(start_line, end_line):
+        line = lines[i]
+        start_char = text_range.start.character if i == text_range.start.line else 0
+        end_char = text_range.end.character if i == text_range.end.line else len(line)
+        result.append(line[start_char:end_char])
+
+    return "".join(result)

{sqlmesh-0.189.2.dev5 → sqlmesh-0.189.3.dev6}/sqlmesh/core/linter/rule.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import abc
+from dataclasses import dataclass
 
 from sqlmesh.core.model import Model
 
@@ -22,6 +23,22 @@ class RuleLocation(PydanticModel):
     start_line: t.Optional[int] = None
 
 
+@dataclass(frozen=True)
+class Position:
+    """The position of a rule violation in a file, the position follows the LSP standard."""
+
+    line: int
+    character: int
+
+
+@dataclass(frozen=True)
+class Range:
+    """The range of a rule violation in a file. The range follows the LSP standard."""
+
+    start: Position
+    end: Position
+
+
 class _Rule(abc.ABCMeta):
     def __new__(cls: Type[_Rule], clsname: str, bases: t.Tuple, attrs: t.Dict) -> _Rule:
         attrs["name"] = clsname.lower()
@@ -45,9 +62,15 @@ class Rule(abc.ABC, metaclass=_Rule):
         """A summary of what this rule checks for."""
         return self.__doc__ or ""
 
-    def violation(self, violation_msg: t.Optional[str] = None) -> RuleViolation:
+    def violation(
+        self,
+        violation_msg: t.Optional[str] = None,
+        violation_range: t.Optional[Range] = None,
+    ) -> RuleViolation:
         """Create a RuleViolation instance for this rule"""
-        return RuleViolation(rule=self, violation_msg=violation_msg or self.summary)
+        return RuleViolation(
+            rule=self, violation_msg=violation_msg or self.summary, violation_range=violation_range
+        )
 
     def get_definition_location(self) -> RuleLocation:
         """Return the file path and position information for this rule.
@@ -79,9 +102,12 @@ class Rule(abc.ABC, metaclass=_Rule):
 
 
 class RuleViolation:
-    def __init__(self, rule: Rule, violation_msg: str) -> None:
+    def __init__(
+        self, rule: Rule, violation_msg: str, violation_range: t.Optional[Range] = None
+    ) -> None:
         self.rule = rule
         self.violation_msg = violation_msg
+        self.violation_range = violation_range
 
     def __repr__(self) -> str:
         return f"{self.rule.name}: {self.violation_msg}"

{sqlmesh-0.189.2.dev5 → sqlmesh-0.189.3.dev6}/sqlmesh/core/linter/rules/builtin.py

@@ -4,9 +4,11 @@ from __future__ import annotations
 
 import typing as t
 
+from sqlglot.expressions import Star
 from sqlglot.helper import subclasses
 
-from sqlmesh.core.linter.rule import Rule, RuleViolation
+from sqlmesh.core.linter.helpers import TokenPositionDetails
+from sqlmesh.core.linter.rule import Rule, RuleViolation, Range
 from sqlmesh.core.linter.definition import RuleSet
 from sqlmesh.core.model import Model, SqlModel
 
@@ -15,10 +17,25 @@ class NoSelectStar(Rule):
     """Query should not contain SELECT * on its outer most projections, even if it can be expanded."""
 
     def check_model(self, model: Model) -> t.Optional[RuleViolation]:
+        # Only applies to SQL models, as other model types do not have a query.
         if not isinstance(model, SqlModel):
             return None
-
-        return self.violation() if model.query.is_star else None
+        if model.query.is_star:
+            violation_range = self._get_range(model)
+            return self.violation(violation_range=violation_range)
+        return None
+
+    def _get_range(self, model: SqlModel) -> t.Optional[Range]:
+        """Get the range of the violation if available."""
+        try:
+            if len(model.query.expressions) == 1 and isinstance(model.query.expressions[0], Star):
+                return TokenPositionDetails.from_meta(model.query.expressions[0].meta).to_range(
+                    None
+                )
+        except Exception:
+            pass
+
+        return None
 
 
 class InvalidSelectStarExpansion(Rule):

{sqlmesh-0.189.2.dev5 → sqlmesh-0.189.3.dev6}/sqlmesh/core/table_diff.py

@@ -36,29 +36,78 @@ class SchemaDiff(PydanticModel, frozen=True):
     source_alias: t.Optional[str] = None
     target_alias: t.Optional[str] = None
     model_name: t.Optional[str] = None
+    ignore_case: bool = False
+
+    @property
+    def _comparable_source_schema(self) -> t.Dict[str, exp.DataType]:
+        return (
+            self._lowercase_schema_names(self.source_schema)
+            if self.ignore_case
+            else self.source_schema
+        )
+
+    @property
+    def _comparable_target_schema(self) -> t.Dict[str, exp.DataType]:
+        return (
+            self._lowercase_schema_names(self.target_schema)
+            if self.ignore_case
+            else self.target_schema
+        )
+
+    def _lowercase_schema_names(
+        self, schema: t.Dict[str, exp.DataType]
+    ) -> t.Dict[str, exp.DataType]:
+        return {c.lower(): t for c, t in schema.items()}
+
+    def _original_column_name(
+        self, maybe_lowercased_column_name: str, schema: t.Dict[str, exp.DataType]
+    ) -> str:
+        if not self.ignore_case:
+            return maybe_lowercased_column_name
+
+        return next(c for c in schema if c.lower() == maybe_lowercased_column_name)
 
     @property
     def added(self) -> t.List[t.Tuple[str, exp.DataType]]:
         """Added columns."""
-        return [(c, t) for c, t in self.target_schema.items() if c not in self.source_schema]
+        return [
+            (self._original_column_name(c, self.target_schema), t)
+            for c, t in self._comparable_target_schema.items()
+            if c not in self._comparable_source_schema
+        ]
 
     @property
     def removed(self) -> t.List[t.Tuple[str, exp.DataType]]:
         """Removed columns."""
-        return [(c, t) for c, t in self.source_schema.items() if c not in self.target_schema]
+        return [
+            (self._original_column_name(c, self.source_schema), t)
+            for c, t in self._comparable_source_schema.items()
+            if c not in self._comparable_target_schema
+        ]
 
     @property
     def modified(self) -> t.Dict[str, t.Tuple[exp.DataType, exp.DataType]]:
         """Columns with modified types."""
         modified = {}
-        for column in self.source_schema.keys() & self.target_schema.keys():
-            source_type = self.source_schema[column]
-            target_type = self.target_schema[column]
+        for column in self._comparable_source_schema.keys() & self._comparable_target_schema.keys():
+            source_type = self._comparable_source_schema[column]
+            target_type = self._comparable_target_schema[column]
 
             if source_type != target_type:
                 modified[column] = (source_type, target_type)
+
+        if self.ignore_case:
+            modified = {
+                self._original_column_name(c, self.source_schema): dt for c, dt in modified.items()
+            }
+
         return modified
 
+    @property
+    def has_changes(self) -> bool:
+        """Does the schema contain any changes at all between source and target"""
+        return bool(self.added or self.removed or self.modified)
+
 
 class RowDiff(PydanticModel, frozen=True):
     """Summary statistics and a sample dataframe."""
@@ -183,6 +232,7 @@ class TableDiff:
         model_name: t.Optional[str] = None,
         model_dialect: t.Optional[str] = None,
         decimals: int = 3,
+        schema_diff_ignore_case: bool = False,
     ):
         if not isinstance(adapter, RowDiffMixin):
             raise ValueError(f"Engine {adapter} doesnt support RowDiff")
@@ -198,6 +248,7 @@ class TableDiff:
         self.model_name = model_name
         self.model_dialect = model_dialect
         self.decimals = decimals
+        self.schema_diff_ignore_case = schema_diff_ignore_case
 
         # Support environment aliases for diff output improvement in certain cases
         self.source_alias = source_alias
@@ -282,6 +333,7 @@ class TableDiff:
             source_alias=self.source_alias,
             target_alias=self.target_alias,
             model_name=self.model_name,
+            ignore_case=self.schema_diff_ignore_case,
         )
 
     def row_diff(

{sqlmesh-0.189.2.dev5 → sqlmesh-0.189.3.dev6}/sqlmesh/lsp/context.py

@@ -5,7 +5,7 @@ import typing as t
 
 from sqlmesh.core.model.definition import SqlModel
 from sqlmesh.core.linter.definition import AnnotatedRuleViolation
-from sqlmesh.lsp.custom import RenderModelEntry, ModelForRendering
+from sqlmesh.lsp.custom import ModelForRendering
 from sqlmesh.lsp.custom import AllModelsResponse, RenderModelEntry
 from sqlmesh.lsp.uri import URI
 

sqlmesh-0.189.3.dev6/sqlmesh/lsp/hints.py

@@ -0,0 +1,136 @@
+"""Type hinting on SQLMesh models"""
+
+import typing as t
+
+from lsprotocol import types
+
+from sqlglot import exp
+from sqlglot.expressions import Expression
+from sqlglot.optimizer.normalize_identifiers import normalize_identifiers
+from sqlmesh.core.model.definition import SqlModel
+from sqlmesh.lsp.context import LSPContext, ModelTarget
+from sqlmesh.lsp.uri import URI
+
+
+def get_hints(
+    lsp_context: LSPContext,
+    document_uri: URI,
+    start_line: int,
+    end_line: int,
+) -> t.List[types.InlayHint]:
+    """
+    Get type hints for certain lines in a document
+
+    Args:
+        lint_context: The LSP context
+        document_uri: The URI of the document
+        start_line: the starting line to get hints for
+        end_line: the ending line to get hints for
+
+    Returns:
+        A list of hints to apply to the document
+    """
+    path = document_uri.to_path()
+    if path.suffix != ".sql":
+        return []
+
+    if path not in lsp_context.map:
+        return []
+
+    file_info = lsp_context.map[path]
+
+    # Process based on whether it's a model or standalone audit
+    if not isinstance(file_info, ModelTarget):
+        return []
+
+    # It's a model
+    model = lsp_context.context.get_model(
+        model_or_snapshot=file_info.names[0], raise_if_missing=False
+    )
+    if not isinstance(model, SqlModel):
+        return []
+
+    query = model.query
+    dialect = model.dialect
+    columns_to_types = model.columns_to_types or {}
+
+    return _get_type_hints_for_model_from_query(
+        query, dialect, columns_to_types, start_line, end_line
+    )
+
+
+def _get_type_hints_for_select(
+    expression: exp.Expression,
+    dialect: str,
+    columns_to_types: t.Dict[str, exp.DataType],
+    start_line: int,
+    end_line: int,
+) -> t.List[types.InlayHint]:
+    hints: t.List[types.InlayHint] = []
+
+    for select_exp in expression.expressions:
+        if isinstance(select_exp, exp.Alias):
+            if isinstance(select_exp.this, exp.Cast):
+                continue
+
+            meta = select_exp.args["alias"].meta
+
+        elif isinstance(select_exp, exp.Column):
+            meta = select_exp.parts[-1].meta
+        else:
+            continue
+
+        if "line" not in meta or "col" not in meta:
+            continue
+
+        line = meta["line"]
+        col = meta["col"]
+
+        # Lines from sqlglot are 1 based
+        line -= 1
+
+        if line < start_line or line > end_line:
+            continue
+
+        name = select_exp.alias_or_name
+        data_type = columns_to_types.get(name)
+
+        if not data_type or data_type.is_type(exp.DataType.Type.UNKNOWN):
+            continue
+
+        type_label = data_type.sql(dialect)
+        hints.append(
+            types.InlayHint(
+                label=f"::{type_label}",
+                kind=types.InlayHintKind.Type,
+                padding_left=False,
+                padding_right=True,
+                position=types.Position(line=line, character=col),
+            )
+        )
+
+    return hints
+
+
+def _get_type_hints_for_model_from_query(
+    query: Expression,
+    dialect: str,
+    columns_to_types: t.Dict[str, exp.DataType],
+    start_line: int,
+    end_line: int,
+) -> t.List[types.InlayHint]:
+    hints: t.List[types.InlayHint] = []
+    try:
+        query = normalize_identifiers(query.copy(), dialect=dialect)
+
+        # Return the hints for top level selects (model definition columns only)
+        return [
+            hint
+            for q in query.walk(prune=lambda n: not isinstance(n, exp.SetOperation))
+            if isinstance(select := q.unnest(), exp.Select)
+            for hint in _get_type_hints_for_select(
+                q, dialect, columns_to_types, start_line, end_line
+            )
+        ]
+    except Exception:
+        return []