sqlmesh 0.188.2.dev17__py3-none-any.whl → 0.188.2.dev18__py3-none-any.whl

sqlmesh/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.188.2.dev17'
-__version_tuple__ = version_tuple = (0, 188, 2, 'dev17')
+__version__ = version = '0.188.2.dev18'
+__version_tuple__ = version_tuple = (0, 188, 2, 'dev18')

sqlmesh/core/console.py

@@ -8,7 +8,6 @@ import uuid
 import logging
 import textwrap
 from pathlib import Path
-import pandas as pd
 import numpy as np
 from hyperscript import h
 from rich.console import Console as RichConsole
@@ -2497,6 +2496,7 @@ class TerminalConsole(Console):
 
 def _cells_match(x: t.Any, y: t.Any) -> bool:
     """Helper function to compare two cells and returns true if they're equal, handling array objects."""
+    import pandas as pd
 
     # Convert array-like objects to list for consistent comparison
     def _normalize(val: t.Any) -> t.Any:

sqlmesh/core/context.py

@@ -48,7 +48,6 @@ from pathlib import Path
 from shutil import rmtree
 from types import MappingProxyType
 
-import pandas as pd
 from sqlglot import Dialect, exp
 from sqlglot.helper import first
 from sqlglot.lineage import GraphHTML
@@ -134,6 +133,7 @@ from sqlmesh.utils.config import print_config
 from sqlmesh.utils.jinja import JinjaMacroRegistry
 
 if t.TYPE_CHECKING:
+    import pandas as pd
     from typing_extensions import Literal
 
     from sqlmesh.core.engine_adapter._typing import (
@@ -1007,6 +1007,8 @@ class GenericContext(BaseContext, t.Generic[C]):
         expand = self.dag.upstream(model.fqn) if expand is True else expand or []
 
         if model.is_seed:
+            import pandas as pd
+
             df = next(
                 model.render(
                     context=self.execution_context(
@@ -2013,6 +2015,8 @@ class GenericContext(BaseContext, t.Generic[C]):
     ) -> ModelTextTestResult:
         """Discover and run model tests"""
         if verbosity >= Verbosity.VERBOSE:
+            import pandas as pd
+
             pd.set_option("display.max_columns", None)
 
         test_meta = self.load_model_tests(tests=tests, patterns=match_patterns)

sqlmesh/core/dialect.py

@@ -10,7 +10,6 @@ from difflib import unified_diff
 from enum import Enum, auto
 from functools import lru_cache
 
-import pandas as pd
 from sqlglot import Dialect, Generator, ParseError, Parser, Tokenizer, TokenType, exp
 from sqlglot.dialects.dialect import DialectType
 from sqlglot.dialects import DuckDB, Snowflake
@@ -27,6 +26,8 @@ from sqlmesh.utils.errors import SQLMeshError, ConfigError
 from sqlmesh.utils.pandas import columns_to_types_from_df
 
 if t.TYPE_CHECKING:
+    import pandas as pd
+
     from sqlglot._typing import E
 
 

sqlmesh/core/engine_adapter/_typing.py

@@ -1,11 +1,11 @@
 import typing as t
 
-import pandas as pd
 from sqlglot import exp
 
 from sqlmesh.utils import optional_import
 
 if t.TYPE_CHECKING:
+    import pandas as pd
     import pyspark
     import pyspark.sql.connect.dataframe
     from bigframes.session import Session as BigframeSession  # noqa

sqlmesh/core/engine_adapter/base.py

@@ -16,7 +16,6 @@ import sys
 import typing as t
 from functools import partial
 
-import pandas as pd
 from sqlglot import Dialect, exp
 from sqlglot.errors import ErrorLevel
 from sqlglot.helper import ensure_list
@@ -51,6 +50,8 @@ from sqlmesh.utils.errors import (
 from sqlmesh.utils.pandas import columns_to_types_from_df
 
 if t.TYPE_CHECKING:
+    import pandas as pd
+
     from sqlmesh.core._typing import SchemaName, SessionProperties, TableName
     from sqlmesh.core.engine_adapter._typing import (
         BigframeSession,
@@ -217,6 +218,8 @@ class EngineAdapter:
         *,
         batch_size: t.Optional[int] = None,
     ) -> t.List[SourceQuery]:
+        import pandas as pd
+
         batch_size = self.DEFAULT_BATCH_SIZE if batch_size is None else batch_size
         if isinstance(query_or_df, (exp.Query, exp.DerivedTable)):
             return [SourceQuery(query_factory=lambda: query_or_df)]  # type: ignore
@@ -244,6 +247,8 @@ class EngineAdapter:
         batch_size: int,
         target_table: TableName,
     ) -> t.List[SourceQuery]:
+        import pandas as pd
+
         assert isinstance(df, pd.DataFrame)
         num_rows = len(df.index)
         batch_size = sys.maxsize if batch_size == 0 else batch_size
@@ -257,7 +262,7 @@ class EngineAdapter:
             SourceQuery(
                 query_factory=partial(
                     self._values_to_sql,
-                    values=values,
+                    values=values,  # type: ignore
                     columns_to_types=columns_to_types,
                     batch_start=i,
                     batch_end=min(i + batch_size, num_rows),
@@ -295,6 +300,8 @@ class EngineAdapter:
     def _columns_to_types(
         self, query_or_df: QueryOrDF, columns_to_types: t.Optional[t.Dict[str, exp.DataType]] = None
     ) -> t.Optional[t.Dict[str, exp.DataType]]:
+        import pandas as pd
+
         if columns_to_types:
             return columns_to_types
         if isinstance(query_or_df, pd.DataFrame):
@@ -1006,6 +1013,8 @@ class EngineAdapter:
             view_properties: Optional view properties to add to the view.
             create_kwargs: Additional kwargs to pass into the Create expression
         """
+        import pandas as pd
+
         if materialized_properties and not materialized:
             raise SQLMeshError("Materialized properties are only supported for materialized views")
 
@@ -2012,6 +2021,8 @@ class EngineAdapter:
         """
         Take a "native" DataFrame (eg Pyspark, Bigframe, Snowpark etc) and convert it to Pandas
         """
+        import pandas as pd
+
         if isinstance(query_or_df, (exp.Query, exp.DerivedTable, pd.DataFrame)):
             return query_or_df
 
@@ -2022,6 +2033,8 @@ class EngineAdapter:
         self, query: t.Union[exp.Expression, str], quote_identifiers: bool = False
     ) -> pd.DataFrame:
         """Fetches a Pandas DataFrame from the cursor"""
+        import pandas as pd
+
         df = self._fetch_native_df(query, quote_identifiers=quote_identifiers)
         if not isinstance(df, pd.DataFrame):
             raise NotImplementedError(

sqlmesh/core/engine_adapter/bigquery.py

@@ -4,7 +4,6 @@ import logging
 import typing as t
 from collections import defaultdict
 
-import pandas as pd
 from sqlglot import exp, parse_one
 from sqlglot.transforms import remove_precision_parameterized_types
 
@@ -29,6 +28,7 @@ from sqlmesh.utils.errors import SQLMeshError
 from sqlmesh.utils.pandas import columns_to_types_from_dtypes
 
 if t.TYPE_CHECKING:
+    import pandas as pd
     from google.api_core.retry import Retry
     from google.cloud import bigquery
     from google.cloud.bigquery import StandardSqlDataType
@@ -147,6 +147,8 @@ class BigQueryEngineAdapter(InsertOverwriteWithMergeMixin, ClusteredByMixin, Row
         batch_size: int,
         target_table: TableName,
     ) -> t.List[SourceQuery]:
+        import pandas as pd
+
         temp_bq_table = self.__get_temp_bq_table(
             self._get_temp_table(target_table or "pandas"), columns_to_types
         )

sqlmesh/core/engine_adapter/clickhouse.py

@@ -2,7 +2,6 @@ from __future__ import annotations
 
 import typing as t
 import logging
-import pandas as pd
 import re
 from sqlglot import exp, maybe_parse
 from sqlmesh.core.dialect import to_schema
@@ -19,6 +18,8 @@ from sqlmesh.core.engine_adapter.shared import (
 from sqlmesh.core.schema_diff import SchemaDiffer
 
 if t.TYPE_CHECKING:
+    import pandas as pd
+
     from sqlmesh.core._typing import SchemaName, TableName
     from sqlmesh.core.engine_adapter._typing import DF, Query, QueryOrDF
 

sqlmesh/core/engine_adapter/databricks.py

@@ -4,7 +4,6 @@ import logging
 import typing as t
 from functools import partial
 
-import pandas as pd
 from sqlglot import exp
 from sqlmesh.core.dialect import to_schema
 from sqlmesh.core.engine_adapter.shared import (
@@ -21,6 +20,8 @@ from sqlmesh.engines.spark.db_api.spark_session import connection, SparkSessionC
 from sqlmesh.utils.errors import SQLMeshError, MissingDefaultCatalogError
 
 if t.TYPE_CHECKING:
+    import pandas as pd
+
     from sqlmesh.core._typing import SchemaName, TableName, SessionProperties
     from sqlmesh.core.engine_adapter._typing import DF, PySparkSession, Query
 
@@ -199,6 +200,8 @@ class DatabricksEngineAdapter(SparkEngineAdapter):
         """
         Returns a Pandas DataFrame from a query or expression.
         """
+        import pandas as pd
+
         df = self._fetch_native_df(query, quote_identifiers=quote_identifiers)
         if not isinstance(df, pd.DataFrame):
             return df.toPandas()

sqlmesh/core/engine_adapter/mssql.py

@@ -5,8 +5,6 @@ from __future__ import annotations
 import typing as t
 
 import numpy as np
-import pandas as pd
-from pandas.api.types import is_datetime64_any_dtype  # type: ignore
 from sqlglot import exp
 
 from sqlmesh.core.dialect import to_schema
@@ -189,6 +187,9 @@ class MSSQLEngineAdapter(
         super().drop_schema(schema_name, ignore_if_not_exists=ignore_if_not_exists, cascade=False)
 
     def _convert_df_datetime(self, df: DF, columns_to_types: t.Dict[str, exp.DataType]) -> None:
+        import pandas as pd
+        from pandas.api.types import is_datetime64_any_dtype  # type: ignore
+
         # pymssql doesn't convert Pandas Timestamp (datetime64) types
         # - this code is based on snowflake adapter implementation
         for column, kind in columns_to_types.items():
@@ -213,6 +214,8 @@ class MSSQLEngineAdapter(
         batch_size: int,
         target_table: TableName,
     ) -> t.List[SourceQuery]:
+        import pandas as pd
+
         assert isinstance(df, pd.DataFrame)
         temp_table = self._get_temp_table(target_table or "pandas")
 
@@ -247,6 +250,8 @@ class MSSQLEngineAdapter(
         """
         Returns all the data objects that exist in the given schema and catalog.
         """
+        import pandas as pd
+
         catalog = self.get_current_catalog()
         query = (
             exp.select(

sqlmesh/core/engine_adapter/redshift.py

@@ -3,7 +3,6 @@ from __future__ import annotations
 import logging
 import typing as t
 
-import pandas as pd
 from sqlglot import exp
 
 from sqlmesh.core.dialect import to_schema
@@ -27,6 +26,8 @@ from sqlmesh.core.schema_diff import SchemaDiffer
 from sqlmesh.utils.errors import SQLMeshError
 
 if t.TYPE_CHECKING:
+    import pandas as pd
+
     from sqlmesh.core._typing import SchemaName, TableName
     from sqlmesh.core.engine_adapter.base import QueryOrDF, Query
 
@@ -142,6 +143,8 @@ class RedshiftEngineAdapter(
         self, query: t.Union[exp.Expression, str], quote_identifiers: bool = False
     ) -> pd.DataFrame:
         """Fetches a Pandas DataFrame from the cursor"""
+        import pandas as pd
+
         self.execute(query, quote_identifiers=quote_identifiers)
 
         # We manually build the `DataFrame` here because the driver's `fetch_dataframe`
@@ -257,6 +260,8 @@ class RedshiftEngineAdapter(
         If it does exist then we need to do the:
             `CREATE TABLE...`, `INSERT INTO...`, `RENAME TABLE...`, `RENAME TABLE...`, DROP TABLE...`  dance.
         """
+        import pandas as pd
+
         if not isinstance(query_or_df, pd.DataFrame) or not self.table_exists(table_name):
             return super().replace_query(
                 table_name,

sqlmesh/core/engine_adapter/snowflake.py

@@ -4,8 +4,6 @@ import contextlib
 import logging
 import typing as t
 
-import pandas as pd
-from pandas.api.types import is_datetime64_any_dtype  # type: ignore
 from sqlglot import exp
 from sqlglot.helper import ensure_list
 from sqlglot.optimizer.normalize_identifiers import normalize_identifiers
@@ -33,6 +31,8 @@ logger = logging.getLogger(__name__)
 snowpark = optional_import("snowflake.snowpark")
 
 if t.TYPE_CHECKING:
+    import pandas as pd
+
     from sqlmesh.core._typing import SchemaName, SessionProperties, TableName
     from sqlmesh.core.engine_adapter._typing import DF, Query, QueryOrDF, SnowparkSession
     from sqlmesh.core.node import IntervalUnit
@@ -290,6 +290,9 @@ class SnowflakeEngineAdapter(GetCurrentCatalogFromFunctionMixin, ClusteredByMixi
         batch_size: int,
         target_table: TableName,
     ) -> t.List[SourceQuery]:
+        import pandas as pd
+        from pandas.api.types import is_datetime64_any_dtype
+
         temp_table = self._get_temp_table(
             target_table or "pandas", quoted=False
         )  # write_pandas() re-quotes everything without checking if its already quoted
@@ -391,6 +394,7 @@ class SnowflakeEngineAdapter(GetCurrentCatalogFromFunctionMixin, ClusteredByMixi
     def _fetch_native_df(
         self, query: t.Union[exp.Expression, str], quote_identifiers: bool = False
     ) -> DF:
+        import pandas as pd
         from snowflake.connector.errors import NotSupportedError
 
         self.execute(query, quote_identifiers=quote_identifiers)

sqlmesh/core/engine_adapter/spark.py

@@ -4,7 +4,6 @@ import logging
 import typing as t
 from functools import partial
 
-import pandas as pd
 from sqlglot import exp
 
 from sqlmesh.core.dialect import to_schema
@@ -28,6 +27,7 @@ from sqlmesh.utils import classproperty
 from sqlmesh.utils.errors import SQLMeshError
 
 if t.TYPE_CHECKING:
+    import pandas as pd
     from pyspark.sql import types as spark_types
 
     from sqlmesh.core._typing import SchemaName, TableName
@@ -232,6 +232,8 @@ class SparkEngineAdapter(
 
     @classmethod
     def try_get_pandas_df(cls, value: t.Any) -> t.Optional[pd.DataFrame]:
+        import pandas as pd
+
         if isinstance(value, pd.DataFrame):
             return value
         return None

sqlmesh/core/engine_adapter/trino.py

@@ -2,8 +2,7 @@ from __future__ import annotations
 import re
 import typing as t
 from functools import lru_cache
-import pandas as pd
-from pandas.api.types import is_datetime64_any_dtype  # type: ignore
+
 from sqlglot import exp
 from sqlglot.helper import seq_get
 from tenacity import retry, wait_fixed, stop_after_attempt, retry_if_result
@@ -194,6 +193,9 @@ class TrinoEngineAdapter(
         batch_size: int,
         target_table: TableName,
     ) -> t.List[SourceQuery]:
+        import pandas as pd
+        from pandas.api.types import is_datetime64_any_dtype  # type: ignore
+
         assert isinstance(df, pd.DataFrame)
 
         # Trino does not accept timestamps in ISOFORMAT that include the "T". `execution_time` is stored in

sqlmesh/core/model/definition.py

@@ -8,7 +8,6 @@ import typing as t
 from functools import cached_property, partial
 from pathlib import Path
 
-import pandas as pd
 import numpy as np
 from pydantic import Field
 from sqlglot import diff, exp
@@ -1596,6 +1595,8 @@ class SeedModel(_Model):
 
             # convert all date/time types to native pandas timestamp
             for column in [*date_columns, *datetime_columns]:
+                import pandas as pd
+
                 df[column] = pd.to_datetime(df[column])
 
             # extract datetime.date from pandas timestamp for DATE columns

sqlmesh/core/model/seed.py

@@ -6,7 +6,6 @@ import zlib
 from io import StringIO
 from pathlib import Path
 
-import pandas as pd
 from sqlglot import exp
 from sqlglot.dialects.dialect import UNESCAPED_SEQUENCES
 from sqlglot.helper import seq_get
@@ -16,6 +15,9 @@ from sqlmesh.core.model.common import parse_bool
 from sqlmesh.utils.pandas import columns_to_types_from_df
 from sqlmesh.utils.pydantic import PydanticModel, field_validator
 
+if t.TYPE_CHECKING:
+    import pandas as pd
+
 logger = logging.getLogger(__name__)
 
 NaHashables = t.List[t.Union[int, str, bool, t.Literal[None]]]
@@ -115,6 +117,8 @@ class CsvSeedReader:
             batch_start += batch_size
 
     def _get_df(self) -> pd.DataFrame:
+        import pandas as pd
+
         if self._df is None:
             self._df = pd.read_csv(
                 StringIO(self.content),

sqlmesh/core/snapshot/evaluator.py

@@ -30,7 +30,6 @@ from collections import defaultdict
 from contextlib import contextmanager
 from functools import reduce
 
-import pandas as pd
 from sqlglot import exp, select
 from sqlglot.executor import execute
 
@@ -736,6 +735,8 @@ class SnapshotEvaluator:
             )
 
             if limit is not None:
+                import pandas as pd
+
                 query_or_df = next(queries_or_dfs)
                 if isinstance(query_or_df, pd.DataFrame):
                     return query_or_df.head(limit)
@@ -769,6 +770,8 @@ class SnapshotEvaluator:
                 )
                 and snapshot.is_incremental_by_time_range
             ):
+                import pandas as pd
+
                 query_or_df = reduce(
                     lambda a, b: (
                         pd.concat([a, b], ignore_index=True)  # type: ignore

sqlmesh/core/state_sync/db/environment.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import typing as t
-import pandas as pd
 import json
 import logging
 from sqlglot import exp
@@ -17,6 +16,9 @@ from sqlmesh.utils.migration import index_text_type, blob_text_type
 from sqlmesh.utils.date import now_timestamp, time_like_to_str
 from sqlmesh.utils.errors import SQLMeshError
 
+if t.TYPE_CHECKING:
+    import pandas as pd
+
 
 logger = logging.getLogger(__name__)
 
@@ -330,6 +332,8 @@ class EnvironmentState:
 
 
 def _environment_to_df(environment: Environment) -> pd.DataFrame:
+    import pandas as pd
+
     return pd.DataFrame(
         [
             {
@@ -364,6 +368,8 @@ def _environment_to_df(environment: Environment) -> pd.DataFrame:
 def _environment_statements_to_df(
     environment_name: str, plan_id: str, environment_statements: t.List[EnvironmentStatements]
 ) -> pd.DataFrame:
+    import pandas as pd
+
     return pd.DataFrame(
         [
             {

sqlmesh/core/state_sync/db/interval.py

@@ -2,7 +2,6 @@ from __future__ import annotations
 
 import typing as t
 import logging
-import pandas as pd
 
 from sqlglot import exp
 
@@ -27,6 +26,9 @@ from sqlmesh.utils.migration import index_text_type
 from sqlmesh.utils import random_id
 from sqlmesh.utils.date import now_timestamp
 
+if t.TYPE_CHECKING:
+    import pandas as pd
+
 
 logger = logging.getLogger(__name__)
 
@@ -205,6 +207,8 @@ class IntervalState:
         snapshots: t.Iterable[t.Union[Snapshot, SnapshotIntervals]],
         is_compacted: bool = False,
     ) -> None:
+        import pandas as pd
+
         new_intervals = []
         for snapshot in snapshots:
             logger.info("Pushing intervals for snapshot %s", snapshot.snapshot_id)
@@ -430,6 +434,8 @@ def _intervals_to_df(
     is_dev: bool,
     is_removed: bool,
 ) -> pd.DataFrame:
+    import pandas as pd
+
     return pd.DataFrame(
         [
             _interval_to_df(

sqlmesh/core/state_sync/db/snapshot.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import typing as t
-import pandas as pd
 import json
 import logging
 from pathlib import Path
@@ -39,6 +38,8 @@ from sqlmesh.utils.pydantic import PydanticModel
 from sqlmesh.utils import unique
 
 if t.TYPE_CHECKING:
+    import pandas as pd
+
     from sqlmesh.core.state_sync.db.interval import IntervalState
 
 
@@ -674,6 +675,8 @@ def _snapshot_to_json(snapshot: Snapshot) -> str:
 
 
 def _snapshots_to_df(snapshots: t.Iterable[Snapshot]) -> pd.DataFrame:
+    import pandas as pd
+
     return pd.DataFrame(
         [
             {
@@ -693,6 +696,8 @@ def _snapshots_to_df(snapshots: t.Iterable[Snapshot]) -> pd.DataFrame:
 
 
 def _auto_restatements_to_df(auto_restatements: t.Dict[SnapshotNameVersion, int]) -> pd.DataFrame:
+    import pandas as pd
+
     return pd.DataFrame(
         [
             {

sqlmesh/core/state_sync/db/version.py

@@ -3,7 +3,6 @@ from __future__ import annotations
 import logging
 import typing as t
 
-import pandas as pd
 from sqlglot import __version__ as SQLGLOT_VERSION
 from sqlglot import exp
 from sqlglot.helper import seq_get
@@ -40,6 +39,8 @@ class VersionState:
         sqlglot_version: str = SQLGLOT_VERSION,
         sqlmesh_version: str = SQLMESH_VERSION,
     ) -> None:
+        import pandas as pd
+
         self.engine_adapter.delete_from(self.versions_table, "TRUE")
 
         self.engine_adapter.insert_append(

sqlmesh/core/table_diff.py

@@ -4,8 +4,6 @@ import math
 import typing as t
 from functools import cached_property
 
-import pandas as pd
-
 from sqlmesh.core.dialect import to_schema
 from sqlmesh.core.engine_adapter.mixins import RowDiffMixin
 from sqlmesh.core.engine_adapter.athena import AthenaEngineAdapter
@@ -18,7 +16,10 @@ from sqlglot.optimizer.scope import find_all_in_scope
 from sqlmesh.utils.pydantic import PydanticModel
 from sqlmesh.utils.errors import SQLMeshError
 
+
 if t.TYPE_CHECKING:
+    import pandas as pd
+
     from sqlmesh.core._typing import TableName
     from sqlmesh.core.engine_adapter import EngineAdapter
 
@@ -75,6 +76,21 @@ class RowDiff(PydanticModel, frozen=True):
     model_name: t.Optional[str] = None
     decimals: int = 3
 
+    _types_resolved: t.ClassVar[bool] = False
+
+    def __new__(cls, *args: t.Any, **kwargs: t.Any) -> RowDiff:
+        if not cls._types_resolved:
+            cls._resolve_types()
+        return super().__new__(cls)
+
+    @classmethod
+    def _resolve_types(cls) -> None:
+        # Pandas is imported by type checking so we need to resolve the types with the real import before instantiating
+        import pandas as pd  # noqa
+
+        cls.model_rebuild()
+        cls._types_resolved = True
+
     @property
     def source_count(self) -> int:
         """Count of the source."""