sqlmesh 0.213.1.dev1__py3-none-any.whl → 0.227.2.dev4__py3-none-any.whl

sqlmesh/core/state_sync/__init__.py

@@ -20,5 +20,4 @@ from sqlmesh.core.state_sync.base import (
     Versions as Versions,
 )
 from sqlmesh.core.state_sync.cache import CachingStateSync as CachingStateSync
-from sqlmesh.core.state_sync.common import cleanup_expired_views as cleanup_expired_views
 from sqlmesh.core.state_sync.db import EngineAdapterStateSync as EngineAdapterStateSync

sqlmesh/core/state_sync/base.py

@@ -11,7 +11,6 @@ from sqlglot import __version__ as SQLGLOT_VERSION
 from sqlmesh import migrations
 from sqlmesh.core.environment import (
     Environment,
-    EnvironmentNamingInfo,
     EnvironmentStatements,
     EnvironmentSummary,
 )
@@ -19,17 +18,22 @@ from sqlmesh.core.snapshot import (
     Snapshot,
     SnapshotId,
     SnapshotIdLike,
+    SnapshotIdAndVersionLike,
     SnapshotInfoLike,
-    SnapshotTableCleanupTask,
-    SnapshotTableInfo,
     SnapshotNameVersion,
+    SnapshotIdAndVersion,
 )
 from sqlmesh.core.snapshot.definition import Interval, SnapshotIntervals
 from sqlmesh.utils import major_minor
 from sqlmesh.utils.date import TimeLike
 from sqlmesh.utils.errors import SQLMeshError
-from sqlmesh.utils.pydantic import PydanticModel, ValidationInfo, field_validator
-from sqlmesh.core.state_sync.common import StateStream
+from sqlmesh.utils.pydantic import PydanticModel, field_validator
+from sqlmesh.core.state_sync.common import (
+    StateStream,
+    ExpiredSnapshotBatch,
+    PromotionResult,
+    ExpiredBatchRange,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -60,25 +64,14 @@ class Versions(PydanticModel):
         return 0 if v is None else int(v)
 
 
+MIN_SCHEMA_VERSION = 60
+MIN_SQLMESH_VERSION = "0.134.0"
 MIGRATIONS = [
     importlib.import_module(f"sqlmesh.migrations.{migration}")
     for migration in sorted(info.name for info in pkgutil.iter_modules(migrations.__path__))
 ]
-SCHEMA_VERSION: int = len(MIGRATIONS)
-
-
-class PromotionResult(PydanticModel):
-    added: t.List[SnapshotTableInfo]
-    removed: t.List[SnapshotTableInfo]
-    removed_environment_naming_info: t.Optional[EnvironmentNamingInfo]
-
-    @field_validator("removed_environment_naming_info")
-    def _validate_removed_environment_naming_info(
-        cls, v: t.Optional[EnvironmentNamingInfo], info: ValidationInfo
-    ) -> t.Optional[EnvironmentNamingInfo]:
-        if v and not info.data.get("removed"):
-            raise ValueError("removed_environment_naming_info must be None if removed is empty")
-        return v
+# -1 to account for the baseline script
+SCHEMA_VERSION: int = MIN_SCHEMA_VERSION + len(MIGRATIONS) - 1
 
 
 class StateReader(abc.ABC):
@@ -97,6 +90,24 @@ class StateReader(abc.ABC):
             A dictionary of snapshot ids to snapshots for ones that could be found.
         """
 
+    @abc.abstractmethod
+    def get_snapshots_by_names(
+        self,
+        snapshot_names: t.Iterable[str],
+        current_ts: t.Optional[int] = None,
+        exclude_expired: bool = True,
+    ) -> t.Set[SnapshotIdAndVersion]:
+        """Return the snapshot records for all versions of the specified snapshot names.
+
+        Args:
+            snapshot_names: Iterable of snapshot names to fetch all snapshot records for
+            current_ts: Sets the current time for identifying which snapshots have expired so they can be excluded (only relevant if :exclude_expired=True)
+            exclude_expired: Whether or not to return the snapshot id's of expired snapshots in the result
+
+        Returns:
+            A set containing all the matched snapshot records. To fetch full snapshots, pass it into StateSync.get_snapshots()
+        """
+
     @abc.abstractmethod
     def snapshots_exist(self, snapshot_ids: t.Iterable[SnapshotIdLike]) -> t.Set[SnapshotId]:
         """Checks if multiple snapshots exist in the state sync.
@@ -292,15 +303,21 @@ class StateReader(abc.ABC):
 
     @abc.abstractmethod
     def get_expired_snapshots(
-        self, current_ts: int, ignore_ttl: bool = False
-    ) -> t.List[SnapshotTableCleanupTask]:
-        """Aggregates the id's of the expired snapshots and creates a list of table cleanup tasks.
+        self,
+        *,
+        batch_range: ExpiredBatchRange,
+        current_ts: t.Optional[int] = None,
+        ignore_ttl: bool = False,
+    ) -> t.Optional[ExpiredSnapshotBatch]:
+        """Returns a single batch of expired snapshots ordered by (updated_ts, name, identifier).
 
-        Expired snapshots are snapshots that have exceeded their time-to-live
-        and are no longer in use within an environment.
+        Args:
+            current_ts: Timestamp used to evaluate expiration.
+            ignore_ttl: If True, include snapshots regardless of TTL (only checks if unreferenced).
+            batch_range: The range of the batch to fetch.
 
         Returns:
-           The list of table cleanup tasks.
+            A batch describing expired snapshots or None if no snapshots are pending cleanup.
         """
 
     @abc.abstractmethod
@@ -340,19 +357,21 @@ class StateSync(StateReader, abc.ABC):
 
     @abc.abstractmethod
     def delete_expired_snapshots(
-        self, ignore_ttl: bool = False, current_ts: t.Optional[int] = None
-    ) -> t.List[SnapshotTableCleanupTask]:
+        self,
+        batch_range: ExpiredBatchRange,
+        ignore_ttl: bool = False,
+        current_ts: t.Optional[int] = None,
+    ) -> None:
         """Removes expired snapshots.
 
         Expired snapshots are snapshots that have exceeded their time-to-live
         and are no longer in use within an environment.
 
         Args:
+            batch_range: The range of snapshots to delete in this batch.
             ignore_ttl: Ignore the TTL on the snapshot when considering it expired. This has the effect of deleting
                 all snapshots that are not referenced in any environment
-
-        Returns:
-            The list of snapshot table cleanup tasks.
+            current_ts: Timestamp used to evaluate expiration.
         """
 
     @abc.abstractmethod
@@ -371,7 +390,7 @@ class StateSync(StateReader, abc.ABC):
     @abc.abstractmethod
     def remove_intervals(
         self,
-        snapshot_intervals: t.Sequence[t.Tuple[SnapshotInfoLike, Interval]],
+        snapshot_intervals: t.Sequence[t.Tuple[SnapshotIdAndVersionLike, Interval]],
         remove_shared_versions: bool = False,
     ) -> None:
         """Remove an interval from a list of snapshots and sync it to the store.
@@ -453,7 +472,6 @@ class StateSync(StateReader, abc.ABC):
     @abc.abstractmethod
     def migrate(
         self,
-        default_catalog: t.Optional[str],
         skip_backup: bool = False,
         promoted_snapshots_only: bool = True,
     ) -> None:
@@ -477,6 +495,7 @@ class StateSync(StateReader, abc.ABC):
         start: TimeLike,
         end: TimeLike,
         is_dev: bool = False,
+        last_altered_ts: t.Optional[int] = None,
     ) -> None:
         """Add an interval to a snapshot and sync it to the store.
 
@@ -485,6 +504,7 @@ class StateSync(StateReader, abc.ABC):
             start: The start of the interval to add.
             end: The end of the interval to add.
             is_dev: Indicates whether the given interval is being added while in development mode
+            last_altered_ts: The timestamp of the last modification of the physical table
         """
         start_ts, end_ts = snapshot.inclusive_exclusive(start, end, strict=False, expand=False)
         if not snapshot.version:
@@ -497,6 +517,8 @@ class StateSync(StateReader, abc.ABC):
             dev_version=snapshot.dev_version,
             intervals=intervals if not is_dev else [],
             dev_intervals=intervals if is_dev else [],
+            last_altered_ts=last_altered_ts if not is_dev else None,
+            dev_last_altered_ts=last_altered_ts if is_dev else None,
         )
         self.add_snapshots_intervals([snapshot_intervals])
 

sqlmesh/core/state_sync/cache.py

@@ -7,11 +7,12 @@ from sqlmesh.core.snapshot import (
     Snapshot,
     SnapshotId,
     SnapshotIdLike,
+    SnapshotIdAndVersionLike,
     SnapshotInfoLike,
-    SnapshotTableCleanupTask,
 )
 from sqlmesh.core.snapshot.definition import Interval, SnapshotIntervals
 from sqlmesh.core.state_sync.base import DelegatingStateSync, StateSync
+from sqlmesh.core.state_sync.common import ExpiredBatchRange
 from sqlmesh.utils.date import TimeLike, now_timestamp
 
 
@@ -108,12 +109,16 @@ class CachingStateSync(DelegatingStateSync):
         self.state_sync.delete_snapshots(snapshot_ids)
 
     def delete_expired_snapshots(
-        self, ignore_ttl: bool = False, current_ts: t.Optional[int] = None
-    ) -> t.List[SnapshotTableCleanupTask]:
-        current_ts = current_ts or now_timestamp()
+        self,
+        batch_range: ExpiredBatchRange,
+        ignore_ttl: bool = False,
+        current_ts: t.Optional[int] = None,
+    ) -> None:
         self.snapshot_cache.clear()
-        return self.state_sync.delete_expired_snapshots(
-            current_ts=current_ts, ignore_ttl=ignore_ttl
+        self.state_sync.delete_expired_snapshots(
+            batch_range=batch_range,
+            ignore_ttl=ignore_ttl,
+            current_ts=current_ts,
         )
 
     def add_snapshots_intervals(self, snapshots_intervals: t.Sequence[SnapshotIntervals]) -> None:
@@ -131,7 +136,7 @@ class CachingStateSync(DelegatingStateSync):
 
     def remove_intervals(
         self,
-        snapshot_intervals: t.Sequence[t.Tuple[SnapshotInfoLike, Interval]],
+        snapshot_intervals: t.Sequence[t.Tuple[SnapshotIdAndVersionLike, Interval]],
         remove_shared_versions: bool = False,
     ) -> None:
         for s, _ in snapshot_intervals:

sqlmesh/core/state_sync/common.py

@@ -7,124 +7,25 @@ import itertools
 import abc
 
 from dataclasses import dataclass
+
+from pydantic_core.core_schema import ValidationInfo
 from sqlglot import exp
 
-from sqlmesh.core.console import Console
-from sqlmesh.core.dialect import schema_
-from sqlmesh.utils.pydantic import PydanticModel
-from sqlmesh.core.environment import Environment, EnvironmentStatements
-from sqlmesh.utils.errors import SQLMeshError
-from sqlmesh.core.snapshot import Snapshot
+from sqlmesh.utils.pydantic import PydanticModel, field_validator
+from sqlmesh.core.environment import Environment, EnvironmentStatements, EnvironmentNamingInfo
+from sqlmesh.core.snapshot import (
+    Snapshot,
+    SnapshotId,
+    SnapshotTableCleanupTask,
+    SnapshotTableInfo,
+)
 
 if t.TYPE_CHECKING:
-    from sqlmesh.core.engine_adapter.base import EngineAdapter
-    from sqlmesh.core.state_sync.base import Versions
+    from sqlmesh.core.state_sync.base import Versions, StateReader
 
 logger = logging.getLogger(__name__)
 
-
-def cleanup_expired_views(
-    default_adapter: EngineAdapter,
-    engine_adapters: t.Dict[str, EngineAdapter],
-    environments: t.List[Environment],
-    warn_on_delete_failure: bool = False,
-    console: t.Optional[Console] = None,
-) -> None:
-    expired_schema_or_catalog_environments = [
-        environment
-        for environment in environments
-        if environment.suffix_target.is_schema or environment.suffix_target.is_catalog
-    ]
-    expired_table_environments = [
-        environment for environment in environments if environment.suffix_target.is_table
-    ]
-
-    # We have to use the corresponding adapter if the virtual layer is gateway managed
-    def get_adapter(gateway_managed: bool, gateway: t.Optional[str] = None) -> EngineAdapter:
-        if gateway_managed and gateway:
-            return engine_adapters.get(gateway, default_adapter)
-        return default_adapter
-
-    catalogs_to_drop: t.Set[t.Tuple[EngineAdapter, str]] = set()
-    schemas_to_drop: t.Set[t.Tuple[EngineAdapter, exp.Table]] = set()
-
-    # Collect schemas and catalogs to drop
-    for engine_adapter, expired_catalog, expired_schema, suffix_target in {
-        (
-            (engine_adapter := get_adapter(environment.gateway_managed, snapshot.model_gateway)),
-            snapshot.qualified_view_name.catalog_for_environment(
-                environment.naming_info, dialect=engine_adapter.dialect
-            ),
-            snapshot.qualified_view_name.schema_for_environment(
-                environment.naming_info, dialect=engine_adapter.dialect
-            ),
-            environment.suffix_target,
-        )
-        for environment in expired_schema_or_catalog_environments
-        for snapshot in environment.snapshots
-        if snapshot.is_model and not snapshot.is_symbolic
-    }:
-        if suffix_target.is_catalog:
-            if expired_catalog:
-                catalogs_to_drop.add((engine_adapter, expired_catalog))
-        else:
-            schema = schema_(expired_schema, expired_catalog)
-            schemas_to_drop.add((engine_adapter, schema))
-
-    # Drop the views for the expired environments
-    for engine_adapter, expired_view in {
-        (
-            (engine_adapter := get_adapter(environment.gateway_managed, snapshot.model_gateway)),
-            snapshot.qualified_view_name.for_environment(
-                environment.naming_info, dialect=engine_adapter.dialect
-            ),
-        )
-        for environment in expired_table_environments
-        for snapshot in environment.snapshots
-        if snapshot.is_model and not snapshot.is_symbolic
-    }:
-        try:
-            engine_adapter.drop_view(expired_view, ignore_if_not_exists=True)
-            if console:
-                console.update_cleanup_progress(expired_view)
-        except Exception as e:
-            message = f"Failed to drop the expired environment view '{expired_view}': {e}"
-            if warn_on_delete_failure:
-                logger.warning(message)
-            else:
-                raise SQLMeshError(message) from e
-
-    # Drop the schemas for the expired environments
-    for engine_adapter, schema in schemas_to_drop:
-        try:
-            engine_adapter.drop_schema(
-                schema,
-                ignore_if_not_exists=True,
-                cascade=True,
-            )
-            if console:
-                console.update_cleanup_progress(schema.sql(dialect=engine_adapter.dialect))
-        except Exception as e:
-            message = f"Failed to drop the expired environment schema '{schema}': {e}"
-            if warn_on_delete_failure:
-                logger.warning(message)
-            else:
-                raise SQLMeshError(message) from e
-
-    # Drop any catalogs that were associated with a snapshot where the engine adapter supports dropping catalogs
-    # catalogs_to_drop is only populated when environment_suffix_target is set to 'catalog'
-    for engine_adapter, catalog in catalogs_to_drop:
-        if engine_adapter.SUPPORTS_CREATE_DROP_CATALOG:
-            try:
-                engine_adapter.drop_catalog(catalog)
-                if console:
-                    console.update_cleanup_progress(catalog)
-            except Exception as e:
-                message = f"Failed to drop the expired environment catalog '{catalog}': {e}"
-                if warn_on_delete_failure:
-                    logger.warning(message)
-                else:
-                    raise SQLMeshError(message) from e
+EXPIRED_SNAPSHOT_DEFAULT_BATCH_SIZE = 200
 
 
 def transactional() -> t.Callable[[t.Callable], t.Callable]:
@@ -215,3 +116,207 @@ class StateStream(abc.ABC):
                 yield EnvironmentsChunk(environments)
 
         return _StateStream()
+
+
+class ExpiredBatchRange(PydanticModel):
+    start: RowBoundary
+    end: t.Union[RowBoundary, LimitBoundary]
+
+    @classmethod
+    def init_batch_range(cls, batch_size: int) -> ExpiredBatchRange:
+        return ExpiredBatchRange(
+            start=RowBoundary.lowest_boundary(),
+            end=LimitBoundary(batch_size=batch_size),
+        )
+
+    @classmethod
+    def all_batch_range(cls) -> ExpiredBatchRange:
+        return ExpiredBatchRange(
+            start=RowBoundary.lowest_boundary(),
+            end=RowBoundary.highest_boundary(),
+        )
+
+    @classmethod
+    def _expanded_tuple_comparison(
+        cls,
+        columns: t.List[exp.Column],
+        values: t.List[exp.Literal],
+        operator: t.Type[exp.Expression],
+    ) -> exp.Expression:
+        """Generate expanded tuple comparison that works across all SQL engines.
+
+        Converts tuple comparisons like (a, b, c) OP (x, y, z) into an expanded form
+        that's compatible with all SQL engines, since native tuple comparisons have
+        inconsistent support across engines (especially DuckDB, MySQL, SQLite).
+
+        Repro of problem with DuckDB:
+            "SELECT * FROM VALUES(1,'2') as test(a,b) WHERE ((a, b) > (1, 'foo')) AND ((a, b) <= (10, 'baz'))"
+
+        Args:
+            columns: List of column expressions to compare
+            values: List of value expressions to compare against
+            operator: The comparison operator class (exp.GT, exp.GTE, exp.LT, exp.LTE)
+
+        Examples:
+            (a, b, c) > (x, y, z) expands to:
+                a > x OR (a = x AND b > y) OR (a = x AND b = y AND c > z)
+
+            (a, b, c) <= (x, y, z) expands to:
+                a < x OR (a = x AND b < y) OR (a = x AND b = y AND c <= z)
+
+            (a, b, c) >= (x, y, z) expands to:
+                a > x OR (a = x AND b > y) OR (a = x AND b = y AND c >= z)
+
+        Returns:
+            An expanded OR expression representing the tuple comparison
+        """
+        if operator not in (exp.GT, exp.GTE, exp.LT, exp.LTE):
+            raise ValueError(f"Unsupported operator: {operator}. Use GT, GTE, LT, or LTE.")
+
+        # For <= and >=, we use the strict operator for all but the last column
+        # e.g., (a, b) <= (x, y) becomes: a < x OR (a = x AND b <= y)
+        # For < and >, we use the strict operator throughout
+        # e.g., (a, b) > (x, y) becomes: a > x OR (a = x AND b > x)
+        strict_operator: t.Type[exp.Expression]
+        final_operator: t.Type[exp.Expression]
+
+        if operator in (exp.LTE, exp.GTE):
+            # For inclusive operators (<=, >=), use strict form for intermediate columns
+            # but keep inclusive form for the last column
+            strict_operator = exp.LT if operator == exp.LTE else exp.GT
+            final_operator = operator  # Keep LTE/GTE for last column
+        else:
+            # For strict operators (<, >), use them throughout
+            strict_operator = operator
+            final_operator = operator
+
+        conditions: t.List[exp.Expression] = []
+        for i in range(len(columns)):
+            # Build equality conditions for all columns before current
+            equality_conditions = [exp.EQ(this=columns[j], expression=values[j]) for j in range(i)]
+
+            # Use the final operator for the last column, strict for others
+            comparison_op = final_operator if i == len(columns) - 1 else strict_operator
+            comparison_condition = comparison_op(this=columns[i], expression=values[i])
+
+            if equality_conditions:
+                conditions.append(exp.and_(*equality_conditions, comparison_condition))
+            else:
+                conditions.append(comparison_condition)
+
+        return exp.or_(*conditions) if len(conditions) > 1 else conditions[0]
+
+    @property
+    def where_filter(self) -> exp.Expression:
+        # Use expanded tuple comparisons for cross-engine compatibility
+        # Native tuple comparisons like (a, b) > (x, y) don't work reliably across all SQL engines
+        columns = [
+            exp.column("updated_ts"),
+            exp.column("name"),
+            exp.column("identifier"),
+        ]
+        start_values = [
+            exp.Literal.number(self.start.updated_ts),
+            exp.Literal.string(self.start.name),
+            exp.Literal.string(self.start.identifier),
+        ]
+
+        start_condition = self._expanded_tuple_comparison(columns, start_values, exp.GT)
+
+        range_filter: exp.Expression
+        if isinstance(self.end, RowBoundary):
+            end_values = [
+                exp.Literal.number(self.end.updated_ts),
+                exp.Literal.string(self.end.name),
+                exp.Literal.string(self.end.identifier),
+            ]
+            end_condition = self._expanded_tuple_comparison(columns, end_values, exp.LTE)
+            range_filter = exp.and_(start_condition, end_condition)
+        else:
+            range_filter = start_condition
+        return range_filter
+
+
+class RowBoundary(PydanticModel):
+    updated_ts: int
+    name: str
+    identifier: str
+
+    @classmethod
+    def lowest_boundary(cls) -> RowBoundary:
+        return RowBoundary(updated_ts=0, name="", identifier="")
+
+    @classmethod
+    def highest_boundary(cls) -> RowBoundary:
+        # 9999-12-31T23:59:59.999Z in epoch milliseconds
+        return RowBoundary(updated_ts=253_402_300_799_999, name="", identifier="")
+
+
+class LimitBoundary(PydanticModel):
+    batch_size: int
+
+    @classmethod
+    def init_batch_boundary(cls, batch_size: int) -> LimitBoundary:
+        return LimitBoundary(batch_size=batch_size)
+
+
+class PromotionResult(PydanticModel):
+    added: t.List[SnapshotTableInfo]
+    removed: t.List[SnapshotTableInfo]
+    removed_environment_naming_info: t.Optional[EnvironmentNamingInfo]
+
+    @field_validator("removed_environment_naming_info")
+    def _validate_removed_environment_naming_info(
+        cls, v: t.Optional[EnvironmentNamingInfo], info: ValidationInfo
+    ) -> t.Optional[EnvironmentNamingInfo]:
+        if v and not info.data.get("removed"):
+            raise ValueError("removed_environment_naming_info must be None if removed is empty")
+        return v
+
+
+class ExpiredSnapshotBatch(PydanticModel):
+    """A batch of expired snapshots to be cleaned up."""
+
+    expired_snapshot_ids: t.Set[SnapshotId]
+    cleanup_tasks: t.List[SnapshotTableCleanupTask]
+    batch_range: ExpiredBatchRange
+
+
+def iter_expired_snapshot_batches(
+    state_reader: StateReader,
+    *,
+    current_ts: int,
+    ignore_ttl: bool = False,
+    batch_size: t.Optional[int] = None,
+) -> t.Iterator[ExpiredSnapshotBatch]:
+    """Yields expired snapshot batches.
+
+    Args:
+        state_reader: StateReader instance to query expired snapshots from.
+        current_ts: Timestamp used to evaluate expiration.
+        ignore_ttl: If True, include snapshots regardless of TTL (only checks if unreferenced).
+        batch_size: Maximum number of snapshots to fetch per batch.
+    """
+
+    batch_size = batch_size if batch_size is not None else EXPIRED_SNAPSHOT_DEFAULT_BATCH_SIZE
+    batch_range = ExpiredBatchRange.init_batch_range(batch_size=batch_size)
+
+    while True:
+        batch = state_reader.get_expired_snapshots(
+            current_ts=current_ts,
+            ignore_ttl=ignore_ttl,
+            batch_range=batch_range,
+        )
+
+        if batch is None:
+            return
+
+        yield batch
+
+        assert isinstance(batch.batch_range.end, RowBoundary), (
+            "Only RowBoundary is supported for pagination currently"
+        )
+        batch_range = ExpiredBatchRange(
+            start=batch.batch_range.end,
+            end=LimitBoundary(batch_size=batch_size),
+        )

sqlmesh/core/state_sync/db/environment.py

@@ -285,11 +285,13 @@ class EnvironmentState:
         return []
 
     def _environment_from_row(self, row: t.Tuple[str, ...]) -> Environment:
-        return Environment(**{field: row[i] for i, field in enumerate(Environment.all_fields())})
+        return Environment(
+            **{field: row[i] for i, field in enumerate(sorted(Environment.all_fields()))}
+        )
 
     def _environment_summmary_from_row(self, row: t.Tuple[str, ...]) -> EnvironmentSummary:
         return EnvironmentSummary(
-            **{field: row[i] for i, field in enumerate(EnvironmentSummary.all_fields())}
+            **{field: row[i] for i, field in enumerate(sorted(EnvironmentSummary.all_fields()))}
         )
 
     def _environments_query(
@@ -298,7 +300,7 @@ class EnvironmentState:
         lock_for_update: bool = False,
         required_fields: t.Optional[t.List[str]] = None,
     ) -> exp.Select:
-        query_fields = required_fields if required_fields else Environment.all_fields()
+        query_fields = required_fields if required_fields else sorted(Environment.all_fields())
         query = (
             exp.select(*(exp.to_identifier(field) for field in query_fields))
             .from_(self.environments_table)
@@ -328,7 +330,7 @@ class EnvironmentState:
                 self.engine_adapter,
                 self._environments_query(
                     where=where,
-                    required_fields=list(EnvironmentSummary.all_fields()),
+                    required_fields=sorted(EnvironmentSummary.all_fields()),
                 ),
             )
         ]