dagster 1.12.11__py3-none-any.whl → 1.12.13__py3-none-any.whl

dagster/_core/storage/event_log/sql_event_log.py

@@ -38,6 +38,7 @@ from dagster._core.errors import (
 )
 from dagster._core.event_api import (
     EventRecordsResult,
+    PartitionKeyFilter,
     RunShardedEventsCursor,
     RunStatusChangeRecordsFilter,
 )
@@ -58,6 +59,7 @@ from dagster._core.storage.asset_check_execution_record import (
     COMPLETED_ASSET_CHECK_EXECUTION_RECORD_STATUSES,
     AssetCheckExecutionRecord,
     AssetCheckExecutionRecordStatus,
+    AssetCheckPartitionInfo,
 )
 from dagster._core.storage.dagster_run import DagsterRunStatsSnapshot
 from dagster._core.storage.event_log.base import (
@@ -2992,15 +2994,25 @@ class SqlEventLogStorage(EventLogStorage):
         planned = cast(
             "AssetCheckEvaluationPlanned", check.not_none(event.dagster_event).event_specific_data
         )
+        partition_keys = (
+            planned.partitions_subset.get_partition_keys() if planned.partitions_subset else [None]
+        )
         with self.index_connection() as conn:
             conn.execute(
                 AssetCheckExecutionsTable.insert().values(
-                    asset_key=planned.asset_key.to_string(),
-                    check_name=planned.check_name,
-                    run_id=event.run_id,
-                    execution_status=AssetCheckExecutionRecordStatus.PLANNED.value,
-                    evaluation_event=serialize_value(event),
-                    evaluation_event_timestamp=self._event_insert_timestamp(event),
+                    [
+                        dict(
+                            asset_key=planned.asset_key.to_string(),
+                            check_name=planned.check_name,
+                            partition=partition_key,
+                            run_id=event.run_id,
+                            execution_status=AssetCheckExecutionRecordStatus.PLANNED.value,
+                            evaluation_event=serialize_value(event),
+                            evaluation_event_timestamp=self._event_insert_timestamp(event),
+                            evaluation_event_storage_id=event_id,
+                        )
+                        for partition_key in partition_keys
+                    ]
                 )
             )
 
@@ -3033,6 +3045,7 @@ class SqlEventLogStorage(EventLogStorage):
                         if evaluation.target_materialization_data
                         else None
                     ),
+                    partition=evaluation.partition,
                 )
             )
 
@@ -3049,6 +3062,9 @@ class SqlEventLogStorage(EventLogStorage):
                         AssetCheckExecutionsTable.c.asset_key == evaluation.asset_key.to_string(),
                         AssetCheckExecutionsTable.c.check_name == evaluation.check_name,
                         AssetCheckExecutionsTable.c.run_id == event.run_id,
+                        self._get_asset_check_partition_filter_clause(
+                            PartitionKeyFilter(key=evaluation.partition)
+                        ),
                     )
                 )
                 .values(
@@ -3065,6 +3081,7 @@ class SqlEventLogStorage(EventLogStorage):
                         if evaluation.target_materialization_data
                         else None
                     ),
+                    partition=evaluation.partition,
                 )
             ).rowcount
 
@@ -3089,6 +3106,7 @@ class SqlEventLogStorage(EventLogStorage):
                             if evaluation.target_materialization_data
                             else None
                         ),
+                        partition=evaluation.partition,
                     )
                 ).rowcount
 
@@ -3100,12 +3118,23 @@ class SqlEventLogStorage(EventLogStorage):
                 "as a result of duplicate AssetCheckPlanned events."
             )
 
+    def _get_asset_check_partition_filter_clause(
+        self, partition_filter: Optional[PartitionKeyFilter]
+    ):
+        if partition_filter is None:
+            return True
+        elif partition_filter.key is None:
+            return AssetCheckExecutionsTable.c.partition.is_(None)
+        else:
+            return AssetCheckExecutionsTable.c.partition == partition_filter.key
+
     def get_asset_check_execution_history(
         self,
         check_key: AssetCheckKey,
         limit: int,
         cursor: Optional[int] = None,
         status: Optional[AbstractSet[AssetCheckExecutionRecordStatus]] = None,
+        partition_filter: Optional[PartitionKeyFilter] = None,
     ) -> Sequence[AssetCheckExecutionRecord]:
         check.inst_param(check_key, "key", AssetCheckKey)
         check.int_param(limit, "limit")
@@ -3119,12 +3148,14 @@ class SqlEventLogStorage(EventLogStorage):
                     AssetCheckExecutionsTable.c.execution_status,
                     AssetCheckExecutionsTable.c.evaluation_event,
                     AssetCheckExecutionsTable.c.create_timestamp,
+                    AssetCheckExecutionsTable.c.partition,
                 ]
             )
             .where(
                 db.and_(
                     AssetCheckExecutionsTable.c.asset_key == check_key.asset_key.to_string(),
                     AssetCheckExecutionsTable.c.check_name == check_key.name,
+                    self._get_asset_check_partition_filter_clause(partition_filter),
                 )
             )
             .order_by(AssetCheckExecutionsTable.c.id.desc())
@@ -3144,8 +3175,13 @@ class SqlEventLogStorage(EventLogStorage):
         return [AssetCheckExecutionRecord.from_db_row(row, key=check_key) for row in rows]
 
     def get_latest_asset_check_execution_by_key(
-        self, check_keys: Sequence[AssetCheckKey]
+        self,
+        check_keys: Sequence[AssetCheckKey],
+        partition_filter: Optional[PartitionKeyFilter] = None,
     ) -> Mapping[AssetCheckKey, AssetCheckExecutionRecord]:
+        """Returns the latest AssetCheckExecutionRecord for each check key. By default, returns the latest
+        record regardless of partitioning.
+        """
         if not check_keys:
             return {}
 
@@ -3161,6 +3197,7 @@ class SqlEventLogStorage(EventLogStorage):
                         [key.asset_key.to_string() for key in check_keys]
                     ),
                     AssetCheckExecutionsTable.c.check_name.in_([key.name for key in check_keys]),
+                    self._get_asset_check_partition_filter_clause(partition_filter),
                 )
             )
             .group_by(
@@ -3178,6 +3215,7 @@ class SqlEventLogStorage(EventLogStorage):
                 AssetCheckExecutionsTable.c.execution_status,
                 AssetCheckExecutionsTable.c.evaluation_event,
                 AssetCheckExecutionsTable.c.create_timestamp,
+                AssetCheckExecutionsTable.c.partition,
             ]
         ).select_from(
             AssetCheckExecutionsTable.join(
@@ -3200,6 +3238,135 @@ class SqlEventLogStorage(EventLogStorage):
             results[check_key] = AssetCheckExecutionRecord.from_db_row(row, key=check_key)
         return results
 
+    def _get_asset_check_partition_info_for_key(
+        self,
+        check_key: AssetCheckKey,
+        after_storage_id: Optional[int],
+        partition_keys: Optional[Sequence[str]],
+        latest_unpartitioned_materialization_storage_ids: Mapping[AssetKey, int],
+    ) -> Sequence[AssetCheckPartitionInfo]:
+        # Build the base filter conditions
+        filter_conditions = [
+            AssetCheckExecutionsTable.c.asset_key == check_key.asset_key.to_string(),
+            AssetCheckExecutionsTable.c.check_name == check_key.name,
+            # Historical records may have NULL in the evaluation_event_storage_id column for
+            # PLANNED events
+            AssetCheckExecutionsTable.c.evaluation_event_storage_id.isnot(None),
+        ]
+        if partition_keys is not None:
+            filter_conditions.append(AssetCheckExecutionsTable.c.partition.in_(partition_keys))
+
+        # Subquery to find the max id for each partition
+        latest_check_ids_subquery = db_subquery(
+            db_select(
+                [
+                    db.func.max(AssetCheckExecutionsTable.c.id).label("id"),
+                    AssetCheckExecutionsTable.c.partition.label("partition"),
+                ]
+            )
+            .where(db.and_(*filter_conditions))
+            .group_by(AssetCheckExecutionsTable.c.partition),
+            "latest_check_ids_subquery",
+        )
+
+        # Subquery to find the latest materialization storage id for each partition of the
+        # target asset. Note: we don't filter by after_storage_id here because we always want
+        # to return the latest materialization storage id, even if it's older than after_storage_id.
+        latest_materialization_ids_subquery = self._latest_event_ids_by_partition_subquery(
+            check_key.asset_key,
+            [DagsterEventType.ASSET_MATERIALIZATION],
+            asset_partitions=partition_keys,
+        )
+
+        # Main query to get all columns for the latest records, joined with latest
+        # materialization storage ids
+        query = db_select(
+            [
+                AssetCheckExecutionsTable.c.id,
+                AssetCheckExecutionsTable.c.partition,
+                AssetCheckExecutionsTable.c.execution_status,
+                AssetCheckExecutionsTable.c.evaluation_event_storage_id,
+                AssetCheckExecutionsTable.c.materialization_event_storage_id,
+                AssetCheckExecutionsTable.c.run_id,
+                latest_materialization_ids_subquery.c.id.label("latest_materialization_storage_id"),
+            ]
+        ).select_from(
+            AssetCheckExecutionsTable.join(
+                latest_check_ids_subquery,
+                AssetCheckExecutionsTable.c.id == latest_check_ids_subquery.c.id,
+            ).join(
+                latest_materialization_ids_subquery,
+                AssetCheckExecutionsTable.c.partition
+                == latest_materialization_ids_subquery.c.partition,
+                isouter=True,
+            )
+        )
+
+        # these filters are applied to the main query rather than the individual subqueries to ensure
+        # we don't miss records that only have a new materialization or a new check execution but not both
+        if after_storage_id is not None:
+            query = query.where(
+                db.or_(
+                    AssetCheckExecutionsTable.c.evaluation_event_storage_id > after_storage_id,
+                    latest_materialization_ids_subquery.c.id > after_storage_id,
+                )
+            )
+
+        with self.index_connection() as conn:
+            rows = db_fetch_mappings(conn, query)
+
+        return [
+            AssetCheckPartitionInfo(
+                check_key=check_key,
+                partition_key=row["partition"],
+                latest_execution_status=AssetCheckExecutionRecordStatus(row["execution_status"]),
+                latest_target_materialization_storage_id=row["materialization_event_storage_id"],
+                latest_planned_run_id=row["run_id"],
+                latest_check_event_storage_id=row["evaluation_event_storage_id"],
+                latest_materialization_storage_id=max(
+                    filter(
+                        None,
+                        [
+                            row["latest_materialization_storage_id"],
+                            latest_unpartitioned_materialization_storage_ids.get(
+                                check_key.asset_key
+                            ),
+                        ],
+                    ),
+                    default=None,
+                ),
+            )
+            for row in rows
+        ]
+
+    def get_asset_check_partition_info(
+        self,
+        keys: Sequence[AssetCheckKey],
+        after_storage_id: Optional[int] = None,
+        partition_keys: Optional[Sequence[str]] = None,
+    ) -> Sequence[AssetCheckPartitionInfo]:
+        check.list_param(keys, "keys", of_type=AssetCheckKey)
+        check.opt_int_param(after_storage_id, "after_storage_id")
+
+        infos = []
+        latest_unpartitioned_materialization_storage_ids = (
+            self._get_latest_unpartitioned_materialization_storage_ids(
+                list(set(key.asset_key for key in keys))
+            )
+        )
+        # the inner query is not feasible to join in a single query because the latest materialization ids subquery,
+        # so for now we fetch the info for each key separately
+        for key in keys:
+            infos.extend(
+                self._get_asset_check_partition_info_for_key(
+                    key,
+                    after_storage_id,
+                    partition_keys,
+                    latest_unpartitioned_materialization_storage_ids,
+                )
+            )
+        return infos
+
     @property
     def supports_asset_checks(self):  # pyright: ignore[reportIncompatibleMethodOverride]
         return self.has_table(AssetCheckExecutionsTable.name)

dagster/_core/storage/event_log/sqlite/sqlite_event_log.py

@@ -276,7 +276,12 @@ class SqliteEventLogStorage(SqlEventLogStorage, ConfigurableClass):
             self.store_asset_event_tags([event], [event_id])
 
         if event.is_dagster_event and event.dagster_event_type in ASSET_CHECK_EVENTS:
-            self.store_asset_check_event(event, None)
+            # mirror the event in the cross-run index database
+            with self.index_connection() as conn:
+                result = conn.execute(insert_event_statement)
+                event_id = result.inserted_primary_key[0]
+
+            self.store_asset_check_event(event, event_id)
 
         if event.is_dagster_event and event.dagster_event_type in EVENT_TYPE_TO_PIPELINE_RUN_STATUS:
             # should mirror run status change events in the index shard

dagster/_core/storage/legacy_storage.py

@@ -4,17 +4,17 @@ from typing import TYPE_CHECKING, AbstractSet, Optional, Union  # noqa: UP035
 
 from dagster import _check as check
 from dagster._config.config_schema import UserConfigSchema
-from dagster._core.definitions.asset_checks.asset_check_spec import AssetCheckKey
 from dagster._core.definitions.asset_key import EntityKey
 from dagster._core.definitions.declarative_automation.serialized_objects import (
     AutomationConditionEvaluationWithRunIds,
 )
 from dagster._core.definitions.events import AssetKey
 from dagster._core.definitions.freshness import FreshnessStateRecord
-from dagster._core.event_api import EventHandlerFn
+from dagster._core.event_api import EventHandlerFn, PartitionKeyFilter
 from dagster._core.storage.asset_check_execution_record import (
     AssetCheckExecutionRecord,
     AssetCheckExecutionRecordStatus,
+    AssetCheckPartitionInfo,
 )
 from dagster._core.storage.base_storage import DagsterStorage
 from dagster._core.storage.event_log.base import (
@@ -60,6 +60,7 @@ if TYPE_CHECKING:
     )
     from dagster._core.snap.execution_plan_snapshot import ExecutionPlanSnapshot
     from dagster._core.snap.job_snapshot import JobSnap
+    from dagster._core.storage.asset_check_state import AssetCheckState
     from dagster._core.storage.dagster_run import (
         DagsterRun,
         DagsterRunStatsSnapshot,
@@ -745,19 +746,39 @@ class LegacyEventLogStorage(EventLogStorage, ConfigurableClass):
         limit: int,
         cursor: Optional[int] = None,
         status: Optional[AbstractSet[AssetCheckExecutionRecordStatus]] = None,
+        partition_filter: Optional[PartitionKeyFilter] = None,
     ) -> Sequence[AssetCheckExecutionRecord]:
         return self._storage.event_log_storage.get_asset_check_execution_history(
             check_key=check_key,
             limit=limit,
             cursor=cursor,
             status=status,
+            partition_filter=partition_filter,
         )
 
-    def get_latest_asset_check_execution_by_key(  # pyright: ignore[reportIncompatibleMethodOverride]
+    def get_latest_asset_check_execution_by_key(
         self,
         check_keys: Sequence["AssetCheckKey"],
-    ) -> Mapping["AssetCheckKey", Optional[AssetCheckExecutionRecord]]:
-        return self._storage.event_log_storage.get_latest_asset_check_execution_by_key(check_keys)
+        partition_filter: Optional[PartitionKeyFilter] = None,
+    ) -> Mapping["AssetCheckKey", AssetCheckExecutionRecord]:
+        return self._storage.event_log_storage.get_latest_asset_check_execution_by_key(
+            check_keys, partition_filter=partition_filter
+        )
+
+    def get_asset_check_partition_info(
+        self,
+        keys: Sequence["AssetCheckKey"],
+        after_storage_id: Optional[int] = None,
+        partition_keys: Optional[Sequence[str]] = None,
+    ) -> Sequence[AssetCheckPartitionInfo]:
+        return self._storage.event_log_storage.get_asset_check_partition_info(
+            keys=keys, after_storage_id=after_storage_id, partition_keys=partition_keys
+        )
+
+    def get_checkpointed_asset_check_state(
+        self, keys: Sequence["AssetCheckKey"]
+    ) -> Mapping["AssetCheckKey", "AssetCheckState"]:
+        return self._storage.event_log_storage.get_checkpointed_asset_check_state(keys)
 
 
 class LegacyScheduleStorage(ScheduleStorage, ConfigurableClass):

dagster/_core/telemetry.py

@@ -25,6 +25,7 @@ from dagster_shared.telemetry import (
     TelemetrySettings,
     dagster_home_if_set,
     get_or_set_instance_id,
+    get_or_set_user_id,
     log_telemetry_action,
     write_telemetry_log_line,
 )
@@ -379,6 +380,7 @@ def log_remote_repo_stats(
                 client_time=str(datetime.datetime.now()),
                 event_id=str(uuid.uuid4()),
                 instance_id=instance_id,
+                user_id=get_or_set_user_id(),
                 metadata={
                     **get_stats_from_remote_repo(remote_repo),
                     "source": source,
@@ -450,6 +452,7 @@ def log_repo_stats(
                 client_time=str(datetime.datetime.now()),
                 event_id=str(uuid.uuid4()),
                 instance_id=instance_id,
+                user_id=get_or_set_user_id(),
                 metadata={
                     "source": source,
                     "pipeline_name_hash": job_name_hash,

dagster/_core/workspace/load_target.py

@@ -67,7 +67,7 @@ def validate_dagster_block_for_module_name_or_modules(dagster_block):
     modules_present = "modules" in dagster_block and isinstance(dagster_block.get("modules"), list)
 
     if module_name_present and modules_present:
-        # Here we have the check only for list; to be a bit more forgiving in comparison to 'is_valid_modules_list' in case it's an empty list next to 'module_name' existance
+        # Here we have the check only for list; to be a bit more forgiving in comparison to 'is_valid_modules_list' in case it's an empty list next to 'module_name' existence
         if len(dagster_block["modules"]) > 0:
             raise ValueError(
                 "Only one of 'module_name' or 'modules' should be specified, not both."

dagster/_daemon/monitoring/run_monitoring.py

@@ -235,7 +235,11 @@ def check_run_timeout(
         MAX_RUNTIME_SECONDS_TAG, run_record.dagster_run.tags.get("dagster/max_runtime_seconds")
     )
     if max_time_str:
-        max_time = float(max_time_str)
+        try:
+            max_time = float(max_time_str)
+        except ValueError:
+            logger.warning(f"Invalid max runtime value: {max_time_str}")
+            max_time = None
     else:
         max_time = default_timeout_seconds
 

dagster/_generate/download.py

@@ -43,6 +43,7 @@ AVAILABLE_EXAMPLES = [
     "project_analytics",
     "project_fully_featured",
     "quickstart_etl",
+    "snowflake_cortex",
     "tutorial_notebook_assets",
     "with_great_expectations",
     "with_openai",

dagster/_utils/__init__.py

@@ -465,6 +465,17 @@ def git_repository_root() -> str:
     return subprocess.check_output(["git", "rev-parse", "--show-toplevel"]).decode("utf-8").strip()
 
 
+_DAGSTER_OSS_SUBDIRECTORY = "dagster-oss"
+
+
+def discover_oss_root(path: Path) -> Path:
+    while path != path.parent:
+        if (path / ".git").exists() or path.name == _DAGSTER_OSS_SUBDIRECTORY:
+            return path
+        path = path.parent
+    raise ValueError("Could not find OSS root")
+
+
 def segfault() -> None:
     """Reliable cross-Python version segfault.
 

dagster/components/list/list.py

@@ -28,6 +28,7 @@ from dagster._cli.workspace.cli_target import get_repository_python_origin_from_
 from dagster._config.pythonic_config.resource import get_resource_type_name
 from dagster._core.definitions.asset_selection import AssetSelection
 from dagster._core.definitions.assets.job.asset_job import is_reserved_asset_job_name
+from dagster._core.definitions.declarative_automation.serialized_objects import get_expanded_label
 from dagster._core.definitions.definitions_load_context import (
     DefinitionsLoadContext,
     DefinitionsLoadType,
@@ -177,7 +178,9 @@ def list_definitions(
                     group=node.group_name,
                     kinds=sorted(list(node.kinds)),
                     description=node.description,
-                    automation_condition=node.automation_condition.get_label()
+                    automation_condition=" ".join(
+                        get_expanded_label(node.automation_condition.get_snapshot())
+                    )
                     if node.automation_condition
                     else None,
                     tags=sorted(f'"{k}"="{v}"' for k, v in node.tags.items() if _tag_filter(k)),

dagster/version.py

@@ -1 +1 @@
-__version__ = "1.12.11"
+__version__ = "1.12.13"

{dagster-1.12.11.dist-info → dagster-1.12.13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dagster
-Version: 1.12.11
+Version: 1.12.13
 Summary: Dagster is an orchestration platform for the development, production, and observation of data assets.
 Author: Dagster Labs
 Author-email: hello@dagsterlabs.com
@@ -60,8 +60,8 @@ Requires-Dist: universal_pathlib; python_version < "3.12"
 Requires-Dist: universal_pathlib>=0.2.0; python_version >= "3.12"
 Requires-Dist: rich
 Requires-Dist: filelock
-Requires-Dist: dagster-pipes==1.12.11
-Requires-Dist: dagster-shared==1.12.11
+Requires-Dist: dagster-pipes==1.12.13
+Requires-Dist: dagster-shared==1.12.13
 Requires-Dist: antlr4-python3-runtime
 Provides-Extra: docker
 Requires-Dist: docker; extra == "docker"
@@ -88,7 +88,7 @@ Requires-Dist: ruff==0.11.5; extra == "test"
 Provides-Extra: test-components
 Requires-Dist: tomlkit; extra == "test-components"
 Requires-Dist: jsonschema; extra == "test-components"
-Requires-Dist: pandas; extra == "test-components"
+Requires-Dist: pandas<3.0.0; extra == "test-components"
 Requires-Dist: duckdb; extra == "test-components"
 Provides-Extra: mypy
 Requires-Dist: mypy==1.8.0; extra == "mypy"