dagster-cloud 1.8.2__py3-none-any.whl → 1.12.6__py3-none-any.whl

dagster_cloud/storage/event_logs/utils.py

@@ -1,19 +1,100 @@
-from os import environ
+import os
+from typing import Optional
 
+from dagster import DagsterEvent
+from dagster._core.events import (
+    EngineEventData,
+    HookErroredData,
+    JobCanceledData,
+    JobFailureData,
+    StepFailureData,
+    StepRetryData,
+)
 from dagster._core.events.log import EventLogEntry
+from dagster._utils.error import SerializableErrorInfo, truncate_serialized_error
+
+
+def _get_error_character_size_limit() -> int:
+    return int(os.getenv("DAGSTER_CLOUD_MAXIMUM_EVENT_ERROR_SIZE", "500000"))
+
+
+def _get_maximum_event_message_characters() -> int:
+    return int(os.getenv("DAGSTER_CLOUD_MAXIMUM_EVENT_MESSAGE_CHARACTERS", "50000"))
 
-MAXIMUM_EVENT_MESSAGE_CHARACTERS = int(
-    environ.get("DAGSTER_CLOUD_MAXIMUM_EVENT_MESSAGE_CHARACTERS", 50000)
-)
+
+def _truncate_dagster_event_error(
+    error_info: Optional[SerializableErrorInfo],
+    truncations: list[str],
+) -> Optional[SerializableErrorInfo]:
+    if not error_info:
+        return error_info
+
+    return truncate_serialized_error(
+        error_info,
+        _get_error_character_size_limit(),
+        max_depth=5,
+        truncations=truncations,
+    )
+
+
+def _truncate_dagster_event(
+    dagster_event: Optional[DagsterEvent],
+    truncations: list[str],
+) -> Optional[DagsterEvent]:
+    if not dagster_event:
+        return dagster_event
+    event_specific_data = dagster_event.event_specific_data
+
+    if isinstance(event_specific_data, JobFailureData):
+        event_specific_data = event_specific_data._replace(
+            error=_truncate_dagster_event_error(
+                event_specific_data.error,
+                truncations,
+            ),
+            first_step_failure_event=_truncate_dagster_event(
+                event_specific_data.first_step_failure_event,
+                truncations,
+            ),
+        )
+    elif isinstance(
+        event_specific_data,
+        (JobCanceledData, EngineEventData, HookErroredData, StepFailureData, StepRetryData),
+    ):
+        event_specific_data = event_specific_data._replace(
+            error=_truncate_dagster_event_error(
+                event_specific_data.error,
+                truncations,
+            ),
+        )
+
+    return dagster_event._replace(event_specific_data=event_specific_data)
 
 
 def truncate_event(
-    event: EventLogEntry, maximum_length=MAXIMUM_EVENT_MESSAGE_CHARACTERS
+    event: EventLogEntry,
+    maximum_length=None,
+    truncations: Optional[list[str]] = None,
 ) -> EventLogEntry:
-    if len(event.user_message) > maximum_length:
+    truncations = [] if truncations is None else truncations
+
+    if event.dagster_event:
+        event = event._replace(
+            dagster_event=_truncate_dagster_event(
+                event.dagster_event,
+                truncations,
+            )
+        )
+
+    maximum_length = (
+        maximum_length if maximum_length is not None else _get_maximum_event_message_characters()
+    )
+
+    len_usr_msg = len(event.user_message)
+    if len_usr_msg > maximum_length:
+        truncations.append(f"user_message {len_usr_msg} to {maximum_length}")
         return event._replace(
             user_message=(
-                f"[TRUNCATED from {len(event.user_message)} characters to"
+                f"[TRUNCATED from {len_usr_msg} characters to"
                 f" {maximum_length}]"
                 f" {event.user_message[:maximum_length]} [TRUNCATED]"
             ),

dagster_cloud/storage/runs/__init__.py

@@ -1 +1 @@
-from .storage import GraphQLRunStorage as GraphQLRunStorage
+from dagster_cloud.storage.runs.storage import GraphQLRunStorage as GraphQLRunStorage

dagster_cloud/storage/runs/queries.py

@@ -43,6 +43,21 @@ ADD_RUN_MUTATION = (
 """
 )
 
+ADD_HISTORICAL_RUN_MUTATION = (
+    ERROR_FRAGMENT
+    + """
+    mutation addHistoricalRunMutation($serializedPipelineRun: String!, $runCreationTime: Float!) {
+        runs {
+            addHistoricalRun(serializedPipelineRun: $serializedPipelineRun, runCreationTime: $runCreationTime) {
+                ok
+                error {
+                    ...errorFragment
+                }
+            }
+        }
+    }
+    """
+)
 
 GET_RUNS_QUERY = """
     query getRunsQuery($filters: RunsFilter, $cursor: String, $limit: Int, $bucketBy: RunBucket, $ascending: Boolean) {
@@ -226,9 +241,9 @@ ADD_RUN_TELEMETRY_MUTATION = """
 """
 
 GET_BACKFILLS_QUERY = """
-    query getBackfillsQuery($status: String, $cursor: String, $limit: Int) {
+    query getBackfillsQuery($status: String, $cursor: String, $limit: Int, $filters: BulkActionsFilter) {
         runs {
-            getBackfills(status: $status, cursor: $cursor, limit: $limit)
+            getBackfills(status: $status, cursor: $cursor, limit: $limit, filters: $filters)
         }
     }
 """

dagster_cloud/storage/runs/storage.py

@@ -1,17 +1,7 @@
 import json
-from typing import (
-    Any,
-    Callable,
-    Dict,
-    Iterable,
-    List,
-    Mapping,
-    Optional,
-    Sequence,
-    Set,
-    Tuple,
-    Union,
-)
+from collections.abc import Iterable, Mapping, Sequence
+from datetime import datetime
+from typing import Any, Callable, Optional, Union
 
 import dagster._check as check
 from dagster._core.errors import (
@@ -21,15 +11,10 @@ from dagster._core.errors import (
     DagsterSnapshotDoesNotExist,
 )
 from dagster._core.events import DagsterEvent
-from dagster._core.execution.backfill import BulkActionStatus, PartitionBackfill
+from dagster._core.execution.backfill import BulkActionsFilter, BulkActionStatus, PartitionBackfill
 from dagster._core.execution.telemetry import RunTelemetryData
-from dagster._core.remote_representation.origin import RemoteJobOrigin
-from dagster._core.snap import (
-    ExecutionPlanSnapshot,
-    JobSnapshot,
-    create_execution_plan_snapshot_id,
-    create_job_snapshot_id,
-)
+from dagster._core.remote_origin import RemoteJobOrigin
+from dagster._core.snap import ExecutionPlanSnapshot, JobSnap, create_execution_plan_snapshot_id
 from dagster._core.storage.dagster_run import (
     DagsterRun,
     JobBucket,
@@ -51,10 +36,11 @@ from dagster._utils.merger import merge_dicts
 from dagster_cloud_cli.core.errors import DagsterCloudAgentServerError
 from typing_extensions import Self
 
-from .queries import (
+from dagster_cloud.storage.runs.queries import (
     ADD_BACKFILL_MUTATION,
     ADD_DAEMON_HEARTBEAT_MUTATION,
     ADD_EXECUTION_PLAN_SNAPSHOT_MUTATION,
+    ADD_HISTORICAL_RUN_MUTATION,
     ADD_PIPELINE_SNAPSHOT_MUTATION,
     ADD_RUN_MUTATION,
     ADD_RUN_TAGS_MUTATION,
@@ -81,11 +67,13 @@ from .queries import (
 )
 
 
-def _get_filters_input(filters: Optional[RunsFilter]) -> Optional[Dict[str, Any]]:
+def _get_filters_input(filters: Optional[RunsFilter]) -> Optional[dict[str, Any]]:
     filters = check.opt_inst_param(filters, "filters", RunsFilter)
 
     if filters is None:
         return None
+
+    check.invariant(filters.exclude_subruns is None, "RunsFilter.exclude_subruns is not supported")
     return {
         "runIds": filters.run_ids,
         "pipelineName": filters.job_name,
@@ -109,7 +97,33 @@ def _get_filters_input(filters: Optional[RunsFilter]) -> Optional[Dict[str, Any]
     }
 
 
-def _run_record_from_graphql(graphene_run_record: Dict) -> RunRecord:
+def _get_bulk_actions_filters_input(
+    filters: Optional[BulkActionsFilter],
+) -> Optional[dict[str, Any]]:
+    filters = check.opt_inst_param(filters, "filters", BulkActionsFilter)
+    unsupported_filters = []
+    if filters and filters.job_name:
+        unsupported_filters.append("job_name")
+    if filters and filters.backfill_ids:
+        unsupported_filters.append("backfill_ids")
+    if filters and filters.tags:
+        unsupported_filters.append("tags")
+
+    check.invariant(
+        len(unsupported_filters) == 0,
+        f"Used the following unsupported filters: {', '.join(unsupported_filters)}.",
+    )
+
+    if filters is None:
+        return None
+    return {
+        "statuses": [status.value for status in filters.statuses] if filters.statuses else None,
+        "createdAfter": filters.created_after.timestamp() if filters.created_after else None,
+        "createdBefore": filters.created_before.timestamp() if filters.created_before else None,
+    }
+
+
+def _run_record_from_graphql(graphene_run_record: dict) -> RunRecord:
     check.dict_param(graphene_run_record, "graphene_run_record")
     return RunRecord(
         storage_id=check.int_elem(graphene_run_record, "storageId"),
@@ -128,7 +142,7 @@ def _run_record_from_graphql(graphene_run_record: Dict) -> RunRecord:
     )
 
 
-def _get_bucket_input(bucket_by: Optional[Union[JobBucket, TagBucket]]) -> Optional[Dict[str, Any]]:
+def _get_bucket_input(bucket_by: Optional[Union[JobBucket, TagBucket]]) -> Optional[dict[str, Any]]:
     if not bucket_by:
         return None
 
@@ -170,7 +184,7 @@ class GraphQLRunStorage(RunStorage, ConfigurableClass):
         return (
             self._override_graphql_client
             if self._override_graphql_client
-            else self._instance.graphql_client
+            else self._instance.graphql_client  # pyright: ignore[reportAttributeAccessIssue]
         )
 
     def _execute_query(self, query, variables=None, idempotent_mutation=False):
@@ -178,6 +192,29 @@ class GraphQLRunStorage(RunStorage, ConfigurableClass):
             query, variable_values=variables, idempotent_mutation=idempotent_mutation
         )
 
+    def add_historical_run(
+        self, dagster_run: DagsterRun, run_creation_time: datetime
+    ) -> DagsterRun:
+        check.inst_param(dagster_run, "dagster_run", DagsterRun)
+        res = self._execute_query(
+            ADD_HISTORICAL_RUN_MUTATION,
+            variables={
+                "serializedPipelineRun": serialize_value(dagster_run),
+                "runCreationTime": run_creation_time.timestamp(),
+            },
+        )
+        result = res["data"]["runs"]["addHistoricalRun"]
+        error = result.get("error")
+        # Special-case some errors to match the RunStorage API
+        if error:
+            if error["className"] == "DagsterRunAlreadyExists":
+                raise DagsterRunAlreadyExists(error["message"])
+            if error["className"] == "DagsterSnapshotDoesNotExist":
+                raise DagsterSnapshotDoesNotExist(error["message"])
+            else:
+                raise DagsterCloudAgentServerError(res)
+        return dagster_run
+
     def add_run(self, dagster_run: DagsterRun):
         check.inst_param(dagster_run, "dagster_run", DagsterRun)
         res = self._execute_query(
@@ -199,9 +236,10 @@ class GraphQLRunStorage(RunStorage, ConfigurableClass):
 
         return dagster_run
 
-    def handle_run_event(self, run_id: str, event: DagsterEvent):
-        # no-op, handled by store_event
-        pass
+    def handle_run_event(
+        self, run_id: str, event: DagsterEvent, update_timestamp: Optional[datetime] = None
+    ):
+        raise NotImplementedError("Should never be called by an agent client")
 
     @property
     def supports_bucket_queries(self) -> bool:
@@ -252,7 +290,7 @@ class GraphQLRunStorage(RunStorage, ConfigurableClass):
         )
         return res["data"]["runs"]["getRunsCount"]
 
-    def get_run_group(self, run_id: str) -> Optional[Tuple[str, Iterable[DagsterRun]]]:  # pyright: ignore[reportIncompatibleMethodOverride], fix me!
+    def get_run_group(self, run_id: str) -> Optional[tuple[str, Iterable[DagsterRun]]]:  # pyright: ignore[reportIncompatibleMethodOverride], fix me!
         res = self._execute_query(
             GET_RUN_GROUP_QUERY, variables={"runId": check.str_param(run_id, "run_id")}
         )
@@ -288,7 +326,7 @@ class GraphQLRunStorage(RunStorage, ConfigurableClass):
         ascending: bool = False,
         cursor: Optional[str] = None,
         bucket_by: Optional[Union[JobBucket, TagBucket]] = None,
-    ) -> List[RunRecord]:
+    ) -> list[RunRecord]:
         res = self._execute_query(
             GET_RUN_RECORDS_QUERY,
             variables={
@@ -307,7 +345,7 @@ class GraphQLRunStorage(RunStorage, ConfigurableClass):
         tag_keys: Sequence[str],
         value_prefix: Optional[str] = None,
         limit: Optional[int] = None,
-    ) -> Sequence[Tuple[str, Set[str]]]:
+    ) -> Sequence[tuple[str, set[str]]]:
         res = self._execute_query(
             GET_RUN_TAGS_QUERY,
             variables={
@@ -353,27 +391,27 @@ class GraphQLRunStorage(RunStorage, ConfigurableClass):
         return res["data"]["runs"]["hasPipelineSnapshot"]
 
     def add_job_snapshot(  # pyright: ignore[reportIncompatibleMethodOverride], fix me!
-        self, pipeline_snapshot: JobSnapshot, snapshot_id: Optional[str] = None
+        self, pipeline_snapshot: JobSnap, snapshot_id: Optional[str] = None
     ) -> str:
         self._execute_query(
             ADD_PIPELINE_SNAPSHOT_MUTATION,
             variables={
                 "serializedPipelineSnapshot": serialize_value(
-                    check.inst_param(pipeline_snapshot, "pipeline_snapshot", JobSnapshot)
+                    check.inst_param(pipeline_snapshot, "pipeline_snapshot", JobSnap)
                 ),
                 "snapshotId": snapshot_id,
             },
         )
-        return snapshot_id if snapshot_id else create_job_snapshot_id(pipeline_snapshot)
+        return snapshot_id if snapshot_id else pipeline_snapshot.snapshot_id
 
-    def get_job_snapshot(self, pipeline_snapshot_id: str) -> JobSnapshot:  # pyright: ignore[reportIncompatibleMethodOverride], fix me!
+    def get_job_snapshot(self, pipeline_snapshot_id: str) -> JobSnap:  # pyright: ignore[reportIncompatibleMethodOverride], fix me!
         res = self._execute_query(
             GET_PIPELINE_SNAPSHOT_QUERY,
             variables={
                 "pipelineSnapshotId": check.str_param(pipeline_snapshot_id, "pipeline_snapshot_id")
             },
         )
-        return deserialize_value(res["data"]["runs"]["getPipelineSnapshot"], JobSnapshot)
+        return deserialize_value(res["data"]["runs"]["getPipelineSnapshot"], JobSnap)
 
     def has_execution_plan_snapshot(self, execution_plan_snapshot_id: str) -> bool:
         res = self._execute_query(
@@ -419,7 +457,7 @@ class GraphQLRunStorage(RunStorage, ConfigurableClass):
             res["data"]["runs"]["getExecutionPlanSnapshot"], ExecutionPlanSnapshot
         )
 
-    def get_run_partition_data(self, runs_filter: RunsFilter) -> List[RunPartitionData]:
+    def get_run_partition_data(self, runs_filter: RunsFilter) -> list[RunPartitionData]:
         res = self._execute_query(
             GET_RUN_PARTITION_DATA_QUERY,
             variables={
@@ -441,10 +479,13 @@ class GraphQLRunStorage(RunStorage, ConfigurableClass):
             },
         )
 
+    def supports_run_telemetry(self) -> bool:
+        return True
+
     def add_run_telemetry(
         self,
         run_telemetry: RunTelemetryData,
-        tags: Optional[Dict[str, str]] = None,
+        tags: Optional[dict[str, str]] = None,
     ) -> None:
         if tags is None:
             tags = {}
@@ -466,7 +507,7 @@ class GraphQLRunStorage(RunStorage, ConfigurableClass):
     ):
         raise Exception("Not allowed to build indexes from user cloud")
 
-    def get_daemon_heartbeats(self) -> Dict[str, DaemonHeartbeat]:
+    def get_daemon_heartbeats(self) -> dict[str, DaemonHeartbeat]:
         res = self._execute_query(GET_DAEMON_HEARTBEATS_QUERY)
         return {
             key: deserialize_value(heartbeat, DaemonHeartbeat)
@@ -487,9 +528,10 @@ class GraphQLRunStorage(RunStorage, ConfigurableClass):
 
     def get_backfills(
         self,
-        status: Optional[BulkActionStatus] = None,
+        filters: Optional[BulkActionsFilter] = None,
         cursor: Optional[str] = None,
         limit: Optional[int] = None,
+        status: Optional[BulkActionStatus] = None,
     ):
         """Get a list of partition backfills."""
         res = self._execute_query(
@@ -498,6 +540,7 @@ class GraphQLRunStorage(RunStorage, ConfigurableClass):
                 "status": status.value if status else None,
                 "cursor": check.opt_str_param(cursor, "cursor"),
                 "limit": check.opt_int_param(limit, "limit"),
+                "filters": _get_bulk_actions_filters_input(filters),
             },
         )
         return [
@@ -505,6 +548,9 @@ class GraphQLRunStorage(RunStorage, ConfigurableClass):
             for backfill in res["data"]["runs"]["getBackfills"]
         ]
 
+    def get_backfills_count(self, filters: Optional[BulkActionsFilter] = None) -> int:
+        raise NotImplementedError("get_backfills_count is not callable from user cloud.")
+
     def get_backfill(self, backfill_id: str) -> PartitionBackfill:
         """Get a single partition backfill."""
         res = self._execute_query(GET_BACKFILL_QUERY, variables={"backfillId": backfill_id})
@@ -525,7 +571,7 @@ class GraphQLRunStorage(RunStorage, ConfigurableClass):
             variables={"serializedPartitionBackfill": serialize_value(partition_backfill)},
         )
 
-    def get_cursor_values(self, keys: Set[str]):  # pyright: ignore[reportIncompatibleMethodOverride], fix me!
+    def get_cursor_values(self, keys: set[str]):  # pyright: ignore[reportIncompatibleMethodOverride], fix me!
         return NotImplementedError("KVS is not supported from the user cloud")
 
     def set_cursor_values(self, pairs: Mapping[str, str]):  # pyright: ignore[reportIncompatibleMethodOverride], fix me!

dagster_cloud/storage/schedules/__init__.py

@@ -1 +1 @@
-from .storage import GraphQLScheduleStorage as GraphQLScheduleStorage
+from dagster_cloud.storage.schedules.storage import GraphQLScheduleStorage as GraphQLScheduleStorage

dagster_cloud/storage/schedules/storage.py

@@ -1,7 +1,8 @@
-from typing import TYPE_CHECKING, Any, Iterable, Optional, Sequence, Set
+from collections.abc import Iterable, Sequence
+from typing import TYPE_CHECKING, Any, Optional
 
 import dagster._check as check
-from dagster import AssetKey
+from dagster._core.definitions.asset_key import EntityKey
 from dagster._core.definitions.declarative_automation.serialized_objects import (
     AutomationConditionEvaluationWithRunIds,
 )
@@ -23,7 +24,7 @@ from dagster._serdes import (
 )
 from typing_extensions import Self
 
-from .queries import (
+from dagster_cloud.storage.schedules.queries import (
     ADD_JOB_STATE_MUTATION,
     ALL_STORED_JOB_STATE_QUERY,
     CREATE_JOB_TICK_MUTATION,
@@ -76,7 +77,7 @@ class GraphQLScheduleStorage(ScheduleStorage["DagsterCloudAgentInstance"], Confi
         repository_origin_id: Optional[str] = None,
         repository_selector_id: Optional[str] = None,
         instigator_type: Optional[InstigatorType] = None,
-        instigator_statuses: Optional[Set[InstigatorStatus]] = None,
+        instigator_statuses: Optional[set[InstigatorStatus]] = None,
     ) -> Iterable[InstigatorState]:
         res = self._execute_query(
             ALL_STORED_JOB_STATE_QUERY,
@@ -206,7 +207,7 @@ class GraphQLScheduleStorage(ScheduleStorage["DagsterCloudAgentInstance"], Confi
         raise NotImplementedError("Not callable from user cloud")
 
     def get_auto_materialize_asset_evaluations(
-        self, asset_key: AssetKey, limit: int, cursor: Optional[int] = None
+        self, key: EntityKey, limit: int, cursor: Optional[int] = None
     ) -> Sequence[AutoMaterializeAssetEvaluationRecord]:
         raise NotImplementedError("Not callable from user cloud")
 
@@ -220,6 +221,3 @@ class GraphQLScheduleStorage(ScheduleStorage["DagsterCloudAgentInstance"], Confi
 
     def upgrade(self):
         raise NotImplementedError("Not callable from user cloud")
-
-    def optimize_for_dagit(self, statement_timeout: int, pool_recycle: int):
-        raise NotImplementedError("Not callable from user cloud")

dagster_cloud/storage/tags.py

@@ -1,4 +1,18 @@
-from dagster._core.storage.tags import HIDDEN_TAG_PREFIX, SYSTEM_TAG_PREFIX
+from collections.abc import Mapping
+
+from dagster._core.storage.tags import (
+    AUTO_MATERIALIZE_TAG,
+    AUTO_OBSERVE_TAG,
+    BACKFILL_ID_TAG,
+    HIDDEN_TAG_PREFIX,
+    PARTITION_NAME_TAG,
+    PARTITION_SET_TAG,
+    REPOSITORY_LABEL_TAG,
+    ROOT_RUN_ID_TAG,
+    SCHEDULE_NAME_TAG,
+    SENSOR_NAME_TAG,
+    SYSTEM_TAG_PREFIX,
+)
 
 CLOUD_SYSTEM_TAG_PREFIX = "dagster-cloud/"
 
@@ -6,7 +20,58 @@ PEX_METADATA_TAG = f"{HIDDEN_TAG_PREFIX}pex_metadata"
 PEX_TAG_TAG = f"{SYSTEM_TAG_PREFIX}pex_tag"
 IGNORE_ALERTS_TAG = f"{CLOUD_SYSTEM_TAG_PREFIX}ignore-alerts"
 
+RUN_TAG_RUN_COLUMN_TAG_KEYS = {
+    AUTO_MATERIALIZE_TAG,
+    AUTO_OBSERVE_TAG,
+    REPOSITORY_LABEL_TAG,
+    ROOT_RUN_ID_TAG,
+    PARTITION_NAME_TAG,
+    PARTITION_SET_TAG,
+    SCHEDULE_NAME_TAG,
+    SENSOR_NAME_TAG,
+    BACKFILL_ID_TAG,
+}
+
+DID_ALERT_TAG = f"{CLOUD_SYSTEM_TAG_PREFIX}triggered_alert"
+TRIGGERED_ALERT_POLICY_TAG_PREFIX = f"{HIDDEN_TAG_PREFIX}triggered_alert_policy/"
+TRIGGERED_ALERT_ID_TAG_PREFIX = f"{HIDDEN_TAG_PREFIX}triggered_alert/"
+
+TRIGGERED_NOTIFICATION_TAG_PREFIX = f"{HIDDEN_TAG_PREFIX}triggered_notification/"
+
+
+def get_triggered_alert_policy_key(alert_policy_id: str) -> str:
+    return f"{TRIGGERED_ALERT_POLICY_TAG_PREFIX}{alert_policy_id}"
+
+
+def get_triggered_alert_id_key(alert_id: str) -> str:
+    return f"{TRIGGERED_ALERT_ID_TAG_PREFIX}{alert_id}"
+
+
+def get_triggered_notification_key(notification_id: str) -> str:
+    return f"{TRIGGERED_NOTIFICATION_TAG_PREFIX}{notification_id}"
+
+
+def get_triggered_notification_key_value(notification_id: str) -> Mapping[str, str]:
+    return {f"{TRIGGERED_NOTIFICATION_TAG_PREFIX}{notification_id}": "true"}
+
 
 def get_policy_names_from_tag_value(policies_str):
     """From a comma-delineated string (whitespace allowed), retrieve alert policy names."""
     return [policy_name.strip() for policy_name in policies_str.split(",")]
+
+
+def should_tag_be_used_for_indexing_filtering(tag_key: str) -> bool:
+    """Determines whether a tag should be added to the run_tags table,
+    which is used for indexing and filtering runs.
+    Tags not added to the table will still be stored on the run object, but
+    will not be directly queryable via the run_tags table.
+    """
+    if tag_key in RUN_TAG_RUN_COLUMN_TAG_KEYS:
+        return False
+
+    if tag_key.startswith(TRIGGERED_ALERT_ID_TAG_PREFIX) or tag_key.startswith(
+        TRIGGERED_ALERT_POLICY_TAG_PREFIX
+    ):
+        return False
+
+    return True

dagster_cloud/util/__init__.py

@@ -1,8 +1,9 @@
 import zlib
 from collections import namedtuple
+from collections.abc import Mapping
 from contextlib import contextmanager
 from io import BytesIO
-from typing import Any, Dict, List, Mapping, NamedTuple
+from typing import Any, Callable
 
 from dagster import (
     Field,
@@ -10,7 +11,7 @@ from dagster import (
 )
 from dagster._config import BoolSourceType, IntSourceType, StringSourceType
 from dagster._serdes import serialize_value
-from dagster._serdes.utils import create_snapshot_id
+from dagster_shared.serdes.serdes import PackableValue
 
 
 class SerializableNamedtupleMapDiff(
@@ -25,7 +26,7 @@ class SerializableNamedtupleMapDiff(
         to_update,
         to_remove,
     ):
-        return super(SerializableNamedtupleMapDiff, cls).__new__(
+        return super().__new__(
             cls,
             check.set_param(to_add, "to_add", tuple),
             check.set_param(to_update, "to_update", tuple),
@@ -33,10 +34,9 @@ class SerializableNamedtupleMapDiff(
         )
 
 
-def diff_serializable_namedtuple_map(desired_map, actual_map, force_update_keys=None):
+def diff_serializable_namedtuple_map(desired_map, actual_map, update_key_fn: Callable):
     desired_keys = set(desired_map.keys())
     actual_keys = set(actual_map.keys())
-    force_update_keys = check.opt_set_param(force_update_keys, "force_update_keys", tuple)
 
     to_add = desired_keys.difference(actual_keys)
     to_remove = actual_keys.difference(desired_keys)
@@ -46,17 +46,15 @@ def diff_serializable_namedtuple_map(desired_map, actual_map, force_update_keys=
     to_update = {
         existing_key
         for existing_key in existing
-        if create_snapshot_id(desired_map[existing_key])
-        != create_snapshot_id(actual_map[existing_key])
-        or existing_key in force_update_keys
+        if update_key_fn(desired_map[existing_key]) != update_key_fn(actual_map[existing_key])
     }
 
     return SerializableNamedtupleMapDiff(to_add, to_update, to_remove)
 
 
 def get_env_names_from_config(
-    config_schema: Dict[str, Field], config_dict: Dict[str, Any]
-) -> List[str]:
+    config_schema: dict[str, Field], config_dict: dict[str, Any]
+) -> list[str]:
     env_vars = []
     for field_name, field in config_schema.items():
         config_type = field.config_type
@@ -81,14 +79,14 @@ SERVER_HANDLE_TAG = ".dagster/server_handle"
 
 
 def keys_not_none(
-    keys: List[str],
+    keys: list[str],
     dictionary: Mapping[str, Any],
 ) -> bool:
     return all(key in dictionary and dictionary[key] is not None for key in keys)
 
 
 @contextmanager
-def compressed_namedtuple_upload_file(to_serialize: NamedTuple):
+def compressed_namedtuple_upload_file(to_serialize: PackableValue):
     compressed_data = zlib.compress(serialize_value(to_serialize).encode("utf-8"))
     with BytesIO(compressed_data) as f:
         yield f