dagster 1.12.10__py3-none-any.whl → 1.12.12__py3-none-any.whl

dagster/_cli/asset.py

@@ -43,7 +43,11 @@ def asset_cli():
 @click.option("--partition", help="Asset partition to target", required=False)
 @click.option(
     "--partition-range",
-    help="Asset partition range to target i.e. <start>...<end>",
+    help=(
+        "Asset partition range to materialize in the format <start>...<end>. "
+        "Requires all assets to have a BackfillPolicy.single_run() policy, which allows "
+        "the partition range to be executed in a single run. For example: 2025-01-01...2025-01-05"
+    ),
     required=False,
 )
 @click.option(
@@ -163,12 +167,18 @@ def execute_materialize_command(
         for asset_key in asset_keys:
             backfill_policy = implicit_job_def.asset_layer.get(asset_key).backfill_policy
             if (
-                backfill_policy is not None
-                and backfill_policy.policy_type != BackfillPolicyType.SINGLE_RUN
+                backfill_policy is None
+                or backfill_policy.policy_type != BackfillPolicyType.SINGLE_RUN
             ):
                 check.failed(
-                    "Provided partition range, but not all assets have a single-run backfill policy."
+                    "Partition ranges with the CLI require all selected assets to have a "
+                    "BackfillPolicy.single_run() policy. This allows the partition range to be "
+                    "executed in a single run. Assets without this policy would require creating "
+                    "a backfill with separate runs per partition, which needs a running daemon "
+                    "process. Consider using the Dagster UI or a running daemon to execute "
+                    "partition ranges for assets without a single-run backfill policy."
                 )
+
         try:
             implicit_job_def.validate_partition_key(
                 partition_range_start, selected_asset_keys=asset_keys, context=context
@@ -181,6 +191,7 @@ def execute_materialize_command(
                 "All selected assets must have a PartitionsDefinition containing the passed"
                 f" partition key `{partition_range_start}` or have no PartitionsDefinition."
             )
+
         tags = {
             ASSET_PARTITION_RANGE_START_TAG: partition_range_start,
             ASSET_PARTITION_RANGE_END_TAG: partition_range_end,

dagster/_cli/job.py

@@ -405,11 +405,16 @@ def execute_execute_command(
             for asset_key in job_def.asset_layer.executable_asset_keys:
                 backfill_policy = job_def.asset_layer.get(asset_key).backfill_policy
                 if (
-                    backfill_policy is not None
-                    and backfill_policy.policy_type != BackfillPolicyType.SINGLE_RUN
+                    backfill_policy is None
+                    or backfill_policy.policy_type != BackfillPolicyType.SINGLE_RUN
                 ):
                     check.failed(
-                        "Provided partition range, but not all assets have a single-run backfill policy."
+                        "Partition ranges with the CLI require all selected assets to have a "
+                        "BackfillPolicy.single_run() policy. This allows the partition range to be "
+                        "executed in a single run. Assets without this policy would require creating "
+                        "a backfill with separate runs per partition, which needs a running daemon "
+                        "process. Consider using the Dagster UI or a running daemon to execute "
+                        "partition ranges for assets without a single-run backfill policy."
                     )
             try:
                 job_def.validate_partition_key(

dagster/_core/asset_graph_view/serializable_entity_subset.py

@@ -132,6 +132,10 @@ class SerializableEntitySubset(Generic[T_EntityKey]):
     def is_compatible_with_partitions_def(
         self, partitions_def: Optional[PartitionsDefinition]
     ) -> bool:
+        from dagster._core.definitions.partitions.definition.time_window import (
+            TimeWindowPartitionsDefinition,
+        )
+
         if self.is_partitioned:
             # for some PartitionSubset types, we have access to the underlying partitions
             # definitions, so we can ensure those are identical
@@ -150,6 +154,11 @@ class SerializableEntitySubset(Generic[T_EntityKey]):
                     and partitions_def.has_partition_key(r.end)
                     for r in self.value.key_ranges
                 )
+            elif isinstance(self.value, DefaultPartitionsSubset) and isinstance(
+                partitions_def, TimeWindowPartitionsDefinition
+            ):
+                return all(partitions_def.has_partition_key(k) for k in self.value.subset)
+
             else:
                 return partitions_def is not None
         else:

dagster/_core/code_pointer.py

@@ -2,6 +2,7 @@ import importlib
 import inspect
 import os
 import sys
+import uuid
 from abc import ABC, abstractmethod
 from collections.abc import Callable, Sequence
 from pathlib import Path
@@ -59,7 +60,11 @@ def rebase_file(relative_path_in_file: str, file_path_resides_in: str) -> str:
     )
 
 
-def load_python_file(python_file: Union[str, Path], working_directory: Optional[str]) -> ModuleType:
+def load_python_file(
+    python_file: Union[str, Path],
+    working_directory: Optional[str],
+    add_uuid_suffix: bool = False,
+) -> ModuleType:
     """Takes a path to a python file and returns a loaded module."""
     check.inst_param(python_file, "python_file", (str, Path))
     check.opt_str_param(working_directory, "working_directory")
@@ -68,6 +73,8 @@ def load_python_file(python_file: Union[str, Path], working_directory: Optional[
     os.stat(python_file)
 
     module_name = os.path.splitext(os.path.basename(python_file))[0]
+    if add_uuid_suffix:
+        module_name = f"{module_name}_{uuid.uuid4().hex}"
 
     # Use the passed in working directory for local imports (sys.path[0] isn't
     # consistently set in the different entry points that Dagster uses to import code)

dagster/_core/definitions/assets/graph/remote_asset_graph.py

@@ -520,7 +520,7 @@ class RemoteAssetGraph(BaseAssetGraph[TRemoteAssetNode], ABC, Generic[TRemoteAss
 
         by_table_name = defaultdict(set)
         for node in self.asset_nodes:
-            normalized_table_name = TableMetadataSet.extract(node.metadata).normalized_table_name
+            normalized_table_name = TableMetadataSet.extract_normalized_table_name(node.metadata)
             if normalized_table_name:
                 by_table_name[normalized_table_name.lower()].add(node.key)
 
@@ -547,7 +547,7 @@ class RemoteAssetGraph(BaseAssetGraph[TRemoteAssetNode], ABC, Generic[TRemoteAss
         from dagster._core.definitions.metadata.metadata_set import TableMetadataSet
 
         input_node = self.get(asset_key)
-        input_table_name = TableMetadataSet.extract(input_node.metadata).normalized_table_name
+        input_table_name = TableMetadataSet.extract_normalized_table_name(input_node.metadata)
 
         if not input_table_name:
             return set()

dagster/_core/definitions/declarative_automation/serialized_objects.py

@@ -1,3 +1,4 @@
+import itertools
 from collections.abc import Iterator, Mapping, Sequence
 from dataclasses import dataclass
 from typing import (  # noqa: UP035
@@ -312,3 +313,38 @@ class AutomationConditionEvaluationState:
     @property
     def true_subset(self) -> SerializableEntitySubset:
         return self.previous_evaluation.true_subset
+
+
+def get_expanded_label(
+    item: Union[AutomationConditionEvaluation, AutomationConditionSnapshot],
+    use_label=False,
+) -> Sequence[str]:
+    if isinstance(item, AutomationConditionSnapshot):
+        label, name, description, children = (
+            item.node_snapshot.label,
+            item.node_snapshot.name,
+            item.node_snapshot.description,
+            item.children,
+        )
+    else:
+        snapshot = item.condition_snapshot
+        label, name, description, children = (
+            snapshot.label,
+            snapshot.name,
+            snapshot.description,
+            item.child_evaluations,
+        )
+
+    if use_label and label is not None:
+        return [label]
+    node_text = name or description
+    child_labels = [f"({' '.join(get_expanded_label(c, use_label=True))})" for c in children]
+    if len(child_labels) == 0:
+        return [node_text]
+    elif len(child_labels) == 1:
+        return [node_text, f"{child_labels[0]}"]
+    else:
+        # intersperses node_text (e.g. AND) between each child label
+        return list(itertools.chain(*itertools.zip_longest(child_labels, [], fillvalue=node_text)))[
+            :-1
+        ]

dagster/_core/definitions/decorators/asset_decorator.py

@@ -82,11 +82,11 @@ def asset(
     io_manager_def: Optional[object] = ...,
     io_manager_key: Optional[str] = ...,
     dagster_type: Optional[DagsterType] = ...,
-    partitions_def: Optional[PartitionsDefinition] = ...,
+    partitions_def: Optional[PartitionsDefinition[str]] = ...,
     op_tags: Optional[Mapping[str, Any]] = ...,
     group_name: Optional[str] = ...,
     output_required: bool = ...,
-    automation_condition: Optional[AutomationCondition] = ...,
+    automation_condition: Optional[AutomationCondition[AssetKey]] = ...,
     backfill_policy: Optional[BackfillPolicy] = ...,
     retry_policy: Optional[RetryPolicy] = ...,
     code_version: Optional[str] = ...,
@@ -95,14 +95,14 @@ def asset(
     owners: Optional[Sequence[str]] = ...,
     kinds: Optional[AbstractSet[str]] = ...,
     pool: Optional[str] = ...,
-    **kwargs,
+    **kwargs: Any,
 ) -> Callable[[Callable[..., Any]], AssetsDefinition]: ...
 
 
 @overload
 def asset(
     compute_fn: Callable[..., Any],
-    **kwargs,
+    **kwargs: Any,
 ) -> AssetsDefinition: ...
 
 
@@ -168,11 +168,11 @@ def asset(
     io_manager_def: Optional[object] = None,
     io_manager_key: Optional[str] = None,
     dagster_type: Optional[DagsterType] = None,
-    partitions_def: Optional[PartitionsDefinition] = None,
+    partitions_def: Optional[PartitionsDefinition[str]] = None,
     op_tags: Optional[Mapping[str, Any]] = None,
     group_name: Optional[str] = None,
     output_required: bool = True,
-    automation_condition: Optional[AutomationCondition] = None,
+    automation_condition: Optional[AutomationCondition[AssetKey]] = None,
     freshness_policy: Optional[FreshnessPolicy] = None,
     backfill_policy: Optional[BackfillPolicy] = None,
     retry_policy: Optional[RetryPolicy] = None,
@@ -182,7 +182,7 @@ def asset(
     owners: Optional[Sequence[str]] = None,
     kinds: Optional[AbstractSet[str]] = None,
     pool: Optional[str] = None,
-    **kwargs,
+    **kwargs: Any,
 ) -> Union[AssetsDefinition, Callable[[Callable[..., Any]], AssetsDefinition]]:
     """Create a definition for how to compute an asset.
 
@@ -590,7 +590,7 @@ def multi_asset(
     config_schema: Optional[UserConfigSchema] = None,
     required_resource_keys: Optional[AbstractSet[str]] = None,
     internal_asset_deps: Optional[Mapping[str, set[AssetKey]]] = None,
-    partitions_def: Optional[PartitionsDefinition] = None,
+    partitions_def: Optional[PartitionsDefinition[str]] = None,
     hooks: Optional[AbstractSet[HookDefinition]] = None,
     backfill_policy: Optional[BackfillPolicy] = None,
     op_tags: Optional[Mapping[str, Any]] = None,
@@ -769,7 +769,7 @@ def graph_asset(
     config: Optional[Union[ConfigMapping, Mapping[str, Any]]] = None,
     key_prefix: Optional[CoercibleToAssetKeyPrefix] = None,
     group_name: Optional[str] = None,
-    partitions_def: Optional[PartitionsDefinition] = None,
+    partitions_def: Optional[PartitionsDefinition[str]] = None,
     hooks: Optional[AbstractSet[HookDefinition]] = None,
     metadata: Optional[RawMetadataMapping] = ...,
     tags: Optional[Mapping[str, str]] = ...,
@@ -777,7 +777,7 @@ def graph_asset(
     kinds: Optional[AbstractSet[str]] = None,
     legacy_freshness_policy: Optional[LegacyFreshnessPolicy] = ...,
     auto_materialize_policy: Optional[AutoMaterializePolicy] = ...,
-    automation_condition: Optional[AutomationCondition] = ...,
+    automation_condition: Optional[AutomationCondition[AssetKey]] = ...,
     backfill_policy: Optional[BackfillPolicy] = ...,
     resource_defs: Optional[Mapping[str, ResourceDefinition]] = ...,
     check_specs: Optional[Sequence[AssetCheckSpec]] = None,
@@ -806,19 +806,19 @@ def graph_asset(
     config: Optional[Union[ConfigMapping, Mapping[str, Any]]] = None,
     key_prefix: Optional[CoercibleToAssetKeyPrefix] = None,
     group_name: Optional[str] = None,
-    partitions_def: Optional[PartitionsDefinition] = None,
+    partitions_def: Optional[PartitionsDefinition[str]] = None,
     hooks: Optional[AbstractSet[HookDefinition]] = None,
     metadata: Optional[RawMetadataMapping] = None,
     tags: Optional[Mapping[str, str]] = None,
     owners: Optional[Sequence[str]] = None,
-    automation_condition: Optional[AutomationCondition] = None,
+    automation_condition: Optional[AutomationCondition[AssetKey]] = None,
     backfill_policy: Optional[BackfillPolicy] = None,
     resource_defs: Optional[Mapping[str, ResourceDefinition]] = None,
     check_specs: Optional[Sequence[AssetCheckSpec]] = None,
     code_version: Optional[str] = None,
     key: Optional[CoercibleToAssetKey] = None,
     kinds: Optional[AbstractSet[str]] = None,
-    **kwargs,
+    **kwargs: Any,
 ) -> Union[AssetsDefinition, Callable[[Callable[..., Any]], AssetsDefinition]]:
     """Creates a software-defined asset that's computed using a graph of ops.
 

dagster/_core/definitions/inference.py

@@ -1,14 +1,16 @@
 from collections.abc import Callable, Mapping, Sequence
 from inspect import Parameter, Signature, isgeneratorfunction, signature
-from typing import Any, NamedTuple, Optional
+from typing import Any, Optional
 
+from dagster_shared.record import record
 from docstring_parser import parse
 
 from dagster._core.decorator_utils import get_type_hints
 from dagster._core.definitions.utils import NoValueSentinel
 
 
-class InferredInputProps(NamedTuple):
+@record
+class InferredInputProps:
     """The information about an input that can be inferred from the function signature."""
 
     name: str
@@ -17,7 +19,8 @@ class InferredInputProps(NamedTuple):
     default_value: Any = NoValueSentinel
 
 
-class InferredOutputProps(NamedTuple):
+@record
+class InferredOutputProps:
     """The information about an input that can be inferred from the function signature."""
 
     annotation: Any
@@ -80,15 +83,15 @@ def _infer_inputs_from_params(
     for param in params:
         if param.default is not Parameter.empty:
             input_def = InferredInputProps(
-                param.name,
-                type_hints.get(param.name, param.annotation),
+                name=param.name,
+                annotation=type_hints.get(param.name, param.annotation),
                 default_value=param.default,
                 description=_descriptions.get(param.name),
             )
         else:
             input_def = InferredInputProps(
-                param.name,
-                type_hints.get(param.name, param.annotation),
+                name=param.name,
+                annotation=type_hints.get(param.name, param.annotation),
                 description=_descriptions.get(param.name),
             )
 

dagster/_core/definitions/input.py

@@ -1,12 +1,13 @@
 import inspect
 from collections.abc import Callable, Mapping
-from types import FunctionType, UnionType
-from typing import TYPE_CHECKING, Any, NamedTuple, Optional, TypeVar, Union
+from types import UnionType
+from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union
 
 from dagster_shared.error import DagsterError
+from dagster_shared.record import IHaveNew, record, record_custom
 
 import dagster._check as check
-from dagster._annotations import PublicAttr, deprecated_param, public, superseded
+from dagster._annotations import PublicAttr, public, superseded
 from dagster._core.definitions.events import AssetKey
 from dagster._core.definitions.inference import InferredInputProps
 from dagster._core.definitions.metadata import (
@@ -309,39 +310,22 @@ def _checked_inferred_type(inferred: InferredInputProps) -> DagsterType:
     return resolved_type
 
 
-class InputPointer(NamedTuple("_InputPointer", [("node_name", str), ("input_name", str)])):
-    def __new__(cls, node_name: str, input_name: str):
-        return super().__new__(
-            cls,
-            check.str_param(node_name, "node_name"),
-            check.str_param(input_name, "input_name"),
-        )
+@record(kw_only=False)
+class InputPointer:
+    node_name: str
+    input_name: str
 
 
-class FanInInputPointer(
-    NamedTuple(
-        "_FanInInputPointer", [("node_name", str), ("input_name", str), ("fan_in_index", int)]
-    )
-):
-    def __new__(cls, node_name: str, input_name: str, fan_in_index: int):
-        return super().__new__(
-            cls,
-            check.str_param(node_name, "node_name"),
-            check.str_param(input_name, "input_name"),
-            check.int_param(fan_in_index, "fan_in_index"),
-        )
+@record(kw_only=False)
+class FanInInputPointer:
+    node_name: str
+    input_name: str
+    fan_in_index: int
 
 
-@deprecated_param(
-    param="dagster_type",
-    breaking_version="2.0",
-    additional_warn_text="Any defined `dagster_type` should come from the upstream op `Output`.",
-    # Disabling warning here since we're passing this internally and I'm not sure whether it is
-    # actually used or discarded.
-    emit_runtime_warning=False,
-)
 @public
-class InputMapping(NamedTuple):
+@record
+class InputMapping:
     """Defines an input mapping for a graph.
 
     Args:
@@ -350,8 +334,8 @@ class InputMapping(NamedTuple):
         mapped_node_input_name (str): Name of the input in the node (op/graph) that is being mapped to.
         fan_in_index (Optional[int]): The index in to a fanned input, otherwise None.
         graph_input_description (Optional[str]): A description of the input in the graph being mapped from.
-        dagster_type (Optional[DagsterType]): The dagster type of the graph's input
-            being mapped from.
+        dagster_type (Optional[DagsterType]): DEPRECATED, to be removed in 2.0.
+            The dagster type of the graph's input being mapped from.
 
     Examples:
         .. code-block:: python
@@ -384,6 +368,8 @@ class InputMapping(NamedTuple):
     mapped_node_input_name: str
     fan_in_index: Optional[int] = None
     graph_input_description: Optional[str] = None
+
+    # DEPRECATED: Any defined `dagster_type` should come from the upstream op `Output`.
     dagster_type: Optional[DagsterType] = None
 
     @property
@@ -411,26 +397,8 @@ class InputMapping(NamedTuple):
 
 
 @public
-class In(
-    NamedTuple(
-        "_In",
-        [
-            ("dagster_type", PublicAttr[Union[DagsterType, type[NoValueSentinel]]]),
-            ("description", PublicAttr[Optional[str]]),
-            ("default_value", PublicAttr[Any]),
-            ("metadata", PublicAttr[Optional[Mapping[str, Any]]]),
-            (
-                "asset_key",
-                PublicAttr[Optional[Union[AssetKey, Callable[["InputContext"], AssetKey]]]],
-            ),
-            (
-                "asset_partitions",
-                PublicAttr[Optional[Union[set[str], Callable[["InputContext"], set[str]]]]],
-            ),
-            ("input_manager_key", PublicAttr[Optional[str]]),
-        ],
-    )
-):
+@record_custom
+class In(IHaveNew):
     """Defines an argument to an op's compute function.
 
     Inputs may flow from previous op's outputs, or be stubbed using config. They may optionally
@@ -454,6 +422,14 @@ class In(
             upstream output.
     """
 
+    dagster_type: PublicAttr[Union[DagsterType, type[NoValueSentinel]]]
+    description: PublicAttr[Optional[str]]
+    default_value: PublicAttr[Any]
+    metadata: PublicAttr[Mapping[str, Any]]
+    asset_key: PublicAttr[Optional[Union[AssetKey, Callable[["InputContext"], AssetKey]]]]
+    asset_partitions: PublicAttr[Optional[Union[set[str], Callable[["InputContext"], set[str]]]]]
+    input_manager_key: PublicAttr[Optional[str]]
+
     def __new__(
         cls,
         dagster_type: Union[type, UnionType, DagsterType] = NoValueSentinel,
@@ -471,12 +447,12 @@ class In(
                 if dagster_type is NoValueSentinel
                 else resolve_dagster_type(dagster_type)
             ),
-            description=check.opt_str_param(description, "description"),
+            description=description,
             default_value=default_value,
-            metadata=check.opt_mapping_param(metadata, "metadata", key_type=str),
-            asset_key=check.opt_inst_param(asset_key, "asset_key", (AssetKey, FunctionType)),
+            metadata={} if metadata is None else metadata,
+            asset_key=asset_key,
             asset_partitions=asset_partitions,
-            input_manager_key=check.opt_str_param(input_manager_key, "input_manager_key"),
+            input_manager_key=input_manager_key,
         )
 
     @staticmethod
@@ -506,15 +482,15 @@ class In(
 
 
 @public
-class GraphIn(NamedTuple("_GraphIn", [("description", PublicAttr[Optional[str]])])):
+@record
+class GraphIn:
     """Represents information about an input that a graph maps.
 
     Args:
         description (Optional[str]): Human-readable description of the input.
     """
 
-    def __new__(cls, description: Optional[str] = None):
-        return super().__new__(cls, description=description)
+    description: PublicAttr[Optional[str]] = None
 
     def to_definition(self, name: str) -> InputDefinition:
         return InputDefinition(name=name, description=self.description)

dagster/_core/definitions/metadata/metadata_set.py

@@ -197,9 +197,20 @@ class TableMetadataSet(NamespacedMetadataSet):
     def current_key_by_legacy_key(cls) -> Mapping[str, str]:
         return {"relation_identifier": "table_name"}
 
-    @property
-    def normalized_table_name(self) -> Optional[str]:
-        return self.table_name.lower() if self.table_name else None
+    @classmethod
+    def extract_normalized_table_name(cls, metadata: Mapping[str, Any]) -> Optional[str]:
+        from pydantic import ValidationError
+
+        metadata_subset = {
+            key: metadata[key]
+            for key in {"dagster/table_name", "dagster/relation_identifier"}
+            if key in metadata
+        }
+        try:
+            table_name = TableMetadataSet.extract(metadata_subset).table_name
+            return table_name.lower() if table_name else None
+        except ValidationError:
+            return None
 
 
 class UriMetadataSet(NamespacedMetadataSet):

dagster/_core/execution/run_cancellation_thread.py

@@ -21,6 +21,7 @@ def _kill_on_cancel(instance_ref: InstanceRef, run_id, shutdown_event):
             if run.status in [
                 DagsterRunStatus.CANCELING,
                 DagsterRunStatus.CANCELED,
+                DagsterRunStatus.FAILURE,
             ]:
                 print(  # noqa: T201
                     f"Detected run status {run.status}, sending interrupt to main thread"

dagster/_core/telemetry.py

@@ -25,6 +25,7 @@ from dagster_shared.telemetry import (
     TelemetrySettings,
     dagster_home_if_set,
     get_or_set_instance_id,
+    get_or_set_user_id,
     log_telemetry_action,
     write_telemetry_log_line,
 )
@@ -379,6 +380,7 @@ def log_remote_repo_stats(
                 client_time=str(datetime.datetime.now()),
                 event_id=str(uuid.uuid4()),
                 instance_id=instance_id,
+                user_id=get_or_set_user_id(),
                 metadata={
                     **get_stats_from_remote_repo(remote_repo),
                     "source": source,
@@ -450,6 +452,7 @@ def log_repo_stats(
                 client_time=str(datetime.datetime.now()),
                 event_id=str(uuid.uuid4()),
                 instance_id=instance_id,
+                user_id=get_or_set_user_id(),
                 metadata={
                     "source": source,
                     "pipeline_name_hash": job_name_hash,

dagster/_generate/download.py

@@ -43,6 +43,7 @@ AVAILABLE_EXAMPLES = [
     "project_analytics",
     "project_fully_featured",
     "quickstart_etl",
+    "snowflake_cortex",
     "tutorial_notebook_assets",
     "with_great_expectations",
     "with_openai",