dagster 1.12.2__py3-none-any.whl → 1.12.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dagster/_config/pythonic_config/conversion_utils.py +2 -1
- dagster/_config/pythonic_config/resource.py +4 -2
- dagster/_config/source.py +17 -6
- dagster/_core/definitions/asset_daemon_cursor.py +4 -3
- dagster/_core/definitions/asset_sensor_definition.py +61 -1
- dagster/_core/definitions/assets/graph/remote_asset_graph.py +14 -11
- dagster/_core/definitions/automation_condition_sensor_definition.py +59 -2
- dagster/_core/definitions/declarative_automation/automation_condition.py +40 -6
- dagster/_core/definitions/declarative_automation/automation_context.py +8 -2
- dagster/_core/definitions/declarative_automation/legacy/legacy_context.py +10 -4
- dagster/_core/definitions/declarative_automation/legacy/rule_condition.py +8 -2
- dagster/_core/definitions/declarative_automation/operators/check_operators.py +18 -4
- dagster/_core/definitions/declarative_automation/operators/dep_operators.py +18 -4
- dagster/_core/definitions/declarative_automation/operators/newly_true_operator.py +27 -1
- dagster/_core/definitions/declarative_automation/operators/since_operator.py +27 -1
- dagster/_core/definitions/metadata/metadata_value.py +4 -3
- dagster/_core/definitions/multi_asset_sensor_definition.py +64 -2
- dagster/_core/definitions/op_definition.py +10 -2
- dagster/_core/definitions/reconstruct.py +0 -6
- dagster/_core/definitions/run_status_sensor_definition.py +79 -1
- dagster/_core/definitions/selector.py +4 -0
- dagster/_core/definitions/sensor_definition.py +32 -21
- dagster/_core/execution/backfill.py +29 -4
- dagster/_core/execution/context/output.py +26 -26
- dagster/_core/execution/plan/objects.py +3 -1
- dagster/_core/remote_representation/code_location.py +11 -13
- dagster/_core/remote_representation/handle.py +4 -2
- dagster/_core/workspace/context.py +9 -3
- dagster/_core/workspace/workspace.py +6 -0
- dagster/_daemon/asset_daemon.py +56 -11
- dagster/_daemon/sensor.py +11 -3
- dagster/_utils/error.py +1 -1
- dagster/components/component/component.py +31 -7
- dagster/components/core/component_tree.py +41 -28
- dagster/components/core/context.py +50 -15
- dagster/components/lib/definitions_component/__init__.py +2 -0
- dagster/components/lib/executable_component/function_component.py +26 -23
- dagster/components/lib/executable_component/python_script_component.py +2 -0
- dagster/components/lib/executable_component/uv_run_component.py +2 -0
- dagster/components/lib/sql_component/sql_component.py +1 -0
- dagster/components/list/list.py +1 -1
- dagster/components/resolved/context.py +15 -36
- dagster/components/resolved/scopes.py +161 -0
- dagster/components/testing/__init__.py +1 -0
- dagster/components/testing/utils.py +19 -18
- dagster/version.py +1 -1
- {dagster-1.12.2.dist-info → dagster-1.12.4.dist-info}/METADATA +3 -3
- {dagster-1.12.2.dist-info → dagster-1.12.4.dist-info}/RECORD +52 -51
- {dagster-1.12.2.dist-info → dagster-1.12.4.dist-info}/WHEEL +0 -0
- {dagster-1.12.2.dist-info → dagster-1.12.4.dist-info}/entry_points.txt +0 -0
- {dagster-1.12.2.dist-info → dagster-1.12.4.dist-info}/licenses/LICENSE +0 -0
- {dagster-1.12.2.dist-info → dagster-1.12.4.dist-info}/top_level.txt +0 -0
|
@@ -148,7 +148,8 @@ def _convert_pydantic_field(
|
|
|
148
148
|
config=config_type,
|
|
149
149
|
description=pydantic_field.description,
|
|
150
150
|
is_required=pydantic_field.is_required()
|
|
151
|
-
and not is_closed_python_optional_type(field_type)
|
|
151
|
+
and not is_closed_python_optional_type(field_type)
|
|
152
|
+
and default_to_pass == FIELD_NO_DEFAULT_PROVIDED,
|
|
152
153
|
default_value=default_to_pass,
|
|
153
154
|
)
|
|
154
155
|
|
|
@@ -704,7 +704,7 @@ class PartialResource(
|
|
|
704
704
|
resource_cls: type[ConfigurableResourceFactory[TResValue]],
|
|
705
705
|
data: dict[str, Any],
|
|
706
706
|
):
|
|
707
|
-
resource_pointers,
|
|
707
|
+
resource_pointers, data_without_resources = separate_resource_params(resource_cls, data)
|
|
708
708
|
|
|
709
709
|
super().__init__(data=data, resource_cls=resource_cls) # type: ignore # extends BaseModel, takes kwargs
|
|
710
710
|
|
|
@@ -724,7 +724,9 @@ class PartialResource(
|
|
|
724
724
|
k: v for k, v in resource_pointers.items() if (not _is_fully_configured(v))
|
|
725
725
|
},
|
|
726
726
|
config_schema=infer_schema_from_config_class(
|
|
727
|
-
resource_cls,
|
|
727
|
+
resource_cls,
|
|
728
|
+
fields_to_omit=set(resource_pointers.keys()),
|
|
729
|
+
default=data_without_resources,
|
|
728
730
|
),
|
|
729
731
|
resource_fn=resource_fn,
|
|
730
732
|
description=resource_cls.__doc__,
|
dagster/_config/source.py
CHANGED
|
@@ -60,12 +60,17 @@ class IntSourceType(ScalarUnion):
|
|
|
60
60
|
key, cfg = next(iter(value.items()))
|
|
61
61
|
check.invariant(key == "env", "Only valid key is env")
|
|
62
62
|
value = _ensure_env_variable(cfg)
|
|
63
|
+
validation_failed = False
|
|
63
64
|
try:
|
|
64
65
|
return int(value)
|
|
65
|
-
except ValueError
|
|
66
|
+
except ValueError:
|
|
67
|
+
# raise exception separately to ensure the exception chain doesn't leak the value of the env var
|
|
68
|
+
validation_failed = True
|
|
69
|
+
|
|
70
|
+
if validation_failed:
|
|
66
71
|
raise PostProcessingError(
|
|
67
|
-
f'Value
|
|
68
|
-
)
|
|
72
|
+
f'Value stored in env variable "{cfg}" cannot be coerced into an int.'
|
|
73
|
+
)
|
|
69
74
|
|
|
70
75
|
|
|
71
76
|
class BoolSourceType(ScalarUnion):
|
|
@@ -87,12 +92,18 @@ class BoolSourceType(ScalarUnion):
|
|
|
87
92
|
key, cfg = next(iter(value.items()))
|
|
88
93
|
check.invariant(key == "env", "Only valid key is env")
|
|
89
94
|
value = _ensure_env_variable(cfg)
|
|
95
|
+
validation_failed = False
|
|
96
|
+
|
|
90
97
|
try:
|
|
91
98
|
return get_boolean_string_value(value)
|
|
92
|
-
except ValueError
|
|
99
|
+
except ValueError:
|
|
100
|
+
# raise exception separately to ensure the exception chain doesn't leak the value of the env var
|
|
101
|
+
validation_failed = True
|
|
102
|
+
|
|
103
|
+
if validation_failed:
|
|
93
104
|
raise PostProcessingError(
|
|
94
|
-
f'Value
|
|
95
|
-
)
|
|
105
|
+
f'Value stored in env variable "{cfg}" cannot be coerced into an bool.'
|
|
106
|
+
)
|
|
96
107
|
|
|
97
108
|
|
|
98
109
|
StringSource: StringSourceType = StringSourceType()
|
|
@@ -11,9 +11,10 @@ from dagster_shared.serdes.serdes import (
|
|
|
11
11
|
PackableValue,
|
|
12
12
|
SerializableNonScalarKeyMapping,
|
|
13
13
|
UnpackContext,
|
|
14
|
+
UnpackedValue,
|
|
14
15
|
WhitelistMap,
|
|
16
|
+
inner_unpack_value,
|
|
15
17
|
pack_value,
|
|
16
|
-
unpack_value,
|
|
17
18
|
whitelist_for_serdes,
|
|
18
19
|
)
|
|
19
20
|
|
|
@@ -53,8 +54,8 @@ class ObserveRequestTimestampSerializer(FieldSerializer):
|
|
|
53
54
|
unpacked_value: JsonSerializableValue,
|
|
54
55
|
whitelist_map: WhitelistMap,
|
|
55
56
|
context: UnpackContext,
|
|
56
|
-
) ->
|
|
57
|
-
return
|
|
57
|
+
) -> UnpackedValue:
|
|
58
|
+
return inner_unpack_value(unpacked_value, whitelist_map, context)
|
|
58
59
|
|
|
59
60
|
|
|
60
61
|
@whitelist_for_serdes(
|
|
@@ -14,10 +14,12 @@ from dagster._core.definitions.sensor_definition import (
|
|
|
14
14
|
SensorDefinition,
|
|
15
15
|
SensorReturnTypesUnion,
|
|
16
16
|
SensorType,
|
|
17
|
+
resolve_jobs_from_targets_for_with_attributes,
|
|
17
18
|
validate_and_get_resource_dict,
|
|
18
19
|
)
|
|
19
20
|
from dagster._core.definitions.target import ExecutableDefinition
|
|
20
21
|
from dagster._core.definitions.utils import check_valid_name
|
|
22
|
+
from dagster._utils import IHasInternalInit
|
|
21
23
|
|
|
22
24
|
|
|
23
25
|
class AssetSensorParamNames(NamedTuple):
|
|
@@ -44,7 +46,7 @@ def get_asset_sensor_param_names(fn: Callable[..., Any]) -> AssetSensorParamName
|
|
|
44
46
|
|
|
45
47
|
|
|
46
48
|
@public
|
|
47
|
-
class AssetSensorDefinition(SensorDefinition):
|
|
49
|
+
class AssetSensorDefinition(SensorDefinition, IHasInternalInit):
|
|
48
50
|
"""Define an asset sensor that initiates a set of runs based on the materialization of a given
|
|
49
51
|
asset.
|
|
50
52
|
|
|
@@ -95,6 +97,8 @@ class AssetSensorDefinition(SensorDefinition):
|
|
|
95
97
|
metadata: Optional[RawMetadataMapping] = None,
|
|
96
98
|
):
|
|
97
99
|
self._asset_key = check.inst_param(asset_key, "asset_key", AssetKey)
|
|
100
|
+
self._asset_materialization_fn = asset_materialization_fn
|
|
101
|
+
self._job_name = job_name
|
|
98
102
|
|
|
99
103
|
from dagster._core.event_api import AssetRecordsFilter
|
|
100
104
|
|
|
@@ -106,6 +110,7 @@ class AssetSensorDefinition(SensorDefinition):
|
|
|
106
110
|
check.opt_set_param(required_resource_keys, "required_resource_keys", of_type=str)
|
|
107
111
|
| resource_arg_names
|
|
108
112
|
)
|
|
113
|
+
self._raw_required_resource_keys = combined_required_resource_keys
|
|
109
114
|
|
|
110
115
|
def _wrap_asset_fn(materialization_fn) -> Any:
|
|
111
116
|
def _fn(context) -> Any:
|
|
@@ -184,3 +189,58 @@ class AssetSensorDefinition(SensorDefinition):
|
|
|
184
189
|
@property
|
|
185
190
|
def sensor_type(self) -> SensorType:
|
|
186
191
|
return SensorType.ASSET
|
|
192
|
+
|
|
193
|
+
@staticmethod
|
|
194
|
+
def dagster_internal_init( # type: ignore
|
|
195
|
+
*,
|
|
196
|
+
name: str,
|
|
197
|
+
asset_key: AssetKey,
|
|
198
|
+
job_name: Optional[str],
|
|
199
|
+
asset_materialization_fn: Callable[..., SensorReturnTypesUnion],
|
|
200
|
+
minimum_interval_seconds: Optional[int],
|
|
201
|
+
description: Optional[str],
|
|
202
|
+
job: Optional[ExecutableDefinition],
|
|
203
|
+
jobs: Optional[Sequence[ExecutableDefinition]],
|
|
204
|
+
default_status: DefaultSensorStatus,
|
|
205
|
+
required_resource_keys: Optional[set[str]],
|
|
206
|
+
tags: Optional[Mapping[str, str]],
|
|
207
|
+
metadata: Optional[RawMetadataMapping],
|
|
208
|
+
) -> "AssetSensorDefinition":
|
|
209
|
+
return AssetSensorDefinition(
|
|
210
|
+
name=name,
|
|
211
|
+
asset_key=asset_key,
|
|
212
|
+
job_name=job_name,
|
|
213
|
+
asset_materialization_fn=asset_materialization_fn,
|
|
214
|
+
minimum_interval_seconds=minimum_interval_seconds,
|
|
215
|
+
description=description,
|
|
216
|
+
job=job,
|
|
217
|
+
jobs=jobs,
|
|
218
|
+
default_status=default_status,
|
|
219
|
+
required_resource_keys=required_resource_keys,
|
|
220
|
+
tags=tags,
|
|
221
|
+
metadata=metadata,
|
|
222
|
+
)
|
|
223
|
+
|
|
224
|
+
def with_attributes(
|
|
225
|
+
self,
|
|
226
|
+
*,
|
|
227
|
+
jobs: Optional[Sequence[ExecutableDefinition]] = None,
|
|
228
|
+
metadata: Optional[RawMetadataMapping] = None,
|
|
229
|
+
) -> "AssetSensorDefinition":
|
|
230
|
+
"""Returns a copy of this sensor with the attributes replaced."""
|
|
231
|
+
job_name, new_job, new_jobs = resolve_jobs_from_targets_for_with_attributes(self, jobs)
|
|
232
|
+
|
|
233
|
+
return AssetSensorDefinition.dagster_internal_init(
|
|
234
|
+
name=self.name,
|
|
235
|
+
asset_key=self._asset_key,
|
|
236
|
+
job_name=job_name,
|
|
237
|
+
asset_materialization_fn=self._asset_materialization_fn,
|
|
238
|
+
minimum_interval_seconds=self.minimum_interval_seconds,
|
|
239
|
+
description=self.description,
|
|
240
|
+
job=new_job,
|
|
241
|
+
jobs=new_jobs,
|
|
242
|
+
default_status=self.default_status,
|
|
243
|
+
required_resource_keys=self._raw_required_resource_keys,
|
|
244
|
+
tags=self._tags,
|
|
245
|
+
metadata=metadata if metadata is not None else self._metadata,
|
|
246
|
+
)
|
|
@@ -40,9 +40,10 @@ from dagster._core.definitions.freshness_policy import LegacyFreshnessPolicy
|
|
|
40
40
|
from dagster._core.definitions.metadata import ArbitraryMetadataMapping
|
|
41
41
|
from dagster._core.definitions.partitions.definition import PartitionsDefinition
|
|
42
42
|
from dagster._core.definitions.partitions.mapping import PartitionMapping
|
|
43
|
+
from dagster._core.definitions.selector import ScheduleSelector, SensorSelector
|
|
43
44
|
from dagster._core.definitions.utils import DEFAULT_GROUP_NAME
|
|
44
45
|
from dagster._core.remote_representation.external import RemoteRepository
|
|
45
|
-
from dagster._core.remote_representation.handle import
|
|
46
|
+
from dagster._core.remote_representation.handle import RepositoryHandle
|
|
46
47
|
from dagster._core.workspace.workspace import CurrentWorkspace
|
|
47
48
|
from dagster._record import ImportFrom, record
|
|
48
49
|
from dagster._utils.cached_method import cached_method
|
|
@@ -395,31 +396,33 @@ class RemoteWorkspaceAssetNode(RemoteAssetNode):
|
|
|
395
396
|
)
|
|
396
397
|
)
|
|
397
398
|
|
|
398
|
-
def
|
|
399
|
+
def get_targeting_schedule_selectors(
|
|
399
400
|
self,
|
|
400
|
-
) -> Sequence[
|
|
401
|
+
) -> Sequence[ScheduleSelector]:
|
|
401
402
|
selectors = []
|
|
402
403
|
for node in self.repo_scoped_asset_infos:
|
|
403
404
|
for schedule_name in node.targeting_schedule_names:
|
|
404
405
|
selectors.append(
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
406
|
+
ScheduleSelector(
|
|
407
|
+
location_name=node.handle.location_name,
|
|
408
|
+
repository_name=node.handle.repository_name,
|
|
409
|
+
schedule_name=schedule_name,
|
|
408
410
|
)
|
|
409
411
|
)
|
|
410
412
|
|
|
411
413
|
return selectors
|
|
412
414
|
|
|
413
|
-
def
|
|
415
|
+
def get_targeting_sensor_selectors(
|
|
414
416
|
self,
|
|
415
|
-
) -> Sequence[
|
|
417
|
+
) -> Sequence[SensorSelector]:
|
|
416
418
|
selectors = []
|
|
417
419
|
for node in self.repo_scoped_asset_infos:
|
|
418
420
|
for sensor_name in node.targeting_sensor_names:
|
|
419
421
|
selectors.append(
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
422
|
+
SensorSelector(
|
|
423
|
+
location_name=node.handle.location_name,
|
|
424
|
+
repository_name=node.handle.repository_name,
|
|
425
|
+
sensor_name=sensor_name,
|
|
423
426
|
)
|
|
424
427
|
)
|
|
425
428
|
return selectors
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from collections.abc import Mapping
|
|
1
|
+
from collections.abc import Mapping, Sequence
|
|
2
2
|
from functools import partial
|
|
3
3
|
from typing import Any, Optional, cast
|
|
4
4
|
|
|
@@ -16,8 +16,10 @@ from dagster._core.definitions.sensor_definition import (
|
|
|
16
16
|
SensorEvaluationContext,
|
|
17
17
|
SensorType,
|
|
18
18
|
)
|
|
19
|
+
from dagster._core.definitions.target import ExecutableDefinition
|
|
19
20
|
from dagster._core.definitions.utils import check_valid_name
|
|
20
21
|
from dagster._core.errors import DagsterInvalidInvocationError
|
|
22
|
+
from dagster._utils import IHasInternalInit
|
|
21
23
|
from dagster._utils.tags import normalize_tags
|
|
22
24
|
|
|
23
25
|
MAX_ENTITIES = 500
|
|
@@ -79,7 +81,7 @@ def not_supported(context) -> None:
|
|
|
79
81
|
@public
|
|
80
82
|
@beta_param(param="use_user_code_server")
|
|
81
83
|
@beta_param(param="default_condition")
|
|
82
|
-
class AutomationConditionSensorDefinition(SensorDefinition):
|
|
84
|
+
class AutomationConditionSensorDefinition(SensorDefinition, IHasInternalInit):
|
|
83
85
|
"""Targets a set of assets and repeatedly evaluates all the AutomationConditions on all of
|
|
84
86
|
those assets to determine which to request runs for.
|
|
85
87
|
|
|
@@ -171,6 +173,8 @@ class AutomationConditionSensorDefinition(SensorDefinition):
|
|
|
171
173
|
)
|
|
172
174
|
|
|
173
175
|
self._run_tags = normalize_tags(run_tags)
|
|
176
|
+
self._sensor_target = target
|
|
177
|
+
self._emit_backfills = emit_backfills
|
|
174
178
|
|
|
175
179
|
# only store this value in the metadata if it's True
|
|
176
180
|
if emit_backfills:
|
|
@@ -210,3 +214,56 @@ class AutomationConditionSensorDefinition(SensorDefinition):
|
|
|
210
214
|
@property
|
|
211
215
|
def sensor_type(self) -> SensorType:
|
|
212
216
|
return SensorType.AUTOMATION if self._use_user_code_server else SensorType.AUTO_MATERIALIZE
|
|
217
|
+
|
|
218
|
+
@staticmethod
|
|
219
|
+
def dagster_internal_init( # type: ignore
|
|
220
|
+
*,
|
|
221
|
+
name: str,
|
|
222
|
+
target: CoercibleToAssetSelection,
|
|
223
|
+
tags: Optional[Mapping[str, str]],
|
|
224
|
+
run_tags: Optional[Mapping[str, Any]],
|
|
225
|
+
default_status: DefaultSensorStatus,
|
|
226
|
+
minimum_interval_seconds: Optional[int],
|
|
227
|
+
description: Optional[str],
|
|
228
|
+
metadata: Optional[RawMetadataMapping],
|
|
229
|
+
emit_backfills: bool,
|
|
230
|
+
use_user_code_server: bool,
|
|
231
|
+
default_condition: Optional[AutomationCondition],
|
|
232
|
+
) -> "AutomationConditionSensorDefinition":
|
|
233
|
+
return AutomationConditionSensorDefinition(
|
|
234
|
+
name=name,
|
|
235
|
+
target=target,
|
|
236
|
+
tags=tags,
|
|
237
|
+
run_tags=run_tags,
|
|
238
|
+
default_status=default_status,
|
|
239
|
+
minimum_interval_seconds=minimum_interval_seconds,
|
|
240
|
+
description=description,
|
|
241
|
+
metadata=metadata,
|
|
242
|
+
emit_backfills=emit_backfills,
|
|
243
|
+
use_user_code_server=use_user_code_server,
|
|
244
|
+
default_condition=default_condition,
|
|
245
|
+
)
|
|
246
|
+
|
|
247
|
+
def with_attributes(
|
|
248
|
+
self,
|
|
249
|
+
*,
|
|
250
|
+
jobs: Optional[Sequence[ExecutableDefinition]] = None,
|
|
251
|
+
metadata: Optional[RawMetadataMapping] = None,
|
|
252
|
+
) -> "AutomationConditionSensorDefinition":
|
|
253
|
+
"""Returns a copy of this sensor with the attributes replaced.
|
|
254
|
+
|
|
255
|
+
Note: jobs parameter is ignored for AutomationConditionSensorDefinition as it doesn't use jobs.
|
|
256
|
+
"""
|
|
257
|
+
return AutomationConditionSensorDefinition.dagster_internal_init(
|
|
258
|
+
name=self.name,
|
|
259
|
+
target=self._sensor_target,
|
|
260
|
+
tags=self._tags,
|
|
261
|
+
run_tags=self._run_tags,
|
|
262
|
+
default_status=self.default_status,
|
|
263
|
+
minimum_interval_seconds=self.minimum_interval_seconds,
|
|
264
|
+
description=self.description,
|
|
265
|
+
metadata=metadata if metadata is not None else self._metadata,
|
|
266
|
+
emit_backfills=self._emit_backfills,
|
|
267
|
+
use_user_code_server=self._use_user_code_server,
|
|
268
|
+
default_condition=self._default_condition,
|
|
269
|
+
)
|
|
@@ -15,6 +15,7 @@ from dagster._core.definitions.asset_key import (
|
|
|
15
15
|
AssetCheckKey,
|
|
16
16
|
AssetKey,
|
|
17
17
|
CoercibleToAssetKey,
|
|
18
|
+
EntityKey,
|
|
18
19
|
T_EntityKey,
|
|
19
20
|
)
|
|
20
21
|
from dagster._core.definitions.declarative_automation.serialized_objects import (
|
|
@@ -136,7 +137,9 @@ class AutomationCondition(ABC, Generic[T_EntityKey]):
|
|
|
136
137
|
self, *, parent_unique_id: Optional[str] = None, index: Optional[int] = None
|
|
137
138
|
) -> AutomationConditionSnapshot:
|
|
138
139
|
"""Returns a serializable snapshot of the entire AutomationCondition tree."""
|
|
139
|
-
unique_id = self.get_node_unique_id(
|
|
140
|
+
unique_id = self.get_node_unique_id(
|
|
141
|
+
parent_unique_id=parent_unique_id, index=index, target_key=None
|
|
142
|
+
)
|
|
140
143
|
node_snapshot = self.get_node_snapshot(unique_id)
|
|
141
144
|
children = [
|
|
142
145
|
child.get_snapshot(parent_unique_id=unique_id, index=i)
|
|
@@ -144,12 +147,22 @@ class AutomationCondition(ABC, Generic[T_EntityKey]):
|
|
|
144
147
|
]
|
|
145
148
|
return AutomationConditionSnapshot(node_snapshot=node_snapshot, children=children)
|
|
146
149
|
|
|
147
|
-
def get_node_unique_id(
|
|
150
|
+
def get_node_unique_id(
|
|
151
|
+
self,
|
|
152
|
+
*,
|
|
153
|
+
parent_unique_id: Optional[str],
|
|
154
|
+
index: Optional[int],
|
|
155
|
+
target_key: Optional[EntityKey],
|
|
156
|
+
) -> str:
|
|
148
157
|
"""Returns a unique identifier for this condition within the broader condition tree."""
|
|
149
158
|
return non_secure_md5_hash_str(f"{parent_unique_id}{index}{self.name}".encode())
|
|
150
159
|
|
|
151
160
|
def get_backcompat_node_unique_ids(
|
|
152
|
-
self,
|
|
161
|
+
self,
|
|
162
|
+
*,
|
|
163
|
+
parent_unique_id: Optional[str] = None,
|
|
164
|
+
index: Optional[int] = None,
|
|
165
|
+
target_key: Optional[EntityKey] = None,
|
|
153
166
|
) -> Sequence[str]:
|
|
154
167
|
"""Used for backwards compatibility when condition unique id logic changes."""
|
|
155
168
|
return []
|
|
@@ -159,6 +172,7 @@ class AutomationCondition(ABC, Generic[T_EntityKey]):
|
|
|
159
172
|
*,
|
|
160
173
|
parent_unique_ids: Sequence[Optional[str]],
|
|
161
174
|
child_indices: Sequence[Optional[int]],
|
|
175
|
+
target_key: Optional[EntityKey],
|
|
162
176
|
) -> Sequence[str]:
|
|
163
177
|
unique_ids = []
|
|
164
178
|
for parent_unique_id in parent_unique_ids:
|
|
@@ -166,10 +180,14 @@ class AutomationCondition(ABC, Generic[T_EntityKey]):
|
|
|
166
180
|
unique_ids.extend(
|
|
167
181
|
[
|
|
168
182
|
self.get_node_unique_id(
|
|
169
|
-
parent_unique_id=parent_unique_id,
|
|
183
|
+
parent_unique_id=parent_unique_id,
|
|
184
|
+
index=child_index,
|
|
185
|
+
target_key=target_key,
|
|
170
186
|
),
|
|
171
187
|
*self.get_backcompat_node_unique_ids(
|
|
172
|
-
parent_unique_id=parent_unique_id,
|
|
188
|
+
parent_unique_id=parent_unique_id,
|
|
189
|
+
index=child_index,
|
|
190
|
+
target_key=target_key,
|
|
173
191
|
),
|
|
174
192
|
]
|
|
175
193
|
)
|
|
@@ -180,7 +198,7 @@ class AutomationCondition(ABC, Generic[T_EntityKey]):
|
|
|
180
198
|
) -> str:
|
|
181
199
|
"""Returns a unique identifier for the entire subtree."""
|
|
182
200
|
node_unique_id = self.get_node_unique_id(
|
|
183
|
-
parent_unique_id=parent_node_unique_id, index=index
|
|
201
|
+
parent_unique_id=parent_node_unique_id, index=index, target_key=None
|
|
184
202
|
)
|
|
185
203
|
child_unique_ids = [
|
|
186
204
|
child.get_unique_id(parent_node_unique_id=node_unique_id, index=i)
|
|
@@ -845,6 +863,22 @@ class BuiltinAutomationCondition(AutomationCondition[T_EntityKey]):
|
|
|
845
863
|
"""Returns a copy of this AutomationCondition with a human-readable label."""
|
|
846
864
|
return copy(self, label=label)
|
|
847
865
|
|
|
866
|
+
def _get_stable_unique_id(self, target_key: Optional[EntityKey]) -> str:
|
|
867
|
+
"""Returns an identifier that is stable regardless of where it exists in the broader condition tree.
|
|
868
|
+
This should only be used for conditions that don't change their output based on what conditions are
|
|
869
|
+
evaluated before them (i.e. they explicitly set their candidate subset to the entire subset).
|
|
870
|
+
"""
|
|
871
|
+
child_ids = [
|
|
872
|
+
child.get_node_unique_id(
|
|
873
|
+
parent_unique_id=None,
|
|
874
|
+
index=i,
|
|
875
|
+
target_key=target_key,
|
|
876
|
+
)
|
|
877
|
+
for i, child in enumerate(self.children)
|
|
878
|
+
]
|
|
879
|
+
parts = [self.name, *child_ids, target_key.to_user_string() if target_key else ""]
|
|
880
|
+
return non_secure_md5_hash_str("".join(parts).encode())
|
|
881
|
+
|
|
848
882
|
|
|
849
883
|
@public
|
|
850
884
|
@hidden_param(param="subsets_with_metadata", breaking_version="", emit_runtime_warning=False)
|
|
@@ -73,7 +73,9 @@ class AutomationContext(Generic[T_EntityKey]):
|
|
|
73
73
|
condition = check.not_none(
|
|
74
74
|
evaluator.asset_graph.get(key).automation_condition or evaluator.default_condition
|
|
75
75
|
)
|
|
76
|
-
unique_ids = condition.get_node_unique_ids(
|
|
76
|
+
unique_ids = condition.get_node_unique_ids(
|
|
77
|
+
parent_unique_ids=[None], child_indices=[None], target_key=None
|
|
78
|
+
)
|
|
77
79
|
|
|
78
80
|
return AutomationContext(
|
|
79
81
|
condition=condition,
|
|
@@ -101,7 +103,11 @@ class AutomationContext(Generic[T_EntityKey]):
|
|
|
101
103
|
check.invariant(len(child_indices) > 0, "Must be at least one child index")
|
|
102
104
|
|
|
103
105
|
unique_ids = child_condition.get_node_unique_ids(
|
|
104
|
-
parent_unique_ids=self.condition_unique_ids,
|
|
106
|
+
parent_unique_ids=self.condition_unique_ids,
|
|
107
|
+
child_indices=child_indices,
|
|
108
|
+
target_key=candidate_subset.key
|
|
109
|
+
if candidate_subset.key != self.root_context.key
|
|
110
|
+
else None,
|
|
105
111
|
)
|
|
106
112
|
return AutomationContext(
|
|
107
113
|
condition=child_condition,
|
|
@@ -88,7 +88,7 @@ class LegacyRuleEvaluationContext:
|
|
|
88
88
|
condition=condition,
|
|
89
89
|
cursor=cursor,
|
|
90
90
|
node_cursor=cursor.node_cursors_by_unique_id.get(
|
|
91
|
-
condition.get_node_unique_id(parent_unique_id=None, index=0)
|
|
91
|
+
condition.get_node_unique_id(parent_unique_id=None, index=0, target_key=None)
|
|
92
92
|
)
|
|
93
93
|
if cursor
|
|
94
94
|
else None,
|
|
@@ -224,17 +224,23 @@ class LegacyRuleEvaluationContext:
|
|
|
224
224
|
# Or(MaterializeCond, Not(SkipCond), Not(DiscardCond))
|
|
225
225
|
if len(self.condition.children) != 3:
|
|
226
226
|
return None
|
|
227
|
-
unique_id = self.condition.get_node_unique_id(
|
|
227
|
+
unique_id = self.condition.get_node_unique_id(
|
|
228
|
+
parent_unique_id=None, index=None, target_key=None
|
|
229
|
+
)
|
|
228
230
|
|
|
229
231
|
# get Not(DiscardCond)
|
|
230
232
|
not_discard_condition = self.condition.children[2]
|
|
231
|
-
unique_id = not_discard_condition.get_node_unique_id(
|
|
233
|
+
unique_id = not_discard_condition.get_node_unique_id(
|
|
234
|
+
parent_unique_id=unique_id, index=2, target_key=None
|
|
235
|
+
)
|
|
232
236
|
if not isinstance(not_discard_condition, NotAutomationCondition):
|
|
233
237
|
return None
|
|
234
238
|
|
|
235
239
|
# get DiscardCond
|
|
236
240
|
discard_condition = not_discard_condition.children[0]
|
|
237
|
-
unique_id = discard_condition.get_node_unique_id(
|
|
241
|
+
unique_id = discard_condition.get_node_unique_id(
|
|
242
|
+
parent_unique_id=unique_id, index=0, target_key=None
|
|
243
|
+
)
|
|
238
244
|
if not isinstance(discard_condition, RuleCondition) or not isinstance(
|
|
239
245
|
discard_condition.rule, DiscardOnMaxMaterializationsExceededRule
|
|
240
246
|
):
|
|
@@ -2,7 +2,7 @@ from typing import TYPE_CHECKING, Optional
|
|
|
2
2
|
|
|
3
3
|
from dagster_shared.serdes import whitelist_for_serdes
|
|
4
4
|
|
|
5
|
-
from dagster._core.definitions.asset_key import AssetKey
|
|
5
|
+
from dagster._core.definitions.asset_key import AssetKey, EntityKey
|
|
6
6
|
from dagster._core.definitions.auto_materialize_rule import AutoMaterializeRule
|
|
7
7
|
from dagster._core.definitions.declarative_automation.automation_condition import (
|
|
8
8
|
AutomationResult,
|
|
@@ -24,7 +24,13 @@ class RuleCondition(BuiltinAutomationCondition[AssetKey]):
|
|
|
24
24
|
|
|
25
25
|
rule: AutoMaterializeRule
|
|
26
26
|
|
|
27
|
-
def get_node_unique_id(
|
|
27
|
+
def get_node_unique_id(
|
|
28
|
+
self,
|
|
29
|
+
*,
|
|
30
|
+
parent_unique_id: Optional[str],
|
|
31
|
+
index: Optional[int],
|
|
32
|
+
target_key: Optional[EntityKey],
|
|
33
|
+
) -> str:
|
|
28
34
|
# preserves old (bad) behavior of not including the parent_unique_id to avoid invalidating
|
|
29
35
|
# old serialized information
|
|
30
36
|
parts = [self.rule.__class__.__name__, self.description]
|
|
@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, AbstractSet, Any, Optional, Sequence # noqa:
|
|
|
5
5
|
from dagster_shared.serdes import whitelist_for_serdes
|
|
6
6
|
|
|
7
7
|
import dagster._check as check
|
|
8
|
-
from dagster._core.definitions.asset_key import AssetCheckKey, AssetKey
|
|
8
|
+
from dagster._core.definitions.asset_key import AssetCheckKey, AssetKey, EntityKey
|
|
9
9
|
from dagster._core.definitions.assets.graph.base_asset_graph import BaseAssetGraph, BaseAssetNode
|
|
10
10
|
from dagster._core.definitions.declarative_automation.automation_condition import (
|
|
11
11
|
AutomationCondition,
|
|
@@ -56,16 +56,30 @@ class ChecksAutomationCondition(BuiltinAutomationCondition[AssetKey]):
|
|
|
56
56
|
def requires_cursor(self) -> bool:
|
|
57
57
|
return False
|
|
58
58
|
|
|
59
|
-
def get_node_unique_id(
|
|
59
|
+
def get_node_unique_id(
|
|
60
|
+
self,
|
|
61
|
+
*,
|
|
62
|
+
parent_unique_id: Optional[str],
|
|
63
|
+
index: Optional[int],
|
|
64
|
+
target_key: Optional[EntityKey],
|
|
65
|
+
) -> str:
|
|
60
66
|
"""Ignore allow_selection / ignore_selection for the cursor hash."""
|
|
61
67
|
parts = [str(parent_unique_id), str(index), self.base_name]
|
|
62
68
|
return non_secure_md5_hash_str("".join(parts).encode())
|
|
63
69
|
|
|
64
70
|
def get_backcompat_node_unique_ids(
|
|
65
|
-
self,
|
|
71
|
+
self,
|
|
72
|
+
*,
|
|
73
|
+
parent_unique_id: Optional[str] = None,
|
|
74
|
+
index: Optional[int] = None,
|
|
75
|
+
target_key: Optional[EntityKey] = None,
|
|
66
76
|
) -> Sequence[str]:
|
|
67
77
|
# backcompat for previous cursors where the allow/ignore selection influenced the hash
|
|
68
|
-
return [
|
|
78
|
+
return [
|
|
79
|
+
super().get_node_unique_id(
|
|
80
|
+
parent_unique_id=parent_unique_id, index=index, target_key=target_key
|
|
81
|
+
)
|
|
82
|
+
]
|
|
69
83
|
|
|
70
84
|
def allow(self, selection: "AssetSelection") -> "ChecksAutomationCondition":
|
|
71
85
|
"""Returns a copy of this condition that will only consider dependencies within the provided
|
|
@@ -8,7 +8,7 @@ from typing_extensions import Self
|
|
|
8
8
|
import dagster._check as check
|
|
9
9
|
from dagster._annotations import public
|
|
10
10
|
from dagster._core.asset_graph_view.asset_graph_view import U_EntityKey
|
|
11
|
-
from dagster._core.definitions.asset_key import AssetKey, T_EntityKey
|
|
11
|
+
from dagster._core.definitions.asset_key import AssetKey, EntityKey, T_EntityKey
|
|
12
12
|
from dagster._core.definitions.assets.graph.base_asset_graph import BaseAssetGraph, BaseAssetNode
|
|
13
13
|
from dagster._core.definitions.declarative_automation.automation_condition import (
|
|
14
14
|
AutomationCondition,
|
|
@@ -123,16 +123,30 @@ class DepsAutomationCondition(BuiltinAutomationCondition[T_EntityKey]):
|
|
|
123
123
|
def requires_cursor(self) -> bool:
|
|
124
124
|
return False
|
|
125
125
|
|
|
126
|
-
def get_node_unique_id(
|
|
126
|
+
def get_node_unique_id(
|
|
127
|
+
self,
|
|
128
|
+
*,
|
|
129
|
+
parent_unique_id: Optional[str],
|
|
130
|
+
index: Optional[int],
|
|
131
|
+
target_key: Optional[EntityKey],
|
|
132
|
+
) -> str:
|
|
127
133
|
"""Ignore allow_selection / ignore_selection for the cursor hash."""
|
|
128
134
|
parts = [str(parent_unique_id), str(index), self.base_name]
|
|
129
135
|
return non_secure_md5_hash_str("".join(parts).encode())
|
|
130
136
|
|
|
131
137
|
def get_backcompat_node_unique_ids(
|
|
132
|
-
self,
|
|
138
|
+
self,
|
|
139
|
+
*,
|
|
140
|
+
parent_unique_id: Optional[str] = None,
|
|
141
|
+
index: Optional[int] = None,
|
|
142
|
+
target_key: Optional[EntityKey] = None,
|
|
133
143
|
) -> Sequence[str]:
|
|
134
144
|
# backcompat for previous cursors where the allow/ignore selection influenced the hash
|
|
135
|
-
return [
|
|
145
|
+
return [
|
|
146
|
+
super().get_node_unique_id(
|
|
147
|
+
parent_unique_id=parent_unique_id, index=index, target_key=target_key
|
|
148
|
+
)
|
|
149
|
+
]
|
|
136
150
|
|
|
137
151
|
@public
|
|
138
152
|
def allow(self, selection: "AssetSelection") -> "DepsAutomationCondition":
|
|
@@ -5,7 +5,7 @@ from dagster_shared.serdes import whitelist_for_serdes
|
|
|
5
5
|
|
|
6
6
|
from dagster._core.asset_graph_view.entity_subset import EntitySubset
|
|
7
7
|
from dagster._core.asset_graph_view.serializable_entity_subset import SerializableEntitySubset
|
|
8
|
-
from dagster._core.definitions.asset_key import T_EntityKey
|
|
8
|
+
from dagster._core.definitions.asset_key import EntityKey, T_EntityKey
|
|
9
9
|
from dagster._core.definitions.declarative_automation.automation_condition import (
|
|
10
10
|
AutomationCondition,
|
|
11
11
|
AutomationResult,
|
|
@@ -39,6 +39,32 @@ class NewlyTrueCondition(BuiltinAutomationCondition[T_EntityKey]):
|
|
|
39
39
|
return None
|
|
40
40
|
return context.asset_graph_view.get_subset_from_serializable_subset(true_subset)
|
|
41
41
|
|
|
42
|
+
def get_node_unique_id(
|
|
43
|
+
self,
|
|
44
|
+
*,
|
|
45
|
+
parent_unique_id: Optional[str],
|
|
46
|
+
index: Optional[int],
|
|
47
|
+
target_key: Optional[EntityKey],
|
|
48
|
+
) -> str:
|
|
49
|
+
# newly true conditions should have stable cursoring logic regardless of where they
|
|
50
|
+
# exist in the broader condition tree, as they're always evaluated over the entire
|
|
51
|
+
# subset
|
|
52
|
+
return self._get_stable_unique_id(target_key)
|
|
53
|
+
|
|
54
|
+
def get_backcompat_node_unique_ids(
|
|
55
|
+
self,
|
|
56
|
+
*,
|
|
57
|
+
parent_unique_id: Optional[str] = None,
|
|
58
|
+
index: Optional[int] = None,
|
|
59
|
+
target_key: Optional[EntityKey] = None,
|
|
60
|
+
) -> Sequence[str]:
|
|
61
|
+
return [
|
|
62
|
+
# get the standard globally-aware unique id for backcompat purposes
|
|
63
|
+
super().get_node_unique_id(
|
|
64
|
+
parent_unique_id=parent_unique_id, index=index, target_key=target_key
|
|
65
|
+
)
|
|
66
|
+
]
|
|
67
|
+
|
|
42
68
|
async def evaluate(self, context: AutomationContext) -> AutomationResult: # pyright: ignore[reportIncompatibleMethodOverride]
|
|
43
69
|
# evaluate child condition
|
|
44
70
|
child_result = await context.for_child_condition(
|