zenml-nightly 0.83.0.dev20250604__py3-none-any.whl → 0.83.0.dev20250606__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- zenml/VERSION +1 -1
- zenml/artifacts/utils.py +21 -3
- zenml/client.py +0 -19
- zenml/integrations/aws/orchestrators/sagemaker_orchestrator.py +83 -39
- zenml/integrations/aws/step_operators/sagemaker_step_operator.py +18 -13
- zenml/models/v2/core/stack.py +56 -10
- zenml/zen_stores/sql_zen_store.py +84 -79
- {zenml_nightly-0.83.0.dev20250604.dist-info → zenml_nightly-0.83.0.dev20250606.dist-info}/METADATA +1 -1
- {zenml_nightly-0.83.0.dev20250604.dist-info → zenml_nightly-0.83.0.dev20250606.dist-info}/RECORD +12 -12
- {zenml_nightly-0.83.0.dev20250604.dist-info → zenml_nightly-0.83.0.dev20250606.dist-info}/LICENSE +0 -0
- {zenml_nightly-0.83.0.dev20250604.dist-info → zenml_nightly-0.83.0.dev20250606.dist-info}/WHEEL +0 -0
- {zenml_nightly-0.83.0.dev20250604.dist-info → zenml_nightly-0.83.0.dev20250606.dist-info}/entry_points.txt +0 -0
zenml/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.83.0.
|
1
|
+
0.83.0.dev20250606
|
zenml/artifacts/utils.py
CHANGED
@@ -37,6 +37,7 @@ from zenml.artifacts.preexisting_data_materializer import (
|
|
37
37
|
)
|
38
38
|
from zenml.client import Client
|
39
39
|
from zenml.constants import (
|
40
|
+
ENV_ZENML_SERVER,
|
40
41
|
MODEL_METADATA_YAML_FILE_NAME,
|
41
42
|
)
|
42
43
|
from zenml.enums import (
|
@@ -820,11 +821,18 @@ def _load_artifact_store(
|
|
820
821
|
an artifact store.
|
821
822
|
NotImplementedError: If the artifact store could not be loaded.
|
822
823
|
"""
|
824
|
+
client = Client()
|
823
825
|
if isinstance(artifact_store_id, str):
|
824
826
|
artifact_store_id = UUID(artifact_store_id)
|
825
827
|
|
828
|
+
if (
|
829
|
+
ENV_ZENML_SERVER not in os.environ
|
830
|
+
and client.active_stack.artifact_store.id == artifact_store_id
|
831
|
+
):
|
832
|
+
return client.active_stack.artifact_store
|
833
|
+
|
826
834
|
if zen_store is None:
|
827
|
-
zen_store =
|
835
|
+
zen_store = client.zen_store
|
828
836
|
|
829
837
|
try:
|
830
838
|
artifact_store_model = zen_store.get_stack_component(artifact_store_id)
|
@@ -857,9 +865,18 @@ def _load_artifact_store(
|
|
857
865
|
def _get_artifact_store_from_response_or_from_active_stack(
|
858
866
|
artifact: ArtifactVersionResponse,
|
859
867
|
) -> "BaseArtifactStore":
|
868
|
+
client = Client()
|
869
|
+
|
860
870
|
if artifact.artifact_store_id:
|
871
|
+
if (
|
872
|
+
ENV_ZENML_SERVER not in os.environ
|
873
|
+
and client.active_stack.artifact_store.id
|
874
|
+
== artifact.artifact_store_id
|
875
|
+
):
|
876
|
+
return client.active_stack.artifact_store
|
877
|
+
|
861
878
|
try:
|
862
|
-
artifact_store_model =
|
879
|
+
artifact_store_model = client.get_stack_component(
|
863
880
|
component_type=StackComponentType.ARTIFACT_STORE,
|
864
881
|
name_id_or_prefix=artifact.artifact_store_id,
|
865
882
|
)
|
@@ -881,7 +898,8 @@ def _get_artifact_store_from_response_or_from_active_stack(
|
|
881
898
|
"the environment that you are loading this artifact from "
|
882
899
|
"has the right dependencies."
|
883
900
|
)
|
884
|
-
|
901
|
+
|
902
|
+
return client.active_stack.artifact_store
|
885
903
|
|
886
904
|
|
887
905
|
def _load_file_from_artifact_store(
|
zenml/client.py
CHANGED
@@ -1480,9 +1480,6 @@ class Client(metaclass=ClientMetaClass):
|
|
1480
1480
|
|
1481
1481
|
Returns:
|
1482
1482
|
The model of the active stack for this client.
|
1483
|
-
|
1484
|
-
Raises:
|
1485
|
-
RuntimeError: If the active stack is not set.
|
1486
1483
|
"""
|
1487
1484
|
if env_stack_id := os.environ.get(ENV_ZENML_ACTIVE_STACK_ID):
|
1488
1485
|
if not self._active_stack or self._active_stack.id != UUID(
|
@@ -1509,12 +1506,6 @@ class Client(metaclass=ClientMetaClass):
|
|
1509
1506
|
|
1510
1507
|
stack_id = GlobalConfiguration().get_active_stack_id()
|
1511
1508
|
|
1512
|
-
if not stack_id:
|
1513
|
-
raise RuntimeError(
|
1514
|
-
"No active stack is configured. Run "
|
1515
|
-
"`zenml stack set STACK_NAME` to set the active stack."
|
1516
|
-
)
|
1517
|
-
|
1518
1509
|
return self.get_stack(stack_id)
|
1519
1510
|
|
1520
1511
|
def activate_stack(
|
@@ -1550,9 +1541,6 @@ class Client(metaclass=ClientMetaClass):
|
|
1550
1541
|
|
1551
1542
|
Args:
|
1552
1543
|
stack: The stack to validate.
|
1553
|
-
|
1554
|
-
Raises:
|
1555
|
-
ValidationError: If the stack configuration is invalid.
|
1556
1544
|
"""
|
1557
1545
|
local_components: List[str] = []
|
1558
1546
|
remote_components: List[str] = []
|
@@ -1617,13 +1605,6 @@ class Client(metaclass=ClientMetaClass):
|
|
1617
1605
|
f"local resources."
|
1618
1606
|
)
|
1619
1607
|
|
1620
|
-
if not stack.is_valid:
|
1621
|
-
raise ValidationError(
|
1622
|
-
"Stack configuration is invalid. A valid"
|
1623
|
-
"stack must contain an Artifact Store and "
|
1624
|
-
"an Orchestrator."
|
1625
|
-
)
|
1626
|
-
|
1627
1608
|
# ----------------------------- Services -----------------------------------
|
1628
1609
|
|
1629
1610
|
def create_service(
|
@@ -20,24 +20,35 @@ from typing import (
|
|
20
20
|
Any,
|
21
21
|
Dict,
|
22
22
|
Iterator,
|
23
|
+
List,
|
23
24
|
Optional,
|
24
25
|
Tuple,
|
25
26
|
Type,
|
27
|
+
Union,
|
26
28
|
cast,
|
27
29
|
)
|
28
30
|
from uuid import UUID
|
29
31
|
|
30
32
|
import boto3
|
31
|
-
import sagemaker
|
32
33
|
from botocore.exceptions import WaiterError
|
34
|
+
from sagemaker.estimator import Estimator
|
35
|
+
from sagemaker.inputs import TrainingInput
|
33
36
|
from sagemaker.network import NetworkConfig
|
34
|
-
from sagemaker.processing import ProcessingInput, ProcessingOutput
|
37
|
+
from sagemaker.processing import ProcessingInput, ProcessingOutput, Processor
|
38
|
+
from sagemaker.session import Session
|
39
|
+
from sagemaker.workflow.entities import PipelineVariable
|
35
40
|
from sagemaker.workflow.execution_variables import (
|
36
|
-
ExecutionVariable,
|
37
41
|
ExecutionVariables,
|
38
42
|
)
|
39
43
|
from sagemaker.workflow.pipeline import Pipeline
|
40
|
-
from sagemaker.workflow.
|
44
|
+
from sagemaker.workflow.step_collections import (
|
45
|
+
StepCollection,
|
46
|
+
)
|
47
|
+
from sagemaker.workflow.steps import (
|
48
|
+
ProcessingStep,
|
49
|
+
Step,
|
50
|
+
TrainingStep,
|
51
|
+
)
|
41
52
|
from sagemaker.workflow.triggers import PipelineSchedule
|
42
53
|
|
43
54
|
from zenml.client import Client
|
@@ -215,7 +226,7 @@ class SagemakerOrchestrator(ContainerizedOrchestrator):
|
|
215
226
|
"""
|
216
227
|
return SagemakerOrchestratorSettings
|
217
228
|
|
218
|
-
def _get_sagemaker_session(self) ->
|
229
|
+
def _get_sagemaker_session(self) -> Session:
|
219
230
|
"""Method to create the sagemaker session with proper authentication.
|
220
231
|
|
221
232
|
Returns:
|
@@ -258,7 +269,7 @@ class SagemakerOrchestrator(ContainerizedOrchestrator):
|
|
258
269
|
aws_session_token=credentials["SessionToken"],
|
259
270
|
region_name=self.config.region,
|
260
271
|
)
|
261
|
-
return
|
272
|
+
return Session(
|
262
273
|
boto_session=boto_session, default_bucket=self.config.bucket
|
263
274
|
)
|
264
275
|
|
@@ -324,12 +335,8 @@ class SagemakerOrchestrator(ContainerizedOrchestrator):
|
|
324
335
|
SagemakerOrchestratorSettings, self.get_settings(step)
|
325
336
|
)
|
326
337
|
|
327
|
-
environment[ENV_ZENML_SAGEMAKER_RUN_ID] = (
|
328
|
-
ExecutionVariables.PIPELINE_EXECUTION_ARN
|
329
|
-
)
|
330
|
-
|
331
338
|
if step_settings.environment:
|
332
|
-
|
339
|
+
split_environment = step_settings.environment.copy()
|
333
340
|
# Sagemaker does not allow environment variables longer than 256
|
334
341
|
# characters to be passed to Processor steps. If an environment variable
|
335
342
|
# is longer than 256 characters, we split it into multiple environment
|
@@ -337,9 +344,9 @@ class SagemakerOrchestrator(ContainerizedOrchestrator):
|
|
337
344
|
# custom entrypoint configuration.
|
338
345
|
split_environment_variables(
|
339
346
|
size_limit=SAGEMAKER_PROCESSOR_STEP_ENV_VAR_SIZE_LIMIT,
|
340
|
-
env=
|
347
|
+
env=split_environment,
|
341
348
|
)
|
342
|
-
environment.update(
|
349
|
+
environment.update(split_environment)
|
343
350
|
|
344
351
|
use_training_step = (
|
345
352
|
step_settings.use_training_step
|
@@ -421,28 +428,51 @@ class SagemakerOrchestrator(ContainerizedOrchestrator):
|
|
421
428
|
)
|
422
429
|
|
423
430
|
# Construct S3 inputs to container for step
|
424
|
-
|
431
|
+
training_inputs: Optional[
|
432
|
+
Union[TrainingInput, Dict[str, TrainingInput]]
|
433
|
+
] = None
|
434
|
+
processing_inputs: Optional[List[ProcessingInput]] = None
|
425
435
|
|
426
436
|
if step_settings.input_data_s3_uri is None:
|
427
437
|
pass
|
428
438
|
elif isinstance(step_settings.input_data_s3_uri, str):
|
429
|
-
|
430
|
-
|
431
|
-
|
432
|
-
|
433
|
-
s3_input_mode=step_settings.input_data_s3_mode,
|
439
|
+
if use_training_step:
|
440
|
+
training_inputs = TrainingInput(
|
441
|
+
s3_data=step_settings.input_data_s3_uri,
|
442
|
+
input_mode=step_settings.input_data_s3_mode,
|
434
443
|
)
|
435
|
-
|
436
|
-
|
437
|
-
inputs = []
|
438
|
-
for channel, s3_uri in step_settings.input_data_s3_uri.items():
|
439
|
-
inputs.append(
|
444
|
+
else:
|
445
|
+
processing_inputs = [
|
440
446
|
ProcessingInput(
|
441
|
-
source=
|
442
|
-
destination=
|
447
|
+
source=step_settings.input_data_s3_uri,
|
448
|
+
destination="/opt/ml/processing/input/data",
|
443
449
|
s3_input_mode=step_settings.input_data_s3_mode,
|
444
450
|
)
|
445
|
-
|
451
|
+
]
|
452
|
+
elif isinstance(step_settings.input_data_s3_uri, dict):
|
453
|
+
if use_training_step:
|
454
|
+
training_inputs = {}
|
455
|
+
for (
|
456
|
+
channel,
|
457
|
+
s3_uri,
|
458
|
+
) in step_settings.input_data_s3_uri.items():
|
459
|
+
training_inputs[channel] = TrainingInput(
|
460
|
+
s3_data=s3_uri,
|
461
|
+
input_mode=step_settings.input_data_s3_mode,
|
462
|
+
)
|
463
|
+
else:
|
464
|
+
processing_inputs = []
|
465
|
+
for (
|
466
|
+
channel,
|
467
|
+
s3_uri,
|
468
|
+
) in step_settings.input_data_s3_uri.items():
|
469
|
+
processing_inputs.append(
|
470
|
+
ProcessingInput(
|
471
|
+
source=s3_uri,
|
472
|
+
destination=f"/opt/ml/processing/input/data/{channel}",
|
473
|
+
s3_input_mode=step_settings.input_data_s3_mode,
|
474
|
+
)
|
475
|
+
)
|
446
476
|
|
447
477
|
# Construct S3 outputs from container for step
|
448
478
|
outputs = None
|
@@ -475,42 +505,56 @@ class SagemakerOrchestrator(ContainerizedOrchestrator):
|
|
475
505
|
)
|
476
506
|
)
|
477
507
|
|
478
|
-
|
479
|
-
environment = {
|
508
|
+
step_environment: Dict[str, Union[str, PipelineVariable]] = {
|
480
509
|
key: str(value)
|
481
|
-
if not isinstance(value,
|
510
|
+
if not isinstance(value, PipelineVariable)
|
482
511
|
else value
|
483
512
|
for key, value in environment.items()
|
484
513
|
}
|
485
514
|
|
515
|
+
step_environment[ENV_ZENML_SAGEMAKER_RUN_ID] = (
|
516
|
+
ExecutionVariables.PIPELINE_EXECUTION_ARN
|
517
|
+
)
|
518
|
+
|
519
|
+
sagemaker_step: Step
|
486
520
|
if use_training_step:
|
487
521
|
# Create Estimator and TrainingStep
|
488
|
-
estimator =
|
522
|
+
estimator = Estimator(
|
489
523
|
keep_alive_period_in_seconds=step_settings.keep_alive_period_in_seconds,
|
490
524
|
output_path=output_path,
|
491
|
-
environment=
|
525
|
+
environment=step_environment,
|
492
526
|
container_entry_point=entrypoint,
|
493
527
|
**args_for_step_executor,
|
494
528
|
)
|
529
|
+
|
495
530
|
sagemaker_step = TrainingStep(
|
496
531
|
name=step_name,
|
497
|
-
depends_on=
|
498
|
-
|
532
|
+
depends_on=cast(
|
533
|
+
Optional[List[Union[str, Step, StepCollection]]],
|
534
|
+
step.spec.upstream_steps,
|
535
|
+
),
|
536
|
+
inputs=training_inputs,
|
499
537
|
estimator=estimator,
|
500
538
|
)
|
501
539
|
else:
|
502
540
|
# Create Processor and ProcessingStep
|
503
|
-
processor =
|
504
|
-
entrypoint=
|
505
|
-
|
541
|
+
processor = Processor(
|
542
|
+
entrypoint=cast(
|
543
|
+
Optional[List[Union[str, PipelineVariable]]],
|
544
|
+
entrypoint,
|
545
|
+
),
|
546
|
+
env=step_environment,
|
506
547
|
**args_for_step_executor,
|
507
548
|
)
|
508
549
|
|
509
550
|
sagemaker_step = ProcessingStep(
|
510
551
|
name=step_name,
|
511
552
|
processor=processor,
|
512
|
-
depends_on=
|
513
|
-
|
553
|
+
depends_on=cast(
|
554
|
+
Optional[List[Union[str, Step, StepCollection]]],
|
555
|
+
step.spec.upstream_steps,
|
556
|
+
),
|
557
|
+
inputs=processing_inputs,
|
514
558
|
outputs=outputs,
|
515
559
|
)
|
516
560
|
|
@@ -13,10 +13,21 @@
|
|
13
13
|
# permissions and limitations under the License.
|
14
14
|
"""Implementation of the Sagemaker Step Operator."""
|
15
15
|
|
16
|
-
from typing import
|
16
|
+
from typing import (
|
17
|
+
TYPE_CHECKING,
|
18
|
+
Dict,
|
19
|
+
List,
|
20
|
+
Optional,
|
21
|
+
Tuple,
|
22
|
+
Type,
|
23
|
+
Union,
|
24
|
+
cast,
|
25
|
+
)
|
17
26
|
|
18
27
|
import boto3
|
19
|
-
import
|
28
|
+
from sagemaker.estimator import Estimator
|
29
|
+
from sagemaker.inputs import TrainingInput
|
30
|
+
from sagemaker.session import Session
|
20
31
|
|
21
32
|
from zenml.client import Client
|
22
33
|
from zenml.config.build_configuration import BuildConfiguration
|
@@ -216,7 +227,7 @@ class SagemakerStepOperator(BaseStepOperator):
|
|
216
227
|
else:
|
217
228
|
boto_session = boto3.Session()
|
218
229
|
|
219
|
-
session =
|
230
|
+
session = Session(
|
220
231
|
boto_session=boto_session, default_bucket=self.config.bucket
|
221
232
|
)
|
222
233
|
|
@@ -232,9 +243,7 @@ class SagemakerStepOperator(BaseStepOperator):
|
|
232
243
|
estimator_args["sagemaker_session"] = session
|
233
244
|
|
234
245
|
# Create Estimator
|
235
|
-
estimator =
|
236
|
-
image_name, self.config.role, **estimator_args
|
237
|
-
)
|
246
|
+
estimator = Estimator(image_name, self.config.role, **estimator_args)
|
238
247
|
|
239
248
|
# SageMaker allows 63 characters at maximum for job name - ZenML uses 60 for safety margin.
|
240
249
|
step_name = Client().get_run_step(info.step_run_id).name
|
@@ -248,18 +257,14 @@ class SagemakerStepOperator(BaseStepOperator):
|
|
248
257
|
)
|
249
258
|
|
250
259
|
# Construct training input object, if necessary
|
251
|
-
inputs = None
|
260
|
+
inputs: Optional[Union[TrainingInput, Dict[str, TrainingInput]]] = None
|
252
261
|
|
253
262
|
if isinstance(settings.input_data_s3_uri, str):
|
254
|
-
inputs =
|
255
|
-
s3_data=settings.input_data_s3_uri
|
256
|
-
)
|
263
|
+
inputs = TrainingInput(s3_data=settings.input_data_s3_uri)
|
257
264
|
elif isinstance(settings.input_data_s3_uri, dict):
|
258
265
|
inputs = {}
|
259
266
|
for channel, s3_uri in settings.input_data_s3_uri.items():
|
260
|
-
inputs[channel] =
|
261
|
-
s3_data=s3_uri
|
262
|
-
)
|
267
|
+
inputs[channel] = TrainingInput(s3_data=s3_uri)
|
263
268
|
|
264
269
|
experiment_config = {}
|
265
270
|
if settings.experiment_name:
|
zenml/models/v2/core/stack.py
CHANGED
@@ -27,7 +27,7 @@ from typing import (
|
|
27
27
|
)
|
28
28
|
from uuid import UUID
|
29
29
|
|
30
|
-
from pydantic import Field, model_validator
|
30
|
+
from pydantic import Field, field_validator, model_validator
|
31
31
|
from sqlmodel import and_
|
32
32
|
|
33
33
|
from zenml.constants import STR_FIELD_MAX_LENGTH
|
@@ -94,18 +94,31 @@ class StackRequest(UserScopedRequest):
|
|
94
94
|
"scratch.",
|
95
95
|
)
|
96
96
|
|
97
|
-
@
|
98
|
-
def
|
99
|
-
|
97
|
+
@field_validator("components")
|
98
|
+
def _validate_components(
|
99
|
+
cls, value: Dict[StackComponentType, List[Union[UUID, ComponentInfo]]]
|
100
|
+
) -> Dict[StackComponentType, List[Union[UUID, ComponentInfo]]]:
|
101
|
+
"""Validate the components of the stack.
|
102
|
+
|
103
|
+
Args:
|
104
|
+
value: The components of the stack.
|
105
|
+
|
106
|
+
Raises:
|
107
|
+
ValueError: If the stack does not contain an orchestrator and
|
108
|
+
artifact store.
|
100
109
|
|
101
110
|
Returns:
|
102
|
-
|
111
|
+
The components of the stack.
|
103
112
|
"""
|
104
|
-
if
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
113
|
+
if value:
|
114
|
+
artifact_stores = value.get(StackComponentType.ARTIFACT_STORE, [])
|
115
|
+
orchestrators = value.get(StackComponentType.ORCHESTRATOR, [])
|
116
|
+
|
117
|
+
if orchestrators and artifact_stores:
|
118
|
+
return value
|
119
|
+
|
120
|
+
raise ValueError(
|
121
|
+
"Stack must contain at least an orchestrator and artifact store."
|
109
122
|
)
|
110
123
|
|
111
124
|
@model_validator(mode="after")
|
@@ -163,6 +176,39 @@ class StackUpdate(BaseUpdate):
|
|
163
176
|
title="The stack labels.",
|
164
177
|
)
|
165
178
|
|
179
|
+
@field_validator("components")
|
180
|
+
def _validate_components(
|
181
|
+
cls,
|
182
|
+
value: Optional[
|
183
|
+
Dict[StackComponentType, List[Union[UUID, ComponentInfo]]]
|
184
|
+
],
|
185
|
+
) -> Optional[Dict[StackComponentType, List[Union[UUID, ComponentInfo]]]]:
|
186
|
+
"""Validate the components of the stack.
|
187
|
+
|
188
|
+
Args:
|
189
|
+
value: The components of the stack.
|
190
|
+
|
191
|
+
Raises:
|
192
|
+
ValueError: If the stack does not contain an orchestrator and
|
193
|
+
artifact store.
|
194
|
+
|
195
|
+
Returns:
|
196
|
+
The components of the stack.
|
197
|
+
"""
|
198
|
+
if value is None:
|
199
|
+
return None
|
200
|
+
|
201
|
+
if value:
|
202
|
+
artifact_stores = value.get(StackComponentType.ARTIFACT_STORE, [])
|
203
|
+
orchestrators = value.get(StackComponentType.ORCHESTRATOR, [])
|
204
|
+
|
205
|
+
if orchestrators and artifact_stores:
|
206
|
+
return value
|
207
|
+
|
208
|
+
raise ValueError(
|
209
|
+
"Stack must contain at least an orchestrator and artifact store."
|
210
|
+
)
|
211
|
+
|
166
212
|
|
167
213
|
# ------------------ Response Model ------------------
|
168
214
|
|
@@ -8925,6 +8925,7 @@ class SqlZenStore(BaseZenStore):
|
|
8925
8925
|
|
8926
8926
|
pipeline_run = session.exec(
|
8927
8927
|
select(PipelineRunSchema)
|
8928
|
+
.with_for_update()
|
8928
8929
|
.options(
|
8929
8930
|
joinedload(
|
8930
8931
|
jl_arg(PipelineRunSchema.deployment), innerjoin=True
|
@@ -8950,96 +8951,100 @@ class SqlZenStore(BaseZenStore):
|
|
8950
8951
|
num_steps=num_steps,
|
8951
8952
|
)
|
8952
8953
|
|
8953
|
-
if pipeline_run.
|
8954
|
-
|
8955
|
-
|
8954
|
+
if new_status == pipeline_run.status or (
|
8955
|
+
pipeline_run.is_placeholder_run() and not new_status.is_finished
|
8956
|
+
):
|
8957
|
+
# The status hasn't changed -> no need to update the status.
|
8958
|
+
# If the pipeline run is a placeholder run (=no step has been started
|
8959
|
+
# for the run yet), this means the orchestrator hasn't started
|
8956
8960
|
# running yet, and this method is most likely being called as
|
8957
8961
|
# part of the creation of some cached steps. In this case, we don't
|
8958
8962
|
# update the status unless the run is finished.
|
8963
|
+
|
8964
|
+
# Commit so that we release the lock on the pipeline run.
|
8965
|
+
session.commit()
|
8959
8966
|
return
|
8960
8967
|
|
8961
|
-
|
8962
|
-
|
8963
|
-
|
8964
|
-
|
8965
|
-
|
8966
|
-
|
8967
|
-
|
8968
|
-
|
8969
|
-
|
8970
|
-
|
8971
|
-
|
8972
|
-
|
8973
|
-
|
8974
|
-
|
8975
|
-
|
8968
|
+
run_update = PipelineRunUpdate(status=new_status)
|
8969
|
+
if new_status.is_finished:
|
8970
|
+
run_update.end_time = utc_now()
|
8971
|
+
|
8972
|
+
pipeline_run.update(run_update)
|
8973
|
+
session.add(pipeline_run)
|
8974
|
+
# Commit so that we release the lock on the pipeline run.
|
8975
|
+
session.commit()
|
8976
|
+
|
8977
|
+
if new_status.is_finished:
|
8978
|
+
assert run_update.end_time
|
8979
|
+
if pipeline_run.start_time:
|
8980
|
+
duration_time = run_update.end_time - pipeline_run.start_time
|
8981
|
+
duration_seconds = duration_time.total_seconds()
|
8982
|
+
start_time_str = pipeline_run.start_time.strftime(
|
8983
|
+
"%Y-%m-%dT%H:%M:%S.%fZ"
|
8984
|
+
)
|
8985
|
+
else:
|
8986
|
+
start_time_str = None
|
8987
|
+
duration_seconds = None
|
8988
|
+
|
8989
|
+
stack = pipeline_run.deployment.stack
|
8990
|
+
assert stack
|
8991
|
+
stack_metadata = {
|
8992
|
+
str(component.type): component.flavor
|
8993
|
+
for component in stack.components
|
8994
|
+
}
|
8995
|
+
with track_handler(
|
8996
|
+
AnalyticsEvent.RUN_PIPELINE_ENDED
|
8997
|
+
) as analytics_handler:
|
8998
|
+
analytics_handler.metadata = {
|
8999
|
+
"project_id": pipeline_run.project_id,
|
9000
|
+
"pipeline_run_id": pipeline_run_id,
|
9001
|
+
"template_id": pipeline_run.deployment.template_id,
|
9002
|
+
"status": new_status,
|
9003
|
+
"num_steps": num_steps,
|
9004
|
+
"start_time": start_time_str,
|
9005
|
+
"end_time": run_update.end_time.strftime(
|
8976
9006
|
"%Y-%m-%dT%H:%M:%S.%fZ"
|
8977
|
-
)
|
8978
|
-
|
8979
|
-
|
8980
|
-
duration_seconds = None
|
8981
|
-
|
8982
|
-
stack = pipeline_run.deployment.stack
|
8983
|
-
assert stack
|
8984
|
-
stack_metadata = {
|
8985
|
-
str(component.type): component.flavor
|
8986
|
-
for component in stack.components
|
9007
|
+
),
|
9008
|
+
"duration_seconds": duration_seconds,
|
9009
|
+
**stack_metadata,
|
8987
9010
|
}
|
8988
|
-
with track_handler(
|
8989
|
-
AnalyticsEvent.RUN_PIPELINE_ENDED
|
8990
|
-
) as analytics_handler:
|
8991
|
-
analytics_handler.metadata = {
|
8992
|
-
"project_id": pipeline_run.project_id,
|
8993
|
-
"pipeline_run_id": pipeline_run_id,
|
8994
|
-
"template_id": pipeline_run.deployment.template_id,
|
8995
|
-
"status": new_status,
|
8996
|
-
"num_steps": num_steps,
|
8997
|
-
"start_time": start_time_str,
|
8998
|
-
"end_time": run_update.end_time.strftime(
|
8999
|
-
"%Y-%m-%dT%H:%M:%S.%fZ"
|
9000
|
-
),
|
9001
|
-
"duration_seconds": duration_seconds,
|
9002
|
-
**stack_metadata,
|
9003
|
-
}
|
9004
9011
|
|
9005
|
-
|
9006
|
-
|
9007
|
-
|
9008
|
-
|
9009
|
-
|
9010
|
-
|
9011
|
-
|
9012
|
-
|
9013
|
-
|
9014
|
-
|
9015
|
-
|
9016
|
-
|
9012
|
+
completed_onboarding_steps: Set[str] = {
|
9013
|
+
OnboardingStep.PIPELINE_RUN,
|
9014
|
+
OnboardingStep.STARTER_SETUP_COMPLETED,
|
9015
|
+
}
|
9016
|
+
if stack_metadata["orchestrator"] not in {
|
9017
|
+
"local",
|
9018
|
+
"local_docker",
|
9019
|
+
}:
|
9020
|
+
completed_onboarding_steps.update(
|
9021
|
+
{
|
9022
|
+
OnboardingStep.PIPELINE_RUN_WITH_REMOTE_ORCHESTRATOR,
|
9023
|
+
}
|
9024
|
+
)
|
9025
|
+
if stack_metadata["artifact_store"] != "local":
|
9026
|
+
completed_onboarding_steps.update(
|
9027
|
+
{
|
9028
|
+
OnboardingStep.PIPELINE_RUN_WITH_REMOTE_ARTIFACT_STORE,
|
9029
|
+
OnboardingStep.PRODUCTION_SETUP_COMPLETED,
|
9030
|
+
}
|
9031
|
+
)
|
9032
|
+
if OnboardingStep.THIRD_PIPELINE_RUN not in (
|
9033
|
+
self._cached_onboarding_state or {}
|
9034
|
+
):
|
9035
|
+
onboarding_state = self.get_onboarding_state()
|
9036
|
+
if OnboardingStep.PIPELINE_RUN in onboarding_state:
|
9037
|
+
completed_onboarding_steps.add(
|
9038
|
+
OnboardingStep.SECOND_PIPELINE_RUN
|
9017
9039
|
)
|
9018
|
-
if
|
9019
|
-
completed_onboarding_steps.
|
9020
|
-
|
9021
|
-
OnboardingStep.PIPELINE_RUN_WITH_REMOTE_ARTIFACT_STORE,
|
9022
|
-
OnboardingStep.PRODUCTION_SETUP_COMPLETED,
|
9023
|
-
}
|
9040
|
+
if OnboardingStep.SECOND_PIPELINE_RUN in onboarding_state:
|
9041
|
+
completed_onboarding_steps.add(
|
9042
|
+
OnboardingStep.THIRD_PIPELINE_RUN
|
9024
9043
|
)
|
9025
|
-
if OnboardingStep.THIRD_PIPELINE_RUN not in (
|
9026
|
-
self._cached_onboarding_state or {}
|
9027
|
-
):
|
9028
|
-
onboarding_state = self.get_onboarding_state()
|
9029
|
-
if OnboardingStep.PIPELINE_RUN in onboarding_state:
|
9030
|
-
completed_onboarding_steps.add(
|
9031
|
-
OnboardingStep.SECOND_PIPELINE_RUN
|
9032
|
-
)
|
9033
|
-
if OnboardingStep.SECOND_PIPELINE_RUN in onboarding_state:
|
9034
|
-
completed_onboarding_steps.add(
|
9035
|
-
OnboardingStep.THIRD_PIPELINE_RUN
|
9036
|
-
)
|
9037
9044
|
|
9038
|
-
|
9039
|
-
|
9040
|
-
|
9041
|
-
pipeline_run.update(run_update)
|
9042
|
-
session.add(pipeline_run)
|
9045
|
+
self._update_onboarding_state(
|
9046
|
+
completed_steps=completed_onboarding_steps, session=session
|
9047
|
+
)
|
9043
9048
|
|
9044
9049
|
# --------------------------- Triggers ---------------------------
|
9045
9050
|
|
{zenml_nightly-0.83.0.dev20250604.dist-info → zenml_nightly-0.83.0.dev20250606.dist-info}/RECORD
RENAMED
@@ -1,5 +1,5 @@
|
|
1
1
|
zenml/README.md,sha256=827dekbOWAs1BpW7VF1a4d7EbwPbjwccX-2zdXBENZo,1777
|
2
|
-
zenml/VERSION,sha256=
|
2
|
+
zenml/VERSION,sha256=dOlZWzCTAp-20wgwKM0CFZujsv4yk51NATYHpGjvRRw,19
|
3
3
|
zenml/__init__.py,sha256=CKEyepFK-7akXYiMrNVh92Nb01Cjs23w4_YyI6sgdc8,2242
|
4
4
|
zenml/actions/__init__.py,sha256=mrt6wPo73iKRxK754_NqsGyJ3buW7RnVeIGXr1xEw8Y,681
|
5
5
|
zenml/actions/base_action.py,sha256=UcaHev6BTuLDwuswnyaPjdA8AgUqB5xPZ-lRtuvf2FU,25553
|
@@ -25,7 +25,7 @@ zenml/artifacts/external_artifact.py,sha256=7nLANV0vsGC36H1s_B_awX4hnZgXHCGIscQ2
|
|
25
25
|
zenml/artifacts/external_artifact_config.py,sha256=P172p0JOu8Xx1F8RCQut1dnRpt5lpWXGNqMbY-V90sI,3323
|
26
26
|
zenml/artifacts/preexisting_data_materializer.py,sha256=dcahDcHUD3Lvn0-6zE2BG84bkyo_ydAgzBWxtbyJJZQ,3325
|
27
27
|
zenml/artifacts/unmaterialized_artifact.py,sha256=JNPKq_sNifQx5wP8jEw7TGBEi26zwKirPGlWX9uxbJI,1300
|
28
|
-
zenml/artifacts/utils.py,sha256=
|
28
|
+
zenml/artifacts/utils.py,sha256=44wDliyVDmfNQSRpdUwI4JhhsmGP18FF4mWgyEmN0E8,35933
|
29
29
|
zenml/cli/__init__.py,sha256=nxq4ifwLV5qT7Qghb42h02XUOcOihBkZp2xBqgiykM8,75670
|
30
30
|
zenml/cli/annotator.py,sha256=JRR7_TJOWKyiKGv1kwSjG1Ay6RBWPVgm0X-D0uSBlyE,6976
|
31
31
|
zenml/cli/artifact.py,sha256=7lsAS52DroBTFkFWxkyb-lIDOGP5jPL_Se_RDG_2jgg,9564
|
@@ -55,7 +55,7 @@ zenml/cli/text_utils.py,sha256=bY1GIjoULt1cW2FyrPlMoAXNS2R7cSOjDFEZQqrpVQ8,3553
|
|
55
55
|
zenml/cli/user_management.py,sha256=sNnhaUxH-cHecbZBR1L0mEU0TnLNZHzI6ZBCUSQa7OY,13078
|
56
56
|
zenml/cli/utils.py,sha256=vMAb9f6GDfNVGmZWOz9UOyPRpKI3KfnYpRl_w9YUBNE,86501
|
57
57
|
zenml/cli/version.py,sha256=nm1iSU_1V6-MUwpMKeXcwFhLYGUMLswvQL67cEuCpxA,3635
|
58
|
-
zenml/client.py,sha256=
|
58
|
+
zenml/client.py,sha256=cf4sYrVCx9xbX3ZcQKF2j-jRroCxy-MKUNkEOhZD1yk,293336
|
59
59
|
zenml/client_lazy_loader.py,sha256=oyxKvBWVB7k2pHMavdhNEOfR2Vk4IS3XUu43SBzDPsI,7152
|
60
60
|
zenml/code_repositories/__init__.py,sha256=W5bDfzAG8OXIKZSV1L-VHuzMcSCYa9qzTdPb3jqfyYw,920
|
61
61
|
zenml/code_repositories/base_code_repository.py,sha256=Id6VjbUu8N3ZpNvBGhOgbahtoMiCAtYXed3G7YQ_iAc,5225
|
@@ -146,12 +146,12 @@ zenml/integrations/aws/flavors/sagemaker_step_operator_flavor.py,sha256=e3locb2O
|
|
146
146
|
zenml/integrations/aws/image_builders/__init__.py,sha256=91hgH1OphG2i-vk-G8N4yKBFIzK89Wu7BK4-T5yOA7E,786
|
147
147
|
zenml/integrations/aws/image_builders/aws_image_builder.py,sha256=UcPYYYjJjfsicY3hV4OZeJt552AVmwPZPlv-AsG1g1I,11489
|
148
148
|
zenml/integrations/aws/orchestrators/__init__.py,sha256=Wh0Fhtt_uo6YrkvXY9kL0M478FL7XpapjoFreUZbgUg,794
|
149
|
-
zenml/integrations/aws/orchestrators/sagemaker_orchestrator.py,sha256=
|
149
|
+
zenml/integrations/aws/orchestrators/sagemaker_orchestrator.py,sha256=EfDFT2WG0t-6x1CU0n57BS7YCC0wPA8CfTGpb4-G0Tc,39282
|
150
150
|
zenml/integrations/aws/orchestrators/sagemaker_orchestrator_entrypoint_config.py,sha256=WXCWdVojIZxx5_3-g1T95S2vsJ-RLNGcp-V409wgme0,1555
|
151
151
|
zenml/integrations/aws/service_connectors/__init__.py,sha256=w2Md40yG89PwmU9eBceh6dGy3XYZ3MKusNAZ51sGOgE,783
|
152
152
|
zenml/integrations/aws/service_connectors/aws_service_connector.py,sha256=7H69IoOYmyn5QcXEfL1-OmC0UaQ54TfNNhv2t8A6Daw,92107
|
153
153
|
zenml/integrations/aws/step_operators/__init__.py,sha256=HK5oIqLixYKUKoiN4LnqTyGjAZJUdUqWYGqJhFSWyo0,817
|
154
|
-
zenml/integrations/aws/step_operators/sagemaker_step_operator.py,sha256=
|
154
|
+
zenml/integrations/aws/step_operators/sagemaker_step_operator.py,sha256=1RCHSzHcgTldChlk4c8iCrKTeeHrdQlQUJpMepk9ShE,10384
|
155
155
|
zenml/integrations/aws/step_operators/sagemaker_step_operator_entrypoint_config.py,sha256=2cXroe6-bXyHVkO5yPnZagNpqx6MrMDcvuvNr8J2j-A,1581
|
156
156
|
zenml/integrations/azure/__init__.py,sha256=iJ8smVn76fIuOeeJV5txrBfBS0PyRKUNt9aVkL3Ucic,3103
|
157
157
|
zenml/integrations/azure/artifact_stores/__init__.py,sha256=dlIwbpgjE0Hy4rhMbelNJHVKm4t8tj_hRu9mQ_cEIAg,820
|
@@ -658,7 +658,7 @@ zenml/models/v2/core/server_settings.py,sha256=NNNsYM2AwfsPmA2kfTMwqxn0__9WDoii5
|
|
658
658
|
zenml/models/v2/core/service.py,sha256=PeI036PIVG0zX5EiYPsx7h5LTRC8hlmfdeClKff-IaU,16106
|
659
659
|
zenml/models/v2/core/service_account.py,sha256=-1c9Et9Ma4_OHOZxIHjVnLJAaLLxhpoLEyGKkwc1Yx8,6619
|
660
660
|
zenml/models/v2/core/service_connector.py,sha256=5JBRqd3iL0SFnupUO8fSgbsAZY9u2FDkMaY44azV5FQ,37048
|
661
|
-
zenml/models/v2/core/stack.py,sha256=
|
661
|
+
zenml/models/v2/core/stack.py,sha256=3IKofEaKprAtYNbIrKvcYs5IABTbCqxcfJSXC3Q0fYM,13332
|
662
662
|
zenml/models/v2/core/step_run.py,sha256=NLA9cGk76_nGfxhRuXeEJFv4LzWiIl70AtSsVkJ6NCU,21020
|
663
663
|
zenml/models/v2/core/tag.py,sha256=60hCZuHAHy7lsYFkTDw7hK0zZz2eG7jx8l9VY7Hh-LQ,7100
|
664
664
|
zenml/models/v2/core/tag_resource.py,sha256=H7CPj9oaUpym8vSkwuYSJ6_rWfHMEKVRedPw2kyxmII,2640
|
@@ -1324,11 +1324,11 @@ zenml/zen_stores/secrets_stores/hashicorp_secrets_store.py,sha256=5err1a-TrV3SR5
|
|
1324
1324
|
zenml/zen_stores/secrets_stores/secrets_store_interface.py,sha256=Q2Jbnt2Pp7NGlR-u1YBfRZV2g8su2Fd0ArBMdksAE-Q,2819
|
1325
1325
|
zenml/zen_stores/secrets_stores/service_connector_secrets_store.py,sha256=S87ne23D08PAwtfRVlVnBn8R0ilTpEh6r8blauNV5WQ,6941
|
1326
1326
|
zenml/zen_stores/secrets_stores/sql_secrets_store.py,sha256=LPFW757WCJLP1S8vrvjsrl2Tf1yo281xUTjSBsos4qk,8788
|
1327
|
-
zenml/zen_stores/sql_zen_store.py,sha256=
|
1327
|
+
zenml/zen_stores/sql_zen_store.py,sha256=zgdaIUTG2JYTr6_sW_yhYar_ZgPQZiKWEO3rmM0ys1Y,466071
|
1328
1328
|
zenml/zen_stores/template_utils.py,sha256=GbJ7LgGVYHSCKPEA8RNTxPoVTWqpC77F_lGzjJ4O1Fw,9220
|
1329
1329
|
zenml/zen_stores/zen_store_interface.py,sha256=_ap55L3_mrHgegsLkMRSmmNXVasYC53LwjcEeuS1YT4,92411
|
1330
|
-
zenml_nightly-0.83.0.
|
1331
|
-
zenml_nightly-0.83.0.
|
1332
|
-
zenml_nightly-0.83.0.
|
1333
|
-
zenml_nightly-0.83.0.
|
1334
|
-
zenml_nightly-0.83.0.
|
1330
|
+
zenml_nightly-0.83.0.dev20250606.dist-info/LICENSE,sha256=wbnfEnXnafPbqwANHkV6LUsPKOtdpsd-SNw37rogLtc,11359
|
1331
|
+
zenml_nightly-0.83.0.dev20250606.dist-info/METADATA,sha256=GcnWjjJZNPcjOoYHllarFKXh6zn0RmaoyeAeGwyS9xY,24317
|
1332
|
+
zenml_nightly-0.83.0.dev20250606.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
|
1333
|
+
zenml_nightly-0.83.0.dev20250606.dist-info/entry_points.txt,sha256=QK3ETQE0YswAM2mWypNMOv8TLtr7EjnqAFq1br_jEFE,43
|
1334
|
+
zenml_nightly-0.83.0.dev20250606.dist-info/RECORD,,
|
{zenml_nightly-0.83.0.dev20250604.dist-info → zenml_nightly-0.83.0.dev20250606.dist-info}/LICENSE
RENAMED
File without changes
|
{zenml_nightly-0.83.0.dev20250604.dist-info → zenml_nightly-0.83.0.dev20250606.dist-info}/WHEEL
RENAMED
File without changes
|
File without changes
|