zenml-nightly 0.80.2.dev20250414__py3-none-any.whl → 0.80.2.dev20250416__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- zenml/VERSION +1 -1
- zenml/artifacts/utils.py +7 -2
- zenml/cli/utils.py +13 -11
- zenml/config/compiler.py +1 -0
- zenml/config/global_config.py +1 -1
- zenml/config/pipeline_configurations.py +1 -0
- zenml/config/pipeline_run_configuration.py +1 -0
- zenml/config/server_config.py +7 -0
- zenml/constants.py +8 -0
- zenml/integrations/gcp/orchestrators/vertex_orchestrator.py +47 -5
- zenml/integrations/gcp/vertex_custom_job_parameters.py +15 -1
- zenml/integrations/kubernetes/flavors/kubernetes_step_operator_flavor.py +12 -0
- zenml/integrations/kubernetes/orchestrators/kube_utils.py +92 -0
- zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator.py +12 -3
- zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator_entrypoint.py +11 -65
- zenml/integrations/kubernetes/step_operators/kubernetes_step_operator.py +11 -3
- zenml/logging/step_logging.py +41 -21
- zenml/login/credentials_store.py +31 -0
- zenml/materializers/path_materializer.py +17 -2
- zenml/models/v2/base/base.py +8 -4
- zenml/models/v2/base/filter.py +1 -1
- zenml/models/v2/core/pipeline_run.py +19 -0
- zenml/orchestrators/step_launcher.py +2 -3
- zenml/orchestrators/step_runner.py +2 -2
- zenml/orchestrators/utils.py +2 -5
- zenml/pipelines/pipeline_context.py +1 -0
- zenml/pipelines/pipeline_decorator.py +4 -0
- zenml/pipelines/pipeline_definition.py +83 -22
- zenml/pipelines/run_utils.py +4 -0
- zenml/steps/utils.py +1 -1
- zenml/utils/io_utils.py +23 -0
- zenml/zen_server/auth.py +96 -64
- zenml/zen_server/cloud_utils.py +7 -1
- zenml/zen_server/download_utils.py +123 -0
- zenml/zen_server/jwt.py +0 -14
- zenml/zen_server/rbac/rbac_interface.py +10 -3
- zenml/zen_server/rbac/utils.py +13 -3
- zenml/zen_server/rbac/zenml_cloud_rbac.py +14 -8
- zenml/zen_server/routers/artifact_version_endpoints.py +86 -3
- zenml/zen_server/routers/auth_endpoints.py +5 -36
- zenml/zen_server/routers/pipeline_deployments_endpoints.py +63 -26
- zenml/zen_server/routers/runs_endpoints.py +57 -0
- zenml/zen_server/routers/users_endpoints.py +13 -8
- zenml/zen_server/template_execution/utils.py +3 -3
- zenml/zen_stores/migrations/versions/ff538a321a92_migrate_onboarding_state.py +123 -0
- zenml/zen_stores/rest_zen_store.py +16 -13
- zenml/zen_stores/schemas/pipeline_run_schemas.py +1 -0
- zenml/zen_stores/schemas/server_settings_schemas.py +4 -1
- zenml/zen_stores/sql_zen_store.py +18 -0
- {zenml_nightly-0.80.2.dev20250414.dist-info → zenml_nightly-0.80.2.dev20250416.dist-info}/METADATA +2 -1
- {zenml_nightly-0.80.2.dev20250414.dist-info → zenml_nightly-0.80.2.dev20250416.dist-info}/RECORD +54 -52
- {zenml_nightly-0.80.2.dev20250414.dist-info → zenml_nightly-0.80.2.dev20250416.dist-info}/LICENSE +0 -0
- {zenml_nightly-0.80.2.dev20250414.dist-info → zenml_nightly-0.80.2.dev20250416.dist-info}/WHEEL +0 -0
- {zenml_nightly-0.80.2.dev20250414.dist-info → zenml_nightly-0.80.2.dev20250416.dist-info}/entry_points.txt +0 -0
zenml/logging/step_logging.py
CHANGED
@@ -13,6 +13,7 @@
|
|
13
13
|
# permissions and limitations under the License.
|
14
14
|
"""ZenML logging handler."""
|
15
15
|
|
16
|
+
import logging
|
16
17
|
import os
|
17
18
|
import re
|
18
19
|
import sys
|
@@ -48,6 +49,7 @@ logger = get_logger(__name__)
|
|
48
49
|
redirected: ContextVar[bool] = ContextVar("redirected", default=False)
|
49
50
|
|
50
51
|
LOGS_EXTENSION = ".log"
|
52
|
+
PIPELINE_RUN_LOGS_FOLDER = "pipeline_runs"
|
51
53
|
|
52
54
|
|
53
55
|
def remove_ansi_escape_codes(text: str) -> str:
|
@@ -65,14 +67,14 @@ def remove_ansi_escape_codes(text: str) -> str:
|
|
65
67
|
|
66
68
|
def prepare_logs_uri(
|
67
69
|
artifact_store: "BaseArtifactStore",
|
68
|
-
step_name: str,
|
70
|
+
step_name: Optional[str] = None,
|
69
71
|
log_key: Optional[str] = None,
|
70
72
|
) -> str:
|
71
73
|
"""Generates and prepares a URI for the log file or folder for a step.
|
72
74
|
|
73
75
|
Args:
|
74
76
|
artifact_store: The artifact store on which the artifact will be stored.
|
75
|
-
step_name: Name of the step.
|
77
|
+
step_name: Name of the step. Skipped for global pipeline run logs.
|
76
78
|
log_key: The unique identification key of the log file.
|
77
79
|
|
78
80
|
Returns:
|
@@ -81,11 +83,8 @@ def prepare_logs_uri(
|
|
81
83
|
if log_key is None:
|
82
84
|
log_key = str(uuid4())
|
83
85
|
|
84
|
-
|
85
|
-
|
86
|
-
step_name,
|
87
|
-
"logs",
|
88
|
-
)
|
86
|
+
subfolder = step_name or PIPELINE_RUN_LOGS_FOLDER
|
87
|
+
logs_base_uri = os.path.join(artifact_store.path, subfolder, "logs")
|
89
88
|
|
90
89
|
# Create the dir
|
91
90
|
if not artifact_store.exists(logs_base_uri):
|
@@ -210,7 +209,7 @@ def fetch_logs(
|
|
210
209
|
artifact_store.cleanup()
|
211
210
|
|
212
211
|
|
213
|
-
class
|
212
|
+
class PipelineLogsStorage:
|
214
213
|
"""Helper class which buffers and stores logs to a given URI."""
|
215
214
|
|
216
215
|
def __init__(
|
@@ -324,6 +323,18 @@ class StepLogsStorage:
|
|
324
323
|
self.disabled = True
|
325
324
|
|
326
325
|
try:
|
326
|
+
# The configured logging handler uses a lock to ensure that
|
327
|
+
# logs generated by different threads are not interleaved.
|
328
|
+
# Given that most artifact stores are based on fsspec, which
|
329
|
+
# use a separate thread for async operations, it may happen that
|
330
|
+
# the fsspec library itself will log something, which will end
|
331
|
+
# up in a deadlock.
|
332
|
+
# To avoid this, we temporarily disable the lock in the logging
|
333
|
+
# handler while writing to the file.
|
334
|
+
logging_handler = logging.getLogger().handlers[0]
|
335
|
+
logging_lock = logging_handler.lock
|
336
|
+
logging_handler.lock = None
|
337
|
+
|
327
338
|
if self.buffer:
|
328
339
|
if self.artifact_store.config.IS_IMMUTABLE_FILESYSTEM:
|
329
340
|
_logs_uri = self._get_timestamped_filename()
|
@@ -353,6 +364,9 @@ class StepLogsStorage:
|
|
353
364
|
# I/O errors.
|
354
365
|
logger.error(f"Error while trying to write logs: {e}")
|
355
366
|
finally:
|
367
|
+
# Restore the original logging handler lock
|
368
|
+
logging_handler.lock = logging_lock
|
369
|
+
|
356
370
|
self.buffer = []
|
357
371
|
self.last_save_time = time.time()
|
358
372
|
|
@@ -418,27 +432,32 @@ class StepLogsStorage:
|
|
418
432
|
)
|
419
433
|
|
420
434
|
|
421
|
-
class
|
422
|
-
"""Context manager which patches stdout and stderr during
|
435
|
+
class PipelineLogsStorageContext:
|
436
|
+
"""Context manager which patches stdout and stderr during pipeline run execution."""
|
423
437
|
|
424
438
|
def __init__(
|
425
|
-
self,
|
439
|
+
self,
|
440
|
+
logs_uri: str,
|
441
|
+
artifact_store: "BaseArtifactStore",
|
442
|
+
prepend_step_name: bool = True,
|
426
443
|
) -> None:
|
427
444
|
"""Initializes and prepares a storage object.
|
428
445
|
|
429
446
|
Args:
|
430
447
|
logs_uri: the URI of the logs file.
|
431
|
-
artifact_store: Artifact Store from the current
|
448
|
+
artifact_store: Artifact Store from the current pipeline run context.
|
449
|
+
prepend_step_name: Whether to prepend the step name to the logs.
|
432
450
|
"""
|
433
|
-
self.storage =
|
451
|
+
self.storage = PipelineLogsStorage(
|
434
452
|
logs_uri=logs_uri, artifact_store=artifact_store
|
435
453
|
)
|
454
|
+
self.prepend_step_name = prepend_step_name
|
436
455
|
|
437
|
-
def __enter__(self) -> "
|
456
|
+
def __enter__(self) -> "PipelineLogsStorageContext":
|
438
457
|
"""Enter condition of the context manager.
|
439
458
|
|
440
459
|
Wraps the `write` method of both stderr and stdout, so each incoming
|
441
|
-
message gets stored in the
|
460
|
+
message gets stored in the pipeline logs storage.
|
442
461
|
|
443
462
|
Returns:
|
444
463
|
self
|
@@ -499,14 +518,17 @@ class StepLogsStorageContext:
|
|
499
518
|
"""
|
500
519
|
|
501
520
|
def wrapped_write(*args: Any, **kwargs: Any) -> Any:
|
502
|
-
|
503
|
-
|
504
|
-
|
521
|
+
step_names_disabled = (
|
522
|
+
handle_bool_env_var(
|
523
|
+
ENV_ZENML_DISABLE_STEP_NAMES_IN_LOGS, default=False
|
524
|
+
)
|
525
|
+
or not self.prepend_step_name
|
505
526
|
)
|
506
527
|
|
507
528
|
if step_names_disabled:
|
508
529
|
output = method(*args, **kwargs)
|
509
530
|
else:
|
531
|
+
message = args[0]
|
510
532
|
# Try to get step context if not available yet
|
511
533
|
step_context = None
|
512
534
|
try:
|
@@ -515,9 +537,7 @@ class StepLogsStorageContext:
|
|
515
537
|
pass
|
516
538
|
|
517
539
|
if step_context and args[0] != "\n":
|
518
|
-
message = f"[{step_context.step_name}] " +
|
519
|
-
else:
|
520
|
-
message = args[0]
|
540
|
+
message = f"[{step_context.step_name}] " + message
|
521
541
|
|
522
542
|
output = method(message, *args[1:], **kwargs)
|
523
543
|
|
zenml/login/credentials_store.py
CHANGED
@@ -25,6 +25,7 @@ from zenml.constants import (
|
|
25
25
|
from zenml.io import fileio
|
26
26
|
from zenml.logger import get_logger
|
27
27
|
from zenml.login.credentials import APIToken, ServerCredentials, ServerType
|
28
|
+
from zenml.login.pro.constants import ZENML_PRO_API_URL
|
28
29
|
from zenml.login.pro.workspace.models import WorkspaceRead
|
29
30
|
from zenml.models import OAuthTokenResponse, ServerModel
|
30
31
|
from zenml.utils import yaml_utils
|
@@ -396,6 +397,36 @@ class CredentialsStore(metaclass=SingletonMetaClass):
|
|
396
397
|
"""
|
397
398
|
return self.get_pro_token(pro_api_url) is not None
|
398
399
|
|
400
|
+
def can_login(self, server_url: str) -> bool:
|
401
|
+
"""Check if credentials to login to the given server exist.
|
402
|
+
|
403
|
+
Args:
|
404
|
+
server_url: The server URL for which to check the authentication.
|
405
|
+
|
406
|
+
Returns:
|
407
|
+
True if the credentials store contains credentials that can be used
|
408
|
+
to login to the given server URL, False otherwise.
|
409
|
+
"""
|
410
|
+
self.check_and_reload_from_file()
|
411
|
+
credentials = self.get_credentials(server_url)
|
412
|
+
if not credentials:
|
413
|
+
return False
|
414
|
+
|
415
|
+
if credentials.api_key is not None:
|
416
|
+
return True
|
417
|
+
elif (
|
418
|
+
credentials.username is not None
|
419
|
+
and credentials.password is not None
|
420
|
+
):
|
421
|
+
return True
|
422
|
+
elif credentials.type == ServerType.PRO:
|
423
|
+
pro_api_url = credentials.pro_api_url or ZENML_PRO_API_URL
|
424
|
+
pro_token = self.get_pro_token(pro_api_url, allow_expired=False)
|
425
|
+
if pro_token:
|
426
|
+
return True
|
427
|
+
|
428
|
+
return False
|
429
|
+
|
399
430
|
def set_api_key(
|
400
431
|
self,
|
401
432
|
server_url: str,
|
@@ -26,6 +26,7 @@ from zenml.constants import (
|
|
26
26
|
from zenml.enums import ArtifactType
|
27
27
|
from zenml.io import fileio
|
28
28
|
from zenml.materializers.base_materializer import BaseMaterializer
|
29
|
+
from zenml.utils.io_utils import is_path_within_directory
|
29
30
|
|
30
31
|
|
31
32
|
class PathMaterializer(BaseMaterializer):
|
@@ -71,7 +72,15 @@ class PathMaterializer(BaseMaterializer):
|
|
71
72
|
|
72
73
|
# Extract the archive to the temporary directory
|
73
74
|
with tarfile.open(archive_path_local, "r:gz") as tar:
|
74
|
-
|
75
|
+
# Validate archive members to prevent path traversal attacks
|
76
|
+
# Filter members to only those with safe paths
|
77
|
+
safe_members = []
|
78
|
+
for member in tar.getmembers():
|
79
|
+
if is_path_within_directory(member.name, directory):
|
80
|
+
safe_members.append(member)
|
81
|
+
|
82
|
+
# Extract only safe members
|
83
|
+
tar.extractall(path=directory, members=safe_members) # nosec B202 - members are filtered through is_path_within_directory
|
75
84
|
|
76
85
|
# Clean up the archive file
|
77
86
|
os.remove(archive_path_local)
|
@@ -93,8 +102,14 @@ class PathMaterializer(BaseMaterializer):
|
|
93
102
|
|
94
103
|
Args:
|
95
104
|
data: Path to a local directory or file to store. Must be a Path object.
|
105
|
+
|
106
|
+
Raises:
|
107
|
+
TypeError: If data is not a Path object.
|
96
108
|
"""
|
97
|
-
|
109
|
+
if not isinstance(data, Path):
|
110
|
+
raise TypeError(
|
111
|
+
f"Expected a Path object, got {type(data).__name__}"
|
112
|
+
)
|
98
113
|
|
99
114
|
if data.is_dir():
|
100
115
|
# Handle directory artifact
|
zenml/models/v2/base/base.py
CHANGED
@@ -134,7 +134,7 @@ class BaseResponse(BaseZenModel, Generic[AnyBody, AnyMetadata, AnyResources]):
|
|
134
134
|
)
|
135
135
|
|
136
136
|
# Check if the name has changed
|
137
|
-
if "name" in self.model_fields:
|
137
|
+
if "name" in type(self).model_fields:
|
138
138
|
original_name = getattr(self, "name")
|
139
139
|
hydrated_name = getattr(hydrated_model, "name")
|
140
140
|
|
@@ -172,7 +172,7 @@ class BaseResponse(BaseZenModel, Generic[AnyBody, AnyMetadata, AnyResources]):
|
|
172
172
|
)
|
173
173
|
|
174
174
|
# Check all the fields in the body
|
175
|
-
for field in self.get_body().model_fields:
|
175
|
+
for field in type(self.get_body()).model_fields:
|
176
176
|
original_value = getattr(self.get_body(), field)
|
177
177
|
hydrated_value = getattr(hydrated_model.get_body(), field)
|
178
178
|
|
@@ -255,7 +255,9 @@ class BaseResponse(BaseZenModel, Generic[AnyBody, AnyMetadata, AnyResources]):
|
|
255
255
|
"""
|
256
256
|
if self.metadata is None:
|
257
257
|
# If the metadata is not there, check the class first.
|
258
|
-
metadata_annotation =
|
258
|
+
metadata_annotation = (
|
259
|
+
type(self).model_fields["metadata"].annotation
|
260
|
+
)
|
259
261
|
assert metadata_annotation is not None, (
|
260
262
|
"For each response model, an annotated metadata"
|
261
263
|
"field should exist."
|
@@ -293,7 +295,9 @@ class BaseResponse(BaseZenModel, Generic[AnyBody, AnyMetadata, AnyResources]):
|
|
293
295
|
"""
|
294
296
|
if self.resources is None:
|
295
297
|
# If the resources are not there, check the class first.
|
296
|
-
resources_annotation =
|
298
|
+
resources_annotation = (
|
299
|
+
type(self).model_fields["resources"].annotation
|
300
|
+
)
|
297
301
|
assert resources_annotation is not None, (
|
298
302
|
"For each response model, an annotated resources"
|
299
303
|
"field should exist."
|
zenml/models/v2/base/filter.py
CHANGED
@@ -665,7 +665,7 @@ class BaseFilter(BaseModel):
|
|
665
665
|
A list of Filter models.
|
666
666
|
"""
|
667
667
|
return self._generate_filter_list(
|
668
|
-
{key: getattr(self, key) for key in self.model_fields}
|
668
|
+
{key: getattr(self, key) for key in type(self).model_fields}
|
669
669
|
)
|
670
670
|
|
671
671
|
@property
|
@@ -45,6 +45,7 @@ from zenml.models.v2.base.scoped import (
|
|
45
45
|
RunMetadataFilterMixin,
|
46
46
|
TaggableFilter,
|
47
47
|
)
|
48
|
+
from zenml.models.v2.core.logs import LogsRequest
|
48
49
|
from zenml.models.v2.core.model_version import ModelVersionResponse
|
49
50
|
from zenml.models.v2.core.tag import TagResponse
|
50
51
|
from zenml.utils.tag_utils import Tag
|
@@ -55,6 +56,7 @@ if TYPE_CHECKING:
|
|
55
56
|
from zenml.models import TriggerExecutionResponse
|
56
57
|
from zenml.models.v2.core.artifact_version import ArtifactVersionResponse
|
57
58
|
from zenml.models.v2.core.code_reference import CodeReferenceResponse
|
59
|
+
from zenml.models.v2.core.logs import LogsResponse
|
58
60
|
from zenml.models.v2.core.pipeline import PipelineResponse
|
59
61
|
from zenml.models.v2.core.pipeline_build import (
|
60
62
|
PipelineBuildResponse,
|
@@ -124,6 +126,10 @@ class PipelineRunRequest(ProjectScopedRequest):
|
|
124
126
|
default=None,
|
125
127
|
title="Tags of the pipeline run.",
|
126
128
|
)
|
129
|
+
logs: Optional[LogsRequest] = Field(
|
130
|
+
default=None,
|
131
|
+
title="Logs of the pipeline run.",
|
132
|
+
)
|
127
133
|
|
128
134
|
model_config = ConfigDict(protected_namespaces=())
|
129
135
|
|
@@ -252,6 +258,10 @@ class PipelineRunResponseResources(ProjectScopedResponseResources):
|
|
252
258
|
tags: List[TagResponse] = Field(
|
253
259
|
title="Tags associated with the pipeline run.",
|
254
260
|
)
|
261
|
+
logs: Optional["LogsResponse"] = Field(
|
262
|
+
title="Logs associated with this pipeline run.",
|
263
|
+
default=None,
|
264
|
+
)
|
255
265
|
|
256
266
|
# TODO: In Pydantic v2, the `model_` is a protected namespaces for all
|
257
267
|
# fields defined under base models. If not handled, this raises a warning.
|
@@ -579,6 +589,15 @@ class PipelineRunResponse(
|
|
579
589
|
"""
|
580
590
|
return self.get_resources().tags
|
581
591
|
|
592
|
+
@property
|
593
|
+
def logs(self) -> Optional["LogsResponse"]:
|
594
|
+
"""The `logs` property.
|
595
|
+
|
596
|
+
Returns:
|
597
|
+
the value of the property.
|
598
|
+
"""
|
599
|
+
return self.get_resources().logs
|
600
|
+
|
582
601
|
|
583
602
|
# ------------------ Filter Model ------------------
|
584
603
|
|
@@ -158,7 +158,7 @@ class StepLauncher:
|
|
158
158
|
step_name=self._step_name,
|
159
159
|
)
|
160
160
|
|
161
|
-
logs_context = step_logging.
|
161
|
+
logs_context = step_logging.PipelineLogsStorageContext(
|
162
162
|
logs_uri=logs_uri, artifact_store=self._stack.artifact_store
|
163
163
|
) # type: ignore[assignment]
|
164
164
|
|
@@ -240,7 +240,7 @@ class StepLauncher:
|
|
240
240
|
# the external jobs in step operators
|
241
241
|
if isinstance(
|
242
242
|
logs_context,
|
243
|
-
step_logging.
|
243
|
+
step_logging.PipelineLogsStorageContext,
|
244
244
|
):
|
245
245
|
force_write_logs = partial(
|
246
246
|
logs_context.storage.save_to_file,
|
@@ -421,7 +421,6 @@ class StepLauncher:
|
|
421
421
|
)
|
422
422
|
environment = orchestrator_utils.get_config_environment_vars(
|
423
423
|
pipeline_run_id=step_run_info.run_id,
|
424
|
-
step_run_id=step_run_info.step_run_id,
|
425
424
|
)
|
426
425
|
if last_retry:
|
427
426
|
environment[ENV_ZENML_IGNORE_FAILURE_HOOK] = str(False)
|
@@ -40,7 +40,7 @@ from zenml.constants import (
|
|
40
40
|
from zenml.enums import ArtifactSaveType
|
41
41
|
from zenml.exceptions import StepInterfaceError
|
42
42
|
from zenml.logger import get_logger
|
43
|
-
from zenml.logging.step_logging import
|
43
|
+
from zenml.logging.step_logging import PipelineLogsStorageContext, redirected
|
44
44
|
from zenml.materializers.base_materializer import BaseMaterializer
|
45
45
|
from zenml.models.v2.core.step_run import StepRunInputResponse
|
46
46
|
from zenml.orchestrators.publish_utils import (
|
@@ -136,7 +136,7 @@ class StepRunner:
|
|
136
136
|
logs_context = nullcontext()
|
137
137
|
if step_logging_enabled and not redirected.get():
|
138
138
|
if step_run.logs:
|
139
|
-
logs_context =
|
139
|
+
logs_context = PipelineLogsStorageContext( # type: ignore[assignment]
|
140
140
|
logs_uri=step_run.logs.uri,
|
141
141
|
artifact_store=self._stack.artifact_store,
|
142
142
|
)
|
zenml/orchestrators/utils.py
CHANGED
@@ -105,7 +105,6 @@ def is_setting_enabled(
|
|
105
105
|
def get_config_environment_vars(
|
106
106
|
schedule_id: Optional[UUID] = None,
|
107
107
|
pipeline_run_id: Optional[UUID] = None,
|
108
|
-
step_run_id: Optional[UUID] = None,
|
109
108
|
) -> Dict[str, str]:
|
110
109
|
"""Gets environment variables to set for mirroring the active config.
|
111
110
|
|
@@ -119,7 +118,6 @@ def get_config_environment_vars(
|
|
119
118
|
schedule_id: Optional schedule ID to use to generate a new API token.
|
120
119
|
pipeline_run_id: Optional pipeline run ID to use to generate a new API
|
121
120
|
token.
|
122
|
-
step_run_id: Optional step run ID to use to generate a new API token.
|
123
121
|
|
124
122
|
Returns:
|
125
123
|
Environment variable dict.
|
@@ -138,7 +136,7 @@ def get_config_environment_vars(
|
|
138
136
|
credentials_store = get_credentials_store()
|
139
137
|
url = global_config.store_configuration.url
|
140
138
|
api_token = credentials_store.get_token(url, allow_expired=False)
|
141
|
-
if schedule_id or pipeline_run_id
|
139
|
+
if schedule_id or pipeline_run_id:
|
142
140
|
assert isinstance(global_config.zen_store, RestZenStore)
|
143
141
|
|
144
142
|
# The user has the option to manually set an expiration for the API
|
@@ -173,7 +171,7 @@ def get_config_environment_vars(
|
|
173
171
|
# If only a schedule is given, the pipeline run credentials will
|
174
172
|
# be valid for the entire duration of the schedule.
|
175
173
|
api_key = credentials_store.get_api_key(url)
|
176
|
-
if not api_key and not pipeline_run_id
|
174
|
+
if not api_key and not pipeline_run_id:
|
177
175
|
logger.warning(
|
178
176
|
"An API token without an expiration time will be generated "
|
179
177
|
"and used to run this pipeline on a schedule. This is very "
|
@@ -194,7 +192,6 @@ def get_config_environment_vars(
|
|
194
192
|
token_type=APITokenType.WORKLOAD,
|
195
193
|
schedule_id=schedule_id,
|
196
194
|
pipeline_run_id=pipeline_run_id,
|
197
|
-
step_run_id=step_run_id,
|
198
195
|
)
|
199
196
|
|
200
197
|
environment_vars[ENV_ZENML_STORE_PREFIX + "API_TOKEN"] = (
|
@@ -109,6 +109,7 @@ class PipelineContext:
|
|
109
109
|
pipeline_configuration.enable_artifact_visualization
|
110
110
|
)
|
111
111
|
self.enable_step_logs = pipeline_configuration.enable_step_logs
|
112
|
+
self.enable_pipeline_logs = pipeline_configuration.enable_pipeline_logs
|
112
113
|
self.settings = pipeline_configuration.settings
|
113
114
|
self.extra = pipeline_configuration.extra
|
114
115
|
self.model = pipeline_configuration.model
|
@@ -50,6 +50,7 @@ def pipeline(
|
|
50
50
|
enable_cache: Optional[bool] = None,
|
51
51
|
enable_artifact_metadata: Optional[bool] = None,
|
52
52
|
enable_step_logs: Optional[bool] = None,
|
53
|
+
enable_pipeline_logs: Optional[bool] = None,
|
53
54
|
settings: Optional[Dict[str, "SettingsOrDict"]] = None,
|
54
55
|
tags: Optional[List[Union[str, "Tag"]]] = None,
|
55
56
|
extra: Optional[Dict[str, Any]] = None,
|
@@ -67,6 +68,7 @@ def pipeline(
|
|
67
68
|
enable_cache: Optional[bool] = None,
|
68
69
|
enable_artifact_metadata: Optional[bool] = None,
|
69
70
|
enable_step_logs: Optional[bool] = None,
|
71
|
+
enable_pipeline_logs: Optional[bool] = None,
|
70
72
|
settings: Optional[Dict[str, "SettingsOrDict"]] = None,
|
71
73
|
tags: Optional[List[Union[str, "Tag"]]] = None,
|
72
74
|
extra: Optional[Dict[str, Any]] = None,
|
@@ -84,6 +86,7 @@ def pipeline(
|
|
84
86
|
enable_cache: Whether to use caching or not.
|
85
87
|
enable_artifact_metadata: Whether to enable artifact metadata or not.
|
86
88
|
enable_step_logs: If step logs should be enabled for this pipeline.
|
89
|
+
enable_pipeline_logs: If pipeline logs should be enabled for this pipeline.
|
87
90
|
settings: Settings for this pipeline.
|
88
91
|
tags: Tags to apply to runs of the pipeline.
|
89
92
|
extra: Extra configurations for this pipeline.
|
@@ -108,6 +111,7 @@ def pipeline(
|
|
108
111
|
enable_cache=enable_cache,
|
109
112
|
enable_artifact_metadata=enable_artifact_metadata,
|
110
113
|
enable_step_logs=enable_step_logs,
|
114
|
+
enable_pipeline_logs=enable_pipeline_logs,
|
111
115
|
settings=settings,
|
112
116
|
tags=tags,
|
113
117
|
extra=extra,
|
@@ -16,7 +16,7 @@
|
|
16
16
|
import copy
|
17
17
|
import hashlib
|
18
18
|
import inspect
|
19
|
-
from contextlib import contextmanager
|
19
|
+
from contextlib import contextmanager, nullcontext
|
20
20
|
from pathlib import Path
|
21
21
|
from typing import (
|
22
22
|
TYPE_CHECKING,
|
@@ -56,8 +56,13 @@ from zenml.enums import StackComponentType
|
|
56
56
|
from zenml.exceptions import EntityExistsError
|
57
57
|
from zenml.hooks.hook_validators import resolve_and_validate_hook
|
58
58
|
from zenml.logger import get_logger
|
59
|
+
from zenml.logging.step_logging import (
|
60
|
+
PipelineLogsStorageContext,
|
61
|
+
prepare_logs_uri,
|
62
|
+
)
|
59
63
|
from zenml.models import (
|
60
64
|
CodeReferenceRequest,
|
65
|
+
LogsRequest,
|
61
66
|
PipelineBuildBase,
|
62
67
|
PipelineBuildResponse,
|
63
68
|
PipelineDeploymentBase,
|
@@ -130,6 +135,7 @@ class Pipeline:
|
|
130
135
|
enable_artifact_metadata: Optional[bool] = None,
|
131
136
|
enable_artifact_visualization: Optional[bool] = None,
|
132
137
|
enable_step_logs: Optional[bool] = None,
|
138
|
+
enable_pipeline_logs: Optional[bool] = None,
|
133
139
|
settings: Optional[Mapping[str, "SettingsOrDict"]] = None,
|
134
140
|
tags: Optional[List[Union[str, "Tag"]]] = None,
|
135
141
|
extra: Optional[Dict[str, Any]] = None,
|
@@ -149,6 +155,7 @@ class Pipeline:
|
|
149
155
|
enable_artifact_visualization: If artifact visualization should be
|
150
156
|
enabled for this pipeline.
|
151
157
|
enable_step_logs: If step logs should be enabled for this pipeline.
|
158
|
+
enable_pipeline_logs: If pipeline logs should be enabled for this pipeline.
|
152
159
|
settings: Settings for this pipeline.
|
153
160
|
tags: Tags to apply to runs of this pipeline.
|
154
161
|
extra: Extra configurations for this pipeline.
|
@@ -174,6 +181,7 @@ class Pipeline:
|
|
174
181
|
enable_artifact_metadata=enable_artifact_metadata,
|
175
182
|
enable_artifact_visualization=enable_artifact_visualization,
|
176
183
|
enable_step_logs=enable_step_logs,
|
184
|
+
enable_pipeline_logs=enable_pipeline_logs,
|
177
185
|
settings=settings,
|
178
186
|
tags=tags,
|
179
187
|
extra=extra,
|
@@ -293,6 +301,7 @@ class Pipeline:
|
|
293
301
|
enable_artifact_metadata: Optional[bool] = None,
|
294
302
|
enable_artifact_visualization: Optional[bool] = None,
|
295
303
|
enable_step_logs: Optional[bool] = None,
|
304
|
+
enable_pipeline_logs: Optional[bool] = None,
|
296
305
|
settings: Optional[Mapping[str, "SettingsOrDict"]] = None,
|
297
306
|
tags: Optional[List[Union[str, "Tag"]]] = None,
|
298
307
|
extra: Optional[Dict[str, Any]] = None,
|
@@ -322,6 +331,7 @@ class Pipeline:
|
|
322
331
|
enable_artifact_visualization: If artifact visualization should be
|
323
332
|
enabled for this pipeline.
|
324
333
|
enable_step_logs: If step logs should be enabled for this pipeline.
|
334
|
+
enable_pipeline_logs: If pipeline logs should be enabled for this pipeline.
|
325
335
|
settings: settings for this pipeline.
|
326
336
|
tags: Tags to apply to runs of this pipeline.
|
327
337
|
extra: Extra configurations for this pipeline.
|
@@ -364,6 +374,7 @@ class Pipeline:
|
|
364
374
|
"enable_artifact_metadata": enable_artifact_metadata,
|
365
375
|
"enable_artifact_visualization": enable_artifact_visualization,
|
366
376
|
"enable_step_logs": enable_step_logs,
|
377
|
+
"enable_pipeline_logs": enable_pipeline_logs,
|
367
378
|
"settings": settings,
|
368
379
|
"tags": tags,
|
369
380
|
"extra": extra,
|
@@ -588,6 +599,7 @@ To avoid this consider setting pipeline parameters only in one place (config or
|
|
588
599
|
enable_artifact_metadata: Optional[bool] = None,
|
589
600
|
enable_artifact_visualization: Optional[bool] = None,
|
590
601
|
enable_step_logs: Optional[bool] = None,
|
602
|
+
enable_pipeline_logs: Optional[bool] = None,
|
591
603
|
schedule: Optional[Schedule] = None,
|
592
604
|
build: Union[str, "UUID", "PipelineBuildBase", None] = None,
|
593
605
|
settings: Optional[Mapping[str, "SettingsOrDict"]] = None,
|
@@ -610,6 +622,8 @@ To avoid this consider setting pipeline parameters only in one place (config or
|
|
610
622
|
enable_artifact_visualization: If artifact visualization should be
|
611
623
|
enabled for this pipeline run.
|
612
624
|
enable_step_logs: If step logs should be enabled for this pipeline.
|
625
|
+
enable_pipeline_logs: If pipeline logs should be enabled for this
|
626
|
+
pipeline run.
|
613
627
|
schedule: Optional schedule to use for the run.
|
614
628
|
build: Optional build to use for the run.
|
615
629
|
settings: Settings for this pipeline run.
|
@@ -641,6 +655,7 @@ To avoid this consider setting pipeline parameters only in one place (config or
|
|
641
655
|
enable_artifact_metadata=enable_artifact_metadata,
|
642
656
|
enable_artifact_visualization=enable_artifact_visualization,
|
643
657
|
enable_step_logs=enable_step_logs,
|
658
|
+
enable_pipeline_logs=enable_pipeline_logs,
|
644
659
|
steps=step_configurations,
|
645
660
|
settings=settings,
|
646
661
|
schedule=schedule,
|
@@ -723,7 +738,7 @@ To avoid this consider setting pipeline parameters only in one place (config or
|
|
723
738
|
if prevent_build_reuse:
|
724
739
|
logger.warning(
|
725
740
|
"Passing `prevent_build_reuse=True` to "
|
726
|
-
"`pipeline.
|
741
|
+
"`pipeline.with_options(...)` is deprecated. Use "
|
727
742
|
"`DockerSettings.prevent_build_reuse` instead."
|
728
743
|
)
|
729
744
|
|
@@ -806,31 +821,77 @@ To avoid this consider setting pipeline parameters only in one place (config or
|
|
806
821
|
|
807
822
|
with track_handler(AnalyticsEvent.RUN_PIPELINE) as analytics_handler:
|
808
823
|
stack = Client().active_stack
|
809
|
-
deployment = self._create_deployment(**self._run_args)
|
810
824
|
|
811
|
-
|
812
|
-
|
825
|
+
# Enable or disable pipeline run logs storage
|
826
|
+
if self._run_args.get("schedule"):
|
827
|
+
# Pipeline runs scheduled to run in the future are not logged
|
828
|
+
# via the client.
|
829
|
+
logging_enabled = False
|
830
|
+
elif constants.handle_bool_env_var(
|
831
|
+
constants.ENV_ZENML_DISABLE_PIPELINE_LOGS_STORAGE, False
|
832
|
+
):
|
833
|
+
logging_enabled = False
|
834
|
+
else:
|
835
|
+
logging_enabled = self._run_args.get(
|
836
|
+
"enable_pipeline_logs",
|
837
|
+
self.configuration.enable_pipeline_logs
|
838
|
+
if self.configuration.enable_pipeline_logs is not None
|
839
|
+
else True,
|
840
|
+
)
|
813
841
|
|
814
|
-
|
815
|
-
|
816
|
-
stack=stack,
|
817
|
-
run_id=run.id if run else None,
|
818
|
-
)
|
842
|
+
logs_context = nullcontext()
|
843
|
+
logs_model = None
|
819
844
|
|
820
|
-
if
|
821
|
-
|
822
|
-
|
823
|
-
|
824
|
-
|
825
|
-
|
826
|
-
|
827
|
-
|
828
|
-
|
845
|
+
if logging_enabled:
|
846
|
+
# Configure the logs
|
847
|
+
logs_uri = prepare_logs_uri(
|
848
|
+
stack.artifact_store,
|
849
|
+
)
|
850
|
+
|
851
|
+
logs_context = PipelineLogsStorageContext(
|
852
|
+
logs_uri=logs_uri,
|
853
|
+
artifact_store=stack.artifact_store,
|
854
|
+
prepend_step_name=False,
|
855
|
+
) # type: ignore[assignment]
|
856
|
+
|
857
|
+
logs_model = LogsRequest(
|
858
|
+
uri=logs_uri,
|
859
|
+
artifact_store_id=stack.artifact_store.id,
|
860
|
+
)
|
861
|
+
|
862
|
+
with logs_context:
|
863
|
+
deployment = self._create_deployment(**self._run_args)
|
864
|
+
|
865
|
+
self.log_pipeline_deployment_metadata(deployment)
|
866
|
+
run = create_placeholder_run(
|
867
|
+
deployment=deployment, logs=logs_model
|
868
|
+
)
|
869
|
+
|
870
|
+
analytics_handler.metadata = (
|
871
|
+
self._get_pipeline_analytics_metadata(
|
872
|
+
deployment=deployment,
|
873
|
+
stack=stack,
|
874
|
+
run_id=run.id if run else None,
|
829
875
|
)
|
876
|
+
)
|
877
|
+
|
878
|
+
if run:
|
879
|
+
run_url = dashboard_utils.get_run_url(run)
|
880
|
+
if run_url:
|
881
|
+
logger.info(
|
882
|
+
f"Dashboard URL for Pipeline Run: {run_url}"
|
883
|
+
)
|
884
|
+
else:
|
885
|
+
logger.info(
|
886
|
+
"You can visualize your pipeline runs in the `ZenML "
|
887
|
+
"Dashboard`. In order to try it locally, please run "
|
888
|
+
"`zenml login --local`."
|
889
|
+
)
|
890
|
+
|
891
|
+
deploy_pipeline(
|
892
|
+
deployment=deployment, stack=stack, placeholder_run=run
|
893
|
+
)
|
830
894
|
|
831
|
-
deploy_pipeline(
|
832
|
-
deployment=deployment, stack=stack, placeholder_run=run
|
833
|
-
)
|
834
895
|
if run:
|
835
896
|
return Client().get_pipeline_run(run.id)
|
836
897
|
return None
|