zenml-nightly 0.80.2.dev20250414__py3-none-any.whl → 0.80.2.dev20250416__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. zenml/VERSION +1 -1
  2. zenml/artifacts/utils.py +7 -2
  3. zenml/cli/utils.py +13 -11
  4. zenml/config/compiler.py +1 -0
  5. zenml/config/global_config.py +1 -1
  6. zenml/config/pipeline_configurations.py +1 -0
  7. zenml/config/pipeline_run_configuration.py +1 -0
  8. zenml/config/server_config.py +7 -0
  9. zenml/constants.py +8 -0
  10. zenml/integrations/gcp/orchestrators/vertex_orchestrator.py +47 -5
  11. zenml/integrations/gcp/vertex_custom_job_parameters.py +15 -1
  12. zenml/integrations/kubernetes/flavors/kubernetes_step_operator_flavor.py +12 -0
  13. zenml/integrations/kubernetes/orchestrators/kube_utils.py +92 -0
  14. zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator.py +12 -3
  15. zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator_entrypoint.py +11 -65
  16. zenml/integrations/kubernetes/step_operators/kubernetes_step_operator.py +11 -3
  17. zenml/logging/step_logging.py +41 -21
  18. zenml/login/credentials_store.py +31 -0
  19. zenml/materializers/path_materializer.py +17 -2
  20. zenml/models/v2/base/base.py +8 -4
  21. zenml/models/v2/base/filter.py +1 -1
  22. zenml/models/v2/core/pipeline_run.py +19 -0
  23. zenml/orchestrators/step_launcher.py +2 -3
  24. zenml/orchestrators/step_runner.py +2 -2
  25. zenml/orchestrators/utils.py +2 -5
  26. zenml/pipelines/pipeline_context.py +1 -0
  27. zenml/pipelines/pipeline_decorator.py +4 -0
  28. zenml/pipelines/pipeline_definition.py +83 -22
  29. zenml/pipelines/run_utils.py +4 -0
  30. zenml/steps/utils.py +1 -1
  31. zenml/utils/io_utils.py +23 -0
  32. zenml/zen_server/auth.py +96 -64
  33. zenml/zen_server/cloud_utils.py +7 -1
  34. zenml/zen_server/download_utils.py +123 -0
  35. zenml/zen_server/jwt.py +0 -14
  36. zenml/zen_server/rbac/rbac_interface.py +10 -3
  37. zenml/zen_server/rbac/utils.py +13 -3
  38. zenml/zen_server/rbac/zenml_cloud_rbac.py +14 -8
  39. zenml/zen_server/routers/artifact_version_endpoints.py +86 -3
  40. zenml/zen_server/routers/auth_endpoints.py +5 -36
  41. zenml/zen_server/routers/pipeline_deployments_endpoints.py +63 -26
  42. zenml/zen_server/routers/runs_endpoints.py +57 -0
  43. zenml/zen_server/routers/users_endpoints.py +13 -8
  44. zenml/zen_server/template_execution/utils.py +3 -3
  45. zenml/zen_stores/migrations/versions/ff538a321a92_migrate_onboarding_state.py +123 -0
  46. zenml/zen_stores/rest_zen_store.py +16 -13
  47. zenml/zen_stores/schemas/pipeline_run_schemas.py +1 -0
  48. zenml/zen_stores/schemas/server_settings_schemas.py +4 -1
  49. zenml/zen_stores/sql_zen_store.py +18 -0
  50. {zenml_nightly-0.80.2.dev20250414.dist-info → zenml_nightly-0.80.2.dev20250416.dist-info}/METADATA +2 -1
  51. {zenml_nightly-0.80.2.dev20250414.dist-info → zenml_nightly-0.80.2.dev20250416.dist-info}/RECORD +54 -52
  52. {zenml_nightly-0.80.2.dev20250414.dist-info → zenml_nightly-0.80.2.dev20250416.dist-info}/LICENSE +0 -0
  53. {zenml_nightly-0.80.2.dev20250414.dist-info → zenml_nightly-0.80.2.dev20250416.dist-info}/WHEEL +0 -0
  54. {zenml_nightly-0.80.2.dev20250414.dist-info → zenml_nightly-0.80.2.dev20250416.dist-info}/entry_points.txt +0 -0
@@ -13,6 +13,7 @@
13
13
  # permissions and limitations under the License.
14
14
  """ZenML logging handler."""
15
15
 
16
+ import logging
16
17
  import os
17
18
  import re
18
19
  import sys
@@ -48,6 +49,7 @@ logger = get_logger(__name__)
48
49
  redirected: ContextVar[bool] = ContextVar("redirected", default=False)
49
50
 
50
51
  LOGS_EXTENSION = ".log"
52
+ PIPELINE_RUN_LOGS_FOLDER = "pipeline_runs"
51
53
 
52
54
 
53
55
  def remove_ansi_escape_codes(text: str) -> str:
@@ -65,14 +67,14 @@ def remove_ansi_escape_codes(text: str) -> str:
65
67
 
66
68
  def prepare_logs_uri(
67
69
  artifact_store: "BaseArtifactStore",
68
- step_name: str,
70
+ step_name: Optional[str] = None,
69
71
  log_key: Optional[str] = None,
70
72
  ) -> str:
71
73
  """Generates and prepares a URI for the log file or folder for a step.
72
74
 
73
75
  Args:
74
76
  artifact_store: The artifact store on which the artifact will be stored.
75
- step_name: Name of the step.
77
+ step_name: Name of the step. Skipped for global pipeline run logs.
76
78
  log_key: The unique identification key of the log file.
77
79
 
78
80
  Returns:
@@ -81,11 +83,8 @@ def prepare_logs_uri(
81
83
  if log_key is None:
82
84
  log_key = str(uuid4())
83
85
 
84
- logs_base_uri = os.path.join(
85
- artifact_store.path,
86
- step_name,
87
- "logs",
88
- )
86
+ subfolder = step_name or PIPELINE_RUN_LOGS_FOLDER
87
+ logs_base_uri = os.path.join(artifact_store.path, subfolder, "logs")
89
88
 
90
89
  # Create the dir
91
90
  if not artifact_store.exists(logs_base_uri):
@@ -210,7 +209,7 @@ def fetch_logs(
210
209
  artifact_store.cleanup()
211
210
 
212
211
 
213
- class StepLogsStorage:
212
+ class PipelineLogsStorage:
214
213
  """Helper class which buffers and stores logs to a given URI."""
215
214
 
216
215
  def __init__(
@@ -324,6 +323,18 @@ class StepLogsStorage:
324
323
  self.disabled = True
325
324
 
326
325
  try:
326
+ # The configured logging handler uses a lock to ensure that
327
+ # logs generated by different threads are not interleaved.
328
+ # Given that most artifact stores are based on fsspec, which
329
+ # use a separate thread for async operations, it may happen that
330
+ # the fsspec library itself will log something, which will end
331
+ # up in a deadlock.
332
+ # To avoid this, we temporarily disable the lock in the logging
333
+ # handler while writing to the file.
334
+ logging_handler = logging.getLogger().handlers[0]
335
+ logging_lock = logging_handler.lock
336
+ logging_handler.lock = None
337
+
327
338
  if self.buffer:
328
339
  if self.artifact_store.config.IS_IMMUTABLE_FILESYSTEM:
329
340
  _logs_uri = self._get_timestamped_filename()
@@ -353,6 +364,9 @@ class StepLogsStorage:
353
364
  # I/O errors.
354
365
  logger.error(f"Error while trying to write logs: {e}")
355
366
  finally:
367
+ # Restore the original logging handler lock
368
+ logging_handler.lock = logging_lock
369
+
356
370
  self.buffer = []
357
371
  self.last_save_time = time.time()
358
372
 
@@ -418,27 +432,32 @@ class StepLogsStorage:
418
432
  )
419
433
 
420
434
 
421
- class StepLogsStorageContext:
422
- """Context manager which patches stdout and stderr during step execution."""
435
+ class PipelineLogsStorageContext:
436
+ """Context manager which patches stdout and stderr during pipeline run execution."""
423
437
 
424
438
  def __init__(
425
- self, logs_uri: str, artifact_store: "BaseArtifactStore"
439
+ self,
440
+ logs_uri: str,
441
+ artifact_store: "BaseArtifactStore",
442
+ prepend_step_name: bool = True,
426
443
  ) -> None:
427
444
  """Initializes and prepares a storage object.
428
445
 
429
446
  Args:
430
447
  logs_uri: the URI of the logs file.
431
- artifact_store: Artifact Store from the current step context.
448
+ artifact_store: Artifact Store from the current pipeline run context.
449
+ prepend_step_name: Whether to prepend the step name to the logs.
432
450
  """
433
- self.storage = StepLogsStorage(
451
+ self.storage = PipelineLogsStorage(
434
452
  logs_uri=logs_uri, artifact_store=artifact_store
435
453
  )
454
+ self.prepend_step_name = prepend_step_name
436
455
 
437
- def __enter__(self) -> "StepLogsStorageContext":
456
+ def __enter__(self) -> "PipelineLogsStorageContext":
438
457
  """Enter condition of the context manager.
439
458
 
440
459
  Wraps the `write` method of both stderr and stdout, so each incoming
441
- message gets stored in the step logs storage.
460
+ message gets stored in the pipeline logs storage.
442
461
 
443
462
  Returns:
444
463
  self
@@ -499,14 +518,17 @@ class StepLogsStorageContext:
499
518
  """
500
519
 
501
520
  def wrapped_write(*args: Any, **kwargs: Any) -> Any:
502
- # Check if step names in logs are disabled via env var
503
- step_names_disabled = handle_bool_env_var(
504
- ENV_ZENML_DISABLE_STEP_NAMES_IN_LOGS, default=False
521
+ step_names_disabled = (
522
+ handle_bool_env_var(
523
+ ENV_ZENML_DISABLE_STEP_NAMES_IN_LOGS, default=False
524
+ )
525
+ or not self.prepend_step_name
505
526
  )
506
527
 
507
528
  if step_names_disabled:
508
529
  output = method(*args, **kwargs)
509
530
  else:
531
+ message = args[0]
510
532
  # Try to get step context if not available yet
511
533
  step_context = None
512
534
  try:
@@ -515,9 +537,7 @@ class StepLogsStorageContext:
515
537
  pass
516
538
 
517
539
  if step_context and args[0] != "\n":
518
- message = f"[{step_context.step_name}] " + args[0]
519
- else:
520
- message = args[0]
540
+ message = f"[{step_context.step_name}] " + message
521
541
 
522
542
  output = method(message, *args[1:], **kwargs)
523
543
 
@@ -25,6 +25,7 @@ from zenml.constants import (
25
25
  from zenml.io import fileio
26
26
  from zenml.logger import get_logger
27
27
  from zenml.login.credentials import APIToken, ServerCredentials, ServerType
28
+ from zenml.login.pro.constants import ZENML_PRO_API_URL
28
29
  from zenml.login.pro.workspace.models import WorkspaceRead
29
30
  from zenml.models import OAuthTokenResponse, ServerModel
30
31
  from zenml.utils import yaml_utils
@@ -396,6 +397,36 @@ class CredentialsStore(metaclass=SingletonMetaClass):
396
397
  """
397
398
  return self.get_pro_token(pro_api_url) is not None
398
399
 
400
+ def can_login(self, server_url: str) -> bool:
401
+ """Check if credentials to login to the given server exist.
402
+
403
+ Args:
404
+ server_url: The server URL for which to check the authentication.
405
+
406
+ Returns:
407
+ True if the credentials store contains credentials that can be used
408
+ to login to the given server URL, False otherwise.
409
+ """
410
+ self.check_and_reload_from_file()
411
+ credentials = self.get_credentials(server_url)
412
+ if not credentials:
413
+ return False
414
+
415
+ if credentials.api_key is not None:
416
+ return True
417
+ elif (
418
+ credentials.username is not None
419
+ and credentials.password is not None
420
+ ):
421
+ return True
422
+ elif credentials.type == ServerType.PRO:
423
+ pro_api_url = credentials.pro_api_url or ZENML_PRO_API_URL
424
+ pro_token = self.get_pro_token(pro_api_url, allow_expired=False)
425
+ if pro_token:
426
+ return True
427
+
428
+ return False
429
+
399
430
  def set_api_key(
400
431
  self,
401
432
  server_url: str,
@@ -26,6 +26,7 @@ from zenml.constants import (
26
26
  from zenml.enums import ArtifactType
27
27
  from zenml.io import fileio
28
28
  from zenml.materializers.base_materializer import BaseMaterializer
29
+ from zenml.utils.io_utils import is_path_within_directory
29
30
 
30
31
 
31
32
  class PathMaterializer(BaseMaterializer):
@@ -71,7 +72,15 @@ class PathMaterializer(BaseMaterializer):
71
72
 
72
73
  # Extract the archive to the temporary directory
73
74
  with tarfile.open(archive_path_local, "r:gz") as tar:
74
- tar.extractall(path=directory)
75
+ # Validate archive members to prevent path traversal attacks
76
+ # Filter members to only those with safe paths
77
+ safe_members = []
78
+ for member in tar.getmembers():
79
+ if is_path_within_directory(member.name, directory):
80
+ safe_members.append(member)
81
+
82
+ # Extract only safe members
83
+ tar.extractall(path=directory, members=safe_members) # nosec B202 - members are filtered through is_path_within_directory
75
84
 
76
85
  # Clean up the archive file
77
86
  os.remove(archive_path_local)
@@ -93,8 +102,14 @@ class PathMaterializer(BaseMaterializer):
93
102
 
94
103
  Args:
95
104
  data: Path to a local directory or file to store. Must be a Path object.
105
+
106
+ Raises:
107
+ TypeError: If data is not a Path object.
96
108
  """
97
- assert isinstance(data, Path)
109
+ if not isinstance(data, Path):
110
+ raise TypeError(
111
+ f"Expected a Path object, got {type(data).__name__}"
112
+ )
98
113
 
99
114
  if data.is_dir():
100
115
  # Handle directory artifact
@@ -134,7 +134,7 @@ class BaseResponse(BaseZenModel, Generic[AnyBody, AnyMetadata, AnyResources]):
134
134
  )
135
135
 
136
136
  # Check if the name has changed
137
- if "name" in self.model_fields:
137
+ if "name" in type(self).model_fields:
138
138
  original_name = getattr(self, "name")
139
139
  hydrated_name = getattr(hydrated_model, "name")
140
140
 
@@ -172,7 +172,7 @@ class BaseResponse(BaseZenModel, Generic[AnyBody, AnyMetadata, AnyResources]):
172
172
  )
173
173
 
174
174
  # Check all the fields in the body
175
- for field in self.get_body().model_fields:
175
+ for field in type(self.get_body()).model_fields:
176
176
  original_value = getattr(self.get_body(), field)
177
177
  hydrated_value = getattr(hydrated_model.get_body(), field)
178
178
 
@@ -255,7 +255,9 @@ class BaseResponse(BaseZenModel, Generic[AnyBody, AnyMetadata, AnyResources]):
255
255
  """
256
256
  if self.metadata is None:
257
257
  # If the metadata is not there, check the class first.
258
- metadata_annotation = self.model_fields["metadata"].annotation
258
+ metadata_annotation = (
259
+ type(self).model_fields["metadata"].annotation
260
+ )
259
261
  assert metadata_annotation is not None, (
260
262
  "For each response model, an annotated metadata"
261
263
  "field should exist."
@@ -293,7 +295,9 @@ class BaseResponse(BaseZenModel, Generic[AnyBody, AnyMetadata, AnyResources]):
293
295
  """
294
296
  if self.resources is None:
295
297
  # If the resources are not there, check the class first.
296
- resources_annotation = self.model_fields["resources"].annotation
298
+ resources_annotation = (
299
+ type(self).model_fields["resources"].annotation
300
+ )
297
301
  assert resources_annotation is not None, (
298
302
  "For each response model, an annotated resources"
299
303
  "field should exist."
@@ -665,7 +665,7 @@ class BaseFilter(BaseModel):
665
665
  A list of Filter models.
666
666
  """
667
667
  return self._generate_filter_list(
668
- {key: getattr(self, key) for key in self.model_fields}
668
+ {key: getattr(self, key) for key in type(self).model_fields}
669
669
  )
670
670
 
671
671
  @property
@@ -45,6 +45,7 @@ from zenml.models.v2.base.scoped import (
45
45
  RunMetadataFilterMixin,
46
46
  TaggableFilter,
47
47
  )
48
+ from zenml.models.v2.core.logs import LogsRequest
48
49
  from zenml.models.v2.core.model_version import ModelVersionResponse
49
50
  from zenml.models.v2.core.tag import TagResponse
50
51
  from zenml.utils.tag_utils import Tag
@@ -55,6 +56,7 @@ if TYPE_CHECKING:
55
56
  from zenml.models import TriggerExecutionResponse
56
57
  from zenml.models.v2.core.artifact_version import ArtifactVersionResponse
57
58
  from zenml.models.v2.core.code_reference import CodeReferenceResponse
59
+ from zenml.models.v2.core.logs import LogsResponse
58
60
  from zenml.models.v2.core.pipeline import PipelineResponse
59
61
  from zenml.models.v2.core.pipeline_build import (
60
62
  PipelineBuildResponse,
@@ -124,6 +126,10 @@ class PipelineRunRequest(ProjectScopedRequest):
124
126
  default=None,
125
127
  title="Tags of the pipeline run.",
126
128
  )
129
+ logs: Optional[LogsRequest] = Field(
130
+ default=None,
131
+ title="Logs of the pipeline run.",
132
+ )
127
133
 
128
134
  model_config = ConfigDict(protected_namespaces=())
129
135
 
@@ -252,6 +258,10 @@ class PipelineRunResponseResources(ProjectScopedResponseResources):
252
258
  tags: List[TagResponse] = Field(
253
259
  title="Tags associated with the pipeline run.",
254
260
  )
261
+ logs: Optional["LogsResponse"] = Field(
262
+ title="Logs associated with this pipeline run.",
263
+ default=None,
264
+ )
255
265
 
256
266
  # TODO: In Pydantic v2, the `model_` is a protected namespaces for all
257
267
  # fields defined under base models. If not handled, this raises a warning.
@@ -579,6 +589,15 @@ class PipelineRunResponse(
579
589
  """
580
590
  return self.get_resources().tags
581
591
 
592
+ @property
593
+ def logs(self) -> Optional["LogsResponse"]:
594
+ """The `logs` property.
595
+
596
+ Returns:
597
+ the value of the property.
598
+ """
599
+ return self.get_resources().logs
600
+
582
601
 
583
602
  # ------------------ Filter Model ------------------
584
603
 
@@ -158,7 +158,7 @@ class StepLauncher:
158
158
  step_name=self._step_name,
159
159
  )
160
160
 
161
- logs_context = step_logging.StepLogsStorageContext(
161
+ logs_context = step_logging.PipelineLogsStorageContext(
162
162
  logs_uri=logs_uri, artifact_store=self._stack.artifact_store
163
163
  ) # type: ignore[assignment]
164
164
 
@@ -240,7 +240,7 @@ class StepLauncher:
240
240
  # the external jobs in step operators
241
241
  if isinstance(
242
242
  logs_context,
243
- step_logging.StepLogsStorageContext,
243
+ step_logging.PipelineLogsStorageContext,
244
244
  ):
245
245
  force_write_logs = partial(
246
246
  logs_context.storage.save_to_file,
@@ -421,7 +421,6 @@ class StepLauncher:
421
421
  )
422
422
  environment = orchestrator_utils.get_config_environment_vars(
423
423
  pipeline_run_id=step_run_info.run_id,
424
- step_run_id=step_run_info.step_run_id,
425
424
  )
426
425
  if last_retry:
427
426
  environment[ENV_ZENML_IGNORE_FAILURE_HOOK] = str(False)
@@ -40,7 +40,7 @@ from zenml.constants import (
40
40
  from zenml.enums import ArtifactSaveType
41
41
  from zenml.exceptions import StepInterfaceError
42
42
  from zenml.logger import get_logger
43
- from zenml.logging.step_logging import StepLogsStorageContext, redirected
43
+ from zenml.logging.step_logging import PipelineLogsStorageContext, redirected
44
44
  from zenml.materializers.base_materializer import BaseMaterializer
45
45
  from zenml.models.v2.core.step_run import StepRunInputResponse
46
46
  from zenml.orchestrators.publish_utils import (
@@ -136,7 +136,7 @@ class StepRunner:
136
136
  logs_context = nullcontext()
137
137
  if step_logging_enabled and not redirected.get():
138
138
  if step_run.logs:
139
- logs_context = StepLogsStorageContext( # type: ignore[assignment]
139
+ logs_context = PipelineLogsStorageContext( # type: ignore[assignment]
140
140
  logs_uri=step_run.logs.uri,
141
141
  artifact_store=self._stack.artifact_store,
142
142
  )
@@ -105,7 +105,6 @@ def is_setting_enabled(
105
105
  def get_config_environment_vars(
106
106
  schedule_id: Optional[UUID] = None,
107
107
  pipeline_run_id: Optional[UUID] = None,
108
- step_run_id: Optional[UUID] = None,
109
108
  ) -> Dict[str, str]:
110
109
  """Gets environment variables to set for mirroring the active config.
111
110
 
@@ -119,7 +118,6 @@ def get_config_environment_vars(
119
118
  schedule_id: Optional schedule ID to use to generate a new API token.
120
119
  pipeline_run_id: Optional pipeline run ID to use to generate a new API
121
120
  token.
122
- step_run_id: Optional step run ID to use to generate a new API token.
123
121
 
124
122
  Returns:
125
123
  Environment variable dict.
@@ -138,7 +136,7 @@ def get_config_environment_vars(
138
136
  credentials_store = get_credentials_store()
139
137
  url = global_config.store_configuration.url
140
138
  api_token = credentials_store.get_token(url, allow_expired=False)
141
- if schedule_id or pipeline_run_id or step_run_id:
139
+ if schedule_id or pipeline_run_id:
142
140
  assert isinstance(global_config.zen_store, RestZenStore)
143
141
 
144
142
  # The user has the option to manually set an expiration for the API
@@ -173,7 +171,7 @@ def get_config_environment_vars(
173
171
  # If only a schedule is given, the pipeline run credentials will
174
172
  # be valid for the entire duration of the schedule.
175
173
  api_key = credentials_store.get_api_key(url)
176
- if not api_key and not pipeline_run_id and not step_run_id:
174
+ if not api_key and not pipeline_run_id:
177
175
  logger.warning(
178
176
  "An API token without an expiration time will be generated "
179
177
  "and used to run this pipeline on a schedule. This is very "
@@ -194,7 +192,6 @@ def get_config_environment_vars(
194
192
  token_type=APITokenType.WORKLOAD,
195
193
  schedule_id=schedule_id,
196
194
  pipeline_run_id=pipeline_run_id,
197
- step_run_id=step_run_id,
198
195
  )
199
196
 
200
197
  environment_vars[ENV_ZENML_STORE_PREFIX + "API_TOKEN"] = (
@@ -109,6 +109,7 @@ class PipelineContext:
109
109
  pipeline_configuration.enable_artifact_visualization
110
110
  )
111
111
  self.enable_step_logs = pipeline_configuration.enable_step_logs
112
+ self.enable_pipeline_logs = pipeline_configuration.enable_pipeline_logs
112
113
  self.settings = pipeline_configuration.settings
113
114
  self.extra = pipeline_configuration.extra
114
115
  self.model = pipeline_configuration.model
@@ -50,6 +50,7 @@ def pipeline(
50
50
  enable_cache: Optional[bool] = None,
51
51
  enable_artifact_metadata: Optional[bool] = None,
52
52
  enable_step_logs: Optional[bool] = None,
53
+ enable_pipeline_logs: Optional[bool] = None,
53
54
  settings: Optional[Dict[str, "SettingsOrDict"]] = None,
54
55
  tags: Optional[List[Union[str, "Tag"]]] = None,
55
56
  extra: Optional[Dict[str, Any]] = None,
@@ -67,6 +68,7 @@ def pipeline(
67
68
  enable_cache: Optional[bool] = None,
68
69
  enable_artifact_metadata: Optional[bool] = None,
69
70
  enable_step_logs: Optional[bool] = None,
71
+ enable_pipeline_logs: Optional[bool] = None,
70
72
  settings: Optional[Dict[str, "SettingsOrDict"]] = None,
71
73
  tags: Optional[List[Union[str, "Tag"]]] = None,
72
74
  extra: Optional[Dict[str, Any]] = None,
@@ -84,6 +86,7 @@ def pipeline(
84
86
  enable_cache: Whether to use caching or not.
85
87
  enable_artifact_metadata: Whether to enable artifact metadata or not.
86
88
  enable_step_logs: If step logs should be enabled for this pipeline.
89
+ enable_pipeline_logs: If pipeline logs should be enabled for this pipeline.
87
90
  settings: Settings for this pipeline.
88
91
  tags: Tags to apply to runs of the pipeline.
89
92
  extra: Extra configurations for this pipeline.
@@ -108,6 +111,7 @@ def pipeline(
108
111
  enable_cache=enable_cache,
109
112
  enable_artifact_metadata=enable_artifact_metadata,
110
113
  enable_step_logs=enable_step_logs,
114
+ enable_pipeline_logs=enable_pipeline_logs,
111
115
  settings=settings,
112
116
  tags=tags,
113
117
  extra=extra,
@@ -16,7 +16,7 @@
16
16
  import copy
17
17
  import hashlib
18
18
  import inspect
19
- from contextlib import contextmanager
19
+ from contextlib import contextmanager, nullcontext
20
20
  from pathlib import Path
21
21
  from typing import (
22
22
  TYPE_CHECKING,
@@ -56,8 +56,13 @@ from zenml.enums import StackComponentType
56
56
  from zenml.exceptions import EntityExistsError
57
57
  from zenml.hooks.hook_validators import resolve_and_validate_hook
58
58
  from zenml.logger import get_logger
59
+ from zenml.logging.step_logging import (
60
+ PipelineLogsStorageContext,
61
+ prepare_logs_uri,
62
+ )
59
63
  from zenml.models import (
60
64
  CodeReferenceRequest,
65
+ LogsRequest,
61
66
  PipelineBuildBase,
62
67
  PipelineBuildResponse,
63
68
  PipelineDeploymentBase,
@@ -130,6 +135,7 @@ class Pipeline:
130
135
  enable_artifact_metadata: Optional[bool] = None,
131
136
  enable_artifact_visualization: Optional[bool] = None,
132
137
  enable_step_logs: Optional[bool] = None,
138
+ enable_pipeline_logs: Optional[bool] = None,
133
139
  settings: Optional[Mapping[str, "SettingsOrDict"]] = None,
134
140
  tags: Optional[List[Union[str, "Tag"]]] = None,
135
141
  extra: Optional[Dict[str, Any]] = None,
@@ -149,6 +155,7 @@ class Pipeline:
149
155
  enable_artifact_visualization: If artifact visualization should be
150
156
  enabled for this pipeline.
151
157
  enable_step_logs: If step logs should be enabled for this pipeline.
158
+ enable_pipeline_logs: If pipeline logs should be enabled for this pipeline.
152
159
  settings: Settings for this pipeline.
153
160
  tags: Tags to apply to runs of this pipeline.
154
161
  extra: Extra configurations for this pipeline.
@@ -174,6 +181,7 @@ class Pipeline:
174
181
  enable_artifact_metadata=enable_artifact_metadata,
175
182
  enable_artifact_visualization=enable_artifact_visualization,
176
183
  enable_step_logs=enable_step_logs,
184
+ enable_pipeline_logs=enable_pipeline_logs,
177
185
  settings=settings,
178
186
  tags=tags,
179
187
  extra=extra,
@@ -293,6 +301,7 @@ class Pipeline:
293
301
  enable_artifact_metadata: Optional[bool] = None,
294
302
  enable_artifact_visualization: Optional[bool] = None,
295
303
  enable_step_logs: Optional[bool] = None,
304
+ enable_pipeline_logs: Optional[bool] = None,
296
305
  settings: Optional[Mapping[str, "SettingsOrDict"]] = None,
297
306
  tags: Optional[List[Union[str, "Tag"]]] = None,
298
307
  extra: Optional[Dict[str, Any]] = None,
@@ -322,6 +331,7 @@ class Pipeline:
322
331
  enable_artifact_visualization: If artifact visualization should be
323
332
  enabled for this pipeline.
324
333
  enable_step_logs: If step logs should be enabled for this pipeline.
334
+ enable_pipeline_logs: If pipeline logs should be enabled for this pipeline.
325
335
  settings: settings for this pipeline.
326
336
  tags: Tags to apply to runs of this pipeline.
327
337
  extra: Extra configurations for this pipeline.
@@ -364,6 +374,7 @@ class Pipeline:
364
374
  "enable_artifact_metadata": enable_artifact_metadata,
365
375
  "enable_artifact_visualization": enable_artifact_visualization,
366
376
  "enable_step_logs": enable_step_logs,
377
+ "enable_pipeline_logs": enable_pipeline_logs,
367
378
  "settings": settings,
368
379
  "tags": tags,
369
380
  "extra": extra,
@@ -588,6 +599,7 @@ To avoid this consider setting pipeline parameters only in one place (config or
588
599
  enable_artifact_metadata: Optional[bool] = None,
589
600
  enable_artifact_visualization: Optional[bool] = None,
590
601
  enable_step_logs: Optional[bool] = None,
602
+ enable_pipeline_logs: Optional[bool] = None,
591
603
  schedule: Optional[Schedule] = None,
592
604
  build: Union[str, "UUID", "PipelineBuildBase", None] = None,
593
605
  settings: Optional[Mapping[str, "SettingsOrDict"]] = None,
@@ -610,6 +622,8 @@ To avoid this consider setting pipeline parameters only in one place (config or
610
622
  enable_artifact_visualization: If artifact visualization should be
611
623
  enabled for this pipeline run.
612
624
  enable_step_logs: If step logs should be enabled for this pipeline.
625
+ enable_pipeline_logs: If pipeline logs should be enabled for this
626
+ pipeline run.
613
627
  schedule: Optional schedule to use for the run.
614
628
  build: Optional build to use for the run.
615
629
  settings: Settings for this pipeline run.
@@ -641,6 +655,7 @@ To avoid this consider setting pipeline parameters only in one place (config or
641
655
  enable_artifact_metadata=enable_artifact_metadata,
642
656
  enable_artifact_visualization=enable_artifact_visualization,
643
657
  enable_step_logs=enable_step_logs,
658
+ enable_pipeline_logs=enable_pipeline_logs,
644
659
  steps=step_configurations,
645
660
  settings=settings,
646
661
  schedule=schedule,
@@ -723,7 +738,7 @@ To avoid this consider setting pipeline parameters only in one place (config or
723
738
  if prevent_build_reuse:
724
739
  logger.warning(
725
740
  "Passing `prevent_build_reuse=True` to "
726
- "`pipeline.with_opitions(...)` is deprecated. Use "
741
+ "`pipeline.with_options(...)` is deprecated. Use "
727
742
  "`DockerSettings.prevent_build_reuse` instead."
728
743
  )
729
744
 
@@ -806,31 +821,77 @@ To avoid this consider setting pipeline parameters only in one place (config or
806
821
 
807
822
  with track_handler(AnalyticsEvent.RUN_PIPELINE) as analytics_handler:
808
823
  stack = Client().active_stack
809
- deployment = self._create_deployment(**self._run_args)
810
824
 
811
- self.log_pipeline_deployment_metadata(deployment)
812
- run = create_placeholder_run(deployment=deployment)
825
+ # Enable or disable pipeline run logs storage
826
+ if self._run_args.get("schedule"):
827
+ # Pipeline runs scheduled to run in the future are not logged
828
+ # via the client.
829
+ logging_enabled = False
830
+ elif constants.handle_bool_env_var(
831
+ constants.ENV_ZENML_DISABLE_PIPELINE_LOGS_STORAGE, False
832
+ ):
833
+ logging_enabled = False
834
+ else:
835
+ logging_enabled = self._run_args.get(
836
+ "enable_pipeline_logs",
837
+ self.configuration.enable_pipeline_logs
838
+ if self.configuration.enable_pipeline_logs is not None
839
+ else True,
840
+ )
813
841
 
814
- analytics_handler.metadata = self._get_pipeline_analytics_metadata(
815
- deployment=deployment,
816
- stack=stack,
817
- run_id=run.id if run else None,
818
- )
842
+ logs_context = nullcontext()
843
+ logs_model = None
819
844
 
820
- if run:
821
- run_url = dashboard_utils.get_run_url(run)
822
- if run_url:
823
- logger.info(f"Dashboard URL for Pipeline Run: {run_url}")
824
- else:
825
- logger.info(
826
- "You can visualize your pipeline runs in the `ZenML "
827
- "Dashboard`. In order to try it locally, please run "
828
- "`zenml login --local`."
845
+ if logging_enabled:
846
+ # Configure the logs
847
+ logs_uri = prepare_logs_uri(
848
+ stack.artifact_store,
849
+ )
850
+
851
+ logs_context = PipelineLogsStorageContext(
852
+ logs_uri=logs_uri,
853
+ artifact_store=stack.artifact_store,
854
+ prepend_step_name=False,
855
+ ) # type: ignore[assignment]
856
+
857
+ logs_model = LogsRequest(
858
+ uri=logs_uri,
859
+ artifact_store_id=stack.artifact_store.id,
860
+ )
861
+
862
+ with logs_context:
863
+ deployment = self._create_deployment(**self._run_args)
864
+
865
+ self.log_pipeline_deployment_metadata(deployment)
866
+ run = create_placeholder_run(
867
+ deployment=deployment, logs=logs_model
868
+ )
869
+
870
+ analytics_handler.metadata = (
871
+ self._get_pipeline_analytics_metadata(
872
+ deployment=deployment,
873
+ stack=stack,
874
+ run_id=run.id if run else None,
829
875
  )
876
+ )
877
+
878
+ if run:
879
+ run_url = dashboard_utils.get_run_url(run)
880
+ if run_url:
881
+ logger.info(
882
+ f"Dashboard URL for Pipeline Run: {run_url}"
883
+ )
884
+ else:
885
+ logger.info(
886
+ "You can visualize your pipeline runs in the `ZenML "
887
+ "Dashboard`. In order to try it locally, please run "
888
+ "`zenml login --local`."
889
+ )
890
+
891
+ deploy_pipeline(
892
+ deployment=deployment, stack=stack, placeholder_run=run
893
+ )
830
894
 
831
- deploy_pipeline(
832
- deployment=deployment, stack=stack, placeholder_run=run
833
- )
834
895
  if run:
835
896
  return Client().get_pipeline_run(run.id)
836
897
  return None