zenml-nightly 0.62.0.dev20240729__py3-none-any.whl → 0.62.0.dev20240730__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. zenml/VERSION +1 -1
  2. zenml/actions/pipeline_run/pipeline_run_action.py +19 -17
  3. zenml/analytics/enums.py +4 -0
  4. zenml/cli/__init__.py +28 -15
  5. zenml/cli/base.py +1 -1
  6. zenml/cli/pipeline.py +54 -61
  7. zenml/cli/stack.py +6 -8
  8. zenml/client.py +232 -99
  9. zenml/config/compiler.py +14 -22
  10. zenml/config/pipeline_run_configuration.py +3 -0
  11. zenml/config/server_config.py +3 -0
  12. zenml/config/source.py +2 -1
  13. zenml/constants.py +2 -0
  14. zenml/enums.py +3 -0
  15. zenml/integrations/aws/orchestrators/sagemaker_orchestrator.py +13 -4
  16. zenml/integrations/databricks/flavors/databricks_orchestrator_flavor.py +11 -2
  17. zenml/integrations/databricks/orchestrators/databricks_orchestrator.py +19 -13
  18. zenml/models/__init__.py +26 -10
  19. zenml/models/v2/base/filter.py +32 -0
  20. zenml/models/v2/core/pipeline.py +73 -89
  21. zenml/models/v2/core/pipeline_build.py +15 -11
  22. zenml/models/v2/core/pipeline_deployment.py +56 -0
  23. zenml/models/v2/core/pipeline_run.py +52 -1
  24. zenml/models/v2/core/run_template.py +393 -0
  25. zenml/models/v2/misc/stack_deployment.py +5 -0
  26. zenml/new/pipelines/build_utils.py +34 -58
  27. zenml/new/pipelines/pipeline.py +17 -76
  28. zenml/new/pipelines/run_utils.py +12 -0
  29. zenml/post_execution/pipeline.py +1 -4
  30. zenml/service_connectors/service_connector_utils.py +4 -2
  31. zenml/stack_deployments/aws_stack_deployment.py +6 -5
  32. zenml/stack_deployments/azure_stack_deployment.py +118 -11
  33. zenml/stack_deployments/gcp_stack_deployment.py +12 -5
  34. zenml/stack_deployments/stack_deployment.py +6 -5
  35. zenml/steps/utils.py +0 -4
  36. zenml/utils/package_utils.py +39 -0
  37. zenml/zen_server/rbac/models.py +1 -0
  38. zenml/zen_server/rbac/utils.py +4 -0
  39. zenml/zen_server/routers/pipeline_builds_endpoints.py +2 -66
  40. zenml/zen_server/routers/pipeline_deployments_endpoints.py +2 -53
  41. zenml/zen_server/routers/pipelines_endpoints.py +1 -74
  42. zenml/zen_server/routers/run_templates_endpoints.py +212 -0
  43. zenml/zen_server/routers/workspaces_endpoints.py +79 -0
  44. zenml/zen_server/{pipeline_deployment → template_execution}/runner_entrypoint_configuration.py +1 -8
  45. zenml/zen_server/{pipeline_deployment → template_execution}/utils.py +214 -92
  46. zenml/zen_server/utils.py +2 -2
  47. zenml/zen_server/zen_server_api.py +2 -1
  48. zenml/zen_stores/migrations/versions/7d1919bb1ef0_add_run_templates.py +100 -0
  49. zenml/zen_stores/migrations/versions/b59aa68fdb1f_simplify_pipelines.py +139 -0
  50. zenml/zen_stores/rest_zen_store.py +107 -36
  51. zenml/zen_stores/schemas/__init__.py +2 -0
  52. zenml/zen_stores/schemas/pipeline_build_schemas.py +3 -3
  53. zenml/zen_stores/schemas/pipeline_deployment_schemas.py +29 -2
  54. zenml/zen_stores/schemas/pipeline_run_schemas.py +26 -3
  55. zenml/zen_stores/schemas/pipeline_schemas.py +29 -30
  56. zenml/zen_stores/schemas/run_template_schemas.py +264 -0
  57. zenml/zen_stores/schemas/step_run_schemas.py +11 -4
  58. zenml/zen_stores/sql_zen_store.py +364 -150
  59. zenml/zen_stores/template_utils.py +261 -0
  60. zenml/zen_stores/zen_store_interface.py +93 -20
  61. {zenml_nightly-0.62.0.dev20240729.dist-info → zenml_nightly-0.62.0.dev20240730.dist-info}/METADATA +1 -1
  62. {zenml_nightly-0.62.0.dev20240729.dist-info → zenml_nightly-0.62.0.dev20240730.dist-info}/RECORD +67 -63
  63. zenml/models/v2/core/pipeline_namespace.py +0 -113
  64. zenml/new/pipelines/deserialization_utils.py +0 -292
  65. /zenml/zen_server/{pipeline_deployment → template_execution}/__init__.py +0 -0
  66. /zenml/zen_server/{pipeline_deployment → template_execution}/workload_manager_interface.py +0 -0
  67. {zenml_nightly-0.62.0.dev20240729.dist-info → zenml_nightly-0.62.0.dev20240730.dist-info}/LICENSE +0 -0
  68. {zenml_nightly-0.62.0.dev20240729.dist-info → zenml_nightly-0.62.0.dev20240730.dist-info}/WHEEL +0 -0
  69. {zenml_nightly-0.62.0.dev20240729.dist-info → zenml_nightly-0.62.0.dev20240730.dist-info}/entry_points.txt +0 -0
zenml/client.py CHANGED
@@ -139,6 +139,10 @@ from zenml.models import (
139
139
  RunMetadataFilter,
140
140
  RunMetadataRequest,
141
141
  RunMetadataResponse,
142
+ RunTemplateFilter,
143
+ RunTemplateRequest,
144
+ RunTemplateResponse,
145
+ RunTemplateUpdate,
142
146
  ScheduleFilter,
143
147
  ScheduleResponse,
144
148
  SecretFilter,
@@ -2350,11 +2354,9 @@ class Client(metaclass=ClientMetaClass):
2350
2354
  created: Optional[Union[datetime, str]] = None,
2351
2355
  updated: Optional[Union[datetime, str]] = None,
2352
2356
  name: Optional[str] = None,
2353
- version: Optional[str] = None,
2354
- version_hash: Optional[str] = None,
2355
- docstring: Optional[str] = None,
2356
2357
  workspace_id: Optional[Union[str, UUID]] = None,
2357
2358
  user_id: Optional[Union[str, UUID]] = None,
2359
+ tag: Optional[str] = None,
2358
2360
  hydrate: bool = False,
2359
2361
  ) -> Page[PipelineResponse]:
2360
2362
  """List all pipelines.
@@ -2368,11 +2370,9 @@ class Client(metaclass=ClientMetaClass):
2368
2370
  created: Use to filter by time of creation
2369
2371
  updated: Use the last updated date for filtering
2370
2372
  name: The name of the pipeline to filter by.
2371
- version: The version of the pipeline to filter by.
2372
- version_hash: The version hash of the pipeline to filter by.
2373
- docstring: The docstring of the pipeline to filter by.
2374
2373
  workspace_id: The id of the workspace to filter by.
2375
2374
  user_id: The id of the user to filter by.
2375
+ tag: Tag to filter by.
2376
2376
  hydrate: Flag deciding whether to hydrate the output model(s)
2377
2377
  by including metadata fields in the response.
2378
2378
 
@@ -2388,11 +2388,9 @@ class Client(metaclass=ClientMetaClass):
2388
2388
  created=created,
2389
2389
  updated=updated,
2390
2390
  name=name,
2391
- version=version,
2392
- version_hash=version_hash,
2393
- docstring=docstring,
2394
2391
  workspace_id=workspace_id,
2395
2392
  user_id=user_id,
2393
+ tag=tag,
2396
2394
  )
2397
2395
  pipeline_filter_model.set_scope_workspace(self.active_workspace.id)
2398
2396
  return self.zen_store.list_pipelines(
@@ -2403,124 +2401,83 @@ class Client(metaclass=ClientMetaClass):
2403
2401
  def get_pipeline(
2404
2402
  self,
2405
2403
  name_id_or_prefix: Union[str, UUID],
2406
- version: Optional[str] = None,
2407
2404
  hydrate: bool = True,
2408
2405
  ) -> PipelineResponse:
2409
2406
  """Get a pipeline by name, id or prefix.
2410
2407
 
2411
2408
  Args:
2412
2409
  name_id_or_prefix: The name, ID or ID prefix of the pipeline.
2413
- version: The pipeline version. If not specified, the latest
2414
- version is returned.
2415
2410
  hydrate: Flag deciding whether to hydrate the output model(s)
2416
2411
  by including metadata fields in the response.
2417
2412
 
2418
2413
  Returns:
2419
2414
  The pipeline.
2420
2415
  """
2421
- return self._get_entity_version_by_id_or_name_or_prefix(
2416
+ return self._get_entity_by_id_or_name_or_prefix(
2422
2417
  get_method=self.zen_store.get_pipeline,
2423
2418
  list_method=self.list_pipelines,
2424
2419
  name_id_or_prefix=name_id_or_prefix,
2425
- version=version,
2426
2420
  hydrate=hydrate,
2427
2421
  )
2428
2422
 
2429
2423
  def delete_pipeline(
2430
2424
  self,
2431
2425
  name_id_or_prefix: Union[str, UUID],
2432
- version: Optional[str] = None,
2433
- all_versions: bool = False,
2434
2426
  ) -> None:
2435
2427
  """Delete a pipeline.
2436
2428
 
2437
2429
  Args:
2438
2430
  name_id_or_prefix: The name, ID or ID prefix of the pipeline.
2439
- version: The pipeline version. If left empty, will delete
2440
- the latest version.
2441
- all_versions: If `True`, delete all versions of the pipeline.
2442
-
2443
- Raises:
2444
- ValueError: If an ID is supplied when trying to delete all versions
2445
- of a pipeline.
2446
2431
  """
2447
- if all_versions:
2448
- if is_valid_uuid(name_id_or_prefix):
2449
- raise ValueError(
2450
- "You need to supply a name (not an ID) when trying to "
2451
- "delete all versions of a pipeline."
2452
- )
2453
-
2454
- for pipeline in depaginate(
2455
- Client().list_pipelines, name=name_id_or_prefix
2456
- ):
2457
- Client().delete_pipeline(pipeline.id)
2458
- else:
2459
- pipeline = self.get_pipeline(
2460
- name_id_or_prefix=name_id_or_prefix, version=version
2461
- )
2462
- self.zen_store.delete_pipeline(pipeline_id=pipeline.id)
2432
+ pipeline = self.get_pipeline(name_id_or_prefix=name_id_or_prefix)
2433
+ self.zen_store.delete_pipeline(pipeline_id=pipeline.id)
2463
2434
 
2464
2435
  @_fail_for_sql_zen_store
2465
2436
  def trigger_pipeline(
2466
2437
  self,
2467
2438
  pipeline_name_or_id: Union[str, UUID, None] = None,
2468
- pipeline_version: Optional[str] = None,
2469
2439
  run_configuration: Optional[PipelineRunConfiguration] = None,
2470
2440
  config_path: Optional[str] = None,
2471
- deployment_id: Optional[UUID] = None,
2472
- build_id: Optional[UUID] = None,
2441
+ template_id: Optional[UUID] = None,
2473
2442
  stack_name_or_id: Union[str, UUID, None] = None,
2474
2443
  synchronous: bool = False,
2475
2444
  ) -> PipelineRunResponse:
2476
2445
  """Trigger a pipeline from the server.
2477
2446
 
2478
2447
  Usage examples:
2479
- * Run the latest runnable build for the latest version of a pipeline:
2448
+ * Run the latest runnable template for a pipeline:
2480
2449
  ```python
2481
2450
  Client().trigger_pipeline(pipeline_name_or_id=<NAME>)
2482
2451
  ```
2483
- * Run the latest runnable build for a specific version of a pipeline:
2452
+ * Run the latest runnable template for a pipeline on a specific stack:
2484
2453
  ```python
2485
2454
  Client().trigger_pipeline(
2486
2455
  pipeline_name_or_id=<NAME>,
2487
- pipeline_version=<VERSION>
2488
- )
2489
- ```
2490
- * Run a specific pipeline version on a specific stack:
2491
- ```python
2492
- Client().trigger_pipeline(
2493
- pipeline_name_or_id=<ID>,
2494
- stack_name_or_id=<ID>
2456
+ stack_name_or_id=<STACK_NAME_OR_ID>
2495
2457
  )
2496
2458
  ```
2497
- * Run a specific deployment:
2459
+ * Run a specific template:
2498
2460
  ```python
2499
- Client().trigger_pipeline(deployment_id=<ID>)
2500
- ```
2501
- * Run a specific build:
2502
- ```python
2503
- Client().trigger_pipeline(build_id=<ID>)
2461
+ Client().trigger_pipeline(template_id=<ID>)
2504
2462
  ```
2505
2463
 
2506
2464
  Args:
2507
- pipeline_name_or_id: Name or ID of the pipeline. If not given,
2508
- either the build or deployment that should be run needs to be
2509
- specified.
2510
- pipeline_version: Version of the pipeline. This is only used if a
2511
- pipeline name is given.
2465
+ pipeline_name_or_id: Name or ID of the pipeline. If this is
2466
+ specified, the latest runnable template for this pipeline will
2467
+ be used for the run (Runnable here means that the build
2468
+ associated with the template is for a remote stack without any
2469
+ custom flavor stack components). If not given, a template ID
2470
+ that should be run needs to be specified.
2512
2471
  run_configuration: Configuration for the run. Either this or a
2513
2472
  path to a config file can be specified.
2514
2473
  config_path: Path to a YAML configuration file. This file will be
2515
2474
  parsed as a `PipelineRunConfiguration` object. Either this or
2516
2475
  the configuration in code can be specified.
2517
- deployment_id: ID of the deployment to run. Either this or a build
2518
- to run can be specified.
2519
- build_id: ID of the build to run. Either this or a deployment to
2520
- run can be specified.
2476
+ template_id: ID of the template to run. Either this or a pipeline
2477
+ can be specified.
2521
2478
  stack_name_or_id: Name or ID of the stack on which to run the
2522
2479
  pipeline. If not specified, this method will try to find a
2523
- runnable build on any stack.
2480
+ runnable template on any stack.
2524
2481
  synchronous: If `True`, this method will wait until the triggered
2525
2482
  run is finished.
2526
2483
 
@@ -2536,10 +2493,10 @@ class Client(metaclass=ClientMetaClass):
2536
2493
  wait_for_pipeline_run_to_finish,
2537
2494
  )
2538
2495
 
2539
- if Counter([build_id, deployment_id, pipeline_name_or_id])[None] != 2:
2496
+ if Counter([template_id, pipeline_name_or_id])[None] != 1:
2540
2497
  raise RuntimeError(
2541
- "You need to specify exactly one of pipeline, build or "
2542
- "deployment to trigger."
2498
+ "You need to specify exactly one of pipeline or template "
2499
+ "to trigger."
2543
2500
  )
2544
2501
 
2545
2502
  if run_configuration and config_path:
@@ -2553,32 +2510,20 @@ class Client(metaclass=ClientMetaClass):
2553
2510
  if run_configuration:
2554
2511
  validate_run_config_is_runnable_from_server(run_configuration)
2555
2512
 
2556
- if deployment_id:
2513
+ if template_id:
2557
2514
  if stack_name_or_id:
2558
2515
  logger.warning(
2559
- "Deployment ID and stack specified, ignoring the stack and "
2560
- "using stack from deployment instead."
2516
+ "Template ID and stack specified, ignoring the stack and "
2517
+ "using stack associated with the template instead."
2561
2518
  )
2562
2519
 
2563
- run = self.zen_store.run_deployment(
2564
- deployment_id=deployment_id,
2520
+ run = self.zen_store.run_template(
2521
+ template_id=template_id,
2565
2522
  run_configuration=run_configuration,
2566
2523
  )
2567
- elif build_id:
2568
- if stack_name_or_id:
2569
- logger.warning(
2570
- "Build ID and stack specified, ignoring the stack and "
2571
- "using stack from build instead."
2572
- )
2573
-
2574
- run = self.zen_store.run_build(
2575
- build_id=build_id, run_configuration=run_configuration
2576
- )
2577
2524
  else:
2578
2525
  assert pipeline_name_or_id
2579
- pipeline = self.get_pipeline(
2580
- name_id_or_prefix=pipeline_name_or_id, version=pipeline_version
2581
- )
2526
+ pipeline = self.get_pipeline(name_id_or_prefix=pipeline_name_or_id)
2582
2527
 
2583
2528
  stack = None
2584
2529
  if stack_name_or_id:
@@ -2589,34 +2534,36 @@ class Client(metaclass=ClientMetaClass):
2589
2534
  zen_store=self.zen_store, stack=stack
2590
2535
  )
2591
2536
 
2592
- builds = depaginate(
2593
- self.list_builds,
2537
+ templates = depaginate(
2538
+ self.list_run_templates,
2594
2539
  pipeline_id=pipeline.id,
2595
2540
  stack_id=stack.id if stack else None,
2596
2541
  )
2597
2542
 
2598
- for build in builds:
2599
- if not build.template_deployment_id:
2543
+ for template in templates:
2544
+ if not template.build:
2600
2545
  continue
2601
2546
 
2602
- if not build.stack:
2547
+ stack = template.build.stack
2548
+ if not stack:
2603
2549
  continue
2604
2550
 
2605
2551
  try:
2606
2552
  validate_stack_is_runnable_from_server(
2607
- zen_store=self.zen_store, stack=build.stack
2553
+ zen_store=self.zen_store, stack=stack
2608
2554
  )
2609
2555
  except ValueError:
2610
2556
  continue
2611
2557
 
2612
- run = self.zen_store.run_build(
2613
- build_id=build.id, run_configuration=run_configuration
2558
+ run = self.zen_store.run_template(
2559
+ template_id=template.id,
2560
+ run_configuration=run_configuration,
2614
2561
  )
2615
2562
  break
2616
2563
  else:
2617
2564
  raise RuntimeError(
2618
- "Unable to find a runnable build for the given stack and "
2619
- "pipeline."
2565
+ "Unable to find a runnable template for the given stack "
2566
+ "and pipeline."
2620
2567
  )
2621
2568
 
2622
2569
  if synchronous:
@@ -3400,6 +3347,7 @@ class Client(metaclass=ClientMetaClass):
3400
3347
  pipeline_id: Optional[Union[str, UUID]] = None,
3401
3348
  stack_id: Optional[Union[str, UUID]] = None,
3402
3349
  build_id: Optional[Union[str, UUID]] = None,
3350
+ template_id: Optional[Union[str, UUID]] = None,
3403
3351
  hydrate: bool = False,
3404
3352
  ) -> Page[PipelineDeploymentResponse]:
3405
3353
  """List all deployments.
@@ -3417,6 +3365,7 @@ class Client(metaclass=ClientMetaClass):
3417
3365
  pipeline_id: The id of the pipeline to filter by.
3418
3366
  stack_id: The id of the stack to filter by.
3419
3367
  build_id: The id of the build to filter by.
3368
+ template_id: The ID of the template to filter by.
3420
3369
  hydrate: Flag deciding whether to hydrate the output model(s)
3421
3370
  by including metadata fields in the response.
3422
3371
 
@@ -3436,6 +3385,7 @@ class Client(metaclass=ClientMetaClass):
3436
3385
  pipeline_id=pipeline_id,
3437
3386
  stack_id=stack_id,
3438
3387
  build_id=build_id,
3388
+ template_id=template_id,
3439
3389
  )
3440
3390
  deployment_filter_model.set_scope_workspace(self.active_workspace.id)
3441
3391
  return self.zen_store.list_deployments(
@@ -3452,6 +3402,183 @@ class Client(metaclass=ClientMetaClass):
3452
3402
  deployment = self.get_deployment(id_or_prefix=id_or_prefix)
3453
3403
  self.zen_store.delete_deployment(deployment_id=deployment.id)
3454
3404
 
3405
+ # ------------------------------ Run templates -----------------------------
3406
+
3407
+ def create_run_template(
3408
+ self,
3409
+ name: str,
3410
+ deployment_id: UUID,
3411
+ description: Optional[str] = None,
3412
+ tags: Optional[List[str]] = None,
3413
+ ) -> RunTemplateResponse:
3414
+ """Create a run template.
3415
+
3416
+ Args:
3417
+ name: The name of the run template.
3418
+ deployment_id: ID of the deployment which this template should be
3419
+ based off of.
3420
+ description: The description of the run template.
3421
+ tags: Tags associated with the run template.
3422
+
3423
+ Returns:
3424
+ The created run template.
3425
+ """
3426
+ return self.zen_store.create_run_template(
3427
+ template=RunTemplateRequest(
3428
+ name=name,
3429
+ description=description,
3430
+ source_deployment_id=deployment_id,
3431
+ tags=tags,
3432
+ user=self.active_user.id,
3433
+ workspace=self.active_workspace.id,
3434
+ )
3435
+ )
3436
+
3437
+ def get_run_template(
3438
+ self,
3439
+ name_id_or_prefix: Union[str, UUID],
3440
+ hydrate: bool = True,
3441
+ ) -> RunTemplateResponse:
3442
+ """Get a run template.
3443
+
3444
+ Args:
3445
+ name_id_or_prefix: Name/ID/ID prefix of the template to get.
3446
+ hydrate: Flag deciding whether to hydrate the output model(s)
3447
+ by including metadata fields in the response.
3448
+
3449
+ Returns:
3450
+ The run template.
3451
+ """
3452
+ return self._get_entity_by_id_or_name_or_prefix(
3453
+ get_method=self.zen_store.get_run_template,
3454
+ list_method=self.list_run_templates,
3455
+ name_id_or_prefix=name_id_or_prefix,
3456
+ allow_name_prefix_match=False,
3457
+ hydrate=hydrate,
3458
+ )
3459
+
3460
+ def list_run_templates(
3461
+ self,
3462
+ sort_by: str = "created",
3463
+ page: int = PAGINATION_STARTING_PAGE,
3464
+ size: int = PAGE_SIZE_DEFAULT,
3465
+ logical_operator: LogicalOperators = LogicalOperators.AND,
3466
+ created: Optional[Union[datetime, str]] = None,
3467
+ updated: Optional[Union[datetime, str]] = None,
3468
+ name: Optional[str] = None,
3469
+ tag: Optional[str] = None,
3470
+ workspace_id: Optional[Union[str, UUID]] = None,
3471
+ user_id: Optional[Union[str, UUID]] = None,
3472
+ pipeline_id: Optional[Union[str, UUID]] = None,
3473
+ build_id: Optional[Union[str, UUID]] = None,
3474
+ stack_id: Optional[Union[str, UUID]] = None,
3475
+ code_repository_id: Optional[Union[str, UUID]] = None,
3476
+ hydrate: bool = False,
3477
+ ) -> Page[RunTemplateResponse]:
3478
+ """Get a page of run templates.
3479
+
3480
+ Args:
3481
+ sort_by: The column to sort by.
3482
+ page: The page of items.
3483
+ size: The maximum size of all pages.
3484
+ logical_operator: Which logical operator to use [and, or].
3485
+ created: Filter by the creation date.
3486
+ updated: Filter by the last updated date.
3487
+ name: Filter by run template name.
3488
+ tag: Filter by run template tags.
3489
+ workspace_id: Filter by workspace ID.
3490
+ user_id: Filter by user ID.
3491
+ pipeline_id: Filter by pipeline ID.
3492
+ build_id: Filter by build ID.
3493
+ stack_id: Filter by stack ID.
3494
+ code_repository_id: Filter by code repository ID.
3495
+ hydrate: Flag deciding whether to hydrate the output model(s)
3496
+ by including metadata fields in the response.
3497
+
3498
+ Returns:
3499
+ A page of run templates.
3500
+ """
3501
+ filter = RunTemplateFilter(
3502
+ sort_by=sort_by,
3503
+ page=page,
3504
+ size=size,
3505
+ logical_operator=logical_operator,
3506
+ created=created,
3507
+ updated=updated,
3508
+ name=name,
3509
+ tag=tag,
3510
+ workspace_id=workspace_id,
3511
+ user_id=user_id,
3512
+ pipeline_id=pipeline_id,
3513
+ build_id=build_id,
3514
+ stack_id=stack_id,
3515
+ code_repository_id=code_repository_id,
3516
+ )
3517
+
3518
+ return self.zen_store.list_run_templates(
3519
+ template_filter_model=filter, hydrate=hydrate
3520
+ )
3521
+
3522
+ def update_run_template(
3523
+ self,
3524
+ name_id_or_prefix: Union[str, UUID],
3525
+ name: Optional[str] = None,
3526
+ description: Optional[str] = None,
3527
+ add_tags: Optional[List[str]] = None,
3528
+ remove_tags: Optional[List[str]] = None,
3529
+ ) -> RunTemplateResponse:
3530
+ """Update a run template.
3531
+
3532
+ Args:
3533
+ name_id_or_prefix: Name/ID/ID prefix of the template to update.
3534
+ name: The new name of the run template.
3535
+ description: The new description of the run template.
3536
+ add_tags: Tags to add to the run template.
3537
+ remove_tags: Tags to remove from the run template.
3538
+
3539
+ Returns:
3540
+ The updated run template.
3541
+ """
3542
+ if is_valid_uuid(name_id_or_prefix):
3543
+ template_id = (
3544
+ UUID(name_id_or_prefix)
3545
+ if isinstance(name_id_or_prefix, str)
3546
+ else name_id_or_prefix
3547
+ )
3548
+ else:
3549
+ template_id = self.get_run_template(
3550
+ name_id_or_prefix, hydrate=False
3551
+ ).id
3552
+
3553
+ return self.zen_store.update_run_template(
3554
+ template_id=template_id,
3555
+ template_update=RunTemplateUpdate(
3556
+ name=name,
3557
+ description=description,
3558
+ add_tags=add_tags,
3559
+ remove_tags=remove_tags,
3560
+ ),
3561
+ )
3562
+
3563
+ def delete_run_template(self, name_id_or_prefix: Union[str, UUID]) -> None:
3564
+ """Delete a run template.
3565
+
3566
+ Args:
3567
+ name_id_or_prefix: Name/ID/ID prefix of the template to delete.
3568
+ """
3569
+ if is_valid_uuid(name_id_or_prefix):
3570
+ template_id = (
3571
+ UUID(name_id_or_prefix)
3572
+ if isinstance(name_id_or_prefix, str)
3573
+ else name_id_or_prefix
3574
+ )
3575
+ else:
3576
+ template_id = self.get_run_template(
3577
+ name_id_or_prefix, hydrate=False
3578
+ ).id
3579
+
3580
+ self.zen_store.delete_run_template(template_id=template_id)
3581
+
3455
3582
  # ------------------------------- Schedules --------------------------------
3456
3583
 
3457
3584
  def get_schedule(
@@ -3620,12 +3747,14 @@ class Client(metaclass=ClientMetaClass):
3620
3747
  build_id: Optional[Union[str, UUID]] = None,
3621
3748
  deployment_id: Optional[Union[str, UUID]] = None,
3622
3749
  code_repository_id: Optional[Union[str, UUID]] = None,
3750
+ template_id: Optional[Union[str, UUID]] = None,
3623
3751
  orchestrator_run_id: Optional[str] = None,
3624
3752
  status: Optional[str] = None,
3625
3753
  start_time: Optional[Union[datetime, str]] = None,
3626
3754
  end_time: Optional[Union[datetime, str]] = None,
3627
3755
  num_steps: Optional[Union[int, str]] = None,
3628
3756
  unlisted: Optional[bool] = None,
3757
+ tag: Optional[str] = None,
3629
3758
  hydrate: bool = False,
3630
3759
  ) -> Page[PipelineRunResponse]:
3631
3760
  """List all pipeline runs.
@@ -3647,6 +3776,7 @@ class Client(metaclass=ClientMetaClass):
3647
3776
  build_id: The id of the build to filter by.
3648
3777
  deployment_id: The id of the deployment to filter by.
3649
3778
  code_repository_id: The id of the code repository to filter by.
3779
+ template_id: The ID of the template to filter by.
3650
3780
  orchestrator_run_id: The run id of the orchestrator to filter by.
3651
3781
  name: The name of the run to filter by.
3652
3782
  status: The status of the pipeline run
@@ -3654,6 +3784,7 @@ class Client(metaclass=ClientMetaClass):
3654
3784
  end_time: The end_time for the pipeline run
3655
3785
  num_steps: The number of steps for the pipeline run
3656
3786
  unlisted: If the runs should be unlisted or not.
3787
+ tag: Tag to filter by.
3657
3788
  hydrate: Flag deciding whether to hydrate the output model(s)
3658
3789
  by including metadata fields in the response.
3659
3790
 
@@ -3676,6 +3807,7 @@ class Client(metaclass=ClientMetaClass):
3676
3807
  build_id=build_id,
3677
3808
  deployment_id=deployment_id,
3678
3809
  code_repository_id=code_repository_id,
3810
+ template_id=template_id,
3679
3811
  orchestrator_run_id=orchestrator_run_id,
3680
3812
  user_id=user_id,
3681
3813
  stack_id=stack_id,
@@ -3683,6 +3815,7 @@ class Client(metaclass=ClientMetaClass):
3683
3815
  start_time=start_time,
3684
3816
  end_time=end_time,
3685
3817
  num_steps=num_steps,
3818
+ tag=tag,
3686
3819
  unlisted=unlisted,
3687
3820
  )
3688
3821
  runs_filter_model.set_scope_workspace(self.active_workspace.id)
zenml/config/compiler.py CHANGED
@@ -39,6 +39,7 @@ from zenml.config.step_configurations import (
39
39
  from zenml.environment import get_run_environment_dict
40
40
  from zenml.exceptions import StackValidationError
41
41
  from zenml.models import PipelineDeploymentBase
42
+ from zenml.new.pipelines.run_utils import get_default_run_name
42
43
  from zenml.utils import pydantic_utils, settings_utils
43
44
 
44
45
  if TYPE_CHECKING:
@@ -74,7 +75,7 @@ class Compiler:
74
75
  pipeline: "Pipeline",
75
76
  stack: "Stack",
76
77
  run_configuration: PipelineRunConfiguration,
77
- ) -> Tuple[PipelineDeploymentBase, PipelineSpec]:
78
+ ) -> PipelineDeploymentBase:
78
79
  """Compiles a ZenML pipeline to a serializable representation.
79
80
 
80
81
  Args:
@@ -83,7 +84,7 @@ class Compiler:
83
84
  run_configuration: The run configuration for this pipeline.
84
85
 
85
86
  Returns:
86
- The compiled pipeline deployment and spec
87
+ The compiled pipeline deployment.
87
88
  """
88
89
  logger.debug("Compiling pipeline `%s`.", pipeline.name)
89
90
  # Copy the pipeline before we apply any run-level configurations, so
@@ -127,12 +128,17 @@ class Compiler:
127
128
 
128
129
  self._ensure_required_stack_components_exist(stack=stack, steps=steps)
129
130
 
130
- run_name = run_configuration.run_name or self._get_default_run_name(
131
+ run_name = run_configuration.run_name or get_default_run_name(
131
132
  pipeline_name=pipeline.name
132
133
  )
133
134
 
134
135
  client_version, server_version = get_zenml_versions()
135
136
 
137
+ step_specs = [step.spec for step in steps.values()]
138
+ pipeline_spec = self._compute_pipeline_spec(
139
+ pipeline=pipeline, step_specs=step_specs
140
+ )
141
+
136
142
  deployment = PipelineDeploymentBase(
137
143
  run_name_template=run_name,
138
144
  pipeline_configuration=pipeline.configuration,
@@ -140,17 +146,15 @@ class Compiler:
140
146
  client_environment=get_run_environment_dict(),
141
147
  client_version=client_version,
142
148
  server_version=server_version,
143
- )
144
-
145
- step_specs = [step.spec for step in steps.values()]
146
- pipeline_spec = self._compute_pipeline_spec(
147
- pipeline=pipeline, step_specs=step_specs
149
+ pipeline_version_hash=pipeline._compute_unique_identifier(
150
+ pipeline_spec=pipeline_spec
151
+ ),
152
+ pipeline_spec=pipeline_spec,
148
153
  )
149
154
 
150
155
  logger.debug("Compiled pipeline deployment: %s", deployment)
151
- logger.debug("Compiled pipeline spec: %s", pipeline_spec)
152
156
 
153
- return deployment, pipeline_spec
157
+ return deployment
154
158
 
155
159
  def compile_spec(self, pipeline: "Pipeline") -> PipelineSpec:
156
160
  """Compiles a ZenML pipeline to a pipeline spec.
@@ -474,18 +478,6 @@ class Compiler:
474
478
  )
475
479
  return Step(spec=step_spec, config=complete_step_configuration)
476
480
 
477
- @staticmethod
478
- def _get_default_run_name(pipeline_name: str) -> str:
479
- """Gets the default name for a pipeline run.
480
-
481
- Args:
482
- pipeline_name: Name of the pipeline which will be run.
483
-
484
- Returns:
485
- Run name.
486
- """
487
- return f"{pipeline_name}-{{date}}-{{time}}"
488
-
489
481
  def _get_sorted_invocations(
490
482
  self,
491
483
  pipeline: "Pipeline",
@@ -21,6 +21,7 @@ from pydantic import Field, SerializeAsAny
21
21
  from zenml.config.base_settings import BaseSettings
22
22
  from zenml.config.retry_config import StepRetryConfig
23
23
  from zenml.config.schedule import Schedule
24
+ from zenml.config.source import SourceWithValidator
24
25
  from zenml.config.step_configurations import StepConfigurationUpdate
25
26
  from zenml.config.strict_base_model import StrictBaseModel
26
27
  from zenml.model.model import Model
@@ -48,3 +49,5 @@ class PipelineRunConfiguration(
48
49
  model: Optional[Model] = None
49
50
  parameters: Optional[Dict[str, Any]] = None
50
51
  retry: Optional[StepRetryConfig] = None
52
+ failure_hook_source: Optional[SourceWithValidator] = None
53
+ success_hook_source: Optional[SourceWithValidator] = None
@@ -65,6 +65,9 @@ def generate_jwt_secret_key() -> str:
65
65
  class ServerConfiguration(BaseModel):
66
66
  """ZenML Server configuration attributes.
67
67
 
68
+ All these attributes can be set through the environment with the `ZENML_SERVER_`-Prefix.
69
+ The value of the `ZENML_SERVER_DEPLOYMENT_TYPE` environment variable will be extracted to deployment_type.
70
+
68
71
  Attributes:
69
72
  deployment_type: The type of ZenML server deployment that is running.
70
73
  server_url: The URL where the ZenML server API is reachable. Must be
zenml/config/source.py CHANGED
@@ -245,5 +245,6 @@ def convert_source(source: Any) -> Any:
245
245
 
246
246
 
247
247
  SourceWithValidator = Annotated[
248
- SerializeAsAny[Source], BeforeValidator(convert_source)
248
+ SerializeAsAny[Source],
249
+ BeforeValidator(convert_source),
249
250
  ]
zenml/constants.py CHANGED
@@ -364,6 +364,7 @@ PIPELINES = "/pipelines"
364
364
  PIPELINE_SPEC = "/pipeline-spec"
365
365
  PLUGIN_FLAVORS = "/plugin-flavors"
366
366
  RUNS = "/runs"
367
+ RUN_TEMPLATES = "/run_templates"
367
368
  RUN_METADATA = "/run-metadata"
368
369
  SCHEDULES = "/schedules"
369
370
  SECRETS = "/secrets"
@@ -424,6 +425,7 @@ PAGE_SIZE_MAXIMUM: int = handle_int_env_var(
424
425
  ENV_ZENML_PAGINATION_DEFAULT_LIMIT, default=10000
425
426
  )
426
427
  FILTERING_DATETIME_FORMAT: str = "%Y-%m-%d %H:%M:%S"
428
+ SORT_PIPELINES_BY_LATEST_RUN_KEY = "latest_run"
427
429
 
428
430
  # Metadata constants
429
431
  METADATA_ORCHESTRATOR_URL = "orchestrator_url"