zenml-nightly 0.70.0.dev20241201__py3-none-any.whl → 0.71.0.dev20241223__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (193) hide show
  1. README.md +4 -4
  2. RELEASE_NOTES.md +112 -0
  3. zenml/VERSION +1 -1
  4. zenml/artifacts/artifact_config.py +8 -5
  5. zenml/artifacts/utils.py +3 -1
  6. zenml/cli/__init__.py +4 -4
  7. zenml/cli/base.py +1 -1
  8. zenml/cli/pipeline.py +48 -79
  9. zenml/cli/server.py +19 -19
  10. zenml/client.py +54 -2
  11. zenml/config/secret_reference_mixin.py +1 -1
  12. zenml/config/server_config.py +4 -0
  13. zenml/constants.py +10 -0
  14. zenml/image_builders/base_image_builder.py +5 -2
  15. zenml/image_builders/build_context.py +7 -16
  16. zenml/integrations/aws/__init__.py +3 -0
  17. zenml/integrations/aws/flavors/__init__.py +6 -0
  18. zenml/integrations/aws/flavors/aws_image_builder_flavor.py +146 -0
  19. zenml/integrations/aws/image_builders/__init__.py +20 -0
  20. zenml/integrations/aws/image_builders/aws_image_builder.py +307 -0
  21. zenml/integrations/gcp/orchestrators/vertex_orchestrator.py +1 -1
  22. zenml/integrations/kaniko/image_builders/kaniko_image_builder.py +2 -1
  23. zenml/integrations/kubernetes/flavors/kubernetes_orchestrator_flavor.py +11 -0
  24. zenml/integrations/kubernetes/step_operators/kubernetes_step_operator.py +0 -1
  25. zenml/integrations/lightning/flavors/lightning_orchestrator_flavor.py +11 -0
  26. zenml/integrations/neptune/experiment_trackers/neptune_experiment_tracker.py +7 -5
  27. zenml/integrations/neptune/experiment_trackers/run_state.py +69 -53
  28. zenml/integrations/registry.py +2 -2
  29. zenml/integrations/skypilot/flavors/skypilot_orchestrator_base_vm_config.py +12 -0
  30. zenml/integrations/wandb/flavors/wandb_experiment_tracker_flavor.py +13 -5
  31. zenml/materializers/built_in_materializer.py +1 -1
  32. zenml/model/model.py +12 -16
  33. zenml/model/utils.py +3 -1
  34. zenml/models/v2/base/filter.py +26 -30
  35. zenml/models/v2/base/scoped.py +258 -5
  36. zenml/models/v2/core/artifact_version.py +15 -26
  37. zenml/models/v2/core/code_repository.py +1 -12
  38. zenml/models/v2/core/component.py +5 -46
  39. zenml/models/v2/core/flavor.py +1 -11
  40. zenml/models/v2/core/model.py +1 -57
  41. zenml/models/v2/core/model_version.py +5 -33
  42. zenml/models/v2/core/model_version_artifact.py +11 -3
  43. zenml/models/v2/core/model_version_pipeline_run.py +14 -3
  44. zenml/models/v2/core/pipeline.py +47 -55
  45. zenml/models/v2/core/pipeline_build.py +67 -12
  46. zenml/models/v2/core/pipeline_deployment.py +0 -10
  47. zenml/models/v2/core/pipeline_run.py +91 -29
  48. zenml/models/v2/core/run_template.py +21 -29
  49. zenml/models/v2/core/schedule.py +0 -10
  50. zenml/models/v2/core/secret.py +0 -14
  51. zenml/models/v2/core/service.py +9 -16
  52. zenml/models/v2/core/service_connector.py +0 -11
  53. zenml/models/v2/core/stack.py +21 -30
  54. zenml/models/v2/core/step_run.py +18 -14
  55. zenml/models/v2/core/trigger.py +19 -3
  56. zenml/orchestrators/base_orchestrator.py +13 -1
  57. zenml/orchestrators/output_utils.py +5 -1
  58. zenml/orchestrators/step_launcher.py +9 -13
  59. zenml/orchestrators/step_run_utils.py +8 -204
  60. zenml/orchestrators/utils.py +55 -27
  61. zenml/pipelines/build_utils.py +12 -0
  62. zenml/service_connectors/service_connector_utils.py +3 -9
  63. zenml/stack/stack_component.py +1 -1
  64. zenml/stack_deployments/aws_stack_deployment.py +22 -0
  65. zenml/utils/archivable.py +65 -36
  66. zenml/utils/code_utils.py +8 -4
  67. zenml/utils/docker_utils.py +9 -0
  68. zenml/zen_server/auth.py +9 -10
  69. zenml/zen_server/dashboard/assets/{404-NVXKFp-x.js → 404-Cqu3EDCm.js} +1 -1
  70. zenml/zen_server/dashboard/assets/{@reactflow-CK0KJUen.js → @reactflow-D2Y7BWwz.js} +1 -1
  71. zenml/zen_server/dashboard/assets/{AlertDialogDropdownItem-DezXKmDf.js → AlertDialogDropdownItem-BHd71pVS.js} +1 -1
  72. zenml/zen_server/dashboard/assets/{CodeSnippet-JzR8CEtw.js → CodeSnippet-DIonwetW.js} +1 -1
  73. zenml/zen_server/dashboard/assets/{CollapsibleCard-DQW_ktMO.js → CollapsibleCard-CDnC97pB.js} +1 -1
  74. zenml/zen_server/dashboard/assets/{Commands-DL2kwkRd.js → Commands-BVEXKAOj.js} +1 -1
  75. zenml/zen_server/dashboard/assets/{ComponentBadge-D_g62Wv8.js → ComponentBadge-CrRvovox.js} +1 -1
  76. zenml/zen_server/dashboard/assets/{CopyButton-LNcWaa14.js → CopyButton-B6wGAhQv.js} +1 -1
  77. zenml/zen_server/dashboard/assets/{CsvVizualization-DknpE5ej.js → CsvVizualization-CjcT7LMm.js} +5 -5
  78. zenml/zen_server/dashboard/assets/DeleteAlertDialog-D2ELtM2W.js +1 -0
  79. zenml/zen_server/dashboard/assets/{DialogItem-Bxf8FuAT.js → DialogItem-DXIMhBgU.js} +1 -1
  80. zenml/zen_server/dashboard/assets/{Error-DYflYyps.js → Error-B8uUfTpL.js} +1 -1
  81. zenml/zen_server/dashboard/assets/{ExecutionStatus-C7zyIQKZ.js → ExecutionStatus-ibAdY-dG.js} +1 -1
  82. zenml/zen_server/dashboard/assets/{Helpbox-oYSGpLqd.js → Helpbox-BfAfhKHw.js} +1 -1
  83. zenml/zen_server/dashboard/assets/{Infobox-Cx4xGoXR.js → Infobox-M_SMOu96.js} +1 -1
  84. zenml/zen_server/dashboard/assets/{InlineAvatar-DiGOWNKF.js → InlineAvatar-DBA0a0-a.js} +1 -1
  85. zenml/zen_server/dashboard/assets/{NestedCollapsible-DYbgyKxK.js → NestedCollapsible-DpgmEFKw.js} +1 -1
  86. zenml/zen_server/dashboard/assets/{Partials-03iZf8-N.js → Partials-D_ldD9if.js} +1 -1
  87. zenml/zen_server/dashboard/assets/{ProBadge-D_EB8HNo.js → ProBadge-DQbfFotM.js} +1 -1
  88. zenml/zen_server/dashboard/assets/{ProCta-DqNS4v3x.js → ProCta-Bcpb4rcY.js} +1 -1
  89. zenml/zen_server/dashboard/assets/{ProviderIcon-Bki2aw8w.js → ProviderIcon-BZpgPigN.js} +1 -1
  90. zenml/zen_server/dashboard/assets/{ProviderRadio-8f43sPD4.js → ProviderRadio-DWPnMuQ1.js} +1 -1
  91. zenml/zen_server/dashboard/assets/RunSelector-DgRGaAc6.js +1 -0
  92. zenml/zen_server/dashboard/assets/{RunsBody-07YEO7qI.js → RunsBody-KecfSkjY.js} +1 -1
  93. zenml/zen_server/dashboard/assets/{SearchField-lp1KgU4e.js → SearchField-n-ILHnaP.js} +1 -1
  94. zenml/zen_server/dashboard/assets/{SecretTooltip-CgnbyeOx.js → SecretTooltip-B8MrX5yu.js} +1 -1
  95. zenml/zen_server/dashboard/assets/{SetPassword-CpP418A2.js → SetPassword-B_IVq_wg.js} +1 -1
  96. zenml/zen_server/dashboard/assets/StackList-TWPBYnkF.js +1 -0
  97. zenml/zen_server/dashboard/assets/{Tabs-BktHkCJJ.js → Tabs-Rg857zmd.js} +1 -1
  98. zenml/zen_server/dashboard/assets/{Tick-BlMoIlJT.js → Tick-COg4A-xo.js} +1 -1
  99. zenml/zen_server/dashboard/assets/{UpdatePasswordSchemas-Sc0A0pP-.js → UpdatePasswordSchemas-C6Aj3hm6.js} +1 -1
  100. zenml/zen_server/dashboard/assets/{UsageReason-YYduL4fj.js → UsageReason-BTLbx7w4.js} +1 -1
  101. zenml/zen_server/dashboard/assets/{WizardFooter-dgmizSJC.js → WizardFooter-BCAj69Vj.js} +1 -1
  102. zenml/zen_server/dashboard/assets/{all-pipeline-runs-query-D-c2G6lV.js → all-pipeline-runs-query-DMXkDrV2.js} +1 -1
  103. zenml/zen_server/dashboard/assets/code-snippets-CqONne41.js +13 -0
  104. zenml/zen_server/dashboard/assets/{create-stack-DM_JPgef.js → create-stack-HfdbhLs4.js} +1 -1
  105. zenml/zen_server/dashboard/assets/dates-3pMLCNrD.js +1 -0
  106. zenml/zen_server/dashboard/assets/delete-run-DZ4hIXff.js +1 -0
  107. zenml/zen_server/dashboard/assets/{form-schemas-K6FYKjwa.js → form-schemas-B0AVEd9b.js} +1 -1
  108. zenml/zen_server/dashboard/assets/{index-BAkC7FXi.js → index-DPqSWjug.js} +1 -1
  109. zenml/zen_server/dashboard/assets/{index-CEV4Cvaf.js → index-DScjfBRb.js} +1 -1
  110. zenml/zen_server/dashboard/assets/index-DXvT1_Um.css +1 -0
  111. zenml/zen_server/dashboard/assets/{index-CCOPpudF.js → index-FO-p0GU7.js} +5 -5
  112. zenml/zen_server/dashboard/assets/{index-B1mVPYxf.js → index-I3bKUGUj.js} +1 -1
  113. zenml/zen_server/dashboard/assets/key-icon-aH-QIa5R.js +1 -0
  114. zenml/zen_server/dashboard/assets/login-command-CkqxPtV3.js +1 -0
  115. zenml/zen_server/dashboard/assets/{login-mutation-hf-lK87O.js → login-mutation-BQeo4wTY.js} +1 -1
  116. zenml/zen_server/dashboard/assets/{not-found-BGirLjU-.js → not-found-gAJ5aDdR.js} +1 -1
  117. zenml/zen_server/dashboard/assets/page-9Y9-gig0.js +1 -0
  118. zenml/zen_server/dashboard/assets/{page-DjRJCGb3.js → page-AUwiQ14W.js} +1 -1
  119. zenml/zen_server/dashboard/assets/page-B6XU7yYT.js +2 -0
  120. zenml/zen_server/dashboard/assets/{page-C00YAkaB.js → page-BKZYc2Zv.js} +1 -1
  121. zenml/zen_server/dashboard/assets/{page-CdMWnQak.js → page-BU9FG4sR.js} +1 -1
  122. zenml/zen_server/dashboard/assets/{page-D7S3aCbF.js → page-B_Apk3xg.js} +1 -1
  123. zenml/zen_server/dashboard/assets/{page-Djikxq_S.js → page-BdowiCbr.js} +1 -1
  124. zenml/zen_server/dashboard/assets/page-Bg8OjTRe.js +1 -0
  125. zenml/zen_server/dashboard/assets/page-BxL4qD4_.js +1 -0
  126. zenml/zen_server/dashboard/assets/{page-DakHVWXF.js → page-CWxT5K5J.js} +1 -1
  127. zenml/zen_server/dashboard/assets/page-CXuQufSe.js +1 -0
  128. zenml/zen_server/dashboard/assets/{page-DLC-bNBP.js → page-CcQr8CPP.js} +1 -1
  129. zenml/zen_server/dashboard/assets/{page-CD-DcWoy.js → page-Ce4Hrjnr.js} +1 -1
  130. zenml/zen_server/dashboard/assets/page-CiYxgZP_.js +1 -0
  131. zenml/zen_server/dashboard/assets/page-Cldq1mpe.js +1 -0
  132. zenml/zen_server/dashboard/assets/{page-BDigxVpo.js → page-D4wdonLm.js} +1 -1
  133. zenml/zen_server/dashboard/assets/{page-D6uU2ax4.js → page-D8ObrbH8.js} +1 -1
  134. zenml/zen_server/dashboard/assets/{page-DXSTpqRD.js → page-DFuAUGt4.js} +1 -1
  135. zenml/zen_server/dashboard/assets/{page-CbpvrsDL.js → page-DGazBpuP.js} +1 -1
  136. zenml/zen_server/dashboard/assets/{page-COXXJj1k.js → page-DO1UcqPX.js} +1 -1
  137. zenml/zen_server/dashboard/assets/page-DRYXdL5o.js +1 -0
  138. zenml/zen_server/dashboard/assets/{page-Df-Fw0aq.js → page-DYEquBC2.js} +1 -1
  139. zenml/zen_server/dashboard/assets/page-Dk32IeZm.js +1 -0
  140. zenml/zen_server/dashboard/assets/{page-yYC9OI-E.js → page-I3nKFGie.js} +1 -1
  141. zenml/zen_server/dashboard/assets/{page-6m6yHHlE.js → page-M0w-n6vn.js} +1 -1
  142. zenml/zen_server/dashboard/assets/{page-Vcxara9U.js → page-R5dx3xGF.js} +1 -1
  143. zenml/zen_server/dashboard/assets/{page-BR68V0V1.js → page-bT5pOvcB.js} +1 -1
  144. zenml/zen_server/dashboard/assets/page-hUqK889I.js +6 -0
  145. zenml/zen_server/dashboard/assets/{page-CjGdWY13.js → page-h_Stveon.js} +1 -1
  146. zenml/zen_server/dashboard/assets/{page-D01JhjQB.js → page-r8XK5vR7.js} +1 -1
  147. zenml/zen_server/dashboard/assets/page-u_-ZXBKb.js +1 -0
  148. zenml/zen_server/dashboard/assets/page-zaMqB_ao.js +1 -0
  149. zenml/zen_server/dashboard/assets/{persist-GjC8PZoC.js → persist-AppN1B0J.js} +1 -1
  150. zenml/zen_server/dashboard/assets/{persist-Coz7ZWvz.js → persist-DAUi_3za.js} +1 -1
  151. zenml/zen_server/dashboard/assets/service-BqqeXLEe.js +2 -0
  152. zenml/zen_server/dashboard/assets/{sharedSchema-CQb14VSr.js → sharedSchema-uXN9FLLk.js} +1 -1
  153. zenml/zen_server/dashboard/assets/{stack-detail-query-OPEW-cDJ.js → stack-detail-query-XfZBiBP2.js} +1 -1
  154. zenml/zen_server/dashboard/assets/{update-server-settings-mutation-LwuQfHYn.js → update-server-settings-mutation-BWmgVJwA.js} +1 -1
  155. zenml/zen_server/dashboard/assets/{url-CkvKAnwF.js → url-BLwMbzES.js} +1 -1
  156. zenml/zen_server/dashboard/index.html +4 -4
  157. zenml/zen_server/deploy/helm/Chart.yaml +1 -1
  158. zenml/zen_server/deploy/helm/README.md +2 -2
  159. zenml/zen_server/rbac/rbac_sql_zen_store.py +173 -0
  160. zenml/zen_server/routers/auth_endpoints.py +22 -11
  161. zenml/zen_server/routers/steps_endpoints.py +7 -1
  162. zenml/zen_server/template_execution/utils.py +3 -1
  163. zenml/zen_server/utils.py +4 -3
  164. zenml/zen_stores/base_zen_store.py +10 -2
  165. zenml/zen_stores/migrations/versions/0.71.0_release.py +23 -0
  166. zenml/zen_stores/migrations/versions/26351d482b9e_add_step_run_unique_constraint.py +37 -0
  167. zenml/zen_stores/migrations/versions/a1237ba94fd8_add_model_version_producer_run_unique_.py +68 -0
  168. zenml/zen_stores/rest_zen_store.py +76 -43
  169. zenml/zen_stores/schemas/model_schemas.py +42 -6
  170. zenml/zen_stores/schemas/pipeline_deployment_schemas.py +7 -7
  171. zenml/zen_stores/schemas/pipeline_run_schemas.py +12 -6
  172. zenml/zen_stores/schemas/pipeline_schemas.py +5 -0
  173. zenml/zen_stores/schemas/step_run_schemas.py +8 -1
  174. zenml/zen_stores/sql_zen_store.py +332 -100
  175. {zenml_nightly-0.70.0.dev20241201.dist-info → zenml_nightly-0.71.0.dev20241223.dist-info}/METADATA +5 -5
  176. {zenml_nightly-0.70.0.dev20241201.dist-info → zenml_nightly-0.71.0.dev20241223.dist-info}/RECORD +179 -164
  177. zenml/zen_server/dashboard/assets/RunSelector-DkPiIiNr.js +0 -1
  178. zenml/zen_server/dashboard/assets/StackList-WvuKQusZ.js +0 -1
  179. zenml/zen_server/dashboard/assets/delete-run-CJdh1P_h.js +0 -1
  180. zenml/zen_server/dashboard/assets/index-DlGvJQPn.css +0 -1
  181. zenml/zen_server/dashboard/assets/page-0JE_-Ec1.js +0 -1
  182. zenml/zen_server/dashboard/assets/page-BRLpxOt0.js +0 -1
  183. zenml/zen_server/dashboard/assets/page-BU7huvKw.js +0 -6
  184. zenml/zen_server/dashboard/assets/page-BvqLv2Ky.js +0 -1
  185. zenml/zen_server/dashboard/assets/page-CwxrFarU.js +0 -1
  186. zenml/zen_server/dashboard/assets/page-DfbXf_8s.js +0 -1
  187. zenml/zen_server/dashboard/assets/page-Dnovpa0i.js +0 -3
  188. zenml/zen_server/dashboard/assets/page-Dot3LPmL.js +0 -1
  189. zenml/zen_server/dashboard/assets/page-Xynx4btY.js +0 -14
  190. zenml/zen_server/dashboard/assets/page-YpKAqVSa.js +0 -1
  191. {zenml_nightly-0.70.0.dev20241201.dist-info → zenml_nightly-0.71.0.dev20241223.dist-info}/LICENSE +0 -0
  192. {zenml_nightly-0.70.0.dev20241201.dist-info → zenml_nightly-0.71.0.dev20241223.dist-info}/WHEEL +0 -0
  193. {zenml_nightly-0.70.0.dev20241201.dist-info → zenml_nightly-0.71.0.dev20241223.dist-info}/entry_points.txt +0 -0
@@ -14,7 +14,7 @@
14
14
  """Utilities for creating step runs."""
15
15
 
16
16
  from datetime import datetime
17
- from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple
17
+ from typing import Dict, List, Optional, Set, Tuple
18
18
 
19
19
  from zenml.client import Client
20
20
  from zenml.config.step_configurations import Step
@@ -24,21 +24,13 @@ from zenml.logger import get_logger
24
24
  from zenml.model.utils import link_artifact_version_to_model_version
25
25
  from zenml.models import (
26
26
  ArtifactVersionResponse,
27
- ModelVersionPipelineRunRequest,
28
27
  ModelVersionResponse,
29
28
  PipelineDeploymentResponse,
30
29
  PipelineRunResponse,
31
- PipelineRunUpdate,
32
30
  StepRunRequest,
33
- StepRunResponse,
34
- StepRunUpdate,
35
31
  )
36
32
  from zenml.orchestrators import cache_utils, input_utils, utils
37
33
  from zenml.stack import Stack
38
- from zenml.utils import pagination_utils, string_utils
39
-
40
- if TYPE_CHECKING:
41
- from zenml.model.model import Model
42
34
 
43
35
  logger = get_logger(__name__)
44
36
 
@@ -293,10 +285,6 @@ def create_cached_step_runs(
293
285
  deployment=deployment, pipeline_run=pipeline_run, stack=stack
294
286
  )
295
287
 
296
- pipeline_model_version, pipeline_run = prepare_pipeline_run_model_version(
297
- pipeline_run=pipeline_run
298
- )
299
-
300
288
  while (
301
289
  cache_candidates := find_cacheable_invocation_candidates(
302
290
  deployment=deployment,
@@ -311,7 +299,9 @@ def create_cached_step_runs(
311
299
 
312
300
  # Make sure the request factory has the most up to date pipeline
313
301
  # run to avoid hydration calls
314
- request_factory.pipeline_run = pipeline_run
302
+ request_factory.pipeline_run = Client().get_pipeline_run(
303
+ pipeline_run.id
304
+ )
315
305
  try:
316
306
  step_run_request = request_factory.create_request(
317
307
  invocation_id
@@ -336,15 +326,10 @@ def create_cached_step_runs(
336
326
 
337
327
  step_run = Client().zen_store.create_run_step(step_run_request)
338
328
 
339
- # Refresh the pipeline run here to make sure we have the latest
340
- # state
341
- pipeline_run = Client().get_pipeline_run(pipeline_run.id)
342
-
343
- step_model_version, step_run = prepare_step_run_model_version(
344
- step_run=step_run, pipeline_run=pipeline_run
345
- )
346
-
347
- if model_version := step_model_version or pipeline_model_version:
329
+ if (
330
+ model_version := step_run.model_version
331
+ or pipeline_run.model_version
332
+ ):
348
333
  link_output_artifacts_to_model_version(
349
334
  artifacts=step_run.outputs,
350
335
  model_version=model_version,
@@ -356,169 +341,6 @@ def create_cached_step_runs(
356
341
  return cached_invocations
357
342
 
358
343
 
359
- def get_or_create_model_version_for_pipeline_run(
360
- model: "Model",
361
- pipeline_run: PipelineRunResponse,
362
- substitutions: Dict[str, str],
363
- ) -> Tuple[ModelVersionResponse, bool]:
364
- """Get or create a model version as part of a pipeline run.
365
-
366
- Args:
367
- model: The model to get or create.
368
- pipeline_run: The pipeline run for which the model should be created.
369
- substitutions: Substitutions to apply to the model version name.
370
-
371
- Returns:
372
- The model version and a boolean indicating whether it was newly created
373
- or not.
374
- """
375
- # Copy the model before modifying it so we don't accidently modify
376
- # configurations in which the model object is potentially referenced
377
- model = model.model_copy()
378
-
379
- if model.model_version_id:
380
- return model._get_model_version(), False
381
- elif model.version:
382
- if isinstance(model.version, str):
383
- model.version = string_utils.format_name_template(
384
- model.version,
385
- substitutions=substitutions,
386
- )
387
- model.name = string_utils.format_name_template(
388
- model.name,
389
- substitutions=substitutions,
390
- )
391
-
392
- return (
393
- model._get_or_create_model_version(),
394
- model._created_model_version,
395
- )
396
-
397
- # The model version should be created as part of this run
398
- # -> We first check if it was already created as part of this run, and if
399
- # not we do create it. If this is running in two parallel steps, we might
400
- # run into issues that this will create two versions. Ideally, all model
401
- # versions required for a pipeline run and its steps could be created
402
- # server-side at run creation time before the first step starts.
403
- if model_version := get_model_version_created_by_pipeline_run(
404
- model_name=model.name, pipeline_run=pipeline_run
405
- ):
406
- return model_version, False
407
- else:
408
- return model._get_or_create_model_version(), True
409
-
410
-
411
- def get_model_version_created_by_pipeline_run(
412
- model_name: str, pipeline_run: PipelineRunResponse
413
- ) -> Optional[ModelVersionResponse]:
414
- """Get a model version that was created by a specific pipeline run.
415
-
416
- This function does not refresh the pipeline run, so it will only try to
417
- fetch the model version from existing steps if they're already part of the
418
- response.
419
-
420
- Args:
421
- model_name: The model name for which to get the version.
422
- pipeline_run: The pipeline run for which to get the version.
423
-
424
- Returns:
425
- A model version with the given name created by the run, or None if such
426
- a model version does not exist.
427
- """
428
- if pipeline_run.config.model and pipeline_run.model_version:
429
- if (
430
- pipeline_run.config.model.name == model_name
431
- and pipeline_run.config.model.version is None
432
- ):
433
- return pipeline_run.model_version
434
-
435
- # We fetch a list of hydrated step runs here in order to avoid hydration
436
- # calls for each step separately.
437
- candidate_step_runs = pagination_utils.depaginate(
438
- Client().list_run_steps,
439
- pipeline_run_id=pipeline_run.id,
440
- model=model_name,
441
- hydrate=True,
442
- )
443
- for step_run in candidate_step_runs:
444
- if step_run.config.model and step_run.model_version:
445
- if (
446
- step_run.config.model.name == model_name
447
- and step_run.config.model.version is None
448
- ):
449
- return step_run.model_version
450
-
451
- return None
452
-
453
-
454
- def prepare_pipeline_run_model_version(
455
- pipeline_run: PipelineRunResponse,
456
- ) -> Tuple[Optional[ModelVersionResponse], PipelineRunResponse]:
457
- """Prepare the model version for a pipeline run.
458
-
459
- Args:
460
- pipeline_run: The pipeline run for which to prepare the model version.
461
-
462
- Returns:
463
- The prepared model version and the updated pipeline run.
464
- """
465
- model_version = None
466
-
467
- if pipeline_run.model_version:
468
- model_version = pipeline_run.model_version
469
- elif config_model := pipeline_run.config.model:
470
- model_version, _ = get_or_create_model_version_for_pipeline_run(
471
- model=config_model,
472
- pipeline_run=pipeline_run,
473
- substitutions=pipeline_run.config.substitutions,
474
- )
475
- pipeline_run = Client().zen_store.update_run(
476
- run_id=pipeline_run.id,
477
- run_update=PipelineRunUpdate(model_version_id=model_version.id),
478
- )
479
- link_pipeline_run_to_model_version(
480
- pipeline_run=pipeline_run, model_version=model_version
481
- )
482
- log_model_version_dashboard_url(model_version)
483
-
484
- return model_version, pipeline_run
485
-
486
-
487
- def prepare_step_run_model_version(
488
- step_run: StepRunResponse, pipeline_run: PipelineRunResponse
489
- ) -> Tuple[Optional[ModelVersionResponse], StepRunResponse]:
490
- """Prepare the model version for a step run.
491
-
492
- Args:
493
- step_run: The step run for which to prepare the model version.
494
- pipeline_run: The pipeline run of the step.
495
-
496
- Returns:
497
- The prepared model version and the updated step run.
498
- """
499
- model_version = None
500
-
501
- if step_run.model_version:
502
- model_version = step_run.model_version
503
- elif config_model := step_run.config.model:
504
- model_version, created = get_or_create_model_version_for_pipeline_run(
505
- model=config_model,
506
- pipeline_run=pipeline_run,
507
- substitutions=step_run.config.substitutions,
508
- )
509
- step_run = Client().zen_store.update_run_step(
510
- step_run_id=step_run.id,
511
- step_run_update=StepRunUpdate(model_version_id=model_version.id),
512
- )
513
- link_pipeline_run_to_model_version(
514
- pipeline_run=pipeline_run, model_version=model_version
515
- )
516
- if created:
517
- log_model_version_dashboard_url(model_version)
518
-
519
- return model_version, step_run
520
-
521
-
522
344
  def log_model_version_dashboard_url(
523
345
  model_version: ModelVersionResponse,
524
346
  ) -> None:
@@ -546,24 +368,6 @@ def log_model_version_dashboard_url(
546
368
  )
547
369
 
548
370
 
549
- def link_pipeline_run_to_model_version(
550
- pipeline_run: PipelineRunResponse, model_version: ModelVersionResponse
551
- ) -> None:
552
- """Link a pipeline run to a model version.
553
-
554
- Args:
555
- pipeline_run: The pipeline run to link.
556
- model_version: The model version to link.
557
- """
558
- client = Client()
559
- client.zen_store.create_model_version_pipeline_run_link(
560
- ModelVersionPipelineRunRequest(
561
- pipeline_run=pipeline_run.id,
562
- model_version=model_version.id,
563
- )
564
- )
565
-
566
-
567
371
  def link_output_artifacts_to_model_version(
568
372
  artifacts: Dict[str, List[ArtifactVersionResponse]],
569
373
  model_version: ModelVersionResponse,
@@ -26,10 +26,12 @@ from zenml.constants import (
26
26
  ENV_ZENML_ACTIVE_STACK_ID,
27
27
  ENV_ZENML_ACTIVE_WORKSPACE_ID,
28
28
  ENV_ZENML_DISABLE_CREDENTIALS_DISK_CACHING,
29
+ ENV_ZENML_PIPELINE_RUN_API_TOKEN_EXPIRATION,
29
30
  ENV_ZENML_SERVER,
30
31
  ENV_ZENML_STORE_PREFIX,
32
+ ZENML_PIPELINE_RUN_API_TOKEN_EXPIRATION,
31
33
  )
32
- from zenml.enums import AuthScheme, StackComponentType, StoreType
34
+ from zenml.enums import APITokenType, AuthScheme, StackComponentType, StoreType
33
35
  from zenml.logger import get_logger
34
36
  from zenml.stack import StackComponent
35
37
 
@@ -137,37 +139,63 @@ def get_config_environment_vars(
137
139
  url = global_config.store_configuration.url
138
140
  api_token = credentials_store.get_token(url, allow_expired=False)
139
141
  if schedule_id or pipeline_run_id or step_run_id:
140
- # When connected to an authenticated ZenML server, if a schedule ID,
141
- # pipeline run ID or step run ID is supplied, we need to fetch a new
142
- # workload API token scoped to the schedule, pipeline run or step
143
- # run.
144
142
  assert isinstance(global_config.zen_store, RestZenStore)
145
143
 
146
- # If only a schedule is given, the pipeline run credentials will
147
- # be valid for the entire duration of the schedule.
148
- api_key = credentials_store.get_api_key(url)
149
- if not api_key and not pipeline_run_id and not step_run_id:
144
+ # The user has the option to manually set an expiration for the API
145
+ # token generated for a pipeline run. In this case, we generate a new
146
+ # generic API token that will be valid for the indicated duration.
147
+ if (
148
+ pipeline_run_id
149
+ and ZENML_PIPELINE_RUN_API_TOKEN_EXPIRATION != 0
150
+ ):
150
151
  logger.warning(
151
- "An API token without an expiration time will be generated "
152
- "and used to run this pipeline on a schedule. This is very "
153
- "insecure because the API token will be valid for the "
154
- "entire lifetime of the schedule and can be used to access "
155
- "your user account if accidentally leaked. When deploying "
156
- "a pipeline on a schedule, it is strongly advised to use a "
157
- "service account API key to authenticate to the ZenML "
158
- "server instead of your regular user account. For more "
159
- "information, see "
160
- "https://docs.zenml.io/how-to/connecting-to-zenml/connect-with-a-service-account"
152
+ f"An unscoped API token will be generated for this pipeline "
153
+ f"run that will expire after "
154
+ f"{ZENML_PIPELINE_RUN_API_TOKEN_EXPIRATION} "
155
+ f"seconds instead of being scoped to the pipeline run "
156
+ f"and not having an expiration time. This is more insecure "
157
+ f"because the API token will remain valid even after the "
158
+ f"pipeline run completes its execution. This option has "
159
+ "been explicitly enabled by setting the "
160
+ f"{ENV_ZENML_PIPELINE_RUN_API_TOKEN_EXPIRATION} environment "
161
+ f"variable"
162
+ )
163
+ new_api_token = global_config.zen_store.get_api_token(
164
+ token_type=APITokenType.GENERIC,
165
+ expires_in=ZENML_PIPELINE_RUN_API_TOKEN_EXPIRATION,
161
166
  )
162
167
 
163
- # The schedule, pipeline run or step run credentials are scoped to
164
- # the schedule, pipeline run or step run and will only be valid for
165
- # the duration of the schedule/pipeline run/step run.
166
- new_api_token = global_config.zen_store.get_api_token(
167
- schedule_id=schedule_id,
168
- pipeline_run_id=pipeline_run_id,
169
- step_run_id=step_run_id,
170
- )
168
+ else:
169
+ # If a schedule ID, pipeline run ID or step run ID is supplied,
170
+ # we need to fetch a new workload API token scoped to the
171
+ # schedule, pipeline run or step run.
172
+
173
+ # If only a schedule is given, the pipeline run credentials will
174
+ # be valid for the entire duration of the schedule.
175
+ api_key = credentials_store.get_api_key(url)
176
+ if not api_key and not pipeline_run_id and not step_run_id:
177
+ logger.warning(
178
+ "An API token without an expiration time will be generated "
179
+ "and used to run this pipeline on a schedule. This is very "
180
+ "insecure because the API token will be valid for the "
181
+ "entire lifetime of the schedule and can be used to access "
182
+ "your user account if accidentally leaked. When deploying "
183
+ "a pipeline on a schedule, it is strongly advised to use a "
184
+ "service account API key to authenticate to the ZenML "
185
+ "server instead of your regular user account. For more "
186
+ "information, see "
187
+ "https://docs.zenml.io/how-to/connecting-to-zenml/connect-with-a-service-account"
188
+ )
189
+
190
+ # The schedule, pipeline run or step run credentials are scoped to
191
+ # the schedule, pipeline run or step run and will only be valid for
192
+ # the duration of the schedule/pipeline run/step run.
193
+ new_api_token = global_config.zen_store.get_api_token(
194
+ token_type=APITokenType.WORKLOAD,
195
+ schedule_id=schedule_id,
196
+ pipeline_run_id=pipeline_run_id,
197
+ step_run_id=step_run_id,
198
+ )
171
199
 
172
200
  environment_vars[ENV_ZENML_STORE_PREFIX + "API_TOKEN"] = (
173
201
  new_api_token
@@ -249,6 +249,11 @@ def find_existing_build(
249
249
  client = Client()
250
250
  stack = client.active_stack
251
251
 
252
+ if not stack.container_registry:
253
+ # There can be no non-local builds that we can reuse if there is no
254
+ # container registry in the stack.
255
+ return None
256
+
252
257
  python_version_prefix = ".".join(platform.python_version_tuple()[:2])
253
258
  required_builds = stack.get_docker_builds(deployment=deployment)
254
259
 
@@ -263,6 +268,13 @@ def find_existing_build(
263
268
  sort_by="desc:created",
264
269
  size=1,
265
270
  stack_id=stack.id,
271
+ # Until we implement stack versioning, users can still update their
272
+ # stack to update/remove the container registry. In that case, we might
273
+ # try to pull an image from a container registry that we don't have
274
+ # access to. This is why we add an additional check for the container
275
+ # registry ID here. (This is still not perfect as users can update the
276
+ # container registry URI or config, but the best we can do)
277
+ container_registry_id=stack.container_registry.id,
266
278
  # The build is local and it's not clear whether the images
267
279
  # exist on the current machine or if they've been overwritten.
268
280
  # TODO: Should we support this by storing the unique Docker ID for
@@ -60,15 +60,9 @@ def _raise_specific_cloud_exception_if_needed(
60
60
  orchestrators: List[ResourcesInfo],
61
61
  container_registries: List[ResourcesInfo],
62
62
  ) -> None:
63
- AWS_DOCS = (
64
- "https://docs.zenml.io/how-to/auth-management/aws-service-connector"
65
- )
66
- GCP_DOCS = (
67
- "https://docs.zenml.io/how-to/auth-management/gcp-service-connector"
68
- )
69
- AZURE_DOCS = (
70
- "https://docs.zenml.io/how-to/auth-management/azure-service-connector"
71
- )
63
+ AWS_DOCS = "https://docs.zenml.io/how-to/infrastructure-deployment/auth-management/aws-service-connector"
64
+ GCP_DOCS = "https://docs.zenml.io/how-to/infrastructure-deployment/auth-management/gcp-service-connector"
65
+ AZURE_DOCS = "https://docs.zenml.io/how-to/infrastructure-deployment/auth-management/azure-service-connector"
72
66
 
73
67
  if not artifact_stores:
74
68
  error_msg = (
@@ -102,7 +102,7 @@ class StackComponentConfig(BaseModel, ABC):
102
102
  "in sensitive information as secrets. Check out the "
103
103
  "documentation on how to configure your stack "
104
104
  "components with secrets here: "
105
- "https://docs.zenml.io/getting-started/deploying-zenml/manage-the-deployed-services/secret-management"
105
+ "https://docs.zenml.io/getting-started/deploying-zenml/secret-management"
106
106
  )
107
107
  continue
108
108
 
@@ -73,6 +73,7 @@ of any potential costs:
73
73
  - An ECR repository registered as a [ZenML container registry](https://docs.zenml.io/stack-components/container-registries/aws).
74
74
  - Sagemaker registered as a [ZenML orchestrator](https://docs.zenml.io/stack-components/orchestrators/sagemaker)
75
75
  as well as a [ZenML step operator](https://docs.zenml.io/stack-components/step-operators/sagemaker).
76
+ - A CodeBuild project registered as a [ZenML image builder](https://docs.zenml.io/stack-components/image-builder/aws).
76
77
  - An IAM user and IAM role with the minimum necessary permissions to access the
77
78
  above resources.
78
79
  - An AWS access key used to give access to ZenML to connect to the above
@@ -158,6 +159,26 @@ console.
158
159
  "ecr:PutImage",
159
160
  "ecr:GetAuthorizationToken",
160
161
  ],
162
+ "CloudBuild (Client)": [
163
+ "codebuild:CreateProject",
164
+ "codebuild:BatchGetBuilds",
165
+ ],
166
+ "CloudBuild (Service)": [
167
+ "s3:GetObject",
168
+ "s3:GetObjectVersion",
169
+ "logs:CreateLogGroup",
170
+ "logs:CreateLogStream",
171
+ "logs:PutLogEvents",
172
+ "ecr:BatchGetImage",
173
+ "ecr:DescribeImages",
174
+ "ecr:BatchCheckLayerAvailability",
175
+ "ecr:GetDownloadUrlForLayer",
176
+ "ecr:InitiateLayerUpload",
177
+ "ecr:UploadLayerPart",
178
+ "ecr:CompleteLayerUpload",
179
+ "ecr:PutImage",
180
+ "ecr:GetAuthorizationToken",
181
+ ],
161
182
  "SageMaker (Client)": [
162
183
  "sagemaker:CreatePipeline",
163
184
  "sagemaker:StartPipelineExecution",
@@ -243,6 +264,7 @@ console.
243
264
  param_ResourceName=f"zenml-{random_str(6).lower()}",
244
265
  param_ZenMLServerURL=self.zenml_server_url,
245
266
  param_ZenMLServerAPIToken=self.zenml_server_api_token,
267
+ param_CodeBuild="true",
246
268
  )
247
269
  # Encode the parameters as URL query parameters
248
270
  query_params = "&".join([f"{k}={v}" for k, v in params.items()])
zenml/utils/archivable.py CHANGED
@@ -15,11 +15,21 @@
15
15
 
16
16
  import io
17
17
  import tarfile
18
+ import zipfile
18
19
  from abc import ABC, abstractmethod
19
20
  from pathlib import Path
20
- from typing import IO, Any, Dict
21
+ from typing import IO, Any, Dict, Optional
21
22
 
22
23
  from zenml.io import fileio
24
+ from zenml.utils.enum_utils import StrEnum
25
+
26
+
27
+ class ArchiveType(StrEnum):
28
+ """Archive types supported by the ZenML build context."""
29
+
30
+ TAR = "tar"
31
+ TAR_GZ = "tar.gz"
32
+ ZIP = "zip"
23
33
 
24
34
 
25
35
  class Archivable(ABC):
@@ -81,52 +91,71 @@ class Archivable(ABC):
81
91
  self._extra_files[file_destination.as_posix()] = f.read()
82
92
 
83
93
  def write_archive(
84
- self, output_file: IO[bytes], use_gzip: bool = True
94
+ self,
95
+ output_file: IO[bytes],
96
+ archive_type: ArchiveType = ArchiveType.TAR_GZ,
85
97
  ) -> None:
86
98
  """Writes an archive of the build context to the given file.
87
99
 
88
100
  Args:
89
101
  output_file: The file to write the archive to.
90
- use_gzip: Whether to use `gzip` to compress the file.
102
+ archive_type: The type of archive to create.
91
103
  """
92
104
  files = self.get_files()
93
105
  extra_files = self.get_extra_files()
106
+ close_fileobj: Optional[Any] = None
107
+ fileobj: Any = output_file
94
108
 
95
- if use_gzip:
96
- from gzip import GzipFile
97
-
98
- # We don't use the builtin gzip functionality of the `tarfile`
99
- # library as that one includes the tar filename and creation
100
- # timestamp in the archive which causes the hash of the resulting
101
- # file to be different each time. We use this hash to avoid
102
- # duplicate uploads, which is why we pass empty values for filename
103
- # and mtime here.
104
- fileobj: Any = GzipFile(
105
- filename="", mode="wb", fileobj=output_file, mtime=0.0
106
- )
109
+ if archive_type == ArchiveType.ZIP:
110
+ fileobj = zipfile.ZipFile(output_file, "w", zipfile.ZIP_DEFLATED)
107
111
  else:
108
- fileobj = output_file
109
-
110
- with tarfile.open(mode="w", fileobj=fileobj) as tf:
111
- for archive_path, file_path in files.items():
112
- if archive_path in extra_files:
113
- continue
114
-
115
- if info := tf.gettarinfo(file_path, arcname=archive_path):
116
- if info.isfile():
117
- with open(file_path, "rb") as f:
118
- tf.addfile(info, f)
112
+ if archive_type == ArchiveType.TAR_GZ:
113
+ from gzip import GzipFile
114
+
115
+ # We don't use the builtin gzip functionality of the `tarfile`
116
+ # library as that one includes the tar filename and creation
117
+ # timestamp in the archive which causes the hash of the resulting
118
+ # file to be different each time. We use this hash to avoid
119
+ # duplicate uploads, which is why we pass empty values for filename
120
+ # and mtime here.
121
+ close_fileobj = fileobj = GzipFile(
122
+ filename="", mode="wb", fileobj=output_file, mtime=0.0
123
+ )
124
+ fileobj = tarfile.open(mode="w", fileobj=fileobj)
125
+
126
+ try:
127
+ with fileobj as af:
128
+ for archive_path, file_path in files.items():
129
+ if archive_path in extra_files:
130
+ continue
131
+ if archive_type == ArchiveType.ZIP:
132
+ assert isinstance(af, zipfile.ZipFile)
133
+ af.write(file_path, arcname=archive_path)
119
134
  else:
120
- tf.addfile(info, None)
121
-
122
- for archive_path, contents in extra_files.items():
123
- info = tarfile.TarInfo(archive_path)
124
- contents_encoded = contents.encode("utf-8")
125
- info.size = len(contents_encoded)
126
- tf.addfile(info, io.BytesIO(contents_encoded))
127
-
128
- if use_gzip:
129
- fileobj.close()
135
+ assert isinstance(af, tarfile.TarFile)
136
+ if info := af.gettarinfo(
137
+ file_path, arcname=archive_path
138
+ ):
139
+ if info.isfile():
140
+ with open(file_path, "rb") as f:
141
+ af.addfile(info, f)
142
+ else:
143
+ af.addfile(info, None)
144
+
145
+ for archive_path, contents in extra_files.items():
146
+ contents_encoded = contents.encode("utf-8")
147
+
148
+ if archive_type == ArchiveType.ZIP:
149
+ assert isinstance(af, zipfile.ZipFile)
150
+ af.writestr(archive_path, contents_encoded)
151
+ else:
152
+ assert isinstance(af, tarfile.TarFile)
153
+ info = tarfile.TarInfo(archive_path)
154
+ info.size = len(contents_encoded)
155
+ af.addfile(info, io.BytesIO(contents_encoded))
156
+ finally:
157
+ if close_fileobj:
158
+ close_fileobj.close()
130
159
 
131
160
  output_file.seek(0)
132
161
 
zenml/utils/code_utils.py CHANGED
@@ -25,7 +25,7 @@ from zenml.client import Client
25
25
  from zenml.io import fileio
26
26
  from zenml.logger import get_logger
27
27
  from zenml.utils import source_utils, string_utils
28
- from zenml.utils.archivable import Archivable
28
+ from zenml.utils.archivable import Archivable, ArchiveType
29
29
 
30
30
  if TYPE_CHECKING:
31
31
  from git.repo.base import Repo
@@ -152,15 +152,19 @@ class CodeArchive(Archivable):
152
152
  return all_files
153
153
 
154
154
  def write_archive(
155
- self, output_file: IO[bytes], use_gzip: bool = True
155
+ self,
156
+ output_file: IO[bytes],
157
+ archive_type: ArchiveType = ArchiveType.TAR_GZ,
156
158
  ) -> None:
157
159
  """Writes an archive of the build context to the given file.
158
160
 
159
161
  Args:
160
162
  output_file: The file to write the archive to.
161
- use_gzip: Whether to use `gzip` to compress the file.
163
+ archive_type: The type of archive to create.
162
164
  """
163
- super().write_archive(output_file=output_file, use_gzip=use_gzip)
165
+ super().write_archive(
166
+ output_file=output_file, archive_type=archive_type
167
+ )
164
168
  archive_size = os.path.getsize(output_file.name)
165
169
  if archive_size > 20 * 1024 * 1024:
166
170
  logger.warning(
@@ -266,6 +266,14 @@ def push_image(
266
266
  logger.info("Finished pushing Docker image.")
267
267
 
268
268
  image_name_without_tag, _ = image_name.rsplit(":", maxsplit=1)
269
+
270
+ image = docker_client.images.get(image_name)
271
+ repo_digests: List[str] = image.attrs["RepoDigests"]
272
+
273
+ for digest in repo_digests:
274
+ if digest.startswith(f"{image_name_without_tag}@"):
275
+ return digest
276
+
269
277
  for info in reversed(aux_info):
270
278
  try:
271
279
  repo_digest = info["Digest"]
@@ -304,6 +312,7 @@ def get_image_digest(image_name: str) -> Optional[str]:
304
312
 
305
313
  image = docker_client.images.get(image_name)
306
314
  repo_digests = image.attrs["RepoDigests"]
315
+
307
316
  if len(repo_digests) == 1:
308
317
  return cast(str, repo_digests[0])
309
318
  else: