zenml-nightly 0.81.0.dev20250428__py3-none-any.whl → 0.81.0.dev20250430__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (168) hide show
  1. zenml/VERSION +1 -1
  2. zenml/cli/model.py +2 -7
  3. zenml/client.py +17 -2
  4. zenml/config/build_configuration.py +4 -0
  5. zenml/config/docker_settings.py +1 -1
  6. zenml/config/resource_settings.py +1 -1
  7. zenml/config/strict_base_model.py +1 -1
  8. zenml/entrypoints/base_entrypoint_configuration.py +1 -1
  9. zenml/environment.py +37 -34
  10. zenml/exceptions.py +0 -78
  11. zenml/integrations/aws/__init__.py +1 -1
  12. zenml/integrations/evidently/__init__.py +1 -0
  13. zenml/integrations/evidently/metrics.py +1 -1
  14. zenml/integrations/evidently/tests.py +1 -1
  15. zenml/integrations/gcp/flavors/vertex_orchestrator_flavor.py +6 -0
  16. zenml/integrations/gcp/orchestrators/vertex_orchestrator.py +49 -28
  17. zenml/integrations/kubeflow/orchestrators/kubeflow_orchestrator.py +7 -29
  18. zenml/integrations/kubernetes/flavors/kubernetes_orchestrator_flavor.py +6 -0
  19. zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator.py +12 -5
  20. zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator_entrypoint.py +31 -18
  21. zenml/integrations/kubernetes/orchestrators/manifest_utils.py +21 -0
  22. zenml/integrations/kubernetes/pod_settings.py +6 -1
  23. zenml/integrations/numpy/__init__.py +4 -1
  24. zenml/integrations/numpy/materializers/numpy_materializer.py +75 -6
  25. zenml/integrations/pandas/materializers/pandas_materializer.py +118 -22
  26. zenml/integrations/tekton/orchestrators/tekton_orchestrator.py +8 -28
  27. zenml/models/v2/core/model_version.py +28 -33
  28. zenml/models/v2/core/pipeline_deployment.py +4 -4
  29. zenml/models/v2/core/pipeline_run.py +5 -12
  30. zenml/models/v2/core/user.py +1 -1
  31. zenml/orchestrators/dag_runner.py +50 -14
  32. zenml/secret/base_secret.py +1 -1
  33. zenml/stack/stack_component.py +1 -1
  34. zenml/stack/utils.py +1 -1
  35. zenml/utils/code_utils.py +1 -1
  36. zenml/zen_server/dashboard/assets/{404-o3nB5iwI.js → 404-D4aYbspS.js} +1 -1
  37. zenml/zen_server/dashboard/assets/{@react-router-BGgzhn5D.js → @react-router-CNP6g_RL.js} +1 -1
  38. zenml/zen_server/dashboard/assets/{AlertDialogDropdownItem-C_qWs-4u.js → AlertDialogDropdownItem-CRZjthRL.js} +1 -1
  39. zenml/zen_server/dashboard/assets/{CodeSnippet-nRnOvbOJ.js → CodeSnippet-Di-loaZu.js} +1 -1
  40. zenml/zen_server/dashboard/assets/CollapsibleCard-Dr5zb5T5.js +1 -0
  41. zenml/zen_server/dashboard/assets/{Commands-BwBzFj-5.js → Commands-CD9Y7FOE.js} +1 -1
  42. zenml/zen_server/dashboard/assets/{ComponentBadge-CMx2DbsV.js → ComponentBadge-BgGnzcLu.js} +1 -1
  43. zenml/zen_server/dashboard/assets/{ComponentIcon-CSxdApv8.js → ComponentIcon-B_7Y_oV-.js} +1 -1
  44. zenml/zen_server/dashboard/assets/{CsvVizualization-lxB2iyEh.js → CsvVizualization-CDrjL6bW.js} +1 -1
  45. zenml/zen_server/dashboard/assets/{DeleteAlertDialog-Bp0o-UXZ.js → DeleteAlertDialog-9RTDnEx3.js} +1 -1
  46. zenml/zen_server/dashboard/assets/DialogItem-0fWTH7ki.js +1 -0
  47. zenml/zen_server/dashboard/assets/{Error-DRhVeVHQ.js → Error-vBjUYjb-.js} +1 -1
  48. zenml/zen_server/dashboard/assets/{ExecutionStatus-HnZXi9bW.js → ExecutionStatus-B-ysY113.js} +1 -1
  49. zenml/zen_server/dashboard/assets/{Helpbox-Dsr3ZYcL.js → Helpbox-Dt3q7NGd.js} +1 -1
  50. zenml/zen_server/dashboard/assets/{Infobox-C4Op55B8.js → Infobox-Bai0JtFs.js} +1 -1
  51. zenml/zen_server/dashboard/assets/{InlineAvatar-CrAwe7Pl.js → InlineAvatar-Cpj0lVRQ.js} +1 -1
  52. zenml/zen_server/dashboard/assets/NestedCollapsible-DbhKdWx-.js +1 -0
  53. zenml/zen_server/dashboard/assets/{Partials-DjTM0Oe-.js → Partials-BNBtCjuT.js} +1 -1
  54. zenml/zen_server/dashboard/assets/ProBadge-3vtouJQf.js +1 -0
  55. zenml/zen_server/dashboard/assets/{ProCta-BYSSivgY.js → ProCta-BoTLdAdV.js} +1 -1
  56. zenml/zen_server/dashboard/assets/{ProviderIcon-B8W7AGNk.js → ProviderIcon-kkODO9tx.js} +1 -1
  57. zenml/zen_server/dashboard/assets/{ProviderRadio-BWFJPEY2.js → ProviderRadio-26EB5tY-.js} +1 -1
  58. zenml/zen_server/dashboard/assets/RunSelector-j0C_TPVp.js +1 -0
  59. zenml/zen_server/dashboard/assets/RunsBody-DexNouV4.js +1 -0
  60. zenml/zen_server/dashboard/assets/SearchField-DMCywawn.js +1 -0
  61. zenml/zen_server/dashboard/assets/{SecretTooltip-CCn9HfEh.js → SecretTooltip-LLGP7AIC.js} +1 -1
  62. zenml/zen_server/dashboard/assets/{SetPassword-BDFVwnNB.js → SetPassword-CBHW-Su8.js} +1 -1
  63. zenml/zen_server/dashboard/assets/StackList-S-9I1R4n.js +1 -0
  64. zenml/zen_server/dashboard/assets/Tabs-C2ufJX4t.js +1 -0
  65. zenml/zen_server/dashboard/assets/Tick-ay3Bgtro.js +1 -0
  66. zenml/zen_server/dashboard/assets/{UpdatePasswordSchemas-3gg2tC-H.js → UpdatePasswordSchemas-p3RX0P-Q.js} +1 -1
  67. zenml/zen_server/dashboard/assets/{UsageReason-DOMAzO02.js → UsageReason-C8sD2bOU.js} +1 -1
  68. zenml/zen_server/dashboard/assets/{Wizard-BSH_lIfi.js → Wizard-DIllc07n.js} +1 -1
  69. zenml/zen_server/dashboard/assets/{WizardFooter-CnM-50jL.js → WizardFooter-dNDpuero.js} +1 -1
  70. zenml/zen_server/dashboard/assets/{all-pipeline-runs-query-D9XfZpmV.js → all-pipeline-runs-query-BUT1PiTp.js} +1 -1
  71. zenml/zen_server/dashboard/assets/{configuration-form-Ci_QlfZC.js → configuration-form-DBZSb0FV.js} +1 -1
  72. zenml/zen_server/dashboard/assets/{create-stack-CoQ5SEJL.js → create-stack-BXLyjE58.js} +1 -1
  73. zenml/zen_server/dashboard/assets/{delete-run-B5xUZwdP.js → delete-run-DzcYxUnd.js} +1 -1
  74. zenml/zen_server/dashboard/assets/dots-horizontal-BGRJCPCs.js +1 -0
  75. zenml/zen_server/dashboard/assets/{flavor-select-DOLAl-S4.js → flavor-select-BTEwByv6.js} +1 -1
  76. zenml/zen_server/dashboard/assets/{form-schemas-BaEJ1z87.js → form-schemas-BJN_25Ua.js} +1 -1
  77. zenml/zen_server/dashboard/assets/{index-e8udi7QT.js → index-8uhUi12k.js} +1 -1
  78. zenml/zen_server/dashboard/assets/{index-hsSXyS3H.js → index-BygJE88j.js} +8 -8
  79. zenml/zen_server/dashboard/assets/{index-ojAuWXMf.js → index-D9-ukAem.js} +1 -1
  80. zenml/zen_server/dashboard/assets/index-DmTFrHJm.css +1 -0
  81. zenml/zen_server/dashboard/assets/{index-BoWZ9rXk.js → index-wYtmKLnQ.js} +1 -1
  82. zenml/zen_server/dashboard/assets/{login-mutation-CwraNa5G.js → login-mutation-DywLqguW.js} +1 -1
  83. zenml/zen_server/dashboard/assets/{not-found-BuK1QrF2.js → not-found-BOoklIG4.js} +1 -1
  84. zenml/zen_server/dashboard/assets/{page-0AIX3VK8.js → page--hSXKqyG.js} +1 -1
  85. zenml/zen_server/dashboard/assets/{page-CIDo1Ajk.js → page-2BsNK_sZ.js} +1 -1
  86. zenml/zen_server/dashboard/assets/page-4nVakAEl.js +1 -0
  87. zenml/zen_server/dashboard/assets/{page-CjRoKEQs.js → page-8AzB83Py.js} +1 -1
  88. zenml/zen_server/dashboard/assets/{page-BQVPbNps.js → page-B0r9dBEU.js} +1 -1
  89. zenml/zen_server/dashboard/assets/{page-hVng6ANV.js → page-B1Uq1S1Q.js} +1 -1
  90. zenml/zen_server/dashboard/assets/page-B3DscprQ.js +1 -0
  91. zenml/zen_server/dashboard/assets/{page-KRP_w5zH.js → page-B7Aj2XbV.js} +1 -1
  92. zenml/zen_server/dashboard/assets/{page-CZgeF_wQ.js → page-BE8jdyaM.js} +1 -1
  93. zenml/zen_server/dashboard/assets/page-BIYqUwTI.js +1 -0
  94. zenml/zen_server/dashboard/assets/page-BTzhAVEn.js +1 -0
  95. zenml/zen_server/dashboard/assets/{page-_Zt6UwiG.js → page-ByXQN8A-.js} +1 -1
  96. zenml/zen_server/dashboard/assets/page-CI4a9CXs.js +1 -0
  97. zenml/zen_server/dashboard/assets/page-CPLlVRXx.js +6 -0
  98. zenml/zen_server/dashboard/assets/page-CSQOpvvK.js +3 -0
  99. zenml/zen_server/dashboard/assets/{page-DwIlrsF0.js → page-Ccnk_5ji.js} +1 -1
  100. zenml/zen_server/dashboard/assets/{page-C-6XUKSp.js → page-Ci_n_x3a.js} +1 -1
  101. zenml/zen_server/dashboard/assets/page-D-ry_oLs.js +1 -0
  102. zenml/zen_server/dashboard/assets/{page-DbEn9V9E.js → page-D2TtSfnb.js} +1 -1
  103. zenml/zen_server/dashboard/assets/page-DHnkTzqs.js +1 -0
  104. zenml/zen_server/dashboard/assets/{page-B5clbTo4.js → page-DP2Ed-2m.js} +1 -1
  105. zenml/zen_server/dashboard/assets/{page-C30HZCTR.js → page-D_5R6E7b.js} +1 -1
  106. zenml/zen_server/dashboard/assets/page-DfTsntQI.js +1 -0
  107. zenml/zen_server/dashboard/assets/page-DkY7AIhj.js +1 -0
  108. zenml/zen_server/dashboard/assets/page-DmteSSj6.js +1 -0
  109. zenml/zen_server/dashboard/assets/page-Dt3wHeWX.js +1 -0
  110. zenml/zen_server/dashboard/assets/page-EXvws1ss.js +1 -0
  111. zenml/zen_server/dashboard/assets/{page-Czj8fu8q.js → page-K9yjbnd4.js} +1 -1
  112. zenml/zen_server/dashboard/assets/{page-DHDeVaMO.js → page-RV8wpZ_4.js} +1 -1
  113. zenml/zen_server/dashboard/assets/{page-DLVtrRa5.js → page-bUVHi7DE.js} +1 -1
  114. zenml/zen_server/dashboard/assets/page-cOEvFs_l.js +1 -0
  115. zenml/zen_server/dashboard/assets/page-kzSGEVUs.js +1 -0
  116. zenml/zen_server/dashboard/assets/{page-C7buw8xJ.js → page-uDGvpa5g.js} +2 -2
  117. zenml/zen_server/dashboard/assets/{page-BbygQpf-.js → page-uN0n_DE2.js} +1 -1
  118. zenml/zen_server/dashboard/assets/{page-CgJcperk.js → page-xa1EAPcI.js} +1 -1
  119. zenml/zen_server/dashboard/assets/{persist-xOAq24Bm.js → persist-CEeDYHW5.js} +1 -1
  120. zenml/zen_server/dashboard/assets/{persist-DLvsEXh0.js → persist-SSTTV5q0.js} +1 -1
  121. zenml/zen_server/dashboard/assets/{service-MHyqhRgp.js → service-BRoAA1Kq.js} +1 -1
  122. zenml/zen_server/dashboard/assets/sharedSchema-BACERDat.js +14 -0
  123. zenml/zen_server/dashboard/assets/{stack-detail-query-C1GoxBeK.js → stack-detail-query-ihiJr6nq.js} +1 -1
  124. zenml/zen_server/dashboard/assets/{update-server-settings-mutation-DduehrZn.js → update-server-settings-mutation-CK6COmw9.js} +1 -1
  125. zenml/zen_server/dashboard/index.html +4 -4
  126. zenml/zen_server/deploy/daemon/daemon_zen_server.py +1 -1
  127. zenml/zen_server/deploy/docker/docker_zen_server.py +1 -1
  128. zenml/zen_server/exceptions.py +0 -2
  129. zenml/zen_server/routers/model_versions_endpoints.py +10 -19
  130. zenml/zen_server/routers/runs_endpoints.py +7 -1
  131. zenml/zen_stores/schemas/pipeline_deployment_schemas.py +20 -3
  132. zenml/zen_stores/schemas/pipeline_run_schemas.py +21 -1
  133. zenml/zen_stores/secrets_stores/hashicorp_secrets_store.py +1 -1
  134. zenml/zen_stores/secrets_stores/sql_secrets_store.py +1 -1
  135. zenml/zen_stores/sql_zen_store.py +10 -34
  136. {zenml_nightly-0.81.0.dev20250428.dist-info → zenml_nightly-0.81.0.dev20250430.dist-info}/METADATA +2 -2
  137. {zenml_nightly-0.81.0.dev20250428.dist-info → zenml_nightly-0.81.0.dev20250430.dist-info}/RECORD +140 -139
  138. zenml/zen_server/dashboard/assets/CollapsibleCard-9sSz7n1R.js +0 -1
  139. zenml/zen_server/dashboard/assets/DialogItem-DHw2ihTx.js +0 -1
  140. zenml/zen_server/dashboard/assets/NestedCollapsible-Ce1bmT8N.js +0 -1
  141. zenml/zen_server/dashboard/assets/ProBadge-CDEqgzh8.js +0 -1
  142. zenml/zen_server/dashboard/assets/RunSelector-DoL_gMHJ.js +0 -1
  143. zenml/zen_server/dashboard/assets/RunsBody-B6mBjv-_.js +0 -1
  144. zenml/zen_server/dashboard/assets/SearchField-DHl5flXv.js +0 -1
  145. zenml/zen_server/dashboard/assets/StackList-dhdUtky3.js +0 -1
  146. zenml/zen_server/dashboard/assets/Tabs-BuZG8ft1.js +0 -1
  147. zenml/zen_server/dashboard/assets/Tick-C4gtED35.js +0 -1
  148. zenml/zen_server/dashboard/assets/index-6mLFgFwe.css +0 -1
  149. zenml/zen_server/dashboard/assets/page-3lW9eIer.js +0 -1
  150. zenml/zen_server/dashboard/assets/page-9PzQok8O.js +0 -1
  151. zenml/zen_server/dashboard/assets/page-BEibNYiA.js +0 -6
  152. zenml/zen_server/dashboard/assets/page-BOE2XBFy.js +0 -1
  153. zenml/zen_server/dashboard/assets/page-Bki79xdd.js +0 -1
  154. zenml/zen_server/dashboard/assets/page-BnKwJnOb.js +0 -1
  155. zenml/zen_server/dashboard/assets/page-CBGeo-7W.js +0 -2
  156. zenml/zen_server/dashboard/assets/page-Co-FJ5ds.js +0 -1
  157. zenml/zen_server/dashboard/assets/page-Cu8vuohQ.js +0 -1
  158. zenml/zen_server/dashboard/assets/page-D0QcEBi1.js +0 -1
  159. zenml/zen_server/dashboard/assets/page-DKY_U5Sz.js +0 -1
  160. zenml/zen_server/dashboard/assets/page-DQHU5o_C.js +0 -1
  161. zenml/zen_server/dashboard/assets/page-DpY1koQY.js +0 -1
  162. zenml/zen_server/dashboard/assets/page-DrS4WOup.js +0 -1
  163. zenml/zen_server/dashboard/assets/page-HIAPXq4w.js +0 -1
  164. zenml/zen_server/dashboard/assets/page-Jw2QAdxa.js +0 -1
  165. zenml/zen_server/dashboard/assets/sharedSchema-CQA5lnlU.js +0 -14
  166. {zenml_nightly-0.81.0.dev20250428.dist-info → zenml_nightly-0.81.0.dev20250430.dist-info}/LICENSE +0 -0
  167. {zenml_nightly-0.81.0.dev20250428.dist-info → zenml_nightly-0.81.0.dev20250430.dist-info}/WHEEL +0 -0
  168. {zenml_nightly-0.81.0.dev20250428.dist-info → zenml_nightly-0.81.0.dev20250430.dist-info}/entry_points.txt +0 -0
@@ -15,7 +15,7 @@
15
15
 
16
16
  import argparse
17
17
  import socket
18
- from typing import Any, Dict
18
+ from typing import Any, Dict, cast
19
19
  from uuid import UUID
20
20
 
21
21
  from kubernetes import client as k8s_client
@@ -41,7 +41,10 @@ from zenml.integrations.kubernetes.orchestrators.manifest_utils import (
41
41
  from zenml.logger import get_logger
42
42
  from zenml.orchestrators import publish_utils
43
43
  from zenml.orchestrators.dag_runner import NodeStatus, ThreadedDagRunner
44
- from zenml.orchestrators.utils import get_config_environment_vars
44
+ from zenml.orchestrators.utils import (
45
+ get_config_environment_vars,
46
+ get_orchestrator_run_name,
47
+ )
45
48
 
46
49
  logger = get_logger(__name__)
47
50
 
@@ -103,8 +106,27 @@ def main() -> None:
103
106
  Raises:
104
107
  Exception: If the pod fails to start.
105
108
  """
106
- # Define Kubernetes pod name.
107
- pod_name = f"{orchestrator_run_id}-{step_name}"
109
+ step_config = deployment_config.step_configurations[step_name].config
110
+ settings = step_config.settings.get("orchestrator.kubernetes", None)
111
+ settings = KubernetesOrchestratorSettings.model_validate(
112
+ settings.model_dump() if settings else {}
113
+ )
114
+
115
+ if settings.pod_name_prefix and not orchestrator_run_id.startswith(
116
+ settings.pod_name_prefix
117
+ ):
118
+ max_length = (
119
+ kube_utils.calculate_max_pod_name_length_for_namespace(
120
+ namespace=args.kubernetes_namespace
121
+ )
122
+ )
123
+ pod_name_prefix = get_orchestrator_run_name(
124
+ settings.pod_name_prefix, max_length=max_length
125
+ )
126
+ pod_name = f"{pod_name_prefix}-{step_name}"
127
+ else:
128
+ pod_name = f"{orchestrator_run_id}-{step_name}"
129
+
108
130
  pod_name = kube_utils.sanitize_pod_name(
109
131
  pod_name, namespace=args.kubernetes_namespace
110
132
  )
@@ -116,20 +138,6 @@ def main() -> None:
116
138
  step_name=step_name, deployment_id=deployment_config.id
117
139
  )
118
140
 
119
- step_config = deployment_config.step_configurations[step_name].config
120
-
121
- kubernetes_settings = step_config.settings.get(
122
- "orchestrator.kubernetes", None
123
- )
124
-
125
- orchestrator_settings = {}
126
- if kubernetes_settings is not None:
127
- orchestrator_settings = kubernetes_settings.model_dump()
128
-
129
- settings = KubernetesOrchestratorSettings.model_validate(
130
- orchestrator_settings
131
- )
132
-
133
141
  # We set some default minimum memory resource requests for the step pod
134
142
  # here if the user has not specified any, because the step pod takes up
135
143
  # some memory resources itself and, if not specified, the pod will be
@@ -274,12 +282,17 @@ def main() -> None:
274
282
  parallel_node_startup_waiting_period = (
275
283
  orchestrator.config.parallel_step_startup_waiting_period or 0.0
276
284
  )
285
+ settings = cast(
286
+ KubernetesOrchestratorSettings,
287
+ orchestrator.get_settings(deployment_config),
288
+ )
277
289
  try:
278
290
  ThreadedDagRunner(
279
291
  dag=pipeline_dag,
280
292
  run_fn=run_step_on_kubernetes,
281
293
  finalize_fn=finalize_run,
282
294
  parallel_node_startup_waiting_period=parallel_node_startup_waiting_period,
295
+ max_parallelism=settings.max_parallelism,
283
296
  ).run()
284
297
  logger.info("Orchestration pod completed.")
285
298
  finally:
@@ -28,6 +28,9 @@ from zenml.integrations.airflow.orchestrators.dag_generator import (
28
28
  )
29
29
  from zenml.integrations.kubernetes.orchestrators import kube_utils
30
30
  from zenml.integrations.kubernetes.pod_settings import KubernetesPodSettings
31
+ from zenml.logger import get_logger
32
+
33
+ logger = get_logger(__name__)
31
34
 
32
35
 
33
36
  def add_local_stores_mount(
@@ -244,6 +247,24 @@ def add_pod_settings(
244
247
  if settings.host_ipc:
245
248
  pod_spec.host_ipc = settings.host_ipc
246
249
 
250
+ if settings.scheduler_name:
251
+ pod_spec.scheduler_name = settings.scheduler_name
252
+
253
+ for key, value in settings.additional_pod_spec_args.items():
254
+ if not hasattr(pod_spec, key):
255
+ logger.warning(f"Ignoring invalid Pod Spec argument `{key}`.")
256
+ else:
257
+ if value is None:
258
+ continue
259
+
260
+ existing_value = getattr(pod_spec, key)
261
+ if isinstance(existing_value, list):
262
+ existing_value.extend(value)
263
+ elif isinstance(existing_value, dict):
264
+ existing_value.update(value)
265
+ else:
266
+ setattr(pod_spec, key, value)
267
+
247
268
 
248
269
  def build_cron_job_manifest(
249
270
  cron_expression: str,
@@ -13,7 +13,7 @@
13
13
  # permissions and limitations under the License.
14
14
  """Kubernetes pod settings."""
15
15
 
16
- from typing import Any, Dict, List
16
+ from typing import Any, Dict, List, Optional
17
17
 
18
18
  from pydantic import field_validator
19
19
 
@@ -33,10 +33,13 @@ class KubernetesPodSettings(BaseSettings):
33
33
  volumes: Volumes to mount in the pod.
34
34
  volume_mounts: Volume mounts to apply to the pod containers.
35
35
  host_ipc: Whether to enable host IPC for the pod.
36
+ scheduler_name: The name of the scheduler to use for the pod.
36
37
  image_pull_secrets: Image pull secrets to use for the pod.
37
38
  labels: Labels to apply to the pod.
38
39
  env: Environment variables to apply to the container.
39
40
  env_from: Environment variables to apply to the container.
41
+ additional_pod_spec_args: Additional arguments to pass to the pod. These
42
+ will be applied to the pod spec.
40
43
  """
41
44
 
42
45
  node_selectors: Dict[str, str] = {}
@@ -47,10 +50,12 @@ class KubernetesPodSettings(BaseSettings):
47
50
  volumes: List[Dict[str, Any]] = []
48
51
  volume_mounts: List[Dict[str, Any]] = []
49
52
  host_ipc: bool = False
53
+ scheduler_name: Optional[str] = None
50
54
  image_pull_secrets: List[str] = []
51
55
  labels: Dict[str, str] = {}
52
56
  env: List[Dict[str, Any]] = []
53
57
  env_from: List[Dict[str, Any]] = []
58
+ additional_pod_spec_args: Dict[str, Any] = {}
54
59
 
55
60
  @field_validator("volumes", mode="before")
56
61
  @classmethod
@@ -21,7 +21,10 @@ class NumpyIntegration(Integration):
21
21
  """Definition of Numpy integration for ZenML."""
22
22
 
23
23
  NAME = NUMPY
24
- REQUIREMENTS = ["numpy<2.0.0"]
24
+ # Support both NumPy 1.x and 2.x
25
+ # Note: Some other integrations (like evidently) may require NumPy <2.0
26
+ # until they are updated for NumPy 2.0 compatibility
27
+ REQUIREMENTS = ["numpy<3.0"]
25
28
 
26
29
  @classmethod
27
30
  def activate(cls) -> None:
@@ -15,7 +15,16 @@
15
15
 
16
16
  import os
17
17
  from collections import Counter
18
- from typing import TYPE_CHECKING, Any, ClassVar, Dict, Tuple, Type
18
+ from typing import (
19
+ TYPE_CHECKING,
20
+ Any,
21
+ ClassVar,
22
+ Dict,
23
+ Optional,
24
+ Tuple,
25
+ Type,
26
+ Union,
27
+ )
19
28
 
20
29
  import numpy as np
21
30
 
@@ -36,6 +45,48 @@ DATA_FILENAME = "data.parquet"
36
45
  SHAPE_FILENAME = "shape.json"
37
46
  DATA_VAR = "data_var"
38
47
 
48
+ # Check NumPy version for compatibility handling
49
+ IS_NUMPY_2 = np.lib.NumpyVersion(np.__version__) >= "2.0.0"
50
+
51
+ # In NumPy 2.0, np.object_ is deprecated in favor of object
52
+ # Let's use the right type based on the NumPy version
53
+ NUMPY_OBJECT_TYPE = object if IS_NUMPY_2 else np.object_
54
+
55
+
56
+ def _ensure_dtype_compatibility(arr: "NDArray[Any]") -> "NDArray[Any]":
57
+ """Ensure consistent dtype handling across NumPy versions.
58
+
59
+ Args:
60
+ arr: NumPy array to ensure compatible dtype handling
61
+
62
+ Returns:
63
+ NumPy array with consistent dtype behavior
64
+ """
65
+ if IS_NUMPY_2:
66
+ return arr # NumPy 2.0 already preserves precision
67
+ else:
68
+ # For 1.x, explicitly preserve precision when needed
69
+ return arr.astype(arr.dtype, copy=False)
70
+
71
+
72
+ def _create_array(
73
+ data: Any, dtype: Optional[Union["np.dtype[Any]", Type[Any]]] = None
74
+ ) -> "NDArray[Any]":
75
+ """Create arrays with consistent behavior across NumPy versions.
76
+
77
+ Args:
78
+ data: Data to convert to array
79
+ dtype: Optional dtype to use
80
+
81
+ Returns:
82
+ NumPy array with consistent creation behavior
83
+ """
84
+ if IS_NUMPY_2:
85
+ return np.asarray(data, dtype=dtype)
86
+ else:
87
+ # In NumPy 1.x, copy behavior is different
88
+ return np.array(data, dtype=dtype, copy=False)
89
+
39
90
 
40
91
  class NumpyMaterializer(BaseMaterializer):
41
92
  """Materializer to read data to and from pandas."""
@@ -60,7 +111,9 @@ class NumpyMaterializer(BaseMaterializer):
60
111
 
61
112
  if self.artifact_store.exists(numpy_file):
62
113
  with self.artifact_store.open(numpy_file, "rb") as f:
63
- return np.load(f, allow_pickle=True)
114
+ arr = np.load(f, allow_pickle=True)
115
+ # Ensure consistent dtype handling
116
+ return _ensure_dtype_compatibility(arr)
64
117
  elif self.artifact_store.exists(os.path.join(self.uri, DATA_FILENAME)):
65
118
  logger.warning(
66
119
  "A legacy artifact was found. "
@@ -86,7 +139,9 @@ class NumpyMaterializer(BaseMaterializer):
86
139
  input_stream = pa.input_stream(f)
87
140
  data = pq.read_table(input_stream)
88
141
  vals = getattr(data.to_pandas(), DATA_VAR).values
89
- return np.reshape(vals, shape_tuple)
142
+ arr = np.reshape(vals, shape_tuple)
143
+ # Ensure consistent dtype handling
144
+ return _ensure_dtype_compatibility(arr)
90
145
  except ImportError:
91
146
  raise ImportError(
92
147
  "You have an old version of a `NumpyMaterializer` ",
@@ -101,6 +156,9 @@ class NumpyMaterializer(BaseMaterializer):
101
156
  Args:
102
157
  arr: The numpy array to write.
103
158
  """
159
+ # Ensure consistent dtype handling before saving
160
+ arr = _ensure_dtype_compatibility(arr)
161
+
104
162
  with self.artifact_store.open(
105
163
  os.path.join(self.uri, NUMPY_FILENAME), "wb"
106
164
  ) as f:
@@ -186,8 +244,8 @@ class NumpyMaterializer(BaseMaterializer):
186
244
  """
187
245
  if np.issubdtype(arr.dtype, np.number):
188
246
  return self._extract_numeric_metadata(arr)
189
- elif np.issubdtype(arr.dtype, np.unicode_) or np.issubdtype(
190
- arr.dtype, np.object_
247
+ elif np.issubdtype(arr.dtype, np.str_) or np.issubdtype(
248
+ arr.dtype, NUMPY_OBJECT_TYPE
191
249
  ):
192
250
  return self._extract_text_metadata(arr)
193
251
  else:
@@ -204,6 +262,9 @@ class NumpyMaterializer(BaseMaterializer):
204
262
  Returns:
205
263
  A dictionary of metadata.
206
264
  """
265
+ # Ensure consistent precision handling
266
+ arr = _ensure_dtype_compatibility(arr)
267
+
207
268
  min_val = np.min(arr).item()
208
269
  max_val = np.max(arr).item()
209
270
 
@@ -228,7 +289,15 @@ class NumpyMaterializer(BaseMaterializer):
228
289
  Returns:
229
290
  A dictionary of metadata.
230
291
  """
231
- text = " ".join(arr)
292
+ # Convert all array elements to strings explicitly to handle
293
+ # mixed types and ensure NumPy 2.0 compatibility
294
+ str_items = [str(item) for item in arr.flat]
295
+ # Use dtype='U' (unicode string) instead of str to avoid type issues
296
+ str_arr = _create_array(str_items, dtype=np.dtype("U")).reshape(
297
+ arr.shape
298
+ )
299
+
300
+ text = " ".join(str_arr)
232
301
  words = text.split()
233
302
  word_counts = Counter(words)
234
303
  unique_words = len(word_counts)
@@ -15,6 +15,11 @@
15
15
 
16
16
  This materializer handles pandas DataFrame and Series objects.
17
17
 
18
+ Special features:
19
+ - Handles pandas DataFrames and Series with various data types
20
+ - Provides helpful error messages for custom data type errors
21
+ - Warns when custom data types are detected that might need additional libraries
22
+
18
23
  Environment Variables:
19
24
  ZENML_PANDAS_SAMPLE_ROWS: Controls the number of sample rows to include in
20
25
  visualizations. Defaults to 10 if not set.
@@ -41,6 +46,32 @@ CSV_FILENAME = "df.csv"
41
46
  # Default number of sample rows to display in visualizations
42
47
  DEFAULT_SAMPLE_ROWS = 10
43
48
 
49
+ # List of standard pandas/numpy dtype prefixes for type checking
50
+ STANDARD_DTYPE_PREFIXES = [
51
+ "int",
52
+ "float",
53
+ "bool",
54
+ "datetime",
55
+ "timedelta",
56
+ "object",
57
+ "category",
58
+ "string",
59
+ "complex",
60
+ ]
61
+
62
+
63
+ def is_standard_dtype(dtype_str: str) -> bool:
64
+ """Check if a dtype string represents a standard pandas/numpy dtype.
65
+
66
+ Args:
67
+ dtype_str: String representation of the dtype
68
+
69
+ Returns:
70
+ bool: True if it's a standard dtype, False otherwise
71
+ """
72
+ dtype_str = dtype_str.lower()
73
+ return any(prefix in dtype_str for prefix in STANDARD_DTYPE_PREFIXES)
74
+
44
75
 
45
76
  class PandasMaterializer(BaseMaterializer):
46
77
  """Materializer to read data to and from pandas."""
@@ -86,27 +117,60 @@ class PandasMaterializer(BaseMaterializer):
86
117
 
87
118
  Raises:
88
119
  ImportError: If pyarrow or fastparquet is not installed.
120
+ TypeError: Raised if there is an error when reading parquet files.
121
+ zenml_type_error: If the data type is a custom data type.
89
122
 
90
123
  Returns:
91
124
  The pandas dataframe or series.
92
125
  """
93
- if self.artifact_store.exists(self.parquet_path):
94
- if self.pyarrow_exists:
95
- with self.artifact_store.open(
96
- self.parquet_path, mode="rb"
97
- ) as f:
98
- df = pd.read_parquet(f)
126
+ try:
127
+ # First try normal loading
128
+ if self.artifact_store.exists(self.parquet_path):
129
+ if self.pyarrow_exists:
130
+ with self.artifact_store.open(
131
+ self.parquet_path, mode="rb"
132
+ ) as f:
133
+ df = pd.read_parquet(f)
134
+ else:
135
+ raise ImportError(
136
+ "You have an old version of a `PandasMaterializer` "
137
+ "data artifact stored in the artifact store "
138
+ "as a `.parquet` file, which requires `pyarrow` "
139
+ "for reading, You can install `pyarrow` by running "
140
+ "'`pip install pyarrow fastparquet`'."
141
+ )
99
142
  else:
100
- raise ImportError(
101
- "You have an old version of a `PandasMaterializer` "
102
- "data artifact stored in the artifact store "
103
- "as a `.parquet` file, which requires `pyarrow` "
104
- "for reading, You can install `pyarrow` by running "
105
- "'`pip install pyarrow fastparquet`'."
143
+ with self.artifact_store.open(self.csv_path, mode="rb") as f:
144
+ df = pd.read_csv(f, index_col=0, parse_dates=True)
145
+ except TypeError as e:
146
+ # Check for common data type error patterns
147
+ error_str = str(e).lower()
148
+ is_dtype_error = (
149
+ "not understood" in error_str
150
+ or "no type" in error_str
151
+ or "cannot deserialize" in error_str
152
+ or "data type" in error_str
153
+ )
154
+
155
+ if is_dtype_error:
156
+ # If the error is due to a custom data type, raise a ZenML TypeError
157
+ # This is to avoid the original error from being swallowed
158
+ # and to provide a more helpful error message
159
+ zenml_type_error = TypeError(
160
+ "Encountered an error with custom data types. This may be due to "
161
+ "missing libraries that were used when the data was originally created. "
162
+ "For example, you might need to install libraries like 'geopandas' for "
163
+ "GeoPandas data types, 'pandas-gbq' for BigQuery data types, or "
164
+ "'pyarrow' for Arrow data types. Make sure to import these libraries "
165
+ "in your step code as well as adding them to your step requirements, "
166
+ "even if you're not directly using them in your code. Pandas needs "
167
+ "these libraries to be imported to properly load the custom data types. "
168
+ "Try installing any packages that were used in previous pipeline steps "
169
+ "but might not be available in the current environment."
106
170
  )
107
- else:
108
- with self.artifact_store.open(self.csv_path, mode="rb") as f:
109
- df = pd.read_csv(f, index_col=0, parse_dates=True)
171
+ raise zenml_type_error from e
172
+ # We don't know how to handle this error, so re-raise the original error
173
+ raise e
110
174
 
111
175
  # validate the type of the data.
112
176
  def is_dataframe_or_series(
@@ -204,14 +268,46 @@ class PandasMaterializer(BaseMaterializer):
204
268
  Returns:
205
269
  The extracted metadata as a dictionary.
206
270
  """
207
- pandas_metadata: Dict[str, "MetadataType"] = {"shape": df.shape}
271
+ # Store whether it's a Series for later reference
272
+ is_series = isinstance(df, pd.Series)
208
273
 
209
- if isinstance(df, pd.Series):
210
- pandas_metadata["dtype"] = DType(df.dtype.type)
211
- pandas_metadata["mean"] = float(df.mean().item())
212
- pandas_metadata["std"] = float(df.std().item())
213
- pandas_metadata["min"] = float(df.min().item())
214
- pandas_metadata["max"] = float(df.max().item())
274
+ # Store original shape before conversion
275
+ original_shape = df.shape
276
+
277
+ # Convert Series to DataFrame for consistent handling of dtypes
278
+ if is_series:
279
+ series_obj = df # Keep original Series for some calculations
280
+ df = df.to_frame(name="series")
281
+
282
+ pandas_metadata: Dict[str, "MetadataType"] = {"shape": original_shape}
283
+
284
+ # Add information about custom data types to metadata
285
+ custom_types = {}
286
+ try:
287
+ for col, dtype in df.dtypes.items():
288
+ dtype_str = str(dtype)
289
+ if not is_standard_dtype(dtype_str):
290
+ col_name = "series" if is_series else str(col)
291
+ custom_types[col_name] = dtype_str
292
+ # Try to get module information if available
293
+ try:
294
+ module_name = dtype.type.__module__
295
+ custom_types[f"{col_name}_module"] = module_name
296
+ except (AttributeError, TypeError):
297
+ pass
298
+
299
+ if custom_types:
300
+ pandas_metadata["custom_types"] = custom_types
301
+ except Exception as e:
302
+ logger.debug(f"Error extracting custom type metadata: {e}")
303
+
304
+ if is_series:
305
+ # For Series, use the original series object for statistics
306
+ pandas_metadata["dtype"] = DType(series_obj.dtype.type)
307
+ pandas_metadata["mean"] = float(series_obj.mean().item())
308
+ pandas_metadata["std"] = float(series_obj.std().item())
309
+ pandas_metadata["min"] = float(series_obj.min().item())
310
+ pandas_metadata["max"] = float(series_obj.max().item())
215
311
 
216
312
  else:
217
313
  pandas_metadata["dtype"] = {
@@ -524,36 +524,16 @@ class TektonOrchestrator(ContainerizedOrchestrator):
524
524
  node_selector_constraint: Optional[Tuple[str, str]] = None
525
525
  pod_settings = step_settings.pod_settings
526
526
  if pod_settings:
527
- if pod_settings.host_ipc:
527
+ ignored_fields = pod_settings.model_fields_set - {
528
+ "node_selectors"
529
+ }
530
+ if ignored_fields:
528
531
  logger.warning(
529
- "Host IPC is set to `True` but not supported in "
530
- "this orchestrator. Ignoring..."
531
- )
532
- if pod_settings.affinity:
533
- logger.warning(
534
- "Affinity is set but not supported in Tekton with "
535
- "Tekton Pipelines 2.x. Ignoring..."
536
- )
537
- if pod_settings.tolerations:
538
- logger.warning(
539
- "Tolerations are set but not supported in "
540
- "Tekton with Tekton Pipelines 2.x. Ignoring..."
541
- )
542
- if pod_settings.volumes:
543
- logger.warning(
544
- "Volumes are set but not supported in Tekton with "
545
- "Tekton Pipelines 2.x. Ignoring..."
546
- )
547
- if pod_settings.volume_mounts:
548
- logger.warning(
549
- "Volume mounts are set but not supported in "
550
- "Tekton with Tekton Pipelines 2.x. Ignoring..."
551
- )
552
- if pod_settings.env or pod_settings.env_from:
553
- logger.warning(
554
- "Environment variables are set but not supported "
555
- "in Tekton with Tekton Pipelines 2.x. Ignoring..."
532
+ f"The following pod settings are not supported in "
533
+ f"Tekton with Tekton Pipelines 2.x and will be "
534
+ f"ignored: {list(ignored_fields)}."
556
535
  )
536
+
557
537
  # apply pod settings
558
538
  if (
559
539
  KFP_ACCELERATOR_NODE_SELECTOR_CONSTRAINT_LABEL
@@ -31,7 +31,6 @@ from zenml.constants import STR_FIELD_MAX_LENGTH, TEXT_FIELD_MAX_LENGTH
31
31
  from zenml.enums import ArtifactType, ModelStages
32
32
  from zenml.metadata.metadata_types import MetadataType
33
33
  from zenml.models.v2.base.base import BaseUpdate
34
- from zenml.models.v2.base.filter import AnyQuery
35
34
  from zenml.models.v2.base.page import Page
36
35
  from zenml.models.v2.base.scoped import (
37
36
  ProjectScopedFilter,
@@ -47,11 +46,15 @@ from zenml.models.v2.core.service import ServiceResponse
47
46
  from zenml.models.v2.core.tag import TagResponse
48
47
 
49
48
  if TYPE_CHECKING:
49
+ from sqlalchemy.sql.elements import ColumnElement
50
+
50
51
  from zenml.model.model import Model
51
52
  from zenml.models.v2.core.artifact_version import ArtifactVersionResponse
52
53
  from zenml.models.v2.core.model import ModelResponse
53
54
  from zenml.models.v2.core.pipeline_run import PipelineRunResponse
54
- from zenml.zen_stores.schemas import BaseSchema
55
+ from zenml.zen_stores.schemas import (
56
+ BaseSchema,
57
+ )
55
58
 
56
59
  AnySchema = TypeVar("AnySchema", bound=BaseSchema)
57
60
 
@@ -585,7 +588,6 @@ class ModelVersionFilter(
585
588
  *ProjectScopedFilter.CLI_EXCLUDE_FIELDS,
586
589
  *TaggableFilter.CLI_EXCLUDE_FIELDS,
587
590
  *RunMetadataFilterMixin.CLI_EXCLUDE_FIELDS,
588
- "model",
589
591
  ]
590
592
  API_MULTI_INPUT_PARAMS: ClassVar[List[str]] = [
591
593
  *ProjectScopedFilter.API_MULTI_INPUT_PARAMS,
@@ -615,41 +617,34 @@ class ModelVersionFilter(
615
617
  union_mode="left_to_right",
616
618
  )
617
619
 
618
- def apply_filter(
619
- self,
620
- query: AnyQuery,
621
- table: Type["AnySchema"],
622
- ) -> AnyQuery:
623
- """Applies the filter to a query.
620
+ def get_custom_filters(
621
+ self, table: Type["AnySchema"]
622
+ ) -> List[Union["ColumnElement[bool]"]]:
623
+ """Get custom filters.
624
624
 
625
625
  Args:
626
- query: The query to which to apply the filter.
627
626
  table: The query table.
628
627
 
629
628
  Returns:
630
- The query with filter applied.
631
-
632
- Raises:
633
- ValueError: if the filter is not scoped to a model.
629
+ A list of custom filters.
634
630
  """
635
- query = super().apply_filter(query=query, table=table)
636
-
637
- # The model scope must always be set and must be a UUID. If the
638
- # client sets this to a string, the server will try to resolve it to a
639
- # model ID.
640
- #
641
- # If not set by the client, the server will raise a ValueError.
642
- #
643
- # See: SqlZenStore._set_filter_model_id
644
-
645
- if not self.model:
646
- raise ValueError("Model scope missing from the filter.")
647
-
648
- if not isinstance(self.model, UUID):
649
- raise ValueError(
650
- f"Model scope must be a UUID, got {type(self.model)}."
651
- )
631
+ from sqlalchemy import and_
632
+
633
+ from zenml.zen_stores.schemas import (
634
+ ModelSchema,
635
+ ModelVersionSchema,
636
+ )
652
637
 
653
- query = query.where(getattr(table, "model_id") == self.model)
638
+ custom_filters = super().get_custom_filters(table)
639
+
640
+ if self.model:
641
+ value, operator = self._resolve_operator(self.model)
642
+ model_filter = and_(
643
+ ModelVersionSchema.model_id == ModelSchema.id, # type: ignore[arg-type]
644
+ self.generate_name_or_id_query_conditions(
645
+ value=self.model, table=ModelSchema
646
+ ),
647
+ )
648
+ custom_filters.append(model_filter)
654
649
 
655
- return query
650
+ return custom_filters
@@ -13,7 +13,7 @@
13
13
  # permissions and limitations under the License.
14
14
  """Models representing pipeline deployments."""
15
15
 
16
- from typing import Dict, Optional, TypeVar, Union
16
+ from typing import Any, Dict, Optional, TypeVar, Union
17
17
  from uuid import UUID
18
18
 
19
19
  from pydantic import Field
@@ -60,7 +60,7 @@ class PipelineDeploymentBase(BaseZenModel):
60
60
  step_configurations: Dict[str, Step] = Field(
61
61
  default={}, title="The step configurations for this deployment."
62
62
  )
63
- client_environment: Dict[str, str] = Field(
63
+ client_environment: Dict[str, Any] = Field(
64
64
  default={}, title="The client environment for this deployment."
65
65
  )
66
66
  client_version: Optional[str] = Field(
@@ -143,7 +143,7 @@ class PipelineDeploymentResponseMetadata(ProjectScopedResponseMetadata):
143
143
  step_configurations: Dict[str, Step] = Field(
144
144
  default={}, title="The step configurations for this deployment."
145
145
  )
146
- client_environment: Dict[str, str] = Field(
146
+ client_environment: Dict[str, Any] = Field(
147
147
  default={}, title="The client environment for this deployment."
148
148
  )
149
149
  client_version: Optional[str] = Field(
@@ -242,7 +242,7 @@ class PipelineDeploymentResponse(
242
242
  return self.get_metadata().step_configurations
243
243
 
244
244
  @property
245
- def client_environment(self) -> Dict[str, str]:
245
+ def client_environment(self) -> Dict[str, Any]:
246
246
  """The `client_environment` property.
247
247
 
248
248
  Returns: