zenml-nightly 0.68.0.dev20241027__py3-none-any.whl → 0.68.1.dev20241101__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. README.md +17 -11
  2. RELEASE_NOTES.md +9 -0
  3. zenml/VERSION +1 -1
  4. zenml/__init__.py +1 -1
  5. zenml/analytics/context.py +16 -1
  6. zenml/analytics/utils.py +18 -7
  7. zenml/artifacts/utils.py +40 -216
  8. zenml/cli/__init__.py +63 -90
  9. zenml/cli/base.py +3 -3
  10. zenml/cli/login.py +951 -0
  11. zenml/cli/server.py +462 -353
  12. zenml/cli/service_accounts.py +4 -4
  13. zenml/cli/stack.py +77 -2
  14. zenml/cli/stack_components.py +5 -16
  15. zenml/cli/user_management.py +0 -12
  16. zenml/cli/utils.py +24 -77
  17. zenml/client.py +46 -14
  18. zenml/config/compiler.py +1 -0
  19. zenml/config/global_config.py +9 -0
  20. zenml/config/pipeline_configurations.py +2 -1
  21. zenml/config/pipeline_run_configuration.py +2 -1
  22. zenml/constants.py +3 -9
  23. zenml/enums.py +1 -1
  24. zenml/exceptions.py +11 -0
  25. zenml/integrations/github/code_repositories/github_code_repository.py +1 -1
  26. zenml/login/__init__.py +16 -0
  27. zenml/login/credentials.py +346 -0
  28. zenml/login/credentials_store.py +603 -0
  29. zenml/login/pro/__init__.py +16 -0
  30. zenml/login/pro/client.py +496 -0
  31. zenml/login/pro/constants.py +34 -0
  32. zenml/login/pro/models.py +25 -0
  33. zenml/login/pro/organization/__init__.py +14 -0
  34. zenml/login/pro/organization/client.py +79 -0
  35. zenml/login/pro/organization/models.py +32 -0
  36. zenml/login/pro/tenant/__init__.py +14 -0
  37. zenml/login/pro/tenant/client.py +92 -0
  38. zenml/login/pro/tenant/models.py +174 -0
  39. zenml/login/pro/utils.py +121 -0
  40. zenml/{cli → login}/web_login.py +64 -28
  41. zenml/materializers/base_materializer.py +43 -9
  42. zenml/materializers/built_in_materializer.py +1 -1
  43. zenml/metadata/metadata_types.py +49 -0
  44. zenml/model/model.py +0 -38
  45. zenml/models/__init__.py +3 -0
  46. zenml/models/v2/base/base.py +12 -8
  47. zenml/models/v2/base/filter.py +9 -0
  48. zenml/models/v2/core/artifact_version.py +49 -10
  49. zenml/models/v2/core/component.py +54 -19
  50. zenml/models/v2/core/flavor.py +13 -13
  51. zenml/models/v2/core/model.py +3 -1
  52. zenml/models/v2/core/model_version.py +3 -5
  53. zenml/models/v2/core/model_version_artifact.py +3 -1
  54. zenml/models/v2/core/model_version_pipeline_run.py +3 -1
  55. zenml/models/v2/core/pipeline.py +3 -1
  56. zenml/models/v2/core/pipeline_run.py +23 -1
  57. zenml/models/v2/core/run_template.py +3 -1
  58. zenml/models/v2/core/stack.py +7 -3
  59. zenml/models/v2/core/step_run.py +43 -2
  60. zenml/models/v2/misc/auth_models.py +11 -2
  61. zenml/models/v2/misc/server_models.py +2 -0
  62. zenml/orchestrators/base_orchestrator.py +8 -4
  63. zenml/orchestrators/step_launcher.py +1 -0
  64. zenml/orchestrators/step_run_utils.py +10 -2
  65. zenml/orchestrators/step_runner.py +67 -55
  66. zenml/orchestrators/utils.py +45 -22
  67. zenml/pipelines/pipeline_decorator.py +5 -0
  68. zenml/pipelines/pipeline_definition.py +206 -160
  69. zenml/pipelines/run_utils.py +11 -10
  70. zenml/services/local/local_daemon_entrypoint.py +4 -4
  71. zenml/services/service.py +2 -2
  72. zenml/stack/stack.py +2 -6
  73. zenml/stack/stack_component.py +2 -7
  74. zenml/stack/utils.py +26 -14
  75. zenml/steps/base_step.py +8 -2
  76. zenml/steps/step_context.py +0 -3
  77. zenml/steps/step_invocation.py +14 -5
  78. zenml/steps/utils.py +1 -0
  79. zenml/utils/materializer_utils.py +1 -1
  80. zenml/utils/requirements_utils.py +71 -0
  81. zenml/utils/singleton.py +15 -3
  82. zenml/utils/source_utils.py +39 -2
  83. zenml/utils/visualization_utils.py +1 -1
  84. zenml/zen_server/auth.py +44 -39
  85. zenml/zen_server/deploy/__init__.py +7 -7
  86. zenml/zen_server/deploy/base_provider.py +46 -73
  87. zenml/zen_server/deploy/{local → daemon}/__init__.py +3 -3
  88. zenml/zen_server/deploy/{local/local_provider.py → daemon/daemon_provider.py} +44 -63
  89. zenml/zen_server/deploy/{local/local_zen_server.py → daemon/daemon_zen_server.py} +50 -22
  90. zenml/zen_server/deploy/deployer.py +90 -171
  91. zenml/zen_server/deploy/deployment.py +20 -12
  92. zenml/zen_server/deploy/docker/docker_provider.py +9 -28
  93. zenml/zen_server/deploy/docker/docker_zen_server.py +19 -3
  94. zenml/zen_server/deploy/helm/Chart.yaml +1 -1
  95. zenml/zen_server/deploy/helm/README.md +2 -2
  96. zenml/zen_server/exceptions.py +11 -0
  97. zenml/zen_server/jwt.py +9 -9
  98. zenml/zen_server/routers/auth_endpoints.py +30 -8
  99. zenml/zen_server/routers/stack_components_endpoints.py +1 -1
  100. zenml/zen_server/routers/workspaces_endpoints.py +1 -1
  101. zenml/zen_server/template_execution/runner_entrypoint_configuration.py +7 -4
  102. zenml/zen_server/template_execution/utils.py +6 -61
  103. zenml/zen_server/utils.py +64 -36
  104. zenml/zen_stores/base_zen_store.py +4 -49
  105. zenml/zen_stores/migrations/versions/0.68.1_release.py +23 -0
  106. zenml/zen_stores/migrations/versions/c22561cbb3a9_add_artifact_unique_constraints.py +86 -0
  107. zenml/zen_stores/rest_zen_store.py +325 -147
  108. zenml/zen_stores/schemas/api_key_schemas.py +9 -4
  109. zenml/zen_stores/schemas/artifact_schemas.py +21 -2
  110. zenml/zen_stores/schemas/artifact_visualization_schemas.py +1 -1
  111. zenml/zen_stores/schemas/component_schemas.py +49 -6
  112. zenml/zen_stores/schemas/device_schemas.py +9 -4
  113. zenml/zen_stores/schemas/flavor_schemas.py +1 -1
  114. zenml/zen_stores/schemas/model_schemas.py +1 -1
  115. zenml/zen_stores/schemas/service_schemas.py +1 -1
  116. zenml/zen_stores/schemas/step_run_schemas.py +1 -1
  117. zenml/zen_stores/schemas/trigger_schemas.py +1 -1
  118. zenml/zen_stores/sql_zen_store.py +393 -140
  119. zenml/zen_stores/template_utils.py +3 -1
  120. {zenml_nightly-0.68.0.dev20241027.dist-info → zenml_nightly-0.68.1.dev20241101.dist-info}/METADATA +18 -12
  121. {zenml_nightly-0.68.0.dev20241027.dist-info → zenml_nightly-0.68.1.dev20241101.dist-info}/RECORD +124 -107
  122. zenml/api.py +0 -60
  123. {zenml_nightly-0.68.0.dev20241027.dist-info → zenml_nightly-0.68.1.dev20241101.dist-info}/LICENSE +0 -0
  124. {zenml_nightly-0.68.0.dev20241027.dist-info → zenml_nightly-0.68.1.dev20241101.dist-info}/WHEEL +0 -0
  125. {zenml_nightly-0.68.0.dev20241027.dist-info → zenml_nightly-0.68.1.dev20241101.dist-info}/entry_points.txt +0 -0
zenml/exceptions.py CHANGED
@@ -47,6 +47,17 @@ class AuthorizationException(ZenMLBaseException):
47
47
  """Raised when an authorization error occurred while trying to access a ZenML resource ."""
48
48
 
49
49
 
50
+ class CredentialsNotValid(AuthorizationException):
51
+ """Raised when the credentials provided are invalid.
52
+
53
+ This is a subclass of AuthorizationException and should only be raised when
54
+ the authentication credentials are invalid (e.g. expired API token, invalid
55
+ username/password, invalid signature). If caught by the ZenML client, it
56
+ will trigger an invalidation of the currently cached API token and a
57
+ re-authentication flow.
58
+ """
59
+
60
+
50
61
  class DoesNotExistException(ZenMLBaseException):
51
62
  """Raises exception when the entity does not exist in the system but an action is being done that requires it to be present."""
52
63
 
@@ -164,7 +164,7 @@ class GitHubCodeRepository(BaseCodeRepository):
164
164
  try:
165
165
  with open(local_path, "wb") as f:
166
166
  f.write(content.decoded_content)
167
- except (GithubException, IOError) as e:
167
+ except (GithubException, IOError, AssertionError) as e:
168
168
  logger.error("Error processing %s: %s", content.path, e)
169
169
 
170
170
  def get_local_context(self, path: str) -> Optional[LocalRepositoryContext]:
@@ -0,0 +1,16 @@
1
+ # Copyright (c) ZenML GmbH 2024. All Rights Reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at:
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
12
+ # or implied. See the License for the specific language governing
13
+ # permissions and limitations under the License.
14
+ """ZenML login utilities."""
15
+
16
+
@@ -0,0 +1,346 @@
1
+ # Copyright (c) ZenML GmbH 2024. All Rights Reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at:
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
12
+ # or implied. See the License for the specific language governing
13
+ # permissions and limitations under the License.
14
+ """ZenML login credentials models."""
15
+
16
+ from datetime import datetime, timedelta, timezone
17
+ from typing import Any, Dict, Optional, Union
18
+ from urllib.parse import urlparse
19
+ from uuid import UUID
20
+
21
+ from pydantic import BaseModel, ConfigDict
22
+
23
+ from zenml.login.pro.constants import ZENML_PRO_API_URL, ZENML_PRO_URL
24
+ from zenml.login.pro.tenant.models import TenantRead, TenantStatus
25
+ from zenml.models import ServerModel
26
+ from zenml.services.service_status import ServiceState
27
+ from zenml.utils.enum_utils import StrEnum
28
+ from zenml.utils.string_utils import get_human_readable_time
29
+
30
+
31
+ class ServerType(StrEnum):
32
+ """The type of server."""
33
+
34
+ PRO_API = "PRO_API"
35
+ PRO = "PRO"
36
+ REMOTE = "REMOTE"
37
+ LOCAL = "LOCAL"
38
+
39
+
40
+ class APIToken(BaseModel):
41
+ """Cached API Token."""
42
+
43
+ access_token: str
44
+ expires_in: Optional[int] = None
45
+ expires_at: Optional[datetime] = None
46
+ leeway: Optional[int] = None
47
+ cookie_name: Optional[str] = None
48
+ device_id: Optional[UUID] = None
49
+ device_metadata: Optional[Dict[str, Any]] = None
50
+
51
+ @property
52
+ def expires_at_with_leeway(self) -> Optional[datetime]:
53
+ """Get the token expiration time with leeway.
54
+
55
+ Returns:
56
+ The token expiration time with leeway.
57
+ """
58
+ if not self.expires_at:
59
+ return None
60
+ if not self.leeway:
61
+ return self.expires_at
62
+ return self.expires_at - timedelta(seconds=self.leeway)
63
+
64
+ @property
65
+ def expired(self) -> bool:
66
+ """Check if the token is expired.
67
+
68
+ Returns:
69
+ bool: True if the token is expired, False otherwise.
70
+ """
71
+ expires_at = self.expires_at_with_leeway
72
+ if not expires_at:
73
+ return False
74
+ return expires_at < datetime.now(timezone.utc)
75
+
76
+ model_config = ConfigDict(
77
+ # Allow extra attributes to allow backwards compatibility
78
+ extra="allow",
79
+ )
80
+
81
+
82
+ class ServerCredentials(BaseModel):
83
+ """Cached Server Credentials."""
84
+
85
+ url: str
86
+ api_key: Optional[str] = None
87
+ api_token: Optional[APIToken] = None
88
+ username: Optional[str] = None
89
+ password: Optional[str] = None
90
+
91
+ # Extra server attributes
92
+ server_id: Optional[UUID] = None
93
+ server_name: Optional[str] = None
94
+ organization_name: Optional[str] = None
95
+ organization_id: Optional[UUID] = None
96
+ status: Optional[str] = None
97
+ version: Optional[str] = None
98
+
99
+ @property
100
+ def id(self) -> str:
101
+ """Get the server identifier.
102
+
103
+ Returns:
104
+ The server identifier.
105
+ """
106
+ if self.server_id:
107
+ return str(self.server_id)
108
+ return self.url
109
+
110
+ @property
111
+ def type(self) -> ServerType:
112
+ """Get the server type.
113
+
114
+ Returns:
115
+ The server type.
116
+ """
117
+ from zenml.login.pro.utils import is_zenml_pro_server_url
118
+
119
+ if self.url == ZENML_PRO_API_URL:
120
+ return ServerType.PRO_API
121
+ if self.organization_id or is_zenml_pro_server_url(self.url):
122
+ return ServerType.PRO
123
+ if urlparse(self.url).hostname in [
124
+ "localhost",
125
+ "127.0.0.1",
126
+ "host.docker.internal",
127
+ ]:
128
+ return ServerType.LOCAL
129
+ return ServerType.REMOTE
130
+
131
+ def update_server_info(
132
+ self, server_info: Union[ServerModel, TenantRead]
133
+ ) -> None:
134
+ """Update with server information received from the server itself or from a ZenML Pro tenant descriptor.
135
+
136
+ Args:
137
+ server_info: The server information to update with.
138
+ """
139
+ if isinstance(server_info, ServerModel):
140
+ # The server ID doesn't change during the lifetime of the server
141
+ self.server_id = self.server_id or server_info.id
142
+
143
+ # All other attributes can change during the lifetime of the server
144
+ server_name = (
145
+ server_info.metadata.get("tenant_name") or server_info.name
146
+ )
147
+ if server_name:
148
+ self.server_name = server_name
149
+ organization_id = server_info.metadata.get("organization_id")
150
+ if organization_id:
151
+ self.organization_id = UUID(organization_id)
152
+ self.version = server_info.version or self.version
153
+ # The server information was retrieved from the server itself, so we
154
+ # can assume that the server is available
155
+ self.status = "available"
156
+ else:
157
+ self.server_id = server_info.id
158
+ self.server_name = server_info.name
159
+ self.organization_name = server_info.organization_name
160
+ self.organization_id = server_info.organization_id
161
+ self.status = server_info.status
162
+ self.version = server_info.version
163
+
164
+ @property
165
+ def is_available(self) -> bool:
166
+ """Check if the server is available (running and authenticated).
167
+
168
+ Returns:
169
+ True if the server is available, False otherwise.
170
+ """
171
+ if self.status not in [TenantStatus.AVAILABLE, ServiceState.ACTIVE]:
172
+ return False
173
+ if (
174
+ self.api_key
175
+ or self.api_token
176
+ or self.username
177
+ and self.password is not None
178
+ or self.type in [ServerType.PRO, ServerType.LOCAL]
179
+ ):
180
+ return True
181
+ if self.api_token and not self.api_token.expired:
182
+ return True
183
+ return False
184
+
185
+ @property
186
+ def auth_status(self) -> str:
187
+ """Get the authentication status.
188
+
189
+ Returns:
190
+ The authentication status.
191
+ """
192
+ if self.api_key:
193
+ return "API key"
194
+ if self.username and self.password is not None:
195
+ return "password"
196
+ if not self.api_token:
197
+ if self.type == ServerType.LOCAL:
198
+ return "no authentication required"
199
+ return "N/A"
200
+ expires_at = self.api_token.expires_at_with_leeway
201
+ if not expires_at:
202
+ return "never expires"
203
+ if expires_at < datetime.now(timezone.utc):
204
+ return "expired at " + self.expires_at
205
+
206
+ return f"valid until {self.expires_at} (in {self.expires_in})"
207
+
208
+ @property
209
+ def expires_at(self) -> str:
210
+ """Get the expiration time of the token as a string.
211
+
212
+ Returns:
213
+ The expiration time of the token as a string.
214
+ """
215
+ if not self.api_token:
216
+ return "N/A"
217
+ expires_at = self.api_token.expires_at_with_leeway
218
+ if not expires_at:
219
+ return "never"
220
+
221
+ # Convert the date in the local timezone
222
+ local_expires_at = expires_at.astimezone()
223
+ return local_expires_at.strftime("%Y-%m-%d %H:%M:%S %Z")
224
+
225
+ @property
226
+ def expires_in(self) -> str:
227
+ """Get the time remaining until the token expires.
228
+
229
+ Returns:
230
+ The time remaining until the token expires.
231
+ """
232
+ if not self.api_token:
233
+ return "N/A"
234
+ expires_at = self.api_token.expires_at_with_leeway
235
+ if not expires_at:
236
+ return "never"
237
+
238
+ # Get the time remaining until the token expires
239
+ expires_in = expires_at - datetime.now(timezone.utc)
240
+ return get_human_readable_time(expires_in.total_seconds())
241
+
242
+ @property
243
+ def dashboard_url(self) -> str:
244
+ """Get the URL to the ZenML dashboard for this server.
245
+
246
+ Returns:
247
+ The URL to the ZenML dashboard for this server.
248
+ """
249
+ if self.organization_id and self.server_id:
250
+ return (
251
+ ZENML_PRO_URL
252
+ + f"/organizations/{str(self.organization_id)}/tenants/{str(self.server_id)}"
253
+ )
254
+ return self.url
255
+
256
+ @property
257
+ def dashboard_organization_url(self) -> str:
258
+ """Get the URL to the ZenML Pro dashboard for this tenant's organization.
259
+
260
+ Returns:
261
+ The URL to the ZenML Pro dashboard for this tenant's organization.
262
+ """
263
+ if self.organization_id:
264
+ return (
265
+ ZENML_PRO_URL + f"/organizations/{str(self.organization_id)}"
266
+ )
267
+ return ""
268
+
269
+ @property
270
+ def dashboard_hyperlink(self) -> str:
271
+ """Get the hyperlink to the ZenML dashboard for this tenant.
272
+
273
+ Returns:
274
+ The hyperlink to the ZenML dashboard for this tenant.
275
+ """
276
+ return f"[link={self.dashboard_url}]{self.dashboard_url}[/link]"
277
+
278
+ @property
279
+ def api_hyperlink(self) -> str:
280
+ """Get the hyperlink to the ZenML OpenAPI dashboard for this tenant.
281
+
282
+ Returns:
283
+ The hyperlink to the ZenML OpenAPI dashboard for this tenant.
284
+ """
285
+ api_url = self.url + "/docs"
286
+ return f"[link={api_url}]{self.url}[/link]"
287
+
288
+ @property
289
+ def server_name_hyperlink(self) -> str:
290
+ """Get the hyperlink to the ZenML dashboard for this server using its name.
291
+
292
+ Returns:
293
+ The hyperlink to the ZenML dashboard for this server using its name.
294
+ """
295
+ if self.server_name is None:
296
+ return "N/A"
297
+ return f"[link={self.dashboard_url}]{self.server_name}[/link]"
298
+
299
+ @property
300
+ def server_id_hyperlink(self) -> str:
301
+ """Get the hyperlink to the ZenML dashboard for this server using its ID.
302
+
303
+ Returns:
304
+ The hyperlink to the ZenML dashboard for this server using its ID.
305
+ """
306
+ if self.server_id is None:
307
+ return "N/A"
308
+ return f"[link={self.dashboard_url}]{str(self.server_id)}[/link]"
309
+
310
+ @property
311
+ def organization_hyperlink(self) -> str:
312
+ """Get the hyperlink to the ZenML Pro dashboard for this server's organization.
313
+
314
+ Returns:
315
+ The hyperlink to the ZenML Pro dashboard for this server's
316
+ organization.
317
+ """
318
+ if self.organization_name:
319
+ return self.organization_name_hyperlink
320
+ if self.organization_id:
321
+ return self.organization_id_hyperlink
322
+ return "N/A"
323
+
324
+ @property
325
+ def organization_name_hyperlink(self) -> str:
326
+ """Get the hyperlink to the ZenML Pro dashboard for this server's organization using its name.
327
+
328
+ Returns:
329
+ The hyperlink to the ZenML Pro dashboard for this server's
330
+ organization using its name.
331
+ """
332
+ if self.organization_name is None:
333
+ return "N/A"
334
+ return f"[link={self.dashboard_organization_url}]{self.organization_name}[/link]"
335
+
336
+ @property
337
+ def organization_id_hyperlink(self) -> str:
338
+ """Get the hyperlink to the ZenML Pro dashboard for this tenant's organization using its ID.
339
+
340
+ Returns:
341
+ The hyperlink to the ZenML Pro dashboard for this tenant's
342
+ organization using its ID.
343
+ """
344
+ if self.organization_id is None:
345
+ return "N/A"
346
+ return f"[link={self.dashboard_organization_url}]{self.organization_id}[/link]"