polyaxon 2.0.0rc49__py3-none-any.whl → 2.4.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (177) hide show
  1. polyaxon/_auxiliaries/cleaner.py +8 -3
  2. polyaxon/_auxiliaries/init.py +7 -2
  3. polyaxon/_auxiliaries/notifier.py +8 -2
  4. polyaxon/_auxiliaries/sidecar.py +30 -2
  5. polyaxon/_cli/artifacts.py +96 -11
  6. polyaxon/_cli/components.py +96 -11
  7. polyaxon/_cli/config.py +118 -22
  8. polyaxon/_cli/dashboard.py +15 -2
  9. polyaxon/_cli/init.py +1 -1
  10. polyaxon/_cli/models.py +96 -11
  11. polyaxon/_cli/operations.py +267 -90
  12. polyaxon/_cli/project_versions.py +139 -6
  13. polyaxon/_cli/projects.py +23 -9
  14. polyaxon/_cli/run.py +37 -9
  15. polyaxon/_cli/services/agent.py +2 -2
  16. polyaxon/_cli/services/clean_artifacts.py +1 -1
  17. polyaxon/_cli/services/sidecar.py +8 -1
  18. polyaxon/_client/client.py +17 -0
  19. polyaxon/_client/mixin.py +39 -0
  20. polyaxon/_client/project.py +218 -23
  21. polyaxon/_client/run.py +131 -33
  22. polyaxon/_compiler/contexts/contexts.py +2 -2
  23. polyaxon/_compiler/contexts/ray_job.py +4 -2
  24. polyaxon/_compiler/resolver/agent.py +12 -2
  25. polyaxon/_compiler/resolver/runtime.py +2 -2
  26. polyaxon/_contexts/paths.py +4 -7
  27. polyaxon/_deploy/operators/compose.py +1 -27
  28. polyaxon/_deploy/schemas/deployment.py +4 -1
  29. polyaxon/_deploy/schemas/intervals.py +0 -7
  30. polyaxon/_deploy/schemas/proxy.py +1 -0
  31. polyaxon/_deploy/schemas/service.py +11 -1
  32. polyaxon/_docker/converter/base/base.py +8 -0
  33. polyaxon/_docker/executor.py +10 -4
  34. polyaxon/_env_vars/getters/owner_entity.py +4 -2
  35. polyaxon/_env_vars/getters/project.py +4 -2
  36. polyaxon/_env_vars/getters/run.py +5 -2
  37. polyaxon/_env_vars/keys.py +7 -1
  38. polyaxon/_flow/__init__.py +2 -0
  39. polyaxon/_flow/builds/__init__.py +19 -6
  40. polyaxon/_flow/component/base.py +1 -0
  41. polyaxon/_flow/component/component.py +14 -0
  42. polyaxon/_flow/environment/__init__.py +8 -8
  43. polyaxon/_flow/hooks/__init__.py +19 -6
  44. polyaxon/_flow/init/__init__.py +6 -6
  45. polyaxon/_flow/matrix/iterative.py +0 -1
  46. polyaxon/_flow/matrix/tuner.py +18 -6
  47. polyaxon/_flow/operations/operation.py +44 -17
  48. polyaxon/_flow/plugins/__init__.py +6 -0
  49. polyaxon/_flow/run/__init__.py +2 -2
  50. polyaxon/_flow/run/dag.py +2 -2
  51. polyaxon/_flow/run/dask/dask.py +0 -1
  52. polyaxon/_flow/run/dask/replica.py +3 -3
  53. polyaxon/_flow/run/enums.py +5 -0
  54. polyaxon/_flow/run/job.py +4 -4
  55. polyaxon/_flow/run/kubeflow/mpi_job.py +1 -2
  56. polyaxon/_flow/run/kubeflow/mx_job.py +1 -2
  57. polyaxon/_flow/run/kubeflow/paddle_job.py +35 -4
  58. polyaxon/_flow/run/kubeflow/pytorch_job.py +51 -5
  59. polyaxon/_flow/run/kubeflow/replica.py +4 -4
  60. polyaxon/_flow/run/kubeflow/scheduling_policy.py +12 -0
  61. polyaxon/_flow/run/kubeflow/tf_job.py +3 -3
  62. polyaxon/_flow/run/kubeflow/xgboost_job.py +1 -2
  63. polyaxon/_flow/run/ray/ray.py +2 -3
  64. polyaxon/_flow/run/ray/replica.py +3 -3
  65. polyaxon/_flow/run/service.py +4 -4
  66. polyaxon/_fs/fs.py +7 -2
  67. polyaxon/_fs/utils.py +3 -2
  68. polyaxon/_k8s/converter/base/base.py +2 -1
  69. polyaxon/_k8s/converter/base/main.py +1 -0
  70. polyaxon/_k8s/converter/base/sidecar.py +16 -1
  71. polyaxon/_k8s/converter/common/accelerators.py +7 -4
  72. polyaxon/_k8s/converter/converters/job.py +1 -1
  73. polyaxon/_k8s/converter/converters/kubeflow/paddle_job.py +1 -0
  74. polyaxon/_k8s/converter/converters/kubeflow/pytroch_job.py +2 -0
  75. polyaxon/_k8s/converter/converters/kubeflow/tf_job.py +1 -0
  76. polyaxon/_k8s/converter/converters/ray_job.py +4 -2
  77. polyaxon/_k8s/custom_resources/dask_job.py +3 -0
  78. polyaxon/_k8s/custom_resources/kubeflow/common.py +4 -1
  79. polyaxon/_k8s/custom_resources/kubeflow/paddle_job.py +10 -1
  80. polyaxon/_k8s/custom_resources/kubeflow/pytorch_job.py +14 -1
  81. polyaxon/_k8s/custom_resources/kubeflow/tf_job.py +4 -0
  82. polyaxon/_k8s/custom_resources/ray_job.py +3 -0
  83. polyaxon/_k8s/custom_resources/setter.py +1 -1
  84. polyaxon/_k8s/executor/async_executor.py +2 -0
  85. polyaxon/_k8s/executor/base.py +23 -6
  86. polyaxon/_k8s/logging/async_monitor.py +150 -5
  87. polyaxon/_k8s/manager/async_manager.py +96 -23
  88. polyaxon/_k8s/manager/base.py +4 -0
  89. polyaxon/_k8s/manager/manager.py +282 -134
  90. polyaxon/_local_process/__init__.py +0 -0
  91. polyaxon/_local_process/agent.py +6 -0
  92. polyaxon/_local_process/converter/__init__.py +1 -0
  93. polyaxon/_local_process/converter/base/__init__.py +1 -0
  94. polyaxon/_local_process/converter/base/base.py +140 -0
  95. polyaxon/_local_process/converter/base/containers.py +69 -0
  96. polyaxon/_local_process/converter/base/env_vars.py +253 -0
  97. polyaxon/_local_process/converter/base/init.py +414 -0
  98. polyaxon/_local_process/converter/base/main.py +74 -0
  99. polyaxon/_local_process/converter/base/mounts.py +82 -0
  100. polyaxon/_local_process/converter/converters/__init__.py +8 -0
  101. polyaxon/_local_process/converter/converters/job.py +40 -0
  102. polyaxon/_local_process/converter/converters/service.py +41 -0
  103. polyaxon/_local_process/converter/mixins.py +38 -0
  104. polyaxon/_local_process/executor.py +132 -0
  105. polyaxon/_local_process/process_types.py +39 -0
  106. polyaxon/_managers/agent.py +2 -0
  107. polyaxon/_managers/home.py +2 -1
  108. polyaxon/_operations/tuner.py +1 -0
  109. polyaxon/_polyaxonfile/check.py +2 -0
  110. polyaxon/_polyaxonfile/manager/operations.py +3 -0
  111. polyaxon/_polyaxonfile/manager/workflows.py +2 -0
  112. polyaxon/_polyaxonfile/specs/compiled_operation.py +1 -0
  113. polyaxon/_polyaxonfile/specs/operation.py +1 -0
  114. polyaxon/_polyaxonfile/specs/sections.py +3 -0
  115. polyaxon/_pql/manager.py +1 -1
  116. polyaxon/_runner/agent/async_agent.py +97 -21
  117. polyaxon/_runner/agent/base_agent.py +27 -9
  118. polyaxon/_runner/agent/client.py +15 -1
  119. polyaxon/_runner/agent/sync_agent.py +85 -20
  120. polyaxon/_runner/converter/converter.py +6 -2
  121. polyaxon/_runner/executor.py +13 -7
  122. polyaxon/_schemas/agent.py +27 -1
  123. polyaxon/_schemas/client.py +30 -3
  124. polyaxon/_schemas/installation.py +4 -3
  125. polyaxon/_schemas/lifecycle.py +10 -5
  126. polyaxon/_schemas/log_handler.py +2 -3
  127. polyaxon/_schemas/types/artifacts.py +3 -3
  128. polyaxon/_schemas/types/dockerfile.py +3 -3
  129. polyaxon/_schemas/types/file.py +3 -3
  130. polyaxon/_schemas/types/git.py +3 -3
  131. polyaxon/_schemas/types/tensorboard.py +3 -3
  132. polyaxon/_sdk/api/agents_v1_api.py +1076 -73
  133. polyaxon/_sdk/api/organizations_v1_api.py +371 -10
  134. polyaxon/_sdk/api/project_dashboards_v1_api.py +12 -12
  135. polyaxon/_sdk/api/project_searches_v1_api.py +12 -12
  136. polyaxon/_sdk/api/projects_v1_api.py +221 -44
  137. polyaxon/_sdk/api/runs_v1_api.py +917 -445
  138. polyaxon/_sdk/api/service_accounts_v1_api.py +16 -16
  139. polyaxon/_sdk/api/teams_v1_api.py +2827 -375
  140. polyaxon/_sdk/api/users_v1_api.py +231 -55
  141. polyaxon/_sdk/async_client/api_client.py +4 -0
  142. polyaxon/_sdk/schemas/__init__.py +10 -2
  143. polyaxon/_sdk/schemas/v1_agent.py +2 -1
  144. polyaxon/_sdk/schemas/v1_agent_reconcile_body_request.py +14 -0
  145. polyaxon/_sdk/schemas/v1_artifact_tree.py +1 -1
  146. polyaxon/_sdk/schemas/v1_dashboard_spec.py +4 -0
  147. polyaxon/_sdk/schemas/v1_events_response.py +4 -0
  148. polyaxon/_sdk/schemas/v1_organization.py +1 -0
  149. polyaxon/_sdk/schemas/v1_preset.py +8 -0
  150. polyaxon/_sdk/schemas/v1_project.py +1 -0
  151. polyaxon/_sdk/schemas/v1_project_settings.py +4 -2
  152. polyaxon/_sdk/schemas/v1_run.py +2 -2
  153. polyaxon/_sdk/schemas/v1_run_edge_lineage.py +14 -0
  154. polyaxon/_sdk/schemas/v1_run_edges_graph.py +9 -0
  155. polyaxon/_sdk/schemas/v1_section_spec.py +7 -2
  156. polyaxon/_sdk/schemas/v1_settings_catalog.py +1 -0
  157. polyaxon/_sdk/schemas/v1_team.py +3 -0
  158. polyaxon/_sdk/schemas/v1_user.py +1 -2
  159. polyaxon/_sdk/schemas/v1_user_access.py +17 -0
  160. polyaxon/_services/values.py +1 -0
  161. polyaxon/_sidecar/container/__init__.py +39 -18
  162. polyaxon/_sidecar/container/monitors/__init__.py +1 -0
  163. polyaxon/_sidecar/container/monitors/logs.py +10 -13
  164. polyaxon/_sidecar/container/monitors/spec.py +24 -0
  165. polyaxon/_sidecar/ignore.py +0 -1
  166. polyaxon/_utils/fqn_utils.py +25 -2
  167. polyaxon/client.py +1 -1
  168. polyaxon/pkg.py +1 -1
  169. polyaxon/schemas.py +8 -1
  170. polyaxon/settings.py +6 -0
  171. {polyaxon-2.0.0rc49.dist-info → polyaxon-2.4.0rc1.dist-info}/METADATA +43 -43
  172. {polyaxon-2.0.0rc49.dist-info → polyaxon-2.4.0rc1.dist-info}/RECORD +176 -155
  173. {polyaxon-2.0.0rc49.dist-info → polyaxon-2.4.0rc1.dist-info}/WHEEL +1 -1
  174. polyaxon/_sdk/schemas/v1_project_user_access.py +0 -10
  175. {polyaxon-2.0.0rc49.dist-info → polyaxon-2.4.0rc1.dist-info}/LICENSE +0 -0
  176. {polyaxon-2.0.0rc49.dist-info → polyaxon-2.4.0rc1.dist-info}/entry_points.txt +0 -0
  177. {polyaxon-2.0.0rc49.dist-info → polyaxon-2.4.0rc1.dist-info}/top_level.txt +0 -0
@@ -38,7 +38,7 @@ class AsyncK8sManager(BaseK8sManager):
38
38
  if self.api_client:
39
39
  await self.api_client.close()
40
40
 
41
- async def get_version(self, reraise=False):
41
+ async def get_version(self, reraise: bool = False):
42
42
  try:
43
43
  version = await self.k8s_version_api.get_code()
44
44
  return version.to_dict()
@@ -47,48 +47,68 @@ class AsyncK8sManager(BaseK8sManager):
47
47
  if reraise:
48
48
  raise PolyaxonK8sError("Connection error: %s" % e) from e
49
49
 
50
- async def get_pod(self, name, reraise=False) -> Optional[client.V1Pod]:
50
+ async def get_pod(
51
+ self, name, reraise: bool = False, namespace: str = None
52
+ ) -> Optional[client.V1Pod]:
51
53
  try:
52
54
  return await self.k8s_api.read_namespaced_pod( # type: ignore[attr-defined]
53
- name=name, namespace=self.namespace
55
+ name=name, namespace=namespace or self.namespace
54
56
  )
55
57
  except ApiException as e:
56
58
  if reraise:
57
59
  raise PolyaxonK8sError("Connection error: %s" % e) from e
58
60
  return None
59
61
 
60
- async def is_pod_running(self, pod_id: str, container_id: str) -> bool:
61
- event = await self.k8s_api.read_namespaced_pod_status(pod_id, self.namespace) # type: ignore[attr-defined]
62
+ async def is_pod_running(
63
+ self, pod_id: str, container_id: str, namespace: str = None
64
+ ) -> bool:
65
+ event = await self.k8s_api.read_namespaced_pod_status(pod_id, namespace=namespace or self.namespace) # type: ignore[attr-defined]
62
66
  return is_pod_running(event, container_id)
63
67
 
64
68
  async def _list_namespace_resource(
65
- self, resource_api, reraise=False, **kwargs
69
+ self, resource_api, reraise: bool = False, namespace: str = None, **kwargs
66
70
  ) -> List:
67
71
  try:
68
- res = await resource_api(namespace=self.namespace, **kwargs)
69
- return [p for p in res.items]
72
+ res = await resource_api(namespace=namespace or self.namespace, **kwargs)
73
+ if isinstance(res, dict):
74
+ items = res["items"]
75
+ else:
76
+ items = res.items
77
+ return [p for p in items]
70
78
  except ApiException as e:
71
79
  logger.error("K8S error: {}".format(e))
72
80
  if reraise:
73
81
  raise PolyaxonK8sError("Connection error: %s" % e) from e
74
82
  return []
75
83
 
76
- async def list_pods(self, reraise=False, **kwargs) -> List[client.V1Pod]:
84
+ async def list_pods(
85
+ self, reraise: bool = False, namespace: str = None, **kwargs
86
+ ) -> List[client.V1Pod]:
77
87
  return await self._list_namespace_resource(
78
88
  resource_api=self.k8s_api.list_namespaced_pod, # type: ignore[attr-defined]
79
89
  reraise=reraise,
90
+ namespace=namespace,
80
91
  **kwargs,
81
92
  )
82
93
 
83
- async def list_jobs(self, reraise=False, **kwargs) -> List[client.V1Job]:
94
+ async def list_jobs(
95
+ self, reraise: bool = False, namespace: str = None, **kwargs
96
+ ) -> List[client.V1Job]:
84
97
  return await self._list_namespace_resource(
85
98
  resource_api=self.k8s_batch_api.list_namespaced_job, # type: ignore[attr-defined]
86
99
  reraise=reraise,
100
+ namespace=namespace,
87
101
  **kwargs,
88
102
  )
89
103
 
90
104
  async def list_custom_objects(
91
- self, group, version, plural, reraise=False, **kwargs
105
+ self,
106
+ group,
107
+ version,
108
+ plural,
109
+ reraise: bool = False,
110
+ namespace: str = None,
111
+ **kwargs
92
112
  ) -> List:
93
113
  return await self._list_namespace_resource(
94
114
  resource_api=self.k8s_custom_object_api.list_namespaced_custom_object, # type: ignore[attr-defined]
@@ -96,49 +116,64 @@ class AsyncK8sManager(BaseK8sManager):
96
116
  group=group,
97
117
  version=version,
98
118
  plural=plural,
119
+ namespace=namespace,
99
120
  **kwargs,
100
121
  )
101
122
 
102
123
  async def list_services(
103
- self, reraise: bool = False, **kwargs
124
+ self, reraise: bool = False, namespace: str = None, **kwargs
104
125
  ) -> List[client.V1Service]:
105
126
  return await self._list_namespace_resource(
106
127
  resource_api=self.k8s_api.list_namespaced_service, # type: ignore[attr-defined]
107
128
  reraise=reraise,
129
+ namespace=namespace,
108
130
  **kwargs,
109
131
  )
110
132
 
111
133
  async def list_deployments(
112
- self, reraise: bool = False, **kwargs
134
+ self, reraise: bool = False, namespace: str = None, **kwargs
113
135
  ) -> List[client.V1Deployment]:
114
136
  return await self._list_namespace_resource(
115
137
  resource_api=self.k8s_apps_api.list_namespaced_deployment, # type: ignore[attr-defined]
116
138
  reraise=reraise,
139
+ namespace=namespace,
117
140
  **kwargs,
118
141
  )
119
142
 
120
143
  async def create_custom_object(
121
- self, name: str, group: str, version: str, plural: str, body: Dict
144
+ self,
145
+ name: str,
146
+ group: str,
147
+ version: str,
148
+ plural: str,
149
+ body: Dict,
150
+ namespace: str = None,
122
151
  ) -> Dict:
123
152
  resp = await self.k8s_custom_object_api.create_namespaced_custom_object( # type: ignore[attr-defined]
124
153
  group=group,
125
154
  version=version,
126
155
  plural=plural,
127
- namespace=self.namespace,
156
+ namespace=namespace or self.namespace,
128
157
  body=body,
129
158
  )
130
159
  logger.debug("Custom object `{}` was created".format(name))
131
160
  return resp
132
161
 
133
162
  async def update_custom_object(
134
- self, name: str, group: str, version: str, plural: str, body: Dict
163
+ self,
164
+ name: str,
165
+ group: str,
166
+ version: str,
167
+ plural: str,
168
+ body: Dict,
169
+ namespace: str = None,
135
170
  ) -> Dict:
136
171
  resp = await self.k8s_custom_object_api.patch_namespaced_custom_object( # type: ignore[attr-defined]
137
172
  name=name,
138
173
  group=group,
139
174
  version=version,
140
175
  plural=plural,
141
- namespace=self.namespace,
176
+ namespace=namespace or self.namespace,
142
177
  body=body,
143
178
  _content_type="application/merge-patch+json",
144
179
  )
@@ -153,17 +188,28 @@ class AsyncK8sManager(BaseK8sManager):
153
188
  plural: str,
154
189
  body: Dict,
155
190
  reraise: bool = False,
191
+ namespace: str = None,
156
192
  ) -> Tuple[Dict, bool]:
157
193
  try:
158
194
  create = await self.create_custom_object(
159
- name=name, group=group, version=version, plural=plural, body=body
195
+ name=name,
196
+ group=group,
197
+ version=version,
198
+ plural=plural,
199
+ body=body,
200
+ namespace=namespace,
160
201
  )
161
202
  return create, True
162
203
 
163
204
  except ApiException as e_create:
164
205
  try:
165
206
  update = await self.update_custom_object(
166
- name=name, group=group, version=version, plural=plural, body=body
207
+ name=name,
208
+ group=group,
209
+ version=version,
210
+ plural=plural,
211
+ body=body,
212
+ namespace=namespace,
167
213
  )
168
214
  return update, False
169
215
  except ApiException as e:
@@ -176,7 +222,13 @@ class AsyncK8sManager(BaseK8sManager):
176
222
  return {}, False
177
223
 
178
224
  async def get_custom_object(
179
- self, name: str, group: str, version: str, plural: str, reraise: bool = False
225
+ self,
226
+ name: str,
227
+ group: str,
228
+ version: str,
229
+ plural: str,
230
+ reraise: bool = False,
231
+ namespace: str = None,
180
232
  ) -> Optional[Dict]:
181
233
  try:
182
234
  return await self.k8s_custom_object_api.get_namespaced_custom_object(
@@ -184,7 +236,7 @@ class AsyncK8sManager(BaseK8sManager):
184
236
  group=group,
185
237
  version=version,
186
238
  plural=plural,
187
- namespace=self.namespace,
239
+ namespace=namespace or self.namespace,
188
240
  )
189
241
  except ApiException as e:
190
242
  if reraise:
@@ -192,7 +244,13 @@ class AsyncK8sManager(BaseK8sManager):
192
244
  return None
193
245
 
194
246
  async def delete_custom_object(
195
- self, name: str, group: str, version: str, plural: str, reraise: bool = False
247
+ self,
248
+ name: str,
249
+ group: str,
250
+ version: str,
251
+ plural: str,
252
+ reraise: bool = False,
253
+ namespace: str = None,
196
254
  ):
197
255
  try:
198
256
  await self.k8s_custom_object_api.delete_namespaced_custom_object(
@@ -200,7 +258,7 @@ class AsyncK8sManager(BaseK8sManager):
200
258
  group=group,
201
259
  version=version,
202
260
  plural=plural,
203
- namespace=self.namespace,
261
+ namespace=namespace or self.namespace,
204
262
  body=client.V1DeleteOptions(),
205
263
  )
206
264
  logger.debug("Custom object `{}` deleted".format(name))
@@ -209,3 +267,18 @@ class AsyncK8sManager(BaseK8sManager):
209
267
  raise PolyaxonK8sError("Connection error: %s" % e) from e
210
268
  else:
211
269
  logger.debug("Custom object `{}` was not found".format(name))
270
+
271
+ async def list_namespaced_events(
272
+ self,
273
+ field_selector: str = None,
274
+ namespace: str = None,
275
+ reraise: bool = False,
276
+ **kwargs
277
+ ) -> List[client.CoreV1EventList]:
278
+ return await self._list_namespace_resource(
279
+ resource_api=self.k8s_api.list_namespaced_event, # type: ignore[attr-defined]
280
+ reraise=reraise,
281
+ namespace=namespace,
282
+ field_selector=field_selector,
283
+ **kwargs,
284
+ )
@@ -28,6 +28,10 @@ class BaseK8sManager:
28
28
  instance
29
29
  )
30
30
 
31
+ @staticmethod
32
+ def get_core_polyaxon() -> str:
33
+ return "app.kubernetes.io/part-of=polyaxon-core"
34
+
31
35
  @classmethod
32
36
  def get_config_auth(cls, k8s_config: Optional[Configuration] = None) -> str:
33
37
  if not k8s_config or not k8s_config.api_key: