polyaxon 2.0.0rc49__py3-none-any.whl → 2.4.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- polyaxon/_auxiliaries/cleaner.py +8 -3
- polyaxon/_auxiliaries/init.py +7 -2
- polyaxon/_auxiliaries/notifier.py +8 -2
- polyaxon/_auxiliaries/sidecar.py +30 -2
- polyaxon/_cli/artifacts.py +96 -11
- polyaxon/_cli/components.py +96 -11
- polyaxon/_cli/config.py +118 -22
- polyaxon/_cli/dashboard.py +15 -2
- polyaxon/_cli/init.py +1 -1
- polyaxon/_cli/models.py +96 -11
- polyaxon/_cli/operations.py +267 -90
- polyaxon/_cli/project_versions.py +139 -6
- polyaxon/_cli/projects.py +23 -9
- polyaxon/_cli/run.py +37 -9
- polyaxon/_cli/services/agent.py +2 -2
- polyaxon/_cli/services/clean_artifacts.py +1 -1
- polyaxon/_cli/services/sidecar.py +8 -1
- polyaxon/_client/client.py +17 -0
- polyaxon/_client/mixin.py +39 -0
- polyaxon/_client/project.py +218 -23
- polyaxon/_client/run.py +131 -33
- polyaxon/_compiler/contexts/contexts.py +2 -2
- polyaxon/_compiler/contexts/ray_job.py +4 -2
- polyaxon/_compiler/resolver/agent.py +12 -2
- polyaxon/_compiler/resolver/runtime.py +2 -2
- polyaxon/_contexts/paths.py +4 -7
- polyaxon/_deploy/operators/compose.py +1 -27
- polyaxon/_deploy/schemas/deployment.py +4 -1
- polyaxon/_deploy/schemas/intervals.py +0 -7
- polyaxon/_deploy/schemas/proxy.py +1 -0
- polyaxon/_deploy/schemas/service.py +11 -1
- polyaxon/_docker/converter/base/base.py +8 -0
- polyaxon/_docker/executor.py +10 -4
- polyaxon/_env_vars/getters/owner_entity.py +4 -2
- polyaxon/_env_vars/getters/project.py +4 -2
- polyaxon/_env_vars/getters/run.py +5 -2
- polyaxon/_env_vars/keys.py +7 -1
- polyaxon/_flow/__init__.py +2 -0
- polyaxon/_flow/builds/__init__.py +19 -6
- polyaxon/_flow/component/base.py +1 -0
- polyaxon/_flow/component/component.py +14 -0
- polyaxon/_flow/environment/__init__.py +8 -8
- polyaxon/_flow/hooks/__init__.py +19 -6
- polyaxon/_flow/init/__init__.py +6 -6
- polyaxon/_flow/matrix/iterative.py +0 -1
- polyaxon/_flow/matrix/tuner.py +18 -6
- polyaxon/_flow/operations/operation.py +44 -17
- polyaxon/_flow/plugins/__init__.py +6 -0
- polyaxon/_flow/run/__init__.py +2 -2
- polyaxon/_flow/run/dag.py +2 -2
- polyaxon/_flow/run/dask/dask.py +0 -1
- polyaxon/_flow/run/dask/replica.py +3 -3
- polyaxon/_flow/run/enums.py +5 -0
- polyaxon/_flow/run/job.py +4 -4
- polyaxon/_flow/run/kubeflow/mpi_job.py +1 -2
- polyaxon/_flow/run/kubeflow/mx_job.py +1 -2
- polyaxon/_flow/run/kubeflow/paddle_job.py +35 -4
- polyaxon/_flow/run/kubeflow/pytorch_job.py +51 -5
- polyaxon/_flow/run/kubeflow/replica.py +4 -4
- polyaxon/_flow/run/kubeflow/scheduling_policy.py +12 -0
- polyaxon/_flow/run/kubeflow/tf_job.py +3 -3
- polyaxon/_flow/run/kubeflow/xgboost_job.py +1 -2
- polyaxon/_flow/run/ray/ray.py +2 -3
- polyaxon/_flow/run/ray/replica.py +3 -3
- polyaxon/_flow/run/service.py +4 -4
- polyaxon/_fs/fs.py +7 -2
- polyaxon/_fs/utils.py +3 -2
- polyaxon/_k8s/converter/base/base.py +2 -1
- polyaxon/_k8s/converter/base/main.py +1 -0
- polyaxon/_k8s/converter/base/sidecar.py +16 -1
- polyaxon/_k8s/converter/common/accelerators.py +7 -4
- polyaxon/_k8s/converter/converters/job.py +1 -1
- polyaxon/_k8s/converter/converters/kubeflow/paddle_job.py +1 -0
- polyaxon/_k8s/converter/converters/kubeflow/pytroch_job.py +2 -0
- polyaxon/_k8s/converter/converters/kubeflow/tf_job.py +1 -0
- polyaxon/_k8s/converter/converters/ray_job.py +4 -2
- polyaxon/_k8s/custom_resources/dask_job.py +3 -0
- polyaxon/_k8s/custom_resources/kubeflow/common.py +4 -1
- polyaxon/_k8s/custom_resources/kubeflow/paddle_job.py +10 -1
- polyaxon/_k8s/custom_resources/kubeflow/pytorch_job.py +14 -1
- polyaxon/_k8s/custom_resources/kubeflow/tf_job.py +4 -0
- polyaxon/_k8s/custom_resources/ray_job.py +3 -0
- polyaxon/_k8s/custom_resources/setter.py +1 -1
- polyaxon/_k8s/executor/async_executor.py +2 -0
- polyaxon/_k8s/executor/base.py +23 -6
- polyaxon/_k8s/logging/async_monitor.py +150 -5
- polyaxon/_k8s/manager/async_manager.py +96 -23
- polyaxon/_k8s/manager/base.py +4 -0
- polyaxon/_k8s/manager/manager.py +282 -134
- polyaxon/_local_process/__init__.py +0 -0
- polyaxon/_local_process/agent.py +6 -0
- polyaxon/_local_process/converter/__init__.py +1 -0
- polyaxon/_local_process/converter/base/__init__.py +1 -0
- polyaxon/_local_process/converter/base/base.py +140 -0
- polyaxon/_local_process/converter/base/containers.py +69 -0
- polyaxon/_local_process/converter/base/env_vars.py +253 -0
- polyaxon/_local_process/converter/base/init.py +414 -0
- polyaxon/_local_process/converter/base/main.py +74 -0
- polyaxon/_local_process/converter/base/mounts.py +82 -0
- polyaxon/_local_process/converter/converters/__init__.py +8 -0
- polyaxon/_local_process/converter/converters/job.py +40 -0
- polyaxon/_local_process/converter/converters/service.py +41 -0
- polyaxon/_local_process/converter/mixins.py +38 -0
- polyaxon/_local_process/executor.py +132 -0
- polyaxon/_local_process/process_types.py +39 -0
- polyaxon/_managers/agent.py +2 -0
- polyaxon/_managers/home.py +2 -1
- polyaxon/_operations/tuner.py +1 -0
- polyaxon/_polyaxonfile/check.py +2 -0
- polyaxon/_polyaxonfile/manager/operations.py +3 -0
- polyaxon/_polyaxonfile/manager/workflows.py +2 -0
- polyaxon/_polyaxonfile/specs/compiled_operation.py +1 -0
- polyaxon/_polyaxonfile/specs/operation.py +1 -0
- polyaxon/_polyaxonfile/specs/sections.py +3 -0
- polyaxon/_pql/manager.py +1 -1
- polyaxon/_runner/agent/async_agent.py +97 -21
- polyaxon/_runner/agent/base_agent.py +27 -9
- polyaxon/_runner/agent/client.py +15 -1
- polyaxon/_runner/agent/sync_agent.py +85 -20
- polyaxon/_runner/converter/converter.py +6 -2
- polyaxon/_runner/executor.py +13 -7
- polyaxon/_schemas/agent.py +27 -1
- polyaxon/_schemas/client.py +30 -3
- polyaxon/_schemas/installation.py +4 -3
- polyaxon/_schemas/lifecycle.py +10 -5
- polyaxon/_schemas/log_handler.py +2 -3
- polyaxon/_schemas/types/artifacts.py +3 -3
- polyaxon/_schemas/types/dockerfile.py +3 -3
- polyaxon/_schemas/types/file.py +3 -3
- polyaxon/_schemas/types/git.py +3 -3
- polyaxon/_schemas/types/tensorboard.py +3 -3
- polyaxon/_sdk/api/agents_v1_api.py +1076 -73
- polyaxon/_sdk/api/organizations_v1_api.py +371 -10
- polyaxon/_sdk/api/project_dashboards_v1_api.py +12 -12
- polyaxon/_sdk/api/project_searches_v1_api.py +12 -12
- polyaxon/_sdk/api/projects_v1_api.py +221 -44
- polyaxon/_sdk/api/runs_v1_api.py +917 -445
- polyaxon/_sdk/api/service_accounts_v1_api.py +16 -16
- polyaxon/_sdk/api/teams_v1_api.py +2827 -375
- polyaxon/_sdk/api/users_v1_api.py +231 -55
- polyaxon/_sdk/async_client/api_client.py +4 -0
- polyaxon/_sdk/schemas/__init__.py +10 -2
- polyaxon/_sdk/schemas/v1_agent.py +2 -1
- polyaxon/_sdk/schemas/v1_agent_reconcile_body_request.py +14 -0
- polyaxon/_sdk/schemas/v1_artifact_tree.py +1 -1
- polyaxon/_sdk/schemas/v1_dashboard_spec.py +4 -0
- polyaxon/_sdk/schemas/v1_events_response.py +4 -0
- polyaxon/_sdk/schemas/v1_organization.py +1 -0
- polyaxon/_sdk/schemas/v1_preset.py +8 -0
- polyaxon/_sdk/schemas/v1_project.py +1 -0
- polyaxon/_sdk/schemas/v1_project_settings.py +4 -2
- polyaxon/_sdk/schemas/v1_run.py +2 -2
- polyaxon/_sdk/schemas/v1_run_edge_lineage.py +14 -0
- polyaxon/_sdk/schemas/v1_run_edges_graph.py +9 -0
- polyaxon/_sdk/schemas/v1_section_spec.py +7 -2
- polyaxon/_sdk/schemas/v1_settings_catalog.py +1 -0
- polyaxon/_sdk/schemas/v1_team.py +3 -0
- polyaxon/_sdk/schemas/v1_user.py +1 -2
- polyaxon/_sdk/schemas/v1_user_access.py +17 -0
- polyaxon/_services/values.py +1 -0
- polyaxon/_sidecar/container/__init__.py +39 -18
- polyaxon/_sidecar/container/monitors/__init__.py +1 -0
- polyaxon/_sidecar/container/monitors/logs.py +10 -13
- polyaxon/_sidecar/container/monitors/spec.py +24 -0
- polyaxon/_sidecar/ignore.py +0 -1
- polyaxon/_utils/fqn_utils.py +25 -2
- polyaxon/client.py +1 -1
- polyaxon/pkg.py +1 -1
- polyaxon/schemas.py +8 -1
- polyaxon/settings.py +6 -0
- {polyaxon-2.0.0rc49.dist-info → polyaxon-2.4.0rc1.dist-info}/METADATA +43 -43
- {polyaxon-2.0.0rc49.dist-info → polyaxon-2.4.0rc1.dist-info}/RECORD +176 -155
- {polyaxon-2.0.0rc49.dist-info → polyaxon-2.4.0rc1.dist-info}/WHEEL +1 -1
- polyaxon/_sdk/schemas/v1_project_user_access.py +0 -10
- {polyaxon-2.0.0rc49.dist-info → polyaxon-2.4.0rc1.dist-info}/LICENSE +0 -0
- {polyaxon-2.0.0rc49.dist-info → polyaxon-2.4.0rc1.dist-info}/entry_points.txt +0 -0
- {polyaxon-2.0.0rc49.dist-info → polyaxon-2.4.0rc1.dist-info}/top_level.txt +0 -0
@@ -38,7 +38,7 @@ class AsyncK8sManager(BaseK8sManager):
|
|
38
38
|
if self.api_client:
|
39
39
|
await self.api_client.close()
|
40
40
|
|
41
|
-
async def get_version(self, reraise=False):
|
41
|
+
async def get_version(self, reraise: bool = False):
|
42
42
|
try:
|
43
43
|
version = await self.k8s_version_api.get_code()
|
44
44
|
return version.to_dict()
|
@@ -47,48 +47,68 @@ class AsyncK8sManager(BaseK8sManager):
|
|
47
47
|
if reraise:
|
48
48
|
raise PolyaxonK8sError("Connection error: %s" % e) from e
|
49
49
|
|
50
|
-
async def get_pod(
|
50
|
+
async def get_pod(
|
51
|
+
self, name, reraise: bool = False, namespace: str = None
|
52
|
+
) -> Optional[client.V1Pod]:
|
51
53
|
try:
|
52
54
|
return await self.k8s_api.read_namespaced_pod( # type: ignore[attr-defined]
|
53
|
-
name=name, namespace=self.namespace
|
55
|
+
name=name, namespace=namespace or self.namespace
|
54
56
|
)
|
55
57
|
except ApiException as e:
|
56
58
|
if reraise:
|
57
59
|
raise PolyaxonK8sError("Connection error: %s" % e) from e
|
58
60
|
return None
|
59
61
|
|
60
|
-
async def is_pod_running(
|
61
|
-
|
62
|
+
async def is_pod_running(
|
63
|
+
self, pod_id: str, container_id: str, namespace: str = None
|
64
|
+
) -> bool:
|
65
|
+
event = await self.k8s_api.read_namespaced_pod_status(pod_id, namespace=namespace or self.namespace) # type: ignore[attr-defined]
|
62
66
|
return is_pod_running(event, container_id)
|
63
67
|
|
64
68
|
async def _list_namespace_resource(
|
65
|
-
self, resource_api, reraise=False, **kwargs
|
69
|
+
self, resource_api, reraise: bool = False, namespace: str = None, **kwargs
|
66
70
|
) -> List:
|
67
71
|
try:
|
68
|
-
res = await resource_api(namespace=self.namespace, **kwargs)
|
69
|
-
|
72
|
+
res = await resource_api(namespace=namespace or self.namespace, **kwargs)
|
73
|
+
if isinstance(res, dict):
|
74
|
+
items = res["items"]
|
75
|
+
else:
|
76
|
+
items = res.items
|
77
|
+
return [p for p in items]
|
70
78
|
except ApiException as e:
|
71
79
|
logger.error("K8S error: {}".format(e))
|
72
80
|
if reraise:
|
73
81
|
raise PolyaxonK8sError("Connection error: %s" % e) from e
|
74
82
|
return []
|
75
83
|
|
76
|
-
async def list_pods(
|
84
|
+
async def list_pods(
|
85
|
+
self, reraise: bool = False, namespace: str = None, **kwargs
|
86
|
+
) -> List[client.V1Pod]:
|
77
87
|
return await self._list_namespace_resource(
|
78
88
|
resource_api=self.k8s_api.list_namespaced_pod, # type: ignore[attr-defined]
|
79
89
|
reraise=reraise,
|
90
|
+
namespace=namespace,
|
80
91
|
**kwargs,
|
81
92
|
)
|
82
93
|
|
83
|
-
async def list_jobs(
|
94
|
+
async def list_jobs(
|
95
|
+
self, reraise: bool = False, namespace: str = None, **kwargs
|
96
|
+
) -> List[client.V1Job]:
|
84
97
|
return await self._list_namespace_resource(
|
85
98
|
resource_api=self.k8s_batch_api.list_namespaced_job, # type: ignore[attr-defined]
|
86
99
|
reraise=reraise,
|
100
|
+
namespace=namespace,
|
87
101
|
**kwargs,
|
88
102
|
)
|
89
103
|
|
90
104
|
async def list_custom_objects(
|
91
|
-
self,
|
105
|
+
self,
|
106
|
+
group,
|
107
|
+
version,
|
108
|
+
plural,
|
109
|
+
reraise: bool = False,
|
110
|
+
namespace: str = None,
|
111
|
+
**kwargs
|
92
112
|
) -> List:
|
93
113
|
return await self._list_namespace_resource(
|
94
114
|
resource_api=self.k8s_custom_object_api.list_namespaced_custom_object, # type: ignore[attr-defined]
|
@@ -96,49 +116,64 @@ class AsyncK8sManager(BaseK8sManager):
|
|
96
116
|
group=group,
|
97
117
|
version=version,
|
98
118
|
plural=plural,
|
119
|
+
namespace=namespace,
|
99
120
|
**kwargs,
|
100
121
|
)
|
101
122
|
|
102
123
|
async def list_services(
|
103
|
-
self, reraise: bool = False, **kwargs
|
124
|
+
self, reraise: bool = False, namespace: str = None, **kwargs
|
104
125
|
) -> List[client.V1Service]:
|
105
126
|
return await self._list_namespace_resource(
|
106
127
|
resource_api=self.k8s_api.list_namespaced_service, # type: ignore[attr-defined]
|
107
128
|
reraise=reraise,
|
129
|
+
namespace=namespace,
|
108
130
|
**kwargs,
|
109
131
|
)
|
110
132
|
|
111
133
|
async def list_deployments(
|
112
|
-
self, reraise: bool = False, **kwargs
|
134
|
+
self, reraise: bool = False, namespace: str = None, **kwargs
|
113
135
|
) -> List[client.V1Deployment]:
|
114
136
|
return await self._list_namespace_resource(
|
115
137
|
resource_api=self.k8s_apps_api.list_namespaced_deployment, # type: ignore[attr-defined]
|
116
138
|
reraise=reraise,
|
139
|
+
namespace=namespace,
|
117
140
|
**kwargs,
|
118
141
|
)
|
119
142
|
|
120
143
|
async def create_custom_object(
|
121
|
-
self,
|
144
|
+
self,
|
145
|
+
name: str,
|
146
|
+
group: str,
|
147
|
+
version: str,
|
148
|
+
plural: str,
|
149
|
+
body: Dict,
|
150
|
+
namespace: str = None,
|
122
151
|
) -> Dict:
|
123
152
|
resp = await self.k8s_custom_object_api.create_namespaced_custom_object( # type: ignore[attr-defined]
|
124
153
|
group=group,
|
125
154
|
version=version,
|
126
155
|
plural=plural,
|
127
|
-
namespace=self.namespace,
|
156
|
+
namespace=namespace or self.namespace,
|
128
157
|
body=body,
|
129
158
|
)
|
130
159
|
logger.debug("Custom object `{}` was created".format(name))
|
131
160
|
return resp
|
132
161
|
|
133
162
|
async def update_custom_object(
|
134
|
-
self,
|
163
|
+
self,
|
164
|
+
name: str,
|
165
|
+
group: str,
|
166
|
+
version: str,
|
167
|
+
plural: str,
|
168
|
+
body: Dict,
|
169
|
+
namespace: str = None,
|
135
170
|
) -> Dict:
|
136
171
|
resp = await self.k8s_custom_object_api.patch_namespaced_custom_object( # type: ignore[attr-defined]
|
137
172
|
name=name,
|
138
173
|
group=group,
|
139
174
|
version=version,
|
140
175
|
plural=plural,
|
141
|
-
namespace=self.namespace,
|
176
|
+
namespace=namespace or self.namespace,
|
142
177
|
body=body,
|
143
178
|
_content_type="application/merge-patch+json",
|
144
179
|
)
|
@@ -153,17 +188,28 @@ class AsyncK8sManager(BaseK8sManager):
|
|
153
188
|
plural: str,
|
154
189
|
body: Dict,
|
155
190
|
reraise: bool = False,
|
191
|
+
namespace: str = None,
|
156
192
|
) -> Tuple[Dict, bool]:
|
157
193
|
try:
|
158
194
|
create = await self.create_custom_object(
|
159
|
-
name=name,
|
195
|
+
name=name,
|
196
|
+
group=group,
|
197
|
+
version=version,
|
198
|
+
plural=plural,
|
199
|
+
body=body,
|
200
|
+
namespace=namespace,
|
160
201
|
)
|
161
202
|
return create, True
|
162
203
|
|
163
204
|
except ApiException as e_create:
|
164
205
|
try:
|
165
206
|
update = await self.update_custom_object(
|
166
|
-
name=name,
|
207
|
+
name=name,
|
208
|
+
group=group,
|
209
|
+
version=version,
|
210
|
+
plural=plural,
|
211
|
+
body=body,
|
212
|
+
namespace=namespace,
|
167
213
|
)
|
168
214
|
return update, False
|
169
215
|
except ApiException as e:
|
@@ -176,7 +222,13 @@ class AsyncK8sManager(BaseK8sManager):
|
|
176
222
|
return {}, False
|
177
223
|
|
178
224
|
async def get_custom_object(
|
179
|
-
self,
|
225
|
+
self,
|
226
|
+
name: str,
|
227
|
+
group: str,
|
228
|
+
version: str,
|
229
|
+
plural: str,
|
230
|
+
reraise: bool = False,
|
231
|
+
namespace: str = None,
|
180
232
|
) -> Optional[Dict]:
|
181
233
|
try:
|
182
234
|
return await self.k8s_custom_object_api.get_namespaced_custom_object(
|
@@ -184,7 +236,7 @@ class AsyncK8sManager(BaseK8sManager):
|
|
184
236
|
group=group,
|
185
237
|
version=version,
|
186
238
|
plural=plural,
|
187
|
-
namespace=self.namespace,
|
239
|
+
namespace=namespace or self.namespace,
|
188
240
|
)
|
189
241
|
except ApiException as e:
|
190
242
|
if reraise:
|
@@ -192,7 +244,13 @@ class AsyncK8sManager(BaseK8sManager):
|
|
192
244
|
return None
|
193
245
|
|
194
246
|
async def delete_custom_object(
|
195
|
-
self,
|
247
|
+
self,
|
248
|
+
name: str,
|
249
|
+
group: str,
|
250
|
+
version: str,
|
251
|
+
plural: str,
|
252
|
+
reraise: bool = False,
|
253
|
+
namespace: str = None,
|
196
254
|
):
|
197
255
|
try:
|
198
256
|
await self.k8s_custom_object_api.delete_namespaced_custom_object(
|
@@ -200,7 +258,7 @@ class AsyncK8sManager(BaseK8sManager):
|
|
200
258
|
group=group,
|
201
259
|
version=version,
|
202
260
|
plural=plural,
|
203
|
-
namespace=self.namespace,
|
261
|
+
namespace=namespace or self.namespace,
|
204
262
|
body=client.V1DeleteOptions(),
|
205
263
|
)
|
206
264
|
logger.debug("Custom object `{}` deleted".format(name))
|
@@ -209,3 +267,18 @@ class AsyncK8sManager(BaseK8sManager):
|
|
209
267
|
raise PolyaxonK8sError("Connection error: %s" % e) from e
|
210
268
|
else:
|
211
269
|
logger.debug("Custom object `{}` was not found".format(name))
|
270
|
+
|
271
|
+
async def list_namespaced_events(
|
272
|
+
self,
|
273
|
+
field_selector: str = None,
|
274
|
+
namespace: str = None,
|
275
|
+
reraise: bool = False,
|
276
|
+
**kwargs
|
277
|
+
) -> List[client.CoreV1EventList]:
|
278
|
+
return await self._list_namespace_resource(
|
279
|
+
resource_api=self.k8s_api.list_namespaced_event, # type: ignore[attr-defined]
|
280
|
+
reraise=reraise,
|
281
|
+
namespace=namespace,
|
282
|
+
field_selector=field_selector,
|
283
|
+
**kwargs,
|
284
|
+
)
|
polyaxon/_k8s/manager/base.py
CHANGED
@@ -28,6 +28,10 @@ class BaseK8sManager:
|
|
28
28
|
instance
|
29
29
|
)
|
30
30
|
|
31
|
+
@staticmethod
|
32
|
+
def get_core_polyaxon() -> str:
|
33
|
+
return "app.kubernetes.io/part-of=polyaxon-core"
|
34
|
+
|
31
35
|
@classmethod
|
32
36
|
def get_config_auth(cls, k8s_config: Optional[Configuration] = None) -> str:
|
33
37
|
if not k8s_config or not k8s_config.api_key:
|