polyaxon 2.0.0rc49__py3-none-any.whl → 2.4.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- polyaxon/_auxiliaries/cleaner.py +8 -3
- polyaxon/_auxiliaries/init.py +7 -2
- polyaxon/_auxiliaries/notifier.py +8 -2
- polyaxon/_auxiliaries/sidecar.py +30 -2
- polyaxon/_cli/artifacts.py +96 -11
- polyaxon/_cli/components.py +96 -11
- polyaxon/_cli/config.py +118 -22
- polyaxon/_cli/dashboard.py +15 -2
- polyaxon/_cli/init.py +1 -1
- polyaxon/_cli/models.py +96 -11
- polyaxon/_cli/operations.py +267 -90
- polyaxon/_cli/project_versions.py +139 -6
- polyaxon/_cli/projects.py +23 -9
- polyaxon/_cli/run.py +37 -9
- polyaxon/_cli/services/agent.py +2 -2
- polyaxon/_cli/services/clean_artifacts.py +1 -1
- polyaxon/_cli/services/sidecar.py +8 -1
- polyaxon/_client/client.py +17 -0
- polyaxon/_client/mixin.py +39 -0
- polyaxon/_client/project.py +218 -23
- polyaxon/_client/run.py +131 -33
- polyaxon/_compiler/contexts/contexts.py +2 -2
- polyaxon/_compiler/contexts/ray_job.py +4 -2
- polyaxon/_compiler/resolver/agent.py +12 -2
- polyaxon/_compiler/resolver/runtime.py +2 -2
- polyaxon/_contexts/paths.py +4 -7
- polyaxon/_deploy/operators/compose.py +1 -27
- polyaxon/_deploy/schemas/deployment.py +4 -1
- polyaxon/_deploy/schemas/intervals.py +0 -7
- polyaxon/_deploy/schemas/proxy.py +1 -0
- polyaxon/_deploy/schemas/service.py +11 -1
- polyaxon/_docker/converter/base/base.py +8 -0
- polyaxon/_docker/executor.py +10 -4
- polyaxon/_env_vars/getters/owner_entity.py +4 -2
- polyaxon/_env_vars/getters/project.py +4 -2
- polyaxon/_env_vars/getters/run.py +5 -2
- polyaxon/_env_vars/keys.py +7 -1
- polyaxon/_flow/__init__.py +2 -0
- polyaxon/_flow/builds/__init__.py +19 -6
- polyaxon/_flow/component/base.py +1 -0
- polyaxon/_flow/component/component.py +14 -0
- polyaxon/_flow/environment/__init__.py +8 -8
- polyaxon/_flow/hooks/__init__.py +19 -6
- polyaxon/_flow/init/__init__.py +6 -6
- polyaxon/_flow/matrix/iterative.py +0 -1
- polyaxon/_flow/matrix/tuner.py +18 -6
- polyaxon/_flow/operations/operation.py +44 -17
- polyaxon/_flow/plugins/__init__.py +6 -0
- polyaxon/_flow/run/__init__.py +2 -2
- polyaxon/_flow/run/dag.py +2 -2
- polyaxon/_flow/run/dask/dask.py +0 -1
- polyaxon/_flow/run/dask/replica.py +3 -3
- polyaxon/_flow/run/enums.py +5 -0
- polyaxon/_flow/run/job.py +4 -4
- polyaxon/_flow/run/kubeflow/mpi_job.py +1 -2
- polyaxon/_flow/run/kubeflow/mx_job.py +1 -2
- polyaxon/_flow/run/kubeflow/paddle_job.py +35 -4
- polyaxon/_flow/run/kubeflow/pytorch_job.py +51 -5
- polyaxon/_flow/run/kubeflow/replica.py +4 -4
- polyaxon/_flow/run/kubeflow/scheduling_policy.py +12 -0
- polyaxon/_flow/run/kubeflow/tf_job.py +3 -3
- polyaxon/_flow/run/kubeflow/xgboost_job.py +1 -2
- polyaxon/_flow/run/ray/ray.py +2 -3
- polyaxon/_flow/run/ray/replica.py +3 -3
- polyaxon/_flow/run/service.py +4 -4
- polyaxon/_fs/fs.py +7 -2
- polyaxon/_fs/utils.py +3 -2
- polyaxon/_k8s/converter/base/base.py +2 -1
- polyaxon/_k8s/converter/base/main.py +1 -0
- polyaxon/_k8s/converter/base/sidecar.py +16 -1
- polyaxon/_k8s/converter/common/accelerators.py +7 -4
- polyaxon/_k8s/converter/converters/job.py +1 -1
- polyaxon/_k8s/converter/converters/kubeflow/paddle_job.py +1 -0
- polyaxon/_k8s/converter/converters/kubeflow/pytroch_job.py +2 -0
- polyaxon/_k8s/converter/converters/kubeflow/tf_job.py +1 -0
- polyaxon/_k8s/converter/converters/ray_job.py +4 -2
- polyaxon/_k8s/custom_resources/dask_job.py +3 -0
- polyaxon/_k8s/custom_resources/kubeflow/common.py +4 -1
- polyaxon/_k8s/custom_resources/kubeflow/paddle_job.py +10 -1
- polyaxon/_k8s/custom_resources/kubeflow/pytorch_job.py +14 -1
- polyaxon/_k8s/custom_resources/kubeflow/tf_job.py +4 -0
- polyaxon/_k8s/custom_resources/ray_job.py +3 -0
- polyaxon/_k8s/custom_resources/setter.py +1 -1
- polyaxon/_k8s/executor/async_executor.py +2 -0
- polyaxon/_k8s/executor/base.py +23 -6
- polyaxon/_k8s/logging/async_monitor.py +150 -5
- polyaxon/_k8s/manager/async_manager.py +96 -23
- polyaxon/_k8s/manager/base.py +4 -0
- polyaxon/_k8s/manager/manager.py +282 -134
- polyaxon/_local_process/__init__.py +0 -0
- polyaxon/_local_process/agent.py +6 -0
- polyaxon/_local_process/converter/__init__.py +1 -0
- polyaxon/_local_process/converter/base/__init__.py +1 -0
- polyaxon/_local_process/converter/base/base.py +140 -0
- polyaxon/_local_process/converter/base/containers.py +69 -0
- polyaxon/_local_process/converter/base/env_vars.py +253 -0
- polyaxon/_local_process/converter/base/init.py +414 -0
- polyaxon/_local_process/converter/base/main.py +74 -0
- polyaxon/_local_process/converter/base/mounts.py +82 -0
- polyaxon/_local_process/converter/converters/__init__.py +8 -0
- polyaxon/_local_process/converter/converters/job.py +40 -0
- polyaxon/_local_process/converter/converters/service.py +41 -0
- polyaxon/_local_process/converter/mixins.py +38 -0
- polyaxon/_local_process/executor.py +132 -0
- polyaxon/_local_process/process_types.py +39 -0
- polyaxon/_managers/agent.py +2 -0
- polyaxon/_managers/home.py +2 -1
- polyaxon/_operations/tuner.py +1 -0
- polyaxon/_polyaxonfile/check.py +2 -0
- polyaxon/_polyaxonfile/manager/operations.py +3 -0
- polyaxon/_polyaxonfile/manager/workflows.py +2 -0
- polyaxon/_polyaxonfile/specs/compiled_operation.py +1 -0
- polyaxon/_polyaxonfile/specs/operation.py +1 -0
- polyaxon/_polyaxonfile/specs/sections.py +3 -0
- polyaxon/_pql/manager.py +1 -1
- polyaxon/_runner/agent/async_agent.py +97 -21
- polyaxon/_runner/agent/base_agent.py +27 -9
- polyaxon/_runner/agent/client.py +15 -1
- polyaxon/_runner/agent/sync_agent.py +85 -20
- polyaxon/_runner/converter/converter.py +6 -2
- polyaxon/_runner/executor.py +13 -7
- polyaxon/_schemas/agent.py +27 -1
- polyaxon/_schemas/client.py +30 -3
- polyaxon/_schemas/installation.py +4 -3
- polyaxon/_schemas/lifecycle.py +10 -5
- polyaxon/_schemas/log_handler.py +2 -3
- polyaxon/_schemas/types/artifacts.py +3 -3
- polyaxon/_schemas/types/dockerfile.py +3 -3
- polyaxon/_schemas/types/file.py +3 -3
- polyaxon/_schemas/types/git.py +3 -3
- polyaxon/_schemas/types/tensorboard.py +3 -3
- polyaxon/_sdk/api/agents_v1_api.py +1076 -73
- polyaxon/_sdk/api/organizations_v1_api.py +371 -10
- polyaxon/_sdk/api/project_dashboards_v1_api.py +12 -12
- polyaxon/_sdk/api/project_searches_v1_api.py +12 -12
- polyaxon/_sdk/api/projects_v1_api.py +221 -44
- polyaxon/_sdk/api/runs_v1_api.py +917 -445
- polyaxon/_sdk/api/service_accounts_v1_api.py +16 -16
- polyaxon/_sdk/api/teams_v1_api.py +2827 -375
- polyaxon/_sdk/api/users_v1_api.py +231 -55
- polyaxon/_sdk/async_client/api_client.py +4 -0
- polyaxon/_sdk/schemas/__init__.py +10 -2
- polyaxon/_sdk/schemas/v1_agent.py +2 -1
- polyaxon/_sdk/schemas/v1_agent_reconcile_body_request.py +14 -0
- polyaxon/_sdk/schemas/v1_artifact_tree.py +1 -1
- polyaxon/_sdk/schemas/v1_dashboard_spec.py +4 -0
- polyaxon/_sdk/schemas/v1_events_response.py +4 -0
- polyaxon/_sdk/schemas/v1_organization.py +1 -0
- polyaxon/_sdk/schemas/v1_preset.py +8 -0
- polyaxon/_sdk/schemas/v1_project.py +1 -0
- polyaxon/_sdk/schemas/v1_project_settings.py +4 -2
- polyaxon/_sdk/schemas/v1_run.py +2 -2
- polyaxon/_sdk/schemas/v1_run_edge_lineage.py +14 -0
- polyaxon/_sdk/schemas/v1_run_edges_graph.py +9 -0
- polyaxon/_sdk/schemas/v1_section_spec.py +7 -2
- polyaxon/_sdk/schemas/v1_settings_catalog.py +1 -0
- polyaxon/_sdk/schemas/v1_team.py +3 -0
- polyaxon/_sdk/schemas/v1_user.py +1 -2
- polyaxon/_sdk/schemas/v1_user_access.py +17 -0
- polyaxon/_services/values.py +1 -0
- polyaxon/_sidecar/container/__init__.py +39 -18
- polyaxon/_sidecar/container/monitors/__init__.py +1 -0
- polyaxon/_sidecar/container/monitors/logs.py +10 -13
- polyaxon/_sidecar/container/monitors/spec.py +24 -0
- polyaxon/_sidecar/ignore.py +0 -1
- polyaxon/_utils/fqn_utils.py +25 -2
- polyaxon/client.py +1 -1
- polyaxon/pkg.py +1 -1
- polyaxon/schemas.py +8 -1
- polyaxon/settings.py +6 -0
- {polyaxon-2.0.0rc49.dist-info → polyaxon-2.4.0rc1.dist-info}/METADATA +43 -43
- {polyaxon-2.0.0rc49.dist-info → polyaxon-2.4.0rc1.dist-info}/RECORD +176 -155
- {polyaxon-2.0.0rc49.dist-info → polyaxon-2.4.0rc1.dist-info}/WHEEL +1 -1
- polyaxon/_sdk/schemas/v1_project_user_access.py +0 -10
- {polyaxon-2.0.0rc49.dist-info → polyaxon-2.4.0rc1.dist-info}/LICENSE +0 -0
- {polyaxon-2.0.0rc49.dist-info → polyaxon-2.4.0rc1.dist-info}/entry_points.txt +0 -0
- {polyaxon-2.0.0rc49.dist-info → polyaxon-2.4.0rc1.dist-info}/top_level.txt +0 -0
@@ -6,6 +6,13 @@ from clipped.compact.pydantic import StrictStr
|
|
6
6
|
from clipped.config.schema import BaseAllowSchemaModel
|
7
7
|
from clipped.types.uuids import UUIDStr
|
8
8
|
|
9
|
+
from polyaxon._sdk.schemas.v1_settings_catalog import V1SettingsCatalog
|
10
|
+
|
11
|
+
|
12
|
+
class V1PresetSettings(BaseAllowSchemaModel):
|
13
|
+
projects: Optional[List[V1SettingsCatalog]]
|
14
|
+
runs: Optional[List[V1SettingsCatalog]]
|
15
|
+
|
9
16
|
|
10
17
|
class V1Preset(BaseAllowSchemaModel):
|
11
18
|
uuid: Optional[UUIDStr]
|
@@ -17,3 +24,4 @@ class V1Preset(BaseAllowSchemaModel):
|
|
17
24
|
frozen: Optional[bool]
|
18
25
|
live_state: Optional[int]
|
19
26
|
content: Optional[StrictStr]
|
27
|
+
settings: Optional[V1PresetSettings]
|
@@ -22,6 +22,7 @@ class V1Project(BaseAllowSchemaModel):
|
|
22
22
|
readme: Optional[StrictStr]
|
23
23
|
excluded_features: Optional[List[StrictStr]]
|
24
24
|
excluded_runtimes: Optional[List[StrictStr]]
|
25
|
+
archived_deletion_interval: Optional[int]
|
25
26
|
settings: Optional[V1ProjectSettings]
|
26
27
|
role: Optional[StrictStr]
|
27
28
|
contributors: Optional[Dict[str, Any]]
|
@@ -3,7 +3,7 @@ from typing import List, Optional
|
|
3
3
|
from clipped.compact.pydantic import StrictStr
|
4
4
|
from clipped.config.schema import BaseAllowSchemaModel
|
5
5
|
|
6
|
-
from polyaxon._sdk.schemas.
|
6
|
+
from polyaxon._sdk.schemas.v1_user_access import V1UserAccess
|
7
7
|
|
8
8
|
|
9
9
|
class V1ProjectSettings(BaseAllowSchemaModel):
|
@@ -13,6 +13,8 @@ class V1ProjectSettings(BaseAllowSchemaModel):
|
|
13
13
|
queue: Optional[StrictStr]
|
14
14
|
queues: Optional[List[StrictStr]]
|
15
15
|
agents: Optional[List[StrictStr]]
|
16
|
-
|
16
|
+
namespaces: Optional[List[StrictStr]]
|
17
|
+
user_accesses: Optional[List[V1UserAccess]]
|
17
18
|
teams: Optional[List[StrictStr]]
|
18
19
|
projects: Optional[List[StrictStr]]
|
20
|
+
policy: Optional[StrictStr]
|
polyaxon/_sdk/schemas/v1_run.py
CHANGED
@@ -32,8 +32,8 @@ class V1Run(BaseAllowSchemaModel):
|
|
32
32
|
updated_at: Optional[datetime.datetime]
|
33
33
|
started_at: Optional[datetime.datetime]
|
34
34
|
finished_at: Optional[datetime.datetime]
|
35
|
-
wait_time: Optional[
|
36
|
-
duration: Optional[
|
35
|
+
wait_time: Optional[float]
|
36
|
+
duration: Optional[float]
|
37
37
|
managed_by: Optional[ManagedBy]
|
38
38
|
is_managed: Optional[bool]
|
39
39
|
is_approved: Optional[bool]
|
@@ -0,0 +1,14 @@
|
|
1
|
+
from typing import Any, Dict, Optional
|
2
|
+
|
3
|
+
from clipped.config.schema import BaseAllowSchemaModel
|
4
|
+
from clipped.types.uuids import UUIDStr
|
5
|
+
|
6
|
+
|
7
|
+
class V1RunEdgeLineage(BaseAllowSchemaModel):
|
8
|
+
"""
|
9
|
+
V1RunEdgeLineage
|
10
|
+
"""
|
11
|
+
|
12
|
+
uuid: Optional[UUIDStr]
|
13
|
+
is_upstream: Optional[bool]
|
14
|
+
values: Optional[Dict[str, Any]]
|
@@ -7,8 +7,13 @@ from clipped.config.schema import BaseAllowSchemaModel
|
|
7
7
|
class V1SectionSpec(BaseAllowSchemaModel):
|
8
8
|
name: Optional[StrictStr]
|
9
9
|
is_minimized: Optional[bool]
|
10
|
+
is_frozen: Optional[str]
|
10
11
|
columns: Optional[int]
|
11
12
|
height: Optional[int]
|
13
|
+
xaxis: Optional[str]
|
14
|
+
smoothing: Optional[int]
|
15
|
+
ignore_outliers: Optional[bool]
|
16
|
+
sample_size: Optional[int]
|
12
17
|
widgets: Optional[List[Dict[str, Any]]]
|
13
|
-
page_index: Optional[int]
|
14
|
-
page_size: Optional[int]
|
18
|
+
page_index: Optional[int]
|
19
|
+
page_size: Optional[int]
|
polyaxon/_sdk/schemas/v1_team.py
CHANGED
@@ -11,10 +11,13 @@ from polyaxon._sdk.schemas.v1_team_settings import V1TeamSettings
|
|
11
11
|
|
12
12
|
class V1Team(BaseAllowSchemaModel):
|
13
13
|
uuid: Optional[UUIDStr]
|
14
|
+
owner: Optional[StrictStr]
|
14
15
|
name: Optional[StrictStr]
|
15
16
|
projects: Optional[List[StrictStr]]
|
16
17
|
component_hubs: Optional[List[StrictStr]]
|
17
18
|
model_registries: Optional[List[StrictStr]]
|
18
19
|
settings: Optional[V1TeamSettings]
|
20
|
+
policy: Optional[StrictStr]
|
21
|
+
role: Optional[StrictStr]
|
19
22
|
created_at: Optional[datetime.datetime]
|
20
23
|
updated_at: Optional[datetime.datetime]
|
polyaxon/_sdk/schemas/v1_user.py
CHANGED
@@ -2,12 +2,11 @@ from typing import Optional
|
|
2
2
|
|
3
3
|
from clipped.compact.pydantic import StrictStr
|
4
4
|
from clipped.config.schema import BaseAllowSchemaModel
|
5
|
-
from clipped.types.email import EmailStr
|
6
5
|
|
7
6
|
|
8
7
|
class V1User(BaseAllowSchemaModel):
|
9
8
|
username: Optional[StrictStr]
|
10
|
-
email: Optional[
|
9
|
+
email: Optional[StrictStr]
|
11
10
|
name: Optional[StrictStr]
|
12
11
|
kind: Optional[StrictStr]
|
13
12
|
theme: Optional[int]
|
@@ -0,0 +1,17 @@
|
|
1
|
+
from typing import Optional
|
2
|
+
|
3
|
+
from clipped.compact.pydantic import StrictStr
|
4
|
+
from clipped.config.schema import BaseAllowSchemaModel
|
5
|
+
|
6
|
+
|
7
|
+
class V1UserAccessData(BaseAllowSchemaModel):
|
8
|
+
username: Optional[StrictStr]
|
9
|
+
is_sa: Optional[bool]
|
10
|
+
|
11
|
+
|
12
|
+
class V1UserAccess(BaseAllowSchemaModel):
|
13
|
+
user: Optional[StrictStr]
|
14
|
+
user_data: Optional[V1UserAccessData]
|
15
|
+
queue: Optional[StrictStr]
|
16
|
+
preset: Optional[StrictStr]
|
17
|
+
namespace: Optional[StrictStr]
|
polyaxon/_services/values.py
CHANGED
@@ -15,8 +15,8 @@ from polyaxon._fs.fs import (
|
|
15
15
|
from polyaxon._fs.watcher import FSWatcher
|
16
16
|
from polyaxon._k8s.manager.async_manager import AsyncK8sManager
|
17
17
|
from polyaxon._sidecar.container.intervals import get_sync_interval
|
18
|
-
from polyaxon._sidecar.container.monitors import sync_artifacts, sync_logs
|
19
|
-
from polyaxon._sidecar.ignore import
|
18
|
+
from polyaxon._sidecar.container.monitors import sync_artifacts, sync_logs, sync_spec
|
19
|
+
from polyaxon._sidecar.ignore import IGNORE_FOLDERS
|
20
20
|
from polyaxon.client import RunClient
|
21
21
|
from polyaxon.exceptions import PolyaxonClientException, PolyaxonContainerException
|
22
22
|
from polyaxon.logger import logger
|
@@ -29,10 +29,14 @@ async def start_sidecar(
|
|
29
29
|
sync_interval: int,
|
30
30
|
monitor_outputs: bool,
|
31
31
|
monitor_logs: bool,
|
32
|
+
monitor_spec: bool,
|
32
33
|
):
|
33
34
|
sync_interval = get_sync_interval(
|
34
35
|
interval=sync_interval, sleep_interval=sleep_interval
|
35
36
|
)
|
37
|
+
fs_refresh_interval = get_sync_interval(
|
38
|
+
interval=60 * 60 * 2, sleep_interval=sleep_interval
|
39
|
+
)
|
36
40
|
try:
|
37
41
|
pod_id = os.environ[ENV_KEYS_K8S_POD_ID]
|
38
42
|
except KeyError as e:
|
@@ -60,46 +64,56 @@ async def start_sidecar(
|
|
60
64
|
retry = 0
|
61
65
|
is_running = True
|
62
66
|
counter = 0
|
67
|
+
fs_refresh_counter = 0
|
63
68
|
state = {
|
64
69
|
"last_artifacts_check": None,
|
65
70
|
"last_logs_check": None,
|
66
71
|
}
|
67
72
|
|
68
73
|
async def monitor():
|
74
|
+
if monitor_spec and pod.metadata.annotations:
|
75
|
+
await sync_spec(
|
76
|
+
k8s_manager=k8s_manager,
|
77
|
+
run_uuid=run_uuid,
|
78
|
+
run_kind=pod.metadata.annotations.get("operation.polyaxon.com/kind"),
|
79
|
+
)
|
69
80
|
if monitor_logs:
|
70
|
-
await sync_logs(
|
81
|
+
state["last_logs_check"] = await sync_logs(
|
71
82
|
run_uuid=run_uuid,
|
72
83
|
k8s_manager=k8s_manager,
|
73
84
|
pod=pod,
|
74
|
-
last_time=
|
75
|
-
stream=
|
76
|
-
is_running=is_running,
|
85
|
+
last_time=state["last_logs_check"],
|
86
|
+
stream=False,
|
77
87
|
)
|
78
88
|
if monitor_outputs:
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
89
|
+
try:
|
90
|
+
await sync_artifacts(
|
91
|
+
fs=fs,
|
92
|
+
fw=fw,
|
93
|
+
store_path=connection.store_path,
|
94
|
+
run_uuid=run_uuid,
|
95
|
+
exclude=IGNORE_FOLDERS,
|
96
|
+
)
|
97
|
+
except Exception as e:
|
98
|
+
logger.debug(
|
99
|
+
"An error occurred while syncing artifacts, Exception %s" % repr(e)
|
100
|
+
)
|
87
101
|
try:
|
88
102
|
client.sync_events_summaries(
|
89
|
-
last_check=
|
103
|
+
last_check=state["last_artifacts_check"],
|
90
104
|
events_path=ctx_paths.CONTEXT_MOUNT_RUN_EVENTS_FORMAT.format(
|
91
105
|
run_uuid
|
92
106
|
),
|
93
107
|
)
|
94
108
|
client.sync_system_events_summaries(
|
95
|
-
last_check=
|
109
|
+
last_check=state["last_artifacts_check"],
|
96
110
|
events_path=ctx_paths.CONTEXT_MOUNT_RUN_SYSTEM_RESOURCES_EVENTS_FORMAT.format(
|
97
111
|
run_uuid
|
98
112
|
),
|
99
113
|
)
|
100
114
|
update_last_check = True
|
101
115
|
except Exception as e:
|
102
|
-
logger.
|
116
|
+
logger.debug(
|
103
117
|
"An error occurred while syncing events summaries, "
|
104
118
|
"Exception %s" % repr(e)
|
105
119
|
)
|
@@ -124,7 +138,14 @@ async def start_sidecar(
|
|
124
138
|
retry = 0
|
125
139
|
|
126
140
|
counter += 1
|
127
|
-
|
141
|
+
fs_refresh_counter += 1
|
142
|
+
|
143
|
+
if fs_refresh_counter >= fs_refresh_interval:
|
144
|
+
fs_refresh_counter = 0
|
145
|
+
await close_fs(fs)
|
146
|
+
fs = await get_async_fs_from_connection(connection=connection)
|
147
|
+
|
148
|
+
if counter >= sync_interval:
|
128
149
|
counter = 0
|
129
150
|
try:
|
130
151
|
await monitor()
|
@@ -4,7 +4,7 @@ from typing import Optional
|
|
4
4
|
|
5
5
|
import aiofiles
|
6
6
|
|
7
|
-
from clipped.utils.paths import check_or_create_path
|
7
|
+
from clipped.utils.paths import check_or_create_path
|
8
8
|
from kubernetes_asyncio.client.models import V1Pod
|
9
9
|
|
10
10
|
from polyaxon._contexts import paths as ctx_paths
|
@@ -19,26 +19,23 @@ async def sync_logs(
|
|
19
19
|
pod: V1Pod,
|
20
20
|
last_time: Optional[datetime.datetime],
|
21
21
|
stream: bool = False,
|
22
|
-
|
23
|
-
):
|
22
|
+
) -> Optional[datetime.datetime]:
|
24
23
|
path_from = ctx_paths.CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(run_uuid)
|
25
|
-
path_from = "{}
|
24
|
+
path_from = "{}/plxlogs".format(path_from)
|
26
25
|
|
27
|
-
|
28
|
-
delete_path(path_from)
|
29
|
-
return
|
30
|
-
|
31
|
-
logs, _ = await query_k8s_pod_logs(
|
26
|
+
logs, last_time = await query_k8s_pod_logs(
|
32
27
|
k8s_manager=k8s_manager,
|
33
28
|
pod=pod,
|
34
29
|
last_time=last_time,
|
35
30
|
stream=stream,
|
36
31
|
)
|
37
32
|
if not logs:
|
38
|
-
return
|
33
|
+
return last_time
|
39
34
|
|
40
|
-
path_from = "{}/{}.
|
35
|
+
path_from = "{}/{}.jsonl".format(path_from, pod.metadata.name)
|
41
36
|
check_or_create_path(path_from, is_dir=False)
|
42
|
-
async with aiofiles.open(path_from, "
|
37
|
+
async with aiofiles.open(path_from, "a") as filepath:
|
43
38
|
_logs = V1Logs.construct(logs=logs)
|
44
|
-
await filepath.write(
|
39
|
+
await filepath.write(_logs.get_jsonl_events())
|
40
|
+
|
41
|
+
return last_time
|
@@ -0,0 +1,24 @@
|
|
1
|
+
import aiofiles
|
2
|
+
|
3
|
+
from clipped.utils.json import orjson_dumps
|
4
|
+
from clipped.utils.paths import check_or_create_path
|
5
|
+
|
6
|
+
from polyaxon._contexts import paths as ctx_paths
|
7
|
+
from polyaxon._flow import V1RunKind
|
8
|
+
from polyaxon._k8s.logging.async_monitor import get_op_spec
|
9
|
+
from polyaxon._k8s.manager.async_manager import AsyncK8sManager
|
10
|
+
|
11
|
+
|
12
|
+
async def sync_spec(
|
13
|
+
run_uuid: str,
|
14
|
+
k8s_manager: AsyncK8sManager,
|
15
|
+
run_kind: V1RunKind,
|
16
|
+
):
|
17
|
+
op_spec, _, _ = await get_op_spec(
|
18
|
+
k8s_manager=k8s_manager, run_uuid=run_uuid, run_kind=run_kind
|
19
|
+
)
|
20
|
+
path_from = ctx_paths.CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(run_uuid)
|
21
|
+
path_from = "{}/outputs/spec.json".format(path_from)
|
22
|
+
check_or_create_path(path_from, is_dir=False)
|
23
|
+
async with aiofiles.open(path_from, "w") as filepath:
|
24
|
+
await filepath.write(orjson_dumps(op_spec))
|
polyaxon/_sidecar/ignore.py
CHANGED
polyaxon/_utils/fqn_utils.py
CHANGED
@@ -9,15 +9,32 @@ from clipped.utils.paths import get_relative_path_to
|
|
9
9
|
from polyaxon.exceptions import PolyaxonSchemaError
|
10
10
|
|
11
11
|
|
12
|
+
def get_owner_team_space(owner: str, team: Optional[str] = None) -> str:
|
13
|
+
if team:
|
14
|
+
return "{}/{}".format(owner, team)
|
15
|
+
return owner
|
16
|
+
|
17
|
+
|
18
|
+
def split_owner_team_space(owner: str) -> Tuple[str, Optional[str]]:
|
19
|
+
return owner.split("/") if owner and "/" in owner else (owner, None)
|
20
|
+
|
21
|
+
|
22
|
+
def _remove_team(owner: str) -> str:
|
23
|
+
return split_owner_team_space(owner)[0]
|
24
|
+
|
25
|
+
|
12
26
|
def get_project_instance(owner: str, project: str) -> str:
|
27
|
+
owner = _remove_team(owner)
|
13
28
|
return "{}.{}".format(owner, project)
|
14
29
|
|
15
30
|
|
16
31
|
def get_run_instance(owner: str, project: str, run_uuid: str) -> str:
|
32
|
+
owner = _remove_team(owner)
|
17
33
|
return "{}.{}.runs.{}".format(owner, project, run_uuid)
|
18
34
|
|
19
35
|
|
20
36
|
def get_cleaner_instance(owner: str, project: str, run_uuid: str) -> str:
|
37
|
+
owner = _remove_team(owner)
|
21
38
|
return "{}.{}.cleaners.{}".format(owner, project, run_uuid)
|
22
39
|
|
23
40
|
|
@@ -53,6 +70,7 @@ def to_fqn_name(name: str) -> str:
|
|
53
70
|
def get_entity_full_name(
|
54
71
|
owner: Optional[str] = None, entity: Optional[str] = None
|
55
72
|
) -> str:
|
73
|
+
owner = _remove_team(owner)
|
56
74
|
if owner and entity:
|
57
75
|
return "{}/{}".format(owner, entity)
|
58
76
|
return entity
|
@@ -65,11 +83,14 @@ def get_entity_info(entity: str) -> Tuple[str, str]:
|
|
65
83
|
)
|
66
84
|
|
67
85
|
parts = entity.replace(".", "/").split("/")
|
68
|
-
if len(parts) >
|
86
|
+
if len(parts) > 3:
|
69
87
|
raise PolyaxonSchemaError(
|
70
88
|
"Received an invalid entity reference: `{}`".format(entity)
|
71
89
|
)
|
72
|
-
if len(parts) ==
|
90
|
+
if len(parts) == 3:
|
91
|
+
owner = "/".join(parts[:2])
|
92
|
+
entity_name = parts[2]
|
93
|
+
elif len(parts) == 2:
|
73
94
|
owner, entity_name = parts
|
74
95
|
else:
|
75
96
|
owner = None
|
@@ -81,6 +102,8 @@ def get_entity_info(entity: str) -> Tuple[str, str]:
|
|
81
102
|
def get_versioned_entity_full_name(
|
82
103
|
owner: Optional[str], component: str, tag: Optional[str] = None
|
83
104
|
) -> str:
|
105
|
+
owner = _remove_team(owner)
|
106
|
+
|
84
107
|
if tag:
|
85
108
|
component = "{}:{}".format(component, tag)
|
86
109
|
if owner:
|
polyaxon/client.py
CHANGED
@@ -29,5 +29,5 @@ from polyaxon._sdk.api import (
|
|
29
29
|
)
|
30
30
|
from polyaxon._sdk.async_client.api_client import AsyncApiClient
|
31
31
|
from polyaxon._sdk.configuration import Configuration
|
32
|
-
from polyaxon._sdk.schemas import *
|
33
32
|
from polyaxon._sdk.sync_client.api_client import ApiClient
|
33
|
+
from polyaxon.schemas import *
|
polyaxon/pkg.py
CHANGED
polyaxon/schemas.py
CHANGED
@@ -1,5 +1,11 @@
|
|
1
1
|
from clipped.config.patch_strategy import PatchStrategy as V1PatchStrategy
|
2
2
|
|
3
|
+
from polyaxon._auxiliaries import (
|
4
|
+
V1PolyaxonCleaner,
|
5
|
+
V1PolyaxonInitContainer,
|
6
|
+
V1PolyaxonNotifier,
|
7
|
+
V1PolyaxonSidecarContainer,
|
8
|
+
)
|
3
9
|
from polyaxon._connections.kinds import V1ConnectionKind
|
4
10
|
from polyaxon._connections.schemas import (
|
5
11
|
V1BucketConnection,
|
@@ -195,6 +201,7 @@ from polyaxon._sdk.schemas import (
|
|
195
201
|
V1ListTeamMembersResponse,
|
196
202
|
V1ListTeamsResponse,
|
197
203
|
V1ListTokenResponse,
|
204
|
+
V1MultiEventsResponse,
|
198
205
|
V1OperationBody,
|
199
206
|
V1Organization,
|
200
207
|
V1OrganizationMember,
|
@@ -203,7 +210,6 @@ from polyaxon._sdk.schemas import (
|
|
203
210
|
V1Preset,
|
204
211
|
V1Project,
|
205
212
|
V1ProjectSettings,
|
206
|
-
V1ProjectUserAccess,
|
207
213
|
V1ProjectVersion,
|
208
214
|
V1Queue,
|
209
215
|
V1Run,
|
@@ -223,6 +229,7 @@ from polyaxon._sdk.schemas import (
|
|
223
229
|
V1Token,
|
224
230
|
V1TrialStart,
|
225
231
|
V1User,
|
232
|
+
V1UserAccess,
|
226
233
|
V1UserEmail,
|
227
234
|
V1UserSingup,
|
228
235
|
V1Uuids,
|
polyaxon/settings.py
CHANGED
@@ -38,6 +38,12 @@ AGENT_CONFIG: Optional[AgentConfig] = None
|
|
38
38
|
PolyaxonServices.set_service_name()
|
39
39
|
|
40
40
|
|
41
|
+
def set_home_config(config: Optional[HomeConfig] = None):
|
42
|
+
global HOME_CONFIG
|
43
|
+
|
44
|
+
HOME_CONFIG = config or HomeConfigManager.get_config_from_env()
|
45
|
+
|
46
|
+
|
41
47
|
def set_agent_config(config: Optional[AgentConfig] = None):
|
42
48
|
from polyaxon._connections import CONNECTION_CONFIG
|
43
49
|
from polyaxon._managers.agent import AgentConfigManager
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: polyaxon
|
3
|
-
Version: 2.
|
3
|
+
Version: 2.4.0rc1
|
4
4
|
Summary: Command Line Interface (CLI) and client to interact with Polyaxon API.
|
5
5
|
Home-page: https://github.com/polyaxon/polyaxon
|
6
6
|
Author: Polyaxon, Inc.
|
@@ -33,67 +33,67 @@ Classifier: Programming Language :: Python :: 3.11
|
|
33
33
|
Requires-Python: >=3.8
|
34
34
|
Description-Content-Type: text/markdown
|
35
35
|
License-File: LICENSE
|
36
|
-
Requires-Dist: click
|
37
|
-
Requires-Dist: clipped
|
38
|
-
Requires-Dist: vents
|
39
|
-
Requires-Dist: Jinja2
|
40
|
-
Requires-Dist: kubernetes
|
41
|
-
Requires-Dist: python-dateutil
|
42
|
-
Requires-Dist: pytz
|
43
|
-
Requires-Dist: PyYAML
|
44
|
-
Requires-Dist: orjson
|
45
|
-
Requires-Dist: psutil
|
46
|
-
Requires-Dist: requests
|
47
|
-
Requires-Dist: requests-toolbelt
|
48
|
-
Requires-Dist: rich
|
49
|
-
Requires-Dist: sentry-sdk
|
50
|
-
Requires-Dist: urllib3
|
51
|
-
Requires-Dist: certifi
|
52
|
-
Requires-Dist: pydantic
|
53
|
-
Requires-Dist: traceml
|
54
|
-
Requires-Dist: hypertune
|
36
|
+
Requires-Dist: click<9.0.0,>=7.1.1
|
37
|
+
Requires-Dist: clipped==0.8.*
|
38
|
+
Requires-Dist: vents==0.4.*
|
39
|
+
Requires-Dist: Jinja2<3.1.5,>=2.10.3
|
40
|
+
Requires-Dist: kubernetes>=10.0.1
|
41
|
+
Requires-Dist: python-dateutil>=2.7.3
|
42
|
+
Requires-Dist: pytz>=2019.2
|
43
|
+
Requires-Dist: PyYAML>=5.1
|
44
|
+
Requires-Dist: orjson>=3.7
|
45
|
+
Requires-Dist: psutil>=5.4.7
|
46
|
+
Requires-Dist: requests>=2.20.1
|
47
|
+
Requires-Dist: requests-toolbelt>=0.8.0
|
48
|
+
Requires-Dist: rich>=12.0.0
|
49
|
+
Requires-Dist: sentry-sdk<2.6,>=1.2.0
|
50
|
+
Requires-Dist: urllib3>=1.25.6
|
51
|
+
Requires-Dist: certifi>=2022.12.7
|
52
|
+
Requires-Dist: pydantic>=1.10.2
|
53
|
+
Requires-Dist: traceml==1.1.*
|
54
|
+
Requires-Dist: hypertune==1.1.*
|
55
55
|
Provides-Extra: azure
|
56
|
-
Requires-Dist: adlfs
|
56
|
+
Requires-Dist: adlfs; extra == "azure"
|
57
57
|
Provides-Extra: dev
|
58
|
-
Requires-Dist: moto
|
58
|
+
Requires-Dist: moto==2.0.5; extra == "dev"
|
59
59
|
Provides-Extra: docker
|
60
|
-
Requires-Dist: docker
|
60
|
+
Requires-Dist: docker; extra == "docker"
|
61
61
|
Provides-Extra: fs
|
62
|
-
Requires-Dist: adlfs
|
63
|
-
Requires-Dist: fsspec
|
64
|
-
Requires-Dist: gcsfs
|
65
|
-
Requires-Dist: s3fs
|
62
|
+
Requires-Dist: adlfs==2024.4.1; extra == "fs"
|
63
|
+
Requires-Dist: fsspec==2024.6.1; extra == "fs"
|
64
|
+
Requires-Dist: gcsfs==2024.6.1; extra == "fs"
|
65
|
+
Requires-Dist: s3fs==2024.6.1; extra == "fs"
|
66
66
|
Provides-Extra: fsspec
|
67
|
-
Requires-Dist: fsspec
|
67
|
+
Requires-Dist: fsspec; extra == "fsspec"
|
68
68
|
Provides-Extra: gcs
|
69
|
-
Requires-Dist: gcsfs
|
69
|
+
Requires-Dist: gcsfs; extra == "gcs"
|
70
70
|
Provides-Extra: git
|
71
|
-
Requires-Dist: gitpython
|
71
|
+
Requires-Dist: gitpython; extra == "git"
|
72
72
|
Provides-Extra: init
|
73
|
-
Requires-Dist: docker
|
74
|
-
Requires-Dist: GitPython
|
75
|
-
Requires-Dist: aiofiles
|
73
|
+
Requires-Dist: docker; extra == "init"
|
74
|
+
Requires-Dist: GitPython<3.2.0; extra == "init"
|
75
|
+
Requires-Dist: aiofiles==23.2.1; extra == "init"
|
76
76
|
Provides-Extra: k8s_async
|
77
|
-
Requires-Dist: kubernetes-asyncio
|
78
|
-
Requires-Dist: aiohttp
|
79
|
-
Requires-Dist: anyio
|
77
|
+
Requires-Dist: kubernetes-asyncio>=28.2.1; extra == "k8s-async"
|
78
|
+
Requires-Dist: aiohttp>=3.0.0; extra == "k8s-async"
|
79
|
+
Requires-Dist: anyio; extra == "k8s-async"
|
80
80
|
Provides-Extra: numpy
|
81
|
-
Requires-Dist: numpy
|
81
|
+
Requires-Dist: numpy; extra == "numpy"
|
82
82
|
Provides-Extra: s3
|
83
|
-
Requires-Dist: s3fs
|
83
|
+
Requires-Dist: s3fs; extra == "s3"
|
84
84
|
Provides-Extra: sandbox
|
85
|
-
Requires-Dist: haupt[sandbox]
|
85
|
+
Requires-Dist: haupt[sandbox]; extra == "sandbox"
|
86
86
|
Provides-Extra: sidecar
|
87
|
-
Requires-Dist: aiofiles
|
88
|
-
Requires-Dist: pandas
|
89
|
-
Requires-Dist: anyio
|
87
|
+
Requires-Dist: aiofiles==23.2.1; extra == "sidecar"
|
88
|
+
Requires-Dist: pandas; extra == "sidecar"
|
89
|
+
Requires-Dist: anyio; extra == "sidecar"
|
90
90
|
|
91
91
|
[](LICENSE)
|
92
92
|
[](https://hub.docker.com/r/polyaxon/polyaxon-api)
|
93
93
|
[](https://polyaxon.com/slack/)
|
94
94
|
|
95
95
|
[](https://polyaxon.com/docs/)
|
96
|
-
[](https://polyaxon.com/docs/releases/2-1/)
|
97
97
|
[](https://github.com/polyaxon/polyaxon/issues)
|
98
98
|
[](https://github.com/orgs/polyaxon/projects/5)
|
99
99
|
|