polyaxon 2.0.0rc49__py3-none-any.whl → 2.4.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- polyaxon/_auxiliaries/cleaner.py +8 -3
- polyaxon/_auxiliaries/init.py +7 -2
- polyaxon/_auxiliaries/notifier.py +8 -2
- polyaxon/_auxiliaries/sidecar.py +30 -2
- polyaxon/_cli/artifacts.py +96 -11
- polyaxon/_cli/components.py +96 -11
- polyaxon/_cli/config.py +118 -22
- polyaxon/_cli/dashboard.py +15 -2
- polyaxon/_cli/init.py +1 -1
- polyaxon/_cli/models.py +96 -11
- polyaxon/_cli/operations.py +267 -90
- polyaxon/_cli/project_versions.py +139 -6
- polyaxon/_cli/projects.py +23 -9
- polyaxon/_cli/run.py +37 -9
- polyaxon/_cli/services/agent.py +2 -2
- polyaxon/_cli/services/clean_artifacts.py +1 -1
- polyaxon/_cli/services/sidecar.py +8 -1
- polyaxon/_client/client.py +17 -0
- polyaxon/_client/mixin.py +39 -0
- polyaxon/_client/project.py +218 -23
- polyaxon/_client/run.py +131 -33
- polyaxon/_compiler/contexts/contexts.py +2 -2
- polyaxon/_compiler/contexts/ray_job.py +4 -2
- polyaxon/_compiler/resolver/agent.py +12 -2
- polyaxon/_compiler/resolver/runtime.py +2 -2
- polyaxon/_contexts/paths.py +4 -7
- polyaxon/_deploy/operators/compose.py +1 -27
- polyaxon/_deploy/schemas/deployment.py +4 -1
- polyaxon/_deploy/schemas/intervals.py +0 -7
- polyaxon/_deploy/schemas/proxy.py +1 -0
- polyaxon/_deploy/schemas/service.py +11 -1
- polyaxon/_docker/converter/base/base.py +8 -0
- polyaxon/_docker/executor.py +10 -4
- polyaxon/_env_vars/getters/owner_entity.py +4 -2
- polyaxon/_env_vars/getters/project.py +4 -2
- polyaxon/_env_vars/getters/run.py +5 -2
- polyaxon/_env_vars/keys.py +7 -1
- polyaxon/_flow/__init__.py +2 -0
- polyaxon/_flow/builds/__init__.py +19 -6
- polyaxon/_flow/component/base.py +1 -0
- polyaxon/_flow/component/component.py +14 -0
- polyaxon/_flow/environment/__init__.py +8 -8
- polyaxon/_flow/hooks/__init__.py +19 -6
- polyaxon/_flow/init/__init__.py +6 -6
- polyaxon/_flow/matrix/iterative.py +0 -1
- polyaxon/_flow/matrix/tuner.py +18 -6
- polyaxon/_flow/operations/operation.py +44 -17
- polyaxon/_flow/plugins/__init__.py +6 -0
- polyaxon/_flow/run/__init__.py +2 -2
- polyaxon/_flow/run/dag.py +2 -2
- polyaxon/_flow/run/dask/dask.py +0 -1
- polyaxon/_flow/run/dask/replica.py +3 -3
- polyaxon/_flow/run/enums.py +5 -0
- polyaxon/_flow/run/job.py +4 -4
- polyaxon/_flow/run/kubeflow/mpi_job.py +1 -2
- polyaxon/_flow/run/kubeflow/mx_job.py +1 -2
- polyaxon/_flow/run/kubeflow/paddle_job.py +35 -4
- polyaxon/_flow/run/kubeflow/pytorch_job.py +51 -5
- polyaxon/_flow/run/kubeflow/replica.py +4 -4
- polyaxon/_flow/run/kubeflow/scheduling_policy.py +12 -0
- polyaxon/_flow/run/kubeflow/tf_job.py +3 -3
- polyaxon/_flow/run/kubeflow/xgboost_job.py +1 -2
- polyaxon/_flow/run/ray/ray.py +2 -3
- polyaxon/_flow/run/ray/replica.py +3 -3
- polyaxon/_flow/run/service.py +4 -4
- polyaxon/_fs/fs.py +7 -2
- polyaxon/_fs/utils.py +3 -2
- polyaxon/_k8s/converter/base/base.py +2 -1
- polyaxon/_k8s/converter/base/main.py +1 -0
- polyaxon/_k8s/converter/base/sidecar.py +16 -1
- polyaxon/_k8s/converter/common/accelerators.py +7 -4
- polyaxon/_k8s/converter/converters/job.py +1 -1
- polyaxon/_k8s/converter/converters/kubeflow/paddle_job.py +1 -0
- polyaxon/_k8s/converter/converters/kubeflow/pytroch_job.py +2 -0
- polyaxon/_k8s/converter/converters/kubeflow/tf_job.py +1 -0
- polyaxon/_k8s/converter/converters/ray_job.py +4 -2
- polyaxon/_k8s/custom_resources/dask_job.py +3 -0
- polyaxon/_k8s/custom_resources/kubeflow/common.py +4 -1
- polyaxon/_k8s/custom_resources/kubeflow/paddle_job.py +10 -1
- polyaxon/_k8s/custom_resources/kubeflow/pytorch_job.py +14 -1
- polyaxon/_k8s/custom_resources/kubeflow/tf_job.py +4 -0
- polyaxon/_k8s/custom_resources/ray_job.py +3 -0
- polyaxon/_k8s/custom_resources/setter.py +1 -1
- polyaxon/_k8s/executor/async_executor.py +2 -0
- polyaxon/_k8s/executor/base.py +23 -6
- polyaxon/_k8s/logging/async_monitor.py +150 -5
- polyaxon/_k8s/manager/async_manager.py +96 -23
- polyaxon/_k8s/manager/base.py +4 -0
- polyaxon/_k8s/manager/manager.py +282 -134
- polyaxon/_local_process/__init__.py +0 -0
- polyaxon/_local_process/agent.py +6 -0
- polyaxon/_local_process/converter/__init__.py +1 -0
- polyaxon/_local_process/converter/base/__init__.py +1 -0
- polyaxon/_local_process/converter/base/base.py +140 -0
- polyaxon/_local_process/converter/base/containers.py +69 -0
- polyaxon/_local_process/converter/base/env_vars.py +253 -0
- polyaxon/_local_process/converter/base/init.py +414 -0
- polyaxon/_local_process/converter/base/main.py +74 -0
- polyaxon/_local_process/converter/base/mounts.py +82 -0
- polyaxon/_local_process/converter/converters/__init__.py +8 -0
- polyaxon/_local_process/converter/converters/job.py +40 -0
- polyaxon/_local_process/converter/converters/service.py +41 -0
- polyaxon/_local_process/converter/mixins.py +38 -0
- polyaxon/_local_process/executor.py +132 -0
- polyaxon/_local_process/process_types.py +39 -0
- polyaxon/_managers/agent.py +2 -0
- polyaxon/_managers/home.py +2 -1
- polyaxon/_operations/tuner.py +1 -0
- polyaxon/_polyaxonfile/check.py +2 -0
- polyaxon/_polyaxonfile/manager/operations.py +3 -0
- polyaxon/_polyaxonfile/manager/workflows.py +2 -0
- polyaxon/_polyaxonfile/specs/compiled_operation.py +1 -0
- polyaxon/_polyaxonfile/specs/operation.py +1 -0
- polyaxon/_polyaxonfile/specs/sections.py +3 -0
- polyaxon/_pql/manager.py +1 -1
- polyaxon/_runner/agent/async_agent.py +97 -21
- polyaxon/_runner/agent/base_agent.py +27 -9
- polyaxon/_runner/agent/client.py +15 -1
- polyaxon/_runner/agent/sync_agent.py +85 -20
- polyaxon/_runner/converter/converter.py +6 -2
- polyaxon/_runner/executor.py +13 -7
- polyaxon/_schemas/agent.py +27 -1
- polyaxon/_schemas/client.py +30 -3
- polyaxon/_schemas/installation.py +4 -3
- polyaxon/_schemas/lifecycle.py +10 -5
- polyaxon/_schemas/log_handler.py +2 -3
- polyaxon/_schemas/types/artifacts.py +3 -3
- polyaxon/_schemas/types/dockerfile.py +3 -3
- polyaxon/_schemas/types/file.py +3 -3
- polyaxon/_schemas/types/git.py +3 -3
- polyaxon/_schemas/types/tensorboard.py +3 -3
- polyaxon/_sdk/api/agents_v1_api.py +1076 -73
- polyaxon/_sdk/api/organizations_v1_api.py +371 -10
- polyaxon/_sdk/api/project_dashboards_v1_api.py +12 -12
- polyaxon/_sdk/api/project_searches_v1_api.py +12 -12
- polyaxon/_sdk/api/projects_v1_api.py +221 -44
- polyaxon/_sdk/api/runs_v1_api.py +917 -445
- polyaxon/_sdk/api/service_accounts_v1_api.py +16 -16
- polyaxon/_sdk/api/teams_v1_api.py +2827 -375
- polyaxon/_sdk/api/users_v1_api.py +231 -55
- polyaxon/_sdk/async_client/api_client.py +4 -0
- polyaxon/_sdk/schemas/__init__.py +10 -2
- polyaxon/_sdk/schemas/v1_agent.py +2 -1
- polyaxon/_sdk/schemas/v1_agent_reconcile_body_request.py +14 -0
- polyaxon/_sdk/schemas/v1_artifact_tree.py +1 -1
- polyaxon/_sdk/schemas/v1_dashboard_spec.py +4 -0
- polyaxon/_sdk/schemas/v1_events_response.py +4 -0
- polyaxon/_sdk/schemas/v1_organization.py +1 -0
- polyaxon/_sdk/schemas/v1_preset.py +8 -0
- polyaxon/_sdk/schemas/v1_project.py +1 -0
- polyaxon/_sdk/schemas/v1_project_settings.py +4 -2
- polyaxon/_sdk/schemas/v1_run.py +2 -2
- polyaxon/_sdk/schemas/v1_run_edge_lineage.py +14 -0
- polyaxon/_sdk/schemas/v1_run_edges_graph.py +9 -0
- polyaxon/_sdk/schemas/v1_section_spec.py +7 -2
- polyaxon/_sdk/schemas/v1_settings_catalog.py +1 -0
- polyaxon/_sdk/schemas/v1_team.py +3 -0
- polyaxon/_sdk/schemas/v1_user.py +1 -2
- polyaxon/_sdk/schemas/v1_user_access.py +17 -0
- polyaxon/_services/values.py +1 -0
- polyaxon/_sidecar/container/__init__.py +39 -18
- polyaxon/_sidecar/container/monitors/__init__.py +1 -0
- polyaxon/_sidecar/container/monitors/logs.py +10 -13
- polyaxon/_sidecar/container/monitors/spec.py +24 -0
- polyaxon/_sidecar/ignore.py +0 -1
- polyaxon/_utils/fqn_utils.py +25 -2
- polyaxon/client.py +1 -1
- polyaxon/pkg.py +1 -1
- polyaxon/schemas.py +8 -1
- polyaxon/settings.py +6 -0
- {polyaxon-2.0.0rc49.dist-info → polyaxon-2.4.0rc1.dist-info}/METADATA +43 -43
- {polyaxon-2.0.0rc49.dist-info → polyaxon-2.4.0rc1.dist-info}/RECORD +176 -155
- {polyaxon-2.0.0rc49.dist-info → polyaxon-2.4.0rc1.dist-info}/WHEEL +1 -1
- polyaxon/_sdk/schemas/v1_project_user_access.py +0 -10
- {polyaxon-2.0.0rc49.dist-info → polyaxon-2.4.0rc1.dist-info}/LICENSE +0 -0
- {polyaxon-2.0.0rc49.dist-info → polyaxon-2.4.0rc1.dist-info}/entry_points.txt +0 -0
- {polyaxon-2.0.0rc49.dist-info → polyaxon-2.4.0rc1.dist-info}/top_level.txt +0 -0
@@ -175,7 +175,7 @@ def validate_deployment_chart(
|
|
175
175
|
if deployment_chart == DeploymentCharts.AGENT and not agent:
|
176
176
|
raise ValueError("Agent deployment requires a valid `agent` key configuration.")
|
177
177
|
|
178
|
-
if deployment_chart == DeploymentCharts.PLATFORM and agent:
|
178
|
+
if deployment_chart == DeploymentCharts.PLATFORM and agent and agent.enabled:
|
179
179
|
Printer.header("Deployment has agent enabled!")
|
180
180
|
|
181
181
|
|
@@ -198,6 +198,7 @@ class DeploymentConfig(BaseSchemaModel):
|
|
198
198
|
polyaxon_secret: Optional[StrictStr] = Field(alias="polyaxonSecret")
|
199
199
|
internal_token: Optional[StrictStr] = Field(alias="internalToken")
|
200
200
|
password_length: Optional[StrictInt] = Field(alias="passwordLength")
|
201
|
+
password_auth: Optional[bool] = Field(alias="passwordAuth")
|
201
202
|
ssl: Optional[SSLConfig]
|
202
203
|
encryption_secret: Optional[StrictStr] = Field(alias="encryptionSecret")
|
203
204
|
platform_secret: Optional[StrictStr] = Field(alias="platformSecret")
|
@@ -254,6 +255,7 @@ class DeploymentConfig(BaseSchemaModel):
|
|
254
255
|
allowed_hosts: Optional[List[StrictStr]] = Field(alias="allowedHosts")
|
255
256
|
include_host_ips: Optional[bool] = Field(alias="includeHostIps")
|
256
257
|
intervals: Optional[IntervalsConfig]
|
258
|
+
cleaning_intervals: Optional[Dict] = Field(alias="cleaningIntervals")
|
257
259
|
artifacts_store: Optional[V1Connection] = Field(alias="artifactsStore")
|
258
260
|
connections: Optional[List[V1Connection]]
|
259
261
|
mount_connections: Optional[List[str]] = Field(alias="mountConnections")
|
@@ -293,6 +295,7 @@ class DeploymentConfig(BaseSchemaModel):
|
|
293
295
|
if values.get("deployment_chart") == DeploymentCharts.AGENT:
|
294
296
|
wrong_agent_deployment_keys(
|
295
297
|
password_length=values.get("password_length"),
|
298
|
+
password_auth=values.get("password_auth"),
|
296
299
|
platform_secret=values.get("platform_secret"),
|
297
300
|
encryption_secret=values.get("encryption_secret"),
|
298
301
|
user=values.get("user"),
|
@@ -6,11 +6,4 @@ from polyaxon._schemas.base import BaseSchemaModel
|
|
6
6
|
|
7
7
|
|
8
8
|
class IntervalsConfig(BaseSchemaModel):
|
9
|
-
runs_scheduler: Optional[StrictInt] = Field(alias="runsScheduler")
|
10
|
-
operations_default_retry_delay: Optional[StrictInt] = Field(
|
11
|
-
alias="operationsDefaultRetryDelay"
|
12
|
-
)
|
13
|
-
operations_max_retry_delay: Optional[StrictInt] = Field(
|
14
|
-
alias="operationsMaxRetryDelay"
|
15
|
-
)
|
16
9
|
compatibility_check: Optional[StrictInt] = Field(alias="compatibilityCheck")
|
@@ -12,6 +12,7 @@ class DeploymentService(BaseServiceConfig):
|
|
12
12
|
enabled: Optional[bool]
|
13
13
|
replicas: Optional[StrictInt]
|
14
14
|
concurrency: Optional[StrictInt]
|
15
|
+
per_core: Optional[bool] = Field(alias="perCore")
|
15
16
|
scheme: Optional[StrictStr]
|
16
17
|
|
17
18
|
|
@@ -22,6 +23,11 @@ class WorkerServiceConfig(DeploymentService):
|
|
22
23
|
class AgentServiceConfig(DeploymentService):
|
23
24
|
instance: Optional[StrictStr]
|
24
25
|
token: Optional[StrictStr]
|
26
|
+
watch_cluster: Optional[bool] = Field(alias="watchCluster")
|
27
|
+
additional_namespaces: Optional[List[str]] = Field(alias="additionalNamespaces")
|
28
|
+
service_account_annotations: Optional[Dict] = Field(
|
29
|
+
alias="serviceAccountAnnotations"
|
30
|
+
)
|
25
31
|
is_replica: Optional[bool] = Field(alias="isReplica")
|
26
32
|
|
27
33
|
|
@@ -50,6 +56,7 @@ class HooksConfig(DeploymentService):
|
|
50
56
|
tables: Optional[bool] = Field(alias="tables")
|
51
57
|
sync_db: Optional[bool] = Field(alias="syncdb")
|
52
58
|
admin_user: Optional[bool] = Field(alias="adminUser")
|
59
|
+
default_org: Optional[bool] = Field(alias="defaultOrg")
|
53
60
|
|
54
61
|
|
55
62
|
class ThirdPartyService(DeploymentService):
|
@@ -92,6 +99,7 @@ class ExternalService(BaseSchemaModel):
|
|
92
99
|
class ExternalBackend(BaseSchemaModel):
|
93
100
|
enabled: Optional[bool]
|
94
101
|
backend: Optional[StrictStr]
|
102
|
+
url: Optional[StrictStr]
|
95
103
|
options: Optional[Dict]
|
96
104
|
|
97
105
|
|
@@ -100,7 +108,9 @@ class AuthServicesConfig(BaseSchemaModel):
|
|
100
108
|
gitlab: Optional[ExternalBackend]
|
101
109
|
bitbucket: Optional[ExternalBackend]
|
102
110
|
google: Optional[ExternalBackend]
|
103
|
-
|
111
|
+
okta: Optional[ExternalBackend]
|
112
|
+
onelogin: Optional[ExternalBackend]
|
113
|
+
azuread: Optional[ExternalBackend]
|
104
114
|
|
105
115
|
|
106
116
|
class ExternalServicesConfig(BaseSchemaModel):
|
@@ -123,6 +123,14 @@ class BaseConverter(
|
|
123
123
|
if not resources:
|
124
124
|
return None
|
125
125
|
|
126
|
+
if isinstance(resources, dict):
|
127
|
+
try:
|
128
|
+
resources = k8s_schemas.V1ResourceRequirements(**resources)
|
129
|
+
except (ValueError, TypeError) as e:
|
130
|
+
raise PolyaxonConverterError(
|
131
|
+
f"Could not parse resources value `{resources}` from the K8S schema in container section"
|
132
|
+
) from e
|
133
|
+
|
126
134
|
cpus = None
|
127
135
|
memory = None
|
128
136
|
gpus = None
|
polyaxon/_docker/executor.py
CHANGED
@@ -35,6 +35,7 @@ class Executor(BaseExecutor):
|
|
35
35
|
run_uuid: str,
|
36
36
|
run_kind: str,
|
37
37
|
resource: List[docker_types.V1Container],
|
38
|
+
namespace: str = None,
|
38
39
|
) -> Dict:
|
39
40
|
logger.info(f"[Executor] Starting operation {run_uuid} {run_kind}.")
|
40
41
|
self._ops[run_uuid] = []
|
@@ -62,13 +63,15 @@ class Executor(BaseExecutor):
|
|
62
63
|
self._clean_temp_execution_path(run_uuid)
|
63
64
|
return {"status": V1Statuses.SUCCEEDED, "tasks": self._ops[run_uuid]}
|
64
65
|
|
65
|
-
def apply(
|
66
|
+
def apply(
|
67
|
+
self, run_uuid: str, run_kind: str, resource: Dict, namespace: str = None
|
68
|
+
) -> Dict:
|
66
69
|
raise PolyaxonAgentError(
|
67
70
|
"Docker executor does not support apply method.\n"
|
68
71
|
"Run: <kind: {}, uuid: {}>".format(run_kind, run_uuid)
|
69
72
|
)
|
70
73
|
|
71
|
-
def stop(self, run_uuid: str, run_kind: str):
|
74
|
+
def stop(self, run_uuid: str, run_kind: str, namespace: str = None):
|
72
75
|
proc = self._get_op_proc(run_uuid)
|
73
76
|
if proc.poll() is None:
|
74
77
|
# Kill the process tree rooted at the child if it's the leader of its own process
|
@@ -85,7 +88,7 @@ class Executor(BaseExecutor):
|
|
85
88
|
logger.debug(_msg)
|
86
89
|
proc.wait()
|
87
90
|
|
88
|
-
def clean(self, run_uuid: str, run_kind: str):
|
91
|
+
def clean(self, run_uuid: str, run_kind: str, namespace: str = None):
|
89
92
|
return self.apply(
|
90
93
|
run_uuid=run_uuid,
|
91
94
|
run_kind=run_kind,
|
@@ -100,6 +103,9 @@ class Executor(BaseExecutor):
|
|
100
103
|
return V1Statuses.SUCCEEDED
|
101
104
|
return V1Statuses.FAILED
|
102
105
|
|
103
|
-
def get(self, run_uuid: str, run_kind: str) -> V1Statuses:
|
106
|
+
def get(self, run_uuid: str, run_kind: str, namespace: str = None) -> V1Statuses:
|
104
107
|
procs = self._get_op_proc(run_uuid)
|
105
108
|
return self._get_task_status(procs[-1])
|
109
|
+
|
110
|
+
def list_ops(self, namespace: str = None):
|
111
|
+
return []
|
@@ -5,7 +5,7 @@ from clipped.utils.strings import validate_slug
|
|
5
5
|
|
6
6
|
from polyaxon._constants.globals import DEFAULT
|
7
7
|
from polyaxon._env_vars.getters.user import get_local_owner
|
8
|
-
from polyaxon._utils.fqn_utils import get_entity_info
|
8
|
+
from polyaxon._utils.fqn_utils import get_entity_info, split_owner_team_space
|
9
9
|
from polyaxon.exceptions import PolyaxonClientException, PolyaxonSchemaError
|
10
10
|
|
11
11
|
|
@@ -31,6 +31,8 @@ def resolve_entity_info(entity: str, entity_name: str, is_cli: bool = False):
|
|
31
31
|
if not owner:
|
32
32
|
owner = settings.AUTH_CONFIG.username if settings.AUTH_CONFIG else None
|
33
33
|
|
34
|
+
owner, team = split_owner_team_space(owner)
|
35
|
+
|
34
36
|
if not all([owner, entity_value]):
|
35
37
|
message = "Please provide a valid {}.".format(entity_name)
|
36
38
|
if is_cli:
|
@@ -49,4 +51,4 @@ def resolve_entity_info(entity: str, entity_name: str, is_cli: bool = False):
|
|
49
51
|
entity_name, entity_value
|
50
52
|
)
|
51
53
|
)
|
52
|
-
return owner, entity_value
|
54
|
+
return owner, team, entity_value
|
@@ -7,7 +7,7 @@ from polyaxon._constants.globals import DEFAULT
|
|
7
7
|
from polyaxon._env_vars.getters.user import get_local_owner
|
8
8
|
from polyaxon._managers.project import ProjectConfigManager
|
9
9
|
from polyaxon._utils.cache import get_local_project
|
10
|
-
from polyaxon._utils.fqn_utils import get_entity_info
|
10
|
+
from polyaxon._utils.fqn_utils import get_entity_info, split_owner_team_space
|
11
11
|
from polyaxon.exceptions import PolyaxonClientException, PolyaxonSchemaError
|
12
12
|
|
13
13
|
|
@@ -52,6 +52,8 @@ def get_project_or_local(project=None, is_cli: bool = False):
|
|
52
52
|
if not owner and (not settings.CLI_CONFIG or settings.CLI_CONFIG.is_community):
|
53
53
|
owner = DEFAULT
|
54
54
|
|
55
|
+
owner, team = split_owner_team_space(owner)
|
56
|
+
|
55
57
|
if not all([owner, project_name]):
|
56
58
|
error_message = get_project_error_message(owner, project_name)
|
57
59
|
if is_cli:
|
@@ -79,4 +81,4 @@ def get_project_or_local(project=None, is_cli: bool = False):
|
|
79
81
|
sys.exit(1)
|
80
82
|
else:
|
81
83
|
raise PolyaxonSchemaError(error_message)
|
82
|
-
return owner, project_name
|
84
|
+
return owner, team, project_name
|
@@ -37,9 +37,9 @@ def get_run_or_local(run_uuid=None, is_cli: bool = False):
|
|
37
37
|
|
38
38
|
|
39
39
|
def get_project_run_or_local(project=None, run_uuid=None, is_cli: bool = True):
|
40
|
-
|
40
|
+
owner, team, project_name = get_project_or_local(project, is_cli=is_cli)
|
41
41
|
run_uuid = get_run_or_local(run_uuid, is_cli=is_cli)
|
42
|
-
return
|
42
|
+
return owner, team, project_name, run_uuid
|
43
43
|
|
44
44
|
|
45
45
|
def get_collect_artifacts(arg: Optional[bool] = None, default: Optional[bool] = None):
|
@@ -74,7 +74,10 @@ def get_run_info(run_instance: Optional[str] = None):
|
|
74
74
|
"Could not get run info, "
|
75
75
|
"please make sure this is run is correctly started by Polyaxon."
|
76
76
|
)
|
77
|
+
return get_run_info_from_instance(run_instance)
|
77
78
|
|
79
|
+
|
80
|
+
def get_run_info_from_instance(run_instance: str):
|
78
81
|
parts = run_instance.split(".")
|
79
82
|
if not len(parts) == 4:
|
80
83
|
raise PolyaxonClientException(
|
polyaxon/_env_vars/keys.py
CHANGED
@@ -78,6 +78,9 @@ ENV_KEYS_AGENT_ARTIFACTS_STORE = "POLYAXON_AGENT_ARTIFACTS_STORE"
|
|
78
78
|
ENV_KEYS_AGENT_CONNECTIONS = "POLYAXON_AGENT_CONNECTIONS"
|
79
79
|
ENV_KEYS_SET_AGENT = "POLYAXON_SET_AGENT"
|
80
80
|
ENV_KEYS_K8S_APP_SECRET_NAME = "POLYAXON_K8S_APP_SECRET_NAME" # noqa
|
81
|
+
ENV_KEYS_WATCH_CLUSTER = "POLYAXON_WATCH_CLUSTER"
|
82
|
+
ENV_KEYS_SINGLE_NAMESPACE = "POLYAXON_SINGLE_NAMESPACE"
|
83
|
+
ENV_KEYS_ADDITIONAL_NAMESPACES = "POLYAXON_ADDITIONAL_NAMESPACES"
|
81
84
|
ENV_KEYS_AGENT_SECRET_NAME = "POLYAXON_AGENT_SECRET_NAME" # noqa
|
82
85
|
ENV_KEYS_AGENT_RUNS_SA = "POLYAXON_AGENT_RUNS_SA"
|
83
86
|
ENV_KEYS_AGENT_ENABLE_HEALTH_CHECKS = "POLYAXON_AGENT_ENABLE_HEALTH_CHECKS"
|
@@ -112,14 +115,16 @@ ENV_KEYS_SANDBOX_DEBUG = "POLYAXON_SANDBOX_DEBUG"
|
|
112
115
|
ENV_KEYS_SANDBOX_SSL_ENABLED = "POLYAXON_SANDBOX_SSL_ENABLED"
|
113
116
|
ENV_KEYS_SANDBOX_WORKERS = "POLYAXON_SANDBOX_WORKERS"
|
114
117
|
ENV_KEYS_SANDBOX_PER_CORE = "POLYAXON_SANDBOX_PER_CORE"
|
115
|
-
ENV_KEYS_SANDBOX_IS_LOCAL = "POLYAXON_SANDBOX_IS_LOCAL"
|
116
118
|
ENV_KEYS_SANDBOX_ROOT = "POLYAXON_SANDBOX_ROOT"
|
119
|
+
ENV_KEYS_SERVICE_MODE = "POLYAXON_SERVICE_MODE"
|
117
120
|
|
118
121
|
# Proxies
|
119
122
|
ENV_KEYS_PROXY_NAMESPACES = "POLYAXON_PROXY_NAMESPACES"
|
120
123
|
ENV_KEYS_PROXY_GATEWAY_PORT = "POLYAXON_PROXY_GATEWAY_PORT"
|
121
124
|
ENV_KEYS_PROXY_GATEWAY_TARGET_PORT = "POLYAXON_PROXY_GATEWAY_TARGET_PORT"
|
122
125
|
ENV_KEYS_PROXY_GATEWAY_HOST = "POLYAXON_PROXY_GATEWAY_HOST"
|
126
|
+
ENV_KEYS_PROXY_GATEWAY_CONCURRENCY = "POLYAXON_PROXY_GATEWAY_CONCURRENCY"
|
127
|
+
ENV_KEYS_PROXY_GATEWAY_PER_CORE = "POLYAXON_PROXY_GATEWAY_PER_CORE"
|
123
128
|
ENV_KEYS_PROXY_STREAMS_PORT = "POLYAXON_PROXY_STREAMS_PORT"
|
124
129
|
ENV_KEYS_PROXY_STREAMS_TARGET_PORT = "POLYAXON_PROXY_STREAMS_TARGET_PORT"
|
125
130
|
ENV_KEYS_PROXY_STREAMS_HOST = "POLYAXON_PROXY_STREAMS_HOST"
|
@@ -136,6 +141,7 @@ ENV_KEYS_PROXY_AUTH_USE_RESOLVER = "POLYAXON_PROXY_AUTH_USE_RESOLVER"
|
|
136
141
|
ENV_KEYS_PROXY_HAS_FORWARD_PROXY = "POLYAXON_PROXY_HAS_FORWARD_PROXY"
|
137
142
|
ENV_KEYS_PROXY_FORWARD_PROXY_PORT = "POLYAXON_PROXY_FORWARD_PROXY_PORT"
|
138
143
|
ENV_KEYS_PROXY_FORWARD_PROXY_HOST = "POLYAXON_PROXY_FORWARD_PROXY_HOST"
|
144
|
+
ENV_KEYS_PROXY_FORWARD_PROXY_PROTOCOL = "POLYAXON_PROXY_FORWARD_PROXY_PROTOCOL"
|
139
145
|
ENV_KEYS_PROXY_FORWARD_PROXY_KIND = "POLYAXON_PROXY_FORWARD_PROXY_KIND"
|
140
146
|
ENV_KEYS_UI_IN_SANDBOX = "POLYAXON_UI_IN_SANDBOX"
|
141
147
|
ENV_KEYS_UI_ADMIN_ENABLED = "POLYAXON_UI_ADMIN_ENABLED"
|
polyaxon/_flow/__init__.py
CHANGED
@@ -28,6 +28,7 @@ class V1Build(BaseSchemaModel):
|
|
28
28
|
hub_ref: str
|
29
29
|
connection: str
|
30
30
|
queue: str, optional
|
31
|
+
namespace: str, optional
|
31
32
|
presets: List[str], optional
|
32
33
|
cache: [V1Cache](/docs/automation/helpers/cache/), optional
|
33
34
|
params: Dict[str, [V1Param](/docs/core/specification/params/)], optional
|
@@ -94,6 +95,17 @@ class V1Build(BaseSchemaModel):
|
|
94
95
|
...
|
95
96
|
```
|
96
97
|
|
98
|
+
### presets
|
99
|
+
|
100
|
+
The [presets](/docs/management/organizations/presets/) to use for the hook operation,
|
101
|
+
if provided, it will override the component's presets otherwise
|
102
|
+
the presets of the component will be used if available.
|
103
|
+
|
104
|
+
```yaml
|
105
|
+
>>> build:
|
106
|
+
>>> presets: [test]
|
107
|
+
```
|
108
|
+
|
97
109
|
### queue
|
98
110
|
|
99
111
|
The [queue](/docs/core/scheduling-strategies/queues/) to use.
|
@@ -112,15 +124,15 @@ class V1Build(BaseSchemaModel):
|
|
112
124
|
>>> queue: queue-name
|
113
125
|
```
|
114
126
|
|
115
|
-
###
|
127
|
+
### namespace
|
116
128
|
|
117
|
-
|
118
|
-
|
119
|
-
|
129
|
+
> **Note**: Please note that this field is only available in some commercial editions.
|
130
|
+
|
131
|
+
The namespace to use, if not provided, it will default to the agent's namespace.
|
120
132
|
|
121
133
|
```yaml
|
122
134
|
>>> build:
|
123
|
-
>>>
|
135
|
+
>>> namespace: polyaxon
|
124
136
|
```
|
125
137
|
|
126
138
|
### cache
|
@@ -211,8 +223,9 @@ class V1Build(BaseSchemaModel):
|
|
211
223
|
_IDENTIFIER = "build"
|
212
224
|
hub_ref: StrictStr = Field(alias="hubRef")
|
213
225
|
connection: Optional[StrictStr]
|
214
|
-
queue: Optional[StrictStr]
|
215
226
|
presets: Optional[Union[List[StrictStr], RefField]]
|
227
|
+
queue: Optional[StrictStr]
|
228
|
+
namespace: Optional[StrictStr]
|
216
229
|
cache: Optional[Union[V1Cache, RefField]]
|
217
230
|
params: Optional[Dict[str, Union[V1Param, RefField]]]
|
218
231
|
run_patch: Optional[Dict[str, Any]] = Field(alias="runPatch")
|
polyaxon/_flow/component/base.py
CHANGED
@@ -20,6 +20,7 @@ class BaseComponent(BaseSchemaModel):
|
|
20
20
|
tags: Optional[List[StrictStr]]
|
21
21
|
presets: Optional[List[StrictStr]]
|
22
22
|
queue: Optional[StrictStr]
|
23
|
+
namespace: Optional[StrictStr]
|
23
24
|
cache: Optional[Union[V1Cache, RefField]]
|
24
25
|
termination: Optional[Union[V1Termination, RefField]]
|
25
26
|
plugins: Optional[Union[V1Plugins, RefField]]
|
@@ -39,6 +39,7 @@ class V1Component(
|
|
39
39
|
tags: List[str], optional
|
40
40
|
presets: List[str], optional
|
41
41
|
queue: str, optional
|
42
|
+
namespace: str, optional
|
42
43
|
cache: [V1Cache](/docs/automation/helpers/cache/), optional
|
43
44
|
termination: [V1Termination](/docs/core/specification/termination/), optional
|
44
45
|
plugins: [V1Plugins](/docs/core/specification/plugins/), optional
|
@@ -60,6 +61,7 @@ class V1Component(
|
|
60
61
|
>>> tags:
|
61
62
|
>>> presets:
|
62
63
|
>>> queue:
|
64
|
+
>>> namespace:
|
63
65
|
>>> cache:
|
64
66
|
>>> termination:
|
65
67
|
>>> plugins:
|
@@ -85,6 +87,7 @@ class V1Component(
|
|
85
87
|
>>> tags=["test"],
|
86
88
|
>>> presets=["test"],
|
87
89
|
>>> queue="test",
|
90
|
+
>>> namespace="test",
|
88
91
|
>>> cache=V1Cache(...),
|
89
92
|
>>> termination=V1Termination(...),
|
90
93
|
>>> plugins=V1Plugins(...),
|
@@ -197,6 +200,17 @@ class V1Component(
|
|
197
200
|
>>> queue: queue-name
|
198
201
|
```
|
199
202
|
|
203
|
+
### namespace
|
204
|
+
|
205
|
+
> **Note**: Please note that this field is only available in some commercial editions.
|
206
|
+
|
207
|
+
The namespace to use, if not provided, it will default to the agent's namespace.
|
208
|
+
|
209
|
+
```yaml
|
210
|
+
>>> component:
|
211
|
+
>>> namespace: polyaxon
|
212
|
+
```
|
213
|
+
|
200
214
|
### cache
|
201
215
|
|
202
216
|
The default component [cache](/docs/automation/helpers/cache/).
|
@@ -207,14 +207,14 @@ class V1Environment(BaseSchemaModel):
|
|
207
207
|
```yaml
|
208
208
|
>>> rules:
|
209
209
|
>>> - apiGroups: [""]
|
210
|
-
>>> resources: ["pods"]
|
210
|
+
>>> resources: ["pods", "services", "events", "pods/status", "pods/log"]
|
211
211
|
>>> verbs: ["get", "watch", "list"]
|
212
212
|
>>> - apiGroups: ["metrics.k8s.io"]
|
213
|
-
>>> resources: ["pods", "nodes"
|
213
|
+
>>> resources: ["pods", "nodes"]
|
214
214
|
>>> verbs: ["get", "list", "watch"]
|
215
|
-
>>> - apiGroups: [""
|
216
|
-
>>> resources: ["
|
217
|
-
>>> verbs: ["
|
215
|
+
>>> - apiGroups: ["core.polyaxon.com"]
|
216
|
+
>>> resources: ["operations"]
|
217
|
+
>>> verbs: ["get", "watch", "list"]
|
218
218
|
```
|
219
219
|
|
220
220
|
### hostAliases
|
@@ -408,9 +408,9 @@ class V1Environment(BaseSchemaModel):
|
|
408
408
|
scheduler_name: Optional[StrictStr] = Field(alias="schedulerName")
|
409
409
|
priority_class_name: Optional[StrictStr] = Field(alias="priorityClassName")
|
410
410
|
priority: Optional[int]
|
411
|
-
restart_policy: Optional[
|
412
|
-
|
413
|
-
)
|
411
|
+
restart_policy: Optional[
|
412
|
+
Literal["Always", "OnFailure", "Never", "ExitCode"]
|
413
|
+
] = Field(alias="restartPolicy")
|
414
414
|
|
415
415
|
@validator("affinity", always=True, pre=True)
|
416
416
|
def validate_affinity(cls, v):
|
polyaxon/_flow/hooks/__init__.py
CHANGED
@@ -49,6 +49,7 @@ class V1Hook(BaseSchemaModel):
|
|
49
49
|
hub_ref: str
|
50
50
|
conditions: str, optional
|
51
51
|
queue: str, optional
|
52
|
+
namespace: str, optional
|
52
53
|
presets: List[str], optional
|
53
54
|
disableDefaults: bool, optional
|
54
55
|
params: Dict[str, [V1Param](/docs/core/specification/params/)], optional
|
@@ -131,6 +132,17 @@ class V1Hook(BaseSchemaModel):
|
|
131
132
|
In the example above, the hook will only run if a param is passed, or an output is logged and
|
132
133
|
is equal to "some-value".
|
133
134
|
|
135
|
+
### presets
|
136
|
+
|
137
|
+
The [presets](/docs/management/organizations/presets/) to use for the hook operation,
|
138
|
+
if provided, it will override the component's presets otherwise
|
139
|
+
the presets of the component will be used if available.
|
140
|
+
|
141
|
+
```yaml
|
142
|
+
>>> hook:
|
143
|
+
>>> presets: [test]
|
144
|
+
```
|
145
|
+
|
134
146
|
### queue
|
135
147
|
|
136
148
|
The [queue](/docs/core/scheduling-strategies/queues/) to use.
|
@@ -149,15 +161,15 @@ class V1Hook(BaseSchemaModel):
|
|
149
161
|
>>> queue: queue-name
|
150
162
|
```
|
151
163
|
|
152
|
-
###
|
164
|
+
### namespace
|
153
165
|
|
154
|
-
|
155
|
-
|
156
|
-
|
166
|
+
> **Note**: Please note that this field is only available in some commercial editions.
|
167
|
+
|
168
|
+
The namespace to use, if not provided, it will default to the agent's namespace.
|
157
169
|
|
158
170
|
```yaml
|
159
171
|
>>> hook:
|
160
|
-
>>>
|
172
|
+
>>> namespace: polyaxon
|
161
173
|
```
|
162
174
|
|
163
175
|
### disableDefaults
|
@@ -199,8 +211,9 @@ class V1Hook(BaseSchemaModel):
|
|
199
211
|
connection: Optional[StrictStr]
|
200
212
|
trigger: Optional[V1Statuses]
|
201
213
|
conditions: Optional[StrictStr]
|
202
|
-
queue: Optional[StrictStr]
|
203
214
|
presets: Optional[List[StrictStr]]
|
215
|
+
queue: Optional[StrictStr]
|
216
|
+
namespace: Optional[StrictStr]
|
204
217
|
params: Optional[Union[Dict[str, V1Param], RefField]]
|
205
218
|
disable_defaults: Optional[BoolOrRef] = Field(alias="disableDefaults")
|
206
219
|
|
polyaxon/_flow/init/__init__.py
CHANGED
@@ -106,8 +106,8 @@ class V1Init(BaseSchemaModel):
|
|
106
106
|
|
107
107
|
```python
|
108
108
|
>>> from polyaxon.schemas import V1Component, V1Init, V1Job
|
109
|
-
>>> from polyaxon.
|
110
|
-
>>> from polyaxon
|
109
|
+
>>> from polyaxon.types import V1ArtifactsType, V1DockerfileType, V1GitType
|
110
|
+
>>> from polyaxon import k8s
|
111
111
|
>>> component = V1Component(
|
112
112
|
>>> run=V1Job(
|
113
113
|
>>> init=[
|
@@ -132,8 +132,8 @@ class V1Init(BaseSchemaModel):
|
|
132
132
|
>>> V1Init(
|
133
133
|
>>> connection="gcs-large-datasets",
|
134
134
|
>>> artifacts=V1ArtifactsType(dirs=["data"]),
|
135
|
-
>>> container=
|
136
|
-
>>> resources=
|
135
|
+
>>> container=k8s.V1Container(
|
136
|
+
>>> resources=k8s.V1ResourceRequirements(requests={"memory": "256Mi", "cpu": "500m"}), # noqa
|
137
137
|
>>> )
|
138
138
|
>>> ),
|
139
139
|
>>> V1Init(
|
@@ -163,14 +163,14 @@ class V1Init(BaseSchemaModel):
|
|
163
163
|
>>> )
|
164
164
|
>>> ),
|
165
165
|
>>> V1Init(
|
166
|
-
>>> container=
|
166
|
+
>>> container=k8s.V1Container(
|
167
167
|
>>> name="myapp-container",
|
168
168
|
>>> image="busybox:1.28",
|
169
169
|
>>> command=['sh', '-c', 'echo custom init container']
|
170
170
|
>>> )
|
171
171
|
>>> ),
|
172
172
|
>>> ],
|
173
|
-
>>> container=
|
173
|
+
>>> container=k8s.V1Container(...)
|
174
174
|
>>> )
|
175
175
|
>>> )
|
176
176
|
```
|
polyaxon/_flow/matrix/tuner.py
CHANGED
@@ -60,6 +60,17 @@ class V1Tuner(BaseSchemaModel):
|
|
60
60
|
...
|
61
61
|
```
|
62
62
|
|
63
|
+
### presets
|
64
|
+
|
65
|
+
The [presets](/docs/management/organizations/presets/) to use for the tuner operation,
|
66
|
+
if provided, it will override the component's presets otherwise
|
67
|
+
the presets of the component will be used if available.
|
68
|
+
|
69
|
+
```yaml
|
70
|
+
>>> tuner:
|
71
|
+
>>> presets: [test]
|
72
|
+
```
|
73
|
+
|
63
74
|
### queue
|
64
75
|
|
65
76
|
The [queue](/docs/core/scheduling-strategies/queues/) to use.
|
@@ -78,15 +89,15 @@ class V1Tuner(BaseSchemaModel):
|
|
78
89
|
>>> queue: queue-name
|
79
90
|
```
|
80
91
|
|
81
|
-
###
|
92
|
+
### namespace
|
82
93
|
|
83
|
-
|
84
|
-
|
85
|
-
|
94
|
+
> **Note**: Please note that this field is only available in some commercial editions.
|
95
|
+
|
96
|
+
The namespace to use, if not provided, it will default to the agent's namespace.
|
86
97
|
|
87
98
|
```yaml
|
88
99
|
>>> tuner:
|
89
|
-
>>>
|
100
|
+
>>> namespace: polyaxon
|
90
101
|
```
|
91
102
|
|
92
103
|
### params
|
@@ -108,6 +119,7 @@ class V1Tuner(BaseSchemaModel):
|
|
108
119
|
_IDENTIFIER = "tuner"
|
109
120
|
|
110
121
|
hub_ref: StrictStr = Field(alias="hubRef")
|
111
|
-
queue: Optional[StrictStr]
|
112
122
|
presets: Optional[Union[List[StrictStr], RefField]]
|
123
|
+
queue: Optional[StrictStr]
|
124
|
+
namespace: Optional[StrictStr]
|
113
125
|
params: Optional[Union[Dict[str, V1Param], RefField]]
|