polyaxon 2.0.0rc49__py3-none-any.whl → 2.4.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- polyaxon/_auxiliaries/cleaner.py +8 -3
- polyaxon/_auxiliaries/init.py +7 -2
- polyaxon/_auxiliaries/notifier.py +8 -2
- polyaxon/_auxiliaries/sidecar.py +30 -2
- polyaxon/_cli/artifacts.py +96 -11
- polyaxon/_cli/components.py +96 -11
- polyaxon/_cli/config.py +118 -22
- polyaxon/_cli/dashboard.py +15 -2
- polyaxon/_cli/init.py +1 -1
- polyaxon/_cli/models.py +96 -11
- polyaxon/_cli/operations.py +267 -90
- polyaxon/_cli/project_versions.py +139 -6
- polyaxon/_cli/projects.py +23 -9
- polyaxon/_cli/run.py +37 -9
- polyaxon/_cli/services/agent.py +2 -2
- polyaxon/_cli/services/clean_artifacts.py +1 -1
- polyaxon/_cli/services/sidecar.py +8 -1
- polyaxon/_client/client.py +17 -0
- polyaxon/_client/mixin.py +39 -0
- polyaxon/_client/project.py +218 -23
- polyaxon/_client/run.py +131 -33
- polyaxon/_compiler/contexts/contexts.py +2 -2
- polyaxon/_compiler/contexts/ray_job.py +4 -2
- polyaxon/_compiler/resolver/agent.py +12 -2
- polyaxon/_compiler/resolver/runtime.py +2 -2
- polyaxon/_contexts/paths.py +4 -7
- polyaxon/_deploy/operators/compose.py +1 -27
- polyaxon/_deploy/schemas/deployment.py +4 -1
- polyaxon/_deploy/schemas/intervals.py +0 -7
- polyaxon/_deploy/schemas/proxy.py +1 -0
- polyaxon/_deploy/schemas/service.py +11 -1
- polyaxon/_docker/converter/base/base.py +8 -0
- polyaxon/_docker/executor.py +10 -4
- polyaxon/_env_vars/getters/owner_entity.py +4 -2
- polyaxon/_env_vars/getters/project.py +4 -2
- polyaxon/_env_vars/getters/run.py +5 -2
- polyaxon/_env_vars/keys.py +7 -1
- polyaxon/_flow/__init__.py +2 -0
- polyaxon/_flow/builds/__init__.py +19 -6
- polyaxon/_flow/component/base.py +1 -0
- polyaxon/_flow/component/component.py +14 -0
- polyaxon/_flow/environment/__init__.py +8 -8
- polyaxon/_flow/hooks/__init__.py +19 -6
- polyaxon/_flow/init/__init__.py +6 -6
- polyaxon/_flow/matrix/iterative.py +0 -1
- polyaxon/_flow/matrix/tuner.py +18 -6
- polyaxon/_flow/operations/operation.py +44 -17
- polyaxon/_flow/plugins/__init__.py +6 -0
- polyaxon/_flow/run/__init__.py +2 -2
- polyaxon/_flow/run/dag.py +2 -2
- polyaxon/_flow/run/dask/dask.py +0 -1
- polyaxon/_flow/run/dask/replica.py +3 -3
- polyaxon/_flow/run/enums.py +5 -0
- polyaxon/_flow/run/job.py +4 -4
- polyaxon/_flow/run/kubeflow/mpi_job.py +1 -2
- polyaxon/_flow/run/kubeflow/mx_job.py +1 -2
- polyaxon/_flow/run/kubeflow/paddle_job.py +35 -4
- polyaxon/_flow/run/kubeflow/pytorch_job.py +51 -5
- polyaxon/_flow/run/kubeflow/replica.py +4 -4
- polyaxon/_flow/run/kubeflow/scheduling_policy.py +12 -0
- polyaxon/_flow/run/kubeflow/tf_job.py +3 -3
- polyaxon/_flow/run/kubeflow/xgboost_job.py +1 -2
- polyaxon/_flow/run/ray/ray.py +2 -3
- polyaxon/_flow/run/ray/replica.py +3 -3
- polyaxon/_flow/run/service.py +4 -4
- polyaxon/_fs/fs.py +7 -2
- polyaxon/_fs/utils.py +3 -2
- polyaxon/_k8s/converter/base/base.py +2 -1
- polyaxon/_k8s/converter/base/main.py +1 -0
- polyaxon/_k8s/converter/base/sidecar.py +16 -1
- polyaxon/_k8s/converter/common/accelerators.py +7 -4
- polyaxon/_k8s/converter/converters/job.py +1 -1
- polyaxon/_k8s/converter/converters/kubeflow/paddle_job.py +1 -0
- polyaxon/_k8s/converter/converters/kubeflow/pytroch_job.py +2 -0
- polyaxon/_k8s/converter/converters/kubeflow/tf_job.py +1 -0
- polyaxon/_k8s/converter/converters/ray_job.py +4 -2
- polyaxon/_k8s/custom_resources/dask_job.py +3 -0
- polyaxon/_k8s/custom_resources/kubeflow/common.py +4 -1
- polyaxon/_k8s/custom_resources/kubeflow/paddle_job.py +10 -1
- polyaxon/_k8s/custom_resources/kubeflow/pytorch_job.py +14 -1
- polyaxon/_k8s/custom_resources/kubeflow/tf_job.py +4 -0
- polyaxon/_k8s/custom_resources/ray_job.py +3 -0
- polyaxon/_k8s/custom_resources/setter.py +1 -1
- polyaxon/_k8s/executor/async_executor.py +2 -0
- polyaxon/_k8s/executor/base.py +23 -6
- polyaxon/_k8s/logging/async_monitor.py +150 -5
- polyaxon/_k8s/manager/async_manager.py +96 -23
- polyaxon/_k8s/manager/base.py +4 -0
- polyaxon/_k8s/manager/manager.py +282 -134
- polyaxon/_local_process/__init__.py +0 -0
- polyaxon/_local_process/agent.py +6 -0
- polyaxon/_local_process/converter/__init__.py +1 -0
- polyaxon/_local_process/converter/base/__init__.py +1 -0
- polyaxon/_local_process/converter/base/base.py +140 -0
- polyaxon/_local_process/converter/base/containers.py +69 -0
- polyaxon/_local_process/converter/base/env_vars.py +253 -0
- polyaxon/_local_process/converter/base/init.py +414 -0
- polyaxon/_local_process/converter/base/main.py +74 -0
- polyaxon/_local_process/converter/base/mounts.py +82 -0
- polyaxon/_local_process/converter/converters/__init__.py +8 -0
- polyaxon/_local_process/converter/converters/job.py +40 -0
- polyaxon/_local_process/converter/converters/service.py +41 -0
- polyaxon/_local_process/converter/mixins.py +38 -0
- polyaxon/_local_process/executor.py +132 -0
- polyaxon/_local_process/process_types.py +39 -0
- polyaxon/_managers/agent.py +2 -0
- polyaxon/_managers/home.py +2 -1
- polyaxon/_operations/tuner.py +1 -0
- polyaxon/_polyaxonfile/check.py +2 -0
- polyaxon/_polyaxonfile/manager/operations.py +3 -0
- polyaxon/_polyaxonfile/manager/workflows.py +2 -0
- polyaxon/_polyaxonfile/specs/compiled_operation.py +1 -0
- polyaxon/_polyaxonfile/specs/operation.py +1 -0
- polyaxon/_polyaxonfile/specs/sections.py +3 -0
- polyaxon/_pql/manager.py +1 -1
- polyaxon/_runner/agent/async_agent.py +97 -21
- polyaxon/_runner/agent/base_agent.py +27 -9
- polyaxon/_runner/agent/client.py +15 -1
- polyaxon/_runner/agent/sync_agent.py +85 -20
- polyaxon/_runner/converter/converter.py +6 -2
- polyaxon/_runner/executor.py +13 -7
- polyaxon/_schemas/agent.py +27 -1
- polyaxon/_schemas/client.py +30 -3
- polyaxon/_schemas/installation.py +4 -3
- polyaxon/_schemas/lifecycle.py +10 -5
- polyaxon/_schemas/log_handler.py +2 -3
- polyaxon/_schemas/types/artifacts.py +3 -3
- polyaxon/_schemas/types/dockerfile.py +3 -3
- polyaxon/_schemas/types/file.py +3 -3
- polyaxon/_schemas/types/git.py +3 -3
- polyaxon/_schemas/types/tensorboard.py +3 -3
- polyaxon/_sdk/api/agents_v1_api.py +1076 -73
- polyaxon/_sdk/api/organizations_v1_api.py +371 -10
- polyaxon/_sdk/api/project_dashboards_v1_api.py +12 -12
- polyaxon/_sdk/api/project_searches_v1_api.py +12 -12
- polyaxon/_sdk/api/projects_v1_api.py +221 -44
- polyaxon/_sdk/api/runs_v1_api.py +917 -445
- polyaxon/_sdk/api/service_accounts_v1_api.py +16 -16
- polyaxon/_sdk/api/teams_v1_api.py +2827 -375
- polyaxon/_sdk/api/users_v1_api.py +231 -55
- polyaxon/_sdk/async_client/api_client.py +4 -0
- polyaxon/_sdk/schemas/__init__.py +10 -2
- polyaxon/_sdk/schemas/v1_agent.py +2 -1
- polyaxon/_sdk/schemas/v1_agent_reconcile_body_request.py +14 -0
- polyaxon/_sdk/schemas/v1_artifact_tree.py +1 -1
- polyaxon/_sdk/schemas/v1_dashboard_spec.py +4 -0
- polyaxon/_sdk/schemas/v1_events_response.py +4 -0
- polyaxon/_sdk/schemas/v1_organization.py +1 -0
- polyaxon/_sdk/schemas/v1_preset.py +8 -0
- polyaxon/_sdk/schemas/v1_project.py +1 -0
- polyaxon/_sdk/schemas/v1_project_settings.py +4 -2
- polyaxon/_sdk/schemas/v1_run.py +2 -2
- polyaxon/_sdk/schemas/v1_run_edge_lineage.py +14 -0
- polyaxon/_sdk/schemas/v1_run_edges_graph.py +9 -0
- polyaxon/_sdk/schemas/v1_section_spec.py +7 -2
- polyaxon/_sdk/schemas/v1_settings_catalog.py +1 -0
- polyaxon/_sdk/schemas/v1_team.py +3 -0
- polyaxon/_sdk/schemas/v1_user.py +1 -2
- polyaxon/_sdk/schemas/v1_user_access.py +17 -0
- polyaxon/_services/values.py +1 -0
- polyaxon/_sidecar/container/__init__.py +39 -18
- polyaxon/_sidecar/container/monitors/__init__.py +1 -0
- polyaxon/_sidecar/container/monitors/logs.py +10 -13
- polyaxon/_sidecar/container/monitors/spec.py +24 -0
- polyaxon/_sidecar/ignore.py +0 -1
- polyaxon/_utils/fqn_utils.py +25 -2
- polyaxon/client.py +1 -1
- polyaxon/pkg.py +1 -1
- polyaxon/schemas.py +8 -1
- polyaxon/settings.py +6 -0
- {polyaxon-2.0.0rc49.dist-info → polyaxon-2.4.0rc1.dist-info}/METADATA +43 -43
- {polyaxon-2.0.0rc49.dist-info → polyaxon-2.4.0rc1.dist-info}/RECORD +176 -155
- {polyaxon-2.0.0rc49.dist-info → polyaxon-2.4.0rc1.dist-info}/WHEEL +1 -1
- polyaxon/_sdk/schemas/v1_project_user_access.py +0 -10
- {polyaxon-2.0.0rc49.dist-info → polyaxon-2.4.0rc1.dist-info}/LICENSE +0 -0
- {polyaxon-2.0.0rc49.dist-info → polyaxon-2.4.0rc1.dist-info}/entry_points.txt +0 -0
- {polyaxon-2.0.0rc49.dist-info → polyaxon-2.4.0rc1.dist-info}/top_level.txt +0 -0
polyaxon/_cli/operations.py
CHANGED
@@ -21,7 +21,11 @@ from clipped.utils.versions import compare_versions
|
|
21
21
|
from urllib3.exceptions import HTTPError
|
22
22
|
|
23
23
|
from polyaxon import settings
|
24
|
-
from polyaxon._cli.dashboard import
|
24
|
+
from polyaxon._cli.dashboard import (
|
25
|
+
get_dashboard,
|
26
|
+
get_dashboard_url,
|
27
|
+
get_project_subpath_url,
|
28
|
+
)
|
25
29
|
from polyaxon._cli.errors import handle_cli_error, is_in_ce
|
26
30
|
from polyaxon._cli.options import (
|
27
31
|
OPTIONS_NAME,
|
@@ -47,7 +51,7 @@ from polyaxon.api import (
|
|
47
51
|
REWRITE_SERVICES_V1,
|
48
52
|
SERVICES_V1,
|
49
53
|
)
|
50
|
-
from polyaxon.client import RunClient, V1Run, get_run_logs
|
54
|
+
from polyaxon.client import RunClient, V1Run, V1RunSettings, get_run_logs
|
51
55
|
from polyaxon.exceptions import (
|
52
56
|
ApiException,
|
53
57
|
PolyaxonClientException,
|
@@ -82,6 +86,14 @@ DEFAULT_EXCLUDE = [
|
|
82
86
|
]
|
83
87
|
|
84
88
|
|
89
|
+
def get_op_agent_host(runSettings: V1RunSettings):
|
90
|
+
host_kwargs = {}
|
91
|
+
if runSettings and runSettings.agent and runSettings.agent.url:
|
92
|
+
host_kwargs["host"] = runSettings.agent.url
|
93
|
+
|
94
|
+
return host_kwargs
|
95
|
+
|
96
|
+
|
85
97
|
def handle_run_statuses(status, conditions, table):
|
86
98
|
if not conditions:
|
87
99
|
return False
|
@@ -286,7 +298,7 @@ def ls(
|
|
286
298
|
else:
|
287
299
|
Printer.warning(f"Skipping run {uid}, offline data not found.")
|
288
300
|
else:
|
289
|
-
owner, project_name = get_project_or_local(
|
301
|
+
owner, _, project_name = get_project_or_local(
|
290
302
|
project or ctx.obj.get("project"), is_cli=True
|
291
303
|
)
|
292
304
|
|
@@ -431,7 +443,7 @@ def get(ctx, project, uid, offline, path, output):
|
|
431
443
|
sys.exit(1)
|
432
444
|
run_data = RunConfigManager.read_from_path(offline_path)
|
433
445
|
else:
|
434
|
-
owner, project_name, run_uuid = get_project_run_or_local(
|
446
|
+
owner, team, project_name, run_uuid = get_project_run_or_local(
|
435
447
|
project or ctx.obj.get("project"),
|
436
448
|
uid,
|
437
449
|
is_cli=True,
|
@@ -456,7 +468,9 @@ def get(ctx, project, uid, offline, path, output):
|
|
456
468
|
)
|
457
469
|
if output:
|
458
470
|
run_url = get_dashboard_url(
|
459
|
-
subpath="{}/
|
471
|
+
subpath="{}/runs/{}".format(
|
472
|
+
get_project_subpath_url(owner, team, project_name), run_uuid
|
473
|
+
)
|
460
474
|
)
|
461
475
|
config["url"] = run_url
|
462
476
|
handle_output(config, output)
|
@@ -529,7 +543,7 @@ def delete(ctx, project, uid, yes, offline, path):
|
|
529
543
|
return
|
530
544
|
|
531
545
|
# Resume normal flow
|
532
|
-
owner, project_name, run_uuid = get_project_run_or_local(
|
546
|
+
owner, _, project_name, run_uuid = get_project_run_or_local(
|
533
547
|
project or ctx.obj.get("project"),
|
534
548
|
uid or ctx.obj.get("run_uuid"),
|
535
549
|
is_cli=True,
|
@@ -624,7 +638,7 @@ def update(ctx, project, uid, name, description, tags, offline, path):
|
|
624
638
|
)
|
625
639
|
sys.exit(1)
|
626
640
|
else:
|
627
|
-
owner, project_name, run_uuid = get_project_run_or_local(
|
641
|
+
owner, _, project_name, run_uuid = get_project_run_or_local(
|
628
642
|
project or ctx.obj.get("project"),
|
629
643
|
uid or ctx.obj.get("run_uuid"),
|
630
644
|
is_cli=True,
|
@@ -663,7 +677,7 @@ def approve(ctx, project, uid):
|
|
663
677
|
\b
|
664
678
|
$ polyaxon ops approve --uid 8aac02e3a62a4f0aaa257c59da5eab80
|
665
679
|
"""
|
666
|
-
owner, project_name, run_uuid = get_project_run_or_local(
|
680
|
+
owner, _, project_name, run_uuid = get_project_run_or_local(
|
667
681
|
project or ctx.obj.get("project"),
|
668
682
|
uid or ctx.obj.get("run_uuid"),
|
669
683
|
is_cli=True,
|
@@ -710,7 +724,7 @@ def stop(ctx, project, uid, yes):
|
|
710
724
|
\b
|
711
725
|
$ polyaxon ops stop --uid 8aac02e3a62a4f0aaa257c59da5eab80
|
712
726
|
"""
|
713
|
-
owner, project_name, run_uuid = get_project_run_or_local(
|
727
|
+
owner, _, project_name, run_uuid = get_project_run_or_local(
|
714
728
|
project or ctx.obj.get("project"),
|
715
729
|
uid or ctx.obj.get("run_uuid"),
|
716
730
|
is_cli=True,
|
@@ -736,6 +750,58 @@ def stop(ctx, project, uid, yes):
|
|
736
750
|
Printer.success("Run `{}` is being stopped ...".format(run_uuid))
|
737
751
|
|
738
752
|
|
753
|
+
@ops.command()
|
754
|
+
@click.option(*OPTIONS_PROJECT["args"], **OPTIONS_PROJECT["kwargs"])
|
755
|
+
@click.option(*OPTIONS_RUN_UID["args"], **OPTIONS_RUN_UID["kwargs"])
|
756
|
+
@click.option(
|
757
|
+
"--yes",
|
758
|
+
"-y",
|
759
|
+
is_flag=True,
|
760
|
+
default=False,
|
761
|
+
help="Automatic yes to prompts. "
|
762
|
+
'Assume "yes" as answer to all prompts and run non-interactively.',
|
763
|
+
)
|
764
|
+
@click.pass_context
|
765
|
+
@clean_outputs
|
766
|
+
def skip(ctx, project, uid, yes):
|
767
|
+
"""Skip run.
|
768
|
+
|
769
|
+
Uses /docs/core/cli/#caching
|
770
|
+
|
771
|
+
Examples:
|
772
|
+
|
773
|
+
\b
|
774
|
+
$ polyaxon ops skip
|
775
|
+
|
776
|
+
\b
|
777
|
+
$ polyaxon ops skip --uid 8aac02e3a62a4f0aaa257c59da5eab80
|
778
|
+
"""
|
779
|
+
owner, _, project_name, run_uuid = get_project_run_or_local(
|
780
|
+
project or ctx.obj.get("project"),
|
781
|
+
uid or ctx.obj.get("run_uuid"),
|
782
|
+
is_cli=True,
|
783
|
+
)
|
784
|
+
if not yes and not click.confirm(
|
785
|
+
"Are sure you want to stop " "run `{}`".format(run_uuid)
|
786
|
+
):
|
787
|
+
Printer.print("Exiting without stopping run.")
|
788
|
+
sys.exit(0)
|
789
|
+
|
790
|
+
try:
|
791
|
+
polyaxon_client = RunClient(
|
792
|
+
owner=owner,
|
793
|
+
project=project_name,
|
794
|
+
run_uuid=run_uuid,
|
795
|
+
manual_exceptions_handling=True,
|
796
|
+
)
|
797
|
+
polyaxon_client.skip()
|
798
|
+
except (ApiException, HTTPError) as e:
|
799
|
+
handle_cli_error(e, message="Could not skip run `{}`.".format(run_uuid))
|
800
|
+
sys.exit(1)
|
801
|
+
|
802
|
+
Printer.success("Run `{}` is skipped.".format(run_uuid))
|
803
|
+
|
804
|
+
|
739
805
|
@ops.command()
|
740
806
|
@click.option(*OPTIONS_PROJECT["args"], **OPTIONS_PROJECT["kwargs"])
|
741
807
|
@click.option(*OPTIONS_RUN_UID["args"], **OPTIONS_RUN_UID["kwargs"])
|
@@ -814,7 +880,7 @@ def restart(
|
|
814
880
|
if polyaxonfile:
|
815
881
|
content = OperationSpecification.read(polyaxonfile, is_preset=True).to_json()
|
816
882
|
|
817
|
-
owner, project_name, run_uuid = get_project_run_or_local(
|
883
|
+
owner, _, project_name, run_uuid = get_project_run_or_local(
|
818
884
|
project or ctx.obj.get("project"),
|
819
885
|
uid or ctx.obj.get("run_uuid"),
|
820
886
|
is_cli=True,
|
@@ -899,7 +965,7 @@ def resume(
|
|
899
965
|
if polyaxonfile:
|
900
966
|
content = OperationSpecification.read(polyaxonfile, is_preset=True).to_json()
|
901
967
|
|
902
|
-
owner, project_name, run_uuid = get_project_run_or_local(
|
968
|
+
owner, _, project_name, run_uuid = get_project_run_or_local(
|
903
969
|
project or ctx.obj.get("project"),
|
904
970
|
uid or ctx.obj.get("run_uuid"),
|
905
971
|
is_cli=True,
|
@@ -942,7 +1008,7 @@ def invalidate(ctx, project, uid):
|
|
942
1008
|
\b
|
943
1009
|
$ polyaxon ops invalidate --uid 8aac02e3a62a4f0aaa257c59da5eab80
|
944
1010
|
"""
|
945
|
-
owner, project_name, run_uuid = get_project_run_or_local(
|
1011
|
+
owner, _, project_name, run_uuid = get_project_run_or_local(
|
946
1012
|
project or ctx.obj.get("project"),
|
947
1013
|
uid or ctx.obj.get("run_uuid"),
|
948
1014
|
is_cli=True,
|
@@ -968,7 +1034,7 @@ def invalidate(ctx, project, uid):
|
|
968
1034
|
"--executor",
|
969
1035
|
"-exc",
|
970
1036
|
type=str,
|
971
|
-
help="The
|
1037
|
+
help="The executor to use, possible values are: docker, k8s, process.",
|
972
1038
|
)
|
973
1039
|
@click.pass_context
|
974
1040
|
@clean_outputs
|
@@ -986,50 +1052,74 @@ def execute(ctx, project, uid, executor):
|
|
986
1052
|
$ polyaxon ops execute -uid 8aac02e3a62a4f0aaa257c59da5eab80
|
987
1053
|
"""
|
988
1054
|
|
989
|
-
|
1055
|
+
Printer.warning(
|
1056
|
+
"The `ops execute` command is experimental and might change in the future!"
|
1057
|
+
)
|
1058
|
+
|
1059
|
+
def _execute_on_docker(response: V1Run, run_client: RunClient):
|
990
1060
|
from polyaxon._docker.executor import Executor
|
991
1061
|
|
992
|
-
|
1062
|
+
run_client.log_status(
|
993
1063
|
V1Statuses.STARTING,
|
994
1064
|
reason="CliDockerExecutor",
|
995
1065
|
message="Operation is starting",
|
996
1066
|
)
|
997
1067
|
executor = Executor()
|
998
1068
|
if not executor.manager.check():
|
999
|
-
|
1069
|
+
run_client.log_failed(
|
1000
1070
|
reason="CliDockerExecutor",
|
1001
1071
|
message="Docker is required to run this operation.",
|
1002
1072
|
)
|
1003
1073
|
raise Printer.error(
|
1004
1074
|
"Docker is required to run this command.", sys_exit=True
|
1005
1075
|
)
|
1006
|
-
|
1076
|
+
run_client.log_status(
|
1007
1077
|
V1Statuses.RUNNING,
|
1008
|
-
reason="
|
1078
|
+
reason="CliDockerExecutor",
|
1009
1079
|
message="Operation is running",
|
1010
1080
|
)
|
1011
|
-
|
1081
|
+
if V1RunKind.has_pipeline(response.kind):
|
1082
|
+
has_children = True
|
1083
|
+
while has_children:
|
1084
|
+
pipeline_runs = run_client.list_children(
|
1085
|
+
query="status:created", sort="created_at", limit=1
|
1086
|
+
)
|
1087
|
+
if pipeline_runs.results:
|
1088
|
+
current_run_client = RunClient(
|
1089
|
+
owner=owner,
|
1090
|
+
project=project_name,
|
1091
|
+
run_uuid=pipeline_runs.results[0].uuid,
|
1092
|
+
manual_exceptions_handling=True,
|
1093
|
+
)
|
1094
|
+
current_run_client.refresh_data()
|
1095
|
+
_prepare(current_run_client.run_data, current_run_client)
|
1096
|
+
_execute(current_run_client.run_data, current_run_client)
|
1097
|
+
else:
|
1098
|
+
has_children = False
|
1099
|
+
return
|
1100
|
+
else:
|
1101
|
+
result = executor.create_from_run(response, default_auth=not is_in_ce())
|
1012
1102
|
if result["status"] == V1Statuses.SUCCEEDED:
|
1013
|
-
|
1103
|
+
run_client.log_succeeded(
|
1014
1104
|
reason="CliDockerExecutor", message="Operation was succeeded"
|
1015
1105
|
)
|
1016
1106
|
else:
|
1017
|
-
|
1107
|
+
run_client.log_failed(
|
1018
1108
|
reason="CliDockerExecutor",
|
1019
|
-
message="Operation
|
1109
|
+
message="Operation failed.\n{}".format(result["message"]),
|
1020
1110
|
)
|
1021
1111
|
|
1022
|
-
def
|
1112
|
+
def _execute_on_k8s(response: V1Run, run_client: RunClient):
|
1023
1113
|
from polyaxon._k8s.executor.executor import Executor
|
1024
1114
|
|
1025
|
-
|
1115
|
+
run_client.log_status(
|
1026
1116
|
V1Statuses.STARTING,
|
1027
1117
|
reason="CliK8SExecutor",
|
1028
1118
|
message="Operation is starting",
|
1029
1119
|
)
|
1030
1120
|
if not settings.AGENT_CONFIG.namespace:
|
1031
|
-
|
1032
|
-
reason="
|
1121
|
+
run_client.log_failed(
|
1122
|
+
reason="CliK8SExecutor",
|
1033
1123
|
message="an agent config with defined namespace is required.",
|
1034
1124
|
)
|
1035
1125
|
raise Printer.error(
|
@@ -1038,51 +1128,76 @@ def execute(ctx, project, uid, executor):
|
|
1038
1128
|
|
1039
1129
|
executor = Executor(namespace=settings.AGENT_CONFIG.namespace)
|
1040
1130
|
if not executor.manager.get_version():
|
1041
|
-
|
1042
|
-
reason="
|
1131
|
+
run_client.log_failed(
|
1132
|
+
reason="CliK8SExecutor",
|
1043
1133
|
message="Kubernetes is required to run this operation.",
|
1044
1134
|
)
|
1045
1135
|
raise Printer.error(
|
1046
1136
|
"Kubernetes is required to run this command.", sys_exit=True
|
1047
1137
|
)
|
1048
|
-
|
1138
|
+
run_client.log_status(
|
1049
1139
|
V1Statuses.RUNNING,
|
1050
1140
|
reason="CliK8SExecutor",
|
1051
1141
|
message="Operation is running",
|
1052
1142
|
)
|
1053
1143
|
executor.create_from_run(response, default_auth=not is_in_ce())
|
1054
1144
|
|
1055
|
-
def _execute_on_local_process(response: V1Run):
|
1056
|
-
from polyaxon.
|
1145
|
+
def _execute_on_local_process(response: V1Run, run_client: RunClient):
|
1146
|
+
from polyaxon._local_process.executor import Executor
|
1057
1147
|
|
1058
|
-
|
1148
|
+
run_client.log_status(
|
1059
1149
|
V1Statuses.STARTING,
|
1060
1150
|
reason="CliProcessExecutor",
|
1061
1151
|
message="Operation is starting",
|
1062
1152
|
)
|
1063
1153
|
executor = Executor()
|
1064
1154
|
if not executor.manager.check():
|
1065
|
-
|
1066
|
-
|
1067
|
-
|
1068
|
-
|
1069
|
-
|
1070
|
-
|
1071
|
-
|
1072
|
-
|
1073
|
-
|
1074
|
-
|
1075
|
-
|
1076
|
-
|
1077
|
-
|
1078
|
-
|
1079
|
-
|
1080
|
-
|
1081
|
-
|
1082
|
-
|
1083
|
-
|
1084
|
-
|
1085
|
-
|
1155
|
+
run_client.log_failed(
|
1156
|
+
reason="CliProcessExecutor",
|
1157
|
+
message="Local process can't be used to run this operation.",
|
1158
|
+
)
|
1159
|
+
raise Printer.error(
|
1160
|
+
"Local process can't be used to run this operation.", sys_exit=True
|
1161
|
+
)
|
1162
|
+
|
1163
|
+
run_client.log_status(
|
1164
|
+
V1Statuses.RUNNING,
|
1165
|
+
reason="CliProcessExecutor",
|
1166
|
+
message="Operation is running",
|
1167
|
+
)
|
1168
|
+
|
1169
|
+
if V1RunKind.has_pipeline(response.kind):
|
1170
|
+
has_children = True
|
1171
|
+
while has_children:
|
1172
|
+
pipeline_runs = run_client.list_children(
|
1173
|
+
query="status:created", sort="created_at", limit=1
|
1174
|
+
)
|
1175
|
+
if pipeline_runs.results:
|
1176
|
+
current_run_client = RunClient(
|
1177
|
+
owner=owner,
|
1178
|
+
project=project_name,
|
1179
|
+
run_uuid=pipeline_runs.results[0].uuid,
|
1180
|
+
manual_exceptions_handling=True,
|
1181
|
+
)
|
1182
|
+
current_run_client.refresh_data()
|
1183
|
+
_prepare(current_run_client.run_data, current_run_client)
|
1184
|
+
_execute(current_run_client.run_data, current_run_client)
|
1185
|
+
else:
|
1186
|
+
has_children = False
|
1187
|
+
return
|
1188
|
+
else:
|
1189
|
+
result = executor.create_from_run(response, default_auth=not is_in_ce())
|
1190
|
+
if result["status"] == V1Statuses.SUCCEEDED:
|
1191
|
+
run_client.log_succeeded(
|
1192
|
+
reason="CliProcessExecutor", message="Operation was succeeded"
|
1193
|
+
)
|
1194
|
+
else:
|
1195
|
+
run_client.log_failed(
|
1196
|
+
reason="CliProcessExecutor",
|
1197
|
+
message="Operation failed.\n{}".format(result["message"]),
|
1198
|
+
)
|
1199
|
+
|
1200
|
+
owner, _, project_name, run_uuid = get_project_run_or_local(
|
1086
1201
|
project or ctx.obj.get("project"),
|
1087
1202
|
uid or ctx.obj.get("run_uuid"),
|
1088
1203
|
is_cli=True,
|
@@ -1114,22 +1229,48 @@ def execute(ctx, project, uid, executor):
|
|
1114
1229
|
)
|
1115
1230
|
settings.AGENT_CONFIG.set_default_artifacts_store()
|
1116
1231
|
|
1117
|
-
|
1118
|
-
V1Statuses.
|
1119
|
-
|
1120
|
-
|
1121
|
-
|
1122
|
-
|
1123
|
-
|
1124
|
-
|
1125
|
-
|
1126
|
-
|
1127
|
-
|
1128
|
-
|
1232
|
+
def _prepare(response: V1Run, client: RunClient):
|
1233
|
+
if response.status == V1Statuses.CREATED:
|
1234
|
+
try:
|
1235
|
+
client.approve()
|
1236
|
+
client.refresh_data()
|
1237
|
+
except (ApiException, HTTPError) as e:
|
1238
|
+
handle_cli_error(
|
1239
|
+
e, message="Could not approve run `{}`.".format(response.uuid)
|
1240
|
+
)
|
1241
|
+
sys.exit(1)
|
1242
|
+
|
1243
|
+
def _execute(response: V1Run, client: RunClient):
|
1244
|
+
if LifeCycle.is_done(response.status):
|
1245
|
+
return
|
1246
|
+
if response.status != V1Statuses.COMPILED:
|
1247
|
+
Printer.error("Run must be compiled.", sys_exit=True)
|
1248
|
+
client.log_status(
|
1249
|
+
V1Statuses.SCHEDULED,
|
1250
|
+
reason="CliExecutor",
|
1251
|
+
message="Operation is pulled by CLI.",
|
1252
|
+
)
|
1253
|
+
if executor == RunnerKind.DOCKER:
|
1254
|
+
_execute_on_docker(response, client)
|
1255
|
+
elif executor == RunnerKind.K8S:
|
1256
|
+
_execute_on_k8s(response, client)
|
1257
|
+
elif executor == RunnerKind.PROCESS:
|
1258
|
+
_execute_on_local_process(response, client)
|
1259
|
+
else:
|
1260
|
+
client.log_failed(
|
1261
|
+
reason="CliExecutor",
|
1262
|
+
message="Local executor is not supported {}.".format(executor),
|
1263
|
+
)
|
1264
|
+
|
1265
|
+
try:
|
1266
|
+
_prepare(polyaxon_client.run_data, polyaxon_client)
|
1267
|
+
_execute(polyaxon_client.run_data, polyaxon_client)
|
1268
|
+
except KeyboardInterrupt:
|
1129
1269
|
polyaxon_client.log_failed(
|
1130
1270
|
reason="CliExecutor",
|
1131
|
-
message="
|
1271
|
+
message="Operation was canceled with `KeyboardInterrupt`.",
|
1132
1272
|
)
|
1273
|
+
sys.exit(1)
|
1133
1274
|
|
1134
1275
|
|
1135
1276
|
@ops.command()
|
@@ -1180,7 +1321,7 @@ def statuses(ctx, project, uid, watch, offline, path):
|
|
1180
1321
|
sys.exit(1)
|
1181
1322
|
return
|
1182
1323
|
|
1183
|
-
owner, project_name, run_uuid = get_project_run_or_local(
|
1324
|
+
owner, _, project_name, run_uuid = get_project_run_or_local(
|
1184
1325
|
project or ctx.obj.get("project"),
|
1185
1326
|
uid or ctx.obj.get("run_uuid"),
|
1186
1327
|
is_cli=True,
|
@@ -1248,7 +1389,7 @@ def statuses(ctx, project, uid, watch, offline, path):
|
|
1248
1389
|
# )
|
1249
1390
|
# sys.exit(1)
|
1250
1391
|
#
|
1251
|
-
# owner, project_name, run_uuid = get_project_run_or_local(
|
1392
|
+
# owner, team, project_name, run_uuid = get_project_run_or_local(
|
1252
1393
|
# ctx.obj.get("project"), ctx.obj.get("run_uuid"), is_cli=True,
|
1253
1394
|
# )
|
1254
1395
|
#
|
@@ -1340,7 +1481,7 @@ def logs(ctx, project, uid, follow, hide_time, all_containers, all_info, offline
|
|
1340
1481
|
sys.exit(1)
|
1341
1482
|
return
|
1342
1483
|
|
1343
|
-
owner, project_name, run_uuid = get_project_run_or_local(
|
1484
|
+
owner, _, project_name, run_uuid = get_project_run_or_local(
|
1344
1485
|
project or ctx.obj.get("project"),
|
1345
1486
|
uid or ctx.obj.get("run_uuid"),
|
1346
1487
|
is_cli=True,
|
@@ -1386,7 +1527,7 @@ def inspect(ctx, project, uid):
|
|
1386
1527
|
\b
|
1387
1528
|
$ polyaxon ops inspect -p acme/project -uid 8aac02e3a62a4f0aaa257c59da5eab80
|
1388
1529
|
"""
|
1389
|
-
owner, project_name, run_uuid = get_project_run_or_local(
|
1530
|
+
owner, _, project_name, run_uuid = get_project_run_or_local(
|
1390
1531
|
project or ctx.obj.get("project"),
|
1391
1532
|
uid or ctx.obj.get("run_uuid"),
|
1392
1533
|
is_cli=True,
|
@@ -1445,7 +1586,7 @@ def shell(ctx, project, uid, command, pod, container):
|
|
1445
1586
|
"""
|
1446
1587
|
from polyaxon._vendor.shell_pty import PseudoTerminal
|
1447
1588
|
|
1448
|
-
owner, project_name, run_uuid = get_project_run_or_local(
|
1589
|
+
owner, _, project_name, run_uuid = get_project_run_or_local(
|
1449
1590
|
project or ctx.obj.get("project"),
|
1450
1591
|
uid or ctx.obj.get("run_uuid"),
|
1451
1592
|
is_cli=True,
|
@@ -1550,7 +1691,7 @@ def artifacts(
|
|
1550
1691
|
\b
|
1551
1692
|
$ polyaxon ops artifacts -uid 8aac02e3a62a4f0aaa257c59da5eab80 -l-kind model -l-kind env --path="this/path"
|
1552
1693
|
"""
|
1553
|
-
owner, project_name, run_uuid = get_project_run_or_local(
|
1694
|
+
owner, _, project_name, run_uuid = get_project_run_or_local(
|
1554
1695
|
project or ctx.obj.get("project"),
|
1555
1696
|
uid or ctx.obj.get("run_uuid"),
|
1556
1697
|
is_cli=True,
|
@@ -1710,9 +1851,12 @@ def artifacts(
|
|
1710
1851
|
default=False,
|
1711
1852
|
help="To set the run to failed if the upload fails.",
|
1712
1853
|
)
|
1854
|
+
@click.option(
|
1855
|
+
"--agent", "-ag", type=str, help="Optional, uuid reference of an agent to use."
|
1856
|
+
)
|
1713
1857
|
@click.pass_context
|
1714
1858
|
@clean_outputs
|
1715
|
-
def upload(ctx, project, uid, path_from, path_to, sync_failure):
|
1859
|
+
def upload(ctx, project, uid, path_from, path_to, sync_failure, agent):
|
1716
1860
|
"""Upload runs' artifacts.
|
1717
1861
|
|
1718
1862
|
Uses /docs/core/cli/#caching
|
@@ -1728,7 +1872,7 @@ def upload(ctx, project, uid, path_from, path_to, sync_failure):
|
|
1728
1872
|
\b
|
1729
1873
|
$ polyaxon ops upload -uid 8aac02e3a62a4f0aaa257c59da5eab80 --path-to="path/to/upload/to"
|
1730
1874
|
"""
|
1731
|
-
owner, project_name, run_uuid = get_project_run_or_local(
|
1875
|
+
owner, _, project_name, run_uuid = get_project_run_or_local(
|
1732
1876
|
project or ctx.obj.get("project"),
|
1733
1877
|
uid or ctx.obj.get("run_uuid"),
|
1734
1878
|
is_cli=True,
|
@@ -1743,7 +1887,7 @@ def upload(ctx, project, uid, path_from, path_to, sync_failure):
|
|
1743
1887
|
)
|
1744
1888
|
if is_file:
|
1745
1889
|
response = client.upload_artifact(
|
1746
|
-
filepath=path_from, path=path_to, overwrite=True
|
1890
|
+
filepath=path_from, path=path_to, overwrite=True, agent=agent
|
1747
1891
|
)
|
1748
1892
|
else:
|
1749
1893
|
response = client.upload_artifacts_dir(
|
@@ -1751,6 +1895,7 @@ def upload(ctx, project, uid, path_from, path_to, sync_failure):
|
|
1751
1895
|
path=path_to,
|
1752
1896
|
overwrite=True,
|
1753
1897
|
relative_to=path_from,
|
1898
|
+
agent=agent,
|
1754
1899
|
)
|
1755
1900
|
except (
|
1756
1901
|
ApiException,
|
@@ -1802,7 +1947,7 @@ def transfer(ctx, project, uid, to_project):
|
|
1802
1947
|
\b
|
1803
1948
|
$ polyaxon ops transfer -p acme/foobar -uid 8aac02e3a62a4f0aaa257c59da5eab80 -to=dest-project
|
1804
1949
|
"""
|
1805
|
-
owner, project_name, run_uuid = get_project_run_or_local(
|
1950
|
+
owner, _, project_name, run_uuid = get_project_run_or_local(
|
1806
1951
|
project or ctx.obj.get("project"),
|
1807
1952
|
uid or ctx.obj.get("run_uuid"),
|
1808
1953
|
is_cli=True,
|
@@ -1870,19 +2015,22 @@ def dashboard(ctx, project, uid, yes, url, offline, path, server_config):
|
|
1870
2015
|
sys.exit(1)
|
1871
2016
|
run_data = RunConfigManager.read_from_path(offline_path)
|
1872
2017
|
owner, project_name, run_uuid = run_data.owner, run_data.project, run_data.uuid
|
2018
|
+
team = None
|
1873
2019
|
else:
|
1874
|
-
owner, project_name, run_uuid = get_project_run_or_local(
|
2020
|
+
owner, team, project_name, run_uuid = get_project_run_or_local(
|
1875
2021
|
project or ctx.obj.get("project"),
|
1876
2022
|
uid or ctx.obj.get("run_uuid"),
|
1877
2023
|
is_cli=True,
|
1878
2024
|
)
|
1879
|
-
subpath = "{}/
|
1880
|
-
|
1881
|
-
dashboard_url=get_dashboard_url(subpath=subpath), url_only=url, yes=yes
|
2025
|
+
subpath = "{}/runs/{}".format(
|
2026
|
+
get_project_subpath_url(owner, team, project_name), run_uuid
|
1882
2027
|
)
|
2028
|
+
dashboard_url = get_dashboard_url(subpath=subpath)
|
2029
|
+
get_dashboard(dashboard_url=dashboard_url, url_only=url, yes=yes)
|
1883
2030
|
if offline:
|
1884
2031
|
from haupt.cli.viewer import sanitize_server_config, viewer
|
1885
2032
|
|
2033
|
+
os.environ["POLYAXON_DASHBOARD_URL"] = dashboard_url
|
1886
2034
|
server_config = sanitize_server_config(server_config)
|
1887
2035
|
ctx.invoke(viewer, path=path, **server_config)
|
1888
2036
|
|
@@ -1920,7 +2068,7 @@ def service(ctx, project, uid, yes, external, url):
|
|
1920
2068
|
You can open the service embedded in Polyaxon UI or using the real service URL,
|
1921
2069
|
please use the `--external` flag.
|
1922
2070
|
"""
|
1923
|
-
owner, project_name, run_uuid = get_project_run_or_local(
|
2071
|
+
owner, team, project_name, run_uuid = get_project_run_or_local(
|
1924
2072
|
project or ctx.obj.get("project"),
|
1925
2073
|
uid or ctx.obj.get("run_uuid"),
|
1926
2074
|
is_cli=True,
|
@@ -1953,8 +2101,12 @@ def service(ctx, project, uid, yes, external, url):
|
|
1953
2101
|
|
1954
2102
|
wait_for_running_condition(client)
|
1955
2103
|
|
2104
|
+
host_kwargs = get_op_agent_host(client.settings)
|
1956
2105
|
run_url = get_dashboard_url(
|
1957
|
-
subpath="{}/
|
2106
|
+
subpath="{}/runs/{}/service".format(
|
2107
|
+
get_project_subpath_url(owner, team, project_name), run_uuid
|
2108
|
+
),
|
2109
|
+
**host_kwargs,
|
1958
2110
|
)
|
1959
2111
|
|
1960
2112
|
namespace = client.run_data.settings.namespace
|
@@ -1964,8 +2116,8 @@ def service(ctx, project, uid, yes, external, url):
|
|
1964
2116
|
if rewrite_path:
|
1965
2117
|
service_endpoint = REWRITE_EXTERNAL_V1 if is_external else REWRITE_SERVICES_V1
|
1966
2118
|
|
1967
|
-
service_subpath = "{}/{}/
|
1968
|
-
namespace, owner, project_name, run_uuid
|
2119
|
+
service_subpath = "{}/{}/runs/{}/".format(
|
2120
|
+
get_project_subpath_url(namespace, team, owner), project_name, run_uuid
|
1969
2121
|
)
|
1970
2122
|
port = 80
|
1971
2123
|
if client.settings and client.settings.agent:
|
@@ -1978,9 +2130,9 @@ def service(ctx, project, uid, yes, external, url):
|
|
1978
2130
|
if port:
|
1979
2131
|
service_subpath = "{}{}/".format(service_subpath, port)
|
1980
2132
|
|
2133
|
+
host_kwargs = get_op_agent_host(client.settings)
|
1981
2134
|
external_run_url = get_dashboard_url(
|
1982
|
-
base=service_endpoint,
|
1983
|
-
subpath=service_subpath,
|
2135
|
+
base=service_endpoint, subpath=service_subpath, **host_kwargs
|
1984
2136
|
)
|
1985
2137
|
|
1986
2138
|
if url:
|
@@ -2061,7 +2213,7 @@ def pull(
|
|
2061
2213
|
\b
|
2062
2214
|
$ polyaxon ops pull -a
|
2063
2215
|
"""
|
2064
|
-
owner, project_name = get_project_or_local(
|
2216
|
+
owner, _, project_name = get_project_or_local(
|
2065
2217
|
project or ctx.obj.get("project"), is_cli=True
|
2066
2218
|
)
|
2067
2219
|
|
@@ -2154,10 +2306,32 @@ def pull(
|
|
2154
2306
|
help="Optional, to ignore the owner/project of the local "
|
2155
2307
|
"run and use the owner/project provided or resolved from the current project.",
|
2156
2308
|
)
|
2309
|
+
@click.option(
|
2310
|
+
"--reset-uuid",
|
2311
|
+
is_flag=True,
|
2312
|
+
default=False,
|
2313
|
+
help="Optional, to ignore the uuid of the local " "run and generate a new uuid.",
|
2314
|
+
)
|
2315
|
+
@click.option(
|
2316
|
+
"--agent", "-ag", type=str, help="Optional, uuid reference of an agent to use."
|
2317
|
+
)
|
2318
|
+
@click.option("--name", "-n", type=str, help="Optional, a new name to set for the run.")
|
2157
2319
|
@click.pass_context
|
2158
2320
|
@clean_outputs
|
2159
|
-
def push(
|
2160
|
-
|
2321
|
+
def push(
|
2322
|
+
ctx,
|
2323
|
+
project,
|
2324
|
+
uid,
|
2325
|
+
all_runs,
|
2326
|
+
no_artifacts,
|
2327
|
+
clean,
|
2328
|
+
path,
|
2329
|
+
reset_project,
|
2330
|
+
reset_uuid,
|
2331
|
+
agent,
|
2332
|
+
name,
|
2333
|
+
):
|
2334
|
+
"""Push a local run (or all runs) to a remove server.
|
2161
2335
|
|
2162
2336
|
Uses /docs/core/cli/#caching
|
2163
2337
|
|
@@ -2181,7 +2355,7 @@ def push(ctx, project, uid, all_runs, no_artifacts, clean, path, reset_project):
|
|
2181
2355
|
\b
|
2182
2356
|
$ polyaxon ops push -uid 8aac02e3a62a4f0aaa257c59da5eab80 --reset-project -p send-to-project
|
2183
2357
|
"""
|
2184
|
-
owner, project_name = get_project_or_local(
|
2358
|
+
owner, _, project_name = get_project_or_local(
|
2185
2359
|
project or ctx.obj.get("project"), is_cli=True
|
2186
2360
|
)
|
2187
2361
|
|
@@ -2204,6 +2378,8 @@ def push(ctx, project, uid, all_runs, no_artifacts, clean, path, reset_project):
|
|
2204
2378
|
path=artifacts_path,
|
2205
2379
|
run_client=client,
|
2206
2380
|
reset_project=reset_project,
|
2381
|
+
name=name,
|
2382
|
+
reset_uuid=reset_uuid,
|
2207
2383
|
raise_if_not_found=True,
|
2208
2384
|
)
|
2209
2385
|
except Exception as e:
|
@@ -2220,6 +2396,7 @@ def push(ctx, project, uid, all_runs, no_artifacts, clean, path, reset_project):
|
|
2220
2396
|
path=artifacts_path,
|
2221
2397
|
upload_artifacts=not no_artifacts,
|
2222
2398
|
clean=clean,
|
2399
|
+
agent=agent,
|
2223
2400
|
)
|
2224
2401
|
Printer.success(
|
2225
2402
|
f"Finished pushing offline run {uid} to {client.owner}/{client.project}"
|