polyaxon 2.0.0rc49__py3-none-any.whl → 2.4.0rc1__py3-none-any.whl

polyaxon/_cli/project_versions.py

@@ -1,3 +1,4 @@
+import os
 import sys
 
 from typing import Callable, List, Optional, Union
@@ -11,8 +12,9 @@ from clipped.utils.responses import get_meta_response
 from clipped.utils.validation import validate_tags
 from urllib3.exceptions import HTTPError
 
-from polyaxon._cli.dashboard import get_dashboard_url
+from polyaxon._cli.dashboard import get_dashboard_url, get_project_subpath_url
 from polyaxon._cli.errors import handle_cli_error
+from polyaxon._contexts.paths import get_offline_base_path
 from polyaxon._schemas.lifecycle import V1ProjectVersionKind
 from polyaxon._utils.fqn_utils import get_versioned_entity_full_name
 from polyaxon.client import PolyaxonClient, ProjectClient
@@ -168,6 +170,7 @@ def list_project_versions(
 
 def register_project_version(
     owner: str,
+    team: Optional[str],
     project_name: str,
     version: str,
     kind: V1ProjectVersionKind,
@@ -207,7 +210,9 @@ def register_project_version(
     Printer.print(
         "You can view this version on Polyaxon UI: {}".format(
             get_dashboard_url(
-                subpath="{}/{}/{}s/{}".format(owner, project_name, kind, version)
+                subpath="{}/{}s/{}".format(
+                    get_project_subpath_url(owner, team, project_name), kind, version
+                )
             )
         )
     )
@@ -215,6 +220,7 @@ def register_project_version(
 
 def copy_project_version(
     owner: str,
+    team: Optional[str],
     project_name: str,
     version: str,
     kind: V1ProjectVersionKind,
@@ -259,8 +265,10 @@ def copy_project_version(
     Printer.print(
         "You can view this version on Polyaxon UI: {}".format(
             get_dashboard_url(
-                subpath="{}/{}/{}s/{}".format(
-                    owner, to_project or project_name, kind, _version.name
+                subpath="{}/{}s/{}".format(
+                    get_project_subpath_url(owner, team, to_project or project_name),
+                    kind,
+                    _version.name,
                 )
             )
         )
@@ -269,6 +277,7 @@ def copy_project_version(
 
 def get_project_version(
     owner: str,
+    team: Optional[str],
     project_name: str,
     kind: V1ProjectVersionKind,
     version: str,
@@ -286,6 +295,17 @@ def get_project_version(
     try:
         response = polyaxon_client.get_version(kind, version)
         get_version_details(response, content_callback)
+        Printer.print(
+            "You can view this version on Polyaxon UI: {}".format(
+                get_dashboard_url(
+                    subpath="{}/{}s/{}".format(
+                        get_project_subpath_url(owner, team, project_name),
+                        kind,
+                        version,
+                    )
+                )
+            )
+        )
     except (ApiException, HTTPError) as e:
         handle_cli_error(
             e,
@@ -468,13 +488,16 @@ def stage_project_version(
 
 def open_project_version_dashboard(
     owner: str,
+    team: Optional[str],
     project_name: str,
     kind: V1ProjectVersionKind,
     version: str,
     url: str,
     yes: bool = False,
 ):
-    subpath = "{}/{}/{}s/{}".format(owner, project_name, kind, version)
+    subpath = "{}/{}s/{}".format(
+        get_project_subpath_url(owner, team, project_name), kind, version
+    )
 
     artifact_url = get_dashboard_url(subpath=subpath)
     if url:
@@ -526,7 +549,7 @@ def pull_one_or_many_project_versions(
     owner: str,
     project_name: str,
     kind: V1ProjectVersionKind,
-    version: str,
+    version: Optional[str] = None,
     all_versions: Optional[bool] = None,
     query: Optional[str] = None,
     limit: Optional[int] = None,
@@ -566,3 +589,113 @@ def pull_one_or_many_project_versions(
             "or pass the flag `-a/--all` to pull versions.",
             sys_exit=True,
         )
+
+
+def push_project_version(
+    owner: str,
+    project_name: str,
+    kind: V1ProjectVersionKind,
+    version: str,
+    path: str,
+    reset_project: bool = False,
+    force: bool = False,
+    clean: bool = False,
+    sys_exit: bool = False,
+):
+    fqn_version = get_versioned_entity_full_name(owner, project_name, version)
+    polyaxon_client = ProjectClient(
+        owner=owner, project=project_name, manual_exceptions_handling=True
+    )
+
+    try:
+        try:
+            polyaxon_client.load_offline_version(
+                kind=kind,
+                version=version,
+                path=path,
+                project_client=polyaxon_client,
+                reset_project=reset_project,
+                raise_if_not_found=True,
+            )
+        except Exception as _:
+            Printer.error(
+                "Could not load offline version `{}`.".format(version),
+                sys_exit=sys_exit,
+            )
+            return
+
+        Printer.header(
+            "Pushing {} version [white]`{}`[/white] ...".format(kind, fqn_version),
+        )
+        polyaxon_client.push_version(
+            kind,
+            version,
+            path=path,
+            force=force,
+            clean=clean,
+        )
+        Printer.success(
+            "Finished pushing the {} version `{}` from `{}`".format(
+                kind, fqn_version, path
+            )
+        )
+    except (ApiException, HTTPError) as e:
+        handle_cli_error(
+            e,
+            message="Could not push the {} version `{}`".format(kind, fqn_version),
+        )
+
+
+def push_one_or_many_project_versions(
+    owner: str,
+    project_name: str,
+    kind: V1ProjectVersionKind,
+    path: str,
+    version: Optional[str] = None,
+    all_versions: Optional[bool] = None,
+    reset_project: bool = False,
+    force: bool = False,
+    clean: bool = False,
+):
+    def _push(version_name: str):
+        push_project_version(
+            owner=owner,
+            project_name=project_name,
+            kind=kind,
+            version=version_name,
+            path=path,
+            reset_project=reset_project,
+            force=force,
+            clean=clean,
+        )
+
+    offline_path = get_offline_base_path(
+        entity_kind=kind,
+        path=path,
+    )
+
+    if all_versions:
+        if (
+            not os.path.exists(offline_path)
+            or not os.path.isdir(offline_path)
+            or not os.listdir(offline_path)
+        ):
+            Printer.error(
+                f"Could not push offline {kind} versions, the path `{offline_path}` "
+                f"does not exist, is not a directory, or is empty."
+            )
+            sys.exit(1)
+        version_paths = os.listdir(offline_path)
+        Printer.header(
+            f"Pushing local {kind} versions (total: {len(version_paths)}) ..."
+        )
+        for idx, uid in enumerate(version_paths):
+            Printer.heading(f"Pushing {kind} version {idx + 1}/{len(offline_path)} ...")
+            _push(uid)
+    elif version:
+        _push(version)
+    else:
+        Printer.error(
+            "Please provide a version name, or pass the flag `-a/--all` to pull versions.",
+            sys_exit=True,
+        )

polyaxon/_cli/projects.py

@@ -8,7 +8,7 @@ from clipped.utils.responses import get_meta_response
 from clipped.utils.validation import validate_tags
 from urllib3.exceptions import HTTPError
 
-from polyaxon._cli.dashboard import get_dashboard_url
+from polyaxon._cli.dashboard import get_dashboard_url, get_project_subpath_url
 from polyaxon._cli.errors import handle_cli_error
 from polyaxon._cli.init import init as init_project
 from polyaxon._cli.options import OPTIONS_NAME, OPTIONS_OWNER, OPTIONS_PROJECT
@@ -19,6 +19,7 @@ from polyaxon._env_vars.getters.user import get_local_owner
 from polyaxon._managers.project import ProjectConfigManager
 from polyaxon._utils import cache
 from polyaxon._utils.cache import get_local_project
+from polyaxon._utils.fqn_utils import get_owner_team_space
 from polyaxon.client import ProjectClient, V1Project
 from polyaxon.exceptions import ApiException
 from polyaxon.logger import clean_outputs
@@ -74,7 +75,7 @@ def create(ctx, name, description, tags, public, init):
             command_help="project create",
             sys_exit=True,
         )
-    owner, project_name = resolve_entity_info(
+    owner, team, project_name = resolve_entity_info(
         name or ctx.obj.get("project"), is_cli=True, entity_name="project"
     )
 
@@ -90,7 +91,9 @@ def create(ctx, name, description, tags, public, init):
         project_config = V1Project(
             name=project_name, description=description, tags=tags, is_public=public
         )
-        polyaxon_client = ProjectClient(owner=owner, manual_exceptions_handling=True)
+        polyaxon_client = ProjectClient(
+            owner=get_owner_team_space(owner, team), manual_exceptions_handling=True
+        )
         _project = polyaxon_client.create(project_config)
         config = polyaxon_client.client.sanitize_for_serialization(_project)
         cache.cache(config_manager=ProjectConfigManager, config=config)
@@ -103,7 +106,9 @@ def create(ctx, name, description, tags, public, init):
     Printer.success("Project `{}` was created successfully.".format(_project.name))
     Printer.print(
         "You can view this project on Polyaxon UI: {}".format(
-            get_dashboard_url(subpath="{}/{}".format(owner, _project.name))
+            get_dashboard_url(
+                subpath=get_project_subpath_url(owner, team, _project.name)
+            )
         )
     )
 
@@ -195,7 +200,7 @@ def get(ctx, _project):
     \b
     $ polyaxon project get -p owner/project
     """
-    owner, project_name = get_project_or_local(
+    owner, team, project_name = get_project_or_local(
         _project or ctx.obj.get("project"), is_cli=True
     )
 
@@ -214,6 +219,13 @@ def get(ctx, _project):
             project=project_name,
         )
         get_entity_details(polyaxon_client.project_data, "Project")
+        Printer.print(
+            "You can view this project on Polyaxon UI: {}".format(
+                get_dashboard_url(
+                    subpath=get_project_subpath_url(owner, team, project_name)
+                )
+            )
+        )
     except (ApiException, HTTPError) as e:
         handle_cli_error(
             e, message="Could not get project `{}`.".format(project_name), sys_exit=True
@@ -237,7 +249,7 @@ def delete(ctx, _project, yes):
 
     Uses /docs/core/cli/#caching
     """
-    owner, project_name = get_project_or_local(
+    owner, _, project_name = get_project_or_local(
         _project or ctx.obj.get("project"), is_cli=True
     )
 
@@ -300,7 +312,7 @@ def update(ctx, _project, name, description, tags, private):
     \b
     $ polyaxon update --tags="foo, bar"
     """
-    owner, project_name = get_project_or_local(
+    owner, _, project_name = get_project_or_local(
         _project or ctx.obj.get("project"), is_cli=True
     )
 
@@ -357,10 +369,12 @@ def update(ctx, _project, name, description, tags, private):
 @clean_outputs
 def dashboard(ctx, _project, yes, url):
     """Open this project's dashboard details in browser."""
-    owner, project_name = get_project_or_local(
+    owner, team, project_name = get_project_or_local(
         _project or ctx.obj.get("project"), is_cli=True
     )
-    project_url = get_dashboard_url(subpath="{}/{}".format(owner, project_name))
+    project_url = get_dashboard_url(
+        subpath=get_project_subpath_url(owner, team, project_name)
+    )
     if url:
         Printer.header("The dashboard is available at: {}".format(project_url))
         sys.exit(0)

polyaxon/_cli/run.py

@@ -10,10 +10,11 @@ from clipped.utils import git as git_utils
 from clipped.utils.validation import validate_tags
 from urllib3.exceptions import HTTPError
 
-from polyaxon._cli.dashboard import get_dashboard_url
+from polyaxon._cli.dashboard import get_dashboard_url, get_project_subpath_url
 from polyaxon._cli.errors import handle_cli_error
 from polyaxon._cli.operations import approve
 from polyaxon._cli.operations import execute as run_execute
+from polyaxon._cli.operations import get_op_agent_host
 from polyaxon._cli.operations import logs as run_logs
 from polyaxon._cli.operations import shell as run_shell
 from polyaxon._cli.operations import statuses
@@ -30,6 +31,7 @@ from polyaxon._polyaxonfile import check_polyaxonfile
 from polyaxon._runner.kinds import RunnerKind
 from polyaxon._schemas.lifecycle import ManagedBy
 from polyaxon._utils import cache
+from polyaxon._utils.fqn_utils import get_owner_team_space
 from polyaxon.client import RunClient
 from polyaxon.exceptions import ApiException
 from polyaxon.logger import clean_outputs
@@ -43,6 +45,7 @@ def _run(
     ctx,
     name: str,
     owner: str,
+    team: Optional[str],
     project_name: str,
     description: str,
     tags: List[str],
@@ -61,11 +64,14 @@ def _run(
         owner=owner, project=project_name, manual_exceptions_handling=True
     )
 
+    owner_team = get_owner_team_space(owner, team)
+    project_url = get_project_subpath_url(owner, team, project_name)
+
     def get_instance_info(r):
         rn = (
             f"[white]{r.name}[/white]@[white]{r.uuid}[/white]"
             if r.name
-            else "[white]{r.uuid}[/white]"
+            else f"[white]{r.uuid}[/white]"
         )
         return f"[white]{owner}[/white]/[white]{project_name}[/white]:{rn}"
 
@@ -93,8 +99,9 @@ def _run(
                 meta_info=meta_info,
                 pending=pending,
             )
+            host_kwargs = get_op_agent_host(response.settings)
             run_url = get_dashboard_url(
-                subpath="{}/{}/runs/{}".format(owner, project_name, response.uuid)
+                subpath="{}/runs/{}".format(project_url, response.uuid), **host_kwargs
             )
             if output:
                 response_data = polyaxon_client.client.sanitize_for_serialization(
@@ -125,26 +132,38 @@ def _run(
 
     def execute_run_locally(run_uuid: str):
         ctx.obj = {
-            "project": "{}/{}".format(owner, project_name),
+            "project": "{}/{}".format(owner_team, project_name),
             "run_uuid": run_uuid,
             "executor": executor,
         }
         ctx.invoke(run_execute)
 
     def watch_run_statuses(run_uuid: str):
-        ctx.obj = {"project": "{}/{}".format(owner, project_name), "run_uuid": run_uuid}
+        ctx.obj = {
+            "project": "{}/{}".format(owner_team, project_name),
+            "run_uuid": run_uuid,
+        }
         ctx.invoke(statuses, watch=True)
 
     def watch_run_logs(run_uuid: str):
-        ctx.obj = {"project": "{}/{}".format(owner, project_name), "run_uuid": run_uuid}
+        ctx.obj = {
+            "project": "{}/{}".format(owner_team, project_name),
+            "run_uuid": run_uuid,
+        }
         ctx.invoke(run_logs)
 
     def start_run_shell(run_uuid: str):
-        ctx.obj = {"project": "{}/{}".format(owner, project_name), "run_uuid": run_uuid}
+        ctx.obj = {
+            "project": "{}/{}".format(owner_team, project_name),
+            "run_uuid": run_uuid,
+        }
         ctx.invoke(run_shell)
 
     def upload_run(run_uuid: str):
-        ctx.obj = {"project": "{}/{}".format(owner, project_name), "run_uuid": run_uuid}
+        ctx.obj = {
+            "project": "{}/{}".format(owner_team, project_name),
+            "run_uuid": run_uuid,
+        }
         ctx.invoke(
             run_upload, path_to=upload_to, path_from=upload_from, sync_failure=True
         )
@@ -334,6 +353,12 @@ def _run(
     "If the name is not namespaced by the agent name the default agent is used: "
     "queue-name or agent-name/queue-name",
 )
+@click.option(
+    "--namespace",
+    "-ns",
+    type=str,
+    help="Namespace to use for this run. By default it will use the agent's namespace.",
+)
 @click.option(
     "--nocache",
     is_flag=True,
@@ -412,6 +437,7 @@ def run(
     matrix_num_runs,
     presets,
     queue,
+    namespace,
     nocache,
     cache,
     approved,
@@ -533,6 +559,7 @@ def run(
         matrix_num_runs=matrix_num_runs,
         presets=presets,
         queue=queue,
+        namespace=namespace,
         cache=cache,
         nocache=nocache,
         approved=approved,
@@ -547,13 +574,14 @@ def run(
         Printer.print("Please customize the specification or disable the template!")
         sys.exit(1)
 
-    owner, project_name = get_project_or_local(project, is_cli=True)
+    owner, team, project_name = get_project_or_local(project, is_cli=True)
     tags = validate_tags(tags, validate_yaml=True)
 
     _run(
         ctx=ctx,
         name=name,
         owner=owner,
+        team=team,
         project_name=project_name,
         description=description,
         tags=tags,

polyaxon/_cli/services/agent.py

@@ -33,7 +33,7 @@ def agent():
     help="Number of times to retry the process.",
 )
 @coroutine
-async def start(kind, max_retries, max_interval):
+async def start(kind, max_interval, max_retries):
     from polyaxon import settings
     from polyaxon._env_vars.getters import get_agent_info
     from polyaxon._runner.kinds import RunnerKind
@@ -65,7 +65,7 @@ async def start(kind, max_retries, max_interval):
                 await ag.start()
                 return
         except Exception as e:
-            logger.warning("Polyaxon agent retrying, error %s", e)
+            logger.warning("Polyaxon agent retrying, error %r", e)
             retry += 1
 
 

polyaxon/_cli/services/clean_artifacts.py

@@ -5,7 +5,7 @@ import click
 from clipped.formatting import Printer
 from clipped.utils.lists import to_list
 
-from polyaxon._connections import V1Connection, V1ConnectionKind
+from polyaxon._connections import V1ConnectionKind
 
 
 @click.group()

polyaxon/_cli/services/sidecar.py

@@ -36,9 +36,15 @@ from polyaxon.logger import logger
     default=False,
     help="Enable logs monitoring.",
 )
+@click.option(
+    "--monitor-spec",
+    is_flag=True,
+    default=False,
+    help="Enable spec monitoring.",
+)
 @coroutine
 async def sidecar(
-    container_id, max_retries, sleep_interval, sync_interval, monitor_logs
+    container_id, max_retries, sleep_interval, sync_interval, monitor_logs, monitor_spec
 ):
     """
     Start Polyaxon's sidecar command.
@@ -56,6 +62,7 @@ async def sidecar(
                 sync_interval=sync_interval,
                 monitor_outputs=True,
                 monitor_logs=monitor_logs,
+                monitor_spec=monitor_spec,
             )
             return
         except Exception as e:

polyaxon/_client/client.py

@@ -92,6 +92,7 @@ class PolyaxonClient:
         self._users_v1 = None
         self._versions_v1 = None
         self._agents_v1 = None
+        self._internal_agents_v1 = None
         self._queues_v1 = None
         self._service_accounts_v1 = None
         self._presets_v1 = None
@@ -109,6 +110,15 @@ class PolyaxonClient:
             )
         return ApiClient(self.config.sdk_config, **self.config.client_header)
 
+    def _get_internal_client(self):
+        if self.is_async:
+            return AsyncApiClient(
+                self.config.internal_sdk_config, **self.config.get_internal_header()
+            )
+        return ApiClient(
+            self.config.internal_sdk_config, **self.config.get_internal_header()
+        )
+
     def reset(self):
         self._projects_v1 = None
         self._runs_v1 = None
@@ -118,6 +128,7 @@ class PolyaxonClient:
         self._users_v1 = None
         self._versions_v1 = None
         self._agents_v1 = None
+        self._internal_agents_v1 = None
         self._queues_v1 = None
         self._service_accounts_v1 = None
         self._presets_v1 = None
@@ -169,6 +180,12 @@ class PolyaxonClient:
             self._agents_v1 = AgentsV1Api(self.api_client)
         return self._agents_v1
 
+    @property
+    def internal_agents_v1(self):
+        if not self._internal_agents_v1:
+            self._internal_agents_v1 = AgentsV1Api(self._get_internal_client())
+        return self._internal_agents_v1
+
     @property
     def queues_v1(self):
         if not self._queues_v1:

polyaxon/_client/mixin.py

@@ -0,0 +1,39 @@
+from polyaxon import settings
+from polyaxon._client.client import PolyaxonClient
+from polyaxon._schemas.client import ClientConfig
+from polyaxon._utils.fqn_utils import split_owner_team_space
+
+
+class ClientMixin:
+    @property
+    def client(self):
+        if self._client:
+            return self._client
+        self._client = PolyaxonClient()
+        return self._client
+
+    def reset_client(self, **kwargs):
+        if not settings.CLIENT_CONFIG.in_cluster:
+            self._client = PolyaxonClient(
+                ClientConfig.patch_from(settings.CLIENT_CONFIG, **kwargs)
+            )
+
+    @property
+    def owner(self) -> str:
+        return self._owner
+
+    @property
+    def team(self):
+        return self._team
+
+    @property
+    def project(self) -> str:
+        return self._project
+
+    def set_project(self, project: str):
+        self._project = project
+
+    def set_owner(self, owner: str):
+        owner, team = split_owner_team_space(owner)
+        self._owner = owner
+        self._team = team