polyaxon 2.1.7.post1__py3-none-any.whl → 2.2.0__py3-none-any.whl

polyaxon/_local_process/executor.py

@@ -0,0 +1,132 @@
+import os
+import signal
+import subprocess
+
+from typing import Dict, List
+
+from polyaxon._deploy.operators.cmd_operator import CmdOperator
+from polyaxon._deploy.operators.conda import CondaOperator
+from polyaxon._local_process import process_types
+from polyaxon._local_process.converter.converters import CONVERTERS
+from polyaxon._local_process.converter.mixins import MIXIN_MAPPING
+from polyaxon._runner.executor import BaseExecutor
+from polyaxon._runner.kinds import RunnerKind
+from polyaxon._schemas.lifecycle import V1Statuses
+from polyaxon.exceptions import PolyaxonAgentError
+from polyaxon.logger import logger
+
+
+class Executor(BaseExecutor):
+    MIXIN_MAPPING = MIXIN_MAPPING
+    CONVERTERS = CONVERTERS
+    RUNNER_KIND = RunnerKind.PROCESS
+
+    def __init__(self, conda_env: str = None, venv: str = None):
+        super().__init__()
+        self._ops = {}
+        self._conda_env = conda_env
+        self._venv = venv
+
+    def _get_manager(self):
+        if self._conda_env:
+            return CondaOperator()
+        return CmdOperator()
+
+    def _check_conda(self):
+        if not self.manager.check():
+            raise logger.error("Conda is required to run this command.")
+
+        envs = self.manager.execute(["env", "list", "--json"], is_json=True)
+        env_names = [os.path.basename(env) for env in envs["envs"]]
+        if self._conda_env not in env_names:
+            raise logger.error(
+                "Conda env `{}` is not installed.".format(self._conda_env),
+                sys_exit=True,
+            )
+
+    def _run_in_conda(self, cmd_bash, cmd_args):
+        cmd_args = ["source activate {}".format(self._conda_env)] + cmd_args
+        subprocess.Popen(cmd_bash + [" && ".join(cmd_args)], close_fds=True)
+
+    def _get_op_proc(self, run_uuid: str) -> List[subprocess.Popen]:
+        return self._ops.get(run_uuid)
+
+    def create(
+        self,
+        run_uuid: str,
+        run_kind: str,
+        resource: List[process_types.V1Container],
+        namespace: str = None,
+    ) -> Dict:
+        logger.info(f"[Executor] Starting operation {run_uuid} {run_kind}.")
+        self._ops[run_uuid] = []
+        for task in resource:
+            logger.info(
+                f"[Executor] Starting task container {task.name} {task.image} ."
+            )
+            proc = self.manager.execute(
+                task.get_cmd_args(), env=os.environ, output_only=False
+            )
+            self._ops[run_uuid].append(proc)
+            proc.wait()
+            task_status = self._get_task_status(proc)
+            message = f"Task container {task.name} {task.image} with id {proc.pid} {task_status}"
+            if task_status == V1Statuses.SUCCEEDED:
+                logger.info(f"[Executor] {message}")
+            else:
+                logger.warning(f"[Executor] {message}")
+                self._clean_temp_execution_path(run_uuid)
+                return {
+                    "status": V1Statuses.FAILED,
+                    "tasks": self._ops[run_uuid],
+                    "message": message,
+                }
+        self._clean_temp_execution_path(run_uuid)
+        return {"status": V1Statuses.SUCCEEDED, "tasks": self._ops[run_uuid]}
+
+    def apply(
+        self, run_uuid: str, run_kind: str, resource: Dict, namespace: str = None
+    ) -> Dict:
+        raise PolyaxonAgentError(
+            "Docker executor does not support apply method.\n"
+            "Run: <kind: {}, uuid: {}>".format(run_kind, run_uuid)
+        )
+
+    def stop(self, run_uuid: str, run_kind: str, namespace: str = None):
+        proc = self._get_op_proc(run_uuid)
+        if proc.poll() is None:
+            # Kill the process tree rooted at the child if it's the leader of its own process
+            # group, otherwise just kill the child
+            try:
+                if proc.pid == os.getpgid(proc.pid):
+                    os.killpg(proc.pid, signal.SIGTERM)
+                else:
+                    proc.terminate()
+            except OSError:
+                # The child process may have exited before we attempted to terminate it, so we
+                # ignore OSErrors raised during child process termination
+                _msg = f"Failed to terminate operation {run_kind} {run_uuid} child process PID {proc.pid}"
+                logger.debug(_msg)
+            proc.wait()
+
+    def clean(self, run_uuid: str, run_kind: str, namespace: str = None):
+        return self.apply(
+            run_uuid=run_uuid,
+            run_kind=run_kind,
+            resource={"metadata": {"finalizers": None}},
+        )
+
+    def _get_task_status(self, proc) -> V1Statuses:
+        exit_code = proc.poll()
+        if exit_code is None:
+            return V1Statuses.RUNNING
+        if exit_code == 0:
+            return V1Statuses.SUCCEEDED
+        return V1Statuses.FAILED
+
+    def get(self, run_uuid: str, run_kind: str, namespace: str = None) -> V1Statuses:
+        procs = self._get_op_proc(run_uuid)
+        return self._get_task_status(procs[-1])
+
+    def list_ops(self, namespace: str = None):
+        return []

polyaxon/_local_process/process_types.py

@@ -0,0 +1,39 @@
+from typing import Dict, List, Optional, Tuple, Union
+
+from clipped.compact.pydantic import Field
+
+from polyaxon._schemas.base import BaseSchemaModel
+
+
+class V1EnvVar(BaseSchemaModel):
+    __root__: Union[Tuple[str, str], Dict[str, str]]
+
+    def to_cmd(self):
+        if isinstance(self.__root__, tuple):
+            value = self.__root__
+        else:
+            value = self.__root__.items()
+        return [f"{value[0]}={value[1]}"]
+
+
+class V1Container(BaseSchemaModel):
+    name: Optional[str]
+    command: Optional[List[str]]
+    args: Optional[List[str]]
+    env: Optional[List[V1EnvVar]]
+    working_dir: Optional[str] = Field(alias="workingDir")
+
+    def get_cmd_args(self):
+        cmd_args = ["run", "--rm"]
+        for env in self.env:
+            cmd_args += ["-e"] + env.to_cmd()
+        if self.working_dir:
+            cmd_args += ["-w", self.working_dir]
+        if self.command:
+            cmd_args += ["--entrypoint", self.command[0]]
+        cmd_args += [self.image]
+        if self.command:
+            cmd_args += self.command[1:]
+        if self.args:
+            cmd_args += self.args
+        return cmd_args

polyaxon/_sdk/api/organizations_v1_api.py

@@ -1270,7 +1270,7 @@ class OrganizationsV1Api(BaseApi):
     def delete_organization_member(
         self,
         owner: Annotated[StrictStr, Field(..., description="Owner of the namespace")],
-        name: Annotated[StrictStr, Field(..., description="Component under namesapce")],
+        name: Annotated[StrictStr, Field(..., description="Component under namespace")],
         **kwargs
     ) -> None:  # noqa: E501
         """Delete organization member details  # noqa: E501
@@ -1283,7 +1283,7 @@ class OrganizationsV1Api(BaseApi):
 
         :param owner: Owner of the namespace (required)
         :type owner: str
-        :param name: Component under namesapce (required)
+        :param name: Component under namespace (required)
         :type name: str
         :param async_req: Whether to execute the request asynchronously.
         :type async_req: bool, optional
@@ -1309,7 +1309,7 @@ class OrganizationsV1Api(BaseApi):
     def delete_organization_member_with_http_info(
         self,
         owner: Annotated[StrictStr, Field(..., description="Owner of the namespace")],
-        name: Annotated[StrictStr, Field(..., description="Component under namesapce")],
+        name: Annotated[StrictStr, Field(..., description="Component under namespace")],
         **kwargs
     ):  # noqa: E501
         """Delete organization member details  # noqa: E501
@@ -1322,7 +1322,7 @@ class OrganizationsV1Api(BaseApi):
 
         :param owner: Owner of the namespace (required)
         :type owner: str
-        :param name: Component under namesapce (required)
+        :param name: Component under namespace (required)
         :type name: str
         :param async_req: Whether to execute the request asynchronously.
         :type async_req: bool, optional
@@ -2249,7 +2249,7 @@ class OrganizationsV1Api(BaseApi):
     def get_organization_member(
         self,
         owner: Annotated[StrictStr, Field(..., description="Owner of the namespace")],
-        name: Annotated[StrictStr, Field(..., description="Component under namesapce")],
+        name: Annotated[StrictStr, Field(..., description="Component under namespace")],
         **kwargs
     ) -> V1OrganizationMember:  # noqa: E501
         """Get organization member details  # noqa: E501
@@ -2262,7 +2262,7 @@ class OrganizationsV1Api(BaseApi):
 
         :param owner: Owner of the namespace (required)
         :type owner: str
-        :param name: Component under namesapce (required)
+        :param name: Component under namespace (required)
         :type name: str
         :param async_req: Whether to execute the request asynchronously.
         :type async_req: bool, optional
@@ -2288,7 +2288,7 @@ class OrganizationsV1Api(BaseApi):
     def get_organization_member_with_http_info(
         self,
         owner: Annotated[StrictStr, Field(..., description="Owner of the namespace")],
-        name: Annotated[StrictStr, Field(..., description="Component under namesapce")],
+        name: Annotated[StrictStr, Field(..., description="Component under namespace")],
         **kwargs
     ):  # noqa: E501
         """Get organization member details  # noqa: E501
@@ -2301,7 +2301,7 @@ class OrganizationsV1Api(BaseApi):
 
         :param owner: Owner of the namespace (required)
         :type owner: str
-        :param name: Component under namesapce (required)
+        :param name: Component under namespace (required)
         :type name: str
         :param async_req: Whether to execute the request asynchronously.
         :type async_req: bool, optional

polyaxon/_sdk/api/project_dashboards_v1_api.py

@@ -15,7 +15,7 @@ class ProjectDashboardsV1Api(BaseApi):
         self,
         owner: Annotated[StrictStr, Field(..., description="Owner of the namespace")],
         project: Annotated[
-            StrictStr, Field(..., description="Project under namesapce")
+            StrictStr, Field(..., description="Project under namespace")
         ],
         body: Annotated[V1Dashboard, Field(..., description="Dashboard body")],
         **kwargs
@@ -30,7 +30,7 @@ class ProjectDashboardsV1Api(BaseApi):
 
         :param owner: Owner of the namespace (required)
         :type owner: str
-        :param project: Project under namesapce (required)
+        :param project: Project under namespace (required)
         :type project: str
         :param body: Dashboard body (required)
         :type body: V1Dashboard
@@ -59,7 +59,7 @@ class ProjectDashboardsV1Api(BaseApi):
         self,
         owner: Annotated[StrictStr, Field(..., description="Owner of the namespace")],
         project: Annotated[
-            StrictStr, Field(..., description="Project under namesapce")
+            StrictStr, Field(..., description="Project under namespace")
         ],
         body: Annotated[V1Dashboard, Field(..., description="Dashboard body")],
         **kwargs
@@ -74,7 +74,7 @@ class ProjectDashboardsV1Api(BaseApi):
 
         :param owner: Owner of the namespace (required)
         :type owner: str
-        :param project: Project under namesapce (required)
+        :param project: Project under namespace (required)
         :type project: str
         :param body: Dashboard body (required)
         :type body: V1Dashboard
@@ -1069,7 +1069,7 @@ class ProjectDashboardsV1Api(BaseApi):
         self,
         owner: Annotated[StrictStr, Field(..., description="Owner of the namespace")],
         project: Annotated[
-            StrictStr, Field(..., description="Project under namesapce")
+            StrictStr, Field(..., description="Project under namespace")
         ],
         dashboard_uuid: Annotated[StrictStr, Field(..., description="UUID")],
         body: Annotated[V1Dashboard, Field(..., description="Dashboard body")],
@@ -1085,7 +1085,7 @@ class ProjectDashboardsV1Api(BaseApi):
 
         :param owner: Owner of the namespace (required)
         :type owner: str
-        :param project: Project under namesapce (required)
+        :param project: Project under namespace (required)
         :type project: str
         :param dashboard_uuid: UUID (required)
         :type dashboard_uuid: str
@@ -1116,7 +1116,7 @@ class ProjectDashboardsV1Api(BaseApi):
         self,
         owner: Annotated[StrictStr, Field(..., description="Owner of the namespace")],
         project: Annotated[
-            StrictStr, Field(..., description="Project under namesapce")
+            StrictStr, Field(..., description="Project under namespace")
         ],
         dashboard_uuid: Annotated[StrictStr, Field(..., description="UUID")],
         body: Annotated[V1Dashboard, Field(..., description="Dashboard body")],
@@ -1132,7 +1132,7 @@ class ProjectDashboardsV1Api(BaseApi):
 
         :param owner: Owner of the namespace (required)
         :type owner: str
-        :param project: Project under namesapce (required)
+        :param project: Project under namespace (required)
         :type project: str
         :param dashboard_uuid: UUID (required)
         :type dashboard_uuid: str
@@ -1440,7 +1440,7 @@ class ProjectDashboardsV1Api(BaseApi):
         self,
         owner: Annotated[StrictStr, Field(..., description="Owner of the namespace")],
         project: Annotated[
-            StrictStr, Field(..., description="Project under namesapce")
+            StrictStr, Field(..., description="Project under namespace")
         ],
         dashboard_uuid: Annotated[StrictStr, Field(..., description="UUID")],
         body: Annotated[V1Dashboard, Field(..., description="Dashboard body")],
@@ -1456,7 +1456,7 @@ class ProjectDashboardsV1Api(BaseApi):
 
         :param owner: Owner of the namespace (required)
         :type owner: str
-        :param project: Project under namesapce (required)
+        :param project: Project under namespace (required)
         :type project: str
         :param dashboard_uuid: UUID (required)
         :type dashboard_uuid: str
@@ -1487,7 +1487,7 @@ class ProjectDashboardsV1Api(BaseApi):
         self,
         owner: Annotated[StrictStr, Field(..., description="Owner of the namespace")],
         project: Annotated[
-            StrictStr, Field(..., description="Project under namesapce")
+            StrictStr, Field(..., description="Project under namespace")
         ],
         dashboard_uuid: Annotated[StrictStr, Field(..., description="UUID")],
         body: Annotated[V1Dashboard, Field(..., description="Dashboard body")],
@@ -1503,7 +1503,7 @@ class ProjectDashboardsV1Api(BaseApi):
 
         :param owner: Owner of the namespace (required)
         :type owner: str
-        :param project: Project under namesapce (required)
+        :param project: Project under namespace (required)
         :type project: str
         :param dashboard_uuid: UUID (required)
         :type dashboard_uuid: str

polyaxon/_sdk/api/project_searches_v1_api.py

@@ -15,7 +15,7 @@ class ProjectSearchesV1Api(BaseApi):
         self,
         owner: Annotated[StrictStr, Field(..., description="Owner of the namespace")],
         project: Annotated[
-            StrictStr, Field(..., description="Project under namesapce")
+            StrictStr, Field(..., description="Project under namespace")
         ],
         body: Annotated[V1Search, Field(..., description="Search body")],
         **kwargs
@@ -30,7 +30,7 @@ class ProjectSearchesV1Api(BaseApi):
 
         :param owner: Owner of the namespace (required)
         :type owner: str
-        :param project: Project under namesapce (required)
+        :param project: Project under namespace (required)
         :type project: str
         :param body: Search body (required)
         :type body: V1Search
@@ -59,7 +59,7 @@ class ProjectSearchesV1Api(BaseApi):
         self,
         owner: Annotated[StrictStr, Field(..., description="Owner of the namespace")],
         project: Annotated[
-            StrictStr, Field(..., description="Project under namesapce")
+            StrictStr, Field(..., description="Project under namespace")
         ],
         body: Annotated[V1Search, Field(..., description="Search body")],
         **kwargs
@@ -74,7 +74,7 @@ class ProjectSearchesV1Api(BaseApi):
 
         :param owner: Owner of the namespace (required)
         :type owner: str
-        :param project: Project under namesapce (required)
+        :param project: Project under namespace (required)
         :type project: str
         :param body: Search body (required)
         :type body: V1Search
@@ -1069,7 +1069,7 @@ class ProjectSearchesV1Api(BaseApi):
         self,
         owner: Annotated[StrictStr, Field(..., description="Owner of the namespace")],
         project: Annotated[
-            StrictStr, Field(..., description="Project under namesapce")
+            StrictStr, Field(..., description="Project under namespace")
         ],
         search_uuid: Annotated[StrictStr, Field(..., description="UUID")],
         body: Annotated[V1Search, Field(..., description="Search body")],
@@ -1085,7 +1085,7 @@ class ProjectSearchesV1Api(BaseApi):
 
         :param owner: Owner of the namespace (required)
         :type owner: str
-        :param project: Project under namesapce (required)
+        :param project: Project under namespace (required)
         :type project: str
         :param search_uuid: UUID (required)
         :type search_uuid: str
@@ -1116,7 +1116,7 @@ class ProjectSearchesV1Api(BaseApi):
         self,
         owner: Annotated[StrictStr, Field(..., description="Owner of the namespace")],
         project: Annotated[
-            StrictStr, Field(..., description="Project under namesapce")
+            StrictStr, Field(..., description="Project under namespace")
         ],
         search_uuid: Annotated[StrictStr, Field(..., description="UUID")],
         body: Annotated[V1Search, Field(..., description="Search body")],
@@ -1132,7 +1132,7 @@ class ProjectSearchesV1Api(BaseApi):
 
         :param owner: Owner of the namespace (required)
         :type owner: str
-        :param project: Project under namesapce (required)
+        :param project: Project under namespace (required)
         :type project: str
         :param search_uuid: UUID (required)
         :type search_uuid: str
@@ -1440,7 +1440,7 @@ class ProjectSearchesV1Api(BaseApi):
         self,
         owner: Annotated[StrictStr, Field(..., description="Owner of the namespace")],
         project: Annotated[
-            StrictStr, Field(..., description="Project under namesapce")
+            StrictStr, Field(..., description="Project under namespace")
         ],
         search_uuid: Annotated[StrictStr, Field(..., description="UUID")],
         body: Annotated[V1Search, Field(..., description="Search body")],
@@ -1456,7 +1456,7 @@ class ProjectSearchesV1Api(BaseApi):
 
         :param owner: Owner of the namespace (required)
         :type owner: str
-        :param project: Project under namesapce (required)
+        :param project: Project under namespace (required)
         :type project: str
         :param search_uuid: UUID (required)
         :type search_uuid: str
@@ -1487,7 +1487,7 @@ class ProjectSearchesV1Api(BaseApi):
         self,
         owner: Annotated[StrictStr, Field(..., description="Owner of the namespace")],
         project: Annotated[
-            StrictStr, Field(..., description="Project under namesapce")
+            StrictStr, Field(..., description="Project under namespace")
         ],
         search_uuid: Annotated[StrictStr, Field(..., description="UUID")],
         body: Annotated[V1Search, Field(..., description="Search body")],
@@ -1503,7 +1503,7 @@ class ProjectSearchesV1Api(BaseApi):
 
         :param owner: Owner of the namespace (required)
         :type owner: str
-        :param project: Project under namesapce (required)
+        :param project: Project under namespace (required)
         :type project: str
         :param search_uuid: UUID (required)
         :type search_uuid: str