polyaxon 2.0.6rc8__py3-none-any.whl → 2.1.0__py3-none-any.whl

polyaxon/_flow/run/kubeflow/pytorch_job.py

@@ -1,8 +1,8 @@
-from typing import Optional, Union
+from typing import Dict, List, Optional, Union
 from typing_extensions import Literal
 
-from clipped.compact.pydantic import Field
-from clipped.types.ref_or_obj import RefField
+from clipped.compact.pydantic import Field, StrictStr
+from clipped.types.ref_or_obj import BoolOrRef, IntOrRef, RefField
 
 from polyaxon._flow.run.base import BaseRun
 from polyaxon._flow.run.enums import V1RunKind
@@ -12,6 +12,39 @@ from polyaxon._flow.run.kubeflow.scheduling_policy import V1SchedulingPolicy
 from polyaxon._flow.run.resources import V1RunResources
 from polyaxon._flow.run.utils import DestinationImageMixin
 from polyaxon._k8s.k8s_schemas import V1Container
+from polyaxon._schemas.base import BaseSchemaModel
+
+
+class V1PytorchElasticPolicy(BaseSchemaModel):
+    """Elastic policy for Pytorch distributed runs.
+
+    Args:
+        min_replicas: int, optional
+        max_replicas: int, optional
+        rdvz_backend: str, optional
+        rdvz_port: int, optional
+        rdvz_host: str, optional
+        rdvz_id: str, optional
+        rdvz_conf: List[Dict], optional
+        standalone: bool, optional
+        n_proc_per_node: int, optional
+        max_restarts: int, optional
+        metrics: List[Dict], optional
+    """
+
+    _IDENTIFIER = "elasticPolicy"
+
+    min_replicas: Optional[IntOrRef] = Field(alias="minReplicas")
+    max_replicas: Optional[IntOrRef] = Field(alias="maxReplicas")
+    rdvz_backend: Optional[StrictStr] = Field(alias="rdvzBackend")
+    rdvz_port: Optional[IntOrRef] = Field(alias="rdvzPort")
+    rdvz_host: Optional[StrictStr] = Field(alias="rdvzHost")
+    rdvz_id: Optional[StrictStr] = Field(alias="rdvzId")
+    rdvz_conf: Optional[List[Dict]] = Field(alias="rdvzConf")
+    standalone: Optional[BoolOrRef]
+    n_proc_per_node: Optional[IntOrRef] = Field(alias="nProcPerNode")
+    max_restarts: Optional[IntOrRef] = Field(alias="maxRestarts")
+    metrics: Optional[List[Dict]] = Field(alias="Metrics")
 
 
 class V1PytorchJob(BaseRun, DestinationImageMixin):
@@ -88,6 +121,18 @@ class V1PytorchJob(BaseRun, DestinationImageMixin):
     >>>  ...
     ```
 
+    ### elasticPolicy
+
+    ElasticPolicy encapsulates various policies for elastic distributed training job.
+
+    ```yaml
+    >>> run:
+    >>>   kind: pytorchjob
+    >>>   elasticPolicy:
+    >>>     ...
+    >>>  ...
+    ```
+
     ### master
 
     The master replica in the distributed PytorchJob
@@ -122,6 +167,8 @@ class V1PytorchJob(BaseRun, DestinationImageMixin):
     kind: Literal[_IDENTIFIER] = _IDENTIFIER
     clean_pod_policy: Optional[V1CleanPodPolicy] = Field(alias="cleanPodPolicy")
     scheduling_policy: Optional[V1SchedulingPolicy] = Field(alias="schedulingPolicy")
+    elastic_policy: Optional[V1PytorchElasticPolicy] = Field(alias="elasticPolicy")
+    n_proc_per_node: Optional[IntOrRef] = Field(alias="nProcPerNode")
     master: Optional[Union[V1KFReplica, RefField]]
     worker: Optional[Union[V1KFReplica, RefField]]
 

polyaxon/_flow/run/kubeflow/scheduling_policy.py

@@ -12,11 +12,15 @@ class V1SchedulingPolicy(BaseSchemaModel):
     Args:
         min_available: int, optional
         queue: str, optional
+        min_resources: int, optional
         priority_class: str, optional
+        schedule_timeout_seconds: int, optional
     """
 
     _IDENTIFIER = "schedulingPolicy"
 
     min_available: Optional[IntOrRef] = Field(alias="minAvailable")
     queue: Optional[StrictStr]
+    min_resources: Optional[IntOrRef] = Field(alias="minResources")
     priority_class: Optional[StrictStr] = Field(alias="priorityClass")
+    schedule_timeout_seconds: Optional[IntOrRef] = Field(alias="scheduleTimeoutSeconds")

polyaxon/_flow/run/kubeflow/tf_job.py

@@ -172,8 +172,9 @@ class V1TFJob(BaseRun, DestinationImageMixin):
 
     kind: Literal[_IDENTIFIER] = _IDENTIFIER
     clean_pod_policy: Optional[V1CleanPodPolicy] = Field(alias="cleanPodPolicy")
-    enable_dynamic_worker: Optional[bool] = Field(alias="enableDynamicWorker")
     scheduling_policy: Optional[V1SchedulingPolicy] = Field(alias="schedulingPolicy")
+    enable_dynamic_worker: Optional[bool] = Field(alias="enableDynamicWorker")
+    success_policy: Optional[str] = Field(alias="successPolicy")
     chief: Optional[Union[V1KFReplica, RefField]]
     ps: Optional[Union[V1KFReplica, RefField]]
     worker: Optional[Union[V1KFReplica, RefField]]

polyaxon/_fs/fs.py

@@ -72,6 +72,11 @@ def get_fs_from_name(connection_name: str, asynchronous: bool = False, **kwargs)
     )
 
 
+def get_sync_default_fs(**kwargs):
+    connection = get_artifacts_connection()
+    return get_sync_fs_from_connection(connection=connection, **kwargs)
+
+
 async def get_default_fs(**kwargs):
     connection = get_artifacts_connection()
     return await get_async_fs_from_connection(connection=connection, **kwargs)

polyaxon/_k8s/converter/converters/job.py

@@ -39,7 +39,7 @@ class JobConverter(JobMixin, BaseConverter):
             default_sa=default_sa,
         )
         return get_job_custom_resource(
-            namespace=self.namespace,
+            namespace=compiled_operation.namespace or self.namespace,
             main_container=replica_spec.main_container,
             sidecar_containers=replica_spec.sidecar_containers,
             init_containers=replica_spec.init_containers,

polyaxon/_k8s/converter/converters/kubeflow/paddle_job.py

@@ -58,6 +58,7 @@ class PaddleJobConverter(PaddleJobMixin, BaseConverter):
             termination=compiled_operation.termination,
             clean_pod_policy=job.clean_pod_policy,
             scheduling_policy=job.scheduling_policy,
+            elastic_policy=job.elastic_policy,
             collect_logs=plugins.collect_logs,
             sync_statuses=plugins.sync_statuses,
             notifications=plugins.notifications,

polyaxon/_k8s/converter/converters/kubeflow/pytroch_job.py

@@ -58,6 +58,8 @@ class PytorchJobConverter(PytorchJobMixin, BaseConverter):
             termination=compiled_operation.termination,
             clean_pod_policy=job.clean_pod_policy,
             scheduling_policy=job.scheduling_policy,
+            elastic_policy=job.elastic_policy,
+            n_proc_per_node=job.n_proc_per_node,
             collect_logs=plugins.collect_logs,
             sync_statuses=plugins.sync_statuses,
             notifications=plugins.notifications,

polyaxon/_k8s/converter/converters/kubeflow/tf_job.py

@@ -63,6 +63,7 @@ class TfJobConverter(TFJobMixin, BaseConverter):
             collect_logs=plugins.collect_logs,
             clean_pod_policy=job.clean_pod_policy,
             scheduling_policy=job.scheduling_policy,
+            success_policy=job.success_policy,
             enable_dynamic_worker=job.enable_dynamic_worker,
             sync_statuses=plugins.sync_statuses,
             notifications=plugins.notifications,

polyaxon/_k8s/custom_resources/kubeflow/paddle_job.py

@@ -1,6 +1,11 @@
 from typing import Dict, List, Optional
 
-from polyaxon._flow import V1Notification, V1SchedulingPolicy, V1Termination
+from polyaxon._flow import (
+    V1Notification,
+    V1PaddleElasticPolicy,
+    V1SchedulingPolicy,
+    V1Termination,
+)
 from polyaxon._k8s.custom_resources.kubeflow.common import get_kf_replicas_template
 from polyaxon._k8s.custom_resources.operation import get_operation_custom_object
 from polyaxon._k8s.custom_resources.setter import (
@@ -25,6 +30,7 @@ def get_paddle_job_custom_resource(
     notifications: List[V1Notification],
     clean_pod_policy: Optional[str],
     scheduling_policy: Optional[V1SchedulingPolicy],
+    elastic_policy: Optional[V1PaddleElasticPolicy],
     labels: Dict[str, str],
     annotations: Dict[str, str],
 ) -> Dict:
@@ -58,6 +64,9 @@ def get_paddle_job_custom_resource(
         template_spec=template_spec, scheduling_policy=scheduling_policy
     )
 
+    if elastic_policy:
+        template_spec["elasticPolicy"] = elastic_policy.to_light_dict()
+
     custom_object = {"paddleJobSpec": template_spec}
     custom_object = set_termination(
         custom_object=custom_object, termination=termination

polyaxon/_k8s/custom_resources/kubeflow/pytorch_job.py

@@ -1,6 +1,11 @@
 from typing import Dict, List, Optional
 
-from polyaxon._flow import V1Notification, V1SchedulingPolicy, V1Termination
+from polyaxon._flow import (
+    V1Notification,
+    V1PytorchElasticPolicy,
+    V1SchedulingPolicy,
+    V1Termination,
+)
 from polyaxon._k8s.custom_resources.kubeflow.common import get_kf_replicas_template
 from polyaxon._k8s.custom_resources.operation import get_operation_custom_object
 from polyaxon._k8s.custom_resources.setter import (
@@ -25,6 +30,8 @@ def get_pytorch_job_custom_resource(
     notifications: List[V1Notification],
     clean_pod_policy: Optional[str],
     scheduling_policy: Optional[V1SchedulingPolicy],
+    elastic_policy: Optional[V1PytorchElasticPolicy],
+    n_proc_per_node: Optional[int],
     labels: Dict[str, str],
     annotations: Dict[str, str],
 ) -> Dict:
@@ -58,6 +65,12 @@ def get_pytorch_job_custom_resource(
         template_spec=template_spec, scheduling_policy=scheduling_policy
     )
 
+    if elastic_policy:
+        template_spec["elasticPolicy"] = elastic_policy.to_light_dict()
+
+    if n_proc_per_node is not None:
+        template_spec["nProcPerNode"] = str(n_proc_per_node)
+
     custom_object = {"pytorchJobSpec": template_spec}
     custom_object = set_termination(
         custom_object=custom_object, termination=termination

polyaxon/_k8s/custom_resources/kubeflow/tf_job.py

@@ -28,6 +28,7 @@ def get_tf_job_custom_resource(
     clean_pod_policy: Optional[str],
     scheduling_policy: Optional[V1SchedulingPolicy],
     enable_dynamic_worker: bool,
+    success_policy: Optional[str],
     labels: Dict[str, str],
     annotations: Dict[str, str],
 ) -> Dict:
@@ -73,6 +74,9 @@ def get_tf_job_custom_resource(
     if enable_dynamic_worker:
         template_spec["enableDynamicWorker"] = enable_dynamic_worker
 
+    if success_policy:
+        template_spec["successPolicy"] = success_policy
+
     template_spec = {"replicaSpecs": template_spec}
 
     template_spec = set_clean_pod_policy(

polyaxon/_k8s/executor/base.py

@@ -5,7 +5,7 @@ from kubernetes.client import Configuration
 
 from polyaxon import settings
 from polyaxon._k8s.converter.converters import CONVERTERS
-from polyaxon._k8s.converter.mixins import MIXIN_MAPPING
+from polyaxon._k8s.converter.mixins import MIXIN_MAPPING, BaseMixin
 from polyaxon._runner.executor import BaseExecutor as _BaseExecutor
 from polyaxon._runner.kinds import RunnerKind
 from polyaxon._utils.fqn_utils import get_resource_name
@@ -56,7 +56,9 @@ class BaseExecutor(_BaseExecutor):
         api = k8s_client.ApiClient()
         return api.sanitize_for_serialization(resource)
 
-    def create(self, run_uuid: str, run_kind: str, resource: Dict) -> Dict:
+    def create(
+        self, run_uuid: str, run_kind: str, resource: Dict, namespace: str = None
+    ) -> Dict:
         mixin = self._get_mixin_for_kind(kind=run_kind)
         resource_name = get_resource_name(run_uuid)
         return self.manager.create_custom_object(
@@ -65,9 +67,12 @@ class BaseExecutor(_BaseExecutor):
             version=mixin.API_VERSION,
             plural=mixin.PLURAL,
             body=resource,
+            namespace=namespace,
         )
 
-    def apply(self, run_uuid: str, run_kind: str, resource: Dict) -> Dict:
+    def apply(
+        self, run_uuid: str, run_kind: str, resource: Dict, namespace: str = None
+    ) -> Dict:
         mixin = self._get_mixin_for_kind(kind=run_kind)
         resource_name = get_resource_name(run_uuid)
         return self.manager.update_custom_object(
@@ -76,9 +81,10 @@ class BaseExecutor(_BaseExecutor):
             version=mixin.API_VERSION,
             plural=mixin.PLURAL,
             body=resource,
+            namespace=namespace,
         )
 
-    def stop(self, run_uuid: str, run_kind: str):
+    def stop(self, run_uuid: str, run_kind: str, namespace: str = None):
         mixin = self._get_mixin_for_kind(kind=run_kind)
         resource_name = get_resource_name(run_uuid)
         return self.manager.delete_custom_object(
@@ -86,16 +92,18 @@ class BaseExecutor(_BaseExecutor):
             group=mixin.GROUP,
             version=mixin.API_VERSION,
             plural=mixin.PLURAL,
+            namespace=namespace,
         )
 
-    def clean(self, run_uuid: str, run_kind: str):
+    def clean(self, run_uuid: str, run_kind: str, namespace: str = None):
         return self.apply(
             run_uuid=run_uuid,
             run_kind=run_kind,
             resource={"metadata": {"finalizers": None}},
+            namespace=namespace,
         )
 
-    def get(self, run_uuid: str, run_kind: str):
+    def get(self, run_uuid: str, run_kind: str, namespace: str = None):
         mixin = self._get_mixin_for_kind(kind=run_kind)
         resource_name = get_resource_name(run_uuid)
         return self.manager.get_custom_object(
@@ -103,4 +111,13 @@ class BaseExecutor(_BaseExecutor):
             group=mixin.GROUP,
             version=mixin.API_VERSION,
             plural=mixin.PLURAL,
+            namespace=namespace,
+        )
+
+    def list_ops(self, namespace: str = None):
+        return self.manager.list_custom_objects(
+            group=BaseMixin.GROUP,
+            version=BaseMixin.API_VERSION,
+            plural=BaseMixin.PLURAL,
+            namespace=namespace,
         )

polyaxon/_k8s/logging/async_monitor.py

@@ -86,6 +86,20 @@ async def query_k8s_operation_logs(
     return logs, new_time
 
 
+async def collect_agent_service_logs(
+    k8s_manager: AsyncK8sManager, pod: V1Pod
+) -> List[V1Log]:
+    if not pod or not pod.spec.containers:
+        return []
+    container = pod.spec.containers[0]
+    return await handle_container_logs(
+        k8s_manager=k8s_manager,
+        pod=pod,
+        container_name=container.name,
+        tail_lines=V1Logs._CHUNK_SIZE * 3,
+    )
+
+
 async def query_k8s_pod_logs(
     k8s_manager: AsyncK8sManager,
     pod: V1Pod,
@@ -107,7 +121,7 @@ async def query_k8s_pod_logs(
     return logs, last_time
 
 
-async def get_op_spec(
+async def get_op_pos_and_services(
     k8s_manager: AsyncK8sManager,
     run_uuid: str,
     run_kind: str,
@@ -115,20 +129,67 @@ async def get_op_spec(
     pods = await k8s_manager.list_pods(
         label_selector=k8s_manager.get_managed_by_polyaxon(run_uuid)
     )
+    services = []
+    if V1RunKind.has_service(run_kind):
+        services = await k8s_manager.list_services(
+            label_selector=k8s_manager.get_managed_by_polyaxon(run_uuid)
+        )
+
+    return pods, services
+
+
+async def get_op_spec(
+    k8s_manager: AsyncK8sManager,
+    run_uuid: str,
+    run_kind: str,
+):
+    pods, services = await get_op_pos_and_services(
+        k8s_manager=k8s_manager,
+        run_uuid=run_uuid,
+        run_kind=run_kind,
+    )
+    pods_list = {}
+    for pod in pods or []:
+        pods_list[
+            pod.metadata.name
+        ] = k8s_manager.api_client.sanitize_for_serialization(pod)
+    services_list = {}
+    for service in services or []:
+        services_list[
+            service.metadata.name
+        ] = k8s_manager.api_client.sanitize_for_serialization(service)
+    data = {"pods": pods_list, "services": services_list}
+    return data, pods, services
+
+
+async def get_agent_pods_and_services(
+    k8s_manager: AsyncK8sManager,
+):
+    pods = await k8s_manager.list_pods(
+        label_selector=k8s_manager.get_core_polyaxon(),
+    )
+    services = await k8s_manager.list_services(
+        label_selector=k8s_manager.get_core_polyaxon(),
+    )
+    return pods, services
+
+
+async def get_agent_spec(
+    k8s_manager: AsyncK8sManager,
+):
+    pods, services = await get_agent_pods_and_services(
+        k8s_manager=k8s_manager,
+    )
     pods_list = {}
     for pod in pods or []:
         pods_list[
             pod.metadata.name
         ] = k8s_manager.api_client.sanitize_for_serialization(pod)
     data = {"pods": pods_list}
-    if V1RunKind.has_service(run_kind):
-        services = await k8s_manager.list_services(
-            label_selector=k8s_manager.get_managed_by_polyaxon(run_uuid)
-        )
-        services_list = {}
-        for service in services or []:
-            services_list[
-                service.metadata.name
-            ] = k8s_manager.api_client.sanitize_for_serialization(service)
-        data["services"] = services_list
-    return data
+    services_list = {}
+    for service in services or []:
+        services_list[
+            service.metadata.name
+        ] = k8s_manager.api_client.sanitize_for_serialization(service)
+    data["services"] = services_list
+    return data, pods, services

polyaxon/_k8s/manager/async_manager.py

@@ -38,7 +38,7 @@ class AsyncK8sManager(BaseK8sManager):
         if self.api_client:
             await self.api_client.close()
 
-    async def get_version(self, reraise=False):
+    async def get_version(self, reraise: bool = False):
         try:
             version = await self.k8s_version_api.get_code()
             return version.to_dict()
@@ -47,48 +47,68 @@ class AsyncK8sManager(BaseK8sManager):
             if reraise:
                 raise PolyaxonK8sError("Connection error: %s" % e) from e
 
-    async def get_pod(self, name, reraise=False) -> Optional[client.V1Pod]:
+    async def get_pod(
+        self, name, reraise: bool = False, namespace: str = None
+    ) -> Optional[client.V1Pod]:
         try:
             return await self.k8s_api.read_namespaced_pod(  # type: ignore[attr-defined]
-                name=name, namespace=self.namespace
+                name=name, namespace=namespace or self.namespace
             )
         except ApiException as e:
             if reraise:
                 raise PolyaxonK8sError("Connection error: %s" % e) from e
             return None
 
-    async def is_pod_running(self, pod_id: str, container_id: str) -> bool:
-        event = await self.k8s_api.read_namespaced_pod_status(pod_id, self.namespace)  # type: ignore[attr-defined]
+    async def is_pod_running(
+        self, pod_id: str, container_id: str, namespace: str = None
+    ) -> bool:
+        event = await self.k8s_api.read_namespaced_pod_status(pod_id, namespace=namespace or self.namespace)  # type: ignore[attr-defined]
         return is_pod_running(event, container_id)
 
     async def _list_namespace_resource(
-        self, resource_api, reraise=False, **kwargs
+        self, resource_api, reraise: bool = False, namespace: str = None, **kwargs
     ) -> List:
         try:
-            res = await resource_api(namespace=self.namespace, **kwargs)
-            return [p for p in res.items]
+            res = await resource_api(namespace=namespace or self.namespace, **kwargs)
+            if isinstance(res, dict):
+                items = res["items"]
+            else:
+                items = res.items
+            return [p for p in items]
         except ApiException as e:
             logger.error("K8S error: {}".format(e))
             if reraise:
                 raise PolyaxonK8sError("Connection error: %s" % e) from e
             return []
 
-    async def list_pods(self, reraise=False, **kwargs) -> List[client.V1Pod]:
+    async def list_pods(
+        self, reraise: bool = False, namespace: str = None, **kwargs
+    ) -> List[client.V1Pod]:
         return await self._list_namespace_resource(
             resource_api=self.k8s_api.list_namespaced_pod,  # type: ignore[attr-defined]
             reraise=reraise,
+            namespace=namespace,
             **kwargs,
         )
 
-    async def list_jobs(self, reraise=False, **kwargs) -> List[client.V1Job]:
+    async def list_jobs(
+        self, reraise: bool = False, namespace: str = None, **kwargs
+    ) -> List[client.V1Job]:
         return await self._list_namespace_resource(
             resource_api=self.k8s_batch_api.list_namespaced_job,  # type: ignore[attr-defined]
             reraise=reraise,
+            namespace=namespace,
             **kwargs,
         )
 
     async def list_custom_objects(
-        self, group, version, plural, reraise=False, **kwargs
+        self,
+        group,
+        version,
+        plural,
+        reraise: bool = False,
+        namespace: str = None,
+        **kwargs
     ) -> List:
         return await self._list_namespace_resource(
             resource_api=self.k8s_custom_object_api.list_namespaced_custom_object,  # type: ignore[attr-defined]
@@ -96,49 +116,64 @@ class AsyncK8sManager(BaseK8sManager):
             group=group,
             version=version,
             plural=plural,
+            namespace=namespace,
             **kwargs,
         )
 
     async def list_services(
-        self, reraise: bool = False, **kwargs
+        self, reraise: bool = False, namespace: str = None, **kwargs
     ) -> List[client.V1Service]:
         return await self._list_namespace_resource(
             resource_api=self.k8s_api.list_namespaced_service,  # type: ignore[attr-defined]
             reraise=reraise,
+            namespace=namespace,
             **kwargs,
         )
 
     async def list_deployments(
-        self, reraise: bool = False, **kwargs
+        self, reraise: bool = False, namespace: str = None, **kwargs
     ) -> List[client.V1Deployment]:
         return await self._list_namespace_resource(
             resource_api=self.k8s_apps_api.list_namespaced_deployment,  # type: ignore[attr-defined]
             reraise=reraise,
+            namespace=namespace,
             **kwargs,
         )
 
     async def create_custom_object(
-        self, name: str, group: str, version: str, plural: str, body: Dict
+        self,
+        name: str,
+        group: str,
+        version: str,
+        plural: str,
+        body: Dict,
+        namespace: str = None,
     ) -> Dict:
         resp = await self.k8s_custom_object_api.create_namespaced_custom_object(  # type: ignore[attr-defined]
             group=group,
             version=version,
             plural=plural,
-            namespace=self.namespace,
+            namespace=namespace or self.namespace,
             body=body,
         )
         logger.debug("Custom object `{}` was created".format(name))
         return resp
 
     async def update_custom_object(
-        self, name: str, group: str, version: str, plural: str, body: Dict
+        self,
+        name: str,
+        group: str,
+        version: str,
+        plural: str,
+        body: Dict,
+        namespace: str = None,
     ) -> Dict:
         resp = await self.k8s_custom_object_api.patch_namespaced_custom_object(  # type: ignore[attr-defined]
             name=name,
             group=group,
             version=version,
             plural=plural,
-            namespace=self.namespace,
+            namespace=namespace or self.namespace,
             body=body,
             _content_type="application/merge-patch+json",
         )
@@ -153,17 +188,28 @@ class AsyncK8sManager(BaseK8sManager):
         plural: str,
         body: Dict,
         reraise: bool = False,
+        namespace: str = None,
     ) -> Tuple[Dict, bool]:
         try:
             create = await self.create_custom_object(
-                name=name, group=group, version=version, plural=plural, body=body
+                name=name,
+                group=group,
+                version=version,
+                plural=plural,
+                body=body,
+                namespace=namespace,
             )
             return create, True
 
         except ApiException as e_create:
             try:
                 update = await self.update_custom_object(
-                    name=name, group=group, version=version, plural=plural, body=body
+                    name=name,
+                    group=group,
+                    version=version,
+                    plural=plural,
+                    body=body,
+                    namespace=namespace,
                 )
                 return update, False
             except ApiException as e:
@@ -176,7 +222,13 @@ class AsyncK8sManager(BaseK8sManager):
         return {}, False
 
     async def get_custom_object(
-        self, name: str, group: str, version: str, plural: str, reraise: bool = False
+        self,
+        name: str,
+        group: str,
+        version: str,
+        plural: str,
+        reraise: bool = False,
+        namespace: str = None,
     ) -> Optional[Dict]:
         try:
             return await self.k8s_custom_object_api.get_namespaced_custom_object(
@@ -184,7 +236,7 @@ class AsyncK8sManager(BaseK8sManager):
                 group=group,
                 version=version,
                 plural=plural,
-                namespace=self.namespace,
+                namespace=namespace or self.namespace,
             )
         except ApiException as e:
             if reraise:
@@ -192,7 +244,13 @@ class AsyncK8sManager(BaseK8sManager):
             return None
 
     async def delete_custom_object(
-        self, name: str, group: str, version: str, plural: str, reraise: bool = False
+        self,
+        name: str,
+        group: str,
+        version: str,
+        plural: str,
+        reraise: bool = False,
+        namespace: str = None,
     ):
         try:
             await self.k8s_custom_object_api.delete_namespaced_custom_object(
@@ -200,7 +258,7 @@ class AsyncK8sManager(BaseK8sManager):
                 group=group,
                 version=version,
                 plural=plural,
-                namespace=self.namespace,
+                namespace=namespace or self.namespace,
                 body=client.V1DeleteOptions(),
             )
             logger.debug("Custom object `{}` deleted".format(name))

polyaxon/_k8s/manager/base.py

@@ -28,6 +28,10 @@ class BaseK8sManager:
             instance
         )
 
+    @staticmethod
+    def get_core_polyaxon() -> str:
+        return "app.kubernetes.io/part-of=polyaxon-core"
+
     @classmethod
     def get_config_auth(cls, k8s_config: Optional[Configuration] = None) -> str:
         if not k8s_config or not k8s_config.api_key: