polyaxon 2.0.6rc8__py3-none-any.whl → 2.1.0__py3-none-any.whl

polyaxon/_operations/tuner.py

@@ -31,6 +31,7 @@ def get_tuner(
 
     return V1Operation(
         queue=tuner.queue,
+        namespace=tuner.namespace,
         joins=[join],
         params=params,
         hub_ref=tuner.hub_ref,

polyaxon/_polyaxonfile/check.py

@@ -80,6 +80,7 @@ def check_polyaxonfile(
     matrix: Optional[Union[Dict, V1Matrix]] = None,
     presets: Optional[List[str]] = None,
     queue: Optional[str] = None,
+    namespace: Optional[str] = None,
     nocache: Optional[bool] = None,
     cache: Optional[Union[int, str, bool]] = None,
     verbose: bool = True,
@@ -177,6 +178,7 @@ def check_polyaxonfile(
                 matrix=matrix,
                 presets=presets,
                 queue=queue,
+                namespace=namespace,
                 nocache=nocache,
                 cache=cache,
                 approved=approved,

polyaxon/_polyaxonfile/manager/operations.py

@@ -29,6 +29,7 @@ def get_op_specification(
     matrix: Optional[Union[Dict, V1Matrix]] = None,
     presets: Optional[List[str]] = None,
     queue: Optional[str] = None,
+    namespace: Optional[str] = None,
     nocache: Optional[bool] = None,
     cache: Optional[Union[int, str, bool]] = None,
     approved: Optional[Union[int, str, bool]] = None,
@@ -70,6 +71,8 @@ def get_op_specification(
         # Check only
         get_queue_info(queue)
         op_data["queue"] = queue
+    if namespace:
+        op_data["namespace"] = namespace
     if cache is not None:
         op_data["cache"] = {"disable": not to_bool(cache)}
     if nocache:

polyaxon/_polyaxonfile/manager/workflows.py

@@ -20,6 +20,7 @@ def get_op_from_schedule(
     op_spec.skip_on_upstream_skip = None
     op_spec.cache = compiled_operation.cache
     op_spec.queue = compiled_operation.queue
+    op_spec.namespace = compiled_operation.namespace
     op_spec.component.inputs = compiled_operation.inputs
     op_spec.component.outputs = compiled_operation.outputs
     op_spec.component.run = compiled_operation.run
@@ -59,6 +60,7 @@ def get_ops_from_suggestions(
         op_spec.skip_on_upstream_skip = None
         op_spec.cache = compiled_operation.cache
         op_spec.queue = compiled_operation.queue
+        op_spec.namespace = compiled_operation.namespace
         op_spec.params = params
         op_spec.component.inputs = compiled_operation.inputs
         op_spec.component.outputs = compiled_operation.outputs

polyaxon/_polyaxonfile/specs/compiled_operation.py

@@ -333,6 +333,7 @@ class CompiledOperationSpecification(BaseSpecification):
             "is_approved",
             "presets",
             "queue",
+            "namespace",
             "cache",
             "build",
             "hooks",

polyaxon/_polyaxonfile/specs/operation.py

@@ -92,6 +92,7 @@ class OperationSpecification(BaseSpecification):
             "is_approved",
             "presets",
             "queue",
+            "namespace",
             "cache",
             "build",
             "hooks",

polyaxon/_polyaxonfile/specs/sections.py

@@ -13,6 +13,7 @@ class Sections:
     QUEUE = "queue"
     CACHE = "cache"
     PLUGINS = "plugins"
+    NAMESPACE = "namespace"
     BUILD = "build"
     HOOKS = "hooks"
     EVENTS = "events"
@@ -55,6 +56,7 @@ class Sections:
         CACHE,
         QUEUE,
         PLUGINS,
+        NAMESPACE,
         BUILD,
         HOOKS,
         EVENTS,
@@ -89,6 +91,7 @@ class Sections:
         CACHE,
         CONNECTIONS,
         PLUGINS,
+        NAMESPACE,
         TERMINATION,
         SCHEDULE,
         DEPENDENCIES,

polyaxon/_runner/agent/async_agent.py

@@ -14,6 +14,7 @@ from polyaxon._env_vars.getters import get_run_info
 from polyaxon._runner.agent.base_agent import BaseAgent
 from polyaxon._sdk.schemas.v1_agent import V1Agent
 from polyaxon._sdk.schemas.v1_agent_state_response import V1AgentStateResponse
+from polyaxon._utils.fqn_utils import get_run_instance
 from polyaxon.exceptions import ApiException as SDKApiException
 from polyaxon.exceptions import PolyaxonAgentError, PolyaxonConverterError
 from polyaxon.logger import logger
@@ -78,6 +79,49 @@ class BaseAsyncAgent(BaseAgent):
             ),
         )
 
+    async def reconcile(self):
+        if (
+            now() - self._last_reconciled_at
+        ).total_seconds() > self.SLEEP_AGENT_DATA_COLLECT_TIME:
+            return
+
+        # Collect data
+        await self.collect_agent_data()
+
+        # Update reconcile
+        namespaces = [settings.AGENT_CONFIG.namespace]
+        namespaces += settings.AGENT_CONFIG.additional_namespaces or []
+        ops = []
+        for namespace in namespaces:
+            _ops = await self.executor.list_ops(namespace=namespace)
+            if _ops:
+                ops += [
+                    (
+                        get_run_instance(
+                            owner=op["metadata"]["annotations"][
+                                "operation.polyaxon.com/owner"
+                            ],
+                            project=op["metadata"]["annotations"][
+                                "operation.polyaxon.com/project"
+                            ],
+                            run_uuid=op["metadata"]["labels"][
+                                "app.kubernetes.io/instance"
+                            ],
+                        ),
+                        op["metadata"]["annotations"]["operation.polyaxon.com/kind"],
+                        op["metadata"]["annotations"]["operation.polyaxon.com/name"],
+                        namespace,
+                    )
+                    for op in _ops
+                ]
+        if not ops:
+            return None
+
+        logger.info("Reconcile agent.")
+        return await self.client.reconcile_agent(
+            reconcile={"ops": ops},
+        )
+
     async def start(self):
         try:
             async with async_exit_context() as exit_event:
@@ -91,7 +135,9 @@ class BaseAsyncAgent(BaseAgent):
                     except asyncio.TimeoutError:
                         index += 1
                         await self.refresh_executor()
-                        if not self._default_auth:
+                        if self._default_auth:
+                            await self.reconcile()
+                        else:
                             await self.cron()
                         agent_state = await self.process()
                         if not agent_state:
@@ -118,7 +164,7 @@ class BaseAsyncAgent(BaseAgent):
                 self.sync_compatible_updates(agent_state.compatible_updates)
 
             if agent_state:
-                logger.info("Starting runs submission process.")
+                logger.info("Checking agent state.")
             else:
                 logger.info("No state was found.")
                 return V1AgentStateResponse.construct()
@@ -185,7 +231,7 @@ class BaseAsyncAgent(BaseAgent):
             )
         return None
 
-    async def submit_run(self, run_data: Tuple[str, str, str, str]):
+    async def submit_run(self, run_data: Tuple[str, str, str, str, str]):
         run_owner, run_project, run_uuid = get_run_info(run_instance=run_data[0])
         resource = await self.prepare_run_resource(
             owner_name=run_owner,
@@ -197,9 +243,13 @@ class BaseAsyncAgent(BaseAgent):
         if not resource:
             return
 
+        namespace = None if len(run_data) < 5 else run_data[4]
         try:
             await self.executor.create(
-                run_uuid=run_uuid, run_kind=run_data[1], resource=resource
+                run_uuid=run_uuid,
+                run_kind=run_data[1],
+                resource=resource,
+                namespace=namespace,
             )
         except ApiException as e:
             if e.status == 409:
@@ -222,7 +272,7 @@ class BaseAsyncAgent(BaseAgent):
             )
 
     async def make_and_create_run(
-        self, run_data: Tuple[str, str, str, str], default_auth: bool = False
+        self, run_data: Tuple[str, str, str, str, str], default_auth: bool = False
     ):
         run_owner, run_project, run_uuid = get_run_info(run_instance=run_data[0])
         resource = await self.make_run_resource(
@@ -236,9 +286,14 @@ class BaseAsyncAgent(BaseAgent):
         if not resource:
             return
 
+        namepsace = None if len(run_data) < 5 else run_data[4]
+
         try:
             await self.executor.create(
-                run_uuid=run_uuid, run_kind=run_data[1], resource=resource
+                run_uuid=run_uuid,
+                run_kind=run_data[1],
+                resource=resource,
+                namespace=namepsace,
             )
         except ApiException as e:
             if e.status == 409:
@@ -252,7 +307,7 @@ class BaseAsyncAgent(BaseAgent):
                 )
             )
 
-    async def apply_run(self, run_data: Tuple[str, str, str, str]):
+    async def apply_run(self, run_data: Tuple[str, str, str, str, str]):
         run_owner, run_project, run_uuid = get_run_info(run_instance=run_data[0])
         resource = await self.prepare_run_resource(
             owner_name=run_owner,
@@ -264,9 +319,14 @@ class BaseAsyncAgent(BaseAgent):
         if not resource:
             return
 
+        namespace = None if len(run_data) < 5 else run_data[4]
+
         try:
             await self.executor.apply(
-                run_uuid=run_uuid, run_kind=run_data[1], resource=resource
+                run_uuid=run_uuid,
+                run_kind=run_data[1],
+                resource=resource,
+                namespace=namespace,
             )
             await self.client.log_run_running(
                 run_owner=run_owner, run_project=run_project, run_uuid=run_uuid
@@ -275,12 +335,17 @@ class BaseAsyncAgent(BaseAgent):
             await self.client.log_run_failed(
                 run_owner=run_owner, run_project=run_project, run_uuid=run_uuid, exc=e
             )
-            await self.clean_run(run_uuid=run_uuid, run_kind=run_data[1])
+            await self.clean_run(
+                run_uuid=run_uuid, run_kind=run_data[1], namespace=namespace
+            )
 
-    async def check_run(self, run_data: Tuple[str, str]):
+    async def check_run(self, run_data: Tuple[str, str, str]):
         run_owner, run_project, run_uuid = get_run_info(run_instance=run_data[0])
+        namespace = None if len(run_data) < 3 else run_data[2]
         try:
-            await self.executor.get(run_uuid=run_uuid, run_kind=run_data[1])
+            await self.executor.get(
+                run_uuid=run_uuid, run_kind=run_data[1], namespace=namespace
+            )
         except ApiException as e:
             if e.status == 404:
                 logger.info(
@@ -290,10 +355,13 @@ class BaseAsyncAgent(BaseAgent):
                     run_owner=run_owner, run_project=run_project, run_uuid=run_uuid
                 )
 
-    async def stop_run(self, run_data: Tuple[str, str]):
+    async def stop_run(self, run_data: Tuple[str, str, str]):
         run_owner, run_project, run_uuid = get_run_info(run_instance=run_data[0])
+        namespace = None if len(run_data) < 3 else run_data[2]
         try:
-            await self.executor.stop(run_uuid=run_uuid, run_kind=run_data[1])
+            await self.executor.stop(
+                run_uuid=run_uuid, run_kind=run_data[1], namespace=namespace
+            )
         except ApiException as e:
             if e.status == 404:
                 logger.info("Run does not exist anymore, it could have been stopped.")
@@ -309,16 +377,24 @@ class BaseAsyncAgent(BaseAgent):
                 message="Agent failed stopping run.\n",
             )
 
-    async def delete_run(self, run_data: Tuple[str, str, str, str]):
+    async def delete_run(self, run_data: Tuple[str, str, str, str, str]):
         run_owner, run_project, run_uuid = get_run_info(run_instance=run_data[0])
-        await self.clean_run(run_uuid=run_uuid, run_kind=run_data[1])
+        namespace = None if len(run_data) < 5 else run_data[4]
         if run_data[3]:
             await self.make_and_create_run(run_data)
+        else:
+            await self.clean_run(
+                run_uuid=run_uuid, run_kind=run_data[1], namespace=namespace
+            )
 
-    async def clean_run(self, run_uuid: str, run_kind: str):
+    async def clean_run(self, run_uuid: str, run_kind: str, namespace: str = None):
         try:
-            await self.executor.clean(run_uuid=run_uuid, run_kind=run_kind)
-            await self.executor.stop(run_uuid=run_uuid, run_kind=run_kind)
+            await self.executor.clean(
+                run_uuid=run_uuid, run_kind=run_kind, namespace=namespace
+            )
+            await self.executor.stop(
+                run_uuid=run_uuid, run_kind=run_kind, namespace=namespace
+            )
         except ApiException as e:
             if e.status == 404:
                 logger.info("Run does not exist.")

polyaxon/_runner/agent/base_agent.py

@@ -24,6 +24,7 @@ class BaseAgent:
     HEALTH_FILE = "/tmp/.healthz"
     SLEEP_STOP_TIME = 60 * 5
     SLEEP_ARCHIVED_TIME = 60 * 60
+    SLEEP_AGENT_DATA_COLLECT_TIME = 60 * 30
     IS_ASYNC = False
 
     def __init__(
@@ -41,6 +42,7 @@ class BaseAgent:
         self._default_auth = bool(agent_uuid)
         self._executor_refreshed_at = now()
         self._graceful_shutdown = False
+        self._last_reconciled_at = now()
         self.client = AgentClient(
             owner=owner, agent_uuid=agent_uuid, is_async=self.IS_ASYNC
         )
@@ -50,9 +52,25 @@ class BaseAgent:
     def sync(self):
         raise NotImplementedError
 
+    def reconcile(self):
+        raise NotImplementedError
+
     def cron(self):
         return self.client.cron_agent()
 
+    def collect_agent_data(self):
+        logger.info("Collecting agent data.")
+        self._last_reconciled_at = now()
+        try:
+            return self.client.collect_agent_data(
+                namespace=settings.CLIENT_CONFIG.namespace
+            )
+        except Exception as e:
+            logger.warning(
+                "Agent failed to collect agent data: {}\n"
+                "Retrying ...".format(repr(e))
+            )
+
     def sync_compatible_updates(self, compatible_updates: Dict):
         if compatible_updates and settings.AGENT_CONFIG:
             init = compatible_updates.get("init")
@@ -173,25 +191,25 @@ class BaseAgent:
     ) -> Optional[Any]:
         raise NotImplementedError
 
-    def submit_run(self, run_data: Tuple[str, str, str, str]):
+    def submit_run(self, run_data: Tuple[str, str, str, str, str]):
         raise NotImplementedError
 
     def make_and_create_run(
-        self, run_data: Tuple[str, str, str, str], default_auth: bool = False
+        self, run_data: Tuple[str, str, str, str, str], default_auth: bool = False
     ):
         raise NotImplementedError
 
-    def apply_run(self, run_data: Tuple[str, str, str, str]):
+    def apply_run(self, run_data: Tuple[str, str, str, str, str]):
         raise NotImplementedError
 
-    def check_run(self, run_data: Tuple[str, str]):
+    def check_run(self, run_data: Tuple[str, str, str]):
         raise NotImplementedError
 
-    def stop_run(self, run_data: Tuple[str, str]):
+    def stop_run(self, run_data: Tuple[str, str, str]):
         raise NotImplementedError
 
-    def delete_run(self, run_data: Tuple[str, str, str, str]):
+    def delete_run(self, run_data: Tuple[str, str, str, str, str]):
         raise NotImplementedError
 
-    def clean_run(self, run_uuid: str, run_kind: str):
+    def clean_run(self, run_uuid: str, run_kind: str, namespace: str = None):
         raise NotImplementedError

polyaxon/_runner/agent/client.py

@@ -1,6 +1,6 @@
 import traceback
 
-from typing import Optional
+from typing import Dict, Optional
 
 from polyaxon._schemas.lifecycle import V1StatusCondition, V1Statuses
 from polyaxon.client import PolyaxonClient, V1Agent, V1AgentStateResponse
@@ -65,6 +65,20 @@ class AgentClient:
     def cron_agent(self):
         return self.client.agents_v1.cron_agent(owner=self.owner, _request_timeout=10)
 
+    def collect_agent_data(self, namespace: str):
+        return self.client.internal_agents_v1.collect_agent_data(
+            owner=self.owner,
+            uuid=self.agent_uuid,
+            namespace=namespace,
+        )
+
+    def reconcile_agent(self, reconcile: Dict):
+        return self.client.agents_v1.reconcile_agent(
+            owner=self.owner,
+            uuid=self.agent_uuid,
+            body={"reconcile": reconcile},
+        )
+
     def log_agent_running(self):
         return self.log_agent_status(status=V1Statuses.RUNNING, reason="AgentLogger")
 

polyaxon/_runner/agent/sync_agent.py

@@ -13,6 +13,7 @@ from urllib3.exceptions import HTTPError
 from polyaxon import pkg, settings
 from polyaxon._env_vars.getters import get_run_info
 from polyaxon._runner.agent.base_agent import BaseAgent
+from polyaxon._utils.fqn_utils import get_run_instance
 from polyaxon.client import V1Agent, V1AgentStateResponse
 from polyaxon.exceptions import ApiException as SDKApiException
 from polyaxon.exceptions import PolyaxonAgentError, PolyaxonConverterError
@@ -76,6 +77,49 @@ class BaseSyncAgent(BaseAgent):
             ),
         )
 
+    def reconcile(self):
+        if (
+            now() - self._last_reconciled_at
+        ).total_seconds() > self.SLEEP_AGENT_DATA_COLLECT_TIME:
+            return
+
+        # Collect data
+        self.collect_agent_data()
+
+        # Update reconcile
+        namespaces = [settings.AGENT_CONFIG.namespace]
+        namespaces += settings.AGENT_CONFIG.additional_namespaces or []
+        ops = []
+        for namespace in namespaces:
+            _ops = self.executor.list_ops(namespace=namespace)
+            if _ops:
+                ops += [
+                    (
+                        get_run_instance(
+                            owner=op["metadata"]["annotations"][
+                                "operation.polyaxon.com/owner"
+                            ],
+                            project=op["metadata"]["annotations"][
+                                "operation.polyaxon.com/project"
+                            ],
+                            run_uuid=op["metadata"]["labels"][
+                                "app.kubernetes.io/instance"
+                            ],
+                        ),
+                        op["metadata"]["annotations"]["operation.polyaxon.com/kind"],
+                        op["metadata"]["annotations"]["operation.polyaxon.com/name"],
+                        namespace,
+                    )
+                    for op in _ops
+                ]
+        if not ops:
+            return None
+
+        logger.info("Reconcile agent.")
+        return self.client.reconcile_agent(
+            reconcile={"ops": ops},
+        )
+
     def start(self):
         try:
             with sync_exit_context() as exit_event:
@@ -88,7 +132,9 @@ class BaseSyncAgent(BaseAgent):
                     while not exit_event.wait(timeout=timeout):
                         index += 1
                         self.refresh_executor()
-                        if not self._default_auth:
+                        if self._default_auth:
+                            self.reconcile()
+                        else:
                             self.cron()
                         agent_state = self.process(pool)
                         if not agent_state:
@@ -113,7 +159,7 @@ class BaseSyncAgent(BaseAgent):
                 self.sync_compatible_updates(agent_state.compatible_updates)
 
             if agent_state:
-                logger.info("Starting runs submission process.")
+                logger.info("Checking agent state.")
             else:
                 logger.info("No state was found.")
                 return V1AgentStateResponse.construct()
@@ -180,7 +226,7 @@ class BaseSyncAgent(BaseAgent):
             )
         return None
 
-    def submit_run(self, run_data: Tuple[str, str, str, str]):
+    def submit_run(self, run_data: Tuple[str, str, str, str, str]):
         run_owner, run_project, run_uuid = get_run_info(run_instance=run_data[0])
         resource = self.prepare_run_resource(
             owner_name=run_owner,
@@ -194,7 +240,10 @@ class BaseSyncAgent(BaseAgent):
 
         try:
             self.executor.create(
-                run_uuid=run_uuid, run_kind=run_data[1], resource=resource
+                run_uuid=run_uuid,
+                run_kind=run_data[1],
+                resource=resource,
+                namespace=run_data[4],
             )
         except ApiException as e:
             if e.status == 409:
@@ -217,7 +266,7 @@ class BaseSyncAgent(BaseAgent):
             )
 
     def make_and_create_run(
-        self, run_data: Tuple[str, str, str, str], default_auth: bool = False
+        self, run_data: Tuple[str, str, str, str, str], default_auth: bool = False
     ):
         run_owner, run_project, run_uuid = get_run_info(run_instance=run_data[0])
         resource = self.make_run_resource(
@@ -233,7 +282,10 @@ class BaseSyncAgent(BaseAgent):
 
         try:
             self.executor.create(
-                run_uuid=run_uuid, run_kind=run_data[1], resource=resource
+                run_uuid=run_uuid,
+                run_kind=run_data[1],
+                resource=resource,
+                namespace=run_data[4],
             )
         except ApiException as e:
             if e.status == 409:
@@ -247,7 +299,7 @@ class BaseSyncAgent(BaseAgent):
                 )
             )
 
-    def apply_run(self, run_data: Tuple[str, str, str, str]):
+    def apply_run(self, run_data: Tuple[str, str, str, str, str]):
         run_owner, run_project, run_uuid = get_run_info(run_instance=run_data[0])
         resource = self.prepare_run_resource(
             owner_name=run_owner,
@@ -261,7 +313,10 @@ class BaseSyncAgent(BaseAgent):
 
         try:
             self.executor.apply(
-                run_uuid=run_uuid, run_kind=run_data[1], resource=resource
+                run_uuid=run_uuid,
+                run_kind=run_data[1],
+                resource=resource,
+                namespace=run_data[4],
             )
             self.client.log_run_running(
                 run_owner=run_owner, run_project=run_project, run_uuid=run_uuid
@@ -270,12 +325,16 @@ class BaseSyncAgent(BaseAgent):
             self.client.log_run_failed(
                 run_owner=run_owner, run_project=run_project, run_uuid=run_uuid, exc=e
             )
-            self.clean_run(run_uuid=run_uuid, run_kind=run_data[1])
+            self.clean_run(
+                run_uuid=run_uuid, run_kind=run_data[1], namespace=run_data[4]
+            )
 
-    def check_run(self, run_data: Tuple[str, str]):
+    def check_run(self, run_data: Tuple[str, str, str]):
         run_owner, run_project, run_uuid = get_run_info(run_instance=run_data[0])
         try:
-            self.executor.get(run_uuid=run_uuid, run_kind=run_data[1])
+            self.executor.get(
+                run_uuid=run_uuid, run_kind=run_data[1], namespace=run_data[2]
+            )
         except ApiException as e:
             if e.status == 404:
                 logger.info(
@@ -285,10 +344,12 @@ class BaseSyncAgent(BaseAgent):
                     run_owner=run_owner, run_project=run_project, run_uuid=run_uuid
                 )
 
-    def stop_run(self, run_data: Tuple[str, str]):
+    def stop_run(self, run_data: Tuple[str, str, str]):
         run_owner, run_project, run_uuid = get_run_info(run_instance=run_data[0])
         try:
-            self.executor.stop(run_uuid=run_uuid, run_kind=run_data[1])
+            self.executor.stop(
+                run_uuid=run_uuid, run_kind=run_data[1], namespace=run_data[2]
+            )
         except ApiException as e:
             if e.status == 404:
                 logger.info("Run does not exist anymore, it could have been stopped.")
@@ -304,16 +365,20 @@ class BaseSyncAgent(BaseAgent):
                 message="Agent failed stopping run.\n",
             )
 
-    def delete_run(self, run_data: Tuple[str, str, str, str]):
+    def delete_run(self, run_data: Tuple[str, str, str, str, str]):
         run_owner, run_project, run_uuid = get_run_info(run_instance=run_data[0])
-        self.clean_run(run_uuid=run_uuid, run_kind=run_data[1])
+        self.clean_run(run_uuid=run_uuid, run_kind=run_data[1], namespace=run_data[4])
         if run_data[3]:
             self.make_and_create_run(run_data)
 
-    def clean_run(self, run_uuid: str, run_kind: str):
+    def clean_run(self, run_uuid: str, run_kind: str, namespace: str = None):
         try:
-            self.executor.clean(run_uuid=run_uuid, run_kind=run_kind)
-            self.executor.stop(run_uuid=run_uuid, run_kind=run_kind)
+            self.executor.clean(
+                run_uuid=run_uuid, run_kind=run_kind, namespace=namespace
+            )
+            self.executor.stop(
+                run_uuid=run_uuid, run_kind=run_kind, namespace=namespace
+            )
         except ApiException as e:
             if e.status == 404:
                 logger.info("Run does not exist.")

polyaxon/_runner/executor.py

@@ -50,19 +50,21 @@ class BaseExecutor:
         self._manager = None
         return self.manager
 
-    def get(self, run_uuid: str, run_kind: str):
+    def get(self, run_uuid: str, run_kind: str, namespace: str = None):
         raise NotImplementedError
 
-    def create(self, run_uuid: str, run_kind: str, resource: Any):
+    def create(
+        self, run_uuid: str, run_kind: str, resource: Any, namespace: str = None
+    ):
         raise NotImplementedError
 
-    def apply(self, run_uuid: str, run_kind: str, resource: Any):
+    def apply(self, run_uuid: str, run_kind: str, resource: Any, namespace: str = None):
         raise NotImplementedError
 
-    def stop(self, run_uuid: str, run_kind: str):
+    def stop(self, run_uuid: str, run_kind: str, namespace: str = None):
         raise NotImplementedError
 
-    def clean(self, run_uuid: str, run_kind: str):
+    def clean(self, run_uuid: str, run_kind: str, namespace: str = None):
         raise NotImplementedError
 
     def _clean_temp_execution_path(self, run_uuid: str):
@@ -154,7 +156,7 @@ class BaseExecutor:
             run_name=run_name,
             run_uuid=run_uuid,
             run_path=run_uuid,
-            namespace=agent_env.namespace,
+            namespace=compiled_operation.namespace or agent_env.namespace,
             compiled_operation=compiled_operation,
             polyaxon_init=agent_env.polyaxon_init,
             polyaxon_sidecar=agent_env.polyaxon_sidecar,
@@ -191,7 +193,7 @@ class BaseExecutor:
             params=operation.params,
         )
         return cls.get_resource(
-            namespace=resolver_obj.namespace,
+            namespace=compiled_operation.namespace or resolver_obj.namespace,
             owner_name=resolver_obj.owner_name,
             project_name=resolver_obj.project_name,
             run_name=resolver_obj.run_name,
@@ -223,4 +225,8 @@ class BaseExecutor:
             run_uuid=response.uuid,
             run_kind=response.kind,
             resource=resource,
+            namespace=response.namespace,
         )
+
+    def list_ops(self, namespace: str = None):
+        raise NotImplementedError