indexify 0.3.13__py3-none-any.whl → 0.3.15__py3-none-any.whl

indexify/cli/cli.py

@@ -208,13 +208,17 @@ def executor(
             help="Port where to run Executor Monitoring server",
         ),
     ] = 7000,
-    disable_automatic_function_executor_management: Annotated[
-        bool,
+    grpc_server_addr: Annotated[
+        Optional[str],
         typer.Option(
-            "--disable-automatic-function-executor-management",
-            help="Disable automatic Function Executor management by Executor",
+            "--grpc-server-addr",
+            help=(
+                "(exprimental) Address of server gRPC API to connect to, e.g. 'localhost:8901'.\n"
+                "If set disables automatic Function Executor management on Executor and uses the Server gRPC API\n"
+                "for Function Executor management and placement of tasks on them."
+            ),
         ),
-    ] = False,
+    ] = None,
 ):
     if dev:
         configure_development_mode_logging()
@@ -247,7 +251,7 @@ def executor(
         dev_mode=dev,
         monitoring_server_host=monitoring_server_host,
         monitoring_server_port=monitoring_server_port,
-        disable_automatic_function_executor_management=disable_automatic_function_executor_management,
+        grpc_server_addr=grpc_server_addr,
     )
 
     executor_cache = Path(executor_cache).expanduser().absolute()
@@ -285,7 +289,7 @@ def executor(
         config_path=config_path,
         monitoring_server_host=monitoring_server_host,
         monitoring_server_port=monitoring_server_port,
-        disable_automatic_function_executor_management=disable_automatic_function_executor_management,
+        grpc_server_addr=grpc_server_addr,
     ).run()
 
 

indexify/executor/downloader.py

@@ -3,7 +3,7 @@ import os
 from typing import Any, Optional
 
 import httpx
-import structlog
+import nanoid
 from tensorlake.function_executor.proto.function_executor_pb2 import SerializedObject
 from tensorlake.utils.http_client import get_httpx_client
 
@@ -33,41 +33,81 @@ class Downloader:
         self._base_url = base_url
         self._client = get_httpx_client(config_path, make_async=True)
 
-    async def download_graph(self, task: Task) -> SerializedObject:
+    async def download_graph(
+        self, namespace: str, graph_name: str, graph_version: str, logger: Any
+    ) -> SerializedObject:
+        logger = logger.bind(module=__name__)
         with (
             metric_graph_download_errors.count_exceptions(),
             metric_tasks_downloading_graphs.track_inprogress(),
             metric_graph_download_latency.time(),
         ):
             metric_graph_downloads.inc()
-            return await self._download_graph(task)
+            return await self._download_graph(
+                namespace=namespace,
+                graph_name=graph_name,
+                graph_version=graph_version,
+                logger=logger,
+            )
 
-    async def download_input(self, task: Task) -> SerializedObject:
+    async def download_input(
+        self,
+        namespace: str,
+        graph_name: str,
+        graph_invocation_id: str,
+        input_key: str,
+        logger: Any,
+    ) -> SerializedObject:
+        logger = logger.bind(module=__name__)
         with (
             metric_task_input_download_errors.count_exceptions(),
             metric_tasks_downloading_inputs.track_inprogress(),
             metric_task_input_download_latency.time(),
         ):
             metric_task_input_downloads.inc()
-            return await self._download_input(task)
+            return await self._download_input(
+                namespace=namespace,
+                graph_name=graph_name,
+                graph_invocation_id=graph_invocation_id,
+                input_key=input_key,
+                logger=logger,
+            )
 
-    async def download_init_value(self, task: Task) -> SerializedObject:
+    async def download_init_value(
+        self,
+        namespace: str,
+        graph_name: str,
+        function_name: str,
+        graph_invocation_id: str,
+        reducer_output_key: str,
+        logger: Any,
+    ) -> SerializedObject:
+        logger = logger.bind(module=__name__)
         with (
             metric_reducer_init_value_download_errors.count_exceptions(),
             metric_tasks_downloading_reducer_init_value.track_inprogress(),
             metric_reducer_init_value_download_latency.time(),
         ):
             metric_reducer_init_value_downloads.inc()
-            return await self._download_init_value(task)
+            return await self._fetch_function_init_value(
+                namespace=namespace,
+                graph_name=graph_name,
+                function_name=function_name,
+                graph_invocation_id=graph_invocation_id,
+                reducer_output_key=reducer_output_key,
+                logger=logger,
+            )
 
-    async def _download_graph(self, task: Task) -> SerializedObject:
+    async def _download_graph(
+        self, namespace: str, graph_name: str, graph_version: str, logger: Any
+    ) -> SerializedObject:
         # Cache graph to reduce load on the server.
         graph_path = os.path.join(
             self.code_path,
             "graph_cache",
-            task.namespace,
-            task.compute_graph,
-            task.graph_version,
+            namespace,
+            graph_name,
+            graph_version,
         )
         # Filesystem operations are synchronous.
         # Run in a separate thread to not block the main event loop.
@@ -78,13 +118,17 @@ class Downloader:
             metric_graphs_from_cache.inc()
             return graph
 
-        logger = self._task_logger(task)
-        graph: SerializedObject = await self._fetch_graph(task, logger)
+        graph: SerializedObject = await self._fetch_graph(
+            namespace=namespace,
+            graph_name=graph_name,
+            graph_version=graph_version,
+            logger=logger,
+        )
         # Filesystem operations are synchronous.
         # Run in a separate thread to not block the main event loop.
         # We don't need to wait for the write completion so we use create_task.
         asyncio.create_task(
-            asyncio.to_thread(self._write_cached_graph, task, graph_path, graph)
+            asyncio.to_thread(self._write_cached_graph, graph_path, graph)
         )
 
         return graph
@@ -96,14 +140,12 @@ class Downloader:
         with open(path, "rb") as f:
             return SerializedObject.FromString(f.read())
 
-    def _write_cached_graph(
-        self, task: Task, path: str, graph: SerializedObject
-    ) -> None:
+    def _write_cached_graph(self, path: str, graph: SerializedObject) -> None:
         if os.path.exists(path):
             # Another task already cached the graph.
             return None
 
-        tmp_path = os.path.join(self.code_path, "task_graph_cache", task.id)
+        tmp_path = os.path.join(self.code_path, "task_graph_cache", nanoid.generate())
         os.makedirs(os.path.dirname(tmp_path), exist_ok=True)
         with open(tmp_path, "wb") as f:
             f.write(graph.SerializeToString())
@@ -114,60 +156,67 @@ class Downloader:
         # This also allows to share the same cache between multiple Executors.
         os.replace(tmp_path, path)
 
-    async def _download_input(self, task: Task) -> SerializedObject:
-        logger = self._task_logger(task)
-
-        first_function_in_graph = task.invocation_id == task.input_key.split("|")[-1]
+    async def _download_input(
+        self,
+        namespace: str,
+        graph_name: str,
+        graph_invocation_id: str,
+        input_key: str,
+        logger: Any,
+    ) -> SerializedObject:
+        first_function_in_graph = graph_invocation_id == input_key.split("|")[-1]
         if first_function_in_graph:
             # The first function in Graph gets its input from graph invocation payload.
-            return await self._fetch_graph_invocation_payload(task, logger)
+            return await self._fetch_graph_invocation_payload(
+                namespace=namespace,
+                graph_name=graph_name,
+                graph_invocation_id=graph_invocation_id,
+                logger=logger,
+            )
         else:
-            return await self._fetch_function_input(task, logger)
-
-    async def _download_init_value(self, task: Task) -> SerializedObject:
-        logger = self._task_logger(task)
-        return await self._fetch_function_init_value(task, logger)
-
-    def _task_logger(self, task: Task) -> Any:
-        return structlog.get_logger(
-            module=__name__,
-            namespace=task.namespace,
-            name=task.compute_graph,
-            version=task.graph_version,
-            task_id=task.id,
-        )
+            return await self._fetch_function_input(input_key=input_key, logger=logger)
 
-    async def _fetch_graph(self, task: Task, logger: Any) -> SerializedObject:
+    async def _fetch_graph(
+        self, namespace: str, graph_name: str, graph_version: str, logger: Any
+    ) -> SerializedObject:
         """Downloads the compute graph for the task and returns it."""
         return await self._fetch_url(
-            url=f"{self._base_url}/internal/namespaces/{task.namespace}/compute_graphs/{task.compute_graph}/versions/{task.graph_version}/code",
-            resource_description=f"compute graph: {task.compute_graph}",
+            url=f"{self._base_url}/internal/namespaces/{namespace}/compute_graphs/{graph_name}/versions/{graph_version}/code",
+            resource_description=f"compute graph: {graph_name}",
             logger=logger,
         )
 
     async def _fetch_graph_invocation_payload(
-        self, task: Task, logger: Any
+        self, namespace: str, graph_name: str, graph_invocation_id: str, logger: Any
     ) -> SerializedObject:
         return await self._fetch_url(
-            url=f"{self._base_url}/namespaces/{task.namespace}/compute_graphs/{task.compute_graph}/invocations/{task.invocation_id}/payload",
-            resource_description=f"graph invocation payload: {task.invocation_id}",
+            url=f"{self._base_url}/namespaces/{namespace}/compute_graphs/{graph_name}/invocations/{graph_invocation_id}/payload",
+            resource_description=f"graph invocation payload: {graph_invocation_id}",
             logger=logger,
         )
 
-    async def _fetch_function_input(self, task: Task, logger: Any) -> SerializedObject:
+    async def _fetch_function_input(
+        self, input_key: str, logger: Any
+    ) -> SerializedObject:
         return await self._fetch_url(
-            url=f"{self._base_url}/internal/fn_outputs/{task.input_key}",
-            resource_description=f"function input: {task.input_key}",
+            url=f"{self._base_url}/internal/fn_outputs/{input_key}",
+            resource_description=f"function input: {input_key}",
             logger=logger,
         )
 
     async def _fetch_function_init_value(
-        self, task: Task, logger: Any
+        self,
+        namespace: str,
+        graph_name: str,
+        function_name: str,
+        graph_invocation_id: str,
+        reducer_output_key: str,
+        logger: Any,
     ) -> SerializedObject:
         return await self._fetch_url(
-            url=f"{self._base_url}/namespaces/{task.namespace}/compute_graphs/{task.compute_graph}"
-            f"/invocations/{task.invocation_id}/fn/{task.compute_fn}/output/{task.reducer_output_id}",
-            resource_description=f"reducer output: {task.reducer_output_id}",
+            url=f"{self._base_url}/namespaces/{namespace}/compute_graphs/{graph_name}"
+            f"/invocations/{graph_invocation_id}/fn/{function_name}/output/{reducer_output_key}",
+            resource_description=f"reducer output: {reducer_output_key}",
             logger=logger,
         )
 

indexify/executor/executor.py

@@ -5,6 +5,7 @@ from pathlib import Path
 from socket import gethostname
 from typing import Any, Dict, List, Optional
 
+import grpc
 import structlog
 from tensorlake.function_executor.proto.function_executor_pb2 import SerializedObject
 from tensorlake.utils.logging import suppress as suppress_logging
@@ -38,10 +39,14 @@ from .monitoring.health_checker.health_checker import HealthChecker
 from .monitoring.prometheus_metrics_handler import PrometheusMetricsHandler
 from .monitoring.server import MonitoringServer
 from .monitoring.startup_probe_handler import StartupProbeHandler
+from .state_reconciler import ExecutorStateReconciler
+from .state_reporter import ExecutorStateReporter
 from .task_fetcher import TaskFetcher
 from .task_reporter import TaskReporter
 from .task_runner import TaskInput, TaskOutput, TaskRunner
 
+EXECUTOR_GRPC_SERVER_READY_TIMEOUT_SEC = 10
+
 metric_executor_state.state("starting")
 
 
@@ -58,7 +63,7 @@ class Executor:
         config_path: Optional[str],
         monitoring_server_host: str,
         monitoring_server_port: int,
-        disable_automatic_function_executor_management: bool,
+        grpc_server_addr: Optional[str],
     ):
         self._logger = structlog.get_logger(module=__name__)
         self._is_shutdown: bool = False
@@ -83,39 +88,45 @@ class Executor:
         health_checker.set_function_executor_states_container(
             self._function_executor_states
         )
-        self._task_runner = TaskRunner(
-            executor_id=id,
-            function_executor_server_factory=function_executor_server_factory,
-            base_url=self._base_url,
-            disable_automatic_function_executor_management=disable_automatic_function_executor_management,
-            function_executor_states=self._function_executor_states,
-            config_path=config_path,
-        )
         self._downloader = Downloader(
             code_path=code_path, base_url=self._base_url, config_path=config_path
         )
-        self._task_fetcher = TaskFetcher(
-            executor_id=id,
-            executor_version=version,
-            function_allowlist=function_allowlist,
-            protocol=protocol,
-            indexify_server_addr=self._server_addr,
-            config_path=config_path,
-        )
         self._task_reporter = TaskReporter(
             base_url=self._base_url,
             executor_id=id,
             config_path=self._config_path,
         )
+        self._grpc_server_addr: Optional[str] = grpc_server_addr
+        self._id = id
+        self._function_allowlist: Optional[List[FunctionURI]] = function_allowlist
+        self._function_executor_server_factory = function_executor_server_factory
+        self._state_reporter: Optional[ExecutorStateReporter] = None
+        self._state_reconciler: Optional[ExecutorStateReconciler] = None
+
+        if self._grpc_server_addr is None:
+            self._task_runner: Optional[TaskRunner] = TaskRunner(
+                executor_id=id,
+                function_executor_server_factory=function_executor_server_factory,
+                base_url=self._base_url,
+                function_executor_states=self._function_executor_states,
+                config_path=config_path,
+            )
+            self._task_fetcher: Optional[TaskFetcher] = TaskFetcher(
+                executor_id=id,
+                executor_version=version,
+                function_allowlist=function_allowlist,
+                protocol=protocol,
+                indexify_server_addr=self._server_addr,
+                config_path=config_path,
+            )
+
         executor_info: Dict[str, str] = {
             "id": id,
             "version": version,
             "code_path": str(code_path),
             "server_addr": server_addr,
             "config_path": str(config_path),
-            "disable_automatic_function_executor_management": str(
-                disable_automatic_function_executor_management
-            ),
+            "grpc_server_addr": str(grpc_server_addr),
             "hostname": gethostname(),
         }
         executor_info.update(function_allowlist_to_info_dict(function_allowlist))
@@ -137,18 +148,88 @@ class Executor:
         asyncio.get_event_loop().create_task(self._monitoring_server.run())
 
         try:
-            asyncio.get_event_loop().run_until_complete(self._run_tasks_loop())
+            if self._grpc_server_addr is None:
+                asyncio.get_event_loop().run_until_complete(self._http_mode_loop())
+            else:
+                asyncio.get_event_loop().run_until_complete(self._grpc_mode_loop())
         except asyncio.CancelledError:
             pass  # Suppress this expected exception and return without error (normally).
 
-    async def _run_tasks_loop(self):
+    async def _grpc_mode_loop(self):
+        metric_executor_state.state("running")
+        self._startup_probe_handler.set_ready()
+
+        while not self._is_shutdown:
+            async with self._establish_grpc_server_channel() as server_channel:
+                server_channel: grpc.aio.Channel
+                await self._run_grpc_mode_services(server_channel)
+                self._logger.warning(
+                    "grpc mode services exited, retrying in 5 seconds",
+                )
+                await asyncio.sleep(5)
+
+    async def _establish_grpc_server_channel(self) -> grpc.aio.Channel:
+        try:
+            channel = grpc.aio.insecure_channel(self._grpc_server_addr)
+            await asyncio.wait_for(
+                channel.channel_ready(),
+                timeout=EXECUTOR_GRPC_SERVER_READY_TIMEOUT_SEC,
+            )
+            return channel
+        except Exception as e:
+            self._logger.error("failed establishing grpc server channel", exc_info=e)
+            raise
+
+    async def _run_grpc_mode_services(self, server_channel: grpc.aio.Channel):
+        """Runs the gRPC mode services.
+
+        Never raises any exceptions."""
+        try:
+            self._state_reporter = ExecutorStateReporter(
+                executor_id=self._id,
+                function_allowlist=self._function_allowlist,
+                function_executor_states=self._function_executor_states,
+                server_channel=server_channel,
+                logger=self._logger,
+            )
+            self._state_reconciler = ExecutorStateReconciler(
+                executor_id=self._id,
+                function_executor_server_factory=self._function_executor_server_factory,
+                base_url=self._base_url,
+                function_executor_states=self._function_executor_states,
+                config_path=self._config_path,
+                downloader=self._downloader,
+                task_reporter=self._task_reporter,
+                server_channel=server_channel,
+                logger=self._logger,
+            )
+
+            # Task group ensures that:
+            # 1. If one of the tasks fails then the other tasks are cancelled.
+            # 2. If Executor shuts down then all the tasks are cancelled and this function returns.
+            async with asyncio.TaskGroup() as tg:
+                tg.create_task(self._state_reporter.run())
+                tg.create_task(self._state_reconciler.run())
+        except Exception as e:
+            self._logger.error("failed running grpc mode services", exc_info=e)
+        finally:
+            # Handle task cancellation using finally.
+            if self._state_reporter is not None:
+                self._state_reporter.shutdown()
+                self._state_reporter = None
+            if self._state_reconciler is not None:
+                self._state_reconciler.shutdown()
+                self._state_reconciler = None
+
+    async def _http_mode_loop(self):
         metric_executor_state.state("running")
         self._startup_probe_handler.set_ready()
         while not self._is_shutdown:
             try:
                 async for task in self._task_fetcher.run():
                     metric_tasks_fetched.inc()
-                    asyncio.create_task(self._run_task(task))
+                    if not self._is_shutdown:
+                        asyncio.create_task(self._run_task(task))
             except Exception as e:
                 self._logger.error(
                     "failed fetching tasks, retrying in 5 seconds", exc_info=e
@@ -167,7 +248,14 @@ class Executor:
             output = await self._run_task_and_get_output(task, logger)
             logger.info("task execution finished", success=output.success)
         except Exception as e:
-            output = TaskOutput.internal_error(task)
+            output = TaskOutput.internal_error(
+                task_id=task.id,
+                namespace=task.namespace,
+                graph_name=task.compute_graph,
+                function_name=task.compute_fn,
+                graph_version=task.graph_version,
+                graph_invocation_id=task.invocation_id,
+            )
             logger.error("task execution failed", exc_info=e)
 
         with (
@@ -180,12 +268,32 @@ class Executor:
         metric_task_completion_latency.observe(time.monotonic() - start_time)
 
     async def _run_task_and_get_output(self, task: Task, logger: Any) -> TaskOutput:
-        graph: SerializedObject = await self._downloader.download_graph(task)
-        input: SerializedObject = await self._downloader.download_input(task)
+        graph: SerializedObject = await self._downloader.download_graph(
+            namespace=task.namespace,
+            graph_name=task.compute_graph,
+            graph_version=task.graph_version,
+            logger=logger,
+        )
+        input: SerializedObject = await self._downloader.download_input(
+            namespace=task.namespace,
+            graph_name=task.compute_graph,
+            graph_invocation_id=task.invocation_id,
+            input_key=task.input_key,
+            logger=logger,
+        )
         init_value: Optional[SerializedObject] = (
             None
             if task.reducer_output_id is None
-            else (await self._downloader.download_init_value(task))
+            else (
+                await self._downloader.download_init_value(
+                    namespace=task.namespace,
+                    graph_name=task.compute_graph,
+                    function_name=task.compute_fn,
+                    graph_invocation_id=task.invocation_id,
+                    reducer_output_key=task.reducer_output_id,
+                    logger=logger,
+                )
+            )
         )
         return await self._task_runner.run(
             TaskInput(
@@ -241,11 +349,24 @@ class Executor:
 
         self._is_shutdown = True
         await self._monitoring_server.shutdown()
-        await self._task_runner.shutdown()
+
+        if self._task_runner is not None:
+            await self._task_runner.shutdown()
+        if self._state_reporter is not None:
+            await self._state_reporter.shutdown()
+            self._state_reporter = None
+        if self._state_reconciler is not None:
+            await self._state_reconciler.shutdown()
+            self._state_reconciler = None
+
+        # We need to shutdown all users of FE states first,
+        # otherwise states might disappear unexpectedly and we might
+        # report errors, etc that are expected.
         await self._function_executor_states.shutdown()
-        # We mainly need to cancel the task that runs _run_tasks_loop().
+        # We mainly need to cancel the task that runs _.*_mode_loop().
         for task in asyncio.all_tasks(loop):
             task.cancel()
+        # The current task is cancelled, the code after this line will not run.
 
     def shutdown(self, loop):
         loop.create_task(self._shutdown(loop))

indexify/executor/function_executor/function_executor_state.py

@@ -1,6 +1,8 @@
 import asyncio
 from typing import Optional
 
+from indexify.task_scheduler.proto.task_scheduler_pb2 import FunctionExecutorStatus
+
 from .function_executor import FunctionExecutor
 from .metrics.function_executor_state import (
     metric_function_executor_state_not_locked_errors,
@@ -15,14 +17,31 @@ class FunctionExecutorState:
     under the lock.
     """
 
-    def __init__(self, function_id_with_version: str, function_id_without_version: str):
-        self.function_id_with_version: str = function_id_with_version
-        self.function_id_without_version: str = function_id_without_version
-        # All the fields below are protected by the lock.
+    def __init__(
+        self,
+        id: str,
+        namespace: str,
+        graph_name: str,
+        graph_version: str,
+        function_name: str,
+        image_uri: Optional[str],
+    ):
+        # Read only fields.
+        self.id: str = id
+        self.namespace: str = namespace
+        self.graph_name: str = graph_name
+        self.function_name: str = function_name
+        self.image_uri: Optional[str] = image_uri
+        # The lock must be held while modifying the fields below.
         self.lock: asyncio.Lock = asyncio.Lock()
+        self.graph_version: str = graph_version
         self.is_shutdown: bool = False
         # Set to True if a Function Executor health check ever failed.
         self.health_check_failed: bool = False
+        # TODO: remove fields that duplicate this status field.
+        self.status: FunctionExecutorStatus = (
+            FunctionExecutorStatus.FUNCTION_EXECUTOR_STATUS_STOPPED
+        )
         self.function_executor: Optional[FunctionExecutor] = None
         self.running_tasks: int = 0
         self.running_tasks_change_notifier: asyncio.Condition = asyncio.Condition(

indexify/executor/function_executor/function_executor_states_container.py

@@ -1,7 +1,6 @@
 import asyncio
-from typing import AsyncGenerator, Dict
+from typing import AsyncGenerator, Dict, Optional
 
-from ..api_objects import Task
 from .function_executor_state import FunctionExecutorState
 from .metrics.function_executor_state_container import (
     metric_function_executor_states_count,
@@ -17,19 +16,33 @@ class FunctionExecutorStatesContainer:
         self._states: Dict[str, FunctionExecutorState] = {}
         self._is_shutdown: bool = False
 
-    async def get_or_create_state(self, task: Task) -> FunctionExecutorState:
-        """Get or create a function executor state for the given task.
+    async def get_or_create_state(
+        self,
+        id: str,
+        namespace: str,
+        graph_name: str,
+        graph_version: str,
+        function_name: str,
+        image_uri: Optional[str],
+    ) -> FunctionExecutorState:
+        """Get or create a function executor state with the given ID.
 
+        If the state already exists, it is returned. Otherwise, a new state is created from the supplied task.
         Raises Exception if it's not possible to create a new state at this time."""
         async with self._lock:
             if self._is_shutdown:
-                raise RuntimeError("Task runner is shutting down.")
+                raise RuntimeError(
+                    "Function Executor states container is shutting down."
+                )
 
-            id = function_id_without_version(task)
             if id not in self._states:
                 state = FunctionExecutorState(
-                    function_id_with_version=function_id_with_version(task),
-                    function_id_without_version=id,
+                    id=id,
+                    namespace=namespace,
+                    graph_name=graph_name,
+                    graph_version=graph_version,
+                    function_name=function_name,
+                    image_uri=image_uri,
                 )
                 self._states[id] = state
                 metric_function_executor_states_count.set(len(self._states))
@@ -41,6 +54,13 @@ class FunctionExecutorStatesContainer:
             for state in self._states.values():
                 yield state
 
+    async def pop(self, id: str) -> FunctionExecutorState:
+        """Removes the state with the given ID and returns it."""
+        async with self._lock:
+            state = self._states.pop(id)
+            metric_function_executor_states_count.set(len(self._states))
+            return state
+
     async def shutdown(self):
         # Function Executors are outside the Executor process
         # so they need to get cleaned up explicitly and reliably.
@@ -54,11 +74,3 @@ class FunctionExecutorStatesContainer:
                 async with state.lock:
                     await state.shutdown()
                     # The task running inside the Function Executor will fail because it's destroyed.
-
-
-def function_id_with_version(task: Task) -> str:
-    return f"versioned/{task.namespace}/{task.compute_graph}/{task.graph_version}/{task.compute_fn}"
-
-
-def function_id_without_version(task: Task) -> str:
-    return f"not_versioned/{task.namespace}/{task.compute_graph}/{task.compute_fn}"