indexify 0.3.6__tar.gz → 0.3.8__tar.gz

{indexify-0.3.6 → indexify-0.3.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: indexify
-Version: 0.3.6
+Version: 0.3.8
 Summary: Open Source Indexify components and helper tools
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0
@@ -22,7 +22,7 @@ Requires-Dist: pydantic (==2.10.4)
 Requires-Dist: pyyaml (>=6,<7)
 Requires-Dist: rich (>=13.9.2,<14.0.0)
 Requires-Dist: structlog (>=24.4.0,<25.0.0)
-Requires-Dist: tensorlake (>=0.1.13)
+Requires-Dist: tensorlake (>=0.1.16)
 Requires-Dist: typer (>=0.12,<0.13)
 Project-URL: Repository, https://github.com/tensorlakeai/indexify
 Description-Content-Type: text/markdown

{indexify-0.3.6 → indexify-0.3.8}/pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 name = "indexify"
 # Incremented if any of the components provided in this packages are updated.
-version = "0.3.6"
+version = "0.3.8"
 description = "Open Source Indexify components and helper tools"
 authors = ["Tensorlake Inc. <support@tensorlake.ai>"]
 license = "Apache 2.0"
@@ -24,7 +24,7 @@ grpcio = "1.70.0"
 pydantic = "2.10.4"
 httpx-sse = "^0.4.0"
 # Adds function-executor binary and utils lib.
-tensorlake = ">=0.1.13"
+tensorlake = ">=0.1.16"
 
 # CLI only
 rich = "^13.9.2"

{indexify-0.3.6 → indexify-0.3.8}/src/indexify/cli/cli.py

@@ -34,8 +34,6 @@ from indexify.executor.function_executor.server.subprocess_function_executor_ser
     SubprocessFunctionExecutorServerFactory,
 )
 
-logger = structlog.get_logger(module=__name__)
-
 custom_theme = Theme(
     {
         "info": "cyan",
@@ -208,6 +206,13 @@ def executor(
     ports: Tuple[int, int] = typer.Option(
         (50000, 51000), help="Range of localhost TCP ports to be used by the executor"
     ),
+    disable_automatic_function_executor_management: Annotated[
+        bool,
+        typer.Option(
+            "--disable-automatic-function-executor-management",
+            help="Disable automatic Function Executor management by Executor",
+        ),
+    ] = False,
 ):
     if dev:
         configure_development_mode_logging()
@@ -218,18 +223,20 @@ def executor(
                 "At least one function must be specified when not running in development mode"
             )
 
-    id = nanoid.generate()
     executor_version = version("indexify")
+    id = nanoid.generate()
+    logger = structlog.get_logger(module=__name__, executor_id=id)
+
     logger.info(
         "starting executor",
         server_addr=server_addr,
         config_path=config_path,
-        executor_id=id,
         executor_version=executor_version,
         executor_cache=executor_cache,
         ports=ports,
         functions=function_uris,
         dev_mode=dev,
+        disable_automatic_function_executor_management=disable_automatic_function_executor_management,
     )
 
     executor_cache = Path(executor_cache).expanduser().absolute()
@@ -258,6 +265,7 @@ def executor(
             development_mode=dev,
             server_ports=range(ports[0], ports[1]),
         ),
+        disable_automatic_function_executor_management=disable_automatic_function_executor_management,
     ).run()
 
 

{indexify-0.3.6 → indexify-0.3.8}/src/indexify/executor/executor.py

@@ -27,6 +27,7 @@ class Executor:
         function_executor_server_factory: FunctionExecutorServerFactory,
         server_addr: str = "localhost:8900",
         config_path: Optional[str] = None,
+        disable_automatic_function_executor_management: bool = False,
     ):
         self._logger = structlog.get_logger(module=__name__)
         self._is_shutdown: bool = False
@@ -40,9 +41,11 @@ class Executor:
         self._base_url = f"{protocol}://{self._server_addr}"
         self._code_path = code_path
         self._task_runner = TaskRunner(
+            executor_id=id,
             function_executor_server_factory=function_executor_server_factory,
             base_url=self._base_url,
             config_path=config_path,
+            disable_automatic_function_executor_management=disable_automatic_function_executor_management,
         )
         self._downloader = Downloader(
             code_path=code_path, base_url=self._base_url, config_path=config_path

{indexify-0.3.6 → indexify-0.3.8}/src/indexify/executor/function_executor/server/function_executor_server_factory.py

@@ -14,7 +14,8 @@ class FunctionExecutorServerConfiguration:
     configuration parameters or raise an exception if it can't implement
     them."""
 
-    def __init__(self, image_uri: Optional[str]):
+    def __init__(self, executor_id: str, image_uri: Optional[str]):
+        self.executor_id: str = executor_id
         # Container image URI of the Function Executor Server.
         self.image_uri: Optional[str] = image_uri
 

{indexify-0.3.6 → indexify-0.3.8}/src/indexify/executor/function_executor/server/subprocess_function_executor_server_factory.py

@@ -28,6 +28,8 @@ class SubprocessFunctionExecutorServerFactory(FunctionExecutorServerFactory):
         try:
             port = self._allocate_port()
             args = [
+                "--executor-id",
+                config.executor_id,
                 "--address",
                 _server_address(port),
             ]

{indexify-0.3.6 → indexify-0.3.8}/src/indexify/executor/function_executor/single_task_runner.py

@@ -25,6 +25,7 @@ from .task_output import TaskOutput
 class SingleTaskRunner:
     def __init__(
         self,
+        executor_id: str,
         function_executor_state: FunctionExecutorState,
         task_input: TaskInput,
         function_executor_server_factory: FunctionExecutorServerFactory,
@@ -32,6 +33,7 @@ class SingleTaskRunner:
         config_path: Optional[str],
         logger: Any,
     ):
+        self._executor_id: str = executor_id
         self._state: FunctionExecutorState = function_executor_state
         self._task_input: TaskInput = task_input
         self._factory: FunctionExecutorServerFactory = function_executor_server_factory
@@ -76,6 +78,7 @@ class SingleTaskRunner:
         )
         config: FunctionExecutorServerConfiguration = (
             FunctionExecutorServerConfiguration(
+                executor_id=self._executor_id,
                 image_uri=self._task_input.task.image_uri,
             )
         )

{indexify-0.3.6 → indexify-0.3.8}/src/indexify/executor/task_reporter.py

@@ -1,4 +1,5 @@
 import asyncio
+import time
 from typing import Any, List, Optional, Tuple
 
 import nanoid
@@ -74,10 +75,18 @@ class TaskReporter:
             ),
             "files": output_files if len(output_files) > 0 else FORCE_MULTIPART,
         }
+
+        start_time = time.time()
         # Run in a separate thread to not block the main event loop.
         response = await asyncio.to_thread(
             self._client.post, url=f"{self._base_url}/internal/ingest_files", **kwargs
         )
+        end_time = time.time()
+        logger.info(
+            "task_outcome_reported",
+            response_time=end_time - start_time,
+            response_code=response.status_code,
+        )
 
         try:
             response.raise_for_status()

{indexify-0.3.6 → indexify-0.3.8}/src/indexify/executor/task_runner.py

@@ -18,13 +18,19 @@ class TaskRunner:
 
     def __init__(
         self,
+        executor_id: str,
         function_executor_server_factory: FunctionExecutorServerFactory,
         base_url: str,
         config_path: Optional[str],
+        disable_automatic_function_executor_management: bool,
     ):
+        self._executor_id: str = executor_id
         self._factory: FunctionExecutorServerFactory = function_executor_server_factory
         self._base_url: str = base_url
         self._config_path: Optional[str] = config_path
+        self._disable_automatic_function_executor_management: bool = (
+            disable_automatic_function_executor_management
+        )
         # The fields below are protected by the lock.
         self._lock: asyncio.Lock = asyncio.Lock()
         self._is_shutdown: bool = False
@@ -71,6 +77,9 @@ class TaskRunner:
         #   - Each Function Executor rans at most 1 task concurrently.
         await state.wait_running_tasks_less(1)
 
+        if self._disable_automatic_function_executor_management:
+            return  # Disable Function Executor destroy in manual management mode.
+
         if state.function_id_with_version != _function_id_with_version(task):
             await state.destroy_function_executor()
             state.function_id_with_version = _function_id_with_version(task)
@@ -81,6 +90,7 @@ class TaskRunner:
         self, state: FunctionExecutorState, task_input: TaskInput, logger: Any
     ) -> TaskOutput:
         runner: SingleTaskRunner = SingleTaskRunner(
+            executor_id=self._executor_id,
             function_executor_state=state,
             task_input=task_input,
             function_executor_server_factory=self._factory,