indexify 0.3.6__py3-none-any.whl → 0.3.8__py3-none-any.whl

indexify/cli/cli.py

@@ -34,8 +34,6 @@ from indexify.executor.function_executor.server.subprocess_function_executor_ser
     SubprocessFunctionExecutorServerFactory,
 )
 
-logger = structlog.get_logger(module=__name__)
-
 custom_theme = Theme(
     {
         "info": "cyan",
@@ -208,6 +206,13 @@ def executor(
     ports: Tuple[int, int] = typer.Option(
         (50000, 51000), help="Range of localhost TCP ports to be used by the executor"
     ),
+    disable_automatic_function_executor_management: Annotated[
+        bool,
+        typer.Option(
+            "--disable-automatic-function-executor-management",
+            help="Disable automatic Function Executor management by Executor",
+        ),
+    ] = False,
 ):
     if dev:
         configure_development_mode_logging()
@@ -218,18 +223,20 @@ def executor(
                 "At least one function must be specified when not running in development mode"
             )
 
-    id = nanoid.generate()
     executor_version = version("indexify")
+    id = nanoid.generate()
+    logger = structlog.get_logger(module=__name__, executor_id=id)
+
     logger.info(
         "starting executor",
         server_addr=server_addr,
         config_path=config_path,
-        executor_id=id,
         executor_version=executor_version,
         executor_cache=executor_cache,
         ports=ports,
         functions=function_uris,
         dev_mode=dev,
+        disable_automatic_function_executor_management=disable_automatic_function_executor_management,
     )
 
     executor_cache = Path(executor_cache).expanduser().absolute()
@@ -258,6 +265,7 @@ def executor(
             development_mode=dev,
             server_ports=range(ports[0], ports[1]),
         ),
+        disable_automatic_function_executor_management=disable_automatic_function_executor_management,
     ).run()
 
 

indexify/executor/executor.py

@@ -27,6 +27,7 @@ class Executor:
         function_executor_server_factory: FunctionExecutorServerFactory,
         server_addr: str = "localhost:8900",
         config_path: Optional[str] = None,
+        disable_automatic_function_executor_management: bool = False,
     ):
         self._logger = structlog.get_logger(module=__name__)
         self._is_shutdown: bool = False
@@ -40,9 +41,11 @@ class Executor:
         self._base_url = f"{protocol}://{self._server_addr}"
         self._code_path = code_path
         self._task_runner = TaskRunner(
+            executor_id=id,
             function_executor_server_factory=function_executor_server_factory,
             base_url=self._base_url,
             config_path=config_path,
+            disable_automatic_function_executor_management=disable_automatic_function_executor_management,
         )
         self._downloader = Downloader(
             code_path=code_path, base_url=self._base_url, config_path=config_path

indexify/executor/function_executor/server/function_executor_server_factory.py

@@ -14,7 +14,8 @@ class FunctionExecutorServerConfiguration:
     configuration parameters or raise an exception if it can't implement
     them."""
 
-    def __init__(self, image_uri: Optional[str]):
+    def __init__(self, executor_id: str, image_uri: Optional[str]):
+        self.executor_id: str = executor_id
         # Container image URI of the Function Executor Server.
         self.image_uri: Optional[str] = image_uri
 

indexify/executor/function_executor/server/subprocess_function_executor_server_factory.py

@@ -28,6 +28,8 @@ class SubprocessFunctionExecutorServerFactory(FunctionExecutorServerFactory):
         try:
             port = self._allocate_port()
             args = [
+                "--executor-id",
+                config.executor_id,
                 "--address",
                 _server_address(port),
             ]

indexify/executor/function_executor/single_task_runner.py

@@ -25,6 +25,7 @@ from .task_output import TaskOutput
 class SingleTaskRunner:
     def __init__(
         self,
+        executor_id: str,
         function_executor_state: FunctionExecutorState,
         task_input: TaskInput,
         function_executor_server_factory: FunctionExecutorServerFactory,
@@ -32,6 +33,7 @@ class SingleTaskRunner:
         config_path: Optional[str],
         logger: Any,
     ):
+        self._executor_id: str = executor_id
         self._state: FunctionExecutorState = function_executor_state
         self._task_input: TaskInput = task_input
         self._factory: FunctionExecutorServerFactory = function_executor_server_factory
@@ -76,6 +78,7 @@ class SingleTaskRunner:
         )
         config: FunctionExecutorServerConfiguration = (
             FunctionExecutorServerConfiguration(
+                executor_id=self._executor_id,
                 image_uri=self._task_input.task.image_uri,
             )
         )

indexify/executor/task_reporter.py

@@ -1,4 +1,5 @@
 import asyncio
+import time
 from typing import Any, List, Optional, Tuple
 
 import nanoid
@@ -74,10 +75,18 @@ class TaskReporter:
             ),
             "files": output_files if len(output_files) > 0 else FORCE_MULTIPART,
         }
+
+        start_time = time.time()
         # Run in a separate thread to not block the main event loop.
         response = await asyncio.to_thread(
             self._client.post, url=f"{self._base_url}/internal/ingest_files", **kwargs
         )
+        end_time = time.time()
+        logger.info(
+            "task_outcome_reported",
+            response_time=end_time - start_time,
+            response_code=response.status_code,
+        )
 
         try:
             response.raise_for_status()

indexify/executor/task_runner.py

@@ -18,13 +18,19 @@ class TaskRunner:
 
     def __init__(
         self,
+        executor_id: str,
         function_executor_server_factory: FunctionExecutorServerFactory,
         base_url: str,
         config_path: Optional[str],
+        disable_automatic_function_executor_management: bool,
     ):
+        self._executor_id: str = executor_id
         self._factory: FunctionExecutorServerFactory = function_executor_server_factory
         self._base_url: str = base_url
         self._config_path: Optional[str] = config_path
+        self._disable_automatic_function_executor_management: bool = (
+            disable_automatic_function_executor_management
+        )
         # The fields below are protected by the lock.
         self._lock: asyncio.Lock = asyncio.Lock()
         self._is_shutdown: bool = False
@@ -71,6 +77,9 @@ class TaskRunner:
         #   - Each Function Executor rans at most 1 task concurrently.
         await state.wait_running_tasks_less(1)
 
+        if self._disable_automatic_function_executor_management:
+            return  # Disable Function Executor destroy in manual management mode.
+
         if state.function_id_with_version != _function_id_with_version(task):
             await state.destroy_function_executor()
             state.function_id_with_version = _function_id_with_version(task)
@@ -81,6 +90,7 @@ class TaskRunner:
         self, state: FunctionExecutorState, task_input: TaskInput, logger: Any
     ) -> TaskOutput:
         runner: SingleTaskRunner = SingleTaskRunner(
+            executor_id=self._executor_id,
             function_executor_state=state,
             task_input=task_input,
             function_executor_server_factory=self._factory,

{indexify-0.3.6.dist-info → indexify-0.3.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: indexify
-Version: 0.3.6
+Version: 0.3.8
 Summary: Open Source Indexify components and helper tools
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0
@@ -22,7 +22,7 @@ Requires-Dist: pydantic (==2.10.4)
 Requires-Dist: pyyaml (>=6,<7)
 Requires-Dist: rich (>=13.9.2,<14.0.0)
 Requires-Dist: structlog (>=24.4.0,<25.0.0)
-Requires-Dist: tensorlake (>=0.1.13)
+Requires-Dist: tensorlake (>=0.1.16)
 Requires-Dist: typer (>=0.12,<0.13)
 Project-URL: Repository, https://github.com/tensorlakeai/indexify
 Description-Content-Type: text/markdown

{indexify-0.3.6.dist-info → indexify-0.3.8.dist-info}/RECORD

@@ -1,25 +1,25 @@
-indexify/cli/cli.py,sha256=uZjcbl41cfBCYrXYp3ggplz3rI7enf-I8pwSYCeNsTo,11502
+indexify/cli/cli.py,sha256=Um413vABP7i-UgH5-AUgt0O5oO2n13os_VdzE9JD4Vg,11972
 indexify/executor/README.md,sha256=ozC6_hMkhQQNVCMEpBxwiUALz6lwErPQxNxQfQDqnG4,2029
 indexify/executor/api_objects.py,sha256=k5tKYxaWml0sSECoEDzamCYeJnlD7zO2M7E_qGwyMrg,1032
 indexify/executor/downloader.py,sha256=a4f7M_Npfvy5Y-XLqmrPRUdPYvonl4qbK0dDw3EvpZ8,6460
-indexify/executor/executor.py,sha256=np6qfwGrCAOmupefoIiGsznJcppZ62vagQuQId4L1Gg,5895
+indexify/executor/executor.py,sha256=YlUr-BNdsRueJx-T3YB9Z-KIFJIiaz26so2C0wLQzrE,6100
 indexify/executor/function_executor/function_executor.py,sha256=00ILKA4kctsky4v3y26xispJh5Bl07H2VBN9fsaTbOA,5838
 indexify/executor/function_executor/function_executor_state.py,sha256=_85dpaudYM0sekOqwjMxKGdK7MNQdTGUhHi67sqVHyY,2853
 indexify/executor/function_executor/health_checker.py,sha256=mYR0WRL-gDFXoyaTBRBnnl9ZfJlL4d-RRe48HEZtU0g,2750
 indexify/executor/function_executor/invocation_state_client.py,sha256=CfIcVKJZoFMQFOyi3R_dtMgHs5VcGo2V4FKH0or6n80,8586
 indexify/executor/function_executor/server/client_configuration.py,sha256=gOywMus0cotlX6NKIadEJwvOmBE-LbGE_wvoMi5-HzY,994
 indexify/executor/function_executor/server/function_executor_server.py,sha256=_DLivLDikupZusRk8gVWDk7fWPT9XjZ4un1yWSlOObs,883
-indexify/executor/function_executor/server/function_executor_server_factory.py,sha256=pGbJMQfC5TNvyWOs6VDKdqd2PK5OHQh5_wSDP-E7DbI,1677
+indexify/executor/function_executor/server/function_executor_server_factory.py,sha256=oBEuOwuGsZrvOZsv9v8M4JEtfproirE-cH28XCr0HoY,1739
 indexify/executor/function_executor/server/subprocess_function_executor_server.py,sha256=JekDOqF7oFD4J6zcN3xB0Dxd1cgpEXMOsb_rKZOeBlI,668
-indexify/executor/function_executor/server/subprocess_function_executor_server_factory.py,sha256=FYExuYZZ7CUcznUobtnxvd2hVjUjpB9Dkav0FFcA0hM,4118
-indexify/executor/function_executor/single_task_runner.py,sha256=asaBLQlY0P5KigA7waX3Lra1aWi8oFeNi7jVj5KvjDI,7764
+indexify/executor/function_executor/server/subprocess_function_executor_server_factory.py,sha256=xm_EL2Gouwi0qAuKsyJVAQJRBZ0VB-NYsWUM4mOi3nQ,4187
+indexify/executor/function_executor/single_task_runner.py,sha256=uHEDCo4ae8npyelKQdkBIHtypuH6fExWShXRFjJoPkY,7882
 indexify/executor/function_executor/task_input.py,sha256=wSrHR4m0juiGClQyeVdhRC37QzDt6Rrjq-ZXJkfBi9k,584
 indexify/executor/function_executor/task_output.py,sha256=FLtqpmfhv6faAls0HCzyiZyyOjvENlrslpfl4Mz9JvI,1066
 indexify/executor/runtime_probes.py,sha256=bo6Dq6AGZpJH099j0DHtVSDEH80tv3j9MXf3VXSx_p8,2182
 indexify/executor/task_fetcher.py,sha256=hng1moUwRl4bUMwKum8eGgcAd9djU5PJqHT9hxFnhtU,2912
-indexify/executor/task_reporter.py,sha256=m0JrXYnBw-wUqeZHdtewCW9H8ojtcphCbvAkGQRPe8g,6640
-indexify/executor/task_runner.py,sha256=RSFeJYhQ_agXXPBm8u13HErSUPZGsCGwV1stgS7g258,5035
-indexify-0.3.6.dist-info/METADATA,sha256=zvi_iL28kPeHzKrmJiuV928j9oKPuZ3mJAgJQZf9v8M,1334
-indexify-0.3.6.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
-indexify-0.3.6.dist-info/entry_points.txt,sha256=GU9wmsgvN7nQw3N2X0PMYn1RSvF6CrhH9RuC2D8d3Gk,53
-indexify-0.3.6.dist-info/RECORD,,
+indexify/executor/task_reporter.py,sha256=7VgHA5W1He9Ur0s1bPgXKP75GWhtnIMoUz3_HRPpajE,6882
+indexify/executor/task_runner.py,sha256=w_om0LloMd5cV-K4NrMOJai0vjP7ebsjVEwvP-jDGgg,5500
+indexify-0.3.8.dist-info/METADATA,sha256=Zty_YeCUe6inMfaSoABWn_2CDqBvb5tKLBSK1F4_SVA,1334
+indexify-0.3.8.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
+indexify-0.3.8.dist-info/entry_points.txt,sha256=GU9wmsgvN7nQw3N2X0PMYn1RSvF6CrhH9RuC2D8d3Gk,53
+indexify-0.3.8.dist-info/RECORD,,