indexify 0.3.7__tar.gz → 0.3.9__tar.gz

{indexify-0.3.7 → indexify-0.3.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: indexify
-Version: 0.3.7
+Version: 0.3.9
 Summary: Open Source Indexify components and helper tools
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0
@@ -22,7 +22,7 @@ Requires-Dist: pydantic (==2.10.4)
 Requires-Dist: pyyaml (>=6,<7)
 Requires-Dist: rich (>=13.9.2,<14.0.0)
 Requires-Dist: structlog (>=24.4.0,<25.0.0)
-Requires-Dist: tensorlake (>=0.1.14)
+Requires-Dist: tensorlake (>=0.1.16)
 Requires-Dist: typer (>=0.12,<0.13)
 Project-URL: Repository, https://github.com/tensorlakeai/indexify
 Description-Content-Type: text/markdown

{indexify-0.3.7 → indexify-0.3.9}/pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 name = "indexify"
 # Incremented if any of the components provided in this packages are updated.
-version = "0.3.7"
+version = "0.3.9"
 description = "Open Source Indexify components and helper tools"
 authors = ["Tensorlake Inc. <support@tensorlake.ai>"]
 license = "Apache 2.0"
@@ -24,7 +24,7 @@ grpcio = "1.70.0"
 pydantic = "2.10.4"
 httpx-sse = "^0.4.0"
 # Adds function-executor binary and utils lib.
-tensorlake = ">=0.1.14"
+tensorlake = ">=0.1.16"
 
 # CLI only
 rich = "^13.9.2"

{indexify-0.3.7 → indexify-0.3.9}/src/indexify/cli/cli.py

@@ -34,8 +34,6 @@ from indexify.executor.function_executor.server.subprocess_function_executor_ser
     SubprocessFunctionExecutorServerFactory,
 )
 
-logger = structlog.get_logger(module=__name__)
-
 custom_theme = Theme(
     {
         "info": "cyan",
@@ -160,25 +158,6 @@ def build_image(
                 _create_image(obj, python_sdk_path)
 
 
-@app.command(help="Build default image for indexify")
-def build_default_image(
-    python_version: Optional[str] = typer.Option(
-        f"{sys.version_info.major}.{sys.version_info.minor}",
-        help="Python version to use in the base image",
-    )
-):
-    image = GetDefaultPythonImage(python_version)
-
-    _build_image(image=image)
-
-    console.print(
-        Text(f"Built default indexify image with hash {image.hash()}\n", style="cyan"),
-        Text(
-            f"Don't forget to update your executors to run this image!", style="yellow"
-        ),
-    )
-
-
 @app.command(
     help="Runs Executor that connects to the Indexify server and starts running its tasks"
 )
@@ -208,6 +187,13 @@ def executor(
     ports: Tuple[int, int] = typer.Option(
         (50000, 51000), help="Range of localhost TCP ports to be used by the executor"
     ),
+    disable_automatic_function_executor_management: Annotated[
+        bool,
+        typer.Option(
+            "--disable-automatic-function-executor-management",
+            help="Disable automatic Function Executor management by Executor",
+        ),
+    ] = False,
 ):
     if dev:
         configure_development_mode_logging()
@@ -218,18 +204,20 @@ def executor(
                 "At least one function must be specified when not running in development mode"
             )
 
-    id = nanoid.generate()
     executor_version = version("indexify")
+    id = nanoid.generate()
+    logger = structlog.get_logger(module=__name__, executor_id=id)
+
     logger.info(
         "starting executor",
         server_addr=server_addr,
         config_path=config_path,
-        executor_id=id,
         executor_version=executor_version,
         executor_cache=executor_cache,
         ports=ports,
         functions=function_uris,
         dev_mode=dev,
+        disable_automatic_function_executor_management=disable_automatic_function_executor_management,
     )
 
     executor_cache = Path(executor_cache).expanduser().absolute()
@@ -258,6 +246,7 @@ def executor(
             development_mode=dev,
             server_ports=range(ports[0], ports[1]),
         ),
+        disable_automatic_function_executor_management=disable_automatic_function_executor_management,
     ).run()
 
 
@@ -299,57 +288,7 @@ def _create_image(image: Image, python_sdk_path):
 
 
 def _build_image(image: Image, python_sdk_path: Optional[str] = None):
-    docker_file = _generate_dockerfile(image, python_sdk_path=python_sdk_path)
-    image_name = f"{image._image_name}:{image._tag}"
-
-    # low_level_client = docker.APIClient(base_url=docker_client.api.base_url)
-    docker_host = os.getenv("DOCKER_HOST", "unix:///var/run/docker.sock")
-    low_level_client = docker.APIClient(base_url=docker_host)
-    docker.api.build.process_dockerfile = lambda dockerfile, path: (
-        "Dockerfile",
-        dockerfile,
-    )
-    generator = low_level_client.build(
-        dockerfile=docker_file,
-        rm=True,
-        path=".",
-        tag=image_name,
-    )
-
+    built_image, generator = image.build(python_sdk_path=python_sdk_path)
     for output in generator:
-        for line in output.decode().splitlines():
-            json_line = json.loads(line)
-            if "stream" in json_line:
-                print(json_line["stream"], end="")
-
-            elif "errorDetail" in json_line:
-                print(json_line["errorDetail"]["message"])
-
-
-def _generate_dockerfile(image, python_sdk_path: Optional[str] = None):
-    docker_contents = [
-        f"FROM {image._base_image}",
-        "RUN mkdir -p ~/.indexify",
-        f"RUN echo {image._image_name} > ~/.indexify/image_name",  # TODO: Do we still use this in executors?
-        f"RUN echo {image.hash()} > ~/.indexify/image_hash",  # TODO: Do we still use this in executors?
-        "WORKDIR /app",
-    ]
-
-    for build_op in image._build_ops:
-        docker_contents.append(build_op.render())
-
-    if python_sdk_path is not None:
-        print(f"Building image {image._image_name} with local version of the SDK")
-
-        if not os.path.exists(python_sdk_path):
-            print(f"error: {python_sdk_path} does not exist")
-            os.exit(1)
-        docker_contents.append(f"COPY {python_sdk_path} /app/python-sdk")
-        docker_contents.append("RUN (cd /app/python-sdk && pip install .)")
-    else:
-        docker_contents.append(
-            f"RUN pip install indexify=={importlib.metadata.version('indexify')}"
-        )
-
-    docker_file = "\n".join(docker_contents)
-    return docker_file
+        print(output)
+    print(f"built image: {built_image.tags[0]}")

{indexify-0.3.7 → indexify-0.3.9}/src/indexify/executor/executor.py

@@ -27,6 +27,7 @@ class Executor:
         function_executor_server_factory: FunctionExecutorServerFactory,
         server_addr: str = "localhost:8900",
         config_path: Optional[str] = None,
+        disable_automatic_function_executor_management: bool = False,
     ):
         self._logger = structlog.get_logger(module=__name__)
         self._is_shutdown: bool = False
@@ -40,9 +41,11 @@ class Executor:
         self._base_url = f"{protocol}://{self._server_addr}"
         self._code_path = code_path
         self._task_runner = TaskRunner(
+            executor_id=id,
             function_executor_server_factory=function_executor_server_factory,
             base_url=self._base_url,
             config_path=config_path,
+            disable_automatic_function_executor_management=disable_automatic_function_executor_management,
         )
         self._downloader = Downloader(
             code_path=code_path, base_url=self._base_url, config_path=config_path

{indexify-0.3.7 → indexify-0.3.9}/src/indexify/executor/function_executor/server/function_executor_server_factory.py

@@ -14,7 +14,8 @@ class FunctionExecutorServerConfiguration:
     configuration parameters or raise an exception if it can't implement
     them."""
 
-    def __init__(self, image_uri: Optional[str]):
+    def __init__(self, executor_id: str, image_uri: Optional[str]):
+        self.executor_id: str = executor_id
         # Container image URI of the Function Executor Server.
         self.image_uri: Optional[str] = image_uri
 

{indexify-0.3.7 → indexify-0.3.9}/src/indexify/executor/function_executor/server/subprocess_function_executor_server_factory.py

@@ -28,6 +28,8 @@ class SubprocessFunctionExecutorServerFactory(FunctionExecutorServerFactory):
         try:
             port = self._allocate_port()
             args = [
+                "--executor-id",
+                config.executor_id,
                 "--address",
                 _server_address(port),
             ]

{indexify-0.3.7 → indexify-0.3.9}/src/indexify/executor/function_executor/single_task_runner.py

@@ -25,6 +25,7 @@ from .task_output import TaskOutput
 class SingleTaskRunner:
     def __init__(
         self,
+        executor_id: str,
         function_executor_state: FunctionExecutorState,
         task_input: TaskInput,
         function_executor_server_factory: FunctionExecutorServerFactory,
@@ -32,6 +33,7 @@ class SingleTaskRunner:
         config_path: Optional[str],
         logger: Any,
     ):
+        self._executor_id: str = executor_id
         self._state: FunctionExecutorState = function_executor_state
         self._task_input: TaskInput = task_input
         self._factory: FunctionExecutorServerFactory = function_executor_server_factory
@@ -76,6 +78,7 @@ class SingleTaskRunner:
         )
         config: FunctionExecutorServerConfiguration = (
             FunctionExecutorServerConfiguration(
+                executor_id=self._executor_id,
                 image_uri=self._task_input.task.image_uri,
             )
         )

{indexify-0.3.7 → indexify-0.3.9}/src/indexify/executor/task_reporter.py

@@ -1,4 +1,5 @@
 import asyncio
+import time
 from typing import Any, List, Optional, Tuple
 
 import nanoid
@@ -74,10 +75,18 @@ class TaskReporter:
             ),
             "files": output_files if len(output_files) > 0 else FORCE_MULTIPART,
         }
+
+        start_time = time.time()
         # Run in a separate thread to not block the main event loop.
         response = await asyncio.to_thread(
             self._client.post, url=f"{self._base_url}/internal/ingest_files", **kwargs
         )
+        end_time = time.time()
+        logger.info(
+            "task_outcome_reported",
+            response_time=end_time - start_time,
+            response_code=response.status_code,
+        )
 
         try:
             response.raise_for_status()

{indexify-0.3.7 → indexify-0.3.9}/src/indexify/executor/task_runner.py

@@ -18,13 +18,19 @@ class TaskRunner:
 
     def __init__(
         self,
+        executor_id: str,
         function_executor_server_factory: FunctionExecutorServerFactory,
         base_url: str,
         config_path: Optional[str],
+        disable_automatic_function_executor_management: bool,
     ):
+        self._executor_id: str = executor_id
         self._factory: FunctionExecutorServerFactory = function_executor_server_factory
         self._base_url: str = base_url
         self._config_path: Optional[str] = config_path
+        self._disable_automatic_function_executor_management: bool = (
+            disable_automatic_function_executor_management
+        )
         # The fields below are protected by the lock.
         self._lock: asyncio.Lock = asyncio.Lock()
         self._is_shutdown: bool = False
@@ -71,6 +77,9 @@ class TaskRunner:
         #   - Each Function Executor rans at most 1 task concurrently.
         await state.wait_running_tasks_less(1)
 
+        if self._disable_automatic_function_executor_management:
+            return  # Disable Function Executor destroy in manual management mode.
+
         if state.function_id_with_version != _function_id_with_version(task):
             await state.destroy_function_executor()
             state.function_id_with_version = _function_id_with_version(task)
@@ -81,6 +90,7 @@ class TaskRunner:
         self, state: FunctionExecutorState, task_input: TaskInput, logger: Any
     ) -> TaskOutput:
         runner: SingleTaskRunner = SingleTaskRunner(
+            executor_id=self._executor_id,
             function_executor_state=state,
             task_input=task_input,
             function_executor_server_factory=self._factory,