indexify 0.3.7__py3-none-any.whl → 0.3.9__py3-none-any.whl

indexify/cli/cli.py

@@ -34,8 +34,6 @@ from indexify.executor.function_executor.server.subprocess_function_executor_ser
     SubprocessFunctionExecutorServerFactory,
 )
 
-logger = structlog.get_logger(module=__name__)
-
 custom_theme = Theme(
     {
         "info": "cyan",
@@ -160,25 +158,6 @@ def build_image(
                 _create_image(obj, python_sdk_path)
 
 
-@app.command(help="Build default image for indexify")
-def build_default_image(
-    python_version: Optional[str] = typer.Option(
-        f"{sys.version_info.major}.{sys.version_info.minor}",
-        help="Python version to use in the base image",
-    )
-):
-    image = GetDefaultPythonImage(python_version)
-
-    _build_image(image=image)
-
-    console.print(
-        Text(f"Built default indexify image with hash {image.hash()}\n", style="cyan"),
-        Text(
-            f"Don't forget to update your executors to run this image!", style="yellow"
-        ),
-    )
-
-
 @app.command(
     help="Runs Executor that connects to the Indexify server and starts running its tasks"
 )
@@ -208,6 +187,13 @@ def executor(
     ports: Tuple[int, int] = typer.Option(
         (50000, 51000), help="Range of localhost TCP ports to be used by the executor"
     ),
+    disable_automatic_function_executor_management: Annotated[
+        bool,
+        typer.Option(
+            "--disable-automatic-function-executor-management",
+            help="Disable automatic Function Executor management by Executor",
+        ),
+    ] = False,
 ):
     if dev:
         configure_development_mode_logging()
@@ -218,18 +204,20 @@ def executor(
                 "At least one function must be specified when not running in development mode"
             )
 
-    id = nanoid.generate()
     executor_version = version("indexify")
+    id = nanoid.generate()
+    logger = structlog.get_logger(module=__name__, executor_id=id)
+
     logger.info(
         "starting executor",
         server_addr=server_addr,
         config_path=config_path,
-        executor_id=id,
         executor_version=executor_version,
         executor_cache=executor_cache,
         ports=ports,
         functions=function_uris,
         dev_mode=dev,
+        disable_automatic_function_executor_management=disable_automatic_function_executor_management,
     )
 
     executor_cache = Path(executor_cache).expanduser().absolute()
@@ -258,6 +246,7 @@ def executor(
             development_mode=dev,
             server_ports=range(ports[0], ports[1]),
         ),
+        disable_automatic_function_executor_management=disable_automatic_function_executor_management,
     ).run()
 
 
@@ -299,57 +288,7 @@ def _create_image(image: Image, python_sdk_path):
 
 
 def _build_image(image: Image, python_sdk_path: Optional[str] = None):
-    docker_file = _generate_dockerfile(image, python_sdk_path=python_sdk_path)
-    image_name = f"{image._image_name}:{image._tag}"
-
-    # low_level_client = docker.APIClient(base_url=docker_client.api.base_url)
-    docker_host = os.getenv("DOCKER_HOST", "unix:///var/run/docker.sock")
-    low_level_client = docker.APIClient(base_url=docker_host)
-    docker.api.build.process_dockerfile = lambda dockerfile, path: (
-        "Dockerfile",
-        dockerfile,
-    )
-    generator = low_level_client.build(
-        dockerfile=docker_file,
-        rm=True,
-        path=".",
-        tag=image_name,
-    )
-
+    built_image, generator = image.build(python_sdk_path=python_sdk_path)
     for output in generator:
-        for line in output.decode().splitlines():
-            json_line = json.loads(line)
-            if "stream" in json_line:
-                print(json_line["stream"], end="")
-
-            elif "errorDetail" in json_line:
-                print(json_line["errorDetail"]["message"])
-
-
-def _generate_dockerfile(image, python_sdk_path: Optional[str] = None):
-    docker_contents = [
-        f"FROM {image._base_image}",
-        "RUN mkdir -p ~/.indexify",
-        f"RUN echo {image._image_name} > ~/.indexify/image_name",  # TODO: Do we still use this in executors?
-        f"RUN echo {image.hash()} > ~/.indexify/image_hash",  # TODO: Do we still use this in executors?
-        "WORKDIR /app",
-    ]
-
-    for build_op in image._build_ops:
-        docker_contents.append(build_op.render())
-
-    if python_sdk_path is not None:
-        print(f"Building image {image._image_name} with local version of the SDK")
-
-        if not os.path.exists(python_sdk_path):
-            print(f"error: {python_sdk_path} does not exist")
-            os.exit(1)
-        docker_contents.append(f"COPY {python_sdk_path} /app/python-sdk")
-        docker_contents.append("RUN (cd /app/python-sdk && pip install .)")
-    else:
-        docker_contents.append(
-            f"RUN pip install indexify=={importlib.metadata.version('indexify')}"
-        )
-
-    docker_file = "\n".join(docker_contents)
-    return docker_file
+        print(output)
+    print(f"built image: {built_image.tags[0]}")

indexify/executor/executor.py

@@ -27,6 +27,7 @@ class Executor:
         function_executor_server_factory: FunctionExecutorServerFactory,
         server_addr: str = "localhost:8900",
         config_path: Optional[str] = None,
+        disable_automatic_function_executor_management: bool = False,
     ):
         self._logger = structlog.get_logger(module=__name__)
         self._is_shutdown: bool = False
@@ -40,9 +41,11 @@ class Executor:
         self._base_url = f"{protocol}://{self._server_addr}"
         self._code_path = code_path
         self._task_runner = TaskRunner(
+            executor_id=id,
             function_executor_server_factory=function_executor_server_factory,
             base_url=self._base_url,
             config_path=config_path,
+            disable_automatic_function_executor_management=disable_automatic_function_executor_management,
         )
         self._downloader = Downloader(
             code_path=code_path, base_url=self._base_url, config_path=config_path

indexify/executor/function_executor/server/function_executor_server_factory.py

@@ -14,7 +14,8 @@ class FunctionExecutorServerConfiguration:
     configuration parameters or raise an exception if it can't implement
     them."""
 
-    def __init__(self, image_uri: Optional[str]):
+    def __init__(self, executor_id: str, image_uri: Optional[str]):
+        self.executor_id: str = executor_id
         # Container image URI of the Function Executor Server.
         self.image_uri: Optional[str] = image_uri
 

indexify/executor/function_executor/server/subprocess_function_executor_server_factory.py

@@ -28,6 +28,8 @@ class SubprocessFunctionExecutorServerFactory(FunctionExecutorServerFactory):
         try:
             port = self._allocate_port()
             args = [
+                "--executor-id",
+                config.executor_id,
                 "--address",
                 _server_address(port),
             ]

indexify/executor/function_executor/single_task_runner.py

@@ -25,6 +25,7 @@ from .task_output import TaskOutput
 class SingleTaskRunner:
     def __init__(
         self,
+        executor_id: str,
         function_executor_state: FunctionExecutorState,
         task_input: TaskInput,
         function_executor_server_factory: FunctionExecutorServerFactory,
@@ -32,6 +33,7 @@ class SingleTaskRunner:
         config_path: Optional[str],
         logger: Any,
     ):
+        self._executor_id: str = executor_id
         self._state: FunctionExecutorState = function_executor_state
         self._task_input: TaskInput = task_input
         self._factory: FunctionExecutorServerFactory = function_executor_server_factory
@@ -76,6 +78,7 @@ class SingleTaskRunner:
         )
         config: FunctionExecutorServerConfiguration = (
             FunctionExecutorServerConfiguration(
+                executor_id=self._executor_id,
                 image_uri=self._task_input.task.image_uri,
             )
         )

indexify/executor/task_reporter.py

@@ -1,4 +1,5 @@
 import asyncio
+import time
 from typing import Any, List, Optional, Tuple
 
 import nanoid
@@ -74,10 +75,18 @@ class TaskReporter:
             ),
             "files": output_files if len(output_files) > 0 else FORCE_MULTIPART,
         }
+
+        start_time = time.time()
         # Run in a separate thread to not block the main event loop.
         response = await asyncio.to_thread(
             self._client.post, url=f"{self._base_url}/internal/ingest_files", **kwargs
         )
+        end_time = time.time()
+        logger.info(
+            "task_outcome_reported",
+            response_time=end_time - start_time,
+            response_code=response.status_code,
+        )
 
         try:
             response.raise_for_status()

indexify/executor/task_runner.py

@@ -18,13 +18,19 @@ class TaskRunner:
 
     def __init__(
         self,
+        executor_id: str,
         function_executor_server_factory: FunctionExecutorServerFactory,
         base_url: str,
         config_path: Optional[str],
+        disable_automatic_function_executor_management: bool,
     ):
+        self._executor_id: str = executor_id
         self._factory: FunctionExecutorServerFactory = function_executor_server_factory
         self._base_url: str = base_url
         self._config_path: Optional[str] = config_path
+        self._disable_automatic_function_executor_management: bool = (
+            disable_automatic_function_executor_management
+        )
         # The fields below are protected by the lock.
         self._lock: asyncio.Lock = asyncio.Lock()
         self._is_shutdown: bool = False
@@ -71,6 +77,9 @@ class TaskRunner:
         #   - Each Function Executor rans at most 1 task concurrently.
         await state.wait_running_tasks_less(1)
 
+        if self._disable_automatic_function_executor_management:
+            return  # Disable Function Executor destroy in manual management mode.
+
         if state.function_id_with_version != _function_id_with_version(task):
             await state.destroy_function_executor()
             state.function_id_with_version = _function_id_with_version(task)
@@ -81,6 +90,7 @@ class TaskRunner:
         self, state: FunctionExecutorState, task_input: TaskInput, logger: Any
     ) -> TaskOutput:
         runner: SingleTaskRunner = SingleTaskRunner(
+            executor_id=self._executor_id,
             function_executor_state=state,
             task_input=task_input,
             function_executor_server_factory=self._factory,

{indexify-0.3.7.dist-info → indexify-0.3.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: indexify
-Version: 0.3.7
+Version: 0.3.9
 Summary: Open Source Indexify components and helper tools
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0
@@ -22,7 +22,7 @@ Requires-Dist: pydantic (==2.10.4)
 Requires-Dist: pyyaml (>=6,<7)
 Requires-Dist: rich (>=13.9.2,<14.0.0)
 Requires-Dist: structlog (>=24.4.0,<25.0.0)
-Requires-Dist: tensorlake (>=0.1.14)
+Requires-Dist: tensorlake (>=0.1.16)
 Requires-Dist: typer (>=0.12,<0.13)
 Project-URL: Repository, https://github.com/tensorlakeai/indexify
 Description-Content-Type: text/markdown

{indexify-0.3.7.dist-info → indexify-0.3.9.dist-info}/RECORD

@@ -1,25 +1,25 @@
-indexify/cli/cli.py,sha256=uZjcbl41cfBCYrXYp3ggplz3rI7enf-I8pwSYCeNsTo,11502
+indexify/cli/cli.py,sha256=Pk7DwzoaMfHyF-M8CoorBB1iCDUl6Gsh_kf4Dc7SLsM,9551
 indexify/executor/README.md,sha256=ozC6_hMkhQQNVCMEpBxwiUALz6lwErPQxNxQfQDqnG4,2029
 indexify/executor/api_objects.py,sha256=k5tKYxaWml0sSECoEDzamCYeJnlD7zO2M7E_qGwyMrg,1032
 indexify/executor/downloader.py,sha256=a4f7M_Npfvy5Y-XLqmrPRUdPYvonl4qbK0dDw3EvpZ8,6460
-indexify/executor/executor.py,sha256=np6qfwGrCAOmupefoIiGsznJcppZ62vagQuQId4L1Gg,5895
+indexify/executor/executor.py,sha256=YlUr-BNdsRueJx-T3YB9Z-KIFJIiaz26so2C0wLQzrE,6100
 indexify/executor/function_executor/function_executor.py,sha256=00ILKA4kctsky4v3y26xispJh5Bl07H2VBN9fsaTbOA,5838
 indexify/executor/function_executor/function_executor_state.py,sha256=_85dpaudYM0sekOqwjMxKGdK7MNQdTGUhHi67sqVHyY,2853
 indexify/executor/function_executor/health_checker.py,sha256=mYR0WRL-gDFXoyaTBRBnnl9ZfJlL4d-RRe48HEZtU0g,2750
 indexify/executor/function_executor/invocation_state_client.py,sha256=CfIcVKJZoFMQFOyi3R_dtMgHs5VcGo2V4FKH0or6n80,8586
 indexify/executor/function_executor/server/client_configuration.py,sha256=gOywMus0cotlX6NKIadEJwvOmBE-LbGE_wvoMi5-HzY,994
 indexify/executor/function_executor/server/function_executor_server.py,sha256=_DLivLDikupZusRk8gVWDk7fWPT9XjZ4un1yWSlOObs,883
-indexify/executor/function_executor/server/function_executor_server_factory.py,sha256=pGbJMQfC5TNvyWOs6VDKdqd2PK5OHQh5_wSDP-E7DbI,1677
+indexify/executor/function_executor/server/function_executor_server_factory.py,sha256=oBEuOwuGsZrvOZsv9v8M4JEtfproirE-cH28XCr0HoY,1739
 indexify/executor/function_executor/server/subprocess_function_executor_server.py,sha256=JekDOqF7oFD4J6zcN3xB0Dxd1cgpEXMOsb_rKZOeBlI,668
-indexify/executor/function_executor/server/subprocess_function_executor_server_factory.py,sha256=FYExuYZZ7CUcznUobtnxvd2hVjUjpB9Dkav0FFcA0hM,4118
-indexify/executor/function_executor/single_task_runner.py,sha256=asaBLQlY0P5KigA7waX3Lra1aWi8oFeNi7jVj5KvjDI,7764
+indexify/executor/function_executor/server/subprocess_function_executor_server_factory.py,sha256=xm_EL2Gouwi0qAuKsyJVAQJRBZ0VB-NYsWUM4mOi3nQ,4187
+indexify/executor/function_executor/single_task_runner.py,sha256=uHEDCo4ae8npyelKQdkBIHtypuH6fExWShXRFjJoPkY,7882
 indexify/executor/function_executor/task_input.py,sha256=wSrHR4m0juiGClQyeVdhRC37QzDt6Rrjq-ZXJkfBi9k,584
 indexify/executor/function_executor/task_output.py,sha256=FLtqpmfhv6faAls0HCzyiZyyOjvENlrslpfl4Mz9JvI,1066
 indexify/executor/runtime_probes.py,sha256=bo6Dq6AGZpJH099j0DHtVSDEH80tv3j9MXf3VXSx_p8,2182
 indexify/executor/task_fetcher.py,sha256=hng1moUwRl4bUMwKum8eGgcAd9djU5PJqHT9hxFnhtU,2912
-indexify/executor/task_reporter.py,sha256=m0JrXYnBw-wUqeZHdtewCW9H8ojtcphCbvAkGQRPe8g,6640
-indexify/executor/task_runner.py,sha256=RSFeJYhQ_agXXPBm8u13HErSUPZGsCGwV1stgS7g258,5035
-indexify-0.3.7.dist-info/METADATA,sha256=Rv9YkcMGVCmqgY6IC3tceZOUQRqARrNC-Yd_QcR45ak,1334
-indexify-0.3.7.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
-indexify-0.3.7.dist-info/entry_points.txt,sha256=GU9wmsgvN7nQw3N2X0PMYn1RSvF6CrhH9RuC2D8d3Gk,53
-indexify-0.3.7.dist-info/RECORD,,
+indexify/executor/task_reporter.py,sha256=7VgHA5W1He9Ur0s1bPgXKP75GWhtnIMoUz3_HRPpajE,6882
+indexify/executor/task_runner.py,sha256=w_om0LloMd5cV-K4NrMOJai0vjP7ebsjVEwvP-jDGgg,5500
+indexify-0.3.9.dist-info/METADATA,sha256=2WZDAjnyCJ7imbyvM9fS3MV2hxQ8_3eCDhdZ_3ehKkM,1334
+indexify-0.3.9.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
+indexify-0.3.9.dist-info/entry_points.txt,sha256=GU9wmsgvN7nQw3N2X0PMYn1RSvF6CrhH9RuC2D8d3Gk,53
+indexify-0.3.9.dist-info/RECORD,,