indexify 0.2.40__tar.gz → 0.2.41__tar.gz

{indexify-0.2.40 → indexify-0.2.41}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: indexify
-Version: 0.2.40
+Version: 0.2.41
 Summary: Python Client for Indexify
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0
@@ -16,6 +16,8 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: cloudpickle (>=3.1.0,<4.0.0)
 Requires-Dist: docker (>=7.1.0,<8.0.0)
+Requires-Dist: grpcio (==1.68.1)
+Requires-Dist: grpcio-tools (==1.68.1)
 Requires-Dist: httpx-sse (>=0.4.0,<0.5.0)
 Requires-Dist: httpx[http2] (==0.27.2)
 Requires-Dist: nanoid (>=2.0.0,<3.0.0)

{indexify-0.2.40 → indexify-0.2.41}/indexify/cli.py

@@ -1,3 +1,8 @@
+from .logging import configure_logging_early, configure_production_logging
+
+configure_logging_early()
+
+
 import asyncio
 import os
 import shutil
@@ -18,12 +23,18 @@ from rich.text import Text
 from rich.theme import Theme
 
 from indexify.executor.agent import ExtractorAgent
-from indexify.executor.function_worker import FunctionWorker
+from indexify.function_executor.function_executor_service import (
+    FunctionExecutorService,
+)
+from indexify.function_executor.server import Server as FunctionExecutorServer
 from indexify.functions_sdk.image import (
-    DEFAULT_IMAGE_3_10,
-    DEFAULT_IMAGE_3_11,
+    LOCAL_PYTHON_VERSION,
+    GetDefaultPythonImage,
     Image,
 )
+from indexify.http_client import IndexifyClient
+
+logger = structlog.get_logger(module=__name__)
 
 custom_theme = Theme(
     {
@@ -34,10 +45,12 @@ custom_theme = Theme(
     }
 )
 
-logging = structlog.get_logger(module=__name__)
 console = Console(theme=custom_theme)
 
 app = typer.Typer(pretty_exceptions_enable=False, no_args_is_help=True)
+config_path_option: Optional[str] = typer.Option(
+    None, help="Path to the TLS configuration file"
+)
 
 
 @app.command(
@@ -149,12 +162,21 @@ def build_image(
 
 
 @app.command(help="Build default image for indexify")
-def build_default_image():
-    _build_image(image=DEFAULT_IMAGE_3_10)
-    _build_image(image=DEFAULT_IMAGE_3_11)
+def build_default_image(
+    python_version: Optional[str] = typer.Option(
+        f"{sys.version_info.major}.{sys.version_info.minor}",
+        help="Python version to use in the base image",
+    )
+):
+    image = GetDefaultPythonImage(python_version)
+
+    _build_image(image=image)
 
     console.print(
-        Text(f"Built default indexify image", style="cyan"),
+        Text(f"Built default indexify image with hash {image.hash()}\n", style="cyan"),
+        Text(
+            f"Don't forget to update your executors to run this image!", style="yellow"
+        ),
     )
 
 
@@ -164,42 +186,32 @@ def executor(
     dev: Annotated[
         bool, typer.Option("--dev", "-d", help="Run the executor in development mode")
     ] = False,
-    workers: Annotated[
-        int, typer.Option(help="number of worker processes for extraction")
-    ] = 1,
-    config_path: Optional[str] = typer.Option(
-        None, help="Path to the TLS configuration file"
-    ),
+    config_path: Optional[str] = config_path_option,
     executor_cache: Optional[str] = typer.Option(
         "~/.indexify/executor_cache", help="Path to the executor cache directory"
     ),
     name_alias: Optional[str] = typer.Option(
-        None, help="Name alias for the executor if it's spun up with the base image"
+        None, help="Image name override for the executor"
     ),
-    image_version: Optional[int] = typer.Option(
-        "1", help="Requested Image Version for this executor"
+    image_hash: Optional[str] = typer.Option(
+        None, help="Image hash override for the executor"
     ),
 ):
-    # configure structured logging
     if not dev:
-        processors = [
-            structlog.processors.dict_tracebacks,
-            structlog.processors.JSONRenderer(),
-        ]
-        structlog.configure(processors=processors)
+        configure_production_logging()
 
     id = nanoid.generate()
     executor_version = version("indexify")
-    logging.info(
+    logger.info(
         "executor started",
-        workers=workers,
         server_addr=server_addr,
         config_path=config_path,
         executor_id=id,
         executor_version=executor_version,
         executor_cache=executor_cache,
         name_alias=name_alias,
-        image_version=image_version,
+        image_hash=image_hash,
+        dev_mode=dev,
     )
 
     from pathlib import Path
@@ -211,18 +223,47 @@ def executor(
 
     agent = ExtractorAgent(
         id,
-        num_workers=workers,
         server_addr=server_addr,
         config_path=config_path,
         code_path=executor_cache,
         name_alias=name_alias,
-        image_version=image_version,
+        image_hash=image_hash,
+        development_mode=dev,
     )
 
     try:
         asyncio.get_event_loop().run_until_complete(agent.run())
     except asyncio.CancelledError:
-        logging.info("graceful shutdown")
+        logger.info("graceful shutdown")
+
+
+@app.command(help="Runs a Function Executor server")
+def function_executor(
+    function_executor_server_address: str = typer.Option(
+        help="Function Executor server address"
+    ),
+    indexify_server_address: str = typer.Option(help="Indexify server address"),
+    dev: Annotated[
+        bool, typer.Option("--dev", "-d", help="Run the executor in development mode")
+    ] = False,
+    config_path: Optional[str] = config_path_option,
+):
+    if not dev:
+        configure_production_logging()
+
+    logger.info(
+        "starting function executor server",
+        function_executor_server_address=function_executor_server_address,
+        indexify_server_address=indexify_server_address,
+        config_path=config_path,
+    )
+
+    FunctionExecutorServer(
+        server_address=function_executor_server_address,
+        service=FunctionExecutorService(
+            indexify_server_address=indexify_server_address, config_path=config_path
+        ),
+    ).run()
 
 
 def _create_image(image: Image, python_sdk_path):
@@ -234,6 +275,7 @@ def _create_image(image: Image, python_sdk_path):
 
 
 def _build_image(image: Image, python_sdk_path: Optional[str] = None):
+
     try:
         import docker
 
@@ -246,24 +288,31 @@ def _build_image(image: Image, python_sdk_path: Optional[str] = None):
         )
         exit(-1)
 
-    docker_file = f"""
-FROM {image._base_image}
-
-RUN mkdir -p ~/.indexify
+    docker_contents = [
+        f"FROM {image._base_image}",
+        "RUN mkdir -p ~/.indexify",
+        "RUN touch ~/.indexify/image_name",
+        f"RUN echo {image._image_name} > ~/.indexify/image_name",
+        f"RUN echo {image.hash()} > ~/.indexify/image_hash",
+        "WORKDIR /app",
+    ]
 
-RUN touch ~/.indexify/image_name
+    docker_contents.extend(["RUN " + i for i in image._run_strs])
 
-RUN  echo {image._image_name} > ~/.indexify/image_name
-
-WORKDIR /app
-
-"""
+    if python_sdk_path is not None:
+        logging.info(
+            f"Building image {image._image_name} with local version of the SDK"
+        )
+        if not os.path.exists(python_sdk_path):
+            print(f"error: {python_sdk_path} does not exist")
+            os.exit(1)
+        docker_contents.append(f"COPY {python_sdk_path} /app/python-sdk")
+        docker_contents.append("RUN (cd /app/python-sdk && pip install .)")
+    else:
+        docker_contents.append(f"RUN pip install indexify=={image._sdk_version}")
 
-    run_strs = ["RUN " + i for i in image._run_strs]
+    docker_file = "\n".join(docker_contents)
 
-    docker_file += "\n".join(run_strs)
-    print(os.getcwd())
-    import docker
     import docker.api.build
 
     docker.api.build.process_dockerfile = lambda dockerfile, path: (
@@ -271,15 +320,6 @@ WORKDIR /app
         dockerfile,
     )
 
-    if python_sdk_path is not None:
-        if not os.path.exists(python_sdk_path):
-            print(f"error: {python_sdk_path} does not exist")
-            os.exit(1)
-        docker_file += f"\nCOPY {python_sdk_path} /app/python-sdk"
-        docker_file += f"\nRUN (cd /app/python-sdk && pip install .)"
-    else:
-        docker_file += f"\nRUN pip install indexify"
-
     console.print("Creating image using Dockerfile contents:", style="cyan bold")
     print(f"{docker_file}")
 

indexify-0.2.41/indexify/executor/agent.py

@@ -0,0 +1,262 @@
+import asyncio
+from pathlib import Path
+from typing import Dict, List, Optional
+
+import structlog
+
+from .downloader import Downloader
+from .executor_tasks import DownloadGraphTask, DownloadInputsTask, RunTask
+from .function_executor.process_function_executor_factory import (
+    ProcessFunctionExecutorFactory,
+)
+from .function_worker import (
+    FunctionWorker,
+    FunctionWorkerInput,
+    FunctionWorkerOutput,
+)
+from .task_fetcher import TaskFetcher
+from .task_reporter import TaskReporter
+from .task_store import CompletedTask, TaskStore
+
+logger = structlog.get_logger(module=__name__)
+
+
+class ExtractorAgent:
+    def __init__(
+        self,
+        executor_id: str,
+        code_path: Path,
+        server_addr: str = "localhost:8900",
+        development_mode: bool = False,
+        config_path: Optional[str] = None,
+        name_alias: Optional[str] = None,
+        image_hash: Optional[str] = None,
+    ):
+        self._config_path = config_path
+        protocol: str = "http"
+        if config_path:
+            logger.info("running the extractor with TLS enabled")
+            protocol = "https"
+
+        self._task_store: TaskStore = TaskStore()
+        self._function_worker = FunctionWorker(
+            function_executor_factory=ProcessFunctionExecutorFactory(
+                indexify_server_address=server_addr,
+                development_mode=development_mode,
+                config_path=config_path,
+            )
+        )
+        self._has_registered = False
+        self._server_addr = server_addr
+        self._base_url = f"{protocol}://{self._server_addr}"
+        self._code_path = code_path
+        self._downloader = Downloader(
+            code_path=code_path, base_url=self._base_url, config_path=config_path
+        )
+        self._task_fetcher = TaskFetcher(
+            protocol=protocol,
+            indexify_server_addr=self._server_addr,
+            executor_id=executor_id,
+            name_alias=name_alias,
+            image_hash=image_hash,
+            config_path=config_path,
+        )
+        self._task_reporter = TaskReporter(
+            base_url=self._base_url,
+            executor_id=executor_id,
+            config_path=self._config_path,
+        )
+
+    async def task_completion_reporter(self):
+        logger.info("starting task completion reporter")
+        # We should copy only the keys and not the values
+        while True:
+            outcomes = await self._task_store.task_outcomes()
+            for task_outcome in outcomes:
+                logger.info(
+                    "reporting_task_outcome",
+                    task_id=task_outcome.task.id,
+                    fn_name=task_outcome.task.compute_fn,
+                    num_outputs=(
+                        len(task_outcome.function_output.outputs)
+                        if task_outcome.function_output is not None
+                        else 0
+                    ),
+                    router_output=task_outcome.router_output,
+                    outcome=task_outcome.task_outcome,
+                    retries=task_outcome.reporting_retries,
+                )
+
+                try:
+                    # Send task outcome to the server
+                    self._task_reporter.report_task_outcome(completed_task=task_outcome)
+                except Exception as e:
+                    # The connection was dropped in the middle of the reporting, process, retry
+                    logger.error(
+                        "failed_to_report_task",
+                        task_id=task_outcome.task.id,
+                        exc_info=e,
+                        retries=task_outcome.reporting_retries,
+                    )
+                    task_outcome.reporting_retries += 1
+                    await asyncio.sleep(5)
+                    continue
+
+                self._task_store.mark_reported(task_id=task_outcome.task.id)
+
+    async def task_launcher(self):
+        async_tasks: List[asyncio.Task] = [
+            asyncio.create_task(
+                self._task_store.get_runnable_tasks(), name="get_runnable_tasks"
+            )
+        ]
+
+        while True:
+            done, pending = await asyncio.wait(
+                async_tasks, return_when=asyncio.FIRST_COMPLETED
+            )
+
+            async_tasks: List[asyncio.Task] = list(pending)
+            for async_task in done:
+                if async_task.get_name() == "get_runnable_tasks":
+                    if async_task.exception():
+                        logger.error(
+                            "task_launcher_error, failed to get runnable tasks",
+                            exc_info=async_task.exception(),
+                        )
+                        continue
+                    result: Dict[str, Task] = await async_task
+                    task: Task
+                    for _, task in result.items():
+                        async_tasks.append(
+                            DownloadGraphTask(
+                                function_worker_input=FunctionWorkerInput(task=task),
+                                downloader=self._downloader,
+                            )
+                        )
+                    async_tasks.append(
+                        asyncio.create_task(
+                            self._task_store.get_runnable_tasks(),
+                            name="get_runnable_tasks",
+                        )
+                    )
+                elif async_task.get_name() == "download_graph":
+                    if async_task.exception():
+                        logger.error(
+                            "task_launcher_error, failed to download graph",
+                            exc_info=async_task.exception(),
+                        )
+                        completed_task = CompletedTask(
+                            task=async_task.function_worker_input.task,
+                            task_outcome="failure",
+                        )
+                        self._task_store.complete(outcome=completed_task)
+                        continue
+                    async_task: DownloadGraphTask
+                    function_worker_input: FunctionWorkerInput = (
+                        async_task.function_worker_input
+                    )
+                    function_worker_input.graph = await async_task
+                    async_tasks.append(
+                        DownloadInputsTask(
+                            function_worker_input=function_worker_input,
+                            downloader=self._downloader,
+                        )
+                    )
+                elif async_task.get_name() == "download_inputs":
+                    if async_task.exception():
+                        logger.error(
+                            "task_launcher_error, failed to download inputs",
+                            exc_info=async_task.exception(),
+                        )
+                        completed_task = CompletedTask(
+                            task=async_task.function_worker_input.task,
+                            task_outcome="failure",
+                        )
+                        self._task_store.complete(outcome=completed_task)
+                        continue
+                    async_task: DownloadInputsTask
+                    function_worker_input: FunctionWorkerInput = (
+                        async_task.function_worker_input
+                    )
+                    function_worker_input.function_input = await async_task
+                    async_tasks.append(
+                        RunTask(
+                            function_worker=self._function_worker,
+                            function_worker_input=function_worker_input,
+                        )
+                    )
+                elif async_task.get_name() == "run_task":
+                    if async_task.exception():
+                        completed_task = CompletedTask(
+                            task=async_task.function_worker_input.task,
+                            task_outcome="failure",
+                            stderr=str(async_task.exception()),
+                        )
+                        self._task_store.complete(outcome=completed_task)
+                        continue
+                    async_task: RunTask
+                    try:
+                        outputs: FunctionWorkerOutput = await async_task
+                        if not outputs.success:
+                            task_outcome = "failure"
+                        else:
+                            task_outcome = "success"
+
+                        completed_task = CompletedTask(
+                            task=async_task.function_worker_input.task,
+                            task_outcome=task_outcome,
+                            function_output=outputs.function_output,
+                            router_output=outputs.router_output,
+                            stdout=outputs.stdout,
+                            stderr=outputs.stderr,
+                            reducer=outputs.reducer,
+                        )
+                        self._task_store.complete(outcome=completed_task)
+                    except Exception as e:
+                        logger.error(
+                            "failed to execute task",
+                            task_id=async_task.function_worker_input.task.id,
+                            exc_info=e,
+                        )
+                        completed_task = CompletedTask(
+                            task=async_task.function_worker_input.task,
+                            task_outcome="failure",
+                        )
+                        self._task_store.complete(outcome=completed_task)
+                        continue
+
+    async def _main_loop(self):
+        """Fetches incoming tasks from the server and starts their processing."""
+        self._should_run = True
+        while self._should_run:
+            try:
+                async for task in self._task_fetcher.run():
+                    self._task_store.add_tasks([task])
+            except Exception as e:
+                logger.error("failed fetching tasks, retrying in 5 seconds", exc_info=e)
+                await asyncio.sleep(5)
+                continue
+
+    async def run(self):
+        import signal
+
+        asyncio.get_event_loop().add_signal_handler(
+            signal.SIGINT, self.shutdown, asyncio.get_event_loop()
+        )
+        asyncio.get_event_loop().add_signal_handler(
+            signal.SIGTERM, self.shutdown, asyncio.get_event_loop()
+        )
+        asyncio.create_task(self.task_launcher())
+        asyncio.create_task(self.task_completion_reporter())
+        await self._main_loop()
+
+    async def _shutdown(self, loop):
+        logger.info("shutting_down")
+        self._should_run = False
+        await self._function_worker.shutdown()
+        for task in asyncio.all_tasks(loop):
+            task.cancel()
+
+    def shutdown(self, loop):
+        loop.create_task(self._shutdown(loop))

{indexify-0.2.40 → indexify-0.2.41}/indexify/executor/api_objects.py

@@ -1,8 +1,6 @@
 from typing import Any, Dict, List, Optional
 
-from pydantic import BaseModel, Json
-
-from indexify.functions_sdk.data_objects import IndexifyData
+from pydantic import BaseModel
 
 
 class Task(BaseModel):
@@ -21,7 +19,7 @@ class ExecutorMetadata(BaseModel):
     executor_version: str
     addr: str
     image_name: str
-    image_version: int
+    image_hash: str
     labels: Dict[str, Any]
 
 
@@ -29,10 +27,6 @@ class RouterOutput(BaseModel):
     edges: List[str]
 
 
-class FnOutput(BaseModel):
-    payload: Json
-
-
 class TaskResult(BaseModel):
     router_output: Optional[RouterOutput] = None
     outcome: str