indexify 0.3.30__tar.gz → 0.4.2__tar.gz

{indexify-0.3.30 → indexify-0.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: indexify
-Version: 0.3.30
+Version: 0.4.2
 Summary: Open Source Indexify components and helper tools
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0
@@ -18,9 +18,7 @@ Requires-Dist: aiohttp (>=3.11.0,<4.0.0)
 Requires-Dist: boto3 (>=1.37.30,<2.0.0)
 Requires-Dist: prometheus-client (>=0.21.1,<0.22.0)
 Requires-Dist: psutil (>=7.0.0,<8.0.0)
-Requires-Dist: rich (>=13.9.2,<14.0.0)
 Requires-Dist: tensorlake (>=0.1)
-Requires-Dist: typer (>=0.12,<0.13)
 Project-URL: Repository, https://github.com/tensorlakeai/indexify
 Description-Content-Type: text/markdown
 

{indexify-0.3.30 → indexify-0.4.2}/pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 name = "indexify"
 # Incremented if any of the components provided in this packages are updated.
-version = "0.3.30"
+version = "0.4.2"
 description = "Open Source Indexify components and helper tools"
 authors = ["Tensorlake Inc. <support@tensorlake.ai>"]
 license = "Apache 2.0"
@@ -10,7 +10,7 @@ homepage = "https://github.com/tensorlakeai/indexify"
 repository = "https://github.com/tensorlakeai/indexify"
 
 [tool.poetry.scripts]
-indexify-cli = "indexify.cli.cli:app"
+indexify-cli = "indexify.cli:cli"
 
 [tool.poetry.dependencies]
 # Common dependencies
@@ -28,14 +28,12 @@ tensorlake = ">=0.1"
 # Uncomment the next line to use local tensorlake package (only for development!)
 # tensorlake = { path = "../tensorlake", develop = true }
 # pydantic is provided by tensorlake
-# httpx-sse is provided by tensorlake
 # grpcio is provided by tensorlake
 # grpcio-tools is provided by tensorlake
 
 # CLI only
-rich = "^13.9.2"
-typer = "^0.12"
 # nanoid is provided by tensorlake
+# click is provided by tensorlake
 boto3 = "^1.37.30"
 
 [tool.poetry.group.dev.dependencies]

indexify-0.4.2/src/indexify/cli/__init__.py

@@ -0,0 +1,18 @@
+import click
+
+from . import build_image, deploy, executor
+
+
+@click.group()
+@click.version_option(package_name="indexify", prog_name="indexify-cli")
+@click.pass_context
+def cli(ctx: click.Context):
+    """
+    Indexify CLI to manage and deploy workflows to Indexify Server and run Indexify Executors.
+    """
+    pass
+
+
+cli.add_command(build_image.build_image)
+cli.add_command(deploy.deploy)
+cli.add_command(executor.executor)

indexify-0.4.2/src/indexify/cli/build_image.py

@@ -0,0 +1,51 @@
+import click
+from tensorlake.functions_sdk.image import Image
+from tensorlake.functions_sdk.workflow_module import (
+    WorkflowModuleInfo,
+    load_workflow_module_info,
+)
+
+
+@click.command(
+    short_help="Build images for graphs/workflows defined in the workflow file"
+)
+# Path to the file where the graphs/workflows are defined as global variables
+@click.argument(
+    "workflow-file-path",
+    type=click.Path(exists=True, file_okay=True, dir_okay=False),
+)
+@click.option(
+    "-i",
+    "--image-names",
+    multiple=True,
+    help="Names of images to build. Can be specified multiple times. If not provided, all images will be built.",
+)
+def build_image(
+    workflow_file_path: str,
+    image_names: tuple[str, ...] = None,
+):
+    try:
+        workflow_module_info: WorkflowModuleInfo = load_workflow_module_info(
+            workflow_file_path
+        )
+    except Exception as e:
+        click.secho(
+            f"Failed loading workflow file, please check the error message: {e}",
+            fg="red",
+        )
+        raise click.Abort
+
+    for image in workflow_module_info.images.keys():
+        image: Image
+        if image_names is not None and image.image_name not in image_names:
+            click.echo(
+                f"Skipping image `{image.image_name}` as it is not in the provided image names."
+            )
+            continue
+
+        click.echo(f"Building image `{image.image_name}`")
+        built_image, generator = image.build()
+        for output in generator:
+            click.secho(output)
+
+        click.secho(f"built image: {built_image.tags[0]}", fg="green")

indexify-0.4.2/src/indexify/cli/deploy.py

@@ -0,0 +1,57 @@
+import click
+from tensorlake import Graph
+from tensorlake.functions_sdk.graph_serialization import graph_code_dir_path
+from tensorlake.functions_sdk.workflow_module import (
+    WorkflowModuleInfo,
+    load_workflow_module_info,
+)
+from tensorlake.remote_graph import RemoteGraph
+
+
+@click.command(
+    short_help="Deploy all graphs/workflows defined in the workflow file to Indexify"
+)
+# Path to the file where the graphs/workflows are defined as global variables
+@click.argument(
+    "workflow-file-path",
+    type=click.Path(exists=True, file_okay=True, dir_okay=False),
+)
+@click.option(
+    "-u",
+    "--upgrade-queued-requests",
+    is_flag=True,
+    default=False,
+    help="Upgrade invocations that are already queued or running to use the deployed version of the graphs/workflows",
+)
+def deploy(
+    workflow_file_path: str,
+    upgrade_queued_invocations: bool,
+):
+    click.echo(f"Preparing deployment for {workflow_file_path}")
+    try:
+        workflow_module_info: WorkflowModuleInfo = load_workflow_module_info(
+            workflow_file_path
+        )
+    except Exception as e:
+        click.secho(
+            f"Failed loading workflow file, please check the error message: {e}",
+            fg="red",
+        )
+        raise click.Abort
+
+    for graph in workflow_module_info.graphs:
+        graph: Graph
+        try:
+            RemoteGraph.deploy(
+                graph,
+                code_dir_path=graph_code_dir_path(workflow_file_path),
+                upgrade_tasks_to_latest_version=upgrade_queued_invocations,
+            )
+        except Exception as e:
+            click.secho(
+                f"Graph {graph.name} could not be deployed, please check the error message: {e}",
+                fg="red",
+            )
+            raise click.Abort
+
+        click.secho(f"Deployed {graph.name}", fg="green")

indexify-0.4.2/src/indexify/cli/executor.py

@@ -0,0 +1,205 @@
+from tensorlake.utils.logging import (
+    configure_development_mode_logging,
+    configure_logging_early,
+    configure_production_mode_logging,
+)
+
+configure_logging_early()
+
+import shutil
+from importlib.metadata import version
+from pathlib import Path
+from socket import gethostname
+from typing import Dict, List, Optional
+
+import click
+import nanoid
+import prometheus_client
+import structlog
+
+from indexify.executor.blob_store.blob_store import BLOBStore
+from indexify.executor.blob_store.local_fs_blob_store import LocalFSBLOBStore
+from indexify.executor.blob_store.s3_blob_store import S3BLOBStore
+from indexify.executor.executor import Executor
+from indexify.executor.function_executor.server.subprocess_function_executor_server_factory import (
+    SubprocessFunctionExecutorServerFactory,
+)
+from indexify.executor.host_resources.host_resources import HostResourcesProvider
+from indexify.executor.host_resources.nvidia_gpu_allocator import NvidiaGPUAllocator
+from indexify.executor.monitoring.health_checker.generic_health_checker import (
+    GenericHealthChecker,
+)
+
+
+@click.command(
+    context_settings={"allow_extra_args": True, "ignore_unknown_options": True},
+    short_help="Runs Executor that connects to the Indexify server and starts running its tasks",
+)
+@click.option(
+    "--server-addr",
+    "server_address",
+    default="localhost:8900",
+    help="Address of Indexify HTTP Server to connect to",
+)
+@click.option(
+    "--grpc-server-addr",
+    "grpc_server_address",
+    default="localhost:8901",
+    help="Address of Indexify gRPC Server to connect to",
+)
+@click.option(
+    "-v",
+    "--verbose",
+    is_flag=True,
+    default=False,
+    help="Verbose logging",
+)
+@click.option(
+    "-vv",
+    "--very-verbose",
+    is_flag=True,
+    default=False,
+    help="Very verbose logging",
+)
+@click.option(
+    "-f",
+    "--function",
+    "function_uris",
+    default=[],
+    multiple=True,
+    help="Functions that the executor will run "
+    "specified as <namespace>:<workflow>:<function>:<version>"
+    "version is optional, not specifying it will make the server send any version"
+    "of the function. Any number of --function arguments can be passed.",
+)
+@click.option(
+    "--config-path",
+    type=click.Path(exists=True, dir_okay=False, file_okay=True, readable=True),
+    default=None,
+    help="Path to the TLS configuration file",
+)
+@click.option(
+    "--executor-cache-path",
+    type=click.Path(exists=False, dir_okay=True, readable=True, writable=True),
+    default="~/.indexify/executor_cache",
+    help="Path to the executor cache directory",
+)
+@click.option(
+    "--monitoring-server-host",
+    default="localhost",
+    help="IP address or hostname where to run Executor Monitoring server",
+)
+@click.option(
+    "--monitoring-server-port",
+    default=7000,
+    type=int,
+    help="Port where to run Executor Monitoring server",
+)
+@click.option(
+    "-l",
+    "--label",
+    "labels",
+    default=[],
+    multiple=True,
+    help="Executor key-value label to be sent to the Server. "
+    "Specified as <key>=<value>",
+)
+@click.pass_context
+def executor(
+    ctx: click.Context,
+    server_address: str,
+    grpc_server_address: str,
+    verbose: bool,
+    very_verbose: bool,
+    function_uris: List[str],
+    config_path: Optional[str],
+    executor_cache_path: str,
+    monitoring_server_host: str,
+    monitoring_server_port: int,
+    labels: List[str],
+):
+    if verbose or very_verbose:
+        configure_development_mode_logging(compact_tracebacks=not very_verbose)
+    else:
+        configure_production_mode_logging()
+
+    kv_labels: Dict[str, str] = {}
+    for label in labels:
+        key, value = label.split("=")
+        kv_labels[key] = value
+
+    executor_id: str = nanoid.generate()
+    executor_version = version("indexify")
+    logger = structlog.get_logger(module=__name__, executor_id=executor_id)
+
+    logger.info(
+        "starting executor",
+        hostname=gethostname(),
+        server_address=server_address,
+        grpc_server_address=grpc_server_address,
+        config_path=config_path,
+        executor_version=executor_version,
+        labels=kv_labels,
+        executor_cache_path=executor_cache_path,
+        functions=function_uris,
+        verbose=verbose,
+        very_verbose=very_verbose,
+        monitoring_server_host=monitoring_server_host,
+        monitoring_server_port=monitoring_server_port,
+    )
+    if ctx.args:
+        logger.warning(
+            "Unknown arguments passed to the executor",
+            unknown_args=ctx.args,
+        )
+    if len(function_uris) == 0:
+        logger.warning(
+            "No --function arguments were passed. Executor will run all functions. This scenario is only supported for testing purposes.",
+        )
+
+    executor_cache_path: Path = Path(executor_cache_path).expanduser().absolute()
+    if executor_cache_path.exists():
+        shutil.rmtree(str(executor_cache_path))
+    executor_cache_path.mkdir(parents=True, exist_ok=True)
+
+    blob_store: BLOBStore = BLOBStore(
+        # Local FS mode is used in tests and in cases when user wants to store data on NFS.
+        local=LocalFSBLOBStore(),
+        # S3 is initiliazed lazily so it's okay to create it even if the user is not going to use it.
+        s3=S3BLOBStore(),
+    )
+
+    host_resources_provider: HostResourcesProvider = HostResourcesProvider(
+        gpu_allocator=NvidiaGPUAllocator(logger),
+        # Assuming a simple setup in OSS where Executor container has a single file system
+        # used by all Function Executors and all the container resources are available to all Function Executors.
+        function_executors_ephimeral_disks_path="/",
+        host_overhead_cpus=0,
+        host_overhead_memory_gb=0,
+        host_overhead_function_executors_ephimeral_disks_gb=0,
+    )
+
+    prometheus_client.Info("cli", "CLI information").info(
+        {
+            "package": "indexify",
+        }
+    )
+
+    Executor(
+        id=executor_id,
+        version=executor_version,
+        labels=kv_labels,
+        health_checker=GenericHealthChecker(),
+        cache_path=executor_cache_path,
+        function_uris=function_uris,
+        function_executor_server_factory=SubprocessFunctionExecutorServerFactory(
+            verbose_logs=verbose or very_verbose
+        ),
+        server_addr=server_address,
+        grpc_server_addr=grpc_server_address,
+        config_path=config_path,
+        monitoring_server_host=monitoring_server_host,
+        monitoring_server_port=monitoring_server_port,
+        blob_store=blob_store,
+        host_resources_provider=host_resources_provider,
+    ).run()

{indexify-0.3.30/src/indexify/executor/grpc → indexify-0.4.2/src/indexify/executor}/channel_manager.py

@@ -84,15 +84,28 @@ class ChannelManager:
         # Use the lock to ensure that we only create one channel without race conditions.
         async with self._lock:
             if self._channel is None:
-                self._channel = await self._create_channel()
+                self._channel = await self._create_ready_channel()
             elif not await self._locked_channel_is_healthy():
                 self._logger.info("grpc channel to server is unhealthy")
                 await self._destroy_locked_channel()
-                self._channel = await self._create_channel()
+                self._channel = await self._create_ready_channel()
 
             return self._channel
 
-    async def _create_channel(self) -> grpc.aio.Channel:
+    def create_channel(self) -> grpc.aio.Channel:
+        """Creates a new channel to the gRPC server.
+
+        The channel is not be ready to use. Raises an exception on failure.
+        """
+        if self._channel_credentials is None:
+            return grpc.aio.insecure_channel(target=self._server_address)
+        else:
+            return grpc.aio.secure_channel(
+                target=self._server_address,
+                credentials=self._channel_credentials,
+            )
+
+    async def _create_ready_channel(self) -> grpc.aio.Channel:
         """Creates a new channel to the gRPC server."
 
         Returns a ready to use channel. Blocks until the channel
@@ -104,14 +117,7 @@ class ChannelManager:
             metric_grpc_server_channel_creations.inc()
             while True:
                 try:
-                    if self._channel_credentials is None:
-                        channel = grpc.aio.insecure_channel(target=self._server_address)
-                    else:
-                        channel = grpc.aio.secure_channel(
-                            target=self._server_address,
-                            credentials=self._channel_credentials,
-                        )
-
+                    channel = self.create_channel()
                     await asyncio.wait_for(
                         channel.channel_ready(),
                         timeout=_CONNECT_TIMEOUT_SEC,

indexify-0.4.2/src/indexify/executor/executor.py

@@ -0,0 +1,175 @@
+import asyncio
+import signal
+from pathlib import Path
+from socket import gethostname
+from typing import Dict, List, Optional
+
+import structlog
+
+from indexify.proto.executor_api_pb2 import ExecutorStatus
+
+from .blob_store.blob_store import BLOBStore
+from .channel_manager import ChannelManager
+from .function_allowlist import (
+    FunctionURI,
+    function_allowlist_to_indexed_dict,
+    parse_function_uris,
+)
+from .function_executor.server.function_executor_server_factory import (
+    FunctionExecutorServerFactory,
+)
+from .host_resources.host_resources import HostResourcesProvider
+from .metrics.executor import (
+    metric_executor_info,
+    metric_executor_state,
+)
+from .monitoring.health_check_handler import HealthCheckHandler
+from .monitoring.health_checker.health_checker import HealthChecker
+from .monitoring.prometheus_metrics_handler import PrometheusMetricsHandler
+from .monitoring.server import MonitoringServer
+from .monitoring.startup_probe_handler import StartupProbeHandler
+from .state_reconciler import ExecutorStateReconciler
+from .state_reporter import ExecutorStateReporter
+
+metric_executor_state.state("starting")
+
+
+class Executor:
+    def __init__(
+        self,
+        id: str,
+        version: str,
+        labels: Dict[str, str],
+        cache_path: Path,
+        health_checker: HealthChecker,
+        function_uris: List[str],
+        function_executor_server_factory: FunctionExecutorServerFactory,
+        server_addr: str,
+        grpc_server_addr: str,
+        config_path: Optional[str],
+        monitoring_server_host: str,
+        monitoring_server_port: int,
+        blob_store: BLOBStore,
+        host_resources_provider: HostResourcesProvider,
+    ):
+        self._logger = structlog.get_logger(module=__name__)
+        protocol: str = "http"
+        if config_path:
+            self._logger.info("running the extractor with TLS enabled")
+            protocol = "https"
+
+        self._startup_probe_handler = StartupProbeHandler()
+        self._monitoring_server = MonitoringServer(
+            host=monitoring_server_host,
+            port=monitoring_server_port,
+            startup_probe_handler=self._startup_probe_handler,
+            health_probe_handler=HealthCheckHandler(health_checker),
+            metrics_handler=PrometheusMetricsHandler(),
+        )
+        self._channel_manager = ChannelManager(
+            server_address=grpc_server_addr,
+            config_path=config_path,
+            logger=self._logger,
+        )
+        function_allowlist: List[FunctionURI] = parse_function_uris(function_uris)
+        self._state_reporter = ExecutorStateReporter(
+            executor_id=id,
+            version=version,
+            labels=labels,
+            function_allowlist=function_allowlist,
+            channel_manager=self._channel_manager,
+            host_resources_provider=host_resources_provider,
+            logger=self._logger,
+        )
+        self._state_reporter.update_executor_status(
+            ExecutorStatus.EXECUTOR_STATUS_STARTING_UP
+        )
+        self._state_reconciler = ExecutorStateReconciler(
+            executor_id=id,
+            function_executor_server_factory=function_executor_server_factory,
+            base_url=f"{protocol}://{server_addr}",
+            config_path=config_path,
+            cache_path=cache_path,
+            blob_store=blob_store,
+            channel_manager=self._channel_manager,
+            state_reporter=self._state_reporter,
+            logger=self._logger,
+        )
+        self._run_aio_task: Optional[asyncio.Task] = None
+        self._shutdown_aio_task: Optional[asyncio.Task] = None
+
+        executor_info: Dict[str, str] = {
+            "id": id,
+            "version": version,
+            "cache_path": str(cache_path),
+            "server_addr": server_addr,
+            "grpc_server_addr": str(grpc_server_addr),
+            "config_path": str(config_path),
+            "hostname": gethostname(),
+        }
+        for key, value in labels.items():
+            executor_info["label_" + key] = value
+        executor_info.update(function_allowlist_to_indexed_dict(function_allowlist))
+        metric_executor_info.info(executor_info)
+
+    def run(self):
+        asyncio.new_event_loop()
+
+        self._run_aio_task = asyncio.get_event_loop().create_task(
+            self._run(),
+            name="executor startup and run loop",
+        )
+
+        try:
+            asyncio.get_event_loop().run_until_complete(self._run_aio_task)
+        except asyncio.CancelledError:
+            pass  # Expected exception on shutdown
+
+    async def _run(self):
+        for signum in [
+            signal.SIGABRT,
+            signal.SIGINT,
+            signal.SIGTERM,
+            signal.SIGQUIT,
+            signal.SIGHUP,
+        ]:
+            asyncio.get_event_loop().add_signal_handler(
+                signum, self._shutdown_signal_handler, asyncio.get_event_loop()
+            )
+
+        asyncio.create_task(
+            self._monitoring_server.run(), name="monitoring server runner"
+        )
+        self._state_reporter.update_executor_status(
+            ExecutorStatus.EXECUTOR_STATUS_RUNNING
+        )
+        self._state_reporter.run()
+        self._state_reconciler.run()
+        metric_executor_state.state("running")
+        self._startup_probe_handler.set_ready()
+
+        # Run the Executor forever until it is shut down.
+        while True:
+            await asyncio.sleep(10)
+
+    def _shutdown_signal_handler(self, loop):
+        if self._shutdown_aio_task is None:
+            self._shutdown_aio_task = loop.create_task(
+                self._shutdown(), name="executor shutdown"
+            )
+
+    async def _shutdown(self):
+        self._logger.info("shutting down Executor")
+        metric_executor_state.state("shutting_down")
+
+        # Shutdown state reconciler first because it changes reported state on shutdown.
+        await self._state_reconciler.shutdown()
+
+        # Do one last state report with STOPPED status. This reduces latency in the system.
+        self._state_reporter.update_executor_status(
+            ExecutorStatus.EXECUTOR_STATUS_STOPPED
+        )
+        await self._state_reporter.shutdown()
+        await self._channel_manager.destroy()
+        await self._monitoring_server.shutdown()
+        self._run_aio_task.cancel()

indexify-0.4.2/src/indexify/executor/function_allowlist.py

@@ -0,0 +1,59 @@
+from dataclasses import dataclass
+from typing import Dict, List, Optional
+
+
+@dataclass
+class FunctionURI:
+    namespace: str
+    compute_graph: str
+    compute_fn: str
+    version: Optional[str] = None
+
+
+def function_allowlist_to_indexed_dict(
+    function_allowlist: List[FunctionURI],
+) -> Dict[str, str]:
+    """Returns a dictionary with each function URI in the allowlist as a key-value pair.
+
+    The keys are prefixed indexes in function allowlist, and the values are the function URIs
+    """
+    indexed_dict = {}
+    counter = 0
+    for function_uri in function_allowlist:
+        function_uri: FunctionURI
+        indexed_dict[f"function_allowlist_{counter}"] = ":".join(
+            [
+                function_uri.namespace,
+                function_uri.compute_graph,
+                function_uri.compute_fn,
+                str(function_uri.version),
+            ]
+        )
+        counter += 1
+    return indexed_dict
+
+
+def parse_function_uris(function_uri_strs: List[str]) -> List[FunctionURI]:
+    """Parses a list of function URIs from strings to FunctionURI objects."""
+    uris: List[FunctionURI] = []
+    for uri_str in function_uri_strs:
+        tokens = uri_str.split(":")
+        if len(tokens) < 3 or len(tokens) > 4:
+            raise ValueError(
+                "Function should be specified as <namespace>:<workflow>:<function>:<version> or"
+                "<namespace>:<workflow>:<function>"
+            )
+        version: Optional[str] = None
+        if len(tokens) == 4:
+            version = tokens[3]
+
+        uris.append(
+            FunctionURI(
+                namespace=tokens[0],
+                compute_graph=tokens[1],
+                compute_fn=tokens[2],
+                version=version,
+            )
+        )
+
+    return uris