indexify 0.3.18__tar.gz → 0.3.20__tar.gz

{indexify-0.3.18 → indexify-0.3.20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: indexify
-Version: 0.3.18
+Version: 0.3.20
 Summary: Open Source Indexify components and helper tools
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0
@@ -15,6 +15,7 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: aiohttp (>=3.11.0,<4.0.0)
+Requires-Dist: boto3 (>=1.37.30,<2.0.0)
 Requires-Dist: prometheus-client (>=0.21.1,<0.22.0)
 Requires-Dist: rich (>=13.9.2,<14.0.0)
 Requires-Dist: tensorlake (>=0.1)

{indexify-0.3.18 → indexify-0.3.20}/pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 name = "indexify"
 # Incremented if any of the components provided in this packages are updated.
-version = "0.3.18"
+version = "0.3.20"
 description = "Open Source Indexify components and helper tools"
 authors = ["Tensorlake Inc. <support@tensorlake.ai>"]
 license = "Apache 2.0"
@@ -24,6 +24,8 @@ aiohttp = "^3.11.0"
 prometheus-client = "^0.21.1"
 # Adds function-executor binary and utils lib.
 tensorlake = ">=0.1"
+# Uncomment the next line to use local tensorlake package (only for development!)
+# tensorlake = { path = "../tensorlake", develop = true }
 # pydantic is provided by tensorlake
 # httpx-sse is provided by tensorlake
 # grpcio is provided by tensorlake
@@ -33,6 +35,7 @@ tensorlake = ">=0.1"
 rich = "^13.9.2"
 typer = "^0.12"
 # nanoid is provided by tensorlake
+boto3 = "^1.37.30"
 
 [tool.poetry.group.dev.dependencies]
 black = "^24.10.0"

{indexify-0.3.18 → indexify-0.3.20}/src/indexify/cli/cli.py

@@ -25,6 +25,9 @@ from rich.theme import Theme
 from tensorlake.functions_sdk.image import Image
 
 from indexify.executor.api_objects import FunctionURI
+from indexify.executor.blob_store.blob_store import BLOBStore
+from indexify.executor.blob_store.local_fs_blob_store import LocalFSBLOBStore
+from indexify.executor.blob_store.s3_blob_store import S3BLOBStore
 from indexify.executor.executor import Executor
 from indexify.executor.executor_flavor import ExecutorFlavor
 from indexify.executor.function_executor.server.subprocess_function_executor_server_factory import (
@@ -78,6 +81,7 @@ def build_image(
 )
 def executor(
     server_addr: str = "localhost:8900",
+    grpc_server_addr: str = "localhost:8901",
     dev: Annotated[
         bool, typer.Option("--dev", "-d", help="Run the executor in development mode")
     ] = False,
@@ -120,16 +124,6 @@ def executor(
             help="Port where to run Executor Monitoring server",
         ),
     ] = 7000,
-    grpc_server_addr: Annotated[
-        Optional[str],
-        typer.Option(
-            "--grpc-server-addr",
-            help=(
-                "(exprimental) Address of server gRPC API to connect to, e.g. 'localhost:8901'.\n"
-                "Enables gRPC state reporter that will periodically report the state of the Function Executors to Server\n"
-            ),
-        ),
-    ] = None,
     enable_grpc_state_reconciler: Annotated[
         bool,
         typer.Option(
@@ -166,11 +160,6 @@ def executor(
             "--executor-id should be at least 10 characters long and only include characters _-[0-9][a-z][A-Z]"
         )
 
-    if enable_grpc_state_reconciler and grpc_server_addr is None:
-        raise typer.BadParameter(
-            "--grpc-server-addr must be set when --enable-grpc-state-reconciler is set"
-        )
-
     kv_labels: Dict[str, str] = {}
     for label in labels:
         key, value = label.split("=")
@@ -183,6 +172,7 @@ def executor(
         "starting executor",
         hostname=gethostname(),
         server_addr=server_addr,
+        grpc_server_addr=grpc_server_addr,
         config_path=config_path,
         executor_version=executor_version,
         labels=kv_labels,
@@ -192,7 +182,6 @@ def executor(
         dev_mode=dev,
         monitoring_server_host=monitoring_server_host,
         monitoring_server_port=monitoring_server_port,
-        grpc_server_addr=grpc_server_addr,
         enable_grpc_state_reconciler=enable_grpc_state_reconciler,
     )
 
@@ -211,6 +200,14 @@ def executor(
         )
         exit(1)
 
+    # Enable all available blob stores in OSS because we don't know which one is going to be used.
+    blob_store: BLOBStore = BLOBStore(
+        # Local FS mode is used in tests and in cases when user wants to store data on NFS.
+        local=LocalFSBLOBStore(),
+        # S3 is initiliazed lazily so it's okay to create it even if the user is not going to use it.
+        s3=S3BLOBStore(),
+    )
+
     prometheus_client.Info("cli", "CLI information").info(
         {
             "package": "indexify",
@@ -231,11 +228,12 @@ def executor(
             server_ports=range(ports[0], ports[1]),
         ),
         server_addr=server_addr,
+        grpc_server_addr=grpc_server_addr,
         config_path=config_path,
         monitoring_server_host=monitoring_server_host,
         monitoring_server_port=monitoring_server_port,
-        grpc_server_addr=grpc_server_addr,
         enable_grpc_state_reconciler=enable_grpc_state_reconciler,
+        blob_store=blob_store,
     ).run()
 
 

{indexify-0.3.18 → indexify-0.3.20}/src/indexify/executor/api_objects.py

@@ -49,5 +49,17 @@ class TaskResult(BaseModel):
     reducer: bool = False
 
 
+class DataPayload(BaseModel):
+    path: str
+    size: int
+    sha256_hash: str
+
+
+class IngestFnOutputsResponse(BaseModel):
+    data_payloads: List[DataPayload]
+    stdout: Optional[DataPayload] = None
+    stderr: Optional[DataPayload] = None
+
+
 TASK_OUTCOME_SUCCESS = "success"
 TASK_OUTCOME_FAILURE = "failure"

indexify-0.3.20/src/indexify/executor/blob_store/blob_store.py

@@ -0,0 +1,69 @@
+from typing import Any, Optional
+
+from .local_fs_blob_store import LocalFSBLOBStore
+from .metrics.blob_store import (
+    metric_get_blob_errors,
+    metric_get_blob_latency,
+    metric_get_blob_requests,
+    metric_put_blob_errors,
+    metric_put_blob_latency,
+    metric_put_blob_requests,
+)
+from .s3_blob_store import S3BLOBStore
+
+
+class BLOBStore:
+    """Dispatches generic BLOB store calls to their real backends."""
+
+    def __init__(
+        self, local: Optional[LocalFSBLOBStore] = None, s3: Optional[S3BLOBStore] = None
+    ):
+        """Creates a BLOB store that uses the supplied BLOB stores."""
+        self._local: Optional[LocalFSBLOBStore] = local
+        self._s3: Optional[S3BLOBStore] = s3
+
+    async def get(self, uri: str, logger: Any) -> bytes:
+        """Returns binary value stored in BLOB with the supplied URI.
+
+        Raises Exception on error. Raises KeyError if the BLOB doesn't exist.
+        """
+        with (
+            metric_get_blob_errors.count_exceptions(),
+            metric_get_blob_latency.time(),
+        ):
+            metric_get_blob_requests.inc()
+            if _is_file_uri(uri):
+                self._check_local_is_available()
+                return await self._local.get(uri, logger)
+            else:
+                self._check_s3_is_available()
+                return await self._s3.get(uri, logger)
+
+    async def put(self, uri: str, value: bytes, logger: Any) -> None:
+        """Stores the supplied binary value in a BLOB with the supplied URI.
+
+        Overwrites existing BLOB. Raises Exception on error.
+        """
+        with (
+            metric_put_blob_errors.count_exceptions(),
+            metric_put_blob_latency.time(),
+        ):
+            metric_put_blob_requests.inc()
+            if _is_file_uri(uri):
+                self._check_local_is_available()
+                await self._local.put(uri, value, logger)
+            else:
+                self._check_s3_is_available()
+                await self._s3.put(uri, value, logger)
+
+    def _check_local_is_available(self):
+        if self._local is None:
+            raise RuntimeError("Local file system BLOB store is not available")
+
+    def _check_s3_is_available(self):
+        if self._s3 is None:
+            raise RuntimeError("S3 BLOB store is not available")
+
+
+def _is_file_uri(uri: str) -> bool:
+    return uri.startswith("file://")

indexify-0.3.20/src/indexify/executor/blob_store/local_fs_blob_store.py

@@ -0,0 +1,48 @@
+import asyncio
+import os
+import os.path
+from typing import Any
+
+
+class LocalFSBLOBStore:
+    """BLOB store that stores BLOBs in local file system."""
+
+    async def get(self, uri: str, logger: Any) -> bytes:
+        """Returns binary value stored in file at the supplied URI.
+
+        The URI must be a file URI (starts with "file://"). The path must be absolute.
+        Raises Exception on error. Raises KeyError if the file doesn't exist.
+        """
+        # Run synchronous code in a thread to not block the event loop.
+        return await asyncio.to_thread(self._sync_get, _path_from_file_uri(uri))
+
+    async def put(self, uri: str, value: bytes, logger: Any) -> None:
+        """Stores the supplied binary value in a file at the supplied URI.
+
+        The URI must be a file URI (starts with "file://"). The path must be absolute.
+        Overwrites existing file. Raises Exception on error.
+        """
+        # Run synchronous code in a thread to not block the event loop.
+        return await asyncio.to_thread(self._sync_put, _path_from_file_uri(uri), value)
+
+    def _sync_get(self, path: str) -> bytes:
+        if not os.path.isabs(path):
+            raise ValueError(f"Path {path} is not absolute")
+
+        if os.path.exists(path):
+            with open(path, mode="rb") as blob_file:
+                return blob_file.read()
+        else:
+            raise KeyError(f"File at {path} does not exist")
+
+    def _sync_put(self, path: str, value: bytes) -> None:
+        if not os.path.isabs(path):
+            raise ValueError(f"Path {path} is not absolute")
+
+        os.makedirs(os.path.dirname(path), exist_ok=True)
+        with open(path, mode="wb") as blob_file:
+            blob_file.write(value)
+
+
+def _path_from_file_uri(uri: str) -> str:
+    return uri[7:]  # strip "file://" prefix

indexify-0.3.20/src/indexify/executor/blob_store/metrics/blob_store.py

@@ -0,0 +1,33 @@
+import prometheus_client
+
+from ...monitoring.metrics import latency_metric_for_fast_operation
+
+metric_get_blob_requests: prometheus_client.Counter = prometheus_client.Counter(
+    "get_blob_requests",
+    "Number of get blob requests",
+)
+metric_get_blob_errors: prometheus_client.Counter = prometheus_client.Counter(
+    "get_blob_request_errors",
+    "Number of get blob request errors",
+)
+metric_get_blob_latency: prometheus_client.Histogram = (
+    latency_metric_for_fast_operation(
+        "get_blob_request",
+        "get blob request",
+    )
+)
+
+metric_put_blob_requests: prometheus_client.Counter = prometheus_client.Counter(
+    "put_blob_requests",
+    "Number of put blob requests",
+)
+metric_put_blob_errors: prometheus_client.Counter = prometheus_client.Counter(
+    "put_blob_request_errors",
+    "Number of put blob request errors",
+)
+metric_put_blob_latency: prometheus_client.Histogram = (
+    latency_metric_for_fast_operation(
+        "put_blob_request",
+        "put blob request",
+    )
+)

indexify-0.3.20/src/indexify/executor/blob_store/s3_blob_store.py

@@ -0,0 +1,85 @@
+import asyncio
+from typing import Any, Optional
+
+import boto3
+from botocore.config import Config as BotoConfig
+from botocore.exceptions import ClientError as BotoClientError
+
+_MAX_RETRIES = 3
+
+
+class S3BLOBStore:
+    def __init__(self):
+        self._s3_client: Optional[Any] = None
+
+    def _lazy_create_client(self):
+        """Creates S3 client if it doesn't exist.
+
+        We create the client lazily only if S3 is used.
+        This is because S3 BLOB store is always created by Executor
+        and the creation will fail if user didn't configure S3 credentials and etc.
+        """
+        if self._s3_client is not None:
+            return
+
+        # The credentials and etc are fetched by boto3 library automatically following
+        # https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html#configuring-credentials
+        # This provides a lot of flexibility for the user and follows a well-known and documented logic.
+        self._s3_client = boto3.client(
+            "s3",
+            config=BotoConfig(
+                # https://boto3.amazonaws.com/v1/documentation/api/latest/guide/retries.html#standard-retry-mode
+                retries={
+                    "max_attempts": _MAX_RETRIES,
+                    "mode": "standard",
+                }
+            ),
+        )
+
+    async def get(self, uri: str, logger: Any) -> bytes:
+        """Returns binary value stored in S3 object at the supplied URI.
+
+        The URI must be S3 URI (starts with "s3://").
+        Raises Exception on error. Raises KeyError if the object doesn't exist.
+        """
+        try:
+            self._lazy_create_client()
+            bucket_name, key = _bucket_name_and_object_key_from_uri(uri)
+            response = await asyncio.to_thread(
+                self._s3_client.get_object, Bucket=bucket_name, Key=key
+            )
+            return response["Body"].read()
+        except BotoClientError as e:
+            logger.error("failed to get S3 object", uri=uri, exc_info=e)
+
+            if e.response["Error"]["Code"] == "NoSuchKey":
+                raise KeyError(f"Object {key} does not exist in bucket {bucket_name}")
+            raise
+        except Exception as e:
+            logger.error("failed to get S3 object", uri=uri, exc_info=e)
+            raise
+
+    async def put(self, uri: str, value: bytes, logger: Any) -> None:
+        """Stores the supplied binary value in a S3 object at the supplied URI.
+
+        The URI must be S3 URI (starts with "s3://").
+        Overwrites existing object. Raises Exception on error.
+        """
+        try:
+            self._lazy_create_client()
+            bucket_name, key = _bucket_name_and_object_key_from_uri(uri)
+            await asyncio.to_thread(
+                self._s3_client.put_object, Bucket=bucket_name, Key=key, Body=value
+            )
+        except Exception as e:
+            logger.error("failed to set S3 object", uri=uri, exc_info=e)
+            raise
+
+
+def _bucket_name_and_object_key_from_uri(uri: str) -> tuple[str, str]:
+    if not uri.startswith("s3://"):
+        raise ValueError(f"S3 URI '{uri}' is missing 's3://' prefix")
+    parts = uri[5:].split("/", 1)
+    if len(parts) != 2:
+        raise ValueError(f"Failed parsing bucket name from S3 URI '{uri}'")
+    return parts[0], parts[1]  # bucket_name, key