indexify 0.4.22__tar.gz → 0.4.24__tar.gz

{indexify-0.4.22 → indexify-0.4.24}/PKG-INFO

@@ -1,20 +1,19 @@
 Metadata-Version: 2.3
 Name: indexify
-Version: 0.4.22
+Version: 0.4.24
 Summary: Open Source Indexify components and helper tools
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0
 Author: Tensorlake Inc.
 Author-email: support@tensorlake.ai
-Requires-Python: >=3.10,<4.0
+Requires-Python: >=3.11,<4.0
 Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
-Requires-Dist: aiohttp (>=3.12.14,<4.0.0)
-Requires-Dist: boto3 (>=1.39.15,<2.0.0)
+Requires-Dist: aiohttp (>=3.12.15,<4.0.0)
+Requires-Dist: boto3 (>=1.40.6,<2.0.0)
 Requires-Dist: docker (>=7.1.0,<8.0.0)
 Requires-Dist: httpx[http2] (==0.27.2)
 Requires-Dist: nanoid (>=2.0.0,<3.0.0)
@@ -22,7 +21,8 @@ Requires-Dist: prometheus-client (>=0.22.1,<0.23.0)
 Requires-Dist: psutil (>=7.0.0,<8.0.0)
 Requires-Dist: pydantic (>=2.11,<3.0)
 Requires-Dist: requests (>=2.32.4,<3.0.0)
-Requires-Dist: tensorlake (==0.2.37)
+Requires-Dist: structlog (==25.4.0)
+Requires-Dist: tensorlake (==0.2.39)
 Requires-Dist: urllib3 (>=2.5.0,<3.0.0)
 Project-URL: Repository, https://github.com/tensorlakeai/indexify
 Description-Content-Type: text/markdown

{indexify-0.4.22 → indexify-0.4.24}/pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 name = "indexify"
 # Incremented if any of the components provided in this packages are updated.
-version = "0.4.22"
+version = "0.4.24"
 description = "Open Source Indexify components and helper tools"
 authors = ["Tensorlake Inc. <support@tensorlake.ai>"]
 license = "Apache 2.0"
@@ -14,22 +14,22 @@ indexify-cli = "indexify.cli:cli"
 
 [tool.poetry.dependencies]
 # Common dependencies
-python = "^3.10"
+python = "^3.11"
 nanoid = "^2.0.0"
-# structlog is provided by tensorlake
 # pyyaml is provided by tensorlake
 
 # Executor only
-aiohttp = "^3.12.14"
+aiohttp = "^3.12.15"
 # mTLS support for httpx 0.28.1 is broken, wait for 0.28.2 to see if the bug is fixed
 httpx = { version = "0.27.2", extras = ["http2"] }
 pydantic = "^2.11"
 prometheus-client = "^0.22.1"
 psutil = "^7.0.0"
-boto3 = "^1.39.15"
+boto3 = "^1.40.6"
+structlog = "25.4.0"
 # Adds function-executor binary, utils lib, sdk used in indexify-cli commands.
 # We need to specify the tensorlake version exactly because pip install doesn't respect poetry.lock files.
-tensorlake = "0.2.37"
+tensorlake = "0.2.39"
 # Uncomment the next line to use local tensorlake package (only for development!)
 # tensorlake = { path = "../tensorlake", develop = true }
 # grpcio is provided by tensorlake

{indexify-0.4.22 → indexify-0.4.24}/src/indexify/cli/executor.py

@@ -1,4 +1,4 @@
-from tensorlake.utils.logging import (
+from indexify.executor.logging import (
     configure_development_mode_logging,
     configure_logging_early,
     configure_production_mode_logging,
@@ -162,13 +162,6 @@ def executor(
         shutil.rmtree(str(executor_cache_path))
     executor_cache_path.mkdir(parents=True, exist_ok=True)
 
-    blob_store: BLOBStore = BLOBStore(
-        # Local FS mode is used in tests and in cases when user wants to store data on NFS.
-        local=LocalFSBLOBStore(),
-        # S3 is initiliazed lazily so it's okay to create it even if the user is not going to use it.
-        s3=S3BLOBStore(),
-    )
-
     host_resources_provider: HostResourcesProvider = HostResourcesProvider(
         gpu_allocator=NvidiaGPUAllocator(logger),
         # Assuming a simple setup in OSS where Executor container has a single file system
@@ -200,6 +193,6 @@ def executor(
         config_path=config_path,
         monitoring_server_host=monitoring_server_host,
         monitoring_server_port=monitoring_server_port,
-        blob_store=blob_store,
+        blob_store=BLOBStore(),
         host_resources_provider=host_resources_provider,
     ).run()

indexify-0.4.24/src/indexify/executor/blob_store/blob_store.py

@@ -0,0 +1,153 @@
+from typing import Any
+
+from .local_fs_blob_store import LocalFSBLOBStore
+from .metrics.blob_store import (
+    metric_abort_multipart_upload_errors,
+    metric_abort_multipart_upload_latency,
+    metric_abort_multipart_upload_requests,
+    metric_complete_multipart_upload_errors,
+    metric_complete_multipart_upload_latency,
+    metric_complete_multipart_upload_requests,
+    metric_create_multipart_upload_errors,
+    metric_create_multipart_upload_latency,
+    metric_create_multipart_upload_requests,
+    metric_get_blob_errors,
+    metric_get_blob_latency,
+    metric_get_blob_requests,
+    metric_presign_uri_errors,
+    metric_presign_uri_latency,
+    metric_presign_uri_requests,
+    metric_upload_blob_errors,
+    metric_upload_blob_latency,
+    metric_upload_blob_requests,
+)
+from .s3_blob_store import S3BLOBStore
+
+
+class BLOBStore:
+    """Dispatches generic BLOB store calls to their real backends."""
+
+    def __init__(self):
+        self._local: LocalFSBLOBStore = LocalFSBLOBStore()
+        self._s3: S3BLOBStore = S3BLOBStore()
+
+    async def get(self, uri: str, logger: Any) -> bytes:
+        """Returns binary value stored in BLOB with the supplied URI.
+
+        Raises Exception on error. Raises KeyError if the BLOB doesn't exist.
+        """
+        with (
+            metric_get_blob_errors.count_exceptions(),
+            metric_get_blob_latency.time(),
+        ):
+            metric_get_blob_requests.inc()
+            if _is_file_uri(uri):
+                return await self._local.get(uri, logger)
+            else:
+                return await self._s3.get(uri, logger)
+
+    async def presign_get_uri(self, uri: str, expires_in_sec: int, logger: Any) -> str:
+        """Returns a presigned URI for getting the BLOB with the supplied URI.
+
+        The URI allows to read any byte range in the BLOB."""
+        with (
+            metric_presign_uri_errors.count_exceptions(),
+            metric_presign_uri_latency.time(),
+        ):
+            metric_presign_uri_requests.inc()
+            if _is_file_uri(uri):
+                return await self._local.presign_get_uri(uri, expires_in_sec, logger)
+            else:
+                return await self._s3.presign_get_uri(uri, expires_in_sec, logger)
+
+    async def upload(self, uri: str, value: bytes, logger: Any) -> None:
+        """Stores the supplied binary value in a BLOB with the supplied URI.
+
+        Overwrites existing BLOB. Raises Exception on error.
+        """
+        with (
+            metric_upload_blob_errors.count_exceptions(),
+            metric_upload_blob_latency.time(),
+        ):
+            metric_upload_blob_requests.inc()
+            if _is_file_uri(uri):
+                await self._local.upload(uri, value, logger)
+            else:
+                await self._s3.upload(uri, value, logger)
+
+    async def create_multipart_upload(self, uri: str, logger: Any) -> str:
+        """Creates a multipart upload for BLOB with the supplied URI and returns the upload ID."""
+        with (
+            metric_create_multipart_upload_errors.count_exceptions(),
+            metric_create_multipart_upload_latency.time(),
+        ):
+            metric_create_multipart_upload_requests.inc()
+            if _is_file_uri(uri):
+                return await self._local.create_multipart_upload(uri, logger)
+            else:
+                return await self._s3.create_multipart_upload(uri, logger)
+
+    async def complete_multipart_upload(
+        self, uri: str, upload_id: str, parts_etags: list[str], logger: Any
+    ) -> None:
+        """Completes a multipart upload for BLOB with the supplied URI.
+
+        parts_etags is a list of ETags for the parts that were uploaded.
+        The list is ordered by part number starting from 1.
+        """
+        with (
+            metric_complete_multipart_upload_errors.count_exceptions(),
+            metric_complete_multipart_upload_latency.time(),
+        ):
+            metric_complete_multipart_upload_requests.inc()
+            if _is_file_uri(uri):
+                await self._local.complete_multipart_upload(
+                    uri, upload_id, parts_etags, logger
+                )
+            else:
+                await self._s3.complete_multipart_upload(
+                    uri, upload_id, parts_etags, logger
+                )
+
+    async def abort_multipart_upload(
+        self, uri: str, upload_id: str, logger: Any
+    ) -> None:
+        """Aborts a multipart upload for BLOB with the supplied URI."""
+        with (
+            metric_abort_multipart_upload_errors.count_exceptions(),
+            metric_abort_multipart_upload_latency.time(),
+        ):
+            metric_abort_multipart_upload_requests.inc()
+            if _is_file_uri(uri):
+                await self._local.abort_multipart_upload(uri, upload_id, logger)
+            else:
+                await self._s3.abort_multipart_upload(uri, upload_id, logger)
+
+    async def presign_upload_part_uri(
+        self,
+        uri: str,
+        part_number: int,
+        upload_id: str,
+        expires_in_sec: int,
+        logger: Any,
+    ) -> str:
+        """Returns a presigned URI for uploading a part in a multipart upload.
+
+        part_number starts from 1."""
+        with (
+            metric_presign_uri_errors.count_exceptions(),
+            metric_presign_uri_latency.time(),
+        ):
+            metric_presign_uri_requests.inc()
+            if _is_file_uri(uri):
+                return await self._local.presign_upload_part_uri(
+                    uri, part_number, upload_id, expires_in_sec, logger
+                )
+            else:
+                return await self._s3.presign_upload_part_uri(
+                    uri, part_number, upload_id, expires_in_sec, logger
+                )
+
+
+def _is_file_uri(uri: str) -> bool:
+    return uri.startswith("file://")

{indexify-0.4.22 → indexify-0.4.24}/src/indexify/executor/blob_store/local_fs_blob_store.py

@@ -16,7 +16,14 @@ class LocalFSBLOBStore:
         # Run synchronous code in a thread to not block the event loop.
         return await asyncio.to_thread(self._sync_get, _path_from_file_uri(uri))
 
-    async def put(self, uri: str, value: bytes, logger: Any) -> None:
+    async def presign_get_uri(self, uri: str, expires_in_sec: int, logger: Any) -> str:
+        """Returns a presigned URI for getting the file at the supplied URI.
+
+        For local files, just returns the file URI itself.
+        """
+        return uri
+
+    async def upload(self, uri: str, value: bytes, logger: Any) -> None:
         """Stores the supplied binary value in a file at the supplied URI.
 
         The URI must be a file URI (starts with "file://"). The path must be absolute.
@@ -25,6 +32,39 @@ class LocalFSBLOBStore:
         # Run synchronous code in a thread to not block the event loop.
         return await asyncio.to_thread(self._sync_put, _path_from_file_uri(uri), value)
 
+    async def create_multipart_upload(self, uri: str, logger: Any) -> str:
+        """Creates a multipart upload for local file and returns a dummy upload ID."""
+        # Local files do not require multipart upload, return a dummy ID
+        return "local-multipart-upload-id"
+
+    async def complete_multipart_upload(
+        self, uri: str, upload_id: str, parts_etags: list[str], logger: Any
+    ) -> None:
+        """Completes a multipart upload for local file. No-op for local files."""
+        # No action needed for local files
+        return None
+
+    async def abort_multipart_upload(
+        self, uri: str, upload_id: str, logger: Any
+    ) -> None:
+        """Aborts a multipart upload for local file. No-op for local files."""
+        # No action needed for local files
+        return None
+
+    async def presign_upload_part_uri(
+        self,
+        uri: str,
+        part_number: int,
+        upload_id: str,
+        expires_in_sec: int,
+        logger: Any,
+    ) -> str:
+        """Returns a presigned URI for uploading a part in a multipart upload for local file.
+
+        For local files, just returns the file URI itself.
+        """
+        return uri
+
     def _sync_get(self, path: str) -> bytes:
         if not os.path.isabs(path):
             raise ValueError(f"Path {path} must be absolute")

indexify-0.4.24/src/indexify/executor/blob_store/metrics/blob_store.py

@@ -0,0 +1,105 @@
+import prometheus_client
+
+from ...monitoring.metrics import latency_metric_for_fast_operation
+
+metric_get_blob_requests: prometheus_client.Counter = prometheus_client.Counter(
+    "blob_store_get_requests",
+    "Number of get BLOB requests in BLOB store",
+)
+metric_get_blob_errors: prometheus_client.Counter = prometheus_client.Counter(
+    "blob_store_get_request_errors",
+    "Number of get BLOB request errors in BLOB store",
+)
+metric_get_blob_latency: prometheus_client.Histogram = (
+    latency_metric_for_fast_operation(
+        "blob_store_get",
+        "BLOB store get BLOB request",
+    )
+)
+
+metric_presign_uri_requests: prometheus_client.Counter = prometheus_client.Counter(
+    "blob_store_presign_uri_requests",
+    "Number of presign URI requests in BLOB store",
+)
+metric_presign_uri_errors: prometheus_client.Counter = prometheus_client.Counter(
+    "blob_store_presign_uri_request_errors",
+    "Number of presign URI request errors in BLOB store",
+)
+metric_presign_uri_latency: prometheus_client.Histogram = (
+    latency_metric_for_fast_operation(
+        "blob_store_presign_uri",
+        "BLOB store presign URI request",
+    )
+)
+
+metric_upload_blob_requests: prometheus_client.Counter = prometheus_client.Counter(
+    "blob_store_upload_requests",
+    "Number of upload BLOB requests in BLOB store",
+)
+metric_upload_blob_errors: prometheus_client.Counter = prometheus_client.Counter(
+    "blob_store_upload_request_errors",
+    "Number of upload BLOB request errors in BLOB store",
+)
+metric_upload_blob_latency: prometheus_client.Histogram = (
+    latency_metric_for_fast_operation(
+        "blob_store_upload",
+        "BLOB store upload BLOB request",
+    )
+)
+
+metric_create_multipart_upload_requests: prometheus_client.Counter = (
+    prometheus_client.Counter(
+        "blob_store_create_multipart_upload_requests",
+        "Number of create multipart upload requests in BLOB store",
+    )
+)
+metric_create_multipart_upload_errors: prometheus_client.Counter = (
+    prometheus_client.Counter(
+        "blob_store_create_multipart_upload_request_errors",
+        "Number of create multipart upload request errors in BLOB store",
+    )
+)
+metric_create_multipart_upload_latency: prometheus_client.Histogram = (
+    latency_metric_for_fast_operation(
+        "blob_store_create_multipart_upload_request",
+        "create multipart upload request in BLOB store",
+    )
+)
+
+metric_complete_multipart_upload_requests: prometheus_client.Counter = (
+    prometheus_client.Counter(
+        "blob_store_complete_multipart_upload_requests",
+        "Number of complete multipart upload requests in BLOB store",
+    )
+)
+metric_complete_multipart_upload_errors: prometheus_client.Counter = (
+    prometheus_client.Counter(
+        "blob_store_complete_multipart_upload_request_errors",
+        "Number of complete multipart upload request errors in BLOB store",
+    )
+)
+metric_complete_multipart_upload_latency: prometheus_client.Histogram = (
+    latency_metric_for_fast_operation(
+        "blob_store_complete_multipart_upload_request",
+        "complete multipart upload request in BLOB store",
+    )
+)
+
+metric_abort_multipart_upload_requests: prometheus_client.Counter = (
+    prometheus_client.Counter(
+        "blob_store_abort_multipart_upload_requests",
+        "Number of abort multipart upload requests in BLOB store",
+    )
+)
+metric_abort_multipart_upload_errors: prometheus_client.Counter = (
+    prometheus_client.Counter(
+        "blob_store_abort_multipart_upload_request_errors",
+        "Number of abort multipart upload request errors in BLOB store",
+    )
+)
+metric_abort_multipart_upload_latency: prometheus_client.Histogram = (
+    latency_metric_for_fast_operation(
+        "blob_store_abort_multipart_upload_request",
+        "abort multipart upload request in BLOB store",
+    )
+)

indexify-0.4.24/src/indexify/executor/blob_store/s3_blob_store.py

@@ -0,0 +1,199 @@
+import asyncio
+from typing import Any, Optional
+
+import boto3
+from botocore.config import Config as BotoConfig
+from botocore.exceptions import ClientError as BotoClientError
+
+_MAX_RETRIES = 3
+
+
+class S3BLOBStore:
+    def __init__(self):
+        self._s3_client: Optional[Any] = None
+
+    def _lazy_create_client(self):
+        """Creates S3 client if it doesn't exist.
+
+        We create the client lazily only if S3 is used.
+        This is because S3 BLOB store is always created by Executor
+        and the creation will fail if user didn't configure S3 credentials and etc.
+        """
+        if self._s3_client is not None:
+            return
+
+        # The credentials and etc are fetched by boto3 library automatically following
+        # https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html#configuring-credentials
+        # This provides a lot of flexibility for the user and follows a well-known and documented logic.
+        self._s3_client = boto3.client(
+            "s3",
+            config=BotoConfig(
+                # https://boto3.amazonaws.com/v1/documentation/api/latest/guide/retries.html#standard-retry-mode
+                retries={
+                    "max_attempts": _MAX_RETRIES,
+                    "mode": "standard",
+                }
+            ),
+        )
+
+    async def get(self, uri: str, logger: Any) -> bytes:
+        """Returns binary value stored in S3 object at the supplied URI.
+
+        The URI must be S3 URI (starts with "s3://").
+        Raises Exception on error. Raises KeyError if the object doesn't exist.
+        """
+        try:
+            self._lazy_create_client()
+            bucket_name, key = _bucket_name_and_object_key_from_uri(uri)
+            response = await asyncio.to_thread(
+                self._s3_client.get_object, Bucket=bucket_name, Key=key
+            )
+            return response["Body"].read()
+        except BotoClientError as e:
+            logger.error("failed to get S3 object", uri=uri, exc_info=e)
+
+            if e.response["Error"]["Code"] == "NoSuchKey":
+                raise KeyError(f"Object {key} does not exist in bucket {bucket_name}")
+            raise
+        except Exception as e:
+            logger.error("failed to get S3 object", uri=uri, exc_info=e)
+            raise
+
+    async def presign_get_uri(self, uri: str, expires_in_sec: int, logger: Any) -> str:
+        """Returns a presigned URI for getting the S3 object at the supplied URI."""
+        self._lazy_create_client()
+        bucket_name, key = _bucket_name_and_object_key_from_uri(uri)
+        try:
+            s3_uri: str = await asyncio.to_thread(
+                self._s3_client.generate_presigned_url,
+                ClientMethod="get_object",
+                Params={"Bucket": bucket_name, "Key": key},
+                ExpiresIn=expires_in_sec,
+            )
+            return s3_uri.replace("https://", "s3://", 1)
+        except Exception as e:
+            logger.error(
+                "failed to presign URI for get_object operation",
+                uri=uri,
+                exc_info=e,
+                expires_in_sec=expires_in_sec,
+            )
+            raise
+
+    async def upload(self, uri: str, value: bytes, logger: Any) -> None:
+        """Stores the supplied binary value in a S3 object at the supplied URI.
+
+        The URI must be S3 URI (starts with "s3://").
+        Overwrites existing object. Raises Exception on error.
+        """
+        try:
+            self._lazy_create_client()
+            bucket_name, key = _bucket_name_and_object_key_from_uri(uri)
+            await asyncio.to_thread(
+                self._s3_client.put_object, Bucket=bucket_name, Key=key, Body=value
+            )
+        except Exception as e:
+            logger.error("failed to set S3 object", uri=uri, exc_info=e)
+            raise
+
+    async def create_multipart_upload(self, uri: str, logger: Any) -> str:
+        """Creates a multipart upload for S3 object and returns the upload ID."""
+        self._lazy_create_client()
+        bucket_name, key = _bucket_name_and_object_key_from_uri(uri)
+        try:
+            response = await asyncio.to_thread(
+                self._s3_client.create_multipart_upload,
+                Bucket=bucket_name,
+                Key=key,
+            )
+            return response["UploadId"]
+        except Exception as e:
+            logger.error("failed to create multipart upload", uri=uri, exc_info=e)
+            raise
+
+    async def complete_multipart_upload(
+        self, uri: str, upload_id: str, parts_etags: list[str], logger: Any
+    ) -> None:
+        """Completes a multipart upload for S3 object."""
+        self._lazy_create_client()
+        bucket_name, key = _bucket_name_and_object_key_from_uri(uri)
+        try:
+            await asyncio.to_thread(
+                self._s3_client.complete_multipart_upload,
+                Bucket=bucket_name,
+                Key=key,
+                UploadId=upload_id,
+                MultipartUpload={
+                    "Parts": [
+                        {"ETag": etag, "PartNumber": i + 1}
+                        for i, etag in enumerate(parts_etags)
+                    ]
+                },
+            )
+        except Exception as e:
+            logger.error("failed to complete multipart upload", uri=uri, exc_info=e)
+            raise
+
+    async def abort_multipart_upload(
+        self, uri: str, upload_id: str, logger: Any
+    ) -> None:
+        """Aborts a multipart upload for S3 object."""
+        self._lazy_create_client()
+        bucket_name, key = _bucket_name_and_object_key_from_uri(uri)
+        try:
+            await asyncio.to_thread(
+                self._s3_client.abort_multipart_upload,
+                Bucket=bucket_name,
+                Key=key,
+                UploadId=upload_id,
+            )
+        except Exception as e:
+            logger.error("failed to abort multipart upload", uri=uri, exc_info=e)
+            raise
+
+    async def presign_upload_part_uri(
+        self,
+        uri: str,
+        part_number: int,
+        upload_id: str,
+        expires_in_sec: int,
+        logger: Any,
+    ) -> str:
+        """Returns a presigned URI for uploading a part in a multipart upload for S3 object."""
+        self._lazy_create_client()
+        bucket_name, key = _bucket_name_and_object_key_from_uri(uri)
+        try:
+            response = await asyncio.to_thread(
+                self._s3_client.generate_presigned_url,
+                ClientMethod="upload_part",
+                Params={
+                    "Bucket": bucket_name,
+                    "Key": key,
+                    "UploadId": upload_id,
+                    "PartNumber": part_number,
+                },
+                ExpiresIn=expires_in_sec,
+            )
+            return response
+        except Exception as e:
+            logger.error(
+                "failed to presign URI for upload_part operation",
+                uri=uri,
+                exc_info=e,
+                part_number=part_number,
+                upload_id=upload_id,
+                expires_in_sec=expires_in_sec,
+            )
+            raise
+
+
+def _bucket_name_and_object_key_from_uri(uri: str) -> tuple[str, str]:
+    # Example S3 object URI:
+    # s3://test-indexify-server-blob-store-eugene-20250411/225b83f4-2aed-40a7-adee-b7a681f817f2
+    if not uri.startswith("s3://"):
+        raise ValueError(f"S3 URI '{uri}' is missing 's3://' prefix")
+
+    parts = uri[5:].split("/", 1)
+    if len(parts) != 2:
+        raise ValueError(f"Failed parsing bucket name from S3 URI '{uri}'")
+    return parts[0], parts[1]  # bucket_name, key