miniopy-async 1.23.2__tar.gz → 1.23.3__tar.gz

{miniopy_async-1.23.2 → miniopy_async-1.23.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: miniopy-async
-Version: 1.23.2
+Version: 1.23.3
 Summary: Asynchronous MinIO Client SDK for Python
 Home-page: https://github.com/hlf20010508/miniopy-async
 License: Apache-2.0

{miniopy_async-1.23.2 → miniopy_async-1.23.3}/miniopy_async/__init__.py

@@ -36,7 +36,7 @@ miniopy-async - Asynchronous MinIO Client SDK for Python
 
 __title__ = "miniopy-async"
 __author__ = "L-ING"
-__version__ = "1.23.2"
+__version__ = "1.23.3"
 __license__ = "Apache 2.0"
 __copyright__ = "(C) 2022 L-ING <hlf01@icloud.com>"
 

{miniopy_async-1.23.2 → miniopy_async-1.23.3}/miniopy_async/api.py

@@ -36,6 +36,7 @@ from urllib.parse import urlunsplit
 from xml.etree import ElementTree as ET
 
 import certifi
+import yarl
 from aiohttp import ClientResponse, ClientSession, ClientTimeout, TCPConnector
 from aiohttp.typedefs import LooseHeaders
 from aiohttp_retry import ExponentialRetry, RetryClient
@@ -54,6 +55,7 @@ from .datatypes import (
     ListMultipartUploadsResult,
     ListObjects,
     ListPartsResult,
+    MultipartUploader,
     Object,
     Part,
     PostPolicy,
@@ -361,9 +363,10 @@ class Minio:  # pylint: disable=too-many-public-methods
         self._ensure_session()
         session = cast(ClientSession | RetryClient, self._session)
 
+        yarl_url = yarl.URL(urlunsplit(url), encoded=True)
         response = await session.request(
             method,
-            urlunsplit(url),
+            yarl_url,
             data=body,
             headers=http_headers,
         )
@@ -2296,11 +2299,140 @@ class Minio:  # pylint: disable=too-many-public-methods
         element = ET.fromstring(await response.text())
         return cast(str, findtext(element, "UploadId", True))
 
+    def multipart_uploader(
+        self,
+        bucket_name: str,
+        object_name: str,
+        content_type: str = "application/octet-stream",
+        metadata: DictType | None = None,
+        sse: Sse | None = None,
+        tags: Tags | None = None,
+        retention: Retention | None = None,
+        legal_hold: bool = False,
+        extra_headers: DictType | None = None,
+    ) -> MultipartUploader:
+        """
+        Create a multipart uploader for large data.
+
+        :param str bucket_name: Name of the bucket.
+        :param str object_name: Object name in the bucket.
+        :param str content_type: Content type of the object.
+        :param DictType | None metadata: Any additional metadata to be uploaded along
+            with your PUT request.
+        :param Sse | None sse: Server-side encryption.
+        :param Tags | None tags: :class:`Tags` for the object.
+        :param Retention | None retention: :class:`Retention` configuration object.
+        :param bool legal_hold: Flag to set legal hold for the object.
+        :param DictType | None extra_headers: Any additional headers to be added with GET request.
+        :return: :class:`MultipartUploader` object.
+        :rtype: MultipartUploader
+
+        .. code-block:: python
+
+            import asyncio
+
+            from miniopy_async import Minio
+
+            client = Minio(
+                "play.min.io",
+                access_key="Q3AM3UQ867SPQQA43P2F",
+                secret_key="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG",
+                secure=True,  # http for False, https for True
+            )
+
+
+            async def main():
+                # Use context manager
+                print("example one")
+                async with client.multipart_uploader(
+                    "my-bucket",
+                    "my-object",
+                ) as uploader:
+                    await uploader.upload_part(b"hello" * 1024 * 1024, 1)
+                    await uploader.upload_part(b"world", 2)
+                result = uploader.result
+                print(
+                    "created {0} object; etag: {1}, version-id: {2}".format(
+                        result.object_name,
+                        result.etag,
+                        result.version_id,
+                    ),
+                )
+
+                # Use directly
+                print("example two")
+                uploader = client.multipart_uploader(
+                    "my-bucket",
+                    "my-object",
+                )
+                await uploader.upload_part(b"hello" * 1024 * 1024, 1)
+                await uploader.upload_part(b"world", 2)
+                result = await uploader.complete()
+                print(
+                    "created {0} object; etag: {1}, version-id: {2}".format(
+                        result.object_name,
+                        result.etag,
+                        result.version_id,
+                    ),
+                )
+
+                # Abort
+                print("example three")
+                uploader = client.multipart_uploader(
+                    "my-bucket",
+                    "my-object",
+                )
+                await uploader.upload_part(b"hello", 1)
+                await uploader.abort()
+
+                # Parallel upload
+                print("example four")
+                tasks = []
+                async with client.multipart_uploader(
+                    "my-bucket",
+                    "my-object",
+                ) as uploader:
+                    for i in range(1, 11):
+                        data = (f"part{i}" * 1024 * 1024).encode("utf-8")
+                        tasks.append(uploader.upload_part(data, i))
+                    await asyncio.gather(*tasks)
+                result = uploader.result
+                print(
+                    "created {0} object; etag: {1}, version-id: {2}".format(
+                        result.object_name,
+                        result.etag,
+                        result.version_id,
+                    ),
+                )
+
+
+            asyncio.run(main())
+        """
+        check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
+        check_object_name(object_name)
+        check_sse(sse)
+        if tags is not None and not isinstance(tags, Tags):
+            raise ValueError("tags must be Tags type")
+        if retention is not None and not isinstance(retention, Retention):
+            raise ValueError("retention must be Retention type")
+
+        headers = genheaders(metadata, sse, tags, retention, legal_hold)
+        headers["Content-Type"] = content_type or "application/octet-stream"
+        if extra_headers:
+            headers.update(extra_headers)
+
+        return MultipartUploader(
+            self,
+            bucket_name,
+            object_name,
+            headers,
+        )
+
     async def _put_object(
         self,
         bucket_name: str,
         object_name: str,
-        data: Substream,
+        data: bytes | BytesIO | Substream,
         headers: DictType | None,
         query_params: DictType | None = None,
     ) -> ObjectWriteResult:
@@ -2326,7 +2458,7 @@ class Minio:  # pylint: disable=too-many-public-methods
         self,
         bucket_name: str,
         object_name: str,
-        data: Substream,
+        data: bytes | BytesIO | Substream,
         headers: DictType | None,
         upload_id: str,
         part_number: int,
@@ -2371,6 +2503,7 @@ class Minio:  # pylint: disable=too-many-public-methods
         tags: Tags | None = None,
         retention: Retention | None = None,
         legal_hold: bool = False,
+        write_offset: int | None = None,
     ) -> ObjectWriteResult:
         """
         Uploads data from a stream to an object in a bucket.
@@ -2389,6 +2522,7 @@ class Minio:  # pylint: disable=too-many-public-methods
         :param Tags | None tags: :class:`Tags` for the object.
         :param Retention | None retention: :class:`Retention` configuration object.
         :param bool legal_hold: Flag to set legal hold for the object.
+        :param int | None write_offset: Offset byte for appending data to existing object.
         :return: :class:`ObjectWriteResult` object.
         :rtype: ObjectWriteResult
         :raise ValueError: If tags is not of type :class:`Tags`.
@@ -2493,6 +2627,12 @@ class Minio:  # pylint: disable=too-many-public-methods
             raise ValueError("retention must be Retention type")
         if not callable(getattr(data, "read")):
             raise ValueError("input data must have callable read()")
+        if write_offset is not None:
+            if write_offset < 0:
+                raise ValueError("write offset should not be negative")
+            if length < 0:
+                raise ValueError("length must be provided for write offset")
+            part_size = length if length > MIN_PART_SIZE else MIN_PART_SIZE
         part_size, part_count = get_part_info(length, part_size)
         if progress:
             # Set progress bar length and object name before upload
@@ -2500,6 +2640,8 @@ class Minio:  # pylint: disable=too-many-public-methods
 
         headers = genheaders(metadata, sse, tags, retention, legal_hold)
         headers["Content-Type"] = content_type or "application/octet-stream"
+        if write_offset:
+            headers["x-amz-write-offset-bytes"] = str(write_offset)
 
         object_size = length
         uploaded_size = 0
@@ -2606,6 +2748,169 @@ class Minio:  # pylint: disable=too-many-public-methods
                 )
             raise exc
 
+    async def append_object(
+        self,
+        bucket_name: str,
+        object_name: str,
+        data: BinaryIO,
+        length: int,
+        chunk_size: int | None = None,
+        progress: ProgressType | None = None,
+        extra_headers: DictType | None = None,
+    ) -> ObjectWriteResult:
+        """
+        Appends from a stream to existing object in a bucket.
+
+        :param str bucket_name: Name of the bucket.
+        :param str object_name: Object name in the bucket.
+        :param BinaryIO data: An object having callable read() returning bytes object.
+        :param int length: Data size; -1 for unknown size.
+        :param int chunk_size: Chunk size to optimize uploads.
+        :param ProgressType | None progress: A progress object.
+        :param DictType | None extra_headers: Any additional headers to be added with GET request.
+        :return: :class:`ObjectWriteResult` object.
+        :rtype: ObjectWriteResult
+        :raise ValueError: If length is 0.
+        :raise ValueError: If chunk_size is less than 5 MiB or greater than 5 GiB.
+        :raise IOError: If stream doesn't have enough data to read.
+
+        .. code-block:: python
+
+            import asyncio
+            import io
+            from urllib.request import urlopen
+
+            from miniopy_async import Minio
+
+            client = Minio(
+                "play.min.io",
+                access_key="Q3AM3UQ867SPQQA43P2F",
+                secret_key="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG",
+                secure=True,  # http for False, https for True
+            )
+
+
+            async def main():
+                # Upload data.
+                result = await client.put_object(
+                    "my-bucket",
+                    "my-object",
+                    io.BytesIO(b"hello, "),
+                    7,
+                )
+                print(f"created {result.object_name} object; etag: {result.etag}")
+
+                # Append data.
+                result = await client.append_object(
+                    "my-bucket",
+                    "my-object",
+                    io.BytesIO(b"world"),
+                    5,
+                )
+                print(f"appended {result.object_name} object; etag: {result.etag}")
+
+                # Append data in chunks.
+                data = urlopen(
+                    "https://www.kernel.org/pub/linux/kernel/v6.x/linux-6.13.12.tar.xz",
+                )
+                result = await client.append_object(
+                    "my-bucket",
+                    "my-object",
+                    data,
+                    148611164,
+                    5 * 1024 * 1024,
+                )
+                print(f"appended {result.object_name} object; etag: {result.etag}")
+
+                # Append unknown sized data.
+                data = urlopen(
+                    "https://www.kernel.org/pub/linux/kernel/v6.x/linux-6.14.3.tar.xz",
+                )
+                result = await client.append_object(
+                    "my-bucket",
+                    "my-object",
+                    data,
+                    149426584,
+                    5 * 1024 * 1024,
+                )
+                print(f"appended {result.object_name} object; etag: {result.etag}")
+
+
+            asyncio.run(main())
+        """
+        if length == 0:
+            raise ValueError("length should not be zero")
+        if chunk_size is not None:
+            if chunk_size < MIN_PART_SIZE:
+                raise ValueError("chunk size must be minimum of 5 MiB")
+            if chunk_size > MAX_PART_SIZE:
+                raise ValueError("chunk size must be less than 5 GiB")
+        else:
+            chunk_size = length if length > MIN_PART_SIZE else MIN_PART_SIZE
+
+        chunk_count = -1
+        if length > 0:
+            chunk_count = int(length / chunk_size)
+            if (chunk_count * chunk_size) < length:
+                chunk_count += 1
+            chunk_count = chunk_count or 1
+
+        object_size = length
+        uploaded_size = 0
+        chunk_number = 0
+        stop = False
+
+        stat = await self.stat_object(bucket_name, object_name)
+        write_offset = cast(int, stat.size)
+        upload_result = None
+
+        while not stop:
+            chunk_number += 1
+            if chunk_count > 0:
+                if chunk_number == chunk_count:
+                    chunk_size = object_size - uploaded_size
+                    stop = True
+                chunk_data = Substream(data, uploaded_size, chunk_size)
+                if chunk_data.size != chunk_size:
+                    raise IOError(
+                        f"stream having not enough data;"
+                        f"expected: {chunk_size}, "
+                        f"got: {chunk_data.size} bytes"
+                    )
+            else:
+                chunk_data = Substream(data, uploaded_size, chunk_size)
+                if chunk_data.size == 0:
+                    break
+                if chunk_data.size < chunk_size:
+                    chunk_count = chunk_number
+                    stop = True
+                    chunk_size = chunk_data.size
+
+            if progress:
+                await progress.update(chunk_size)
+
+            uploaded_size += chunk_size
+
+            headers = extra_headers or {}
+            headers["x-amz-write-offset-bytes"] = str(write_offset)
+            upload_result = await self._put_object(
+                bucket_name,
+                object_name,
+                chunk_data,
+                headers=headers,
+            )
+            write_offset += chunk_size
+
+        upload_result = cast(ObjectWriteResult, upload_result)
+        return ObjectWriteResult(
+            cast(str, upload_result.bucket_name),
+            cast(str, upload_result.object_name),
+            upload_result.version_id,
+            upload_result.etag,
+            upload_result.http_headers,
+            location=upload_result.location,
+        )
+
     def list_objects(
         self,
         bucket_name: str,
@@ -4228,9 +4533,10 @@ class Minio:  # pylint: disable=too-many-public-methods
         metadata["X-Amz-Meta-Snowball-Auto-Extract"] = "true"
 
         name = staging_filename
-        mode = "w:gz" if compression else "w"
         fileobj = None if name else BytesIO()
-        with tarfile.open(name=name, mode=mode, fileobj=fileobj) as tar:
+        with tarfile.open(
+            name=name, mode="w:gz" if compression else "w", fileobj=fileobj
+        ) as tar:
             for obj in object_list:
                 if obj.filename:
                     tar.add(obj.filename, obj.object_name)

{miniopy_async-1.23.2 → miniopy_async-1.23.3}/miniopy_async/datatypes.py

@@ -21,6 +21,7 @@ Response of ListBuckets, ListObjects, ListObjectsV2 and ListObjectVersions API.
 from __future__ import absolute_import, annotations
 
 import base64
+import io
 import itertools
 import json
 from collections import OrderedDict
@@ -1286,3 +1287,100 @@ class DeleteErrors:
         async for error in self:
             errors.append(error)
         return errors
+
+
+class MultipartUploader:
+    def __init__(
+        self,
+        client: Minio,
+        bucket_name: str,
+        object_name: str,
+        headers: DictType,
+    ):
+        self._client = client
+        self._bucket_name = bucket_name
+        self._object_name = object_name
+        self._headers = headers
+        self._upload_id = ""
+        self._parts = []
+        self._result: CompleteMultipartUploadResult | None = None
+
+    @property
+    def result(self) -> CompleteMultipartUploadResult:
+        """
+        Get multipart upload result.
+
+        :return: Multipart upload result.
+        :rtype: CompleteMultipartUploadResult
+        :raises ValueError: If multipart upload is not completed.
+        """
+        if not self._result:
+            raise ValueError("Multipart upload not completed")
+        return self._result
+
+    async def _init(self):
+        self._upload_id = await self._client._create_multipart_upload(
+            self._bucket_name, self._object_name, self._headers
+        )
+
+    async def complete(self) -> CompleteMultipartUploadResult:
+        """
+        Complete multipart upload.
+
+        :return: Multipart upload result.
+        :rtype: CompleteMultipartUploadResult
+        """
+        self._result = await self._client._complete_multipart_upload(
+            self._bucket_name, self._object_name, self._upload_id, self._parts
+        )
+        return self.result
+
+    async def abort(self):
+        """Abort multipart upload."""
+        await self._client._abort_multipart_upload(
+            self._bucket_name, self._object_name, self._upload_id
+        )
+
+    async def __aenter__(self):
+        await self._init()
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        await self.complete()
+
+    async def upload_part(self, data: bytes | io.BytesIO, part_number: int) -> Part:
+        """
+        Upload a part of the object.
+
+        The data size must be larger than 5 MiB and less than 5 GiB except the last part.
+
+        :param bytes | io.BytesIO data: Data to upload as part.
+        :param int part_number: Number of the part.
+        :return: Part information.
+        :rtype: Part
+        :raises ValueError: If part_number is not a positive integer.
+        """
+        if not self._upload_id:
+            await self._init()
+        try:
+            if not isinstance(part_number, int) or part_number < 1:
+                raise ValueError("part_number must be a positive integer")
+
+            etag = await self._client._upload_part(
+                self._bucket_name,
+                self._object_name,
+                data,
+                self._headers,
+                self._upload_id,
+                part_number,
+            )
+
+            part = Part(part_number, etag)
+            self._parts.append(part)
+
+            return part
+
+        except Exception as e:
+            await self.abort()
+
+            raise e

{miniopy_async-1.23.2 → miniopy_async-1.23.3}/miniopy_async/helpers.py

@@ -26,9 +26,10 @@ import math
 import os
 import platform
 import re
+import threading
 import urllib.parse
 from datetime import datetime
-from typing import BinaryIO, Dict, List, Protocol, Tuple, Union
+from typing import BinaryIO, Dict, List, Protocol, Tuple, Union, cast
 
 from aiohttp.typedefs import LooseHeaders
 
@@ -865,36 +866,67 @@ class Substream(io.IOBase):
 
         self._base = base
         self._start = offset
-        self._size = self._get_size(part_size)
         self._pos = 0
         self._closed = False
 
+        self._lock = threading.RLock()
+        self._fd: int | None = None
+        self._has_pread: bool = False
+        try:
+            if hasattr(base, "fileno"):
+                self._fd = base.fileno()
+                self._has_pread = self._fd is not None and hasattr(os, "pread")
+        except Exception:
+            self._fd = None
+            self._has_pread = False
+
+        self._size = self._get_size(part_size)
+
+    def _read_at(self, abs_offset: int, n: int) -> bytes:
+        """Read n bytes from the underlying stream at absolute position abs_offset without changing other readers' pointers."""
+        if n <= 0:
+            return b""
+        if self._has_pread:
+            # POSIX: thread-safe, does not affect file pointer
+            return os.pread(cast(int, self._fd), n, abs_offset)
+        # Fallback: save/restore pointer + lock, ensure atomicity within this object
+        with self._lock:
+            curr = self._base.tell()
+            try:
+                self._base.seek(abs_offset)
+                return self._base.read(n)
+            finally:
+                try:
+                    self._base.seek(curr)
+                except Exception:
+                    pass
+
     def _get_size(self, part_size: int) -> int:
         """Get size of the substream using bisection."""
-        head = self._start
-        tail = self._start + part_size - 1
+        if part_size == 0:
+            return 0
+
+        start = self._start
+        end = start + part_size
 
-        self._base.seek(tail)
-        data = self._base.read(1)
-        if data:
+        if self._read_at(end - 1, 1):
             return part_size
 
-        self._base.seek(head)
-        data = self._base.read(1)
-        if not data:
+        if not self._read_at(start, 1):
             return 0
 
         # bisection
-        while tail >= head:
-            center = (head + tail) // 2
-            self._base.seek(center)
-            data = self._base.read(2)
-            if not data:
-                tail = center - 1
-            elif len(data) == 2:
-                head = center + 1
+        lo = start
+        hi = end - 1
+        while lo <= hi:
+            mid = (lo + hi) // 2
+            if self._read_at(mid, 1):
+                if self._read_at(mid + 1, 1):
+                    lo = mid + 1
+                else:
+                    return mid - start + 1
             else:
-                return center - self._start + 1
+                hi = mid - 1
 
         raise UnreachableError
 
@@ -906,27 +938,20 @@ class Substream(io.IOBase):
     def read(self, size: int = -1) -> bytes:
         """Read data from the substream."""
         self._check_closed()
-        self._base.seek(self._start + self._pos)
-
         remaining = self._size - self._pos
         if remaining <= 0:
             return b""
         if size < 0 or size > remaining:
             size = remaining
-        data = self._base.read(size)
+        abs_off = self._start + self._pos
+        data = self._read_at(abs_off, size)
+
         self._pos += len(data)
         return data
 
     def is_empty(self) -> bool:
         """Check if the substream is empty."""
-        pos = self._pos
-
-        self.seek(0)
-        if not self.read(1):
-            return True
-
-        self._pos = pos
-        return False
+        return not bool(self._read_at(self._start, 1))
 
     def seek(self, offset: int, whence: int = io.SEEK_SET) -> int:
         """Seek to a position in the substream."""

{miniopy_async-1.23.2 → miniopy_async-1.23.3}/miniopy_async/minioadmin.py

@@ -55,6 +55,7 @@ from .signer import sign_v4_s3
 class _COMMAND(Enum):
     """Admin Command enumerations."""
 
+    ACCOUNT_INFO = "accountinfo"
     ADD_USER = "add-user"
     USER_INFO = "user-info"
     LIST_USERS = "list-users"
@@ -324,6 +325,15 @@ class MinioAdmin:
         )
         return await response.text()
 
+    async def account_info(self, prefix_usage: bool = False) -> str:
+        """Get usage information for the authenticating account"""
+        response = await self._url_open(
+            "GET",
+            _COMMAND.ACCOUNT_INFO,
+            query_params={"prefix-usage": "true"} if prefix_usage else None,
+        )
+        return await response.text()
+
     async def user_add(self, access_key: str, secret_key: str) -> str:
         """Create user with access and secret keys"""
         body = json.dumps({"status": "enabled", "secretKey": secret_key}).encode()

{miniopy_async-1.23.2 → miniopy_async-1.23.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "miniopy-async"
-version = "1.23.2"
+version = "1.23.3"
 description = "Asynchronous MinIO Client SDK for Python"
 authors = [
     "L-ING <hlf01@icloud.com>",