miniopy-async 1.16__tar.gz → 1.18__tar.gz

{miniopy-async-1.16 → miniopy-async-1.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: miniopy-async
-Version: 1.16
+Version: 1.18
 Summary: Asynchronous MinIO Client SDK for Python
 Home-page: https://github.com/hlf20010508/miniopy-async
 Author: L-ING
@@ -87,7 +87,7 @@ pipenv install miniopy-async
 
 Github Repository
 ```sh
-pipenv install git+https://github.com/hlf20010508/miniopy-async.git#egg=miniopy-async
+pipenv install "miniopy-async@ git+https://github.com/hlf20010508/miniopy-async.git"
 ```
 
 <span id="example"></span>

{miniopy-async-1.16 → miniopy-async-1.18}/README.md

@@ -60,7 +60,7 @@ pipenv install miniopy-async
 
 Github Repository
 ```sh
-pipenv install git+https://github.com/hlf20010508/miniopy-async.git#egg=miniopy-async
+pipenv install "miniopy-async@ git+https://github.com/hlf20010508/miniopy-async.git"
 ```
 
 <span id="example"></span>

{miniopy-async-1.16 → miniopy-async-1.18}/miniopy_async/__init__.py

@@ -45,7 +45,7 @@ miniopy-async - Asynchronous MinIO Client SDK for Python
 
 __title__ = "miniopy-async"
 __author__ = "L-ING."
-__version__ = "1.16"
+__version__ = "1.18"
 __license__ = "Apache 2.0"
 __copyright__ = "(C) 2022 L-ING <hlf01@icloud.com>"
 

{miniopy-async-1.16 → miniopy-async-1.18}/miniopy_async/api.py

@@ -33,6 +33,9 @@ import asyncio
 import itertools
 import os
 import platform
+from random import random
+from io import BytesIO
+import tarfile
 
 # import weakref
 from concurrent.futures import ThreadPoolExecutor
@@ -44,7 +47,7 @@ import aiofile
 import aiohttp
 
 from . import __title__, __version__, time
-from .commonconfig import COPY, ComposeSource, CopySource, REPLACE, Tags
+from .commonconfig import COPY, ComposeSource, CopySource, REPLACE, Tags, SnowballObject
 from .credentials import StaticProvider
 from .datatypes import (
     AsyncEventIterable,
@@ -376,7 +379,7 @@ class Minio:  # pylint: disable=too-many-public-methods
                         query_params=query_params,
                         session=session,
                     )
-                return response
+                    return response
             else:
                 response = await self._url_open(
                     method,
@@ -407,7 +410,7 @@ class Minio:  # pylint: disable=too-many-public-methods
                         query_params=query_params,
                         session=session,
                     )
-                return response
+                    return response
             else:
                 response = await self._url_open(
                     method,
@@ -2502,6 +2505,7 @@ class Minio:  # pylint: disable=too-many-public-methods
         object_name,
         ssec=None,
         version_id=None,
+        request_headers=None,
         extra_query_params=None,
     ):
         """
@@ -2511,6 +2515,7 @@ class Minio:  # pylint: disable=too-many-public-methods
         :param object_name: Object name in the bucket.
         :param ssec: Server-side encryption customer key.
         :param version_id: Version ID of the object.
+        :param request_headers: Any additional headers to be added with GET request.
         :param extra_query_params: Extra query parameters for advanced usage.
         :return: :class:`Object <Object>`.
 
@@ -2570,6 +2575,8 @@ class Minio:  # pylint: disable=too-many-public-methods
         check_ssec(ssec)
 
         headers = ssec.headers() if ssec else {}
+        if request_headers:
+            headers.update(request_headers)
         query_params = extra_query_params or {}
         query_params.update({"versionId": version_id} if version_id else {})
         async with aiohttp.ClientSession() as session:
@@ -3973,8 +3980,11 @@ class Minio:  # pylint: disable=too-many-public-methods
             days=10))
 
             async def main():
-                await client.set_object_retention("my-bucket", "my-object",
-                config)
+                await client.set_object_retention(
+                    "my-bucket",
+                    "my-object",
+                    config
+                )
 
             loop = asyncio.get_event_loop()
             loop.run_until_complete(main())
@@ -3996,6 +4006,127 @@ class Minio:  # pylint: disable=too-many-public-methods
             query_params=query_params,
         )
 
+    async def upload_snowball_objects(
+        self,
+        bucket_name,
+        object_list,
+        metadata=None,
+        sse=None,
+        tags=None,
+        retention=None,
+        legal_hold=False,
+        staging_filename=None,
+        compression=False,
+    ):
+        """
+        Uploads multiple objects in a single put call. It is done by creating
+        intermediate TAR file optionally compressed which is uploaded to S3
+        service.
+
+        :param bucket_name: Name of the bucket.
+        :param object_list: An iterable containing
+            :class:`SnowballObject <SnowballObject>` object.
+        :param metadata: Any additional metadata to be uploaded along
+            with your PUT request.
+        :param sse: Server-side encryption.
+        :param tags: :class:`Tags` for the object.
+        :param retention: :class:`Retention` configuration object.
+        :param legal_hold: Flag to set legal hold for the object.
+        :param staging_filename: A staging filename to create intermediate tarball.
+        :param compression: Flag to compress TAR ball.
+        :return: :class:`ObjectWriteResult` object.
+
+        Example::
+            from miniopy_async import Minio
+            from miniopy_async.commonconfig import SnowballObject
+            import io
+            from datetime import datetime
+            import asyncio
+
+            client = Minio(
+                "play.min.io",
+                access_key="Q3AM3UQ867SPQQA43P2F",
+                secret_key="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG",
+                secure=True,  # http for False, https for True
+            )
+
+            async def main():
+                result = await client.upload_snowball_objects(
+                    "my-bucket",
+                    [
+                        SnowballObject("my-object1", filename="filename"),
+                        SnowballObject(
+                            "my-object2", data=io.BytesIO(b"hello"), length=5,
+                        ),
+                        SnowballObject(
+                            "my-object3", data=io.BytesIO(b"world"), length=5,
+                            mod_time=datetime.now(),
+                        ),
+                    ],
+                )
+
+            loop = asyncio.get_event_loop()
+            loop.run_until_complete(main())
+            loop.close()
+        """
+        check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
+
+        object_name = f"snowball.{random()}.tar"
+
+        # turn list like objects into an iterator.
+        object_list = itertools.chain(object_list)
+
+        metadata = metadata or {}
+        metadata["X-Amz-Meta-Snowball-Auto-Extract"] = "true"
+
+        name = staging_filename
+        mode = "w:gz" if compression else "w"
+        fileobj = None if name else BytesIO()
+        with tarfile.open(name=name, mode=mode, fileobj=fileobj) as tar:
+            for obj in object_list:
+                if obj.filename:
+                    tar.add(obj.filename, obj.object_name)
+                else:
+                    info = tarfile.TarInfo(obj.object_name)
+                    info.size = obj.length
+                    info.mtime = int(
+                        time.to_float(obj.mod_time or time.utcnow()),
+                    )
+                    tar.addfile(info, obj.data)
+
+        if not name:
+            length = fileobj.tell()
+            fileobj.seek(0)
+        else:
+            length = os.stat(name).st_size
+
+        part_size = 0 if length < MIN_PART_SIZE else length
+
+        if name:
+            return await self.fput_object(
+                bucket_name,
+                object_name,
+                staging_filename,
+                metadata=metadata,
+                sse=sse,
+                tags=tags,
+                retention=retention,
+                legal_hold=legal_hold,
+                part_size=part_size,
+            )
+        return await self.put_object(
+            bucket_name,
+            object_name,
+            fileobj,
+            length,
+            metadata=metadata,
+            sse=sse,
+            tags=tags,
+            retention=retention,
+            legal_hold=legal_hold,
+            part_size=part_size,
+        )
+
     async def _list_objects(
         # pylint: disable=too-many-arguments,too-many-branches
         self,

{miniopy-async-1.16 → miniopy-async-1.18}/miniopy_async/commonconfig.py

@@ -508,3 +508,51 @@ class ComposeSource(ObjectConditionalReadArgs):
             src.modified_since,
             src.unmodified_since,
         )
+
+
+class SnowballObject:
+    """A source object definition for upload_snowball_objects method."""
+
+    def __init__(
+            self,
+            object_name,
+            filename=None,
+            data=None,
+            length=None,
+            mod_time=None,
+    ):
+        self._object_name = object_name
+        if (filename is not None) ^ (data is not None):
+            self._filename = filename
+            self._data = data
+            self._length = length
+        else:
+            raise ValueError("only one of filename or data must be provided")
+        if mod_time is not None and not isinstance(mod_time, datetime.datetime):
+            raise ValueError("mod_time must be datetime.datetime type")
+        self._mod_time = mod_time
+
+    @property
+    def object_name(self):
+        """Get object name."""
+        return self._object_name
+
+    @property
+    def filename(self):
+        """Get filename."""
+        return self._filename
+
+    @property
+    def data(self):
+        """Get data."""
+        return self._data
+
+    @property
+    def length(self):
+        """Get length."""
+        return self._length
+
+    @property
+    def mod_time(self):
+        """Get modification time."""
+        return self._mod_time

{miniopy-async-1.16 → miniopy-async-1.18}/miniopy_async/helpers.py

@@ -43,19 +43,16 @@ MAX_MULTIPART_OBJECT_SIZE = 5 * 1024 * 1024 * 1024 * 1024  # 5TiB
 MAX_PART_SIZE = 5 * 1024 * 1024 * 1024  # 5GiB
 MIN_PART_SIZE = 5 * 1024 * 1024  # 5MiB
 
-_VALID_BUCKETNAME_REGEX = re.compile(
-    "^[A-Za-z0-9][A-Za-z0-9\\.\\-\\_\\:]{1,61}[A-Za-z0-9]$"
+_BUCKET_NAME_REGEX = re.compile(r'^[a-z0-9][a-z0-9\.\-]{1,61}[a-z0-9]$')
+_OLD_BUCKET_NAME_REGEX = re.compile(
+    r'^[a-z0-9][a-z0-9_\.\-\:]{1,61}[a-z0-9]$',
+    re.IGNORECASE
 )
-_VALID_BUCKETNAME_STRICT_REGEX = re.compile("^[a-z0-9][a-z0-9\\.\\-]{1,61}[a-z0-9]$")
-_VALID_IP_ADDRESS = re.compile(r"^(\d+\.){3}\d+$")
-_ALLOWED_HOSTNAME_REGEX = re.compile(
-    "^((?!-)(?!_)[A-Z_\\d-]{1,63}(?<!-)(?<!_)\\.)*((?!_)(?!-)"
-    + "[A-Z_\\d-]{1,63}(?<!-)(?<!_))$",
-    re.IGNORECASE,
-)
-
-_EXTRACT_REGION_REGEX = re.compile("s3[.-]?(.+?).amazonaws.com")
 
+_IPV4_REGEX = re.compile(
+    r'^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[0-9])\.){3}'
+    r'(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[0-9])$'
+)
 
 def quote(resource, safe="/", encoding=None, errors=None):
     """
@@ -191,44 +188,38 @@ def makedirs(path):
             ) from exc
 
 
-def check_bucket_name(bucket_name, strict=False):
+def check_bucket_name(
+        bucket_name: str,
+        strict: bool = False,
+        s3_check: bool = False,
+):
     """Check whether bucket name is valid optional with strict check or not."""
 
-    # Verify bucket name is not empty
-    bucket_name = str(bucket_name).strip()
-    if not bucket_name:
-        raise ValueError("Bucket name cannot be empty.")
-
-    # Verify bucket name length.
-    if len(bucket_name) < 3:
-        raise ValueError("Bucket name cannot be less than" " 3 characters.")
-    if len(bucket_name) > 63:
-        raise ValueError("Bucket name cannot be greater than" " 63 characters.")
+    if strict:
+        if not _BUCKET_NAME_REGEX.match(bucket_name):
+            raise ValueError(f'invalid bucket name {bucket_name}')
+    else:
+        if not _OLD_BUCKET_NAME_REGEX.match(bucket_name):
+            raise ValueError(f'invalid bucket name {bucket_name}')
 
-    match = _VALID_IP_ADDRESS.match(bucket_name)
-    if match:
-        raise ValueError("Bucket name cannot be an ip address")
+    if _IPV4_REGEX.match(bucket_name):
+        raise ValueError(f'bucket name {bucket_name} must not be formatted '
+                         'as an IP address')
 
-    unallowed_successive_chars = ["..", ".-", "-."]
+    unallowed_successive_chars = ['..', '.-', '-.']
     if any(x in bucket_name for x in unallowed_successive_chars):
-        raise ValueError(
-            "Bucket name contains invalid "
-            "successive chars " + str(unallowed_successive_chars) + "."
-        )
-
-    if strict:
-        match = _VALID_BUCKETNAME_STRICT_REGEX.match(bucket_name)
-        if (not match) or match.end() != len(bucket_name):
-            raise ValueError(
-                "Bucket name contains invalid " "characters (strictly enforced)."
-            )
-
-    match = _VALID_BUCKETNAME_REGEX.match(bucket_name)
-    if (not match) or match.end() != len(bucket_name):
-        raise ValueError(
-            "Bucket name does not follow S3 standards."
-            " Bucket: {0}".format(bucket_name)
-        )
+        raise ValueError(f'bucket name {bucket_name} contains invalid '
+                         'successive characters')
+
+    if (
+            s3_check and
+            bucket_name.startswith("xn--") or
+            bucket_name.endswith("-s3alias") or
+            bucket_name.endswith("--ol-s3")
+    ):
+        raise ValueError(f"bucket name {bucket_name} must not start with "
+                         "'xn--' and must not end with '--s3alias' or "
+                         "'--ol-s3'")
 
 
 def check_non_empty_string(string):

{miniopy-async-1.16 → miniopy-async-1.18}/miniopy_async/time.py

@@ -22,6 +22,7 @@ from __future__ import absolute_import
 
 import locale
 from contextlib import contextmanager
+import time as ctime
 from datetime import datetime, timezone
 
 from . import __LOCALE_LOCK__
@@ -96,3 +97,7 @@ def utcnow():
 def to_signer_date(value):
     """Format datetime into SignatureV4 date formatted string."""
     return _to_utc(value).strftime("%Y%m%d")
+
+def to_float(value: datetime) -> float:
+    """Convert datetime into float value."""
+    return ctime.mktime(value.timetuple()) + value.microsecond * 1e-6

{miniopy-async-1.16 → miniopy-async-1.18}/miniopy_async.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: miniopy-async
-Version: 1.16
+Version: 1.18
 Summary: Asynchronous MinIO Client SDK for Python
 Home-page: https://github.com/hlf20010508/miniopy-async
 Author: L-ING
@@ -87,7 +87,7 @@ pipenv install miniopy-async
 
 Github Repository
 ```sh
-pipenv install git+https://github.com/hlf20010508/miniopy-async.git#egg=miniopy-async
+pipenv install "miniopy-async@ git+https://github.com/hlf20010508/miniopy-async.git"
 ```
 
 <span id="example"></span>