megfile 3.1.0.post2__py3-none-any.whl → 3.1.2__py3-none-any.whl

megfile/lib/http_prefetch_reader.py

@@ -3,8 +3,17 @@ from typing import Optional
 
 import requests
 
-from megfile.config import DEFAULT_BLOCK_CAPACITY, DEFAULT_BLOCK_SIZE, HTTP_MAX_RETRY_TIMES
-from megfile.errors import HttpBodyIncompleteError, UnsupportedError, http_should_retry, patch_method
+from megfile.config import (
+    DEFAULT_BLOCK_CAPACITY,
+    DEFAULT_BLOCK_SIZE,
+    HTTP_MAX_RETRY_TIMES,
+)
+from megfile.errors import (
+    HttpBodyIncompleteError,
+    UnsupportedError,
+    http_should_retry,
+    patch_method,
+)
 from megfile.lib.base_prefetch_reader import BasePrefetchReader
 from megfile.lib.compat import fspath
 from megfile.pathlike import PathLike
@@ -13,24 +22,29 @@ DEFAULT_TIMEOUT = (60, 60 * 60 * 24)
 
 
 class HttpPrefetchReader(BasePrefetchReader):
-    '''
-    Reader to fast read the http content, service must support Accept-Ranges. 
-    This will divide the file content into equal parts of block_size size, and will use LRU to cache at most block_capacity blocks in memory.
-    open(), seek() and read() will trigger prefetch read. 
-    The prefetch will cached block_forward blocks of data from offset position (the position after reading if the called function is read).
-    '''
+    """
+    Reader to fast read the http content, service must support Accept-Ranges.
 
-    def __init__(
-            self,
-            url: PathLike,
-            *,
-            content_size: Optional[int] = None,
-            block_size: int = DEFAULT_BLOCK_SIZE,
-            block_capacity: int = DEFAULT_BLOCK_CAPACITY,
-            block_forward: Optional[int] = None,
-            max_retries: int = HTTP_MAX_RETRY_TIMES,
-            max_workers: Optional[int] = None):
+    This will divide the file content into equal parts of block_size size, and will use
+    LRU to cache at most block_capacity blocks in memory.
+
+    open(), seek() and read() will trigger prefetch read.
+
+    The prefetch will cached block_forward blocks of data from offset position
+    (the position after reading if the called function is read).
+    """
 
+    def __init__(
+        self,
+        url: PathLike,
+        *,
+        content_size: Optional[int] = None,
+        block_size: int = DEFAULT_BLOCK_SIZE,
+        block_capacity: int = DEFAULT_BLOCK_CAPACITY,
+        block_forward: Optional[int] = None,
+        max_retries: int = HTTP_MAX_RETRY_TIMES,
+        max_workers: Optional[int] = None,
+    ):
         self._url = url
         self._content_size = content_size
 
@@ -39,68 +53,77 @@ class HttpPrefetchReader(BasePrefetchReader):
             block_capacity=block_capacity,
             block_forward=block_forward,
             max_retries=max_retries,
-            max_workers=max_workers)
+            max_workers=max_workers,
+        )
 
     def _get_content_size(self) -> int:
         if self._content_size is not None:
             return self._content_size
 
         first_index_response = self._fetch_response()
-        if first_index_response['Headers'].get('Accept-Ranges') != 'bytes':
+        if first_index_response["Headers"].get("Accept-Ranges") != "bytes":
             raise UnsupportedError(
-                f'Unsupported server, server must support Accept-Ranges: {self._url}',
+                f"Unsupported server, server must support Accept-Ranges: {self._url}",
                 path=fspath(self._url),
             )
-        return first_index_response['Headers']['Content-Length']
+        return first_index_response["Headers"]["Content-Length"]
 
     @property
     def name(self) -> str:
         return fspath(self._url)
 
     def _fetch_response(
-            self,
-            start: Optional[int] = None,
-            end: Optional[int] = None) -> dict:
-
+        self, start: Optional[int] = None, end: Optional[int] = None
+    ) -> dict:
         def fetch_response() -> dict:
             request_kwargs = {}
-            if hasattr(self._url, 'request_kwargs'):
+            if hasattr(self._url, "request_kwargs"):
                 request_kwargs = self._url.request_kwargs  # pyre-ignore[16]
-            timeout = request_kwargs.pop('timeout', DEFAULT_TIMEOUT)
-            stream = request_kwargs.pop('stream', True)
+            timeout = request_kwargs.pop("timeout", DEFAULT_TIMEOUT)
+            stream = request_kwargs.pop("stream", True)
 
             if start is None or end is None:
-                with requests.get(fspath(self._url), timeout=timeout,
-                                  stream=stream, **request_kwargs) as response:
+                with requests.get(
+                    fspath(self._url), timeout=timeout, stream=stream, **request_kwargs
+                ) as response:
                     return {
-                        'Headers': response.headers,
-                        'Cookies': response.cookies,
-                        'StatusCode': response.status_code,
+                        "Headers": response.headers,
+                        "Cookies": response.cookies,
+                        "StatusCode": response.status_code,
                     }
             else:
                 range_end = end
                 if self._content_size is not None:
                     range_end = min(range_end, self._content_size - 1)
-                headers = request_kwargs.pop('headers', {})
+                headers = request_kwargs.pop("headers", {})
                 headers["Range"] = f"bytes={start}-{range_end}"
-                with requests.get(fspath(self._url), timeout=timeout,
-                                  headers=headers, stream=stream,
-                                  **request_kwargs) as response:
-                    if len(response.content) != int(
-                            response.headers['Content-Length']):
+                with requests.get(
+                    fspath(self._url),
+                    timeout=timeout,
+                    headers=headers,
+                    stream=stream,
+                    **request_kwargs,
+                ) as response:
+                    if len(response.content) != int(response.headers["Content-Length"]):
                         raise HttpBodyIncompleteError(
-                            f"The downloaded content is incomplete, expected size: {response.headers['Content-Length']}, actual size: {len(response.content)}",
+                            "The downloaded content is incomplete, "
+                            "expected size: %s, actual size: %d"
+                            % (
+                                response.headers["Content-Length"],
+                                len(response.content),
+                            )
                         )
                     return {
-                        'Body': BytesIO(response.content),
-                        'Headers': response.headers,
-                        'Cookies': response.cookies,
-                        'StatusCode': response.status_code,
+                        "Body": BytesIO(response.content),
+                        "Headers": response.headers,
+                        "Cookies": response.cookies,
+                        "StatusCode": response.status_code,
                     }
 
         fetch_response = patch_method(
             fetch_response,
             max_retries=self._max_retries,
-            should_retry=http_should_retry)
+            should_retry=http_should_retry,
+        )
 
         return fetch_response()

megfile/lib/joinpath.py

@@ -10,22 +10,22 @@ def uri_join(path: str, *other_paths: str) -> str:
         return path
 
     first_path = path
-    if first_path.endswith('/'):
+    if first_path.endswith("/"):
         first_path = first_path[:-1]
 
     last_path = other_paths[-1]
-    if last_path.startswith('/'):
+    if last_path.startswith("/"):
         last_path = last_path[1:]
 
     middle_paths = []
     for other_path in other_paths[:-1]:
-        if other_path.startswith('/'):
+        if other_path.startswith("/"):
             other_path = other_path[1:]
-        if other_path.endswith('/'):
+        if other_path.endswith("/"):
             other_path = other_path[:-1]
         middle_paths.append(other_path)
 
-    return '/'.join([first_path, *middle_paths, last_path])
+    return "/".join([first_path, *middle_paths, last_path])
 
     # Imp. 2
     # other_paths = (other_path.lstrip('/') for other_path in other_paths)

megfile/lib/lazy_handler.py

@@ -7,9 +7,13 @@ from megfile.utils import get_content_size
 
 
 class LazyHandler(Readable, Seekable, Writable):
-    ''' Create a File-Like Object, maintaining file pointer, to avoid misunderstanding the position when read / write / seek 
-    It can be roughly regarded as the copy function of the file handle, but you need to be careful with the write handle, because no matter which copy will modify the data itself
-    '''
+    """Create a File-Like Object, maintaining file pointer,
+    to avoid misunderstanding the position when read / write / seek.
+
+    It can be roughly regarded as the copy function of the file handle,
+    but you need to be careful with the write handle,
+    because no matter which copy will modify the data itself.
+    """
 
     def __init__(self, path: str, mode: str, open_func: Callable, **options):
         self._open_func = open_func

megfile/lib/s3_buffered_writer.py

@@ -5,13 +5,20 @@ from logging import getLogger as get_logger
 from threading import Lock
 from typing import NamedTuple, Optional
 
-from megfile.config import BACKOFF_FACTOR, BACKOFF_INITIAL, DEFAULT_MAX_BLOCK_SIZE, DEFAULT_MAX_BUFFER_SIZE, DEFAULT_MIN_BLOCK_SIZE, GLOBAL_MAX_WORKERS
+from megfile.config import (
+    BACKOFF_FACTOR,
+    BACKOFF_INITIAL,
+    DEFAULT_MAX_BLOCK_SIZE,
+    DEFAULT_MAX_BUFFER_SIZE,
+    DEFAULT_MIN_BLOCK_SIZE,
+    GLOBAL_MAX_WORKERS,
+)
 from megfile.errors import raise_s3_error
 from megfile.interfaces import Writable
 from megfile.utils import get_human_size, process_local
 
 _logger = get_logger(__name__)
-'''
+"""
 class PartResult(NamedTuple):
 
     etag: str
@@ -19,41 +26,39 @@ class PartResult(NamedTuple):
     content_size: int
 
 in Python 3.6+
-'''
+"""
 
 _PartResult = NamedTuple(
-    'PartResult', [('etag', str), ('part_number', int), ('content_size', int)])
+    "PartResult", [("etag", str), ("part_number", int), ("content_size", int)]
+)
 
 
 class PartResult(_PartResult):
-
     def asdict(self):
-        return {
-            'PartNumber': self.part_number,
-            'ETag': self.etag,
-        }
+        return {"PartNumber": self.part_number, "ETag": self.etag}
 
 
 class S3BufferedWriter(Writable[bytes]):
-
     def __init__(
-            self,
-            bucket: str,
-            key: str,
-            *,
-            s3_client,
-            block_size: int = DEFAULT_MIN_BLOCK_SIZE,
-            max_block_size: int = DEFAULT_MAX_BLOCK_SIZE,
-            max_buffer_size: int = DEFAULT_MAX_BUFFER_SIZE,
-            max_workers: Optional[int] = None,
-            profile_name: Optional[str] = None):
-
+        self,
+        bucket: str,
+        key: str,
+        *,
+        s3_client,
+        block_size: int = DEFAULT_MIN_BLOCK_SIZE,
+        max_block_size: int = DEFAULT_MAX_BLOCK_SIZE,
+        max_buffer_size: int = DEFAULT_MAX_BUFFER_SIZE,
+        max_workers: Optional[int] = None,
+        profile_name: Optional[str] = None,
+    ):
         self._bucket = bucket
         self._key = key
         self._client = s3_client
         self._profile_name = profile_name
 
-        self._block_size = block_size
+        # user maybe put block_size with 'numpy.uint64' type
+        self._block_size = int(block_size)
+
         self._max_block_size = max_block_size
         self._max_buffer_size = max_buffer_size
         self._total_buffer_size = 0
@@ -66,9 +71,10 @@ class S3BufferedWriter(Writable[bytes]):
         self._is_global_executor = False
         if max_workers is None:
             self._executor = process_local(
-                'S3BufferedWriter.executor',
+                "S3BufferedWriter.executor",
                 ThreadPoolExecutor,
-                max_workers=GLOBAL_MAX_WORKERS)
+                max_workers=GLOBAL_MAX_WORKERS,
+            )
             self._is_global_executor = True
         else:
             self._executor = ThreadPoolExecutor(max_workers=max_workers)
@@ -77,17 +83,19 @@ class S3BufferedWriter(Writable[bytes]):
         self.__upload_id = None
         self.__upload_id_lock = Lock()
 
-        _logger.debug('open file: %r, mode: %s' % (self.name, self.mode))
+        _logger.debug("open file: %r, mode: %s" % (self.name, self.mode))
 
     @property
     def name(self) -> str:
-        return 's3%s://%s/%s' % (
+        return "s3%s://%s/%s" % (
             f"+{self._profile_name}" if self._profile_name else "",
-            self._bucket, self._key)
+            self._bucket,
+            self._key,
+        )
 
     @property
     def mode(self) -> str:
-        return 'wb'
+        return "wb"
 
     def tell(self) -> int:
         return self._offset
@@ -100,8 +108,9 @@ class S3BufferedWriter(Writable[bytes]):
     def _content_size(self, value: int):
         if value > self._backoff_size:
             _logger.debug(
-                'writing file: %r, current size: %s' %
-                (self.name, get_human_size(value)))
+                "writing file: %r, current size: %s"
+                % (self.name, get_human_size(value))
+            )
         while value > self._backoff_size:
             self._backoff_size *= BACKOFF_FACTOR
         self.__content_size = value
@@ -116,9 +125,8 @@ class S3BufferedWriter(Writable[bytes]):
             if self.__upload_id is None:
                 with raise_s3_error(self.name):
                     self.__upload_id = self._client.create_multipart_upload(
-                        Bucket=self._bucket,
-                        Key=self._key,
-                    )['UploadId']
+                        Bucket=self._bucket, Key=self._key
+                    )["UploadId"]
             return self.__upload_id
 
     @property
@@ -126,22 +134,19 @@ class S3BufferedWriter(Writable[bytes]):
         return self._total_buffer_size - sum(
             future.result().content_size
             for future in self._futures.values()
-            if future.done())
+            if future.done()
+        )
 
     @property
     def _uploading_futures(self):
-        return [
-            future for future in self._futures.values() if not future.done()
-        ]
+        return [future for future in self._futures.values() if not future.done()]
 
     @property
     def _multipart_upload(self):
         return {
-            'Parts':
-                [
-                    future.result().asdict()
-                    for _, future in sorted(self._futures.items())
-                ],
+            "Parts": [
+                future.result().asdict() for _, future in sorted(self._futures.items())
+            ]
         }
 
     def _upload_buffer(self, part_number, content):
@@ -153,24 +158,29 @@ class S3BufferedWriter(Writable[bytes]):
                     UploadId=self._upload_id,
                     PartNumber=part_number,
                     Body=content,
-                )['ETag'], part_number, len(content))
+                )["ETag"],
+                part_number,
+                len(content),
+            )
 
     def _submit_upload_buffer(self, part_number, content):
         self._futures[part_number] = self._executor.submit(
-            self._upload_buffer, part_number, content)
+            self._upload_buffer, part_number, content
+        )
         self._total_buffer_size += len(content)
         while self._buffer_size > self._max_buffer_size:
             wait(self._uploading_futures, return_when=FIRST_COMPLETED)
 
     def _submit_upload_content(self, content: bytes):
-        # s3 part needs at least 5MB, so we need to divide content into equal-size parts, and give last part more size
+        # s3 part needs at least 5MB,
+        # so we need to divide content into equal-size parts,
+        # and give last part more size.
         # e.g. 257MB can be divided into 2 parts, 128MB and 129MB
         offset = 0
         while len(content) - offset - self._max_block_size > self._block_size:
             self._part_number += 1
             offset_stop = offset + self._max_block_size
-            self._submit_upload_buffer(
-                self._part_number, content[offset:offset_stop])
+            self._submit_upload_buffer(self._part_number, content[offset:offset_stop])
             offset = offset_stop
         self._part_number += 1
         self._submit_upload_buffer(self._part_number, content[offset:])
@@ -184,7 +194,7 @@ class S3BufferedWriter(Writable[bytes]):
 
     def write(self, data: bytes) -> int:
         if self.closed:
-            raise IOError('file already closed: %r' % self.name)
+            raise IOError("file already closed: %r" % self.name)
 
         result = self._buffer.write(data)
         if self._buffer.tell() >= self._block_size:
@@ -198,14 +208,13 @@ class S3BufferedWriter(Writable[bytes]):
             self._executor.shutdown()
 
     def _close(self):
-        _logger.debug('close file: %r' % self.name)
+        _logger.debug("close file: %r" % self.name)
 
         if not self._is_multipart:
             with raise_s3_error(self.name):
                 self._client.put_object(
-                    Bucket=self._bucket,
-                    Key=self._key,
-                    Body=self._buffer.getvalue())
+                    Bucket=self._bucket, Key=self._key, Body=self._buffer.getvalue()
+                )
             self._shutdown()
             return
 

megfile/lib/s3_cached_handler.py

@@ -7,19 +7,20 @@ from megfile.utils import generate_cache_path
 
 
 class S3CachedHandler(S3MemoryHandler):
-
     def __init__(
-            self,
-            bucket: str,
-            key: str,
-            mode: str,
-            *,
-            s3_client,
-            cache_path: Optional[str] = None,
-            remove_cache_when_open: bool = True,
-            profile_name: Optional[str] = None):
-
-        assert mode in ('rb', 'wb', 'ab', 'rb+', 'wb+', 'ab+')
+        self,
+        bucket: str,
+        key: str,
+        mode: str,
+        *,
+        s3_client,
+        cache_path: Optional[str] = None,
+        remove_cache_when_open: bool = True,
+        profile_name: Optional[str] = None,
+    ):
+        if mode not in ("rb", "wb", "ab", "rb+", "wb+", "ab+"):
+            # TODO: replace AssertionError with ValueError in 4.0.0
+            raise AssertionError("unacceptable mode: %r" % mode)
 
         self._bucket = bucket
         self._key = key
@@ -31,7 +32,7 @@ class S3CachedHandler(S3MemoryHandler):
             cache_path = generate_cache_path(self.name)
 
         self._cache_path = cache_path
-        self._fileobj = open(self._cache_path, 'wb+')
+        self._fileobj = open(self._cache_path, "wb+")
         self._download_fileobj()
 
         if remove_cache_when_open:

megfile/lib/s3_limited_seekable_writer.py

@@ -3,7 +3,11 @@ from io import BytesIO
 from logging import getLogger as get_logger
 from typing import Optional
 
-from megfile.config import DEFAULT_MAX_BLOCK_SIZE, DEFAULT_MAX_BUFFER_SIZE, DEFAULT_MIN_BLOCK_SIZE
+from megfile.config import (
+    DEFAULT_MAX_BLOCK_SIZE,
+    DEFAULT_MAX_BUFFER_SIZE,
+    DEFAULT_MIN_BLOCK_SIZE,
+)
 from megfile.errors import raise_s3_error
 from megfile.interfaces import Seekable
 from megfile.lib.s3_buffered_writer import S3BufferedWriter
@@ -12,27 +16,27 @@ _logger = get_logger(__name__)
 
 
 class S3LimitedSeekableWriter(S3BufferedWriter, Seekable):
-    ''' For file format like msgpack and mp4, it's a pain that you need to write
+    """For file format like msgpack and mp4, it's a pain that you need to write
     header before writing the data. So it's kind of hard to make streaming write
     to unseekable file system like s3. In this case, we will try to keep the first
     and last parts of data in memory, so we can come back to head again and write
     the header at the last second.
-    '''
+    """
 
     def __init__(
-            self,
-            bucket: str,
-            key: str,
-            *,
-            s3_client,
-            block_size: int = DEFAULT_MIN_BLOCK_SIZE,
-            head_block_size: Optional[int] = None,
-            tail_block_size: Optional[int] = None,
-            max_block_size: int = DEFAULT_MAX_BLOCK_SIZE,
-            max_buffer_size: int = DEFAULT_MAX_BUFFER_SIZE,
-            max_workers: Optional[int] = None,
-            profile_name: Optional[str] = None):
-
+        self,
+        bucket: str,
+        key: str,
+        *,
+        s3_client,
+        block_size: int = DEFAULT_MIN_BLOCK_SIZE,
+        head_block_size: Optional[int] = None,
+        tail_block_size: Optional[int] = None,
+        max_block_size: int = DEFAULT_MAX_BLOCK_SIZE,
+        max_buffer_size: int = DEFAULT_MAX_BUFFER_SIZE,
+        max_workers: Optional[int] = None,
+        profile_name: Optional[str] = None,
+    ):
         super().__init__(
             bucket,
             key,
@@ -41,7 +45,8 @@ class S3LimitedSeekableWriter(S3BufferedWriter, Seekable):
             max_block_size=max_block_size,
             max_buffer_size=max_buffer_size,
             max_workers=max_workers,
-            profile_name=profile_name)
+            profile_name=profile_name,
+        )
 
         self._head_block_size = head_block_size or block_size
         self._tail_block_size = tail_block_size or block_size
@@ -61,8 +66,9 @@ class S3LimitedSeekableWriter(S3BufferedWriter, Seekable):
 
     def seek(self, offset: int, whence: int = os.SEEK_SET) -> int:
         if self.closed:
-            raise IOError('file already closed: %r' % self.name)
+            raise IOError("file already closed: %r" % self.name)
 
+        offset = int(offset)  # user maybe put offset with 'numpy.uint64' type
         if whence == os.SEEK_SET:
             target_offset = offset
         elif whence == os.SEEK_CUR:
@@ -70,7 +76,7 @@ class S3LimitedSeekableWriter(S3BufferedWriter, Seekable):
         elif whence == os.SEEK_END:
             target_offset = self._content_size + offset
         else:
-            raise OSError('Unsupported whence value: %d' % whence)
+            raise OSError("Unsupported whence value: %d" % whence)
 
         if target_offset < self._head_block_size:
             self._head_buffer.seek(target_offset)
@@ -78,15 +84,16 @@ class S3LimitedSeekableWriter(S3BufferedWriter, Seekable):
             self._buffer.seek(target_offset - self._tail_offset)
         else:
             raise OSError(
-                'Can only seek inside of head, or seek to tail, target offset: %d'
-                % target_offset)
+                "Can only seek inside of head, or seek to tail, target offset: %d"
+                % target_offset
+            )
 
         self._offset = target_offset
         return self._offset
 
     def write(self, data: bytes) -> int:
         if self.closed:
-            raise IOError('file already closed: %r' % self.name)
+            raise IOError("file already closed: %r" % self.name)
 
         if self._head_size != self._head_block_size:  # no tail part yet
             self._write_to_head(data)
@@ -96,8 +103,9 @@ class S3LimitedSeekableWriter(S3BufferedWriter, Seekable):
             self._write_to_tail(data)
         else:
             raise OSError(
-                'Can only write inside of head, or write to tail, current offset: %d'
-                % self._offset)
+                "Can only write inside of head, or write to tail, current offset: %d"
+                % self._offset
+            )
         return len(data)
 
     def _write_to_head(self, data: bytes):
@@ -116,8 +124,9 @@ class S3LimitedSeekableWriter(S3BufferedWriter, Seekable):
     def _write_to_head_after_tail_part_created(self, data: bytes):
         if self._offset + len(data) > self._head_block_size:
             raise Exception(
-                'Head part overflow, %d bytes left but try to write %d bytes' %
-                (self._head_block_size - self._offset, len(data)))
+                "Head part overflow, %d bytes left but try to write %d bytes"
+                % (self._head_block_size - self._offset, len(data))
+            )
         self._head_buffer.write(data)
         self._offset += len(data)
 
@@ -139,14 +148,15 @@ class S3LimitedSeekableWriter(S3BufferedWriter, Seekable):
         self._submit_upload_content(content[:offset])
 
     def _close(self):
-        _logger.debug('close file: %r' % self.name)
+        _logger.debug("close file: %r" % self.name)
 
         if not self._is_multipart:
             with raise_s3_error(self.name):
                 self._client.put_object(
                     Bucket=self._bucket,
                     Key=self._key,
-                    Body=self._head_buffer.getvalue() + self._buffer.getvalue())
+                    Body=self._head_buffer.getvalue() + self._buffer.getvalue(),
+                )
             self._shutdown()
             return