megfile 3.1.1__py3-none-any.whl → 3.1.2__py3-none-any.whl

megfile/lib/s3_memory_handler.py

@@ -2,24 +2,28 @@ import os
 from io import BytesIO, UnsupportedOperation
 from typing import Iterable, List, Optional
 
-from megfile.errors import S3ConfigError, UnknownError, raise_s3_error, translate_s3_error
+from megfile.errors import (
+    S3ConfigError,
+    UnknownError,
+    raise_s3_error,
+    translate_s3_error,
+)
 from megfile.interfaces import Readable, Seekable, Writable
 
 
 class S3MemoryHandler(Readable[bytes], Seekable, Writable[bytes]):
-
     def __init__(
-            self,
-            bucket: str,
-            key: str,
-            mode: str,
-            *,
-            s3_client,
-            profile_name: Optional[str] = None):
-
-        if mode not in ('rb', 'wb', 'ab', 'rb+', 'wb+', 'ab+'):
+        self,
+        bucket: str,
+        key: str,
+        mode: str,
+        *,
+        s3_client,
+        profile_name: Optional[str] = None,
+    ):
+        if mode not in ("rb", "wb", "ab", "rb+", "wb+", "ab+"):
             # TODO: replace AssertionError with ValueError in 4.0.0
-            raise AssertionError('unacceptable mode: %r' % mode)
+            raise AssertionError("unacceptable mode: %r" % mode)
 
         self._bucket = bucket
         self._key = key
@@ -32,9 +36,11 @@ class S3MemoryHandler(Readable[bytes], Seekable, Writable[bytes]):
 
     @property
     def name(self) -> str:
-        return 's3%s://%s/%s' % (
+        return "s3%s://%s/%s" % (
             f"+{self._profile_name}" if self._profile_name else "",
-            self._bucket, self._key)
+            self._bucket,
+            self._key,
+        )
 
     @property
     def mode(self) -> str:
@@ -47,46 +53,44 @@ class S3MemoryHandler(Readable[bytes], Seekable, Writable[bytes]):
         return self._fileobj.seek(offset, whence)
 
     def readable(self) -> bool:
-        return self._mode[0] == 'r' or self._mode[-1] == '+'
+        return self._mode[0] == "r" or self._mode[-1] == "+"
 
     def read(self, size: Optional[int] = None) -> bytes:
         if not self.readable():
-            raise UnsupportedOperation('not readable')
+            raise UnsupportedOperation("not readable")
         return self._fileobj.read(size)
 
     def readline(self, size: Optional[int] = None) -> bytes:
         if not self.readable():
-            raise UnsupportedOperation('not readable')
+            raise UnsupportedOperation("not readable")
         if size is None:
             size = -1
         return self._fileobj.readline(size)
 
     def readlines(self, hint: Optional[int] = None) -> List[bytes]:
         if not self.readable():
-            raise UnsupportedOperation('not readable')
+            raise UnsupportedOperation("not readable")
         if hint is None:
             hint = -1
         return self._fileobj.readlines(hint)
 
     def writable(self) -> bool:
-        return self._mode[0] == 'w' or \
-            self._mode[0] == 'a' or \
-            self._mode[-1] == '+'
+        return self._mode[0] == "w" or self._mode[0] == "a" or self._mode[-1] == "+"
 
     def flush(self):
         self._fileobj.flush()
 
     def write(self, data: bytes) -> int:
         if not self.writable():
-            raise UnsupportedOperation('not writable')
-        if self._mode[0] == 'a':
+            raise UnsupportedOperation("not writable")
+        if self._mode[0] == "a":
             self.seek(0, os.SEEK_END)
         return self._fileobj.write(data)
 
     def writelines(self, lines: Iterable[bytes]):
         if not self.writable():
-            raise UnsupportedOperation('not writable')
-        if self._mode[0] == 'a':
+            raise UnsupportedOperation("not writable")
+        if self._mode[0] == "a":
             self.seek(0, os.SEEK_END)
         self._fileobj.writelines(lines)
 
@@ -104,17 +108,17 @@ class S3MemoryHandler(Readable[bytes], Seekable, Writable[bytes]):
         return True
 
     def _download_fileobj(self):
-        need_download = self._mode[0] == 'r' or (
-            self._mode[0] == 'a' and self._file_exists())
+        need_download = self._mode[0] == "r" or (
+            self._mode[0] == "a" and self._file_exists()
+        )
         if not need_download:
             return
         # directly download to the file handle
         try:
-            self._client.download_fileobj(
-                self._bucket, self._key, self._fileobj)
+            self._client.download_fileobj(self._bucket, self._key, self._fileobj)
         except Exception as error:
             raise self._translate_error(error)
-        if self._mode[0] == 'r':
+        if self._mode[0] == "r":
             self.seek(0, os.SEEK_SET)
 
     def _upload_fileobj(self):

megfile/lib/s3_pipe_handler.py

@@ -1,5 +1,5 @@
 import atexit
-import concurrent.futures  # don't delete this import, to ensure the _close_s3_pipes registration is earlier than concurrent.futures._python_exit
+import concurrent.futures  # noqa: F401 # don't delete this import, to ensure the _close_s3_pipes registration is earlier than concurrent.futures._python_exit
 import os
 from threading import Thread
 from typing import Optional
@@ -12,7 +12,6 @@ _s3_opened_pipes = []
 
 @atexit.register
 def _close_s3_pipes():  # pragma: no cover
-
     def try_close_pipe(fd):
         try:
             os.close(fd)
@@ -25,20 +24,19 @@ def _close_s3_pipes():  # pragma: no cover
 
 
 class S3PipeHandler(Readable[bytes], Writable[bytes]):
-
     def __init__(
-            self,
-            bucket: str,
-            key: str,
-            mode: str,
-            *,
-            s3_client,
-            join_thread: bool = True,
-            profile_name: Optional[str] = None):
-
-        if mode not in ('rb', 'wb'):
+        self,
+        bucket: str,
+        key: str,
+        mode: str,
+        *,
+        s3_client,
+        join_thread: bool = True,
+        profile_name: Optional[str] = None,
+    ):
+        if mode not in ("rb", "wb"):
             # TODO: replace AssertionError with ValueError in 4.0.0
-            raise AssertionError('unacceptable mode: %r' % mode)
+            raise AssertionError("unacceptable mode: %r" % mode)
 
         self._bucket = bucket
         self._key = key
@@ -52,20 +50,21 @@ class S3PipeHandler(Readable[bytes], Writable[bytes]):
         self._pipe = os.pipe()
         _s3_opened_pipes.append(self._pipe)
 
-        if self._mode == 'rb':
-            self._fileobj = os.fdopen(self._pipe[0], 'rb')
-            self._async_task = Thread(
-                target=self._download_fileobj, daemon=True)
+        if self._mode == "rb":
+            self._fileobj = os.fdopen(self._pipe[0], "rb")
+            self._async_task = Thread(target=self._download_fileobj, daemon=True)
         else:
-            self._fileobj = os.fdopen(self._pipe[1], 'wb')
+            self._fileobj = os.fdopen(self._pipe[1], "wb")
             self._async_task = Thread(target=self._upload_fileobj, daemon=True)
         self._async_task.start()
 
     @property
     def name(self) -> str:
-        return 's3%s://%s/%s' % (
+        return "s3%s://%s/%s" % (
             f"+{self._profile_name}" if self._profile_name else "",
-            self._bucket, self._key)
+            self._bucket,
+            self._key,
+        )
 
     @property
     def mode(self) -> str:
@@ -76,7 +75,7 @@ class S3PipeHandler(Readable[bytes], Writable[bytes]):
 
     def _download_fileobj(self):
         try:
-            with os.fdopen(self._pipe[1], 'wb') as buffer:
+            with os.fdopen(self._pipe[1], "wb") as buffer:
                 self._client.download_fileobj(self._bucket, self._key, buffer)
         except BrokenPipeError:
             if self._fileobj.closed:
@@ -87,7 +86,7 @@ class S3PipeHandler(Readable[bytes], Writable[bytes]):
 
     def _upload_fileobj(self):
         try:
-            with os.fdopen(self._pipe[0], 'rb') as buffer:
+            with os.fdopen(self._pipe[0], "rb") as buffer:
                 self._client.upload_fileobj(buffer, self._bucket, self._key)
         except Exception as error:
             self._exc = error
@@ -97,7 +96,7 @@ class S3PipeHandler(Readable[bytes], Writable[bytes]):
             raise translate_s3_error(self._exc, self.name)
 
     def readable(self) -> bool:
-        return self._mode == 'rb'
+        return self._mode == "rb"
 
     def read(self, size: Optional[int] = None) -> bytes:
         self._raise_exception()
@@ -112,7 +111,7 @@ class S3PipeHandler(Readable[bytes], Writable[bytes]):
         return data
 
     def writable(self) -> bool:
-        return self._mode == 'wb'
+        return self._mode == "wb"
 
     def flush(self):
         self._fileobj.flush()

megfile/lib/s3_prefetch_reader.py

@@ -2,41 +2,61 @@ from concurrent.futures import Future
 from io import BytesIO
 from typing import Optional
 
-from megfile.config import BACKOFF_FACTOR, BACKOFF_INITIAL, DEFAULT_BLOCK_CAPACITY, DEFAULT_BLOCK_SIZE, GLOBAL_MAX_WORKERS, NEWLINE, S3_MAX_RETRY_TIMES
-from megfile.errors import S3FileChangedError, S3InvalidRangeError, patch_method, raise_s3_error, s3_should_retry
+from megfile.config import (
+    BACKOFF_FACTOR,
+    BACKOFF_INITIAL,
+    DEFAULT_BLOCK_CAPACITY,
+    DEFAULT_BLOCK_SIZE,
+    GLOBAL_MAX_WORKERS,
+    NEWLINE,
+    S3_MAX_RETRY_TIMES,
+)
+from megfile.errors import (
+    S3FileChangedError,
+    S3InvalidRangeError,
+    patch_method,
+    raise_s3_error,
+    s3_should_retry,
+)
 from megfile.lib.base_prefetch_reader import BasePrefetchReader, LRUCacheFutureManager
 
 __all__ = [
-    'DEFAULT_BLOCK_CAPACITY',
-    'DEFAULT_BLOCK_SIZE',
-    'GLOBAL_MAX_WORKERS',
-    'BACKOFF_INITIAL',
-    'BACKOFF_FACTOR',
-    'NEWLINE',
-    'S3PrefetchReader',
-    'LRUCacheFutureManager',
+    "DEFAULT_BLOCK_CAPACITY",
+    "DEFAULT_BLOCK_SIZE",
+    "GLOBAL_MAX_WORKERS",
+    "BACKOFF_INITIAL",
+    "BACKOFF_FACTOR",
+    "NEWLINE",
+    "S3PrefetchReader",
+    "LRUCacheFutureManager",
 ]
 
 
 class S3PrefetchReader(BasePrefetchReader):
-    '''
-    Reader to fast read the s3 content. This will divide the file content into equal parts of block_size size, and will use LRU to cache at most block_capacity blocks in memory.
-    open(), seek() and read() will trigger prefetch read. The prefetch will cached block_forward blocks of data from offset position (the position after reading if the called function is read).
-    '''
+    """
+    Reader to fast read the s3 content.
 
-    def __init__(
-            self,
-            bucket: str,
-            key: str,
-            *,
-            s3_client,
-            block_size: int = DEFAULT_BLOCK_SIZE,
-            block_capacity: int = DEFAULT_BLOCK_CAPACITY,
-            block_forward: Optional[int] = None,
-            max_retries: int = S3_MAX_RETRY_TIMES,
-            max_workers: Optional[int] = None,
-            profile_name: Optional[str] = None):
+    This will divide the file content into equalparts of block_size size,
+    and will use LRU to cache at most block_capacity blocks in memory.
+
+    open(), seek() and read() will trigger prefetch read.
+    The prefetch will cached block_forward blocks of data from offset position
+    (the position after reading if the called function is read).
+    """
 
+    def __init__(
+        self,
+        bucket: str,
+        key: str,
+        *,
+        s3_client,
+        block_size: int = DEFAULT_BLOCK_SIZE,
+        block_capacity: int = DEFAULT_BLOCK_CAPACITY,
+        block_forward: Optional[int] = None,
+        max_retries: int = S3_MAX_RETRY_TIMES,
+        max_workers: Optional[int] = None,
+        profile_name: Optional[str] = None,
+    ):
         self._bucket = bucket
         self._key = key
         self._client = s3_client
@@ -49,65 +69,64 @@ class S3PrefetchReader(BasePrefetchReader):
             block_capacity=block_capacity,
             block_forward=block_forward,
             max_retries=max_retries,
-            max_workers=max_workers)
+            max_workers=max_workers,
+        )
 
     def _get_content_size(self):
         try:
             start, end = 0, self._block_size - 1
             first_index_response = self._fetch_response(start=start, end=end)
-            content_size = int(
-                first_index_response['ContentRange'].split('/')[-1])
+            content_size = int(first_index_response["ContentRange"].split("/")[-1])
         except S3InvalidRangeError:
             # usually when read a empty file
             # can use minio test empty file: https://hub.docker.com/r/minio/minio
             first_index_response = self._fetch_response()
-            content_size = int(first_index_response['ContentLength'])
+            content_size = int(first_index_response["ContentLength"])
 
         first_future = Future()
-        first_future.set_result(first_index_response['Body'])
+        first_future.set_result(first_index_response["Body"])
         self._insert_futures(index=0, future=first_future)
-        self._content_etag = first_index_response['ETag']
+        self._content_etag = first_index_response["ETag"]
         self._content_info = first_index_response
         return content_size
 
     @property
     def name(self) -> str:
-        return 's3%s://%s/%s' % (
+        return "s3%s://%s/%s" % (
             f"+{self._profile_name}" if self._profile_name else "",
-            self._bucket, self._key)
+            self._bucket,
+            self._key,
+        )
 
     def _fetch_response(
-            self,
-            start: Optional[int] = None,
-            end: Optional[int] = None) -> dict:
-
+        self, start: Optional[int] = None, end: Optional[int] = None
+    ) -> dict:
         def fetch_response() -> dict:
             if start is None or end is None:
-                return self._client.get_object(
-                    Bucket=self._bucket, Key=self._key)
+                return self._client.get_object(Bucket=self._bucket, Key=self._key)
 
-            range_str = f'bytes={start}-{end}'
+            range_str = f"bytes={start}-{end}"
             response = self._client.get_object(
-                Bucket=self._bucket, Key=self._key, Range=range_str)
-            response['Body'] = BytesIO(response['Body'].read())
+                Bucket=self._bucket, Key=self._key, Range=range_str
+            )
+            response["Body"] = BytesIO(response["Body"].read())
             return response
 
         fetch_response = patch_method(
-            fetch_response,
-            max_retries=self._max_retries,
-            should_retry=s3_should_retry)
+            fetch_response, max_retries=self._max_retries, should_retry=s3_should_retry
+        )
 
         with raise_s3_error(self.name):
             return fetch_response()
 
     def _fetch_buffer(self, index: int) -> BytesIO:
-        start, end = index * self._block_size, (
-            index + 1) * self._block_size - 1
+        start, end = index * self._block_size, (index + 1) * self._block_size - 1
         response = self._fetch_response(start=start, end=end)
-        etag = response.get('ETag', None)
+        etag = response.get("ETag", None)
         if etag is not None and etag != self._content_etag:
             raise S3FileChangedError(
-                'File changed: %r, etag before: %s, after: %s' %
-                (self.name, self._content_info, response))
+                "File changed: %r, etag before: %s, after: %s"
+                % (self.name, self._content_info, response)
+            )
 
-        return response['Body']
+        return response["Body"]

megfile/lib/s3_share_cache_reader.py

@@ -3,7 +3,11 @@ from concurrent.futures import Future
 from logging import getLogger as get_logger
 from typing import Optional
 
-from megfile.config import DEFAULT_BLOCK_CAPACITY, DEFAULT_BLOCK_SIZE, S3_MAX_RETRY_TIMES
+from megfile.config import (
+    DEFAULT_BLOCK_CAPACITY,
+    DEFAULT_BLOCK_SIZE,
+    S3_MAX_RETRY_TIMES,
+)
 from megfile.lib.s3_prefetch_reader import LRUCacheFutureManager, S3PrefetchReader
 from megfile.utils import thread_local
 
@@ -11,25 +15,31 @@ _logger = get_logger(__name__)
 
 
 class S3ShareCacheReader(S3PrefetchReader):
-    '''
-    Reader to fast read the s3 content. This will divide the file content into equal parts of block_size size, and will use LRU to cache at most block_capacity blocks in memory.
-    open(), seek() and read() will trigger prefetch read. The prefetch will cached block_forward blocks of data from offset position (the position after reading if the called function is read).
-    '''
+    """
+    Reader to fast read the s3 content.
 
-    def __init__(
-            self,
-            bucket: str,
-            key: str,
-            *,
-            s3_client,
-            block_size: int = DEFAULT_BLOCK_SIZE,
-            block_capacity: int = DEFAULT_BLOCK_CAPACITY,
-            block_forward: Optional[int] = None,
-            max_retries: int = S3_MAX_RETRY_TIMES,
-            cache_key: str = 'lru',
-            max_workers: Optional[int] = None,
-            profile_name: Optional[str] = None):
+    This will divide the file content into equal parts of block_size size,
+    and will use LRU to cache at most block_capacity blocks in memory.
+
+    open(), seek() and read() will trigger prefetch read.
+    The prefetch will cached block_forward blocks of data from offset position
+    (the position after reading if the called function is read).
+    """
 
+    def __init__(
+        self,
+        bucket: str,
+        key: str,
+        *,
+        s3_client,
+        block_size: int = DEFAULT_BLOCK_SIZE,
+        block_capacity: int = DEFAULT_BLOCK_CAPACITY,
+        block_forward: Optional[int] = None,
+        max_retries: int = S3_MAX_RETRY_TIMES,
+        cache_key: str = "lru",
+        max_workers: Optional[int] = None,
+        profile_name: Optional[str] = None,
+    ):
         self._cache_key = cache_key
 
         super().__init__(
@@ -44,14 +54,17 @@ class S3ShareCacheReader(S3PrefetchReader):
             profile_name=profile_name,
         )
 
-    def _get_futures(self) -> 'ShareCacheFutureManager':
+    def _get_futures(self) -> "ShareCacheFutureManager":
         futures = thread_local(
-            'S3ShareCacheReader.' + self._cache_key, ShareCacheFutureManager)
+            "S3ShareCacheReader." + self._cache_key, ShareCacheFutureManager
+        )
         futures.register(self.name)
         return futures
 
     def _seek_buffer(self, index: int, offset: int = 0):
-        # The corresponding block is probably not downloaded when sought to a new position
+        # The corresponding block is probably not downloaded
+        # when sought to a new position
+        #
         # So record the offset first, set it when it is accessed
         self._cached_offset = offset
         self._block_index = index
@@ -60,7 +73,8 @@ class S3ShareCacheReader(S3PrefetchReader):
         if index < 0 or index >= self._block_stop:
             return
         self._futures.submit(
-            self._executor, (self.name, index), self._fetch_buffer, index)
+            self._executor, (self.name, index), self._fetch_buffer, index
+        )
 
     def _insert_futures(self, index: int, future: Future):
         self._futures[(self.name, index)] = future
@@ -72,7 +86,7 @@ class S3ShareCacheReader(S3PrefetchReader):
         self._futures.cleanup(DEFAULT_BLOCK_CAPACITY)
 
     def _close(self):
-        _logger.debug('close file: %r' % self.name)
+        _logger.debug("close file: %r" % self.name)
 
         if not self._is_global_executor:
             self._executor.shutdown()
@@ -80,7 +94,6 @@ class S3ShareCacheReader(S3PrefetchReader):
 
 
 class ShareCacheFutureManager(LRUCacheFutureManager):
-
     def __init__(self):
         super().__init__()
         self._references = Counter()

megfile/lib/shadow_handler.py

@@ -12,9 +12,13 @@ class BaseShadowHandler(RawIOBase):
 
 
 class ShadowHandler(Readable, Seekable, Writable, BaseShadowHandler):
-    ''' Create a File-Like Object, maintaining file pointer, to avoid misunderstanding the position when read / write / seek 
-    It can be roughly regarded as the copy function of the file handle, but you need to be careful with the write handle, because no matter which copy will modify the data itself
-    '''
+    """Create a File-Like Object, maintaining file pointer,
+    to avoid misunderstanding the position when read / write / seek.
+
+    It can be roughly regarded as the copy function of the file handle,
+    but you need to be careful with the write handle,
+    because no matter which copy will modify the data itself.
+    """
 
     def __init__(self, file_object: IO, intrusive: bool = True):
         self._file_object = file_object

megfile/lib/stdio_handler.py

@@ -6,27 +6,27 @@ from megfile.interfaces import Readable, Writable
 
 
 class STDHandler:
-
     def tell(self):
-        raise UnsupportedOperation('not tellable')
+        raise UnsupportedOperation("not tellable")
 
     def _close(self):
         pass
 
 
 class STDReader(STDHandler, Readable):
-    """ megfile encapsulation of stdin. Avoid direct operation on sys.stdin
+    """megfile encapsulation of stdin. Avoid direct operation on sys.stdin
 
     .. note ::
 
         1. For convenience, use buffer by default
 
-        2. There is currently no demand and no design for seek, so seek is not allowed now
+        2. There is currently no demand and no design for seek,
+           so seek is not allowed now
     """
 
     def __init__(self, mode: str):
         handler = sys.stdin
-        if 'b' in mode:
+        if "b" in mode:
             handler = handler.buffer
 
         self._handler = handler
@@ -48,13 +48,14 @@ class STDReader(STDHandler, Readable):
 
 
 class STDWriter(STDHandler, Writable):
-    """ megfile encapsulation of stdin. Avoid direct operation on sys.stdin
+    """megfile encapsulation of stdin. Avoid direct operation on sys.stdin
 
     .. note ::
 
         1. For convenience, use buffer by default
 
-        2. There is currently no demand and no design for seek, so seek is not allowed now
+        2. There is currently no demand and no design for seek,
+           so seek is not allowed now
     """
 
     def __init__(self, path: str, mode: str):
@@ -64,7 +65,7 @@ class STDWriter(STDHandler, Writable):
         else:
             name = "stdout"
             handler = sys.stdout
-        if 'b' in mode:
+        if "b" in mode:
             handler = handler.buffer
 
         self._handler = handler

megfile/lib/url.py

@@ -1,4 +1,4 @@
 def get_url_scheme(url: str):
-    if '://' in url:
-        return url.split('://', 1)[0]
-    return ''
+    if "://" in url:
+        return url.split("://", 1)[0]
+    return ""