megfile 3.1.1__py3-none-any.whl → 3.1.3__py3-none-any.whl

megfile/interfaces.py

@@ -3,7 +3,16 @@ from abc import ABC, abstractmethod
 from io import IOBase, UnsupportedOperation
 from typing import IO, AnyStr, Iterable, List, Optional
 
-from megfile.pathlike import Access, BasePath, BaseURIPath, FileEntry, PathLike, Self, StatResult, URIPath
+from megfile.pathlike import (
+    Access,
+    BasePath,
+    BaseURIPath,
+    FileEntry,
+    PathLike,
+    Self,
+    StatResult,
+    URIPath,
+)
 
 __all__ = [
     "Access",
@@ -28,33 +37,32 @@ __all__ = [
 def fullname(o):
     klass = o.__class__
     module = klass.__module__
-    if module == 'builtins':
+    if module == "builtins":
         return klass.__qualname__  # avoid outputs like 'builtins.str'
-    return module + '.' + klass.__qualname__
+    return module + "." + klass.__qualname__
 
 
 # 1. Default value of closed is False
 # 2. closed is set to True when close() are called
 # 3. close() will only be called once
 class Closable(ABC):
-
     @property
     def closed(self) -> bool:
-        '''Return True if the file-like object is closed.'''
-        return getattr(self, '__closed__', False)
+        """Return True if the file-like object is closed."""
+        return getattr(self, "__closed__", False)
 
     @abstractmethod
     def _close(self) -> None:
         pass  # pragma: no cover
 
     def close(self) -> None:
-        '''Flush and close the file-like object.
+        """Flush and close the file-like object.
 
         This method has no effect if the file is already closed.
-        '''
-        if not getattr(self, '__closed__', False):
+        """
+        if not getattr(self, "__closed__", False):
             self._close()
-            setattr(self, '__closed__', True)
+            setattr(self, "__closed__", True)
 
     def __enter__(self: Self) -> Self:
         return self
@@ -64,23 +72,25 @@ class Closable(ABC):
 
 
 class FileLike(Closable, IOBase, IO[AnyStr], ABC):  # pytype: disable=signature-mismatch
-
     def fileno(self) -> int:
-        raise UnsupportedOperation('not a local file')
+        raise UnsupportedOperation("not a local file")
 
     def isatty(self) -> bool:
         return False
 
     def __repr__(self) -> str:
-        return '<%s name=%r mode=%r>' % (
-            fullname(self), self.name, self.mode)  # pragma: no cover
+        return "<%s name=%r mode=%r>" % (
+            fullname(self),
+            self.name,
+            self.mode,
+        )  # pragma: no cover
 
     def seekable(self) -> bool:
-        '''Return True if the file-like object can be sought.'''
+        """Return True if the file-like object can be sought."""
         return False
 
     def seek(self, offset: int, whence: int = os.SEEK_SET) -> int:
-        '''Change stream position.
+        """Change stream position.
 
         Seek to byte `offset` relative to position indicated by `whence`:
             0  Start of stream (the default).  `offset` should be >= 0;
@@ -88,37 +98,37 @@ class FileLike(Closable, IOBase, IO[AnyStr], ABC):  # pytype: disable=signature-
             2  End of stream - `offset` usually negative.
 
         Return the new absolute position.
-        '''
-        raise UnsupportedOperation('not seekable')  # pragma: no cover
+        """
+        raise UnsupportedOperation("not seekable")  # pragma: no cover
 
     def readable(self) -> bool:
-        '''Return True if the file-like object can be read.'''
+        """Return True if the file-like object can be read."""
         return False  # pragma: no cover
 
     def writable(self) -> bool:
-        '''Return True if the file-like object can be written.'''
+        """Return True if the file-like object can be written."""
         return False
 
     def flush(self) -> None:
-        '''Flush write buffers, if applicable.
+        """Flush write buffers, if applicable.
 
         This is not implemented for read-only and non-blocking streams.
-        '''
+        """
 
     def __del__(self) -> None:
-        # TODO: Next version should turn on __del__ for auto closing, and disable this in child class like CombineReader
+        # TODO: Next version should turn on __del__ for auto closing,
+        # and disable this in child class like CombineReader
         pass
 
 
 class Seekable(FileLike, ABC):
-
     def seekable(self) -> bool:
-        '''Return True if the file-like object can be sought.'''
+        """Return True if the file-like object can be sought."""
         return True
 
     @abstractmethod
     def seek(self, offset: int, whence: int = os.SEEK_SET) -> int:
-        '''Change stream position.
+        """Change stream position.
 
         Seek to byte offset `cookie` relative to position indicated by `whence`:
             0  Start of stream (the default).  `cookie` should be >= 0;
@@ -126,42 +136,41 @@ class Seekable(FileLike, ABC):
             2  End of stream - `cookie` usually negative.
 
         Return the new absolute position.
-        '''
+        """
 
 
 class Readable(FileLike[AnyStr], ABC):
-
     def readable(self) -> bool:
-        '''Return True if the file-like object can be read.'''
+        """Return True if the file-like object can be read."""
         return True
 
     @abstractmethod
     def read(self, size: Optional[int] = None) -> AnyStr:
-        '''Read at most `size` bytes or string, returned as a bytes or string object.
+        """Read at most `size` bytes or string, returned as a bytes or string object.
 
         If the `size` argument is negative, read until EOF is reached.
         Return an empty bytes or string object at EOF.
-        '''
+        """
 
     @abstractmethod
     def readline(self, size: Optional[int] = None) -> AnyStr:  # pyre-ignore[15]
-        '''Next line from the file, as a bytes or string object.
+        """Next line from the file, as a bytes or string object.
 
-        Retain newline. A non-negative `size` argument limits the maximum number of bytes or string to return (an incomplete line may be returned then).
+        Retain newline. A non-negative `size` argument limits the maximum number of
+        bytes or string to return (an incomplete line may be returned then).
         Return an empty bytes object at EOF.
-        '''
+        """
 
-    def readlines(  # pyre-ignore[15]
-            self, hint: Optional[int] = None) -> List[AnyStr]:
-        '''Return a list of lines from the stream.'''
+    def readlines(self, hint: Optional[int] = None) -> List[AnyStr]:  # pyre-ignore[15]
+        """Return a list of lines from the stream."""
         return self.read(size=hint).splitlines(True)  # pyre-ignore[7]
 
     def readinto(self, buffer: bytearray) -> int:
-        '''Read bytes into buffer.
+        """Read bytes into buffer.
 
         Returns number of bytes read (0 for EOF), or None if the object
         is set not to block and has no data to read.
-        '''
+        """
         if "b" not in self.mode:
             raise OSError("'readinto' only works on binary files")
 
@@ -180,41 +189,40 @@ class Readable(FileLike[AnyStr], ABC):
         return self
 
     def truncate(self, size: Optional[int] = None) -> int:
-        raise OSError('not writable')
+        raise OSError("not writable")
 
     def write(self, data: AnyStr) -> int:
-        raise OSError('not writable')
+        raise OSError("not writable")
 
     def writelines(self, lines: Iterable[AnyStr]) -> None:  # pyre-ignore[14]
-        raise OSError('not writable')
+        raise OSError("not writable")
 
 
 class Writable(FileLike[AnyStr], ABC):
-
     def writable(self) -> bool:
-        '''Return True if the file-like object can be written.'''
+        """Return True if the file-like object can be written."""
         return True
 
     @abstractmethod
     def write(self, data: AnyStr) -> int:
-        '''Write bytes or string to file.
+        """Write bytes or string to file.
 
         Return the number of bytes or string written.
-        '''
+        """
 
     def writelines(self, lines: Iterable[AnyStr]) -> None:  # pyre-ignore[14]
-        '''Write `lines` to the file.
+        """Write `lines` to the file.
 
-        Note that newlines are not added. 
-        `lines` can be any iterable object producing bytes-like or string-like objects. 
+        Note that newlines are not added.
+        `lines` can be any iterable object producing bytes-like or string-like objects.
         This is equivalent to calling write() for each element.
-        '''
+        """
         for line in lines:
             self.write(line)
 
     def truncate(self, size: Optional[int] = None) -> int:
         """
-        Resize the stream to the given size in bytes. 
+        Resize the stream to the given size in bytes.
 
         :param size: resize size, defaults to None
         :type size: int, optional
@@ -223,21 +231,19 @@ class Writable(FileLike[AnyStr], ABC):
         :return: The new file size.
         :rtype: int
         """
-        raise UnsupportedOperation('not support truncate')
+        raise UnsupportedOperation("not support truncate")
 
     def read(self, size: Optional[int] = None) -> AnyStr:
-        raise OSError('not readable')
+        raise OSError("not readable")
 
     def readline(self, size: Optional[int] = None) -> AnyStr:  # pyre-ignore[15]
-        raise OSError('not readable')
+        raise OSError("not readable")
 
-    def readlines(  # pyre-ignore[15]
-            self, hint: Optional[int] = None) -> List[AnyStr]:
-        raise OSError('not readable')
+    def readlines(self, hint: Optional[int] = None) -> List[AnyStr]:  # pyre-ignore[15]
+        raise OSError("not readable")
 
 
 class FileCacher(ABC):
-
     @property
     @abstractmethod
     def cache_path(self) -> str:
@@ -245,21 +251,21 @@ class FileCacher(ABC):
 
     @property
     def closed(self) -> bool:
-        '''Return True if the file-like object is closed.'''
-        return getattr(self, '__closed__', False)
+        """Return True if the file-like object is closed."""
+        return getattr(self, "__closed__", False)
 
     @abstractmethod
     def _close(self) -> None:
         pass  # pragma: no cover
 
     def close(self) -> None:
-        '''Flush and close the file-like object.
+        """Flush and close the file-like object.
 
         This method has no effect if the file is already closed.
-        '''
-        if not getattr(self, '__closed__', False):
+        """
+        if not getattr(self, "__closed__", False):
             self._close()
-            setattr(self, '__closed__', True)
+            setattr(self, "__closed__", True)
 
     def __enter__(self) -> str:
         return self.cache_path
@@ -282,7 +288,6 @@ class NullCacher(FileCacher):
 
 
 class ContextIterator(Closable):
-
     def __init__(self, iterable: Iterable) -> None:
         self._iter = iter(iterable)
 

megfile/lib/base_prefetch_reader.py

@@ -8,15 +8,22 @@ from math import ceil
 from statistics import mean
 from typing import Optional
 
-from megfile.config import BACKOFF_FACTOR, BACKOFF_INITIAL, DEFAULT_BLOCK_CAPACITY, DEFAULT_BLOCK_SIZE, DEFAULT_MAX_RETRY_TIMES, GLOBAL_MAX_WORKERS, NEWLINE
+from megfile.config import (
+    BACKOFF_FACTOR,
+    BACKOFF_INITIAL,
+    DEFAULT_BLOCK_CAPACITY,
+    DEFAULT_BLOCK_SIZE,
+    DEFAULT_MAX_RETRY_TIMES,
+    GLOBAL_MAX_WORKERS,
+    NEWLINE,
+)
 from megfile.interfaces import Readable, Seekable
-from megfile.utils import get_human_size, process_local
+from megfile.utils import ProcessLocal, get_human_size, process_local
 
 _logger = get_logger(__name__)
 
 
 class SeekRecord:
-
     def __init__(self, seek_index: int):
         self.seek_index = seek_index
         self.seek_count = 0
@@ -24,25 +31,25 @@ class SeekRecord:
 
 
 class BasePrefetchReader(Readable[bytes], Seekable, ABC):
-    '''
-    Reader to fast read the remote file content. 
-    This will divide the file content into equal parts of block_size size, 
+    """
+    Reader to fast read the remote file content.
+    This will divide the file content into equal parts of block_size size,
     and will use LRU to cache at most block_capacity blocks in memory.
-    open(), seek() and read() will trigger prefetch read. 
-    The prefetch will cached block_forward blocks of data from offset position 
+    open(), seek() and read() will trigger prefetch read.
+    The prefetch will cached block_forward blocks of data from offset position
     (the position after reading if the called function is read).
-    '''
+    """
 
     def __init__(
-            self,
-            *,
-            block_size: int = DEFAULT_BLOCK_SIZE,
-            block_capacity: int = DEFAULT_BLOCK_CAPACITY,
-            block_forward: Optional[int] = None,
-            max_retries: int = DEFAULT_MAX_RETRY_TIMES,
-            max_workers: Optional[int] = None,
-            **kwargs):
-
+        self,
+        *,
+        block_size: int = DEFAULT_BLOCK_SIZE,
+        block_capacity: int = DEFAULT_BLOCK_CAPACITY,
+        block_forward: Optional[int] = None,
+        max_retries: int = DEFAULT_MAX_RETRY_TIMES,
+        max_workers: Optional[int] = None,
+        **kwargs,
+    ):
         self._is_auto_scaling = block_forward is None
         if block_forward is None:
             block_forward = max(block_capacity - 1, 1)
@@ -50,9 +57,10 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
         if block_capacity <= block_forward:
             # TODO: replace AssertionError with ValueError in 4.0.0
             raise AssertionError(
-                'block_capacity should greater than block_forward, '
-                'got: block_capacity=%s, block_forward=%s' %
-                (block_capacity, block_forward))
+                "block_capacity should greater than block_forward, "
+                "got: block_capacity=%s, block_forward=%s"
+                % (block_capacity, block_forward)
+            )
 
         # user maybe put block_size with 'numpy.uint64' type
         block_size = int(block_size)
@@ -60,9 +68,12 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
         self._max_retries = max_retries
         self._block_size = block_size
         self._block_capacity = block_capacity  # Max number of blocks
-        self._block_forward = block_forward  # Number of blocks every prefetch, which should be smaller than block_capacity
 
-        self._futures = self._get_futures()
+        # Number of blocks every prefetch, which should be smaller than block_capacity
+        self._block_forward = block_forward
+
+        self._process_local = ProcessLocal()
+
         self._content_size = self._get_content_size()
         self._block_stop = ceil(self._content_size / block_size)
 
@@ -75,20 +86,25 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
         self._is_global_executor = False
         if max_workers is None:
             self._executor = process_local(
-                f'{self.__class__.__name__}.executor',
+                f"{self.__class__.__name__}.executor",
                 ThreadPoolExecutor,
-                max_workers=GLOBAL_MAX_WORKERS)
+                max_workers=GLOBAL_MAX_WORKERS,
+            )
             self._is_global_executor = True
         else:
             self._executor = ThreadPoolExecutor(max_workers=max_workers)
         self._seek_buffer(0)
 
-        _logger.debug('open file: %r, mode: %s' % (self.name, self.mode))
+        _logger.debug("open file: %r, mode: %s" % (self.name, self.mode))
 
     @abstractmethod
     def _get_content_size(self):
         pass
 
+    @property
+    def _futures(self):
+        return self._process_local("futures", self._get_futures)
+
     def _get_futures(self):
         return LRUCacheFutureManager()
 
@@ -99,7 +115,7 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
 
     @property
     def mode(self) -> str:
-        return 'rb'
+        return "rb"
 
     def tell(self) -> int:
         return self._offset
@@ -112,15 +128,15 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
     def _offset(self, value: int):
         if value > self._backoff_size:
             _logger.debug(
-                'reading file: %r, current offset / total size: %s / %s' % (
-                    self.name, get_human_size(value),
-                    get_human_size(self._content_size)))
+                "reading file: %r, current offset / total size: %s / %s"
+                % (self.name, get_human_size(value), get_human_size(self._content_size))
+            )
         while value > self._backoff_size:
             self._backoff_size *= BACKOFF_FACTOR
         self.__offset = value
 
     def seek(self, offset: int, whence: int = os.SEEK_SET) -> int:
-        '''Change stream position.
+        """Change stream position.
 
         Seek to byte offset pos relative to position indicated by whence:
 
@@ -129,10 +145,10 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
             2  End of stream - pos usually negative.
 
         Returns the new absolute position.
-        '''
+        """
         offset = int(offset)  # user maybe put offset with 'numpy.uint64' type
         if self.closed:
-            raise IOError('file already closed: %r' % self.name)
+            raise IOError("file already closed: %r" % self.name)
         if whence == os.SEEK_CUR:
             target_offset = self._offset + offset
         elif whence == os.SEEK_END:
@@ -140,7 +156,7 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
         elif whence == os.SEEK_SET:
             target_offset = offset
         else:
-            raise ValueError('invalid whence: %r' % whence)
+            raise ValueError("invalid whence: %r" % whence)
 
         if target_offset == self._offset:
             return target_offset
@@ -152,19 +168,19 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
         return self._offset
 
     def read(self, size: Optional[int] = None) -> bytes:
-        '''Read at most size bytes, returned as a bytes object.
+        """Read at most size bytes, returned as a bytes object.
 
         If the size argument is negative, read until EOF is reached.
         Return an empty bytes object at EOF.
-        '''
+        """
         if self.closed:
-            raise IOError('file already closed: %r' % self.name)
+            raise IOError("file already closed: %r" % self.name)
 
         if len(self._seek_history) > 0:
             self._seek_history[-1].read_count += 1
 
         if self._offset >= self._content_size:
-            return b''
+            return b""
 
         if size is None or size < 0:
             size = self._content_size - self._offset
@@ -174,12 +190,12 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
         if self._block_forward == 1:
             block_index = self._offset // self._block_size
             if len(self._seek_history) > 0:
-                mean_read_count = mean(
-                    item.read_count for item in self._seek_history)
+                mean_read_count = mean(item.read_count for item in self._seek_history)
             else:
                 mean_read_count = 0
             if block_index not in self._futures and mean_read_count < 3:
-                # No using LRP will be better if read() are always called less than 3 times after seek()
+                # No using LRP will be better if read() are always called less than 3
+                # times after seek()
                 return self._read(size)
 
         data = self._buffer.read(size)
@@ -198,20 +214,20 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
         return buffer.getvalue()
 
     def readline(self, size: Optional[int] = None) -> bytes:
-        '''Next line from the file, as a bytes object.
+        """Next line from the file, as a bytes object.
 
         Retain newline.  A non-negative size argument limits the maximum
         number of bytes to return (an incomplete line may be returned then).
         If the size argument is negative, read until EOF is reached.
         Return an empty bytes object at EOF.
-        '''
+        """
         if self.closed:
-            raise IOError('file already closed: %r' % self.name)
+            raise IOError("file already closed: %r" % self.name)
 
         if len(self._seek_history) > 0:
             self._seek_history[-1].read_count += 1
         if self._offset >= self._content_size:
-            return b''
+            return b""
 
         if size is None or size < 0:
             size = self._content_size - self._offset
@@ -237,21 +253,22 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
 
     def _read(self, size: int) -> bytes:
         if size == 0 or self._offset >= self._content_size:
-            return b''
+            return b""
 
-        data = self._fetch_response(
-            start=self._offset, end=self._offset + size - 1)['Body'].read()
+        data = self._fetch_response(start=self._offset, end=self._offset + size - 1)[
+            "Body"
+        ].read()
         self.seek(size, os.SEEK_CUR)
         return data
 
     def readinto(self, buffer: bytearray) -> int:
-        '''Read bytes into buffer.
+        """Read bytes into buffer.
 
         Returns number of bytes read (0 for EOF), or None if the object
         is set not to block and has no data to read.
-        '''
+        """
         if self.closed:
-            raise IOError('file already closed: %r' % self.name)
+            raise IOError("file already closed: %r" % self.name)
 
         if self._offset >= self._content_size:
             return 0
@@ -260,7 +277,7 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
         size = min(size, self._content_size - self._offset)
 
         data = self._buffer.read(size)
-        buffer[:len(data)] = data
+        buffer[: len(data)] = data
         if len(data) == size:
             self._offset += len(data)
             return size
@@ -269,7 +286,7 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
         while offset < size:
             remain_size = size - offset
             data = self._next_buffer.read(remain_size)
-            buffer[offset:offset + len(data)] = data
+            buffer[offset : offset + len(data)] = data
             offset += len(data)
 
         self._offset += offset
@@ -306,8 +323,11 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
 
     @property
     def _next_buffer(self) -> BytesIO:
-        # Get next buffer by this function when finished reading current buffer (self._buffer)
-        # Make sure that _buffer is used before using _next_buffer(), or will make _cached_offset invalid
+        # Get next buffer by this function when finished reading current buffer
+        # (self._buffer)
+        #
+        # Make sure that _buffer is used before using _next_buffer(), or will make
+        # _cached_offset invalid
         self._block_index += 1
         self._cached_offset = 0
         return self._buffer
@@ -319,7 +339,8 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
             history = []
             for item in self._seek_history:
                 if item.seek_count > self._block_capacity * 2:
-                    # seek interval is bigger than self._block_capacity * 2, drop it from self._seek_history
+                    # seek interval is bigger than self._block_capacity * 2, drop it
+                    # from self._seek_history
                     continue
                 if index - 1 < item.seek_index < index + 2:
                     continue
@@ -328,23 +349,22 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
             history.append(SeekRecord(index))
             self._seek_history = history
             self._block_forward = max(
-                (self._block_capacity - 1) // len(self._seek_history), 1)
+                (self._block_capacity - 1) // len(self._seek_history), 1
+            )
 
         self._cached_offset = offset
         self._block_index = index
 
     @abstractmethod
     def _fetch_response(
-            self,
-            start: Optional[int] = None,
-            end: Optional[int] = None) -> dict:
+        self, start: Optional[int] = None, end: Optional[int] = None
+    ) -> dict:
         pass
 
     def _fetch_buffer(self, index: int) -> BytesIO:
-        start, end = index * self._block_size, (
-            index + 1) * self._block_size - 1
+        start, end = index * self._block_size, (index + 1) * self._block_size - 1
         response = self._fetch_response(start=start, end=end)
-        return response['Body']
+        return response["Body"]
 
     def _submit_future(self, index: int):
         if index < 0 or index >= self._block_stop:
@@ -361,7 +381,7 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
         self._futures.cleanup(self._block_capacity)
 
     def _close(self):
-        _logger.debug('close file: %r' % self.name)
+        _logger.debug("close file: %r" % self.name)
 
         if not self._is_global_executor:
             self._executor.shutdown()
@@ -369,7 +389,6 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
 
 
 class LRUCacheFutureManager(OrderedDict):
-
     def __init__(self):
         super().__init__()