megfile 3.1.0.post2__py3-none-any.whl → 3.1.2__py3-none-any.whl

megfile/interfaces.py

@@ -3,7 +3,16 @@ from abc import ABC, abstractmethod
 from io import IOBase, UnsupportedOperation
 from typing import IO, AnyStr, Iterable, List, Optional
 
-from megfile.pathlike import Access, BasePath, BaseURIPath, FileEntry, PathLike, Self, StatResult, URIPath
+from megfile.pathlike import (
+    Access,
+    BasePath,
+    BaseURIPath,
+    FileEntry,
+    PathLike,
+    Self,
+    StatResult,
+    URIPath,
+)
 
 __all__ = [
     "Access",
@@ -28,33 +37,32 @@ __all__ = [
 def fullname(o):
     klass = o.__class__
     module = klass.__module__
-    if module == 'builtins':
+    if module == "builtins":
         return klass.__qualname__  # avoid outputs like 'builtins.str'
-    return module + '.' + klass.__qualname__
+    return module + "." + klass.__qualname__
 
 
 # 1. Default value of closed is False
 # 2. closed is set to True when close() are called
 # 3. close() will only be called once
 class Closable(ABC):
-
     @property
     def closed(self) -> bool:
-        '''Return True if the file-like object is closed.'''
-        return getattr(self, '__closed__', False)
+        """Return True if the file-like object is closed."""
+        return getattr(self, "__closed__", False)
 
     @abstractmethod
     def _close(self) -> None:
         pass  # pragma: no cover
 
     def close(self) -> None:
-        '''Flush and close the file-like object.
+        """Flush and close the file-like object.
 
         This method has no effect if the file is already closed.
-        '''
-        if not getattr(self, '__closed__', False):
+        """
+        if not getattr(self, "__closed__", False):
             self._close()
-            setattr(self, '__closed__', True)
+            setattr(self, "__closed__", True)
 
     def __enter__(self: Self) -> Self:
         return self
@@ -64,23 +72,25 @@ class Closable(ABC):
 
 
 class FileLike(Closable, IOBase, IO[AnyStr], ABC):  # pytype: disable=signature-mismatch
-
     def fileno(self) -> int:
-        raise UnsupportedOperation('not a local file')
+        raise UnsupportedOperation("not a local file")
 
     def isatty(self) -> bool:
         return False
 
     def __repr__(self) -> str:
-        return '<%s name=%r mode=%r>' % (
-            fullname(self), self.name, self.mode)  # pragma: no cover
+        return "<%s name=%r mode=%r>" % (
+            fullname(self),
+            self.name,
+            self.mode,
+        )  # pragma: no cover
 
     def seekable(self) -> bool:
-        '''Return True if the file-like object can be sought.'''
+        """Return True if the file-like object can be sought."""
         return False
 
     def seek(self, offset: int, whence: int = os.SEEK_SET) -> int:
-        '''Change stream position.
+        """Change stream position.
 
         Seek to byte `offset` relative to position indicated by `whence`:
             0  Start of stream (the default).  `offset` should be >= 0;
@@ -88,37 +98,37 @@ class FileLike(Closable, IOBase, IO[AnyStr], ABC):  # pytype: disable=signature-
             2  End of stream - `offset` usually negative.
 
         Return the new absolute position.
-        '''
-        raise UnsupportedOperation('not seekable')  # pragma: no cover
+        """
+        raise UnsupportedOperation("not seekable")  # pragma: no cover
 
     def readable(self) -> bool:
-        '''Return True if the file-like object can be read.'''
+        """Return True if the file-like object can be read."""
         return False  # pragma: no cover
 
     def writable(self) -> bool:
-        '''Return True if the file-like object can be written.'''
+        """Return True if the file-like object can be written."""
         return False
 
     def flush(self) -> None:
-        '''Flush write buffers, if applicable.
+        """Flush write buffers, if applicable.
 
         This is not implemented for read-only and non-blocking streams.
-        '''
+        """
 
     def __del__(self) -> None:
-        # TODO: Next version should turn on __del__ for auto closing, and disable this in child class like CombineReader
+        # TODO: Next version should turn on __del__ for auto closing,
+        # and disable this in child class like CombineReader
         pass
 
 
 class Seekable(FileLike, ABC):
-
     def seekable(self) -> bool:
-        '''Return True if the file-like object can be sought.'''
+        """Return True if the file-like object can be sought."""
         return True
 
     @abstractmethod
     def seek(self, offset: int, whence: int = os.SEEK_SET) -> int:
-        '''Change stream position.
+        """Change stream position.
 
         Seek to byte offset `cookie` relative to position indicated by `whence`:
             0  Start of stream (the default).  `cookie` should be >= 0;
@@ -126,42 +136,41 @@ class Seekable(FileLike, ABC):
             2  End of stream - `cookie` usually negative.
 
         Return the new absolute position.
-        '''
+        """
 
 
 class Readable(FileLike[AnyStr], ABC):
-
     def readable(self) -> bool:
-        '''Return True if the file-like object can be read.'''
+        """Return True if the file-like object can be read."""
         return True
 
     @abstractmethod
     def read(self, size: Optional[int] = None) -> AnyStr:
-        '''Read at most `size` bytes or string, returned as a bytes or string object.
+        """Read at most `size` bytes or string, returned as a bytes or string object.
 
         If the `size` argument is negative, read until EOF is reached.
         Return an empty bytes or string object at EOF.
-        '''
+        """
 
     @abstractmethod
     def readline(self, size: Optional[int] = None) -> AnyStr:  # pyre-ignore[15]
-        '''Next line from the file, as a bytes or string object.
+        """Next line from the file, as a bytes or string object.
 
-        Retain newline. A non-negative `size` argument limits the maximum number of bytes or string to return (an incomplete line may be returned then).
+        Retain newline. A non-negative `size` argument limits the maximum number of
+        bytes or string to return (an incomplete line may be returned then).
         Return an empty bytes object at EOF.
-        '''
+        """
 
-    def readlines(  # pyre-ignore[15]
-            self, hint: Optional[int] = None) -> List[AnyStr]:
-        '''Return a list of lines from the stream.'''
+    def readlines(self, hint: Optional[int] = None) -> List[AnyStr]:  # pyre-ignore[15]
+        """Return a list of lines from the stream."""
         return self.read(size=hint).splitlines(True)  # pyre-ignore[7]
 
     def readinto(self, buffer: bytearray) -> int:
-        '''Read bytes into buffer.
+        """Read bytes into buffer.
 
         Returns number of bytes read (0 for EOF), or None if the object
         is set not to block and has no data to read.
-        '''
+        """
         if "b" not in self.mode:
             raise OSError("'readinto' only works on binary files")
 
@@ -180,41 +189,40 @@ class Readable(FileLike[AnyStr], ABC):
         return self
 
     def truncate(self, size: Optional[int] = None) -> int:
-        raise OSError('not writable')
+        raise OSError("not writable")
 
     def write(self, data: AnyStr) -> int:
-        raise OSError('not writable')
+        raise OSError("not writable")
 
     def writelines(self, lines: Iterable[AnyStr]) -> None:  # pyre-ignore[14]
-        raise OSError('not writable')
+        raise OSError("not writable")
 
 
 class Writable(FileLike[AnyStr], ABC):
-
     def writable(self) -> bool:
-        '''Return True if the file-like object can be written.'''
+        """Return True if the file-like object can be written."""
         return True
 
     @abstractmethod
     def write(self, data: AnyStr) -> int:
-        '''Write bytes or string to file.
+        """Write bytes or string to file.
 
         Return the number of bytes or string written.
-        '''
+        """
 
     def writelines(self, lines: Iterable[AnyStr]) -> None:  # pyre-ignore[14]
-        '''Write `lines` to the file.
+        """Write `lines` to the file.
 
-        Note that newlines are not added. 
-        `lines` can be any iterable object producing bytes-like or string-like objects. 
+        Note that newlines are not added.
+        `lines` can be any iterable object producing bytes-like or string-like objects.
         This is equivalent to calling write() for each element.
-        '''
+        """
         for line in lines:
             self.write(line)
 
     def truncate(self, size: Optional[int] = None) -> int:
         """
-        Resize the stream to the given size in bytes. 
+        Resize the stream to the given size in bytes.
 
         :param size: resize size, defaults to None
         :type size: int, optional
@@ -223,21 +231,19 @@ class Writable(FileLike[AnyStr], ABC):
         :return: The new file size.
         :rtype: int
         """
-        raise UnsupportedOperation('not support truncate')
+        raise UnsupportedOperation("not support truncate")
 
     def read(self, size: Optional[int] = None) -> AnyStr:
-        raise OSError('not readable')
+        raise OSError("not readable")
 
     def readline(self, size: Optional[int] = None) -> AnyStr:  # pyre-ignore[15]
-        raise OSError('not readable')
+        raise OSError("not readable")
 
-    def readlines(  # pyre-ignore[15]
-            self, hint: Optional[int] = None) -> List[AnyStr]:
-        raise OSError('not readable')
+    def readlines(self, hint: Optional[int] = None) -> List[AnyStr]:  # pyre-ignore[15]
+        raise OSError("not readable")
 
 
 class FileCacher(ABC):
-
     @property
     @abstractmethod
     def cache_path(self) -> str:
@@ -245,21 +251,21 @@ class FileCacher(ABC):
 
     @property
     def closed(self) -> bool:
-        '''Return True if the file-like object is closed.'''
-        return getattr(self, '__closed__', False)
+        """Return True if the file-like object is closed."""
+        return getattr(self, "__closed__", False)
 
     @abstractmethod
     def _close(self) -> None:
         pass  # pragma: no cover
 
     def close(self) -> None:
-        '''Flush and close the file-like object.
+        """Flush and close the file-like object.
 
         This method has no effect if the file is already closed.
-        '''
-        if not getattr(self, '__closed__', False):
+        """
+        if not getattr(self, "__closed__", False):
             self._close()
-            setattr(self, '__closed__', True)
+            setattr(self, "__closed__", True)
 
     def __enter__(self) -> str:
         return self.cache_path
@@ -282,7 +288,6 @@ class NullCacher(FileCacher):
 
 
 class ContextIterator(Closable):
-
     def __init__(self, iterable: Iterable) -> None:
         self._iter = iter(iterable)
 

megfile/lib/base_prefetch_reader.py

@@ -8,15 +8,22 @@ from math import ceil
 from statistics import mean
 from typing import Optional
 
-from megfile.config import BACKOFF_FACTOR, BACKOFF_INITIAL, DEFAULT_BLOCK_CAPACITY, DEFAULT_BLOCK_SIZE, DEFAULT_MAX_RETRY_TIMES, GLOBAL_MAX_WORKERS, NEWLINE
+from megfile.config import (
+    BACKOFF_FACTOR,
+    BACKOFF_INITIAL,
+    DEFAULT_BLOCK_CAPACITY,
+    DEFAULT_BLOCK_SIZE,
+    DEFAULT_MAX_RETRY_TIMES,
+    GLOBAL_MAX_WORKERS,
+    NEWLINE,
+)
 from megfile.interfaces import Readable, Seekable
-from megfile.utils import get_human_size, process_local
+from megfile.utils import ProcessLocal, get_human_size, process_local
 
 _logger = get_logger(__name__)
 
 
 class SeekRecord:
-
     def __init__(self, seek_index: int):
         self.seek_index = seek_index
         self.seek_count = 0
@@ -24,38 +31,49 @@ class SeekRecord:
 
 
 class BasePrefetchReader(Readable[bytes], Seekable, ABC):
-    '''
-    Reader to fast read the remote file content. 
-    This will divide the file content into equal parts of block_size size, 
+    """
+    Reader to fast read the remote file content.
+    This will divide the file content into equal parts of block_size size,
     and will use LRU to cache at most block_capacity blocks in memory.
-    open(), seek() and read() will trigger prefetch read. 
-    The prefetch will cached block_forward blocks of data from offset position 
+    open(), seek() and read() will trigger prefetch read.
+    The prefetch will cached block_forward blocks of data from offset position
     (the position after reading if the called function is read).
-    '''
+    """
 
     def __init__(
-            self,
-            *,
-            block_size: int = DEFAULT_BLOCK_SIZE,
-            block_capacity: int = DEFAULT_BLOCK_CAPACITY,
-            block_forward: Optional[int] = None,
-            max_retries: int = DEFAULT_MAX_RETRY_TIMES,
-            max_workers: Optional[int] = None,
-            **kwargs):
-
+        self,
+        *,
+        block_size: int = DEFAULT_BLOCK_SIZE,
+        block_capacity: int = DEFAULT_BLOCK_CAPACITY,
+        block_forward: Optional[int] = None,
+        max_retries: int = DEFAULT_MAX_RETRY_TIMES,
+        max_workers: Optional[int] = None,
+        **kwargs,
+    ):
         self._is_auto_scaling = block_forward is None
         if block_forward is None:
             block_forward = max(block_capacity - 1, 1)
 
-        assert block_capacity > block_forward, 'block_capacity should greater than block_forward, got: block_capacity=%s, block_forward=%s' % (
-            block_capacity, block_forward)
+        if block_capacity <= block_forward:
+            # TODO: replace AssertionError with ValueError in 4.0.0
+            raise AssertionError(
+                "block_capacity should greater than block_forward, "
+                "got: block_capacity=%s, block_forward=%s"
+                % (block_capacity, block_forward)
+            )
+
+        # user maybe put block_size with 'numpy.uint64' type
+        block_size = int(block_size)
 
         self._max_retries = max_retries
         self._block_size = block_size
         self._block_capacity = block_capacity  # Max number of blocks
-        self._block_forward = block_forward  # Number of blocks every prefetch, which should be smaller than block_capacity
 
-        self._futures = self._get_futures()
+        # Number of blocks every prefetch, which should be smaller than block_capacity
+        self._block_forward = block_forward
+
+        self._process_local = ProcessLocal()
+
         self._content_size = self._get_content_size()
         self._block_stop = ceil(self._content_size / block_size)
 
@@ -68,20 +86,25 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
         self._is_global_executor = False
         if max_workers is None:
             self._executor = process_local(
-                f'{self.__class__.__name__}.executor',
+                f"{self.__class__.__name__}.executor",
                 ThreadPoolExecutor,
-                max_workers=GLOBAL_MAX_WORKERS)
+                max_workers=GLOBAL_MAX_WORKERS,
+            )
             self._is_global_executor = True
         else:
             self._executor = ThreadPoolExecutor(max_workers=max_workers)
         self._seek_buffer(0)
 
-        _logger.debug('open file: %r, mode: %s' % (self.name, self.mode))
+        _logger.debug("open file: %r, mode: %s" % (self.name, self.mode))
 
     @abstractmethod
     def _get_content_size(self):
         pass
 
+    @property
+    def _futures(self):
+        return self._process_local("futures", self._get_futures)
+
     def _get_futures(self):
         return LRUCacheFutureManager()
 
@@ -92,7 +115,7 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
 
     @property
     def mode(self) -> str:
-        return 'rb'
+        return "rb"
 
     def tell(self) -> int:
         return self._offset
@@ -105,15 +128,15 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
     def _offset(self, value: int):
         if value > self._backoff_size:
             _logger.debug(
-                'reading file: %r, current offset / total size: %s / %s' % (
-                    self.name, get_human_size(value),
-                    get_human_size(self._content_size)))
+                "reading file: %r, current offset / total size: %s / %s"
+                % (self.name, get_human_size(value), get_human_size(self._content_size))
+            )
         while value > self._backoff_size:
             self._backoff_size *= BACKOFF_FACTOR
         self.__offset = value
 
     def seek(self, offset: int, whence: int = os.SEEK_SET) -> int:
-        '''Change stream position.
+        """Change stream position.
 
         Seek to byte offset pos relative to position indicated by whence:
 
@@ -122,10 +145,10 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
             2  End of stream - pos usually negative.
 
         Returns the new absolute position.
-        '''
+        """
+        offset = int(offset)  # user maybe put offset with 'numpy.uint64' type
         if self.closed:
-            raise IOError('file already closed: %r' % self.name)
-
+            raise IOError("file already closed: %r" % self.name)
         if whence == os.SEEK_CUR:
             target_offset = self._offset + offset
         elif whence == os.SEEK_END:
@@ -133,7 +156,7 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
         elif whence == os.SEEK_SET:
             target_offset = offset
         else:
-            raise ValueError('invalid whence: %r' % whence)
+            raise ValueError("invalid whence: %r" % whence)
 
         if target_offset == self._offset:
             return target_offset
@@ -145,35 +168,34 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
         return self._offset
 
     def read(self, size: Optional[int] = None) -> bytes:
-        '''Read at most size bytes, returned as a bytes object.
+        """Read at most size bytes, returned as a bytes object.
 
         If the size argument is negative, read until EOF is reached.
         Return an empty bytes object at EOF.
-        '''
+        """
         if self.closed:
-            raise IOError('file already closed: %r' % self.name)
+            raise IOError("file already closed: %r" % self.name)
 
         if len(self._seek_history) > 0:
             self._seek_history[-1].read_count += 1
 
         if self._offset >= self._content_size:
-            return b''
+            return b""
 
-        if size is None:
+        if size is None or size < 0:
             size = self._content_size - self._offset
         else:
-            assert size >= 0, 'size should greater than 0, got: %r' % size
             size = min(size, self._content_size - self._offset)
 
         if self._block_forward == 1:
             block_index = self._offset // self._block_size
             if len(self._seek_history) > 0:
-                mean_read_count = mean(
-                    item.read_count for item in self._seek_history)
+                mean_read_count = mean(item.read_count for item in self._seek_history)
             else:
                 mean_read_count = 0
             if block_index not in self._futures and mean_read_count < 3:
-                # No using LRP will be better if read() are always called less than 3 times after seek()
+                # No using LRP will be better if read() are always called less than 3
+                # times after seek()
                 return self._read(size)
 
         data = self._buffer.read(size)
@@ -192,24 +214,24 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
         return buffer.getvalue()
 
     def readline(self, size: Optional[int] = None) -> bytes:
-        '''Next line from the file, as a bytes object.
+        """Next line from the file, as a bytes object.
 
         Retain newline.  A non-negative size argument limits the maximum
         number of bytes to return (an incomplete line may be returned then).
+        If the size argument is negative, read until EOF is reached.
         Return an empty bytes object at EOF.
-        '''
+        """
         if self.closed:
-            raise IOError('file already closed: %r' % self.name)
+            raise IOError("file already closed: %r" % self.name)
 
         if len(self._seek_history) > 0:
             self._seek_history[-1].read_count += 1
         if self._offset >= self._content_size:
-            return b''
+            return b""
 
-        if size is None:
+        if size is None or size < 0:
             size = self._content_size - self._offset
         else:
-            assert size >= 0, 'size should greater than 0, got: %r' % size
             size = min(size, self._content_size - self._offset)
 
         data = self._buffer.readline(size)
@@ -231,21 +253,22 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
 
     def _read(self, size: int) -> bytes:
         if size == 0 or self._offset >= self._content_size:
-            return b''
+            return b""
 
-        data = self._fetch_response(
-            start=self._offset, end=self._offset + size - 1)['Body'].read()
+        data = self._fetch_response(start=self._offset, end=self._offset + size - 1)[
+            "Body"
+        ].read()
         self.seek(size, os.SEEK_CUR)
         return data
 
     def readinto(self, buffer: bytearray) -> int:
-        '''Read bytes into buffer.
+        """Read bytes into buffer.
 
         Returns number of bytes read (0 for EOF), or None if the object
         is set not to block and has no data to read.
-        '''
+        """
         if self.closed:
-            raise IOError('file already closed: %r' % self.name)
+            raise IOError("file already closed: %r" % self.name)
 
         if self._offset >= self._content_size:
             return 0
@@ -254,7 +277,7 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
         size = min(size, self._content_size - self._offset)
 
         data = self._buffer.read(size)
-        buffer[:len(data)] = data
+        buffer[: len(data)] = data
         if len(data) == size:
             self._offset += len(data)
             return size
@@ -263,7 +286,7 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
         while offset < size:
             remain_size = size - offset
             data = self._next_buffer.read(remain_size)
-            buffer[offset:offset + len(data)] = data
+            buffer[offset : offset + len(data)] = data
             offset += len(data)
 
         self._offset += offset
@@ -300,8 +323,11 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
 
     @property
     def _next_buffer(self) -> BytesIO:
-        # Get next buffer by this function when finished reading current buffer (self._buffer)
-        # Make sure that _buffer is used before using _next_buffer(), or will make _cached_offset invalid
+        # Get next buffer by this function when finished reading current buffer
+        # (self._buffer)
+        #
+        # Make sure that _buffer is used before using _next_buffer(), or will make
+        # _cached_offset invalid
         self._block_index += 1
         self._cached_offset = 0
         return self._buffer
@@ -313,7 +339,8 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
             history = []
             for item in self._seek_history:
                 if item.seek_count > self._block_capacity * 2:
-                    # seek interval is bigger than self._block_capacity * 2, drop it from self._seek_history
+                    # seek interval is bigger than self._block_capacity * 2, drop it
+                    # from self._seek_history
                     continue
                 if index - 1 < item.seek_index < index + 2:
                     continue
@@ -322,23 +349,22 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
             history.append(SeekRecord(index))
             self._seek_history = history
             self._block_forward = max(
-                (self._block_capacity - 1) // len(self._seek_history), 1)
+                (self._block_capacity - 1) // len(self._seek_history), 1
+            )
 
         self._cached_offset = offset
         self._block_index = index
 
     @abstractmethod
     def _fetch_response(
-            self,
-            start: Optional[int] = None,
-            end: Optional[int] = None) -> dict:
+        self, start: Optional[int] = None, end: Optional[int] = None
+    ) -> dict:
         pass
 
     def _fetch_buffer(self, index: int) -> BytesIO:
-        start, end = index * self._block_size, (
-            index + 1) * self._block_size - 1
+        start, end = index * self._block_size, (index + 1) * self._block_size - 1
         response = self._fetch_response(start=start, end=end)
-        return response['Body']
+        return response["Body"]
 
     def _submit_future(self, index: int):
         if index < 0 or index >= self._block_stop:
@@ -355,7 +381,7 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
         self._futures.cleanup(self._block_capacity)
 
     def _close(self):
-        _logger.debug('close file: %r' % self.name)
+        _logger.debug("close file: %r" % self.name)
 
         if not self._is_global_executor:
             self._executor.shutdown()
@@ -363,7 +389,6 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
 
 
 class LRUCacheFutureManager(OrderedDict):
-
     def __init__(self):
         super().__init__()