megfile 3.0.6.post1__py3-none-any.whl → 3.1.0.post1__py3-none-any.whl

megfile/http.py

@@ -1,4 +1,4 @@
-from megfile.http_path import HttpPath, HttpsPath, get_http_session, http_open, is_http
+from megfile.http_path import HttpPath, get_http_session, http_open, is_http
 from megfile.interfaces import PathLike, StatResult
 
 __all__ = [

megfile/http_path.py

@@ -33,8 +33,8 @@ max_retries = HTTP_MAX_RETRY_TIMES
 
 
 def get_http_session(
-        timeout: Optional[Union[int, Tuple[int, int]]] = DEFAULT_TIMEOUT,
-        status_forcelist: Iterable[int] = (500, 502, 503, 504)
+    timeout: Optional[Union[int, Tuple[int, int]]] = DEFAULT_TIMEOUT,
+    status_forcelist: Iterable[int] = (500, 502, 503, 504)
 ) -> requests.Session:
     session = requests.Session()
 
@@ -49,24 +49,24 @@ def get_http_session(
             kwargs)
 
     def retry_callback(
-            error,
-            method,
-            url,
-            params=None,
-            data=None,
-            headers=None,
-            cookies=None,
-            files=None,
-            auth=None,
-            timeout=None,
-            allow_redirects=True,
-            proxies=None,
-            hooks=None,
-            stream=None,
-            verify=None,
-            cert=None,
-            json=None,
-            **kwargs,
+        error,
+        method,
+        url,
+        params=None,
+        data=None,
+        headers=None,
+        cookies=None,
+        files=None,
+        auth=None,
+        timeout=None,
+        allow_redirects=True,
+        proxies=None,
+        hooks=None,
+        stream=None,
+        verify=None,
+        cert=None,
+        json=None,
+        **kwargs,
     ):
         if data and hasattr(data, 'seek'):
             data.seek(0)
@@ -86,7 +86,7 @@ def get_http_session(
                         return BytesIO(f.read())
                 else:
                     _logger.warning(
-                        f'Can not retry http request, because the file object is not seekable and unsupport "name"'
+                        f'Can not retry http request, because the file object is not seekable and not support "name"'
                     )
                     raise
 
@@ -243,8 +243,8 @@ class HttpPath(URIPath):
                 block_forward=block_forward,
                 block_size=block_size,
             )
-            if _is_pickle(reader):  # pytype: disable=wrong-arg-types
-                reader = BufferedReader(reader)  # pytype: disable=wrong-arg-types
+            if _is_pickle(reader):
+                reader = BufferedReader(reader)  # type: ignore
             return reader
 
         response.raw.name = self.path_with_protocol
@@ -252,7 +252,7 @@ class HttpPath(URIPath):
         # response.raw.auto_close = False
         # response.raw.decode_content = True
         # return BufferedReader(response.raw)
-        return BufferedReader(Response(response.raw))  # pytype: disable=wrong-arg-types
+        return BufferedReader(Response(response.raw))  # type: ignore
 
     def stat(self, follow_symlinks=True) -> StatResult:
         '''
@@ -279,15 +279,22 @@ class HttpPath(URIPath):
         size = headers.get('Content-Length')
         if size:
             size = int(size)
+        else:
+            size = 0
 
         last_modified = headers.get('Last-Modified')
         if last_modified:
             last_modified = time.mktime(
                 time.strptime(last_modified, "%a, %d %b %Y %H:%M:%S %Z"))
+        else:
+            last_modified = 0.0
 
-        return StatResult(  # pyre-ignore[20]
-            size=size, mtime=last_modified, isdir=False,
-            islnk=False, extra=headers)
+        return StatResult(
+            size=size,
+            mtime=last_modified,
+            isdir=False,
+            islnk=False,
+            extra=headers)
 
     def getsize(self, follow_symlinks: bool = False) -> int:
         '''
@@ -342,7 +349,7 @@ class HttpsPath(HttpPath):
     protocol = "https"
 
 
-class Response(Readable):
+class Response(Readable[bytes]):
 
     def __init__(self, raw: HTTPResponse) -> None:
         super().__init__()

megfile/interfaces.py

@@ -1,9 +1,28 @@
 import os
 from abc import ABC, abstractmethod
-from io import UnsupportedOperation
-from typing import Iterable, Iterator, List, Optional
-
-from megfile.pathlike import Access, BasePath, BaseURIPath, FileEntry, PathLike, StatResult, URIPath
+from io import IOBase, UnsupportedOperation
+from typing import IO, AnyStr, Iterable, List, Optional
+
+from megfile.pathlike import Access, BasePath, BaseURIPath, FileEntry, PathLike, Self, StatResult, URIPath
+
+__all__ = [
+    "Access",
+    "BasePath",
+    "BaseURIPath",
+    "FileEntry",
+    "PathLike",
+    "StatResult",
+    "URIPath",
+    "fullname",
+    "Closable",
+    "FileLike",
+    "Seekable",
+    "Readable",
+    "Writable",
+    "FileCacher",
+    "NullCacher",
+    "ContextIterator",
+]
 
 
 def fullname(o):
@@ -37,52 +56,41 @@ class Closable(ABC):
             self._close()
             setattr(self, '__closed__', True)
 
-    def __enter__(self) -> 'Closable':
+    def __enter__(self: Self) -> Self:
         return self
 
     def __exit__(self, type, value, traceback) -> None:
         self.close()
 
 
-class FileLike(Closable, ABC):
-
-    @property
-    @abstractmethod
-    def name(self) -> str:
-        pass  # pragma: no cover
-
-    @property
-    @abstractmethod
-    def mode(self) -> str:
-        pass  # pragma: no cover
+class FileLike(Closable, IOBase, IO[AnyStr], ABC):  # pytype: disable=signature-mismatch
 
     def fileno(self) -> int:
         raise UnsupportedOperation('not a local file')
 
+    def isatty(self) -> bool:
+        return False
+
     def __repr__(self) -> str:
         return '<%s name=%r mode=%r>' % (
             fullname(self), self.name, self.mode)  # pragma: no cover
 
     def seekable(self) -> bool:
-        '''Return True if the file-like object can be seeked.'''
+        '''Return True if the file-like object can be sought.'''
         return False
 
-    def seek(self, cookie: int, whence: int = os.SEEK_SET) -> int:
+    def seek(self, offset: int, whence: int = os.SEEK_SET) -> int:
         '''Change stream position.
 
-        Seek to byte offset `cookie` relative to position indicated by `whence`:
-            0  Start of stream (the default).  `cookie` should be >= 0;
-            1  Current position - `cookie` may be negative;
-            2  End of stream - `cookie` usually negative.
+        Seek to byte `offset` relative to position indicated by `whence`:
+            0  Start of stream (the default).  `offset` should be >= 0;
+            1  Current position - `offset` may be negative;
+            2  End of stream - `offset` usually negative.
 
         Return the new absolute position.
         '''
         raise UnsupportedOperation('not seekable')  # pragma: no cover
 
-    @abstractmethod
-    def tell(self) -> int:
-        '''Return current stream position.'''
-
     def readable(self) -> bool:
         '''Return True if the file-like object can be read.'''
         return False  # pragma: no cover
@@ -97,15 +105,19 @@ class FileLike(Closable, ABC):
         This is not implemented for read-only and non-blocking streams.
         '''
 
+    def __del__(self) -> None:
+        # TODO: Next version should turn on __del__ for auto closing, and disable this in child class like CombineReader
+        pass
+
 
 class Seekable(FileLike, ABC):
 
     def seekable(self) -> bool:
-        '''Return True if the file-like object can be seeked.'''
+        '''Return True if the file-like object can be sought.'''
         return True
 
     @abstractmethod
-    def seek(self, cookie: int, whence: int = os.SEEK_SET) -> int:
+    def seek(self, offset: int, whence: int = os.SEEK_SET) -> int:
         '''Change stream position.
 
         Seek to byte offset `cookie` relative to position indicated by `whence`:
@@ -117,31 +129,32 @@ class Seekable(FileLike, ABC):
         '''
 
 
-class Readable(FileLike, ABC):
+class Readable(FileLike[AnyStr], ABC):
 
     def readable(self) -> bool:
         '''Return True if the file-like object can be read.'''
         return True
 
     @abstractmethod
-    def read(self, size: Optional[int] = None) -> bytes:
-        '''Read at most `size` bytes, returned as a bytes object.
+    def read(self, size: Optional[int] = None) -> AnyStr:
+        '''Read at most `size` bytes or string, returned as a bytes or string object.
 
         If the `size` argument is negative, read until EOF is reached.
-        Return an empty bytes object at EOF.
+        Return an empty bytes or string object at EOF.
         '''
 
     @abstractmethod
-    def readline(self, size: Optional[int] = None) -> bytes:
-        '''Next line from the file, as a bytes object.
+    def readline(self, size: Optional[int] = None) -> AnyStr:  # pyre-ignore[15]
+        '''Next line from the file, as a bytes or string object.
 
-        Retain newline. A non-negative `size` argument limits the maximum number of bytes to return (an incomplete line may be returned then).
+        Retain newline. A non-negative `size` argument limits the maximum number of bytes or string to return (an incomplete line may be returned then).
         Return an empty bytes object at EOF.
         '''
 
-    def readlines(self) -> List[bytes]:
+    def readlines(  # pyre-ignore[15]
+            self, hint: Optional[int] = None) -> List[AnyStr]:
         '''Return a list of lines from the stream.'''
-        return self.read().splitlines(True)
+        return self.read(size=hint).splitlines(True)  # pyre-ignore[7]
 
     def readinto(self, buffer: bytearray) -> int:
         '''Read bytes into buffer.
@@ -149,53 +162,111 @@ class Readable(FileLike, ABC):
         Returns number of bytes read (0 for EOF), or None if the object
         is set not to block and has no data to read.
         '''
+        if "b" not in self.mode:
+            raise OSError("'readinto' only works on binary files")
+
         data = self.read(len(buffer))
         size = len(data)
-        buffer[:size] = data
+        buffer[:size] = data  # pyre-ignore[6]
         return size
 
-    def __next__(self) -> bytes:
+    def __next__(self) -> AnyStr:  # pyre-ignore[15]
         line = self.readline()
         if not line:
             raise StopIteration
         return line
 
-    def __iter__(self) -> Iterator[bytes]:
+    def __iter__(self: Self) -> Self:  # pyre-ignore[15]
         return self
 
+    def truncate(self, size: Optional[int] = None) -> int:
+        raise OSError('not writable')
+
+    def write(self, data: AnyStr) -> int:
+        raise OSError('not writable')
+
+    def writelines(self, lines: Iterable[AnyStr]) -> None:  # pyre-ignore[14]
+        raise OSError('not writable')
 
-class Writable(FileLike, ABC):
+
+class Writable(FileLike[AnyStr], ABC):
 
     def writable(self) -> bool:
         '''Return True if the file-like object can be written.'''
         return True
 
     @abstractmethod
-    def write(self, data: bytes) -> int:
-        '''Write bytes to file.
+    def write(self, data: AnyStr) -> int:
+        '''Write bytes or string to file.
 
-        Return the number of bytes written.
+        Return the number of bytes or string written.
         '''
 
-    def writelines(self, lines: Iterable[bytes]) -> None:
+    def writelines(self, lines: Iterable[AnyStr]) -> None:  # pyre-ignore[14]
         '''Write `lines` to the file.
 
-        Note that newlines are not added. `lines` can be any iterable object producing bytes-like objects. This is equivalent to calling write() for each element.
+        Note that newlines are not added. 
+        `lines` can be any iterable object producing bytes-like or string-like objects. 
+        This is equivalent to calling write() for each element.
         '''
         for line in lines:
             self.write(line)
 
+    def truncate(self, size: Optional[int] = None) -> int:
+        """
+        Resize the stream to the given size in bytes. 
+
+        :param size: resize size, defaults to None
+        :type size: int, optional
+
+        :raises OSError: When the stream is not support truncate.
+        :return: The new file size.
+        :rtype: int
+        """
+        raise UnsupportedOperation('not support truncate')
 
-class FileCacher(Closable):
+    def read(self, size: Optional[int] = None) -> AnyStr:
+        raise OSError('not readable')
+
+    def readline(self, size: Optional[int] = None) -> AnyStr:  # pyre-ignore[15]
+        raise OSError('not readable')
+
+    def readlines(  # pyre-ignore[15]
+            self, hint: Optional[int] = None) -> List[AnyStr]:
+        raise OSError('not readable')
+
+
+class FileCacher(ABC):
 
     @property
     @abstractmethod
     def cache_path(self) -> str:
         pass  # pragma: no cover
 
-    def __enter__(self) -> str:  # pytype: disable=signature-mismatch
+    @property
+    def closed(self) -> bool:
+        '''Return True if the file-like object is closed.'''
+        return getattr(self, '__closed__', False)
+
+    @abstractmethod
+    def _close(self) -> None:
+        pass  # pragma: no cover
+
+    def close(self) -> None:
+        '''Flush and close the file-like object.
+
+        This method has no effect if the file is already closed.
+        '''
+        if not getattr(self, '__closed__', False):
+            self._close()
+            setattr(self, '__closed__', True)
+
+    def __enter__(self) -> str:
         return self.cache_path
 
+    def __exit__(self, type, value, traceback) -> None:
+        self.close()
+
     def __del__(self):
         self.close()
 

megfile/lib/base_prefetch_reader.py

@@ -23,7 +23,7 @@ class SeekRecord:
         self.read_count = 0
 
 
-class BasePrefetchReader(Readable, Seekable, ABC):
+class BasePrefetchReader(Readable[bytes], Seekable, ABC):
     '''
     Reader to fast read the remote file content. 
     This will divide the file content into equal parts of block_size size, 
@@ -112,7 +112,7 @@ class BasePrefetchReader(Readable, Seekable, ABC):
             self._backoff_size *= BACKOFF_FACTOR
         self.__offset = value
 
-    def seek(self, cookie: int, whence: int = os.SEEK_SET) -> int:
+    def seek(self, offset: int, whence: int = os.SEEK_SET) -> int:
         '''Change stream position.
 
         Seek to byte offset pos relative to position indicated by whence:
@@ -127,11 +127,11 @@ class BasePrefetchReader(Readable, Seekable, ABC):
             raise IOError('file already closed: %r' % self.name)
 
         if whence == os.SEEK_CUR:
-            target_offset = self._offset + cookie
+            target_offset = self._offset + offset
         elif whence == os.SEEK_END:
-            target_offset = self._content_size + cookie
+            target_offset = self._content_size + offset
         elif whence == os.SEEK_SET:
-            target_offset = cookie
+            target_offset = offset
         else:
             raise ValueError('invalid whence: %r' % whence)
 
@@ -271,7 +271,7 @@ class BasePrefetchReader(Readable, Seekable, ABC):
 
     @property
     def _is_alive(self):
-        return not self._executor._shutdown  # pytype: disable=attribute-error
+        return not self._executor._shutdown
 
     @property
     def _is_downloading(self):
@@ -307,7 +307,7 @@ class BasePrefetchReader(Readable, Seekable, ABC):
         return self._buffer
 
     def _seek_buffer(self, index: int, offset: int = 0):
-        # The corresponding block is probably not downloaded when seeked to a new position
+        # The corresponding block is probably not downloaded when seek to a new position
         # So record the offset first, set it when it is accessed
         if self._is_auto_scaling:  # When user doesn't define forward
             history = []
@@ -329,7 +329,8 @@ class BasePrefetchReader(Readable, Seekable, ABC):
 
     @abstractmethod
     def _fetch_response(
-            self, start: Optional[int] = None,
+            self,
+            start: Optional[int] = None,
             end: Optional[int] = None) -> dict:
         pass
 

megfile/lib/combine_reader.py

@@ -49,17 +49,17 @@ class CombineReader(Readable, Seekable):
     def tell(self) -> int:
         return self._offset
 
-    def _empty_bytes(self) -> AnyStr:  # pytype: disable=signature-mismatch
+    def _empty_bytes(self) -> AnyStr:  # pyre-ignore[34]
         if 'b' in self._mode:
-            return b''
-        return ''
+            return b''  # pyre-ignore[7]
+        return ''  # pyre-ignore[7]
 
     def _empty_buffer(self) -> Union[BytesIO, StringIO]:
         if 'b' in self._mode:
             return BytesIO()
         return StringIO()
 
-    def read(self, size: Optional[int] = None) -> AnyStr:  # pytype: disable=signature-mismatch
+    def read(self, size: Optional[int] = None) -> AnyStr:  # pyre-ignore[34]
         if self._offset >= self._content_size:
             return self._empty_bytes()
         if size is None or size < 0:
@@ -72,9 +72,9 @@ class CombineReader(Readable, Seekable):
             buffer.write(data)
             size -= len(data)
             self._offset += len(data)
-        return buffer.getvalue()
+        return buffer.getvalue()  # pyre-ignore[7]
 
-    def readline(self, size: Optional[int] = None) -> AnyStr:  # pytype: disable=signature-mismatch
+    def readline(self, size: Optional[int] = None) -> AnyStr:  # pyre-ignore[34]
         if self._offset >= self._content_size:
             return self._empty_bytes()
         if size is None or size < 0:
@@ -96,7 +96,7 @@ class CombineReader(Readable, Seekable):
             self._offset += len(data)
             if buffer.tell() == size or data[-1] == NEWLINE:
                 break
-        return buffer.getvalue()
+        return buffer.getvalue()  # pyre-ignore[7]
 
     def seek(self, offset: int, whence: int = os.SEEK_SET) -> int:
         if whence == os.SEEK_SET:

megfile/lib/fnmatch.py

@@ -14,7 +14,6 @@ corresponding to PATTERN.  (It does not compile it.)
 import functools
 import os
 import re
-import sys
 from typing import Callable, List, Match, Optional
 
 
@@ -127,7 +126,8 @@ def translate(pat: str) -> str:
                 res = res + r'\{'
             else:
                 stuff = pat[i:j].replace('\\', r'\\')
-                stuff = r'|'.join(map(re.escape, stuff.split(',')))
+                stuff = r'|'.join(
+                    map(re.escape, stuff.split(',')))  # pyre-ignore[6]
                 res = r'%s(%s)' % (res, stuff)
                 i = j + 1
         else:

megfile/lib/glob.py

@@ -5,7 +5,7 @@ import os
 import re
 from collections import OrderedDict
 from collections import namedtuple as NamedTuple
-from typing import Callable, Iterator, List, Tuple
+from typing import Iterator, List, Tuple
 
 from megfile.lib import fnmatch
 
@@ -131,7 +131,7 @@ def _glob1(dirname: str, pattern: str, dironly: bool, fs: FSFunc) -> List[str]:
     names = list(_iterdir(dirname, dironly, fs))
     if not _ishidden(pattern):
         names = (x for x in names if not _ishidden(x))
-    return fnmatch.filter(names, pattern)
+    return fnmatch.filter(names, pattern)  # pyre-ignore[6]
 
 
 def _glob0(dirname: str, basename: str, dironly: bool, fs: FSFunc) -> List[str]:
@@ -165,7 +165,7 @@ def _iterdir(dirname: str, dironly: bool, fs: FSFunc) -> Iterator[str]:
         for name, isdir in fs.scandir(dirname):
             try:
                 if not dironly or isdir:
-                    yield name  # type: ignore
+                    yield name
             except OSError:
                 pass
     except OSError:

megfile/lib/hdfs_prefetch_reader.py

@@ -44,7 +44,8 @@ class HdfsPrefetchReader(BasePrefetchReader):
             f"+{self._profile_name}" if self._profile_name else "", self._path)
 
     def _fetch_response(
-            self, start: Optional[int] = None,
+            self,
+            start: Optional[int] = None,
             end: Optional[int] = None) -> dict:
         with raise_hdfs_error(self.name):
             with self._client.read(self._path, offset=start or 0, length=end -

megfile/lib/http_prefetch_reader.py

@@ -58,13 +58,14 @@ class HttpPrefetchReader(BasePrefetchReader):
         return fspath(self._url)
 
     def _fetch_response(
-            self, start: Optional[int] = None,
+            self,
+            start: Optional[int] = None,
             end: Optional[int] = None) -> dict:
 
         def fetch_response() -> dict:
             request_kwargs = {}
             if hasattr(self._url, 'request_kwargs'):
-                request_kwargs = self._url.request_kwargs
+                request_kwargs = self._url.request_kwargs  # pyre-ignore[16]
             timeout = request_kwargs.pop('timeout', DEFAULT_TIMEOUT)
             stream = request_kwargs.pop('stream', True)
 

megfile/lib/lazy_handler.py

@@ -1,8 +1,9 @@
 import os
+from functools import cached_property
 from typing import AnyStr, Callable, Optional
 
 from megfile.interfaces import Readable, Seekable, Writable
-from megfile.utils import cachedproperty, get_content_size
+from megfile.utils import get_content_size
 
 
 class LazyHandler(Readable, Seekable, Writable):
@@ -24,7 +25,7 @@ class LazyHandler(Readable, Seekable, Writable):
     def mode(self) -> str:
         return self._mode
 
-    @cachedproperty
+    @cached_property
     def _file_object(self):
         return self._open_func(self._path, self._mode, **self._options)
 
@@ -41,16 +42,16 @@ class LazyHandler(Readable, Seekable, Writable):
     def readable(self) -> bool:
         return self._file_object.readable()
 
-    def read(self, size: Optional[int] = None) -> AnyStr:  # pytype: disable=signature-mismatch
+    def read(self, size: Optional[int] = None) -> AnyStr:  # pyre-ignore[34]
         return self._file_object.read(size)
 
-    def readline(self, size: Optional[int] = None) -> AnyStr:  # pytype: disable=signature-mismatch
+    def readline(self, size: Optional[int] = None) -> AnyStr:  # pyre-ignore[34]
         return self._file_object.readline(size)
 
     def writable(self) -> bool:
         return self._file_object.writable()
 
-    def write(self, data: AnyStr):  # pytype: disable=signature-mismatch
+    def write(self, data: AnyStr):
         return self._file_object.write(data)
 
     def _close(self):

megfile/lib/s3_buffered_writer.py

@@ -5,7 +5,7 @@ from logging import getLogger as get_logger
 from threading import Lock
 from typing import NamedTuple, Optional
 
-from megfile.config import BACKOFF_FACTOR, BACKOFF_INITIAL, DEFAULT_BLOCK_SIZE, DEFAULT_MAX_BLOCK_SIZE, DEFAULT_MAX_BUFFER_SIZE, GLOBAL_MAX_WORKERS
+from megfile.config import BACKOFF_FACTOR, BACKOFF_INITIAL, DEFAULT_MAX_BLOCK_SIZE, DEFAULT_MAX_BUFFER_SIZE, DEFAULT_MIN_BLOCK_SIZE, GLOBAL_MAX_WORKERS
 from megfile.errors import raise_s3_error
 from megfile.interfaces import Writable
 from megfile.utils import get_human_size, process_local
@@ -34,7 +34,7 @@ class PartResult(_PartResult):
         }
 
 
-class S3BufferedWriter(Writable):
+class S3BufferedWriter(Writable[bytes]):
 
     def __init__(
             self,
@@ -42,7 +42,7 @@ class S3BufferedWriter(Writable):
             key: str,
             *,
             s3_client,
-            block_size: int = DEFAULT_BLOCK_SIZE,
+            block_size: int = DEFAULT_MIN_BLOCK_SIZE,
             max_block_size: int = DEFAULT_MAX_BLOCK_SIZE,
             max_buffer_size: int = DEFAULT_MAX_BUFFER_SIZE,
             max_workers: Optional[int] = None,
@@ -137,10 +137,11 @@ class S3BufferedWriter(Writable):
     @property
     def _multipart_upload(self):
         return {
-            'Parts': [
-                future.result().asdict()
-                for _, future in sorted(self._futures.items())
-            ],
+            'Parts':
+                [
+                    future.result().asdict()
+                    for _, future in sorted(self._futures.items())
+                ],
         }
 
     def _upload_buffer(self, part_number, content):