megfile 4.0.4__py3-none-any.whl → 4.1.0.post1__py3-none-any.whl

megfile/errors.py

@@ -273,8 +273,8 @@ class S3InvalidRangeError(S3Exception):
 
 
 class S3UnknownError(S3Exception, UnknownError):
-    def __init__(self, error: Exception, path: PathLike):
-        super().__init__(error, path, "endpoint: %r" % s3_endpoint_url(path))
+    def __init__(self, error: Exception, path: PathLike, extra: Optional[str] = None):
+        super().__init__(error, path, extra or "endpoint: %r" % s3_endpoint_url(path))
 
 
 class HttpException(Exception):
@@ -413,12 +413,12 @@ def translate_http_error(http_error: Exception, http_url: str) -> Exception:
 
 
 @contextmanager
-def raise_s3_error(s3_url: PathLike, suppress_errors=()):
+def raise_s3_error(s3_url: PathLike, suppress_error_callback=None):
     try:
         yield
     except Exception as error:
         error = translate_s3_error(error, s3_url)
-        if suppress_errors and isinstance(error, suppress_errors):
+        if suppress_error_callback and suppress_error_callback(error):
             return
         raise error
 

megfile/fs.py

@@ -317,7 +317,7 @@ def fs_walk(
     return FSPath(path).walk(followlinks)
 
 
-def fs_getmd5(path: PathLike, recalculate: bool = False, followlinks: bool = True):
+def fs_getmd5(path: PathLike, recalculate: bool = False, followlinks: bool = False):
     """
     Calculate the md5 value of the file
 

megfile/fs_path.py

@@ -392,13 +392,15 @@ class FSPath(URIPath):
 
     def iterdir(self) -> Iterator["FSPath"]:
         """
-        Get all contents of given fs path.
-        The result is in ascending alphabetical order.
+        Get all contents of given fs path. The order of result is in arbitrary order.
 
-        :returns: All contents have in the path in ascending alphabetical order
+        :returns: All contents have in the path.
         """
-        for path in self.listdir():
-            yield self.joinpath(path)
+        self._check_int_path()
+        for path in pathlib.Path(
+            self.path_without_protocol  # pyre-ignore[6]
+        ).iterdir():
+            yield self.from_path(fspath(path))
 
     def load(self) -> BinaryIO:
         """Read all content on specified path and write into memory
@@ -568,7 +570,7 @@ class FSPath(URIPath):
                 "No match any file in: %r" % self.path_without_protocol
             )
 
-    def scandir(self) -> Iterator[FileEntry]:
+    def scandir(self) -> ContextIterator:
         """
         Get all content of given file path.
 
@@ -704,7 +706,7 @@ class FSPath(URIPath):
             )
         )
 
-    def md5(self, recalculate: bool = False, followlinks: bool = True):
+    def md5(self, recalculate: bool = False, followlinks: bool = False):
         """
         Calculate the md5 value of the file
 
@@ -713,11 +715,11 @@ class FSPath(URIPath):
 
         returns: md5 of file
         """
-        if os.path.isdir(self.path_without_protocol):
+        if self.is_dir():
             hash_md5 = hashlib.md5()  # nosec
             for file_name in self.listdir():
                 chunk = (
-                    FSPath(self.path_without_protocol, file_name)
+                    self.joinpath(file_name)
                     .md5(recalculate=recalculate, followlinks=followlinks)
                     .encode()
                 )
@@ -794,8 +796,13 @@ class FSPath(URIPath):
         except FileNotFoundError as error:
             # Prevent the dst_path directory from being created when src_path does not
             # exist
-            if dst_path == error.filename:
-                FSPath(os.path.dirname(dst_path)).mkdir(parents=True, exist_ok=True)
+            dst_parent_dir = os.path.dirname(dst_path)
+            if (
+                dst_parent_dir
+                and dst_parent_dir != "."
+                and error.filename in (dst_path, dst_parent_dir)
+            ):
+                self.from_path(dst_parent_dir).mkdir(parents=True, exist_ok=True)
                 self._copyfile(dst_path, callback=callback, followlinks=followlinks)
             else:
                 raise

megfile/hdfs.py

@@ -125,7 +125,7 @@ def hdfs_isfile(path: PathLike, followlinks: bool = False) -> bool:
     return HdfsPath(path).is_file(followlinks)
 
 
-def hdfs_listdir(path: PathLike, followlinks: bool = False) -> List[str]:
+def hdfs_listdir(path: PathLike) -> List[str]:
     """
     Get all contents of given path.
 
@@ -133,10 +133,10 @@ def hdfs_listdir(path: PathLike, followlinks: bool = False) -> List[str]:
     :returns: All contents have prefix of path.
     :raises: FileNotFoundError, NotADirectoryError
     """
-    return HdfsPath(path).listdir(followlinks)
+    return HdfsPath(path).listdir()
 
 
-def hdfs_load_from(path: PathLike, followlinks: bool = False) -> BinaryIO:
+def hdfs_load_from(path: PathLike) -> BinaryIO:
     """Read all content in binary on specified path and write into memory
 
     User should close the BinaryIO manually
@@ -144,7 +144,7 @@ def hdfs_load_from(path: PathLike, followlinks: bool = False) -> BinaryIO:
     :param path: Given path
     :returns: BinaryIO
     """
-    return HdfsPath(path).load(followlinks)
+    return HdfsPath(path).load()
 
 
 def hdfs_move(src_path: PathLike, dst_path: PathLike, overwrite: bool = True) -> None:
@@ -209,15 +209,15 @@ def hdfs_scan_stat(
     return HdfsPath(path).scan_stat(missing_ok, followlinks)
 
 
-def hdfs_scandir(path: PathLike, followlinks: bool = False) -> Iterator[FileEntry]:
+def hdfs_scandir(path: PathLike) -> Iterator[FileEntry]:
     """
-    Get all contents of given path, the order of result is not guaranteed.
+    Get all contents of given path, the order of result is in arbitrary order.
 
     :param path: Given path
     :returns: All contents have prefix of path
     :raises: FileNotFoundError, NotADirectoryError
     """
-    return HdfsPath(path).scandir(followlinks)
+    return HdfsPath(path).scandir()
 
 
 def hdfs_unlink(path: PathLike, missing_ok: bool = False) -> None:

megfile/hdfs_path.py

@@ -12,7 +12,7 @@ from megfile.config import (
     READER_MAX_BUFFER_SIZE,
 )
 from megfile.errors import _create_missing_ok_generator, raise_hdfs_error
-from megfile.interfaces import FileEntry, PathLike, StatResult, URIPath
+from megfile.interfaces import ContextIterator, FileEntry, PathLike, StatResult, URIPath
 from megfile.lib.compat import fspath
 from megfile.lib.glob import FSFunc, iglob
 from megfile.lib.hdfs_prefetch_reader import HdfsPrefetchReader
@@ -299,7 +299,12 @@ class HdfsPath(URIPath):
             Because hdfs symlink not support dir.
         :returns: True if path is hdfs directory, else False
         """
-        return self.stat().is_dir()
+        try:
+            stat = self.stat(follow_symlinks=followlinks)
+            return stat.is_dir()
+        except FileNotFoundError:
+            pass
+        return False
 
     def is_file(self, followlinks: bool = False) -> bool:
         """
@@ -307,9 +312,14 @@ class HdfsPath(URIPath):
 
         :returns: True if path is hdfs file, else False
         """
-        return self.stat().is_file()
+        try:
+            stat = self.stat(follow_symlinks=followlinks)
+            return stat.is_file()
+        except FileNotFoundError:
+            pass
+        return False
 
-    def listdir(self, followlinks: bool = False) -> List[str]:
+    def listdir(self) -> List[str]:
         """
         Get all contents of given path.
 
@@ -319,19 +329,19 @@ class HdfsPath(URIPath):
         if not self.is_dir():
             raise NotADirectoryError("Not a directory: %r" % self.path)
         with raise_hdfs_error(self.path_with_protocol):
-            return self._client.list(self.path_without_protocol)
+            return sorted(self._client.list(self.path_without_protocol))
 
-    def iterdir(self, followlinks: bool = False) -> Iterator["HdfsPath"]:
+    def iterdir(self) -> Iterator["HdfsPath"]:
         """
         Get all contents of given path.
 
         :returns: All contents have prefix of path.
         :raises: FileNotFoundError, NotADirectoryError
         """
-        for filename in self.listdir(followlinks=followlinks):
+        for filename in self.listdir():
             yield self.joinpath(filename)
 
-    def load(self, followlinks: bool = False) -> BinaryIO:
+    def load(self) -> BinaryIO:
         """Read all content in binary on specified path and write into memory
 
         User should close the BinaryIO manually
@@ -466,28 +476,32 @@ class HdfsPath(URIPath):
                         ),
                     )
 
-    def scandir(self, followlinks: bool = False) -> Iterator[FileEntry]:
+    def scandir(self) -> ContextIterator:
         """
-        Get all contents of given path, the order of result is not guaranteed.
+        Get all contents of given path, the order of result is in arbitrary order.
 
         :returns: All contents have prefix of path
         :raises: FileNotFoundError, NotADirectoryError
         """
-        with raise_hdfs_error(self.path_with_protocol):
-            for filename, stat_data in self._client.list(
-                self.path_without_protocol, status=True
-            ):
-                yield FileEntry(
-                    name=filename,
-                    path=self.joinpath(filename).path_with_protocol,
-                    stat=StatResult(
-                        size=stat_data["length"],
-                        mtime=stat_data["modificationTime"] / 1000,
-                        isdir=stat_data["type"] == "DIRECTORY",
-                        islnk=False,
-                        extra=stat_data,
-                    ),
-                )
+
+        def create_generator():
+            with raise_hdfs_error(self.path_with_protocol):
+                for filename, stat_data in self._client.list(
+                    self.path_without_protocol, status=True
+                ):
+                    yield FileEntry(
+                        name=filename,
+                        path=self.joinpath(filename).path_with_protocol,
+                        stat=StatResult(
+                            size=stat_data["length"],
+                            mtime=stat_data["modificationTime"] / 1000,
+                            isdir=stat_data["type"] == "DIRECTORY",
+                            islnk=False,
+                            extra=stat_data,
+                        ),
+                    )
+
+        return ContextIterator(create_generator())
 
     def unlink(self, missing_ok: bool = False) -> None:
         """

megfile/lib/base_prefetch_reader.py

@@ -5,7 +5,6 @@ from concurrent.futures import Future, ThreadPoolExecutor
 from io import BytesIO
 from logging import getLogger as get_logger
 from math import ceil
-from statistics import mean
 from typing import Optional
 
 from megfile.config import (
@@ -92,7 +91,7 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
 
     @abstractmethod
     def _get_content_size(self):
-        pass
+        pass  # pragma: no cover
 
     @property
     def _futures(self) -> "LRUCacheFutureManager":
@@ -104,7 +103,7 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
     @property
     @abstractmethod
     def name(self) -> str:
-        pass
+        pass  # pragma: no cover
 
     @property
     def mode(self) -> str:
@@ -238,13 +237,7 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
 
         if self._block_forward == 0:
             block_index = self._offset // self._block_size
-            if len(self._seek_history) > 0:
-                mean_read_count = mean(item.read_count for item in self._seek_history)
-            else:
-                mean_read_count = 0
-            if block_index not in self._futures and mean_read_count < 3:
-                # No using LRP will be better if read() are always called less than 3
-                # times after seek()
+            if block_index not in self._futures:
                 buffer[:size] = self._read(size)
                 return size
 
@@ -329,8 +322,9 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
                 history.append(item)
             history.append(SeekRecord(index))
             self._seek_history = history
-            self._block_forward = max(
-                self._block_capacity // len(self._seek_history), 0
+            self._block_forward = min(
+                max(self._block_capacity // len(self._seek_history), 0),
+                self._block_capacity - 1,
             )
             if self._block_forward == 0:
                 self._is_auto_scaling = False
@@ -343,7 +337,7 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
     def _fetch_response(
         self, start: Optional[int] = None, end: Optional[int] = None
     ) -> dict:
-        pass
+        pass  # pragma: no cover
 
     def _fetch_buffer(self, index: int) -> BytesIO:
         start, end = index * self._block_size, (index + 1) * self._block_size - 1

megfile/lib/combine_reader.py

@@ -36,7 +36,7 @@ class CombineReader(Readable, Seekable):
         for index, size in enumerate(self._blocks_sizes):
             if self._offset < size:
                 return index - 1, self._offset - self._blocks_sizes[index - 1]
-        raise IOError("offset out of range: %d" % self._offset)
+        raise IOError("offset out of range: %d" % self._offset)  # pragma: no cover
 
     @property
     def name(self) -> str:

megfile/lib/glob.py

@@ -5,22 +5,15 @@
 import os
 import re
 from collections import OrderedDict
-from collections import namedtuple as NamedTuple
-from typing import Iterator, List, Tuple
+from typing import Callable, Iterator, List, NamedTuple, Tuple
 
 from megfile.lib import fnmatch
 
-# Python 3.5+ Compatible
-"""
+
 class FSFunc(NamedTuple):
     exists: Callable[[str], bool]
     isdir: Callable[[str], bool]
-    scandir: Callable[[str], Iterator[Tuple[str, bool]]]  # name, isdir
-
-in Python 3.6+
-"""
-
-FSFunc = NamedTuple("FSFunc", ["exists", "isdir", "scandir"])
+    scandir: Callable[[str], Iterator[Tuple[str, bool]]]
 
 
 def _exists(path: str) -> bool:
@@ -72,7 +65,7 @@ def iglob(
     if recursive and _isrecursive(pathname):
         s = next(it)  # skip empty string
         if s:
-            raise OSError("iglob with recursive=True error")
+            raise OSError("iglob with recursive=True error")  # pragma: no cover
     return it
 
 
@@ -161,11 +154,8 @@ def _iterdir(dirname: str, dironly: bool, fs: FSFunc) -> Iterator[str]:
     try:
         # dirname may be non-existent, raise OSError
         for name, isdir in fs.scandir(dirname):
-            try:
-                if not dironly or isdir:
-                    yield name
-            except OSError:
-                pass
+            if not dironly or isdir:
+                yield name
     except OSError:
         return
 

megfile/lib/s3_cached_handler.py

@@ -18,15 +18,15 @@ class S3CachedHandler(S3MemoryHandler):
         remove_cache_when_open: bool = True,
         profile_name: Optional[str] = None,
     ):
-        if mode not in ("rb", "wb", "ab", "rb+", "wb+", "ab+"):
-            raise ValueError("unacceptable mode: %r" % mode)
-
         self._bucket = bucket
         self._key = key
         self._mode = mode
         self._client = s3_client
         self._profile_name = profile_name
 
+        if mode not in ("rb", "wb", "ab", "rb+", "wb+", "ab+"):
+            raise ValueError("unacceptable mode: %r" % mode)
+
         if cache_path is None:
             cache_path = generate_cache_path(self.name)
 

megfile/lib/s3_limited_seekable_writer.py

@@ -137,7 +137,7 @@ class S3LimitedSeekableWriter(S3BufferedWriter, Seekable):
     def _submit_futures(self):
         content = self._buffer.getvalue()
         if len(content) == 0:
-            return
+            return  # pragma: no cover
         offset = len(content) - self._tail_block_size
         self._buffer = BytesIO(content[offset:])
         self._buffer.seek(0, os.SEEK_END)

megfile/lib/s3_memory_handler.py

@@ -21,15 +21,15 @@ class S3MemoryHandler(Readable[bytes], Seekable, Writable[bytes]):
         s3_client,
         profile_name: Optional[str] = None,
     ):
-        if mode not in ("rb", "wb", "ab", "rb+", "wb+", "ab+"):
-            raise ValueError("unacceptable mode: %r" % mode)
-
         self._bucket = bucket
         self._key = key
         self._mode = mode
         self._client = s3_client
         self._profile_name = profile_name
 
+        if mode not in ("rb", "wb", "ab", "rb+", "wb+", "ab+"):
+            raise ValueError("unacceptable mode: %r" % mode)
+
         self._fileobj = BytesIO()
         self._download_fileobj()
 

megfile/lib/s3_pipe_handler.py

@@ -34,9 +34,6 @@ class S3PipeHandler(Readable[bytes], Writable[bytes]):
         join_thread: bool = True,
         profile_name: Optional[str] = None,
     ):
-        if mode not in ("rb", "wb"):
-            raise ValueError("unacceptable mode: %r" % mode)
-
         self._bucket = bucket
         self._key = key
         self._mode = mode
@@ -45,6 +42,9 @@ class S3PipeHandler(Readable[bytes], Writable[bytes]):
         self._offset = 0
         self._profile_name = profile_name
 
+        if mode not in ("rb", "wb"):
+            raise ValueError("unacceptable mode: %r" % mode)
+
         self._exc = None
         self._pipe = os.pipe()
         _s3_opened_pipes.append(self._pipe)
@@ -76,7 +76,7 @@ class S3PipeHandler(Readable[bytes], Writable[bytes]):
         try:
             with os.fdopen(self._pipe[1], "wb") as buffer:
                 self._client.download_fileobj(self._bucket, self._key, buffer)
-        except BrokenPipeError:
+        except BrokenPipeError:  # pragma: no cover
             if self._fileobj.closed:
                 return
             raise

megfile/pathlike.py

@@ -412,7 +412,7 @@ class BasePath:
         raw_suffix = self.suffix
         return self.from_path(path[: len(path) - len(raw_suffix)] + suffix)
 
-    def relpath(self, start=None):
+    def relpath(self, start: Optional[str] = None):
         """Return the relative path."""
         if start is None:
             raise TypeError("start is required")
@@ -485,7 +485,7 @@ class BasePath:
         """Return the canonical path of the path."""
         return self.path_with_protocol
 
-    def resolve(self):
+    def resolve(self, strict=False):
         """Alias of realpath."""
         return self.path_with_protocol
 
@@ -615,18 +615,18 @@ class BasePath:
         """Return the names of the entries in the directory the path points to."""
         raise NotImplementedError('method "listdir" not implemented: %r' % self)
 
-    def scandir(self) -> Iterator[FileEntry]:
+    def scandir(self):
         """
         Return an iterator of FileEntry objects corresponding to the entries
         in the directory.
         """
         raise NotImplementedError('method "scandir" not implemented: %r' % self)
 
-    def getsize(self, follow_symlinks: bool = True) -> int:
+    def getsize(self, follow_symlinks: bool = False) -> int:
         """Return the size, in bytes."""
         raise NotImplementedError('method "getsize" not implemented: %r' % self)
 
-    def getmtime(self, follow_symlinks: bool = True) -> float:
+    def getmtime(self, follow_symlinks: bool = False) -> float:
         """Return the time of last modification."""
         raise NotImplementedError('method "getmtime" not implemented: %r' % self)
 

megfile/s3.py

@@ -73,9 +73,7 @@ __all__ = [
 ]
 
 
-def s3_access(
-    path: PathLike, mode: Access = Access.READ, followlinks: bool = False
-) -> bool:
+def s3_access(path: PathLike, mode: Access = Access.READ) -> bool:
     """
     Test if path has access permission described by mode
 
@@ -83,7 +81,7 @@ def s3_access(
     :param mode: access mode
     :returns: bool, if the bucket of s3_url has read/write access.
     """
-    return S3Path(path).access(mode, followlinks)
+    return S3Path(path).access(mode)
 
 
 def s3_exists(path: PathLike, followlinks: bool = False) -> bool:
@@ -161,9 +159,7 @@ def s3_isfile(path: PathLike, followlinks: bool = False) -> bool:
     return S3Path(path).is_file(followlinks)
 
 
-def s3_listdir(
-    path: PathLike, followlinks: bool = False, missing_ok: bool = True
-) -> List[str]:
+def s3_listdir(path: PathLike) -> List[str]:
     """
     Get all contents of given s3_url. The result is in ascending alphabetical order.
 
@@ -171,10 +167,10 @@ def s3_listdir(
     :returns: All contents have prefix of s3_url in ascending alphabetical order
     :raises: S3FileNotFoundError, S3NotADirectoryError
     """
-    return S3Path(path).listdir(followlinks, missing_ok)
+    return S3Path(path).listdir()
 
 
-def s3_load_from(path: PathLike, followlinks: bool = False) -> BinaryIO:
+def s3_load_from(path: PathLike) -> BinaryIO:
     """Read all content in binary on specified path and write into memory
 
     User should close the BinaryIO manually
@@ -182,7 +178,7 @@ def s3_load_from(path: PathLike, followlinks: bool = False) -> BinaryIO:
     :param path: Given path
     :returns: BinaryIO
     """
-    return S3Path(path).load(followlinks)
+    return S3Path(path).load()
 
 
 def s3_hasbucket(path: PathLike) -> bool:
@@ -262,17 +258,15 @@ def s3_scan_stat(
     return S3Path(path).scan_stat(missing_ok, followlinks)
 
 
-def s3_scandir(
-    path: PathLike, followlinks: bool = False, missing_ok: bool = True
-) -> Iterator[FileEntry]:
+def s3_scandir(path: PathLike) -> Iterator[FileEntry]:
     """
-    Get all contents of given s3_url, the order of result is not guaranteed.
+    Get all contents of given s3_url, the order of result is in arbitrary order.
 
     :param path: Given path
     :returns: All contents have prefix of s3_url
     :raises: S3FileNotFoundError, S3NotADirectoryError
     """
-    return S3Path(path).scandir(followlinks, missing_ok)
+    return S3Path(path).scandir()
 
 
 def s3_stat(path: PathLike, follow_symlinks=True) -> StatResult:
@@ -457,7 +451,6 @@ def s3_glob(
     path: PathLike,
     recursive: bool = True,
     missing_ok: bool = True,
-    followlinks: bool = False,
 ) -> List[str]:
     """Return s3 path list in ascending alphabetical order,
     in which path matches glob pattern
@@ -476,7 +469,6 @@ def s3_glob(
             path=path,
             recursive=recursive,
             missing_ok=missing_ok,
-            followlinks=followlinks,
         )
     )
 
@@ -485,7 +477,6 @@ def s3_glob_stat(
     path: PathLike,
     recursive: bool = True,
     missing_ok: bool = True,
-    followlinks: bool = False,
 ) -> Iterator[FileEntry]:
     """Return a generator contains tuples of path and file stat,
     in ascending alphabetical order, in which path matches glob pattern
@@ -501,7 +492,7 @@ def s3_glob_stat(
         in which paths match `s3_pathname`
     """
     return S3Path(path).glob_stat(
-        pattern="", recursive=recursive, missing_ok=missing_ok, followlinks=followlinks
+        pattern="", recursive=recursive, missing_ok=missing_ok
     )
 
 
@@ -509,7 +500,6 @@ def s3_iglob(
     path: PathLike,
     recursive: bool = True,
     missing_ok: bool = True,
-    followlinks: bool = False,
 ) -> Iterator[str]:
     """Return s3 path iterator in ascending alphabetical order,
     in which path matches glob pattern
@@ -524,7 +514,7 @@ def s3_iglob(
     :returns: An iterator contains paths match `s3_pathname`
     """
     for path_obj in S3Path(path).iglob(
-        pattern="", recursive=recursive, missing_ok=missing_ok, followlinks=followlinks
+        pattern="", recursive=recursive, missing_ok=missing_ok
     ):
         yield path_obj.path_with_protocol