megfile 4.0.3__py3-none-any.whl → 4.1.0__py3-none-any.whl

megfile/cli.py

@@ -46,6 +46,7 @@ from megfile.version import VERSION
 
 options = {}
 set_log_level()
+max_file_object_catch_count = 1024 * 128
 
 
 @click.group()
@@ -124,12 +125,15 @@ def _sftp_prompt_host_key(path):
         hostname = (
             path.pathlike._urlsplit_parts.hostname  # pytype: disable=attribute-error
         )
-        port = path.pathlike._urlsplit_parts.port  # pytype: disable=attribute-error
-        sftp_add_host_key(
-            hostname=hostname,
-            port=port,
-            prompt=True,
+        port = (
+            path.pathlike._urlsplit_parts.port or 22  # pytype: disable=attribute-error
         )
+        if hostname:
+            sftp_add_host_key(
+                hostname=hostname,
+                port=port,
+                prompt=True,
+            )
 
 
 def _ls(path: str, long: bool, recursive: bool, human_readable: bool):
@@ -161,11 +165,7 @@ def _ls(path: str, long: bool, recursive: bool, human_readable: bool):
         total_count += 1
         output = echo_func(file_stat, base_path, full_path=full_path)
         if file_stat.is_symlink():
-            try:
-                link = smart_readlink(file_stat.path)
-            except FileNotFoundError as e:
-                link = repr(e)
-            output += " -> %s" % link
+            output += " -> %s" % smart_readlink(file_stat.path)
         click.echo(output)
     if long:
         click.echo(f"total({total_count}): {get_human_size(total_size)}")
@@ -417,7 +417,7 @@ def sync(
             file_entries = []
             total_count = total_size = 0
             for total_count, file_entry in enumerate(scan_func(src_path), start=1):
-                if total_count > 1024 * 128:
+                if total_count > max_file_object_catch_count:
                     file_entries = []
                 else:
                     file_entries.append(file_entry)
@@ -506,15 +506,21 @@ def head(path: str, lines: int):
 
     with smart_open(path, "rb") as f:
         for _ in range(lines):
-            try:
-                content = f.readline()
-                if not content:
-                    break
-            except EOFError:
+            content = f.readline()
+            if not content:
                 break
             click.echo(content.strip(b"\n"))
 
 
+def _tail_follow_content(path, offset):
+    with smart_open(path, "rb") as f:
+        f.seek(offset)
+        for line in f.readlines():
+            click.echo(line, nl=False)
+        offset = f.tell()
+    return offset
+
+
 @cli.command(
     short_help="Concatenate any files and send last n lines of them to stdout."
 )
@@ -554,17 +560,14 @@ def tail(path: str, lines: int, follow: bool):
     if line_list:
         click.echo(line_list[-1], nl=False)
 
-    if follow:
+    if follow:  # pragma: no cover
         offset = file_size
         while True:
-            with smart_open(path, "rb") as f:
-                f.seek(offset)
-                line = f.readline()
-                offset = f.tell()
-                if not line:
-                    time.sleep(1)
-                    continue
-                click.echo(line, nl=False)
+            new_offset = _tail_follow_content(path, offset)
+            if new_offset == offset:
+                time.sleep(1)
+            else:
+                offset = new_offset
 
 
 @cli.command(short_help="Write bytes from stdin to file.")

megfile/config.py

@@ -46,18 +46,13 @@ def parse_quantity(quantity: T.Union[str, int]) -> int:
 
     if suffix.endswith("i"):
         base = 1024
-    elif len(suffix) == 1:
-        base = 1000
     else:
-        raise ValueError("{} has unknown suffix".format(quantity))
+        base = 1000
 
     # handle SI inconsistency
     if suffix == "ki":
         raise ValueError("{} has unknown suffix".format(quantity))
 
-    if suffix[0] not in exponents:
-        raise ValueError("{} has unknown suffix".format(quantity))
-
     exponent = int(exponents[suffix[0]])
     return number * (base**exponent)  # pytype: disable=bad-return-type
 

megfile/errors.py

@@ -177,17 +177,19 @@ def patch_method(
 
 def _create_missing_ok_generator(generator, missing_ok: bool, error: Exception):
     if missing_ok:
-        yield from generator
-        return
-
-    zero_elem = True
-    for item in generator:
-        zero_elem = False
-        yield item
+        return generator
 
-    if zero_elem:
+    try:
+        first = next(generator)
+    except StopIteration:
         raise error
 
+    def create_generator():
+        yield first
+        yield from generator
+
+    return create_generator()
+
 
 class UnknownError(Exception):
     def __init__(self, error: Exception, path: PathLike, extra: Optional[str] = None):
@@ -411,11 +413,14 @@ def translate_http_error(http_error: Exception, http_url: str) -> Exception:
 
 
 @contextmanager
-def raise_s3_error(s3_url: PathLike):
+def raise_s3_error(s3_url: PathLike, suppress_error_callback=None):
     try:
         yield
     except Exception as error:
-        raise translate_s3_error(error, s3_url)
+        error = translate_s3_error(error, s3_url)
+        if suppress_error_callback and suppress_error_callback(error):
+            return
+        raise error
 
 
 def s3_error_code_should_retry(error: str) -> bool:

megfile/fs.py

@@ -317,7 +317,7 @@ def fs_walk(
     return FSPath(path).walk(followlinks)
 
 
-def fs_getmd5(path: PathLike, recalculate: bool = False, followlinks: bool = True):
+def fs_getmd5(path: PathLike, recalculate: bool = False, followlinks: bool = False):
     """
     Calculate the md5 value of the file
 

megfile/fs_path.py

@@ -392,13 +392,15 @@ class FSPath(URIPath):
 
     def iterdir(self) -> Iterator["FSPath"]:
         """
-        Get all contents of given fs path.
-        The result is in ascending alphabetical order.
+        Get all contents of given fs path. The order of result is in arbitrary order.
 
-        :returns: All contents have in the path in ascending alphabetical order
+        :returns: All contents have in the path.
         """
-        for path in self.listdir():
-            yield self.joinpath(path)
+        self._check_int_path()
+        for path in pathlib.Path(
+            self.path_without_protocol  # pyre-ignore[6]
+        ).iterdir():
+            yield self.from_path(fspath(path))
 
     def load(self) -> BinaryIO:
         """Read all content on specified path and write into memory
@@ -469,6 +471,8 @@ class FSPath(URIPath):
         src_path, dst_path = fspath(self.path_without_protocol), fspath(dst_path)
         if os.path.isfile(src_path):
             _fs_rename_file(src_path, dst_path, overwrite)
+            if os.path.exists(src_path):
+                os.remove(src_path)
             return self.from_path(dst_path)
         else:
             os.makedirs(dst_path, exist_ok=True)
@@ -485,10 +489,7 @@ class FSPath(URIPath):
                 else:
                     _fs_rename_file(src_file_path, dst_file_path, overwrite)
 
-            if os.path.isdir(src_path):
-                shutil.rmtree(src_path)
-            else:
-                os.remove(src_path)
+            shutil.rmtree(src_path, ignore_errors=True)
 
         return self.from_path(dst_path)
 
@@ -518,6 +519,8 @@ class FSPath(URIPath):
     def _scan(
         self, missing_ok: bool = True, followlinks: bool = False
     ) -> Iterator[str]:
+        self._check_int_path()
+
         if self.is_file(followlinks=followlinks):
             path = fspath(self.path_without_protocol)
             yield path
@@ -567,12 +570,13 @@ class FSPath(URIPath):
                 "No match any file in: %r" % self.path_without_protocol
             )
 
-    def scandir(self) -> Iterator[FileEntry]:
+    def scandir(self) -> ContextIterator:
         """
         Get all content of given file path.
 
         :returns: An iterator contains all contents have prefix path
         """
+        self._check_int_path()
 
         def create_generator():
             with os.scandir(self.path_without_protocol) as entries:
@@ -702,7 +706,7 @@ class FSPath(URIPath):
             )
         )
 
-    def md5(self, recalculate: bool = False, followlinks: bool = True):
+    def md5(self, recalculate: bool = False, followlinks: bool = False):
         """
         Calculate the md5 value of the file
 
@@ -711,11 +715,11 @@ class FSPath(URIPath):
 
         returns: md5 of file
         """
-        if os.path.isdir(self.path_without_protocol):
+        if self.is_dir():
             hash_md5 = hashlib.md5()  # nosec
             for file_name in self.listdir():
                 chunk = (
-                    FSPath(self.path_without_protocol, file_name)
+                    self.joinpath(file_name)
                     .md5(recalculate=recalculate, followlinks=followlinks)
                     .encode()
                 )
@@ -740,9 +744,8 @@ class FSPath(URIPath):
                     if not buf:
                         break
                     fdst.write(buf)
-                    if callback is None:
-                        continue
-                    callback(len(buf))
+                    if callback:
+                        callback(len(buf))
         else:
             shutil.copy2(
                 self.path_without_protocol,  # pyre-ignore[6]
@@ -793,8 +796,13 @@ class FSPath(URIPath):
         except FileNotFoundError as error:
             # Prevent the dst_path directory from being created when src_path does not
             # exist
-            if dst_path == error.filename:
-                FSPath(os.path.dirname(dst_path)).mkdir(parents=True, exist_ok=True)
+            dst_parent_dir = os.path.dirname(dst_path)
+            if (
+                dst_parent_dir
+                and dst_parent_dir != "."
+                and error.filename in (dst_path, dst_parent_dir)
+            ):
+                self.from_path(dst_parent_dir).mkdir(parents=True, exist_ok=True)
                 self._copyfile(dst_path, callback=callback, followlinks=followlinks)
             else:
                 raise
@@ -814,15 +822,15 @@ class FSPath(URIPath):
             priority is higher than 'overwrite', default is False
         :param overwrite: whether or not overwrite file when exists, default is True
         """
+        self._check_int_path()
+
         if self.is_dir(followlinks=followlinks):
 
             def ignore_same_file(src: str, names: List[str]) -> List[str]:
                 ignore_files = []
                 for name in names:
                     dst_obj = self.from_path(dst_path).joinpath(name)
-                    if force:
-                        pass
-                    elif not overwrite and dst_obj.exists():
+                    if not overwrite and dst_obj.exists():
                         ignore_files.append(name)
                     elif dst_obj.exists() and is_same_file(
                         self.joinpath(name).stat(), dst_obj.stat(), "copy"
@@ -833,11 +841,11 @@ class FSPath(URIPath):
             shutil.copytree(
                 self.path_without_protocol,  # pyre-ignore[6]
                 dst_path,
-                ignore=ignore_same_file,
+                ignore=None if force else ignore_same_file,
                 dirs_exist_ok=True,
             )
         else:
-            self.copy(dst_path, followlinks=followlinks, overwrite=overwrite)
+            self.copy(dst_path, followlinks=followlinks, overwrite=force or overwrite)
 
     def symlink(self, dst_path: PathLike) -> None:
         """
@@ -893,6 +901,8 @@ class FSPath(URIPath):
         return self.from_path(os.path.expanduser("~"))
 
     def joinpath(self, *other_paths: PathLike) -> "FSPath":
+        self._check_int_path()
+
         path = fspath(self)
         if path == ".":
             path = ""

megfile/hdfs.py

@@ -125,7 +125,7 @@ def hdfs_isfile(path: PathLike, followlinks: bool = False) -> bool:
     return HdfsPath(path).is_file(followlinks)
 
 
-def hdfs_listdir(path: PathLike, followlinks: bool = False) -> List[str]:
+def hdfs_listdir(path: PathLike) -> List[str]:
     """
     Get all contents of given path.
 
@@ -133,10 +133,10 @@ def hdfs_listdir(path: PathLike, followlinks: bool = False) -> List[str]:
     :returns: All contents have prefix of path.
     :raises: FileNotFoundError, NotADirectoryError
     """
-    return HdfsPath(path).listdir(followlinks)
+    return HdfsPath(path).listdir()
 
 
-def hdfs_load_from(path: PathLike, followlinks: bool = False) -> BinaryIO:
+def hdfs_load_from(path: PathLike) -> BinaryIO:
     """Read all content in binary on specified path and write into memory
 
     User should close the BinaryIO manually
@@ -144,7 +144,7 @@ def hdfs_load_from(path: PathLike, followlinks: bool = False) -> BinaryIO:
     :param path: Given path
     :returns: BinaryIO
     """
-    return HdfsPath(path).load(followlinks)
+    return HdfsPath(path).load()
 
 
 def hdfs_move(src_path: PathLike, dst_path: PathLike, overwrite: bool = True) -> None:
@@ -209,15 +209,15 @@ def hdfs_scan_stat(
     return HdfsPath(path).scan_stat(missing_ok, followlinks)
 
 
-def hdfs_scandir(path: PathLike, followlinks: bool = False) -> Iterator[FileEntry]:
+def hdfs_scandir(path: PathLike) -> Iterator[FileEntry]:
     """
-    Get all contents of given path, the order of result is not guaranteed.
+    Get all contents of given path, the order of result is in arbitrary order.
 
     :param path: Given path
     :returns: All contents have prefix of path
     :raises: FileNotFoundError, NotADirectoryError
     """
-    return HdfsPath(path).scandir(followlinks)
+    return HdfsPath(path).scandir()
 
 
 def hdfs_unlink(path: PathLike, missing_ok: bool = False) -> None:

megfile/hdfs_path.py

@@ -12,7 +12,7 @@ from megfile.config import (
     READER_MAX_BUFFER_SIZE,
 )
 from megfile.errors import _create_missing_ok_generator, raise_hdfs_error
-from megfile.interfaces import FileEntry, PathLike, StatResult, URIPath
+from megfile.interfaces import ContextIterator, FileEntry, PathLike, StatResult, URIPath
 from megfile.lib.compat import fspath
 from megfile.lib.glob import FSFunc, iglob
 from megfile.lib.hdfs_prefetch_reader import HdfsPrefetchReader
@@ -299,7 +299,12 @@ class HdfsPath(URIPath):
             Because hdfs symlink not support dir.
         :returns: True if path is hdfs directory, else False
         """
-        return self.stat().is_dir()
+        try:
+            stat = self.stat(follow_symlinks=followlinks)
+            return stat.is_dir()
+        except FileNotFoundError:
+            pass
+        return False
 
     def is_file(self, followlinks: bool = False) -> bool:
         """
@@ -307,9 +312,14 @@ class HdfsPath(URIPath):
 
         :returns: True if path is hdfs file, else False
         """
-        return self.stat().is_file()
+        try:
+            stat = self.stat(follow_symlinks=followlinks)
+            return stat.is_file()
+        except FileNotFoundError:
+            pass
+        return False
 
-    def listdir(self, followlinks: bool = False) -> List[str]:
+    def listdir(self) -> List[str]:
         """
         Get all contents of given path.
 
@@ -319,19 +329,19 @@ class HdfsPath(URIPath):
         if not self.is_dir():
             raise NotADirectoryError("Not a directory: %r" % self.path)
         with raise_hdfs_error(self.path_with_protocol):
-            return self._client.list(self.path_without_protocol)
+            return sorted(self._client.list(self.path_without_protocol))
 
-    def iterdir(self, followlinks: bool = False) -> Iterator["HdfsPath"]:
+    def iterdir(self) -> Iterator["HdfsPath"]:
         """
         Get all contents of given path.
 
         :returns: All contents have prefix of path.
         :raises: FileNotFoundError, NotADirectoryError
         """
-        for filename in self.listdir(followlinks=followlinks):
+        for filename in self.listdir():
             yield self.joinpath(filename)
 
-    def load(self, followlinks: bool = False) -> BinaryIO:
+    def load(self) -> BinaryIO:
         """Read all content in binary on specified path and write into memory
 
         User should close the BinaryIO manually
@@ -372,7 +382,10 @@ class HdfsPath(URIPath):
         dst_path = self.from_path(dst_path)
         if self.is_dir():
             for filename in self.iterdir():
-                self.joinpath(filename).rename(dst_path.joinpath(filename))
+                filename.rename(
+                    dst_path.joinpath(filename.relative_to(self.path_with_protocol)),
+                    overwrite=overwrite,
+                )
         else:
             if overwrite:
                 dst_path.remove(missing_ok=True)
@@ -463,28 +476,32 @@ class HdfsPath(URIPath):
                         ),
                     )
 
-    def scandir(self, followlinks: bool = False) -> Iterator[FileEntry]:
+    def scandir(self) -> ContextIterator:
         """
-        Get all contents of given path, the order of result is not guaranteed.
+        Get all contents of given path, the order of result is in arbitrary order.
 
         :returns: All contents have prefix of path
         :raises: FileNotFoundError, NotADirectoryError
         """
-        with raise_hdfs_error(self.path_with_protocol):
-            for filename, stat_data in self._client.list(
-                self.path_without_protocol, status=True
-            ):
-                yield FileEntry(
-                    name=filename,
-                    path=self.joinpath(filename).path_with_protocol,
-                    stat=StatResult(
-                        size=stat_data["length"],
-                        mtime=stat_data["modificationTime"] / 1000,
-                        isdir=stat_data["type"] == "DIRECTORY",
-                        islnk=False,
-                        extra=stat_data,
-                    ),
-                )
+
+        def create_generator():
+            with raise_hdfs_error(self.path_with_protocol):
+                for filename, stat_data in self._client.list(
+                    self.path_without_protocol, status=True
+                ):
+                    yield FileEntry(
+                        name=filename,
+                        path=self.joinpath(filename).path_with_protocol,
+                        stat=StatResult(
+                            size=stat_data["length"],
+                            mtime=stat_data["modificationTime"] / 1000,
+                            isdir=stat_data["type"] == "DIRECTORY",
+                            islnk=False,
+                            extra=stat_data,
+                        ),
+                    )
+
+        return ContextIterator(create_generator())
 
     def unlink(self, missing_ok: bool = False) -> None:
         """

megfile/http_path.py

@@ -96,14 +96,7 @@ def get_http_session(
                     file_info.seek(0)
                 elif isinstance(file_info, (tuple, list)) and len(file_info) >= 2:
                     file_info = list(file_info)
-                    if (
-                        isinstance(file_info[1], (tuple, list))
-                        and len(file_info[1]) >= 2
-                    ):
-                        file_info[1] = list(file_info[1])
-                        file_info[1] = seek_or_reopen(file_info[1])
-                    else:
-                        file_info[1] = seek_or_reopen(file_info[1])
+                    file_info[1] = seek_or_reopen(file_info[1])
                     files[key] = file_info
 
     session.request = patch_method(

megfile/lib/base_prefetch_reader.py

@@ -5,7 +5,6 @@ from concurrent.futures import Future, ThreadPoolExecutor
 from io import BytesIO
 from logging import getLogger as get_logger
 from math import ceil
-from statistics import mean
 from typing import Optional
 
 from megfile.config import (
@@ -92,7 +91,7 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
 
     @abstractmethod
     def _get_content_size(self):
-        pass
+        pass  # pragma: no cover
 
     @property
     def _futures(self) -> "LRUCacheFutureManager":
@@ -104,7 +103,7 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
     @property
     @abstractmethod
     def name(self) -> str:
-        pass
+        pass  # pragma: no cover
 
     @property
     def mode(self) -> str:
@@ -238,13 +237,7 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
 
         if self._block_forward == 0:
             block_index = self._offset // self._block_size
-            if len(self._seek_history) > 0:
-                mean_read_count = mean(item.read_count for item in self._seek_history)
-            else:
-                mean_read_count = 0
-            if block_index not in self._futures and mean_read_count < 3:
-                # No using LRP will be better if read() are always called less than 3
-                # times after seek()
+            if block_index not in self._futures:
                 buffer[:size] = self._read(size)
                 return size
 
@@ -329,8 +322,9 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
                 history.append(item)
             history.append(SeekRecord(index))
             self._seek_history = history
-            self._block_forward = max(
-                self._block_capacity // len(self._seek_history), 0
+            self._block_forward = min(
+                max(self._block_capacity // len(self._seek_history), 0),
+                self._block_capacity - 1,
             )
             if self._block_forward == 0:
                 self._is_auto_scaling = False
@@ -343,7 +337,7 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
     def _fetch_response(
         self, start: Optional[int] = None, end: Optional[int] = None
     ) -> dict:
-        pass
+        pass  # pragma: no cover
 
     def _fetch_buffer(self, index: int) -> BytesIO:
         start, end = index * self._block_size, (index + 1) * self._block_size - 1

megfile/lib/combine_reader.py

@@ -36,7 +36,7 @@ class CombineReader(Readable, Seekable):
         for index, size in enumerate(self._blocks_sizes):
             if self._offset < size:
                 return index - 1, self._offset - self._blocks_sizes[index - 1]
-        raise IOError("offset out of range: %d" % self._offset)
+        raise IOError("offset out of range: %d" % self._offset)  # pragma: no cover
 
     @property
     def name(self) -> str:

megfile/lib/glob.py

@@ -5,22 +5,15 @@
 import os
 import re
 from collections import OrderedDict
-from collections import namedtuple as NamedTuple
-from typing import Iterator, List, Tuple
+from typing import Callable, Iterator, List, NamedTuple, Tuple
 
 from megfile.lib import fnmatch
 
-# Python 3.5+ Compatible
-"""
+
 class FSFunc(NamedTuple):
     exists: Callable[[str], bool]
     isdir: Callable[[str], bool]
-    scandir: Callable[[str], Iterator[Tuple[str, bool]]]  # name, isdir
-
-in Python 3.6+
-"""
-
-FSFunc = NamedTuple("FSFunc", ["exists", "isdir", "scandir"])
+    scandir: Callable[[str], Iterator[Tuple[str, bool]]]
 
 
 def _exists(path: str) -> bool:
@@ -72,7 +65,7 @@ def iglob(
     if recursive and _isrecursive(pathname):
         s = next(it)  # skip empty string
         if s:
-            raise OSError("iglob with recursive=True error")
+            raise OSError("iglob with recursive=True error")  # pragma: no cover
     return it
 
 
@@ -161,11 +154,8 @@ def _iterdir(dirname: str, dironly: bool, fs: FSFunc) -> Iterator[str]:
     try:
         # dirname may be non-existent, raise OSError
         for name, isdir in fs.scandir(dirname):
-            try:
-                if not dironly or isdir:
-                    yield name
-            except OSError:
-                pass
+            if not dironly or isdir:
+                yield name
     except OSError:
         return
 

megfile/lib/s3_cached_handler.py

@@ -18,15 +18,15 @@ class S3CachedHandler(S3MemoryHandler):
         remove_cache_when_open: bool = True,
         profile_name: Optional[str] = None,
     ):
-        if mode not in ("rb", "wb", "ab", "rb+", "wb+", "ab+"):
-            raise ValueError("unacceptable mode: %r" % mode)
-
         self._bucket = bucket
         self._key = key
         self._mode = mode
         self._client = s3_client
         self._profile_name = profile_name
 
+        if mode not in ("rb", "wb", "ab", "rb+", "wb+", "ab+"):
+            raise ValueError("unacceptable mode: %r" % mode)
+
         if cache_path is None:
             cache_path = generate_cache_path(self.name)
 

megfile/lib/s3_limited_seekable_writer.py

@@ -137,7 +137,7 @@ class S3LimitedSeekableWriter(S3BufferedWriter, Seekable):
     def _submit_futures(self):
         content = self._buffer.getvalue()
         if len(content) == 0:
-            return
+            return  # pragma: no cover
         offset = len(content) - self._tail_block_size
         self._buffer = BytesIO(content[offset:])
         self._buffer.seek(0, os.SEEK_END)