megfile 5.0.1__py3-none-any.whl → 5.0.3__py3-none-any.whl

megfile/__init__.py

@@ -1,5 +1,5 @@
 import megfile.config  # noqa: F401  # make sure env config is loaded
-from megfile.fs_path import FSPath, is_fs
+from megfile.fs_path import FSPath, fs_copy, is_fs
 from megfile.hdfs_path import HdfsPath, is_hdfs
 from megfile.http_path import HttpPath, HttpsPath, is_http
 from megfile.s3_path import (
@@ -7,13 +7,26 @@ from megfile.s3_path import (
     is_s3,
     s3_buffered_open,
     s3_cached_open,
+    s3_concat,
+    s3_copy,
+    s3_download,
+    s3_load_content,
     s3_memory_open,
     s3_open,
     s3_pipe_open,
     s3_prefetch_open,
     s3_share_cache_open,
+    s3_upload,
+)
+from megfile.sftp_path import (
+    SftpPath,
+    is_sftp,
+    sftp_add_host_key,
+    sftp_concat,
+    sftp_copy,
+    sftp_download,
+    sftp_upload,
 )
-from megfile.sftp_path import SftpPath, is_sftp, sftp_add_host_key
 from megfile.smart import (
     smart_access,
     smart_cache,
@@ -73,9 +86,12 @@ except ImportError:
 __all__ = [
     "smart_access",
     "smart_cache",
+    "smart_cache",
     "smart_combine_open",
+    "smart_concat",
     "smart_copy",
     "smart_exists",
+    "smart_getmd5",
     "smart_getmtime",
     "smart_getsize",
     "smart_glob_stat",
@@ -86,55 +102,62 @@ __all__ = [
     "smart_islink",
     "smart_listdir",
     "smart_load_content",
-    "smart_save_content",
     "smart_load_from",
     "smart_load_text",
-    "smart_save_text",
+    "smart_lstat",
     "smart_makedirs",
+    "smart_move",
     "smart_open",
     "smart_path_join",
+    "smart_readlink",
     "smart_realpath",
     "smart_remove",
-    "smart_move",
     "smart_rename",
     "smart_save_as",
+    "smart_save_content",
+    "smart_save_text",
     "smart_scan_stat",
     "smart_scan",
     "smart_scandir",
     "smart_stat",
+    "smart_symlink",
     "smart_sync",
     "smart_touch",
     "smart_unlink",
     "smart_walk",
-    "smart_cache",
-    "smart_getmd5",
-    "smart_symlink",
-    "smart_readlink",
-    "smart_lstat",
-    "smart_concat",
+    "is_fs",
+    "fs_copy",
+    "is_hdfs",
+    "is_http",
     "is_s3",
     "s3_buffered_open",
     "s3_cached_open",
+    "s3_concat",
+    "s3_copy",
+    "s3_download",
+    "s3_load_content",
     "s3_memory_open",
     "s3_open",
     "s3_pipe_open",
     "s3_prefetch_open",
     "s3_share_cache_open",
-    "is_fs",
-    "is_http",
-    "is_stdio",
-    "stdio_open",
+    "s3_upload",
     "is_sftp",
     "sftp_add_host_key",
-    "is_hdfs",
+    "sftp_concat",
+    "sftp_copy",
+    "sftp_download",
+    "sftp_upload",
+    "is_stdio",
+    "stdio_open",
     "is_webdav",
-    "WebdavPath",
-    "S3Path",
     "FSPath",
+    "HdfsPath",
     "HttpPath",
     "HttpsPath",
-    "StdioPath",
-    "SmartPath",
+    "S3Path",
     "SftpPath",
-    "HdfsPath",
+    "SmartPath",
+    "StdioPath",
+    "WebdavPath",
 ]

megfile/cli.py

@@ -92,6 +92,8 @@ def safe_cli():  # pragma: no cover
 
 
 def get_echo_path(file_stat, base_path: str = "", full: bool = False):
+    if base_path.startswith("file://"):
+        base_path = base_path[7:]
     if base_path == file_stat.path:
         path = file_stat.name
     elif full:
@@ -150,6 +152,8 @@ def _sftp_prompt_host_key(path):
 
 def _ls(path: str, long: bool, full: bool, recursive: bool, human_readable: bool):
     base_path = path
+    if path == "file://":
+        path = "./"
     if has_magic(path):
         scan_func = smart_glob_stat
         base_path = get_non_glob_dir(path)
@@ -186,7 +190,7 @@ class PathType(ParamType):
     name = "path"
 
     def shell_complete(self, ctx, param, incomplete):
-        if "://" not in incomplete and not incomplete.startswith("/"):
+        if not incomplete:
             completions = [
                 CompletionItem(f"{protocol}://")
                 for protocol in SmartPath._registered_protocols
@@ -196,6 +200,15 @@ class PathType(ParamType):
                     continue
                 completions.append(CompletionItem(f"s3+{name}://"))
             return completions
+        if incomplete.startswith("file://"):
+            return [
+                CompletionItem(
+                    f"file://{entry.path}/"
+                    if entry.is_dir()
+                    else f"file://{entry.path}"
+                )
+                for entry in islice(smart_glob_stat(incomplete[7:] + "*"), 128)
+            ]
         try:
             return [
                 CompletionItem(f"{entry.path}/" if entry.is_dir() else entry.path)

megfile/errors.py

@@ -182,7 +182,7 @@ def patch_method(
                         f"Cannot handle error {full_error_message(error)} "
                         f"after {retries} tries"
                     )
-                    raise
+                    raise MaxRetriesExceededError(error, retries=retries)
                 retry_interval = min(0.1 * 2**retries, 30)
                 _logger.info(
                     f"unknown error encountered: {full_error_message(error)}, "
@@ -209,14 +209,34 @@ def _create_missing_ok_generator(generator, missing_ok: bool, error: Exception):
     return create_generator()
 
 
-class UnknownError(Exception):
-    def __init__(self, error: Exception, path: PathLike, extra: Optional[str] = None):
-        message = "Unknown error encountered: %r, error: %s" % (
-            path,
+class MaxRetriesExceededError(Exception):
+    def __init__(self, error: Exception, retries: int = 1):
+        while isinstance(error, MaxRetriesExceededError):
+            retries *= error.retries
+            error = error.__cause__
+        message = "Max retires exceeded: %s, after %d tries" % (
             full_error_message(error),
+            retries,
         )
+        super().__init__(message)
+        self.retries = retries
+        self.__cause__ = error
+
+    def __reduce__(self):
+        return (self.__class__, (self.__cause__, self.retries))
+
+
+class UnknownError(Exception):
+    def __init__(self, error: Exception, path: PathLike, extra: Optional[str] = None):
+        parts = [f"Unknown error encountered: {path!r}"]
+        if isinstance(error, MaxRetriesExceededError):
+            parts.append(f"error: {full_error_message(error.__cause__)}")
+            parts.append(f"after {error.retries} tries")
+        else:
+            parts.append(f"error: {full_error_message(error)}")
         if extra is not None:
-            message += ", " + extra
+            parts.append(extra)
+        message = ", ".join(parts)
         super().__init__(message)
         self.path = path
         self.extra = extra
@@ -350,6 +370,8 @@ def translate_fs_error(fs_error: Exception, fs_path: PathLike) -> Exception:
         if fs_error.filename is None:
             fs_error.filename = fs_path
         return fs_error
+    if isinstance(fs_error, MaxRetriesExceededError):
+        return fs_error.__cause__
     return fs_error
 
 
@@ -359,7 +381,10 @@ def translate_s3_error(s3_error: Exception, s3_url: PathLike) -> Exception:
     """
     if isinstance(s3_error, S3Exception):
         return s3_error
-    elif isinstance(s3_error, ClientError):
+    ori_error = s3_error
+    if isinstance(s3_error, MaxRetriesExceededError):
+        s3_error = s3_error.__cause__
+    if isinstance(s3_error, ClientError):
         code = client_error_code(s3_error)
         if code in ("NoSuchBucket"):
             bucket_or_url = (
@@ -419,7 +444,7 @@ def translate_s3_error(s3_error: Exception, s3_url: PathLike) -> Exception:
             return S3InvalidRangeError("Invalid range: %r" % s3_url)
         elif "AccessDenied" in str(s3_error):
             return S3PermissionError("Access denied: %r" % s3_url)
-    return S3UnknownError(s3_error, s3_url)
+    return S3UnknownError(ori_error, s3_url)
 
 
 def translate_http_error(http_error: Exception, http_url: str) -> Exception:
@@ -434,13 +459,16 @@ def translate_http_error(http_error: Exception, http_url: str) -> Exception:
     """
     if isinstance(http_error, HttpException):
         return http_error
+    ori_error = http_error
+    if isinstance(http_error, MaxRetriesExceededError):
+        http_error = http_error.__cause__
     if isinstance(http_error, HTTPError):
         status_code = http_error.response.status_code
         if status_code == 401 or status_code == 403:
             return HttpPermissionError("Permission denied: %r" % http_url)
         elif status_code == 404:
             return HttpFileNotFoundError("No such file: %r" % http_url)
-    return HttpUnknownError(http_error, http_url)
+    return HttpUnknownError(ori_error, http_url)
 
 
 @contextmanager
@@ -476,6 +504,8 @@ def translate_hdfs_error(hdfs_error: Exception, hdfs_path: PathLike) -> Exceptio
         elif hdfs_error.status_code == 404:
             return FileNotFoundError(f"No match file: {hdfs_path}")
     # pytype: enable=attribute-error
+    if isinstance(hdfs_error, MaxRetriesExceededError):
+        return hdfs_error.__cause__
     return hdfs_error
 
 

megfile/pathlike.py

@@ -565,7 +565,7 @@ class BasePath:
 
     @cached_property
     def anchor(self) -> str:
-        return self.root  # pyre-ignore[7]
+        return self.root
 
     def joinpath(self: Self, *other_paths: "PathLike") -> Self:
         """
@@ -863,7 +863,7 @@ class URIPath(BasePath):
         path = path.lstrip("/")
         if path != "":
             parts.extend(path.split("/"))
-        return tuple(parts)  # pyre-ignore[7]
+        return tuple(parts)
 
     @cached_property
     def parents(self) -> "URIPathParents":
@@ -917,10 +917,18 @@ class URIPathParents(Sequence):
             self.parts = parts
 
     def __len__(self):
+        if (
+            (self.prefix == "" or "://" in self.prefix)
+            and len(self.parts) > 0
+            and self.parts[0] != "/"
+        ):
+            return len(self.parts)
         return max(len(self.parts) - 1, 0)
 
     def __getitem__(self, idx):
-        if idx < 0 or idx > len(self):
+        if idx < 0:
+            idx += len(self)
+        if idx < 0 or idx >= len(self):
             raise IndexError(idx)
 
         if len(self.parts[: -idx - 1]) > 1:

megfile/version.py

@@ -1 +1 @@
-VERSION = "5.0.1"
+VERSION = "5.0.3"

{megfile-5.0.1.dist-info → megfile-5.0.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: megfile
-Version: 5.0.1
+Version: 5.0.3
 Summary: Megvii file operation library
 Author-email: megvii-reng <megvii-reng@googlegroups.com>
 Project-URL: Homepage, https://github.com/megvii-research/megfile

{megfile-5.0.1.dist-info → megfile-5.0.3.dist-info}/RECORD

@@ -1,19 +1,19 @@
-megfile/__init__.py,sha256=YvF3lLDTnxYxeICLYwzwJHQWfNTizBx8B0BwjWvHnRU,2934
-megfile/cli.py,sha256=HdOIAMFXdZw79kayVIri5-_rbq_KwOGSdMYTx51ABp0,30204
+megfile/__init__.py,sha256=4XbMsR-lM7MxbnPGBI16m2sq6ghuA2-zZj2XF4bbX2Y,3291
+megfile/cli.py,sha256=TjUhfcOB_weEBa_e55nmiajq_XObAhQa7sx5oivrQCw,30620
 megfile/config.py,sha256=K3B_o2dnI7qGsGnK8Jg18-S5YYLYuzskfNJowlSMkQM,5065
-megfile/errors.py,sha256=eC7z-2-QqE12pYgTGAOIlEBsemqcMyeBFVUEDz3gBS0,15585
+megfile/errors.py,sha256=zKwM5r5j89mlbWZNeax26Hq63NmQhl9iGMfTtgyvYNA,16830
 megfile/fs_path.py,sha256=tt2__W6E4vep0lmVreTLIW63njl-EzyQEEkEGziyAb4,41015
 megfile/hdfs_path.py,sha256=OmUe3vA3Qoxnqtcq0Rs3ygBvzAtqUz3fGo8iP5sWneE,26058
 megfile/http_path.py,sha256=08OmzmRMyLSyq1Yr1K2HbzexesURJrIoA6AibwYzUiA,13844
 megfile/interfaces.py,sha256=XU46U5pl4k1Gse63i4z5SvxcjWeKLj0xyB0Y6fYiWWo,9887
-megfile/pathlike.py,sha256=kacTG55j3dq3b7YP7MT7jxeJGbDwvK3XuZIvF-UUE0o,31244
+megfile/pathlike.py,sha256=4RuYHqUc5_6rZDCcVo_18il0Hy7BlOYt-rtYwCtp9Gg,31446
 megfile/s3_path.py,sha256=LINHnHnpesXnf9wxbV6n0xQVT0wPwyjLc7xAasakefU,94467
 megfile/sftp2_path.py,sha256=K90bnMVAx0MQPGXP6LogGuDRzaD4MPR6lMOfdY9C9-0,37942
 megfile/sftp_path.py,sha256=_KU7_-Mq2m7lcLY1mpiGrju0SP-OsdEXlRjFhZH25UA,51223
 megfile/smart.py,sha256=Lab2jxprj-zvPw5GqUWlWiEY8bcpRlviks_qp9r-km8,38224
 megfile/smart_path.py,sha256=kGidkM5S58ChE3LVZMcUACs3IQgsqh9m04sp6-wxuhk,12615
 megfile/stdio_path.py,sha256=cxaDr8rtisTPnN-rjtaEpqQnshwiqwXFUJBM9xWY7Cg,2711
-megfile/version.py,sha256=LPuRF_X3cat_YjhbR3eY8fMsZ1mTDe7rRMdxF3ozYRY,19
+megfile/version.py,sha256=ayvDTXMtNAcyGKXxogED3ZAvUOOsfQYG6VqgcDYL7o8,19
 megfile/webdav_path.py,sha256=xQmZMt-hDA7PfHzuSjRYaIoJA_Nbi1jsg952KZJhs-E,31364
 megfile/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 megfile/lib/base_memory_handler.py,sha256=i5-YHpL0k2tuFBnP9VMzb3_OsaU6D6j6thkmWgptnFg,2680
@@ -42,10 +42,10 @@ megfile/lib/webdav_memory_handler.py,sha256=_UccPYPpvfTd4gSEhBFL1BHeyFtsBJdhVINk
 megfile/lib/webdav_prefetch_reader.py,sha256=M0X6E6t-DS5q9KiLvjVZx_AZuiW9SaIkBnIPLc774GQ,3941
 megfile/utils/__init__.py,sha256=4hBVSXbNTbDj7Je0y9SbwgcPm_s41H9v3eHUMr9JNGo,12700
 megfile/utils/mutex.py,sha256=asb8opGLgK22RiuBJUnfsvB8LnMmodP8KzCVHKmQBWA,2561
-megfile-5.0.1.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-megfile-5.0.1.dist-info/licenses/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
-megfile-5.0.1.dist-info/METADATA,sha256=r-7Od-pbEExL11PMsE0VhgxUvkv_O0kGz0s8bzzczRE,9225
-megfile-5.0.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-megfile-5.0.1.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
-megfile-5.0.1.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
-megfile-5.0.1.dist-info/RECORD,,
+megfile-5.0.3.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+megfile-5.0.3.dist-info/licenses/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
+megfile-5.0.3.dist-info/METADATA,sha256=qc9JcTR52y09F_lY2fiCuwCkeRzfekculMSNeXBEP_g,9225
+megfile-5.0.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+megfile-5.0.3.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
+megfile-5.0.3.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
+megfile-5.0.3.dist-info/RECORD,,