megfile 4.2.1__py3-none-any.whl → 4.2.3__py3-none-any.whl

megfile/errors.py

@@ -9,6 +9,10 @@ from typing import Callable, Optional
 import botocore.exceptions
 import requests.exceptions
 import urllib3.exceptions
+from boto3.exceptions import (  # TODO: test different boto3 version
+    S3TransferFailedError,
+    S3UploadFailedError,
+)
 from botocore.exceptions import ClientError, NoCredentialsError, ParamValidationError
 from requests.exceptions import HTTPError
 
@@ -402,6 +406,19 @@ def translate_s3_error(s3_error: Exception, s3_url: PathLike) -> Exception:
         return S3UnknownError(s3_error, s3_url)
     elif isinstance(s3_error, NoCredentialsError):
         return S3ConfigError(str(s3_error))
+    elif isinstance(s3_error, (S3UploadFailedError, S3TransferFailedError)):
+        if "NoSuchBucket" in str(s3_error):
+            return S3BucketNotFoundError("No such bucket: %r" % s3_url)
+        elif "NoSuchKey" in str(s3_error):
+            return S3FileNotFoundError("No such file: %r" % s3_url)
+        elif "InvalidAccessKeyId" in str(s3_error) or "SignatureDoesNotMatch" in str(
+            s3_error
+        ):
+            return S3ConfigError("Invalid access key id: %r" % s3_url)
+        elif "InvalidRange" in str(s3_error):
+            return S3InvalidRangeError("Invalid range: %r" % s3_url)
+        elif "AccessDenied" in str(s3_error):
+            return S3PermissionError("Access denied: %r" % s3_url)
     return S3UnknownError(s3_error, s3_url)
 
 

megfile/lib/compare.py

@@ -5,10 +5,10 @@ from megfile.pathlike import StatResult
 
 
 def get_sync_type(src_protocol, dst_protocol):
-    if src_protocol == "s3" and dst_protocol != "s3":
-        return "download"
-    elif src_protocol != "s3" and dst_protocol == "s3":
+    if dst_protocol == "s3" or dst_protocol.startswith("s3+"):
         return "upload"
+    elif src_protocol == "s3" or src_protocol.startswith("s3+"):
+        return "download"
     else:
         return "copy"
 

megfile/lib/glob.py

@@ -289,6 +289,9 @@ def get_non_glob_dir(glob: str):
     root_dir = []
     if glob.startswith("/"):
         root_dir.append("/")
+    elif "://" in glob:
+        protocol, glob = glob.split("://", 1)
+        root_dir.append(f"{protocol}://")
     for name in glob.split("/"):
         if has_magic(name):
             break

megfile/s3_path.py

@@ -610,6 +610,7 @@ def _s3_glob_stat_single_path(
                         yield FileEntry(S3Path(path).name, path, _make_stat(content))
                     dirname = os.path.dirname(path)
                     while dirname not in dirnames and dirname != top_dir:
+                        # TODO: optimize memory usage and file path order
                         dirnames.add(dirname)
                         path = dirname + "/" if search_dir else dirname
                         if pattern.match(path):
@@ -1201,6 +1202,10 @@ def s3_upload(
     src_path = FSPath(src_url)
     if followlinks and src_path.is_symlink():
         src_path = src_path.readlink()
+    if not src_path.exists():
+        raise FileNotFoundError("No such file or directory: %r" % src_url)
+    elif src_path.is_dir():
+        raise IsADirectoryError("Is a directory: %r" % src_url)
 
     dst_bucket, dst_key = parse_s3_url(dst_url)
     if not dst_bucket:
@@ -1212,8 +1217,8 @@ def s3_upload(
         return
 
     client = get_s3_client_with_cache(profile_name=S3Path(dst_url)._profile_name)
-    upload_fileobj = patch_method(
-        client.upload_fileobj, max_retries=max_retries, should_retry=s3_should_retry
+    upload_file = patch_method(
+        client.upload_file, max_retries=max_retries, should_retry=s3_should_retry
     )
 
     transfer_config = TransferConfig(
@@ -1221,9 +1226,9 @@ def s3_upload(
         max_concurrency=GLOBAL_MAX_WORKERS,
         multipart_chunksize=WRITER_BLOCK_SIZE,
     )
-    with open(src_path.path_without_protocol, "rb") as src, raise_s3_error(dst_url):
-        upload_fileobj(
-            src,
+    with raise_s3_error(dst_url):
+        upload_file(
+            src_path.path_without_protocol,
             Bucket=dst_bucket,
             Key=dst_key,
             Callback=callback,

megfile/smart.py

@@ -308,8 +308,18 @@ def _default_copy_func(
 
 
 def _get_copy_func(src_protocol, dst_protocol):
-    if src_protocol.startswith("s3+") and src_protocol == dst_protocol:
+    def _is_s3_plus(protocol: str):
+        return protocol.startswith("s3+")
+
+    def _is_s3_or_s3_plus(protocol: str):
+        return protocol.startswith("s3+") or protocol == "s3"
+
+    if _is_s3_plus(src_protocol) and src_protocol == dst_protocol:
         src_protocol = dst_protocol = "s3"
+    elif _is_s3_plus(src_protocol) and not _is_s3_or_s3_plus(dst_protocol):
+        src_protocol = "s3"
+    elif _is_s3_plus(dst_protocol) and not _is_s3_or_s3_plus(src_protocol):
+        dst_protocol = "s3"
 
     try:
         return _copy_funcs[src_protocol][dst_protocol]

megfile/version.py

@@ -1 +1 @@
-VERSION = "4.2.1"
+VERSION = "4.2.3"

{megfile-4.2.1.dist-info → megfile-4.2.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: megfile
-Version: 4.2.1
+Version: 4.2.3
 Summary: Megvii file operation library
 Author-email: megvii <megfile@megvii.com>
 Project-URL: Homepage, https://github.com/megvii-research/megfile
@@ -24,7 +24,7 @@ License-File: LICENSE.pyre
 Requires-Dist: boto3
 Requires-Dist: botocore>=1.13.0
 Requires-Dist: requests
-Requires-Dist: paramiko
+Requires-Dist: paramiko<4.0.0
 Requires-Dist: tqdm
 Requires-Dist: pyyaml
 Provides-Extra: hdfs

{megfile-4.2.1.dist-info → megfile-4.2.3.dist-info}/RECORD

@@ -1,7 +1,7 @@
 megfile/__init__.py,sha256=7oEfu410CFKzDWZ9RjL5xEJ1gtkJkTfvPrL_7TWdJuY,7366
 megfile/cli.py,sha256=VxY0__M19Ti_S7ZIozp9l0FxvdLwdd6eQL-wfpYOi_0,29160
 megfile/config.py,sha256=2MMj5QkhlDJQFZRbCQL2c9iDdeMAVctiaPszRBkg5vM,3988
-megfile/errors.py,sha256=aw4BX-pU1uj0-dNc6Tq-IeSwK_PWJNi2ZL5ZcRSo7aQ,14739
+megfile/errors.py,sha256=cGSYyB7VBRKi1Gehgt9IO-wDvtzICV4XgKOkoMvLU5w,15583
 megfile/fs.py,sha256=KMEqAE35alpcxiy6du5nPFYcaorhUM_kPJMah3q76ng,19160
 megfile/fs_path.py,sha256=Hozl9LAJ8EMuSWBSZXGj2GNmPZ1sJp9PZs-7hPrLgm8,39341
 megfile/hdfs.py,sha256=owXr4d3j1frCvlbhmhENcSBnKKDky5cJZzWLOF4ZJMo,13251
@@ -11,21 +11,21 @@ megfile/http_path.py,sha256=08OmzmRMyLSyq1Yr1K2HbzexesURJrIoA6AibwYzUiA,13844
 megfile/interfaces.py,sha256=p4UvVZpeLx5djd6bqqDaygIx_s-_AxIVj-gudTch4JE,8467
 megfile/pathlike.py,sha256=3Hnw-fn6RcIe9iPrJt00QdHSA--UfDyxnVBuZ_ymYYQ,31278
 megfile/s3.py,sha256=abBxnI7RIyn7n7qjGszP1VruYd6Gi9I8QnUOvsHkx1Y,16325
-megfile/s3_path.py,sha256=NKOJLxy1gxdOc-jdEvxNl45tham5un8YWmZC-CZXHuU,93663
+megfile/s3_path.py,sha256=yulypUpJ2k0WzOPbeXyx8Q75YGJa3R5qXBt2QmMt9H0,93901
 megfile/sftp.py,sha256=uBcLQs-j6Q-q-sWAdd-pgi5Qmb_kq7boJM-0sCfcNO0,26540
 megfile/sftp_path.py,sha256=CgirHWmNdXdqyIL9ufmlaMpwFhlkQVZhqmfvjUaj7qU,43845
-megfile/smart.py,sha256=Miz3jyLKmwWBja-8GrSzrumpTarPrFPqXaFQJKwrK1Y,36627
+megfile/smart.py,sha256=GnabQVb_NU7a-etKfF-NgpJ9JM2rT0uZBeA9UoNz_wM,37014
 megfile/smart_path.py,sha256=Up_6xNZ2019iSzMn_JAU_1H--z-AP6O7SxdXGdeTG0c,7659
 megfile/stdio.py,sha256=ZwxsnJNJYIT7Iyg5pIw4qiyH8bszG6oAhEJuR-hXGG4,658
 megfile/stdio_path.py,sha256=cxaDr8rtisTPnN-rjtaEpqQnshwiqwXFUJBM9xWY7Cg,2711
-megfile/version.py,sha256=e0k-2Nn8dpoGmVJ3Wg03fcRMzM9q9jfKeY7f3OkI7LE,19
+megfile/version.py,sha256=DjJW-M4YIrKI_sFscLAPh7I9pVVFC24RH3yBxS8RkNI,19
 megfile/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 megfile/lib/base_prefetch_reader.py,sha256=uxVwYknOjc8hLF7q_T2QKMsBqFcrf411ZsuK25CN1eQ,12848
 megfile/lib/combine_reader.py,sha256=Kp2wEloOUpTlIU7dve87MBpSzmIM-F9OtpTawAjFkiU,4828
-megfile/lib/compare.py,sha256=n_dtLxgoskYnsIZMKdKmVhQoVn8qYUrUhkS1JH2_X3o,2170
+megfile/lib/compare.py,sha256=CPSbyqsQ396oSfxa7h0NdUUqBw5A3WOn6fHrNKkuinw,2188
 megfile/lib/compat.py,sha256=SynEeHluys3tCK-lb_1oV3o_ft83yZvunqM_AjibLgE,207
 megfile/lib/fnmatch.py,sha256=4MvGzEahMRA-u8Z7mxaD-Yw1idOwBoJJpVywQy29jwY,4162
-megfile/lib/glob.py,sha256=anr7JjCJz1T0G5hYQkJwV5mCWzlt5YaxGTyyyQB28zA,9663
+megfile/lib/glob.py,sha256=BSpYm6w16rVuIBCI9hNOCxn83BN7UuKGBp2wFRFdwp4,9775
 megfile/lib/hdfs_prefetch_reader.py,sha256=yCNpcXcTiC2SHKHC-Qp50KQx1ObSLmOgwNUKlG-4ADg,2131
 megfile/lib/hdfs_tools.py,sha256=4K-OdMYFFSLBGmDzjatioHvuZuUbKVy7ACeJl-l0HLQ,435
 megfile/lib/http_prefetch_reader.py,sha256=OQPZ7kWFImqpynjaiTtmadtgtab5fCeQmu51UYHZfgs,4135
@@ -43,10 +43,10 @@ megfile/lib/stdio_handler.py,sha256=IDdgENLQlhigEwkLL4zStueVSzdWg7xVcTF_koof_Ek,
 megfile/lib/url.py,sha256=ER32pWy9Q2MAk3TraAaNEBWIqUeBmLuM57ol2cs7-Ks,103
 megfile/utils/__init__.py,sha256=pawmXnCNokWLj338a60b_hK21koYavpEiEohZhsOaGQ,10156
 megfile/utils/mutex.py,sha256=asb8opGLgK22RiuBJUnfsvB8LnMmodP8KzCVHKmQBWA,2561
-megfile-4.2.1.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-megfile-4.2.1.dist-info/licenses/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
-megfile-4.2.1.dist-info/METADATA,sha256=8-Gzye2STZTocHieLky8XR0nAae8sHEmanRVeTWKWYM,9595
-megfile-4.2.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-megfile-4.2.1.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
-megfile-4.2.1.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
-megfile-4.2.1.dist-info/RECORD,,
+megfile-4.2.3.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+megfile-4.2.3.dist-info/licenses/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
+megfile-4.2.3.dist-info/METADATA,sha256=EKObM2zjGcOsvRyMILjA4qgEvJg72mPB8icyZhBbZ9o,9601
+megfile-4.2.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+megfile-4.2.3.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
+megfile-4.2.3.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
+megfile-4.2.3.dist-info/RECORD,,