megfile 4.2.0__py3-none-any.whl → 4.2.2__py3-none-any.whl

megfile/errors.py

@@ -9,6 +9,10 @@ from typing import Callable, Optional
 import botocore.exceptions
 import requests.exceptions
 import urllib3.exceptions
+from boto3.exceptions import (  # TODO: test different boto3 version
+    S3TransferFailedError,
+    S3UploadFailedError,
+)
 from botocore.exceptions import ClientError, NoCredentialsError, ParamValidationError
 from requests.exceptions import HTTPError
 
@@ -402,6 +406,19 @@ def translate_s3_error(s3_error: Exception, s3_url: PathLike) -> Exception:
         return S3UnknownError(s3_error, s3_url)
     elif isinstance(s3_error, NoCredentialsError):
         return S3ConfigError(str(s3_error))
+    elif isinstance(s3_error, (S3UploadFailedError, S3TransferFailedError)):
+        if "NoSuchBucket" in str(s3_error):
+            return S3BucketNotFoundError("No such bucket: %r" % s3_url)
+        elif "NoSuchKey" in str(s3_error):
+            return S3FileNotFoundError("No such file: %r" % s3_url)
+        elif "InvalidAccessKeyId" in str(s3_error) or "SignatureDoesNotMatch" in str(
+            s3_error
+        ):
+            return S3ConfigError("Invalid access key id: %r" % s3_url)
+        elif "InvalidRange" in str(s3_error):
+            return S3InvalidRangeError("Invalid range: %r" % s3_url)
+        elif "AccessDenied" in str(s3_error):
+            return S3PermissionError("Access denied: %r" % s3_url)
     return S3UnknownError(s3_error, s3_url)
 
 

megfile/s3_path.py

@@ -1201,6 +1201,10 @@ def s3_upload(
     src_path = FSPath(src_url)
     if followlinks and src_path.is_symlink():
         src_path = src_path.readlink()
+    if not src_path.exists():
+        raise FileNotFoundError("No such file or directory: %r" % src_url)
+    elif src_path.is_dir():
+        raise IsADirectoryError("Is a directory: %r" % src_url)
 
     dst_bucket, dst_key = parse_s3_url(dst_url)
     if not dst_bucket:
@@ -1212,8 +1216,8 @@ def s3_upload(
         return
 
     client = get_s3_client_with_cache(profile_name=S3Path(dst_url)._profile_name)
-    upload_fileobj = patch_method(
-        client.upload_fileobj, max_retries=max_retries, should_retry=s3_should_retry
+    upload_file = patch_method(
+        client.upload_file, max_retries=max_retries, should_retry=s3_should_retry
     )
 
     transfer_config = TransferConfig(
@@ -1221,9 +1225,9 @@ def s3_upload(
         max_concurrency=GLOBAL_MAX_WORKERS,
         multipart_chunksize=WRITER_BLOCK_SIZE,
     )
-    with open(src_path.path_without_protocol, "rb") as src, raise_s3_error(dst_url):
-        upload_fileobj(
-            src,
+    with raise_s3_error(dst_url):
+        upload_file(
+            src_path.path_without_protocol,
             Bucket=dst_bucket,
             Key=dst_key,
             Callback=callback,
@@ -1676,15 +1680,18 @@ class S3Path(URIPath):
         :returns: True if path is s3 file, else False
         """
         s3_url = self.path_with_protocol
+        bucket, key = parse_s3_url(s3_url)
+        if not bucket or not key or key.endswith("/"):
+            # s3://, s3:///key, s3://bucket, s3://bucket/prefix/
+            return False
+
         if followlinks:
             try:
                 s3_url = self.readlink().path_with_protocol
+                bucket, key = parse_s3_url(s3_url)
             except S3NotALinkError:
                 pass
-        bucket, key = parse_s3_url(s3_url)
-        if not bucket or not key or key.endswith("/"):
-            # s3://, s3:///key, s3://bucket, s3://bucket/prefix/
-            return False
+
         try:
             self._client.head_object(Bucket=bucket, Key=key)
         except Exception as error:
@@ -1829,18 +1836,18 @@ class S3Path(URIPath):
             )
         if not key:
             raise UnsupportedError("Remove bucket", self.path_with_protocol)
-        if not self.exists():
-            if missing_ok:
-                return
-            raise S3FileNotFoundError(
-                "No such file or directory: %r" % self.path_with_protocol
-            )
 
         client = self._client
+        if self.is_file():
+            try:
+                with raise_s3_error(self.path_with_protocol):
+                    client.delete_object(Bucket=bucket, Key=key)
+            except S3FileNotFoundError:
+                if not missing_ok:
+                    raise
+            return
+
         with raise_s3_error(self.path_with_protocol):
-            if self.is_file():
-                client.delete_object(Bucket=bucket, Key=key)
-                return
             prefix = _become_prefix(key)
             total_count, error_count = 0, 0
             for resp in _list_objects_recursive(client, bucket, prefix):
@@ -1891,6 +1898,10 @@ class S3Path(URIPath):
                     % (self.path_with_protocol, total_count, error_count)
                 )
                 raise S3UnknownError(Exception(error_msg), self.path_with_protocol)
+            if total_count == 0 and not missing_ok:
+                raise S3FileNotFoundError(
+                    "No such file or directory: %r" % self.path_with_protocol
+                )
 
     def rename(self, dst_path: PathLike, overwrite: bool = True) -> "S3Path":
         """

megfile/smart.py

@@ -308,8 +308,18 @@ def _default_copy_func(
 
 
 def _get_copy_func(src_protocol, dst_protocol):
-    if src_protocol.startswith("s3+") and src_protocol == dst_protocol:
+    def _is_s3_plus(protocol: str):
+        return protocol.startswith("s3+")
+
+    def _is_s3_or_s3_plus(protocol: str):
+        return protocol.startswith("s3+") or protocol == "s3"
+
+    if _is_s3_plus(src_protocol) and src_protocol == dst_protocol:
         src_protocol = dst_protocol = "s3"
+    elif _is_s3_plus(src_protocol) and not _is_s3_or_s3_plus(dst_protocol):
+        src_protocol = "s3"
+    elif _is_s3_plus(dst_protocol) and not _is_s3_or_s3_plus(src_protocol):
+        dst_protocol = "s3"
 
     try:
         return _copy_funcs[src_protocol][dst_protocol]

megfile/version.py

@@ -1 +1 @@
-VERSION = "4.2.0"
+VERSION = "4.2.2"

{megfile-4.2.0.dist-info → megfile-4.2.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: megfile
-Version: 4.2.0
+Version: 4.2.2
 Summary: Megvii file operation library
 Author-email: megvii <megfile@megvii.com>
 Project-URL: Homepage, https://github.com/megvii-research/megfile
@@ -24,7 +24,7 @@ License-File: LICENSE.pyre
 Requires-Dist: boto3
 Requires-Dist: botocore>=1.13.0
 Requires-Dist: requests
-Requires-Dist: paramiko
+Requires-Dist: paramiko<4.0.0
 Requires-Dist: tqdm
 Requires-Dist: pyyaml
 Provides-Extra: hdfs

{megfile-4.2.0.dist-info → megfile-4.2.2.dist-info}/RECORD

@@ -1,7 +1,7 @@
 megfile/__init__.py,sha256=7oEfu410CFKzDWZ9RjL5xEJ1gtkJkTfvPrL_7TWdJuY,7366
 megfile/cli.py,sha256=VxY0__M19Ti_S7ZIozp9l0FxvdLwdd6eQL-wfpYOi_0,29160
 megfile/config.py,sha256=2MMj5QkhlDJQFZRbCQL2c9iDdeMAVctiaPszRBkg5vM,3988
-megfile/errors.py,sha256=aw4BX-pU1uj0-dNc6Tq-IeSwK_PWJNi2ZL5ZcRSo7aQ,14739
+megfile/errors.py,sha256=cGSYyB7VBRKi1Gehgt9IO-wDvtzICV4XgKOkoMvLU5w,15583
 megfile/fs.py,sha256=KMEqAE35alpcxiy6du5nPFYcaorhUM_kPJMah3q76ng,19160
 megfile/fs_path.py,sha256=Hozl9LAJ8EMuSWBSZXGj2GNmPZ1sJp9PZs-7hPrLgm8,39341
 megfile/hdfs.py,sha256=owXr4d3j1frCvlbhmhENcSBnKKDky5cJZzWLOF4ZJMo,13251
@@ -11,14 +11,14 @@ megfile/http_path.py,sha256=08OmzmRMyLSyq1Yr1K2HbzexesURJrIoA6AibwYzUiA,13844
 megfile/interfaces.py,sha256=p4UvVZpeLx5djd6bqqDaygIx_s-_AxIVj-gudTch4JE,8467
 megfile/pathlike.py,sha256=3Hnw-fn6RcIe9iPrJt00QdHSA--UfDyxnVBuZ_ymYYQ,31278
 megfile/s3.py,sha256=abBxnI7RIyn7n7qjGszP1VruYd6Gi9I8QnUOvsHkx1Y,16325
-megfile/s3_path.py,sha256=nA1FXFwfhSvZN8W0mF2PKGbnaTDouZfp6anUZkxvOBc,93449
+megfile/s3_path.py,sha256=fzqXahIEKGbyArU-2Yl5YpczwkImUiZy7Y1RYQOKNKM,93827
 megfile/sftp.py,sha256=uBcLQs-j6Q-q-sWAdd-pgi5Qmb_kq7boJM-0sCfcNO0,26540
 megfile/sftp_path.py,sha256=CgirHWmNdXdqyIL9ufmlaMpwFhlkQVZhqmfvjUaj7qU,43845
-megfile/smart.py,sha256=Miz3jyLKmwWBja-8GrSzrumpTarPrFPqXaFQJKwrK1Y,36627
+megfile/smart.py,sha256=GnabQVb_NU7a-etKfF-NgpJ9JM2rT0uZBeA9UoNz_wM,37014
 megfile/smart_path.py,sha256=Up_6xNZ2019iSzMn_JAU_1H--z-AP6O7SxdXGdeTG0c,7659
 megfile/stdio.py,sha256=ZwxsnJNJYIT7Iyg5pIw4qiyH8bszG6oAhEJuR-hXGG4,658
 megfile/stdio_path.py,sha256=cxaDr8rtisTPnN-rjtaEpqQnshwiqwXFUJBM9xWY7Cg,2711
-megfile/version.py,sha256=nmN7kc0bNHgtVq4jv4MJKEo-L3X7QgGHAv_z6zkMVSQ,19
+megfile/version.py,sha256=xnzpVSLKRvpgVFWjV-1xKKQitlJEcAepuzOjV7CqQ6I,19
 megfile/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 megfile/lib/base_prefetch_reader.py,sha256=uxVwYknOjc8hLF7q_T2QKMsBqFcrf411ZsuK25CN1eQ,12848
 megfile/lib/combine_reader.py,sha256=Kp2wEloOUpTlIU7dve87MBpSzmIM-F9OtpTawAjFkiU,4828
@@ -43,10 +43,10 @@ megfile/lib/stdio_handler.py,sha256=IDdgENLQlhigEwkLL4zStueVSzdWg7xVcTF_koof_Ek,
 megfile/lib/url.py,sha256=ER32pWy9Q2MAk3TraAaNEBWIqUeBmLuM57ol2cs7-Ks,103
 megfile/utils/__init__.py,sha256=pawmXnCNokWLj338a60b_hK21koYavpEiEohZhsOaGQ,10156
 megfile/utils/mutex.py,sha256=asb8opGLgK22RiuBJUnfsvB8LnMmodP8KzCVHKmQBWA,2561
-megfile-4.2.0.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-megfile-4.2.0.dist-info/licenses/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
-megfile-4.2.0.dist-info/METADATA,sha256=0IuIzy3B0sO0YhPrMdEK5kCkK16K1ZCX6w3vZO-SlB4,9595
-megfile-4.2.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-megfile-4.2.0.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
-megfile-4.2.0.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
-megfile-4.2.0.dist-info/RECORD,,
+megfile-4.2.2.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+megfile-4.2.2.dist-info/licenses/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
+megfile-4.2.2.dist-info/METADATA,sha256=GCTiJb_unoYia-PwLFfqjOhm-7hjU73R-pJIpDChSAk,9601
+megfile-4.2.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+megfile-4.2.2.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
+megfile-4.2.2.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
+megfile-4.2.2.dist-info/RECORD,,