megfile 4.1.8.post1__py3-none-any.whl → 4.2.1__py3-none-any.whl

megfile/cli.py

@@ -70,17 +70,16 @@ def cli(debug, log_level):
     """
     options["debug"] = debug
     options["log_level"] = log_level or ("DEBUG" if debug else "INFO")
+    if not debug:
+        signal.signal(signal.SIGINT, signal.SIG_DFL)
     set_log_level(options["log_level"])
 
 
 def safe_cli():  # pragma: no cover
-    debug = options.get("debug", False)
-    if not debug:
-        signal.signal(signal.SIGINT, signal.SIG_DFL)
     try:
         cli()
     except Exception as e:
-        if debug:
+        if options.get("debug", False):
             raise
         else:
             click.echo(f"\n[{type(e).__name__}] {e}", err=True)

megfile/s3_path.py

@@ -605,6 +605,8 @@ def _s3_glob_stat_single_path(
                 for content in resp.get("Contents", []):
                     path = s3_path_join(f"{protocol}://", bucket, content["Key"])
                     if not search_dir and pattern.match(path):
+                        if path.endswith("/"):
+                            continue
                         yield FileEntry(S3Path(path).name, path, _make_stat(content))
                     dirname = os.path.dirname(path)
                     while dirname not in dirnames and dirname != top_dir:
@@ -1674,15 +1676,18 @@ class S3Path(URIPath):
         :returns: True if path is s3 file, else False
         """
         s3_url = self.path_with_protocol
+        bucket, key = parse_s3_url(s3_url)
+        if not bucket or not key or key.endswith("/"):
+            # s3://, s3:///key, s3://bucket, s3://bucket/prefix/
+            return False
+
         if followlinks:
             try:
                 s3_url = self.readlink().path_with_protocol
+                bucket, key = parse_s3_url(s3_url)
             except S3NotALinkError:
                 pass
-        bucket, key = parse_s3_url(s3_url)
-        if not bucket or not key or key.endswith("/"):
-            # s3://, s3:///key, s3://bucket, s3://bucket/prefix/
-            return False
+
         try:
             self._client.head_object(Bucket=bucket, Key=key)
         except Exception as error:
@@ -1827,18 +1832,18 @@ class S3Path(URIPath):
             )
         if not key:
             raise UnsupportedError("Remove bucket", self.path_with_protocol)
-        if not self.exists():
-            if missing_ok:
-                return
-            raise S3FileNotFoundError(
-                "No such file or directory: %r" % self.path_with_protocol
-            )
 
         client = self._client
+        if self.is_file():
+            try:
+                with raise_s3_error(self.path_with_protocol):
+                    client.delete_object(Bucket=bucket, Key=key)
+            except S3FileNotFoundError:
+                if not missing_ok:
+                    raise
+            return
+
         with raise_s3_error(self.path_with_protocol):
-            if self.is_file():
-                client.delete_object(Bucket=bucket, Key=key)
-                return
             prefix = _become_prefix(key)
             total_count, error_count = 0, 0
             for resp in _list_objects_recursive(client, bucket, prefix):
@@ -1889,6 +1894,10 @@ class S3Path(URIPath):
                     % (self.path_with_protocol, total_count, error_count)
                 )
                 raise S3UnknownError(Exception(error_msg), self.path_with_protocol)
+            if total_count == 0 and not missing_ok:
+                raise S3FileNotFoundError(
+                    "No such file or directory: %r" % self.path_with_protocol
+                )
 
     def rename(self, dst_path: PathLike, overwrite: bool = True) -> "S3Path":
         """
@@ -1960,6 +1969,7 @@ class S3Path(URIPath):
                 )
 
             prefix = _become_prefix(key)
+            protocol = self._protocol_with_profile
             client = self._client
 
             def suppress_error_callback(e):
@@ -1970,13 +1980,13 @@ class S3Path(URIPath):
             with raise_s3_error(self.path_with_protocol, suppress_error_callback):
                 for resp in _list_objects_recursive(client, bucket, prefix):
                     for content in resp.get("Contents", []):
-                        full_path = s3_path_join(
-                            f"{self._protocol_with_profile}://", bucket, content["Key"]
-                        )
+                        if content["Key"].endswith("/"):
+                            continue
+                        path = s3_path_join(f"{protocol}://", bucket, content["Key"])
 
                         if followlinks:
                             try:
-                                origin_path = self.from_path(full_path).readlink()
+                                origin_path = self.from_path(path).readlink()
                                 yield FileEntry(
                                     origin_path.name,
                                     origin_path.path_with_protocol,
@@ -1986,9 +1996,7 @@ class S3Path(URIPath):
                             except S3NotALinkError:
                                 pass
 
-                        yield FileEntry(
-                            S3Path(full_path).name, full_path, _make_stat(content)
-                        )
+                        yield FileEntry(S3Path(path).name, path, _make_stat(content))
 
         return _create_missing_ok_generator(
             create_generator(),
@@ -2016,17 +2024,15 @@ class S3Path(URIPath):
         # we need to wrap the iterator in another function
         def create_generator() -> Iterator[FileEntry]:
             prefix = _become_prefix(key)
+            protocol = self._protocol_with_profile
             client = self._client
 
-            def generate_s3_path(protocol: str, bucket: str, key: str) -> str:
-                return "%s://%s/%s" % (protocol, bucket, key)
-
             if not bucket and not key:  # list buckets
                 response = client.list_buckets()
                 for content in response["Buckets"]:
                     yield FileEntry(
                         content["Name"],
-                        f"{self._protocol_with_profile}://{content['Name']}",
+                        f"{protocol}://{content['Name']}",
                         StatResult(
                             ctime=content["CreationDate"].timestamp(),
                             isdir=True,
@@ -2039,21 +2045,17 @@ class S3Path(URIPath):
                 for common_prefix in resp.get("CommonPrefixes", []):
                     yield FileEntry(
                         common_prefix["Prefix"][len(prefix) : -1],
-                        generate_s3_path(
-                            self._protocol_with_profile,
-                            bucket,
-                            common_prefix["Prefix"],
-                        ),
+                        f"{protocol}://{bucket}/{common_prefix['Prefix']}",
                         StatResult(isdir=True, extra=common_prefix),
                     )
                 for content in resp.get("Contents", []):
-                    src_url = generate_s3_path(
-                        self._protocol_with_profile, bucket, content["Key"]
-                    )
+                    if content["Key"].endswith("/"):
+                        continue
+                    path = f"{protocol}://{bucket}/{content['Key']}"
                     yield FileEntry(  # pytype: disable=wrong-arg-types
                         content["Key"][len(prefix) :],
-                        src_url,
-                        _make_stat_without_metadata(content, self.from_path(src_url)),
+                        path,
+                        _make_stat_without_metadata(content, self.from_path(path)),
                     )
 
         def missing_ok_generator():
@@ -2220,6 +2222,8 @@ class S3Path(URIPath):
                     for common_prefix in resp.get("CommonPrefixes", []):
                         dirs.append(common_prefix["Prefix"][:-1])
                     for content in resp.get("Contents", []):
+                        if content["Key"].endswith("/"):
+                            continue
                         files.append(content["Key"])
 
                 dirs = sorted(dirs)

megfile/version.py

@@ -1 +1 @@
-VERSION = "4.1.8.post1"
+VERSION = "4.2.1"

{megfile-4.1.8.post1.dist-info → megfile-4.2.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: megfile
-Version: 4.1.8.post1
+Version: 4.2.1
 Summary: Megvii file operation library
 Author-email: megvii <megfile@megvii.com>
 Project-URL: Homepage, https://github.com/megvii-research/megfile

{megfile-4.1.8.post1.dist-info → megfile-4.2.1.dist-info}/RECORD

@@ -1,5 +1,5 @@
 megfile/__init__.py,sha256=7oEfu410CFKzDWZ9RjL5xEJ1gtkJkTfvPrL_7TWdJuY,7366
-megfile/cli.py,sha256=iwwlwVNu_yXgqnaURPHqGPrx1WeReqN0zi9C4rqK5Ag,29178
+megfile/cli.py,sha256=VxY0__M19Ti_S7ZIozp9l0FxvdLwdd6eQL-wfpYOi_0,29160
 megfile/config.py,sha256=2MMj5QkhlDJQFZRbCQL2c9iDdeMAVctiaPszRBkg5vM,3988
 megfile/errors.py,sha256=aw4BX-pU1uj0-dNc6Tq-IeSwK_PWJNi2ZL5ZcRSo7aQ,14739
 megfile/fs.py,sha256=KMEqAE35alpcxiy6du5nPFYcaorhUM_kPJMah3q76ng,19160
@@ -11,14 +11,14 @@ megfile/http_path.py,sha256=08OmzmRMyLSyq1Yr1K2HbzexesURJrIoA6AibwYzUiA,13844
 megfile/interfaces.py,sha256=p4UvVZpeLx5djd6bqqDaygIx_s-_AxIVj-gudTch4JE,8467
 megfile/pathlike.py,sha256=3Hnw-fn6RcIe9iPrJt00QdHSA--UfDyxnVBuZ_ymYYQ,31278
 megfile/s3.py,sha256=abBxnI7RIyn7n7qjGszP1VruYd6Gi9I8QnUOvsHkx1Y,16325
-megfile/s3_path.py,sha256=4vKdn5RupvSMYelYznCMDpE1t-87H3iwgzY4iZHWmIE,93519
+megfile/s3_path.py,sha256=NKOJLxy1gxdOc-jdEvxNl45tham5un8YWmZC-CZXHuU,93663
 megfile/sftp.py,sha256=uBcLQs-j6Q-q-sWAdd-pgi5Qmb_kq7boJM-0sCfcNO0,26540
 megfile/sftp_path.py,sha256=CgirHWmNdXdqyIL9ufmlaMpwFhlkQVZhqmfvjUaj7qU,43845
 megfile/smart.py,sha256=Miz3jyLKmwWBja-8GrSzrumpTarPrFPqXaFQJKwrK1Y,36627
 megfile/smart_path.py,sha256=Up_6xNZ2019iSzMn_JAU_1H--z-AP6O7SxdXGdeTG0c,7659
 megfile/stdio.py,sha256=ZwxsnJNJYIT7Iyg5pIw4qiyH8bszG6oAhEJuR-hXGG4,658
 megfile/stdio_path.py,sha256=cxaDr8rtisTPnN-rjtaEpqQnshwiqwXFUJBM9xWY7Cg,2711
-megfile/version.py,sha256=FceFVYf0TTvPdz_-wQYGd2Nu7cQw9bu1MClPA-Hgh0E,25
+megfile/version.py,sha256=e0k-2Nn8dpoGmVJ3Wg03fcRMzM9q9jfKeY7f3OkI7LE,19
 megfile/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 megfile/lib/base_prefetch_reader.py,sha256=uxVwYknOjc8hLF7q_T2QKMsBqFcrf411ZsuK25CN1eQ,12848
 megfile/lib/combine_reader.py,sha256=Kp2wEloOUpTlIU7dve87MBpSzmIM-F9OtpTawAjFkiU,4828
@@ -43,10 +43,10 @@ megfile/lib/stdio_handler.py,sha256=IDdgENLQlhigEwkLL4zStueVSzdWg7xVcTF_koof_Ek,
 megfile/lib/url.py,sha256=ER32pWy9Q2MAk3TraAaNEBWIqUeBmLuM57ol2cs7-Ks,103
 megfile/utils/__init__.py,sha256=pawmXnCNokWLj338a60b_hK21koYavpEiEohZhsOaGQ,10156
 megfile/utils/mutex.py,sha256=asb8opGLgK22RiuBJUnfsvB8LnMmodP8KzCVHKmQBWA,2561
-megfile-4.1.8.post1.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-megfile-4.1.8.post1.dist-info/licenses/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
-megfile-4.1.8.post1.dist-info/METADATA,sha256=7pRHrmLqja2l5oNGGx19IPRYbM-Z0ruvJ-znSi7NOVQ,9601
-megfile-4.1.8.post1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-megfile-4.1.8.post1.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
-megfile-4.1.8.post1.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
-megfile-4.1.8.post1.dist-info/RECORD,,
+megfile-4.2.1.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+megfile-4.2.1.dist-info/licenses/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
+megfile-4.2.1.dist-info/METADATA,sha256=8-Gzye2STZTocHieLky8XR0nAae8sHEmanRVeTWKWYM,9595
+megfile-4.2.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+megfile-4.2.1.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
+megfile-4.2.1.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
+megfile-4.2.1.dist-info/RECORD,,