megfile 4.1.8__py3-none-any.whl → 4.2.0__py3-none-any.whl

megfile/cli.py

@@ -70,17 +70,16 @@ def cli(debug, log_level):
     """
     options["debug"] = debug
     options["log_level"] = log_level or ("DEBUG" if debug else "INFO")
+    if not debug:
+        signal.signal(signal.SIGINT, signal.SIG_DFL)
     set_log_level(options["log_level"])
 
 
 def safe_cli():  # pragma: no cover
-    debug = options.get("debug", False)
-    if not debug:
-        signal.signal(signal.SIGINT, signal.SIG_DFL)
     try:
         cli()
     except Exception as e:
-        if debug:
+        if options.get("debug", False):
             raise
         else:
             click.echo(f"\n[{type(e).__name__}] {e}", err=True)

megfile/http_path.py

@@ -187,11 +187,12 @@ class HttpPath(URIPath):
                 response.close()
             raise translate_http_error(error, self.path_with_protocol)
 
-        content_size = int(response.headers["Content-Length"])
+        headers = response.headers
+        content_size = int(headers.get("Content-Length", 0))
         if (
-            response.headers.get("Accept-Ranges") == "bytes"
+            headers.get("Accept-Ranges") == "bytes"
             and content_size >= block_size * 2
-            and not response.headers.get("Content-Encoding")
+            and not headers.get("Content-Encoding")
         ):
             response.close()
 

megfile/s3_path.py

@@ -605,6 +605,8 @@ def _s3_glob_stat_single_path(
                 for content in resp.get("Contents", []):
                     path = s3_path_join(f"{protocol}://", bucket, content["Key"])
                     if not search_dir and pattern.match(path):
+                        if path.endswith("/"):
+                            continue
                         yield FileEntry(S3Path(path).name, path, _make_stat(content))
                     dirname = os.path.dirname(path)
                     while dirname not in dirnames and dirname != top_dir:
@@ -1960,6 +1962,7 @@ class S3Path(URIPath):
                 )
 
             prefix = _become_prefix(key)
+            protocol = self._protocol_with_profile
             client = self._client
 
             def suppress_error_callback(e):
@@ -1970,13 +1973,13 @@ class S3Path(URIPath):
             with raise_s3_error(self.path_with_protocol, suppress_error_callback):
                 for resp in _list_objects_recursive(client, bucket, prefix):
                     for content in resp.get("Contents", []):
-                        full_path = s3_path_join(
-                            f"{self._protocol_with_profile}://", bucket, content["Key"]
-                        )
+                        if content["Key"].endswith("/"):
+                            continue
+                        path = s3_path_join(f"{protocol}://", bucket, content["Key"])
 
                         if followlinks:
                             try:
-                                origin_path = self.from_path(full_path).readlink()
+                                origin_path = self.from_path(path).readlink()
                                 yield FileEntry(
                                     origin_path.name,
                                     origin_path.path_with_protocol,
@@ -1986,9 +1989,7 @@ class S3Path(URIPath):
                             except S3NotALinkError:
                                 pass
 
-                        yield FileEntry(
-                            S3Path(full_path).name, full_path, _make_stat(content)
-                        )
+                        yield FileEntry(S3Path(path).name, path, _make_stat(content))
 
         return _create_missing_ok_generator(
             create_generator(),
@@ -2016,17 +2017,15 @@ class S3Path(URIPath):
         # we need to wrap the iterator in another function
         def create_generator() -> Iterator[FileEntry]:
             prefix = _become_prefix(key)
+            protocol = self._protocol_with_profile
             client = self._client
 
-            def generate_s3_path(protocol: str, bucket: str, key: str) -> str:
-                return "%s://%s/%s" % (protocol, bucket, key)
-
             if not bucket and not key:  # list buckets
                 response = client.list_buckets()
                 for content in response["Buckets"]:
                     yield FileEntry(
                         content["Name"],
-                        f"{self._protocol_with_profile}://{content['Name']}",
+                        f"{protocol}://{content['Name']}",
                         StatResult(
                             ctime=content["CreationDate"].timestamp(),
                             isdir=True,
@@ -2039,21 +2038,17 @@ class S3Path(URIPath):
                 for common_prefix in resp.get("CommonPrefixes", []):
                     yield FileEntry(
                         common_prefix["Prefix"][len(prefix) : -1],
-                        generate_s3_path(
-                            self._protocol_with_profile,
-                            bucket,
-                            common_prefix["Prefix"],
-                        ),
+                        f"{protocol}://{bucket}/{common_prefix['Prefix']}",
                         StatResult(isdir=True, extra=common_prefix),
                     )
                 for content in resp.get("Contents", []):
-                    src_url = generate_s3_path(
-                        self._protocol_with_profile, bucket, content["Key"]
-                    )
+                    if content["Key"].endswith("/"):
+                        continue
+                    path = f"{protocol}://{bucket}/{content['Key']}"
                     yield FileEntry(  # pytype: disable=wrong-arg-types
                         content["Key"][len(prefix) :],
-                        src_url,
-                        _make_stat_without_metadata(content, self.from_path(src_url)),
+                        path,
+                        _make_stat_without_metadata(content, self.from_path(path)),
                     )
 
         def missing_ok_generator():
@@ -2220,6 +2215,8 @@ class S3Path(URIPath):
                     for common_prefix in resp.get("CommonPrefixes", []):
                         dirs.append(common_prefix["Prefix"][:-1])
                     for content in resp.get("Contents", []):
+                        if content["Key"].endswith("/"):
+                            continue
                         files.append(content["Key"])
 
                 dirs = sorted(dirs)

megfile/version.py

@@ -1 +1 @@
-VERSION = "4.1.8"
+VERSION = "4.2.0"

{megfile-4.1.8.dist-info → megfile-4.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: megfile
-Version: 4.1.8
+Version: 4.2.0
 Summary: Megvii file operation library
 Author-email: megvii <megfile@megvii.com>
 Project-URL: Homepage, https://github.com/megvii-research/megfile

{megfile-4.1.8.dist-info → megfile-4.2.0.dist-info}/RECORD

@@ -1,5 +1,5 @@
 megfile/__init__.py,sha256=7oEfu410CFKzDWZ9RjL5xEJ1gtkJkTfvPrL_7TWdJuY,7366
-megfile/cli.py,sha256=iwwlwVNu_yXgqnaURPHqGPrx1WeReqN0zi9C4rqK5Ag,29178
+megfile/cli.py,sha256=VxY0__M19Ti_S7ZIozp9l0FxvdLwdd6eQL-wfpYOi_0,29160
 megfile/config.py,sha256=2MMj5QkhlDJQFZRbCQL2c9iDdeMAVctiaPszRBkg5vM,3988
 megfile/errors.py,sha256=aw4BX-pU1uj0-dNc6Tq-IeSwK_PWJNi2ZL5ZcRSo7aQ,14739
 megfile/fs.py,sha256=KMEqAE35alpcxiy6du5nPFYcaorhUM_kPJMah3q76ng,19160
@@ -7,18 +7,18 @@ megfile/fs_path.py,sha256=Hozl9LAJ8EMuSWBSZXGj2GNmPZ1sJp9PZs-7hPrLgm8,39341
 megfile/hdfs.py,sha256=owXr4d3j1frCvlbhmhENcSBnKKDky5cJZzWLOF4ZJMo,13251
 megfile/hdfs_path.py,sha256=OmUe3vA3Qoxnqtcq0Rs3ygBvzAtqUz3fGo8iP5sWneE,26058
 megfile/http.py,sha256=1nuGe-JbnwMFyV3s35CJxByED3uoRoS9y8Y8cSGP9Kw,3865
-megfile/http_path.py,sha256=ZGug-bTWq8GqEA-R6zIXQmbcOVYvILxEweTXzHGnBuk,13829
+megfile/http_path.py,sha256=08OmzmRMyLSyq1Yr1K2HbzexesURJrIoA6AibwYzUiA,13844
 megfile/interfaces.py,sha256=p4UvVZpeLx5djd6bqqDaygIx_s-_AxIVj-gudTch4JE,8467
 megfile/pathlike.py,sha256=3Hnw-fn6RcIe9iPrJt00QdHSA--UfDyxnVBuZ_ymYYQ,31278
 megfile/s3.py,sha256=abBxnI7RIyn7n7qjGszP1VruYd6Gi9I8QnUOvsHkx1Y,16325
-megfile/s3_path.py,sha256=4vKdn5RupvSMYelYznCMDpE1t-87H3iwgzY4iZHWmIE,93519
+megfile/s3_path.py,sha256=nA1FXFwfhSvZN8W0mF2PKGbnaTDouZfp6anUZkxvOBc,93449
 megfile/sftp.py,sha256=uBcLQs-j6Q-q-sWAdd-pgi5Qmb_kq7boJM-0sCfcNO0,26540
 megfile/sftp_path.py,sha256=CgirHWmNdXdqyIL9ufmlaMpwFhlkQVZhqmfvjUaj7qU,43845
 megfile/smart.py,sha256=Miz3jyLKmwWBja-8GrSzrumpTarPrFPqXaFQJKwrK1Y,36627
 megfile/smart_path.py,sha256=Up_6xNZ2019iSzMn_JAU_1H--z-AP6O7SxdXGdeTG0c,7659
 megfile/stdio.py,sha256=ZwxsnJNJYIT7Iyg5pIw4qiyH8bszG6oAhEJuR-hXGG4,658
 megfile/stdio_path.py,sha256=cxaDr8rtisTPnN-rjtaEpqQnshwiqwXFUJBM9xWY7Cg,2711
-megfile/version.py,sha256=Q7vzX4XpnfI8xIxWcYQYkZJUE1GiBtK_NmvaAfIJfC8,19
+megfile/version.py,sha256=nmN7kc0bNHgtVq4jv4MJKEo-L3X7QgGHAv_z6zkMVSQ,19
 megfile/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 megfile/lib/base_prefetch_reader.py,sha256=uxVwYknOjc8hLF7q_T2QKMsBqFcrf411ZsuK25CN1eQ,12848
 megfile/lib/combine_reader.py,sha256=Kp2wEloOUpTlIU7dve87MBpSzmIM-F9OtpTawAjFkiU,4828
@@ -43,10 +43,10 @@ megfile/lib/stdio_handler.py,sha256=IDdgENLQlhigEwkLL4zStueVSzdWg7xVcTF_koof_Ek,
 megfile/lib/url.py,sha256=ER32pWy9Q2MAk3TraAaNEBWIqUeBmLuM57ol2cs7-Ks,103
 megfile/utils/__init__.py,sha256=pawmXnCNokWLj338a60b_hK21koYavpEiEohZhsOaGQ,10156
 megfile/utils/mutex.py,sha256=asb8opGLgK22RiuBJUnfsvB8LnMmodP8KzCVHKmQBWA,2561
-megfile-4.1.8.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-megfile-4.1.8.dist-info/licenses/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
-megfile-4.1.8.dist-info/METADATA,sha256=tuE4RMIMSYdGykVRvC0SoEGNl6N0xvv5KsNWXk7xhG4,9595
-megfile-4.1.8.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-megfile-4.1.8.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
-megfile-4.1.8.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
-megfile-4.1.8.dist-info/RECORD,,
+megfile-4.2.0.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+megfile-4.2.0.dist-info/licenses/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
+megfile-4.2.0.dist-info/METADATA,sha256=0IuIzy3B0sO0YhPrMdEK5kCkK16K1ZCX6w3vZO-SlB4,9595
+megfile-4.2.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+megfile-4.2.0.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
+megfile-4.2.0.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
+megfile-4.2.0.dist-info/RECORD,,