megfile 4.1.3.post1__py3-none-any.whl → 4.1.4__py3-none-any.whl

megfile/s3_path.py

@@ -290,14 +290,23 @@ def get_s3_client(
             profile_name=profile_name,
         )
 
+    try:
+        default_config = botocore.config.Config(
+            connect_timeout=5,
+            max_pool_connections=GLOBAL_MAX_WORKERS,
+            request_checksum_calculation="when_required",
+            response_checksum_validation="when_required",
+        )
+    except TypeError:  # botocore < 1.36.0
+        default_config = botocore.config.Config(
+            connect_timeout=5,
+            max_pool_connections=GLOBAL_MAX_WORKERS,
+        )
+
     if config:
-        config = botocore.config.Config(
-            connect_timeout=5, max_pool_connections=GLOBAL_MAX_WORKERS
-        ).merge(config)
+        config = default_config.merge(config)
     else:
-        config = botocore.config.Config(
-            connect_timeout=5, max_pool_connections=GLOBAL_MAX_WORKERS
-        )
+        config = default_config
 
     addressing_style = get_env_var("AWS_S3_ADDRESSING_STYLE", profile_name=profile_name)
     if addressing_style:
@@ -366,12 +375,9 @@ def _list_all_buckets(profile_name: Optional[str] = None) -> List[str]:
     return [content["Name"] for content in response["Buckets"]]
 
 
-def _parse_s3_url_ignore_brace(s3_url: str) -> Tuple[str, str]:
-    s3_url = fspath(s3_url)
-    s3_scheme, right_part = s3_url[:5], s3_url[5:]
-    if s3_scheme != "s3://":
-        raise ValueError("Not a s3 url: %r" % s3_url)
+def _parse_s3_url_ignore_brace(s3_pathname: str) -> Tuple[str, str]:
     left_brace = False
+    right_part = s3_pathname.split("://", maxsplit=1)[1]
     for current_index, current_character in enumerate(right_part):
         if current_character == "/" and left_brace is False:
             return right_part[:current_index], right_part[current_index + 1 :]
@@ -382,9 +388,14 @@ def _parse_s3_url_ignore_brace(s3_url: str) -> Tuple[str, str]:
     return right_part, ""
 
 
-def _group_s3path_by_bucket(
-    s3_pathname: str, profile_name: Optional[str] = None
-) -> List[str]:
+def _parse_s3_url_profile(s3_pathname: str) -> Tuple[str, Optional[str]]:
+    protocol = s3_pathname.split("://", maxsplit=1)[0]
+    profile_name = protocol[3:] if protocol.startswith("s3+") else None
+    return protocol, profile_name
+
+
+def _group_s3path_by_bucket(s3_pathname: str) -> List[str]:
+    protocol, profile_name = _parse_s3_url_profile(s3_pathname)
     bucket, key = _parse_s3_url_ignore_brace(s3_pathname)
     if not bucket:
         if not key:
@@ -395,8 +406,8 @@ def _group_s3path_by_bucket(
 
     def generate_s3_path(bucket: str, key: str) -> str:
         if key:
-            return "s3://%s/%s" % (bucket, key)
-        return "s3://%s%s" % (bucket, "/" if s3_pathname.endswith("/") else "")
+            return f"{protocol}://{bucket}/{key}"
+        return f"{protocol}://{bucket}{'/' if s3_pathname.endswith('/') else ''}"
 
     all_bucket = lru_cache(maxsize=1)(_list_all_buckets)
     for bucket_name in ungloblize(bucket):
@@ -418,23 +429,11 @@ def _group_s3path_by_bucket(
 
 
 def _s3_split_magic_ignore_brace(s3_pathname: str) -> Tuple[str, str]:
-    if not s3_pathname:
-        raise ValueError("s3_pathname: %s", s3_pathname)
-
-    has_protocol = False
-    if s3_pathname.startswith("s3://"):
-        has_protocol = True
-        s3_pathname = s3_pathname[5:]
-
-    has_delimiter = False
-    if s3_pathname.endswith("/"):
-        has_delimiter = True
-        s3_pathname = s3_pathname[:-1]
-
-    normal_parts = []
-    magic_parts = []
-    left_brace = False
-    left_index = 0
+    left_brace, left_index = False, 0
+    normal_parts, magic_parts = [], []
+    s3_pathname_with_suffix = s3_pathname
+    s3_pathname = s3_pathname.rstrip("/")
+    suffix = (len(s3_pathname_with_suffix) - len(s3_pathname)) * "/"
     for current_index, current_character in enumerate(s3_pathname):
         if current_character == "/" and left_brace is False:
             if has_magic_ignore_brace(s3_pathname[left_index:current_index]):
@@ -454,18 +453,13 @@ def _s3_split_magic_ignore_brace(s3_pathname: str) -> Tuple[str, str]:
             magic_parts.append(s3_pathname[left_index:])
         else:
             normal_parts.append(s3_pathname[left_index:])
-
-    if has_protocol and normal_parts:
-        normal_parts.insert(0, "s3:/")
-    elif has_protocol:
-        magic_parts.insert(0, "s3:/")
-
-    if has_delimiter and magic_parts:
-        magic_parts.append("")
-    elif has_delimiter:
-        normal_parts.append("")
-
-    return "/".join(normal_parts), "/".join(magic_parts)
+    top_dir, magic_part = "/".join(normal_parts), "/".join(magic_parts)
+    if suffix:
+        if magic_part:
+            magic_part += suffix
+        else:
+            top_dir += suffix
+    return top_dir, magic_part
 
 
 def _group_s3path_by_prefix(s3_pathname: str) -> List[str]:
@@ -493,17 +487,15 @@ def _become_prefix(prefix: str) -> str:
 def _s3_split_magic(s3_pathname: str) -> Tuple[str, str]:
     if not has_magic(s3_pathname):
         return s3_pathname, ""
-    delimiter = "/"
     normal_parts = []
     magic_parts = []
-    all_parts = s3_pathname.split(delimiter)
+    all_parts = s3_pathname.split("/")
     for i, part in enumerate(all_parts):
-        if not has_magic(part):
-            normal_parts.append(part)
-        else:
+        if has_magic(part):
             magic_parts = all_parts[i:]
             break
-    return delimiter.join(normal_parts), delimiter.join(magic_parts)
+        normal_parts.append(part)
+    return "/".join(normal_parts), "/".join(magic_parts)
 
 
 def _list_objects_recursive(s3_client, bucket: str, prefix: str, delimiter: str = ""):
@@ -574,12 +566,12 @@ def _s3_glob_stat_single_path(
     recursive: bool = True,
     missing_ok: bool = True,
     followlinks: bool = False,
-    profile_name: Optional[str] = None,
 ) -> Iterator[FileEntry]:
     s3_pathname = fspath(s3_pathname)
     if not recursive:
         # If not recursive, replace ** with *
         s3_pathname = re.sub(r"\*{2,}", "*", s3_pathname)
+    protocol, profile_name = _parse_s3_url_profile(s3_pathname)
     top_dir, wildcard_part = _s3_split_magic(s3_pathname)
     search_dir = wildcard_part.endswith("/")
 
@@ -596,7 +588,7 @@ def _s3_glob_stat_single_path(
         if not has_magic(_s3_pathname):
             _s3_pathname_obj = S3Path(_s3_pathname)
             if _s3_pathname_obj.is_file():
-                stat = S3Path(_s3_pathname).stat(follow_symlinks=followlinks)
+                stat = _s3_pathname_obj.stat(follow_symlinks=followlinks)
                 yield FileEntry(_s3_pathname_obj.name, _s3_pathname_obj.path, stat)
             if _s3_pathname_obj.is_dir():
                 yield FileEntry(
@@ -616,7 +608,7 @@ def _s3_glob_stat_single_path(
         with raise_s3_error(_s3_pathname, S3BucketNotFoundError):
             for resp in _list_objects_recursive(client, bucket, prefix, delimiter):
                 for content in resp.get("Contents", []):
-                    path = s3_path_join("s3://", bucket, content["Key"])
+                    path = s3_path_join(f"{protocol}://", bucket, content["Key"])
                     if not search_dir and pattern.match(path):
                         yield FileEntry(S3Path(path).name, path, _make_stat(content))
                     dirname = os.path.dirname(path)
@@ -629,7 +621,9 @@ def _s3_glob_stat_single_path(
                             )
                         dirname = os.path.dirname(dirname)
                 for common_prefix in resp.get("CommonPrefixes", []):
-                    path = s3_path_join("s3://", bucket, common_prefix["Prefix"])
+                    path = s3_path_join(
+                        f"{protocol}://", bucket, common_prefix["Prefix"]
+                    )
                     dirname = os.path.dirname(path)
                     if dirname not in dirnames and dirname != top_dir:
                         dirnames.add(dirname)
@@ -1595,26 +1589,19 @@ class S3Path(URIPath):
         :returns: A generator contains tuples of path and file stat,
             in which paths match `s3_pathname`
         """
-        glob_path = self._s3_path
+        glob_path = self.path_with_protocol
         if pattern:
-            glob_path = self.joinpath(pattern)._s3_path
+            glob_path = self.joinpath(pattern).path_with_protocol
         s3_pathname = fspath(glob_path)
 
         def create_generator():
-            for group_s3_pathname_1 in _group_s3path_by_bucket(
-                s3_pathname, self._profile_name
-            ):
+            for group_s3_pathname_1 in _group_s3path_by_bucket(s3_pathname):
                 for group_s3_pathname_2 in _group_s3path_by_prefix(group_s3_pathname_1):
                     for file_entry in _s3_glob_stat_single_path(
                         group_s3_pathname_2,
                         recursive,
                         missing_ok,
-                        profile_name=self._profile_name,
                     ):
-                        if self._profile_name:
-                            file_entry = file_entry._replace(
-                                path=f"{self._protocol_with_profile}://{file_entry.path[5:]}"
-                            )
                         yield file_entry
 
         return _create_missing_ok_generator(
@@ -2044,7 +2031,7 @@ class S3Path(URIPath):
                 for content in response["Buckets"]:
                     yield FileEntry(
                         content["Name"],
-                        f"s3://{content['Name']}",
+                        f"{self._protocol_with_profile}://{content['Name']}",
                         StatResult(
                             ctime=content["CreationDate"].timestamp(),
                             isdir=True,

megfile/version.py

@@ -1 +1 @@
-VERSION = "4.1.3.post1"
+VERSION = "4.1.4"

{megfile-4.1.3.post1.dist-info → megfile-4.1.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: megfile
-Version: 4.1.3.post1
+Version: 4.1.4
 Summary: Megvii file operation library
 Author-email: megvii <megfile@megvii.com>
 Project-URL: Homepage, https://github.com/megvii-research/megfile

{megfile-4.1.3.post1.dist-info → megfile-4.1.4.dist-info}/RECORD

@@ -11,14 +11,14 @@ megfile/http_path.py,sha256=yRIk-fNbrsY8rUS5KVOfocS_PS520dX5KOs8lImpLaY,14173
 megfile/interfaces.py,sha256=p4UvVZpeLx5djd6bqqDaygIx_s-_AxIVj-gudTch4JE,8467
 megfile/pathlike.py,sha256=3Hnw-fn6RcIe9iPrJt00QdHSA--UfDyxnVBuZ_ymYYQ,31278
 megfile/s3.py,sha256=abBxnI7RIyn7n7qjGszP1VruYd6Gi9I8QnUOvsHkx1Y,16325
-megfile/s3_path.py,sha256=zelXhlRVOVSWBE6HJz0vXrrcRzSuj6Cnjd9HLGwPbCM,93644
+megfile/s3_path.py,sha256=S8iulho1btVLLvNU-OtcskkbaAC8yNXnBrFNnF3fwS8,93510
 megfile/sftp.py,sha256=uBcLQs-j6Q-q-sWAdd-pgi5Qmb_kq7boJM-0sCfcNO0,26540
 megfile/sftp_path.py,sha256=Wz4VcQ0pBUuWDGMSxPpPbutrT09mnY6jZNiAqTi5tO4,43840
 megfile/smart.py,sha256=Sae2KJzaU0k_qV_Bk0YifOMq8WsV5qQ2pGInDRF546I,36411
 megfile/smart_path.py,sha256=Up_6xNZ2019iSzMn_JAU_1H--z-AP6O7SxdXGdeTG0c,7659
 megfile/stdio.py,sha256=ZwxsnJNJYIT7Iyg5pIw4qiyH8bszG6oAhEJuR-hXGG4,658
 megfile/stdio_path.py,sha256=cxaDr8rtisTPnN-rjtaEpqQnshwiqwXFUJBM9xWY7Cg,2711
-megfile/version.py,sha256=eKLHLHdKHcrWOl_XPErEVRYmoIEhUtehLCugzxSUR-E,25
+megfile/version.py,sha256=oeG77LWvRtozNToerGYb2jXBRGCoB_YTBpQNMOpT1Gc,19
 megfile/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 megfile/lib/base_prefetch_reader.py,sha256=uxVwYknOjc8hLF7q_T2QKMsBqFcrf411ZsuK25CN1eQ,12848
 megfile/lib/combine_reader.py,sha256=Kp2wEloOUpTlIU7dve87MBpSzmIM-F9OtpTawAjFkiU,4828
@@ -43,10 +43,10 @@ megfile/lib/stdio_handler.py,sha256=IDdgENLQlhigEwkLL4zStueVSzdWg7xVcTF_koof_Ek,
 megfile/lib/url.py,sha256=ER32pWy9Q2MAk3TraAaNEBWIqUeBmLuM57ol2cs7-Ks,103
 megfile/utils/__init__.py,sha256=xAzmICA0MtAbg-I2yPfeHjA1N4CiMP4sBrC9BgrfZLw,10151
 megfile/utils/mutex.py,sha256=asb8opGLgK22RiuBJUnfsvB8LnMmodP8KzCVHKmQBWA,2561
-megfile-4.1.3.post1.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-megfile-4.1.3.post1.dist-info/licenses/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
-megfile-4.1.3.post1.dist-info/METADATA,sha256=oI1b4MMW9GRz3mav_TZ2MSX6C8y1z1QebHkcCZcrcB0,9601
-megfile-4.1.3.post1.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-megfile-4.1.3.post1.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
-megfile-4.1.3.post1.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
-megfile-4.1.3.post1.dist-info/RECORD,,
+megfile-4.1.4.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+megfile-4.1.4.dist-info/licenses/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
+megfile-4.1.4.dist-info/METADATA,sha256=eK8epRuIQh0igo6DhBw30mRrSw4ssvgt4_voFo-zJYc,9595
+megfile-4.1.4.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+megfile-4.1.4.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
+megfile-4.1.4.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
+megfile-4.1.4.dist-info/RECORD,,