megfile 4.1.3__py3-none-any.whl → 4.1.3.post2__py3-none-any.whl

megfile/s3_path.py

@@ -366,12 +366,9 @@ def _list_all_buckets(profile_name: Optional[str] = None) -> List[str]:
     return [content["Name"] for content in response["Buckets"]]
 
 
-def _parse_s3_url_ignore_brace(s3_url: str) -> Tuple[str, str]:
-    s3_url = fspath(s3_url)
-    s3_scheme, right_part = s3_url[:5], s3_url[5:]
-    if s3_scheme != "s3://":
-        raise ValueError("Not a s3 url: %r" % s3_url)
+def _parse_s3_url_ignore_brace(s3_pathname: str) -> Tuple[str, str]:
     left_brace = False
+    right_part = s3_pathname.split("://", maxsplit=1)[1]
     for current_index, current_character in enumerate(right_part):
         if current_character == "/" and left_brace is False:
             return right_part[:current_index], right_part[current_index + 1 :]
@@ -382,9 +379,14 @@ def _parse_s3_url_ignore_brace(s3_url: str) -> Tuple[str, str]:
     return right_part, ""
 
 
-def _group_s3path_by_bucket(
-    s3_pathname: str, profile_name: Optional[str] = None
-) -> List[str]:
+def _parse_s3_url_profile(s3_pathname: str) -> Tuple[str, Optional[str]]:
+    protocol = s3_pathname.split("://", maxsplit=1)[0]
+    profile_name = protocol[3:] if protocol.startswith("s3+") else None
+    return protocol, profile_name
+
+
+def _group_s3path_by_bucket(s3_pathname: str) -> List[str]:
+    protocol, profile_name = _parse_s3_url_profile(s3_pathname)
     bucket, key = _parse_s3_url_ignore_brace(s3_pathname)
     if not bucket:
         if not key:
@@ -395,8 +397,8 @@ def _group_s3path_by_bucket(
 
     def generate_s3_path(bucket: str, key: str) -> str:
         if key:
-            return "s3://%s/%s" % (bucket, key)
-        return "s3://%s%s" % (bucket, "/" if s3_pathname.endswith("/") else "")
+            return f"{protocol}://{bucket}/{key}"
+        return f"{protocol}://{bucket}{'/' if s3_pathname.endswith('/') else ''}"
 
     all_bucket = lru_cache(maxsize=1)(_list_all_buckets)
     for bucket_name in ungloblize(bucket):
@@ -418,23 +420,11 @@ def _group_s3path_by_bucket(
 
 
 def _s3_split_magic_ignore_brace(s3_pathname: str) -> Tuple[str, str]:
-    if not s3_pathname:
-        raise ValueError("s3_pathname: %s", s3_pathname)
-
-    has_protocol = False
-    if s3_pathname.startswith("s3://"):
-        has_protocol = True
-        s3_pathname = s3_pathname[5:]
-
-    has_delimiter = False
-    if s3_pathname.endswith("/"):
-        has_delimiter = True
-        s3_pathname = s3_pathname[:-1]
-
-    normal_parts = []
-    magic_parts = []
-    left_brace = False
-    left_index = 0
+    left_brace, left_index = False, 0
+    normal_parts, magic_parts = [], []
+    s3_pathname_with_suffix = s3_pathname
+    s3_pathname = s3_pathname.rstrip("/")
+    suffix = (len(s3_pathname_with_suffix) - len(s3_pathname)) * "/"
     for current_index, current_character in enumerate(s3_pathname):
         if current_character == "/" and left_brace is False:
             if has_magic_ignore_brace(s3_pathname[left_index:current_index]):
@@ -454,18 +444,13 @@ def _s3_split_magic_ignore_brace(s3_pathname: str) -> Tuple[str, str]:
             magic_parts.append(s3_pathname[left_index:])
         else:
             normal_parts.append(s3_pathname[left_index:])
-
-    if has_protocol and normal_parts:
-        normal_parts.insert(0, "s3:/")
-    elif has_protocol:
-        magic_parts.insert(0, "s3:/")
-
-    if has_delimiter and magic_parts:
-        magic_parts.append("")
-    elif has_delimiter:
-        normal_parts.append("")
-
-    return "/".join(normal_parts), "/".join(magic_parts)
+    top_dir, magic_part = "/".join(normal_parts), "/".join(magic_parts)
+    if suffix:
+        if magic_part:
+            magic_part += suffix
+        else:
+            top_dir += suffix
+    return top_dir, magic_part
 
 
 def _group_s3path_by_prefix(s3_pathname: str) -> List[str]:
@@ -493,17 +478,15 @@ def _become_prefix(prefix: str) -> str:
 def _s3_split_magic(s3_pathname: str) -> Tuple[str, str]:
     if not has_magic(s3_pathname):
         return s3_pathname, ""
-    delimiter = "/"
     normal_parts = []
     magic_parts = []
-    all_parts = s3_pathname.split(delimiter)
+    all_parts = s3_pathname.split("/")
     for i, part in enumerate(all_parts):
-        if not has_magic(part):
-            normal_parts.append(part)
-        else:
+        if has_magic(part):
             magic_parts = all_parts[i:]
             break
-    return delimiter.join(normal_parts), delimiter.join(magic_parts)
+        normal_parts.append(part)
+    return "/".join(normal_parts), "/".join(magic_parts)
 
 
 def _list_objects_recursive(s3_client, bucket: str, prefix: str, delimiter: str = ""):
@@ -574,12 +557,12 @@ def _s3_glob_stat_single_path(
     recursive: bool = True,
     missing_ok: bool = True,
     followlinks: bool = False,
-    profile_name: Optional[str] = None,
 ) -> Iterator[FileEntry]:
     s3_pathname = fspath(s3_pathname)
     if not recursive:
         # If not recursive, replace ** with *
         s3_pathname = re.sub(r"\*{2,}", "*", s3_pathname)
+    protocol, profile_name = _parse_s3_url_profile(s3_pathname)
     top_dir, wildcard_part = _s3_split_magic(s3_pathname)
     search_dir = wildcard_part.endswith("/")
 
@@ -596,7 +579,7 @@ def _s3_glob_stat_single_path(
         if not has_magic(_s3_pathname):
             _s3_pathname_obj = S3Path(_s3_pathname)
             if _s3_pathname_obj.is_file():
-                stat = S3Path(_s3_pathname).stat(follow_symlinks=followlinks)
+                stat = _s3_pathname_obj.stat(follow_symlinks=followlinks)
                 yield FileEntry(_s3_pathname_obj.name, _s3_pathname_obj.path, stat)
             if _s3_pathname_obj.is_dir():
                 yield FileEntry(
@@ -616,7 +599,7 @@ def _s3_glob_stat_single_path(
         with raise_s3_error(_s3_pathname, S3BucketNotFoundError):
             for resp in _list_objects_recursive(client, bucket, prefix, delimiter):
                 for content in resp.get("Contents", []):
-                    path = s3_path_join("s3://", bucket, content["Key"])
+                    path = s3_path_join(f"{protocol}://", bucket, content["Key"])
                     if not search_dir and pattern.match(path):
                         yield FileEntry(S3Path(path).name, path, _make_stat(content))
                     dirname = os.path.dirname(path)
@@ -629,7 +612,9 @@ def _s3_glob_stat_single_path(
                             )
                         dirname = os.path.dirname(dirname)
                 for common_prefix in resp.get("CommonPrefixes", []):
-                    path = s3_path_join("s3://", bucket, common_prefix["Prefix"])
+                    path = s3_path_join(
+                        f"{protocol}://", bucket, common_prefix["Prefix"]
+                    )
                     dirname = os.path.dirname(path)
                     if dirname not in dirnames and dirname != top_dir:
                         dirnames.add(dirname)
@@ -1595,26 +1580,19 @@ class S3Path(URIPath):
         :returns: A generator contains tuples of path and file stat,
             in which paths match `s3_pathname`
         """
-        glob_path = self._s3_path
+        glob_path = self.path_with_protocol
         if pattern:
-            glob_path = self.joinpath(pattern)._s3_path
+            glob_path = self.joinpath(pattern).path_with_protocol
         s3_pathname = fspath(glob_path)
 
         def create_generator():
-            for group_s3_pathname_1 in _group_s3path_by_bucket(
-                s3_pathname, self._profile_name
-            ):
+            for group_s3_pathname_1 in _group_s3path_by_bucket(s3_pathname):
                 for group_s3_pathname_2 in _group_s3path_by_prefix(group_s3_pathname_1):
                     for file_entry in _s3_glob_stat_single_path(
                         group_s3_pathname_2,
                         recursive,
                         missing_ok,
-                        profile_name=self._profile_name,
                     ):
-                        if self._profile_name:
-                            file_entry = file_entry._replace(
-                                path=f"{self._protocol_with_profile}://{file_entry.path[5:]}"
-                            )
                         yield file_entry
 
         return _create_missing_ok_generator(
@@ -2044,7 +2022,7 @@ class S3Path(URIPath):
                 for content in response["Buckets"]:
                     yield FileEntry(
                         content["Name"],
-                        f"s3://{content['Name']}",
+                        f"{self._protocol_with_profile}://{content['Name']}",
                         StatResult(
                             ctime=content["CreationDate"].timestamp(),
                             isdir=True,

megfile/version.py

@@ -1 +1 @@
-VERSION = "4.1.3"
+VERSION = "4.1.3.post2"

{megfile-4.1.3.dist-info → megfile-4.1.3.post2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: megfile
-Version: 4.1.3
+Version: 4.1.3.post2
 Summary: Megvii file operation library
 Author-email: megvii <megfile@megvii.com>
 Project-URL: Homepage, https://github.com/megvii-research/megfile

{megfile-4.1.3.dist-info → megfile-4.1.3.post2.dist-info}/RECORD

@@ -11,14 +11,14 @@ megfile/http_path.py,sha256=yRIk-fNbrsY8rUS5KVOfocS_PS520dX5KOs8lImpLaY,14173
 megfile/interfaces.py,sha256=p4UvVZpeLx5djd6bqqDaygIx_s-_AxIVj-gudTch4JE,8467
 megfile/pathlike.py,sha256=3Hnw-fn6RcIe9iPrJt00QdHSA--UfDyxnVBuZ_ymYYQ,31278
 megfile/s3.py,sha256=abBxnI7RIyn7n7qjGszP1VruYd6Gi9I8QnUOvsHkx1Y,16325
-megfile/s3_path.py,sha256=zelXhlRVOVSWBE6HJz0vXrrcRzSuj6Cnjd9HLGwPbCM,93644
+megfile/s3_path.py,sha256=Pkz5TjtY7-hh8Gtbd6cxef9c9EwYJCW5BqFo1rI1oog,93235
 megfile/sftp.py,sha256=uBcLQs-j6Q-q-sWAdd-pgi5Qmb_kq7boJM-0sCfcNO0,26540
 megfile/sftp_path.py,sha256=Wz4VcQ0pBUuWDGMSxPpPbutrT09mnY6jZNiAqTi5tO4,43840
 megfile/smart.py,sha256=Sae2KJzaU0k_qV_Bk0YifOMq8WsV5qQ2pGInDRF546I,36411
 megfile/smart_path.py,sha256=Up_6xNZ2019iSzMn_JAU_1H--z-AP6O7SxdXGdeTG0c,7659
 megfile/stdio.py,sha256=ZwxsnJNJYIT7Iyg5pIw4qiyH8bszG6oAhEJuR-hXGG4,658
 megfile/stdio_path.py,sha256=cxaDr8rtisTPnN-rjtaEpqQnshwiqwXFUJBM9xWY7Cg,2711
-megfile/version.py,sha256=VxmH81mTbRqT3UnqXh5OXAB3IfXRrcnMl0YgLgOqhPw,19
+megfile/version.py,sha256=SMg7F3UjVjLLIP3ibFt2p-0gK8lmy7rF2Y0Ehz9wx9M,25
 megfile/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 megfile/lib/base_prefetch_reader.py,sha256=uxVwYknOjc8hLF7q_T2QKMsBqFcrf411ZsuK25CN1eQ,12848
 megfile/lib/combine_reader.py,sha256=Kp2wEloOUpTlIU7dve87MBpSzmIM-F9OtpTawAjFkiU,4828
@@ -43,20 +43,10 @@ megfile/lib/stdio_handler.py,sha256=IDdgENLQlhigEwkLL4zStueVSzdWg7xVcTF_koof_Ek,
 megfile/lib/url.py,sha256=ER32pWy9Q2MAk3TraAaNEBWIqUeBmLuM57ol2cs7-Ks,103
 megfile/utils/__init__.py,sha256=xAzmICA0MtAbg-I2yPfeHjA1N4CiMP4sBrC9BgrfZLw,10151
 megfile/utils/mutex.py,sha256=asb8opGLgK22RiuBJUnfsvB8LnMmodP8KzCVHKmQBWA,2561
-megfile-4.1.3.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-megfile-4.1.3.dist-info/licenses/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
-scripts/benchmark/code/iopath_read.py,sha256=O1Qs3mpvD9S_XCuRH2A2FpGWxCOSw6qZvEBrtPeRL1E,661
-scripts/benchmark/code/iopath_write.py,sha256=Mm0efW1J09RJ_CK5i1xtG2hJuaaslikin8qVpuRFP_Q,704
-scripts/benchmark/code/megfile_read.py,sha256=sAMebUiMColHDv3JEkXplImAHvn_IF1-g3BIJxhcQYE,239
-scripts/benchmark/code/megfile_write.py,sha256=bzn-i2bGH4eRcsVvkhXK35KsQkX2v2oEsOJ0Ft5saj0,257
-scripts/benchmark/code/pyarrow_read.py,sha256=2QBGKjGV2Dvl2ukOntLSag33pF55l3tfZ2Z6dLUjovw,305
-scripts/benchmark/code/pyarrow_write.py,sha256=U1puLh-ljSXv772bZYAyhzmxhPOq4aR4j-QtwdM9hG0,328
-scripts/benchmark/code/s3fs_read.py,sha256=XiTA-qrYblUs-jQWXSnvNg5Wo722C_g47aMMfo5XJBY,380
-scripts/benchmark/code/s3fs_write.py,sha256=gdXKkWXYGjLJlRT_J64pJN85XvRg3bZexcAJQEMXwtw,402
-scripts/benchmark/code/smart_open_read.py,sha256=SA02jHwS9Y31yFtV9CoJcfND5dR0eA_HsGmGNUrpQls,515
-scripts/benchmark/code/smart_open_write.py,sha256=jDxFJdY97yNH889jz3pawBoei3yaqy8pEMvC_ymHFtM,537
-megfile-4.1.3.dist-info/METADATA,sha256=hKZKmMsgZAysVrZJ3aRQtiGRrvHnRfEPx68ayMEt3EM,9595
-megfile-4.1.3.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-megfile-4.1.3.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
-megfile-4.1.3.dist-info/top_level.txt,sha256=TR6xUw8Px5Ms_UENhEmLNmxOdfHAhTlSt9yTw9LRgsQ,35
-megfile-4.1.3.dist-info/RECORD,,
+megfile-4.1.3.post2.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+megfile-4.1.3.post2.dist-info/licenses/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
+megfile-4.1.3.post2.dist-info/METADATA,sha256=c4_nmj6sTfOJDTLB9kKn0c-st_kCklsMRXKOuQi6v2o,9601
+megfile-4.1.3.post2.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+megfile-4.1.3.post2.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
+megfile-4.1.3.post2.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
+megfile-4.1.3.post2.dist-info/RECORD,,

megfile-4.1.3.post2.dist-info/top_level.txt

@@ -0,0 +1 @@
+megfile

megfile-4.1.3.dist-info/top_level.txt

@@ -1,5 +0,0 @@
-dist
-docs
-html_cov
-megfile
-scripts

scripts/benchmark/code/iopath_read.py

@@ -1,29 +0,0 @@
-import os
-import time
-
-import boto3
-from iopath.common.file_io import PathManager
-from iopath.common.s3 import S3PathHandler
-
-times = 10240
-s3_path = "s3://bucketA/large.txt"
-
-start = time.time()
-
-path_manager = PathManager()
-
-session = boto3.Session(
-    aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
-    aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
-)
-client = session.client("s3", endpoint_url=os.environ["OSS_ENDPOINT"])
-handler = S3PathHandler()
-handler.client = client
-
-path_manager.register_handler(handler)
-
-with path_manager.open(s3_path, "rb") as f:
-    for i in range(times):
-        f.read(1024 * 1024)
-
-print(time.time() - start)

scripts/benchmark/code/iopath_write.py

@@ -1,30 +0,0 @@
-import os
-import time
-
-import boto3
-from iopath.common.file_io import PathManager
-from iopath.common.s3 import S3PathHandler
-
-times = 10240
-s3_path = "s3://bucketA/large.txt"
-block = b"1" * 1024 * 1024
-
-start = time.time()
-
-path_manager = PathManager()
-
-session = boto3.Session(
-    aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
-    aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
-)
-client = session.client("s3", endpoint_url=os.environ["OSS_ENDPOINT"])
-handler = S3PathHandler()
-handler.client = client
-
-path_manager.register_handler(handler)
-
-with path_manager.open(s3_path, "wb") as f:
-    for i in range(times):
-        f.write(block)
-
-print(time.time() - start)  # write 10GB 91.642

scripts/benchmark/code/megfile_read.py

@@ -1,13 +0,0 @@
-import time
-
-from megfile import smart_open
-
-times = 10240
-s3_path = "s3://bucketA/large.txt"
-
-start = time.time()
-with smart_open(s3_path, "rb") as f:
-    for i in range(times):
-        f.read(1024 * 1024 * 1)
-
-print(time.time() - start)

scripts/benchmark/code/megfile_write.py

@@ -1,14 +0,0 @@
-import time
-
-from megfile import smart_open
-
-times = 10240
-s3_path = "s3://bucketA/large.txt"
-block = b"1" * 1024 * 1024
-
-start = time.time()
-with smart_open(s3_path, "wb") as f:
-    for i in range(times):
-        f.write(block)
-
-print(time.time() - start)

scripts/benchmark/code/pyarrow_read.py

@@ -1,17 +0,0 @@
-import os
-import time
-
-from pyarrow import fs
-
-times = 10240
-s3_path = "bucketA/large.txt"
-
-start = time.time()
-
-s3 = fs.S3FileSystem(endpoint_override=os.environ["OSS_ENDPOINT"])
-
-with s3.open_input_stream(s3_path) as f:
-    for i in range(times):
-        f.read(1024 * 1024)
-
-print(time.time() - start)

scripts/benchmark/code/pyarrow_write.py

@@ -1,18 +0,0 @@
-import os
-import time
-
-from pyarrow import fs
-
-times = 10240
-block = b"1" * 1024 * 1024
-s3_path = "bucketA/large.txt"
-
-start = time.time()
-
-s3 = fs.S3FileSystem(endpoint_override=os.environ["OSS_ENDPOINT"])
-
-with s3.open_output_stream(s3_path) as f:
-    for i in range(times):
-        f.write(block)
-
-print(time.time() - start)

scripts/benchmark/code/s3fs_read.py

@@ -1,21 +0,0 @@
-import os
-import time
-
-import s3fs
-
-times = 10240
-s3_path = "bucketA/large.txt"
-
-start = time.time()
-
-s3 = s3fs.S3FileSystem(
-    endpoint_url=os.environ["OSS_ENDPOINT"],
-    key=os.environ["AWS_ACCESS_KEY_ID"],
-    secret=os.environ["AWS_SECRET_ACCESS_KEY"],
-)
-
-with s3.open(s3_path, "rb") as f:
-    for i in range(times):
-        f.read(1024 * 1024)
-
-print(time.time() - start)

scripts/benchmark/code/s3fs_write.py

@@ -1,22 +0,0 @@
-import os
-import time
-
-import s3fs
-
-times = 10240
-block = b"1" * 1024 * 1024
-s3_path = "bucketA/large.txt"
-
-start = time.time()
-
-s3 = s3fs.S3FileSystem(
-    endpoint_url=os.environ["OSS_ENDPOINT"],
-    key=os.environ["AWS_ACCESS_KEY_ID"],
-    secret=os.environ["AWS_SECRET_ACCESS_KEY"],
-)
-
-with s3.open(s3_path, "wb") as f:
-    for i in range(times):
-        f.write(block)
-
-print(time.time() - start)

scripts/benchmark/code/smart_open_read.py

@@ -1,25 +0,0 @@
-import os
-import time
-
-import boto3
-from smart_open import open
-
-times = 10240
-s3_path = "s3://bucketA/large.txt"
-
-start = time.time()
-session = boto3.Session(
-    aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
-    aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
-)
-with open(
-    s3_path,
-    "rb",
-    transport_params={
-        "client": session.client("s3", endpoint_url=os.environ["OSS_ENDPOINT"])
-    },
-) as f:
-    for i in range(times):
-        f.read(1024 * 1024)
-
-print(time.time() - start)

scripts/benchmark/code/smart_open_write.py

@@ -1,26 +0,0 @@
-import os
-import time
-
-import boto3
-from smart_open import open
-
-times = 10240
-s3_path = "s3://bucketA/large.txt"
-block = b"1" * 1024 * 1024
-
-start = time.time()
-session = boto3.Session(
-    aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
-    aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
-)
-with open(
-    s3_path,
-    "wb",
-    transport_params={
-        "client": session.client("s3", endpoint_url=os.environ["OSS_ENDPOINT"])
-    },
-) as f:
-    for i in range(times):
-        f.write(block)
-
-print(time.time() - start)