megfile 4.1.6__py3-none-any.whl → 4.1.8__py3-none-any.whl

megfile/lib/glob.py

@@ -247,19 +247,8 @@ def globlize(path_list: List[str]) -> str:
     path_list = sorted(path_list)
     if path_list[0] == path_list[-1]:
         return path_list[0]
-    first_path = path_list[0].split("/")
-    last_path = path_list[-1].split("/")
-    prefix = []
 
-    for i in range(0, min(len(first_path), len(last_path))):
-        if first_path[i] == last_path[i]:
-            prefix.append(first_path[i])
-        else:
-            break
-    if len(prefix) == 0:
-        prefix = ""
-    else:
-        prefix = "/".join(prefix) + "/"
+    prefix = os.path.commonprefix(path_list)
     suffix = _find_suffix(path_list, prefix, "/")
 
     if len(suffix) == 0:

megfile/s3_path.py

@@ -215,10 +215,12 @@ def get_endpoint_url(profile_name: Optional[str] = None) -> str:
 
     :returns: S3 endpoint url
     """
+    profile_name = profile_name or os.environ.get("AWS_PROFILE")
+    environ_keys = ("OSS_ENDPOINT", "AWS_ENDPOINT_URL_S3", "AWS_ENDPOINT_URL")
     if profile_name:
-        environ_keys = (f"{profile_name}__OSS_ENDPOINT".upper(),)
-    else:
-        environ_keys = ("OSS_ENDPOINT", "AWS_ENDPOINT_URL_S3", "AWS_ENDPOINT_URL")
+        environ_keys = tuple(
+            f"{profile_name}__{environ_key}".upper() for environ_key in environ_keys
+        )
     for environ_key in environ_keys:
         environ_endpoint_url = os.environ.get(environ_key)
         if environ_endpoint_url:
@@ -244,6 +246,7 @@ def get_s3_session(profile_name=None) -> boto3.Session:
 
 
 def get_env_var(env_name: str, profile_name=None):
+    profile_name = profile_name or os.environ.get("AWS_PROFILE")
     if profile_name:
         return os.getenv(f"{profile_name}__{env_name}".upper())
     return os.getenv(env_name.upper())
@@ -257,16 +260,15 @@ def get_access_token(profile_name=None):
         return access_key, secret_key, session_token
 
     try:
-        credentials = get_s3_session(profile_name=profile_name).get_credentials()
-    except botocore.exceptions.ProfileNotFound:
-        credentials = None
-    if credentials:
+        config = get_scoped_config(profile_name=profile_name)
         if not access_key:
-            access_key = credentials.access_key
+            access_key = config.get("aws_access_key_id")
         if not secret_key:
-            secret_key = credentials.secret_key
+            secret_key = config.get("aws_secret_access_key")
         if not session_token:
-            session_token = credentials.token
+            session_token = config.get("aws_session_token")
+    except botocore.exceptions.ProfileNotFound:
+        pass
     return access_key, secret_key, session_token
 
 
@@ -485,17 +487,10 @@ def _become_prefix(prefix: str) -> str:
 
 
 def _s3_split_magic(s3_pathname: str) -> Tuple[str, str]:
-    if not has_magic(s3_pathname):
-        return s3_pathname, ""
-    normal_parts = []
-    magic_parts = []
-    all_parts = s3_pathname.split("/")
-    for i, part in enumerate(all_parts):
-        if has_magic(part):
-            magic_parts = all_parts[i:]
-            break
-        normal_parts.append(part)
-    return "/".join(normal_parts), "/".join(magic_parts)
+    for i in range(len(s3_pathname) - 1, 0, -1):
+        if not has_magic(s3_pathname[:i]):
+            return s3_pathname[:i], s3_pathname[i:]
+    return s3_pathname, ""
 
 
 def _list_objects_recursive(s3_client, bucket: str, prefix: str, delimiter: str = ""):
@@ -572,7 +567,8 @@ def _s3_glob_stat_single_path(
         # If not recursive, replace ** with *
         s3_pathname = re.sub(r"\*{2,}", "*", s3_pathname)
     protocol, profile_name = _parse_s3_url_profile(s3_pathname)
-    top_dir, wildcard_part = _s3_split_magic(s3_pathname)
+    top_prefix, wildcard_part = _s3_split_magic(s3_pathname)
+    top_dir = os.path.dirname(top_prefix) if wildcard_part else top_prefix
     search_dir = wildcard_part.endswith("/")
 
     def should_recursive(wildcard_part: str) -> bool:
@@ -602,8 +598,7 @@ def _s3_glob_stat_single_path(
 
         dirnames = set()
         pattern = re.compile(translate(_s3_pathname))
-        bucket, key = parse_s3_url(top_dir)
-        prefix = _become_prefix(key)
+        bucket, prefix = parse_s3_url(top_prefix)
         client = get_s3_client_with_cache(profile_name=profile_name)
         with raise_s3_error(_s3_pathname, S3BucketNotFoundError):
             for resp in _list_objects_recursive(client, bucket, prefix, delimiter):

megfile/smart.py

@@ -307,6 +307,16 @@ def _default_copy_func(
         pass
 
 
+def _get_copy_func(src_protocol, dst_protocol):
+    if src_protocol.startswith("s3+") and src_protocol == dst_protocol:
+        src_protocol = dst_protocol = "s3"
+
+    try:
+        return _copy_funcs[src_protocol][dst_protocol]
+    except KeyError:
+        return _default_copy_func
+
+
 def smart_copy(
     src_path: PathLike,
     dst_path: PathLike,
@@ -351,10 +361,8 @@ def smart_copy(
     src_protocol, _ = SmartPath._extract_protocol(src_path)
     dst_protocol, _ = SmartPath._extract_protocol(dst_path)
 
-    try:
-        copy_func = _copy_funcs[src_protocol][dst_protocol]
-    except KeyError:
-        copy_func = _default_copy_func
+    copy_func = _get_copy_func(src_protocol, dst_protocol)
+
     try:
         copy_func(
             src_path,

megfile/version.py

@@ -1 +1 @@
-VERSION = "4.1.6"
+VERSION = "4.1.8"

{megfile-4.1.6.dist-info → megfile-4.1.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: megfile
-Version: 4.1.6
+Version: 4.1.8
 Summary: Megvii file operation library
 Author-email: megvii <megfile@megvii.com>
 Project-URL: Homepage, https://github.com/megvii-research/megfile

{megfile-4.1.6.dist-info → megfile-4.1.8.dist-info}/RECORD

@@ -11,21 +11,21 @@ megfile/http_path.py,sha256=ZGug-bTWq8GqEA-R6zIXQmbcOVYvILxEweTXzHGnBuk,13829
 megfile/interfaces.py,sha256=p4UvVZpeLx5djd6bqqDaygIx_s-_AxIVj-gudTch4JE,8467
 megfile/pathlike.py,sha256=3Hnw-fn6RcIe9iPrJt00QdHSA--UfDyxnVBuZ_ymYYQ,31278
 megfile/s3.py,sha256=abBxnI7RIyn7n7qjGszP1VruYd6Gi9I8QnUOvsHkx1Y,16325
-megfile/s3_path.py,sha256=S8iulho1btVLLvNU-OtcskkbaAC8yNXnBrFNnF3fwS8,93510
+megfile/s3_path.py,sha256=4vKdn5RupvSMYelYznCMDpE1t-87H3iwgzY4iZHWmIE,93519
 megfile/sftp.py,sha256=uBcLQs-j6Q-q-sWAdd-pgi5Qmb_kq7boJM-0sCfcNO0,26540
 megfile/sftp_path.py,sha256=CgirHWmNdXdqyIL9ufmlaMpwFhlkQVZhqmfvjUaj7qU,43845
-megfile/smart.py,sha256=Sae2KJzaU0k_qV_Bk0YifOMq8WsV5qQ2pGInDRF546I,36411
+megfile/smart.py,sha256=Miz3jyLKmwWBja-8GrSzrumpTarPrFPqXaFQJKwrK1Y,36627
 megfile/smart_path.py,sha256=Up_6xNZ2019iSzMn_JAU_1H--z-AP6O7SxdXGdeTG0c,7659
 megfile/stdio.py,sha256=ZwxsnJNJYIT7Iyg5pIw4qiyH8bszG6oAhEJuR-hXGG4,658
 megfile/stdio_path.py,sha256=cxaDr8rtisTPnN-rjtaEpqQnshwiqwXFUJBM9xWY7Cg,2711
-megfile/version.py,sha256=IhKvX1UU6Hj5y2NPRPLmOC6AwOxQcN87pBebR6XrP7I,19
+megfile/version.py,sha256=Q7vzX4XpnfI8xIxWcYQYkZJUE1GiBtK_NmvaAfIJfC8,19
 megfile/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 megfile/lib/base_prefetch_reader.py,sha256=uxVwYknOjc8hLF7q_T2QKMsBqFcrf411ZsuK25CN1eQ,12848
 megfile/lib/combine_reader.py,sha256=Kp2wEloOUpTlIU7dve87MBpSzmIM-F9OtpTawAjFkiU,4828
 megfile/lib/compare.py,sha256=n_dtLxgoskYnsIZMKdKmVhQoVn8qYUrUhkS1JH2_X3o,2170
 megfile/lib/compat.py,sha256=SynEeHluys3tCK-lb_1oV3o_ft83yZvunqM_AjibLgE,207
 megfile/lib/fnmatch.py,sha256=4MvGzEahMRA-u8Z7mxaD-Yw1idOwBoJJpVywQy29jwY,4162
-megfile/lib/glob.py,sha256=-N75Phx1c8IpZ2hNIMwsHIx6BxMKVVr7N82cpWhs4lQ,9987
+megfile/lib/glob.py,sha256=anr7JjCJz1T0G5hYQkJwV5mCWzlt5YaxGTyyyQB28zA,9663
 megfile/lib/hdfs_prefetch_reader.py,sha256=yCNpcXcTiC2SHKHC-Qp50KQx1ObSLmOgwNUKlG-4ADg,2131
 megfile/lib/hdfs_tools.py,sha256=4K-OdMYFFSLBGmDzjatioHvuZuUbKVy7ACeJl-l0HLQ,435
 megfile/lib/http_prefetch_reader.py,sha256=OQPZ7kWFImqpynjaiTtmadtgtab5fCeQmu51UYHZfgs,4135
@@ -43,10 +43,10 @@ megfile/lib/stdio_handler.py,sha256=IDdgENLQlhigEwkLL4zStueVSzdWg7xVcTF_koof_Ek,
 megfile/lib/url.py,sha256=ER32pWy9Q2MAk3TraAaNEBWIqUeBmLuM57ol2cs7-Ks,103
 megfile/utils/__init__.py,sha256=pawmXnCNokWLj338a60b_hK21koYavpEiEohZhsOaGQ,10156
 megfile/utils/mutex.py,sha256=asb8opGLgK22RiuBJUnfsvB8LnMmodP8KzCVHKmQBWA,2561
-megfile-4.1.6.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-megfile-4.1.6.dist-info/licenses/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
-megfile-4.1.6.dist-info/METADATA,sha256=_YzyQHnfYQYQRtbqc5__xrF5AtBBdIBDMs0WujErjBk,9595
-megfile-4.1.6.dist-info/WHEEL,sha256=ooBFpIzZCPdw3uqIQsOo4qqbA4ZRPxHnOH7peeONza0,91
-megfile-4.1.6.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
-megfile-4.1.6.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
-megfile-4.1.6.dist-info/RECORD,,
+megfile-4.1.8.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+megfile-4.1.8.dist-info/licenses/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
+megfile-4.1.8.dist-info/METADATA,sha256=tuE4RMIMSYdGykVRvC0SoEGNl6N0xvv5KsNWXk7xhG4,9595
+megfile-4.1.8.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+megfile-4.1.8.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
+megfile-4.1.8.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
+megfile-4.1.8.dist-info/RECORD,,

{megfile-4.1.6.dist-info → megfile-4.1.8.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.0.1)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any