megfile 3.1.5__py3-none-any.whl → 3.1.6__py3-none-any.whl

megfile/cli.py

@@ -29,6 +29,7 @@ from megfile.smart import (
     smart_open,
     smart_path_join,
     smart_readlink,
+    smart_relpath,
     smart_remove,
     smart_rename,
     smart_scan_stat,
@@ -78,7 +79,7 @@ def get_echo_path(file_stat, base_path: str = "", full_path: bool = False):
     elif full_path:
         path = file_stat.path
     else:
-        path = os.path.relpath(file_stat.path, start=base_path)
+        path = smart_relpath(file_stat.path, start=base_path)
     return path
 
 

megfile/errors.py

@@ -366,7 +366,7 @@ def translate_s3_error(s3_error: Exception, s3_url: PathLike) -> Exception:
                 "Invalid configuration: %r, code: %r, message: %r, endpoint: %r"
                 % (s3_url, code, message, s3_endpoint_url(s3_url))
             )
-        if code in ("InvalidRange"):
+        if code in ("InvalidRange", "Requested Range Not Satisfiable"):
             return S3InvalidRangeError(
                 "Index out of range: %r, code: %r, message: %r, endpoint: %r"
                 % (

megfile/pathlike.py

@@ -636,8 +636,8 @@ class URIPath(BaseURIPath):
             relative = path[len(other_path) :]
             relative = relative.lstrip("/")
             return type(self)(relative)  # pyre-ignore[19]
-        else:
-            raise ValueError("%r does not start with %r" % (path, other))
+
+        raise ValueError("%r does not start with %r" % (path, other))
 
     def with_name(self: Self, name: str) -> Self:
         """Return a new path with the name changed"""
@@ -655,6 +655,21 @@ class URIPath(BaseURIPath):
         raw_suffix = self.suffix
         return self.from_path(path[: len(path) - len(raw_suffix)] + suffix)
 
+    def relpath(self, start=None):
+        """Return the relative path."""
+        if start is None:
+            raise TypeError("start is required")
+
+        other_path = self.from_path(start).path_with_protocol
+        path = self.path_with_protocol
+
+        if path.startswith(other_path):
+            relative = path[len(other_path) :]
+            relative = relative.lstrip("/")
+            return relative
+
+        raise ValueError("%r does not start with %r" % (path, other_path))
+
     def is_absolute(self) -> bool:
         return True
 

megfile/s3_path.py

@@ -122,7 +122,7 @@ max_retries = S3_MAX_RETRY_TIMES
 max_keys = 1000
 
 
-def _patch_make_request(client: botocore.client.BaseClient):
+def _patch_make_request(client: botocore.client.BaseClient, redirect: bool = False):
     def after_callback(result: Tuple[AWSResponse, dict], *args, **kwargs):
         if (
             not isinstance(result, tuple)
@@ -161,6 +161,26 @@ def _patch_make_request(client: botocore.client.BaseClient):
         before_callback=before_callback,
         retry_callback=retry_callback,
     )
+
+    def patch_send_request(send_request):
+        def patched_send_request(request_dict, operation_model):
+            http, parsed_response = send_request(request_dict, operation_model)
+            if (
+                request_dict["method"] == "GET"  # only support GET method for now
+                and http.status_code in (301, 302, 307, 308)
+                and "Location" in http.headers
+            ):
+                request_dict["url"] = http.headers["Location"]
+                http, parsed_response = send_request(request_dict, operation_model)
+            return http, parsed_response
+
+        return patched_send_request
+
+    if redirect:
+        client._endpoint._send_request = patch_send_request(
+            client._endpoint._send_request
+        )
+
     return client
 
 
@@ -180,7 +200,11 @@ def parse_s3_url(s3_url: PathLike) -> Tuple[str, str]:
 
 
 def get_scoped_config(profile_name: Optional[str] = None) -> Dict:
-    return get_s3_session(profile_name=profile_name)._session.get_scoped_config()
+    try:
+        session = get_s3_session(profile_name=profile_name)
+    except botocore.exceptions.ProfileNotFound:
+        session = get_s3_session()
+    return session._session.get_scoped_config()
 
 
 @lru_cache()
@@ -224,25 +248,22 @@ def get_s3_session(profile_name=None) -> boto3.Session:
     )
 
 
+def get_env_var(env_name: str, profile_name=None):
+    if profile_name:
+        return os.getenv(f"{profile_name}__{env_name}".upper())
+    return os.getenv(env_name.upper())
+
+
+def parse_boolean(value: Optional[str], default: bool = False) -> bool:
+    if value is None:
+        return default
+    return value.lower() in ("true", "yes", "1")
+
+
 def get_access_token(profile_name=None):
-    access_key_env_name = (
-        f"{profile_name}__AWS_ACCESS_KEY_ID".upper()
-        if profile_name
-        else "AWS_ACCESS_KEY_ID"
-    )
-    secret_key_env_name = (
-        f"{profile_name}__AWS_SECRET_ACCESS_KEY".upper()
-        if profile_name
-        else "AWS_SECRET_ACCESS_KEY"
-    )
-    session_token_env_name = (
-        f"{profile_name}__AWS_SESSION_TOKEN".upper()
-        if profile_name
-        else "AWS_SESSION_TOKEN"
-    )
-    access_key = os.getenv(access_key_env_name)
-    secret_key = os.getenv(secret_key_env_name)
-    session_token = os.getenv(session_token_env_name)
+    access_key = get_env_var("AWS_ACCESS_KEY_ID", profile_name=profile_name)
+    secret_key = get_env_var("AWS_SECRET_ACCESS_KEY", profile_name=profile_name)
+    session_token = get_env_var("AWS_SESSION_TOKEN", profile_name=profile_name)
     if access_key and secret_key:
         return access_key, secret_key, session_token
 
@@ -289,10 +310,7 @@ def get_s3_client(
             connect_timeout=5, max_pool_connections=GLOBAL_MAX_WORKERS
         )
 
-    addressing_style_env_key = "AWS_S3_ADDRESSING_STYLE"
-    if profile_name:
-        addressing_style_env_key = f"{profile_name}__AWS_S3_ADDRESSING_STYLE".upper()
-    addressing_style = os.environ.get(addressing_style_env_key)
+    addressing_style = get_env_var("AWS_S3_ADDRESSING_STYLE", profile_name=profile_name)
     if addressing_style:
         config = config.merge(
             botocore.config.Config(s3={"addressing_style": addressing_style})
@@ -303,15 +321,25 @@ def get_s3_client(
         session = get_s3_session(profile_name=profile_name)
     except botocore.exceptions.ProfileNotFound:
         session = get_s3_session()
+
+    s3_config = get_scoped_config(profile_name=profile_name).get("s3", {})
+    verify = get_env_var("AWS_S3_VERIFY", profile_name=profile_name)
+    verify = verify or s3_config.get("verify")
+    verify = parse_boolean(verify, default=True)
+    redirect = get_env_var("AWS_S3_REDIRECT", profile_name=profile_name)
+    redirect = redirect or s3_config.get("redirect")
+    redirect = parse_boolean(redirect, default=False)
+
     client = session.client(
         "s3",
         endpoint_url=get_endpoint_url(profile_name=profile_name),
+        verify=verify,
         config=config,
         aws_access_key_id=access_key,
         aws_secret_access_key=secret_key,
         aws_session_token=session_token,
     )
-    client = _patch_make_request(client)
+    client = _patch_make_request(client, redirect=redirect)
     return client
 
 

megfile/smart_path.py

@@ -1,11 +1,11 @@
 import os
 from configparser import ConfigParser
 from pathlib import PurePath
-from typing import Dict, Tuple, Union
+from typing import Dict, Optional, Tuple, Union
 
 from megfile.lib.compat import fspath
 from megfile.lib.url import get_url_scheme
-from megfile.utils import classproperty
+from megfile.utils import cached_classproperty
 
 from .errors import ProtocolExistsError, ProtocolNotFoundError
 from .interfaces import BasePath, BaseURIPath, PathLike
@@ -54,15 +54,15 @@ class SmartPath(BasePath):
             self.path = str(pathlike)
         self.pathlike = pathlike
 
-    @classproperty
+    @cached_classproperty
     def _aliases(cls) -> Dict[str, Dict[str, str]]:
         config_path = os.path.expanduser(aliases_config)
-        aliases = _load_aliases_config(config_path)
-        setattr(cls, "_aliases", aliases)
-        return aliases
+        return _load_aliases_config(config_path)
 
-    @staticmethod
-    def _extract_protocol(path: Union[PathLike, int]) -> Tuple[str, Union[str, int]]:
+    @classmethod
+    def _extract_protocol(
+        cls, path: Union[PathLike, int]
+    ) -> Tuple[str, Union[str, int]]:
         if isinstance(path, int):
             protocol = "file"
             path_without_protocol = path
@@ -74,27 +74,26 @@ class SmartPath(BasePath):
             else:
                 path_without_protocol = path[len(protocol) + 3 :]
         elif isinstance(path, (BaseURIPath, SmartPath)):
-            protocol = path.protocol
-            path_without_protocol = str(path)
+            return str(path.protocol), str(path)
         elif isinstance(path, (PurePath, BasePath)):
-            protocol, path_without_protocol = SmartPath._extract_protocol(fspath(path))
+            return SmartPath._extract_protocol(fspath(path))
         else:
             raise ProtocolNotFoundError("protocol not found: %r" % path)
-        return protocol, path_without_protocol
-
-    @classmethod
-    def _create_pathlike(cls, path: Union[PathLike, int]) -> BaseURIPath:
-        protocol, path_without_protocol = cls._extract_protocol(path)
         aliases: Dict[str, Dict[str, str]] = cls._aliases  # pyre-ignore[9]
         if protocol in aliases:
             protocol = aliases[protocol]["protocol"]
-            path = protocol + "://" + str(path_without_protocol)
+            path = "%s://%s" % (protocol, path_without_protocol)
+        return protocol, path
+
+    @classmethod
+    def _create_pathlike(cls, path: Union[PathLike, int]) -> BaseURIPath:
+        protocol, unaliased_path = cls._extract_protocol(path)
         if protocol.startswith("s3+"):
             protocol = "s3"
         if protocol not in cls._registered_protocols:
             raise ProtocolNotFoundError("protocol %r not found: %r" % (protocol, path))
         path_class = cls._registered_protocols[protocol]
-        return path_class(path)
+        return path_class(unaliased_path)
 
     @classmethod
     def register(cls, path_class, override_ok: bool = False):
@@ -137,7 +136,6 @@ class SmartPath(BasePath):
     joinpath = _bind_function("joinpath")
     abspath = _bind_function("abspath")
     realpath = _bind_function("realpath")
-    relpath = _bind_function("relpath")
     is_absolute = _bind_function("is_absolute")
     is_mount = _bind_function("is_mount")
     md5 = _bind_function("md5")
@@ -150,6 +148,16 @@ class SmartPath(BasePath):
     def from_uri(cls, path: str):
         return cls(path)
 
+    def relpath(self, start: Optional[str] = None) -> str:
+        """Return the relative path of given path
+
+        :param start: Given start directory
+        :returns: Relative path from start
+        """
+        if start is not None:
+            _, start = SmartPath._extract_protocol(fspath(start))
+        return self.pathlike.relpath(start=start)
+
     as_uri = _bind_function("as_uri")
     as_posix = _bind_function("as_posix")
     __lt__ = _bind_function("__lt__")
@@ -170,7 +178,6 @@ class SmartPath(BasePath):
     is_mount = _bind_function("is_mount")
     abspath = _bind_function("abspath")
     realpath = _bind_function("realpath")
-    relpath = _bind_function("relpath")
     iterdir = _bind_function("iterdir")
     cwd = _bind_function("cwd")
     home = _bind_function("home")

megfile/utils/__init__.py

@@ -4,7 +4,7 @@ import math
 import os
 import uuid
 from copy import copy
-from functools import wraps
+from functools import cached_property, wraps
 from io import (
     BufferedIOBase,
     BufferedRandom,
@@ -15,6 +15,7 @@ from io import (
     TextIOBase,
     TextIOWrapper,
 )
+from threading import RLock
 from typing import IO, Callable, Optional
 
 from megfile.utils.mutex import ProcessLocal, ThreadLocal
@@ -298,3 +299,50 @@ class classproperty(property):
         """
         # call this method only on the class, not the instance
         super(classproperty, self).__delete__(_get_class(cls_or_obj))
+
+
+class cached_classproperty(cached_property):
+    """
+    The use this class as a decorator for your class property with cache.
+    Example:
+        @cached_classproperty
+        def prop(cls):
+            return "value"
+    """
+
+    def __init__(self, func: Callable) -> None:
+        """
+        This method initializes the cached_classproperty instance.
+        @param func: The function to be called when the property value is requested.
+        """
+        super().__init__(func)
+        # Python 3.12 removed the lock attribute from cached_property.
+        # Maybe we should remove this in the future.
+        # See also: https://github.com/python/cpython/pull/101890
+        #        https://github.com/python/cpython/issues/87634
+        if not hasattr(func, "lock"):
+            self.lock = RLock()
+
+    def __get__(  # pyre-ignore[14]
+        self,
+        _,
+        cls,  # pytype: disable=signature-mismatch
+    ) -> object:
+        """
+        This method gets called when a property value is requested.
+        @param cls: The class type of the above instance.
+        @return: The value of the property.
+        """
+        if self.attrname is None:
+            raise TypeError(
+                "Cannot use cached_classproperty instance without calling "
+                "__set_name__ on it."
+            )
+        with self.lock:
+            # check if another thread filled cache while we awaited lock
+            # cannot use getattr since it will cause RecursionError
+            val = cls.__dict__[self.attrname]
+            if val is self:
+                val = self.func(cls)
+                setattr(cls, self.attrname, val)
+        return val

megfile/version.py

@@ -1 +1 @@
-VERSION = "3.1.5"
+VERSION = "3.1.6"

{megfile-3.1.5.dist-info → megfile-3.1.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: megfile
-Version: 3.1.5
+Version: 3.1.6
 Summary: Megvii file operation library
 Author-email: megvii <megfile@megvii.com>
 Project-URL: Homepage, https://github.com/megvii-research/megfile

{megfile-3.1.5.dist-info → megfile-3.1.6.dist-info}/RECORD

@@ -1,8 +1,8 @@
 docs/conf.py,sha256=sfDSly5jO8W_RmuAptOIp4hd8dNcO-9a5XrHTbxFnNo,2448
 megfile/__init__.py,sha256=i2Lbq_VxIgppaqwkxG0_H35dRfcjJ4mCYWjprOf4hHo,7318
-megfile/cli.py,sha256=miI1fEHY3zBxWwwN5F5AHjO14-AI920oPxD4sfi94gY,23465
+megfile/cli.py,sha256=0Sgbz3jeUryVll9Aa6R0MpJdQJUGENvz55YF7Jm1Uxc,23482
 megfile/config.py,sha256=_SkJRaVWUdfW1Q9uX0vao-6YVQKJtfej22Z8DykuRps,2331
-megfile/errors.py,sha256=KyHvK3CgiSgiXT_5kqKa7Nb8Bdwh7U6mBh_H7Gj-Z84,14125
+megfile/errors.py,sha256=a55qKQgyfiLmV-qnojUFzq2gu9JXpj3ZiC2qVaWyUTA,14160
 megfile/fs.py,sha256=dgj5fW-EEzQNdjMF2tkB5DjXu3iHQbtLi5PSIMxR8fc,11966
 megfile/fs_path.py,sha256=Ffvukc176beH5aQMZXXtwH6ApwLYXPViCIUP0pijgT0,41590
 megfile/hdfs.py,sha256=latguOuuzAmg-yWOy3Sm723CJ0ybN_eSHRubVNqhcMU,9202
@@ -10,16 +10,16 @@ megfile/hdfs_path.py,sha256=0XLtABufwqL-y8igOxzOJz6zOGppuBp2f2SwXIMvvYg,27299
 megfile/http.py,sha256=2Z2yqyhU-zcJCJwSNyBsxsZ7f2FT9X6fcednsbHDsFM,2025
 megfile/http_path.py,sha256=BhMNjQVB85IaCGGIKzgEfY73mAVdCzJP08W1RuGeMRA,16119
 megfile/interfaces.py,sha256=7C53Q2FAVFmOEnplfplvWqHab29HJE5RQnpfdb4loVY,8679
-megfile/pathlike.py,sha256=vKuCMlSAPYNSojp03wEj2i3Cq3E3ROp_-UkkdgBElws,30802
+megfile/pathlike.py,sha256=5VAKIArm2UqrpMBJMoNAEydFxLd1mjCZ8iQnKFUIYu0,31274
 megfile/s3.py,sha256=7SdfLjAePVh-bpRyuj566VB4Qa7KP86rCJGzYANR7wQ,13008
-megfile/s3_path.py,sha256=YvIRVI4d6kd2m2wvcIqiu43nPOYZpaPgG9-BospF81w,93803
+megfile/s3_path.py,sha256=x-_wxVvVpv56LtDOwRAWJCOq3XMk7oAB6xdi22TBlmY,95029
 megfile/sftp.py,sha256=vyDnYXX3i1j2fhXMC8YCeX-66MDb9wrBQQjQVhZx0uo,13004
 megfile/sftp_path.py,sha256=4tByWvUJK1KBJoa3t5aoWYnZpaRWN9nQIE6ZyiGHrbk,53519
 megfile/smart.py,sha256=Vr4R7HpjXjt587KOc2-1QGbQ5EsZ48YRzCaK0rz3IS0,36108
-megfile/smart_path.py,sha256=22ZTrA7j9Kd5PJsUJOxBdGg0Uu5FxwdT_XQjQDxUHo4,7637
+megfile/smart_path.py,sha256=Wsn6fR9g7NTwNwwvZ_0H39NLHIlOLnCqK-ZY0n5CvKk,7812
 megfile/stdio.py,sha256=UYe-h440Wc4f5COOzOTG1svnp5nFzrfpixehJ0_0_NY,653
 megfile/stdio_path.py,sha256=7jzVdreamO18yBWZM7Pp71cO7GmrYb0M0qyQde2Ypq4,2706
-megfile/version.py,sha256=kh8D3nOB4xuR3KqckCcukx60UxptlmvEx5kjmgZinIA,19
+megfile/version.py,sha256=GfRGwuY_9YYywKtd8Nc8lbkE6hlfX3NUsNtuHN-c8Gs,19
 megfile/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 megfile/lib/base_prefetch_reader.py,sha256=CaYWuiKmlk4Utr0IFDPwPC58wV2jBAhqpxhwhRHc734,13652
 megfile/lib/combine_reader.py,sha256=uSzo3PmhD5ck6_Vv6dFU5vVx4boeA97VS-puPyhF_BE,4657
@@ -42,14 +42,14 @@ megfile/lib/s3_share_cache_reader.py,sha256=jhGL1B6NPv68cQnW1Jf7ey-zTQ8XfiJg5ILD
 megfile/lib/shadow_handler.py,sha256=TntewlvIW9ZxCfmqASDQREHoiZ8v42faOe9sovQYQz0,2779
 megfile/lib/stdio_handler.py,sha256=IDdgENLQlhigEwkLL4zStueVSzdWg7xVcTF_koof_Ek,1987
 megfile/lib/url.py,sha256=ER32pWy9Q2MAk3TraAaNEBWIqUeBmLuM57ol2cs7-Ks,103
-megfile/utils/__init__.py,sha256=NfO5vNxfeceGvMB3dgZNudyPFTmPY096JbC4iYroX6o,9003
+megfile/utils/__init__.py,sha256=9rD_SoD--XWt7-EJi5-L80Y7YeoFdr-tUp-5ATB85oA,10717
 megfile/utils/mutex.py,sha256=asb8opGLgK22RiuBJUnfsvB8LnMmodP8KzCVHKmQBWA,2561
 scripts/convert_results_to_sarif.py,sha256=nDiOfsedb22Ps7ZodmYdlXZlxv54fRxCQgOZsB2OkNk,2833
 scripts/generate_file.py,sha256=-mTcBiqiQ1juvqojVfVZ-uZWgpANHJNdhrF7s68zNfc,10903
-megfile-3.1.5.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-megfile-3.1.5.dist-info/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
-megfile-3.1.5.dist-info/METADATA,sha256=jPBOY1WWbh-Jxdam48x2RxcXGVV2liPW4kQTw8VtBl0,9178
-megfile-3.1.5.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
-megfile-3.1.5.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
-megfile-3.1.5.dist-info/top_level.txt,sha256=oTnYXo1Z3V61qSWAKtnY9RkDgRSHvfRN38FQae6E0W0,50
-megfile-3.1.5.dist-info/RECORD,,
+megfile-3.1.6.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+megfile-3.1.6.dist-info/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
+megfile-3.1.6.dist-info/METADATA,sha256=XuomkBWbASrj60L6untH8oN32sNezHse1WfuPetTpYg,9178
+megfile-3.1.6.dist-info/WHEEL,sha256=R06PA3UVYHThwHvxuRWMqaGcr-PuniXahwjmQRFMEkY,91
+megfile-3.1.6.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
+megfile-3.1.6.dist-info/top_level.txt,sha256=oTnYXo1Z3V61qSWAKtnY9RkDgRSHvfRN38FQae6E0W0,50
+megfile-3.1.6.dist-info/RECORD,,

{megfile-3.1.5.dist-info → megfile-3.1.6.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.3.0)
+Generator: setuptools (75.5.0)
 Root-Is-Purelib: true
 Tag: py3-none-any