megfile 4.0.3__py3-none-any.whl → 4.1.0__py3-none-any.whl

megfile/s3_path.py

@@ -200,10 +200,9 @@ def parse_s3_url(s3_url: PathLike) -> Tuple[str, str]:
 
 def get_scoped_config(profile_name: Optional[str] = None) -> Dict:
     try:
-        session = get_s3_session(profile_name=profile_name)
+        return get_s3_session(profile_name=profile_name)._session.get_scoped_config()
     except botocore.exceptions.ProfileNotFound:
-        session = get_s3_session()
-    return session._session.get_scoped_config()
+        return {}
 
 
 @lru_cache()
@@ -225,15 +224,12 @@ def get_endpoint_url(profile_name: Optional[str] = None) -> str:
         if environ_endpoint_url:
             warning_endpoint_url(environ_key, environ_endpoint_url)
             return environ_endpoint_url
-    try:
-        config = get_scoped_config(profile_name=profile_name)
-        config_endpoint_url = config.get("s3", {}).get("endpoint_url")
-        config_endpoint_url = config_endpoint_url or config.get("endpoint_url")
-        if config_endpoint_url:
-            warning_endpoint_url("~/.aws/config", config_endpoint_url)
-            return config_endpoint_url
-    except botocore.exceptions.ProfileNotFound:
-        pass
+    config = get_scoped_config(profile_name=profile_name)
+    config_endpoint_url = config.get("s3", {}).get("endpoint_url")
+    config_endpoint_url = config_endpoint_url or config.get("endpoint_url")
+    if config_endpoint_url:
+        warning_endpoint_url("~/.aws/config", config_endpoint_url)
+        return config_endpoint_url
     return endpoint_url
 
 
@@ -540,6 +536,40 @@ def _make_stat(content: Dict[str, Any]):
     )
 
 
+class StatResultForIsLink:
+    def __init__(self, path: "S3Path", *args, **kwargs):
+        self._islnk = None
+        self._path = path
+        self._stat_result = StatResult(*args, **kwargs)
+
+    def __getattr__(self, name):
+        return getattr(self._stat_result, name)
+
+    @property
+    def islnk(self) -> bool:
+        if self._islnk is None:
+            self._islnk = self._path.is_symlink()
+        return self._islnk
+
+    def is_file(self) -> bool:
+        return not self._stat_result.isdir or self.islnk
+
+    def is_dir(self) -> bool:
+        return self._stat_result.isdir and not self.islnk
+
+    def is_symlink(self) -> bool:
+        return self.islnk
+
+
+def _make_stat_without_metadata(content, path):
+    return StatResultForIsLink(
+        path=path,
+        size=content["Size"],
+        mtime=content["LastModified"].timestamp(),
+        extra=content,
+    )
+
+
 def _s3_glob_stat_single_path(
     s3_pathname: PathLike,
     recursive: bool = True,
@@ -564,11 +594,6 @@ def _s3_glob_stat_single_path(
         return False
 
     def create_generator(_s3_pathname) -> Iterator[FileEntry]:
-        top_dir_with_profile = top_dir
-        if profile_name:
-            top_dir_with_profile = f"s3+{profile_name}://{top_dir[5:]}"
-        if not S3Path(top_dir_with_profile).exists():
-            return
         if not has_magic(_s3_pathname):
             _s3_pathname_obj = S3Path(_s3_pathname)
             if _s3_pathname_obj.is_file():
@@ -589,7 +614,7 @@ def _s3_glob_stat_single_path(
         bucket, key = parse_s3_url(top_dir)
         prefix = _become_prefix(key)
         client = get_s3_client_with_cache(profile_name=profile_name)
-        with raise_s3_error(_s3_pathname):
+        with raise_s3_error(_s3_pathname, S3BucketNotFoundError):
             for resp in _list_objects_recursive(client, bucket, prefix, delimiter):
                 for content in resp.get("Contents", []):
                     path = s3_path_join("s3://", bucket, content["Key"])
@@ -1123,10 +1148,9 @@ def s3_download(
             "Empty bucket name: %r" % src_url.path_with_protocol
         )
 
-    if not src_url.exists():
-        raise S3FileNotFoundError("File not found: %r" % src_url.path_with_protocol)
-
     if not src_url.is_file():
+        if not src_url.is_dir():
+            raise S3FileNotFoundError("File not found: %r" % src_url.path_with_protocol)
         raise S3IsADirectoryError("Is a directory: %r" % src_url.path_with_protocol)
 
     dst_directory = os.path.dirname(dst_path.path_without_protocol)  # pyre-ignore[6]
@@ -1221,7 +1245,6 @@ def s3_load_content(
     s3_url,
     start: Optional[int] = None,
     stop: Optional[int] = None,
-    followlinks: bool = False,
 ) -> bytes:
     """
     Get specified file from [start, stop) in bytes
@@ -1236,11 +1259,6 @@ def s3_load_content(
         return client.get_object(Bucket=bucket, Key=key, Range=range_str)["Body"].read()
 
     s3_url = S3Path(s3_url)
-    if followlinks:
-        try:
-            s3_url = s3_url.readlink()
-        except S3NotALinkError:
-            pass
 
     bucket, key = parse_s3_url(s3_url.path_with_protocol)
     if not bucket:
@@ -1430,7 +1448,7 @@ class S3Path(URIPath):
                 raise error
             return {}
 
-    def access(self, mode: Access = Access.READ, followlinks: bool = False) -> bool:
+    def access(self, mode: Access = Access.READ) -> bool:
         """
         Test if path has access permission described by mode
 
@@ -1438,11 +1456,6 @@ class S3Path(URIPath):
         :returns: bool, if the bucket of s3_url has read/write access.
         """
         s3_url = self.path_with_protocol
-        if followlinks:
-            try:
-                s3_url = self.readlink().path_with_protocol
-            except S3NotALinkError:
-                pass
         bucket, key = parse_s3_url(s3_url)  # only check bucket accessibility
         if not bucket:
             raise Exception("No available bucket")
@@ -1537,7 +1550,6 @@ class S3Path(URIPath):
         pattern,
         recursive: bool = True,
         missing_ok: bool = True,
-        followlinks: bool = False,
     ) -> List["S3Path"]:
         """Return s3 path list in ascending alphabetical order,
         in which path matches glob pattern
@@ -1558,7 +1570,6 @@ class S3Path(URIPath):
                 pattern=pattern,
                 recursive=recursive,
                 missing_ok=missing_ok,
-                followlinks=followlinks,
             )
         )
 
@@ -1567,7 +1578,6 @@ class S3Path(URIPath):
         pattern,
         recursive: bool = True,
         missing_ok: bool = True,
-        followlinks: bool = False,
     ) -> Iterator[FileEntry]:
         """Return a generator contains tuples of path and file stat,
         in ascending alphabetical order, in which path matches glob pattern
@@ -1598,7 +1608,6 @@ class S3Path(URIPath):
                         group_s3_pathname_2,
                         recursive,
                         missing_ok,
-                        followlinks=followlinks,
                         profile_name=self._profile_name,
                     ):
                         if self._profile_name:
@@ -1618,7 +1627,6 @@ class S3Path(URIPath):
         pattern,
         recursive: bool = True,
         missing_ok: bool = True,
-        followlinks: bool = False,
     ) -> Iterator["S3Path"]:
         """Return s3 path iterator in ascending alphabetical order,
         in which path matches glob pattern
@@ -1638,7 +1646,6 @@ class S3Path(URIPath):
             pattern=pattern,
             recursive=recursive,
             missing_ok=missing_ok,
-            followlinks=followlinks,
         ):
             yield self.from_path(file_entry.path)
 
@@ -1702,27 +1709,30 @@ class S3Path(URIPath):
             return False
         return True
 
-    def listdir(self, followlinks: bool = False) -> List[str]:
+    def listdir(self) -> List[str]:
         """
         Get all contents of given s3_url. The result is in ascending alphabetical order.
 
+        :param missing_ok: if True and target directory not exists return empty list,
+            default is True.
         :returns: All contents have prefix of s3_url in ascending alphabetical order
         :raises: S3FileNotFoundError, S3NotADirectoryError
         """
-        entries = list(self.scandir(followlinks=followlinks))
-        return sorted([entry.name for entry in entries])
+        with self.scandir() as entries:
+            return sorted([entry.name for entry in entries])
 
-    def iterdir(self, followlinks: bool = False) -> Iterator["S3Path"]:
+    def iterdir(self) -> Iterator["S3Path"]:
         """
-        Get all contents of given s3_url. The result is in ascending alphabetical order.
+        Get all contents of given s3_url. The order of result is in arbitrary order.
 
-        :returns: All contents have prefix of s3_url in ascending alphabetical order
+        :returns: All contents have prefix of s3_url
         :raises: S3FileNotFoundError, S3NotADirectoryError
         """
-        for path in self.listdir(followlinks=followlinks):
-            yield self.joinpath(path)
+        with self.scandir() as entries:
+            for entry in entries:
+                yield self.joinpath(entry.name)
 
-    def load(self, followlinks: bool = False) -> BinaryIO:
+    def load(self) -> BinaryIO:
         """Read all content in binary on specified path and write into memory
 
         User should close the BinaryIO manually
@@ -1730,11 +1740,6 @@ class S3Path(URIPath):
         :returns: BinaryIO
         """
         s3_url = self.path_with_protocol
-        if followlinks:
-            try:
-                s3_url = self.readlink().path_with_protocol
-            except S3NotALinkError:
-                pass
         bucket, key = parse_s3_url(s3_url)
         if not bucket:
             raise S3BucketNotFoundError("Empty bucket name: %r" % s3_url)
@@ -1797,11 +1802,14 @@ class S3Path(URIPath):
             raise S3BucketNotFoundError(
                 "Empty bucket name: %r" % self.path_with_protocol
             )
-        if not self.hasbucket():
-            raise S3BucketNotFoundError("No such bucket: %r" % self.path_with_protocol)
+        try:
+            if not self.hasbucket():
+                raise S3BucketNotFoundError(
+                    "No such bucket: %r" % self.path_with_protocol
+                )
+        except S3PermissionError:
+            pass
         if exist_ok:
-            if self.is_file():
-                raise S3FileExistsError("File exists: %r" % self.path_with_protocol)
             return
         if self.exists():
             raise S3FileExistsError("File exists: %r" % self.path_with_protocol)
@@ -1959,17 +1967,8 @@ class S3Path(URIPath):
             raise UnsupportedError("Scan whole s3", self.path_with_protocol)
 
         def create_generator() -> Iterator[FileEntry]:
-            if not self.is_dir():
-                if self.is_file():
-                    # On s3, file and directory may be of same name and level, so need
-                    # to test the path is file or directory
-                    yield FileEntry(
-                        self.name,
-                        fspath(self.path_with_protocol),
-                        self.stat(follow_symlinks=followlinks),
-                    )
-                return
-
+            # On s3, file and directory may be of same name and level, so need
+            # to test the path is file or directory
             if not key.endswith("/") and self.is_file():
                 yield FileEntry(
                     self.name,
@@ -1979,7 +1978,13 @@ class S3Path(URIPath):
 
             prefix = _become_prefix(key)
             client = self._client
-            with raise_s3_error(self.path_with_protocol):
+
+            def suppress_error_callback(e):
+                if missing_ok and isinstance(e, S3BucketNotFoundError):
+                    return True
+                return False
+
+            with raise_s3_error(self.path_with_protocol, suppress_error_callback):
                 for resp in _list_objects_recursive(client, bucket, prefix):
                     for content in resp.get("Contents", []):
                         full_path = s3_path_join(
@@ -2008,12 +2013,12 @@ class S3Path(URIPath):
             S3FileNotFoundError("No match any file in: %r" % self.path_with_protocol),
         )
 
-    def scandir(self, followlinks: bool = False) -> Iterator[FileEntry]:
+    def scandir(self) -> ContextIterator:
         """
-        Get all contents of given s3_url, the order of result is not guaranteed.
+        Get all contents of given s3_url, the order of result is in arbitrary order.
 
         :returns: All contents have prefix of s3_url
-        :raises: S3FileNotFoundError, S3NotADirectoryError
+        :raises: S3BucketNotFoundError, S3FileNotFoundError, S3NotADirectoryError
         """
         bucket, key = parse_s3_url(self.path_with_protocol)
         if not bucket and key:
@@ -2023,66 +2028,69 @@ class S3Path(URIPath):
 
         if self.is_file():
             raise S3NotADirectoryError("Not a directory: %r" % self.path_with_protocol)
-        elif not self.is_dir():
-            raise S3FileNotFoundError("No such directory: %r" % self.path_with_protocol)
-        prefix = _become_prefix(key)
-        client = self._client
 
         # In order to do check on creation,
         # we need to wrap the iterator in another function
         def create_generator() -> Iterator[FileEntry]:
-            with raise_s3_error(self.path_with_protocol):
+            prefix = _become_prefix(key)
+            client = self._client
 
-                def generate_s3_path(protocol: str, bucket: str, key: str) -> str:
-                    return "%s://%s/%s" % (protocol, bucket, key)
+            def generate_s3_path(protocol: str, bucket: str, key: str) -> str:
+                return "%s://%s/%s" % (protocol, bucket, key)
 
-                if not bucket and not key:  # list buckets
-                    response = client.list_buckets()
-                    for content in response["Buckets"]:
-                        yield FileEntry(
-                            content["Name"],
-                            f"s3://{content['Name']}",
-                            StatResult(
-                                ctime=content["CreationDate"].timestamp(),
-                                isdir=True,
-                                extra=content,
-                            ),
-                        )
-                    return
+            if not bucket and not key:  # list buckets
+                response = client.list_buckets()
+                for content in response["Buckets"]:
+                    yield FileEntry(
+                        content["Name"],
+                        f"s3://{content['Name']}",
+                        StatResult(
+                            ctime=content["CreationDate"].timestamp(),
+                            isdir=True,
+                            extra=content,
+                        ),
+                    )
+                return
 
-                for resp in _list_objects_recursive(client, bucket, prefix, "/"):
-                    for common_prefix in resp.get("CommonPrefixes", []):
-                        yield FileEntry(
-                            common_prefix["Prefix"][len(prefix) : -1],
-                            generate_s3_path(
-                                self._protocol_with_profile,
-                                bucket,
-                                common_prefix["Prefix"],
-                            ),
-                            StatResult(isdir=True, extra=common_prefix),
-                        )
-                    for content in resp.get("Contents", []):
-                        src_url = generate_s3_path(
-                            self._protocol_with_profile, bucket, content["Key"]
-                        )
+            for resp in _list_objects_recursive(client, bucket, prefix, "/"):
+                for common_prefix in resp.get("CommonPrefixes", []):
+                    yield FileEntry(
+                        common_prefix["Prefix"][len(prefix) : -1],
+                        generate_s3_path(
+                            self._protocol_with_profile,
+                            bucket,
+                            common_prefix["Prefix"],
+                        ),
+                        StatResult(isdir=True, extra=common_prefix),
+                    )
+                for content in resp.get("Contents", []):
+                    src_url = generate_s3_path(
+                        self._protocol_with_profile, bucket, content["Key"]
+                    )
+                    yield FileEntry(  # pytype: disable=wrong-arg-types
+                        content["Key"][len(prefix) :],
+                        src_url,
+                        _make_stat_without_metadata(content, self.from_path(src_url)),
+                    )
 
-                        if followlinks:
-                            try:
-                                origin_path = self.from_path(src_url).readlink()
-                                yield FileEntry(
-                                    origin_path.name,
-                                    origin_path.path_with_protocol,
-                                    origin_path.lstat(),
-                                )
-                                continue
-                            except S3NotALinkError:
-                                pass
+        def missing_ok_generator():
+            def suppress_error_callback(e):
+                if isinstance(e, S3BucketNotFoundError):
+                    return False
+                elif not key and isinstance(e, S3FileNotFoundError):
+                    return True
+                return False
 
-                        yield FileEntry(
-                            content["Key"][len(prefix) :], src_url, _make_stat(content)
-                        )
+            with raise_s3_error(self.path_with_protocol, suppress_error_callback):
+                yield from _create_missing_ok_generator(
+                    create_generator(),
+                    missing_ok=False,
+                    error=S3FileNotFoundError(
+                        "No such directory: %r" % self.path_with_protocol
+                    ),
+                )
 
-        return ContextIterator(create_generator())
+        return ContextIterator(missing_ok_generator())
 
     def _get_dir_stat(self) -> StatResult:
         """
@@ -2097,24 +2105,24 @@ class S3Path(URIPath):
 
         :returns: An int indicates size in Bytes
         """
-        if not self.is_dir():
-            raise S3FileNotFoundError(
-                "No such file or directory: %r" % self.path_with_protocol
-            )
-
         bucket, key = parse_s3_url(self.path_with_protocol)
         prefix = _become_prefix(key)
         client = self._client
-        size = 0
-        mtime = 0.0
+        count, size, mtime = 0, 0, 0.0
         with raise_s3_error(self.path_with_protocol):
             for resp in _list_objects_recursive(client, bucket, prefix):
                 for content in resp.get("Contents", []):
+                    count += 1
                     size += content["Size"]
                     last_modified = content["LastModified"].timestamp()
                     if mtime < last_modified:
                         mtime = last_modified
 
+        if count == 0:
+            raise S3FileNotFoundError(
+                "No such file or directory: %r" % self.path_with_protocol
+            )
+
         return StatResult(size=size, mtime=mtime, isdir=True)
 
     def stat(self, follow_symlinks=True) -> StatResult:
@@ -2219,29 +2227,28 @@ class S3Path(URIPath):
         if not bucket:
             raise UnsupportedError("Walk whole s3", self.path_with_protocol)
 
-        if not self.is_dir():
-            return
-
-        stack = [key]
-        client = self._client
-        while len(stack) > 0:
-            current = _become_prefix(stack.pop())
-            dirs, files = [], []
-            for resp in _list_objects_recursive(client, bucket, current, "/"):
-                for common_prefix in resp.get("CommonPrefixes", []):
-                    dirs.append(common_prefix["Prefix"][:-1])
-                for content in resp.get("Contents", []):
-                    files.append(content["Key"])
+        with raise_s3_error(self.path_with_protocol, S3BucketNotFoundError):
+            stack = [key]
+            client = self._client
+            while len(stack) > 0:
+                current = _become_prefix(stack.pop())
+                dirs, files = [], []
+                for resp in _list_objects_recursive(client, bucket, current, "/"):
+                    for common_prefix in resp.get("CommonPrefixes", []):
+                        dirs.append(common_prefix["Prefix"][:-1])
+                    for content in resp.get("Contents", []):
+                        files.append(content["Key"])
 
-            dirs = sorted(dirs)
-            stack.extend(reversed(dirs))
+                dirs = sorted(dirs)
+                stack.extend(reversed(dirs))
 
-            root = s3_path_join(f"{self._protocol_with_profile}://", bucket, current)[
-                :-1
-            ]
-            dirs = [path[len(current) :] for path in dirs]
-            files = sorted(path[len(current) :] for path in files)
-            yield root, dirs, files
+                root = s3_path_join(
+                    f"{self._protocol_with_profile}://", bucket, current
+                )[:-1]
+                dirs = [path[len(current) :] for path in dirs]
+                files = sorted(path[len(current) :] for path in files)
+                if files or dirs or not current:
+                    yield root, dirs, files
 
     def md5(self, recalculate: bool = False, followlinks: bool = False) -> str:
         """
@@ -2258,13 +2265,15 @@ class S3Path(URIPath):
             raise S3BucketNotFoundError(
                 "Empty bucket name: %r" % self.path_with_protocol
             )
-        stat = self.stat(follow_symlinks=followlinks)
-        if stat.isdir:
+        stat = self.stat(follow_symlinks=False)
+        if followlinks and stat.is_symlink():
+            return self.readlink().md5(recalculate=recalculate, followlinks=followlinks)
+        elif stat.is_dir():
             hash_md5 = hashlib.md5()  # nosec
             for file_name in self.listdir():
                 chunk = (
-                    S3Path(s3_path_join(self.path_with_protocol, file_name))
-                    .md5(recalculate=recalculate)
+                    self.joinpath(file_name)
+                    .md5(recalculate=recalculate, followlinks=followlinks)
                     .encode()
                 )
                 hash_md5.update(chunk)
@@ -2308,9 +2317,6 @@ class S3Path(URIPath):
 
         if not src_bucket:
             raise S3BucketNotFoundError("Empty bucket name: %r" % src_url)
-        if self.is_dir():
-            raise S3IsADirectoryError("Is a directory: %r" % src_url)
-
         if not dst_bucket:
             raise S3BucketNotFoundError("Empty bucket name: %r" % dst_url)
         if not dst_key or dst_key.endswith("/"):
@@ -2323,13 +2329,18 @@ class S3Path(URIPath):
             except S3NotALinkError:
                 pass
 
-        with raise_s3_error(f"'{src_url}' or '{dst_url}'"):
-            self._client.copy(
-                {"Bucket": src_bucket, "Key": src_key},
-                Bucket=dst_bucket,
-                Key=dst_key,
-                Callback=callback,
-            )
+        try:
+            with raise_s3_error(f"'{src_url}' or '{dst_url}'"):
+                self._client.copy(
+                    {"Bucket": src_bucket, "Key": src_key},
+                    Bucket=dst_bucket,
+                    Key=dst_key,
+                    Callback=callback,
+                )
+        except S3FileNotFoundError:
+            if self.is_dir():
+                raise S3IsADirectoryError("Is a directory: %r" % src_url)
+            raise
 
     def sync(
         self,
@@ -2362,7 +2373,11 @@ class S3Path(URIPath):
             ):
                 continue
 
-            src_file_path.copy(dst_file_path, followlinks=followlinks)
+            src_file_path.copy(
+                dst_file_path,
+                followlinks=followlinks,
+                overwrite=True,
+            )
 
     def symlink(self, dst_path: PathLike) -> None:
         """

megfile/sftp.py

@@ -613,7 +613,7 @@ def sftp_walk(
     return SftpPath(path).walk(followlinks)
 
 
-def sftp_getmd5(path: PathLike, recalculate: bool = False, followlinks: bool = True):
+def sftp_getmd5(path: PathLike, recalculate: bool = False, followlinks: bool = False):
     """
     Calculate the md5 value of the file
 
@@ -659,6 +659,7 @@ def sftp_save_as(file_object: BinaryIO, path: PathLike):
 def sftp_open(
     path: PathLike,
     mode: str = "r",
+    *,
     buffering=-1,
     encoding: Optional[str] = None,
     errors: Optional[str] = None,
@@ -676,10 +677,12 @@ def sftp_open(
         decoding errors are to be handled—this cannot be used in binary mode.
     :returns: File-Like object
     """
-    return SftpPath(path).open(mode, buffering, encoding, errors)
+    return SftpPath(path).open(
+        mode, buffering=buffering, encoding=encoding, errors=errors
+    )
 
 
-def sftp_chmod(path: PathLike, mode: int, follow_symlinks: bool = True):
+def sftp_chmod(path: PathLike, mode: int, *, follow_symlinks: bool = True):
     """
     Change the file mode and permissions, like os.chmod().
 
@@ -687,7 +690,7 @@ def sftp_chmod(path: PathLike, mode: int, follow_symlinks: bool = True):
     :param mode: the file mode you want to change
     :param followlinks: Ignore this parameter, just for compatibility
     """
-    return SftpPath(path).chmod(mode, follow_symlinks)
+    return SftpPath(path).chmod(mode, follow_symlinks=follow_symlinks)
 
 
 def sftp_absolute(path: PathLike) -> "SftpPath":