megfile 4.2.4__py3-none-any.whl → 5.0.0__py3-none-any.whl

megfile/s3_path.py

@@ -93,7 +93,6 @@ __all__ = [
     "get_endpoint_url",
     "get_s3_session",
     "get_s3_client",
-    "s3_path_join",
     "is_s3",
     "s3_buffered_open",
     "s3_cached_open",
@@ -104,6 +103,7 @@ __all__ = [
     "s3_open",
     "S3Cacher",
     "s3_upload",
+    "s3_copy",
     "s3_download",
     "s3_load_content",
     "s3_concat",
@@ -230,7 +230,7 @@ def get_endpoint_url(profile_name: Optional[str] = None) -> str:
     config_endpoint_url = config.get("s3", {}).get("endpoint_url")
     config_endpoint_url = config_endpoint_url or config.get("endpoint_url")
     if config_endpoint_url:
-        warning_endpoint_url("~/.aws/config", config_endpoint_url)
+        warning_endpoint_url("~/.aws/config or ~/.aws/credentials", config_endpoint_url)
         return config_endpoint_url
     return endpoint_url
 
@@ -351,7 +351,7 @@ def get_s3_client_with_cache(
     )
 
 
-def s3_path_join(path: PathLike, *other_paths: PathLike) -> str:
+def _s3_path_join(path: PathLike, *other_paths: PathLike) -> str:
     """
     Concat 2 or more path to a complete path
 
@@ -366,7 +366,7 @@ def s3_path_join(path: PathLike, *other_paths: PathLike) -> str:
         and will directly concat.
 
         e.g. os.path.join('/path', 'to', '/file') => '/file',
-        but s3_path_join('/path', 'to', '/file') => '/path/to/file'
+        but _s3_path_join('/path', 'to', '/file') => '/path/to/file'
     """
     return uri_join(fspath(path), *map(fspath, other_paths))
 
@@ -603,7 +603,7 @@ def _s3_glob_stat_single_path(
         with raise_s3_error(_s3_pathname, S3BucketNotFoundError):
             for resp in _list_objects_recursive(client, bucket, prefix, delimiter):
                 for content in resp.get("Contents", []):
-                    path = s3_path_join(f"{protocol}://", bucket, content["Key"])
+                    path = _s3_path_join(f"{protocol}://", bucket, content["Key"])
                     if not search_dir and pattern.match(path):
                         if path.endswith("/"):
                             continue
@@ -619,7 +619,7 @@ def _s3_glob_stat_single_path(
                             )
                         dirname = os.path.dirname(dirname)
                 for common_prefix in resp.get("CommonPrefixes", []):
-                    path = s3_path_join(
+                    path = _s3_path_join(
                         f"{protocol}://", bucket, common_prefix["Prefix"]
                     )
                     dirname = os.path.dirname(path)
@@ -640,7 +640,7 @@ def _s3_scan_pairs(
     for src_file_path in S3Path(src_url).scan():
         content_path = src_file_path[len(fspath(src_url)) :]
         if len(content_path) > 0:
-            dst_file_path = s3_path_join(dst_url, content_path)
+            dst_file_path = _s3_path_join(dst_url, content_path)
         else:
             dst_file_path = dst_url
         yield src_file_path, dst_file_path
@@ -937,6 +937,7 @@ def s3_buffered_open(
     buffered: bool = False,
     share_cache_key: Optional[str] = None,
     cache_path: Optional[str] = None,
+    atomic: bool = False,
 ) -> IO:
     """Open an asynchronous prefetch reader, to support fast sequential read
 
@@ -1045,6 +1046,7 @@ def s3_buffered_open(
             block_size=block_size,
             max_buffer_size=max_buffer_size,
             profile_name=s3_url._profile_name,
+            atomic=atomic,
         )
     else:
         if max_buffer_size is None:
@@ -1057,6 +1059,7 @@ def s3_buffered_open(
             block_size=block_size,
             max_buffer_size=max_buffer_size,
             profile_name=s3_url._profile_name,
+            atomic=atomic,
         )
     if buffered or _is_pickle(writer):
         writer = io.BufferedWriter(writer)  # type: ignore
@@ -1100,6 +1103,27 @@ def s3_memory_open(
 s3_open = s3_buffered_open
 
 
+def s3_copy(
+    src_url: PathLike,
+    dst_url: PathLike,
+    callback: Optional[Callable[[int], None]] = None,
+    followlinks: bool = False,
+    overwrite: bool = True,
+) -> None:
+    """File copy on S3
+    Copy content of file on `src_path` to `dst_path`.
+    It's caller's responsibility to ensure the s3_isfile(src_url) is True
+
+    :param src_url: Given path
+    :param dst_path: Target file path
+    :param callback: Called periodically during copy, and the input parameter is
+        the data size (in bytes) of copy since the last call
+    :param followlinks: False if regard symlink as file, else True
+    :param overwrite: whether or not overwrite file when exists, default is True
+    """
+    return S3Path(src_url).copy(dst_url, callback, followlinks, overwrite)
+
+
 def s3_download(
     src_url: PathLike,
     dst_url: PathLike,
@@ -1117,8 +1141,7 @@ def s3_download(
     :param followlinks: False if regard symlink as file, else True
     :param overwrite: whether or not overwrite file when exists, default is True
     """
-    from megfile.fs import is_fs
-    from megfile.fs_path import FSPath
+    from megfile.fs_path import FSPath, is_fs
 
     dst_url = fspath(dst_url)
     if not is_fs(dst_url):
@@ -1198,8 +1221,7 @@ def s3_upload(
     :param followlinks: False if regard symlink as file, else True
     :param overwrite: whether or not overwrite file when exists, default is True
     """
-    from megfile.fs import is_fs
-    from megfile.fs_path import FSPath
+    from megfile.fs_path import FSPath, is_fs
 
     if not is_fs(src_url):
         raise OSError(f"src_url is not fs path: {src_url}")
@@ -1396,11 +1418,9 @@ class S3Path(URIPath):
     @cached_property
     def path_with_protocol(self) -> str:
         """Return path with protocol, like file:///root, s3://bucket/key"""
-        path = self.path
-        protocol_prefix = self._protocol_with_profile + "://"
-        if path.startswith(protocol_prefix):
-            return path
-        return protocol_prefix + path.lstrip("/")
+        if self.path.startswith(self.root):
+            return self.path
+        return self.root + self.path.lstrip("/")
 
     @cached_property
     def path_without_protocol(self) -> str:
@@ -1408,21 +1428,14 @@ class S3Path(URIPath):
         Return path without protocol, example: if path is s3://bucket/key,
         return bucket/key
         """
-        path = self.path
-        protocol_prefix = self._protocol_with_profile + "://"
-        if path.startswith(protocol_prefix):
-            path = path[len(protocol_prefix) :]
-        return path
+        if self.path.startswith(self.root):
+            return self.path[len(self.root) :]
+        return self.path
 
     @cached_property
-    def parts(self) -> Tuple[str, ...]:
-        """A tuple giving access to the path’s various components"""
-        parts = [f"{self._protocol_with_profile}://"]
-        path = self.path_without_protocol
-        path = path.lstrip("/")
-        if path != "":
-            parts.extend(path.split("/"))
-        return tuple(parts)
+    def root(self) -> str:
+        """Return root of the path, like s3://"""
+        return f"{self._protocol_with_profile}://"
 
     @cached_property
     def _client(self):
@@ -1991,7 +2004,7 @@ class S3Path(URIPath):
                     for content in resp.get("Contents", []):
                         if content["Key"].endswith("/"):
                             continue
-                        path = s3_path_join(f"{protocol}://", bucket, content["Key"])
+                        path = _s3_path_join(f"{protocol}://", bucket, content["Key"])
 
                         if followlinks:
                             try:
@@ -2238,9 +2251,7 @@ class S3Path(URIPath):
                 dirs = sorted(dirs)
                 stack.extend(reversed(dirs))
 
-                root = s3_path_join(
-                    f"{self._protocol_with_profile}://", bucket, current
-                )[:-1]
+                root = _s3_path_join(self.root, bucket, current)[:-1]
                 dirs = [path[len(current) :] for path in dirs]
                 files = sorted(path[len(current) :] for path in files)
                 if files or dirs or not current:

megfile/sftp2_path.py

@@ -14,6 +14,7 @@ from urllib.parse import urlsplit, urlunsplit
 import ssh2.session  # type: ignore
 import ssh2.sftp  # type: ignore
 from ssh2.exceptions import SFTPProtocolError  # type: ignore
+from ssh2.sftp_handle import SFTPAttributes  # type: ignore
 
 from megfile.config import SFTP_MAX_RETRY_TIMES
 from megfile.errors import SameFileError, _create_missing_ok_generator
@@ -32,12 +33,12 @@ __all__ = [
     "is_sftp2",
 ]
 
-SFTP2_USERNAME = "SFTP2_USERNAME"
-SFTP2_PASSWORD = "SFTP2_PASSWORD"
-SFTP2_PRIVATE_KEY_PATH = "SFTP2_PRIVATE_KEY_PATH"
-SFTP2_PRIVATE_KEY_TYPE = "SFTP2_PRIVATE_KEY_TYPE"
-SFTP2_PRIVATE_KEY_PASSWORD = "SFTP2_PRIVATE_KEY_PASSWORD"
-SFTP2_MAX_UNAUTH_CONN = "SFTP2_MAX_UNAUTH_CONN"
+SFTP_USERNAME = "SFTP_USERNAME"
+SFTP_PASSWORD = "SFTP_PASSWORD"
+SFTP_PRIVATE_KEY_PATH = "SFTP_PRIVATE_KEY_PATH"
+SFTP_PRIVATE_KEY_TYPE = "SFTP_PRIVATE_KEY_TYPE"
+SFTP_PRIVATE_KEY_PASSWORD = "SFTP_PRIVATE_KEY_PASSWORD"
+SFTP_MAX_UNAUTH_CONN = "SFTP_MAX_UNAUTH_CONN"
 MAX_RETRIES = SFTP_MAX_RETRY_TIMES
 DEFAULT_SSH_CONNECT_TIMEOUT = 5
 DEFAULT_SSH_KEEPALIVE_INTERVAL = 15
@@ -65,11 +66,11 @@ def _make_stat(stat) -> StatResult:
 
 def get_private_key():
     """Get private key for SSH authentication"""
-    private_key_path = os.getenv(SFTP2_PRIVATE_KEY_PATH)
+    private_key_path = os.getenv(SFTP_PRIVATE_KEY_PATH)
     if private_key_path:
         if not os.path.exists(private_key_path):
             raise FileNotFoundError(f"Private key file not exist: '{private_key_path}'")
-        private_key_password = os.getenv(SFTP2_PRIVATE_KEY_PASSWORD)
+        private_key_password = os.getenv(SFTP_PRIVATE_KEY_PASSWORD)
         if private_key_password:
             return private_key_path, private_key_password
         return private_key_path, ""
@@ -86,12 +87,12 @@ def provide_connect_info(
     if not port:
         port = 22
     if not username:
-        username = os.getenv(SFTP2_USERNAME)
+        username = os.getenv(SFTP_USERNAME)
         if not username:
             # 如果没有指定用户名，使用当前系统用户名
             username = getpass.getuser()
     if not password:
-        password = os.getenv(SFTP2_PASSWORD)
+        password = os.getenv(SFTP_PASSWORD)
     private_key = get_private_key()
     return hostname, port, username, password, private_key
 
@@ -411,10 +412,7 @@ class Sftp2Path(URIPath):
     """sftp2 protocol
 
     uri format:
-        - absolute path
-            - sftp2://[username[:password]@]hostname[:port]//file_path
-        - relative path
-            - sftp2://[username[:password]@]hostname[:port]/file_path
+        - sftp2://[username[:password]@]hostname[:port]/file_path
     """
 
     protocol = "sftp2"
@@ -423,27 +421,12 @@ class Sftp2Path(URIPath):
         super().__init__(path, *other_paths)
         parts = urlsplit(self.path)
         self._urlsplit_parts = parts
-        self._real_path = parts.path
-        if parts.path.startswith("//"):
-            self._root_dir = "/"
-        else:
-            self._root_dir = "/"  # Default to absolute path for ssh2
-        self._real_path = (
-            parts.path.lstrip("/")
-            if not parts.path.startswith("//")
-            else parts.path[2:]
-        )
-        if not self._real_path.startswith("/"):
-            self._real_path = f"/{self._real_path}"
+        self._remote_path = parts.path or "/"
 
     @cached_property
     def parts(self) -> Tuple[str, ...]:
         """A tuple giving access to the path's various components"""
-        if self._urlsplit_parts.path.startswith("//"):
-            new_parts = self._urlsplit_parts._replace(path="//")
-        else:
-            new_parts = self._urlsplit_parts._replace(path="/")
-        parts = [urlunsplit(new_parts)]
+        parts = [urlunsplit(self._urlsplit_parts._replace(path=""))]
         path = self._urlsplit_parts.path.lstrip("/")
         if path != "":
             parts.extend(path.split("/"))
@@ -511,12 +494,6 @@ class Sftp2Path(URIPath):
         )
 
     def _generate_path_object(self, sftp_local_path: str, resolve: bool = False):
-        if resolve or self._root_dir == "/":
-            sftp_local_path = f"//{sftp_local_path.lstrip('/')}"
-        else:
-            sftp_local_path = os.path.relpath(sftp_local_path, start=self._root_dir)
-            if sftp_local_path == ".":
-                sftp_local_path = "/"
         new_parts = self._urlsplit_parts._replace(path=sftp_local_path)
         return self.from_path(urlunsplit(new_parts))
 
@@ -580,12 +557,12 @@ class Sftp2Path(URIPath):
             return self.from_path(path).is_dir(followlinks=followlinks)
 
         fs = FSFunc(_exist, _is_dir, _scandir)
-        for real_path in _create_missing_ok_generator(
+        for remote_path in _create_missing_ok_generator(
             iglob(fspath(glob_path), recursive=recursive, fs=fs),
             missing_ok,
             FileNotFoundError(f"No match any file: {glob_path!r}"),
         ):
-            yield self.from_path(real_path)
+            yield self.from_path(remote_path)
 
     def is_dir(self, followlinks: bool = False) -> bool:
         """Test if a path is directory"""
@@ -639,7 +616,7 @@ class Sftp2Path(URIPath):
             for parent_path_object in parent_path_objects[::-1]:
                 parent_path_object.mkdir(mode=mode, parents=False, exist_ok=True)
         try:
-            self._client.mkdir(self._real_path, mode)
+            self._client.mkdir(self._remote_path, mode)
         except OSError:
             if not self.exists():
                 raise
@@ -670,7 +647,7 @@ class Sftp2Path(URIPath):
         if self._is_same_backend(dst_path):
             if overwrite:
                 dst_path.remove(missing_ok=True)
-                self._client.rename(self._real_path, dst_path._real_path)
+                self._client.rename(self._remote_path, dst_path._remote_path)
             else:
                 self.sync(dst_path, overwrite=overwrite)
                 self.remove(missing_ok=True)
@@ -680,7 +657,7 @@ class Sftp2Path(URIPath):
                     self.from_path(file_entry.path).rename(
                         dst_path.joinpath(file_entry.name)
                     )
-                self._client.rmdir(self._real_path)
+                self._client.rmdir(self._remote_path)
             else:
                 if overwrite or not dst_path.exists():
                     with self.open("rb") as fsrc:
@@ -703,9 +680,9 @@ class Sftp2Path(URIPath):
         if self.is_dir():
             for file_entry in self.scandir():
                 self.from_path(file_entry.path).remove(missing_ok=missing_ok)
-            self._client.rmdir(self._real_path)
+            self._client.rmdir(self._remote_path)
         else:
-            self._client.unlink(self._real_path)
+            self._client.unlink(self._remote_path)
 
     def scan(self, missing_ok: bool = True, followlinks: bool = False) -> Iterator[str]:
         """Iteratively traverse only files in given directory"""
@@ -752,7 +729,7 @@ class Sftp2Path(URIPath):
 
     def scandir(self) -> ContextIterator:
         """Get all content of given file path"""
-        real_path = self._real_path
+        remote_path = self._remote_path
         stat_result = None
         try:
             stat_result = self.stat(follow_symlinks=False)
@@ -760,13 +737,13 @@ class Sftp2Path(URIPath):
             raise NotADirectoryError(f"Not a directory: '{self.path_with_protocol}'")
 
         if stat_result.is_symlink():
-            real_path = self.readlink()._real_path
+            remote_path = self.readlink()._remote_path
         elif not stat_result.is_dir():
             raise NotADirectoryError(f"Not a directory: '{self.path_with_protocol}'")
 
         def create_generator():
             # Use opendir and readdir from ssh2-python
-            dir_handle = self._client.opendir(real_path)
+            dir_handle = self._client.opendir(remote_path)
             try:
                 # ssh2-python's readdir returns a generator
                 # First call returns all entries, subsequent calls return empty
@@ -794,9 +771,9 @@ class Sftp2Path(URIPath):
         """Get StatResult of file on sftp2"""
         try:
             if follow_symlinks:
-                stat = self._client.stat(self._real_path)
+                stat = self._client.stat(self._remote_path)
             else:
-                stat = self._client.lstat(self._real_path)
+                stat = self._client.lstat(self._remote_path)
             return _make_stat(stat)
         except SFTPProtocolError as e:  # pytype: disable=mro-error
             raise FileNotFoundError(
@@ -811,7 +788,7 @@ class Sftp2Path(URIPath):
         """Remove the file on sftp2"""
         if missing_ok and not self.exists():
             return
-        self._client.unlink(self._real_path)
+        self._client.unlink(self._remote_path)
 
     def walk(
         self, followlinks: bool = False
@@ -823,7 +800,7 @@ class Sftp2Path(URIPath):
         if self.is_file(followlinks=followlinks):
             return
 
-        stack = [self._real_path]
+        stack = [self._remote_path]
         while stack:
             root = stack.pop()
             dirs, files = [], []
@@ -848,8 +825,8 @@ class Sftp2Path(URIPath):
 
     def resolve(self, strict=False) -> "Sftp2Path":
         """Return the canonical path"""
-        path = self._client.realpath(self._real_path)
-        return self._generate_path_object(path, resolve=True)
+        path = self._client.realpath(self._remote_path)
+        return self._generate_path_object(path)
 
     def md5(self, recalculate: bool = False, followlinks: bool = False):
         """Calculate the md5 value of the file"""
@@ -872,7 +849,7 @@ class Sftp2Path(URIPath):
         dst_path = self.from_path(dst_path)
         if dst_path.exists(followlinks=False):
             raise FileExistsError(f"File exists: '{dst_path.path_with_protocol}'")
-        return self._client.symlink(self._real_path, dst_path._real_path)
+        return self._client.symlink(self._remote_path, dst_path._remote_path)
 
     def readlink(self) -> "Sftp2Path":
         """Return a Sftp2Path instance representing the path to which the
@@ -884,7 +861,7 @@ class Sftp2Path(URIPath):
         if not self.is_symlink():
             raise OSError(f"Not a symlink: {self.path_with_protocol!r}")
         try:
-            path = self._client.readlink(self._real_path)
+            path = self._client.realpath(self._remote_path)
             if not path:
                 raise OSError(f"Not a symlink: {self.path_with_protocol!r}")
             if not path.startswith("/"):
@@ -948,7 +925,7 @@ class Sftp2Path(URIPath):
                 | ssh2.sftp.LIBSSH2_FXF_APPEND
             )
 
-        sftp_handle = self._client.open(self._real_path, ssh2_mode, 0o644)
+        sftp_handle = self._client.open(self._remote_path, ssh2_mode, 0o644)
 
         # Create raw file wrapper
         raw_file = Sftp2RawFile(sftp_handle, self.path, mode)
@@ -976,7 +953,9 @@ class Sftp2Path(URIPath):
 
     def chmod(self, mode: int, *, follow_symlinks: bool = True):
         """Change the file mode and permissions"""
-        return self._client.setstat(self._real_path, mode)
+        stat = SFTPAttributes()
+        stat.permissions = int(mode)
+        return self._client.setstat(self._remote_path, stat)
 
     def absolute(self) -> "Sftp2Path":
         """Make the path absolute"""
@@ -986,7 +965,7 @@ class Sftp2Path(URIPath):
         """Remove this directory. The directory must be empty"""
         if len(self.listdir()) > 0:
             raise OSError(f"Directory not empty: '{self.path_with_protocol}'")
-        return self._client.rmdir(self._real_path)
+        return self._client.rmdir(self._remote_path)
 
     def copy(
         self,
@@ -1014,7 +993,7 @@ class Sftp2Path(URIPath):
         dst_path = self.from_path(dst_path)
 
         if self._is_same_backend(dst_path):
-            if self._real_path == dst_path._real_path:
+            if self._remote_path == dst_path._remote_path:
                 raise SameFileError(
                     f"'{self.path}' and '{dst_path.path}' are the same file"
                 )
@@ -1022,8 +1001,8 @@ class Sftp2Path(URIPath):
             exec_result = self._exec_command(
                 [
                     "cp",
-                    self._real_path,
-                    dst_path._real_path,
+                    self._remote_path,
+                    dst_path._remote_path,
                 ]
             )
 
@@ -1081,4 +1060,7 @@ class Sftp2Path(URIPath):
 
     def utime(self, atime: Union[float, int], mtime: Union[float, int]) -> None:
         """Set the access and modified times of the file"""
-        self._client.utime(self._real_path, (atime, mtime))
+        stat = SFTPAttributes()
+        stat.atime = int(atime)
+        stat.mtime = int(mtime)
+        self._client.setstat(self._remote_path, stat)