megfile 3.0.4__py3-none-any.whl → 3.0.6__py3-none-any.whl

megfile/s3_path.py

@@ -12,7 +12,7 @@ import boto3
 import botocore
 from botocore.awsrequest import AWSResponse
 
-from megfile.config import DEFAULT_BLOCK_SIZE, GLOBAL_MAX_WORKERS, S3_CLIENT_CACHE_MODE
+from megfile.config import DEFAULT_BLOCK_SIZE, GLOBAL_MAX_WORKERS, S3_CLIENT_CACHE_MODE, S3_MAX_RETRY_TIMES
 from megfile.errors import S3BucketNotFoundError, S3ConfigError, S3FileExistsError, S3FileNotFoundError, S3IsADirectoryError, S3NameTooLongError, S3NotADirectoryError, S3NotALinkError, S3PermissionError, S3UnknownError, SameFileError, UnsupportedError, _create_missing_ok_generator
 from megfile.errors import _logger as error_logger
 from megfile.errors import patch_method, raise_s3_error, s3_error_code_should_retry, s3_should_retry, translate_fs_error, translate_s3_error
@@ -69,7 +69,7 @@ _logger = get_logger(__name__)
 content_md5_header = 'megfile-content-md5'
 endpoint_url = 'https://s3.amazonaws.com'
 max_pool_connections = GLOBAL_MAX_WORKERS  # for compatibility
-max_retries = 10
+max_retries = S3_MAX_RETRY_TIMES
 max_keys = 1000
 
 
@@ -930,16 +930,29 @@ def s3_download(
         src_url: PathLike,
         dst_url: PathLike,
         followlinks: bool = False,
-        callback: Optional[Callable[[int], None]] = None) -> None:
+        callback: Optional[Callable[[int], None]] = None,
+        overwrite: bool = True) -> None:
     '''
     Downloads a file from s3 to local filesystem.
     :param src_url: source s3 path
     :param dst_url: target fs path
     :param callback: Called periodically during copy, and the input parameter is the data size (in bytes) of copy since the last call
+    :param followlinks: False if regard symlink as file, else True
+    :param overwrite: whether or not overwrite file when exists, default is True
     '''
     from megfile.fs import is_fs
     from megfile.fs_path import FSPath
 
+    dst_url = fspath(dst_url)
+    if not is_fs(dst_url):
+        raise OSError(f'dst_url is not fs path: {dst_url}')
+    if not dst_url or dst_url.endswith('/'):
+        raise S3IsADirectoryError('Is a directory: %r' % dst_url)
+
+    dst_path = FSPath(dst_url)
+    if not overwrite and dst_path.exists():
+        return
+
     if not isinstance(src_url, S3Path):
         src_url = S3Path(src_url)
     if followlinks:
@@ -960,13 +973,6 @@ def s3_download(
         raise S3IsADirectoryError(
             'Is a directory: %r' % src_url.path_with_protocol)
 
-    dst_url = fspath(dst_url)
-    if not is_fs(dst_url):
-        raise OSError(f'dst_url is not fs path: {dst_url}')
-    if not dst_url or dst_url.endswith('/'):
-        raise S3IsADirectoryError('Is a directory: %r' % dst_url)
-
-    dst_path = FSPath(dst_url)
     dst_directory = os.path.dirname(dst_path.path_without_protocol)
     if dst_directory != '':
         os.makedirs(dst_directory, exist_ok=True)
@@ -997,12 +1003,14 @@ def s3_upload(
         src_url: PathLike,
         dst_url: PathLike,
         callback: Optional[Callable[[int], None]] = None,
+        overwrite: bool = True,
         **kwargs) -> None:
     '''
     Uploads a file from local filesystem to s3.
     :param src_url: source fs path
     :param dst_url: target s3 path
     :param callback: Called periodically during copy, and the input parameter is the data size (in bytes) of copy since the last call
+    :param overwrite: whether or not overwrite file when exists, default is True
     '''
     from megfile.fs import is_fs
     from megfile.fs_path import FSPath
@@ -1017,6 +1025,9 @@ def s3_upload(
     if not dst_key or dst_key.endswith('/'):
         raise S3IsADirectoryError('Is a directory: %r' % dst_url)
 
+    if not overwrite and S3Path(dst_url).is_file():
+        return
+
     client = get_s3_client_with_cache(
         profile_name=S3Path(dst_url)._profile_name)
     upload_fileobj = patch_method(
@@ -1086,18 +1097,15 @@ def s3_readlink(path) -> str:
     return S3Path(path).readlink().path_with_protocol
 
 
-def s3_rename(src_url: PathLike, dst_url: PathLike):
+def s3_rename(
+        src_url: PathLike, dst_url: PathLike, overwrite: bool = True) -> None:
     '''
     Move s3 file path from src_url to dst_url
 
     :param dst_url: Given destination path
+    :param overwrite: whether or not overwrite file when exists
     '''
-    src_path_ins = S3Path(src_url)
-    if src_path_ins.is_file():
-        src_path_ins.copy(dst_url)
-    else:
-        src_path_ins.sync(dst_url)
-    src_path_ins.remove()
+    S3Path(src_url).rename(dst_url, overwrite)
 
 
 class S3Cacher(FileCacher):
@@ -1692,15 +1700,16 @@ class S3Path(URIPath):
         if self.exists():
             raise S3FileExistsError('File exists: %r' % self.path_with_protocol)
 
-    def move(self, dst_url: PathLike) -> None:
+    def move(self, dst_url: PathLike, overwrite: bool = True) -> None:
         '''
         Move file/directory path from src_url to dst_url
 
         :param dst_url: Given destination path
+        :param overwrite: whether or not overwrite file when exists
         '''
         for src_file_path, dst_file_path in _s3_scan_pairs(
                 self.path_with_protocol, dst_url):
-            S3Path(src_file_path).rename(dst_file_path)
+            S3Path(src_file_path).rename(dst_file_path, overwrite)
 
     def remove(self, missing_ok: bool = False) -> None:
         '''
@@ -1774,13 +1783,18 @@ class S3Path(URIPath):
                 raise S3UnknownError(
                     Exception(error_msg), self.path_with_protocol)
 
-    def rename(self, dst_path: PathLike) -> 'S3Path':
+    def rename(self, dst_path: PathLike, overwrite: bool = True) -> 'S3Path':
         '''
         Move s3 file path from src_url to dst_url
 
         :param dst_path: Given destination path
+        :param overwrite: whether or not overwrite file when exists
         '''
-        s3_rename(self.path_with_protocol, dst_path)
+        if self.is_file():
+            self.copy(dst_path, overwrite=overwrite)
+        else:
+            self.sync(dst_path, overwrite=overwrite)
+        self.remove(missing_ok=True)
         return self.from_path(dst_path)
 
     def scan(self, missing_ok: bool = True,
@@ -2111,15 +2125,21 @@ class S3Path(URIPath):
     def copy(
             self,
             dst_url: PathLike,
+            callback: Optional[Callable[[int], None]] = None,
             followlinks: bool = False,
-            callback: Optional[Callable[[int], None]] = None) -> None:
+            overwrite: bool = True) -> None:
         ''' File copy on S3
         Copy content of file on `src_path` to `dst_path`.
         It's caller's responsebility to ensure the s3_isfile(src_url) == True
 
         :param dst_path: Target file path
         :param callback: Called periodically during copy, and the input parameter is the data size (in bytes) of copy since the last call
+        :param followlinks: False if regard symlink as file, else True
+        :param overwrite: whether or not overwrite file when exists, default is True
         '''
+        if not overwrite and self.from_path(dst_url).is_file():
+            return
+
         src_url = self.path_with_protocol
         src_bucket, src_key = parse_s3_url(src_url)
         dst_bucket, dst_key = parse_s3_url(dst_url)
@@ -2156,22 +2176,32 @@ class S3Path(URIPath):
                 Callback=callback)
 
     def sync(
-            self, dst_url: PathLike, followlinks: bool = False,
-            force: bool = False) -> None:
+            self,
+            dst_url: PathLike,
+            followlinks: bool = False,
+            force: bool = False,
+            overwrite: bool = True) -> None:
         '''
         Copy file/directory on src_url to dst_url
 
         :param dst_url: Given destination path
         :param followlinks: False if regard symlink as file, else True
-        :param force: Sync file forcely, do not ignore same files
+        :param force: Sync file forcely, do not ignore same files, priority is higher than 'overwrite', default is False
+        :param overwrite: whether or not overwrite file when exists, default is True
         '''
         for src_file_path, dst_file_path in _s3_scan_pairs(
                 self.path_with_protocol, dst_url):
             src_file_path = self.from_path(src_file_path)
             dst_file_path = self.from_path(dst_file_path)
-            if not force and dst_file_path.exists() and is_same_file(
+
+            if force:
+                pass
+            elif not overwrite and dst_file_path.exists():
+                continue
+            elif dst_file_path.exists() and is_same_file(
                     src_file_path.stat(), dst_file_path.stat(), 'copy'):
                 continue
+
             src_file_path.copy(dst_file_path, followlinks=followlinks)
 
     def symlink(self, dst_path: PathLike) -> None:
@@ -2181,7 +2211,7 @@ class S3Path(URIPath):
         :param dst_path: Desination path
         :raises: S3NameTooLongError, S3BucketNotFoundError, S3IsADirectoryError
         '''
-        if len(str(self._s3_path).encode()) > 1024:
+        if len(fspath(self._s3_path).encode()) > 1024:
             raise S3NameTooLongError('File name too long: %r' % dst_path)
         src_bucket, src_key = parse_s3_url(self.path_with_protocol)
         dst_bucket, dst_key = parse_s3_url(dst_path)

megfile/sftp.py

@@ -162,24 +162,30 @@ def sftp_realpath(path: PathLike) -> str:
     return SftpPath(path).realpath()
 
 
-def sftp_rename(src_path: PathLike, dst_path: PathLike) -> 'SftpPath':
+def sftp_rename(
+        src_path: PathLike, dst_path: PathLike,
+        overwrite: bool = True) -> 'SftpPath':
     '''
     rename file on sftp
 
     :param src_path: Given path
     :param dst_path: Given destination path
+    :param overwrite: whether or not overwrite file when exists
     '''
-    return SftpPath(src_path).rename(dst_path)
+    return SftpPath(src_path).rename(dst_path, overwrite)
 
 
-def sftp_move(src_path: PathLike, dst_path: PathLike) -> 'SftpPath':
+def sftp_move(
+        src_path: PathLike, dst_path: PathLike,
+        overwrite: bool = True) -> 'SftpPath':
     '''
     move file on sftp
 
     :param src_path: Given path
     :param dst_path: Given destination path
+    :param overwrite: whether or not overwrite file when exists
     '''
-    return SftpPath(src_path).replace(dst_path)
+    return SftpPath(src_path).replace(dst_path, overwrite)
 
 
 def sftp_remove(path: PathLike, missing_ok: bool = False) -> None:
@@ -369,7 +375,8 @@ def sftp_copy(
         src_path: PathLike,
         dst_path: PathLike,
         callback: Optional[Callable[[int], None]] = None,
-        followlinks: bool = False):
+        followlinks: bool = False,
+        overwrite: bool = True):
     """
     Copy the file to the given destination path.
 
@@ -381,19 +388,21 @@ def sftp_copy(
     :raises IsADirectoryError: If the source is a directory.
     :raises OSError: If there is an error copying the file.
     """
-    return SftpPath(src_path).copy(dst_path, callback, followlinks)
+    return SftpPath(src_path).copy(dst_path, callback, followlinks, overwrite)
 
 
 def sftp_sync(
         src_path: PathLike,
         dst_path: PathLike,
         followlinks: bool = False,
-        force: bool = False):
+        force: bool = False,
+        overwrite: bool = True):
     '''Copy file/directory on src_url to dst_url
 
     :param src_path: Given path
     :param dst_url: Given destination path
     :param followlinks: False if regard symlink as file, else True
-    :param force: Sync file forcely, do not ignore same files
+    :param force: Sync file forcely, do not ignore same files, priority is higher than 'overwrite', default is False
+    :param overwrite: whether or not overwrite file when exists, default is True
     '''
-    return SftpPath(src_path).sync(dst_path, followlinks, force)
+    return SftpPath(src_path).sync(dst_path, followlinks, force, overwrite)

megfile/sftp_path.py

@@ -14,6 +14,7 @@ from urllib.parse import urlsplit, urlunsplit
 
 import paramiko
 
+from megfile.config import SFTP_MAX_RETRY_TIMES
 from megfile.errors import SameFileError, _create_missing_ok_generator, patch_method
 from megfile.interfaces import ContextIterator, FileEntry, PathLike, StatResult
 from megfile.lib.compare import is_same_file
@@ -47,7 +48,7 @@ SFTP_PRIVATE_KEY_PATH = "SFTP_PRIVATE_KEY_PATH"
 SFTP_PRIVATE_KEY_TYPE = "SFTP_PRIVATE_KEY_TYPE"
 SFTP_PRIVATE_KEY_PASSWORD = "SFTP_PRIVATE_KEY_PASSWORD"
 SFTP_MAX_UNAUTH_CONN = "SFTP_MAX_UNAUTH_CONN"
-MAX_RETRIES = 10
+MAX_RETRIES = SFTP_MAX_RETRY_TIMES
 DEFAULT_SSH_CONNECT_TIMEOUT = 5
 DEFAULT_SSH_KEEPALIVE_INTERVAL = 15
 
@@ -416,9 +417,15 @@ def sftp_download(
         src_url: PathLike,
         dst_url: PathLike,
         callback: Optional[Callable[[int], None]] = None,
-        followlinks: bool = False):
+        followlinks: bool = False,
+        overwrite: bool = True):
     '''
-    File download
+    Downloads a file from sftp to local filesystem.
+    :param src_url: source sftp path
+    :param dst_url: target fs path
+    :param callback: Called periodically during copy, and the input parameter is the data size (in bytes) of copy since the last call
+    :param followlinks: False if regard symlink as file, else True
+    :param overwrite: whether or not overwrite file when exists, default is True
     '''
     from megfile.fs import is_fs
     from megfile.fs_path import FSPath
@@ -428,6 +435,10 @@ def sftp_download(
     if not is_sftp(src_url) and not isinstance(src_url, SftpPath):
         raise OSError(f'src_url is not sftp path: {src_url}')
 
+    dst_path = FSPath(dst_url)
+    if not overwrite and dst_path.exists():
+        return
+
     if isinstance(src_url, SftpPath):
         src_path = src_url
     else:
@@ -440,7 +451,6 @@ def sftp_download(
     if str(dst_url).endswith('/'):
         raise IsADirectoryError('Is a directory: %r' % dst_url)
 
-    dst_path = FSPath(dst_url)
     dst_path.parent.makedirs(exist_ok=True)
 
     sftp_callback = None
@@ -466,9 +476,14 @@ def sftp_upload(
         src_url: PathLike,
         dst_url: PathLike,
         callback: Optional[Callable[[int], None]] = None,
-        followlinks: bool = False):
+        followlinks: bool = False,
+        overwrite: bool = True):
     '''
-    File upload
+    Uploads a file from local filesystem to sftp server.
+    :param src_url: source fs path
+    :param dst_url: target sftp path
+    :param callback: Called periodically during copy, and the input parameter is the data size (in bytes) of copy since the last call
+    :param overwrite: whether or not overwrite file when exists, default is True
     '''
     from megfile.fs import is_fs
     from megfile.fs_path import FSPath
@@ -490,6 +505,9 @@ def sftp_upload(
         dst_path = dst_url
     else:
         dst_path = SftpPath(dst_url)
+    if not overwrite and dst_path.exists():
+        return
+
     dst_path.parent.makedirs(exist_ok=True)
 
     sftp_callback = None
@@ -858,53 +876,57 @@ class SftpPath(URIPath):
     def _is_same_protocol(self, path):
         return is_sftp(path)
 
-    def rename(self, dst_path: PathLike) -> 'SftpPath':
+    def rename(self, dst_path: PathLike, overwrite: bool = True) -> 'SftpPath':
         '''
         rename file on sftp
 
         :param dst_path: Given destination path
+        :param overwrite: whether or not overwrite file when exists
         '''
         if not self._is_same_protocol(dst_path):
             raise OSError('Not a %s path: %r' % (self.protocol, dst_path))
-        if str(dst_path).endswith('/'):
-            raise IsADirectoryError('Is a directory: %r' % dst_path)
 
-        dst_path = self.from_path(dst_path)
+        dst_path = self.from_path(str(dst_path).rstrip('/'))
+
         src_stat = self.stat()
 
         if self._is_same_backend(dst_path):
-            try:
+            if overwrite:
+                dst_path.remove(missing_ok=True)
                 self._client.rename(self._real_path, dst_path._real_path)
-            except OSError:
-                if dst_path.exists():
-                    raise FileExistsError('File exists: %s' % dst_path)
+            else:
+                self.sync(dst_path, overwrite=overwrite)
+                self.remove(missing_ok=True)
         else:
             if self.is_dir():
                 for file_entry in self.scandir():
                     self.from_path(file_entry.path).rename(
                         dst_path.joinpath(file_entry.name))
+                self._client.rmdir(self._real_path)
             else:
-                with self.open('rb') as fsrc:
-                    with dst_path.open('wb') as fdst:
-                        length = 16 * 1024
-                        while True:
-                            buf = fsrc.read(length)
-                            if not buf:
-                                break
-                            fdst.write(buf)
+                if overwrite or not dst_path.exists():
+                    with self.open('rb') as fsrc:
+                        with dst_path.open('wb') as fdst:
+                            length = 16 * 1024
+                            while True:
+                                buf = fsrc.read(length)
+                                if not buf:
+                                    break
+                                fdst.write(buf)
                 self.unlink()
 
         dst_path.utime(src_stat.st_atime, src_stat.st_mtime)
         dst_path.chmod(src_stat.st_mode)
         return dst_path
 
-    def replace(self, dst_path: PathLike) -> 'SftpPath':
+    def replace(self, dst_path: PathLike, overwrite: bool = True) -> 'SftpPath':
         '''
         move file on sftp
 
         :param dst_path: Given destination path
+        :param overwrite: whether or not overwrite file when exists
         '''
-        return self.rename(dst_path=dst_path)
+        return self.rename(dst_path=dst_path, overwrite=overwrite)
 
     def remove(self, missing_ok: bool = False) -> None:
         '''
@@ -1234,7 +1256,8 @@ class SftpPath(URIPath):
             self,
             dst_path: PathLike,
             callback: Optional[Callable[[int], None]] = None,
-            followlinks: bool = False):
+            followlinks: bool = False,
+            overwrite: bool = True):
         """
         Copy the file to the given destination path.
 
@@ -1257,6 +1280,9 @@ class SftpPath(URIPath):
             raise IsADirectoryError(
                 'Is a directory: %r' % self.path_with_protocol)
 
+        if not overwrite and self.from_path(dst_path).exists():
+            return
+
         self.from_path(os.path.dirname(dst_path)).makedirs(exist_ok=True)
         dst_path = self.from_path(dst_path)
         if self._is_same_backend(dst_path):
@@ -1291,12 +1317,14 @@ class SftpPath(URIPath):
             self,
             dst_path: PathLike,
             followlinks: bool = False,
-            force: bool = False):
+            force: bool = False,
+            overwrite: bool = True):
         '''Copy file/directory on src_url to dst_url
 
         :param dst_url: Given destination path
         :param followlinks: False if regard symlink as file, else True
-        :param force: Sync file forcely, do not ignore same files
+        :param force: Sync file forcely, do not ignore same files, priority is higher than 'overwrite', default is False
+        :param overwrite: whether or not overwrite file when exists, default is True
         '''
         if not self._is_same_protocol(dst_path):
             raise OSError('Not a %s path: %r' % (self.protocol, dst_path))
@@ -1305,11 +1333,15 @@ class SftpPath(URIPath):
                 self.path_with_protocol, dst_path):
             dst_path = self.from_path(dst_file_path)
             src_path = self.from_path(src_file_path)
-            if not force and dst_path.exists() and is_same_file(
-                    src_path.stat(), dst_path.stat(), 'copy'):
+
+            if force:
+                pass
+            elif not overwrite and dst_path.exists():
                 continue
-            self.from_path(os.path.dirname(dst_file_path)).mkdir(
-                parents=True, exist_ok=True)
+            elif dst_path.exists() and is_same_file(src_path.stat(),
+                                                    dst_path.stat(), 'copy'):
+                continue
+
             self.from_path(src_file_path).copy(
                 dst_file_path, followlinks=followlinks)