megfile 3.0.5__py3-none-any.whl → 3.0.6.post1__py3-none-any.whl

megfile/s3.py

@@ -169,14 +169,16 @@ def s3_hasbucket(path: PathLike) -> bool:
     return S3Path(path).hasbucket()
 
 
-def s3_move(src_url: PathLike, dst_url: PathLike) -> None:
+def s3_move(
+        src_url: PathLike, dst_url: PathLike, overwrite: bool = True) -> None:
     '''
     Move file/directory path from src_url to dst_url
 
     :param src_url: Given path
     :param dst_url: Given destination path
+    :param overwrite: whether or not overwrite file when exists
     '''
-    return S3Path(src_url).move(dst_url)
+    return S3Path(src_url).move(dst_url, overwrite)
 
 
 def s3_remove(path: PathLike, missing_ok: bool = False) -> None:
@@ -305,8 +307,9 @@ def s3_getmd5(
 def s3_copy(
         src_url: PathLike,
         dst_url: PathLike,
+        callback: Optional[Callable[[int], None]] = None,
         followlinks: bool = False,
-        callback: Optional[Callable[[int], None]] = None) -> None:
+        overwrite: bool = True) -> None:
     ''' File copy on S3
     Copy content of file on `src_path` to `dst_path`.
     It's caller's responsebility to ensure the s3_isfile(src_url) == True
@@ -314,24 +317,28 @@ def s3_copy(
     :param src_url: Given path
     :param dst_path: Target file path
     :param callback: Called periodically during copy, and the input parameter is the data size (in bytes) of copy since the last call
+    :param followlinks: False if regard symlink as file, else True
+    :param overwrite: whether or not overwrite file when exists, default is True
     '''
-    return S3Path(src_url).copy(dst_url, followlinks, callback)
+    return S3Path(src_url).copy(dst_url, callback, followlinks, overwrite)
 
 
 def s3_sync(
         src_url: PathLike,
         dst_url: PathLike,
         followlinks: bool = False,
-        force: bool = False) -> None:
+        force: bool = False,
+        overwrite: bool = True) -> None:
     '''
     Copy file/directory on src_url to dst_url
 
     :param src_url: Given path
     :param dst_url: Given destination path
     :param followlinks: False if regard symlink as file, else True
-    :param force: Sync file forcely, do not ignore same files
+    :param force: Sync file forcely, do not ignore same files, priority is higher than 'overwrite', default is False
+    :param overwrite: whether or not overwrite file when exists, default is True
     '''
-    return S3Path(src_url).sync(dst_url, followlinks, force)
+    return S3Path(src_url).sync(dst_url, followlinks, force, overwrite)
 
 
 def s3_symlink(src_path: PathLike, dst_path: PathLike) -> None:

megfile/s3_path.py

@@ -930,16 +930,29 @@ def s3_download(
         src_url: PathLike,
         dst_url: PathLike,
         followlinks: bool = False,
-        callback: Optional[Callable[[int], None]] = None) -> None:
+        callback: Optional[Callable[[int], None]] = None,
+        overwrite: bool = True) -> None:
     '''
     Downloads a file from s3 to local filesystem.
     :param src_url: source s3 path
     :param dst_url: target fs path
     :param callback: Called periodically during copy, and the input parameter is the data size (in bytes) of copy since the last call
+    :param followlinks: False if regard symlink as file, else True
+    :param overwrite: whether or not overwrite file when exists, default is True
     '''
     from megfile.fs import is_fs
     from megfile.fs_path import FSPath
 
+    dst_url = fspath(dst_url)
+    if not is_fs(dst_url):
+        raise OSError(f'dst_url is not fs path: {dst_url}')
+    if not dst_url or dst_url.endswith('/'):
+        raise S3IsADirectoryError('Is a directory: %r' % dst_url)
+
+    dst_path = FSPath(dst_url)
+    if not overwrite and dst_path.exists():
+        return
+
     if not isinstance(src_url, S3Path):
         src_url = S3Path(src_url)
     if followlinks:
@@ -960,13 +973,6 @@ def s3_download(
         raise S3IsADirectoryError(
             'Is a directory: %r' % src_url.path_with_protocol)
 
-    dst_url = fspath(dst_url)
-    if not is_fs(dst_url):
-        raise OSError(f'dst_url is not fs path: {dst_url}')
-    if not dst_url or dst_url.endswith('/'):
-        raise S3IsADirectoryError('Is a directory: %r' % dst_url)
-
-    dst_path = FSPath(dst_url)
     dst_directory = os.path.dirname(dst_path.path_without_protocol)
     if dst_directory != '':
         os.makedirs(dst_directory, exist_ok=True)
@@ -997,12 +1003,14 @@ def s3_upload(
         src_url: PathLike,
         dst_url: PathLike,
         callback: Optional[Callable[[int], None]] = None,
+        overwrite: bool = True,
         **kwargs) -> None:
     '''
     Uploads a file from local filesystem to s3.
     :param src_url: source fs path
     :param dst_url: target s3 path
     :param callback: Called periodically during copy, and the input parameter is the data size (in bytes) of copy since the last call
+    :param overwrite: whether or not overwrite file when exists, default is True
     '''
     from megfile.fs import is_fs
     from megfile.fs_path import FSPath
@@ -1017,6 +1025,9 @@ def s3_upload(
     if not dst_key or dst_key.endswith('/'):
         raise S3IsADirectoryError('Is a directory: %r' % dst_url)
 
+    if not overwrite and S3Path(dst_url).is_file():
+        return
+
     client = get_s3_client_with_cache(
         profile_name=S3Path(dst_url)._profile_name)
     upload_fileobj = patch_method(
@@ -1086,18 +1097,15 @@ def s3_readlink(path) -> str:
     return S3Path(path).readlink().path_with_protocol
 
 
-def s3_rename(src_url: PathLike, dst_url: PathLike):
+def s3_rename(
+        src_url: PathLike, dst_url: PathLike, overwrite: bool = True) -> None:
     '''
     Move s3 file path from src_url to dst_url
 
     :param dst_url: Given destination path
+    :param overwrite: whether or not overwrite file when exists
     '''
-    src_path_ins = S3Path(src_url)
-    if src_path_ins.is_file():
-        src_path_ins.copy(dst_url)
-    else:
-        src_path_ins.sync(dst_url)
-    src_path_ins.remove()
+    S3Path(src_url).rename(dst_url, overwrite)
 
 
 class S3Cacher(FileCacher):
@@ -1692,15 +1700,16 @@ class S3Path(URIPath):
         if self.exists():
             raise S3FileExistsError('File exists: %r' % self.path_with_protocol)
 
-    def move(self, dst_url: PathLike) -> None:
+    def move(self, dst_url: PathLike, overwrite: bool = True) -> None:
         '''
         Move file/directory path from src_url to dst_url
 
         :param dst_url: Given destination path
+        :param overwrite: whether or not overwrite file when exists
         '''
         for src_file_path, dst_file_path in _s3_scan_pairs(
                 self.path_with_protocol, dst_url):
-            S3Path(src_file_path).rename(dst_file_path)
+            S3Path(src_file_path).rename(dst_file_path, overwrite)
 
     def remove(self, missing_ok: bool = False) -> None:
         '''
@@ -1774,13 +1783,18 @@ class S3Path(URIPath):
                 raise S3UnknownError(
                     Exception(error_msg), self.path_with_protocol)
 
-    def rename(self, dst_path: PathLike) -> 'S3Path':
+    def rename(self, dst_path: PathLike, overwrite: bool = True) -> 'S3Path':
         '''
         Move s3 file path from src_url to dst_url
 
         :param dst_path: Given destination path
+        :param overwrite: whether or not overwrite file when exists
         '''
-        s3_rename(self.path_with_protocol, dst_path)
+        if self.is_file():
+            self.copy(dst_path, overwrite=overwrite)
+        else:
+            self.sync(dst_path, overwrite=overwrite)
+        self.remove(missing_ok=True)
         return self.from_path(dst_path)
 
     def scan(self, missing_ok: bool = True,
@@ -2111,15 +2125,21 @@ class S3Path(URIPath):
     def copy(
             self,
             dst_url: PathLike,
+            callback: Optional[Callable[[int], None]] = None,
             followlinks: bool = False,
-            callback: Optional[Callable[[int], None]] = None) -> None:
+            overwrite: bool = True) -> None:
         ''' File copy on S3
         Copy content of file on `src_path` to `dst_path`.
         It's caller's responsebility to ensure the s3_isfile(src_url) == True
 
         :param dst_path: Target file path
         :param callback: Called periodically during copy, and the input parameter is the data size (in bytes) of copy since the last call
+        :param followlinks: False if regard symlink as file, else True
+        :param overwrite: whether or not overwrite file when exists, default is True
         '''
+        if not overwrite and self.from_path(dst_url).is_file():
+            return
+
         src_url = self.path_with_protocol
         src_bucket, src_key = parse_s3_url(src_url)
         dst_bucket, dst_key = parse_s3_url(dst_url)
@@ -2156,22 +2176,32 @@ class S3Path(URIPath):
                 Callback=callback)
 
     def sync(
-            self, dst_url: PathLike, followlinks: bool = False,
-            force: bool = False) -> None:
+            self,
+            dst_url: PathLike,
+            followlinks: bool = False,
+            force: bool = False,
+            overwrite: bool = True) -> None:
         '''
         Copy file/directory on src_url to dst_url
 
         :param dst_url: Given destination path
         :param followlinks: False if regard symlink as file, else True
-        :param force: Sync file forcely, do not ignore same files
+        :param force: Sync file forcely, do not ignore same files, priority is higher than 'overwrite', default is False
+        :param overwrite: whether or not overwrite file when exists, default is True
         '''
         for src_file_path, dst_file_path in _s3_scan_pairs(
                 self.path_with_protocol, dst_url):
             src_file_path = self.from_path(src_file_path)
             dst_file_path = self.from_path(dst_file_path)
-            if not force and dst_file_path.exists() and is_same_file(
+
+            if force:
+                pass
+            elif not overwrite and dst_file_path.exists():
+                continue
+            elif dst_file_path.exists() and is_same_file(
                     src_file_path.stat(), dst_file_path.stat(), 'copy'):
                 continue
+
             src_file_path.copy(dst_file_path, followlinks=followlinks)
 
     def symlink(self, dst_path: PathLike) -> None:
@@ -2181,7 +2211,7 @@ class S3Path(URIPath):
         :param dst_path: Desination path
         :raises: S3NameTooLongError, S3BucketNotFoundError, S3IsADirectoryError
         '''
-        if len(str(self._s3_path).encode()) > 1024:
+        if len(fspath(self._s3_path).encode()) > 1024:
             raise S3NameTooLongError('File name too long: %r' % dst_path)
         src_bucket, src_key = parse_s3_url(self.path_with_protocol)
         dst_bucket, dst_key = parse_s3_url(dst_path)

megfile/sftp.py

@@ -162,24 +162,30 @@ def sftp_realpath(path: PathLike) -> str:
     return SftpPath(path).realpath()
 
 
-def sftp_rename(src_path: PathLike, dst_path: PathLike) -> 'SftpPath':
+def sftp_rename(
+        src_path: PathLike, dst_path: PathLike,
+        overwrite: bool = True) -> 'SftpPath':
     '''
     rename file on sftp
 
     :param src_path: Given path
     :param dst_path: Given destination path
+    :param overwrite: whether or not overwrite file when exists
     '''
-    return SftpPath(src_path).rename(dst_path)
+    return SftpPath(src_path).rename(dst_path, overwrite)
 
 
-def sftp_move(src_path: PathLike, dst_path: PathLike) -> 'SftpPath':
+def sftp_move(
+        src_path: PathLike, dst_path: PathLike,
+        overwrite: bool = True) -> 'SftpPath':
     '''
     move file on sftp
 
     :param src_path: Given path
     :param dst_path: Given destination path
+    :param overwrite: whether or not overwrite file when exists
     '''
-    return SftpPath(src_path).replace(dst_path)
+    return SftpPath(src_path).replace(dst_path, overwrite)
 
 
 def sftp_remove(path: PathLike, missing_ok: bool = False) -> None:
@@ -369,7 +375,8 @@ def sftp_copy(
         src_path: PathLike,
         dst_path: PathLike,
         callback: Optional[Callable[[int], None]] = None,
-        followlinks: bool = False):
+        followlinks: bool = False,
+        overwrite: bool = True):
     """
     Copy the file to the given destination path.
 
@@ -381,19 +388,21 @@ def sftp_copy(
     :raises IsADirectoryError: If the source is a directory.
     :raises OSError: If there is an error copying the file.
     """
-    return SftpPath(src_path).copy(dst_path, callback, followlinks)
+    return SftpPath(src_path).copy(dst_path, callback, followlinks, overwrite)
 
 
 def sftp_sync(
         src_path: PathLike,
         dst_path: PathLike,
         followlinks: bool = False,
-        force: bool = False):
+        force: bool = False,
+        overwrite: bool = True):
     '''Copy file/directory on src_url to dst_url
 
     :param src_path: Given path
     :param dst_url: Given destination path
     :param followlinks: False if regard symlink as file, else True
-    :param force: Sync file forcely, do not ignore same files
+    :param force: Sync file forcely, do not ignore same files, priority is higher than 'overwrite', default is False
+    :param overwrite: whether or not overwrite file when exists, default is True
     '''
-    return SftpPath(src_path).sync(dst_path, followlinks, force)
+    return SftpPath(src_path).sync(dst_path, followlinks, force, overwrite)

megfile/sftp_path.py

@@ -417,9 +417,15 @@ def sftp_download(
         src_url: PathLike,
         dst_url: PathLike,
         callback: Optional[Callable[[int], None]] = None,
-        followlinks: bool = False):
+        followlinks: bool = False,
+        overwrite: bool = True):
     '''
-    File download
+    Downloads a file from sftp to local filesystem.
+    :param src_url: source sftp path
+    :param dst_url: target fs path
+    :param callback: Called periodically during copy, and the input parameter is the data size (in bytes) of copy since the last call
+    :param followlinks: False if regard symlink as file, else True
+    :param overwrite: whether or not overwrite file when exists, default is True
     '''
     from megfile.fs import is_fs
     from megfile.fs_path import FSPath
@@ -429,6 +435,10 @@ def sftp_download(
     if not is_sftp(src_url) and not isinstance(src_url, SftpPath):
         raise OSError(f'src_url is not sftp path: {src_url}')
 
+    dst_path = FSPath(dst_url)
+    if not overwrite and dst_path.exists():
+        return
+
     if isinstance(src_url, SftpPath):
         src_path = src_url
     else:
@@ -441,7 +451,6 @@ def sftp_download(
     if str(dst_url).endswith('/'):
         raise IsADirectoryError('Is a directory: %r' % dst_url)
 
-    dst_path = FSPath(dst_url)
     dst_path.parent.makedirs(exist_ok=True)
 
     sftp_callback = None
@@ -467,9 +476,14 @@ def sftp_upload(
         src_url: PathLike,
         dst_url: PathLike,
         callback: Optional[Callable[[int], None]] = None,
-        followlinks: bool = False):
+        followlinks: bool = False,
+        overwrite: bool = True):
     '''
-    File upload
+    Uploads a file from local filesystem to sftp server.
+    :param src_url: source fs path
+    :param dst_url: target sftp path
+    :param callback: Called periodically during copy, and the input parameter is the data size (in bytes) of copy since the last call
+    :param overwrite: whether or not overwrite file when exists, default is True
     '''
     from megfile.fs import is_fs
     from megfile.fs_path import FSPath
@@ -491,6 +505,9 @@ def sftp_upload(
         dst_path = dst_url
     else:
         dst_path = SftpPath(dst_url)
+    if not overwrite and dst_path.exists():
+        return
+
     dst_path.parent.makedirs(exist_ok=True)
 
     sftp_callback = None
@@ -859,53 +876,57 @@ class SftpPath(URIPath):
     def _is_same_protocol(self, path):
         return is_sftp(path)
 
-    def rename(self, dst_path: PathLike) -> 'SftpPath':
+    def rename(self, dst_path: PathLike, overwrite: bool = True) -> 'SftpPath':
         '''
         rename file on sftp
 
         :param dst_path: Given destination path
+        :param overwrite: whether or not overwrite file when exists
         '''
         if not self._is_same_protocol(dst_path):
             raise OSError('Not a %s path: %r' % (self.protocol, dst_path))
-        if str(dst_path).endswith('/'):
-            raise IsADirectoryError('Is a directory: %r' % dst_path)
 
-        dst_path = self.from_path(dst_path)
+        dst_path = self.from_path(str(dst_path).rstrip('/'))
+
         src_stat = self.stat()
 
         if self._is_same_backend(dst_path):
-            try:
+            if overwrite:
+                dst_path.remove(missing_ok=True)
                 self._client.rename(self._real_path, dst_path._real_path)
-            except OSError:
-                if dst_path.exists():
-                    raise FileExistsError('File exists: %s' % dst_path)
+            else:
+                self.sync(dst_path, overwrite=overwrite)
+                self.remove(missing_ok=True)
         else:
             if self.is_dir():
                 for file_entry in self.scandir():
                     self.from_path(file_entry.path).rename(
                         dst_path.joinpath(file_entry.name))
+                self._client.rmdir(self._real_path)
             else:
-                with self.open('rb') as fsrc:
-                    with dst_path.open('wb') as fdst:
-                        length = 16 * 1024
-                        while True:
-                            buf = fsrc.read(length)
-                            if not buf:
-                                break
-                            fdst.write(buf)
+                if overwrite or not dst_path.exists():
+                    with self.open('rb') as fsrc:
+                        with dst_path.open('wb') as fdst:
+                            length = 16 * 1024
+                            while True:
+                                buf = fsrc.read(length)
+                                if not buf:
+                                    break
+                                fdst.write(buf)
                 self.unlink()
 
         dst_path.utime(src_stat.st_atime, src_stat.st_mtime)
         dst_path.chmod(src_stat.st_mode)
         return dst_path
 
-    def replace(self, dst_path: PathLike) -> 'SftpPath':
+    def replace(self, dst_path: PathLike, overwrite: bool = True) -> 'SftpPath':
         '''
         move file on sftp
 
         :param dst_path: Given destination path
+        :param overwrite: whether or not overwrite file when exists
         '''
-        return self.rename(dst_path=dst_path)
+        return self.rename(dst_path=dst_path, overwrite=overwrite)
 
     def remove(self, missing_ok: bool = False) -> None:
         '''
@@ -1235,7 +1256,8 @@ class SftpPath(URIPath):
             self,
             dst_path: PathLike,
             callback: Optional[Callable[[int], None]] = None,
-            followlinks: bool = False):
+            followlinks: bool = False,
+            overwrite: bool = True):
         """
         Copy the file to the given destination path.
 
@@ -1258,6 +1280,9 @@ class SftpPath(URIPath):
             raise IsADirectoryError(
                 'Is a directory: %r' % self.path_with_protocol)
 
+        if not overwrite and self.from_path(dst_path).exists():
+            return
+
         self.from_path(os.path.dirname(dst_path)).makedirs(exist_ok=True)
         dst_path = self.from_path(dst_path)
         if self._is_same_backend(dst_path):
@@ -1292,12 +1317,14 @@ class SftpPath(URIPath):
             self,
             dst_path: PathLike,
             followlinks: bool = False,
-            force: bool = False):
+            force: bool = False,
+            overwrite: bool = True):
         '''Copy file/directory on src_url to dst_url
 
         :param dst_url: Given destination path
         :param followlinks: False if regard symlink as file, else True
-        :param force: Sync file forcely, do not ignore same files
+        :param force: Sync file forcely, do not ignore same files, priority is higher than 'overwrite', default is False
+        :param overwrite: whether or not overwrite file when exists, default is True
         '''
         if not self._is_same_protocol(dst_path):
             raise OSError('Not a %s path: %r' % (self.protocol, dst_path))
@@ -1306,11 +1333,15 @@ class SftpPath(URIPath):
                 self.path_with_protocol, dst_path):
             dst_path = self.from_path(dst_file_path)
             src_path = self.from_path(src_file_path)
-            if not force and dst_path.exists() and is_same_file(
-                    src_path.stat(), dst_path.stat(), 'copy'):
+
+            if force:
+                pass
+            elif not overwrite and dst_path.exists():
                 continue
-            self.from_path(os.path.dirname(dst_file_path)).mkdir(
-                parents=True, exist_ok=True)
+            elif dst_path.exists() and is_same_file(src_path.stat(),
+                                                    dst_path.stat(), 'copy'):
+                continue
+
             self.from_path(src_file_path).copy(
                 dst_file_path, followlinks=followlinks)