megfile 3.0.4__py3-none-any.whl → 3.0.6__py3-none-any.whl

megfile/smart.py

@@ -270,7 +270,11 @@ def _default_copy_func(
         src_path: PathLike,
         dst_path: PathLike,
         callback: Optional[Callable[[int], None]] = None,
-        followlinks: bool = False) -> None:
+        followlinks: bool = False,
+        overwrite: bool = True) -> None:
+    if not overwrite and smart_exists(dst_path):
+        return
+
     with smart_open(src_path, 'rb', followlinks=followlinks) as fsrc:
         with smart_open(dst_path, 'wb') as fdst:
             # This magic number is copied from  copyfileobj
@@ -295,7 +299,8 @@ def smart_copy(
         src_path: PathLike,
         dst_path: PathLike,
         callback: Optional[Callable[[int], None]] = None,
-        followlinks: bool = False) -> None:
+        followlinks: bool = False,
+        overwrite: bool = True) -> None:
     '''
     Copy file from source path to destination path
 
@@ -319,6 +324,7 @@ def smart_copy(
     :param dst_path: Given destination path
     :param callback: Called periodically during copy, and the input parameter is the data size (in bytes) of copy since the last call
     :param followlinks: False if regard symlink as file, else True
+    :param overwrite: whether or not overwrite file when exists, default is True
     '''
     # this function contains plenty of manual polymorphism
     if smart_islink(src_path) and is_s3(dst_path) and not followlinks:
@@ -333,13 +339,19 @@ def smart_copy(
         copy_func = _default_copy_func
     try:
         copy_func(
-            src_path, dst_path, callback=callback,
-            followlinks=followlinks)  # type: ignore
+            src_path,
+            dst_path,
+            callback=callback,
+            followlinks=followlinks,
+            overwrite=overwrite)  # type: ignore
     except S3UnknownError as e:
         if 'cannot schedule new futures after interpreter shutdown' in str(e):
             _default_copy_func(
-                src_path, dst_path, callback=callback,
-                followlinks=followlinks)  # type: ignore
+                src_path,
+                dst_path,
+                callback=callback,
+                followlinks=followlinks,
+                overwrite=overwrite)  # type: ignore
         else:
             raise
 
@@ -352,6 +364,7 @@ def _smart_sync_single_file(items: dict):
     followlinks = items['followlinks']
     callback_after_copy_file = items['callback_after_copy_file']
     force = items['force']
+    overwrite = items['overwrite']
 
     content_path = os.path.relpath(src_file_path, start=src_root_path)
     if len(content_path) and content_path != '.':
@@ -365,7 +378,11 @@ def _smart_sync_single_file(items: dict):
     dst_protocol, _ = SmartPath._extract_protocol(dst_abs_file_path)
     should_sync = True
     try:
-        if not force and smart_exists(dst_abs_file_path) and is_same_file(
+        if force:
+            pass
+        elif not overwrite and smart_exists(dst_abs_file_path):
+            should_sync = False
+        elif smart_exists(dst_abs_file_path) and is_same_file(
                 smart_stat(src_file_path, follow_symlinks=followlinks),
                 smart_stat(dst_abs_file_path, follow_symlinks=followlinks),
                 get_sync_type(src_protocol, dst_protocol)):
@@ -382,6 +399,7 @@ def _smart_sync_single_file(items: dict):
             followlinks=followlinks)
     if callback_after_copy_file:
         callback_after_copy_file(src_file_path, dst_abs_file_path)
+    return should_sync
 
 
 def smart_sync(
@@ -392,9 +410,10 @@ def smart_sync(
         callback_after_copy_file: Optional[Callable[[str, str], None]] = None,
         src_file_stats: Optional[Iterable[FileEntry]] = None,
         map_func: Callable[[Callable, Iterable], Any] = map,
-        force: bool = False) -> None:
+        force: bool = False,
+        overwrite: bool = True) -> None:
     '''
-    Sync file or directory on s3 and fs
+    Sync file or directory
 
     .. note ::
 
@@ -437,6 +456,8 @@ def smart_sync(
             This parameter is in order to reduce file traversal times.
     :param map_func: A Callable func like `map`. You can use ThreadPoolExecutor.map, Pool.map and so on if you need concurrent capability.
             default is standard library `map`.
+    :param force: Sync file forcely, do not ignore same files, priority is higher than 'overwrite', default is False
+    :param overwrite: whether or not overwrite file when exists, default is True
     '''
     if not smart_exists(src_path):
         raise FileNotFoundError(f'No match file: {src_path}')
@@ -458,6 +479,7 @@ def smart_sync(
                     followlinks=followlinks,
                     callback_after_copy_file=callback_after_copy_file,
                     force=force,
+                    overwrite=overwrite,
                 )
 
     for _ in map_func(_smart_sync_single_file, create_generator()):
@@ -470,7 +492,24 @@ def smart_sync_with_progress(
         callback: Optional[Callable[[str, int], None]] = None,
         followlinks: bool = False,
         map_func: Callable[[Callable, Iterable], Iterator] = map,
-        force: bool = False):
+        force: bool = False,
+        overwrite: bool = True):
+    '''
+    Sync file or directory with progress bar
+
+    :param src_path: Given source path
+    :param dst_path: Given destination path
+    :param callback: Called periodically during copy, and the input parameter is the data size (in bytes) of copy since the last call
+    :param followlinks: False if regard symlink as file, else True
+    :param callback_after_copy_file: Called after copy success, and the input parameter is src file path and dst file path
+    :param src_file_stats: If this parameter is not None, only this parameter's files will be synced, 
+            and src_path is the root_path of these files used to calculate the path of the target file. 
+            This parameter is in order to reduce file traversal times.
+    :param map_func: A Callable func like `map`. You can use ThreadPoolExecutor.map, Pool.map and so on if you need concurrent capability.
+            default is standard library `map`.
+    :param force: Sync file forcely, do not ignore same files, priority is higher than 'overwrite', default is False
+    :param overwrite: whether or not overwrite file when exists, default is True
+    '''
     if not smart_exists(src_path):
         raise FileNotFoundError(f'No match file: {src_path}')
 
@@ -497,6 +536,7 @@ def smart_sync_with_progress(
         src_file_stats=file_stats,
         map_func=map_func,
         force=force,
+        overwrite=overwrite,
     )
     tbar.close()
     sbar.close()
@@ -513,37 +553,41 @@ def smart_remove(path: PathLike, missing_ok: bool = False) -> None:
     SmartPath(path).remove(missing_ok=missing_ok)
 
 
-def smart_rename(src_path: PathLike, dst_path: PathLike) -> None:
+def smart_rename(
+        src_path: PathLike, dst_path: PathLike, overwrite: bool = True) -> None:
     '''
     Move file on s3 or fs. `s3://` or `s3://bucket` is not allowed to move
 
     :param src_path: Given source path
     :param dst_path: Given destination path
+    :param overwrite: whether or not overwrite file when exists
     '''
     if smart_isdir(src_path):
         raise IsADirectoryError('%r is a directory' % src_path)
     src_protocol, _ = SmartPath._extract_protocol(src_path)
     dst_protocol, _ = SmartPath._extract_protocol(dst_path)
     if src_protocol == dst_protocol:
-        SmartPath(src_path).rename(dst_path)
+        SmartPath(src_path).rename(dst_path, overwrite=overwrite)
         return
-    smart_copy(src_path, dst_path)
+    smart_copy(src_path, dst_path, overwrite=overwrite)
     smart_unlink(src_path)
 
 
-def smart_move(src_path: PathLike, dst_path: PathLike) -> None:
+def smart_move(
+        src_path: PathLike, dst_path: PathLike, overwrite: bool = True) -> None:
     '''
     Move file/directory on s3 or fs. `s3://` or `s3://bucket` is not allowed to move
 
     :param src_path: Given source path
     :param dst_path: Given destination path
+    :param overwrite: whether or not overwrite file when exists
     '''
     src_protocol, _ = SmartPath._extract_protocol(src_path)
     dst_protocol, _ = SmartPath._extract_protocol(dst_path)
     if src_protocol == dst_protocol:
-        SmartPath(src_path).rename(dst_path)
+        SmartPath(src_path).rename(dst_path, overwrite=overwrite)
         return
-    smart_sync(src_path, dst_path, followlinks=True)
+    smart_sync(src_path, dst_path, followlinks=True, overwrite=overwrite)
     smart_remove(src_path)
 
 

megfile/version.py

@@ -1 +1 @@
-VERSION = "3.0.4"
+VERSION = "3.0.6"

{megfile-3.0.4.dist-info → megfile-3.0.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: megfile
-Version: 3.0.4
+Version: 3.0.6
 Summary: Megvii file operation library
 Home-page: https://github.com/megvii-research/megfile
 Author: megvii

{megfile-3.0.4.dist-info → megfile-3.0.6.dist-info}/RECORD

@@ -1,34 +1,34 @@
 megfile/__init__.py,sha256=MT8SIXsmEUvtSpd1GHv6e3fFfR1gRnlEdkNNqv3gngo,6534
-megfile/cli.py,sha256=yB2VnSaxrjNS67KL6CYRaMo72VNgyvuxCXuSfcGu6hM,21880
-megfile/config.py,sha256=Mz91CpRJJtepidOPXs98wYskmbbHRByG1VCeKJ81kVg,658
-megfile/errors.py,sha256=Sbx3UEKnzuyUmB1tFU9cZv61Yr4dRa79J6D0UMmkvj4,13323
-megfile/fs.py,sha256=OfY0z4GSl8fT3mDGdeqP2hWFsd1QJl-h8RkSbg6-M8I,11547
-megfile/fs_path.py,sha256=sHn-sBcvq7SvYN71onkA_ssLs71NzM1MM3d3Sug8uzo,38237
-megfile/hdfs.py,sha256=aAkHobOO0nDcLoqj0tx_1tvgoLOCooTWuukq0pO-nQA,9156
-megfile/hdfs_path.py,sha256=obfMMKSuBcSGgtEN18jXhEldm4hawMhxk8QoRv4k790,26859
+megfile/cli.py,sha256=vPrFVWVx1iQOzv6QE1BChix5pCpbzkiGlB2S4eBW8gM,22445
+megfile/config.py,sha256=YeLuxsFoJz6ZmZq7Pufe_021PeRWs4H6GN-v0MvsV6E,1136
+megfile/errors.py,sha256=jVkQAv3XAmAxh2l2gGc5wptNsx61uGCHwX02ugQynns,13733
+megfile/fs.py,sha256=xLVSS4VWp2xcA7-vlVPbgJBG6s0zxH3JGaINwd9x57U,11850
+megfile/fs_path.py,sha256=2GbJYB4qbRhThWedM7VgbhjaNTKCENYUfsPBFp9JGfU,40651
+megfile/hdfs.py,sha256=9kEf6b-TVy6I3B_j5jufODzjNA5ngdTqW_mEForGFas,9200
+megfile/hdfs_path.py,sha256=iQGIS81PhV7bl9xAZkV14WjCp7O9Fb9yz12RfL2Z41w,27402
 megfile/http.py,sha256=a3oAuARSSaIU8VMx86Mui0N5Vh-EI0AoHnwxRU5DSMU,2032
-megfile/http_path.py,sha256=WJd8-s_xsF71rC2QtXikH--FCsqn2u-e83YF8OarKeY,11593
+megfile/http_path.py,sha256=nCMx_8ETZynfDbNOnNkwY8vzHzCnFCx_H33PHVJQlug,16261
 megfile/interfaces.py,sha256=h3tWE8hVt5S-HopaMAX6lunPJ97vzhv6jH_2HubcDNc,6219
-megfile/pathlike.py,sha256=Ere6tMf2nsI7bDsZo0WBzl_2HRrS_4iKOpYp0zZltAU,29487
-megfile/s3.py,sha256=siBZfveWX1TDA4Mp41UvugcG3zlrhl_iPUbixUp1TmI,12352
-megfile/s3_path.py,sha256=spksQesniQOYWmfIPfJeU4kMSiFAIlB8v2yIHVRLliA,91305
-megfile/sftp.py,sha256=JCkF2v1ZbHuIy_Bg3l85AesjFDimDzx9Gh1gRoMsahc,12524
-megfile/sftp_path.py,sha256=ErPKmwgaCOvvhp3aKhqX9WKIAGbWR30QUWvptQWtag8,51666
-megfile/smart.py,sha256=y5Dzr7_f0jS2FJDF4tWbEO4SPf39zqYftqkVgMhiJds,33725
+megfile/pathlike.py,sha256=JhSP_bxRI7bwVGkubVvIkAqSYDcxdJr9h1DfZXEgc-g,29785
+megfile/s3.py,sha256=RmfUsmMVUD9-7A_O2d55dHBZ8_QX2FiFV4QAd_a1DZs,12830
+megfile/s3_path.py,sha256=S0rsYibPxcLOTOnz3nBS4OmsPcbL23570UsrN4NrJrM,92664
+megfile/sftp.py,sha256=dp0x84RYnxEX43NNIHqNZBCd69HgPqVg0EZPKT6vsCg,12978
+megfile/sftp_path.py,sha256=B_uOcKBdhZaYKh7mTieY1v-cvAheixrE3UFhA_JF6Z0,53253
+megfile/smart.py,sha256=vnm1sgxYShPSMVUl0qbVFnXu7I8NgCLHcxAXyPxXRr8,35967
 megfile/smart_path.py,sha256=Y0UFh4J2ccydRY2W-wX2ubaf9zzJx1M2nf-VLBGe4mk,6749
 megfile/stdio.py,sha256=yRhlfUA2DHi3bq-9cXsSlbLCnHvS_zvglO2IYYyPsGc,707
 megfile/stdio_path.py,sha256=eQulTXUwHvUKA-5PKCGfVNiEPkJhG9YtVhtU58OcmoM,2873
-megfile/version.py,sha256=DnboapDM00MYgi1PslOAcm9O5A6fEv0tG12NA4yH9TI,19
+megfile/version.py,sha256=aH_mbFf1JWAamOobRndDlivMY_OR5K3g0b3oshnbkyc,19
 megfile/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-megfile/lib/base_prefetch_reader.py,sha256=WKwrpuniO8a1iU2k5mIHEpuBP8caLtIeqLrpKbmqX_A,13185
+megfile/lib/base_prefetch_reader.py,sha256=rrHB9FdVX_MKOJD2lGZ00LruTjRY9u0Hj4UdI-CxNO0,13231
 megfile/lib/combine_reader.py,sha256=XFSqEY5A5X5Uf7eQ6AXAzrvNteESSXvKNVPktGjo3KY,4546
 megfile/lib/compare.py,sha256=yG2fZve_gMg32rQVCdwixBdqgYRsjn-24TqhALQaOrA,2233
 megfile/lib/compat.py,sha256=0wt3_atcYhSLCxUj_WuDlQa3E1atjZfwJQ12thiFh5Q,234
 megfile/lib/fnmatch.py,sha256=HgdlnEWBsdFUOZqnW_v1kj1jeH_9lMcCqW85pyMu4vM,4054
 megfile/lib/glob.py,sha256=7i9dIput9rI9JIPyTZX-JDmFS7IP_THlX1k-35foAfw,9732
-megfile/lib/hdfs_prefetch_reader.py,sha256=peLUau0xas33IWOoNTOJECud_r3oJYEHBXblUALO1lY,2040
+megfile/lib/hdfs_prefetch_reader.py,sha256=50oVBNmEB8KRNwHnyDzxpxvwNABWxbOaIjyiTkmcemU,2080
 megfile/lib/hdfs_tools.py,sha256=t4GeoBxO0HPahIQDrsK17WBsLZtcfAaNwWfappzZ5q8,442
-megfile/lib/http_prefetch_reader.py,sha256=Tc1SkZ541LocxG5e2EjbM9GvdW2-TGUd99nSduI1GhY,3352
+megfile/lib/http_prefetch_reader.py,sha256=1F94fFY7ZtyP7Bm9Wphu0iLtvOt771rehhouflqVojk,4328
 megfile/lib/joinpath.py,sha256=D4Px6-lnDDpYs1LMUHkTIGqMPJQ0oCBGfTzREs373iU,929
 megfile/lib/lazy_handler.py,sha256=f1rip2_T57vVo0WRNXve2bAa4LArvVheMfQg1S0vFzg,1915
 megfile/lib/s3_buffered_writer.py,sha256=ZvCtaJsGZdFy-8K6EMHFakzu3oIKXnKof-a6nBa6AY4,7017
@@ -36,17 +36,17 @@ megfile/lib/s3_cached_handler.py,sha256=xuWiThi6pJtGL_ErSBmcu8rDv1XyXNmEhiFBnRF4
 megfile/lib/s3_limited_seekable_writer.py,sha256=14gySEdbS349hpSdUcX5uvRr93HACv7BRZbgIki4b34,6150
 megfile/lib/s3_memory_handler.py,sha256=6Tj89xzc8z-FycVggGpjF_8lEbPsqRVB6undZwWsugo,3971
 megfile/lib/s3_pipe_handler.py,sha256=hG8sEajO9dv9bLTeXsERxDioHHhzi4t8NC61lSbYk94,3557
-megfile/lib/s3_prefetch_reader.py,sha256=ol_gS0J3Yr4lMBkmok3JNuhzFfTthVzmZy5z94NFGeM,4249
-megfile/lib/s3_share_cache_reader.py,sha256=ecx-62wGqgTn5vHIpKhNACCHdYWm6oc1_xwy3vkV4m0,3564
+megfile/lib/s3_prefetch_reader.py,sha256=iLvPaJGlJHaZ6jcyNPf-8A8URJgoJ0RN0C1YD--6wZ4,4285
+megfile/lib/s3_share_cache_reader.py,sha256=jyx6f3LAR05qtf-qq_RTu8wE6wqTrqg4qkwWRSdPGQ4,3600
 megfile/lib/shadow_handler.py,sha256=IbFyTw107t-yWH0cGrDjAJX-CS3xeEr77_PTGsnSgk4,2683
 megfile/lib/stdio_handler.py,sha256=QDWtcZxz-hzi-rqQUiSlR3NrihX1fjK_Rj9T2mdTFEg,2044
 megfile/lib/url.py,sha256=VbQLjo0s4AaV0iSk66BcjI68aUTcN9zBZ5x6-cM4Qvs,103
 megfile/utils/__init__.py,sha256=xrBIJcVJTb_yR68ekAyd9u4HQ46s3xgIjUZoH_Lx7hU,9531
 megfile/utils/mutex.py,sha256=-2KH3bNovKRd9zvsXq9n3bWM7rQdoG9hO7tUPxVG_Po,2538
-megfile-3.0.4.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-megfile-3.0.4.dist-info/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
-megfile-3.0.4.dist-info/METADATA,sha256=ULNUWUSidVxRQfUIJquDdSrV87Os1OQByXqRhO-7TPw,8963
-megfile-3.0.4.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-megfile-3.0.4.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
-megfile-3.0.4.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
-megfile-3.0.4.dist-info/RECORD,,
+megfile-3.0.6.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+megfile-3.0.6.dist-info/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
+megfile-3.0.6.dist-info/METADATA,sha256=GyU4MB9acbaLDMV5LS7gYt9p0py0yIn3bk13g-A38xs,8963
+megfile-3.0.6.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+megfile-3.0.6.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
+megfile-3.0.6.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
+megfile-3.0.6.dist-info/RECORD,,