megfile 4.1.0.post3__py3-none-any.whl → 4.1.2__py3-none-any.whl

megfile/cli.py

@@ -1,10 +1,12 @@
 import configparser
 import os
 import shutil
+import signal
 import sys
 import time
 from concurrent.futures import ThreadPoolExecutor
 from functools import partial
+from queue import Queue
 
 import click
 from tqdm import tqdm
@@ -69,10 +71,13 @@ def cli(debug, log_level):
 
 
 def safe_cli():  # pragma: no cover
+    debug = options.get("debug", False)
+    if not debug:
+        signal.signal(signal.SIGINT, signal.SIG_DFL)
     try:
         cli()
     except Exception as e:
-        if options.get("debug", False):
+        if debug:
             raise
         else:
             click.echo(f"\n[{type(e).__name__}] {e}", err=True)
@@ -364,23 +369,25 @@ def rm(path: str, recursive: bool):
 @cli.command(short_help="Make source and dest identical, modifying destination only.")
 @click.argument("src_path")
 @click.argument("dst_path")
-@click.option("-g", "--progress-bar", is_flag=True, help="Show progress bar.")
 @click.option(
-    "-w", "--worker", type=click.INT, default=8, help="Number of concurrent workers."
+    "-f", "--force", is_flag=True, help="Copy files forcible, ignore same files."
 )
+@click.option("--skip", is_flag=True, help="Skip existed files.")
 @click.option(
-    "-f", "--force", is_flag=True, help="Copy files forcible, ignore same files."
+    "-w", "--worker", type=click.INT, default=-1, help="Number of concurrent workers."
 )
+@click.option("-g", "--progress-bar", is_flag=True, help="Show progress bar.")
+@click.option("-v", "--verbose", is_flag=True, help="Show more progress log.")
 @click.option("-q", "--quiet", is_flag=True, help="Not show any progress log.")
-@click.option("--skip", is_flag=True, help="Skip existed files.")
 def sync(
     src_path: str,
     dst_path: str,
-    progress_bar: bool,
-    worker: int,
     force: bool,
-    quiet: bool,
     skip: bool,
+    worker: int,
+    progress_bar: bool,
+    verbose: bool,
+    quiet: bool,
 ):
     _sftp_prompt_host_key(src_path)
     _sftp_prompt_host_key(dst_path)
@@ -388,7 +395,8 @@ def sync(
     if not smart_exists(dst_path):
         force = True
 
-    with ThreadPoolExecutor(max_workers=worker) as executor:
+    max_workers = worker if worker > 0 else (os.cpu_count() or 1) * 2
+    with ThreadPoolExecutor(max_workers=max_workers + 1) as executor:  # +1 for scan
         if has_magic(src_path):
             src_root_path = get_non_glob_dir(src_path)
             if not smart_exists(src_root_path):
@@ -410,42 +418,66 @@ def sync(
             src_root_path = src_path
             scan_func = partial(smart_scan_stat, followlinks=True)
 
-        if progress_bar and not quiet:
-            print("building progress bar", end="\r")
-            file_entries = []
-            total_count = total_size = 0
-            for total_count, file_entry in enumerate(scan_func(src_path), start=1):
-                if total_count > max_file_object_catch_count:
-                    file_entries = []
-                else:
-                    file_entries.append(file_entry)
-                total_size += file_entry.stat.size
-                print(f"building progress bar, find {total_count} files", end="\r")
-
-            if not file_entries:
-                file_entries = scan_func(src_path)
-        else:
-            total_count = total_size = None
-            file_entries = scan_func(src_path)
-
         if quiet:
+            progress_bar = False
+            verbose = False
+
+        if not progress_bar:
             callback = callback_after_copy_file = None
+
+            if verbose:
+
+                def callback_after_copy_file(src_file_path, dst_file_path):
+                    print(f"copy {src_file_path} to {dst_file_path} done")
+
+            file_entries = scan_func(src_path)
         else:
-            tbar = tqdm(total=total_count, ascii=True)
+            tbar = tqdm(
+                total=0,
+                ascii=True,
+                desc="Files (scaning)",
+            )
             sbar = tqdm(
-                unit="B",
+                total=0,
                 ascii=True,
+                unit="B",
                 unit_scale=True,
                 unit_divisor=1024,
-                total=total_size,
+                desc="File size (scaning)",
             )
 
-            def callback(_filename: str, length: int):
-                sbar.update(length)
-
             def callback_after_copy_file(src_file_path, dst_file_path):
+                if verbose:
+                    tqdm.write(f"copy {src_file_path} to {dst_file_path} done")
                 tbar.update(1)
 
+            def callback(src_file_path: str, length: int):
+                sbar.update(length)
+
+            file_entry_queue = Queue(maxsize=max_file_object_catch_count)
+
+            def scan_and_put_file_entry_to_queue():
+                for file_entry in scan_func(src_path):
+                    tbar.total += 1
+                    sbar.total += file_entry.stat.size
+                    tbar.refresh()
+                    sbar.refresh()
+                    file_entry_queue.put(file_entry)
+                file_entry_queue.put(None)
+                tbar.set_description_str("Files")
+                sbar.set_description_str("File size")
+
+            executor.submit(scan_and_put_file_entry_to_queue)
+
+            def get_file_entry_from_queue():
+                while True:
+                    file_entry = file_entry_queue.get()
+                    if file_entry is None:
+                        break
+                    yield file_entry
+
+            file_entries = get_file_entry_from_queue()
+
         params_iter = (
             dict(
                 src_root_path=src_root_path,
@@ -460,10 +492,10 @@ def sync(
             for file_entry in file_entries
         )
         list(executor.map(_smart_sync_single_file, params_iter))
-    if not quiet:
+
+    if progress_bar:
+        sbar.update(sbar.total - sbar.n)
         tbar.close()
-        if progress_bar:
-            sbar.update(sbar.total - sbar.n)
         sbar.close()
 
 

megfile/errors.py

@@ -110,29 +110,41 @@ if hasattr(botocore.exceptions, "ResponseStreamingError"):  # backport botocore=
     )
 s3_retry_exceptions = tuple(s3_retry_exceptions)  # pyre-ignore[9]
 
+s3_retry_error_codes = (
+    "429",  # noqa: E501 # TOS ExceedAccountQPSLimit
+    "499",  # noqa: E501 # Some cloud providers may send response with http code 499 if the connection not send data in 1 min.
+    "500",
+    "501",
+    "502",
+    "503",
+    "InternalError",
+    "ServiceUnavailable",
+    "SlowDown",
+    "ContextCanceled",
+    "Timeout",  # noqa: E501 # TOS Timeout
+    "RequestTimeout",
+    "RequestTimeTooSkewed",
+    "ExceedAccountQPSLimit",
+    "ExceedAccountRateLimit",
+    "ExceedBucketQPSLimit",
+    "ExceedBucketRateLimit",
+    "DownloadTrafficRateLimitExceeded",  # noqa: E501 # OSS RateLimitExceeded
+    "UploadTrafficRateLimitExceeded",
+    "MetaOperationQpsLimitExceeded",
+    "TotalQpsLimitExceeded",
+    "ActiveRequestLimitExceeded",
+    "CpuLimitExceeded",
+    "QpsLimitExceeded",
+)
+
 
 def s3_should_retry(error: Exception) -> bool:
     if isinstance(error, s3_retry_exceptions):  # pyre-ignore[6]
         return True
+    if isinstance(error, botocore.exceptions.SSLError):
+        return "EOF" in str(error)
     if isinstance(error, botocore.exceptions.ClientError):
-        return client_error_code(error) in (
-            "429",  # noqa: E501 # TOS ExceedAccountQPSLimit
-            "499",  # noqa: E501 # Some cloud providers may send response with http code 499 if the connection not send data in 1 min.
-            "500",
-            "501",
-            "502",
-            "503",
-            "InternalError",
-            "ServiceUnavailable",
-            "SlowDown",
-            "ContextCanceled",
-            "Timeout",  # noqa: E501 # TOS Timeout
-            "RequestTimeout",
-            "ExceedAccountQPSLimit",
-            "ExceedAccountRateLimit",
-            "ExceedBucketQPSLimit",
-            "ExceedBucketRateLimit",
-        )
+        return client_error_code(error) in s3_retry_error_codes
     return False
 
 
@@ -424,7 +436,7 @@ def raise_s3_error(s3_url: PathLike, suppress_error_callback=None):
 
 
 def s3_error_code_should_retry(error: str) -> bool:
-    if error in ["InternalError", "ServiceUnavailable", "SlowDown"]:
+    if error in s3_retry_error_codes:
         return True
     return False
 

megfile/fs.py

@@ -1,4 +1,6 @@
 import os
+from stat import S_ISDIR as stat_isdir
+from stat import S_ISLNK as stat_islnk
 from typing import BinaryIO, Callable, Iterator, List, Optional, Tuple
 
 from megfile.fs_path import (
@@ -7,7 +9,7 @@ from megfile.fs_path import (
     fs_path_join,
     is_fs,
 )
-from megfile.interfaces import Access, FileEntry, PathLike, StatResult
+from megfile.interfaces import Access, ContextIterator, FileEntry, PathLike, StatResult
 
 __all__ = [
     "is_fs",
@@ -169,7 +171,7 @@ def fs_isfile(path: PathLike, followlinks: bool = False) -> bool:
     return FSPath(path).is_file(followlinks)
 
 
-def fs_listdir(path: PathLike) -> List[str]:
+def fs_listdir(path: Optional[PathLike] = None) -> List[str]:
     """
     Get all contents of given fs path.
     The result is in ascending alphabetical order.
@@ -177,6 +179,8 @@ def fs_listdir(path: PathLike) -> List[str]:
     :param path: Given path
     :returns: All contents have in the path in ascending alphabetical order
     """
+    if path is None:
+        return sorted(os.listdir(path))
     return FSPath(path).listdir()
 
 
@@ -255,13 +259,34 @@ def fs_scan_stat(
     return FSPath(path).scan_stat(missing_ok, followlinks)
 
 
-def fs_scandir(path: PathLike) -> Iterator[FileEntry]:
+def fs_scandir(path: Optional[PathLike] = None) -> Iterator[FileEntry]:
     """
     Get all content of given file path.
 
     :param path: Given path
     :returns: An iterator contains all contents have prefix path
     """
+    if path is None:
+
+        def create_generator():
+            with os.scandir(None) as entries:
+                for entry in entries:
+                    stat = entry.stat()
+                    yield FileEntry(
+                        entry.name,
+                        entry.path,
+                        StatResult(
+                            size=stat.st_size,
+                            ctime=stat.st_ctime,
+                            mtime=stat.st_mtime,
+                            isdir=stat_isdir(stat.st_mode),
+                            islnk=stat_islnk(stat.st_mode),
+                            extra=stat,
+                        ),
+                    )
+
+        return ContextIterator(create_generator())
+
     return FSPath(path).scandir()
 
 

megfile/smart.py

@@ -1,8 +1,6 @@
 import os
 from collections import defaultdict
 from functools import partial
-from stat import S_ISDIR as stat_isdir
-from stat import S_ISLNK as stat_islnk
 from typing import (
     IO,
     Any,
@@ -18,10 +16,14 @@ from typing import (
 from tqdm import tqdm
 
 from megfile.errors import S3UnknownError
-from megfile.fs import fs_copy, is_fs
+from megfile.fs import (
+    fs_copy,
+    fs_listdir,
+    fs_scandir,
+    is_fs,
+)
 from megfile.interfaces import (
     Access,
-    ContextIterator,
     FileCacher,
     FileEntry,
     NullCacher,
@@ -170,7 +172,7 @@ def smart_listdir(path: Optional[PathLike] = None) -> List[str]:
     :raises: FileNotFoundError, NotADirectoryError
     """
     if path is None:
-        return sorted(os.listdir(path))
+        return fs_listdir()
     return SmartPath(path).listdir()
 
 
@@ -183,25 +185,7 @@ def smart_scandir(path: Optional[PathLike] = None) -> Iterator[FileEntry]:
     :raises: FileNotFoundError, NotADirectoryError
     """
     if path is None:
-
-        def create_generator():
-            with os.scandir(None) as entries:
-                for entry in entries:
-                    stat = entry.stat()
-                    yield FileEntry(
-                        entry.name,
-                        entry.path,
-                        StatResult(
-                            size=stat.st_size,
-                            ctime=stat.st_ctime,
-                            mtime=stat.st_mtime,
-                            isdir=stat_isdir(stat.st_mode),
-                            islnk=stat_islnk(stat.st_mode),
-                            extra=stat,
-                        ),
-                    )
-
-        return ContextIterator(create_generator())
+        return fs_scandir()
     return SmartPath(path).scandir()
 
 
@@ -404,7 +388,7 @@ def _smart_sync_single_file(items: dict):
     force = items["force"]
     overwrite = items["overwrite"]
 
-    content_path = os.path.relpath(src_file_path, start=src_root_path)
+    content_path = smart_relpath(src_file_path, start=src_root_path)
     if len(content_path) and content_path != ".":
         content_path = content_path.lstrip("/")
         dst_abs_file_path = smart_path_join(dst_root_path, content_path)
@@ -438,6 +422,8 @@ def _smart_sync_single_file(items: dict):
             callback=copy_callback,
             followlinks=followlinks,
         )
+    elif callback:
+        callback(src_file_path, src_file_stat.size)
     if callback_after_copy_file:
         callback_after_copy_file(src_file_path, dst_abs_file_path)
     return should_sync
@@ -580,19 +566,21 @@ def smart_sync_with_progress(
     def callback_after_copy_file(src_file_path, dst_file_path):
         tbar.update(1)
 
-    smart_sync(
-        src_path,
-        dst_path,
-        callback=tqdm_callback,
-        followlinks=followlinks,
-        callback_after_copy_file=callback_after_copy_file,
-        src_file_stats=file_stats,
-        map_func=map_func,
-        force=force,
-        overwrite=overwrite,
-    )
-    tbar.close()
-    sbar.close()
+    try:
+        smart_sync(
+            src_path,
+            dst_path,
+            callback=tqdm_callback,
+            followlinks=followlinks,
+            callback_after_copy_file=callback_after_copy_file,
+            src_file_stats=file_stats,
+            map_func=map_func,
+            force=force,
+            overwrite=overwrite,
+        )
+    finally:
+        tbar.close()
+        sbar.close()
 
 
 def smart_remove(path: PathLike, missing_ok: bool = False) -> None:

megfile/smart_path.py

@@ -210,4 +210,4 @@ class SmartPath(BasePath):
 
 
 def get_traditional_path(path: PathLike) -> str:
-    return fspath(SmartPath(path).path)
+    return fspath(SmartPath(path).pathlike.path)

megfile/version.py

@@ -1 +1 @@
-VERSION = "4.1.0.post3"
+VERSION = "4.1.2"

{megfile-4.1.0.post3.dist-info → megfile-4.1.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: megfile
-Version: 4.1.0.post3
+Version: 4.1.2
 Summary: Megvii file operation library
 Author-email: megvii <megfile@megvii.com>
 Project-URL: Homepage, https://github.com/megvii-research/megfile
@@ -31,6 +31,7 @@ Provides-Extra: hdfs
 Requires-Dist: hdfs; extra == "hdfs"
 Provides-Extra: cli
 Requires-Dist: click; extra == "cli"
+Dynamic: license-file
 
 megfile - Megvii FILE library
 ---

{megfile-4.1.0.post3.dist-info → megfile-4.1.2.dist-info}/RECORD

@@ -1,8 +1,8 @@
 megfile/__init__.py,sha256=7oEfu410CFKzDWZ9RjL5xEJ1gtkJkTfvPrL_7TWdJuY,7366
-megfile/cli.py,sha256=5VF4m7ZcfYgsDdszUYsOytUdJFvoEAkafsqK-HwpTww,24893
+megfile/cli.py,sha256=X9nWGke-VM-FCFdRoEoxVH_SNFkKQesBxhSrRv7jAsI,25884
 megfile/config.py,sha256=2MMj5QkhlDJQFZRbCQL2c9iDdeMAVctiaPszRBkg5vM,3988
-megfile/errors.py,sha256=qSIiqiJmDLb2k1qKiTLdNQzj4VNgfM9bHkSClyCp1Kg,14385
-megfile/fs.py,sha256=TJ0ifJ57IEL-gFgQhJD_WVEqmf28zC5kAvmH6c1QzbU,18240
+megfile/errors.py,sha256=-HuHCMj6AUPlXZnTWlUScZPoiw_DS6xx0NKsSuhztxw,14646
+megfile/fs.py,sha256=KMEqAE35alpcxiy6du5nPFYcaorhUM_kPJMah3q76ng,19160
 megfile/fs_path.py,sha256=Hozl9LAJ8EMuSWBSZXGj2GNmPZ1sJp9PZs-7hPrLgm8,39341
 megfile/hdfs.py,sha256=owXr4d3j1frCvlbhmhENcSBnKKDky5cJZzWLOF4ZJMo,13251
 megfile/hdfs_path.py,sha256=OmUe3vA3Qoxnqtcq0Rs3ygBvzAtqUz3fGo8iP5sWneE,26058
@@ -14,11 +14,11 @@ megfile/s3.py,sha256=abBxnI7RIyn7n7qjGszP1VruYd6Gi9I8QnUOvsHkx1Y,16325
 megfile/s3_path.py,sha256=zelXhlRVOVSWBE6HJz0vXrrcRzSuj6Cnjd9HLGwPbCM,93644
 megfile/sftp.py,sha256=uBcLQs-j6Q-q-sWAdd-pgi5Qmb_kq7boJM-0sCfcNO0,26540
 megfile/sftp_path.py,sha256=Wz4VcQ0pBUuWDGMSxPpPbutrT09mnY6jZNiAqTi5tO4,43840
-megfile/smart.py,sha256=YXjCWp000wimSFXE8oqI6dIE8y0RZJo9I046zIkh0ag,37014
-megfile/smart_path.py,sha256=Bqg95T2-XZrRXWhH7GT-jMCYzD7i1SIXdczQxtOxiPs,7583
+megfile/smart.py,sha256=Sae2KJzaU0k_qV_Bk0YifOMq8WsV5qQ2pGInDRF546I,36411
+megfile/smart_path.py,sha256=HqCOlDwekqqIyJAll-U9YKmaXjjfCGZD5n5aG80lOKw,7592
 megfile/stdio.py,sha256=ZwxsnJNJYIT7Iyg5pIw4qiyH8bszG6oAhEJuR-hXGG4,658
 megfile/stdio_path.py,sha256=cxaDr8rtisTPnN-rjtaEpqQnshwiqwXFUJBM9xWY7Cg,2711
-megfile/version.py,sha256=wNRjMrkurG2-lYRZx4EZUY9SjbBWGLHy8zGyNvzRmuA,25
+megfile/version.py,sha256=kYS0NJJrKL79xx4qm3iMYFp2WaSU6ttQqSdCIGu0WVA,19
 megfile/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 megfile/lib/base_prefetch_reader.py,sha256=uxVwYknOjc8hLF7q_T2QKMsBqFcrf411ZsuK25CN1eQ,12848
 megfile/lib/combine_reader.py,sha256=Kp2wEloOUpTlIU7dve87MBpSzmIM-F9OtpTawAjFkiU,4828
@@ -43,6 +43,8 @@ megfile/lib/stdio_handler.py,sha256=IDdgENLQlhigEwkLL4zStueVSzdWg7xVcTF_koof_Ek,
 megfile/lib/url.py,sha256=ER32pWy9Q2MAk3TraAaNEBWIqUeBmLuM57ol2cs7-Ks,103
 megfile/utils/__init__.py,sha256=xAzmICA0MtAbg-I2yPfeHjA1N4CiMP4sBrC9BgrfZLw,10151
 megfile/utils/mutex.py,sha256=asb8opGLgK22RiuBJUnfsvB8LnMmodP8KzCVHKmQBWA,2561
+megfile-4.1.2.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+megfile-4.1.2.dist-info/licenses/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
 scripts/benchmark/code/iopath_read.py,sha256=O1Qs3mpvD9S_XCuRH2A2FpGWxCOSw6qZvEBrtPeRL1E,661
 scripts/benchmark/code/iopath_write.py,sha256=Mm0efW1J09RJ_CK5i1xtG2hJuaaslikin8qVpuRFP_Q,704
 scripts/benchmark/code/megfile_read.py,sha256=sAMebUiMColHDv3JEkXplImAHvn_IF1-g3BIJxhcQYE,239
@@ -53,10 +55,8 @@ scripts/benchmark/code/s3fs_read.py,sha256=XiTA-qrYblUs-jQWXSnvNg5Wo722C_g47aMMf
 scripts/benchmark/code/s3fs_write.py,sha256=gdXKkWXYGjLJlRT_J64pJN85XvRg3bZexcAJQEMXwtw,402
 scripts/benchmark/code/smart_open_read.py,sha256=SA02jHwS9Y31yFtV9CoJcfND5dR0eA_HsGmGNUrpQls,515
 scripts/benchmark/code/smart_open_write.py,sha256=jDxFJdY97yNH889jz3pawBoei3yaqy8pEMvC_ymHFtM,537
-megfile-4.1.0.post3.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-megfile-4.1.0.post3.dist-info/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
-megfile-4.1.0.post3.dist-info/METADATA,sha256=Y1lKXJkvmVgl-O_VwFqMYtumMDM7LiJA7xR9i6sRrY8,9579
-megfile-4.1.0.post3.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-megfile-4.1.0.post3.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
-megfile-4.1.0.post3.dist-info/top_level.txt,sha256=fVg49lk5B9L7jyfWUXWxb0DDSuw5pbr0OU62Tvx8J8M,44
-megfile-4.1.0.post3.dist-info/RECORD,,
+megfile-4.1.2.dist-info/METADATA,sha256=7Bxhk9yqd8P_kSNqVhvqCP1yif4ScV2x63YE6vzsznQ,9595
+megfile-4.1.2.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+megfile-4.1.2.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
+megfile-4.1.2.dist-info/top_level.txt,sha256=TR6xUw8Px5Ms_UENhEmLNmxOdfHAhTlSt9yTw9LRgsQ,35
+megfile-4.1.2.dist-info/RECORD,,

{megfile-4.1.0.post3.dist-info → megfile-4.1.2.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.8.0)
+Generator: setuptools (78.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{megfile-4.1.0.post3.dist-info → megfile-4.1.2.dist-info}/top_level.txt

@@ -1,6 +1,5 @@
 dist
 docs
 html_cov
-html_doc
 megfile
 scripts