megfile 3.0.0.post1__py3-none-any.whl → 3.0.2__py3-none-any.whl

megfile/cli.py

@@ -104,8 +104,10 @@ def _ls(path: str, long: bool, recursive: bool, human_readable: bool):
         echo_func = simple_echo
 
     total_size = 0
+    total_count = 0
     for file_stat in scan_func(path):
         total_size += file_stat.stat.size
+        total_count += 1
         output = echo_func(file_stat, base_path, full_path=full_path)
         if file_stat.is_symlink():
             try:
@@ -115,7 +117,7 @@ def _ls(path: str, long: bool, recursive: bool, human_readable: bool):
             output += ' -> %s' % link
         click.echo(output)
     if long:
-        click.echo(f'total: {get_human_size(total_size)}')
+        click.echo(f'total({total_count}): {get_human_size(total_size)}')
 
 
 @cli.command(short_help='List all the objects in the path.')
@@ -441,7 +443,12 @@ def head(path: str, lines: int):
     type=click.INT,
     default=10,
     help='print the last NUM lines')
-def tail(path: str, lines: int):
+@click.option(
+    '-f',
+    '--follow',
+    is_flag=True,
+    help='output appended data as the file grows')
+def tail(path: str, lines: int, follow: bool):
     line_list = []
     with smart_open(path, 'rb') as f:
         f.seek(0, os.SEEK_END)
@@ -462,8 +469,23 @@ def tail(path: str, lines: int):
                 break
             else:
                 line_list = block_lines
-    for line in line_list:
+
+    for line in line_list[:-1]:
         click.echo(line)
+    if line_list:
+        click.echo(line_list[-1], nl=False)
+
+    if follow:
+        offset = file_size
+        while True:
+            with smart_open(path, 'rb') as f:
+                f.seek(offset)
+                line = f.readline()
+                offset = f.tell()
+                if not line:
+                    time.sleep(1)
+                    continue
+                click.echo(line, nl=False)
 
 
 @cli.command(short_help='Write bytes from stdin to file.')

megfile/sftp_path.py

@@ -105,8 +105,13 @@ def sftp_should_retry(error: Exception) -> bool:
             socket.timeout,
     )):
         return True
-    elif isinstance(error, OSError) and str(error) == 'Socket is closed':
-        return True
+    elif isinstance(error, OSError):
+        for err_msg in [
+                'Socket is closed',
+                'Cannot assign requested address',
+        ]:
+            if err_msg in str(error):
+                return True
     return False
 
 
@@ -1117,6 +1122,8 @@ class SftpPath(URIPath):
         if not self.is_symlink():
             raise OSError('Not a symlink: %s' % self.path_with_protocol)
         path = self._client.readlink(self._real_path)
+        if not path:
+            raise OSError('Not a symlink: %s' % self.path_with_protocol)
         if not path.startswith('/'):
             return self.parent.joinpath(path)
         return self._generate_path_object(path)

megfile/smart.py

@@ -7,7 +7,7 @@ from typing import IO, Any, AnyStr, BinaryIO, Callable, Iterable, Iterator, List
 
 from tqdm import tqdm
 
-from megfile.errors import SameFileError
+from megfile.errors import S3UnknownError
 from megfile.fs import fs_copy, is_fs
 from megfile.interfaces import Access, ContextIterator, FileCacher, FileEntry, NullCacher, PathLike, StatResult
 from megfile.lib.combine_reader import CombineReader
@@ -331,9 +331,17 @@ def smart_copy(
         copy_func = _copy_funcs[src_protocol][dst_protocol]
     except KeyError:
         copy_func = _default_copy_func
-    copy_func(
-        src_path, dst_path, callback=callback,
-        followlinks=followlinks)  # type: ignore
+    try:
+        copy_func(
+            src_path, dst_path, callback=callback,
+            followlinks=followlinks)  # type: ignore
+    except S3UnknownError as e:
+        if 'cannot schedule new futures after interpreter shutdown' in str(e):
+            _default_copy_func(
+                src_path, dst_path, callback=callback,
+                followlinks=followlinks)  # type: ignore
+        else:
+            raise
 
 
 def _smart_sync_single_file(items: dict):

megfile/version.py

@@ -1 +1 @@
-VERSION = "3.0.0.post1"
+VERSION = "3.0.2"

{megfile-3.0.0.post1.dist-info → megfile-3.0.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: megfile
-Version: 3.0.0.post1
+Version: 3.0.2
 Summary: Megvii file operation library
 Home-page: https://github.com/megvii-research/megfile
 Author: megvii

{megfile-3.0.0.post1.dist-info → megfile-3.0.2.dist-info}/RECORD

@@ -1,5 +1,5 @@
 megfile/__init__.py,sha256=MT8SIXsmEUvtSpd1GHv6e3fFfR1gRnlEdkNNqv3gngo,6534
-megfile/cli.py,sha256=yaECF7bn-_QjeK6MLnZgvgJjIHcUIQrrKVl_JaxOEDo,21283
+megfile/cli.py,sha256=yB2VnSaxrjNS67KL6CYRaMo72VNgyvuxCXuSfcGu6hM,21880
 megfile/config.py,sha256=Mz91CpRJJtepidOPXs98wYskmbbHRByG1VCeKJ81kVg,658
 megfile/errors.py,sha256=Sbx3UEKnzuyUmB1tFU9cZv61Yr4dRa79J6D0UMmkvj4,13323
 megfile/fs.py,sha256=OfY0z4GSl8fT3mDGdeqP2hWFsd1QJl-h8RkSbg6-M8I,11547
@@ -13,12 +13,12 @@ megfile/pathlike.py,sha256=Ere6tMf2nsI7bDsZo0WBzl_2HRrS_4iKOpYp0zZltAU,29487
 megfile/s3.py,sha256=siBZfveWX1TDA4Mp41UvugcG3zlrhl_iPUbixUp1TmI,12352
 megfile/s3_path.py,sha256=FYGjifwbhfraWZKKGRcuRAyllJwFcv_H1wjN5kYGzcw,90791
 megfile/sftp.py,sha256=JCkF2v1ZbHuIy_Bg3l85AesjFDimDzx9Gh1gRoMsahc,12524
-megfile/sftp_path.py,sha256=L26usr4jHhjxzAfNRoguxm3J8JNNaFWyrRYG2W7zCk0,51440
-megfile/smart.py,sha256=YoRvxEaZ_C-A5gQlFEs4sGBJ6D26s0qYEBJEcny_jBg,33416
+megfile/sftp_path.py,sha256=ErPKmwgaCOvvhp3aKhqX9WKIAGbWR30QUWvptQWtag8,51666
+megfile/smart.py,sha256=y5Dzr7_f0jS2FJDF4tWbEO4SPf39zqYftqkVgMhiJds,33725
 megfile/smart_path.py,sha256=Y0UFh4J2ccydRY2W-wX2ubaf9zzJx1M2nf-VLBGe4mk,6749
 megfile/stdio.py,sha256=yRhlfUA2DHi3bq-9cXsSlbLCnHvS_zvglO2IYYyPsGc,707
 megfile/stdio_path.py,sha256=eQulTXUwHvUKA-5PKCGfVNiEPkJhG9YtVhtU58OcmoM,2873
-megfile/version.py,sha256=kMj-Zcr5CPQqqnyAXYysUF5loOzPHPIw9kUQCBiOepA,25
+megfile/version.py,sha256=HGcItPfK5-rkjDD8Y1bmaADbd3qo7wyXrM39-uif4RY,19
 megfile/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 megfile/lib/base_prefetch_reader.py,sha256=9HT0tcgXa95BdbyclpDEDw8WwKi6091GRQerS90-pjE,13191
 megfile/lib/combine_reader.py,sha256=XFSqEY5A5X5Uf7eQ6AXAzrvNteESSXvKNVPktGjo3KY,4546
@@ -43,10 +43,10 @@ megfile/lib/stdio_handler.py,sha256=QDWtcZxz-hzi-rqQUiSlR3NrihX1fjK_Rj9T2mdTFEg,
 megfile/lib/url.py,sha256=VbQLjo0s4AaV0iSk66BcjI68aUTcN9zBZ5x6-cM4Qvs,103
 megfile/utils/__init__.py,sha256=qdX8FF_dYFKwp1BIWx3JeSGd91s7AKUDSEpDv9tORcM,9162
 megfile/utils/mutex.py,sha256=-2KH3bNovKRd9zvsXq9n3bWM7rQdoG9hO7tUPxVG_Po,2538
-megfile-3.0.0.post1.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-megfile-3.0.0.post1.dist-info/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
-megfile-3.0.0.post1.dist-info/METADATA,sha256=8JE1q74gWq4onYy2ws28_CiXNsNHdRgS0_bf4TQvky8,8922
-megfile-3.0.0.post1.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
-megfile-3.0.0.post1.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
-megfile-3.0.0.post1.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
-megfile-3.0.0.post1.dist-info/RECORD,,
+megfile-3.0.2.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+megfile-3.0.2.dist-info/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
+megfile-3.0.2.dist-info/METADATA,sha256=DnkmGRjSQJP0v3CgkHNwp7th5pKwJcF7qyKUN1tKaDs,8916
+megfile-3.0.2.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+megfile-3.0.2.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
+megfile-3.0.2.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
+megfile-3.0.2.dist-info/RECORD,,

{megfile-3.0.0.post1.dist-info → megfile-3.0.2.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.41.2)
+Generator: bdist_wheel (0.42.0)
 Root-Is-Purelib: true
 Tag: py3-none-any