megfile 3.0.1__py3-none-any.whl → 3.0.2.post1__py3-none-any.whl

megfile/cli.py

@@ -443,7 +443,12 @@ def head(path: str, lines: int):
     type=click.INT,
     default=10,
     help='print the last NUM lines')
-def tail(path: str, lines: int):
+@click.option(
+    '-f',
+    '--follow',
+    is_flag=True,
+    help='output appended data as the file grows')
+def tail(path: str, lines: int, follow: bool):
     line_list = []
     with smart_open(path, 'rb') as f:
         f.seek(0, os.SEEK_END)
@@ -464,8 +469,23 @@ def tail(path: str, lines: int):
                 break
             else:
                 line_list = block_lines
-    for line in line_list:
+
+    for line in line_list[:-1]:
         click.echo(line)
+    if line_list:
+        click.echo(line_list[-1], nl=False)
+
+    if follow:
+        offset = file_size
+        while True:
+            with smart_open(path, 'rb') as f:
+                f.seek(offset)
+                line = f.readline()
+                offset = f.tell()
+                if not line:
+                    time.sleep(1)
+                    continue
+                click.echo(line, nl=False)
 
 
 @cli.command(short_help='Write bytes from stdin to file.')

megfile/s3_path.py

@@ -461,7 +461,10 @@ def _s3_glob_stat_single_path(
         return False
 
     def create_generator(_s3_pathname) -> Iterator[FileEntry]:
-        if not S3Path(top_dir).exists():
+        top_dir_with_profile = top_dir
+        if profile_name:
+            top_dir_with_profile = f's3+{profile_name}://{top_dir[5:]}'
+        if not S3Path(top_dir_with_profile).exists():
             return
         if not has_magic(_s3_pathname):
             _s3_pathname_obj = S3Path(_s3_pathname)

megfile/sftp_path.py

@@ -1122,6 +1122,8 @@ class SftpPath(URIPath):
         if not self.is_symlink():
             raise OSError('Not a symlink: %s' % self.path_with_protocol)
         path = self._client.readlink(self._real_path)
+        if not path:
+            raise OSError('Not a symlink: %s' % self.path_with_protocol)
         if not path.startswith('/'):
             return self.parent.joinpath(path)
         return self._generate_path_object(path)

megfile/smart.py

@@ -7,7 +7,7 @@ from typing import IO, Any, AnyStr, BinaryIO, Callable, Iterable, Iterator, List
 
 from tqdm import tqdm
 
-from megfile.errors import SameFileError
+from megfile.errors import S3UnknownError
 from megfile.fs import fs_copy, is_fs
 from megfile.interfaces import Access, ContextIterator, FileCacher, FileEntry, NullCacher, PathLike, StatResult
 from megfile.lib.combine_reader import CombineReader
@@ -331,9 +331,17 @@ def smart_copy(
         copy_func = _copy_funcs[src_protocol][dst_protocol]
     except KeyError:
         copy_func = _default_copy_func
-    copy_func(
-        src_path, dst_path, callback=callback,
-        followlinks=followlinks)  # type: ignore
+    try:
+        copy_func(
+            src_path, dst_path, callback=callback,
+            followlinks=followlinks)  # type: ignore
+    except S3UnknownError as e:
+        if 'cannot schedule new futures after interpreter shutdown' in str(e):
+            _default_copy_func(
+                src_path, dst_path, callback=callback,
+                followlinks=followlinks)  # type: ignore
+        else:
+            raise
 
 
 def _smart_sync_single_file(items: dict):

megfile/version.py

@@ -1 +1 @@
-VERSION = "3.0.1"
+VERSION = "3.0.2.post1"

{megfile-3.0.1.dist-info → megfile-3.0.2.post1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: megfile
-Version: 3.0.1
+Version: 3.0.2.post1
 Summary: Megvii file operation library
 Home-page: https://github.com/megvii-research/megfile
 Author: megvii

{megfile-3.0.1.dist-info → megfile-3.0.2.post1.dist-info}/RECORD

@@ -1,5 +1,5 @@
 megfile/__init__.py,sha256=MT8SIXsmEUvtSpd1GHv6e3fFfR1gRnlEdkNNqv3gngo,6534
-megfile/cli.py,sha256=F-w-wmjjQOfy20b0vb7kOC8CGfUxFbHDWj0UQ43GcgY,21343
+megfile/cli.py,sha256=yB2VnSaxrjNS67KL6CYRaMo72VNgyvuxCXuSfcGu6hM,21880
 megfile/config.py,sha256=Mz91CpRJJtepidOPXs98wYskmbbHRByG1VCeKJ81kVg,658
 megfile/errors.py,sha256=Sbx3UEKnzuyUmB1tFU9cZv61Yr4dRa79J6D0UMmkvj4,13323
 megfile/fs.py,sha256=OfY0z4GSl8fT3mDGdeqP2hWFsd1QJl-h8RkSbg6-M8I,11547
@@ -11,14 +11,14 @@ megfile/http_path.py,sha256=ciqS32V0G3A7GVOU_lqVzRjZeyr9skXfsU87SPrkyS4,11396
 megfile/interfaces.py,sha256=h3tWE8hVt5S-HopaMAX6lunPJ97vzhv6jH_2HubcDNc,6219
 megfile/pathlike.py,sha256=Ere6tMf2nsI7bDsZo0WBzl_2HRrS_4iKOpYp0zZltAU,29487
 megfile/s3.py,sha256=siBZfveWX1TDA4Mp41UvugcG3zlrhl_iPUbixUp1TmI,12352
-megfile/s3_path.py,sha256=FYGjifwbhfraWZKKGRcuRAyllJwFcv_H1wjN5kYGzcw,90791
+megfile/s3_path.py,sha256=Rkl3cppYn8Ri1nTd_ILts0bHwtkhaLTMzj-q2pWxvEA,90940
 megfile/sftp.py,sha256=JCkF2v1ZbHuIy_Bg3l85AesjFDimDzx9Gh1gRoMsahc,12524
-megfile/sftp_path.py,sha256=gIVTITfhKEYKyqlzn5WuAsZ_uiT1YElivWtmj-Gfh3k,51572
-megfile/smart.py,sha256=YoRvxEaZ_C-A5gQlFEs4sGBJ6D26s0qYEBJEcny_jBg,33416
+megfile/sftp_path.py,sha256=ErPKmwgaCOvvhp3aKhqX9WKIAGbWR30QUWvptQWtag8,51666
+megfile/smart.py,sha256=y5Dzr7_f0jS2FJDF4tWbEO4SPf39zqYftqkVgMhiJds,33725
 megfile/smart_path.py,sha256=Y0UFh4J2ccydRY2W-wX2ubaf9zzJx1M2nf-VLBGe4mk,6749
 megfile/stdio.py,sha256=yRhlfUA2DHi3bq-9cXsSlbLCnHvS_zvglO2IYYyPsGc,707
 megfile/stdio_path.py,sha256=eQulTXUwHvUKA-5PKCGfVNiEPkJhG9YtVhtU58OcmoM,2873
-megfile/version.py,sha256=HLZDfxYBhUytEECoyqu7fod5oNr7PzV4p7FrLz2ifpM,19
+megfile/version.py,sha256=dhVp6aZyv8wvEr5NjELL4oQ0Jg_1yxifeqUh9OuUM_c,25
 megfile/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 megfile/lib/base_prefetch_reader.py,sha256=9HT0tcgXa95BdbyclpDEDw8WwKi6091GRQerS90-pjE,13191
 megfile/lib/combine_reader.py,sha256=XFSqEY5A5X5Uf7eQ6AXAzrvNteESSXvKNVPktGjo3KY,4546
@@ -43,10 +43,10 @@ megfile/lib/stdio_handler.py,sha256=QDWtcZxz-hzi-rqQUiSlR3NrihX1fjK_Rj9T2mdTFEg,
 megfile/lib/url.py,sha256=VbQLjo0s4AaV0iSk66BcjI68aUTcN9zBZ5x6-cM4Qvs,103
 megfile/utils/__init__.py,sha256=qdX8FF_dYFKwp1BIWx3JeSGd91s7AKUDSEpDv9tORcM,9162
 megfile/utils/mutex.py,sha256=-2KH3bNovKRd9zvsXq9n3bWM7rQdoG9hO7tUPxVG_Po,2538
-megfile-3.0.1.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-megfile-3.0.1.dist-info/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
-megfile-3.0.1.dist-info/METADATA,sha256=rST56VqiyiU3doAoX66z1SmlK14KtR6Gl3osEpU_ksY,8916
-megfile-3.0.1.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-megfile-3.0.1.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
-megfile-3.0.1.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
-megfile-3.0.1.dist-info/RECORD,,
+megfile-3.0.2.post1.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+megfile-3.0.2.post1.dist-info/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
+megfile-3.0.2.post1.dist-info/METADATA,sha256=qDIxaqhLqBlaV0b6GCn0e7OJr0p6phvlFIhPeqpm508,8922
+megfile-3.0.2.post1.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+megfile-3.0.2.post1.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
+megfile-3.0.2.post1.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
+megfile-3.0.2.post1.dist-info/RECORD,,