megfile 2.2.0__py3-none-any.whl → 2.2.0.post1__py3-none-any.whl

megfile/cli.py

@@ -319,13 +319,14 @@ def cat(path: str):
     help='print the first NUM lines')
 def head(path: str, lines: int):
     with smart_open(path, 'rb') as f:
-        f.seek(0, os.SEEK_END)
-        file_size = f.tell()
-        f.seek(0, os.SEEK_SET)
         for _ in range(lines):
-            click.echo(f.readline().strip(b'\n'))
-            if f.tell() >= file_size:
+            try:
+                content = f.readline()
+                if not content:
+                    break
+            except EOFError:
                 break
+            click.echo(content.strip(b'\n'))
 
 
 @cli.command(

megfile/errors.py

@@ -139,6 +139,9 @@ def patch_method(
                 result = func(*args, **kwargs)
                 if after_callback is not None:
                     result = after_callback(result, *args, **kwargs)
+                if retries > 1:
+                    _logger.info(
+                        f'Error already fixed by retry {retries - 1} times')
                 return result
             except Exception as error:
                 if not should_retry(error):
@@ -148,7 +151,7 @@ def patch_method(
                 if retries == max_retries:
                     raise
                 retry_interval = min(0.1 * 2**retries, 30)
-                _logger.debug(
+                _logger.info(
                     'unknown error encountered: %s, retry in %0.1f seconds after %d tries'
                     % (full_error_message(error), retry_interval, retries))
                 time.sleep(retry_interval)

megfile/lib/s3_prefetch_reader.py

@@ -361,8 +361,10 @@ class S3PrefetchReader(Readable, Seekable):
 
             range_str = 'bytes=%d-%d' % (
                 index * self._block_size, (index + 1) * self._block_size - 1)
-            return self._client.get_object(
+            response = self._client.get_object(
                 Bucket=self._bucket, Key=self._key, Range=range_str)
+            response['Body'] = BytesIO(response['Body'].read())
+            return response
 
         fetch_response = patch_method(
             fetch_response,
@@ -380,7 +382,7 @@ class S3PrefetchReader(Readable, Seekable):
                 'File changed: %r, etag before: %s, after: %s' %
                 (self.name, self._content_info, response))
 
-        return BytesIO(response['Body'].read())
+        return response['Body']
 
     def _submit_future(self, index: int):
         if index < 0 or index >= self._block_stop:

megfile/s3_path.py

@@ -864,6 +864,9 @@ def s3_download(
     :param dst_url: target fs path
     :param callback: Called periodically during copy, and the input parameter is the data size (in bytes) of copy since the last call
     '''
+    from megfile.fs import is_fs
+    from megfile.fs_path import FSPath
+
     src_url = S3Path(src_url)
     if followlinks:
         try:
@@ -884,23 +887,31 @@ def s3_download(
             'Is a directory: %r' % src_url.path_with_protocol)
 
     dst_url = fspath(dst_url)
+    if not is_fs(dst_url):
+        raise OSError(f'dst_url is not fs path: {dst_url}')
     if not dst_url or dst_url.endswith('/'):
         raise S3IsADirectoryError('Is a directory: %r' % dst_url)
 
-    dst_directory = os.path.dirname(dst_url)
+    dst_path = FSPath(dst_url)
+    dst_directory = os.path.dirname(dst_path.path_without_protocol)
     if dst_directory != '':
         os.makedirs(dst_directory, exist_ok=True)
 
     client = get_s3_client(profile_name=src_url._profile_name)
     try:
-        client.download_file(src_bucket, src_key, dst_url, Callback=callback)
+        client.download_file(
+            src_bucket,
+            src_key,
+            dst_path.path_without_protocol,
+            Callback=callback)
     except Exception as error:
         error = translate_fs_error(error, dst_url)
         error = translate_s3_error(error, src_url.path_with_protocol)
         raise error
 
     src_stat = src_url.stat()
-    os.utime(dst_url, (src_stat.st_mtime, src_stat.st_mtime))
+    os.utime(
+        dst_path.path_without_protocol, (src_stat.st_mtime, src_stat.st_mtime))
 
 
 def s3_upload(
@@ -914,6 +925,13 @@ def s3_upload(
     :param dst_url: target s3 path
     :param callback: Called periodically during copy, and the input parameter is the data size (in bytes) of copy since the last call
     '''
+    from megfile.fs import is_fs
+    from megfile.fs_path import FSPath
+
+    if not is_fs(src_url):
+        raise OSError(f'src_url is not fs path: {src_url}')
+    src_path = FSPath(src_url)
+
     dst_bucket, dst_key = parse_s3_url(dst_url)
     if not dst_bucket:
         raise S3BucketNotFoundError('Empty bucket name: %r' % dst_url)
@@ -921,7 +939,7 @@ def s3_upload(
         raise S3IsADirectoryError('Is a directory: %r' % dst_url)
 
     client = get_s3_client(profile_name=S3Path(dst_url)._profile_name)
-    with open(src_url, 'rb') as src:
+    with open(src_path.path_without_protocol, 'rb') as src:
         with raise_s3_error(dst_url):
             client.upload_fileobj(
                 src, Bucket=dst_bucket, Key=dst_key, Callback=callback)

megfile/sftp_path.py

@@ -367,21 +367,24 @@ def sftp_download(
     '''
     File download
     '''
-    from megfile.fs_path import FSPath, is_fs
+    from megfile.fs import is_fs
+    from megfile.fs_path import FSPath
+
     if not is_fs(dst_url):
         raise OSError(f'dst_url is not fs path: {dst_url}')
     if not is_sftp(src_url):
         raise OSError(f'src_url is not sftp path: {src_url}')
 
-    src_url = SftpPath(src_url)
-    if followlinks and src_url.is_symlink():
-        src_url = src_url.readlink()
-    if src_url.is_dir():
+    src_path = SftpPath(src_url)
+    if followlinks and src_path.is_symlink():
+        src_path = src_path.readlink()
+    if src_path.is_dir():
         raise IsADirectoryError('Is a directory: %r' % src_url)
     if str(dst_url).endswith('/'):
         raise IsADirectoryError('Is a directory: %r' % dst_url)
 
-    os.makedirs(os.path.dirname(dst_url), exist_ok=True)
+    dst_path = FSPath(dst_url)
+    dst_path.parent.makedirs(exist_ok=True)
 
     sftp_callback = None
     if callback:
@@ -389,10 +392,12 @@ def sftp_download(
         def sftp_callback(bytes_transferred: int, _total_bytes: int):
             callback(bytes_transferred)
 
-    src_url._client.get(src_url._real_path, dst_url, callback=sftp_callback)
+    src_path._client.get(
+        src_path._real_path,
+        dst_path.path_without_protocol,
+        callback=sftp_callback)
 
-    src_stat = src_url.stat()
-    dst_path = FSPath(dst_url)
+    src_stat = src_path.stat()
     dst_path.utime(src_stat.st_atime, src_stat.st_mtime)
     dst_path.chmod(src_stat.st_mode)
 
@@ -405,7 +410,9 @@ def sftp_upload(
     '''
     File upload
     '''
-    from megfile.fs import fs_stat, is_fs
+    from megfile.fs import is_fs
+    from megfile.fs_path import FSPath
+
     if not is_fs(src_url):
         raise OSError(f'src_url is not fs path: {src_url}')
     if not is_sftp(dst_url):
@@ -418,8 +425,9 @@ def sftp_upload(
     if str(dst_url).endswith('/'):
         raise IsADirectoryError('Is a directory: %r' % dst_url)
 
-    dst_url = SftpPath(dst_url)
-    dst_url.parent.makedirs(exist_ok=True)
+    src_path = FSPath(src_url)
+    dst_path = SftpPath(dst_url)
+    dst_path.parent.makedirs(exist_ok=True)
 
     sftp_callback = None
     if callback:
@@ -427,11 +435,14 @@ def sftp_upload(
         def sftp_callback(bytes_transferred: int, _total_bytes: int):
             callback(bytes_transferred)
 
-    dst_url._client.put(src_url, dst_url._real_path, callback=sftp_callback)
+    dst_path._client.put(
+        src_path.path_without_protocol,
+        dst_path._real_path,
+        callback=sftp_callback)
 
-    src_stat = fs_stat(src_url)
-    dst_url.utime(src_stat.st_atime, src_stat.st_mtime)
-    dst_url.chmod(src_stat.st_mode)
+    src_stat = src_path.stat()
+    dst_path.utime(src_stat.st_atime, src_stat.st_mtime)
+    dst_path.chmod(src_stat.st_mode)
 
 
 def sftp_path_join(path: PathLike, *other_paths: PathLike) -> str:

megfile/version.py

@@ -1 +1 @@
-VERSION = "2.2.0"
+VERSION = "2.2.0.post1"

{megfile-2.2.0.dist-info → megfile-2.2.0.post1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: megfile
-Version: 2.2.0
+Version: 2.2.0.post1
 Summary: Megvii file operation library
 Home-page: https://github.com/megvii-research/megfile
 Author: megvii

{megfile-2.2.0.dist-info → megfile-2.2.0.post1.dist-info}/RECORD

@@ -1,6 +1,6 @@
 megfile/__init__.py,sha256=Qsi3XNP_0XYoSol-1AGutZqo0rfBnzaiZ-HVXll4fB0,5721
-megfile/cli.py,sha256=zB6a1QjvIXqmd4nQR9YSTy3RH61pTX14E63uICjIRG8,12797
-megfile/errors.py,sha256=H5wCiVBvOtQo5cu3JLhAUCpQJsCkPNm1KNw9o8y4Las,11349
+megfile/cli.py,sha256=E3NoSZ8D-k3OCRXPDysFkjeHRunZXw4QvqjP5QNsgUg,12806
+megfile/errors.py,sha256=y0PJ4lzkm-uqgVD1EqakmN0IHBMt1CCzDbdbq84sVb4,11491
 megfile/fs.py,sha256=sjkm_LsvNCw8hj9Ee-1HSNzvg7bRHTPem8KTDDBQlCw,11621
 megfile/fs_path.py,sha256=Lz-u8XL2RMViHl_39yapAwX5XSE_ZpQsuhAvVve6rk8,38532
 megfile/http.py,sha256=Juz95GscschvmElieLIWqO2A4BrqJnrtTWTa9uzNN_4,2477
@@ -8,14 +8,14 @@ megfile/http_path.py,sha256=XtpbYHdtHZysjkz6Xdr4fe8E6hlsV4iLNoIZaBk-_P4,5360
 megfile/interfaces.py,sha256=h3tWE8hVt5S-HopaMAX6lunPJ97vzhv6jH_2HubcDNc,6219
 megfile/pathlike.py,sha256=52e6POtMfUCC3Hb9XNwBLXM3Gkz3aGP8H8AagGKlDSg,29307
 megfile/s3.py,sha256=6d9bBVkVdXIvmJLfpOgJkK4nVAsLR3lirrB6x1-P2y8,12437
-megfile/s3_path.py,sha256=2AY_Rcn0ZR2TpPSqWxBcbUuQ3JuS7XerZ-wBK4wXyMo,84754
+megfile/s3_path.py,sha256=-odQ7rg38khf1Bc8webKBZewIfKIV5I1a-qsdshLLzA,85287
 megfile/sftp.py,sha256=qzfgI-MGzNsr7adUQaiVKS7zRHGuK-Hs6o45wNIAcww,11943
-megfile/sftp_path.py,sha256=h_NeZtkKJ44hBIuhvBbLUlADm02uEkRT2T4PBU44uP4,47103
+megfile/sftp_path.py,sha256=lbestqN_FXwIJbqFeHXLkq306B2zqVILFda4peKOQwY,47288
 megfile/smart.py,sha256=xjRRujYAPzXlrYxV7ZCAPXl0biUNn4730ATKc_tzIzE,32694
 megfile/smart_path.py,sha256=Rwb1McXsshi9-F6miTRqE6j8FO2j1edjmSxZF32YZ6E,6708
 megfile/stdio.py,sha256=qmi1c7VTll6Y6ya9MCd-dSKffocX2GafA-YUncZRU1Y,559
 megfile/stdio_path.py,sha256=ULMzGOj7SBMn3c7fYVSDepT_1nEUiVetc-xRt2pR5o4,2682
-megfile/version.py,sha256=SX94SfG1HNW6IdIYW_iqO17A0qZmHZUfyXnyAzilMUs,19
+megfile/version.py,sha256=wEKo6RBa-JfaOX7tcmNU28H0l1ZzziU5CEqvjD8Y1AU,25
 megfile/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 megfile/lib/combine_reader.py,sha256=XFSqEY5A5X5Uf7eQ6AXAzrvNteESSXvKNVPktGjo3KY,4546
 megfile/lib/compare.py,sha256=yG2fZve_gMg32rQVCdwixBdqgYRsjn-24TqhALQaOrA,2233
@@ -29,17 +29,17 @@ megfile/lib/s3_cached_handler.py,sha256=e4KamTLMIPKa96yp7HGF05wDl2Yfoizz9pBrz-uA
 megfile/lib/s3_limited_seekable_writer.py,sha256=NSBh5cZCCtwZM0DTwfPkpOA_h0NXd9-6qjasOby5zxc,6037
 megfile/lib/s3_memory_handler.py,sha256=jLBA5HVuI-UBPYMYN-B8iX_VGr8bC9brAqY-KbEGrtw,3725
 megfile/lib/s3_pipe_handler.py,sha256=38x8oPrxlXVWqMgDOqScPj0Pt2w2cQFSNoTK27EtBSw,3384
-megfile/lib/s3_prefetch_reader.py,sha256=9YBnxKeOZ5RPbUw9vZ_PpYZzevfhw-GLkw08HIy4KVk,15033
+megfile/lib/s3_prefetch_reader.py,sha256=uffgYsNy7sfiZdoZdY67LYiQiPagtohl4BeQ6jSTwEM,15113
 megfile/lib/s3_share_cache_reader.py,sha256=QyZvzVpTswgiXv-E8QhV_jFdQjCBxVcgbgybXo6d1cM,3771
 megfile/lib/shadow_handler.py,sha256=IbFyTw107t-yWH0cGrDjAJX-CS3xeEr77_PTGsnSgk4,2683
 megfile/lib/stdio_handler.py,sha256=QDWtcZxz-hzi-rqQUiSlR3NrihX1fjK_Rj9T2mdTFEg,2044
 megfile/lib/url.py,sha256=VbQLjo0s4AaV0iSk66BcjI68aUTcN9zBZ5x6-cM4Qvs,103
 megfile/utils/__init__.py,sha256=DEVSwUQ1-1rkHBIuXBxgEYiepq8W25JIWYQrC1gcFrc,9005
 megfile/utils/mutex.py,sha256=-2KH3bNovKRd9zvsXq9n3bWM7rQdoG9hO7tUPxVG_Po,2538
-megfile-2.2.0.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-megfile-2.2.0.dist-info/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
-megfile-2.2.0.dist-info/METADATA,sha256=5hJWi8RWnKtRNaoubN8ZWAtJ-x62XVGsDUEjN9XVI-4,10595
-megfile-2.2.0.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
-megfile-2.2.0.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
-megfile-2.2.0.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
-megfile-2.2.0.dist-info/RECORD,,
+megfile-2.2.0.post1.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+megfile-2.2.0.post1.dist-info/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
+megfile-2.2.0.post1.dist-info/METADATA,sha256=WUowRbsO-dMPhEJYA1YQY0DNYRGc2jw6kdYRYQLnoBQ,10601
+megfile-2.2.0.post1.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
+megfile-2.2.0.post1.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
+megfile-2.2.0.post1.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
+megfile-2.2.0.post1.dist-info/RECORD,,