megfile 2.2.8__py3-none-any.whl → 2.2.8.post1__py3-none-any.whl

megfile/cli.py

@@ -13,7 +13,7 @@ from tqdm import tqdm
 from megfile.hdfs_path import DEFAULT_HDFS_TIMEOUT
 from megfile.interfaces import FileEntry
 from megfile.lib.glob import get_non_glob_dir, has_magic
-from megfile.smart import _smart_sync_single_file, smart_copy, smart_getmd5, smart_getmtime, smart_getsize, smart_glob_stat, smart_isdir, smart_isfile, smart_makedirs, smart_move, smart_open, smart_path_join, smart_remove, smart_rename, smart_scan_stat, smart_scandir, smart_stat, smart_sync, smart_sync_with_progress, smart_touch, smart_unlink
+from megfile.smart import _smart_sync_single_file, smart_copy, smart_exists, smart_getmd5, smart_getmtime, smart_getsize, smart_glob_stat, smart_isdir, smart_isfile, smart_makedirs, smart_move, smart_open, smart_path_join, smart_remove, smart_rename, smart_scan_stat, smart_scandir, smart_stat, smart_sync, smart_sync_with_progress, smart_touch, smart_unlink
 from megfile.smart_path import SmartPath
 from megfile.utils import get_human_size
 from megfile.version import VERSION
@@ -288,7 +288,7 @@ def sync(
             src_root_path = get_non_glob_dir(src_path)
 
             def scan_func(path):
-                for glob_file_entry in smart_glob_stat(path):
+                for glob_file_entry in smart_glob_stat(path, missing_ok=False):
                     if glob_file_entry.is_file():
                         yield glob_file_entry
                     else:
@@ -297,7 +297,8 @@ def sync(
                             yield file_entry
         else:
             src_root_path = src_path
-            scan_func = partial(smart_scan_stat, followlinks=True)
+            scan_func = partial(
+                smart_scan_stat, followlinks=True, missing_ok=False)
 
         if progress_bar and not quiet:
             print('building progress bar', end='\r')

megfile/http_path.py

@@ -1,8 +1,9 @@
+import io
 import time
 from functools import partial
 from io import BufferedReader
 from logging import getLogger as get_logger
-from typing import Iterable, Optional, Union
+from typing import Iterable, Iterator, Optional, Union
 
 import requests
 
@@ -31,7 +32,8 @@ max_retries = 10
 
 def get_http_session(
         timeout: int = 10,
-        status_forcelist: Iterable[int] = (502, 503, 504)) -> requests.Session:
+        status_forcelist: Iterable[int] = (500, 502, 503,
+                                           504)) -> requests.Session:
     session = requests.Session()
 
     def after_callback(response, *args, **kwargs):
@@ -44,12 +46,69 @@ def get_http_session(
             'send http request: %s %r, with parameters: %s', method, url,
             kwargs)
 
+    def retry_callback(
+            error,
+            method,
+            url,
+            params=None,
+            data=None,
+            headers=None,
+            cookies=None,
+            files=None,
+            auth=None,
+            timeout=None,
+            allow_redirects=True,
+            proxies=None,
+            hooks=None,
+            stream=None,
+            verify=None,
+            cert=None,
+            json=None,
+            **kwargs,
+    ):
+        if data and hasattr(data, 'seek'):
+            data.seek(0)
+        elif isinstance(data, Iterator):
+            _logger.warning(f'Can not retry http request with iterator data')
+            raise
+        if files:
+
+            def seek_or_reopen(file_object):
+                if isinstance(file_object, (str, bytes)):
+                    return file_object
+                elif hasattr(file_object, 'seek'):
+                    file_object.seek(0)
+                    return file_object
+                elif hasattr(file_object, 'name'):
+                    with SmartPath(file_object.name).open('rb') as f:
+                        return io.BytesIO(f.read())
+                else:
+                    _logger.warning(
+                        f'Can not retry http request, because the file object is not seekable and unsupport "name"'
+                    )
+                    raise
+
+            for key, file_info in files.items():
+                if hasattr(file_info, 'seek'):
+                    file_info.seek(0)
+                elif isinstance(file_info,
+                                (tuple, list)) and len(file_info) >= 2:
+                    file_info = list(file_info)
+                    if isinstance(file_info[1],
+                                  (tuple, list)) and len(file_info[1]) >= 2:
+                        file_info[1] = list(file_info[1])
+                        file_info[1] = seek_or_reopen(file_info[1])
+                    else:
+                        file_info[1] = seek_or_reopen(file_info[1])
+                    files[key] = file_info
+
     session.request = patch_method(
         partial(session.request, timeout=timeout),
         max_retries=max_retries,
         should_retry=http_should_retry,
         before_callback=before_callback,
         after_callback=after_callback,
+        retry_callback=retry_callback,
     )
     return session
 

megfile/sftp_path.py

@@ -214,6 +214,7 @@ def _get_ssh_client(
         fcntl.flock(fd, fcntl.LOCK_EX)
     ssh_client.connect(
         hostname=hostname,
+        port=port,
         username=username,
         password=password,
         pkey=private_key,

megfile/version.py

@@ -1 +1 @@
-VERSION = "2.2.8"
+VERSION = "2.2.8.post1"

{megfile-2.2.8.dist-info → megfile-2.2.8.post1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: megfile
-Version: 2.2.8
+Version: 2.2.8.post1
 Summary: Megvii file operation library
 Home-page: https://github.com/megvii-research/megfile
 Author: megvii

{megfile-2.2.8.dist-info → megfile-2.2.8.post1.dist-info}/RECORD

@@ -1,23 +1,23 @@
 megfile/__init__.py,sha256=MT8SIXsmEUvtSpd1GHv6e3fFfR1gRnlEdkNNqv3gngo,6534
-megfile/cli.py,sha256=EmUv8Km3gKnT9poLUZ2f9QwHjR7miwkrZTmL2DN721Y,19935
+megfile/cli.py,sha256=lzIMM13Kqd06H7tudM6Yw2OKxXnLC369cv15ggVtMyc,20002
 megfile/errors.py,sha256=ATLYeKgfppA3cb1lNwzWf0AcKAJksr6KXPglpJa2j2w,13198
 megfile/fs.py,sha256=OfY0z4GSl8fT3mDGdeqP2hWFsd1QJl-h8RkSbg6-M8I,11547
 megfile/fs_path.py,sha256=CyOd_hOFGaYeq8CLSqF3B3ns52GMuugy9b5PD-yK0I4,38789
 megfile/hdfs.py,sha256=aAkHobOO0nDcLoqj0tx_1tvgoLOCooTWuukq0pO-nQA,9156
 megfile/hdfs_path.py,sha256=tgB2-lOqscnLRttyhm6gaD3WE-txYV6bqTH0kHeW2T8,26644
 megfile/http.py,sha256=a3oAuARSSaIU8VMx86Mui0N5Vh-EI0AoHnwxRU5DSMU,2032
-megfile/http_path.py,sha256=9X9klm6CtEfu4V0_LUWtSXC8JRYiNjVAJjdzEBfxUa8,9272
+megfile/http_path.py,sha256=pbMPE5bK7rjoFXBHVVD-O9BtPnFwsK1boHBJg6YERcQ,11420
 megfile/interfaces.py,sha256=h3tWE8hVt5S-HopaMAX6lunPJ97vzhv6jH_2HubcDNc,6219
 megfile/pathlike.py,sha256=Ere6tMf2nsI7bDsZo0WBzl_2HRrS_4iKOpYp0zZltAU,29487
 megfile/s3.py,sha256=siBZfveWX1TDA4Mp41UvugcG3zlrhl_iPUbixUp1TmI,12352
 megfile/s3_path.py,sha256=-u4g6uQ2JuMuzgvjw4KMmdF3jXAGkRy2kRRgAGcEgQ8,88778
 megfile/sftp.py,sha256=JCkF2v1ZbHuIy_Bg3l85AesjFDimDzx9Gh1gRoMsahc,12524
-megfile/sftp_path.py,sha256=KkyIbP6haFdU_HZJw7EqwnyQpxBnI6ZZkWW15MEMg0s,50557
+megfile/sftp_path.py,sha256=gz0tW3FUSY44kqoEygBMfq-WJR4vxaOuu850j-zZbtI,50576
 megfile/smart.py,sha256=JH5zed90eQchS8GNQ7mbXvif-pNKjPjvnadMazKfQMs,33278
 megfile/smart_path.py,sha256=Y0UFh4J2ccydRY2W-wX2ubaf9zzJx1M2nf-VLBGe4mk,6749
 megfile/stdio.py,sha256=yRhlfUA2DHi3bq-9cXsSlbLCnHvS_zvglO2IYYyPsGc,707
 megfile/stdio_path.py,sha256=eQulTXUwHvUKA-5PKCGfVNiEPkJhG9YtVhtU58OcmoM,2873
-megfile/version.py,sha256=BA6fabqDK7PExn5DEk-AIhmk2JFAkU5EstpThqstkrA,19
+megfile/version.py,sha256=9139fPUG0s02fyUGMvGdaft44dmi26CtzBhSkOtlTw0,25
 megfile/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 megfile/lib/base_prefetch_reader.py,sha256=SjrBffHVgvJnYtr8HNqiOozP9OJRYS37Eu1KQcZu1Z8,13221
 megfile/lib/combine_reader.py,sha256=XFSqEY5A5X5Uf7eQ6AXAzrvNteESSXvKNVPktGjo3KY,4546
@@ -42,10 +42,10 @@ megfile/lib/stdio_handler.py,sha256=QDWtcZxz-hzi-rqQUiSlR3NrihX1fjK_Rj9T2mdTFEg,
 megfile/lib/url.py,sha256=VbQLjo0s4AaV0iSk66BcjI68aUTcN9zBZ5x6-cM4Qvs,103
 megfile/utils/__init__.py,sha256=qdX8FF_dYFKwp1BIWx3JeSGd91s7AKUDSEpDv9tORcM,9162
 megfile/utils/mutex.py,sha256=-2KH3bNovKRd9zvsXq9n3bWM7rQdoG9hO7tUPxVG_Po,2538
-megfile-2.2.8.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-megfile-2.2.8.dist-info/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
-megfile-2.2.8.dist-info/METADATA,sha256=q6UT8Ujl7S8gdtRVySIiY2PWMCdpgcM8CBOo-xKZxmA,8962
-megfile-2.2.8.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
-megfile-2.2.8.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
-megfile-2.2.8.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
-megfile-2.2.8.dist-info/RECORD,,
+megfile-2.2.8.post1.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+megfile-2.2.8.post1.dist-info/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
+megfile-2.2.8.post1.dist-info/METADATA,sha256=tEAeR70I3W2r9ZDfy5uxC5tAhG43ghUfmU84qnFggYg,8968
+megfile-2.2.8.post1.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
+megfile-2.2.8.post1.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
+megfile-2.2.8.post1.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
+megfile-2.2.8.post1.dist-info/RECORD,,