megfile 4.1.4.post1__py3-none-any.whl → 4.1.6__py3-none-any.whl

megfile/errors.py

@@ -174,12 +174,15 @@ def patch_method(
                 if retry_callback is not None:
                     retry_callback(error, *args, **kwargs)
                 if retries == max_retries:
+                    _logger.error(
+                        f"Cannot handle error {full_error_message(error)} "
+                        f"after {retries} tries"
+                    )
                     raise
                 retry_interval = min(0.1 * 2**retries, 30)
                 _logger.info(
-                    "unknown error encountered: %s, retry in %0.1f seconds "
-                    "after %d tries"
-                    % (full_error_message(error), retry_interval, retries)
+                    f"unknown error encountered: {full_error_message(error)}, "
+                    f"retry in {retry_interval:.1f}s after {retries} tries"
                 )
                 time.sleep(retry_interval)
 
@@ -313,7 +316,7 @@ class HttpBodyIncompleteError(HttpException):
 
 http_retry_exceptions = (
     requests.exceptions.ReadTimeout,
-    requests.exceptions.ConnectTimeout,
+    requests.exceptions.ConnectionError,
     requests.exceptions.ChunkedEncodingError,
     requests.exceptions.HTTPError,
     requests.exceptions.ProxyError,

megfile/http_path.py

@@ -4,7 +4,7 @@ from functools import partial
 from io import BufferedReader, BytesIO
 from logging import getLogger as get_logger
 from threading import Lock
-from typing import Iterable, Iterator, Optional, Tuple, Union
+from typing import Dict, Iterable, Iterator, Optional, Tuple, Union
 
 import requests
 from urllib3 import HTTPResponse
@@ -20,7 +20,7 @@ from megfile.lib.compat import fspath
 from megfile.lib.http_prefetch_reader import DEFAULT_TIMEOUT, HttpPrefetchReader
 from megfile.lib.url import get_url_scheme
 from megfile.smart_path import SmartPath
-from megfile.utils import _is_pickle, binary_open
+from megfile.utils import _is_pickle, binary_open, cached_property
 
 __all__ = [
     "HttpPath",
@@ -31,12 +31,25 @@ __all__ = [
 
 _logger = get_logger(__name__)
 
+DEFAULT_REQUEST_KWARGS = {
+    "timeout": DEFAULT_TIMEOUT,
+}
+
 
 def get_http_session(
     timeout: Optional[Union[int, Tuple[int, int]]] = DEFAULT_TIMEOUT,
     status_forcelist: Iterable[int] = (500, 502, 503, 504),
+    params: Optional[Dict[str, str]] = None,
+    headers: Optional[Dict[str, str]] = None,
+    cookies: Optional[Dict[str, str]] = None,
+    trust_env: bool = True,
+    **kwargs,
 ) -> requests.Session:
     session = requests.Session()
+    session.params.update(params or {})  # type: ignore
+    session.headers.update(headers or {})
+    session.cookies.update(cookies or {})
+    session.trust_env = trust_env
 
     def after_callback(response, *args, **kwargs):
         if response.status_code in status_forcelist:
@@ -52,20 +65,8 @@ def get_http_session(
         error,
         method,
         url,
-        params=None,
         data=None,
-        headers=None,
-        cookies=None,
         files=None,
-        auth=None,
-        timeout=None,
-        allow_redirects=True,
-        proxies=None,
-        hooks=None,
-        stream=None,
-        verify=None,
-        cert=None,
-        json=None,
         **kwargs,
     ):
         if data and hasattr(data, "seek"):
@@ -100,7 +101,7 @@ def get_http_session(
                     files[key] = file_info
 
     session.request = patch_method(
-        partial(session.request, timeout=timeout),
+        partial(session.request, timeout=timeout, **kwargs),
         max_retries=HTTP_MAX_RETRY_TIMES,
         should_retry=http_should_retry,
         before_callback=before_callback,
@@ -136,7 +137,11 @@ class HttpPath(URIPath):
 
         if fspath(path).startswith("https://"):
             self.protocol = "https"
-        self.request_kwargs = {}
+        self.request_kwargs = deepcopy(DEFAULT_REQUEST_KWARGS)
+
+    @cached_property
+    def session(self):
+        return get_http_session(status_forcelist=(), **self.request_kwargs)
 
     @binary_open
     def open(
@@ -174,13 +179,8 @@ class HttpPath(URIPath):
             raise ValueError("unacceptable mode: %r" % mode)
 
         response = None
-        request_kwargs = deepcopy(self.request_kwargs)
-        timeout = request_kwargs.pop("timeout", DEFAULT_TIMEOUT)
-        stream = request_kwargs.pop("stream", True)
         try:
-            response = get_http_session(timeout=timeout, status_forcelist=()).get(
-                self.path_with_protocol, stream=stream, **request_kwargs
-            )
+            response = self.session.get(self.path_with_protocol, stream=True)
             response.raise_for_status()
         except Exception as error:
             if response:
@@ -197,6 +197,7 @@ class HttpPath(URIPath):
 
             reader = HttpPrefetchReader(
                 self,
+                session=self.session,
                 content_size=content_size,
                 block_size=block_size,
                 max_buffer_size=max_buffer_size,
@@ -225,15 +226,8 @@ class HttpPath(URIPath):
         :returns: StatResult
         :raises: HttpPermissionError, HttpFileNotFoundError
         """
-
-        request_kwargs = deepcopy(self.request_kwargs)
-        timeout = request_kwargs.pop("timeout", DEFAULT_TIMEOUT)
-        stream = request_kwargs.pop("stream", True)
-
         try:
-            with get_http_session(timeout=timeout, status_forcelist=()).get(
-                self.path_with_protocol, stream=stream, **request_kwargs
-            ) as response:
+            with self.session.get(self.path_with_protocol, stream=True) as response:
                 response.raise_for_status()
                 headers = response.headers
         except Exception as error:
@@ -254,7 +248,11 @@ class HttpPath(URIPath):
             last_modified = 0.0
 
         return StatResult(
-            size=size, mtime=last_modified, isdir=False, islnk=False, extra=headers
+            size=size,
+            mtime=last_modified,
+            isdir=False,
+            islnk=False,
+            extra=headers,
         )
 
     def getsize(self, follow_symlinks: bool = False) -> int:
@@ -281,7 +279,7 @@ class HttpPath(URIPath):
         """
         return self.stat().mtime
 
-    def exists(self, followlinks: bool = False) -> bool:
+    def is_file(self, followlinks: bool = False) -> bool:
         """Test if http path exists
 
         :param followlinks: ignore this parameter, just for compatibility
@@ -289,20 +287,16 @@ class HttpPath(URIPath):
         :return: return True if exists
         :rtype: bool
         """
-        request_kwargs = deepcopy(self.request_kwargs)
-        timeout = request_kwargs.pop("timeout", DEFAULT_TIMEOUT)
-        stream = request_kwargs.pop("stream", True)
-
         try:
-            with get_http_session(timeout=timeout, status_forcelist=()).get(
-                self.path_with_protocol, stream=stream, **request_kwargs
-            ) as response:
+            with self.session.get(self.path_with_protocol, stream=True) as response:
                 if response.status_code == 404:
                     return False
                 return True
         except requests.exceptions.ConnectionError:
             return False
 
+    exists = is_file
+
 
 @SmartPath.register
 class HttpsPath(HttpPath):

megfile/lib/http_prefetch_reader.py

@@ -38,6 +38,7 @@ class HttpPrefetchReader(BasePrefetchReader):
         self,
         url: PathLike,
         *,
+        session: Optional[requests.Session] = None,
         content_size: Optional[int] = None,
         block_size: int = READER_BLOCK_SIZE,
         max_buffer_size: int = READER_MAX_BUFFER_SIZE,
@@ -47,6 +48,7 @@ class HttpPrefetchReader(BasePrefetchReader):
     ):
         self._url = url
         self._content_size = content_size
+        self._session = session or requests.Session()
 
         super().__init__(
             block_size=block_size,
@@ -76,16 +78,8 @@ class HttpPrefetchReader(BasePrefetchReader):
         self, start: Optional[int] = None, end: Optional[int] = None
     ) -> dict:
         def fetch_response() -> dict:
-            request_kwargs = {}
-            if hasattr(self._url, "request_kwargs"):
-                request_kwargs = self._url.request_kwargs  # pyre-ignore[16]
-            timeout = request_kwargs.pop("timeout", DEFAULT_TIMEOUT)
-            stream = request_kwargs.pop("stream", True)
-
             if start is None or end is None:
-                with requests.get(
-                    fspath(self._url), timeout=timeout, stream=stream, **request_kwargs
-                ) as response:
+                with self._session.get(fspath(self._url), stream=True) as response:
                     return {
                         "Headers": response.headers,
                         "Cookies": response.cookies,
@@ -95,14 +89,9 @@ class HttpPrefetchReader(BasePrefetchReader):
                 range_end = end
                 if self._content_size is not None:
                     range_end = min(range_end, self._content_size - 1)
-                headers = request_kwargs.pop("headers", {})
-                headers["Range"] = f"bytes={start}-{range_end}"
-                with requests.get(
-                    fspath(self._url),
-                    timeout=timeout,
-                    headers=headers,
-                    stream=stream,
-                    **request_kwargs,
+                headers = {"Range": f"bytes={start}-{range_end}"}
+                with self._session.get(
+                    fspath(self._url), headers=headers, stream=True
                 ) as response:
                     if len(response.content) != int(response.headers["Content-Length"]):
                         raise HttpBodyIncompleteError(

megfile/sftp_path.py

@@ -1,5 +1,4 @@
 import atexit
-import fcntl
 import hashlib
 import io
 import os
@@ -198,6 +197,8 @@ def _get_ssh_client(
     password: Optional[str] = None,
     default_policy: Type[paramiko.MissingHostKeyPolicy] = paramiko.RejectPolicy,
 ) -> paramiko.SSHClient:
+    import fcntl
+
     hostname, port, username, password, private_key = provide_connect_info(
         hostname=hostname, port=port, username=username, password=password
     )

megfile/utils/__init__.py

@@ -216,7 +216,7 @@ def get_human_size(size_bytes: float) -> str:
     index = int(math.floor(math.log(size_bytes, 1024)))
     base = math.pow(1024, index)
     if base == 1:
-        size = size_bytes
+        size = int(size_bytes)
     else:
         size = round(size_bytes / base, 2)
     return "%s %s" % (size, size_name[index])

megfile/version.py

@@ -1 +1 @@
-VERSION = "4.1.4.post1"
+VERSION = "4.1.6"

{megfile-4.1.4.post1.dist-info → megfile-4.1.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: megfile
-Version: 4.1.4.post1
+Version: 4.1.6
 Summary: Megvii file operation library
 Author-email: megvii <megfile@megvii.com>
 Project-URL: Homepage, https://github.com/megvii-research/megfile

{megfile-4.1.4.post1.dist-info → megfile-4.1.6.dist-info}/RECORD

@@ -1,24 +1,24 @@
 megfile/__init__.py,sha256=7oEfu410CFKzDWZ9RjL5xEJ1gtkJkTfvPrL_7TWdJuY,7366
 megfile/cli.py,sha256=iwwlwVNu_yXgqnaURPHqGPrx1WeReqN0zi9C4rqK5Ag,29178
 megfile/config.py,sha256=2MMj5QkhlDJQFZRbCQL2c9iDdeMAVctiaPszRBkg5vM,3988
-megfile/errors.py,sha256=A4qX2h1rk6UOOlNy4E81RmI_zuQd5vjh_rlqTfccwag,14589
+megfile/errors.py,sha256=aw4BX-pU1uj0-dNc6Tq-IeSwK_PWJNi2ZL5ZcRSo7aQ,14739
 megfile/fs.py,sha256=KMEqAE35alpcxiy6du5nPFYcaorhUM_kPJMah3q76ng,19160
 megfile/fs_path.py,sha256=Hozl9LAJ8EMuSWBSZXGj2GNmPZ1sJp9PZs-7hPrLgm8,39341
 megfile/hdfs.py,sha256=owXr4d3j1frCvlbhmhENcSBnKKDky5cJZzWLOF4ZJMo,13251
 megfile/hdfs_path.py,sha256=OmUe3vA3Qoxnqtcq0Rs3ygBvzAtqUz3fGo8iP5sWneE,26058
 megfile/http.py,sha256=1nuGe-JbnwMFyV3s35CJxByED3uoRoS9y8Y8cSGP9Kw,3865
-megfile/http_path.py,sha256=yRIk-fNbrsY8rUS5KVOfocS_PS520dX5KOs8lImpLaY,14173
+megfile/http_path.py,sha256=ZGug-bTWq8GqEA-R6zIXQmbcOVYvILxEweTXzHGnBuk,13829
 megfile/interfaces.py,sha256=p4UvVZpeLx5djd6bqqDaygIx_s-_AxIVj-gudTch4JE,8467
 megfile/pathlike.py,sha256=3Hnw-fn6RcIe9iPrJt00QdHSA--UfDyxnVBuZ_ymYYQ,31278
 megfile/s3.py,sha256=abBxnI7RIyn7n7qjGszP1VruYd6Gi9I8QnUOvsHkx1Y,16325
 megfile/s3_path.py,sha256=S8iulho1btVLLvNU-OtcskkbaAC8yNXnBrFNnF3fwS8,93510
 megfile/sftp.py,sha256=uBcLQs-j6Q-q-sWAdd-pgi5Qmb_kq7boJM-0sCfcNO0,26540
-megfile/sftp_path.py,sha256=Wz4VcQ0pBUuWDGMSxPpPbutrT09mnY6jZNiAqTi5tO4,43840
+megfile/sftp_path.py,sha256=CgirHWmNdXdqyIL9ufmlaMpwFhlkQVZhqmfvjUaj7qU,43845
 megfile/smart.py,sha256=Sae2KJzaU0k_qV_Bk0YifOMq8WsV5qQ2pGInDRF546I,36411
 megfile/smart_path.py,sha256=Up_6xNZ2019iSzMn_JAU_1H--z-AP6O7SxdXGdeTG0c,7659
 megfile/stdio.py,sha256=ZwxsnJNJYIT7Iyg5pIw4qiyH8bszG6oAhEJuR-hXGG4,658
 megfile/stdio_path.py,sha256=cxaDr8rtisTPnN-rjtaEpqQnshwiqwXFUJBM9xWY7Cg,2711
-megfile/version.py,sha256=n7uJo8sxqI4-yPdovTHcDolsukxzF_l9KEtT9E_KR3o,25
+megfile/version.py,sha256=IhKvX1UU6Hj5y2NPRPLmOC6AwOxQcN87pBebR6XrP7I,19
 megfile/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 megfile/lib/base_prefetch_reader.py,sha256=uxVwYknOjc8hLF7q_T2QKMsBqFcrf411ZsuK25CN1eQ,12848
 megfile/lib/combine_reader.py,sha256=Kp2wEloOUpTlIU7dve87MBpSzmIM-F9OtpTawAjFkiU,4828
@@ -28,7 +28,7 @@ megfile/lib/fnmatch.py,sha256=4MvGzEahMRA-u8Z7mxaD-Yw1idOwBoJJpVywQy29jwY,4162
 megfile/lib/glob.py,sha256=-N75Phx1c8IpZ2hNIMwsHIx6BxMKVVr7N82cpWhs4lQ,9987
 megfile/lib/hdfs_prefetch_reader.py,sha256=yCNpcXcTiC2SHKHC-Qp50KQx1ObSLmOgwNUKlG-4ADg,2131
 megfile/lib/hdfs_tools.py,sha256=4K-OdMYFFSLBGmDzjatioHvuZuUbKVy7ACeJl-l0HLQ,435
-megfile/lib/http_prefetch_reader.py,sha256=OjP5pdWK_e1QiFIt2xmflceLnrz3S7B7ePBZFK-OwQE,4558
+megfile/lib/http_prefetch_reader.py,sha256=OQPZ7kWFImqpynjaiTtmadtgtab5fCeQmu51UYHZfgs,4135
 megfile/lib/joinpath.py,sha256=gaPNtBi8fzd5LZNyZp5zrHzaybcqKJ1xlntGmVNyFEM,929
 megfile/lib/lazy_handler.py,sha256=bE7RGt1x_xYWMgGAvHr7dwEt52qy-D3z90X3oyCvE6g,1875
 megfile/lib/s3_buffered_writer.py,sha256=kXvz1bdoaVIxjnEQeg4dxEzAXMYNrk-5uIMww86ty00,7860
@@ -41,12 +41,12 @@ megfile/lib/s3_share_cache_reader.py,sha256=LVWKxHdHo0_zUIW4o8yqNvplqqwezUPeYEt0
 megfile/lib/shadow_handler.py,sha256=TntewlvIW9ZxCfmqASDQREHoiZ8v42faOe9sovQYQz0,2779
 megfile/lib/stdio_handler.py,sha256=IDdgENLQlhigEwkLL4zStueVSzdWg7xVcTF_koof_Ek,1987
 megfile/lib/url.py,sha256=ER32pWy9Q2MAk3TraAaNEBWIqUeBmLuM57ol2cs7-Ks,103
-megfile/utils/__init__.py,sha256=xAzmICA0MtAbg-I2yPfeHjA1N4CiMP4sBrC9BgrfZLw,10151
+megfile/utils/__init__.py,sha256=pawmXnCNokWLj338a60b_hK21koYavpEiEohZhsOaGQ,10156
 megfile/utils/mutex.py,sha256=asb8opGLgK22RiuBJUnfsvB8LnMmodP8KzCVHKmQBWA,2561
-megfile-4.1.4.post1.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-megfile-4.1.4.post1.dist-info/licenses/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
-megfile-4.1.4.post1.dist-info/METADATA,sha256=-gzxqJGLKfx6MOtJu4nARQ-wj6e46niHR0rXZQoqPXE,9601
-megfile-4.1.4.post1.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
-megfile-4.1.4.post1.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
-megfile-4.1.4.post1.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
-megfile-4.1.4.post1.dist-info/RECORD,,
+megfile-4.1.6.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+megfile-4.1.6.dist-info/licenses/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
+megfile-4.1.6.dist-info/METADATA,sha256=_YzyQHnfYQYQRtbqc5__xrF5AtBBdIBDMs0WujErjBk,9595
+megfile-4.1.6.dist-info/WHEEL,sha256=ooBFpIzZCPdw3uqIQsOo4qqbA4ZRPxHnOH7peeONza0,91
+megfile-4.1.6.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
+megfile-4.1.6.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
+megfile-4.1.6.dist-info/RECORD,,

{megfile-4.1.4.post1.dist-info → megfile-4.1.6.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (79.0.1)
+Generator: setuptools (80.0.1)
 Root-Is-Purelib: true
 Tag: py3-none-any