megfile 3.1.1__py3-none-any.whl → 3.1.2__py3-none-any.whl

megfile/config.py

@@ -3,32 +3,33 @@ from logging import getLogger
 
 _logger = getLogger(__name__)
 
-DEFAULT_BLOCK_SIZE = int(os.getenv('MEGFILE_BLOCK_SIZE') or 8 * 2**20)
+DEFAULT_BLOCK_SIZE = int(os.getenv("MEGFILE_BLOCK_SIZE") or 8 * 2**20)
 
-if os.getenv('MEGFILE_MAX_BUFFER_SIZE'):
-    DEFAULT_MAX_BUFFER_SIZE = int(os.environ['MEGFILE_MAX_BUFFER_SIZE'])
+if os.getenv("MEGFILE_MAX_BUFFER_SIZE"):
+    DEFAULT_MAX_BUFFER_SIZE = int(os.environ["MEGFILE_MAX_BUFFER_SIZE"])
     if DEFAULT_MAX_BUFFER_SIZE < DEFAULT_BLOCK_SIZE:
         DEFAULT_MAX_BUFFER_SIZE = DEFAULT_BLOCK_SIZE
         _logger.warning(
-            "Env 'MEGFILE_MAX_BUFFER_SIZE' is smaller than block size, will not use buffer."
+            "Env 'MEGFILE_MAX_BUFFER_SIZE' is smaller than block size, "
+            "will not use buffer."
         )
     DEFAULT_BLOCK_CAPACITY = DEFAULT_MAX_BUFFER_SIZE // DEFAULT_BLOCK_SIZE
-    if os.getenv('MEGFILE_BLOCK_CAPACITY'):
+    if os.getenv("MEGFILE_BLOCK_CAPACITY"):
         _logger.warning(
-            "Env 'MEGFILE_MAX_BUFFER_SIZE' and 'MEGFILE_BLOCK_CAPACITY' are both set, 'MEGFILE_BLOCK_CAPACITY' will be ignored."
+            "Env 'MEGFILE_MAX_BUFFER_SIZE' and 'MEGFILE_BLOCK_CAPACITY' are both set, "
+            "'MEGFILE_BLOCK_CAPACITY' will be ignored."
         )
-elif os.getenv('MEGFILE_BLOCK_CAPACITY'):
-    DEFAULT_BLOCK_CAPACITY = int(os.environ['MEGFILE_BLOCK_CAPACITY'])
+elif os.getenv("MEGFILE_BLOCK_CAPACITY"):
+    DEFAULT_BLOCK_CAPACITY = int(os.environ["MEGFILE_BLOCK_CAPACITY"])
     DEFAULT_MAX_BUFFER_SIZE = DEFAULT_BLOCK_SIZE * DEFAULT_BLOCK_CAPACITY
 else:
     DEFAULT_MAX_BUFFER_SIZE = 128 * 2**20
     DEFAULT_BLOCK_CAPACITY = 16
 
-DEFAULT_MIN_BLOCK_SIZE = int(
-    os.getenv('MEGFILE_MIN_BLOCK_SIZE') or DEFAULT_BLOCK_SIZE)
+DEFAULT_MIN_BLOCK_SIZE = int(os.getenv("MEGFILE_MIN_BLOCK_SIZE") or DEFAULT_BLOCK_SIZE)
 
-if os.getenv('MEGFILE_MAX_BLOCK_SIZE'):
-    DEFAULT_MAX_BLOCK_SIZE = int(os.environ['MEGFILE_MAX_BLOCK_SIZE'])
+if os.getenv("MEGFILE_MAX_BLOCK_SIZE"):
+    DEFAULT_MAX_BLOCK_SIZE = int(os.environ["MEGFILE_MAX_BLOCK_SIZE"])
     if DEFAULT_MAX_BLOCK_SIZE < DEFAULT_BLOCK_SIZE:
         DEFAULT_MAX_BLOCK_SIZE = DEFAULT_BLOCK_SIZE
         _logger.warning(
@@ -37,22 +38,25 @@ if os.getenv('MEGFILE_MAX_BLOCK_SIZE'):
 else:
     DEFAULT_MAX_BLOCK_SIZE = max(128 * 2**20, DEFAULT_BLOCK_SIZE)
 
-GLOBAL_MAX_WORKERS = int(os.getenv('MEGFILE_MAX_WORKERS') or 32)
-DEFAULT_MAX_RETRY_TIMES = int(os.getenv('MEGFILE_MAX_RETRY_TIMES') or 10)
+GLOBAL_MAX_WORKERS = int(os.getenv("MEGFILE_MAX_WORKERS") or 32)
+DEFAULT_MAX_RETRY_TIMES = int(os.getenv("MEGFILE_MAX_RETRY_TIMES") or 10)
 
 # for logging the size of file had read or wrote
 BACKOFF_INITIAL = 64 * 2**20  # 64MB
 BACKOFF_FACTOR = 4
 
-NEWLINE = ord('\n')
+NEWLINE = ord("\n")
 
-S3_CLIENT_CACHE_MODE = os.getenv(
-    'MEGFILE_S3_CLIENT_CACHE_MODE') or 'thread_local'
+S3_CLIENT_CACHE_MODE = os.getenv("MEGFILE_S3_CLIENT_CACHE_MODE") or "thread_local"
 S3_MAX_RETRY_TIMES = int(
-    os.getenv('MEGFILE_S3_MAX_RETRY_TIMES') or DEFAULT_MAX_RETRY_TIMES)
+    os.getenv("MEGFILE_S3_MAX_RETRY_TIMES") or DEFAULT_MAX_RETRY_TIMES
+)
 HTTP_MAX_RETRY_TIMES = int(
-    os.getenv('MEGFILE_HTTP_MAX_RETRY_TIMES') or DEFAULT_MAX_RETRY_TIMES)
+    os.getenv("MEGFILE_HTTP_MAX_RETRY_TIMES") or DEFAULT_MAX_RETRY_TIMES
+)
 HDFS_MAX_RETRY_TIMES = int(
-    os.getenv('MEGFILE_HDFS_MAX_RETRY_TIMES') or DEFAULT_MAX_RETRY_TIMES)
+    os.getenv("MEGFILE_HDFS_MAX_RETRY_TIMES") or DEFAULT_MAX_RETRY_TIMES
+)
 SFTP_MAX_RETRY_TIMES = int(
-    os.getenv('MEGFILE_SFTP_MAX_RETRY_TIMES') or DEFAULT_MAX_RETRY_TIMES)
+    os.getenv("MEGFILE_SFTP_MAX_RETRY_TIMES") or DEFAULT_MAX_RETRY_TIMES
+)

megfile/errors.py

@@ -15,31 +15,31 @@ from requests.exceptions import HTTPError
 from megfile.interfaces import PathLike
 
 __all__ = [
-    'S3FileNotFoundError',
-    'S3BucketNotFoundError',
-    'S3FileExistsError',
-    'S3NotADirectoryError',
-    'S3IsADirectoryError',
-    'S3PermissionError',
-    'S3ConfigError',
-    'UnknownError',
-    'UnsupportedError',
-    'HttpPermissionError',
-    'HttpFileNotFoundError',
-    'HttpBodyIncompleteError',
-    'HttpUnknownError',
-    'HttpException',
-    'ProtocolExistsError',
-    'ProtocolNotFoundError',
-    'S3UnknownError',
-    'SameFileError',
-    'translate_http_error',
-    'translate_s3_error',
-    'patch_method',
-    'raise_s3_error',
-    's3_should_retry',
-    'translate_fs_error',
-    'http_should_retry',
+    "S3FileNotFoundError",
+    "S3BucketNotFoundError",
+    "S3FileExistsError",
+    "S3NotADirectoryError",
+    "S3IsADirectoryError",
+    "S3PermissionError",
+    "S3ConfigError",
+    "UnknownError",
+    "UnsupportedError",
+    "HttpPermissionError",
+    "HttpFileNotFoundError",
+    "HttpBodyIncompleteError",
+    "HttpUnknownError",
+    "HttpException",
+    "ProtocolExistsError",
+    "ProtocolNotFoundError",
+    "S3UnknownError",
+    "SameFileError",
+    "translate_http_error",
+    "translate_s3_error",
+    "patch_method",
+    "raise_s3_error",
+    "s3_should_retry",
+    "translate_fs_error",
+    "http_should_retry",
 ]
 
 _logger = getLogger(__name__)
@@ -54,8 +54,7 @@ def s3_endpoint_url(path: Optional[PathLike] = None):
         profile_name = S3Path(path)._profile_name
     endpoint_url = get_endpoint_url(profile_name=profile_name)
     if endpoint_url is None:
-        endpoint_url = get_s3_client(
-            profile_name=profile_name).meta.endpoint_url
+        endpoint_url = get_s3_client(profile_name=profile_name).meta.endpoint_url
     return endpoint_url
 
 
@@ -71,24 +70,24 @@ def full_class_name(obj):
     if module is None or module == str.__class__.__module__:
         return obj.__class__.__name__  # Avoid reporting __builtin__
     else:
-        return module + '.' + obj.__class__.__name__
+        return module + "." + obj.__class__.__name__
 
 
 def full_error_message(error):
-    return '%s(%r)' % (full_class_name(error), str(error))
+    return "%s(%r)" % (full_class_name(error), str(error))
 
 
 def client_error_code(error: ClientError) -> str:
-    error_data = error.response.get('Error', {})
-    return error_data.get('Code') or error_data.get('code', 'Unknown')
+    error_data = error.response.get("Error", {})
+    return error_data.get("Code") or error_data.get("code", "Unknown")
 
 
 def client_error_message(error: ClientError) -> str:
-    return error.response.get('Error', {}).get('Message', 'Unknown')
+    return error.response.get("Error", {}).get("Message", "Unknown")
 
 
 def param_validation_error_report(error: ParamValidationError) -> str:
-    return error.kwargs.get('report', 'Unknown')
+    return error.kwargs.get("report", "Unknown")
 
 
 s3_retry_exceptions = [
@@ -105,10 +104,10 @@ s3_retry_exceptions = [
     urllib3.exceptions.ReadTimeoutError,
     urllib3.exceptions.HeaderParsingError,
 ]
-if hasattr(botocore.exceptions,
-           'ResponseStreamingError'):  # backport botocore==1.23.24
+if hasattr(botocore.exceptions, "ResponseStreamingError"):  # backport botocore==1.23.24
     s3_retry_exceptions.append(
-        botocore.exceptions.ResponseStreamingError)  # pyre-ignore[6]
+        botocore.exceptions.ResponseStreamingError  # pyre-ignore[6]
+    )
 s3_retry_exceptions = tuple(s3_retry_exceptions)  # pyre-ignore[9]
 
 
@@ -117,26 +116,32 @@ def s3_should_retry(error: Exception) -> bool:
         return True
     if isinstance(error, botocore.exceptions.ClientError):
         return client_error_code(error) in (
-            '499',  # Some cloud providers may send response with http code 499 if the connection not send data in 1 min.
-            '500',
-            '501',
-            '502',
-            '503',
-            'InternalError',
-            'ServiceUnavailable',
-            'SlowDown',
-            'ContextCanceled')
+            "429",  # noqa: E501 # TOS ExceedAccountQPSLimit
+            "499",  # noqa: E501 # Some cloud providers may send response with http code 499 if the connection not send data in 1 min.
+            "500",
+            "501",
+            "502",
+            "503",
+            "InternalError",
+            "ServiceUnavailable",
+            "SlowDown",
+            "ContextCanceled",
+            "ExceedAccountQPSLimit",
+            "ExceedAccountRateLimit",
+            "ExceedBucketQPSLimit",
+            "ExceedBucketRateLimit",
+        )
     return False
 
 
 def patch_method(
-        func: Callable,
-        max_retries: int,
-        should_retry: Callable[[Exception], bool],
-        before_callback: Optional[Callable] = None,
-        after_callback: Optional[Callable] = None,
-        retry_callback: Optional[Callable] = None):
-
+    func: Callable,
+    max_retries: int,
+    should_retry: Callable[[Exception], bool],
+    before_callback: Optional[Callable] = None,
+    after_callback: Optional[Callable] = None,
+    retry_callback: Optional[Callable] = None,
+):
     @wraps(func)
     def wrapper(*args, **kwargs):
         if before_callback is not None:
@@ -148,8 +153,7 @@ def patch_method(
                 if after_callback is not None:
                     result = after_callback(result, *args, **kwargs)
                 if retries > 1:
-                    _logger.info(
-                        f'Error already fixed by retry {retries - 1} times')
+                    _logger.info(f"Error already fixed by retry {retries - 1} times")
                 return result
             except Exception as error:
                 if not should_retry(error):
@@ -160,8 +164,10 @@ def patch_method(
                     raise
                 retry_interval = min(0.1 * 2**retries, 30)
                 _logger.info(
-                    'unknown error encountered: %s, retry in %0.1f seconds after %d tries'
-                    % (full_error_message(error), retry_interval, retries))
+                    "unknown error encountered: %s, retry in %0.1f seconds "
+                    "after %d tries"
+                    % (full_error_message(error), retry_interval, retries)
+                )
                 time.sleep(retry_interval)
 
     return wrapper
@@ -182,16 +188,13 @@ def _create_missing_ok_generator(generator, missing_ok: bool, error: Exception):
 
 
 class UnknownError(Exception):
-
-    def __init__(
-            self,
-            error: Exception,
-            path: PathLike,
-            extra: Optional[str] = None):
-        message = 'Unknown error encountered: %r, error: %s' % (
-            path, full_error_message(error))
+    def __init__(self, error: Exception, path: PathLike, extra: Optional[str] = None):
+        message = "Unknown error encountered: %r, error: %s" % (
+            path,
+            full_error_message(error),
+        )
         if extra is not None:
-            message += ', ' + extra
+            message += ", " + extra
         super().__init__(message)
         self.path = path
         self.extra = extra
@@ -202,10 +205,8 @@ class UnknownError(Exception):
 
 
 class UnsupportedError(Exception):
-
     def __init__(self, operation: str, path: PathLike):
-        super().__init__(
-            'Unsupported operation: %r, operation: %r' % (path, operation))
+        super().__init__("Unsupported operation: %r, operation: %r" % (path, operation))
         self.path = path
         self.operation = operation
 
@@ -214,10 +215,10 @@ class UnsupportedError(Exception):
 
 
 class S3Exception(Exception):
-    '''
+    """
     Base type for all s3 errors, should NOT be constructed directly.
     When you try to do so, consider adding a new type of error.
-    '''
+    """
 
 
 class S3FileNotFoundError(S3Exception, FileNotFoundError):
@@ -249,8 +250,10 @@ class S3PermissionError(S3Exception, PermissionError):
 
 
 class S3ConfigError(S3Exception, EnvironmentError):
-    '''Error raised by wrong S3 config, including wrong config file format, wrong aws_secret_access_key / aws_access_key_id, and etc.
-    '''
+    """
+    Error raised by wrong S3 config, including wrong config file format,
+    wrong aws_secret_access_key / aws_access_key_id, and etc.
+    """
 
 
 class S3NotALinkError(S3FileNotFoundError, PermissionError):
@@ -266,16 +269,15 @@ class S3InvalidRangeError(S3Exception):
 
 
 class S3UnknownError(S3Exception, UnknownError):
-
     def __init__(self, error: Exception, path: PathLike):
-        super().__init__(error, path, 'endpoint: %r' % s3_endpoint_url(path))
+        super().__init__(error, path, "endpoint: %r" % s3_endpoint_url(path))
 
 
 class HttpException(Exception):
-    '''
+    """
     Base type for all http errors, should NOT be constructed directly.
     When you try to do so, consider adding a new type of error.
-    '''
+    """
 
 
 class HttpPermissionError(HttpException, PermissionError):
@@ -330,43 +332,52 @@ def translate_fs_error(fs_error: Exception, fs_path: PathLike) -> Exception:
 
 
 def translate_s3_error(s3_error: Exception, s3_url: PathLike) -> Exception:
-    ''' :param s3_error: error raised by boto3
-        :param s3_url: s3_url
-    '''
+    """:param s3_error: error raised by boto3
+    :param s3_url: s3_url
+    """
     if isinstance(s3_error, S3Exception):
         return s3_error
     elif isinstance(s3_error, ClientError):
         code = client_error_code(s3_error)
-        if code in ('NoSuchBucket'):
+        if code in ("NoSuchBucket"):
             return S3BucketNotFoundError(
-                'No such bucket: %r' %
-                s3_error.response.get('Error', {}).get('BucketName') or s3_url)  # pytype: disable=attribute-error
-        if code in ('404', 'NoSuchKey'):
-            return S3FileNotFoundError('No such file: %r' % s3_url)
-        if code in ('401', '403', 'AccessDenied'):
+                "No such bucket: %r"
+                % s3_error.response.get(  # pytype: disable=attribute-error
+                    "Error", {}
+                ).get("BucketName")
+                or s3_url
+            )
+        if code in ("404", "NoSuchKey"):
+            return S3FileNotFoundError("No such file: %r" % s3_url)
+        if code in ("401", "403", "AccessDenied"):
             message = client_error_message(s3_error)
             return S3PermissionError(
-                'Permission denied: %r, code: %r, message: %r, endpoint: %r' %
-                (s3_url, code, message, s3_endpoint_url(s3_url)))
-        if code in ('InvalidAccessKeyId', 'SignatureDoesNotMatch'):
+                "Permission denied: %r, code: %r, message: %r, endpoint: %r"
+                % (s3_url, code, message, s3_endpoint_url(s3_url))
+            )
+        if code in ("InvalidAccessKeyId", "SignatureDoesNotMatch"):
             message = client_error_message(s3_error)
             return S3ConfigError(
-                'Invalid configuration: %r, code: %r, message: %r, endpoint: %r'
-                % (s3_url, code, message, s3_endpoint_url(s3_url)))
-        if code in ('InvalidRange'):
+                "Invalid configuration: %r, code: %r, message: %r, endpoint: %r"
+                % (s3_url, code, message, s3_endpoint_url(s3_url))
+            )
+        if code in ("InvalidRange"):
             return S3InvalidRangeError(
-                'Index out of range: %r, code: %r, message: %r, endpoint: %r' %
-                (
-                    s3_url, code, client_error_message(s3_error),
-                    s3_endpoint_url(s3_url)))
+                "Index out of range: %r, code: %r, message: %r, endpoint: %r"
+                % (
+                    s3_url,
+                    code,
+                    client_error_message(s3_error),
+                    s3_endpoint_url(s3_url),
+                )
+            )
         return S3UnknownError(s3_error, s3_url)
     elif isinstance(s3_error, ParamValidationError):
         report = param_validation_error_report(s3_error)
-        if 'Invalid bucket name' in report:
-            return S3BucketNotFoundError('Invalid bucket name: %r' % s3_url)
-        if 'Invalid length for parameter Key' in report:
-            return S3FileNotFoundError(
-                'Invalid length for parameter Key: %r' % s3_url)
+        if "Invalid bucket name" in report:
+            return S3BucketNotFoundError("Invalid bucket name: %r" % s3_url)
+        if "Invalid length for parameter Key" in report:
+            return S3FileNotFoundError("Invalid length for parameter Key: %r" % s3_url)
         return S3UnknownError(s3_error, s3_url)
     elif isinstance(s3_error, NoCredentialsError):
         return S3ConfigError(str(s3_error))
@@ -374,7 +385,7 @@ def translate_s3_error(s3_error: Exception, s3_url: PathLike) -> Exception:
 
 
 def translate_http_error(http_error: Exception, http_url: str) -> Exception:
-    '''Generate exception according to http_error and status_code
+    """Generate exception according to http_error and status_code
 
     .. note ::
 
@@ -382,15 +393,15 @@ def translate_http_error(http_error: Exception, http_url: str) -> Exception:
 
     :param http_error: error raised by requests
     :param http_url: http url
-    '''
+    """
     if isinstance(http_error, HttpException):
         return http_error
     if isinstance(http_error, HTTPError):
         status_code = http_error.response.status_code
         if status_code == 401 or status_code == 403:
-            return HttpPermissionError('Permission denied: %r' % http_url)
+            return HttpPermissionError("Permission denied: %r" % http_url)
         elif status_code == 404:
-            return HttpFileNotFoundError('No such file: %r' % http_url)
+            return HttpFileNotFoundError("No such file: %r" % http_url)
     return HttpUnknownError(http_error, http_url)
 
 
@@ -403,27 +414,26 @@ def raise_s3_error(s3_url: PathLike):
 
 
 def s3_error_code_should_retry(error: str) -> bool:
-    if error in ['InternalError', 'ServiceUnavailable', 'SlowDown']:
+    if error in ["InternalError", "ServiceUnavailable", "SlowDown"]:
         return True
     return False
 
 
-def translate_hdfs_error(
-        hdfs_error: Exception, hdfs_path: PathLike) -> Exception:
+def translate_hdfs_error(hdfs_error: Exception, hdfs_path: PathLike) -> Exception:
     from megfile.lib.hdfs_tools import hdfs_api
 
     # pytype: disable=attribute-error
     if hdfs_api and isinstance(hdfs_error, hdfs_api.HdfsError):
-        if hdfs_error.message and 'Path is not a file' in hdfs_error.message:
-            return IsADirectoryError('Is a directory: %r' % hdfs_path)
-        elif hdfs_error.message and 'Path is not a directory' in hdfs_error.message:
-            return NotADirectoryError('Not a directory: %r' % hdfs_path)
+        if hdfs_error.message and "Path is not a file" in hdfs_error.message:
+            return IsADirectoryError("Is a directory: %r" % hdfs_path)
+        elif hdfs_error.message and "Path is not a directory" in hdfs_error.message:
+            return NotADirectoryError("Not a directory: %r" % hdfs_path)
         elif hdfs_error.status_code in (401, 403):
-            return PermissionError('Permission denied: %r' % hdfs_path)
+            return PermissionError("Permission denied: %r" % hdfs_path)
         elif hdfs_error.status_code == 400:
-            return ValueError(f'{hdfs_error.message}, path: {hdfs_path}')
+            return ValueError(f"{hdfs_error.message}, path: {hdfs_path}")
         elif hdfs_error.status_code == 404:
-            return FileNotFoundError(f'No match file: {hdfs_path}')
+            return FileNotFoundError(f"No match file: {hdfs_path}")
     # pytype: enable=attribute-error
     return hdfs_error