megfile 3.0.6__py3-none-any.whl → 3.1.0__py3-none-any.whl

docs/conf.py

@@ -0,0 +1,71 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# http://www.sphinx-doc.org/en/master/config
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+import sys
+
+sys.path.insert(0, os.path.abspath('../megfile'))
+
+# -- Project information -----------------------------------------------------
+
+project = 'megfile'
+copyright = '2019, r-eng'
+author = 'r-eng'
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'sphinx.ext.autodoc',
+    'sphinx.ext.viewcode',
+    'sphinx.ext.todo',
+    'm2r2',
+    'sphinx_rtd_theme',
+    'sphinx_click',
+    'sphinxcontrib.jquery',
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = 'en'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = "sphinx_rtd_theme"
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# -- Extension configuration -------------------------------------------------
+
+# -- Options for todo extension ----------------------------------------------
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = True

megfile/cli.py

@@ -14,7 +14,7 @@ from megfile.config import DEFAULT_BLOCK_SIZE
 from megfile.hdfs_path import DEFAULT_HDFS_TIMEOUT
 from megfile.interfaces import FileEntry
 from megfile.lib.glob import get_non_glob_dir, has_magic
-from megfile.smart import _smart_sync_single_file, smart_copy, smart_exists, smart_getmd5, smart_getmtime, smart_getsize, smart_glob_stat, smart_isdir, smart_isfile, smart_makedirs, smart_move, smart_open, smart_path_join, smart_readlink, smart_realpath, smart_remove, smart_rename, smart_scan_stat, smart_scandir, smart_stat, smart_sync, smart_sync_with_progress, smart_touch, smart_unlink
+from megfile.smart import _smart_sync_single_file, smart_copy, smart_exists, smart_getmd5, smart_getmtime, smart_getsize, smart_glob_stat, smart_isdir, smart_isfile, smart_makedirs, smart_move, smart_open, smart_path_join, smart_readlink, smart_remove, smart_rename, smart_scan_stat, smart_scandir, smart_stat, smart_sync, smart_sync_with_progress, smart_touch, smart_unlink
 from megfile.smart_path import SmartPath
 from megfile.utils import get_human_size
 from megfile.version import VERSION
@@ -176,12 +176,12 @@ def ll(path: str, recursive: bool):
 @click.option('-g', '--progress-bar', is_flag=True, help='Show progress bar.')
 @click.option('--skip', is_flag=True, help='Skip existed files.')
 def cp(
-        src_path: str,
-        dst_path: str,
-        recursive: bool,
-        no_target_directory: bool,
-        progress_bar: bool,
-        skip: bool,
+    src_path: str,
+    dst_path: str,
+    recursive: bool,
+    no_target_directory: bool,
+    progress_bar: bool,
+    skip: bool,
 ):
     if not no_target_directory and (dst_path.endswith('/') or
                                     smart_isdir(dst_path)):
@@ -243,12 +243,12 @@ def cp(
 @click.option('-g', '--progress-bar', is_flag=True, help='Show progress bar.')
 @click.option('--skip', is_flag=True, help='Skip existed files.')
 def mv(
-        src_path: str,
-        dst_path: str,
-        recursive: bool,
-        no_target_directory: bool,
-        progress_bar: bool,
-        skip: bool,
+    src_path: str,
+    dst_path: str,
+    recursive: bool,
+    no_target_directory: bool,
+    progress_bar: bool,
+    skip: bool,
 ):
     if not no_target_directory and (dst_path.endswith('/') or
                                     smart_isdir(dst_path)):
@@ -321,7 +321,7 @@ def rm(path: str, recursive: bool):
     '-f',
     '--force',
     is_flag=True,
-    help='Copy files forcely, ignore same files.')
+    help='Copy files forcible, ignore same files.')
 @click.option('-q', '--quiet', is_flag=True, help='Not show any progress log.')
 @click.option('--skip', is_flag=True, help='Skip existed files.')
 def sync(
@@ -419,8 +419,8 @@ def touch(path: str):
 @cli.command(short_help='Concatenate any files and send them to stdout.')
 @click.argument('path')
 def cat(path: str):
-    with smart_open(path, 'rb') as file:
-        shutil.copyfileobj(file, sys.stdout.buffer)
+    with smart_open(path, 'rb') as f:
+        shutil.copyfileobj(f, sys.stdout.buffer)  # pytype: disable=wrong-arg-types
 
 
 @cli.command(

megfile/config.py

@@ -1,12 +1,43 @@
 import os
+from logging import getLogger
+
+_logger = getLogger(__name__)
+
+DEFAULT_BLOCK_SIZE = int(os.getenv('MEGFILE_BLOCK_SIZE') or 8 * 2**20)
+
+if os.getenv('MEGFILE_MAX_BUFFER_SIZE'):
+    DEFAULT_MAX_BUFFER_SIZE = int(os.environ['MEGFILE_MAX_BUFFER_SIZE'])
+    if DEFAULT_MAX_BUFFER_SIZE < DEFAULT_BLOCK_SIZE:
+        DEFAULT_MAX_BUFFER_SIZE = DEFAULT_BLOCK_SIZE
+        _logger.warning(
+            "Env 'MEGFILE_MAX_BUFFER_SIZE' is smaller than block size, will not use buffer."
+        )
+    DEFAULT_BLOCK_CAPACITY = DEFAULT_MAX_BUFFER_SIZE // DEFAULT_BLOCK_SIZE
+    if os.getenv('MEGFILE_BLOCK_CAPACITY'):
+        _logger.warning(
+            "Env 'MEGFILE_MAX_BUFFER_SIZE' and 'MEGFILE_BLOCK_CAPACITY' are both set, 'MEGFILE_BLOCK_CAPACITY' will be ignored."
+        )
+elif os.getenv('MEGFILE_BLOCK_CAPACITY'):
+    DEFAULT_BLOCK_CAPACITY = int(os.environ['MEGFILE_BLOCK_CAPACITY'])
+    DEFAULT_MAX_BUFFER_SIZE = DEFAULT_BLOCK_SIZE * DEFAULT_BLOCK_CAPACITY
+else:
+    DEFAULT_MAX_BUFFER_SIZE = 128 * 2**20
+    DEFAULT_BLOCK_CAPACITY = 16
+
+DEFAULT_MIN_BLOCK_SIZE = int(
+    os.getenv('MEGFILE_MIN_BLOCK_SIZE') or DEFAULT_BLOCK_SIZE)
+
+if os.getenv('MEGFILE_MAX_BLOCK_SIZE'):
+    DEFAULT_MAX_BLOCK_SIZE = int(os.environ['MEGFILE_MAX_BLOCK_SIZE'])
+    if DEFAULT_MAX_BLOCK_SIZE < DEFAULT_BLOCK_SIZE:
+        DEFAULT_MAX_BLOCK_SIZE = DEFAULT_BLOCK_SIZE
+        _logger.warning(
+            "Env 'MEGFILE_MAX_BLOCK_SIZE' is smaller than block size, will be ignored."
+        )
+else:
+    DEFAULT_MAX_BLOCK_SIZE = max(128 * 2**20, DEFAULT_BLOCK_SIZE)
 
-DEFAULT_BLOCK_SIZE = int(os.getenv('MEGFILE_BLOCK_SIZE') or 8 * 2**20)  # 8MB
-DEFAULT_MAX_BLOCK_SIZE = int(
-    os.getenv('MEGFILE_MAX_BLOCK_SIZE') or DEFAULT_BLOCK_SIZE * 16)  # 128MB
-DEFAULT_MAX_BUFFER_SIZE = int(
-    os.getenv('MEGFILE_MAX_BUFFER_SIZE') or DEFAULT_BLOCK_SIZE * 16)  # 128MB
 GLOBAL_MAX_WORKERS = int(os.getenv('MEGFILE_MAX_WORKERS') or 32)
-DEFAULT_BLOCK_CAPACITY = int(os.getenv('MEGFILE_BLOCK_CAPACITY') or 16)
 DEFAULT_MAX_RETRY_TIMES = int(os.getenv('MEGFILE_MAX_RETRY_TIMES') or 10)
 
 # for logging the size of file had read or wrote

megfile/errors.py

@@ -1,3 +1,4 @@
+# pyre-ignore-all-errors[16]
 import time
 from contextlib import contextmanager
 from functools import wraps
@@ -78,16 +79,16 @@ def full_error_message(error):
 
 
 def client_error_code(error: ClientError) -> str:
-    error_data = error.response.get('Error', {})  # pytype: disable=attribute-error
+    error_data = error.response.get('Error', {})
     return error_data.get('Code') or error_data.get('code', 'Unknown')
 
 
 def client_error_message(error: ClientError) -> str:
-    return error.response.get('Error', {}).get('Message', 'Unknown')  # pytype: disable=attribute-error
+    return error.response.get('Error', {}).get('Message', 'Unknown')
 
 
 def param_validation_error_report(error: ParamValidationError) -> str:
-    return error.kwargs.get('report', 'Unknown')  # pytype: disable=attribute-error
+    return error.kwargs.get('report', 'Unknown')
 
 
 s3_retry_exceptions = [
@@ -106,12 +107,13 @@ s3_retry_exceptions = [
 ]
 if hasattr(botocore.exceptions,
            'ResponseStreamingError'):  # backport botocore==1.23.24
-    s3_retry_exceptions.append(botocore.exceptions.ResponseStreamingError)
-s3_retry_exceptions = tuple(s3_retry_exceptions)
+    s3_retry_exceptions.append(
+        botocore.exceptions.ResponseStreamingError)  # pyre-ignore[6]
+s3_retry_exceptions = tuple(s3_retry_exceptions)  # pyre-ignore[9]
 
 
 def s3_should_retry(error: Exception) -> bool:
-    if isinstance(error, s3_retry_exceptions):
+    if isinstance(error, s3_retry_exceptions):  # pyre-ignore[6]
         return True
     if isinstance(error, botocore.exceptions.ClientError):
         return client_error_code(error) in (
@@ -170,19 +172,21 @@ def _create_missing_ok_generator(generator, missing_ok: bool, error: Exception):
         yield from generator
         return
 
-    zero_elum = True
+    zero_elem = True
     for item in generator:
-        zero_elum = False
+        zero_elem = False
         yield item
 
-    if zero_elum:
+    if zero_elem:
         raise error
 
 
 class UnknownError(Exception):
 
     def __init__(
-            self, error: Exception, path: PathLike,
+            self,
+            error: Exception,
+            path: PathLike,
             extra: Optional[str] = None):
         message = 'Unknown error encountered: %r, error: %s' % (
             path, full_error_message(error))
@@ -317,7 +321,7 @@ class ProtocolNotFoundError(Exception):
     pass
 
 
-def translate_fs_error(fs_error: Exception, fs_path: PathLike):
+def translate_fs_error(fs_error: Exception, fs_path: PathLike) -> Exception:
     if isinstance(fs_error, OSError):
         if fs_error.filename is None:
             fs_error.filename = fs_path
@@ -369,8 +373,7 @@ def translate_s3_error(s3_error: Exception, s3_url: PathLike) -> Exception:
     return S3UnknownError(s3_error, s3_url)
 
 
-def translate_http_error(
-        http_error: Optional[Exception], http_url: str) -> Exception:
+def translate_http_error(http_error: Exception, http_url: str) -> Exception:
     '''Generate exception according to http_error and status_code
 
     .. note ::
@@ -405,20 +408,23 @@ def s3_error_code_should_retry(error: str) -> bool:
     return False
 
 
-def translate_hdfs_error(hdfs_error: Exception, hdfs_path: PathLike):
+def translate_hdfs_error(
+        hdfs_error: Exception, hdfs_path: PathLike) -> Exception:
     from megfile.lib.hdfs_tools import hdfs_api
 
+    # pytype: disable=attribute-error
     if hdfs_api and isinstance(hdfs_error, hdfs_api.HdfsError):
-        if hdfs_error.message and 'Path is not a file' in hdfs_error.message:  # pytype: disable=attribute-error
+        if hdfs_error.message and 'Path is not a file' in hdfs_error.message:
             return IsADirectoryError('Is a directory: %r' % hdfs_path)
-        elif hdfs_error.message and 'Path is not a directory' in hdfs_error.message:  # pytype: disable=attribute-error
+        elif hdfs_error.message and 'Path is not a directory' in hdfs_error.message:
             return NotADirectoryError('Not a directory: %r' % hdfs_path)
-        elif hdfs_error.status_code in (401, 403):  # pytype: disable=attribute-error
+        elif hdfs_error.status_code in (401, 403):
             return PermissionError('Permission denied: %r' % hdfs_path)
-        elif hdfs_error.status_code == 400:  # pytype: disable=attribute-error
-            return ValueError(f'{hdfs_error.message}, path: {hdfs_path}')  # pytype: disable=attribute-error
-        elif hdfs_error.status_code == 404:  # pytype: disable=attribute-error
+        elif hdfs_error.status_code == 400:
+            return ValueError(f'{hdfs_error.message}, path: {hdfs_path}')
+        elif hdfs_error.status_code == 404:
             return FileNotFoundError(f'No match file: {hdfs_path}')
+    # pytype: enable=attribute-error
     return hdfs_error
 
 

megfile/fs.py

@@ -161,10 +161,10 @@ def fs_isfile(path: PathLike, followlinks: bool = False) -> bool:
 
 def fs_listdir(path: PathLike) -> List[str]:
     '''
-    Get all contents of given fs path. The result is in acsending alphabetical order.
+    Get all contents of given fs path. The result is in ascending alphabetical order.
 
     :param path: Given path
-    :returns: All contents have in the path in acsending alphabetical order
+    :returns: All contents have in the path in ascending alphabetical order
     '''
     return FSPath(path).listdir()
 
@@ -209,7 +209,8 @@ def fs_remove(path: PathLike, missing_ok: bool = False) -> None:
     return FSPath(path).remove(missing_ok)
 
 
-def fs_scan(path: PathLike, missing_ok: bool = True,
+def fs_scan(path: PathLike,
+            missing_ok: bool = True,
             followlinks: bool = False) -> Iterator[str]:
     '''
     Iteratively traverse only files in given directory, in alphabetical order.
@@ -227,7 +228,8 @@ def fs_scan(path: PathLike, missing_ok: bool = True,
 
 
 def fs_scan_stat(
-        path: PathLike, missing_ok: bool = True,
+        path: PathLike,
+        missing_ok: bool = True,
         followlinks: bool = False) -> Iterator[FileEntry]:
     '''
     Iteratively traverse only files in given directory, in alphabetical order.
@@ -270,8 +272,10 @@ def fs_unlink(path: PathLike, missing_ok: bool = False) -> None:
     return FSPath(path).unlink(missing_ok)
 
 
-def fs_walk(path: PathLike, followlinks: bool = False
-           ) -> Iterator[Tuple[str, List[str], List[str]]]:
+def fs_walk(
+        path: PathLike,
+        followlinks: bool = False
+) -> Iterator[Tuple[str, List[str], List[str]]]:
     '''
     Generate the file names in a directory tree by walking the tree top-down.
     For each directory in the tree rooted at directory path (including path itself),
@@ -302,6 +306,7 @@ def fs_getmd5(
     :param path: Given path
     :param recalculate: Ignore this parameter, just for compatibility
     :param followlinks: Ignore this parameter, just for compatibility
+
     returns: md5 of file
     '''
     return FSPath(path).md5(recalculate, followlinks)
@@ -348,7 +353,7 @@ def fs_sync(
     :param src_path: Given path
     :param dst_path: Target file path
     :param followlinks: False if regard symlink as file, else True
-    :param force: Sync file forcely, do not ignore same files, priority is higher than 'overwrite', default is False
+    :param force: Sync file forcible, do not ignore same files, priority is higher than 'overwrite', default is False
     :param overwrite: whether or not overwrite file when exists, default is True
     '''
     return FSPath(src_path).sync(dst_path, followlinks, force, overwrite)
@@ -359,7 +364,7 @@ def fs_symlink(src_path: PathLike, dst_path: PathLike) -> None:
     Create a symbolic link pointing to src_path named dst_path.
 
     :param src_path: Given path
-    :param dst_path: Desination path
+    :param dst_path: Destination path
     '''
     return FSPath(src_path).symlink(dst_path)