megfile 4.2.5__py3-none-any.whl → 5.0.1__py3-none-any.whl

megfile/__init__.py

@@ -1,164 +1,19 @@
-from megfile.fs import (
-    fs_abspath,
-    fs_access,
-    fs_cwd,
-    fs_exists,
-    fs_expanduser,
-    fs_getmd5,
-    fs_getmtime,
-    fs_getsize,
-    fs_glob,
-    fs_glob_stat,
-    fs_home,
-    fs_iglob,
-    fs_isabs,
-    fs_isdir,
-    fs_isfile,
-    fs_islink,
-    fs_ismount,
-    fs_listdir,
-    fs_load_from,
-    fs_lstat,
-    fs_makedirs,
-    fs_move,
-    fs_readlink,
-    fs_realpath,
-    fs_relpath,
-    fs_remove,
-    fs_rename,
-    fs_resolve,
-    fs_save_as,
-    fs_scan,
-    fs_scan_stat,
-    fs_scandir,
-    fs_stat,
-    fs_symlink,
-    fs_sync,
-    fs_unlink,
-    fs_walk,
-    is_fs,
-)
-from megfile.fs_path import FSPath
-from megfile.hdfs import (
-    hdfs_exists,
-    hdfs_getmd5,
-    hdfs_getmtime,
-    hdfs_getsize,
-    hdfs_glob,
-    hdfs_glob_stat,
-    hdfs_iglob,
-    hdfs_isdir,
-    hdfs_isfile,
-    hdfs_listdir,
-    hdfs_load_from,
-    hdfs_makedirs,
-    hdfs_move,
-    hdfs_open,
-    hdfs_remove,
-    hdfs_save_as,
-    hdfs_scan,
-    hdfs_scan_stat,
-    hdfs_scandir,
-    hdfs_stat,
-    hdfs_unlink,
-    hdfs_walk,
-    is_hdfs,
-)
-from megfile.hdfs_path import HdfsPath
-from megfile.http import (
-    http_exists,
-    http_getmtime,
-    http_getsize,
-    http_open,
-    http_stat,
-    is_http,
-)
-from megfile.http_path import HttpPath, HttpsPath
-from megfile.s3 import (
+import megfile.config  # noqa: F401  # make sure env config is loaded
+from megfile.fs_path import FSPath, is_fs
+from megfile.hdfs_path import HdfsPath, is_hdfs
+from megfile.http_path import HttpPath, HttpsPath, is_http
+from megfile.s3_path import (
+    S3Path,
     is_s3,
-    s3_access,
     s3_buffered_open,
     s3_cached_open,
-    s3_concat,
-    s3_copy,
-    s3_download,
-    s3_exists,
-    s3_getmd5,
-    s3_getmtime,
-    s3_getsize,
-    s3_glob,
-    s3_glob_stat,
-    s3_hasbucket,
-    s3_iglob,
-    s3_isdir,
-    s3_isfile,
-    s3_listdir,
-    s3_load_content,
-    s3_load_from,
-    s3_lstat,
-    s3_makedirs,
     s3_memory_open,
-    s3_move,
     s3_open,
-    s3_path_join,
     s3_pipe_open,
     s3_prefetch_open,
-    s3_readlink,
-    s3_remove,
-    s3_rename,
-    s3_save_as,
-    s3_scan,
-    s3_scan_stat,
-    s3_scandir,
-    s3_stat,
-    s3_symlink,
-    s3_sync,
-    s3_unlink,
-    s3_upload,
-    s3_walk,
-)
-from megfile.s3_path import S3Path
-from megfile.sftp import (
-    is_sftp,
-    sftp_absolute,
-    sftp_add_host_key,
-    sftp_chmod,
-    sftp_concat,
-    sftp_copy,
-    sftp_exists,
-    sftp_getmd5,
-    sftp_getmtime,
-    sftp_getsize,
-    sftp_glob,
-    sftp_glob_stat,
-    sftp_iglob,
-    sftp_isdir,
-    sftp_isfile,
-    sftp_islink,
-    sftp_listdir,
-    sftp_load_from,
-    sftp_lstat,
-    sftp_makedirs,
-    sftp_move,
-    sftp_open,
-    sftp_path_join,
-    sftp_readlink,
-    sftp_realpath,
-    sftp_remove,
-    sftp_rename,
-    sftp_resolve,
-    sftp_rmdir,
-    sftp_save_as,
-    sftp_scan,
-    sftp_scan_stat,
-    sftp_scandir,
-    sftp_stat,
-    sftp_symlink,
-    sftp_sync,
-    sftp_unlink,
-    sftp_walk,
+    s3_share_cache_open,
 )
-from megfile.sftp_path import SftpPath
+from megfile.sftp_path import SftpPath, is_sftp, sftp_add_host_key
 from megfile.smart import (
     smart_access,
     smart_cache,
@@ -202,8 +57,7 @@ from megfile.smart import (
     smart_walk,
 )
 from megfile.smart_path import SmartPath
-from megfile.stdio import is_stdio, stdio_open
-from megfile.stdio_path import StdioPath
+from megfile.stdio_path import StdioPath, is_stdio
 from megfile.version import VERSION as __version__  # noqa: F401
 
 try:
@@ -212,9 +66,9 @@ except ImportError:
     Sftp2Path = None
 
 try:
-    from megfile.webdav_path import WebdavPath
+    from megfile.webdav_path import WebdavPath, is_webdav
 except ImportError:
-    WebdavPath = None
+    WebdavPath = is_webdav = None
 
 __all__ = [
     "smart_access",
@@ -259,153 +113,22 @@ __all__ = [
     "smart_lstat",
     "smart_concat",
     "is_s3",
-    "s3_access",
     "s3_buffered_open",
     "s3_cached_open",
-    "s3_copy",
-    "s3_download",
-    "s3_exists",
-    "s3_getmd5",
-    "s3_getmtime",
-    "s3_getsize",
-    "s3_glob_stat",
-    "s3_glob",
-    "s3_hasbucket",
-    "s3_iglob",
-    "s3_isdir",
-    "s3_isfile",
-    "s3_listdir",
-    "s3_load_content",
-    "s3_load_from",
-    "s3_makedirs",
     "s3_memory_open",
     "s3_open",
-    "s3_path_join",
     "s3_pipe_open",
     "s3_prefetch_open",
-    "s3_remove",
-    "s3_rename",
-    "s3_move",
-    "s3_sync",
-    "s3_save_as",
-    "s3_scan_stat",
-    "s3_scan",
-    "s3_scandir",
-    "s3_stat",
-    "s3_lstat",
-    "s3_unlink",
-    "s3_upload",
-    "s3_walk",
-    "s3_symlink",
-    "s3_readlink",
-    "s3_concat",
+    "s3_share_cache_open",
     "is_fs",
-    "fs_abspath",
-    "fs_access",
-    "fs_exists",
-    "fs_getmtime",
-    "fs_getsize",
-    "fs_glob_stat",
-    "fs_glob",
-    "fs_iglob",
-    "fs_isabs",
-    "fs_isdir",
-    "fs_isfile",
-    "fs_islink",
-    "fs_ismount",
-    "fs_listdir",
-    "fs_load_from",
-    "fs_makedirs",
-    "fs_realpath",
-    "fs_relpath",
-    "fs_remove",
-    "fs_rename",
-    "fs_move",
-    "fs_sync",
-    "fs_save_as",
-    "fs_scan_stat",
-    "fs_scan",
-    "fs_scandir",
-    "fs_stat",
-    "fs_lstat",
-    "fs_unlink",
-    "fs_walk",
-    "fs_cwd",
-    "fs_home",
-    "fs_expanduser",
-    "fs_resolve",
-    "fs_getmd5",
-    "fs_symlink",
-    "fs_readlink",
     "is_http",
-    "http_open",
-    "http_stat",
-    "http_getsize",
-    "http_getmtime",
-    "http_exists",
     "is_stdio",
     "stdio_open",
     "is_sftp",
-    "sftp_readlink",
-    "sftp_absolute",
-    "sftp_glob",
-    "sftp_iglob",
-    "sftp_glob_stat",
-    "sftp_resolve",
-    "sftp_isdir",
-    "sftp_exists",
-    "sftp_scandir",
-    "sftp_getmtime",
-    "sftp_getsize",
-    "sftp_isfile",
-    "sftp_listdir",
-    "sftp_load_from",
-    "sftp_makedirs",
-    "sftp_realpath",
-    "sftp_rename",
-    "sftp_move",
-    "sftp_remove",
-    "sftp_scan",
-    "sftp_scan_stat",
-    "sftp_stat",
-    "sftp_lstat",
-    "sftp_unlink",
-    "sftp_walk",
-    "sftp_path_join",
-    "sftp_getmd5",
-    "sftp_symlink",
-    "sftp_islink",
-    "sftp_save_as",
-    "sftp_open",
-    "sftp_chmod",
-    "sftp_rmdir",
-    "sftp_copy",
-    "sftp_sync",
-    "sftp_concat",
     "sftp_add_host_key",
     "is_hdfs",
-    "hdfs_exists",
-    "hdfs_stat",
-    "hdfs_getmtime",
-    "hdfs_getsize",
-    "hdfs_isdir",
-    "hdfs_isfile",
-    "hdfs_listdir",
-    "hdfs_load_from",
-    "hdfs_move",
-    "hdfs_remove",
-    "hdfs_scan",
-    "hdfs_scan_stat",
-    "hdfs_scandir",
-    "hdfs_unlink",
-    "hdfs_walk",
-    "hdfs_getmd5",
-    "hdfs_save_as",
-    "hdfs_open",
-    "hdfs_glob",
-    "hdfs_glob_stat",
-    "hdfs_iglob",
-    "hdfs_makedirs",
+    "is_webdav",
+    "WebdavPath",
     "S3Path",
     "FSPath",
     "HttpPath",

megfile/cli.py

@@ -1,4 +1,3 @@
-import configparser
 import os
 import shutil
 import signal
@@ -14,12 +13,18 @@ from click import ParamType
 from click.shell_completion import CompletionItem, ZshComplete
 from tqdm import tqdm
 
-from megfile.config import READER_BLOCK_SIZE, SFTP_HOST_KEY_POLICY, set_log_level
+from megfile.config import (
+    CONFIG_PATH,
+    READER_BLOCK_SIZE,
+    SFTP_HOST_KEY_POLICY,
+    CaseSensitiveConfigParser,
+    set_log_level,
+)
 from megfile.hdfs_path import DEFAULT_HDFS_TIMEOUT
 from megfile.interfaces import FileEntry
 from megfile.lib.glob import get_non_glob_dir, has_magic
 from megfile.s3_path import get_s3_session
-from megfile.sftp import sftp_add_host_key
+from megfile.sftp_path import sftp_add_host_key
 from megfile.smart import (
     _smart_sync_single_file,
     smart_copy,
@@ -86,33 +91,33 @@ def safe_cli():  # pragma: no cover
             sys.exit(1)
 
 
-def get_echo_path(file_stat, base_path: str = "", full_path: bool = False):
+def get_echo_path(file_stat, base_path: str = "", full: bool = False):
     if base_path == file_stat.path:
         path = file_stat.name
-    elif full_path:
+    elif full:
         path = file_stat.path
     else:
         path = smart_relpath(file_stat.path, start=base_path)
     return path
 
 
-def simple_echo(file_stat, base_path: str = "", full_path: bool = False):
-    return get_echo_path(file_stat, base_path, full_path)
+def simple_echo(file_stat, base_path: str = "", full: bool = False):
+    return get_echo_path(file_stat, base_path, full)
 
 
-def long_echo(file_stat, base_path: str = "", full_path: bool = False):
+def long_echo(file_stat, base_path: str = "", full: bool = False):
     return "%12d %s %s" % (
         file_stat.stat.size,
         time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(file_stat.stat.mtime)),
-        get_echo_path(file_stat, base_path, full_path),
+        get_echo_path(file_stat, base_path, full),
     )
 
 
-def human_echo(file_stat, base_path: str = "", full_path: bool = False):
+def human_echo(file_stat, base_path: str = "", full: bool = False):
     return "%10s %s %s" % (
         get_human_size(file_stat.stat.size),
         time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(file_stat.stat.mtime)),
-        get_echo_path(file_stat, base_path, full_path),
+        get_echo_path(file_stat, base_path, full),
     )
 
 
@@ -143,13 +148,12 @@ def _sftp_prompt_host_key(path):
             )
 
 
-def _ls(path: str, long: bool, recursive: bool, human_readable: bool):
+def _ls(path: str, long: bool, full: bool, recursive: bool, human_readable: bool):
     base_path = path
-    full_path = False
     if has_magic(path):
         scan_func = smart_glob_stat
         base_path = get_non_glob_dir(path)
-        full_path = True
+        full = True
     elif recursive:
         scan_func = smart_scan_stat
     else:
@@ -170,8 +174,8 @@ def _ls(path: str, long: bool, recursive: bool, human_readable: bool):
     for file_stat in scan_func(path):
         total_size += file_stat.stat.size
         total_count += 1
-        output = echo_func(file_stat, base_path, full_path=full_path)
-        if file_stat.is_symlink():
+        output = echo_func(file_stat, base_path, full=full)
+        if long and file_stat.is_symlink():
             output += " -> %s" % smart_readlink(file_stat.path)
         click.echo(output)
     if long:
@@ -215,6 +219,12 @@ ZshComplete.source_template = ZshComplete.source_template.replace(
     is_flag=True,
     help="List all the objects in the path with size, modification time and path.",
 )
+@click.option(
+    "-f",
+    "--full",
+    is_flag=True,
+    help="Displays the full path of each file.",
+)
 @click.option(
     "-r",
     "--recursive",
@@ -228,12 +238,24 @@ ZshComplete.source_template = ZshComplete.source_template.replace(
     is_flag=True,
     help="Displays file sizes in human readable format.",
 )
-def ls(path: str, long: bool, recursive: bool, human_readable: bool):
-    _ls(path, long=long, recursive=recursive, human_readable=human_readable)
+def ls(path: str, long: bool, full: bool, recursive: bool, human_readable: bool):
+    _ls(
+        path,
+        long=long,
+        full=full,
+        recursive=recursive,
+        human_readable=human_readable,
+    )
 
 
 @cli.command(short_help="List all the objects in the path.")
 @click.argument("path", type=PathType())
+@click.option(
+    "-f",
+    "--full",
+    is_flag=True,
+    help="Displays the full path of each file.",
+)
 @click.option(
     "-r",
     "--recursive",
@@ -241,8 +263,8 @@ def ls(path: str, long: bool, recursive: bool, human_readable: bool):
     help="Command is performed on all files or objects under "
     "the specified directory or prefix.",
 )
-def ll(path: str, recursive: bool):
-    _ls(path, long=True, recursive=recursive, human_readable=True)
+def ll(path: str, recursive: bool, full: bool):
+    _ls(path, long=True, full=full, recursive=recursive, human_readable=True)
 
 
 @cli.command(short_help="Copy files from source to dest, skipping already copied.")
@@ -342,8 +364,8 @@ def mv(
     _sftp_prompt_host_key(dst_path)
 
     if progress_bar:
-        src_protocol, _ = SmartPath._extract_protocol(src_path)
-        dst_protocol, _ = SmartPath._extract_protocol(dst_path)
+        src_protocol = SmartPath._extract_protocol(src_path)
+        dst_protocol = SmartPath._extract_protocol(dst_path)
 
         if recursive:
             if src_protocol == dst_protocol:
@@ -824,7 +846,7 @@ def hdfs(path, url, profile_name, user, root, token, timeout, no_cover):
         "timeout": timeout,
     }
     profile_name = f"{profile_name}.alias"
-    config = configparser.ConfigParser()
+    config = CaseSensitiveConfigParser()
     if os.path.exists(path):
         config.read(path)
     if "global" not in config.sections():
@@ -848,37 +870,58 @@ def hdfs(path, url, profile_name, user, root, token, timeout, no_cover):
 @click.option(
     "-p",
     "--path",
-    default="~/.config/megfile/aliases.conf",
-    help="alias config file, default is $HOME/.config/megfile/aliases.conf",
+    default=CONFIG_PATH,
+    help=f"megfile config file, default is {CONFIG_PATH}",
 )
 @click.argument("name")
 @click.argument("protocol_or_path")
 @click.option("--no-cover", is_flag=True, help="Not cover the same-name config")
 def alias(path, name, protocol_or_path, no_cover):
     path = os.path.expanduser(path)
-    config = configparser.ConfigParser()
+    config = CaseSensitiveConfigParser()
     if os.path.exists(path):
         config.read(path)
-    if name in config.sections() and no_cover:
-        raise NameError(f"alias-name has been used: {name}")
-
-    if "://" in protocol_or_path:
-        protocol, prefix = protocol_or_path.split("://", maxsplit=1)
-        config[name] = {
-            "protocol": protocol,
-            "prefix": prefix,
-        }
-    else:
-        config[name] = {
-            "protocol": protocol_or_path,
-        }
-
+    config.setdefault("alias", {})
+    if config.has_option("alias", name) and no_cover:
+        value = config.get("alias", name)
+        raise NameError(f"alias-name has been used: {name} = {value}")
+    config.set("alias", name, protocol_or_path)
     _safe_makedirs(os.path.dirname(path))  # make sure dirpath exist
     with open(path, "w") as fp:
         config.write(fp)
     click.echo(f"Your alias config has been saved into {path}")
 
 
+@config.command(short_help="Update the config file for envs")
+@click.option(
+    "-p",
+    "--path",
+    default=CONFIG_PATH,
+    help=f"megfile config file, default is {CONFIG_PATH}",
+)
+@click.argument("expr")
+@click.option("--no-cover", is_flag=True, help="Not cover the same-name config")
+def env(path, expr, no_cover):
+    if "=" not in expr:
+        raise ValueError("Invalid env format: {}".format(expr))
+    name, value = expr.split("=", 1)
+
+    path = os.path.expanduser(path)
+
+    config = CaseSensitiveConfigParser()
+    if os.path.exists(path):
+        config.read(path)
+    config.setdefault("env", {})
+    if config.has_option("env", name) and no_cover:
+        value = config.get("env", name)
+        raise NameError(f"env has been set: {name} = {value}")
+    config.set("env", name, value)
+    _safe_makedirs(os.path.dirname(path))  # make sure dirpath exist
+    with open(path, "w") as fp:
+        config.write(fp)
+    click.echo(f"Your env config has been saved into {path}")
+
+
 @cli.group(short_help="Return the completion file")
 def completion():
     pass

megfile/config.py

@@ -1,8 +1,14 @@
+import configparser
 import logging
 import os
 import typing as T
 
 
+class CaseSensitiveConfigParser(configparser.ConfigParser):
+    def optionxform(self, optionstr: str) -> str:
+        return optionstr
+
+
 def parse_quantity(quantity: T.Union[str, int]) -> int:
     """
     Parse kubernetes canonical form quantity like 200Mi to a int number.
@@ -75,6 +81,25 @@ def set_log_level(level: T.Optional[T.Union[int, str]] = None):
     logging.getLogger("megfile").setLevel(level)
 
 
+CONFIG_PATH = "~/.config/megfile/megfile.conf"
+
+
+def load_megfile_config(section) -> T.Dict[str, str]:
+    path = os.path.expanduser(CONFIG_PATH)
+    if not os.path.isfile(path):
+        return {}
+    config = CaseSensitiveConfigParser()
+    if os.path.exists(path):
+        config.read(path)
+    if not config.has_section(section):
+        return {}
+    return dict(config.items(section))
+
+
+for key, value in load_megfile_config("env").items():
+    os.environ.setdefault(key.upper(), value)
+
+
 READER_BLOCK_SIZE = parse_quantity(os.getenv("MEGFILE_READER_BLOCK_SIZE") or 8 * 2**20)
 if READER_BLOCK_SIZE <= 0:
     raise ValueError(
@@ -125,10 +150,19 @@ HDFS_MAX_RETRY_TIMES = int(
 SFTP_MAX_RETRY_TIMES = int(
     os.getenv("MEGFILE_SFTP_MAX_RETRY_TIMES") or DEFAULT_MAX_RETRY_TIMES
 )
+WEBDAV_MAX_RETRY_TIMES = int(
+    os.getenv("MEGFILE_WEBDAV_MAX_RETRY_TIMES") or DEFAULT_MAX_RETRY_TIMES
+)
 
 SFTP_HOST_KEY_POLICY = os.getenv("MEGFILE_SFTP_HOST_KEY_POLICY")
 
-HTTP_AUTH_HEADERS = ("Authorization", "Www-Authenticate", "Cookie", "Cookie2")
+HTTP_AUTH_HEADERS = (
+    "Authorization",
+    "Www-Authenticate",
+    "Cookie",
+    "Cookie2",
+    "X-Amz-Security-Token",
+)
 
 if os.getenv("MEGFILE_LOG_LEVEL"):
     set_log_level()

megfile/errors.py

@@ -50,8 +50,7 @@ _logger = getLogger(__name__)
 
 
 def s3_endpoint_url(path: Optional[PathLike] = None):
-    from megfile.s3 import get_endpoint_url, get_s3_client
-    from megfile.s3_path import S3Path
+    from megfile.s3_path import S3Path, get_endpoint_url, get_s3_client
 
     profile_name = None
     if path:
@@ -137,6 +136,7 @@ s3_retry_error_codes = (
     "UploadTrafficRateLimitExceeded",
     "MetaOperationQpsLimitExceeded",
     "TotalQpsLimitExceeded",
+    "PartitionQpsLimitted",
     "ActiveRequestLimitExceeded",
     "CpuLimitExceeded",
     "QpsLimitExceeded",