megfile 5.0.0__py3-none-any.whl → 5.0.2__py3-none-any.whl

megfile/__init__.py

@@ -1,4 +1,5 @@
-from megfile.fs_path import FSPath, is_fs
+import megfile.config  # noqa: F401  # make sure env config is loaded
+from megfile.fs_path import FSPath, fs_copy, is_fs
 from megfile.hdfs_path import HdfsPath, is_hdfs
 from megfile.http_path import HttpPath, HttpsPath, is_http
 from megfile.s3_path import (
@@ -6,12 +7,26 @@ from megfile.s3_path import (
     is_s3,
     s3_buffered_open,
     s3_cached_open,
+    s3_concat,
+    s3_copy,
+    s3_download,
+    s3_load_content,
     s3_memory_open,
+    s3_open,
     s3_pipe_open,
     s3_prefetch_open,
     s3_share_cache_open,
+    s3_upload,
+)
+from megfile.sftp_path import (
+    SftpPath,
+    is_sftp,
+    sftp_add_host_key,
+    sftp_concat,
+    sftp_copy,
+    sftp_download,
+    sftp_upload,
 )
-from megfile.sftp_path import SftpPath, is_sftp, sftp_add_host_key
 from megfile.smart import (
     smart_access,
     smart_cache,
@@ -71,9 +86,12 @@ except ImportError:
 __all__ = [
     "smart_access",
     "smart_cache",
+    "smart_cache",
     "smart_combine_open",
+    "smart_concat",
     "smart_copy",
     "smart_exists",
+    "smart_getmd5",
     "smart_getmtime",
     "smart_getsize",
     "smart_glob_stat",
@@ -84,54 +102,62 @@ __all__ = [
     "smart_islink",
     "smart_listdir",
     "smart_load_content",
-    "smart_save_content",
     "smart_load_from",
     "smart_load_text",
-    "smart_save_text",
+    "smart_lstat",
     "smart_makedirs",
+    "smart_move",
     "smart_open",
     "smart_path_join",
+    "smart_readlink",
     "smart_realpath",
     "smart_remove",
-    "smart_move",
     "smart_rename",
     "smart_save_as",
+    "smart_save_content",
+    "smart_save_text",
     "smart_scan_stat",
     "smart_scan",
     "smart_scandir",
     "smart_stat",
+    "smart_symlink",
     "smart_sync",
     "smart_touch",
     "smart_unlink",
     "smart_walk",
-    "smart_cache",
-    "smart_getmd5",
-    "smart_symlink",
-    "smart_readlink",
-    "smart_lstat",
-    "smart_concat",
+    "is_fs",
+    "fs_copy",
+    "is_hdfs",
+    "is_http",
     "is_s3",
     "s3_buffered_open",
     "s3_cached_open",
+    "s3_concat",
+    "s3_copy",
+    "s3_download",
+    "s3_load_content",
     "s3_memory_open",
+    "s3_open",
     "s3_pipe_open",
     "s3_prefetch_open",
     "s3_share_cache_open",
-    "is_fs",
-    "is_http",
-    "is_stdio",
-    "stdio_open",
+    "s3_upload",
     "is_sftp",
     "sftp_add_host_key",
-    "is_hdfs",
+    "sftp_concat",
+    "sftp_copy",
+    "sftp_download",
+    "sftp_upload",
+    "is_stdio",
+    "stdio_open",
     "is_webdav",
-    "WebdavPath",
-    "S3Path",
     "FSPath",
+    "HdfsPath",
     "HttpPath",
     "HttpsPath",
-    "StdioPath",
-    "SmartPath",
+    "S3Path",
     "SftpPath",
-    "HdfsPath",
+    "SmartPath",
+    "StdioPath",
+    "WebdavPath",
 ]

megfile/cli.py

@@ -1,4 +1,3 @@
-import configparser
 import os
 import shutil
 import signal
@@ -14,7 +13,13 @@ from click import ParamType
 from click.shell_completion import CompletionItem, ZshComplete
 from tqdm import tqdm
 
-from megfile.config import READER_BLOCK_SIZE, SFTP_HOST_KEY_POLICY, set_log_level
+from megfile.config import (
+    CONFIG_PATH,
+    READER_BLOCK_SIZE,
+    SFTP_HOST_KEY_POLICY,
+    CaseSensitiveConfigParser,
+    set_log_level,
+)
 from megfile.hdfs_path import DEFAULT_HDFS_TIMEOUT
 from megfile.interfaces import FileEntry
 from megfile.lib.glob import get_non_glob_dir, has_magic
@@ -841,7 +846,7 @@ def hdfs(path, url, profile_name, user, root, token, timeout, no_cover):
         "timeout": timeout,
     }
     profile_name = f"{profile_name}.alias"
-    config = configparser.ConfigParser()
+    config = CaseSensitiveConfigParser()
     if os.path.exists(path):
         config.read(path)
     if "global" not in config.sections():
@@ -865,37 +870,58 @@ def hdfs(path, url, profile_name, user, root, token, timeout, no_cover):
 @click.option(
     "-p",
     "--path",
-    default="~/.config/megfile/aliases.conf",
-    help="alias config file, default is $HOME/.config/megfile/aliases.conf",
+    default=CONFIG_PATH,
+    help=f"megfile config file, default is {CONFIG_PATH}",
 )
 @click.argument("name")
 @click.argument("protocol_or_path")
 @click.option("--no-cover", is_flag=True, help="Not cover the same-name config")
 def alias(path, name, protocol_or_path, no_cover):
     path = os.path.expanduser(path)
-    config = configparser.ConfigParser()
+    config = CaseSensitiveConfigParser()
     if os.path.exists(path):
         config.read(path)
-    if name in config.sections() and no_cover:
-        raise NameError(f"alias-name has been used: {name}")
-
-    if "://" in protocol_or_path:
-        protocol, prefix = protocol_or_path.split("://", maxsplit=1)
-        config[name] = {
-            "protocol": protocol,
-            "prefix": prefix,
-        }
-    else:
-        config[name] = {
-            "protocol": protocol_or_path,
-        }
-
+    config.setdefault("alias", {})
+    if config.has_option("alias", name) and no_cover:
+        value = config.get("alias", name)
+        raise NameError(f"alias-name has been used: {name} = {value}")
+    config.set("alias", name, protocol_or_path)
     _safe_makedirs(os.path.dirname(path))  # make sure dirpath exist
     with open(path, "w") as fp:
         config.write(fp)
     click.echo(f"Your alias config has been saved into {path}")
 
 
+@config.command(short_help="Update the config file for envs")
+@click.option(
+    "-p",
+    "--path",
+    default=CONFIG_PATH,
+    help=f"megfile config file, default is {CONFIG_PATH}",
+)
+@click.argument("expr")
+@click.option("--no-cover", is_flag=True, help="Not cover the same-name config")
+def env(path, expr, no_cover):
+    if "=" not in expr:
+        raise ValueError("Invalid env format: {}".format(expr))
+    name, value = expr.split("=", 1)
+
+    path = os.path.expanduser(path)
+
+    config = CaseSensitiveConfigParser()
+    if os.path.exists(path):
+        config.read(path)
+    config.setdefault("env", {})
+    if config.has_option("env", name) and no_cover:
+        value = config.get("env", name)
+        raise NameError(f"env has been set: {name} = {value}")
+    config.set("env", name, value)
+    _safe_makedirs(os.path.dirname(path))  # make sure dirpath exist
+    with open(path, "w") as fp:
+        config.write(fp)
+    click.echo(f"Your env config has been saved into {path}")
+
+
 @cli.group(short_help="Return the completion file")
 def completion():
     pass

megfile/config.py

@@ -1,8 +1,14 @@
+import configparser
 import logging
 import os
 import typing as T
 
 
+class CaseSensitiveConfigParser(configparser.ConfigParser):
+    def optionxform(self, optionstr: str) -> str:
+        return optionstr
+
+
 def parse_quantity(quantity: T.Union[str, int]) -> int:
     """
     Parse kubernetes canonical form quantity like 200Mi to a int number.
@@ -75,6 +81,25 @@ def set_log_level(level: T.Optional[T.Union[int, str]] = None):
     logging.getLogger("megfile").setLevel(level)
 
 
+CONFIG_PATH = "~/.config/megfile/megfile.conf"
+
+
+def load_megfile_config(section) -> T.Dict[str, str]:
+    path = os.path.expanduser(CONFIG_PATH)
+    if not os.path.isfile(path):
+        return {}
+    config = CaseSensitiveConfigParser()
+    if os.path.exists(path):
+        config.read(path)
+    if not config.has_section(section):
+        return {}
+    return dict(config.items(section))
+
+
+for key, value in load_megfile_config("env").items():
+    os.environ.setdefault(key.upper(), value)
+
+
 READER_BLOCK_SIZE = parse_quantity(os.getenv("MEGFILE_READER_BLOCK_SIZE") or 8 * 2**20)
 if READER_BLOCK_SIZE <= 0:
     raise ValueError(

megfile/lib/webdav_memory_handler.py

@@ -19,7 +19,7 @@ def _webdav_stat(client: WebdavClient, remote_path: str):
     info = WebDavXmlUtils.parse_info_response(
         response.content, path, client.webdav.hostname
     )
-    info["is_dir"] = WebDavXmlUtils.parse_is_dir_response(
+    info["isdir"] = WebDavXmlUtils.parse_is_dir_response(
         response.content, path, client.webdav.hostname
     )
     return info
@@ -60,7 +60,7 @@ class WebdavMemoryHandler(BaseMemoryHandler):
 
     def _file_exists(self) -> bool:
         try:
-            return not _webdav_stat(self._client, self._remote_path)["is_dir"]
+            return not _webdav_stat(self._client, self._remote_path)["isdir"]
         except RemoteResourceNotFound:
             return False
 

megfile/smart_path.py

@@ -1,18 +1,19 @@
 import os
-from configparser import ConfigParser
 from functools import cached_property
 from pathlib import PurePath
 from typing import Dict, Optional, Tuple, Union
 
+from megfile.config import load_megfile_config
 from megfile.lib.compat import fspath
 from megfile.lib.url import get_url_scheme
 from megfile.pathlike import URIPathParents
 from megfile.utils import cached_classproperty
 
+from .config import CaseSensitiveConfigParser
 from .errors import ProtocolExistsError, ProtocolNotFoundError
 from .interfaces import BasePath, PathLike
 
-aliases_config = "~/.config/megfile/aliases.conf"
+LEGACY_ALIASES_CONFIG = "~/.config/megfile/aliases.conf"
 
 
 def _bind_function(name, after_callback=None, before_callback=None):
@@ -54,14 +55,20 @@ def _bind_property(name, callback=None):
     return smart_property
 
 
-def _load_aliases_config(config_path) -> Dict[str, Dict[str, str]]:
-    if not os.path.exists(config_path):
-        return {}
-    parser = ConfigParser()
-    parser.read(config_path)
+def _load_aliases_config() -> Dict[str, Dict[str, str]]:
     configs = {}
-    for section in parser.sections():
-        configs[section] = dict(parser.items(section))
+    config_path = os.path.expanduser(LEGACY_ALIASES_CONFIG)
+    if os.path.isfile(config_path):
+        parser = CaseSensitiveConfigParser()
+        parser.read(config_path)
+        for section in parser.sections():
+            configs[section] = dict(parser.items(section))
+    for name, protocol_or_path in load_megfile_config("alias").items():
+        if "://" in protocol_or_path:
+            protocol, prefix = protocol_or_path.split("://", maxsplit=1)
+            configs[name] = {"protocol": protocol, "prefix": prefix}
+        else:
+            configs[name] = {"protocol": protocol_or_path}
     return configs
 
 
@@ -151,8 +158,7 @@ class SmartPath(BasePath):
 
     @cached_classproperty
     def _aliases(cls) -> Dict[str, Dict[str, str]]:
-        config_path = os.path.expanduser(aliases_config)
-        return _load_aliases_config(config_path)
+        return _load_aliases_config()
 
     @classmethod
     def _split_protocol(cls, path: Union[PathLike, int]) -> Tuple[str, Union[str, int]]:

megfile/version.py

@@ -1 +1 @@
-VERSION = "5.0.0"
+VERSION = "5.0.2"

megfile/webdav_path.py

@@ -78,7 +78,7 @@ def _make_stat(info: dict) -> StatResult:
     except Exception:
         mtime = 0.0
 
-    isdir = info.get("is_dir", False)
+    isdir = info.get("isdir", False)
 
     return StatResult(
         size=size,
@@ -144,6 +144,7 @@ def _patch_execute_request(
         cmds = shlex.split(client.webdav.token_command)
         client.webdav.token_command_last_call = time.time()
         client.webdav.token = subprocess.check_output(cmds).decode().strip()
+        _logger.debug("update webdav token by command: %s", client.webdav.token_command)
 
     def webdav_should_retry(error: Exception) -> bool:
         if http_should_retry(error):
@@ -457,7 +458,7 @@ class WebdavPath(URIPath):
         :returns: True if the path is a directory, else False
         """
         try:
-            return _webdav_stat(self._client, self._remote_path)["is_dir"]
+            return _webdav_stat(self._client, self._remote_path)["isdir"]
         except RemoteResourceNotFound:
             return False
 
@@ -469,7 +470,7 @@ class WebdavPath(URIPath):
         :returns: True if the path is a file, else False
         """
         try:
-            return not _webdav_stat(self._client, self._remote_path)["is_dir"]
+            return not _webdav_stat(self._client, self._remote_path)["isdir"]
         except RemoteResourceNotFound:
             return False
 

{megfile-5.0.0.dist-info → megfile-5.0.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: megfile
-Version: 5.0.0
+Version: 5.0.2
 Summary: Megvii file operation library
 Author-email: megvii-reng <megvii-reng@googlegroups.com>
 Project-URL: Homepage, https://github.com/megvii-research/megfile

{megfile-5.0.0.dist-info → megfile-5.0.2.dist-info}/RECORD

@@ -1,6 +1,6 @@
-megfile/__init__.py,sha256=2QtWAC9XFlKDhY0NTHsCJd2wQ-8LKz0q1ng4-8cXo38,2836
-megfile/cli.py,sha256=yRrnXLYejCz1928JCjjiqOe50YH8lD1xuwaiej_67Ng,29352
-megfile/config.py,sha256=r_QmInU_gFiSTV1nLJ5SyBVCX5bsANqOoNM82bRESnQ,4414
+megfile/__init__.py,sha256=4XbMsR-lM7MxbnPGBI16m2sq6ghuA2-zZj2XF4bbX2Y,3291
+megfile/cli.py,sha256=HdOIAMFXdZw79kayVIri5-_rbq_KwOGSdMYTx51ABp0,30204
+megfile/config.py,sha256=K3B_o2dnI7qGsGnK8Jg18-S5YYLYuzskfNJowlSMkQM,5065
 megfile/errors.py,sha256=eC7z-2-QqE12pYgTGAOIlEBsemqcMyeBFVUEDz3gBS0,15585
 megfile/fs_path.py,sha256=tt2__W6E4vep0lmVreTLIW63njl-EzyQEEkEGziyAb4,41015
 megfile/hdfs_path.py,sha256=OmUe3vA3Qoxnqtcq0Rs3ygBvzAtqUz3fGo8iP5sWneE,26058
@@ -11,10 +11,10 @@ megfile/s3_path.py,sha256=LINHnHnpesXnf9wxbV6n0xQVT0wPwyjLc7xAasakefU,94467
 megfile/sftp2_path.py,sha256=K90bnMVAx0MQPGXP6LogGuDRzaD4MPR6lMOfdY9C9-0,37942
 megfile/sftp_path.py,sha256=_KU7_-Mq2m7lcLY1mpiGrju0SP-OsdEXlRjFhZH25UA,51223
 megfile/smart.py,sha256=Lab2jxprj-zvPw5GqUWlWiEY8bcpRlviks_qp9r-km8,38224
-megfile/smart_path.py,sha256=cI8LSD1M7p-pHFJ-yDIgen8g9FyfbfDLrDfQwkkKT-Q,12240
+megfile/smart_path.py,sha256=kGidkM5S58ChE3LVZMcUACs3IQgsqh9m04sp6-wxuhk,12615
 megfile/stdio_path.py,sha256=cxaDr8rtisTPnN-rjtaEpqQnshwiqwXFUJBM9xWY7Cg,2711
-megfile/version.py,sha256=sWZ_cOSUdsW9iQos7P4NME_qRop6_sbMdttJcWoYa7o,19
-megfile/webdav_path.py,sha256=96rt3jN_XfDXY0UYC_Kufrz5Uc5KySo34QnlVbDIQK8,31278
+megfile/version.py,sha256=2z_YV6M_rFsVnaPW0uk4dTnt4IynxvGR09iBXCfO_Po,19
+megfile/webdav_path.py,sha256=xQmZMt-hDA7PfHzuSjRYaIoJA_Nbi1jsg952KZJhs-E,31364
 megfile/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 megfile/lib/base_memory_handler.py,sha256=i5-YHpL0k2tuFBnP9VMzb3_OsaU6D6j6thkmWgptnFg,2680
 megfile/lib/base_prefetch_reader.py,sha256=MYaWOkXc3geZMYNPHlPZmmpOR6uSz-AMuCZwYdoz7t0,13296
@@ -38,14 +38,14 @@ megfile/lib/s3_share_cache_reader.py,sha256=8uip5IdVjPXCquXrskjocsZx2-TiXqWZPY0g
 megfile/lib/shadow_handler.py,sha256=TntewlvIW9ZxCfmqASDQREHoiZ8v42faOe9sovQYQz0,2779
 megfile/lib/stdio_handler.py,sha256=IDdgENLQlhigEwkLL4zStueVSzdWg7xVcTF_koof_Ek,1987
 megfile/lib/url.py,sha256=ER32pWy9Q2MAk3TraAaNEBWIqUeBmLuM57ol2cs7-Ks,103
-megfile/lib/webdav_memory_handler.py,sha256=KDSU-Z3ECiH5p6mo8TsbqHxjSBCZNx-LtFW9nIimdOY,2508
+megfile/lib/webdav_memory_handler.py,sha256=_UccPYPpvfTd4gSEhBFL1BHeyFtsBJdhVINkjNNtyaw,2506
 megfile/lib/webdav_prefetch_reader.py,sha256=M0X6E6t-DS5q9KiLvjVZx_AZuiW9SaIkBnIPLc774GQ,3941
 megfile/utils/__init__.py,sha256=4hBVSXbNTbDj7Je0y9SbwgcPm_s41H9v3eHUMr9JNGo,12700
 megfile/utils/mutex.py,sha256=asb8opGLgK22RiuBJUnfsvB8LnMmodP8KzCVHKmQBWA,2561
-megfile-5.0.0.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-megfile-5.0.0.dist-info/licenses/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
-megfile-5.0.0.dist-info/METADATA,sha256=xwRH2pja42oQN9jJHaiTsFeKO0Ib8Ih4Thvi2cyNns0,9225
-megfile-5.0.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-megfile-5.0.0.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
-megfile-5.0.0.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
-megfile-5.0.0.dist-info/RECORD,,
+megfile-5.0.2.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+megfile-5.0.2.dist-info/licenses/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
+megfile-5.0.2.dist-info/METADATA,sha256=TvjBMScDXQdbke5lhUufQg2B-KY7XywLn-VpQkfQsfw,9225
+megfile-5.0.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+megfile-5.0.2.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
+megfile-5.0.2.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
+megfile-5.0.2.dist-info/RECORD,,