megfile 4.2.4__py3-none-any.whl → 5.0.0__py3-none-any.whl

megfile/__init__.py

@@ -1,164 +1,17 @@
-from megfile.fs import (
-    fs_abspath,
-    fs_access,
-    fs_cwd,
-    fs_exists,
-    fs_expanduser,
-    fs_getmd5,
-    fs_getmtime,
-    fs_getsize,
-    fs_glob,
-    fs_glob_stat,
-    fs_home,
-    fs_iglob,
-    fs_isabs,
-    fs_isdir,
-    fs_isfile,
-    fs_islink,
-    fs_ismount,
-    fs_listdir,
-    fs_load_from,
-    fs_lstat,
-    fs_makedirs,
-    fs_move,
-    fs_readlink,
-    fs_realpath,
-    fs_relpath,
-    fs_remove,
-    fs_rename,
-    fs_resolve,
-    fs_save_as,
-    fs_scan,
-    fs_scan_stat,
-    fs_scandir,
-    fs_stat,
-    fs_symlink,
-    fs_sync,
-    fs_unlink,
-    fs_walk,
-    is_fs,
-)
-from megfile.fs_path import FSPath
-from megfile.hdfs import (
-    hdfs_exists,
-    hdfs_getmd5,
-    hdfs_getmtime,
-    hdfs_getsize,
-    hdfs_glob,
-    hdfs_glob_stat,
-    hdfs_iglob,
-    hdfs_isdir,
-    hdfs_isfile,
-    hdfs_listdir,
-    hdfs_load_from,
-    hdfs_makedirs,
-    hdfs_move,
-    hdfs_open,
-    hdfs_remove,
-    hdfs_save_as,
-    hdfs_scan,
-    hdfs_scan_stat,
-    hdfs_scandir,
-    hdfs_stat,
-    hdfs_unlink,
-    hdfs_walk,
-    is_hdfs,
-)
-from megfile.hdfs_path import HdfsPath
-from megfile.http import (
-    http_exists,
-    http_getmtime,
-    http_getsize,
-    http_open,
-    http_stat,
-    is_http,
-)
-from megfile.http_path import HttpPath, HttpsPath
-from megfile.s3 import (
+from megfile.fs_path import FSPath, is_fs
+from megfile.hdfs_path import HdfsPath, is_hdfs
+from megfile.http_path import HttpPath, HttpsPath, is_http
+from megfile.s3_path import (
+    S3Path,
     is_s3,
-    s3_access,
     s3_buffered_open,
     s3_cached_open,
-    s3_concat,
-    s3_copy,
-    s3_download,
-    s3_exists,
-    s3_getmd5,
-    s3_getmtime,
-    s3_getsize,
-    s3_glob,
-    s3_glob_stat,
-    s3_hasbucket,
-    s3_iglob,
-    s3_isdir,
-    s3_isfile,
-    s3_listdir,
-    s3_load_content,
-    s3_load_from,
-    s3_lstat,
-    s3_makedirs,
     s3_memory_open,
-    s3_move,
-    s3_open,
-    s3_path_join,
     s3_pipe_open,
     s3_prefetch_open,
-    s3_readlink,
-    s3_remove,
-    s3_rename,
-    s3_save_as,
-    s3_scan,
-    s3_scan_stat,
-    s3_scandir,
-    s3_stat,
-    s3_symlink,
-    s3_sync,
-    s3_unlink,
-    s3_upload,
-    s3_walk,
-)
-from megfile.s3_path import S3Path
-from megfile.sftp import (
-    is_sftp,
-    sftp_absolute,
-    sftp_add_host_key,
-    sftp_chmod,
-    sftp_concat,
-    sftp_copy,
-    sftp_exists,
-    sftp_getmd5,
-    sftp_getmtime,
-    sftp_getsize,
-    sftp_glob,
-    sftp_glob_stat,
-    sftp_iglob,
-    sftp_isdir,
-    sftp_isfile,
-    sftp_islink,
-    sftp_listdir,
-    sftp_load_from,
-    sftp_lstat,
-    sftp_makedirs,
-    sftp_move,
-    sftp_open,
-    sftp_path_join,
-    sftp_readlink,
-    sftp_realpath,
-    sftp_remove,
-    sftp_rename,
-    sftp_resolve,
-    sftp_rmdir,
-    sftp_save_as,
-    sftp_scan,
-    sftp_scan_stat,
-    sftp_scandir,
-    sftp_stat,
-    sftp_symlink,
-    sftp_sync,
-    sftp_unlink,
-    sftp_walk,
+    s3_share_cache_open,
 )
-from megfile.sftp_path import SftpPath
+from megfile.sftp_path import SftpPath, is_sftp, sftp_add_host_key
 from megfile.smart import (
     smart_access,
     smart_cache,
@@ -202,8 +55,7 @@ from megfile.smart import (
     smart_walk,
 )
 from megfile.smart_path import SmartPath
-from megfile.stdio import is_stdio, stdio_open
-from megfile.stdio_path import StdioPath
+from megfile.stdio_path import StdioPath, is_stdio
 from megfile.version import VERSION as __version__  # noqa: F401
 
 try:
@@ -211,6 +63,11 @@ try:
 except ImportError:
     Sftp2Path = None
 
+try:
+    from megfile.webdav_path import WebdavPath, is_webdav
+except ImportError:
+    WebdavPath = is_webdav = None
+
 __all__ = [
     "smart_access",
     "smart_cache",
@@ -254,153 +111,21 @@ __all__ = [
     "smart_lstat",
     "smart_concat",
     "is_s3",
-    "s3_access",
     "s3_buffered_open",
     "s3_cached_open",
-    "s3_copy",
-    "s3_download",
-    "s3_exists",
-    "s3_getmd5",
-    "s3_getmtime",
-    "s3_getsize",
-    "s3_glob_stat",
-    "s3_glob",
-    "s3_hasbucket",
-    "s3_iglob",
-    "s3_isdir",
-    "s3_isfile",
-    "s3_listdir",
-    "s3_load_content",
-    "s3_load_from",
-    "s3_makedirs",
     "s3_memory_open",
-    "s3_open",
-    "s3_path_join",
     "s3_pipe_open",
     "s3_prefetch_open",
-    "s3_remove",
-    "s3_rename",
-    "s3_move",
-    "s3_sync",
-    "s3_save_as",
-    "s3_scan_stat",
-    "s3_scan",
-    "s3_scandir",
-    "s3_stat",
-    "s3_lstat",
-    "s3_unlink",
-    "s3_upload",
-    "s3_walk",
-    "s3_symlink",
-    "s3_readlink",
-    "s3_concat",
+    "s3_share_cache_open",
     "is_fs",
-    "fs_abspath",
-    "fs_access",
-    "fs_exists",
-    "fs_getmtime",
-    "fs_getsize",
-    "fs_glob_stat",
-    "fs_glob",
-    "fs_iglob",
-    "fs_isabs",
-    "fs_isdir",
-    "fs_isfile",
-    "fs_islink",
-    "fs_ismount",
-    "fs_listdir",
-    "fs_load_from",
-    "fs_makedirs",
-    "fs_realpath",
-    "fs_relpath",
-    "fs_remove",
-    "fs_rename",
-    "fs_move",
-    "fs_sync",
-    "fs_save_as",
-    "fs_scan_stat",
-    "fs_scan",
-    "fs_scandir",
-    "fs_stat",
-    "fs_lstat",
-    "fs_unlink",
-    "fs_walk",
-    "fs_cwd",
-    "fs_home",
-    "fs_expanduser",
-    "fs_resolve",
-    "fs_getmd5",
-    "fs_symlink",
-    "fs_readlink",
     "is_http",
-    "http_open",
-    "http_stat",
-    "http_getsize",
-    "http_getmtime",
-    "http_exists",
     "is_stdio",
     "stdio_open",
     "is_sftp",
-    "sftp_readlink",
-    "sftp_absolute",
-    "sftp_glob",
-    "sftp_iglob",
-    "sftp_glob_stat",
-    "sftp_resolve",
-    "sftp_isdir",
-    "sftp_exists",
-    "sftp_scandir",
-    "sftp_getmtime",
-    "sftp_getsize",
-    "sftp_isfile",
-    "sftp_listdir",
-    "sftp_load_from",
-    "sftp_makedirs",
-    "sftp_realpath",
-    "sftp_rename",
-    "sftp_move",
-    "sftp_remove",
-    "sftp_scan",
-    "sftp_scan_stat",
-    "sftp_stat",
-    "sftp_lstat",
-    "sftp_unlink",
-    "sftp_walk",
-    "sftp_path_join",
-    "sftp_getmd5",
-    "sftp_symlink",
-    "sftp_islink",
-    "sftp_save_as",
-    "sftp_open",
-    "sftp_chmod",
-    "sftp_rmdir",
-    "sftp_copy",
-    "sftp_sync",
-    "sftp_concat",
     "sftp_add_host_key",
     "is_hdfs",
-    "hdfs_exists",
-    "hdfs_stat",
-    "hdfs_getmtime",
-    "hdfs_getsize",
-    "hdfs_isdir",
-    "hdfs_isfile",
-    "hdfs_listdir",
-    "hdfs_load_from",
-    "hdfs_move",
-    "hdfs_remove",
-    "hdfs_scan",
-    "hdfs_scan_stat",
-    "hdfs_scandir",
-    "hdfs_unlink",
-    "hdfs_walk",
-    "hdfs_getmd5",
-    "hdfs_save_as",
-    "hdfs_open",
-    "hdfs_glob",
-    "hdfs_glob_stat",
-    "hdfs_iglob",
-    "hdfs_makedirs",
+    "is_webdav",
+    "WebdavPath",
     "S3Path",
     "FSPath",
     "HttpPath",

megfile/cli.py

@@ -19,7 +19,7 @@ from megfile.hdfs_path import DEFAULT_HDFS_TIMEOUT
 from megfile.interfaces import FileEntry
 from megfile.lib.glob import get_non_glob_dir, has_magic
 from megfile.s3_path import get_s3_session
-from megfile.sftp import sftp_add_host_key
+from megfile.sftp_path import sftp_add_host_key
 from megfile.smart import (
     _smart_sync_single_file,
     smart_copy,
@@ -86,33 +86,33 @@ def safe_cli():  # pragma: no cover
             sys.exit(1)
 
 
-def get_echo_path(file_stat, base_path: str = "", full_path: bool = False):
+def get_echo_path(file_stat, base_path: str = "", full: bool = False):
     if base_path == file_stat.path:
         path = file_stat.name
-    elif full_path:
+    elif full:
         path = file_stat.path
     else:
         path = smart_relpath(file_stat.path, start=base_path)
     return path
 
 
-def simple_echo(file_stat, base_path: str = "", full_path: bool = False):
-    return get_echo_path(file_stat, base_path, full_path)
+def simple_echo(file_stat, base_path: str = "", full: bool = False):
+    return get_echo_path(file_stat, base_path, full)
 
 
-def long_echo(file_stat, base_path: str = "", full_path: bool = False):
+def long_echo(file_stat, base_path: str = "", full: bool = False):
     return "%12d %s %s" % (
         file_stat.stat.size,
         time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(file_stat.stat.mtime)),
-        get_echo_path(file_stat, base_path, full_path),
+        get_echo_path(file_stat, base_path, full),
     )
 
 
-def human_echo(file_stat, base_path: str = "", full_path: bool = False):
+def human_echo(file_stat, base_path: str = "", full: bool = False):
     return "%10s %s %s" % (
         get_human_size(file_stat.stat.size),
         time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(file_stat.stat.mtime)),
-        get_echo_path(file_stat, base_path, full_path),
+        get_echo_path(file_stat, base_path, full),
     )
 
 
@@ -143,13 +143,12 @@ def _sftp_prompt_host_key(path):
             )
 
 
-def _ls(path: str, long: bool, recursive: bool, human_readable: bool):
+def _ls(path: str, long: bool, full: bool, recursive: bool, human_readable: bool):
     base_path = path
-    full_path = False
     if has_magic(path):
         scan_func = smart_glob_stat
         base_path = get_non_glob_dir(path)
-        full_path = True
+        full = True
     elif recursive:
         scan_func = smart_scan_stat
     else:
@@ -170,8 +169,8 @@ def _ls(path: str, long: bool, recursive: bool, human_readable: bool):
     for file_stat in scan_func(path):
         total_size += file_stat.stat.size
         total_count += 1
-        output = echo_func(file_stat, base_path, full_path=full_path)
-        if file_stat.is_symlink():
+        output = echo_func(file_stat, base_path, full=full)
+        if long and file_stat.is_symlink():
             output += " -> %s" % smart_readlink(file_stat.path)
         click.echo(output)
     if long:
@@ -215,6 +214,12 @@ ZshComplete.source_template = ZshComplete.source_template.replace(
     is_flag=True,
     help="List all the objects in the path with size, modification time and path.",
 )
+@click.option(
+    "-f",
+    "--full",
+    is_flag=True,
+    help="Displays the full path of each file.",
+)
 @click.option(
     "-r",
     "--recursive",
@@ -228,12 +233,24 @@ ZshComplete.source_template = ZshComplete.source_template.replace(
     is_flag=True,
     help="Displays file sizes in human readable format.",
 )
-def ls(path: str, long: bool, recursive: bool, human_readable: bool):
-    _ls(path, long=long, recursive=recursive, human_readable=human_readable)
+def ls(path: str, long: bool, full: bool, recursive: bool, human_readable: bool):
+    _ls(
+        path,
+        long=long,
+        full=full,
+        recursive=recursive,
+        human_readable=human_readable,
+    )
 
 
 @cli.command(short_help="List all the objects in the path.")
 @click.argument("path", type=PathType())
+@click.option(
+    "-f",
+    "--full",
+    is_flag=True,
+    help="Displays the full path of each file.",
+)
 @click.option(
     "-r",
     "--recursive",
@@ -241,8 +258,8 @@ def ls(path: str, long: bool, recursive: bool, human_readable: bool):
     help="Command is performed on all files or objects under "
     "the specified directory or prefix.",
 )
-def ll(path: str, recursive: bool):
-    _ls(path, long=True, recursive=recursive, human_readable=True)
+def ll(path: str, recursive: bool, full: bool):
+    _ls(path, long=True, full=full, recursive=recursive, human_readable=True)
 
 
 @cli.command(short_help="Copy files from source to dest, skipping already copied.")
@@ -342,8 +359,8 @@ def mv(
     _sftp_prompt_host_key(dst_path)
 
     if progress_bar:
-        src_protocol, _ = SmartPath._extract_protocol(src_path)
-        dst_protocol, _ = SmartPath._extract_protocol(dst_path)
+        src_protocol = SmartPath._extract_protocol(src_path)
+        dst_protocol = SmartPath._extract_protocol(dst_path)
 
         if recursive:
             if src_protocol == dst_protocol:

megfile/config.py

@@ -125,10 +125,19 @@ HDFS_MAX_RETRY_TIMES = int(
 SFTP_MAX_RETRY_TIMES = int(
     os.getenv("MEGFILE_SFTP_MAX_RETRY_TIMES") or DEFAULT_MAX_RETRY_TIMES
 )
+WEBDAV_MAX_RETRY_TIMES = int(
+    os.getenv("MEGFILE_WEBDAV_MAX_RETRY_TIMES") or DEFAULT_MAX_RETRY_TIMES
+)
 
 SFTP_HOST_KEY_POLICY = os.getenv("MEGFILE_SFTP_HOST_KEY_POLICY")
 
-HTTP_AUTH_HEADERS = ("Authorization", "Www-Authenticate", "Cookie", "Cookie2")
+HTTP_AUTH_HEADERS = (
+    "Authorization",
+    "Www-Authenticate",
+    "Cookie",
+    "Cookie2",
+    "X-Amz-Security-Token",
+)
 
 if os.getenv("MEGFILE_LOG_LEVEL"):
     set_log_level()

megfile/errors.py

@@ -50,8 +50,7 @@ _logger = getLogger(__name__)
 
 
 def s3_endpoint_url(path: Optional[PathLike] = None):
-    from megfile.s3 import get_endpoint_url, get_s3_client
-    from megfile.s3_path import S3Path
+    from megfile.s3_path import S3Path, get_endpoint_url, get_s3_client
 
     profile_name = None
     if path:
@@ -137,6 +136,7 @@ s3_retry_error_codes = (
     "UploadTrafficRateLimitExceeded",
     "MetaOperationQpsLimitExceeded",
     "TotalQpsLimitExceeded",
+    "PartitionQpsLimitted",
     "ActiveRequestLimitExceeded",
     "CpuLimitExceeded",
     "QpsLimitExceeded",

megfile/fs_path.py

@@ -17,6 +17,7 @@ from megfile.interfaces import (
     Access,
     ContextIterator,
     FileEntry,
+    FileLike,
     PathLike,
     StatResult,
     URIPath,
@@ -32,7 +33,7 @@ from megfile.utils import calculate_md5, copyfd
 __all__ = [
     "FSPath",
     "is_fs",
-    "fs_path_join",
+    "fs_copy",
 ]
 
 
@@ -60,8 +61,37 @@ def is_fs(path: Union["PathLike", int]) -> bool:
     return scheme == "" or scheme == "file"
 
 
-def fs_path_join(path: PathLike, *other_paths: PathLike) -> str:
-    return path_join(fspath(path), *map(fspath, other_paths))
+def fs_copy(
+    src_path: PathLike,
+    dst_path: PathLike,
+    callback: Optional[Callable[[int], None]] = None,
+    followlinks: bool = False,
+    overwrite: bool = True,
+):
+    """File copy on file system
+    Copy content (excluding meta date) of file on `src_path` to `dst_path`.
+    `dst_path` must be a complete file name
+
+    .. note ::
+
+        The differences between this function and shutil.copyfile are:
+
+            1. If parent directory of dst_path doesn't exist, create it
+
+            2. Allow callback function, None by default.
+                callback: Optional[Callable[[int], None]], the int data is means
+                the size (in bytes) of the written data that is passed periodically
+
+            3. This function is thread-unsafe
+
+    :param src_path: Given path
+    :param dst_path: Target file path
+    :param callback: Called periodically during copy, and the input parameter is
+        the data size (in bytes) of copy since the last call
+    :param followlinks: False if regard symlink as file, else True
+    :param overwrite: whether or not overwrite file when exists, default is True
+    """
+    return FSPath(src_path).copy(dst_path, callback, followlinks, overwrite)
 
 
 def _fs_rename_file(
@@ -85,6 +115,36 @@ def _fs_rename_file(
     shutil.move(src_path, dst_path)
 
 
+class WrapAtomic(FileLike):
+    __atomic__ = True
+
+    def __init__(self, fileobj):
+        self.fileobj = fileobj
+        self.temp_name = f"{self.name}.temp"
+        os.rename(self.name, self.temp_name)
+
+    @property
+    def name(self):
+        return self.fileobj.name
+
+    @property
+    def mode(self):
+        return self.fileobj.mode
+
+    def _close(self):
+        self.fileobj.close()
+        os.rename(self.temp_name, self.name)
+
+    def _abort(self):
+        try:
+            os.unlink(self.temp_name)
+        except FileNotFoundError:
+            pass
+
+    def __getattr__(self, name: str):
+        return getattr(self.fileobj, name)
+
+
 @SmartPath.register
 class FSPath(URIPath):
     """file protocol
@@ -627,9 +687,11 @@ class FSPath(URIPath):
         """
         self._check_int_path()
 
-        if missing_ok and not self.exists():
-            return
-        os.unlink(self.path_without_protocol)  # pyre-ignore[6]
+        try:
+            os.unlink(self.path_without_protocol)  # pyre-ignore[6]
+        except FileNotFoundError:
+            if not missing_ok:
+                raise
 
     def walk(
         self, followlinks: bool = False
@@ -917,11 +979,12 @@ class FSPath(URIPath):
     def open(
         self,
         mode: str = "r",
-        buffering=-1,
-        encoding=None,
-        errors=None,
-        newline=None,
-        closefd=True,
+        buffering: int = -1,
+        encoding: Optional[str] = None,
+        errors: Optional[str] = None,
+        newline: Optional[str] = None,
+        closefd: bool = True,
+        atomic: bool = False,
         **kwargs,
     ) -> IO:
         if not isinstance(self.path_without_protocol, int) and (
@@ -932,7 +995,7 @@ class FSPath(URIPath):
                     self.path_without_protocol  # pyre-ignore[6]
                 )
             ).mkdir(parents=True, exist_ok=True)
-        return io.open(
+        fp = io.open(
             self.path_without_protocol,
             mode,
             buffering=buffering,
@@ -941,6 +1004,9 @@ class FSPath(URIPath):
             newline=newline,
             closefd=closefd,
         )
+        if atomic and ("w" in mode or "x" in mode or "a" in mode):
+            return WrapAtomic(fp)
+        return fp
 
     @cached_property
     def parts(self) -> Tuple[str, ...]: