megfile 3.1.6__py3-none-any.whl → 4.0.0__py3-none-any.whl

megfile/cli.py

@@ -10,7 +10,7 @@ from functools import partial
 import click
 from tqdm import tqdm
 
-from megfile.config import DEFAULT_BLOCK_SIZE
+from megfile.config import READER_BLOCK_SIZE
 from megfile.hdfs_path import DEFAULT_HDFS_TIMEOUT
 from megfile.interfaces import FileEntry
 from megfile.lib.glob import get_non_glob_dir, has_magic
@@ -349,6 +349,9 @@ def sync(
     quiet: bool,
     skip: bool,
 ):
+    if not smart_exists(dst_path):
+        force = True
+
     with ThreadPoolExecutor(max_workers=worker) as executor:
         if has_magic(src_path):
             src_root_path = get_non_glob_dir(src_path)
@@ -411,7 +414,7 @@ def sync(
             dict(
                 src_root_path=src_root_path,
                 dst_root_path=dst_path,
-                src_file_path=file_entry.path,
+                src_file_entry=file_entry,
                 callback=callback,
                 followlinks=True,
                 callback_after_copy_file=callback_after_copy_file,
@@ -484,11 +487,11 @@ def tail(path: str, lines: int, follow: bool):
         f.seek(0, os.SEEK_SET)
 
         for current_offset in range(
-            file_size - DEFAULT_BLOCK_SIZE, 0 - DEFAULT_BLOCK_SIZE, -DEFAULT_BLOCK_SIZE
+            file_size - READER_BLOCK_SIZE, 0 - READER_BLOCK_SIZE, -READER_BLOCK_SIZE
         ):
             current_offset = max(0, current_offset)
             f.seek(current_offset)
-            block_lines = f.read(DEFAULT_BLOCK_SIZE).split(b"\n")
+            block_lines = f.read(READER_BLOCK_SIZE).split(b"\n")
             if len(line_list) > 0:
                 block_lines[-1] += line_list[0]
                 block_lines.extend(line_list[1:])
@@ -524,9 +527,11 @@ def to(path: str, append: bool, stdout: bool):
     mode = "wb"
     if append:
         mode = "ab"
-    with smart_open("stdio://0", "rb") as stdin, smart_open(
-        path, mode
-    ) as f, smart_open("stdio://1", "wb") as stdout_fd:
+    with (
+        smart_open("stdio://0", "rb") as stdin,
+        smart_open(path, mode) as f,
+        smart_open("stdio://1", "wb") as stdout_fd,
+    ):
         length = 16 * 1024
         while True:
             buf = stdin.read(length)

megfile/config.py

@@ -1,53 +1,39 @@
 import os
-from logging import getLogger
-
-_logger = getLogger(__name__)
-
-DEFAULT_BLOCK_SIZE = int(os.getenv("MEGFILE_BLOCK_SIZE") or 8 * 2**20)
-
-if os.getenv("MEGFILE_MAX_BUFFER_SIZE"):
-    DEFAULT_MAX_BUFFER_SIZE = int(os.environ["MEGFILE_MAX_BUFFER_SIZE"])
-    if DEFAULT_MAX_BUFFER_SIZE < DEFAULT_BLOCK_SIZE:
-        DEFAULT_MAX_BUFFER_SIZE = DEFAULT_BLOCK_SIZE
-        _logger.warning(
-            "Env 'MEGFILE_MAX_BUFFER_SIZE' is smaller than block size, "
-            "will not use buffer."
-        )
-    DEFAULT_BLOCK_CAPACITY = DEFAULT_MAX_BUFFER_SIZE // DEFAULT_BLOCK_SIZE
-    if os.getenv("MEGFILE_BLOCK_CAPACITY"):
-        _logger.warning(
-            "Env 'MEGFILE_MAX_BUFFER_SIZE' and 'MEGFILE_BLOCK_CAPACITY' are both set, "
-            "'MEGFILE_BLOCK_CAPACITY' will be ignored."
-        )
-elif os.getenv("MEGFILE_BLOCK_CAPACITY"):
-    DEFAULT_BLOCK_CAPACITY = int(os.environ["MEGFILE_BLOCK_CAPACITY"])
-    DEFAULT_MAX_BUFFER_SIZE = DEFAULT_BLOCK_SIZE * DEFAULT_BLOCK_CAPACITY
-else:
-    DEFAULT_MAX_BUFFER_SIZE = 128 * 2**20
-    DEFAULT_BLOCK_CAPACITY = 16
-
-DEFAULT_MIN_BLOCK_SIZE = int(os.getenv("MEGFILE_MIN_BLOCK_SIZE") or DEFAULT_BLOCK_SIZE)
-
-if os.getenv("MEGFILE_MAX_BLOCK_SIZE"):
-    DEFAULT_MAX_BLOCK_SIZE = int(os.environ["MEGFILE_MAX_BLOCK_SIZE"])
-    if DEFAULT_MAX_BLOCK_SIZE < DEFAULT_BLOCK_SIZE:
-        DEFAULT_MAX_BLOCK_SIZE = DEFAULT_BLOCK_SIZE
-        _logger.warning(
-            "Env 'MEGFILE_MAX_BLOCK_SIZE' is smaller than block size, will be ignored."
-        )
-else:
-    DEFAULT_MAX_BLOCK_SIZE = max(128 * 2**20, DEFAULT_BLOCK_SIZE)
-
-GLOBAL_MAX_WORKERS = int(os.getenv("MEGFILE_MAX_WORKERS") or 32)
-DEFAULT_MAX_RETRY_TIMES = int(os.getenv("MEGFILE_MAX_RETRY_TIMES") or 10)
 
-# for logging the size of file had read or wrote
-BACKOFF_INITIAL = 64 * 2**20  # 64MB
-BACKOFF_FACTOR = 4
+
+def to_boolean(value):
+    return value.lower() in ("true", "yes", "1")
+
+
+READER_BLOCK_SIZE = int(os.getenv("MEGFILE_READER_BLOCK_SIZE") or 8 * 2**20)
+if READER_BLOCK_SIZE <= 0:
+    raise ValueError(
+        f"'MEGFILE_READER_BLOCK_SIZE' must bigger than 0, got {READER_BLOCK_SIZE}"
+    )
+READER_MAX_BUFFER_SIZE = int(os.getenv("MEGFILE_READER_MAX_BUFFER_SIZE") or 128 * 2**20)
+
+# Multi-upload in aws s3 has a maximum of 10,000 parts,
+# so the maximum supported file size is MEGFILE_WRITE_BLOCK_SIZE * 10,000,
+# the largest object that can be uploaded in a single PUT is 5 TB in aws s3.
+WRITER_BLOCK_SIZE = int(os.getenv("MEGFILE_WRITER_BLOCK_SIZE") or 8 * 2**20)
+if WRITER_BLOCK_SIZE <= 0:
+    raise ValueError(
+        f"'MEGFILE_WRITER_BLOCK_SIZE' must bigger than 0, got {WRITER_BLOCK_SIZE}"
+    )
+WRITER_MAX_BUFFER_SIZE = int(os.getenv("MEGFILE_WRITER_MAX_BUFFER_SIZE") or 128 * 2**20)
+DEFAULT_WRITER_BLOCK_AUTOSCALE = not os.getenv("MEGFILE_WRITER_BLOCK_SIZE")
+if os.getenv("MEGFILE_WRITER_BLOCK_AUTOSCALE"):
+    DEFAULT_WRITER_BLOCK_AUTOSCALE = to_boolean(
+        os.environ["MEGFILE_WRITER_BLOCK_AUTOSCALE"].lower()
+    )
+
+GLOBAL_MAX_WORKERS = int(os.getenv("MEGFILE_MAX_WORKERS") or 8)
 
 NEWLINE = ord("\n")
 
 S3_CLIENT_CACHE_MODE = os.getenv("MEGFILE_S3_CLIENT_CACHE_MODE") or "thread_local"
+
+DEFAULT_MAX_RETRY_TIMES = int(os.getenv("MEGFILE_MAX_RETRY_TIMES") or 10)
 S3_MAX_RETRY_TIMES = int(
     os.getenv("MEGFILE_S3_MAX_RETRY_TIMES") or DEFAULT_MAX_RETRY_TIMES
 )
@@ -60,3 +46,7 @@ HDFS_MAX_RETRY_TIMES = int(
 SFTP_MAX_RETRY_TIMES = int(
     os.getenv("MEGFILE_SFTP_MAX_RETRY_TIMES") or DEFAULT_MAX_RETRY_TIMES
 )
+
+SFTP_HOST_KEY_POLICY = os.getenv("MEGFILE_SFTP_HOST_KEY_POLICY")
+
+HTTP_AUTH_HEADERS = ("Authorization", "Www-Authenticate", "Cookie", "Cookie2")

megfile/fs.py

@@ -1,20 +1,10 @@
+import os
 from typing import BinaryIO, Callable, Iterator, List, Optional, Tuple
 
 from megfile.fs_path import (
     FSPath,
     _make_stat,
-    fs_cwd,
-    fs_glob,
-    fs_glob_stat,
-    fs_home,
-    fs_iglob,
-    fs_lstat,
-    fs_makedirs,
-    fs_move,
     fs_path_join,
-    fs_readlink,
-    fs_rename,
-    fs_resolve,
     is_fs,
 )
 from megfile.interfaces import Access, FileEntry, PathLike, StatResult
@@ -430,3 +420,171 @@ def fs_save_as(file_object: BinaryIO, path: PathLike):
     :param file_object: stream to be read
     """
     return FSPath(path).save(file_object)
+
+
+def fs_readlink(path) -> str:
+    """
+    Return a string representing the path to which the symbolic link points.
+    :returns: Return a string representing the path to which the symbolic link points.
+    """
+    return FSPath(path).readlink().path_without_protocol  # pyre-ignore[7]
+
+
+def fs_cwd() -> str:
+    """Return current working directory
+
+    returns: Current working directory
+    """
+    return os.getcwd()
+
+
+def fs_home():
+    """Return the home directory
+
+    returns: Home directory path
+    """
+    return os.path.expanduser("~")
+
+
+def fs_iglob(
+    path: PathLike, recursive: bool = True, missing_ok: bool = True
+) -> Iterator[str]:
+    """Return path iterator in ascending alphabetical order,
+    in which path matches glob pattern
+
+    1. If doesn't match any path, return empty list
+        Notice:  ``glob.glob`` in standard library returns ['a/'] instead of empty list
+        when pathname is like `a/**`, recursive is True and directory 'a' doesn't exist.
+        fs_glob behaves like ``glob.glob`` in standard library under such circumstance.
+    2. No guarantee that each path in result is different, which means:
+        Assume there exists a path `/a/b/c/b/d.txt`
+        use path pattern like `/**/b/**/*.txt` to glob,
+        the path above will be returned twice
+    3. `**` will match any matched file, directory, symlink and '' by default,
+        when recursive is `True`
+    4. fs_glob returns same as glob.glob(pathname, recursive=True)
+        in ascending alphabetical order.
+    5. Hidden files (filename stars with '.') will not be found in the result
+
+    :param recursive: If False, `**` will not search directory recursively
+    :param missing_ok: If False and target path doesn't match any file,
+        raise FileNotFoundError
+    :returns: An iterator contains paths match `pathname`
+    """
+    for path_obj in FSPath(path).iglob(
+        pattern="", recursive=recursive, missing_ok=missing_ok
+    ):
+        yield path_obj.path_without_protocol  # pyre-ignore[7]
+
+
+def fs_glob(
+    path: PathLike, recursive: bool = True, missing_ok: bool = True
+) -> List[str]:
+    """Return path list in ascending alphabetical order,
+    in which path matches glob pattern
+
+    1. If doesn't match any path, return empty list
+        Notice:  ``glob.glob`` in standard library returns ['a/'] instead of empty list
+        when pathname is like `a/**`, recursive is True and directory 'a' doesn't exist.
+        fs_glob behaves like ``glob.glob`` in standard library under such circumstance.
+    2. No guarantee that each path in result is different, which means:
+        Assume there exists a path `/a/b/c/b/d.txt`
+        use path pattern like `/**/b/**/*.txt` to glob,
+        the path above will be returned twice
+    3. `**` will match any matched file, directory, symlink and '' by default,
+        when recursive is `True`
+    4. fs_glob returns same as glob.glob(pathname, recursive=True)
+        in ascending alphabetical order.
+    5. Hidden files (filename stars with '.') will not be found in the result
+
+    :param recursive: If False, `**` will not search directory recursively
+    :param missing_ok: If False and target path doesn't match any file,
+        raise FileNotFoundError
+    :returns: A list contains paths match `pathname`
+    """
+    return list(fs_iglob(path=path, recursive=recursive, missing_ok=missing_ok))
+
+
+def fs_glob_stat(
+    path: PathLike, recursive: bool = True, missing_ok: bool = True
+) -> Iterator[FileEntry]:
+    """Return a list contains tuples of path and file stat,
+    in ascending alphabetical order, in which path matches glob pattern
+
+    1. If doesn't match any path, return empty list
+        Notice:  ``glob.glob`` in standard library returns ['a/'] instead of empty list
+        when pathname is like `a/**`, recursive is True and directory 'a' doesn't exist.
+        fs_glob behaves like ``glob.glob`` in standard library under such circumstance.
+    2. No guarantee that each path in result is different, which means:
+        Assume there exists a path `/a/b/c/b/d.txt`
+        use path pattern like `/**/b/**/*.txt` to glob,
+        the path above will be returned twice.
+    3. `**` will match any matched file, directory, symlink and '' by default,
+        when recursive is `True`
+    4. fs_glob returns same as glob.glob(pathname, recursive=True)
+        in ascending alphabetical order.
+    5. Hidden files (filename stars with '.') will not be found in the result
+
+    :param recursive: If False, `**` will not search directory recursively
+    :param missing_ok: If False and target path doesn't match any file,
+        raise FileNotFoundError
+    :returns: A list contains tuples of path and file stat,
+        in which paths match `pathname`
+    """
+    for path in fs_iglob(path=path, recursive=recursive, missing_ok=missing_ok):
+        yield FileEntry(os.path.basename(path), path, _make_stat(os.lstat(path)))
+
+
+def fs_resolve(path: PathLike) -> str:
+    """Equal to fs_realpath, return the real path of given path
+
+    :param path: Given path
+    :returns: Real path of given path
+    """
+    return FSPath(path).realpath()
+
+
+def fs_makedirs(path: PathLike, exist_ok: bool = False):
+    """
+    make a directory on fs, including parent directory
+
+    If there exists a file on the path, raise FileExistsError
+
+    :param path: Given path
+    :param exist_ok: If False and target directory exists, raise FileExistsError
+    :raises: FileExistsError
+    """
+    return FSPath(path).mkdir(parents=True, exist_ok=exist_ok)
+
+
+def fs_lstat(path: PathLike) -> StatResult:
+    """
+    Like Path.stat() but, if the path points to a symbolic link,
+    return the symbolic link’s information rather than its target’s.
+
+    :param path: Given path
+    :returns: StatResult
+    """
+    return FSPath(path).lstat()
+
+
+def fs_rename(src_path: PathLike, dst_path: PathLike, overwrite: bool = True) -> None:
+    """
+    rename file on fs
+
+    :param src_path: Given path
+    :param dst_path: Given destination path
+    :param overwrite: whether or not overwrite file when exists
+    """
+    FSPath(src_path).rename(dst_path, overwrite)
+
+
+def fs_move(src_path: PathLike, dst_path: PathLike, overwrite: bool = True) -> None:
+    """
+    rename file on fs
+
+    :param src_path: Given path
+    :param dst_path: Given destination path
+    :param overwrite: whether or not overwrite file when exists
+    """
+    return fs_rename(src_path, dst_path, overwrite)