megfile 4.0.0.post1__py3-none-any.whl → 4.0.2__py3-none-any.whl

megfile/__init__.py

@@ -121,6 +121,7 @@ from megfile.s3_path import S3Path
 from megfile.sftp import (
     is_sftp,
     sftp_absolute,
+    sftp_add_host_key,
     sftp_chmod,
     sftp_concat,
     sftp_copy,
@@ -371,6 +372,7 @@ __all__ = [
     "sftp_copy",
     "sftp_sync",
     "sftp_concat",
+    "sftp_add_host_key",
     "is_hdfs",
     "hdfs_exists",
     "hdfs_stat",

megfile/cli.py

@@ -1,5 +1,4 @@
 import configparser
-import logging
 import os
 import shutil
 import sys
@@ -10,10 +9,11 @@ from functools import partial
 import click
 from tqdm import tqdm
 
-from megfile.config import READER_BLOCK_SIZE
+from megfile.config import READER_BLOCK_SIZE, SFTP_HOST_KEY_POLICY, set_log_level
 from megfile.hdfs_path import DEFAULT_HDFS_TIMEOUT
 from megfile.interfaces import FileEntry
 from megfile.lib.glob import get_non_glob_dir, has_magic
+from megfile.sftp import sftp_add_host_key
 from megfile.smart import (
     _smart_sync_single_file,
     smart_copy,
@@ -44,29 +44,34 @@ from megfile.smart_path import SmartPath
 from megfile.utils import get_human_size
 from megfile.version import VERSION
 
-logging.basicConfig(level=logging.ERROR)
-logging.getLogger("megfile").setLevel(level=logging.INFO)
-DEBUG = False
+options = {}
+set_log_level()
 
 
 @click.group()
 @click.option("--debug", is_flag=True, help="Enable debug mode.")
-def cli(debug):
+@click.option(
+    "--log-level",
+    type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR"]),
+    help="Set logging level.",
+)
+def cli(debug, log_level):
     """
     Client for megfile.
 
     If you install megfile with ``--user``,
     you also need configure ``$HOME/.local/bin`` into ``$PATH``.
     """
-    global DEBUG
-    DEBUG = debug
+    options["debug"] = debug
+    options["log_level"] = log_level or ("DEBUG" if debug else "INFO")
+    set_log_level(options["log_level"])
 
 
 def safe_cli():  # pragma: no cover
     try:
         cli()
     except Exception as e:
-        if DEBUG:
+        if options.get("debug", False):
             raise
         else:
             click.echo(f"\n[{type(e).__name__}] {e}", err=True)
@@ -110,6 +115,23 @@ def smart_list_stat(path):
         yield from smart_scandir(path)
 
 
+def _sftp_prompt_host_key(path):
+    if SFTP_HOST_KEY_POLICY == "auto":
+        return
+
+    path = SmartPath(path)
+    if path.protocol == "sftp":
+        hostname = (
+            path.pathlike._urlsplit_parts.hostname  # pytype: disable=attribute-error
+        )
+        port = path.pathlike._urlsplit_parts.port  # pytype: disable=attribute-error
+        sftp_add_host_key(
+            hostname=hostname,
+            port=port,
+            prompt=True,
+        )
+
+
 def _ls(path: str, long: bool, recursive: bool, human_readable: bool):
     base_path = path
     full_path = False
@@ -121,6 +143,9 @@ def _ls(path: str, long: bool, recursive: bool, human_readable: bool):
         scan_func = smart_scan_stat
     else:
         scan_func = smart_list_stat
+
+    _sftp_prompt_host_key(base_path)
+
     if long:
         if human_readable:
             echo_func = human_echo
@@ -209,6 +234,10 @@ def cp(
 ):
     if not no_target_directory and (dst_path.endswith("/") or smart_isdir(dst_path)):
         dst_path = smart_path_join(dst_path, os.path.basename(src_path))
+
+    _sftp_prompt_host_key(src_path)
+    _sftp_prompt_host_key(dst_path)
+
     if recursive:
         with ThreadPoolExecutor(max_workers=(os.cpu_count() or 1) * 2) as executor:
             if progress_bar:
@@ -274,6 +303,10 @@ def mv(
 ):
     if not no_target_directory and (dst_path.endswith("/") or smart_isdir(dst_path)):
         dst_path = smart_path_join(dst_path, os.path.basename(src_path))
+
+    _sftp_prompt_host_key(src_path)
+    _sftp_prompt_host_key(dst_path)
+
     if progress_bar:
         src_protocol, _ = SmartPath._extract_protocol(src_path)
         dst_protocol, _ = SmartPath._extract_protocol(dst_path)
@@ -324,6 +357,8 @@ def mv(
     "under the specified directory or prefix.",
 )
 def rm(path: str, recursive: bool):
+    _sftp_prompt_host_key(path)
+
     remove_func = smart_remove if recursive else smart_unlink
     remove_func(path)
 
@@ -349,6 +384,9 @@ def sync(
     quiet: bool,
     skip: bool,
 ):
+    _sftp_prompt_host_key(src_path)
+    _sftp_prompt_host_key(dst_path)
+
     if not smart_exists(dst_path):
         force = True
 
@@ -434,18 +472,24 @@ def sync(
 @cli.command(short_help="Make the path if it doesn't already exist.")
 @click.argument("path")
 def mkdir(path: str):
+    _sftp_prompt_host_key(path)
+
     smart_makedirs(path)
 
 
 @cli.command(short_help="Make the file if it doesn't already exist.")
 @click.argument("path")
 def touch(path: str):
+    _sftp_prompt_host_key(path)
+
     smart_touch(path)
 
 
 @cli.command(short_help="Concatenate any files and send them to stdout.")
 @click.argument("path")
 def cat(path: str):
+    _sftp_prompt_host_key(path)
+
     with smart_open(path, "rb") as f:
         shutil.copyfileobj(f, sys.stdout.buffer)  # pytype: disable=wrong-arg-types
 
@@ -458,6 +502,8 @@ def cat(path: str):
     "-n", "--lines", type=click.INT, default=10, help="print the first NUM lines"
 )
 def head(path: str, lines: int):
+    _sftp_prompt_host_key(path)
+
     with smart_open(path, "rb") as f:
         for _ in range(lines):
             try:
@@ -480,6 +526,8 @@ def head(path: str, lines: int):
     "-f", "--follow", is_flag=True, help="output appended data as the file grows"
 )
 def tail(path: str, lines: int, follow: bool):
+    _sftp_prompt_host_key(path)
+
     line_list = []
     with smart_open(path, "rb") as f:
         f.seek(0, os.SEEK_END)
@@ -524,6 +572,8 @@ def tail(path: str, lines: int, follow: bool):
 @click.option("-a", "--append", is_flag=True, help="Append to the given file")
 @click.option("-o", "--stdout", is_flag=True, help="File content to standard output")
 def to(path: str, append: bool, stdout: bool):
+    _sftp_prompt_host_key(path)
+
     mode = "wb"
     if append:
         mode = "ab"
@@ -545,24 +595,32 @@ def to(path: str, append: bool, stdout: bool):
 @cli.command(short_help="Produce an md5sum file for all the objects in the path.")
 @click.argument("path")
 def md5sum(path: str):
+    _sftp_prompt_host_key(path)
+
     click.echo(smart_getmd5(path, recalculate=True))
 
 
 @cli.command(short_help="Return the total size and number of objects in remote:path.")
 @click.argument("path")
 def size(path: str):
+    _sftp_prompt_host_key(path)
+
     click.echo(smart_getsize(path))
 
 
 @cli.command(short_help="Return the mtime and number of objects in remote:path.")
 @click.argument("path")
 def mtime(path: str):
+    _sftp_prompt_host_key(path)
+
     click.echo(smart_getmtime(path))
 
 
 @cli.command(short_help="Return the stat and number of objects in remote:path.")
 @click.argument("path")
 def stat(path: str):
+    _sftp_prompt_host_key(path)
+
     click.echo(smart_stat(path))
 
 

megfile/config.py

@@ -1,30 +1,109 @@
+import logging
 import os
+import typing as T
 
 
-def to_boolean(value):
+def parse_quantity(quantity: T.Union[str, int]) -> int:
+    """
+    Parse kubernetes canonical form quantity like 200Mi to a int number.
+    Supported SI suffixes:
+    base1024: Ki | Mi | Gi | Ti | Pi | Ei
+    base1000: "" | k | M | G | T | P | E
+
+    (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)
+
+    Input:
+    quantity: string. kubernetes canonical form quantity
+
+    Returns:
+    Int
+
+    Raises:
+    ValueError on invalid or unknown input
+    """
+    if isinstance(quantity, int):
+        return quantity
+
+    exponents = {"K": 1, "k": 1, "M": 2, "G": 3, "T": 4, "P": 5, "E": 6}
+
+    number = quantity
+    suffix = None
+    if len(quantity) >= 2 and quantity[-1] == "i":
+        if quantity[-2] in exponents:
+            number = quantity[:-2]
+            suffix = quantity[-2:]
+    elif len(quantity) >= 1 and quantity[-1] in exponents:
+        number = quantity[:-1]
+        suffix = quantity[-1:]
+
+    try:
+        number = int(number)
+    except ValueError:
+        raise ValueError("Invalid number format: {}".format(number))
+
+    if suffix is None:
+        return number
+
+    if suffix.endswith("i"):
+        base = 1024
+    elif len(suffix) == 1:
+        base = 1000
+    else:
+        raise ValueError("{} has unknown suffix".format(quantity))
+
+    # handle SI inconsistency
+    if suffix == "ki":
+        raise ValueError("{} has unknown suffix".format(quantity))
+
+    if suffix[0] not in exponents:
+        raise ValueError("{} has unknown suffix".format(quantity))
+
+    exponent = int(exponents[suffix[0]])
+    return number * (base**exponent)  # pytype: disable=bad-return-type
+
+
+def parse_boolean(value: T.Optional[str], default: bool = False):
+    if value is None:
+        return default
     return value.lower() in ("true", "yes", "1")
 
 
-READER_BLOCK_SIZE = int(os.getenv("MEGFILE_READER_BLOCK_SIZE") or 8 * 2**20)
+def set_log_level(level: T.Optional[T.Union[int, str]] = None):
+    logging.basicConfig(
+        level=logging.ERROR,
+        format=(
+            "%(asctime)s | %(levelname)-8s | "
+            "%(name)s:%(funcName)s:%(lineno)d - %(message)s"
+        ),
+    )
+    level = level or os.getenv("MEGFILE_LOG_LEVEL") or logging.INFO
+    logging.getLogger("megfile").setLevel(level)
+
+
+READER_BLOCK_SIZE = parse_quantity(os.getenv("MEGFILE_READER_BLOCK_SIZE") or 8 * 2**20)
 if READER_BLOCK_SIZE <= 0:
     raise ValueError(
         f"'MEGFILE_READER_BLOCK_SIZE' must bigger than 0, got {READER_BLOCK_SIZE}"
     )
-READER_MAX_BUFFER_SIZE = int(os.getenv("MEGFILE_READER_MAX_BUFFER_SIZE") or 128 * 2**20)
+READER_MAX_BUFFER_SIZE = parse_quantity(
+    os.getenv("MEGFILE_READER_MAX_BUFFER_SIZE") or 128 * 2**20
+)
 
 # Multi-upload in aws s3 has a maximum of 10,000 parts,
 # so the maximum supported file size is MEGFILE_WRITE_BLOCK_SIZE * 10,000,
 # the largest object that can be uploaded in a single PUT is 5 TB in aws s3.
-WRITER_BLOCK_SIZE = int(os.getenv("MEGFILE_WRITER_BLOCK_SIZE") or 8 * 2**20)
+WRITER_BLOCK_SIZE = parse_quantity(os.getenv("MEGFILE_WRITER_BLOCK_SIZE") or 8 * 2**20)
 if WRITER_BLOCK_SIZE <= 0:
     raise ValueError(
         f"'MEGFILE_WRITER_BLOCK_SIZE' must bigger than 0, got {WRITER_BLOCK_SIZE}"
     )
-WRITER_MAX_BUFFER_SIZE = int(os.getenv("MEGFILE_WRITER_MAX_BUFFER_SIZE") or 128 * 2**20)
+WRITER_MAX_BUFFER_SIZE = parse_quantity(
+    os.getenv("MEGFILE_WRITER_MAX_BUFFER_SIZE") or 128 * 2**20
+)
 DEFAULT_WRITER_BLOCK_AUTOSCALE = not os.getenv("MEGFILE_WRITER_BLOCK_SIZE")
 if os.getenv("MEGFILE_WRITER_BLOCK_AUTOSCALE"):
-    DEFAULT_WRITER_BLOCK_AUTOSCALE = to_boolean(
-        os.environ["MEGFILE_WRITER_BLOCK_AUTOSCALE"].lower()
+    DEFAULT_WRITER_BLOCK_AUTOSCALE = parse_boolean(
+        os.environ["MEGFILE_WRITER_BLOCK_AUTOSCALE"]
     )
 
 GLOBAL_MAX_WORKERS = int(os.getenv("MEGFILE_MAX_WORKERS") or 8)
@@ -50,3 +129,6 @@ SFTP_MAX_RETRY_TIMES = int(
 SFTP_HOST_KEY_POLICY = os.getenv("MEGFILE_SFTP_HOST_KEY_POLICY")
 
 HTTP_AUTH_HEADERS = ("Authorization", "Www-Authenticate", "Cookie", "Cookie2")
+
+if os.getenv("MEGFILE_LOG_LEVEL"):
+    set_log_level()

megfile/lib/s3_prefetch_reader.py

@@ -52,7 +52,6 @@ class S3PrefetchReader(BasePrefetchReader):
         self._client = s3_client
         self._profile_name = profile_name
         self._content_etag = None
-        self._content_info = None
 
         super().__init__(
             block_size=block_size,
@@ -63,6 +62,11 @@ class S3PrefetchReader(BasePrefetchReader):
         )
 
     def _get_content_size(self):
+        if self._block_capacity <= 0:
+            response = self._client.head_object(Bucket=self._bucket, Key=self._key)
+            self._content_etag = response["ETag"]
+            return int(response["ContentLength"])
+
         try:
             start, end = 0, self._block_size - 1
             first_index_response = self._fetch_response(start=start, end=end)
@@ -77,12 +81,10 @@ class S3PrefetchReader(BasePrefetchReader):
             first_index_response = self._fetch_response()
             content_size = int(first_index_response["ContentLength"])
 
-        if self._block_capacity > 0:
-            first_future = Future()
-            first_future.set_result(first_index_response["Body"])
-            self._insert_futures(index=0, future=first_future)
+        first_future = Future()
+        first_future.set_result(first_index_response["Body"])
+        self._insert_futures(index=0, future=first_future)
         self._content_etag = first_index_response["ETag"]
-        self._content_info = first_index_response
         return content_size
 
     @property
@@ -121,7 +123,7 @@ class S3PrefetchReader(BasePrefetchReader):
         if etag is not None and etag != self._content_etag:
             raise S3FileChangedError(
                 "File changed: %r, etag before: %s, after: %s"
-                % (self.name, self._content_info, response)
+                % (self.name, self._content_etag, etag)
             )
 
         return response["Body"]

megfile/s3_path.py

@@ -23,7 +23,7 @@ from megfile.config import (
     S3_MAX_RETRY_TIMES,
     WRITER_BLOCK_SIZE,
     WRITER_MAX_BUFFER_SIZE,
-    to_boolean,
+    parse_boolean,
 )
 from megfile.errors import (
     S3BucketNotFoundError,
@@ -253,12 +253,6 @@ def get_env_var(env_name: str, profile_name=None):
     return os.getenv(env_name.upper())
 
 
-def parse_boolean(value: Optional[str], default: bool = False) -> bool:
-    if value is None:
-        return default
-    return to_boolean(value)
-
-
 def get_access_token(profile_name=None):
     access_key = get_env_var("AWS_ACCESS_KEY_ID", profile_name=profile_name)
     secret_key = get_env_var("AWS_SECRET_ACCESS_KEY", profile_name=profile_name)
@@ -1003,13 +997,15 @@ def s3_buffered_open(
                 profile_name=s3_url._profile_name,
             )
         else:
+            if max_buffer_size is None:
+                max_buffer_size = READER_MAX_BUFFER_SIZE
             reader = S3PrefetchReader(
                 bucket,
                 key,
                 s3_client=client,
                 max_retries=max_retries,
                 max_workers=max_workers,
-                max_buffer_size=max_buffer_size or READER_MAX_BUFFER_SIZE,
+                max_buffer_size=max_buffer_size,
                 block_forward=block_forward,
                 block_size=block_size or READER_BLOCK_SIZE,
                 profile_name=s3_url._profile_name,
@@ -1019,23 +1015,27 @@ def s3_buffered_open(
         return reader
 
     if limited_seekable:
+        if max_buffer_size is None:
+            max_buffer_size = WRITER_MAX_BUFFER_SIZE
         writer = S3LimitedSeekableWriter(
             bucket,
             key,
             s3_client=client,
             max_workers=max_workers,
             block_size=block_size or WRITER_BLOCK_SIZE,
-            max_buffer_size=max_buffer_size or WRITER_MAX_BUFFER_SIZE,
+            max_buffer_size=max_buffer_size,
             profile_name=s3_url._profile_name,
         )
     else:
+        if max_buffer_size is None:
+            max_buffer_size = WRITER_MAX_BUFFER_SIZE
         writer = S3BufferedWriter(
             bucket,
             key,
             s3_client=client,
             max_workers=max_workers,
             block_size=block_size or WRITER_BLOCK_SIZE,
-            max_buffer_size=max_buffer_size or WRITER_MAX_BUFFER_SIZE,
+            max_buffer_size=max_buffer_size,
             profile_name=s3_url._profile_name,
         )
     if buffered or _is_pickle(writer):

megfile/sftp.py

@@ -1,7 +1,11 @@
+import base64
+import hashlib
 import os
 from logging import getLogger as get_logger
 from typing import IO, BinaryIO, Callable, Iterator, List, Optional, Tuple
 
+import paramiko
+
 from megfile.interfaces import FileEntry, PathLike, StatResult
 from megfile.lib.compat import fspath
 from megfile.lib.joinpath import uri_join
@@ -52,6 +56,7 @@ __all__ = [
     "sftp_rmdir",
     "sftp_copy",
     "sftp_sync",
+    "sftp_add_host_key",
 ]
 
 
@@ -739,3 +744,75 @@ def sftp_sync(
     :param overwrite: whether or not overwrite file when exists, default is True
     """
     return SftpPath(src_path).sync(dst_path, followlinks, force, overwrite)
+
+
+def _check_input(input_str: str, fingerprint: str, times: int = 0) -> bool:
+    answers = input_str.strip()
+    if answers.lower() in ("yes", "y") or answers == fingerprint:
+        return True
+    elif answers.lower() in ("no", "n"):
+        return False
+    elif times >= 10:
+        _logger.warning("Retried more than 10 times, give up")
+        return False
+    else:
+        input_str = input("Please type 'yes', 'no' or the fingerprint: ")
+        return _check_input(input_str, fingerprint, times=times + 1)
+
+
+def _prompt_add_to_known_hosts(hostname, key) -> bool:
+    fingerprint = hashlib.sha256(key.asbytes()).digest()
+    fingerprint = f"SHA256:{base64.b64encode(fingerprint).decode('utf-8')}"
+    answers = input(f"""The authenticity of host '{hostname}' can't be established.
+{key.get_name().upper()} key fingerprint is {fingerprint}.
+This key is not known by any other names.
+Are you sure you want to continue connecting (yes/no/[fingerprint])? """)
+    return _check_input(answers, fingerprint)
+
+
+def sftp_add_host_key(
+    hostname: str,
+    port: int = 22,
+    prompt: bool = False,
+    host_key_path: Optional["str"] = None,
+):
+    """Add a host key to known_hosts.
+
+    :param hostname: hostname
+    :param port: port, default is 22
+    :param prompt: If True, requires user input of 'yes' or 'no' to decide whether to
+        add this host key
+    :param host_key_path: path of known_hosts, default is ~/.ssh/known_hosts
+    """
+    if not host_key_path:
+        host_key_path = os.path.expanduser("~/.ssh/known_hosts")
+
+    if not os.path.exists(host_key_path):
+        dirname = os.path.dirname(host_key_path)
+        if dirname and dirname != ".":
+            os.makedirs(dirname, exist_ok=True, mode=0o700)
+        with open(host_key_path, "w"):
+            pass
+        os.chmod(host_key_path, 0o600)
+
+    host_key = paramiko.hostkeys.HostKeys(host_key_path)
+    if host_key.lookup(hostname):
+        return
+
+    transport = paramiko.Transport(
+        (
+            hostname,
+            port,
+        )
+    )
+    transport.connect()
+    key = transport.get_remote_server_key()
+    transport.close()
+
+    if prompt:
+        result = _prompt_add_to_known_hosts(hostname, key)
+        if not result:
+            return
+
+    host_key.add(hostname, key.get_name(), key)
+    host_key.save(host_key_path)

megfile/sftp_path.py

@@ -208,6 +208,7 @@ def _get_ssh_client(
     policy = policies.get(SFTP_HOST_KEY_POLICY, default_policy)()  # pyre-ignore[29]
 
     ssh_client = paramiko.SSHClient()
+    ssh_client.load_system_host_keys()
     ssh_client.set_missing_host_key_policy(policy)
     max_unauth_connections = int(os.getenv(SFTP_MAX_UNAUTH_CONN, 10))
     try:

megfile/smart.py

@@ -1001,11 +1001,13 @@ def smart_load_content(
         return s3_load_content(path, start, stop)
 
     with smart_open(path, "rb") as fd:
-        if start:
+        if start is not None:
             fd.seek(start)
         offset = -1
-        if start and stop:
-            offset = stop - start
+        if stop is not None:
+            offset = stop - (start or 0)  # start may be None
+            if offset < 0:
+                raise ValueError("stop should be greater than start")
         return fd.read(offset)  # pytype: disable=bad-return-type
 
 

megfile/version.py

@@ -1 +1 @@
-VERSION = "4.0.0.post1"
+VERSION = "4.0.2"

{megfile-4.0.0.post1.dist-info → megfile-4.0.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: megfile
-Version: 4.0.0.post1
+Version: 4.0.2
 Summary: Megvii file operation library
 Author-email: megvii <megfile@megvii.com>
 Project-URL: Homepage, https://github.com/megvii-research/megfile
@@ -200,10 +200,10 @@ s3 =
 $ megfile config s3 accesskey secretkey \
 --addressing-style virtual \
 --endpoint-url https://tos-s3-cn-beijing.ivolces.com \
---profile tos
+--profile-name tos
 
 # create alias
-$ megfile alias tos s3+tos
+$ megfile config alias tos s3+tos
 ```
 
 You can get the configuration from `~/.config/megfile/aliases.conf`, like:
@@ -212,6 +212,9 @@ You can get the configuration from `~/.config/megfile/aliases.conf`, like:
 protocol = s3+tos
 ```
 
+## Speed Test
+[![10GiB](https://github.com/megvii-research/megfile/blob/main/scripts/speed_test/10GiB.png?raw=true)](https://megvii-research.github.io/megfile/speed_test.html)
+[![10MiB](https://github.com/megvii-research/megfile/blob/main/scripts/speed_test/10MiB.png?raw=true)](https://megvii-research.github.io/megfile/speed_test.html)
 
 ## How to Contribute
 * We welcome everyone to contribute code to the `megfile` project, but the contributed code needs to meet the following conditions as much as possible:

{megfile-4.0.0.post1.dist-info → megfile-4.0.2.dist-info}/RECORD

@@ -1,6 +1,6 @@
-megfile/__init__.py,sha256=i2Lbq_VxIgppaqwkxG0_H35dRfcjJ4mCYWjprOf4hHo,7318
-megfile/cli.py,sha256=yRRd555mk-yxxg0XShnot068oEuvX1E4qY-hb-uVLcc,23549
-megfile/config.py,sha256=kcnqVGrUKSuZ3YqLHpeJPYj94WO2SIH2S6n755lUXew,1942
+megfile/__init__.py,sha256=7oEfu410CFKzDWZ9RjL5xEJ1gtkJkTfvPrL_7TWdJuY,7366
+megfile/cli.py,sha256=e3VVr8oe8iR7L_PtpNtyqAvQL_WgJzzEz8oewSAlgX4,24887
+megfile/config.py,sha256=_6HiGeXEyk6RjPdjA0eEj1unq9iLJV_vQJBzQ-eHNvs,4185
 megfile/errors.py,sha256=a55qKQgyfiLmV-qnojUFzq2gu9JXpj3ZiC2qVaWyUTA,14160
 megfile/fs.py,sha256=bPGbapv41FzME87X3MhSNQRjNmHrI23FuFnjPT0ukQs,18239
 megfile/fs_path.py,sha256=ZK-po1xqhHocMb9Vrxf5K9tDx3zxQmGxNIHY3Z7Akp8,39085
@@ -11,14 +11,14 @@ megfile/http_path.py,sha256=c-xAu5wDxcTevmIUmrNEy-m-QiCfDJToaVI7y8SVIUI,14492
 megfile/interfaces.py,sha256=p4UvVZpeLx5djd6bqqDaygIx_s-_AxIVj-gudTch4JE,8467
 megfile/pathlike.py,sha256=vfuTBqSTIciRxkkqMfLfnBxWTEl9yns1yR8zgK4Raw0,31268
 megfile/s3.py,sha256=zqAegH5tijcztEKcfHXmOYhAR880nTxaAzc2O0JJnjc,16661
-megfile/s3_path.py,sha256=oBA9GdOseEtQJmh7LMDOf1sGamsEERs6Sm1jHpdksO8,93343
-megfile/sftp.py,sha256=WP7soS6WuSzHPc72iRHklkBoQG_4lNPbOCGwEy2Zwhw,24075
-megfile/sftp_path.py,sha256=jzLEt1RnzX00GCApyy1NSCGj7eQKen_bz0DETD6oVyg,43565
-megfile/smart.py,sha256=WS9ZZbMAMPrOR2z8sb8hxVYCVsCRMhGGUvcboQjCkBw,36771
+megfile/s3_path.py,sha256=lpUKy4n5DTf6hK6TvPhMjt_ZgdIXO4vcyK_VLaGkvhg,93395
+megfile/sftp.py,sha256=0ZnQlmhgvs7pYjFTcvvOyxTo2IUurE-hp1GN0hnIrdQ,26473
+megfile/sftp_path.py,sha256=4rcbn3wqcOEs71W6qWu1efcj6MZUgrZm6U0Jan-eB70,43604
+megfile/smart.py,sha256=Ps8acPx6jeG1UJnRD8xL2aQjRp7IMW8sV6VFkMF0TQk,36910
 megfile/smart_path.py,sha256=Bqg95T2-XZrRXWhH7GT-jMCYzD7i1SIXdczQxtOxiPs,7583
 megfile/stdio.py,sha256=C_cGID_npthpwoPcsJMMEqqbVUPUnDxxJV9jLY2_D7c,635
 megfile/stdio_path.py,sha256=L8ODNIwO79UIv13YYc2OTr6f4XTv4ZPyvBeRk83-AjA,2700
-megfile/version.py,sha256=4XWC9d4BvU4TuG3QOaPY1dppiaFLKYtd6zGozoT0xdM,25
+megfile/version.py,sha256=secXeY4K_CdlbaUs9G--LGWfWlf1VqdtopDxqRzq2JQ,19
 megfile/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 megfile/lib/base_prefetch_reader.py,sha256=6Dy2ZwlowqAvyUUa7bpQLCKOclmmUDhqEF-_CDDp0Og,13100
 megfile/lib/combine_reader.py,sha256=nKGAug29lOpNIZuLKu7_qVrJJRpXL_J4jxLglWbGJ1w,4808
@@ -36,17 +36,27 @@ megfile/lib/s3_cached_handler.py,sha256=X8PdeRC-BY6eSmOO5f2BeyjTPxyEwNtHgmAm9Vgm
 megfile/lib/s3_limited_seekable_writer.py,sha256=mUeoTS98LHluwDN7zxdCVcsjOGBT1bOYV8nRvi9QMGE,6212
 megfile/lib/s3_memory_handler.py,sha256=4uzBzz2jfRI_u6jl0CpOGAhpNJhDQo18FSAweauCUFs,4136
 megfile/lib/s3_pipe_handler.py,sha256=dm7NnZd1Ym5ABS1GvOQtoCJEO_CB8e6p4sUhLiid0go,3622
-megfile/lib/s3_prefetch_reader.py,sha256=wnc-0K6USDBXKgVRZtp-W8aWf5J87J_ediJK6IJWLho,4332
+megfile/lib/s3_prefetch_reader.py,sha256=dHltiM5Ui-SY4pqhvIsmC0iNmprXwlczDD4lNHB5WrQ,4418
 megfile/lib/s3_share_cache_reader.py,sha256=LVWKxHdHo0_zUIW4o8yqNvplqqwezUPeYEt02Vj-WNM,3754
 megfile/lib/shadow_handler.py,sha256=TntewlvIW9ZxCfmqASDQREHoiZ8v42faOe9sovQYQz0,2779
 megfile/lib/stdio_handler.py,sha256=IDdgENLQlhigEwkLL4zStueVSzdWg7xVcTF_koof_Ek,1987
 megfile/lib/url.py,sha256=ER32pWy9Q2MAk3TraAaNEBWIqUeBmLuM57ol2cs7-Ks,103
 megfile/utils/__init__.py,sha256=sATf_NlsSTYIMEiA8-gM6K1M-Q1K6_7rx2VM31hrqaA,10838
 megfile/utils/mutex.py,sha256=asb8opGLgK22RiuBJUnfsvB8LnMmodP8KzCVHKmQBWA,2561
-megfile-4.0.0.post1.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-megfile-4.0.0.post1.dist-info/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
-megfile-4.0.0.post1.dist-info/METADATA,sha256=wBX578pPpvPTAxGBK4kiszH6Qi7kwhUMruNOOIIFQHE,9234
-megfile-4.0.0.post1.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-megfile-4.0.0.post1.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
-megfile-4.0.0.post1.dist-info/top_level.txt,sha256=IaHHoRXeemLL6kTM5YuC3H0UyOnTdZH9J324TkeBneo,36
-megfile-4.0.0.post1.dist-info/RECORD,,
+scripts/speed_test/code/iopath_read.py,sha256=O1Qs3mpvD9S_XCuRH2A2FpGWxCOSw6qZvEBrtPeRL1E,661
+scripts/speed_test/code/iopath_write.py,sha256=Mm0efW1J09RJ_CK5i1xtG2hJuaaslikin8qVpuRFP_Q,704
+scripts/speed_test/code/megfile_read.py,sha256=sAMebUiMColHDv3JEkXplImAHvn_IF1-g3BIJxhcQYE,239
+scripts/speed_test/code/megfile_write.py,sha256=bzn-i2bGH4eRcsVvkhXK35KsQkX2v2oEsOJ0Ft5saj0,257
+scripts/speed_test/code/pyarrow_read.py,sha256=2QBGKjGV2Dvl2ukOntLSag33pF55l3tfZ2Z6dLUjovw,305
+scripts/speed_test/code/pyarrow_write.py,sha256=U1puLh-ljSXv772bZYAyhzmxhPOq4aR4j-QtwdM9hG0,328
+scripts/speed_test/code/s3fs_read.py,sha256=XiTA-qrYblUs-jQWXSnvNg5Wo722C_g47aMMfo5XJBY,380
+scripts/speed_test/code/s3fs_write.py,sha256=gdXKkWXYGjLJlRT_J64pJN85XvRg3bZexcAJQEMXwtw,402
+scripts/speed_test/code/smart_open_read.py,sha256=SA02jHwS9Y31yFtV9CoJcfND5dR0eA_HsGmGNUrpQls,515
+scripts/speed_test/code/smart_open_write.py,sha256=jDxFJdY97yNH889jz3pawBoei3yaqy8pEMvC_ymHFtM,537
+megfile-4.0.2.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+megfile-4.0.2.dist-info/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
+megfile-4.0.2.dist-info/METADATA,sha256=A8TXdy5RWnQFzA7e2r9h757InExc7l7JaSN3O3felgk,9578
+megfile-4.0.2.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+megfile-4.0.2.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
+megfile-4.0.2.dist-info/top_level.txt,sha256=fVg49lk5B9L7jyfWUXWxb0DDSuw5pbr0OU62Tvx8J8M,44
+megfile-4.0.2.dist-info/RECORD,,

{megfile-4.0.0.post1.dist-info → megfile-4.0.2.dist-info}/top_level.txt

@@ -3,3 +3,4 @@ docs
 html_cov
 html_doc
 megfile
+scripts

scripts/speed_test/code/iopath_read.py

@@ -0,0 +1,29 @@
+import os
+import time
+
+import boto3
+from iopath.common.file_io import PathManager
+from iopath.common.s3 import S3PathHandler
+
+times = 10240
+s3_path = "s3://bucketA/large.txt"
+
+start = time.time()
+
+path_manager = PathManager()
+
+session = boto3.Session(
+    aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
+    aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
+)
+client = session.client("s3", endpoint_url=os.environ["OSS_ENDPOINT"])
+handler = S3PathHandler()
+handler.client = client
+
+path_manager.register_handler(handler)
+
+with path_manager.open(s3_path, "rb") as f:
+    for i in range(times):
+        f.read(1024 * 1024)
+
+print(time.time() - start)

scripts/speed_test/code/iopath_write.py

@@ -0,0 +1,30 @@
+import os
+import time
+
+import boto3
+from iopath.common.file_io import PathManager
+from iopath.common.s3 import S3PathHandler
+
+times = 10240
+s3_path = "s3://bucketA/large.txt"
+block = b"1" * 1024 * 1024
+
+start = time.time()
+
+path_manager = PathManager()
+
+session = boto3.Session(
+    aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
+    aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
+)
+client = session.client("s3", endpoint_url=os.environ["OSS_ENDPOINT"])
+handler = S3PathHandler()
+handler.client = client
+
+path_manager.register_handler(handler)
+
+with path_manager.open(s3_path, "wb") as f:
+    for i in range(times):
+        f.write(block)
+
+print(time.time() - start)  # write 10GB 91.642

scripts/speed_test/code/megfile_read.py

@@ -0,0 +1,13 @@
+import time
+
+from megfile import smart_open
+
+times = 10240
+s3_path = "s3://bucketA/large.txt"
+
+start = time.time()
+with smart_open(s3_path, "rb") as f:
+    for i in range(times):
+        f.read(1024 * 1024 * 1)
+
+print(time.time() - start)

scripts/speed_test/code/megfile_write.py

@@ -0,0 +1,14 @@
+import time
+
+from megfile import smart_open
+
+times = 10240
+s3_path = "s3://bucketA/large.txt"
+block = b"1" * 1024 * 1024
+
+start = time.time()
+with smart_open(s3_path, "wb") as f:
+    for i in range(times):
+        f.write(block)
+
+print(time.time() - start)

scripts/speed_test/code/pyarrow_read.py

@@ -0,0 +1,17 @@
+import os
+import time
+
+from pyarrow import fs
+
+times = 10240
+s3_path = "bucketA/large.txt"
+
+start = time.time()
+
+s3 = fs.S3FileSystem(endpoint_override=os.environ["OSS_ENDPOINT"])
+
+with s3.open_input_stream(s3_path) as f:
+    for i in range(times):
+        f.read(1024 * 1024)
+
+print(time.time() - start)

scripts/speed_test/code/pyarrow_write.py

@@ -0,0 +1,18 @@
+import os
+import time
+
+from pyarrow import fs
+
+times = 10240
+block = b"1" * 1024 * 1024
+s3_path = "bucketA/large.txt"
+
+start = time.time()
+
+s3 = fs.S3FileSystem(endpoint_override=os.environ["OSS_ENDPOINT"])
+
+with s3.open_output_stream(s3_path) as f:
+    for i in range(times):
+        f.write(block)
+
+print(time.time() - start)

scripts/speed_test/code/s3fs_read.py

@@ -0,0 +1,21 @@
+import os
+import time
+
+import s3fs
+
+times = 10240
+s3_path = "bucketA/large.txt"
+
+start = time.time()
+
+s3 = s3fs.S3FileSystem(
+    endpoint_url=os.environ["OSS_ENDPOINT"],
+    key=os.environ["AWS_ACCESS_KEY_ID"],
+    secret=os.environ["AWS_SECRET_ACCESS_KEY"],
+)
+
+with s3.open(s3_path, "rb") as f:
+    for i in range(times):
+        f.read(1024 * 1024)
+
+print(time.time() - start)

scripts/speed_test/code/s3fs_write.py

@@ -0,0 +1,22 @@
+import os
+import time
+
+import s3fs
+
+times = 10240
+block = b"1" * 1024 * 1024
+s3_path = "bucketA/large.txt"
+
+start = time.time()
+
+s3 = s3fs.S3FileSystem(
+    endpoint_url=os.environ["OSS_ENDPOINT"],
+    key=os.environ["AWS_ACCESS_KEY_ID"],
+    secret=os.environ["AWS_SECRET_ACCESS_KEY"],
+)
+
+with s3.open(s3_path, "wb") as f:
+    for i in range(times):
+        f.write(block)
+
+print(time.time() - start)

scripts/speed_test/code/smart_open_read.py

@@ -0,0 +1,25 @@
+import os
+import time
+
+import boto3
+from smart_open import open
+
+times = 10240
+s3_path = "s3://bucketA/large.txt"
+
+start = time.time()
+session = boto3.Session(
+    aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
+    aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
+)
+with open(
+    s3_path,
+    "rb",
+    transport_params={
+        "client": session.client("s3", endpoint_url=os.environ["OSS_ENDPOINT"])
+    },
+) as f:
+    for i in range(times):
+        f.read(1024 * 1024)
+
+print(time.time() - start)

scripts/speed_test/code/smart_open_write.py

@@ -0,0 +1,26 @@
+import os
+import time
+
+import boto3
+from smart_open import open
+
+times = 10240
+s3_path = "s3://bucketA/large.txt"
+block = b"1" * 1024 * 1024
+
+start = time.time()
+session = boto3.Session(
+    aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
+    aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
+)
+with open(
+    s3_path,
+    "wb",
+    transport_params={
+        "client": session.client("s3", endpoint_url=os.environ["OSS_ENDPOINT"])
+    },
+) as f:
+    for i in range(times):
+        f.write(block)
+
+print(time.time() - start)