megfile 4.0.1__py3-none-any.whl → 4.0.3__py3-none-any.whl

megfile/cli.py

@@ -1,5 +1,4 @@
 import configparser
-import logging
 import os
 import shutil
 import sys
@@ -10,7 +9,7 @@ from functools import partial
 import click
 from tqdm import tqdm
 
-from megfile.config import READER_BLOCK_SIZE, SFTP_HOST_KEY_POLICY
+from megfile.config import READER_BLOCK_SIZE, SFTP_HOST_KEY_POLICY, set_log_level
 from megfile.hdfs_path import DEFAULT_HDFS_TIMEOUT
 from megfile.interfaces import FileEntry
 from megfile.lib.glob import get_non_glob_dir, has_magic
@@ -45,29 +44,34 @@ from megfile.smart_path import SmartPath
 from megfile.utils import get_human_size
 from megfile.version import VERSION
 
-logging.basicConfig(level=logging.ERROR)
-logging.getLogger("megfile").setLevel(level=logging.INFO)
-DEBUG = False
+options = {}
+set_log_level()
 
 
 @click.group()
 @click.option("--debug", is_flag=True, help="Enable debug mode.")
-def cli(debug):
+@click.option(
+    "--log-level",
+    type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR"]),
+    help="Set logging level.",
+)
+def cli(debug, log_level):
     """
     Client for megfile.
 
     If you install megfile with ``--user``,
     you also need configure ``$HOME/.local/bin`` into ``$PATH``.
     """
-    global DEBUG
-    DEBUG = debug
+    options["debug"] = debug
+    options["log_level"] = log_level or ("DEBUG" if debug else "INFO")
+    set_log_level(options["log_level"])
 
 
 def safe_cli():  # pragma: no cover
     try:
         cli()
     except Exception as e:
-        if DEBUG:
+        if options.get("debug", False):
             raise
         else:
             click.echo(f"\n[{type(e).__name__}] {e}", err=True)

megfile/config.py

@@ -1,3 +1,4 @@
+import logging
 import os
 import typing as T
 
@@ -61,10 +62,24 @@ def parse_quantity(quantity: T.Union[str, int]) -> int:
     return number * (base**exponent)  # pytype: disable=bad-return-type
 
 
-def to_boolean(value):
+def parse_boolean(value: T.Optional[str], default: bool = False):
+    if value is None:
+        return default
     return value.lower() in ("true", "yes", "1")
 
 
+def set_log_level(level: T.Optional[T.Union[int, str]] = None):
+    logging.basicConfig(
+        level=logging.ERROR,
+        format=(
+            "%(asctime)s | %(levelname)-8s | "
+            "%(name)s:%(funcName)s:%(lineno)d - %(message)s"
+        ),
+    )
+    level = level or os.getenv("MEGFILE_LOG_LEVEL") or logging.INFO
+    logging.getLogger("megfile").setLevel(level)
+
+
 READER_BLOCK_SIZE = parse_quantity(os.getenv("MEGFILE_READER_BLOCK_SIZE") or 8 * 2**20)
 if READER_BLOCK_SIZE <= 0:
     raise ValueError(
@@ -87,8 +102,8 @@ WRITER_MAX_BUFFER_SIZE = parse_quantity(
 )
 DEFAULT_WRITER_BLOCK_AUTOSCALE = not os.getenv("MEGFILE_WRITER_BLOCK_SIZE")
 if os.getenv("MEGFILE_WRITER_BLOCK_AUTOSCALE"):
-    DEFAULT_WRITER_BLOCK_AUTOSCALE = to_boolean(
-        os.environ["MEGFILE_WRITER_BLOCK_AUTOSCALE"].lower()
+    DEFAULT_WRITER_BLOCK_AUTOSCALE = parse_boolean(
+        os.environ["MEGFILE_WRITER_BLOCK_AUTOSCALE"]
     )
 
 GLOBAL_MAX_WORKERS = int(os.getenv("MEGFILE_MAX_WORKERS") or 8)
@@ -114,3 +129,6 @@ SFTP_MAX_RETRY_TIMES = int(
 SFTP_HOST_KEY_POLICY = os.getenv("MEGFILE_SFTP_HOST_KEY_POLICY")
 
 HTTP_AUTH_HEADERS = ("Authorization", "Www-Authenticate", "Cookie", "Cookie2")
+
+if os.getenv("MEGFILE_LOG_LEVEL"):
+    set_log_level()

megfile/lib/s3_prefetch_reader.py

@@ -62,31 +62,29 @@ class S3PrefetchReader(BasePrefetchReader):
         )
 
     def _get_content_size(self):
-        if self._block_capacity > 0:
-            try:
-                start, end = 0, self._block_size - 1
-                first_index_response = self._fetch_response(start=start, end=end)
-                if "ContentRange" in first_index_response:
-                    content_size = int(
-                        first_index_response["ContentRange"].split("/")[-1]
-                    )
-                else:
-                    # usually when read a file only have one block
-                    content_size = int(first_index_response["ContentLength"])
-            except S3InvalidRangeError:
-                # usually when read a empty file
-                # can use minio test empty file: https://hub.docker.com/r/minio/minio
-                first_index_response = self._fetch_response()
-                content_size = int(first_index_response["ContentLength"])
-
-            first_future = Future()
-            first_future.set_result(first_index_response["Body"])
-            self._insert_futures(index=0, future=first_future)
-            self._content_etag = first_index_response["ETag"]
-        else:
+        if self._block_capacity <= 0:
             response = self._client.head_object(Bucket=self._bucket, Key=self._key)
-            self._content_etag = response["ETag"]
-            content_size = int(response["ContentLength"])
+            self._content_etag = response.get("ETag")
+            return int(response["ContentLength"])
+
+        try:
+            start, end = 0, self._block_size - 1
+            first_index_response = self._fetch_response(start=start, end=end)
+            if "ContentRange" in first_index_response:
+                content_size = int(first_index_response["ContentRange"].split("/")[-1])
+            else:
+                # usually when read a file only have one block
+                content_size = int(first_index_response["ContentLength"])
+        except S3InvalidRangeError:
+            # usually when read a empty file
+            # can use minio test empty file: https://hub.docker.com/r/minio/minio
+            first_index_response = self._fetch_response()
+            content_size = int(first_index_response["ContentLength"])
+
+        first_future = Future()
+        first_future.set_result(first_index_response["Body"])
+        self._insert_futures(index=0, future=first_future)
+        self._content_etag = first_index_response.get("ETag")
         return content_size
 
     @property
@@ -122,7 +120,7 @@ class S3PrefetchReader(BasePrefetchReader):
         start, end = index * self._block_size, (index + 1) * self._block_size - 1
         response = self._fetch_response(start=start, end=end)
         etag = response.get("ETag", None)
-        if etag is not None and etag != self._content_etag:
+        if self._content_etag and etag and etag != self._content_etag:
             raise S3FileChangedError(
                 "File changed: %r, etag before: %s, after: %s"
                 % (self.name, self._content_etag, etag)

megfile/s3_path.py

@@ -23,7 +23,7 @@ from megfile.config import (
     S3_MAX_RETRY_TIMES,
     WRITER_BLOCK_SIZE,
     WRITER_MAX_BUFFER_SIZE,
-    to_boolean,
+    parse_boolean,
 )
 from megfile.errors import (
     S3BucketNotFoundError,
@@ -253,12 +253,6 @@ def get_env_var(env_name: str, profile_name=None):
     return os.getenv(env_name.upper())
 
 
-def parse_boolean(value: Optional[str], default: bool = False) -> bool:
-    if value is None:
-        return default
-    return to_boolean(value)
-
-
 def get_access_token(profile_name=None):
     access_key = get_env_var("AWS_ACCESS_KEY_ID", profile_name=profile_name)
     secret_key = get_env_var("AWS_SECRET_ACCESS_KEY", profile_name=profile_name)
@@ -1003,13 +997,15 @@ def s3_buffered_open(
                 profile_name=s3_url._profile_name,
             )
         else:
+            if max_buffer_size is None:
+                max_buffer_size = READER_MAX_BUFFER_SIZE
             reader = S3PrefetchReader(
                 bucket,
                 key,
                 s3_client=client,
                 max_retries=max_retries,
                 max_workers=max_workers,
-                max_buffer_size=max_buffer_size or READER_MAX_BUFFER_SIZE,
+                max_buffer_size=max_buffer_size,
                 block_forward=block_forward,
                 block_size=block_size or READER_BLOCK_SIZE,
                 profile_name=s3_url._profile_name,
@@ -1019,23 +1015,27 @@ def s3_buffered_open(
         return reader
 
     if limited_seekable:
+        if max_buffer_size is None:
+            max_buffer_size = WRITER_MAX_BUFFER_SIZE
         writer = S3LimitedSeekableWriter(
             bucket,
             key,
             s3_client=client,
             max_workers=max_workers,
             block_size=block_size or WRITER_BLOCK_SIZE,
-            max_buffer_size=max_buffer_size or WRITER_MAX_BUFFER_SIZE,
+            max_buffer_size=max_buffer_size,
             profile_name=s3_url._profile_name,
         )
     else:
+        if max_buffer_size is None:
+            max_buffer_size = WRITER_MAX_BUFFER_SIZE
         writer = S3BufferedWriter(
             bucket,
             key,
             s3_client=client,
             max_workers=max_workers,
             block_size=block_size or WRITER_BLOCK_SIZE,
-            max_buffer_size=max_buffer_size or WRITER_MAX_BUFFER_SIZE,
+            max_buffer_size=max_buffer_size,
             profile_name=s3_url._profile_name,
         )
     if buffered or _is_pickle(writer):

megfile/smart.py

@@ -1006,7 +1006,8 @@ def smart_load_content(
         offset = -1
         if stop is not None:
             offset = stop - (start or 0)  # start may be None
-            assert offset >= 0, "stop should be greater than start"
+            if offset < 0:
+                raise ValueError("stop should be greater than start")
         return fd.read(offset)  # pytype: disable=bad-return-type
 
 

megfile/version.py

@@ -1 +1 @@
-VERSION = "4.0.1"
+VERSION = "4.0.3"

{megfile-4.0.1.dist-info → megfile-4.0.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: megfile
-Version: 4.0.1
+Version: 4.0.3
 Summary: Megvii file operation library
 Author-email: megvii <megfile@megvii.com>
 Project-URL: Homepage, https://github.com/megvii-research/megfile
@@ -212,6 +212,9 @@ You can get the configuration from `~/.config/megfile/aliases.conf`, like:
 protocol = s3+tos
 ```
 
+## Benchmark
+[![10GiB](https://github.com/megvii-research/megfile/blob/main/scripts/benchmark/10GiB.png?raw=true)](https://megvii-research.github.io/megfile/benchmark.html)
+[![10MiB](https://github.com/megvii-research/megfile/blob/main/scripts/benchmark/10MiB.png?raw=true)](https://megvii-research.github.io/megfile/benchmark.html)
 
 ## How to Contribute
 * We welcome everyone to contribute code to the `megfile` project, but the contributed code needs to meet the following conditions as much as possible:

{megfile-4.0.1.dist-info → megfile-4.0.3.dist-info}/RECORD

@@ -1,6 +1,6 @@
 megfile/__init__.py,sha256=7oEfu410CFKzDWZ9RjL5xEJ1gtkJkTfvPrL_7TWdJuY,7366
-megfile/cli.py,sha256=FuwxjU7-Z_pIU8sX8Jf-0QNSLCR8JqLD4fqXyj7TrQI,24704
-megfile/config.py,sha256=4aXs2fw59ep1a6PfMT3Vq-AaOxTwaS8yfgMNcb3Cvw8,3652
+megfile/cli.py,sha256=e3VVr8oe8iR7L_PtpNtyqAvQL_WgJzzEz8oewSAlgX4,24887
+megfile/config.py,sha256=_6HiGeXEyk6RjPdjA0eEj1unq9iLJV_vQJBzQ-eHNvs,4185
 megfile/errors.py,sha256=a55qKQgyfiLmV-qnojUFzq2gu9JXpj3ZiC2qVaWyUTA,14160
 megfile/fs.py,sha256=bPGbapv41FzME87X3MhSNQRjNmHrI23FuFnjPT0ukQs,18239
 megfile/fs_path.py,sha256=ZK-po1xqhHocMb9Vrxf5K9tDx3zxQmGxNIHY3Z7Akp8,39085
@@ -11,14 +11,14 @@ megfile/http_path.py,sha256=c-xAu5wDxcTevmIUmrNEy-m-QiCfDJToaVI7y8SVIUI,14492
 megfile/interfaces.py,sha256=p4UvVZpeLx5djd6bqqDaygIx_s-_AxIVj-gudTch4JE,8467
 megfile/pathlike.py,sha256=vfuTBqSTIciRxkkqMfLfnBxWTEl9yns1yR8zgK4Raw0,31268
 megfile/s3.py,sha256=zqAegH5tijcztEKcfHXmOYhAR880nTxaAzc2O0JJnjc,16661
-megfile/s3_path.py,sha256=oBA9GdOseEtQJmh7LMDOf1sGamsEERs6Sm1jHpdksO8,93343
+megfile/s3_path.py,sha256=lpUKy4n5DTf6hK6TvPhMjt_ZgdIXO4vcyK_VLaGkvhg,93395
 megfile/sftp.py,sha256=0ZnQlmhgvs7pYjFTcvvOyxTo2IUurE-hp1GN0hnIrdQ,26473
 megfile/sftp_path.py,sha256=4rcbn3wqcOEs71W6qWu1efcj6MZUgrZm6U0Jan-eB70,43604
-megfile/smart.py,sha256=h42U8WQvCy9UXOX0X3V1eOZ500wDeZy8YKfr-uygTaA,36881
+megfile/smart.py,sha256=Ps8acPx6jeG1UJnRD8xL2aQjRp7IMW8sV6VFkMF0TQk,36910
 megfile/smart_path.py,sha256=Bqg95T2-XZrRXWhH7GT-jMCYzD7i1SIXdczQxtOxiPs,7583
 megfile/stdio.py,sha256=C_cGID_npthpwoPcsJMMEqqbVUPUnDxxJV9jLY2_D7c,635
 megfile/stdio_path.py,sha256=L8ODNIwO79UIv13YYc2OTr6f4XTv4ZPyvBeRk83-AjA,2700
-megfile/version.py,sha256=AvvDxCXX9bKN1IZtn0pXhbfwx56BkY-VhwBBiNl-1JE,19
+megfile/version.py,sha256=aJJATI8dDtIUcu59WVztQ3az97BkUrCKd86VVa6W_aM,19
 megfile/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 megfile/lib/base_prefetch_reader.py,sha256=6Dy2ZwlowqAvyUUa7bpQLCKOclmmUDhqEF-_CDDp0Og,13100
 megfile/lib/combine_reader.py,sha256=nKGAug29lOpNIZuLKu7_qVrJJRpXL_J4jxLglWbGJ1w,4808
@@ -36,17 +36,27 @@ megfile/lib/s3_cached_handler.py,sha256=X8PdeRC-BY6eSmOO5f2BeyjTPxyEwNtHgmAm9Vgm
 megfile/lib/s3_limited_seekable_writer.py,sha256=mUeoTS98LHluwDN7zxdCVcsjOGBT1bOYV8nRvi9QMGE,6212
 megfile/lib/s3_memory_handler.py,sha256=4uzBzz2jfRI_u6jl0CpOGAhpNJhDQo18FSAweauCUFs,4136
 megfile/lib/s3_pipe_handler.py,sha256=dm7NnZd1Ym5ABS1GvOQtoCJEO_CB8e6p4sUhLiid0go,3622
-megfile/lib/s3_prefetch_reader.py,sha256=ujqQMXAxsaNtWA_VdqfwqcEKBJFFv0GauAaj2WGLTCs,4552
+megfile/lib/s3_prefetch_reader.py,sha256=R37-y_L9l8IKJhpT8HwBrZEbo2X72vCqEV6fvqPCBug,4437
 megfile/lib/s3_share_cache_reader.py,sha256=LVWKxHdHo0_zUIW4o8yqNvplqqwezUPeYEt02Vj-WNM,3754
 megfile/lib/shadow_handler.py,sha256=TntewlvIW9ZxCfmqASDQREHoiZ8v42faOe9sovQYQz0,2779
 megfile/lib/stdio_handler.py,sha256=IDdgENLQlhigEwkLL4zStueVSzdWg7xVcTF_koof_Ek,1987
 megfile/lib/url.py,sha256=ER32pWy9Q2MAk3TraAaNEBWIqUeBmLuM57ol2cs7-Ks,103
 megfile/utils/__init__.py,sha256=sATf_NlsSTYIMEiA8-gM6K1M-Q1K6_7rx2VM31hrqaA,10838
 megfile/utils/mutex.py,sha256=asb8opGLgK22RiuBJUnfsvB8LnMmodP8KzCVHKmQBWA,2561
-megfile-4.0.1.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-megfile-4.0.1.dist-info/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
-megfile-4.0.1.dist-info/METADATA,sha256=GVa381PxFA9AaKcRRWpUdxvOwBxBSbPhZud6_HP_1eE,9240
-megfile-4.0.1.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-megfile-4.0.1.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
-megfile-4.0.1.dist-info/top_level.txt,sha256=IaHHoRXeemLL6kTM5YuC3H0UyOnTdZH9J324TkeBneo,36
-megfile-4.0.1.dist-info/RECORD,,
+scripts/benchmark/code/iopath_read.py,sha256=O1Qs3mpvD9S_XCuRH2A2FpGWxCOSw6qZvEBrtPeRL1E,661
+scripts/benchmark/code/iopath_write.py,sha256=Mm0efW1J09RJ_CK5i1xtG2hJuaaslikin8qVpuRFP_Q,704
+scripts/benchmark/code/megfile_read.py,sha256=sAMebUiMColHDv3JEkXplImAHvn_IF1-g3BIJxhcQYE,239
+scripts/benchmark/code/megfile_write.py,sha256=bzn-i2bGH4eRcsVvkhXK35KsQkX2v2oEsOJ0Ft5saj0,257
+scripts/benchmark/code/pyarrow_read.py,sha256=2QBGKjGV2Dvl2ukOntLSag33pF55l3tfZ2Z6dLUjovw,305
+scripts/benchmark/code/pyarrow_write.py,sha256=U1puLh-ljSXv772bZYAyhzmxhPOq4aR4j-QtwdM9hG0,328
+scripts/benchmark/code/s3fs_read.py,sha256=XiTA-qrYblUs-jQWXSnvNg5Wo722C_g47aMMfo5XJBY,380
+scripts/benchmark/code/s3fs_write.py,sha256=gdXKkWXYGjLJlRT_J64pJN85XvRg3bZexcAJQEMXwtw,402
+scripts/benchmark/code/smart_open_read.py,sha256=SA02jHwS9Y31yFtV9CoJcfND5dR0eA_HsGmGNUrpQls,515
+scripts/benchmark/code/smart_open_write.py,sha256=jDxFJdY97yNH889jz3pawBoei3yaqy8pEMvC_ymHFtM,537
+megfile-4.0.3.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+megfile-4.0.3.dist-info/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
+megfile-4.0.3.dist-info/METADATA,sha256=At2cDDQKTUSj0QmjmPAjPRpqG_M5TZdefe-_2xMNjX8,9573
+megfile-4.0.3.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+megfile-4.0.3.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
+megfile-4.0.3.dist-info/top_level.txt,sha256=fVg49lk5B9L7jyfWUXWxb0DDSuw5pbr0OU62Tvx8J8M,44
+megfile-4.0.3.dist-info/RECORD,,

{megfile-4.0.1.dist-info → megfile-4.0.3.dist-info}/top_level.txt

@@ -3,3 +3,4 @@ docs
 html_cov
 html_doc
 megfile
+scripts

scripts/benchmark/code/iopath_read.py

@@ -0,0 +1,29 @@
+import os
+import time
+
+import boto3
+from iopath.common.file_io import PathManager
+from iopath.common.s3 import S3PathHandler
+
+times = 10240
+s3_path = "s3://bucketA/large.txt"
+
+start = time.time()
+
+path_manager = PathManager()
+
+session = boto3.Session(
+    aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
+    aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
+)
+client = session.client("s3", endpoint_url=os.environ["OSS_ENDPOINT"])
+handler = S3PathHandler()
+handler.client = client
+
+path_manager.register_handler(handler)
+
+with path_manager.open(s3_path, "rb") as f:
+    for i in range(times):
+        f.read(1024 * 1024)
+
+print(time.time() - start)

scripts/benchmark/code/iopath_write.py

@@ -0,0 +1,30 @@
+import os
+import time
+
+import boto3
+from iopath.common.file_io import PathManager
+from iopath.common.s3 import S3PathHandler
+
+times = 10240
+s3_path = "s3://bucketA/large.txt"
+block = b"1" * 1024 * 1024
+
+start = time.time()
+
+path_manager = PathManager()
+
+session = boto3.Session(
+    aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
+    aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
+)
+client = session.client("s3", endpoint_url=os.environ["OSS_ENDPOINT"])
+handler = S3PathHandler()
+handler.client = client
+
+path_manager.register_handler(handler)
+
+with path_manager.open(s3_path, "wb") as f:
+    for i in range(times):
+        f.write(block)
+
+print(time.time() - start)  # write 10GB 91.642

scripts/benchmark/code/megfile_read.py

@@ -0,0 +1,13 @@
+import time
+
+from megfile import smart_open
+
+times = 10240
+s3_path = "s3://bucketA/large.txt"
+
+start = time.time()
+with smart_open(s3_path, "rb") as f:
+    for i in range(times):
+        f.read(1024 * 1024 * 1)
+
+print(time.time() - start)

scripts/benchmark/code/megfile_write.py

@@ -0,0 +1,14 @@
+import time
+
+from megfile import smart_open
+
+times = 10240
+s3_path = "s3://bucketA/large.txt"
+block = b"1" * 1024 * 1024
+
+start = time.time()
+with smart_open(s3_path, "wb") as f:
+    for i in range(times):
+        f.write(block)
+
+print(time.time() - start)

scripts/benchmark/code/pyarrow_read.py

@@ -0,0 +1,17 @@
+import os
+import time
+
+from pyarrow import fs
+
+times = 10240
+s3_path = "bucketA/large.txt"
+
+start = time.time()
+
+s3 = fs.S3FileSystem(endpoint_override=os.environ["OSS_ENDPOINT"])
+
+with s3.open_input_stream(s3_path) as f:
+    for i in range(times):
+        f.read(1024 * 1024)
+
+print(time.time() - start)

scripts/benchmark/code/pyarrow_write.py

@@ -0,0 +1,18 @@
+import os
+import time
+
+from pyarrow import fs
+
+times = 10240
+block = b"1" * 1024 * 1024
+s3_path = "bucketA/large.txt"
+
+start = time.time()
+
+s3 = fs.S3FileSystem(endpoint_override=os.environ["OSS_ENDPOINT"])
+
+with s3.open_output_stream(s3_path) as f:
+    for i in range(times):
+        f.write(block)
+
+print(time.time() - start)

scripts/benchmark/code/s3fs_read.py

@@ -0,0 +1,21 @@
+import os
+import time
+
+import s3fs
+
+times = 10240
+s3_path = "bucketA/large.txt"
+
+start = time.time()
+
+s3 = s3fs.S3FileSystem(
+    endpoint_url=os.environ["OSS_ENDPOINT"],
+    key=os.environ["AWS_ACCESS_KEY_ID"],
+    secret=os.environ["AWS_SECRET_ACCESS_KEY"],
+)
+
+with s3.open(s3_path, "rb") as f:
+    for i in range(times):
+        f.read(1024 * 1024)
+
+print(time.time() - start)

scripts/benchmark/code/s3fs_write.py

@@ -0,0 +1,22 @@
+import os
+import time
+
+import s3fs
+
+times = 10240
+block = b"1" * 1024 * 1024
+s3_path = "bucketA/large.txt"
+
+start = time.time()
+
+s3 = s3fs.S3FileSystem(
+    endpoint_url=os.environ["OSS_ENDPOINT"],
+    key=os.environ["AWS_ACCESS_KEY_ID"],
+    secret=os.environ["AWS_SECRET_ACCESS_KEY"],
+)
+
+with s3.open(s3_path, "wb") as f:
+    for i in range(times):
+        f.write(block)
+
+print(time.time() - start)

scripts/benchmark/code/smart_open_read.py

@@ -0,0 +1,25 @@
+import os
+import time
+
+import boto3
+from smart_open import open
+
+times = 10240
+s3_path = "s3://bucketA/large.txt"
+
+start = time.time()
+session = boto3.Session(
+    aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
+    aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
+)
+with open(
+    s3_path,
+    "rb",
+    transport_params={
+        "client": session.client("s3", endpoint_url=os.environ["OSS_ENDPOINT"])
+    },
+) as f:
+    for i in range(times):
+        f.read(1024 * 1024)
+
+print(time.time() - start)

scripts/benchmark/code/smart_open_write.py

@@ -0,0 +1,26 @@
+import os
+import time
+
+import boto3
+from smart_open import open
+
+times = 10240
+s3_path = "s3://bucketA/large.txt"
+block = b"1" * 1024 * 1024
+
+start = time.time()
+session = boto3.Session(
+    aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
+    aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
+)
+with open(
+    s3_path,
+    "wb",
+    transport_params={
+        "client": session.client("s3", endpoint_url=os.environ["OSS_ENDPOINT"])
+    },
+) as f:
+    for i in range(times):
+        f.write(block)
+
+print(time.time() - start)