megfile 4.1.3__py3-none-any.whl → 4.1.3.post1__py3-none-any.whl

megfile/version.py

@@ -1 +1 @@
-VERSION = "4.1.3"
+VERSION = "4.1.3.post1"

{megfile-4.1.3.dist-info → megfile-4.1.3.post1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: megfile
-Version: 4.1.3
+Version: 4.1.3.post1
 Summary: Megvii file operation library
 Author-email: megvii <megfile@megvii.com>
 Project-URL: Homepage, https://github.com/megvii-research/megfile

{megfile-4.1.3.dist-info → megfile-4.1.3.post1.dist-info}/RECORD

@@ -18,7 +18,7 @@ megfile/smart.py,sha256=Sae2KJzaU0k_qV_Bk0YifOMq8WsV5qQ2pGInDRF546I,36411
 megfile/smart_path.py,sha256=Up_6xNZ2019iSzMn_JAU_1H--z-AP6O7SxdXGdeTG0c,7659
 megfile/stdio.py,sha256=ZwxsnJNJYIT7Iyg5pIw4qiyH8bszG6oAhEJuR-hXGG4,658
 megfile/stdio_path.py,sha256=cxaDr8rtisTPnN-rjtaEpqQnshwiqwXFUJBM9xWY7Cg,2711
-megfile/version.py,sha256=VxmH81mTbRqT3UnqXh5OXAB3IfXRrcnMl0YgLgOqhPw,19
+megfile/version.py,sha256=eKLHLHdKHcrWOl_XPErEVRYmoIEhUtehLCugzxSUR-E,25
 megfile/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 megfile/lib/base_prefetch_reader.py,sha256=uxVwYknOjc8hLF7q_T2QKMsBqFcrf411ZsuK25CN1eQ,12848
 megfile/lib/combine_reader.py,sha256=Kp2wEloOUpTlIU7dve87MBpSzmIM-F9OtpTawAjFkiU,4828
@@ -43,20 +43,10 @@ megfile/lib/stdio_handler.py,sha256=IDdgENLQlhigEwkLL4zStueVSzdWg7xVcTF_koof_Ek,
 megfile/lib/url.py,sha256=ER32pWy9Q2MAk3TraAaNEBWIqUeBmLuM57ol2cs7-Ks,103
 megfile/utils/__init__.py,sha256=xAzmICA0MtAbg-I2yPfeHjA1N4CiMP4sBrC9BgrfZLw,10151
 megfile/utils/mutex.py,sha256=asb8opGLgK22RiuBJUnfsvB8LnMmodP8KzCVHKmQBWA,2561
-megfile-4.1.3.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-megfile-4.1.3.dist-info/licenses/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
-scripts/benchmark/code/iopath_read.py,sha256=O1Qs3mpvD9S_XCuRH2A2FpGWxCOSw6qZvEBrtPeRL1E,661
-scripts/benchmark/code/iopath_write.py,sha256=Mm0efW1J09RJ_CK5i1xtG2hJuaaslikin8qVpuRFP_Q,704
-scripts/benchmark/code/megfile_read.py,sha256=sAMebUiMColHDv3JEkXplImAHvn_IF1-g3BIJxhcQYE,239
-scripts/benchmark/code/megfile_write.py,sha256=bzn-i2bGH4eRcsVvkhXK35KsQkX2v2oEsOJ0Ft5saj0,257
-scripts/benchmark/code/pyarrow_read.py,sha256=2QBGKjGV2Dvl2ukOntLSag33pF55l3tfZ2Z6dLUjovw,305
-scripts/benchmark/code/pyarrow_write.py,sha256=U1puLh-ljSXv772bZYAyhzmxhPOq4aR4j-QtwdM9hG0,328
-scripts/benchmark/code/s3fs_read.py,sha256=XiTA-qrYblUs-jQWXSnvNg5Wo722C_g47aMMfo5XJBY,380
-scripts/benchmark/code/s3fs_write.py,sha256=gdXKkWXYGjLJlRT_J64pJN85XvRg3bZexcAJQEMXwtw,402
-scripts/benchmark/code/smart_open_read.py,sha256=SA02jHwS9Y31yFtV9CoJcfND5dR0eA_HsGmGNUrpQls,515
-scripts/benchmark/code/smart_open_write.py,sha256=jDxFJdY97yNH889jz3pawBoei3yaqy8pEMvC_ymHFtM,537
-megfile-4.1.3.dist-info/METADATA,sha256=hKZKmMsgZAysVrZJ3aRQtiGRrvHnRfEPx68ayMEt3EM,9595
-megfile-4.1.3.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-megfile-4.1.3.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
-megfile-4.1.3.dist-info/top_level.txt,sha256=TR6xUw8Px5Ms_UENhEmLNmxOdfHAhTlSt9yTw9LRgsQ,35
-megfile-4.1.3.dist-info/RECORD,,
+megfile-4.1.3.post1.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+megfile-4.1.3.post1.dist-info/licenses/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
+megfile-4.1.3.post1.dist-info/METADATA,sha256=oI1b4MMW9GRz3mav_TZ2MSX6C8y1z1QebHkcCZcrcB0,9601
+megfile-4.1.3.post1.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+megfile-4.1.3.post1.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
+megfile-4.1.3.post1.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
+megfile-4.1.3.post1.dist-info/RECORD,,

megfile-4.1.3.post1.dist-info/top_level.txt

@@ -0,0 +1 @@
+megfile

megfile-4.1.3.dist-info/top_level.txt

@@ -1,5 +0,0 @@
-dist
-docs
-html_cov
-megfile
-scripts

scripts/benchmark/code/iopath_read.py

@@ -1,29 +0,0 @@
-import os
-import time
-
-import boto3
-from iopath.common.file_io import PathManager
-from iopath.common.s3 import S3PathHandler
-
-times = 10240
-s3_path = "s3://bucketA/large.txt"
-
-start = time.time()
-
-path_manager = PathManager()
-
-session = boto3.Session(
-    aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
-    aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
-)
-client = session.client("s3", endpoint_url=os.environ["OSS_ENDPOINT"])
-handler = S3PathHandler()
-handler.client = client
-
-path_manager.register_handler(handler)
-
-with path_manager.open(s3_path, "rb") as f:
-    for i in range(times):
-        f.read(1024 * 1024)
-
-print(time.time() - start)

scripts/benchmark/code/iopath_write.py

@@ -1,30 +0,0 @@
-import os
-import time
-
-import boto3
-from iopath.common.file_io import PathManager
-from iopath.common.s3 import S3PathHandler
-
-times = 10240
-s3_path = "s3://bucketA/large.txt"
-block = b"1" * 1024 * 1024
-
-start = time.time()
-
-path_manager = PathManager()
-
-session = boto3.Session(
-    aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
-    aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
-)
-client = session.client("s3", endpoint_url=os.environ["OSS_ENDPOINT"])
-handler = S3PathHandler()
-handler.client = client
-
-path_manager.register_handler(handler)
-
-with path_manager.open(s3_path, "wb") as f:
-    for i in range(times):
-        f.write(block)
-
-print(time.time() - start)  # write 10GB 91.642

scripts/benchmark/code/megfile_read.py

@@ -1,13 +0,0 @@
-import time
-
-from megfile import smart_open
-
-times = 10240
-s3_path = "s3://bucketA/large.txt"
-
-start = time.time()
-with smart_open(s3_path, "rb") as f:
-    for i in range(times):
-        f.read(1024 * 1024 * 1)
-
-print(time.time() - start)

scripts/benchmark/code/megfile_write.py

@@ -1,14 +0,0 @@
-import time
-
-from megfile import smart_open
-
-times = 10240
-s3_path = "s3://bucketA/large.txt"
-block = b"1" * 1024 * 1024
-
-start = time.time()
-with smart_open(s3_path, "wb") as f:
-    for i in range(times):
-        f.write(block)
-
-print(time.time() - start)

scripts/benchmark/code/pyarrow_read.py

@@ -1,17 +0,0 @@
-import os
-import time
-
-from pyarrow import fs
-
-times = 10240
-s3_path = "bucketA/large.txt"
-
-start = time.time()
-
-s3 = fs.S3FileSystem(endpoint_override=os.environ["OSS_ENDPOINT"])
-
-with s3.open_input_stream(s3_path) as f:
-    for i in range(times):
-        f.read(1024 * 1024)
-
-print(time.time() - start)

scripts/benchmark/code/pyarrow_write.py

@@ -1,18 +0,0 @@
-import os
-import time
-
-from pyarrow import fs
-
-times = 10240
-block = b"1" * 1024 * 1024
-s3_path = "bucketA/large.txt"
-
-start = time.time()
-
-s3 = fs.S3FileSystem(endpoint_override=os.environ["OSS_ENDPOINT"])
-
-with s3.open_output_stream(s3_path) as f:
-    for i in range(times):
-        f.write(block)
-
-print(time.time() - start)

scripts/benchmark/code/s3fs_read.py

@@ -1,21 +0,0 @@
-import os
-import time
-
-import s3fs
-
-times = 10240
-s3_path = "bucketA/large.txt"
-
-start = time.time()
-
-s3 = s3fs.S3FileSystem(
-    endpoint_url=os.environ["OSS_ENDPOINT"],
-    key=os.environ["AWS_ACCESS_KEY_ID"],
-    secret=os.environ["AWS_SECRET_ACCESS_KEY"],
-)
-
-with s3.open(s3_path, "rb") as f:
-    for i in range(times):
-        f.read(1024 * 1024)
-
-print(time.time() - start)

scripts/benchmark/code/s3fs_write.py

@@ -1,22 +0,0 @@
-import os
-import time
-
-import s3fs
-
-times = 10240
-block = b"1" * 1024 * 1024
-s3_path = "bucketA/large.txt"
-
-start = time.time()
-
-s3 = s3fs.S3FileSystem(
-    endpoint_url=os.environ["OSS_ENDPOINT"],
-    key=os.environ["AWS_ACCESS_KEY_ID"],
-    secret=os.environ["AWS_SECRET_ACCESS_KEY"],
-)
-
-with s3.open(s3_path, "wb") as f:
-    for i in range(times):
-        f.write(block)
-
-print(time.time() - start)

scripts/benchmark/code/smart_open_read.py

@@ -1,25 +0,0 @@
-import os
-import time
-
-import boto3
-from smart_open import open
-
-times = 10240
-s3_path = "s3://bucketA/large.txt"
-
-start = time.time()
-session = boto3.Session(
-    aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
-    aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
-)
-with open(
-    s3_path,
-    "rb",
-    transport_params={
-        "client": session.client("s3", endpoint_url=os.environ["OSS_ENDPOINT"])
-    },
-) as f:
-    for i in range(times):
-        f.read(1024 * 1024)
-
-print(time.time() - start)

scripts/benchmark/code/smart_open_write.py

@@ -1,26 +0,0 @@
-import os
-import time
-
-import boto3
-from smart_open import open
-
-times = 10240
-s3_path = "s3://bucketA/large.txt"
-block = b"1" * 1024 * 1024
-
-start = time.time()
-session = boto3.Session(
-    aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
-    aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
-)
-with open(
-    s3_path,
-    "wb",
-    transport_params={
-        "client": session.client("s3", endpoint_url=os.environ["OSS_ENDPOINT"])
-    },
-) as f:
-    for i in range(times):
-        f.write(block)
-
-print(time.time() - start)