megfile 5.0.5__py3-none-any.whl → 5.0.7__py3-none-any.whl

megfile/lib/base_prefetch_reader.py

@@ -274,7 +274,11 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
     def _buffer(self) -> BytesIO:
         if self._block_capacity == 0:
             buffer = self._fetch_buffer(index=self._block_index)
-            buffer.seek(self._cached_offset)
+            if self._cached_offset is not None:
+                offset = self._cached_offset
+            else:
+                offset = self._offset % self._block_size
+            buffer.seek(offset)
             self._cached_offset = None
             return buffer
 

megfile/s3_path.py

@@ -13,6 +13,7 @@ import boto3
 import botocore
 from boto3.s3.transfer import TransferConfig
 from botocore.awsrequest import AWSPreparedRequest, AWSResponse
+from botocore.utils import calculate_md5 as botocore_calculate_md5
 
 from megfile.config import (
     GLOBAL_MAX_WORKERS,
@@ -114,6 +115,11 @@ endpoint_url = "https://s3.amazonaws.com"
 max_retries = S3_MAX_RETRY_TIMES
 max_keys = 1000
 
+# Patch for https://github.com/aws/aws-cli/issues/9214
+CALCULATE_MD5_FOR_OPERATIONS = {
+    "DeleteObjects",
+}
+
 
 def _patch_make_request(client: botocore.client.BaseClient, redirect: bool = False):
     def after_callback(result: Tuple[AWSResponse, dict], *args, **kwargs):
@@ -140,6 +146,10 @@ def _patch_make_request(client: botocore.client.BaseClient, redirect: bool = Fal
             request_dict["body"].seek(0)
 
     def before_callback(operation_model, request_dict, request_context):
+        if operation_model.name in CALCULATE_MD5_FOR_OPERATIONS:
+            if "Content-MD5" not in request_dict["headers"]:
+                md5_digest = botocore_calculate_md5(request_dict["body"])
+                request_dict["headers"]["Content-MD5"] = md5_digest
         _logger.debug(
             "send s3 request: %r, with parameters: %s",
             operation_model.name,

megfile/utils/__init__.py

@@ -266,7 +266,7 @@ def _get_class(cls_or_obj) -> type:
     return type(cls_or_obj)
 
 
-def calculate_md5(file_object):
+def calculate_md5(file_object) -> str:
     hash_md5 = hashlib.md5()  # nosec
     for chunk in iter(lambda: file_object.read(DEFAULT_HASH_BUFFER_SIZE), b""):
         hash_md5.update(chunk)

megfile/version.py

@@ -1 +1 @@
-VERSION = "5.0.5"
+VERSION = "5.0.7"

{megfile-5.0.5.dist-info → megfile-5.0.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: megfile
-Version: 5.0.5
+Version: 5.0.7
 Summary: Megvii file operation library
 Author-email: megvii-reng <megvii-reng@googlegroups.com>
 Project-URL: Homepage, https://github.com/megvii-research/megfile
@@ -210,6 +210,11 @@ You can use alias in path, like `tos://bucket/key`, the same as `s3+tos://bucket
 [![10GiB](https://github.com/megvii-research/megfile/blob/main/scripts/benchmark/10GiB.png?raw=true)](https://megvii-research.github.io/megfile/benchmark.html)
 [![10MiB](https://github.com/megvii-research/megfile/blob/main/scripts/benchmark/10MiB.png?raw=true)](https://megvii-research.github.io/megfile/benchmark.html)
 
+## Skill for AI coding
+```bash
+git clone -b gh-skill --single-branch https://github.com/megvii-research/megfile.git path/to/your/ai/skills/megfile
+```
+
 ## How to Contribute
 * We welcome everyone to contribute code to the `megfile` project, but the contributed code needs to meet the following conditions as much as possible:
 

{megfile-5.0.5.dist-info → megfile-5.0.7.dist-info}/RECORD

@@ -7,17 +7,17 @@ megfile/hdfs_path.py,sha256=PWqws54Ou136VxaYp9K_UFRr5BoiZWsO330n9ig5IG0,26338
 megfile/http_path.py,sha256=08OmzmRMyLSyq1Yr1K2HbzexesURJrIoA6AibwYzUiA,13844
 megfile/interfaces.py,sha256=XU46U5pl4k1Gse63i4z5SvxcjWeKLj0xyB0Y6fYiWWo,9887
 megfile/pathlike.py,sha256=4RuYHqUc5_6rZDCcVo_18il0Hy7BlOYt-rtYwCtp9Gg,31446
-megfile/s3_path.py,sha256=C6z6pqZb0LMlNqUbZiaC8_deKX3feGxfPfSYjusqJq0,94507
+megfile/s3_path.py,sha256=yScDLBUomWqK7ph8n_xXN2nj1m8RjGxvnyQ2tMFjAZE,94954
 megfile/sftp2_path.py,sha256=K90bnMVAx0MQPGXP6LogGuDRzaD4MPR6lMOfdY9C9-0,37942
 megfile/sftp_path.py,sha256=zxuT1hk7sgoOUwq6KBXS__caX8Hk_LgPjINQheTZWAU,52063
 megfile/smart.py,sha256=Lab2jxprj-zvPw5GqUWlWiEY8bcpRlviks_qp9r-km8,38224
 megfile/smart_path.py,sha256=kGidkM5S58ChE3LVZMcUACs3IQgsqh9m04sp6-wxuhk,12615
 megfile/stdio_path.py,sha256=cxaDr8rtisTPnN-rjtaEpqQnshwiqwXFUJBM9xWY7Cg,2711
-megfile/version.py,sha256=KZmXhcDHicScZCqwBzPprRNi1hfyVjIjbAsyTGuyt_g,19
+megfile/version.py,sha256=my9FRCmD9195OJwXusjWmUyOZeaGWmueiOoYCz7bChw,19
 megfile/webdav_path.py,sha256=QrRYKBGWXkUZXEeHxAfVJkxnCfnczocBSRkVgDC_qC4,31421
 megfile/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 megfile/lib/base_memory_handler.py,sha256=VEfw2RuynHFPtQ6KZv8hv8rnUwsj566Omg4cHajhARk,2863
-megfile/lib/base_prefetch_reader.py,sha256=MYaWOkXc3geZMYNPHlPZmmpOR6uSz-AMuCZwYdoz7t0,13296
+megfile/lib/base_prefetch_reader.py,sha256=rURgAGFDbUx8M-5GnNfKJXPImq76WAZbHQYHyoLCqKQ,13451
 megfile/lib/combine_reader.py,sha256=Kp2wEloOUpTlIU7dve87MBpSzmIM-F9OtpTawAjFkiU,4828
 megfile/lib/compare.py,sha256=CPSbyqsQ396oSfxa7h0NdUUqBw5A3WOn6fHrNKkuinw,2188
 megfile/lib/compat.py,sha256=SynEeHluys3tCK-lb_1oV3o_ft83yZvunqM_AjibLgE,207
@@ -40,13 +40,13 @@ megfile/lib/stdio_handler.py,sha256=IDdgENLQlhigEwkLL4zStueVSzdWg7xVcTF_koof_Ek,
 megfile/lib/url.py,sha256=ER32pWy9Q2MAk3TraAaNEBWIqUeBmLuM57ol2cs7-Ks,103
 megfile/lib/webdav_memory_handler.py,sha256=7nq4o69ck_7dFh6xlYTBG8-rj49Q7gxwa3V2bHXEQz4,2551
 megfile/lib/webdav_prefetch_reader.py,sha256=M0X6E6t-DS5q9KiLvjVZx_AZuiW9SaIkBnIPLc774GQ,3941
-megfile/utils/__init__.py,sha256=lfJze58nO18ug8EUfSJgTTxOwj1p7FQdsnO1keBeMSo,12740
+megfile/utils/__init__.py,sha256=dCyiRgTYScx8FdxWOP2-N1gONbBg76Snrv8tuJ7Ykek,12747
 megfile/utils/atomic.py,sha256=V3gqUQjHIx5j6LCrU2_0tLVlX1ossetxdJJK48g9-rc,4084
 megfile/utils/mutex.py,sha256=asb8opGLgK22RiuBJUnfsvB8LnMmodP8KzCVHKmQBWA,2561
-megfile-5.0.5.dist-info/licenses/LICENSE,sha256=xuY_rHyygMLmf0LgkKj_-wb-BxveHp9rTN0VDE73PrE,11365
-megfile-5.0.5.dist-info/licenses/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
-megfile-5.0.5.dist-info/METADATA,sha256=A5wTdggi4xoo1jf9Jn9PzkOCglLOQ1TBPBBX07_yLtE,9226
-megfile-5.0.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-megfile-5.0.5.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
-megfile-5.0.5.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
-megfile-5.0.5.dist-info/RECORD,,
+megfile-5.0.7.dist-info/licenses/LICENSE,sha256=xuY_rHyygMLmf0LgkKj_-wb-BxveHp9rTN0VDE73PrE,11365
+megfile-5.0.7.dist-info/licenses/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
+megfile-5.0.7.dist-info/METADATA,sha256=kBUxkMNJ70BgXxloM5OwpsdLWZ8Y1QK03V2EOwG1UM8,9378
+megfile-5.0.7.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+megfile-5.0.7.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
+megfile-5.0.7.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
+megfile-5.0.7.dist-info/RECORD,,

{megfile-5.0.5.dist-info → megfile-5.0.7.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.9.0)
+Generator: setuptools (80.10.2)
 Root-Is-Purelib: true
 Tag: py3-none-any