megfile 5.0.4.post1__py3-none-any.whl → 5.0.6__py3-none-any.whl

megfile/cli.py

@@ -1,6 +1,8 @@
 import os
+import shlex
 import shutil
 import signal
+import subprocess
 import sys
 import time
 from concurrent.futures import ThreadPoolExecutor
@@ -27,6 +29,7 @@ from megfile.s3_path import get_s3_session
 from megfile.sftp_path import sftp_add_host_key
 from megfile.smart import (
     _smart_sync_single_file,
+    smart_cache,
     smart_copy,
     smart_exists,
     smart_getmd5,
@@ -719,6 +722,16 @@ def stat(path: str):
     click.echo(smart_stat(path))
 
 
+@cli.command(short_help="Edit the file.")
+@click.argument("path", type=PathType())
+@click.option("-e", "--editor", type=str, default="vim", help="Editor to use.")
+def edit(path: str, editor: str):
+    with smart_cache(path, mode="a") as cache_path:
+        cmds = shlex.split(editor)
+        cmds.append(cache_path)
+        subprocess.check_call(cmds)
+
+
 @cli.command(short_help="Return the megfile version.")
 def version():
     click.echo(VERSION)

megfile/s3_path.py

@@ -13,6 +13,7 @@ import boto3
 import botocore
 from boto3.s3.transfer import TransferConfig
 from botocore.awsrequest import AWSPreparedRequest, AWSResponse
+from botocore.utils import calculate_md5 as botocore_calculate_md5
 
 from megfile.config import (
     GLOBAL_MAX_WORKERS,
@@ -114,6 +115,11 @@ endpoint_url = "https://s3.amazonaws.com"
 max_retries = S3_MAX_RETRY_TIMES
 max_keys = 1000
 
+# Patch for https://github.com/aws/aws-cli/issues/9214
+CALCULATE_MD5_FOR_OPERATIONS = {
+    "DeleteObjects",
+}
+
 
 def _patch_make_request(client: botocore.client.BaseClient, redirect: bool = False):
     def after_callback(result: Tuple[AWSResponse, dict], *args, **kwargs):
@@ -140,6 +146,10 @@ def _patch_make_request(client: botocore.client.BaseClient, redirect: bool = Fal
             request_dict["body"].seek(0)
 
     def before_callback(operation_model, request_dict, request_context):
+        if operation_model.name in CALCULATE_MD5_FOR_OPERATIONS:
+            if "Content-MD5" not in request_dict["headers"]:
+                md5_digest = botocore_calculate_md5(request_dict["body"])
+                request_dict["headers"]["Content-MD5"] = md5_digest
         _logger.debug(
             "send s3 request: %r, with parameters: %s",
             operation_model.name,

megfile/utils/__init__.py

@@ -266,7 +266,7 @@ def _get_class(cls_or_obj) -> type:
     return type(cls_or_obj)
 
 
-def calculate_md5(file_object):
+def calculate_md5(file_object) -> str:
     hash_md5 = hashlib.md5()  # nosec
     for chunk in iter(lambda: file_object.read(DEFAULT_HASH_BUFFER_SIZE), b""):
         hash_md5.update(chunk)

megfile/utils/atomic.py

@@ -133,15 +133,14 @@ class AtomicTextIOWrapper(TextIOWrapper):
         super().__exit__(exc_type, exc_val, exc_tb)
 
     def __del__(self):
+        if self.closed:
+            return
+
         if self.atomic:
             if self.abort():
                 _logger.warning(
                     f"skip closing atomic file-like object before deletion: {self}"
                 )
             return
-        try:
-            if self._raw.writable():
-                self.flush()
-            self.close()
-        except Exception:
-            pass
+
+        self.close()

megfile/version.py

@@ -1 +1 @@
-VERSION = "5.0.4.post1"
+VERSION = "5.0.6"

{megfile-5.0.4.post1.dist-info → megfile-5.0.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: megfile
-Version: 5.0.4.post1
+Version: 5.0.6
 Summary: Megvii file operation library
 Author-email: megvii-reng <megvii-reng@googlegroups.com>
 Project-URL: Homepage, https://github.com/megvii-research/megfile
@@ -210,6 +210,11 @@ You can use alias in path, like `tos://bucket/key`, the same as `s3+tos://bucket
 [![10GiB](https://github.com/megvii-research/megfile/blob/main/scripts/benchmark/10GiB.png?raw=true)](https://megvii-research.github.io/megfile/benchmark.html)
 [![10MiB](https://github.com/megvii-research/megfile/blob/main/scripts/benchmark/10MiB.png?raw=true)](https://megvii-research.github.io/megfile/benchmark.html)
 
+## Skill for AI coding
+```bash
+git clone -b gh-skill --single-branch https://github.com/megvii-research/megfile.git path/to/your/ai/skills/megfile
+```
+
 ## How to Contribute
 * We welcome everyone to contribute code to the `megfile` project, but the contributed code needs to meet the following conditions as much as possible:
 

{megfile-5.0.4.post1.dist-info → megfile-5.0.6.dist-info}/RECORD

@@ -1,5 +1,5 @@
 megfile/__init__.py,sha256=4XbMsR-lM7MxbnPGBI16m2sq6ghuA2-zZj2XF4bbX2Y,3291
-megfile/cli.py,sha256=TjUhfcOB_weEBa_e55nmiajq_XObAhQa7sx5oivrQCw,30620
+megfile/cli.py,sha256=Quj5uM7UUukiDGLiG6Z1BwtKrbaWw_dHHNiVD-l649E,31022
 megfile/config.py,sha256=K3B_o2dnI7qGsGnK8Jg18-S5YYLYuzskfNJowlSMkQM,5065
 megfile/errors.py,sha256=zKwM5r5j89mlbWZNeax26Hq63NmQhl9iGMfTtgyvYNA,16830
 megfile/fs_path.py,sha256=RxdhMDoc1HRmQtyaCehEGk_UJtHGLrrwiUIHrS4LJiY,41027
@@ -7,13 +7,13 @@ megfile/hdfs_path.py,sha256=PWqws54Ou136VxaYp9K_UFRr5BoiZWsO330n9ig5IG0,26338
 megfile/http_path.py,sha256=08OmzmRMyLSyq1Yr1K2HbzexesURJrIoA6AibwYzUiA,13844
 megfile/interfaces.py,sha256=XU46U5pl4k1Gse63i4z5SvxcjWeKLj0xyB0Y6fYiWWo,9887
 megfile/pathlike.py,sha256=4RuYHqUc5_6rZDCcVo_18il0Hy7BlOYt-rtYwCtp9Gg,31446
-megfile/s3_path.py,sha256=C6z6pqZb0LMlNqUbZiaC8_deKX3feGxfPfSYjusqJq0,94507
+megfile/s3_path.py,sha256=yScDLBUomWqK7ph8n_xXN2nj1m8RjGxvnyQ2tMFjAZE,94954
 megfile/sftp2_path.py,sha256=K90bnMVAx0MQPGXP6LogGuDRzaD4MPR6lMOfdY9C9-0,37942
 megfile/sftp_path.py,sha256=zxuT1hk7sgoOUwq6KBXS__caX8Hk_LgPjINQheTZWAU,52063
 megfile/smart.py,sha256=Lab2jxprj-zvPw5GqUWlWiEY8bcpRlviks_qp9r-km8,38224
 megfile/smart_path.py,sha256=kGidkM5S58ChE3LVZMcUACs3IQgsqh9m04sp6-wxuhk,12615
 megfile/stdio_path.py,sha256=cxaDr8rtisTPnN-rjtaEpqQnshwiqwXFUJBM9xWY7Cg,2711
-megfile/version.py,sha256=MnVREJsp4Ik25QAKczs0kjFzRp3Aegh4XyEkdhOqwzI,25
+megfile/version.py,sha256=BZ8yMCCXoKX9dKkNpe3u2B651Z01j5nsP3q98uQPOtk,19
 megfile/webdav_path.py,sha256=QrRYKBGWXkUZXEeHxAfVJkxnCfnczocBSRkVgDC_qC4,31421
 megfile/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 megfile/lib/base_memory_handler.py,sha256=VEfw2RuynHFPtQ6KZv8hv8rnUwsj566Omg4cHajhARk,2863
@@ -40,13 +40,13 @@ megfile/lib/stdio_handler.py,sha256=IDdgENLQlhigEwkLL4zStueVSzdWg7xVcTF_koof_Ek,
 megfile/lib/url.py,sha256=ER32pWy9Q2MAk3TraAaNEBWIqUeBmLuM57ol2cs7-Ks,103
 megfile/lib/webdav_memory_handler.py,sha256=7nq4o69ck_7dFh6xlYTBG8-rj49Q7gxwa3V2bHXEQz4,2551
 megfile/lib/webdav_prefetch_reader.py,sha256=M0X6E6t-DS5q9KiLvjVZx_AZuiW9SaIkBnIPLc774GQ,3941
-megfile/utils/__init__.py,sha256=lfJze58nO18ug8EUfSJgTTxOwj1p7FQdsnO1keBeMSo,12740
-megfile/utils/atomic.py,sha256=1skKRE7_tQP1FAgmYWlDRZyHNY5iyYYi0V8XQfSWeyI,4165
+megfile/utils/__init__.py,sha256=dCyiRgTYScx8FdxWOP2-N1gONbBg76Snrv8tuJ7Ykek,12747
+megfile/utils/atomic.py,sha256=V3gqUQjHIx5j6LCrU2_0tLVlX1ossetxdJJK48g9-rc,4084
 megfile/utils/mutex.py,sha256=asb8opGLgK22RiuBJUnfsvB8LnMmodP8KzCVHKmQBWA,2561
-megfile-5.0.4.post1.dist-info/licenses/LICENSE,sha256=xuY_rHyygMLmf0LgkKj_-wb-BxveHp9rTN0VDE73PrE,11365
-megfile-5.0.4.post1.dist-info/licenses/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
-megfile-5.0.4.post1.dist-info/METADATA,sha256=3GY4kMU9g3APFaQ2a7z0cexM4SX0HOesW2WOKdbCiMs,9232
-megfile-5.0.4.post1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-megfile-5.0.4.post1.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
-megfile-5.0.4.post1.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
-megfile-5.0.4.post1.dist-info/RECORD,,
+megfile-5.0.6.dist-info/licenses/LICENSE,sha256=xuY_rHyygMLmf0LgkKj_-wb-BxveHp9rTN0VDE73PrE,11365
+megfile-5.0.6.dist-info/licenses/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
+megfile-5.0.6.dist-info/METADATA,sha256=E6WNTVXh9RB1rnOZRIc33ERNyuMw1P2JX4l1i25ROqE,9378
+megfile-5.0.6.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+megfile-5.0.6.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
+megfile-5.0.6.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
+megfile-5.0.6.dist-info/RECORD,,