megfile 4.2.2__py3-none-any.whl → 4.2.4__py3-none-any.whl

megfile/sftp_path.py

@@ -22,7 +22,7 @@ from megfile.lib.compat import fspath
 from megfile.lib.glob import FSFunc, iglob
 from megfile.pathlike import URIPath
 from megfile.smart_path import SmartPath
-from megfile.utils import calculate_md5, thread_local
+from megfile.utils import calculate_md5, copyfileobj, thread_local
 
 _logger = get_logger(__name__)
 
@@ -713,12 +713,7 @@ class SftpPath(URIPath):
                 if overwrite or not dst_path.exists():
                     with self.open("rb") as fsrc:
                         with dst_path.open("wb") as fdst:
-                            length = 16 * 1024
-                            while True:
-                                buf = fsrc.read(length)
-                                if not buf:
-                                    break
-                                fdst.write(buf)
+                            copyfileobj(fsrc, fdst)
                 self.unlink()
 
         dst_path.utime(src_stat.st_atime, src_stat.st_mtime)
@@ -1134,14 +1129,7 @@ class SftpPath(URIPath):
         else:
             with self.open("rb") as fsrc:
                 with dst_path.open("wb") as fdst:
-                    length = 16 * 1024
-                    while True:
-                        buf = fsrc.read(length)
-                        if not buf:
-                            break
-                        fdst.write(buf)
-                        if callback:
-                            callback(len(buf))
+                    copyfileobj(fsrc, fdst, callback)
 
         src_stat = self.stat()
         dst_path.utime(src_stat.st_atime, src_stat.st_mtime)

megfile/smart.py

@@ -45,7 +45,7 @@ from megfile.s3 import (
 )
 from megfile.sftp import sftp_concat, sftp_copy, sftp_download, sftp_upload
 from megfile.smart_path import SmartPath, get_traditional_path
-from megfile.utils import combine, generate_cache_path
+from megfile.utils import combine, copyfileobj, generate_cache_path
 
 __all__ = [
     "smart_access",
@@ -289,16 +289,7 @@ def _default_copy_func(
 
     with smart_open(src_path, "rb", followlinks=followlinks) as fsrc:
         with smart_open(dst_path, "wb") as fdst:
-            # This magic number is copied from  copyfileobj
-            length = 16 * 1024
-            while True:
-                buf = fsrc.read(length)
-                if not buf:
-                    break
-                fdst.write(buf)
-                if callback is None:
-                    continue
-                callback(len(buf))
+            copyfileobj(fsrc, fdst, callback)
     try:
         src_stat = smart_stat(src_path)
         dst_path = SmartPath(dst_path)
@@ -723,6 +714,8 @@ def smart_open(
         read mode. Only be used in s3, http, hdfs.
     :param block_size: Size of single block. Each block will be uploaded by single
         thread. Only be used in s3, http, hdfs.
+    :param buffered: If you are operating pickle file without .pkl or .pickle extension,
+        please set this to True to avoid the performance issue.
 
     :returns: File-Like object
     :raises: FileNotFoundError, IsADirectoryError, ValueError
@@ -1101,15 +1094,10 @@ _concat_funcs = {"s3": s3_concat, "sftp": sftp_concat}
 
 
 def _default_concat_func(src_paths: List[PathLike], dst_path: PathLike) -> None:
-    length = 16 * 1024
     with smart_open(dst_path, "wb") as dst_fd:
         for src_path in src_paths:
             with smart_open(src_path, "rb") as src_fd:
-                while True:
-                    buf = src_fd.read(length)
-                    if not buf:
-                        break
-                    dst_fd.write(buf)
+                copyfileobj(src_fd, dst_fd)
 
 
 def smart_concat(src_paths: List[PathLike], dst_path: PathLike) -> None:

megfile/utils/__init__.py

@@ -16,8 +16,13 @@ from io import (
     TextIOWrapper,
 )
 from threading import RLock
-from typing import IO, Callable, Optional
+from typing import IO, Callable, List, Optional
 
+from megfile.config import (
+    DEFAULT_COPY_BUFFER_SIZE,
+    DEFAULT_HASH_BUFFER_SIZE,
+    READER_LAZY_PREFETCH,
+)
 from megfile.utils.mutex import ProcessLocal, ThreadLocal
 
 
@@ -81,16 +86,22 @@ def is_writable(fileobj: IO) -> bool:
 
 def _is_pickle(fileobj) -> bool:
     """Test if File Object is pickle"""
+    if READER_LAZY_PREFETCH:
+        return False
+
     if fileobj.name.endswith(".pkl") or fileobj.name.endswith(".pickle"):
         return True
 
-    if "r" in fileobj.mode and "b" in fileobj.mode:
-        offset = fileobj.tell()
-        fileobj.seek(0)
-        data = fileobj.read(2)
-        fileobj.seek(offset)
-        if len(data) >= 2 and data[0] == 128 and 2 <= data[1] <= 5:
-            return True
+    # XXX: read 2 bytes will trigger pre read in prefetch reader,
+    #      so comment out the code for now.
+    #
+    # if "r" in fileobj.mode and "b" in fileobj.mode:
+    #     offset = fileobj.tell()
+    #     fileobj.seek(0)
+    #     data = fileobj.read(2)
+    #     fileobj.seek(offset)
+    #     if len(data) >= 2 and data[0] == 128 and 2 <= data[1] <= 5:
+    #         return True
     return False
 
 
@@ -257,7 +268,7 @@ def _get_class(cls_or_obj) -> type:
 
 def calculate_md5(file_object):
     hash_md5 = hashlib.md5()  # nosec
-    for chunk in iter(lambda: file_object.read(4096), b""):
+    for chunk in iter(lambda: file_object.read(DEFAULT_HASH_BUFFER_SIZE), b""):
         hash_md5.update(chunk)
     return hash_md5.hexdigest()
 
@@ -337,3 +348,75 @@ def is_domain_or_subdomain(sub, parent):
     if sub.endswith(f".{parent}"):
         return True
     return False
+
+
+def copyfileobj(
+    fsrc: IO,
+    fdst: IO,
+    callback: Optional[Callable[[int], None]] = None,
+    length: int = DEFAULT_COPY_BUFFER_SIZE,
+) -> None:
+    """Copy data from fsrc to fdst with optional progress callback.
+
+    This is similar to shutil.copyfileobj but with callback support.
+
+    Args:
+        fsrc: Source file-like object (opened for reading)
+        fdst: Destination file-like object (opened for writing)
+        callback: Optional callback function called with number of bytes written
+        length: Buffer size for copying (default: DEFAULT_COPY_BUFFER_SIZE)
+    """
+    while True:
+        buf = fsrc.read(length)
+        if not buf:
+            break
+        fdst.write(buf)
+        if callback:
+            callback(len(buf))
+
+
+def copyfd(
+    src_fd: int,
+    fdst: IO,
+    callback: Optional[Callable[[int], None]] = None,
+    length: int = DEFAULT_COPY_BUFFER_SIZE,
+) -> None:
+    """Copy data from file descriptor to file object with optional progress callback.
+
+    Args:
+        src_fd: Source file descriptor (integer)
+        fdst: Destination file-like object (opened for writing)
+        callback: Optional callback function called with number of bytes written
+        length: Buffer size for copying (default: DEFAULT_COPY_BUFFER_SIZE)
+    """
+    while True:
+        buf = os.read(src_fd, length)
+        if not buf:
+            break
+        fdst.write(buf)
+        if callback:
+            callback(len(buf))
+
+
+def copyfileobj_multi(
+    fsrc: IO,
+    fdst_list: List[IO],
+    callback: Optional[Callable[[int], None]] = None,
+    length: int = DEFAULT_COPY_BUFFER_SIZE,
+) -> None:
+    """Copy data from fsrc to multiple destinations with optional progress callback.
+
+    Args:
+        fsrc: Source file-like object (opened for reading)
+        fdst_list: List of destination file-like objects (opened for writing)
+        callback: Optional callback function called with number of bytes written
+        length: Buffer size for copying (default: DEFAULT_COPY_BUFFER_SIZE)
+    """
+    while True:
+        buf = fsrc.read(length)
+        if not buf:
+            break
+        for fdst in fdst_list:
+            fdst.write(buf)
+        if callback:
+            callback(len(buf))

megfile/version.py

@@ -1 +1 @@
-VERSION = "4.2.2"
+VERSION = "4.2.4"

{megfile-4.2.2.dist-info → megfile-4.2.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: megfile
-Version: 4.2.2
+Version: 4.2.4
 Summary: Megvii file operation library
 Author-email: megvii <megfile@megvii.com>
 Project-URL: Homepage, https://github.com/megvii-research/megfile
@@ -29,6 +29,8 @@ Requires-Dist: tqdm
 Requires-Dist: pyyaml
 Provides-Extra: hdfs
 Requires-Dist: hdfs; extra == "hdfs"
+Provides-Extra: sftp2
+Requires-Dist: ssh2-python>=1.0.0; extra == "sftp2"
 Provides-Extra: cli
 Requires-Dist: click; extra == "cli"
 Dynamic: license-file

{megfile-4.2.2.dist-info → megfile-4.2.4.dist-info}/RECORD

@@ -1,9 +1,9 @@
-megfile/__init__.py,sha256=7oEfu410CFKzDWZ9RjL5xEJ1gtkJkTfvPrL_7TWdJuY,7366
-megfile/cli.py,sha256=VxY0__M19Ti_S7ZIozp9l0FxvdLwdd6eQL-wfpYOi_0,29160
-megfile/config.py,sha256=2MMj5QkhlDJQFZRbCQL2c9iDdeMAVctiaPszRBkg5vM,3988
+megfile/__init__.py,sha256=K7ZXETL1HuXwR-xeeRWd7xtvDCCczB-DffkVTFlqxLY,7458
+megfile/cli.py,sha256=5HBBRu0T3lF53AhPJLQXF_pzHy0G6E5yH8m1e4NYfJg,29100
+megfile/config.py,sha256=0h5miOJo35Wdk67XgaFHieaVlbvaiciil9IjYkL1Geg,4260
 megfile/errors.py,sha256=cGSYyB7VBRKi1Gehgt9IO-wDvtzICV4XgKOkoMvLU5w,15583
 megfile/fs.py,sha256=KMEqAE35alpcxiy6du5nPFYcaorhUM_kPJMah3q76ng,19160
-megfile/fs_path.py,sha256=Hozl9LAJ8EMuSWBSZXGj2GNmPZ1sJp9PZs-7hPrLgm8,39341
+megfile/fs_path.py,sha256=J7rx6vkIFFbWFb2B2MHALa-p9yABLAMv-FUOqrg8VSk,39027
 megfile/hdfs.py,sha256=owXr4d3j1frCvlbhmhENcSBnKKDky5cJZzWLOF4ZJMo,13251
 megfile/hdfs_path.py,sha256=OmUe3vA3Qoxnqtcq0Rs3ygBvzAtqUz3fGo8iP5sWneE,26058
 megfile/http.py,sha256=1nuGe-JbnwMFyV3s35CJxByED3uoRoS9y8Y8cSGP9Kw,3865
@@ -11,21 +11,23 @@ megfile/http_path.py,sha256=08OmzmRMyLSyq1Yr1K2HbzexesURJrIoA6AibwYzUiA,13844
 megfile/interfaces.py,sha256=p4UvVZpeLx5djd6bqqDaygIx_s-_AxIVj-gudTch4JE,8467
 megfile/pathlike.py,sha256=3Hnw-fn6RcIe9iPrJt00QdHSA--UfDyxnVBuZ_ymYYQ,31278
 megfile/s3.py,sha256=abBxnI7RIyn7n7qjGszP1VruYd6Gi9I8QnUOvsHkx1Y,16325
-megfile/s3_path.py,sha256=fzqXahIEKGbyArU-2Yl5YpczwkImUiZy7Y1RYQOKNKM,93827
+megfile/s3_path.py,sha256=-CPqLQzhkqmmhWLTd6sbd1-JULzseDayhcH9HqQ_5LE,94072
 megfile/sftp.py,sha256=uBcLQs-j6Q-q-sWAdd-pgi5Qmb_kq7boJM-0sCfcNO0,26540
-megfile/sftp_path.py,sha256=CgirHWmNdXdqyIL9ufmlaMpwFhlkQVZhqmfvjUaj7qU,43845
-megfile/smart.py,sha256=GnabQVb_NU7a-etKfF-NgpJ9JM2rT0uZBeA9UoNz_wM,37014
+megfile/sftp2.py,sha256=9l6TJfXUAsTydgV60HXgpAaBNjfzSgDcpdUexhHu2Do,26935
+megfile/sftp2_path.py,sha256=D38d4q_b9N79Mwe7m348OQB9V9hPQOoSe-rpRXWa-6M,38675
+megfile/sftp_path.py,sha256=U-5NIVnqyoVEjQFEy8pkAeM1UGqPhaAzq5U_dkxK85c,43374
+megfile/smart.py,sha256=g9WeIxUsTa_4kuXpYJ65ZtqYfheqsaMZX-HUA1A7_jc,36731
 megfile/smart_path.py,sha256=Up_6xNZ2019iSzMn_JAU_1H--z-AP6O7SxdXGdeTG0c,7659
 megfile/stdio.py,sha256=ZwxsnJNJYIT7Iyg5pIw4qiyH8bszG6oAhEJuR-hXGG4,658
 megfile/stdio_path.py,sha256=cxaDr8rtisTPnN-rjtaEpqQnshwiqwXFUJBM9xWY7Cg,2711
-megfile/version.py,sha256=xnzpVSLKRvpgVFWjV-1xKKQitlJEcAepuzOjV7CqQ6I,19
+megfile/version.py,sha256=hnyIOxGx-9adm33CcvJetPS1oMoaaHkTCvydnNjsOLc,19
 megfile/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-megfile/lib/base_prefetch_reader.py,sha256=uxVwYknOjc8hLF7q_T2QKMsBqFcrf411ZsuK25CN1eQ,12848
+megfile/lib/base_prefetch_reader.py,sha256=MYaWOkXc3geZMYNPHlPZmmpOR6uSz-AMuCZwYdoz7t0,13296
 megfile/lib/combine_reader.py,sha256=Kp2wEloOUpTlIU7dve87MBpSzmIM-F9OtpTawAjFkiU,4828
-megfile/lib/compare.py,sha256=n_dtLxgoskYnsIZMKdKmVhQoVn8qYUrUhkS1JH2_X3o,2170
+megfile/lib/compare.py,sha256=CPSbyqsQ396oSfxa7h0NdUUqBw5A3WOn6fHrNKkuinw,2188
 megfile/lib/compat.py,sha256=SynEeHluys3tCK-lb_1oV3o_ft83yZvunqM_AjibLgE,207
 megfile/lib/fnmatch.py,sha256=4MvGzEahMRA-u8Z7mxaD-Yw1idOwBoJJpVywQy29jwY,4162
-megfile/lib/glob.py,sha256=anr7JjCJz1T0G5hYQkJwV5mCWzlt5YaxGTyyyQB28zA,9663
+megfile/lib/glob.py,sha256=BSpYm6w16rVuIBCI9hNOCxn83BN7UuKGBp2wFRFdwp4,9775
 megfile/lib/hdfs_prefetch_reader.py,sha256=yCNpcXcTiC2SHKHC-Qp50KQx1ObSLmOgwNUKlG-4ADg,2131
 megfile/lib/hdfs_tools.py,sha256=4K-OdMYFFSLBGmDzjatioHvuZuUbKVy7ACeJl-l0HLQ,435
 megfile/lib/http_prefetch_reader.py,sha256=OQPZ7kWFImqpynjaiTtmadtgtab5fCeQmu51UYHZfgs,4135
@@ -36,17 +38,17 @@ megfile/lib/s3_cached_handler.py,sha256=MkNt6AAapd5x8BH2gnW5_S0cLofN-mshEdb0qSoL
 megfile/lib/s3_limited_seekable_writer.py,sha256=joVcjoTx48jataXAwEqOI3toRxxO3-XQRvyWHtJ23lQ,6232
 megfile/lib/s3_memory_handler.py,sha256=epIzQgTlE_deuVlP4LTmsh1T3cmAgK8ATEwrit9j7X8,4186
 megfile/lib/s3_pipe_handler.py,sha256=g3iAN1P9pCdvSNsGeJBGcBa10S62oqIg_9W3b3wc7os,3809
-megfile/lib/s3_prefetch_reader.py,sha256=txJbXyxaFVS3wPvut5yW4ppVdLeDtVQIvCXFDlQfBMc,4474
-megfile/lib/s3_share_cache_reader.py,sha256=LVWKxHdHo0_zUIW4o8yqNvplqqwezUPeYEt02Vj-WNM,3754
+megfile/lib/s3_prefetch_reader.py,sha256=AqfADmbbZYA6nw4vxBOiFWX5q5CSYOd0hq1LWcf1PY0,4524
+megfile/lib/s3_share_cache_reader.py,sha256=8uip5IdVjPXCquXrskjocsZx2-TiXqWZPY0gX8JC144,4020
 megfile/lib/shadow_handler.py,sha256=TntewlvIW9ZxCfmqASDQREHoiZ8v42faOe9sovQYQz0,2779
 megfile/lib/stdio_handler.py,sha256=IDdgENLQlhigEwkLL4zStueVSzdWg7xVcTF_koof_Ek,1987
 megfile/lib/url.py,sha256=ER32pWy9Q2MAk3TraAaNEBWIqUeBmLuM57ol2cs7-Ks,103
-megfile/utils/__init__.py,sha256=pawmXnCNokWLj338a60b_hK21koYavpEiEohZhsOaGQ,10156
+megfile/utils/__init__.py,sha256=4hBVSXbNTbDj7Je0y9SbwgcPm_s41H9v3eHUMr9JNGo,12700
 megfile/utils/mutex.py,sha256=asb8opGLgK22RiuBJUnfsvB8LnMmodP8KzCVHKmQBWA,2561
-megfile-4.2.2.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-megfile-4.2.2.dist-info/licenses/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
-megfile-4.2.2.dist-info/METADATA,sha256=GCTiJb_unoYia-PwLFfqjOhm-7hjU73R-pJIpDChSAk,9601
-megfile-4.2.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-megfile-4.2.2.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
-megfile-4.2.2.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
-megfile-4.2.2.dist-info/RECORD,,
+megfile-4.2.4.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+megfile-4.2.4.dist-info/licenses/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
+megfile-4.2.4.dist-info/METADATA,sha256=7zDLJtMR3UfawCuutBu_LFtyKmeuXlVwnZJO6CY3nE8,9675
+megfile-4.2.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+megfile-4.2.4.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
+megfile-4.2.4.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
+megfile-4.2.4.dist-info/RECORD,,