megfile 3.0.2__py3-none-any.whl → 3.0.3__py3-none-any.whl

megfile/hdfs_path.py

@@ -14,7 +14,7 @@ from megfile.lib.hdfs_tools import hdfs_api
 from megfile.lib.url import get_url_scheme
 from megfile.pathlike import PathLike, URIPath
 from megfile.smart_path import SmartPath
-from megfile.utils import cachedproperty
+from megfile.utils import _is_pickle, cachedproperty
 
 __all__ = [
     'HdfsPath',
@@ -603,6 +603,8 @@ class HdfsPath(URIPath):
                     client=self._client,
                     profile_name=self._profile_name,
                     **input_kwargs)
+                if _is_pickle(file_obj):  # pytype: disable=wrong-arg-types
+                    file_obj = io.BufferedReader(file_obj)  # pytype: disable=wrong-arg-types
                 if 'b' not in mode:
                     file_obj = io.TextIOWrapper(
                         file_obj, encoding=encoding, errors=errors)  # pytype: disable=wrong-arg-types

megfile/http_path.py

@@ -16,7 +16,7 @@ from megfile.lib.s3_buffered_writer import DEFAULT_MAX_BUFFER_SIZE
 from megfile.lib.url import get_url_scheme
 from megfile.pathlike import PathLike
 from megfile.smart_path import SmartPath
-from megfile.utils import binary_open
+from megfile.utils import _is_pickle, binary_open
 
 __all__ = [
     'HttpPath',
@@ -223,7 +223,7 @@ class HttpPath(URIPath):
             else:
                 block_forward = max(int(block_capacity * forward_ratio), 1)
 
-            return HttpPrefetchReader(
+            reader = HttpPrefetchReader(
                 self.path_with_protocol,
                 content_size=content_size,
                 max_retries=max_retries,
@@ -232,6 +232,10 @@ class HttpPath(URIPath):
                 block_forward=block_forward,
                 block_size=block_size,
             )
+            if _is_pickle(reader):  # pytype: disable=wrong-arg-types
+                reader = io.BufferedReader(reader)  # pytype: disable=wrong-arg-types
+            return reader
+
         response.raw.auto_close = False
         response.raw.name = self.path_with_protocol
         return BufferedReader(response.raw)

megfile/lib/base_prefetch_reader.py

@@ -1,5 +1,5 @@
 import os
-from abc import ABC, abstractmethod, abstractproperty
+from abc import ABC, abstractmethod
 from collections import OrderedDict
 from concurrent.futures import Future, ThreadPoolExecutor
 from io import BytesIO
@@ -85,7 +85,8 @@ class BasePrefetchReader(Readable, Seekable, ABC):
     def _get_futures(self):
         return LRUCacheFutureManager()
 
-    @abstractproperty
+    @property
+    @abstractmethod
     def name(self) -> str:
         pass
 

megfile/s3_path.py

@@ -31,7 +31,7 @@ from megfile.lib.s3_prefetch_reader import S3PrefetchReader
 from megfile.lib.s3_share_cache_reader import S3ShareCacheReader
 from megfile.lib.url import get_url_scheme
 from megfile.smart_path import SmartPath
-from megfile.utils import cachedproperty, calculate_md5, generate_cache_path, get_binary_mode, get_content_offset, is_readable, necessary_params, process_local, thread_local
+from megfile.utils import _is_pickle, cachedproperty, calculate_md5, generate_cache_path, get_binary_mode, get_content_offset, is_readable, necessary_params, process_local, thread_local
 
 __all__ = [
     'S3Path',
@@ -461,7 +461,10 @@ def _s3_glob_stat_single_path(
         return False
 
     def create_generator(_s3_pathname) -> Iterator[FileEntry]:
-        if not S3Path(top_dir).exists():
+        top_dir_with_profile = top_dir
+        if profile_name:
+            top_dir_with_profile = f's3+{profile_name}://{top_dir[5:]}'
+        if not S3Path(top_dir_with_profile).exists():
             return
         if not has_magic(_s3_pathname):
             _s3_pathname_obj = S3Path(_s3_pathname)
@@ -772,7 +775,7 @@ def s3_buffered_open(
         forward_ratio: Optional[float] = None,
         block_size: int = DEFAULT_BLOCK_SIZE,
         limited_seekable: bool = False,
-        buffered: bool = True,
+        buffered: bool = False,
         share_cache_key: Optional[str] = None,
         cache_path: Optional[str] = None
 ) -> Union[S3PrefetchReader, S3BufferedWriter, io.BufferedReader, io.
@@ -855,7 +858,7 @@ def s3_buffered_open(
                 block_forward=block_forward,
                 block_size=block_size,
                 profile_name=s3_url._profile_name)
-        if buffered:
+        if buffered or _is_pickle(reader):  # pytype: disable=wrong-arg-types
             reader = io.BufferedReader(reader)  # pytype: disable=wrong-arg-types
         return reader
 
@@ -877,7 +880,7 @@ def s3_buffered_open(
             max_buffer_size=max_buffer_size,
             block_size=block_size,
             profile_name=s3_url._profile_name)
-    if buffered:
+    if buffered or _is_pickle(writer):  # pytype: disable=wrong-arg-types
         writer = io.BufferedWriter(writer)  # pytype: disable=wrong-arg-types
     return writer
 

megfile/utils/__init__.py

@@ -6,9 +6,6 @@ import uuid
 from copy import copy
 from functools import wraps
 from io import BufferedIOBase, BufferedRandom, BufferedReader, BufferedWriter, BytesIO, StringIO, TextIOBase, TextIOWrapper
-from multiprocessing.util import register_after_fork
-from threading import RLock as _RLock
-from threading import local
 from typing import IO, Callable, Optional
 
 from megfile.utils.mutex import ProcessLocal, ThreadLocal
@@ -72,6 +69,20 @@ def is_writable(fileobj: IO) -> bool:
     return hasattr(fileobj, 'write')
 
 
+def _is_pickle(fileobj: IO) -> bool:
+    ''' Test if File Object is pickle'''
+    if fileobj.name.endswith('.pkl') or fileobj.name.endswith('.pickle'):
+        return True
+
+    if 'r' in fileobj.mode and 'b' in fileobj.mode:
+        offset = fileobj.tell()
+        data = fileobj.read(2)
+        fileobj.seek(offset)
+        if len(data) >= 2 and data[0] == 128 and 2 <= data[1] <= 5:
+            return True
+    return False
+
+
 def get_content_offset(start: Optional[int], stop: Optional[int], size: int):
     if start is None:
         start = 0
@@ -94,7 +105,7 @@ def get_mode(fileobj, default='r'):
     return getattr(fileobj, 'mode', default)
 
 
-def shadow_copy(fileobj: IO, intrusive: bool = True, buffered: bool = True):
+def shadow_copy(fileobj: IO, intrusive: bool = True, buffered: bool = False):
     ''' Create a File-Like Object, maintaining file pointer, to avoid misunderstanding the position when read / write / seek
 
     :param intrusive: If is intrusive. If True, move file pointer to the original position after every read / write / seek. If False, then not.
@@ -103,7 +114,7 @@ def shadow_copy(fileobj: IO, intrusive: bool = True, buffered: bool = True):
     from megfile.lib.shadow_handler import ShadowHandler
     result = ShadowHandler(fileobj, intrusive=intrusive)
     mode = get_mode(fileobj)
-    if buffered and "b" in mode:
+    if "b" in mode and (buffered or _is_pickle(result)):  # pytype: disable=wrong-arg-types
         if "+" in mode:
             result = BufferedRandom(result)
         elif "x" in mode or "w" in mode or "a" in mode:

megfile/version.py

@@ -1 +1 @@
-VERSION = "3.0.2"
+VERSION = "3.0.3"

{megfile-3.0.2.dist-info → megfile-3.0.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: megfile
-Version: 3.0.2
+Version: 3.0.3
 Summary: Megvii file operation library
 Home-page: https://github.com/megvii-research/megfile
 Author: megvii

{megfile-3.0.2.dist-info → megfile-3.0.3.dist-info}/RECORD

@@ -5,22 +5,22 @@ megfile/errors.py,sha256=Sbx3UEKnzuyUmB1tFU9cZv61Yr4dRa79J6D0UMmkvj4,13323
 megfile/fs.py,sha256=OfY0z4GSl8fT3mDGdeqP2hWFsd1QJl-h8RkSbg6-M8I,11547
 megfile/fs_path.py,sha256=sHn-sBcvq7SvYN71onkA_ssLs71NzM1MM3d3Sug8uzo,38237
 megfile/hdfs.py,sha256=aAkHobOO0nDcLoqj0tx_1tvgoLOCooTWuukq0pO-nQA,9156
-megfile/hdfs_path.py,sha256=rVmdHydhe0x6Vn7hblewghiWZIdZswe4hfxLOHm8vCM,26677
+megfile/hdfs_path.py,sha256=obfMMKSuBcSGgtEN18jXhEldm4hawMhxk8QoRv4k790,26859
 megfile/http.py,sha256=a3oAuARSSaIU8VMx86Mui0N5Vh-EI0AoHnwxRU5DSMU,2032
-megfile/http_path.py,sha256=ciqS32V0G3A7GVOU_lqVzRjZeyr9skXfsU87SPrkyS4,11396
+megfile/http_path.py,sha256=WJd8-s_xsF71rC2QtXikH--FCsqn2u-e83YF8OarKeY,11593
 megfile/interfaces.py,sha256=h3tWE8hVt5S-HopaMAX6lunPJ97vzhv6jH_2HubcDNc,6219
 megfile/pathlike.py,sha256=Ere6tMf2nsI7bDsZo0WBzl_2HRrS_4iKOpYp0zZltAU,29487
 megfile/s3.py,sha256=siBZfveWX1TDA4Mp41UvugcG3zlrhl_iPUbixUp1TmI,12352
-megfile/s3_path.py,sha256=FYGjifwbhfraWZKKGRcuRAyllJwFcv_H1wjN5kYGzcw,90791
+megfile/s3_path.py,sha256=vypCpnNzkObng6BVcq4j3Y_W3J2btq16B8154bWVPjg,91067
 megfile/sftp.py,sha256=JCkF2v1ZbHuIy_Bg3l85AesjFDimDzx9Gh1gRoMsahc,12524
 megfile/sftp_path.py,sha256=ErPKmwgaCOvvhp3aKhqX9WKIAGbWR30QUWvptQWtag8,51666
 megfile/smart.py,sha256=y5Dzr7_f0jS2FJDF4tWbEO4SPf39zqYftqkVgMhiJds,33725
 megfile/smart_path.py,sha256=Y0UFh4J2ccydRY2W-wX2ubaf9zzJx1M2nf-VLBGe4mk,6749
 megfile/stdio.py,sha256=yRhlfUA2DHi3bq-9cXsSlbLCnHvS_zvglO2IYYyPsGc,707
 megfile/stdio_path.py,sha256=eQulTXUwHvUKA-5PKCGfVNiEPkJhG9YtVhtU58OcmoM,2873
-megfile/version.py,sha256=HGcItPfK5-rkjDD8Y1bmaADbd3qo7wyXrM39-uif4RY,19
+megfile/version.py,sha256=UE7hhbkRaCCOli6ABmNbLoQp1rQZubN3Q_YGWfVH4sY,19
 megfile/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-megfile/lib/base_prefetch_reader.py,sha256=9HT0tcgXa95BdbyclpDEDw8WwKi6091GRQerS90-pjE,13191
+megfile/lib/base_prefetch_reader.py,sha256=WKwrpuniO8a1iU2k5mIHEpuBP8caLtIeqLrpKbmqX_A,13185
 megfile/lib/combine_reader.py,sha256=XFSqEY5A5X5Uf7eQ6AXAzrvNteESSXvKNVPktGjo3KY,4546
 megfile/lib/compare.py,sha256=yG2fZve_gMg32rQVCdwixBdqgYRsjn-24TqhALQaOrA,2233
 megfile/lib/compat.py,sha256=0wt3_atcYhSLCxUj_WuDlQa3E1atjZfwJQ12thiFh5Q,234
@@ -41,12 +41,12 @@ megfile/lib/s3_share_cache_reader.py,sha256=ecx-62wGqgTn5vHIpKhNACCHdYWm6oc1_xwy
 megfile/lib/shadow_handler.py,sha256=IbFyTw107t-yWH0cGrDjAJX-CS3xeEr77_PTGsnSgk4,2683
 megfile/lib/stdio_handler.py,sha256=QDWtcZxz-hzi-rqQUiSlR3NrihX1fjK_Rj9T2mdTFEg,2044
 megfile/lib/url.py,sha256=VbQLjo0s4AaV0iSk66BcjI68aUTcN9zBZ5x6-cM4Qvs,103
-megfile/utils/__init__.py,sha256=qdX8FF_dYFKwp1BIWx3JeSGd91s7AKUDSEpDv9tORcM,9162
+megfile/utils/__init__.py,sha256=xrBIJcVJTb_yR68ekAyd9u4HQ46s3xgIjUZoH_Lx7hU,9531
 megfile/utils/mutex.py,sha256=-2KH3bNovKRd9zvsXq9n3bWM7rQdoG9hO7tUPxVG_Po,2538
-megfile-3.0.2.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-megfile-3.0.2.dist-info/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
-megfile-3.0.2.dist-info/METADATA,sha256=DnkmGRjSQJP0v3CgkHNwp7th5pKwJcF7qyKUN1tKaDs,8916
-megfile-3.0.2.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-megfile-3.0.2.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
-megfile-3.0.2.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
-megfile-3.0.2.dist-info/RECORD,,
+megfile-3.0.3.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+megfile-3.0.3.dist-info/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
+megfile-3.0.3.dist-info/METADATA,sha256=icEVsbrmXVTDGlH9axr1walAIB0pHrCPb6Jj0FpPAhA,8916
+megfile-3.0.3.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+megfile-3.0.3.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
+megfile-3.0.3.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
+megfile-3.0.3.dist-info/RECORD,,