megfile 3.0.2.post1__py3-none-any.whl → 3.0.4__py3-none-any.whl

megfile/hdfs_path.py

@@ -14,7 +14,7 @@ from megfile.lib.hdfs_tools import hdfs_api
 from megfile.lib.url import get_url_scheme
 from megfile.pathlike import PathLike, URIPath
 from megfile.smart_path import SmartPath
-from megfile.utils import cachedproperty
+from megfile.utils import _is_pickle, cachedproperty
 
 __all__ = [
     'HdfsPath',
@@ -603,6 +603,8 @@ class HdfsPath(URIPath):
                     client=self._client,
                     profile_name=self._profile_name,
                     **input_kwargs)
+                if _is_pickle(file_obj):  # pytype: disable=wrong-arg-types
+                    file_obj = io.BufferedReader(file_obj)  # pytype: disable=wrong-arg-types
                 if 'b' not in mode:
                     file_obj = io.TextIOWrapper(
                         file_obj, encoding=encoding, errors=errors)  # pytype: disable=wrong-arg-types

megfile/http_path.py

@@ -16,7 +16,7 @@ from megfile.lib.s3_buffered_writer import DEFAULT_MAX_BUFFER_SIZE
 from megfile.lib.url import get_url_scheme
 from megfile.pathlike import PathLike
 from megfile.smart_path import SmartPath
-from megfile.utils import binary_open
+from megfile.utils import _is_pickle, binary_open
 
 __all__ = [
     'HttpPath',
@@ -223,7 +223,7 @@ class HttpPath(URIPath):
             else:
                 block_forward = max(int(block_capacity * forward_ratio), 1)
 
-            return HttpPrefetchReader(
+            reader = HttpPrefetchReader(
                 self.path_with_protocol,
                 content_size=content_size,
                 max_retries=max_retries,
@@ -232,6 +232,10 @@ class HttpPath(URIPath):
                 block_forward=block_forward,
                 block_size=block_size,
             )
+            if _is_pickle(reader):  # pytype: disable=wrong-arg-types
+                reader = io.BufferedReader(reader)  # pytype: disable=wrong-arg-types
+            return reader
+
         response.raw.auto_close = False
         response.raw.name = self.path_with_protocol
         return BufferedReader(response.raw)

megfile/lib/base_prefetch_reader.py

@@ -1,5 +1,5 @@
 import os
-from abc import ABC, abstractmethod, abstractproperty
+from abc import ABC, abstractmethod
 from collections import OrderedDict
 from concurrent.futures import Future, ThreadPoolExecutor
 from io import BytesIO
@@ -85,7 +85,8 @@ class BasePrefetchReader(Readable, Seekable, ABC):
     def _get_futures(self):
         return LRUCacheFutureManager()
 
-    @abstractproperty
+    @property
+    @abstractmethod
     def name(self) -> str:
         pass
 

megfile/s3_path.py

@@ -31,7 +31,7 @@ from megfile.lib.s3_prefetch_reader import S3PrefetchReader
 from megfile.lib.s3_share_cache_reader import S3ShareCacheReader
 from megfile.lib.url import get_url_scheme
 from megfile.smart_path import SmartPath
-from megfile.utils import cachedproperty, calculate_md5, generate_cache_path, get_binary_mode, get_content_offset, is_readable, necessary_params, process_local, thread_local
+from megfile.utils import _is_pickle, cachedproperty, calculate_md5, generate_cache_path, get_binary_mode, get_content_offset, is_readable, necessary_params, process_local, thread_local
 
 __all__ = [
     'S3Path',
@@ -138,15 +138,20 @@ def get_endpoint_url(profile_name: Optional[str] = None) -> str:
 
     :returns: S3 endpoint url
     '''
-    environ_key = f'{profile_name}__OSS_ENDPOINT'.upper(
-    ) if profile_name else 'OSS_ENDPOINT'
-    environ_endpoint_url = os.environ.get(environ_key)
-    if environ_endpoint_url:
-        warning_endpoint_url(environ_key, environ_endpoint_url)
-        return environ_endpoint_url
+    if profile_name:
+        environ_keys = (f'{profile_name}__OSS_ENDPOINT'.upper(),)
+    else:
+        environ_keys = (
+            'OSS_ENDPOINT', 'AWS_ENDPOINT_URL_S3', 'AWS_ENDPOINT_URL')
+    for environ_key in environ_keys:
+        environ_endpoint_url = os.environ.get(environ_key)
+        if environ_endpoint_url:
+            warning_endpoint_url(environ_key, environ_endpoint_url)
+            return environ_endpoint_url
     try:
-        config_endpoint_url = get_scoped_config(profile_name=profile_name).get(
-            's3', {}).get('endpoint_url')
+        config = get_scoped_config(profile_name=profile_name)
+        config_endpoint_url = config.get('s3', {}).get('endpoint_url')
+        config_endpoint_url = config_endpoint_url or config.get('endpoint_url')
         if config_endpoint_url:
             warning_endpoint_url('~/.aws/config', config_endpoint_url)
             return config_endpoint_url
@@ -775,7 +780,7 @@ def s3_buffered_open(
         forward_ratio: Optional[float] = None,
         block_size: int = DEFAULT_BLOCK_SIZE,
         limited_seekable: bool = False,
-        buffered: bool = True,
+        buffered: bool = False,
         share_cache_key: Optional[str] = None,
         cache_path: Optional[str] = None
 ) -> Union[S3PrefetchReader, S3BufferedWriter, io.BufferedReader, io.
@@ -858,7 +863,7 @@ def s3_buffered_open(
                 block_forward=block_forward,
                 block_size=block_size,
                 profile_name=s3_url._profile_name)
-        if buffered:
+        if buffered or _is_pickle(reader):  # pytype: disable=wrong-arg-types
             reader = io.BufferedReader(reader)  # pytype: disable=wrong-arg-types
         return reader
 
@@ -880,7 +885,7 @@ def s3_buffered_open(
             max_buffer_size=max_buffer_size,
             block_size=block_size,
             profile_name=s3_url._profile_name)
-    if buffered:
+    if buffered or _is_pickle(writer):  # pytype: disable=wrong-arg-types
         writer = io.BufferedWriter(writer)  # pytype: disable=wrong-arg-types
     return writer
 

megfile/utils/__init__.py

@@ -6,9 +6,6 @@ import uuid
 from copy import copy
 from functools import wraps
 from io import BufferedIOBase, BufferedRandom, BufferedReader, BufferedWriter, BytesIO, StringIO, TextIOBase, TextIOWrapper
-from multiprocessing.util import register_after_fork
-from threading import RLock as _RLock
-from threading import local
 from typing import IO, Callable, Optional
 
 from megfile.utils.mutex import ProcessLocal, ThreadLocal
@@ -72,6 +69,20 @@ def is_writable(fileobj: IO) -> bool:
     return hasattr(fileobj, 'write')
 
 
+def _is_pickle(fileobj: IO) -> bool:
+    ''' Test if File Object is pickle'''
+    if fileobj.name.endswith('.pkl') or fileobj.name.endswith('.pickle'):
+        return True
+
+    if 'r' in fileobj.mode and 'b' in fileobj.mode:
+        offset = fileobj.tell()
+        data = fileobj.read(2)
+        fileobj.seek(offset)
+        if len(data) >= 2 and data[0] == 128 and 2 <= data[1] <= 5:
+            return True
+    return False
+
+
 def get_content_offset(start: Optional[int], stop: Optional[int], size: int):
     if start is None:
         start = 0
@@ -94,7 +105,7 @@ def get_mode(fileobj, default='r'):
     return getattr(fileobj, 'mode', default)
 
 
-def shadow_copy(fileobj: IO, intrusive: bool = True, buffered: bool = True):
+def shadow_copy(fileobj: IO, intrusive: bool = True, buffered: bool = False):
     ''' Create a File-Like Object, maintaining file pointer, to avoid misunderstanding the position when read / write / seek
 
     :param intrusive: If is intrusive. If True, move file pointer to the original position after every read / write / seek. If False, then not.
@@ -103,7 +114,7 @@ def shadow_copy(fileobj: IO, intrusive: bool = True, buffered: bool = True):
     from megfile.lib.shadow_handler import ShadowHandler
     result = ShadowHandler(fileobj, intrusive=intrusive)
     mode = get_mode(fileobj)
-    if buffered and "b" in mode:
+    if "b" in mode and (buffered or _is_pickle(result)):  # pytype: disable=wrong-arg-types
         if "+" in mode:
             result = BufferedRandom(result)
         elif "x" in mode or "w" in mode or "a" in mode:

megfile/version.py

@@ -1 +1 @@
-VERSION = "3.0.2.post1"
+VERSION = "3.0.4"

{megfile-3.0.2.post1.dist-info → megfile-3.0.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: megfile
-Version: 3.0.2.post1
+Version: 3.0.4
 Summary: Megvii file operation library
 Home-page: https://github.com/megvii-research/megfile
 Author: megvii
@@ -160,10 +160,10 @@ Using `s3` as an example, the following describes the configuration methods. For
 You can use environments and configuration file for configuration, and priority is that environment variables take precedence over configuration file.
 
 ### Use environments
-You can use environments to setup authentication credentials for your s3 account:
+You can use environments to setup authentication credentials for your `s3` account:
 - `AWS_ACCESS_KEY_ID`: access key
 - `AWS_SECRET_ACCESS_KEY`: secret key
-- `OSS_ENDPOINT`: endpoint url of s3
+- `OSS_ENDPOINT` / `AWS_ENDPOINT_URL_S3` / `AWS_ENDPOINT_URL`: endpoint url of s3
 - `AWS_S3_ADDRESSING_STYLE`: addressing style
 
 ### Use command

{megfile-3.0.2.post1.dist-info → megfile-3.0.4.dist-info}/RECORD

@@ -5,22 +5,22 @@ megfile/errors.py,sha256=Sbx3UEKnzuyUmB1tFU9cZv61Yr4dRa79J6D0UMmkvj4,13323
 megfile/fs.py,sha256=OfY0z4GSl8fT3mDGdeqP2hWFsd1QJl-h8RkSbg6-M8I,11547
 megfile/fs_path.py,sha256=sHn-sBcvq7SvYN71onkA_ssLs71NzM1MM3d3Sug8uzo,38237
 megfile/hdfs.py,sha256=aAkHobOO0nDcLoqj0tx_1tvgoLOCooTWuukq0pO-nQA,9156
-megfile/hdfs_path.py,sha256=rVmdHydhe0x6Vn7hblewghiWZIdZswe4hfxLOHm8vCM,26677
+megfile/hdfs_path.py,sha256=obfMMKSuBcSGgtEN18jXhEldm4hawMhxk8QoRv4k790,26859
 megfile/http.py,sha256=a3oAuARSSaIU8VMx86Mui0N5Vh-EI0AoHnwxRU5DSMU,2032
-megfile/http_path.py,sha256=ciqS32V0G3A7GVOU_lqVzRjZeyr9skXfsU87SPrkyS4,11396
+megfile/http_path.py,sha256=WJd8-s_xsF71rC2QtXikH--FCsqn2u-e83YF8OarKeY,11593
 megfile/interfaces.py,sha256=h3tWE8hVt5S-HopaMAX6lunPJ97vzhv6jH_2HubcDNc,6219
 megfile/pathlike.py,sha256=Ere6tMf2nsI7bDsZo0WBzl_2HRrS_4iKOpYp0zZltAU,29487
 megfile/s3.py,sha256=siBZfveWX1TDA4Mp41UvugcG3zlrhl_iPUbixUp1TmI,12352
-megfile/s3_path.py,sha256=Rkl3cppYn8Ri1nTd_ILts0bHwtkhaLTMzj-q2pWxvEA,90940
+megfile/s3_path.py,sha256=spksQesniQOYWmfIPfJeU4kMSiFAIlB8v2yIHVRLliA,91305
 megfile/sftp.py,sha256=JCkF2v1ZbHuIy_Bg3l85AesjFDimDzx9Gh1gRoMsahc,12524
 megfile/sftp_path.py,sha256=ErPKmwgaCOvvhp3aKhqX9WKIAGbWR30QUWvptQWtag8,51666
 megfile/smart.py,sha256=y5Dzr7_f0jS2FJDF4tWbEO4SPf39zqYftqkVgMhiJds,33725
 megfile/smart_path.py,sha256=Y0UFh4J2ccydRY2W-wX2ubaf9zzJx1M2nf-VLBGe4mk,6749
 megfile/stdio.py,sha256=yRhlfUA2DHi3bq-9cXsSlbLCnHvS_zvglO2IYYyPsGc,707
 megfile/stdio_path.py,sha256=eQulTXUwHvUKA-5PKCGfVNiEPkJhG9YtVhtU58OcmoM,2873
-megfile/version.py,sha256=dhVp6aZyv8wvEr5NjELL4oQ0Jg_1yxifeqUh9OuUM_c,25
+megfile/version.py,sha256=DnboapDM00MYgi1PslOAcm9O5A6fEv0tG12NA4yH9TI,19
 megfile/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-megfile/lib/base_prefetch_reader.py,sha256=9HT0tcgXa95BdbyclpDEDw8WwKi6091GRQerS90-pjE,13191
+megfile/lib/base_prefetch_reader.py,sha256=WKwrpuniO8a1iU2k5mIHEpuBP8caLtIeqLrpKbmqX_A,13185
 megfile/lib/combine_reader.py,sha256=XFSqEY5A5X5Uf7eQ6AXAzrvNteESSXvKNVPktGjo3KY,4546
 megfile/lib/compare.py,sha256=yG2fZve_gMg32rQVCdwixBdqgYRsjn-24TqhALQaOrA,2233
 megfile/lib/compat.py,sha256=0wt3_atcYhSLCxUj_WuDlQa3E1atjZfwJQ12thiFh5Q,234
@@ -41,12 +41,12 @@ megfile/lib/s3_share_cache_reader.py,sha256=ecx-62wGqgTn5vHIpKhNACCHdYWm6oc1_xwy
 megfile/lib/shadow_handler.py,sha256=IbFyTw107t-yWH0cGrDjAJX-CS3xeEr77_PTGsnSgk4,2683
 megfile/lib/stdio_handler.py,sha256=QDWtcZxz-hzi-rqQUiSlR3NrihX1fjK_Rj9T2mdTFEg,2044
 megfile/lib/url.py,sha256=VbQLjo0s4AaV0iSk66BcjI68aUTcN9zBZ5x6-cM4Qvs,103
-megfile/utils/__init__.py,sha256=qdX8FF_dYFKwp1BIWx3JeSGd91s7AKUDSEpDv9tORcM,9162
+megfile/utils/__init__.py,sha256=xrBIJcVJTb_yR68ekAyd9u4HQ46s3xgIjUZoH_Lx7hU,9531
 megfile/utils/mutex.py,sha256=-2KH3bNovKRd9zvsXq9n3bWM7rQdoG9hO7tUPxVG_Po,2538
-megfile-3.0.2.post1.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-megfile-3.0.2.post1.dist-info/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
-megfile-3.0.2.post1.dist-info/METADATA,sha256=qDIxaqhLqBlaV0b6GCn0e7OJr0p6phvlFIhPeqpm508,8922
-megfile-3.0.2.post1.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-megfile-3.0.2.post1.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
-megfile-3.0.2.post1.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
-megfile-3.0.2.post1.dist-info/RECORD,,
+megfile-3.0.4.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+megfile-3.0.4.dist-info/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
+megfile-3.0.4.dist-info/METADATA,sha256=ULNUWUSidVxRQfUIJquDdSrV87Os1OQByXqRhO-7TPw,8963
+megfile-3.0.4.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+megfile-3.0.4.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
+megfile-3.0.4.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
+megfile-3.0.4.dist-info/RECORD,,