megfile 3.1.0.post1__py3-none-any.whl → 3.1.1__py3-none-any.whl

megfile/lib/base_prefetch_reader.py

@@ -47,8 +47,15 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
         if block_forward is None:
             block_forward = max(block_capacity - 1, 1)
 
-        assert block_capacity > block_forward, 'block_capacity should greater than block_forward, got: block_capacity=%s, block_forward=%s' % (
-            block_capacity, block_forward)
+        if block_capacity <= block_forward:
+            # TODO: replace AssertionError with ValueError in 4.0.0
+            raise AssertionError(
+                'block_capacity should greater than block_forward, '
+                'got: block_capacity=%s, block_forward=%s' %
+                (block_capacity, block_forward))
+
+        # user maybe put block_size with 'numpy.uint64' type
+        block_size = int(block_size)
 
         self._max_retries = max_retries
         self._block_size = block_size
@@ -123,9 +130,9 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
 
         Returns the new absolute position.
         '''
+        offset = int(offset)  # user maybe put offset with 'numpy.uint64' type
         if self.closed:
             raise IOError('file already closed: %r' % self.name)
-
         if whence == os.SEEK_CUR:
             target_offset = self._offset + offset
         elif whence == os.SEEK_END:
@@ -159,10 +166,9 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
         if self._offset >= self._content_size:
             return b''
 
-        if size is None:
+        if size is None or size < 0:
             size = self._content_size - self._offset
         else:
-            assert size >= 0, 'size should greater than 0, got: %r' % size
             size = min(size, self._content_size - self._offset)
 
         if self._block_forward == 1:
@@ -196,6 +202,7 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
 
         Retain newline.  A non-negative size argument limits the maximum
         number of bytes to return (an incomplete line may be returned then).
+        If the size argument is negative, read until EOF is reached.
         Return an empty bytes object at EOF.
         '''
         if self.closed:
@@ -206,10 +213,9 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
         if self._offset >= self._content_size:
             return b''
 
-        if size is None:
+        if size is None or size < 0:
             size = self._content_size - self._offset
         else:
-            assert size >= 0, 'size should greater than 0, got: %r' % size
             size = min(size, self._content_size - self._offset)
 
         data = self._buffer.readline(size)

megfile/lib/combine_reader.py

@@ -99,6 +99,7 @@ class CombineReader(Readable, Seekable):
         return buffer.getvalue()  # pyre-ignore[7]
 
     def seek(self, offset: int, whence: int = os.SEEK_SET) -> int:
+        offset = int(offset)  # user maybe put offset with 'numpy.uint64' type
         if whence == os.SEEK_SET:
             target_offset = offset
         elif whence == os.SEEK_CUR:

megfile/lib/glob.py

@@ -74,7 +74,9 @@ def iglob(
     it = _iglob(pathname, recursive, False, fs)
     if recursive and _isrecursive(pathname):
         s = next(it)  # skip empty string
-        assert not s
+        if s:
+            # TODO: replace AssertionError with OSError in 4.0.0
+            raise AssertionError("iglob with recursive=True error")
     return it
 
 
@@ -88,7 +90,10 @@ def _iglob(pathname: str, recursive: bool, dironly: bool,
     if protocol:
         dirname = "://".join([protocol, dirname])
     if not has_magic(pathname):
-        assert not dironly
+        if dironly:
+            # TODO: replace AssertionError with OSError in 4.0.0
+            raise AssertionError(
+                "can't use dironly with non-magic patterns in _iglob")
         if basename:
             if fs.exists(pathname):
                 yield pathname
@@ -150,7 +155,9 @@ def _glob0(dirname: str, basename: str, dironly: bool, fs: FSFunc) -> List[str]:
 # directory.
 def _glob2(dirname: str, pattern: str, dironly: bool,
            fs: FSFunc) -> Iterator[str]:
-    assert _isrecursive(pattern)
+    if not _isrecursive(pattern):
+        # TODO: replace AssertionError with OSError in 4.0.0
+        raise AssertionError("error call '_glob2' with non-glob pattern")
     yield pattern[:0]
     yield from _rlistdir(dirname, dironly, fs)
 

megfile/lib/s3_buffered_writer.py

@@ -53,7 +53,9 @@ class S3BufferedWriter(Writable[bytes]):
         self._client = s3_client
         self._profile_name = profile_name
 
-        self._block_size = block_size
+        # user maybe put block_size with 'numpy.uint64' type
+        self._block_size = int(block_size)
+
         self._max_block_size = max_block_size
         self._max_buffer_size = max_buffer_size
         self._total_buffer_size = 0

megfile/lib/s3_cached_handler.py

@@ -19,7 +19,9 @@ class S3CachedHandler(S3MemoryHandler):
             remove_cache_when_open: bool = True,
             profile_name: Optional[str] = None):
 
-        assert mode in ('rb', 'wb', 'ab', 'rb+', 'wb+', 'ab+')
+        if mode not in ('rb', 'wb', 'ab', 'rb+', 'wb+', 'ab+'):
+            # TODO: replace AssertionError with ValueError in 4.0.0
+            raise AssertionError('unacceptable mode: %r' % mode)
 
         self._bucket = bucket
         self._key = key

megfile/lib/s3_limited_seekable_writer.py

@@ -63,6 +63,7 @@ class S3LimitedSeekableWriter(S3BufferedWriter, Seekable):
         if self.closed:
             raise IOError('file already closed: %r' % self.name)
 
+        offset = int(offset)  # user maybe put offset with 'numpy.uint64' type
         if whence == os.SEEK_SET:
             target_offset = offset
         elif whence == os.SEEK_CUR:

megfile/lib/s3_memory_handler.py

@@ -17,7 +17,9 @@ class S3MemoryHandler(Readable[bytes], Seekable, Writable[bytes]):
             s3_client,
             profile_name: Optional[str] = None):
 
-        assert mode in ('rb', 'wb', 'ab', 'rb+', 'wb+', 'ab+')
+        if mode not in ('rb', 'wb', 'ab', 'rb+', 'wb+', 'ab+'):
+            # TODO: replace AssertionError with ValueError in 4.0.0
+            raise AssertionError('unacceptable mode: %r' % mode)
 
         self._bucket = bucket
         self._key = key

megfile/lib/s3_pipe_handler.py

@@ -36,7 +36,9 @@ class S3PipeHandler(Readable[bytes], Writable[bytes]):
             join_thread: bool = True,
             profile_name: Optional[str] = None):
 
-        assert mode in ('rb', 'wb')
+        if mode not in ('rb', 'wb'):
+            # TODO: replace AssertionError with ValueError in 4.0.0
+            raise AssertionError('unacceptable mode: %r' % mode)
 
         self._bucket = bucket
         self._key = key

megfile/lib/shadow_handler.py

@@ -44,6 +44,7 @@ class ShadowHandler(Readable, Seekable, Writable, BaseShadowHandler):
         return get_content_size(self._file_object, intrusive=self._intrusive)
 
     def seek(self, offset: int, whence: int = os.SEEK_SET) -> int:
+        offset = int(offset)  # user maybe put offset with 'numpy.uint64' type
         if whence == os.SEEK_SET:
             self._offset = offset
         elif whence == os.SEEK_CUR:

megfile/s3_path.py

@@ -779,12 +779,12 @@ def s3_buffered_open(
         max_concurrency: Optional[int] = None,
         max_buffer_size: int = DEFAULT_MAX_BUFFER_SIZE,
         forward_ratio: Optional[float] = None,
-        block_size: int = DEFAULT_BLOCK_SIZE,
+        block_size: Optional[int] = None,
         limited_seekable: bool = False,
         buffered: bool = False,
         share_cache_key: Optional[str] = None,
         cache_path: Optional[str] = None,
-        min_block_size: int = DEFAULT_MIN_BLOCK_SIZE,
+        min_block_size: Optional[int] = None,
         max_block_size: int = DEFAULT_MAX_BLOCK_SIZE) -> IO:
     '''Open an asynchronous prefetch reader, to support fast sequential read
 
@@ -814,6 +814,8 @@ def s3_buffered_open(
             s3_url = s3_url.readlink()
         except S3NotALinkError:
             pass
+    min_block_size = min_block_size or block_size or DEFAULT_MIN_BLOCK_SIZE
+    block_size = block_size or DEFAULT_BLOCK_SIZE
 
     bucket, key = parse_s3_url(s3_url.path_with_protocol)
     config = botocore.config.Config(max_pool_connections=max_pool_connections)
@@ -1124,9 +1126,9 @@ class S3Cacher(FileCacher):
             self, path: str, cache_path: Optional[str] = None, mode: str = 'r'):
         if mode not in ('r', 'w', 'a'):
             raise ValueError('unacceptable mode: %r' % mode)
+        if cache_path is None:
+            cache_path = generate_cache_path(path)
         if mode in ('r', 'a'):
-            if cache_path is None:
-                cache_path = generate_cache_path(path)
             s3_download(path, cache_path)
         self.name = path
         self.mode = mode

megfile/smart.py

@@ -966,9 +966,9 @@ class SmartCacher(FileCacher):
             self, path: str, cache_path: Optional[str] = None, mode: str = 'r'):
         if mode not in ('r', 'w', 'a'):
             raise ValueError('unacceptable mode: %r' % mode)
+        if cache_path is None:
+            cache_path = generate_cache_path(path)
         if mode in ('r', 'a'):
-            if cache_path is None:
-                cache_path = generate_cache_path(path)
             smart_copy(path, cache_path)
         self.name = path
         self.mode = mode

megfile/utils/__init__.py

@@ -190,7 +190,9 @@ def binary_open(open_func):
 
 def get_human_size(size_bytes: float) -> str:
     '''Get human-readable size, e.g. `100MB`'''
-    assert size_bytes >= 0, 'negative size: %r' % size_bytes
+    if size_bytes < 0:
+        # TODO: replace AssertionError with ValueError in 4.0.0
+        raise AssertionError('negative size: %r' % size_bytes)
     if size_bytes == 0:
         return '0 B'
     size_name = ('B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB')

megfile/version.py

@@ -1 +1 @@
-VERSION = "3.1.0.post1"
+VERSION = "3.1.1"

{megfile-3.1.0.post1.dist-info → megfile-3.1.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: megfile
-Version: 3.1.0.post1
+Version: 3.1.1
 Summary: Megvii file operation library
 Author-email: megvii <megfile@megvii.com>
 Project-URL: Homepage, https://github.com/megvii-research/megfile

{megfile-3.1.0.post1.dist-info → megfile-3.1.1.dist-info}/RECORD

@@ -12,44 +12,44 @@ megfile/http_path.py,sha256=mnoN5hHJro23cLny1r5ePMJ28r2gnBrzqA0S-grpwrE,16215
 megfile/interfaces.py,sha256=PLNv0hDBTrTFJGTwT5mbDKYTsCjNryrHHMJCYhL9Wp0,8522
 megfile/pathlike.py,sha256=FjY88X1IA4-ue5jcekwlIO740D0oT_wNo95y3eo6vNA,30685
 megfile/s3.py,sha256=JfYWFHNJrBAoncB5RPDMyhjrIwVYGZ2lh8prGst-L_M,12874
-megfile/s3_path.py,sha256=WvP6AsFpS0bR11ABbOYSMUOVm3H2ffYcKrBbXxZJdrI,93001
+megfile/s3_path.py,sha256=UV-rOV-j1uWhTjsClqNmYbqpPrrBd0CZldo717pd63o,93107
 megfile/sftp.py,sha256=cXS1LZh4gtJgDCmR9p2_uM1wlycV34ebuTJGaceatVE,12984
 megfile/sftp_path.py,sha256=QcV4DK192c5GL2CeK07JgpELpBXpGVAKswQ1JtZ2DZQ,53336
-megfile/smart.py,sha256=3Rm-nI3HCLdljdo3mGn1JWFnw1qGwuxUunb1MIEJK0I,36032
+megfile/smart.py,sha256=jOTBBbDKpLuRbob9-Omj_GRsk66Vpql7LPJX4xCm5fs,36024
 megfile/smart_path.py,sha256=OMTM9JeowIa6j2Yn8XEH-HS4gWD3gBkKgSj8GBD08Bk,6734
 megfile/stdio.py,sha256=rsBMGr4LaFYOQjL-6k4GE_WznLsa8cPzMhSVNEEDh5o,671
 megfile/stdio_path.py,sha256=2n3gTEN1-7d8cCRsQJ7zj8rdP1VPlgr78oSClYndxXI,2806
-megfile/version.py,sha256=HgxsH8zUeKbwsczoAuIOhPyGbI_oqu-LqLFys9BLbJQ,25
+megfile/version.py,sha256=_DO4G2zWDcG7gSWIzdTDMHAx4XjUc5Vq4U89Oj2xeWY,19
 megfile/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-megfile/lib/base_prefetch_reader.py,sha256=iTHdWvpERMELgv2sEAsf9tiiJFDYp2mHzodszFrB9T4,13213
-megfile/lib/combine_reader.py,sha256=dV1AblB8LTHK2a4aM03w1eBEcTpxMBWln9g_fuz0v7Y,4561
+megfile/lib/base_prefetch_reader.py,sha256=Di2xCkf-Pci7BvJs5sosymKT8sJfdgYY8fTHIk_LEBc,13471
+megfile/lib/combine_reader.py,sha256=ePUMwDxdL1oPqgZLAj7RvHeJ5NROcjPWNwLMuiv7Z5M,4641
 megfile/lib/compare.py,sha256=yG2fZve_gMg32rQVCdwixBdqgYRsjn-24TqhALQaOrA,2233
 megfile/lib/compat.py,sha256=0wt3_atcYhSLCxUj_WuDlQa3E1atjZfwJQ12thiFh5Q,234
 megfile/lib/fnmatch.py,sha256=qtlTdOoowgUvQuZNDrzvEl7hA4Sdtifa9eXd47LEJZY,4082
-megfile/lib/glob.py,sha256=bqJ13xIBTbgS1mnqUZcI1Tga2M6TvJrWZP9vmFL2P0U,9724
+megfile/lib/glob.py,sha256=imsjV_lgCLteEacrId0uJU7w77A0tpRrAcKtWfUrzDU,10149
 megfile/lib/hdfs_prefetch_reader.py,sha256=ytp0td83M8NheMCBtQjrQpvGaBvConpRqgd8xSqdi-E,2092
 megfile/lib/hdfs_tools.py,sha256=t4GeoBxO0HPahIQDrsK17WBsLZtcfAaNwWfappzZ5q8,442
 megfile/lib/http_prefetch_reader.py,sha256=Owc_eaDGS752QnzatOnZ3zp19DuuftxWPS8kTXidi1A,4359
 megfile/lib/joinpath.py,sha256=D4Px6-lnDDpYs1LMUHkTIGqMPJQ0oCBGfTzREs373iU,929
 megfile/lib/lazy_handler.py,sha256=V_dQR2ac7LXJ6u__R5_VauvmxqQVdjydNIKwJjYUBeQ,1862
-megfile/lib/s3_buffered_writer.py,sha256=i1cmUePalrhVXIzT_H-vS1mmhh4Bh_9QDrLJs8UwA1s,7060
-megfile/lib/s3_cached_handler.py,sha256=7VmDgMjv9PlL_gBoCkEMJx2PQxUoo_dEPQLOTVNu1Bk,1396
-megfile/lib/s3_limited_seekable_writer.py,sha256=_uDEtfWluurqULAtsWa6HbexKH3xcJG-ukhQUgfPryk,6239
-megfile/lib/s3_memory_handler.py,sha256=HaQMzjtbhFHsjpymQ9lAXHQo7drSj7-q-Amo5Z2j_bw,4091
-megfile/lib/s3_pipe_handler.py,sha256=BicJ0EyOwBDWIpvpSWk-ZWDrUAd2QZlTR3LVHl4_pPs,3571
+megfile/lib/s3_buffered_writer.py,sha256=UW_C6dtvzIEydapsHnPYWMn2xmz0Ah0jD7hrdUxqcoM,7127
+megfile/lib/s3_cached_handler.py,sha256=1laCBQxG_PMp7Tk9_wY9XXDF-Ev4EREt1LPojetiA6A,1530
+megfile/lib/s3_limited_seekable_writer.py,sha256=6Hzy8sdiRDQMQ0os175weu5vYUfcEP-L4ztknCQETf8,6318
+megfile/lib/s3_memory_handler.py,sha256=02yivynhVWLKWtdf0_3N1-ipYOSGgFEmutYkpXzrXUU,4225
+megfile/lib/s3_pipe_handler.py,sha256=9dcW0WKmjETLjK-m6QW4EvuvaEjCqUI9BavOeWmuB6Y,3705
 megfile/lib/s3_prefetch_reader.py,sha256=ofzSn_MC6INAZY83oA3Dq1QNI0SS2b-NP6Ov8ajjLos,4283
 megfile/lib/s3_share_cache_reader.py,sha256=AFwpyFQq4y-PXTnbFoXxLMQOJFIcJXUgVTrDMCuAMUg,3629
-megfile/lib/shadow_handler.py,sha256=w0K2g31598DPonJORa-wTB5B34hKq2Zt7IIYDZsuhOY,2643
+megfile/lib/shadow_handler.py,sha256=Kb4CcfoMBdp822A5teSjNVG_6waAy5AUUHrW1qaRqEI,2722
 megfile/lib/stdio_handler.py,sha256=t2hxBE1vo25-kS6QQRMiDEG95rdMVx2z6wTnwuN6NUc,1968
 megfile/lib/url.py,sha256=VbQLjo0s4AaV0iSk66BcjI68aUTcN9zBZ5x6-cM4Qvs,103
-megfile/utils/__init__.py,sha256=QFDDhVAmptdMzYE91O6GRARHTalj_ZfSKZNF71uCj7w,8857
+megfile/utils/__init__.py,sha256=5gf_KfoqCvX6sVNLLZYUsGJL_zzesuQPj0znGC-HG8I,8946
 megfile/utils/mutex.py,sha256=-2KH3bNovKRd9zvsXq9n3bWM7rQdoG9hO7tUPxVG_Po,2538
 scripts/convert_results_to_sarif.py,sha256=SI0vw4TrpJ9gAZ2pH8ywu-lXMYeZ-TYW2dLEHw5Rx84,4218
 scripts/generate_file.py,sha256=OLAH9iZKQlHTjuV8zpC6exCXLS6bNdu4Njoc3IKO9Sc,10173
-megfile-3.1.0.post1.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-megfile-3.1.0.post1.dist-info/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
-megfile-3.1.0.post1.dist-info/METADATA,sha256=0Q1RFftQQ8ewo_hz4GleF1x5ucyCzIPbMseirlbX6ls,9112
-megfile-3.1.0.post1.dist-info/WHEEL,sha256=Z4pYXqR_rTB7OWNDYFOm1qRk0RX6GFP2o8LgvP453Hk,91
-megfile-3.1.0.post1.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
-megfile-3.1.0.post1.dist-info/top_level.txt,sha256=oTnYXo1Z3V61qSWAKtnY9RkDgRSHvfRN38FQae6E0W0,50
-megfile-3.1.0.post1.dist-info/RECORD,,
+megfile-3.1.1.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+megfile-3.1.1.dist-info/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
+megfile-3.1.1.dist-info/METADATA,sha256=honpFcwkQykVvHhlx_PQlEfOh6TgkPgk2FbdspMjc64,9106
+megfile-3.1.1.dist-info/WHEEL,sha256=Wyh-_nZ0DJYolHNn1_hMa4lM7uDedD_RGVwbmTjyItk,91
+megfile-3.1.1.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
+megfile-3.1.1.dist-info/top_level.txt,sha256=oTnYXo1Z3V61qSWAKtnY9RkDgRSHvfRN38FQae6E0W0,50
+megfile-3.1.1.dist-info/RECORD,,

{megfile-3.1.0.post1.dist-info → megfile-3.1.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (70.3.0)
+Generator: setuptools (71.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any