megfile 4.1.0__py3-none-any.whl → 4.1.0.post2__py3-none-any.whl

megfile/errors.py

@@ -273,8 +273,8 @@ class S3InvalidRangeError(S3Exception):
 
 
 class S3UnknownError(S3Exception, UnknownError):
-    def __init__(self, error: Exception, path: PathLike):
-        super().__init__(error, path, "endpoint: %r" % s3_endpoint_url(path))
+    def __init__(self, error: Exception, path: PathLike, extra: Optional[str] = None):
+        super().__init__(error, path, extra or "endpoint: %r" % s3_endpoint_url(path))
 
 
 class HttpException(Exception):

megfile/lib/base_prefetch_reader.py

@@ -38,6 +38,18 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
         max_workers: Optional[int] = None,
         **kwargs,
     ):
+        self._is_global_executor = False
+        if max_workers is None:
+            self._executor = process_local(
+                f"{self.__class__.__name__}.executor",
+                ThreadPoolExecutor,
+                max_workers=GLOBAL_MAX_WORKERS,
+            )
+            self._is_global_executor = True
+        else:
+            self._executor = ThreadPoolExecutor(max_workers=max_workers)
+        self._process_local = ProcessLocal()
+
         if max_buffer_size == 0:
             block_capacity = block_forward = 0
         else:
@@ -65,8 +77,6 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
         # Number of blocks every prefetch, which should be smaller than block_capacity
         self._block_forward = block_forward
 
-        self._process_local = ProcessLocal()
-
         self._content_size = self._get_content_size()
         self._block_stop = ceil(self._content_size / block_size)
 
@@ -75,16 +85,6 @@ class BasePrefetchReader(Readable[bytes], Seekable, ABC):
         self._block_index = None  # Current block index
         self._seek_history = []
 
-        self._is_global_executor = False
-        if max_workers is None:
-            self._executor = process_local(
-                f"{self.__class__.__name__}.executor",
-                ThreadPoolExecutor,
-                max_workers=GLOBAL_MAX_WORKERS,
-            )
-            self._is_global_executor = True
-        else:
-            self._executor = ThreadPoolExecutor(max_workers=max_workers)
         self._seek_buffer(0)
 
         _logger.debug("open file: %r, mode: %s" % (self.name, self.mode))

megfile/lib/glob.py

@@ -176,6 +176,7 @@ def _rlistdir(dirname: str, dironly: bool, fs: FSFunc) -> Iterator[str]:
 
 
 magic_check = re.compile(r"([*?[{])")
+magic_check_only_brace = re.compile(r"([{])")
 magic_decheck = re.compile(r"\[(.)\]")
 brace_check = re.compile(r"(\{.*\})")
 unbrace_check = re.compile(r"([*?[])")
@@ -215,6 +216,13 @@ def unescape(pathname):
     return drive + pathname
 
 
+def escape_brace(pathname):
+    """Escape brace."""
+    drive, pathname = os.path.splitdrive(pathname)
+    pathname = magic_check_only_brace.sub(r"[\1]", pathname)
+    return drive + pathname
+
+
 def _find_suffix(path_list: List[str], prefix: str, split_sign: str) -> List[str]:
     suffix = []
     temp_path_list = []
@@ -275,13 +283,15 @@ def ungloblize(glob: str) -> List[str]:
     while True:
         temp_path = path_list[0]
         begin = temp_path.find("{")
+        while temp_path[begin - 1 : begin + 2] == "[{]":
+            begin = temp_path.find("{", begin + 1)
         end = temp_path.find("}", begin)
         if end == -1:
             break
         path_list.pop(0)
         subpath_list = temp_path[begin + 1 : end].split(",")
         for subpath in subpath_list:
-            path = temp_path[:begin] + escape(subpath) + temp_path[end + 1 :]
+            path = temp_path[:begin] + escape_brace(subpath) + temp_path[end + 1 :]
             path_list.append(path)
     return path_list
 

megfile/lib/s3_memory_handler.py

@@ -130,6 +130,7 @@ class S3MemoryHandler(Readable[bytes], Seekable, Writable[bytes]):
             self._client.upload_fileobj(self._fileobj, self._bucket, self._key)
 
     def _close(self, need_upload: bool = True):
-        if need_upload:
-            self._upload_fileobj()
-        self._fileobj.close()
+        if hasattr(self, "_fileobj"):
+            if need_upload:
+                self._upload_fileobj()
+            self._fileobj.close()

megfile/lib/s3_pipe_handler.py

@@ -91,7 +91,7 @@ class S3PipeHandler(Readable[bytes], Writable[bytes]):
             self._exc = error
 
     def _raise_exception(self):
-        if self._exc is not None:
+        if getattr(self, "_exc", None) is not None:
             raise translate_s3_error(self._exc, self.name)
 
     def readable(self) -> bool:
@@ -121,8 +121,10 @@ class S3PipeHandler(Readable[bytes], Writable[bytes]):
         return self._fileobj.write(data)
 
     def _close(self):
-        self._fileobj.close()
-        if self._join_thread:
+        if hasattr(self, "_fileobj"):
+            self._fileobj.close()
+        if self._join_thread and hasattr(self, "_async_task"):
             self._async_task.join()
-        _s3_opened_pipes.remove(self._pipe)
+        if hasattr(self, "_pipe") and self._pipe in _s3_opened_pipes:
+            _s3_opened_pipes.remove(self._pipe)
         self._raise_exception()

megfile/s3_path.py

@@ -406,12 +406,11 @@ def _group_s3path_by_bucket(
             if len(split_bucket_name) == 2:
                 bucket_name, path_part = split_bucket_name
             pattern = re.compile(translate(re.sub(r"\*{2,}", "*", bucket_name)))
-
-            for bucket in all_bucket(profile_name):
-                if pattern.fullmatch(bucket) is not None:
+            for current_bucket in all_bucket(profile_name):
+                if pattern.fullmatch(current_bucket) is not None:
                     if path_part is not None:
-                        bucket = "%s/%s" % (bucket, path_part)
-                    grouped_path.append(generate_s3_path(bucket, key))
+                        current_bucket = "%s/%s" % (current_bucket, path_part)
+                    grouped_path.append(generate_s3_path(current_bucket, key))
         else:
             grouped_path.append(generate_s3_path(bucket_name, key))
 
@@ -1293,7 +1292,9 @@ class S3Cacher(FileCacher):
         self.cache_path = cache_path
 
     def _close(self):
-        if self.cache_path is not None and os.path.exists(self.cache_path):
+        if getattr(self, "cache_path", None) is not None and os.path.exists(
+            self.cache_path
+        ):
             if self.mode in ("w", "a"):
                 s3_upload(self.cache_path, self.name)
             os.unlink(self.cache_path)
@@ -1313,14 +1314,14 @@ def _group_src_paths_by_block(
             if len(groups) == 0:
                 if current_group_size + current_file_size > 2 * block_size:
                     group_lack_size = block_size - current_group_size
-                    current_group.append((src_path, f"bytes=0-{group_lack_size-1}"))
+                    current_group.append((src_path, f"bytes=0-{group_lack_size - 1}"))
                     groups.extend(
                         [
                             current_group,
                             [
                                 (
                                     src_path,
-                                    f"bytes={group_lack_size}-{current_file_size-1}",
+                                    f"bytes={group_lack_size}-{current_file_size - 1}",
                                 )
                             ],
                         ]
@@ -1386,7 +1387,7 @@ class S3Path(URIPath):
         if protocol.startswith("s3+"):
             self._protocol_with_profile = protocol
             self._profile_name = protocol[3:]
-            self._s3_path = f"s3://{self.path[len(protocol)+3:]}"
+            self._s3_path = f"s3://{self.path[len(protocol) + 3 :]}"
         elif not protocol:
             self._s3_path = f"s3://{self.path.lstrip('/')}"
         else:

megfile/smart.py

@@ -1052,7 +1052,9 @@ class SmartCacher(FileCacher):
         self.cache_path = cache_path
 
     def _close(self):
-        if self.cache_path is not None and os.path.exists(self.cache_path):
+        if getattr(self, "cache_path", None) is not None and os.path.exists(
+            self.cache_path
+        ):
             if self.mode in ("w", "a"):
                 smart_copy(self.cache_path, self.name)
             os.unlink(self.cache_path)

megfile/version.py

@@ -1 +1 @@
-VERSION = "4.1.0"
+VERSION = "4.1.0.post2"

{megfile-4.1.0.dist-info → megfile-4.1.0.post2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: megfile
-Version: 4.1.0
+Version: 4.1.0.post2
 Summary: Megvii file operation library
 Author-email: megvii <megfile@megvii.com>
 Project-URL: Homepage, https://github.com/megvii-research/megfile

{megfile-4.1.0.dist-info → megfile-4.1.0.post2.dist-info}/RECORD

@@ -1,7 +1,7 @@
 megfile/__init__.py,sha256=7oEfu410CFKzDWZ9RjL5xEJ1gtkJkTfvPrL_7TWdJuY,7366
 megfile/cli.py,sha256=Y7ZyWsYmzMUjUrJ1Vz-WMWJQcpK-4og_5xyRzfKI91U,24957
 megfile/config.py,sha256=2MMj5QkhlDJQFZRbCQL2c9iDdeMAVctiaPszRBkg5vM,3988
-megfile/errors.py,sha256=su85hEFEk18gwUJW_GNdgCe4FeXuuy7Vf3b1pmNkHIk,14347
+megfile/errors.py,sha256=qSIiqiJmDLb2k1qKiTLdNQzj4VNgfM9bHkSClyCp1Kg,14385
 megfile/fs.py,sha256=TJ0ifJ57IEL-gFgQhJD_WVEqmf28zC5kAvmH6c1QzbU,18240
 megfile/fs_path.py,sha256=Hozl9LAJ8EMuSWBSZXGj2GNmPZ1sJp9PZs-7hPrLgm8,39341
 megfile/hdfs.py,sha256=owXr4d3j1frCvlbhmhENcSBnKKDky5cJZzWLOF4ZJMo,13251
@@ -11,21 +11,21 @@ megfile/http_path.py,sha256=yRIk-fNbrsY8rUS5KVOfocS_PS520dX5KOs8lImpLaY,14173
 megfile/interfaces.py,sha256=p4UvVZpeLx5djd6bqqDaygIx_s-_AxIVj-gudTch4JE,8467
 megfile/pathlike.py,sha256=3Hnw-fn6RcIe9iPrJt00QdHSA--UfDyxnVBuZ_ymYYQ,31278
 megfile/s3.py,sha256=abBxnI7RIyn7n7qjGszP1VruYd6Gi9I8QnUOvsHkx1Y,16325
-megfile/s3_path.py,sha256=zqXrS4aclIE0454AjMsD64Lxy3BlXlh4kAdi_5Bj19E,93558
+megfile/s3_path.py,sha256=zelXhlRVOVSWBE6HJz0vXrrcRzSuj6Cnjd9HLGwPbCM,93644
 megfile/sftp.py,sha256=uBcLQs-j6Q-q-sWAdd-pgi5Qmb_kq7boJM-0sCfcNO0,26540
 megfile/sftp_path.py,sha256=Wz4VcQ0pBUuWDGMSxPpPbutrT09mnY6jZNiAqTi5tO4,43840
-megfile/smart.py,sha256=xkNVEa3DAThM4zJf3aoHJg_t8PQeVv9yz66GhGIdhpA,36974
+megfile/smart.py,sha256=YXjCWp000wimSFXE8oqI6dIE8y0RZJo9I046zIkh0ag,37014
 megfile/smart_path.py,sha256=Bqg95T2-XZrRXWhH7GT-jMCYzD7i1SIXdczQxtOxiPs,7583
 megfile/stdio.py,sha256=ZwxsnJNJYIT7Iyg5pIw4qiyH8bszG6oAhEJuR-hXGG4,658
 megfile/stdio_path.py,sha256=cxaDr8rtisTPnN-rjtaEpqQnshwiqwXFUJBM9xWY7Cg,2711
-megfile/version.py,sha256=5UUfPKQaA2LgHzk7jZgvPthpTs-MIguSBfg3cfykU3k,19
+megfile/version.py,sha256=bGSGJb1sKjmDAZKMCver-Yiujxub2hCFXyCS_i7sl5U,25
 megfile/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-megfile/lib/base_prefetch_reader.py,sha256=oSiQq6wlV9DADQapkf3Up3-86zl6QrtusVu93MAt1KE,12848
+megfile/lib/base_prefetch_reader.py,sha256=uxVwYknOjc8hLF7q_T2QKMsBqFcrf411ZsuK25CN1eQ,12848
 megfile/lib/combine_reader.py,sha256=Kp2wEloOUpTlIU7dve87MBpSzmIM-F9OtpTawAjFkiU,4828
 megfile/lib/compare.py,sha256=n_dtLxgoskYnsIZMKdKmVhQoVn8qYUrUhkS1JH2_X3o,2170
 megfile/lib/compat.py,sha256=SynEeHluys3tCK-lb_1oV3o_ft83yZvunqM_AjibLgE,207
 megfile/lib/fnmatch.py,sha256=4MvGzEahMRA-u8Z7mxaD-Yw1idOwBoJJpVywQy29jwY,4162
-megfile/lib/glob.py,sha256=1u0j_DoqT7VmTUsDAdcKhPbr4XOwIBpOkCnyAFEGw1I,9633
+megfile/lib/glob.py,sha256=-N75Phx1c8IpZ2hNIMwsHIx6BxMKVVr7N82cpWhs4lQ,9987
 megfile/lib/hdfs_prefetch_reader.py,sha256=yCNpcXcTiC2SHKHC-Qp50KQx1ObSLmOgwNUKlG-4ADg,2131
 megfile/lib/hdfs_tools.py,sha256=4K-OdMYFFSLBGmDzjatioHvuZuUbKVy7ACeJl-l0HLQ,435
 megfile/lib/http_prefetch_reader.py,sha256=OjP5pdWK_e1QiFIt2xmflceLnrz3S7B7ePBZFK-OwQE,4558
@@ -34,8 +34,8 @@ megfile/lib/lazy_handler.py,sha256=bE7RGt1x_xYWMgGAvHr7dwEt52qy-D3z90X3oyCvE6g,1
 megfile/lib/s3_buffered_writer.py,sha256=kXvz1bdoaVIxjnEQeg4dxEzAXMYNrk-5uIMww86ty00,7860
 megfile/lib/s3_cached_handler.py,sha256=MkNt6AAapd5x8BH2gnW5_S0cLofN-mshEdb0qSoLho8,1426
 megfile/lib/s3_limited_seekable_writer.py,sha256=joVcjoTx48jataXAwEqOI3toRxxO3-XQRvyWHtJ23lQ,6232
-megfile/lib/s3_memory_handler.py,sha256=_2JCLbbPUuIGGCeUEuSRZ-mdDRtGow2ofgtAQwzru4s,4136
-megfile/lib/s3_pipe_handler.py,sha256=QKhcjXWkedM4-eX0BhZVmkmCjawfyBJfYzmGFUclO8w,3642
+megfile/lib/s3_memory_handler.py,sha256=epIzQgTlE_deuVlP4LTmsh1T3cmAgK8ATEwrit9j7X8,4186
+megfile/lib/s3_pipe_handler.py,sha256=g3iAN1P9pCdvSNsGeJBGcBa10S62oqIg_9W3b3wc7os,3809
 megfile/lib/s3_prefetch_reader.py,sha256=R37-y_L9l8IKJhpT8HwBrZEbo2X72vCqEV6fvqPCBug,4437
 megfile/lib/s3_share_cache_reader.py,sha256=LVWKxHdHo0_zUIW4o8yqNvplqqwezUPeYEt02Vj-WNM,3754
 megfile/lib/shadow_handler.py,sha256=TntewlvIW9ZxCfmqASDQREHoiZ8v42faOe9sovQYQz0,2779
@@ -53,10 +53,10 @@ scripts/benchmark/code/s3fs_read.py,sha256=XiTA-qrYblUs-jQWXSnvNg5Wo722C_g47aMMf
 scripts/benchmark/code/s3fs_write.py,sha256=gdXKkWXYGjLJlRT_J64pJN85XvRg3bZexcAJQEMXwtw,402
 scripts/benchmark/code/smart_open_read.py,sha256=SA02jHwS9Y31yFtV9CoJcfND5dR0eA_HsGmGNUrpQls,515
 scripts/benchmark/code/smart_open_write.py,sha256=jDxFJdY97yNH889jz3pawBoei3yaqy8pEMvC_ymHFtM,537
-megfile-4.1.0.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-megfile-4.1.0.dist-info/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
-megfile-4.1.0.dist-info/METADATA,sha256=Q6vaVV6xtJSdmFIOdBaPHRyvhcVPhI0NUlUpwhUUZfA,9573
-megfile-4.1.0.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-megfile-4.1.0.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
-megfile-4.1.0.dist-info/top_level.txt,sha256=fVg49lk5B9L7jyfWUXWxb0DDSuw5pbr0OU62Tvx8J8M,44
-megfile-4.1.0.dist-info/RECORD,,
+megfile-4.1.0.post2.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+megfile-4.1.0.post2.dist-info/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
+megfile-4.1.0.post2.dist-info/METADATA,sha256=_oXpfxMbIgcf75-z8J-aZ2j-BcnxjmJkWax7Ruj1ctM,9579
+megfile-4.1.0.post2.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+megfile-4.1.0.post2.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
+megfile-4.1.0.post2.dist-info/top_level.txt,sha256=fVg49lk5B9L7jyfWUXWxb0DDSuw5pbr0OU62Tvx8J8M,44
+megfile-4.1.0.post2.dist-info/RECORD,,