megfile 4.1.1__py3-none-any.whl → 4.1.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (9) hide show
  1. megfile/errors.py +29 -20
  2. megfile/version.py +1 -1
  3. {megfile-4.1.1.dist-info → megfile-4.1.2.dist-info}/METADATA +3 -2
  4. {megfile-4.1.1.dist-info → megfile-4.1.2.dist-info}/RECORD +9 -9
  5. {megfile-4.1.1.dist-info → megfile-4.1.2.dist-info}/WHEEL +1 -1
  6. {megfile-4.1.1.dist-info → megfile-4.1.2.dist-info}/top_level.txt +0 -1
  7. {megfile-4.1.1.dist-info → megfile-4.1.2.dist-info}/entry_points.txt +0 -0
  8. {megfile-4.1.1.dist-info → megfile-4.1.2.dist-info/licenses}/LICENSE +0 -0
  9. {megfile-4.1.1.dist-info → megfile-4.1.2.dist-info/licenses}/LICENSE.pyre +0 -0
megfile/errors.py CHANGED
@@ -110,6 +110,33 @@ if hasattr(botocore.exceptions, "ResponseStreamingError"): # backport botocore=
110
110
  )
111
111
  s3_retry_exceptions = tuple(s3_retry_exceptions) # pyre-ignore[9]
112
112
 
113
+ s3_retry_error_codes = (
114
+ "429", # noqa: E501 # TOS ExceedAccountQPSLimit
115
+ "499", # noqa: E501 # Some cloud providers may send response with http code 499 if the connection not send data in 1 min.
116
+ "500",
117
+ "501",
118
+ "502",
119
+ "503",
120
+ "InternalError",
121
+ "ServiceUnavailable",
122
+ "SlowDown",
123
+ "ContextCanceled",
124
+ "Timeout", # noqa: E501 # TOS Timeout
125
+ "RequestTimeout",
126
+ "RequestTimeTooSkewed",
127
+ "ExceedAccountQPSLimit",
128
+ "ExceedAccountRateLimit",
129
+ "ExceedBucketQPSLimit",
130
+ "ExceedBucketRateLimit",
131
+ "DownloadTrafficRateLimitExceeded", # noqa: E501 # OSS RateLimitExceeded
132
+ "UploadTrafficRateLimitExceeded",
133
+ "MetaOperationQpsLimitExceeded",
134
+ "TotalQpsLimitExceeded",
135
+ "ActiveRequestLimitExceeded",
136
+ "CpuLimitExceeded",
137
+ "QpsLimitExceeded",
138
+ )
139
+
113
140
 
114
141
  def s3_should_retry(error: Exception) -> bool:
115
142
  if isinstance(error, s3_retry_exceptions): # pyre-ignore[6]
@@ -117,25 +144,7 @@ def s3_should_retry(error: Exception) -> bool:
117
144
  if isinstance(error, botocore.exceptions.SSLError):
118
145
  return "EOF" in str(error)
119
146
  if isinstance(error, botocore.exceptions.ClientError):
120
- return client_error_code(error) in (
121
- "429", # noqa: E501 # TOS ExceedAccountQPSLimit
122
- "499", # noqa: E501 # Some cloud providers may send response with http code 499 if the connection not send data in 1 min.
123
- "500",
124
- "501",
125
- "502",
126
- "503",
127
- "InternalError",
128
- "ServiceUnavailable",
129
- "SlowDown",
130
- "ContextCanceled",
131
- "Timeout", # noqa: E501 # TOS Timeout
132
- "RequestTimeout",
133
- "RequestTimeTooSkewed",
134
- "ExceedAccountQPSLimit",
135
- "ExceedAccountRateLimit",
136
- "ExceedBucketQPSLimit",
137
- "ExceedBucketRateLimit",
138
- )
147
+ return client_error_code(error) in s3_retry_error_codes
139
148
  return False
140
149
 
141
150
 
@@ -427,7 +436,7 @@ def raise_s3_error(s3_url: PathLike, suppress_error_callback=None):
427
436
 
428
437
 
429
438
  def s3_error_code_should_retry(error: str) -> bool:
430
- if error in ["InternalError", "ServiceUnavailable", "SlowDown"]:
439
+ if error in s3_retry_error_codes:
431
440
  return True
432
441
  return False
433
442
 
megfile/version.py CHANGED
@@ -1 +1 @@
1
- VERSION = "4.1.1"
1
+ VERSION = "4.1.2"
{megfile-4.1.1.dist-info → megfile-4.1.2.dist-info}/METADATA RENAMED
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.2
1
+ Metadata-Version: 2.4
2
2
  Name: megfile
3
- Version: 4.1.1
3
+ Version: 4.1.2
4
4
  Summary: Megvii file operation library
5
5
  Author-email: megvii <megfile@megvii.com>
6
6
  Project-URL: Homepage, https://github.com/megvii-research/megfile
@@ -31,6 +31,7 @@ Provides-Extra: hdfs
31
31
  Requires-Dist: hdfs; extra == "hdfs"
32
32
  Provides-Extra: cli
33
33
  Requires-Dist: click; extra == "cli"
34
+ Dynamic: license-file
34
35
 
35
36
  megfile - Megvii FILE library
36
37
  ---
{megfile-4.1.1.dist-info → megfile-4.1.2.dist-info}/RECORD RENAMED
@@ -1,7 +1,7 @@
1
1
  megfile/__init__.py,sha256=7oEfu410CFKzDWZ9RjL5xEJ1gtkJkTfvPrL_7TWdJuY,7366
2
2
  megfile/cli.py,sha256=X9nWGke-VM-FCFdRoEoxVH_SNFkKQesBxhSrRv7jAsI,25884
3
3
  megfile/config.py,sha256=2MMj5QkhlDJQFZRbCQL2c9iDdeMAVctiaPszRBkg5vM,3988
4
- megfile/errors.py,sha256=ZHp9zEltrmuUqP4Jtg8SY31cQeU0UKzKp1h9Jj8197E,14512
4
+ megfile/errors.py,sha256=-HuHCMj6AUPlXZnTWlUScZPoiw_DS6xx0NKsSuhztxw,14646
5
5
  megfile/fs.py,sha256=KMEqAE35alpcxiy6du5nPFYcaorhUM_kPJMah3q76ng,19160
6
6
  megfile/fs_path.py,sha256=Hozl9LAJ8EMuSWBSZXGj2GNmPZ1sJp9PZs-7hPrLgm8,39341
7
7
  megfile/hdfs.py,sha256=owXr4d3j1frCvlbhmhENcSBnKKDky5cJZzWLOF4ZJMo,13251
@@ -18,7 +18,7 @@ megfile/smart.py,sha256=Sae2KJzaU0k_qV_Bk0YifOMq8WsV5qQ2pGInDRF546I,36411
18
18
  megfile/smart_path.py,sha256=HqCOlDwekqqIyJAll-U9YKmaXjjfCGZD5n5aG80lOKw,7592
19
19
  megfile/stdio.py,sha256=ZwxsnJNJYIT7Iyg5pIw4qiyH8bszG6oAhEJuR-hXGG4,658
20
20
  megfile/stdio_path.py,sha256=cxaDr8rtisTPnN-rjtaEpqQnshwiqwXFUJBM9xWY7Cg,2711
21
- megfile/version.py,sha256=JRp6vi1OeDHnCmegsFVWveiyO4FzOzG-JLUcsx9DHSQ,19
21
+ megfile/version.py,sha256=kYS0NJJrKL79xx4qm3iMYFp2WaSU6ttQqSdCIGu0WVA,19
22
22
  megfile/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
23
23
  megfile/lib/base_prefetch_reader.py,sha256=uxVwYknOjc8hLF7q_T2QKMsBqFcrf411ZsuK25CN1eQ,12848
24
24
  megfile/lib/combine_reader.py,sha256=Kp2wEloOUpTlIU7dve87MBpSzmIM-F9OtpTawAjFkiU,4828
@@ -43,6 +43,8 @@ megfile/lib/stdio_handler.py,sha256=IDdgENLQlhigEwkLL4zStueVSzdWg7xVcTF_koof_Ek,
43
43
  megfile/lib/url.py,sha256=ER32pWy9Q2MAk3TraAaNEBWIqUeBmLuM57ol2cs7-Ks,103
44
44
  megfile/utils/__init__.py,sha256=xAzmICA0MtAbg-I2yPfeHjA1N4CiMP4sBrC9BgrfZLw,10151
45
45
  megfile/utils/mutex.py,sha256=asb8opGLgK22RiuBJUnfsvB8LnMmodP8KzCVHKmQBWA,2561
46
+ megfile-4.1.2.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
47
+ megfile-4.1.2.dist-info/licenses/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
46
48
  scripts/benchmark/code/iopath_read.py,sha256=O1Qs3mpvD9S_XCuRH2A2FpGWxCOSw6qZvEBrtPeRL1E,661
47
49
  scripts/benchmark/code/iopath_write.py,sha256=Mm0efW1J09RJ_CK5i1xtG2hJuaaslikin8qVpuRFP_Q,704
48
50
  scripts/benchmark/code/megfile_read.py,sha256=sAMebUiMColHDv3JEkXplImAHvn_IF1-g3BIJxhcQYE,239
@@ -53,10 +55,8 @@ scripts/benchmark/code/s3fs_read.py,sha256=XiTA-qrYblUs-jQWXSnvNg5Wo722C_g47aMMf
53
55
  scripts/benchmark/code/s3fs_write.py,sha256=gdXKkWXYGjLJlRT_J64pJN85XvRg3bZexcAJQEMXwtw,402
54
56
  scripts/benchmark/code/smart_open_read.py,sha256=SA02jHwS9Y31yFtV9CoJcfND5dR0eA_HsGmGNUrpQls,515
55
57
  scripts/benchmark/code/smart_open_write.py,sha256=jDxFJdY97yNH889jz3pawBoei3yaqy8pEMvC_ymHFtM,537
56
- megfile-4.1.1.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
57
- megfile-4.1.1.dist-info/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
58
- megfile-4.1.1.dist-info/METADATA,sha256=FvGLIQyfs2_dp0um9bzkW3A6mDDVlVjFi9GMs0a_DMc,9573
59
- megfile-4.1.1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
60
- megfile-4.1.1.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
61
- megfile-4.1.1.dist-info/top_level.txt,sha256=fVg49lk5B9L7jyfWUXWxb0DDSuw5pbr0OU62Tvx8J8M,44
62
- megfile-4.1.1.dist-info/RECORD,,
58
+ megfile-4.1.2.dist-info/METADATA,sha256=7Bxhk9yqd8P_kSNqVhvqCP1yif4ScV2x63YE6vzsznQ,9595
59
+ megfile-4.1.2.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
60
+ megfile-4.1.2.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
61
+ megfile-4.1.2.dist-info/top_level.txt,sha256=TR6xUw8Px5Ms_UENhEmLNmxOdfHAhTlSt9yTw9LRgsQ,35
62
+ megfile-4.1.2.dist-info/RECORD,,
{megfile-4.1.1.dist-info → megfile-4.1.2.dist-info}/WHEEL RENAMED
@@ -1,5 +1,5 @@
1
1
  Wheel-Version: 1.0
2
- Generator: setuptools (75.8.0)
2
+ Generator: setuptools (78.1.0)
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
5
5
 
{megfile-4.1.1.dist-info → megfile-4.1.2.dist-info}/top_level.txt RENAMED
@@ -1,6 +1,5 @@
1
1
  dist
2
2
  docs
3
3
  html_cov
4
- html_doc
5
4
  megfile
6
5
  scripts