megfile 3.1.4__py3-none-any.whl → 3.1.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- megfile/cli.py +9 -3
- megfile/errors.py +1 -1
- megfile/pathlike.py +17 -2
- megfile/s3_path.py +59 -22
- megfile/smart_path.py +27 -20
- megfile/utils/__init__.py +49 -1
- megfile/version.py +1 -1
- {megfile-3.1.4.dist-info → megfile-3.1.6.dist-info}/METADATA +2 -1
- {megfile-3.1.4.dist-info → megfile-3.1.6.dist-info}/RECORD +14 -14
- {megfile-3.1.4.dist-info → megfile-3.1.6.dist-info}/WHEEL +1 -1
- {megfile-3.1.4.dist-info → megfile-3.1.6.dist-info}/LICENSE +0 -0
- {megfile-3.1.4.dist-info → megfile-3.1.6.dist-info}/LICENSE.pyre +0 -0
- {megfile-3.1.4.dist-info → megfile-3.1.6.dist-info}/entry_points.txt +0 -0
- {megfile-3.1.4.dist-info → megfile-3.1.6.dist-info}/top_level.txt +0 -0
megfile/cli.py
CHANGED
|
@@ -29,6 +29,7 @@ from megfile.smart import (
|
|
|
29
29
|
smart_open,
|
|
30
30
|
smart_path_join,
|
|
31
31
|
smart_readlink,
|
|
32
|
+
smart_relpath,
|
|
32
33
|
smart_remove,
|
|
33
34
|
smart_rename,
|
|
34
35
|
smart_scan_stat,
|
|
@@ -78,7 +79,7 @@ def get_echo_path(file_stat, base_path: str = "", full_path: bool = False):
|
|
|
78
79
|
elif full_path:
|
|
79
80
|
path = file_stat.path
|
|
80
81
|
else:
|
|
81
|
-
path =
|
|
82
|
+
path = smart_relpath(file_stat.path, start=base_path)
|
|
82
83
|
return path
|
|
83
84
|
|
|
84
85
|
|
|
@@ -589,6 +590,7 @@ def _safe_makedirs(path: str):
|
|
|
589
590
|
@click.argument("aws_access_key_id")
|
|
590
591
|
@click.argument("aws_secret_access_key")
|
|
591
592
|
@click.option("-e", "--endpoint-url", help="endpoint-url")
|
|
593
|
+
@click.option("-st", "--session-token", help="session-token")
|
|
592
594
|
@click.option("-as", "--addressing-style", help="addressing-style")
|
|
593
595
|
@click.option("-sv", "--signature-version", help="signature-version")
|
|
594
596
|
@click.option("--no-cover", is_flag=True, help="Not cover the same-name config")
|
|
@@ -598,6 +600,7 @@ def s3(
|
|
|
598
600
|
aws_access_key_id,
|
|
599
601
|
aws_secret_access_key,
|
|
600
602
|
endpoint_url,
|
|
603
|
+
session_token,
|
|
601
604
|
addressing_style,
|
|
602
605
|
signature_version,
|
|
603
606
|
no_cover,
|
|
@@ -608,6 +611,7 @@ def s3(
|
|
|
608
611
|
"name": profile_name,
|
|
609
612
|
"aws_access_key_id": aws_access_key_id,
|
|
610
613
|
"aws_secret_access_key": aws_secret_access_key,
|
|
614
|
+
"aws_session_token": session_token,
|
|
611
615
|
}
|
|
612
616
|
s3_config_dict = {
|
|
613
617
|
"endpoint_url": endpoint_url,
|
|
@@ -615,14 +619,16 @@ def s3(
|
|
|
615
619
|
"signature_version": signature_version,
|
|
616
620
|
}
|
|
617
621
|
|
|
622
|
+
config_dict = {k: v for k, v in config_dict.items() if v}
|
|
618
623
|
s3_config_dict = {k: v for k, v in s3_config_dict.items() if v}
|
|
619
624
|
if s3_config_dict:
|
|
620
625
|
config_dict["s3"] = s3_config_dict
|
|
621
626
|
|
|
622
627
|
def dumps(config_dict: dict) -> str:
|
|
623
628
|
content = "[{}]\n".format(config_dict["name"])
|
|
624
|
-
for key in ("aws_access_key_id", "aws_secret_access_key"):
|
|
625
|
-
|
|
629
|
+
for key in ("aws_access_key_id", "aws_secret_access_key", "session_token"):
|
|
630
|
+
if key in config_dict:
|
|
631
|
+
content += "{} = {}\n".format(key, config_dict[key])
|
|
626
632
|
if "s3" in config_dict.keys():
|
|
627
633
|
content += "\ns3 = \n"
|
|
628
634
|
for key, value in config_dict["s3"].items():
|
megfile/errors.py
CHANGED
|
@@ -366,7 +366,7 @@ def translate_s3_error(s3_error: Exception, s3_url: PathLike) -> Exception:
|
|
|
366
366
|
"Invalid configuration: %r, code: %r, message: %r, endpoint: %r"
|
|
367
367
|
% (s3_url, code, message, s3_endpoint_url(s3_url))
|
|
368
368
|
)
|
|
369
|
-
if code in ("InvalidRange"):
|
|
369
|
+
if code in ("InvalidRange", "Requested Range Not Satisfiable"):
|
|
370
370
|
return S3InvalidRangeError(
|
|
371
371
|
"Index out of range: %r, code: %r, message: %r, endpoint: %r"
|
|
372
372
|
% (
|
megfile/pathlike.py
CHANGED
|
@@ -636,8 +636,8 @@ class URIPath(BaseURIPath):
|
|
|
636
636
|
relative = path[len(other_path) :]
|
|
637
637
|
relative = relative.lstrip("/")
|
|
638
638
|
return type(self)(relative) # pyre-ignore[19]
|
|
639
|
-
|
|
640
|
-
|
|
639
|
+
|
|
640
|
+
raise ValueError("%r does not start with %r" % (path, other))
|
|
641
641
|
|
|
642
642
|
def with_name(self: Self, name: str) -> Self:
|
|
643
643
|
"""Return a new path with the name changed"""
|
|
@@ -655,6 +655,21 @@ class URIPath(BaseURIPath):
|
|
|
655
655
|
raw_suffix = self.suffix
|
|
656
656
|
return self.from_path(path[: len(path) - len(raw_suffix)] + suffix)
|
|
657
657
|
|
|
658
|
+
def relpath(self, start=None):
|
|
659
|
+
"""Return the relative path."""
|
|
660
|
+
if start is None:
|
|
661
|
+
raise TypeError("start is required")
|
|
662
|
+
|
|
663
|
+
other_path = self.from_path(start).path_with_protocol
|
|
664
|
+
path = self.path_with_protocol
|
|
665
|
+
|
|
666
|
+
if path.startswith(other_path):
|
|
667
|
+
relative = path[len(other_path) :]
|
|
668
|
+
relative = relative.lstrip("/")
|
|
669
|
+
return relative
|
|
670
|
+
|
|
671
|
+
raise ValueError("%r does not start with %r" % (path, other_path))
|
|
672
|
+
|
|
658
673
|
def is_absolute(self) -> bool:
|
|
659
674
|
return True
|
|
660
675
|
|
megfile/s3_path.py
CHANGED
|
@@ -122,7 +122,7 @@ max_retries = S3_MAX_RETRY_TIMES
|
|
|
122
122
|
max_keys = 1000
|
|
123
123
|
|
|
124
124
|
|
|
125
|
-
def _patch_make_request(client: botocore.client.BaseClient):
|
|
125
|
+
def _patch_make_request(client: botocore.client.BaseClient, redirect: bool = False):
|
|
126
126
|
def after_callback(result: Tuple[AWSResponse, dict], *args, **kwargs):
|
|
127
127
|
if (
|
|
128
128
|
not isinstance(result, tuple)
|
|
@@ -161,6 +161,26 @@ def _patch_make_request(client: botocore.client.BaseClient):
|
|
|
161
161
|
before_callback=before_callback,
|
|
162
162
|
retry_callback=retry_callback,
|
|
163
163
|
)
|
|
164
|
+
|
|
165
|
+
def patch_send_request(send_request):
|
|
166
|
+
def patched_send_request(request_dict, operation_model):
|
|
167
|
+
http, parsed_response = send_request(request_dict, operation_model)
|
|
168
|
+
if (
|
|
169
|
+
request_dict["method"] == "GET" # only support GET method for now
|
|
170
|
+
and http.status_code in (301, 302, 307, 308)
|
|
171
|
+
and "Location" in http.headers
|
|
172
|
+
):
|
|
173
|
+
request_dict["url"] = http.headers["Location"]
|
|
174
|
+
http, parsed_response = send_request(request_dict, operation_model)
|
|
175
|
+
return http, parsed_response
|
|
176
|
+
|
|
177
|
+
return patched_send_request
|
|
178
|
+
|
|
179
|
+
if redirect:
|
|
180
|
+
client._endpoint._send_request = patch_send_request(
|
|
181
|
+
client._endpoint._send_request
|
|
182
|
+
)
|
|
183
|
+
|
|
164
184
|
return client
|
|
165
185
|
|
|
166
186
|
|
|
@@ -180,7 +200,11 @@ def parse_s3_url(s3_url: PathLike) -> Tuple[str, str]:
|
|
|
180
200
|
|
|
181
201
|
|
|
182
202
|
def get_scoped_config(profile_name: Optional[str] = None) -> Dict:
|
|
183
|
-
|
|
203
|
+
try:
|
|
204
|
+
session = get_s3_session(profile_name=profile_name)
|
|
205
|
+
except botocore.exceptions.ProfileNotFound:
|
|
206
|
+
session = get_s3_session()
|
|
207
|
+
return session._session.get_scoped_config()
|
|
184
208
|
|
|
185
209
|
|
|
186
210
|
@lru_cache()
|
|
@@ -224,21 +248,24 @@ def get_s3_session(profile_name=None) -> boto3.Session:
|
|
|
224
248
|
)
|
|
225
249
|
|
|
226
250
|
|
|
251
|
+
def get_env_var(env_name: str, profile_name=None):
|
|
252
|
+
if profile_name:
|
|
253
|
+
return os.getenv(f"{profile_name}__{env_name}".upper())
|
|
254
|
+
return os.getenv(env_name.upper())
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def parse_boolean(value: Optional[str], default: bool = False) -> bool:
|
|
258
|
+
if value is None:
|
|
259
|
+
return default
|
|
260
|
+
return value.lower() in ("true", "yes", "1")
|
|
261
|
+
|
|
262
|
+
|
|
227
263
|
def get_access_token(profile_name=None):
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
else "AWS_ACCESS_KEY_ID"
|
|
232
|
-
)
|
|
233
|
-
secret_key_env_name = (
|
|
234
|
-
f"{profile_name}__AWS_SECRET_ACCESS_KEY".upper()
|
|
235
|
-
if profile_name
|
|
236
|
-
else "AWS_SECRET_ACCESS_KEY"
|
|
237
|
-
)
|
|
238
|
-
access_key = os.getenv(access_key_env_name)
|
|
239
|
-
secret_key = os.getenv(secret_key_env_name)
|
|
264
|
+
access_key = get_env_var("AWS_ACCESS_KEY_ID", profile_name=profile_name)
|
|
265
|
+
secret_key = get_env_var("AWS_SECRET_ACCESS_KEY", profile_name=profile_name)
|
|
266
|
+
session_token = get_env_var("AWS_SESSION_TOKEN", profile_name=profile_name)
|
|
240
267
|
if access_key and secret_key:
|
|
241
|
-
return access_key, secret_key
|
|
268
|
+
return access_key, secret_key, session_token
|
|
242
269
|
|
|
243
270
|
try:
|
|
244
271
|
credentials = get_s3_session(profile_name=profile_name).get_credentials()
|
|
@@ -249,7 +276,9 @@ def get_access_token(profile_name=None):
|
|
|
249
276
|
access_key = credentials.access_key
|
|
250
277
|
if not secret_key:
|
|
251
278
|
secret_key = credentials.secret_key
|
|
252
|
-
|
|
279
|
+
if not session_token:
|
|
280
|
+
session_token = credentials.token
|
|
281
|
+
return access_key, secret_key, session_token
|
|
253
282
|
|
|
254
283
|
|
|
255
284
|
def get_s3_client(
|
|
@@ -281,28 +310,36 @@ def get_s3_client(
|
|
|
281
310
|
connect_timeout=5, max_pool_connections=GLOBAL_MAX_WORKERS
|
|
282
311
|
)
|
|
283
312
|
|
|
284
|
-
|
|
285
|
-
if profile_name:
|
|
286
|
-
addressing_style_env_key = f"{profile_name}__AWS_S3_ADDRESSING_STYLE".upper()
|
|
287
|
-
addressing_style = os.environ.get(addressing_style_env_key)
|
|
313
|
+
addressing_style = get_env_var("AWS_S3_ADDRESSING_STYLE", profile_name=profile_name)
|
|
288
314
|
if addressing_style:
|
|
289
315
|
config = config.merge(
|
|
290
316
|
botocore.config.Config(s3={"addressing_style": addressing_style})
|
|
291
317
|
)
|
|
292
318
|
|
|
293
|
-
access_key, secret_key = get_access_token(profile_name)
|
|
319
|
+
access_key, secret_key, session_token = get_access_token(profile_name)
|
|
294
320
|
try:
|
|
295
321
|
session = get_s3_session(profile_name=profile_name)
|
|
296
322
|
except botocore.exceptions.ProfileNotFound:
|
|
297
323
|
session = get_s3_session()
|
|
324
|
+
|
|
325
|
+
s3_config = get_scoped_config(profile_name=profile_name).get("s3", {})
|
|
326
|
+
verify = get_env_var("AWS_S3_VERIFY", profile_name=profile_name)
|
|
327
|
+
verify = verify or s3_config.get("verify")
|
|
328
|
+
verify = parse_boolean(verify, default=True)
|
|
329
|
+
redirect = get_env_var("AWS_S3_REDIRECT", profile_name=profile_name)
|
|
330
|
+
redirect = redirect or s3_config.get("redirect")
|
|
331
|
+
redirect = parse_boolean(redirect, default=False)
|
|
332
|
+
|
|
298
333
|
client = session.client(
|
|
299
334
|
"s3",
|
|
300
335
|
endpoint_url=get_endpoint_url(profile_name=profile_name),
|
|
336
|
+
verify=verify,
|
|
301
337
|
config=config,
|
|
302
338
|
aws_access_key_id=access_key,
|
|
303
339
|
aws_secret_access_key=secret_key,
|
|
340
|
+
aws_session_token=session_token,
|
|
304
341
|
)
|
|
305
|
-
client = _patch_make_request(client)
|
|
342
|
+
client = _patch_make_request(client, redirect=redirect)
|
|
306
343
|
return client
|
|
307
344
|
|
|
308
345
|
|
megfile/smart_path.py
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import os
|
|
2
2
|
from configparser import ConfigParser
|
|
3
3
|
from pathlib import PurePath
|
|
4
|
-
from typing import Dict, Tuple, Union
|
|
4
|
+
from typing import Dict, Optional, Tuple, Union
|
|
5
5
|
|
|
6
6
|
from megfile.lib.compat import fspath
|
|
7
7
|
from megfile.lib.url import get_url_scheme
|
|
8
|
-
from megfile.utils import
|
|
8
|
+
from megfile.utils import cached_classproperty
|
|
9
9
|
|
|
10
10
|
from .errors import ProtocolExistsError, ProtocolNotFoundError
|
|
11
11
|
from .interfaces import BasePath, BaseURIPath, PathLike
|
|
@@ -54,15 +54,15 @@ class SmartPath(BasePath):
|
|
|
54
54
|
self.path = str(pathlike)
|
|
55
55
|
self.pathlike = pathlike
|
|
56
56
|
|
|
57
|
-
@
|
|
57
|
+
@cached_classproperty
|
|
58
58
|
def _aliases(cls) -> Dict[str, Dict[str, str]]:
|
|
59
59
|
config_path = os.path.expanduser(aliases_config)
|
|
60
|
-
|
|
61
|
-
setattr(cls, "_aliases", aliases)
|
|
62
|
-
return aliases
|
|
60
|
+
return _load_aliases_config(config_path)
|
|
63
61
|
|
|
64
|
-
@
|
|
65
|
-
def _extract_protocol(
|
|
62
|
+
@classmethod
|
|
63
|
+
def _extract_protocol(
|
|
64
|
+
cls, path: Union[PathLike, int]
|
|
65
|
+
) -> Tuple[str, Union[str, int]]:
|
|
66
66
|
if isinstance(path, int):
|
|
67
67
|
protocol = "file"
|
|
68
68
|
path_without_protocol = path
|
|
@@ -74,27 +74,26 @@ class SmartPath(BasePath):
|
|
|
74
74
|
else:
|
|
75
75
|
path_without_protocol = path[len(protocol) + 3 :]
|
|
76
76
|
elif isinstance(path, (BaseURIPath, SmartPath)):
|
|
77
|
-
|
|
78
|
-
path_without_protocol = str(path)
|
|
77
|
+
return str(path.protocol), str(path)
|
|
79
78
|
elif isinstance(path, (PurePath, BasePath)):
|
|
80
|
-
|
|
79
|
+
return SmartPath._extract_protocol(fspath(path))
|
|
81
80
|
else:
|
|
82
81
|
raise ProtocolNotFoundError("protocol not found: %r" % path)
|
|
83
|
-
return protocol, path_without_protocol
|
|
84
|
-
|
|
85
|
-
@classmethod
|
|
86
|
-
def _create_pathlike(cls, path: Union[PathLike, int]) -> BaseURIPath:
|
|
87
|
-
protocol, path_without_protocol = cls._extract_protocol(path)
|
|
88
82
|
aliases: Dict[str, Dict[str, str]] = cls._aliases # pyre-ignore[9]
|
|
89
83
|
if protocol in aliases:
|
|
90
84
|
protocol = aliases[protocol]["protocol"]
|
|
91
|
-
path =
|
|
85
|
+
path = "%s://%s" % (protocol, path_without_protocol)
|
|
86
|
+
return protocol, path
|
|
87
|
+
|
|
88
|
+
@classmethod
|
|
89
|
+
def _create_pathlike(cls, path: Union[PathLike, int]) -> BaseURIPath:
|
|
90
|
+
protocol, unaliased_path = cls._extract_protocol(path)
|
|
92
91
|
if protocol.startswith("s3+"):
|
|
93
92
|
protocol = "s3"
|
|
94
93
|
if protocol not in cls._registered_protocols:
|
|
95
94
|
raise ProtocolNotFoundError("protocol %r not found: %r" % (protocol, path))
|
|
96
95
|
path_class = cls._registered_protocols[protocol]
|
|
97
|
-
return path_class(
|
|
96
|
+
return path_class(unaliased_path)
|
|
98
97
|
|
|
99
98
|
@classmethod
|
|
100
99
|
def register(cls, path_class, override_ok: bool = False):
|
|
@@ -137,7 +136,6 @@ class SmartPath(BasePath):
|
|
|
137
136
|
joinpath = _bind_function("joinpath")
|
|
138
137
|
abspath = _bind_function("abspath")
|
|
139
138
|
realpath = _bind_function("realpath")
|
|
140
|
-
relpath = _bind_function("relpath")
|
|
141
139
|
is_absolute = _bind_function("is_absolute")
|
|
142
140
|
is_mount = _bind_function("is_mount")
|
|
143
141
|
md5 = _bind_function("md5")
|
|
@@ -150,6 +148,16 @@ class SmartPath(BasePath):
|
|
|
150
148
|
def from_uri(cls, path: str):
|
|
151
149
|
return cls(path)
|
|
152
150
|
|
|
151
|
+
def relpath(self, start: Optional[str] = None) -> str:
|
|
152
|
+
"""Return the relative path of given path
|
|
153
|
+
|
|
154
|
+
:param start: Given start directory
|
|
155
|
+
:returns: Relative path from start
|
|
156
|
+
"""
|
|
157
|
+
if start is not None:
|
|
158
|
+
_, start = SmartPath._extract_protocol(fspath(start))
|
|
159
|
+
return self.pathlike.relpath(start=start)
|
|
160
|
+
|
|
153
161
|
as_uri = _bind_function("as_uri")
|
|
154
162
|
as_posix = _bind_function("as_posix")
|
|
155
163
|
__lt__ = _bind_function("__lt__")
|
|
@@ -170,7 +178,6 @@ class SmartPath(BasePath):
|
|
|
170
178
|
is_mount = _bind_function("is_mount")
|
|
171
179
|
abspath = _bind_function("abspath")
|
|
172
180
|
realpath = _bind_function("realpath")
|
|
173
|
-
relpath = _bind_function("relpath")
|
|
174
181
|
iterdir = _bind_function("iterdir")
|
|
175
182
|
cwd = _bind_function("cwd")
|
|
176
183
|
home = _bind_function("home")
|
megfile/utils/__init__.py
CHANGED
|
@@ -4,7 +4,7 @@ import math
|
|
|
4
4
|
import os
|
|
5
5
|
import uuid
|
|
6
6
|
from copy import copy
|
|
7
|
-
from functools import wraps
|
|
7
|
+
from functools import cached_property, wraps
|
|
8
8
|
from io import (
|
|
9
9
|
BufferedIOBase,
|
|
10
10
|
BufferedRandom,
|
|
@@ -15,6 +15,7 @@ from io import (
|
|
|
15
15
|
TextIOBase,
|
|
16
16
|
TextIOWrapper,
|
|
17
17
|
)
|
|
18
|
+
from threading import RLock
|
|
18
19
|
from typing import IO, Callable, Optional
|
|
19
20
|
|
|
20
21
|
from megfile.utils.mutex import ProcessLocal, ThreadLocal
|
|
@@ -298,3 +299,50 @@ class classproperty(property):
|
|
|
298
299
|
"""
|
|
299
300
|
# call this method only on the class, not the instance
|
|
300
301
|
super(classproperty, self).__delete__(_get_class(cls_or_obj))
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
class cached_classproperty(cached_property):
|
|
305
|
+
"""
|
|
306
|
+
The use this class as a decorator for your class property with cache.
|
|
307
|
+
Example:
|
|
308
|
+
@cached_classproperty
|
|
309
|
+
def prop(cls):
|
|
310
|
+
return "value"
|
|
311
|
+
"""
|
|
312
|
+
|
|
313
|
+
def __init__(self, func: Callable) -> None:
|
|
314
|
+
"""
|
|
315
|
+
This method initializes the cached_classproperty instance.
|
|
316
|
+
@param func: The function to be called when the property value is requested.
|
|
317
|
+
"""
|
|
318
|
+
super().__init__(func)
|
|
319
|
+
# Python 3.12 removed the lock attribute from cached_property.
|
|
320
|
+
# Maybe we should remove this in the future.
|
|
321
|
+
# See also: https://github.com/python/cpython/pull/101890
|
|
322
|
+
# https://github.com/python/cpython/issues/87634
|
|
323
|
+
if not hasattr(func, "lock"):
|
|
324
|
+
self.lock = RLock()
|
|
325
|
+
|
|
326
|
+
def __get__( # pyre-ignore[14]
|
|
327
|
+
self,
|
|
328
|
+
_,
|
|
329
|
+
cls, # pytype: disable=signature-mismatch
|
|
330
|
+
) -> object:
|
|
331
|
+
"""
|
|
332
|
+
This method gets called when a property value is requested.
|
|
333
|
+
@param cls: The class type of the above instance.
|
|
334
|
+
@return: The value of the property.
|
|
335
|
+
"""
|
|
336
|
+
if self.attrname is None:
|
|
337
|
+
raise TypeError(
|
|
338
|
+
"Cannot use cached_classproperty instance without calling "
|
|
339
|
+
"__set_name__ on it."
|
|
340
|
+
)
|
|
341
|
+
with self.lock:
|
|
342
|
+
# check if another thread filled cache while we awaited lock
|
|
343
|
+
# cannot use getattr since it will cause RecursionError
|
|
344
|
+
val = cls.__dict__[self.attrname]
|
|
345
|
+
if val is self:
|
|
346
|
+
val = self.func(cls)
|
|
347
|
+
setattr(cls, self.attrname, val)
|
|
348
|
+
return val
|
megfile/version.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
VERSION = "3.1.
|
|
1
|
+
VERSION = "3.1.6"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: megfile
|
|
3
|
-
Version: 3.1.
|
|
3
|
+
Version: 3.1.6
|
|
4
4
|
Summary: Megvii file operation library
|
|
5
5
|
Author-email: megvii <megfile@megvii.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/megvii-research/megfile
|
|
@@ -166,6 +166,7 @@ You can use environments and configuration file for configuration, and priority
|
|
|
166
166
|
You can use environments to setup authentication credentials for your `s3` account:
|
|
167
167
|
- `AWS_ACCESS_KEY_ID`: access key
|
|
168
168
|
- `AWS_SECRET_ACCESS_KEY`: secret key
|
|
169
|
+
- `AWS_SESSION_TOKEN`: session token
|
|
169
170
|
- `OSS_ENDPOINT` / `AWS_ENDPOINT_URL_S3` / `AWS_ENDPOINT_URL`: endpoint url of s3
|
|
170
171
|
- `AWS_S3_ADDRESSING_STYLE`: addressing style
|
|
171
172
|
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
docs/conf.py,sha256=sfDSly5jO8W_RmuAptOIp4hd8dNcO-9a5XrHTbxFnNo,2448
|
|
2
2
|
megfile/__init__.py,sha256=i2Lbq_VxIgppaqwkxG0_H35dRfcjJ4mCYWjprOf4hHo,7318
|
|
3
|
-
megfile/cli.py,sha256=
|
|
3
|
+
megfile/cli.py,sha256=0Sgbz3jeUryVll9Aa6R0MpJdQJUGENvz55YF7Jm1Uxc,23482
|
|
4
4
|
megfile/config.py,sha256=_SkJRaVWUdfW1Q9uX0vao-6YVQKJtfej22Z8DykuRps,2331
|
|
5
|
-
megfile/errors.py,sha256=
|
|
5
|
+
megfile/errors.py,sha256=a55qKQgyfiLmV-qnojUFzq2gu9JXpj3ZiC2qVaWyUTA,14160
|
|
6
6
|
megfile/fs.py,sha256=dgj5fW-EEzQNdjMF2tkB5DjXu3iHQbtLi5PSIMxR8fc,11966
|
|
7
7
|
megfile/fs_path.py,sha256=Ffvukc176beH5aQMZXXtwH6ApwLYXPViCIUP0pijgT0,41590
|
|
8
8
|
megfile/hdfs.py,sha256=latguOuuzAmg-yWOy3Sm723CJ0ybN_eSHRubVNqhcMU,9202
|
|
@@ -10,16 +10,16 @@ megfile/hdfs_path.py,sha256=0XLtABufwqL-y8igOxzOJz6zOGppuBp2f2SwXIMvvYg,27299
|
|
|
10
10
|
megfile/http.py,sha256=2Z2yqyhU-zcJCJwSNyBsxsZ7f2FT9X6fcednsbHDsFM,2025
|
|
11
11
|
megfile/http_path.py,sha256=BhMNjQVB85IaCGGIKzgEfY73mAVdCzJP08W1RuGeMRA,16119
|
|
12
12
|
megfile/interfaces.py,sha256=7C53Q2FAVFmOEnplfplvWqHab29HJE5RQnpfdb4loVY,8679
|
|
13
|
-
megfile/pathlike.py,sha256=
|
|
13
|
+
megfile/pathlike.py,sha256=5VAKIArm2UqrpMBJMoNAEydFxLd1mjCZ8iQnKFUIYu0,31274
|
|
14
14
|
megfile/s3.py,sha256=7SdfLjAePVh-bpRyuj566VB4Qa7KP86rCJGzYANR7wQ,13008
|
|
15
|
-
megfile/s3_path.py,sha256=
|
|
15
|
+
megfile/s3_path.py,sha256=x-_wxVvVpv56LtDOwRAWJCOq3XMk7oAB6xdi22TBlmY,95029
|
|
16
16
|
megfile/sftp.py,sha256=vyDnYXX3i1j2fhXMC8YCeX-66MDb9wrBQQjQVhZx0uo,13004
|
|
17
17
|
megfile/sftp_path.py,sha256=4tByWvUJK1KBJoa3t5aoWYnZpaRWN9nQIE6ZyiGHrbk,53519
|
|
18
18
|
megfile/smart.py,sha256=Vr4R7HpjXjt587KOc2-1QGbQ5EsZ48YRzCaK0rz3IS0,36108
|
|
19
|
-
megfile/smart_path.py,sha256=
|
|
19
|
+
megfile/smart_path.py,sha256=Wsn6fR9g7NTwNwwvZ_0H39NLHIlOLnCqK-ZY0n5CvKk,7812
|
|
20
20
|
megfile/stdio.py,sha256=UYe-h440Wc4f5COOzOTG1svnp5nFzrfpixehJ0_0_NY,653
|
|
21
21
|
megfile/stdio_path.py,sha256=7jzVdreamO18yBWZM7Pp71cO7GmrYb0M0qyQde2Ypq4,2706
|
|
22
|
-
megfile/version.py,sha256=
|
|
22
|
+
megfile/version.py,sha256=GfRGwuY_9YYywKtd8Nc8lbkE6hlfX3NUsNtuHN-c8Gs,19
|
|
23
23
|
megfile/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
24
24
|
megfile/lib/base_prefetch_reader.py,sha256=CaYWuiKmlk4Utr0IFDPwPC58wV2jBAhqpxhwhRHc734,13652
|
|
25
25
|
megfile/lib/combine_reader.py,sha256=uSzo3PmhD5ck6_Vv6dFU5vVx4boeA97VS-puPyhF_BE,4657
|
|
@@ -42,14 +42,14 @@ megfile/lib/s3_share_cache_reader.py,sha256=jhGL1B6NPv68cQnW1Jf7ey-zTQ8XfiJg5ILD
|
|
|
42
42
|
megfile/lib/shadow_handler.py,sha256=TntewlvIW9ZxCfmqASDQREHoiZ8v42faOe9sovQYQz0,2779
|
|
43
43
|
megfile/lib/stdio_handler.py,sha256=IDdgENLQlhigEwkLL4zStueVSzdWg7xVcTF_koof_Ek,1987
|
|
44
44
|
megfile/lib/url.py,sha256=ER32pWy9Q2MAk3TraAaNEBWIqUeBmLuM57ol2cs7-Ks,103
|
|
45
|
-
megfile/utils/__init__.py,sha256=
|
|
45
|
+
megfile/utils/__init__.py,sha256=9rD_SoD--XWt7-EJi5-L80Y7YeoFdr-tUp-5ATB85oA,10717
|
|
46
46
|
megfile/utils/mutex.py,sha256=asb8opGLgK22RiuBJUnfsvB8LnMmodP8KzCVHKmQBWA,2561
|
|
47
47
|
scripts/convert_results_to_sarif.py,sha256=nDiOfsedb22Ps7ZodmYdlXZlxv54fRxCQgOZsB2OkNk,2833
|
|
48
48
|
scripts/generate_file.py,sha256=-mTcBiqiQ1juvqojVfVZ-uZWgpANHJNdhrF7s68zNfc,10903
|
|
49
|
-
megfile-3.1.
|
|
50
|
-
megfile-3.1.
|
|
51
|
-
megfile-3.1.
|
|
52
|
-
megfile-3.1.
|
|
53
|
-
megfile-3.1.
|
|
54
|
-
megfile-3.1.
|
|
55
|
-
megfile-3.1.
|
|
49
|
+
megfile-3.1.6.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
|
|
50
|
+
megfile-3.1.6.dist-info/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
|
|
51
|
+
megfile-3.1.6.dist-info/METADATA,sha256=XuomkBWbASrj60L6untH8oN32sNezHse1WfuPetTpYg,9178
|
|
52
|
+
megfile-3.1.6.dist-info/WHEEL,sha256=R06PA3UVYHThwHvxuRWMqaGcr-PuniXahwjmQRFMEkY,91
|
|
53
|
+
megfile-3.1.6.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
|
|
54
|
+
megfile-3.1.6.dist-info/top_level.txt,sha256=oTnYXo1Z3V61qSWAKtnY9RkDgRSHvfRN38FQae6E0W0,50
|
|
55
|
+
megfile-3.1.6.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|