megfile 3.1.3__py3-none-any.whl → 3.1.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- megfile/cli.py +62 -24
- megfile/errors.py +6 -3
- megfile/interfaces.py +6 -2
- megfile/lib/fnmatch.py +24 -19
- megfile/lib/shadow_handler.py +3 -1
- megfile/s3_path.py +12 -3
- megfile/smart_path.py +29 -2
- megfile/version.py +1 -1
- {megfile-3.1.3.dist-info → megfile-3.1.5.dist-info}/METADATA +23 -3
- {megfile-3.1.3.dist-info → megfile-3.1.5.dist-info}/RECORD +15 -15
- {megfile-3.1.3.dist-info → megfile-3.1.5.dist-info}/WHEEL +1 -1
- {megfile-3.1.3.dist-info → megfile-3.1.5.dist-info}/LICENSE +0 -0
- {megfile-3.1.3.dist-info → megfile-3.1.5.dist-info}/LICENSE.pyre +0 -0
- {megfile-3.1.3.dist-info → megfile-3.1.5.dist-info}/entry_points.txt +0 -0
- {megfile-3.1.3.dist-info → megfile-3.1.5.dist-info}/top_level.txt +0 -0
megfile/cli.py
CHANGED
|
@@ -570,7 +570,12 @@ def config():
|
|
|
570
570
|
pass
|
|
571
571
|
|
|
572
572
|
|
|
573
|
-
|
|
573
|
+
def _safe_makedirs(path: str):
|
|
574
|
+
if path not in ("", ".", "/"):
|
|
575
|
+
os.makedirs(path, exist_ok=True)
|
|
576
|
+
|
|
577
|
+
|
|
578
|
+
@config.command(short_help="Update the config file for s3")
|
|
574
579
|
@click.option(
|
|
575
580
|
"-p",
|
|
576
581
|
"--path",
|
|
@@ -584,7 +589,9 @@ def config():
|
|
|
584
589
|
@click.argument("aws_access_key_id")
|
|
585
590
|
@click.argument("aws_secret_access_key")
|
|
586
591
|
@click.option("-e", "--endpoint-url", help="endpoint-url")
|
|
587
|
-
@click.option("-
|
|
592
|
+
@click.option("-st", "--session-token", help="session-token")
|
|
593
|
+
@click.option("-as", "--addressing-style", help="addressing-style")
|
|
594
|
+
@click.option("-sv", "--signature-version", help="signature-version")
|
|
588
595
|
@click.option("--no-cover", is_flag=True, help="Not cover the same-name config")
|
|
589
596
|
def s3(
|
|
590
597
|
path,
|
|
@@ -592,7 +599,9 @@ def s3(
|
|
|
592
599
|
aws_access_key_id,
|
|
593
600
|
aws_secret_access_key,
|
|
594
601
|
endpoint_url,
|
|
602
|
+
session_token,
|
|
595
603
|
addressing_style,
|
|
604
|
+
signature_version,
|
|
596
605
|
no_cover,
|
|
597
606
|
):
|
|
598
607
|
path = os.path.expanduser(path)
|
|
@@ -601,31 +610,31 @@ def s3(
|
|
|
601
610
|
"name": profile_name,
|
|
602
611
|
"aws_access_key_id": aws_access_key_id,
|
|
603
612
|
"aws_secret_access_key": aws_secret_access_key,
|
|
613
|
+
"aws_session_token": session_token,
|
|
614
|
+
}
|
|
615
|
+
s3_config_dict = {
|
|
616
|
+
"endpoint_url": endpoint_url,
|
|
617
|
+
"addressing_style": addressing_style,
|
|
618
|
+
"signature_version": signature_version,
|
|
604
619
|
}
|
|
605
|
-
|
|
606
|
-
if
|
|
607
|
-
|
|
608
|
-
if
|
|
609
|
-
s3
|
|
610
|
-
if s3:
|
|
611
|
-
config_dict.update({"s3": s3})
|
|
620
|
+
|
|
621
|
+
config_dict = {k: v for k, v in config_dict.items() if v}
|
|
622
|
+
s3_config_dict = {k: v for k, v in s3_config_dict.items() if v}
|
|
623
|
+
if s3_config_dict:
|
|
624
|
+
config_dict["s3"] = s3_config_dict
|
|
612
625
|
|
|
613
626
|
def dumps(config_dict: dict) -> str:
|
|
614
627
|
content = "[{}]\n".format(config_dict["name"])
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
)
|
|
628
|
+
for key in ("aws_access_key_id", "aws_secret_access_key", "session_token"):
|
|
629
|
+
if key in config_dict:
|
|
630
|
+
content += "{} = {}\n".format(key, config_dict[key])
|
|
619
631
|
if "s3" in config_dict.keys():
|
|
620
632
|
content += "\ns3 = \n"
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
content += " endpoint_url = {}\n".format(s3["endpoint_url"])
|
|
624
|
-
if "addressing_style" in s3.keys():
|
|
625
|
-
content += " addressing_style = {}\n".format(s3["addressing_style"])
|
|
633
|
+
for key, value in config_dict["s3"].items():
|
|
634
|
+
content += " {} = {}\n".format(key, value)
|
|
626
635
|
return content
|
|
627
636
|
|
|
628
|
-
|
|
637
|
+
_safe_makedirs(os.path.dirname(path)) # make sure dirpath exist
|
|
629
638
|
if not os.path.exists(path): # If this file doesn't exist.
|
|
630
639
|
content_str = dumps(config_dict)
|
|
631
640
|
with open(path, "w") as fp:
|
|
@@ -663,15 +672,15 @@ def s3(
|
|
|
663
672
|
click.echo(f"Your oss config has been saved into {path}")
|
|
664
673
|
|
|
665
674
|
|
|
666
|
-
@config.command(short_help="
|
|
667
|
-
@click.argument("url")
|
|
675
|
+
@config.command(short_help="Update the config file for hdfs")
|
|
668
676
|
@click.option(
|
|
669
677
|
"-p",
|
|
670
678
|
"--path",
|
|
671
679
|
default="~/.hdfscli.cfg",
|
|
672
|
-
help="
|
|
680
|
+
help="hdfs config file, default is $HOME/.hdfscli.cfg",
|
|
673
681
|
)
|
|
674
|
-
@click.
|
|
682
|
+
@click.argument("url")
|
|
683
|
+
@click.option("-n", "--profile-name", default="default", help="hdfs config file")
|
|
675
684
|
@click.option("-u", "--user", help="user name")
|
|
676
685
|
@click.option("-r", "--root", help="hdfs path's root dir")
|
|
677
686
|
@click.option("-t", "--token", help="token for requesting hdfs server")
|
|
@@ -681,7 +690,7 @@ def s3(
|
|
|
681
690
|
help=f"request hdfs server timeout, default {DEFAULT_HDFS_TIMEOUT}",
|
|
682
691
|
)
|
|
683
692
|
@click.option("--no-cover", is_flag=True, help="Not cover the same-name config")
|
|
684
|
-
def hdfs(
|
|
693
|
+
def hdfs(path, url, profile_name, user, root, token, timeout, no_cover):
|
|
685
694
|
path = os.path.expanduser(path)
|
|
686
695
|
current_config = {
|
|
687
696
|
"url": url,
|
|
@@ -704,11 +713,40 @@ def hdfs(url, path, profile_name, user, root, token, timeout, no_cover):
|
|
|
704
713
|
for key, value in current_config.items():
|
|
705
714
|
if value:
|
|
706
715
|
config[profile_name][key] = value
|
|
716
|
+
|
|
717
|
+
_safe_makedirs(os.path.dirname(path)) # make sure dirpath exist
|
|
707
718
|
with open(path, "w") as fp:
|
|
708
719
|
config.write(fp)
|
|
709
720
|
click.echo(f"Your hdfs config has been saved into {path}")
|
|
710
721
|
|
|
711
722
|
|
|
723
|
+
@config.command(short_help="Update the config file for aliases")
|
|
724
|
+
@click.option(
|
|
725
|
+
"-p",
|
|
726
|
+
"--path",
|
|
727
|
+
default="~/.config/megfile/aliases.conf",
|
|
728
|
+
help="alias config file, default is $HOME/.config/megfile/aliases.conf",
|
|
729
|
+
)
|
|
730
|
+
@click.argument("name")
|
|
731
|
+
@click.argument("protocol")
|
|
732
|
+
@click.option("--no-cover", is_flag=True, help="Not cover the same-name config")
|
|
733
|
+
def alias(path, name, protocol, no_cover):
|
|
734
|
+
path = os.path.expanduser(path)
|
|
735
|
+
config = configparser.ConfigParser()
|
|
736
|
+
if os.path.exists(path):
|
|
737
|
+
config.read(path)
|
|
738
|
+
if name in config.sections() and no_cover:
|
|
739
|
+
raise NameError(f"alias-name has been used: {name}")
|
|
740
|
+
config[name] = {
|
|
741
|
+
"protocol": protocol,
|
|
742
|
+
}
|
|
743
|
+
|
|
744
|
+
_safe_makedirs(os.path.dirname(path)) # make sure dirpath exist
|
|
745
|
+
with open(path, "w") as fp:
|
|
746
|
+
config.write(fp)
|
|
747
|
+
click.echo(f"Your alias config has been saved into {path}")
|
|
748
|
+
|
|
749
|
+
|
|
712
750
|
if __name__ == "__main__":
|
|
713
751
|
# Usage: python -m megfile.cli
|
|
714
752
|
safe_cli() # pragma: no cover
|
megfile/errors.py
CHANGED
|
@@ -342,13 +342,16 @@ def translate_s3_error(s3_error: Exception, s3_url: PathLike) -> Exception:
|
|
|
342
342
|
elif isinstance(s3_error, ClientError):
|
|
343
343
|
code = client_error_code(s3_error)
|
|
344
344
|
if code in ("NoSuchBucket"):
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
% s3_error.response.get( # pytype: disable=attribute-error
|
|
345
|
+
bucket_or_url = (
|
|
346
|
+
s3_error.response.get( # pytype: disable=attribute-error
|
|
348
347
|
"Error", {}
|
|
349
348
|
).get("BucketName")
|
|
350
349
|
or s3_url
|
|
351
350
|
)
|
|
351
|
+
return S3BucketNotFoundError(
|
|
352
|
+
"No such bucket: %r, endpoint: %r"
|
|
353
|
+
% (bucket_or_url, s3_endpoint_url(s3_url))
|
|
354
|
+
)
|
|
352
355
|
if code in ("404", "NoSuchKey"):
|
|
353
356
|
return S3FileNotFoundError("No such file: %r" % s3_url)
|
|
354
357
|
if code in ("401", "403", "AccessDenied"):
|
megfile/interfaces.py
CHANGED
|
@@ -194,7 +194,9 @@ class Readable(FileLike[AnyStr], ABC):
|
|
|
194
194
|
def write(self, data: AnyStr) -> int:
|
|
195
195
|
raise OSError("not writable")
|
|
196
196
|
|
|
197
|
-
def writelines(
|
|
197
|
+
def writelines( # pyre-ignore[14] # pytype: disable=signature-mismatch
|
|
198
|
+
self, lines: Iterable[AnyStr]
|
|
199
|
+
) -> None:
|
|
198
200
|
raise OSError("not writable")
|
|
199
201
|
|
|
200
202
|
|
|
@@ -210,7 +212,9 @@ class Writable(FileLike[AnyStr], ABC):
|
|
|
210
212
|
Return the number of bytes or string written.
|
|
211
213
|
"""
|
|
212
214
|
|
|
213
|
-
def writelines(
|
|
215
|
+
def writelines( # pyre-ignore[14] # pytype: disable=signature-mismatch
|
|
216
|
+
self, lines: Iterable[AnyStr]
|
|
217
|
+
) -> None:
|
|
214
218
|
"""Write `lines` to the file.
|
|
215
219
|
|
|
216
220
|
Note that newlines are not added.
|
megfile/lib/fnmatch.py
CHANGED
|
@@ -13,6 +13,7 @@ corresponding to PATTERN. (It does not compile it.)
|
|
|
13
13
|
"""Compared with the standard library, syntax '{seq1,seq2}' is supported"""
|
|
14
14
|
|
|
15
15
|
import functools
|
|
16
|
+
import io
|
|
16
17
|
import os
|
|
17
18
|
import re
|
|
18
19
|
from typing import Callable, List, Match, Optional
|
|
@@ -71,14 +72,9 @@ def _compat(res: str) -> str:
|
|
|
71
72
|
return r"(?s:%s)\Z" % res
|
|
72
73
|
|
|
73
74
|
|
|
74
|
-
def
|
|
75
|
-
"""Translate a shell PATTERN to a regular expression.
|
|
76
|
-
|
|
77
|
-
There is no way to quote meta-characters.
|
|
78
|
-
"""
|
|
79
|
-
|
|
75
|
+
def _translate(pat: str, match_curly: bool) -> str:
|
|
80
76
|
i, n = 0, len(pat)
|
|
81
|
-
|
|
77
|
+
buf = io.StringIO()
|
|
82
78
|
while i < n:
|
|
83
79
|
c = pat[i]
|
|
84
80
|
i = i + 1
|
|
@@ -90,14 +86,14 @@ def translate(pat: str) -> str:
|
|
|
90
86
|
if (j < n and pat[j] == "/") and (i <= 1 or pat[i - 2] == "/"):
|
|
91
87
|
# hit /**/ instead of /seq**/
|
|
92
88
|
j = j + 1
|
|
93
|
-
|
|
89
|
+
buf.write(r"(.*/)?")
|
|
94
90
|
else:
|
|
95
|
-
|
|
91
|
+
buf.write(r".*")
|
|
96
92
|
else:
|
|
97
|
-
|
|
93
|
+
buf.write(r"[^/]*")
|
|
98
94
|
i = j
|
|
99
95
|
elif c == "?":
|
|
100
|
-
|
|
96
|
+
buf.write(r".")
|
|
101
97
|
elif c == "[":
|
|
102
98
|
j = i
|
|
103
99
|
if j < n and pat[j] == "!":
|
|
@@ -107,7 +103,7 @@ def translate(pat: str) -> str:
|
|
|
107
103
|
while j < n and pat[j] != "]":
|
|
108
104
|
j = j + 1
|
|
109
105
|
if j >= n:
|
|
110
|
-
|
|
106
|
+
buf.write(r"\[")
|
|
111
107
|
else:
|
|
112
108
|
stuff = pat[i:j].replace("\\", r"\\")
|
|
113
109
|
i = j + 1
|
|
@@ -115,20 +111,29 @@ def translate(pat: str) -> str:
|
|
|
115
111
|
stuff = r"^" + stuff[1:]
|
|
116
112
|
elif stuff[0] == "^":
|
|
117
113
|
stuff = "\\" + stuff
|
|
118
|
-
|
|
119
|
-
elif c == "{":
|
|
114
|
+
buf.write(r"[%s]" % stuff)
|
|
115
|
+
elif match_curly and c == "{":
|
|
120
116
|
j = i
|
|
121
117
|
if j < n and pat[j] == "}":
|
|
122
118
|
j = j + 1
|
|
123
119
|
while j < n and pat[j] != "}":
|
|
124
120
|
j = j + 1
|
|
125
121
|
if j >= n:
|
|
126
|
-
|
|
122
|
+
buf.write(r"\{")
|
|
127
123
|
else:
|
|
128
124
|
stuff = pat[i:j].replace("\\", r"\\")
|
|
129
|
-
stuff = r"|".join(
|
|
130
|
-
|
|
125
|
+
stuff = r"|".join(_translate(part, False) for part in stuff.split(","))
|
|
126
|
+
buf.write(r"(%s)" % stuff)
|
|
131
127
|
i = j + 1
|
|
132
128
|
else:
|
|
133
|
-
|
|
134
|
-
return
|
|
129
|
+
buf.write(re.escape(c))
|
|
130
|
+
return buf.getvalue()
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def translate(pat: str) -> str:
|
|
134
|
+
"""Translate a shell PATTERN to a regular expression.
|
|
135
|
+
|
|
136
|
+
There is no way to quote meta-characters.
|
|
137
|
+
"""
|
|
138
|
+
|
|
139
|
+
return _compat(_translate(pat, True))
|
megfile/lib/shadow_handler.py
CHANGED
|
@@ -11,7 +11,9 @@ class BaseShadowHandler(RawIOBase):
|
|
|
11
11
|
"""ShadowHandler using RawIOBase's interface. (avoid type checking error)"""
|
|
12
12
|
|
|
13
13
|
|
|
14
|
-
class ShadowHandler(
|
|
14
|
+
class ShadowHandler( # pytype: disable=signature-mismatch
|
|
15
|
+
Readable, Seekable, Writable, BaseShadowHandler
|
|
16
|
+
):
|
|
15
17
|
"""Create a File-Like Object, maintaining file pointer,
|
|
16
18
|
to avoid misunderstanding the position when read / write / seek.
|
|
17
19
|
|
megfile/s3_path.py
CHANGED
|
@@ -235,10 +235,16 @@ def get_access_token(profile_name=None):
|
|
|
235
235
|
if profile_name
|
|
236
236
|
else "AWS_SECRET_ACCESS_KEY"
|
|
237
237
|
)
|
|
238
|
+
session_token_env_name = (
|
|
239
|
+
f"{profile_name}__AWS_SESSION_TOKEN".upper()
|
|
240
|
+
if profile_name
|
|
241
|
+
else "AWS_SESSION_TOKEN"
|
|
242
|
+
)
|
|
238
243
|
access_key = os.getenv(access_key_env_name)
|
|
239
244
|
secret_key = os.getenv(secret_key_env_name)
|
|
245
|
+
session_token = os.getenv(session_token_env_name)
|
|
240
246
|
if access_key and secret_key:
|
|
241
|
-
return access_key, secret_key
|
|
247
|
+
return access_key, secret_key, session_token
|
|
242
248
|
|
|
243
249
|
try:
|
|
244
250
|
credentials = get_s3_session(profile_name=profile_name).get_credentials()
|
|
@@ -249,7 +255,9 @@ def get_access_token(profile_name=None):
|
|
|
249
255
|
access_key = credentials.access_key
|
|
250
256
|
if not secret_key:
|
|
251
257
|
secret_key = credentials.secret_key
|
|
252
|
-
|
|
258
|
+
if not session_token:
|
|
259
|
+
session_token = credentials.token
|
|
260
|
+
return access_key, secret_key, session_token
|
|
253
261
|
|
|
254
262
|
|
|
255
263
|
def get_s3_client(
|
|
@@ -290,7 +298,7 @@ def get_s3_client(
|
|
|
290
298
|
botocore.config.Config(s3={"addressing_style": addressing_style})
|
|
291
299
|
)
|
|
292
300
|
|
|
293
|
-
access_key, secret_key = get_access_token(profile_name)
|
|
301
|
+
access_key, secret_key, session_token = get_access_token(profile_name)
|
|
294
302
|
try:
|
|
295
303
|
session = get_s3_session(profile_name=profile_name)
|
|
296
304
|
except botocore.exceptions.ProfileNotFound:
|
|
@@ -301,6 +309,7 @@ def get_s3_client(
|
|
|
301
309
|
config=config,
|
|
302
310
|
aws_access_key_id=access_key,
|
|
303
311
|
aws_secret_access_key=secret_key,
|
|
312
|
+
aws_session_token=session_token,
|
|
304
313
|
)
|
|
305
314
|
client = _patch_make_request(client)
|
|
306
315
|
return client
|
megfile/smart_path.py
CHANGED
|
@@ -1,12 +1,17 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from configparser import ConfigParser
|
|
1
3
|
from pathlib import PurePath
|
|
2
|
-
from typing import Tuple, Union
|
|
4
|
+
from typing import Dict, Tuple, Union
|
|
3
5
|
|
|
4
6
|
from megfile.lib.compat import fspath
|
|
5
7
|
from megfile.lib.url import get_url_scheme
|
|
8
|
+
from megfile.utils import classproperty
|
|
6
9
|
|
|
7
10
|
from .errors import ProtocolExistsError, ProtocolNotFoundError
|
|
8
11
|
from .interfaces import BasePath, BaseURIPath, PathLike
|
|
9
12
|
|
|
13
|
+
aliases_config = "~/.config/megfile/aliases.conf"
|
|
14
|
+
|
|
10
15
|
|
|
11
16
|
def _bind_function(name):
|
|
12
17
|
def smart_method(self, *args, **kwargs):
|
|
@@ -25,6 +30,17 @@ def _bind_property(name):
|
|
|
25
30
|
return smart_property
|
|
26
31
|
|
|
27
32
|
|
|
33
|
+
def _load_aliases_config(config_path) -> Dict[str, Dict[str, str]]:
|
|
34
|
+
if not os.path.exists(config_path):
|
|
35
|
+
return {}
|
|
36
|
+
parser = ConfigParser()
|
|
37
|
+
parser.read(config_path)
|
|
38
|
+
configs = {}
|
|
39
|
+
for section in parser.sections():
|
|
40
|
+
configs[section] = dict(parser.items(section))
|
|
41
|
+
return configs
|
|
42
|
+
|
|
43
|
+
|
|
28
44
|
class SmartPath(BasePath):
|
|
29
45
|
_registered_protocols = dict()
|
|
30
46
|
|
|
@@ -38,6 +54,13 @@ class SmartPath(BasePath):
|
|
|
38
54
|
self.path = str(pathlike)
|
|
39
55
|
self.pathlike = pathlike
|
|
40
56
|
|
|
57
|
+
@classproperty
|
|
58
|
+
def _aliases(cls) -> Dict[str, Dict[str, str]]:
|
|
59
|
+
config_path = os.path.expanduser(aliases_config)
|
|
60
|
+
aliases = _load_aliases_config(config_path)
|
|
61
|
+
setattr(cls, "_aliases", aliases)
|
|
62
|
+
return aliases
|
|
63
|
+
|
|
41
64
|
@staticmethod
|
|
42
65
|
def _extract_protocol(path: Union[PathLike, int]) -> Tuple[str, Union[str, int]]:
|
|
43
66
|
if isinstance(path, int):
|
|
@@ -61,7 +84,11 @@ class SmartPath(BasePath):
|
|
|
61
84
|
|
|
62
85
|
@classmethod
|
|
63
86
|
def _create_pathlike(cls, path: Union[PathLike, int]) -> BaseURIPath:
|
|
64
|
-
protocol,
|
|
87
|
+
protocol, path_without_protocol = cls._extract_protocol(path)
|
|
88
|
+
aliases: Dict[str, Dict[str, str]] = cls._aliases # pyre-ignore[9]
|
|
89
|
+
if protocol in aliases:
|
|
90
|
+
protocol = aliases[protocol]["protocol"]
|
|
91
|
+
path = protocol + "://" + str(path_without_protocol)
|
|
65
92
|
if protocol.startswith("s3+"):
|
|
66
93
|
protocol = "s3"
|
|
67
94
|
if protocol not in cls._registered_protocols:
|
megfile/version.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
VERSION = "3.1.
|
|
1
|
+
VERSION = "3.1.5"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: megfile
|
|
3
|
-
Version: 3.1.
|
|
3
|
+
Version: 3.1.5
|
|
4
4
|
Summary: Megvii file operation library
|
|
5
5
|
Author-email: megvii <megfile@megvii.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/megvii-research/megfile
|
|
@@ -166,6 +166,7 @@ You can use environments and configuration file for configuration, and priority
|
|
|
166
166
|
You can use environments to setup authentication credentials for your `s3` account:
|
|
167
167
|
- `AWS_ACCESS_KEY_ID`: access key
|
|
168
168
|
- `AWS_SECRET_ACCESS_KEY`: secret key
|
|
169
|
+
- `AWS_SESSION_TOKEN`: session token
|
|
169
170
|
- `OSS_ENDPOINT` / `AWS_ENDPOINT_URL_S3` / `AWS_ENDPOINT_URL`: endpoint url of s3
|
|
170
171
|
- `AWS_S3_ADDRESSING_STYLE`: addressing style
|
|
171
172
|
|
|
@@ -176,10 +177,10 @@ You can update config file with `megfile` command easyly:
|
|
|
176
177
|
```
|
|
177
178
|
$ megfile config s3 accesskey secretkey
|
|
178
179
|
|
|
179
|
-
# for aliyun
|
|
180
|
+
# for aliyun oss
|
|
180
181
|
$ megfile config s3 accesskey secretkey \
|
|
181
182
|
--addressing-style virtual \
|
|
182
|
-
--endpoint-url http://oss-cn-hangzhou.aliyuncs.com
|
|
183
|
+
--endpoint-url http://oss-cn-hangzhou.aliyuncs.com
|
|
183
184
|
```
|
|
184
185
|
|
|
185
186
|
You can get the configuration from `~/.aws/credentials`, like:
|
|
@@ -193,6 +194,25 @@ s3 =
|
|
|
193
194
|
endpoint_url = http://oss-cn-hangzhou.aliyuncs.com
|
|
194
195
|
```
|
|
195
196
|
|
|
197
|
+
### Create aliases
|
|
198
|
+
```
|
|
199
|
+
# for volcengine tos
|
|
200
|
+
$ megfile config s3 accesskey secretkey \
|
|
201
|
+
--addressing-style virtual \
|
|
202
|
+
--endpoint-url https://tos-s3-cn-beijing.ivolces.com \
|
|
203
|
+
--profile tos
|
|
204
|
+
|
|
205
|
+
# create alias
|
|
206
|
+
$ megfile config tos s3+tos
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
You can get the configuration from `~/.config/megfile/aliases.conf`, like:
|
|
210
|
+
```
|
|
211
|
+
[tos]
|
|
212
|
+
protocol = s3+tos
|
|
213
|
+
```
|
|
214
|
+
|
|
215
|
+
|
|
196
216
|
## How to Contribute
|
|
197
217
|
* We welcome everyone to contribute code to the `megfile` project, but the contributed code needs to meet the following conditions as much as possible:
|
|
198
218
|
|
|
@@ -1,31 +1,31 @@
|
|
|
1
1
|
docs/conf.py,sha256=sfDSly5jO8W_RmuAptOIp4hd8dNcO-9a5XrHTbxFnNo,2448
|
|
2
2
|
megfile/__init__.py,sha256=i2Lbq_VxIgppaqwkxG0_H35dRfcjJ4mCYWjprOf4hHo,7318
|
|
3
|
-
megfile/cli.py,sha256=
|
|
3
|
+
megfile/cli.py,sha256=miI1fEHY3zBxWwwN5F5AHjO14-AI920oPxD4sfi94gY,23465
|
|
4
4
|
megfile/config.py,sha256=_SkJRaVWUdfW1Q9uX0vao-6YVQKJtfej22Z8DykuRps,2331
|
|
5
|
-
megfile/errors.py,sha256=
|
|
5
|
+
megfile/errors.py,sha256=KyHvK3CgiSgiXT_5kqKa7Nb8Bdwh7U6mBh_H7Gj-Z84,14125
|
|
6
6
|
megfile/fs.py,sha256=dgj5fW-EEzQNdjMF2tkB5DjXu3iHQbtLi5PSIMxR8fc,11966
|
|
7
7
|
megfile/fs_path.py,sha256=Ffvukc176beH5aQMZXXtwH6ApwLYXPViCIUP0pijgT0,41590
|
|
8
8
|
megfile/hdfs.py,sha256=latguOuuzAmg-yWOy3Sm723CJ0ybN_eSHRubVNqhcMU,9202
|
|
9
9
|
megfile/hdfs_path.py,sha256=0XLtABufwqL-y8igOxzOJz6zOGppuBp2f2SwXIMvvYg,27299
|
|
10
10
|
megfile/http.py,sha256=2Z2yqyhU-zcJCJwSNyBsxsZ7f2FT9X6fcednsbHDsFM,2025
|
|
11
11
|
megfile/http_path.py,sha256=BhMNjQVB85IaCGGIKzgEfY73mAVdCzJP08W1RuGeMRA,16119
|
|
12
|
-
megfile/interfaces.py,sha256=
|
|
12
|
+
megfile/interfaces.py,sha256=7C53Q2FAVFmOEnplfplvWqHab29HJE5RQnpfdb4loVY,8679
|
|
13
13
|
megfile/pathlike.py,sha256=vKuCMlSAPYNSojp03wEj2i3Cq3E3ROp_-UkkdgBElws,30802
|
|
14
14
|
megfile/s3.py,sha256=7SdfLjAePVh-bpRyuj566VB4Qa7KP86rCJGzYANR7wQ,13008
|
|
15
|
-
megfile/s3_path.py,sha256=
|
|
15
|
+
megfile/s3_path.py,sha256=YvIRVI4d6kd2m2wvcIqiu43nPOYZpaPgG9-BospF81w,93803
|
|
16
16
|
megfile/sftp.py,sha256=vyDnYXX3i1j2fhXMC8YCeX-66MDb9wrBQQjQVhZx0uo,13004
|
|
17
17
|
megfile/sftp_path.py,sha256=4tByWvUJK1KBJoa3t5aoWYnZpaRWN9nQIE6ZyiGHrbk,53519
|
|
18
18
|
megfile/smart.py,sha256=Vr4R7HpjXjt587KOc2-1QGbQ5EsZ48YRzCaK0rz3IS0,36108
|
|
19
|
-
megfile/smart_path.py,sha256=
|
|
19
|
+
megfile/smart_path.py,sha256=22ZTrA7j9Kd5PJsUJOxBdGg0Uu5FxwdT_XQjQDxUHo4,7637
|
|
20
20
|
megfile/stdio.py,sha256=UYe-h440Wc4f5COOzOTG1svnp5nFzrfpixehJ0_0_NY,653
|
|
21
21
|
megfile/stdio_path.py,sha256=7jzVdreamO18yBWZM7Pp71cO7GmrYb0M0qyQde2Ypq4,2706
|
|
22
|
-
megfile/version.py,sha256=
|
|
22
|
+
megfile/version.py,sha256=kh8D3nOB4xuR3KqckCcukx60UxptlmvEx5kjmgZinIA,19
|
|
23
23
|
megfile/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
24
24
|
megfile/lib/base_prefetch_reader.py,sha256=CaYWuiKmlk4Utr0IFDPwPC58wV2jBAhqpxhwhRHc734,13652
|
|
25
25
|
megfile/lib/combine_reader.py,sha256=uSzo3PmhD5ck6_Vv6dFU5vVx4boeA97VS-puPyhF_BE,4657
|
|
26
26
|
megfile/lib/compare.py,sha256=n_dtLxgoskYnsIZMKdKmVhQoVn8qYUrUhkS1JH2_X3o,2170
|
|
27
27
|
megfile/lib/compat.py,sha256=SynEeHluys3tCK-lb_1oV3o_ft83yZvunqM_AjibLgE,207
|
|
28
|
-
megfile/lib/fnmatch.py,sha256=
|
|
28
|
+
megfile/lib/fnmatch.py,sha256=4MvGzEahMRA-u8Z7mxaD-Yw1idOwBoJJpVywQy29jwY,4162
|
|
29
29
|
megfile/lib/glob.py,sha256=iJ0NvFh7b07MDru36YY1j3ZWPCNBLAECzUkoqYfJWgY,10052
|
|
30
30
|
megfile/lib/hdfs_prefetch_reader.py,sha256=UrcUmTM1IZwD95oZMJXuY2dYEpE7uUjs_6dHyTMYDbg,2129
|
|
31
31
|
megfile/lib/hdfs_tools.py,sha256=4K-OdMYFFSLBGmDzjatioHvuZuUbKVy7ACeJl-l0HLQ,435
|
|
@@ -39,17 +39,17 @@ megfile/lib/s3_memory_handler.py,sha256=NGKWbI4LG2cmV06CP7KOVPqS_BNpm3ApqKi5ibgI
|
|
|
39
39
|
megfile/lib/s3_pipe_handler.py,sha256=DY1UTNCq8oD3QWXNb4orOiz3EoEAo6dhwmZZdk6h1bU,3694
|
|
40
40
|
megfile/lib/s3_prefetch_reader.py,sha256=YZA6JOQXcioREh_z1E-kZ2WRPTm02v0dCEVqyaOMHns,4287
|
|
41
41
|
megfile/lib/s3_share_cache_reader.py,sha256=jhGL1B6NPv68cQnW1Jf7ey-zTQ8XfiJg5ILDNgRWHy0,3671
|
|
42
|
-
megfile/lib/shadow_handler.py,sha256=
|
|
42
|
+
megfile/lib/shadow_handler.py,sha256=TntewlvIW9ZxCfmqASDQREHoiZ8v42faOe9sovQYQz0,2779
|
|
43
43
|
megfile/lib/stdio_handler.py,sha256=IDdgENLQlhigEwkLL4zStueVSzdWg7xVcTF_koof_Ek,1987
|
|
44
44
|
megfile/lib/url.py,sha256=ER32pWy9Q2MAk3TraAaNEBWIqUeBmLuM57ol2cs7-Ks,103
|
|
45
45
|
megfile/utils/__init__.py,sha256=NfO5vNxfeceGvMB3dgZNudyPFTmPY096JbC4iYroX6o,9003
|
|
46
46
|
megfile/utils/mutex.py,sha256=asb8opGLgK22RiuBJUnfsvB8LnMmodP8KzCVHKmQBWA,2561
|
|
47
47
|
scripts/convert_results_to_sarif.py,sha256=nDiOfsedb22Ps7ZodmYdlXZlxv54fRxCQgOZsB2OkNk,2833
|
|
48
48
|
scripts/generate_file.py,sha256=-mTcBiqiQ1juvqojVfVZ-uZWgpANHJNdhrF7s68zNfc,10903
|
|
49
|
-
megfile-3.1.
|
|
50
|
-
megfile-3.1.
|
|
51
|
-
megfile-3.1.
|
|
52
|
-
megfile-3.1.
|
|
53
|
-
megfile-3.1.
|
|
54
|
-
megfile-3.1.
|
|
55
|
-
megfile-3.1.
|
|
49
|
+
megfile-3.1.5.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
|
|
50
|
+
megfile-3.1.5.dist-info/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
|
|
51
|
+
megfile-3.1.5.dist-info/METADATA,sha256=jPBOY1WWbh-Jxdam48x2RxcXGVV2liPW4kQTw8VtBl0,9178
|
|
52
|
+
megfile-3.1.5.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
|
|
53
|
+
megfile-3.1.5.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
|
|
54
|
+
megfile-3.1.5.dist-info/top_level.txt,sha256=oTnYXo1Z3V61qSWAKtnY9RkDgRSHvfRN38FQae6E0W0,50
|
|
55
|
+
megfile-3.1.5.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|