megfile 2.1.4__py3-none-any.whl → 2.2.0.post1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- megfile/__init__.py +12 -2
- megfile/cli.py +80 -2
- megfile/errors.py +4 -1
- megfile/lib/compat.py +1 -1
- megfile/lib/s3_prefetch_reader.py +4 -2
- megfile/s3_path.py +22 -4
- megfile/sftp_path.py +47 -25
- megfile/smart.py +1 -0
- megfile/version.py +1 -1
- {megfile-2.1.4.dist-info → megfile-2.2.0.post1.dist-info}/METADATA +25 -2
- {megfile-2.1.4.dist-info → megfile-2.2.0.post1.dist-info}/RECORD +16 -16
- {megfile-2.1.4.dist-info → megfile-2.2.0.post1.dist-info}/LICENSE +0 -0
- {megfile-2.1.4.dist-info → megfile-2.2.0.post1.dist-info}/LICENSE.pyre +0 -0
- {megfile-2.1.4.dist-info → megfile-2.2.0.post1.dist-info}/WHEEL +0 -0
- {megfile-2.1.4.dist-info → megfile-2.2.0.post1.dist-info}/entry_points.txt +0 -0
- {megfile-2.1.4.dist-info → megfile-2.2.0.post1.dist-info}/top_level.txt +0 -0
megfile/__init__.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
from megfile.fs import fs_abspath, fs_access, fs_cwd, fs_exists, fs_expanduser, fs_getmd5, fs_getmtime, fs_getsize, fs_glob, fs_glob_stat, fs_home, fs_iglob, fs_isabs, fs_isdir, fs_isfile, fs_islink, fs_ismount, fs_listdir, fs_load_from, fs_lstat, fs_makedirs, fs_move, fs_readlink, fs_realpath, fs_relpath, fs_remove, fs_rename, fs_resolve, fs_save_as, fs_scan, fs_scan_stat, fs_scandir, fs_stat, fs_symlink, fs_sync, fs_unlink, fs_walk, is_fs
|
|
2
2
|
from megfile.fs_path import FSPath
|
|
3
|
+
from megfile.http import http_exists, http_getmtime, http_getsize, http_open, http_stat, is_http
|
|
3
4
|
from megfile.http_path import HttpPath, HttpsPath
|
|
4
5
|
from megfile.s3 import is_s3, s3_access, s3_buffered_open, s3_cached_open, s3_concat, s3_copy, s3_download, s3_exists, s3_getmd5, s3_getmtime, s3_getsize, s3_glob, s3_glob_stat, s3_hasbucket, s3_iglob, s3_isdir, s3_isfile, s3_listdir, s3_load_content, s3_load_from, s3_lstat, s3_makedirs, s3_memory_open, s3_move, s3_open, s3_path_join, s3_pipe_open, s3_prefetch_open, s3_readlink, s3_remove, s3_rename, s3_save_as, s3_scan, s3_scan_stat, s3_scandir, s3_stat, s3_symlink, s3_sync, s3_unlink, s3_upload, s3_walk
|
|
5
6
|
from megfile.s3_path import S3Path
|
|
@@ -7,12 +8,11 @@ from megfile.sftp import is_sftp, sftp_absolute, sftp_chmod, sftp_concat, sftp_c
|
|
|
7
8
|
from megfile.sftp_path import SftpPath
|
|
8
9
|
from megfile.smart import smart_access, smart_cache, smart_combine_open, smart_concat, smart_copy, smart_exists, smart_getmd5, smart_getmtime, smart_getsize, smart_glob, smart_glob_stat, smart_iglob, smart_isdir, smart_isfile, smart_islink, smart_listdir, smart_load_content, smart_load_from, smart_load_text, smart_lstat, smart_makedirs, smart_move, smart_open, smart_path_join, smart_readlink, smart_realpath, smart_remove, smart_rename, smart_save_as, smart_save_content, smart_save_text, smart_scan, smart_scan_stat, smart_scandir, smart_stat, smart_symlink, smart_sync, smart_touch, smart_unlink, smart_walk
|
|
9
10
|
from megfile.smart_path import SmartPath
|
|
11
|
+
from megfile.stdio import is_stdio, stdio_open
|
|
10
12
|
from megfile.stdio_path import StdioPath
|
|
11
13
|
from megfile.version import VERSION as __version__
|
|
12
14
|
|
|
13
15
|
__all__ = [
|
|
14
|
-
'is_fs',
|
|
15
|
-
'is_s3',
|
|
16
16
|
'smart_access',
|
|
17
17
|
'smart_cache',
|
|
18
18
|
'smart_combine_open',
|
|
@@ -54,6 +54,7 @@ __all__ = [
|
|
|
54
54
|
'smart_readlink',
|
|
55
55
|
'smart_lstat',
|
|
56
56
|
'smart_concat',
|
|
57
|
+
'is_s3',
|
|
57
58
|
's3_access',
|
|
58
59
|
's3_buffered_open',
|
|
59
60
|
's3_cached_open',
|
|
@@ -94,6 +95,7 @@ __all__ = [
|
|
|
94
95
|
's3_symlink',
|
|
95
96
|
's3_readlink',
|
|
96
97
|
's3_concat',
|
|
98
|
+
'is_fs',
|
|
97
99
|
'fs_abspath',
|
|
98
100
|
'fs_access',
|
|
99
101
|
'fs_exists',
|
|
@@ -131,6 +133,14 @@ __all__ = [
|
|
|
131
133
|
'fs_getmd5',
|
|
132
134
|
'fs_symlink',
|
|
133
135
|
'fs_readlink',
|
|
136
|
+
'is_http',
|
|
137
|
+
'http_open',
|
|
138
|
+
'http_stat',
|
|
139
|
+
'http_getsize',
|
|
140
|
+
'http_getmtime',
|
|
141
|
+
'http_exists',
|
|
142
|
+
'is_stdio',
|
|
143
|
+
'stdio_open',
|
|
134
144
|
'is_sftp',
|
|
135
145
|
'sftp_readlink',
|
|
136
146
|
'sftp_absolute',
|
megfile/cli.py
CHANGED
|
@@ -17,11 +17,12 @@ from megfile.version import VERSION
|
|
|
17
17
|
|
|
18
18
|
logging.basicConfig(level=logging.ERROR)
|
|
19
19
|
logging.getLogger('megfile').setLevel(level=logging.INFO)
|
|
20
|
+
DEFAULT_BLOCK_SIZE = 8 * 2**20 # 8MB
|
|
20
21
|
|
|
21
22
|
|
|
22
23
|
@click.group()
|
|
23
24
|
def cli():
|
|
24
|
-
"""
|
|
25
|
+
"""Client"""
|
|
25
26
|
|
|
26
27
|
|
|
27
28
|
def safe_cli(): # pragma: no cover
|
|
@@ -306,11 +307,88 @@ def cat(path: str):
|
|
|
306
307
|
shutil.copyfileobj(file, sys.stdout.buffer)
|
|
307
308
|
|
|
308
309
|
|
|
310
|
+
@cli.command(
|
|
311
|
+
short_help='Concatenate any files and send first n lines of them to stdout.'
|
|
312
|
+
)
|
|
313
|
+
@click.argument('path')
|
|
314
|
+
@click.option(
|
|
315
|
+
'-n',
|
|
316
|
+
'--lines',
|
|
317
|
+
type=click.INT,
|
|
318
|
+
default=10,
|
|
319
|
+
help='print the first NUM lines')
|
|
320
|
+
def head(path: str, lines: int):
|
|
321
|
+
with smart_open(path, 'rb') as f:
|
|
322
|
+
for _ in range(lines):
|
|
323
|
+
try:
|
|
324
|
+
content = f.readline()
|
|
325
|
+
if not content:
|
|
326
|
+
break
|
|
327
|
+
except EOFError:
|
|
328
|
+
break
|
|
329
|
+
click.echo(content.strip(b'\n'))
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
@cli.command(
|
|
333
|
+
short_help='Concatenate any files and send last n lines of them to stdout.')
|
|
334
|
+
@click.argument('path')
|
|
335
|
+
@click.option(
|
|
336
|
+
'-n',
|
|
337
|
+
'--lines',
|
|
338
|
+
type=click.INT,
|
|
339
|
+
default=10,
|
|
340
|
+
help='print the last NUM lines')
|
|
341
|
+
def tail(path: str, lines: int):
|
|
342
|
+
line_list = []
|
|
343
|
+
with smart_open(path, 'rb') as f:
|
|
344
|
+
f.seek(0, os.SEEK_END)
|
|
345
|
+
file_size = f.tell()
|
|
346
|
+
f.seek(0, os.SEEK_SET)
|
|
347
|
+
|
|
348
|
+
for current_offset in range(file_size - DEFAULT_BLOCK_SIZE,
|
|
349
|
+
0 - DEFAULT_BLOCK_SIZE,
|
|
350
|
+
-DEFAULT_BLOCK_SIZE):
|
|
351
|
+
current_offset = max(0, current_offset)
|
|
352
|
+
f.seek(current_offset)
|
|
353
|
+
block_lines = f.read(DEFAULT_BLOCK_SIZE).split(b'\n')
|
|
354
|
+
if len(line_list) > 0:
|
|
355
|
+
block_lines[-1] += line_list[0]
|
|
356
|
+
block_lines.extend(line_list[1:])
|
|
357
|
+
if len(block_lines) > lines:
|
|
358
|
+
line_list = block_lines[-lines:]
|
|
359
|
+
break
|
|
360
|
+
else:
|
|
361
|
+
line_list = block_lines
|
|
362
|
+
for line in line_list:
|
|
363
|
+
click.echo(line)
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
@cli.command(short_help='Write bytes from stdin to file.')
|
|
367
|
+
@click.argument('path')
|
|
368
|
+
@click.option('-a', '--append', is_flag=True, help='Append to the given file')
|
|
369
|
+
@click.option(
|
|
370
|
+
'-o', '--stdout', is_flag=True, help='File content to standard output')
|
|
371
|
+
def to(path: str, append: bool, stdout: bool):
|
|
372
|
+
mode = 'wb'
|
|
373
|
+
if append:
|
|
374
|
+
mode = 'ab'
|
|
375
|
+
with smart_open('stdio://0', 'rb') as stdin, smart_open(
|
|
376
|
+
path, mode) as f, smart_open('stdio://1', 'wb') as stdout_fd:
|
|
377
|
+
length = 16 * 1024
|
|
378
|
+
while True:
|
|
379
|
+
buf = stdin.read(length)
|
|
380
|
+
if not buf:
|
|
381
|
+
break
|
|
382
|
+
f.write(buf)
|
|
383
|
+
if stdout:
|
|
384
|
+
stdout_fd.write(buf)
|
|
385
|
+
|
|
386
|
+
|
|
309
387
|
@cli.command(
|
|
310
388
|
short_help='Produce an md5sum file for all the objects in the path.')
|
|
311
389
|
@click.argument('path')
|
|
312
390
|
def md5sum(path: str):
|
|
313
|
-
click.echo(smart_getmd5(path))
|
|
391
|
+
click.echo(smart_getmd5(path, recalculate=True))
|
|
314
392
|
|
|
315
393
|
|
|
316
394
|
@cli.command(
|
megfile/errors.py
CHANGED
|
@@ -139,6 +139,9 @@ def patch_method(
|
|
|
139
139
|
result = func(*args, **kwargs)
|
|
140
140
|
if after_callback is not None:
|
|
141
141
|
result = after_callback(result, *args, **kwargs)
|
|
142
|
+
if retries > 1:
|
|
143
|
+
_logger.info(
|
|
144
|
+
f'Error already fixed by retry {retries - 1} times')
|
|
142
145
|
return result
|
|
143
146
|
except Exception as error:
|
|
144
147
|
if not should_retry(error):
|
|
@@ -148,7 +151,7 @@ def patch_method(
|
|
|
148
151
|
if retries == max_retries:
|
|
149
152
|
raise
|
|
150
153
|
retry_interval = min(0.1 * 2**retries, 30)
|
|
151
|
-
_logger.
|
|
154
|
+
_logger.info(
|
|
152
155
|
'unknown error encountered: %s, retry in %0.1f seconds after %d tries'
|
|
153
156
|
% (full_error_message(error), retry_interval, retries))
|
|
154
157
|
time.sleep(retry_interval)
|
megfile/lib/compat.py
CHANGED
|
@@ -361,8 +361,10 @@ class S3PrefetchReader(Readable, Seekable):
|
|
|
361
361
|
|
|
362
362
|
range_str = 'bytes=%d-%d' % (
|
|
363
363
|
index * self._block_size, (index + 1) * self._block_size - 1)
|
|
364
|
-
|
|
364
|
+
response = self._client.get_object(
|
|
365
365
|
Bucket=self._bucket, Key=self._key, Range=range_str)
|
|
366
|
+
response['Body'] = BytesIO(response['Body'].read())
|
|
367
|
+
return response
|
|
366
368
|
|
|
367
369
|
fetch_response = patch_method(
|
|
368
370
|
fetch_response,
|
|
@@ -380,7 +382,7 @@ class S3PrefetchReader(Readable, Seekable):
|
|
|
380
382
|
'File changed: %r, etag before: %s, after: %s' %
|
|
381
383
|
(self.name, self._content_info, response))
|
|
382
384
|
|
|
383
|
-
return
|
|
385
|
+
return response['Body']
|
|
384
386
|
|
|
385
387
|
def _submit_future(self, index: int):
|
|
386
388
|
if index < 0 or index >= self._block_stop:
|
megfile/s3_path.py
CHANGED
|
@@ -864,6 +864,9 @@ def s3_download(
|
|
|
864
864
|
:param dst_url: target fs path
|
|
865
865
|
:param callback: Called periodically during copy, and the input parameter is the data size (in bytes) of copy since the last call
|
|
866
866
|
'''
|
|
867
|
+
from megfile.fs import is_fs
|
|
868
|
+
from megfile.fs_path import FSPath
|
|
869
|
+
|
|
867
870
|
src_url = S3Path(src_url)
|
|
868
871
|
if followlinks:
|
|
869
872
|
try:
|
|
@@ -884,23 +887,31 @@ def s3_download(
|
|
|
884
887
|
'Is a directory: %r' % src_url.path_with_protocol)
|
|
885
888
|
|
|
886
889
|
dst_url = fspath(dst_url)
|
|
890
|
+
if not is_fs(dst_url):
|
|
891
|
+
raise OSError(f'dst_url is not fs path: {dst_url}')
|
|
887
892
|
if not dst_url or dst_url.endswith('/'):
|
|
888
893
|
raise S3IsADirectoryError('Is a directory: %r' % dst_url)
|
|
889
894
|
|
|
890
|
-
|
|
895
|
+
dst_path = FSPath(dst_url)
|
|
896
|
+
dst_directory = os.path.dirname(dst_path.path_without_protocol)
|
|
891
897
|
if dst_directory != '':
|
|
892
898
|
os.makedirs(dst_directory, exist_ok=True)
|
|
893
899
|
|
|
894
900
|
client = get_s3_client(profile_name=src_url._profile_name)
|
|
895
901
|
try:
|
|
896
|
-
client.download_file(
|
|
902
|
+
client.download_file(
|
|
903
|
+
src_bucket,
|
|
904
|
+
src_key,
|
|
905
|
+
dst_path.path_without_protocol,
|
|
906
|
+
Callback=callback)
|
|
897
907
|
except Exception as error:
|
|
898
908
|
error = translate_fs_error(error, dst_url)
|
|
899
909
|
error = translate_s3_error(error, src_url.path_with_protocol)
|
|
900
910
|
raise error
|
|
901
911
|
|
|
902
912
|
src_stat = src_url.stat()
|
|
903
|
-
os.utime(
|
|
913
|
+
os.utime(
|
|
914
|
+
dst_path.path_without_protocol, (src_stat.st_mtime, src_stat.st_mtime))
|
|
904
915
|
|
|
905
916
|
|
|
906
917
|
def s3_upload(
|
|
@@ -914,6 +925,13 @@ def s3_upload(
|
|
|
914
925
|
:param dst_url: target s3 path
|
|
915
926
|
:param callback: Called periodically during copy, and the input parameter is the data size (in bytes) of copy since the last call
|
|
916
927
|
'''
|
|
928
|
+
from megfile.fs import is_fs
|
|
929
|
+
from megfile.fs_path import FSPath
|
|
930
|
+
|
|
931
|
+
if not is_fs(src_url):
|
|
932
|
+
raise OSError(f'src_url is not fs path: {src_url}')
|
|
933
|
+
src_path = FSPath(src_url)
|
|
934
|
+
|
|
917
935
|
dst_bucket, dst_key = parse_s3_url(dst_url)
|
|
918
936
|
if not dst_bucket:
|
|
919
937
|
raise S3BucketNotFoundError('Empty bucket name: %r' % dst_url)
|
|
@@ -921,7 +939,7 @@ def s3_upload(
|
|
|
921
939
|
raise S3IsADirectoryError('Is a directory: %r' % dst_url)
|
|
922
940
|
|
|
923
941
|
client = get_s3_client(profile_name=S3Path(dst_url)._profile_name)
|
|
924
|
-
with open(
|
|
942
|
+
with open(src_path.path_without_protocol, 'rb') as src:
|
|
925
943
|
with raise_s3_error(dst_url):
|
|
926
944
|
client.upload_fileobj(
|
|
927
945
|
src, Bucket=dst_bucket, Key=dst_key, Callback=callback)
|
megfile/sftp_path.py
CHANGED
|
@@ -367,21 +367,24 @@ def sftp_download(
|
|
|
367
367
|
'''
|
|
368
368
|
File download
|
|
369
369
|
'''
|
|
370
|
-
from megfile.
|
|
370
|
+
from megfile.fs import is_fs
|
|
371
|
+
from megfile.fs_path import FSPath
|
|
372
|
+
|
|
371
373
|
if not is_fs(dst_url):
|
|
372
374
|
raise OSError(f'dst_url is not fs path: {dst_url}')
|
|
373
375
|
if not is_sftp(src_url):
|
|
374
376
|
raise OSError(f'src_url is not sftp path: {src_url}')
|
|
375
377
|
|
|
376
|
-
|
|
377
|
-
if followlinks and
|
|
378
|
-
|
|
379
|
-
if
|
|
378
|
+
src_path = SftpPath(src_url)
|
|
379
|
+
if followlinks and src_path.is_symlink():
|
|
380
|
+
src_path = src_path.readlink()
|
|
381
|
+
if src_path.is_dir():
|
|
380
382
|
raise IsADirectoryError('Is a directory: %r' % src_url)
|
|
381
383
|
if str(dst_url).endswith('/'):
|
|
382
384
|
raise IsADirectoryError('Is a directory: %r' % dst_url)
|
|
383
385
|
|
|
384
|
-
|
|
386
|
+
dst_path = FSPath(dst_url)
|
|
387
|
+
dst_path.parent.makedirs(exist_ok=True)
|
|
385
388
|
|
|
386
389
|
sftp_callback = None
|
|
387
390
|
if callback:
|
|
@@ -389,10 +392,12 @@ def sftp_download(
|
|
|
389
392
|
def sftp_callback(bytes_transferred: int, _total_bytes: int):
|
|
390
393
|
callback(bytes_transferred)
|
|
391
394
|
|
|
392
|
-
|
|
395
|
+
src_path._client.get(
|
|
396
|
+
src_path._real_path,
|
|
397
|
+
dst_path.path_without_protocol,
|
|
398
|
+
callback=sftp_callback)
|
|
393
399
|
|
|
394
|
-
src_stat =
|
|
395
|
-
dst_path = FSPath(dst_url)
|
|
400
|
+
src_stat = src_path.stat()
|
|
396
401
|
dst_path.utime(src_stat.st_atime, src_stat.st_mtime)
|
|
397
402
|
dst_path.chmod(src_stat.st_mode)
|
|
398
403
|
|
|
@@ -405,7 +410,9 @@ def sftp_upload(
|
|
|
405
410
|
'''
|
|
406
411
|
File upload
|
|
407
412
|
'''
|
|
408
|
-
from megfile.fs import
|
|
413
|
+
from megfile.fs import is_fs
|
|
414
|
+
from megfile.fs_path import FSPath
|
|
415
|
+
|
|
409
416
|
if not is_fs(src_url):
|
|
410
417
|
raise OSError(f'src_url is not fs path: {src_url}')
|
|
411
418
|
if not is_sftp(dst_url):
|
|
@@ -418,8 +425,9 @@ def sftp_upload(
|
|
|
418
425
|
if str(dst_url).endswith('/'):
|
|
419
426
|
raise IsADirectoryError('Is a directory: %r' % dst_url)
|
|
420
427
|
|
|
421
|
-
|
|
422
|
-
dst_url
|
|
428
|
+
src_path = FSPath(src_url)
|
|
429
|
+
dst_path = SftpPath(dst_url)
|
|
430
|
+
dst_path.parent.makedirs(exist_ok=True)
|
|
423
431
|
|
|
424
432
|
sftp_callback = None
|
|
425
433
|
if callback:
|
|
@@ -427,11 +435,14 @@ def sftp_upload(
|
|
|
427
435
|
def sftp_callback(bytes_transferred: int, _total_bytes: int):
|
|
428
436
|
callback(bytes_transferred)
|
|
429
437
|
|
|
430
|
-
|
|
438
|
+
dst_path._client.put(
|
|
439
|
+
src_path.path_without_protocol,
|
|
440
|
+
dst_path._real_path,
|
|
441
|
+
callback=sftp_callback)
|
|
431
442
|
|
|
432
|
-
src_stat =
|
|
433
|
-
|
|
434
|
-
|
|
443
|
+
src_stat = src_path.stat()
|
|
444
|
+
dst_path.utime(src_stat.st_atime, src_stat.st_mtime)
|
|
445
|
+
dst_path.chmod(src_stat.st_mode)
|
|
435
446
|
|
|
436
447
|
|
|
437
448
|
def sftp_path_join(path: PathLike, *other_paths: PathLike) -> str:
|
|
@@ -475,8 +486,11 @@ def sftp_concat(src_paths: List[PathLike], dst_path: PathLike) -> None:
|
|
|
475
486
|
class SftpPath(URIPath):
|
|
476
487
|
"""sftp protocol
|
|
477
488
|
|
|
478
|
-
uri format:
|
|
479
|
-
|
|
489
|
+
uri format:
|
|
490
|
+
- absolute path
|
|
491
|
+
- sftp://[username[:password]@]hostname[:port]//file_path
|
|
492
|
+
- relative path
|
|
493
|
+
- - sftp://[username[:password]@]hostname[:port]/file_path
|
|
480
494
|
"""
|
|
481
495
|
|
|
482
496
|
protocol = "sftp"
|
|
@@ -484,10 +498,13 @@ class SftpPath(URIPath):
|
|
|
484
498
|
def __init__(self, path: "PathLike", *other_paths: "PathLike"):
|
|
485
499
|
super().__init__(path, *other_paths)
|
|
486
500
|
parts = urlsplit(self.path)
|
|
487
|
-
self._real_path = parts.path
|
|
488
|
-
if not self._real_path.startswith('/'):
|
|
489
|
-
self._real_path = f"/{self._real_path}"
|
|
490
501
|
self._urlsplit_parts = parts
|
|
502
|
+
self._real_path = parts.path
|
|
503
|
+
if parts.path.startswith('//'):
|
|
504
|
+
self._root_dir = '/'
|
|
505
|
+
else:
|
|
506
|
+
self._root_dir = self._client.normalize('.')
|
|
507
|
+
self._real_path = os.path.join(self._root_dir, parts.path.lstrip('/'))
|
|
491
508
|
|
|
492
509
|
@property
|
|
493
510
|
def _client(self):
|
|
@@ -497,9 +514,14 @@ class SftpPath(URIPath):
|
|
|
497
514
|
username=self._urlsplit_parts.username,
|
|
498
515
|
password=self._urlsplit_parts.password)
|
|
499
516
|
|
|
500
|
-
def _generate_path_object(
|
|
501
|
-
|
|
502
|
-
|
|
517
|
+
def _generate_path_object(
|
|
518
|
+
self, sftp_local_path: str, resolve: bool = False):
|
|
519
|
+
if resolve or self._root_dir == '/':
|
|
520
|
+
sftp_local_path = f"//{sftp_local_path.lstrip('/')}"
|
|
521
|
+
else:
|
|
522
|
+
sftp_local_path = os.path.relpath(
|
|
523
|
+
sftp_local_path, start=self._root_dir)
|
|
524
|
+
new_parts = self._urlsplit_parts._replace(path=sftp_local_path)
|
|
503
525
|
return self.from_path(urlunsplit(new_parts))
|
|
504
526
|
|
|
505
527
|
def exists(self, followlinks: bool = False) -> bool:
|
|
@@ -971,7 +993,7 @@ class SftpPath(URIPath):
|
|
|
971
993
|
:rtype: SftpPath
|
|
972
994
|
'''
|
|
973
995
|
path = self._client.normalize(self._real_path)
|
|
974
|
-
return self._generate_path_object(path)
|
|
996
|
+
return self._generate_path_object(path, resolve=True)
|
|
975
997
|
|
|
976
998
|
def md5(self, recalculate: bool = False, followlinks: bool = True):
|
|
977
999
|
'''
|
megfile/smart.py
CHANGED
megfile/version.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
VERSION = "2.
|
|
1
|
+
VERSION = "2.2.0.post1"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: megfile
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.2.0.post1
|
|
4
4
|
Summary: Megvii file operation library
|
|
5
5
|
Home-page: https://github.com/megvii-research/megfile
|
|
6
6
|
Author: megvii
|
|
@@ -62,6 +62,29 @@ megfile - Megvii FILE library
|
|
|
62
62
|
|
|
63
63
|
Here's an example of writing a file to s3 / sftp / fs, syncing to local, reading and finally deleting it.
|
|
64
64
|
|
|
65
|
+
### Path Format
|
|
66
|
+
- local file
|
|
67
|
+
- unix filesystem path
|
|
68
|
+
- examples:
|
|
69
|
+
- `/data/test.txt`
|
|
70
|
+
- `test.txt`
|
|
71
|
+
- 1
|
|
72
|
+
- oss
|
|
73
|
+
- `s3[+profile_name]://bucket/key`
|
|
74
|
+
- sftp
|
|
75
|
+
- `sftp://[username[:password]@]hostname[:port]//absolute_file_path`
|
|
76
|
+
- `sftp://[username[:password]@]hostname[:port]/relative_file_path`
|
|
77
|
+
- http
|
|
78
|
+
- http / https url
|
|
79
|
+
- examples:
|
|
80
|
+
- `http://hostname/test`
|
|
81
|
+
- `https://hostname/test`
|
|
82
|
+
- stdio
|
|
83
|
+
- `stdio://-`
|
|
84
|
+
- `stdio://0`
|
|
85
|
+
- `stdio://1`
|
|
86
|
+
- `stdio://2`
|
|
87
|
+
|
|
65
88
|
### Functional Interface
|
|
66
89
|
```python
|
|
67
90
|
from megfile import smart_open, smart_exists, smart_sync, smart_remove, smart_glob
|
|
@@ -201,7 +224,7 @@ s3 =
|
|
|
201
224
|
megfile.smart_copy('s3+profile1://bucket/key', 's3+profile2://bucket/key')
|
|
202
225
|
```
|
|
203
226
|
|
|
204
|
-
sftp
|
|
227
|
+
sftp support some environments:
|
|
205
228
|
```
|
|
206
229
|
# If you are not set username or password in path, you can set them in environments
|
|
207
230
|
$ export SFTP_USERNAME=user
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
megfile/__init__.py,sha256=
|
|
2
|
-
megfile/cli.py,sha256=
|
|
3
|
-
megfile/errors.py,sha256=
|
|
1
|
+
megfile/__init__.py,sha256=Qsi3XNP_0XYoSol-1AGutZqo0rfBnzaiZ-HVXll4fB0,5721
|
|
2
|
+
megfile/cli.py,sha256=E3NoSZ8D-k3OCRXPDysFkjeHRunZXw4QvqjP5QNsgUg,12806
|
|
3
|
+
megfile/errors.py,sha256=y0PJ4lzkm-uqgVD1EqakmN0IHBMt1CCzDbdbq84sVb4,11491
|
|
4
4
|
megfile/fs.py,sha256=sjkm_LsvNCw8hj9Ee-1HSNzvg7bRHTPem8KTDDBQlCw,11621
|
|
5
5
|
megfile/fs_path.py,sha256=Lz-u8XL2RMViHl_39yapAwX5XSE_ZpQsuhAvVve6rk8,38532
|
|
6
6
|
megfile/http.py,sha256=Juz95GscschvmElieLIWqO2A4BrqJnrtTWTa9uzNN_4,2477
|
|
@@ -8,18 +8,18 @@ megfile/http_path.py,sha256=XtpbYHdtHZysjkz6Xdr4fe8E6hlsV4iLNoIZaBk-_P4,5360
|
|
|
8
8
|
megfile/interfaces.py,sha256=h3tWE8hVt5S-HopaMAX6lunPJ97vzhv6jH_2HubcDNc,6219
|
|
9
9
|
megfile/pathlike.py,sha256=52e6POtMfUCC3Hb9XNwBLXM3Gkz3aGP8H8AagGKlDSg,29307
|
|
10
10
|
megfile/s3.py,sha256=6d9bBVkVdXIvmJLfpOgJkK4nVAsLR3lirrB6x1-P2y8,12437
|
|
11
|
-
megfile/s3_path.py,sha256
|
|
11
|
+
megfile/s3_path.py,sha256=-odQ7rg38khf1Bc8webKBZewIfKIV5I1a-qsdshLLzA,85287
|
|
12
12
|
megfile/sftp.py,sha256=qzfgI-MGzNsr7adUQaiVKS7zRHGuK-Hs6o45wNIAcww,11943
|
|
13
|
-
megfile/sftp_path.py,sha256=
|
|
14
|
-
megfile/smart.py,sha256=
|
|
13
|
+
megfile/sftp_path.py,sha256=lbestqN_FXwIJbqFeHXLkq306B2zqVILFda4peKOQwY,47288
|
|
14
|
+
megfile/smart.py,sha256=xjRRujYAPzXlrYxV7ZCAPXl0biUNn4730ATKc_tzIzE,32694
|
|
15
15
|
megfile/smart_path.py,sha256=Rwb1McXsshi9-F6miTRqE6j8FO2j1edjmSxZF32YZ6E,6708
|
|
16
16
|
megfile/stdio.py,sha256=qmi1c7VTll6Y6ya9MCd-dSKffocX2GafA-YUncZRU1Y,559
|
|
17
17
|
megfile/stdio_path.py,sha256=ULMzGOj7SBMn3c7fYVSDepT_1nEUiVetc-xRt2pR5o4,2682
|
|
18
|
-
megfile/version.py,sha256=
|
|
18
|
+
megfile/version.py,sha256=wEKo6RBa-JfaOX7tcmNU28H0l1ZzziU5CEqvjD8Y1AU,25
|
|
19
19
|
megfile/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
20
20
|
megfile/lib/combine_reader.py,sha256=XFSqEY5A5X5Uf7eQ6AXAzrvNteESSXvKNVPktGjo3KY,4546
|
|
21
21
|
megfile/lib/compare.py,sha256=yG2fZve_gMg32rQVCdwixBdqgYRsjn-24TqhALQaOrA,2233
|
|
22
|
-
megfile/lib/compat.py,sha256=
|
|
22
|
+
megfile/lib/compat.py,sha256=rYjfzQ3svuY7pB37W1JGyWH1kxd9aT4RtIe90npPtXI,3033
|
|
23
23
|
megfile/lib/fnmatch.py,sha256=HgdlnEWBsdFUOZqnW_v1kj1jeH_9lMcCqW85pyMu4vM,4054
|
|
24
24
|
megfile/lib/glob.py,sha256=7i9dIput9rI9JIPyTZX-JDmFS7IP_THlX1k-35foAfw,9732
|
|
25
25
|
megfile/lib/joinpath.py,sha256=D4Px6-lnDDpYs1LMUHkTIGqMPJQ0oCBGfTzREs373iU,929
|
|
@@ -29,17 +29,17 @@ megfile/lib/s3_cached_handler.py,sha256=e4KamTLMIPKa96yp7HGF05wDl2Yfoizz9pBrz-uA
|
|
|
29
29
|
megfile/lib/s3_limited_seekable_writer.py,sha256=NSBh5cZCCtwZM0DTwfPkpOA_h0NXd9-6qjasOby5zxc,6037
|
|
30
30
|
megfile/lib/s3_memory_handler.py,sha256=jLBA5HVuI-UBPYMYN-B8iX_VGr8bC9brAqY-KbEGrtw,3725
|
|
31
31
|
megfile/lib/s3_pipe_handler.py,sha256=38x8oPrxlXVWqMgDOqScPj0Pt2w2cQFSNoTK27EtBSw,3384
|
|
32
|
-
megfile/lib/s3_prefetch_reader.py,sha256=
|
|
32
|
+
megfile/lib/s3_prefetch_reader.py,sha256=uffgYsNy7sfiZdoZdY67LYiQiPagtohl4BeQ6jSTwEM,15113
|
|
33
33
|
megfile/lib/s3_share_cache_reader.py,sha256=QyZvzVpTswgiXv-E8QhV_jFdQjCBxVcgbgybXo6d1cM,3771
|
|
34
34
|
megfile/lib/shadow_handler.py,sha256=IbFyTw107t-yWH0cGrDjAJX-CS3xeEr77_PTGsnSgk4,2683
|
|
35
35
|
megfile/lib/stdio_handler.py,sha256=QDWtcZxz-hzi-rqQUiSlR3NrihX1fjK_Rj9T2mdTFEg,2044
|
|
36
36
|
megfile/lib/url.py,sha256=VbQLjo0s4AaV0iSk66BcjI68aUTcN9zBZ5x6-cM4Qvs,103
|
|
37
37
|
megfile/utils/__init__.py,sha256=DEVSwUQ1-1rkHBIuXBxgEYiepq8W25JIWYQrC1gcFrc,9005
|
|
38
38
|
megfile/utils/mutex.py,sha256=-2KH3bNovKRd9zvsXq9n3bWM7rQdoG9hO7tUPxVG_Po,2538
|
|
39
|
-
megfile-2.
|
|
40
|
-
megfile-2.
|
|
41
|
-
megfile-2.
|
|
42
|
-
megfile-2.
|
|
43
|
-
megfile-2.
|
|
44
|
-
megfile-2.
|
|
45
|
-
megfile-2.
|
|
39
|
+
megfile-2.2.0.post1.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
|
|
40
|
+
megfile-2.2.0.post1.dist-info/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
|
|
41
|
+
megfile-2.2.0.post1.dist-info/METADATA,sha256=WUowRbsO-dMPhEJYA1YQY0DNYRGc2jw6kdYRYQLnoBQ,10601
|
|
42
|
+
megfile-2.2.0.post1.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
|
|
43
|
+
megfile-2.2.0.post1.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
|
|
44
|
+
megfile-2.2.0.post1.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
|
|
45
|
+
megfile-2.2.0.post1.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|