megfile 3.0.6.post1__py3-none-any.whl → 3.1.0.post1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- docs/conf.py +67 -0
- megfile/cli.py +16 -16
- megfile/config.py +37 -6
- megfile/errors.py +26 -20
- megfile/fs.py +13 -8
- megfile/fs_path.py +69 -49
- megfile/hdfs.py +13 -8
- megfile/hdfs_path.py +49 -41
- megfile/http.py +1 -1
- megfile/http_path.py +35 -28
- megfile/interfaces.py +119 -48
- megfile/lib/base_prefetch_reader.py +9 -8
- megfile/lib/combine_reader.py +7 -7
- megfile/lib/fnmatch.py +2 -2
- megfile/lib/glob.py +3 -3
- megfile/lib/hdfs_prefetch_reader.py +2 -1
- megfile/lib/http_prefetch_reader.py +3 -2
- megfile/lib/lazy_handler.py +6 -5
- megfile/lib/s3_buffered_writer.py +8 -7
- megfile/lib/s3_cached_handler.py +3 -4
- megfile/lib/s3_limited_seekable_writer.py +5 -3
- megfile/lib/s3_memory_handler.py +10 -6
- megfile/lib/s3_pipe_handler.py +1 -1
- megfile/lib/s3_prefetch_reader.py +7 -5
- megfile/lib/s3_share_cache_reader.py +2 -2
- megfile/lib/shadow_handler.py +5 -5
- megfile/lib/stdio_handler.py +3 -3
- megfile/pathlike.py +156 -170
- megfile/s3.py +19 -13
- megfile/s3_path.py +98 -83
- megfile/sftp.py +25 -16
- megfile/sftp_path.py +109 -94
- megfile/smart.py +38 -28
- megfile/smart_path.py +6 -6
- megfile/stdio.py +3 -3
- megfile/stdio_path.py +5 -5
- megfile/utils/__init__.py +8 -27
- megfile/version.py +1 -1
- {megfile-3.0.6.post1.dist-info → megfile-3.1.0.post1.dist-info}/METADATA +4 -5
- megfile-3.1.0.post1.dist-info/RECORD +55 -0
- {megfile-3.0.6.post1.dist-info → megfile-3.1.0.post1.dist-info}/WHEEL +1 -1
- megfile-3.1.0.post1.dist-info/top_level.txt +7 -0
- scripts/convert_results_to_sarif.py +124 -0
- scripts/generate_file.py +268 -0
- megfile-3.0.6.post1.dist-info/RECORD +0 -52
- megfile-3.0.6.post1.dist-info/top_level.txt +0 -1
- {megfile-3.0.6.post1.dist-info → megfile-3.1.0.post1.dist-info}/LICENSE +0 -0
- {megfile-3.0.6.post1.dist-info → megfile-3.1.0.post1.dist-info}/LICENSE.pyre +0 -0
- {megfile-3.0.6.post1.dist-info → megfile-3.1.0.post1.dist-info}/entry_points.txt +0 -0
megfile/sftp.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from typing import IO,
|
|
1
|
+
from typing import IO, BinaryIO, Callable, Iterator, List, Optional, Tuple
|
|
2
2
|
|
|
3
3
|
from megfile.interfaces import FileEntry, PathLike, StatResult
|
|
4
4
|
from megfile.sftp_path import SftpPath, is_sftp, sftp_concat, sftp_download, sftp_glob, sftp_glob_stat, sftp_iglob, sftp_lstat, sftp_path_join, sftp_readlink, sftp_resolve, sftp_upload
|
|
@@ -116,10 +116,10 @@ def sftp_isfile(path: PathLike, followlinks: bool = False) -> bool:
|
|
|
116
116
|
|
|
117
117
|
def sftp_listdir(path: PathLike) -> List[str]:
|
|
118
118
|
'''
|
|
119
|
-
Get all contents of given sftp path. The result is in
|
|
119
|
+
Get all contents of given sftp path. The result is in ascending alphabetical order.
|
|
120
120
|
|
|
121
121
|
:param path: Given path
|
|
122
|
-
:returns: All contents have in the path in
|
|
122
|
+
:returns: All contents have in the path in ascending alphabetical order
|
|
123
123
|
'''
|
|
124
124
|
return SftpPath(path).listdir()
|
|
125
125
|
|
|
@@ -136,18 +136,20 @@ def sftp_load_from(path: PathLike) -> BinaryIO:
|
|
|
136
136
|
|
|
137
137
|
|
|
138
138
|
def sftp_makedirs(
|
|
139
|
-
path: PathLike,
|
|
139
|
+
path: PathLike,
|
|
140
|
+
mode=0o777,
|
|
141
|
+
parents: bool = False,
|
|
140
142
|
exist_ok: bool = False):
|
|
141
143
|
'''
|
|
142
|
-
make a directory on sftp, including parent directory
|
|
143
|
-
|
|
144
|
+
make a directory on sftp, including parent directory.
|
|
144
145
|
If there exists a file on the path, raise FileExistsError
|
|
145
146
|
|
|
146
147
|
:param path: Given path
|
|
147
148
|
:param mode: If mode is given, it is combined with the process’ umask value to determine the file mode and access flags.
|
|
148
149
|
:param parents: If parents is true, any missing parents of this path are created as needed;
|
|
149
|
-
|
|
150
|
+
If parents is false (the default), a missing parent raises FileNotFoundError.
|
|
150
151
|
:param exist_ok: If False and target directory exists, raise FileExistsError
|
|
152
|
+
|
|
151
153
|
:raises: FileExistsError
|
|
152
154
|
'''
|
|
153
155
|
return SftpPath(path).mkdir(mode, parents, exist_ok)
|
|
@@ -163,7 +165,8 @@ def sftp_realpath(path: PathLike) -> str:
|
|
|
163
165
|
|
|
164
166
|
|
|
165
167
|
def sftp_rename(
|
|
166
|
-
src_path: PathLike,
|
|
168
|
+
src_path: PathLike,
|
|
169
|
+
dst_path: PathLike,
|
|
167
170
|
overwrite: bool = True) -> 'SftpPath':
|
|
168
171
|
'''
|
|
169
172
|
rename file on sftp
|
|
@@ -176,7 +179,8 @@ def sftp_rename(
|
|
|
176
179
|
|
|
177
180
|
|
|
178
181
|
def sftp_move(
|
|
179
|
-
src_path: PathLike,
|
|
182
|
+
src_path: PathLike,
|
|
183
|
+
dst_path: PathLike,
|
|
180
184
|
overwrite: bool = True) -> 'SftpPath':
|
|
181
185
|
'''
|
|
182
186
|
move file on sftp
|
|
@@ -199,7 +203,8 @@ def sftp_remove(path: PathLike, missing_ok: bool = False) -> None:
|
|
|
199
203
|
|
|
200
204
|
|
|
201
205
|
def sftp_scan(
|
|
202
|
-
path: PathLike,
|
|
206
|
+
path: PathLike,
|
|
207
|
+
missing_ok: bool = True,
|
|
203
208
|
followlinks: bool = False) -> Iterator[str]:
|
|
204
209
|
'''
|
|
205
210
|
Iteratively traverse only files in given directory, in alphabetical order.
|
|
@@ -217,7 +222,8 @@ def sftp_scan(
|
|
|
217
222
|
|
|
218
223
|
|
|
219
224
|
def sftp_scan_stat(
|
|
220
|
-
path: PathLike,
|
|
225
|
+
path: PathLike,
|
|
226
|
+
missing_ok: bool = True,
|
|
221
227
|
followlinks: bool = False) -> Iterator[FileEntry]:
|
|
222
228
|
'''
|
|
223
229
|
Iteratively traverse only files in given directory, in alphabetical order.
|
|
@@ -260,8 +266,10 @@ def sftp_unlink(path: PathLike, missing_ok: bool = False) -> None:
|
|
|
260
266
|
return SftpPath(path).unlink(missing_ok)
|
|
261
267
|
|
|
262
268
|
|
|
263
|
-
def sftp_walk(
|
|
264
|
-
|
|
269
|
+
def sftp_walk(
|
|
270
|
+
path: PathLike,
|
|
271
|
+
followlinks: bool = False
|
|
272
|
+
) -> Iterator[Tuple[str, List[str], List[str]]]:
|
|
265
273
|
'''
|
|
266
274
|
Generate the file names in a directory tree by walking the tree top-down.
|
|
267
275
|
For each directory in the tree rooted at directory path (including path itself),
|
|
@@ -292,6 +300,7 @@ def sftp_getmd5(
|
|
|
292
300
|
:param path: Given path
|
|
293
301
|
:param recalculate: Ignore this parameter, just for compatibility
|
|
294
302
|
:param followlinks: Ignore this parameter, just for compatibility
|
|
303
|
+
|
|
295
304
|
returns: md5 of file
|
|
296
305
|
'''
|
|
297
306
|
return SftpPath(path).md5(recalculate, followlinks)
|
|
@@ -302,7 +311,7 @@ def sftp_symlink(src_path: PathLike, dst_path: PathLike) -> None:
|
|
|
302
311
|
Create a symbolic link pointing to src_path named dst_path.
|
|
303
312
|
|
|
304
313
|
:param src_path: Given path
|
|
305
|
-
:param dst_path:
|
|
314
|
+
:param dst_path: Destination path
|
|
306
315
|
'''
|
|
307
316
|
return SftpPath(src_path).symlink(dst_path)
|
|
308
317
|
|
|
@@ -333,7 +342,7 @@ def sftp_open(
|
|
|
333
342
|
buffering=-1,
|
|
334
343
|
encoding: Optional[str] = None,
|
|
335
344
|
errors: Optional[str] = None,
|
|
336
|
-
**kwargs) -> IO
|
|
345
|
+
**kwargs) -> IO:
|
|
337
346
|
'''Open a file on the path.
|
|
338
347
|
|
|
339
348
|
:param path: Given path
|
|
@@ -402,7 +411,7 @@ def sftp_sync(
|
|
|
402
411
|
:param src_path: Given path
|
|
403
412
|
:param dst_url: Given destination path
|
|
404
413
|
:param followlinks: False if regard symlink as file, else True
|
|
405
|
-
:param force: Sync file
|
|
414
|
+
:param force: Sync file forcible, do not ignore same files, priority is higher than 'overwrite', default is False
|
|
406
415
|
:param overwrite: whether or not overwrite file when exists, default is True
|
|
407
416
|
'''
|
|
408
417
|
return SftpPath(src_path).sync(dst_path, followlinks, force, overwrite)
|
megfile/sftp_path.py
CHANGED
|
@@ -7,9 +7,10 @@ import random
|
|
|
7
7
|
import shlex
|
|
8
8
|
import socket
|
|
9
9
|
import subprocess
|
|
10
|
+
from functools import cached_property
|
|
10
11
|
from logging import getLogger as get_logger
|
|
11
12
|
from stat import S_ISDIR, S_ISLNK, S_ISREG
|
|
12
|
-
from typing import IO,
|
|
13
|
+
from typing import IO, BinaryIO, Callable, Iterator, List, Optional, Tuple, Union
|
|
13
14
|
from urllib.parse import urlsplit, urlunsplit
|
|
14
15
|
|
|
15
16
|
import paramiko
|
|
@@ -23,7 +24,7 @@ from megfile.lib.glob import FSFunc, iglob
|
|
|
23
24
|
from megfile.lib.joinpath import uri_join
|
|
24
25
|
from megfile.pathlike import PathLike, URIPath
|
|
25
26
|
from megfile.smart_path import SmartPath
|
|
26
|
-
from megfile.utils import
|
|
27
|
+
from megfile.utils import calculate_md5, thread_local
|
|
27
28
|
|
|
28
29
|
_logger = get_logger(__name__)
|
|
29
30
|
|
|
@@ -55,10 +56,10 @@ DEFAULT_SSH_KEEPALIVE_INTERVAL = 15
|
|
|
55
56
|
|
|
56
57
|
def _make_stat(stat: paramiko.SFTPAttributes) -> StatResult:
|
|
57
58
|
return StatResult(
|
|
58
|
-
size=stat.st_size,
|
|
59
|
-
mtime=stat.st_mtime,
|
|
60
|
-
isdir=S_ISDIR(stat.st_mode),
|
|
61
|
-
islnk=S_ISLNK(stat.st_mode),
|
|
59
|
+
size=stat.st_size or 0,
|
|
60
|
+
mtime=stat.st_mtime or 0.0,
|
|
61
|
+
isdir=S_ISDIR(stat.st_mode) if stat.st_mode is not None else False,
|
|
62
|
+
islnk=S_ISLNK(stat.st_mode) if stat.st_mode is not None else False,
|
|
62
63
|
extra=stat,
|
|
63
64
|
)
|
|
64
65
|
|
|
@@ -82,10 +83,10 @@ def get_private_key():
|
|
|
82
83
|
|
|
83
84
|
|
|
84
85
|
def provide_connect_info(
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
86
|
+
hostname: str,
|
|
87
|
+
port: Optional[int] = None,
|
|
88
|
+
username: Optional[str] = None,
|
|
89
|
+
password: Optional[str] = None,
|
|
89
90
|
):
|
|
90
91
|
if not port:
|
|
91
92
|
port = 22
|
|
@@ -117,11 +118,11 @@ def sftp_should_retry(error: Exception) -> bool:
|
|
|
117
118
|
|
|
118
119
|
|
|
119
120
|
def _patch_sftp_client_request(
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
121
|
+
client: paramiko.SFTPClient,
|
|
122
|
+
hostname: str,
|
|
123
|
+
port: Optional[int] = None,
|
|
124
|
+
username: Optional[str] = None,
|
|
125
|
+
password: Optional[str] = None,
|
|
125
126
|
):
|
|
126
127
|
|
|
127
128
|
def retry_callback(error, *args, **kwargs):
|
|
@@ -144,7 +145,7 @@ def _patch_sftp_client_request(
|
|
|
144
145
|
)
|
|
145
146
|
client.sock = new_sftp_client.sock
|
|
146
147
|
|
|
147
|
-
client._request = patch_method(
|
|
148
|
+
client._request = patch_method( # pyre-ignore[16]
|
|
148
149
|
client._request, # pytype: disable=attribute-error
|
|
149
150
|
max_retries=MAX_RETRIES,
|
|
150
151
|
should_retry=sftp_should_retry,
|
|
@@ -153,10 +154,10 @@ def _patch_sftp_client_request(
|
|
|
153
154
|
|
|
154
155
|
|
|
155
156
|
def _get_sftp_client(
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
157
|
+
hostname: str,
|
|
158
|
+
port: Optional[int] = None,
|
|
159
|
+
username: Optional[str] = None,
|
|
160
|
+
password: Optional[str] = None,
|
|
160
161
|
) -> paramiko.SFTPClient:
|
|
161
162
|
'''Get sftp client
|
|
162
163
|
|
|
@@ -175,10 +176,10 @@ def _get_sftp_client(
|
|
|
175
176
|
|
|
176
177
|
|
|
177
178
|
def get_sftp_client(
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
179
|
+
hostname: str,
|
|
180
|
+
port: Optional[int] = None,
|
|
181
|
+
username: Optional[str] = None,
|
|
182
|
+
password: Optional[str] = None,
|
|
182
183
|
) -> paramiko.SFTPClient:
|
|
183
184
|
'''Get sftp client
|
|
184
185
|
|
|
@@ -190,10 +191,10 @@ def get_sftp_client(
|
|
|
190
191
|
|
|
191
192
|
|
|
192
193
|
def _get_ssh_client(
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
194
|
+
hostname: str,
|
|
195
|
+
port: Optional[int] = None,
|
|
196
|
+
username: Optional[str] = None,
|
|
197
|
+
password: Optional[str] = None,
|
|
197
198
|
) -> paramiko.SSHClient:
|
|
198
199
|
hostname, port, username, password, private_key = provide_connect_info(
|
|
199
200
|
hostname=hostname,
|
|
@@ -236,10 +237,10 @@ def _get_ssh_client(
|
|
|
236
237
|
|
|
237
238
|
|
|
238
239
|
def get_ssh_client(
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
240
|
+
hostname: str,
|
|
241
|
+
port: Optional[int] = None,
|
|
242
|
+
username: Optional[str] = None,
|
|
243
|
+
password: Optional[str] = None,
|
|
243
244
|
) -> paramiko.SSHClient:
|
|
244
245
|
return thread_local(
|
|
245
246
|
f'ssh_client:{hostname},{port},{username},{password}', _get_ssh_client,
|
|
@@ -247,10 +248,10 @@ def get_ssh_client(
|
|
|
247
248
|
|
|
248
249
|
|
|
249
250
|
def get_ssh_session(
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
251
|
+
hostname: str,
|
|
252
|
+
port: Optional[int] = None,
|
|
253
|
+
username: Optional[str] = None,
|
|
254
|
+
password: Optional[str] = None,
|
|
254
255
|
) -> paramiko.Channel:
|
|
255
256
|
|
|
256
257
|
def retry_callback(error, *args, **kwargs):
|
|
@@ -265,7 +266,7 @@ def get_ssh_session(
|
|
|
265
266
|
del thread_local[sftp_key]
|
|
266
267
|
|
|
267
268
|
return patch_method(
|
|
268
|
-
_open_session,
|
|
269
|
+
_open_session,
|
|
269
270
|
max_retries=MAX_RETRIES,
|
|
270
271
|
should_retry=sftp_should_retry,
|
|
271
272
|
retry_callback=retry_callback)(
|
|
@@ -277,10 +278,10 @@ def get_ssh_session(
|
|
|
277
278
|
|
|
278
279
|
|
|
279
280
|
def _open_session(
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
281
|
+
hostname: str,
|
|
282
|
+
port: Optional[int] = None,
|
|
283
|
+
username: Optional[str] = None,
|
|
284
|
+
password: Optional[str] = None,
|
|
284
285
|
) -> paramiko.Channel:
|
|
285
286
|
ssh_client = get_ssh_client(hostname, port, username, password)
|
|
286
287
|
transport = ssh_client.get_transport()
|
|
@@ -314,7 +315,8 @@ def sftp_readlink(path: PathLike) -> 'str':
|
|
|
314
315
|
return SftpPath(path).readlink().path_with_protocol
|
|
315
316
|
|
|
316
317
|
|
|
317
|
-
def sftp_glob(path: PathLike,
|
|
318
|
+
def sftp_glob(path: PathLike,
|
|
319
|
+
recursive: bool = True,
|
|
318
320
|
missing_ok: bool = True) -> List[str]:
|
|
319
321
|
'''Return path list in ascending alphabetical order, in which path matches glob pattern
|
|
320
322
|
|
|
@@ -324,7 +326,7 @@ def sftp_glob(path: PathLike, recursive: bool = True,
|
|
|
324
326
|
Assume there exists a path `/a/b/c/b/d.txt`
|
|
325
327
|
use path pattern like `/**/b/**/*.txt` to glob, the path above will be returned twice
|
|
326
328
|
3. `**` will match any matched file, directory, symlink and '' by default, when recursive is `True`
|
|
327
|
-
4. fs_glob returns same as glob.glob(pathname, recursive=True) in
|
|
329
|
+
4. fs_glob returns same as glob.glob(pathname, recursive=True) in ascending alphabetical order.
|
|
328
330
|
5. Hidden files (filename stars with '.') will not be found in the result
|
|
329
331
|
|
|
330
332
|
:param path: Given path
|
|
@@ -338,7 +340,8 @@ def sftp_glob(path: PathLike, recursive: bool = True,
|
|
|
338
340
|
|
|
339
341
|
|
|
340
342
|
def sftp_glob_stat(
|
|
341
|
-
path: PathLike,
|
|
343
|
+
path: PathLike,
|
|
344
|
+
recursive: bool = True,
|
|
342
345
|
missing_ok: bool = True) -> Iterator[FileEntry]:
|
|
343
346
|
'''Return a list contains tuples of path and file stat, in ascending alphabetical order, in which path matches glob pattern
|
|
344
347
|
|
|
@@ -348,7 +351,7 @@ def sftp_glob_stat(
|
|
|
348
351
|
Assume there exists a path `/a/b/c/b/d.txt`
|
|
349
352
|
use path pattern like `/**/b/**/*.txt` to glob, the path above will be returned twice
|
|
350
353
|
3. `**` will match any matched file, directory, symlink and '' by default, when recursive is `True`
|
|
351
|
-
4. fs_glob returns same as glob.glob(pathname, recursive=True) in
|
|
354
|
+
4. fs_glob returns same as glob.glob(pathname, recursive=True) in ascending alphabetical order.
|
|
352
355
|
5. Hidden files (filename stars with '.') will not be found in the result
|
|
353
356
|
|
|
354
357
|
:param path: Given path
|
|
@@ -365,7 +368,8 @@ def sftp_glob_stat(
|
|
|
365
368
|
path_object.lstat())
|
|
366
369
|
|
|
367
370
|
|
|
368
|
-
def sftp_iglob(path: PathLike,
|
|
371
|
+
def sftp_iglob(path: PathLike,
|
|
372
|
+
recursive: bool = True,
|
|
369
373
|
missing_ok: bool = True) -> Iterator[str]:
|
|
370
374
|
'''Return path iterator in ascending alphabetical order, in which path matches glob pattern
|
|
371
375
|
|
|
@@ -375,7 +379,7 @@ def sftp_iglob(path: PathLike, recursive: bool = True,
|
|
|
375
379
|
Assume there exists a path `/a/b/c/b/d.txt`
|
|
376
380
|
use path pattern like `/**/b/**/*.txt` to glob, the path above will be returned twice
|
|
377
381
|
3. `**` will match any matched file, directory, symlink and '' by default, when recursive is `True`
|
|
378
|
-
4. fs_glob returns same as glob.glob(pathname, recursive=True) in
|
|
382
|
+
4. fs_glob returns same as glob.glob(pathname, recursive=True) in ascending alphabetical order.
|
|
379
383
|
5. Hidden files (filename stars with '.') will not be found in the result
|
|
380
384
|
|
|
381
385
|
:param path: Given path
|
|
@@ -459,7 +463,8 @@ def sftp_download(
|
|
|
459
463
|
|
|
460
464
|
def sftp_callback(bytes_transferred: int, _total_bytes: int):
|
|
461
465
|
nonlocal bytes_transferred_before
|
|
462
|
-
callback(
|
|
466
|
+
callback( # pyre-ignore[29]
|
|
467
|
+
bytes_transferred - bytes_transferred_before)
|
|
463
468
|
bytes_transferred_before = bytes_transferred
|
|
464
469
|
|
|
465
470
|
src_path._client.get(
|
|
@@ -516,7 +521,8 @@ def sftp_upload(
|
|
|
516
521
|
|
|
517
522
|
def sftp_callback(bytes_transferred: int, _total_bytes: int):
|
|
518
523
|
nonlocal bytes_transferred_before
|
|
519
|
-
callback(
|
|
524
|
+
callback( # pyre-ignore[29]
|
|
525
|
+
bytes_transferred - bytes_transferred_before)
|
|
520
526
|
bytes_transferred_before = bytes_transferred
|
|
521
527
|
|
|
522
528
|
dst_path._client.put(
|
|
@@ -581,10 +587,10 @@ class SftpPath(URIPath):
|
|
|
581
587
|
"""sftp protocol
|
|
582
588
|
|
|
583
589
|
uri format:
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
590
|
+
- absolute path
|
|
591
|
+
- sftp://[username[:password]@]hostname[:port]//file_path
|
|
592
|
+
- relative path
|
|
593
|
+
- sftp://[username[:password]@]hostname[:port]/file_path
|
|
588
594
|
"""
|
|
589
595
|
|
|
590
596
|
protocol = "sftp"
|
|
@@ -600,8 +606,8 @@ class SftpPath(URIPath):
|
|
|
600
606
|
self._root_dir = self._client.normalize('.')
|
|
601
607
|
self._real_path = os.path.join(self._root_dir, parts.path.lstrip('/'))
|
|
602
608
|
|
|
603
|
-
@
|
|
604
|
-
def parts(self) -> Tuple[str]:
|
|
609
|
+
@cached_property
|
|
610
|
+
def parts(self) -> Tuple[str, ...]:
|
|
605
611
|
'''A tuple giving access to the path’s various components'''
|
|
606
612
|
if self._urlsplit_parts.path.startswith('//'):
|
|
607
613
|
new_parts = self._urlsplit_parts._replace(path='//')
|
|
@@ -611,7 +617,7 @@ class SftpPath(URIPath):
|
|
|
611
617
|
path = self._urlsplit_parts.path.lstrip('/')
|
|
612
618
|
if path != '':
|
|
613
619
|
parts.extend(path.split('/'))
|
|
614
|
-
return tuple(parts)
|
|
620
|
+
return tuple(parts) # pyre-ignore[7]
|
|
615
621
|
|
|
616
622
|
@property
|
|
617
623
|
def _client(self):
|
|
@@ -631,7 +637,7 @@ class SftpPath(URIPath):
|
|
|
631
637
|
if sftp_local_path == ".":
|
|
632
638
|
sftp_local_path = "/"
|
|
633
639
|
new_parts = self._urlsplit_parts._replace(path=sftp_local_path)
|
|
634
|
-
return self.from_path(urlunsplit(new_parts))
|
|
640
|
+
return self.from_path(urlunsplit(new_parts)) # pyre-ignore[6]
|
|
635
641
|
|
|
636
642
|
def exists(self, followlinks: bool = False) -> bool:
|
|
637
643
|
'''
|
|
@@ -670,7 +676,9 @@ class SftpPath(URIPath):
|
|
|
670
676
|
'''
|
|
671
677
|
return self.stat(follow_symlinks=follow_symlinks).size
|
|
672
678
|
|
|
673
|
-
def glob(self,
|
|
679
|
+
def glob(self,
|
|
680
|
+
pattern,
|
|
681
|
+
recursive: bool = True,
|
|
674
682
|
missing_ok: bool = True) -> List['SftpPath']:
|
|
675
683
|
'''Return path list in ascending alphabetical order, in which path matches glob pattern
|
|
676
684
|
|
|
@@ -680,7 +688,7 @@ class SftpPath(URIPath):
|
|
|
680
688
|
Assume there exists a path `/a/b/c/b/d.txt`
|
|
681
689
|
use path pattern like `/**/b/**/*.txt` to glob, the path above will be returned twice
|
|
682
690
|
3. `**` will match any matched file, directory, symlink and '' by default, when recursive is `True`
|
|
683
|
-
4. fs_glob returns same as glob.glob(pathname, recursive=True) in
|
|
691
|
+
4. fs_glob returns same as glob.glob(pathname, recursive=True) in ascending alphabetical order.
|
|
684
692
|
5. Hidden files (filename stars with '.') will not be found in the result
|
|
685
693
|
|
|
686
694
|
:param pattern: Glob the given relative pattern in the directory represented by this path
|
|
@@ -693,7 +701,9 @@ class SftpPath(URIPath):
|
|
|
693
701
|
pattern=pattern, recursive=recursive, missing_ok=missing_ok))
|
|
694
702
|
|
|
695
703
|
def glob_stat(
|
|
696
|
-
self,
|
|
704
|
+
self,
|
|
705
|
+
pattern,
|
|
706
|
+
recursive: bool = True,
|
|
697
707
|
missing_ok: bool = True) -> Iterator[FileEntry]:
|
|
698
708
|
'''Return a list contains tuples of path and file stat, in ascending alphabetical order, in which path matches glob pattern
|
|
699
709
|
|
|
@@ -703,7 +713,7 @@ class SftpPath(URIPath):
|
|
|
703
713
|
Assume there exists a path `/a/b/c/b/d.txt`
|
|
704
714
|
use path pattern like `/**/b/**/*.txt` to glob, the path above will be returned twice
|
|
705
715
|
3. `**` will match any matched file, directory, symlink and '' by default, when recursive is `True`
|
|
706
|
-
4. fs_glob returns same as glob.glob(pathname, recursive=True) in
|
|
716
|
+
4. fs_glob returns same as glob.glob(pathname, recursive=True) in ascending alphabetical order.
|
|
707
717
|
5. Hidden files (filename stars with '.') will not be found in the result
|
|
708
718
|
|
|
709
719
|
:param pattern: Glob the given relative pattern in the directory represented by this path
|
|
@@ -715,7 +725,9 @@ class SftpPath(URIPath):
|
|
|
715
725
|
missing_ok=missing_ok):
|
|
716
726
|
yield FileEntry(path_obj.name, path_obj.path, path_obj.lstat())
|
|
717
727
|
|
|
718
|
-
def iglob(self,
|
|
728
|
+
def iglob(self,
|
|
729
|
+
pattern,
|
|
730
|
+
recursive: bool = True,
|
|
719
731
|
missing_ok: bool = True) -> Iterator['SftpPath']:
|
|
720
732
|
'''Return path iterator in ascending alphabetical order, in which path matches glob pattern
|
|
721
733
|
|
|
@@ -725,7 +737,7 @@ class SftpPath(URIPath):
|
|
|
725
737
|
Assume there exists a path `/a/b/c/b/d.txt`
|
|
726
738
|
use path pattern like `/**/b/**/*.txt` to glob, the path above will be returned twice
|
|
727
739
|
3. `**` will match any matched file, directory, symlink and '' by default, when recursive is `True`
|
|
728
|
-
4. fs_glob returns same as glob.glob(pathname, recursive=True) in
|
|
740
|
+
4. fs_glob returns same as glob.glob(pathname, recursive=True) in ascending alphabetical order.
|
|
729
741
|
5. Hidden files (filename stars with '.') will not be found in the result
|
|
730
742
|
|
|
731
743
|
:param pattern: Glob the given relative pattern in the directory represented by this path
|
|
@@ -796,9 +808,9 @@ class SftpPath(URIPath):
|
|
|
796
808
|
|
|
797
809
|
def listdir(self) -> List[str]:
|
|
798
810
|
'''
|
|
799
|
-
Get all contents of given sftp path. The result is in
|
|
811
|
+
Get all contents of given sftp path. The result is in ascending alphabetical order.
|
|
800
812
|
|
|
801
|
-
:returns: All contents have in the path in
|
|
813
|
+
:returns: All contents have in the path in ascending alphabetical order
|
|
802
814
|
'''
|
|
803
815
|
if not self.is_dir():
|
|
804
816
|
raise NotADirectoryError(
|
|
@@ -807,15 +819,15 @@ class SftpPath(URIPath):
|
|
|
807
819
|
|
|
808
820
|
def iterdir(self) -> Iterator['SftpPath']:
|
|
809
821
|
'''
|
|
810
|
-
Get all contents of given sftp path. The result is in
|
|
822
|
+
Get all contents of given sftp path. The result is in ascending alphabetical order.
|
|
811
823
|
|
|
812
|
-
:returns: All contents have in the path in
|
|
824
|
+
:returns: All contents have in the path in ascending alphabetical order
|
|
813
825
|
'''
|
|
814
826
|
if not self.is_dir():
|
|
815
827
|
raise NotADirectoryError(
|
|
816
828
|
f"Not a directory: '{self.path_with_protocol}'")
|
|
817
829
|
for path in self.listdir():
|
|
818
|
-
yield self.joinpath(path)
|
|
830
|
+
yield self.joinpath(path)
|
|
819
831
|
|
|
820
832
|
def load(self) -> BinaryIO:
|
|
821
833
|
'''Read all content on specified path and write into memory
|
|
@@ -830,14 +842,14 @@ class SftpPath(URIPath):
|
|
|
830
842
|
|
|
831
843
|
def mkdir(self, mode=0o777, parents: bool = False, exist_ok: bool = False):
|
|
832
844
|
'''
|
|
833
|
-
make a directory on sftp, including parent directory
|
|
834
|
-
|
|
845
|
+
make a directory on sftp, including parent directory.
|
|
835
846
|
If there exists a file on the path, raise FileExistsError
|
|
836
847
|
|
|
837
848
|
:param mode: If mode is given, it is combined with the process’ umask value to determine the file mode and access flags.
|
|
838
849
|
:param parents: If parents is true, any missing parents of this path are created as needed;
|
|
839
|
-
|
|
850
|
+
If parents is false (the default), a missing parent raises FileNotFoundError.
|
|
840
851
|
:param exist_ok: If False and target directory exists, raise FileExistsError
|
|
852
|
+
|
|
841
853
|
:raises: FileExistsError
|
|
842
854
|
'''
|
|
843
855
|
if self.exists():
|
|
@@ -943,7 +955,8 @@ class SftpPath(URIPath):
|
|
|
943
955
|
else:
|
|
944
956
|
self._client.unlink(self._real_path)
|
|
945
957
|
|
|
946
|
-
def scan(self,
|
|
958
|
+
def scan(self,
|
|
959
|
+
missing_ok: bool = True,
|
|
947
960
|
followlinks: bool = False) -> Iterator[str]:
|
|
948
961
|
'''
|
|
949
962
|
Iteratively traverse only files in given directory, in alphabetical order.
|
|
@@ -962,7 +975,8 @@ class SftpPath(URIPath):
|
|
|
962
975
|
for file_entry in scan_stat_iter:
|
|
963
976
|
yield file_entry.path
|
|
964
977
|
|
|
965
|
-
def scan_stat(self,
|
|
978
|
+
def scan_stat(self,
|
|
979
|
+
missing_ok: bool = True,
|
|
966
980
|
followlinks: bool = False) -> Iterator[FileEntry]:
|
|
967
981
|
'''
|
|
968
982
|
Iteratively traverse only files in given directory, in alphabetical order.
|
|
@@ -993,8 +1007,7 @@ class SftpPath(URIPath):
|
|
|
993
1007
|
)
|
|
994
1008
|
else:
|
|
995
1009
|
yield FileEntry(
|
|
996
|
-
current_path.name,
|
|
997
|
-
current_path.path_with_protocol,
|
|
1010
|
+
current_path.name, current_path.path_with_protocol,
|
|
998
1011
|
current_path.stat(follow_symlinks=followlinks))
|
|
999
1012
|
|
|
1000
1013
|
return _create_missing_ok_generator(
|
|
@@ -1020,9 +1033,8 @@ class SftpPath(URIPath):
|
|
|
1020
1033
|
for name in self.listdir():
|
|
1021
1034
|
current_path = self.joinpath(name)
|
|
1022
1035
|
yield FileEntry(
|
|
1023
|
-
current_path.name,
|
|
1024
|
-
current_path.
|
|
1025
|
-
current_path.lstat()) # type: ignore
|
|
1036
|
+
current_path.name, current_path.path_with_protocol,
|
|
1037
|
+
current_path.lstat())
|
|
1026
1038
|
|
|
1027
1039
|
return ContextIterator(create_generator())
|
|
1028
1040
|
|
|
@@ -1048,8 +1060,10 @@ class SftpPath(URIPath):
|
|
|
1048
1060
|
return
|
|
1049
1061
|
self._client.unlink(self._real_path)
|
|
1050
1062
|
|
|
1051
|
-
def walk(
|
|
1052
|
-
|
|
1063
|
+
def walk(
|
|
1064
|
+
self,
|
|
1065
|
+
followlinks: bool = False
|
|
1066
|
+
) -> Iterator[Tuple[str, List[str], List[str]]]:
|
|
1053
1067
|
'''
|
|
1054
1068
|
Generate the file names in a directory tree by walking the tree top-down.
|
|
1055
1069
|
For each directory in the tree rooted at directory path (including path itself),
|
|
@@ -1111,16 +1125,17 @@ class SftpPath(URIPath):
|
|
|
1111
1125
|
|
|
1112
1126
|
:param recalculate: Ignore this parameter, just for compatibility
|
|
1113
1127
|
:param followlinks: Ignore this parameter, just for compatibility
|
|
1128
|
+
|
|
1114
1129
|
returns: md5 of file
|
|
1115
1130
|
'''
|
|
1116
1131
|
if self.is_dir():
|
|
1117
1132
|
hash_md5 = hashlib.md5() # nosec
|
|
1118
1133
|
for file_name in self.listdir():
|
|
1119
|
-
chunk = self.joinpath(file_name).md5(
|
|
1134
|
+
chunk = self.joinpath(file_name).md5(
|
|
1120
1135
|
recalculate=recalculate, followlinks=followlinks).encode()
|
|
1121
1136
|
hash_md5.update(chunk)
|
|
1122
1137
|
return hash_md5.hexdigest()
|
|
1123
|
-
with self.open('rb') as src:
|
|
1138
|
+
with self.open('rb') as src:
|
|
1124
1139
|
md5 = calculate_md5(src)
|
|
1125
1140
|
return md5
|
|
1126
1141
|
|
|
@@ -1128,7 +1143,7 @@ class SftpPath(URIPath):
|
|
|
1128
1143
|
'''
|
|
1129
1144
|
Create a symbolic link pointing to src_path named dst_path.
|
|
1130
1145
|
|
|
1131
|
-
:param dst_path:
|
|
1146
|
+
:param dst_path: Destination path
|
|
1132
1147
|
'''
|
|
1133
1148
|
dst_path = self.from_path(dst_path)
|
|
1134
1149
|
if dst_path.exists(followlinks=False):
|
|
@@ -1180,7 +1195,7 @@ class SftpPath(URIPath):
|
|
|
1180
1195
|
buffering=-1,
|
|
1181
1196
|
encoding: Optional[str] = None,
|
|
1182
1197
|
errors: Optional[str] = None,
|
|
1183
|
-
**kwargs) -> IO
|
|
1198
|
+
**kwargs) -> IO:
|
|
1184
1199
|
'''Open a file on the path.
|
|
1185
1200
|
|
|
1186
1201
|
:param mode: Mode to open file
|
|
@@ -1200,9 +1215,8 @@ class SftpPath(URIPath):
|
|
|
1200
1215
|
fileobj = self._client.open(self._real_path, mode, bufsize=buffering)
|
|
1201
1216
|
fileobj.name = self.path
|
|
1202
1217
|
if 'r' in mode and 'b' not in mode:
|
|
1203
|
-
return io.TextIOWrapper(
|
|
1204
|
-
|
|
1205
|
-
return fileobj # type: ignore
|
|
1218
|
+
return io.TextIOWrapper(fileobj, encoding=encoding, errors=errors) # pytype: disable=wrong-arg-types
|
|
1219
|
+
return fileobj # pytype: disable=bad-return-type
|
|
1206
1220
|
|
|
1207
1221
|
def chmod(self, mode: int, follow_symlinks: bool = True):
|
|
1208
1222
|
'''
|
|
@@ -1228,11 +1242,11 @@ class SftpPath(URIPath):
|
|
|
1228
1242
|
return self._client.rmdir(self._real_path)
|
|
1229
1243
|
|
|
1230
1244
|
def _exec_command(
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1245
|
+
self,
|
|
1246
|
+
command: List[str],
|
|
1247
|
+
bufsize: int = -1,
|
|
1248
|
+
timeout: Optional[int] = None,
|
|
1249
|
+
environment: Optional[dict] = None,
|
|
1236
1250
|
) -> subprocess.CompletedProcess:
|
|
1237
1251
|
with get_ssh_session(
|
|
1238
1252
|
hostname=self._urlsplit_parts.hostname,
|
|
@@ -1283,7 +1297,8 @@ class SftpPath(URIPath):
|
|
|
1283
1297
|
if not overwrite and self.from_path(dst_path).exists():
|
|
1284
1298
|
return
|
|
1285
1299
|
|
|
1286
|
-
self.from_path(os.path.dirname(
|
|
1300
|
+
self.from_path(os.path.dirname(
|
|
1301
|
+
fspath(dst_path))).makedirs(exist_ok=True)
|
|
1287
1302
|
dst_path = self.from_path(dst_path)
|
|
1288
1303
|
if self._is_same_backend(dst_path):
|
|
1289
1304
|
if self._real_path == dst_path._real_path:
|
|
@@ -1323,7 +1338,7 @@ class SftpPath(URIPath):
|
|
|
1323
1338
|
|
|
1324
1339
|
:param dst_url: Given destination path
|
|
1325
1340
|
:param followlinks: False if regard symlink as file, else True
|
|
1326
|
-
:param force: Sync file
|
|
1341
|
+
:param force: Sync file forcible, do not ignore same files, priority is higher than 'overwrite', default is False
|
|
1327
1342
|
:param overwrite: whether or not overwrite file when exists, default is True
|
|
1328
1343
|
'''
|
|
1329
1344
|
if not self._is_same_protocol(dst_path):
|