megfile 4.2.5__py3-none-any.whl → 5.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- megfile/__init__.py +14 -291
- megfile/cli.py +83 -40
- megfile/config.py +35 -1
- megfile/errors.py +2 -2
- megfile/fs_path.py +32 -3
- megfile/interfaces.py +21 -10
- megfile/lib/base_memory_handler.py +92 -0
- megfile/lib/glob.py +3 -3
- megfile/lib/http_prefetch_reader.py +22 -22
- megfile/lib/s3_memory_handler.py +14 -81
- megfile/lib/webdav_memory_handler.py +83 -0
- megfile/lib/webdav_prefetch_reader.py +115 -0
- megfile/pathlike.py +3 -4
- megfile/s3_path.py +40 -32
- megfile/sftp2_path.py +38 -62
- megfile/sftp_path.py +238 -1
- megfile/smart.py +70 -29
- megfile/smart_path.py +198 -96
- megfile/version.py +1 -1
- megfile/webdav_path.py +161 -166
- {megfile-4.2.5.dist-info → megfile-5.0.1.dist-info}/METADATA +27 -39
- megfile-5.0.1.dist-info/RECORD +51 -0
- megfile/fs.py +0 -627
- megfile/hdfs.py +0 -408
- megfile/http.py +0 -114
- megfile/s3.py +0 -540
- megfile/sftp.py +0 -821
- megfile/sftp2.py +0 -827
- megfile/stdio.py +0 -30
- megfile/webdav.py +0 -552
- megfile-4.2.5.dist-info/RECORD +0 -56
- {megfile-4.2.5.dist-info → megfile-5.0.1.dist-info}/WHEEL +0 -0
- {megfile-4.2.5.dist-info → megfile-5.0.1.dist-info}/entry_points.txt +0 -0
- {megfile-4.2.5.dist-info → megfile-5.0.1.dist-info}/licenses/LICENSE +0 -0
- {megfile-4.2.5.dist-info → megfile-5.0.1.dist-info}/licenses/LICENSE.pyre +0 -0
- {megfile-4.2.5.dist-info → megfile-5.0.1.dist-info}/top_level.txt +0 -0
megfile/sftp.py
DELETED
|
@@ -1,821 +0,0 @@
|
|
|
1
|
-
import base64
|
|
2
|
-
import hashlib
|
|
3
|
-
import os
|
|
4
|
-
from logging import getLogger as get_logger
|
|
5
|
-
from typing import IO, BinaryIO, Callable, Iterator, List, Optional, Tuple
|
|
6
|
-
|
|
7
|
-
import paramiko
|
|
8
|
-
|
|
9
|
-
from megfile.interfaces import FileEntry, PathLike, StatResult
|
|
10
|
-
from megfile.lib.compat import fspath
|
|
11
|
-
from megfile.lib.joinpath import uri_join
|
|
12
|
-
from megfile.sftp_path import (
|
|
13
|
-
SftpPath,
|
|
14
|
-
is_sftp,
|
|
15
|
-
)
|
|
16
|
-
|
|
17
|
-
_logger = get_logger(__name__)
|
|
18
|
-
|
|
19
|
-
__all__ = [
|
|
20
|
-
"is_sftp",
|
|
21
|
-
"sftp_readlink",
|
|
22
|
-
"sftp_glob",
|
|
23
|
-
"sftp_iglob",
|
|
24
|
-
"sftp_glob_stat",
|
|
25
|
-
"sftp_resolve",
|
|
26
|
-
"sftp_download",
|
|
27
|
-
"sftp_upload",
|
|
28
|
-
"sftp_path_join",
|
|
29
|
-
"sftp_concat",
|
|
30
|
-
"sftp_lstat",
|
|
31
|
-
"sftp_exists",
|
|
32
|
-
"sftp_getmtime",
|
|
33
|
-
"sftp_getsize",
|
|
34
|
-
"sftp_isdir",
|
|
35
|
-
"sftp_isfile",
|
|
36
|
-
"sftp_listdir",
|
|
37
|
-
"sftp_load_from",
|
|
38
|
-
"sftp_makedirs",
|
|
39
|
-
"sftp_realpath",
|
|
40
|
-
"sftp_rename",
|
|
41
|
-
"sftp_move",
|
|
42
|
-
"sftp_remove",
|
|
43
|
-
"sftp_scan",
|
|
44
|
-
"sftp_scan_stat",
|
|
45
|
-
"sftp_scandir",
|
|
46
|
-
"sftp_stat",
|
|
47
|
-
"sftp_unlink",
|
|
48
|
-
"sftp_walk",
|
|
49
|
-
"sftp_getmd5",
|
|
50
|
-
"sftp_symlink",
|
|
51
|
-
"sftp_islink",
|
|
52
|
-
"sftp_save_as",
|
|
53
|
-
"sftp_open",
|
|
54
|
-
"sftp_chmod",
|
|
55
|
-
"sftp_absolute",
|
|
56
|
-
"sftp_rmdir",
|
|
57
|
-
"sftp_copy",
|
|
58
|
-
"sftp_sync",
|
|
59
|
-
"sftp_add_host_key",
|
|
60
|
-
]
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
def sftp_readlink(path: PathLike) -> "str":
|
|
64
|
-
"""
|
|
65
|
-
Return a SftpPath instance representing the path to which the symbolic link points.
|
|
66
|
-
|
|
67
|
-
:param path: Given path
|
|
68
|
-
:returns: Return a SftpPath instance representing the path to
|
|
69
|
-
which the symbolic link points.
|
|
70
|
-
"""
|
|
71
|
-
return SftpPath(path).readlink().path_with_protocol
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
def sftp_glob(
|
|
75
|
-
path: PathLike, recursive: bool = True, missing_ok: bool = True
|
|
76
|
-
) -> List[str]:
|
|
77
|
-
"""Return path list in ascending alphabetical order,
|
|
78
|
-
in which path matches glob pattern
|
|
79
|
-
|
|
80
|
-
1. If doesn't match any path, return empty list
|
|
81
|
-
Notice: ``glob.glob`` in standard library returns ['a/'] instead of empty list
|
|
82
|
-
when pathname is like `a/**`, recursive is True and directory 'a' doesn't exist.
|
|
83
|
-
fs_glob behaves like ``glob.glob`` in standard library under such circumstance.
|
|
84
|
-
2. No guarantee that each path in result is different, which means:
|
|
85
|
-
Assume there exists a path `/a/b/c/b/d.txt`
|
|
86
|
-
use path pattern like `/**/b/**/*.txt` to glob,
|
|
87
|
-
the path above will be returned twice
|
|
88
|
-
3. `**` will match any matched file, directory, symlink and '' by default,
|
|
89
|
-
when recursive is `True`
|
|
90
|
-
4. fs_glob returns same as glob.glob(pathname, recursive=True)
|
|
91
|
-
in ascending alphabetical order.
|
|
92
|
-
5. Hidden files (filename stars with '.') will not be found in the result
|
|
93
|
-
|
|
94
|
-
:param path: Given path
|
|
95
|
-
:param pattern: Glob the given relative pattern in the directory represented
|
|
96
|
-
by this path
|
|
97
|
-
:param recursive: If False, `**` will not search directory recursively
|
|
98
|
-
:param missing_ok: If False and target path doesn't match any file,
|
|
99
|
-
raise FileNotFoundError
|
|
100
|
-
:returns: A list contains paths match `pathname`
|
|
101
|
-
"""
|
|
102
|
-
return list(sftp_iglob(path=path, recursive=recursive, missing_ok=missing_ok))
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
def sftp_glob_stat(
|
|
106
|
-
path: PathLike, recursive: bool = True, missing_ok: bool = True
|
|
107
|
-
) -> Iterator[FileEntry]:
|
|
108
|
-
"""Return a list contains tuples of path and file stat, in ascending alphabetical
|
|
109
|
-
order, in which path matches glob pattern
|
|
110
|
-
|
|
111
|
-
1. If doesn't match any path, return empty list
|
|
112
|
-
Notice: ``glob.glob`` in standard library returns ['a/'] instead of empty list
|
|
113
|
-
when pathname is like `a/**`, recursive is True and directory 'a' doesn't exist.
|
|
114
|
-
sftp_glob behaves like ``glob.glob`` in standard library under such circumstance.
|
|
115
|
-
2. No guarantee that each path in result is different, which means:
|
|
116
|
-
Assume there exists a path `/a/b/c/b/d.txt`
|
|
117
|
-
use path pattern like `/**/b/**/*.txt` to glob,
|
|
118
|
-
the path above will be returned twice
|
|
119
|
-
3. `**` will match any matched file, directory, symlink and '' by default,
|
|
120
|
-
when recursive is `True`
|
|
121
|
-
4. fs_glob returns same as glob.glob(pathname, recursive=True) in
|
|
122
|
-
ascending alphabetical order.
|
|
123
|
-
5. Hidden files (filename stars with '.') will not be found in the result
|
|
124
|
-
|
|
125
|
-
:param path: Given path
|
|
126
|
-
:param pattern: Glob the given relative pattern in the directory represented
|
|
127
|
-
by this path
|
|
128
|
-
:param recursive: If False, `**` will not search directory recursively
|
|
129
|
-
:param missing_ok: If False and target path doesn't match any file,
|
|
130
|
-
raise FileNotFoundError
|
|
131
|
-
:returns: A list contains tuples of path and file stat,
|
|
132
|
-
in which paths match `pathname`
|
|
133
|
-
"""
|
|
134
|
-
for path in sftp_iglob(path=path, recursive=recursive, missing_ok=missing_ok):
|
|
135
|
-
path_object = SftpPath(path)
|
|
136
|
-
yield FileEntry(
|
|
137
|
-
path_object.name, path_object.path_with_protocol, path_object.lstat()
|
|
138
|
-
)
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
def sftp_iglob(
|
|
142
|
-
path: PathLike, recursive: bool = True, missing_ok: bool = True
|
|
143
|
-
) -> Iterator[str]:
|
|
144
|
-
"""Return path iterator in ascending alphabetical order,
|
|
145
|
-
in which path matches glob pattern
|
|
146
|
-
|
|
147
|
-
1. If doesn't match any path, return empty list
|
|
148
|
-
Notice: ``glob.glob`` in standard library returns ['a/'] instead of empty list
|
|
149
|
-
when pathname is like `a/**`, recursive is True and directory 'a' doesn't exist.
|
|
150
|
-
fs_glob behaves like ``glob.glob`` in standard library under such circumstance.
|
|
151
|
-
2. No guarantee that each path in result is different, which means:
|
|
152
|
-
Assume there exists a path `/a/b/c/b/d.txt`
|
|
153
|
-
use path pattern like `/**/b/**/*.txt` to glob,
|
|
154
|
-
the path above will be returned twice
|
|
155
|
-
3. `**` will match any matched file, directory, symlink and '' by default,
|
|
156
|
-
when recursive is `True`
|
|
157
|
-
4. fs_glob returns same as glob.glob(pathname, recursive=True) in
|
|
158
|
-
ascending alphabetical order.
|
|
159
|
-
5. Hidden files (filename stars with '.') will not be found in the result
|
|
160
|
-
|
|
161
|
-
:param path: Given path
|
|
162
|
-
:param pattern: Glob the given relative pattern in the directory represented
|
|
163
|
-
by this path
|
|
164
|
-
:param recursive: If False, `**` will not search directory recursively
|
|
165
|
-
:param missing_ok: If False and target path doesn't match any file,
|
|
166
|
-
raise FileNotFoundError
|
|
167
|
-
:returns: An iterator contains paths match `pathname`
|
|
168
|
-
"""
|
|
169
|
-
|
|
170
|
-
for path in SftpPath(path).iglob(
|
|
171
|
-
pattern="", recursive=recursive, missing_ok=missing_ok
|
|
172
|
-
):
|
|
173
|
-
yield path.path_with_protocol
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
def sftp_resolve(path: PathLike, strict=False) -> "str":
|
|
177
|
-
"""Equal to fs_realpath
|
|
178
|
-
|
|
179
|
-
:param path: Given path
|
|
180
|
-
:param strict: Ignore this parameter, just for compatibility
|
|
181
|
-
:return: Return the canonical path of the specified filename,
|
|
182
|
-
eliminating any symbolic links encountered in the path.
|
|
183
|
-
:rtype: SftpPath
|
|
184
|
-
"""
|
|
185
|
-
return SftpPath(path).resolve(strict).path_with_protocol
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
def sftp_download(
|
|
189
|
-
src_url: PathLike,
|
|
190
|
-
dst_url: PathLike,
|
|
191
|
-
callback: Optional[Callable[[int], None]] = None,
|
|
192
|
-
followlinks: bool = False,
|
|
193
|
-
overwrite: bool = True,
|
|
194
|
-
):
|
|
195
|
-
"""
|
|
196
|
-
Downloads a file from sftp to local filesystem.
|
|
197
|
-
|
|
198
|
-
:param src_url: source sftp path
|
|
199
|
-
:param dst_url: target fs path
|
|
200
|
-
:param callback: Called periodically during copy, and the input parameter is
|
|
201
|
-
the data size (in bytes) of copy since the last call
|
|
202
|
-
:param followlinks: False if regard symlink as file, else True
|
|
203
|
-
:param overwrite: whether or not overwrite file when exists, default is True
|
|
204
|
-
"""
|
|
205
|
-
from megfile.fs import is_fs
|
|
206
|
-
from megfile.fs_path import FSPath
|
|
207
|
-
|
|
208
|
-
if not is_fs(dst_url):
|
|
209
|
-
raise OSError(f"dst_url is not fs path: {dst_url}")
|
|
210
|
-
if not is_sftp(src_url) and not isinstance(src_url, SftpPath):
|
|
211
|
-
raise OSError(f"src_url is not sftp path: {src_url}")
|
|
212
|
-
|
|
213
|
-
dst_path = FSPath(dst_url)
|
|
214
|
-
if not overwrite and dst_path.exists():
|
|
215
|
-
return
|
|
216
|
-
|
|
217
|
-
if isinstance(src_url, SftpPath):
|
|
218
|
-
src_path: SftpPath = src_url
|
|
219
|
-
else:
|
|
220
|
-
src_path: SftpPath = SftpPath(src_url)
|
|
221
|
-
|
|
222
|
-
if followlinks and src_path.is_symlink():
|
|
223
|
-
src_path = src_path.readlink()
|
|
224
|
-
if src_path.is_dir():
|
|
225
|
-
raise IsADirectoryError("Is a directory: %r" % src_url)
|
|
226
|
-
if str(dst_url).endswith("/"):
|
|
227
|
-
raise IsADirectoryError("Is a directory: %r" % dst_url)
|
|
228
|
-
|
|
229
|
-
dst_path.parent.makedirs(exist_ok=True)
|
|
230
|
-
|
|
231
|
-
sftp_callback = None
|
|
232
|
-
if callback:
|
|
233
|
-
bytes_transferred_before = 0
|
|
234
|
-
|
|
235
|
-
def sftp_callback(bytes_transferred: int, _total_bytes: int):
|
|
236
|
-
nonlocal bytes_transferred_before
|
|
237
|
-
callback(bytes_transferred - bytes_transferred_before) # pyre-ignore[29]
|
|
238
|
-
bytes_transferred_before = bytes_transferred
|
|
239
|
-
|
|
240
|
-
src_path._client.get(
|
|
241
|
-
src_path._real_path, dst_path.path_without_protocol, callback=sftp_callback
|
|
242
|
-
)
|
|
243
|
-
|
|
244
|
-
src_stat = src_path.stat()
|
|
245
|
-
dst_path.utime(src_stat.st_atime, src_stat.st_mtime)
|
|
246
|
-
dst_path.chmod(src_stat.st_mode)
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
def sftp_upload(
|
|
250
|
-
src_url: PathLike,
|
|
251
|
-
dst_url: PathLike,
|
|
252
|
-
callback: Optional[Callable[[int], None]] = None,
|
|
253
|
-
followlinks: bool = False,
|
|
254
|
-
overwrite: bool = True,
|
|
255
|
-
):
|
|
256
|
-
"""
|
|
257
|
-
Uploads a file from local filesystem to sftp server.
|
|
258
|
-
|
|
259
|
-
:param src_url: source fs path
|
|
260
|
-
:param dst_url: target sftp path
|
|
261
|
-
:param callback: Called periodically during copy, and the input parameter is
|
|
262
|
-
the data size (in bytes) of copy since the last call
|
|
263
|
-
:param overwrite: whether or not overwrite file when exists, default is True
|
|
264
|
-
"""
|
|
265
|
-
from megfile.fs import is_fs
|
|
266
|
-
from megfile.fs_path import FSPath
|
|
267
|
-
|
|
268
|
-
if not is_fs(src_url):
|
|
269
|
-
raise OSError(f"src_url is not fs path: {src_url}")
|
|
270
|
-
if not is_sftp(dst_url) and not isinstance(dst_url, SftpPath):
|
|
271
|
-
raise OSError(f"dst_url is not sftp path: {dst_url}")
|
|
272
|
-
|
|
273
|
-
if followlinks and os.path.islink(src_url):
|
|
274
|
-
src_url = os.readlink(src_url)
|
|
275
|
-
if os.path.isdir(src_url):
|
|
276
|
-
raise IsADirectoryError("Is a directory: %r" % src_url)
|
|
277
|
-
if str(dst_url).endswith("/"):
|
|
278
|
-
raise IsADirectoryError("Is a directory: %r" % dst_url)
|
|
279
|
-
|
|
280
|
-
src_path = FSPath(src_url)
|
|
281
|
-
if isinstance(dst_url, SftpPath):
|
|
282
|
-
dst_path: SftpPath = dst_url
|
|
283
|
-
else:
|
|
284
|
-
dst_path: SftpPath = SftpPath(dst_url)
|
|
285
|
-
if not overwrite and dst_path.exists():
|
|
286
|
-
return
|
|
287
|
-
|
|
288
|
-
dst_path.parent.makedirs(exist_ok=True)
|
|
289
|
-
|
|
290
|
-
sftp_callback = None
|
|
291
|
-
if callback:
|
|
292
|
-
bytes_transferred_before = 0
|
|
293
|
-
|
|
294
|
-
def sftp_callback(bytes_transferred: int, _total_bytes: int):
|
|
295
|
-
nonlocal bytes_transferred_before
|
|
296
|
-
callback(bytes_transferred - bytes_transferred_before) # pyre-ignore[29]
|
|
297
|
-
bytes_transferred_before = bytes_transferred
|
|
298
|
-
|
|
299
|
-
dst_path._client.put(
|
|
300
|
-
src_path.path_without_protocol, dst_path._real_path, callback=sftp_callback
|
|
301
|
-
)
|
|
302
|
-
|
|
303
|
-
src_stat = src_path.stat()
|
|
304
|
-
dst_path.utime(src_stat.st_atime, src_stat.st_mtime)
|
|
305
|
-
dst_path.chmod(src_stat.st_mode)
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
def sftp_path_join(path: PathLike, *other_paths: PathLike) -> str:
|
|
309
|
-
"""
|
|
310
|
-
Concat 2 or more path to a complete path
|
|
311
|
-
|
|
312
|
-
:param path: Given path
|
|
313
|
-
:param other_paths: Paths to be concatenated
|
|
314
|
-
:returns: Concatenated complete path
|
|
315
|
-
|
|
316
|
-
.. note ::
|
|
317
|
-
|
|
318
|
-
The difference between this function and ``os.path.join`` is that this function
|
|
319
|
-
ignores left side slash (which indicates absolute path) in ``other_paths``
|
|
320
|
-
and will directly concat.
|
|
321
|
-
|
|
322
|
-
e.g. os.path.join('/path', 'to', '/file') => '/file',
|
|
323
|
-
but sftp_path_join('/path', 'to', '/file') => '/path/to/file'
|
|
324
|
-
"""
|
|
325
|
-
return uri_join(fspath(path), *map(fspath, other_paths))
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
def sftp_concat(src_paths: List[PathLike], dst_path: PathLike) -> None:
|
|
329
|
-
"""Concatenate sftp files to one file.
|
|
330
|
-
|
|
331
|
-
:param src_paths: Given source paths
|
|
332
|
-
:param dst_path: Given destination path
|
|
333
|
-
"""
|
|
334
|
-
dst_path_obj = SftpPath(dst_path)
|
|
335
|
-
|
|
336
|
-
def get_real_path(path: PathLike) -> str:
|
|
337
|
-
return SftpPath(path)._real_path
|
|
338
|
-
|
|
339
|
-
command = ["cat", *map(get_real_path, src_paths), ">", get_real_path(dst_path)]
|
|
340
|
-
exec_result = dst_path_obj._exec_command(command)
|
|
341
|
-
if exec_result.returncode != 0:
|
|
342
|
-
_logger.error(exec_result.stderr)
|
|
343
|
-
raise OSError(f"Failed to concat {src_paths} to {dst_path}")
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
def sftp_lstat(path: PathLike) -> StatResult:
|
|
347
|
-
"""
|
|
348
|
-
Get StatResult of file on sftp, including file size and mtime,
|
|
349
|
-
referring to fs_getsize and fs_getmtime
|
|
350
|
-
|
|
351
|
-
:param path: Given path
|
|
352
|
-
:returns: StatResult
|
|
353
|
-
"""
|
|
354
|
-
return SftpPath(path).lstat()
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
def sftp_exists(path: PathLike, followlinks: bool = False) -> bool:
|
|
358
|
-
"""
|
|
359
|
-
Test if the path exists
|
|
360
|
-
|
|
361
|
-
:param path: Given path
|
|
362
|
-
:param followlinks: False if regard symlink as file, else True
|
|
363
|
-
:returns: True if the path exists, else False
|
|
364
|
-
|
|
365
|
-
"""
|
|
366
|
-
return SftpPath(path).exists(followlinks)
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
def sftp_getmtime(path: PathLike, follow_symlinks: bool = False) -> float:
|
|
370
|
-
"""
|
|
371
|
-
Get last-modified time of the file on the given path (in Unix timestamp format).
|
|
372
|
-
|
|
373
|
-
If the path is an existent directory,
|
|
374
|
-
return the latest modified time of all file in it.
|
|
375
|
-
|
|
376
|
-
:param path: Given path
|
|
377
|
-
:returns: last-modified time
|
|
378
|
-
"""
|
|
379
|
-
return SftpPath(path).getmtime(follow_symlinks)
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
def sftp_getsize(path: PathLike, follow_symlinks: bool = False) -> int:
|
|
383
|
-
"""
|
|
384
|
-
Get file size on the given file path (in bytes).
|
|
385
|
-
|
|
386
|
-
If the path in a directory, return the sum of all file size in it,
|
|
387
|
-
including file in subdirectories (if exist).
|
|
388
|
-
|
|
389
|
-
The result excludes the size of directory itself. In other words,
|
|
390
|
-
return 0 Byte on an empty directory path.
|
|
391
|
-
|
|
392
|
-
:param path: Given path
|
|
393
|
-
:returns: File size
|
|
394
|
-
|
|
395
|
-
"""
|
|
396
|
-
return SftpPath(path).getsize(follow_symlinks)
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
def sftp_isdir(path: PathLike, followlinks: bool = False) -> bool:
|
|
400
|
-
"""
|
|
401
|
-
Test if a path is directory
|
|
402
|
-
|
|
403
|
-
.. note::
|
|
404
|
-
|
|
405
|
-
The difference between this function and ``os.path.isdir`` is that
|
|
406
|
-
this function regard symlink as file
|
|
407
|
-
|
|
408
|
-
:param path: Given path
|
|
409
|
-
:param followlinks: False if regard symlink as file, else True
|
|
410
|
-
:returns: True if the path is a directory, else False
|
|
411
|
-
|
|
412
|
-
"""
|
|
413
|
-
return SftpPath(path).is_dir(followlinks)
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
def sftp_isfile(path: PathLike, followlinks: bool = False) -> bool:
|
|
417
|
-
"""
|
|
418
|
-
Test if a path is file
|
|
419
|
-
|
|
420
|
-
.. note::
|
|
421
|
-
|
|
422
|
-
The difference between this function and ``os.path.isfile`` is that
|
|
423
|
-
this function regard symlink as file
|
|
424
|
-
|
|
425
|
-
:param path: Given path
|
|
426
|
-
:param followlinks: False if regard symlink as file, else True
|
|
427
|
-
:returns: True if the path is a file, else False
|
|
428
|
-
|
|
429
|
-
"""
|
|
430
|
-
return SftpPath(path).is_file(followlinks)
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
def sftp_listdir(path: PathLike) -> List[str]:
|
|
434
|
-
"""
|
|
435
|
-
Get all contents of given sftp path.
|
|
436
|
-
The result is in ascending alphabetical order.
|
|
437
|
-
|
|
438
|
-
:param path: Given path
|
|
439
|
-
:returns: All contents have in the path in ascending alphabetical order
|
|
440
|
-
"""
|
|
441
|
-
return SftpPath(path).listdir()
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
def sftp_load_from(path: PathLike) -> BinaryIO:
|
|
445
|
-
"""Read all content on specified path and write into memory
|
|
446
|
-
|
|
447
|
-
User should close the BinaryIO manually
|
|
448
|
-
|
|
449
|
-
:param path: Given path
|
|
450
|
-
:returns: Binary stream
|
|
451
|
-
"""
|
|
452
|
-
return SftpPath(path).load()
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
def sftp_makedirs(
|
|
456
|
-
path: PathLike, mode=0o777, parents: bool = False, exist_ok: bool = False
|
|
457
|
-
):
|
|
458
|
-
"""
|
|
459
|
-
make a directory on sftp, including parent directory.
|
|
460
|
-
If there exists a file on the path, raise FileExistsError
|
|
461
|
-
|
|
462
|
-
:param path: Given path
|
|
463
|
-
:param mode: If mode is given, it is combined with the process’ umask value to
|
|
464
|
-
determine the file mode and access flags.
|
|
465
|
-
:param parents: If parents is true, any missing parents of this path
|
|
466
|
-
are created as needed; If parents is false (the default),
|
|
467
|
-
a missing parent raises FileNotFoundError.
|
|
468
|
-
:param exist_ok: If False and target directory exists, raise FileExistsError
|
|
469
|
-
|
|
470
|
-
:raises: FileExistsError
|
|
471
|
-
"""
|
|
472
|
-
return SftpPath(path).mkdir(mode, parents, exist_ok)
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
def sftp_realpath(path: PathLike) -> str:
|
|
476
|
-
"""Return the real path of given path
|
|
477
|
-
|
|
478
|
-
:param path: Given path
|
|
479
|
-
:returns: Real path of given path
|
|
480
|
-
"""
|
|
481
|
-
return SftpPath(path).realpath()
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
def sftp_rename(
|
|
485
|
-
src_path: PathLike, dst_path: PathLike, overwrite: bool = True
|
|
486
|
-
) -> "SftpPath":
|
|
487
|
-
"""
|
|
488
|
-
rename file on sftp
|
|
489
|
-
|
|
490
|
-
:param src_path: Given path
|
|
491
|
-
:param dst_path: Given destination path
|
|
492
|
-
:param overwrite: whether or not overwrite file when exists
|
|
493
|
-
"""
|
|
494
|
-
return SftpPath(src_path).rename(dst_path, overwrite)
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
def sftp_move(
|
|
498
|
-
src_path: PathLike, dst_path: PathLike, overwrite: bool = True
|
|
499
|
-
) -> "SftpPath":
|
|
500
|
-
"""
|
|
501
|
-
move file on sftp
|
|
502
|
-
|
|
503
|
-
:param src_path: Given path
|
|
504
|
-
:param dst_path: Given destination path
|
|
505
|
-
:param overwrite: whether or not overwrite file when exists
|
|
506
|
-
"""
|
|
507
|
-
return SftpPath(src_path).replace(dst_path, overwrite)
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
def sftp_remove(path: PathLike, missing_ok: bool = False) -> None:
|
|
511
|
-
"""
|
|
512
|
-
Remove the file or directory on sftp
|
|
513
|
-
|
|
514
|
-
:param path: Given path
|
|
515
|
-
:param missing_ok: if False and target file/directory not exists,
|
|
516
|
-
raise FileNotFoundError
|
|
517
|
-
"""
|
|
518
|
-
return SftpPath(path).remove(missing_ok)
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
def sftp_scan(
|
|
522
|
-
path: PathLike, missing_ok: bool = True, followlinks: bool = False
|
|
523
|
-
) -> Iterator[str]:
|
|
524
|
-
"""
|
|
525
|
-
Iteratively traverse only files in given directory, in alphabetical order.
|
|
526
|
-
Every iteration on generator yields a path string.
|
|
527
|
-
|
|
528
|
-
If path is a file path, yields the file only
|
|
529
|
-
If path is a non-existent path, return an empty generator
|
|
530
|
-
If path is a bucket path, return all file paths in the bucket
|
|
531
|
-
|
|
532
|
-
:param path: Given path
|
|
533
|
-
:param missing_ok: If False and there's no file in the directory,
|
|
534
|
-
raise FileNotFoundError
|
|
535
|
-
:returns: A file path generator
|
|
536
|
-
"""
|
|
537
|
-
return SftpPath(path).scan(missing_ok, followlinks)
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
def sftp_scan_stat(
|
|
541
|
-
path: PathLike, missing_ok: bool = True, followlinks: bool = False
|
|
542
|
-
) -> Iterator[FileEntry]:
|
|
543
|
-
"""
|
|
544
|
-
Iteratively traverse only files in given directory, in alphabetical order.
|
|
545
|
-
Every iteration on generator yields a tuple of path string and file stat
|
|
546
|
-
|
|
547
|
-
:param path: Given path
|
|
548
|
-
:param missing_ok: If False and there's no file in the directory,
|
|
549
|
-
raise FileNotFoundError
|
|
550
|
-
:returns: A file path generator
|
|
551
|
-
"""
|
|
552
|
-
return SftpPath(path).scan_stat(missing_ok, followlinks)
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
def sftp_scandir(path: PathLike) -> Iterator[FileEntry]:
|
|
556
|
-
"""
|
|
557
|
-
Get all content of given file path.
|
|
558
|
-
|
|
559
|
-
:param path: Given path
|
|
560
|
-
:returns: An iterator contains all contents have prefix path
|
|
561
|
-
"""
|
|
562
|
-
return SftpPath(path).scandir()
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
def sftp_stat(path: PathLike, follow_symlinks=True) -> StatResult:
|
|
566
|
-
"""
|
|
567
|
-
Get StatResult of file on sftp, including file size and mtime,
|
|
568
|
-
referring to fs_getsize and fs_getmtime
|
|
569
|
-
|
|
570
|
-
:param path: Given path
|
|
571
|
-
:returns: StatResult
|
|
572
|
-
"""
|
|
573
|
-
return SftpPath(path).stat(follow_symlinks)
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
def sftp_unlink(path: PathLike, missing_ok: bool = False) -> None:
|
|
577
|
-
"""
|
|
578
|
-
Remove the file on sftp
|
|
579
|
-
|
|
580
|
-
:param path: Given path
|
|
581
|
-
:param missing_ok: if False and target file not exists, raise FileNotFoundError
|
|
582
|
-
"""
|
|
583
|
-
return SftpPath(path).unlink(missing_ok)
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
def sftp_walk(
|
|
587
|
-
path: PathLike, followlinks: bool = False
|
|
588
|
-
) -> Iterator[Tuple[str, List[str], List[str]]]:
|
|
589
|
-
"""
|
|
590
|
-
Generate the file names in a directory tree by walking the tree top-down.
|
|
591
|
-
For each directory in the tree rooted at directory path (including path itself),
|
|
592
|
-
it yields a 3-tuple (root, dirs, files).
|
|
593
|
-
|
|
594
|
-
- root: a string of current path
|
|
595
|
-
- dirs: name list of subdirectories (excluding '.' and '..' if they exist)
|
|
596
|
-
in 'root'. The list is sorted by ascending alphabetical order
|
|
597
|
-
- files: name list of non-directory files (link is regarded as file) in 'root'.
|
|
598
|
-
The list is sorted by ascending alphabetical order
|
|
599
|
-
|
|
600
|
-
If path not exists, or path is a file (link is regarded as file),
|
|
601
|
-
return an empty generator
|
|
602
|
-
|
|
603
|
-
.. note::
|
|
604
|
-
|
|
605
|
-
Be aware that setting ``followlinks`` to True can lead to infinite recursion
|
|
606
|
-
if a link points to a parent directory of itself. fs_walk() does not keep
|
|
607
|
-
track of the directories it visited already.
|
|
608
|
-
|
|
609
|
-
:param path: Given path
|
|
610
|
-
:param followlinks: False if regard symlink as file, else True
|
|
611
|
-
:returns: A 3-tuple generator
|
|
612
|
-
"""
|
|
613
|
-
return SftpPath(path).walk(followlinks)
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
def sftp_getmd5(path: PathLike, recalculate: bool = False, followlinks: bool = False):
|
|
617
|
-
"""
|
|
618
|
-
Calculate the md5 value of the file
|
|
619
|
-
|
|
620
|
-
:param path: Given path
|
|
621
|
-
:param recalculate: Ignore this parameter, just for compatibility
|
|
622
|
-
:param followlinks: Ignore this parameter, just for compatibility
|
|
623
|
-
|
|
624
|
-
returns: md5 of file
|
|
625
|
-
"""
|
|
626
|
-
return SftpPath(path).md5(recalculate, followlinks)
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
def sftp_symlink(src_path: PathLike, dst_path: PathLike) -> None:
|
|
630
|
-
"""
|
|
631
|
-
Create a symbolic link pointing to src_path named dst_path.
|
|
632
|
-
|
|
633
|
-
:param src_path: Given path
|
|
634
|
-
:param dst_path: Destination path
|
|
635
|
-
"""
|
|
636
|
-
return SftpPath(src_path).symlink(dst_path)
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
def sftp_islink(path: PathLike) -> bool:
|
|
640
|
-
"""Test whether a path is a symbolic link
|
|
641
|
-
|
|
642
|
-
:param path: Given path
|
|
643
|
-
:return: If path is a symbolic link return True, else False
|
|
644
|
-
:rtype: bool
|
|
645
|
-
"""
|
|
646
|
-
return SftpPath(path).is_symlink()
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
def sftp_save_as(file_object: BinaryIO, path: PathLike):
|
|
650
|
-
"""Write the opened binary stream to path
|
|
651
|
-
If parent directory of path doesn't exist, it will be created.
|
|
652
|
-
|
|
653
|
-
:param path: Given path
|
|
654
|
-
:param file_object: stream to be read
|
|
655
|
-
"""
|
|
656
|
-
return SftpPath(path).save(file_object)
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
def sftp_open(
|
|
660
|
-
path: PathLike,
|
|
661
|
-
mode: str = "r",
|
|
662
|
-
*,
|
|
663
|
-
buffering=-1,
|
|
664
|
-
encoding: Optional[str] = None,
|
|
665
|
-
errors: Optional[str] = None,
|
|
666
|
-
**kwargs,
|
|
667
|
-
) -> IO:
|
|
668
|
-
"""Open a file on the path.
|
|
669
|
-
|
|
670
|
-
:param path: Given path
|
|
671
|
-
:param mode: Mode to open file
|
|
672
|
-
:param buffering: buffering is an optional integer used to
|
|
673
|
-
set the buffering policy.
|
|
674
|
-
:param encoding: encoding is the name of the encoding used to decode or encode
|
|
675
|
-
the file. This should only be used in text mode.
|
|
676
|
-
:param errors: errors is an optional string that specifies how encoding and
|
|
677
|
-
decoding errors are to be handled—this cannot be used in binary mode.
|
|
678
|
-
:returns: File-Like object
|
|
679
|
-
"""
|
|
680
|
-
return SftpPath(path).open(
|
|
681
|
-
mode, buffering=buffering, encoding=encoding, errors=errors
|
|
682
|
-
)
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
def sftp_chmod(path: PathLike, mode: int, *, follow_symlinks: bool = True):
|
|
686
|
-
"""
|
|
687
|
-
Change the file mode and permissions, like os.chmod().
|
|
688
|
-
|
|
689
|
-
:param path: Given path
|
|
690
|
-
:param mode: the file mode you want to change
|
|
691
|
-
:param followlinks: Ignore this parameter, just for compatibility
|
|
692
|
-
"""
|
|
693
|
-
return SftpPath(path).chmod(mode, follow_symlinks=follow_symlinks)
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
def sftp_absolute(path: PathLike) -> "SftpPath":
|
|
697
|
-
"""
|
|
698
|
-
Make the path absolute, without normalization or resolving symlinks.
|
|
699
|
-
Returns a new path object
|
|
700
|
-
"""
|
|
701
|
-
return SftpPath(path).absolute()
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
def sftp_rmdir(path: PathLike):
|
|
705
|
-
"""
|
|
706
|
-
Remove this directory. The directory must be empty.
|
|
707
|
-
"""
|
|
708
|
-
return SftpPath(path).rmdir()
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
def sftp_copy(
|
|
712
|
-
src_path: PathLike,
|
|
713
|
-
dst_path: PathLike,
|
|
714
|
-
callback: Optional[Callable[[int], None]] = None,
|
|
715
|
-
followlinks: bool = False,
|
|
716
|
-
overwrite: bool = True,
|
|
717
|
-
):
|
|
718
|
-
"""
|
|
719
|
-
Copy the file to the given destination path.
|
|
720
|
-
|
|
721
|
-
:param src_path: Given path
|
|
722
|
-
:param dst_path: The destination path to copy the file to.
|
|
723
|
-
:param callback: An optional callback function that takes an integer parameter
|
|
724
|
-
and is called periodically during the copy operation to report the number
|
|
725
|
-
of bytes copied.
|
|
726
|
-
:param followlinks: Whether to follow symbolic links when copying directories.
|
|
727
|
-
:raises IsADirectoryError: If the source is a directory.
|
|
728
|
-
:raises OSError: If there is an error copying the file.
|
|
729
|
-
"""
|
|
730
|
-
return SftpPath(src_path).copy(dst_path, callback, followlinks, overwrite)
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
def sftp_sync(
|
|
734
|
-
src_path: PathLike,
|
|
735
|
-
dst_path: PathLike,
|
|
736
|
-
followlinks: bool = False,
|
|
737
|
-
force: bool = False,
|
|
738
|
-
overwrite: bool = True,
|
|
739
|
-
):
|
|
740
|
-
"""Copy file/directory on src_url to dst_url
|
|
741
|
-
|
|
742
|
-
:param src_path: Given path
|
|
743
|
-
:param dst_url: Given destination path
|
|
744
|
-
:param followlinks: False if regard symlink as file, else True
|
|
745
|
-
:param force: Sync file forcible, do not ignore same files,
|
|
746
|
-
priority is higher than 'overwrite', default is False
|
|
747
|
-
:param overwrite: whether or not overwrite file when exists, default is True
|
|
748
|
-
"""
|
|
749
|
-
return SftpPath(src_path).sync(dst_path, followlinks, force, overwrite)
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
def _check_input(input_str: str, fingerprint: str, times: int = 0) -> bool:
|
|
753
|
-
answers = input_str.strip()
|
|
754
|
-
if answers.lower() in ("yes", "y") or answers == fingerprint:
|
|
755
|
-
return True
|
|
756
|
-
elif answers.lower() in ("no", "n"):
|
|
757
|
-
return False
|
|
758
|
-
elif times >= 10:
|
|
759
|
-
_logger.warning("Retried more than 10 times, give up")
|
|
760
|
-
return False
|
|
761
|
-
else:
|
|
762
|
-
input_str = input("Please type 'yes', 'no' or the fingerprint: ")
|
|
763
|
-
return _check_input(input_str, fingerprint, times=times + 1)
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
def _prompt_add_to_known_hosts(hostname, key) -> bool:
|
|
767
|
-
fingerprint = hashlib.sha256(key.asbytes()).digest()
|
|
768
|
-
fingerprint = f"SHA256:{base64.b64encode(fingerprint).decode('utf-8')}"
|
|
769
|
-
answers = input(f"""The authenticity of host '{hostname}' can't be established.
|
|
770
|
-
{key.get_name().upper()} key fingerprint is {fingerprint}.
|
|
771
|
-
This key is not known by any other names.
|
|
772
|
-
Are you sure you want to continue connecting (yes/no/[fingerprint])? """)
|
|
773
|
-
return _check_input(answers, fingerprint)
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
def sftp_add_host_key(
|
|
777
|
-
hostname: str,
|
|
778
|
-
port: int = 22,
|
|
779
|
-
prompt: bool = False,
|
|
780
|
-
host_key_path: Optional["str"] = None,
|
|
781
|
-
):
|
|
782
|
-
"""Add a host key to known_hosts.
|
|
783
|
-
|
|
784
|
-
:param hostname: hostname
|
|
785
|
-
:param port: port, default is 22
|
|
786
|
-
:param prompt: If True, requires user input of 'yes' or 'no' to decide whether to
|
|
787
|
-
add this host key
|
|
788
|
-
:param host_key_path: path of known_hosts, default is ~/.ssh/known_hosts
|
|
789
|
-
"""
|
|
790
|
-
if not host_key_path:
|
|
791
|
-
host_key_path = os.path.expanduser("~/.ssh/known_hosts")
|
|
792
|
-
|
|
793
|
-
if not os.path.exists(host_key_path):
|
|
794
|
-
dirname = os.path.dirname(host_key_path)
|
|
795
|
-
if dirname and dirname != ".":
|
|
796
|
-
os.makedirs(dirname, exist_ok=True, mode=0o700)
|
|
797
|
-
with open(host_key_path, "w"):
|
|
798
|
-
pass
|
|
799
|
-
os.chmod(host_key_path, 0o600)
|
|
800
|
-
|
|
801
|
-
host_key = paramiko.hostkeys.HostKeys(host_key_path)
|
|
802
|
-
if host_key.lookup(hostname):
|
|
803
|
-
return
|
|
804
|
-
|
|
805
|
-
transport = paramiko.Transport(
|
|
806
|
-
(
|
|
807
|
-
hostname,
|
|
808
|
-
port,
|
|
809
|
-
)
|
|
810
|
-
)
|
|
811
|
-
transport.connect()
|
|
812
|
-
key = transport.get_remote_server_key()
|
|
813
|
-
transport.close()
|
|
814
|
-
|
|
815
|
-
if prompt:
|
|
816
|
-
result = _prompt_add_to_known_hosts(hostname, key)
|
|
817
|
-
if not result:
|
|
818
|
-
return
|
|
819
|
-
|
|
820
|
-
host_key.add(hostname, key.get_name(), key)
|
|
821
|
-
host_key.save(host_key_path)
|