megfile 4.2.5__py3-none-any.whl → 5.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- megfile/__init__.py +14 -291
- megfile/cli.py +83 -40
- megfile/config.py +35 -1
- megfile/errors.py +2 -2
- megfile/fs_path.py +32 -3
- megfile/interfaces.py +21 -10
- megfile/lib/base_memory_handler.py +92 -0
- megfile/lib/glob.py +3 -3
- megfile/lib/http_prefetch_reader.py +22 -22
- megfile/lib/s3_memory_handler.py +14 -81
- megfile/lib/webdav_memory_handler.py +83 -0
- megfile/lib/webdav_prefetch_reader.py +115 -0
- megfile/pathlike.py +3 -4
- megfile/s3_path.py +40 -32
- megfile/sftp2_path.py +38 -62
- megfile/sftp_path.py +238 -1
- megfile/smart.py +70 -29
- megfile/smart_path.py +198 -96
- megfile/version.py +1 -1
- megfile/webdav_path.py +161 -166
- {megfile-4.2.5.dist-info → megfile-5.0.1.dist-info}/METADATA +27 -39
- megfile-5.0.1.dist-info/RECORD +51 -0
- megfile/fs.py +0 -627
- megfile/hdfs.py +0 -408
- megfile/http.py +0 -114
- megfile/s3.py +0 -540
- megfile/sftp.py +0 -821
- megfile/sftp2.py +0 -827
- megfile/stdio.py +0 -30
- megfile/webdav.py +0 -552
- megfile-4.2.5.dist-info/RECORD +0 -56
- {megfile-4.2.5.dist-info → megfile-5.0.1.dist-info}/WHEEL +0 -0
- {megfile-4.2.5.dist-info → megfile-5.0.1.dist-info}/entry_points.txt +0 -0
- {megfile-4.2.5.dist-info → megfile-5.0.1.dist-info}/licenses/LICENSE +0 -0
- {megfile-4.2.5.dist-info → megfile-5.0.1.dist-info}/licenses/LICENSE.pyre +0 -0
- {megfile-4.2.5.dist-info → megfile-5.0.1.dist-info}/top_level.txt +0 -0
megfile/sftp2.py
DELETED
|
@@ -1,827 +0,0 @@
|
|
|
1
|
-
import base64
|
|
2
|
-
import hashlib
|
|
3
|
-
import os
|
|
4
|
-
from logging import getLogger as get_logger
|
|
5
|
-
from typing import IO, BinaryIO, Callable, Iterator, List, Optional, Tuple
|
|
6
|
-
|
|
7
|
-
import ssh2.session # type: ignore
|
|
8
|
-
import ssh2.utils # type: ignore
|
|
9
|
-
|
|
10
|
-
from megfile.interfaces import FileEntry, PathLike, StatResult
|
|
11
|
-
from megfile.lib.compat import fspath
|
|
12
|
-
from megfile.lib.joinpath import uri_join
|
|
13
|
-
from megfile.sftp2_path import (
|
|
14
|
-
Sftp2Path,
|
|
15
|
-
is_sftp2,
|
|
16
|
-
)
|
|
17
|
-
from megfile.utils import copyfileobj
|
|
18
|
-
|
|
19
|
-
_logger = get_logger(__name__)
|
|
20
|
-
|
|
21
|
-
__all__ = [
|
|
22
|
-
"is_sftp2",
|
|
23
|
-
"sftp2_readlink",
|
|
24
|
-
"sftp2_glob",
|
|
25
|
-
"sftp2_iglob",
|
|
26
|
-
"sftp2_glob_stat",
|
|
27
|
-
"sftp2_resolve",
|
|
28
|
-
"sftp2_download",
|
|
29
|
-
"sftp2_upload",
|
|
30
|
-
"sftp2_path_join",
|
|
31
|
-
"sftp2_concat",
|
|
32
|
-
"sftp2_lstat",
|
|
33
|
-
"sftp2_exists",
|
|
34
|
-
"sftp2_getmtime",
|
|
35
|
-
"sftp2_getsize",
|
|
36
|
-
"sftp2_isdir",
|
|
37
|
-
"sftp2_isfile",
|
|
38
|
-
"sftp2_listdir",
|
|
39
|
-
"sftp2_load_from",
|
|
40
|
-
"sftp2_makedirs",
|
|
41
|
-
"sftp2_realpath",
|
|
42
|
-
"sftp2_rename",
|
|
43
|
-
"sftp2_move",
|
|
44
|
-
"sftp2_remove",
|
|
45
|
-
"sftp2_scan",
|
|
46
|
-
"sftp2_scan_stat",
|
|
47
|
-
"sftp2_scandir",
|
|
48
|
-
"sftp2_stat",
|
|
49
|
-
"sftp2_unlink",
|
|
50
|
-
"sftp2_walk",
|
|
51
|
-
"sftp2_getmd5",
|
|
52
|
-
"sftp2_symlink",
|
|
53
|
-
"sftp2_islink",
|
|
54
|
-
"sftp2_save_as",
|
|
55
|
-
"sftp2_open",
|
|
56
|
-
"sftp2_chmod",
|
|
57
|
-
"sftp2_absolute",
|
|
58
|
-
"sftp2_rmdir",
|
|
59
|
-
"sftp2_copy",
|
|
60
|
-
"sftp2_sync",
|
|
61
|
-
"sftp2_add_host_key",
|
|
62
|
-
]
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
def sftp2_readlink(path: PathLike) -> "str":
|
|
66
|
-
"""
|
|
67
|
-
Return a Sftp2Path instance representing the path to which the symbolic link points.
|
|
68
|
-
|
|
69
|
-
:param path: Given path
|
|
70
|
-
:returns: Return a Sftp2Path instance representing the path to
|
|
71
|
-
which the symbolic link points.
|
|
72
|
-
"""
|
|
73
|
-
return Sftp2Path(path).readlink().path_with_protocol
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
def sftp2_glob(
|
|
77
|
-
path: PathLike, recursive: bool = True, missing_ok: bool = True
|
|
78
|
-
) -> List[str]:
|
|
79
|
-
"""Return path list in ascending alphabetical order,
|
|
80
|
-
in which path matches glob pattern
|
|
81
|
-
|
|
82
|
-
1. If doesn't match any path, return empty list
|
|
83
|
-
Notice: ``glob.glob`` in standard library returns ['a/'] instead of empty list
|
|
84
|
-
when pathname is like `a/**`, recursive is True and directory 'a' doesn't exist.
|
|
85
|
-
fs_glob behaves like ``glob.glob`` in standard library under such circumstance.
|
|
86
|
-
2. No guarantee that each path in result is different, which means:
|
|
87
|
-
Assume there exists a path `/a/b/c/b/d.txt`
|
|
88
|
-
use path pattern like `/**/b/**/*.txt` to glob,
|
|
89
|
-
the path above will be returned twice
|
|
90
|
-
3. `**` will match any matched file, directory, symlink and '' by default,
|
|
91
|
-
when recursive is `True`
|
|
92
|
-
4. fs_glob returns same as glob.glob(pathname, recursive=True)
|
|
93
|
-
in ascending alphabetical order.
|
|
94
|
-
5. Hidden files (filename stars with '.') will not be found in the result
|
|
95
|
-
|
|
96
|
-
:param path: Given path
|
|
97
|
-
:param pattern: Glob the given relative pattern in the directory represented
|
|
98
|
-
by this path
|
|
99
|
-
:param recursive: If False, `**` will not search directory recursively
|
|
100
|
-
:param missing_ok: If False and target path doesn't match any file,
|
|
101
|
-
raise FileNotFoundError
|
|
102
|
-
:returns: A list contains paths match `pathname`
|
|
103
|
-
"""
|
|
104
|
-
return list(sftp2_iglob(path=path, recursive=recursive, missing_ok=missing_ok))
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
def sftp2_glob_stat(
|
|
108
|
-
path: PathLike, recursive: bool = True, missing_ok: bool = True
|
|
109
|
-
) -> Iterator[FileEntry]:
|
|
110
|
-
"""Return a list contains tuples of path and file stat, in ascending alphabetical
|
|
111
|
-
order, in which path matches glob pattern
|
|
112
|
-
|
|
113
|
-
1. If doesn't match any path, return empty list
|
|
114
|
-
Notice: ``glob.glob`` in standard library returns ['a/'] instead of empty list
|
|
115
|
-
when pathname is like `a/**`, recursive is True and directory 'a' doesn't exist.
|
|
116
|
-
sftp2_glob behaves like ``glob.glob`` in standard library under such
|
|
117
|
-
circumstance.
|
|
118
|
-
2. No guarantee that each path in result is different, which means:
|
|
119
|
-
Assume there exists a path `/a/b/c/b/d.txt`
|
|
120
|
-
use path pattern like `/**/b/**/*.txt` to glob,
|
|
121
|
-
the path above will be returned twice
|
|
122
|
-
3. `**` will match any matched file, directory, symlink and '' by default,
|
|
123
|
-
when recursive is `True`
|
|
124
|
-
4. fs_glob returns same as glob.glob(pathname, recursive=True) in
|
|
125
|
-
ascending alphabetical order.
|
|
126
|
-
5. Hidden files (filename stars with '.') will not be found in the result
|
|
127
|
-
|
|
128
|
-
:param path: Given path
|
|
129
|
-
:param pattern: Glob the given relative pattern in the directory represented
|
|
130
|
-
by this path
|
|
131
|
-
:param recursive: If False, `**` will not search directory recursively
|
|
132
|
-
:param missing_ok: If False and target path doesn't match any file,
|
|
133
|
-
raise FileNotFoundError
|
|
134
|
-
:returns: A list contains tuples of path and file stat,
|
|
135
|
-
in which paths match `pathname`
|
|
136
|
-
"""
|
|
137
|
-
for path in sftp2_iglob(path=path, recursive=recursive, missing_ok=missing_ok):
|
|
138
|
-
path_object = Sftp2Path(path)
|
|
139
|
-
yield FileEntry(
|
|
140
|
-
path_object.name, path_object.path_with_protocol, path_object.lstat()
|
|
141
|
-
)
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
def sftp2_iglob(
|
|
145
|
-
path: PathLike, recursive: bool = True, missing_ok: bool = True
|
|
146
|
-
) -> Iterator[str]:
|
|
147
|
-
"""Return path iterator in ascending alphabetical order,
|
|
148
|
-
in which path matches glob pattern
|
|
149
|
-
|
|
150
|
-
1. If doesn't match any path, return empty list
|
|
151
|
-
Notice: ``glob.glob`` in standard library returns ['a/'] instead of empty list
|
|
152
|
-
when pathname is like `a/**`, recursive is True and directory 'a' doesn't exist.
|
|
153
|
-
fs_glob behaves like ``glob.glob`` in standard library under such circumstance.
|
|
154
|
-
2. No guarantee that each path in result is different, which means:
|
|
155
|
-
Assume there exists a path `/a/b/c/b/d.txt`
|
|
156
|
-
use path pattern like `/**/b/**/*.txt` to glob,
|
|
157
|
-
the path above will be returned twice
|
|
158
|
-
3. `**` will match any matched file, directory, symlink and '' by default,
|
|
159
|
-
when recursive is `True`
|
|
160
|
-
4. fs_glob returns same as glob.glob(pathname, recursive=True) in
|
|
161
|
-
ascending alphabetical order.
|
|
162
|
-
5. Hidden files (filename stars with '.') will not be found in the result
|
|
163
|
-
|
|
164
|
-
:param path: Given path
|
|
165
|
-
:param pattern: Glob the given relative pattern in the directory represented
|
|
166
|
-
by this path
|
|
167
|
-
:param recursive: If False, `**` will not search directory recursively
|
|
168
|
-
:param missing_ok: If False and target path doesn't match any file,
|
|
169
|
-
raise FileNotFoundError
|
|
170
|
-
:returns: An iterator contains paths match `pathname`
|
|
171
|
-
"""
|
|
172
|
-
|
|
173
|
-
for path in Sftp2Path(path).iglob(
|
|
174
|
-
pattern="", recursive=recursive, missing_ok=missing_ok
|
|
175
|
-
):
|
|
176
|
-
yield path.path_with_protocol
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
def sftp2_resolve(path: PathLike, strict=False) -> "str":
|
|
180
|
-
"""Equal to fs_realpath
|
|
181
|
-
|
|
182
|
-
:param path: Given path
|
|
183
|
-
:param strict: Ignore this parameter, just for compatibility
|
|
184
|
-
:return: Return the canonical path of the specified filename,
|
|
185
|
-
eliminating any symbolic links encountered in the path.
|
|
186
|
-
:rtype: Sftp2Path
|
|
187
|
-
"""
|
|
188
|
-
return Sftp2Path(path).resolve(strict).path_with_protocol
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
def sftp2_download(
|
|
192
|
-
src_url: PathLike,
|
|
193
|
-
dst_url: PathLike,
|
|
194
|
-
callback: Optional[Callable[[int], None]] = None,
|
|
195
|
-
followlinks: bool = False,
|
|
196
|
-
overwrite: bool = True,
|
|
197
|
-
):
|
|
198
|
-
"""
|
|
199
|
-
Downloads a file from sftp2 to local filesystem.
|
|
200
|
-
|
|
201
|
-
:param src_url: source sftp2 path
|
|
202
|
-
:param dst_url: target fs path
|
|
203
|
-
:param callback: Called periodically during copy, and the input parameter is
|
|
204
|
-
the data size (in bytes) of copy since the last call
|
|
205
|
-
:param followlinks: False if regard symlink as file, else True
|
|
206
|
-
:param overwrite: whether or not overwrite file when exists, default is True
|
|
207
|
-
"""
|
|
208
|
-
from megfile.fs import is_fs
|
|
209
|
-
from megfile.fs_path import FSPath
|
|
210
|
-
|
|
211
|
-
if not is_fs(dst_url):
|
|
212
|
-
raise OSError(f"dst_url is not fs path: {dst_url}")
|
|
213
|
-
if not is_sftp2(src_url) and not isinstance(src_url, Sftp2Path):
|
|
214
|
-
raise OSError(f"src_url is not sftp2 path: {src_url}")
|
|
215
|
-
|
|
216
|
-
dst_path = FSPath(dst_url)
|
|
217
|
-
if not overwrite and dst_path.exists():
|
|
218
|
-
return
|
|
219
|
-
|
|
220
|
-
if isinstance(src_url, Sftp2Path):
|
|
221
|
-
src_path: Sftp2Path = src_url
|
|
222
|
-
else:
|
|
223
|
-
src_path: Sftp2Path = Sftp2Path(src_url)
|
|
224
|
-
|
|
225
|
-
if followlinks and src_path.is_symlink():
|
|
226
|
-
src_path = src_path.readlink()
|
|
227
|
-
if src_path.is_dir():
|
|
228
|
-
raise IsADirectoryError(f"Is a directory: {src_url!r}")
|
|
229
|
-
if str(dst_url).endswith("/"):
|
|
230
|
-
raise IsADirectoryError(f"Is a directory: {dst_url!r}")
|
|
231
|
-
|
|
232
|
-
dst_path.parent.makedirs(exist_ok=True)
|
|
233
|
-
|
|
234
|
-
with src_path.open("rb") as src_file, dst_path.open("wb") as dst_file:
|
|
235
|
-
copyfileobj(src_file, dst_file, callback)
|
|
236
|
-
|
|
237
|
-
src_stat = src_path.stat()
|
|
238
|
-
dst_path.utime(src_stat.st_atime, src_stat.st_mtime)
|
|
239
|
-
dst_path.chmod(src_stat.st_mode)
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
def sftp2_upload(
|
|
243
|
-
src_url: PathLike,
|
|
244
|
-
dst_url: PathLike,
|
|
245
|
-
callback: Optional[Callable[[int], None]] = None,
|
|
246
|
-
followlinks: bool = False,
|
|
247
|
-
overwrite: bool = True,
|
|
248
|
-
):
|
|
249
|
-
"""
|
|
250
|
-
Uploads a file from local filesystem to sftp2 server.
|
|
251
|
-
|
|
252
|
-
:param src_url: source fs path
|
|
253
|
-
:param dst_url: target sftp2 path
|
|
254
|
-
:param callback: Called periodically during copy, and the input parameter is
|
|
255
|
-
the data size (in bytes) of copy since the last call
|
|
256
|
-
:param overwrite: whether or not overwrite file when exists, default is True
|
|
257
|
-
"""
|
|
258
|
-
from megfile.fs import is_fs
|
|
259
|
-
from megfile.fs_path import FSPath
|
|
260
|
-
|
|
261
|
-
if not is_fs(src_url):
|
|
262
|
-
raise OSError(f"src_url is not fs path: {src_url}")
|
|
263
|
-
if not is_sftp2(dst_url) and not isinstance(dst_url, Sftp2Path):
|
|
264
|
-
raise OSError(f"dst_url is not sftp2 path: {dst_url}")
|
|
265
|
-
|
|
266
|
-
if followlinks and os.path.islink(src_url):
|
|
267
|
-
src_url = os.readlink(src_url)
|
|
268
|
-
if os.path.isdir(src_url):
|
|
269
|
-
raise IsADirectoryError(f"Is a directory: {src_url!r}")
|
|
270
|
-
if str(dst_url).endswith("/"):
|
|
271
|
-
raise IsADirectoryError(f"Is a directory: {dst_url!r}")
|
|
272
|
-
|
|
273
|
-
src_path = FSPath(src_url)
|
|
274
|
-
if isinstance(dst_url, Sftp2Path):
|
|
275
|
-
dst_path: Sftp2Path = dst_url
|
|
276
|
-
else:
|
|
277
|
-
dst_path: Sftp2Path = Sftp2Path(dst_url)
|
|
278
|
-
if not overwrite and dst_path.exists():
|
|
279
|
-
return
|
|
280
|
-
|
|
281
|
-
dst_path.parent.makedirs(exist_ok=True)
|
|
282
|
-
|
|
283
|
-
with src_path.open("rb") as src_file, dst_path.open("wb") as dst_file:
|
|
284
|
-
copyfileobj(src_file, dst_file, callback)
|
|
285
|
-
|
|
286
|
-
src_stat = src_path.stat()
|
|
287
|
-
dst_path.utime(src_stat.st_atime, src_stat.st_mtime)
|
|
288
|
-
dst_path.chmod(src_stat.st_mode)
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
def sftp2_path_join(path: PathLike, *other_paths: PathLike) -> str:
|
|
292
|
-
"""
|
|
293
|
-
Concat 2 or more path to a complete path
|
|
294
|
-
|
|
295
|
-
:param path: Given path
|
|
296
|
-
:param other_paths: Paths to be concatenated
|
|
297
|
-
:returns: Concatenated complete path
|
|
298
|
-
|
|
299
|
-
.. note ::
|
|
300
|
-
|
|
301
|
-
The difference between this function and ``os.path.join`` is that this function
|
|
302
|
-
ignores left side slash (which indicates absolute path) in ``other_paths``
|
|
303
|
-
and will directly concat.
|
|
304
|
-
|
|
305
|
-
e.g. os.path.join('/path', 'to', '/file') => '/file',
|
|
306
|
-
but sftp2_path_join('/path', 'to', '/file') => '/path/to/file'
|
|
307
|
-
"""
|
|
308
|
-
return uri_join(fspath(path), *map(fspath, other_paths))
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
def sftp2_concat(src_paths: List[PathLike], dst_path: PathLike) -> None:
|
|
312
|
-
"""Concatenate sftp2 files to one file.
|
|
313
|
-
|
|
314
|
-
:param src_paths: Given source paths
|
|
315
|
-
:param dst_path: Given destination path
|
|
316
|
-
"""
|
|
317
|
-
dst_path_obj = Sftp2Path(dst_path)
|
|
318
|
-
|
|
319
|
-
if len(src_paths) == 0:
|
|
320
|
-
return
|
|
321
|
-
|
|
322
|
-
# Check if all sources are on the same server as destination
|
|
323
|
-
all_same_backend = all(
|
|
324
|
-
dst_path_obj._is_same_backend(Sftp2Path(src_path)) for src_path in src_paths
|
|
325
|
-
)
|
|
326
|
-
|
|
327
|
-
if all_same_backend and len(src_paths) > 1:
|
|
328
|
-
# Use server-side cat command for efficiency
|
|
329
|
-
def get_real_path(path: PathLike) -> str:
|
|
330
|
-
return Sftp2Path(path)._real_path
|
|
331
|
-
|
|
332
|
-
exec_result = dst_path_obj._exec_command(
|
|
333
|
-
[
|
|
334
|
-
"cat",
|
|
335
|
-
*map(get_real_path, src_paths),
|
|
336
|
-
">",
|
|
337
|
-
get_real_path(dst_path),
|
|
338
|
-
]
|
|
339
|
-
)
|
|
340
|
-
|
|
341
|
-
if exec_result.returncode != 0:
|
|
342
|
-
# Log the failure but fall back to SFTP method
|
|
343
|
-
_logger.error(exec_result.stderr)
|
|
344
|
-
raise OSError(
|
|
345
|
-
f"Failed to concat files, returncode: {exec_result.returncode}, "
|
|
346
|
-
f"{exec_result.stderr}"
|
|
347
|
-
)
|
|
348
|
-
|
|
349
|
-
# Fallback to traditional SFTP concat (download then upload)
|
|
350
|
-
with dst_path_obj.open("wb") as dst_file:
|
|
351
|
-
for src_path in src_paths:
|
|
352
|
-
src_path_obj = Sftp2Path(src_path)
|
|
353
|
-
with src_path_obj.open("rb") as src_file:
|
|
354
|
-
# Use the copyfileobj utility function
|
|
355
|
-
copyfileobj(src_file, dst_file)
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
def sftp2_lstat(path: PathLike) -> StatResult:
|
|
359
|
-
"""
|
|
360
|
-
Get StatResult of file on sftp2, including file size and mtime,
|
|
361
|
-
referring to fs_getsize and fs_getmtime
|
|
362
|
-
|
|
363
|
-
:param path: Given path
|
|
364
|
-
:returns: StatResult
|
|
365
|
-
"""
|
|
366
|
-
return Sftp2Path(path).lstat()
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
def sftp2_exists(path: PathLike, followlinks: bool = False) -> bool:
|
|
370
|
-
"""
|
|
371
|
-
Test if the path exists
|
|
372
|
-
|
|
373
|
-
:param path: Given path
|
|
374
|
-
:param followlinks: False if regard symlink as file, else True
|
|
375
|
-
:returns: True if the path exists, else False
|
|
376
|
-
|
|
377
|
-
"""
|
|
378
|
-
return Sftp2Path(path).exists(followlinks)
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
def sftp2_getmtime(path: PathLike, follow_symlinks: bool = False) -> float:
|
|
382
|
-
"""
|
|
383
|
-
Get last-modified time of the file on the given path (in Unix timestamp format).
|
|
384
|
-
|
|
385
|
-
If the path is an existent directory,
|
|
386
|
-
return the latest modified time of all file in it.
|
|
387
|
-
|
|
388
|
-
:param path: Given path
|
|
389
|
-
:returns: last-modified time
|
|
390
|
-
"""
|
|
391
|
-
return Sftp2Path(path).getmtime(follow_symlinks)
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
def sftp2_getsize(path: PathLike, follow_symlinks: bool = False) -> int:
|
|
395
|
-
"""
|
|
396
|
-
Get file size on the given file path (in bytes).
|
|
397
|
-
|
|
398
|
-
If the path in a directory, return the sum of all file size in it,
|
|
399
|
-
including file in subdirectories (if exist).
|
|
400
|
-
|
|
401
|
-
The result excludes the size of directory itself. In other words,
|
|
402
|
-
return 0 Byte on an empty directory path.
|
|
403
|
-
|
|
404
|
-
:param path: Given path
|
|
405
|
-
:returns: File size
|
|
406
|
-
|
|
407
|
-
"""
|
|
408
|
-
return Sftp2Path(path).getsize(follow_symlinks)
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
def sftp2_isdir(path: PathLike, followlinks: bool = False) -> bool:
|
|
412
|
-
"""
|
|
413
|
-
Test if a path is directory
|
|
414
|
-
|
|
415
|
-
.. note::
|
|
416
|
-
|
|
417
|
-
The difference between this function and ``os.path.isdir`` is that
|
|
418
|
-
this function regard symlink as file
|
|
419
|
-
|
|
420
|
-
:param path: Given path
|
|
421
|
-
:param followlinks: False if regard symlink as file, else True
|
|
422
|
-
:returns: True if the path is a directory, else False
|
|
423
|
-
|
|
424
|
-
"""
|
|
425
|
-
return Sftp2Path(path).is_dir(followlinks)
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
def sftp2_isfile(path: PathLike, followlinks: bool = False) -> bool:
|
|
429
|
-
"""
|
|
430
|
-
Test if a path is file
|
|
431
|
-
|
|
432
|
-
.. note::
|
|
433
|
-
|
|
434
|
-
The difference between this function and ``os.path.isfile`` is that
|
|
435
|
-
this function regard symlink as file
|
|
436
|
-
|
|
437
|
-
:param path: Given path
|
|
438
|
-
:param followlinks: False if regard symlink as file, else True
|
|
439
|
-
:returns: True if the path is a file, else False
|
|
440
|
-
|
|
441
|
-
"""
|
|
442
|
-
return Sftp2Path(path).is_file(followlinks)
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
def sftp2_listdir(path: PathLike) -> List[str]:
|
|
446
|
-
"""
|
|
447
|
-
Get all contents of given sftp2 path.
|
|
448
|
-
The result is in ascending alphabetical order.
|
|
449
|
-
|
|
450
|
-
:param path: Given path
|
|
451
|
-
:returns: All contents have in the path in ascending alphabetical order
|
|
452
|
-
"""
|
|
453
|
-
return Sftp2Path(path).listdir()
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
def sftp2_load_from(path: PathLike) -> BinaryIO:
|
|
457
|
-
"""Read all content on specified path and write into memory
|
|
458
|
-
|
|
459
|
-
User should close the BinaryIO manually
|
|
460
|
-
|
|
461
|
-
:param path: Given path
|
|
462
|
-
:returns: Binary stream
|
|
463
|
-
"""
|
|
464
|
-
return Sftp2Path(path).load()
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
def sftp2_makedirs(
|
|
468
|
-
path: PathLike, mode=0o777, parents: bool = False, exist_ok: bool = False
|
|
469
|
-
):
|
|
470
|
-
"""
|
|
471
|
-
make a directory on sftp2, including parent directory.
|
|
472
|
-
If there exists a file on the path, raise FileExistsError
|
|
473
|
-
|
|
474
|
-
:param path: Given path
|
|
475
|
-
:param mode: If mode is given, it is combined with the process' umask value to
|
|
476
|
-
determine the file mode and access flags.
|
|
477
|
-
:param parents: If parents is true, any missing parents of this path
|
|
478
|
-
are created as needed; If parents is false (the default),
|
|
479
|
-
a missing parent raises FileNotFoundError.
|
|
480
|
-
:param exist_ok: If False and target directory exists, raise FileExistsError
|
|
481
|
-
|
|
482
|
-
:raises: FileExistsError
|
|
483
|
-
"""
|
|
484
|
-
return Sftp2Path(path).mkdir(mode, parents, exist_ok)
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
def sftp2_realpath(path: PathLike) -> str:
|
|
488
|
-
"""Return the real path of given path
|
|
489
|
-
|
|
490
|
-
:param path: Given path
|
|
491
|
-
:returns: Real path of given path
|
|
492
|
-
"""
|
|
493
|
-
return Sftp2Path(path).realpath()
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
def sftp2_rename(
|
|
497
|
-
src_path: PathLike, dst_path: PathLike, overwrite: bool = True
|
|
498
|
-
) -> "Sftp2Path":
|
|
499
|
-
"""
|
|
500
|
-
rename file on sftp2
|
|
501
|
-
|
|
502
|
-
:param src_path: Given path
|
|
503
|
-
:param dst_path: Given destination path
|
|
504
|
-
:param overwrite: whether or not overwrite file when exists
|
|
505
|
-
"""
|
|
506
|
-
return Sftp2Path(src_path).rename(dst_path, overwrite)
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
def sftp2_move(
|
|
510
|
-
src_path: PathLike, dst_path: PathLike, overwrite: bool = True
|
|
511
|
-
) -> "Sftp2Path":
|
|
512
|
-
"""
|
|
513
|
-
move file on sftp2
|
|
514
|
-
|
|
515
|
-
:param src_path: Given path
|
|
516
|
-
:param dst_path: Given destination path
|
|
517
|
-
:param overwrite: whether or not overwrite file when exists
|
|
518
|
-
"""
|
|
519
|
-
return Sftp2Path(src_path).replace(dst_path, overwrite)
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
def sftp2_remove(path: PathLike, missing_ok: bool = False) -> None:
|
|
523
|
-
"""
|
|
524
|
-
Remove the file or directory on sftp2
|
|
525
|
-
|
|
526
|
-
:param path: Given path
|
|
527
|
-
:param missing_ok: if False and target file/directory not exists,
|
|
528
|
-
raise FileNotFoundError
|
|
529
|
-
"""
|
|
530
|
-
return Sftp2Path(path).remove(missing_ok)
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
def sftp2_scan(
|
|
534
|
-
path: PathLike, missing_ok: bool = True, followlinks: bool = False
|
|
535
|
-
) -> Iterator[str]:
|
|
536
|
-
"""
|
|
537
|
-
Iteratively traverse only files in given directory, in alphabetical order.
|
|
538
|
-
Every iteration on generator yields a path string.
|
|
539
|
-
|
|
540
|
-
If path is a file path, yields the file only
|
|
541
|
-
If path is a non-existent path, return an empty generator
|
|
542
|
-
If path is a bucket path, return all file paths in the bucket
|
|
543
|
-
|
|
544
|
-
:param path: Given path
|
|
545
|
-
:param missing_ok: If False and there's no file in the directory,
|
|
546
|
-
raise FileNotFoundError
|
|
547
|
-
:returns: A file path generator
|
|
548
|
-
"""
|
|
549
|
-
return Sftp2Path(path).scan(missing_ok, followlinks)
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
def sftp2_scan_stat(
|
|
553
|
-
path: PathLike, missing_ok: bool = True, followlinks: bool = False
|
|
554
|
-
) -> Iterator[FileEntry]:
|
|
555
|
-
"""
|
|
556
|
-
Iteratively traverse only files in given directory, in alphabetical order.
|
|
557
|
-
Every iteration on generator yields a tuple of path string and file stat
|
|
558
|
-
|
|
559
|
-
:param path: Given path
|
|
560
|
-
:param missing_ok: If False and there's no file in the directory,
|
|
561
|
-
raise FileNotFoundError
|
|
562
|
-
:returns: A file path generator
|
|
563
|
-
"""
|
|
564
|
-
return Sftp2Path(path).scan_stat(missing_ok, followlinks)
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
def sftp2_scandir(path: PathLike) -> Iterator[FileEntry]:
|
|
568
|
-
"""
|
|
569
|
-
Get all content of given file path.
|
|
570
|
-
|
|
571
|
-
:param path: Given path
|
|
572
|
-
:returns: An iterator contains all contents have prefix path
|
|
573
|
-
"""
|
|
574
|
-
return Sftp2Path(path).scandir()
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
def sftp2_stat(path: PathLike, follow_symlinks=True) -> StatResult:
|
|
578
|
-
"""
|
|
579
|
-
Get StatResult of file on sftp2, including file size and mtime,
|
|
580
|
-
referring to fs_getsize and fs_getmtime
|
|
581
|
-
|
|
582
|
-
:param path: Given path
|
|
583
|
-
:returns: StatResult
|
|
584
|
-
"""
|
|
585
|
-
return Sftp2Path(path).stat(follow_symlinks)
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
def sftp2_unlink(path: PathLike, missing_ok: bool = False) -> None:
|
|
589
|
-
"""
|
|
590
|
-
Remove the file on sftp2
|
|
591
|
-
|
|
592
|
-
:param path: Given path
|
|
593
|
-
:param missing_ok: if False and target file not exists, raise FileNotFoundError
|
|
594
|
-
"""
|
|
595
|
-
return Sftp2Path(path).unlink(missing_ok)
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
def sftp2_walk(
|
|
599
|
-
path: PathLike, followlinks: bool = False
|
|
600
|
-
) -> Iterator[Tuple[str, List[str], List[str]]]:
|
|
601
|
-
"""
|
|
602
|
-
Generate the file names in a directory tree by walking the tree top-down.
|
|
603
|
-
For each directory in the tree rooted at directory path (including path itself),
|
|
604
|
-
it yields a 3-tuple (root, dirs, files).
|
|
605
|
-
|
|
606
|
-
- root: a string of current path
|
|
607
|
-
- dirs: name list of subdirectories (excluding '.' and '..' if they exist)
|
|
608
|
-
in 'root'. The list is sorted by ascending alphabetical order
|
|
609
|
-
- files: name list of non-directory files (link is regarded as file) in 'root'.
|
|
610
|
-
The list is sorted by ascending alphabetical order
|
|
611
|
-
|
|
612
|
-
If path not exists, or path is a file (link is regarded as file),
|
|
613
|
-
return an empty generator
|
|
614
|
-
|
|
615
|
-
.. note::
|
|
616
|
-
|
|
617
|
-
Be aware that setting ``followlinks`` to True can lead to infinite recursion
|
|
618
|
-
if a link points to a parent directory of itself. fs_walk() does not keep
|
|
619
|
-
track of the directories it visited already.
|
|
620
|
-
|
|
621
|
-
:param path: Given path
|
|
622
|
-
:param followlinks: False if regard symlink as file, else True
|
|
623
|
-
:returns: A 3-tuple generator
|
|
624
|
-
"""
|
|
625
|
-
return Sftp2Path(path).walk(followlinks)
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
def sftp2_getmd5(path: PathLike, recalculate: bool = False, followlinks: bool = False):
|
|
629
|
-
"""
|
|
630
|
-
Calculate the md5 value of the file
|
|
631
|
-
|
|
632
|
-
:param path: Given path
|
|
633
|
-
:param recalculate: Ignore this parameter, just for compatibility
|
|
634
|
-
:param followlinks: Ignore this parameter, just for compatibility
|
|
635
|
-
|
|
636
|
-
returns: md5 of file
|
|
637
|
-
"""
|
|
638
|
-
return Sftp2Path(path).md5(recalculate, followlinks)
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
def sftp2_symlink(src_path: PathLike, dst_path: PathLike) -> None:
|
|
642
|
-
"""
|
|
643
|
-
Create a symbolic link pointing to src_path named dst_path.
|
|
644
|
-
|
|
645
|
-
:param src_path: Given path
|
|
646
|
-
:param dst_path: Destination path
|
|
647
|
-
"""
|
|
648
|
-
return Sftp2Path(src_path).symlink(dst_path)
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
def sftp2_islink(path: PathLike) -> bool:
|
|
652
|
-
"""Test whether a path is a symbolic link
|
|
653
|
-
|
|
654
|
-
:param path: Given path
|
|
655
|
-
:return: If path is a symbolic link return True, else False
|
|
656
|
-
:rtype: bool
|
|
657
|
-
"""
|
|
658
|
-
return Sftp2Path(path).is_symlink()
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
def sftp2_save_as(file_object: BinaryIO, path: PathLike):
|
|
662
|
-
"""Write the opened binary stream to path
|
|
663
|
-
If parent directory of path doesn't exist, it will be created.
|
|
664
|
-
|
|
665
|
-
:param path: Given path
|
|
666
|
-
:param file_object: stream to be read
|
|
667
|
-
"""
|
|
668
|
-
return Sftp2Path(path).save(file_object)
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
def sftp2_open(
|
|
672
|
-
path: PathLike,
|
|
673
|
-
mode: str = "r",
|
|
674
|
-
*,
|
|
675
|
-
buffering=-1,
|
|
676
|
-
encoding: Optional[str] = None,
|
|
677
|
-
errors: Optional[str] = None,
|
|
678
|
-
**kwargs,
|
|
679
|
-
) -> IO:
|
|
680
|
-
"""Open a file on the path.
|
|
681
|
-
|
|
682
|
-
:param path: Given path
|
|
683
|
-
:param mode: Mode to open file
|
|
684
|
-
:param buffering: buffering is an optional integer used to
|
|
685
|
-
set the buffering policy.
|
|
686
|
-
:param encoding: encoding is the name of the encoding used to decode or encode
|
|
687
|
-
the file. This should only be used in text mode.
|
|
688
|
-
:param errors: errors is an optional string that specifies how encoding and
|
|
689
|
-
decoding errors are to be handled—this cannot be used in binary mode.
|
|
690
|
-
:returns: File-Like object
|
|
691
|
-
"""
|
|
692
|
-
return Sftp2Path(path).open(
|
|
693
|
-
mode, buffering=buffering, encoding=encoding, errors=errors
|
|
694
|
-
)
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
def sftp2_chmod(path: PathLike, mode: int, *, follow_symlinks: bool = True):
|
|
698
|
-
"""
|
|
699
|
-
Change the file mode and permissions, like os.chmod().
|
|
700
|
-
|
|
701
|
-
:param path: Given path
|
|
702
|
-
:param mode: the file mode you want to change
|
|
703
|
-
:param followlinks: Ignore this parameter, just for compatibility
|
|
704
|
-
"""
|
|
705
|
-
return Sftp2Path(path).chmod(mode, follow_symlinks=follow_symlinks)
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
def sftp2_absolute(path: PathLike) -> "Sftp2Path":
|
|
709
|
-
"""
|
|
710
|
-
Make the path absolute, without normalization or resolving symlinks.
|
|
711
|
-
Returns a new path object
|
|
712
|
-
"""
|
|
713
|
-
return Sftp2Path(path).absolute()
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
def sftp2_rmdir(path: PathLike):
|
|
717
|
-
"""
|
|
718
|
-
Remove this directory. The directory must be empty.
|
|
719
|
-
"""
|
|
720
|
-
return Sftp2Path(path).rmdir()
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
def sftp2_copy(
|
|
724
|
-
src_path: PathLike,
|
|
725
|
-
dst_path: PathLike,
|
|
726
|
-
callback: Optional[Callable[[int], None]] = None,
|
|
727
|
-
followlinks: bool = False,
|
|
728
|
-
overwrite: bool = True,
|
|
729
|
-
):
|
|
730
|
-
"""
|
|
731
|
-
Copy the file to the given destination path.
|
|
732
|
-
|
|
733
|
-
:param src_path: Given path
|
|
734
|
-
:param dst_path: The destination path to copy the file to.
|
|
735
|
-
:param callback: An optional callback function that takes an integer parameter
|
|
736
|
-
and is called periodically during the copy operation to report the number
|
|
737
|
-
of bytes copied.
|
|
738
|
-
:param followlinks: Whether to follow symbolic links when copying directories.
|
|
739
|
-
:raises IsADirectoryError: If the source is a directory.
|
|
740
|
-
:raises OSError: If there is an error copying the file.
|
|
741
|
-
"""
|
|
742
|
-
return Sftp2Path(src_path).copy(dst_path, callback, followlinks, overwrite)
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
def sftp2_sync(
|
|
746
|
-
src_path: PathLike,
|
|
747
|
-
dst_path: PathLike,
|
|
748
|
-
followlinks: bool = False,
|
|
749
|
-
force: bool = False,
|
|
750
|
-
overwrite: bool = True,
|
|
751
|
-
):
|
|
752
|
-
"""Copy file/directory on src_url to dst_url
|
|
753
|
-
|
|
754
|
-
:param src_path: Given path
|
|
755
|
-
:param dst_url: Given destination path
|
|
756
|
-
:param followlinks: False if regard symlink as file, else True
|
|
757
|
-
:param force: Sync file forcible, do not ignore same files,
|
|
758
|
-
priority is higher than 'overwrite', default is False
|
|
759
|
-
:param overwrite: whether or not overwrite file when exists, default is True
|
|
760
|
-
"""
|
|
761
|
-
return Sftp2Path(src_path).sync(dst_path, followlinks, force, overwrite)
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
def _check_input(input_str: str, fingerprint: str, times: int = 0) -> bool:
|
|
765
|
-
answers = input_str.strip()
|
|
766
|
-
if answers.lower() in ("yes", "y") or answers == fingerprint:
|
|
767
|
-
return True
|
|
768
|
-
elif answers.lower() in ("no", "n"):
|
|
769
|
-
return False
|
|
770
|
-
elif times >= 10:
|
|
771
|
-
_logger.warning("Retried more than 10 times, give up")
|
|
772
|
-
return False
|
|
773
|
-
else:
|
|
774
|
-
input_str = input("Please type 'yes', 'no' or the fingerprint: ")
|
|
775
|
-
return _check_input(input_str, fingerprint, times=times + 1)
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
def _prompt_add_to_known_hosts(hostname, key) -> bool:
|
|
779
|
-
fingerprint = hashlib.sha256(key).digest()
|
|
780
|
-
fingerprint = f"SHA256:{base64.b64encode(fingerprint).decode('utf-8')}"
|
|
781
|
-
answers = input(
|
|
782
|
-
f"""The authenticity of host '{hostname}' can't be established.
|
|
783
|
-
SSH key fingerprint is {fingerprint}.
|
|
784
|
-
This key is not known by any other names.
|
|
785
|
-
Are you sure you want to continue connecting (yes/no/[fingerprint])? """
|
|
786
|
-
)
|
|
787
|
-
return _check_input(answers, fingerprint)
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
def sftp2_add_host_key(
|
|
791
|
-
hostname: str,
|
|
792
|
-
port: int = 22,
|
|
793
|
-
prompt: bool = False,
|
|
794
|
-
host_key_path: Optional["str"] = None,
|
|
795
|
-
):
|
|
796
|
-
"""Add a host key to known_hosts.
|
|
797
|
-
|
|
798
|
-
:param hostname: hostname
|
|
799
|
-
:param port: port, default is 22
|
|
800
|
-
:param prompt: If True, requires user input of 'yes' or 'no' to decide whether to
|
|
801
|
-
add this host key
|
|
802
|
-
:param host_key_path: path of known_hosts, default is ~/.ssh/known_hosts
|
|
803
|
-
"""
|
|
804
|
-
if not host_key_path:
|
|
805
|
-
host_key_path = os.path.expanduser("~/.ssh/known_hosts")
|
|
806
|
-
|
|
807
|
-
if not os.path.exists(host_key_path):
|
|
808
|
-
dirname = os.path.dirname(host_key_path)
|
|
809
|
-
if dirname and dirname != ".":
|
|
810
|
-
os.makedirs(dirname, exist_ok=True, mode=0o700)
|
|
811
|
-
with open(host_key_path, "w"):
|
|
812
|
-
pass
|
|
813
|
-
os.chmod(host_key_path, 0o600)
|
|
814
|
-
|
|
815
|
-
sock = ssh2.utils.Socket.open(hostname, port)
|
|
816
|
-
session = ssh2.session.Session()
|
|
817
|
-
session.handshake(sock)
|
|
818
|
-
|
|
819
|
-
hostkey = session.hostkey()
|
|
820
|
-
|
|
821
|
-
if prompt:
|
|
822
|
-
result = _prompt_add_to_known_hosts(hostname, hostkey)
|
|
823
|
-
if not result:
|
|
824
|
-
return
|
|
825
|
-
|
|
826
|
-
with open(host_key_path, "a") as f:
|
|
827
|
-
f.write(f"{hostname} {base64.b64encode(hostkey).decode('ascii')}\n")
|