megfile 3.1.1__py3-none-any.whl → 3.1.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- docs/conf.py +2 -4
- megfile/__init__.py +394 -203
- megfile/cli.py +258 -238
- megfile/config.py +25 -21
- megfile/errors.py +124 -114
- megfile/fs.py +174 -140
- megfile/fs_path.py +462 -354
- megfile/hdfs.py +133 -101
- megfile/hdfs_path.py +290 -236
- megfile/http.py +15 -14
- megfile/http_path.py +111 -107
- megfile/interfaces.py +70 -65
- megfile/lib/base_prefetch_reader.py +84 -65
- megfile/lib/combine_reader.py +12 -12
- megfile/lib/compare.py +17 -13
- megfile/lib/compat.py +1 -5
- megfile/lib/fnmatch.py +29 -30
- megfile/lib/glob.py +46 -54
- megfile/lib/hdfs_prefetch_reader.py +40 -25
- megfile/lib/hdfs_tools.py +1 -3
- megfile/lib/http_prefetch_reader.py +69 -46
- megfile/lib/joinpath.py +5 -5
- megfile/lib/lazy_handler.py +7 -3
- megfile/lib/s3_buffered_writer.py +58 -51
- megfile/lib/s3_cached_handler.py +13 -14
- megfile/lib/s3_limited_seekable_writer.py +37 -28
- megfile/lib/s3_memory_handler.py +34 -30
- megfile/lib/s3_pipe_handler.py +24 -25
- megfile/lib/s3_prefetch_reader.py +71 -52
- megfile/lib/s3_share_cache_reader.py +37 -24
- megfile/lib/shadow_handler.py +7 -3
- megfile/lib/stdio_handler.py +9 -8
- megfile/lib/url.py +3 -3
- megfile/pathlike.py +259 -228
- megfile/s3.py +220 -153
- megfile/s3_path.py +977 -802
- megfile/sftp.py +190 -156
- megfile/sftp_path.py +540 -450
- megfile/smart.py +397 -330
- megfile/smart_path.py +100 -105
- megfile/stdio.py +10 -9
- megfile/stdio_path.py +32 -35
- megfile/utils/__init__.py +73 -54
- megfile/utils/mutex.py +11 -14
- megfile/version.py +1 -1
- {megfile-3.1.1.dist-info → megfile-3.1.2.dist-info}/METADATA +5 -8
- megfile-3.1.2.dist-info/RECORD +55 -0
- {megfile-3.1.1.dist-info → megfile-3.1.2.dist-info}/WHEEL +1 -1
- scripts/convert_results_to_sarif.py +45 -78
- scripts/generate_file.py +140 -64
- megfile-3.1.1.dist-info/RECORD +0 -55
- {megfile-3.1.1.dist-info → megfile-3.1.2.dist-info}/LICENSE +0 -0
- {megfile-3.1.1.dist-info → megfile-3.1.2.dist-info}/LICENSE.pyre +0 -0
- {megfile-3.1.1.dist-info → megfile-3.1.2.dist-info}/entry_points.txt +0 -0
- {megfile-3.1.1.dist-info → megfile-3.1.2.dist-info}/top_level.txt +0 -0
megfile/fs_path.py
CHANGED
|
@@ -9,34 +9,38 @@ from stat import S_ISLNK as stat_islnk
|
|
|
9
9
|
from typing import IO, BinaryIO, Callable, Iterator, List, Optional, Tuple, Union
|
|
10
10
|
|
|
11
11
|
from megfile.errors import _create_missing_ok_generator
|
|
12
|
-
from megfile.interfaces import
|
|
12
|
+
from megfile.interfaces import (
|
|
13
|
+
Access,
|
|
14
|
+
ContextIterator,
|
|
15
|
+
FileEntry,
|
|
16
|
+
PathLike,
|
|
17
|
+
StatResult,
|
|
18
|
+
URIPath,
|
|
19
|
+
)
|
|
13
20
|
from megfile.lib.compare import is_same_file
|
|
21
|
+
from megfile.lib.compat import fspath
|
|
14
22
|
from megfile.lib.glob import iglob
|
|
23
|
+
from megfile.lib.joinpath import path_join
|
|
15
24
|
from megfile.lib.url import get_url_scheme
|
|
25
|
+
from megfile.smart_path import SmartPath
|
|
16
26
|
from megfile.utils import calculate_md5
|
|
17
27
|
|
|
18
|
-
from .interfaces import PathLike, URIPath
|
|
19
|
-
from .lib.compat import fspath
|
|
20
|
-
from .lib.joinpath import path_join
|
|
21
|
-
from .smart_path import SmartPath
|
|
22
|
-
|
|
23
28
|
__all__ = [
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
'fs_lstat',
|
|
29
|
+
"FSPath",
|
|
30
|
+
"is_fs",
|
|
31
|
+
"fs_path_join",
|
|
32
|
+
"_make_stat",
|
|
33
|
+
"fs_readlink",
|
|
34
|
+
"fs_cwd",
|
|
35
|
+
"fs_home",
|
|
36
|
+
"fs_iglob",
|
|
37
|
+
"fs_glob",
|
|
38
|
+
"fs_glob_stat",
|
|
39
|
+
"fs_rename",
|
|
40
|
+
"fs_resolve",
|
|
41
|
+
"fs_move",
|
|
42
|
+
"fs_makedirs",
|
|
43
|
+
"fs_lstat",
|
|
40
44
|
]
|
|
41
45
|
|
|
42
46
|
|
|
@@ -52,16 +56,16 @@ def _make_stat(stat: os.stat_result) -> StatResult:
|
|
|
52
56
|
|
|
53
57
|
|
|
54
58
|
def is_fs(path: Union["PathLike", int]) -> bool:
|
|
55
|
-
|
|
59
|
+
"""Test if a path is fs path
|
|
56
60
|
|
|
57
61
|
:param path: Path to be tested
|
|
58
62
|
:returns: True of a path is fs path, else False
|
|
59
|
-
|
|
63
|
+
"""
|
|
60
64
|
if isinstance(path, int):
|
|
61
65
|
return True
|
|
62
66
|
path = fspath(path)
|
|
63
67
|
scheme = get_url_scheme(path)
|
|
64
|
-
return scheme ==
|
|
68
|
+
return scheme == "" or scheme == "file"
|
|
65
69
|
|
|
66
70
|
|
|
67
71
|
def fs_path_join(path: PathLike, *other_paths: PathLike) -> str:
|
|
@@ -69,107 +73,130 @@ def fs_path_join(path: PathLike, *other_paths: PathLike) -> str:
|
|
|
69
73
|
|
|
70
74
|
|
|
71
75
|
def fs_readlink(path) -> str:
|
|
72
|
-
|
|
76
|
+
"""
|
|
73
77
|
Return a string representing the path to which the symbolic link points.
|
|
74
78
|
:returns: Return a string representing the path to which the symbolic link points.
|
|
75
|
-
|
|
79
|
+
"""
|
|
76
80
|
return os.readlink(path)
|
|
77
81
|
|
|
78
82
|
|
|
79
83
|
def fs_cwd() -> str:
|
|
80
|
-
|
|
84
|
+
"""Return current working directory
|
|
81
85
|
|
|
82
86
|
returns: Current working directory
|
|
83
|
-
|
|
87
|
+
"""
|
|
84
88
|
return os.getcwd()
|
|
85
89
|
|
|
86
90
|
|
|
87
91
|
def fs_home():
|
|
88
|
-
|
|
92
|
+
"""Return the home directory
|
|
89
93
|
|
|
90
94
|
returns: Home directory path
|
|
91
|
-
|
|
92
|
-
return os.path.expanduser(
|
|
95
|
+
"""
|
|
96
|
+
return os.path.expanduser("~")
|
|
93
97
|
|
|
94
98
|
|
|
95
|
-
def fs_iglob(
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
+
def fs_iglob(
|
|
100
|
+
path: PathLike, recursive: bool = True, missing_ok: bool = True
|
|
101
|
+
) -> Iterator[str]:
|
|
102
|
+
"""Return path iterator in ascending alphabetical order,
|
|
103
|
+
in which path matches glob pattern
|
|
99
104
|
|
|
100
105
|
1. If doesn't match any path, return empty list
|
|
101
|
-
Notice: ``glob.glob`` in standard library returns ['a/'] instead of empty list
|
|
106
|
+
Notice: ``glob.glob`` in standard library returns ['a/'] instead of empty list
|
|
107
|
+
when pathname is like `a/**`, recursive is True and directory 'a' doesn't exist.
|
|
108
|
+
fs_glob behaves like ``glob.glob`` in standard library under such circumstance.
|
|
102
109
|
2. No guarantee that each path in result is different, which means:
|
|
103
110
|
Assume there exists a path `/a/b/c/b/d.txt`
|
|
104
|
-
use path pattern like `/**/b/**/*.txt` to glob,
|
|
105
|
-
|
|
106
|
-
|
|
111
|
+
use path pattern like `/**/b/**/*.txt` to glob,
|
|
112
|
+
the path above will be returned twice
|
|
113
|
+
3. `**` will match any matched file, directory, symlink and '' by default,
|
|
114
|
+
when recursive is `True`
|
|
115
|
+
4. fs_glob returns same as glob.glob(pathname, recursive=True)
|
|
116
|
+
in ascending alphabetical order.
|
|
107
117
|
5. Hidden files (filename stars with '.') will not be found in the result
|
|
108
118
|
|
|
109
119
|
:param recursive: If False, `**` will not search directory recursively
|
|
110
|
-
:param missing_ok: If False and target path doesn't match any file,
|
|
120
|
+
:param missing_ok: If False and target path doesn't match any file,
|
|
121
|
+
raise FileNotFoundError
|
|
111
122
|
:returns: An iterator contains paths match `pathname`
|
|
112
|
-
|
|
123
|
+
"""
|
|
113
124
|
for path in _create_missing_ok_generator(
|
|
114
|
-
|
|
115
|
-
|
|
125
|
+
iglob(fspath(path), recursive=recursive),
|
|
126
|
+
missing_ok,
|
|
127
|
+
FileNotFoundError("No match any file: %r" % path),
|
|
128
|
+
):
|
|
116
129
|
yield path
|
|
117
130
|
|
|
118
131
|
|
|
119
|
-
def fs_glob(
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
132
|
+
def fs_glob(
|
|
133
|
+
path: PathLike, recursive: bool = True, missing_ok: bool = True
|
|
134
|
+
) -> List[str]:
|
|
135
|
+
"""Return path list in ascending alphabetical order,
|
|
136
|
+
in which path matches glob pattern
|
|
123
137
|
|
|
124
138
|
1. If doesn't match any path, return empty list
|
|
125
|
-
Notice: ``glob.glob`` in standard library returns ['a/'] instead of empty list
|
|
139
|
+
Notice: ``glob.glob`` in standard library returns ['a/'] instead of empty list
|
|
140
|
+
when pathname is like `a/**`, recursive is True and directory 'a' doesn't exist.
|
|
141
|
+
fs_glob behaves like ``glob.glob`` in standard library under such circumstance.
|
|
126
142
|
2. No guarantee that each path in result is different, which means:
|
|
127
143
|
Assume there exists a path `/a/b/c/b/d.txt`
|
|
128
|
-
use path pattern like `/**/b/**/*.txt` to glob,
|
|
129
|
-
|
|
130
|
-
|
|
144
|
+
use path pattern like `/**/b/**/*.txt` to glob,
|
|
145
|
+
the path above will be returned twice
|
|
146
|
+
3. `**` will match any matched file, directory, symlink and '' by default,
|
|
147
|
+
when recursive is `True`
|
|
148
|
+
4. fs_glob returns same as glob.glob(pathname, recursive=True)
|
|
149
|
+
in ascending alphabetical order.
|
|
131
150
|
5. Hidden files (filename stars with '.') will not be found in the result
|
|
132
151
|
|
|
133
152
|
:param recursive: If False, `**` will not search directory recursively
|
|
134
|
-
:param missing_ok: If False and target path doesn't match any file,
|
|
153
|
+
:param missing_ok: If False and target path doesn't match any file,
|
|
154
|
+
raise FileNotFoundError
|
|
135
155
|
:returns: A list contains paths match `pathname`
|
|
136
|
-
|
|
156
|
+
"""
|
|
137
157
|
return list(fs_iglob(path=path, recursive=recursive, missing_ok=missing_ok))
|
|
138
158
|
|
|
139
159
|
|
|
140
160
|
def fs_glob_stat(
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
161
|
+
path: PathLike, recursive: bool = True, missing_ok: bool = True
|
|
162
|
+
) -> Iterator[FileEntry]:
|
|
163
|
+
"""Return a list contains tuples of path and file stat,
|
|
164
|
+
in ascending alphabetical order, in which path matches glob pattern
|
|
145
165
|
|
|
146
166
|
1. If doesn't match any path, return empty list
|
|
147
|
-
Notice: ``glob.glob`` in standard library returns ['a/'] instead of empty list
|
|
167
|
+
Notice: ``glob.glob`` in standard library returns ['a/'] instead of empty list
|
|
168
|
+
when pathname is like `a/**`, recursive is True and directory 'a' doesn't exist.
|
|
169
|
+
fs_glob behaves like ``glob.glob`` in standard library under such circumstance.
|
|
148
170
|
2. No guarantee that each path in result is different, which means:
|
|
149
171
|
Assume there exists a path `/a/b/c/b/d.txt`
|
|
150
|
-
use path pattern like `/**/b/**/*.txt` to glob,
|
|
151
|
-
|
|
152
|
-
|
|
172
|
+
use path pattern like `/**/b/**/*.txt` to glob,
|
|
173
|
+
the path above will be returned twice.
|
|
174
|
+
3. `**` will match any matched file, directory, symlink and '' by default,
|
|
175
|
+
when recursive is `True`
|
|
176
|
+
4. fs_glob returns same as glob.glob(pathname, recursive=True)
|
|
177
|
+
in ascending alphabetical order.
|
|
153
178
|
5. Hidden files (filename stars with '.') will not be found in the result
|
|
154
179
|
|
|
155
180
|
:param recursive: If False, `**` will not search directory recursively
|
|
156
|
-
:param missing_ok: If False and target path doesn't match any file,
|
|
157
|
-
|
|
158
|
-
|
|
181
|
+
:param missing_ok: If False and target path doesn't match any file,
|
|
182
|
+
raise FileNotFoundError
|
|
183
|
+
:returns: A list contains tuples of path and file stat,
|
|
184
|
+
in which paths match `pathname`
|
|
185
|
+
"""
|
|
159
186
|
for path in fs_iglob(path=path, recursive=recursive, missing_ok=missing_ok):
|
|
160
|
-
yield FileEntry(
|
|
161
|
-
os.path.basename(path), path, _make_stat(os.lstat(path)))
|
|
187
|
+
yield FileEntry(os.path.basename(path), path, _make_stat(os.lstat(path)))
|
|
162
188
|
|
|
163
189
|
|
|
164
190
|
def _fs_rename_file(
|
|
165
|
-
|
|
166
|
-
|
|
191
|
+
src_path: PathLike, dst_path: PathLike, overwrite: bool = True
|
|
192
|
+
) -> None:
|
|
193
|
+
"""
|
|
167
194
|
rename file on fs
|
|
168
195
|
|
|
169
196
|
:param src_path: Given path
|
|
170
197
|
:param dst_path: Given destination path
|
|
171
198
|
:param overwrite: whether or not overwrite file when exists
|
|
172
|
-
|
|
199
|
+
"""
|
|
173
200
|
src_path, dst_path = fspath(src_path), fspath(dst_path)
|
|
174
201
|
|
|
175
202
|
if not overwrite and os.path.exists(dst_path):
|
|
@@ -181,15 +208,14 @@ def _fs_rename_file(
|
|
|
181
208
|
shutil.move(src_path, dst_path)
|
|
182
209
|
|
|
183
210
|
|
|
184
|
-
def fs_rename(
|
|
185
|
-
|
|
186
|
-
'''
|
|
211
|
+
def fs_rename(src_path: PathLike, dst_path: PathLike, overwrite: bool = True) -> None:
|
|
212
|
+
"""
|
|
187
213
|
rename file on fs
|
|
188
214
|
|
|
189
215
|
:param src_path: Given path
|
|
190
216
|
:param dst_path: Given destination path
|
|
191
217
|
:param overwrite: whether or not overwrite file when exists
|
|
192
|
-
|
|
218
|
+
"""
|
|
193
219
|
src_path, dst_path = fspath(src_path), fspath(dst_path)
|
|
194
220
|
if os.path.isfile(src_path):
|
|
195
221
|
return _fs_rename_file(src_path, dst_path, overwrite)
|
|
@@ -201,7 +227,7 @@ def fs_rename(
|
|
|
201
227
|
src_file_path = file_entry.path
|
|
202
228
|
dst_file_path = dst_path
|
|
203
229
|
relative_path = os.path.relpath(src_file_path, start=src_path)
|
|
204
|
-
if relative_path and relative_path !=
|
|
230
|
+
if relative_path and relative_path != ".":
|
|
205
231
|
dst_file_path = os.path.join(dst_file_path, relative_path)
|
|
206
232
|
if os.path.exists(dst_file_path) and file_entry.is_dir():
|
|
207
233
|
fs_rename(src_file_path, dst_file_path, overwrite)
|
|
@@ -214,29 +240,28 @@ def fs_rename(
|
|
|
214
240
|
os.remove(src_path)
|
|
215
241
|
|
|
216
242
|
|
|
217
|
-
def fs_move(
|
|
218
|
-
|
|
219
|
-
'''
|
|
243
|
+
def fs_move(src_path: PathLike, dst_path: PathLike, overwrite: bool = True) -> None:
|
|
244
|
+
"""
|
|
220
245
|
rename file on fs
|
|
221
246
|
|
|
222
247
|
:param src_path: Given path
|
|
223
248
|
:param dst_path: Given destination path
|
|
224
249
|
:param overwrite: whether or not overwrite file when exists
|
|
225
|
-
|
|
250
|
+
"""
|
|
226
251
|
return fs_rename(src_path, dst_path, overwrite)
|
|
227
252
|
|
|
228
253
|
|
|
229
254
|
def fs_resolve(path: PathLike) -> str:
|
|
230
|
-
|
|
255
|
+
"""Equal to fs_realpath, return the real path of given path
|
|
231
256
|
|
|
232
257
|
:param path: Given path
|
|
233
258
|
:returns: Real path of given path
|
|
234
|
-
|
|
259
|
+
"""
|
|
235
260
|
return FSPath(path).realpath()
|
|
236
261
|
|
|
237
262
|
|
|
238
263
|
def fs_makedirs(path: PathLike, exist_ok: bool = False):
|
|
239
|
-
|
|
264
|
+
"""
|
|
240
265
|
make a directory on fs, including parent directory
|
|
241
266
|
|
|
242
267
|
If there exists a file on the path, raise FileExistsError
|
|
@@ -244,17 +269,18 @@ def fs_makedirs(path: PathLike, exist_ok: bool = False):
|
|
|
244
269
|
:param path: Given path
|
|
245
270
|
:param exist_ok: If False and target directory exists, raise FileExistsError
|
|
246
271
|
:raises: FileExistsError
|
|
247
|
-
|
|
272
|
+
"""
|
|
248
273
|
return FSPath(path).mkdir(parents=True, exist_ok=exist_ok)
|
|
249
274
|
|
|
250
275
|
|
|
251
276
|
def fs_lstat(path: PathLike) -> StatResult:
|
|
252
|
-
|
|
253
|
-
Like Path.stat() but, if the path points to a symbolic link,
|
|
277
|
+
"""
|
|
278
|
+
Like Path.stat() but, if the path points to a symbolic link,
|
|
279
|
+
return the symbolic link’s information rather than its target’s.
|
|
254
280
|
|
|
255
281
|
:param path: Given path
|
|
256
282
|
:returns: StatResult
|
|
257
|
-
|
|
283
|
+
"""
|
|
258
284
|
return FSPath(path).lstat()
|
|
259
285
|
|
|
260
286
|
|
|
@@ -302,275 +328,318 @@ class FSPath(URIPath):
|
|
|
302
328
|
return protocol_prefix + self.path # pyre-ignore[58]
|
|
303
329
|
|
|
304
330
|
def is_absolute(self) -> bool:
|
|
305
|
-
|
|
331
|
+
"""Test whether a path is absolute
|
|
306
332
|
|
|
307
333
|
:returns: True if a path is absolute, else False
|
|
308
|
-
|
|
334
|
+
"""
|
|
309
335
|
return os.path.isabs(self.path_without_protocol)
|
|
310
336
|
|
|
311
337
|
def abspath(self) -> str:
|
|
312
|
-
|
|
338
|
+
"""Return the absolute path of given path
|
|
313
339
|
|
|
314
340
|
:returns: Absolute path of given path
|
|
315
|
-
|
|
341
|
+
"""
|
|
316
342
|
return fspath(os.path.abspath(self.path_without_protocol))
|
|
317
343
|
|
|
318
344
|
def access(self, mode: Access = Access.READ) -> bool:
|
|
319
|
-
|
|
345
|
+
"""
|
|
320
346
|
Test if path has access permission described by mode
|
|
321
347
|
Using ``os.access``
|
|
322
348
|
|
|
323
349
|
:param mode: access mode
|
|
324
350
|
:returns: Access: Enum, the read/write access that path has.
|
|
325
|
-
|
|
351
|
+
"""
|
|
326
352
|
if mode == Access.READ:
|
|
327
353
|
return os.access(self.path_without_protocol, os.R_OK)
|
|
328
354
|
elif mode == Access.WRITE:
|
|
329
355
|
return os.access(self.path_without_protocol, os.W_OK)
|
|
330
356
|
else:
|
|
331
357
|
raise TypeError(
|
|
332
|
-
|
|
333
|
-
|
|
358
|
+
"Unsupported mode: {} -- Mode should use one of "
|
|
359
|
+
"the enums belonging to: {}".format(
|
|
360
|
+
mode, ", ".join([str(a) for a in Access])
|
|
361
|
+
)
|
|
362
|
+
)
|
|
334
363
|
|
|
335
364
|
def exists(self, followlinks: bool = False) -> bool:
|
|
336
|
-
|
|
365
|
+
"""
|
|
337
366
|
Test if the path exists
|
|
338
367
|
|
|
339
368
|
.. note::
|
|
340
369
|
|
|
341
|
-
The difference between this function and ``os.path.exists`` is that
|
|
370
|
+
The difference between this function and ``os.path.exists`` is that
|
|
371
|
+
this function regard symlink as file.
|
|
342
372
|
In other words, this function is equal to ``os.path.lexists``
|
|
343
373
|
|
|
344
374
|
:param followlinks: False if regard symlink as file, else True
|
|
345
375
|
:returns: True if the path exists, else False
|
|
346
376
|
|
|
347
|
-
|
|
377
|
+
"""
|
|
348
378
|
if followlinks:
|
|
349
379
|
return os.path.exists(self.path_without_protocol)
|
|
350
380
|
return os.path.lexists(self.path_without_protocol)
|
|
351
381
|
|
|
352
382
|
def getmtime(self, follow_symlinks: bool = False) -> float:
|
|
353
|
-
|
|
383
|
+
"""
|
|
354
384
|
Get last-modified time of the file on the given path (in Unix timestamp format).
|
|
355
|
-
If the path is an existent directory,
|
|
385
|
+
If the path is an existent directory,
|
|
386
|
+
return the latest modified time of all file in it.
|
|
356
387
|
|
|
357
388
|
:returns: last-modified time
|
|
358
|
-
|
|
389
|
+
"""
|
|
359
390
|
return self.stat(follow_symlinks=follow_symlinks).mtime
|
|
360
391
|
|
|
361
392
|
def getsize(self, follow_symlinks: bool = False) -> int:
|
|
362
|
-
|
|
393
|
+
"""
|
|
363
394
|
Get file size on the given file path (in bytes).
|
|
364
|
-
If the path in a directory, return the sum of all file size in it,
|
|
365
|
-
|
|
395
|
+
If the path in a directory, return the sum of all file size in it,
|
|
396
|
+
including file in subdirectories (if exist).
|
|
397
|
+
The result excludes the size of directory itself.
|
|
398
|
+
In other words, return 0 Byte on an empty directory path.
|
|
366
399
|
|
|
367
400
|
:returns: File size
|
|
368
401
|
|
|
369
|
-
|
|
402
|
+
"""
|
|
370
403
|
return self.stat(follow_symlinks=follow_symlinks).size
|
|
371
404
|
|
|
372
|
-
def glob(
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
405
|
+
def glob(
|
|
406
|
+
self, pattern, recursive: bool = True, missing_ok: bool = True
|
|
407
|
+
) -> List["FSPath"]:
|
|
408
|
+
"""Return path list in ascending alphabetical order,
|
|
409
|
+
in which path matches glob pattern
|
|
377
410
|
|
|
378
411
|
1. If doesn't match any path, return empty list
|
|
379
|
-
Notice: ``glob.glob`` in standard library returns ['a/'] instead of
|
|
412
|
+
Notice: ``glob.glob`` in standard library returns ['a/'] instead of
|
|
413
|
+
empty list when pathname is like `a/**`,
|
|
414
|
+
recursive is True and directory 'a' doesn't exist.
|
|
415
|
+
fs_glob behaves like ``glob.glob`` in standard library
|
|
416
|
+
under such circumstance.
|
|
380
417
|
2. No guarantee that each path in result is different, which means:
|
|
381
418
|
Assume there exists a path `/a/b/c/b/d.txt`
|
|
382
|
-
use path pattern like `/**/b/**/*.txt` to glob,
|
|
383
|
-
|
|
384
|
-
|
|
419
|
+
use path pattern like `/**/b/**/*.txt` to glob,
|
|
420
|
+
the path above will be returned twice
|
|
421
|
+
3. `**` will match any matched file, directory, symlink and '' by default,
|
|
422
|
+
when recursive is `True`
|
|
423
|
+
4. fs_glob returns same as glob.glob(pathname, recursive=True)
|
|
424
|
+
in ascending alphabetical order.
|
|
385
425
|
5. Hidden files (filename stars with '.') will not be found in the result
|
|
386
426
|
|
|
387
|
-
:param pattern: Glob the given relative pattern in the directory represented
|
|
427
|
+
:param pattern: Glob the given relative pattern in the directory represented
|
|
428
|
+
by this path
|
|
388
429
|
:param recursive: If False, `**` will not search directory recursively
|
|
389
|
-
:param missing_ok: If False and target path doesn't match any file,
|
|
430
|
+
:param missing_ok: If False and target path doesn't match any file,
|
|
431
|
+
raise FileNotFoundError
|
|
390
432
|
:returns: A list contains paths match `pathname`
|
|
391
|
-
|
|
433
|
+
"""
|
|
392
434
|
return list(
|
|
393
|
-
self.iglob(
|
|
394
|
-
|
|
435
|
+
self.iglob(pattern=pattern, recursive=recursive, missing_ok=missing_ok)
|
|
436
|
+
)
|
|
395
437
|
|
|
396
438
|
def glob_stat(
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
'''Return a list contains tuples of path and file stat, in ascending alphabetical order, in which path matches glob pattern
|
|
439
|
+
self, pattern, recursive: bool = True, missing_ok: bool = True
|
|
440
|
+
) -> Iterator[FileEntry]:
|
|
441
|
+
"""Return a list contains tuples of path and file stat,
|
|
442
|
+
in ascending alphabetical order, in which path matches glob pattern
|
|
402
443
|
|
|
403
444
|
1. If doesn't match any path, return empty list
|
|
404
|
-
Notice: ``glob.glob`` in standard library returns ['a/'] instead of
|
|
445
|
+
Notice: ``glob.glob`` in standard library returns ['a/'] instead of
|
|
446
|
+
empty list when pathname is like `a/**`,
|
|
447
|
+
recursive is True and directory 'a' doesn't exist.
|
|
448
|
+
fs_glob behaves like ``glob.glob`` in standard library
|
|
449
|
+
under such circumstance.
|
|
405
450
|
2. No guarantee that each path in result is different, which means:
|
|
406
451
|
Assume there exists a path `/a/b/c/b/d.txt`
|
|
407
|
-
use path pattern like `/**/b/**/*.txt` to glob,
|
|
408
|
-
|
|
409
|
-
|
|
452
|
+
use path pattern like `/**/b/**/*.txt` to glob,
|
|
453
|
+
the path above will be returned twice
|
|
454
|
+
3. `**` will match any matched file, directory, symlink and '' by default,
|
|
455
|
+
when recursive is `True`
|
|
456
|
+
4. fs_glob returns same as glob.glob(pathname, recursive=True)
|
|
457
|
+
in ascending alphabetical order.
|
|
410
458
|
5. Hidden files (filename stars with '.') will not be found in the result
|
|
411
459
|
|
|
412
|
-
:param pattern: Glob the given relative pattern in the directory represented
|
|
460
|
+
:param pattern: Glob the given relative pattern in the directory represented
|
|
461
|
+
by this path
|
|
413
462
|
:param recursive: If False, `**` will not search directory recursively
|
|
414
|
-
:param missing_ok: If False and target path doesn't match any file,
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
463
|
+
:param missing_ok: If False and target path doesn't match any file,
|
|
464
|
+
raise FileNotFoundError
|
|
465
|
+
:returns: A list contains tuples of path and file stat,
|
|
466
|
+
in which paths match `pathname`
|
|
467
|
+
"""
|
|
468
|
+
for path_obj in self.iglob(
|
|
469
|
+
pattern=pattern, recursive=recursive, missing_ok=missing_ok
|
|
470
|
+
):
|
|
419
471
|
yield FileEntry(
|
|
420
472
|
path_obj.name,
|
|
421
473
|
path_obj.path, # pyre-ignore[6]
|
|
422
|
-
_make_stat(os.lstat(path_obj.path))
|
|
474
|
+
_make_stat(os.lstat(path_obj.path)), # pyre-ignore[6]
|
|
475
|
+
)
|
|
423
476
|
|
|
424
477
|
def expanduser(self):
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
478
|
+
"""Expand ~ and ~user constructions. If user or $HOME is unknown,
|
|
479
|
+
do nothing.
|
|
480
|
+
"""
|
|
428
481
|
return os.path.expanduser(self.path_without_protocol)
|
|
429
482
|
|
|
430
|
-
def iglob(
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
483
|
+
def iglob(
|
|
484
|
+
self, pattern, recursive: bool = True, missing_ok: bool = True
|
|
485
|
+
) -> Iterator["FSPath"]:
|
|
486
|
+
"""Return path iterator in ascending alphabetical order,
|
|
487
|
+
in which path matches glob pattern
|
|
435
488
|
|
|
436
489
|
1. If doesn't match any path, return empty list
|
|
437
|
-
Notice: ``glob.glob`` in standard library returns ['a/'] instead of
|
|
490
|
+
Notice: ``glob.glob`` in standard library returns ['a/'] instead of
|
|
491
|
+
empty list when pathname is like `a/**`,
|
|
492
|
+
recursive is True and directory 'a' doesn't exist.
|
|
493
|
+
fs_glob behaves like ``glob.glob`` in standard library
|
|
494
|
+
under such circumstance.
|
|
438
495
|
2. No guarantee that each path in result is different, which means:
|
|
439
496
|
Assume there exists a path `/a/b/c/b/d.txt`
|
|
440
|
-
use path pattern like `/**/b/**/*.txt` to glob,
|
|
441
|
-
|
|
442
|
-
|
|
497
|
+
use path pattern like `/**/b/**/*.txt` to glob,
|
|
498
|
+
the path above will be returned twice
|
|
499
|
+
3. `**` will match any matched file, directory, symlink and '' by default,
|
|
500
|
+
when recursive is `True`
|
|
501
|
+
4. fs_glob returns same as glob.glob(pathname, recursive=True)
|
|
502
|
+
in ascending alphabetical order.
|
|
443
503
|
5. Hidden files (filename stars with '.') will not be found in the result
|
|
444
504
|
|
|
445
|
-
:param pattern: Glob the given relative pattern in the directory represented
|
|
505
|
+
:param pattern: Glob the given relative pattern in the directory represented
|
|
506
|
+
by this path
|
|
446
507
|
:param recursive: If False, `**` will not search directory recursively
|
|
447
|
-
:param missing_ok: If False and target path doesn't match any file,
|
|
508
|
+
:param missing_ok: If False and target path doesn't match any file,
|
|
509
|
+
raise FileNotFoundError
|
|
448
510
|
:returns: An iterator contains paths match `pathname`
|
|
449
|
-
|
|
511
|
+
"""
|
|
450
512
|
glob_path = self.path_without_protocol
|
|
451
513
|
if pattern:
|
|
452
514
|
glob_path = self.joinpath(pattern).path_without_protocol
|
|
453
|
-
for path in fs_iglob(glob_path, recursive=recursive,
|
|
454
|
-
missing_ok=missing_ok):
|
|
515
|
+
for path in fs_iglob(glob_path, recursive=recursive, missing_ok=missing_ok):
|
|
455
516
|
yield self.from_path(path)
|
|
456
517
|
|
|
457
518
|
def is_dir(self, followlinks: bool = False) -> bool:
|
|
458
|
-
|
|
519
|
+
"""
|
|
459
520
|
Test if a path is directory
|
|
460
521
|
|
|
461
522
|
.. note::
|
|
462
523
|
|
|
463
|
-
The difference between this function and ``os.path.isdir`` is that
|
|
524
|
+
The difference between this function and ``os.path.isdir`` is that
|
|
525
|
+
this function regard symlink as file
|
|
464
526
|
|
|
465
527
|
:param followlinks: False if regard symlink as file, else True
|
|
466
528
|
:returns: True if the path is a directory, else False
|
|
467
529
|
|
|
468
|
-
|
|
530
|
+
"""
|
|
469
531
|
if os.path.islink(self.path_without_protocol) and not followlinks:
|
|
470
532
|
return False
|
|
471
533
|
return os.path.isdir(self.path_without_protocol)
|
|
472
534
|
|
|
473
535
|
def is_file(self, followlinks: bool = False) -> bool:
|
|
474
|
-
|
|
536
|
+
"""
|
|
475
537
|
Test if a path is file
|
|
476
538
|
|
|
477
539
|
.. note::
|
|
478
|
-
|
|
479
|
-
The difference between this function and ``os.path.isfile`` is that
|
|
540
|
+
|
|
541
|
+
The difference between this function and ``os.path.isfile`` is that
|
|
542
|
+
this function regard symlink as file
|
|
480
543
|
|
|
481
544
|
:param followlinks: False if regard symlink as file, else True
|
|
482
545
|
:returns: True if the path is a file, else False
|
|
483
546
|
|
|
484
|
-
|
|
547
|
+
"""
|
|
485
548
|
if os.path.islink(self.path_without_protocol) and not followlinks:
|
|
486
549
|
return True
|
|
487
550
|
return os.path.isfile(self.path_without_protocol)
|
|
488
551
|
|
|
489
552
|
def listdir(self) -> List[str]:
|
|
490
|
-
|
|
491
|
-
Get all contents of given fs path.
|
|
553
|
+
"""
|
|
554
|
+
Get all contents of given fs path.
|
|
555
|
+
The result is in ascending alphabetical order.
|
|
492
556
|
|
|
493
557
|
:returns: All contents have in the path in ascending alphabetical order
|
|
494
|
-
|
|
558
|
+
"""
|
|
495
559
|
return sorted(os.listdir(self.path_without_protocol))
|
|
496
560
|
|
|
497
|
-
def iterdir(self) -> Iterator[
|
|
498
|
-
|
|
499
|
-
Get all contents of given fs path.
|
|
561
|
+
def iterdir(self) -> Iterator["FSPath"]:
|
|
562
|
+
"""
|
|
563
|
+
Get all contents of given fs path.
|
|
564
|
+
The result is in ascending alphabetical order.
|
|
500
565
|
|
|
501
566
|
:returns: All contents have in the path in ascending alphabetical order
|
|
502
|
-
|
|
567
|
+
"""
|
|
503
568
|
for path in self.listdir():
|
|
504
569
|
yield self.joinpath(path)
|
|
505
570
|
|
|
506
571
|
def load(self) -> BinaryIO:
|
|
507
|
-
|
|
572
|
+
"""Read all content on specified path and write into memory
|
|
508
573
|
|
|
509
574
|
User should close the BinaryIO manually
|
|
510
575
|
|
|
511
576
|
:returns: Binary stream
|
|
512
|
-
|
|
513
|
-
with open(self.path_without_protocol,
|
|
577
|
+
"""
|
|
578
|
+
with open(self.path_without_protocol, "rb") as f:
|
|
514
579
|
data = f.read()
|
|
515
580
|
return io.BytesIO(data)
|
|
516
581
|
|
|
517
582
|
def mkdir(self, mode=0o777, parents: bool = False, exist_ok: bool = False):
|
|
518
|
-
|
|
583
|
+
"""
|
|
519
584
|
make a directory on fs, including parent directory.
|
|
520
585
|
If there exists a file on the path, raise FileExistsError
|
|
521
586
|
|
|
522
|
-
:param mode: If mode is given, it is combined with the process’ umask value to
|
|
523
|
-
|
|
524
|
-
|
|
587
|
+
:param mode: If mode is given, it is combined with the process’ umask value to
|
|
588
|
+
determine the file mode and access flags.
|
|
589
|
+
:param parents: If parents is true, any missing parents of this path
|
|
590
|
+
are created as needed; If parents is false (the default),
|
|
591
|
+
a missing parent raises FileNotFoundError.
|
|
525
592
|
:param exist_ok: If False and target directory exists, raise FileExistsError
|
|
526
593
|
|
|
527
594
|
:raises: FileExistsError
|
|
528
|
-
|
|
529
|
-
if exist_ok and self.path_without_protocol ==
|
|
595
|
+
"""
|
|
596
|
+
if exist_ok and self.path_without_protocol == "":
|
|
530
597
|
return
|
|
531
598
|
return pathlib.Path(self.path_without_protocol).mkdir(
|
|
532
|
-
mode=mode, parents=parents, exist_ok=exist_ok
|
|
599
|
+
mode=mode, parents=parents, exist_ok=exist_ok
|
|
600
|
+
)
|
|
533
601
|
|
|
534
602
|
def realpath(self) -> str:
|
|
535
|
-
|
|
603
|
+
"""Return the real path of given path
|
|
536
604
|
|
|
537
605
|
:returns: Real path of given path
|
|
538
|
-
|
|
606
|
+
"""
|
|
539
607
|
return fspath(os.path.realpath(self.path_without_protocol))
|
|
540
608
|
|
|
541
609
|
def relpath(self, start: Optional[str] = None) -> str:
|
|
542
|
-
|
|
610
|
+
"""Return the relative path of given path
|
|
543
611
|
|
|
544
612
|
:param start: Given start directory
|
|
545
613
|
:returns: Relative path from start
|
|
546
|
-
|
|
614
|
+
"""
|
|
547
615
|
return fspath(os.path.relpath(self.path_without_protocol, start=start))
|
|
548
616
|
|
|
549
|
-
def rename(self, dst_path: PathLike, overwrite: bool = True) ->
|
|
550
|
-
|
|
617
|
+
def rename(self, dst_path: PathLike, overwrite: bool = True) -> "FSPath":
|
|
618
|
+
"""
|
|
551
619
|
rename file on fs
|
|
552
620
|
|
|
553
621
|
:param dst_path: Given destination path
|
|
554
622
|
:param overwrite: whether or not overwrite file when exists
|
|
555
|
-
|
|
623
|
+
"""
|
|
556
624
|
fs_rename(self.path_without_protocol, dst_path, overwrite)
|
|
557
625
|
return self.from_path(dst_path)
|
|
558
626
|
|
|
559
|
-
def replace(self, dst_path: PathLike, overwrite: bool = True) ->
|
|
560
|
-
|
|
627
|
+
def replace(self, dst_path: PathLike, overwrite: bool = True) -> "FSPath":
|
|
628
|
+
"""
|
|
561
629
|
move file on fs
|
|
562
630
|
|
|
563
631
|
:param dst_path: Given destination path
|
|
564
632
|
:param overwrite: whether or not overwrite file when exists
|
|
565
|
-
|
|
633
|
+
"""
|
|
566
634
|
return self.rename(dst_path=dst_path, overwrite=overwrite)
|
|
567
635
|
|
|
568
636
|
def remove(self, missing_ok: bool = False) -> None:
|
|
569
|
-
|
|
637
|
+
"""
|
|
570
638
|
Remove the file or directory on fs
|
|
571
639
|
|
|
572
|
-
:param missing_ok: if False and target file/directory not exists,
|
|
573
|
-
|
|
640
|
+
:param missing_ok: if False and target file/directory not exists,
|
|
641
|
+
raise FileNotFoundError
|
|
642
|
+
"""
|
|
574
643
|
if missing_ok and not self.exists():
|
|
575
644
|
return
|
|
576
645
|
if self.is_dir():
|
|
@@ -578,9 +647,9 @@ class FSPath(URIPath):
|
|
|
578
647
|
else:
|
|
579
648
|
os.remove(self.path_without_protocol)
|
|
580
649
|
|
|
581
|
-
def _scan(
|
|
582
|
-
|
|
583
|
-
|
|
650
|
+
def _scan(
|
|
651
|
+
self, missing_ok: bool = True, followlinks: bool = False
|
|
652
|
+
) -> Iterator[str]:
|
|
584
653
|
if self.is_file(followlinks=followlinks):
|
|
585
654
|
path = fspath(self.path_without_protocol)
|
|
586
655
|
yield path
|
|
@@ -589,10 +658,8 @@ class FSPath(URIPath):
|
|
|
589
658
|
for filename in files:
|
|
590
659
|
yield os.path.join(root, filename)
|
|
591
660
|
|
|
592
|
-
def scan(self,
|
|
593
|
-
|
|
594
|
-
followlinks: bool = False) -> Iterator[str]:
|
|
595
|
-
'''
|
|
661
|
+
def scan(self, missing_ok: bool = True, followlinks: bool = False) -> Iterator[str]:
|
|
662
|
+
"""
|
|
596
663
|
Iteratively traverse only files in given directory, in alphabetical order.
|
|
597
664
|
Every iteration on generator yields a path string.
|
|
598
665
|
|
|
@@ -600,56 +667,63 @@ class FSPath(URIPath):
|
|
|
600
667
|
If path is a non-existent path, return an empty generator
|
|
601
668
|
If path is a bucket path, return all file paths in the bucket
|
|
602
669
|
|
|
603
|
-
:param missing_ok: If False and there's no file in the directory,
|
|
670
|
+
:param missing_ok: If False and there's no file in the directory,
|
|
671
|
+
raise FileNotFoundError
|
|
604
672
|
:returns: A file path generator
|
|
605
|
-
|
|
673
|
+
"""
|
|
606
674
|
return _create_missing_ok_generator(
|
|
607
|
-
self._scan(followlinks=followlinks),
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
675
|
+
self._scan(followlinks=followlinks),
|
|
676
|
+
missing_ok,
|
|
677
|
+
FileNotFoundError("No match any file in: %r" % self.path),
|
|
678
|
+
)
|
|
679
|
+
|
|
680
|
+
def scan_stat(
|
|
681
|
+
self, missing_ok: bool = True, followlinks: bool = False
|
|
682
|
+
) -> Iterator[FileEntry]:
|
|
683
|
+
"""
|
|
614
684
|
Iteratively traverse only files in given directory, in alphabetical order.
|
|
615
685
|
Every iteration on generator yields a tuple of path string and file stat
|
|
616
686
|
|
|
617
|
-
:param missing_ok: If False and there's no file in the directory,
|
|
687
|
+
:param missing_ok: If False and there's no file in the directory,
|
|
688
|
+
raise FileNotFoundError
|
|
618
689
|
:returns: A file path generator
|
|
619
|
-
|
|
690
|
+
"""
|
|
620
691
|
file_not_found = True
|
|
621
692
|
for path in self._scan(followlinks=followlinks):
|
|
622
|
-
yield FileEntry(
|
|
623
|
-
os.path.basename(path), path, _make_stat(os.lstat(path)))
|
|
693
|
+
yield FileEntry(os.path.basename(path), path, _make_stat(os.lstat(path)))
|
|
624
694
|
file_not_found = False
|
|
625
695
|
if file_not_found:
|
|
626
696
|
if missing_ok:
|
|
627
697
|
return
|
|
628
698
|
raise FileNotFoundError(
|
|
629
|
-
|
|
699
|
+
"No match any file in: %r" % self.path_without_protocol
|
|
700
|
+
)
|
|
630
701
|
|
|
631
702
|
def scandir(self) -> Iterator[FileEntry]:
|
|
632
|
-
|
|
703
|
+
"""
|
|
633
704
|
Get all content of given file path.
|
|
634
705
|
|
|
635
706
|
:returns: An iterator contains all contents have prefix path
|
|
636
|
-
|
|
707
|
+
"""
|
|
637
708
|
|
|
638
709
|
def create_generator():
|
|
639
710
|
with os.scandir(self.path_without_protocol) as entries:
|
|
640
711
|
for entry in entries:
|
|
641
712
|
yield FileEntry(
|
|
642
|
-
entry.name,
|
|
643
|
-
|
|
713
|
+
entry.name,
|
|
714
|
+
entry.path,
|
|
715
|
+
_make_stat(entry.stat(follow_symlinks=False)),
|
|
716
|
+
)
|
|
644
717
|
|
|
645
718
|
return ContextIterator(create_generator())
|
|
646
719
|
|
|
647
720
|
def stat(self, follow_symlinks=True) -> StatResult:
|
|
648
|
-
|
|
649
|
-
Get StatResult of file on fs, including file size and mtime,
|
|
721
|
+
"""
|
|
722
|
+
Get StatResult of file on fs, including file size and mtime,
|
|
723
|
+
referring to fs_getsize and fs_getmtime
|
|
650
724
|
|
|
651
725
|
:returns: StatResult
|
|
652
|
-
|
|
726
|
+
"""
|
|
653
727
|
if follow_symlinks:
|
|
654
728
|
result = _make_stat(os.stat(self.path_without_protocol))
|
|
655
729
|
else:
|
|
@@ -673,37 +747,42 @@ class FSPath(URIPath):
|
|
|
673
747
|
return result._replace(size=size, ctime=ctime, mtime=mtime)
|
|
674
748
|
|
|
675
749
|
def unlink(self, missing_ok: bool = False) -> None:
|
|
676
|
-
|
|
750
|
+
"""
|
|
677
751
|
Remove the file on fs
|
|
678
752
|
|
|
679
753
|
:param missing_ok: if False and target file not exists, raise FileNotFoundError
|
|
680
|
-
|
|
754
|
+
"""
|
|
681
755
|
if missing_ok and not self.exists():
|
|
682
756
|
return
|
|
683
757
|
os.unlink(self.path_without_protocol)
|
|
684
758
|
|
|
685
759
|
def walk(
|
|
686
|
-
self,
|
|
687
|
-
followlinks: bool = False
|
|
760
|
+
self, followlinks: bool = False
|
|
688
761
|
) -> Iterator[Tuple[str, List[str], List[str]]]:
|
|
689
|
-
|
|
762
|
+
"""
|
|
690
763
|
Generate the file names in a directory tree by walking the tree top-down.
|
|
691
764
|
For each directory in the tree rooted at directory path (including path itself),
|
|
692
765
|
it yields a 3-tuple (root, dirs, files).
|
|
693
766
|
|
|
694
|
-
root: a string of current path
|
|
695
|
-
dirs: name list of subdirectories (excluding '.' and '..' if they exist)
|
|
696
|
-
|
|
767
|
+
- root: a string of current path
|
|
768
|
+
- dirs: name list of subdirectories (excluding '.' and '..' if they exist)
|
|
769
|
+
in 'root'. The list is sorted by ascending alphabetical order
|
|
770
|
+
- files: name list of non-directory files (link is regarded as file) in 'root'.
|
|
771
|
+
The list is sorted by ascending alphabetical order
|
|
697
772
|
|
|
698
|
-
If path not exists, or path is a file (link is regarded as file),
|
|
773
|
+
If path not exists, or path is a file (link is regarded as file),
|
|
774
|
+
return an empty generator
|
|
699
775
|
|
|
700
776
|
.. note::
|
|
701
777
|
|
|
702
|
-
Be aware that setting ``followlinks`` to True can lead to infinite recursion
|
|
778
|
+
Be aware that setting ``followlinks`` to True can lead to infinite recursion
|
|
779
|
+
if a link points to a parent directory of itself. fs_walk() does not keep
|
|
780
|
+
track of the directories it visited already.
|
|
703
781
|
|
|
704
782
|
:param followlinks: False if regard symlink as file, else True
|
|
783
|
+
|
|
705
784
|
:returns: A 3-tuple generator
|
|
706
|
-
|
|
785
|
+
"""
|
|
707
786
|
if not self.exists(followlinks=followlinks):
|
|
708
787
|
return
|
|
709
788
|
|
|
@@ -731,62 +810,68 @@ class FSPath(URIPath):
|
|
|
731
810
|
yield root, dirs, files
|
|
732
811
|
|
|
733
812
|
stack.extend(
|
|
734
|
-
(os.path.join(root, directory) for directory in reversed(dirs))
|
|
813
|
+
(os.path.join(root, directory) for directory in reversed(dirs))
|
|
814
|
+
)
|
|
735
815
|
|
|
736
|
-
def resolve(self, strict=False) ->
|
|
737
|
-
|
|
816
|
+
def resolve(self, strict=False) -> "FSPath":
|
|
817
|
+
"""Equal to fs_realpath
|
|
738
818
|
|
|
739
|
-
:return: Return the canonical path of the specified filename,
|
|
819
|
+
:return: Return the canonical path of the specified filename,
|
|
820
|
+
eliminating any symbolic links encountered in the path.
|
|
740
821
|
:rtype: FSPath
|
|
741
|
-
|
|
822
|
+
"""
|
|
742
823
|
return self.from_path(
|
|
743
|
-
fspath(
|
|
744
|
-
|
|
745
|
-
self.path_without_protocol).resolve(strict=strict)))
|
|
824
|
+
fspath(pathlib.Path(self.path_without_protocol).resolve(strict=strict))
|
|
825
|
+
)
|
|
746
826
|
|
|
747
827
|
def md5(self, recalculate: bool = False, followlinks: bool = True):
|
|
748
|
-
|
|
828
|
+
"""
|
|
749
829
|
Calculate the md5 value of the file
|
|
750
830
|
|
|
751
831
|
:param recalculate: Ignore this parameter, just for compatibility
|
|
752
832
|
:param followlinks: Ignore this parameter, just for compatibility
|
|
753
833
|
|
|
754
834
|
returns: md5 of file
|
|
755
|
-
|
|
835
|
+
"""
|
|
756
836
|
if os.path.isdir(self.path_without_protocol):
|
|
757
837
|
hash_md5 = hashlib.md5() # nosec
|
|
758
838
|
for file_name in self.listdir():
|
|
759
|
-
chunk =
|
|
760
|
-
|
|
839
|
+
chunk = (
|
|
840
|
+
FSPath(self.path_without_protocol, file_name)
|
|
841
|
+
.md5(recalculate=recalculate, followlinks=followlinks)
|
|
842
|
+
.encode()
|
|
843
|
+
)
|
|
761
844
|
hash_md5.update(chunk)
|
|
762
845
|
return hash_md5.hexdigest()
|
|
763
|
-
with open(self.path_without_protocol,
|
|
846
|
+
with open(self.path_without_protocol, "rb") as src:
|
|
764
847
|
md5 = calculate_md5(src)
|
|
765
848
|
return md5
|
|
766
849
|
|
|
767
850
|
def _copyfile(
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
851
|
+
self,
|
|
852
|
+
dst_path: PathLike,
|
|
853
|
+
callback: Optional[Callable[[int], None]] = None,
|
|
854
|
+
followlinks: bool = False,
|
|
855
|
+
):
|
|
773
856
|
shutil.copy2(
|
|
774
|
-
self.path_without_protocol,
|
|
775
|
-
|
|
776
|
-
follow_symlinks=followlinks)
|
|
857
|
+
self.path_without_protocol, fspath(dst_path), follow_symlinks=followlinks
|
|
858
|
+
)
|
|
777
859
|
|
|
778
|
-
# After python3.8, patch `shutil.copyfile` is not a good way,
|
|
860
|
+
# After python3.8, patch `shutil.copyfile` is not a good way,
|
|
861
|
+
# because `shutil.copy2` will not call it in some cases.
|
|
779
862
|
if callback:
|
|
780
863
|
callback(self.stat(follow_symlinks=followlinks).size)
|
|
781
864
|
|
|
782
865
|
def copy(
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
866
|
+
self,
|
|
867
|
+
dst_path: PathLike,
|
|
868
|
+
callback: Optional[Callable[[int], None]] = None,
|
|
869
|
+
followlinks: bool = False,
|
|
870
|
+
overwrite: bool = True,
|
|
871
|
+
):
|
|
872
|
+
"""File copy on file system
|
|
873
|
+
Copy content (excluding meta date) of file on `src_path` to `dst_path`.
|
|
874
|
+
`dst_path` must be a complete file name
|
|
790
875
|
|
|
791
876
|
.. note ::
|
|
792
877
|
|
|
@@ -794,17 +879,18 @@ class FSPath(URIPath):
|
|
|
794
879
|
|
|
795
880
|
1. If parent directory of dst_path doesn't exist, create it
|
|
796
881
|
|
|
797
|
-
2. Allow callback function, None by default.
|
|
798
|
-
|
|
799
|
-
|
|
882
|
+
2. Allow callback function, None by default.
|
|
883
|
+
callback: Optional[Callable[[int], None]], the int data is means
|
|
884
|
+
the size (in bytes) of the written data that is passed periodically
|
|
800
885
|
|
|
801
886
|
3. This function is thread-unsafe
|
|
802
887
|
|
|
803
888
|
:param dst_path: Target file path
|
|
804
|
-
:param callback: Called periodically during copy, and the input parameter is
|
|
889
|
+
:param callback: Called periodically during copy, and the input parameter is
|
|
890
|
+
the data size (in bytes) of copy since the last call
|
|
805
891
|
:param followlinks: False if regard symlink as file, else True
|
|
806
892
|
:param overwrite: whether or not overwrite file when exists, default is True
|
|
807
|
-
|
|
893
|
+
"""
|
|
808
894
|
dst_path = fspath(dst_path)
|
|
809
895
|
if not overwrite and os.path.exists((dst_path)):
|
|
810
896
|
return
|
|
@@ -812,28 +898,29 @@ class FSPath(URIPath):
|
|
|
812
898
|
try:
|
|
813
899
|
self._copyfile(dst_path, callback=callback, followlinks=followlinks)
|
|
814
900
|
except FileNotFoundError as error:
|
|
815
|
-
# Prevent the dst_path directory from being created when src_path does not
|
|
901
|
+
# Prevent the dst_path directory from being created when src_path does not
|
|
902
|
+
# exist
|
|
816
903
|
if dst_path == error.filename:
|
|
817
|
-
FSPath(os.path.dirname(dst_path)).mkdir(
|
|
818
|
-
|
|
819
|
-
self._copyfile(
|
|
820
|
-
dst_path, callback=callback, followlinks=followlinks)
|
|
904
|
+
FSPath(os.path.dirname(dst_path)).mkdir(parents=True, exist_ok=True)
|
|
905
|
+
self._copyfile(dst_path, callback=callback, followlinks=followlinks)
|
|
821
906
|
else:
|
|
822
907
|
raise
|
|
823
908
|
|
|
824
909
|
def sync(
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
910
|
+
self,
|
|
911
|
+
dst_path: PathLike,
|
|
912
|
+
followlinks: bool = False,
|
|
913
|
+
force: bool = False,
|
|
914
|
+
overwrite: bool = True,
|
|
915
|
+
) -> None:
|
|
916
|
+
"""Force write of everything to disk.
|
|
831
917
|
|
|
832
918
|
:param dst_path: Target file path
|
|
833
919
|
:param followlinks: False if regard symlink as file, else True
|
|
834
|
-
:param force: Sync file forcible, do not ignore same files,
|
|
920
|
+
:param force: Sync file forcible, do not ignore same files,
|
|
921
|
+
priority is higher than 'overwrite', default is False
|
|
835
922
|
:param overwrite: whether or not overwrite file when exists, default is True
|
|
836
|
-
|
|
923
|
+
"""
|
|
837
924
|
if self.is_dir(followlinks=followlinks):
|
|
838
925
|
|
|
839
926
|
def ignore_same_file(src: str, names: List[str]) -> List[str]:
|
|
@@ -845,7 +932,8 @@ class FSPath(URIPath):
|
|
|
845
932
|
elif not overwrite and dst_obj.exists():
|
|
846
933
|
ignore_files.append(name)
|
|
847
934
|
elif dst_obj.exists() and is_same_file(
|
|
848
|
-
|
|
935
|
+
self.joinpath(name).stat(), dst_obj.stat(), "copy"
|
|
936
|
+
):
|
|
849
937
|
ignore_files.append(name)
|
|
850
938
|
return ignore_files
|
|
851
939
|
|
|
@@ -853,85 +941,92 @@ class FSPath(URIPath):
|
|
|
853
941
|
self.path_without_protocol,
|
|
854
942
|
dst_path,
|
|
855
943
|
ignore=ignore_same_file,
|
|
856
|
-
dirs_exist_ok=True
|
|
944
|
+
dirs_exist_ok=True,
|
|
945
|
+
)
|
|
857
946
|
else:
|
|
858
947
|
self.copy(dst_path, followlinks=followlinks, overwrite=overwrite)
|
|
859
948
|
|
|
860
949
|
def symlink(self, dst_path: PathLike) -> None:
|
|
861
|
-
|
|
950
|
+
"""
|
|
862
951
|
Create a symbolic link pointing to src_path named dst_path.
|
|
863
952
|
|
|
864
953
|
:param dst_path: Destination path
|
|
865
|
-
|
|
954
|
+
"""
|
|
866
955
|
return os.symlink(self.path_without_protocol, dst_path)
|
|
867
956
|
|
|
868
|
-
def readlink(self) ->
|
|
869
|
-
|
|
870
|
-
Return a FSPath instance representing the path to which
|
|
871
|
-
|
|
872
|
-
|
|
957
|
+
def readlink(self) -> "FSPath":
|
|
958
|
+
"""
|
|
959
|
+
Return a FSPath instance representing the path to which
|
|
960
|
+
the symbolic link points.
|
|
961
|
+
|
|
962
|
+
:returns: Return a FSPath instance representing the path to which
|
|
963
|
+
the symbolic link points.
|
|
964
|
+
"""
|
|
873
965
|
return self.from_path(fs_readlink(self.path_without_protocol))
|
|
874
966
|
|
|
875
967
|
def is_symlink(self) -> bool:
|
|
876
|
-
|
|
968
|
+
"""Test whether a path is a symbolic link
|
|
877
969
|
|
|
878
970
|
:return: If path is a symbolic link return True, else False
|
|
879
971
|
:rtype: bool
|
|
880
|
-
|
|
972
|
+
"""
|
|
881
973
|
return os.path.islink(self.path_without_protocol)
|
|
882
974
|
|
|
883
975
|
def is_mount(self) -> bool:
|
|
884
|
-
|
|
976
|
+
"""Test whether a path is a mount point
|
|
885
977
|
|
|
886
978
|
:returns: True if a path is a mount point, else False
|
|
887
|
-
|
|
979
|
+
"""
|
|
888
980
|
return os.path.ismount(self.path_without_protocol)
|
|
889
981
|
|
|
890
|
-
def cwd(self) ->
|
|
891
|
-
|
|
982
|
+
def cwd(self) -> "FSPath":
|
|
983
|
+
"""Return current working directory
|
|
892
984
|
|
|
893
985
|
returns: Current working directory
|
|
894
|
-
|
|
986
|
+
"""
|
|
895
987
|
return self.from_path(fs_cwd())
|
|
896
988
|
|
|
897
989
|
def home(self):
|
|
898
|
-
|
|
990
|
+
"""Return the home directory
|
|
899
991
|
|
|
900
992
|
returns: Home directory path
|
|
901
|
-
|
|
993
|
+
"""
|
|
902
994
|
return self.from_path(fs_home())
|
|
903
995
|
|
|
904
996
|
def joinpath(self, *other_paths: PathLike) -> "FSPath":
|
|
905
997
|
path = fspath(self)
|
|
906
|
-
if path ==
|
|
907
|
-
path =
|
|
998
|
+
if path == ".":
|
|
999
|
+
path = ""
|
|
908
1000
|
return self.from_path(path_join(path, *map(fspath, other_paths)))
|
|
909
1001
|
|
|
910
1002
|
def save(self, file_object: BinaryIO):
|
|
911
|
-
|
|
1003
|
+
"""Write the opened binary stream to path
|
|
912
1004
|
If parent directory of path doesn't exist, it will be created.
|
|
913
1005
|
|
|
914
1006
|
:param file_object: stream to be read
|
|
915
|
-
|
|
1007
|
+
"""
|
|
916
1008
|
FSPath(os.path.dirname(self.path_without_protocol)).mkdir(
|
|
917
|
-
parents=True, exist_ok=True
|
|
918
|
-
|
|
1009
|
+
parents=True, exist_ok=True
|
|
1010
|
+
)
|
|
1011
|
+
with open(self.path_without_protocol, "wb") as output:
|
|
919
1012
|
output.write(file_object.read())
|
|
920
1013
|
|
|
921
1014
|
def open(
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
1015
|
+
self,
|
|
1016
|
+
mode: str = "r",
|
|
1017
|
+
buffering=-1,
|
|
1018
|
+
encoding=None,
|
|
1019
|
+
errors=None,
|
|
1020
|
+
newline=None,
|
|
1021
|
+
closefd=True,
|
|
1022
|
+
**kwargs,
|
|
1023
|
+
) -> IO:
|
|
1024
|
+
if not isinstance(self.path_without_protocol, int) and (
|
|
1025
|
+
"w" in mode or "x" in mode or "a" in mode
|
|
1026
|
+
):
|
|
933
1027
|
FSPath(os.path.dirname(self.path_without_protocol)).mkdir(
|
|
934
|
-
parents=True, exist_ok=True
|
|
1028
|
+
parents=True, exist_ok=True
|
|
1029
|
+
)
|
|
935
1030
|
return io.open(
|
|
936
1031
|
self.path_without_protocol,
|
|
937
1032
|
mode,
|
|
@@ -939,98 +1034,111 @@ class FSPath(URIPath):
|
|
|
939
1034
|
encoding=encoding,
|
|
940
1035
|
errors=errors,
|
|
941
1036
|
newline=newline,
|
|
942
|
-
closefd=closefd
|
|
1037
|
+
closefd=closefd,
|
|
1038
|
+
)
|
|
943
1039
|
|
|
944
1040
|
@cached_property
|
|
945
1041
|
def parts(self) -> Tuple[str, ...]:
|
|
946
|
-
|
|
1042
|
+
"""
|
|
947
1043
|
A tuple giving access to the path’s various components
|
|
948
|
-
|
|
1044
|
+
"""
|
|
949
1045
|
return pathlib.Path(self.path_without_protocol).parts
|
|
950
1046
|
|
|
951
1047
|
def chmod(self, mode: int, *, follow_symlinks: bool = True):
|
|
952
|
-
|
|
1048
|
+
"""
|
|
953
1049
|
Change the file mode and permissions, like os.chmod().
|
|
954
1050
|
|
|
955
|
-
This method normally follows symlinks.
|
|
1051
|
+
This method normally follows symlinks.
|
|
956
1052
|
Some Unix flavours support changing permissions on the symlink itself;
|
|
957
|
-
on these platforms you may add the argument follow_symlinks=False,
|
|
958
|
-
|
|
1053
|
+
on these platforms you may add the argument follow_symlinks=False,
|
|
1054
|
+
or use lchmod().
|
|
1055
|
+
"""
|
|
959
1056
|
return os.chmod(
|
|
960
|
-
path=self.path_without_protocol,
|
|
961
|
-
|
|
962
|
-
follow_symlinks=follow_symlinks)
|
|
1057
|
+
path=self.path_without_protocol, mode=mode, follow_symlinks=follow_symlinks
|
|
1058
|
+
)
|
|
963
1059
|
|
|
964
1060
|
def group(self) -> str:
|
|
965
|
-
|
|
966
|
-
Return the name of the group owning the file. KeyError is raised if
|
|
967
|
-
|
|
1061
|
+
"""
|
|
1062
|
+
Return the name of the group owning the file. KeyError is raised if
|
|
1063
|
+
the file’s gid isn’t found in the system database.
|
|
1064
|
+
"""
|
|
968
1065
|
return pathlib.Path(self.path_without_protocol).group()
|
|
969
1066
|
|
|
970
1067
|
def is_socket(self) -> bool:
|
|
971
|
-
|
|
972
|
-
Return True if the path points to a Unix socket (or a symbolic link pointing to
|
|
1068
|
+
"""
|
|
1069
|
+
Return True if the path points to a Unix socket (or a symbolic link pointing to
|
|
1070
|
+
a Unix socket), False if it points to another kind of file.
|
|
973
1071
|
|
|
974
|
-
False is also returned if the path doesn’t exist or is a broken symlink;
|
|
975
|
-
|
|
1072
|
+
False is also returned if the path doesn’t exist or is a broken symlink;
|
|
1073
|
+
other errors (such as permission errors) are propagated.
|
|
1074
|
+
"""
|
|
976
1075
|
return pathlib.Path(self.path_without_protocol).is_socket()
|
|
977
1076
|
|
|
978
1077
|
def is_fifo(self) -> bool:
|
|
979
|
-
|
|
980
|
-
Return True if the path points to a FIFO (or a symbolic link pointing to a FIFO)
|
|
1078
|
+
"""
|
|
1079
|
+
Return True if the path points to a FIFO (or a symbolic link pointing to a FIFO)
|
|
1080
|
+
Return False if it points to another kind of file.
|
|
981
1081
|
|
|
982
|
-
False is also returned if the path doesn’t exist or is a broken symlink;
|
|
983
|
-
|
|
1082
|
+
False is also returned if the path doesn’t exist or is a broken symlink;
|
|
1083
|
+
other errors (such as permission errors) are propagated.
|
|
1084
|
+
"""
|
|
984
1085
|
return pathlib.Path(self.path_without_protocol).is_fifo()
|
|
985
1086
|
|
|
986
1087
|
def is_block_device(self) -> bool:
|
|
987
|
-
|
|
988
|
-
Return True if the path points to a block device (or a symbolic link pointing to
|
|
1088
|
+
"""
|
|
1089
|
+
Return True if the path points to a block device (or a symbolic link pointing to
|
|
1090
|
+
a block device), False if it points to another kind of file.
|
|
989
1091
|
|
|
990
|
-
False is also returned if the path doesn’t exist or is a broken symlink;
|
|
991
|
-
|
|
1092
|
+
False is also returned if the path doesn’t exist or is a broken symlink;
|
|
1093
|
+
other errors (such as permission errors) are propagated.
|
|
1094
|
+
"""
|
|
992
1095
|
return pathlib.Path(self.path_without_protocol).is_block_device()
|
|
993
1096
|
|
|
994
1097
|
def is_char_device(self) -> bool:
|
|
995
|
-
|
|
996
|
-
Return True if the path points to a character device (or a symbolic link
|
|
1098
|
+
"""
|
|
1099
|
+
Return True if the path points to a character device (or a symbolic link
|
|
1100
|
+
pointing to a character device), False if it points to another kind of file.
|
|
997
1101
|
|
|
998
|
-
False is also returned if the path doesn’t exist or is a broken symlink;
|
|
999
|
-
|
|
1102
|
+
False is also returned if the path doesn’t exist or is a broken symlink;
|
|
1103
|
+
other errors (such as permission errors) are propagated.
|
|
1104
|
+
"""
|
|
1000
1105
|
return pathlib.Path(self.path_without_protocol).is_char_device()
|
|
1001
1106
|
|
|
1002
1107
|
def owner(self) -> str:
|
|
1003
|
-
|
|
1004
|
-
Return the name of the user owning the file. KeyError is raised if the file’s
|
|
1005
|
-
|
|
1108
|
+
"""
|
|
1109
|
+
Return the name of the user owning the file. KeyError is raised if the file’s
|
|
1110
|
+
uid isn’t found in the system database.
|
|
1111
|
+
"""
|
|
1006
1112
|
return pathlib.Path(self.path_without_protocol).owner()
|
|
1007
1113
|
|
|
1008
|
-
def absolute(self) ->
|
|
1009
|
-
|
|
1010
|
-
Make the path absolute, without normalization or resolving symlinks.
|
|
1011
|
-
|
|
1114
|
+
def absolute(self) -> "FSPath":
|
|
1115
|
+
"""
|
|
1116
|
+
Make the path absolute, without normalization or resolving symlinks.
|
|
1117
|
+
Returns a new path object
|
|
1118
|
+
"""
|
|
1012
1119
|
return self.from_path(os.path.abspath(self.path_without_protocol))
|
|
1013
1120
|
|
|
1014
1121
|
def rmdir(self):
|
|
1015
|
-
|
|
1122
|
+
"""
|
|
1016
1123
|
Remove this directory. The directory must be empty.
|
|
1017
|
-
|
|
1124
|
+
"""
|
|
1018
1125
|
return os.rmdir(self.path_without_protocol)
|
|
1019
1126
|
|
|
1020
1127
|
def hardlink_to(self, target):
|
|
1021
|
-
|
|
1128
|
+
"""
|
|
1022
1129
|
Make this path a hard link to the same file as target.
|
|
1023
|
-
|
|
1130
|
+
"""
|
|
1024
1131
|
return os.link(target, self.path)
|
|
1025
1132
|
|
|
1026
1133
|
def utime(self, atime: Union[float, int], mtime: Union[float, int]):
|
|
1027
1134
|
"""
|
|
1028
1135
|
Set the access and modified times of the file specified by path.
|
|
1029
1136
|
|
|
1030
|
-
:param atime: a float or int representing the access time to be set.
|
|
1031
|
-
access time is set to the current time.
|
|
1032
|
-
:param mtime: a float or int representing the modified time to be set.
|
|
1033
|
-
|
|
1137
|
+
:param atime: a float or int representing the access time to be set.
|
|
1138
|
+
If it is set to None, the access time is set to the current time.
|
|
1139
|
+
:param mtime: a float or int representing the modified time to be set.
|
|
1140
|
+
If it is set to None, the modified time is set
|
|
1141
|
+
to the current time.
|
|
1034
1142
|
:return: None
|
|
1035
1143
|
"""
|
|
1036
1144
|
return os.utime(self.path_without_protocol, times=(atime, mtime))
|