megfile 3.1.6__py3-none-any.whl → 4.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- megfile/cli.py +12 -7
- megfile/config.py +34 -44
- megfile/fs.py +169 -11
- megfile/fs_path.py +183 -259
- megfile/hdfs.py +106 -5
- megfile/hdfs_path.py +34 -90
- megfile/http.py +50 -1
- megfile/http_path.py +27 -65
- megfile/interfaces.py +1 -8
- megfile/lib/base_prefetch_reader.py +62 -78
- megfile/lib/combine_reader.py +5 -0
- megfile/lib/glob.py +3 -6
- megfile/lib/hdfs_prefetch_reader.py +7 -7
- megfile/lib/http_prefetch_reader.py +6 -6
- megfile/lib/s3_buffered_writer.py +67 -64
- megfile/lib/s3_cached_handler.py +1 -2
- megfile/lib/s3_limited_seekable_writer.py +3 -7
- megfile/lib/s3_memory_handler.py +1 -2
- megfile/lib/s3_pipe_handler.py +1 -2
- megfile/lib/s3_prefetch_reader.py +15 -20
- megfile/lib/s3_share_cache_reader.py +8 -5
- megfile/pathlike.py +397 -401
- megfile/s3.py +118 -17
- megfile/s3_path.py +150 -224
- megfile/sftp.py +300 -10
- megfile/sftp_path.py +46 -322
- megfile/smart.py +33 -27
- megfile/smart_path.py +9 -14
- megfile/stdio.py +1 -1
- megfile/stdio_path.py +2 -2
- megfile/utils/__init__.py +11 -4
- megfile/version.py +1 -1
- {megfile-3.1.6.dist-info → megfile-4.0.0.dist-info}/METADATA +7 -7
- megfile-4.0.0.dist-info/RECORD +52 -0
- {megfile-3.1.6.dist-info → megfile-4.0.0.dist-info}/WHEEL +1 -1
- {megfile-3.1.6.dist-info → megfile-4.0.0.dist-info}/top_level.txt +0 -2
- docs/conf.py +0 -65
- megfile-3.1.6.dist-info/RECORD +0 -55
- scripts/convert_results_to_sarif.py +0 -91
- scripts/generate_file.py +0 -344
- {megfile-3.1.6.dist-info → megfile-4.0.0.dist-info}/LICENSE +0 -0
- {megfile-3.1.6.dist-info → megfile-4.0.0.dist-info}/LICENSE.pyre +0 -0
- {megfile-3.1.6.dist-info → megfile-4.0.0.dist-info}/entry_points.txt +0 -0
megfile/fs_path.py
CHANGED
|
@@ -4,8 +4,12 @@ import os
|
|
|
4
4
|
import pathlib
|
|
5
5
|
import shutil
|
|
6
6
|
from functools import cached_property
|
|
7
|
+
from stat import S_ISBLK as stat_isblk
|
|
8
|
+
from stat import S_ISCHR as stat_ischr
|
|
7
9
|
from stat import S_ISDIR as stat_isdir
|
|
10
|
+
from stat import S_ISFIFO as stat_isfifo
|
|
8
11
|
from stat import S_ISLNK as stat_islnk
|
|
12
|
+
from stat import S_ISSOCK as stat_issock
|
|
9
13
|
from typing import IO, BinaryIO, Callable, Iterator, List, Optional, Tuple, Union
|
|
10
14
|
|
|
11
15
|
from megfile.errors import _create_missing_ok_generator
|
|
@@ -30,24 +34,13 @@ __all__ = [
|
|
|
30
34
|
"is_fs",
|
|
31
35
|
"fs_path_join",
|
|
32
36
|
"_make_stat",
|
|
33
|
-
"fs_readlink",
|
|
34
|
-
"fs_cwd",
|
|
35
|
-
"fs_home",
|
|
36
|
-
"fs_iglob",
|
|
37
|
-
"fs_glob",
|
|
38
|
-
"fs_glob_stat",
|
|
39
|
-
"fs_rename",
|
|
40
|
-
"fs_resolve",
|
|
41
|
-
"fs_move",
|
|
42
|
-
"fs_makedirs",
|
|
43
|
-
"fs_lstat",
|
|
44
37
|
]
|
|
45
38
|
|
|
46
39
|
|
|
47
40
|
def _make_stat(stat: os.stat_result) -> StatResult:
|
|
48
41
|
return StatResult(
|
|
49
42
|
size=stat.st_size,
|
|
50
|
-
ctime=stat.st_ctime,
|
|
43
|
+
ctime=stat.st_ctime, # pyre-ignore[16]
|
|
51
44
|
mtime=stat.st_mtime,
|
|
52
45
|
isdir=stat_isdir(stat.st_mode),
|
|
53
46
|
islnk=stat_islnk(stat.st_mode),
|
|
@@ -72,121 +65,6 @@ def fs_path_join(path: PathLike, *other_paths: PathLike) -> str:
|
|
|
72
65
|
return path_join(fspath(path), *map(fspath, other_paths))
|
|
73
66
|
|
|
74
67
|
|
|
75
|
-
def fs_readlink(path) -> str:
|
|
76
|
-
"""
|
|
77
|
-
Return a string representing the path to which the symbolic link points.
|
|
78
|
-
:returns: Return a string representing the path to which the symbolic link points.
|
|
79
|
-
"""
|
|
80
|
-
return os.readlink(path)
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
def fs_cwd() -> str:
|
|
84
|
-
"""Return current working directory
|
|
85
|
-
|
|
86
|
-
returns: Current working directory
|
|
87
|
-
"""
|
|
88
|
-
return os.getcwd()
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
def fs_home():
|
|
92
|
-
"""Return the home directory
|
|
93
|
-
|
|
94
|
-
returns: Home directory path
|
|
95
|
-
"""
|
|
96
|
-
return os.path.expanduser("~")
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
def fs_iglob(
|
|
100
|
-
path: PathLike, recursive: bool = True, missing_ok: bool = True
|
|
101
|
-
) -> Iterator[str]:
|
|
102
|
-
"""Return path iterator in ascending alphabetical order,
|
|
103
|
-
in which path matches glob pattern
|
|
104
|
-
|
|
105
|
-
1. If doesn't match any path, return empty list
|
|
106
|
-
Notice: ``glob.glob`` in standard library returns ['a/'] instead of empty list
|
|
107
|
-
when pathname is like `a/**`, recursive is True and directory 'a' doesn't exist.
|
|
108
|
-
fs_glob behaves like ``glob.glob`` in standard library under such circumstance.
|
|
109
|
-
2. No guarantee that each path in result is different, which means:
|
|
110
|
-
Assume there exists a path `/a/b/c/b/d.txt`
|
|
111
|
-
use path pattern like `/**/b/**/*.txt` to glob,
|
|
112
|
-
the path above will be returned twice
|
|
113
|
-
3. `**` will match any matched file, directory, symlink and '' by default,
|
|
114
|
-
when recursive is `True`
|
|
115
|
-
4. fs_glob returns same as glob.glob(pathname, recursive=True)
|
|
116
|
-
in ascending alphabetical order.
|
|
117
|
-
5. Hidden files (filename stars with '.') will not be found in the result
|
|
118
|
-
|
|
119
|
-
:param recursive: If False, `**` will not search directory recursively
|
|
120
|
-
:param missing_ok: If False and target path doesn't match any file,
|
|
121
|
-
raise FileNotFoundError
|
|
122
|
-
:returns: An iterator contains paths match `pathname`
|
|
123
|
-
"""
|
|
124
|
-
for path in _create_missing_ok_generator(
|
|
125
|
-
iglob(fspath(path), recursive=recursive),
|
|
126
|
-
missing_ok,
|
|
127
|
-
FileNotFoundError("No match any file: %r" % path),
|
|
128
|
-
):
|
|
129
|
-
yield path
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
def fs_glob(
|
|
133
|
-
path: PathLike, recursive: bool = True, missing_ok: bool = True
|
|
134
|
-
) -> List[str]:
|
|
135
|
-
"""Return path list in ascending alphabetical order,
|
|
136
|
-
in which path matches glob pattern
|
|
137
|
-
|
|
138
|
-
1. If doesn't match any path, return empty list
|
|
139
|
-
Notice: ``glob.glob`` in standard library returns ['a/'] instead of empty list
|
|
140
|
-
when pathname is like `a/**`, recursive is True and directory 'a' doesn't exist.
|
|
141
|
-
fs_glob behaves like ``glob.glob`` in standard library under such circumstance.
|
|
142
|
-
2. No guarantee that each path in result is different, which means:
|
|
143
|
-
Assume there exists a path `/a/b/c/b/d.txt`
|
|
144
|
-
use path pattern like `/**/b/**/*.txt` to glob,
|
|
145
|
-
the path above will be returned twice
|
|
146
|
-
3. `**` will match any matched file, directory, symlink and '' by default,
|
|
147
|
-
when recursive is `True`
|
|
148
|
-
4. fs_glob returns same as glob.glob(pathname, recursive=True)
|
|
149
|
-
in ascending alphabetical order.
|
|
150
|
-
5. Hidden files (filename stars with '.') will not be found in the result
|
|
151
|
-
|
|
152
|
-
:param recursive: If False, `**` will not search directory recursively
|
|
153
|
-
:param missing_ok: If False and target path doesn't match any file,
|
|
154
|
-
raise FileNotFoundError
|
|
155
|
-
:returns: A list contains paths match `pathname`
|
|
156
|
-
"""
|
|
157
|
-
return list(fs_iglob(path=path, recursive=recursive, missing_ok=missing_ok))
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
def fs_glob_stat(
|
|
161
|
-
path: PathLike, recursive: bool = True, missing_ok: bool = True
|
|
162
|
-
) -> Iterator[FileEntry]:
|
|
163
|
-
"""Return a list contains tuples of path and file stat,
|
|
164
|
-
in ascending alphabetical order, in which path matches glob pattern
|
|
165
|
-
|
|
166
|
-
1. If doesn't match any path, return empty list
|
|
167
|
-
Notice: ``glob.glob`` in standard library returns ['a/'] instead of empty list
|
|
168
|
-
when pathname is like `a/**`, recursive is True and directory 'a' doesn't exist.
|
|
169
|
-
fs_glob behaves like ``glob.glob`` in standard library under such circumstance.
|
|
170
|
-
2. No guarantee that each path in result is different, which means:
|
|
171
|
-
Assume there exists a path `/a/b/c/b/d.txt`
|
|
172
|
-
use path pattern like `/**/b/**/*.txt` to glob,
|
|
173
|
-
the path above will be returned twice.
|
|
174
|
-
3. `**` will match any matched file, directory, symlink and '' by default,
|
|
175
|
-
when recursive is `True`
|
|
176
|
-
4. fs_glob returns same as glob.glob(pathname, recursive=True)
|
|
177
|
-
in ascending alphabetical order.
|
|
178
|
-
5. Hidden files (filename stars with '.') will not be found in the result
|
|
179
|
-
|
|
180
|
-
:param recursive: If False, `**` will not search directory recursively
|
|
181
|
-
:param missing_ok: If False and target path doesn't match any file,
|
|
182
|
-
raise FileNotFoundError
|
|
183
|
-
:returns: A list contains tuples of path and file stat,
|
|
184
|
-
in which paths match `pathname`
|
|
185
|
-
"""
|
|
186
|
-
for path in fs_iglob(path=path, recursive=recursive, missing_ok=missing_ok):
|
|
187
|
-
yield FileEntry(os.path.basename(path), path, _make_stat(os.lstat(path)))
|
|
188
|
-
|
|
189
|
-
|
|
190
68
|
def _fs_rename_file(
|
|
191
69
|
src_path: PathLike, dst_path: PathLike, overwrite: bool = True
|
|
192
70
|
) -> None:
|
|
@@ -208,82 +86,6 @@ def _fs_rename_file(
|
|
|
208
86
|
shutil.move(src_path, dst_path)
|
|
209
87
|
|
|
210
88
|
|
|
211
|
-
def fs_rename(src_path: PathLike, dst_path: PathLike, overwrite: bool = True) -> None:
|
|
212
|
-
"""
|
|
213
|
-
rename file on fs
|
|
214
|
-
|
|
215
|
-
:param src_path: Given path
|
|
216
|
-
:param dst_path: Given destination path
|
|
217
|
-
:param overwrite: whether or not overwrite file when exists
|
|
218
|
-
"""
|
|
219
|
-
src_path, dst_path = fspath(src_path), fspath(dst_path)
|
|
220
|
-
if os.path.isfile(src_path):
|
|
221
|
-
return _fs_rename_file(src_path, dst_path, overwrite)
|
|
222
|
-
else:
|
|
223
|
-
os.makedirs(dst_path, exist_ok=True)
|
|
224
|
-
|
|
225
|
-
with os.scandir(src_path) as entries:
|
|
226
|
-
for file_entry in entries:
|
|
227
|
-
src_file_path = file_entry.path
|
|
228
|
-
dst_file_path = dst_path
|
|
229
|
-
relative_path = os.path.relpath(src_file_path, start=src_path)
|
|
230
|
-
if relative_path and relative_path != ".":
|
|
231
|
-
dst_file_path = os.path.join(dst_file_path, relative_path)
|
|
232
|
-
if os.path.exists(dst_file_path) and file_entry.is_dir():
|
|
233
|
-
fs_rename(src_file_path, dst_file_path, overwrite)
|
|
234
|
-
else:
|
|
235
|
-
_fs_rename_file(src_file_path, dst_file_path, overwrite)
|
|
236
|
-
|
|
237
|
-
if os.path.isdir(src_path):
|
|
238
|
-
shutil.rmtree(src_path)
|
|
239
|
-
else:
|
|
240
|
-
os.remove(src_path)
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
def fs_move(src_path: PathLike, dst_path: PathLike, overwrite: bool = True) -> None:
|
|
244
|
-
"""
|
|
245
|
-
rename file on fs
|
|
246
|
-
|
|
247
|
-
:param src_path: Given path
|
|
248
|
-
:param dst_path: Given destination path
|
|
249
|
-
:param overwrite: whether or not overwrite file when exists
|
|
250
|
-
"""
|
|
251
|
-
return fs_rename(src_path, dst_path, overwrite)
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
def fs_resolve(path: PathLike) -> str:
|
|
255
|
-
"""Equal to fs_realpath, return the real path of given path
|
|
256
|
-
|
|
257
|
-
:param path: Given path
|
|
258
|
-
:returns: Real path of given path
|
|
259
|
-
"""
|
|
260
|
-
return FSPath(path).realpath()
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
def fs_makedirs(path: PathLike, exist_ok: bool = False):
|
|
264
|
-
"""
|
|
265
|
-
make a directory on fs, including parent directory
|
|
266
|
-
|
|
267
|
-
If there exists a file on the path, raise FileExistsError
|
|
268
|
-
|
|
269
|
-
:param path: Given path
|
|
270
|
-
:param exist_ok: If False and target directory exists, raise FileExistsError
|
|
271
|
-
:raises: FileExistsError
|
|
272
|
-
"""
|
|
273
|
-
return FSPath(path).mkdir(parents=True, exist_ok=exist_ok)
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
def fs_lstat(path: PathLike) -> StatResult:
|
|
277
|
-
"""
|
|
278
|
-
Like Path.stat() but, if the path points to a symbolic link,
|
|
279
|
-
return the symbolic link’s information rather than its target’s.
|
|
280
|
-
|
|
281
|
-
:param path: Given path
|
|
282
|
-
:returns: StatResult
|
|
283
|
-
"""
|
|
284
|
-
return FSPath(path).lstat()
|
|
285
|
-
|
|
286
|
-
|
|
287
89
|
@SmartPath.register
|
|
288
90
|
class FSPath(URIPath):
|
|
289
91
|
"""file protocol
|
|
@@ -299,27 +101,39 @@ class FSPath(URIPath):
|
|
|
299
101
|
path = str(path)
|
|
300
102
|
self.path = path
|
|
301
103
|
|
|
104
|
+
def _check_int_path(self) -> None:
|
|
105
|
+
if isinstance(self.path_without_protocol, int):
|
|
106
|
+
raise TypeError("not support the path of int type")
|
|
107
|
+
|
|
302
108
|
def __fspath__(self) -> str:
|
|
303
|
-
|
|
109
|
+
self._check_int_path()
|
|
110
|
+
return os.path.normpath(self.path_without_protocol) # pyre-ignore[6]
|
|
304
111
|
|
|
305
112
|
@cached_property
|
|
306
113
|
def root(self) -> str:
|
|
307
|
-
|
|
114
|
+
if isinstance(self.path_without_protocol, int):
|
|
115
|
+
return "/"
|
|
116
|
+
return pathlib.Path(self.path_without_protocol).root # pyre-ignore[6]
|
|
308
117
|
|
|
309
118
|
@cached_property
|
|
310
119
|
def anchor(self) -> str:
|
|
311
|
-
|
|
120
|
+
if isinstance(self.path_without_protocol, int):
|
|
121
|
+
return "/"
|
|
122
|
+
return pathlib.Path(self.path_without_protocol).anchor # pyre-ignore[6]
|
|
312
123
|
|
|
313
124
|
@cached_property
|
|
314
125
|
def drive(self) -> str:
|
|
315
|
-
|
|
126
|
+
if isinstance(self.path_without_protocol, int):
|
|
127
|
+
return ""
|
|
128
|
+
return pathlib.Path(self.path_without_protocol).drive # pyre-ignore[6]
|
|
316
129
|
|
|
317
130
|
@classmethod
|
|
318
131
|
def from_uri(cls, path: PathLike) -> "FSPath":
|
|
319
132
|
return cls.from_path(path)
|
|
320
133
|
|
|
321
|
-
@
|
|
134
|
+
@cached_property
|
|
322
135
|
def path_with_protocol(self) -> Union[str, int]:
|
|
136
|
+
"""Return path with protocol, like file:///root"""
|
|
323
137
|
if isinstance(self.path, int):
|
|
324
138
|
return self.path
|
|
325
139
|
protocol_prefix = self.protocol + "://"
|
|
@@ -327,19 +141,32 @@ class FSPath(URIPath):
|
|
|
327
141
|
return self.path # pyre-ignore[7]
|
|
328
142
|
return protocol_prefix + self.path # pyre-ignore[58]
|
|
329
143
|
|
|
144
|
+
@cached_property
|
|
145
|
+
def path_without_protocol(self) -> Union[str, int]:
|
|
146
|
+
"""
|
|
147
|
+
Return path without protocol, example: if path is file:///root,
|
|
148
|
+
return /root
|
|
149
|
+
"""
|
|
150
|
+
if isinstance(self.path, int):
|
|
151
|
+
return self.path
|
|
152
|
+
return super().path_without_protocol
|
|
153
|
+
|
|
330
154
|
def is_absolute(self) -> bool:
|
|
331
155
|
"""Test whether a path is absolute
|
|
332
156
|
|
|
333
157
|
:returns: True if a path is absolute, else False
|
|
334
158
|
"""
|
|
335
|
-
|
|
159
|
+
if isinstance(self.path_without_protocol, int):
|
|
160
|
+
return False
|
|
161
|
+
return os.path.isabs(self.path_without_protocol) # pyre-ignore[6]
|
|
336
162
|
|
|
337
163
|
def abspath(self) -> str:
|
|
338
164
|
"""Return the absolute path of given path
|
|
339
165
|
|
|
340
166
|
:returns: Absolute path of given path
|
|
341
167
|
"""
|
|
342
|
-
|
|
168
|
+
self._check_int_path()
|
|
169
|
+
return fspath(os.path.abspath(self.path_without_protocol)) # pyre-ignore[6]
|
|
343
170
|
|
|
344
171
|
def access(self, mode: Access = Access.READ) -> bool:
|
|
345
172
|
"""
|
|
@@ -512,7 +339,12 @@ class FSPath(URIPath):
|
|
|
512
339
|
glob_path = self.path_without_protocol
|
|
513
340
|
if pattern:
|
|
514
341
|
glob_path = self.joinpath(pattern).path_without_protocol
|
|
515
|
-
|
|
342
|
+
|
|
343
|
+
for path in _create_missing_ok_generator(
|
|
344
|
+
iglob(fspath(glob_path), recursive=recursive),
|
|
345
|
+
missing_ok,
|
|
346
|
+
FileNotFoundError("No match any file: %r" % glob_path),
|
|
347
|
+
):
|
|
516
348
|
yield self.from_path(path)
|
|
517
349
|
|
|
518
350
|
def is_dir(self, followlinks: bool = False) -> bool:
|
|
@@ -556,7 +388,8 @@ class FSPath(URIPath):
|
|
|
556
388
|
|
|
557
389
|
:returns: All contents have in the path in ascending alphabetical order
|
|
558
390
|
"""
|
|
559
|
-
|
|
391
|
+
self._check_int_path()
|
|
392
|
+
return sorted(os.listdir(self.path_without_protocol)) # pyre-ignore[6]
|
|
560
393
|
|
|
561
394
|
def iterdir(self) -> Iterator["FSPath"]:
|
|
562
395
|
"""
|
|
@@ -593,9 +426,13 @@ class FSPath(URIPath):
|
|
|
593
426
|
|
|
594
427
|
:raises: FileExistsError
|
|
595
428
|
"""
|
|
596
|
-
if exist_ok and
|
|
429
|
+
if exist_ok and (
|
|
430
|
+
self.path_without_protocol == ""
|
|
431
|
+
or isinstance(self.path_without_protocol, int)
|
|
432
|
+
):
|
|
597
433
|
return
|
|
598
|
-
|
|
434
|
+
self._check_int_path()
|
|
435
|
+
return pathlib.Path(self.path_without_protocol).mkdir( # pyre-ignore[6]
|
|
599
436
|
mode=mode, parents=parents, exist_ok=exist_ok
|
|
600
437
|
)
|
|
601
438
|
|
|
@@ -604,7 +441,8 @@ class FSPath(URIPath):
|
|
|
604
441
|
|
|
605
442
|
:returns: Real path of given path
|
|
606
443
|
"""
|
|
607
|
-
|
|
444
|
+
self._check_int_path()
|
|
445
|
+
return fspath(os.path.realpath(self.path_without_protocol)) # pyre-ignore[6]
|
|
608
446
|
|
|
609
447
|
def relpath(self, start: Optional[str] = None) -> str:
|
|
610
448
|
"""Return the relative path of given path
|
|
@@ -612,7 +450,13 @@ class FSPath(URIPath):
|
|
|
612
450
|
:param start: Given start directory
|
|
613
451
|
:returns: Relative path from start
|
|
614
452
|
"""
|
|
615
|
-
|
|
453
|
+
self._check_int_path()
|
|
454
|
+
return fspath(
|
|
455
|
+
os.path.relpath(
|
|
456
|
+
self.path_without_protocol, # pyre-ignore[6]
|
|
457
|
+
start=start,
|
|
458
|
+
)
|
|
459
|
+
)
|
|
616
460
|
|
|
617
461
|
def rename(self, dst_path: PathLike, overwrite: bool = True) -> "FSPath":
|
|
618
462
|
"""
|
|
@@ -621,7 +465,32 @@ class FSPath(URIPath):
|
|
|
621
465
|
:param dst_path: Given destination path
|
|
622
466
|
:param overwrite: whether or not overwrite file when exists
|
|
623
467
|
"""
|
|
624
|
-
|
|
468
|
+
self._check_int_path()
|
|
469
|
+
|
|
470
|
+
src_path, dst_path = fspath(self.path_without_protocol), fspath(dst_path)
|
|
471
|
+
if os.path.isfile(src_path):
|
|
472
|
+
_fs_rename_file(src_path, dst_path, overwrite)
|
|
473
|
+
return self.from_path(dst_path)
|
|
474
|
+
else:
|
|
475
|
+
os.makedirs(dst_path, exist_ok=True)
|
|
476
|
+
|
|
477
|
+
with os.scandir(src_path) as entries:
|
|
478
|
+
for file_entry in entries:
|
|
479
|
+
src_file_path = file_entry.path
|
|
480
|
+
dst_file_path = dst_path
|
|
481
|
+
relative_path = os.path.relpath(src_file_path, start=src_path)
|
|
482
|
+
if relative_path and relative_path != ".":
|
|
483
|
+
dst_file_path = os.path.join(dst_file_path, relative_path)
|
|
484
|
+
if os.path.exists(dst_file_path) and file_entry.is_dir():
|
|
485
|
+
self.from_path(src_file_path).rename(dst_file_path, overwrite)
|
|
486
|
+
else:
|
|
487
|
+
_fs_rename_file(src_file_path, dst_file_path, overwrite)
|
|
488
|
+
|
|
489
|
+
if os.path.isdir(src_path):
|
|
490
|
+
shutil.rmtree(src_path)
|
|
491
|
+
else:
|
|
492
|
+
os.remove(src_path)
|
|
493
|
+
|
|
625
494
|
return self.from_path(dst_path)
|
|
626
495
|
|
|
627
496
|
def replace(self, dst_path: PathLike, overwrite: bool = True) -> "FSPath":
|
|
@@ -643,9 +512,9 @@ class FSPath(URIPath):
|
|
|
643
512
|
if missing_ok and not self.exists():
|
|
644
513
|
return
|
|
645
514
|
if self.is_dir():
|
|
646
|
-
shutil.rmtree(self.path_without_protocol)
|
|
515
|
+
shutil.rmtree(self.path_without_protocol) # pyre-ignore[6]
|
|
647
516
|
else:
|
|
648
|
-
os.remove(self.path_without_protocol)
|
|
517
|
+
os.remove(self.path_without_protocol) # pyre-ignore[6]
|
|
649
518
|
|
|
650
519
|
def _scan(
|
|
651
520
|
self, missing_ok: bool = True, followlinks: bool = False
|
|
@@ -724,10 +593,10 @@ class FSPath(URIPath):
|
|
|
724
593
|
|
|
725
594
|
:returns: StatResult
|
|
726
595
|
"""
|
|
727
|
-
if follow_symlinks:
|
|
596
|
+
if follow_symlinks or isinstance(self.path_without_protocol, int):
|
|
728
597
|
result = _make_stat(os.stat(self.path_without_protocol))
|
|
729
598
|
else:
|
|
730
|
-
result = _make_stat(os.lstat(self.path_without_protocol))
|
|
599
|
+
result = _make_stat(os.lstat(self.path_without_protocol)) # pyre-ignore[6]
|
|
731
600
|
|
|
732
601
|
if result.islnk or not result.isdir:
|
|
733
602
|
return result
|
|
@@ -735,15 +604,16 @@ class FSPath(URIPath):
|
|
|
735
604
|
size = 0
|
|
736
605
|
ctime = result.ctime
|
|
737
606
|
mtime = result.mtime
|
|
738
|
-
|
|
739
|
-
for
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
ctime
|
|
745
|
-
|
|
746
|
-
mtime
|
|
607
|
+
if not isinstance(self.path_without_protocol, int):
|
|
608
|
+
for root, _, files in os.walk(self.path_without_protocol): # pyre-ignore[6]
|
|
609
|
+
for filename in files:
|
|
610
|
+
canonical_path = os.path.join(root, filename)
|
|
611
|
+
stat = os.lstat(canonical_path)
|
|
612
|
+
size += stat.st_size
|
|
613
|
+
if ctime > stat.st_ctime: # pyre-ignore[16]
|
|
614
|
+
ctime = stat.st_ctime
|
|
615
|
+
if mtime < stat.st_mtime:
|
|
616
|
+
mtime = stat.st_mtime
|
|
747
617
|
return result._replace(size=size, ctime=ctime, mtime=mtime)
|
|
748
618
|
|
|
749
619
|
def unlink(self, missing_ok: bool = False) -> None:
|
|
@@ -752,9 +622,11 @@ class FSPath(URIPath):
|
|
|
752
622
|
|
|
753
623
|
:param missing_ok: if False and target file not exists, raise FileNotFoundError
|
|
754
624
|
"""
|
|
625
|
+
self._check_int_path()
|
|
626
|
+
|
|
755
627
|
if missing_ok and not self.exists():
|
|
756
628
|
return
|
|
757
|
-
os.unlink(self.path_without_protocol)
|
|
629
|
+
os.unlink(self.path_without_protocol) # pyre-ignore[6]
|
|
758
630
|
|
|
759
631
|
def walk(
|
|
760
632
|
self, followlinks: bool = False
|
|
@@ -783,6 +655,8 @@ class FSPath(URIPath):
|
|
|
783
655
|
|
|
784
656
|
:returns: A 3-tuple generator
|
|
785
657
|
"""
|
|
658
|
+
self._check_int_path()
|
|
659
|
+
|
|
786
660
|
if not self.exists(followlinks=followlinks):
|
|
787
661
|
return
|
|
788
662
|
|
|
@@ -790,7 +664,7 @@ class FSPath(URIPath):
|
|
|
790
664
|
return
|
|
791
665
|
|
|
792
666
|
path = fspath(self.path_without_protocol)
|
|
793
|
-
path = os.path.normpath(self.path_without_protocol)
|
|
667
|
+
path = os.path.normpath(self.path_without_protocol) # pyre-ignore[6]
|
|
794
668
|
|
|
795
669
|
stack = [path]
|
|
796
670
|
while stack:
|
|
@@ -820,8 +694,13 @@ class FSPath(URIPath):
|
|
|
820
694
|
eliminating any symbolic links encountered in the path.
|
|
821
695
|
:rtype: FSPath
|
|
822
696
|
"""
|
|
697
|
+
self._check_int_path()
|
|
823
698
|
return self.from_path(
|
|
824
|
-
fspath(
|
|
699
|
+
fspath(
|
|
700
|
+
pathlib.Path(
|
|
701
|
+
self.path_without_protocol # pyre-ignore[6]
|
|
702
|
+
).resolve(strict=strict)
|
|
703
|
+
)
|
|
825
704
|
)
|
|
826
705
|
|
|
827
706
|
def md5(self, recalculate: bool = False, followlinks: bool = True):
|
|
@@ -853,14 +732,29 @@ class FSPath(URIPath):
|
|
|
853
732
|
callback: Optional[Callable[[int], None]] = None,
|
|
854
733
|
followlinks: bool = False,
|
|
855
734
|
):
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
735
|
+
if isinstance(self.path_without_protocol, int):
|
|
736
|
+
with open(fspath(dst_path), "wb") as fdst:
|
|
737
|
+
# This magic number is copied from copyfileobj
|
|
738
|
+
length = 16 * 1024
|
|
739
|
+
while True:
|
|
740
|
+
buf = os.read(self.path_without_protocol, length) # pyre-ignore[6]
|
|
741
|
+
if not buf:
|
|
742
|
+
break
|
|
743
|
+
fdst.write(buf)
|
|
744
|
+
if callback is None:
|
|
745
|
+
continue
|
|
746
|
+
callback(len(buf))
|
|
747
|
+
else:
|
|
748
|
+
shutil.copy2(
|
|
749
|
+
self.path_without_protocol, # pyre-ignore[6]
|
|
750
|
+
fspath(dst_path),
|
|
751
|
+
follow_symlinks=followlinks,
|
|
752
|
+
)
|
|
859
753
|
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
754
|
+
# After python3.8, patch `shutil.copyfile` is not a good way,
|
|
755
|
+
# because `shutil.copy2` will not call it in some cases.
|
|
756
|
+
if callback:
|
|
757
|
+
callback(self.stat(follow_symlinks=followlinks).size)
|
|
864
758
|
|
|
865
759
|
def copy(
|
|
866
760
|
self,
|
|
@@ -938,7 +832,7 @@ class FSPath(URIPath):
|
|
|
938
832
|
return ignore_files
|
|
939
833
|
|
|
940
834
|
shutil.copytree(
|
|
941
|
-
self.path_without_protocol,
|
|
835
|
+
self.path_without_protocol, # pyre-ignore[6]
|
|
942
836
|
dst_path,
|
|
943
837
|
ignore=ignore_same_file,
|
|
944
838
|
dirs_exist_ok=True,
|
|
@@ -952,7 +846,8 @@ class FSPath(URIPath):
|
|
|
952
846
|
|
|
953
847
|
:param dst_path: Destination path
|
|
954
848
|
"""
|
|
955
|
-
|
|
849
|
+
self._check_int_path()
|
|
850
|
+
return os.symlink(self.path_without_protocol, dst_path) # pyre-ignore[6]
|
|
956
851
|
|
|
957
852
|
def readlink(self) -> "FSPath":
|
|
958
853
|
"""
|
|
@@ -962,7 +857,12 @@ class FSPath(URIPath):
|
|
|
962
857
|
:returns: Return a FSPath instance representing the path to which
|
|
963
858
|
the symbolic link points.
|
|
964
859
|
"""
|
|
965
|
-
|
|
860
|
+
self._check_int_path()
|
|
861
|
+
return self.from_path(
|
|
862
|
+
os.readlink(
|
|
863
|
+
self.path_without_protocol # pyre-ignore[6]
|
|
864
|
+
)
|
|
865
|
+
)
|
|
966
866
|
|
|
967
867
|
def is_symlink(self) -> bool:
|
|
968
868
|
"""Test whether a path is a symbolic link
|
|
@@ -984,14 +884,14 @@ class FSPath(URIPath):
|
|
|
984
884
|
|
|
985
885
|
returns: Current working directory
|
|
986
886
|
"""
|
|
987
|
-
return self.from_path(
|
|
887
|
+
return self.from_path(os.getcwd())
|
|
988
888
|
|
|
989
889
|
def home(self):
|
|
990
890
|
"""Return the home directory
|
|
991
891
|
|
|
992
892
|
returns: Home directory path
|
|
993
893
|
"""
|
|
994
|
-
return self.from_path(
|
|
894
|
+
return self.from_path(os.path.expanduser("~"))
|
|
995
895
|
|
|
996
896
|
def joinpath(self, *other_paths: PathLike) -> "FSPath":
|
|
997
897
|
path = fspath(self)
|
|
@@ -1005,9 +905,11 @@ class FSPath(URIPath):
|
|
|
1005
905
|
|
|
1006
906
|
:param file_object: stream to be read
|
|
1007
907
|
"""
|
|
1008
|
-
FSPath(
|
|
1009
|
-
|
|
1010
|
-
|
|
908
|
+
FSPath(
|
|
909
|
+
os.path.dirname(
|
|
910
|
+
self.path_without_protocol # pyre-ignore[6]
|
|
911
|
+
)
|
|
912
|
+
).mkdir(parents=True, exist_ok=True)
|
|
1011
913
|
with open(self.path_without_protocol, "wb") as output:
|
|
1012
914
|
output.write(file_object.read())
|
|
1013
915
|
|
|
@@ -1024,9 +926,11 @@ class FSPath(URIPath):
|
|
|
1024
926
|
if not isinstance(self.path_without_protocol, int) and (
|
|
1025
927
|
"w" in mode or "x" in mode or "a" in mode
|
|
1026
928
|
):
|
|
1027
|
-
FSPath(
|
|
1028
|
-
|
|
1029
|
-
|
|
929
|
+
FSPath(
|
|
930
|
+
os.path.dirname(
|
|
931
|
+
self.path_without_protocol # pyre-ignore[6]
|
|
932
|
+
)
|
|
933
|
+
).mkdir(parents=True, exist_ok=True)
|
|
1030
934
|
return io.open(
|
|
1031
935
|
self.path_without_protocol,
|
|
1032
936
|
mode,
|
|
@@ -1042,7 +946,8 @@ class FSPath(URIPath):
|
|
|
1042
946
|
"""
|
|
1043
947
|
A tuple giving access to the path’s various components
|
|
1044
948
|
"""
|
|
1045
|
-
|
|
949
|
+
self._check_int_path()
|
|
950
|
+
return pathlib.Path(self.path_without_protocol).parts # pyre-ignore[6]
|
|
1046
951
|
|
|
1047
952
|
def chmod(self, mode: int, *, follow_symlinks: bool = True):
|
|
1048
953
|
"""
|
|
@@ -1062,7 +967,8 @@ class FSPath(URIPath):
|
|
|
1062
967
|
Return the name of the group owning the file. KeyError is raised if
|
|
1063
968
|
the file’s gid isn’t found in the system database.
|
|
1064
969
|
"""
|
|
1065
|
-
|
|
970
|
+
self._check_int_path()
|
|
971
|
+
return pathlib.Path(self.path_without_protocol).group() # pyre-ignore[6]
|
|
1066
972
|
|
|
1067
973
|
def is_socket(self) -> bool:
|
|
1068
974
|
"""
|
|
@@ -1072,7 +978,9 @@ class FSPath(URIPath):
|
|
|
1072
978
|
False is also returned if the path doesn’t exist or is a broken symlink;
|
|
1073
979
|
other errors (such as permission errors) are propagated.
|
|
1074
980
|
"""
|
|
1075
|
-
|
|
981
|
+
if isinstance(self.path_without_protocol, int):
|
|
982
|
+
return bool(stat_issock(os.stat(self.path_without_protocol).st_mode))
|
|
983
|
+
return pathlib.Path(self.path_without_protocol).is_socket() # pyre-ignore[6]
|
|
1076
984
|
|
|
1077
985
|
def is_fifo(self) -> bool:
|
|
1078
986
|
"""
|
|
@@ -1082,7 +990,9 @@ class FSPath(URIPath):
|
|
|
1082
990
|
False is also returned if the path doesn’t exist or is a broken symlink;
|
|
1083
991
|
other errors (such as permission errors) are propagated.
|
|
1084
992
|
"""
|
|
1085
|
-
|
|
993
|
+
if isinstance(self.path_without_protocol, int):
|
|
994
|
+
return bool(stat_isfifo(os.stat(self.path_without_protocol).st_mode))
|
|
995
|
+
return pathlib.Path(self.path_without_protocol).is_fifo() # pyre-ignore[6]
|
|
1086
996
|
|
|
1087
997
|
def is_block_device(self) -> bool:
|
|
1088
998
|
"""
|
|
@@ -1092,7 +1002,11 @@ class FSPath(URIPath):
|
|
|
1092
1002
|
False is also returned if the path doesn’t exist or is a broken symlink;
|
|
1093
1003
|
other errors (such as permission errors) are propagated.
|
|
1094
1004
|
"""
|
|
1095
|
-
|
|
1005
|
+
if isinstance(self.path_without_protocol, int):
|
|
1006
|
+
return bool(stat_isblk(os.stat(self.path_without_protocol).st_mode))
|
|
1007
|
+
return pathlib.Path(
|
|
1008
|
+
self.path_without_protocol # pyre-ignore[6]
|
|
1009
|
+
).is_block_device()
|
|
1096
1010
|
|
|
1097
1011
|
def is_char_device(self) -> bool:
|
|
1098
1012
|
"""
|
|
@@ -1102,21 +1016,31 @@ class FSPath(URIPath):
|
|
|
1102
1016
|
False is also returned if the path doesn’t exist or is a broken symlink;
|
|
1103
1017
|
other errors (such as permission errors) are propagated.
|
|
1104
1018
|
"""
|
|
1105
|
-
|
|
1019
|
+
if isinstance(self.path_without_protocol, int):
|
|
1020
|
+
return bool(stat_ischr(os.stat(self.path_without_protocol).st_mode))
|
|
1021
|
+
return pathlib.Path(
|
|
1022
|
+
self.path_without_protocol # pyre-ignore[6]
|
|
1023
|
+
).is_char_device()
|
|
1106
1024
|
|
|
1107
1025
|
def owner(self) -> str:
|
|
1108
1026
|
"""
|
|
1109
1027
|
Return the name of the user owning the file. KeyError is raised if the file’s
|
|
1110
1028
|
uid isn’t found in the system database.
|
|
1111
1029
|
"""
|
|
1112
|
-
|
|
1030
|
+
self._check_int_path()
|
|
1031
|
+
return pathlib.Path(self.path_without_protocol).owner() # pyre-ignore[6]
|
|
1113
1032
|
|
|
1114
1033
|
def absolute(self) -> "FSPath":
|
|
1115
1034
|
"""
|
|
1116
1035
|
Make the path absolute, without normalization or resolving symlinks.
|
|
1117
1036
|
Returns a new path object
|
|
1118
1037
|
"""
|
|
1119
|
-
|
|
1038
|
+
self._check_int_path()
|
|
1039
|
+
return self.from_path(
|
|
1040
|
+
os.path.abspath(
|
|
1041
|
+
self.path_without_protocol # pyre-ignore[6]
|
|
1042
|
+
)
|
|
1043
|
+
)
|
|
1120
1044
|
|
|
1121
1045
|
def rmdir(self):
|
|
1122
1046
|
"""
|