megfile 3.1.6.post1__py3-none-any.whl → 4.0.0.post1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- megfile/cli.py +12 -7
- megfile/config.py +27 -39
- megfile/fs.py +169 -12
- megfile/fs_path.py +183 -260
- megfile/hdfs.py +106 -5
- megfile/hdfs_path.py +34 -90
- megfile/http.py +50 -1
- megfile/http_path.py +27 -65
- megfile/interfaces.py +1 -8
- megfile/lib/base_prefetch_reader.py +62 -78
- megfile/lib/combine_reader.py +5 -0
- megfile/lib/glob.py +3 -6
- megfile/lib/hdfs_prefetch_reader.py +7 -7
- megfile/lib/http_prefetch_reader.py +6 -6
- megfile/lib/s3_buffered_writer.py +71 -65
- megfile/lib/s3_cached_handler.py +1 -2
- megfile/lib/s3_limited_seekable_writer.py +3 -7
- megfile/lib/s3_memory_handler.py +1 -2
- megfile/lib/s3_pipe_handler.py +1 -2
- megfile/lib/s3_prefetch_reader.py +10 -19
- megfile/lib/s3_share_cache_reader.py +8 -5
- megfile/pathlike.py +397 -401
- megfile/s3.py +118 -17
- megfile/s3_path.py +126 -209
- megfile/sftp.py +300 -10
- megfile/sftp_path.py +46 -322
- megfile/smart.py +33 -27
- megfile/smart_path.py +9 -14
- megfile/stdio.py +1 -1
- megfile/stdio_path.py +2 -2
- megfile/utils/__init__.py +3 -4
- megfile/version.py +1 -1
- {megfile-3.1.6.post1.dist-info → megfile-4.0.0.post1.dist-info}/METADATA +7 -7
- megfile-4.0.0.post1.dist-info/RECORD +52 -0
- {megfile-3.1.6.post1.dist-info → megfile-4.0.0.post1.dist-info}/WHEEL +1 -1
- {megfile-3.1.6.post1.dist-info → megfile-4.0.0.post1.dist-info}/top_level.txt +0 -2
- docs/conf.py +0 -65
- megfile-3.1.6.post1.dist-info/RECORD +0 -55
- scripts/convert_results_to_sarif.py +0 -91
- scripts/generate_file.py +0 -344
- {megfile-3.1.6.post1.dist-info → megfile-4.0.0.post1.dist-info}/LICENSE +0 -0
- {megfile-3.1.6.post1.dist-info → megfile-4.0.0.post1.dist-info}/LICENSE.pyre +0 -0
- {megfile-3.1.6.post1.dist-info → megfile-4.0.0.post1.dist-info}/entry_points.txt +0 -0
megfile/fs_path.py
CHANGED
|
@@ -4,8 +4,12 @@ import os
|
|
|
4
4
|
import pathlib
|
|
5
5
|
import shutil
|
|
6
6
|
from functools import cached_property
|
|
7
|
+
from stat import S_ISBLK as stat_isblk
|
|
8
|
+
from stat import S_ISCHR as stat_ischr
|
|
7
9
|
from stat import S_ISDIR as stat_isdir
|
|
10
|
+
from stat import S_ISFIFO as stat_isfifo
|
|
8
11
|
from stat import S_ISLNK as stat_islnk
|
|
12
|
+
from stat import S_ISSOCK as stat_issock
|
|
9
13
|
from typing import IO, BinaryIO, Callable, Iterator, List, Optional, Tuple, Union
|
|
10
14
|
|
|
11
15
|
from megfile.errors import _create_missing_ok_generator
|
|
@@ -29,25 +33,13 @@ __all__ = [
|
|
|
29
33
|
"FSPath",
|
|
30
34
|
"is_fs",
|
|
31
35
|
"fs_path_join",
|
|
32
|
-
"_make_stat",
|
|
33
|
-
"fs_readlink",
|
|
34
|
-
"fs_cwd",
|
|
35
|
-
"fs_home",
|
|
36
|
-
"fs_iglob",
|
|
37
|
-
"fs_glob",
|
|
38
|
-
"fs_glob_stat",
|
|
39
|
-
"fs_rename",
|
|
40
|
-
"fs_resolve",
|
|
41
|
-
"fs_move",
|
|
42
|
-
"fs_makedirs",
|
|
43
|
-
"fs_lstat",
|
|
44
36
|
]
|
|
45
37
|
|
|
46
38
|
|
|
47
39
|
def _make_stat(stat: os.stat_result) -> StatResult:
|
|
48
40
|
return StatResult(
|
|
49
41
|
size=stat.st_size,
|
|
50
|
-
ctime=stat.st_ctime,
|
|
42
|
+
ctime=stat.st_ctime, # pyre-ignore[16]
|
|
51
43
|
mtime=stat.st_mtime,
|
|
52
44
|
isdir=stat_isdir(stat.st_mode),
|
|
53
45
|
islnk=stat_islnk(stat.st_mode),
|
|
@@ -72,121 +64,6 @@ def fs_path_join(path: PathLike, *other_paths: PathLike) -> str:
|
|
|
72
64
|
return path_join(fspath(path), *map(fspath, other_paths))
|
|
73
65
|
|
|
74
66
|
|
|
75
|
-
def fs_readlink(path) -> str:
|
|
76
|
-
"""
|
|
77
|
-
Return a string representing the path to which the symbolic link points.
|
|
78
|
-
:returns: Return a string representing the path to which the symbolic link points.
|
|
79
|
-
"""
|
|
80
|
-
return os.readlink(path)
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
def fs_cwd() -> str:
|
|
84
|
-
"""Return current working directory
|
|
85
|
-
|
|
86
|
-
returns: Current working directory
|
|
87
|
-
"""
|
|
88
|
-
return os.getcwd()
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
def fs_home():
|
|
92
|
-
"""Return the home directory
|
|
93
|
-
|
|
94
|
-
returns: Home directory path
|
|
95
|
-
"""
|
|
96
|
-
return os.path.expanduser("~")
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
def fs_iglob(
|
|
100
|
-
path: PathLike, recursive: bool = True, missing_ok: bool = True
|
|
101
|
-
) -> Iterator[str]:
|
|
102
|
-
"""Return path iterator in ascending alphabetical order,
|
|
103
|
-
in which path matches glob pattern
|
|
104
|
-
|
|
105
|
-
1. If doesn't match any path, return empty list
|
|
106
|
-
Notice: ``glob.glob`` in standard library returns ['a/'] instead of empty list
|
|
107
|
-
when pathname is like `a/**`, recursive is True and directory 'a' doesn't exist.
|
|
108
|
-
fs_glob behaves like ``glob.glob`` in standard library under such circumstance.
|
|
109
|
-
2. No guarantee that each path in result is different, which means:
|
|
110
|
-
Assume there exists a path `/a/b/c/b/d.txt`
|
|
111
|
-
use path pattern like `/**/b/**/*.txt` to glob,
|
|
112
|
-
the path above will be returned twice
|
|
113
|
-
3. `**` will match any matched file, directory, symlink and '' by default,
|
|
114
|
-
when recursive is `True`
|
|
115
|
-
4. fs_glob returns same as glob.glob(pathname, recursive=True)
|
|
116
|
-
in ascending alphabetical order.
|
|
117
|
-
5. Hidden files (filename stars with '.') will not be found in the result
|
|
118
|
-
|
|
119
|
-
:param recursive: If False, `**` will not search directory recursively
|
|
120
|
-
:param missing_ok: If False and target path doesn't match any file,
|
|
121
|
-
raise FileNotFoundError
|
|
122
|
-
:returns: An iterator contains paths match `pathname`
|
|
123
|
-
"""
|
|
124
|
-
for path in _create_missing_ok_generator(
|
|
125
|
-
iglob(fspath(path), recursive=recursive),
|
|
126
|
-
missing_ok,
|
|
127
|
-
FileNotFoundError("No match any file: %r" % path),
|
|
128
|
-
):
|
|
129
|
-
yield path
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
def fs_glob(
|
|
133
|
-
path: PathLike, recursive: bool = True, missing_ok: bool = True
|
|
134
|
-
) -> List[str]:
|
|
135
|
-
"""Return path list in ascending alphabetical order,
|
|
136
|
-
in which path matches glob pattern
|
|
137
|
-
|
|
138
|
-
1. If doesn't match any path, return empty list
|
|
139
|
-
Notice: ``glob.glob`` in standard library returns ['a/'] instead of empty list
|
|
140
|
-
when pathname is like `a/**`, recursive is True and directory 'a' doesn't exist.
|
|
141
|
-
fs_glob behaves like ``glob.glob`` in standard library under such circumstance.
|
|
142
|
-
2. No guarantee that each path in result is different, which means:
|
|
143
|
-
Assume there exists a path `/a/b/c/b/d.txt`
|
|
144
|
-
use path pattern like `/**/b/**/*.txt` to glob,
|
|
145
|
-
the path above will be returned twice
|
|
146
|
-
3. `**` will match any matched file, directory, symlink and '' by default,
|
|
147
|
-
when recursive is `True`
|
|
148
|
-
4. fs_glob returns same as glob.glob(pathname, recursive=True)
|
|
149
|
-
in ascending alphabetical order.
|
|
150
|
-
5. Hidden files (filename stars with '.') will not be found in the result
|
|
151
|
-
|
|
152
|
-
:param recursive: If False, `**` will not search directory recursively
|
|
153
|
-
:param missing_ok: If False and target path doesn't match any file,
|
|
154
|
-
raise FileNotFoundError
|
|
155
|
-
:returns: A list contains paths match `pathname`
|
|
156
|
-
"""
|
|
157
|
-
return list(fs_iglob(path=path, recursive=recursive, missing_ok=missing_ok))
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
def fs_glob_stat(
|
|
161
|
-
path: PathLike, recursive: bool = True, missing_ok: bool = True
|
|
162
|
-
) -> Iterator[FileEntry]:
|
|
163
|
-
"""Return a list contains tuples of path and file stat,
|
|
164
|
-
in ascending alphabetical order, in which path matches glob pattern
|
|
165
|
-
|
|
166
|
-
1. If doesn't match any path, return empty list
|
|
167
|
-
Notice: ``glob.glob`` in standard library returns ['a/'] instead of empty list
|
|
168
|
-
when pathname is like `a/**`, recursive is True and directory 'a' doesn't exist.
|
|
169
|
-
fs_glob behaves like ``glob.glob`` in standard library under such circumstance.
|
|
170
|
-
2. No guarantee that each path in result is different, which means:
|
|
171
|
-
Assume there exists a path `/a/b/c/b/d.txt`
|
|
172
|
-
use path pattern like `/**/b/**/*.txt` to glob,
|
|
173
|
-
the path above will be returned twice.
|
|
174
|
-
3. `**` will match any matched file, directory, symlink and '' by default,
|
|
175
|
-
when recursive is `True`
|
|
176
|
-
4. fs_glob returns same as glob.glob(pathname, recursive=True)
|
|
177
|
-
in ascending alphabetical order.
|
|
178
|
-
5. Hidden files (filename stars with '.') will not be found in the result
|
|
179
|
-
|
|
180
|
-
:param recursive: If False, `**` will not search directory recursively
|
|
181
|
-
:param missing_ok: If False and target path doesn't match any file,
|
|
182
|
-
raise FileNotFoundError
|
|
183
|
-
:returns: A list contains tuples of path and file stat,
|
|
184
|
-
in which paths match `pathname`
|
|
185
|
-
"""
|
|
186
|
-
for path in fs_iglob(path=path, recursive=recursive, missing_ok=missing_ok):
|
|
187
|
-
yield FileEntry(os.path.basename(path), path, _make_stat(os.lstat(path)))
|
|
188
|
-
|
|
189
|
-
|
|
190
67
|
def _fs_rename_file(
|
|
191
68
|
src_path: PathLike, dst_path: PathLike, overwrite: bool = True
|
|
192
69
|
) -> None:
|
|
@@ -208,82 +85,6 @@ def _fs_rename_file(
|
|
|
208
85
|
shutil.move(src_path, dst_path)
|
|
209
86
|
|
|
210
87
|
|
|
211
|
-
def fs_rename(src_path: PathLike, dst_path: PathLike, overwrite: bool = True) -> None:
|
|
212
|
-
"""
|
|
213
|
-
rename file on fs
|
|
214
|
-
|
|
215
|
-
:param src_path: Given path
|
|
216
|
-
:param dst_path: Given destination path
|
|
217
|
-
:param overwrite: whether or not overwrite file when exists
|
|
218
|
-
"""
|
|
219
|
-
src_path, dst_path = fspath(src_path), fspath(dst_path)
|
|
220
|
-
if os.path.isfile(src_path):
|
|
221
|
-
return _fs_rename_file(src_path, dst_path, overwrite)
|
|
222
|
-
else:
|
|
223
|
-
os.makedirs(dst_path, exist_ok=True)
|
|
224
|
-
|
|
225
|
-
with os.scandir(src_path) as entries:
|
|
226
|
-
for file_entry in entries:
|
|
227
|
-
src_file_path = file_entry.path
|
|
228
|
-
dst_file_path = dst_path
|
|
229
|
-
relative_path = os.path.relpath(src_file_path, start=src_path)
|
|
230
|
-
if relative_path and relative_path != ".":
|
|
231
|
-
dst_file_path = os.path.join(dst_file_path, relative_path)
|
|
232
|
-
if os.path.exists(dst_file_path) and file_entry.is_dir():
|
|
233
|
-
fs_rename(src_file_path, dst_file_path, overwrite)
|
|
234
|
-
else:
|
|
235
|
-
_fs_rename_file(src_file_path, dst_file_path, overwrite)
|
|
236
|
-
|
|
237
|
-
if os.path.isdir(src_path):
|
|
238
|
-
shutil.rmtree(src_path)
|
|
239
|
-
else:
|
|
240
|
-
os.remove(src_path)
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
def fs_move(src_path: PathLike, dst_path: PathLike, overwrite: bool = True) -> None:
|
|
244
|
-
"""
|
|
245
|
-
rename file on fs
|
|
246
|
-
|
|
247
|
-
:param src_path: Given path
|
|
248
|
-
:param dst_path: Given destination path
|
|
249
|
-
:param overwrite: whether or not overwrite file when exists
|
|
250
|
-
"""
|
|
251
|
-
return fs_rename(src_path, dst_path, overwrite)
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
def fs_resolve(path: PathLike) -> str:
|
|
255
|
-
"""Equal to fs_realpath, return the real path of given path
|
|
256
|
-
|
|
257
|
-
:param path: Given path
|
|
258
|
-
:returns: Real path of given path
|
|
259
|
-
"""
|
|
260
|
-
return FSPath(path).realpath()
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
def fs_makedirs(path: PathLike, exist_ok: bool = False):
|
|
264
|
-
"""
|
|
265
|
-
make a directory on fs, including parent directory
|
|
266
|
-
|
|
267
|
-
If there exists a file on the path, raise FileExistsError
|
|
268
|
-
|
|
269
|
-
:param path: Given path
|
|
270
|
-
:param exist_ok: If False and target directory exists, raise FileExistsError
|
|
271
|
-
:raises: FileExistsError
|
|
272
|
-
"""
|
|
273
|
-
return FSPath(path).mkdir(parents=True, exist_ok=exist_ok)
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
def fs_lstat(path: PathLike) -> StatResult:
|
|
277
|
-
"""
|
|
278
|
-
Like Path.stat() but, if the path points to a symbolic link,
|
|
279
|
-
return the symbolic link’s information rather than its target’s.
|
|
280
|
-
|
|
281
|
-
:param path: Given path
|
|
282
|
-
:returns: StatResult
|
|
283
|
-
"""
|
|
284
|
-
return FSPath(path).lstat()
|
|
285
|
-
|
|
286
|
-
|
|
287
88
|
@SmartPath.register
|
|
288
89
|
class FSPath(URIPath):
|
|
289
90
|
"""file protocol
|
|
@@ -299,27 +100,39 @@ class FSPath(URIPath):
|
|
|
299
100
|
path = str(path)
|
|
300
101
|
self.path = path
|
|
301
102
|
|
|
103
|
+
def _check_int_path(self) -> None:
|
|
104
|
+
if isinstance(self.path_without_protocol, int):
|
|
105
|
+
raise TypeError("not support the path of int type")
|
|
106
|
+
|
|
302
107
|
def __fspath__(self) -> str:
|
|
303
|
-
|
|
108
|
+
self._check_int_path()
|
|
109
|
+
return os.path.normpath(self.path_without_protocol) # pyre-ignore[6]
|
|
304
110
|
|
|
305
111
|
@cached_property
|
|
306
112
|
def root(self) -> str:
|
|
307
|
-
|
|
113
|
+
if isinstance(self.path_without_protocol, int):
|
|
114
|
+
return "/"
|
|
115
|
+
return pathlib.Path(self.path_without_protocol).root # pyre-ignore[6]
|
|
308
116
|
|
|
309
117
|
@cached_property
|
|
310
118
|
def anchor(self) -> str:
|
|
311
|
-
|
|
119
|
+
if isinstance(self.path_without_protocol, int):
|
|
120
|
+
return "/"
|
|
121
|
+
return pathlib.Path(self.path_without_protocol).anchor # pyre-ignore[6]
|
|
312
122
|
|
|
313
123
|
@cached_property
|
|
314
124
|
def drive(self) -> str:
|
|
315
|
-
|
|
125
|
+
if isinstance(self.path_without_protocol, int):
|
|
126
|
+
return ""
|
|
127
|
+
return pathlib.Path(self.path_without_protocol).drive # pyre-ignore[6]
|
|
316
128
|
|
|
317
129
|
@classmethod
|
|
318
130
|
def from_uri(cls, path: PathLike) -> "FSPath":
|
|
319
131
|
return cls.from_path(path)
|
|
320
132
|
|
|
321
|
-
@
|
|
133
|
+
@cached_property
|
|
322
134
|
def path_with_protocol(self) -> Union[str, int]:
|
|
135
|
+
"""Return path with protocol, like file:///root"""
|
|
323
136
|
if isinstance(self.path, int):
|
|
324
137
|
return self.path
|
|
325
138
|
protocol_prefix = self.protocol + "://"
|
|
@@ -327,19 +140,32 @@ class FSPath(URIPath):
|
|
|
327
140
|
return self.path # pyre-ignore[7]
|
|
328
141
|
return protocol_prefix + self.path # pyre-ignore[58]
|
|
329
142
|
|
|
143
|
+
@cached_property
|
|
144
|
+
def path_without_protocol(self) -> Union[str, int]:
|
|
145
|
+
"""
|
|
146
|
+
Return path without protocol, example: if path is file:///root,
|
|
147
|
+
return /root
|
|
148
|
+
"""
|
|
149
|
+
if isinstance(self.path, int):
|
|
150
|
+
return self.path
|
|
151
|
+
return super().path_without_protocol
|
|
152
|
+
|
|
330
153
|
def is_absolute(self) -> bool:
|
|
331
154
|
"""Test whether a path is absolute
|
|
332
155
|
|
|
333
156
|
:returns: True if a path is absolute, else False
|
|
334
157
|
"""
|
|
335
|
-
|
|
158
|
+
if isinstance(self.path_without_protocol, int):
|
|
159
|
+
return False
|
|
160
|
+
return os.path.isabs(self.path_without_protocol) # pyre-ignore[6]
|
|
336
161
|
|
|
337
162
|
def abspath(self) -> str:
|
|
338
163
|
"""Return the absolute path of given path
|
|
339
164
|
|
|
340
165
|
:returns: Absolute path of given path
|
|
341
166
|
"""
|
|
342
|
-
|
|
167
|
+
self._check_int_path()
|
|
168
|
+
return fspath(os.path.abspath(self.path_without_protocol)) # pyre-ignore[6]
|
|
343
169
|
|
|
344
170
|
def access(self, mode: Access = Access.READ) -> bool:
|
|
345
171
|
"""
|
|
@@ -512,7 +338,12 @@ class FSPath(URIPath):
|
|
|
512
338
|
glob_path = self.path_without_protocol
|
|
513
339
|
if pattern:
|
|
514
340
|
glob_path = self.joinpath(pattern).path_without_protocol
|
|
515
|
-
|
|
341
|
+
|
|
342
|
+
for path in _create_missing_ok_generator(
|
|
343
|
+
iglob(fspath(glob_path), recursive=recursive),
|
|
344
|
+
missing_ok,
|
|
345
|
+
FileNotFoundError("No match any file: %r" % glob_path),
|
|
346
|
+
):
|
|
516
347
|
yield self.from_path(path)
|
|
517
348
|
|
|
518
349
|
def is_dir(self, followlinks: bool = False) -> bool:
|
|
@@ -556,7 +387,8 @@ class FSPath(URIPath):
|
|
|
556
387
|
|
|
557
388
|
:returns: All contents have in the path in ascending alphabetical order
|
|
558
389
|
"""
|
|
559
|
-
|
|
390
|
+
self._check_int_path()
|
|
391
|
+
return sorted(os.listdir(self.path_without_protocol)) # pyre-ignore[6]
|
|
560
392
|
|
|
561
393
|
def iterdir(self) -> Iterator["FSPath"]:
|
|
562
394
|
"""
|
|
@@ -593,9 +425,13 @@ class FSPath(URIPath):
|
|
|
593
425
|
|
|
594
426
|
:raises: FileExistsError
|
|
595
427
|
"""
|
|
596
|
-
if exist_ok and
|
|
428
|
+
if exist_ok and (
|
|
429
|
+
self.path_without_protocol == ""
|
|
430
|
+
or isinstance(self.path_without_protocol, int)
|
|
431
|
+
):
|
|
597
432
|
return
|
|
598
|
-
|
|
433
|
+
self._check_int_path()
|
|
434
|
+
return pathlib.Path(self.path_without_protocol).mkdir( # pyre-ignore[6]
|
|
599
435
|
mode=mode, parents=parents, exist_ok=exist_ok
|
|
600
436
|
)
|
|
601
437
|
|
|
@@ -604,7 +440,8 @@ class FSPath(URIPath):
|
|
|
604
440
|
|
|
605
441
|
:returns: Real path of given path
|
|
606
442
|
"""
|
|
607
|
-
|
|
443
|
+
self._check_int_path()
|
|
444
|
+
return fspath(os.path.realpath(self.path_without_protocol)) # pyre-ignore[6]
|
|
608
445
|
|
|
609
446
|
def relpath(self, start: Optional[str] = None) -> str:
|
|
610
447
|
"""Return the relative path of given path
|
|
@@ -612,7 +449,13 @@ class FSPath(URIPath):
|
|
|
612
449
|
:param start: Given start directory
|
|
613
450
|
:returns: Relative path from start
|
|
614
451
|
"""
|
|
615
|
-
|
|
452
|
+
self._check_int_path()
|
|
453
|
+
return fspath(
|
|
454
|
+
os.path.relpath(
|
|
455
|
+
self.path_without_protocol, # pyre-ignore[6]
|
|
456
|
+
start=start,
|
|
457
|
+
)
|
|
458
|
+
)
|
|
616
459
|
|
|
617
460
|
def rename(self, dst_path: PathLike, overwrite: bool = True) -> "FSPath":
|
|
618
461
|
"""
|
|
@@ -621,7 +464,32 @@ class FSPath(URIPath):
|
|
|
621
464
|
:param dst_path: Given destination path
|
|
622
465
|
:param overwrite: whether or not overwrite file when exists
|
|
623
466
|
"""
|
|
624
|
-
|
|
467
|
+
self._check_int_path()
|
|
468
|
+
|
|
469
|
+
src_path, dst_path = fspath(self.path_without_protocol), fspath(dst_path)
|
|
470
|
+
if os.path.isfile(src_path):
|
|
471
|
+
_fs_rename_file(src_path, dst_path, overwrite)
|
|
472
|
+
return self.from_path(dst_path)
|
|
473
|
+
else:
|
|
474
|
+
os.makedirs(dst_path, exist_ok=True)
|
|
475
|
+
|
|
476
|
+
with os.scandir(src_path) as entries:
|
|
477
|
+
for file_entry in entries:
|
|
478
|
+
src_file_path = file_entry.path
|
|
479
|
+
dst_file_path = dst_path
|
|
480
|
+
relative_path = os.path.relpath(src_file_path, start=src_path)
|
|
481
|
+
if relative_path and relative_path != ".":
|
|
482
|
+
dst_file_path = os.path.join(dst_file_path, relative_path)
|
|
483
|
+
if os.path.exists(dst_file_path) and file_entry.is_dir():
|
|
484
|
+
self.from_path(src_file_path).rename(dst_file_path, overwrite)
|
|
485
|
+
else:
|
|
486
|
+
_fs_rename_file(src_file_path, dst_file_path, overwrite)
|
|
487
|
+
|
|
488
|
+
if os.path.isdir(src_path):
|
|
489
|
+
shutil.rmtree(src_path)
|
|
490
|
+
else:
|
|
491
|
+
os.remove(src_path)
|
|
492
|
+
|
|
625
493
|
return self.from_path(dst_path)
|
|
626
494
|
|
|
627
495
|
def replace(self, dst_path: PathLike, overwrite: bool = True) -> "FSPath":
|
|
@@ -643,9 +511,9 @@ class FSPath(URIPath):
|
|
|
643
511
|
if missing_ok and not self.exists():
|
|
644
512
|
return
|
|
645
513
|
if self.is_dir():
|
|
646
|
-
shutil.rmtree(self.path_without_protocol)
|
|
514
|
+
shutil.rmtree(self.path_without_protocol) # pyre-ignore[6]
|
|
647
515
|
else:
|
|
648
|
-
os.remove(self.path_without_protocol)
|
|
516
|
+
os.remove(self.path_without_protocol) # pyre-ignore[6]
|
|
649
517
|
|
|
650
518
|
def _scan(
|
|
651
519
|
self, missing_ok: bool = True, followlinks: bool = False
|
|
@@ -724,10 +592,10 @@ class FSPath(URIPath):
|
|
|
724
592
|
|
|
725
593
|
:returns: StatResult
|
|
726
594
|
"""
|
|
727
|
-
if follow_symlinks:
|
|
595
|
+
if follow_symlinks or isinstance(self.path_without_protocol, int):
|
|
728
596
|
result = _make_stat(os.stat(self.path_without_protocol))
|
|
729
597
|
else:
|
|
730
|
-
result = _make_stat(os.lstat(self.path_without_protocol))
|
|
598
|
+
result = _make_stat(os.lstat(self.path_without_protocol)) # pyre-ignore[6]
|
|
731
599
|
|
|
732
600
|
if result.islnk or not result.isdir:
|
|
733
601
|
return result
|
|
@@ -735,15 +603,16 @@ class FSPath(URIPath):
|
|
|
735
603
|
size = 0
|
|
736
604
|
ctime = result.ctime
|
|
737
605
|
mtime = result.mtime
|
|
738
|
-
|
|
739
|
-
for
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
ctime
|
|
745
|
-
|
|
746
|
-
mtime
|
|
606
|
+
if not isinstance(self.path_without_protocol, int):
|
|
607
|
+
for root, _, files in os.walk(self.path_without_protocol): # pyre-ignore[6]
|
|
608
|
+
for filename in files:
|
|
609
|
+
canonical_path = os.path.join(root, filename)
|
|
610
|
+
stat = os.lstat(canonical_path)
|
|
611
|
+
size += stat.st_size
|
|
612
|
+
if ctime > stat.st_ctime: # pyre-ignore[16]
|
|
613
|
+
ctime = stat.st_ctime
|
|
614
|
+
if mtime < stat.st_mtime:
|
|
615
|
+
mtime = stat.st_mtime
|
|
747
616
|
return result._replace(size=size, ctime=ctime, mtime=mtime)
|
|
748
617
|
|
|
749
618
|
def unlink(self, missing_ok: bool = False) -> None:
|
|
@@ -752,9 +621,11 @@ class FSPath(URIPath):
|
|
|
752
621
|
|
|
753
622
|
:param missing_ok: if False and target file not exists, raise FileNotFoundError
|
|
754
623
|
"""
|
|
624
|
+
self._check_int_path()
|
|
625
|
+
|
|
755
626
|
if missing_ok and not self.exists():
|
|
756
627
|
return
|
|
757
|
-
os.unlink(self.path_without_protocol)
|
|
628
|
+
os.unlink(self.path_without_protocol) # pyre-ignore[6]
|
|
758
629
|
|
|
759
630
|
def walk(
|
|
760
631
|
self, followlinks: bool = False
|
|
@@ -783,6 +654,8 @@ class FSPath(URIPath):
|
|
|
783
654
|
|
|
784
655
|
:returns: A 3-tuple generator
|
|
785
656
|
"""
|
|
657
|
+
self._check_int_path()
|
|
658
|
+
|
|
786
659
|
if not self.exists(followlinks=followlinks):
|
|
787
660
|
return
|
|
788
661
|
|
|
@@ -790,7 +663,7 @@ class FSPath(URIPath):
|
|
|
790
663
|
return
|
|
791
664
|
|
|
792
665
|
path = fspath(self.path_without_protocol)
|
|
793
|
-
path = os.path.normpath(self.path_without_protocol)
|
|
666
|
+
path = os.path.normpath(self.path_without_protocol) # pyre-ignore[6]
|
|
794
667
|
|
|
795
668
|
stack = [path]
|
|
796
669
|
while stack:
|
|
@@ -820,8 +693,13 @@ class FSPath(URIPath):
|
|
|
820
693
|
eliminating any symbolic links encountered in the path.
|
|
821
694
|
:rtype: FSPath
|
|
822
695
|
"""
|
|
696
|
+
self._check_int_path()
|
|
823
697
|
return self.from_path(
|
|
824
|
-
fspath(
|
|
698
|
+
fspath(
|
|
699
|
+
pathlib.Path(
|
|
700
|
+
self.path_without_protocol # pyre-ignore[6]
|
|
701
|
+
).resolve(strict=strict)
|
|
702
|
+
)
|
|
825
703
|
)
|
|
826
704
|
|
|
827
705
|
def md5(self, recalculate: bool = False, followlinks: bool = True):
|
|
@@ -853,14 +731,29 @@ class FSPath(URIPath):
|
|
|
853
731
|
callback: Optional[Callable[[int], None]] = None,
|
|
854
732
|
followlinks: bool = False,
|
|
855
733
|
):
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
734
|
+
if isinstance(self.path_without_protocol, int):
|
|
735
|
+
with open(fspath(dst_path), "wb") as fdst:
|
|
736
|
+
# This magic number is copied from copyfileobj
|
|
737
|
+
length = 16 * 1024
|
|
738
|
+
while True:
|
|
739
|
+
buf = os.read(self.path_without_protocol, length) # pyre-ignore[6]
|
|
740
|
+
if not buf:
|
|
741
|
+
break
|
|
742
|
+
fdst.write(buf)
|
|
743
|
+
if callback is None:
|
|
744
|
+
continue
|
|
745
|
+
callback(len(buf))
|
|
746
|
+
else:
|
|
747
|
+
shutil.copy2(
|
|
748
|
+
self.path_without_protocol, # pyre-ignore[6]
|
|
749
|
+
fspath(dst_path),
|
|
750
|
+
follow_symlinks=followlinks,
|
|
751
|
+
)
|
|
859
752
|
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
753
|
+
# After python3.8, patch `shutil.copyfile` is not a good way,
|
|
754
|
+
# because `shutil.copy2` will not call it in some cases.
|
|
755
|
+
if callback:
|
|
756
|
+
callback(self.stat(follow_symlinks=followlinks).size)
|
|
864
757
|
|
|
865
758
|
def copy(
|
|
866
759
|
self,
|
|
@@ -938,7 +831,7 @@ class FSPath(URIPath):
|
|
|
938
831
|
return ignore_files
|
|
939
832
|
|
|
940
833
|
shutil.copytree(
|
|
941
|
-
self.path_without_protocol,
|
|
834
|
+
self.path_without_protocol, # pyre-ignore[6]
|
|
942
835
|
dst_path,
|
|
943
836
|
ignore=ignore_same_file,
|
|
944
837
|
dirs_exist_ok=True,
|
|
@@ -952,7 +845,8 @@ class FSPath(URIPath):
|
|
|
952
845
|
|
|
953
846
|
:param dst_path: Destination path
|
|
954
847
|
"""
|
|
955
|
-
|
|
848
|
+
self._check_int_path()
|
|
849
|
+
return os.symlink(self.path_without_protocol, dst_path) # pyre-ignore[6]
|
|
956
850
|
|
|
957
851
|
def readlink(self) -> "FSPath":
|
|
958
852
|
"""
|
|
@@ -962,7 +856,12 @@ class FSPath(URIPath):
|
|
|
962
856
|
:returns: Return a FSPath instance representing the path to which
|
|
963
857
|
the symbolic link points.
|
|
964
858
|
"""
|
|
965
|
-
|
|
859
|
+
self._check_int_path()
|
|
860
|
+
return self.from_path(
|
|
861
|
+
os.readlink(
|
|
862
|
+
self.path_without_protocol # pyre-ignore[6]
|
|
863
|
+
)
|
|
864
|
+
)
|
|
966
865
|
|
|
967
866
|
def is_symlink(self) -> bool:
|
|
968
867
|
"""Test whether a path is a symbolic link
|
|
@@ -984,14 +883,14 @@ class FSPath(URIPath):
|
|
|
984
883
|
|
|
985
884
|
returns: Current working directory
|
|
986
885
|
"""
|
|
987
|
-
return self.from_path(
|
|
886
|
+
return self.from_path(os.getcwd())
|
|
988
887
|
|
|
989
888
|
def home(self):
|
|
990
889
|
"""Return the home directory
|
|
991
890
|
|
|
992
891
|
returns: Home directory path
|
|
993
892
|
"""
|
|
994
|
-
return self.from_path(
|
|
893
|
+
return self.from_path(os.path.expanduser("~"))
|
|
995
894
|
|
|
996
895
|
def joinpath(self, *other_paths: PathLike) -> "FSPath":
|
|
997
896
|
path = fspath(self)
|
|
@@ -1005,9 +904,11 @@ class FSPath(URIPath):
|
|
|
1005
904
|
|
|
1006
905
|
:param file_object: stream to be read
|
|
1007
906
|
"""
|
|
1008
|
-
FSPath(
|
|
1009
|
-
|
|
1010
|
-
|
|
907
|
+
FSPath(
|
|
908
|
+
os.path.dirname(
|
|
909
|
+
self.path_without_protocol # pyre-ignore[6]
|
|
910
|
+
)
|
|
911
|
+
).mkdir(parents=True, exist_ok=True)
|
|
1011
912
|
with open(self.path_without_protocol, "wb") as output:
|
|
1012
913
|
output.write(file_object.read())
|
|
1013
914
|
|
|
@@ -1024,9 +925,11 @@ class FSPath(URIPath):
|
|
|
1024
925
|
if not isinstance(self.path_without_protocol, int) and (
|
|
1025
926
|
"w" in mode or "x" in mode or "a" in mode
|
|
1026
927
|
):
|
|
1027
|
-
FSPath(
|
|
1028
|
-
|
|
1029
|
-
|
|
928
|
+
FSPath(
|
|
929
|
+
os.path.dirname(
|
|
930
|
+
self.path_without_protocol # pyre-ignore[6]
|
|
931
|
+
)
|
|
932
|
+
).mkdir(parents=True, exist_ok=True)
|
|
1030
933
|
return io.open(
|
|
1031
934
|
self.path_without_protocol,
|
|
1032
935
|
mode,
|
|
@@ -1042,7 +945,8 @@ class FSPath(URIPath):
|
|
|
1042
945
|
"""
|
|
1043
946
|
A tuple giving access to the path’s various components
|
|
1044
947
|
"""
|
|
1045
|
-
|
|
948
|
+
self._check_int_path()
|
|
949
|
+
return pathlib.Path(self.path_without_protocol).parts # pyre-ignore[6]
|
|
1046
950
|
|
|
1047
951
|
def chmod(self, mode: int, *, follow_symlinks: bool = True):
|
|
1048
952
|
"""
|
|
@@ -1062,7 +966,8 @@ class FSPath(URIPath):
|
|
|
1062
966
|
Return the name of the group owning the file. KeyError is raised if
|
|
1063
967
|
the file’s gid isn’t found in the system database.
|
|
1064
968
|
"""
|
|
1065
|
-
|
|
969
|
+
self._check_int_path()
|
|
970
|
+
return pathlib.Path(self.path_without_protocol).group() # pyre-ignore[6]
|
|
1066
971
|
|
|
1067
972
|
def is_socket(self) -> bool:
|
|
1068
973
|
"""
|
|
@@ -1072,7 +977,9 @@ class FSPath(URIPath):
|
|
|
1072
977
|
False is also returned if the path doesn’t exist or is a broken symlink;
|
|
1073
978
|
other errors (such as permission errors) are propagated.
|
|
1074
979
|
"""
|
|
1075
|
-
|
|
980
|
+
if isinstance(self.path_without_protocol, int):
|
|
981
|
+
return bool(stat_issock(os.stat(self.path_without_protocol).st_mode))
|
|
982
|
+
return pathlib.Path(self.path_without_protocol).is_socket() # pyre-ignore[6]
|
|
1076
983
|
|
|
1077
984
|
def is_fifo(self) -> bool:
|
|
1078
985
|
"""
|
|
@@ -1082,7 +989,9 @@ class FSPath(URIPath):
|
|
|
1082
989
|
False is also returned if the path doesn’t exist or is a broken symlink;
|
|
1083
990
|
other errors (such as permission errors) are propagated.
|
|
1084
991
|
"""
|
|
1085
|
-
|
|
992
|
+
if isinstance(self.path_without_protocol, int):
|
|
993
|
+
return bool(stat_isfifo(os.stat(self.path_without_protocol).st_mode))
|
|
994
|
+
return pathlib.Path(self.path_without_protocol).is_fifo() # pyre-ignore[6]
|
|
1086
995
|
|
|
1087
996
|
def is_block_device(self) -> bool:
|
|
1088
997
|
"""
|
|
@@ -1092,7 +1001,11 @@ class FSPath(URIPath):
|
|
|
1092
1001
|
False is also returned if the path doesn’t exist or is a broken symlink;
|
|
1093
1002
|
other errors (such as permission errors) are propagated.
|
|
1094
1003
|
"""
|
|
1095
|
-
|
|
1004
|
+
if isinstance(self.path_without_protocol, int):
|
|
1005
|
+
return bool(stat_isblk(os.stat(self.path_without_protocol).st_mode))
|
|
1006
|
+
return pathlib.Path(
|
|
1007
|
+
self.path_without_protocol # pyre-ignore[6]
|
|
1008
|
+
).is_block_device()
|
|
1096
1009
|
|
|
1097
1010
|
def is_char_device(self) -> bool:
|
|
1098
1011
|
"""
|
|
@@ -1102,21 +1015,31 @@ class FSPath(URIPath):
|
|
|
1102
1015
|
False is also returned if the path doesn’t exist or is a broken symlink;
|
|
1103
1016
|
other errors (such as permission errors) are propagated.
|
|
1104
1017
|
"""
|
|
1105
|
-
|
|
1018
|
+
if isinstance(self.path_without_protocol, int):
|
|
1019
|
+
return bool(stat_ischr(os.stat(self.path_without_protocol).st_mode))
|
|
1020
|
+
return pathlib.Path(
|
|
1021
|
+
self.path_without_protocol # pyre-ignore[6]
|
|
1022
|
+
).is_char_device()
|
|
1106
1023
|
|
|
1107
1024
|
def owner(self) -> str:
|
|
1108
1025
|
"""
|
|
1109
1026
|
Return the name of the user owning the file. KeyError is raised if the file’s
|
|
1110
1027
|
uid isn’t found in the system database.
|
|
1111
1028
|
"""
|
|
1112
|
-
|
|
1029
|
+
self._check_int_path()
|
|
1030
|
+
return pathlib.Path(self.path_without_protocol).owner() # pyre-ignore[6]
|
|
1113
1031
|
|
|
1114
1032
|
def absolute(self) -> "FSPath":
|
|
1115
1033
|
"""
|
|
1116
1034
|
Make the path absolute, without normalization or resolving symlinks.
|
|
1117
1035
|
Returns a new path object
|
|
1118
1036
|
"""
|
|
1119
|
-
|
|
1037
|
+
self._check_int_path()
|
|
1038
|
+
return self.from_path(
|
|
1039
|
+
os.path.abspath(
|
|
1040
|
+
self.path_without_protocol # pyre-ignore[6]
|
|
1041
|
+
)
|
|
1042
|
+
)
|
|
1120
1043
|
|
|
1121
1044
|
def rmdir(self):
|
|
1122
1045
|
"""
|