megfile 3.1.1__py3-none-any.whl → 3.1.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. docs/conf.py +2 -4
  2. megfile/__init__.py +394 -203
  3. megfile/cli.py +258 -238
  4. megfile/config.py +25 -21
  5. megfile/errors.py +124 -114
  6. megfile/fs.py +174 -140
  7. megfile/fs_path.py +462 -354
  8. megfile/hdfs.py +133 -101
  9. megfile/hdfs_path.py +290 -236
  10. megfile/http.py +15 -14
  11. megfile/http_path.py +111 -107
  12. megfile/interfaces.py +70 -65
  13. megfile/lib/base_prefetch_reader.py +84 -65
  14. megfile/lib/combine_reader.py +12 -12
  15. megfile/lib/compare.py +17 -13
  16. megfile/lib/compat.py +1 -5
  17. megfile/lib/fnmatch.py +29 -30
  18. megfile/lib/glob.py +46 -54
  19. megfile/lib/hdfs_prefetch_reader.py +40 -25
  20. megfile/lib/hdfs_tools.py +1 -3
  21. megfile/lib/http_prefetch_reader.py +69 -46
  22. megfile/lib/joinpath.py +5 -5
  23. megfile/lib/lazy_handler.py +7 -3
  24. megfile/lib/s3_buffered_writer.py +58 -51
  25. megfile/lib/s3_cached_handler.py +13 -14
  26. megfile/lib/s3_limited_seekable_writer.py +37 -28
  27. megfile/lib/s3_memory_handler.py +34 -30
  28. megfile/lib/s3_pipe_handler.py +24 -25
  29. megfile/lib/s3_prefetch_reader.py +71 -52
  30. megfile/lib/s3_share_cache_reader.py +37 -24
  31. megfile/lib/shadow_handler.py +7 -3
  32. megfile/lib/stdio_handler.py +9 -8
  33. megfile/lib/url.py +3 -3
  34. megfile/pathlike.py +259 -228
  35. megfile/s3.py +220 -153
  36. megfile/s3_path.py +977 -802
  37. megfile/sftp.py +190 -156
  38. megfile/sftp_path.py +540 -450
  39. megfile/smart.py +397 -330
  40. megfile/smart_path.py +100 -105
  41. megfile/stdio.py +10 -9
  42. megfile/stdio_path.py +32 -35
  43. megfile/utils/__init__.py +73 -54
  44. megfile/utils/mutex.py +11 -14
  45. megfile/version.py +1 -1
  46. {megfile-3.1.1.dist-info → megfile-3.1.2.dist-info}/METADATA +5 -8
  47. megfile-3.1.2.dist-info/RECORD +55 -0
  48. {megfile-3.1.1.dist-info → megfile-3.1.2.dist-info}/WHEEL +1 -1
  49. scripts/convert_results_to_sarif.py +45 -78
  50. scripts/generate_file.py +140 -64
  51. megfile-3.1.1.dist-info/RECORD +0 -55
  52. {megfile-3.1.1.dist-info → megfile-3.1.2.dist-info}/LICENSE +0 -0
  53. {megfile-3.1.1.dist-info → megfile-3.1.2.dist-info}/LICENSE.pyre +0 -0
  54. {megfile-3.1.1.dist-info → megfile-3.1.2.dist-info}/entry_points.txt +0 -0
  55. {megfile-3.1.1.dist-info → megfile-3.1.2.dist-info}/top_level.txt +0 -0
megfile/pathlike.py CHANGED
@@ -4,7 +4,19 @@ import stat
4
4
  from collections.abc import Sequence
5
5
  from enum import Enum
6
6
  from functools import cached_property
7
- from typing import IO, Any, BinaryIO, Iterator, List, NamedTuple, Optional, Tuple, Type, TypeVar, Union
7
+ from typing import (
8
+ IO,
9
+ Any,
10
+ BinaryIO,
11
+ Iterator,
12
+ List,
13
+ NamedTuple,
14
+ Optional,
15
+ Tuple,
16
+ Type,
17
+ TypeVar,
18
+ Union,
19
+ )
8
20
 
9
21
  from megfile.lib.compat import PathLike as _PathLike
10
22
  from megfile.lib.compat import fspath
@@ -12,7 +24,7 @@ from megfile.lib.fnmatch import _compile_pattern
12
24
  from megfile.lib.joinpath import uri_join
13
25
  from megfile.utils import classproperty
14
26
 
15
- Self = TypeVar('Self')
27
+ Self = TypeVar("Self")
16
28
 
17
29
 
18
30
  class Access(Enum):
@@ -39,11 +51,11 @@ class StatResult(NamedTuple):
39
51
 
40
52
  @property
41
53
  def st_mode(self) -> int:
42
- '''
54
+ """
43
55
  File mode: file type and file mode bits (permissions).
44
56
  Only support fs.
45
- '''
46
- if self.extra and hasattr(self.extra, 'st_mode'):
57
+ """
58
+ if self.extra and hasattr(self.extra, "st_mode"):
47
59
  return self.extra.st_mode
48
60
  if self.is_symlink():
49
61
  return stat.S_IFLNK
@@ -53,138 +65,139 @@ class StatResult(NamedTuple):
53
65
 
54
66
  @property
55
67
  def st_ino(self) -> int:
56
- '''
57
- Platform dependent, but if non-zero, uniquely identifies the file for a given value of st_dev. Typically:
58
-
68
+ """
69
+ Platform dependent, but if non-zero, uniquely identifies the file for
70
+ a given value of st_dev. Typically:
71
+
59
72
  the inode number on Unix,
60
73
  the file index on Windows,
61
74
  the decimal of etag on oss.
62
- '''
75
+ """
63
76
  if self.extra:
64
- if hasattr(self.extra, 'st_ino'):
77
+ if hasattr(self.extra, "st_ino"):
65
78
  return self.extra.st_ino
66
- elif isinstance(self.extra, dict) and self.extra.get('ETag'):
67
- return int(self.extra['ETag'][1:-1], 16)
79
+ elif isinstance(self.extra, dict) and self.extra.get("ETag"):
80
+ return int(self.extra["ETag"][1:-1], 16)
68
81
  return 0
69
82
 
70
83
  @property
71
84
  def st_dev(self) -> int:
72
- '''
85
+ """
73
86
  Identifier of the device on which this file resides.
74
- '''
87
+ """
75
88
  if self.extra:
76
- if hasattr(self.extra, 'st_dev'):
89
+ if hasattr(self.extra, "st_dev"):
77
90
  return self.extra.st_dev
78
91
  return 0
79
92
 
80
93
  @property
81
94
  def st_nlink(self) -> int:
82
- '''
95
+ """
83
96
  Number of hard links.
84
97
  Only support fs.
85
- '''
86
- if self.extra and hasattr(self.extra, 'st_nlink'):
98
+ """
99
+ if self.extra and hasattr(self.extra, "st_nlink"):
87
100
  return self.extra.st_nlink
88
101
  return 0
89
102
 
90
103
  @property
91
104
  def st_uid(self) -> int:
92
- '''
105
+ """
93
106
  User identifier of the file owner.
94
107
  Only support fs.
95
- '''
96
- if self.extra and hasattr(self.extra, 'st_uid'):
108
+ """
109
+ if self.extra and hasattr(self.extra, "st_uid"):
97
110
  return self.extra.st_uid
98
111
  return 0
99
112
 
100
113
  @property
101
114
  def st_gid(self) -> int:
102
- '''
115
+ """
103
116
  Group identifier of the file owner.
104
117
  Only support fs.
105
- '''
106
- if self.extra and hasattr(self.extra, 'st_gid'):
118
+ """
119
+ if self.extra and hasattr(self.extra, "st_gid"):
107
120
  return self.extra.st_gid
108
121
  return 0
109
122
 
110
123
  @property
111
124
  def st_size(self) -> int:
112
- '''
125
+ """
113
126
  Size of the file in bytes.
114
- '''
115
- if self.extra and hasattr(self.extra, 'st_size'):
127
+ """
128
+ if self.extra and hasattr(self.extra, "st_size"):
116
129
  return self.extra.st_size
117
130
  return self.size
118
131
 
119
132
  @property
120
133
  def st_atime(self) -> float:
121
- '''
134
+ """
122
135
  Time of most recent access expressed in seconds.
123
136
  Only support fs.
124
- '''
125
- if self.extra and hasattr(self.extra, 'st_atime'):
137
+ """
138
+ if self.extra and hasattr(self.extra, "st_atime"):
126
139
  return self.extra.st_atime
127
140
  return 0.0
128
141
 
129
142
  @property
130
143
  def st_mtime(self) -> float:
131
- '''
144
+ """
132
145
  Time of most recent content modification expressed in seconds.
133
- '''
134
- if self.extra and hasattr(self.extra, 'st_mtime'):
146
+ """
147
+ if self.extra and hasattr(self.extra, "st_mtime"):
135
148
  return self.extra.st_mtime
136
149
  return self.mtime
137
150
 
138
151
  @property
139
152
  def st_ctime(self) -> float:
140
- '''
153
+ """
141
154
  Platform dependent:
142
155
 
143
156
  the time of most recent metadata change on Unix,
144
157
  the time of creation on Windows, expressed in seconds,
145
- the time of file created on oss; if is dir, return the latest ctime of the files in dir.
146
- '''
147
- if self.extra and hasattr(self.extra, 'st_ctime'):
158
+ the time of file created on oss;
159
+ if is dir, return the latest ctime of the files in dir.
160
+ """
161
+ if self.extra and hasattr(self.extra, "st_ctime"):
148
162
  return self.extra.st_ctime
149
163
  return self.ctime
150
164
 
151
165
  @property
152
166
  def st_atime_ns(self) -> int:
153
- '''
167
+ """
154
168
  Time of most recent access expressed in nanoseconds as an integer.
155
169
  Only support fs.
156
- '''
157
- if self.extra and hasattr(self.extra, 'st_atime_ns'):
170
+ """
171
+ if self.extra and hasattr(self.extra, "st_atime_ns"):
158
172
  return self.extra.st_atime_ns
159
173
  return 0
160
174
 
161
175
  @property
162
176
  def st_mtime_ns(self) -> int:
163
- '''
177
+ """
164
178
  Time of most recent content modification expressed in nanoseconds as an integer.
165
179
  Only support fs.
166
- '''
167
- if self.extra and hasattr(self.extra, 'st_mtime_ns'):
180
+ """
181
+ if self.extra and hasattr(self.extra, "st_mtime_ns"):
168
182
  return self.extra.st_mtime_ns
169
183
  return 0
170
184
 
171
185
  @property
172
186
  def st_ctime_ns(self) -> int:
173
- '''
187
+ """
174
188
  Platform dependent:
175
189
 
176
190
  the time of most recent metadata change on Unix,
177
191
  the time of creation on Windows, expressed in nanoseconds as an integer.
178
192
 
179
193
  Only support fs.
180
- '''
181
- if self.extra and hasattr(self.extra, 'st_ctime_ns'):
194
+ """
195
+ if self.extra and hasattr(self.extra, "st_ctime_ns"):
182
196
  return self.extra.st_ctime_ns
183
197
  return 0
184
198
 
185
199
 
186
200
  class FileEntry(NamedTuple):
187
-
188
201
  name: str
189
202
  path: str
190
203
  stat: StatResult
@@ -203,7 +216,6 @@ class FileEntry(NamedTuple):
203
216
 
204
217
 
205
218
  class BasePath:
206
-
207
219
  def __init__(self, path: "PathLike"):
208
220
  self.path = str(path)
209
221
 
@@ -211,7 +223,7 @@ class BasePath:
211
223
  return self.path
212
224
 
213
225
  def __repr__(self) -> str:
214
- return '%s(%r)' % (self.__class__.__name__, str(self))
226
+ return "%s(%r)" % (self.__class__.__name__, str(self))
215
227
 
216
228
  def __bytes__(self) -> bytes:
217
229
  return str(self).encode()
@@ -249,7 +261,10 @@ class BasePath:
249
261
  raise NotImplementedError('method "listdir" not implemented: %r' % self)
250
262
 
251
263
  def scandir(self) -> Iterator[FileEntry]:
252
- """Return an iterator of FileEntry objects corresponding to the entries in the directory."""
264
+ """
265
+ Return an iterator of FileEntry objects corresponding to the entries
266
+ in the directory.
267
+ """
253
268
  raise NotImplementedError('method "scandir" not implemented: %r' % self)
254
269
 
255
270
  def getsize(self, follow_symlinks: bool = True) -> int:
@@ -258,8 +273,7 @@ class BasePath:
258
273
 
259
274
  def getmtime(self, follow_symlinks: bool = True) -> float:
260
275
  """Return the time of last modification."""
261
- raise NotImplementedError(
262
- 'method "getmtime" not implemented: %r' % self)
276
+ raise NotImplementedError('method "getmtime" not implemented: %r' % self)
263
277
 
264
278
  def stat(self, follow_symlinks=True) -> StatResult:
265
279
  """Get the status of the path."""
@@ -273,11 +287,7 @@ class BasePath:
273
287
  """Remove (delete) the file."""
274
288
  raise NotImplementedError('method "unlink" not implemented: %r' % self)
275
289
 
276
- def mkdir(
277
- self,
278
- mode=0o777,
279
- parents: bool = False,
280
- exist_ok: bool = False) -> None:
290
+ def mkdir(self, mode=0o777, parents: bool = False, exist_ok: bool = False) -> None:
281
291
  """Create a directory."""
282
292
  raise NotImplementedError('method "mkdir" not implemented: %r' % self)
283
293
 
@@ -285,54 +295,43 @@ class BasePath:
285
295
  """Remove (delete) the directory."""
286
296
  raise NotImplementedError('method "rmdir" not implemented: %r' % self)
287
297
 
288
- def open(self, mode: str = 'r', **kwargs) -> IO:
298
+ def open(self, mode: str = "r", **kwargs) -> IO:
289
299
  """Open the file with mode."""
290
300
  raise NotImplementedError('method "open" not implemented: %r' % self)
291
301
 
292
302
  def walk(
293
- self,
294
- followlinks: bool = False
303
+ self, followlinks: bool = False
295
304
  ) -> Iterator[Tuple[str, List[str], List[str]]]:
296
305
  """Generate the file names in a directory tree by walking the tree."""
297
306
  raise NotImplementedError('method "walk" not implemented: %r' % self)
298
307
 
299
- def scan(self,
300
- missing_ok: bool = True,
301
- followlinks: bool = False) -> Iterator[str]:
308
+ def scan(self, missing_ok: bool = True, followlinks: bool = False) -> Iterator[str]:
302
309
  """Iterate through the files in the directory."""
303
310
  raise NotImplementedError('method "scan" not implemented: %r' % self)
304
311
 
305
- def scan_stat(self,
306
- missing_ok: bool = True,
307
- followlinks: bool = False) -> Iterator[FileEntry]:
312
+ def scan_stat(
313
+ self, missing_ok: bool = True, followlinks: bool = False
314
+ ) -> Iterator[FileEntry]:
308
315
  """Iterate through the files in the directory, with file stat."""
309
- raise NotImplementedError(
310
- 'method "scan_stat" not implemented: %r' % self)
316
+ raise NotImplementedError('method "scan_stat" not implemented: %r' % self)
311
317
 
312
318
  def glob(
313
- self: Self,
314
- pattern: str,
315
- recursive: bool = True,
316
- missing_ok: bool = True) -> List[Self]:
319
+ self: Self, pattern: str, recursive: bool = True, missing_ok: bool = True
320
+ ) -> List[Self]:
317
321
  """Return files whose paths match the glob pattern."""
318
322
  raise NotImplementedError('method "glob" not implemented: %r' % self)
319
323
 
320
324
  def iglob(
321
- self: Self,
322
- pattern: str,
323
- recursive: bool = True,
324
- missing_ok: bool = True) -> Iterator[Self]:
325
+ self: Self, pattern: str, recursive: bool = True, missing_ok: bool = True
326
+ ) -> Iterator[Self]:
325
327
  """Return an iterator of files whose paths match the glob pattern."""
326
328
  raise NotImplementedError('method "iglob" not implemented: %r' % self)
327
329
 
328
330
  def glob_stat(
329
- self,
330
- pattern: str,
331
- recursive: bool = True,
332
- missing_ok: bool = True) -> Iterator[FileEntry]:
331
+ self, pattern: str, recursive: bool = True, missing_ok: bool = True
332
+ ) -> Iterator[FileEntry]:
333
333
  """Return an iterator of files with stat whose paths match the glob pattern."""
334
- raise NotImplementedError(
335
- 'method "glob_stat" not implemented: %r' % self)
334
+ raise NotImplementedError('method "glob_stat" not implemented: %r' % self)
336
335
 
337
336
  def load(self) -> BinaryIO:
338
337
  """Read all content in binary."""
@@ -344,8 +343,7 @@ class BasePath:
344
343
 
345
344
  def joinpath(self: Self, *other_paths: "PathLike") -> Self:
346
345
  """Join or or more path."""
347
- raise NotImplementedError(
348
- 'method "joinpath" not implemented: %r' % self)
346
+ raise NotImplementedError('method "joinpath" not implemented: %r' % self)
349
347
 
350
348
  def abspath(self):
351
349
  """Return a normalized absolute version of the path."""
@@ -353,8 +351,7 @@ class BasePath:
353
351
 
354
352
  def realpath(self):
355
353
  """Return the canonical path of the path."""
356
- raise NotImplementedError(
357
- 'method "realpath" not implemented: %r' % self)
354
+ raise NotImplementedError('method "realpath" not implemented: %r' % self)
358
355
 
359
356
  def relpath(self, start=None):
360
357
  """Return the relative path."""
@@ -362,20 +359,18 @@ class BasePath:
362
359
 
363
360
  def is_absolute(self) -> bool:
364
361
  """Return True if the path is an absolute pathname."""
365
- raise NotImplementedError(
366
- 'method "is_absolute" not implemented: %r' % self)
362
+ raise NotImplementedError('method "is_absolute" not implemented: %r' % self)
367
363
 
368
364
  def is_mount(self) -> bool:
369
365
  """Return True if the path is a mount point."""
370
- raise NotImplementedError(
371
- 'method "is_mount" not implemented: %r' % self)
366
+ raise NotImplementedError('method "is_mount" not implemented: %r' % self)
372
367
 
373
368
  def resolve(self):
374
369
  """Alias of realpath."""
375
370
  raise NotImplementedError('method "resolve" not implemented: %r' % self)
376
371
 
377
372
  def touch(self):
378
- with self.open('w'):
373
+ with self.open("w"):
379
374
  pass
380
375
 
381
376
  # TODO: will be deleted in next version
@@ -383,9 +378,10 @@ class BasePath:
383
378
  return self.is_symlink()
384
379
 
385
380
  def makedirs(self, exist_ok: bool = False) -> None:
386
- '''
387
- Recursive directory creation function. Like mkdir(), but makes all intermediate-level directories needed to contain the leaf directory.
388
- '''
381
+ """
382
+ Recursive directory creation function. Like mkdir(), but makes all
383
+ intermediate-level directories needed to contain the leaf directory.
384
+ """
389
385
  self.mkdir(parents=True, exist_ok=exist_ok)
390
386
 
391
387
 
@@ -393,7 +389,6 @@ PathLike = Union[str, BasePath, _PathLike]
393
389
 
394
390
 
395
391
  class BaseURIPath(BasePath):
396
-
397
392
  # #####
398
393
  # TODO: Backwards compatible API, will be removed in megfile 1.0
399
394
  @classmethod
@@ -414,24 +409,27 @@ class BaseURIPath(BasePath):
414
409
 
415
410
  @cached_property
416
411
  def path_with_protocol(self) -> str:
417
- '''Return path with protocol, like file:///root, s3://bucket/key'''
412
+ """Return path with protocol, like file:///root, s3://bucket/key"""
418
413
  path = self.path
419
414
  protocol_prefix = self.protocol + "://" # pyre-ignore[58]
420
415
  if path.startswith(protocol_prefix):
421
416
  return path
422
- return protocol_prefix + path.lstrip('/')
417
+ return protocol_prefix + path.lstrip("/")
423
418
 
424
419
  @cached_property
425
420
  def path_without_protocol(self) -> str:
426
- '''Return path without protocol, example: if path is s3://bucket/key, return bucket/key'''
421
+ """
422
+ Return path without protocol, example: if path is s3://bucket/key,
423
+ return bucket/key
424
+ """
427
425
  path = self.path
428
426
  protocol_prefix = self.protocol + "://" # pyre-ignore[58]
429
427
  if path.startswith(protocol_prefix):
430
- path = path[len(protocol_prefix):]
428
+ path = path[len(protocol_prefix) :]
431
429
  return path
432
430
 
433
431
  def as_posix(self) -> str:
434
- '''Return a string representation of the path with forward slashes (/)'''
432
+ """Return a string representation of the path with forward slashes (/)"""
435
433
  return self.path_with_protocol
436
434
 
437
435
  def __fspath__(self) -> str:
@@ -442,8 +440,9 @@ class BaseURIPath(BasePath):
442
440
  raise TypeError("%r is not 'URIPath'" % other_path)
443
441
  if self.protocol != other_path.protocol:
444
442
  raise TypeError(
445
- "'<' not supported between instances of %r and %r" %
446
- (type(self), type(other_path)))
443
+ "'<' not supported between instances of %r and %r"
444
+ % (type(self), type(other_path))
445
+ )
447
446
  return fspath(self) < fspath(other_path)
448
447
 
449
448
  def __le__(self, other_path: "BaseURIPath") -> bool:
@@ -451,8 +450,9 @@ class BaseURIPath(BasePath):
451
450
  raise TypeError("%r is not 'URIPath'" % other_path)
452
451
  if self.protocol != other_path.protocol:
453
452
  raise TypeError(
454
- "'<=' not supported between instances of %r and %r" %
455
- (type(self), type(other_path)))
453
+ "'<=' not supported between instances of %r and %r"
454
+ % (type(self), type(other_path))
455
+ )
456
456
  return str(self) <= str(other_path)
457
457
 
458
458
  def __gt__(self, other_path: "BaseURIPath") -> bool:
@@ -460,8 +460,9 @@ class BaseURIPath(BasePath):
460
460
  raise TypeError("%r is not 'URIPath'" % other_path)
461
461
  if self.protocol != other_path.protocol:
462
462
  raise TypeError(
463
- "'>' not supported between instances of %r and %r" %
464
- (type(self), type(other_path)))
463
+ "'>' not supported between instances of %r and %r"
464
+ % (type(self), type(other_path))
465
+ )
465
466
  return str(self) > str(other_path)
466
467
 
467
468
  def __ge__(self, other_path: "BaseURIPath") -> bool:
@@ -469,17 +470,18 @@ class BaseURIPath(BasePath):
469
470
  raise TypeError("%r is not 'URIPath'" % other_path)
470
471
  if self.protocol != other_path.protocol:
471
472
  raise TypeError(
472
- "'>=' not supported between instances of %r and %r" %
473
- (type(self), type(other_path)))
473
+ "'>=' not supported between instances of %r and %r"
474
+ % (type(self), type(other_path))
475
+ )
474
476
  return str(self) >= str(other_path)
475
477
 
476
478
  @classproperty
477
479
  def drive(self) -> str:
478
- return ''
480
+ return ""
479
481
 
480
482
  @classproperty
481
483
  def root(self) -> str:
482
- return self.protocol + '://' # pyre-ignore[58]
484
+ return self.protocol + "://" # pyre-ignore[58]
483
485
 
484
486
  @classproperty
485
487
  def anchor(self) -> str:
@@ -487,7 +489,6 @@ class BaseURIPath(BasePath):
487
489
 
488
490
 
489
491
  class URIPath(BaseURIPath):
490
-
491
492
  def __init__(self, path: "PathLike", *other_paths: "PathLike"):
492
493
  if len(other_paths) > 0:
493
494
  path = self.from_path(path).joinpath(*other_paths)
@@ -497,7 +498,7 @@ class URIPath(BaseURIPath):
497
498
  def from_path(cls: Type[Self], path: PathLike) -> Self:
498
499
  """Return new instance of this class
499
500
 
500
- :param path: new path
501
+ :param path: new path
501
502
 
502
503
  :return: new instance of new path
503
504
  :rtype: Self
@@ -508,44 +509,50 @@ class URIPath(BaseURIPath):
508
509
  def from_uri(cls: Type[Self], path: PathLike) -> Self:
509
510
  path = fspath(path)
510
511
  protocol_prefix = cls.protocol + "://"
511
- if path[:len(protocol_prefix)] != protocol_prefix:
512
+ if path[: len(protocol_prefix)] != protocol_prefix:
512
513
  raise ValueError(
513
- "protocol not match, expected: %r, got: %r" %
514
- (cls.protocol, path))
515
- return cls.from_path(path[len(protocol_prefix):])
514
+ "protocol not match, expected: %r, got: %r" % (cls.protocol, path)
515
+ )
516
+ return cls.from_path(path[len(protocol_prefix) :])
516
517
 
517
518
  def __truediv__(self: Self, other_path: PathLike) -> Self:
518
519
  if isinstance(other_path, BaseURIPath):
519
520
  if self.protocol != other_path.protocol:
520
521
  raise TypeError(
521
- "'/' not supported between instances of %r and %r" %
522
- (type(self), type(other_path)))
522
+ "'/' not supported between instances of %r and %r"
523
+ % (type(self), type(other_path))
524
+ )
523
525
  elif not isinstance(other_path, str):
524
526
  raise TypeError("%r is not 'str' nor 'URIPath'" % other_path)
525
527
  return self.joinpath(other_path)
526
528
 
527
529
  def joinpath(self: Self, *other_paths: PathLike) -> Self:
528
- '''Calling this method is equivalent to combining the path with each of the other arguments in turn'''
530
+ """
531
+ Calling this method is equivalent to combining the path
532
+ with each of the other arguments in turn
533
+ """
529
534
  return self.from_path(uri_join(str(self), *map(str, other_paths)))
530
535
 
531
536
  @cached_property
532
537
  def parts(self) -> Tuple[str, ...]:
533
- '''A tuple giving access to the path’s various components'''
538
+ """A tuple giving access to the path’s various components"""
534
539
  parts = [self.root]
535
540
  path = self.path_without_protocol
536
- path = path.lstrip('/')
537
- if path != '':
538
- parts.extend(path.split('/'))
541
+ path = path.lstrip("/")
542
+ if path != "":
543
+ parts.extend(path.split("/"))
539
544
  return tuple(parts) # pyre-ignore[7]
540
545
 
541
546
  @cached_property
542
547
  def parents(self) -> "URIPathParents":
543
- '''An immutable sequence providing access to the logical ancestors of the path'''
548
+ """
549
+ An immutable sequence providing access to the logical ancestors of the path
550
+ """
544
551
  return URIPathParents(self)
545
552
 
546
553
  @cached_property
547
554
  def parent(self: Self) -> Self:
548
- '''The logical parent of the path'''
555
+ """The logical parent of the path"""
549
556
  if self.path_without_protocol == "/":
550
557
  return self
551
558
  elif len(self.parents) > 0:
@@ -554,36 +561,37 @@ class URIPath(BaseURIPath):
554
561
 
555
562
  @cached_property
556
563
  def name(self) -> str:
557
- '''A string representing the final path component, excluding the drive and root'''
564
+ """
565
+ A string representing the final path component, excluding the drive and root
566
+ """
558
567
  parts = self.parts
559
- if len(parts
560
- ) == 1 and parts[0] == self.protocol + "://": # pyre-ignore[58]
561
- return ''
568
+ if len(parts) == 1 and parts[0] == self.protocol + "://": # pyre-ignore[58]
569
+ return ""
562
570
  return parts[-1]
563
571
 
564
572
  @cached_property
565
573
  def suffix(self) -> str:
566
- '''The file extension of the final component'''
574
+ """The file extension of the final component"""
567
575
  name = self.name
568
- i = name.rfind('.')
576
+ i = name.rfind(".")
569
577
  if 0 < i < len(name) - 1:
570
578
  return name[i:]
571
- return ''
579
+ return ""
572
580
 
573
581
  @cached_property
574
582
  def suffixes(self) -> List[str]:
575
- '''A list of the path’s file extensions'''
583
+ """A list of the path’s file extensions"""
576
584
  name = self.name
577
- if name.endswith('.'):
585
+ if name.endswith("."):
578
586
  return []
579
- name = name.lstrip('.')
580
- return ['.' + suffix for suffix in name.split('.')[1:]]
587
+ name = name.lstrip(".")
588
+ return ["." + suffix for suffix in name.split(".")[1:]]
581
589
 
582
590
  @cached_property
583
591
  def stem(self) -> str:
584
- '''The final path component, without its suffix'''
592
+ """The final path component, without its suffix"""
585
593
  name = self.name
586
- i = name.rfind('.')
594
+ i = name.rfind(".")
587
595
  if 0 < i < len(name) - 1:
588
596
  return name[:i]
589
597
  return name
@@ -592,16 +600,18 @@ class URIPath(BaseURIPath):
592
600
  return False
593
601
 
594
602
  def match(self, pattern) -> bool:
595
- '''Match this path against the provided glob-style pattern. Return True if matching is successful, False otherwise'''
603
+ """
604
+ Match this path against the provided glob-style pattern.
605
+ Return True if matching is successful, False otherwise
606
+ """
596
607
  match = _compile_pattern(pattern)
597
608
  for index in range(len(self.parts), 0, -1):
598
- path = '/'.join(self.parts[index:])
609
+ path = "/".join(self.parts[index:])
599
610
  if match(path) is not None:
600
611
  return True
601
612
  return match(self.path_with_protocol) is not None
602
613
 
603
614
  def is_relative_to(self, *other) -> bool:
604
-
605
615
  try:
606
616
  self.relative_to(*other)
607
617
  return True
@@ -609,10 +619,10 @@ class URIPath(BaseURIPath):
609
619
  return False
610
620
 
611
621
  def relative_to(self: Self, *other: str) -> Self:
612
- '''
622
+ """
613
623
  Compute a version of this path relative to the path represented by other.
614
624
  If it’s impossible, ValueError is raised.
615
- '''
625
+ """
616
626
  if not other:
617
627
  raise TypeError("need at least one argument")
618
628
 
@@ -623,68 +633,76 @@ class URIPath(BaseURIPath):
623
633
  path = self.path_with_protocol
624
634
 
625
635
  if path.startswith(other_path):
626
- relative = path[len(other_path):]
627
- relative = relative.lstrip('/')
636
+ relative = path[len(other_path) :]
637
+ relative = relative.lstrip("/")
628
638
  return type(self)(relative) # pyre-ignore[19]
629
639
  else:
630
640
  raise ValueError("%r does not start with %r" % (path, other))
631
641
 
632
642
  def with_name(self: Self, name: str) -> Self:
633
- '''Return a new path with the name changed'''
643
+ """Return a new path with the name changed"""
634
644
  path = str(self)
635
645
  raw_name = self.name
636
- return self.from_path(path[:len(path) - len(raw_name)] + name)
646
+ return self.from_path(path[: len(path) - len(raw_name)] + name)
637
647
 
638
648
  def with_stem(self: Self, stem: str) -> Self:
639
- '''Return a new path with the stem changed'''
649
+ """Return a new path with the stem changed"""
640
650
  return self.with_name("".join([stem, self.suffix]))
641
651
 
642
652
  def with_suffix(self: Self, suffix: str) -> Self:
643
- '''Return a new path with the suffix changed'''
653
+ """Return a new path with the suffix changed"""
644
654
  path = str(self)
645
655
  raw_suffix = self.suffix
646
- return self.from_path(path[:len(path) - len(raw_suffix)] + suffix)
656
+ return self.from_path(path[: len(path) - len(raw_suffix)] + suffix)
647
657
 
648
658
  def is_absolute(self) -> bool:
649
659
  return True
650
660
 
651
661
  def is_mount(self) -> bool:
652
- '''Test whether a path is a mount point
662
+ """Test whether a path is a mount point
653
663
 
654
664
  :returns: True if a path is a mount point, else False
655
- '''
665
+ """
656
666
  return False
657
667
 
658
668
  def is_socket(self) -> bool:
659
- '''
660
- Return True if the path points to a Unix socket (or a symbolic link pointing to a Unix socket), False if it points to another kind of file.
669
+ """
670
+ Return True if the path points to a Unix socket (or a symbolic link pointing
671
+ to a Unix socket), False if it points to another kind of file.
661
672
 
662
- False is also returned if the path doesn’t exist or is a broken symlink; other errors (such as permission errors) are propagated.
663
- '''
673
+ False is also returned if the path doesn’t exist or is a broken symlink;
674
+ other errors (such as permission errors) are propagated.
675
+ """
664
676
  return False
665
677
 
666
678
  def is_fifo(self) -> bool:
667
- '''
668
- Return True if the path points to a FIFO (or a symbolic link pointing to a FIFO), False if it points to another kind of file.
679
+ """
680
+ Return True if the path points to a FIFO (or a symbolic link pointing to a
681
+ FIFO), False if it points to another kind of file.
669
682
 
670
- False is also returned if the path doesn’t exist or is a broken symlink; other errors (such as permission errors) are propagated.
671
- '''
683
+ False is also returned if the path doesn’t exist or is a broken symlink;
684
+ other errors (such as permission errors) are propagated.
685
+ """
672
686
  return False
673
687
 
674
688
  def is_block_device(self) -> bool:
675
- '''
676
- Return True if the path points to a block device (or a symbolic link pointing to a block device), False if it points to another kind of file.
689
+ """
690
+ Return True if the path points to a block device (or a symbolic link pointing
691
+ to a block device), False if it points to another kind of file.
677
692
 
678
- False is also returned if the path doesn’t exist or is a broken symlink; other errors (such as permission errors) are propagated.
679
- '''
693
+ False is also returned if the path doesn’t exist or is a broken symlink;
694
+ other errors (such as permission errors) are propagated.
695
+ """
680
696
  return False
681
697
 
682
698
  def is_char_device(self) -> bool:
683
- '''
684
- Return True if the path points to a character device (or a symbolic link pointing to a character device), False if it points to another kind of file.
699
+ """
700
+ Return True if the path points to a character device (or a symbolic link
701
+ pointing to a character device), False if it points to another kind of file.
685
702
 
686
- False is also returned if the path doesn’t exist or is a broken symlink; other errors (such as permission errors) are propagated.
687
- '''
703
+ False is also returned if the path doesn’t exist or is a broken symlink;
704
+ other errors (such as permission errors) are propagated.
705
+ """
688
706
  return False
689
707
 
690
708
  def abspath(self) -> str:
@@ -703,104 +721,111 @@ class URIPath(BaseURIPath):
703
721
  raise NotImplementedError(f"'chmod' is unsupported on '{type(self)}'")
704
722
 
705
723
  def lchmod(self, mode: int):
706
- '''
707
- Like chmod() but, if the path points to a symbolic link, the symbolic link’s mode is changed rather than its target’s.
708
- '''
724
+ """
725
+ Like chmod() but, if the path points to a symbolic link, the symbolic
726
+ link’s mode is changed rather than its target’s.
727
+ """
709
728
  return self.chmod(mode=mode, follow_symlinks=False)
710
729
 
711
730
  def read_bytes(self) -> bytes:
712
- '''Return the binary contents of the pointed-to file as a bytes object'''
713
- with self.open(mode='rb') as f:
731
+ """Return the binary contents of the pointed-to file as a bytes object"""
732
+ with self.open(mode="rb") as f:
714
733
  return f.read() # pytype: disable=bad-return-type
715
734
 
716
735
  def read_text(self) -> str:
717
- '''Return the decoded contents of the pointed-to file as a string'''
718
- with self.open(mode='r') as f:
736
+ """Return the decoded contents of the pointed-to file as a string"""
737
+ with self.open(mode="r") as f:
719
738
  return f.read() # pytype: disable=bad-return-type
720
739
 
721
740
  def rename(self: Self, dst_path: PathLike, overwrite: bool = True) -> Self:
722
- '''
741
+ """
723
742
  rename file
724
743
 
725
744
  :param dst_path: Given destination path
726
745
  :param overwrite: whether or not overwrite file when exists
727
- '''
746
+ """
728
747
  raise NotImplementedError(f"'rename' is unsupported on '{type(self)}'")
729
748
 
730
749
  def replace(self: Self, dst_path: PathLike, overwrite: bool = True) -> Self:
731
- '''
750
+ """
732
751
  move file
733
752
 
734
753
  :param dst_path: Given destination path
735
754
  :param overwrite: whether or not overwrite file when exists
736
- '''
755
+ """
737
756
  return self.rename(dst_path=dst_path, overwrite=overwrite)
738
757
 
739
758
  def rglob(self: Self, pattern) -> List[Self]:
740
- '''
741
- This is like calling Path.glob() with “**/” added in front of the given relative pattern
742
- '''
759
+ """
760
+ This is like calling Path.glob() with “**/” added in front of
761
+ the given relative pattern
762
+ """
743
763
  if not pattern:
744
764
  pattern = ""
745
- pattern = '**/' + pattern.lstrip('/')
765
+ pattern = "**/" + pattern.lstrip("/")
746
766
  return self.glob(pattern=pattern)
747
767
 
748
768
  def md5(self, recalculate: bool = False, followlinks: bool = False) -> str:
749
769
  raise NotImplementedError(f"'md5' is unsupported on '{type(self)}'")
750
770
 
751
771
  def samefile(self, other_path) -> bool:
752
- '''
772
+ """
753
773
  Return whether this path points to the same file
754
- '''
755
- if hasattr(other_path, 'protocol'):
774
+ """
775
+ if hasattr(other_path, "protocol"):
756
776
  if other_path.protocol != self.protocol:
757
777
  return False
758
778
 
759
779
  stat = self.stat()
760
- if hasattr(other_path, 'stat'):
780
+ if hasattr(other_path, "stat"):
761
781
  other_path_stat = other_path.stat()
762
782
  else:
763
783
  other_path_stat = self.from_path(other_path).stat()
764
784
 
765
- return stat.st_ino == other_path_stat.st_ino and stat.st_dev == other_path_stat.st_dev
785
+ return (
786
+ stat.st_ino == other_path_stat.st_ino
787
+ and stat.st_dev == other_path_stat.st_dev
788
+ )
766
789
 
767
790
  def symlink(self, dst_path: PathLike) -> None:
768
791
  raise NotImplementedError(f"'symlink' is unsupported on '{type(self)}'")
769
792
 
770
793
  def symlink_to(self, target, target_is_directory=False):
771
- '''
772
- Make this path a symbolic link to target.
794
+ """
795
+ Make this path a symbolic link to target.
773
796
  symlink_to's arguments is the reverse of symlink's.
774
797
  Target_is_directory’s value is ignored, only be compatible with pathlib.Path
775
- '''
798
+ """
776
799
  return self.from_path(target).symlink(dst_path=self.path)
777
800
 
778
801
  def hardlink_to(self, target):
779
- '''
802
+ """
780
803
  Make this path a hard link to the same file as target.
781
- '''
782
- raise NotImplementedError(
783
- f"'hardlink_to' is unsupported on '{type(self)}'")
804
+ """
805
+ raise NotImplementedError(f"'hardlink_to' is unsupported on '{type(self)}'")
784
806
 
785
807
  def write_bytes(self, data: bytes):
786
- '''Open the file pointed to in bytes mode, write data to it, and close the file'''
787
- with self.open(mode='wb') as f:
808
+ """
809
+ Open the file pointed to in bytes mode, write data to it, and close the file
810
+ """
811
+ with self.open(mode="wb") as f:
788
812
  return f.write(data)
789
813
 
790
814
  def write_text(self, data: str, encoding=None, errors=None, newline=None):
791
- '''
815
+ """
792
816
  Open the file pointed to in text mode, write data to it, and close the file.
793
817
  The optional parameters have the same meaning as in open().
794
- '''
795
- with self.open(mode='w', encoding=encoding, errors=errors,
796
- newline=newline) as f:
818
+ """
819
+ with self.open(
820
+ mode="w", encoding=encoding, errors=errors, newline=newline
821
+ ) as f:
797
822
  return f.write(data)
798
823
 
799
824
  def home(self):
800
- '''Return the home directory
825
+ """Return the home directory
801
826
 
802
827
  returns: Home directory path
803
- '''
828
+ """
804
829
  raise NotImplementedError(f"'home' is unsupported on '{type(self)}'")
805
830
 
806
831
  def group(self):
@@ -811,59 +836,65 @@ class URIPath(BaseURIPath):
811
836
 
812
837
  def expanduser(self):
813
838
  """
814
- Return a new path with expanded ~ and ~user constructs, as returned by os.path.expanduser().
839
+ Return a new path with expanded ~ and ~user constructs, as returned by
840
+ os.path.expanduser().
841
+
815
842
  Only fs path support this method.
816
843
  """
817
- raise NotImplementedError(
818
- f"'expanduser' is unsupported on '{type(self)}'")
844
+ raise NotImplementedError(f"'expanduser' is unsupported on '{type(self)}'")
819
845
 
820
846
  def cwd(self: Self) -> Self:
821
- '''Return current working directory
847
+ """Return current working directory
822
848
 
823
849
  returns: Current working directory
824
- '''
850
+ """
825
851
  raise NotImplementedError(f"'cwd' is unsupported on '{type(self)}'")
826
852
 
827
853
  def iterdir(self: Self) -> Iterator[Self]:
828
- '''
829
- Get all contents of given fs path. The result is in ascending alphabetical order.
854
+ """
855
+ Get all contents of given fs path.
856
+ The result is in ascending alphabetical order.
830
857
 
831
858
  :returns: All contents have in the path in ascending alphabetical order
832
- '''
859
+ """
833
860
  raise NotImplementedError(f"'iterdir' is unsupported on '{type(self)}'")
834
861
 
835
862
  def owner(self) -> str:
836
- '''
863
+ """
837
864
  Return the name of the user owning the file.
838
- '''
865
+ """
839
866
  raise NotImplementedError(f"'owner' is unsupported on '{type(self)}'")
840
867
 
841
868
  def absolute(self: Self) -> Self:
842
- '''
843
- Make the path absolute, without normalization or resolving symlinks. Returns a new path object
844
- '''
845
- raise NotImplementedError(
846
- f"'absolute' is unsupported on '{type(self)}'")
869
+ """
870
+ Make the path absolute, without normalization or resolving symlinks.
871
+ Returns a new path object
872
+ """
873
+ raise NotImplementedError(f"'absolute' is unsupported on '{type(self)}'")
847
874
 
848
875
  def utime(self, atime: Union[float, int], mtime: Union[float, int]):
849
876
  """
850
- Sets the access and modified times of the file specified by path to the specified values.
851
-
877
+ Sets the access and modified times of the file specified by path to
878
+ the specified values.
879
+
852
880
  :param atime: The access time to be set.
853
881
  :type atime: Union[float, int]
854
882
  :param mtime: The modification time to be set.
855
883
  :type mtime: Union[float, int]
856
- :raises NotImplementedError: Always raised, since the functionality is unsupported.
884
+ :raises NotImplementedError: Always raised,
885
+ since the functionality is unsupported.
857
886
  """
858
887
  raise NotImplementedError(f"'utime' is unsupported on '{type(self)}'")
859
888
 
860
889
  def lstat(self) -> StatResult:
861
- '''Like stat() but, if the path points to a symbolic link, return the symbolic link’s information rather than its target’s.'''
890
+ """
891
+ Like stat() but, if the path points to a symbolic link,
892
+ return the symbolic link’s information rather than its target’s.
893
+ """
862
894
  return self.stat(follow_symlinks=False)
863
895
 
864
896
 
865
897
  class URIPathParents(Sequence):
866
-
867
898
  def __init__(self, path):
868
899
  # We don't store the instance to avoid reference cycles
869
900
  self.cls = type(path)
@@ -872,7 +903,7 @@ class URIPathParents(Sequence):
872
903
  self.prefix = parts[0]
873
904
  self.parts = parts[1:]
874
905
  else:
875
- self.prefix = ''
906
+ self.prefix = ""
876
907
  self.parts = parts
877
908
 
878
909
  def __len__(self):
@@ -882,10 +913,10 @@ class URIPathParents(Sequence):
882
913
  if idx < 0 or idx > len(self):
883
914
  raise IndexError(idx)
884
915
 
885
- if len(self.parts[:-idx - 1]) > 1:
886
- other_path = os.path.join(*self.parts[:-idx - 1])
887
- elif len(self.parts[:-idx - 1]) == 1:
888
- other_path = self.parts[:-idx - 1][0]
916
+ if len(self.parts[: -idx - 1]) > 1:
917
+ other_path = os.path.join(*self.parts[: -idx - 1])
918
+ elif len(self.parts[: -idx - 1]) == 1:
919
+ other_path = self.parts[: -idx - 1][0]
889
920
  else:
890
921
  other_path = ""
891
922
  return self.cls(self.prefix + other_path)