megfile 3.0.6.post1__py3-none-any.whl → 3.1.0.post1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. docs/conf.py +67 -0
  2. megfile/cli.py +16 -16
  3. megfile/config.py +37 -6
  4. megfile/errors.py +26 -20
  5. megfile/fs.py +13 -8
  6. megfile/fs_path.py +69 -49
  7. megfile/hdfs.py +13 -8
  8. megfile/hdfs_path.py +49 -41
  9. megfile/http.py +1 -1
  10. megfile/http_path.py +35 -28
  11. megfile/interfaces.py +119 -48
  12. megfile/lib/base_prefetch_reader.py +9 -8
  13. megfile/lib/combine_reader.py +7 -7
  14. megfile/lib/fnmatch.py +2 -2
  15. megfile/lib/glob.py +3 -3
  16. megfile/lib/hdfs_prefetch_reader.py +2 -1
  17. megfile/lib/http_prefetch_reader.py +3 -2
  18. megfile/lib/lazy_handler.py +6 -5
  19. megfile/lib/s3_buffered_writer.py +8 -7
  20. megfile/lib/s3_cached_handler.py +3 -4
  21. megfile/lib/s3_limited_seekable_writer.py +5 -3
  22. megfile/lib/s3_memory_handler.py +10 -6
  23. megfile/lib/s3_pipe_handler.py +1 -1
  24. megfile/lib/s3_prefetch_reader.py +7 -5
  25. megfile/lib/s3_share_cache_reader.py +2 -2
  26. megfile/lib/shadow_handler.py +5 -5
  27. megfile/lib/stdio_handler.py +3 -3
  28. megfile/pathlike.py +156 -170
  29. megfile/s3.py +19 -13
  30. megfile/s3_path.py +98 -83
  31. megfile/sftp.py +25 -16
  32. megfile/sftp_path.py +109 -94
  33. megfile/smart.py +38 -28
  34. megfile/smart_path.py +6 -6
  35. megfile/stdio.py +3 -3
  36. megfile/stdio_path.py +5 -5
  37. megfile/utils/__init__.py +8 -27
  38. megfile/version.py +1 -1
  39. {megfile-3.0.6.post1.dist-info → megfile-3.1.0.post1.dist-info}/METADATA +4 -5
  40. megfile-3.1.0.post1.dist-info/RECORD +55 -0
  41. {megfile-3.0.6.post1.dist-info → megfile-3.1.0.post1.dist-info}/WHEEL +1 -1
  42. megfile-3.1.0.post1.dist-info/top_level.txt +7 -0
  43. scripts/convert_results_to_sarif.py +124 -0
  44. scripts/generate_file.py +268 -0
  45. megfile-3.0.6.post1.dist-info/RECORD +0 -52
  46. megfile-3.0.6.post1.dist-info/top_level.txt +0 -1
  47. {megfile-3.0.6.post1.dist-info → megfile-3.1.0.post1.dist-info}/LICENSE +0 -0
  48. {megfile-3.0.6.post1.dist-info → megfile-3.1.0.post1.dist-info}/LICENSE.pyre +0 -0
  49. {megfile-3.0.6.post1.dist-info → megfile-3.1.0.post1.dist-info}/entry_points.txt +0 -0
megfile/sftp.py CHANGED
@@ -1,4 +1,4 @@
1
- from typing import IO, AnyStr, BinaryIO, Callable, Iterator, List, Optional, Tuple
1
+ from typing import IO, BinaryIO, Callable, Iterator, List, Optional, Tuple
2
2
 
3
3
  from megfile.interfaces import FileEntry, PathLike, StatResult
4
4
  from megfile.sftp_path import SftpPath, is_sftp, sftp_concat, sftp_download, sftp_glob, sftp_glob_stat, sftp_iglob, sftp_lstat, sftp_path_join, sftp_readlink, sftp_resolve, sftp_upload
@@ -116,10 +116,10 @@ def sftp_isfile(path: PathLike, followlinks: bool = False) -> bool:
116
116
 
117
117
  def sftp_listdir(path: PathLike) -> List[str]:
118
118
  '''
119
- Get all contents of given sftp path. The result is in acsending alphabetical order.
119
+ Get all contents of given sftp path. The result is in ascending alphabetical order.
120
120
 
121
121
  :param path: Given path
122
- :returns: All contents have in the path in acsending alphabetical order
122
+ :returns: All contents have in the path in ascending alphabetical order
123
123
  '''
124
124
  return SftpPath(path).listdir()
125
125
 
@@ -136,18 +136,20 @@ def sftp_load_from(path: PathLike) -> BinaryIO:
136
136
 
137
137
 
138
138
  def sftp_makedirs(
139
- path: PathLike, mode=0o777, parents: bool = False,
139
+ path: PathLike,
140
+ mode=0o777,
141
+ parents: bool = False,
140
142
  exist_ok: bool = False):
141
143
  '''
142
- make a directory on sftp, including parent directory
143
-
144
+ make a directory on sftp, including parent directory.
144
145
  If there exists a file on the path, raise FileExistsError
145
146
 
146
147
  :param path: Given path
147
148
  :param mode: If mode is given, it is combined with the process’ umask value to determine the file mode and access flags.
148
149
  :param parents: If parents is true, any missing parents of this path are created as needed;
149
- If parents is false (the default), a missing parent raises FileNotFoundError.
150
+ If parents is false (the default), a missing parent raises FileNotFoundError.
150
151
  :param exist_ok: If False and target directory exists, raise FileExistsError
152
+
151
153
  :raises: FileExistsError
152
154
  '''
153
155
  return SftpPath(path).mkdir(mode, parents, exist_ok)
@@ -163,7 +165,8 @@ def sftp_realpath(path: PathLike) -> str:
163
165
 
164
166
 
165
167
  def sftp_rename(
166
- src_path: PathLike, dst_path: PathLike,
168
+ src_path: PathLike,
169
+ dst_path: PathLike,
167
170
  overwrite: bool = True) -> 'SftpPath':
168
171
  '''
169
172
  rename file on sftp
@@ -176,7 +179,8 @@ def sftp_rename(
176
179
 
177
180
 
178
181
  def sftp_move(
179
- src_path: PathLike, dst_path: PathLike,
182
+ src_path: PathLike,
183
+ dst_path: PathLike,
180
184
  overwrite: bool = True) -> 'SftpPath':
181
185
  '''
182
186
  move file on sftp
@@ -199,7 +203,8 @@ def sftp_remove(path: PathLike, missing_ok: bool = False) -> None:
199
203
 
200
204
 
201
205
  def sftp_scan(
202
- path: PathLike, missing_ok: bool = True,
206
+ path: PathLike,
207
+ missing_ok: bool = True,
203
208
  followlinks: bool = False) -> Iterator[str]:
204
209
  '''
205
210
  Iteratively traverse only files in given directory, in alphabetical order.
@@ -217,7 +222,8 @@ def sftp_scan(
217
222
 
218
223
 
219
224
  def sftp_scan_stat(
220
- path: PathLike, missing_ok: bool = True,
225
+ path: PathLike,
226
+ missing_ok: bool = True,
221
227
  followlinks: bool = False) -> Iterator[FileEntry]:
222
228
  '''
223
229
  Iteratively traverse only files in given directory, in alphabetical order.
@@ -260,8 +266,10 @@ def sftp_unlink(path: PathLike, missing_ok: bool = False) -> None:
260
266
  return SftpPath(path).unlink(missing_ok)
261
267
 
262
268
 
263
- def sftp_walk(path: PathLike, followlinks: bool = False
264
- ) -> Iterator[Tuple[str, List[str], List[str]]]:
269
+ def sftp_walk(
270
+ path: PathLike,
271
+ followlinks: bool = False
272
+ ) -> Iterator[Tuple[str, List[str], List[str]]]:
265
273
  '''
266
274
  Generate the file names in a directory tree by walking the tree top-down.
267
275
  For each directory in the tree rooted at directory path (including path itself),
@@ -292,6 +300,7 @@ def sftp_getmd5(
292
300
  :param path: Given path
293
301
  :param recalculate: Ignore this parameter, just for compatibility
294
302
  :param followlinks: Ignore this parameter, just for compatibility
303
+
295
304
  returns: md5 of file
296
305
  '''
297
306
  return SftpPath(path).md5(recalculate, followlinks)
@@ -302,7 +311,7 @@ def sftp_symlink(src_path: PathLike, dst_path: PathLike) -> None:
302
311
  Create a symbolic link pointing to src_path named dst_path.
303
312
 
304
313
  :param src_path: Given path
305
- :param dst_path: Desination path
314
+ :param dst_path: Destination path
306
315
  '''
307
316
  return SftpPath(src_path).symlink(dst_path)
308
317
 
@@ -333,7 +342,7 @@ def sftp_open(
333
342
  buffering=-1,
334
343
  encoding: Optional[str] = None,
335
344
  errors: Optional[str] = None,
336
- **kwargs) -> IO[AnyStr]: # pytype: disable=signature-mismatch
345
+ **kwargs) -> IO:
337
346
  '''Open a file on the path.
338
347
 
339
348
  :param path: Given path
@@ -402,7 +411,7 @@ def sftp_sync(
402
411
  :param src_path: Given path
403
412
  :param dst_url: Given destination path
404
413
  :param followlinks: False if regard symlink as file, else True
405
- :param force: Sync file forcely, do not ignore same files, priority is higher than 'overwrite', default is False
414
+ :param force: Sync file forcible, do not ignore same files, priority is higher than 'overwrite', default is False
406
415
  :param overwrite: whether or not overwrite file when exists, default is True
407
416
  '''
408
417
  return SftpPath(src_path).sync(dst_path, followlinks, force, overwrite)
megfile/sftp_path.py CHANGED
@@ -7,9 +7,10 @@ import random
7
7
  import shlex
8
8
  import socket
9
9
  import subprocess
10
+ from functools import cached_property
10
11
  from logging import getLogger as get_logger
11
12
  from stat import S_ISDIR, S_ISLNK, S_ISREG
12
- from typing import IO, AnyStr, BinaryIO, Callable, Iterator, List, Optional, Tuple, Union
13
+ from typing import IO, BinaryIO, Callable, Iterator, List, Optional, Tuple, Union
13
14
  from urllib.parse import urlsplit, urlunsplit
14
15
 
15
16
  import paramiko
@@ -23,7 +24,7 @@ from megfile.lib.glob import FSFunc, iglob
23
24
  from megfile.lib.joinpath import uri_join
24
25
  from megfile.pathlike import PathLike, URIPath
25
26
  from megfile.smart_path import SmartPath
26
- from megfile.utils import cachedproperty, calculate_md5, thread_local
27
+ from megfile.utils import calculate_md5, thread_local
27
28
 
28
29
  _logger = get_logger(__name__)
29
30
 
@@ -55,10 +56,10 @@ DEFAULT_SSH_KEEPALIVE_INTERVAL = 15
55
56
 
56
57
  def _make_stat(stat: paramiko.SFTPAttributes) -> StatResult:
57
58
  return StatResult(
58
- size=stat.st_size,
59
- mtime=stat.st_mtime,
60
- isdir=S_ISDIR(stat.st_mode),
61
- islnk=S_ISLNK(stat.st_mode),
59
+ size=stat.st_size or 0,
60
+ mtime=stat.st_mtime or 0.0,
61
+ isdir=S_ISDIR(stat.st_mode) if stat.st_mode is not None else False,
62
+ islnk=S_ISLNK(stat.st_mode) if stat.st_mode is not None else False,
62
63
  extra=stat,
63
64
  )
64
65
 
@@ -82,10 +83,10 @@ def get_private_key():
82
83
 
83
84
 
84
85
  def provide_connect_info(
85
- hostname: str,
86
- port: Optional[int] = None,
87
- username: Optional[str] = None,
88
- password: Optional[str] = None,
86
+ hostname: str,
87
+ port: Optional[int] = None,
88
+ username: Optional[str] = None,
89
+ password: Optional[str] = None,
89
90
  ):
90
91
  if not port:
91
92
  port = 22
@@ -117,11 +118,11 @@ def sftp_should_retry(error: Exception) -> bool:
117
118
 
118
119
 
119
120
  def _patch_sftp_client_request(
120
- client: paramiko.SFTPClient,
121
- hostname: str,
122
- port: Optional[int] = None,
123
- username: Optional[str] = None,
124
- password: Optional[str] = None,
121
+ client: paramiko.SFTPClient,
122
+ hostname: str,
123
+ port: Optional[int] = None,
124
+ username: Optional[str] = None,
125
+ password: Optional[str] = None,
125
126
  ):
126
127
 
127
128
  def retry_callback(error, *args, **kwargs):
@@ -144,7 +145,7 @@ def _patch_sftp_client_request(
144
145
  )
145
146
  client.sock = new_sftp_client.sock
146
147
 
147
- client._request = patch_method(
148
+ client._request = patch_method( # pyre-ignore[16]
148
149
  client._request, # pytype: disable=attribute-error
149
150
  max_retries=MAX_RETRIES,
150
151
  should_retry=sftp_should_retry,
@@ -153,10 +154,10 @@ def _patch_sftp_client_request(
153
154
 
154
155
 
155
156
  def _get_sftp_client(
156
- hostname: str,
157
- port: Optional[int] = None,
158
- username: Optional[str] = None,
159
- password: Optional[str] = None,
157
+ hostname: str,
158
+ port: Optional[int] = None,
159
+ username: Optional[str] = None,
160
+ password: Optional[str] = None,
160
161
  ) -> paramiko.SFTPClient:
161
162
  '''Get sftp client
162
163
 
@@ -175,10 +176,10 @@ def _get_sftp_client(
175
176
 
176
177
 
177
178
  def get_sftp_client(
178
- hostname: str,
179
- port: Optional[int] = None,
180
- username: Optional[str] = None,
181
- password: Optional[str] = None,
179
+ hostname: str,
180
+ port: Optional[int] = None,
181
+ username: Optional[str] = None,
182
+ password: Optional[str] = None,
182
183
  ) -> paramiko.SFTPClient:
183
184
  '''Get sftp client
184
185
 
@@ -190,10 +191,10 @@ def get_sftp_client(
190
191
 
191
192
 
192
193
  def _get_ssh_client(
193
- hostname: str,
194
- port: Optional[int] = None,
195
- username: Optional[str] = None,
196
- password: Optional[str] = None,
194
+ hostname: str,
195
+ port: Optional[int] = None,
196
+ username: Optional[str] = None,
197
+ password: Optional[str] = None,
197
198
  ) -> paramiko.SSHClient:
198
199
  hostname, port, username, password, private_key = provide_connect_info(
199
200
  hostname=hostname,
@@ -236,10 +237,10 @@ def _get_ssh_client(
236
237
 
237
238
 
238
239
  def get_ssh_client(
239
- hostname: str,
240
- port: Optional[int] = None,
241
- username: Optional[str] = None,
242
- password: Optional[str] = None,
240
+ hostname: str,
241
+ port: Optional[int] = None,
242
+ username: Optional[str] = None,
243
+ password: Optional[str] = None,
243
244
  ) -> paramiko.SSHClient:
244
245
  return thread_local(
245
246
  f'ssh_client:{hostname},{port},{username},{password}', _get_ssh_client,
@@ -247,10 +248,10 @@ def get_ssh_client(
247
248
 
248
249
 
249
250
  def get_ssh_session(
250
- hostname: str,
251
- port: Optional[int] = None,
252
- username: Optional[str] = None,
253
- password: Optional[str] = None,
251
+ hostname: str,
252
+ port: Optional[int] = None,
253
+ username: Optional[str] = None,
254
+ password: Optional[str] = None,
254
255
  ) -> paramiko.Channel:
255
256
 
256
257
  def retry_callback(error, *args, **kwargs):
@@ -265,7 +266,7 @@ def get_ssh_session(
265
266
  del thread_local[sftp_key]
266
267
 
267
268
  return patch_method(
268
- _open_session, # pytype: disable=attribute-error
269
+ _open_session,
269
270
  max_retries=MAX_RETRIES,
270
271
  should_retry=sftp_should_retry,
271
272
  retry_callback=retry_callback)(
@@ -277,10 +278,10 @@ def get_ssh_session(
277
278
 
278
279
 
279
280
  def _open_session(
280
- hostname: str,
281
- port: Optional[int] = None,
282
- username: Optional[str] = None,
283
- password: Optional[str] = None,
281
+ hostname: str,
282
+ port: Optional[int] = None,
283
+ username: Optional[str] = None,
284
+ password: Optional[str] = None,
284
285
  ) -> paramiko.Channel:
285
286
  ssh_client = get_ssh_client(hostname, port, username, password)
286
287
  transport = ssh_client.get_transport()
@@ -314,7 +315,8 @@ def sftp_readlink(path: PathLike) -> 'str':
314
315
  return SftpPath(path).readlink().path_with_protocol
315
316
 
316
317
 
317
- def sftp_glob(path: PathLike, recursive: bool = True,
318
+ def sftp_glob(path: PathLike,
319
+ recursive: bool = True,
318
320
  missing_ok: bool = True) -> List[str]:
319
321
  '''Return path list in ascending alphabetical order, in which path matches glob pattern
320
322
 
@@ -324,7 +326,7 @@ def sftp_glob(path: PathLike, recursive: bool = True,
324
326
  Assume there exists a path `/a/b/c/b/d.txt`
325
327
  use path pattern like `/**/b/**/*.txt` to glob, the path above will be returned twice
326
328
  3. `**` will match any matched file, directory, symlink and '' by default, when recursive is `True`
327
- 4. fs_glob returns same as glob.glob(pathname, recursive=True) in acsending alphabetical order.
329
+ 4. fs_glob returns same as glob.glob(pathname, recursive=True) in ascending alphabetical order.
328
330
  5. Hidden files (filename stars with '.') will not be found in the result
329
331
 
330
332
  :param path: Given path
@@ -338,7 +340,8 @@ def sftp_glob(path: PathLike, recursive: bool = True,
338
340
 
339
341
 
340
342
  def sftp_glob_stat(
341
- path: PathLike, recursive: bool = True,
343
+ path: PathLike,
344
+ recursive: bool = True,
342
345
  missing_ok: bool = True) -> Iterator[FileEntry]:
343
346
  '''Return a list contains tuples of path and file stat, in ascending alphabetical order, in which path matches glob pattern
344
347
 
@@ -348,7 +351,7 @@ def sftp_glob_stat(
348
351
  Assume there exists a path `/a/b/c/b/d.txt`
349
352
  use path pattern like `/**/b/**/*.txt` to glob, the path above will be returned twice
350
353
  3. `**` will match any matched file, directory, symlink and '' by default, when recursive is `True`
351
- 4. fs_glob returns same as glob.glob(pathname, recursive=True) in acsending alphabetical order.
354
+ 4. fs_glob returns same as glob.glob(pathname, recursive=True) in ascending alphabetical order.
352
355
  5. Hidden files (filename stars with '.') will not be found in the result
353
356
 
354
357
  :param path: Given path
@@ -365,7 +368,8 @@ def sftp_glob_stat(
365
368
  path_object.lstat())
366
369
 
367
370
 
368
- def sftp_iglob(path: PathLike, recursive: bool = True,
371
+ def sftp_iglob(path: PathLike,
372
+ recursive: bool = True,
369
373
  missing_ok: bool = True) -> Iterator[str]:
370
374
  '''Return path iterator in ascending alphabetical order, in which path matches glob pattern
371
375
 
@@ -375,7 +379,7 @@ def sftp_iglob(path: PathLike, recursive: bool = True,
375
379
  Assume there exists a path `/a/b/c/b/d.txt`
376
380
  use path pattern like `/**/b/**/*.txt` to glob, the path above will be returned twice
377
381
  3. `**` will match any matched file, directory, symlink and '' by default, when recursive is `True`
378
- 4. fs_glob returns same as glob.glob(pathname, recursive=True) in acsending alphabetical order.
382
+ 4. fs_glob returns same as glob.glob(pathname, recursive=True) in ascending alphabetical order.
379
383
  5. Hidden files (filename stars with '.') will not be found in the result
380
384
 
381
385
  :param path: Given path
@@ -459,7 +463,8 @@ def sftp_download(
459
463
 
460
464
  def sftp_callback(bytes_transferred: int, _total_bytes: int):
461
465
  nonlocal bytes_transferred_before
462
- callback(bytes_transferred - bytes_transferred_before)
466
+ callback( # pyre-ignore[29]
467
+ bytes_transferred - bytes_transferred_before)
463
468
  bytes_transferred_before = bytes_transferred
464
469
 
465
470
  src_path._client.get(
@@ -516,7 +521,8 @@ def sftp_upload(
516
521
 
517
522
  def sftp_callback(bytes_transferred: int, _total_bytes: int):
518
523
  nonlocal bytes_transferred_before
519
- callback(bytes_transferred - bytes_transferred_before)
524
+ callback( # pyre-ignore[29]
525
+ bytes_transferred - bytes_transferred_before)
520
526
  bytes_transferred_before = bytes_transferred
521
527
 
522
528
  dst_path._client.put(
@@ -581,10 +587,10 @@ class SftpPath(URIPath):
581
587
  """sftp protocol
582
588
 
583
589
  uri format:
584
- - absolute path
585
- - sftp://[username[:password]@]hostname[:port]//file_path
586
- - relative path
587
- - - sftp://[username[:password]@]hostname[:port]/file_path
590
+ - absolute path
591
+ - sftp://[username[:password]@]hostname[:port]//file_path
592
+ - relative path
593
+ - sftp://[username[:password]@]hostname[:port]/file_path
588
594
  """
589
595
 
590
596
  protocol = "sftp"
@@ -600,8 +606,8 @@ class SftpPath(URIPath):
600
606
  self._root_dir = self._client.normalize('.')
601
607
  self._real_path = os.path.join(self._root_dir, parts.path.lstrip('/'))
602
608
 
603
- @cachedproperty
604
- def parts(self) -> Tuple[str]:
609
+ @cached_property
610
+ def parts(self) -> Tuple[str, ...]:
605
611
  '''A tuple giving access to the path’s various components'''
606
612
  if self._urlsplit_parts.path.startswith('//'):
607
613
  new_parts = self._urlsplit_parts._replace(path='//')
@@ -611,7 +617,7 @@ class SftpPath(URIPath):
611
617
  path = self._urlsplit_parts.path.lstrip('/')
612
618
  if path != '':
613
619
  parts.extend(path.split('/'))
614
- return tuple(parts)
620
+ return tuple(parts) # pyre-ignore[7]
615
621
 
616
622
  @property
617
623
  def _client(self):
@@ -631,7 +637,7 @@ class SftpPath(URIPath):
631
637
  if sftp_local_path == ".":
632
638
  sftp_local_path = "/"
633
639
  new_parts = self._urlsplit_parts._replace(path=sftp_local_path)
634
- return self.from_path(urlunsplit(new_parts))
640
+ return self.from_path(urlunsplit(new_parts)) # pyre-ignore[6]
635
641
 
636
642
  def exists(self, followlinks: bool = False) -> bool:
637
643
  '''
@@ -670,7 +676,9 @@ class SftpPath(URIPath):
670
676
  '''
671
677
  return self.stat(follow_symlinks=follow_symlinks).size
672
678
 
673
- def glob(self, pattern, recursive: bool = True,
679
+ def glob(self,
680
+ pattern,
681
+ recursive: bool = True,
674
682
  missing_ok: bool = True) -> List['SftpPath']:
675
683
  '''Return path list in ascending alphabetical order, in which path matches glob pattern
676
684
 
@@ -680,7 +688,7 @@ class SftpPath(URIPath):
680
688
  Assume there exists a path `/a/b/c/b/d.txt`
681
689
  use path pattern like `/**/b/**/*.txt` to glob, the path above will be returned twice
682
690
  3. `**` will match any matched file, directory, symlink and '' by default, when recursive is `True`
683
- 4. fs_glob returns same as glob.glob(pathname, recursive=True) in acsending alphabetical order.
691
+ 4. fs_glob returns same as glob.glob(pathname, recursive=True) in ascending alphabetical order.
684
692
  5. Hidden files (filename stars with '.') will not be found in the result
685
693
 
686
694
  :param pattern: Glob the given relative pattern in the directory represented by this path
@@ -693,7 +701,9 @@ class SftpPath(URIPath):
693
701
  pattern=pattern, recursive=recursive, missing_ok=missing_ok))
694
702
 
695
703
  def glob_stat(
696
- self, pattern, recursive: bool = True,
704
+ self,
705
+ pattern,
706
+ recursive: bool = True,
697
707
  missing_ok: bool = True) -> Iterator[FileEntry]:
698
708
  '''Return a list contains tuples of path and file stat, in ascending alphabetical order, in which path matches glob pattern
699
709
 
@@ -703,7 +713,7 @@ class SftpPath(URIPath):
703
713
  Assume there exists a path `/a/b/c/b/d.txt`
704
714
  use path pattern like `/**/b/**/*.txt` to glob, the path above will be returned twice
705
715
  3. `**` will match any matched file, directory, symlink and '' by default, when recursive is `True`
706
- 4. fs_glob returns same as glob.glob(pathname, recursive=True) in acsending alphabetical order.
716
+ 4. fs_glob returns same as glob.glob(pathname, recursive=True) in ascending alphabetical order.
707
717
  5. Hidden files (filename stars with '.') will not be found in the result
708
718
 
709
719
  :param pattern: Glob the given relative pattern in the directory represented by this path
@@ -715,7 +725,9 @@ class SftpPath(URIPath):
715
725
  missing_ok=missing_ok):
716
726
  yield FileEntry(path_obj.name, path_obj.path, path_obj.lstat())
717
727
 
718
- def iglob(self, pattern, recursive: bool = True,
728
+ def iglob(self,
729
+ pattern,
730
+ recursive: bool = True,
719
731
  missing_ok: bool = True) -> Iterator['SftpPath']:
720
732
  '''Return path iterator in ascending alphabetical order, in which path matches glob pattern
721
733
 
@@ -725,7 +737,7 @@ class SftpPath(URIPath):
725
737
  Assume there exists a path `/a/b/c/b/d.txt`
726
738
  use path pattern like `/**/b/**/*.txt` to glob, the path above will be returned twice
727
739
  3. `**` will match any matched file, directory, symlink and '' by default, when recursive is `True`
728
- 4. fs_glob returns same as glob.glob(pathname, recursive=True) in acsending alphabetical order.
740
+ 4. fs_glob returns same as glob.glob(pathname, recursive=True) in ascending alphabetical order.
729
741
  5. Hidden files (filename stars with '.') will not be found in the result
730
742
 
731
743
  :param pattern: Glob the given relative pattern in the directory represented by this path
@@ -796,9 +808,9 @@ class SftpPath(URIPath):
796
808
 
797
809
  def listdir(self) -> List[str]:
798
810
  '''
799
- Get all contents of given sftp path. The result is in acsending alphabetical order.
811
+ Get all contents of given sftp path. The result is in ascending alphabetical order.
800
812
 
801
- :returns: All contents have in the path in acsending alphabetical order
813
+ :returns: All contents have in the path in ascending alphabetical order
802
814
  '''
803
815
  if not self.is_dir():
804
816
  raise NotADirectoryError(
@@ -807,15 +819,15 @@ class SftpPath(URIPath):
807
819
 
808
820
  def iterdir(self) -> Iterator['SftpPath']:
809
821
  '''
810
- Get all contents of given sftp path. The result is in acsending alphabetical order.
822
+ Get all contents of given sftp path. The result is in ascending alphabetical order.
811
823
 
812
- :returns: All contents have in the path in acsending alphabetical order
824
+ :returns: All contents have in the path in ascending alphabetical order
813
825
  '''
814
826
  if not self.is_dir():
815
827
  raise NotADirectoryError(
816
828
  f"Not a directory: '{self.path_with_protocol}'")
817
829
  for path in self.listdir():
818
- yield self.joinpath(path) # type: ignore
830
+ yield self.joinpath(path)
819
831
 
820
832
  def load(self) -> BinaryIO:
821
833
  '''Read all content on specified path and write into memory
@@ -830,14 +842,14 @@ class SftpPath(URIPath):
830
842
 
831
843
  def mkdir(self, mode=0o777, parents: bool = False, exist_ok: bool = False):
832
844
  '''
833
- make a directory on sftp, including parent directory
834
-
845
+ make a directory on sftp, including parent directory.
835
846
  If there exists a file on the path, raise FileExistsError
836
847
 
837
848
  :param mode: If mode is given, it is combined with the process’ umask value to determine the file mode and access flags.
838
849
  :param parents: If parents is true, any missing parents of this path are created as needed;
839
- If parents is false (the default), a missing parent raises FileNotFoundError.
850
+ If parents is false (the default), a missing parent raises FileNotFoundError.
840
851
  :param exist_ok: If False and target directory exists, raise FileExistsError
852
+
841
853
  :raises: FileExistsError
842
854
  '''
843
855
  if self.exists():
@@ -943,7 +955,8 @@ class SftpPath(URIPath):
943
955
  else:
944
956
  self._client.unlink(self._real_path)
945
957
 
946
- def scan(self, missing_ok: bool = True,
958
+ def scan(self,
959
+ missing_ok: bool = True,
947
960
  followlinks: bool = False) -> Iterator[str]:
948
961
  '''
949
962
  Iteratively traverse only files in given directory, in alphabetical order.
@@ -962,7 +975,8 @@ class SftpPath(URIPath):
962
975
  for file_entry in scan_stat_iter:
963
976
  yield file_entry.path
964
977
 
965
- def scan_stat(self, missing_ok: bool = True,
978
+ def scan_stat(self,
979
+ missing_ok: bool = True,
966
980
  followlinks: bool = False) -> Iterator[FileEntry]:
967
981
  '''
968
982
  Iteratively traverse only files in given directory, in alphabetical order.
@@ -993,8 +1007,7 @@ class SftpPath(URIPath):
993
1007
  )
994
1008
  else:
995
1009
  yield FileEntry(
996
- current_path.name, # type: ignore
997
- current_path.path_with_protocol,
1010
+ current_path.name, current_path.path_with_protocol,
998
1011
  current_path.stat(follow_symlinks=followlinks))
999
1012
 
1000
1013
  return _create_missing_ok_generator(
@@ -1020,9 +1033,8 @@ class SftpPath(URIPath):
1020
1033
  for name in self.listdir():
1021
1034
  current_path = self.joinpath(name)
1022
1035
  yield FileEntry(
1023
- current_path.name, # type: ignore
1024
- current_path.path_with_protocol,
1025
- current_path.lstat()) # type: ignore
1036
+ current_path.name, current_path.path_with_protocol,
1037
+ current_path.lstat())
1026
1038
 
1027
1039
  return ContextIterator(create_generator())
1028
1040
 
@@ -1048,8 +1060,10 @@ class SftpPath(URIPath):
1048
1060
  return
1049
1061
  self._client.unlink(self._real_path)
1050
1062
 
1051
- def walk(self, followlinks: bool = False
1052
- ) -> Iterator[Tuple[str, List[str], List[str]]]:
1063
+ def walk(
1064
+ self,
1065
+ followlinks: bool = False
1066
+ ) -> Iterator[Tuple[str, List[str], List[str]]]:
1053
1067
  '''
1054
1068
  Generate the file names in a directory tree by walking the tree top-down.
1055
1069
  For each directory in the tree rooted at directory path (including path itself),
@@ -1111,16 +1125,17 @@ class SftpPath(URIPath):
1111
1125
 
1112
1126
  :param recalculate: Ignore this parameter, just for compatibility
1113
1127
  :param followlinks: Ignore this parameter, just for compatibility
1128
+
1114
1129
  returns: md5 of file
1115
1130
  '''
1116
1131
  if self.is_dir():
1117
1132
  hash_md5 = hashlib.md5() # nosec
1118
1133
  for file_name in self.listdir():
1119
- chunk = self.joinpath(file_name).md5( # type: ignore
1134
+ chunk = self.joinpath(file_name).md5(
1120
1135
  recalculate=recalculate, followlinks=followlinks).encode()
1121
1136
  hash_md5.update(chunk)
1122
1137
  return hash_md5.hexdigest()
1123
- with self.open('rb') as src: # type: ignore
1138
+ with self.open('rb') as src:
1124
1139
  md5 = calculate_md5(src)
1125
1140
  return md5
1126
1141
 
@@ -1128,7 +1143,7 @@ class SftpPath(URIPath):
1128
1143
  '''
1129
1144
  Create a symbolic link pointing to src_path named dst_path.
1130
1145
 
1131
- :param dst_path: Desination path
1146
+ :param dst_path: Destination path
1132
1147
  '''
1133
1148
  dst_path = self.from_path(dst_path)
1134
1149
  if dst_path.exists(followlinks=False):
@@ -1180,7 +1195,7 @@ class SftpPath(URIPath):
1180
1195
  buffering=-1,
1181
1196
  encoding: Optional[str] = None,
1182
1197
  errors: Optional[str] = None,
1183
- **kwargs) -> IO[AnyStr]: # pytype: disable=signature-mismatch
1198
+ **kwargs) -> IO:
1184
1199
  '''Open a file on the path.
1185
1200
 
1186
1201
  :param mode: Mode to open file
@@ -1200,9 +1215,8 @@ class SftpPath(URIPath):
1200
1215
  fileobj = self._client.open(self._real_path, mode, bufsize=buffering)
1201
1216
  fileobj.name = self.path
1202
1217
  if 'r' in mode and 'b' not in mode:
1203
- return io.TextIOWrapper(
1204
- fileobj, encoding=encoding, errors=errors) # type: ignore
1205
- return fileobj # type: ignore
1218
+ return io.TextIOWrapper(fileobj, encoding=encoding, errors=errors) # pytype: disable=wrong-arg-types
1219
+ return fileobj # pytype: disable=bad-return-type
1206
1220
 
1207
1221
  def chmod(self, mode: int, follow_symlinks: bool = True):
1208
1222
  '''
@@ -1228,11 +1242,11 @@ class SftpPath(URIPath):
1228
1242
  return self._client.rmdir(self._real_path)
1229
1243
 
1230
1244
  def _exec_command(
1231
- self,
1232
- command: List[str],
1233
- bufsize: int = -1,
1234
- timeout: Optional[int] = None,
1235
- environment: Optional[dict] = None,
1245
+ self,
1246
+ command: List[str],
1247
+ bufsize: int = -1,
1248
+ timeout: Optional[int] = None,
1249
+ environment: Optional[dict] = None,
1236
1250
  ) -> subprocess.CompletedProcess:
1237
1251
  with get_ssh_session(
1238
1252
  hostname=self._urlsplit_parts.hostname,
@@ -1283,7 +1297,8 @@ class SftpPath(URIPath):
1283
1297
  if not overwrite and self.from_path(dst_path).exists():
1284
1298
  return
1285
1299
 
1286
- self.from_path(os.path.dirname(dst_path)).makedirs(exist_ok=True)
1300
+ self.from_path(os.path.dirname(
1301
+ fspath(dst_path))).makedirs(exist_ok=True)
1287
1302
  dst_path = self.from_path(dst_path)
1288
1303
  if self._is_same_backend(dst_path):
1289
1304
  if self._real_path == dst_path._real_path:
@@ -1323,7 +1338,7 @@ class SftpPath(URIPath):
1323
1338
 
1324
1339
  :param dst_url: Given destination path
1325
1340
  :param followlinks: False if regard symlink as file, else True
1326
- :param force: Sync file forcely, do not ignore same files, priority is higher than 'overwrite', default is False
1341
+ :param force: Sync file forcible, do not ignore same files, priority is higher than 'overwrite', default is False
1327
1342
  :param overwrite: whether or not overwrite file when exists, default is True
1328
1343
  '''
1329
1344
  if not self._is_same_protocol(dst_path):