pyglove 0.5.0.dev202510020810__py3-none-any.whl → 0.5.0.dev202512280810__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of pyglove might be problematic. Click here for more details.

Files changed (40) hide show
  1. pyglove/core/geno/base.py +7 -3
  2. pyglove/core/io/file_system.py +452 -2
  3. pyglove/core/io/file_system_test.py +442 -0
  4. pyglove/core/monitoring.py +213 -90
  5. pyglove/core/monitoring_test.py +82 -29
  6. pyglove/core/symbolic/__init__.py +7 -0
  7. pyglove/core/symbolic/base.py +89 -35
  8. pyglove/core/symbolic/base_test.py +3 -3
  9. pyglove/core/symbolic/dict.py +31 -12
  10. pyglove/core/symbolic/dict_test.py +49 -0
  11. pyglove/core/symbolic/list.py +17 -3
  12. pyglove/core/symbolic/list_test.py +24 -2
  13. pyglove/core/symbolic/object.py +3 -1
  14. pyglove/core/symbolic/object_test.py +13 -10
  15. pyglove/core/symbolic/ref.py +19 -7
  16. pyglove/core/symbolic/ref_test.py +94 -7
  17. pyglove/core/symbolic/unknown_symbols.py +147 -0
  18. pyglove/core/symbolic/unknown_symbols_test.py +100 -0
  19. pyglove/core/typing/annotation_conversion.py +8 -1
  20. pyglove/core/typing/annotation_conversion_test.py +14 -19
  21. pyglove/core/typing/class_schema.py +24 -1
  22. pyglove/core/typing/json_schema.py +221 -8
  23. pyglove/core/typing/json_schema_test.py +508 -12
  24. pyglove/core/typing/type_conversion.py +17 -3
  25. pyglove/core/typing/type_conversion_test.py +7 -2
  26. pyglove/core/typing/value_specs.py +5 -1
  27. pyglove/core/typing/value_specs_test.py +5 -0
  28. pyglove/core/utils/__init__.py +1 -0
  29. pyglove/core/utils/contextual.py +9 -4
  30. pyglove/core/utils/contextual_test.py +10 -0
  31. pyglove/core/utils/json_conversion.py +360 -63
  32. pyglove/core/utils/json_conversion_test.py +146 -13
  33. pyglove/core/views/html/controls/tab.py +33 -0
  34. pyglove/core/views/html/controls/tab_test.py +37 -0
  35. pyglove/ext/evolution/base_test.py +1 -1
  36. {pyglove-0.5.0.dev202510020810.dist-info → pyglove-0.5.0.dev202512280810.dist-info}/METADATA +8 -1
  37. {pyglove-0.5.0.dev202510020810.dist-info → pyglove-0.5.0.dev202512280810.dist-info}/RECORD +40 -38
  38. {pyglove-0.5.0.dev202510020810.dist-info → pyglove-0.5.0.dev202512280810.dist-info}/WHEEL +0 -0
  39. {pyglove-0.5.0.dev202510020810.dist-info → pyglove-0.5.0.dev202512280810.dist-info}/licenses/LICENSE +0 -0
  40. {pyglove-0.5.0.dev202510020810.dist-info → pyglove-0.5.0.dev202512280810.dist-info}/top_level.txt +0 -0
pyglove/core/geno/base.py CHANGED
@@ -1447,6 +1447,7 @@ class DNA(symbolic.Object):
1447
1447
  *,
1448
1448
  allow_partial: bool = False,
1449
1449
  root_path: Optional[utils.KeyPath] = None,
1450
+ **kwargs,
1450
1451
  ) -> 'DNA':
1451
1452
  """Class method that load a DNA from a JSON value.
1452
1453
 
@@ -1454,6 +1455,7 @@ class DNA(symbolic.Object):
1454
1455
  json_value: Input JSON value, only JSON dict is acceptable.
1455
1456
  allow_partial: Whether to allow elements of the list to be partial.
1456
1457
  root_path: KeyPath of loaded object in its object tree.
1458
+ **kwargs: Keyword arguments that will be passed to symbolic.from_json.
1457
1459
 
1458
1460
  Returns:
1459
1461
  A DNA object.
@@ -1463,16 +1465,18 @@ class DNA(symbolic.Object):
1463
1465
  # NOTE(daiyip): DNA.parse will validate the input. Therefore, we can
1464
1466
  # disable runtime type check during constructing the DNA objects.
1465
1467
  with symbolic.enable_type_check(False):
1466
- dna = DNA.parse(symbolic.from_json(json_value.get('value')))
1468
+ dna = DNA.parse(symbolic.from_json(json_value.get('value'), **kwargs))
1467
1469
  if 'metadata' in json_value:
1468
1470
  dna.rebind(
1469
- metadata=symbolic.from_json(json_value.get('metadata')),
1471
+ metadata=symbolic.from_json(json_value.get('metadata'), **kwargs),
1470
1472
  raise_on_no_change=False, skip_notification=True)
1471
1473
  else:
1472
1474
  dna = super(DNA, cls).from_json(
1473
1475
  json_value,
1474
1476
  allow_partial=allow_partial,
1475
- root_path=root_path) # pytype: disable=bad-return-type
1477
+ root_path=root_path,
1478
+ **kwargs,
1479
+ ) # pytype: disable=bad-return-type
1476
1480
  assert isinstance(dna, DNA)
1477
1481
  if cloneable_metadata_keys:
1478
1482
  dna._cloneable_metadata_keys = set(cloneable_metadata_keys) # pylint: disable=protected-access
@@ -14,8 +14,13 @@
14
14
  """Pluggable file system."""
15
15
 
16
16
  import abc
17
+ import datetime
18
+ import fnmatch
19
+ import glob as std_glob
17
20
  import io
18
21
  import os
22
+ import re
23
+ import shutil
19
24
  from typing import Any, Literal, Optional, Union
20
25
 
21
26
 
@@ -84,6 +89,10 @@ class FileSystem(metaclass=abc.ABCMeta):
84
89
  def exists(self, path: Union[str, os.PathLike[str]]) -> bool:
85
90
  """Returns True if a path exists."""
86
91
 
92
+ @abc.abstractmethod
93
+ def glob(self, pattern: Union[str, os.PathLike[str]]) -> list[str]:
94
+ """Lists all files or sub-directories."""
95
+
87
96
  @abc.abstractmethod
88
97
  def listdir(self, path: Union[str, os.PathLike[str]]) -> list[str]:
89
98
  """Lists all files or sub-directories."""
@@ -92,6 +101,14 @@ class FileSystem(metaclass=abc.ABCMeta):
92
101
  def isdir(self, path: Union[str, os.PathLike[str]]) -> bool:
93
102
  """Returns True if a path is a directory."""
94
103
 
104
+ @abc.abstractmethod
105
+ def getctime(self, path: Union[str, os.PathLike[str]]) -> float:
106
+ """Returns the creation time of a file."""
107
+
108
+ @abc.abstractmethod
109
+ def getmtime(self, path: Union[str, os.PathLike[str]]) -> float:
110
+ """Returns the last modification time of a file."""
111
+
95
112
  @abc.abstractmethod
96
113
  def mkdir(
97
114
  self, path: Union[str, os.PathLike[str]], mode: int = 0o777
@@ -111,6 +128,14 @@ class FileSystem(metaclass=abc.ABCMeta):
111
128
  def rm(self, path: Union[str, os.PathLike[str]]) -> None:
112
129
  """Removes a file based on a path."""
113
130
 
131
+ @abc.abstractmethod
132
+ def rename(
133
+ self,
134
+ oldpath: Union[str, os.PathLike[str]],
135
+ newpath: Union[str, os.PathLike[str]],
136
+ ) -> None:
137
+ """Renames a file based on a path."""
138
+
114
139
  @abc.abstractmethod
115
140
  def rmdir(self, path: Union[str, os.PathLike[str]]) -> bool:
116
141
  """Removes a directory based on a path."""
@@ -119,6 +144,14 @@ class FileSystem(metaclass=abc.ABCMeta):
119
144
  def rmdirs(self, path: Union[str, os.PathLike[str]]) -> None:
120
145
  """Removes a directory chain based on a path."""
121
146
 
147
+ @abc.abstractmethod
148
+ def copy(
149
+ self,
150
+ oldpath: Union[str, os.PathLike[str]],
151
+ newpath: Union[str, os.PathLike[str]],
152
+ ) -> None:
153
+ """Copies a file to a new path."""
154
+
122
155
 
123
156
  def resolve_path(path: Union[str, os.PathLike[str]]) -> str:
124
157
  if isinstance(path, str):
@@ -137,9 +170,10 @@ def resolve_path(path: Union[str, os.PathLike[str]]) -> str:
137
170
  class StdFile(File):
138
171
  """The standard file."""
139
172
 
140
- def __init__(self, file_object) -> None:
173
+ def __init__(self, file_object, path: Union[str, os.PathLike[str]]) -> None:
141
174
  super().__init__()
142
175
  self._file_object = file_object
176
+ self._path = path
143
177
 
144
178
  def read(self, size: Optional[int] = None) -> Union[str, bytes]:
145
179
  return self._file_object.read(size)
@@ -169,7 +203,7 @@ class StdFileSystem(FileSystem):
169
203
  def open(
170
204
  self, path: Union[str, os.PathLike[str]], mode: str = 'r', **kwargs
171
205
  ) -> File:
172
- return StdFile(io.open(path, mode, **kwargs))
206
+ return StdFile(io.open(path, mode, **kwargs), path)
173
207
 
174
208
  def chmod(self, path: Union[str, os.PathLike[str]], mode: int) -> None:
175
209
  os.chmod(path, mode)
@@ -177,12 +211,21 @@ class StdFileSystem(FileSystem):
177
211
  def exists(self, path: Union[str, os.PathLike[str]]) -> bool:
178
212
  return os.path.exists(path)
179
213
 
214
+ def glob(self, pattern: Union[str, os.PathLike[str]]) -> list[str]:
215
+ return std_glob.glob(pattern)
216
+
180
217
  def listdir(self, path: Union[str, os.PathLike[str]]) -> list[str]:
181
218
  return os.listdir(path)
182
219
 
183
220
  def isdir(self, path: Union[str, os.PathLike[str]]) -> bool:
184
221
  return os.path.isdir(path)
185
222
 
223
+ def getctime(self, path: Union[str, os.PathLike[str]]) -> float:
224
+ return os.path.getctime(path)
225
+
226
+ def getmtime(self, path: Union[str, os.PathLike[str]]) -> float:
227
+ return os.path.getmtime(path)
228
+
186
229
  def mkdir(
187
230
  self, path: Union[str, os.PathLike[str]], mode: int = 0o777
188
231
  ) -> None:
@@ -196,6 +239,13 @@ class StdFileSystem(FileSystem):
196
239
  ) -> None:
197
240
  os.makedirs(path, mode, exist_ok)
198
241
 
242
+ def rename(
243
+ self,
244
+ oldpath: Union[str, os.PathLike[str]],
245
+ newpath: Union[str, os.PathLike[str]],
246
+ ) -> None:
247
+ os.rename(oldpath, newpath)
248
+
199
249
  def rm(self, path: Union[str, os.PathLike[str]]) -> None:
200
250
  os.remove(path)
201
251
 
@@ -205,6 +255,13 @@ class StdFileSystem(FileSystem):
205
255
  def rmdirs(self, path: Union[str, os.PathLike[str]]) -> None:
206
256
  os.removedirs(path)
207
257
 
258
+ def copy(
259
+ self,
260
+ oldpath: Union[str, os.PathLike[str]],
261
+ newpath: Union[str, os.PathLike[str]],
262
+ ) -> None:
263
+ shutil.copy(oldpath, newpath)
264
+
208
265
 
209
266
  #
210
267
  # Memory file system.
@@ -218,6 +275,9 @@ class MemoryFile(File):
218
275
  super().__init__()
219
276
  self._buffer = buffer
220
277
  self._pos = 0
278
+ now = datetime.datetime.now().timestamp()
279
+ self.ctime = now
280
+ self.mtime = now
221
281
 
222
282
  def read(self, size: Optional[int] = None) -> Union[str, bytes]:
223
283
  return self._buffer.read(size)
@@ -227,6 +287,7 @@ class MemoryFile(File):
227
287
 
228
288
  def write(self, content: Union[str, bytes]) -> None:
229
289
  self._buffer.write(content)
290
+ self.mtime = datetime.datetime.now().timestamp()
230
291
 
231
292
  def seek(self, offset: int, whence: Literal[0, 1, 2] = 0) -> int:
232
293
  return self._buffer.seek(offset, whence)
@@ -286,6 +347,21 @@ class MemoryFileSystem(FileSystem):
286
347
  def exists(self, path: Union[str, os.PathLike[str]]) -> bool:
287
348
  return self._locate(path) is not None
288
349
 
350
+ def glob(self, pattern: Union[str, os.PathLike[str]]) -> list[str]:
351
+ pattern = resolve_path(pattern)
352
+ regex = re.compile(fnmatch.translate(pattern))
353
+
354
+ results = []
355
+ def _traverse(node, prefix_parts):
356
+ for k, v in node.items():
357
+ path = self._prefix + '/'.join(prefix_parts + [k])
358
+ if regex.match(path):
359
+ results.append(path)
360
+ if isinstance(v, dict):
361
+ _traverse(v, prefix_parts + [k])
362
+ _traverse(self._root, [])
363
+ return results
364
+
289
365
  def listdir(self, path: Union[str, os.PathLike[str]]) -> list[str]:
290
366
  d = self._locate(path)
291
367
  if not isinstance(d, dict):
@@ -295,6 +371,24 @@ class MemoryFileSystem(FileSystem):
295
371
  def isdir(self, path: Union[str, os.PathLike[str]]) -> bool:
296
372
  return isinstance(self._locate(path), dict)
297
373
 
374
+ def getctime(self, path: Union[str, os.PathLike[str]]) -> float:
375
+ """Returns the creation time of a file."""
376
+ f = self._locate(path)
377
+ if isinstance(f, dict):
378
+ raise IsADirectoryError(path)
379
+ if f is None:
380
+ raise FileNotFoundError(path)
381
+ return f.ctime
382
+
383
+ def getmtime(self, path: Union[str, os.PathLike[str]]) -> float:
384
+ """Returns the last modification time of a file."""
385
+ f = self._locate(path)
386
+ if isinstance(f, dict):
387
+ raise IsADirectoryError(path)
388
+ if f is None:
389
+ raise FileNotFoundError(path)
390
+ return f.mtime
391
+
298
392
  def _parent_and_name(
299
393
  self, path: Union[str, os.PathLike[str]]
300
394
  ) -> tuple[dict[str, Any], str]:
@@ -338,6 +432,46 @@ class MemoryFileSystem(FileSystem):
338
432
  raise NotADirectoryError(path)
339
433
  current = entry
340
434
 
435
+ def rename(
436
+ self,
437
+ oldpath: Union[str, os.PathLike[str]],
438
+ newpath: Union[str, os.PathLike[str]],
439
+ ) -> None:
440
+ oldpath_str = resolve_path(oldpath)
441
+ newpath_str = resolve_path(newpath)
442
+
443
+ old_parent_dir, old_name = self._parent_and_name(oldpath_str)
444
+ entry = old_parent_dir.get(old_name)
445
+
446
+ if entry is None:
447
+ raise FileNotFoundError(oldpath_str)
448
+
449
+ new_entry = self._locate(newpath_str)
450
+ if new_entry is not None:
451
+ if isinstance(entry, dict): # oldpath is dir
452
+ if not isinstance(new_entry, dict):
453
+ raise NotADirectoryError(
454
+ f'Cannot rename directory {oldpath_str!r} '
455
+ f'to non-directory {newpath_str!r}')
456
+ elif new_entry:
457
+ raise OSError(f'Directory not empty: {newpath_str!r}')
458
+ else:
459
+ self.rmdir(newpath_str)
460
+ else: # oldpath is file
461
+ if isinstance(new_entry, dict):
462
+ raise IsADirectoryError(
463
+ f'Cannot rename non-directory {oldpath_str!r} '
464
+ f'to directory {newpath_str!r}')
465
+ else:
466
+ self.rm(newpath_str)
467
+ elif isinstance(entry, dict) and newpath_str.startswith(oldpath_str + '/'):
468
+ raise OSError(f'Cannot move directory {oldpath_str!r} '
469
+ f'to a subdirectory of itself {newpath_str!r}')
470
+
471
+ new_parent_dir, new_name = self._parent_and_name(newpath_str)
472
+ new_parent_dir[new_name] = entry
473
+ del old_parent_dir[old_name]
474
+
341
475
  def rm(self, path: Union[str, os.PathLike[str]]) -> None:
342
476
  parent_dir, name = self._parent_and_name(path)
343
477
  entry = parent_dir.get(name)
@@ -379,6 +513,37 @@ class MemoryFileSystem(FileSystem):
379
513
  return not dir_dict
380
514
  _rmdir(self._root, self._internal_path(path))
381
515
 
516
+ def copy(
517
+ self,
518
+ oldpath: Union[str, os.PathLike[str]],
519
+ newpath: Union[str, os.PathLike[str]],
520
+ ) -> None:
521
+ old_file = self._locate(oldpath)
522
+ if old_file is None:
523
+ raise FileNotFoundError(oldpath)
524
+ if not isinstance(old_file, MemoryFile):
525
+ raise IsADirectoryError(oldpath)
526
+
527
+ if self.isdir(newpath):
528
+ _, old_file_name = self._parent_and_name(oldpath)
529
+ newpath = resolve_path(newpath)
530
+ newpath = newpath.rstrip('/') + '/' + old_file_name
531
+
532
+ # If newpath exists as file, remove it for overwrite.
533
+ if self.exists(newpath) and not self.isdir(newpath):
534
+ self.rm(newpath)
535
+ elif self.isdir(newpath):
536
+ raise IsADirectoryError(newpath)
537
+
538
+ old_pos = old_file.tell()
539
+ old_file.seek(0)
540
+ content = old_file.read()
541
+ old_file.seek(old_pos)
542
+
543
+ is_binary = isinstance(old_file._buffer, io.BytesIO) # pylint: disable=protected-access
544
+ with self.open(newpath, 'wb' if is_binary else 'w') as f_new:
545
+ f_new.write(content)
546
+
382
547
 
383
548
  class _FileSystemRegistry:
384
549
  """File system registry."""
@@ -412,6 +577,260 @@ def add_file_system(prefix: str, fs: FileSystem) -> None:
412
577
  add_file_system('/mem/', MemoryFileSystem('/mem/'))
413
578
 
414
579
 
580
+ try:
581
+ # pylint: disable=g-import-not-at-top
582
+ # pytype: disable=import-error
583
+ import fsspec
584
+ # pytype: enable=import-error
585
+ # pylint: enable=g-import-not-at-top
586
+ except ImportError:
587
+ fsspec = None
588
+
589
+
590
+ class FsspecFile(File):
591
+ """File object based on fsspec."""
592
+
593
+ def __init__(self, f):
594
+ self._f = f
595
+
596
+ def read(self, size: Optional[int] = None) -> Union[str, bytes]:
597
+ return self._f.read(size)
598
+
599
+ def readline(self) -> Union[str, bytes]:
600
+ return self._f.readline()
601
+
602
+ def write(self, content: Union[str, bytes]) -> None:
603
+ self._f.write(content)
604
+
605
+ def seek(self, offset: int, whence: Literal[0, 1, 2] = 0) -> int:
606
+ return self._f.seek(offset, whence)
607
+
608
+ def tell(self) -> int:
609
+ return self._f.tell()
610
+
611
+ def flush(self) -> None:
612
+ self._f.flush()
613
+
614
+ def close(self) -> None:
615
+ self._f.close()
616
+
617
+
618
+ class FsspecFileSystem(FileSystem):
619
+ """File system based on fsspec."""
620
+
621
+ def open(
622
+ self, path: Union[str, os.PathLike[str]], mode: str = 'r', **kwargs
623
+ ) -> File:
624
+ assert fsspec is not None, '`fsspec` is not installed.'
625
+ return FsspecFile(fsspec.open(path, mode, **kwargs).open())
626
+
627
+ def chmod(self, path: Union[str, os.PathLike[str]], mode: int) -> None:
628
+ assert fsspec is not None, '`fsspec` is not installed.'
629
+ fs, path = fsspec.core.url_to_fs(path)
630
+ if hasattr(fs, 'chmod'):
631
+ fs.chmod(path, mode)
632
+
633
+ def exists(self, path: Union[str, os.PathLike[str]]) -> bool:
634
+ assert fsspec is not None, '`fsspec` is not installed.'
635
+ fs, path = fsspec.core.url_to_fs(path)
636
+ return fs.exists(path)
637
+
638
+ def glob(self, pattern: Union[str, os.PathLike[str]]) -> list[str]:
639
+ assert fsspec is not None, '`fsspec` is not installed.'
640
+ fs, path = fsspec.core.url_to_fs(pattern)
641
+ protocol = fsspec.utils.get_protocol(pattern)
642
+ return [f'{protocol}:///{r.lstrip("/")}' for r in fs.glob(path)]
643
+
644
+ def listdir(self, path: Union[str, os.PathLike[str]]) -> list[str]:
645
+ assert fsspec is not None, '`fsspec` is not installed.'
646
+ fs, path = fsspec.core.url_to_fs(path)
647
+ return [os.path.basename(f) for f in fs.ls(path, detail=False)]
648
+
649
+ def isdir(self, path: Union[str, os.PathLike[str]]) -> bool:
650
+ assert fsspec is not None, '`fsspec` is not installed.'
651
+ fs, path = fsspec.core.url_to_fs(path)
652
+ return fs.isdir(path)
653
+
654
+ def getctime(self, path: Union[str, os.PathLike[str]]) -> float:
655
+ assert fsspec is not None, '`fsspec` is not installed.'
656
+ fs, path_in_fs = fsspec.core.url_to_fs(path)
657
+ created = fs.created(path_in_fs)
658
+ if created is None:
659
+ raise OSError(f'c-time is not available for path {path!r}')
660
+ return created.timestamp()
661
+
662
+ def getmtime(self, path: Union[str, os.PathLike[str]]) -> float:
663
+ assert fsspec is not None, '`fsspec` is not installed.'
664
+ fs, path_in_fs = fsspec.core.url_to_fs(path)
665
+ modified = fs.modified(path_in_fs)
666
+ if modified is None:
667
+ raise OSError(f'm-time is not available for path {path!r}')
668
+ return modified.timestamp()
669
+
670
+ def mkdir(
671
+ self,
672
+ path: Union[str, os.PathLike[str]], mode: int = 0o777
673
+ ) -> None:
674
+ assert fsspec is not None, '`fsspec` is not installed.'
675
+ fs, path = fsspec.core.url_to_fs(path)
676
+ fs.mkdir(path)
677
+
678
+ def mkdirs(
679
+ self,
680
+ path: Union[str, os.PathLike[str]],
681
+ mode: int = 0o777,
682
+ exist_ok: bool = True,
683
+ ) -> None:
684
+ assert fsspec is not None, '`fsspec` is not installed.'
685
+ fs, path = fsspec.core.url_to_fs(path)
686
+ fs.makedirs(path, exist_ok=exist_ok)
687
+
688
+ def rm(self, path: Union[str, os.PathLike[str]]) -> None:
689
+ assert fsspec is not None, '`fsspec` is not installed.'
690
+ fs, path = fsspec.core.url_to_fs(path)
691
+ fs.rm(path)
692
+
693
+ def rename(
694
+ self,
695
+ oldpath: Union[str, os.PathLike[str]],
696
+ newpath: Union[str, os.PathLike[str]],
697
+ ) -> None:
698
+ assert fsspec is not None, '`fsspec` is not installed.'
699
+ fs, old_path = fsspec.core.url_to_fs(oldpath)
700
+ fs2, new_path = fsspec.core.url_to_fs(newpath)
701
+ if fs.__class__ != fs2.__class__:
702
+ raise ValueError(
703
+ f'Rename across different filesystems is not supported: '
704
+ f'{type(fs)} vs {type(fs2)}'
705
+ )
706
+ fs.rename(old_path, new_path)
707
+
708
+ def rmdir(self, path: Union[str, os.PathLike[str]]) -> None: # pytype: disable=signature-mismatch
709
+ assert fsspec is not None, '`fsspec` is not installed.'
710
+ fs, path = fsspec.core.url_to_fs(path)
711
+ fs.rmdir(path)
712
+
713
+ def rmdirs(self, path: Union[str, os.PathLike[str]]) -> None:
714
+ assert fsspec is not None, '`fsspec` is not installed.'
715
+ fs, path = fsspec.core.url_to_fs(path)
716
+ fs.rm(path, recursive=True)
717
+
718
+ def copy(
719
+ self,
720
+ oldpath: Union[str, os.PathLike[str]],
721
+ newpath: Union[str, os.PathLike[str]],
722
+ ) -> None:
723
+ assert fsspec is not None, '`fsspec` is not installed.'
724
+ fs1, old_path_in_fs = fsspec.core.url_to_fs(oldpath)
725
+ fs2, new_path_in_fs = fsspec.core.url_to_fs(newpath)
726
+
727
+ if fs2.isdir(new_path_in_fs):
728
+ new_path_in_fs = os.path.join(
729
+ new_path_in_fs, os.path.basename(old_path_in_fs)
730
+ )
731
+
732
+ if fs1.__class__ == fs2.__class__:
733
+ fs1.copy(old_path_in_fs, new_path_in_fs)
734
+ else:
735
+ with fs1.open(old_path_in_fs, 'rb') as f1:
736
+ with fs2.open(new_path_in_fs, 'wb') as f2:
737
+ while True:
738
+ # Reads the file in 16MB chunks.
739
+ chunk = f1.read(16 * 1024 * 1024)
740
+ if not chunk:
741
+ break
742
+ f2.write(chunk)
743
+
744
+
745
+ class _FsspecUriCatcher(FileSystem):
746
+ """File system to catch URI paths and redirect to FsspecFileSystem."""
747
+
748
+ # Catch all paths that contains '://' but not registered by
749
+ # available_protocols.
750
+ _URI_PATTERN = re.compile(r'^[a-zA-Z][a-zA-Z0-9+-.]*://.*')
751
+
752
+ def __init__(self):
753
+ super().__init__()
754
+ self._std_fs = StdFileSystem()
755
+ self._fsspec_fs = FsspecFileSystem()
756
+
757
+ def get_fs(self, path: Union[str, os.PathLike[str]]) -> FileSystem:
758
+ if self._URI_PATTERN.match(resolve_path(path)):
759
+ return self._fsspec_fs
760
+ return self._std_fs
761
+
762
+ def open(
763
+ self, path: Union[str, os.PathLike[str]], mode: str = 'r', **kwargs
764
+ ) -> File:
765
+ return self.get_fs(path).open(path, mode, **kwargs)
766
+
767
+ def chmod(self, path: Union[str, os.PathLike[str]], mode: int) -> None:
768
+ self.get_fs(path).chmod(path, mode)
769
+
770
+ def exists(self, path: Union[str, os.PathLike[str]]) -> bool:
771
+ return self.get_fs(path).exists(path)
772
+
773
+ def glob(self, pattern: Union[str, os.PathLike[str]]) -> list[str]:
774
+ return self.get_fs(pattern).glob(pattern)
775
+
776
+ def listdir(self, path: Union[str, os.PathLike[str]]) -> list[str]:
777
+ return self.get_fs(path).listdir(path)
778
+
779
+ def isdir(self, path: Union[str, os.PathLike[str]]) -> bool:
780
+ return self.get_fs(path).isdir(path)
781
+
782
+ def getctime(self, path: Union[str, os.PathLike[str]]) -> float:
783
+ return self.get_fs(path).getctime(path)
784
+
785
+ def getmtime(self, path: Union[str, os.PathLike[str]]) -> float:
786
+ return self.get_fs(path).getmtime(path)
787
+
788
+ def mkdir(
789
+ self,
790
+ path: Union[str, os.PathLike[str]],
791
+ mode: int = 0o777
792
+ ) -> None:
793
+ self.get_fs(path).mkdir(path, mode)
794
+
795
+ def mkdirs(
796
+ self,
797
+ path: Union[str, os.PathLike[str]],
798
+ mode: int = 0o777,
799
+ exist_ok: bool = True,
800
+ ) -> None:
801
+ self.get_fs(path).mkdirs(path, mode, exist_ok)
802
+
803
+ def rm(self, path: Union[str, os.PathLike[str]]) -> None:
804
+ self.get_fs(path).rm(path)
805
+
806
+ def rename(
807
+ self,
808
+ oldpath: Union[str, os.PathLike[str]],
809
+ newpath: Union[str, os.PathLike[str]],
810
+ ) -> None:
811
+ self.get_fs(oldpath).rename(oldpath, newpath)
812
+
813
+ def rmdir(self, path: Union[str, os.PathLike[str]]) -> None: # pytype: disable=signature-mismatch
814
+ self.get_fs(path).rmdir(path)
815
+
816
+ def rmdirs(self, path: Union[str, os.PathLike[str]]) -> None:
817
+ self.get_fs(path).rmdirs(path)
818
+
819
+ def copy(
820
+ self,
821
+ oldpath: Union[str, os.PathLike[str]],
822
+ newpath: Union[str, os.PathLike[str]],
823
+ ) -> None:
824
+ self.get_fs(oldpath).copy(oldpath, newpath)
825
+
826
+
827
+ if fsspec is not None:
828
+ fsspec_fs = FsspecFileSystem()
829
+ for p in fsspec.available_protocols():
830
+ add_file_system(p + '://', fsspec_fs)
831
+ add_file_system('', _FsspecUriCatcher())
832
+
833
+
415
834
  #
416
835
  # APIs for file IO.
417
836
  #
@@ -458,16 +877,37 @@ def writefile(
458
877
  chmod(path, perms)
459
878
 
460
879
 
880
+ def rename(
881
+ oldpath: Union[str, os.PathLike[str]],
882
+ newpath: Union[str, os.PathLike[str]],
883
+ ) -> None:
884
+ """Renames a file."""
885
+ _fs.get(oldpath).rename(oldpath, newpath)
886
+
887
+
461
888
  def rm(path: Union[str, os.PathLike[str]]) -> None:
462
889
  """Removes a file."""
463
890
  _fs.get(path).rm(path)
464
891
 
465
892
 
893
+ def copy(
894
+ oldpath: Union[str, os.PathLike[str]],
895
+ newpath: Union[str, os.PathLike[str]],
896
+ ) -> None:
897
+ """Copies a file."""
898
+ _fs.get(oldpath).copy(oldpath, newpath)
899
+
900
+
466
901
  def path_exists(path: Union[str, os.PathLike[str]]) -> bool:
467
902
  """Returns True if path exists."""
468
903
  return _fs.get(path).exists(path)
469
904
 
470
905
 
906
+ def glob(pattern: Union[str, os.PathLike[str]]) -> list[str]:
907
+ """Lists all files or sub-directories."""
908
+ return _fs.get(pattern).glob(pattern)
909
+
910
+
471
911
  def listdir(
472
912
  path: Union[str, os.PathLike[str]], fullpath: bool = False
473
913
  ) -> list[str]: # pylint: disable=redefined-builtin
@@ -483,6 +923,16 @@ def isdir(path: Union[str, os.PathLike[str]]) -> bool:
483
923
  return _fs.get(path).isdir(path)
484
924
 
485
925
 
926
+ def getctime(path: Union[str, os.PathLike[str]]) -> float:
927
+ """Returns the creation time of a file."""
928
+ return _fs.get(path).getctime(path)
929
+
930
+
931
+ def getmtime(path: Union[str, os.PathLike[str]]) -> float:
932
+ """Returns the last modification time of a file."""
933
+ return _fs.get(path).getmtime(path)
934
+
935
+
486
936
  def mkdir(path: Union[str, os.PathLike[str]], mode: int = 0o777) -> None:
487
937
  """Makes a directory."""
488
938
  _fs.get(path).mkdir(path, mode=mode)