pyglove 0.5.0.dev202511240812__py3-none-any.whl → 0.5.0.dev202512080812__py3-none-any.whl

pyglove/core/io/file_system.py

@@ -14,11 +14,13 @@
 """Pluggable file system."""
 
 import abc
+import datetime
 import fnmatch
 import glob as std_glob
 import io
 import os
 import re
+import shutil
 from typing import Any, Literal, Optional, Union
 
 
@@ -99,6 +101,14 @@ class FileSystem(metaclass=abc.ABCMeta):
   def isdir(self, path: Union[str, os.PathLike[str]]) -> bool:
     """Returns True if a path is a directory."""
 
+  @abc.abstractmethod
+  def getctime(self, path: Union[str, os.PathLike[str]]) -> float:
+    """Returns the creation time of a file."""
+
+  @abc.abstractmethod
+  def getmtime(self, path: Union[str, os.PathLike[str]]) -> float:
+    """Returns the last modification time of a file."""
+
   @abc.abstractmethod
   def mkdir(
       self, path: Union[str, os.PathLike[str]], mode: int = 0o777
@@ -134,6 +144,14 @@ class FileSystem(metaclass=abc.ABCMeta):
   def rmdirs(self, path: Union[str, os.PathLike[str]]) -> None:
     """Removes a directory chain based on a path."""
 
+  @abc.abstractmethod
+  def copy(
+      self,
+      oldpath: Union[str, os.PathLike[str]],
+      newpath: Union[str, os.PathLike[str]],
+  ) -> None:
+    """Copies a file to a new path."""
+
 
 def resolve_path(path: Union[str, os.PathLike[str]]) -> str:
   if isinstance(path, str):
@@ -202,6 +220,12 @@ class StdFileSystem(FileSystem):
   def isdir(self, path: Union[str, os.PathLike[str]]) -> bool:
     return os.path.isdir(path)
 
+  def getctime(self, path: Union[str, os.PathLike[str]]) -> float:
+    return os.path.getctime(path)
+
+  def getmtime(self, path: Union[str, os.PathLike[str]]) -> float:
+    return os.path.getmtime(path)
+
   def mkdir(
       self, path: Union[str, os.PathLike[str]], mode: int = 0o777
   ) -> None:
@@ -231,6 +255,13 @@ class StdFileSystem(FileSystem):
   def rmdirs(self, path: Union[str, os.PathLike[str]]) -> None:
     os.removedirs(path)
 
+  def copy(
+      self,
+      oldpath: Union[str, os.PathLike[str]],
+      newpath: Union[str, os.PathLike[str]],
+  ) -> None:
+    shutil.copy(oldpath, newpath)
+
 
 #
 # Memory file system.
@@ -244,6 +275,9 @@ class MemoryFile(File):
     super().__init__()
     self._buffer = buffer
     self._pos = 0
+    now = datetime.datetime.now().timestamp()
+    self.ctime = now
+    self.mtime = now
 
   def read(self, size: Optional[int] = None) -> Union[str, bytes]:
     return self._buffer.read(size)
@@ -253,6 +287,7 @@ class MemoryFile(File):
 
   def write(self, content: Union[str, bytes]) -> None:
     self._buffer.write(content)
+    self.mtime = datetime.datetime.now().timestamp()
 
   def seek(self, offset: int, whence: Literal[0, 1, 2] = 0) -> int:
     return self._buffer.seek(offset, whence)
@@ -336,6 +371,24 @@ class MemoryFileSystem(FileSystem):
   def isdir(self, path: Union[str, os.PathLike[str]]) -> bool:
     return isinstance(self._locate(path), dict)
 
+  def getctime(self, path: Union[str, os.PathLike[str]]) -> float:
+    """Returns the creation time of a file."""
+    f = self._locate(path)
+    if isinstance(f, dict):
+      raise IsADirectoryError(path)
+    if f is None:
+      raise FileNotFoundError(path)
+    return f.ctime
+
+  def getmtime(self, path: Union[str, os.PathLike[str]]) -> float:
+    """Returns the last modification time of a file."""
+    f = self._locate(path)
+    if isinstance(f, dict):
+      raise IsADirectoryError(path)
+    if f is None:
+      raise FileNotFoundError(path)
+    return f.mtime
+
   def _parent_and_name(
       self, path: Union[str, os.PathLike[str]]
   ) -> tuple[dict[str, Any], str]:
@@ -460,6 +513,37 @@ class MemoryFileSystem(FileSystem):
       return not dir_dict
     _rmdir(self._root, self._internal_path(path))
 
+  def copy(
+      self,
+      oldpath: Union[str, os.PathLike[str]],
+      newpath: Union[str, os.PathLike[str]],
+  ) -> None:
+    old_file = self._locate(oldpath)
+    if old_file is None:
+      raise FileNotFoundError(oldpath)
+    if not isinstance(old_file, MemoryFile):
+      raise IsADirectoryError(oldpath)
+
+    if self.isdir(newpath):
+      _, old_file_name = self._parent_and_name(oldpath)
+      newpath = resolve_path(newpath)
+      newpath = newpath.rstrip('/') + '/' + old_file_name
+
+    # If newpath exists as file, remove it for overwrite.
+    if self.exists(newpath) and not self.isdir(newpath):
+      self.rm(newpath)
+    elif self.isdir(newpath):
+      raise IsADirectoryError(newpath)
+
+    old_pos = old_file.tell()
+    old_file.seek(0)
+    content = old_file.read()
+    old_file.seek(old_pos)
+
+    is_binary = isinstance(old_file._buffer, io.BytesIO)  # pylint: disable=protected-access
+    with self.open(newpath, 'wb' if is_binary else 'w') as f_new:
+      f_new.write(content)
+
 
 class _FileSystemRegistry:
   """File system registry."""
@@ -567,6 +651,22 @@ class FsspecFileSystem(FileSystem):
     fs, path = fsspec.core.url_to_fs(path)
     return fs.isdir(path)
 
+  def getctime(self, path: Union[str, os.PathLike[str]]) -> float:
+    assert fsspec is not None, '`fsspec` is not installed.'
+    fs, path_in_fs = fsspec.core.url_to_fs(path)
+    created = fs.created(path_in_fs)
+    if created is None:
+      raise OSError(f'c-time is not available for path {path!r}')
+    return created.timestamp()
+
+  def getmtime(self, path: Union[str, os.PathLike[str]]) -> float:
+    assert fsspec is not None, '`fsspec` is not installed.'
+    fs, path_in_fs = fsspec.core.url_to_fs(path)
+    modified = fs.modified(path_in_fs)
+    if modified is None:
+      raise OSError(f'm-time is not available for path {path!r}')
+    return modified.timestamp()
+
   def mkdir(
       self,
       path: Union[str, os.PathLike[str]], mode: int = 0o777
@@ -615,6 +715,32 @@ class FsspecFileSystem(FileSystem):
     fs, path = fsspec.core.url_to_fs(path)
     fs.rm(path, recursive=True)
 
+  def copy(
+      self,
+      oldpath: Union[str, os.PathLike[str]],
+      newpath: Union[str, os.PathLike[str]],
+  ) -> None:
+    assert fsspec is not None, '`fsspec` is not installed.'
+    fs1, old_path_in_fs = fsspec.core.url_to_fs(oldpath)
+    fs2, new_path_in_fs = fsspec.core.url_to_fs(newpath)
+
+    if fs2.isdir(new_path_in_fs):
+      new_path_in_fs = os.path.join(
+          new_path_in_fs, os.path.basename(old_path_in_fs)
+      )
+
+    if fs1.__class__ == fs2.__class__:
+      fs1.copy(old_path_in_fs, new_path_in_fs)
+    else:
+      with fs1.open(old_path_in_fs, 'rb') as f1:
+        with fs2.open(new_path_in_fs, 'wb') as f2:
+          while True:
+            # Reads the file in 16MB chunks.
+            chunk = f1.read(16 * 1024 * 1024)
+            if not chunk:
+              break
+            f2.write(chunk)
+
 
 class _FsspecUriCatcher(FileSystem):
   """File system to catch URI paths and redirect to FsspecFileSystem."""
@@ -653,6 +779,12 @@ class _FsspecUriCatcher(FileSystem):
   def isdir(self, path: Union[str, os.PathLike[str]]) -> bool:
     return self.get_fs(path).isdir(path)
 
+  def getctime(self, path: Union[str, os.PathLike[str]]) -> float:
+    return self.get_fs(path).getctime(path)
+
+  def getmtime(self, path: Union[str, os.PathLike[str]]) -> float:
+    return self.get_fs(path).getmtime(path)
+
   def mkdir(
       self,
       path: Union[str, os.PathLike[str]],
@@ -684,6 +816,13 @@ class _FsspecUriCatcher(FileSystem):
   def rmdirs(self, path: Union[str, os.PathLike[str]]) -> None:
     self.get_fs(path).rmdirs(path)
 
+  def copy(
+      self,
+      oldpath: Union[str, os.PathLike[str]],
+      newpath: Union[str, os.PathLike[str]],
+  ) -> None:
+    self.get_fs(oldpath).copy(oldpath, newpath)
+
 
 if fsspec is not None:
   fsspec_fs = FsspecFileSystem()
@@ -751,6 +890,14 @@ def rm(path: Union[str, os.PathLike[str]]) -> None:
   _fs.get(path).rm(path)
 
 
+def copy(
+    oldpath: Union[str, os.PathLike[str]],
+    newpath: Union[str, os.PathLike[str]],
+) -> None:
+  """Copies a file."""
+  _fs.get(oldpath).copy(oldpath, newpath)
+
+
 def path_exists(path: Union[str, os.PathLike[str]]) -> bool:
   """Returns True if path exists."""
   return _fs.get(path).exists(path)
@@ -776,6 +923,16 @@ def isdir(path: Union[str, os.PathLike[str]]) -> bool:
   return _fs.get(path).isdir(path)
 
 
+def getctime(path: Union[str, os.PathLike[str]]) -> float:
+  """Returns the creation time of a file."""
+  return _fs.get(path).getctime(path)
+
+
+def getmtime(path: Union[str, os.PathLike[str]]) -> float:
+  """Returns the last modification time of a file."""
+  return _fs.get(path).getmtime(path)
+
+
 def mkdir(path: Union[str, os.PathLike[str]], mode: int = 0o777) -> None:
   """Makes a directory."""
   _fs.get(path).mkdir(path, mode=mode)

pyglove/core/io/file_system_test.py

@@ -12,9 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import datetime
 import os
 import pathlib
+import shutil
 import tempfile
+import time
 import unittest
 from unittest import mock
 import fsspec
@@ -153,6 +156,42 @@ class StdFileSystemTest(unittest.TestCase):
           os.path.join(tmp_dir, 'y')
       )
 
+  def test_copy(self):
+    tmp_dir = tempfile.mkdtemp()
+    fs = file_system.StdFileSystem()
+    foo_txt = os.path.join(tmp_dir, 'foo.txt')
+    bar_txt = os.path.join(tmp_dir, 'bar.txt')
+    sub_dir = os.path.join(tmp_dir, 'sub')
+    sub_foo_txt = os.path.join(sub_dir, 'foo.txt')
+
+    with fs.open(foo_txt, 'w') as f:
+      f.write('hello')
+    fs.copy(foo_txt, bar_txt)
+    with fs.open(bar_txt) as f:
+      self.assertEqual(f.read(), 'hello')
+    fs.mkdir(sub_dir)
+    fs.copy(foo_txt, sub_dir)
+    with fs.open(sub_foo_txt) as f:
+      self.assertEqual(f.read(), 'hello')
+    with fs.open(foo_txt, 'w') as f:
+      f.write('overwrite')
+    fs.copy(foo_txt, bar_txt)
+    with fs.open(bar_txt) as f:
+      self.assertEqual(f.read(), 'overwrite')
+    shutil.rmtree(tmp_dir)
+
+  def test_getctime_getmtime(self):
+    tmp_dir = tempfile.mkdtemp()
+    fs = file_system.StdFileSystem()
+    file1 = os.path.join(tmp_dir, 'file1')
+    with fs.open(file1, 'w') as f:
+      f.write('hello')
+    ctime = fs.getctime(file1)
+    mtime = fs.getmtime(file1)
+    self.assertLess(0, ctime)
+    self.assertLessEqual(ctime, mtime)
+    shutil.rmtree(tmp_dir)
+
 
 class MemoryFileSystemTest(unittest.TestCase):
 
@@ -339,6 +378,57 @@ class MemoryFileSystemTest(unittest.TestCase):
     with self.assertRaises(FileNotFoundError):
       fs.rename('/mem/non-existent', '/mem/y')
 
+  def test_copy(self):
+    fs = file_system.MemoryFileSystem()
+    fs.mkdirs('/mem/a')
+    with fs.open('/mem/a/foo.txt', 'w') as f:
+      f.write('hello')
+    fs.copy('/mem/a/foo.txt', '/mem/a/bar.txt')
+    self.assertEqual(fs.open('/mem/a/bar.txt').read(), 'hello')
+    fs.mkdir('/mem/b')
+    fs.copy('/mem/a/foo.txt', '/mem/b')
+    self.assertEqual(fs.open('/mem/b/foo.txt').read(), 'hello')
+    with fs.open('/mem/a/foo.txt', 'w') as f:
+      f.write('overwrite')
+    fs.copy('/mem/a/foo.txt', '/mem/a/bar.txt')
+    self.assertEqual(fs.open('/mem/a/bar.txt').read(), 'overwrite')
+
+    # Test exceptions
+    with self.assertRaises(FileNotFoundError):
+      fs.copy('/mem/non-existent', '/mem/y')
+    with self.assertRaisesRegex(IsADirectoryError, '/mem/a'):
+      fs.copy('/mem/a', '/mem/y')
+
+    fs.mkdirs('/mem/c/foo.txt')
+    with self.assertRaisesRegex(IsADirectoryError, '/mem/c/foo.txt'):
+      fs.copy('/mem/a/foo.txt', '/mem/c')
+
+  def test_getctime_getmtime(self):
+    fs = file_system.MemoryFileSystem()
+    fs.mkdirs('/mem/a')
+    file1 = '/mem/file1_times'
+    with fs.open(file1, 'w') as f:
+      f.write('hello')
+    ctime = fs.getctime(file1)
+    mtime = fs.getmtime(file1)
+    self.assertLess(0, ctime)
+    self.assertLess(ctime, mtime)
+    time.sleep(0.01)
+    with fs.open(file1, 'w') as f:
+      f.write('world')
+    self.assertEqual(fs.getctime(file1), ctime)
+    self.assertLess(mtime, fs.getmtime(file1))
+
+    with self.assertRaises(IsADirectoryError):
+      fs.getctime('/mem/a')
+    with self.assertRaises(FileNotFoundError):
+      fs.getctime('/mem/non_existent')
+
+    with self.assertRaises(IsADirectoryError):
+      fs.getmtime('/mem/a')
+    with self.assertRaises(FileNotFoundError):
+      fs.getmtime('/mem/non_existent')
+
 
 class FileIoApiTest(unittest.TestCase):
 
@@ -424,6 +514,19 @@ class FileIoApiTest(unittest.TestCase):
         sorted(file_system.glob('/mem/g/a/*')),
         ['/mem/g/a/b', '/mem/g/a/b/bar.txt', '/mem/g/a/foo2.txt'])
 
+  def test_getctime_getmtime(self):
+    # Test with standard file system.
+    std_file = os.path.join(tempfile.mkdtemp(), 'file_ctime_mtime')
+    file_system.writefile(std_file, 'foo')
+    self.assertLess(0, file_system.getctime(std_file))
+    self.assertLess(0, file_system.getmtime(std_file))
+
+    # Test with memory file system.
+    mem_file = '/mem/file_ctime_mtime'
+    file_system.writefile(mem_file, 'foo')
+    self.assertLess(0, file_system.getctime(mem_file))
+    self.assertLess(0, file_system.getmtime(mem_file))
+
 
 class FsspecFileSystemTest(unittest.TestCase):
 
@@ -433,10 +536,12 @@ class FsspecFileSystemTest(unittest.TestCase):
     self.fs.pipe('memory:///a/b/c', b'abc')
     self.fs.pipe('memory:///a/b/d', b'abd')
     self.fs.mkdir('memory:///a/e')
+    self.tmp_dir = tempfile.mkdtemp()
 
   def tearDown(self):
     super().tearDown()
     fsspec.filesystem('memory').rm('/', recursive=True)
+    shutil.rmtree(self.tmp_dir)
 
   def test_read_file(self):
     self.assertEqual(file_system.readfile('memory:///a/b/c', mode='rb'), b'abc')
@@ -513,6 +618,44 @@ class FsspecFileSystemTest(unittest.TestCase):
     file_system.rmdirs('memory:///x')
     self.assertFalse(file_system.path_exists('memory:///x'))
 
+  def test_copy(self):
+    # same FS copy
+    file_system.copy('memory:///a/b/c', 'memory:///a/f')
+    self.assertEqual(file_system.readfile('memory:///a/f', mode='rb'), b'abc')
+
+    # same FS copy to dir
+    file_system.copy('memory:///a/b/d', 'memory:///a/e')
+    self.assertEqual(file_system.readfile('memory:///a/e/d', mode='rb'), b'abd')
+
+    # cross FS copy: memory to local
+    local_path = os.path.join(self.tmp_dir, 'test.txt')
+    file_system.copy('memory:///a/b/c', f'file://{local_path}')
+    self.assertEqual(file_system.readfile(local_path, mode='rb'), b'abc')
+
+    # cross FS copy: local to memory
+    file_system.copy(f'file://{local_path}', 'memory:///a/g')
+    self.assertEqual(file_system.readfile('memory:///a/g', mode='rb'), b'abc')
+
+  def test_getctime_getmtime(self):
+    self.fs.touch('memory:///a/b/c_time')
+    now = datetime.datetime.now()
+    with mock.patch.object(self.fs, 'created', return_value=now):
+      self.assertEqual(
+          file_system.getctime('memory:///a/b/c_time'), now.timestamp()
+      )
+    with mock.patch.object(self.fs, 'modified', return_value=now):
+      self.assertEqual(
+          file_system.getmtime('memory:///a/b/c_time'), now.timestamp()
+      )
+
+    with mock.patch.object(self.fs, 'created', return_value=None):
+      with self.assertRaisesRegex(OSError, 'c-time is not available'):
+        file_system.getctime('memory:///a/b/c_time')
+
+    with mock.patch.object(self.fs, 'modified', return_value=None):
+      with self.assertRaisesRegex(OSError, 'm-time is not available'):
+        file_system.getmtime('memory:///a/b/c_time')
+
   def test_fsspec_uri_catcher(self):
     with mock.patch.object(
         file_system.FsspecFileSystem, 'exists', return_value=True
@@ -539,6 +682,14 @@ class FsspecFileSystemTest(unittest.TestCase):
           'some-proto://foo', 'some-proto://bar'
       )
 
+    with mock.patch.object(
+        file_system.FsspecFileSystem, 'copy', return_value=None
+    ) as mock_fsspec_copy:
+      file_system.copy('some-proto://foo', 'some-proto://bar')
+      mock_fsspec_copy.assert_called_once_with(
+          'some-proto://foo', 'some-proto://bar'
+      )
+
 
 if __name__ == '__main__':
   unittest.main()

{pyglove-0.5.0.dev202511240812.dist-info → pyglove-0.5.0.dev202512080812.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyglove
-Version: 0.5.0.dev202511240812
+Version: 0.5.0.dev202512080812
 Summary: PyGlove: A library for manipulating Python objects.
 Home-page: https://github.com/google/pyglove
 Author: PyGlove Authors

{pyglove-0.5.0.dev202511240812.dist-info → pyglove-0.5.0.dev202512080812.dist-info}/RECORD

@@ -56,8 +56,8 @@ pyglove/core/hyper/numerical_test.py,sha256=UWdH55Bok7bghYDLJOGsgOwV_2LNkhj1AmFw
 pyglove/core/hyper/object_template.py,sha256=YPALTV0mMULa7iuqnryTpA2wMsdyFZ_6g-R525asAr8,22222
 pyglove/core/hyper/object_template_test.py,sha256=TEFX7LIqUvdCdJILnK_gP5xIgNJKzRnioUF0CGVBzcY,9105
 pyglove/core/io/__init__.py,sha256=4ZT1a595DqQuLTNYc2JP_eCp_KesXvHmKRkr777bzpg,785
-pyglove/core/io/file_system.py,sha256=r7x_kDWDynKbAcO8tEkkRHShOPiYXyB9wmCXv2c-Oes,23149
-pyglove/core/io/file_system_test.py,sha256=QvcPEQj3hAk7xc8VkFda1ZzgsV0etBimc1V9CJXr_Rk,19145
+pyglove/core/io/file_system.py,sha256=cggBMIzQxQpTjPsYQPQj8IsO-UpunsEGxaclYQcUHjw,28093
+pyglove/core/io/file_system_test.py,sha256=2c2l3HCNS8Q4mv8k1dqopnGpeVLiWSzmc8eDIx4b5oo,24561
 pyglove/core/io/sequence.py,sha256=XdVpPBuvhUnrTIWMUrak_qdcNdUJBpjgH5c1b5I3E2A,8873
 pyglove/core/io/sequence_test.py,sha256=mnONyNG1M1sCae2tyI-tF8qns96htfZPKycthETPthU,4062
 pyglove/core/patching/__init__.py,sha256=C1Q1cWPV74YL3eXbzGvc-8aPw1DR8EK6lRhQYDCwHek,2059
@@ -220,8 +220,8 @@ pyglove/ext/scalars/randoms.py,sha256=LkMIIx7lOq_lvJvVS3BrgWGuWl7Pi91-lA-O8x_gZs
 pyglove/ext/scalars/randoms_test.py,sha256=nEhiqarg8l_5EOucp59CYrpO2uKxS1pe0hmBdZUzRNM,2000
 pyglove/ext/scalars/step_wise.py,sha256=IDw3tuTpv0KVh7AN44W43zqm1-E0HWPUlytWOQC9w3Y,3789
 pyglove/ext/scalars/step_wise_test.py,sha256=TL1vJ19xVx2t5HKuyIzGoogF7N3Rm8YhLE6JF7i0iy8,2540
-pyglove-0.5.0.dev202511240812.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-pyglove-0.5.0.dev202511240812.dist-info/METADATA,sha256=vaiIKjLvPfyE_gV31sliLGA2SsRx3kLNCjB5cJQTySo,7349
-pyglove-0.5.0.dev202511240812.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-pyglove-0.5.0.dev202511240812.dist-info/top_level.txt,sha256=wITzJSKcj8GZUkbq-MvUQnFadkiuAv_qv5qQMw0fIow,8
-pyglove-0.5.0.dev202511240812.dist-info/RECORD,,
+pyglove-0.5.0.dev202512080812.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+pyglove-0.5.0.dev202512080812.dist-info/METADATA,sha256=E7UuKxo0wIhPc7Z3L0cRpVKXxj9eY_qfcbr0fjPfym8,7349
+pyglove-0.5.0.dev202512080812.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+pyglove-0.5.0.dev202512080812.dist-info/top_level.txt,sha256=wITzJSKcj8GZUkbq-MvUQnFadkiuAv_qv5qQMw0fIow,8
+pyglove-0.5.0.dev202512080812.dist-info/RECORD,,