pyglove 0.5.0.dev202511040811__tar.gz → 0.5.0.dev202511161718__tar.gz

{pyglove-0.5.0.dev202511040811 → pyglove-0.5.0.dev202511161718}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyglove
-Version: 0.5.0.dev202511040811
+Version: 0.5.0.dev202511161718
 Summary: PyGlove: A library for manipulating Python objects.
 Home-page: https://github.com/google/pyglove
 Author: PyGlove Authors

{pyglove-0.5.0.dev202511040811 → pyglove-0.5.0.dev202511161718}/pyglove/core/io/file_system.py

@@ -14,8 +14,12 @@
 """Pluggable file system."""
 
 import abc
+import fnmatch
+import glob as std_glob
 import io
 import os
+import re
+import time
 from typing import Any, Literal, Optional, Union
 
 
@@ -84,6 +88,10 @@ class FileSystem(metaclass=abc.ABCMeta):
   def exists(self, path: Union[str, os.PathLike[str]]) -> bool:
     """Returns True if a path exists."""
 
+  @abc.abstractmethod
+  def glob(self, pattern: Union[str, os.PathLike[str]]) -> list[str]:
+    """Lists all files or sub-directories."""
+
   @abc.abstractmethod
   def listdir(self, path: Union[str, os.PathLike[str]]) -> list[str]:
     """Lists all files or sub-directories."""
@@ -111,6 +119,14 @@ class FileSystem(metaclass=abc.ABCMeta):
   def rm(self, path: Union[str, os.PathLike[str]]) -> None:
     """Removes a file based on a path."""
 
+  @abc.abstractmethod
+  def rename(
+      self,
+      oldpath: Union[str, os.PathLike[str]],
+      newpath: Union[str, os.PathLike[str]],
+  ) -> None:
+    """Renames a file based on a path."""
+
   @abc.abstractmethod
   def rmdir(self, path: Union[str, os.PathLike[str]]) -> bool:
     """Removes a directory based on a path."""
@@ -137,9 +153,10 @@ def resolve_path(path: Union[str, os.PathLike[str]]) -> str:
 class StdFile(File):
   """The standard file."""
 
-  def __init__(self, file_object) -> None:
+  def __init__(self, file_object, path: Union[str, os.PathLike[str]]) -> None:
     super().__init__()
     self._file_object = file_object
+    self._path = path
 
   def read(self, size: Optional[int] = None) -> Union[str, bytes]:
     return self._file_object.read(size)
@@ -162,6 +179,14 @@ class StdFile(File):
   def close(self) -> None:
     self._file_object.close()
 
+    # For some file systems, the file might not be immediately available
+    # after writing. We retry for a few times to ensure the file is
+    # world-readable.
+    while True:
+      if os.path.exists(self._path):
+        break
+      time.sleep(0.1)
+
 
 class StdFileSystem(FileSystem):
   """The standard file system."""
@@ -169,7 +194,7 @@ class StdFileSystem(FileSystem):
   def open(
       self, path: Union[str, os.PathLike[str]], mode: str = 'r', **kwargs
   ) -> File:
-    return StdFile(io.open(path, mode, **kwargs))
+    return StdFile(io.open(path, mode, **kwargs), path)
 
   def chmod(self, path: Union[str, os.PathLike[str]], mode: int) -> None:
     os.chmod(path, mode)
@@ -177,6 +202,9 @@ class StdFileSystem(FileSystem):
   def exists(self, path: Union[str, os.PathLike[str]]) -> bool:
     return os.path.exists(path)
 
+  def glob(self, pattern: Union[str, os.PathLike[str]]) -> list[str]:
+    return std_glob.glob(pattern)
+
   def listdir(self, path: Union[str, os.PathLike[str]]) -> list[str]:
     return os.listdir(path)
 
@@ -196,6 +224,13 @@ class StdFileSystem(FileSystem):
   ) -> None:
     os.makedirs(path, mode, exist_ok)
 
+  def rename(
+      self,
+      oldpath: Union[str, os.PathLike[str]],
+      newpath: Union[str, os.PathLike[str]],
+  ) -> None:
+    os.rename(oldpath, newpath)
+
   def rm(self, path: Union[str, os.PathLike[str]]) -> None:
     os.remove(path)
 
@@ -286,6 +321,21 @@ class MemoryFileSystem(FileSystem):
   def exists(self, path: Union[str, os.PathLike[str]]) -> bool:
     return self._locate(path) is not None
 
+  def glob(self, pattern: Union[str, os.PathLike[str]]) -> list[str]:
+    pattern = resolve_path(pattern)
+    regex = re.compile(fnmatch.translate(pattern))
+
+    results = []
+    def _traverse(node, prefix_parts):
+      for k, v in node.items():
+        path = self._prefix + '/'.join(prefix_parts + [k])
+        if regex.match(path):
+          results.append(path)
+        if isinstance(v, dict):
+          _traverse(v, prefix_parts + [k])
+    _traverse(self._root, [])
+    return results
+
   def listdir(self, path: Union[str, os.PathLike[str]]) -> list[str]:
     d = self._locate(path)
     if not isinstance(d, dict):
@@ -338,6 +388,46 @@ class MemoryFileSystem(FileSystem):
         raise NotADirectoryError(path)
       current = entry
 
+  def rename(
+      self,
+      oldpath: Union[str, os.PathLike[str]],
+      newpath: Union[str, os.PathLike[str]],
+  ) -> None:
+    oldpath_str = resolve_path(oldpath)
+    newpath_str = resolve_path(newpath)
+
+    old_parent_dir, old_name = self._parent_and_name(oldpath_str)
+    entry = old_parent_dir.get(old_name)
+
+    if entry is None:
+      raise FileNotFoundError(oldpath_str)
+
+    new_entry = self._locate(newpath_str)
+    if new_entry is not None:
+      if isinstance(entry, dict):  # oldpath is dir
+        if not isinstance(new_entry, dict):
+          raise NotADirectoryError(
+              f'Cannot rename directory {oldpath_str!r} '
+              f'to non-directory {newpath_str!r}')
+        elif new_entry:
+          raise OSError(f'Directory not empty: {newpath_str!r}')
+        else:
+          self.rmdir(newpath_str)
+      else:  # oldpath is file
+        if isinstance(new_entry, dict):
+          raise IsADirectoryError(
+              f'Cannot rename non-directory {oldpath_str!r} '
+              f'to directory {newpath_str!r}')
+        else:
+          self.rm(newpath_str)
+    elif isinstance(entry, dict) and newpath_str.startswith(oldpath_str + '/'):
+      raise OSError(f'Cannot move directory {oldpath_str!r} '
+                    f'to a subdirectory of itself {newpath_str!r}')
+
+    new_parent_dir, new_name = self._parent_and_name(newpath_str)
+    new_parent_dir[new_name] = entry
+    del old_parent_dir[old_name]
+
   def rm(self, path: Union[str, os.PathLike[str]]) -> None:
     parent_dir, name = self._parent_and_name(path)
     entry = parent_dir.get(name)
@@ -458,6 +548,14 @@ def writefile(
     chmod(path, perms)
 
 
+def rename(
+    oldpath: Union[str, os.PathLike[str]],
+    newpath: Union[str, os.PathLike[str]],
+) -> None:
+  """Renames a file."""
+  _fs.get(oldpath).rename(oldpath, newpath)
+
+
 def rm(path: Union[str, os.PathLike[str]]) -> None:
   """Removes a file."""
   _fs.get(path).rm(path)
@@ -468,6 +566,11 @@ def path_exists(path: Union[str, os.PathLike[str]]) -> bool:
   return _fs.get(path).exists(path)
 
 
+def glob(pattern: Union[str, os.PathLike[str]]) -> list[str]:
+  """Lists all files or sub-directories."""
+  return _fs.get(pattern).glob(pattern)
+
+
 def listdir(
     path: Union[str, os.PathLike[str]], fullpath: bool = False
 ) -> list[str]:  # pylint: disable=redefined-builtin

{pyglove-0.5.0.dev202511040811 → pyglove-0.5.0.dev202511161718}/pyglove/core/io/file_system_test.py

@@ -82,6 +82,75 @@ class StdFileSystemTest(unittest.TestCase):
     fs.rmdirs(os.path.join(dir_a, 'b/c'))
     self.assertEqual(sorted(fs.listdir(dir_a)), ['file1'])   # pylint: disable=g-generic-assert
 
+  def test_rename(self):
+    tmp_dir = tempfile.mkdtemp()
+    fs = file_system.StdFileSystem()
+
+    _ = fs.mkdirs(os.path.join(tmp_dir, 'a/b'))
+    file_foo = os.path.join(tmp_dir, 'a/foo.txt')
+    file_bar = os.path.join(tmp_dir, 'a/bar.txt')
+
+    with fs.open(file_foo, 'w') as f:
+      f.write('foo')
+    with fs.open(file_bar, 'w') as f:
+      f.write('bar')
+
+    # Rename file to a new name.
+    file_foo_new = os.path.join(tmp_dir, 'a/foo-new.txt')
+    fs.rename(file_foo, file_foo_new)
+    self.assertFalse(fs.exists(file_foo))
+    self.assertTrue(fs.exists(file_foo_new))
+
+    # Rename file to an existing file name.
+    fs.rename(file_foo_new, file_bar)
+    self.assertFalse(fs.exists(file_foo_new))
+    with fs.open(file_bar, 'r') as f:
+      self.assertEqual(f.read(), 'foo')
+
+    # Rename directory to a new name.
+    dir_b = os.path.join(tmp_dir, 'a/b')
+    dir_c = os.path.join(tmp_dir, 'a/c')
+    fs.rename(dir_b, dir_c)
+    self.assertFalse(fs.exists(dir_b))
+    self.assertTrue(fs.exists(dir_c))
+    self.assertTrue(fs.isdir(dir_c))
+
+    # Rename directory to an existing empty directory.
+    dir_d = os.path.join(tmp_dir, 'a/d')
+    fs.mkdirs(dir_d)
+    fs.rename(dir_c, dir_d)
+    self.assertFalse(fs.exists(dir_c))
+    self.assertTrue(fs.exists(dir_d))
+
+    # Rename directory to a non-empty directory.
+    dir_x = os.path.join(tmp_dir, 'x')
+    dir_a = os.path.join(tmp_dir, 'a')
+    fs.mkdirs(os.path.join(dir_x, 'y'))
+    with self.assertRaises(OSError):
+      fs.rename(dir_a, dir_x)
+    self.assertTrue(fs.exists(dir_a))
+    self.assertTrue(fs.exists(os.path.join(dir_x, 'y')))
+
+    # Errors
+    dir_u = os.path.join(tmp_dir, 'u')
+    dir_u_v = os.path.join(dir_u, 'v')
+    file_u_a = os.path.join(dir_u, 'a.txt')
+    fs.mkdirs(dir_u_v)
+    with fs.open(file_u_a, 'w') as f:
+      f.write('a')
+
+    with self.assertRaises((OSError, NotADirectoryError)):
+      fs.rename(dir_u, file_u_a)
+
+    with self.assertRaises(IsADirectoryError):
+      fs.rename(file_u_a, dir_u_v)
+
+    with self.assertRaises(FileNotFoundError):
+      fs.rename(
+          os.path.join(tmp_dir, 'non-existent'),
+          os.path.join(tmp_dir, 'y')
+      )
+
 
 class MemoryFileSystemTest(unittest.TestCase):
 
@@ -180,6 +249,94 @@ class MemoryFileSystemTest(unittest.TestCase):
     fs.rmdirs(os.path.join(dir_a, 'b/c'))
     self.assertEqual(fs.listdir(dir_a), ['file1'])   # pylint: disable=g-generic-assert
 
+  def test_glob(self):
+    fs = file_system.MemoryFileSystem()
+    fs.mkdirs('/mem/a/b/c')
+    with fs.open('/mem/a/foo.txt', 'w') as f:
+      f.write('foo')
+    with fs.open('/mem/a/bar.json', 'w') as f:
+      f.write('bar')
+    with fs.open('/mem/a/b/baz.txt', 'w') as f:
+      f.write('baz')
+
+    self.assertEqual(
+        sorted(fs.glob('/mem/a/*')),
+        ['/mem/a/b', '/mem/a/b/baz.txt', '/mem/a/b/c',
+         '/mem/a/bar.json', '/mem/a/foo.txt'])
+    self.assertEqual(
+        sorted(fs.glob('/mem/a/*.txt')),
+        ['/mem/a/b/baz.txt', '/mem/a/foo.txt'])
+    self.assertEqual(
+        sorted(fs.glob('/mem/a/b/*')),
+        ['/mem/a/b/baz.txt', '/mem/a/b/c'])
+    self.assertEqual(fs.glob('/mem/a/b/*.txt'), ['/mem/a/b/baz.txt'])
+    self.assertEqual(fs.glob('/mem/a/b/c/*'), [])
+    self.assertEqual(fs.glob('/mem/a/???.txt'), ['/mem/a/foo.txt'])
+    self.assertEqual(fs.glob('/mem/a/bar.*'), ['/mem/a/bar.json'])
+    self.assertEqual(
+        sorted(fs.glob('/mem/a/*.*')),
+        ['/mem/a/b/baz.txt', '/mem/a/bar.json', '/mem/a/foo.txt'])
+
+  def test_rename(self):
+    fs = file_system.MemoryFileSystem()
+    fs.mkdirs('/mem/a/b')
+    with fs.open('/mem/a/foo.txt', 'w') as f:
+      f.write('foo')
+    with fs.open('/mem/a/bar.txt', 'w') as f:
+      f.write('bar')
+
+    # Rename file to a new name.
+    fs.rename('/mem/a/foo.txt', '/mem/a/foo-new.txt')
+    self.assertFalse(fs.exists('/mem/a/foo.txt'))
+    self.assertTrue(fs.exists('/mem/a/foo-new.txt'))
+
+    # Rename file to an existing file name.
+    fs.rename('/mem/a/foo-new.txt', '/mem/a/bar.txt')
+    self.assertFalse(fs.exists('/mem/a/foo-new.txt'))
+    with fs.open('/mem/a/bar.txt', 'r') as f:
+      self.assertEqual(f.read(), 'foo')
+
+    # Rename directory to a new name.
+    fs.rename('/mem/a/b', '/mem/a/c')
+    self.assertFalse(fs.exists('/mem/a/b'))
+    self.assertTrue(fs.exists('/mem/a/c'))
+    self.assertTrue(fs.isdir('/mem/a/c'))
+
+    # Rename directory to an existing empty directory.
+    fs.mkdirs('/mem/a/d')
+    fs.rename('/mem/a/c', '/mem/a/d')
+    self.assertFalse(fs.exists('/mem/a/c'))
+    self.assertTrue(fs.exists('/mem/a/d'))
+
+    # Rename directory to a non-empty directory.
+    fs.mkdirs('/mem/x/y')
+    with self.assertRaisesRegex(OSError, "Directory not empty: '/mem/x'"):
+      fs.rename('/mem/a', '/mem/x')
+    self.assertTrue(fs.exists('/mem/a'))
+    self.assertTrue(fs.exists('/mem/x/y'))
+
+    # Errors
+    fs.mkdirs('/mem/u/v')
+    with fs.open('/mem/u/a.txt', 'w') as f:
+      f.write('a')
+
+    with self.assertRaisesRegex(
+        OSError, "Cannot move directory '/mem/u' to a subdirectory of itself"):
+      fs.rename('/mem/u', '/mem/u/v/w')
+
+    with self.assertRaisesRegex(
+        NotADirectoryError,
+        "Cannot rename directory '/mem/u' to non-directory '/mem/u/a.txt'"):
+      fs.rename('/mem/u', '/mem/u/a.txt')
+
+    with self.assertRaisesRegex(
+        IsADirectoryError,
+        "Cannot rename non-directory '/mem/u/a.txt' to directory '/mem/u/v'"):
+      fs.rename('/mem/u/a.txt', '/mem/u/v')
+
+    with self.assertRaises(FileNotFoundError):
+      fs.rename('/mem/non-existent', '/mem/y')
+
 
 class FileIoApiTest(unittest.TestCase):
 
@@ -217,6 +374,14 @@ class FileIoApiTest(unittest.TestCase):
     file_system.rm(file2)
     self.assertFalse(file_system.path_exists(file2))
 
+    # Test glob with standard file system.
+    glob_dir = os.path.join(tempfile.mkdtemp(), 'glob')
+    file_system.mkdirs(os.path.join(glob_dir, 'a/b'))
+    file_system.writefile(os.path.join(glob_dir, 'a/foo.txt'), 'foo')
+    self.assertEqual(
+        sorted(file_system.glob(os.path.join(glob_dir, 'a/*'))),
+        [os.path.join(glob_dir, 'a/b'), os.path.join(glob_dir, 'a/foo.txt')])
+
   def test_memory_filesystem(self):
     file1 = pathlib.Path('/mem/file1')
     with self.assertRaises(FileNotFoundError):
@@ -248,6 +413,15 @@ class FileIoApiTest(unittest.TestCase):
     file_system.rm(file2)
     self.assertFalse(file_system.path_exists(file2))
 
+    # Test glob with memory file system.
+    file_system.mkdirs('/mem/g/a/b')
+    file_system.writefile('/mem/g/a/foo.txt', 'foo')
+    file_system.rename('/mem/g/a/foo.txt', '/mem/g/a/foo2.txt')
+    file_system.writefile('/mem/g/a/b/bar.txt', 'bar')
+    self.assertEqual(
+        sorted(file_system.glob('/mem/g/a/*')),
+        ['/mem/g/a/b', '/mem/g/a/b/bar.txt', '/mem/g/a/foo2.txt'])
+
 
 if __name__ == '__main__':
   unittest.main()

{pyglove-0.5.0.dev202511040811 → pyglove-0.5.0.dev202511161718}/pyglove/core/typing/type_conversion_test.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 import calendar
 import datetime
+import sys
 import typing
 import unittest
 
@@ -130,12 +131,16 @@ class BuiltInConversionsTest(unittest.TestCase):
   def test_datetime_to_int(self):
     """Test built-in converter between int and datetime.datetime."""
     timestamp = calendar.timegm(datetime.datetime.now().timetuple())
-    now = datetime.datetime.fromtimestamp(timestamp, datetime.UTC)
+    if sys.version_info >= (3, 11):
+      now = datetime.datetime.fromtimestamp(timestamp, datetime.UTC)
+    else:
+      now = datetime.datetime.utcfromtimestamp(timestamp)
     self.assertEqual(vs.Object(datetime.datetime).apply(timestamp), now)
     self.assertEqual(vs.Int().apply(now), timestamp)
     self.assertEqual(
         type_conversion.get_json_value_converter(datetime.datetime)(now),
-        timestamp)
+        timestamp
+    )
 
   def test_keypath_to_str(self):
     """Test built-in converter between string and KeyPath."""

{pyglove-0.5.0.dev202511040811 → pyglove-0.5.0.dev202511161718}/pyglove.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyglove
-Version: 0.5.0.dev202511040811
+Version: 0.5.0.dev202511161718
 Summary: PyGlove: A library for manipulating Python objects.
 Home-page: https://github.com/google/pyglove
 Author: PyGlove Authors