pyglove 0.4.5.dev202410301815__tar.gz → 0.4.5.dev202410310809__tar.gz

{pyglove-0.4.5.dev202410301815 → pyglove-0.4.5.dev202410310809}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pyglove
-Version: 0.4.5.dev202410301815
+Version: 0.4.5.dev202410310809
 Summary: PyGlove: A library for manipulating Python objects.
 Home-page: https://github.com/google/pyglove
 Author: PyGlove Authors

{pyglove-0.4.5.dev202410301815 → pyglove-0.4.5.dev202410310809}/pyglove/core/io/file_system.py

@@ -116,7 +116,7 @@ class FileSystem(metaclass=abc.ABCMeta):
     """Removes a directory chain based on a path."""
 
 
-def _resolve_path(path: Union[str, os.PathLike[str]]) -> str:
+def resolve_path(path: Union[str, os.PathLike[str]]) -> str:
   if isinstance(path, str):
     return path
   elif hasattr(path, '__fspath__'):
@@ -243,7 +243,7 @@ class MemoryFileSystem(FileSystem):
     self._prefix = prefix
 
   def _internal_path(self, path: Union[str, os.PathLike[str]]) -> str:
-    return '/' + _resolve_path(path).lstrip(self._prefix)
+    return '/' + resolve_path(path).lstrip(self._prefix)
 
   def _locate(self, path: Union[str, os.PathLike[str]]) -> Any:
     current = self._root
@@ -287,7 +287,7 @@ class MemoryFileSystem(FileSystem):
   def _parent_and_name(
       self, path: Union[str, os.PathLike[str]]
   ) -> tuple[dict[str, Any], str]:
-    path = _resolve_path(path)
+    path = resolve_path(path)
     rpos = path.rfind('/')
     assert rpos >= 0, path
     name = path[rpos + 1:]
@@ -382,7 +382,7 @@ class _FileSystemRegistry:
 
   def get(self, path: Union[str, os.PathLike[str]]) -> FileSystem:
     """Gets the file system for a path."""
-    path = _resolve_path(path)
+    path = resolve_path(path)
     for prefix, fs in self._filesystems:
       if path.startswith(prefix):
         return fs

{pyglove-0.4.5.dev202410301815 → pyglove-0.4.5.dev202410310809}/pyglove/core/io/sequence.py

@@ -107,7 +107,7 @@ class _SequenceIORegistry(object):
 
   def get(self, path: Union[str, os.PathLike[str]]) -> SequenceIO:
     """Gets the record IO system for a path."""
-    path = _resolve_path(path)
+    path = file_system.resolve_path(path)
     parts = path.split('.')
     if parts:
       extension = parts[-1].lower()
@@ -126,15 +126,6 @@ def add_sequence_io(extension: str, sequence_io: SequenceIO) -> None:
   _registry.add(extension, sequence_io)
 
 
-def _resolve_path(path: Union[str, os.PathLike[str]]) -> str:
-  if isinstance(path, str):
-    return path
-  elif hasattr(path, '__fspath__'):
-    return path.__fspath__()
-  else:
-    raise ValueError(f'Unsupported path: {path!r}.')
-
-
 def open_sequence(
     path: Union[str, os.PathLike[str]],
     mode: str = 'r',
@@ -145,6 +136,7 @@ def open_sequence(
     deserializer: Optional[
         Callable[[Union[bytes, str]], Any]
     ] = None,
+    make_dirs_if_not_exist: bool = True,
 ) -> Sequence:
   """Open sequence for reading or writing.
 
@@ -155,10 +147,16 @@ def open_sequence(
       object to a string or bytes.
     deserializer: (Optional) A deserializer function for converting a string or
       bytes to a structured object.
+    make_dirs_if_not_exist: (Optional) Whether to create the directories
+      if they do not exist. Applicable when opening in write or append mode.
 
   Returns:
     A sequence for reading or writing.
   """
+  if 'w' in mode or 'a' in mode:
+    parent_dir = os.path.dirname(path)
+    if make_dirs_if_not_exist:
+      file_system.mkdirs(parent_dir, exist_ok=True)
   return _registry.get(path).open(
       path, mode, serializer=serializer, deserializer=deserializer
   )

{pyglove-0.4.5.dev202410301815 → pyglove-0.4.5.dev202410310809}/pyglove/core/io/sequence_test.py

@@ -13,7 +13,6 @@
 # limitations under the License.
 
 import os
-import pathlib
 import tempfile
 import unittest
 from pyglove.core.io import sequence as sequence_io
@@ -25,7 +24,7 @@ class LineSequenceIOTest(unittest.TestCase):
 
   def test_read_write(self):
     tmp_dir = tempfile.gettempdir()
-    file1 = os.path.join(tmp_dir, 'file1')
+    file1 = os.path.join(tmp_dir, 'abc', 'file1')
     with pg_symbolic.open_jsonl(file1, 'w') as f:
       self.assertIsInstance(f, sequence_io.LineSequence)
       f.add(1)
@@ -73,19 +72,7 @@ class LineSequenceIOTest(unittest.TestCase):
       self.assertEqual(list(iter(f)), ['foo', 'bar'])
 
 
-class MemoryRecordIOIOTest(unittest.TestCase):
-
-  def test_resolve_path(self):
-    self.assertEqual(
-        sequence_io._resolve_path('/file1.mem@123'),
-        '/file1.mem@123'
-    )
-    self.assertEqual(
-        sequence_io._resolve_path(pathlib.Path('/file1.mem@123.txt')),
-        '/file1.mem@123.txt'
-    )
-    with self.assertRaisesRegex(ValueError, 'Unsupported path'):
-      sequence_io._resolve_path(1)
+class MemorySequenceIOTest(unittest.TestCase):
 
   def test_read_write(self):
     with sequence_io.open_sequence('/file1.mem@123', 'w') as f:

{pyglove-0.4.5.dev202410301815 → pyglove-0.4.5.dev202410310809}/pyglove/core/object_utils/error_utils.py

@@ -30,19 +30,19 @@ class ErrorInfo(json_conversion.JSONConvertible, formatting.Formattable):
   """Serializable error information.
 
   Attributes:
-    type: The type path of the error. For example,
+    tag: A path of the error types in the exception chain. For example,
       `ValueError.ZeroDivisionError` means the error is a `ZeroDivisionError`
       raised at the first place and then reraised as a `ValueError`.
     description: The description of the error.
     stacktrace: The stacktrace of the error.
   """
 
-  type: str
+  tag: str
   description: str
   stacktrace: str
 
   @classmethod
-  def _get_type(cls, error: BaseException):
+  def _compute_tag(cls, error: BaseException):
     error_types = []
     while error is not None:
       error_types.append(error.__class__.__name__)
@@ -53,7 +53,7 @@ class ErrorInfo(json_conversion.JSONConvertible, formatting.Formattable):
   def from_exception(cls, error: BaseException) -> 'ErrorInfo':
     """Creates an error info from an exception."""
     return cls(
-        type=cls._get_type(error),
+        tag=cls._compute_tag(error),
         description=str(error),
         stacktrace=''.join(
             traceback.format_exception(*sys.exc_info())
@@ -63,7 +63,7 @@ class ErrorInfo(json_conversion.JSONConvertible, formatting.Formattable):
   def to_json(self, **kwargs) -> Dict[str, Any]:
     return self.to_json_dict(
         fields=dict(
-            type=(self.type, None),
+            tag=(self.tag, None),
             description=(self.description, None),
             stacktrace=(self.stacktrace, None),
         ),
@@ -74,7 +74,7 @@ class ErrorInfo(json_conversion.JSONConvertible, formatting.Formattable):
   def format(self, *args, **kwargs) -> str:
     return formatting.kvlist_str(
         [
-            ('type', self.type, None),
+            ('tag', self.tag, None),
             ('description', self.description, None),
             ('stacktrace', self.stacktrace, None),
         ],

{pyglove-0.4.5.dev202410301815 → pyglove-0.4.5.dev202410310809}/pyglove/core/object_utils/error_utils_test.py

@@ -36,13 +36,13 @@ class ErrorInfoTest(unittest.TestCase):
     except ValueError as e:
       error_info = error_utils.ErrorInfo.from_exception(e)
     self.assertIsNotNone(error_info)
-    self.assertEqual(error_info.type, 'ValueError.ZeroDivisionError')
+    self.assertEqual(error_info.tag, 'ValueError.ZeroDivisionError')
     self.assertEqual(error_info.description, 'Bad call to `foo`')
     self.assertIn('Traceback (most recent call last)', error_info.stacktrace)
 
   def test_to_json(self):
     error_info = error_utils.ErrorInfo(
-        type='ValueError.ZeroDivisionError',
+        tag='ValueError.ZeroDivisionError',
         description='Bad call to `foo`',
         stacktrace='Traceback (most recent call last)',
     )
@@ -53,7 +53,7 @@ class ErrorInfoTest(unittest.TestCase):
 
   def test_format(self):
     error_info = error_utils.ErrorInfo(
-        type='ValueError.ZeroDivisionError',
+        tag='ValueError.ZeroDivisionError',
         description='Bad call to `foo`',
         stacktrace='Traceback (most recent call last)',
     )
@@ -62,7 +62,7 @@ class ErrorInfoTest(unittest.TestCase):
         inspect.cleandoc(
             """
             ErrorInfo(
-              type='ValueError.ZeroDivisionError',
+              tag='ValueError.ZeroDivisionError',
               description='Bad call to `foo`',
               stacktrace='Traceback (most recent call last)'
             )

{pyglove-0.4.5.dev202410301815 → pyglove-0.4.5.dev202410310809}/pyglove/core/object_utils/timing.py

@@ -13,10 +13,12 @@
 # limitations under the License.
 """Utilities for timing."""
 
+import collections
 import dataclasses
 import time
 from typing import Any, Dict, List, Optional
 
+from pyglove.core.object_utils import error_utils
 from pyglove.core.object_utils import json_conversion
 from pyglove.core.object_utils import thread_local
 
@@ -30,7 +32,7 @@ class TimeIt:
     name: str
     elapse: float = 0.0
     has_ended: bool = True
-    error: Optional[Exception] = None
+    error: Optional[error_utils.ErrorInfo] = None
 
     @property
     def has_started(self) -> bool:
@@ -66,6 +68,9 @@ class TimeIt:
       num_ended: int = 0
       num_failed: int = 0
       avg_duration: float = 0.0
+      error_tags: Dict[str, int] = dataclasses.field(
+          default_factory=lambda: collections.defaultdict(int)
+      )
 
       def update(self, status: 'TimeIt.Status'):
         self.avg_duration = (
@@ -77,6 +82,8 @@ class TimeIt:
           self.num_ended += 1
         if status.has_error:
           self.num_failed += 1
+          assert status.error is not None
+          self.error_tags[status.error.tag] += 1
 
       def to_json(self, **kwargs) -> Dict[str, Any]:
         return self.to_json_dict(
@@ -85,6 +92,7 @@ class TimeIt:
                 num_ended=(self.num_ended, 0),
                 num_failed=(self.num_failed, 0),
                 avg_duration=(self.avg_duration, 0.0),
+                error_tags=(self.error_tags, {}),
             ),
             exclude_default=True,
             **kwargs,
@@ -113,13 +121,13 @@ class TimeIt:
           **kwargs,
       )
 
-  def __init__(self, name: str):
-    self._name = name
-    self._start_time = None
-    self._end_time = None
-    self._child_contexts = {}
-    self._error = None
-    self._parent = None
+  def __init__(self, name: str = ''):
+    self._name: str = name
+    self._start_time: Optional[float] = None
+    self._end_time: Optional[float] = None
+    self._child_contexts: Dict[str, TimeIt] = {}
+    self._error: Optional[error_utils.ErrorInfo] = None
+    self._parent: Optional[TimeIt] = None
 
   @property
   def name(self) -> str:
@@ -145,7 +153,9 @@ class TimeIt:
     """Ends timing."""
     if not self.has_ended:
       self._end_time = time.time()
-      self._error = error
+      self._error = (
+          None if error is None else error_utils.ErrorInfo.from_exception(error)
+      )
       return True
     return False
 
@@ -170,7 +180,7 @@ class TimeIt:
     return self._end_time
 
   @property
-  def error(self) -> Optional[BaseException]:
+  def error(self) -> Optional[error_utils.ErrorInfo]:
     """Returns error."""
     return self._error
 
@@ -199,7 +209,8 @@ class TimeIt:
     for child in self._child_contexts.values():
       child_result = child.status()
       for k, v in child_result.items():
-        result[f'{self.name}.{k}'] = v
+        key = f'{self.name}.{k}' if self.name else k
+        result[key] = v
     return result
 
   def __enter__(self):
@@ -220,6 +231,6 @@ class TimeIt:
       thread_local.thread_local_set('__timing_context__', self._parent)
 
 
-def timeit(name: str) -> TimeIt:
+def timeit(name: str = '') -> TimeIt:
   """Context manager to time a block of code."""
   return TimeIt(name)

{pyglove-0.4.5.dev202410301815 → pyglove-0.4.5.dev202410310809}/pyglove/core/object_utils/timing_test.py

@@ -100,8 +100,8 @@ class TimeItTest(unittest.TestCase):
     r = t.status()
     self.assertTrue(r['node'].has_error)
     self.assertTrue(t.has_error)
-    self.assertIsInstance(t.error, ValueError)
-    self.assertIsInstance(r['node'].error, ValueError)
+    self.assertTrue(t.error.tag.startswith('ValueError'))
+    self.assertTrue(r['node'].error.tag.startswith('ValueError'))
     self.assertTrue(r['node.child'].has_error)
     self.assertTrue(t1.has_error)
     self.assertTrue(r['node.child.grandchild'].has_error)
@@ -111,7 +111,7 @@ class TimeItTest(unittest.TestCase):
     summary = timing.TimeIt.StatusSummary()
     self.assertFalse(summary)
     for i in range(10):
-      with timing.timeit('node') as t:
+      with timing.timeit() as t:
         time.sleep(0.1)
         with timing.timeit('child'):
           time.sleep(0.1)
@@ -126,7 +126,7 @@ class TimeItTest(unittest.TestCase):
     self.assertTrue(summary)
     self.assertEqual(
         list(summary.breakdown.keys()),
-        ['node', 'node.child', 'node.child.grandchild']
+        ['', 'child', 'child.grandchild']
     )
     self.assertEqual(
         [x.num_started for x in summary.breakdown.values()],
@@ -140,6 +140,10 @@ class TimeItTest(unittest.TestCase):
         [x.num_failed for x in summary.breakdown.values()],
         [0, 0, 2]
     )
+    self.assertEqual(
+        summary.breakdown['child.grandchild'].error_tags,
+        {'ValueError': 2},
+    )
     # Test serialization.
     json_dict = summary.to_json()
     summary2 = timing.TimeIt.StatusSummary.from_json(json_dict)

{pyglove-0.4.5.dev202410301815 → pyglove-0.4.5.dev202410310809}/pyglove.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pyglove
-Version: 0.4.5.dev202410301815
+Version: 0.4.5.dev202410310809
 Summary: PyGlove: A library for manipulating Python objects.
 Home-page: https://github.com/google/pyglove
 Author: PyGlove Authors