pyglove 0.4.5.dev202410301815__py3-none-any.whl → 0.4.5.dev202410310809__py3-none-any.whl

pyglove/core/io/file_system.py

@@ -116,7 +116,7 @@ class FileSystem(metaclass=abc.ABCMeta):
     """Removes a directory chain based on a path."""
 
 
-def _resolve_path(path: Union[str, os.PathLike[str]]) -> str:
+def resolve_path(path: Union[str, os.PathLike[str]]) -> str:
   if isinstance(path, str):
     return path
   elif hasattr(path, '__fspath__'):
@@ -243,7 +243,7 @@ class MemoryFileSystem(FileSystem):
     self._prefix = prefix
 
   def _internal_path(self, path: Union[str, os.PathLike[str]]) -> str:
-    return '/' + _resolve_path(path).lstrip(self._prefix)
+    return '/' + resolve_path(path).lstrip(self._prefix)
 
   def _locate(self, path: Union[str, os.PathLike[str]]) -> Any:
     current = self._root
@@ -287,7 +287,7 @@ class MemoryFileSystem(FileSystem):
   def _parent_and_name(
       self, path: Union[str, os.PathLike[str]]
   ) -> tuple[dict[str, Any], str]:
-    path = _resolve_path(path)
+    path = resolve_path(path)
     rpos = path.rfind('/')
     assert rpos >= 0, path
     name = path[rpos + 1:]
@@ -382,7 +382,7 @@ class _FileSystemRegistry:
 
   def get(self, path: Union[str, os.PathLike[str]]) -> FileSystem:
     """Gets the file system for a path."""
-    path = _resolve_path(path)
+    path = resolve_path(path)
     for prefix, fs in self._filesystems:
       if path.startswith(prefix):
         return fs

pyglove/core/io/sequence.py

@@ -107,7 +107,7 @@ class _SequenceIORegistry(object):
 
   def get(self, path: Union[str, os.PathLike[str]]) -> SequenceIO:
     """Gets the record IO system for a path."""
-    path = _resolve_path(path)
+    path = file_system.resolve_path(path)
     parts = path.split('.')
     if parts:
       extension = parts[-1].lower()
@@ -126,15 +126,6 @@ def add_sequence_io(extension: str, sequence_io: SequenceIO) -> None:
   _registry.add(extension, sequence_io)
 
 
-def _resolve_path(path: Union[str, os.PathLike[str]]) -> str:
-  if isinstance(path, str):
-    return path
-  elif hasattr(path, '__fspath__'):
-    return path.__fspath__()
-  else:
-    raise ValueError(f'Unsupported path: {path!r}.')
-
-
 def open_sequence(
     path: Union[str, os.PathLike[str]],
     mode: str = 'r',
@@ -145,6 +136,7 @@ def open_sequence(
     deserializer: Optional[
         Callable[[Union[bytes, str]], Any]
     ] = None,
+    make_dirs_if_not_exist: bool = True,
 ) -> Sequence:
   """Open sequence for reading or writing.
 
@@ -155,10 +147,16 @@ def open_sequence(
       object to a string or bytes.
     deserializer: (Optional) A deserializer function for converting a string or
       bytes to a structured object.
+    make_dirs_if_not_exist: (Optional) Whether to create the directories
+      if they do not exist. Applicable when opening in write or append mode.
 
   Returns:
     A sequence for reading or writing.
   """
+  if 'w' in mode or 'a' in mode:
+    parent_dir = os.path.dirname(path)
+    if make_dirs_if_not_exist:
+      file_system.mkdirs(parent_dir, exist_ok=True)
   return _registry.get(path).open(
       path, mode, serializer=serializer, deserializer=deserializer
   )

pyglove/core/io/sequence_test.py

@@ -13,7 +13,6 @@
 # limitations under the License.
 
 import os
-import pathlib
 import tempfile
 import unittest
 from pyglove.core.io import sequence as sequence_io
@@ -25,7 +24,7 @@ class LineSequenceIOTest(unittest.TestCase):
 
   def test_read_write(self):
     tmp_dir = tempfile.gettempdir()
-    file1 = os.path.join(tmp_dir, 'file1')
+    file1 = os.path.join(tmp_dir, 'abc', 'file1')
     with pg_symbolic.open_jsonl(file1, 'w') as f:
       self.assertIsInstance(f, sequence_io.LineSequence)
       f.add(1)
@@ -73,19 +72,7 @@ class LineSequenceIOTest(unittest.TestCase):
       self.assertEqual(list(iter(f)), ['foo', 'bar'])
 
 
-class MemoryRecordIOIOTest(unittest.TestCase):
-
-  def test_resolve_path(self):
-    self.assertEqual(
-        sequence_io._resolve_path('/file1.mem@123'),
-        '/file1.mem@123'
-    )
-    self.assertEqual(
-        sequence_io._resolve_path(pathlib.Path('/file1.mem@123.txt')),
-        '/file1.mem@123.txt'
-    )
-    with self.assertRaisesRegex(ValueError, 'Unsupported path'):
-      sequence_io._resolve_path(1)
+class MemorySequenceIOTest(unittest.TestCase):
 
   def test_read_write(self):
     with sequence_io.open_sequence('/file1.mem@123', 'w') as f:

pyglove/core/object_utils/error_utils.py

@@ -30,19 +30,19 @@ class ErrorInfo(json_conversion.JSONConvertible, formatting.Formattable):
   """Serializable error information.
 
   Attributes:
-    type: The type path of the error. For example,
+    tag: A path of the error types in the exception chain. For example,
       `ValueError.ZeroDivisionError` means the error is a `ZeroDivisionError`
       raised at the first place and then reraised as a `ValueError`.
     description: The description of the error.
     stacktrace: The stacktrace of the error.
   """
 
-  type: str
+  tag: str
   description: str
   stacktrace: str
 
   @classmethod
-  def _get_type(cls, error: BaseException):
+  def _compute_tag(cls, error: BaseException):
     error_types = []
     while error is not None:
       error_types.append(error.__class__.__name__)
@@ -53,7 +53,7 @@ class ErrorInfo(json_conversion.JSONConvertible, formatting.Formattable):
   def from_exception(cls, error: BaseException) -> 'ErrorInfo':
     """Creates an error info from an exception."""
     return cls(
-        type=cls._get_type(error),
+        tag=cls._compute_tag(error),
         description=str(error),
         stacktrace=''.join(
             traceback.format_exception(*sys.exc_info())
@@ -63,7 +63,7 @@ class ErrorInfo(json_conversion.JSONConvertible, formatting.Formattable):
   def to_json(self, **kwargs) -> Dict[str, Any]:
     return self.to_json_dict(
         fields=dict(
-            type=(self.type, None),
+            tag=(self.tag, None),
             description=(self.description, None),
             stacktrace=(self.stacktrace, None),
         ),
@@ -74,7 +74,7 @@ class ErrorInfo(json_conversion.JSONConvertible, formatting.Formattable):
   def format(self, *args, **kwargs) -> str:
     return formatting.kvlist_str(
         [
-            ('type', self.type, None),
+            ('tag', self.tag, None),
             ('description', self.description, None),
             ('stacktrace', self.stacktrace, None),
         ],

pyglove/core/object_utils/error_utils_test.py

@@ -36,13 +36,13 @@ class ErrorInfoTest(unittest.TestCase):
     except ValueError as e:
       error_info = error_utils.ErrorInfo.from_exception(e)
     self.assertIsNotNone(error_info)
-    self.assertEqual(error_info.type, 'ValueError.ZeroDivisionError')
+    self.assertEqual(error_info.tag, 'ValueError.ZeroDivisionError')
     self.assertEqual(error_info.description, 'Bad call to `foo`')
     self.assertIn('Traceback (most recent call last)', error_info.stacktrace)
 
   def test_to_json(self):
     error_info = error_utils.ErrorInfo(
-        type='ValueError.ZeroDivisionError',
+        tag='ValueError.ZeroDivisionError',
         description='Bad call to `foo`',
         stacktrace='Traceback (most recent call last)',
     )
@@ -53,7 +53,7 @@ class ErrorInfoTest(unittest.TestCase):
 
   def test_format(self):
     error_info = error_utils.ErrorInfo(
-        type='ValueError.ZeroDivisionError',
+        tag='ValueError.ZeroDivisionError',
         description='Bad call to `foo`',
         stacktrace='Traceback (most recent call last)',
     )
@@ -62,7 +62,7 @@ class ErrorInfoTest(unittest.TestCase):
         inspect.cleandoc(
             """
             ErrorInfo(
-              type='ValueError.ZeroDivisionError',
+              tag='ValueError.ZeroDivisionError',
               description='Bad call to `foo`',
               stacktrace='Traceback (most recent call last)'
             )

pyglove/core/object_utils/timing.py

@@ -13,10 +13,12 @@
 # limitations under the License.
 """Utilities for timing."""
 
+import collections
 import dataclasses
 import time
 from typing import Any, Dict, List, Optional
 
+from pyglove.core.object_utils import error_utils
 from pyglove.core.object_utils import json_conversion
 from pyglove.core.object_utils import thread_local
 
@@ -30,7 +32,7 @@ class TimeIt:
     name: str
     elapse: float = 0.0
     has_ended: bool = True
-    error: Optional[Exception] = None
+    error: Optional[error_utils.ErrorInfo] = None
 
     @property
     def has_started(self) -> bool:
@@ -66,6 +68,9 @@ class TimeIt:
       num_ended: int = 0
       num_failed: int = 0
       avg_duration: float = 0.0
+      error_tags: Dict[str, int] = dataclasses.field(
+          default_factory=lambda: collections.defaultdict(int)
+      )
 
       def update(self, status: 'TimeIt.Status'):
         self.avg_duration = (
@@ -77,6 +82,8 @@ class TimeIt:
           self.num_ended += 1
         if status.has_error:
           self.num_failed += 1
+          assert status.error is not None
+          self.error_tags[status.error.tag] += 1
 
       def to_json(self, **kwargs) -> Dict[str, Any]:
         return self.to_json_dict(
@@ -85,6 +92,7 @@ class TimeIt:
                 num_ended=(self.num_ended, 0),
                 num_failed=(self.num_failed, 0),
                 avg_duration=(self.avg_duration, 0.0),
+                error_tags=(self.error_tags, {}),
             ),
             exclude_default=True,
             **kwargs,
@@ -113,13 +121,13 @@ class TimeIt:
           **kwargs,
       )
 
-  def __init__(self, name: str):
-    self._name = name
-    self._start_time = None
-    self._end_time = None
-    self._child_contexts = {}
-    self._error = None
-    self._parent = None
+  def __init__(self, name: str = ''):
+    self._name: str = name
+    self._start_time: Optional[float] = None
+    self._end_time: Optional[float] = None
+    self._child_contexts: Dict[str, TimeIt] = {}
+    self._error: Optional[error_utils.ErrorInfo] = None
+    self._parent: Optional[TimeIt] = None
 
   @property
   def name(self) -> str:
@@ -145,7 +153,9 @@ class TimeIt:
     """Ends timing."""
     if not self.has_ended:
       self._end_time = time.time()
-      self._error = error
+      self._error = (
+          None if error is None else error_utils.ErrorInfo.from_exception(error)
+      )
       return True
     return False
 
@@ -170,7 +180,7 @@ class TimeIt:
     return self._end_time
 
   @property
-  def error(self) -> Optional[BaseException]:
+  def error(self) -> Optional[error_utils.ErrorInfo]:
     """Returns error."""
     return self._error
 
@@ -199,7 +209,8 @@ class TimeIt:
     for child in self._child_contexts.values():
       child_result = child.status()
       for k, v in child_result.items():
-        result[f'{self.name}.{k}'] = v
+        key = f'{self.name}.{k}' if self.name else k
+        result[key] = v
     return result
 
   def __enter__(self):
@@ -220,6 +231,6 @@ class TimeIt:
       thread_local.thread_local_set('__timing_context__', self._parent)
 
 
-def timeit(name: str) -> TimeIt:
+def timeit(name: str = '') -> TimeIt:
   """Context manager to time a block of code."""
   return TimeIt(name)

pyglove/core/object_utils/timing_test.py

@@ -100,8 +100,8 @@ class TimeItTest(unittest.TestCase):
     r = t.status()
     self.assertTrue(r['node'].has_error)
     self.assertTrue(t.has_error)
-    self.assertIsInstance(t.error, ValueError)
-    self.assertIsInstance(r['node'].error, ValueError)
+    self.assertTrue(t.error.tag.startswith('ValueError'))
+    self.assertTrue(r['node'].error.tag.startswith('ValueError'))
     self.assertTrue(r['node.child'].has_error)
     self.assertTrue(t1.has_error)
     self.assertTrue(r['node.child.grandchild'].has_error)
@@ -111,7 +111,7 @@ class TimeItTest(unittest.TestCase):
     summary = timing.TimeIt.StatusSummary()
     self.assertFalse(summary)
     for i in range(10):
-      with timing.timeit('node') as t:
+      with timing.timeit() as t:
         time.sleep(0.1)
         with timing.timeit('child'):
           time.sleep(0.1)
@@ -126,7 +126,7 @@ class TimeItTest(unittest.TestCase):
     self.assertTrue(summary)
     self.assertEqual(
         list(summary.breakdown.keys()),
-        ['node', 'node.child', 'node.child.grandchild']
+        ['', 'child', 'child.grandchild']
     )
     self.assertEqual(
         [x.num_started for x in summary.breakdown.values()],
@@ -140,6 +140,10 @@ class TimeItTest(unittest.TestCase):
         [x.num_failed for x in summary.breakdown.values()],
         [0, 0, 2]
     )
+    self.assertEqual(
+        summary.breakdown['child.grandchild'].error_tags,
+        {'ValueError': 2},
+    )
     # Test serialization.
     json_dict = summary.to_json()
     summary2 = timing.TimeIt.StatusSummary.from_json(json_dict)

{pyglove-0.4.5.dev202410301815.dist-info → pyglove-0.4.5.dev202410310809.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pyglove
-Version: 0.4.5.dev202410301815
+Version: 0.4.5.dev202410310809
 Summary: PyGlove: A library for manipulating Python objects.
 Home-page: https://github.com/google/pyglove
 Author: PyGlove Authors

{pyglove-0.4.5.dev202410301815.dist-info → pyglove-0.4.5.dev202410310809.dist-info}/RECORD

@@ -43,10 +43,10 @@ pyglove/core/hyper/numerical_test.py,sha256=_xZe0qe3wJdrQL4F8pgt0zf04zpbc_9c_5va
 pyglove/core/hyper/object_template.py,sha256=x6aBe-6ueo3CsFJ2yhbXbcdJKfId6wm_KsgkHHbpSoY,22325
 pyglove/core/hyper/object_template_test.py,sha256=TEFX7LIqUvdCdJILnK_gP5xIgNJKzRnioUF0CGVBzcY,9105
 pyglove/core/io/__init__.py,sha256=4ZT1a595DqQuLTNYc2JP_eCp_KesXvHmKRkr777bzpg,785
-pyglove/core/io/file_system.py,sha256=KR1riWNBVZpkgIcqPUhR5xqjCN5Lhp937x7Ubjpv-jE,13283
+pyglove/core/io/file_system.py,sha256=ZfeQeZ1HF1WjV1REC9wCaut5Wkdu_FjxPzulUju7Z_E,13279
 pyglove/core/io/file_system_test.py,sha256=FX0ySuh_Xcg1RO68do7ikD4pvslKUzfSpwZ6P4wIP7c,8691
-pyglove/core/io/sequence.py,sha256=Kir8JM8lxjrlKr1gs3hvhCB8F08PPz4K8-aDI_US9Vo,8042
-pyglove/core/io/sequence_test.py,sha256=iqDR-MPm64VbUPKFYocVKH91Y6Z6tP1TZB0PzCn92tE,4265
+pyglove/core/io/sequence.py,sha256=7QWMGXPtJzHyGPgqkT3yJ01FxKJ4mP4lF5HRDiIHNbQ,8165
+pyglove/core/io/sequence_test.py,sha256=6tmnS7frBuDR8ussT5jugeh23TDsPDrDnbfxHh81Gy4,3891
 pyglove/core/object_utils/__init__.py,sha256=bFyt-_uNzpOjJGDfYFTRP1kV6rL4du9xGx_OaFMcIxI,8758
 pyglove/core/object_utils/codegen.py,sha256=HQlpjz42oh_2lrGSflwuAcEneE9Bv8pgPahQzvJm1KI,1568
 pyglove/core/object_utils/codegen_test.py,sha256=ThPVyE805plNvM8vwlDxMrcjV-GGdDgbBLSqSOpCvJ0,2166
@@ -54,8 +54,8 @@ pyglove/core/object_utils/common_traits.py,sha256=9wfWvcq3uuVgL8mQexgJmWdF2aaQ-w
 pyglove/core/object_utils/common_traits_test.py,sha256=wIeVsF3kon9K0Kbblcaib9hBJeZ76LyGZDD-5A1BD9w,1182
 pyglove/core/object_utils/docstr_utils.py,sha256=5BY40kXozPKVGOB0eN8jy1P5_GHIzqFJ9FXAu_kzxaw,5119
 pyglove/core/object_utils/docstr_utils_test.py,sha256=1NIsXXpp87HGC8LB7vx1KEMVVrmeGi733Xpuf6yTUq0,4287
-pyglove/core/object_utils/error_utils.py,sha256=wnGX_aR4TdbCylvUOh-x6PPYzGSJBnfyX9MANpk03u8,5780
-pyglove/core/object_utils/error_utils_test.py,sha256=vlFkR41jNLzUnwqF6IXy7HmXhpLlwOvUvlVMoMSOitk,4175
+pyglove/core/object_utils/error_utils.py,sha256=5mcD61La1tTYmmKodsWfnt8R-yV3yfBfbW4o5rZzCTg,5801
+pyglove/core/object_utils/error_utils_test.py,sha256=rnX4HYsZaMphWPm4ppgzCQ49LAXfJJ9KDTiDz4f9Ahc,4171
 pyglove/core/object_utils/formatting.py,sha256=cXGWJd6zleS9QiEfY3w4fqWfFYgi_QF8S2IapTOzAj8,14994
 pyglove/core/object_utils/formatting_test.py,sha256=QGbpUS8u70NoF8-3v5IYmPWEsML3ZbIwkQpUMwQdTkQ,13589
 pyglove/core/object_utils/hierarchical.py,sha256=za9sA6XnlUOKUHMKNwQPNSOK9E3wDqXFQB0AND2Yghw,19754
@@ -66,8 +66,8 @@ pyglove/core/object_utils/missing.py,sha256=0liMs9iEyQxxu6UohdJ5hEM246e9Nu05qp0h
 pyglove/core/object_utils/missing_test.py,sha256=B36p-vqUvAnXWMszAj9GOPBN0_8cq7vVF61AkcsZ9qU,1396
 pyglove/core/object_utils/thread_local.py,sha256=i-CnyY3VREtLfAj4_JndBnsKuQLIgwG29ma8dAyRxbI,4839
 pyglove/core/object_utils/thread_local_test.py,sha256=EvU1-TF7KqpLQxxBvHd7dxtuY22YUQSIwQ0UcR-NORA,6816
-pyglove/core/object_utils/timing.py,sha256=PeSN59g8vQDpenPgsWf4oYP45c0UDSpnFYDKhrtcSBc,6665
-pyglove/core/object_utils/timing_test.py,sha256=Ye4qBrNl5ix2C_K-qT5TxKHui4Kvwmea3uUTx8t26qk,4828
+pyglove/core/object_utils/timing.py,sha256=ihOdcZaJo96wpe4U-bXEu-H1PZMWeWOaU-nSW9DFStY,7262
+pyglove/core/object_utils/timing_test.py,sha256=jym-jV55kRujhgoxmDUUIXuqRLDnBGYiNKN3lUMbnfg,4943
 pyglove/core/object_utils/value_location.py,sha256=lSFQNTazY2M6_nRLmMbouqZAcZSiOLZQnmQPMD2FDMs,26770
 pyglove/core/object_utils/value_location_test.py,sha256=wCZYqWf_tq3l4ZM6dnkAs1obmq8LfYawkns2NKos9kk,21406
 pyglove/core/patching/__init__.py,sha256=C1Q1cWPV74YL3eXbzGvc-8aPw1DR8EK6lRhQYDCwHek,2059
@@ -186,8 +186,8 @@ pyglove/ext/scalars/randoms.py,sha256=LkMIIx7lOq_lvJvVS3BrgWGuWl7Pi91-lA-O8x_gZs
 pyglove/ext/scalars/randoms_test.py,sha256=nEhiqarg8l_5EOucp59CYrpO2uKxS1pe0hmBdZUzRNM,2000
 pyglove/ext/scalars/step_wise.py,sha256=IDw3tuTpv0KVh7AN44W43zqm1-E0HWPUlytWOQC9w3Y,3789
 pyglove/ext/scalars/step_wise_test.py,sha256=TL1vJ19xVx2t5HKuyIzGoogF7N3Rm8YhLE6JF7i0iy8,2540
-pyglove-0.4.5.dev202410301815.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-pyglove-0.4.5.dev202410301815.dist-info/METADATA,sha256=jIeDOiGyeIig84equ3ZG2EjO4s1his-CfD1PjMETKpU,6666
-pyglove-0.4.5.dev202410301815.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
-pyglove-0.4.5.dev202410301815.dist-info/top_level.txt,sha256=wITzJSKcj8GZUkbq-MvUQnFadkiuAv_qv5qQMw0fIow,8
-pyglove-0.4.5.dev202410301815.dist-info/RECORD,,
+pyglove-0.4.5.dev202410310809.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+pyglove-0.4.5.dev202410310809.dist-info/METADATA,sha256=HtMHbHQS4WSDH7njjSK3TS_F8CDrrhyGpPFI5v2Jphs,6666
+pyglove-0.4.5.dev202410310809.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
+pyglove-0.4.5.dev202410310809.dist-info/top_level.txt,sha256=wITzJSKcj8GZUkbq-MvUQnFadkiuAv_qv5qQMw0fIow,8
+pyglove-0.4.5.dev202410310809.dist-info/RECORD,,