langfun 0.1.2.dev202504110804__py3-none-any.whl → 0.1.2.dev202504120803__py3-none-any.whl

langfun/core/eval/v2/evaluation.py

@@ -724,18 +724,15 @@ class EvaluationState:
       filter: Callable[[example_lib.Example], bool] | None = None,  # pylint: disable=redefined-builtin
   ) -> None:
     """Loads the state from the example sequence file."""
-    with pg.io.sequence.open_sequence(state_file) as f:
-      for record in f:
-        example = pg.from_json_str(
-            record,
-            example_input_by_id=example_input_by_id,
-            load_example_metadata=load_example_metadata
-        )
-        assert isinstance(example, example_lib.Example), example
-        if filter is not None and not filter(example):
-          continue
-        example.newly_processed = False
-        self._ckpt_examples[example.id] = example
+    for example in example_lib.Example.iter_ckpts(
+        state_file,
+        example_input_by_id=example_input_by_id,
+        load_example_metadata=load_example_metadata,
+    ):
+      if filter is not None and not filter(example):
+        continue
+      example.newly_processed = False
+      self._ckpt_examples[example.id] = example
 
   @property
   def evaluation_status(self) -> dict[int, ExampleStatus]:

langfun/core/eval/v2/example.py

@@ -15,7 +15,7 @@
 
 import dataclasses
 import inspect
-from typing import Any, Callable
+from typing import Any, Callable, Iterator
 import langfun.core as lf
 import pyglove as pg
 
@@ -145,6 +145,26 @@ class Example(pg.JSONConvertible, pg.views.HtmlTreeView.Extension):
         example.metadata = pg.from_json(metadata_dict, **kwargs)
       return example
 
+  @classmethod
+  def iter_ckpts(
+      cls,
+      ckpt_file: str | list[str],
+      example_input_by_id: Callable[[int], Any] | None = None,
+      load_example_metadata: bool = True,
+  ) -> Iterator['Example']:
+    """Iterates Examples from the checkpoint files."""
+    ckpt_files = [ckpt_file] if isinstance(ckpt_file, str) else ckpt_file
+    for ckpt_file in ckpt_files:
+      with pg.io.open_sequence(ckpt_file) as f:
+        for record in f:
+          example = pg.from_json_str(
+              record,
+              example_input_by_id=example_input_by_id,
+              load_example_metadata=load_example_metadata
+          )
+          assert isinstance(example, cls), example
+          yield example
+
   #
   # HTML rendering.
   #

{langfun-0.1.2.dev202504110804.dist-info → langfun-0.1.2.dev202504120803.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langfun
-Version: 0.1.2.dev202504110804
+Version: 0.1.2.dev202504120803
 Summary: Langfun: Language as Functions.
 Home-page: https://github.com/google/langfun
 Author: Langfun Authors

{langfun-0.1.2.dev202504110804.dist-info → langfun-0.1.2.dev202504120803.dist-info}/RECORD

@@ -63,9 +63,9 @@ langfun/core/eval/v2/__init__.py,sha256=9lNKJwbvl0lcFblAXYT_OHI8fOubJsTOdSkxEqsP
 langfun/core/eval/v2/checkpointing.py,sha256=t47rBfzGZYgIqWW1N1Ak9yQnNtHd-IRbEO0cZjG2VRo,11755
 langfun/core/eval/v2/checkpointing_test.py,sha256=NggOSJ_6XSa4cNP6nGIu9wLsK59dUwe8SPWDiXtGGDE,9197
 langfun/core/eval/v2/eval_test_helper.py,sha256=sKFi_wPYCNmr96WyTduuXY0KnxjFxcJyEhXey-_nGX8,3962
-langfun/core/eval/v2/evaluation.py,sha256=faL-G98rdGBW9hMSlzZJtkGMytBqvTZRue1k9m_BWRQ,24029
+langfun/core/eval/v2/evaluation.py,sha256=E5PlPb9AA4VV12wHkwH-TXj-3BmYzr1GqhGFcY8gVvs,23882
 langfun/core/eval/v2/evaluation_test.py,sha256=TKbeyeenoaUDH23E3TXXJ4otuq3VZBuWkWdV7c5vMk4,6804
-langfun/core/eval/v2/example.py,sha256=4-LNr8Ke-fhaF6gyeXX4JMyw0s8YkVTC63pXZ-CXKrE,10144
+langfun/core/eval/v2/example.py,sha256=Jegt-viQSNYzPVkOZE_M19GON2TYGTct4Cp9HnJ7DGo,10861
 langfun/core/eval/v2/example_test.py,sha256=1DNm6EuyZOq827DKvf3oTRVFkMNM_qTnLUpvOjpgz5I,3419
 langfun/core/eval/v2/experiment.py,sha256=hHNyIw1uSV-hW2nMMx4hV-uul7KwXCmp7jAWNRyT5fU,32938
 langfun/core/eval/v2/experiment_test.py,sha256=UmCobeS6ifPcaGkTJp0WPISolXrVFbeFCBiyJeA0Lt4,13666
@@ -156,8 +156,8 @@ langfun/core/templates/demonstration.py,sha256=vCrgYubdZM5Umqcgp8NUVGXgr4P_c-fik
 langfun/core/templates/demonstration_test.py,sha256=SafcDQ0WgI7pw05EmPI2S4v1t3ABKzup8jReCljHeK4,2162
 langfun/core/templates/selfplay.py,sha256=yhgrJbiYwq47TgzThmHrDQTF4nDrTI09CWGhuQPNv-s,2273
 langfun/core/templates/selfplay_test.py,sha256=Ot__1P1M8oJfoTp-M9-PQ6HUXqZKyMwvZ5f7yQ3yfyM,2326
-langfun-0.1.2.dev202504110804.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-langfun-0.1.2.dev202504110804.dist-info/METADATA,sha256=UuGetAgxCOtGnl_sRK-qCacVQ47Sol26gShG223KEgw,7692
-langfun-0.1.2.dev202504110804.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-langfun-0.1.2.dev202504110804.dist-info/top_level.txt,sha256=RhlEkHxs1qtzmmtWSwYoLVJAc1YrbPtxQ52uh8Z9VvY,8
-langfun-0.1.2.dev202504110804.dist-info/RECORD,,
+langfun-0.1.2.dev202504120803.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+langfun-0.1.2.dev202504120803.dist-info/METADATA,sha256=425wS7mJbBj9ud9wfkWPDmCw9fmluo-qREBCcBZ06tk,7692
+langfun-0.1.2.dev202504120803.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+langfun-0.1.2.dev202504120803.dist-info/top_level.txt,sha256=RhlEkHxs1qtzmmtWSwYoLVJAc1YrbPtxQ52uh8Z9VvY,8
+langfun-0.1.2.dev202504120803.dist-info/RECORD,,