langfun 0.1.2.dev202508050805__tar.gz → 0.1.2.dev202508070805__tar.gz

{langfun-0.1.2.dev202508050805 → langfun-0.1.2.dev202508070805}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langfun
-Version: 0.1.2.dev202508050805
+Version: 0.1.2.dev202508070805
 Summary: Langfun: Language as Functions.
 Home-page: https://github.com/google/langfun
 Author: Langfun Authors

{langfun-0.1.2.dev202508050805 → langfun-0.1.2.dev202508070805}/langfun/core/eval/v2/checkpointing.py

@@ -82,11 +82,7 @@ class Checkpointer(experiment_lib.Plugin):
       example: Example,
   ) -> None:
     """Saves the example to the checkpoint file."""
-    if example.has_error:
-      experiment.warning(
-          f'Example {example.id} has error. Skipping checkpointing.'
-      )
-    elif example.newly_processed:
+    if example.newly_processed:
       self._save_example(runner, experiment, example)
 
   def _load_experiment(

{langfun-0.1.2.dev202508050805 → langfun-0.1.2.dev202508070805}/langfun/core/eval/v2/checkpointing_test.py

@@ -101,12 +101,15 @@ class PerExampleCheckpointerTest(CheckpointerTest):
         self.assertIn(i + 1, collector.examples)
         example = collector.examples[i + 1]
         ckpt = run.output_path_for(leaf, f'checkpoint_{example.id}.jsonl')
-        if example.has_error:
-          self.assertFalse(pg.io.path_exists(ckpt))
-        else:
-          self.assertTrue(pg.io.path_exists(ckpt))
-          with pg.io.open_sequence(ckpt) as f:
-            self.assertEqual(len(list(iter(f))), 1)
+        self.assertTrue(pg.io.path_exists(ckpt))
+        with pg.io.open_sequence(ckpt) as f:
+          examples_from_ckpt = list(iter(f))
+          # `eval_test_helper.test_experiment` has two TestEvaluation with
+          # llm offset 0, which makes that evaluation run twice.
+          if example.has_error and leaf.lm.offset == 0:
+            self.assertEqual(len(examples_from_ckpt), 2)
+          else:
+            self.assertEqual(len(examples_from_ckpt), 1)
       if leaf.id not in num_processed:
         self.assertEqual(leaf.progress.num_skipped, 0)
         num_processed[leaf.id] = leaf.progress.num_processed
@@ -222,7 +225,10 @@ class BulkCheckpointerTest(CheckpointerTest):
       with pg.io.open_sequence(ckpt) as f:
         self.assertEqual(
             len(list(iter(f))),
-            leaf.progress.num_completed - leaf.progress.num_failed
+            # `eval_test_helper.test_experiment` has two TestEvaluation with
+            # llm offset 0, which makes that evaluation run twice.
+            leaf.progress.num_completed
+            + (leaf.progress.num_failed if leaf.lm.offset == 0 else 0)
         )
       if leaf.id not in num_processed:
         self.assertEqual(leaf.progress.num_skipped, 0)

{langfun-0.1.2.dev202508050805 → langfun-0.1.2.dev202508070805}/langfun/core/eval/v2/evaluation.py

@@ -149,12 +149,15 @@ class Evaluation(experiment_lib.Experiment):
       self,
       example: example_lib.Example | int,
       raise_if_has_error: bool = False,
+      reevaluate_upon_previous_errors: bool = True,
   ) -> example_lib.Example:
     """Evaluates a single example input.
 
     Args:
       example: An example ID or an example object with ID.
       raise_if_has_error: Whether to raise an error if the example has error.
+      reevaluate_upon_previous_errors: Whether to reevaluate the example if
+        the previous checkpointed run has error.
 
     Returns:
       The evaluated example with the output and metric metadata populated.
@@ -169,7 +172,9 @@ class Evaluation(experiment_lib.Experiment):
     checkpointed = self._state.ckpt_example(example.id)
 
     with pg.timeit('evaluate') as timeit, lf.track_usages() as usage_summary:
-      if checkpointed is None or checkpointed.has_error:
+      if checkpointed is None or (
+          reevaluate_upon_previous_errors and checkpointed.has_error
+      ):
         if checkpointed is None:
           self.info(
               f'Example {example.id} is being processed for the first time '
@@ -184,17 +189,27 @@ class Evaluation(experiment_lib.Experiment):
         self._state.update(example, in_progress=True)
         self._process(example, raise_if_has_error=raise_if_has_error)
       else:
-        self.info(
-            f'Example {example.id} skipped processing as prior run '
-            'is available and error free.'
-        )
+        if checkpointed.has_error:
+          self.warning(
+              f'Example {example.id} skipped processing despite its checkpoint '
+              f'has error: {checkpointed.error}, as '
+              '`reevaluate_upon_previous_errors` is set to False.'
+          )
+        else:
+          self.info(
+              f'Example {example.id} skipped processing as prior run '
+              'is available and error free.'
+          )
         example.start_time = checkpointed.start_time
         self._state.update(example, in_progress=True)
 
         # Use the output and metadata obtained from the previous processing.
         example.output = checkpointed.output
         example.metadata = checkpointed.metadata
+        example.error = checkpointed.error
         example.newly_processed = False
+        example.execution_status = checkpointed.execution_status
+        example.end_time = checkpointed.end_time
 
         # For previously processed examples, we merge previous usages as
         # cached, so the usage summary will account previous usages, but as

{langfun-0.1.2.dev202508050805 → langfun-0.1.2.dev202508070805}/langfun/core/eval/v2/evaluation_test.py

@@ -128,10 +128,17 @@ class EvaluationTest(unittest.TestCase):
       self.assertEqual(len(exp.state.evaluation_status), 1)
       f.add(pg.to_json_str(example))
 
+      example2 = exp.evaluate(6)
+      self.assertTrue(example2.newly_processed)
+      self.assertEqual(example2.input, pg.Dict(x=5, y=25, groundtruth=30))
+      self.assertTrue(example2.has_error)
+      self.assertEqual(len(exp.state.evaluation_status), 2)
+      f.add(pg.to_json_str(example2))
+
     exp.reset()
     self.assertEqual(len(exp.state.ckpt_examples), 0)
     exp.load_state(state_file)
-    self.assertEqual(len(exp.state.ckpt_examples), 1)
+    self.assertEqual(len(exp.state.ckpt_examples), 2)
     self.assertEqual(len(exp.state.evaluation_status), 0)
     example = exp.evaluate(3)
     self.assertFalse(example.newly_processed)
@@ -142,6 +149,19 @@ class EvaluationTest(unittest.TestCase):
     self.assertEqual(example.usage_summary.cached.total.num_requests, 1)
     self.assertEqual(example.usage_summary.uncached.total.total_tokens, 0)
     self.assertEqual(example.usage_summary.uncached.total.num_requests, 0)
+    self.assertIsNotNone(example.start_time)
+    self.assertIsNotNone(example.end_time)
+    self.assertIn('evaluate.process', example.execution_status)
+    self.assertIn('evaluate.metric', example.execution_status)
+
+    example2 = exp.evaluate(6, reevaluate_upon_previous_errors=False)
+    self.assertFalse(example2.newly_processed)
+    self.assertEqual(example2.input, pg.Dict(x=5, y=25, groundtruth=30))
+    self.assertTrue(example2.has_error)
+    self.assertIsNotNone(example2.start_time)
+    self.assertIsNotNone(example2.end_time)
+    self.assertIn('evaluate.process', example2.execution_status)
+    self.assertIn('evaluate.metric', example2.execution_status)
 
     # Test load_state with filter.
     exp.reset()

{langfun-0.1.2.dev202508050805 → langfun-0.1.2.dev202508070805}/langfun/core/eval/v2/experiment.py

@@ -384,6 +384,7 @@ class Experiment(lf.Component, pg.views.HtmlTreeView.Extension):
       example_ids: list[int] | None = None,
       shuffle_inputs: bool = False,
       raise_if_has_error: bool = False,
+      reevaluate_upon_previous_errors: bool = True,
       reprocess: bool | list[int] = False,
       generate_example_html: Literal['new', 'all', 'no'] | list[int] = 'new',
       process_timeout: int | None = None,
@@ -436,6 +437,9 @@ class Experiment(lf.Component, pg.views.HtmlTreeView.Extension):
         Neverthless, the example ID remains unchanged for each example.
       raise_if_has_error: If True, it will raise an error if any example fails.
         Otherwise, it will continue and report the error in the output.
+      reevaluate_upon_previous_errors: If True, it will reevaluate the example
+        if the previous checkpointed run has error when experiment runs with
+        existing checkpoints.
       reprocess: A boolean or a list of example IDs. If boolean, it indicates
         that whether all the examples to be evaluated will be reprocessed,
         meaning that existing checkpoints will be ignored. If a list of
@@ -475,6 +479,7 @@ class Experiment(lf.Component, pg.views.HtmlTreeView.Extension):
             example_ids=example_ids,
             shuffle_inputs=shuffle_inputs,
             raise_if_has_error=raise_if_has_error,
+            reevaluate_upon_previous_errors=reevaluate_upon_previous_errors,
             reprocess=reprocess,
             generate_example_html=generate_example_html,
             use_cache=use_cache,
@@ -835,6 +840,14 @@ class Run(pg.Object, pg.views.html.HtmlTreeView.Extension):
       )
   ] = False
 
+  reevaluate_upon_previous_errors: Annotated[
+      bool,
+      (
+          'If True, it will reevaluate the example if the previously '
+          'checkpointed run has error. Otherwise, it will skip the example.'
+      )
+  ] = True
+
   note: Annotated[
       str | None,
       'The user note for the current run.'

{langfun-0.1.2.dev202508050805 → langfun-0.1.2.dev202508070805}/langfun/core/eval/v2/runners.py

@@ -384,6 +384,7 @@ class RunnerBase(Runner):
     item = evaluation.evaluate(
         item,
         raise_if_has_error=self.current_run.raise_if_has_error,
+        reevaluate_upon_previous_errors=self.current_run.reevaluate_upon_previous_errors,
     )
     self.on_example_complete(evaluation, item)
     return item

{langfun-0.1.2.dev202508050805 → langfun-0.1.2.dev202508070805}/langfun/core/template.py

@@ -307,7 +307,7 @@ class Template(
                 else:
                   raise ValueError(
                       f'The value for template variable {var_name!r} is not '
-                      'provided.'
+                      f'provided. Template: {self.template_str!r}'
                   )
               inputs[var_name] = var_value
 

{langfun-0.1.2.dev202508050805 → langfun-0.1.2.dev202508070805}/langfun.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langfun
-Version: 0.1.2.dev202508050805
+Version: 0.1.2.dev202508070805
 Summary: Langfun: Language as Functions.
 Home-page: https://github.com/google/langfun
 Author: Langfun Authors