langfun 0.1.2.dev202512180805__tar.gz → 0.1.2.dev202601120805__tar.gz

{langfun-0.1.2.dev202512180805 → langfun-0.1.2.dev202601120805}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langfun
-Version: 0.1.2.dev202512180805
+Version: 0.1.2.dev202601120805
 Summary: Langfun: Language as Functions.
 Home-page: https://github.com/google/langfun
 Author: Langfun Authors

{langfun-0.1.2.dev202512180805 → langfun-0.1.2.dev202601120805}/langfun/core/eval/v2/checkpointing.py

@@ -122,7 +122,7 @@ class Checkpointer(experiment_lib.Plugin):
       example: Example,
   ) -> None:
     """Saves the example to the checkpoint file."""
-    if example.newly_processed:
+    if example.newly_processed or runner.current_run.force_recompute_metrics:
       self._save_example(runner, experiment, example)
 
   def _load_experiment(
@@ -173,9 +173,9 @@ class Checkpointer(experiment_lib.Plugin):
         output_ckpt_file = current_run.output_path_for(
             experiment, os.path.basename(ckpt_file)
         )
-        if ckpt_file != output_ckpt_file and any(
-            e for e in loaded_examples if not e.has_error
-        ):
+        if (not runner.current_run.force_recompute_metrics
+            and ckpt_file != output_ckpt_file
+            and any(e for e in loaded_examples if not e.has_error)):
           # Write the error-free warm-start examples to the output checkpoint
           # file.
           with SequenceWriter(output_ckpt_file) as writer:
@@ -373,7 +373,6 @@ class BulkCheckpointer(Checkpointer):
       )
       if pg.io.path_exists(input_ckpt_file):
         return [input_ckpt_file]
-    print('CCC', experiment.hash, [])
     return []
 
   def on_experiment_complete(

{langfun-0.1.2.dev202512180805 → langfun-0.1.2.dev202601120805}/langfun/core/eval/v2/experiment.py

@@ -376,23 +376,24 @@ class Experiment(lf.Component, pg.views.HtmlTreeView.Extension):
   def run(
       self,
       root_dir: str,
-      id: str | None = None,   # pylint: disable=redefined-builtin
+      id: str | None = None,  # pylint: disable=redefined-builtin
       *,
       runner: str = 'parallel',
       warm_start_from: str | None = None,
-      filter: Callable[['Experiment'], bool] | None = None,   # pylint: disable=redefined-builtin
+      filter: Callable[['Experiment'], bool] | None = None,  # pylint: disable=redefined-builtin
       example_ids: list[int] | None = None,
       shuffle_inputs: bool = False,
       raise_if_has_error: bool = False,
       reevaluate_upon_previous_errors: bool = True,
       reprocess: bool | list[int] = False,
+      force_recompute_metrics: bool = False,
       generate_example_html: Literal['new', 'all', 'no'] | list[int] = 'new',
       process_timeout: int | None = None,
       use_cache: Literal['global', 'per_dataset', 'no'] = 'per_dataset',
       note: str | None = None,
       tags: list[str] | None = None,
       plugins: list['Plugin'] | None = None,
-      **kwargs
+      **kwargs,
   ) -> 'Run':
     """Runs the experiment.
 
@@ -445,6 +446,8 @@ class Experiment(lf.Component, pg.views.HtmlTreeView.Extension):
         meaning that existing checkpoints will be ignored. If a list of
         example IDs, it indicates that only the specified examples will be
         reprocessed.
+      force_recompute_metrics: If True, it will recompute the metrics for all
+        examples, even if the previous checkpoints have metric metadata.
       generate_example_html: Among 'new', 'all', 'no' or a list of example IDs.
         If 'new', generate HTML files for all newly processed examples, and
           keep/copy existing HTML files for unchanged examples.
@@ -481,6 +484,7 @@ class Experiment(lf.Component, pg.views.HtmlTreeView.Extension):
             raise_if_has_error=raise_if_has_error,
             reevaluate_upon_previous_errors=reevaluate_upon_previous_errors,
             reprocess=reprocess,
+            force_recompute_metrics=force_recompute_metrics,
             generate_example_html=generate_example_html,
             use_cache=use_cache,
             process_timeout=process_timeout,
@@ -884,6 +888,14 @@ class Run(pg.Object, pg.views.html.HtmlTreeView.Extension):
       )
   ] = True
 
+  force_recompute_metrics: Annotated[
+      bool,
+      (
+          'If True, force recompute the metrics even if metric metadata is '
+          'already present from previous checkpoint.'
+      )
+  ] = False
+
   note: Annotated[
       str | None,
       'The user note for the current run.'
@@ -1003,7 +1015,7 @@ class Run(pg.Object, pg.views.html.HtmlTreeView.Extension):
     load_metadata_ids = set()
     if isinstance(self.generate_example_html, list):
       load_metadata_ids = set(self.generate_example_html)
-    elif self.generate_example_html == 'all':
+    elif self.generate_example_html == 'all' or self.force_recompute_metrics:
       load_metadata_ids = self.examples_to_evaluate(experiment)
     load_metadata_ids -= self.examples_to_reprocess(experiment)
     return load_metadata_ids

{langfun-0.1.2.dev202512180805 → langfun-0.1.2.dev202601120805}/langfun/core/eval/v2/progress.py

@@ -227,7 +227,11 @@ class Progress(pg.Object, pg.views.HtmlTreeView.Extension):
 
   def merge_from(self, other: 'Progress') -> None:
     """Merges the progress from another progress."""
-    with pg.notify_on_change(False), pg.allow_writable_accessors(True):
+    with (
+        self._lock,
+        pg.notify_on_change(False),
+        pg.allow_writable_accessors(True)
+    ):
       if other.start_time is not None and (
           self.start_time is None or self.start_time > other.start_time):
         self.start_time = other.start_time
@@ -268,16 +272,17 @@ class Progress(pg.Object, pg.views.HtmlTreeView.Extension):
         stop_time=self.stop_time_str,
     )
     if self.execution_summary:
-      time_info['execution'] = pg.Dict(
-          {
-              k: pg.Dict(
-                  num_started=v.num_started,
-                  num_ended=v.num_ended,
-                  num_failed=v.num_failed,
-                  avg_duration=round(v.avg_duration, 2),
-              ) for k, v in self.execution_summary.breakdown.items()
-          }
-      )
+      with self._lock:
+        time_info['execution'] = pg.Dict(
+            {
+                k: pg.Dict(
+                    num_started=v.num_started,
+                    num_ended=v.num_ended,
+                    num_failed=v.num_failed,
+                    avg_duration=round(v.avg_duration, 2),
+                ) for k, v in self.execution_summary.breakdown.items()
+            }
+        )
     return pg.format(time_info, verbose=False)
 
   def _html_tree_view(

{langfun-0.1.2.dev202512180805 → langfun-0.1.2.dev202601120805}/langfun/core/eval/v2/reporting.py

@@ -99,6 +99,7 @@ class ExampleHtmlGenerator(experiment_lib.Plugin):
 
     generate_example_html = current_run.generate_example_html
     if (generate_example_html == 'all'
+        or runner.current_run.force_recompute_metrics
         or (generate_example_html == 'new' and example.newly_processed)
         or (isinstance(generate_example_html, list)
             and example.id in generate_example_html)):

{langfun-0.1.2.dev202512180805 → langfun-0.1.2.dev202601120805}/langfun/core/eval/v2/runners/base.py

@@ -396,6 +396,7 @@ class RunnerBase(Runner):
         item,
         raise_if_has_error=self.current_run.raise_if_has_error,
         reevaluate_upon_previous_errors=self.current_run.reevaluate_upon_previous_errors,
+        force_recompute_metrics=self.current_run.force_recompute_metrics,
     )
     self.on_example_complete(evaluation, item)
     return item

{langfun-0.1.2.dev202512180805 → langfun-0.1.2.dev202601120805}/langfun.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langfun
-Version: 0.1.2.dev202512180805
+Version: 0.1.2.dev202601120805
 Summary: Langfun: Language as Functions.
 Home-page: https://github.com/google/langfun
 Author: Langfun Authors