langfun 0.1.2.dev202512040805__py3-none-any.whl → 0.1.2.dev202601030804__py3-none-any.whl

langfun/core/agentic/action.py

@@ -288,6 +288,8 @@ class Action(pg.Object):
 
     with session.track_action(self, max_execution_time=max_execution_time):
       try:
+        # Early terminate the action if the execution time is exceeded.
+        session.check_execution_time()
         result = self.call(session=session, **kwargs)
         self._invocation.end(result)
       except BaseException as e:
@@ -2005,9 +2007,6 @@ class Session(pg.Object, pg.views.html.HtmlTreeView.Extension):
           'signal the start and end of the session.'
       )
 
-    # Early terminate the action if the execution time is exceeded.
-    self.check_execution_time()
-
     invocation = ActionInvocation(
         pg.maybe_ref(action),
         max_execution_time=self._child_max_execution_time(max_execution_time)

langfun/core/eval/v2/__init__.py

@@ -41,6 +41,7 @@ from langfun.core.eval.v2.checkpointing import PerExampleCheckpointer
 from langfun.core.eval.v2.reporting import HtmlReporter
 from langfun.core.eval.v2.reporting import ExampleHtmlGenerator
 
+# Google-internal imports.
 
 # pylint: enable=g-bad-import-order
 # pylint: enable=g-importing-member

langfun/core/eval/v2/checkpointing.py

@@ -38,7 +38,7 @@ class Checkpointer(experiment_lib.Plugin):
   later. When an experiment starts, the checkpointer loads any previously saved
   examples from an earlier run (or a warm-start run) into `experiment.state`,
   so the runner can skip processing them again.
-  Subclasses should implement `_list_checkpoint_filenames` to identify
+  Subclasses should implement `_list_checkpoint_files` to identify
   checkpoint files to load, and `_save_example` to save a newly processed
   example.
   """
@@ -122,7 +122,7 @@ class Checkpointer(experiment_lib.Plugin):
       example: Example,
   ) -> None:
     """Saves the example to the checkpoint file."""
-    if example.newly_processed:
+    if example.newly_processed or runner.current_run.force_recompute_metrics:
       self._save_example(runner, experiment, example)
 
   def _load_experiment(
@@ -131,7 +131,7 @@ class Checkpointer(experiment_lib.Plugin):
       experiment: Experiment,
   ) -> None:
     """Creates the checkpoint file."""
-    ckpt_files = self._list_checkpoint_filenames(runner, experiment)
+    ckpt_files = self._list_checkpoint_files(runner, experiment)
     experiment.info(f'Found {len(ckpt_files)} checkpoint files to load.')
 
     # Load the checkpoint files in parallel.
@@ -141,18 +141,18 @@ class Checkpointer(experiment_lib.Plugin):
         experiment
     )
     context = dict(counter=0, counter_lock=threading.Lock())
-    copy_ckpt = current_run.input_root != current_run.output_root
 
     def _load_state(ckpt_file):
       error = None
       with pg.timeit() as t:
         try:
-          experiment.load_state(
-              current_run.input_path_for(experiment, ckpt_file),
+          loaded_examples = experiment.load_state(
+              ckpt_file,
               filter=lambda x: x.id in examples_to_load,
               load_example_metadata=lambda x: x.id in examples_to_load_metadata,
           )
         except BaseException as e:  # pylint: disable=broad-except
+          loaded_examples = []
           error = e
         finally:
           with context['counter_lock']:
@@ -170,22 +170,18 @@ class Checkpointer(experiment_lib.Plugin):
                 f'Skipping the file. ({progress_str})'
             )
 
-        if not copy_ckpt:
-          return
-
-        # Copy the checkpoint records to the output directory.
-        try:
-          with pg.io.open_sequence(
-              current_run.output_path_for(experiment, ckpt_file), 'w'
-          ) as o, pg.io.open_sequence(
-              current_run.input_path_for(experiment, ckpt_file), 'r'
-          ) as i:
-            for x in i:
-              o.add(x)
-        except BaseException as e:  # pylint: disable=broad-except
-          experiment.warning(
-              f'Failed to copy checkpoint {ckpt_file!r}: {e}.'
-          )
+        output_ckpt_file = current_run.output_path_for(
+            experiment, os.path.basename(ckpt_file)
+        )
+        if (not runner.current_run.force_recompute_metrics
+            and ckpt_file != output_ckpt_file
+            and any(e for e in loaded_examples if not e.has_error)):
+          # Write the error-free warm-start examples to the output checkpoint
+          # file.
+          with SequenceWriter(output_ckpt_file) as writer:
+            for example in loaded_examples:
+              if not example.has_error:
+                writer.add(example)
 
     _ = list(
         lf.concurrent_map(
@@ -197,10 +193,10 @@ class Checkpointer(experiment_lib.Plugin):
     )
 
   @abc.abstractmethod
-  def _list_checkpoint_filenames(
+  def _list_checkpoint_files(
       self, runner: Runner, experiment: Experiment
   ) -> list[str]:
-    """Lists the checkpoint filenames to restore."""
+    """Lists the checkpoint file paths to restore."""
 
   @abc.abstractmethod
   def _save_example(
@@ -226,22 +222,41 @@ class PerExampleCheckpointer(Checkpointer):
     self._checkpoint_file_prefix = prefix
     self._checkpoint_file_ext = ext
 
-  def _list_checkpoint_filenames(
+  def _list_checkpoint_files(
       self, runner: Runner, experiment: Experiment
   ) -> list[str]:
-    experiment_dir = runner.current_run.input_dir(experiment)
-    filenames = []
+
+    def _list_checkpoints_from(ckpt_dir: str, examples_to_load: set[int]):
+      ckpt_files = []
+      if pg.io.path_exists(ckpt_dir):
+        regex = re.compile(
+            f'{self._checkpoint_file_prefix}_(\\d+){self._checkpoint_file_ext}'
+            .replace('.', '\\.')
+        )
+        for filename in pg.io.listdir(ckpt_dir):
+          match = regex.match(filename)
+          if match and int(match.group(1)) in examples_to_load:
+            examples_to_load.remove(int(match.group(1)))
+            ckpt_files.append(os.path.join(ckpt_dir, filename))
+      return ckpt_files
+
     examples_to_load = runner.current_run.examples_to_load(experiment)
-    if pg.io.path_exists(experiment_dir):
-      regex = re.compile(
-          f'{self._checkpoint_file_prefix}_(\\d+){self._checkpoint_file_ext}'
-          .replace('.', '\\.')
+
+    # Take output directory as the first priority to checkpoints processed in
+    # this run.
+    ckpt_files = _list_checkpoints_from(
+        runner.current_run.output_dir(experiment), examples_to_load
+    )
+    # If the input and output directories are different, also load from the
+    # input directory.
+    if (examples_to_load
+        and runner.current_run.input_root != runner.current_run.output_root):
+      ckpt_files.extend(
+          _list_checkpoints_from(
+              runner.current_run.input_dir(experiment), examples_to_load
+          )
       )
-      for filename in pg.io.listdir(experiment_dir):
-        match = regex.match(filename)
-        if match and int(match.group(1)) in examples_to_load:
-          filenames.append(filename)
-    return filenames
+    return ckpt_files
 
   def _save_example(
       self,
@@ -341,13 +356,23 @@ class BulkCheckpointer(Checkpointer):
         if self._sequence_writer is not None:
           self._sequence_writer[experiment.id] = sequence_writer
 
-  def _list_checkpoint_filenames(
+  def _list_checkpoint_files(
       self, runner: Runner, experiment: Experiment
   ) -> list[str]:
-    if pg.io.path_exists(
-        runner.current_run.input_path_for(experiment, self.checkpoint_filename)
-    ):
-      return [self.checkpoint_filename]
+    # Always honor the output directory if it's present, as it contains both
+    # the warm-started examples and newly processed examples.
+    output_ckpt_file = runner.current_run.output_path_for(
+        experiment, self.checkpoint_filename
+    )
+    if pg.io.path_exists(output_ckpt_file):
+      return [output_ckpt_file]
+
+    if runner.current_run.input_root != runner.current_run.output_root:
+      input_ckpt_file = runner.current_run.input_path_for(
+          experiment, self.checkpoint_filename
+      )
+      if pg.io.path_exists(input_ckpt_file):
+        return [input_ckpt_file]
     return []
 
   def on_experiment_complete(
@@ -441,5 +466,12 @@ class SequenceWriter:
       self._sequence_writer = None
       pg.io.rename(self._tmp_path, self._path)
 
+  def __enter__(self):
+    return self
+
+  def __exit__(self, *args, **kwargs):
+    del args, kwargs
+    self.close()
+
   def __del__(self):
     self.close()

langfun/core/eval/v2/evaluation.py

@@ -114,6 +114,13 @@ class Evaluation(experiment_lib.Experiment):
     self._log_entries = []
     self._log_lock = threading.Lock()
 
+  def _identity(self) -> str:
+    """Returns the definition of the evaluation."""
+    return self.format(
+        compact=True, hide_default_values=True, use_inferred=True,
+        exclude_keys=('plugins', 'progress', 'usage_summary')
+    )
+
   #
   # Handling evaluation hierarchy (materialized vs. hyper evaluations).
   #
@@ -379,10 +386,10 @@ class Evaluation(experiment_lib.Experiment):
       load_example_metadata: bool = True,
       filter: Callable[[example_lib.Example], bool] | None = None,  # pylint: disable=redefined-builtin
       raise_if_not_exist: bool = False
-  ) -> None:
+  ) -> list[example_lib.Example]:
     """Loads saved state from a sequence IO file."""
     if pg.io.path_exists(state_file):
-      self._state.load(
+      return self._state.load(
           state_file,
           example_input_by_id=self.example_input_by_id,
           load_example_metadata=load_example_metadata,
@@ -390,6 +397,7 @@ class Evaluation(experiment_lib.Experiment):
       )
     elif raise_if_not_exist:
       raise ValueError(f'State file {state_file} does not exist.')
+    return []
 
   def _reset(self) -> None:
     """Resets the state of the evaluation."""

langfun/core/eval/v2/experiment.py

@@ -268,11 +268,11 @@ class Experiment(lf.Component, pg.views.HtmlTreeView.Extension):
   @functools.cached_property
   def hash(self) -> str:
     """A 8-byte MD5 hash computed from experiment identity."""
-    identity = self.format(
-        compact=True, hide_default_values=True, use_inferred=True,
-        exclude_keys=('plugins', 'progress', 'usage_summary')
-    )
-    return hashlib.md5(identity.encode()).hexdigest()[:8]
+    return hashlib.md5(self._identity().encode()).hexdigest()[:8]
+
+  @abc.abstractmethod
+  def _identity(self) -> str:
+    """Returns the identity of the experiment."""
 
   @classmethod
   def link(cls, path: str) -> str:
@@ -376,23 +376,24 @@ class Experiment(lf.Component, pg.views.HtmlTreeView.Extension):
   def run(
       self,
       root_dir: str,
-      id: str | None = None,   # pylint: disable=redefined-builtin
+      id: str | None = None,  # pylint: disable=redefined-builtin
       *,
       runner: str = 'parallel',
       warm_start_from: str | None = None,
-      filter: Callable[['Experiment'], bool] | None = None,   # pylint: disable=redefined-builtin
+      filter: Callable[['Experiment'], bool] | None = None,  # pylint: disable=redefined-builtin
       example_ids: list[int] | None = None,
       shuffle_inputs: bool = False,
       raise_if_has_error: bool = False,
       reevaluate_upon_previous_errors: bool = True,
       reprocess: bool | list[int] = False,
+      force_recompute_metrics: bool = False,
       generate_example_html: Literal['new', 'all', 'no'] | list[int] = 'new',
       process_timeout: int | None = None,
       use_cache: Literal['global', 'per_dataset', 'no'] = 'per_dataset',
       note: str | None = None,
       tags: list[str] | None = None,
       plugins: list['Plugin'] | None = None,
-      **kwargs
+      **kwargs,
   ) -> 'Run':
     """Runs the experiment.
 
@@ -445,6 +446,8 @@ class Experiment(lf.Component, pg.views.HtmlTreeView.Extension):
         meaning that existing checkpoints will be ignored. If a list of
         example IDs, it indicates that only the specified examples will be
         reprocessed.
+      force_recompute_metrics: If True, it will recompute the metrics for all
+        examples, even if the previous checkpoints have metric metadata.
       generate_example_html: Among 'new', 'all', 'no' or a list of example IDs.
         If 'new', generate HTML files for all newly processed examples, and
           keep/copy existing HTML files for unchanged examples.
@@ -481,6 +484,7 @@ class Experiment(lf.Component, pg.views.HtmlTreeView.Extension):
             raise_if_has_error=raise_if_has_error,
             reevaluate_upon_previous_errors=reevaluate_upon_previous_errors,
             reprocess=reprocess,
+            force_recompute_metrics=force_recompute_metrics,
             generate_example_html=generate_example_html,
             use_cache=use_cache,
             process_timeout=process_timeout,
@@ -691,6 +695,12 @@ class Suite(Experiment):
     """Returns whether the task is a leaf."""
     return False
 
+  def _identity(self) -> str:
+    """Returns the definition of the evaluation."""
+    return '[' + ', '.join(
+        [child._identity() for child in self.children]  # pylint: disable=protected-access
+    ) + ']'
+
 
 class RunId(pg.Object):
   """Structured repreesentation a experiment run ID."""
@@ -878,6 +888,14 @@ class Run(pg.Object, pg.views.html.HtmlTreeView.Extension):
       )
   ] = True
 
+  force_recompute_metrics: Annotated[
+      bool,
+      (
+          'If True, force recompute the metrics even if metric metadata is '
+          'already present from previous checkpoint.'
+      )
+  ] = False
+
   note: Annotated[
       str | None,
       'The user note for the current run.'
@@ -997,7 +1015,7 @@ class Run(pg.Object, pg.views.html.HtmlTreeView.Extension):
     load_metadata_ids = set()
     if isinstance(self.generate_example_html, list):
       load_metadata_ids = set(self.generate_example_html)
-    elif self.generate_example_html == 'all':
+    elif self.generate_example_html == 'all' or self.force_recompute_metrics:
       load_metadata_ids = self.examples_to_evaluate(experiment)
     load_metadata_ids -= self.examples_to_reprocess(experiment)
     return load_metadata_ids

langfun/core/eval/v2/progress.py

@@ -227,7 +227,11 @@ class Progress(pg.Object, pg.views.HtmlTreeView.Extension):
 
   def merge_from(self, other: 'Progress') -> None:
     """Merges the progress from another progress."""
-    with pg.notify_on_change(False), pg.allow_writable_accessors(True):
+    with (
+        self._lock,
+        pg.notify_on_change(False),
+        pg.allow_writable_accessors(True)
+    ):
       if other.start_time is not None and (
           self.start_time is None or self.start_time > other.start_time):
         self.start_time = other.start_time
@@ -268,16 +272,17 @@ class Progress(pg.Object, pg.views.HtmlTreeView.Extension):
         stop_time=self.stop_time_str,
     )
     if self.execution_summary:
-      time_info['execution'] = pg.Dict(
-          {
-              k: pg.Dict(
-                  num_started=v.num_started,
-                  num_ended=v.num_ended,
-                  num_failed=v.num_failed,
-                  avg_duration=round(v.avg_duration, 2),
-              ) for k, v in self.execution_summary.breakdown.items()
-          }
-      )
+      with self._lock:
+        time_info['execution'] = pg.Dict(
+            {
+                k: pg.Dict(
+                    num_started=v.num_started,
+                    num_ended=v.num_ended,
+                    num_failed=v.num_failed,
+                    avg_duration=round(v.avg_duration, 2),
+                ) for k, v in self.execution_summary.breakdown.items()
+            }
+        )
     return pg.format(time_info, verbose=False)
 
   def _html_tree_view(

langfun/core/eval/v2/reporting.py

@@ -86,10 +86,8 @@ class ExampleHtmlGenerator(experiment_lib.Plugin):
         return
 
       try:
-        with pg.timeit() as t, pg.io.open(src_file, 'r') as src:
-          content = src.read()
-          with pg.io.open(dest_file, 'w') as dest:
-            dest.write(content)
+        with pg.timeit() as t:
+          pg.io.copy(src_file, dest_file)
         experiment.info(
             f'\'{example.id}.html\' copied in {t.elapse:.2f} seconds.'
         )
@@ -101,6 +99,7 @@ class ExampleHtmlGenerator(experiment_lib.Plugin):
 
     generate_example_html = current_run.generate_example_html
     if (generate_example_html == 'all'
+        or runner.current_run.force_recompute_metrics
         or (generate_example_html == 'new' and example.newly_processed)
         or (isinstance(generate_example_html, list)
             and example.id in generate_example_html)):

langfun/core/eval/v2/runners/base.py

@@ -139,6 +139,7 @@ class RunnerBase(Runner):
           self.current_run.examples_to_evaluate(experiment)
       )
       experiment.progress.start(total=num_examples_to_evaluate)
+      pg.io.mkdirs(self.current_run.output_dir(experiment))
     else:
       experiment.progress.start(total=len(experiment.leaf_nodes))
 
@@ -395,6 +396,7 @@ class RunnerBase(Runner):
         item,
         raise_if_has_error=self.current_run.raise_if_has_error,
         reevaluate_upon_previous_errors=self.current_run.reevaluate_upon_previous_errors,
+        force_recompute_metrics=self.current_run.force_recompute_metrics,
     )
     self.on_example_complete(evaluation, item)
     return item

langfun/core/eval/v2/runners/ckpt_monitor.py

@@ -60,6 +60,19 @@ class CheckpointMonitor(base.RunnerBase):
       'The maximum number of threads to aggregate checkpoints.'
   ] = 128
 
+  bypass_old_ckpt_files_with_non_oop_errors: Annotated[
+      bool,
+      'If True, ignore old checkpoint files with non-oop errors.'
+  ] = True
+
+  ckpt_start_time: Annotated[
+      float | None,
+      (
+          'The timestamp to treat checkpoint files modified before this '
+          'time as old.'
+      )
+  ] = None
+
   @dataclasses.dataclass
   class _AggregationEntry:
     evaluation: evaluation_lib.Evaluation
@@ -78,6 +91,9 @@ class CheckpointMonitor(base.RunnerBase):
     self._aggregation_entries = []
     self._aggregator_pool = None
     self._error = None
+    if self.ckpt_start_time is None:
+      self.rebind(ckpt_start_time=time.time(), skip_notification=True)
+    self._ckpt_bypass_timestamp: dict[str, int] = {}
 
   def start(self):
     # Reset the experiment state before getting started.
@@ -165,6 +181,14 @@ class CheckpointMonitor(base.RunnerBase):
               os.path.basename(filepath).split('.')[0].split('_')[-1]
           )
           if example_id in entry.example_ids_to_be_aggregated:
+            last_modified_time = pg.io.getmtime(filepath)
+            bypass_timestamp = self._ckpt_bypass_timestamp.get(filepath)
+            if (
+                bypass_timestamp is not None
+                and last_modified_time <= bypass_timestamp
+            ):
+              continue
+
             # Remove example ID from the set to avoid duplicate processing.
             entry.example_ids_to_be_aggregated.remove(example_id)
             entry.example_ids_being_aggregated.add(example_id)
@@ -177,7 +201,7 @@ class CheckpointMonitor(base.RunnerBase):
               entry.example_ids_inprogress.add(example_id)
 
             self._aggregator_pool.submit(
-                self._aggregate, entry, filepath, example_id
+                self._aggregate, entry, filepath, example_id, last_modified_time
             )
             pg.logging.info(
                 '[%s] Aggregating example %d from %s...',
@@ -196,7 +220,8 @@ class CheckpointMonitor(base.RunnerBase):
       self,
       entry: _AggregationEntry,
       ckpt_filepath: str,
-      example_id: int
+      example_id: int,
+      last_modified_time: float,
   ):
     """Aggregate an example from a checkpoint file."""
     try:
@@ -212,6 +237,25 @@ class CheckpointMonitor(base.RunnerBase):
       # example processed multiple times. We only need to aggregate the last
       # example.
       example = loaded_examples[-1]
+      if (
+          self.bypass_old_ckpt_files_with_non_oop_errors
+          and last_modified_time < self.ckpt_start_time
+          and example.error is not None
+          and not example.error.tag.startswith('MappingError')
+      ):
+        entry.example_ids_being_aggregated.remove(example_id)
+        entry.example_ids_to_be_aggregated.add(example_id)
+        self._ckpt_bypass_timestamp[ckpt_filepath] = last_modified_time
+        pg.logging.info(
+            '[%s] Bypassing old checkpoint file with non-oop errors (%s) '
+            'for example %d, last_modified_time: %s, ckpt_start_time: %s',
+            entry.evaluation.id,
+            ckpt_filepath,
+            example_id,
+            last_modified_time,
+            self.ckpt_start_time,
+        )
+        return
     except BaseException as e:  # pylint: disable=broad-except
       error_info = pg.ErrorInfo.from_exception(e)
       pg.logging.error(
@@ -229,9 +273,21 @@ class CheckpointMonitor(base.RunnerBase):
     # This will skip processing but still allow metrics to be collected.
     # `process` will never be called for evaluation, thus we do not
     # need to setup/teardown evaluation.
-    example = entry.evaluation.evaluate(
-        example, reevaluate_upon_previous_errors=False
-    )
+    try:
+      example = entry.evaluation.evaluate(
+          example, reevaluate_upon_previous_errors=False
+      )
+    except BaseException as e:  # pylint: disable=broad-except
+      pg.logging.error(
+          '[%s] Unexpected error found during evaluating example %d from %s.',
+          entry.evaluation.id,
+          example_id,
+          ckpt_filepath,
+      )
+      self._error = e
+      entry.example_ids_being_aggregated.remove(example_id)
+      return
+
     example.newly_processed = True
     pg.logging.info(
         '[%s] Successfully aggregated example %d from %s.',

langfun/core/eval/v2/runners/ckpt_monitor_test.py

@@ -13,8 +13,10 @@
 # limitations under the License.
 import os
 import tempfile
+import time
 import unittest
 
+import langfun.core as lf
 from langfun.core.eval.v2 import checkpointing
 from langfun.core.eval.v2 import eval_test_helper
 from langfun.core.eval.v2 import example as example_lib
@@ -33,6 +35,7 @@ class CheckpointMonitorTest(unittest.TestCase):
   def test_aggregate(self):
     exp = eval_test_helper.test_experiment()
     root_dir = os.path.join(self.test_dir, 'test_aggregate')
+    ckpt_start_time = time.time()
     run = exp.run(
         root_dir,
         runner='sequential',
@@ -55,6 +58,7 @@ class CheckpointMonitorTest(unittest.TestCase):
         plugins=[plugin],
         checkpoint_pattern='checkpoint_*.jsonl',
         monitor_inprogress_files=True,
+        ckpt_start_time=ckpt_start_time,
     )
     monitor.run()
 
@@ -70,7 +74,50 @@ class CheckpointMonitorTest(unittest.TestCase):
     for e in exp.leaf_nodes:
       self.assertEqual(e.progress.num_completed, 10)
 
+  def test_ignore_old_ckpt_files_with_non_oop_errors(self):
+    exp = eval_test_helper.test_evaluation()
+    root_dir = os.path.join(self.test_dir, 'test_ignore_old_ckpt_files')
+    run = exp.run(
+        root_dir,
+        runner='sequential',
+        progress_tracker=None,
+        plugins=[
+            checkpointing.PerExampleCheckpointer(
+                checkpoint_filename='checkpoint.jsonl'
+            )
+        ],
+        use_cache='no',
+    )
+    monitor = ckpt_monitor.CheckpointMonitor(
+        run,
+        plugins=[],
+        checkpoint_pattern='checkpoint_*.jsonl',
+        monitor_inprogress_files=True
+    )
+    monitor.start()
+    time.sleep(2)
+    # Example 6 is a non-oop error, we simulate a re-evaluation.
+    ex = example_lib.Example(
+        id=6, output=1, metric_metadata={'match': {'is_correct': True}},
+        start_time=time.time() - 2, end_time=time.time(),
+        usage_summary=lf.UsageSummary(),
+        execution_status={
+            'evaluate': pg.utils.TimeIt.Status(name='evaluate', elapse=1)
+        }
+    )
+    with pg.io.open_sequence(
+        run.output_path_for(exp, 'checkpoint_6.jsonl'),
+        mode='w'
+    ) as f:
+      f.add(pg.to_json_str(ex))
+    print(time.time(), pg.io.listdir(run.output_dir(exp)))
+    monitor.join()
+    self.assertEqual(exp.progress.num_processed, 10)
+    self.assertEqual(exp.progress.num_completed, 10)
+    self.assertEqual(exp.progress.num_failed, 0)
+
   def test_aggregate_with_filter(self):
+    ckpt_start_time = time.time()
     exp = eval_test_helper.test_experiment()
     root_dir = os.path.join(self.test_dir, 'test_aggregate_with_filter')
 
@@ -93,6 +140,7 @@ class CheckpointMonitorTest(unittest.TestCase):
         run,
         plugins=[plugin],
         checkpoint_pattern='checkpoint_*.jsonl',
+        ckpt_start_time=ckpt_start_time,
     )
     monitor.run()
 
@@ -127,6 +175,7 @@ class CheckpointMonitorTest(unittest.TestCase):
         if self.simulate_raise_on_experiment_complete:
           raise ValueError('experiment complete error')
 
+    ckpt_start_time = time.time()
     exp = eval_test_helper.test_evaluation()
     root_dir = os.path.join(self.test_dir, 'test_plugin_raise')
 
@@ -148,6 +197,7 @@ class CheckpointMonitorTest(unittest.TestCase):
           run,
           plugins=[TestPlugin(simulate_raise_on_example_complete=True)],
           checkpoint_pattern='checkpoint_*.jsonl',
+          ckpt_start_time=ckpt_start_time,
       ).run()
 
     with self.assertRaisesRegex(ValueError, 'experiment complete error'):
@@ -155,6 +205,7 @@ class CheckpointMonitorTest(unittest.TestCase):
           run,
           plugins=[TestPlugin(simulate_raise_on_experiment_complete=True)],
           checkpoint_pattern='checkpoint_*.jsonl',
+          ckpt_start_time=ckpt_start_time,
       ).run()
 
 

langfun/core/llms/__init__.py

@@ -43,6 +43,7 @@ from langfun.core.llms.azure_openai import AzureOpenAI
 # Gemini models.
 from langfun.core.llms.google_genai import GenAI
 from langfun.core.llms.google_genai import Gemini3ProPreview
+from langfun.core.llms.google_genai import Gemini3FlashPreview
 from langfun.core.llms.google_genai import Gemini25Pro
 from langfun.core.llms.google_genai import Gemini25Flash
 from langfun.core.llms.google_genai import Gemini25ProPreview_20250605
@@ -65,6 +66,7 @@ from langfun.core.llms.google_genai import Gemini2ProExp_20250205
 from langfun.core.llms.google_genai import Gemini2FlashThinkingExp_20250121
 from langfun.core.llms.google_genai import GeminiExp_20241206
 from langfun.core.llms.google_genai import Gemini25FlashImagePreview
+from langfun.core.llms.google_genai import Gemini3ProImagePreview
 
 from langfun.core.llms.vertexai import VertexAIGemini
 from langfun.core.llms.vertexai import VertexAIGemini2Flash
@@ -92,6 +94,8 @@ from langfun.core.llms.vertexai import VertexAIGemini25Pro
 from langfun.core.llms.vertexai import VertexAIGemini25Flash
 from langfun.core.llms.vertexai import VertexAIGemini25FlashImagePreview
 from langfun.core.llms.vertexai import VertexAIGemini3ProPreview
+from langfun.core.llms.vertexai import VertexAIGemini3ProImagePreview
+from langfun.core.llms.vertexai import VertexAIGemini3FlashPreview
 
 # For backward compatibility.
 GeminiPro1_5 = Gemini15Pro
@@ -158,6 +162,7 @@ from langfun.core.llms.openai import Gpt35
 from langfun.core.llms.anthropic import Claude45
 from langfun.core.llms.anthropic import Claude45Haiku_20251001
 from langfun.core.llms.anthropic import Claude45Sonnet_20250929
+from langfun.core.llms.anthropic import Claude45Opus_20251101
 from langfun.core.llms.anthropic import Claude4
 from langfun.core.llms.anthropic import Claude4Sonnet_20250514
 from langfun.core.llms.anthropic import Claude4Opus_20250514
@@ -177,6 +182,7 @@ from langfun.core.llms.anthropic import Claude3Haiku_20240307
 from langfun.core.llms.vertexai import VertexAIAnthropic
 from langfun.core.llms.vertexai import VertexAIClaude45Haiku_20251001
 from langfun.core.llms.vertexai import VertexAIClaude45Sonnet_20250929
+from langfun.core.llms.vertexai import VertexAIClaude45Opus_20251101
 from langfun.core.llms.vertexai import VertexAIClaude4Opus_20250514
 from langfun.core.llms.vertexai import VertexAIClaude4Sonnet_20250514
 from langfun.core.llms.vertexai import VertexAIClaude37Sonnet_20250219

langfun/core/llms/anthropic.py

@@ -113,6 +113,32 @@ SUPPORTED_MODELS = [
             max_output_tokens_per_minute=400_000,
         ),
     ),
+    AnthropicModelInfo(
+        model_id='claude-opus-4-5-20251101',
+        provider='Anthropic',
+        in_service=True,
+        description='Claude 4.5 Opus model (11/01/2025).',
+        release_date=datetime.datetime(2025, 11, 1),
+        input_modalities=(
+            AnthropicModelInfo.INPUT_IMAGE_TYPES
+            + AnthropicModelInfo.INPUT_DOC_TYPES
+        ),
+        context_length=lf.ModelInfo.ContextLength(
+            max_input_tokens=200_000,
+            max_output_tokens=64_000,
+        ),
+        pricing=lf.ModelInfo.Pricing(
+            cost_per_1m_cached_input_tokens=0.5,
+            cost_per_1m_input_tokens=5,
+            cost_per_1m_output_tokens=25,
+        ),
+        rate_limits=AnthropicModelInfo.RateLimits(
+            # Tier 4 rate limits
+            max_requests_per_minute=2000,
+            max_input_tokens_per_minute=1_000_000,
+            max_output_tokens_per_minute=400_000,
+        ),
+    ),
     AnthropicModelInfo(
         model_id='claude-4-opus-20250514',
         provider='Anthropic',
@@ -300,6 +326,32 @@ SUPPORTED_MODELS = [
             max_output_tokens_per_minute=0,
         ),
     ),
+    AnthropicModelInfo(
+        model_id='claude-opus-4-5@20251101',
+        alias_for='claude-opus-4-5-20251101',
+        provider='VertexAI',
+        in_service=True,
+        description='Claude 4.5 Opus model served on VertexAI (11/01/2025).',
+        release_date=datetime.datetime(2025, 11, 1),
+        input_modalities=(
+            AnthropicModelInfo.INPUT_IMAGE_TYPES
+            + AnthropicModelInfo.INPUT_DOC_TYPES
+        ),
+        context_length=lf.ModelInfo.ContextLength(
+            max_input_tokens=200_000,
+            max_output_tokens=64_000,
+        ),
+        pricing=lf.ModelInfo.Pricing(
+            cost_per_1m_cached_input_tokens=0.5,
+            cost_per_1m_input_tokens=5,
+            cost_per_1m_output_tokens=25,
+        ),
+        rate_limits=AnthropicModelInfo.RateLimits(
+            max_requests_per_minute=100,
+            max_input_tokens_per_minute=1_000_000,
+            max_output_tokens_per_minute=80_000,
+        ),
+    ),
     AnthropicModelInfo(
         model_id='claude-opus-4@20250514',
         alias_for='claude-opus-4-20250514',
@@ -834,6 +886,13 @@ class Claude45Sonnet_20250929(Claude45):
   model = 'claude-sonnet-4-5-20250929'
 
 
+# pylint: disable=invalid-name
+class Claude45Opus_20251101(Claude45):
+  """Claude 4.5 Opus model 20251101."""
+
+  model = 'claude-opus-4-5-20251101'
+
+
 class Claude4(Anthropic):
   """Base class for Claude 4 models."""
 

langfun/core/llms/gemini.py

@@ -177,6 +177,55 @@ SUPPORTED_MODELS = [
             max_tokens_per_minute=4_000_000,
         ),
     ),
+    # Gemini 3 Pro Image Preview
+    GeminiModelInfo(
+        model_id='gemini-3-pro-image-preview',
+        in_service=True,
+        experimental=True,
+        provider=pg.oneof(['Google GenAI', 'VertexAI']),
+        model_type='instruction-tuned',
+        description=(
+            'Gemini 3 Pro Image Preview for high-fidelity image generation,'
+            ' editing, and visual reasoning.'
+        ),
+        release_date=datetime.datetime(2025, 12, 9),
+        input_modalities=GeminiModelInfo.INPUT_IMAGE_TYPES
+        + GeminiModelInfo.INPUT_DOC_TYPES,
+        context_length=lf.ModelInfo.ContextLength(
+            max_input_tokens=65_536,
+            max_output_tokens=32_768,
+        ),
+        rate_limits=lf.ModelInfo.RateLimits(
+            max_requests_per_minute=200,
+            max_tokens_per_minute=1_000_000,
+        ),
+    ),
+    # Gemini 3 Flash Preview
+    GeminiModelInfo(
+        model_id='gemini-3-flash-preview',
+        in_service=True,
+        provider=pg.oneof(['Google GenAI', 'VertexAI']),
+        model_type='instruction-tuned',
+        description=(
+            'Gemini 3 Flash Preview: High-efficiency, low-latency multimodal'
+            ' model optimized for agentic workflows.'
+        ),
+        release_date=datetime.datetime(2025, 12, 17),
+        input_modalities=GeminiModelInfo.ALL_SUPPORTED_INPUT_TYPES,
+        context_length=lf.ModelInfo.ContextLength(
+            max_input_tokens=1_048_576,
+            max_output_tokens=65_536,
+        ),
+        pricing=GeminiModelInfo.Pricing(
+            cost_per_1m_cached_input_tokens=0.05,
+            cost_per_1m_input_tokens=0.50,
+            cost_per_1m_output_tokens=3.00,
+        ),
+        rate_limits=lf.ModelInfo.RateLimits(
+            max_requests_per_minute=2_000,
+            max_tokens_per_minute=4_000_000,
+        ),
+    ),
     # Gemini 2.5 Flash
     GeminiModelInfo(
         model_id='gemini-2.5-flash',
@@ -834,7 +883,10 @@ class Gemini(rest.REST):
       config['thinkingConfig'] = thinking_config_data
 
     # This is the new feature since Gemini 3.
-    if self.model_id.startswith('gemini-3'):
+    # Skip for image generation models as they don't support mediaResolution.
+    if self.model_id.startswith('gemini-3') and not (
+        self.response_modalities and 'IMAGE' in self.response_modalities
+    ):
       config['mediaResolution'] = 'MEDIA_RESOLUTION_HIGH'
 
     if self.response_modalities:

langfun/core/llms/google_genai.py

@@ -125,6 +125,30 @@ class Gemini3ProPreview(GenAI):
   model = 'gemini-3-pro-preview'
 
 
+class Gemini3ProImagePreview(GenAI):
+  """Gemini 3 Pro Image Preview model for high-fidelity image generation.
+
+  This model supports:
+  - Text-to-image generation
+  - Image editing (multimodal input)
+  - Visual reasoning
+
+  Key Requirements:
+  - responseModalities must include 'IMAGE'
+  - Supported aspect ratios: 1:1, 16:9, 9:16, 4:3, 3:4
+  - Image sizes: 1K (default), 2K, 4K
+  """
+
+  model = 'gemini-3-pro-image-preview'
+  response_modalities = ['TEXT', 'IMAGE']
+
+
+class Gemini3FlashPreview(GenAI):
+  """Gemini 3 Flash Preview model."""
+
+  model = 'gemini-3-flash-preview'
+
+
 class Gemini25FlashImagePreview(GenAI):
   """Gemini 2.5 Flash Image Preview model."""
   model = 'gemini-2.5-flash-image-preview'

langfun/core/llms/vertexai.py

@@ -220,6 +220,33 @@ class VertexAIGemini3ProPreview(VertexAIGemini):  # pylint: disable=invalid-name
   location = 'global'
 
 
+class VertexAIGemini3ProImagePreview(VertexAIGemini):  # pylint: disable=invalid-name
+  """Gemini 3 Pro Image Preview model for high-fidelity image generation.
+
+  This model supports:
+  - Text-to-image generation
+  - Image editing (multimodal input)
+  - Visual reasoning
+
+  Key Requirements:
+  - Uses v1beta1 API endpoint
+  - responseModalities must include 'IMAGE'
+  - Supported aspect ratios: 1:1, 16:9, 9:16, 4:3, 3:4
+  - Image sizes: 1K (default), 2K, 4K
+  """
+
+  model = 'gemini-3-pro-image-preview'
+  location = 'global'
+  response_modalities = ['TEXT', 'IMAGE']
+
+
+class VertexAIGemini3FlashPreview(VertexAIGemini):  # pylint: disable=invalid-name
+  """Gemini 3 Flash Preview model launched on 12/17/2025."""
+
+  model = 'gemini-3-flash-preview'
+  location = 'global'
+
+
 class VertexAIGemini25Pro(VertexAIGemini):  # pylint: disable=invalid-name
   """Gemini 2.5 Pro GA model launched on 06/17/2025."""
 
@@ -419,6 +446,12 @@ class VertexAIClaude45Sonnet_20250929(VertexAIAnthropic):
   model = 'claude-sonnet-4-5@20250929'
 
 
+class VertexAIClaude45Opus_20251101(VertexAIAnthropic):
+  """Anthropic's Claude 4.5 Opus model on VertexAI."""
+
+  model = 'claude-opus-4-5@20251101'
+
+
 class VertexAIClaude4Opus_20250514(VertexAIAnthropic):
   """Anthropic's Claude 4 Opus model on VertexAI."""
   model = 'claude-opus-4@20250514'

langfun/core/modalities/mime.py

@@ -135,7 +135,20 @@ class Mime(lf.Modality):
       raise lf.ModalityError(
           f'MIME type {self.mime_type!r} cannot be converted to text.'
       )
-    return self.to_bytes().decode()
+    content = self.to_bytes()
+    # Try UTF-8 first (most common encoding).
+    try:
+      return content.decode('utf-8')
+    except UnicodeDecodeError:
+      pass
+    # Check for UTF-16 BOM (0xff 0xfe or 0xfe 0xff).
+    if content[:2] in (b'\xff\xfe', b'\xfe\xff'):
+      try:
+        return content.decode('utf-16')
+      except UnicodeDecodeError:
+        pass
+    # Fallback: decode with error replacement to avoid crashing.
+    return content.decode('utf-8', errors='replace')
 
   def is_compatible(
       self, mime_types: str | Iterable[str]

langfun/core/modalities/mime_test.py

@@ -163,5 +163,53 @@ class CustomMimeTest(unittest.TestCase):
     )
 
 
+class ToTextEncodingTest(unittest.TestCase):
+  """Tests for to_text() encoding handling."""
+
+  def test_utf8_decoding(self):
+    """Test that valid UTF-8 content is decoded correctly."""
+    content = mime.Custom('text/plain', b'Hello, World!')
+    self.assertEqual(content.to_text(), 'Hello, World!')
+
+    # UTF-8 with multi-byte characters.
+    utf8_content = 'こんにちは'.encode('utf-8')
+    content = mime.Custom('text/plain', utf8_content)
+    self.assertEqual(content.to_text(), 'こんにちは')
+
+  def test_utf16_le_bom_decoding(self):
+    """Test that UTF-16 Little Endian with BOM is decoded correctly."""
+    # UTF-16 LE BOM: 0xff 0xfe
+    utf16_le_content = 'Hello'.encode('utf-16-le')
+    content_with_bom = b'\xff\xfe' + utf16_le_content
+    content = mime.Custom('text/plain', content_with_bom)
+    self.assertEqual(content.to_text(), 'Hello')
+
+  def test_utf16_be_bom_decoding(self):
+    """Test that UTF-16 Big Endian with BOM is decoded correctly."""
+    # UTF-16 BE BOM: 0xfe 0xff
+    utf16_be_content = 'Hello'.encode('utf-16-be')
+    content_with_bom = b'\xfe\xff' + utf16_be_content
+    content = mime.Custom('text/plain', content_with_bom)
+    self.assertEqual(content.to_text(), 'Hello')
+
+  def test_invalid_bytes_fallback_with_replacement(self):
+    """Test that invalid bytes are replaced with replacement character."""
+    # 0xff alone is invalid in UTF-8 and doesn't have UTF-16 BOM pattern.
+    invalid_content = b'\xff\xfdHello'
+    content = mime.Custom('text/plain', invalid_content)
+    result = content.to_text()
+    # Invalid bytes should be replaced with U+FFFD (replacement character).
+    self.assertIn('\ufffd', result)
+    self.assertIn('Hello', result)
+
+  def test_binary_mime_type_raises_error(self):
+    """Test that binary MIME types raise ModalityError."""
+    content = mime.Custom('application/octet-stream', b'\x00\x01\x02')
+    with self.assertRaisesRegex(
+        lf.ModalityError, 'cannot be converted to text'
+    ):
+      content.to_text()
+
+
 if __name__ == '__main__':
   unittest.main()

{langfun-0.1.2.dev202512040805.dist-info → langfun-0.1.2.dev202601030804.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langfun
-Version: 0.1.2.dev202512040805
+Version: 0.1.2.dev202601030804
 Summary: Langfun: Language as Functions.
 Home-page: https://github.com/google/langfun
 Author: Langfun Authors

{langfun-0.1.2.dev202512040805.dist-info → langfun-0.1.2.dev202601030804.dist-info}/RECORD

@@ -35,7 +35,7 @@ langfun/core/subscription_test.py,sha256=Y4ZdbZEwm83YNZBxHff0QR4QUa4rdaNXA3_jfIc
 langfun/core/template.py,sha256=KIRLEhijPf5UP5auJKf9x6HKKW2E1Ki83Dcs9W8eKs8,29571
 langfun/core/template_test.py,sha256=K7gx1CsDlau2CCvrE4BeKV26n0GyovhbsadWf8pDMek,20400
 langfun/core/agentic/__init__.py,sha256=ajI1SGcQWXfBp2MFH13Fr9OkSN4slSKDlJSHPDp4P_c,1573
-langfun/core/agentic/action.py,sha256=mgRUU6_dQVntZP3OmCoQCbrnNRHA_KVnOcKSy9GCQQM,72375
+langfun/core/agentic/action.py,sha256=R3KQNzyuv2ME7ciK7PP_mB_5HaKTyFGMLeaCIZRRn5A,72385
 langfun/core/agentic/action_eval.py,sha256=Mjk5QBFInjIK3VlDR2RT__pugmtAW-iv-SMtH5GMcNo,5258
 langfun/core/agentic/action_eval_test.py,sha256=7AkOwNbUX-ZgR1R0a7bvUZ5abNTUV7blf_8Mnrwb-II,2811
 langfun/core/agentic/action_test.py,sha256=sKaU8IZ9whgBeV7oMB-0s8TdQ6PqDkI_eT9Yw3EZSYU,24123
@@ -68,42 +68,42 @@ langfun/core/eval/patching.py,sha256=wJqqML_z_hXQQ65f9oJpdtiNEkUvwWWdNgGiIcV1Jq4
 langfun/core/eval/patching_test.py,sha256=8kCd54Egjju22FMgtJuxEsrXkW8ifs-UUBHtrCG1L6w,4775
 langfun/core/eval/scoring.py,sha256=1C7e7gR8Wai7M9oBXRZifntxy5HEik5qjVo9gY8B7KI,6423
 langfun/core/eval/scoring_test.py,sha256=UcBH0R6vAovZ0A4yM22s5cBHL1qVKASubrbu1t8dYBw,4529
-langfun/core/eval/v2/__init__.py,sha256=q2e_WUmZmWWigqBx6n-2koncbe2ZLk4lmO5hLevp97w,1851
-langfun/core/eval/v2/checkpointing.py,sha256=h_h-nOC-NXi1p85KYvv2nYfsVWuCxayJnRQCn2ErMlo,14310
+langfun/core/eval/v2/__init__.py,sha256=XbkBqoyJBH_khtAS01gP6_V4KnWLY3bFJ7D0rtHa1BU,1878
+langfun/core/eval/v2/checkpointing.py,sha256=_WJydQn43LDHTpjyt7qIh5XQl1--Jvq03lrwktcgy6U,15526
 langfun/core/eval/v2/checkpointing_test.py,sha256=s_E94dOPNO1zYzXyQI37wvCF3suez-r4Nls9popN58w,9787
 langfun/core/eval/v2/config_saver.py,sha256=nsuG0pqTikIlsL-Mij6swteUBif-zxJUdGxTHZsOVeQ,1205
 langfun/core/eval/v2/config_saver_test.py,sha256=OD0zl26YHjNibFD67YxwrZ7-zT9V7p-3zLDItWBAgic,1261
 langfun/core/eval/v2/eval_test_helper.py,sha256=baew3-cqomy1p7mF1_Xw7AvEWUwCimi3J7-8Ay3eEPo,6539
-langfun/core/eval/v2/evaluation.py,sha256=A_5go1bsZuSfkLueo24jmIf4Tao2rfW-VPE2x1onG8U,30456
+langfun/core/eval/v2/evaluation.py,sha256=1T0lxTu9gy329Mq4ii16ktARbtvbBGY9IUtsUIkNXeY,30740
 langfun/core/eval/v2/evaluation_test.py,sha256=gurFzSfPECZ_FMQOnf3bzKOHmQ7C4IUxEfbyZy50bjM,7966
 langfun/core/eval/v2/example.py,sha256=VZeBqMWnfEtn1mmdPW2w2u2XbAWVll1q1-50qL8DjS8,11606
 langfun/core/eval/v2/example_test.py,sha256=RwtBcUumPBWynA8BLMoZetSHdgvFywlHXuyvInf1y_s,3576
-langfun/core/eval/v2/experiment.py,sha256=Qkff7hJgaeQJvv9mUrQ5kp8Pn38Z1sWyeUizFajMr1I,36596
+langfun/core/eval/v2/experiment.py,sha256=_tmahrRoFxSf8oPzsS-VcxqbCT2KvYafUoQuBP4yQ4s,37256
 langfun/core/eval/v2/experiment_test.py,sha256=7prE4ASKlbwQIXiLzEqjgaF4yQDL7KjxX-dBUPT84VA,14145
 langfun/core/eval/v2/metric_values.py,sha256=WAL1BdHaU_oq7d_k1KyjhiQDK32dNLSyn1L2yEkz0o4,6040
 langfun/core/eval/v2/metric_values_test.py,sha256=5ffwnqrbLIBh1hdUl3L9mpJlUvsmd2VQ8UWPOJcQj4s,3630
 langfun/core/eval/v2/metrics.py,sha256=cdFqrhRlxqpBk_04Mmhk21NcOD0kor5H0iFX54_rO4s,14486
 langfun/core/eval/v2/metrics_test.py,sha256=gf8hT5V5OeM-Ah-Wa4aLtgrYZmlMStKPjEhCTS0VMHQ,6812
-langfun/core/eval/v2/progress.py,sha256=Cd79j8fhumW5QOuISiSXOJKOZ5-I9IkmGLgvqRmoULA,11677
+langfun/core/eval/v2/progress.py,sha256=-kYzdiVbw56eb8mc0yCS_2d5aIUdZ1oVi1QWBuhFW74,11764
 langfun/core/eval/v2/progress_test.py,sha256=MzJ7wa65XYZ0chArA-lSg1eRSvQ_TzZJIHMk85Kwz7o,3208
 langfun/core/eval/v2/progress_tracking.py,sha256=yMYlOMJF8M4FUhyjGRkM6O6TXiMwKPsEn3wbpftxcss,6376
 langfun/core/eval/v2/progress_tracking_test.py,sha256=37v42y4kh2GfDXBrkugEupW6IRAzA774wwPJaOyefUs,2597
-langfun/core/eval/v2/reporting.py,sha256=-IV-RzR-QriPZz2Ke6nNNRXu6avxpX48zr9c-j815iA,9001
+langfun/core/eval/v2/reporting.py,sha256=U6ToanG_y-zFZ1W1CRaju6T_zLlbmknLR148nIvl8mw,8949
 langfun/core/eval/v2/reporting_test.py,sha256=q3LBfPk7jvEWXB3sdk2CycbMKqNRyXhs5z6BokfwDIE,6096
 langfun/core/eval/v2/runners/__init__.py,sha256=2TcCLW32OsmXQINcVKa2ZJY8Ca7j3NnT0yy9hXYUDn8,1115
-langfun/core/eval/v2/runners/base.py,sha256=XY9biSNMW17UvaqyPQU1yc6uo41O6pHqCiK-LLBunNc,14012
+langfun/core/eval/v2/runners/base.py,sha256=8KZlkSuuzlSxeoQfsG32OM6E2WUbtgNuz9p5MyfVFQY,14146
 langfun/core/eval/v2/runners/beam.py,sha256=LQK9bZCFJR9j9DJ-mAudhphumItGwXc5bbGwadl9kxY,11782
 langfun/core/eval/v2/runners/beam_test.py,sha256=cI5WaQQObnRrPnGjED3OFT3JXYOE3thQ640H08TG_dw,5306
-langfun/core/eval/v2/runners/ckpt_monitor.py,sha256=jZCb4nrMXBbOe9cNKdPrj6X8AQmVB2R09jCTSRnWsoU,10100
-langfun/core/eval/v2/runners/ckpt_monitor_test.py,sha256=38sdIAObTdYUibD27cmyDCKrl_DhOZLynDKVkOrvbB4,5440
+langfun/core/eval/v2/runners/ckpt_monitor.py,sha256=KaaDYvHNOewUrJqJ4FHjdMeS7okpX7FYdjCx558joPU,12071
+langfun/core/eval/v2/runners/ckpt_monitor_test.py,sha256=Xqd30PF0XIOrqBSZ53_7ozxYR3Wc3SiIaKuwwj1AXQ8,7176
 langfun/core/eval/v2/runners/debug.py,sha256=ExsBcAvmhFsaaS3VLjxE70HImHe2YVs0IpoefM01onY,1442
 langfun/core/eval/v2/runners/debug_test.py,sha256=kDWs4Fu7itzBxbRwFc-UKEP2hAV0iVFp2wWkEuZNEcg,2577
 langfun/core/eval/v2/runners/parallel.py,sha256=PSdOY3i2ot94TWVCZY0iJSWFAT0CCxa1wxk7KpI_GfI,7794
 langfun/core/eval/v2/runners/parallel_test.py,sha256=8M8OTpsDd-wQYZRRSPCYGkwjt7gUvkgze8NMCTKydUw,6146
 langfun/core/eval/v2/runners/sequential.py,sha256=hebMZd6EVraY9zAwariT9WfsWQyX5AYuRsFdRo-knKU,1631
 langfun/core/eval/v2/runners/sequential_test.py,sha256=apbNC0-Pi6r17_OQlHqqOZM0OVo1mZlaPk2B4vUteRg,6064
-langfun/core/llms/__init__.py,sha256=WDcrh0whpAB07zhW4NOzfHLNaYhjlQdCGVEk65KbJdY,10353
-langfun/core/llms/anthropic.py,sha256=6uE1EC9YWtbiFwZNNPEFv-QzeGQQ7G27kheTTE15Ewg,31175
+langfun/core/llms/__init__.py,sha256=BPPV18zMOvLuAhStVVbef13Pe59CsEWGo9YvRAyeFAo,10750
+langfun/core/llms/anthropic.py,sha256=HwgzvMkVdQwmwfDtI_X6lxW0gBxsbYAdMKes8u8Jv_Y,33224
 langfun/core/llms/anthropic_test.py,sha256=qA9vByp_cwwXNlXzcwHpPWFnO9lfFo8NKfDi5nBNqgI,9052
 langfun/core/llms/azure_openai.py,sha256=LEc7-ay2fOOCwwL3SfxDr3KCdH8-2i1EtD-PBvr4kfk,2777
 langfun/core/llms/azure_openai_test.py,sha256=lkMZkQdJBV97fTM4C4z8qNfvr6spgiN5G4hvVUIVr0M,1735
@@ -113,9 +113,9 @@ langfun/core/llms/deepseek.py,sha256=jQsotTUk4161EJIcoQOV7iOWBZfQ3Ukh9GOh31A0HYU
 langfun/core/llms/deepseek_test.py,sha256=DvROWPlDuow5E1lfoSkhyGt_ELA19JoQoDsTnRgDtTg,1847
 langfun/core/llms/fake.py,sha256=NH8Zlezmx3eacao4D7wihrZjRuyBJuHR5rdyp94PrAw,4409
 langfun/core/llms/fake_test.py,sha256=lC-C2TpEsnf2kmZpa3OiH2H944I4hMWTAaHEXzRj1DU,7855
-langfun/core/llms/gemini.py,sha256=VcVNMIFv9iY--pYPAf2rngtQt-XB882_-2k0PO9nBGM,32229
+langfun/core/llms/gemini.py,sha256=6ys2nm9WnhyO27YY3c6AI7reOs-rMk7yPBBh9EC07-s,34225
 langfun/core/llms/gemini_test.py,sha256=bv-Ulv3vjGhxd8nJD_UDhWDMK3K3TM7b5powBcYrv1c,10844
-langfun/core/llms/google_genai.py,sha256=0MaLyh4ndwWEFsMZZZymHxImvao9_kaWmXEElWWXAkE,6516
+langfun/core/llms/google_genai.py,sha256=YObrVUhHJcGCiUlL4LuoGyjEiyf_WM2y4iBxQIV0GHY,7096
 langfun/core/llms/google_genai_test.py,sha256=NKNtpebArQ9ZR7Qsnhd2prFIpMjleojy6o6VMXkJ1zY,1502
 langfun/core/llms/groq.py,sha256=O-kv2_R_IkC8wGIT086xin8jYi7QnsakPCGVLR58lMw,12517
 langfun/core/llms/groq_test.py,sha256=P4EgexCqsh4K2x11w0UL_vz-YYNaPdQU0WsDAdnTRQ8,2045
@@ -127,7 +127,7 @@ langfun/core/llms/openai_compatible_test.py,sha256=8yr_jGmHCDyMwp-VcJwThFgh7B_56
 langfun/core/llms/openai_test.py,sha256=1o5rxiHZj-UEgugWN8JmfJtznhUmDywy6dU3Euax-Ts,2639
 langfun/core/llms/rest.py,sha256=eR-M1st5ZnzuitICyYfxSRcmQWmy_eeOoe2bHLalzN0,5351
 langfun/core/llms/rest_test.py,sha256=_zM7nV8DEVyoXNiQOnuwJ917mWjki0614H88rNmDboE,5020
-langfun/core/llms/vertexai.py,sha256=TvF4nO8p-JS1GvwDfAZL_xmzbqkdnQX1IFhujap7D4Y,21563
+langfun/core/llms/vertexai.py,sha256=zQO-zi1HVLwLJU9r5_hmCvyfxdox7FKT_43XjILRwvA,22489
 langfun/core/llms/vertexai_test.py,sha256=_e-acnNBAf9C3WO6i1b2J_mhRzdDdYQTorD9hIVZKOg,5034
 langfun/core/llms/cache/__init__.py,sha256=QAo3InUMDM_YpteNnVCSejI4zOsnjSMWKJKzkb3VY64,993
 langfun/core/llms/cache/base.py,sha256=qLGlEMi5cfsDxRTsOWrmwbxjvvwUaq4Y8MxlXr69wpw,5060
@@ -150,8 +150,8 @@ langfun/core/modalities/audio.py,sha256=cb95FzDE-IIQf7kXy7D4AAXtziQF0FYkZUe4pw5E
 langfun/core/modalities/audio_test.py,sha256=tW1vEy-Cumhf-HgDgCxlSNZqgJb2HTgqOixGWLiwOmw,2065
 langfun/core/modalities/image.py,sha256=SS_FSrYSjb1uL0IAVyUu2TZX2-FcI_x9ZTB8im5Amzk,4030
 langfun/core/modalities/image_test.py,sha256=91LpEOvr_v6SGdtzSuCw3ks62L8vxJVIZwgAKxk7UmY,8476
-langfun/core/modalities/mime.py,sha256=dc0pmWl4S_fmVlHYXT7kx_WcBXxGCfk654w1vOsDAt8,10809
-langfun/core/modalities/mime_test.py,sha256=9Juvs4KQgPfP72nn8TbPNwqShV7Bs8oFe--Qvt0QR5A,6390
+langfun/core/modalities/mime.py,sha256=9YK-uRGYN6YG3ux7zSYl5XGZEDLBiXyTax1cLMhissY,11255
+langfun/core/modalities/mime_test.py,sha256=e6p-XW47yNXbvPS2R4-0afZd84bOqCc6DnIzqZCkPZk,8391
 langfun/core/modalities/pdf.py,sha256=rc-uIKRVkTTa0j7jC6WRwKM9WqiS5NxF-H6PPunVeXM,1231
 langfun/core/modalities/pdf_test.py,sha256=ulZ0FbnlsU0wkrdckJ4ONZPTYRyMPO9Aob1UO6FXygk,1950
 langfun/core/modalities/video.py,sha256=ZopyDf-8bi0V-QZDAg-_8S3HkMNiEQL9aWmGuI6Fkrs,1506
@@ -210,8 +210,8 @@ langfun/env/event_handlers/event_logger.py,sha256=ga8RN8qjwtAOCnV_MnhNPTktN8EJ-x
 langfun/env/event_handlers/event_logger_test.py,sha256=qSAcirtRz00H-1RL9ShELBiZKiPxsk_v6cVA6XdAk4k,9274
 langfun/env/event_handlers/metric_writer.py,sha256=7ZrUp0rYvs7TfNpQ16Xbxg8vp-6ZbjuJ-qrhVSbhv2I,21085
 langfun/env/event_handlers/metric_writer_test.py,sha256=bjdYXoXMPWpWz_-HUPM6vFP1ez5G386u0fmPfe-SR_M,5952
-langfun-0.1.2.dev202512040805.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-langfun-0.1.2.dev202512040805.dist-info/METADATA,sha256=-73ceSaomQOr4ZxkeUCJqXmXlWj3MGCFyhJLo9CUohA,7522
-langfun-0.1.2.dev202512040805.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-langfun-0.1.2.dev202512040805.dist-info/top_level.txt,sha256=RhlEkHxs1qtzmmtWSwYoLVJAc1YrbPtxQ52uh8Z9VvY,8
-langfun-0.1.2.dev202512040805.dist-info/RECORD,,
+langfun-0.1.2.dev202601030804.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+langfun-0.1.2.dev202601030804.dist-info/METADATA,sha256=FRXTPzGp4Bi8NFAGVHUUWOaEuO-5nUIy-X2y1CH_bds,7522
+langfun-0.1.2.dev202601030804.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+langfun-0.1.2.dev202601030804.dist-info/top_level.txt,sha256=RhlEkHxs1qtzmmtWSwYoLVJAc1YrbPtxQ52uh8Z9VvY,8
+langfun-0.1.2.dev202601030804.dist-info/RECORD,,