langfun 0.1.2.dev202510230805__py3-none-any.whl → 0.1.2.dev202511160804__py3-none-any.whl

langfun/core/eval/v2/reporting.py

@@ -32,8 +32,97 @@ _SUMMARY_FILE = 'summary.html'
 _EVALULATION_DETAIL_FILE = 'index.html'
 
 
+class ExampleHtmlGenerator(experiment_lib.Plugin):
+  """Plugin for generating HTML views for each evaluation example."""
+
+  def on_example_complete(
+      self, runner: Runner, experiment: Experiment, example: Example
+  ):
+    self._save_example_html(runner, experiment, example)
+
+  def _save_example_html(
+      self, runner: Runner, experiment: Experiment, example: Example
+  ) -> None:
+    """Saves the example in HTML format."""
+    current_run = runner.current_run
+    def _generate():
+      try:
+        with pg.timeit() as t:
+          html = example.to_html(
+              collapse_level=None,
+              enable_summary_tooltip=False,
+              extra_flags=dict(
+                  # For properly rendering the next link.
+                  num_examples=getattr(experiment, 'num_examples', None)
+              ),
+          )
+          html.save(
+              runner.current_run.output_path_for(
+                  experiment, f'{example.id}.html'
+              )
+          )
+        experiment.info(
+            f'\'{example.id}.html\' generated in {t.elapse:.2f} seconds. '
+        )
+      except BaseException as e:  # pylint: disable=broad-except
+        experiment.error(
+            f'Failed to generate \'{example.id}.html\'. '
+            f'Error: {e}, Stacktrace: \n{traceback.format_exc()}.',
+        )
+        raise e
+
+    def _copy():
+      src_file = current_run.input_path_for(experiment, f'{example.id}.html')
+      dest_file = current_run.output_path_for(experiment, f'{example.id}.html')
+
+      if src_file == dest_file:
+        return
+
+      if not pg.io.path_exists(src_file):
+        experiment.warning(
+            f'Skip copying \'{example.id}.html\' as '
+            f'{src_file!r} does not exist.'
+        )
+        return
+
+      try:
+        with pg.timeit() as t, pg.io.open(src_file, 'r') as src:
+          content = src.read()
+          with pg.io.open(dest_file, 'w') as dest:
+            dest.write(content)
+        experiment.info(
+            f'\'{example.id}.html\' copied in {t.elapse:.2f} seconds.'
+        )
+      except BaseException as e:  # pylint: disable=broad-except
+        experiment.error(
+            f'Failed to copy {src_file!r} to {dest_file!r}. Error: {e}.'
+        )
+        raise e
+
+    generate_example_html = current_run.generate_example_html
+    if (generate_example_html == 'all'
+        or (generate_example_html == 'new' and example.newly_processed)
+        or (isinstance(generate_example_html, list)
+            and example.id in generate_example_html)):
+      op = _generate
+    else:
+      op = _copy
+    runner.background_run(op)
+
+
 class HtmlReporter(experiment_lib.Plugin):
-  """Plugin for periodically generating HTML reports for the experiment."""
+  """Plugin for periodically generating HTML reports for the experiment.
+
+  The `HtmlReporter` plugin generates several HTML files during an experiment
+  run:
+    - A `summary.html` at the root of the run directory, summarizing all
+      evaluations in the experiment.
+    - An `index.html` for each leaf evaluation, detailing the evaluation
+      definition, metrics, and logs.
+
+  These reports are updated periodically in the background during the run,
+  allowing users to monitor progress in near real-time.
+  """
 
   summary_interval: Annotated[
       int,
@@ -127,7 +216,6 @@ class HtmlReporter(experiment_lib.Plugin):
   def on_example_complete(
       self, runner: Runner, experiment: Experiment, example: Example
   ):
-    self._save_example_html(runner, experiment, example)
     self._maybe_update_experiment_html(runner, experiment)
     self._maybe_update_summary(runner)
 
@@ -197,72 +285,3 @@ class HtmlReporter(experiment_lib.Plugin):
         runner.background_run(_save)
       else:
         _save()
-
-  def _save_example_html(
-      self, runner: Runner, experiment: Experiment, example: Example
-  ) -> None:
-    """Saves the example in HTML format."""
-    current_run = runner.current_run
-    def _generate():
-      try:
-        with pg.timeit() as t:
-          html = example.to_html(
-              collapse_level=None,
-              enable_summary_tooltip=False,
-              extra_flags=dict(
-                  # For properly rendering the next link.
-                  num_examples=getattr(experiment, 'num_examples', None)
-              ),
-          )
-          html.save(
-              runner.current_run.output_path_for(
-                  experiment, f'{example.id}.html'
-              )
-          )
-        experiment.info(
-            f'\'{example.id}.html\' generated in {t.elapse:.2f} seconds. '
-        )
-      except BaseException as e:  # pylint: disable=broad-except
-        experiment.error(
-            f'Failed to generate \'{example.id}.html\'. '
-            f'Error: {e}, Stacktrace: \n{traceback.format_exc()}.',
-        )
-        raise e
-
-    def _copy():
-      src_file = current_run.input_path_for(experiment, f'{example.id}.html')
-      dest_file = current_run.output_path_for(experiment, f'{example.id}.html')
-
-      if src_file == dest_file:
-        return
-
-      if not pg.io.path_exists(src_file):
-        experiment.warning(
-            f'Skip copying \'{example.id}.html\' as '
-            f'{src_file!r} does not exist.'
-        )
-        return
-
-      try:
-        with pg.timeit() as t, pg.io.open(src_file, 'r') as src:
-          content = src.read()
-          with pg.io.open(dest_file, 'w') as dest:
-            dest.write(content)
-        experiment.info(
-            f'\'{example.id}.html\' copied in {t.elapse:.2f} seconds.'
-        )
-      except BaseException as e:  # pylint: disable=broad-except
-        experiment.error(
-            f'Failed to copy {src_file!r} to {dest_file!r}. Error: {e}.'
-        )
-        raise e
-
-    generate_example_html = current_run.generate_example_html
-    if (generate_example_html == 'all'
-        or (generate_example_html == 'new' and example.newly_processed)
-        or (isinstance(generate_example_html, list)
-            and example.id in generate_example_html)):
-      op = _generate
-    else:
-      op = _copy
-    runner.background_run(op)

langfun/core/eval/v2/reporting_test.py

@@ -29,7 +29,12 @@ class ReportingTest(unittest.TestCase):
     experiment = eval_test_helper.test_experiment()
     checkpointer = checkpointing.BulkCheckpointer('checkpoint.jsonl')
     reporter = reporting.HtmlReporter()
-    run = experiment.run(root_dir, 'new', plugins=[checkpointer, reporter])
+    example_html_generator = reporting.ExampleHtmlGenerator()
+    run = experiment.run(
+        root_dir,
+        'new',
+        plugins=[checkpointer, reporter, example_html_generator]
+    )
     self.assertTrue(
         pg.io.path_exists(os.path.join(run.output_root, 'summary.html'))
     )
@@ -52,8 +57,10 @@ class ReportingTest(unittest.TestCase):
     root_dir = os.path.join(tempfile.mkdtemp(), 'test_reporting2')
     experiment = eval_test_helper.test_experiment()
     run = experiment.run(
-        root_dir, 'new', plugins=[checkpointer, reporter],
-        warm_start_from=run.output_root
+        root_dir,
+        'new',
+        plugins=[checkpointer, reporter, example_html_generator],
+        warm_start_from=run.output_root,
     )
     self.assertTrue(
         pg.io.path_exists(os.path.join(run.output_root, 'summary.html'))
@@ -105,7 +112,12 @@ class ReportingTest(unittest.TestCase):
                   .test_experiment_with_example_html_generation_error())
     checkpointer = checkpointing.BulkCheckpointer('checkpoint.jsonl')
     reporter = reporting.HtmlReporter()
-    run = experiment.run(root_dir, 'new', plugins=[checkpointer, reporter])
+    example_html_generator = reporting.ExampleHtmlGenerator()
+    run = experiment.run(
+        root_dir,
+        'new',
+        plugins=[checkpointer, reporter, example_html_generator]
+    )
     self.assertTrue(
         pg.io.path_exists(os.path.join(run.output_root, 'summary.html'))
     )
@@ -132,8 +144,10 @@ class ReportingTest(unittest.TestCase):
     experiment = (eval_test_helper
                   .test_experiment_with_example_html_generation_error())
     run = experiment.run(
-        root_dir, 'new', plugins=[checkpointer, reporter],
-        warm_start_from=run.output_root
+        root_dir,
+        'new',
+        plugins=[checkpointer, reporter, example_html_generator],
+        warm_start_from=run.output_root,
     )
     self.assertTrue(
         pg.io.path_exists(os.path.join(run.output_root, 'summary.html'))

langfun/core/eval/v2/runners/__init__.py

@@ -0,0 +1,26 @@
+# Copyright 2024 The Langfun Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Langfun evaluation runners."""
+
+from langfun.core.eval.v2.runners.base import RunnerBase
+from langfun.core.eval.v2.runners.debug import DebugRunner
+from langfun.core.eval.v2.runners.parallel import ParallelRunner
+from langfun.core.eval.v2.runners.sequential import SequentialRunner
+
+__all__ = [
+    'RunnerBase',
+    'DebugRunner',
+    'ParallelRunner',
+    'SequentialRunner',
+]

langfun/core/eval/v2/{runners.py → runners/base.py}

@@ -11,13 +11,12 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""Evaluation experiment runners."""
+"""Base experiment runner."""
+
 import abc
-import collections
 import concurrent.futures
 import random
 import threading
-import time
 import traceback
 from typing import Any, Annotated, Callable, Iterator
 
@@ -42,7 +41,14 @@ _RUN_MANIFEST = 'run.json'
 
 
 class RunnerBase(Runner):
-  """A simple runner that runs evaluations and their examples sequentially."""
+  """Base class for runners with plugin support and IO pooling.
+
+  `RunnerBase` provides the basic runner functionalities such as plugin
+  integration for checkpointing, reporting and progress tracking.
+  It also manages a thread pool for background IO operations.
+  Subclasses should implement `_run` and `_evaluate_items` for different
+  execution strategies.
+  """
 
   tqdm: Annotated[
       bool,
@@ -58,6 +64,11 @@ class RunnerBase(Runner):
       reporting.HtmlReporter(),
   ]
 
+  max_background_threads: Annotated[
+      int,
+      'Max number of background threads for IO operations.'
+  ] = 128
+
   def _on_bound(self):
     super()._on_bound()
 
@@ -66,7 +77,9 @@ class RunnerBase(Runner):
       self.plugins.append(progress_tracking.progress_tracker(self.tqdm))
 
     self._io_pool_lock = threading.Lock()
-    self._io_pool = concurrent.futures.ThreadPoolExecutor(max_workers=16)
+    self._io_pool = concurrent.futures.ThreadPoolExecutor(
+        max_workers=self.max_background_threads
+    )
     # TODO(daiyip): render background errors.
     self._background_last_error = None
 
@@ -220,7 +233,7 @@ class RunnerBase(Runner):
       else:
         # A evaluation could be considered as done if it has processed all the
         # examples specified by `example_ids`.
-        assert progress.is_completed
+        assert progress.is_completed, progress
         parent_progress.increment_processed()
 
       if parent_progress.is_completed:
@@ -335,6 +348,7 @@ class RunnerBase(Runner):
   def run_evaluation(self, evaluation: Evaluation) -> None:
     """Runs the evaluation."""
     try:
+      evaluation.setup()
       self.on_experiment_start(evaluation)
 
       per_evaluation_settings = {}
@@ -367,6 +381,8 @@ class RunnerBase(Runner):
     except BaseException as e:  # pylint: disable=broad-except
       self.on_experiment_abort(evaluation, e)
       raise e
+    finally:
+      evaluation.teardown()
 
   @abc.abstractmethod
   def _evaluate_items(
@@ -394,121 +410,3 @@ class RunnerBase(Runner):
     return in_memory.InMemory(
         self.current_run.output_path_for(experiment, 'cache.json')
     )
-
-
-class SequentialRunner(RunnerBase):
-  """Sequential runner.
-
-  Sequential runner runs all evaluations and their examples in sequence,
-  as well as the background tasks, it allows the developer to catch all
-  exceptions thrown from the background tasks, making it easier to debug.
-  """
-
-  NAME = 'sequential'
-
-  def background_run(
-      self, func: Callable[..., Any], *args: Any, **kwargs: Any
-  ) -> None:
-    """Runs the function with the IO pool."""
-    func(*args, **kwargs)
-
-  def _run(self, evaluations: list[Evaluation]) -> None:
-    """Runs the experiment in sequence."""
-    for e in evaluations:
-      self.run_evaluation(e)
-
-  def _evaluate_items(
-      self, evaluation: Evaluation, items: Iterator[Example]
-  ) -> None:
-    """Runs the evaluation items in sequence."""
-    for item in items:
-      self.evaluate_item(evaluation, item)
-
-
-class DebugRunner(SequentialRunner):
-  """Debug runner."""
-
-  NAME = 'debug'
-
-  # Do not use the checkpointer for debug runner.
-  plugins = []
-
-  def _on_bound(self):
-    super()._on_bound()
-    if self.current_run.example_ids is None:
-      self.current_run.rebind(example_ids=[1], skip_notification=True)
-    self.current_run.rebind(raise_if_has_error=True, skip_notification=True)
-
-  def _save_run_manifest(self) -> None:
-    """Do nothing to avoid overriden existing runs."""
-
-
-class ParallelRunner(RunnerBase):
-  """Parallel runner."""
-
-  NAME = 'parallel'
-
-  timeout: Annotated[
-      int | None,
-      'Timeout for each evaluation example.'
-  ] = None
-
-  concurrent_startup_delay: Annotated[
-      tuple[int, int] | None,
-      (
-          'A range of seconds to delay the initial evaluation of each thread '
-          'in the thread pool, helping to prevent a burst in LLM QPS at '
-          'startup. If set to None, no delay will be applied.'
-      )
-  ] = None
-
-  def _run(self, evaluations: list[Evaluation]) -> None:
-    """Runs the evaluations in parallel."""
-    def _run_group(evaluation_group: list[Evaluation]):
-      for e in evaluation_group:
-        self.run_evaluation(e)
-
-    # Run evaluations in parallel groupped by resource key.
-    groups: dict[str, list[Evaluation]] = collections.defaultdict(list)
-    for e in evaluations:
-      resource_ids = e.resource_ids()
-      if not resource_ids:
-        group_id = e.id
-      else:
-        # TODO(daiyip): support group that requires multiple resources.
-        group_id = resource_ids.pop()
-      groups[group_id].append(e)
-
-    for _, _, _ in lf.concurrent_map(
-        _run_group,
-        groups.values(),
-        max_workers=max(64, len(groups)),
-        timeout=self.timeout,
-        silence_on_errors=None,
-    ):
-      pass
-
-  def _evaluate_items(
-      self, evaluation: Evaluation, items: Iterator[Example]
-  ) -> None:
-    """Override run items to run in parallel."""
-    if self.concurrent_startup_delay is not None:
-      thread_delayed = {}
-      def _evaluate_item(item: Example):
-        thread_id = threading.current_thread().ident
-        if thread_id not in thread_delayed:
-          thread_delayed[thread_id] = True
-          time.sleep(random.randint(*self.concurrent_startup_delay))
-        return self.evaluate_item(evaluation, item)
-    else:
-      def _evaluate_item(item: Example):
-        return self.evaluate_item(evaluation, item)
-
-    for _, _, _ in lf.concurrent_map(
-        _evaluate_item,
-        items,
-        max_workers=evaluation.max_workers,
-        timeout=self.timeout,
-        silence_on_errors=None,
-    ):
-      pass

langfun/core/eval/v2/runners/debug.py

@@ -0,0 +1,40 @@
+# Copyright 2024 The Langfun Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Debug runner."""
+
+from langfun.core.eval.v2.runners import sequential
+
+
+class DebugRunner(sequential.SequentialRunner):
+  """A runner for debugging evaluations.
+
+  The debug runner is a sequential runner that only runs the first example
+  of each evaluation, with `raise_if_has_error` enabled. This is useful for
+  quickly identifying issues in evaluation logic during development.
+  Checkpointers are disabled for this runner.
+  """
+
+  NAME = 'debug'
+
+  # Do not use the checkpointer for debug runner.
+  plugins = []
+
+  def _on_bound(self):
+    super()._on_bound()
+    if self.current_run.example_ids is None:
+      self.current_run.rebind(example_ids=[1], skip_notification=True)
+    self.current_run.rebind(raise_if_has_error=True, skip_notification=True)
+
+  def _save_run_manifest(self) -> None:
+    """Do nothing to avoid overriden existing runs."""

langfun/core/eval/v2/runners/debug_test.py

@@ -0,0 +1,79 @@
+# Copyright 2024 The Langfun Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for debug runner."""
+import os
+import tempfile
+from typing import Any
+import unittest
+
+from langfun.core.eval.v2 import eval_test_helper
+from langfun.core.eval.v2.runners import debug  # pylint: disable=unused-import
+
+import pyglove as pg
+
+
+class RunnerTest(unittest.TestCase):
+
+  def assert_same_list(self, actual: list[Any], expected: list[Any]):
+    self.assertEqual(len(actual), len(expected))
+    for i, (x, y) in enumerate(zip(actual, expected)):
+      if x is not y:
+        print(i, pg.diff(x, y))
+      self.assertIs(x, y)
+
+
+class DebugRunnerTest(RunnerTest):
+
+  def test_debug_runner(self):
+    plugin = eval_test_helper.TestPlugin()
+    exp = eval_test_helper.test_experiment()
+    root_dir = os.path.join(tempfile.mkdtemp(), 'test_debug_runner')
+    run = exp.run(root_dir, runner='debug', plugins=[plugin])
+
+    self.assertIsNotNone(plugin.start_time)
+    self.assertIsNotNone(plugin.complete_time)
+    self.assertGreater(plugin.complete_time, plugin.start_time)
+
+    self.assertEqual(
+        len(plugin.started_experiments), len(exp.nodes)
+    )
+    self.assertEqual(
+        len(plugin.completed_experiments), len(exp.nodes)
+    )
+    self.assertEqual(
+        len(plugin.started_example_ids), 6 * 1
+    )
+    self.assertEqual(
+        len(plugin.completed_example_ids), 6 * 1
+    )
+    self.assert_same_list(plugin.skipped_experiments, [])
+    self.assertFalse(
+        pg.io.path_exists(os.path.join(run.output_root, 'run.json'))
+    )
+
+    for node in exp.nodes:
+      self.assertTrue(node.progress.is_started)
+      self.assertTrue(node.progress.is_completed)
+      if node.is_leaf:
+        self.assertEqual(node.progress.num_skipped, 0)
+        self.assertEqual(node.progress.num_completed, 1)
+        self.assertEqual(node.progress.num_failed, 0)
+      else:
+        self.assertEqual(node.progress.num_skipped, 0)
+        self.assertEqual(node.progress.num_failed, 0)
+        self.assertEqual(node.progress.num_processed, node.progress.num_total)
+
+
+if __name__ == '__main__':
+  unittest.main()

langfun/core/eval/v2/runners/parallel.py

@@ -0,0 +1,100 @@
+# Copyright 2025 The Langfun Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Parallel runner."""
+
+import collections
+import random
+import threading
+import time
+
+from typing import Annotated, Iterator
+import langfun.core as lf
+from langfun.core.eval.v2.runners import base
+
+
+class ParallelRunner(base.RunnerBase):
+  """A runner that executes evaluations and examples in parallel.
+
+  The parallel runner groups evaluations by their required resources
+  (e.g., specific LLMs) and runs evaluations that do not share resources in
+  parallel. Within each evaluation, examples are also processed in parallel
+  using threads, up to `Evaluation.max_workers`.
+  """
+
+  NAME = 'parallel'
+
+  timeout: Annotated[
+      int | None,
+      'Timeout for each evaluation example.'
+  ] = None
+
+  concurrent_startup_delay: Annotated[
+      tuple[int, int] | None,
+      (
+          'A range of seconds to delay the initial evaluation of each thread '
+          'in the thread pool, helping to prevent a burst in LLM QPS at '
+          'startup. If set to None, no delay will be applied.'
+      )
+  ] = None
+
+  def _run(self, evaluations: list[base.Evaluation]) -> None:
+    """Runs the evaluations in parallel."""
+    def _run_group(evaluation_group: list[base.Evaluation]):
+      for e in evaluation_group:
+        self.run_evaluation(e)
+
+    # Run evaluations in parallel groupped by resource key.
+    groups: dict[str, list[base.Evaluation]] = collections.defaultdict(list)
+    for e in evaluations:
+      resource_ids = e.resource_ids()
+      if not resource_ids:
+        group_id = e.id
+      else:
+        # TODO(daiyip): support group that requires multiple resources.
+        group_id = resource_ids.pop()
+      groups[group_id].append(e)
+
+    for _, _, _ in lf.concurrent_map(
+        _run_group,
+        groups.values(),
+        max_workers=max(64, len(groups)),
+        timeout=self.timeout,
+        silence_on_errors=None,
+    ):
+      pass
+
+  def _evaluate_items(
+      self, evaluation: base.Evaluation, items: Iterator[base.Example]
+  ) -> None:
+    """Override run items to run in parallel."""
+    if self.concurrent_startup_delay is not None:
+      thread_delayed = {}
+      def _evaluate_item(item: base.Example):
+        thread_id = threading.current_thread().ident
+        if thread_id not in thread_delayed:
+          thread_delayed[thread_id] = True
+          time.sleep(random.randint(*self.concurrent_startup_delay))
+        return self.evaluate_item(evaluation, item)
+    else:
+      def _evaluate_item(item: base.Example):
+        return self.evaluate_item(evaluation, item)
+
+    for _, _, _ in lf.concurrent_map(
+        _evaluate_item,
+        items,
+        max_workers=evaluation.max_workers,
+        timeout=self.timeout,
+        silence_on_errors=None,
+    ):
+      pass