langfun 0.0.2.dev20240330__py3-none-any.whl → 0.1.2.dev202501140804__py3-none-any.whl

langfun/core/eval/v2/progress.py

@@ -0,0 +1,348 @@
+# Copyright 2024 The Langfun Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Progress reporting for evaluation."""
+
+import datetime
+import threading
+import time
+from typing import Annotated, Any
+import pyglove as pg
+
+
+class Progress(pg.Object, pg.views.HtmlTreeView.Extension):
+  """Evaluation progress."""
+
+  num_total: Annotated[
+      int | None,
+      (
+          'Total number of items to be processed. '
+          'If None, the progress is not started.'
+      )
+  ] = None
+  num_processed: Annotated[
+      int,
+      (
+          'Number of items that have been processed without errors.'
+      )
+  ] = 0
+  num_failed: Annotated[
+      int,
+      (
+          'Number of items that have failed.'
+      )
+  ] = 0
+  num_skipped: Annotated[
+      int,
+      (
+          'Number of items that have been skipped.'
+      )
+  ] = 0
+  start_time: Annotated[
+      float | None,
+      (
+          'The start time of the progress. '
+          'If None, the progress is not started.'
+      )
+  ] = None
+  stop_time: Annotated[
+      float | None,
+      (
+          'The stop time of the progress. '
+          'If None, the progress is not stopped.'
+      )
+  ] = None
+  execution_summary: Annotated[
+      pg.object_utils.TimeIt.StatusSummary,
+      'The execution summary of the progress.'
+  ] = pg.object_utils.TimeIt.StatusSummary()
+
+  def _on_bound(self):
+    super()._on_bound()
+    self._progress_bar = None
+    self._time_label = None
+    self._lock = threading.Lock()
+
+  def reset(self) -> None:
+    """Resets the progress."""
+    self._sync_members(
+        num_total=None,
+        num_processed=0,
+        num_failed=0,
+        num_skipped=0,
+        start_time=None,
+        stop_time=None,
+        execution_summary=pg.object_utils.TimeIt.StatusSummary(),
+    )
+
+  @property
+  def num_completed(self) -> int:
+    """Returns the number of completed examples."""
+    return self.num_processed + self.num_failed + self.num_skipped
+
+  def __float__(self) -> float:
+    """Returns the complete rate in range [0, 1]."""
+    if self.num_total is None:
+      return float('nan')
+    return self.num_completed / self.num_total
+
+  @property
+  def is_started(self) -> bool:
+    """Returns whether the evaluation is started."""
+    return self.start_time is not None
+
+  @property
+  def is_stopped(self) -> bool:
+    """Returns whether the evaluation is stopped."""
+    return self.stop_time is not None
+
+  @property
+  def is_completed(self) -> bool:
+    """Returns whether the evaluation is completed."""
+    return (
+        self.num_total is not None
+        and self.num_completed == self.num_total
+    )
+
+  @property
+  def is_skipped(self) -> bool:
+    """Returns whether the evaluation is skipped."""
+    return (
+        self.num_total is not None
+        and self.num_skipped == self.num_total
+    )
+
+  @property
+  def is_failed(self) -> bool:
+    """Returns whether the evaluation is failed."""
+    return (
+        self.num_failed > 0
+        and self.num_failed + self.num_skipped == self.num_total
+    )
+
+  @property
+  def elapse(self) -> float | None:
+    """Returns the elapse time in seconds."""
+    if self.start_time is None:
+      return None
+    if self.stop_time is None:
+      return time.time() - self.start_time
+    return self.stop_time - self.start_time
+
+  @property
+  def start_time_str(self) -> str | None:
+    """Returns the start time string of the evaluation."""
+    if self.start_time is None:
+      return None
+    return time.strftime('%Y/%m/%d %H:%M:%S', time.localtime(self.start_time))
+
+  @property
+  def stop_time_str(self) -> str | None:
+    """Returns the complete time string of the evaluation."""
+    if self.stop_time is None:
+      return None
+    return time.strftime(
+        '%Y/%m/%d %H:%M:%S', time.localtime(self.stop_time)
+    )
+
+  def start(self, total: int) -> None:
+    """Marks the evaluation as started."""
+    assert self.start_time is None, self
+    self._sync_members(start_time=time.time(), num_total=total)
+    if self._progress_bar is not None:
+      self._progress_bar.update(total=total)
+    self._update_time_label()
+
+  def stop(self) -> None:
+    """Marks the evaluation as stopped."""
+    assert self.stop_time is None, self
+    self._sync_members(stop_time=time.time())
+    self._update_time_label()
+
+  def _sync_members(self, **kwargs: Any):
+    """Synchronizes the members of the progress."""
+    self.rebind(
+        **kwargs,
+        skip_notification=True,
+        raise_on_no_change=False,
+    )
+
+  def increment_processed(self, delta: int = 1) -> None:
+    """Updates the number of processed examples."""
+    assert self.is_started and not self.is_stopped, self
+    with self._lock:
+      self._sync_members(num_processed=self.num_processed + delta)
+    if self._progress_bar is not None:
+      self._progress_bar['Processed'].increment(delta)
+    self._update_time_label()
+
+  def increment_failed(self, delta: int = 1) -> None:
+    """Updates the number of failed examples."""
+    assert self.is_started and not self.is_stopped, self
+    with self._lock:
+      self._sync_members(num_failed=self.num_failed + delta)
+    if self._progress_bar is not None:
+      self._progress_bar['Failed'].increment(delta)
+    self._update_time_label()
+
+  def increment_skipped(self, delta: int = 1) -> None:
+    """Updates the number of skipped examples."""
+    assert self.is_started and not self.is_stopped, self
+    with self._lock:
+      self._sync_members(num_skipped=self.num_skipped + delta)
+    if self._progress_bar is not None:
+      self._progress_bar['Skipped'].increment(delta)
+    self._update_time_label()
+
+  def update_execution_summary(
+      self,
+      execution_status: dict[str, pg.object_utils.TimeIt.Status]
+  ) -> None:
+    """Updates the execution summary of the progress."""
+    with self._lock:
+      self.execution_summary.aggregate(execution_status)
+
+  def _sym_nondefault(self) -> dict[str, Any]:
+    """Overrides nondefault values so volatile values are not included."""
+    return dict()
+
+  #
+  # HTML view.
+  #
+
+  def _duration_text(self) -> str:
+    if self.start_time is None:
+      return '00:00:00'
+    return str(datetime.timedelta(seconds=self.elapse)).split('.')[0]
+
+  def _time_tooltip(self) -> pg.Html.WritableTypes:
+    time_info = pg.Dict(
+        duration=self._duration_text(),
+        last_update=(
+            time.strftime(    # pylint: disable=g-long-ternary
+                '%Y/%m/%d %H:%M:%S',
+                time.localtime(time.time())
+            ) if not self.is_stopped else self.stop_time_str
+        ),
+        start_time=self.start_time_str,
+        stop_time=self.stop_time_str,
+    )
+    if self.execution_summary:
+      time_info['execution'] = pg.Dict(
+          {
+              k: pg.Dict(
+                  num_started=v.num_started,
+                  num_ended=v.num_ended,
+                  num_failed=v.num_failed,
+                  avg_duration=round(v.avg_duration, 2),
+              ) for k, v in self.execution_summary.breakdown.items()
+          }
+      )
+    return pg.format(time_info, verbose=False)
+
+  def _html_tree_view(
+      self,
+      *,
+      view: pg.views.HtmlTreeView,
+      extra_flags: dict[str, Any] | None = None,
+      **kwargs
+  ) -> pg.Html:
+    """Renders the content of the progress bar."""
+    def _progress_bar():
+      return pg.views.html.controls.ProgressBar(
+          [
+              pg.views.html.controls.SubProgress(
+                  name='Skipped', value=self.num_skipped,
+              ),
+              pg.views.html.controls.SubProgress(
+                  name='Processed', value=self.num_processed,
+              ),
+              pg.views.html.controls.SubProgress(
+                  name='Failed', value=self.num_failed,
+              ),
+          ],
+          total=self.num_total,
+          interactive=interactive,
+      )
+
+    def _time_label():
+      css_class = 'not-started'
+      if self.is_started and not self.is_stopped:
+        css_class = 'started'
+      elif self.is_stopped:
+        css_class = 'stopped'
+      return pg.views.html.controls.Label(
+          self._duration_text(),
+          tooltip=self._time_tooltip(),
+          css_classes=[
+              'progress-time', css_class
+          ],
+          interactive=interactive,
+      )
+
+    extra_flags = extra_flags or {}
+    interactive = extra_flags.pop('interactive', True)
+    if interactive:
+      if self._progress_bar is None:
+        self._progress_bar = _progress_bar()
+      if self._time_label is None:
+        self._time_label = _time_label()
+      progress_bar = self._progress_bar
+      time_label = self._time_label
+    else:
+      progress_bar = _progress_bar()
+      time_label = _time_label()
+    return pg.Html.element(
+        'div', [progress_bar, time_label], css_classes=['eval-progress'],
+    )
+
+  def _update_time_label(self):
+    """Updates the time label of the progress."""
+    if self._time_label is None:
+      return
+    self._time_label.update(
+        text=self._duration_text(),
+        tooltip=self._time_tooltip(),
+        styles=dict(
+            color=(
+                'dodgerblue' if self.is_started
+                and not self.is_stopped else '#ccc'
+            ),
+        ),
+    )
+
+  @classmethod
+  def _html_tree_view_css_styles(cls) -> list[str]:
+    return super()._html_tree_view_css_styles() + [
+        """
+        .eval-progress {
+          display: inline-block;
+        }
+        .sub-progress.skipped {
+          background-color:yellow;
+        }
+        .sub-progress.processed {
+          background-color:#00B000;
+        }
+        .sub-progress.failed {
+          background-color:red;
+        }
+        .progress-time {
+          font-weight: normal;
+          margin-left: 10px;
+          border-radius: 5px;
+          color: #CCC;
+          padding: 5px;
+        }
+        """
+    ]

langfun/core/eval/v2/progress_test.py

@@ -0,0 +1,82 @@
+# Copyright 2024 The Langfun Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import math
+import time
+import unittest
+
+from langfun.core.eval.v2 import progress as progress_lib
+import pyglove as pg
+
+Progress = progress_lib.Progress
+
+
+class ProgressTest(unittest.TestCase):
+
+  def test_basic(self):
+    p = Progress()
+    self.assertFalse(p.is_started)
+    self.assertFalse(p.is_stopped)
+    self.assertFalse(p.is_completed)
+    self.assertFalse(p.is_skipped)
+    self.assertFalse(p.is_failed)
+    self.assertEqual(p.num_completed, 0)
+    self.assertIsNone(p.elapse)
+    self.assertIsNone(p.start_time_str)
+    self.assertIsNone(p.stop_time_str)
+    self.assertTrue(math.isnan(float(p)))
+
+    p.start(10)
+    self.assertEqual(p.num_total, 10)
+    self.assertTrue(p.is_started)
+    self.assertFalse(p.is_stopped)
+    self.assertIsNotNone(p.start_time_str)
+    self.assertIsNotNone(p.elapse)
+    self.assertEqual(float(p), 0.0)
+
+    with pg.views.html.controls.HtmlControl.track_scripts() as scripts:
+      p.increment_processed()
+      p.increment_failed()
+      p.increment_skipped()
+    # Does not triggger scripts as progress is not rendered yet.
+    self.assertEqual(len(scripts), 0)
+    self.assertEqual(p.num_completed, 3)
+    self.assertIn(
+        '3/10',
+        p.to_html(extra_flags=dict(interactive=True)).content,
+    )
+    with pg.views.html.controls.HtmlControl.track_scripts() as scripts:
+      p.increment_processed()
+      p.increment_failed()
+      p.increment_skipped()
+    self.assertEqual(len(scripts), 24)
+    self.assertEqual(p.num_completed, 6)
+    self.assertIn(
+        '6/10',
+        p.to_html(extra_flags=dict(interactive=False)).content,
+    )
+    p.update_execution_summary(
+        dict(
+            evaluate=pg.object_utils.TimeIt.Status(name='evaluate', elapse=1.0)
+        )
+    )
+    p.stop()
+    elapse1 = p.elapse
+    time.sleep(0.1)
+    self.assertEqual(p.elapse, elapse1)
+    self.assertTrue(p.is_stopped)
+    self.assertIsNotNone(p.stop_time_str)
+
+
+if __name__ == '__main__':
+  unittest.main()

langfun/core/eval/v2/progress_tracking.py

@@ -0,0 +1,210 @@
+# Copyright 2024 The Langfun Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tracking evaluation run progress."""
+
+import langfun.core as lf
+from langfun.core.eval.v2 import example as example_lib
+from langfun.core.eval.v2 import experiment as experiment_lib
+import pyglove as pg
+
+Runner = experiment_lib.Runner
+Experiment = experiment_lib.Experiment
+Example = example_lib.Example
+
+
+def progress_tracker(tqdm: bool = False) -> experiment_lib.Plugin:
+  """Creates a progress tracker as a plugin.
+
+  Args:
+    tqdm: If True, force using tqdm for progress update.
+
+  Returns:
+    The progress tracker plugin.
+  """
+  if tqdm or not lf.console.under_notebook():
+    return _TqdmProgressTracker()
+  else:
+    return _HtmlProgressTracker()
+
+
+class _HtmlProgressTracker(experiment_lib.Plugin):
+  """HTML progress tracker plugin."""
+
+  def on_run_start(
+      self,
+      runner: Runner,
+      root: Experiment
+  ) -> None:
+    # Display the experiment if running under notebook and not using tqdm.
+    assert lf.console.under_notebook()
+    with pg.view_options(
+        collapse_level=None,
+        extra_flags=dict(
+            current_run=runner.current_run,
+            interactive=True,
+        )
+    ):
+      lf.console.display(runner.current_run.experiment)
+
+
+ProgressBarId = int
+
+
+class _TqdmProgressTracker(experiment_lib.Plugin):
+  """Tqdm process updater plugin."""
+
+  def _on_bound(self):
+    super()._on_bound()
+    self._overall_progress: ProgressBarId | None = None
+    self._leaf_progresses: dict[str, ProgressBarId] = {}
+
+  def experiment_progress(
+      self, experiment: Experiment) -> lf.concurrent.ProgressBar:
+    """Returns the progress of the experiment."""
+    assert experiment.is_leaf
+    return self._leaf_progresses[experiment.id]
+
+  def on_run_start(
+      self,
+      runner: Runner,
+      root: Experiment
+  ) -> None:
+    """Called when a runner is started."""
+    self._overall_progress = lf.concurrent.ProgressBar.install(
+        label='All', total=len(root.leaf_nodes), color='blue'
+    )
+    self._leaf_progresses = {
+        leaf.id: lf.concurrent.ProgressBar.install(
+            label=f'[#{i + 1} - {leaf.id}]',
+            total=(len(runner.current_run.example_ids)
+                   if runner.current_run.example_ids else leaf.num_examples),
+            color='cyan',
+            status=None
+        )
+        for i, leaf in enumerate(root.leaf_nodes)
+    }
+    summary_link = Experiment.link(
+        runner.current_run.output_path_for(root, 'summary.html')
+    )
+    lf.console.write(f'Summary: {summary_link}.', color='green')
+
+  def on_run_complete(
+      self,
+      runner: Runner,
+      root: Experiment
+  ) -> None:
+    """Called when a runner is complete."""
+    lf.concurrent.ProgressBar.update(
+        self._overall_progress,
+        color='green',
+        status='ALL COMPLETED.',
+    )
+    lf.concurrent.ProgressBar.uninstall(self._overall_progress)
+    self._overall_progress = None
+    for progress in self._leaf_progresses.values():
+      lf.concurrent.ProgressBar.uninstall(progress)
+    self._leaf_progresses = {}
+
+  def on_experiment_start(
+      self,
+      runner: Runner,
+      experiment: Experiment
+  ) -> None:
+    """Called when an evaluation is started."""
+
+  def on_experiment_skipped(
+      self,
+      runner: Runner,
+      experiment: Experiment
+  ) -> None:
+    """Called when an evaluation is skipped."""
+    if experiment.is_leaf:
+      lf.concurrent.ProgressBar.update(
+          self.experiment_progress(experiment),
+          delta=experiment.progress.num_total,
+          status='Skipped.',
+      )
+      lf.concurrent.ProgressBar.update(
+          self._overall_progress,
+          status=f'Skipped {experiment.id}.',
+      )
+
+  def on_experiment_complete(
+      self,
+      runner: Runner,
+      experiment: Experiment
+  ) -> None:
+    """Called when an evaluation is complete."""
+    if experiment.is_leaf:
+      lf.concurrent.ProgressBar.update(
+          self.experiment_progress(experiment),
+          color='green',
+      )
+      lf.concurrent.ProgressBar.update(
+          self._overall_progress,
+          delta=1,
+          status=f'{experiment.id} COMPLETED.',
+      )
+
+  def on_example_start(
+      self,
+      runner: Runner,
+      experiment: Experiment,
+      example: Example
+  ) -> None:
+    """Called when an evaluation example is started."""
+
+  def on_example_skipped(
+      self,
+      runner: Runner,
+      experiment: Experiment,
+      example: Example
+  ) -> None:
+    """Called when an evaluation example is skipped."""
+    del runner, example
+    lf.concurrent.ProgressBar.update(
+        self.experiment_progress(experiment),
+        delta=1,
+    )
+
+  def on_example_complete(
+      self,
+      runner: Runner,
+      experiment: Experiment,
+      example: Example
+  ) -> None:
+    """Called when an evaluation example is complete."""
+    lf.concurrent.ProgressBar.update(
+        self.experiment_progress(experiment),
+        delta=1,
+        status=self.status(experiment),
+    )
+
+  def status(self, experiment: Experiment) -> str:
+    """Returns the progress text of the evaluation."""
+    items = []
+    for metric in experiment.metrics:
+      for metric_value in metric.values():
+        items.append(
+            f'{metric_value.sym_path.key}={metric_value.format(verbose=True)}'
+        )
+    error_tags = {}
+    for entry in experiment.progress.execution_summary.breakdown.values():
+      error_tags.update(entry.error_tags)
+
+    if error_tags:
+      items.extend(
+          [f'{k}={v}' for k, v in error_tags.items() if v > 0]
+      )
+    return ', '.join(items)

langfun/core/eval/v2/progress_tracking_test.py

@@ -0,0 +1,66 @@
+# Copyright 2024 The Langfun Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import contextlib
+import io
+import os
+import tempfile
+import unittest
+
+from langfun.core import console as lf_console
+from langfun.core.eval.v2 import eval_test_helper
+from langfun.core.eval.v2 import progress_tracking    # pylint: disable=unused-import
+from langfun.core.eval.v2 import runners as runners_lib  # pylint: disable=unused-import
+import pyglove as pg
+
+
+class HtmlProgressTrackerTest(unittest.TestCase):
+
+  def test_track_progress(self):
+    result = pg.Dict()
+    def display(x):
+      result['view'] = x.to_html()
+
+    lf_console._notebook = pg.Dict(
+        display=display
+    )
+    root_dir = os.path.join(tempfile.gettempdir(), 'test_html_progress_tracker')
+    experiment = eval_test_helper.test_experiment()
+    _ = experiment.run(root_dir, 'new', plugins=[])
+    self.assertIsInstance(result['view'], pg.Html)
+    lf_console._notebook = None
+
+
+class TqdmProgressTrackerTest(unittest.TestCase):
+
+  def test_basic(self):
+    root_dir = os.path.join(tempfile.gettempdir(), 'test_tqdm_progress_tracker')
+    experiment = eval_test_helper.test_experiment()
+    string_io = io.StringIO()
+    with contextlib.redirect_stderr(string_io):
+      _ = experiment.run(root_dir, 'new', plugins=[])
+    self.assertIn('All: 100%', string_io.getvalue())
+
+  def test_with_example_ids(self):
+    root_dir = os.path.join(
+        tempfile.gettempdir(), 'test_tqdm_progress_tracker_with_example_ids'
+    )
+    experiment = eval_test_helper.test_experiment()
+    string_io = io.StringIO()
+    with contextlib.redirect_stderr(string_io):
+      _ = experiment.run(root_dir, 'new', example_ids=[1], plugins=[])
+    self.assertIn('All: 100%', string_io.getvalue())
+
+
+if __name__ == '__main__':
+  unittest.main()