langfun 0.1.2.dev202509020804__py3-none-any.whl → 0.1.2.dev202511110805__py3-none-any.whl

langfun/core/eval/v2/reporting.py

@@ -32,8 +32,97 @@ _SUMMARY_FILE = 'summary.html'
 _EVALULATION_DETAIL_FILE = 'index.html'
 
 
+class ExampleHtmlGenerator(experiment_lib.Plugin):
+  """Plugin for generating HTML views for each evaluation example."""
+
+  def on_example_complete(
+      self, runner: Runner, experiment: Experiment, example: Example
+  ):
+    self._save_example_html(runner, experiment, example)
+
+  def _save_example_html(
+      self, runner: Runner, experiment: Experiment, example: Example
+  ) -> None:
+    """Saves the example in HTML format."""
+    current_run = runner.current_run
+    def _generate():
+      try:
+        with pg.timeit() as t:
+          html = example.to_html(
+              collapse_level=None,
+              enable_summary_tooltip=False,
+              extra_flags=dict(
+                  # For properly rendering the next link.
+                  num_examples=getattr(experiment, 'num_examples', None)
+              ),
+          )
+          html.save(
+              runner.current_run.output_path_for(
+                  experiment, f'{example.id}.html'
+              )
+          )
+        experiment.info(
+            f'\'{example.id}.html\' generated in {t.elapse:.2f} seconds. '
+        )
+      except BaseException as e:  # pylint: disable=broad-except
+        experiment.error(
+            f'Failed to generate \'{example.id}.html\'. '
+            f'Error: {e}, Stacktrace: \n{traceback.format_exc()}.',
+        )
+        raise e
+
+    def _copy():
+      src_file = current_run.input_path_for(experiment, f'{example.id}.html')
+      dest_file = current_run.output_path_for(experiment, f'{example.id}.html')
+
+      if src_file == dest_file:
+        return
+
+      if not pg.io.path_exists(src_file):
+        experiment.warning(
+            f'Skip copying \'{example.id}.html\' as '
+            f'{src_file!r} does not exist.'
+        )
+        return
+
+      try:
+        with pg.timeit() as t, pg.io.open(src_file, 'r') as src:
+          content = src.read()
+          with pg.io.open(dest_file, 'w') as dest:
+            dest.write(content)
+        experiment.info(
+            f'\'{example.id}.html\' copied in {t.elapse:.2f} seconds.'
+        )
+      except BaseException as e:  # pylint: disable=broad-except
+        experiment.error(
+            f'Failed to copy {src_file!r} to {dest_file!r}. Error: {e}.'
+        )
+        raise e
+
+    generate_example_html = current_run.generate_example_html
+    if (generate_example_html == 'all'
+        or (generate_example_html == 'new' and example.newly_processed)
+        or (isinstance(generate_example_html, list)
+            and example.id in generate_example_html)):
+      op = _generate
+    else:
+      op = _copy
+    runner.background_run(op)
+
+
 class HtmlReporter(experiment_lib.Plugin):
-  """Plugin for periodically generating HTML reports for the experiment."""
+  """Plugin for periodically generating HTML reports for the experiment.
+
+  The `HtmlReporter` plugin generates several HTML files during an experiment
+  run:
+    - A `summary.html` at the root of the run directory, summarizing all
+      evaluations in the experiment.
+    - An `index.html` for each leaf evaluation, detailing the evaluation
+      definition, metrics, and logs.
+
+  These reports are updated periodically in the background during the run,
+  allowing users to monitor progress in near real-time.
+  """
 
   summary_interval: Annotated[
       int,
@@ -127,7 +216,6 @@ class HtmlReporter(experiment_lib.Plugin):
   def on_example_complete(
       self, runner: Runner, experiment: Experiment, example: Example
   ):
-    self._save_example_html(runner, experiment, example)
     self._maybe_update_experiment_html(runner, experiment)
     self._maybe_update_summary(runner)
 
@@ -197,72 +285,3 @@ class HtmlReporter(experiment_lib.Plugin):
         runner.background_run(_save)
       else:
         _save()
-
-  def _save_example_html(
-      self, runner: Runner, experiment: Experiment, example: Example
-  ) -> None:
-    """Saves the example in HTML format."""
-    current_run = runner.current_run
-    def _generate():
-      try:
-        with pg.timeit() as t:
-          html = example.to_html(
-              collapse_level=None,
-              enable_summary_tooltip=False,
-              extra_flags=dict(
-                  # For properly rendering the next link.
-                  num_examples=getattr(experiment, 'num_examples', None)
-              ),
-          )
-          html.save(
-              runner.current_run.output_path_for(
-                  experiment, f'{example.id}.html'
-              )
-          )
-        experiment.info(
-            f'\'{example.id}.html\' generated in {t.elapse:.2f} seconds. '
-        )
-      except BaseException as e:  # pylint: disable=broad-except
-        experiment.error(
-            f'Failed to generate \'{example.id}.html\'. '
-            f'Error: {e}, Stacktrace: \n{traceback.format_exc()}.',
-        )
-        raise e
-
-    def _copy():
-      src_file = current_run.input_path_for(experiment, f'{example.id}.html')
-      dest_file = current_run.output_path_for(experiment, f'{example.id}.html')
-
-      if src_file == dest_file:
-        return
-
-      if not pg.io.path_exists(src_file):
-        experiment.warning(
-            f'Skip copying \'{example.id}.html\' as '
-            f'{src_file!r} does not exist.'
-        )
-        return
-
-      try:
-        with pg.timeit() as t, pg.io.open(src_file, 'r') as src:
-          content = src.read()
-          with pg.io.open(dest_file, 'w') as dest:
-            dest.write(content)
-        experiment.info(
-            f'\'{example.id}.html\' copied in {t.elapse:.2f} seconds.'
-        )
-      except BaseException as e:  # pylint: disable=broad-except
-        experiment.error(
-            f'Failed to copy {src_file!r} to {dest_file!r}. Error: {e}.'
-        )
-        raise e
-
-    generate_example_html = current_run.generate_example_html
-    if (generate_example_html == 'all'
-        or (generate_example_html == 'new' and example.newly_processed)
-        or (isinstance(generate_example_html, list)
-            and example.id in generate_example_html)):
-      op = _generate
-    else:
-      op = _copy
-    runner.background_run(op)

langfun/core/eval/v2/reporting_test.py

@@ -29,7 +29,12 @@ class ReportingTest(unittest.TestCase):
     experiment = eval_test_helper.test_experiment()
     checkpointer = checkpointing.BulkCheckpointer('checkpoint.jsonl')
     reporter = reporting.HtmlReporter()
-    run = experiment.run(root_dir, 'new', plugins=[checkpointer, reporter])
+    example_html_generator = reporting.ExampleHtmlGenerator()
+    run = experiment.run(
+        root_dir,
+        'new',
+        plugins=[checkpointer, reporter, example_html_generator]
+    )
     self.assertTrue(
         pg.io.path_exists(os.path.join(run.output_root, 'summary.html'))
     )
@@ -52,8 +57,10 @@ class ReportingTest(unittest.TestCase):
     root_dir = os.path.join(tempfile.mkdtemp(), 'test_reporting2')
     experiment = eval_test_helper.test_experiment()
     run = experiment.run(
-        root_dir, 'new', plugins=[checkpointer, reporter],
-        warm_start_from=run.output_root
+        root_dir,
+        'new',
+        plugins=[checkpointer, reporter, example_html_generator],
+        warm_start_from=run.output_root,
     )
     self.assertTrue(
         pg.io.path_exists(os.path.join(run.output_root, 'summary.html'))
@@ -105,7 +112,12 @@ class ReportingTest(unittest.TestCase):
                   .test_experiment_with_example_html_generation_error())
     checkpointer = checkpointing.BulkCheckpointer('checkpoint.jsonl')
     reporter = reporting.HtmlReporter()
-    run = experiment.run(root_dir, 'new', plugins=[checkpointer, reporter])
+    example_html_generator = reporting.ExampleHtmlGenerator()
+    run = experiment.run(
+        root_dir,
+        'new',
+        plugins=[checkpointer, reporter, example_html_generator]
+    )
     self.assertTrue(
         pg.io.path_exists(os.path.join(run.output_root, 'summary.html'))
     )
@@ -132,8 +144,10 @@ class ReportingTest(unittest.TestCase):
     experiment = (eval_test_helper
                   .test_experiment_with_example_html_generation_error())
     run = experiment.run(
-        root_dir, 'new', plugins=[checkpointer, reporter],
-        warm_start_from=run.output_root
+        root_dir,
+        'new',
+        plugins=[checkpointer, reporter, example_html_generator],
+        warm_start_from=run.output_root,
     )
     self.assertTrue(
         pg.io.path_exists(os.path.join(run.output_root, 'summary.html'))

langfun/core/eval/v2/runners.py

@@ -42,7 +42,14 @@ _RUN_MANIFEST = 'run.json'
 
 
 class RunnerBase(Runner):
-  """A simple runner that runs evaluations and their examples sequentially."""
+  """Base class for runners with plugin support and IO pooling.
+
+  `RunnerBase` provides the basic runner functionalities such as plugin
+  integration for checkpointing, reporting and progress tracking.
+  It also manages a thread pool for background IO operations.
+  Subclasses should implement `_run` and `_evaluate_items` for different
+  execution strategies.
+  """
 
   tqdm: Annotated[
       bool,
@@ -397,11 +404,12 @@ class RunnerBase(Runner):
 
 
 class SequentialRunner(RunnerBase):
-  """Sequential runner.
+  """A runner that executes evaluations and examples sequentially.
 
-  Sequential runner runs all evaluations and their examples in sequence,
-  as well as the background tasks, it allows the developer to catch all
-  exceptions thrown from the background tasks, making it easier to debug.
+  The sequential runner executes all evaluations and their examples in the
+  calling thread. Background tasks are also run sequentially, which makes it
+  easier to debug as exceptions from background tasks will be raised
+  immediately.
   """
 
   NAME = 'sequential'
@@ -426,7 +434,13 @@ class SequentialRunner(RunnerBase):
 
 
 class DebugRunner(SequentialRunner):
-  """Debug runner."""
+  """A runner for debugging evaluations.
+
+  The debug runner is a sequential runner that only runs the first example
+  of each evaluation, with `raise_if_has_error` enabled. This is useful for
+  quickly identifying issues in evaluation logic during development.
+  Checkpointers are disabled for this runner.
+  """
 
   NAME = 'debug'
 
@@ -444,7 +458,13 @@ class DebugRunner(SequentialRunner):
 
 
 class ParallelRunner(RunnerBase):
-  """Parallel runner."""
+  """A runner that executes evaluations and examples in parallel.
+
+  The parallel runner groups evaluations by their required resources
+  (e.g., specific LLMs) and runs evaluations that do not share resources in
+  parallel. Within each evaluation, examples are also processed in parallel
+  using threads, up to `Evaluation.max_workers`.
+  """
 
   NAME = 'parallel'
 

langfun/core/eval/v2/runners_test.py

@@ -100,6 +100,9 @@ class RunnerTest(unittest.TestCase):
         print(i, pg.diff(x, y))
       self.assertIs(x, y)
 
+
+class SequentialRunnerTest(RunnerTest):
+
   def test_basic(self):
     plugin = TestPlugin()
     exp = eval_test_helper.test_experiment()

langfun/core/langfunc.py

@@ -32,146 +32,43 @@ _TLS_LFUN_CALL_STACK = '_langfunc_callstack'
 # NOTE(daiyip): Only the template string belongs to the positional arguments,
 # all others are keyword-only for clarity.
 @pg.use_init_args(['template_str'])
-class LangFunc(
-    template_lib.Template,
-):
-  r"""Base class for natural-language driven component.
-
-  ``LangFunc`` is a language-driven component that enables users to
-  seamlessly interact with Language Models (LLMs) using a blend of natural
-  language and code. It empowers users to easily modularize prompt/execution
-  logics, compose them, and simplify the creation of Language Model (LLM)-based
-  components and applications.
-
-  LangFunc can be conceptualized as a string template with embeddable code,
-  but it distinguishes itself from traditional template systems in four key
-  ways.
-
-  Firstly, it enables easy modularization of templates along with the required
-  values with OO principles, providing a reusable way for LLM-based content
-  generation. For example:
-
-    ```
-    class FewshotExamples(lf.LangFunc):
-      '''Base for fewshot prompt.
-
-      {% for example in examples %}
-      {{ example }}
-      {% endfor %}
-      '''
-
-    # Usage 1: __init__ time binding.
-    assert FewshotPrompt(examples=['foo', 'bar'])() == 'foo\nbar'
-
-    # Usage 2: __call__ time binding.
-    assert FewshotPrompt()(examples=['foo', 'bar']) == 'foo\nbar'
-
-    class ToolDescription(lf.LangFunc):
-      '''Tool descriptions.
-
-      {% for tool in tools %}
-      {{ tool.description }}
-      {% endfor %}
-      '''
-      # We want to constrain tools to be a list of `Tool` objects.
-      tools: list[Tool]
-
-    # Raises: runtime type checking will fail on [1, 2, 3].
-    ToolDescription(tools=[1, 2, 3])
-    ```
-
-  Secondly, it has the capability to compose multiple LangFuncs together,
-  enabling the accomplishment of complex language tasks with maximum reuse.
-  It allows users to provide program inputs to all the LangFuncs within a
-  composition at the top level, significantly simplifying the process of
-  providing context for users. For example:
-
-    ```
-    class ReAct(lf.LangFunc):
-      '''ReAct prompt for tool-use.
-
-      {{ preamble }}
-      {{ tool_description }}
-      {{ tool_examples }}
-      {{ user_input }}
-      '''
-      # Default preamble, which could be overriden from subclass
-      # or parsed from the `__init__` argument.
-      preamble = 'Please help me on my task based on the following tools.',
-
-    react = ReAct(
-        tool_description=ToolDescription()
-        tool_examples=FewshotExamples(),
-        # Partially bind `tools` and `examples`.
-        tools=my_tools,
-        examples=[t.examples for t in my_tools]
-        )
-
-    # Late bind `user_input` at __call__ time.
-    react(user_input='Help me get a lunch to go, veggie please.' )
-    ```
-
-  Thirdly, it allows the flexibility to encapsulate complex compositions to
-  reusable classes and modify them. For example:
-
-    ```
-    # The compound decorator converts a function into a LangFunc.
-    @lf.compound
-    def react_with_tools(preamble, tools: list[Tool]):
-      return ReAct(
-          preamble=preamble,
-          tool_description=ToolDescription()
-          tool_examples=FewshotExamples(),
-          # Partially bind `tools` and `examples`.
-          tools=my_tools,
-          examples=[t.examples for t in my_tools]
-      )
+class LangFunc(template_lib.Template):
+  r"""Base class for Language-based functions.
 
-    # Actually, the entire chat application is a LangFunc.
-    class Chat(lt.LangFunc):
-      '''LLM-based Chat application.
+  LangFunc represents a function powered by a language model. It is a subclass
+  of `lf.Template` and can be thought of as a `lf.Template` augmented with an LM
+  and an output transformation. Calling a `lf.LangFunc` is equivalent to calling
+  the LM with the rendered prompt and transforming the output.
 
-      llm({{ prompt }})
-      '''
+  LangFunc can be directly constructed and used.
 
-    chat = Chat(
-        llm=Bard24B(),
-        prompt=react_with_tools(
-            preamble=(
-                f'Please help me solve my problem using tools. '
-                f'Current time is {{datetime.datetime.now()}}'),
-            tools=my_tools))
+  ```python
+  import langfun as lf
 
-    chat(user_input='Help me get a lunch to go, veggie please.')
-    ```
+  func = lf.LangFunc("Hello, {{name}}!")
+  print(func(name="Gemini", lm=lf.llms.Gemini25Flash()))
+  # Output: Hello, how are you today?
+  ```
 
-  Fourthly, LangFunc is built on top of PyGlove symbolic programming power,
-  it could be manipulated programmatically, turned into a space for data
-  sampling, or even tuned by AutoML. For example:
+  Or it can be subclassed:
 
-    ```
-    import pyglove as pg
+  ```python
+  import langfun as lf
 
-    prompt_space = react_with_tools(
-        preamble=pg.oneof([
-            'Help me solve my problem using the following tools:',
-            'Help me with the tools below:',
-            ...
-        ])
-        # Choose any two of the tools for generating data.
-        tools=pg.manyof(2, [
-            google_search(...),
-            doordash(...),
-            ...
-        ])
+  class Compute(lf.LangFunc):
+    '''Compute a simple arithmetic expression.
 
-    for prompt in pg.random_sample(prompt_space):
-      print(prompt(user_input='Help me book a conf room please.'))
+    {{expression}} = ?
+    '''
+    expression: str
 
-    ```
+    def transform_output(self, lm_output: lf.Message) -> lf.Message:
+      lm_output.metadata.result = float(lm_output.text)
+      return lm_output
 
-  For more capabilities on symbolic programming with PyGlove, please checkout
-  https://pyglove.readthedocs.io/en/latest/.
+  r = Compute(expression="1 + 1")(lm=lf.llms.Gemini25Flash())
+  print(r.result)
+  # Output: 2.0
 
   Final note: always include these capitalized words if you don't want to treat
   the docstr as the template str: THIS IS NOT A TEMPLATE. So as a result, this
@@ -305,6 +202,24 @@ class LangFunc(
       message_cls: Type[message_lib.Message] = message_lib.UserMessage,
       **kwargs,
   ) -> message_lib.Message:
+    """Renders the template and transforms it as LM input message.
+
+    Args:
+      allow_partial: If True, allows partial rendering, which leaves unresolved
+        variables in place in the output text. Otherwise, raises error when
+        there are unresolved variables.
+      implicit: If True, reuse the rendering output if a parent `lf.Template`
+        is rendering current `lf.Template` multiple times. This is important
+        for making sure all references to the same `lf.Template` within a single
+        top-level rendering would return the same result. If False, every call
+        to `render` will trigger the actual rendering process.
+      message_cls: The message class used for creating the return value.
+      **kwargs: Values for template variables, which override values from
+        member attributes or context.
+
+    Returns:
+      A Message object containing the rendered result.
+    """
     lm_input = super().render(
         allow_partial=allow_partial,
         implicit=implicit,

langfun/core/langfunc_test.py

@@ -82,7 +82,7 @@ class LangFuncCallTest(unittest.TestCase):
 
     i = l.render()
     self.assertEqual(i, 'Hello')
-    self.assertEqual(i, message.UserMessage('Hello'))
+    self.assertEqual(i, message.UserMessage('Hello', __template_input__={}))
     self.assertEqual(i.tags, ['rendered'])
 
     r = l()
@@ -96,7 +96,9 @@ class LangFuncCallTest(unittest.TestCase):
     self.assertEqual(r.tags, ['lm-response', 'lm-output'])
     self.assertEqual(
         r.source,
-        message.UserMessage('Hello', metadata=dict(cache_seed=0))
+        message.UserMessage(
+            'Hello', metadata=dict(cache_seed=0, __template_input__={})
+        )
     )
     self.assertEqual(r.source.tags, ['rendered', 'lm-input'])
 
@@ -107,8 +109,8 @@ class LangFuncCallTest(unittest.TestCase):
         ' lm=ExcitedEchoer(sampling_options=LMSamplingOptions(temperature=None,'
         ' max_tokens=None, n=1, top_k=40, top_p=None, stop=None,'
         ' random_seed=None, logprobs=False, top_logprobs=None,'
-        ' max_thinking_tokens=None, reasoning_effort=None), cache=None,'
-        ' max_concurrency=None, timeout=120.0, max_attempts=5,'
+        ' max_thinking_tokens=None, reasoning_effort=None, extras={}),'
+        ' cache=None, max_concurrency=None, timeout=120.0, max_attempts=5,'
         ' retry_interval=(5, 60), exponential_backoff=True,'
         ' max_retry_interval=300, debug=False))',
     )