langfun 0.0.2.dev20240330__py3-none-any.whl → 0.1.2.dev202501140804__py3-none-any.whl

langfun/core/eval/matching_test.py

@@ -103,7 +103,7 @@ class MatchingTest(unittest.TestCase):
         s.result,
         dict(
             experiment_setup=dict(
-                id='MyTask@3d87f97f',
+                id='MyTask@739a174b',
                 dir=s.dir,
                 model='StaticSequence',
                 prompt_template='{{example.question}}',
@@ -120,11 +120,19 @@ class MatchingTest(unittest.TestCase):
                 total=4,
                 failures=1,
                 failure_rate=0.25,
+                oop_failures=1,
+                oop_failure_rate=0.25,
+                non_oop_failures=0,
+                non_oop_failure_rate=0.0,
+                failure_breakdown={
+                    'MappingError.SchemaError.TypeError': 1
+                },
                 num_matches=2,
                 match_rate=0.5,
                 num_mismatches=1,
                 mismatch_rate=0.25,
             ),
+            usage=s.result.usage,
         ),
     )
     self.assertTrue(
@@ -144,22 +152,17 @@ class MatchingTest(unittest.TestCase):
             os.path.join(s.dir, matching.Matching.CACHE_JSON)
         )
     )
-    self.assertTrue(
-        os.path.exists(
-            os.path.join(s.dir, matching.Matching.MATCHES_JSON)
-        )
-    )
     self.assertTrue(
         os.path.exists(
             os.path.join(
-                s.dir, matching.Matching.MISMATCHES_JSON
+                s.dir, matching.Matching.OOP_FAILURES_JSON
             )
         )
     )
     self.assertTrue(
         os.path.exists(
             os.path.join(
-                s.dir, matching.Matching.FAILURES_JSON
+                s.dir, matching.Matching.NON_OOP_FAILURES_JSON
             )
         )
     )
@@ -174,7 +177,14 @@ class MatchingTest(unittest.TestCase):
     self.assertTrue(
         os.path.exists(
             os.path.join(
-                s.dir, matching.Matching.FAILURES_HTML
+                s.dir, matching.Matching.OOP_FAILURES_HTML
+            )
+        )
+    )
+    self.assertTrue(
+        os.path.exists(
+            os.path.join(
+                s.dir, matching.Matching.NON_OOP_FAILURES_HTML
             )
         )
     )

langfun/core/eval/patching.py

@@ -0,0 +1,130 @@
+# Copyright 2024 The Langfun Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Experiment patching for Langfun evaluations."""
+
+import inspect
+from typing import Union
+import langfun.core as lf
+from langfun.core import llms as lf_llms
+from langfun.core.eval import base
+import pyglove as pg
+
+
+#
+# Program-based patchers.
+#
+
+
+def patch_member(cls, key, value, parent_key: str | None = None):
+  """Patches a member of a class."""
+
+  def _rebind_fn(k, v, p):
+    if (
+        isinstance(p, cls)
+        and k.key == key
+        and (parent_key is None or (p and p.sym_path.key == parent_key))
+    ):
+      if inspect.isfunction(value):
+        return value(k, v, p)
+      return value
+    return v
+
+  return _rebind_fn
+
+
+def patch_lm(lm: Union[lf.LanguageModel, pg.hyper.OneOf]):  # pylint: disable=redefined-outer-name
+  """Patches the LLM of evaluations."""
+  return patch_member(base.Evaluable, "lm", lm)
+
+
+def patch_parsing_lm(lm: Union[lf.LanguageModel, pg.hyper.OneOf]):  # pylint: disable=redefined-outer-name
+  """Patches the parsing LLM of evaluations."""
+  return patch_member(base.Evaluable, "parsing_lm", lm)
+
+
+def patch_schema_fn(schema_fn: Union[pg.Functor, pg.hyper.OneOf]):
+  """Patches the schema_fn of evaluations."""
+  return patch_member(base.Evaluable, "schema_fn", schema_fn)
+
+
+def patch_prompt(prompt: Union[str, lf.Template, pg.hyper.OneOf]):
+  """Patches the prompt of evaluations."""
+  return patch_member(base.Evaluable, "prompt", prompt)
+
+
+def patch_inputs(inputs: Union[pg.Functor, pg.hyper.OneOf]):
+  """Patches the inputs used in evaluations."""
+  return patch_member(base.Evaluable, "inputs", inputs)
+
+
+def patch_additional_args(**kwargs):
+  """Patches additional_args."""
+
+  def value_fn(k, unused_v, p):
+    # We infer the symbolic value for the old args, as it might be a
+    # contextual attribute referring to its containing object.
+    old_args = p.sym_inferred(k.key)
+    if old_args:
+      old_args = dict(old_args)
+      old_args.update(kwargs)
+      return old_args
+    return kwargs
+
+  return patch_member(base.Evaluable, "additional_args", value_fn)
+
+
+#
+# String-based patching.
+#
+
+_NAMED_MODELS = {
+    # GPT models.
+    "gpt35turbo": lf_llms.Gpt35Turbo,
+    "gpt35turbo16k": lf_llms.Gpt35Turbo16K,
+    "gpt4": lf_llms.Gpt4,
+    "gpt4turbo": lf_llms.Gpt4Turbo,
+    # Anthropic models.
+    "haiku": lf_llms.Claude3Haiku,
+    "claude3haiku": lf_llms.Claude3Haiku,
+    "opus": lf_llms.Claude3Opus,
+    "claude3opus": lf_llms.Claude3Opus,
+    "sonnet": lf_llms.Claude3Sonnet,
+    "claude3sonnet": lf_llms.Claude3Opus,
+}
+
+
+def model_by_name(name: str) -> lf.LanguageModel:
+  """Gets model by name."""
+  name = name.strip().lower()
+  if name in _NAMED_MODELS:
+    return _NAMED_MODELS[name]()
+  raise ValueError(f"Unknown model name: {name}")
+
+
+@pg.patcher(auto_typing=True)
+def lm(unused_eval, models: list[str]):
+  """Patch the LM used for benchmarking."""
+  return patch_lm(pg.oneof([model_by_name(name) for name in models]))
+
+
+@pg.patcher(auto_typing=True)
+def temperature(unused_eval, value: float):
+  """Patch the temperature used for benchmarking."""
+  return patch_member(lf.LMSamplingOptions, "temperature", value)
+
+
+@pg.patcher(auto_typing=True)
+def max_tokens(unused_eval, value: int | None):
+  """Patch the temperature used for benchmarking."""
+  return patch_member(lf.LMSamplingOptions, "max_tokens", value)

langfun/core/eval/patching_test.py

@@ -0,0 +1,170 @@
+# Copyright 2024 The Langfun Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for evaluation patching."""
+
+import unittest
+from langfun.core import llms as lf_llms
+from langfun.core.eval import base
+from langfun.core.eval import patching
+import pyglove as pg
+
+
+class PatchingCommonTest(unittest.TestCase):
+
+  def test_patch_member(self):
+    class A(pg.Object):
+      x: int = 1
+
+    class B(pg.Object):
+      a: A
+
+    b = B(A())
+    pg.patch(b, [patching.patch_member(A, 'x', 2)])
+    self.assertEqual(b, B(A(2)))
+
+  def test_patch_args(self):
+    s = base.Suite(
+        [base.Evaluation(inputs=base.as_inputs([1]))],
+        additional_args=dict(x=1, y=2),
+    )
+    pg.patch(s, [patching.patch_additional_args(x=3, z=4)])
+    self.assertTrue(
+        pg.eq(
+            s,
+            base.Suite(
+                [
+                    base.Evaluation(
+                        inputs=base.as_inputs([1]),
+                        additional_args=dict(x=3, y=2, z=4),
+                    )
+                ],
+                additional_args=dict(x=3, y=2, z=4),
+            ),
+        )
+    )
+
+  def test_patch_lm(self):
+    s = base.Suite(
+        [base.Evaluation(inputs=base.as_inputs([1]))],
+        lm=lf_llms.Gpt35Turbo(),
+    )
+    pg.patch(
+        s, [patching.patch_lm(pg.oneof([lf_llms.Gpt35Turbo(), lf_llms.Gpt4()]))]
+    )
+    self.assertTrue(
+        pg.eq(
+            s,
+            base.Suite(
+                [
+                    base.Evaluation(
+                        inputs=base.as_inputs([1]),
+                        lm=pg.oneof([lf_llms.Gpt35Turbo(), lf_llms.Gpt4()]),
+                    )
+                ],
+                lm=pg.oneof([lf_llms.Gpt35Turbo(), lf_llms.Gpt4()]),
+            ),
+        )
+    )
+
+  def test_patch_parsing_lm(self):
+    s = base.Suite(
+        [base.Evaluation(inputs=base.as_inputs([1]))],
+        lm=lf_llms.Gpt4(),
+    )
+    pg.patch(s, [patching.patch_parsing_lm(lf_llms.Gpt35Turbo())])
+    self.assertTrue(
+        pg.eq(
+            s,
+            base.Suite(
+                [
+                    base.Evaluation(
+                        inputs=base.as_inputs([1]),
+                        lm=lf_llms.Gpt4(),
+                        parsing_lm=lf_llms.Gpt35Turbo(),
+                    )
+                ],
+                # NOTE(daiyip): Suite does not have `parsing_lm` as one of its
+                # variable keyword fields yet, so patching does not add to it.
+                # This is okay since we only care about the leaf nodes.
+                lm=lf_llms.Gpt4(),
+            ),
+        )
+    )
+
+  def test_patch_prompt(self):
+    e = base.Evaluation(inputs=base.as_inputs([1]))
+    pg.patch(e, [patching.patch_prompt('Q: {{example.question}}')])
+    self.assertTrue(
+        pg.eq(
+            e,
+            base.Evaluation(
+                inputs=base.as_inputs([1]),
+                prompt='Q: {{example.question}}',
+            ),
+        )
+    )
+
+  def test_patch_inputs(self):
+    e = base.Evaluation(inputs=base.as_inputs([1]))
+    pg.patch(e, [patching.patch_inputs(base.as_inputs([2]))])
+    self.assertTrue(
+        pg.eq(
+            e,
+            base.Evaluation(
+                inputs=base.as_inputs([2]),
+            ),
+        )
+    )
+
+  def test_patch_schema_fn(self):
+    @pg.functor()
+    def int_schema():
+      return int
+
+    e = base.Evaluation(inputs=base.as_inputs([1]))
+    pg.patch(e, [patching.patch_schema_fn(int_schema())])
+    self.assertTrue(
+        pg.eq(
+            e,
+            base.Evaluation(
+                inputs=base.as_inputs([1]),
+                schema_fn=int_schema(),
+            ),
+        )
+    )
+
+
+class StringPatcheTest(unittest.TestCase):
+
+  def test_lm(self):
+    target = pg.patch(
+        base.Evaluation(inputs=base.as_inputs([1])),
+        ['lm?haiku:gpt4', 'max_tokens?1024', 'temperature?0.7'],
+    )
+    self.assertEqual(
+        target.lm,
+        pg.oneof([
+            lf_llms.Claude3Haiku(temperature=0.7, max_tokens=1024),
+            lf_llms.Gpt4(temperature=0.7, max_tokens=1024),
+        ]),
+    )
+    with self.assertRaisesRegex(ValueError, 'Unknown model name'):
+      pg.patch(
+          base.Evaluation(inputs=base.as_inputs([1])),
+          ['lm?gpt2'],
+      )
+
+
+if __name__ == '__main__':
+  unittest.main()

langfun/core/eval/scoring.py

@@ -61,25 +61,36 @@ class Scoring(base.Evaluation):
     super()._reset()
     self._scored = []
 
-  def audit(self, example: Any, output: Any, message: lf.Message) -> None:
+  def audit_processed(
+      self, example_idx: int, example: Any, output: Any, message: lf.Message,
+      dryrun: bool = False
+  ) -> None:
+    del example_idx
     score = self.score(example, output)
+
+    if dryrun:
+      lf.console.write('')
+      lf.console.write(
+          str(score),
+          title='SCORE',
+          color='blue',
+      )
     self._scored.append((example, output, score, message))
 
   @abc.abstractmethod
   def score(self, example: Any, output: Any) -> float:
     """Scores the output against its input example."""
 
-  def _status(self, progress: lf.concurrent.Progress) -> dict[str, Any]:
+  def _eval_status(self, progress: lf.concurrent.Progress) -> dict[str, Any]:
     del progress
     return {
-        'Model': self.lm.model_id,
         'Average Score': {self.avg_score},
-        'Scored': '%.2f%% (%d/%d)' % (
+        'Scored': '%.3f%% (%d/%d)' % (
             self.score_rate * 100,
             self.num_scored,
             self.num_completed,
         ),
-        'Failed': '%.2f%% (%d/%d)' % (
+        'Failed': '%.3f%% (%d/%d)' % (
             self.failure_rate * 100,
             self.num_failures,
             self.num_completed,
@@ -90,8 +101,8 @@ class Scoring(base.Evaluation):
     assert self.result is not None
     m = self.result.metrics
     return (
-        'COMPLETED(%s): AvgScore=%f Scored=%.2f%% (%d/%d) '
-        'Failures=%.2f%% (%d/%d)'
+        'COMPLETED(%s): AvgScore=%f Scored=%.3f%% (%d/%d) '
+        'Failures=%.3f%% (%d/%d)'
     ) % (
         run_status,
         m.avg_score,
@@ -103,8 +114,8 @@ class Scoring(base.Evaluation):
         m.total,
     )
 
-  def summarize(self) -> pg.Dict:
-    result = super().summarize()
+  def finalize(self) -> pg.Dict:
+    result = super().finalize()
     result.metrics.update(
         num_scored=self.num_scored,
         score_rate=self.score_rate,
@@ -118,16 +129,12 @@ class Scoring(base.Evaluation):
     super().save(definition, result, report)
 
     if result:
-
-      def force_dict(v):
-        return pg.object_utils.json_conversion.strip_types(pg.to_json(v))
-
       # Save scored.
       pg.save(
           [
               # We force the output to be dict as its type may be defined
               # within functors which could be deserialized.
-              pg.Dict(input=input, output=force_dict(output), score=score)
+              pg.Dict(input=input, output=output, score=score)
               for input, output, score, _ in self.scored
           ],
           os.path.join(self.dir, Scoring.SCORED_JSON),
@@ -148,32 +155,30 @@ class Scoring(base.Evaluation):
   def _render_result_row(self, s: io.StringIO):
     super()._render_result_row(s)
     s.write(
-        '<td><span style="color:blue">%.2f</span></td>' % self.avg_score
+        '<td><span style="color:blue">%.3f</span></td>' % self.avg_score
     )
     s.write(
         '<td><span style="color:red">%s</span>%s</td>'
         % (
-            '%.2f%% ' % (self.score_rate * 100),
+            '%.3f%% ' % (self.score_rate * 100),
             '<a href="%s">(%d/%d)</a>'
             % (self.scored_link, self.num_scored, self.num_completed),
         )
     )
 
-  def _render_metric(self, s: io.StringIO) -> None:
+  def _render_summary_metrics(self, s: io.StringIO) -> None:
     """Renders metrics in HTML."""
     assert self.result is not None
     m = self.result.metrics
-    s.write(
-        '<a title="Average score (%d/%d)" href="%s" style="color:green">%s</a>'
-        % (
-            m.num_scored,
-            m.total,
-            self.scored_link,
-            '%.2f%%' % (m.score_rate * 100),
-        )
+    self._render_link(
+        s,
+        'Average score (%d/%d)' % (m.num_scored, m.total),
+        '%.3f (%.3f%%)' % (m.avg_score, m.score_rate * 100),
+        'color:green',
+        lambda: self.scored_link,
     )
     s.write(' | ')
-    super()._render_metric(s)
+    super()._render_summary_metrics(s)
 
   def _render_scored(self, s: io.StringIO) -> None:
     """Formats the matched cases into html."""
@@ -189,9 +194,13 @@ class Scoring(base.Evaluation):
     for i, (example, output, score, message) in enumerate(self.scored):
       bgcolor = 'white' if i % 2 == 0 else '#DDDDDD'
       s.write(f'<tr style="background-color: {bgcolor}"><td>{i + 1}</td>')
-      input_str = pg.format(example, verbose=False)
+      input_str = pg.Html.escape(
+          pg.format(example, verbose=False, max_bytes_len=32)
+      )
       s.write(f'<td style="color:green;white-space:pre-wrap">{input_str}</td>')
-      output_str = pg.format(output, verbose=False)
+      output_str = pg.Html.escape(
+          pg.format(output, verbose=False, max_bytes_len=32)
+      )
       s.write(f'<td style="color:blue;white-space:pre-wrap">{output_str}</td>')
       s.write(f'<td style="color:magenta;white-space:pre-wrap">{score}</td>')
       s.write('<td>')

langfun/core/eval/scoring_test.py

@@ -81,7 +81,7 @@ class ScoringTest(unittest.TestCase):
         s.result,
         dict(
             experiment_setup=dict(
-                id='ConstraintFollowing@9e51bb9e',
+                id='ConstraintFollowing@5c88a5eb',
                 dir=s.dir,
                 model='StaticSequence',
                 prompt_template='{{example}}',
@@ -98,10 +98,16 @@ class ScoringTest(unittest.TestCase):
                 total=2,
                 failures=0,
                 failure_rate=0.0,
+                oop_failures=0,
+                oop_failure_rate=0.0,
+                non_oop_failures=0,
+                non_oop_failure_rate=0.0,
+                failure_breakdown={},
                 num_scored=2,
                 score_rate=1.0,
                 avg_score=0.5,
             ),
+            usage=s.result.usage,
         ),
     )
     self.assertTrue(
@@ -123,7 +129,12 @@ class ScoringTest(unittest.TestCase):
     )
     self.assertTrue(
         os.path.exists(
-            os.path.join(s.dir, scoring.Scoring.FAILURES_JSON)
+            os.path.join(s.dir, scoring.Scoring.OOP_FAILURES_JSON)
+        )
+    )
+    self.assertTrue(
+        os.path.exists(
+            os.path.join(s.dir, scoring.Scoring.NON_OOP_FAILURES_JSON)
         )
     )
     self.assertTrue(
@@ -142,7 +153,14 @@ class ScoringTest(unittest.TestCase):
     self.assertTrue(
         os.path.exists(
             os.path.join(
-                s.dir, scoring.Scoring.FAILURES_HTML
+                s.dir, scoring.Scoring.OOP_FAILURES_HTML
+            )
+        )
+    )
+    self.assertTrue(
+        os.path.exists(
+            os.path.join(
+                s.dir, scoring.Scoring.NON_OOP_FAILURES_HTML
             )
         )
     )

langfun/core/eval/v2/__init__.py

@@ -0,0 +1,42 @@
+# Copyright 2024 The Langfun Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""langfun eval framework v2."""
+
+# pylint: disable=g-importing-member
+# pylint: disable=g-bad-import-order
+from langfun.core.eval.v2.experiment import Experiment
+from langfun.core.eval.v2.experiment import Suite
+from langfun.core.eval.v2.evaluation import Evaluation
+
+from langfun.core.eval.v2.example import Example
+from langfun.core.eval.v2.progress import Progress
+
+from langfun.core.eval.v2.metric_values import MetricValue
+from langfun.core.eval.v2.metric_values import Rate
+from langfun.core.eval.v2.metric_values import Average
+from langfun.core.eval.v2.metrics import Metric
+from langfun.core.eval.v2 import metrics
+
+from langfun.core.eval.v2.experiment import Plugin
+from langfun.core.eval.v2.experiment import Runner
+from langfun.core.eval.v2 import runners
+
+# Plugins
+from langfun.core.eval.v2.checkpointing import BulkCheckpointer
+from langfun.core.eval.v2.checkpointing import PerExampleCheckpointer
+from langfun.core.eval.v2.reporting import HtmlReporter
+
+
+# pylint: enable=g-bad-import-order
+# pylint: enable=g-importing-member