langfun 0.0.2.dev20240330__py3-none-any.whl → 0.0.2.dev20240511__py3-none-any.whl

langfun/core/eval/patching.py

@@ -0,0 +1,130 @@
+# Copyright 2024 The Langfun Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Experiment patching for Langfun evaluations."""
+
+import inspect
+from typing import Union
+import langfun.core as lf
+from langfun.core import llms as lf_llms
+from langfun.core.eval import base
+import pyglove as pg
+
+
+#
+# Program-based patchers.
+#
+
+
+def patch_member(cls, key, value, parent_key: str | None = None):
+  """Patches a member of a class."""
+
+  def _rebind_fn(k, v, p):
+    if (
+        isinstance(p, cls)
+        and k.key == key
+        and (parent_key is None or (p and p.sym_path.key == parent_key))
+    ):
+      if inspect.isfunction(value):
+        return value(k, v, p)
+      return value
+    return v
+
+  return _rebind_fn
+
+
+def patch_lm(lm: Union[lf.LanguageModel, pg.hyper.OneOf]):  # pylint: disable=redefined-outer-name
+  """Patches the LLM of evaluations."""
+  return patch_member(base.Evaluable, "lm", lm)
+
+
+def patch_parsing_lm(lm: Union[lf.LanguageModel, pg.hyper.OneOf]):  # pylint: disable=redefined-outer-name
+  """Patches the parsing LLM of evaluations."""
+  return patch_member(base.Evaluable, "parsing_lm", lm)
+
+
+def patch_schema_fn(schema_fn: Union[pg.Functor, pg.hyper.OneOf]):
+  """Patches the schema_fn of evaluations."""
+  return patch_member(base.Evaluable, "schema_fn", schema_fn)
+
+
+def patch_prompt(prompt: Union[str, lf.Template, pg.hyper.OneOf]):
+  """Patches the prompt of evaluations."""
+  return patch_member(base.Evaluable, "prompt", prompt)
+
+
+def patch_inputs(inputs: Union[pg.Functor, pg.hyper.OneOf]):
+  """Patches the inputs used in evaluations."""
+  return patch_member(base.Evaluable, "inputs", inputs)
+
+
+def patch_additional_args(**kwargs):
+  """Patches additional_args."""
+
+  def value_fn(k, unused_v, p):
+    # We infer the symbolic value for the old args, as it might be a
+    # contextual attribute referring to its containing object.
+    old_args = p.sym_inferred(k.key)
+    if old_args:
+      old_args = dict(old_args)
+      old_args.update(kwargs)
+      return old_args
+    return kwargs
+
+  return patch_member(base.Evaluable, "additional_args", value_fn)
+
+
+#
+# String-based patching.
+#
+
+_NAMED_MODELS = {
+    # GPT models.
+    "gpt35turbo": lf_llms.Gpt35Turbo,
+    "gpt35turbo16k": lf_llms.Gpt35Turbo16K,
+    "gpt4": lf_llms.Gpt4,
+    "gpt4turbo": lf_llms.Gpt4Turbo,
+    # Anthropic models.
+    "haiku": lf_llms.Claude3Haiku,
+    "claude3haiku": lf_llms.Claude3Haiku,
+    "opus": lf_llms.Claude3Opus,
+    "claude3opus": lf_llms.Claude3Opus,
+    "sonnet": lf_llms.Claude3Sonnet,
+    "claude3sonnet": lf_llms.Claude3Opus,
+}
+
+
+def model_by_name(name: str) -> lf.LanguageModel:
+  """Gets model by name."""
+  name = name.strip().lower()
+  if name in _NAMED_MODELS:
+    return _NAMED_MODELS[name]()
+  raise ValueError(f"Unknown model name: {name}")
+
+
+@pg.patcher(auto_typing=True)
+def lm(unused_eval, models: list[str]):
+  """Patch the LM used for benchmarking."""
+  return patch_lm(pg.oneof([model_by_name(name) for name in models]))
+
+
+@pg.patcher(auto_typing=True)
+def temperature(unused_eval, value: float):
+  """Patch the temperature used for benchmarking."""
+  return patch_member(lf.LMSamplingOptions, "temperature", value)
+
+
+@pg.patcher(auto_typing=True)
+def max_tokens(unused_eval, value: int | None):
+  """Patch the temperature used for benchmarking."""
+  return patch_member(lf.LMSamplingOptions, "max_tokens", value)

langfun/core/eval/patching_test.py

@@ -0,0 +1,170 @@
+# Copyright 2024 The Langfun Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for evaluation patching."""
+
+import unittest
+from langfun.core import llms as lf_llms
+from langfun.core.eval import base
+from langfun.core.eval import patching
+import pyglove as pg
+
+
+class PatchingCommonTest(unittest.TestCase):
+
+  def test_patch_member(self):
+    class A(pg.Object):
+      x: int = 1
+
+    class B(pg.Object):
+      a: A
+
+    b = B(A())
+    pg.patch(b, [patching.patch_member(A, 'x', 2)])
+    self.assertEqual(b, B(A(2)))
+
+  def test_patch_args(self):
+    s = base.Suite(
+        [base.Evaluation(inputs=base.as_inputs([1]))],
+        additional_args=dict(x=1, y=2),
+    )
+    pg.patch(s, [patching.patch_additional_args(x=3, z=4)])
+    self.assertTrue(
+        pg.eq(
+            s,
+            base.Suite(
+                [
+                    base.Evaluation(
+                        inputs=base.as_inputs([1]),
+                        additional_args=dict(x=3, y=2, z=4),
+                    )
+                ],
+                additional_args=dict(x=3, y=2, z=4),
+            ),
+        )
+    )
+
+  def test_patch_lm(self):
+    s = base.Suite(
+        [base.Evaluation(inputs=base.as_inputs([1]))],
+        lm=lf_llms.Gpt35Turbo(),
+    )
+    pg.patch(
+        s, [patching.patch_lm(pg.oneof([lf_llms.Gpt35Turbo(), lf_llms.Gpt4()]))]
+    )
+    self.assertTrue(
+        pg.eq(
+            s,
+            base.Suite(
+                [
+                    base.Evaluation(
+                        inputs=base.as_inputs([1]),
+                        lm=pg.oneof([lf_llms.Gpt35Turbo(), lf_llms.Gpt4()]),
+                    )
+                ],
+                lm=pg.oneof([lf_llms.Gpt35Turbo(), lf_llms.Gpt4()]),
+            ),
+        )
+    )
+
+  def test_patch_parsing_lm(self):
+    s = base.Suite(
+        [base.Evaluation(inputs=base.as_inputs([1]))],
+        lm=lf_llms.Gpt4(),
+    )
+    pg.patch(s, [patching.patch_parsing_lm(lf_llms.Gpt35Turbo())])
+    self.assertTrue(
+        pg.eq(
+            s,
+            base.Suite(
+                [
+                    base.Evaluation(
+                        inputs=base.as_inputs([1]),
+                        lm=lf_llms.Gpt4(),
+                        parsing_lm=lf_llms.Gpt35Turbo(),
+                    )
+                ],
+                # NOTE(daiyip): Suite does not have `parsing_lm` as one of its
+                # variable keyword fields yet, so patching does not add to it.
+                # This is okay since we only care about the leaf nodes.
+                lm=lf_llms.Gpt4(),
+            ),
+        )
+    )
+
+  def test_patch_prompt(self):
+    e = base.Evaluation(inputs=base.as_inputs([1]))
+    pg.patch(e, [patching.patch_prompt('Q: {{example.question}}')])
+    self.assertTrue(
+        pg.eq(
+            e,
+            base.Evaluation(
+                inputs=base.as_inputs([1]),
+                prompt='Q: {{example.question}}',
+            ),
+        )
+    )
+
+  def test_patch_inputs(self):
+    e = base.Evaluation(inputs=base.as_inputs([1]))
+    pg.patch(e, [patching.patch_inputs(base.as_inputs([2]))])
+    self.assertTrue(
+        pg.eq(
+            e,
+            base.Evaluation(
+                inputs=base.as_inputs([2]),
+            ),
+        )
+    )
+
+  def test_patch_schema_fn(self):
+    @pg.functor()
+    def int_schema():
+      return int
+
+    e = base.Evaluation(inputs=base.as_inputs([1]))
+    pg.patch(e, [patching.patch_schema_fn(int_schema())])
+    self.assertTrue(
+        pg.eq(
+            e,
+            base.Evaluation(
+                inputs=base.as_inputs([1]),
+                schema_fn=int_schema(),
+            ),
+        )
+    )
+
+
+class StringPatcheTest(unittest.TestCase):
+
+  def test_lm(self):
+    target = pg.patch(
+        base.Evaluation(inputs=base.as_inputs([1])),
+        ['lm?haiku:gpt4', 'max_tokens?1024', 'temperature?0.7'],
+    )
+    self.assertEqual(
+        target.lm,
+        pg.oneof([
+            lf_llms.Claude3Haiku(temperature=0.7, max_tokens=1024),
+            lf_llms.Gpt4(temperature=0.7, max_tokens=1024),
+        ]),
+    )
+    with self.assertRaisesRegex(ValueError, 'Unknown model name'):
+      pg.patch(
+          base.Evaluation(inputs=base.as_inputs([1])),
+          ['lm?gpt2'],
+      )
+
+
+if __name__ == '__main__':
+  unittest.main()

langfun/core/eval/scoring.py

@@ -61,8 +61,18 @@ class Scoring(base.Evaluation):
     super()._reset()
     self._scored = []
 
-  def audit(self, example: Any, output: Any, message: lf.Message) -> None:
+  def audit_processed(
+      self, example: Any, output: Any, message: lf.Message, dryrun: bool = False
+  ) -> None:
     score = self.score(example, output)
+
+    if dryrun:
+      lf.console.write('')
+      lf.console.write(
+          str(score),
+          title='SCORE',
+          color='blue',
+      )
     self._scored.append((example, output, score, message))
 
   @abc.abstractmethod
@@ -103,8 +113,8 @@ class Scoring(base.Evaluation):
         m.total,
     )
 
-  def summarize(self) -> pg.Dict:
-    result = super().summarize()
+  def finalize(self) -> pg.Dict:
+    result = super().finalize()
     result.metrics.update(
         num_scored=self.num_scored,
         score_rate=self.score_rate,
@@ -118,19 +128,18 @@ class Scoring(base.Evaluation):
     super().save(definition, result, report)
 
     if result:
-
-      def force_dict(v):
-        return pg.object_utils.json_conversion.strip_types(pg.to_json(v))
-
       # Save scored.
       pg.save(
           [
               # We force the output to be dict as its type may be defined
               # within functors which could be deserialized.
-              pg.Dict(input=input, output=force_dict(output), score=score)
+              pg.Dict(input=input, output=output, score=score)
               for input, output, score, _ in self.scored
           ],
           os.path.join(self.dir, Scoring.SCORED_JSON),
+          # We force the input and output to be dict so it does not depend on
+          # the downstream to serialize.
+          force_dict=True,
       )
 
     if report:
@@ -159,7 +168,7 @@ class Scoring(base.Evaluation):
         )
     )
 
-  def _render_metric(self, s: io.StringIO) -> None:
+  def _render_summary_metrics(self, s: io.StringIO) -> None:
     """Renders metrics in HTML."""
     assert self.result is not None
     m = self.result.metrics
@@ -173,7 +182,7 @@ class Scoring(base.Evaluation):
         )
     )
     s.write(' | ')
-    super()._render_metric(s)
+    super()._render_summary_metrics(s)
 
   def _render_scored(self, s: io.StringIO) -> None:
     """Formats the matched cases into html."""

langfun/core/eval/scoring_test.py

@@ -81,7 +81,7 @@ class ScoringTest(unittest.TestCase):
         s.result,
         dict(
             experiment_setup=dict(
-                id='ConstraintFollowing@9e51bb9e',
+                id='ConstraintFollowing@5c88a5eb',
                 dir=s.dir,
                 model='StaticSequence',
                 prompt_template='{{example}}',
@@ -98,10 +98,16 @@ class ScoringTest(unittest.TestCase):
                 total=2,
                 failures=0,
                 failure_rate=0.0,
+                oop_failures=0,
+                oop_failure_rate=0.0,
+                non_oop_failures=0,
+                non_oop_failure_rate=0.0,
+                failure_breakdown={},
                 num_scored=2,
                 score_rate=1.0,
                 avg_score=0.5,
             ),
+            usage=s.result.usage,
         ),
     )
     self.assertTrue(
@@ -123,7 +129,12 @@ class ScoringTest(unittest.TestCase):
     )
     self.assertTrue(
         os.path.exists(
-            os.path.join(s.dir, scoring.Scoring.FAILURES_JSON)
+            os.path.join(s.dir, scoring.Scoring.OOP_FAILURES_JSON)
+        )
+    )
+    self.assertTrue(
+        os.path.exists(
+            os.path.join(s.dir, scoring.Scoring.NON_OOP_FAILURES_JSON)
         )
     )
     self.assertTrue(
@@ -142,7 +153,14 @@ class ScoringTest(unittest.TestCase):
     self.assertTrue(
         os.path.exists(
             os.path.join(
-                s.dir, scoring.Scoring.FAILURES_HTML
+                s.dir, scoring.Scoring.OOP_FAILURES_HTML
+            )
+        )
+    )
+    self.assertTrue(
+        os.path.exists(
+            os.path.join(
+                s.dir, scoring.Scoring.NON_OOP_FAILURES_HTML
             )
         )
     )

langfun/core/langfunc.py

@@ -14,7 +14,7 @@
 """LangFunc: Language-based functions."""
 
 import dataclasses
-from typing import Annotated, Type, Union
+from typing import Annotated, Type
 
 from langfun.core import component
 from langfun.core import language_model
@@ -261,7 +261,6 @@ class LangFunc(
         if lm_input is None:
           lm_input = self.render(**kwargs)
 
-        lm_input.tag(message_lib.Message.TAG_LM_INPUT)
         if skip_lm:
           return lm_input
 
@@ -270,10 +269,6 @@ class LangFunc(
         # Send rendered text to LM.
         lm_output = self.lm(lm_input, cache_seed=cache_seed)
 
-        # Track the input as the source of the output.
-        lm_output.source = lm_input
-        lm_output.tag(message_lib.Message.TAG_LM_RESPONSE)
-
         # Transform the output message.
         lm_output = self.transform_output(lm_output)
         lm_output.tag(message_lib.Message.TAG_LM_OUTPUT)
@@ -333,22 +328,6 @@ class LangFunc(
     """Transforms the output message before returning from __call__."""
     return lm_output
 
-  @classmethod
-  def from_value(
-      cls, value: Union[str, template_lib.Template], **kwargs
-  ) -> 'LangFunc':
-    """Create a LangFunc object from a string or template."""
-    if isinstance(value, LangFunc):
-      return value
-    if isinstance(value, template_lib.Template):
-      lfun = LangFunc(value.template_str, **kwargs)
-      # So lfun could acccess all attributes from value.
-      lfun.sym_setparent(value)
-      return lfun
-    if isinstance(value, str):
-      return LangFunc(template_str=value, **kwargs)
-    return LangFunc('{{input}}', input=value, **kwargs)
-
 
 # Register converter from str to LangFunc, therefore we can always
 # pass strs to attributes that accept LangFunc.

langfun/core/langfunc_test.py

@@ -57,6 +57,10 @@ class BasicTest(unittest.TestCase):
     l2 = LangFunc.from_value(l1)
     self.assertIs(l2, l1)
 
+    l3 = LangFunc.from_value(l1, x=1)
+    self.assertIsNot(l3, l1)
+    self.assertTrue(pg.eq(l3, LangFunc('Hello', x=1)))
+
     c = template_lib.Template(
         '{{x}} + {{l}}',
         x=1,
@@ -82,7 +86,9 @@ class LangFuncCallTest(unittest.TestCase):
     self.assertEqual(i.tags, ['rendered'])
 
     r = l()
-    self.assertEqual(r, message.AIMessage('Hello!!!', score=0.0, logprobs=None))
+    self.assertEqual(
+        r, message.AIMessage('Hello!!!', score=0.0, logprobs=None, usage=None)
+    )
     self.assertEqual(r.tags, ['lm-response', 'lm-output'])
     self.assertEqual(r.source, message.UserMessage('Hello'))
     self.assertEqual(r.source.tags, ['rendered', 'lm-input'])
@@ -92,8 +98,8 @@ class LangFuncCallTest(unittest.TestCase):
     self.assertEqual(
         repr(l),
         "LangFunc(template_str='Hello', clean=True,"
-        ' lm=ExcitedEchoer(sampling_options=LMSamplingOptions(temperature=0.0,'
-        ' max_tokens=1024, n=1, top_k=40, top_p=None, stop=None,'
+        ' lm=ExcitedEchoer(sampling_options=LMSamplingOptions(temperature=None,'
+        ' max_tokens=None, n=1, top_k=40, top_p=None, stop=None,'
         ' random_seed=None, logprobs=False, top_logprobs=None), cache=None,'
         ' max_concurrency=None, timeout=120.0, max_attempts=5,'
         ' retry_interval=(5, 60), exponential_backoff=True, debug=False))',
@@ -106,7 +112,7 @@ class LangFuncCallTest(unittest.TestCase):
       self.assertEqual(l.render(), 'Hello')
       r = l()
       self.assertEqual(
-          r, message.AIMessage('Hello!!!', score=0.0, logprobs=None)
+          r, message.AIMessage('Hello!!!', score=0.0, logprobs=None, usage=None)
       )
       self.assertEqual(r.tags, ['lm-response', 'lm-output'])