langfun 0.1.1.dev20240817__py3-none-any.whl → 0.1.1.dev20240820__py3-none-any.whl

langfun/core/eval/base.py

@@ -1061,6 +1061,8 @@ class Evaluation(Evaluable):
     try:
       with lf.use_settings(debug=debug):
         output_message = copy.process(example, **(self.additional_args or {}))
+        self.process_output(example, output_message)
+
         if self.schema is None:
           output = output_message.text
         else:
@@ -1123,7 +1125,9 @@ class Evaluation(Evaluable):
         # generated code with calls to `input` will raise an error, thus not
         # blocking the evaluation.
         with lf_coding.context(input=None):
-          return self.process(example, **(self.additional_args or {}))
+          output_message = self.process(example, **(self.additional_args or {}))
+          self.process_output(example, output_message)
+          return output_message
 
       try:
         for example, message, error in lf.concurrent_map(
@@ -1201,6 +1205,29 @@ class Evaluation(Evaluable):
           **kwargs,
       )
 
+  def process_output(self, example: Any, output: lf.Message) -> None:
+    """Process the output for an example.
+
+    Subclasses can override this method to generate and attach additional
+    metadata for debugging purpose. For example, draw bounding boxes on the
+    input image based on LLM predicted boxes and attach to output_message's
+    metadata.
+
+    Example:
+
+      class BoundingBoxEval(lf.eval.Matching):
+        ...
+        def process_output(example, output):
+          output.metadata.image_with_bbox = draw_bboxes(
+              example.image, output.result)
+
+    Args:
+      example: User input.
+      output: LLM's output message. Users could attach additional
+        information to the message, which will be shown in debugging
+    """
+    del example, output
+
   def _status(self, progress: lf.concurrent.Progress) -> dict[str, Any]:
     return {
         'Model': self.lm.model_id,

langfun/core/structured/mapping.py

@@ -92,6 +92,15 @@ class MappingExample(lf.NaturalLanguageFormattable, lf.Component):
       'The natural language context for this mapping. ',
   ] = None
 
+  metadata: Annotated[
+      dict[str, Any],
+      (
+          'The metadata associated with the mapping example, '
+          'which chould carry structured data, such as tool function input. '
+          'It is a `pg.Dict` object whose keys can be accessed by attributes.'
+      ),
+  ] = pg.Dict()
+
   def schema_repr(
       self, protocol: schema_lib.SchemaProtocol = 'python', **kwargs
   ) -> str:
@@ -157,16 +166,21 @@ class MappingExample(lf.NaturalLanguageFormattable, lf.Component):
 
     result.write(lf.colored('[INPUT]\n', styles=['bold']))
     result.write(lf.colored(self.input_repr(), color='green'))
-    result.write('\n\n')
 
     if self.schema is not None:
+      result.write('\n\n')
       result.write(lf.colored('[SCHEMA]\n', styles=['bold']))
       result.write(lf.colored(self.schema_repr(), color='red'))
-      result.write('\n\n')
 
     if schema_lib.MISSING != self.output:
+      result.write('\n\n')
       result.write(lf.colored('[OUTPUT]\n', styles=['bold']))
       result.write(lf.colored(self.output_repr(), color='blue'))
+
+    if self.metadata:
+      result.write('\n\n')
+      result.write(lf.colored('[METADATA]\n', styles=['bold']))
+      result.write(lf.colored(str(self.metadata), color='cyan'))
     return result.getvalue().strip()
 
 

langfun/core/structured/mapping_test.py

@@ -129,6 +129,33 @@ class MappingExampleTest(unittest.TestCase):
             """),
     )
 
+  def test_str_with_metadata(self):
+    self.assertEqual(
+        str(
+            mapping.MappingExample(
+                '1 + 1 = 2',
+                schema=int,
+                context='Give the answer.',
+                metadata={'foo': 'bar'},
+            )
+        ),
+        inspect.cleandoc("""
+            \x1b[1m[CONTEXT]
+            \x1b[0m\x1b[35mGive the answer.\x1b[0m
+
+            \x1b[1m[INPUT]
+            \x1b[0m\x1b[32m1 + 1 = 2\x1b[0m
+
+            \x1b[1m[SCHEMA]
+            \x1b[0m\x1b[31mint\x1b[0m
+
+            \x1b[1m[METADATA]
+            \x1b[0m\x1b[36m{
+              foo = 'bar'
+            }\x1b[0m
+            """),
+    )
+
   def test_serialization(self):
     example = mapping.MappingExample(
         'the answer is 2', 2, int, context='compute 1 + 1'

{langfun-0.1.1.dev20240817.dist-info → langfun-0.1.1.dev20240820.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: langfun
-Version: 0.1.1.dev20240817
+Version: 0.1.1.dev20240820
 Summary: Langfun: Language as Functions.
 Home-page: https://github.com/google/langfun
 Author: Langfun Authors

{langfun-0.1.1.dev20240817.dist-info → langfun-0.1.1.dev20240820.dist-info}/RECORD

@@ -44,7 +44,7 @@ langfun/core/coding/python/parsing_test.py,sha256=9vAWF484kWIm6JZq8NFiMgKUDhXV-d
 langfun/core/coding/python/permissions.py,sha256=1QWGHvzL8MM0Ok_auQ9tURqZHtdOfJaDpBzZ29GUE-c,2544
 langfun/core/coding/python/permissions_test.py,sha256=w5EDb8QxpxgJyZkojyzVWQvDfg366zn99-g__6TbPQ0,2699
 langfun/core/eval/__init__.py,sha256=Ogdr9OtTywhhLPHi3AZzOD2mXX2oyaHWflrSTMm96uA,1899
-langfun/core/eval/base.py,sha256=0_iaKuQhS49PlbWqCQ5EABUMKavr2R4ltcJZWCVoZZg,73816
+langfun/core/eval/base.py,sha256=BiWColibVo9-4P27Z0hIWXe8_UPocJTSTUdKeOPVwxI,74746
 langfun/core/eval/base_test.py,sha256=p1EfqviHMz_ppQY8FU67h5OCgL0tzhLvXzGIsq0sVyI,26930
 langfun/core/eval/matching.py,sha256=9GX8HfO9jKxgNLAivgy5K88Xhoh6Z7Pptq65pe7vht8,9762
 langfun/core/eval/matching_test.py,sha256=f7iVyXH5KGJBWt4Wp14Bt9J3X59A6Ayfog9MbuFvPew,5532
@@ -96,8 +96,8 @@ langfun/core/structured/description.py,sha256=SXW4MJvshFjbR-0gw6rE21o6WXq12UlRXa
 langfun/core/structured/description_test.py,sha256=UtZGjSFUaQ6130t1E5tcL7ODu0xIefkapb53TbnqsK8,7362
 langfun/core/structured/function_generation.py,sha256=pFgS3vcRAWiuFBol2x5Eeip3XqoudONsOpeJpWyjT3s,7479
 langfun/core/structured/function_generation_test.py,sha256=ZJI-aaGgWWszn92u7h5IZ9Pl70N2DgAGGJrIxPzsvwg,10065
-langfun/core/structured/mapping.py,sha256=QKbSnvOgut-sx2mZPjHJcdlDLxR8b3ZC16ZLWociwog,11298
-langfun/core/structured/mapping_test.py,sha256=PiXklMeIa8L6KtMi3ju7J9Y39gZy0hIGz-Oeq4A_7XE,3835
+langfun/core/structured/mapping.py,sha256=CsflMwm5cKJYZ2ag-neroA4CQlhu2wjFRSxKpd_qQDQ,11778
+langfun/core/structured/mapping_test.py,sha256=zQoVx3kAD5oSm_OJAQA6q41NXLLyn8qs6CIVJgAoP_w,4489
 langfun/core/structured/parsing.py,sha256=keoVqEfzAbdULh6GawWFsTQzU91MzJXYFZjXGXLaD8g,11492
 langfun/core/structured/parsing_test.py,sha256=34wDrXaQ-EYhJLfDL8mX9K53oQMSzh5pVYdKjnESmK8,20895
 langfun/core/structured/prompting.py,sha256=_U6Z65AwXvVvfaQFCY9GawB_QV9S3u7P7BOU2URABmw,8873
@@ -117,8 +117,8 @@ langfun/core/templates/demonstration.py,sha256=vCrgYubdZM5Umqcgp8NUVGXgr4P_c-fik
 langfun/core/templates/demonstration_test.py,sha256=SafcDQ0WgI7pw05EmPI2S4v1t3ABKzup8jReCljHeK4,2162
 langfun/core/templates/selfplay.py,sha256=yhgrJbiYwq47TgzThmHrDQTF4nDrTI09CWGhuQPNv-s,2273
 langfun/core/templates/selfplay_test.py,sha256=rBW2Qr8yi-aWYwoTwRR-n1peKyMX9QXPZXURjLgoiRs,2264
-langfun-0.1.1.dev20240817.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-langfun-0.1.1.dev20240817.dist-info/METADATA,sha256=_j3RVHGW7vU400lTH9B4ncDzlqEmCHOs4GfpwO96quE,5234
-langfun-0.1.1.dev20240817.dist-info/WHEEL,sha256=HiCZjzuy6Dw0hdX5R3LCFPDmFS4BWl8H-8W39XfmgX4,91
-langfun-0.1.1.dev20240817.dist-info/top_level.txt,sha256=RhlEkHxs1qtzmmtWSwYoLVJAc1YrbPtxQ52uh8Z9VvY,8
-langfun-0.1.1.dev20240817.dist-info/RECORD,,
+langfun-0.1.1.dev20240820.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+langfun-0.1.1.dev20240820.dist-info/METADATA,sha256=t3TOnvt67GjpbhKOfk9K19GbEXtGlmNBMwrjqaqbYQU,5234
+langfun-0.1.1.dev20240820.dist-info/WHEEL,sha256=nCVcAvsfA9TDtwGwhYaRrlPhTLV9m-Ga6mdyDtuwK18,91
+langfun-0.1.1.dev20240820.dist-info/top_level.txt,sha256=RhlEkHxs1qtzmmtWSwYoLVJAc1YrbPtxQ52uh8Z9VvY,8
+langfun-0.1.1.dev20240820.dist-info/RECORD,,

{langfun-0.1.1.dev20240817.dist-info → langfun-0.1.1.dev20240820.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (72.2.0)
+Generator: setuptools (73.0.0)
 Root-Is-Purelib: true
 Tag: py3-none-any