langfun 0.1.1.dev20240819__tar.gz → 0.1.1.dev20240820__tar.gz

{langfun-0.1.1.dev20240819 → langfun-0.1.1.dev20240820}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: langfun
-Version: 0.1.1.dev20240819
+Version: 0.1.1.dev20240820
 Summary: Langfun: Language as Functions.
 Home-page: https://github.com/google/langfun
 Author: Langfun Authors

{langfun-0.1.1.dev20240819 → langfun-0.1.1.dev20240820}/langfun/core/eval/base.py

@@ -1061,6 +1061,8 @@ class Evaluation(Evaluable):
     try:
       with lf.use_settings(debug=debug):
         output_message = copy.process(example, **(self.additional_args or {}))
+        self.process_output(example, output_message)
+
         if self.schema is None:
           output = output_message.text
         else:
@@ -1123,7 +1125,9 @@ class Evaluation(Evaluable):
         # generated code with calls to `input` will raise an error, thus not
         # blocking the evaluation.
         with lf_coding.context(input=None):
-          return self.process(example, **(self.additional_args or {}))
+          output_message = self.process(example, **(self.additional_args or {}))
+          self.process_output(example, output_message)
+          return output_message
 
       try:
         for example, message, error in lf.concurrent_map(
@@ -1201,6 +1205,29 @@ class Evaluation(Evaluable):
           **kwargs,
       )
 
+  def process_output(self, example: Any, output: lf.Message) -> None:
+    """Process the output for an example.
+
+    Subclasses can override this method to generate and attach additional
+    metadata for debugging purpose. For example, draw bounding boxes on the
+    input image based on LLM predicted boxes and attach to output_message's
+    metadata.
+
+    Example:
+
+      class BoundingBoxEval(lf.eval.Matching):
+        ...
+        def process_output(example, output):
+          output.metadata.image_with_bbox = draw_bboxes(
+              example.image, output.result)
+
+    Args:
+      example: User input.
+      output: LLM's output message. Users could attach additional
+        information to the message, which will be shown in debugging
+    """
+    del example, output
+
   def _status(self, progress: lf.concurrent.Progress) -> dict[str, Any]:
     return {
         'Model': self.lm.model_id,

{langfun-0.1.1.dev20240819 → langfun-0.1.1.dev20240820}/langfun/core/structured/mapping.py

@@ -92,6 +92,15 @@ class MappingExample(lf.NaturalLanguageFormattable, lf.Component):
       'The natural language context for this mapping. ',
   ] = None
 
+  metadata: Annotated[
+      dict[str, Any],
+      (
+          'The metadata associated with the mapping example, '
+          'which chould carry structured data, such as tool function input. '
+          'It is a `pg.Dict` object whose keys can be accessed by attributes.'
+      ),
+  ] = pg.Dict()
+
   def schema_repr(
       self, protocol: schema_lib.SchemaProtocol = 'python', **kwargs
   ) -> str:
@@ -157,16 +166,21 @@ class MappingExample(lf.NaturalLanguageFormattable, lf.Component):
 
     result.write(lf.colored('[INPUT]\n', styles=['bold']))
     result.write(lf.colored(self.input_repr(), color='green'))
-    result.write('\n\n')
 
     if self.schema is not None:
+      result.write('\n\n')
       result.write(lf.colored('[SCHEMA]\n', styles=['bold']))
       result.write(lf.colored(self.schema_repr(), color='red'))
-      result.write('\n\n')
 
     if schema_lib.MISSING != self.output:
+      result.write('\n\n')
       result.write(lf.colored('[OUTPUT]\n', styles=['bold']))
       result.write(lf.colored(self.output_repr(), color='blue'))
+
+    if self.metadata:
+      result.write('\n\n')
+      result.write(lf.colored('[METADATA]\n', styles=['bold']))
+      result.write(lf.colored(str(self.metadata), color='cyan'))
     return result.getvalue().strip()
 
 

{langfun-0.1.1.dev20240819 → langfun-0.1.1.dev20240820}/langfun/core/structured/mapping_test.py

@@ -129,6 +129,33 @@ class MappingExampleTest(unittest.TestCase):
             """),
     )
 
+  def test_str_with_metadata(self):
+    self.assertEqual(
+        str(
+            mapping.MappingExample(
+                '1 + 1 = 2',
+                schema=int,
+                context='Give the answer.',
+                metadata={'foo': 'bar'},
+            )
+        ),
+        inspect.cleandoc("""
+            \x1b[1m[CONTEXT]
+            \x1b[0m\x1b[35mGive the answer.\x1b[0m
+
+            \x1b[1m[INPUT]
+            \x1b[0m\x1b[32m1 + 1 = 2\x1b[0m
+
+            \x1b[1m[SCHEMA]
+            \x1b[0m\x1b[31mint\x1b[0m
+
+            \x1b[1m[METADATA]
+            \x1b[0m\x1b[36m{
+              foo = 'bar'
+            }\x1b[0m
+            """),
+    )
+
   def test_serialization(self):
     example = mapping.MappingExample(
         'the answer is 2', 2, int, context='compute 1 + 1'

{langfun-0.1.1.dev20240819 → langfun-0.1.1.dev20240820}/langfun.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: langfun
-Version: 0.1.1.dev20240819
+Version: 0.1.1.dev20240820
 Summary: Langfun: Language as Functions.
 Home-page: https://github.com/google/langfun
 Author: Langfun Authors