langfun 0.1.1.dev20240817__py3-none-any.whl → 0.1.1.dev20240820__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
langfun/core/eval/base.py CHANGED
@@ -1061,6 +1061,8 @@ class Evaluation(Evaluable):
1061
1061
  try:
1062
1062
  with lf.use_settings(debug=debug):
1063
1063
  output_message = copy.process(example, **(self.additional_args or {}))
1064
+ self.process_output(example, output_message)
1065
+
1064
1066
  if self.schema is None:
1065
1067
  output = output_message.text
1066
1068
  else:
@@ -1123,7 +1125,9 @@ class Evaluation(Evaluable):
1123
1125
  # generated code with calls to `input` will raise an error, thus not
1124
1126
  # blocking the evaluation.
1125
1127
  with lf_coding.context(input=None):
1126
- return self.process(example, **(self.additional_args or {}))
1128
+ output_message = self.process(example, **(self.additional_args or {}))
1129
+ self.process_output(example, output_message)
1130
+ return output_message
1127
1131
 
1128
1132
  try:
1129
1133
  for example, message, error in lf.concurrent_map(
@@ -1201,6 +1205,29 @@ class Evaluation(Evaluable):
1201
1205
  **kwargs,
1202
1206
  )
1203
1207
 
1208
+ def process_output(self, example: Any, output: lf.Message) -> None:
1209
+ """Process the output for an example.
1210
+
1211
+ Subclasses can override this method to generate and attach additional
1212
+ metadata for debugging purpose. For example, draw bounding boxes on the
1213
+ input image based on LLM predicted boxes and attach to output_message's
1214
+ metadata.
1215
+
1216
+ Example:
1217
+
1218
+ class BoundingBoxEval(lf.eval.Matching):
1219
+ ...
1220
+ def process_output(example, output):
1221
+ output.metadata.image_with_bbox = draw_bboxes(
1222
+ example.image, output.result)
1223
+
1224
+ Args:
1225
+ example: User input.
1226
+ output: LLM's output message. Users could attach additional
1227
+ information to the message, which will be shown in debugging
1228
+ """
1229
+ del example, output
1230
+
1204
1231
  def _status(self, progress: lf.concurrent.Progress) -> dict[str, Any]:
1205
1232
  return {
1206
1233
  'Model': self.lm.model_id,
@@ -92,6 +92,15 @@ class MappingExample(lf.NaturalLanguageFormattable, lf.Component):
92
92
  'The natural language context for this mapping. ',
93
93
  ] = None
94
94
 
95
+ metadata: Annotated[
96
+ dict[str, Any],
97
+ (
98
+ 'The metadata associated with the mapping example, '
99
+ 'which chould carry structured data, such as tool function input. '
100
+ 'It is a `pg.Dict` object whose keys can be accessed by attributes.'
101
+ ),
102
+ ] = pg.Dict()
103
+
95
104
  def schema_repr(
96
105
  self, protocol: schema_lib.SchemaProtocol = 'python', **kwargs
97
106
  ) -> str:
@@ -157,16 +166,21 @@ class MappingExample(lf.NaturalLanguageFormattable, lf.Component):
157
166
 
158
167
  result.write(lf.colored('[INPUT]\n', styles=['bold']))
159
168
  result.write(lf.colored(self.input_repr(), color='green'))
160
- result.write('\n\n')
161
169
 
162
170
  if self.schema is not None:
171
+ result.write('\n\n')
163
172
  result.write(lf.colored('[SCHEMA]\n', styles=['bold']))
164
173
  result.write(lf.colored(self.schema_repr(), color='red'))
165
- result.write('\n\n')
166
174
 
167
175
  if schema_lib.MISSING != self.output:
176
+ result.write('\n\n')
168
177
  result.write(lf.colored('[OUTPUT]\n', styles=['bold']))
169
178
  result.write(lf.colored(self.output_repr(), color='blue'))
179
+
180
+ if self.metadata:
181
+ result.write('\n\n')
182
+ result.write(lf.colored('[METADATA]\n', styles=['bold']))
183
+ result.write(lf.colored(str(self.metadata), color='cyan'))
170
184
  return result.getvalue().strip()
171
185
 
172
186
 
@@ -129,6 +129,33 @@ class MappingExampleTest(unittest.TestCase):
129
129
  """),
130
130
  )
131
131
 
132
+ def test_str_with_metadata(self):
133
+ self.assertEqual(
134
+ str(
135
+ mapping.MappingExample(
136
+ '1 + 1 = 2',
137
+ schema=int,
138
+ context='Give the answer.',
139
+ metadata={'foo': 'bar'},
140
+ )
141
+ ),
142
+ inspect.cleandoc("""
143
+ \x1b[1m[CONTEXT]
144
+ \x1b[0m\x1b[35mGive the answer.\x1b[0m
145
+
146
+ \x1b[1m[INPUT]
147
+ \x1b[0m\x1b[32m1 + 1 = 2\x1b[0m
148
+
149
+ \x1b[1m[SCHEMA]
150
+ \x1b[0m\x1b[31mint\x1b[0m
151
+
152
+ \x1b[1m[METADATA]
153
+ \x1b[0m\x1b[36m{
154
+ foo = 'bar'
155
+ }\x1b[0m
156
+ """),
157
+ )
158
+
132
159
  def test_serialization(self):
133
160
  example = mapping.MappingExample(
134
161
  'the answer is 2', 2, int, context='compute 1 + 1'
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: langfun
3
- Version: 0.1.1.dev20240817
3
+ Version: 0.1.1.dev20240820
4
4
  Summary: Langfun: Language as Functions.
5
5
  Home-page: https://github.com/google/langfun
6
6
  Author: Langfun Authors
@@ -44,7 +44,7 @@ langfun/core/coding/python/parsing_test.py,sha256=9vAWF484kWIm6JZq8NFiMgKUDhXV-d
44
44
  langfun/core/coding/python/permissions.py,sha256=1QWGHvzL8MM0Ok_auQ9tURqZHtdOfJaDpBzZ29GUE-c,2544
45
45
  langfun/core/coding/python/permissions_test.py,sha256=w5EDb8QxpxgJyZkojyzVWQvDfg366zn99-g__6TbPQ0,2699
46
46
  langfun/core/eval/__init__.py,sha256=Ogdr9OtTywhhLPHi3AZzOD2mXX2oyaHWflrSTMm96uA,1899
47
- langfun/core/eval/base.py,sha256=0_iaKuQhS49PlbWqCQ5EABUMKavr2R4ltcJZWCVoZZg,73816
47
+ langfun/core/eval/base.py,sha256=BiWColibVo9-4P27Z0hIWXe8_UPocJTSTUdKeOPVwxI,74746
48
48
  langfun/core/eval/base_test.py,sha256=p1EfqviHMz_ppQY8FU67h5OCgL0tzhLvXzGIsq0sVyI,26930
49
49
  langfun/core/eval/matching.py,sha256=9GX8HfO9jKxgNLAivgy5K88Xhoh6Z7Pptq65pe7vht8,9762
50
50
  langfun/core/eval/matching_test.py,sha256=f7iVyXH5KGJBWt4Wp14Bt9J3X59A6Ayfog9MbuFvPew,5532
@@ -96,8 +96,8 @@ langfun/core/structured/description.py,sha256=SXW4MJvshFjbR-0gw6rE21o6WXq12UlRXa
96
96
  langfun/core/structured/description_test.py,sha256=UtZGjSFUaQ6130t1E5tcL7ODu0xIefkapb53TbnqsK8,7362
97
97
  langfun/core/structured/function_generation.py,sha256=pFgS3vcRAWiuFBol2x5Eeip3XqoudONsOpeJpWyjT3s,7479
98
98
  langfun/core/structured/function_generation_test.py,sha256=ZJI-aaGgWWszn92u7h5IZ9Pl70N2DgAGGJrIxPzsvwg,10065
99
- langfun/core/structured/mapping.py,sha256=QKbSnvOgut-sx2mZPjHJcdlDLxR8b3ZC16ZLWociwog,11298
100
- langfun/core/structured/mapping_test.py,sha256=PiXklMeIa8L6KtMi3ju7J9Y39gZy0hIGz-Oeq4A_7XE,3835
99
+ langfun/core/structured/mapping.py,sha256=CsflMwm5cKJYZ2ag-neroA4CQlhu2wjFRSxKpd_qQDQ,11778
100
+ langfun/core/structured/mapping_test.py,sha256=zQoVx3kAD5oSm_OJAQA6q41NXLLyn8qs6CIVJgAoP_w,4489
101
101
  langfun/core/structured/parsing.py,sha256=keoVqEfzAbdULh6GawWFsTQzU91MzJXYFZjXGXLaD8g,11492
102
102
  langfun/core/structured/parsing_test.py,sha256=34wDrXaQ-EYhJLfDL8mX9K53oQMSzh5pVYdKjnESmK8,20895
103
103
  langfun/core/structured/prompting.py,sha256=_U6Z65AwXvVvfaQFCY9GawB_QV9S3u7P7BOU2URABmw,8873
@@ -117,8 +117,8 @@ langfun/core/templates/demonstration.py,sha256=vCrgYubdZM5Umqcgp8NUVGXgr4P_c-fik
117
117
  langfun/core/templates/demonstration_test.py,sha256=SafcDQ0WgI7pw05EmPI2S4v1t3ABKzup8jReCljHeK4,2162
118
118
  langfun/core/templates/selfplay.py,sha256=yhgrJbiYwq47TgzThmHrDQTF4nDrTI09CWGhuQPNv-s,2273
119
119
  langfun/core/templates/selfplay_test.py,sha256=rBW2Qr8yi-aWYwoTwRR-n1peKyMX9QXPZXURjLgoiRs,2264
120
- langfun-0.1.1.dev20240817.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
121
- langfun-0.1.1.dev20240817.dist-info/METADATA,sha256=_j3RVHGW7vU400lTH9B4ncDzlqEmCHOs4GfpwO96quE,5234
122
- langfun-0.1.1.dev20240817.dist-info/WHEEL,sha256=HiCZjzuy6Dw0hdX5R3LCFPDmFS4BWl8H-8W39XfmgX4,91
123
- langfun-0.1.1.dev20240817.dist-info/top_level.txt,sha256=RhlEkHxs1qtzmmtWSwYoLVJAc1YrbPtxQ52uh8Z9VvY,8
124
- langfun-0.1.1.dev20240817.dist-info/RECORD,,
120
+ langfun-0.1.1.dev20240820.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
121
+ langfun-0.1.1.dev20240820.dist-info/METADATA,sha256=t3TOnvt67GjpbhKOfk9K19GbEXtGlmNBMwrjqaqbYQU,5234
122
+ langfun-0.1.1.dev20240820.dist-info/WHEEL,sha256=nCVcAvsfA9TDtwGwhYaRrlPhTLV9m-Ga6mdyDtuwK18,91
123
+ langfun-0.1.1.dev20240820.dist-info/top_level.txt,sha256=RhlEkHxs1qtzmmtWSwYoLVJAc1YrbPtxQ52uh8Z9VvY,8
124
+ langfun-0.1.1.dev20240820.dist-info/RECORD,,
@@ -1,5 +1,5 @@
1
1
  Wheel-Version: 1.0
2
- Generator: setuptools (72.2.0)
2
+ Generator: setuptools (73.0.0)
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
5
5