langfun 0.1.1.dev20240819__py3-none-any.whl → 0.1.1.dev20240820__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- langfun/core/eval/base.py +28 -1
- langfun/core/structured/mapping.py +16 -2
- langfun/core/structured/mapping_test.py +27 -0
- {langfun-0.1.1.dev20240819.dist-info → langfun-0.1.1.dev20240820.dist-info}/METADATA +1 -1
- {langfun-0.1.1.dev20240819.dist-info → langfun-0.1.1.dev20240820.dist-info}/RECORD +8 -8
- {langfun-0.1.1.dev20240819.dist-info → langfun-0.1.1.dev20240820.dist-info}/WHEEL +1 -1
- {langfun-0.1.1.dev20240819.dist-info → langfun-0.1.1.dev20240820.dist-info}/LICENSE +0 -0
- {langfun-0.1.1.dev20240819.dist-info → langfun-0.1.1.dev20240820.dist-info}/top_level.txt +0 -0
langfun/core/eval/base.py
CHANGED
@@ -1061,6 +1061,8 @@ class Evaluation(Evaluable):
|
|
1061
1061
|
try:
|
1062
1062
|
with lf.use_settings(debug=debug):
|
1063
1063
|
output_message = copy.process(example, **(self.additional_args or {}))
|
1064
|
+
self.process_output(example, output_message)
|
1065
|
+
|
1064
1066
|
if self.schema is None:
|
1065
1067
|
output = output_message.text
|
1066
1068
|
else:
|
@@ -1123,7 +1125,9 @@ class Evaluation(Evaluable):
|
|
1123
1125
|
# generated code with calls to `input` will raise an error, thus not
|
1124
1126
|
# blocking the evaluation.
|
1125
1127
|
with lf_coding.context(input=None):
|
1126
|
-
|
1128
|
+
output_message = self.process(example, **(self.additional_args or {}))
|
1129
|
+
self.process_output(example, output_message)
|
1130
|
+
return output_message
|
1127
1131
|
|
1128
1132
|
try:
|
1129
1133
|
for example, message, error in lf.concurrent_map(
|
@@ -1201,6 +1205,29 @@ class Evaluation(Evaluable):
|
|
1201
1205
|
**kwargs,
|
1202
1206
|
)
|
1203
1207
|
|
1208
|
+
def process_output(self, example: Any, output: lf.Message) -> None:
|
1209
|
+
"""Process the output for an example.
|
1210
|
+
|
1211
|
+
Subclasses can override this method to generate and attach additional
|
1212
|
+
metadata for debugging purpose. For example, draw bounding boxes on the
|
1213
|
+
input image based on LLM predicted boxes and attach to output_message's
|
1214
|
+
metadata.
|
1215
|
+
|
1216
|
+
Example:
|
1217
|
+
|
1218
|
+
class BoundingBoxEval(lf.eval.Matching):
|
1219
|
+
...
|
1220
|
+
def process_output(example, output):
|
1221
|
+
output.metadata.image_with_bbox = draw_bboxes(
|
1222
|
+
example.image, output.result)
|
1223
|
+
|
1224
|
+
Args:
|
1225
|
+
example: User input.
|
1226
|
+
output: LLM's output message. Users could attach additional
|
1227
|
+
information to the message, which will be shown in debugging
|
1228
|
+
"""
|
1229
|
+
del example, output
|
1230
|
+
|
1204
1231
|
def _status(self, progress: lf.concurrent.Progress) -> dict[str, Any]:
|
1205
1232
|
return {
|
1206
1233
|
'Model': self.lm.model_id,
|
@@ -92,6 +92,15 @@ class MappingExample(lf.NaturalLanguageFormattable, lf.Component):
|
|
92
92
|
'The natural language context for this mapping. ',
|
93
93
|
] = None
|
94
94
|
|
95
|
+
metadata: Annotated[
|
96
|
+
dict[str, Any],
|
97
|
+
(
|
98
|
+
'The metadata associated with the mapping example, '
|
99
|
+
'which chould carry structured data, such as tool function input. '
|
100
|
+
'It is a `pg.Dict` object whose keys can be accessed by attributes.'
|
101
|
+
),
|
102
|
+
] = pg.Dict()
|
103
|
+
|
95
104
|
def schema_repr(
|
96
105
|
self, protocol: schema_lib.SchemaProtocol = 'python', **kwargs
|
97
106
|
) -> str:
|
@@ -157,16 +166,21 @@ class MappingExample(lf.NaturalLanguageFormattable, lf.Component):
|
|
157
166
|
|
158
167
|
result.write(lf.colored('[INPUT]\n', styles=['bold']))
|
159
168
|
result.write(lf.colored(self.input_repr(), color='green'))
|
160
|
-
result.write('\n\n')
|
161
169
|
|
162
170
|
if self.schema is not None:
|
171
|
+
result.write('\n\n')
|
163
172
|
result.write(lf.colored('[SCHEMA]\n', styles=['bold']))
|
164
173
|
result.write(lf.colored(self.schema_repr(), color='red'))
|
165
|
-
result.write('\n\n')
|
166
174
|
|
167
175
|
if schema_lib.MISSING != self.output:
|
176
|
+
result.write('\n\n')
|
168
177
|
result.write(lf.colored('[OUTPUT]\n', styles=['bold']))
|
169
178
|
result.write(lf.colored(self.output_repr(), color='blue'))
|
179
|
+
|
180
|
+
if self.metadata:
|
181
|
+
result.write('\n\n')
|
182
|
+
result.write(lf.colored('[METADATA]\n', styles=['bold']))
|
183
|
+
result.write(lf.colored(str(self.metadata), color='cyan'))
|
170
184
|
return result.getvalue().strip()
|
171
185
|
|
172
186
|
|
@@ -129,6 +129,33 @@ class MappingExampleTest(unittest.TestCase):
|
|
129
129
|
"""),
|
130
130
|
)
|
131
131
|
|
132
|
+
def test_str_with_metadata(self):
|
133
|
+
self.assertEqual(
|
134
|
+
str(
|
135
|
+
mapping.MappingExample(
|
136
|
+
'1 + 1 = 2',
|
137
|
+
schema=int,
|
138
|
+
context='Give the answer.',
|
139
|
+
metadata={'foo': 'bar'},
|
140
|
+
)
|
141
|
+
),
|
142
|
+
inspect.cleandoc("""
|
143
|
+
\x1b[1m[CONTEXT]
|
144
|
+
\x1b[0m\x1b[35mGive the answer.\x1b[0m
|
145
|
+
|
146
|
+
\x1b[1m[INPUT]
|
147
|
+
\x1b[0m\x1b[32m1 + 1 = 2\x1b[0m
|
148
|
+
|
149
|
+
\x1b[1m[SCHEMA]
|
150
|
+
\x1b[0m\x1b[31mint\x1b[0m
|
151
|
+
|
152
|
+
\x1b[1m[METADATA]
|
153
|
+
\x1b[0m\x1b[36m{
|
154
|
+
foo = 'bar'
|
155
|
+
}\x1b[0m
|
156
|
+
"""),
|
157
|
+
)
|
158
|
+
|
132
159
|
def test_serialization(self):
|
133
160
|
example = mapping.MappingExample(
|
134
161
|
'the answer is 2', 2, int, context='compute 1 + 1'
|
@@ -44,7 +44,7 @@ langfun/core/coding/python/parsing_test.py,sha256=9vAWF484kWIm6JZq8NFiMgKUDhXV-d
|
|
44
44
|
langfun/core/coding/python/permissions.py,sha256=1QWGHvzL8MM0Ok_auQ9tURqZHtdOfJaDpBzZ29GUE-c,2544
|
45
45
|
langfun/core/coding/python/permissions_test.py,sha256=w5EDb8QxpxgJyZkojyzVWQvDfg366zn99-g__6TbPQ0,2699
|
46
46
|
langfun/core/eval/__init__.py,sha256=Ogdr9OtTywhhLPHi3AZzOD2mXX2oyaHWflrSTMm96uA,1899
|
47
|
-
langfun/core/eval/base.py,sha256=
|
47
|
+
langfun/core/eval/base.py,sha256=BiWColibVo9-4P27Z0hIWXe8_UPocJTSTUdKeOPVwxI,74746
|
48
48
|
langfun/core/eval/base_test.py,sha256=p1EfqviHMz_ppQY8FU67h5OCgL0tzhLvXzGIsq0sVyI,26930
|
49
49
|
langfun/core/eval/matching.py,sha256=9GX8HfO9jKxgNLAivgy5K88Xhoh6Z7Pptq65pe7vht8,9762
|
50
50
|
langfun/core/eval/matching_test.py,sha256=f7iVyXH5KGJBWt4Wp14Bt9J3X59A6Ayfog9MbuFvPew,5532
|
@@ -96,8 +96,8 @@ langfun/core/structured/description.py,sha256=SXW4MJvshFjbR-0gw6rE21o6WXq12UlRXa
|
|
96
96
|
langfun/core/structured/description_test.py,sha256=UtZGjSFUaQ6130t1E5tcL7ODu0xIefkapb53TbnqsK8,7362
|
97
97
|
langfun/core/structured/function_generation.py,sha256=pFgS3vcRAWiuFBol2x5Eeip3XqoudONsOpeJpWyjT3s,7479
|
98
98
|
langfun/core/structured/function_generation_test.py,sha256=ZJI-aaGgWWszn92u7h5IZ9Pl70N2DgAGGJrIxPzsvwg,10065
|
99
|
-
langfun/core/structured/mapping.py,sha256=
|
100
|
-
langfun/core/structured/mapping_test.py,sha256=
|
99
|
+
langfun/core/structured/mapping.py,sha256=CsflMwm5cKJYZ2ag-neroA4CQlhu2wjFRSxKpd_qQDQ,11778
|
100
|
+
langfun/core/structured/mapping_test.py,sha256=zQoVx3kAD5oSm_OJAQA6q41NXLLyn8qs6CIVJgAoP_w,4489
|
101
101
|
langfun/core/structured/parsing.py,sha256=keoVqEfzAbdULh6GawWFsTQzU91MzJXYFZjXGXLaD8g,11492
|
102
102
|
langfun/core/structured/parsing_test.py,sha256=34wDrXaQ-EYhJLfDL8mX9K53oQMSzh5pVYdKjnESmK8,20895
|
103
103
|
langfun/core/structured/prompting.py,sha256=_U6Z65AwXvVvfaQFCY9GawB_QV9S3u7P7BOU2URABmw,8873
|
@@ -117,8 +117,8 @@ langfun/core/templates/demonstration.py,sha256=vCrgYubdZM5Umqcgp8NUVGXgr4P_c-fik
|
|
117
117
|
langfun/core/templates/demonstration_test.py,sha256=SafcDQ0WgI7pw05EmPI2S4v1t3ABKzup8jReCljHeK4,2162
|
118
118
|
langfun/core/templates/selfplay.py,sha256=yhgrJbiYwq47TgzThmHrDQTF4nDrTI09CWGhuQPNv-s,2273
|
119
119
|
langfun/core/templates/selfplay_test.py,sha256=rBW2Qr8yi-aWYwoTwRR-n1peKyMX9QXPZXURjLgoiRs,2264
|
120
|
-
langfun-0.1.1.
|
121
|
-
langfun-0.1.1.
|
122
|
-
langfun-0.1.1.
|
123
|
-
langfun-0.1.1.
|
124
|
-
langfun-0.1.1.
|
120
|
+
langfun-0.1.1.dev20240820.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
|
121
|
+
langfun-0.1.1.dev20240820.dist-info/METADATA,sha256=t3TOnvt67GjpbhKOfk9K19GbEXtGlmNBMwrjqaqbYQU,5234
|
122
|
+
langfun-0.1.1.dev20240820.dist-info/WHEEL,sha256=nCVcAvsfA9TDtwGwhYaRrlPhTLV9m-Ga6mdyDtuwK18,91
|
123
|
+
langfun-0.1.1.dev20240820.dist-info/top_level.txt,sha256=RhlEkHxs1qtzmmtWSwYoLVJAc1YrbPtxQ52uh8Z9VvY,8
|
124
|
+
langfun-0.1.1.dev20240820.dist-info/RECORD,,
|
File without changes
|
File without changes
|