langfun 0.1.2.dev202505070805__py3-none-any.whl → 0.1.2.dev202505090804__py3-none-any.whl

langfun/core/eval/v2/experiment.py

@@ -382,6 +382,7 @@ class Experiment(lf.Component, pg.views.HtmlTreeView.Extension):
       warm_start_from: str | None = None,
       filter: Callable[['Experiment'], bool] | None = None,   # pylint: disable=redefined-builtin
       example_ids: list[int] | None = None,
+      shuffle_inputs: bool = False,
       raise_if_has_error: bool = False,
       reprocess: bool | list[int] = False,
       generate_example_html: Literal['new', 'all', 'no'] | list[int] = 'new',
@@ -431,6 +432,8 @@ class Experiment(lf.Component, pg.views.HtmlTreeView.Extension):
       filter: A filter function to decide whether an experiment should be run
         or not.
       example_ids: The example IDs to run. If None, it will run all examples.
+      shuffle_inputs: If True, the order of evaluatin examples will be shuffled.
+        Neverthless, the example ID remains unchanged for each example.
       raise_if_has_error: If True, it will raise an error if any example fails.
         Otherwise, it will continue and report the error in the output.
       reprocess: A boolean or a list of example IDs. If boolean, it indicates
@@ -470,6 +473,7 @@ class Experiment(lf.Component, pg.views.HtmlTreeView.Extension):
             warm_start_from=warm_start_from,
             filter=filter,
             example_ids=example_ids,
+            shuffle_inputs=shuffle_inputs,
             raise_if_has_error=raise_if_has_error,
             reprocess=reprocess,
             generate_example_html=generate_example_html,
@@ -816,6 +820,14 @@ class Run(pg.Object, pg.views.html.HtmlTreeView.Extension):
       )
   ] = None
 
+  shuffle_inputs: Annotated[
+      bool,
+      (
+          'If True, the order of evaluating examples will be shuffled.'
+          'Otherwise an increasing order will be used.'
+      )
+  ] = False
+
   raise_if_has_error: Annotated[
       bool,
       (

langfun/core/eval/v2/runners.py

@@ -356,6 +356,9 @@ class RunnerBase(Runner):
                   input=evaluation.example_input_by_id(example_id)
               ) for example_id in self.current_run.example_ids
           )
+        if self.current_run.shuffle_inputs:
+          items = list(items)
+          random.shuffle(items)
         self._evaluate_items(evaluation, items)
 
       if cache:

langfun/core/eval/v2/runners_test.py

@@ -163,6 +163,15 @@ class RunnerTest(unittest.TestCase):
     self.assertEqual(plugin.started_example_ids, [5, 7, 9] * 6)
     self.assertEqual(plugin.completed_example_ids, [5, 7, 9] * 6)
 
+  def test_shuffle_inputs(self):
+    root_dir = os.path.join(tempfile.gettempdir(), 'test_shuffle_inputs')
+    exp = eval_test_helper.test_experiment()
+    plugin = TestPlugin()
+    run = exp.run(
+        root_dir, runner='sequential', plugins=[plugin], shuffle_inputs=True
+    )
+    self.assertTrue(run.shuffle_inputs)
+
   def test_filter(self):
     plugin = TestPlugin()
     exp = eval_test_helper.test_experiment()

langfun/core/structured/querying.py

@@ -325,7 +325,7 @@ def query(
 
     ```
     lf.query('1 + 1 = ?', int, lm=lf.llms.Gpt4Turbo())
-    
+
     # Output: 2
     ```
 
@@ -349,7 +349,7 @@ def query(
 
     class Dog(Animal):
       pass
-    
+
     class Entity(pg.Object):
       name: str
 
@@ -562,7 +562,10 @@ def query(
           output_message = lf.AIMessage(processed_text, source=output_message)
     else:
       # Query with structured output.
-      output_message = LfQuery.from_protocol(protocol)(
+      query_cls = LfQuery.from_protocol(protocol)
+      if ':' not in protocol:
+        protocol = f'{protocol}:{query_cls.version}'
+      output_message = query_cls(
           input=(
               query_input.render(lm=lm)
               if isinstance(query_input, lf.Template)
@@ -572,7 +575,7 @@ def query(
           default=default,
           examples=examples,
           response_postprocess=response_postprocess,
-          autofix=autofix if protocol == 'python' else 0,
+          autofix=autofix if protocol.startswith('python:') else 0,
           **kwargs,
       )(
           lm=lm,
@@ -605,6 +608,8 @@ def query(
           ),
           lm=pg.Ref(lm),
           examples=pg.Ref(examples) if examples else [],
+          protocol=protocol,
+          kwargs={k: pg.Ref(v) for k, v in kwargs.items()},
           lm_response=lf.AIMessage(output_message.text, metadata=metadata),
           usage_summary=usage_summary,
           start_time=start_time,
@@ -788,6 +793,14 @@ class QueryInvocation(pg.Object, pg.views.HtmlTreeView.Extension):
       list[mapping.MappingExample],
       'Fewshot exemplars for `lf.query`.'
   ]
+  protocol: Annotated[
+      str,
+      'Protocol of `lf.query`.'
+  ] = 'python'
+  kwargs: Annotated[
+      dict[str, Any],
+      'Kwargs of `lf.query`.'
+  ] = {}
   usage_summary: Annotated[
       lf.UsageSummary,
       'Usage summary for `lf.query`.'
@@ -803,13 +816,17 @@ class QueryInvocation(pg.Object, pg.views.HtmlTreeView.Extension):
 
   @functools.cached_property
   def lm_request(self) -> lf.Message:
-    return query_prompt(self.input, self.schema, examples=self.examples or None)
+    return query_prompt(
+        self.input, self.schema, examples=self.examples or None,
+        protocol=self.protocol,
+        **self.kwargs
+    )
 
   @functools.cached_property
   def output(self) -> Any:
     """The output of `lf.query`. If it failed, returns the `MappingError`."""
     try:
-      return query_output(self.lm_response, self.schema)
+      return query_output(self.lm_response, self.schema, protocol=self.protocol)
     except mapping.MappingError as e:
       return e
 

langfun/core/structured/querying_test.py

@@ -1321,6 +1321,26 @@ class QueryInvocationTest(unittest.TestCase):
     self.assertTrue(queries[0].has_error)
     self.assertIsInstance(queries[0].output, mapping.MappingError)
 
+  def test_kwargs(self):
+    lm = fake.StaticSequence([
+        'Activity(description="hi")',
+    ])
+    with querying.track_queries() as queries:
+      querying.query(
+          'foo {{x}}',
+          Activity,
+          lm=lm,
+          system_message='system message',
+          x=1,
+      )
+    self.assertTrue(queries[0].protocol.startswith('python:'))
+    self.assertEqual(
+        list(queries[0].kwargs.keys()),
+        ['x', 'metadata_system_message']
+    )
+    self.assertIn('foo 1', queries[0].lm_request.text)
+    self.assertEqual(queries[0].lm_request.system_message, 'system message')
+
   def test_to_html(self):
     lm = fake.StaticSequence([
         'Activity(description="hi")',

{langfun-0.1.2.dev202505070805.dist-info → langfun-0.1.2.dev202505090804.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langfun
-Version: 0.1.2.dev202505070805
+Version: 0.1.2.dev202505090804
 Summary: Langfun: Language as Functions.
 Home-page: https://github.com/google/langfun
 Author: Langfun Authors

{langfun-0.1.2.dev202505070805.dist-info → langfun-0.1.2.dev202505090804.dist-info}/RECORD

@@ -67,7 +67,7 @@ langfun/core/eval/v2/evaluation.py,sha256=BijbrYufnlRZLryxezDU32gHjFCCZvLAZMLF6k
 langfun/core/eval/v2/evaluation_test.py,sha256=QNp_HEvRTupvNuLEeYTvylykh1Ut2jpMqHQ-gCUZQ10,6919
 langfun/core/eval/v2/example.py,sha256=Jegt-viQSNYzPVkOZE_M19GON2TYGTct4Cp9HnJ7DGo,10861
 langfun/core/eval/v2/example_test.py,sha256=1DNm6EuyZOq827DKvf3oTRVFkMNM_qTnLUpvOjpgz5I,3419
-langfun/core/eval/v2/experiment.py,sha256=xlQvx-AgPCgHNaBoW1HxddA9wby-odADF0VJ3rQjw_M,32978
+langfun/core/eval/v2/experiment.py,sha256=fb3RHNOSRftV7ZTBfYVV50iEevqdPwRHCt3mgtLzuFw,33408
 langfun/core/eval/v2/experiment_test.py,sha256=UmCobeS6ifPcaGkTJp0WPISolXrVFbeFCBiyJeA0Lt4,13666
 langfun/core/eval/v2/metric_values.py,sha256=_B905bC-jxrYPLSEcP2M8MaHZOVMz_bVrUw8YC4arCE,4660
 langfun/core/eval/v2/metric_values_test.py,sha256=ab2oF_HsIwrSy459108ggyjgefHSPn8UVILR4dRwx14,2634
@@ -79,8 +79,8 @@ langfun/core/eval/v2/progress_tracking.py,sha256=zNhNPGlnJnHELEfFpbTMCSXFn8d1IJ5
 langfun/core/eval/v2/progress_tracking_test.py,sha256=fouMVJkFJqHjbhQJngGLGCmA9x3n0dU4USI2dY163mg,2291
 langfun/core/eval/v2/reporting.py,sha256=yUIPCAMnp7InIzpv1DDWrcLO-75iiOUTpscj7smkfrA,8335
 langfun/core/eval/v2/reporting_test.py,sha256=hcPJJaMtPulqERvHYTpId83WXdqDKnnexmULtK7WKwk,5686
-langfun/core/eval/v2/runners.py,sha256=8PGNlpqSGCW4O-z0E9SHcLklZNceAzwbS7letstljPE,16652
-langfun/core/eval/v2/runners_test.py,sha256=A37fKK2MvAVTiShsg_laluJzJ9AuAQn52k7HPbfD0Ks,11666
+langfun/core/eval/v2/runners.py,sha256=iqbH4jMtnNMhfuv1eHaxJmk1Vvsrz-sAJJFP8U44-tA,16758
+langfun/core/eval/v2/runners_test.py,sha256=DO3xV0sBNB6n65j41xx2i7gqUCJcPF37DFZLEjrmISg,11987
 langfun/core/llms/__init__.py,sha256=QWxRhzVn_vgJvdmW_xs5PcPuDbHsUxTU94YyV4Ofl34,8684
 langfun/core/llms/anthropic.py,sha256=qaclpfX3qeHoZMDxU3Gn-638Vi4IyCbxdow3zgGUHK4,22195
 langfun/core/llms/anthropic_test.py,sha256=dFnNvrgwCYUseDuiuWCBoQ5jloYX9RIlZQf7cCLPNU4,8282
@@ -137,8 +137,8 @@ langfun/core/structured/mapping.py,sha256=gxdcYQP9yqbDRtiJQ1RRAOrKHiCr0h6xBYLCRK
 langfun/core/structured/mapping_test.py,sha256=OntYvfDitAf0tAnzQty3YS90vyEn6FY1Mi93r_ViEk8,9594
 langfun/core/structured/parsing.py,sha256=MGvI7ypXlwfzr5XB8_TFU9Ei0_5reYqkWkv64eAy0EA,12015
 langfun/core/structured/parsing_test.py,sha256=V8Cj1tJK4Lxv_b0YQj6-2hzXZgnYNBa2JR7rOLRBKoQ,22346
-langfun/core/structured/querying.py,sha256=lHSPQ4UKRfhLXRjofG1geIGpQ_ImZf0dIvjXIwr_jNk,30480
-langfun/core/structured/querying_test.py,sha256=_npZ3ztaZc6VerP7nU_QTJscWGgBqiwTE02z_S3Ahd4,40197
+langfun/core/structured/querying.py,sha256=v-C8cU4LNFPwVSVgfO5PFyYxKCTHHh6SdbaR5V5MKKs,30938
+langfun/core/structured/querying_test.py,sha256=6jN3oOSpVEBIlISrCUJtZfY6sBHahF_EBRUEhNNlzW8,40797
 langfun/core/structured/schema.py,sha256=UTonddBx3hVr0Zhm_38jqd8khTSXWy-bkk8l1YOUdLA,28797
 langfun/core/structured/schema_generation.py,sha256=3AcuKvv3VOtKY5zMVqODrxfOuDxzoZtGeBxHlOWDOWw,5308
 langfun/core/structured/schema_generation_test.py,sha256=RM9s71kMNg2jTePwInkiW9fK1ACN37eyPeF8OII-0zw,2950
@@ -156,8 +156,8 @@ langfun/core/templates/demonstration.py,sha256=vCrgYubdZM5Umqcgp8NUVGXgr4P_c-fik
 langfun/core/templates/demonstration_test.py,sha256=SafcDQ0WgI7pw05EmPI2S4v1t3ABKzup8jReCljHeK4,2162
 langfun/core/templates/selfplay.py,sha256=yhgrJbiYwq47TgzThmHrDQTF4nDrTI09CWGhuQPNv-s,2273
 langfun/core/templates/selfplay_test.py,sha256=Ot__1P1M8oJfoTp-M9-PQ6HUXqZKyMwvZ5f7yQ3yfyM,2326
-langfun-0.1.2.dev202505070805.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-langfun-0.1.2.dev202505070805.dist-info/METADATA,sha256=AaKPpm66YjxOfH--Ct9d_I3L_0h5ngugLiU2RwI1CRE,8178
-langfun-0.1.2.dev202505070805.dist-info/WHEEL,sha256=0CuiUZ_p9E4cD6NyLD6UG80LBXYyiSYZOKDm5lp32xk,91
-langfun-0.1.2.dev202505070805.dist-info/top_level.txt,sha256=RhlEkHxs1qtzmmtWSwYoLVJAc1YrbPtxQ52uh8Z9VvY,8
-langfun-0.1.2.dev202505070805.dist-info/RECORD,,
+langfun-0.1.2.dev202505090804.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+langfun-0.1.2.dev202505090804.dist-info/METADATA,sha256=m_vknRs5lJe6eWbqgQG2REHAaNbJn8P1nPpgpxorCsc,8178
+langfun-0.1.2.dev202505090804.dist-info/WHEEL,sha256=0CuiUZ_p9E4cD6NyLD6UG80LBXYyiSYZOKDm5lp32xk,91
+langfun-0.1.2.dev202505090804.dist-info/top_level.txt,sha256=RhlEkHxs1qtzmmtWSwYoLVJAc1YrbPtxQ52uh8Z9VvY,8
+langfun-0.1.2.dev202505090804.dist-info/RECORD,,