langfun 0.1.2.dev202505070805__py3-none-any.whl → 0.1.2.dev202505080804__py3-none-any.whl

langfun/core/eval/v2/experiment.py

@@ -382,6 +382,7 @@ class Experiment(lf.Component, pg.views.HtmlTreeView.Extension):
       warm_start_from: str | None = None,
       filter: Callable[['Experiment'], bool] | None = None,   # pylint: disable=redefined-builtin
       example_ids: list[int] | None = None,
+      shuffle_inputs: bool = False,
       raise_if_has_error: bool = False,
       reprocess: bool | list[int] = False,
       generate_example_html: Literal['new', 'all', 'no'] | list[int] = 'new',
@@ -431,6 +432,8 @@ class Experiment(lf.Component, pg.views.HtmlTreeView.Extension):
       filter: A filter function to decide whether an experiment should be run
         or not.
       example_ids: The example IDs to run. If None, it will run all examples.
+      shuffle_inputs: If True, the order of evaluatin examples will be shuffled.
+        Neverthless, the example ID remains unchanged for each example.
       raise_if_has_error: If True, it will raise an error if any example fails.
         Otherwise, it will continue and report the error in the output.
       reprocess: A boolean or a list of example IDs. If boolean, it indicates
@@ -470,6 +473,7 @@ class Experiment(lf.Component, pg.views.HtmlTreeView.Extension):
             warm_start_from=warm_start_from,
             filter=filter,
             example_ids=example_ids,
+            shuffle_inputs=shuffle_inputs,
             raise_if_has_error=raise_if_has_error,
             reprocess=reprocess,
             generate_example_html=generate_example_html,
@@ -816,6 +820,14 @@ class Run(pg.Object, pg.views.html.HtmlTreeView.Extension):
       )
   ] = None
 
+  shuffle_inputs: Annotated[
+      bool,
+      (
+          'If True, the order of evaluating examples will be shuffled.'
+          'Otherwise an increasing order will be used.'
+      )
+  ] = False
+
   raise_if_has_error: Annotated[
       bool,
       (

langfun/core/eval/v2/runners.py

@@ -356,6 +356,9 @@ class RunnerBase(Runner):
                   input=evaluation.example_input_by_id(example_id)
               ) for example_id in self.current_run.example_ids
           )
+        if self.current_run.shuffle_inputs:
+          items = list(items)
+          random.shuffle(items)
         self._evaluate_items(evaluation, items)
 
       if cache:

langfun/core/eval/v2/runners_test.py

@@ -163,6 +163,15 @@ class RunnerTest(unittest.TestCase):
     self.assertEqual(plugin.started_example_ids, [5, 7, 9] * 6)
     self.assertEqual(plugin.completed_example_ids, [5, 7, 9] * 6)
 
+  def test_shuffle_inputs(self):
+    root_dir = os.path.join(tempfile.gettempdir(), 'test_shuffle_inputs')
+    exp = eval_test_helper.test_experiment()
+    plugin = TestPlugin()
+    run = exp.run(
+        root_dir, runner='sequential', plugins=[plugin], shuffle_inputs=True
+    )
+    self.assertTrue(run.shuffle_inputs)
+
   def test_filter(self):
     plugin = TestPlugin()
     exp = eval_test_helper.test_experiment()

{langfun-0.1.2.dev202505070805.dist-info → langfun-0.1.2.dev202505080804.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langfun
-Version: 0.1.2.dev202505070805
+Version: 0.1.2.dev202505080804
 Summary: Langfun: Language as Functions.
 Home-page: https://github.com/google/langfun
 Author: Langfun Authors

{langfun-0.1.2.dev202505070805.dist-info → langfun-0.1.2.dev202505080804.dist-info}/RECORD

@@ -67,7 +67,7 @@ langfun/core/eval/v2/evaluation.py,sha256=BijbrYufnlRZLryxezDU32gHjFCCZvLAZMLF6k
 langfun/core/eval/v2/evaluation_test.py,sha256=QNp_HEvRTupvNuLEeYTvylykh1Ut2jpMqHQ-gCUZQ10,6919
 langfun/core/eval/v2/example.py,sha256=Jegt-viQSNYzPVkOZE_M19GON2TYGTct4Cp9HnJ7DGo,10861
 langfun/core/eval/v2/example_test.py,sha256=1DNm6EuyZOq827DKvf3oTRVFkMNM_qTnLUpvOjpgz5I,3419
-langfun/core/eval/v2/experiment.py,sha256=xlQvx-AgPCgHNaBoW1HxddA9wby-odADF0VJ3rQjw_M,32978
+langfun/core/eval/v2/experiment.py,sha256=fb3RHNOSRftV7ZTBfYVV50iEevqdPwRHCt3mgtLzuFw,33408
 langfun/core/eval/v2/experiment_test.py,sha256=UmCobeS6ifPcaGkTJp0WPISolXrVFbeFCBiyJeA0Lt4,13666
 langfun/core/eval/v2/metric_values.py,sha256=_B905bC-jxrYPLSEcP2M8MaHZOVMz_bVrUw8YC4arCE,4660
 langfun/core/eval/v2/metric_values_test.py,sha256=ab2oF_HsIwrSy459108ggyjgefHSPn8UVILR4dRwx14,2634
@@ -79,8 +79,8 @@ langfun/core/eval/v2/progress_tracking.py,sha256=zNhNPGlnJnHELEfFpbTMCSXFn8d1IJ5
 langfun/core/eval/v2/progress_tracking_test.py,sha256=fouMVJkFJqHjbhQJngGLGCmA9x3n0dU4USI2dY163mg,2291
 langfun/core/eval/v2/reporting.py,sha256=yUIPCAMnp7InIzpv1DDWrcLO-75iiOUTpscj7smkfrA,8335
 langfun/core/eval/v2/reporting_test.py,sha256=hcPJJaMtPulqERvHYTpId83WXdqDKnnexmULtK7WKwk,5686
-langfun/core/eval/v2/runners.py,sha256=8PGNlpqSGCW4O-z0E9SHcLklZNceAzwbS7letstljPE,16652
-langfun/core/eval/v2/runners_test.py,sha256=A37fKK2MvAVTiShsg_laluJzJ9AuAQn52k7HPbfD0Ks,11666
+langfun/core/eval/v2/runners.py,sha256=iqbH4jMtnNMhfuv1eHaxJmk1Vvsrz-sAJJFP8U44-tA,16758
+langfun/core/eval/v2/runners_test.py,sha256=DO3xV0sBNB6n65j41xx2i7gqUCJcPF37DFZLEjrmISg,11987
 langfun/core/llms/__init__.py,sha256=QWxRhzVn_vgJvdmW_xs5PcPuDbHsUxTU94YyV4Ofl34,8684
 langfun/core/llms/anthropic.py,sha256=qaclpfX3qeHoZMDxU3Gn-638Vi4IyCbxdow3zgGUHK4,22195
 langfun/core/llms/anthropic_test.py,sha256=dFnNvrgwCYUseDuiuWCBoQ5jloYX9RIlZQf7cCLPNU4,8282
@@ -156,8 +156,8 @@ langfun/core/templates/demonstration.py,sha256=vCrgYubdZM5Umqcgp8NUVGXgr4P_c-fik
 langfun/core/templates/demonstration_test.py,sha256=SafcDQ0WgI7pw05EmPI2S4v1t3ABKzup8jReCljHeK4,2162
 langfun/core/templates/selfplay.py,sha256=yhgrJbiYwq47TgzThmHrDQTF4nDrTI09CWGhuQPNv-s,2273
 langfun/core/templates/selfplay_test.py,sha256=Ot__1P1M8oJfoTp-M9-PQ6HUXqZKyMwvZ5f7yQ3yfyM,2326
-langfun-0.1.2.dev202505070805.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-langfun-0.1.2.dev202505070805.dist-info/METADATA,sha256=AaKPpm66YjxOfH--Ct9d_I3L_0h5ngugLiU2RwI1CRE,8178
-langfun-0.1.2.dev202505070805.dist-info/WHEEL,sha256=0CuiUZ_p9E4cD6NyLD6UG80LBXYyiSYZOKDm5lp32xk,91
-langfun-0.1.2.dev202505070805.dist-info/top_level.txt,sha256=RhlEkHxs1qtzmmtWSwYoLVJAc1YrbPtxQ52uh8Z9VvY,8
-langfun-0.1.2.dev202505070805.dist-info/RECORD,,
+langfun-0.1.2.dev202505080804.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+langfun-0.1.2.dev202505080804.dist-info/METADATA,sha256=CAqwJ8wOEhPMNBg-5-9XUOB8qWL3wuZJkycpxgcKugg,8178
+langfun-0.1.2.dev202505080804.dist-info/WHEEL,sha256=0CuiUZ_p9E4cD6NyLD6UG80LBXYyiSYZOKDm5lp32xk,91
+langfun-0.1.2.dev202505080804.dist-info/top_level.txt,sha256=RhlEkHxs1qtzmmtWSwYoLVJAc1YrbPtxQ52uh8Z9VvY,8
+langfun-0.1.2.dev202505080804.dist-info/RECORD,,