langfun 0.1.2.dev202505070805__tar.gz → 0.1.2.dev202505080804__tar.gz

{langfun-0.1.2.dev202505070805 → langfun-0.1.2.dev202505080804}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langfun
-Version: 0.1.2.dev202505070805
+Version: 0.1.2.dev202505080804
 Summary: Langfun: Language as Functions.
 Home-page: https://github.com/google/langfun
 Author: Langfun Authors

{langfun-0.1.2.dev202505070805 → langfun-0.1.2.dev202505080804}/langfun/core/eval/v2/experiment.py

@@ -382,6 +382,7 @@ class Experiment(lf.Component, pg.views.HtmlTreeView.Extension):
       warm_start_from: str | None = None,
       filter: Callable[['Experiment'], bool] | None = None,   # pylint: disable=redefined-builtin
       example_ids: list[int] | None = None,
+      shuffle_inputs: bool = False,
       raise_if_has_error: bool = False,
       reprocess: bool | list[int] = False,
       generate_example_html: Literal['new', 'all', 'no'] | list[int] = 'new',
@@ -431,6 +432,8 @@ class Experiment(lf.Component, pg.views.HtmlTreeView.Extension):
       filter: A filter function to decide whether an experiment should be run
         or not.
       example_ids: The example IDs to run. If None, it will run all examples.
+      shuffle_inputs: If True, the order of evaluatin examples will be shuffled.
+        Neverthless, the example ID remains unchanged for each example.
       raise_if_has_error: If True, it will raise an error if any example fails.
         Otherwise, it will continue and report the error in the output.
       reprocess: A boolean or a list of example IDs. If boolean, it indicates
@@ -470,6 +473,7 @@ class Experiment(lf.Component, pg.views.HtmlTreeView.Extension):
             warm_start_from=warm_start_from,
             filter=filter,
             example_ids=example_ids,
+            shuffle_inputs=shuffle_inputs,
             raise_if_has_error=raise_if_has_error,
             reprocess=reprocess,
             generate_example_html=generate_example_html,
@@ -816,6 +820,14 @@ class Run(pg.Object, pg.views.html.HtmlTreeView.Extension):
       )
   ] = None
 
+  shuffle_inputs: Annotated[
+      bool,
+      (
+          'If True, the order of evaluating examples will be shuffled.'
+          'Otherwise an increasing order will be used.'
+      )
+  ] = False
+
   raise_if_has_error: Annotated[
       bool,
       (

{langfun-0.1.2.dev202505070805 → langfun-0.1.2.dev202505080804}/langfun/core/eval/v2/runners.py

@@ -356,6 +356,9 @@ class RunnerBase(Runner):
                   input=evaluation.example_input_by_id(example_id)
               ) for example_id in self.current_run.example_ids
           )
+        if self.current_run.shuffle_inputs:
+          items = list(items)
+          random.shuffle(items)
         self._evaluate_items(evaluation, items)
 
       if cache:

{langfun-0.1.2.dev202505070805 → langfun-0.1.2.dev202505080804}/langfun/core/eval/v2/runners_test.py

@@ -163,6 +163,15 @@ class RunnerTest(unittest.TestCase):
     self.assertEqual(plugin.started_example_ids, [5, 7, 9] * 6)
     self.assertEqual(plugin.completed_example_ids, [5, 7, 9] * 6)
 
+  def test_shuffle_inputs(self):
+    root_dir = os.path.join(tempfile.gettempdir(), 'test_shuffle_inputs')
+    exp = eval_test_helper.test_experiment()
+    plugin = TestPlugin()
+    run = exp.run(
+        root_dir, runner='sequential', plugins=[plugin], shuffle_inputs=True
+    )
+    self.assertTrue(run.shuffle_inputs)
+
   def test_filter(self):
     plugin = TestPlugin()
     exp = eval_test_helper.test_experiment()

{langfun-0.1.2.dev202505070805 → langfun-0.1.2.dev202505080804}/langfun.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langfun
-Version: 0.1.2.dev202505070805
+Version: 0.1.2.dev202505080804
 Summary: Langfun: Language as Functions.
 Home-page: https://github.com/google/langfun
 Author: Langfun Authors