langfun 0.1.2.dev202411150804__py3-none-any.whl → 0.1.2.dev202411170804__py3-none-any.whl

langfun/core/eval/v2/runners.py

@@ -15,6 +15,9 @@
 import abc
 import collections
 import concurrent.futures
+import random
+import threading
+import time
 from typing import Any, Annotated, Callable, Iterator
 
 from langfun import core as lf
@@ -373,6 +376,15 @@ class ParallelRunner(RunnerBase):
       'Timeout for each evaluation example.'
   ] = None
 
+  concurrent_startup_delay: Annotated[
+      tuple[int, int] | None,
+      (
+          'A range of seconds to delay the initial evaluation of each thread '
+          'in the thread pool, helping to prevent a burst in LLM QPS at '
+          'startup. If set to None, no delay will be applied.'
+      )
+  ] = None
+
   def _run(self, evaluations: list[Evaluation]) -> None:
     """Runs the evaluations in parallel."""
     def _run_group(evaluation_group: list[Evaluation]):
@@ -405,8 +417,20 @@ class ParallelRunner(RunnerBase):
       self, evaluation: Evaluation, items: Iterator[Example]
   ) -> None:
     """Override run items to run in parallel."""
+    if self.concurrent_startup_delay is not None:
+      thread_delayed = {}
+      def _evaluate_item(item: Example):
+        thread_id = threading.current_thread().ident
+        if thread_id not in thread_delayed:
+          thread_delayed[thread_id] = True
+          time.sleep(random.randint(*self.concurrent_startup_delay))
+        return self.evaluate_item(evaluation, item)
+    else:
+      def _evaluate_item(item: Example):
+        return self.evaluate_item(evaluation, item)
+
     for _, _, _ in lf.concurrent_map(
-        lambda item: self.evaluate_item(evaluation, item),
+        _evaluate_item,
         items,
         max_workers=evaluation.max_workers,
         timeout=self.timeout,

langfun/core/eval/v2/runners_test.py

@@ -198,7 +198,9 @@ class RunnerTest(unittest.TestCase):
     )
     # Global cache.
     root_dir = os.path.join(tempfile.gettempdir(), 'global_cache')
-    run = exp.run(root_dir, runner='sequential', use_cache='global', plugins=[])
+    run = exp.run(
+        root_dir, 'new', runner='sequential', use_cache='global', plugins=[]
+    )
     self.assertTrue(pg.io.path_exists(run.output_path_for(exp, 'cache.json')))
     self.assertEqual(exp.usage_summary.cached.total.num_requests, 4)
     self.assertEqual(exp.usage_summary.uncached.total.num_requests, 2)
@@ -206,7 +208,8 @@ class RunnerTest(unittest.TestCase):
     # Per-dataset cache.
     root_dir = os.path.join(tempfile.gettempdir(), 'per_dataset')
     run = exp.run(
-        root_dir, runner='sequential', use_cache='per_dataset', plugins=[]
+        root_dir, 'new', runner='sequential',
+        use_cache='per_dataset', plugins=[]
     )
     for leaf in exp.leaf_nodes:
       self.assertTrue(
@@ -226,6 +229,9 @@ class RunnerTest(unittest.TestCase):
     self.assertEqual(exp.usage_summary.cached.total.num_requests, 0)
     self.assertEqual(exp.usage_summary.uncached.total.num_requests, 6)
 
+
+class ParallelRunnerTest(RunnerTest):
+
   def test_parallel_runner(self):
     plugin = TestPlugin()
     exp = test_helper.test_experiment()
@@ -266,6 +272,22 @@ class RunnerTest(unittest.TestCase):
         self.assertEqual(node.progress.num_failed, 0)
         self.assertEqual(node.progress.num_processed, node.progress.num_total)
 
+  def test_concurrent_startup_delay(self):
+    plugin = TestPlugin()
+    exp = test_helper.test_experiment()
+    root_dir = os.path.join(
+        tempfile.gettempdir(), 'test_concurrent_startup_delay'
+    )
+    _ = exp.run(
+        root_dir,
+        runner='parallel',
+        plugins=[plugin],
+        concurrent_startup_delay=(0, 5),
+    )
+
+
+class DebugRunnerTest(RunnerTest):
+
   def test_debug_runner(self):
     plugin = TestPlugin()
     exp = test_helper.test_experiment()

langfun/core/language_model.py

@@ -124,17 +124,18 @@ class LMSamplingUsage(pg.Object):
   def __add__(self, other: Optional['LMSamplingUsage']) -> 'LMSamplingUsage':
     if other is None:
       return self
+    if self.estimated_cost is None:
+      estimated_cost = other.estimated_cost
+    elif other.estimated_cost is None:
+      estimated_cost = self.estimated_cost
+    else:
+      estimated_cost = self.estimated_cost + other.estimated_cost
     return LMSamplingUsage(
         prompt_tokens=self.prompt_tokens + other.prompt_tokens,
         completion_tokens=self.completion_tokens + other.completion_tokens,
         total_tokens=self.total_tokens + other.total_tokens,
         num_requests=self.num_requests + other.num_requests,
-        estimated_cost=(
-            self.estimated_cost + other.estimated_cost    # pylint: disable=g-long-ternary
-            if (self.estimated_cost is not None
-                and other.estimated_cost is not None)
-            else None
-        )
+        estimated_cost=estimated_cost,
     )
 
   def __radd__(self, other: Optional['LMSamplingUsage']) -> 'LMSamplingUsage':
@@ -956,7 +957,9 @@ class UsageSummary(pg.Object, pg.views.HtmlTreeView.Extension):
     if self._usage_badge is not None:
       self._usage_badge.update(
           self._badge_text(),
-          tooltip=pg.format(self.total, verbose=False),
+          tooltip=pg.format(
+              self, verbose=False, custom_format=self._tooltip_format
+          ),
           styles=dict(color=self._badge_color()),
       )
 
@@ -978,6 +981,14 @@ class UsageSummary(pg.Object, pg.views.HtmlTreeView.Extension):
     green = int(255 * (1 - normalized_value))
     return f'rgb({red}, {green}, 0)'
 
+  def _tooltip_format(self, v, root_indent):
+    del root_indent
+    if isinstance(v, int):
+      return f'{v:,}'
+    if isinstance(v, float):
+      return f'{v:,.3f}'
+    return None
+
   def _html_tree_view(
       self,
       *,
@@ -993,7 +1004,9 @@ class UsageSummary(pg.Object, pg.views.HtmlTreeView.Extension):
       if usage_badge is None:
         usage_badge = pg.views.html.controls.Badge(
             self._badge_text(),
-            tooltip=pg.format(self.total, verbose=False),
+            tooltip=pg.format(
+                self, custom_format=self._tooltip_format, verbose=False
+            ),
             css_classes=['usage-summary'],
             styles=dict(color=self._badge_color()),
             interactive=True,

langfun/core/language_model_test.py

@@ -744,6 +744,13 @@ class LMSamplingUsageTest(unittest.TestCase):
     self.assertEqual(usage1 + usage2, usage1 + usage2)
     self.assertIs(usage1 + None, usage1)
     self.assertIs(None + usage1, usage1)
+    usage3 = lm_lib.LMSamplingUsage(100, 200, 300, 4, None)
+    self.assertEqual(
+        usage1 + usage3, lm_lib.LMSamplingUsage(200, 400, 600, 8, 5.0)
+    )
+    self.assertEqual(
+        usage3 + usage1, lm_lib.LMSamplingUsage(200, 400, 600, 8, 5.0)
+    )
 
   def test_usage_not_available(self):
     usage_not_available = lm_lib.UsageNotAvailable()

{langfun-0.1.2.dev202411150804.dist-info → langfun-0.1.2.dev202411170804.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: langfun
-Version: 0.1.2.dev202411150804
+Version: 0.1.2.dev202411170804
 Summary: Langfun: Language as Functions.
 Home-page: https://github.com/google/langfun
 Author: Langfun Authors

{langfun-0.1.2.dev202411150804.dist-info → langfun-0.1.2.dev202411170804.dist-info}/RECORD

@@ -8,8 +8,8 @@ langfun/core/console.py,sha256=Fra2_MSWZbFh6rY8HZoYgpGLsrNvhaGuL03znOwQbhM,2529
 langfun/core/console_test.py,sha256=pBOcuNMJdVELywvroptfcRtJMsegMm3wSlHAL2TdxVk,1679
 langfun/core/langfunc.py,sha256=G50YgoVZ0y1GFw2ev41MlOqr6qa8YakbvNC0h_E0PiA,11140
 langfun/core/langfunc_test.py,sha256=fKIAqcSNI_7M6nwoZW77HEam8Oa6vcWhsCNgVJanzb4,8822
-langfun/core/language_model.py,sha256=UtLvclKx55_SAKZ-ajaquudLxKorTARAeZFme5IaPi8,33499
-langfun/core/language_model_test.py,sha256=td81wm4zFPeMb16nmIuIZ6eHtpYhH0k6IeiYLfGgR0o,31525
+langfun/core/language_model.py,sha256=b15MZ_qbydnz5vQ09t7sf9tc3C7qWvMSxUrGfT0p99I,33827
+langfun/core/language_model_test.py,sha256=hnYhtw7GM_TbhgsJzHNYTaoDewUlPHpOVlI7xEkCFuI,31783
 langfun/core/logging.py,sha256=uslllP0RTGN223oro1m4nZZ0bFppcL07OwbFKm2iG6k,7519
 langfun/core/logging_test.py,sha256=b5bPTSUoYeICATaO6I8dOVumodwRbxSp1Oz96Sf3KcE,6104
 langfun/core/memory.py,sha256=f-asN1F7Vehgdn_fK84v73GrEUOxRtaW934keutTKjk,2416
@@ -76,8 +76,8 @@ langfun/core/eval/v2/progress_tracking.py,sha256=1imwSbllxHWG3zYrzo2NvytBZsVtjqu
 langfun/core/eval/v2/progress_tracking_test.py,sha256=eY2HvZeEXDA5Zyfi2m5NDWO_9kSfQsaAOEcIhkSbWCY,1874
 langfun/core/eval/v2/reporting.py,sha256=TGkli1IDwqfqsCJ_WslOMGk_24JDg7oRRTGXlAJlWpc,4361
 langfun/core/eval/v2/reporting_test.py,sha256=JxffbUPWInUyLjo-AQVFrllga884Mdfm05R86FtxSss,1482
-langfun/core/eval/v2/runners.py,sha256=2OHAVTbqq9hZ3qZpUEvQ--9X-Cr_z8Ghc3MRXCfclpk,13442
-langfun/core/eval/v2/runners_test.py,sha256=s3GgWA-H9x0JyPhPZq2s9-5GXGHo5dSbDD-4faX0h_E,11164
+langfun/core/eval/v2/runners.py,sha256=zJmu-amUiYv1g0Ek4c3mXkBgp-AFvSF7WpXVZCCf7Y4,14245
+langfun/core/eval/v2/runners_test.py,sha256=UeiUNygux_U6iGVG18rhp68ZE4hoWeoT6XsXvSjxNQg,11620
 langfun/core/eval/v2/test_helper.py,sha256=pDpZTBnWRR5xjJv3Uy3NWEzArqlL8FTMOgeR4C53F5M,2348
 langfun/core/llms/__init__.py,sha256=uR2vLghsnZqY6OjZKAs9Lo-YFNxZNunf3A0q6-1GYlc,6346
 langfun/core/llms/anthropic.py,sha256=uJXVgaFONL8okOSVQ4VGMGht_VZ30m1hoLzmDbIjmks,13990
@@ -148,8 +148,8 @@ langfun/core/templates/demonstration.py,sha256=vCrgYubdZM5Umqcgp8NUVGXgr4P_c-fik
 langfun/core/templates/demonstration_test.py,sha256=SafcDQ0WgI7pw05EmPI2S4v1t3ABKzup8jReCljHeK4,2162
 langfun/core/templates/selfplay.py,sha256=yhgrJbiYwq47TgzThmHrDQTF4nDrTI09CWGhuQPNv-s,2273
 langfun/core/templates/selfplay_test.py,sha256=Ot__1P1M8oJfoTp-M9-PQ6HUXqZKyMwvZ5f7yQ3yfyM,2326
-langfun-0.1.2.dev202411150804.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-langfun-0.1.2.dev202411150804.dist-info/METADATA,sha256=1RftsAD_qubksdeWSmhbcR35HGkcT7opT3L1c1EO5Ok,8890
-langfun-0.1.2.dev202411150804.dist-info/WHEEL,sha256=R06PA3UVYHThwHvxuRWMqaGcr-PuniXahwjmQRFMEkY,91
-langfun-0.1.2.dev202411150804.dist-info/top_level.txt,sha256=RhlEkHxs1qtzmmtWSwYoLVJAc1YrbPtxQ52uh8Z9VvY,8
-langfun-0.1.2.dev202411150804.dist-info/RECORD,,
+langfun-0.1.2.dev202411170804.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+langfun-0.1.2.dev202411170804.dist-info/METADATA,sha256=Sr1olxW-wSjImKrZFtjVkrCe-_-uqVWkPa20-axQGPQ,8890
+langfun-0.1.2.dev202411170804.dist-info/WHEEL,sha256=R06PA3UVYHThwHvxuRWMqaGcr-PuniXahwjmQRFMEkY,91
+langfun-0.1.2.dev202411170804.dist-info/top_level.txt,sha256=RhlEkHxs1qtzmmtWSwYoLVJAc1YrbPtxQ52uh8Z9VvY,8
+langfun-0.1.2.dev202411170804.dist-info/RECORD,,