langfun 0.1.2.dev202411090804__py3-none-any.whl → 0.1.2.dev202411140804__py3-none-any.whl

langfun/core/llms/compositional.py

@@ -0,0 +1,101 @@
+# Copyright 2024 The Langfun Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Compositions of different LLM models."""
+import random
+from typing import Annotated
+
+import langfun.core as lf
+import pyglove as pg
+
+
+@pg.use_init_args(['candidates', 'seed'])
+class RandomChoice(lf.LanguageModel):
+  """Random choice of a list of LLM models."""
+
+  candidates: Annotated[
+      list[lf.LanguageModel],
+      (
+          'A list of LLMs as candidates to choose from.'
+      )
+  ]
+
+  seed: Annotated[
+      int,
+      (
+          'The random seed to use for the random choice.'
+      )
+  ] = 0
+
+  def _on_bound(self):
+    super()._on_bound()
+    self._rand = random.Random(self.seed)
+    # Applying sampling options to all candidates.
+    parent_non_default = self.sampling_options.sym_nondefault()
+    if parent_non_default:
+      for c in self.candidates:
+        c.sampling_options.rebind(
+            parent_non_default, notify_parents=False, raise_on_no_change=False
+        )
+
+  @property
+  def model_id(self) -> str:
+    model_ids = ', '.join(
+        sorted(c.model_id for c in self.candidates)
+    )
+    return f'RandomChoice({model_ids})'
+
+  @property
+  def resource_id(self) -> str:
+    resource_ids = ', '.join(
+        sorted(c.resource_id for c in self.candidates)
+    )
+    return f'RandomChoice({resource_ids})'
+
+  def _select_lm(self) -> lf.LanguageModel:
+    """Selects a random LLM from the candidates."""
+    return self._rand.choice(self.candidates)
+
+  def sample(
+      self,
+      prompts: list[str | lf.Message],
+      *,
+      cache_seed: int = 0,
+      **kwargs,
+  ) -> list[lf.LMSamplingResult]:
+    return self._select_lm().sample(
+        prompts, cache_seed=cache_seed, **kwargs
+    )
+
+  def __call__(
+      self, prompt: lf.Message, *, cache_seed: int = 0, **kwargs
+  ) -> lf.Message:
+    return self._select_lm()(prompt, cache_seed=cache_seed, **kwargs)
+
+  def score(
+      self,
+      prompt: str | lf.Message | list[lf.Message],
+      completions: list[str | lf.Message],
+      **kwargs,
+  ) -> list[lf.LMScoringResult]:
+    return self._select_lm().score(prompt, completions, **kwargs)
+
+  def tokenize(
+      self,
+      prompt: str | lf.Message,
+      **kwargs,
+  ) -> list[tuple[str | bytes, int]]:
+    return self._select_lm().tokenize(prompt, **kwargs)
+
+  def _sample(self, *arg, **kwargs):
+    assert False, 'Should never trigger.'

langfun/core/llms/compositional_test.py

@@ -0,0 +1,73 @@
+# Copyright 2024 The Langfun Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for compositional models."""
+import unittest
+
+import langfun.core as lf
+from langfun.core.llms import compositional
+from langfun.core.llms import fake
+
+
+class RandomChoiceTest(unittest.TestCase):
+
+  def test_basic(self):
+    lm = compositional.RandomChoice([
+        fake.StaticResponse('hi'),
+        fake.StaticSequence(['hello', 'world'])
+    ])
+    self.assertEqual(
+        lm.model_id,
+        'RandomChoice(StaticResponse, StaticSequence)'
+    )
+    self.assertEqual(
+        lm.resource_id,
+        'RandomChoice(StaticResponse, StaticSequence)'
+    )
+    self.assertEqual(
+        [lm('a'), lm('b'), lm('c')],
+        ['hello', 'world', 'hi']
+    )
+    lm = lm.clone()
+    self.assertEqual(
+        [
+            x.samples[0].response for x in [
+                lm.sample(['a'])[0],
+                lm.sample(['b'])[0],
+                lm.sample(['c'])[0],
+            ]
+        ],
+        ['hello', 'world', 'hi']
+    )
+    self.assertEqual(
+        lm.score('hello', ['world']),
+        [lf.LMScoringResult(0.0)]
+    )
+    self.assertEqual(
+        lm.tokenize('hello'),
+        [('hello', 0)]
+    )
+
+  def test_sampling_options(self):
+    lm = compositional.RandomChoice([
+        fake.StaticResponse('hi'),
+        fake.StaticSequence(['hello', 'world'])
+    ], temperature=0.5)
+    self.assertEqual(
+        lm.candidates[0].sampling_options.temperature,
+        0.5
+    )
+
+
+if __name__ == '__main__':
+  unittest.main()

langfun/core/llms/vertexai.py

@@ -51,13 +51,13 @@ AVGERAGE_CHARS_PER_TOEKN = 4
 SUPPORTED_MODELS_AND_SETTINGS = {
     'gemini-1.5-pro-001': pg.Dict(
         api='gemini',
-        rpm=500,
+        rpm=100,
         cost_per_1k_input_chars=0.0003125,
         cost_per_1k_output_chars=0.00125,
     ),
     'gemini-1.5-pro-002': pg.Dict(
         api='gemini',
-        rpm=500,
+        rpm=100,
         cost_per_1k_input_chars=0.0003125,
         cost_per_1k_output_chars=0.00125,
     ),
@@ -75,7 +75,7 @@ SUPPORTED_MODELS_AND_SETTINGS = {
     ),
     'gemini-1.5-pro': pg.Dict(
         api='gemini',
-        rpm=500,
+        rpm=100,
         cost_per_1k_input_chars=0.0003125,
         cost_per_1k_output_chars=0.00125,
     ),
@@ -87,7 +87,7 @@ SUPPORTED_MODELS_AND_SETTINGS = {
     ),
     'gemini-1.5-pro-latest': pg.Dict(
         api='gemini',
-        rpm=500,
+        rpm=100,
         cost_per_1k_input_chars=0.0003125,
         cost_per_1k_output_chars=0.00125,
     ),

{langfun-0.1.2.dev202411090804.dist-info → langfun-0.1.2.dev202411140804.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: langfun
-Version: 0.1.2.dev202411090804
+Version: 0.1.2.dev202411140804
 Summary: Langfun: Language as Functions.
 Home-page: https://github.com/google/langfun
 Author: Langfun Authors

{langfun-0.1.2.dev202411090804.dist-info → langfun-0.1.2.dev202411140804.dist-info}/RECORD

@@ -4,12 +4,12 @@ langfun/core/component.py,sha256=HVrEoTL1Y01iqOHC3FYdbAOnffqfHHtGJXoK1vkdEwo,115
 langfun/core/component_test.py,sha256=sG-T2wpvBfHqWGZE7sc4NayJj2aj5QFBzSwFiwrGEIc,10376
 langfun/core/concurrent.py,sha256=QMNYhB_PyjvVJtabMokpzotZRYvyE9iYu2QsgwDk7M4,29552
 langfun/core/concurrent_test.py,sha256=ILlAjfhV84yJfY1QLe3N9aYry1sCjY-ywfIlXGafenI,17336
-langfun/core/console.py,sha256=bk5rNPNm9rMGW5YT2HixxU04p2umnoabn5SDz6Dqe88,2317
-langfun/core/console_test.py,sha256=5SYJdxpJGLgdSSQqqMPoA1X6jpsLD8rgcyk-EgI65oE,1077
+langfun/core/console.py,sha256=Fra2_MSWZbFh6rY8HZoYgpGLsrNvhaGuL03znOwQbhM,2529
+langfun/core/console_test.py,sha256=pBOcuNMJdVELywvroptfcRtJMsegMm3wSlHAL2TdxVk,1679
 langfun/core/langfunc.py,sha256=G50YgoVZ0y1GFw2ev41MlOqr6qa8YakbvNC0h_E0PiA,11140
 langfun/core/langfunc_test.py,sha256=fKIAqcSNI_7M6nwoZW77HEam8Oa6vcWhsCNgVJanzb4,8822
-langfun/core/language_model.py,sha256=jOvWyiKvcv2yJGBNPWmxV8wyzuWnCcrc1FhldYBtPkE,30219
-langfun/core/language_model_test.py,sha256=cgoSdKwicnvHYo-tQeTdONXAVM-bvWLzgTlqxvace-A,28424
+langfun/core/language_model.py,sha256=UtLvclKx55_SAKZ-ajaquudLxKorTARAeZFme5IaPi8,33499
+langfun/core/language_model_test.py,sha256=td81wm4zFPeMb16nmIuIZ6eHtpYhH0k6IeiYLfGgR0o,31525
 langfun/core/logging.py,sha256=uslllP0RTGN223oro1m4nZZ0bFppcL07OwbFKm2iG6k,7519
 langfun/core/logging_test.py,sha256=b5bPTSUoYeICATaO6I8dOVumodwRbxSp1Oz96Sf3KcE,6104
 langfun/core/memory.py,sha256=f-asN1F7Vehgdn_fK84v73GrEUOxRtaW934keutTKjk,2416
@@ -43,7 +43,7 @@ langfun/core/coding/python/parsing.py,sha256=LMg8REP4VDY0YQjtPAGNAW4rKlMNdSXF8m1
 langfun/core/coding/python/parsing_test.py,sha256=9vAWF484kWIm6JZq8NFiMgKUDhXV-deRl1QMmNERfAA,7386
 langfun/core/coding/python/permissions.py,sha256=1QWGHvzL8MM0Ok_auQ9tURqZHtdOfJaDpBzZ29GUE-c,2544
 langfun/core/coding/python/permissions_test.py,sha256=w5EDb8QxpxgJyZkojyzVWQvDfg366zn99-g__6TbPQ0,2699
-langfun/core/eval/__init__.py,sha256=Ogdr9OtTywhhLPHi3AZzOD2mXX2oyaHWflrSTMm96uA,1899
+langfun/core/eval/__init__.py,sha256=OEXr1ZRuvLuhJJfuQ1ZWQ-SvYzjyrtiAAEogYaB7E6o,1933
 langfun/core/eval/base.py,sha256=ajHUS_BdzBIDKEwAmMzne0lJi8HcDhPmyw_plO0p7G0,75814
 langfun/core/eval/base_test.py,sha256=-LsIV9DXlDal0EnOlaWpibJvfef0NbxtZAm0OH_abAE,27189
 langfun/core/eval/matching.py,sha256=UnjdM_ebPqXKJamY4lvL3AYxrMIz3LqkjRTnHJ5xsYc,9349
@@ -52,9 +52,33 @@ langfun/core/eval/patching.py,sha256=R0s2eAd1m97exQt06dmUL0V_MBG0W2Hxg7fhNB7cXW0
 langfun/core/eval/patching_test.py,sha256=8kCd54Egjju22FMgtJuxEsrXkW8ifs-UUBHtrCG1L6w,4775
 langfun/core/eval/scoring.py,sha256=B69IsIxiPs1xZcOBFIhZF70YmDue2Siik-CPL2bh33s,6254
 langfun/core/eval/scoring_test.py,sha256=O8olHbrUEg60gMxwOkWzKBJZpZoUlmVnBANX5Se2SXM,4546
-langfun/core/llms/__init__.py,sha256=ATayGMO5rQzkPBAf7zl3vfgRQ8fth33TFVXv_tvraCQ,6190
+langfun/core/eval/v2/__init__.py,sha256=XMpkKjd_vL_qzCQYPAwR1NNO0T_Zo5j4kP3eYL-EJLc,1428
+langfun/core/eval/v2/checkpointing.py,sha256=AhChb5U0fTd0NmDHaIaVE9qJ8zFEe7Lhq7KjIt1D2pQ,3766
+langfun/core/eval/v2/checkpointing_test.py,sha256=xU5asikwUn-rCmNd93ssvT6gDiKvvWogYKQa0r_TqOM,3046
+langfun/core/eval/v2/evaluation.py,sha256=hausa1NnUGi56KNrucd61qv9km1eDvcj2GBqiJzb0FA,19371
+langfun/core/eval/v2/evaluation_test.py,sha256=hh6L2HhQPQ6NBv1pXKcNkYraNcV9MLuJ--69t9jbmaI,5846
+langfun/core/eval/v2/example.py,sha256=fURrvdNmMsVMqoEErcsmLmC6Xq3ny16dYsnLH8HVlcY,9626
+langfun/core/eval/v2/example_test.py,sha256=WcJmU7IQQXvjFia63mokySC4CqxzVL9Wso1sC5F0YK8,3032
+langfun/core/eval/v2/experiment.py,sha256=xkROLFYj2Nf6G9wunfVx6nEhdMX_560hRSsB8qT3S_Q,28787
+langfun/core/eval/v2/experiment_test.py,sha256=R0ujPSluvDu5gwxGpiSWJ3eTiVpfleJDwwT1pXN8Nvg,8933
+langfun/core/eval/v2/metric_values.py,sha256=_B905bC-jxrYPLSEcP2M8MaHZOVMz_bVrUw8YC4arCE,4660
+langfun/core/eval/v2/metric_values_test.py,sha256=ab2oF_HsIwrSy459108ggyjgefHSPn8UVILR4dRwx14,2634
+langfun/core/eval/v2/metrics.py,sha256=bl8i6u-ZHRBz4hAc3LzsZ2Dc7ZRQcuTYeUhhH-GxfF0,10628
+langfun/core/eval/v2/metrics_test.py,sha256=p4FzLJsE8XAzAQuyP9hfEf9YeKWZ__PO_ue8a9P0-cc,6082
+langfun/core/eval/v2/progress.py,sha256=azZgssQgNdv3IgjKEaQBuGI5ucFDNbdi02P4z_nQ8GE,10292
+langfun/core/eval/v2/progress_test.py,sha256=YU7VHzmy5knPZwj9vpBN3rQQH2tukj9eKHkuBCI62h8,2540
+langfun/core/eval/v2/progress_tracking.py,sha256=1imwSbllxHWG3zYrzo2NvytBZsVtjqum6bmXGGsvT1E,5987
+langfun/core/eval/v2/progress_tracking_test.py,sha256=eY2HvZeEXDA5Zyfi2m5NDWO_9kSfQsaAOEcIhkSbWCY,1874
+langfun/core/eval/v2/reporting.py,sha256=TGkli1IDwqfqsCJ_WslOMGk_24JDg7oRRTGXlAJlWpc,4361
+langfun/core/eval/v2/reporting_test.py,sha256=JxffbUPWInUyLjo-AQVFrllga884Mdfm05R86FtxSss,1482
+langfun/core/eval/v2/runners.py,sha256=2OHAVTbqq9hZ3qZpUEvQ--9X-Cr_z8Ghc3MRXCfclpk,13442
+langfun/core/eval/v2/runners_test.py,sha256=s3GgWA-H9x0JyPhPZq2s9-5GXGHo5dSbDD-4faX0h_E,11164
+langfun/core/eval/v2/test_helper.py,sha256=pDpZTBnWRR5xjJv3Uy3NWEzArqlL8FTMOgeR4C53F5M,2348
+langfun/core/llms/__init__.py,sha256=i0m-fVpwuIN_Jno1M-5O9ikzbVbvXWJKFQZO22MFPq8,6272
 langfun/core/llms/anthropic.py,sha256=XPQxjfe9O4b-CygCgqvQU0MPSfe1rU7uErNbo8zth7Q,13606
 langfun/core/llms/anthropic_test.py,sha256=-2U4kc_pgBM7wqxu8RuxzyHPGww1EAWqKUvN4PW8Btw,8058
+langfun/core/llms/compositional.py,sha256=csW_FLlgL-tpeyCOTVvfUQkMa_zCN5Y2I-YbSNuK27U,2872
+langfun/core/llms/compositional_test.py,sha256=4eTnOer-DncRKGaIJW2ZQQMLnt5r2R0UIx_DYOvGAQo,2027
 langfun/core/llms/fake.py,sha256=gCHBYBLvBCsC78HI1hpoqXCS-p1FMTgY1P1qh_sGBPk,3070
 langfun/core/llms/fake_test.py,sha256=2h13qkwEz_JR0mtUDPxdAhQo7MueXaFSwsD2DIRDW9g,7653
 langfun/core/llms/google_genai.py,sha256=btUIfWteBoj8Jl0j8d3e8hyI6p3Biq4rldlQYctVQfg,10936
@@ -67,7 +91,7 @@ langfun/core/llms/openai.py,sha256=qrAiJxE0tS7ZqjaVzRgVJKtMtoo1Z5TYpvi5ikTwPpw,2
 langfun/core/llms/openai_test.py,sha256=_8cd3VRNEUfE0-Ko1RiM6MlC5hjalRj7nYTJNhG1p3E,18907
 langfun/core/llms/rest.py,sha256=sWbYUV8S3SuOg9giq7xwD-xDRfaF7NP_ig7bI52-Rj4,3442
 langfun/core/llms/rest_test.py,sha256=NZ3Nf0XQVpT9kLP5cBVo_yBHLI7vWTYhWQxYEJVMGs4,3472
-langfun/core/llms/vertexai.py,sha256=eUFU0JjgpTVCAvQVEWiGqZrlE3Dye-EkzZCF1P8nwc4,18866
+langfun/core/llms/vertexai.py,sha256=-KB880Ovab6CQqI-Y5Y6V7RlEA0tAIazmnnG74Ebp4A,18866
 langfun/core/llms/vertexai_test.py,sha256=7uBVOF5VF86xQ9HFAbSTh4J-0NjYLnuotBS1YRm-vgw,10529
 langfun/core/llms/cache/__init__.py,sha256=QAo3InUMDM_YpteNnVCSejI4zOsnjSMWKJKzkb3VY64,993
 langfun/core/llms/cache/base.py,sha256=rt3zwmyw0y9jsSGW-ZbV1vAfLxQ7_3AVk0l2EySlse4,3918
@@ -119,8 +143,8 @@ langfun/core/templates/demonstration.py,sha256=vCrgYubdZM5Umqcgp8NUVGXgr4P_c-fik
 langfun/core/templates/demonstration_test.py,sha256=SafcDQ0WgI7pw05EmPI2S4v1t3ABKzup8jReCljHeK4,2162
 langfun/core/templates/selfplay.py,sha256=yhgrJbiYwq47TgzThmHrDQTF4nDrTI09CWGhuQPNv-s,2273
 langfun/core/templates/selfplay_test.py,sha256=Ot__1P1M8oJfoTp-M9-PQ6HUXqZKyMwvZ5f7yQ3yfyM,2326
-langfun-0.1.2.dev202411090804.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-langfun-0.1.2.dev202411090804.dist-info/METADATA,sha256=Ez5vUJUc5-XDAdDMyR6Wcgj4fiDL1RwJpu1NI3i3lwE,8890
-langfun-0.1.2.dev202411090804.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
-langfun-0.1.2.dev202411090804.dist-info/top_level.txt,sha256=RhlEkHxs1qtzmmtWSwYoLVJAc1YrbPtxQ52uh8Z9VvY,8
-langfun-0.1.2.dev202411090804.dist-info/RECORD,,
+langfun-0.1.2.dev202411140804.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+langfun-0.1.2.dev202411140804.dist-info/METADATA,sha256=LNyNk_qsiVz-CAbbtkN4jzdSwDWGhlR5RkefV6lclFA,8890
+langfun-0.1.2.dev202411140804.dist-info/WHEEL,sha256=R06PA3UVYHThwHvxuRWMqaGcr-PuniXahwjmQRFMEkY,91
+langfun-0.1.2.dev202411140804.dist-info/top_level.txt,sha256=RhlEkHxs1qtzmmtWSwYoLVJAc1YrbPtxQ52uh8Z9VvY,8
+langfun-0.1.2.dev202411140804.dist-info/RECORD,,

{langfun-0.1.2.dev202411090804.dist-info → langfun-0.1.2.dev202411140804.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.3.0)
+Generator: setuptools (75.5.0)
 Root-Is-Purelib: true
 Tag: py3-none-any