langfun 0.0.2.dev20240515__py3-none-any.whl → 0.0.2.dev20240517__py3-none-any.whl

langfun/core/language_model.py

@@ -566,12 +566,19 @@ class LanguageModel(component.Component):
 
   def score(
       self,
-      prompt: str | message_lib.Message,
+      prompt: str | message_lib.Message | list[message_lib.Message],
       completions: list[str | message_lib.Message],
       **kwargs,
   ) -> list[LMScoringResult]:
     """Scores the given prompt."""
-    prompt = message_lib.UserMessage.from_value(prompt)
+    if isinstance(prompt, list):
+      if len(prompt) != len(completions):
+        raise ValueError(
+            'prompt and completions must have the same length.'
+        )
+      prompt = [message_lib.UserMessage.from_value(p) for p in prompt]
+    else:
+      prompt = message_lib.UserMessage.from_value(prompt)
     completions = [message_lib.UserMessage.from_value(c) for c in completions]
 
     call_counter = self._call_counter
@@ -587,7 +594,8 @@ class LanguageModel(component.Component):
       return scoring_results
 
   def _score(
-      self, prompt: message_lib.Message, completions: list[message_lib.Message]
+      self, prompt: message_lib.Message | list[message_lib.Message],
+      completions: list[message_lib.Message]
   ) -> list[LMScoringResult]:
     """Subclass to implement."""
     raise NotImplementedError(
@@ -596,7 +604,7 @@ class LanguageModel(component.Component):
 
   def _debug_score(
       self,
-      prompt: message_lib.Message,
+      prompt: message_lib.Message | list[message_lib.Message],
       completions: list[message_lib.Message],
       scoring_results: list[LMScoringResult],
       call_counter: int,
@@ -615,15 +623,19 @@ class LanguageModel(component.Component):
           title=f'\n[{call_counter}] SCORING LM WITH PROMPT:',
           color='green',
       )
-      referred_modalities = prompt.referred_modalities()
-      if referred_modalities:
-        console.write(
-            pg.object_utils.kvlist_str(
-                [(k, repr(v), None) for k, v in referred_modalities.items()]
-            ),
-            title=f'\n[{call_counter}] MODALITY OBJECTS SENT TO LM:',
-            color='green',
-        )
+      if isinstance(prompt, list):
+        referred_modalities_lst = [p.referred_modalities() for p in prompt]
+      else:
+        referred_modalities_lst = [prompt.referred_modalities(),]
+      if referred_modalities_lst:
+        for referred_modalities in referred_modalities_lst:
+          console.write(
+              pg.object_utils.kvlist_str(
+                  [(k, repr(v), None) for k, v in referred_modalities.items()]
+              ),
+              title=f'\n[{call_counter}] MODALITY OBJECTS SENT TO LM:',
+              color='green',
+          )
 
     if debug & LMDebugMode.RESPONSE:
       console.write(

langfun/core/language_model_test.py

@@ -68,7 +68,7 @@ class MockScoringModel(MockModel):
 
   def _score(
       self,
-      prompt: message_lib.Message,
+      prompt: message_lib.Message | list[message_lib.Message],
       completions: list[message_lib.Message],
       **kwargs
   ) -> list[lm_lib.LMScoringResult]:
@@ -508,6 +508,17 @@ class LanguageModelTest(unittest.TestCase):
             ],
         )
 
+        self.assertEqual(
+            lm.score(
+                [message_lib.UserMessage('hi {{image}}', image=Image()),
+                 message_lib.UserMessage('hi {{image}}', image=Image())],
+                ['1', '2'], debug=debug_mode),
+            [
+                lm_lib.LMScoringResult(score=-0.0),
+                lm_lib.LMScoringResult(score=-1.0),
+            ],
+        )
+
       debug_info = string_io.getvalue()
       expected_included = [
           debug_prints[f]
@@ -528,6 +539,10 @@ class LanguageModelTest(unittest.TestCase):
       if debug_mode & lm_lib.LMDebugMode.PROMPT:
         self.assertIn('[0] MODALITY OBJECTS SENT TO LM', debug_info)
 
+  def test_score_with_unmatched_prompt_and_completions(self):
+    with self.assertRaises(ValueError):
+      MockScoringModel().score(['hi',], ['1', '2', '3'])
+
   def test_score_with_unsupported_model(self):
     with self.assertRaises(NotImplementedError):
       MockModel().score('hi', ['1', '2'])

langfun/core/llms/fake.py

@@ -21,7 +21,8 @@ import langfun.core as lf
 class Fake(lf.LanguageModel):
   """The base class for all fake language models."""
 
-  def _score(self, prompt: lf.Message, completions: list[lf.Message]):
+  def _score(self, prompt: lf.Message| list[lf.Message],
+             completions: list[lf.Message]):
     return [lf.LMScoringResult(score=-i * 1.0) for i in range(len(completions))]
 
   def _sample(self, prompts: list[lf.Message]) -> list[lf.LMSamplingResult]:

langfun/core/llms/openai.py

@@ -33,17 +33,18 @@ _DEFAULT_RPM = 3000
 SUPPORTED_MODELS_AND_SETTINGS = {
     # Models from https://platform.openai.com/docs/models
     # RPM is from https://platform.openai.com/docs/guides/rate-limits
-    'gpt-4o': pg.Dict(rpm=10000, tpm=1500000),
-    'gpt-4o-2024-05-13': pg.Dict(rpm=10000, tpm=1500000),
-    # GPT-4-Turbo models.
-    'gpt-4-turbo': pg.Dict(rpm=10000, tpm=1500000),
-    'gpt-4-turbo-2024-04-09': pg.Dict(rpm=10000, tpm=1500000),
-    'gpt-4-turbo-preview': pg.Dict(rpm=10000, tpm=1500000),
-    'gpt-4-0125-preview': pg.Dict(rpm=10000, tpm=1500000),
-    'gpt-4-1106-preview': pg.Dict(rpm=10000, tpm=1500000),
-    'gpt-4-vision-preview': pg.Dict(rpm=10000, tpm=1500000),
+    # GPT-4o models
+    'gpt-4o': pg.Dict(rpm=10000, tpm=5000000),
+    'gpt-4o-2024-05-13': pg.Dict(rpm=10000, tpm=5000000),
+    # GPT-4-Turbo models
+    'gpt-4-turbo': pg.Dict(rpm=10000, tpm=2000000),
+    'gpt-4-turbo-2024-04-09': pg.Dict(rpm=10000, tpm=2000000),
+    'gpt-4-turbo-preview': pg.Dict(rpm=10000, tpm=2000000),
+    'gpt-4-0125-preview': pg.Dict(rpm=10000, tpm=2000000),
+    'gpt-4-1106-preview': pg.Dict(rpm=10000, tpm=2000000),
+    'gpt-4-vision-preview': pg.Dict(rpm=10000, tpm=2000000),
     'gpt-4-1106-vision-preview': pg.Dict(
-        rpm=10000, tpm=1500000
+        rpm=10000, tpm=2000000
     ),
     # GPT-4 models
     'gpt-4': pg.Dict(rpm=10000, tpm=300000),

langfun/core/llms/vertexai.py

@@ -99,7 +99,7 @@ class VertexAI(lf.LanguageModel):
 
     credentials = self.credentials
     # Placeholder for Google-internal credentials.
-    from google.cloud.aiplatform import vertexai  # pylint: disable=g-import-not-at-top
+    import vertexai
     vertexai.init(project=project, location=location, credentials=credentials)
     return True
 
@@ -125,7 +125,7 @@ class VertexAI(lf.LanguageModel):
       self, options: lf.LMSamplingOptions
   ) -> Any:  # generative_models.GenerationConfig
     """Creates generation config from langfun sampling options."""
-    from google.cloud.aiplatform.vertexai.preview import generative_models  # pylint: disable=g-import-not-at-top
+    from vertexai import generative_models
     return generative_models.GenerationConfig(
         temperature=options.temperature,
         top_p=options.top_p,
@@ -138,7 +138,7 @@ class VertexAI(lf.LanguageModel):
       self, prompt: lf.Message
   ) -> list[str | Any]:
     """Gets generation input from langfun message."""
-    from google.cloud.aiplatform.vertexai.preview import generative_models  # pylint: disable=g-import-not-at-top
+    from vertexai import generative_models
     chunks = []
     for lf_chunk in prompt.chunk():
       if isinstance(lf_chunk, str):
@@ -239,7 +239,7 @@ class _ModelHub:
     """Gets a generative model by model id."""
     model = self._generative_model_cache.get(model_id, None)
     if model is None:
-      from google.cloud.aiplatform.vertexai.preview import generative_models  # pylint: disable=g-import-not-at-top
+      from vertexai import generative_models
       model = generative_models.GenerativeModel(model_id)
       self._generative_model_cache[model_id] = model
     return model
@@ -250,7 +250,7 @@ class _ModelHub:
     """Gets a text generation model by model id."""
     model = self._text_generation_model_cache.get(model_id, None)
     if model is None:
-      from google.cloud.aiplatform.vertexai import language_models  # pylint: disable=g-import-not-at-top
+      from vertexai import language_models
       model = language_models.TextGenerationModel.from_pretrained(model_id)
       self._text_generation_model_cache[model_id] = model
     return model

langfun/core/llms/vertexai_test.py

@@ -17,7 +17,7 @@ import os
 import unittest
 from unittest import mock
 
-from google.cloud.aiplatform.vertexai.preview import generative_models
+from vertexai import generative_models
 import langfun.core as lf
 from langfun.core import modalities as lf_modalities
 from langfun.core.llms import vertexai
@@ -39,7 +39,6 @@ example_image = (
 def mock_generate_content(content, generation_config, **kwargs):
   del kwargs
   c = pg.Dict(generation_config.to_dict())
-  print('zzz', c)
   return generative_models.GenerationResponse.from_dict({
       'candidates': [
           {
@@ -111,7 +110,7 @@ class VertexAITest(unittest.TestCase):
 
   def test_model_hub(self):
     with mock.patch(
-        'google.cloud.aiplatform.vertexai.preview.generative_models.'
+        'vertexai.generative_models.'
         'GenerativeModel.__init__'
     ) as mock_model_init:
       mock_model_init.side_effect = lambda *args, **kwargs: None
@@ -125,7 +124,7 @@ class VertexAITest(unittest.TestCase):
       )
 
     with mock.patch(
-        'google.cloud.aiplatform.vertexai.language_models.'
+        'vertexai.language_models.'
         'TextGenerationModel.from_pretrained'
     ) as mock_model_init:
 
@@ -163,13 +162,13 @@ class VertexAITest(unittest.TestCase):
 
   def test_call_generative_model(self):
     with mock.patch(
-        'google.cloud.aiplatform.vertexai.preview.generative_models.'
+        'vertexai.generative_models.'
         'GenerativeModel.__init__'
     ) as mock_model_init:
       mock_model_init.side_effect = lambda *args, **kwargs: None
 
       with mock.patch(
-          'google.cloud.aiplatform.vertexai.preview.generative_models.'
+          'vertexai.generative_models.'
           'GenerativeModel.generate_content'
       ) as mock_generate:
         mock_generate.side_effect = mock_generate_content
@@ -192,7 +191,7 @@ class VertexAITest(unittest.TestCase):
 
   def test_call_text_generation_model(self):
     with mock.patch(
-        'google.cloud.aiplatform.vertexai.language_models.'
+        'vertexai.language_models.'
         'TextGenerationModel.from_pretrained'
     ) as mock_model_init:
 

langfun/core/structured/scoring.py

@@ -23,7 +23,7 @@ import pyglove as pg
 
 
 def score(
-    prompt: Union[str, pg.Symbolic],
+    prompt: Union[str, pg.Symbolic] | list[str | pg.Symbolic],
     completions: list[str | pg.Symbolic],
     schema: Union[
         schema_lib.Schema, Type[Any], list[Type[Any]], dict[str, Any], None
@@ -49,15 +49,27 @@ def score(
             f'{[type(c) for c in completions]}.'
         )
 
-  input_message = prompting.query(
-      prompt,
-      schema,
-      examples=examples,
-      protocol=protocol,
-      skip_lm=True,
-      returns_message=True,
-      **kwargs,
-  )
+  if isinstance(prompt, list):
+    prompts = []
+    for p in prompt:
+      prompts.append(
+          prompting.query_prompt(
+              p,
+              schema,
+              examples=examples,
+              protocol=protocol,
+              **kwargs,
+          )
+      )
+    input_message = prompts
+  else:
+    input_message = prompting.query_prompt(
+        prompt,
+        schema,
+        examples=examples,
+        protocol=protocol,
+        **kwargs,
+    )
   if lm is None:
     lm_override = lf.get_contextual_override('lm')
     if lm_override is None:

{langfun-0.0.2.dev20240515.dist-info → langfun-0.0.2.dev20240517.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: langfun
-Version: 0.0.2.dev20240515
+Version: 0.0.2.dev20240517
 Summary: Langfun: Language as Functions.
 Home-page: https://github.com/google/langfun
 Author: Langfun Authors

{langfun-0.0.2.dev20240515.dist-info → langfun-0.0.2.dev20240517.dist-info}/RECORD

@@ -8,8 +8,8 @@ langfun/core/console.py,sha256=bk5rNPNm9rMGW5YT2HixxU04p2umnoabn5SDz6Dqe88,2317
 langfun/core/console_test.py,sha256=5SYJdxpJGLgdSSQqqMPoA1X6jpsLD8rgcyk-EgI65oE,1077
 langfun/core/langfunc.py,sha256=RvIcRjIq0jWYRu1xim-FYe4HSrt97r3GMBO_PuagUmw,11060
 langfun/core/langfunc_test.py,sha256=_mfARnakX3oji5HDigFSLMd6yQ2wma-2Mgbztwqn73g,8501
-langfun/core/language_model.py,sha256=6wtY8RGbOymfo1PYzcYCfOlWuKQcSVFs5R1sFB4-QMQ,20202
-langfun/core/language_model_test.py,sha256=T-itu7Li2smv2dkru0C0neCs2W4VJXlNTYahXU6jF54,19548
+langfun/core/language_model.py,sha256=owNCgefGoPeRCHrxBhMtNdOj3orbeVml4eqLf1n211o,20760
+langfun/core/language_model_test.py,sha256=36evArVJgSQ9lRgHfMmlLW3lwjjDoiAgfTEbk2FIKa4,20122
 langfun/core/memory.py,sha256=f-asN1F7Vehgdn_fK84v73GrEUOxRtaW934keutTKjk,2416
 langfun/core/message.py,sha256=QhvV9t5qaryPcruyxxcXi3gm9QDInkSldwTtK6sVJ3c,15734
 langfun/core/message_test.py,sha256=Z23pUM5vPnDrYkIIibe2KL73D5HKur_awI0ut_EQFQA,9501
@@ -51,7 +51,7 @@ langfun/core/eval/scoring_test.py,sha256=O8olHbrUEg60gMxwOkWzKBJZpZoUlmVnBANX5Se
 langfun/core/llms/__init__.py,sha256=h_kam-0fjWISAQ90KZ_ydBhwADVCzrhLPXmAki3GfU0,4175
 langfun/core/llms/anthropic.py,sha256=7W9YdPN3SlAFhAIQlihMkrpo7tTY_4NvD0KIlCrqcsk,8505
 langfun/core/llms/anthropic_test.py,sha256=TMM30myyEhwF99Le4RvJEXOn8RYl0q1FRkt9Q9nl1jk,5540
-langfun/core/llms/fake.py,sha256=_smsN_CsYbeWrtjpegEPwdAPV9mwaIuH_4oZGeXQwQI,2896
+langfun/core/llms/fake.py,sha256=Dd7-6ka9pFf3fcWZyczamjOqQ91MOI-m7We3Oc9Ffmo,2927
 langfun/core/llms/fake_test.py,sha256=ipKfdOcuqVcJ8lDXVpnBVb9HHG0hAVkFkMoHpWjC2cI,7212
 langfun/core/llms/google_genai.py,sha256=nDI_Adur_K458l6EWoiiAhzjfnjRSqfTiikdu7iLPyU,8808
 langfun/core/llms/google_genai_test.py,sha256=_UcGTfl16-aDUlEWFC2W2F8y9jPUs53RBYA6MOCpGXw,7525
@@ -59,10 +59,10 @@ langfun/core/llms/groq.py,sha256=NaGItVL_pkOpqPpI4bPGU27xLFRoaeizZ49v2s-4ERs,784
 langfun/core/llms/groq_test.py,sha256=M6GtlrsOvDun_j-sR8cPh4W_moHWZNSTiThu3kuwbbc,5281
 langfun/core/llms/llama_cpp.py,sha256=Y_KkMUf3Xfac49koMUtUslKl3h-HWp3-ntq7Jaa3bdo,2385
 langfun/core/llms/llama_cpp_test.py,sha256=ZxC6defGd_HX9SFRU9U4cJiQnBKundbOrchbXuC1Z2M,1683
-langfun/core/llms/openai.py,sha256=u5hlpQlmYvdcuU_TiqH0ektQ0lHD09M3xByE7dIhqrs,13638
+langfun/core/llms/openai.py,sha256=IN46gIqfY6aEEfxCPNmyH1hrep3oWBhJDwVFilfqNkM,13657
 langfun/core/llms/openai_test.py,sha256=asSA1sVy_7hnXioD_2HTxtSDpVTKBUO_EjZuyHpwbn0,14854
-langfun/core/llms/vertexai.py,sha256=O2Lp-F4KJzvQSCjPV--sa6nMS9-GsLj2eiqA-1qGhWQ,9661
-langfun/core/llms/vertexai_test.py,sha256=LBk4luL_N13ZejZebBzQ3tkfjxFhk7uBS4JjEpojJAo,7836
+langfun/core/llms/vertexai.py,sha256=DBH7WzrTL5oIyKWoZTuIR8B_gtgRi7hlS9GteSY0r4E,9317
+langfun/core/llms/vertexai_test.py,sha256=1u813e2esr8HVbOpLTOhIgzaX-GqCRTBouJ1doRn80Q,7642
 langfun/core/llms/cache/__init__.py,sha256=QAo3InUMDM_YpteNnVCSejI4zOsnjSMWKJKzkb3VY64,993
 langfun/core/llms/cache/base.py,sha256=cFfYvOIUae842pncqCAsRvqXCk2AnAsRYVx0mcIoAeY,3338
 langfun/core/llms/cache/in_memory.py,sha256=YfFyJEhLs73cUiB0ZfhMxYpdE8Iuxxw-dvMFwGHTSHw,4742
@@ -94,7 +94,7 @@ langfun/core/structured/schema.py,sha256=Zy9y6Vq9DrFwcuP5o5VL_PvMCmzavF-nuDqyviB
 langfun/core/structured/schema_generation.py,sha256=U3nRQsqmMZg_qIVDh2fiY3K4JLfsAL1LcKzIFP1iXFg,5316
 langfun/core/structured/schema_generation_test.py,sha256=RM9s71kMNg2jTePwInkiW9fK1ACN37eyPeF8OII-0zw,2950
 langfun/core/structured/schema_test.py,sha256=NgQK1zGSliZVx_Af6gDBTqQxXRHvmAvGARv4dUs8IbI,23078
-langfun/core/structured/scoring.py,sha256=5DsMNrWKf98ZYCEkxA4-HvA62nMSNBs9DC5m8dYL7Cs,2442
+langfun/core/structured/scoring.py,sha256=QyT1S8FkLtKICfUbh4AXoKK3YJ_rgejyk6TI2OtOa68,2751
 langfun/core/structured/scoring_test.py,sha256=39_dw6p_FkoqeUccO67yIqos-MccAWezoozS21i8mi0,1732
 langfun/core/templates/__init__.py,sha256=bO0eMsVJbi7sxEB2YlInKRQ2EVP-RyyKUwcD-8msuN4,927
 langfun/core/templates/completion.py,sha256=mUqZHOEV3ag6-A08XghpeEltcrBvCDxXP004eDDfeag,1931
@@ -105,8 +105,8 @@ langfun/core/templates/demonstration.py,sha256=vCrgYubdZM5Umqcgp8NUVGXgr4P_c-fik
 langfun/core/templates/demonstration_test.py,sha256=SafcDQ0WgI7pw05EmPI2S4v1t3ABKzup8jReCljHeK4,2162
 langfun/core/templates/selfplay.py,sha256=yhgrJbiYwq47TgzThmHrDQTF4nDrTI09CWGhuQPNv-s,2273
 langfun/core/templates/selfplay_test.py,sha256=DYVrkk7uNKCqJGEHH31HssU2BPuMItU1vJLzfcXIlYg,2156
-langfun-0.0.2.dev20240515.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-langfun-0.0.2.dev20240515.dist-info/METADATA,sha256=wx28EHVqDU4Xz2qt4i8JYirho8VkBqCb5URcoKVfmhA,3452
-langfun-0.0.2.dev20240515.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-langfun-0.0.2.dev20240515.dist-info/top_level.txt,sha256=RhlEkHxs1qtzmmtWSwYoLVJAc1YrbPtxQ52uh8Z9VvY,8
-langfun-0.0.2.dev20240515.dist-info/RECORD,,
+langfun-0.0.2.dev20240517.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+langfun-0.0.2.dev20240517.dist-info/METADATA,sha256=yNu343UZMun1YltzX9amHxvKX1qwpJi_QZSgtCaa6oU,3452
+langfun-0.0.2.dev20240517.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+langfun-0.0.2.dev20240517.dist-info/top_level.txt,sha256=RhlEkHxs1qtzmmtWSwYoLVJAc1YrbPtxQ52uh8Z9VvY,8
+langfun-0.0.2.dev20240517.dist-info/RECORD,,