langfun 0.0.2.dev20240515__tar.gz → 0.0.2.dev20240517__tar.gz

{langfun-0.0.2.dev20240515 → langfun-0.0.2.dev20240517}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: langfun
-Version: 0.0.2.dev20240515
+Version: 0.0.2.dev20240517
 Summary: Langfun: Language as Functions.
 Home-page: https://github.com/google/langfun
 Author: Langfun Authors

{langfun-0.0.2.dev20240515 → langfun-0.0.2.dev20240517}/langfun/core/language_model.py

@@ -566,12 +566,19 @@ class LanguageModel(component.Component):
 
   def score(
       self,
-      prompt: str | message_lib.Message,
+      prompt: str | message_lib.Message | list[message_lib.Message],
       completions: list[str | message_lib.Message],
       **kwargs,
   ) -> list[LMScoringResult]:
     """Scores the given prompt."""
-    prompt = message_lib.UserMessage.from_value(prompt)
+    if isinstance(prompt, list):
+      if len(prompt) != len(completions):
+        raise ValueError(
+            'prompt and completions must have the same length.'
+        )
+      prompt = [message_lib.UserMessage.from_value(p) for p in prompt]
+    else:
+      prompt = message_lib.UserMessage.from_value(prompt)
     completions = [message_lib.UserMessage.from_value(c) for c in completions]
 
     call_counter = self._call_counter
@@ -587,7 +594,8 @@ class LanguageModel(component.Component):
       return scoring_results
 
   def _score(
-      self, prompt: message_lib.Message, completions: list[message_lib.Message]
+      self, prompt: message_lib.Message | list[message_lib.Message],
+      completions: list[message_lib.Message]
   ) -> list[LMScoringResult]:
     """Subclass to implement."""
     raise NotImplementedError(
@@ -596,7 +604,7 @@ class LanguageModel(component.Component):
 
   def _debug_score(
       self,
-      prompt: message_lib.Message,
+      prompt: message_lib.Message | list[message_lib.Message],
       completions: list[message_lib.Message],
       scoring_results: list[LMScoringResult],
       call_counter: int,
@@ -615,15 +623,19 @@ class LanguageModel(component.Component):
           title=f'\n[{call_counter}] SCORING LM WITH PROMPT:',
           color='green',
       )
-      referred_modalities = prompt.referred_modalities()
-      if referred_modalities:
-        console.write(
-            pg.object_utils.kvlist_str(
-                [(k, repr(v), None) for k, v in referred_modalities.items()]
-            ),
-            title=f'\n[{call_counter}] MODALITY OBJECTS SENT TO LM:',
-            color='green',
-        )
+      if isinstance(prompt, list):
+        referred_modalities_lst = [p.referred_modalities() for p in prompt]
+      else:
+        referred_modalities_lst = [prompt.referred_modalities(),]
+      if referred_modalities_lst:
+        for referred_modalities in referred_modalities_lst:
+          console.write(
+              pg.object_utils.kvlist_str(
+                  [(k, repr(v), None) for k, v in referred_modalities.items()]
+              ),
+              title=f'\n[{call_counter}] MODALITY OBJECTS SENT TO LM:',
+              color='green',
+          )
 
     if debug & LMDebugMode.RESPONSE:
       console.write(

{langfun-0.0.2.dev20240515 → langfun-0.0.2.dev20240517}/langfun/core/language_model_test.py

@@ -68,7 +68,7 @@ class MockScoringModel(MockModel):
 
   def _score(
       self,
-      prompt: message_lib.Message,
+      prompt: message_lib.Message | list[message_lib.Message],
       completions: list[message_lib.Message],
       **kwargs
   ) -> list[lm_lib.LMScoringResult]:
@@ -508,6 +508,17 @@ class LanguageModelTest(unittest.TestCase):
             ],
         )
 
+        self.assertEqual(
+            lm.score(
+                [message_lib.UserMessage('hi {{image}}', image=Image()),
+                 message_lib.UserMessage('hi {{image}}', image=Image())],
+                ['1', '2'], debug=debug_mode),
+            [
+                lm_lib.LMScoringResult(score=-0.0),
+                lm_lib.LMScoringResult(score=-1.0),
+            ],
+        )
+
       debug_info = string_io.getvalue()
       expected_included = [
           debug_prints[f]
@@ -528,6 +539,10 @@ class LanguageModelTest(unittest.TestCase):
       if debug_mode & lm_lib.LMDebugMode.PROMPT:
         self.assertIn('[0] MODALITY OBJECTS SENT TO LM', debug_info)
 
+  def test_score_with_unmatched_prompt_and_completions(self):
+    with self.assertRaises(ValueError):
+      MockScoringModel().score(['hi',], ['1', '2', '3'])
+
   def test_score_with_unsupported_model(self):
     with self.assertRaises(NotImplementedError):
       MockModel().score('hi', ['1', '2'])

{langfun-0.0.2.dev20240515 → langfun-0.0.2.dev20240517}/langfun/core/llms/fake.py

@@ -21,7 +21,8 @@ import langfun.core as lf
 class Fake(lf.LanguageModel):
   """The base class for all fake language models."""
 
-  def _score(self, prompt: lf.Message, completions: list[lf.Message]):
+  def _score(self, prompt: lf.Message| list[lf.Message],
+             completions: list[lf.Message]):
     return [lf.LMScoringResult(score=-i * 1.0) for i in range(len(completions))]
 
   def _sample(self, prompts: list[lf.Message]) -> list[lf.LMSamplingResult]:

{langfun-0.0.2.dev20240515 → langfun-0.0.2.dev20240517}/langfun/core/llms/openai.py

@@ -33,17 +33,18 @@ _DEFAULT_RPM = 3000
 SUPPORTED_MODELS_AND_SETTINGS = {
     # Models from https://platform.openai.com/docs/models
     # RPM is from https://platform.openai.com/docs/guides/rate-limits
-    'gpt-4o': pg.Dict(rpm=10000, tpm=1500000),
-    'gpt-4o-2024-05-13': pg.Dict(rpm=10000, tpm=1500000),
-    # GPT-4-Turbo models.
-    'gpt-4-turbo': pg.Dict(rpm=10000, tpm=1500000),
-    'gpt-4-turbo-2024-04-09': pg.Dict(rpm=10000, tpm=1500000),
-    'gpt-4-turbo-preview': pg.Dict(rpm=10000, tpm=1500000),
-    'gpt-4-0125-preview': pg.Dict(rpm=10000, tpm=1500000),
-    'gpt-4-1106-preview': pg.Dict(rpm=10000, tpm=1500000),
-    'gpt-4-vision-preview': pg.Dict(rpm=10000, tpm=1500000),
+    # GPT-4o models
+    'gpt-4o': pg.Dict(rpm=10000, tpm=5000000),
+    'gpt-4o-2024-05-13': pg.Dict(rpm=10000, tpm=5000000),
+    # GPT-4-Turbo models
+    'gpt-4-turbo': pg.Dict(rpm=10000, tpm=2000000),
+    'gpt-4-turbo-2024-04-09': pg.Dict(rpm=10000, tpm=2000000),
+    'gpt-4-turbo-preview': pg.Dict(rpm=10000, tpm=2000000),
+    'gpt-4-0125-preview': pg.Dict(rpm=10000, tpm=2000000),
+    'gpt-4-1106-preview': pg.Dict(rpm=10000, tpm=2000000),
+    'gpt-4-vision-preview': pg.Dict(rpm=10000, tpm=2000000),
     'gpt-4-1106-vision-preview': pg.Dict(
-        rpm=10000, tpm=1500000
+        rpm=10000, tpm=2000000
     ),
     # GPT-4 models
     'gpt-4': pg.Dict(rpm=10000, tpm=300000),

{langfun-0.0.2.dev20240515 → langfun-0.0.2.dev20240517}/langfun/core/llms/vertexai.py

@@ -99,7 +99,7 @@ class VertexAI(lf.LanguageModel):
 
     credentials = self.credentials
     # Placeholder for Google-internal credentials.
-    from google.cloud.aiplatform import vertexai  # pylint: disable=g-import-not-at-top
+    import vertexai
     vertexai.init(project=project, location=location, credentials=credentials)
     return True
 
@@ -125,7 +125,7 @@ class VertexAI(lf.LanguageModel):
       self, options: lf.LMSamplingOptions
   ) -> Any:  # generative_models.GenerationConfig
     """Creates generation config from langfun sampling options."""
-    from google.cloud.aiplatform.vertexai.preview import generative_models  # pylint: disable=g-import-not-at-top
+    from vertexai import generative_models
     return generative_models.GenerationConfig(
         temperature=options.temperature,
         top_p=options.top_p,
@@ -138,7 +138,7 @@ class VertexAI(lf.LanguageModel):
       self, prompt: lf.Message
   ) -> list[str | Any]:
     """Gets generation input from langfun message."""
-    from google.cloud.aiplatform.vertexai.preview import generative_models  # pylint: disable=g-import-not-at-top
+    from vertexai import generative_models
     chunks = []
     for lf_chunk in prompt.chunk():
       if isinstance(lf_chunk, str):
@@ -239,7 +239,7 @@ class _ModelHub:
     """Gets a generative model by model id."""
     model = self._generative_model_cache.get(model_id, None)
     if model is None:
-      from google.cloud.aiplatform.vertexai.preview import generative_models  # pylint: disable=g-import-not-at-top
+      from vertexai import generative_models
       model = generative_models.GenerativeModel(model_id)
       self._generative_model_cache[model_id] = model
     return model
@@ -250,7 +250,7 @@ class _ModelHub:
     """Gets a text generation model by model id."""
     model = self._text_generation_model_cache.get(model_id, None)
     if model is None:
-      from google.cloud.aiplatform.vertexai import language_models  # pylint: disable=g-import-not-at-top
+      from vertexai import language_models
       model = language_models.TextGenerationModel.from_pretrained(model_id)
       self._text_generation_model_cache[model_id] = model
     return model

{langfun-0.0.2.dev20240515 → langfun-0.0.2.dev20240517}/langfun/core/llms/vertexai_test.py

@@ -17,7 +17,7 @@ import os
 import unittest
 from unittest import mock
 
-from google.cloud.aiplatform.vertexai.preview import generative_models
+from vertexai import generative_models
 import langfun.core as lf
 from langfun.core import modalities as lf_modalities
 from langfun.core.llms import vertexai
@@ -39,7 +39,6 @@ example_image = (
 def mock_generate_content(content, generation_config, **kwargs):
   del kwargs
   c = pg.Dict(generation_config.to_dict())
-  print('zzz', c)
   return generative_models.GenerationResponse.from_dict({
       'candidates': [
           {
@@ -111,7 +110,7 @@ class VertexAITest(unittest.TestCase):
 
   def test_model_hub(self):
     with mock.patch(
-        'google.cloud.aiplatform.vertexai.preview.generative_models.'
+        'vertexai.generative_models.'
         'GenerativeModel.__init__'
     ) as mock_model_init:
       mock_model_init.side_effect = lambda *args, **kwargs: None
@@ -125,7 +124,7 @@ class VertexAITest(unittest.TestCase):
       )
 
     with mock.patch(
-        'google.cloud.aiplatform.vertexai.language_models.'
+        'vertexai.language_models.'
         'TextGenerationModel.from_pretrained'
     ) as mock_model_init:
 
@@ -163,13 +162,13 @@ class VertexAITest(unittest.TestCase):
 
   def test_call_generative_model(self):
     with mock.patch(
-        'google.cloud.aiplatform.vertexai.preview.generative_models.'
+        'vertexai.generative_models.'
         'GenerativeModel.__init__'
     ) as mock_model_init:
       mock_model_init.side_effect = lambda *args, **kwargs: None
 
       with mock.patch(
-          'google.cloud.aiplatform.vertexai.preview.generative_models.'
+          'vertexai.generative_models.'
           'GenerativeModel.generate_content'
       ) as mock_generate:
         mock_generate.side_effect = mock_generate_content
@@ -192,7 +191,7 @@ class VertexAITest(unittest.TestCase):
 
   def test_call_text_generation_model(self):
     with mock.patch(
-        'google.cloud.aiplatform.vertexai.language_models.'
+        'vertexai.language_models.'
         'TextGenerationModel.from_pretrained'
     ) as mock_model_init:
 

{langfun-0.0.2.dev20240515 → langfun-0.0.2.dev20240517}/langfun/core/structured/scoring.py

@@ -23,7 +23,7 @@ import pyglove as pg
 
 
 def score(
-    prompt: Union[str, pg.Symbolic],
+    prompt: Union[str, pg.Symbolic] | list[str | pg.Symbolic],
     completions: list[str | pg.Symbolic],
     schema: Union[
         schema_lib.Schema, Type[Any], list[Type[Any]], dict[str, Any], None
@@ -49,15 +49,27 @@ def score(
             f'{[type(c) for c in completions]}.'
         )
 
-  input_message = prompting.query(
-      prompt,
-      schema,
-      examples=examples,
-      protocol=protocol,
-      skip_lm=True,
-      returns_message=True,
-      **kwargs,
-  )
+  if isinstance(prompt, list):
+    prompts = []
+    for p in prompt:
+      prompts.append(
+          prompting.query_prompt(
+              p,
+              schema,
+              examples=examples,
+              protocol=protocol,
+              **kwargs,
+          )
+      )
+    input_message = prompts
+  else:
+    input_message = prompting.query_prompt(
+        prompt,
+        schema,
+        examples=examples,
+        protocol=protocol,
+        **kwargs,
+    )
   if lm is None:
     lm_override = lf.get_contextual_override('lm')
     if lm_override is None:

{langfun-0.0.2.dev20240515 → langfun-0.0.2.dev20240517}/langfun.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: langfun
-Version: 0.0.2.dev20240515
+Version: 0.0.2.dev20240517
 Summary: Langfun: Language as Functions.
 Home-page: https://github.com/google/langfun
 Author: Langfun Authors