langfun 0.1.2.dev202510240805__py3-none-any.whl → 0.1.2.dev202510250803__py3-none-any.whl

langfun/core/concurrent_test.py

@@ -608,6 +608,7 @@ class ConcurrentMapTest(unittest.TestCase):
         ],
     )
     concurrent.ProgressBar.uninstall(bar_id)
+    concurrent.ProgressBar.refresh()
     self.assertIn('100%', string_io.getvalue())
 
 

langfun/core/data/conversion/anthropic_test.py

@@ -253,14 +253,16 @@ class AnthropicConversionTest(unittest.TestCase):
     )
     self.assertEqual(
         m.text,
-        'What are the common words from <<[[obj0]]>> and <<[[obj1]]>> ?'
+        'What are the common words from <<[[image:dc6e1e43]]>> and'
+        ' <<[[pdf:5daf5f31]]>> ?'
     )
-    self.assertIsInstance(m.obj0, lf_modalities.Image)
-    self.assertEqual(m.obj0.mime_type, 'image/png')
-    self.assertEqual(m.obj0.to_bytes(), image_content)
+    modalities = m.modalities()
+    self.assertIsInstance(modalities[0], lf_modalities.Image)
+    self.assertEqual(modalities[0].mime_type, 'image/png')
+    self.assertEqual(modalities[0].content, image_content)
 
-    self.assertIsInstance(m.obj1, lf_modalities.PDF)
-    self.assertEqual(m.obj1.to_bytes(), pdf_content)
+    self.assertIsInstance(modalities[1], lf_modalities.PDF)
+    self.assertEqual(modalities[1].content, pdf_content)
 
 
 if __name__ == '__main__':

langfun/core/data/conversion/gemini_test.py

@@ -225,19 +225,22 @@ class GeminiConversionTest(unittest.TestCase):
     self.assertEqual(
         m.text,
         (
-            'What are the common words from <<[[obj0]]>> , <<[[obj1]]>> '
-            'and <<[[obj2]]>> ?'
+            'What are the common words from <<[[image:dc6e1e43]]>> , '
+            '<<[[pdf:4dc12e93]]>> and <<[[video:7e169565]]>> ?'
         )
     )
-    self.assertIsInstance(m.obj0, lf_modalities.Image)
-    self.assertEqual(m.obj0.mime_type, 'image/png')
-    self.assertEqual(m.obj0.to_bytes(), image_content)
+    self.assertIsInstance(m.modalities()[0], lf_modalities.Image)
+    self.assertEqual(m.modalities()[0].mime_type, 'image/png')
+    self.assertEqual(m.modalities()[0].to_bytes(), image_content)
 
-    self.assertIsInstance(m.obj1, lf_modalities.PDF)
-    self.assertEqual(m.obj1.uri, 'https://my.pdf')
+    self.assertIsInstance(m.modalities()[1], lf_modalities.PDF)
+    self.assertEqual(m.modalities()[1].uri, 'https://my.pdf')
 
-    self.assertIsInstance(m.obj2, lf_modalities.Video)
-    self.assertEqual(m.obj2.uri, 'https://www.youtube.com/watch?v=abcd')
+    self.assertIsInstance(m.modalities()[2], lf_modalities.Video)
+    self.assertEqual(
+        m.modalities()[2].uri,
+        'https://www.youtube.com/watch?v=abcd'
+    )
 
 
 if __name__ == '__main__':

langfun/core/data/conversion/openai.py

@@ -19,10 +19,13 @@ import langfun.core as lf
 from langfun.core import modalities as lf_modalities
 
 
-class OpenAIMessageConverter(lf.MessageConverter):
-  """Converter to OpenAI API."""
+class OpenAIChatCompletionAPIMessageConverter(lf.MessageConverter):
+  """Converter to OpenAI ChatCompletion API.
 
-  FORMAT_ID = 'openai'
+  See https://platform.openai.com/docs/api-reference/chat
+  """
+
+  FORMAT_ID = 'openai_chat_completion_api'
 
   chunk_preprocessor: Annotated[
       Callable[[str | lf.Modality], Any] | None,
@@ -41,22 +44,29 @@ class OpenAIMessageConverter(lf.MessageConverter):
         chunk = self.chunk_preprocessor(chunk)
         if chunk is None:
           continue
-
-      if isinstance(chunk, str):
-        item = dict(type='text', text=chunk)
-      elif isinstance(chunk, lf_modalities.Image):
-        item = dict(
-            type='image_url', image_url=dict(url=chunk.embeddable_uri)
-        )
-      # TODO(daiyip): Support audio_input.
-      else:
-        raise ValueError(f'Unsupported content type: {chunk!r}.')
-      parts.append(item)
+      parts.append(self.chunk_to_json(type(message), chunk))
     return dict(
         role=self.get_role(message),
         content=parts,
     )
 
+  def chunk_to_json(
+      self,
+      message_cls: type[lf.Message],
+      chunk: str | lf.Modality
+  ) -> dict[str, Any]:
+    """Converts a Langfun chunk to OpenAI chunk."""
+    del message_cls
+    if isinstance(chunk, str):
+      return dict(type='text', text=chunk)
+    elif isinstance(chunk, lf_modalities.Image):
+      return dict(
+          type='image_url', image_url=dict(url=chunk.embeddable_uri)
+      )
+    # TODO(daiyip): Support audio_input.
+    else:
+      raise ValueError(f'Unsupported content type: {chunk!r}.')
+
   def get_role(self, message: lf.Message) -> str:
     """Returns the role of the message."""
     if isinstance(message, lf.SystemMessage):
@@ -92,40 +102,134 @@ class OpenAIMessageConverter(lf.MessageConverter):
     assert isinstance(content, list)
     chunks = []
     for item in content:
-      t = self._safe_read(item, 'type')
-      if t == 'text':
-        chunk = self._safe_read(item, 'text')
-      elif t == 'image_url':
-        chunk = lf_modalities.Image.from_uri(
-            self._safe_read(self._safe_read(item, 'image_url'), 'url')
-        )
-      else:
-        raise ValueError(f'Unsupported content type: {item!r}.')
-      chunks.append(chunk)
+      chunks.append(self.json_to_chunk(item))
     return message_cls.from_chunks(chunks)
 
+  def json_to_chunk(self, json: dict[str, Any]) -> str | lf.Modality:
+    """Returns a Langfun chunk from OpenAI chunk JSON."""
+    t = self._safe_read(json, 'type')
+    if t == 'text':
+      return self._safe_read(json, 'text')
+    elif t == 'image_url':
+      return lf_modalities.Image.from_uri(
+          self._safe_read(self._safe_read(json, 'image_url'), 'url')
+      )
+    else:
+      raise ValueError(f'Unsupported content type: {json!r}.')
+
 
-def _as_openai_format(
+def _as_openai_chat_completion_api_format(
     self,
     chunk_preprocessor: Callable[[str | lf.Modality], Any] | None = None,
     **kwargs
 ) -> dict[str, Any]:
   """Returns an OpenAI format message."""
-  return OpenAIMessageConverter(
+  return OpenAIChatCompletionAPIMessageConverter(
       chunk_preprocessor=chunk_preprocessor, **kwargs
   ).to_value(self)
 
 
 @classmethod
-def _from_openai_format(
+def _from_openai_chat_completion_api_format(
     cls,
     openai_message: dict[str, Any],
     **kwargs
 ) -> lf.Message:
   """Creates a Langfun message from the OpenAI format message."""
   del cls
-  return OpenAIMessageConverter(**kwargs).from_value(openai_message)
+  return OpenAIChatCompletionAPIMessageConverter(
+      **kwargs
+  ).from_value(openai_message)
 
 # Set shortcut methods in lf.Message.
-lf.Message.as_openai_format = _as_openai_format
-lf.Message.from_openai_format = _from_openai_format
+lf.Message.as_openai_chat_completion_api_format = (
+    _as_openai_chat_completion_api_format
+)
+
+lf.Message.from_openai_chat_completion_api_format = (
+    _from_openai_chat_completion_api_format
+)
+
+
+#
+# OpenAI Responses API message converter.
+#
+
+
+class OpenAIResponsesAPIMessageConverter(
+    OpenAIChatCompletionAPIMessageConverter
+):
+  """Converter to OpenAI Responses API.
+
+  See https://platform.openai.com/docs/api-reference/responses/create
+  """
+
+  FORMAT_ID = 'openai_responses_api'
+
+  def to_value(self, message: lf.Message) -> dict[str, Any]:
+    """Converts a Langfun message to OpenAI API."""
+    message_json = super().to_value(message)
+    message_json['type'] = 'message'
+    return message_json
+
+  def chunk_to_json(
+      self,
+      message_cls: type[lf.Message],
+      chunk: str | lf.Modality
+  ) -> dict[str, Any]:
+    """Converts a Langfun chunk to OpenAI chunk."""
+    source = 'output' if issubclass(message_cls, lf.AIMessage) else 'input'
+
+    if isinstance(chunk, str):
+      return dict(type=f'{source}_text', text=chunk)
+    elif isinstance(chunk, lf_modalities.Image):
+      return dict(
+          type=f'{source}_image', image_url=chunk.embeddable_uri
+      )
+    # TODO(daiyip): Support audio_input.
+    else:
+      raise ValueError(f'Unsupported content type: {chunk!r}.')
+
+  def json_to_chunk(self, json: dict[str, Any]) -> str | lf.Modality:
+    """Returns a Langfun chunk from OpenAI chunk JSON."""
+    t = self._safe_read(json, 'type')
+    if t in ('input_text', 'output_text'):
+      return self._safe_read(json, 'text')
+    elif t in ('input_image', 'output_image'):
+      return lf_modalities.Image.from_uri(self._safe_read(json, 'image_url'))
+    else:
+      raise ValueError(f'Unsupported content type: {json!r}.')
+
+
+def _as_openai_responses_api_format(
+    self,
+    chunk_preprocessor: Callable[[str | lf.Modality], Any] | None = None,
+    **kwargs
+) -> dict[str, Any]:
+  """Returns an OpenAI format message."""
+  return OpenAIResponsesAPIMessageConverter(
+      chunk_preprocessor=chunk_preprocessor, **kwargs
+  ).to_value(self)
+
+
+@classmethod
+def _from_openai_responses_api_format(
+    cls,
+    openai_message: dict[str, Any],
+    **kwargs
+) -> lf.Message:
+  """Creates a Langfun message from the OpenAI format message."""
+  del cls
+  return OpenAIResponsesAPIMessageConverter(
+      **kwargs
+  ).from_value(openai_message)
+
+
+# Set shortcut methods in lf.Message.
+lf.Message.as_openai_responses_api_format = (
+    _as_openai_responses_api_format
+)
+
+lf.Message.from_openai_responses_api_format = (
+    _from_openai_responses_api_format
+)

langfun/core/data/conversion/openai_test.py

@@ -30,25 +30,25 @@ image_content = (
 )
 
 
-class OpenAIConversionTest(unittest.TestCase):
+class OpenAIChatCompletionAPIConverterTest(unittest.TestCase):
 
   def test_as_format_with_role(self):
     self.assertEqual(
-        lf.UserMessage('hi').as_format('openai'),
+        lf.UserMessage('hi').as_format('openai_chat_completion_api'),
         {
             'role': 'user',
             'content': [{'type': 'text', 'text': 'hi'}],
         },
     )
     self.assertEqual(
-        lf.AIMessage('hi').as_format('openai'),
+        lf.AIMessage('hi').as_format('openai_chat_completion_api'),
         {
             'role': 'assistant',
             'content': [{'type': 'text', 'text': 'hi'}],
         },
     )
     self.assertEqual(
-        lf.SystemMessage('hi').as_format('openai'),
+        lf.SystemMessage('hi').as_format('openai_chat_completion_api'),
         {
             'role': 'system',
             'content': [{'type': 'text', 'text': 'hi'}],
@@ -60,7 +60,7 @@ class OpenAIConversionTest(unittest.TestCase):
         lf.Template(
             'What is this {{image}}?',
             image=lf_modalities.Image.from_bytes(image_content)
-        ).render().as_format('openai'),
+        ).render().as_format('openai_chat_completion_api'),
         {
             'role': 'user',
             'content': [
@@ -90,7 +90,7 @@ class OpenAIConversionTest(unittest.TestCase):
         lf.Template(
             'What is this {{image}}?',
             image=lf_modalities.Image.from_bytes(image_content)
-        ).render().as_openai_format(
+        ).render().as_openai_chat_completion_api_format(
             chunk_preprocessor=lambda x: x if isinstance(x, str) else None
         ),
         {
@@ -114,7 +114,7 @@ class OpenAIConversionTest(unittest.TestCase):
             {
                 'content': 'this is a text',
             },
-            format='openai',
+            format='openai_chat_completion_api',
         ),
         lf.AIMessage('this is a text'),
     )
@@ -126,7 +126,7 @@ class OpenAIConversionTest(unittest.TestCase):
                 'role': 'user',
                 'content': [{'type': 'text', 'text': 'hi'}],
             },
-            format='openai',
+            format='openai_chat_completion_api',
         ),
         lf.UserMessage('hi'),
     )
@@ -136,7 +136,7 @@ class OpenAIConversionTest(unittest.TestCase):
                 'role': 'assistant',
                 'content': [{'type': 'text', 'text': 'hi'}],
             },
-            format='openai',
+            format='openai_chat_completion_api',
         ),
         lf.AIMessage('hi'),
     )
@@ -146,7 +146,7 @@ class OpenAIConversionTest(unittest.TestCase):
                 'role': 'system',
                 'content': [{'type': 'text', 'text': 'hi'}],
             },
-            format='openai',
+            format='openai_chat_completion_api',
         ),
         lf.SystemMessage('hi'),
     )
@@ -156,21 +156,165 @@ class OpenAIConversionTest(unittest.TestCase):
               'role': 'function',
               'content': [{'type': 'text', 'text': 'hi'}],
           },
-          format='openai',
+          format='openai_chat_completion_api',
       )
 
   def test_from_value_with_image(self):
-    m = lf.Message.from_openai_format(
+    image = lf_modalities.Image.from_bytes(image_content)
+    m = lf.Message.from_openai_chat_completion_api_format(
+        lf.Template(
+            'What is this {{image}}?',
+            image=image
+        ).render().as_format('openai_chat_completion_api'),
+    )
+    self.assertEqual(m.text, f'What is this <<[[{image.id}]]>> ?')
+    self.assertIsInstance(m.images[0], lf_modalities.Image)
+    self.assertEqual(m.images[0].mime_type, 'image/png')
+    self.assertEqual(m.images[0].to_bytes(), image_content)
+
+
+class OpenAIResponsesAPIMessageConverterTest(unittest.TestCase):
+
+  def test_as_format_with_role(self):
+    self.assertEqual(
+        lf.UserMessage('hi').as_format('openai_responses_api'),
+        {
+            'type': 'message',
+            'role': 'user',
+            'content': [{'type': 'input_text', 'text': 'hi'}],
+        },
+    )
+    self.assertEqual(
+        lf.AIMessage('hi').as_format('openai_responses_api'),
+        {
+            'type': 'message',
+            'role': 'assistant',
+            'content': [{'type': 'output_text', 'text': 'hi'}],
+        },
+    )
+    self.assertEqual(
+        lf.SystemMessage('hi').as_format('openai_responses_api'),
+        {
+            'type': 'message',
+            'role': 'system',
+            'content': [{'type': 'input_text', 'text': 'hi'}],
+        },
+    )
+
+  def test_as_format_with_image(self):
+    self.assertEqual(
         lf.Template(
             'What is this {{image}}?',
             image=lf_modalities.Image.from_bytes(image_content)
-        ).render().as_format('openai'),
+        ).render().as_format('openai_responses_api'),
+        {
+            'type': 'message',
+            'role': 'user',
+            'content': [
+                {
+                    'type': 'input_text',
+                    'text': 'What is this'
+                },
+                {
+                    'type': 'input_image',
+                    'image_url': (
+                        'data:image/png;base64,'
+                        + base64.b64encode(image_content).decode('utf-8')
+                    )
+                },
+                {
+                    'type': 'input_text',
+                    'text': '?'
+                }
+            ],
+        },
     )
-    self.assertEqual(m.text, 'What is this <<[[obj0]]>> ?')
-    self.assertIsInstance(m.obj0, lf_modalities.Image)
-    self.assertEqual(m.obj0.mime_type, 'image/png')
-    self.assertEqual(m.obj0.to_bytes(), image_content)
 
+  def test_as_format_with_chunk_preprocessor(self):
+    self.assertEqual(
+        lf.Template(
+            'What is this {{image}}?',
+            image=lf_modalities.Image.from_bytes(image_content)
+        ).render().as_openai_responses_api_format(
+            chunk_preprocessor=lambda x: x if isinstance(x, str) else None
+        ),
+        {
+            'type': 'message',
+            'role': 'user',
+            'content': [
+                {
+                    'type': 'input_text',
+                    'text': 'What is this'
+                },
+                {
+                    'type': 'input_text',
+                    'text': '?'
+                }
+            ],
+        },
+    )
+
+  def test_from_value_with_simple_text(self):
+    self.assertEqual(
+        lf.Message.from_value(
+            {
+                'content': 'this is a text',
+            },
+            format='openai_responses_api',
+        ),
+        lf.AIMessage('this is a text'),
+    )
+
+  def test_from_value_with_role(self):
+    self.assertEqual(
+        lf.Message.from_value(
+            {
+                'role': 'user',
+                'content': [{'type': 'input_text', 'text': 'hi'}],
+            },
+            format='openai_responses_api',
+        ),
+        lf.UserMessage('hi'),
+    )
+    self.assertEqual(
+        lf.Message.from_value(
+            {
+                'role': 'assistant',
+                'content': [{'type': 'output_text', 'text': 'hi'}],
+            },
+            format='openai_responses_api',
+        ),
+        lf.AIMessage('hi'),
+    )
+    self.assertEqual(
+        lf.Message.from_value(
+            {
+                'role': 'system',
+                'content': [{'type': 'input_text', 'text': 'hi'}],
+            },
+            format='openai_responses_api',
+        ),
+        lf.SystemMessage('hi'),
+    )
+    with self.assertRaisesRegex(ValueError, 'Unsupported role: .*'):
+      lf.Message.from_value(
+          {
+              'role': 'function',
+              'content': [{'type': 'input_text', 'text': 'hi'}],
+          },
+          format='openai_responses_api',
+      )
+
+  def test_from_value_with_image(self):
+    image = lf_modalities.Image.from_bytes(image_content)
+    m = lf.Message.from_openai_responses_api_format(
+        lf.Template(
+            'What is this {{image}}?', image=image
+        ).render().as_format('openai_responses_api'),
+    )
+    self.assertEqual(m.text, f'What is this <<[[{image.id}]]>> ?')
+    self.assertIsInstance(m.modalities()[0], lf_modalities.Image)
+    self.assertEqual(m.modalities()[0].content, image_content)
 
 if __name__ == '__main__':
   unittest.main()

langfun/core/eval/v2/progress_tracking_test.py

@@ -18,6 +18,7 @@ import sys
 import tempfile
 import unittest
 
+from langfun.core import concurrent as lf_concurrent
 from langfun.core import console as lf_console
 from langfun.core.eval.v2 import eval_test_helper
 from langfun.core.eval.v2 import progress_tracking    # pylint: disable=unused-import
@@ -51,6 +52,7 @@ class TqdmProgressTrackerTest(unittest.TestCase):
     with contextlib.redirect_stderr(string_io):
       _ = experiment.run(root_dir, 'new', plugins=[])
       sys.stderr.flush()
+    lf_concurrent.ProgressBar.refresh()
     self.assertIn('All: 100%', string_io.getvalue())
 
   def test_with_example_ids(self):
@@ -62,6 +64,7 @@ class TqdmProgressTrackerTest(unittest.TestCase):
     with contextlib.redirect_stderr(string_io):
       _ = experiment.run(root_dir, 'new', example_ids=[1], plugins=[])
       sys.stderr.flush()
+    lf_concurrent.ProgressBar.refresh()
     self.assertIn('All: 100%', string_io.getvalue())
 
 

langfun/core/langfunc_test.py

@@ -82,7 +82,7 @@ class LangFuncCallTest(unittest.TestCase):
 
     i = l.render()
     self.assertEqual(i, 'Hello')
-    self.assertEqual(i, message.UserMessage('Hello'))
+    self.assertEqual(i, message.UserMessage('Hello', __template_input__={}))
     self.assertEqual(i.tags, ['rendered'])
 
     r = l()
@@ -96,7 +96,9 @@ class LangFuncCallTest(unittest.TestCase):
     self.assertEqual(r.tags, ['lm-response', 'lm-output'])
     self.assertEqual(
         r.source,
-        message.UserMessage('Hello', metadata=dict(cache_seed=0))
+        message.UserMessage(
+            'Hello', metadata=dict(cache_seed=0, __template_input__={})
+        )
     )
     self.assertEqual(r.source.tags, ['rendered', 'lm-input'])
 

langfun/core/language_model.py

@@ -1253,11 +1253,11 @@ class LanguageModel(component.Component):
         title=f'\n[{call_counter}] PROMPT SENT TO LM{title_suffix}:',
         color='green',
     )
-    referred_modalities = prompt.referred_modalities()
-    if referred_modalities:
+    if prompt.referred_modalities:
       console.write(
           pg.object_utils.kvlist_str(
-              [(k, repr(v), None) for k, v in referred_modalities.items()]
+              [(k, repr(v), None)
+               for k, v in prompt.referred_modalities.items()]
           ),
           title=f'\n[{call_counter}] MODALITY OBJECTS SENT TO LM:',
           color='green',
@@ -1343,9 +1343,9 @@ class LanguageModel(component.Component):
           color='green',
       )
       if isinstance(prompt, list):
-        referred_modalities_lst = [p.referred_modalities() for p in prompt]
+        referred_modalities_lst = [p.referred_modalities for p in prompt]
       else:
-        referred_modalities_lst = [prompt.referred_modalities(),]
+        referred_modalities_lst = [prompt.referred_modalities,]
       if referred_modalities_lst:
         for referred_modalities in referred_modalities_lst:
           console.write(
@@ -1420,7 +1420,7 @@ class LanguageModel(component.Component):
           title=f'\n[{call_counter}] PROMPT TO TOKENIZE:',
           color='green',
       )
-      referred_modalities_lst = [prompt.referred_modalities(),]
+      referred_modalities_lst = [prompt.referred_modalities,]
       if referred_modalities_lst:
         for referred_modalities in referred_modalities_lst:
           console.write(

langfun/core/language_model_test.py

@@ -656,11 +656,17 @@ class LanguageModelTest(unittest.TestCase):
 
     string_io = io.StringIO()
     lm = MockModel(sampling_options=lm_lib.LMSamplingOptions(top_k=1))
+    image = Image()
     with contextlib.redirect_stdout(string_io):
       self.assertEqual(
-          lm(message_lib.UserMessage(
-              'hi <<[[image]]>>', image=Image()), debug=True),
-          'hi <<[[image]]>>'
+          lm(
+              message_lib.UserMessage(
+                  f'hi <<[[{image.id}]]>>',
+                  referred_modalities=[image],
+              ),
+              debug=True
+          ),
+          f'hi <<[[{image.id}]]>>'
       )
 
     debug_info = string_io.getvalue()

langfun/core/llms/__init__.py

@@ -30,7 +30,8 @@ from langfun.core.llms.compositional import RandomChoice
 
 # Base models by request/response protocol.
 from langfun.core.llms.rest import REST
-from langfun.core.llms.openai_compatible import OpenAICompatible
+from langfun.core.llms.openai_compatible import OpenAIChatCompletionAPI
+from langfun.core.llms.openai_compatible import OpenAIResponsesAPI
 from langfun.core.llms.gemini import Gemini
 from langfun.core.llms.anthropic import Anthropic
 

langfun/core/llms/cache/base.py

@@ -121,4 +121,6 @@ class LMCacheBase(lf.LMCache):
 
 def default_key(lm: lf.LanguageModel, prompt: lf.Message, seed: int) -> Any:
   """Default key for LM cache."""
-  return (prompt.text_with_modality_hash, lm.sampling_options.cache_key(), seed)
+  # prompt text already contains the modality id for referenced modality
+  # objects, so no need to include them in the key.
+  return (prompt.text, lm.sampling_options.cache_key(), seed)