langfun 0.0.2.dev20240527__py3-none-any.whl → 0.0.2.dev20240531__py3-none-any.whl

langfun/__init__.py

@@ -23,6 +23,8 @@ Schema = structured.Schema
 MISSING = structured.MISSING
 UNKNOWN = structured.UNKNOWN
 
+include_method_in_prompt = structured.include_method_in_prompt
+
 MappingExample = structured.MappingExample
 
 call = structured.call

langfun/core/language_model_test.py

@@ -418,8 +418,9 @@ class LanguageModelTest(unittest.TestCase):
     with contextlib.redirect_stdout(string_io):
       self.assertEqual(
           lm(message_lib.UserMessage(
-              'hi {{image}}', image=Image()), debug=True),
-          'hi {{image}}')
+              'hi <<[[image]]>>', image=Image()), debug=True),
+          'hi <<[[image]]>>'
+      )
 
     debug_info = string_io.getvalue()
     self.assertIn('[0] LM INFO', debug_info)

langfun/core/llms/google_genai_test.py

@@ -102,7 +102,7 @@ class GenAITest(unittest.TestCase):
 
   def test_content_from_message_mm(self):
     message = lf.UserMessage(
-        'This is an {{image}}, what is it?',
+        'This is an <<[[image]]>>, what is it?',
         image=lf_modalities.Image.from_bytes(example_image),
     )
 

langfun/core/llms/openai_test.py

@@ -164,7 +164,7 @@ class OpenAITest(unittest.TestCase):
         self.assertEqual(
             lm(
                 lf.UserMessage(
-                    'hello {{image}}',
+                    'hello <<[[image]]>>',
                     image=lf_modalities.Image.from_uri('https://fake/image')
                 ),
                 sampling_options=lf.LMSamplingOptions(n=2)

langfun/core/llms/vertexai_test.py

@@ -74,7 +74,7 @@ class VertexAITest(unittest.TestCase):
 
   def test_content_from_message_mm(self):
     message = lf.UserMessage(
-        'This is an {{image}}, what is it?',
+        'This is an <<[[image]]>>, what is it?',
         image=lf_modalities.Image.from_bytes(example_image),
     )
 

langfun/core/message.py

@@ -144,7 +144,7 @@ class Message(natural_language.NaturalLanguageFormattable, pg.Object):
   def from_value(cls, value: Union[str, 'Message']) -> 'Message':
     """Creates a message from a value or return value itself if a Message."""
     if isinstance(value, modality.Modality):
-      return cls('{{object}}', object=value)
+      return cls('<<[[object]]>>', object=value)
     if isinstance(value, Message):
       return value
     return cls(value)

langfun/core/message_test.py

@@ -54,7 +54,7 @@ class MessageTest(unittest.TestCase):
     self.assertTrue(
         pg.eq(
             message.UserMessage.from_value(CustomModality('foo')),
-            message.UserMessage('{{object}}', object=CustomModality('foo')),
+            message.UserMessage('<<[[object]]>>', object=CustomModality('foo')),
         )
     )
     m = message.UserMessage('hi')
@@ -258,13 +258,16 @@ class MessageTest(unittest.TestCase):
 
   def test_referred_modalities(self):
     m1 = message.UserMessage(
-        'hi, this is a {{img1}} and {{x.img2}}',
+        'hi, this is a <<[[img1]]>> and <<[[x.img2]]>>',
         img1=CustomModality('foo'),
         x=dict(img2=CustomModality('bar')),
     )
     m2 = message.SystemMessage('class Question:\n  image={{img1}}', source=m1)
     m3 = message.AIMessage(
-        'This is the {{output_image}} based on {{x.img2}}, {{unknown_var}}',
+        (
+            'This is the <<[[output_image]]>> based on <<[[x.img2]]>>, '
+            '{{unknown_var}}'
+        ),
         output_image=CustomModality('bar'),
         source=m2,
     )
@@ -279,8 +282,8 @@ class MessageTest(unittest.TestCase):
   def test_chunking(self):
     m = message.UserMessage(
         inspect.cleandoc("""
-            Hi, this is {{a}} and this is {{b}}.
-            {{x.c}} {{something else
+            Hi, this is <<[[a]]>> and this is {{b}}.
+            <<[[x.c]]>> {{something else
             """),
         a=CustomModality('foo'),
         x=dict(c=CustomModality('bar')),
@@ -294,7 +297,7 @@ class MessageTest(unittest.TestCase):
                 CustomModality('foo'),
                 'and this is {{b}}.',
                 CustomModality('bar'),
-                ' {{something else',
+                '{{something else',
             ],
         )
     )
@@ -304,10 +307,10 @@ class MessageTest(unittest.TestCase):
             message.AIMessage(
                 inspect.cleandoc("""
                     Hi, this is
-                    {{obj0}}
+                    <<[[obj0]]>>
                     and this is {{b}}.
-                    {{obj1}}
-                     {{something else
+                    <<[[obj1]]>>
+                    {{something else
                     """),
                 obj0=pg.Ref(m.a),
                 obj1=pg.Ref(m.x.c),

langfun/core/modalities/__init__.py

@@ -17,8 +17,12 @@
 # pylint: disable=g-bad-import-order
 # pylint: disable=g-import-not-at-top
 
+from langfun.core.modalities.audio import Audio
 from langfun.core.modalities.mime import MimeType
 from langfun.core.modalities.mime import Custom
+from langfun.core.modalities.ms_office import Docx
+from langfun.core.modalities.ms_office import Pptx
+from langfun.core.modalities.ms_office import Xlsx
 from langfun.core.modalities.image import Image
 from langfun.core.modalities.pdf import PDF
 from langfun.core.modalities.video import Video

langfun/core/modalities/audio.py

@@ -0,0 +1,30 @@
+# Copyright 2024 The Langfun Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Audio types."""
+
+import functools
+from langfun.core.modalities import mime
+
+
+class Audio(mime.MimeType):
+  """Audio."""
+
+  MIME_PREFIX = 'audio'
+
+  @functools.cached_property
+  def audio_format(self) -> str:
+    return self.mime_type.removeprefix(self.MIME_PREFIX + '/')
+
+  def _html(self, uri: str) -> str:
+    return f'<audio controls> <source src="{uri}"> </audio>'

langfun/core/modalities/audio_test.py

@@ -0,0 +1,63 @@
+# Copyright 2024 The Langfun Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Audio tests."""
+import unittest
+from unittest import mock
+
+from langfun.core.modalities import audio as audio_lib
+import pyglove as pg
+
+
+content_bytes = (
+    b'RIFF$\x00\x00\x00WAVEfmt'
+    b' \x10\x00\x00\x00\x01\x00\x01\x00\x00\x04\x00\x00\x00\x04\x00\x00\x01\x00\x08\x00data\x00\x00\x00\x00'
+)
+
+
+def mock_request(*args, **kwargs):
+  del args, kwargs
+  return pg.Dict(content=content_bytes)
+
+
+class AudioTest(unittest.TestCase):
+
+  def test_audio_content(self):
+    audio = audio_lib.Audio.from_bytes(content_bytes)
+    self.assertEqual(audio.mime_type, 'audio/x-wav')
+    self.assertEqual(audio.audio_format, 'x-wav')
+    self.assertEqual(audio.to_bytes(), content_bytes)
+
+  def test_bad_audio(self):
+    audio = audio_lib.Audio.from_bytes(b'bad')
+    with self.assertRaisesRegex(ValueError, 'Expected MIME type'):
+      _ = audio.audio_format
+
+
+class AudioFileTest(unittest.TestCase):
+
+  def test_audio_file(self):
+    audio = audio_lib.Audio.from_uri('http://mock/web/a.wav')
+    with mock.patch('requests.get') as mock_requests_get:
+      mock_requests_get.side_effect = mock_request
+      self.assertEqual(audio.audio_format, 'x-wav')
+      self.assertEqual(audio.mime_type, 'audio/x-wav')
+      self.assertEqual(
+          audio._repr_html_(),
+          '<audio controls> <source src="http://mock/web/a.wav"> </audio>',
+      )
+      self.assertEqual(audio.to_bytes(), content_bytes)
+
+
+if __name__ == '__main__':
+  unittest.main()

langfun/core/modalities/ms_office.py

@@ -0,0 +1,83 @@
+# Copyright 2023 The Langfun Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Microsoft Office file types."""
+
+import base64
+import io
+import os
+from langfun.core.modalities import mime
+from langfun.core.modalities import pdf
+import requests
+
+
+class Xlsx(mime.MimeType):
+  """Xlsx file type."""
+
+  MIME_PREFIX = (
+      'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
+  )
+
+  def to_html(self) -> str:
+    import pandas as pd  # pylint: disable=g-import-not-at-top
+
+    df = pd.read_excel(io.BytesIO(self.to_bytes()))
+    return df.to_html()
+
+  def _repr_html_(self) -> str:
+    return self.to_html()
+
+
+class Docx(mime.MimeType):
+  """Docx file type."""
+
+  MIME_PREFIX = (
+      'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
+  )
+
+  def to_xml(self) -> str:
+    import docx  # pylint: disable=g-import-not-at-top
+
+    doc = docx.Document(io.BytesIO(self.to_bytes()))
+    return str(doc.element.xml)
+
+  def _repr_html_(self) -> str:
+    return self.to_xml()
+
+
+class Pptx(mime.MimeType):
+  """Pptx file type."""
+
+  MIME_PREFIX = 'application/vnd.openxmlformats-officedocument.presentationml.presentation'
+  API_URL = 'https://v2.convertapi.com/convert/pptx/to/pdf'
+
+  def to_pdf(self, convert_api_key: str | None = None) -> pdf.PDF:
+    filename = os.path.basename(self.uri)
+    file_bytes = self.to_bytes()
+
+    api_key = convert_api_key or os.environ.get('CONVERT_API_KEY')
+    url = f'{self.API_URL}?Secret={api_key}'
+
+    json = {
+        'Parameters': [{
+            'Name': 'File',
+            'FileValue': {
+                'Name': filename,
+                'Data': base64.b64encode(file_bytes),
+            },
+        }]
+    }
+    response = requests.post(url, json=json).json()
+    base64_pdf = response['Files'][0]['FileData']
+    pdf_bytes = base64.b64decode(base64_pdf)
+    return pdf.PDF.from_bytes(content=pdf_bytes)