langfun 0.0.2.dev20240531__py3-none-any.whl → 0.0.2.dev20240601__py3-none-any.whl

langfun/__init__.py

@@ -57,6 +57,8 @@ from langfun.core import memories
 
 from langfun.core import modalities
 
+Mime = modalities.Mime
+MimeType = Mime  # For backwards compatibility.
 Image = modalities.Image
 Video = modalities.Video
 PDF = modalities.PDF

langfun/core/__init__.py

@@ -94,6 +94,7 @@ from langfun.core.message import MemoryRecord
 # Interface for modality.
 from langfun.core.modality import Modality
 from langfun.core.modality import ModalityRef
+from langfun.core.modality import ModalityError
 
 # Interfaces for languge models.
 from langfun.core.language_model import LanguageModel

langfun/core/llms/google_genai.py

@@ -49,9 +49,10 @@ class GenAI(lf.LanguageModel):
       ),
   ] = None
 
-  multimodal: Annotated[bool, 'Whether this model has multimodal support.'] = (
-      False
-  )
+  supported_modalities: Annotated[
+      list[str],
+      'A list of MIME types for supported modalities'
+  ] = []
 
   # Set the default max concurrency to 8 workers.
   max_concurrency = 8
@@ -118,14 +119,27 @@ class GenAI(lf.LanguageModel):
     chunks = []
     for lf_chunk in formatted.chunk():
       if isinstance(lf_chunk, str):
-        chunk = lf_chunk
-      elif self.multimodal and isinstance(lf_chunk, lf_modalities.MimeType):
-        chunk = genai.types.BlobDict(
-            data=lf_chunk.to_bytes(), mime_type=lf_chunk.mime_type
-        )
+        chunks.append(lf_chunk)
+      elif isinstance(lf_chunk, lf_modalities.Mime):
+        try:
+          modalities = lf_chunk.make_compatible(
+              self.supported_modalities + ['text/plain']
+          )
+          if isinstance(modalities, lf_modalities.Mime):
+            modalities = [modalities]
+          for modality in modalities:
+            if modality.is_text:
+              chunk = modality.to_text()
+            else:
+              chunk = genai.types.BlobDict(
+                  data=modality.to_bytes(),
+                  mime_type=modality.mime_type
+              )
+            chunks.append(chunk)
+        except lf.ModalityError as e:
+          raise lf.ModalityError(f'Unsupported modality: {lf_chunk!r}') from e
       else:
-        raise ValueError(f'Unsupported modality: {lf_chunk!r}')
-      chunks.append(chunk)
+        raise lf.ModalityError(f'Unsupported modality: {lf_chunk!r}')
     return chunks
 
   def _response_to_result(
@@ -264,18 +278,57 @@ _GOOGLE_GENAI_MODEL_HUB = _ModelHub()
 #
 
 
+_IMAGE_TYPES = [
+    'image/png',
+    'image/jpeg',
+    'image/webp',
+    'image/heic',
+    'image/heif',
+]
+
+_AUDIO_TYPES = [
+    'audio/aac',
+    'audio/flac',
+    'audio/mp3',
+    'audio/m4a',
+    'audio/mpeg',
+    'audio/mpga',
+    'audio/mp4',
+    'audio/opus',
+    'audio/pcm',
+    'audio/wav',
+    'audio/webm'
+]
+
+_VIDEO_TYPES = [
+    'video/mov',
+    'video/mpeg',
+    'video/mpegps',
+    'video/mpg',
+    'video/mp4',
+    'video/webm',
+    'video/wmv',
+    'video/x-flv',
+    'video/3gpp',
+]
+
+_PDF = [
+    'application/pdf',
+]
+
+
 class GeminiPro1_5(GenAI):  # pylint: disable=invalid-name
   """Gemini Pro latest model."""
 
   model = 'gemini-1.5-pro-latest'
-  multimodal = True
+  supported_modalities = _PDF + _IMAGE_TYPES + _AUDIO_TYPES + _VIDEO_TYPES
 
 
 class GeminiFlash1_5(GenAI):  # pylint: disable=invalid-name
   """Gemini Flash latest model."""
 
   model = 'gemini-1.5-flash-latest'
-  multimodal = True
+  supported_modalities = _PDF + _IMAGE_TYPES + _AUDIO_TYPES + _VIDEO_TYPES
 
 
 class GeminiPro(GenAI):
@@ -288,7 +341,7 @@ class GeminiProVision(GenAI):
   """Gemini Pro vision model."""
 
   model = 'gemini-pro-vision'
-  multimodal = True
+  supported_modalities = _IMAGE_TYPES + _VIDEO_TYPES
 
 
 class Palm2(GenAI):

langfun/core/llms/google_genai_test.py

@@ -107,7 +107,7 @@ class GenAITest(unittest.TestCase):
     )
 
     # Non-multimodal model.
-    with self.assertRaisesRegex(ValueError, 'Unsupported modality'):
+    with self.assertRaisesRegex(lf.ModalityError, 'Unsupported modality'):
       google_genai.GeminiPro()._content_from_message(message)
 
     model = google_genai.GeminiProVision()

langfun/core/llms/vertexai.py

@@ -75,9 +75,10 @@ class VertexAI(lf.LanguageModel):
       ),
   ] = None
 
-  multimodal: Annotated[bool, 'Whether this model has multimodal support.'] = (
-      False
-  )
+  supported_modalities: Annotated[
+      list[str],
+      'A list of MIME types for supported modalities'
+  ] = []
 
   def _on_bound(self):
     super()._on_bound()
@@ -142,16 +143,29 @@ class VertexAI(lf.LanguageModel):
     """Gets generation input from langfun message."""
     from vertexai import generative_models
     chunks = []
+
     for lf_chunk in prompt.chunk():
       if isinstance(lf_chunk, str):
-        chunk = lf_chunk
-      elif self.multimodal and isinstance(lf_chunk, lf_modalities.MimeType):
-        chunk = generative_models.Part.from_data(
-            lf_chunk.to_bytes(), lf_chunk.mime_type
-        )
+        chunks.append(lf_chunk)
+      elif isinstance(lf_chunk, lf_modalities.Mime):
+        try:
+          modalities = lf_chunk.make_compatible(
+              self.supported_modalities + ['text/plain']
+          )
+          if isinstance(modalities, lf_modalities.Mime):
+            modalities = [modalities]
+          for modality in modalities:
+            if modality.is_text:
+              chunk = modality.to_text()
+            else:
+              chunk = generative_models.Part.from_data(
+                  modality.to_bytes(), modality.mime_type
+              )
+            chunks.append(chunk)
+        except lf.ModalityError as e:
+          raise lf.ModalityError(f'Unsupported modality: {lf_chunk!r}') from e
       else:
-        raise ValueError(f'Unsupported modality: {lf_chunk!r}')
-      chunks.append(chunk)
+        raise lf.ModalityError(f'Unsupported modality: {lf_chunk!r}')
     return chunks
 
   def _generation_response_to_message(
@@ -265,25 +279,64 @@ class _ModelHub:
 _VERTEXAI_MODEL_HUB = _ModelHub()
 
 
+_IMAGE_TYPES = [
+    'image/png',
+    'image/jpeg',
+    'image/webp',
+    'image/heic',
+    'image/heif',
+]
+
+_AUDIO_TYPES = [
+    'audio/aac',
+    'audio/flac',
+    'audio/mp3',
+    'audio/m4a',
+    'audio/mpeg',
+    'audio/mpga',
+    'audio/mp4',
+    'audio/opus',
+    'audio/pcm',
+    'audio/wav',
+    'audio/webm'
+]
+
+_VIDEO_TYPES = [
+    'video/mov',
+    'video/mpeg',
+    'video/mpegps',
+    'video/mpg',
+    'video/mp4',
+    'video/webm',
+    'video/wmv',
+    'video/x-flv',
+    'video/3gpp',
+]
+
+_PDF = [
+    'application/pdf',
+]
+
+
 class VertexAIGeminiPro1_5(VertexAI):  # pylint: disable=invalid-name
   """Vertex AI Gemini 1.5 Pro model."""
 
   model = 'gemini-1.5-pro-preview-0514'
-  multimodal = True
+  supported_modalities = _PDF + _IMAGE_TYPES + _AUDIO_TYPES + _VIDEO_TYPES
 
 
 class VertexAIGeminiPro1_5_0409(VertexAI):  # pylint: disable=invalid-name
   """Vertex AI Gemini 1.5 Pro model."""
 
   model = 'gemini-1.5-pro-preview-0409'
-  multimodal = True
+  supported_modalities = _PDF + _IMAGE_TYPES + _AUDIO_TYPES + _VIDEO_TYPES
 
 
 class VertexAIGeminiFlash1_5(VertexAI):  # pylint: disable=invalid-name
   """Vertex AI Gemini 1.5 Flash model."""
 
   model = 'gemini-1.5-flash-preview-0514'
-  multimodal = True
+  supported_modalities = _PDF + _IMAGE_TYPES + _AUDIO_TYPES + _VIDEO_TYPES
 
 
 class VertexAIGeminiPro1(VertexAI):  # pylint: disable=invalid-name
@@ -296,7 +349,7 @@ class VertexAIGeminiPro1Vision(VertexAI):  # pylint: disable=invalid-name
   """Vertex AI Gemini 1.0 Pro model."""
 
   model = 'gemini-1.0-pro-vision'
-  multimodal = True
+  supported_modalities = _IMAGE_TYPES + _VIDEO_TYPES
 
 
 class VertexAIPalm2(VertexAI):  # pylint: disable=invalid-name

langfun/core/llms/vertexai_test.py

@@ -79,7 +79,7 @@ class VertexAITest(unittest.TestCase):
     )
 
     # Non-multimodal model.
-    with self.assertRaisesRegex(ValueError, 'Unsupported modality'):
+    with self.assertRaisesRegex(lf.ModalityError, 'Unsupported modality'):
       vertexai.VertexAIGeminiPro1()._content_from_message(message)
 
     model = vertexai.VertexAIGeminiPro1Vision()

langfun/core/modalities/__init__.py

@@ -18,7 +18,7 @@
 # pylint: disable=g-import-not-at-top
 
 from langfun.core.modalities.audio import Audio
-from langfun.core.modalities.mime import MimeType
+from langfun.core.modalities.mime import Mime
 from langfun.core.modalities.mime import Custom
 from langfun.core.modalities.ms_office import Docx
 from langfun.core.modalities.ms_office import Pptx

langfun/core/modalities/audio.py

@@ -17,7 +17,7 @@ import functools
 from langfun.core.modalities import mime
 
 
-class Audio(mime.MimeType):
+class Audio(mime.Mime):
   """Audio."""
 
   MIME_PREFIX = 'audio'

langfun/core/modalities/image.py

@@ -17,7 +17,7 @@ import functools
 from langfun.core.modalities import mime
 
 
-class Image(mime.MimeType):
+class Image(mime.Mime):
   """Image."""
 
   MIME_PREFIX = 'image'

langfun/core/modalities/image_test.py

@@ -15,7 +15,9 @@
 import unittest
 from unittest import mock
 
+import langfun.core as lf
 from langfun.core.modalities import image as image_lib
+from langfun.core.modalities import mime as mime_lib
 import pyglove as pg
 
 
@@ -36,23 +38,29 @@ def mock_request(*args, **kwargs):
   return pg.Dict(content=image_content)
 
 
-class ImageContentTest(unittest.TestCase):
+class ImageTest(unittest.TestCase):
 
-  def test_image_content(self):
+  def test_from_bytes(self):
     image = image_lib.Image.from_bytes(image_content)
     self.assertEqual(image.image_format, 'png')
     self.assertIn('data:image/png;base64,', image._repr_html_())
     self.assertEqual(image.to_bytes(), image_content)
+    with self.assertRaisesRegex(
+        lf.ModalityError, '.* cannot be converted to text'
+    ):
+      image.to_text()
 
-  def test_bad_image(self):
+  def test_from_bytes_invalid(self):
     image = image_lib.Image.from_bytes(b'bad')
     with self.assertRaisesRegex(ValueError, 'Expected MIME type'):
       _ = image.image_format
 
+  def test_from_bytes_base_cls(self):
+    self.assertIsInstance(
+        mime_lib.Mime.from_bytes(image_content), image_lib.Image
+    )
 
-class ImageFileTest(unittest.TestCase):
-
-  def test_image_file(self):
+  def test_from_uri(self):
     image = image_lib.Image.from_uri('http://mock/web/a.png')
     with mock.patch('requests.get') as mock_requests_get:
       mock_requests_get.side_effect = mock_request
@@ -60,6 +68,15 @@ class ImageFileTest(unittest.TestCase):
       self.assertEqual(image._repr_html_(), '<img src="http://mock/web/a.png">')
       self.assertEqual(image.to_bytes(), image_content)
 
+  def test_from_uri_base_cls(self):
+    with mock.patch('requests.get') as mock_requests_get:
+      mock_requests_get.side_effect = mock_request
+      image = mime_lib.Mime.from_uri('http://mock/web/a.png')
+      self.assertIsInstance(image, image_lib.Image)
+      self.assertEqual(image.image_format, 'png')
+      self.assertEqual(image._repr_html_(), '<img src="http://mock/web/a.png">')
+      self.assertEqual(image.to_bytes(), image_content)
+
 
 if __name__ == '__main__':
   unittest.main()

langfun/core/modalities/mime.py

@@ -15,15 +15,15 @@
 
 import base64
 import functools
-from typing import Annotated, Union
+from typing import Annotated, Iterable, Type, Union
 import langfun.core as lf
 import magic
 import pyglove as pg
 import requests
 
 
-class MimeType(lf.Modality):
-  """Base for MIME type data."""
+class Mime(lf.Modality):
+  """Base for MIME data."""
 
   # The regular expression that describes the MIME type str.
   # If None, the MIME type is dynamic. Subclass could override.
@@ -39,12 +39,80 @@ class MimeType(lf.Modality):
   def mime_type(self) -> str:
     """Returns the MIME type."""
     mime = magic.from_buffer((self.to_bytes()), mime=True)
-    if self.MIME_PREFIX and not mime.lower().startswith(self.MIME_PREFIX):
+    if (
+        self.MIME_PREFIX
+        and not mime.lower().startswith(self.MIME_PREFIX)
+        # NOTE(daiyip): libmagic fails to detect the MIME type of some binary
+        # files.
+        and mime != 'application/octet-stream'
+    ):
       raise ValueError(
           f'Expected MIME type: {self.MIME_PREFIX}, Encountered: {mime}'
       )
     return mime
 
+  @functools.cached_property
+  def is_text(self) -> bool:
+    return self.mime_type.startswith(
+        (
+            'text/',
+            'application/javascript',
+            'application/json',
+            'application/ld+json',
+            'application/plain',
+            'application/xhtml+xml',
+            'application/xml',
+            'application/x-tex',
+            'application/x-yaml',
+        )
+    )
+
+  @property
+  def is_binary(self) -> bool:
+    """Returns True if the MIME type is a binary type."""
+    return not self.is_text
+
+  def to_text(self) -> str:
+    """Returns the text content of the MIME type."""
+    if not self.is_text:
+      raise lf.ModalityError(
+          f'MIME type {self.mime_type!r} cannot be converted to text.'
+      )
+    return self.to_bytes().decode()
+
+  def is_compatible(
+      self, mime_types: str | Iterable[str]
+  ) -> bool:
+    """Returns True if this object is compatible to any of the MIME types."""
+    if isinstance(mime_types, str):
+      mime_types = {mime_types}
+    return self._is_compatible(mime_types)
+
+  def _is_compatible(self, mime_types: Iterable[str]):
+    return self.mime_type in mime_types
+
+  def make_compatible(
+      self,
+      mime_types: str | Iterable[str]
+      ) -> Union['Mime', list['Mime']]:
+    """Makes compatible MIME objects from this object."""
+    if isinstance(mime_types, str):
+      mime_types = {mime_types}
+    if not self._is_compatible(mime_types):
+      raise lf.ModalityError(
+          f'MIME type {self.mime_type!r} cannot be converted to supported '
+          f'types: {mime_types!r}.'
+      )
+    return self._make_compatible(mime_types)
+
+  def _make_compatible(
+      self,
+      mime_types: Iterable[str]
+  ) -> Union['Mime', list['Mime']]:
+    """Makes compatbile MIME objects from this object."""
+    del mime_types
+    return self
+
   def _on_bound(self):
     super()._on_bound()
     if self.uri is None and self.content is None:
@@ -54,15 +122,7 @@ class MimeType(lf.Modality):
     if self.content is not None:
       return self.content
 
-    assert self.uri is not None
-    if self.uri.lower().startswith(('http:', 'https:', 'ftp:')):
-      content = requests.get(
-          self.uri,
-          headers={'User-Agent': 'Langfun'},
-      ).content
-    else:
-      content = pg.io.readfile(self.uri, mode='rb')
-    self.rebind(content=content, skip_notification=True)
+    self.rebind(content=self.download(self.uri), skip_notification=True)
     return self.content
 
   @property
@@ -71,13 +131,42 @@ class MimeType(lf.Modality):
     return f'data:{self.mime_type};base64,{base64_content}'
 
   @classmethod
-  def from_uri(cls, uri: str, **kwargs) -> 'MimeType':
+  def from_uri(cls, uri: str, **kwargs) -> 'Mime':
+    if cls is Mime:
+      content = cls.download(uri)
+      mime = magic.from_buffer(content, mime=True).lower()
+      return cls.class_from_mime_type(mime)(uri=uri, content=content, **kwargs)
     return cls(uri=uri, content=None, **kwargs)
 
   @classmethod
-  def from_bytes(cls, content: bytes | str, **kwargs) -> 'MimeType':
+  def from_bytes(cls, content: bytes | str, **kwargs) -> 'Mime':
+    if cls is Mime:
+      mime = magic.from_buffer(content, mime=True).lower()
+      return cls.class_from_mime_type(mime)(content=content, **kwargs)
     return cls(content=content, **kwargs)
 
+  @classmethod
+  def class_from_mime_type(cls, mime_type: str) -> Type['Mime']:
+    """Subclass from the given MIME type."""
+    for subcls in cls.__subclasses__():
+      if subcls.MIME_PREFIX is not None and mime_type.startswith(
+          subcls.MIME_PREFIX):
+        return subcls
+    return cls
+
+  @classmethod
+  def download(cls, uri: str) -> bytes | str:
+    """Downloads the content of the given URI."""
+    if uri.lower().startswith(('http:', 'https:', 'ftp:')):
+      return requests.get(
+          uri,
+          headers={'User-Agent': 'Mozilla/5.0'},
+      ).content
+    else:
+      content = pg.io.readfile(uri, mode='rb')
+      assert content is not None
+      return content
+
   def _repr_html_(self) -> str:
     if self.uri and self.uri.lower().startswith(('http:', 'https:', 'ftp:')):
       uri = self.uri
@@ -90,7 +179,7 @@ class MimeType(lf.Modality):
 
 
 @pg.use_init_args(['mime', 'content', 'uri'])
-class Custom(MimeType):
+class Custom(Mime):
   """Custom MIME data."""
 
   mime: Annotated[

langfun/core/modalities/mime_test.py

@@ -15,6 +15,7 @@
 import unittest
 from unittest import mock
 
+import langfun.core as lf
 from langfun.core.modalities import mime
 import pyglove as pg
 
@@ -31,10 +32,24 @@ def mock_readfile(*args, **kwargs):
 
 class CustomMimeTest(unittest.TestCase):
 
-  def test_content(self):
-    content = mime.Custom('text/plain', 'foo')
-    self.assertEqual(content.to_bytes(), 'foo')
+  def test_from_byes(self):
+    content = mime.Mime.from_bytes(b'hello')
+    self.assertIs(content.__class__, mime.Mime)
+
+    content = mime.Custom('text/plain', b'foo')
+    self.assertEqual(content.to_bytes(), b'foo')
     self.assertEqual(content.mime_type, 'text/plain')
+    self.assertTrue(content.is_text)
+    self.assertFalse(content.is_binary)
+    self.assertEqual(content.to_text(), 'foo')
+    self.assertTrue(content.is_compatible('text/plain'))
+    self.assertFalse(content.is_compatible('text/xml'))
+    self.assertIs(content.make_compatible('text/plain'), content)
+
+    with self.assertRaisesRegex(
+        lf.ModalityError, '.* cannot be converted to supported types'
+    ):
+      content.make_compatible('application/pdf')
 
     with self.assertRaisesRegex(
         ValueError, 'Either uri or content must be provided.'

langfun/core/modalities/ms_office.py

@@ -16,12 +16,13 @@
 import base64
 import io
 import os
+from typing import Iterable
 from langfun.core.modalities import mime
 from langfun.core.modalities import pdf
 import requests
 
 
-class Xlsx(mime.MimeType):
+class Xlsx(mime.Mime):
   """Xlsx file type."""
 
   MIME_PREFIX = (
@@ -37,8 +38,19 @@ class Xlsx(mime.MimeType):
   def _repr_html_(self) -> str:
     return self.to_html()
 
+  def _is_compatible(self, mime_types: Iterable[str]) -> bool:
+    return bool(set(mime_types).intersection([
+        'text/html',
+        'text/plain',
+    ]))
 
-class Docx(mime.MimeType):
+  def _make_compatible(self, mime_types: Iterable[str]) -> mime.Mime:
+    """Returns the MimeType of the converted file."""
+    del mime_types
+    return mime.Mime(uri=self.uri, content=self.to_html())
+
+
+class Docx(mime.Mime):
   """Docx file type."""
 
   MIME_PREFIX = (
@@ -54,17 +66,26 @@ class Docx(mime.MimeType):
   def _repr_html_(self) -> str:
     return self.to_xml()
 
+  def _is_compatible(self, mime_types: Iterable[str]) -> bool:
+    return bool(set(mime_types).intersection([
+        'application/xml',
+        'text/xml',
+        'text/plain',
+    ]))
+
+  def _make_compatible(self, mime_types: Iterable[str]) -> mime.Mime:
+    """Returns the MimeType of the converted file."""
+    del mime_types
+    return mime.Mime(uri=self.uri, content=self.to_xml())
+
 
-class Pptx(mime.MimeType):
+class Pptx(mime.Mime):
   """Pptx file type."""
 
   MIME_PREFIX = 'application/vnd.openxmlformats-officedocument.presentationml.presentation'
   API_URL = 'https://v2.convertapi.com/convert/pptx/to/pdf'
 
   def to_pdf(self, convert_api_key: str | None = None) -> pdf.PDF:
-    filename = os.path.basename(self.uri)
-    file_bytes = self.to_bytes()
-
     api_key = convert_api_key or os.environ.get('CONVERT_API_KEY')
     url = f'{self.API_URL}?Secret={api_key}'
 
@@ -72,12 +93,19 @@ class Pptx(mime.MimeType):
         'Parameters': [{
             'Name': 'File',
             'FileValue': {
-                'Name': filename,
-                'Data': base64.b64encode(file_bytes),
+                'Name': os.path.basename(self.uri) if self.uri else 'tmp.pptx',
+                'Data': base64.b64encode(self.to_bytes()).decode('utf-8'),
             },
         }]
     }
     response = requests.post(url, json=json).json()
     base64_pdf = response['Files'][0]['FileData']
-    pdf_bytes = base64.b64decode(base64_pdf)
-    return pdf.PDF.from_bytes(content=pdf_bytes)
+    return pdf.PDF.from_bytes(base64.b64decode(base64_pdf))
+
+  def _is_compatible(self, mime_types: Iterable[str]) -> bool:
+    return 'application/pdf' in mime_types
+
+  def _make_compatible(self, mime_types: Iterable[str]) -> mime.Mime:
+    """Returns the MimeType of the converted file."""
+    del mime_types
+    return self.to_pdf()

langfun/core/modalities/ms_office_test.py

@@ -12,11 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Video tests."""
+import base64
 import io
 import unittest
 from unittest import mock
 
 from langfun.core.modalities import ms_office as ms_office_lib
+from langfun.core.modalities import pdf as pdf_lib
 import pyglove as pg
 
 
@@ -243,23 +245,72 @@ def pptx_mock_request(*args, **kwargs):
   return pg.Dict(content=pptx_bytes)
 
 
+pdf_bytes = (
+    b'%PDF-1.1\n%\xc2\xa5\xc2\xb1\xc3\xab\n\n1 0 obj\n'
+    b'<< /Type /Catalog\n     /Pages 2 0 R\n  >>\nendobj\n\n2 0 obj\n '
+    b'<< /Type /Pages\n     /Kids [3 0 R]\n     '
+    b'/Count 1\n     /MediaBox [0 0 300 144]\n  '
+    b'>>\nendobj\n\n3 0 obj\n  '
+    b'<<  /Type /Page\n      /Parent 2 0 R\n      /Resources\n       '
+    b'<< /Font\n'
+    b'<< /F1\n'
+    b'<< /Type /Font\n'
+    b'/Subtype /Type1\n'
+    b'/BaseFont /Times-Roman\n'
+    b'>>\n>>\n>>\n      '
+    b'/Contents 4 0 R\n  >>\nendobj\n\n4 0 obj\n  '
+    b'<< /Length 55 >>\nstream\n  BT\n    /F1 18 Tf\n    0 0 Td\n    '
+    b'(Hello World) Tj\n  ET\nendstream\nendobj\n\nxref\n0 5\n0000000000 '
+    b'65535 f \n0000000018 00000 n \n0000000077 00000 n \n0000000178 00000 n '
+    b'\n0000000457 00000 n \ntrailer\n  <<  /Root 1 0 R\n      /Size 5\n  '
+    b'>>\nstartxref\n565\n%%EOF\n'
+)
+
+
+def convert_mock_request(*args, **kwargs):
+  del args, kwargs
+
+  class Result:
+    def json(self):
+      return {
+          'Files': [
+              {
+                  'FileData': base64.b64encode(pdf_bytes).decode()
+              }
+          ]
+      }
+  return Result()
+
+
 class DocxTest(unittest.TestCase):
 
-  def test_content(self):
+  def test_from_bytes(self):
     content = ms_office_lib.Docx.from_bytes(docx_bytes)
-    self.assertEqual(
+    self.assertIn(
         content.mime_type,
-        'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+        (
+            'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+            'application/octet-stream',
+        ),
     )
     self.assertEqual(content.to_bytes(), docx_bytes)
+    self.assertTrue(content.is_compatible('text/plain'))
+    self.assertFalse(content.is_compatible('application/pdf'))
+    self.assertEqual(
+        content.make_compatible(['image/png', 'text/plain']).mime_type,
+        'text/plain'
+    )
 
-  def test_file(self):
+  def test_from_uri(self):
     content = ms_office_lib.Docx.from_uri('http://mock/web/a.docx')
     with mock.patch('requests.get') as mock_requests_get:
       mock_requests_get.side_effect = docx_mock_request
-      self.assertEqual(
+      self.assertIn(
           content.mime_type,
-          'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+          (
+              'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+              'application/octet-stream',
+          ),
       )
       self.assertEqual(content.to_bytes(), docx_bytes)
       self.assertEqual(content.to_xml(), expected_docx_xml)
@@ -267,21 +318,33 @@ class DocxTest(unittest.TestCase):
 
 class XlsxTest(unittest.TestCase):
 
-  def test_content(self):
+  def test_from_bytes(self):
     content = ms_office_lib.Xlsx.from_bytes(xlsx_bytes)
-    self.assertEqual(
+    self.assertIn(
         content.mime_type,
-        'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+        (
+            'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+            'application/octet-stream',
+        ),
     )
     self.assertEqual(content.to_bytes(), xlsx_bytes)
+    self.assertTrue(content.is_compatible('text/plain'))
+    self.assertFalse(content.is_compatible('application/pdf'))
+    self.assertEqual(
+        content.make_compatible('text/plain').mime_type,
+        'text/html'
+    )
 
-  def test_file(self):
+  def test_from_uri(self):
     content = ms_office_lib.Xlsx.from_uri('http://mock/web/a.xlsx')
     with mock.patch('requests.get') as mock_requests_get:
       mock_requests_get.side_effect = xlsx_mock_request
-      self.assertEqual(
+      self.assertIn(
           content.mime_type,
-          'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+          (
+              'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+              'application/octet-stream',
+          ),
       )
       self.assertEqual(content.to_bytes(), xlsx_bytes)
       self.assertEqual(content.to_html(), expected_xlsx_html)
@@ -291,22 +354,36 @@ class PptxTest(unittest.TestCase):
 
   def test_content(self):
     content = ms_office_lib.Pptx.from_bytes(pptx_bytes)
-    self.assertEqual(
+    self.assertIn(
         content.mime_type,
-        'application/vnd.openxmlformats-officedocument.presentationml.presentation',
+        (
+            'application/vnd.openxmlformats-officedocument.presentationml.presentation',
+            'application/octet-stream',
+        ),
     )
     self.assertEqual(content.to_bytes(), pptx_bytes)
 
   def test_file(self):
     content = ms_office_lib.Pptx.from_uri('http://mock/web/a.pptx')
+    self.assertFalse(content.is_compatible('text/plain'))
+    self.assertTrue(content.is_compatible('application/pdf'))
     with mock.patch('requests.get') as mock_requests_get:
       mock_requests_get.side_effect = pptx_mock_request
-      self.assertEqual(
+      self.assertIn(
           content.mime_type,
-          'application/vnd.openxmlformats-officedocument.presentationml.presentation',
+          (
+              'application/vnd.openxmlformats-officedocument.presentationml.presentation',
+              'application/octet-stream',
+          ),
       )
       self.assertEqual(content.to_bytes(), pptx_bytes)
 
+    with mock.patch('requests.post') as mock_requests_post:
+      mock_requests_post.side_effect = convert_mock_request
+      self.assertIsInstance(
+          content.make_compatible('application/pdf'), pdf_lib.PDF
+      )
+
 
 if __name__ == '__main__':
   unittest.main()

langfun/core/modalities/pdf.py

@@ -16,7 +16,7 @@
 from langfun.core.modalities import mime
 
 
-class PDF(mime.MimeType):
+class PDF(mime.Mime):
   """PDF document."""
 
   MIME_PREFIX = 'application/pdf'

langfun/core/modalities/video.py

@@ -17,7 +17,7 @@ import functools
 from langfun.core.modalities import mime
 
 
-class Video(mime.MimeType):
+class Video(mime.Mime):
   """Video."""
 
   MIME_PREFIX = 'video'

langfun/core/modality.py

@@ -108,3 +108,7 @@ class ModalityRef(pg.Object, pg.typing.CustomTyping):
         return ModalityRef(name=value.sym_path + k)
       return v
     return value.clone().rebind(_placehold, raise_on_no_change=False)
+
+
+class ModalityError(RuntimeError):  # pylint: disable=g-bad-exception-name
+  """Exception raised when modality is not supported."""

{langfun-0.0.2.dev20240531.dist-info → langfun-0.0.2.dev20240601.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: langfun
-Version: 0.0.2.dev20240531
+Version: 0.0.2.dev20240601
 Summary: Langfun: Language as Functions.
 Home-page: https://github.com/google/langfun
 Author: Langfun Authors
@@ -25,13 +25,14 @@ Requires-Dist: google-cloud-aiplatform >=1.5.0
 Requires-Dist: google-generativeai >=0.3.2
 Requires-Dist: jinja2 >=3.1.2
 Requires-Dist: openai ==0.27.2
+Requires-Dist: openpyxl >=3.1.0
+Requires-Dist: pandas >=2.1.4
 Requires-Dist: pyglove >=0.4.5.dev20240423
+Requires-Dist: python-docx >=0.8.11
 Requires-Dist: python-magic >=0.4.27
 Requires-Dist: requests >=2.31.0
 Requires-Dist: termcolor ==1.1.0
 Requires-Dist: tqdm >=4.64.1
-Requires-Dist: python-docx >=0.8.11
-Requires-Dist: pandas >=2.1.4
 
 <div align="center">
 <img src="https://raw.githubusercontent.com/google/langfun/main/docs/_static/logo.svg" width="520px" alt="logo"></img>

{langfun-0.0.2.dev20240531.dist-info → langfun-0.0.2.dev20240601.dist-info}/RECORD

@@ -1,5 +1,5 @@
-langfun/__init__.py,sha256=5RIeFKQLwyCk36n-NfZgbbfgdPLKI0ucy2O7b0B-384,2192
-langfun/core/__init__.py,sha256=6QEuXOZ9BXxm6TjpaMXuLwUBTYO3pkFDqn9QVBXyyPQ,4248
+langfun/__init__.py,sha256=LFsDp22pTeJHmzzKEg2OLmSVOPAym00DyF38LmrL2n4,2263
+langfun/core/__init__.py,sha256=nFJx6X7oB7IIWsAQqjbgZ_ScH-gsKg53YgAkuDvY0cw,4296
 langfun/core/component.py,sha256=oxesbC0BoE_TbtxwW5x-BAZWxZyyJbuPiX5S38RqCv0,9909
 langfun/core/component_test.py,sha256=uR-_Sz_42Jxc5qzLIB-f5_pXmNwnC01Xlbv5NOQSeSU,8021
 langfun/core/concurrent.py,sha256=TRc49pJ3HQro2kb5FtcWkHjhBm8UcgE8RJybU5cU3-0,24537
@@ -13,7 +13,7 @@ langfun/core/language_model_test.py,sha256=NZaSUls6cZdtxiqkqumWbtkx9zgNiJlsviYZO
 langfun/core/memory.py,sha256=f-asN1F7Vehgdn_fK84v73GrEUOxRtaW934keutTKjk,2416
 langfun/core/message.py,sha256=Rw3yC9HyGRjMhfDgyNjGlSCALEyDDbJ0_o6qTXeeDiQ,15738
 langfun/core/message_test.py,sha256=b6DDRoQ5j3uK-dc0QPSLelNTKaXX10MxJrRiI61iGX4,9574
-langfun/core/modality.py,sha256=-BZDYf5d4bmZnhZyS4QVGTSwvU7Xgs_55IOzeRmyacE,3378
+langfun/core/modality.py,sha256=Tla4t86DUYHpbZ2G7dy1r19fTj_Ga5XOvlYp6lbWa-Q,3512
 langfun/core/modality_test.py,sha256=HyZ5xONKQ0Fw18SzoWAq-Ob9njOXIIjBo1hNtw-rudw,2400
 langfun/core/natural_language.py,sha256=3ynSnaYQnjE60LIPK5fyMgdIjubnPYZwzGq4rWPeloE,1177
 langfun/core/natural_language_test.py,sha256=LHGU_1ytbkGuSZQFIFP7vP3dBlcY4-A12fT6dbjUA0E,1424
@@ -53,16 +53,16 @@ langfun/core/llms/anthropic.py,sha256=7W9YdPN3SlAFhAIQlihMkrpo7tTY_4NvD0KIlCrqcs
 langfun/core/llms/anthropic_test.py,sha256=TMM30myyEhwF99Le4RvJEXOn8RYl0q1FRkt9Q9nl1jk,5540
 langfun/core/llms/fake.py,sha256=Dd7-6ka9pFf3fcWZyczamjOqQ91MOI-m7We3Oc9Ffmo,2927
 langfun/core/llms/fake_test.py,sha256=ipKfdOcuqVcJ8lDXVpnBVb9HHG0hAVkFkMoHpWjC2cI,7212
-langfun/core/llms/google_genai.py,sha256=H1GdarpoMb9RjQz7a4BqVF6loQf3S_mMv8G8TFYrCvw,8999
-langfun/core/llms/google_genai_test.py,sha256=VT_MMmyxHMe4sl4uK_UZzWyxKFFMlF3xc3v6SljJQE0,7529
+langfun/core/llms/google_genai.py,sha256=Rl5a5CyF_6Y0BYYArKk8yMaenv1rH3MUQLy6b3dfMRI,10202
+langfun/core/llms/google_genai_test.py,sha256=iTISk3tJ4-3gjWmzcKQhEbH3ke4AkEiCu8rAGtB7SvU,7535
 langfun/core/llms/groq.py,sha256=NaGItVL_pkOpqPpI4bPGU27xLFRoaeizZ49v2s-4ERs,7844
 langfun/core/llms/groq_test.py,sha256=M6GtlrsOvDun_j-sR8cPh4W_moHWZNSTiThu3kuwbbc,5281
 langfun/core/llms/llama_cpp.py,sha256=Y_KkMUf3Xfac49koMUtUslKl3h-HWp3-ntq7Jaa3bdo,2385
 langfun/core/llms/llama_cpp_test.py,sha256=ZxC6defGd_HX9SFRU9U4cJiQnBKundbOrchbXuC1Z2M,1683
 langfun/core/llms/openai.py,sha256=IN46gIqfY6aEEfxCPNmyH1hrep3oWBhJDwVFilfqNkM,13657
 langfun/core/llms/openai_test.py,sha256=QWDzTgi8F2Z9u9ip6alK4rDEp_YraVTxWlDX5XOsKJk,14858
-langfun/core/llms/vertexai.py,sha256=rrwHRtox-gayVBjrkR_lnko98b0iFIyxsRUPgB_09T8,9921
-langfun/core/llms/vertexai_test.py,sha256=PbkUTVYgbFhg5lDd3HgBJM0kr-OLVz10iph2C-SJblk,7645
+langfun/core/llms/vertexai.py,sha256=eILbXoMSza5r4FLGlIdH6-eD8Ggy9Z4PdjLaBDxy29A,11162
+langfun/core/llms/vertexai_test.py,sha256=G18BG36h5KvmX2zutDTLjtYCRjTuP_nWIFm4FMnLnyY,7651
 langfun/core/llms/cache/__init__.py,sha256=QAo3InUMDM_YpteNnVCSejI4zOsnjSMWKJKzkb3VY64,993
 langfun/core/llms/cache/base.py,sha256=cFfYvOIUae842pncqCAsRvqXCk2AnAsRYVx0mcIoAeY,3338
 langfun/core/llms/cache/in_memory.py,sha256=YfFyJEhLs73cUiB0ZfhMxYpdE8Iuxxw-dvMFwGHTSHw,4742
@@ -70,18 +70,18 @@ langfun/core/llms/cache/in_memory_test.py,sha256=D-n26h__rVXQO51WRFhRfq5sw1oifRL
 langfun/core/memories/__init__.py,sha256=HpghfZ-w1NQqzJXBx8Lz0daRhB2rcy2r9Xm491SBhC4,773
 langfun/core/memories/conversation_history.py,sha256=c9amD8hCxGFiZuVAzkP0dOMWSp8L90uvwkOejjuBqO0,1835
 langfun/core/memories/conversation_history_test.py,sha256=AaW8aNoFjxNusanwJDV0r3384Mg0eAweGmPx5DIkM0Y,2052
-langfun/core/modalities/__init__.py,sha256=0RYYHcjD-s-QHGuW0rkjh9F3c0qMDKtH9ZIQsMcKPXE,1273
-langfun/core/modalities/audio.py,sha256=Fbdt9iJOLwtMetfoI9hNjzAMrWfbGE7T3W7vKgL9B0o,956
+langfun/core/modalities/__init__.py,sha256=F8P72IwFiTpEseTR2tYEJyQMlDW7fd9csvGJquLKJNg,1269
+langfun/core/modalities/audio.py,sha256=Qxo7bYjLKQ1gVJVomr9RqR2SvxY826QgXhTzzk437Sk,952
 langfun/core/modalities/audio_test.py,sha256=gWCB9h3FyrdGqro3ajBXqkw0lU0W1sBjOOq6wZbl7Fg,2027
-langfun/core/modalities/image.py,sha256=hRapD4jTdzzQ23zz44K1HMaDpzXVEhrDNKgTXs7cy7k,930
-langfun/core/modalities/image_test.py,sha256=zyx1eyAdcWK3iwk4q4zgPPmkvKOH-1Los3orgQE9aqk,2295
-langfun/core/modalities/mime.py,sha256=W-rsMyCNeVdurEmnX2aogRrlsM4aCZ6hlXK1kTCoGQE,3056
-langfun/core/modalities/mime_test.py,sha256=pN6EHCf9qKMTZaVtQS7M8HWJcH68B3htiWvbq_tLh5I,1881
-langfun/core/modalities/ms_office.py,sha256=m_-PMuY0iB52POkp_0pRztaT4-O57ugzGNdI14APJEk,2384
-langfun/core/modalities/ms_office_test.py,sha256=D8ZGR6jy3wz7KmrCoB8FtsZ_Faknpws1TRxrZWIgp7o,85434
-langfun/core/modalities/pdf.py,sha256=A-lVHiXdELpbciedSVyhIgQVi1kI1QlE-YBjJFFh3oU,731
+langfun/core/modalities/image.py,sha256=qi7B9uYLxBoKvMzApdOQNpVcp_dKaRwLzeshg2nvo9k,926
+langfun/core/modalities/image_test.py,sha256=qU7G4ucUihIQ9ZB453FsUfcOipUYx5TnnuoMB1GIMfE,3034
+langfun/core/modalities/mime.py,sha256=yMpbBAhf7MmEPJm9qj7tTn7_XionZQ4XkgTT8StA7io,5836
+langfun/core/modalities/mime_test.py,sha256=ruEro7Joima2r-zOuQfO0NzBvmaweSQ6F6jsf-w4Bns,2468
+langfun/core/modalities/ms_office.py,sha256=jOidMSdWCaV9RILpGz8VJkpTSpHJNoirD53jzQvcytM,3388
+langfun/core/modalities/ms_office_test.py,sha256=d_NZ0QU23NydenYZgNj6YxgO5ZYzjg-HCbglsVJGp04,87866
+langfun/core/modalities/pdf.py,sha256=mfaeCbUA4JslFVTARiJh8hW7imvL4tLVw9gUhO5bAZA,727
 langfun/core/modalities/pdf_test.py,sha256=KE40zJD3Whe6ty2OULkp1J8jwLmB4ZjGXlGekluTP48,1952
-langfun/core/modalities/video.py,sha256=6qADCwwv-pEtzxMs_1YvhWgX6NqTsjviQv6IZxQPDTY,959
+langfun/core/modalities/video.py,sha256=sKcXxbx9S1ERjH8yEzkbtySpcRJD40QiPIQiIBy-U5I,955
 langfun/core/modalities/video_test.py,sha256=GbsoefSeO7y8kCYhTtp4s9E3ah_eYrb6Z-MXpS01RFc,2046
 langfun/core/structured/__init__.py,sha256=yp60yeDSVlyT0ElmLwbpBHnQtk_JX5udnjG1UGcsXKA,3776
 langfun/core/structured/completion.py,sha256=RzWdHyaqKj-tj6mGwpHXk0s8YbM0UEHSpyT2axmj-o8,7343
@@ -111,8 +111,8 @@ langfun/core/templates/demonstration.py,sha256=vCrgYubdZM5Umqcgp8NUVGXgr4P_c-fik
 langfun/core/templates/demonstration_test.py,sha256=SafcDQ0WgI7pw05EmPI2S4v1t3ABKzup8jReCljHeK4,2162
 langfun/core/templates/selfplay.py,sha256=yhgrJbiYwq47TgzThmHrDQTF4nDrTI09CWGhuQPNv-s,2273
 langfun/core/templates/selfplay_test.py,sha256=DYVrkk7uNKCqJGEHH31HssU2BPuMItU1vJLzfcXIlYg,2156
-langfun-0.0.2.dev20240531.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-langfun-0.0.2.dev20240531.dist-info/METADATA,sha256=HoYUBYL46dYzSPalF8klTh42cE0-Mg7J2OGJRkokxYM,3518
-langfun-0.0.2.dev20240531.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-langfun-0.0.2.dev20240531.dist-info/top_level.txt,sha256=RhlEkHxs1qtzmmtWSwYoLVJAc1YrbPtxQ52uh8Z9VvY,8
-langfun-0.0.2.dev20240531.dist-info/RECORD,,
+langfun-0.0.2.dev20240601.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+langfun-0.0.2.dev20240601.dist-info/METADATA,sha256=V6qAAPX1gt2gDEInFQKJIvYe48IbEztRw5qwpgq_QH0,3550
+langfun-0.0.2.dev20240601.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+langfun-0.0.2.dev20240601.dist-info/top_level.txt,sha256=RhlEkHxs1qtzmmtWSwYoLVJAc1YrbPtxQ52uh8Z9VvY,8
+langfun-0.0.2.dev20240601.dist-info/RECORD,,