langfun 0.0.2.dev20240308__py3-none-any.whl → 0.0.2.dev20240310__py3-none-any.whl

langfun/core/llms/gemini.py

@@ -107,9 +107,9 @@ class Gemini(lf.LanguageModel):
     for lf_chunk in formatted.chunk():
       if isinstance(lf_chunk, str):
         chunk = lf_chunk
-      elif self.multimodal and isinstance(lf_chunk, lf_modalities.Image):
+      elif self.multimodal and isinstance(lf_chunk, lf_modalities.MimeType):
         chunk = genai.types.BlobDict(
-            data=lf_chunk.to_bytes(), mime_type=f'image/{lf_chunk.image_format}'
+            data=lf_chunk.to_bytes(), mime_type=lf_chunk.mime_type
         )
       else:
         raise ValueError(f'Unsupported modality: {lf_chunk!r}')

langfun/core/llms/openai.py

@@ -237,7 +237,7 @@ class OpenAI(lf.LanguageModel):
         for chunk in prompt.chunk():
           if isinstance(chunk, str):
             item = dict(type='text', text=chunk)
-          elif isinstance(chunk, lf_modalities.image.ImageFile):
+          elif isinstance(chunk, lf_modalities.Image) and chunk.uri:
             item = dict(type='image_url', image_url=chunk.uri)
           else:
             raise ValueError(f'Unsupported modality object: {chunk!r}.')

langfun/core/modalities/__init__.py

@@ -17,6 +17,8 @@
 # pylint: disable=g-bad-import-order
 # pylint: disable=g-import-not-at-top
 
+from langfun.core.modalities.mime import MimeType
+from langfun.core.modalities.mime import Custom
 from langfun.core.modalities.image import Image
 
 # pylint: enable=g-import-not-at-top

langfun/core/modalities/image.py

@@ -15,25 +15,13 @@
 
 import base64
 import imghdr
-from typing import Annotated, cast
+from typing import cast
+from langfun.core.modalities import mime
 
-import langfun.core as lf
-import requests
 
-
-class Image(lf.Modality):
+class Image(mime.MimeType):
   """Base class for image."""
 
-  @classmethod
-  def from_bytes(cls, content: bytes) -> 'Image':
-    """Creates an image from bytes."""
-    return ImageContent(content)
-
-  @classmethod
-  def from_uri(cls, uri: str) -> 'Image':
-    """Creates an image from file."""
-    return ImageFile(uri)
-
   @property
   def image_format(self) -> str:
     iformat = imghdr.what(None, self.to_bytes())
@@ -41,36 +29,12 @@ class Image(lf.Modality):
       raise ValueError(f'Unsupported image format: {iformat!r}.')
     return cast(str, iformat)
 
-
-class ImageContent(Image):
-  """Raw image content."""
-
-  content: Annotated[bytes, 'The raw content of the image.']
-
-  def to_bytes(self) -> bytes:
-    return self.content
+  @property
+  def mime_type(self) -> str:
+    return f'image/{self.image_format}'
 
   def _repr_html_(self) -> str:
+    if self.uri:
+      return f'<img src="{self.uri}">'
     image_raw = base64.b64encode(self.to_bytes()).decode()
     return f'<img src="data:image/{self.image_format};base64,{image_raw}">'
-
-
-class ImageFile(Image):
-  """A image file."""
-
-  uri: Annotated[str, 'The URI of the image. (e.g. https://..../a.jpg).']
-
-  def _on_bound(self):
-    super()._on_bound()
-    self._bytes = None
-
-  def to_bytes(self) -> bytes:
-    if self._bytes is None:
-      self._bytes = requests.get(
-          self.uri,
-          headers={'User-Agent': 'Langfun'},
-      ).content
-    return self._bytes
-
-  def _repr_html_(self) -> str:
-    return f'<img src="{self.uri}">'

langfun/core/modalities/mime.py

@@ -0,0 +1,81 @@
+# Copyright 2023 The Langfun Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""MIME type data."""
+
+import abc
+from typing import Annotated, Union
+import langfun.core as lf
+import pyglove as pg
+import requests
+
+
+class MimeType(lf.Modality):
+  """Base for MIME type data."""
+
+  @property
+  @abc.abstractmethod
+  def mime_type(self) -> str:
+    """Returns the MIME type."""
+
+  uri: Annotated[str | None, 'The URI for locating the MIME data. '] = None
+
+  content: Annotated[
+      Union[str, bytes, None], 'The raw content of the MIME type.'
+  ] = None
+
+  def _on_bound(self):
+    super()._on_bound()
+    if self.uri is None and self.content is None:
+      raise ValueError('Either uri or content must be provided.')
+
+  def to_bytes(self) -> bytes:
+    if self.content is not None:
+      return self.content
+
+    assert self.uri is not None
+    if self.uri.lower().startswith(('http:', 'https:', 'ftp:')):
+      content = requests.get(
+          self.uri,
+          headers={'User-Agent': 'Langfun'},
+      ).content
+    else:
+      content = pg.io.readfile(self.uri, mode='rb')
+    self.rebind(content=content, skip_notification=True)
+    return self.content
+
+  @classmethod
+  def from_uri(cls, uri: str, **kwargs) -> 'MimeType':
+    return cls(uri=uri, content=None, **kwargs)
+
+  @classmethod
+  def from_bytes(cls, content: bytes | str, **kwargs) -> 'MimeType':
+    return cls(content=content, **kwargs)
+
+
+@pg.use_init_args(['type', 'content', 'uri'])
+class Custom(MimeType):
+  """Custom MIME data."""
+
+  type: Annotated[
+      str, 'The MIME type of the data. E.g. text/plain, or image/png. '
+  ]
+
+  @property
+  def mime_type(self) -> str:
+    return self.type
+
+
+class Pdf(Custom):
+  """PDF document."""
+  type = 'application/pdf'

langfun/core/modalities/mime_test.py

@@ -0,0 +1,59 @@
+# Copyright 2023 The Langfun Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""MIME tests."""
+import unittest
+from unittest import mock
+
+from langfun.core.modalities import mime
+import pyglove as pg
+
+
+def mock_request(*args, **kwargs):
+  del args, kwargs
+  return pg.Dict(content='foo')
+
+
+def mock_readfile(*args, **kwargs):
+  del args, kwargs
+  return 'bar'
+
+
+class CustomMimeTest(unittest.TestCase):
+
+  def test_content(self):
+    content = mime.Custom('text/plain', 'foo')
+    self.assertEqual(content.to_bytes(), 'foo')
+    self.assertEqual(content.mime_type, 'text/plain')
+
+    with self.assertRaisesRegex(
+        ValueError, 'Either uri or content must be provided.'
+    ):
+      mime.Custom('text/plain')
+
+  def test_from_uri(self):
+    content = mime.Custom.from_uri('http://mock/web/a.txt', type='text/plain')
+    with mock.patch('requests.get') as mock_requests_stub:
+      mock_requests_stub.side_effect = mock_request
+      self.assertEqual(content.to_bytes(), 'foo')
+      self.assertEqual(content.mime_type, 'text/plain')
+
+    content = mime.Custom.from_uri('a.txt', type='text/plain')
+    with mock.patch('pyglove.io.readfile') as mock_readfile_stub:
+      mock_readfile_stub.side_effect = mock_readfile
+      self.assertEqual(content.to_bytes(), 'bar')
+      self.assertEqual(content.mime_type, 'text/plain')
+
+
+if __name__ == '__main__':
+  unittest.main()

{langfun-0.0.2.dev20240308.dist-info → langfun-0.0.2.dev20240310.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: langfun
-Version: 0.0.2.dev20240308
+Version: 0.0.2.dev20240310
 Summary: Langfun: Language as Functions.
 Home-page: https://github.com/google/langfun
 Author: Langfun Authors

{langfun-0.0.2.dev20240308.dist-info → langfun-0.0.2.dev20240310.dist-info}/RECORD

@@ -49,11 +49,11 @@ langfun/core/eval/scoring_test.py,sha256=_L_B40VZkyI2_PJce-jVKYC4llrO4jGUR5j86Gu
 langfun/core/llms/__init__.py,sha256=T4mgT091BLA4mHrOjAvEGhZPHf0tiYgqD88l_JTp1dQ,2386
 langfun/core/llms/fake.py,sha256=dVzOrW27RZ1p3DdQoRCRZs_vfoQcTcNrlWxia7oqmvw,2499
 langfun/core/llms/fake_test.py,sha256=Qk_Yoi4Z7P9o6f8Q_BZkaSlvxH89ZVsDxnVIbSBRBXk,3555
-langfun/core/llms/gemini.py,sha256=p3d4Cl2uET-os1n_V3YNE6-6cYrZjndj7lxZIk2E8_4,5688
+langfun/core/llms/gemini.py,sha256=9HxrTvac_dMbDytNCEo6YcYqs8vsePtScfI_EygpI5Y,5677
 langfun/core/llms/gemini_test.py,sha256=ybNNCn3JW3hYpMe0wT5ILGDrMPaYYU8PN2kSookM0jk,5433
 langfun/core/llms/llama_cpp.py,sha256=EIjJa1-Tg4_VaIxVR88oDWSWc_axc1r2KwSPpl4PSp0,2549
 langfun/core/llms/llama_cpp_test.py,sha256=ZxC6defGd_HX9SFRU9U4cJiQnBKundbOrchbXuC1Z2M,1683
-langfun/core/llms/openai.py,sha256=ufFz1Q2bHVukyAck0dZ5MYZ8IKcVkYzDjQn3YXugLCQ,11711
+langfun/core/llms/openai.py,sha256=-PnJ8OICuPTzr-unIys4HftNVZ6seAhV5nXny4OfVYc,11715
 langfun/core/llms/openai_test.py,sha256=yfw7A-4Zo9u1cIkAMk39evE-tO7z6isNYTXiSnJXDQw,7599
 langfun/core/llms/cache/__init__.py,sha256=QAo3InUMDM_YpteNnVCSejI4zOsnjSMWKJKzkb3VY64,993
 langfun/core/llms/cache/base.py,sha256=cFfYvOIUae842pncqCAsRvqXCk2AnAsRYVx0mcIoAeY,3338
@@ -62,9 +62,11 @@ langfun/core/llms/cache/in_memory_test.py,sha256=WYLg_SlUdkUxIdBYnbksMqwVLFuzcNL
 langfun/core/memories/__init__.py,sha256=HpghfZ-w1NQqzJXBx8Lz0daRhB2rcy2r9Xm491SBhC4,773
 langfun/core/memories/conversation_history.py,sha256=c9amD8hCxGFiZuVAzkP0dOMWSp8L90uvwkOejjuBqO0,1835
 langfun/core/memories/conversation_history_test.py,sha256=AaW8aNoFjxNusanwJDV0r3384Mg0eAweGmPx5DIkM0Y,2052
-langfun/core/modalities/__init__.py,sha256=VI96XGNfXqcJpBh2c17tkTs0gpO5ftc77Ep0jfLOztw,882
-langfun/core/modalities/image.py,sha256=HU0sV4ZTwRnAwQthmdWZwhFZRD86RyvqoS8JUW2Ia-A,2065
+langfun/core/modalities/__init__.py,sha256=mK82mgydGHsQLEfMRO4J-dlFzPVIrB2KPIA3OVUL6GQ,980
+langfun/core/modalities/image.py,sha256=jYkVRoD2yLgh1ywJwbeHjHompvy3RbHIFb_xYgKzbsc,1297
 langfun/core/modalities/image_test.py,sha256=YxDRvC49Bjwyyndd_P7y6XjyS7dOft0Zewwxk-7q4kE,2301
+langfun/core/modalities/mime.py,sha256=mMjDufv2HpnYrLUD05LqhcuinIW8yp0xRSa8TmBW9ow,2238
+langfun/core/modalities/mime_test.py,sha256=cVHxRvJ1QXC1SVhBmWkJdWGpL9Xl0UNfTQq6j0OGGL4,1881
 langfun/core/structured/__init__.py,sha256=LZ5BFLX6VXy1zH17yChWCdg8bvIDrhtL2lqtSCwtZ-M,3187
 langfun/core/structured/completion.py,sha256=skBxt6V_fv2TBUKnzFgnPMbVY8HSYn8sY04MLok2yvs,7299
 langfun/core/structured/completion_test.py,sha256=98UCgA4gzfp6H6HgP2s2kcKs25YH3k4Nxj1rgAvmVBw,19249
@@ -89,8 +91,8 @@ langfun/core/templates/demonstration.py,sha256=vCrgYubdZM5Umqcgp8NUVGXgr4P_c-fik
 langfun/core/templates/demonstration_test.py,sha256=SafcDQ0WgI7pw05EmPI2S4v1t3ABKzup8jReCljHeK4,2162
 langfun/core/templates/selfplay.py,sha256=yhgrJbiYwq47TgzThmHrDQTF4nDrTI09CWGhuQPNv-s,2273
 langfun/core/templates/selfplay_test.py,sha256=IB5rWbjK_9CTkqEo1BclQPzFAKcIiusJckH8J19HFgI,2096
-langfun-0.0.2.dev20240308.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-langfun-0.0.2.dev20240308.dist-info/METADATA,sha256=hzyG6mL6i6mEGWa7q9eDovYRVCH_UmjAA3wlFo79oII,3368
-langfun-0.0.2.dev20240308.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-langfun-0.0.2.dev20240308.dist-info/top_level.txt,sha256=RhlEkHxs1qtzmmtWSwYoLVJAc1YrbPtxQ52uh8Z9VvY,8
-langfun-0.0.2.dev20240308.dist-info/RECORD,,
+langfun-0.0.2.dev20240310.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+langfun-0.0.2.dev20240310.dist-info/METADATA,sha256=TOSROdxEVxAK4gY073Yz2QNk6oskWwfXRO_0_0-Qv5I,3368
+langfun-0.0.2.dev20240310.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+langfun-0.0.2.dev20240310.dist-info/top_level.txt,sha256=RhlEkHxs1qtzmmtWSwYoLVJAc1YrbPtxQ52uh8Z9VvY,8
+langfun-0.0.2.dev20240310.dist-info/RECORD,,