langfun 0.1.2.dev202511030805__py3-none-any.whl → 0.1.2.dev202511050805__py3-none-any.whl

langfun/core/message.py

@@ -32,15 +32,49 @@ class Message(
     pg.Object,
     pg.views.HtmlTreeView.Extension
 ):
-  """Message.
+  """Message between users, LLMs and tools.
 
-  ``Message`` is the protocol for users and the system to interact with
-  LLMs. It consists of a text in the form of natural language, 
-  an identifier of the sender, and a dictionary of Python values as structured
-  meta-data.
+  `lf.Message` is the fundamental unit of communication in Langfun. It
+  standardizes interactions with LLMs by encapsulating not only text but also
+  multi-modal content, as well as the sender's role and structured metadata.
 
-  The subclasses of ``Message`` represent messages sent from different roles.
-  Agents may use the roles to decide the orchastration logic.
+  **Key Components:**
+
+  *   **`text`**: The natural language content of the message.
+  *   **`sender`**: An identifier for the message originator (e.g., 'User',
+    'AI', 'System').
+  *   **`metadata`**: A dictionary for structured data, such as tool inputs/
+    outputs, scores, or other contextual information.
+  *   **`referred_modalities`**: A dictionary of modality objects (e.g.,
+    `lf.Image`, `lf.Audio`) referenced within the message text via placeholders
+    like `<<[[image_id]]>>`.
+
+  Subclasses like `lf.UserMessage`, `lf.AIMessage`, and `lf.ToolMessage`
+  represent messages from specific roles, enabling more complex conversational
+  flows and agentic behaviors.
+
+  **Example:**
+
+  ```python
+  import langfun as lf
+
+  # Creating a user message with an image
+  image = lf.Image.from_path('/path/to/image.png')
+  user_message = lf.UserMessage(
+      f'What is in this image <<[[{image.id}]]>>?',
+      referred_modalities=[image])
+
+  # Creating an AI message with structured results
+  ai_message = lf.AIMessage(
+      'It is a cat.',
+      metadata=dict(result=dict(label='cat', confidence=0.9)))
+
+  print(user_message.chunk())
+  # Output: ['What is in this image', <lf.Image object>, '?']
+
+  print(ai_message.result)
+  # Output: {'label': 'cat', 'confidence': 0.9}
+  ```
   """
 
   #
@@ -239,6 +273,11 @@ class Message(
     """
     return MessageConverter.get(format_or_type, **kwargs).to_value(self)
 
+  @classmethod
+  def is_convertible(cls, format_or_type: str | Type[Any]) -> bool:
+    """Returns True if the value can be converted to a message."""
+    return MessageConverter.is_convertible(format_or_type)
+
   @classmethod
   def convertible_formats(cls) -> list[str]:
     """Returns supported format for message conversion."""
@@ -412,7 +451,7 @@ class Message(
       var_name: str,
       default: Any = None
   ) -> modality.Modality | None:
-    """Gets the modality object referred in the message.
+    """Returns modality object referred in the message by its variable name.
 
     Args:
       var_name: The referred variable name for the modality object.
@@ -424,7 +463,14 @@ class Message(
     return self.referred_modalities.get(var_name, default)
 
   def chunk(self, text: str | None = None) -> list[str | modality.Modality]:
-    """Chunk a message into a list of str or modality objects."""
+    """Chunks message into a list of text and modality chunks.
+
+    Args:
+      text: The text to chunk. If None, use `self.text`.
+
+    Returns:
+      A list of text and modality chunks.
+    """
     chunks = []
 
     def add_text_chunk(text_piece: str) -> None:
@@ -464,7 +510,7 @@ class Message(
   def from_chunks(
       cls, chunks: list[str | modality.Modality], separator: str = ' '
   ) -> 'Message':
-    """Assembly a message from a list of string or modality objects."""
+    """Assembles a message from a list of string or modality objects."""
     fused_text = io.StringIO()
     metadata = dict()
     referred_modalities = dict()
@@ -554,7 +600,7 @@ class Message(
     return self.trace(Message.TAG_LM_OUTPUT)
 
   def last(self, tag: str) -> Optional['Message']:
-    """Return the last message wih certain tag."""
+    """Returns the last message with a given tag."""
     current = self
     while current is not None:
       if tag in current.tags:
@@ -908,6 +954,12 @@ class _MessageConverterRegistry:
     if converter.OUTPUT_TYPE is not None:
       self._type_to_converters[converter.OUTPUT_TYPE].append(converter)
 
+  def unregister(self, converter: Type['MessageConverter']) -> None:
+    """Unregisters a message converter."""
+    self._name_to_converter.pop(converter.FORMAT_ID, None)
+    if converter.OUTPUT_TYPE is not None:
+      self._type_to_converters[converter.OUTPUT_TYPE].remove(converter)
+
   def get_by_type(self, t: Type[Any], **kwargs) -> 'MessageConverter':
     """Returns a message converter for the given type."""
     t = self._type_to_converters[t]
@@ -938,6 +990,13 @@ class _MessageConverterRegistry:
     assert isinstance(format_or_type, type), format_or_type
     return self.get_by_type(format_or_type, **kwargs)
 
+  def is_convertible(self, format_or_type: str | Type[Any]) -> bool:
+    """Returns whether the message is convertible to the given format or type."""
+    if isinstance(format_or_type, str):
+      return format_or_type in self._name_to_converter
+    assert isinstance(format_or_type, type), format_or_type
+    return bool(self._type_to_converters.get(format_or_type))
+
   def convertible_formats(self) -> list[str]:
     """Returns a list of converter names."""
     return sorted(list(self._name_to_converter.keys()))
@@ -1029,6 +1088,11 @@ class MessageConverter(pg.Object):
     """Returns a message converter for the given type."""
     return cls._REGISTRY.get_by_type(t, **kwargs)
 
+  @classmethod
+  def is_convertible(cls, format_or_type: str | Type[Any]) -> bool:
+    """Returns whether the message is convertible to the given format or type."""
+    return cls._REGISTRY.is_convertible(format_or_type)
+
   @classmethod
   def convertible_formats(cls) -> list[str]:
     """Returns a list of converter names."""

langfun/core/message_test.py

@@ -500,6 +500,12 @@ class MessageConverterTest(unittest.TestCase):
     self.assertIn('test_format2', message.Message.convertible_formats())
     self.assertIn('test_format3', message.Message.convertible_formats())
 
+    self.assertTrue(message.Message.is_convertible(int))
+    self.assertFalse(message.Message.is_convertible(dict))
+    self.assertTrue(message.Message.is_convertible('test_format1'))
+    self.assertTrue(message.Message.is_convertible('test_format2'))
+    self.assertTrue(message.Message.is_convertible('test_format3'))
+    self.assertFalse(message.Message.is_convertible('test_format4'))
     self.assertIn(int, message.Message.convertible_types())
     self.assertIn(tuple, message.Message.convertible_types())
     self.assertEqual(
@@ -540,6 +546,9 @@ class MessageConverterTest(unittest.TestCase):
         message.Message.from_value((1, 2, 3)),
         message.UserMessage('1,2,3')
     )
+    message.MessageConverter._REGISTRY.unregister(TestConverter)
+    message.MessageConverter._REGISTRY.unregister(TestConverter2)
+    message.MessageConverter._REGISTRY.unregister(TestConverter3)
 
   def test_get_role(self):
     self.assertEqual(

langfun/core/modalities/audio.py

@@ -18,7 +18,27 @@ from langfun.core.modalities import mime
 
 
 class Audio(mime.Mime):
-  """Audio."""
+  """Represents audio for communicating with language models.
+
+  `lf.Audio` can be initialized from a URI (HTTP/HTTPS URL or local path)
+  using `lf.Audio.from_uri()` or from raw bytes using `lf.Audio.from_bytes()`.
+
+  **Example:**
+
+  ```python
+  import langfun as lf
+
+  # Load audio from path
+  audio = lf.Audio.from_path('/path/to/audio.mp3')
+
+  # Use audio in a prompt
+  prompt = lf.Template(
+      'What is being said in this audio? {{audio}}', audio=audio
+  )
+  response = lf.query(prompt, lm=lf.llms.Gemini25Flash())
+  print(response)
+  ```
+  """
 
   MIME_PREFIX = 'audio'
 

langfun/core/modalities/image.py

@@ -33,7 +33,25 @@ except ImportError:
 
 
 class Image(mime.Mime):
-  """Image."""
+  """Represents an image for communicating with language models.
+
+  `lf.Image` can be initialized from a URI (HTTP/HTTPS URL or local path)
+  using `lf.Image.from_uri()` or from raw bytes using `lf.Image.from_bytes()`.
+
+  **Example:**
+
+  ```python
+  import langfun as lf
+
+  # Load image from path
+  image = lf.Image.from_path('/path/to/image.png')
+
+  # Use image in a prompt
+  prompt = lf.Template('Describe this image: {{image}}', image=image)
+  response = lf.query(prompt, lm=lf.llms.Gemini25Flash())
+  print(response)
+  ```
+  """
 
   MIME_PREFIX = 'image'
 

langfun/core/modalities/mime.py

@@ -37,7 +37,33 @@ def _detect_mime_type(content: bytes) -> str:
 
 
 class Mime(lf.Modality):
-  """Base for MIME data."""
+  """Base class for representing modality data based on MIME types.
+
+  `lf.Mime` is a subclass of `lf.Modality` that serves as a base for
+  handling various data types like images, audio, video, and PDFs,
+  identified by their MIME types. It provides unified methods for
+  loading data from URIs or bytes (`.from_uri()`, `.from_bytes()`) and
+  for accessing content (`.to_bytes()`).
+
+  Subclasses like `lf.Image`, `lf.Audio`, `lf.Video`, and `lf.PDF`
+  specialize in handling specific MIME type prefixes (e.g., 'image/', 'audio/').
+
+  **Example:**
+
+  ```python
+  import langfun as lf
+
+  # Load an image from a path
+  image = lf.Image.from_path('/path/to/image.png')
+  print(image.mime_type)
+  # Output: image/png
+
+  # Create a text document
+  text = lf.Custom.from_bytes(b'hello world', mime='text/plain')
+  print(text.mime_type)
+  # Output: text/plain
+  ```
+  """
 
   # The regular expression that describes the MIME type str.
   # If None, the MIME type is dynamic. Subclass could override.
@@ -49,6 +75,10 @@ class Mime(lf.Modality):
       Union[str, bytes, None], 'The raw content of the MIME type.'
   ] = None
 
+  metadata: Annotated[
+      dict[str, Any], 'Additional metadata attached to this object.'
+  ] = {}
+
   @functools.cached_property
   def mime_type(self) -> str:
     """Returns the MIME type."""
@@ -94,7 +124,10 @@ class Mime(lf.Modality):
     # Hash the URI to avoid downloading the content.
     if self.uri is not None:
       return hashlib.md5(self.uri.encode()).hexdigest()[:8]
-    return super().hash
+    if self.content is not None:
+      return super().hash
+    assert self.metadata
+    return hashlib.md5(str(self.metadata).encode()).hexdigest()[:8]
 
   def to_text(self) -> str:
     """Returns the text content of the MIME type."""
@@ -141,7 +174,7 @@ class Mime(lf.Modality):
 
   def _on_bound(self):
     super()._on_bound()
-    if self.uri is None and self.content is None:
+    if self.uri is None and self.content is None and not self.metadata:
       raise ValueError('Either uri or content must be provided.')
 
   def to_bytes(self) -> bytes:
@@ -281,7 +314,24 @@ class Mime(lf.Modality):
 
 @pg.use_init_args(['mime', 'content', 'uri'])
 class Custom(Mime):
-  """Custom MIME data."""
+  """Represents content of a custom MIME type.
+
+  `lf.modalities.Custom` is useful for representing data with MIME types
+  that do not have dedicated classes like `lf.Image` or `lf.Audio`.
+
+  **Example:**
+
+  ```python
+  import langfun as lf
+
+  # Create a custom MIME object for plain text
+  text_data = lf.Custom.from_bytes(
+      b'This is a text document.', mime='text/plain'
+  )
+  print(text_data.mime_type)
+  # Output: text/plain
+  ```
+  """
 
   mime: Annotated[
       str, 'The MIME type of the data. E.g. text/plain, or image/png. '

langfun/core/modalities/pdf.py

@@ -17,6 +17,24 @@ from langfun.core.modalities import mime
 
 
 class PDF(mime.Mime):
-  """PDF document."""
+  """Represents a PDF document for communicating with language models.
+
+  `lf.PDF` can be initialized from a URI (HTTP/HTTPS URL or local path)
+  using `lf.PDF.from_uri()` or from raw bytes using `lf.PDF.from_bytes()`.
+
+  **Example:**
+
+  ```python
+  import langfun as lf
+
+  # Load PDF from path
+  pdf = lf.PDF.from_path('/path/to/document.pdf')
+
+  # Use PDF in a prompt
+  prompt = lf.Template('Summarize this document: {{pdf}}', pdf=pdf)
+  response = lf.query(prompt, lm=lf.llms.Gemini25Flash())
+  print(response)
+  ```
+  """
 
   MIME_PREFIX = 'application/pdf'

langfun/core/modalities/video.py

@@ -18,7 +18,27 @@ from langfun.core.modalities import mime
 
 
 class Video(mime.Mime):
-  """Video."""
+  """Represents a video for communicating with language models.
+
+  `lf.Video` can be initialized from a URI (HTTP/HTTPS URL or local path)
+  using `lf.Video.from_uri()` or from raw bytes using `lf.Video.from_bytes()`.
+
+  **Example:**
+
+  ```python
+  import langfun as lf
+
+  # Load video from path
+  video = lf.Video.from_path('/path/to/video.mp4')
+
+  # Use video in a prompt
+  prompt = lf.Template(
+      'What is happening in this video? {{video}}', video=video
+  )
+  response = lf.query(prompt, lm=lf.llms.Gemini25Flash())
+  print(response)
+  ```
+  """
 
   MIME_PREFIX = 'video'
 

langfun/core/modality.py

@@ -24,7 +24,35 @@ import pyglove as pg
 
 
 class Modality(component.Component, pg.views.HtmlTreeView.Extension):
-  """Base class for multimodal object."""
+  """Base class for representing non-text content in prompts.
+
+  `lf.Modality` is the base class for multimodal objects such as `lf.Image`,
+  `lf.Audio`, and `lf.Video`. It allows these non-text inputs to be
+  seamlessly embedded within text prompts for processing by multimodal
+  language models.
+
+  When a `Modality` object is rendered within an `lf.Template`, it is
+  replaced by a text marker (e.g., `<<[[image:b10a8db1]]>>`), and the
+  modality object itself is stored in the `referred_modalities` field of
+  the resulting `lf.Message`. This allows language models to associate
+  the placeholder with its content during processing.
+
+  **Example:**
+
+  ```python
+  import langfun as lf
+
+  image = lf.Image.from_path('/path/to/image.png')
+  prompt = lf.Template('What is in this image? {{image}}', image=image)
+
+  message = prompt.render()
+  print(message.text)
+  # Output: What is in this image? <<[[image:b10a8db1]]>>
+
+  print(message.modalities())
+  # Output: [<Image object>]
+  ```
+  """
 
   REF_START = '<<[['
   REF_END = ']]>>'
@@ -87,11 +115,44 @@ class Modality(component.Component, pg.views.HtmlTreeView.Extension):
 
 
 class ModalityRef(pg.Object, pg.typing.CustomTyping):
-  """References of modality objects in a symbolic tree.
+  """Lightweight placeholder for a `lf.Modality` object in a symbolic tree.
+
+  `ModalityRef` acts as a reference to a `Modality` object (like `lf.Image`
+  or `lf.Audio`) within a structured object hierarchy (e.g., a `pg.Object`).
+  Instead of embedding potentially large modality data directly, `ModalityRef`
+  stores only the ID of the modality object.
+
+  This is useful in scenarios where structured objects are serialized or
+  manipulated, and it's more efficient to refer to modalities by ID rather
+  than copying their content. The `lf.ModalityRef.placehold()` class method
+  can be used to replace `Modality` instances in a symbolic object with
+  `ModalityRef` placeholders, while `lf.ModalityRef.restore()` can reinstate
+  the original `Modality` objects using a lookup table.
+
+  **Example:**
+
+  ```python
+  import langfun as lf
+  import pyglove as pg
+
+  class ImagePair(pg.Object):
+    image1: lf.Image
+    image2: lf.Image
+
+  pair = ImagePair(
+      image1=lf.Image(content=b'abc'), image2=lf.Image(content=b'def')
+  )
+  modalities = lf.Modality.from_value(pair)
+
+  # Replace Image objects with ModalityRef placeholders
+  pair_with_refs = lf.ModalityRef.placehold(pair)
+  print(pair_with_refs.image1)
+  # Output: ModalityRef(id='image:d81e5a68')
 
-  `ModalityRef` was introduced to placehold modality objects in a symbolic
-  tree, to prevent message from being chunked in the middle of a Python
-  structure.
+  # Restore Image objects from ModalityRef placeholders
+  pair_restored = lf.ModalityRef.restore(pair_with_refs, modalities)
+  assert pair_restored.image1.content == b'abc'
+  ```
   """
 
   id: str

langfun/core/natural_language.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""Natural language utilities."""
+"""Natural language formatting."""
 
 import abc
 import pyglove as pg

langfun/core/sampling.py

@@ -38,10 +38,10 @@ def sweep(
         Union[message_lib.Message, BaseException, None],  # LM output.
     ],
 ]:
-  """Sweeps the input/output of this LangFunc concurrently.
+  """Sweeps the input/output of a LangFunc search space concurrently.
 
   Args:
-    lfun: An LangFunc object that contains `pg.oneof` as the search space 
+    lfun: An LangFunc object that contains `pg.oneof` as the search space
       for sampling.
     num_examples: Number of examples to sample.
     max_workers: Max number of concurrent workers to do sampling.
@@ -84,10 +84,10 @@ def random_sample(
         Union[message_lib.Message, BaseException, None],  # LM output.
     ],
 ]:
-  """Random samples the input/output of this LangFunc concurrently.
+  """Random samples the input/output of a LangFunc search space concurrently.
 
   Args:
-    lfun: An LangFunc object that contains `pg.oneof` as the search space 
+    lfun: An LangFunc object that contains `pg.oneof` as the search space
       for sampling.
     num_examples: Number of examples to sample.
     max_workers: Max number of concurrent workers to do sampling.

langfun/core/structured/completion.py

@@ -116,7 +116,7 @@ class _CompleteStructure(mapping.Mapping):
     )
 
   def postprocess_result(self, result: Any) -> Any:
-    """Postprocess result."""
+    """Postprocesses result."""
     # Try restore modality objects from the input value to output value.
     if modalities := self.modalities(self.input):
       result = lf.ModalityRef.restore(result, modalities)
@@ -151,7 +151,7 @@ class _CompleteStructure(mapping.Mapping):
   #
 
   def has_modality_refs(self, value: Any) -> bool:
-    """Returns true if the value has modalities."""
+    """Returns True if the value has modalities."""
     return not isinstance(value, lf.Modality) and pg.contains(
         value, type=lf.Modality
     )
@@ -181,41 +181,36 @@ def complete(
     returns_message: bool = False,
     **kwargs,
 ) -> Any:
-  """Complete a symbolic value by filling its missing fields.
-
-  Examples:
-
-    ```
-    class FlightDuration:
-      hours: int
-      minutes: int
-
-    class Flight(pg.Object):
-      airline: str
-      flight_number: str
-      departure_airport_code: str
-      arrival_airport_code: str
-      departure_time: str
-      arrival_time: str
-      duration: FlightDuration
-      stops: int
-      price: float
-
-    prompt = '''
-      Information about flight UA2631.
-      '''
-
-    r = lf.query(prompt, Flight)
-    assert isinstance(r, Flight)
-    assert r.airline == 'United Airlines'
-    assert r.departure_airport_code == 'SFO'
-    assert r.duration.hour = 7
-    ```
+  """Completes a symbolic value by filling its missing fields using an LLM.
+
+  `lf.complete` is used to fill in missing information in structured
+  data. It takes a partially defined `pg.Object` instance where some fields
+  are marked as `lf.MISSING`, and uses a language model to infer and
+  populate those fields based on the provided values.
+
+  **Example:**
+
+  ```python
+  import langfun as lf
+  import pyglove as pg
+
+  class Country(pg.Object):
+    name: str
+    capital: str = lf.MISSING
+    population: int = lf.MISSING
+
+  # Filling missing fields of Country(name='France')
+  country = lf.complete(Country(name='France'), lm=lf.llms.Gemini25Flash())
+  print(country)
+  # Output: Country(name='France', capital='Paris', population=67000000)
+  ```
 
   Args:
-    input_value: A symbolic value that may contain missing values.
-    default: The default value if parsing failed. If not specified, error will
-      be raised.
+    input_value: A symbolic value that may contain missing values marked
+      by `lf.MISSING`.
+    default: The default value to return if parsing fails. If
+      `lf.RAISE_IF_HAS_ERROR` is used (default), an error will be raised
+      instead.
     lm: The language model to use. If not specified, the language model from
       `lf.context` context manager will be used.
     examples: An optional list of fewshot examples for helping parsing. If None,
@@ -231,10 +226,10 @@ def complete(
     returns_message: If True, returns `lf.Message` as the output, instead of
       returning the structured `message.result`.
     **kwargs: Keyword arguments passed to the
-      `lf.structured.NaturalLanguageToStructureed` transform.
+      `lf.structured.Mapping` transform.
 
   Returns:
-    The result based on the schema.
+    The input object with missing fields completed by LLM.
   """
   t = _CompleteStructure(
       input=schema_lib.mark_missing(input_value),