langfun 0.1.2.dev202512050805__tar.gz → 0.1.2.dev202512180805__tar.gz

{langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512180805}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langfun
-Version: 0.1.2.dev202512050805
+Version: 0.1.2.dev202512180805
 Summary: Langfun: Language as Functions.
 Home-page: https://github.com/google/langfun
 Author: Langfun Authors

{langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512180805}/langfun/core/agentic/action.py

@@ -288,6 +288,8 @@ class Action(pg.Object):
 
     with session.track_action(self, max_execution_time=max_execution_time):
       try:
+        # Early terminate the action if the execution time is exceeded.
+        session.check_execution_time()
         result = self.call(session=session, **kwargs)
         self._invocation.end(result)
       except BaseException as e:
@@ -2005,9 +2007,6 @@ class Session(pg.Object, pg.views.html.HtmlTreeView.Extension):
           'signal the start and end of the session.'
       )
 
-    # Early terminate the action if the execution time is exceeded.
-    self.check_execution_time()
-
     invocation = ActionInvocation(
         pg.maybe_ref(action),
         max_execution_time=self._child_max_execution_time(max_execution_time)

{langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512180805}/langfun/core/eval/v2/__init__.py

@@ -41,6 +41,7 @@ from langfun.core.eval.v2.checkpointing import PerExampleCheckpointer
 from langfun.core.eval.v2.reporting import HtmlReporter
 from langfun.core.eval.v2.reporting import ExampleHtmlGenerator
 
+# Google-internal imports.
 
 # pylint: enable=g-bad-import-order
 # pylint: enable=g-importing-member

{langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512180805}/langfun/core/eval/v2/checkpointing.py

@@ -38,7 +38,7 @@ class Checkpointer(experiment_lib.Plugin):
   later. When an experiment starts, the checkpointer loads any previously saved
   examples from an earlier run (or a warm-start run) into `experiment.state`,
   so the runner can skip processing them again.
-  Subclasses should implement `_list_checkpoint_filenames` to identify
+  Subclasses should implement `_list_checkpoint_files` to identify
   checkpoint files to load, and `_save_example` to save a newly processed
   example.
   """
@@ -131,7 +131,7 @@ class Checkpointer(experiment_lib.Plugin):
       experiment: Experiment,
   ) -> None:
     """Creates the checkpoint file."""
-    ckpt_files = self._list_checkpoint_filenames(runner, experiment)
+    ckpt_files = self._list_checkpoint_files(runner, experiment)
     experiment.info(f'Found {len(ckpt_files)} checkpoint files to load.')
 
     # Load the checkpoint files in parallel.
@@ -141,18 +141,18 @@ class Checkpointer(experiment_lib.Plugin):
         experiment
     )
     context = dict(counter=0, counter_lock=threading.Lock())
-    copy_ckpt = current_run.input_root != current_run.output_root
 
     def _load_state(ckpt_file):
       error = None
       with pg.timeit() as t:
         try:
-          experiment.load_state(
-              current_run.input_path_for(experiment, ckpt_file),
+          loaded_examples = experiment.load_state(
+              ckpt_file,
               filter=lambda x: x.id in examples_to_load,
               load_example_metadata=lambda x: x.id in examples_to_load_metadata,
           )
         except BaseException as e:  # pylint: disable=broad-except
+          loaded_examples = []
           error = e
         finally:
           with context['counter_lock']:
@@ -170,22 +170,18 @@ class Checkpointer(experiment_lib.Plugin):
                 f'Skipping the file. ({progress_str})'
             )
 
-        if not copy_ckpt:
-          return
-
-        # Copy the checkpoint records to the output directory.
-        try:
-          with pg.io.open_sequence(
-              current_run.output_path_for(experiment, ckpt_file), 'w'
-          ) as o, pg.io.open_sequence(
-              current_run.input_path_for(experiment, ckpt_file), 'r'
-          ) as i:
-            for x in i:
-              o.add(x)
-        except BaseException as e:  # pylint: disable=broad-except
-          experiment.warning(
-              f'Failed to copy checkpoint {ckpt_file!r}: {e}.'
-          )
+        output_ckpt_file = current_run.output_path_for(
+            experiment, os.path.basename(ckpt_file)
+        )
+        if ckpt_file != output_ckpt_file and any(
+            e for e in loaded_examples if not e.has_error
+        ):
+          # Write the error-free warm-start examples to the output checkpoint
+          # file.
+          with SequenceWriter(output_ckpt_file) as writer:
+            for example in loaded_examples:
+              if not example.has_error:
+                writer.add(example)
 
     _ = list(
         lf.concurrent_map(
@@ -197,10 +193,10 @@ class Checkpointer(experiment_lib.Plugin):
     )
 
   @abc.abstractmethod
-  def _list_checkpoint_filenames(
+  def _list_checkpoint_files(
       self, runner: Runner, experiment: Experiment
   ) -> list[str]:
-    """Lists the checkpoint filenames to restore."""
+    """Lists the checkpoint file paths to restore."""
 
   @abc.abstractmethod
   def _save_example(
@@ -226,22 +222,41 @@ class PerExampleCheckpointer(Checkpointer):
     self._checkpoint_file_prefix = prefix
     self._checkpoint_file_ext = ext
 
-  def _list_checkpoint_filenames(
+  def _list_checkpoint_files(
       self, runner: Runner, experiment: Experiment
   ) -> list[str]:
-    experiment_dir = runner.current_run.input_dir(experiment)
-    filenames = []
+
+    def _list_checkpoints_from(ckpt_dir: str, examples_to_load: set[int]):
+      ckpt_files = []
+      if pg.io.path_exists(ckpt_dir):
+        regex = re.compile(
+            f'{self._checkpoint_file_prefix}_(\\d+){self._checkpoint_file_ext}'
+            .replace('.', '\\.')
+        )
+        for filename in pg.io.listdir(ckpt_dir):
+          match = regex.match(filename)
+          if match and int(match.group(1)) in examples_to_load:
+            examples_to_load.remove(int(match.group(1)))
+            ckpt_files.append(os.path.join(ckpt_dir, filename))
+      return ckpt_files
+
     examples_to_load = runner.current_run.examples_to_load(experiment)
-    if pg.io.path_exists(experiment_dir):
-      regex = re.compile(
-          f'{self._checkpoint_file_prefix}_(\\d+){self._checkpoint_file_ext}'
-          .replace('.', '\\.')
+
+    # Take output directory as the first priority to checkpoints processed in
+    # this run.
+    ckpt_files = _list_checkpoints_from(
+        runner.current_run.output_dir(experiment), examples_to_load
+    )
+    # If the input and output directories are different, also load from the
+    # input directory.
+    if (examples_to_load
+        and runner.current_run.input_root != runner.current_run.output_root):
+      ckpt_files.extend(
+          _list_checkpoints_from(
+              runner.current_run.input_dir(experiment), examples_to_load
+          )
       )
-      for filename in pg.io.listdir(experiment_dir):
-        match = regex.match(filename)
-        if match and int(match.group(1)) in examples_to_load:
-          filenames.append(filename)
-    return filenames
+    return ckpt_files
 
   def _save_example(
       self,
@@ -341,13 +356,24 @@ class BulkCheckpointer(Checkpointer):
         if self._sequence_writer is not None:
           self._sequence_writer[experiment.id] = sequence_writer
 
-  def _list_checkpoint_filenames(
+  def _list_checkpoint_files(
       self, runner: Runner, experiment: Experiment
   ) -> list[str]:
-    if pg.io.path_exists(
-        runner.current_run.input_path_for(experiment, self.checkpoint_filename)
-    ):
-      return [self.checkpoint_filename]
+    # Always honor the output directory if it's present, as it contains both
+    # the warm-started examples and newly processed examples.
+    output_ckpt_file = runner.current_run.output_path_for(
+        experiment, self.checkpoint_filename
+    )
+    if pg.io.path_exists(output_ckpt_file):
+      return [output_ckpt_file]
+
+    if runner.current_run.input_root != runner.current_run.output_root:
+      input_ckpt_file = runner.current_run.input_path_for(
+          experiment, self.checkpoint_filename
+      )
+      if pg.io.path_exists(input_ckpt_file):
+        return [input_ckpt_file]
+    print('CCC', experiment.hash, [])
     return []
 
   def on_experiment_complete(
@@ -441,5 +467,12 @@ class SequenceWriter:
       self._sequence_writer = None
       pg.io.rename(self._tmp_path, self._path)
 
+  def __enter__(self):
+    return self
+
+  def __exit__(self, *args, **kwargs):
+    del args, kwargs
+    self.close()
+
   def __del__(self):
     self.close()

{langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512180805}/langfun/core/eval/v2/evaluation.py

@@ -386,10 +386,10 @@ class Evaluation(experiment_lib.Experiment):
       load_example_metadata: bool = True,
       filter: Callable[[example_lib.Example], bool] | None = None,  # pylint: disable=redefined-builtin
       raise_if_not_exist: bool = False
-  ) -> None:
+  ) -> list[example_lib.Example]:
     """Loads saved state from a sequence IO file."""
     if pg.io.path_exists(state_file):
-      self._state.load(
+      return self._state.load(
           state_file,
           example_input_by_id=self.example_input_by_id,
           load_example_metadata=load_example_metadata,
@@ -397,6 +397,7 @@ class Evaluation(experiment_lib.Experiment):
       )
     elif raise_if_not_exist:
       raise ValueError(f'State file {state_file} does not exist.')
+    return []
 
   def _reset(self) -> None:
     """Resets the state of the evaluation."""

{langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512180805}/langfun/core/eval/v2/reporting.py

@@ -86,10 +86,8 @@ class ExampleHtmlGenerator(experiment_lib.Plugin):
         return
 
       try:
-        with pg.timeit() as t, pg.io.open(src_file, 'r') as src:
-          content = src.read()
-          with pg.io.open(dest_file, 'w') as dest:
-            dest.write(content)
+        with pg.timeit() as t:
+          pg.io.copy(src_file, dest_file)
         experiment.info(
             f'\'{example.id}.html\' copied in {t.elapse:.2f} seconds.'
         )

{langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512180805}/langfun/core/eval/v2/runners/base.py

@@ -139,6 +139,7 @@ class RunnerBase(Runner):
           self.current_run.examples_to_evaluate(experiment)
       )
       experiment.progress.start(total=num_examples_to_evaluate)
+      pg.io.mkdirs(self.current_run.output_dir(experiment))
     else:
       experiment.progress.start(total=len(experiment.leaf_nodes))
 

{langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512180805}/langfun/core/llms/__init__.py

@@ -43,6 +43,7 @@ from langfun.core.llms.azure_openai import AzureOpenAI
 # Gemini models.
 from langfun.core.llms.google_genai import GenAI
 from langfun.core.llms.google_genai import Gemini3ProPreview
+from langfun.core.llms.google_genai import Gemini3FlashPreview
 from langfun.core.llms.google_genai import Gemini25Pro
 from langfun.core.llms.google_genai import Gemini25Flash
 from langfun.core.llms.google_genai import Gemini25ProPreview_20250605
@@ -65,6 +66,7 @@ from langfun.core.llms.google_genai import Gemini2ProExp_20250205
 from langfun.core.llms.google_genai import Gemini2FlashThinkingExp_20250121
 from langfun.core.llms.google_genai import GeminiExp_20241206
 from langfun.core.llms.google_genai import Gemini25FlashImagePreview
+from langfun.core.llms.google_genai import Gemini3ProImagePreview
 
 from langfun.core.llms.vertexai import VertexAIGemini
 from langfun.core.llms.vertexai import VertexAIGemini2Flash
@@ -92,6 +94,8 @@ from langfun.core.llms.vertexai import VertexAIGemini25Pro
 from langfun.core.llms.vertexai import VertexAIGemini25Flash
 from langfun.core.llms.vertexai import VertexAIGemini25FlashImagePreview
 from langfun.core.llms.vertexai import VertexAIGemini3ProPreview
+from langfun.core.llms.vertexai import VertexAIGemini3ProImagePreview
+from langfun.core.llms.vertexai import VertexAIGemini3FlashPreview
 
 # For backward compatibility.
 GeminiPro1_5 = Gemini15Pro
@@ -158,6 +162,7 @@ from langfun.core.llms.openai import Gpt35
 from langfun.core.llms.anthropic import Claude45
 from langfun.core.llms.anthropic import Claude45Haiku_20251001
 from langfun.core.llms.anthropic import Claude45Sonnet_20250929
+from langfun.core.llms.anthropic import Claude45Opus_20251101
 from langfun.core.llms.anthropic import Claude4
 from langfun.core.llms.anthropic import Claude4Sonnet_20250514
 from langfun.core.llms.anthropic import Claude4Opus_20250514
@@ -177,6 +182,7 @@ from langfun.core.llms.anthropic import Claude3Haiku_20240307
 from langfun.core.llms.vertexai import VertexAIAnthropic
 from langfun.core.llms.vertexai import VertexAIClaude45Haiku_20251001
 from langfun.core.llms.vertexai import VertexAIClaude45Sonnet_20250929
+from langfun.core.llms.vertexai import VertexAIClaude45Opus_20251101
 from langfun.core.llms.vertexai import VertexAIClaude4Opus_20250514
 from langfun.core.llms.vertexai import VertexAIClaude4Sonnet_20250514
 from langfun.core.llms.vertexai import VertexAIClaude37Sonnet_20250219

{langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512180805}/langfun/core/llms/anthropic.py

@@ -113,6 +113,32 @@ SUPPORTED_MODELS = [
             max_output_tokens_per_minute=400_000,
         ),
     ),
+    AnthropicModelInfo(
+        model_id='claude-opus-4-5-20251101',
+        provider='Anthropic',
+        in_service=True,
+        description='Claude 4.5 Opus model (11/01/2025).',
+        release_date=datetime.datetime(2025, 11, 1),
+        input_modalities=(
+            AnthropicModelInfo.INPUT_IMAGE_TYPES
+            + AnthropicModelInfo.INPUT_DOC_TYPES
+        ),
+        context_length=lf.ModelInfo.ContextLength(
+            max_input_tokens=200_000,
+            max_output_tokens=64_000,
+        ),
+        pricing=lf.ModelInfo.Pricing(
+            cost_per_1m_cached_input_tokens=0.5,
+            cost_per_1m_input_tokens=5,
+            cost_per_1m_output_tokens=25,
+        ),
+        rate_limits=AnthropicModelInfo.RateLimits(
+            # Tier 4 rate limits
+            max_requests_per_minute=2000,
+            max_input_tokens_per_minute=1_000_000,
+            max_output_tokens_per_minute=400_000,
+        ),
+    ),
     AnthropicModelInfo(
         model_id='claude-4-opus-20250514',
         provider='Anthropic',
@@ -300,6 +326,32 @@ SUPPORTED_MODELS = [
             max_output_tokens_per_minute=0,
         ),
     ),
+    AnthropicModelInfo(
+        model_id='claude-opus-4-5@20251101',
+        alias_for='claude-opus-4-5-20251101',
+        provider='VertexAI',
+        in_service=True,
+        description='Claude 4.5 Opus model served on VertexAI (11/01/2025).',
+        release_date=datetime.datetime(2025, 11, 1),
+        input_modalities=(
+            AnthropicModelInfo.INPUT_IMAGE_TYPES
+            + AnthropicModelInfo.INPUT_DOC_TYPES
+        ),
+        context_length=lf.ModelInfo.ContextLength(
+            max_input_tokens=200_000,
+            max_output_tokens=64_000,
+        ),
+        pricing=lf.ModelInfo.Pricing(
+            cost_per_1m_cached_input_tokens=0.5,
+            cost_per_1m_input_tokens=5,
+            cost_per_1m_output_tokens=25,
+        ),
+        rate_limits=AnthropicModelInfo.RateLimits(
+            max_requests_per_minute=100,
+            max_input_tokens_per_minute=1_000_000,
+            max_output_tokens_per_minute=80_000,
+        ),
+    ),
     AnthropicModelInfo(
         model_id='claude-opus-4@20250514',
         alias_for='claude-opus-4-20250514',
@@ -834,6 +886,13 @@ class Claude45Sonnet_20250929(Claude45):
   model = 'claude-sonnet-4-5-20250929'
 
 
+# pylint: disable=invalid-name
+class Claude45Opus_20251101(Claude45):
+  """Claude 4.5 Opus model 20251101."""
+
+  model = 'claude-opus-4-5-20251101'
+
+
 class Claude4(Anthropic):
   """Base class for Claude 4 models."""
 

{langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512180805}/langfun/core/llms/gemini.py

@@ -177,6 +177,55 @@ SUPPORTED_MODELS = [
             max_tokens_per_minute=4_000_000,
         ),
     ),
+    # Gemini 3 Pro Image Preview
+    GeminiModelInfo(
+        model_id='gemini-3-pro-image-preview',
+        in_service=True,
+        experimental=True,
+        provider=pg.oneof(['Google GenAI', 'VertexAI']),
+        model_type='instruction-tuned',
+        description=(
+            'Gemini 3 Pro Image Preview for high-fidelity image generation,'
+            ' editing, and visual reasoning.'
+        ),
+        release_date=datetime.datetime(2025, 12, 9),
+        input_modalities=GeminiModelInfo.INPUT_IMAGE_TYPES
+        + GeminiModelInfo.INPUT_DOC_TYPES,
+        context_length=lf.ModelInfo.ContextLength(
+            max_input_tokens=65_536,
+            max_output_tokens=32_768,
+        ),
+        rate_limits=lf.ModelInfo.RateLimits(
+            max_requests_per_minute=200,
+            max_tokens_per_minute=1_000_000,
+        ),
+    ),
+    # Gemini 3 Flash Preview
+    GeminiModelInfo(
+        model_id='gemini-3-flash-preview',
+        in_service=True,
+        provider=pg.oneof(['Google GenAI', 'VertexAI']),
+        model_type='instruction-tuned',
+        description=(
+            'Gemini 3 Flash Preview: High-efficiency, low-latency multimodal'
+            ' model optimized for agentic workflows.'
+        ),
+        release_date=datetime.datetime(2025, 12, 17),
+        input_modalities=GeminiModelInfo.ALL_SUPPORTED_INPUT_TYPES,
+        context_length=lf.ModelInfo.ContextLength(
+            max_input_tokens=1_048_576,
+            max_output_tokens=65_536,
+        ),
+        pricing=GeminiModelInfo.Pricing(
+            cost_per_1m_cached_input_tokens=0.05,
+            cost_per_1m_input_tokens=0.50,
+            cost_per_1m_output_tokens=3.00,
+        ),
+        rate_limits=lf.ModelInfo.RateLimits(
+            max_requests_per_minute=2_000,
+            max_tokens_per_minute=4_000_000,
+        ),
+    ),
     # Gemini 2.5 Flash
     GeminiModelInfo(
         model_id='gemini-2.5-flash',
@@ -834,7 +883,10 @@ class Gemini(rest.REST):
       config['thinkingConfig'] = thinking_config_data
 
     # This is the new feature since Gemini 3.
-    if self.model_id.startswith('gemini-3'):
+    # Skip for image generation models as they don't support mediaResolution.
+    if self.model_id.startswith('gemini-3') and not (
+        self.response_modalities and 'IMAGE' in self.response_modalities
+    ):
       config['mediaResolution'] = 'MEDIA_RESOLUTION_HIGH'
 
     if self.response_modalities:

{langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512180805}/langfun/core/llms/google_genai.py

@@ -125,6 +125,30 @@ class Gemini3ProPreview(GenAI):
   model = 'gemini-3-pro-preview'
 
 
+class Gemini3ProImagePreview(GenAI):
+  """Gemini 3 Pro Image Preview model for high-fidelity image generation.
+
+  This model supports:
+  - Text-to-image generation
+  - Image editing (multimodal input)
+  - Visual reasoning
+
+  Key Requirements:
+  - responseModalities must include 'IMAGE'
+  - Supported aspect ratios: 1:1, 16:9, 9:16, 4:3, 3:4
+  - Image sizes: 1K (default), 2K, 4K
+  """
+
+  model = 'gemini-3-pro-image-preview'
+  response_modalities = ['TEXT', 'IMAGE']
+
+
+class Gemini3FlashPreview(GenAI):
+  """Gemini 3 Flash Preview model."""
+
+  model = 'gemini-3-flash-preview'
+
+
 class Gemini25FlashImagePreview(GenAI):
   """Gemini 2.5 Flash Image Preview model."""
   model = 'gemini-2.5-flash-image-preview'

{langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512180805}/langfun/core/llms/vertexai.py

@@ -220,6 +220,33 @@ class VertexAIGemini3ProPreview(VertexAIGemini):  # pylint: disable=invalid-name
   location = 'global'
 
 
+class VertexAIGemini3ProImagePreview(VertexAIGemini):  # pylint: disable=invalid-name
+  """Gemini 3 Pro Image Preview model for high-fidelity image generation.
+
+  This model supports:
+  - Text-to-image generation
+  - Image editing (multimodal input)
+  - Visual reasoning
+
+  Key Requirements:
+  - Uses v1beta1 API endpoint
+  - responseModalities must include 'IMAGE'
+  - Supported aspect ratios: 1:1, 16:9, 9:16, 4:3, 3:4
+  - Image sizes: 1K (default), 2K, 4K
+  """
+
+  model = 'gemini-3-pro-image-preview'
+  location = 'global'
+  response_modalities = ['TEXT', 'IMAGE']
+
+
+class VertexAIGemini3FlashPreview(VertexAIGemini):  # pylint: disable=invalid-name
+  """Gemini 3 Flash Preview model launched on 12/17/2025."""
+
+  model = 'gemini-3-flash-preview'
+  location = 'global'
+
+
 class VertexAIGemini25Pro(VertexAIGemini):  # pylint: disable=invalid-name
   """Gemini 2.5 Pro GA model launched on 06/17/2025."""
 
@@ -419,6 +446,12 @@ class VertexAIClaude45Sonnet_20250929(VertexAIAnthropic):
   model = 'claude-sonnet-4-5@20250929'
 
 
+class VertexAIClaude45Opus_20251101(VertexAIAnthropic):
+  """Anthropic's Claude 4.5 Opus model on VertexAI."""
+
+  model = 'claude-opus-4-5@20251101'
+
+
 class VertexAIClaude4Opus_20250514(VertexAIAnthropic):
   """Anthropic's Claude 4 Opus model on VertexAI."""
   model = 'claude-opus-4@20250514'

{langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512180805}/langfun/core/modalities/mime.py

@@ -135,7 +135,20 @@ class Mime(lf.Modality):
       raise lf.ModalityError(
           f'MIME type {self.mime_type!r} cannot be converted to text.'
       )
-    return self.to_bytes().decode()
+    content = self.to_bytes()
+    # Try UTF-8 first (most common encoding).
+    try:
+      return content.decode('utf-8')
+    except UnicodeDecodeError:
+      pass
+    # Check for UTF-16 BOM (0xff 0xfe or 0xfe 0xff).
+    if content[:2] in (b'\xff\xfe', b'\xfe\xff'):
+      try:
+        return content.decode('utf-16')
+      except UnicodeDecodeError:
+        pass
+    # Fallback: decode with error replacement to avoid crashing.
+    return content.decode('utf-8', errors='replace')
 
   def is_compatible(
       self, mime_types: str | Iterable[str]

{langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512180805}/langfun/core/modalities/mime_test.py

@@ -163,5 +163,53 @@ class CustomMimeTest(unittest.TestCase):
     )
 
 
+class ToTextEncodingTest(unittest.TestCase):
+  """Tests for to_text() encoding handling."""
+
+  def test_utf8_decoding(self):
+    """Test that valid UTF-8 content is decoded correctly."""
+    content = mime.Custom('text/plain', b'Hello, World!')
+    self.assertEqual(content.to_text(), 'Hello, World!')
+
+    # UTF-8 with multi-byte characters.
+    utf8_content = 'こんにちは'.encode('utf-8')
+    content = mime.Custom('text/plain', utf8_content)
+    self.assertEqual(content.to_text(), 'こんにちは')
+
+  def test_utf16_le_bom_decoding(self):
+    """Test that UTF-16 Little Endian with BOM is decoded correctly."""
+    # UTF-16 LE BOM: 0xff 0xfe
+    utf16_le_content = 'Hello'.encode('utf-16-le')
+    content_with_bom = b'\xff\xfe' + utf16_le_content
+    content = mime.Custom('text/plain', content_with_bom)
+    self.assertEqual(content.to_text(), 'Hello')
+
+  def test_utf16_be_bom_decoding(self):
+    """Test that UTF-16 Big Endian with BOM is decoded correctly."""
+    # UTF-16 BE BOM: 0xfe 0xff
+    utf16_be_content = 'Hello'.encode('utf-16-be')
+    content_with_bom = b'\xfe\xff' + utf16_be_content
+    content = mime.Custom('text/plain', content_with_bom)
+    self.assertEqual(content.to_text(), 'Hello')
+
+  def test_invalid_bytes_fallback_with_replacement(self):
+    """Test that invalid bytes are replaced with replacement character."""
+    # 0xff alone is invalid in UTF-8 and doesn't have UTF-16 BOM pattern.
+    invalid_content = b'\xff\xfdHello'
+    content = mime.Custom('text/plain', invalid_content)
+    result = content.to_text()
+    # Invalid bytes should be replaced with U+FFFD (replacement character).
+    self.assertIn('\ufffd', result)
+    self.assertIn('Hello', result)
+
+  def test_binary_mime_type_raises_error(self):
+    """Test that binary MIME types raise ModalityError."""
+    content = mime.Custom('application/octet-stream', b'\x00\x01\x02')
+    with self.assertRaisesRegex(
+        lf.ModalityError, 'cannot be converted to text'
+    ):
+      content.to_text()
+
+
 if __name__ == '__main__':
   unittest.main()

{langfun-0.1.2.dev202512050805 → langfun-0.1.2.dev202512180805}/langfun.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langfun
-Version: 0.1.2.dev202512050805
+Version: 0.1.2.dev202512180805
 Summary: Langfun: Language as Functions.
 Home-page: https://github.com/google/langfun
 Author: Langfun Authors