langfun 0.1.2.dev202509120804__py3-none-any.whl → 0.1.2.dev202512150805__py3-none-any.whl

langfun/core/coding/python/execution.py

@@ -45,17 +45,17 @@ def evaluate(
     global_vars: An optional dict as the globals that could be referenced by the
       code.
     permission: Permission for the Python code to run.
-    returns_stdout: If True, the stdout (a str) will be returned.
+    returns_stdout: If True, the stdout (a string) will be returned.
     outputs_intermediate: Applicable when returns_stdout is False. If True,
-      intermediate output will be outputted as a dict, with the last line's
-      value accessible by key '__result__' and the std output accessible by
+      intermediate output will be output as a dict, with the last line's
+      value accessible by key '__result__' and the stdout accessible by
       key '__stdout__'. Otherwise the value of the last line will be returned.
 
   Returns:
     The value of the last line of the code block. Or a dict of variable
     names of all locals to their evaluated values as the output of the code to
     run. The value for the last line can be accessed by key '__result__'. Or the
-    stdout as a str.
+    stdout as a string.
   """
   return pg.coding.evaluate(
       parsing.clean(code),
@@ -85,28 +85,30 @@ def run(
 
   Args:
     code: Python code to run.
-    global_vars: An optional dict of
+    global_vars: An optional dict as the globals that could be referenced by the
+      code.
     permission: Permission for the Python code to run.
-    returns_stdout: If True, the stdout (a str) will be returned.
+    returns_stdout: If True, the stdout (a string) will be returned.
     outputs_intermediate: Applicable when returns_stdout is False. If True,
-      intermediate output will be outputted as a dict, with the last line's
-      value accessible by key '__result__' and the std output accessible by
+      intermediate output will be output as a dict, with the last line's
+      value accessible by key '__result__' and the stdout accessible by
       key '__stdout__'. Otherwise the value of the last line will be returned.
     sandbox: If True, run code in sandbox; If False, run code in current
       process. If None, run in sandbox first, if the output could not be
-      serialized and pass to current process, run the code again in current
+      serialized and passed to current process, run the code again in current
       process.
-    timeout: Execution timeout in seconds. If None, wait the code the complete.
+    timeout: Execution timeout in seconds. If None, wait for the code to
+      complete.
 
   Returns:
     The value of the last line of the code block. Or a dict of variable
     names of all locals to their evaluated values as the output of the code to
     run. The value for the last line can be accessed by key '__result__'. Or the
-    stdout as a str.
+    stdout as a string.
 
   Raises:
     TimeoutError: If the execution time exceeds the timeout.
-    Exception: Exception  that are raised from the code.
+    Exception: Exceptions that are raised from the code.
   """
   return pg.coding.run(
       parsing.clean(code),

langfun/core/coding/python/generation.py

@@ -22,9 +22,13 @@ import pyglove as pg
 
 
 class PythonCode(pg.Object):
-  """Symbolic class for Python code.
+  """Represents a piece of Python code that can be executed.
 
-  The value of the last expression of the source will be the returned value.
+  When `PythonCode` is instantiated within a `PythonCode.auto_run()` context,
+  it automatically executes the code and returns the result of the last
+  expression. Otherwise, it acts as a container for the source code, which
+  can be executed by calling the instance. The class also supports automatic
+  error correction via `lf.coding.run_with_correction` when called.
   """
 
   source: Annotated[
@@ -56,7 +60,7 @@ class PythonCode(pg.Object):
         Otherwise, auto call will be disabled.
       sandbox: If True, run code in sandbox; If False, run code in current
         process. If None, run in sandbox first, if the output could not be
-        serialized and pass to current process, run the code again in current
+        serialized and passed to current process, run the code again in current
         process. Applicable when `enabled` is set to True.
       timeout: Timeout in seconds. Applicable when both `enabled` and `sandbox`
         are set to True.
@@ -98,17 +102,17 @@ class PythonCode(pg.Object):
     Args:
       sandbox: If True, run code in sandbox; If False, run code in current
         process. If None, run in sandbox first, if the output could not be
-        serialized and pass to current process, run the code again in current
+        serialized and passed to current process, run the code again in current
         process.
       timeout: Timeout in seconds. If None, there is no timeout. Applicable when
         sandbox is set to True.
       global_vars: Global variables that could be accessed from the source code.
-      returns_stdout: If True, the stdout (a str) will be returned.
+      returns_stdout: If True, the stdout (a string) will be returned.
       outputs_intermediate: Applicable when returns_stdout is False. If True,
-        intermediate output will be outputted as a dict, with the last line's
-        value accessible by key '__result__' and the std output accessible by 
+        intermediate output will be output as a dict, with the last line's
+        value accessible by key '__result__' and the stdout accessible by
         key '__stdout__'. Otherwise the value of the last line will be returned.
-      autofix: Number of attempts to auto fix the generated code. If 0, autofix
+      autofix: Number of attempts to autofix the generated code. If 0, autofix
         is disabled.
       autofix_lm: Language model to be used. If not specified, it will try to
         use the `lm` under `lf.context`.
@@ -117,8 +121,8 @@ class PythonCode(pg.Object):
       The value of the last expression in the source code. Or a dict of local
       variable names defined in the source code to their values if
       `outputs_intermediate` is set to True. The value for the last line can be
-      accessed by key '__result__'. Or the stdout as a str if `returns_stdout`
-      is set to True.
+      accessed by key '__result__'. Or the stdout as a string if
+      `returns_stdout` is set to True.
 
     Raises:
       TimeoutError: If `sandbox` is True and timeout has reached.
@@ -152,12 +156,12 @@ class PythonCode(pg.Object):
     Args:
       sandbox: If True, run code in sandbox; If False, run code in current
         process. If None, run in sandbox first, if the output could not be
-        serialized and pass to current process, run the code again in current
+        serialized and passed to current process, run the code again in current
         process.
       timeout: Timeout in seconds. If None, there is no timeout. Applicable when
         sandbox is set to True.
       global_vars: Global variables that could be accessed from the source code.
-      autofix: Number of attempts to auto fix the generated code. If 0, autofix
+      autofix: Number of attempts to autofix the generated code. If 0, autofix
         is disabled. Auto-fix is not supported for 'json' protocol.
       autofix_lm: Language model to be used. If not specified, it will try to
         use the `lm` under `lf.context`.
@@ -182,10 +186,11 @@ class PythonCode(pg.Object):
 
 
 class PythonFunction(pg.Object):
-  """Generated Python function via source code.
+  """Represents a Python function defined by source code.
 
-  The source code will be directly passed into eval() for execution and the
-  output of the function will be returned.
+  This class takes Python source code that defines a function and makes it
+  callable. The source code is evaluated to create a function object, which
+  can then be invoked like a regular Python function.
   """
 
   name: str
@@ -214,7 +219,7 @@ class PythonFunction(pg.Object):
       *args: Positional arguments that will be passed to the implementation.
       sandbox: If True, run code in sandbox; If False, run code in current
         process. If None, run in sandbox first, if the output could not be
-        serialized and pass to current process, run the code again in current
+        serialized and passed to current process, run the code again in current
         process.
       timeout: Timeout in seconds. If None, there is no timeout. Applicable when
         sandbox is set to True.

langfun/core/coding/python/sandboxing.py

@@ -23,7 +23,14 @@ import pyglove as pg
 
 
 class SandboxOutput(pg.Object):
-  """Sandbox output."""
+  """A structure containing the output from a sandbox execution.
+
+  Attributes:
+    stdout: The standard output captured during execution.
+    stderr: The standard error captured during execution.
+    output_files: A dictionary of file names to their byte content for files
+      generated during execution.
+  """
 
   stdout: Annotated[
       str,
@@ -42,7 +49,14 @@ class SandboxOutput(pg.Object):
 
 
 class BaseSandbox(pg.Object):
-  """Interface and partial implementation for Python sandbox."""
+  """Base class for Python code sandboxing.
+
+  A sandbox provides an isolated environment for executing Python code,
+  typically with restrictions on file system access, network calls, or other
+  potentially harmful operations. This base class defines the interface for
+  sandboxes, including methods for running code (`run`), uploading files
+  (`upload`), and managing the sandbox lifecycle (`setup`, `cleanup`).
+  """
 
   def _on_bound(self):
     super()._on_bound()
@@ -111,7 +125,13 @@ class BaseSandbox(pg.Object):
 
 
 class MultiProcessingSandbox(BaseSandbox):
-  """Sandbox using multiprocessing."""
+  """A sandbox implementation using Python's `multiprocessing`.
+
+  This sandbox executes code in a separate process, providing isolation from
+  the main process. It uses a temporary directory for file operations,
+  which is cleaned up when the sandbox is closed. It relies on
+  `pg.coding.run` with `sandbox=True` for execution.
+  """
 
   def _on_bound(self):
     super()._on_bound()

langfun/core/component.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""langfun Component."""
+"""Base component for Langfun."""
 
 from typing import ContextManager
 import pyglove as pg
@@ -22,7 +22,37 @@ RAISE_IF_HAS_ERROR = (pg.MISSING_VALUE,)
 
 
 class Component(pg.ContextualObject):
-  """Base class for langfun components."""
+  """Base class for Langfun components.
+
+  Langfun components are context-aware symbolic objects powered by PyGlove.
+  (See [PyGlove basics](https://pyglove.readthedocs.io/en/latest/basics.html)
+  for more details).
+
+  **Context-awareness**
+
+  Langfun components can have contextual attributes using `lf.contextual`,
+  whose values can be provided or overridden via `lf.context` or
+  `lf.use_settings`.
+
+  Example:
+  ```python
+  import langfun as lf
+
+  class Bar(lf.Component):
+    y = lf.contextual(1)
+
+  class Foo(lf.Component):
+    x = lf.contextual(0)
+    bar = Bar()
+
+  f = Foo()
+  assert f.x == 0 and f.bar.y == 1
+
+  # `lf.context` overrides `lf.contextual` attributes.
+  with lf.context(x=10, y=20):
+    assert f.x == 10 and f.bar.y == 20
+  ```
+  """
 
   # Allow symbolic assignment, which invalidates the object and recomputes
   # states upon update.
@@ -78,6 +108,15 @@ def use_settings(
 ) -> ContextManager[dict[str, pg.utils.ContextualOverride]]:
   """Shortcut method for overriding component attributes.
 
+  Example:
+
+  ```
+  with lf.use_settings(
+      lm=lf.llms.Gpt35(),
+      temperature=0.0):
+    lf.query('who are you?')
+  ```
+
   Args:
     cascade: If True, this override will apply to both current scope and nested
       scope, meaning that this `lf.context` will take precedence over all
@@ -85,6 +124,6 @@ def use_settings(
     **settings: Key/values as override for component attributes.
 
   Returns:
-    A dict of attribute names to their contextual overrides.
+    A context manager for overriding settings.
   """
   return context(cascade=cascade, override_attrs=True, **settings)

langfun/core/concurrent.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""Utility library for handling concurrency in langfun."""
+"""Utilities for concurrency in Langfun."""
 
 import abc
 import collections
@@ -97,7 +97,7 @@ class RetryError(RuntimeError):
 
 
 def with_retry(
-    func: Callable[[Any], Any],
+    func: Callable[..., Any],
     retry_on_errors: Union[
         Union[Type[BaseException], Tuple[Type[BaseException], str]],
         Sequence[Union[Type[BaseException], Tuple[Type[BaseException], str]]],
@@ -108,10 +108,25 @@ def with_retry(
     max_retry_interval: int = 300,
     seed: int | None = None,
 ) -> Callable[..., Any]:
-  """Derives a user function with retry on error.
+  """Decorator-like function to add retry mechanism to a function.
+
+  Example:
+
+  ```
+  def flaky_function():
+    if random.random() < 0.5:
+      raise ValueError('error')
+    return 1
+
+  reliable_function = lf.with_retry(
+      flaky_function,
+      retry_on_errors=ValueError,
+      max_attempts=3)
+  reliable_function()
+  ```
 
   Args:
-    func: A user function.
+    func: The function to add retry mechanism.
     retry_on_errors: A sequence of exception types or tuples of exception type
       and error messages (described in regular expression) as the desired
       exception types to retry.
@@ -128,8 +143,7 @@ def with_retry(
       determined based on current time.
 
   Returns:
-    A function with the same signature of the input function, with the retry
-    capability.
+    A function with the same signature of `func`, but with retry capability.
   """
 
   def _func(*args, **kwargs):
@@ -179,6 +193,24 @@ def concurrent_execute(
 ) -> list[Any]:
   """Executes a function concurrently under current component context.
 
+  `lf.concurrent_execute` applies a function to each item in an iterable of
+  inputs in parallel and returns a list of results in the same order as the
+  inputs. It is a convenient wrapper around `lf.concurrent_map` for synchronous
+  bulk processing.
+
+  **Example:**
+
+  ```python
+  import langfun as lf
+
+  def square(x):
+    return x ** 2
+
+  results = lf.concurrent_execute(square, [1, 2, 3, 4], max_workers=2)
+  print(results)
+  # Output: [1, 4, 9, 16]
+  ```
+
   Args:
     func: A user function.
     parallel_inputs: The inputs for `func` which will be processed in parallel.
@@ -649,6 +681,38 @@ def concurrent_map(
 ) -> Iterator[Any]:
   """Maps inputs to outptus via func concurrently under current context.
 
+  `lf.concurrent_map` applies a function to each item in an iterable of
+  inputs in parallel and yields `(input, output, error)` tuples as they are
+  completed. It supports features like ordered/unordered results, progress
+  bars, timeouts, and automatic retries for transient errors.
+
+  **Example:**
+
+  ```python
+  import langfun as lf
+  import time
+  import random
+
+  def flaky_square(x):
+    time.sleep(random.random())
+    if random.random() < 0.3:
+      raise ValueError("Flaky error")
+    return x ** 2
+
+  # Unordered execution with progress bar and retries
+  for input, output, error in lf.concurrent_map(
+      flaky_square,
+      range(10),
+      max_workers=3,
+      show_progress=True,
+      retry_on_errors=ValueError,
+      max_attempts=3):
+    if error:
+      print(f"Input {input} failed with error: {error}")
+    else:
+      print(f"Input {input} succeeded with output: {output}")
+  ```
+
   Args:
     func: A user function.
     parallel_inputs: The inputs for `func` which will be processed in parallel.

langfun/core/concurrent_test.py

@@ -262,6 +262,7 @@ class ProgressControlTest(unittest.TestCase):
     with contextlib.redirect_stderr(string_io):
       ctrl.update(1)
       ctrl.refresh()
+      sys.stderr.flush()
     self.assertEqual(string_io.getvalue(), '')
     concurrent.progress_bar = 'tqdm'
 
@@ -274,6 +275,7 @@ class ProgressControlTest(unittest.TestCase):
       ctrl.set_status('bar')
       ctrl.update(10)
       ctrl.refresh()
+      sys.stderr.flush()
     self.assertEqual(
         string_io.getvalue(),
         '\x1b[1m\x1b[31mfoo\x1b[0m: \x1b[34m10% (10/100)\x1b[0m : bar\n'
@@ -288,6 +290,7 @@ class ProgressControlTest(unittest.TestCase):
       self.assertIsInstance(ctrl, concurrent._TqdmProgressControl)
       ctrl.update(10)
       ctrl.refresh()
+      sys.stderr.flush()
     self.assertIn('10/100', string_io.getvalue())
 
     tqdm = concurrent.tqdm
@@ -316,6 +319,7 @@ class ProgressBarTest(unittest.TestCase):
       for _ in concurrent.concurrent_execute(fun, range(5)):
         concurrent.ProgressBar.refresh()
       concurrent.ProgressBar.uninstall(bar_id)
+      sys.stderr.flush()
     output_str = string_io.getvalue()
     self.assertIn('100%', output_str)
     self.assertIn('5/5', output_str)
@@ -332,7 +336,7 @@ class ProgressBarTest(unittest.TestCase):
         concurrent.ProgressBar.update(bar_id, 0, status=1)
       concurrent.ProgressBar.uninstall(bar_id)
       sys.stderr.flush()
-      time.sleep(1)
+    time.sleep(1)
     self.assertIn('1/4', string_io.getvalue())
     # TODO(daiyip): Re-enable once flakiness is fixed.
     # self.assertIn('2/4', string_io.getvalue())
@@ -564,7 +568,8 @@ class ConcurrentMapTest(unittest.TestCase):
               fun, [1, 2, 3], timeout=1.5, max_workers=1, show_progress=True
           )
       ], key=lambda x: x[0])
-      string_io.flush()
+      sys.stderr.flush()
+
     self.assertEqual(   # pylint: disable=g-generic-assert
         output,
         [
@@ -592,6 +597,7 @@ class ConcurrentMapTest(unittest.TestCase):
               show_progress=bar_id, status_fn=lambda p: dict(x=1, y=1)
           )
       ], key=lambda x: x[0])
+      sys.stderr.flush()
 
     self.assertEqual(  # pylint: disable=g-generic-assert
         output,
@@ -602,6 +608,7 @@ class ConcurrentMapTest(unittest.TestCase):
         ],
     )
     concurrent.ProgressBar.uninstall(bar_id)
+    concurrent.ProgressBar.refresh()
     self.assertIn('100%', string_io.getvalue())
 
 

langfun/core/console.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""Console utilities."""
+"""Utilities for console output and notebook display."""
 
 import sys
 from typing import Any

langfun/core/data/conversion/anthropic.py

@@ -21,7 +21,14 @@ from langfun.core import modalities as lf_modalities
 
 
 class AnthropicMessageConverter(lf.MessageConverter):
-  """Converter to Anthropic public API."""
+  """Converter between Langfun messages and Anthropic API message format.
+
+  This converter translates `lf.Message` objects into the JSON format required
+  by the Anthropic API and vice versa. It handles text and modalities like
+  images and PDFs by encoding them in base64 format as expected by Anthropic.
+  An optional `chunk_preprocessor` can be provided to modify or filter
+  chunks before conversion.
+  """
 
   FORMAT_ID = 'anthropic'
 
@@ -30,12 +37,12 @@ class AnthropicMessageConverter(lf.MessageConverter):
       (
           'Chunk preprocessor for Langfun chunk to Anthropic chunk conversion. '
           'It will be applied before each Langfun chunk is converted. '
-          'If returns None, the chunk will be skipped.'
+          'If it returns None, the chunk will be skipped.'
       )
   ] = None
 
   def to_value(self, message: lf.Message) -> dict[str, Any]:
-    """Converts a Langfun message to Gemini API."""
+    """Converts a Langfun message to Anthropic API."""
     content = []
     for chunk in message.chunk():
       if self.chunk_preprocessor:
@@ -97,6 +104,8 @@ class AnthropicMessageConverter(lf.MessageConverter):
                 self._safe_read(source, 'media_type')
             ).from_bytes(base64.b64decode(self._safe_read(source, 'data')))
         )
+      elif t in ('server_tool_use', 'web_search_tool_result'):
+        continue
       else:
         raise ValueError(f'Unsupported content part: {part!r}.')
     message = message_cls.from_chunks(chunks)

langfun/core/data/conversion/anthropic_test.py

@@ -253,14 +253,16 @@ class AnthropicConversionTest(unittest.TestCase):
     )
     self.assertEqual(
         m.text,
-        'What are the common words from <<[[obj0]]>> and <<[[obj1]]>> ?'
+        'What are the common words from <<[[image:dc6e1e43]]>> and'
+        ' <<[[pdf:5daf5f31]]>> ?'
     )
-    self.assertIsInstance(m.obj0, lf_modalities.Image)
-    self.assertEqual(m.obj0.mime_type, 'image/png')
-    self.assertEqual(m.obj0.to_bytes(), image_content)
+    modalities = m.modalities()
+    self.assertIsInstance(modalities[0], lf_modalities.Image)
+    self.assertEqual(modalities[0].mime_type, 'image/png')
+    self.assertEqual(modalities[0].content, image_content)
 
-    self.assertIsInstance(m.obj1, lf_modalities.PDF)
-    self.assertEqual(m.obj1.to_bytes(), pdf_content)
+    self.assertIsInstance(modalities[1], lf_modalities.PDF)
+    self.assertEqual(modalities[1].content, pdf_content)
 
 
 if __name__ == '__main__':

langfun/core/data/conversion/gemini.py

@@ -21,7 +21,14 @@ from langfun.core import modalities as lf_modalities
 
 
 class GeminiMessageConverter(lf.MessageConverter):
-  """Converter to Gemini public API."""
+  """Converter between Langfun messages and Gemini API message format.
+
+  This converter translates `lf.Message` objects into the JSON format required
+  by the public Gemini API (e.g., via Vertex AI or Google AI Studio) and
+  vice versa. It handles text and modalities like images, extracting thought
+  chunks if present. An optional `chunk_preprocessor` can be provided to
+  modify or filter chunks before conversion.
+  """
 
   FORMAT_ID = 'gemini'
 
@@ -30,7 +37,7 @@ class GeminiMessageConverter(lf.MessageConverter):
       (
           'Chunk preprocessor for Langfun chunk to Gemini chunk conversion. '
           'It will be applied before each Langfun chunk is converted. '
-          'If returns None, the chunk will be skipped.'
+          'If it returns None, the chunk will be skipped.'
       ),
   ] = None
 
@@ -131,6 +138,8 @@ class GeminiMessageConverter(lf.MessageConverter):
                 self._safe_read(data, 'mimeType')
             ).from_uri(self._safe_read(data, 'fileUri'))
         )
+      elif 'functionCall' in part or 'functionResponse' in part:
+        pass
       else:
         raise ValueError(f'Unsupported content part: {part!r}.')
     message = message_cls.from_chunks(chunks)

langfun/core/data/conversion/gemini_test.py

@@ -225,19 +225,58 @@ class GeminiConversionTest(unittest.TestCase):
     self.assertEqual(
         m.text,
         (
-            'What are the common words from <<[[obj0]]>> , <<[[obj1]]>> '
-            'and <<[[obj2]]>> ?'
+            'What are the common words from <<[[image:dc6e1e43]]>> , '
+            '<<[[pdf:4dc12e93]]>> and <<[[video:7e169565]]>> ?'
         )
     )
-    self.assertIsInstance(m.obj0, lf_modalities.Image)
-    self.assertEqual(m.obj0.mime_type, 'image/png')
-    self.assertEqual(m.obj0.to_bytes(), image_content)
+    self.assertIsInstance(m.modalities()[0], lf_modalities.Image)
+    self.assertEqual(m.modalities()[0].mime_type, 'image/png')
+    self.assertEqual(m.modalities()[0].to_bytes(), image_content)
 
-    self.assertIsInstance(m.obj1, lf_modalities.PDF)
-    self.assertEqual(m.obj1.uri, 'https://my.pdf')
+    self.assertIsInstance(m.modalities()[1], lf_modalities.PDF)
+    self.assertEqual(m.modalities()[1].uri, 'https://my.pdf')
 
-    self.assertIsInstance(m.obj2, lf_modalities.Video)
-    self.assertEqual(m.obj2.uri, 'https://www.youtube.com/watch?v=abcd')
+    self.assertIsInstance(m.modalities()[2], lf_modalities.Video)
+    self.assertEqual(
+        m.modalities()[2].uri,
+        'https://www.youtube.com/watch?v=abcd'
+    )
+
+  def test_from_value_with_function_call(self):
+    message = lf.Message.from_value(
+        {
+            'role': 'model',
+            'parts': [
+                {'text': 'Let me search for that.'},
+                {
+                    'functionCall': {
+                        'name': 'search',
+                        'args': {'query': 'test'},
+                    }
+                },
+            ],
+        },
+        format='gemini',
+    )
+    self.assertEqual(message.text, 'Let me search for that.')
+
+  def test_from_value_with_function_response(self):
+    message = lf.Message.from_value(
+        {
+            'role': 'user',
+            'parts': [
+                {
+                    'functionResponse': {
+                        'name': 'search',
+                        'response': {'results': ['a', 'b']},
+                    }
+                },
+                {'text': 'Here are the results.'},
+            ],
+        },
+        format='gemini',
+    )
+    self.assertEqual(message.text, 'Here are the results.')
 
 
 if __name__ == '__main__':