langfun 0.0.2.dev20240607__py3-none-any.whl → 0.0.2.dev20240609__py3-none-any.whl

langfun/core/__init__.py

@@ -101,6 +101,7 @@ from langfun.core.language_model import LanguageModel
 from langfun.core.language_model import LMSample
 from langfun.core.language_model import LMSamplingOptions
 from langfun.core.language_model import LMSamplingUsage
+from langfun.core.language_model import UsageNotAvailable
 from langfun.core.language_model import LMSamplingResult
 from langfun.core.language_model import LMScoringResult
 from langfun.core.language_model import LMCache
@@ -120,6 +121,9 @@ from langfun.core.memory import Memory
 # Utility for console output.
 from langfun.core import console
 
+# Helpers for implementing _repr_xxx_ methods.
+from langfun.core import repr_utils
+
 # Utility for event logging.
 from langfun.core import logging
 

langfun/core/eval/base.py

@@ -549,7 +549,7 @@ class Evaluable(lf.Component):
         )
         s.write(html.escape(pg.format(m.result)))
         s.write('</div>')
-      if 'usage' in m.metadata and m.usage is not None:
+      if m.metadata.get('usage', None):
         s.write(
             '<div style="background-color: #EEEEEE; color: black; '
             'white-space: pre-wrap; padding: 10px; border: 0px solid; '
@@ -1321,7 +1321,7 @@ class Evaluation(Evaluable):
       self._render_summary_metrics(s)
 
       # Summarize average usage.
-      if self.result.usage is not None:
+      if self.result.usage:
         self._render_summary_usage(s)
 
     s.write('</td></tr></table></div>')
@@ -1441,9 +1441,10 @@ class Evaluation(Evaluable):
   def audit_usage(self, message: lf.Message, dryrun: bool = False) -> None:
     del dryrun
     for m in message.trace():
-      if m.metadata.get('usage', None) is not None:
-        self._total_prompt_tokens += m.usage.prompt_tokens
-        self._total_completion_tokens += m.usage.completion_tokens
+      usage = m.metadata.get('usage', None)
+      if usage:
+        self._total_prompt_tokens += usage.prompt_tokens
+        self._total_completion_tokens += usage.completion_tokens
         self._num_usages += 1
 
   def audit_processed(
@@ -1504,7 +1505,7 @@ class Evaluation(Evaluable):
         '<td>Schema</td>'
         '<td>Additional Args</td>'
     )
-    if self.result.usage is not None:
+    if self.result.usage:
       s.write('<td>Usage</td>')
     s.write('<td>OOP Failures</td>')
     s.write('<td>Non-OOP Failures</td>')
@@ -1533,7 +1534,7 @@ class Evaluation(Evaluable):
         f'{_html_repr(self.additional_args, compact=False)}</td>'
     )
     # Usage.
-    if self.result.usage is not None:
+    if self.result.usage:
       s.write('<td>')
       self._render_summary_usage(s)
       s.write('</td>')

langfun/core/langfunc_test.py

@@ -87,7 +87,11 @@ class LangFuncCallTest(unittest.TestCase):
 
     r = l()
     self.assertEqual(
-        r, message.AIMessage('Hello!!!', score=0.0, logprobs=None, usage=None)
+        r,
+        message.AIMessage(
+            'Hello!!!', score=0.0, logprobs=None,
+            usage=language_model.UsageNotAvailable()
+        )
     )
     self.assertEqual(r.tags, ['lm-response', 'lm-output'])
     self.assertEqual(r.source, message.UserMessage('Hello'))
@@ -112,7 +116,11 @@ class LangFuncCallTest(unittest.TestCase):
       self.assertEqual(l.render(), 'Hello')
       r = l()
       self.assertEqual(
-          r, message.AIMessage('Hello!!!', score=0.0, logprobs=None, usage=None)
+          r,
+          message.AIMessage(
+              'Hello!!!', score=0.0, logprobs=None,
+              usage=language_model.UsageNotAvailable()
+          )
       )
       self.assertEqual(r.tags, ['lm-response', 'lm-output'])
 

langfun/core/language_model.py

@@ -93,6 +93,16 @@ class LMSamplingUsage(pg.Object):
     )
 
 
+class UsageNotAvailable(LMSamplingUsage):
+  """Usage information not available."""
+  prompt_tokens: pg.typing.Int(0).freeze()        # pytype: disable=invalid-annotation
+  completion_tokens: pg.typing.Int(0).freeze()   # pytype: disable=invalid-annotation
+  total_tokens: pg.typing.Int(0).freeze()        # pytype: disable=invalid-annotation
+
+  def __bool__(self) -> bool:
+    return False
+
+
 class LMSamplingResult(pg.Object):
   """Language model response."""
 
@@ -105,9 +115,9 @@ class LMSamplingResult(pg.Object):
   ] = []
 
   usage: Annotated[
-      LMSamplingUsage | None,
+      LMSamplingUsage,
       'Usage information. Currently only OpenAI models are supported.',
-  ] = None
+  ] = UsageNotAvailable()
 
 
 class LMSamplingOptions(component.Component):
@@ -406,7 +416,7 @@ class LanguageModel(component.Component):
           # which is accurate when n=1. For n > 1, we average the usage across
           # multiple samples.
           usage = result.usage
-          if len(result.samples) == 1 or usage is None:
+          if len(result.samples) == 1 or not usage:
             response.metadata.usage = usage
           else:
             n = len(result.samples)
@@ -417,14 +427,11 @@ class LanguageModel(component.Component):
             )
 
           # Track usage.
-          if usage:
-            tracked_usages = component.context_value('__tracked_usages__', [])
+          trackers = component.context_value('__usage_trackers__', [])
+          if trackers:
             model_id = self.model_id
-            for usage_dict in tracked_usages:
-              if model_id in usage_dict:
-                usage_dict[model_id] += usage
-              else:
-                usage_dict[model_id] = usage
+            for tracker in trackers:
+              tracker.track(model_id, usage)
 
           # Track the prompt for corresponding response.
           response.source = prompt
@@ -529,7 +536,7 @@ class LanguageModel(component.Component):
       prompt: message_lib.Message,
       response: message_lib.Message,
       call_counter: int,
-      usage: LMSamplingUsage | None,
+      usage: LMSamplingUsage,
       elapse: float,
   ) -> None:
     """Outputs debugging information."""
@@ -547,12 +554,13 @@ class LanguageModel(component.Component):
       self._debug_response(response, call_counter, usage, elapse)
 
   def _debug_model_info(
-      self, call_counter: int, usage: LMSamplingUsage | None) -> None:
+      self, call_counter: int, usage: LMSamplingUsage) -> None:
     """Outputs debugging information about the model."""
     title_suffix = ''
-    if usage and usage.total_tokens != 0:
+    if usage.total_tokens != 0:
       title_suffix = console.colored(
-          f' (total {usage.total_tokens} tokens)', 'red')
+          f' (total {usage.total_tokens} tokens)', 'red'
+      )
 
     console.write(
         self.format(compact=True, use_inferred=True),
@@ -564,11 +572,11 @@ class LanguageModel(component.Component):
       self,
       prompt: message_lib.Message,
       call_counter: int,
-      usage: LMSamplingUsage | None,
+      usage: LMSamplingUsage,
   ) -> None:
     """Outputs debugging information about the prompt."""
     title_suffix = ''
-    if usage and usage.prompt_tokens != 0:
+    if usage.prompt_tokens != 0:
       title_suffix = console.colored(f' ({usage.prompt_tokens} tokens)', 'red')
 
     console.write(
@@ -592,12 +600,12 @@ class LanguageModel(component.Component):
       self,
       response: message_lib.Message,
       call_counter: int,
-      usage: LMSamplingUsage | None,
+      usage: LMSamplingUsage,
       elapse: float
   ) -> None:
     """Outputs debugging information about the response."""
     title_suffix = ' ('
-    if usage and usage.completion_tokens != 0:
+    if usage.completion_tokens != 0:
       title_suffix += f'{usage.completion_tokens} tokens '
     title_suffix += f'in {elapse:.2f} seconds)'
     title_suffix = console.colored(title_suffix, 'red')
@@ -659,7 +667,7 @@ class LanguageModel(component.Component):
       debug = LMDebugMode.ALL if debug else LMDebugMode.NONE
 
     if debug & LMDebugMode.INFO:
-      self._debug_model_info(call_counter, None)
+      self._debug_model_info(call_counter, UsageNotAvailable())
 
     if debug & LMDebugMode.PROMPT:
       console.write(
@@ -712,13 +720,58 @@ class LanguageModel(component.Component):
       return DEFAULT_MAX_CONCURRENCY  # Default of 1
 
 
+class _UsageTracker:
+  """Usage tracker."""
+
+  def __init__(self, model_ids: set[str] | None):
+    self.model_ids = model_ids
+    self.usages = {
+        m: LMSamplingUsage(0, 0, 0) for m in model_ids
+    } if model_ids else {}
+
+  def track(self, model_id: str, usage: LMSamplingUsage):
+    if self.model_ids is not None and model_id not in self.model_ids:
+      return
+    if not isinstance(usage, UsageNotAvailable) and model_id in self.usages:
+      self.usages[model_id] += usage
+    else:
+      self.usages[model_id] = usage
+
+
 @contextlib.contextmanager
-def track_usages() -> Iterator[dict[str, LMSamplingUsage]]:
-  """Context manager to track the usage of language models by model ID."""
-  tracked_usages = component.context_value('__tracked_usages__', [])
-  current_usage = dict()
-  with component.context(__tracked_usages__=tracked_usages + [current_usage]):
+def track_usages(
+    *lm: Union[str, LanguageModel]
+) -> Iterator[dict[str, LMSamplingUsage]]:
+  """Context manager to track the usages of all language models in scope.
+
+  `lf.track_usages` works with threads spawned by `lf.concurrent_map` and
+  `lf.concurrent_execute`.
+
+  Example:
+    ```
+    lm = lf.llms.GeminiPro1()
+    with lf.track_usages() as usages:
+      # invoke any code that will call LLMs.
+
+    print(usages[lm.model_id])
+    ```
+
+  Args:
+    *lm: The language model(s) to track. If None, track all models in scope.
+
+  Yields:
+    A dictionary of model ID to usage. If a model does not supports usage
+    counting, the dict entry will be None.
+  """
+  if not lm:
+    model_ids = None
+  else:
+    model_ids = [m.model_id if isinstance(m, LanguageModel) else m for m in lm]
+
+  trackers = component.context_value('__usage_trackers__', [])
+  tracker = _UsageTracker(set(model_ids) if model_ids else None)
+  with component.context(__usage_trackers__=trackers + [tracker]):
     try:
-      yield current_usage
+      yield tracker.usages
     finally:
       pass

langfun/core/language_model_test.py

@@ -27,8 +27,8 @@ import pyglove as pg
 @pg.use_init_args(['failures_before_attempt'])
 class MockModel(lm_lib.LanguageModel):
   """A mock model that echo back user prompts."""
-
   failures_before_attempt: int = 0
+  name: str = 'MockModel'
 
   def _sample(self,
               prompts: list[message_lib.Message]
@@ -63,6 +63,10 @@ class MockModel(lm_lib.LanguageModel):
         retry_interval=1,
     )(prompts)
 
+  @property
+  def model_id(self) -> str:
+    return self.name
+
 
 class MockScoringModel(MockModel):
 
@@ -581,17 +585,33 @@ class LanguageModelTest(unittest.TestCase):
     self.assertEqual(lm.rate_to_max_concurrency(tokens_per_min=1), 1)
 
   def test_track_usages(self):
+    lm = MockModel(name='model1')
+    lm2 = MockModel(name='model2')
     with lm_lib.track_usages() as usages1:
-      lm = MockModel()
       _ = lm('hi')
-      with lm_lib.track_usages() as usages2:
-        _ = lm('hi')
+      with lm_lib.track_usages(lm2) as usages2:
+        with lm_lib.track_usages('model1') as usages3:
+          with lm_lib.track_usages('model1', lm2) as usages4:
+            def call_lm(prompt):
+              _ = lm.sample([prompt] * 2)
+            lm2('hi')
+            list(concurrent.concurrent_map(call_lm, ['hi', 'hello']))
+
     self.assertEqual(usages2, {
-        'MockModel': lm_lib.LMSamplingUsage(100, 100, 200),
+        'model2': lm_lib.LMSamplingUsage(100, 100, 200),
+    })
+    self.assertEqual(usages3, {
+        'model1': lm_lib.LMSamplingUsage(100 * 4, 100 * 4, 200 * 4),
+    })
+    self.assertEqual(usages4, {
+        'model1': lm_lib.LMSamplingUsage(100 * 4, 100 * 4, 200 * 4),
+        'model2': lm_lib.LMSamplingUsage(100, 100, 200),
     })
     self.assertEqual(usages1, {
-        'MockModel': lm_lib.LMSamplingUsage(200, 200, 400),
+        'model1': lm_lib.LMSamplingUsage(100 * 5, 100 * 5, 200 * 5),
+        'model2': lm_lib.LMSamplingUsage(100, 100, 200),
     })
 
+
 if __name__ == '__main__':
   unittest.main()

langfun/core/llms/openai.py

@@ -202,15 +202,14 @@ class OpenAI(lf.LanguageModel):
             lf.LMSample(choice.text.strip(), score=choice.logprobs or 0.0)
         )
 
+      n = len(samples_by_index)
       usage = lf.LMSamplingUsage(
-          prompt_tokens=response.usage.prompt_tokens,
-          completion_tokens=response.usage.completion_tokens,
-          total_tokens=response.usage.total_tokens,
+          prompt_tokens=response.usage.prompt_tokens // n,
+          completion_tokens=response.usage.completion_tokens // n,
+          total_tokens=response.usage.total_tokens // n,
       )
       return [
-          lf.LMSamplingResult(
-              samples_by_index[index], usage=usage if index == 0 else None
-          )
+          lf.LMSamplingResult(samples_by_index[index], usage=usage)
           for index in sorted(samples_by_index.keys())
       ]
 

langfun/core/llms/openai_test.py

@@ -191,9 +191,9 @@ class OpenAITest(unittest.TestCase):
                         score=0.0,
                         logprobs=None,
                         usage=lf.LMSamplingUsage(
-                            prompt_tokens=33,
-                            completion_tokens=33,
-                            total_tokens=66
+                            prompt_tokens=16,
+                            completion_tokens=16,
+                            total_tokens=33
                         ),
                         tags=[lf.Message.TAG_LM_RESPONSE],
                     ),
@@ -206,9 +206,9 @@ class OpenAITest(unittest.TestCase):
                         score=0.1,
                         logprobs=None,
                         usage=lf.LMSamplingUsage(
-                            prompt_tokens=33,
-                            completion_tokens=33,
-                            total_tokens=66
+                            prompt_tokens=16,
+                            completion_tokens=16,
+                            total_tokens=33
                         ),
                         tags=[lf.Message.TAG_LM_RESPONSE],
                     ),
@@ -221,9 +221,9 @@ class OpenAITest(unittest.TestCase):
                         score=0.2,
                         logprobs=None,
                         usage=lf.LMSamplingUsage(
-                            prompt_tokens=33,
-                            completion_tokens=33,
-                            total_tokens=66
+                            prompt_tokens=16,
+                            completion_tokens=16,
+                            total_tokens=33
                         ),
                         tags=[lf.Message.TAG_LM_RESPONSE],
                     ),
@@ -232,7 +232,7 @@ class OpenAITest(unittest.TestCase):
                 ),
             ],
             usage=lf.LMSamplingUsage(
-                prompt_tokens=100, completion_tokens=100, total_tokens=200
+                prompt_tokens=50, completion_tokens=50, total_tokens=100
             ),
         ),
     )
@@ -245,7 +245,11 @@ class OpenAITest(unittest.TestCase):
                         'Sample 0 for prompt 1.',
                         score=0.0,
                         logprobs=None,
-                        usage=None,
+                        usage=lf.LMSamplingUsage(
+                            prompt_tokens=16,
+                            completion_tokens=16,
+                            total_tokens=33
+                        ),
                         tags=[lf.Message.TAG_LM_RESPONSE],
                     ),
                     score=0.0,
@@ -256,7 +260,11 @@ class OpenAITest(unittest.TestCase):
                         'Sample 1 for prompt 1.',
                         score=0.1,
                         logprobs=None,
-                        usage=None,
+                        usage=lf.LMSamplingUsage(
+                            prompt_tokens=16,
+                            completion_tokens=16,
+                            total_tokens=33
+                        ),
                         tags=[lf.Message.TAG_LM_RESPONSE],
                     ),
                     score=0.1,
@@ -267,13 +275,20 @@ class OpenAITest(unittest.TestCase):
                         'Sample 2 for prompt 1.',
                         score=0.2,
                         logprobs=None,
-                        usage=None,
+                        usage=lf.LMSamplingUsage(
+                            prompt_tokens=16,
+                            completion_tokens=16,
+                            total_tokens=33
+                        ),
                         tags=[lf.Message.TAG_LM_RESPONSE],
                     ),
                     score=0.2,
                     logprobs=None,
                 ),
             ],
+            usage=lf.LMSamplingUsage(
+                prompt_tokens=50, completion_tokens=50, total_tokens=100
+            ),
         ),
     )
 

langfun/core/llms/rest_test.py

@@ -89,7 +89,7 @@ class RestTest(unittest.TestCase):
               "max_tokens=4096, stop=['\\n']."
           ),
       )
-      self.assertIsNone(response.usage)
+      self.assertEqual(response.usage, lf.UsageNotAvailable())
 
   def test_call_errors(self):
     for status_code, error_type, error_message in [

langfun/core/repr_utils.py

@@ -0,0 +1,83 @@
+# Copyright 2024 The Langfun Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Helpers for implementing _repr_xxx_ methods."""
+
+import collections
+import contextlib
+import io
+from typing import Iterator
+
+from langfun.core import component
+
+
+@contextlib.contextmanager
+def share_parts() -> Iterator[dict[str, int]]:
+  """Context manager for defining the context (scope) of shared content.
+
+  Under the context manager, call to `lf.write_shared` with the same content
+  will be written only once. This is useful for writing shared content such as
+  shared style and script sections in the HTML.
+
+  Example:
+    ```
+    class Foo(pg.Object):
+      def _repr_html_(self) -> str:
+        s = io.StringIO()
+        lf.repr_utils.write_shared_part(s, '<style>..</style>')
+        lf.repr_utils.write_shared_part(s, '<script>..</script>')
+        return s.getvalue()
+
+    with lf.repr_utils.share_parts() as share_parts:
+      # The <style> and <script> section will be written only once.
+      lf.console.display(Foo())
+      lf.console.display(Foo())
+
+    # Assert that the shared content is attempted to be written twice.
+    assert share_parts['<style>..</style>'] == 2
+    ```
+
+  Yields:
+    A dictionary mapping the shared content to the number of times it is
+    attempted to be written.
+  """
+  context = component.context_value(
+      '__shared_parts__', collections.defaultdict(int)
+  )
+  with component.context(__shared_parts__=context):
+    try:
+      yield context
+    finally:
+      pass
+
+
+def write_maybe_shared(s: io.StringIO, content: str) -> bool:
+  """Writes a maybe shared part to an string stream.
+
+  Args:
+    s: The string stream to write to.
+    content: A maybe shared content to write.
+
+  Returns:
+    True if the content is written to the string. False if the content is
+    already written under the same share context.
+  """
+  context = component.context_value('__shared_parts__', None)
+  if context is None:
+    s.write(content)
+    return True
+  written = content in context
+  if not written:
+    s.write(content)
+  context[content] += 1
+  return not written

langfun/core/repr_utils_test.py

@@ -0,0 +1,58 @@
+# Copyright 2024 The Langfun Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for langfun.core.repr_utils."""
+
+import io
+import unittest
+
+from langfun.core import repr_utils
+
+
+class SharingContentTest(unittest.TestCase):
+
+  def test_sharing(self):
+    s = io.StringIO()
+
+    self.assertTrue(repr_utils.write_maybe_shared(s, '<hr>'))
+    self.assertTrue(repr_utils.write_maybe_shared(s, '<hr>'))
+
+    with repr_utils.share_parts() as ctx1:
+      self.assertTrue(repr_utils.write_maybe_shared(s, '<style></style>'))
+      self.assertFalse(repr_utils.write_maybe_shared(s, '<style></style>'))
+
+      with repr_utils.share_parts() as ctx2:
+        self.assertIs(ctx2, ctx1)
+        self.assertFalse(repr_utils.write_maybe_shared(s, '<style></style>'))
+        self.assertTrue(repr_utils.write_maybe_shared(s, '<style>a</style>'))
+        self.assertFalse(repr_utils.write_maybe_shared(s, '<style>a</style>'))
+        self.assertTrue(repr_utils.write_maybe_shared(s, '<style>b</style>'))
+
+      with repr_utils.share_parts() as ctx3:
+        self.assertIs(ctx3, ctx1)
+        self.assertFalse(repr_utils.write_maybe_shared(s, '<style></style>'))
+        self.assertFalse(repr_utils.write_maybe_shared(s, '<style>a</style>'))
+        self.assertFalse(repr_utils.write_maybe_shared(s, '<style>a</style>'))
+        self.assertFalse(repr_utils.write_maybe_shared(s, '<style>b</style>'))
+
+    self.assertEqual(
+        s.getvalue(),
+        '<hr><hr><style></style><style>a</style><style>b</style>'
+    )
+    self.assertEqual(ctx1['<style></style>'], 4)
+    self.assertEqual(ctx1['<style>b</style>'], 2)
+    self.assertEqual(ctx1['<style>a</style>'], 4)
+
+
+if __name__ == '__main__':
+  unittest.main()

langfun/core/templates/selfplay_test.py

@@ -57,7 +57,10 @@ class SelfPlayTest(unittest.TestCase):
 
     with lf.context(lm=NumberGuesser(guesses=[50, 20, 5, 10])):
       self.assertEqual(
-          g(), lf.AIMessage('10', score=0.0, logprobs=None, usage=None)
+          g(),
+          lf.AIMessage(
+              '10', score=0.0, logprobs=None, usage=lf.UsageNotAvailable()
+          )
       )
 
     self.assertEqual(g.num_turns, 4)
@@ -67,7 +70,10 @@ class SelfPlayTest(unittest.TestCase):
 
     with lf.context(lm=NumberGuesser(guesses=[50, 20, 5, 2, 5, 4])):
       self.assertEqual(
-          g(), lf.AIMessage('2', score=0.0, logprobs=None, usage=None)
+          g(),
+          lf.AIMessage(
+              '2', score=0.0, logprobs=None, usage=lf.UsageNotAvailable()
+          )
       )
 
     self.assertEqual(g.num_turns, 10)

{langfun-0.0.2.dev20240607.dist-info → langfun-0.0.2.dev20240609.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: langfun
-Version: 0.0.2.dev20240607
+Version: 0.0.2.dev20240609
 Summary: Langfun: Language as Functions.
 Home-page: https://github.com/google/langfun
 Author: Langfun Authors

{langfun-0.0.2.dev20240607.dist-info → langfun-0.0.2.dev20240609.dist-info}/RECORD

@@ -1,5 +1,5 @@
 langfun/__init__.py,sha256=P62MnqA6-f0h8iYfQ3MT6Yg7a4qRnQeb4GrIn6dcSnY,2274
-langfun/core/__init__.py,sha256=F3WGAww--u0CdQkg4ENBudsd0ZeGdecscF0R3YXSWmE,4670
+langfun/core/__init__.py,sha256=Mdp1a2YnXdSmfTfbUwuAnEWYbjA3rXXGtbxl5fljZyg,4812
 langfun/core/component.py,sha256=Icyoj9ICoJoK2r2PHbrFXbxnseOr9QZZOvKWklLWNo8,10276
 langfun/core/component_test.py,sha256=q15Xn51cVTu2RKxZ9U5VQgT3bm6RQ4638bKhWBtvW5o,8220
 langfun/core/concurrent.py,sha256=TRc49pJ3HQro2kb5FtcWkHjhBm8UcgE8RJybU5cU3-0,24537
@@ -7,9 +7,9 @@ langfun/core/concurrent_test.py,sha256=mwFMZhDUdppnDr7vDSTwcbMHwrdsIoKJwRYNtl4ZW
 langfun/core/console.py,sha256=bk5rNPNm9rMGW5YT2HixxU04p2umnoabn5SDz6Dqe88,2317
 langfun/core/console_test.py,sha256=5SYJdxpJGLgdSSQqqMPoA1X6jpsLD8rgcyk-EgI65oE,1077
 langfun/core/langfunc.py,sha256=RvIcRjIq0jWYRu1xim-FYe4HSrt97r3GMBO_PuagUmw,11060
-langfun/core/langfunc_test.py,sha256=_mfARnakX3oji5HDigFSLMd6yQ2wma-2Mgbztwqn73g,8501
-langfun/core/language_model.py,sha256=geHX9bHLjWGlubgGvSLTGluFyB7x_NIk09p5KPkp1Wg,22328
-langfun/core/language_model_test.py,sha256=UfAg7ExCWSVqFpFGFOvQWSwO343FcvzVImb_F3hTfa0,20517
+langfun/core/langfunc_test.py,sha256=lyt-UzkD8972cxZwzCkps0_RMLeSsOBrcUFIW-fB6us,8653
+langfun/core/language_model.py,sha256=q41EGhbgoCe68eYWWTAzgt3r6SbdCpINRdXnNGstukQ,23775
+langfun/core/language_model_test.py,sha256=HmA0GACK4-6tCH32TFkfYj9w4CxbrynKqXdKBgiqgwo,21255
 langfun/core/logging.py,sha256=FyZRxUy2TTF6tWLhQCRpCvfH55WGUdNgQjUTK_SQLnY,5320
 langfun/core/logging_test.py,sha256=qvm3RObYP3knO2PnXR9evBRl4gH621GnjnwywbGbRfg,1833
 langfun/core/memory.py,sha256=f-asN1F7Vehgdn_fK84v73GrEUOxRtaW934keutTKjk,2416
@@ -19,6 +19,8 @@ langfun/core/modality.py,sha256=Tla4t86DUYHpbZ2G7dy1r19fTj_Ga5XOvlYp6lbWa-Q,3512
 langfun/core/modality_test.py,sha256=HyZ5xONKQ0Fw18SzoWAq-Ob9njOXIIjBo1hNtw-rudw,2400
 langfun/core/natural_language.py,sha256=3ynSnaYQnjE60LIPK5fyMgdIjubnPYZwzGq4rWPeloE,1177
 langfun/core/natural_language_test.py,sha256=LHGU_1ytbkGuSZQFIFP7vP3dBlcY4-A12fT6dbjUA0E,1424
+langfun/core/repr_utils.py,sha256=HrN7FoGUvpTlv5aL_XISouwZN84z9LmrB6_2jEn1ukc,2590
+langfun/core/repr_utils_test.py,sha256=-XId1A72Vbzo289dYuxC6TegNXuZhI28WbNrm1ghiwc,2206
 langfun/core/sampling.py,sha256=vygWvgC8MFw0_AKNSmz-ywMXJYWf8cl0tI8QycvAmyI,5795
 langfun/core/sampling_test.py,sha256=U7PANpMsl9E_pa4_Y4FzesSjcwg-u-LKHGCWSgv-8FY,3663
 langfun/core/subscription.py,sha256=euawEuSZP-BHydaT-AQpfYFL0m5pWPGcW0upFhrojqc,10930
@@ -42,7 +44,7 @@ langfun/core/coding/python/parsing_test.py,sha256=9vAWF484kWIm6JZq8NFiMgKUDhXV-d
 langfun/core/coding/python/permissions.py,sha256=1QWGHvzL8MM0Ok_auQ9tURqZHtdOfJaDpBzZ29GUE-c,2544
 langfun/core/coding/python/permissions_test.py,sha256=w5EDb8QxpxgJyZkojyzVWQvDfg366zn99-g__6TbPQ0,2699
 langfun/core/eval/__init__.py,sha256=Evt-E4FEhZF2tXL6-byh_AyA7Cc_ZoGmvnN7vkAZedk,1898
-langfun/core/eval/base.py,sha256=zcMPBKmcll5O08waEEnvmkEoXgcINhOat9rRJk8X8b4,74268
+langfun/core/eval/base.py,sha256=GM98Zo4gxZui2ORX6Q7Zr94PfiEViQC5X_qz-uj6b2k,74220
 langfun/core/eval/base_test.py,sha256=cHOTIWVW4Dp8gKKIKcZrAcJ-w84j2GIozTzJoiAX7p4,26743
 langfun/core/eval/matching.py,sha256=Y4vFoNTQEOwko6IA8l9OZ52-vt52e3VGmcTtvLA67wM,9782
 langfun/core/eval/matching_test.py,sha256=f7iVyXH5KGJBWt4Wp14Bt9J3X59A6Ayfog9MbuFvPew,5532
@@ -61,10 +63,10 @@ langfun/core/llms/groq.py,sha256=pqtyOZ_1_OJMOg8xATWT_B_SVbuT9nMRf4VkH9GzW8g,630
 langfun/core/llms/groq_test.py,sha256=GYF_Qtq5S1H1TrKH38t6_lkdroqT7v-joYLDKnmS9e0,5274
 langfun/core/llms/llama_cpp.py,sha256=9tXQntSCDtjTF3bnyJrAPCr4N6wycy5nXYvp9uduygE,2843
 langfun/core/llms/llama_cpp_test.py,sha256=MWO_qaOeKjRniGjcaWPDScd7HPaIJemqUZoslrt4FPs,1806
-langfun/core/llms/openai.py,sha256=IN46gIqfY6aEEfxCPNmyH1hrep3oWBhJDwVFilfqNkM,13657
-langfun/core/llms/openai_test.py,sha256=QWDzTgi8F2Z9u9ip6alK4rDEp_YraVTxWlDX5XOsKJk,14858
+langfun/core/llms/openai.py,sha256=0z9qIH9FlWj9VWUnhOX321T6JHO-vjY2IozT7OVI4GY,13654
+langfun/core/llms/openai_test.py,sha256=3muDTnW7UBOSHq694Fi2bofqhe8Pkj0Tl8IShoLCTOM,15525
 langfun/core/llms/rest.py,sha256=laopuq-zD8V-3Y6eFDngftHEbE66VlUkCD2-rvvRaLU,3388
-langfun/core/llms/rest_test.py,sha256=Zw08Xbl_O2OQuUglLXHsPsY5KW2VEcPGl1gR8PRHuFY,3449
+langfun/core/llms/rest_test.py,sha256=NZ3Nf0XQVpT9kLP5cBVo_yBHLI7vWTYhWQxYEJVMGs4,3472
 langfun/core/llms/vertexai.py,sha256=wIpckH-rMHUBA1vhauQk4LVrSsPQEsVntz7kLDKwm9g,11359
 langfun/core/llms/vertexai_test.py,sha256=G18BG36h5KvmX2zutDTLjtYCRjTuP_nWIFm4FMnLnyY,7651
 langfun/core/llms/cache/__init__.py,sha256=QAo3InUMDM_YpteNnVCSejI4zOsnjSMWKJKzkb3VY64,993
@@ -114,9 +116,9 @@ langfun/core/templates/conversation_test.py,sha256=RryYyIhfc34dLWOs6GfPQ8HU8mXpK
 langfun/core/templates/demonstration.py,sha256=vCrgYubdZM5Umqcgp8NUVGXgr4P_c-fikKhwhzwhpKI,1460
 langfun/core/templates/demonstration_test.py,sha256=SafcDQ0WgI7pw05EmPI2S4v1t3ABKzup8jReCljHeK4,2162
 langfun/core/templates/selfplay.py,sha256=yhgrJbiYwq47TgzThmHrDQTF4nDrTI09CWGhuQPNv-s,2273
-langfun/core/templates/selfplay_test.py,sha256=DYVrkk7uNKCqJGEHH31HssU2BPuMItU1vJLzfcXIlYg,2156
-langfun-0.0.2.dev20240607.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-langfun-0.0.2.dev20240607.dist-info/METADATA,sha256=rFmOTbbcV4Jlik6ma8c87fUqXmYpscGfb3_FcMN2Nf0,3550
-langfun-0.0.2.dev20240607.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-langfun-0.0.2.dev20240607.dist-info/top_level.txt,sha256=RhlEkHxs1qtzmmtWSwYoLVJAc1YrbPtxQ52uh8Z9VvY,8
-langfun-0.0.2.dev20240607.dist-info/RECORD,,
+langfun/core/templates/selfplay_test.py,sha256=rBW2Qr8yi-aWYwoTwRR-n1peKyMX9QXPZXURjLgoiRs,2264
+langfun-0.0.2.dev20240609.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+langfun-0.0.2.dev20240609.dist-info/METADATA,sha256=uyk0OoHbseBguRcn-EDVaDAun1goszVD5Dt1WO1BbZc,3550
+langfun-0.0.2.dev20240609.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+langfun-0.0.2.dev20240609.dist-info/top_level.txt,sha256=RhlEkHxs1qtzmmtWSwYoLVJAc1YrbPtxQ52uh8Z9VvY,8
+langfun-0.0.2.dev20240609.dist-info/RECORD,,