langfun 0.0.2.dev20240201__py3-none-any.whl → 0.0.2.dev20240203__py3-none-any.whl

langfun/core/eval/base.py

@@ -57,6 +57,10 @@ class Evaluable(lf.Component):
       ),
   ] = lf.contextual(default=None)
 
+  report_precision: Annotated[
+      int, 'Number of decimals when reporting precision.'
+  ] = lf.contextual(default=1)
+
   @property
   def dir(self) -> str | None:
     """Returns the directory for saving results and details."""
@@ -1045,12 +1049,12 @@ class Evaluation(Evaluable):
   def _status(self, progress: lf.concurrent.Progress) -> dict[str, Any]:
     return {
         'Model': self.lm.model_id,
-        'Succeeded': '%.2f%% (%d/%d)' % (
+        'Succeeded': f'%.{self.report_precision}f%% (%d/%d)' % (
             progress.success_rate * 100,
             progress.succeeded,
             progress.completed,
         ),
-        'Failed': '%.2f%% (%d/%d)' % (
+        'Failed': f'%.{self.report_precision}f%% (%d/%d)' % (
             progress.failure_rate * 100,
             progress.failed,
             progress.completed,
@@ -1060,14 +1064,18 @@ class Evaluation(Evaluable):
   def _completion_status(self, run_status: str) -> str:
     assert self.result is not None
     m = self.result.metrics
-    return 'COMPLETED(%s): Successes=%.2f%% (%d/%d) Failures=%.2f%% (%d/%d)' % (
-        run_status,
-        (1 - m.failure_rate) * 100,
-        m.total - m.failures,
-        m.total,
-        m.failure_rate * 100,
-        m.failures,
-        m.total,
+    return (
+        f'COMPLETED(%s): Successes=%.{self.report_precision}f%% (%d/%d)'
+        f' Failures=%.{self.report_precision}f%% (%d/%d)'
+        % (
+            run_status,
+            (1 - m.failure_rate) * 100,
+            m.total - m.failures,
+            m.total,
+            m.failure_rate * 100,
+            m.failures,
+            m.total,
+        )
     )
 
   def summarize(self) -> pg.Dict:
@@ -1130,7 +1138,7 @@ class Evaluation(Evaluable):
             m.failures,
             m.total,
             self.failures_link,
-            '%.2f%% ' % (m.failure_rate * 100),
+            f'%.{self.report_precision}f%% ' % (m.failure_rate * 100),
         )
     )
 
@@ -1218,7 +1226,7 @@ class Evaluation(Evaluable):
     s.write(
         '<td><span style="color:orange">%s</span>%s</td>'
         % (
-            '%.2f%%' % (self.failure_rate * 100),
+            f'%.{self.report_precision}f%%' % (self.failure_rate * 100),
             '<a href="%s">(%d/%d)</a>'
             % (self.failures_link, self.num_failures, self.num_completed),
         )

langfun/core/eval/base_test.py

@@ -101,7 +101,7 @@ class EvaluationTest(unittest.TestCase):
     self.assertEqual(s.dir, os.path.join(s.root_dir, s.id))
     self.assertEqual(s.hash, s.clone().hash)
     # Test persistent hash.
-    self.assertEqual(s.hash, 'c76d4fe6')
+    self.assertEqual(s.hash, 'abc7c29a')
     self.assertEqual(
         s.hash, s.clone(override={'max_workers': 2, 'lm.timeout': 20}).hash
     )
@@ -195,6 +195,7 @@ class EvaluationTest(unittest.TestCase):
             result=Solution(2),
             cache_seed=0,
             score=1.0,
+            logprobs=None,
             tags=['lm-response', 'lm-output', 'transformed'],
         ),
     )
@@ -323,7 +324,7 @@ class EvaluationTest(unittest.TestCase):
         s.children[0].dir, os.path.join(s.root_dir, s.children[0].id)
     )
     # Test persistent hash.
-    self.assertEqual(s.hash, 'e987475a')
+    self.assertEqual(s.hash, 'ca7f722b')
 
     summary = s.run(verbose=True)
     self.assertEqual(len(summary.evaluations), 2)
@@ -451,7 +452,7 @@ class SuiteTest(unittest.TestCase):
         ],
     )
     # Test for persistent hash.
-    self.assertEqual(s.hash, 'bb86a963')
+    self.assertEqual(s.hash, '7285e52b')
     s.run()
     expected = {
         s.children[0].id: dict(

langfun/core/eval/matching.py

@@ -102,17 +102,17 @@ class Matching(base.Evaluation):
     del progress
     return {
         'Model': self.lm.model_id,
-        'Matches': '%.2f%% (%d/%d)' % (
+        'Matches': f'%.{self.report_precision}f%% (%d/%d)' % (
             self.match_rate * 100,
             self.num_matches,
             self.num_completed,
         ),
-        'Mismatches': '%.2f%% (%d/%d)' % (
+        'Mismatches': f'%.{self.report_precision}f%% (%d/%d)' % (
             self.mismatch_rate * 100,
             self.num_mismatches,
             self.num_completed,
         ),
-        'Failed': '%.2f%% (%d/%d)' % (
+        'Failed': f'%.{self.report_precision}f%% (%d/%d)' % (
             self.failure_rate * 100,
             self.num_failures,
             self.num_completed,
@@ -123,8 +123,9 @@ class Matching(base.Evaluation):
     assert self.result is not None
     m = self.result.metrics
     return (
-        'COMPLETED(%s): Matches=%.2f%% (%d/%d) Mismatches=%.2f%% (%d/%d) '
-        'Failures=%.2f%% (%d/%d)'
+        f'COMPLETED(%s): Matches=%.{self.report_precision}f%% (%d/%d)'
+        f' Mismatches=%.{self.report_precision}f%% (%d/%d)'
+        f' Failures=%.{self.report_precision}f%% (%d/%d)'
     ) % (
         run_status,
         m.match_rate * 100,
@@ -202,7 +203,7 @@ class Matching(base.Evaluation):
     s.write(
         '<td><span style="color:red">%s</span>%s</td>'
         % (
-            '%.2f%% ' % (self.mismatch_rate * 100),
+            f'%.{self.report_precision}f%% ' % (self.mismatch_rate * 100),
             '<a href="%s">(%d/%d)</a>'
             % (self.mismatches_link, self.num_mismatches, self.num_completed),
         )
@@ -210,7 +211,7 @@ class Matching(base.Evaluation):
     s.write(
         '<td><span style="color:green">%s</span>%s</td>'
         % (
-            '%.2f%% ' % (self.match_rate * 100),
+            f'%.{self.report_precision}f%% ' % (self.match_rate * 100),
             '<a href="%s">(%d/%d)</a>'
             % (self.matches_link, self.num_matches, self.num_completed),
         )
@@ -226,7 +227,7 @@ class Matching(base.Evaluation):
             m.num_matches,
             m.total,
             self.matches_link,
-            '%.2f%% ' % (m.match_rate * 100),
+            f'%.{self.report_precision}f%% ' % (m.match_rate * 100),
         )
     )
     s.write(' | ')
@@ -236,7 +237,7 @@ class Matching(base.Evaluation):
             m.num_mismatches,
             m.total,
             self.mismatches_link,
-            '%.2f%% ' % (m.mismatch_rate * 100),
+            f'%.{self.report_precision}f%% ' % (m.mismatch_rate * 100),
         )
     )
     s.write(' | ')

langfun/core/langfunc_test.py

@@ -82,7 +82,7 @@ class LangFuncCallTest(unittest.TestCase):
     self.assertEqual(i.tags, ['rendered'])
 
     r = l()
-    self.assertEqual(r, message.AIMessage('Hello!!!', score=0.0))
+    self.assertEqual(r, message.AIMessage('Hello!!!', score=0.0, logprobs=None))
     self.assertEqual(r.tags, ['lm-response', 'lm-output'])
     self.assertEqual(r.source, message.UserMessage('Hello'))
     self.assertEqual(r.source.tags, ['rendered', 'lm-input'])
@@ -94,8 +94,9 @@ class LangFuncCallTest(unittest.TestCase):
         "LangFunc(template_str='Hello', clean=True,"
         ' lm=ExcitedEchoer(sampling_options=LMSamplingOptions(temperature=0.0,'
         ' max_tokens=1024, n=1, top_k=40, top_p=None, stop=None,'
-        ' random_seed=None), cache=None, timeout=120.0, max_attempts=5,'
-        ' retry_interval=(5, 60), exponential_backoff=True, debug=False))',
+        ' random_seed=None, logprobs=False, top_logprobs=None), cache=None,'
+        ' timeout=120.0, max_attempts=5, retry_interval=(5, 60),'
+        ' exponential_backoff=True, debug=False))',
     )
 
     l = LangFunc('Hello')
@@ -104,7 +105,9 @@ class LangFuncCallTest(unittest.TestCase):
       self.assertEqual(l.natural_language_format(), 'Hello')
       self.assertEqual(l.render(), 'Hello')
       r = l()
-      self.assertEqual(r, message.AIMessage('Hello!!!', score=0.0))
+      self.assertEqual(
+          r, message.AIMessage('Hello!!!', score=0.0, logprobs=None)
+      )
       self.assertEqual(r.tags, ['lm-response', 'lm-output'])
 
     self.assertEqual(str(l), 'Hello')

langfun/core/language_model.py

@@ -31,15 +31,20 @@ class LMSample(pg.Object):
       pg.typing.Object(
           message_lib.Message,
           # Allowing automatic conversion from text to AIMessage.
-          transform=message_lib.AIMessage.from_value
+          transform=message_lib.AIMessage.from_value,
       ),
-      'The natural language response of LM.'
+      'The natural language response of LM.',
   ]
 
   score: Annotated[
       float, 'The score of sampled response. The larger is better'
   ] = 0.0
 
+  logprobs: Annotated[
+      list[tuple[str, float, list[tuple[str, float]]]] | None,
+      '(token, log prob, top tokens and their probs).',
+  ] = None
+
 
 class LMSamplingResult(pg.Object):
   """Language model response."""
@@ -92,6 +97,23 @@ class LMSamplingOptions(component.Component):
   random_seed: Annotated[
       int | None, 'A fixed random seed used during model inference.'
   ] = None
+  logprobs: Annotated[
+      bool,
+      (
+          'Whether to return log probabilities of the output tokens or not. If '
+          'true, returns the log probabilities of each output token returned '
+          'in the content of message.'
+      ),
+  ] = False
+  top_logprobs: Annotated[
+      int | None,
+      (
+          'An integer between 0 and 5 specifying the number of most likely '
+          'tokens to return at each token position, each with an associated '
+          'log probability. logprobs must be set to true if this parameter is '
+          'used.'
+      ),
+  ] = None
 
   def cache_key(self) -> tuple[Any, ...]:
     """Returns a tuple of current values as cache key."""
@@ -339,7 +361,9 @@ class LanguageModel(component.Component):
           [prompt], sampling_options=sampling_options, cache_seed=cache_seed
       )[0]
       response = result.samples[0].response
+      logprobs = result.samples[0].logprobs
       response.set('score', result.samples[0].score)
+      response.metadata.logprobs = logprobs
       elapse = time.time() - request_start
       self._debug(prompt, response, call_counter, elapse)
       return response

langfun/core/llms/openai.py

@@ -44,29 +44,32 @@ SUPPORTED_MODELS_AND_SETTINGS = [
     # Model name, max concurrent requests.
     # The concurrent requests is estimated by TPM/RPM from
     # https://platform.openai.com/account/limits
-    # GPT4 Turbo models.
-    ('gpt-4-1106-preview', 1),  # Gpt4 Turbo.
-    ('gpt-4-vision-preview', 1),  # Gpt4 Turbo with Vision.
-    # GPT4 models.
+    # GPT-4 Turbo models.
+    ('gpt-4-turbo-preview', 1),  # GPT-4 Turbo.
+    ('gpt-4-0125-preview', 1),  # GPT-4 Turbo
+    ('gpt-4-1106-preview', 1),  # GPT-4 Turbo
+    ('gpt-4-vision-preview', 1),  # GPT-4 Turbo with Vision.
+    # GPT-4 models.
     ('gpt-4', 4),
     ('gpt-4-0613', 4),
     ('gpt-4-0314', 4),
     ('gpt-4-32k', 4),
     ('gpt-4-32k-0613', 4),
     ('gpt-4-32k-0314', 4),
-    # GPT3.5 Turbo models.
+    # GPT-3.5 Turbo models.
     ('gpt-3.5-turbo', 16),
+    ('gpt-3.5-turbo-0125', 16),
     ('gpt-3.5-turbo-1106', 16),
     ('gpt-3.5-turbo-0613', 16),
     ('gpt-3.5-turbo-0301', 16),
     ('gpt-3.5-turbo-16k', 16),
     ('gpt-3.5-turbo-16k-0613', 16),
     ('gpt-3.5-turbo-16k-0301', 16),
-    # GPT3.5 models.
-    ('text-davinci-003', 8),  # Gpt3.5, trained with RHLF.
+    # GPT-3.5 models.
+    ('text-davinci-003', 8),  # GPT-3.5, trained with RHLF.
     ('text-davinci-002', 4),  # Trained with SFT but no RHLF.
     ('code-davinci-002', 4),
-    # GPT3 instruction-tuned models.
+    # GPT-3 instruction-tuned models.
     ('text-curie-001', 4),
     ('text-babbage-001', 4),
     ('text-ada-001', 4),
@@ -74,7 +77,7 @@ SUPPORTED_MODELS_AND_SETTINGS = [
     ('curie', 4),
     ('babbage', 4),
     ('ada', 4),
-    # GPT3 base models without instruction tuning.
+    # GPT-3 base models without instruction tuning.
     ('babbage-002', 4),
     ('davinci-002', 4),
 ]
@@ -164,6 +167,8 @@ class OpenAI(lf.LanguageModel):
         max_tokens=options.max_tokens,
         stream=False,
         timeout=self.timeout,
+        logprobs=options.logprobs,
+        top_logprobs=options.top_logprobs,
     )
     # Completion and ChatCompletion uses different parameter name for model.
     args['model' if self.is_chat_model else 'engine'] = self.model
@@ -246,11 +251,28 @@ class OpenAI(lf.LanguageModel):
           **self._get_request_args(self.sampling_options),
       )
       response = cast(openai_object.OpenAIObject, response)
+      samples = []
+      for choice in response.choices:
+        logprobs = None
+        if choice.logprobs:
+          logprobs = [
+              (
+                  t.token,
+                  t.logprob,
+                  [(tt.token, tt.logprob) for tt in t.top_logprobs],
+              )
+              for t in choice.logprobs.content
+          ]
+        samples.append(
+            lf.LMSample(
+                choice.message.content,
+                score=0.0,
+                logprobs=logprobs,
+            )
+        )
+
       return LMSamplingResult(
-          [
-              lf.LMSample(choice.message.content, score=0.0)
-              for choice in response.choices
-          ],
+          samples=samples,
           usage=Usage(
               prompt_tokens=response.usage.prompt_tokens,
               completion_tokens=response.usage.completion_tokens,

langfun/core/llms/openai_test.py

@@ -46,7 +46,8 @@ def mock_chat_completion_query(messages, *, n=1, **kwargs):
     choices.append(pg.Dict(
         message=pg.Dict(
             content=f'Sample {k} for message.'
-        )
+        ),
+        logprobs=None,
     ))
   return pg.Dict(choices=choices, usage=openai.Usage(
       prompt_tokens=100,
@@ -65,7 +66,8 @@ def mock_chat_completion_query_vision(messages, *, n=1, **kwargs):
     choices.append(pg.Dict(
         message=pg.Dict(
             content=f'Sample {k} for message: {"".join(urls)}'
-        )
+        ),
+        logprobs=None,
     ))
   return pg.Dict(choices=choices, usage=openai.Usage(
       prompt_tokens=100,
@@ -99,6 +101,8 @@ class OpenaiTest(unittest.TestCase):
                 top_p=1.0)),
         dict(
             engine='text-davinci-003',
+            logprobs=False,
+            top_logprobs=None,
             n=2,
             temperature=2.0,
             max_tokens=4096,
@@ -113,6 +117,8 @@ class OpenaiTest(unittest.TestCase):
         ),
         dict(
             model='gpt-4',
+            logprobs=False,
+            top_logprobs=None,
             n=1,
             temperature=1.0,
             max_tokens=1024,

langfun/core/structured/completion_test.py

@@ -582,6 +582,7 @@ class CompleteStructureTest(unittest.TestCase):
             text='Activity(description="foo")',
             result=Activity(description='foo'),
             score=1.0,
+            logprobs=None,
             tags=['lm-response', 'lm-output', 'transformed']
         )
     )

langfun/core/structured/parsing_test.py

@@ -286,7 +286,7 @@ class ParseStructurePythonTest(unittest.TestCase):
             returns_message=True
         ),
         lf.AIMessage(
-            '1', score=1.0, result=1,
+            '1', score=1.0, result=1, logprobs=None,
             tags=['lm-response', 'lm-output', 'transformed']
         ),
     )
@@ -640,7 +640,7 @@ class CallTest(unittest.TestCase):
             returns_message=True
         ),
         lf.AIMessage(
-            '3', result=3, score=1.0,
+            '3', result=3, score=1.0, logprobs=None,
             tags=['lm-response', 'lm-output', 'transformed']
         ),
     )

langfun/core/structured/prompting_test.py

@@ -76,6 +76,7 @@ class QueryTest(unittest.TestCase):
             '1',
             result=1,
             score=1.0,
+            logprobs=None,
             tags=['lm-response', 'lm-output', 'transformed'],
         ),
     )

langfun/core/templates/selfplay_test.py

@@ -56,7 +56,7 @@ class SelfPlayTest(unittest.TestCase):
     g = NumberGuess(target_num=10)
 
     with lf.context(lm=NumberGuesser(guesses=[50, 20, 5, 10])):
-      self.assertEqual(g(), lf.AIMessage('10', score=0.0))
+      self.assertEqual(g(), lf.AIMessage('10', score=0.0, logprobs=None))
 
     self.assertEqual(g.num_turns, 4)
 
@@ -64,7 +64,7 @@ class SelfPlayTest(unittest.TestCase):
     g = NumberGuess(target_num=10, max_turns=10)
 
     with lf.context(lm=NumberGuesser(guesses=[50, 20, 5, 2, 5, 4])):
-      self.assertEqual(g(), lf.AIMessage('2', score=0.0))
+      self.assertEqual(g(), lf.AIMessage('2', score=0.0, logprobs=None))
 
     self.assertEqual(g.num_turns, 10)
 

{langfun-0.0.2.dev20240201.dist-info → langfun-0.0.2.dev20240203.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: langfun
-Version: 0.0.2.dev20240201
+Version: 0.0.2.dev20240203
 Summary: Langfun: Language as Functions.
 Home-page: https://github.com/google/langfun
 Author: Langfun Authors

{langfun-0.0.2.dev20240201.dist-info → langfun-0.0.2.dev20240203.dist-info}/RECORD

@@ -7,8 +7,8 @@ langfun/core/concurrent_test.py,sha256=qQT6_Dq5NVz7qXFLzSf2Rhzkfkh07gocjHMBaT1nS
 langfun/core/console.py,sha256=bk5rNPNm9rMGW5YT2HixxU04p2umnoabn5SDz6Dqe88,2317
 langfun/core/console_test.py,sha256=5SYJdxpJGLgdSSQqqMPoA1X6jpsLD8rgcyk-EgI65oE,1077
 langfun/core/langfunc.py,sha256=266xNz8Vgal7K4HSsrYt7z7_qPYV4bWWK626IbbohrE,11573
-langfun/core/langfunc_test.py,sha256=rnU-7JQcY-fIXB5aoKbouPd2LkjdzIjB5g_aLPBHQCE,8069
-langfun/core/language_model.py,sha256=UppuqiwmZ6AfWCNkfom12XqEu1uf8qBB3yp-xezzQ0s,12056
+langfun/core/langfunc_test.py,sha256=ukv5cnad5ZBckM2PhyIFq79BPN0Db4cszMrPqh_CZkA,8163
+langfun/core/language_model.py,sha256=HG2XQhisyiyvMBws5dASH1OkbGJB3twn0p2e_As8hUo,12889
 langfun/core/language_model_test.py,sha256=gcW4OJJjB-V1b4kEF8zG91t36sVn3H0Yuj0LQxi83Ek,9122
 langfun/core/memory.py,sha256=f-asN1F7Vehgdn_fK84v73GrEUOxRtaW934keutTKjk,2416
 langfun/core/message.py,sha256=QhvV9t5qaryPcruyxxcXi3gm9QDInkSldwTtK6sVJ3c,15734
@@ -40,9 +40,9 @@ langfun/core/coding/python/parsing_test.py,sha256=9vAWF484kWIm6JZq8NFiMgKUDhXV-d
 langfun/core/coding/python/permissions.py,sha256=1QWGHvzL8MM0Ok_auQ9tURqZHtdOfJaDpBzZ29GUE-c,2544
 langfun/core/coding/python/permissions_test.py,sha256=w5EDb8QxpxgJyZkojyzVWQvDfg366zn99-g__6TbPQ0,2699
 langfun/core/eval/__init__.py,sha256=iDA2OcJ3kR6ixZizXIY3N9LsjkaVrfTbSClTiSP8ekY,1291
-langfun/core/eval/base.py,sha256=Wa8kqGQJK2R4pvdCFKB4Mmlcfk4c4qGZn_fl_wy9H-Q,52276
-langfun/core/eval/base_test.py,sha256=bGs3VLchkAJFWYJ8FdR7mC6qoDestAvCHOQpClG6Mzw,21248
-langfun/core/eval/matching.py,sha256=INjG-EU-hQa6zDk2_bAbWWAQnxN4lor-Dnas8i2CTAo,9326
+langfun/core/eval/base.py,sha256=wWFDDrf0jBzs9H_5XfdZSeOBGXyUtXAJJouk7cLckSM,52602
+langfun/core/eval/base_test.py,sha256=uXu6EtTagolDIcSadnVyMmlrz6ixx943jkZhquCRQPI,21275
+langfun/core/eval/matching.py,sha256=g2yuBb4FeOlAlB10hqdWvaIg4QVQlJbiViRDcD2Y8go,9567
 langfun/core/eval/matching_test.py,sha256=IfuMF_dEmy4VzK6tIldRzD2Nqlml7SSh4u-baFNcZrw,4912
 langfun/core/eval/scoring.py,sha256=mshqbV_WM0zcp15TSR32ACMBDymlsbf6YH06PPx1Tw0,6139
 langfun/core/eval/scoring_test.py,sha256=_L_B40VZkyI2_PJce-jVKYC4llrO4jGUR5j86Gu6AT0,4046
@@ -53,8 +53,8 @@ langfun/core/llms/gemini.py,sha256=p3d4Cl2uET-os1n_V3YNE6-6cYrZjndj7lxZIk2E8_4,5
 langfun/core/llms/gemini_test.py,sha256=ybNNCn3JW3hYpMe0wT5ILGDrMPaYYU8PN2kSookM0jk,5433
 langfun/core/llms/llama_cpp.py,sha256=EIjJa1-Tg4_VaIxVR88oDWSWc_axc1r2KwSPpl4PSp0,2549
 langfun/core/llms/llama_cpp_test.py,sha256=ZxC6defGd_HX9SFRU9U4cJiQnBKundbOrchbXuC1Z2M,1683
-langfun/core/llms/openai.py,sha256=1wJohy6UZVBiZkbnKsD2Q_qt6-x_9KUnlRALd1TFV9E,10985
-langfun/core/llms/openai_test.py,sha256=JWcMveifVVVEFWdtmNq1irc9wSFQRxXs-SnOF3Urg9Y,7433
+langfun/core/llms/openai.py,sha256=ufhRf-fV-AFq_-pn5sKrcr_xIU7V0VnhjK3JfBG8sF8,11617
+langfun/core/llms/openai_test.py,sha256=yfw7A-4Zo9u1cIkAMk39evE-tO7z6isNYTXiSnJXDQw,7599
 langfun/core/llms/cache/__init__.py,sha256=QAo3InUMDM_YpteNnVCSejI4zOsnjSMWKJKzkb3VY64,993
 langfun/core/llms/cache/base.py,sha256=cFfYvOIUae842pncqCAsRvqXCk2AnAsRYVx0mcIoAeY,3338
 langfun/core/llms/cache/in_memory.py,sha256=YfFyJEhLs73cUiB0ZfhMxYpdE8Iuxxw-dvMFwGHTSHw,4742
@@ -67,15 +67,15 @@ langfun/core/modalities/image.py,sha256=HU0sV4ZTwRnAwQthmdWZwhFZRD86RyvqoS8JUW2I
 langfun/core/modalities/image_test.py,sha256=YxDRvC49Bjwyyndd_P7y6XjyS7dOft0Zewwxk-7q4kE,2301
 langfun/core/structured/__init__.py,sha256=tGH0MYr5vzK0H2DpYQ2bcW2C5bpPUaLzMk2W2Fj29M4,3136
 langfun/core/structured/completion.py,sha256=XERoxtYPXOTlPdZ2bp4i9R4jl3kA3SOeyLmuSqHG9AM,7036
-langfun/core/structured/completion_test.py,sha256=i68c_eK-QJIAW964E2o4W8kNj2ZokEQOPesG5Hw78-E,19222
+langfun/core/structured/completion_test.py,sha256=98UCgA4gzfp6H6HgP2s2kcKs25YH3k4Nxj1rgAvmVBw,19249
 langfun/core/structured/description.py,sha256=vDiW1g2VbvG8ucNjV7Pp3VYCeAnLcp6vLQ0MfURcZFk,4825
 langfun/core/structured/description_test.py,sha256=UtZGjSFUaQ6130t1E5tcL7ODu0xIefkapb53TbnqsK8,7362
 langfun/core/structured/mapping.py,sha256=lGkjhmvVdhBGgJmc5KbfT2xQjC1MuU4OCcCfsAYJjaQ,10192
 langfun/core/structured/mapping_test.py,sha256=07DDCGbwytQHSMm7fCi5-Ly-JNgdV4ubHZq0wthX4A4,3338
 langfun/core/structured/parsing.py,sha256=V3i8-AuBI4zdpsi_f6H-mbQ3h0HPgCle2OnDQolWJnA,10244
-langfun/core/structured/parsing_test.py,sha256=EC9i4hQkcmoneBX7p1fRIpgznB8lgGHFwtKvHlNrwyA,20816
+langfun/core/structured/parsing_test.py,sha256=2_Uf3LYNRON1-5ysEr75xiG_cAxR3ZiixSfvUQu6mOQ,20846
 langfun/core/structured/prompting.py,sha256=OmI21qQQikoQLAmr5W8nBJ8PWCu9w0lmsCnmbQg9hMc,6632
-langfun/core/structured/prompting_test.py,sha256=LOOka3CaID03SXPOSNHVFrnk7Ymkt6GvSDS9pNa-y3M,19116
+langfun/core/structured/prompting_test.py,sha256=8lakKCidYuRlf-U1KexTOqQdKrBXUt8fb2J7epCdt84,19143
 langfun/core/structured/schema.py,sha256=5DKba0LrvXCJFRY-NVfER3p54BLOB7M3Yi2-u5IAJTw,24115
 langfun/core/structured/schema_test.py,sha256=LEtCST5Bfwoke59I6Q1mnOJLf2cFXQwKwTeAkI2hgqM,20912
 langfun/core/templates/__init__.py,sha256=bO0eMsVJbi7sxEB2YlInKRQ2EVP-RyyKUwcD-8msuN4,927
@@ -86,9 +86,9 @@ langfun/core/templates/conversation_test.py,sha256=RryYyIhfc34dLWOs6GfPQ8HU8mXpK
 langfun/core/templates/demonstration.py,sha256=vCrgYubdZM5Umqcgp8NUVGXgr4P_c-fikKhwhzwhpKI,1460
 langfun/core/templates/demonstration_test.py,sha256=SafcDQ0WgI7pw05EmPI2S4v1t3ABKzup8jReCljHeK4,2162
 langfun/core/templates/selfplay.py,sha256=yhgrJbiYwq47TgzThmHrDQTF4nDrTI09CWGhuQPNv-s,2273
-langfun/core/templates/selfplay_test.py,sha256=ZkDfwiW9OtO_MOIdVTRPn6P6vOExQIszqlVQHg5iD3U,2066
-langfun-0.0.2.dev20240201.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-langfun-0.0.2.dev20240201.dist-info/METADATA,sha256=BRcyCLLZnV3G5PLpO7ik2hVlnF0xkm7HQX5ckgyiXN8,3368
-langfun-0.0.2.dev20240201.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-langfun-0.0.2.dev20240201.dist-info/top_level.txt,sha256=RhlEkHxs1qtzmmtWSwYoLVJAc1YrbPtxQ52uh8Z9VvY,8
-langfun-0.0.2.dev20240201.dist-info/RECORD,,
+langfun/core/templates/selfplay_test.py,sha256=IB5rWbjK_9CTkqEo1BclQPzFAKcIiusJckH8J19HFgI,2096
+langfun-0.0.2.dev20240203.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+langfun-0.0.2.dev20240203.dist-info/METADATA,sha256=9wGw6eai5F4auzoA-J2DHC-OXjqeRessnRbVl2voxNY,3368
+langfun-0.0.2.dev20240203.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+langfun-0.0.2.dev20240203.dist-info/top_level.txt,sha256=RhlEkHxs1qtzmmtWSwYoLVJAc1YrbPtxQ52uh8Z9VvY,8
+langfun-0.0.2.dev20240203.dist-info/RECORD,,