langfun 0.0.2.dev20240202__py3-none-any.whl → 0.0.2.dev20240203__py3-none-any.whl

langfun/core/eval/base_test.py

@@ -101,7 +101,7 @@ class EvaluationTest(unittest.TestCase):
     self.assertEqual(s.dir, os.path.join(s.root_dir, s.id))
     self.assertEqual(s.hash, s.clone().hash)
     # Test persistent hash.
-    self.assertEqual(s.hash, 'c76d4fe6')
+    self.assertEqual(s.hash, 'abc7c29a')
     self.assertEqual(
         s.hash, s.clone(override={'max_workers': 2, 'lm.timeout': 20}).hash
     )
@@ -195,6 +195,7 @@ class EvaluationTest(unittest.TestCase):
             result=Solution(2),
             cache_seed=0,
             score=1.0,
+            logprobs=None,
             tags=['lm-response', 'lm-output', 'transformed'],
         ),
     )
@@ -323,7 +324,7 @@ class EvaluationTest(unittest.TestCase):
         s.children[0].dir, os.path.join(s.root_dir, s.children[0].id)
     )
     # Test persistent hash.
-    self.assertEqual(s.hash, 'e987475a')
+    self.assertEqual(s.hash, 'ca7f722b')
 
     summary = s.run(verbose=True)
     self.assertEqual(len(summary.evaluations), 2)
@@ -451,7 +452,7 @@ class SuiteTest(unittest.TestCase):
         ],
     )
     # Test for persistent hash.
-    self.assertEqual(s.hash, 'bb86a963')
+    self.assertEqual(s.hash, '7285e52b')
     s.run()
     expected = {
         s.children[0].id: dict(

langfun/core/langfunc_test.py

@@ -82,7 +82,7 @@ class LangFuncCallTest(unittest.TestCase):
     self.assertEqual(i.tags, ['rendered'])
 
     r = l()
-    self.assertEqual(r, message.AIMessage('Hello!!!', score=0.0))
+    self.assertEqual(r, message.AIMessage('Hello!!!', score=0.0, logprobs=None))
     self.assertEqual(r.tags, ['lm-response', 'lm-output'])
     self.assertEqual(r.source, message.UserMessage('Hello'))
     self.assertEqual(r.source.tags, ['rendered', 'lm-input'])
@@ -94,8 +94,9 @@ class LangFuncCallTest(unittest.TestCase):
         "LangFunc(template_str='Hello', clean=True,"
         ' lm=ExcitedEchoer(sampling_options=LMSamplingOptions(temperature=0.0,'
         ' max_tokens=1024, n=1, top_k=40, top_p=None, stop=None,'
-        ' random_seed=None), cache=None, timeout=120.0, max_attempts=5,'
-        ' retry_interval=(5, 60), exponential_backoff=True, debug=False))',
+        ' random_seed=None, logprobs=False, top_logprobs=None), cache=None,'
+        ' timeout=120.0, max_attempts=5, retry_interval=(5, 60),'
+        ' exponential_backoff=True, debug=False))',
     )
 
     l = LangFunc('Hello')
@@ -104,7 +105,9 @@ class LangFuncCallTest(unittest.TestCase):
       self.assertEqual(l.natural_language_format(), 'Hello')
       self.assertEqual(l.render(), 'Hello')
       r = l()
-      self.assertEqual(r, message.AIMessage('Hello!!!', score=0.0))
+      self.assertEqual(
+          r, message.AIMessage('Hello!!!', score=0.0, logprobs=None)
+      )
       self.assertEqual(r.tags, ['lm-response', 'lm-output'])
 
     self.assertEqual(str(l), 'Hello')

langfun/core/language_model.py

@@ -31,15 +31,20 @@ class LMSample(pg.Object):
       pg.typing.Object(
           message_lib.Message,
           # Allowing automatic conversion from text to AIMessage.
-          transform=message_lib.AIMessage.from_value
+          transform=message_lib.AIMessage.from_value,
       ),
-      'The natural language response of LM.'
+      'The natural language response of LM.',
   ]
 
   score: Annotated[
       float, 'The score of sampled response. The larger is better'
   ] = 0.0
 
+  logprobs: Annotated[
+      list[tuple[str, float, list[tuple[str, float]]]] | None,
+      '(token, log prob, top tokens and their probs).',
+  ] = None
+
 
 class LMSamplingResult(pg.Object):
   """Language model response."""
@@ -92,6 +97,23 @@ class LMSamplingOptions(component.Component):
   random_seed: Annotated[
       int | None, 'A fixed random seed used during model inference.'
   ] = None
+  logprobs: Annotated[
+      bool,
+      (
+          'Whether to return log probabilities of the output tokens or not. If '
+          'true, returns the log probabilities of each output token returned '
+          'in the content of message.'
+      ),
+  ] = False
+  top_logprobs: Annotated[
+      int | None,
+      (
+          'An integer between 0 and 5 specifying the number of most likely '
+          'tokens to return at each token position, each with an associated '
+          'log probability. logprobs must be set to true if this parameter is '
+          'used.'
+      ),
+  ] = None
 
   def cache_key(self) -> tuple[Any, ...]:
     """Returns a tuple of current values as cache key."""
@@ -339,7 +361,9 @@ class LanguageModel(component.Component):
           [prompt], sampling_options=sampling_options, cache_seed=cache_seed
       )[0]
       response = result.samples[0].response
+      logprobs = result.samples[0].logprobs
       response.set('score', result.samples[0].score)
+      response.metadata.logprobs = logprobs
       elapse = time.time() - request_start
       self._debug(prompt, response, call_counter, elapse)
       return response

langfun/core/llms/openai.py

@@ -167,6 +167,8 @@ class OpenAI(lf.LanguageModel):
         max_tokens=options.max_tokens,
         stream=False,
         timeout=self.timeout,
+        logprobs=options.logprobs,
+        top_logprobs=options.top_logprobs,
     )
     # Completion and ChatCompletion uses different parameter name for model.
     args['model' if self.is_chat_model else 'engine'] = self.model
@@ -249,11 +251,28 @@ class OpenAI(lf.LanguageModel):
           **self._get_request_args(self.sampling_options),
       )
       response = cast(openai_object.OpenAIObject, response)
+      samples = []
+      for choice in response.choices:
+        logprobs = None
+        if choice.logprobs:
+          logprobs = [
+              (
+                  t.token,
+                  t.logprob,
+                  [(tt.token, tt.logprob) for tt in t.top_logprobs],
+              )
+              for t in choice.logprobs.content
+          ]
+        samples.append(
+            lf.LMSample(
+                choice.message.content,
+                score=0.0,
+                logprobs=logprobs,
+            )
+        )
+
       return LMSamplingResult(
-          [
-              lf.LMSample(choice.message.content, score=0.0)
-              for choice in response.choices
-          ],
+          samples=samples,
           usage=Usage(
               prompt_tokens=response.usage.prompt_tokens,
               completion_tokens=response.usage.completion_tokens,

langfun/core/llms/openai_test.py

@@ -46,7 +46,8 @@ def mock_chat_completion_query(messages, *, n=1, **kwargs):
     choices.append(pg.Dict(
         message=pg.Dict(
             content=f'Sample {k} for message.'
-        )
+        ),
+        logprobs=None,
     ))
   return pg.Dict(choices=choices, usage=openai.Usage(
       prompt_tokens=100,
@@ -65,7 +66,8 @@ def mock_chat_completion_query_vision(messages, *, n=1, **kwargs):
     choices.append(pg.Dict(
         message=pg.Dict(
             content=f'Sample {k} for message: {"".join(urls)}'
-        )
+        ),
+        logprobs=None,
     ))
   return pg.Dict(choices=choices, usage=openai.Usage(
       prompt_tokens=100,
@@ -99,6 +101,8 @@ class OpenaiTest(unittest.TestCase):
                 top_p=1.0)),
         dict(
             engine='text-davinci-003',
+            logprobs=False,
+            top_logprobs=None,
             n=2,
             temperature=2.0,
             max_tokens=4096,
@@ -113,6 +117,8 @@ class OpenaiTest(unittest.TestCase):
         ),
         dict(
             model='gpt-4',
+            logprobs=False,
+            top_logprobs=None,
             n=1,
             temperature=1.0,
             max_tokens=1024,

langfun/core/structured/completion_test.py

@@ -582,6 +582,7 @@ class CompleteStructureTest(unittest.TestCase):
             text='Activity(description="foo")',
             result=Activity(description='foo'),
             score=1.0,
+            logprobs=None,
             tags=['lm-response', 'lm-output', 'transformed']
         )
     )

langfun/core/structured/parsing_test.py

@@ -286,7 +286,7 @@ class ParseStructurePythonTest(unittest.TestCase):
             returns_message=True
         ),
         lf.AIMessage(
-            '1', score=1.0, result=1,
+            '1', score=1.0, result=1, logprobs=None,
             tags=['lm-response', 'lm-output', 'transformed']
         ),
     )
@@ -640,7 +640,7 @@ class CallTest(unittest.TestCase):
             returns_message=True
         ),
         lf.AIMessage(
-            '3', result=3, score=1.0,
+            '3', result=3, score=1.0, logprobs=None,
             tags=['lm-response', 'lm-output', 'transformed']
         ),
     )

langfun/core/structured/prompting_test.py

@@ -76,6 +76,7 @@ class QueryTest(unittest.TestCase):
             '1',
             result=1,
             score=1.0,
+            logprobs=None,
             tags=['lm-response', 'lm-output', 'transformed'],
         ),
     )

langfun/core/templates/selfplay_test.py

@@ -56,7 +56,7 @@ class SelfPlayTest(unittest.TestCase):
     g = NumberGuess(target_num=10)
 
     with lf.context(lm=NumberGuesser(guesses=[50, 20, 5, 10])):
-      self.assertEqual(g(), lf.AIMessage('10', score=0.0))
+      self.assertEqual(g(), lf.AIMessage('10', score=0.0, logprobs=None))
 
     self.assertEqual(g.num_turns, 4)
 
@@ -64,7 +64,7 @@ class SelfPlayTest(unittest.TestCase):
     g = NumberGuess(target_num=10, max_turns=10)
 
     with lf.context(lm=NumberGuesser(guesses=[50, 20, 5, 2, 5, 4])):
-      self.assertEqual(g(), lf.AIMessage('2', score=0.0))
+      self.assertEqual(g(), lf.AIMessage('2', score=0.0, logprobs=None))
 
     self.assertEqual(g.num_turns, 10)
 

{langfun-0.0.2.dev20240202.dist-info → langfun-0.0.2.dev20240203.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: langfun
-Version: 0.0.2.dev20240202
+Version: 0.0.2.dev20240203
 Summary: Langfun: Language as Functions.
 Home-page: https://github.com/google/langfun
 Author: Langfun Authors

{langfun-0.0.2.dev20240202.dist-info → langfun-0.0.2.dev20240203.dist-info}/RECORD

@@ -7,8 +7,8 @@ langfun/core/concurrent_test.py,sha256=qQT6_Dq5NVz7qXFLzSf2Rhzkfkh07gocjHMBaT1nS
 langfun/core/console.py,sha256=bk5rNPNm9rMGW5YT2HixxU04p2umnoabn5SDz6Dqe88,2317
 langfun/core/console_test.py,sha256=5SYJdxpJGLgdSSQqqMPoA1X6jpsLD8rgcyk-EgI65oE,1077
 langfun/core/langfunc.py,sha256=266xNz8Vgal7K4HSsrYt7z7_qPYV4bWWK626IbbohrE,11573
-langfun/core/langfunc_test.py,sha256=rnU-7JQcY-fIXB5aoKbouPd2LkjdzIjB5g_aLPBHQCE,8069
-langfun/core/language_model.py,sha256=UppuqiwmZ6AfWCNkfom12XqEu1uf8qBB3yp-xezzQ0s,12056
+langfun/core/langfunc_test.py,sha256=ukv5cnad5ZBckM2PhyIFq79BPN0Db4cszMrPqh_CZkA,8163
+langfun/core/language_model.py,sha256=HG2XQhisyiyvMBws5dASH1OkbGJB3twn0p2e_As8hUo,12889
 langfun/core/language_model_test.py,sha256=gcW4OJJjB-V1b4kEF8zG91t36sVn3H0Yuj0LQxi83Ek,9122
 langfun/core/memory.py,sha256=f-asN1F7Vehgdn_fK84v73GrEUOxRtaW934keutTKjk,2416
 langfun/core/message.py,sha256=QhvV9t5qaryPcruyxxcXi3gm9QDInkSldwTtK6sVJ3c,15734
@@ -41,7 +41,7 @@ langfun/core/coding/python/permissions.py,sha256=1QWGHvzL8MM0Ok_auQ9tURqZHtdOfJa
 langfun/core/coding/python/permissions_test.py,sha256=w5EDb8QxpxgJyZkojyzVWQvDfg366zn99-g__6TbPQ0,2699
 langfun/core/eval/__init__.py,sha256=iDA2OcJ3kR6ixZizXIY3N9LsjkaVrfTbSClTiSP8ekY,1291
 langfun/core/eval/base.py,sha256=wWFDDrf0jBzs9H_5XfdZSeOBGXyUtXAJJouk7cLckSM,52602
-langfun/core/eval/base_test.py,sha256=bGs3VLchkAJFWYJ8FdR7mC6qoDestAvCHOQpClG6Mzw,21248
+langfun/core/eval/base_test.py,sha256=uXu6EtTagolDIcSadnVyMmlrz6ixx943jkZhquCRQPI,21275
 langfun/core/eval/matching.py,sha256=g2yuBb4FeOlAlB10hqdWvaIg4QVQlJbiViRDcD2Y8go,9567
 langfun/core/eval/matching_test.py,sha256=IfuMF_dEmy4VzK6tIldRzD2Nqlml7SSh4u-baFNcZrw,4912
 langfun/core/eval/scoring.py,sha256=mshqbV_WM0zcp15TSR32ACMBDymlsbf6YH06PPx1Tw0,6139
@@ -53,8 +53,8 @@ langfun/core/llms/gemini.py,sha256=p3d4Cl2uET-os1n_V3YNE6-6cYrZjndj7lxZIk2E8_4,5
 langfun/core/llms/gemini_test.py,sha256=ybNNCn3JW3hYpMe0wT5ILGDrMPaYYU8PN2kSookM0jk,5433
 langfun/core/llms/llama_cpp.py,sha256=EIjJa1-Tg4_VaIxVR88oDWSWc_axc1r2KwSPpl4PSp0,2549
 langfun/core/llms/llama_cpp_test.py,sha256=ZxC6defGd_HX9SFRU9U4cJiQnBKundbOrchbXuC1Z2M,1683
-langfun/core/llms/openai.py,sha256=ao2sDDoh5ma1GWpLpNPZARIeLZK55gL1Ldc94h1EGtE,11119
-langfun/core/llms/openai_test.py,sha256=JWcMveifVVVEFWdtmNq1irc9wSFQRxXs-SnOF3Urg9Y,7433
+langfun/core/llms/openai.py,sha256=ufhRf-fV-AFq_-pn5sKrcr_xIU7V0VnhjK3JfBG8sF8,11617
+langfun/core/llms/openai_test.py,sha256=yfw7A-4Zo9u1cIkAMk39evE-tO7z6isNYTXiSnJXDQw,7599
 langfun/core/llms/cache/__init__.py,sha256=QAo3InUMDM_YpteNnVCSejI4zOsnjSMWKJKzkb3VY64,993
 langfun/core/llms/cache/base.py,sha256=cFfYvOIUae842pncqCAsRvqXCk2AnAsRYVx0mcIoAeY,3338
 langfun/core/llms/cache/in_memory.py,sha256=YfFyJEhLs73cUiB0ZfhMxYpdE8Iuxxw-dvMFwGHTSHw,4742
@@ -67,15 +67,15 @@ langfun/core/modalities/image.py,sha256=HU0sV4ZTwRnAwQthmdWZwhFZRD86RyvqoS8JUW2I
 langfun/core/modalities/image_test.py,sha256=YxDRvC49Bjwyyndd_P7y6XjyS7dOft0Zewwxk-7q4kE,2301
 langfun/core/structured/__init__.py,sha256=tGH0MYr5vzK0H2DpYQ2bcW2C5bpPUaLzMk2W2Fj29M4,3136
 langfun/core/structured/completion.py,sha256=XERoxtYPXOTlPdZ2bp4i9R4jl3kA3SOeyLmuSqHG9AM,7036
-langfun/core/structured/completion_test.py,sha256=i68c_eK-QJIAW964E2o4W8kNj2ZokEQOPesG5Hw78-E,19222
+langfun/core/structured/completion_test.py,sha256=98UCgA4gzfp6H6HgP2s2kcKs25YH3k4Nxj1rgAvmVBw,19249
 langfun/core/structured/description.py,sha256=vDiW1g2VbvG8ucNjV7Pp3VYCeAnLcp6vLQ0MfURcZFk,4825
 langfun/core/structured/description_test.py,sha256=UtZGjSFUaQ6130t1E5tcL7ODu0xIefkapb53TbnqsK8,7362
 langfun/core/structured/mapping.py,sha256=lGkjhmvVdhBGgJmc5KbfT2xQjC1MuU4OCcCfsAYJjaQ,10192
 langfun/core/structured/mapping_test.py,sha256=07DDCGbwytQHSMm7fCi5-Ly-JNgdV4ubHZq0wthX4A4,3338
 langfun/core/structured/parsing.py,sha256=V3i8-AuBI4zdpsi_f6H-mbQ3h0HPgCle2OnDQolWJnA,10244
-langfun/core/structured/parsing_test.py,sha256=EC9i4hQkcmoneBX7p1fRIpgznB8lgGHFwtKvHlNrwyA,20816
+langfun/core/structured/parsing_test.py,sha256=2_Uf3LYNRON1-5ysEr75xiG_cAxR3ZiixSfvUQu6mOQ,20846
 langfun/core/structured/prompting.py,sha256=OmI21qQQikoQLAmr5W8nBJ8PWCu9w0lmsCnmbQg9hMc,6632
-langfun/core/structured/prompting_test.py,sha256=LOOka3CaID03SXPOSNHVFrnk7Ymkt6GvSDS9pNa-y3M,19116
+langfun/core/structured/prompting_test.py,sha256=8lakKCidYuRlf-U1KexTOqQdKrBXUt8fb2J7epCdt84,19143
 langfun/core/structured/schema.py,sha256=5DKba0LrvXCJFRY-NVfER3p54BLOB7M3Yi2-u5IAJTw,24115
 langfun/core/structured/schema_test.py,sha256=LEtCST5Bfwoke59I6Q1mnOJLf2cFXQwKwTeAkI2hgqM,20912
 langfun/core/templates/__init__.py,sha256=bO0eMsVJbi7sxEB2YlInKRQ2EVP-RyyKUwcD-8msuN4,927
@@ -86,9 +86,9 @@ langfun/core/templates/conversation_test.py,sha256=RryYyIhfc34dLWOs6GfPQ8HU8mXpK
 langfun/core/templates/demonstration.py,sha256=vCrgYubdZM5Umqcgp8NUVGXgr4P_c-fikKhwhzwhpKI,1460
 langfun/core/templates/demonstration_test.py,sha256=SafcDQ0WgI7pw05EmPI2S4v1t3ABKzup8jReCljHeK4,2162
 langfun/core/templates/selfplay.py,sha256=yhgrJbiYwq47TgzThmHrDQTF4nDrTI09CWGhuQPNv-s,2273
-langfun/core/templates/selfplay_test.py,sha256=ZkDfwiW9OtO_MOIdVTRPn6P6vOExQIszqlVQHg5iD3U,2066
-langfun-0.0.2.dev20240202.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-langfun-0.0.2.dev20240202.dist-info/METADATA,sha256=YNH_28StpsuvajP3UyX4a4-RjS3ErqeooJE_l55L1cQ,3368
-langfun-0.0.2.dev20240202.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-langfun-0.0.2.dev20240202.dist-info/top_level.txt,sha256=RhlEkHxs1qtzmmtWSwYoLVJAc1YrbPtxQ52uh8Z9VvY,8
-langfun-0.0.2.dev20240202.dist-info/RECORD,,
+langfun/core/templates/selfplay_test.py,sha256=IB5rWbjK_9CTkqEo1BclQPzFAKcIiusJckH8J19HFgI,2096
+langfun-0.0.2.dev20240203.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+langfun-0.0.2.dev20240203.dist-info/METADATA,sha256=9wGw6eai5F4auzoA-J2DHC-OXjqeRessnRbVl2voxNY,3368
+langfun-0.0.2.dev20240203.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+langfun-0.0.2.dev20240203.dist-info/top_level.txt,sha256=RhlEkHxs1qtzmmtWSwYoLVJAc1YrbPtxQ52uh8Z9VvY,8
+langfun-0.0.2.dev20240203.dist-info/RECORD,,