langfun 0.0.2.dev20240215__tar.gz → 0.0.2.dev20240217__tar.gz

{langfun-0.0.2.dev20240215 → langfun-0.0.2.dev20240217}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: langfun
-Version: 0.0.2.dev20240215
+Version: 0.0.2.dev20240217
 Summary: Langfun: Language as Functions.
 Home-page: https://github.com/google/langfun
 Author: Langfun Authors

{langfun-0.0.2.dev20240215 → langfun-0.0.2.dev20240217}/langfun/__init__.py

@@ -30,6 +30,7 @@ parse = structured.parse
 query = structured.query
 describe = structured.describe
 complete = structured.complete
+score = structured.score
 
 from langfun.core import eval  # pylint: disable=redefined-builtin
 from langfun.core import templates

{langfun-0.0.2.dev20240215 → langfun-0.0.2.dev20240217}/langfun/core/__init__.py

@@ -100,6 +100,7 @@ from langfun.core.language_model import LanguageModel
 from langfun.core.language_model import LMSample
 from langfun.core.language_model import LMSamplingOptions
 from langfun.core.language_model import LMSamplingResult
+from langfun.core.language_model import LMScoringResult
 from langfun.core.language_model import LMCache
 from langfun.core.language_model import LMDebugMode
 

{langfun-0.0.2.dev20240215 → langfun-0.0.2.dev20240217}/langfun/core/langfunc.py

@@ -210,6 +210,7 @@ class LangFunc(
       lm: language_model.LanguageModel | None = None,
       lm_input: message_lib.Message | None = None,
       cache_seed: int | None = 0,
+      skip_lm: bool = False,
       **variables,
   ) -> message_lib.Message:
     """Calls language model with `lm_input` or rendered text.
@@ -223,6 +224,8 @@ class LangFunc(
       cache_seed: Seed for computing cache key. The cache key is determined by a
         tuple of (lm, prompt, cache seed). If None, cache will be disabled for
         the query even cache is configured by the LM.
+      skip_lm: If True, returns the rendered prompt as a UserMessage object.
+        otherwise return the LLM response based on the rendered prompt.
       **variables: Template variables applicable to this or child LangFunc.
 
     Returns:
@@ -232,6 +235,7 @@ class LangFunc(
         lm=lm,
         lm_input=lm_input,
         cache_seed=cache_seed,
+        skip_lm=skip_lm,
         **variables,
     )
 
@@ -241,6 +245,7 @@ class LangFunc(
       lm: language_model.LanguageModel | None = None,
       lm_input: message_lib.Message | None = None,
       cache_seed: int | None = 0,
+      skip_lm: bool = False,
       **variables,
   ) -> message_lib.Message:
     """Call the language model once, with invoking the output transform."""
@@ -256,10 +261,13 @@ class LangFunc(
         if lm_input is None:
           lm_input = self.render(**kwargs)
 
+        lm_input.tag(message_lib.Message.TAG_LM_INPUT)
+        if skip_lm:
+          return lm_input
+
         self._cached_lm_input = lm_input
 
         # Send rendered text to LM.
-        lm_input.tag(message_lib.Message.TAG_LM_INPUT)
         lm_output = self.lm(lm_input, cache_seed=cache_seed)
 
         # Track the input as the source of the output.

{langfun-0.0.2.dev20240215 → langfun-0.0.2.dev20240217}/langfun/core/langfunc_test.py

@@ -194,6 +194,11 @@ class LangFuncCallTest(unittest.TestCase):
       self.assertEqual(l(x=1, cache_seed=None), 'd')
       self.assertEqual(l(x=2), 'b')
 
+  def test_call_with_skip_lm(self):
+    l = LangFunc('hi')
+    with component.context(lm=ExcitedEchoer()):
+      self.assertEqual(l(skip_lm=True), 'hi')
+
 
 class CallEventTest(unittest.TestCase):
 

{langfun-0.0.2.dev20240215 → langfun-0.0.2.dev20240217}/langfun/core/language_model.py

@@ -127,6 +127,15 @@ class LMSamplingOptions(component.Component):
     )
 
 
+class LMScoringResult(pg.Object):
+  """Language model scoring result."""
+
+  score: Annotated[
+      float,
+      'The log likelyhood of the requested completion towards the prompt.',
+  ]
+
+
 class LMCache(pg.Object):
   """Interface for LM cache."""
 
@@ -425,3 +434,83 @@ class LanguageModel(component.Component):
         title=f'\n[{call_counter}] LM RESPONSE (in {elapse:.2f} seconds):',
         color='blue',
     )
+
+  def score(
+      self,
+      prompt: str | message_lib.Message,
+      completions: list[str | message_lib.Message],
+      **kwargs,
+  ) -> list[LMScoringResult]:
+    """Scores the given prompt."""
+    prompt = message_lib.UserMessage.from_value(prompt)
+    completions = [message_lib.UserMessage.from_value(c) for c in completions]
+
+    call_counter = self._call_counter
+    self._call_counter += 1
+    request_start = time.time()
+
+    with component.context(override_attrs=True, **kwargs):
+      scoring_results = self._score(prompt, completions)
+      elapse = time.time() - request_start
+      self._debug_score(
+          prompt, completions, scoring_results, call_counter, elapse
+      )
+      return scoring_results
+
+  def _score(
+      self, prompt: message_lib.Message, completions: list[message_lib.Message]
+  ) -> list[LMScoringResult]:
+    """Subclass to implement."""
+    raise NotImplementedError(
+        f'{self.__class__.__name__} does not support scoring.'
+    )
+
+  def _debug_score(
+      self,
+      prompt: message_lib.Message,
+      completions: list[message_lib.Message],
+      scoring_results: list[LMScoringResult],
+      call_counter: int,
+      elapse: float,
+  ):
+    debug = self.debug
+    if isinstance(debug, bool):
+      debug = LMDebugMode.ALL if debug else LMDebugMode.NONE
+
+    if debug & LMDebugMode.INFO:
+      self._debug_model_info(call_counter)
+
+    if debug & LMDebugMode.PROMPT:
+      console.write(
+          prompt,
+          title=f'\n[{call_counter}] SCORING LM WITH PROMPT:',
+          color='green',
+      )
+      referred_modalities = prompt.referred_modalities()
+      if referred_modalities:
+        console.write(
+            pg.object_utils.kvlist_str(
+                [(k, repr(v), None) for k, v in referred_modalities.items()]
+            ),
+            title=f'\n[{call_counter}] MODALITY OBJECTS SENT TO LM:',
+            color='green',
+        )
+
+    if debug & LMDebugMode.RESPONSE:
+      console.write(
+          '',
+          title=(
+              f'\n[{call_counter}] SCORING COMPLETED (in {elapse:.2f} seconds):'
+          ),
+          color='blue',
+      )
+      for i, (c, r) in enumerate(zip(completions, scoring_results)):
+        console.write(
+            c,
+            title=f'COMPLETION #{i}',
+            color='green',
+        )
+        console.write(
+            f'score: {r.score}',
+            color='blue',
+        )

{langfun-0.0.2.dev20240215 → langfun-0.0.2.dev20240217}/langfun/core/language_model_test.py

@@ -54,6 +54,19 @@ class MockModel(lm_lib.LanguageModel):
     )(prompts)
 
 
+class MockScoringModel(MockModel):
+
+  def _score(
+      self,
+      prompt: message_lib.Message,
+      completions: list[message_lib.Message],
+      **kwargs
+  ) -> list[lm_lib.LMScoringResult]:
+    return [
+        lm_lib.LMScoringResult(score=-i * 1.0) for i in range(len(completions))
+    ]
+
+
 class LMSamplingOptionsTest(unittest.TestCase):
   """Tests for LMSamplingOptions."""
 
@@ -266,6 +279,68 @@ class LanguageModelTest(unittest.TestCase):
       for expected_exclude in expected_excluded:
         self.assertNotIn('[0] ' + expected_exclude, debug_info)
 
+  def test_score(self):
+    info_flag = lm_lib.LMDebugMode.INFO
+    prompt_flag = lm_lib.LMDebugMode.PROMPT
+    response_flag = lm_lib.LMDebugMode.RESPONSE
+    debug_prints = {
+        info_flag: 'LM INFO',
+        prompt_flag: 'SCORING LM WITH PROMPT',
+        response_flag: 'SCORING COMPLETED',
+    }
+    debug_modes = [
+        info_flag,
+        prompt_flag,
+        response_flag,
+        info_flag | prompt_flag,
+        info_flag | response_flag,
+        prompt_flag | response_flag,
+        info_flag | prompt_flag | response_flag,
+    ]
+
+    class Image(modality.Modality):
+      def to_bytes(self):
+        return b'fake_image'
+
+    for debug_mode in debug_modes:
+      string_io = io.StringIO()
+      lm = MockScoringModel()
+
+      with contextlib.redirect_stdout(string_io):
+        self.assertEqual(
+            lm.score(
+                message_lib.UserMessage('hi {{image}}', image=Image()),
+                ['1', '2'], debug=debug_mode),
+            [
+                lm_lib.LMScoringResult(score=-0.0),
+                lm_lib.LMScoringResult(score=-1.0),
+            ],
+        )
+
+      debug_info = string_io.getvalue()
+      expected_included = [
+          debug_prints[f]
+          for f in lm_lib.LMDebugMode
+          if f != lm_lib.LMDebugMode.NONE and f in debug_mode
+      ]
+      expected_excluded = [
+          debug_prints[f]
+          for f in lm_lib.LMDebugMode
+          if f != lm_lib.LMDebugMode.NONE and f not in debug_mode
+      ]
+
+      for expected_include in expected_included:
+        self.assertIn('[0] ' + expected_include, debug_info)
+      for expected_exclude in expected_excluded:
+        self.assertNotIn('[0] ' + expected_exclude, debug_info)
+
+      if debug_mode & lm_lib.LMDebugMode.PROMPT:
+        self.assertIn('[0] MODALITY OBJECTS SENT TO LM', debug_info)
+
+  def test_score_with_unsupported_model(self):
+    with self.assertRaises(NotImplementedError):
+      MockModel().score('hi', ['1', '2'])
+
 
 if __name__ == '__main__':
   unittest.main()

{langfun-0.0.2.dev20240215 → langfun-0.0.2.dev20240217}/langfun/core/llms/__init__.py

@@ -18,6 +18,7 @@
 # pylint: disable=g-import-not-at-top
 
 # LMs for testing.
+from langfun.core.llms.fake import Fake
 from langfun.core.llms.fake import Echo
 from langfun.core.llms.fake import StaticMapping
 from langfun.core.llms.fake import StaticResponse

{langfun-0.0.2.dev20240215 → langfun-0.0.2.dev20240217}/langfun/core/llms/fake.py

@@ -17,7 +17,14 @@ from typing import Annotated
 import langfun.core as lf
 
 
-class Echo(lf.LanguageModel):
+class Fake(lf.LanguageModel):
+  """The base class for all fake language models."""
+
+  def _score(self, prompt: lf.Message, completions: list[lf.Message]):
+    return [lf.LMScoringResult(score=-i * 1.0) for i in range(len(completions))]
+
+
+class Echo(Fake):
   """A simple echo language model for testing."""
 
   def _sample(self, prompts: list[lf.Message]) -> list[lf.LMSamplingResult]:
@@ -28,7 +35,7 @@ class Echo(lf.LanguageModel):
 
 
 @lf.use_init_args(['response'])
-class StaticResponse(lf.LanguageModel):
+class StaticResponse(Fake):
   """Language model that always gives the same canned response."""
 
   response: Annotated[
@@ -44,7 +51,7 @@ class StaticResponse(lf.LanguageModel):
 
 
 @lf.use_init_args(['mapping'])
-class StaticMapping(lf.LanguageModel):
+class StaticMapping(Fake):
   """A static mapping from prompt to response."""
 
   mapping: Annotated[
@@ -60,7 +67,7 @@ class StaticMapping(lf.LanguageModel):
 
 
 @lf.use_init_args(['sequence'])
-class StaticSequence(lf.LanguageModel):
+class StaticSequence(Fake):
   """A static sequence of responses to use."""
 
   sequence: Annotated[

{langfun-0.0.2.dev20240215 → langfun-0.0.2.dev20240217}/langfun/core/llms/fake_test.py

@@ -38,6 +38,13 @@ class EchoTest(unittest.TestCase):
     self.assertIn('[0] PROMPT SENT TO LM:', debug_info)
     self.assertIn('[0] LM RESPONSE', debug_info)
 
+  def test_score(self):
+    lm = fakelm.Echo()
+    self.assertEqual(
+        lm.score('hi', ['hello', 'how are you']),
+        [lf.LMScoringResult(0.0), lf.LMScoringResult(-1.0)],
+    )
+
 
 class StaticResponseTest(unittest.TestCase):
 

{langfun-0.0.2.dev20240215 → langfun-0.0.2.dev20240217}/langfun/core/structured/__init__.py

@@ -64,6 +64,8 @@ from langfun.core.structured.description import describe
 from langfun.core.structured.completion import CompleteStructure
 from langfun.core.structured.completion import complete
 
+from langfun.core.structured.scoring import score
+
 # Expose default examples for structured operations so users could refer to
 # them.
 from langfun.core.structured.parsing import DEFAULT_PARSE_EXAMPLES

{langfun-0.0.2.dev20240215 → langfun-0.0.2.dev20240217}/langfun/core/structured/completion.py

@@ -153,6 +153,7 @@ def complete(
     *,
     lm: lf.LanguageModel | None = None,
     examples: list[mapping.MappingExample] | None = None,
+    cache_seed: int | None = 0,
     autofix: int = 0,
     autofix_lm: lf.LanguageModel | None = None,
     returns_message: bool = False,
@@ -197,6 +198,9 @@ def complete(
       `lf.context` context manager will be used.
     examples: An optional list of fewshot examples for helping parsing. If None,
       the default one-shot example will be added.
+    cache_seed: Seed for computing cache key. The cache key is determined by a
+      tuple of (lm, prompt, cache seed). If None, cache will be disabled for
+      the query even cache is configured by the LM.
     autofix: Number of attempts to auto fix the generated code. If 0, autofix is
       disabled.
     autofix_lm: The language model to use for autofix. If not specified, the
@@ -218,5 +222,5 @@ def complete(
       **kwargs,
   )
 
-  output = t(lm=lm, autofix_lm=autofix_lm or lm)
+  output = t(lm=lm, cache_seed=cache_seed, autofix_lm=autofix_lm or lm)
   return output if returns_message else output.result

{langfun-0.0.2.dev20240215 → langfun-0.0.2.dev20240217}/langfun/core/structured/description.py

@@ -44,6 +44,7 @@ def describe(
     *,
     lm: lf.LanguageModel | None = None,
     examples: list[mapping.MappingExample] | None = None,
+    cache_seed: int | None = 0,
     **kwargs,
 ) -> str:
   """Describes a structured value using natural language.
@@ -97,6 +98,9 @@ def describe(
       `lf.context` context manager will be used.
     examples: An optional list of fewshot examples for helping parsing. If None,
       the default one-shot example will be added.
+    cache_seed: Seed for computing cache key. The cache key is determined by a
+      tuple of (lm, prompt, cache seed). If None, cache will be disabled for
+      the query even cache is configured by the LM.
     **kwargs: Keyword arguments passed to the `lf.structured.DescribeStructure`.
 
   Returns:
@@ -106,7 +110,7 @@ def describe(
     examples = DEFAULT_DESCRIBE_EXAMPLES
   return DescribeStructure(
       input=value, context=context, examples=examples, **kwargs
-  )(lm=lm).text
+  )(lm=lm, cache_seed=cache_seed).text
 
 
 class _Country(pg.Object):

{langfun-0.0.2.dev20240215 → langfun-0.0.2.dev20240217}/langfun/core/structured/parsing.py

@@ -79,6 +79,7 @@ def parse(
     lm: lf.LanguageModel | None = None,
     examples: list[mapping.MappingExample] | None = None,
     include_context: bool = False,
+    cache_seed: int | None = 0,
     autofix: int = 0,
     autofix_lm: lf.LanguageModel | None = None,
     protocol: schema_lib.SchemaProtocol = 'python',
@@ -134,6 +135,9 @@ def parse(
       the default one-shot example will be added.
     include_context: If True, include the request sent to LLM for obtaining the
       response to pares. Otherwise include only the response.
+    cache_seed: Seed for computing cache key. The cache key is determined by a
+      tuple of (lm, prompt, cache seed). If None, cache will be disabled for
+      the query even cache is configured by the LM.
     autofix: Number of attempts to auto fix the generated code. If 0, autofix is
       disabled. Auto-fix is not supported for 'json' protocol.
     autofix_lm: The language model to use for autofix. If not specified, the
@@ -166,7 +170,7 @@ def parse(
   )
 
   # Setting up context.
-  call_context = dict(autofix=autofix)
+  call_context = dict(cache_seed=cache_seed, autofix=autofix)
   if lm is not None:
     call_context['lm'] = lm
   autofix_lm = autofix_lm or lm
@@ -188,6 +192,7 @@ def call(
     parsing_lm: lf.LanguageModel | None = None,
     parsing_examples: list[mapping.MappingExample] | None = None,
     parsing_include_context: bool = False,
+    cache_seed: int | None = 0,
     autofix: int = 0,
     autofix_lm: lf.LanguageModel | None = None,
     response_postprocess: Callable[[str], str] | None = None,
@@ -231,6 +236,9 @@ def call(
       `lf.structured.DEFAULT_PARSE_EXAMPLES` will be used.
     parsing_include_context: If True, include the request sent to LLM for
       obtaining the response to pares. Otherwise include only the response.
+    cache_seed: Seed for computing cache key. The cache key is determined by a
+      tuple of (lm, prompt, cache seed). If None, cache will be disabled for
+      the query even cache is configured by the LM.
     autofix: Number of attempts to auto fix the generated code. If 0, autofix is
       disabled. Auto-fix is not supported for 'json' protocol.
     autofix_lm: The language model to use for autofix. If not specified, the
@@ -253,7 +261,10 @@ def call(
   lm_output = lf.LangFunc.from_value(prompt, **kwargs)(lm=lm)
 
   if response_postprocess is not None:
-    lm_output.set('text', response_postprocess(lm_output.text))
+    postprocessed_text = response_postprocess(lm_output.text)
+    if postprocessed_text != lm_output.text:
+      processed_lm_output = lf.AIMessage(postprocessed_text, source=lm_output)
+      lm_output = processed_lm_output
 
   if schema in (str, None):
     return lm_output if returns_message else lm_output.text
@@ -265,6 +276,7 @@ def call(
       examples=parsing_examples,
       lm=parsing_lm or lm,
       include_context=parsing_include_context,
+      cache_seed=cache_seed,
       autofix=autofix,
       autofix_lm=autofix_lm or lm,
       protocol=protocol,

{langfun-0.0.2.dev20240215 → langfun-0.0.2.dev20240217}/langfun/core/structured/prompting.py

@@ -106,10 +106,12 @@ def query(
     *,
     lm: lf.LanguageModel | None = None,
     examples: list[mapping.MappingExample] | None = None,
+    cache_seed: int | None = 0,
     autofix: int = 0,
     autofix_lm: lf.LanguageModel | None = None,
     protocol: schema_lib.SchemaProtocol = 'python',
     returns_message: bool = False,
+    skip_lm: bool = False,
     **kwargs,
 ) -> Any:
   """Parse a natural langugage message based on schema.
@@ -154,6 +156,9 @@ def query(
       `lf.context` context manager will be used.
     examples: An optional list of fewshot examples for helping parsing. If None,
       the default one-shot example will be added.
+    cache_seed: Seed for computing cache key. The cache key is determined by a
+      tuple of (lm, prompt, cache seed). If None, cache will be disabled for
+      the query even cache is configured by the LM.
     autofix: Number of attempts to auto fix the generated code. If 0, autofix is
       disabled. Auto-fix is not supported for 'json' protocol.
     autofix_lm: The language model to use for autofix. If not specified, the
@@ -163,6 +168,8 @@ def query(
       are 'json' and 'python'. By default `python` will be used.
     returns_message: If True, returns `lf.Message` as the output, instead of
       returning the structured `message.result`.
+    skip_lm: If True, returns the rendered prompt as a UserMessage object.
+      otherwise return the LLM response based on the rendered prompt.
     **kwargs: Keyword arguments passed to the
       `lf.structured.NaturalLanguageToStructureed` transform.
 
@@ -178,7 +185,9 @@ def query(
 
   if schema in (None, str):
     # Query with natural language output.
-    output = lf.LangFunc.from_value(prompt, **kwargs)(lm=lm)
+    output = lf.LangFunc.from_value(prompt, **kwargs)(
+        lm=lm, cache_seed=cache_seed, skip_lm=skip_lm
+    )
     return output if returns_message else output.text
 
   # Query with structured output.
@@ -202,5 +211,7 @@ def query(
   )(
       lm=lm,
       autofix_lm=autofix_lm or lm,
+      cache_seed=cache_seed,
+      skip_lm=skip_lm,
   )
   return output if returns_message else output.result

{langfun-0.0.2.dev20240215 → langfun-0.0.2.dev20240217}/langfun/core/structured/prompting_test.py

@@ -235,6 +235,10 @@ class QueryTest(unittest.TestCase):
         expected_modalities=2,
     )
 
+  def test_bad_protocol(self):
+    with self.assertRaisesRegex(ValueError, 'Unknown protocol'):
+      prompting.query('what is 1 + 1', int, protocol='text')
+
 
 class QueryStructurePythonTest(unittest.TestCase):
 

langfun-0.0.2.dev20240217/langfun/core/structured/scoring.py

@@ -0,0 +1,75 @@
+# Copyright 2023 The Langfun Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Scoring the output objects based on their inputs."""
+
+from typing import Any, Type, Union
+
+import langfun.core as lf
+from langfun.core.structured import mapping
+from langfun.core.structured import prompting
+from langfun.core.structured import schema as schema_lib
+import pyglove as pg
+
+
+def score(
+    prompt: Union[str, pg.Symbolic],
+    completions: list[str | pg.Symbolic],
+    schema: Union[
+        schema_lib.Schema, Type[Any], list[Type[Any]], dict[str, Any], None
+    ] = None,
+    *,
+    lm: lf.LanguageModel | None = None,
+    examples: list[mapping.MappingExample] | None = None,
+    protocol: schema_lib.SchemaProtocol = 'python',
+    **kwargs,
+) -> list[float]:
+  """Scores the outputs based on the prompt."""
+  if not completions:
+    raise ValueError('`completions` must not be empty.')
+
+  if schema is None:
+    for c in completions:
+      if schema is None:
+        schema = type(c)
+      elif schema is not type(c):
+        raise ValueError(
+            '`schema` cannot be inferred from completions of different types: '
+            f'{[type(c) for c in completions]}.'
+        )
+
+  input_message = prompting.query(
+      prompt,
+      schema,
+      examples=examples,
+      protocol=protocol,
+      skip_lm=True,
+      returns_message=True,
+      **kwargs,
+  )
+  if lm is None:
+    lm_override = lf.get_contextual_override('lm')
+    if lm_override is None:
+      raise ValueError('`lm` must be specified or provided from `lf.context`.')
+    lm = lm_override.value
+
+  results = lm.score(
+      input_message,
+      [
+          mapping.MappingExample.value_repr(
+              c, protocol=protocol, compact=False, verbose=False
+          )
+          for c in completions
+      ],
+  )
+  return [r.score for r in results]

langfun-0.0.2.dev20240217/langfun/core/structured/scoring_test.py

@@ -0,0 +1,44 @@
+# Copyright 2023 The Langfun Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+import langfun.core as lf
+from langfun.core.llms import fake
+from langfun.core.structured import scoring
+
+
+class ScoringTest(unittest.TestCase):
+
+  def test_bad_call(self):
+    with self.assertRaisesRegex(ValueError, '`completions` must not be empty'):
+      scoring.score('hi', [])
+
+    with self.assertRaisesRegex(
+        ValueError, '`schema` cannot be inferred from completions'
+    ):
+      scoring.score('hi', [1, 'b'])
+
+    with self.assertRaisesRegex(ValueError, '`lm` must be specified'):
+      scoring.score('hi', [1, 2])
+
+  def test_score(self):
+    self.assertEqual(scoring.score('hi', [1, 2], lm=fake.Echo()), [0.0, -1.0])
+
+  def test_scope_with_lm_from_the_context(self):
+    with lf.context(lm=fake.Echo()):
+      self.assertEqual(scoring.score('hi', [1, 2]), [0.0, -1.0])
+
+
+if __name__ == '__main__':
+  unittest.main()

{langfun-0.0.2.dev20240215 → langfun-0.0.2.dev20240217}/langfun.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: langfun
-Version: 0.0.2.dev20240215
+Version: 0.0.2.dev20240217
 Summary: Langfun: Language as Functions.
 Home-page: https://github.com/google/langfun
 Author: Langfun Authors

{langfun-0.0.2.dev20240215 → langfun-0.0.2.dev20240217}/langfun.egg-info/SOURCES.txt

@@ -86,6 +86,8 @@ langfun/core/structured/prompting.py
 langfun/core/structured/prompting_test.py
 langfun/core/structured/schema.py
 langfun/core/structured/schema_test.py
+langfun/core/structured/scoring.py
+langfun/core/structured/scoring_test.py
 langfun/core/templates/__init__.py
 langfun/core/templates/completion.py
 langfun/core/templates/completion_test.py