langfun 0.0.2.dev20240208__tar.gz → 0.0.2.dev20240210__tar.gz

{langfun-0.0.2.dev20240208 → langfun-0.0.2.dev20240210}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: langfun
-Version: 0.0.2.dev20240208
+Version: 0.0.2.dev20240210
 Summary: Langfun: Language as Functions.
 Home-page: https://github.com/google/langfun
 Author: Langfun Authors
@@ -24,7 +24,7 @@ License-File: LICENSE
 Requires-Dist: google-generativeai>=0.3.2
 Requires-Dist: jinja2>=3.1.2
 Requires-Dist: openai==0.27.2
-Requires-Dist: pyglove>=0.4.5.dev20240109
+Requires-Dist: pyglove>=0.4.5.dev20240201
 Requires-Dist: requests>=2.31.0
 Requires-Dist: termcolor==1.1.0
 Requires-Dist: tqdm>=4.64.1

{langfun-0.0.2.dev20240208 → langfun-0.0.2.dev20240210}/langfun/core/__init__.py

@@ -109,6 +109,8 @@ from langfun.core.memory import Memory
 # Utility for console output.
 from langfun.core import console
 
+# Import internal modules.
+
 # pylint: enable=g-import-not-at-top
 # pylint: enable=g-importing-member
 # pylint: enable=g-bad-import-order

{langfun-0.0.2.dev20240208 → langfun-0.0.2.dev20240210}/langfun/core/coding/python/correction.py

@@ -83,7 +83,7 @@ def run_with_correction(
   # pylint: disable=g-import-not-at-top
   # pytype: disable=import-error
   from langfun.core.structured import prompting
-  # pytype: disable=import-error
+  # pytype: enable=import-error
   # pylint: enable=g-import-not-at-top
 
   code = remove_docstrings(code)

{langfun-0.0.2.dev20240208 → langfun-0.0.2.dev20240210}/langfun/core/concurrent.py

@@ -538,11 +538,12 @@ def concurrent_map(
     An iterator of (input, output, error).
 
   Raises:
-    Exception: Erros that are not in `silence_on_errors` or `retry_on_errors`,
+    Exception: Errors that are not in `silence_on_errors` or `retry_on_errors`,
       or retry on such errors has reached max attempts.
     TimeoutError: Any item timed out while TimeoutError is not silenced via
       `silence_on_errors`.
   """
+    # Internal usage logging.
 
   if retry_on_errors:
     func = with_retry(
@@ -683,7 +684,7 @@ def concurrent_map(
 class ExecutorPool:
   """A pool of managed executors.
 
-  Managed executors are used for controlling the parallism of execution based
+  Managed executors are used for controlling the parallelism of execution based
   on resource id. This design is to honor overall rate limit of LMs globally
   (with current process).
   """

{langfun-0.0.2.dev20240208 → langfun-0.0.2.dev20240210}/langfun/core/eval/base.py

@@ -211,6 +211,8 @@ class Evaluable(lf.Component):
       **kwargs,
   ) -> Union['Summary', pg.Dict]:
     """Run the evaluation, which fills and returns the result."""
+    # Internal usage logging.
+
     if dryrun:
       self.dryrun(filter=filter, verbose=False, debug=debug)
 
@@ -582,8 +584,22 @@ class Suite(Evaluable):
 
   children: Annotated[list[Evaluable], 'Child evaluation sets or suites.']
 
+  __kwargs__: Annotated[
+      Any,
+      (
+          'Wildcard keyword arguments for `__init__` that can be accessed from '
+          'parent suite if the argument is absent from current evaluation set.'
+      ),
+  ]
+
   def _on_bound(self):
     super()._on_bound()
+    overrides = {
+        k: v for k, v in self.sym_init_args.items()
+        if k not in ('id', 'children')
+    }
+    for child in self.children:
+      child.rebind(overrides, notify_parents=False)
     self.__dict__.pop('hash', None)
 
   @functools.cached_property
@@ -618,7 +634,7 @@ class Evaluation(Evaluable):
 
   method: Annotated[
       Literal['call', 'query', 'complete'], 'Method for symbolic prompting.'
-  ]
+  ] = lf.contextual(default='query')
 
   prompt: Annotated[
       lf.Template,
@@ -626,7 +642,7 @@ class Evaluation(Evaluable):
           'Template for rendering the template. Example object could be '
           'accessed via `example`.'
       ),
-  ]
+  ] = lf.contextual()
 
   schema_fn: pg.typing.Annotated[
       pg.typing.Functor().noneable(),
@@ -660,9 +676,11 @@ class Evaluation(Evaluable):
               ```
               """)
       ),
-  ]
+  ] = lf.contextual()
 
-  lm: Annotated[lf.LanguageModel, 'Language model to use for evaluation.']
+  lm: Annotated[lf.LanguageModel, 'Language model to use for evaluation.'] = (
+      lf.contextual()
+  )
 
   parsing_lm: Annotated[
       lf.LanguageModel | None,
@@ -670,7 +688,7 @@ class Evaluation(Evaluable):
           'Language model for parsing. Applicable only when method is set'
           'to `call`. If None, `lm` will also be used for parsing. '
       ),
-  ] = None
+  ] = lf.contextual(default=None)
 
   completion_prompt_field: Annotated[
       str | None,
@@ -680,8 +698,8 @@ class Evaluation(Evaluable):
           'the class, instead the prompt will be passed as the first argument '
           'of the input object to complete. Applicable only when `method` is '
           'set to `complete`.'
-      )
-  ] = None
+      ),
+  ] = lf.contextual(default=None)
 
   autofix: Annotated[
       int,
@@ -690,7 +708,7 @@ class Evaluation(Evaluable):
           'generated code for the output structure. If 0, autofix will be '
           'disabled.'
       ),
-  ] = 0
+  ] = lf.contextual(default=0)
 
   autofix_lm: Annotated[
       lf.LanguageModel | None,
@@ -698,14 +716,16 @@ class Evaluation(Evaluable):
           'Language model for autofix. If None, `lm` will also be used for '
           'autofix.'
       ),
-  ] = None
+  ] = lf.contextual(default=None)
 
   additional_args: Annotated[
       dict[str, Any] | None,
-      'Additional kwargs that will be passed to `self.process`'
-  ] = None
+      'Additional kwargs that will be passed to `self.process`',
+  ] = lf.contextual(default=None)
 
-  use_cache: Annotated[bool, 'If True, LM cache will be enabled.'] = True
+  use_cache: Annotated[bool, 'If True, LM cache will be enabled.'] = (
+      lf.contextual(default=True)
+  )
 
   max_workers: Annotated[
       int, 'Max workers to run the evaluation in parallel.'
@@ -744,7 +764,11 @@ class Evaluation(Evaluable):
   @functools.cached_property
   def examples(self):
     """Returns examples for evaluation."""
-    return self.inputs()
+    kwargs = {}
+    # Allow inputs to be dependent on current evaluation.
+    if 'evaluation' in self.inputs.__signature__.arg_names:
+      kwargs['evaluation'] = self
+    return self.inputs(**kwargs)
 
   @property
   def num_examples(self) -> int:
@@ -784,7 +808,12 @@ class Evaluation(Evaluable):
     if self.schema_fn is None:
       return None
 
-    schema = self.schema_fn()
+    kwargs = {}
+    # Allow schema to be a function based on current evaluation.
+    if 'evaluation' in self.schema_fn.__signature__.arg_names:
+      kwargs['evaluation'] = self
+
+    schema = self._call_schema_fn()
     fewshot_examples = None
     if isinstance(schema, tuple):
       schema, fewshot_examples = schema
@@ -798,13 +827,20 @@ class Evaluation(Evaluable):
     if self.schema_fn is None:
       return None
 
-    schema = self.schema_fn()
+    schema = self._call_schema_fn()
     fewshot_examples = None
     if isinstance(schema, tuple):
       schema, fewshot_examples = schema
     self.__dict__['schema'] = self._formalize_schema(schema)
     return self._maybe_adjust_examples_for_completion(fewshot_examples)
 
+  def _call_schema_fn(self):
+    kwargs = {}
+    # Allow schema to be a function based on current evaluation.
+    if 'evaluation' in self.schema_fn.__signature__.arg_names:
+      kwargs['evaluation'] = self
+    return self.schema_fn(**kwargs)
+
   def _formalize_schema(self, annotation) -> lf_structured.Schema:
     """Formalizes schema from annotation."""
     if self.method == 'complete':
@@ -854,9 +890,9 @@ class Evaluation(Evaluable):
       return []
     children = []
     for i, child in enumerate(pg.iter(self)):
-      child.rebind(id=f'{self.id}@{child.hash}', skip_notification=True)
       child.sym_setparent(self)
       child.sym_setpath(self.sym_path + f'children[{i}]')
+      child.rebind(id=f'{self.id}@{child.hash}', skip_notification=True)
       children.append(child)
     return children
 
@@ -873,9 +909,6 @@ class Evaluation(Evaluable):
 
   def _on_bound(self):
     super()._on_bound()
-    if self.method != 'call' and self.schema_fn is None:
-      raise ValueError(
-          f'`schema_fn` must be specified for method {self.method!r}')
     self.__dict__.pop('hash', None)
     self.__dict__.pop('children', None)
     self.__dict__.pop('examples', None)
@@ -1442,11 +1475,11 @@ class Summary(pg.Object):
       cols = set()
       nonpivot_values = collections.defaultdict(set)
       for e in evaluations:
-        for k, v in e.sym_init_args.items():
+        for k in e.sym_init_args:
           if pivot_field == k:
-            cols.add(SymbolicComparable(v))
+            cols.add(SymbolicComparable(e.sym_inferred(k)))
           elif k not in ('id', 'groundtruth'):
-            nonpivot_values[k].add(SymbolicComparable(v))
+            nonpivot_values[k].add(SymbolicComparable(e.sym_inferred(k)))
 
       cols = sorted(cols)
 

{langfun-0.0.2.dev20240208 → langfun-0.0.2.dev20240210}/langfun/core/eval/base_test.py

@@ -42,7 +42,8 @@ def answer_schema():
 
 
 @pg.functor
-def answer_schema_with_fewshot_examples():
+def answer_schema_with_fewshot_examples(evaluation):
+  del evaluation
   return Solution, [
       lf_structured.MappingExample(
           input='The result of one plus two',
@@ -61,7 +62,7 @@ def eval_set(
     eval_id: str,
     method: str,
     schema_fn,
-    lm: lf.LanguageModel,
+    lm: lf.LanguageModel = pg.MISSING_VALUE,
     use_cache: bool = True,
     cls: Type[base.Evaluation] = base.Evaluation,
     **kwargs,
@@ -170,8 +171,6 @@ class EvaluationTest(unittest.TestCase):
     )
 
   def test_bad_init(self):
-    with self.assertRaisesRegex(ValueError, '.*'):
-      eval_set('bad_init1', 'complete', None, lm=fake.StaticResponse('hi'))
 
     @pg.functor()
     def _bad_completion_schema():
@@ -438,18 +437,18 @@ class SuiteTest(unittest.TestCase):
     lm = fake.StaticSequence([
         'Solution(final_answer=2)',
         '3',
-    ])
+    ] * 5)
     s = base.Suite(
         'suite_run_test',
         [
-            eval_set('run_test_1', 'query', schema_fn=answer_schema(), lm=lm),
+            eval_set('run_test_1', 'query', schema_fn=answer_schema()),
             # A suite of search space. Two of the sub-experiments are identical,
             # thus the result of run_test_2 would include only two keys.
             eval_set('run_test_2',
                      pg.oneof(['call', 'query']),
-                     schema_fn=pg.oneof([answer_schema(), answer_schema()]),
-                     lm=lm),
+                     schema_fn=pg.oneof([answer_schema(), answer_schema()])),
         ],
+        lm=lm
     )
     # Test for persistent hash.
     self.assertEqual(s.hash, '7285e52b')
@@ -482,7 +481,7 @@ class SuiteTest(unittest.TestCase):
                     schema_fn='answer_schema()',
                 ),
                 cache_stats=dict(
-                    use_cache=True, num_queries=3, num_hits=0, num_updates=2
+                    use_cache=True, num_queries=4, num_hits=0, num_updates=4
                 ),
                 metrics=dict(total=2, failures=2, failure_rate=1.0),
             ),

{langfun-0.0.2.dev20240208 → langfun-0.0.2.dev20240210}/langfun/core/language_model.py

@@ -288,6 +288,8 @@ class LanguageModel(component.Component):
       **kwargs,
   ) -> list[LMSamplingResult]:
     """Samples one or multiple prompts."""
+    # Internal usage logging.
+
     prompts = [message_lib.UserMessage.from_value(p) for p in prompts]
 
     with component.context(override_attrs=True, **kwargs):

{langfun-0.0.2.dev20240208 → langfun-0.0.2.dev20240210}/langfun/core/structured/prompting.py

@@ -26,7 +26,7 @@ class QueryStructure(mapping.Mapping):
   """Query an object out from a natural language text."""
 
   context_title = 'CONTEXT'
-  input_title = 'USER_REQUEST'
+  input_title = 'INPUT_OBJECT'
 
   # Mark schema as required.
   schema: pg.typing.Annotated[
@@ -65,11 +65,6 @@ class QueryStructurePython(QueryStructure):
   preamble = """
       Please respond to the last {{ input_title }} with {{ output_title }} according to {{ schema_title }}.
 
-      INSTRUCTIONS:
-        1. Only respond with the required {{ output_title }} encapsulated with python markdown as illustrated by the given example.
-        2. Don't add any comments in the response.
-        3. {{ output_title }} must strictly follow the {{ schema_title }}.
-
       {{ input_title }}:
         1 + 1 =
 
@@ -87,8 +82,8 @@ class QueryStructurePython(QueryStructure):
         ```
       """
   protocol = 'python'
-  schema_title = 'RESULT_TYPE'
-  output_title = 'RESULT_OBJECT'
+  schema_title = 'OUTPUT_TYPE'
+  output_title = 'OUTPUT_OBJECT'
 
 
 def _query_structure_cls(
@@ -174,6 +169,8 @@ def query(
   Returns:
     The result based on the schema.
   """
+    # Internal usage logging.
+
   # When `lf.query` is used for symbolic completion, schema is automatically
   # inferred when it is None.
   if isinstance(prompt, pg.Symbolic) and prompt.sym_partial and schema is None:

{langfun-0.0.2.dev20240208 → langfun-0.0.2.dev20240210}/langfun/core/structured/prompting_test.py

@@ -115,7 +115,14 @@ class QueryTest(unittest.TestCase):
         x=1,
         y=2,
         lm=lm.clone(),
-        expected_snippet='\n\nUSER_REQUEST:\n  What is 1 + 2?\n\n',
+        expected_snippet=(
+            'Please respond to the last INPUT_OBJECT with OUTPUT_OBJECT'
+            ' according to OUTPUT_TYPE.\n\nINPUT_OBJECT:\n  1 + 1'
+            ' =\n\nOUTPUT_TYPE:\n  Answer\n\n  ```python\n  class Answer:\n   '
+            ' final_answer: int\n  ```\n\nOUTPUT_OBJECT:\n  ```python\n '
+            ' Answer(final_answer=2)\n  ```\n\nINPUT_OBJECT:\n  What is 1 +'
+            ' 2?\n\nOUTPUT_TYPE:\n  int\n\nOUTPUT_OBJECT:'
+        ),
     )
 
   def test_str_to_str_render(self):
@@ -139,7 +146,7 @@ class QueryTest(unittest.TestCase):
         y=2,
         lm=lm.clone(),
         expected_snippet=(
-            '\n\nUSER_REQUEST:\n  ```python\n  [\n    1\n  ]\n  ```\n\n'
+            '\n\nINPUT_OBJECT:\n  ```python\n  [\n    1\n  ]\n  ```\n\n'
         ),
     )
 
@@ -155,7 +162,7 @@ class QueryTest(unittest.TestCase):
         modalities.Image.from_bytes(b'mock_image'),
         int,
         lm=lm,
-        expected_snippet='\n\nUSER_REQUEST:\n  {{input}}\n\n',
+        expected_snippet='\n\nINPUT_OBJECT:\n  {{input}}\n\n',
         expected_modalities=1,
     )
 
@@ -207,7 +214,7 @@ class QueryTest(unittest.TestCase):
         list[str],
         lm=lm,
         expected_snippet=inspect.cleandoc("""
-            USER_REQUEST:
+            INPUT_OBJECT:
               ```python
               [
                 ModalityRef(
@@ -238,17 +245,12 @@ class QueryStructurePythonTest(unittest.TestCase):
     self.assertEqual(
         l.render().text,
         inspect.cleandoc("""
-            Please respond to the last USER_REQUEST with RESULT_OBJECT according to RESULT_TYPE.
-
-            INSTRUCTIONS:
-              1. Only respond with the required RESULT_OBJECT encapsulated with python markdown as illustrated by the given example.
-              2. Don't add any comments in the response.
-              3. RESULT_OBJECT must strictly follow the RESULT_TYPE.
+            Please respond to the last INPUT_OBJECT with OUTPUT_OBJECT according to OUTPUT_TYPE.
 
-            USER_REQUEST:
+            INPUT_OBJECT:
               1 + 1 =
 
-            RESULT_TYPE:
+            OUTPUT_TYPE:
               Answer
 
               ```python
@@ -256,18 +258,18 @@ class QueryStructurePythonTest(unittest.TestCase):
                 final_answer: int
               ```
 
-            RESULT_OBJECT:
+            OUTPUT_OBJECT:
               ```python
               Answer(final_answer=2)
               ```
 
-            USER_REQUEST:
+            INPUT_OBJECT:
               Compute 12 / 6 + 2.
 
-            RESULT_TYPE:
+            OUTPUT_TYPE:
               int
 
-            RESULT_OBJECT:
+            OUTPUT_OBJECT:
             """),
     )
 
@@ -287,17 +289,12 @@ class QueryStructurePythonTest(unittest.TestCase):
     self.assertEqual(
         l.render().text,
         inspect.cleandoc("""
-            Please respond to the last USER_REQUEST with RESULT_OBJECT according to RESULT_TYPE.
-
-            INSTRUCTIONS:
-              1. Only respond with the required RESULT_OBJECT encapsulated with python markdown as illustrated by the given example.
-              2. Don't add any comments in the response.
-              3. RESULT_OBJECT must strictly follow the RESULT_TYPE.
+            Please respond to the last INPUT_OBJECT with OUTPUT_OBJECT according to OUTPUT_TYPE.
 
-            USER_REQUEST:
+            INPUT_OBJECT:
               1 + 1 =
 
-            RESULT_TYPE:
+            OUTPUT_TYPE:
               Answer
 
               ```python
@@ -305,41 +302,41 @@ class QueryStructurePythonTest(unittest.TestCase):
                 final_answer: int
               ```
 
-            RESULT_OBJECT:
+            OUTPUT_OBJECT:
               ```python
               Answer(final_answer=2)
               ```
 
-            USER_REQUEST:
+            INPUT_OBJECT:
               What is the answer of 1 plus 1?
 
-            RESULT_TYPE:
+            OUTPUT_TYPE:
               int
 
-            RESULT_OBJECT:
+            OUTPUT_OBJECT:
               ```python
               2
               ```
 
-            USER_REQUEST:
+            INPUT_OBJECT:
               Compute the value of 3 + (2 * 6).
 
-            RESULT_TYPE:
+            OUTPUT_TYPE:
               int
 
-            RESULT_OBJECT:
+            OUTPUT_OBJECT:
               ```python
               15
               ```
 
 
-            USER_REQUEST:
+            INPUT_OBJECT:
               Compute 12 / 6 + 2.
 
-            RESULT_TYPE:
+            OUTPUT_TYPE:
               int
 
-            RESULT_OBJECT:
+            OUTPUT_OBJECT:
             """),
     )
 
@@ -445,13 +442,13 @@ class QueryStructureJsonTest(unittest.TestCase):
     self.assertEqual(
         l.render().text,
         inspect.cleandoc("""
-            Please respond to the last USER_REQUEST with JSON according to SCHEMA:
+            Please respond to the last INPUT_OBJECT with JSON according to SCHEMA:
 
             INSTRUCTIONS:
               1. If the schema has `_type`, carry it over to the JSON output.
               2. If a field from the schema cannot be extracted from the response, use null as the JSON value.
 
-            USER_REQUEST:
+            INPUT_OBJECT:
               1 + 1 =
 
             SCHEMA:
@@ -460,7 +457,7 @@ class QueryStructureJsonTest(unittest.TestCase):
             JSON:
               {"result": {"_type": "langfun.core.structured.prompting.Answer", "final_answer": 2}}
 
-            USER_REQUEST:
+            INPUT_OBJECT:
               Compute 12 / 6 + 2.
 
             SCHEMA:
@@ -482,13 +479,13 @@ class QueryStructureJsonTest(unittest.TestCase):
     self.assertEqual(
         l.render().text,
         inspect.cleandoc("""
-            Please respond to the last USER_REQUEST with JSON according to SCHEMA:
+            Please respond to the last INPUT_OBJECT with JSON according to SCHEMA:
 
             INSTRUCTIONS:
               1. If the schema has `_type`, carry it over to the JSON output.
               2. If a field from the schema cannot be extracted from the response, use null as the JSON value.
 
-            USER_REQUEST:
+            INPUT_OBJECT:
               1 + 1 =
 
             SCHEMA:
@@ -497,7 +494,7 @@ class QueryStructureJsonTest(unittest.TestCase):
             JSON:
               {"result": {"_type": "langfun.core.structured.prompting.Answer", "final_answer": 2}}
 
-            USER_REQUEST:
+            INPUT_OBJECT:
               What is the answer of 1 plus 1?
 
             SCHEMA:
@@ -506,7 +503,7 @@ class QueryStructureJsonTest(unittest.TestCase):
             JSON:
               {"result": 2}
 
-            USER_REQUEST:
+            INPUT_OBJECT:
               Compute the value of 3 + (2 * 6).
 
             SCHEMA:
@@ -516,7 +513,7 @@ class QueryStructureJsonTest(unittest.TestCase):
               {"result": 15}
 
 
-            USER_REQUEST:
+            INPUT_OBJECT:
               Compute 12 / 6 + 2.
 
             SCHEMA:

{langfun-0.0.2.dev20240208 → langfun-0.0.2.dev20240210}/langfun.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: langfun
-Version: 0.0.2.dev20240208
+Version: 0.0.2.dev20240210
 Summary: Langfun: Language as Functions.
 Home-page: https://github.com/google/langfun
 Author: Langfun Authors
@@ -24,7 +24,7 @@ License-File: LICENSE
 Requires-Dist: google-generativeai>=0.3.2
 Requires-Dist: jinja2>=3.1.2
 Requires-Dist: openai==0.27.2
-Requires-Dist: pyglove>=0.4.5.dev20240109
+Requires-Dist: pyglove>=0.4.5.dev20240201
 Requires-Dist: requests>=2.31.0
 Requires-Dist: termcolor==1.1.0
 Requires-Dist: tqdm>=4.64.1

{langfun-0.0.2.dev20240208 → langfun-0.0.2.dev20240210}/langfun.egg-info/requires.txt

@@ -1,7 +1,7 @@
 google-generativeai>=0.3.2
 jinja2>=3.1.2
 openai==0.27.2
-pyglove>=0.4.5.dev20240109
+pyglove>=0.4.5.dev20240201
 requests>=2.31.0
 termcolor==1.1.0
 tqdm>=4.64.1