langfun 0.0.2.dev20240207__py3-none-any.whl → 0.0.2.dev20240209__py3-none-any.whl

langfun/core/__init__.py

@@ -109,6 +109,8 @@ from langfun.core.memory import Memory
 # Utility for console output.
 from langfun.core import console
 
+# Import internal modules.
+
 # pylint: enable=g-import-not-at-top
 # pylint: enable=g-importing-member
 # pylint: enable=g-bad-import-order

langfun/core/coding/python/correction.py

@@ -83,7 +83,7 @@ def run_with_correction(
   # pylint: disable=g-import-not-at-top
   # pytype: disable=import-error
   from langfun.core.structured import prompting
-  # pytype: disable=import-error
+  # pytype: enable=import-error
   # pylint: enable=g-import-not-at-top
 
   code = remove_docstrings(code)

langfun/core/concurrent.py

@@ -538,11 +538,12 @@ def concurrent_map(
     An iterator of (input, output, error).
 
   Raises:
-    Exception: Erros that are not in `silence_on_errors` or `retry_on_errors`,
+    Exception: Errors that are not in `silence_on_errors` or `retry_on_errors`,
       or retry on such errors has reached max attempts.
     TimeoutError: Any item timed out while TimeoutError is not silenced via
       `silence_on_errors`.
   """
+    # Internal usage logging.
 
   if retry_on_errors:
     func = with_retry(
@@ -683,7 +684,7 @@ def concurrent_map(
 class ExecutorPool:
   """A pool of managed executors.
 
-  Managed executors are used for controlling the parallism of execution based
+  Managed executors are used for controlling the parallelism of execution based
   on resource id. This design is to honor overall rate limit of LMs globally
   (with current process).
   """

langfun/core/eval/base.py

@@ -211,6 +211,8 @@ class Evaluable(lf.Component):
       **kwargs,
   ) -> Union['Summary', pg.Dict]:
     """Run the evaluation, which fills and returns the result."""
+    # Internal usage logging.
+
     if dryrun:
       self.dryrun(filter=filter, verbose=False, debug=debug)
 
@@ -582,8 +584,22 @@ class Suite(Evaluable):
 
   children: Annotated[list[Evaluable], 'Child evaluation sets or suites.']
 
+  __kwargs__: Annotated[
+      Any,
+      (
+          'Wildcard keyword arguments for `__init__` that can be accessed from '
+          'parent suite if the argument is absent from current evaluation set.'
+      ),
+  ]
+
   def _on_bound(self):
     super()._on_bound()
+    overrides = {
+        k: v for k, v in self.sym_init_args.items()
+        if k not in ('id', 'children')
+    }
+    for child in self.children:
+      child.rebind(overrides, notify_parents=False)
     self.__dict__.pop('hash', None)
 
   @functools.cached_property
@@ -618,7 +634,7 @@ class Evaluation(Evaluable):
 
   method: Annotated[
       Literal['call', 'query', 'complete'], 'Method for symbolic prompting.'
-  ]
+  ] = lf.contextual(default='query')
 
   prompt: Annotated[
       lf.Template,
@@ -626,7 +642,7 @@ class Evaluation(Evaluable):
           'Template for rendering the template. Example object could be '
           'accessed via `example`.'
       ),
-  ]
+  ] = lf.contextual()
 
   schema_fn: pg.typing.Annotated[
       pg.typing.Functor().noneable(),
@@ -660,9 +676,11 @@ class Evaluation(Evaluable):
               ```
               """)
       ),
-  ]
+  ] = lf.contextual()
 
-  lm: Annotated[lf.LanguageModel, 'Language model to use for evaluation.']
+  lm: Annotated[lf.LanguageModel, 'Language model to use for evaluation.'] = (
+      lf.contextual()
+  )
 
   parsing_lm: Annotated[
       lf.LanguageModel | None,
@@ -670,7 +688,7 @@ class Evaluation(Evaluable):
           'Language model for parsing. Applicable only when method is set'
           'to `call`. If None, `lm` will also be used for parsing. '
       ),
-  ] = None
+  ] = lf.contextual(default=None)
 
   completion_prompt_field: Annotated[
       str | None,
@@ -680,8 +698,8 @@ class Evaluation(Evaluable):
           'the class, instead the prompt will be passed as the first argument '
           'of the input object to complete. Applicable only when `method` is '
           'set to `complete`.'
-      )
-  ] = None
+      ),
+  ] = lf.contextual(default=None)
 
   autofix: Annotated[
       int,
@@ -690,7 +708,7 @@ class Evaluation(Evaluable):
           'generated code for the output structure. If 0, autofix will be '
           'disabled.'
       ),
-  ] = 0
+  ] = lf.contextual(default=0)
 
   autofix_lm: Annotated[
       lf.LanguageModel | None,
@@ -698,14 +716,16 @@ class Evaluation(Evaluable):
           'Language model for autofix. If None, `lm` will also be used for '
           'autofix.'
       ),
-  ] = None
+  ] = lf.contextual(default=None)
 
   additional_args: Annotated[
       dict[str, Any] | None,
-      'Additional kwargs that will be passed to `self.process`'
-  ] = None
+      'Additional kwargs that will be passed to `self.process`',
+  ] = lf.contextual(default=None)
 
-  use_cache: Annotated[bool, 'If True, LM cache will be enabled.'] = True
+  use_cache: Annotated[bool, 'If True, LM cache will be enabled.'] = (
+      lf.contextual(default=True)
+  )
 
   max_workers: Annotated[
       int, 'Max workers to run the evaluation in parallel.'
@@ -744,7 +764,11 @@ class Evaluation(Evaluable):
   @functools.cached_property
   def examples(self):
     """Returns examples for evaluation."""
-    return self.inputs()
+    kwargs = {}
+    # Allow inputs to be dependent on current evaluation.
+    if 'evaluation' in self.inputs.__signature__.arg_names:
+      kwargs['evaluation'] = self
+    return self.inputs(**kwargs)
 
   @property
   def num_examples(self) -> int:
@@ -784,7 +808,12 @@ class Evaluation(Evaluable):
     if self.schema_fn is None:
       return None
 
-    schema = self.schema_fn()
+    kwargs = {}
+    # Allow schema to be a function based on current evaluation.
+    if 'evaluation' in self.schema_fn.__signature__.arg_names:
+      kwargs['evaluation'] = self
+
+    schema = self._call_schema_fn()
     fewshot_examples = None
     if isinstance(schema, tuple):
       schema, fewshot_examples = schema
@@ -798,13 +827,20 @@ class Evaluation(Evaluable):
     if self.schema_fn is None:
       return None
 
-    schema = self.schema_fn()
+    schema = self._call_schema_fn()
     fewshot_examples = None
     if isinstance(schema, tuple):
       schema, fewshot_examples = schema
     self.__dict__['schema'] = self._formalize_schema(schema)
     return self._maybe_adjust_examples_for_completion(fewshot_examples)
 
+  def _call_schema_fn(self):
+    kwargs = {}
+    # Allow schema to be a function based on current evaluation.
+    if 'evaluation' in self.schema_fn.__signature__.arg_names:
+      kwargs['evaluation'] = self
+    return self.schema_fn(**kwargs)
+
   def _formalize_schema(self, annotation) -> lf_structured.Schema:
     """Formalizes schema from annotation."""
     if self.method == 'complete':
@@ -854,9 +890,9 @@ class Evaluation(Evaluable):
       return []
     children = []
     for i, child in enumerate(pg.iter(self)):
-      child.rebind(id=f'{self.id}@{child.hash}', skip_notification=True)
       child.sym_setparent(self)
       child.sym_setpath(self.sym_path + f'children[{i}]')
+      child.rebind(id=f'{self.id}@{child.hash}', skip_notification=True)
       children.append(child)
     return children
 
@@ -873,9 +909,6 @@ class Evaluation(Evaluable):
 
   def _on_bound(self):
     super()._on_bound()
-    if self.method != 'call' and self.schema_fn is None:
-      raise ValueError(
-          f'`schema_fn` must be specified for method {self.method!r}')
     self.__dict__.pop('hash', None)
     self.__dict__.pop('children', None)
     self.__dict__.pop('examples', None)
@@ -1442,11 +1475,11 @@ class Summary(pg.Object):
       cols = set()
       nonpivot_values = collections.defaultdict(set)
       for e in evaluations:
-        for k, v in e.sym_init_args.items():
+        for k in e.sym_init_args:
           if pivot_field == k:
-            cols.add(SymbolicComparable(v))
+            cols.add(SymbolicComparable(e.sym_inferred(k)))
           elif k not in ('id', 'groundtruth'):
-            nonpivot_values[k].add(SymbolicComparable(v))
+            nonpivot_values[k].add(SymbolicComparable(e.sym_inferred(k)))
 
       cols = sorted(cols)
 

langfun/core/eval/base_test.py

@@ -42,7 +42,8 @@ def answer_schema():
 
 
 @pg.functor
-def answer_schema_with_fewshot_examples():
+def answer_schema_with_fewshot_examples(evaluation):
+  del evaluation
   return Solution, [
       lf_structured.MappingExample(
           input='The result of one plus two',
@@ -61,7 +62,7 @@ def eval_set(
     eval_id: str,
     method: str,
     schema_fn,
-    lm: lf.LanguageModel,
+    lm: lf.LanguageModel = pg.MISSING_VALUE,
     use_cache: bool = True,
     cls: Type[base.Evaluation] = base.Evaluation,
     **kwargs,
@@ -170,8 +171,6 @@ class EvaluationTest(unittest.TestCase):
     )
 
   def test_bad_init(self):
-    with self.assertRaisesRegex(ValueError, '.*'):
-      eval_set('bad_init1', 'complete', None, lm=fake.StaticResponse('hi'))
 
     @pg.functor()
     def _bad_completion_schema():
@@ -438,18 +437,18 @@ class SuiteTest(unittest.TestCase):
     lm = fake.StaticSequence([
         'Solution(final_answer=2)',
         '3',
-    ])
+    ] * 5)
     s = base.Suite(
         'suite_run_test',
         [
-            eval_set('run_test_1', 'query', schema_fn=answer_schema(), lm=lm),
+            eval_set('run_test_1', 'query', schema_fn=answer_schema()),
             # A suite of search space. Two of the sub-experiments are identical,
             # thus the result of run_test_2 would include only two keys.
             eval_set('run_test_2',
                      pg.oneof(['call', 'query']),
-                     schema_fn=pg.oneof([answer_schema(), answer_schema()]),
-                     lm=lm),
+                     schema_fn=pg.oneof([answer_schema(), answer_schema()])),
         ],
+        lm=lm
     )
     # Test for persistent hash.
     self.assertEqual(s.hash, '7285e52b')
@@ -482,7 +481,7 @@ class SuiteTest(unittest.TestCase):
                     schema_fn='answer_schema()',
                 ),
                 cache_stats=dict(
-                    use_cache=True, num_queries=3, num_hits=0, num_updates=2
+                    use_cache=True, num_queries=4, num_hits=0, num_updates=4
                 ),
                 metrics=dict(total=2, failures=2, failure_rate=1.0),
             ),

langfun/core/language_model.py

@@ -288,6 +288,8 @@ class LanguageModel(component.Component):
       **kwargs,
   ) -> list[LMSamplingResult]:
     """Samples one or multiple prompts."""
+    # Internal usage logging.
+
     prompts = [message_lib.UserMessage.from_value(p) for p in prompts]
 
     with component.context(override_attrs=True, **kwargs):

langfun/core/structured/prompting.py

@@ -174,6 +174,8 @@ def query(
   Returns:
     The result based on the schema.
   """
+    # Internal usage logging.
+
   # When `lf.query` is used for symbolic completion, schema is automatically
   # inferred when it is None.
   if isinstance(prompt, pg.Symbolic) and prompt.sym_partial and schema is None:

{langfun-0.0.2.dev20240207.dist-info → langfun-0.0.2.dev20240209.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: langfun
-Version: 0.0.2.dev20240207
+Version: 0.0.2.dev20240209
 Summary: Langfun: Language as Functions.
 Home-page: https://github.com/google/langfun
 Author: Langfun Authors
@@ -24,7 +24,7 @@ License-File: LICENSE
 Requires-Dist: google-generativeai >=0.3.2
 Requires-Dist: jinja2 >=3.1.2
 Requires-Dist: openai ==0.27.2
-Requires-Dist: pyglove >=0.4.5.dev20240109
+Requires-Dist: pyglove >=0.4.5.dev20240201
 Requires-Dist: requests >=2.31.0
 Requires-Dist: termcolor ==1.1.0
 Requires-Dist: tqdm >=4.64.1

{langfun-0.0.2.dev20240207.dist-info → langfun-0.0.2.dev20240209.dist-info}/RECORD

@@ -1,14 +1,14 @@
 langfun/__init__.py,sha256=2HUBxiByAEu63XqaF89hQfI4sqFG1qGffua-JPy4XIY,1689
-langfun/core/__init__.py,sha256=0dIlw2ehlzx7XtTTAyDCXIsX_jKB2F6gQMki-bnqqNM,4108
+langfun/core/__init__.py,sha256=dl7itWvZUEvqDeK2EWd-9lGlZu8cLXCO45HcaZKWAo4,4136
 langfun/core/component.py,sha256=VRPfDB_2jEnxcB3-HoiVjG4ID-SMenNPIsytb0uXMPg,9674
 langfun/core/component_test.py,sha256=VAPd6V_-odAe8rBvesW3ogYDd6OSqRq4FaPhfgOM4Zg,7949
-langfun/core/concurrent.py,sha256=If8fYv3BrdAuV5yysmPLz24UKlqiKCExD3l1ZkQqGCA,24114
+langfun/core/concurrent.py,sha256=HQJOseNZ-XZZR5VmC8lHoDNFzlkkCa_-ri7nOKJfV5s,24147
 langfun/core/concurrent_test.py,sha256=qQT6_Dq5NVz7qXFLzSf2Rhzkfkh07gocjHMBaT1nSeE,14928
 langfun/core/console.py,sha256=bk5rNPNm9rMGW5YT2HixxU04p2umnoabn5SDz6Dqe88,2317
 langfun/core/console_test.py,sha256=5SYJdxpJGLgdSSQqqMPoA1X6jpsLD8rgcyk-EgI65oE,1077
 langfun/core/langfunc.py,sha256=266xNz8Vgal7K4HSsrYt7z7_qPYV4bWWK626IbbohrE,11573
 langfun/core/langfunc_test.py,sha256=ukv5cnad5ZBckM2PhyIFq79BPN0Db4cszMrPqh_CZkA,8163
-langfun/core/language_model.py,sha256=HG2XQhisyiyvMBws5dASH1OkbGJB3twn0p2e_As8hUo,12889
+langfun/core/language_model.py,sha256=JHIfW0GxFx1YVEM-drS_Iy4goFJt63LBosCM4CILWTY,12920
 langfun/core/language_model_test.py,sha256=gcW4OJJjB-V1b4kEF8zG91t36sVn3H0Yuj0LQxi83Ek,9122
 langfun/core/memory.py,sha256=f-asN1F7Vehgdn_fK84v73GrEUOxRtaW934keutTKjk,2416
 langfun/core/message.py,sha256=QhvV9t5qaryPcruyxxcXi3gm9QDInkSldwTtK6sVJ3c,15734
@@ -27,7 +27,7 @@ langfun/core/text_formatting.py,sha256=ytjj7opnRJ6w-pkglL2CZUyfYDXLpNf65E42LBb31
 langfun/core/text_formatting_test.py,sha256=nyKC6tn2L4hPJiqQHgxcbQsJJi4A4Nbj8FiO8iT6B80,1514
 langfun/core/coding/__init__.py,sha256=5utju_fwEsImaiftx4oXKl9FAM8p281k8-Esdh_-m1w,835
 langfun/core/coding/python/__init__.py,sha256=MJ-vubliz-ebrZH3OBRKBwMi0S9-FrhGCp8YQLR6_I4,1776
-langfun/core/coding/python/correction.py,sha256=6yO7eIuu2K4tchaAwbVcRro9mCGUzzZ_QbIGZB5P41c,6777
+langfun/core/coding/python/correction.py,sha256=uuQmZCrAl0EA9etIUmn2-FZ-ge8iNcjOAAlm-WgkYfo,6776
 langfun/core/coding/python/correction_test.py,sha256=yLqmQ9BPORsnREkrS10PnljEaLR3BoydTVeT3OGoqfU,3507
 langfun/core/coding/python/errors.py,sha256=fX3Du63uGm25YFXW9D-bV2gntTdTAX3hBFtAnRlmg14,3166
 langfun/core/coding/python/errors_test.py,sha256=_ZbWJCFIb-FkCK7K1zCuH8W3x_NFt-jNe3dfP8yqaD4,2323
@@ -40,8 +40,8 @@ langfun/core/coding/python/parsing_test.py,sha256=9vAWF484kWIm6JZq8NFiMgKUDhXV-d
 langfun/core/coding/python/permissions.py,sha256=1QWGHvzL8MM0Ok_auQ9tURqZHtdOfJaDpBzZ29GUE-c,2544
 langfun/core/coding/python/permissions_test.py,sha256=w5EDb8QxpxgJyZkojyzVWQvDfg366zn99-g__6TbPQ0,2699
 langfun/core/eval/__init__.py,sha256=iDA2OcJ3kR6ixZizXIY3N9LsjkaVrfTbSClTiSP8ekY,1291
-langfun/core/eval/base.py,sha256=wWFDDrf0jBzs9H_5XfdZSeOBGXyUtXAJJouk7cLckSM,52602
-langfun/core/eval/base_test.py,sha256=uXu6EtTagolDIcSadnVyMmlrz6ixx943jkZhquCRQPI,21275
+langfun/core/eval/base.py,sha256=tT_85jpLMCbXufKf64BMslid9FB1TNhe3AIkIpLULhA,53782
+langfun/core/eval/base_test.py,sha256=cep-f0pj2o7Gd5KNQV-IP0blJ-dzhlcvnt4tSbqXun4,21177
 langfun/core/eval/matching.py,sha256=g2yuBb4FeOlAlB10hqdWvaIg4QVQlJbiViRDcD2Y8go,9567
 langfun/core/eval/matching_test.py,sha256=IfuMF_dEmy4VzK6tIldRzD2Nqlml7SSh4u-baFNcZrw,4912
 langfun/core/eval/scoring.py,sha256=mshqbV_WM0zcp15TSR32ACMBDymlsbf6YH06PPx1Tw0,6139
@@ -74,7 +74,7 @@ langfun/core/structured/mapping.py,sha256=lGkjhmvVdhBGgJmc5KbfT2xQjC1MuU4OCcCfsA
 langfun/core/structured/mapping_test.py,sha256=07DDCGbwytQHSMm7fCi5-Ly-JNgdV4ubHZq0wthX4A4,3338
 langfun/core/structured/parsing.py,sha256=V3i8-AuBI4zdpsi_f6H-mbQ3h0HPgCle2OnDQolWJnA,10244
 langfun/core/structured/parsing_test.py,sha256=2_Uf3LYNRON1-5ysEr75xiG_cAxR3ZiixSfvUQu6mOQ,20846
-langfun/core/structured/prompting.py,sha256=OmI21qQQikoQLAmr5W8nBJ8PWCu9w0lmsCnmbQg9hMc,6632
+langfun/core/structured/prompting.py,sha256=wTnbb2gO4MZ7RdwUTY6f7cR35qq8sSM0iZcyTPjwfA8,6663
 langfun/core/structured/prompting_test.py,sha256=8lakKCidYuRlf-U1KexTOqQdKrBXUt8fb2J7epCdt84,19143
 langfun/core/structured/schema.py,sha256=5DKba0LrvXCJFRY-NVfER3p54BLOB7M3Yi2-u5IAJTw,24115
 langfun/core/structured/schema_test.py,sha256=LEtCST5Bfwoke59I6Q1mnOJLf2cFXQwKwTeAkI2hgqM,20912
@@ -87,8 +87,8 @@ langfun/core/templates/demonstration.py,sha256=vCrgYubdZM5Umqcgp8NUVGXgr4P_c-fik
 langfun/core/templates/demonstration_test.py,sha256=SafcDQ0WgI7pw05EmPI2S4v1t3ABKzup8jReCljHeK4,2162
 langfun/core/templates/selfplay.py,sha256=yhgrJbiYwq47TgzThmHrDQTF4nDrTI09CWGhuQPNv-s,2273
 langfun/core/templates/selfplay_test.py,sha256=IB5rWbjK_9CTkqEo1BclQPzFAKcIiusJckH8J19HFgI,2096
-langfun-0.0.2.dev20240207.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-langfun-0.0.2.dev20240207.dist-info/METADATA,sha256=NPHG-e-l1M32PgSNSOdNo8y3pUUJOAOnv3M_dw5nlzs,3368
-langfun-0.0.2.dev20240207.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-langfun-0.0.2.dev20240207.dist-info/top_level.txt,sha256=RhlEkHxs1qtzmmtWSwYoLVJAc1YrbPtxQ52uh8Z9VvY,8
-langfun-0.0.2.dev20240207.dist-info/RECORD,,
+langfun-0.0.2.dev20240209.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+langfun-0.0.2.dev20240209.dist-info/METADATA,sha256=7arY2SNgfb7JcbwaIKwgOwTeRHElh_KNrWKiscHLdiM,3368
+langfun-0.0.2.dev20240209.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+langfun-0.0.2.dev20240209.dist-info/top_level.txt,sha256=RhlEkHxs1qtzmmtWSwYoLVJAc1YrbPtxQ52uh8Z9VvY,8
+langfun-0.0.2.dev20240209.dist-info/RECORD,,