langfun 0.0.2.dev20240325__tar.gz → 0.0.2.dev20240329__tar.gz

{langfun-0.0.2.dev20240325 → langfun-0.0.2.dev20240329}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: langfun
-Version: 0.0.2.dev20240325
+Version: 0.0.2.dev20240329
 Summary: Langfun: Language as Functions.
 Home-page: https://github.com/google/langfun
 Author: Langfun Authors
@@ -24,7 +24,7 @@ License-File: LICENSE
 Requires-Dist: google-generativeai>=0.3.2
 Requires-Dist: jinja2>=3.1.2
 Requires-Dist: openai==0.27.2
-Requires-Dist: pyglove>=0.4.5.dev20240314
+Requires-Dist: pyglove>=0.4.5.dev20240323
 Requires-Dist: python-magic>=0.4.27
 Requires-Dist: requests>=2.31.0
 Requires-Dist: termcolor==1.1.0

{langfun-0.0.2.dev20240325 → langfun-0.0.2.dev20240329}/langfun/core/eval/base.py

@@ -27,6 +27,7 @@ import time
 from typing import Annotated, Any, Callable, Iterator, Literal, Optional, Sequence, Type, Union
 
 import langfun.core as lf
+import langfun.core.coding as lf_coding
 from langfun.core.llms.cache import in_memory
 import langfun.core.structured as lf_structured
 import pyglove as pg
@@ -41,14 +42,6 @@ class Evaluable(lf.Component):
   INDEX_HTML = 'index.html'
   SUMMARY_HTML = 'summary.html'
 
-  id: Annotated[
-      str,
-      (
-          'The ID of the evaluation, which should be unique across all '
-          'evaluations.'
-      ),
-  ]
-
   root_dir: Annotated[
       str | None,
       (
@@ -61,6 +54,18 @@ class Evaluable(lf.Component):
       int, 'Number of decimals when reporting precision.'
   ] = lf.contextual(default=1)
 
+  @property
+  @abc.abstractmethod
+  def id(self) -> str:
+    """Returns the ID of the task.
+
+    Returns:
+      Evaluation task ID. Different evaluation task should have their unique
+      task IDs, for each task will be stored in sub-directoreis identified by
+      their IDs. For suites, the ID could be an empty string as they will not
+      produce sub-directories
+    """
+
   @property
   def dir(self) -> str | None:
     """Returns the directory for saving results and details."""
@@ -578,12 +583,15 @@ class _LeafNode:
   progress_bar: int | None = None
 
 
-@pg.use_init_args(['id', 'children'])
+@pg.use_init_args(['children'])
 class Suite(Evaluable):
   """Evaluation suite."""
 
   children: Annotated[list[Evaluable], 'Child evaluation sets or suites.']
 
+  # Use empty ID as suite is just a container of child evaluations.
+  id: str = ''
+
   __kwargs__: Annotated[
       Any,
       (
@@ -841,8 +849,10 @@ class Evaluation(Evaluable):
       kwargs['evaluation'] = self
     return self.schema_fn(**kwargs)
 
-  def _formalize_schema(self, annotation) -> lf_structured.Schema:
+  def _formalize_schema(self, annotation) -> lf_structured.Schema | None:
     """Formalizes schema from annotation."""
+    if annotation in (str, None):
+      return None
     if self.method == 'complete':
       if not hasattr(annotation, '__schema__'):
         raise TypeError(
@@ -883,6 +893,14 @@ class Evaluation(Evaluable):
       completion_examples.append(ex)
     return completion_examples
 
+  @property
+  def id(self) -> str:
+    """Returns the ID of this evaluation."""
+    id_prefix = self.__class__.__name__
+    if not self.is_deterministic:
+      return id_prefix
+    return f'{id_prefix}@{self.hash}'
+
   @functools.cached_property
   def children(self) -> list['Evaluation']:
     """Returns the trials as child evaluations if this evaluation is a space."""
@@ -892,7 +910,6 @@ class Evaluation(Evaluable):
     for i, child in enumerate(pg.iter(self)):
       child.sym_setparent(self)
       child.sym_setpath(self.sym_path + f'children[{i}]')
-      child.rebind(id=f'{self.id}@{child.hash}', skip_notification=True)
       children.append(child)
     return children
 
@@ -1004,7 +1021,11 @@ class Evaluation(Evaluable):
       self._reset()
 
       def _process(example: Any):
-        return self.process(example, **(self.additional_args or {}))
+        # NOTE(daiyip): set the `input` symbol of the globals to None, so LLM
+        # generated code with calls to `input` will raise an error, thus not
+        # blocking the evaluation.
+        with lf_coding.context(input=None):
+          return self.process(example, **(self.additional_args or {}))
 
       try:
         for example, message, error in lf.concurrent_map(
@@ -1015,10 +1036,7 @@ class Evaluation(Evaluable):
             status_fn=self._status,
         ):
           if error is not None:
-            try:
-              self._failures.append((example, str(error)))
-            except Exception as e:  # pylint: disable=broad-exception-caught
-              self._failures.append((example, str(e)))
+            self._failures.append((example, str(error)))
           else:
             output = message.text if self.schema is None else message.result
             self.audit(example, output, message)
@@ -1521,9 +1539,12 @@ class Summary(pg.Object):
     pivot_field = pivot_field or self.pivot_field
     s = io.StringIO()
     s.write('<html><body>')
-    for task in self.tasks():
+    for task in sorted(self.tasks(), key=lambda cls: cls.__name__):
+      table_id = task.__name__.lower()
       s.write('<div>')
-      s.write(f'<h2>{task.__name__}</h2>')
+      s.write(f'<a id="{table_id}"')
+      s.write(f'<h2><a href="#{table_id}">{task.__name__}</a></h2>')
+      s.write('</a>')
       table = Summary.Table.from_evaluations(
           self.select(task=task).evaluations, pivot_field
       )

{langfun-0.0.2.dev20240325 → langfun-0.0.2.dev20240329}/langfun/core/eval/base_test.py

@@ -70,8 +70,7 @@ def eval_set(
   """Creates an evaluation object for testing."""
   tmp_dir = tempfile.gettempdir()
   return cls(
-      id=eval_id,
-      root_dir=tmp_dir,
+      root_dir=os.path.join(tmp_dir, eval_id),
       inputs=base.as_inputs([
           pg.Dict(question='Compute 1 + 1'),
           pg.Dict(question='Compute 1 + 2'),
@@ -210,7 +209,7 @@ class EvaluationTest(unittest.TestCase):
         s.result,
         dict(
             experiment_setup=dict(
-                id='run_test',
+                id='Evaluation@17915dc6',
                 dir=s.dir,
                 model='StaticSequence',
                 prompt_template='{{example.question}}',
@@ -302,7 +301,6 @@ class EvaluationTest(unittest.TestCase):
         '3',
     ])
     s = base.Evaluation(
-        id='search_space_test',
         root_dir=tempfile.gettempdir(),
         inputs=base.as_inputs([
             pg.Dict(question='Compute 1 + 1'),
@@ -439,7 +437,6 @@ class SuiteTest(unittest.TestCase):
         '3',
     ] * 5)
     s = base.Suite(
-        'suite_run_test',
         [
             eval_set('run_test_1', 'query', schema_fn=answer_schema()),
             # A suite of search space. Two of the sub-experiments are identical,
@@ -548,7 +545,6 @@ class SummaryTest(unittest.TestCase):
   def _eval_set(self, root_dir):
     return base.Suite(id='select_test', children=[
         TaskA(
-            id='task_a',
             inputs=base.as_inputs([
                 pg.Dict(question='Compute 1 + 1'),
             ]),
@@ -569,7 +565,6 @@ class SummaryTest(unittest.TestCase):
             max_workers=1,
         ),
         TaskB(
-            id='task_b',
             inputs=base.as_inputs([
                 pg.Dict(question='Compute 1 + 1'),
             ]),
@@ -650,10 +645,10 @@ class SummaryTest(unittest.TestCase):
         len(base.Summary.from_dirs(root_dir)), 2 * 2 * 2 * 2 + 2 * 1 * 1 * 2
     )
     self.assertEqual(
-        len(base.Summary.from_dirs(root_dir, 'task_b')), 2 * 1 * 1 * 2
+        len(base.Summary.from_dirs(root_dir, 'TaskB')), 2 * 1 * 1 * 2
     )
     self.assertEqual(
-        len(base.Summary.from_dirs(root_dir, ('task_a'))), 2 * 2 * 2 * 2
+        len(base.Summary.from_dirs(root_dir, ('TaskA'))), 2 * 2 * 2 * 2
     )
 
   def test_monitor(self):

{langfun-0.0.2.dev20240325 → langfun-0.0.2.dev20240329}/langfun/core/eval/matching_test.py

@@ -65,10 +65,8 @@ def eval_set(
     use_cache: bool = True,
 ):
   """Creates an evaluation object for testing."""
-  tmp_dir = tempfile.gettempdir()
   return MyTask(
-      id=eval_id,
-      root_dir=tmp_dir,
+      root_dir=os.path.join(tempfile.gettempdir(), eval_id),
       inputs=base.as_inputs([
           pg.Dict(question='Compute 1 + 1', groundtruth=2),
           pg.Dict(question='Compute 1 + 2', groundtruth=3),
@@ -105,7 +103,7 @@ class MatchingTest(unittest.TestCase):
         s.result,
         dict(
             experiment_setup=dict(
-                id='match_run_test',
+                id='MyTask@3d87f97f',
                 dir=s.dir,
                 model='StaticSequence',
                 prompt_template='{{example.question}}',

{langfun-0.0.2.dev20240325 → langfun-0.0.2.dev20240329}/langfun/core/eval/scoring_test.py

@@ -43,7 +43,6 @@ def constrained_by_upperbound(upper_bound: int):
 
 
 class ConstraintFollowing(scoring.Scoring):
-  id = 'constraint_following'
   inputs = constrained_by_upperbound(1)
   prompt = '{{example}}'
   method = 'query'
@@ -82,7 +81,7 @@ class ScoringTest(unittest.TestCase):
         s.result,
         dict(
             experiment_setup=dict(
-                id='constraint_following',
+                id='ConstraintFollowing@9e51bb9e',
                 dir=s.dir,
                 model='StaticSequence',
                 prompt_template='{{example}}',

{langfun-0.0.2.dev20240325 → langfun-0.0.2.dev20240329}/langfun/core/structured/schema.py

@@ -55,10 +55,6 @@ def parse_value_spec(value) -> pg.typing.ValueSpec:
           ),
       ):
         raise ValueError(f'Unsupported schema specification: {v}')
-      if isinstance(spec, pg.typing.Object) and not issubclass(
-          spec.cls, pg.Symbolic
-      ):
-        raise ValueError(f'{v} must be a symbolic class to be parsable.')
       return spec
 
   return _parse_node(value)
@@ -208,7 +204,9 @@ def class_dependencies(
   if isinstance(value_or_spec, Schema):
     return value_or_spec.class_dependencies(include_subclasses)
 
-  if isinstance(value_or_spec, (pg.typing.ValueSpec, pg.symbolic.ObjectMeta)):
+  if inspect.isclass(value_or_spec) or isinstance(
+      value_or_spec, pg.typing.ValueSpec
+  ):
     value_or_spec = (value_or_spec,)
 
   if isinstance(value_or_spec, tuple):
@@ -216,7 +214,7 @@ def class_dependencies(
     for v in value_or_spec:
       if isinstance(v, pg.typing.ValueSpec):
         value_specs.append(v)
-      elif inspect.isclass(v) and issubclass(v, pg.Object):
+      elif inspect.isclass(v):
         value_specs.append(pg.typing.Object(v))
       else:
         raise TypeError(f'Unsupported spec type: {v!r}')
@@ -235,23 +233,20 @@ def class_dependencies(
 
   def _fill_dependencies(vs: pg.typing.ValueSpec, include_subclasses: bool):
     if isinstance(vs, pg.typing.Object):
-      if issubclass(vs.cls, pg.Object) and vs.cls not in seen:
+      if vs.cls not in seen:
         seen.add(vs.cls)
 
         # Add base classes as dependencies.
         for base_cls in vs.cls.__bases__:
           # We only keep track of user-defined symbolic classes.
-          if issubclass(
-              base_cls, pg.Object
-          ) and not base_cls.__module__.startswith('pyglove'):
+          if base_cls is not object and base_cls is not pg.Object:
             _fill_dependencies(
                 pg.typing.Object(base_cls), include_subclasses=False
             )
 
         # Add members as dependencies.
-        if hasattr(vs.cls, '__schema__'):
-          for field in vs.cls.__schema__.values():
-            _fill_dependencies(field.value, include_subclasses)
+        for field in _pg_schema(vs.cls).values():
+          _fill_dependencies(field.value, include_subclasses)
       _add_dependency(vs.cls)
 
       # Check subclasses if available.
@@ -364,17 +359,13 @@ def class_definition(
 ) -> str:
   """Returns the Python class definition."""
   out = io.StringIO()
-  if not issubclass(cls, pg.Object):
-    raise TypeError(
-        'Classes must be `pg.Object` subclasses to be used as schema. '
-        f'Encountered: {cls}.'
-    )
-  schema = cls.__schema__
+  schema = _pg_schema(cls)
   eligible_bases = []
   for base_cls in cls.__bases__:
-    if issubclass(base_cls, pg.Object):
+    if base_cls is not object:
       if include_pg_object_as_base or base_cls is not pg.Object:
         eligible_bases.append(base_cls.__name__)
+
   if eligible_bases:
     base_cls_str = ', '.join(eligible_bases)
     out.write(f'class {cls.__name__}({base_cls_str}):\n')
@@ -839,3 +830,13 @@ class Unknown(pg.Object, pg.typing.CustomTyping):
 
 
 UNKNOWN = Unknown()
+
+
+def _pg_schema(cls: Type[Any]) -> pg.Schema:
+  """Returns PyGlove schema for the constructor of a class."""
+  schema = getattr(cls, '__schema__', None)
+  if schema is None:
+    schema = pg.symbolic.callable_schema(
+        cls.__init__, auto_typing=True, auto_doc=True, remove_self=True
+    )
+  return schema

{langfun-0.0.2.dev20240325 → langfun-0.0.2.dev20240329}/langfun/core/structured/schema_generation.py

@@ -143,14 +143,14 @@ def generate_class(
 
 
 def classgen_example(
-    class_name: str, prompt: str | pg.Symbolic, cls: Type[Any]
+    prompt: str | pg.Symbolic, cls: Type[Any]
 ) -> mapping.MappingExample:
   """Creates a class generation example."""
   if isinstance(prompt, lf.Template):
     prompt = prompt.render()
   return mapping.MappingExample(
       input=prompt,
-      context=class_name,
+      context=cls.__name__,
       output=cls,
   )
 
@@ -168,7 +168,6 @@ def default_classgen_examples() -> list[mapping.MappingExample]:
 
   return [
       classgen_example(
-          'Solution',
           'How to evaluate an arithmetic expression?',
           Solution,
       )

{langfun-0.0.2.dev20240325 → langfun-0.0.2.dev20240329}/langfun/core/structured/schema_test.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 """Tests for structured parsing."""
 
+import dataclasses
 import inspect
 import typing
 import unittest
@@ -101,12 +102,7 @@ class SchemaTest(unittest.TestCase):
 
     self.assert_unsupported_annotation(typing.Type[int])
     self.assert_unsupported_annotation(typing.Union[int, str, bool])
-
-    class X:
-      pass
-
-    # X must be a symbolic type to be parsable.
-    self.assert_unsupported_annotation(X)
+    self.assert_unsupported_annotation(typing.Any)
 
   def test_schema_dict(self):
     schema = schema_lib.Schema([{'x': Itinerary}])
@@ -150,6 +146,25 @@ class SchemaTest(unittest.TestCase):
     schema = schema_lib.Schema([B])
     self.assertEqual(schema.class_dependencies(), [Foo, A, Bar, X, B])
 
+  def test_class_dependencies_non_pyglove(self):
+    class Baz:
+      def __init__(self, x: int):
+        pass
+
+    @dataclasses.dataclass(frozen=True)
+    class AA:
+      foo: tuple[Baz, int]
+
+    class XX(pg.Object):
+      pass
+
+    @dataclasses.dataclass(frozen=True)
+    class BB(AA):
+      foo2: Baz | XX
+
+    schema = schema_lib.Schema([AA])
+    self.assertEqual(schema.class_dependencies(), [Baz, AA, XX, BB])
+
   def test_schema_repr(self):
     schema = schema_lib.Schema([{'x': Itinerary}])
     self.assertEqual(
@@ -440,13 +455,6 @@ class SchemaPythonReprTest(unittest.TestCase):
         'class A(Object):\n  pass\n',
     )
 
-    class B:
-      pass
-
-    with self.assertRaisesRegex(
-        TypeError, 'Classes must be `pg.Object` subclasses.*'):
-      schema_lib.class_definition(B)
-
     class C(pg.Object):
       x: str
       __kwargs__: typing.Any
@@ -459,9 +467,12 @@ class SchemaPythonReprTest(unittest.TestCase):
     class Foo(pg.Object):
       x: int
 
-    class Bar(pg.Object):
+    @dataclasses.dataclass(frozen=True)
+    class Bar:
+      """Class Bar."""
       y: str
 
+    @dataclasses.dataclass(frozen=True)
     class Baz(Bar):  # pylint: disable=unused-variable
       pass
 
@@ -475,7 +486,7 @@ class SchemaPythonReprTest(unittest.TestCase):
     schema = schema_lib.Schema([B])
     self.assertEqual(
         schema_lib.SchemaPythonRepr().class_definitions(schema),
-        inspect.cleandoc("""
+        inspect.cleandoc('''
             class Foo:
               x: int
 
@@ -483,16 +494,18 @@ class SchemaPythonReprTest(unittest.TestCase):
               foo: Foo
 
             class Bar:
+              """Class Bar."""
               y: str
 
             class Baz(Bar):
+              """Baz(y: str)"""
               y: str
 
             class B(A):
               foo: Foo
               bar: Bar
               foo2: Foo
-            """) + '\n',
+            ''') + '\n',
     )
 
     self.assertEqual(
@@ -501,7 +514,7 @@ class SchemaPythonReprTest(unittest.TestCase):
 
     self.assertEqual(
         schema_lib.SchemaPythonRepr().repr(schema),
-        inspect.cleandoc("""
+        inspect.cleandoc('''
             list[B]
 
             ```python
@@ -512,9 +525,11 @@ class SchemaPythonReprTest(unittest.TestCase):
               foo: Foo
 
             class Bar:
+              """Class Bar."""
               y: str
 
             class Baz(Bar):
+              """Baz(y: str)"""
               y: str
 
             class B(A):
@@ -522,7 +537,7 @@ class SchemaPythonReprTest(unittest.TestCase):
               bar: Bar
               foo2: Foo
             ```
-            """),
+            '''),
     )
     self.assertEqual(
         schema_lib.SchemaPythonRepr().repr(
@@ -531,24 +546,26 @@ class SchemaPythonReprTest(unittest.TestCase):
             include_pg_object_as_base=True,
             markdown=False,
         ),
-        inspect.cleandoc("""
+        inspect.cleandoc('''
             class Foo(Object):
               x: int
 
             class A(Object):
               foo: Foo
 
-            class Bar(Object):
+            class Bar:
+              """Class Bar."""
               y: str
 
             class Baz(Bar):
+              """Baz(y: str)"""
               y: str
 
             class B(A):
               foo: Foo
               bar: Bar
               foo2: Foo
-            """),
+            '''),
     )
 
 

{langfun-0.0.2.dev20240325 → langfun-0.0.2.dev20240329}/langfun.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: langfun
-Version: 0.0.2.dev20240325
+Version: 0.0.2.dev20240329
 Summary: Langfun: Language as Functions.
 Home-page: https://github.com/google/langfun
 Author: Langfun Authors
@@ -24,7 +24,7 @@ License-File: LICENSE
 Requires-Dist: google-generativeai>=0.3.2
 Requires-Dist: jinja2>=3.1.2
 Requires-Dist: openai==0.27.2
-Requires-Dist: pyglove>=0.4.5.dev20240314
+Requires-Dist: pyglove>=0.4.5.dev20240323
 Requires-Dist: python-magic>=0.4.27
 Requires-Dist: requests>=2.31.0
 Requires-Dist: termcolor==1.1.0

{langfun-0.0.2.dev20240325 → langfun-0.0.2.dev20240329}/langfun.egg-info/requires.txt

@@ -1,7 +1,7 @@
 google-generativeai>=0.3.2
 jinja2>=3.1.2
 openai==0.27.2
-pyglove>=0.4.5.dev20240314
+pyglove>=0.4.5.dev20240323
 python-magic>=0.4.27
 requests>=2.31.0
 termcolor==1.1.0