langfun 0.1.2.dev202412070804__py3-none-any.whl → 0.1.2.dev202412110804__py3-none-any.whl

langfun/core/agentic/action_test.py

@@ -18,6 +18,8 @@ import unittest
 import langfun.core as lf
 from langfun.core.agentic import action as action_lib
 from langfun.core.llms import fake
+import langfun.core.structured as lf_structured
+import pyglove as pg
 
 
 class SessionTest(unittest.TestCase):
@@ -28,60 +30,94 @@ class SessionTest(unittest.TestCase):
     class Bar(action_lib.Action):
 
       def call(self, session, *, lm, **kwargs):
-        test.assertIs(session.current_invocation.action, self)
+        test.assertIs(session.current_action.action, self)
         session.info('Begin Bar')
         session.query('bar', lm=lm)
+        session.add_metadata(note='bar')
         return 2
 
     class Foo(action_lib.Action):
       x: int
 
       def call(self, session, *, lm, **kwargs):
-        test.assertIs(session.current_invocation.action, self)
-        session.info('Begin Foo', x=1)
-        session.query('foo', lm=lm)
+        test.assertIs(session.current_action.action, self)
+        with session.phase('prepare'):
+          session.info('Begin Foo', x=1)
+          session.query('foo', lm=lm)
+        with session.track_queries():
+          self.make_additional_query(lm)
+        session.add_metadata(note='foo')
         return self.x + Bar()(session, lm=lm)
 
+      def make_additional_query(self, lm):
+        lf_structured.query('additional query', lm=lm)
+
     lm = fake.StaticResponse('lm response')
-    session = action_lib.Session()
-    root = session.root_invocation
-    self.assertIsInstance(root.action, action_lib.RootAction)
-    self.assertIs(session.current_invocation, session.root_invocation)
-    self.assertEqual(Foo(1)(session, lm=lm), 3)
-    self.assertEqual(len(session.root_invocation.child_invocations), 1)
-    self.assertEqual(len(list(session.root_invocation.queries())), 0)
-    self.assertEqual(
-        len(list(session.root_invocation.queries(include_children=True))), 2
-    )
-    self.assertEqual(
-        len(list(session.root_invocation.child_invocations[0].queries())), 1
-    )
-    self.assertEqual(len(session.root_invocation.child_invocations[0].logs), 1)
-    self.assertEqual(
-        len(session.root_invocation.child_invocations[0].child_invocations),
-        1
-    )
-    self.assertEqual(
-        len(session.root_invocation
-            .child_invocations[0].child_invocations[0].logs),
-        1
-    )
-    self.assertEqual(
-        len(list(session.root_invocation
-                 .child_invocations[0].child_invocations[0].queries())),
-        1
-    )
-    self.assertEqual(
-        len(session.root_invocation
-            .child_invocations[0].child_invocations[0].child_invocations),
-        0
-    )
-    self.assertIs(session.current_invocation, session.root_invocation)
-    self.assertIs(session.final_result, 3)
-    self.assertIn(
-        'invocation-final-result',
-        session.to_html().content,
+    foo = Foo(1)
+    self.assertEqual(foo(lm=lm), 3)
+
+    session = foo.session
+    self.assertIsNotNone(session)
+    self.assertIsInstance(session.root.action, action_lib.RootAction)
+    self.assertIs(session.current_action, session.root)
+
+    #
+    # Inspecting the root invocation.
+    #
+
+    root = session.root
+    self.assertEqual(len(root.execution.items), 1)
+    self.assertIs(root.execution.items[0].action, foo)
+
+    self.assertTrue(root.execution.has_started)
+    self.assertTrue(root.execution.has_stopped)
+    self.assertGreater(root.execution.elapse, 0)
+    self.assertEqual(root.result, 3)
+    self.assertEqual(root.result_metadata, dict(note='foo'))
+
+    # The root space should have one action (foo), no queries, and no logs.
+    self.assertEqual(len(list(root.actions)), 1)
+    self.assertEqual(len(list(root.queries)), 0)
+    self.assertEqual(len(list(root.logs)), 0)
+    # 1 query from Bar and 2 from Foo.
+    self.assertEqual(len(list(root.all_queries)), 3)
+    # 1 log from Bar and 1 from Foo.
+    self.assertEqual(len(list(root.all_logs)), 2)
+    self.assertEqual(root.usage_summary.total.num_requests, 3)
+
+    # Inspecting the top-level action (Foo)
+    foo_invocation = root.execution.items[0]
+    self.assertEqual(len(foo_invocation.execution.items), 3)
+
+    # Prepare phase.
+    prepare_phase = foo_invocation.execution.items[0]
+    self.assertIsInstance(
+        prepare_phase, action_lib.ExecutionTrace
     )
+    self.assertEqual(len(prepare_phase.items), 2)
+    self.assertTrue(prepare_phase.has_started)
+    self.assertTrue(prepare_phase.has_stopped)
+    self.assertEqual(prepare_phase.usage_summary.total.num_requests, 1)
+
+    # Tracked queries.
+    query_invocation = foo_invocation.execution.items[1]
+    self.assertIsInstance(query_invocation, lf_structured.QueryInvocation)
+    self.assertIs(query_invocation.lm, lm)
+
+    # Invocation to Bar.
+    bar_invocation = foo_invocation.execution.items[2]
+    self.assertIsInstance(bar_invocation, action_lib.ActionInvocation)
+    self.assertIsInstance(bar_invocation.action, Bar)
+    self.assertEqual(bar_invocation.result, 2)
+    self.assertEqual(bar_invocation.result_metadata, dict(note='bar'))
+    self.assertEqual(len(bar_invocation.execution.items), 2)
+
+    # Save to HTML
+    self.assertIn('invocation-result', session.to_html().content)
+
+    # Save session to JSON
+    json_str = session.to_json_str(save_ref_value=True)
+    self.assertIsInstance(pg.from_json_str(json_str), action_lib.Session)
 
   def test_log(self):
     session = action_lib.Session()

langfun/core/eval/v2/evaluation.py

@@ -556,6 +556,9 @@ class Evaluation(experiment_lib.Experiment):
           border: 0px;
           margin: 0px;
         }
+        .eval-details .tab-control {
+          width: 100%;
+        }
         .eval-details .tab-button {
           font-size: large;
         }

langfun/core/llms/__init__.py

@@ -33,6 +33,7 @@ from langfun.core.llms.rest import REST
 # Gemini models.
 from langfun.core.llms.google_genai import GenAI
 from langfun.core.llms.google_genai import GeminiExp_20241114
+from langfun.core.llms.google_genai import GeminiExp_20241206
 from langfun.core.llms.google_genai import GeminiFlash1_5
 from langfun.core.llms.google_genai import GeminiPro
 from langfun.core.llms.google_genai import GeminiPro1_5

langfun/core/llms/anthropic.py

@@ -35,7 +35,7 @@ except ImportError:
   google_auth = None
   auth_requests = None
   credentials_lib = None
-  Credentials = None  # pylint: disable=invalid-name
+  Credentials = Any  # pylint: disable=invalid-name
 
 
 SUPPORTED_MODELS_AND_SETTINGS = {

langfun/core/llms/google_genai.py

@@ -20,6 +20,7 @@ from typing import Annotated, Any, Literal
 
 import langfun.core as lf
 from langfun.core import modalities as lf_modalities
+from langfun.core.llms import vertexai
 import pyglove as pg
 
 
@@ -54,6 +55,7 @@ class GenAI(lf.LanguageModel):
           'gemini-1.5-pro-latest',
           'gemini-1.5-flash-latest',
           'gemini-exp-1114',
+          'gemini-exp-1206',
       ],
       'Model name.',
   ]
@@ -306,64 +308,52 @@ _GOOGLE_GENAI_MODEL_HUB = _ModelHub()
 #
 
 
-_IMAGE_TYPES = [
-    'image/png',
-    'image/jpeg',
-    'image/webp',
-    'image/heic',
-    'image/heif',
-]
-
-_AUDIO_TYPES = [
-    'audio/aac',
-    'audio/flac',
-    'audio/mp3',
-    'audio/m4a',
-    'audio/mpeg',
-    'audio/mpga',
-    'audio/mp4',
-    'audio/opus',
-    'audio/pcm',
-    'audio/wav',
-    'audio/webm'
-]
-
-_VIDEO_TYPES = [
-    'video/mov',
-    'video/mpeg',
-    'video/mpegps',
-    'video/mpg',
-    'video/mp4',
-    'video/webm',
-    'video/wmv',
-    'video/x-flv',
-    'video/3gpp',
-]
-
-_PDF = [
-    'application/pdf',
-]
+class GeminiExp_20241206(GenAI):  # pylint: disable=invalid-name
+  """Gemini Experimental model launched on 12/06/2024."""
+
+  model = 'gemini-exp-1206'
+  supported_modalities = (
+      vertexai.DOCUMENT_TYPES
+      + vertexai.IMAGE_TYPES
+      + vertexai.AUDIO_TYPES
+      + vertexai.VIDEO_TYPES
+  )
 
 
 class GeminiExp_20241114(GenAI):  # pylint: disable=invalid-name
   """Gemini Experimental model launched on 11/14/2024."""
 
   model = 'gemini-exp-1114'
-  supported_modalities = _PDF + _IMAGE_TYPES + _AUDIO_TYPES + _VIDEO_TYPES
+  supported_modalities = (
+      vertexai.DOCUMENT_TYPES
+      + vertexai.IMAGE_TYPES
+      + vertexai.AUDIO_TYPES
+      + vertexai.VIDEO_TYPES
+  )
 
 
 class GeminiPro1_5(GenAI):  # pylint: disable=invalid-name
   """Gemini Pro latest model."""
 
   model = 'gemini-1.5-pro-latest'
-  supported_modalities = _PDF + _IMAGE_TYPES + _AUDIO_TYPES + _VIDEO_TYPES
+  supported_modalities = (
+      vertexai.DOCUMENT_TYPES
+      + vertexai.IMAGE_TYPES
+      + vertexai.AUDIO_TYPES
+      + vertexai.VIDEO_TYPES
+  )
 
 
 class GeminiFlash1_5(GenAI):  # pylint: disable=invalid-name
   """Gemini Flash latest model."""
 
   model = 'gemini-1.5-flash-latest'
-  supported_modalities = _PDF + _IMAGE_TYPES + _AUDIO_TYPES + _VIDEO_TYPES
+  supported_modalities = (
+      vertexai.DOCUMENT_TYPES
+      + vertexai.IMAGE_TYPES
+      + vertexai.AUDIO_TYPES
+      + vertexai.VIDEO_TYPES
+  )
 
 
 class GeminiPro(GenAI):
@@ -376,7 +366,7 @@ class GeminiProVision(GenAI):
   """Gemini Pro vision model."""
 
   model = 'gemini-pro-vision'
-  supported_modalities = _IMAGE_TYPES + _VIDEO_TYPES
+  supported_modalities = vertexai.IMAGE_TYPES + vertexai.VIDEO_TYPES
 
 
 class Palm2(GenAI):

langfun/core/llms/vertexai.py

@@ -343,7 +343,7 @@ class VertexAI(rest.REST):
     return lf.AIMessage.from_chunks(chunks)
 
 
-_IMAGE_TYPES = [
+IMAGE_TYPES = [
     'image/png',
     'image/jpeg',
     'image/webp',
@@ -351,7 +351,7 @@ _IMAGE_TYPES = [
     'image/heif',
 ]
 
-_AUDIO_TYPES = [
+AUDIO_TYPES = [
     'audio/aac',
     'audio/flac',
     'audio/mp3',
@@ -362,10 +362,10 @@ _AUDIO_TYPES = [
     'audio/opus',
     'audio/pcm',
     'audio/wav',
-    'audio/webm'
+    'audio/webm',
 ]
 
-_VIDEO_TYPES = [
+VIDEO_TYPES = [
     'video/mov',
     'video/mpeg',
     'video/mpegps',
@@ -375,9 +375,10 @@ _VIDEO_TYPES = [
     'video/wmv',
     'video/x-flv',
     'video/3gpp',
+    'video/quicktime',
 ]
 
-_DOCUMENT_TYPES = [
+DOCUMENT_TYPES = [
     'application/pdf',
     'text/plain',
     'text/csv',
@@ -391,8 +392,8 @@ _DOCUMENT_TYPES = [
 class VertexAIGemini1_5(VertexAI):  # pylint: disable=invalid-name
   """Vertex AI Gemini 1.5 model."""
 
-  supported_modalities: pg.typing.List(str).freeze(   # pytype: disable=invalid-annotation
-      _DOCUMENT_TYPES + _IMAGE_TYPES + _AUDIO_TYPES + _VIDEO_TYPES
+  supported_modalities: pg.typing.List(str).freeze(  # pytype: disable=invalid-annotation
+      DOCUMENT_TYPES + IMAGE_TYPES + AUDIO_TYPES + VIDEO_TYPES
   )
 
 
@@ -460,8 +461,8 @@ class VertexAIGeminiPro1Vision(VertexAI):  # pylint: disable=invalid-name
   """Vertex AI Gemini 1.0 Pro Vision model."""
 
   model = 'gemini-1.0-pro-vision'
-  supported_modalities: pg.typing.List(str).freeze(   # pytype: disable=invalid-annotation
-      _IMAGE_TYPES + _VIDEO_TYPES
+  supported_modalities: pg.typing.List(str).freeze(  # pytype: disable=invalid-annotation
+      IMAGE_TYPES + VIDEO_TYPES
   )
 
 

langfun/core/message.py

@@ -769,7 +769,6 @@ class Message(
             padding: 20px;
             margin: 10px 5px 10px 5px;
             font-style: italic;
-            font-size: 1.1em;
             white-space: pre-wrap;
             border: 1px solid #EEE;
             border-radius: 5px;
@@ -778,7 +777,7 @@ class Message(
         .modality-in-text {
             display: inline-block;
         }
-        .modality-in-text > details {
+        .modality-in-text > details.pyglove {
             display: inline-block;
             font-size: 0.8em;
             border: 0;

langfun/core/message_test.py

@@ -380,7 +380,6 @@ class MessageTest(unittest.TestCase):
                 padding: 20px;
                 margin: 10px 5px 10px 5px;
                 font-style: italic;
-                font-size: 1.1em;
                 white-space: pre-wrap;
                 border: 1px solid #EEE;
                 border-radius: 5px;
@@ -389,7 +388,7 @@ class MessageTest(unittest.TestCase):
             .modality-in-text {
                 display: inline-block;
             }
-            .modality-in-text > details {
+            .modality-in-text > details.pyglove {
                 display: inline-block;
                 font-size: 0.8em;
                 border: 0;

langfun/core/structured/function_generation.py

@@ -76,6 +76,7 @@ def unittest_with_test_cases(f, unittests):
 
 def _function_gen(
     func: Callable[..., Any],
+    context: dict[str, Any],
     signature: str,
     lm: language_model.LanguageModel,
     num_retries: int = 1,
@@ -141,21 +142,23 @@ def _function_gen(
   elif isinstance(unittest, list):
     unittest_examples = unittest
 
+  last_error = None
   for _ in range(num_retries):
     try:
       source_code = prompting.query(
           PythonFunctionPrompt(signature=signature), lm=lm
       )
-      f = python.evaluate(source_code)
+      f = python.evaluate(source_code, global_vars=context)
 
       # Check whether the sigantures are the same.
       if inspect.signature(f) != inspect.signature(func):
-        pg.logging.warning(
-            "Signature mismatch. Expected: %s, Actual: %s",
-            inspect.signature(func),
-            inspect.signature(f),
+        raise python.CodeError(
+            code=source_code,
+            cause=TypeError(
+                f"Signature mismatch: Expected: {inspect.signature(func)}, "
+                f"Actual: {inspect.signature(f)}.",
+            ),
         )
-        continue
 
       if callable(unittest):
         unittest(f)
@@ -163,10 +166,12 @@ def _function_gen(
         unittest_with_test_cases(f, unittest_examples)
 
       return f, source_code
-    except Exception:  # pylint: disable=broad-exception-caught
-      pass
-
-  return None, None
+    except python.CodeError as e:
+      last_error = e
+      pg.logging.warning(
+          f"Bad code generated: {e}",
+      )
+  raise last_error
 
 
 def _process_signature(signature):
@@ -220,6 +225,13 @@ def function_gen(
     setattr(func, "__function__", None)
     setattr(func, "__source_code__", None)
 
+    # Prepare the globals/locals for the generated code to be evaluated against.
+    callstack = inspect.stack()
+    assert len(callstack) > 1
+    context = dict(callstack[1][0].f_globals)
+    context.update(callstack[1][0].f_locals)
+    context.pop(func.__name__, None)
+
     @functools.wraps(func)
     def lm_generated_func(*args, **kwargs):
       if func.__function__ is not None:
@@ -238,20 +250,20 @@ def function_gen(
 
         if signature in cache:
           func.__source_code__ = cache[signature]
-          func.__function__ = python.evaluate(func.__source_code__)
+          func.__function__ = python.evaluate(
+              func.__source_code__, global_vars=context
+          )
           return func.__function__(*args, **kwargs)
 
       func.__function__, func.__source_code__ = _function_gen(
           func,
+          context,
           signature,
           lm,
           num_retries=num_retries,
           unittest=unittest,
           unittest_num_retries=unittest_num_retries,
       )
-      if func.__function__ is None:
-        raise ValueError(f"Function generation failed. Signature:\n{signature}")
-
       if cache_filename is not None:
         cache[signature] = func.__source_code__
         cache.save(cache_filename)

langfun/core/structured/function_generation_test.py

@@ -311,6 +311,36 @@ class FunctionGenerationTest(unittest.TestCase):
 
     self.assertEqual(linear_search(['a', 'b', 'c'], 'c'), 2)
 
+  def test_context_passthrough(self):
+
+    class Number(pg.Object):
+      value: int
+
+    function_gen_lm_response = inspect.cleandoc("""
+        ```python
+        def add(a: Number, b: Number) -> Number:
+            \"\"\"Adds two numbers together.\"\"\"
+            return Number(a.value + b.value)
+        ```
+        """)
+
+    lm = fake.StaticSequence(
+        [function_gen_lm_response]
+    )
+
+    def _unittest_fn(func):
+      assert func(Number(1), Number(2)) == Number(3)
+
+    custom_unittest = _unittest_fn
+
+    @function_generation.function_gen(
+        lm=lm, unittest=custom_unittest, num_retries=1
+    )
+    def add(a: Number, b: Number) -> Number:  # pylint: disable=unused-argument
+      """Adds two numbers together."""
+
+    self.assertEqual(add(Number(2), Number(3)), Number(5))
+
   def test_siganture_check(self):
     incorrect_signature_lm_response = inspect.cleandoc("""
         ```python

langfun/core/structured/prompting.py

@@ -264,9 +264,9 @@ def query(
               schema_lib.Schema.from_value(schema)
               if schema not in (None, str) else None
           ),
-          output=pg.Ref(_result(output_message)),
           lm=pg.Ref(lm),
           examples=pg.Ref(examples) if examples else [],
+          lm_response=lf.AIMessage(output_message.text),
           usage_summary=usage_summary,
       )
       for i, (tracker, include_child_scopes) in enumerate(trackers):
@@ -357,7 +357,7 @@ def _reward_fn(cls) -> Callable[
   return _reward
 
 
-class QueryInvocation(pg.Object):
+class QueryInvocation(pg.Object, pg.views.HtmlTreeView.Extension):
   """A class to represent the invocation of `lf.query`."""
 
   input: Annotated[
@@ -368,9 +368,9 @@ class QueryInvocation(pg.Object):
       schema_lib.schema_spec(noneable=True),
       'Schema of `lf.query`.'
   ]
-  output: Annotated[
-      Any,
-      'Mapping output of `lf.query`.'
+  lm_response: Annotated[
+      lf.Message,
+      'Raw LM response.'
   ]
   lm: Annotated[
       lf.LanguageModel,
@@ -385,6 +385,106 @@ class QueryInvocation(pg.Object):
       'Usage summary for `lf.query`.'
   ]
 
+  @functools.cached_property
+  def lm_request(self) -> lf.Message:
+    return query_prompt(self.input, self.schema)
+
+  @functools.cached_property
+  def output(self) -> Any:
+    return query_output(self.lm_response, self.schema)
+
+  def _on_bound(self):
+    super()._on_bound()
+    self.__dict__.pop('lm_request', None)
+    self.__dict__.pop('output', None)
+
+  def _html_tree_view_summary(
+      self,
+      *,
+      view: pg.views.HtmlTreeView,
+      **kwargs: Any
+  ) -> pg.Html | None:
+    return view.summary(
+        value=self,
+        title=pg.Html.element(
+            'div',
+            [
+                pg.views.html.controls.Label(
+                    'lf.query',
+                    css_classes=['query-invocation-type-name']
+                ),
+                pg.views.html.controls.Badge(
+                    f'lm={self.lm.model_id}',
+                    pg.format(
+                        self.lm,
+                        verbose=False,
+                        python_format=True,
+                        hide_default_values=True
+                    ),
+                    css_classes=['query-invocation-lm']
+                ),
+                self.usage_summary.to_html(extra_flags=dict(as_badge=True))
+            ],
+            css_classes=['query-invocation-title']
+        ),
+        enable_summary_tooltip=False
+    )
+
+  def _html_tree_view_content(
+      self,
+      *,
+      view: pg.views.HtmlTreeView,
+      **kwargs: Any
+  ) -> pg.Html:
+    return pg.views.html.controls.TabControl([
+        pg.views.html.controls.Tab(
+            'input',
+            pg.view(self.input, collapse_level=None),
+        ),
+        pg.views.html.controls.Tab(
+            'schema',
+            pg.view(self.schema),
+        ),
+        pg.views.html.controls.Tab(
+            'output',
+            pg.view(self.output, collapse_level=None),
+        ),
+        pg.views.html.controls.Tab(
+            'lm_request',
+            pg.view(
+                self.lm_request,
+                extra_flags=dict(include_message_metadata=False),
+            ),
+        ),
+        pg.views.html.controls.Tab(
+            'lm_response',
+            pg.view(
+                self.lm_response,
+                extra_flags=dict(include_message_metadata=False)
+            ),
+        ),
+    ], tab_position='top').to_html()
+
+  @classmethod
+  def _html_tree_view_css_styles(cls) -> list[str]:
+    return super()._html_tree_view_css_styles() + [
+        """
+        .query-invocation-title {
+            display: inline-block;
+            font-weight: normal;
+        }
+        .query-invocation-type-name {
+            font-style: italic;
+            color: #888;
+        }
+        .query-invocation-lm.badge {
+            margin-left: 5px;
+            margin-right: 5px;
+            background-color: #fff0d6;
+        }
+        """
+    ]
+
 
 @contextlib.contextmanager
 def track_queries(

langfun/core/structured/prompting_test.py

@@ -962,6 +962,18 @@ class QueryStructureJsonTest(unittest.TestCase):
     )
 
 
+class QueryInvocationTest(unittest.TestCase):
+
+  def test_to_html(self):
+    lm = fake.StaticSequence([
+        'Activity(description="hi")',
+    ])
+    with prompting.track_queries() as queries:
+      prompting.query('foo', Activity, lm=lm)
+
+    self.assertIn('schema', queries[0].to_html_str())
+
+
 class TrackQueriesTest(unittest.TestCase):
 
   def test_include_child_scopes(self):