langfun 0.1.1.dev20240729__py3-none-any.whl → 0.1.1.dev20240801__py3-none-any.whl

langfun/core/eval/base.py

@@ -542,6 +542,8 @@ class Evaluable(lf.Component):
           'padding: 10px; border: 1px solid; margin-top: 10px">'
       )
       s.write(html.escape(m.get('formatted_text', m.text)))
+
+      # Write output.
       if m.result is not None:
         s.write(
             '<div style="color: magenta; white-space: pre-wrap;'
@@ -549,6 +551,23 @@ class Evaluable(lf.Component):
         )
         s.write(html.escape(pg.format(m.result)))
         s.write('</div>')
+
+      # Write modality information.
+      if 'lm-input' in m.tags or 'lm-response' in m.tags:
+        modalities = m.referred_modalities()
+        if modalities:
+          s.write(f'<div style="color: {text_color}; white-space: pre-wrap;'
+                  'padding: 10px; border: 1px solid; margin-top: 10px"><table>')
+          for name, modality in modalities.items():
+            s.write(f'<tr><td>{name}</td><td>')
+            if hasattr(modality, '_repr_html_'):
+              s.write(modality._repr_html_())   # pylint: disable=protected-access
+            else:
+              s.write(html.escape(pg.format(modality, max_bytes_len=32)))
+            s.write('</td></tr>')
+          s.write('</table></div>')
+
+      # Write usage information.
       if m.metadata.get('usage', None):
         s.write(
             '<div style="background-color: #EEEEEE; color: black; '
@@ -1304,20 +1323,22 @@ class Evaluation(Evaluable):
     s = io.StringIO()
     definition = _html_repr(self, compact=False, escape=True)
     s.write('<div><table><tr><td>')
+    self._render_link(
+        s,
+        definition,
+        self.hash,
+        '',
+        lambda: self.link(self.dir),
+    )
     if self.result is None:
       s.write(
-          f'<a target="_blank" title="{definition}" '
-          f'href="{self.link(self.dir)}">{self.hash}</a>'
           '</td></tr><tr><td>'
           '<span style="color: gray">(IN-PROGRESS...)</span>'
       )
     else:
-      s.write(
-          f'<a target="_blank" title="{definition}" '
-          f'href="{self.index_link}">{self.hash}</a>'
-          f' &nbsp;[<a href="{self.link(self.dir)}">dir</a>]'
-          '</td></tr><tr><td>'
-      )
+      if self.dir:
+        s.write(f' &nbsp;[<a href="{self.link(self.dir)}">dir</a>]')
+      s.write('</td></tr><tr><td>')
       self._render_summary_metrics(s)
 
       # Summarize average usage.
@@ -1341,6 +1362,20 @@ class Evaluation(Evaluable):
         f'" style="color:gray">({total} tokens)</a>'
     )
 
+  def _render_link(self,
+                   s: io.StringIO,
+                   title: str,
+                   text: str,
+                   style: str,
+                   url_fn: Callable[[], str]) -> None:
+    """Renders a link in HTML."""
+    s.write(
+        f'<a target="_blank" title="{title}" style="{style}"'
+    )
+    if self.dir:
+      s.write(f' href="{url_fn()}"')
+    s.write(f'>{text}</a>')
+
   def _render_summary_metrics(self, s: io.StringIO) -> None:
     """Renders metrics in HTML."""
     assert self.result is not None
@@ -1362,14 +1397,12 @@ class Evaluation(Evaluable):
     extra_style = ''
     if m.oop_failure_rate > 0.1 and m.oop_failures > 3:
       extra_style = ';font-weight:bold'
-    s.write(
-        '<a title="%s" href="%s" style="color:magenta%s">%s</a>'
-        % (
-            oop_failure_title,
-            self.oop_failures_link,
-            extra_style,
-            self._format_rate(m.oop_failure_rate),
-        )
+    self._render_link(
+        s,
+        oop_failure_title,
+        self._format_rate(m.oop_failure_rate),
+        f'color:magenta{extra_style}',
+        lambda: self.oop_failures_link,
     )
     s.write(' | ')
 
@@ -1387,14 +1420,12 @@ class Evaluation(Evaluable):
           )
 
     extra_style = ';font-weight:bold' if m.non_oop_failures > 0 else ''
-    s.write(
-        '<a title="%s" href="%s" style="color:red%s">%s</a>'
-        % (
-            non_oop_failure_title,
-            self.non_oop_failures_link,
-            extra_style,
-            self._format_rate(m.non_oop_failure_rate),
-        )
+    self._render_link(
+        s,
+        non_oop_failure_title,
+        self._format_rate(m.non_oop_failure_rate),
+        f'color:red{extra_style}',
+        lambda: self.non_oop_failures_link,
     )
 
   def _format_rate(self, rate: float) -> str:

langfun/core/eval/matching.py

@@ -239,24 +239,20 @@ class Matching(base.Evaluation):
     """Renders metrics in HTML."""
     assert self.result is not None
     m = self.result.metrics
-    s.write(
-        '<a title="Matches (%d/%d)" href="%s" style="color:green">%s</a>'
-        % (
-            m.num_matches,
-            m.total,
-            self.matches_link,
-            self._format_rate(m.match_rate),
-        )
+    self._render_link(
+        s,
+        'Matches (%d/%d)' % (m.num_matches, m.total),
+        self._format_rate(m.match_rate),
+        'color:green',
+        lambda: self.matches_link,
     )
     s.write(' | ')
-    s.write(
-        '<a title="Mismatches (%d/%d)" href="%s" style="color:orange">%s</a>'
-        % (
-            m.num_mismatches,
-            m.total,
-            self.mismatches_link,
-            self._format_rate(m.mismatch_rate),
-        )
+    self._render_link(
+        s,
+        'Mismatches (%d/%d)' % (m.num_mismatches, m.total),
+        self._format_rate(m.mismatch_rate),
+        'color:orange',
+        lambda: self.mismatches_link,
     )
     s.write(' | ')
     super()._render_summary_metrics(s)
@@ -275,9 +271,9 @@ class Matching(base.Evaluation):
     for i, (example, output, message) in enumerate(self.matches):
       bgcolor = 'white' if i % 2 == 0 else '#DDDDDD'
       s.write(f'<tr style="background-color: {bgcolor}"><td>{i + 1}</td>')
-      input_str = pg.format(example, verbose=False)
+      input_str = pg.format(example, verbose=False, max_bytes_len=32)
       s.write(f'<td style="color:green;white-space:pre-wrap">{input_str}</td>')
-      output_str = pg.format(output, verbose=False)
+      output_str = pg.format(output, verbose=False, max_bytes_len=32)
       s.write(f'<td style="color:blue;white-space:pre-wrap">{output_str}</td>')
       s.write('<td>')
       self._render_message(message, s)
@@ -300,9 +296,9 @@ class Matching(base.Evaluation):
     for i, (example, output, message) in enumerate(self.mismatches):
       bgcolor = 'white' if i % 2 == 0 else '#DDDDDD'
       s.write(f'<tr style="background-color: {bgcolor}"><td>{i + 1}</td>')
-      input_str = pg.format(example, verbose=False)
+      input_str = pg.format(example, verbose=False, max_bytes_len=32)
       s.write(f'<td style="color:green;white-space:pre-wrap">{input_str}</td>')
-      output_str = pg.format(output, verbose=False)
+      output_str = pg.format(output, verbose=False, max_bytes_len=32)
       s.write(
           f'<td style="color:magenta;white-space:pre-wrap">{output_str}</td>'
       )

langfun/core/eval/scoring.py

@@ -172,14 +172,12 @@ class Scoring(base.Evaluation):
     """Renders metrics in HTML."""
     assert self.result is not None
     m = self.result.metrics
-    s.write(
-        '<a title="Average score (%d/%d)" href="%s" style="color:green">%s</a>'
-        % (
-            m.num_scored,
-            m.total,
-            self.scored_link,
-            '%.2f%%' % (m.score_rate * 100),
-        )
+    self._render_link(
+        s,
+        'Average score (%d/%d)' % (m.num_scored, m.total),
+        '%.2f (%.2f%%)' % (m.avg_score, m.score_rate * 100),
+        'color:green',
+        lambda: self.scored_link,
     )
     s.write(' | ')
     super()._render_summary_metrics(s)
@@ -198,9 +196,9 @@ class Scoring(base.Evaluation):
     for i, (example, output, score, message) in enumerate(self.scored):
       bgcolor = 'white' if i % 2 == 0 else '#DDDDDD'
       s.write(f'<tr style="background-color: {bgcolor}"><td>{i + 1}</td>')
-      input_str = pg.format(example, verbose=False)
+      input_str = pg.format(example, verbose=False, max_bytes_len=32)
       s.write(f'<td style="color:green;white-space:pre-wrap">{input_str}</td>')
-      output_str = pg.format(output, verbose=False)
+      output_str = pg.format(output, verbose=False, max_bytes_len=32)
       s.write(f'<td style="color:blue;white-space:pre-wrap">{output_str}</td>')
       s.write(f'<td style="color:magenta;white-space:pre-wrap">{score}</td>')
       s.write('<td>')

langfun/core/llms/__init__.py

@@ -39,6 +39,8 @@ from langfun.core.llms.google_genai import Palm2_IT
 # OpenAI models.
 from langfun.core.llms.openai import OpenAI
 
+from langfun.core.llms.openai import Gpt4oMini
+from langfun.core.llms.openai import Gpt4oMini_20240718
 from langfun.core.llms.openai import Gpt4o
 from langfun.core.llms.openai import Gpt4o_20240513
 

langfun/core/llms/openai.py

@@ -34,6 +34,8 @@ SUPPORTED_MODELS_AND_SETTINGS = {
     # Models from https://platform.openai.com/docs/models
     # RPM is from https://platform.openai.com/docs/guides/rate-limits
     # GPT-4o models
+    'gpt-4o-mini': pg.Dict(rpm=10000, tpm=5000000),
+    'gpt-4o-mini-2024-07-18': pg.Dict(rpm=10000, tpm=5000000),
     'gpt-4o': pg.Dict(rpm=10000, tpm=5000000),
     'gpt-4o-2024-05-13': pg.Dict(rpm=10000, tpm=5000000),
     # GPT-4-Turbo models
@@ -347,6 +349,18 @@ class Gpt4_32K_20230613(Gpt4_32K):    # pylint:disable=invalid-name
   model = 'gpt-4-32k-0613'
 
 
+class Gpt4oMini(OpenAI):
+  """GPT-4o Mini."""
+  model = 'gpt-4o-mini'
+  multimodal = True
+
+
+class Gpt4oMini_20240718(OpenAI):  # pylint:disable=invalid-name
+  """GPT-4o Mini."""
+  model = 'gpt-4o-mini-2024-07-18'
+  multimodal = True
+
+
 class Gpt4o(OpenAI):
   """GPT-4o."""
   model = 'gpt-4o'

{langfun-0.1.1.dev20240729.dist-info → langfun-0.1.1.dev20240801.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: langfun
-Version: 0.1.1.dev20240729
+Version: 0.1.1.dev20240801
 Summary: Langfun: Language as Functions.
 Home-page: https://github.com/google/langfun
 Author: Langfun Authors

{langfun-0.1.1.dev20240729.dist-info → langfun-0.1.1.dev20240801.dist-info}/RECORD

@@ -44,15 +44,15 @@ langfun/core/coding/python/parsing_test.py,sha256=9vAWF484kWIm6JZq8NFiMgKUDhXV-d
 langfun/core/coding/python/permissions.py,sha256=1QWGHvzL8MM0Ok_auQ9tURqZHtdOfJaDpBzZ29GUE-c,2544
 langfun/core/coding/python/permissions_test.py,sha256=w5EDb8QxpxgJyZkojyzVWQvDfg366zn99-g__6TbPQ0,2699
 langfun/core/eval/__init__.py,sha256=Evt-E4FEhZF2tXL6-byh_AyA7Cc_ZoGmvnN7vkAZedk,1898
-langfun/core/eval/base.py,sha256=GM98Zo4gxZui2ORX6Q7Zr94PfiEViQC5X_qz-uj6b2k,74220
+langfun/core/eval/base.py,sha256=zZgebhUubX-149VkwkyQBKkTyv5hF0ubjj9eBd2baLo,75201
 langfun/core/eval/base_test.py,sha256=cHOTIWVW4Dp8gKKIKcZrAcJ-w84j2GIozTzJoiAX7p4,26743
-langfun/core/eval/matching.py,sha256=Y4vFoNTQEOwko6IA8l9OZ52-vt52e3VGmcTtvLA67wM,9782
+langfun/core/eval/matching.py,sha256=9GX8HfO9jKxgNLAivgy5K88Xhoh6Z7Pptq65pe7vht8,9762
 langfun/core/eval/matching_test.py,sha256=f7iVyXH5KGJBWt4Wp14Bt9J3X59A6Ayfog9MbuFvPew,5532
 langfun/core/eval/patching.py,sha256=R0s2eAd1m97exQt06dmUL0V_MBG0W2Hxg7fhNB7cXW0,3866
 langfun/core/eval/patching_test.py,sha256=8kCd54Egjju22FMgtJuxEsrXkW8ifs-UUBHtrCG1L6w,4775
-langfun/core/eval/scoring.py,sha256=1J7IATo-8FXUR0SBqk9icztHiM0lWkBFcWUo-vUURgQ,6376
+langfun/core/eval/scoring.py,sha256=AlCwEVrU6nvURDB1aPxA2XBUmOjWxuNJDXJoS4-6VbU,6386
 langfun/core/eval/scoring_test.py,sha256=O8olHbrUEg60gMxwOkWzKBJZpZoUlmVnBANX5Se2SXM,4546
-langfun/core/llms/__init__.py,sha256=sRD_PjfD5wKuzANCOCjChjuZHUn2Q1WruZeVflqej5M,4609
+langfun/core/llms/__init__.py,sha256=YGILcGi2QTxDG0v-0Gd4uAj1HL_zRhtllOM9EURxzDg,4712
 langfun/core/llms/anthropic.py,sha256=Gon3fOi31RhZFgNd0ijyTnKnUdp9hrWrCoSXyO4UaLw,7316
 langfun/core/llms/anthropic_test.py,sha256=T-swuMkfnlgs8Fpif4rtXs579exGk0TsbLMirXDZCkg,5533
 langfun/core/llms/fake.py,sha256=Dd7-6ka9pFf3fcWZyczamjOqQ91MOI-m7We3Oc9Ffmo,2927
@@ -63,7 +63,7 @@ langfun/core/llms/groq.py,sha256=pqtyOZ_1_OJMOg8xATWT_B_SVbuT9nMRf4VkH9GzW8g,630
 langfun/core/llms/groq_test.py,sha256=GYF_Qtq5S1H1TrKH38t6_lkdroqT7v-joYLDKnmS9e0,5274
 langfun/core/llms/llama_cpp.py,sha256=9tXQntSCDtjTF3bnyJrAPCr4N6wycy5nXYvp9uduygE,2843
 langfun/core/llms/llama_cpp_test.py,sha256=MWO_qaOeKjRniGjcaWPDScd7HPaIJemqUZoslrt4FPs,1806
-langfun/core/llms/openai.py,sha256=0z9qIH9FlWj9VWUnhOX321T6JHO-vjY2IozT7OVI4GY,13654
+langfun/core/llms/openai.py,sha256=jILxfFb3vBuyf1u_2-LVfs_wekPF2RVuNFzNVg25pEA,14004
 langfun/core/llms/openai_test.py,sha256=3muDTnW7UBOSHq694Fi2bofqhe8Pkj0Tl8IShoLCTOM,15525
 langfun/core/llms/rest.py,sha256=laopuq-zD8V-3Y6eFDngftHEbE66VlUkCD2-rvvRaLU,3388
 langfun/core/llms/rest_test.py,sha256=NZ3Nf0XQVpT9kLP5cBVo_yBHLI7vWTYhWQxYEJVMGs4,3472
@@ -117,8 +117,8 @@ langfun/core/templates/demonstration.py,sha256=vCrgYubdZM5Umqcgp8NUVGXgr4P_c-fik
 langfun/core/templates/demonstration_test.py,sha256=SafcDQ0WgI7pw05EmPI2S4v1t3ABKzup8jReCljHeK4,2162
 langfun/core/templates/selfplay.py,sha256=yhgrJbiYwq47TgzThmHrDQTF4nDrTI09CWGhuQPNv-s,2273
 langfun/core/templates/selfplay_test.py,sha256=rBW2Qr8yi-aWYwoTwRR-n1peKyMX9QXPZXURjLgoiRs,2264
-langfun-0.1.1.dev20240729.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-langfun-0.1.1.dev20240729.dist-info/METADATA,sha256=-rNHziPZUnTYVT8VjFZmPLIU5Ss41mBH3AJY62soJbY,5247
-langfun-0.1.1.dev20240729.dist-info/WHEEL,sha256=Rp8gFpivVLXx-k3U95ozHnQw8yDcPxmhOpn_Gx8d5nc,91
-langfun-0.1.1.dev20240729.dist-info/top_level.txt,sha256=RhlEkHxs1qtzmmtWSwYoLVJAc1YrbPtxQ52uh8Z9VvY,8
-langfun-0.1.1.dev20240729.dist-info/RECORD,,
+langfun-0.1.1.dev20240801.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+langfun-0.1.1.dev20240801.dist-info/METADATA,sha256=K0R8xinnLE-4gm4ecUOyBm2oaSGKgmJaDMbJ3MpgZho,5247
+langfun-0.1.1.dev20240801.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
+langfun-0.1.1.dev20240801.dist-info/top_level.txt,sha256=RhlEkHxs1qtzmmtWSwYoLVJAc1YrbPtxQ52uh8Z9VvY,8
+langfun-0.1.1.dev20240801.dist-info/RECORD,,

{langfun-0.1.1.dev20240729.dist-info → langfun-0.1.1.dev20240801.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (72.0.0)
+Generator: setuptools (72.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any