langfun 0.1.1.dev20240729__tar.gz → 0.1.1.dev20240801__tar.gz

{langfun-0.1.1.dev20240729 → langfun-0.1.1.dev20240801}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: langfun
-Version: 0.1.1.dev20240729
+Version: 0.1.1.dev20240801
 Summary: Langfun: Language as Functions.
 Home-page: https://github.com/google/langfun
 Author: Langfun Authors

{langfun-0.1.1.dev20240729 → langfun-0.1.1.dev20240801}/langfun/core/eval/base.py

@@ -542,6 +542,8 @@ class Evaluable(lf.Component):
           'padding: 10px; border: 1px solid; margin-top: 10px">'
       )
       s.write(html.escape(m.get('formatted_text', m.text)))
+
+      # Write output.
       if m.result is not None:
         s.write(
             '<div style="color: magenta; white-space: pre-wrap;'
@@ -549,6 +551,23 @@ class Evaluable(lf.Component):
         )
         s.write(html.escape(pg.format(m.result)))
         s.write('</div>')
+
+      # Write modality information.
+      if 'lm-input' in m.tags or 'lm-response' in m.tags:
+        modalities = m.referred_modalities()
+        if modalities:
+          s.write(f'<div style="color: {text_color}; white-space: pre-wrap;'
+                  'padding: 10px; border: 1px solid; margin-top: 10px"><table>')
+          for name, modality in modalities.items():
+            s.write(f'<tr><td>{name}</td><td>')
+            if hasattr(modality, '_repr_html_'):
+              s.write(modality._repr_html_())   # pylint: disable=protected-access
+            else:
+              s.write(html.escape(pg.format(modality, max_bytes_len=32)))
+            s.write('</td></tr>')
+          s.write('</table></div>')
+
+      # Write usage information.
       if m.metadata.get('usage', None):
         s.write(
             '<div style="background-color: #EEEEEE; color: black; '
@@ -1304,20 +1323,22 @@ class Evaluation(Evaluable):
     s = io.StringIO()
     definition = _html_repr(self, compact=False, escape=True)
     s.write('<div><table><tr><td>')
+    self._render_link(
+        s,
+        definition,
+        self.hash,
+        '',
+        lambda: self.link(self.dir),
+    )
     if self.result is None:
       s.write(
-          f'<a target="_blank" title="{definition}" '
-          f'href="{self.link(self.dir)}">{self.hash}</a>'
           '</td></tr><tr><td>'
           '<span style="color: gray">(IN-PROGRESS...)</span>'
       )
     else:
-      s.write(
-          f'<a target="_blank" title="{definition}" '
-          f'href="{self.index_link}">{self.hash}</a>'
-          f' &nbsp;[<a href="{self.link(self.dir)}">dir</a>]'
-          '</td></tr><tr><td>'
-      )
+      if self.dir:
+        s.write(f' &nbsp;[<a href="{self.link(self.dir)}">dir</a>]')
+      s.write('</td></tr><tr><td>')
       self._render_summary_metrics(s)
 
       # Summarize average usage.
@@ -1341,6 +1362,20 @@ class Evaluation(Evaluable):
         f'" style="color:gray">({total} tokens)</a>'
     )
 
+  def _render_link(self,
+                   s: io.StringIO,
+                   title: str,
+                   text: str,
+                   style: str,
+                   url_fn: Callable[[], str]) -> None:
+    """Renders a link in HTML."""
+    s.write(
+        f'<a target="_blank" title="{title}" style="{style}"'
+    )
+    if self.dir:
+      s.write(f' href="{url_fn()}"')
+    s.write(f'>{text}</a>')
+
   def _render_summary_metrics(self, s: io.StringIO) -> None:
     """Renders metrics in HTML."""
     assert self.result is not None
@@ -1362,14 +1397,12 @@ class Evaluation(Evaluable):
     extra_style = ''
     if m.oop_failure_rate > 0.1 and m.oop_failures > 3:
       extra_style = ';font-weight:bold'
-    s.write(
-        '<a title="%s" href="%s" style="color:magenta%s">%s</a>'
-        % (
-            oop_failure_title,
-            self.oop_failures_link,
-            extra_style,
-            self._format_rate(m.oop_failure_rate),
-        )
+    self._render_link(
+        s,
+        oop_failure_title,
+        self._format_rate(m.oop_failure_rate),
+        f'color:magenta{extra_style}',
+        lambda: self.oop_failures_link,
     )
     s.write(' | ')
 
@@ -1387,14 +1420,12 @@ class Evaluation(Evaluable):
           )
 
     extra_style = ';font-weight:bold' if m.non_oop_failures > 0 else ''
-    s.write(
-        '<a title="%s" href="%s" style="color:red%s">%s</a>'
-        % (
-            non_oop_failure_title,
-            self.non_oop_failures_link,
-            extra_style,
-            self._format_rate(m.non_oop_failure_rate),
-        )
+    self._render_link(
+        s,
+        non_oop_failure_title,
+        self._format_rate(m.non_oop_failure_rate),
+        f'color:red{extra_style}',
+        lambda: self.non_oop_failures_link,
     )
 
   def _format_rate(self, rate: float) -> str:

{langfun-0.1.1.dev20240729 → langfun-0.1.1.dev20240801}/langfun/core/eval/matching.py

@@ -239,24 +239,20 @@ class Matching(base.Evaluation):
     """Renders metrics in HTML."""
     assert self.result is not None
     m = self.result.metrics
-    s.write(
-        '<a title="Matches (%d/%d)" href="%s" style="color:green">%s</a>'
-        % (
-            m.num_matches,
-            m.total,
-            self.matches_link,
-            self._format_rate(m.match_rate),
-        )
+    self._render_link(
+        s,
+        'Matches (%d/%d)' % (m.num_matches, m.total),
+        self._format_rate(m.match_rate),
+        'color:green',
+        lambda: self.matches_link,
     )
     s.write(' | ')
-    s.write(
-        '<a title="Mismatches (%d/%d)" href="%s" style="color:orange">%s</a>'
-        % (
-            m.num_mismatches,
-            m.total,
-            self.mismatches_link,
-            self._format_rate(m.mismatch_rate),
-        )
+    self._render_link(
+        s,
+        'Mismatches (%d/%d)' % (m.num_mismatches, m.total),
+        self._format_rate(m.mismatch_rate),
+        'color:orange',
+        lambda: self.mismatches_link,
     )
     s.write(' | ')
     super()._render_summary_metrics(s)
@@ -275,9 +271,9 @@ class Matching(base.Evaluation):
     for i, (example, output, message) in enumerate(self.matches):
       bgcolor = 'white' if i % 2 == 0 else '#DDDDDD'
       s.write(f'<tr style="background-color: {bgcolor}"><td>{i + 1}</td>')
-      input_str = pg.format(example, verbose=False)
+      input_str = pg.format(example, verbose=False, max_bytes_len=32)
       s.write(f'<td style="color:green;white-space:pre-wrap">{input_str}</td>')
-      output_str = pg.format(output, verbose=False)
+      output_str = pg.format(output, verbose=False, max_bytes_len=32)
       s.write(f'<td style="color:blue;white-space:pre-wrap">{output_str}</td>')
       s.write('<td>')
       self._render_message(message, s)
@@ -300,9 +296,9 @@ class Matching(base.Evaluation):
     for i, (example, output, message) in enumerate(self.mismatches):
       bgcolor = 'white' if i % 2 == 0 else '#DDDDDD'
       s.write(f'<tr style="background-color: {bgcolor}"><td>{i + 1}</td>')
-      input_str = pg.format(example, verbose=False)
+      input_str = pg.format(example, verbose=False, max_bytes_len=32)
       s.write(f'<td style="color:green;white-space:pre-wrap">{input_str}</td>')
-      output_str = pg.format(output, verbose=False)
+      output_str = pg.format(output, verbose=False, max_bytes_len=32)
       s.write(
           f'<td style="color:magenta;white-space:pre-wrap">{output_str}</td>'
       )

{langfun-0.1.1.dev20240729 → langfun-0.1.1.dev20240801}/langfun/core/eval/scoring.py

@@ -172,14 +172,12 @@ class Scoring(base.Evaluation):
     """Renders metrics in HTML."""
     assert self.result is not None
     m = self.result.metrics
-    s.write(
-        '<a title="Average score (%d/%d)" href="%s" style="color:green">%s</a>'
-        % (
-            m.num_scored,
-            m.total,
-            self.scored_link,
-            '%.2f%%' % (m.score_rate * 100),
-        )
+    self._render_link(
+        s,
+        'Average score (%d/%d)' % (m.num_scored, m.total),
+        '%.2f (%.2f%%)' % (m.avg_score, m.score_rate * 100),
+        'color:green',
+        lambda: self.scored_link,
     )
     s.write(' | ')
     super()._render_summary_metrics(s)
@@ -198,9 +196,9 @@ class Scoring(base.Evaluation):
     for i, (example, output, score, message) in enumerate(self.scored):
       bgcolor = 'white' if i % 2 == 0 else '#DDDDDD'
       s.write(f'<tr style="background-color: {bgcolor}"><td>{i + 1}</td>')
-      input_str = pg.format(example, verbose=False)
+      input_str = pg.format(example, verbose=False, max_bytes_len=32)
       s.write(f'<td style="color:green;white-space:pre-wrap">{input_str}</td>')
-      output_str = pg.format(output, verbose=False)
+      output_str = pg.format(output, verbose=False, max_bytes_len=32)
       s.write(f'<td style="color:blue;white-space:pre-wrap">{output_str}</td>')
       s.write(f'<td style="color:magenta;white-space:pre-wrap">{score}</td>')
       s.write('<td>')

{langfun-0.1.1.dev20240729 → langfun-0.1.1.dev20240801}/langfun/core/llms/__init__.py

@@ -39,6 +39,8 @@ from langfun.core.llms.google_genai import Palm2_IT
 # OpenAI models.
 from langfun.core.llms.openai import OpenAI
 
+from langfun.core.llms.openai import Gpt4oMini
+from langfun.core.llms.openai import Gpt4oMini_20240718
 from langfun.core.llms.openai import Gpt4o
 from langfun.core.llms.openai import Gpt4o_20240513
 

{langfun-0.1.1.dev20240729 → langfun-0.1.1.dev20240801}/langfun/core/llms/openai.py

@@ -34,6 +34,8 @@ SUPPORTED_MODELS_AND_SETTINGS = {
     # Models from https://platform.openai.com/docs/models
     # RPM is from https://platform.openai.com/docs/guides/rate-limits
     # GPT-4o models
+    'gpt-4o-mini': pg.Dict(rpm=10000, tpm=5000000),
+    'gpt-4o-mini-2024-07-18': pg.Dict(rpm=10000, tpm=5000000),
     'gpt-4o': pg.Dict(rpm=10000, tpm=5000000),
     'gpt-4o-2024-05-13': pg.Dict(rpm=10000, tpm=5000000),
     # GPT-4-Turbo models
@@ -347,6 +349,18 @@ class Gpt4_32K_20230613(Gpt4_32K):    # pylint:disable=invalid-name
   model = 'gpt-4-32k-0613'
 
 
+class Gpt4oMini(OpenAI):
+  """GPT-4o Mini."""
+  model = 'gpt-4o-mini'
+  multimodal = True
+
+
+class Gpt4oMini_20240718(OpenAI):  # pylint:disable=invalid-name
+  """GPT-4o Mini."""
+  model = 'gpt-4o-mini-2024-07-18'
+  multimodal = True
+
+
 class Gpt4o(OpenAI):
   """GPT-4o."""
   model = 'gpt-4o'

{langfun-0.1.1.dev20240729 → langfun-0.1.1.dev20240801}/langfun.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: langfun
-Version: 0.1.1.dev20240729
+Version: 0.1.1.dev20240801
 Summary: Langfun: Language as Functions.
 Home-page: https://github.com/google/langfun
 Author: Langfun Authors