langfun 0.1.2.dev202501030804__tar.gz → 0.1.2.dev202501040804__tar.gz

{langfun-0.1.2.dev202501030804 → langfun-0.1.2.dev202501040804}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: langfun
-Version: 0.1.2.dev202501030804
+Version: 0.1.2.dev202501040804
 Summary: Langfun: Language as Functions.
 Home-page: https://github.com/google/langfun
 Author: Langfun Authors

{langfun-0.1.2.dev202501030804 → langfun-0.1.2.dev202501040804}/langfun/core/eval/v2/experiment.py

@@ -381,7 +381,7 @@ class Experiment(lf.Component, pg.views.HtmlTreeView.Extension):
       example_ids: list[int] | None = None,
       raise_if_has_error: bool = False,
       reprocess: bool | list[int] = False,
-      regenerate_example_html: bool | list[int] = False,
+      generate_example_html: Literal['new', 'all', 'no'] | list[int] = 'new',
       process_timeout: int | None = None,
       use_cache: Literal['global', 'per_dataset', 'no'] = 'per_dataset',
       note: str | None = None,
@@ -435,11 +435,13 @@ class Experiment(lf.Component, pg.views.HtmlTreeView.Extension):
         meaning that existing checkpoints will be ignored. If a list of
         example IDs, it indicates that only the specified examples will be
         reprocessed.
-      regenerate_example_html: A boolean or a list of example IDs. If boolean,
-        it indicates that whether all the examples to be evaluated will have
-        their HTML files regenerated. If a list of example IDs, it indicates
-        that only the specified examples will have their HTML files
-        regenerated.
+      generate_example_html: Among 'new', 'all', 'no' or a list of example IDs.
+        If 'new', generate HTML files for all newly processed examples, and
+          keep/copy existing HTML files for unchanged examples.
+        If 'all', generate HTML files for all examples.
+        If 'no', do not generate HTML files for any examples.
+        If a list of example IDs, generate HTML files for the specified
+        examples.
       process_timeout: The timeout in seconds for each process. If None, it
         will use the default timeout for the runner.
       use_cache: Whether to use LLM cache for the experiment.
@@ -467,7 +469,7 @@ class Experiment(lf.Component, pg.views.HtmlTreeView.Extension):
             example_ids=example_ids,
             raise_if_has_error=raise_if_has_error,
             reprocess=reprocess,
-            regenerate_example_html=regenerate_example_html,
+            generate_example_html=generate_example_html,
             use_cache=use_cache,
             process_timeout=process_timeout,
             note=note,
@@ -837,14 +839,17 @@ class Run(pg.Object, pg.views.html.HtmlTreeView.Extension):
       )
   ] = False
 
-  regenerate_example_html: Annotated[
-      bool | list[int],
+  generate_example_html: Annotated[
+      Literal['new', 'all', 'no'] | list[int],
       (
-          'If True, it will regenerate the HTML files for previously processed '
-          'examples. If a list of integers, the HTML files for the examples of '
-          'the given IDs will be regenerated'
+          'If "new", generate HTML files for all newly processed examples, '
+          'and keep/copy existing HTML files for unchanged examples. '
+          'If "all", generate HTML files for all examples. '
+          'If "no", do not generate HTML files for any examples. '
+          'If a list of example IDs, generate HTML files for the specified '
+          'examples.'
       )
-  ] = False
+  ] = 'new'
 
   filter: Annotated[
       Callable[[Experiment], bool] | None,
@@ -917,17 +922,17 @@ class Run(pg.Object, pg.views.html.HtmlTreeView.Extension):
   def examples_to_load(self, experiment: Experiment) -> set[int]:
     """Returns the example IDs to load from checkpoint files.."""
     load_ids = self.examples_to_evaluate(experiment)
-    if isinstance(self.regenerate_example_html, list):
-      load_ids |= set(self.regenerate_example_html)
+    if isinstance(self.generate_example_html, list):
+      load_ids |= set(self.generate_example_html)
     load_ids -= self.examples_to_reprocess(experiment)
     return load_ids
 
   def examples_to_load_metadata(self, experiment: Experiment) -> set[int]:
     """Returns the example IDs to load the metadata."""
     load_metadata_ids = set()
-    if isinstance(self.regenerate_example_html, list):
-      load_metadata_ids = set(self.regenerate_example_html)
-    elif self.regenerate_example_html:
+    if isinstance(self.generate_example_html, list):
+      load_metadata_ids = set(self.generate_example_html)
+    elif self.generate_example_html == 'all':
       load_metadata_ids = self.examples_to_evaluate(experiment)
     load_metadata_ids -= self.examples_to_reprocess(experiment)
     return load_metadata_ids

{langfun-0.1.2.dev202501030804 → langfun-0.1.2.dev202501040804}/langfun/core/eval/v2/experiment_test.py

@@ -337,7 +337,7 @@ class RunTest(unittest.TestCase):
     self.assertEqual(run.examples_to_load(exp), set([3, 5]))
     self.assertEqual(run.examples_to_load_metadata(exp), set())
 
-  def test_examples_with_regenerate_example_html_all(self):
+  def test_examples_with_generate_example_html_all(self):
     run = Run(
         '/root',
         RunId.from_id('20241102_0'),
@@ -346,7 +346,7 @@ class RunTest(unittest.TestCase):
         ])),
         example_ids=[1, 3, 5],
         reprocess=[1],
-        regenerate_example_html=True,
+        generate_example_html='all',
     )
     exp = run.experiment.leaf_nodes[0]
     self.assertEqual(run.examples_to_evaluate(exp), set([1, 3, 5]))
@@ -354,7 +354,7 @@ class RunTest(unittest.TestCase):
     self.assertEqual(run.examples_to_load(exp), set([3, 5]))
     self.assertEqual(run.examples_to_load_metadata(exp), set([3, 5]))
 
-  def test_examples_with_regenerate_example_html_some(self):
+  def test_examples_with_generate_example_html_new(self):
     run = Run(
         '/root',
         RunId.from_id('20241102_0'),
@@ -363,7 +363,24 @@ class RunTest(unittest.TestCase):
         ])),
         example_ids=[1, 3, 5],
         reprocess=[1],
-        regenerate_example_html=[1, 2, 3],
+        generate_example_html='new',
+    )
+    exp = run.experiment.leaf_nodes[0]
+    self.assertEqual(run.examples_to_evaluate(exp), set([1, 3, 5]))
+    self.assertEqual(run.examples_to_reprocess(exp), set([1]))
+    self.assertEqual(run.examples_to_load(exp), set([3, 5]))
+    self.assertEqual(run.examples_to_load_metadata(exp), set())
+
+  def test_examples_with_generate_example_html_some(self):
+    run = Run(
+        '/root',
+        RunId.from_id('20241102_0'),
+        pg.Ref(Suite([
+            MyEvaluation(replica_id=0, inputs=sample_inputs(10)),
+        ])),
+        example_ids=[1, 3, 5],
+        reprocess=[1],
+        generate_example_html=[1, 2, 3],
     )
     exp = run.experiment.leaf_nodes[0]
     self.assertEqual(run.examples_to_evaluate(exp), set([1, 3, 5]))

{langfun-0.1.2.dev202501030804 → langfun-0.1.2.dev202501040804}/langfun/core/eval/v2/reporting.py

@@ -195,6 +195,7 @@ class HtmlReporter(experiment_lib.Plugin):
       self, runner: Runner, experiment: Experiment, example: Example
   ) -> None:
     """Saves the example in HTML format."""
+    current_run = runner.current_run
     def _generate():
       try:
         with pg.timeit() as t:
@@ -222,14 +223,19 @@ class HtmlReporter(experiment_lib.Plugin):
         raise e
 
     def _copy():
-      src_file = runner.current_run.input_path_for(
-          experiment, f'{example.id}.html'
-      )
-      dest_file = runner.current_run.output_path_for(
-          experiment, f'{example.id}.html'
-      )
+      src_file = current_run.input_path_for(experiment, f'{example.id}.html')
+      dest_file = current_run.output_path_for(experiment, f'{example.id}.html')
+
       if src_file == dest_file:
         return
+
+      if not pg.io.path_exists(src_file):
+        experiment.warning(
+            f'Skip copying \'{example.id}.html\' as '
+            f'{src_file!r} does not exist.'
+        )
+        return
+
       try:
         with pg.timeit() as t, pg.io.open(src_file, 'r') as src:
           content = src.read()
@@ -244,7 +250,11 @@ class HtmlReporter(experiment_lib.Plugin):
         )
         raise e
 
-    if example.newly_processed or runner.current_run.regenerate_example_html:
+    generate_example_html = current_run.generate_example_html
+    if (generate_example_html == 'all'
+        or (generate_example_html == 'new' and example.newly_processed)
+        or (isinstance(generate_example_html, list)
+            and example.id in generate_example_html)):
       op = _generate
     else:
       op = _copy

{langfun-0.1.2.dev202501030804 → langfun-0.1.2.dev202501040804}/langfun/core/eval/v2/reporting_test.py

@@ -148,7 +148,7 @@ class ReportingTest(unittest.TestCase):
         )
     found_error_log = False
     for log_entry in experiment._log_entries:
-      if log_entry.message.startswith('Failed to copy'):
+      if log_entry.message.startswith('Skip copying'):
         found_error_log = True
         break
     self.assertTrue(found_error_log)

{langfun-0.1.2.dev202501030804 → langfun-0.1.2.dev202501040804}/langfun/core/llms/__init__.py

@@ -32,6 +32,7 @@ from langfun.core.llms.rest import REST
 
 # Gemini models.
 from langfun.core.llms.google_genai import GenAI
+from langfun.core.llms.google_genai import GeminiFlash2_0ThinkingExp
 from langfun.core.llms.google_genai import GeminiFlash2_0Exp
 from langfun.core.llms.google_genai import GeminiExp_20241114
 from langfun.core.llms.google_genai import GeminiExp_20241206
@@ -126,6 +127,7 @@ from langfun.core.llms.groq import GroqWhisper_Large_v3Turbo
 from langfun.core.llms.vertexai import VertexAI
 from langfun.core.llms.vertexai import VertexAIGemini2_0
 from langfun.core.llms.vertexai import VertexAIGeminiFlash2_0Exp
+from langfun.core.llms.vertexai import VertexAIGeminiFlash2_0ThinkingExp
 from langfun.core.llms.vertexai import VertexAIGemini1_5
 from langfun.core.llms.vertexai import VertexAIGeminiPro1_5
 from langfun.core.llms.vertexai import VertexAIGeminiPro1_5_001

{langfun-0.1.2.dev202501030804 → langfun-0.1.2.dev202501040804}/langfun/core/llms/google_genai.py

@@ -48,6 +48,7 @@ class GenAI(lf.LanguageModel):
 
   model: Annotated[
       Literal[
+          'gemini-2.0-flash-thinking-exp-1219',
           'gemini-2.0-flash-exp',
           'gemini-exp-1206',
           'gemini-exp-1114',
@@ -307,6 +308,16 @@ _GOOGLE_GENAI_MODEL_HUB = _ModelHub()
 #
 # Public Gemini models.
 #
+class GeminiFlash2_0ThinkingExp(GenAI):  # pylint: disable=invalid-name
+  """Gemini 2.0 Flash Thinking Experimental model."""
+
+  model = 'gemini-2.0-flash-thinking-exp-1219'
+  supported_modalities = (
+      vertexai.DOCUMENT_TYPES
+      + vertexai.IMAGE_TYPES
+      + vertexai.AUDIO_TYPES
+      + vertexai.VIDEO_TYPES
+  )
 
 
 class GeminiFlash2_0Exp(GenAI):  # pylint: disable=invalid-name

{langfun-0.1.2.dev202501030804 → langfun-0.1.2.dev202501040804}/langfun/core/llms/vertexai.py

@@ -110,7 +110,13 @@ SUPPORTED_MODELS_AND_SETTINGS = {
     ),
     # TODO(sharatsharat): Update costs when published
     'gemini-2.0-flash-exp': pg.Dict(
-        rpm=20,
+        rpm=10,
+        cost_per_1k_input_chars=0.000,
+        cost_per_1k_output_chars=0.000,
+    ),
+    # TODO(yifenglu): Update costs when published
+    'gemini-2.0-flash-thinking-exp-1219': pg.Dict(
+        rpm=10,
         cost_per_1k_input_chars=0.000,
         cost_per_1k_output_chars=0.000,
     ),
@@ -415,6 +421,12 @@ class VertexAIGeminiFlash2_0Exp(VertexAIGemini2_0):  # pylint: disable=invalid-n
   model = 'gemini-2.0-flash-exp'
 
 
+class VertexAIGeminiFlash2_0ThinkingExp(VertexAIGemini2_0):  # pylint: disable=invalid-name
+  """Vertex AI Gemini 2.0 Flash model."""
+
+  model = 'gemini-2.0-flash-thinking-exp-1219'
+
+
 class VertexAIGemini1_5(VertexAI):  # pylint: disable=invalid-name
   """Vertex AI Gemini 1.5 model."""
 

{langfun-0.1.2.dev202501030804 → langfun-0.1.2.dev202501040804}/langfun.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: langfun
-Version: 0.1.2.dev202501030804
+Version: 0.1.2.dev202501040804
 Summary: Langfun: Language as Functions.
 Home-page: https://github.com/google/langfun
 Author: Langfun Authors