langfun 0.1.2.dev202501030804__py3-none-any.whl → 0.1.2.dev202501040804__py3-none-any.whl

langfun/core/eval/v2/experiment.py

@@ -381,7 +381,7 @@ class Experiment(lf.Component, pg.views.HtmlTreeView.Extension):
       example_ids: list[int] | None = None,
       raise_if_has_error: bool = False,
       reprocess: bool | list[int] = False,
-      regenerate_example_html: bool | list[int] = False,
+      generate_example_html: Literal['new', 'all', 'no'] | list[int] = 'new',
       process_timeout: int | None = None,
       use_cache: Literal['global', 'per_dataset', 'no'] = 'per_dataset',
       note: str | None = None,
@@ -435,11 +435,13 @@ class Experiment(lf.Component, pg.views.HtmlTreeView.Extension):
         meaning that existing checkpoints will be ignored. If a list of
         example IDs, it indicates that only the specified examples will be
         reprocessed.
-      regenerate_example_html: A boolean or a list of example IDs. If boolean,
-        it indicates that whether all the examples to be evaluated will have
-        their HTML files regenerated. If a list of example IDs, it indicates
-        that only the specified examples will have their HTML files
-        regenerated.
+      generate_example_html: Among 'new', 'all', 'no' or a list of example IDs.
+        If 'new', generate HTML files for all newly processed examples, and
+          keep/copy existing HTML files for unchanged examples.
+        If 'all', generate HTML files for all examples.
+        If 'no', do not generate HTML files for any examples.
+        If a list of example IDs, generate HTML files for the specified
+        examples.
       process_timeout: The timeout in seconds for each process. If None, it
         will use the default timeout for the runner.
       use_cache: Whether to use LLM cache for the experiment.
@@ -467,7 +469,7 @@ class Experiment(lf.Component, pg.views.HtmlTreeView.Extension):
             example_ids=example_ids,
             raise_if_has_error=raise_if_has_error,
             reprocess=reprocess,
-            regenerate_example_html=regenerate_example_html,
+            generate_example_html=generate_example_html,
             use_cache=use_cache,
             process_timeout=process_timeout,
             note=note,
@@ -837,14 +839,17 @@ class Run(pg.Object, pg.views.html.HtmlTreeView.Extension):
       )
   ] = False
 
-  regenerate_example_html: Annotated[
-      bool | list[int],
+  generate_example_html: Annotated[
+      Literal['new', 'all', 'no'] | list[int],
       (
-          'If True, it will regenerate the HTML files for previously processed '
-          'examples. If a list of integers, the HTML files for the examples of '
-          'the given IDs will be regenerated'
+          'If "new", generate HTML files for all newly processed examples, '
+          'and keep/copy existing HTML files for unchanged examples. '
+          'If "all", generate HTML files for all examples. '
+          'If "no", do not generate HTML files for any examples. '
+          'If a list of example IDs, generate HTML files for the specified '
+          'examples.'
       )
-  ] = False
+  ] = 'new'
 
   filter: Annotated[
       Callable[[Experiment], bool] | None,
@@ -917,17 +922,17 @@ class Run(pg.Object, pg.views.html.HtmlTreeView.Extension):
   def examples_to_load(self, experiment: Experiment) -> set[int]:
     """Returns the example IDs to load from checkpoint files.."""
     load_ids = self.examples_to_evaluate(experiment)
-    if isinstance(self.regenerate_example_html, list):
-      load_ids |= set(self.regenerate_example_html)
+    if isinstance(self.generate_example_html, list):
+      load_ids |= set(self.generate_example_html)
     load_ids -= self.examples_to_reprocess(experiment)
     return load_ids
 
   def examples_to_load_metadata(self, experiment: Experiment) -> set[int]:
     """Returns the example IDs to load the metadata."""
     load_metadata_ids = set()
-    if isinstance(self.regenerate_example_html, list):
-      load_metadata_ids = set(self.regenerate_example_html)
-    elif self.regenerate_example_html:
+    if isinstance(self.generate_example_html, list):
+      load_metadata_ids = set(self.generate_example_html)
+    elif self.generate_example_html == 'all':
       load_metadata_ids = self.examples_to_evaluate(experiment)
     load_metadata_ids -= self.examples_to_reprocess(experiment)
     return load_metadata_ids

langfun/core/eval/v2/experiment_test.py

@@ -337,7 +337,7 @@ class RunTest(unittest.TestCase):
     self.assertEqual(run.examples_to_load(exp), set([3, 5]))
     self.assertEqual(run.examples_to_load_metadata(exp), set())
 
-  def test_examples_with_regenerate_example_html_all(self):
+  def test_examples_with_generate_example_html_all(self):
     run = Run(
         '/root',
         RunId.from_id('20241102_0'),
@@ -346,7 +346,7 @@ class RunTest(unittest.TestCase):
         ])),
         example_ids=[1, 3, 5],
         reprocess=[1],
-        regenerate_example_html=True,
+        generate_example_html='all',
     )
     exp = run.experiment.leaf_nodes[0]
     self.assertEqual(run.examples_to_evaluate(exp), set([1, 3, 5]))
@@ -354,7 +354,7 @@ class RunTest(unittest.TestCase):
     self.assertEqual(run.examples_to_load(exp), set([3, 5]))
     self.assertEqual(run.examples_to_load_metadata(exp), set([3, 5]))
 
-  def test_examples_with_regenerate_example_html_some(self):
+  def test_examples_with_generate_example_html_new(self):
     run = Run(
         '/root',
         RunId.from_id('20241102_0'),
@@ -363,7 +363,24 @@ class RunTest(unittest.TestCase):
         ])),
         example_ids=[1, 3, 5],
         reprocess=[1],
-        regenerate_example_html=[1, 2, 3],
+        generate_example_html='new',
+    )
+    exp = run.experiment.leaf_nodes[0]
+    self.assertEqual(run.examples_to_evaluate(exp), set([1, 3, 5]))
+    self.assertEqual(run.examples_to_reprocess(exp), set([1]))
+    self.assertEqual(run.examples_to_load(exp), set([3, 5]))
+    self.assertEqual(run.examples_to_load_metadata(exp), set())
+
+  def test_examples_with_generate_example_html_some(self):
+    run = Run(
+        '/root',
+        RunId.from_id('20241102_0'),
+        pg.Ref(Suite([
+            MyEvaluation(replica_id=0, inputs=sample_inputs(10)),
+        ])),
+        example_ids=[1, 3, 5],
+        reprocess=[1],
+        generate_example_html=[1, 2, 3],
     )
     exp = run.experiment.leaf_nodes[0]
     self.assertEqual(run.examples_to_evaluate(exp), set([1, 3, 5]))

langfun/core/eval/v2/reporting.py

@@ -195,6 +195,7 @@ class HtmlReporter(experiment_lib.Plugin):
       self, runner: Runner, experiment: Experiment, example: Example
   ) -> None:
     """Saves the example in HTML format."""
+    current_run = runner.current_run
     def _generate():
       try:
         with pg.timeit() as t:
@@ -222,14 +223,19 @@ class HtmlReporter(experiment_lib.Plugin):
         raise e
 
     def _copy():
-      src_file = runner.current_run.input_path_for(
-          experiment, f'{example.id}.html'
-      )
-      dest_file = runner.current_run.output_path_for(
-          experiment, f'{example.id}.html'
-      )
+      src_file = current_run.input_path_for(experiment, f'{example.id}.html')
+      dest_file = current_run.output_path_for(experiment, f'{example.id}.html')
+
       if src_file == dest_file:
         return
+
+      if not pg.io.path_exists(src_file):
+        experiment.warning(
+            f'Skip copying \'{example.id}.html\' as '
+            f'{src_file!r} does not exist.'
+        )
+        return
+
       try:
         with pg.timeit() as t, pg.io.open(src_file, 'r') as src:
           content = src.read()
@@ -244,7 +250,11 @@ class HtmlReporter(experiment_lib.Plugin):
         )
         raise e
 
-    if example.newly_processed or runner.current_run.regenerate_example_html:
+    generate_example_html = current_run.generate_example_html
+    if (generate_example_html == 'all'
+        or (generate_example_html == 'new' and example.newly_processed)
+        or (isinstance(generate_example_html, list)
+            and example.id in generate_example_html)):
       op = _generate
     else:
       op = _copy

langfun/core/eval/v2/reporting_test.py

@@ -148,7 +148,7 @@ class ReportingTest(unittest.TestCase):
         )
     found_error_log = False
     for log_entry in experiment._log_entries:
-      if log_entry.message.startswith('Failed to copy'):
+      if log_entry.message.startswith('Skip copying'):
         found_error_log = True
         break
     self.assertTrue(found_error_log)

langfun/core/llms/__init__.py

@@ -32,6 +32,7 @@ from langfun.core.llms.rest import REST
 
 # Gemini models.
 from langfun.core.llms.google_genai import GenAI
+from langfun.core.llms.google_genai import GeminiFlash2_0ThinkingExp
 from langfun.core.llms.google_genai import GeminiFlash2_0Exp
 from langfun.core.llms.google_genai import GeminiExp_20241114
 from langfun.core.llms.google_genai import GeminiExp_20241206
@@ -126,6 +127,7 @@ from langfun.core.llms.groq import GroqWhisper_Large_v3Turbo
 from langfun.core.llms.vertexai import VertexAI
 from langfun.core.llms.vertexai import VertexAIGemini2_0
 from langfun.core.llms.vertexai import VertexAIGeminiFlash2_0Exp
+from langfun.core.llms.vertexai import VertexAIGeminiFlash2_0ThinkingExp
 from langfun.core.llms.vertexai import VertexAIGemini1_5
 from langfun.core.llms.vertexai import VertexAIGeminiPro1_5
 from langfun.core.llms.vertexai import VertexAIGeminiPro1_5_001

langfun/core/llms/google_genai.py

@@ -48,6 +48,7 @@ class GenAI(lf.LanguageModel):
 
   model: Annotated[
       Literal[
+          'gemini-2.0-flash-thinking-exp-1219',
           'gemini-2.0-flash-exp',
           'gemini-exp-1206',
           'gemini-exp-1114',
@@ -307,6 +308,16 @@ _GOOGLE_GENAI_MODEL_HUB = _ModelHub()
 #
 # Public Gemini models.
 #
+class GeminiFlash2_0ThinkingExp(GenAI):  # pylint: disable=invalid-name
+  """Gemini 2.0 Flash Thinking Experimental model."""
+
+  model = 'gemini-2.0-flash-thinking-exp-1219'
+  supported_modalities = (
+      vertexai.DOCUMENT_TYPES
+      + vertexai.IMAGE_TYPES
+      + vertexai.AUDIO_TYPES
+      + vertexai.VIDEO_TYPES
+  )
 
 
 class GeminiFlash2_0Exp(GenAI):  # pylint: disable=invalid-name

langfun/core/llms/vertexai.py

@@ -110,7 +110,13 @@ SUPPORTED_MODELS_AND_SETTINGS = {
     ),
     # TODO(sharatsharat): Update costs when published
     'gemini-2.0-flash-exp': pg.Dict(
-        rpm=20,
+        rpm=10,
+        cost_per_1k_input_chars=0.000,
+        cost_per_1k_output_chars=0.000,
+    ),
+    # TODO(yifenglu): Update costs when published
+    'gemini-2.0-flash-thinking-exp-1219': pg.Dict(
+        rpm=10,
         cost_per_1k_input_chars=0.000,
         cost_per_1k_output_chars=0.000,
     ),
@@ -415,6 +421,12 @@ class VertexAIGeminiFlash2_0Exp(VertexAIGemini2_0):  # pylint: disable=invalid-n
   model = 'gemini-2.0-flash-exp'
 
 
+class VertexAIGeminiFlash2_0ThinkingExp(VertexAIGemini2_0):  # pylint: disable=invalid-name
+  """Vertex AI Gemini 2.0 Flash model."""
+
+  model = 'gemini-2.0-flash-thinking-exp-1219'
+
+
 class VertexAIGemini1_5(VertexAI):  # pylint: disable=invalid-name
   """Vertex AI Gemini 1.5 model."""
 

{langfun-0.1.2.dev202501030804.dist-info → langfun-0.1.2.dev202501040804.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: langfun
-Version: 0.1.2.dev202501030804
+Version: 0.1.2.dev202501040804
 Summary: Langfun: Language as Functions.
 Home-page: https://github.com/google/langfun
 Author: Langfun Authors

{langfun-0.1.2.dev202501030804.dist-info → langfun-0.1.2.dev202501040804.dist-info}/RECORD

@@ -63,8 +63,8 @@ langfun/core/eval/v2/evaluation.py,sha256=kARf0pG6SrpN__IMeFsw8DV_5mT_tl52pJrr6w
 langfun/core/eval/v2/evaluation_test.py,sha256=0l0DqJTF8PZGA2Q1OlaF4YIax4ZhSk0ewOCvuVW1XAk,6658
 langfun/core/eval/v2/example.py,sha256=4-LNr8Ke-fhaF6gyeXX4JMyw0s8YkVTC63pXZ-CXKrE,10144
 langfun/core/eval/v2/example_test.py,sha256=1DNm6EuyZOq827DKvf3oTRVFkMNM_qTnLUpvOjpgz5I,3419
-langfun/core/eval/v2/experiment.py,sha256=lKM3WQck1RWxYznMn1madZqI7w9rpOmu1O35q-KHSC4,32413
-langfun/core/eval/v2/experiment_test.py,sha256=ToxYzQoAbnqzI8kkJH3nHF1vtrac_1yiNbxLS45jx9s,12931
+langfun/core/eval/v2/experiment.py,sha256=qYWx22KMfoUa4ieSq1bt7NE8L9dgoiJpuNOQGT1IBQw,32723
+langfun/core/eval/v2/experiment_test.py,sha256=CqpDsDai2DiIU-SzpVmqFzM_ZxxkVYKd0Gr1Uvcvkuw,13546
 langfun/core/eval/v2/metric_values.py,sha256=_B905bC-jxrYPLSEcP2M8MaHZOVMz_bVrUw8YC4arCE,4660
 langfun/core/eval/v2/metric_values_test.py,sha256=ab2oF_HsIwrSy459108ggyjgefHSPn8UVILR4dRwx14,2634
 langfun/core/eval/v2/metrics.py,sha256=bl8i6u-ZHRBz4hAc3LzsZ2Dc7ZRQcuTYeUhhH-GxfF0,10628
@@ -73,18 +73,18 @@ langfun/core/eval/v2/progress.py,sha256=azZgssQgNdv3IgjKEaQBuGI5ucFDNbdi02P4z_nQ
 langfun/core/eval/v2/progress_test.py,sha256=YU7VHzmy5knPZwj9vpBN3rQQH2tukj9eKHkuBCI62h8,2540
 langfun/core/eval/v2/progress_tracking.py,sha256=l9fEkz4oP5McpZzf72Ua7PYm3lAWtRru7gRWNf8H0ms,6083
 langfun/core/eval/v2/progress_tracking_test.py,sha256=fouMVJkFJqHjbhQJngGLGCmA9x3n0dU4USI2dY163mg,2291
-langfun/core/eval/v2/reporting.py,sha256=cf3jEJbaXKOiGm1UreA9USVGGIp6h8-7E6L3wi76L9I,7676
-langfun/core/eval/v2/reporting_test.py,sha256=IiWLifDi_bV7fze_23lK8IfU5FE4YB3rXBE4jx4s2fw,5698
+langfun/core/eval/v2/reporting.py,sha256=KF4pE2H1qj3mJcgkv_c5YYFjrU-uJCk_-fuu891Olzs,8061
+langfun/core/eval/v2/reporting_test.py,sha256=UmYSAQvD3AIXsSyWQ-WD2uLtEISYpmBeoKY5u5Qwc8E,5696
 langfun/core/eval/v2/runners.py,sha256=DKEmSlGXjOXKWFdBhTpLy7tMsBHZHd1Brl3hWIngsSQ,15931
 langfun/core/eval/v2/runners_test.py,sha256=A37fKK2MvAVTiShsg_laluJzJ9AuAQn52k7HPbfD0Ks,11666
-langfun/core/llms/__init__.py,sha256=ePV13zQL8aLZjYj4YB4YC4xeK_nz0TBv7L2Bm10e3Cw,6574
+langfun/core/llms/__init__.py,sha256=6mi0IKTNfq6kymZPlPGA2V7YF1xDLrBCPytojeFMMeA,6716
 langfun/core/llms/anthropic.py,sha256=a5MmnFsBA0CbfvwzXT1v_0fqLRMrhUNdh1tx6469PQ4,14357
 langfun/core/llms/anthropic_test.py,sha256=-2U4kc_pgBM7wqxu8RuxzyHPGww1EAWqKUvN4PW8Btw,8058
 langfun/core/llms/compositional.py,sha256=csW_FLlgL-tpeyCOTVvfUQkMa_zCN5Y2I-YbSNuK27U,2872
 langfun/core/llms/compositional_test.py,sha256=4eTnOer-DncRKGaIJW2ZQQMLnt5r2R0UIx_DYOvGAQo,2027
 langfun/core/llms/fake.py,sha256=gCHBYBLvBCsC78HI1hpoqXCS-p1FMTgY1P1qh_sGBPk,3070
 langfun/core/llms/fake_test.py,sha256=2h13qkwEz_JR0mtUDPxdAhQo7MueXaFSwsD2DIRDW9g,7653
-langfun/core/llms/google_genai.py,sha256=AAYOsSyeNIfHduIL9ZBzLhA8_acZUDMzHhS7AwUbOlM,11603
+langfun/core/llms/google_genai.py,sha256=3iAmLMcBXxkfiiI8BN0S6trKCfyfuajCIHIGpnCrtTg,11973
 langfun/core/llms/google_genai_test.py,sha256=zw14sgWmk0P_irHyb7vpPy1WAuLEE0PmyfiFElu03sA,7686
 langfun/core/llms/groq.py,sha256=dCnR3eAECEKuKKAAj-PDTs8NRHl6CQPdf57m1f6a79U,10312
 langfun/core/llms/groq_test.py,sha256=GYF_Qtq5S1H1TrKH38t6_lkdroqT7v-joYLDKnmS9e0,5274
@@ -94,7 +94,7 @@ langfun/core/llms/openai.py,sha256=dLDVBB47nJ30XCwjJpAZMc55ZlZXB__PcfcICCRNuXQ,2
 langfun/core/llms/openai_test.py,sha256=kOWa1nf-nJvtYY10REUw5wojh3ZgfU8tRaCZ8wUgJbA,16623
 langfun/core/llms/rest.py,sha256=sWbYUV8S3SuOg9giq7xwD-xDRfaF7NP_ig7bI52-Rj4,3442
 langfun/core/llms/rest_test.py,sha256=NZ3Nf0XQVpT9kLP5cBVo_yBHLI7vWTYhWQxYEJVMGs4,3472
-langfun/core/llms/vertexai.py,sha256=oEd665IBwzCTlHuLEMrCdwgQzrFB5ERcnxw6nrYNSyk,14990
+langfun/core/llms/vertexai.py,sha256=EPPswgaTfPZQ_GGa_dWsqWPV9uRjCmIH2Iwgm1YXOqM,15377
 langfun/core/llms/vertexai_test.py,sha256=ffcA5yPecnQy_rhkuYAw_6o1iLW8AR8FgswmHt6aAys,6725
 langfun/core/llms/cache/__init__.py,sha256=QAo3InUMDM_YpteNnVCSejI4zOsnjSMWKJKzkb3VY64,993
 langfun/core/llms/cache/base.py,sha256=rt3zwmyw0y9jsSGW-ZbV1vAfLxQ7_3AVk0l2EySlse4,3918
@@ -146,8 +146,8 @@ langfun/core/templates/demonstration.py,sha256=vCrgYubdZM5Umqcgp8NUVGXgr4P_c-fik
 langfun/core/templates/demonstration_test.py,sha256=SafcDQ0WgI7pw05EmPI2S4v1t3ABKzup8jReCljHeK4,2162
 langfun/core/templates/selfplay.py,sha256=yhgrJbiYwq47TgzThmHrDQTF4nDrTI09CWGhuQPNv-s,2273
 langfun/core/templates/selfplay_test.py,sha256=Ot__1P1M8oJfoTp-M9-PQ6HUXqZKyMwvZ5f7yQ3yfyM,2326
-langfun-0.1.2.dev202501030804.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-langfun-0.1.2.dev202501030804.dist-info/METADATA,sha256=NHHnAue9xjnH1TBHqjSBdmY2jPVh-LFa0Uipe0_e_OE,8281
-langfun-0.1.2.dev202501030804.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-langfun-0.1.2.dev202501030804.dist-info/top_level.txt,sha256=RhlEkHxs1qtzmmtWSwYoLVJAc1YrbPtxQ52uh8Z9VvY,8
-langfun-0.1.2.dev202501030804.dist-info/RECORD,,
+langfun-0.1.2.dev202501040804.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+langfun-0.1.2.dev202501040804.dist-info/METADATA,sha256=NwyfthRhAHybN6adGkLwsSDi_A-Tj2i6oP7ys-Kln-0,8281
+langfun-0.1.2.dev202501040804.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+langfun-0.1.2.dev202501040804.dist-info/top_level.txt,sha256=RhlEkHxs1qtzmmtWSwYoLVJAc1YrbPtxQ52uh8Z9VvY,8
+langfun-0.1.2.dev202501040804.dist-info/RECORD,,