langfun 0.0.2.dev20240201__py3-none-any.whl → 0.0.2.dev20240202__py3-none-any.whl

langfun/core/eval/base.py

@@ -57,6 +57,10 @@ class Evaluable(lf.Component):
       ),
   ] = lf.contextual(default=None)
 
+  report_precision: Annotated[
+      int, 'Number of decimals when reporting precision.'
+  ] = lf.contextual(default=1)
+
   @property
   def dir(self) -> str | None:
     """Returns the directory for saving results and details."""
@@ -1045,12 +1049,12 @@ class Evaluation(Evaluable):
   def _status(self, progress: lf.concurrent.Progress) -> dict[str, Any]:
     return {
         'Model': self.lm.model_id,
-        'Succeeded': '%.2f%% (%d/%d)' % (
+        'Succeeded': f'%.{self.report_precision}f%% (%d/%d)' % (
             progress.success_rate * 100,
             progress.succeeded,
             progress.completed,
         ),
-        'Failed': '%.2f%% (%d/%d)' % (
+        'Failed': f'%.{self.report_precision}f%% (%d/%d)' % (
             progress.failure_rate * 100,
             progress.failed,
             progress.completed,
@@ -1060,14 +1064,18 @@ class Evaluation(Evaluable):
   def _completion_status(self, run_status: str) -> str:
     assert self.result is not None
     m = self.result.metrics
-    return 'COMPLETED(%s): Successes=%.2f%% (%d/%d) Failures=%.2f%% (%d/%d)' % (
-        run_status,
-        (1 - m.failure_rate) * 100,
-        m.total - m.failures,
-        m.total,
-        m.failure_rate * 100,
-        m.failures,
-        m.total,
+    return (
+        f'COMPLETED(%s): Successes=%.{self.report_precision}f%% (%d/%d)'
+        f' Failures=%.{self.report_precision}f%% (%d/%d)'
+        % (
+            run_status,
+            (1 - m.failure_rate) * 100,
+            m.total - m.failures,
+            m.total,
+            m.failure_rate * 100,
+            m.failures,
+            m.total,
+        )
     )
 
   def summarize(self) -> pg.Dict:
@@ -1130,7 +1138,7 @@ class Evaluation(Evaluable):
             m.failures,
             m.total,
             self.failures_link,
-            '%.2f%% ' % (m.failure_rate * 100),
+            f'%.{self.report_precision}f%% ' % (m.failure_rate * 100),
         )
     )
 
@@ -1218,7 +1226,7 @@ class Evaluation(Evaluable):
     s.write(
         '<td><span style="color:orange">%s</span>%s</td>'
         % (
-            '%.2f%%' % (self.failure_rate * 100),
+            f'%.{self.report_precision}f%%' % (self.failure_rate * 100),
             '<a href="%s">(%d/%d)</a>'
             % (self.failures_link, self.num_failures, self.num_completed),
         )

langfun/core/eval/matching.py

@@ -102,17 +102,17 @@ class Matching(base.Evaluation):
     del progress
     return {
         'Model': self.lm.model_id,
-        'Matches': '%.2f%% (%d/%d)' % (
+        'Matches': f'%.{self.report_precision}f%% (%d/%d)' % (
             self.match_rate * 100,
             self.num_matches,
             self.num_completed,
         ),
-        'Mismatches': '%.2f%% (%d/%d)' % (
+        'Mismatches': f'%.{self.report_precision}f%% (%d/%d)' % (
             self.mismatch_rate * 100,
             self.num_mismatches,
             self.num_completed,
         ),
-        'Failed': '%.2f%% (%d/%d)' % (
+        'Failed': f'%.{self.report_precision}f%% (%d/%d)' % (
             self.failure_rate * 100,
             self.num_failures,
             self.num_completed,
@@ -123,8 +123,9 @@ class Matching(base.Evaluation):
     assert self.result is not None
     m = self.result.metrics
     return (
-        'COMPLETED(%s): Matches=%.2f%% (%d/%d) Mismatches=%.2f%% (%d/%d) '
-        'Failures=%.2f%% (%d/%d)'
+        f'COMPLETED(%s): Matches=%.{self.report_precision}f%% (%d/%d)'
+        f' Mismatches=%.{self.report_precision}f%% (%d/%d)'
+        f' Failures=%.{self.report_precision}f%% (%d/%d)'
     ) % (
         run_status,
         m.match_rate * 100,
@@ -202,7 +203,7 @@ class Matching(base.Evaluation):
     s.write(
         '<td><span style="color:red">%s</span>%s</td>'
         % (
-            '%.2f%% ' % (self.mismatch_rate * 100),
+            f'%.{self.report_precision}f%% ' % (self.mismatch_rate * 100),
             '<a href="%s">(%d/%d)</a>'
             % (self.mismatches_link, self.num_mismatches, self.num_completed),
         )
@@ -210,7 +211,7 @@ class Matching(base.Evaluation):
     s.write(
         '<td><span style="color:green">%s</span>%s</td>'
         % (
-            '%.2f%% ' % (self.match_rate * 100),
+            f'%.{self.report_precision}f%% ' % (self.match_rate * 100),
             '<a href="%s">(%d/%d)</a>'
             % (self.matches_link, self.num_matches, self.num_completed),
         )
@@ -226,7 +227,7 @@ class Matching(base.Evaluation):
             m.num_matches,
             m.total,
             self.matches_link,
-            '%.2f%% ' % (m.match_rate * 100),
+            f'%.{self.report_precision}f%% ' % (m.match_rate * 100),
         )
     )
     s.write(' | ')
@@ -236,7 +237,7 @@ class Matching(base.Evaluation):
             m.num_mismatches,
             m.total,
             self.mismatches_link,
-            '%.2f%% ' % (m.mismatch_rate * 100),
+            f'%.{self.report_precision}f%% ' % (m.mismatch_rate * 100),
         )
     )
     s.write(' | ')

langfun/core/llms/openai.py

@@ -44,29 +44,32 @@ SUPPORTED_MODELS_AND_SETTINGS = [
     # Model name, max concurrent requests.
     # The concurrent requests is estimated by TPM/RPM from
     # https://platform.openai.com/account/limits
-    # GPT4 Turbo models.
-    ('gpt-4-1106-preview', 1),  # Gpt4 Turbo.
-    ('gpt-4-vision-preview', 1),  # Gpt4 Turbo with Vision.
-    # GPT4 models.
+    # GPT-4 Turbo models.
+    ('gpt-4-turbo-preview', 1),  # GPT-4 Turbo.
+    ('gpt-4-0125-preview', 1),  # GPT-4 Turbo
+    ('gpt-4-1106-preview', 1),  # GPT-4 Turbo
+    ('gpt-4-vision-preview', 1),  # GPT-4 Turbo with Vision.
+    # GPT-4 models.
     ('gpt-4', 4),
     ('gpt-4-0613', 4),
     ('gpt-4-0314', 4),
     ('gpt-4-32k', 4),
     ('gpt-4-32k-0613', 4),
     ('gpt-4-32k-0314', 4),
-    # GPT3.5 Turbo models.
+    # GPT-3.5 Turbo models.
     ('gpt-3.5-turbo', 16),
+    ('gpt-3.5-turbo-0125', 16),
     ('gpt-3.5-turbo-1106', 16),
     ('gpt-3.5-turbo-0613', 16),
     ('gpt-3.5-turbo-0301', 16),
     ('gpt-3.5-turbo-16k', 16),
     ('gpt-3.5-turbo-16k-0613', 16),
     ('gpt-3.5-turbo-16k-0301', 16),
-    # GPT3.5 models.
-    ('text-davinci-003', 8),  # Gpt3.5, trained with RHLF.
+    # GPT-3.5 models.
+    ('text-davinci-003', 8),  # GPT-3.5, trained with RHLF.
     ('text-davinci-002', 4),  # Trained with SFT but no RHLF.
     ('code-davinci-002', 4),
-    # GPT3 instruction-tuned models.
+    # GPT-3 instruction-tuned models.
     ('text-curie-001', 4),
     ('text-babbage-001', 4),
     ('text-ada-001', 4),
@@ -74,7 +77,7 @@ SUPPORTED_MODELS_AND_SETTINGS = [
     ('curie', 4),
     ('babbage', 4),
     ('ada', 4),
-    # GPT3 base models without instruction tuning.
+    # GPT-3 base models without instruction tuning.
     ('babbage-002', 4),
     ('davinci-002', 4),
 ]

{langfun-0.0.2.dev20240201.dist-info → langfun-0.0.2.dev20240202.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: langfun
-Version: 0.0.2.dev20240201
+Version: 0.0.2.dev20240202
 Summary: Langfun: Language as Functions.
 Home-page: https://github.com/google/langfun
 Author: Langfun Authors

{langfun-0.0.2.dev20240201.dist-info → langfun-0.0.2.dev20240202.dist-info}/RECORD

@@ -40,9 +40,9 @@ langfun/core/coding/python/parsing_test.py,sha256=9vAWF484kWIm6JZq8NFiMgKUDhXV-d
 langfun/core/coding/python/permissions.py,sha256=1QWGHvzL8MM0Ok_auQ9tURqZHtdOfJaDpBzZ29GUE-c,2544
 langfun/core/coding/python/permissions_test.py,sha256=w5EDb8QxpxgJyZkojyzVWQvDfg366zn99-g__6TbPQ0,2699
 langfun/core/eval/__init__.py,sha256=iDA2OcJ3kR6ixZizXIY3N9LsjkaVrfTbSClTiSP8ekY,1291
-langfun/core/eval/base.py,sha256=Wa8kqGQJK2R4pvdCFKB4Mmlcfk4c4qGZn_fl_wy9H-Q,52276
+langfun/core/eval/base.py,sha256=wWFDDrf0jBzs9H_5XfdZSeOBGXyUtXAJJouk7cLckSM,52602
 langfun/core/eval/base_test.py,sha256=bGs3VLchkAJFWYJ8FdR7mC6qoDestAvCHOQpClG6Mzw,21248
-langfun/core/eval/matching.py,sha256=INjG-EU-hQa6zDk2_bAbWWAQnxN4lor-Dnas8i2CTAo,9326
+langfun/core/eval/matching.py,sha256=g2yuBb4FeOlAlB10hqdWvaIg4QVQlJbiViRDcD2Y8go,9567
 langfun/core/eval/matching_test.py,sha256=IfuMF_dEmy4VzK6tIldRzD2Nqlml7SSh4u-baFNcZrw,4912
 langfun/core/eval/scoring.py,sha256=mshqbV_WM0zcp15TSR32ACMBDymlsbf6YH06PPx1Tw0,6139
 langfun/core/eval/scoring_test.py,sha256=_L_B40VZkyI2_PJce-jVKYC4llrO4jGUR5j86Gu6AT0,4046
@@ -53,7 +53,7 @@ langfun/core/llms/gemini.py,sha256=p3d4Cl2uET-os1n_V3YNE6-6cYrZjndj7lxZIk2E8_4,5
 langfun/core/llms/gemini_test.py,sha256=ybNNCn3JW3hYpMe0wT5ILGDrMPaYYU8PN2kSookM0jk,5433
 langfun/core/llms/llama_cpp.py,sha256=EIjJa1-Tg4_VaIxVR88oDWSWc_axc1r2KwSPpl4PSp0,2549
 langfun/core/llms/llama_cpp_test.py,sha256=ZxC6defGd_HX9SFRU9U4cJiQnBKundbOrchbXuC1Z2M,1683
-langfun/core/llms/openai.py,sha256=1wJohy6UZVBiZkbnKsD2Q_qt6-x_9KUnlRALd1TFV9E,10985
+langfun/core/llms/openai.py,sha256=ao2sDDoh5ma1GWpLpNPZARIeLZK55gL1Ldc94h1EGtE,11119
 langfun/core/llms/openai_test.py,sha256=JWcMveifVVVEFWdtmNq1irc9wSFQRxXs-SnOF3Urg9Y,7433
 langfun/core/llms/cache/__init__.py,sha256=QAo3InUMDM_YpteNnVCSejI4zOsnjSMWKJKzkb3VY64,993
 langfun/core/llms/cache/base.py,sha256=cFfYvOIUae842pncqCAsRvqXCk2AnAsRYVx0mcIoAeY,3338
@@ -87,8 +87,8 @@ langfun/core/templates/demonstration.py,sha256=vCrgYubdZM5Umqcgp8NUVGXgr4P_c-fik
 langfun/core/templates/demonstration_test.py,sha256=SafcDQ0WgI7pw05EmPI2S4v1t3ABKzup8jReCljHeK4,2162
 langfun/core/templates/selfplay.py,sha256=yhgrJbiYwq47TgzThmHrDQTF4nDrTI09CWGhuQPNv-s,2273
 langfun/core/templates/selfplay_test.py,sha256=ZkDfwiW9OtO_MOIdVTRPn6P6vOExQIszqlVQHg5iD3U,2066
-langfun-0.0.2.dev20240201.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-langfun-0.0.2.dev20240201.dist-info/METADATA,sha256=BRcyCLLZnV3G5PLpO7ik2hVlnF0xkm7HQX5ckgyiXN8,3368
-langfun-0.0.2.dev20240201.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-langfun-0.0.2.dev20240201.dist-info/top_level.txt,sha256=RhlEkHxs1qtzmmtWSwYoLVJAc1YrbPtxQ52uh8Z9VvY,8
-langfun-0.0.2.dev20240201.dist-info/RECORD,,
+langfun-0.0.2.dev20240202.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+langfun-0.0.2.dev20240202.dist-info/METADATA,sha256=YNH_28StpsuvajP3UyX4a4-RjS3ErqeooJE_l55L1cQ,3368
+langfun-0.0.2.dev20240202.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+langfun-0.0.2.dev20240202.dist-info/top_level.txt,sha256=RhlEkHxs1qtzmmtWSwYoLVJAc1YrbPtxQ52uh8Z9VvY,8
+langfun-0.0.2.dev20240202.dist-info/RECORD,,