langfun 0.1.2.dev202410180804__py3-none-any.whl → 0.1.2.dev202410190803__py3-none-any.whl

langfun/core/concurrent.py

@@ -921,5 +921,11 @@ def _progress_control(
     raise ValueError(f'Unsupported progress bar type: {progress_bar}')
 
 
+def get_executor(
+    resource_id: str,
+    max_workers: int | None = None) -> concurrent.futures.ThreadPoolExecutor:
+  """Gets a thread pool executor associated with a resource id."""
+  return _executor_pool.get(resource_id, max_workers)
+
 # The global executor pool based on resource IDs.
 _executor_pool = ExecutorPool()

langfun/core/eval/base.py

@@ -1087,7 +1087,7 @@ class Evaluation(Evaluable):
       )
       error = e
 
-    copy.audit(example, output_message, error, dryrun=True)
+    copy.audit(1, example, output_message, error, dryrun=True)
     result = copy.finalize()
 
     if verbose:
@@ -1124,19 +1124,20 @@ class Evaluation(Evaluable):
     with lf.use_settings(debug=debug, cache=self.cache):
       self._reset()
 
-      def _process(example: Any):
+      def _process(idx_and_example: Any):
         # NOTE(daiyip): set the `input` symbol of the globals to None, so LLM
         # generated code with calls to `input` will raise an error, thus not
         # blocking the evaluation.
+        _, example = idx_and_example
         with lf_coding.context(input=None):
           output_message = self.process(example, **(self.additional_args or {}))
           self.process_output(example, output_message)
           return output_message
 
       try:
-        for example, message, error in lf.concurrent_map(
+        for (idx, example), message, error in lf.concurrent_map(
             _process,
-            examples,
+            enumerate(examples),
             max_workers=self.max_workers,
             show_progress=progress_bar or False,
             status_fn=self._status,
@@ -1148,7 +1149,7 @@ class Evaluation(Evaluable):
                 if isinstance(error, lf_structured.MappingError)
                 else None
             )
-          self.audit(example, message, error)
+          self.audit(idx + 1, example, message, error)
       finally:
         # Save cache upon completion or interruption.
         if self.dir and self.cache:
@@ -1437,6 +1438,7 @@ class Evaluation(Evaluable):
 
   def audit(
       self,
+      example_idx: int,
       example: Any,
       message: lf.Message | None,
       error: Exception | None = None,
@@ -1445,6 +1447,7 @@ class Evaluation(Evaluable):
     """Audits the example against the output. Subclasses should override.
 
     Args:
+      example_idx: 1-based index of the example in its dataset.
       example: The input object.
       message: The entire message returned by the LM, which could be used to
         trace the LM input, response and parsed structure. If error is raised
@@ -1465,7 +1468,7 @@ class Evaluation(Evaluable):
     else:
       assert message is not None
       output = message.text if self.schema is None else message.result
-      self.audit_processed(example, output, message, dryrun=dryrun)
+      self.audit_processed(example_idx, example, output, message, dryrun=dryrun)
 
     # Audit usage.
     if message is not None:
@@ -1482,7 +1485,8 @@ class Evaluation(Evaluable):
         self._num_usages += 1
 
   def audit_processed(
-      self, example: Any, output: Any, message: lf.Message, dryrun: bool = False
+      self, example_idx: int, example: Any, output: Any, message: lf.Message,
+      dryrun: bool = False
   ) -> None:
     """Audits a successfully processed example. Subclass should override."""
 

langfun/core/eval/matching.py

@@ -41,8 +41,8 @@ class Matching(base.Evaluation):
     """Returns the answer from the structure output."""
 
   @property
-  def matches(self) -> list[tuple[Any, Any, lf.Message]]:
-    """Returns the matches examples, outputs and the output messages."""
+  def matches(self) -> list[tuple[int, Any, Any, lf.Message]]:
+    """Returns the matches IDs, examples, outputs and the output messages."""
     return self._matches
 
   @property
@@ -57,7 +57,7 @@ class Matching(base.Evaluation):
     return self.num_matches / self.num_completed
 
   @property
-  def mismatches(self) -> list[tuple[Any, Any, lf.Message]]:
+  def mismatches(self) -> list[tuple[int, Any, Any, lf.Message]]:
     """Returns the mismatches examples, outputs and output messages."""
     return self._mismatches
 
@@ -87,7 +87,8 @@ class Matching(base.Evaluation):
     self._mismatches = []
 
   def audit_processed(
-      self, example: Any, output: Any, message: lf.Message, dryrun: bool = False
+      self, example_idx: int, example: Any, output: Any, message: lf.Message,
+      dryrun: bool = False
   ) -> None:
     groundtruth = self.groundtruth(example)
     answer = self.answer(output, example)
@@ -107,9 +108,9 @@ class Matching(base.Evaluation):
       )
 
     if self.match(answer, groundtruth):
-      self._matches.append((example, output, message))
+      self._matches.append((example_idx, example, output, message))
     else:
-      self._mismatches.append((example, output, message))
+      self._mismatches.append((example_idx, example, output, message))
 
   def match(self, answer: Any, groundtruth: Any) -> bool:
     """Matches answer against the groundtruth. Subclasses can override."""
@@ -247,7 +248,7 @@ class Matching(base.Evaluation):
       # Fall back to the default format.
       return None
 
-    for i, (example, output, message) in enumerate(self.matches):
+    for i, (_, example, output, message) in enumerate(self.matches):
       bgcolor = 'white' if i % 2 == 0 else '#DDDDDD'
       s.write(f'<tr style="background-color: {bgcolor}"><td>{i + 1}</td>')
       input_str = lf.repr_utils.escape_quoted(
@@ -282,7 +283,7 @@ class Matching(base.Evaluation):
         '</tr>'
     )
 
-    for i, (example, output, message) in enumerate(self.mismatches):
+    for i, (_, example, output, message) in enumerate(self.mismatches):
       bgcolor = 'white' if i % 2 == 0 else '#DDDDDD'
       s.write(f'<tr style="background-color: {bgcolor}"><td>{i + 1}</td>')
       input_str = pg.format(example, verbose=False, max_bytes_len=32)

langfun/core/eval/scoring.py

@@ -62,8 +62,10 @@ class Scoring(base.Evaluation):
     self._scored = []
 
   def audit_processed(
-      self, example: Any, output: Any, message: lf.Message, dryrun: bool = False
+      self, example_idx: int, example: Any, output: Any, message: lf.Message,
+      dryrun: bool = False
   ) -> None:
+    del example_idx
     score = self.score(example, output)
 
     if dryrun:

{langfun-0.1.2.dev202410180804.dist-info → langfun-0.1.2.dev202410190803.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: langfun
-Version: 0.1.2.dev202410180804
+Version: 0.1.2.dev202410190803
 Summary: Langfun: Language as Functions.
 Home-page: https://github.com/google/langfun
 Author: Langfun Authors

{langfun-0.1.2.dev202410180804.dist-info → langfun-0.1.2.dev202410190803.dist-info}/RECORD

@@ -2,7 +2,7 @@ langfun/__init__.py,sha256=mCES7t3R7Z-ZQYvG38-yrVqZubrXNfGCa8tI5HGB7mE,2274
 langfun/core/__init__.py,sha256=xlvFTXc7IKUTs8aCFRFhzOLTmmeuhXgk9yx2InBLNiA,4937
 langfun/core/component.py,sha256=kOWdhEYlGw62CO_7aB_oAdivVhnDfyoymRXHr10VtLo,11502
 langfun/core/component_test.py,sha256=sG-T2wpvBfHqWGZE7sc4NayJj2aj5QFBzSwFiwrGEIc,10376
-langfun/core/concurrent.py,sha256=QW-LbgF555lwHYmnirYONVjpqgkn95CuQRuAErgYIIY,29598
+langfun/core/concurrent.py,sha256=ivUwaswRrwldLRZ_p4R3TUy6XqyWY9UdJ8oo2_1Ic0s,29838
 langfun/core/concurrent_test.py,sha256=F9kQKK0D6CHOejckFcVjCB-ThkBN8Oa4P8WV7FOhxIM,17042
 langfun/core/console.py,sha256=bk5rNPNm9rMGW5YT2HixxU04p2umnoabn5SDz6Dqe88,2317
 langfun/core/console_test.py,sha256=5SYJdxpJGLgdSSQqqMPoA1X6jpsLD8rgcyk-EgI65oE,1077
@@ -44,13 +44,13 @@ langfun/core/coding/python/parsing_test.py,sha256=9vAWF484kWIm6JZq8NFiMgKUDhXV-d
 langfun/core/coding/python/permissions.py,sha256=1QWGHvzL8MM0Ok_auQ9tURqZHtdOfJaDpBzZ29GUE-c,2544
 langfun/core/coding/python/permissions_test.py,sha256=w5EDb8QxpxgJyZkojyzVWQvDfg366zn99-g__6TbPQ0,2699
 langfun/core/eval/__init__.py,sha256=Ogdr9OtTywhhLPHi3AZzOD2mXX2oyaHWflrSTMm96uA,1899
-langfun/core/eval/base.py,sha256=3ALt5L51C7jnulLuJMw2uhEvB01d-GNuG-9iMtxM4ic,75230
+langfun/core/eval/base.py,sha256=mF3JUUvuDafNDnt0G_0yv022yGZgTy1NAxDMZonvoV8,75430
 langfun/core/eval/base_test.py,sha256=gAxamZKvHzT3wVr3xVtXLiwBBi5VFX1I29_QuKSR4Wg,26958
-langfun/core/eval/matching.py,sha256=lkI3dtvJe5ql1ppA8Dy6oNVvu5YNtemRos73WQMsgSY,9278
+langfun/core/eval/matching.py,sha256=UnjdM_ebPqXKJamY4lvL3AYxrMIz3LqkjRTnHJ5xsYc,9349
 langfun/core/eval/matching_test.py,sha256=QCoYEuf4b_1bkHqUCuRzKMbXHrV3AB2FCOBivo1stC4,5249
 langfun/core/eval/patching.py,sha256=R0s2eAd1m97exQt06dmUL0V_MBG0W2Hxg7fhNB7cXW0,3866
 langfun/core/eval/patching_test.py,sha256=8kCd54Egjju22FMgtJuxEsrXkW8ifs-UUBHtrCG1L6w,4775
-langfun/core/eval/scoring.py,sha256=wZz90Iw5Sco3cAiA1T71cJEWhD6qmvMeE1Ai-pez_aY,6210
+langfun/core/eval/scoring.py,sha256=SUdMzOkP0n2qGaSuUA4VwFiTw36jgMvgCJHPJS4yYDw,6254
 langfun/core/eval/scoring_test.py,sha256=O8olHbrUEg60gMxwOkWzKBJZpZoUlmVnBANX5Se2SXM,4546
 langfun/core/llms/__init__.py,sha256=baM-YFySnOyijgmMjv1mKPYDvdlny2Md2aZ5bWyxPfc,5859
 langfun/core/llms/anthropic.py,sha256=zPisDmmt6zq-U6hEnoSgReXujmmAj7VGs4KHgzcw1EU,8976
@@ -119,8 +119,8 @@ langfun/core/templates/demonstration.py,sha256=vCrgYubdZM5Umqcgp8NUVGXgr4P_c-fik
 langfun/core/templates/demonstration_test.py,sha256=SafcDQ0WgI7pw05EmPI2S4v1t3ABKzup8jReCljHeK4,2162
 langfun/core/templates/selfplay.py,sha256=yhgrJbiYwq47TgzThmHrDQTF4nDrTI09CWGhuQPNv-s,2273
 langfun/core/templates/selfplay_test.py,sha256=Ot__1P1M8oJfoTp-M9-PQ6HUXqZKyMwvZ5f7yQ3yfyM,2326
-langfun-0.1.2.dev202410180804.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-langfun-0.1.2.dev202410180804.dist-info/METADATA,sha256=8LqnQs02H5_1Bw2RbdEdz5JMYLec6yO04lwkIMHzxE8,8890
-langfun-0.1.2.dev202410180804.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
-langfun-0.1.2.dev202410180804.dist-info/top_level.txt,sha256=RhlEkHxs1qtzmmtWSwYoLVJAc1YrbPtxQ52uh8Z9VvY,8
-langfun-0.1.2.dev202410180804.dist-info/RECORD,,
+langfun-0.1.2.dev202410190803.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+langfun-0.1.2.dev202410190803.dist-info/METADATA,sha256=P6h0k_SY9HTbCZd6foSjU6qj0-YrljfTw2Q97T5WWg0,8890
+langfun-0.1.2.dev202410190803.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
+langfun-0.1.2.dev202410190803.dist-info/top_level.txt,sha256=RhlEkHxs1qtzmmtWSwYoLVJAc1YrbPtxQ52uh8Z9VvY,8
+langfun-0.1.2.dev202410190803.dist-info/RECORD,,