langfun 0.1.2.dev202410270804__py3-none-any.whl → 0.1.2.dev202410310804__py3-none-any.whl

langfun/core/concurrent.py

@@ -260,6 +260,7 @@ class Job:
   func: Callable[[Any], Any]
   arg: Any
   result: Any = pg.MISSING_VALUE
+  error: BaseException | None = None
   timeit: pg.object_utils.TimeIt = dataclasses.field(
       default_factory=lambda: pg.object_utils.TimeIt('job')
   )
@@ -269,22 +270,19 @@ class Job:
     """Returns the running time in seconds since the job get started."""
     return self.timeit.elapse
 
-  @property
-  def error(self) -> BaseException | None:
-    """Returns the error if the job failed."""
-    return self.timeit.error
-
   def __call__(self) -> Any:
     try:
       with self.timeit:
         self.result = self.func(self.arg)
         return self.result
     except BaseException as e:  # pylint: disable=broad-exception-caught
+      self.error = e
       return e
 
   def mark_canceled(self, error: BaseException) -> None:
     """Marks the job as canceled."""
     self.timeit.end(error)
+    self.error = error
 
 
 @dataclasses.dataclass
@@ -292,33 +290,13 @@ class Progress:
   """Concurrent processing progress."""
   total: int
 
-  @dataclasses.dataclass
-  class TimeItSummary:
-    """Execution details for each `pg.timeit`."""
-
-    num_started: int = 0
-    num_ended: int = 0
-    num_failed: int = 0
-    avg_duration: float = 0.0
-
-    def aggregate(self, status: pg.object_utils.TimeIt.Status):
-      self.avg_duration = (
-          (self.avg_duration * self.num_started + status.elapse)
-          / (self.num_started + 1)
-      )
-      self.num_started += 1
-      if status.has_ended:
-        self.num_ended += 1
-      if status.has_error:
-        self.num_failed += 1
-
   _succeeded: int = 0
   _failed: int = 0
   _last_error: BaseException | None = None
   _total_duration: float = 0.0
   _job: Job | None = None
-  _timeit_summary: dict[str, TimeItSummary] = dataclasses.field(
-      default_factory=dict
+  _timeit_summary: pg.object_utils.TimeIt.StatusSummary = dataclasses.field(
+      default_factory=pg.object_utils.TimeIt.StatusSummary
   )
 
   @property
@@ -368,7 +346,7 @@ class Progress:
     return self._total_duration / self.completed
 
   @property
-  def timeit_summary(self) -> dict[str, TimeItSummary]:
+  def timeit_summary(self) -> pg.object_utils.TimeIt.StatusSummary:
     """Returns the aggregated summary for each `pg.timeit`."""
     return self._timeit_summary
 
@@ -377,8 +355,8 @@ class Progress:
       return None
     return ', '.join([
         '%s (%.2fs, %d/%d)' % (
-            k, v.avg_duration, v.num_ended, v.num_started
-        ) for k, v in self.timeit_summary.items()
+            k.lstrip('job.'), v.avg_duration, v.num_ended, v.num_started
+        ) for k, v in self.timeit_summary.breakdown.items() if k != 'job'
     ])
 
   def last_error_str(self) -> str | None:
@@ -398,14 +376,7 @@ class Progress:
       self._failed += 1
       self._last_error = job.error
     self._total_duration += job.elapse
-    self.merge_timeit_summary(job)
-
-  def merge_timeit_summary(self, job: Job):
-    for child in job.timeit.children:
-      for name, status in child.status().items():
-        if name not in self._timeit_summary:
-          self._timeit_summary[name] = Progress.TimeItSummary()
-        self._timeit_summary[name].aggregate(status)
+    self._timeit_summary.aggregate(job.timeit.status())
 
 
 class ProgressBar:

langfun/core/eval/base.py

@@ -1684,6 +1684,8 @@ def inputs_from(path: str | list[str], **kwargs) -> list[Any]:
   if isinstance(path, str):
     if path.endswith('.json'):
       return pg.load(path)
+    elif path.endswith('.jsonl'):
+      return list(iter(pg.open_jsonl(path)))
     elif path.endswith('.csv'):
       import pandas as pd  # pylint: disable=g-import-not-at-top
       dataset_df = pd.read_csv(path, **kwargs)

langfun/core/eval/base_test.py

@@ -599,6 +599,14 @@ class InputsFrom(unittest.TestCase):
     pg.save([1, 2, 3], path)
     self.assertEqual(base.inputs_from(path)(), [1, 2, 3])
 
+    path = os.path.join(tmp_dir, 'input_file.jsonl')
+    with pg.open_jsonl(path, 'w') as f:
+      f.add(pg.Dict(x=1))
+      f.add(dict(y=2))
+    self.assertEqual(
+        base.inputs_from(path)(), [pg.Dict(x=1), dict(y=2)]
+    )
+
   def test_inputs_from_multiple_files(self):
     tmp_dir = tempfile.gettempdir()
     path1 = os.path.join(tmp_dir, 'input_file1.json')

{langfun-0.1.2.dev202410270804.dist-info → langfun-0.1.2.dev202410310804.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: langfun
-Version: 0.1.2.dev202410270804
+Version: 0.1.2.dev202410310804
 Summary: Langfun: Language as Functions.
 Home-page: https://github.com/google/langfun
 Author: Langfun Authors

{langfun-0.1.2.dev202410270804.dist-info → langfun-0.1.2.dev202410310804.dist-info}/RECORD

@@ -2,7 +2,7 @@ langfun/__init__.py,sha256=mCES7t3R7Z-ZQYvG38-yrVqZubrXNfGCa8tI5HGB7mE,2274
 langfun/core/__init__.py,sha256=xlvFTXc7IKUTs8aCFRFhzOLTmmeuhXgk9yx2InBLNiA,4937
 langfun/core/component.py,sha256=kOWdhEYlGw62CO_7aB_oAdivVhnDfyoymRXHr10VtLo,11502
 langfun/core/component_test.py,sha256=sG-T2wpvBfHqWGZE7sc4NayJj2aj5QFBzSwFiwrGEIc,10376
-langfun/core/concurrent.py,sha256=8L-fRc9x5PtrnuHJSEOPiKG4HUsQ2xWDt_1Ws1uGJRE,30324
+langfun/core/concurrent.py,sha256=QMNYhB_PyjvVJtabMokpzotZRYvyE9iYu2QsgwDk7M4,29552
 langfun/core/concurrent_test.py,sha256=ILlAjfhV84yJfY1QLe3N9aYry1sCjY-ywfIlXGafenI,17336
 langfun/core/console.py,sha256=bk5rNPNm9rMGW5YT2HixxU04p2umnoabn5SDz6Dqe88,2317
 langfun/core/console_test.py,sha256=5SYJdxpJGLgdSSQqqMPoA1X6jpsLD8rgcyk-EgI65oE,1077
@@ -44,8 +44,8 @@ langfun/core/coding/python/parsing_test.py,sha256=9vAWF484kWIm6JZq8NFiMgKUDhXV-d
 langfun/core/coding/python/permissions.py,sha256=1QWGHvzL8MM0Ok_auQ9tURqZHtdOfJaDpBzZ29GUE-c,2544
 langfun/core/coding/python/permissions_test.py,sha256=w5EDb8QxpxgJyZkojyzVWQvDfg366zn99-g__6TbPQ0,2699
 langfun/core/eval/__init__.py,sha256=Ogdr9OtTywhhLPHi3AZzOD2mXX2oyaHWflrSTMm96uA,1899
-langfun/core/eval/base.py,sha256=7CPWykFZB4-1a8rMBrE4XNAtRQ9w_O1T3bKeg7ykkw4,75735
-langfun/core/eval/base_test.py,sha256=gAxamZKvHzT3wVr3xVtXLiwBBi5VFX1I29_QuKSR4Wg,26958
+langfun/core/eval/base.py,sha256=ajHUS_BdzBIDKEwAmMzne0lJi8HcDhPmyw_plO0p7G0,75814
+langfun/core/eval/base_test.py,sha256=-LsIV9DXlDal0EnOlaWpibJvfef0NbxtZAm0OH_abAE,27189
 langfun/core/eval/matching.py,sha256=UnjdM_ebPqXKJamY4lvL3AYxrMIz3LqkjRTnHJ5xsYc,9349
 langfun/core/eval/matching_test.py,sha256=QCoYEuf4b_1bkHqUCuRzKMbXHrV3AB2FCOBivo1stC4,5249
 langfun/core/eval/patching.py,sha256=R0s2eAd1m97exQt06dmUL0V_MBG0W2Hxg7fhNB7cXW0,3866
@@ -119,8 +119,8 @@ langfun/core/templates/demonstration.py,sha256=vCrgYubdZM5Umqcgp8NUVGXgr4P_c-fik
 langfun/core/templates/demonstration_test.py,sha256=SafcDQ0WgI7pw05EmPI2S4v1t3ABKzup8jReCljHeK4,2162
 langfun/core/templates/selfplay.py,sha256=yhgrJbiYwq47TgzThmHrDQTF4nDrTI09CWGhuQPNv-s,2273
 langfun/core/templates/selfplay_test.py,sha256=Ot__1P1M8oJfoTp-M9-PQ6HUXqZKyMwvZ5f7yQ3yfyM,2326
-langfun-0.1.2.dev202410270804.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-langfun-0.1.2.dev202410270804.dist-info/METADATA,sha256=P6bilCfUTSSlbJKbHNVnbZcd-9U4Gc1qhlIdt04olfk,8890
-langfun-0.1.2.dev202410270804.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
-langfun-0.1.2.dev202410270804.dist-info/top_level.txt,sha256=RhlEkHxs1qtzmmtWSwYoLVJAc1YrbPtxQ52uh8Z9VvY,8
-langfun-0.1.2.dev202410270804.dist-info/RECORD,,
+langfun-0.1.2.dev202410310804.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+langfun-0.1.2.dev202410310804.dist-info/METADATA,sha256=FfIoC5x6nfrnCtpk2M9kDCBSQwTJLWC6Ob3C_f_WdDs,8890
+langfun-0.1.2.dev202410310804.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
+langfun-0.1.2.dev202410310804.dist-info/top_level.txt,sha256=RhlEkHxs1qtzmmtWSwYoLVJAc1YrbPtxQ52uh8Z9VvY,8
+langfun-0.1.2.dev202410310804.dist-info/RECORD,,

{langfun-0.1.2.dev202410270804.dist-info → langfun-0.1.2.dev202410310804.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.2.0)
+Generator: setuptools (75.3.0)
 Root-Is-Purelib: true
 Tag: py3-none-any