langfun 0.1.2.dev202410270804__py3-none-any.whl → 0.1.2.dev202410310804__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -260,6 +260,7 @@ class Job:
260
260
  func: Callable[[Any], Any]
261
261
  arg: Any
262
262
  result: Any = pg.MISSING_VALUE
263
+ error: BaseException | None = None
263
264
  timeit: pg.object_utils.TimeIt = dataclasses.field(
264
265
  default_factory=lambda: pg.object_utils.TimeIt('job')
265
266
  )
@@ -269,22 +270,19 @@ class Job:
269
270
  """Returns the running time in seconds since the job get started."""
270
271
  return self.timeit.elapse
271
272
 
272
- @property
273
- def error(self) -> BaseException | None:
274
- """Returns the error if the job failed."""
275
- return self.timeit.error
276
-
277
273
  def __call__(self) -> Any:
278
274
  try:
279
275
  with self.timeit:
280
276
  self.result = self.func(self.arg)
281
277
  return self.result
282
278
  except BaseException as e: # pylint: disable=broad-exception-caught
279
+ self.error = e
283
280
  return e
284
281
 
285
282
  def mark_canceled(self, error: BaseException) -> None:
286
283
  """Marks the job as canceled."""
287
284
  self.timeit.end(error)
285
+ self.error = error
288
286
 
289
287
 
290
288
  @dataclasses.dataclass
@@ -292,33 +290,13 @@ class Progress:
292
290
  """Concurrent processing progress."""
293
291
  total: int
294
292
 
295
- @dataclasses.dataclass
296
- class TimeItSummary:
297
- """Execution details for each `pg.timeit`."""
298
-
299
- num_started: int = 0
300
- num_ended: int = 0
301
- num_failed: int = 0
302
- avg_duration: float = 0.0
303
-
304
- def aggregate(self, status: pg.object_utils.TimeIt.Status):
305
- self.avg_duration = (
306
- (self.avg_duration * self.num_started + status.elapse)
307
- / (self.num_started + 1)
308
- )
309
- self.num_started += 1
310
- if status.has_ended:
311
- self.num_ended += 1
312
- if status.has_error:
313
- self.num_failed += 1
314
-
315
293
  _succeeded: int = 0
316
294
  _failed: int = 0
317
295
  _last_error: BaseException | None = None
318
296
  _total_duration: float = 0.0
319
297
  _job: Job | None = None
320
- _timeit_summary: dict[str, TimeItSummary] = dataclasses.field(
321
- default_factory=dict
298
+ _timeit_summary: pg.object_utils.TimeIt.StatusSummary = dataclasses.field(
299
+ default_factory=pg.object_utils.TimeIt.StatusSummary
322
300
  )
323
301
 
324
302
  @property
@@ -368,7 +346,7 @@ class Progress:
368
346
  return self._total_duration / self.completed
369
347
 
370
348
  @property
371
- def timeit_summary(self) -> dict[str, TimeItSummary]:
349
+ def timeit_summary(self) -> pg.object_utils.TimeIt.StatusSummary:
372
350
  """Returns the aggregated summary for each `pg.timeit`."""
373
351
  return self._timeit_summary
374
352
 
@@ -377,8 +355,8 @@ class Progress:
377
355
  return None
378
356
  return ', '.join([
379
357
  '%s (%.2fs, %d/%d)' % (
380
- k, v.avg_duration, v.num_ended, v.num_started
381
- ) for k, v in self.timeit_summary.items()
358
+ k.lstrip('job.'), v.avg_duration, v.num_ended, v.num_started
359
+ ) for k, v in self.timeit_summary.breakdown.items() if k != 'job'
382
360
  ])
383
361
 
384
362
  def last_error_str(self) -> str | None:
@@ -398,14 +376,7 @@ class Progress:
398
376
  self._failed += 1
399
377
  self._last_error = job.error
400
378
  self._total_duration += job.elapse
401
- self.merge_timeit_summary(job)
402
-
403
- def merge_timeit_summary(self, job: Job):
404
- for child in job.timeit.children:
405
- for name, status in child.status().items():
406
- if name not in self._timeit_summary:
407
- self._timeit_summary[name] = Progress.TimeItSummary()
408
- self._timeit_summary[name].aggregate(status)
379
+ self._timeit_summary.aggregate(job.timeit.status())
409
380
 
410
381
 
411
382
  class ProgressBar:
langfun/core/eval/base.py CHANGED
@@ -1684,6 +1684,8 @@ def inputs_from(path: str | list[str], **kwargs) -> list[Any]:
1684
1684
  if isinstance(path, str):
1685
1685
  if path.endswith('.json'):
1686
1686
  return pg.load(path)
1687
+ elif path.endswith('.jsonl'):
1688
+ return list(iter(pg.open_jsonl(path)))
1687
1689
  elif path.endswith('.csv'):
1688
1690
  import pandas as pd # pylint: disable=g-import-not-at-top
1689
1691
  dataset_df = pd.read_csv(path, **kwargs)
@@ -599,6 +599,14 @@ class InputsFrom(unittest.TestCase):
599
599
  pg.save([1, 2, 3], path)
600
600
  self.assertEqual(base.inputs_from(path)(), [1, 2, 3])
601
601
 
602
+ path = os.path.join(tmp_dir, 'input_file.jsonl')
603
+ with pg.open_jsonl(path, 'w') as f:
604
+ f.add(pg.Dict(x=1))
605
+ f.add(dict(y=2))
606
+ self.assertEqual(
607
+ base.inputs_from(path)(), [pg.Dict(x=1), dict(y=2)]
608
+ )
609
+
602
610
  def test_inputs_from_multiple_files(self):
603
611
  tmp_dir = tempfile.gettempdir()
604
612
  path1 = os.path.join(tmp_dir, 'input_file1.json')
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: langfun
3
- Version: 0.1.2.dev202410270804
3
+ Version: 0.1.2.dev202410310804
4
4
  Summary: Langfun: Language as Functions.
5
5
  Home-page: https://github.com/google/langfun
6
6
  Author: Langfun Authors
@@ -2,7 +2,7 @@ langfun/__init__.py,sha256=mCES7t3R7Z-ZQYvG38-yrVqZubrXNfGCa8tI5HGB7mE,2274
2
2
  langfun/core/__init__.py,sha256=xlvFTXc7IKUTs8aCFRFhzOLTmmeuhXgk9yx2InBLNiA,4937
3
3
  langfun/core/component.py,sha256=kOWdhEYlGw62CO_7aB_oAdivVhnDfyoymRXHr10VtLo,11502
4
4
  langfun/core/component_test.py,sha256=sG-T2wpvBfHqWGZE7sc4NayJj2aj5QFBzSwFiwrGEIc,10376
5
- langfun/core/concurrent.py,sha256=8L-fRc9x5PtrnuHJSEOPiKG4HUsQ2xWDt_1Ws1uGJRE,30324
5
+ langfun/core/concurrent.py,sha256=QMNYhB_PyjvVJtabMokpzotZRYvyE9iYu2QsgwDk7M4,29552
6
6
  langfun/core/concurrent_test.py,sha256=ILlAjfhV84yJfY1QLe3N9aYry1sCjY-ywfIlXGafenI,17336
7
7
  langfun/core/console.py,sha256=bk5rNPNm9rMGW5YT2HixxU04p2umnoabn5SDz6Dqe88,2317
8
8
  langfun/core/console_test.py,sha256=5SYJdxpJGLgdSSQqqMPoA1X6jpsLD8rgcyk-EgI65oE,1077
@@ -44,8 +44,8 @@ langfun/core/coding/python/parsing_test.py,sha256=9vAWF484kWIm6JZq8NFiMgKUDhXV-d
44
44
  langfun/core/coding/python/permissions.py,sha256=1QWGHvzL8MM0Ok_auQ9tURqZHtdOfJaDpBzZ29GUE-c,2544
45
45
  langfun/core/coding/python/permissions_test.py,sha256=w5EDb8QxpxgJyZkojyzVWQvDfg366zn99-g__6TbPQ0,2699
46
46
  langfun/core/eval/__init__.py,sha256=Ogdr9OtTywhhLPHi3AZzOD2mXX2oyaHWflrSTMm96uA,1899
47
- langfun/core/eval/base.py,sha256=7CPWykFZB4-1a8rMBrE4XNAtRQ9w_O1T3bKeg7ykkw4,75735
48
- langfun/core/eval/base_test.py,sha256=gAxamZKvHzT3wVr3xVtXLiwBBi5VFX1I29_QuKSR4Wg,26958
47
+ langfun/core/eval/base.py,sha256=ajHUS_BdzBIDKEwAmMzne0lJi8HcDhPmyw_plO0p7G0,75814
48
+ langfun/core/eval/base_test.py,sha256=-LsIV9DXlDal0EnOlaWpibJvfef0NbxtZAm0OH_abAE,27189
49
49
  langfun/core/eval/matching.py,sha256=UnjdM_ebPqXKJamY4lvL3AYxrMIz3LqkjRTnHJ5xsYc,9349
50
50
  langfun/core/eval/matching_test.py,sha256=QCoYEuf4b_1bkHqUCuRzKMbXHrV3AB2FCOBivo1stC4,5249
51
51
  langfun/core/eval/patching.py,sha256=R0s2eAd1m97exQt06dmUL0V_MBG0W2Hxg7fhNB7cXW0,3866
@@ -119,8 +119,8 @@ langfun/core/templates/demonstration.py,sha256=vCrgYubdZM5Umqcgp8NUVGXgr4P_c-fik
119
119
  langfun/core/templates/demonstration_test.py,sha256=SafcDQ0WgI7pw05EmPI2S4v1t3ABKzup8jReCljHeK4,2162
120
120
  langfun/core/templates/selfplay.py,sha256=yhgrJbiYwq47TgzThmHrDQTF4nDrTI09CWGhuQPNv-s,2273
121
121
  langfun/core/templates/selfplay_test.py,sha256=Ot__1P1M8oJfoTp-M9-PQ6HUXqZKyMwvZ5f7yQ3yfyM,2326
122
- langfun-0.1.2.dev202410270804.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
123
- langfun-0.1.2.dev202410270804.dist-info/METADATA,sha256=P6bilCfUTSSlbJKbHNVnbZcd-9U4Gc1qhlIdt04olfk,8890
124
- langfun-0.1.2.dev202410270804.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
125
- langfun-0.1.2.dev202410270804.dist-info/top_level.txt,sha256=RhlEkHxs1qtzmmtWSwYoLVJAc1YrbPtxQ52uh8Z9VvY,8
126
- langfun-0.1.2.dev202410270804.dist-info/RECORD,,
122
+ langfun-0.1.2.dev202410310804.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
123
+ langfun-0.1.2.dev202410310804.dist-info/METADATA,sha256=FfIoC5x6nfrnCtpk2M9kDCBSQwTJLWC6Ob3C_f_WdDs,8890
124
+ langfun-0.1.2.dev202410310804.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
125
+ langfun-0.1.2.dev202410310804.dist-info/top_level.txt,sha256=RhlEkHxs1qtzmmtWSwYoLVJAc1YrbPtxQ52uh8Z9VvY,8
126
+ langfun-0.1.2.dev202410310804.dist-info/RECORD,,
@@ -1,5 +1,5 @@
1
1
  Wheel-Version: 1.0
2
- Generator: setuptools (75.2.0)
2
+ Generator: setuptools (75.3.0)
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
5
5