langfun 0.1.2.dev202410270804__py3-none-any.whl → 0.1.2.dev202410310804__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- langfun/core/concurrent.py +9 -38
- langfun/core/eval/base.py +2 -0
- langfun/core/eval/base_test.py +8 -0
- {langfun-0.1.2.dev202410270804.dist-info → langfun-0.1.2.dev202410310804.dist-info}/METADATA +1 -1
- {langfun-0.1.2.dev202410270804.dist-info → langfun-0.1.2.dev202410310804.dist-info}/RECORD +8 -8
- {langfun-0.1.2.dev202410270804.dist-info → langfun-0.1.2.dev202410310804.dist-info}/WHEEL +1 -1
- {langfun-0.1.2.dev202410270804.dist-info → langfun-0.1.2.dev202410310804.dist-info}/LICENSE +0 -0
- {langfun-0.1.2.dev202410270804.dist-info → langfun-0.1.2.dev202410310804.dist-info}/top_level.txt +0 -0
langfun/core/concurrent.py
CHANGED
@@ -260,6 +260,7 @@ class Job:
|
|
260
260
|
func: Callable[[Any], Any]
|
261
261
|
arg: Any
|
262
262
|
result: Any = pg.MISSING_VALUE
|
263
|
+
error: BaseException | None = None
|
263
264
|
timeit: pg.object_utils.TimeIt = dataclasses.field(
|
264
265
|
default_factory=lambda: pg.object_utils.TimeIt('job')
|
265
266
|
)
|
@@ -269,22 +270,19 @@ class Job:
|
|
269
270
|
"""Returns the running time in seconds since the job get started."""
|
270
271
|
return self.timeit.elapse
|
271
272
|
|
272
|
-
@property
|
273
|
-
def error(self) -> BaseException | None:
|
274
|
-
"""Returns the error if the job failed."""
|
275
|
-
return self.timeit.error
|
276
|
-
|
277
273
|
def __call__(self) -> Any:
|
278
274
|
try:
|
279
275
|
with self.timeit:
|
280
276
|
self.result = self.func(self.arg)
|
281
277
|
return self.result
|
282
278
|
except BaseException as e: # pylint: disable=broad-exception-caught
|
279
|
+
self.error = e
|
283
280
|
return e
|
284
281
|
|
285
282
|
def mark_canceled(self, error: BaseException) -> None:
|
286
283
|
"""Marks the job as canceled."""
|
287
284
|
self.timeit.end(error)
|
285
|
+
self.error = error
|
288
286
|
|
289
287
|
|
290
288
|
@dataclasses.dataclass
|
@@ -292,33 +290,13 @@ class Progress:
|
|
292
290
|
"""Concurrent processing progress."""
|
293
291
|
total: int
|
294
292
|
|
295
|
-
@dataclasses.dataclass
|
296
|
-
class TimeItSummary:
|
297
|
-
"""Execution details for each `pg.timeit`."""
|
298
|
-
|
299
|
-
num_started: int = 0
|
300
|
-
num_ended: int = 0
|
301
|
-
num_failed: int = 0
|
302
|
-
avg_duration: float = 0.0
|
303
|
-
|
304
|
-
def aggregate(self, status: pg.object_utils.TimeIt.Status):
|
305
|
-
self.avg_duration = (
|
306
|
-
(self.avg_duration * self.num_started + status.elapse)
|
307
|
-
/ (self.num_started + 1)
|
308
|
-
)
|
309
|
-
self.num_started += 1
|
310
|
-
if status.has_ended:
|
311
|
-
self.num_ended += 1
|
312
|
-
if status.has_error:
|
313
|
-
self.num_failed += 1
|
314
|
-
|
315
293
|
_succeeded: int = 0
|
316
294
|
_failed: int = 0
|
317
295
|
_last_error: BaseException | None = None
|
318
296
|
_total_duration: float = 0.0
|
319
297
|
_job: Job | None = None
|
320
|
-
_timeit_summary:
|
321
|
-
default_factory=
|
298
|
+
_timeit_summary: pg.object_utils.TimeIt.StatusSummary = dataclasses.field(
|
299
|
+
default_factory=pg.object_utils.TimeIt.StatusSummary
|
322
300
|
)
|
323
301
|
|
324
302
|
@property
|
@@ -368,7 +346,7 @@ class Progress:
|
|
368
346
|
return self._total_duration / self.completed
|
369
347
|
|
370
348
|
@property
|
371
|
-
def timeit_summary(self) ->
|
349
|
+
def timeit_summary(self) -> pg.object_utils.TimeIt.StatusSummary:
|
372
350
|
"""Returns the aggregated summary for each `pg.timeit`."""
|
373
351
|
return self._timeit_summary
|
374
352
|
|
@@ -377,8 +355,8 @@ class Progress:
|
|
377
355
|
return None
|
378
356
|
return ', '.join([
|
379
357
|
'%s (%.2fs, %d/%d)' % (
|
380
|
-
k, v.avg_duration, v.num_ended, v.num_started
|
381
|
-
) for k, v in self.timeit_summary.items()
|
358
|
+
k.lstrip('job.'), v.avg_duration, v.num_ended, v.num_started
|
359
|
+
) for k, v in self.timeit_summary.breakdown.items() if k != 'job'
|
382
360
|
])
|
383
361
|
|
384
362
|
def last_error_str(self) -> str | None:
|
@@ -398,14 +376,7 @@ class Progress:
|
|
398
376
|
self._failed += 1
|
399
377
|
self._last_error = job.error
|
400
378
|
self._total_duration += job.elapse
|
401
|
-
self.
|
402
|
-
|
403
|
-
def merge_timeit_summary(self, job: Job):
|
404
|
-
for child in job.timeit.children:
|
405
|
-
for name, status in child.status().items():
|
406
|
-
if name not in self._timeit_summary:
|
407
|
-
self._timeit_summary[name] = Progress.TimeItSummary()
|
408
|
-
self._timeit_summary[name].aggregate(status)
|
379
|
+
self._timeit_summary.aggregate(job.timeit.status())
|
409
380
|
|
410
381
|
|
411
382
|
class ProgressBar:
|
langfun/core/eval/base.py
CHANGED
@@ -1684,6 +1684,8 @@ def inputs_from(path: str | list[str], **kwargs) -> list[Any]:
|
|
1684
1684
|
if isinstance(path, str):
|
1685
1685
|
if path.endswith('.json'):
|
1686
1686
|
return pg.load(path)
|
1687
|
+
elif path.endswith('.jsonl'):
|
1688
|
+
return list(iter(pg.open_jsonl(path)))
|
1687
1689
|
elif path.endswith('.csv'):
|
1688
1690
|
import pandas as pd # pylint: disable=g-import-not-at-top
|
1689
1691
|
dataset_df = pd.read_csv(path, **kwargs)
|
langfun/core/eval/base_test.py
CHANGED
@@ -599,6 +599,14 @@ class InputsFrom(unittest.TestCase):
|
|
599
599
|
pg.save([1, 2, 3], path)
|
600
600
|
self.assertEqual(base.inputs_from(path)(), [1, 2, 3])
|
601
601
|
|
602
|
+
path = os.path.join(tmp_dir, 'input_file.jsonl')
|
603
|
+
with pg.open_jsonl(path, 'w') as f:
|
604
|
+
f.add(pg.Dict(x=1))
|
605
|
+
f.add(dict(y=2))
|
606
|
+
self.assertEqual(
|
607
|
+
base.inputs_from(path)(), [pg.Dict(x=1), dict(y=2)]
|
608
|
+
)
|
609
|
+
|
602
610
|
def test_inputs_from_multiple_files(self):
|
603
611
|
tmp_dir = tempfile.gettempdir()
|
604
612
|
path1 = os.path.join(tmp_dir, 'input_file1.json')
|
@@ -2,7 +2,7 @@ langfun/__init__.py,sha256=mCES7t3R7Z-ZQYvG38-yrVqZubrXNfGCa8tI5HGB7mE,2274
|
|
2
2
|
langfun/core/__init__.py,sha256=xlvFTXc7IKUTs8aCFRFhzOLTmmeuhXgk9yx2InBLNiA,4937
|
3
3
|
langfun/core/component.py,sha256=kOWdhEYlGw62CO_7aB_oAdivVhnDfyoymRXHr10VtLo,11502
|
4
4
|
langfun/core/component_test.py,sha256=sG-T2wpvBfHqWGZE7sc4NayJj2aj5QFBzSwFiwrGEIc,10376
|
5
|
-
langfun/core/concurrent.py,sha256=
|
5
|
+
langfun/core/concurrent.py,sha256=QMNYhB_PyjvVJtabMokpzotZRYvyE9iYu2QsgwDk7M4,29552
|
6
6
|
langfun/core/concurrent_test.py,sha256=ILlAjfhV84yJfY1QLe3N9aYry1sCjY-ywfIlXGafenI,17336
|
7
7
|
langfun/core/console.py,sha256=bk5rNPNm9rMGW5YT2HixxU04p2umnoabn5SDz6Dqe88,2317
|
8
8
|
langfun/core/console_test.py,sha256=5SYJdxpJGLgdSSQqqMPoA1X6jpsLD8rgcyk-EgI65oE,1077
|
@@ -44,8 +44,8 @@ langfun/core/coding/python/parsing_test.py,sha256=9vAWF484kWIm6JZq8NFiMgKUDhXV-d
|
|
44
44
|
langfun/core/coding/python/permissions.py,sha256=1QWGHvzL8MM0Ok_auQ9tURqZHtdOfJaDpBzZ29GUE-c,2544
|
45
45
|
langfun/core/coding/python/permissions_test.py,sha256=w5EDb8QxpxgJyZkojyzVWQvDfg366zn99-g__6TbPQ0,2699
|
46
46
|
langfun/core/eval/__init__.py,sha256=Ogdr9OtTywhhLPHi3AZzOD2mXX2oyaHWflrSTMm96uA,1899
|
47
|
-
langfun/core/eval/base.py,sha256=
|
48
|
-
langfun/core/eval/base_test.py,sha256
|
47
|
+
langfun/core/eval/base.py,sha256=ajHUS_BdzBIDKEwAmMzne0lJi8HcDhPmyw_plO0p7G0,75814
|
48
|
+
langfun/core/eval/base_test.py,sha256=-LsIV9DXlDal0EnOlaWpibJvfef0NbxtZAm0OH_abAE,27189
|
49
49
|
langfun/core/eval/matching.py,sha256=UnjdM_ebPqXKJamY4lvL3AYxrMIz3LqkjRTnHJ5xsYc,9349
|
50
50
|
langfun/core/eval/matching_test.py,sha256=QCoYEuf4b_1bkHqUCuRzKMbXHrV3AB2FCOBivo1stC4,5249
|
51
51
|
langfun/core/eval/patching.py,sha256=R0s2eAd1m97exQt06dmUL0V_MBG0W2Hxg7fhNB7cXW0,3866
|
@@ -119,8 +119,8 @@ langfun/core/templates/demonstration.py,sha256=vCrgYubdZM5Umqcgp8NUVGXgr4P_c-fik
|
|
119
119
|
langfun/core/templates/demonstration_test.py,sha256=SafcDQ0WgI7pw05EmPI2S4v1t3ABKzup8jReCljHeK4,2162
|
120
120
|
langfun/core/templates/selfplay.py,sha256=yhgrJbiYwq47TgzThmHrDQTF4nDrTI09CWGhuQPNv-s,2273
|
121
121
|
langfun/core/templates/selfplay_test.py,sha256=Ot__1P1M8oJfoTp-M9-PQ6HUXqZKyMwvZ5f7yQ3yfyM,2326
|
122
|
-
langfun-0.1.2.
|
123
|
-
langfun-0.1.2.
|
124
|
-
langfun-0.1.2.
|
125
|
-
langfun-0.1.2.
|
126
|
-
langfun-0.1.2.
|
122
|
+
langfun-0.1.2.dev202410310804.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
|
123
|
+
langfun-0.1.2.dev202410310804.dist-info/METADATA,sha256=FfIoC5x6nfrnCtpk2M9kDCBSQwTJLWC6Ob3C_f_WdDs,8890
|
124
|
+
langfun-0.1.2.dev202410310804.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
|
125
|
+
langfun-0.1.2.dev202410310804.dist-info/top_level.txt,sha256=RhlEkHxs1qtzmmtWSwYoLVJAc1YrbPtxQ52uh8Z9VvY,8
|
126
|
+
langfun-0.1.2.dev202410310804.dist-info/RECORD,,
|
File without changes
|
{langfun-0.1.2.dev202410270804.dist-info → langfun-0.1.2.dev202410310804.dist-info}/top_level.txt
RENAMED
File without changes
|