thds.core 1.45.20251002190135__py3-none-any.whl → 1.45.20251006172702__py3-none-any.whl

thds/core/parallel.py

@@ -73,6 +73,9 @@ def yield_all(
     thunks: ty.Iterable[ty.Tuple[H, ty.Callable[[], R]]],
     *,
     executor_cm: ty.Optional[ty.ContextManager[concurrent.futures.Executor]] = None,
+    fmt: ty.Callable[[str], str] = lambda x: x,
+    named: str = "",
+    progress_logger: ty.Callable[[str], ty.Any] = logger.info,
 ) -> ty.Iterator[ty.Tuple[H, ty.Union[R, Error]]]:
     """Stream your results so that you don't have to load them all into memory at the same
     time (necessarily). Also, yield (rather than raise) Exceptions, wrapped as Errors.
@@ -86,6 +89,9 @@ def yield_all(
     files.bump_limits()
     len_or_none = try_len(thunks)
 
+    num_tasks_log = "" if not len_or_none else f" of {len_or_none}"
+    named = f" {named} " if named else " result "
+
     if PARALLEL_OFF() or (len_or_none == 1 and not executor_cm):
         # don't actually transfer this to an executor we only have one task.
         for key, thunk in thunks:
@@ -101,12 +107,16 @@ def yield_all(
     with executor_cm as executor:
         keys_onto_futures = {key: executor.submit(thunk) for key, thunk in thunks}
         future_ids_onto_keys = {id(future): key for key, future in keys_onto_futures.items()}
-        for future in concurrent.futures.as_completed(keys_onto_futures.values()):
+        for i, future in enumerate(concurrent.futures.as_completed(keys_onto_futures.values()), start=1):
             thunk_key = future_ids_onto_keys[id(future)]
             try:
-                yield thunk_key, ty.cast(R, future.result())
+                result = future.result()
+                yielder: tuple[H, ty.Union[R, Error]] = thunk_key, ty.cast(R, result)
             except Exception as e:
-                yield thunk_key, Error(e)
+                yielder = thunk_key, Error(e)
+            finally:
+                progress_logger(fmt(f"Yielding{named}{i}{num_tasks_log}"))
+                yield yielder
 
 
 def failfast(results: ty.Iterable[ty.Tuple[H, ty.Union[R, Error]]]) -> ty.Iterator[ty.Tuple[H, R]]:
@@ -183,6 +193,7 @@ def yield_results(
             )
 
     summarize_exceptions(error_fmt, exceptions)
+    # TODO - when `core` moves to 3.11 start using an ExceptionGroup here
 
 
 def summarize_exceptions(

{thds_core-1.45.20251002190135.dist-info → thds_core-1.45.20251006172702.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: thds.core
-Version: 1.45.20251002190135
+Version: 1.45.20251006172702
 Summary: Core utilities.
 Author-email: Trilliant Health <info@trillianthealth.com>
 License: MIT

{thds_core-1.45.20251002190135.dist-info → thds_core-1.45.20251006172702.dist-info}/RECORD

@@ -29,7 +29,7 @@ thds/core/link.py,sha256=4-9d22l_oSkKoSzlYEO-rwxO1hvvj6VETY7LwvGcX6M,5534
 thds/core/logical_root.py,sha256=gWkIYRv9kNQfzbpxJaYiwNXVz1neZ2NvnvProtOn9d8,1399
 thds/core/merge_args.py,sha256=7oj7dtO1-XVkfTM3aBlq3QlZbo8tb6X7E3EVIR-60t8,5781
 thds/core/meta.py,sha256=Df0DxV5UzHcEsu5UCYaE1BWipMPTEXycn9Ug4cdquMk,12114
-thds/core/parallel.py,sha256=Oee5kFvL9OsqVodctRwVIQMyQ5dUJHrUWhXszVcnqaE,7595
+thds/core/parallel.py,sha256=uYVmdYvD0v3RrQq6YY1o7qPY3KQIPPdzyxZMnFr2DzU,8139
 thds/core/pickle_visit.py,sha256=QNMWIi5buvk2zsvx1-D-FKL7tkrFUFDs387vxgGebgU,833
 thds/core/prof.py,sha256=5ViolfPsAPwUTHuhAe-bon7IArPGXydpGoB5uZmObDk,8264
 thds/core/progress.py,sha256=tY8tc_6CMnu_O8DVisnsRoDpFJOw5vqyYzLhQDxsLn8,4361
@@ -74,8 +74,8 @@ thds/core/sqlite/structured.py,sha256=8t1B6XbM5NnudKEeBLsdjRVbSXXSr6iHOW0HwEAqtX
 thds/core/sqlite/types.py,sha256=Zb1FtSAmC1L7QPPCV1eRnWpJuF74t9R293ai1SHtdS4,1292
 thds/core/sqlite/upsert.py,sha256=BmKK6fsGVedt43iY-Lp7dnAu8aJ1e9CYlPVEQR2pMj4,5827
 thds/core/sqlite/write.py,sha256=z0219vDkQDCnsV0WLvsj94keItr7H4j7Y_evbcoBrWU,3458
-thds_core-1.45.20251002190135.dist-info/METADATA,sha256=-58oJSy31smZPiekf-fPaavBEKxjX-08UUUC1fRYqLk,2216
-thds_core-1.45.20251002190135.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-thds_core-1.45.20251002190135.dist-info/entry_points.txt,sha256=bOCOVhKZv7azF3FvaWX6uxE6yrjK6FcjqhtxXvLiFY8,161
-thds_core-1.45.20251002190135.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
-thds_core-1.45.20251002190135.dist-info/RECORD,,
+thds_core-1.45.20251006172702.dist-info/METADATA,sha256=qFm2dPu7qUFAr2ei5gt2wHG4UHil9xOOIn1IZzbq104,2216
+thds_core-1.45.20251006172702.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+thds_core-1.45.20251006172702.dist-info/entry_points.txt,sha256=bOCOVhKZv7azF3FvaWX6uxE6yrjK6FcjqhtxXvLiFY8,161
+thds_core-1.45.20251006172702.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
+thds_core-1.45.20251006172702.dist-info/RECORD,,