thds.core 1.31.20250127214246__py3-none-any.whl → 1.31.20250128233053__py3-none-any.whl

thds/core/meta.json

@@ -1,8 +1,8 @@
 {
-  "git_commit": "5cc4eb34e4c8c668ebd2734dcdda84ca26519c08",
+  "git_commit": "7f3310b958fceeb2a3b25bd3ac33c42cc3f70672",
   "git_branch": "main",
   "git_is_clean": true,
-  "pyproject_version": "1.31.20250127214246",
+  "pyproject_version": "1.31.20250128233053",
   "thds_user": "runner",
   "misc": {}
 }

thds/core/parallel.py

@@ -3,6 +3,7 @@
 
 import concurrent.futures
 import itertools
+import traceback
 import typing as ty
 from collections import defaultdict
 from dataclasses import dataclass
@@ -112,6 +113,9 @@ def yield_all(
 
 
 def failfast(results: ty.Iterable[ty.Tuple[H, ty.Union[R, Error]]]) -> ty.Iterator[ty.Tuple[H, R]]:
+    """Use in conjunction with `yield_all` to run things in parallel but to exit at the first sign
+    of failure. More appropriate for small pipeline stages.
+    """
     for key, res in results:
         if isinstance(res, Error):
             raise res.error
@@ -172,7 +176,14 @@ def yield_results(
             yield res
         else:
             exceptions.append(res.error)
-            logger.exception(error_fmt(f"Task {i}{num_tasks_log} errored with {str(res.error)}"))
+            # print tracebacks as we go, so as not to defer potentially-helpful
+            # debugging information while a long run is ongoing.
+            traceback.print_exception(type(res.error), res.error, res.error.__traceback__)
+            logger.error(  # should only use logger.exception from an except block
+                error_fmt(
+                    f"Task {i}{num_tasks_log} errored with {type(res.error).__name__}({res.error})"
+                )
+            )
 
     summarize_exceptions(error_fmt, exceptions)
 
@@ -182,16 +193,16 @@ def summarize_exceptions(
     exceptions: ty.List[Exception],
 ) -> None:
     if exceptions:
-        # summarize them and raise a final exception
+        # group by type
         by_type = defaultdict(list)
         for exc in exceptions:
             by_type[type(exc)].append(exc)
-            logger.error(error_fmt("EXCEPTION"), exc_info=(type(exc), exc, exc.__traceback__))
 
+        # log the count for each Exception type
         most_common_type = None
         max_count = 0
         for _type, excs in by_type.items():
-            logger.error(error_fmt(f"{len(excs)} tasks failed with exception: " + str(_type)))
+            logger.error(error_fmt(f"{len(excs)} tasks failed with exception: " + _type.__name__))
             if len(excs) > max_count:
                 max_count = len(excs)
                 most_common_type = _type

{thds.core-1.31.20250127214246.dist-info → thds.core-1.31.20250128233053.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: thds.core
-Version: 1.31.20250127214246
+Version: 1.31.20250128233053
 Summary: Core utilities.
 Author: Trilliant Health
 Description-Content-Type: text/markdown

{thds.core-1.31.20250127214246.dist-info → thds.core-1.31.20250128233053.dist-info}/RECORD

@@ -22,9 +22,9 @@ thds/core/iterators.py,sha256=d3iTQDR0gCW1nMRmknQeodR_4THzR9Ajmp8F8KCCFgg,208
 thds/core/lazy.py,sha256=e1WvG4LsbEydV0igEr_Vl1cq05zlQNIE8MFYT90yglE,3289
 thds/core/link.py,sha256=kmFJIFvEZc16-7S7IGvtTpzwl3VuvFl3yPlE6WJJ03w,5404
 thds/core/merge_args.py,sha256=7oj7dtO1-XVkfTM3aBlq3QlZbo8tb6X7E3EVIR-60t8,5781
-thds/core/meta.json,sha256=9XulISJdzvYUNkfK0XigSu9RbMvSml0XLyNO5oduoqQ,196
+thds/core/meta.json,sha256=gkLuXVLcI2acVfgpF2lYUzENQZKleKvTJeb-8VDp3Lo,196
 thds/core/meta.py,sha256=IPLAKrH06HooPMNf5FeqJvUcM-JljTGXddrAQ5oAX8E,16896
-thds/core/parallel.py,sha256=lUzM0CRXsfzZgMMOHcwP-Th7bbXAqfl7FEGwdwAoBiM,7098
+thds/core/parallel.py,sha256=HXAn9aIYqNE5rnRN5ypxR6CUucdfzE5T5rJ_MUv-pFk,7590
 thds/core/pickle_visit.py,sha256=QNMWIi5buvk2zsvx1-D-FKL7tkrFUFDs387vxgGebgU,833
 thds/core/prof.py,sha256=atDz7rDxzMzbKwgM48tics2tB5sBijKb_L5heAT6Pk8,8246
 thds/core/progress.py,sha256=4YGbxliDl1i-k-88w4s86uy1E69eQ6xJySGPSkpH1QM,3358
@@ -60,8 +60,8 @@ thds/core/sqlite/structured.py,sha256=swCbDoyVT6cE7Kl79Wh_rg5Z1-yrUDJbiVJF4bjset
 thds/core/sqlite/types.py,sha256=oUkfoKRYNGDPZRk29s09rc9ha3SCk2SKr_K6WKebBFs,1308
 thds/core/sqlite/upsert.py,sha256=BmKK6fsGVedt43iY-Lp7dnAu8aJ1e9CYlPVEQR2pMj4,5827
 thds/core/sqlite/write.py,sha256=h-tld6sC1alkaF92KRdrjyrEOwCt3bZ804muOcV4MEw,3141
-thds.core-1.31.20250127214246.dist-info/METADATA,sha256=B4x41JmMXI1p7T__6daXdDFnlk3EsLCkRNKxRc7P8pA,2123
-thds.core-1.31.20250127214246.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-thds.core-1.31.20250127214246.dist-info/entry_points.txt,sha256=bOCOVhKZv7azF3FvaWX6uxE6yrjK6FcjqhtxXvLiFY8,161
-thds.core-1.31.20250127214246.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
-thds.core-1.31.20250127214246.dist-info/RECORD,,
+thds.core-1.31.20250128233053.dist-info/METADATA,sha256=1uqpGMht5pzq7Pi3lqhZ9wqSjX-AXGn-cTKYYFcHELY,2123
+thds.core-1.31.20250128233053.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+thds.core-1.31.20250128233053.dist-info/entry_points.txt,sha256=bOCOVhKZv7azF3FvaWX6uxE6yrjK6FcjqhtxXvLiFY8,161
+thds.core-1.31.20250128233053.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
+thds.core-1.31.20250128233053.dist-info/RECORD,,