thds.core 1.47.20251024053252__py3-none-any.whl → 1.47.20251024204437__py3-none-any.whl

thds/core/inspect.py

@@ -1,6 +1,7 @@
 import inspect
 import typing as ty
 from dataclasses import dataclass
+from types import ModuleType
 
 
 @dataclass(frozen=True)
@@ -32,7 +33,7 @@ def get_caller_info(skip: int = 2) -> CallerInfo:
     start = 0 + skip
     if len(stack) < start + 1:
         raise RuntimeError(f"The stack has less than f{skip} + 1 frames in it.")
-    parentframe = stack[start][0]
+    parentframe = stack[start].frame
 
     # full dotted name of caller module
     module_info = inspect.getmodule(parentframe)
@@ -68,3 +69,27 @@ def bind_arguments(
 
 def get_argument(arg_name: str, bound_arguments: inspect.BoundArguments) -> ty.Any:
     return bound_arguments.arguments[arg_name]
+
+
+def yield_caller_modules_and_frames(*skip: str) -> ty.Iterator[tuple[ModuleType, inspect.FrameInfo]]:
+    """Yields caller modules and their frame info, skipping any modules in the skip list."""
+    stack = inspect.stack()
+    skip = set(skip) | {__name__}  # type: ignore
+    for frame_info in stack[1:]:  # don't bother with the current frame, obviously
+        module = inspect.getmodule(frame_info.frame)
+        if module:
+            module_name = module.__name__
+            if module_name not in skip:
+                yield module, frame_info
+
+
+def caller_module_name(*skip: str) -> str:
+    """
+    Find the first caller module that is not in the skip list.
+    :param skip: module names to skip
+    :return: the first caller module name not in skip, or empty string if no module can be found
+    """
+    for module, _frame in yield_caller_modules_and_frames(*skip):
+        return module.__name__
+
+    return ""  # this is trivially distinguishable from a module name, so no need to force people to handle None

thds/core/log/__init__.py

@@ -26,4 +26,4 @@ logger.info("testing 5")
 
 from .basic_config import DuplicateFilter, set_logger_to_console_level  # noqa: F401
 from .kw_formatter import ThdsCompactFormatter  # noqa: F401
-from .kw_logger import KwLogger, getLogger, logger_context, make_th_formatters_safe  # noqa: F401
+from .kw_logger import KwLogger, auto, getLogger, logger_context, make_th_formatters_safe  # noqa: F401

thds/core/log/kw_logger.py

@@ -80,6 +80,16 @@ def getLogger(name: Optional[str] = None) -> logging.LoggerAdapter:
     return KwLogger(logging.getLogger(name), dict())
 
 
+def auto(*skip: str) -> logging.LoggerAdapter:
+    from .. import inspect
+
+    module_name = inspect.caller_module_name(__name__, *skip)
+    if not module_name:
+        raise ValueError("Cannot automatically determine caller module name for logger.")
+
+    return getLogger(module_name)
+
+
 def make_th_formatters_safe(logger: logging.Logger):
     """Non-adapted loggers may still run into our root format string,
     which expects _TH_REC_CTXT to be present on every LogRecord.

thds/core/parallel.py

@@ -8,7 +8,7 @@ from collections import defaultdict
 from dataclasses import dataclass
 from uuid import uuid4
 
-from thds.core import concurrency, config, files, log
+from thds.core import concurrency, config, files, inspect, log
 
 PARALLEL_OFF = config.item("off", default=False, parse=config.tobool)
 # if you want to simplify a stack trace, this may be your friend
@@ -17,9 +17,6 @@ R = ty.TypeVar("R")
 T_co = ty.TypeVar("T_co", covariant=True)
 
 
-logger = log.getLogger(__name__)
-
-
 class IterableWithLen(ty.Protocol[T_co]):
     def __iter__(self) -> ty.Iterator[T_co]: ...  # pragma: no cover
 
@@ -69,13 +66,21 @@ class Error:
 H = ty.TypeVar("H", bound=ty.Hashable)
 
 
+def _get_caller_logger(named: str) -> ty.Callable[[str], ty.Any]:
+    module_name = inspect.caller_module_name(__name__)
+    if module_name:
+        return log.getLogger(module_name).info if named else log.getLogger(module_name).debug
+    return log.getLogger(__name__).debug  # if not named, we default to debug level
+
+
 def yield_all(
     thunks: ty.Iterable[ty.Tuple[H, ty.Callable[[], R]]],
     *,
     executor_cm: ty.Optional[ty.ContextManager[concurrent.futures.Executor]] = None,
     fmt: ty.Callable[[str], str] = lambda x: x,
+    error_fmt: ty.Callable[[str], str] = lambda x: x,
     named: str = "",
-    progress_logger: ty.Callable[[str], ty.Any] = logger.info,
+    progress_logger: ty.Optional[ty.Callable[[str], ty.Any]] = None,
 ) -> ty.Iterator[ty.Tuple[H, ty.Union[R, Error]]]:
     """Stream your results so that you don't have to load them all into memory at the same
     time (necessarily). Also, yield (rather than raise) Exceptions, wrapped as Errors.
@@ -104,6 +109,8 @@ def yield_all(
                 yield key, Error(e)
         return  # we're done here
 
+    progress_logger = progress_logger or _get_caller_logger(named)
+
     executor_cm = executor_cm or concurrent.futures.ThreadPoolExecutor(
         max_workers=len_or_none or None, **concurrency.initcontext()
     )  # if len_or_none turns out to be zero, swap in a None which won't kill the executor
@@ -113,8 +120,12 @@ def yield_all(
         # While concurrent.futures.as_completed accepts an iterable as input, it
         # does not yield any completed futures until the input iterable is
         # exhausted.
+        num_exceptions = 0
         for i, future in enumerate(concurrent.futures.as_completed(keys_onto_futures.values()), start=1):
             thunk_key = future_ids_onto_keys[id(future)]
+            error_suffix = (
+                error_fmt(f"; {num_exceptions} tasks have raised exceptions") if num_exceptions else ""
+            )
             try:
                 result = future.result()
                 yielder: tuple[H, ty.Union[R, Error]] = thunk_key, ty.cast(R, result)
@@ -123,7 +134,7 @@ def yield_all(
                 yielder = thunk_key, Error(e)
                 name = named or e.__class__.__name__
             finally:
-                progress_logger(fmt(f"Yielding {name} {i}{num_tasks_log}"))
+                progress_logger(fmt(f"Yielding {name} {i}{num_tasks_log}") + error_suffix)
                 yield yielder
 
 
@@ -137,8 +148,10 @@ def failfast(results: ty.Iterable[ty.Tuple[H, ty.Union[R, Error]]]) -> ty.Iterat
         yield key, res
 
 
-def xf_mapping(thunks: ty.Mapping[H, ty.Callable[[], R]]) -> ty.Iterator[ty.Tuple[H, R]]:
-    return failfast(yield_all(IteratorWithLen(len(thunks), thunks.items())))
+def xf_mapping(
+    thunks: ty.Mapping[H, ty.Callable[[], R]], named: str = ""
+) -> ty.Iterator[ty.Tuple[H, R]]:
+    return failfast(yield_all(IteratorWithLen(len(thunks), thunks.items()), named=named))
 
 
 def create_keys(iterable: ty.Iterable[R]) -> ty.Iterator[ty.Tuple[str, R]]:
@@ -153,6 +166,9 @@ def create_keys(iterable: ty.Iterable[R]) -> ty.Iterator[ty.Tuple[str, R]]:
         return iter(with_keys)
 
 
+ERROR_LOGGER = log.getLogger(__name__)
+
+
 def yield_results(
     thunks: ty.Iterable[ty.Callable[[], R]],
     *,
@@ -160,9 +176,9 @@ def yield_results(
     error_fmt: ty.Callable[[str], str] = lambda x: x,
     success_fmt: ty.Callable[[str], str] = lambda x: x,
     named: str = "",
-    progress_logger: ty.Callable[[str], ty.Any] = logger.info,
+    progress_logger: ty.Optional[ty.Callable[[str], ty.Any]] = None,
 ) -> ty.Iterator[R]:
-    """Yield only the successful results of your Callables/Thunks.
+    """Yield only the successful results of your Callables/Thunks. Continue despite errors.
 
     If your iterable has a length, we will be able to log progress
     information. In most cases, this will be advantageous for you.
@@ -177,27 +193,26 @@ def yield_results(
 
     exceptions: ty.List[Exception] = list()
 
-    num_tasks = try_len(thunks)
-    num_tasks_log = "" if not num_tasks else f" of {num_tasks}"
-    named = f" {named} " if named else " result "
-
     for i, (_key, res) in enumerate(
-        yield_all(create_keys(thunks), executor_cm=executor_cm),
+        yield_all(
+            create_keys(thunks),
+            executor_cm=executor_cm,
+            named=named,
+            progress_logger=progress_logger,
+            fmt=success_fmt,
+            error_fmt=error_fmt,
+        ),
         start=1,
     ):
         if not isinstance(res, Error):
-            errors = error_fmt(f"; {len(exceptions)} tasks have raised exceptions") if exceptions else ""
-            progress_logger(success_fmt(f"Yielding{named}{i}{num_tasks_log} {errors}"))
             yield res
         else:
             exceptions.append(res.error)
             # print tracebacks as we go, so as not to defer potentially-helpful
             # debugging information while a long run is ongoing.
             traceback.print_exception(type(res.error), res.error, res.error.__traceback__)
-            logger.error(  # should only use logger.exception from an except block
-                error_fmt(
-                    f"Task {i}{num_tasks_log} errored with {type(res.error).__name__}({res.error})"
-                )
+            ERROR_LOGGER.error(  # should only use logger.exception from an except block
+                error_fmt(f"Task {i} errored with {type(res.error).__name__}({res.error})")
             )
 
     summarize_exceptions(error_fmt, exceptions)
@@ -218,10 +233,10 @@ def summarize_exceptions(
         most_common_type = None
         max_count = 0
         for _type, excs in by_type.items():
-            logger.error(error_fmt(f"{len(excs)} tasks failed with exception: " + _type.__name__))
+            ERROR_LOGGER.error(error_fmt(f"{len(excs)} tasks failed with exception: " + _type.__name__))
             if len(excs) > max_count:
                 max_count = len(excs)
                 most_common_type = _type
 
-        logger.info("Raising one of the most common exception type.")
+        ERROR_LOGGER.info("Raising one of the most common exception type.")
         raise by_type[most_common_type][0]  # type: ignore

{thds_core-1.47.20251024053252.dist-info → thds_core-1.47.20251024204437.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: thds.core
-Version: 1.47.20251024053252
+Version: 1.47.20251024204437
 Summary: Core utilities.
 Author-email: Trilliant Health <info@trillianthealth.com>
 License: MIT

{thds_core-1.47.20251024053252.dist-info → thds_core-1.47.20251024204437.dist-info}/RECORD

@@ -22,14 +22,14 @@ thds/core/hashing.py,sha256=dR4HEWcAdU8O-9ASGkl8naKs6I1Sd7aps4EcVefvVLQ,4246
 thds/core/home.py,sha256=tTClL_AarIKeri1aNCpuIC6evD7qr83ESGD173B81hU,470
 thds/core/hostname.py,sha256=canFGr-JaaG7nUfsQlyL0JT-2tnZoT1BvXzyaOMK1vA,208
 thds/core/imports.py,sha256=0LVegY8I8_XKZPcqiIp2OVVzEDtyqYA3JETf9OAKNKs,568
-thds/core/inspect.py,sha256=IXzJYkc8XGx9W9Oym9Bus2fGMcWHLAtlzvmQ1PwDKmk,2371
+thds/core/inspect.py,sha256=vBuVJ9aKR_WT0W8SZ59UfZ3NXVS330WCeef1uUe546w,3426
 thds/core/iterators.py,sha256=h0JBu2-rYhKMfJTDlZWfyHQWzgtIO8vp_Sp0gENFo7g,645
 thds/core/lazy.py,sha256=e1WvG4LsbEydV0igEr_Vl1cq05zlQNIE8MFYT90yglE,3289
 thds/core/link.py,sha256=4-9d22l_oSkKoSzlYEO-rwxO1hvvj6VETY7LwvGcX6M,5534
 thds/core/logical_root.py,sha256=gWkIYRv9kNQfzbpxJaYiwNXVz1neZ2NvnvProtOn9d8,1399
 thds/core/merge_args.py,sha256=7oj7dtO1-XVkfTM3aBlq3QlZbo8tb6X7E3EVIR-60t8,5781
 thds/core/meta.py,sha256=Df0DxV5UzHcEsu5UCYaE1BWipMPTEXycn9Ug4cdquMk,12114
-thds/core/parallel.py,sha256=FojyyUtJ3jrYcsPiHlotWqUUybP_HtVgAN59_ipYf2g,8597
+thds/core/parallel.py,sha256=is12mjELQwueUfrpTyu9d4Ow20641nGozw0Qib5bmWA,9108
 thds/core/pickle_visit.py,sha256=QNMWIi5buvk2zsvx1-D-FKL7tkrFUFDs387vxgGebgU,833
 thds/core/prof.py,sha256=5ViolfPsAPwUTHuhAe-bon7IArPGXydpGoB5uZmObDk,8264
 thds/core/progress.py,sha256=tY8tc_6CMnu_O8DVisnsRoDpFJOw5vqyYzLhQDxsLn8,4361
@@ -46,11 +46,11 @@ thds/core/thunks.py,sha256=LxwqUsu3YPVDleGbNk5JWZIncDYwvM8wUBNOS2L09zs,1056
 thds/core/timer.py,sha256=aOpNP-wHKaKs6ONK5fOtIOgx00FChVZquG4PeaEYH_k,5376
 thds/core/tmp.py,sha256=jA8FwDbXo3hx8o4kRjAlkwpcI77X86GY4Sktkps29ho,3166
 thds/core/types.py,sha256=sFqI_8BsB1u85PSizjBZw8PBtplC7U54E19wZZWCEvI,152
-thds/core/log/__init__.py,sha256=bDbZvlxyymY6VrQzD8lCn0egniLEiA9hpNMAXZ7e7wY,1348
+thds/core/log/__init__.py,sha256=wbHgzqHln74wfqM7gLzRURgOn9V_-2dOnoOBi37xczU,1354
 thds/core/log/basic_config.py,sha256=2Y9U_c4PTrIsCmaN7Ps6Xr90AhJPzdYjeUzUMqO7oFU,6704
 thds/core/log/json_formatter.py,sha256=C5bRsSbAqaQqfTm88jc3mYe3vwKZZLAxET8s7_u7aN0,1757
 thds/core/log/kw_formatter.py,sha256=9-MVOd2r5NEkYNne9qWyFMeR5lac3w7mjHXsDa681i0,3379
-thds/core/log/kw_logger.py,sha256=CyZVPnkUMtrUL2Lyk261AIEPmoP-buf_suFAhQlU1io,4063
+thds/core/log/kw_logger.py,sha256=bXajphFJXOtiIOix7a4uOs_gzqTiPu-76uVGIZK7A5E,4351
 thds/core/log/logfmt.py,sha256=i66zoG2oERnE1P_0TVXdlfJ1YgUmvtMjqRtdV5u2SvU,10366
 thds/core/source/__init__.py,sha256=e-cRoLl1HKY3YrDjpV5p_i7zvr1L4q51-t1ISTxdig4,543
 thds/core/source/_construct.py,sha256=jtsh0Du67TslWjCLASZ3pAMeaiowfgm7Bt50zIhwx7k,4330
@@ -74,8 +74,8 @@ thds/core/sqlite/structured.py,sha256=8t1B6XbM5NnudKEeBLsdjRVbSXXSr6iHOW0HwEAqtX
 thds/core/sqlite/types.py,sha256=oq8m0UrvSn1IqWWcQ4FPptfAhdj6DllnCe7puVqSHlY,1297
 thds/core/sqlite/upsert.py,sha256=BmKK6fsGVedt43iY-Lp7dnAu8aJ1e9CYlPVEQR2pMj4,5827
 thds/core/sqlite/write.py,sha256=z0219vDkQDCnsV0WLvsj94keItr7H4j7Y_evbcoBrWU,3458
-thds_core-1.47.20251024053252.dist-info/METADATA,sha256=yNcSdU59kVnfCMtS6Q3XwfMr6Ak3-PPnRYRS9ItJGc4,2216
-thds_core-1.47.20251024053252.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-thds_core-1.47.20251024053252.dist-info/entry_points.txt,sha256=bOCOVhKZv7azF3FvaWX6uxE6yrjK6FcjqhtxXvLiFY8,161
-thds_core-1.47.20251024053252.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
-thds_core-1.47.20251024053252.dist-info/RECORD,,
+thds_core-1.47.20251024204437.dist-info/METADATA,sha256=2bk7jxHjsXiKVXmQD_x-FSSUbw3UZRDNdRG2skxndrY,2216
+thds_core-1.47.20251024204437.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+thds_core-1.47.20251024204437.dist-info/entry_points.txt,sha256=bOCOVhKZv7azF3FvaWX6uxE6yrjK6FcjqhtxXvLiFY8,161
+thds_core-1.47.20251024204437.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
+thds_core-1.47.20251024204437.dist-info/RECORD,,