thds.core 0.0.1__py3-none-any.whl → 1.31.20250116223856__py3-none-any.whl

thds/core/timer.py

@@ -0,0 +1,214 @@
+"""
+A module that adds a simple interface for tracking execution times of various pieces of code.
+
+```python
+tracker = TimeTracker()
+
+@tracker.track()
+def computation(num: int):
+    ...
+
+with tracker(tracker.total):
+    for i in range(10):
+        with tracker("setup"):
+            ...
+            ...
+        computation(i)
+
+print(tracker.percentage_of_totals)
+```
+
+`TimeTracker`s can also be merged.
+
+*module_1.py*
+```python
+
+tracker = TimeTracker()
+
+@tracker.track()
+def foo():
+    ...
+
+@tracker.track()
+def bar():
+    ...
+
+
+@tracker.track()
+def main():
+    foo()
+    bar()
+```
+
+*module_2.py*
+```python
+import time
+import module_1 as mod1
+
+tracker = TimeTracker()
+
+with tracker(tracker.total):
+    for i in range(100):
+        time.sleep(5)
+    mod1.main()
+
+tracker.merge(mod1.tracker)
+```
+"""
+
+import json
+import time
+from collections import defaultdict
+from dataclasses import asdict, dataclass
+from functools import wraps
+from typing import (
+    Callable,
+    DefaultDict,
+    Dict,
+    Generator,
+    Iterator,
+    List,
+    Optional,
+    Tuple,
+    TypeVar,
+    Union,
+    cast,
+)
+
+from typing_extensions import ParamSpec
+
+from thds.core import config, log
+
+T = TypeVar("T")
+P = ParamSpec("P")
+
+F = TypeVar("F", bound=Callable)
+
+
+SINGLE_LINE_JSON_TIMERS = config.item(
+    "unified.single-line-json-timers", default=False, parse=config.tobool
+)
+# setting this to true makes it much easier to extract the data from logs in a production run using Grafana.
+
+
+def timer(func: F) -> F:
+    """
+    Decorator to add logging of timer information to a function invocation. Logs when entering a function and then logs
+    with time information when exiting.
+    :param func:
+        Function to decorate with timing info.
+    :return:
+        Wrapped function.
+    """
+    logger = log.getLogger(func.__module__)
+
+    @wraps(func)
+    def wrapper_timer(*args, **kwargs):
+        start = time.time()
+        start_formatted = time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.localtime(start))
+
+        logger.info("Starting %r at %s", func.__name__, start_formatted)
+        start_time = time.perf_counter()
+        value = func(*args, **kwargs)
+        end_time = time.perf_counter()
+        run_time = end_time - start_time
+
+        end = time.time()
+        end_formatted = time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.localtime(end))
+        logger.info("Finished %r in %0.4f secs at %s", func.__name__, run_time, end_formatted)
+        return value
+
+    return cast(F, wrapper_timer)
+
+
+@dataclass
+class Timer:
+    secs: float = 0.0
+    calls: int = 0
+
+    def __iter__(self) -> Generator[Tuple[str, Union[int, float]], None, None]:
+        for k, v in {**asdict(self), "secs_per_call": self.secs_per_call}.items():
+            yield k, v
+
+    def __add__(self, other: "Timer") -> "Timer":
+        return Timer(self.secs + other.secs, self.calls + other.calls)
+
+    def __radd__(self, other):
+        if other == 0:
+            return self
+        else:
+            return self.__add__(other)
+
+    @property
+    def secs_per_call(self) -> float:
+        return (self.secs / self.calls) / 60.0 if self.calls > 0 else float("nan")
+
+    @property
+    def mins(self) -> float:
+        return self.secs / 60.0
+
+    def pct_of_total(self, total: float) -> float:
+        return (self.secs / total) * 100
+
+
+class TimeTracker:
+    total: str = "total"
+
+    def __init__(self, times: Optional[DefaultDict[str, Timer]] = None):
+        self.tracked_times: DefaultDict[str, Timer] = times or defaultdict(Timer)
+        self._names: List[str] = []
+        self._start_times: List[float] = []
+
+    def reset(self) -> None:
+        self.tracked_times = defaultdict(Timer)
+        self._names = []
+        self._start_times = []
+
+    def to_json(self) -> Iterator[str]:
+        if SINGLE_LINE_JSON_TIMERS:
+            for name, timer in sorted(self.tracked_times.items(), key=lambda x: x[0]):
+                yield json.dumps({name: dict(timer)}, indent=None)
+        else:
+            yield json.dumps(
+                {name: dict(timer) for name, timer in self.tracked_times.items()},
+                sort_keys=True,
+                indent=4,
+            )
+
+    def track(self, component_name: Optional[str] = None):
+        def decorator(func):
+            @wraps(func)
+            def wrapper(*args, **kwargs):
+                start = time.perf_counter()
+                func_result = func(*args, **kwargs)
+                cmpnt = self.tracked_times[component_name or f"{func.__module__}.{func.__qualname__}"]
+                cmpnt.secs += time.perf_counter() - start
+                cmpnt.calls += 1
+                return func_result
+
+            return wrapper
+
+        return decorator
+
+    def __call__(self, component_name: str) -> "TimeTracker":
+        self._names.append(component_name)
+        return self
+
+    def __enter__(self) -> "TimeTracker":
+        self._start_times.append(time.perf_counter())
+        return self
+
+    def __exit__(self, *args, **kwargs):
+        cmpnt = self.tracked_times[self._names.pop()]
+        cmpnt.secs += time.perf_counter() - self._start_times.pop()
+
+    @property
+    def pct_of_totals(self) -> Dict[str, float]:
+        total = self.tracked_times.get(self.total)
+        if total:
+            return {
+                name: timer.pct_of_total(total.secs)
+                for name, timer in self.tracked_times.items()
+                if name != "total"
+            }
+        return {}

thds/core/tmp.py

@@ -0,0 +1,85 @@
+"""Utilities for managing temporary directories and files.
+
+Module is given this name partly because we tend not to name other things internally using
+the word 'tmp', preferring instead 'temp'. Therefore `tmp` will be a little less ambiguous
+overall.
+"""
+import contextlib
+import shutil
+import tempfile
+import typing as ty
+from pathlib import Path
+from random import SystemRandom
+
+from .home import HOMEDIR
+from .types import StrOrPath
+
+
+def _are_same_fs(path1: Path, path2: Path) -> bool:
+    return path1.stat().st_dev == path2.stat().st_dev
+
+
+def _walk_up_to_existing_dir(path: Path) -> Path:
+    existing_dir = path.resolve()
+    while not existing_dir.exists() or not existing_dir.is_dir():
+        # this is guaranteed to terminate at the FS root
+        existing_dir = existing_dir.parent
+        if existing_dir.resolve() == existing_dir.parent.resolve():
+            raise FileNotFoundError(f"{path} does not exist on a known filesystem.")
+    return existing_dir
+
+
+@contextlib.contextmanager
+def temppath_same_fs(lcl_path: StrOrPath = "") -> ty.Iterator[Path]:
+    """Builds a temporary Path (with no file at it) on the same filesystem as the
+    provided local path. No file or directory is created for you at the actual
+    Path.
+
+    Useful for making sure that you can do atomic moves from tempfiles to your final
+    location.
+    """
+    parent_dir = _walk_up_to_existing_dir(Path(lcl_path) if lcl_path else HOMEDIR())
+    basename = Path(lcl_path).name
+
+    def _tempdir_same_filesystem() -> ty.Iterator[Path]:
+        # we would prefer _not_ to reinvent the wheel here, but if the tempfiles are
+        # getting created on a different volume, then moves are no longer atomic, and
+        # that's a huge pain for lots of reasons.
+        fname_parts = [
+            ".thds-core-tmp-home-fs",
+            str(SystemRandom().random())[2:],  # makes a float look like a numeric string
+        ]
+        if basename:
+            fname_parts.append(basename)
+        dpath = parent_dir / "-".join(fname_parts)
+        try:
+            yield dpath
+        finally:
+            if dpath.exists():
+                if dpath.is_file():
+                    dpath.unlink()
+                else:
+                    shutil.rmtree(dpath, ignore_errors=True)
+
+    with tempfile.TemporaryDirectory() as tdir:
+        # actually check whether we're on the same filesystem.
+        if _are_same_fs(parent_dir, Path(tdir)):
+            yield Path(tdir) / "-".join(filter(None, ["thds-core-tmp-def-fs", basename]))
+        else:
+            yield from _tempdir_same_filesystem()
+
+
+@contextlib.contextmanager
+def tempdir_same_fs(lcl_path: StrOrPath = "") -> ty.Iterator[Path]:
+    """Builds a temporary directory on the same filesystem as the
+    provided local path.
+
+    Useful for making sure that you can do atomic moves from tempfiles to your final
+    location.
+
+    If you have no need for the above uses, teh built-in `tempfile.tmpdir` is a functionally
+    equivalent substitute.
+    """
+    with temppath_same_fs(lcl_path) as tpath:
+        tpath.mkdir(parents=True, exist_ok=True)
+        yield tpath

thds/core/types.py

@@ -0,0 +1,4 @@
+import os
+import typing as ty
+
+StrOrPath = ty.Union[str, os.PathLike]

thds.core-1.31.20250116223856.dist-info/METADATA

@@ -0,0 +1,68 @@
+Metadata-Version: 2.2
+Name: thds.core
+Version: 1.31.20250116223856
+Summary: Core utilities.
+Author: Trilliant Health
+Description-Content-Type: text/markdown
+Requires-Dist: attrs>=22.2.0
+Requires-Dist: cattrs>=22.2.0
+Requires-Dist: setuptools
+Requires-Dist: typing-extensions
+
+# core Library
+
+The monorepo successor to `core`
+
+## Development
+
+If making changes to the library please add an entry to `CHANGES.md`, and if the change is more than a
+patch, please bump the version in `pyproject.toml` accordingly.
+
+## Config
+
+`thds.core.config` provides a general-purpose config system designed to regularize how we implement
+configuration both for libraries and applications. Please see its [README here](src/thds/core/CONFIG.md)!
+
+## Logging config
+
+This library handles configuration of all DS loggers. By default, all INFO-and-above messages are written
+(to `stderr`).
+
+### Default output formatter
+
+By default we use a custom formatter intended to make things maximally human-readable.
+
+If you want structured logs, you might try setting `THDS_CORE_LOG_FORMAT=logfmt`, or `json` if you want
+JSON logs.
+
+### File format
+
+To customize what level different modules are logged at, you should create a file that looks like this:
+
+```
+[debug]
+thds.adls.download
+thds.core.link
+[warning]
+thds.mops.pure.pickle_runner
+thds.mops.k8s.watch
+```
+
+You may also/instead add an `*` to change the global default log level, e.g.:
+
+```
+[warning]
+*
+```
+
+> The wildcard syntax is not a generic pattern-matching facility; it _only_ matches the root logger.
+>
+> However, if you wish to match a subtree of the logger hierarchy, this is built in with Python loggers;
+> simply configure `thds.adls` under `[debug]` and all otherwise-unconfigured loggers under `thds.adls`
+> will now log at the DEBUG level.
+
+### `THDS_CORE_LOG_LEVELS_FILE` environment variable
+
+Provide the path to the above-formatted file to `thds.core` via the `THDS_CORE_LOG_LEVELS_FILE`
+environment variable. You may wish to create this file and then set its path via exported envvar in your
+`.bash/zshrc` so that you can permanently tune our logging to meet your preferences.

thds.core-1.31.20250116223856.dist-info/RECORD

@@ -0,0 +1,67 @@
+thds/core/__init__.py,sha256=imKpmnrBV0_7-1d1Pc2yR5jxbvOrmIplLm7Ig_eK1OU,934
+thds/core/ansi_esc.py,sha256=QZ3CptZbX4N_hyP2IgqfTbNt9tBPaqy7ReTMQIzGbrc,870
+thds/core/cache.py,sha256=nL0oAyZrhPqyBBLevnOWSWVoEBrftaG3aE6Qq6tvmAA,7153
+thds/core/calgitver.py,sha256=HklIz-SczK92Vm2rXtTSDiVxAcxUW_GPVCRRGt4BmBA,2324
+thds/core/concurrency.py,sha256=NQunF_tJ_z8cfVyhzkTPlb-nZrgu-vIk9_3XffgscKQ,3520
+thds/core/config.py,sha256=N-WVpPDrfTSFKz0m7WrqZPBdd17dycpDx9nhbATkf3c,9092
+thds/core/decos.py,sha256=VpFTKTArXepICxN4U8C8J6Z5KDq-yVjFZQzqs2jeVAk,1341
+thds/core/dict_utils.py,sha256=MAVkGJg4KQN1UGBLEKuPdQucZaXg_jJakujQ-GUrYzw,6471
+thds/core/env.py,sha256=M36CYkPZ5AUf_-n8EqjsMGwWOzaKEn0KgRwnqUK7jS4,1094
+thds/core/exit_after.py,sha256=0lz63nz2NTiIdyBDYyRa9bQShxQKe7eISy8VhXeW4HU,3485
+thds/core/files.py,sha256=8amNymTRuW17o6Qay73juxwkcW1DysWXeAByeofadkM,4540
+thds/core/fretry.py,sha256=Tui2q6vXV6c7mjTa1czLrXiugHUEwQp-sZdiwXfxvmM,3829
+thds/core/generators.py,sha256=rcdFpPj0NMJWSaSZTnBfTeZxTTORNB633Lng-BW1284,1939
+thds/core/git.py,sha256=I6kaEvwcvVxCLYHhTTfnHle-GkmgOR9_fHs03QxgBfI,2792
+thds/core/hash_cache.py,sha256=bkdV8HZOl66zNffnIjRvMQH1A0_xNqowMDLBtRXGFl8,3723
+thds/core/hashing.py,sha256=5I4vLBjgkn9yOXMjBduLGdXsywBob-35qJTk_lks6aw,3505
+thds/core/home.py,sha256=tTClL_AarIKeri1aNCpuIC6evD7qr83ESGD173B81hU,470
+thds/core/hostname.py,sha256=canFGr-JaaG7nUfsQlyL0JT-2tnZoT1BvXzyaOMK1vA,208
+thds/core/imports.py,sha256=0LVegY8I8_XKZPcqiIp2OVVzEDtyqYA3JETf9OAKNKs,568
+thds/core/inspect.py,sha256=vCxKqw8XG2W1cuj0MwjdXhe9TLQrGdjRraS6UEYsbf8,1955
+thds/core/iterators.py,sha256=d3iTQDR0gCW1nMRmknQeodR_4THzR9Ajmp8F8KCCFgg,208
+thds/core/lazy.py,sha256=e1WvG4LsbEydV0igEr_Vl1cq05zlQNIE8MFYT90yglE,3289
+thds/core/link.py,sha256=kmFJIFvEZc16-7S7IGvtTpzwl3VuvFl3yPlE6WJJ03w,5404
+thds/core/merge_args.py,sha256=7oj7dtO1-XVkfTM3aBlq3QlZbo8tb6X7E3EVIR-60t8,5781
+thds/core/meta.json,sha256=S_OtvqShWHyPjzX5ODjtw4SFDBQmP20V4QXw5fr2_a0,219
+thds/core/meta.py,sha256=IPLAKrH06HooPMNf5FeqJvUcM-JljTGXddrAQ5oAX8E,16896
+thds/core/parallel.py,sha256=lUzM0CRXsfzZgMMOHcwP-Th7bbXAqfl7FEGwdwAoBiM,7098
+thds/core/pickle_visit.py,sha256=QNMWIi5buvk2zsvx1-D-FKL7tkrFUFDs387vxgGebgU,833
+thds/core/prof.py,sha256=atDz7rDxzMzbKwgM48tics2tB5sBijKb_L5heAT6Pk8,8246
+thds/core/progress.py,sha256=4YGbxliDl1i-k-88w4s86uy1E69eQ6xJySGPSkpH1QM,3358
+thds/core/protocols.py,sha256=4na2EeWUDWfLn5-SxfMmKegDSndJ5z-vwMhDavhCpEM,409
+thds/core/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+thds/core/scaling.py,sha256=f7CtdgK0sN6nroTq5hLAkG8xwbWhbCZUULstSKjoxO0,1615
+thds/core/scope.py,sha256=iPRhS-lIe-axDctqxBtEPeF0PM_w-0tRS-9kPweUGBY,7205
+thds/core/source.py,sha256=ACMe_e-0tuwUknOobomC9_q6vHM3hMTouWsSsJ-qHQk,9884
+thds/core/source_serde.py,sha256=TqW4MTrXQ49JJaXrkmFcymSBIWvBlkxO2JOPNQap_F4,3523
+thds/core/stack_context.py,sha256=17lPOuYWclUpZ-VXRkPgI4WbiMzq7_ZY6Kj1QK_1oNo,1332
+thds/core/thunks.py,sha256=p1OvMBJ4VGMsD8BVA7zwPeAp0L3y_nxVozBF2E78t3M,1053
+thds/core/timer.py,sha256=1FfcQ4-Gp6WQFXR0GKeT_8jwtamEfnTukdSbDKTAJVM,5432
+thds/core/tmp.py,sha256=KgBAwQCmpm7I762eLRu-3MSfH3dKnqlrJkZ5nmPcRbc,3110
+thds/core/types.py,sha256=Pbo5bF3b14V8eeabpnffWOPJDAGrMcKBCh5e-ZvipJk,70
+thds/core/log/__init__.py,sha256=bDbZvlxyymY6VrQzD8lCn0egniLEiA9hpNMAXZ7e7wY,1348
+thds/core/log/basic_config.py,sha256=WQDpLKFgu-BTdR22LMnGDK3jk4CUbP94JrjSsHgAjjg,6216
+thds/core/log/json_formatter.py,sha256=C5bRsSbAqaQqfTm88jc3mYe3vwKZZLAxET8s7_u7aN0,1757
+thds/core/log/kw_formatter.py,sha256=9-MVOd2r5NEkYNne9qWyFMeR5lac3w7mjHXsDa681i0,3379
+thds/core/log/kw_logger.py,sha256=k9nckB0RRYU-X8Ols-pnxU_UuL6kqvtDs6MOi5JE7XY,3841
+thds/core/log/logfmt.py,sha256=qS6BbdlOZPRnxmHenVL3uK43OQ30NJUnz92S6d_Yh2A,10360
+thds/core/sqlite/__init__.py,sha256=tDMzuO76qTtckJHldPQ6nPZ6kcvhhoJrVuuW42JtaSQ,606
+thds/core/sqlite/connect.py,sha256=l4QaSAI8RjP7Qh2FjmJ3EwRgfGf65Z3-LjtC9ocHM_U,977
+thds/core/sqlite/copy.py,sha256=y3IRQTBrWDfKuVIfW7fYuEgwRCRKHjN0rxVFkIb9VrQ,1155
+thds/core/sqlite/ddl.py,sha256=k9BvmDzb0rrlhmEpXkB6ESaZAUWtbL58x-70sPyoFk4,201
+thds/core/sqlite/functions.py,sha256=AOIRzb7lNxmFm1J5JS6R8Nl-dSv3Dy47UNZVVjl1rvk,2158
+thds/core/sqlite/index.py,sha256=Vc7qxPqQ69A6GO5gmVQf5e3y8f8IqOTHgyEDoVZxTFM,903
+thds/core/sqlite/insert_utils.py,sha256=LUVcznl-xCVoh_L_6tabVYUAYnEnaVDmBX2PeopLMKU,884
+thds/core/sqlite/merge.py,sha256=NxettDMJ_mcrWfteQn_ERY7MUB5ETR-yJLKg7uvF6zA,3779
+thds/core/sqlite/meta.py,sha256=4P65PAmCjagHYO1Z6nWM-wkjEWv3hxw5qVa4cIpcH_8,5859
+thds/core/sqlite/read.py,sha256=5pWvrbed3XNWgSy-79-8ONWkkt4jWbTzFNW6SnOrdYQ,2576
+thds/core/sqlite/sqlmap.py,sha256=YzLsTXwvEWcPI_KmFjA4tKfPoSP3gpHThyIthmPVyK0,6778
+thds/core/sqlite/structured.py,sha256=swCbDoyVT6cE7Kl79Wh_rg5Z1-yrUDJbiVJF4bjsetU,4641
+thds/core/sqlite/types.py,sha256=oUkfoKRYNGDPZRk29s09rc9ha3SCk2SKr_K6WKebBFs,1308
+thds/core/sqlite/upsert.py,sha256=BmKK6fsGVedt43iY-Lp7dnAu8aJ1e9CYlPVEQR2pMj4,5827
+thds/core/sqlite/write.py,sha256=h-tld6sC1alkaF92KRdrjyrEOwCt3bZ804muOcV4MEw,3141
+thds.core-1.31.20250116223856.dist-info/METADATA,sha256=vSPFjc1mry08Oj6Cy6Ouwk0G4r7sLRPml9RCVBoGWgU,2123
+thds.core-1.31.20250116223856.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+thds.core-1.31.20250116223856.dist-info/entry_points.txt,sha256=bOCOVhKZv7azF3FvaWX6uxE6yrjK6FcjqhtxXvLiFY8,161
+thds.core-1.31.20250116223856.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
+thds.core-1.31.20250116223856.dist-info/RECORD,,

{thds.core-0.0.1.dist-info → thds.core-1.31.20250116223856.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.41.2)
+Generator: setuptools (75.8.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

thds.core-1.31.20250116223856.dist-info/entry_points.txt

@@ -0,0 +1,4 @@
+[console_scripts]
+new-calgitver = thds.core.meta:print_calgitver
+show-thds-config = thds.core.config:show_config_cli
+thds-exit-after = thds.core.exit_after:main

thds.core-1.31.20250116223856.dist-info/top_level.txt

@@ -0,0 +1 @@
+thds

thds.core-0.0.1.dist-info/METADATA

@@ -1,8 +0,0 @@
-Metadata-Version: 2.1
-Name: thds.core
-Version: 0.0.1
-Home-page: https://www.trillianthealth.com
-Author: Trilliant Health Data Science
-Author-email: datascience@trillianthealth.com
-Classifier: Development Status :: 1 - Planning
-

thds.core-0.0.1.dist-info/RECORD

@@ -1,4 +0,0 @@
-thds.core-0.0.1.dist-info/METADATA,sha256=58tW_pFtdIcYkNB1KrZ6kYpTpg_kLOesVF_9G8vPtLY,228
-thds.core-0.0.1.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
-thds.core-0.0.1.dist-info/top_level.txt,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-thds.core-0.0.1.dist-info/RECORD,,

thds.core-0.0.1.dist-info/top_level.txt

@@ -1 +0,0 @@
-