thds.core 1.32.20250314195422__py3-none-any.whl → 1.32.20250320220920__py3-none-any.whl

thds/core/cpus.py

@@ -0,0 +1,81 @@
+import multiprocessing
+import os
+import typing as ty
+from pathlib import Path
+
+from . import log
+
+_CPU_QUOTA_PATH = Path("/sys/fs/cgroup/cpu/cpu.cfs_quota_us")
+_CPU_PERIOD_PATH = Path("/sys/fs/cgroup/cpu/cpu.cfs_period_us")
+# standard linux kernel cgroup config files
+# https://www.kernel.org/doc/html/latest/scheduler/sched-bwc.html#management
+_CPU_MAX_PATH_V2 = Path("/sys/fs/cgroup/cpu.max")
+# cgroups v2: https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html#cgroup-v2-cpu
+_CPU_SHARES_PATH = Path("/sys/fs/cgroup/cpu/cpu.shares")
+# some kind of redhat thing: https://www.redhat.com/en/blog/cgroups-part-two
+
+logger = log.getLogger(__name__)
+
+T = ty.TypeVar("T")
+
+
+def _try_read_value(config_path: Path, parse: ty.Callable[[str], T]) -> ty.Optional[T]:
+    if config_path.is_file():
+        logger.info(f"Found {config_path}; attempting to read value")
+        with config_path.open() as f:
+            contents = f.read().strip()
+            try:
+                value = parse(contents)
+            except Exception as e:
+                logger.error(f"Could not parse value from {contents} with {parse}: {e}")
+                # if the file exists but we can't parse the contents, something is very wrong with our assumptions;
+                # better to fail loudly than risk silent CPU oversubscription
+                raise e
+            else:
+                logger.info(f"Read value {value} from {config_path}")
+                return value
+    return None
+
+
+def _parse_cpu_quota_and_period_v2(s: str) -> ty.Tuple[int, int]:
+    """Parse both CPU quota and period from kernel cgroup v2 config file."""
+    quota, period = map(int, s.split())
+    return quota, period
+
+
+def available_cpu_count() -> int:
+    """Attempt to determine number of available CPUs, accounting for the possibility of running inside a docker
+    container. Ideally, this serves as a drop-in replacement for os.cpu_count() in that context.
+
+    Partially cribbed from a suggestion in https://bugs.python.org/issue36054, and partially from joblib (specifically
+    handling v2 of the kernel cgroup spec).
+    """
+    if hasattr(os, "sched_getaffinity"):
+        cpu_count = len(os.sched_getaffinity(0))
+    else:
+        cpu_count = multiprocessing.cpu_count()
+
+    cpu_quota_us: ty.Optional[int]
+    cpu_period_us: ty.Optional[int]
+    if (
+        quota_and_period_v2 := _try_read_value(_CPU_MAX_PATH_V2, _parse_cpu_quota_and_period_v2)
+    ) is not None:
+        # this file contains both values
+        cpu_quota_us, cpu_period_us = quota_and_period_v2
+    else:
+        cpu_quota_us = _try_read_value(_CPU_QUOTA_PATH, int)
+        cpu_period_us = _try_read_value(_CPU_PERIOD_PATH, int)
+
+    if cpu_quota_us is not None and cpu_period_us is not None and cpu_quota_us != -1:
+        cpu_shares = int(cpu_quota_us / cpu_period_us)
+    elif cpu_shares_ := _try_read_value(_CPU_SHARES_PATH, int):
+        cpu_shares = int(cpu_shares_ / 1024)
+    else:
+        logger.info(f"Using naive CPU count: {cpu_count}")
+        return cpu_count
+
+    logger.info(
+        f"Determined CPU shares from quota and period: {cpu_shares}; returning lesser of this and naive "
+        f"CPU count: {cpu_count}"
+    )
+    return min(cpu_shares, cpu_count)

thds/core/source/_construct.py

@@ -1,25 +1,26 @@
+import os
 import typing as ty
 from functools import partial
 from pathlib import Path
 
 from ..files import is_file_uri, path_from_uri, to_uri
 from ..hashing import Hash
-from ..types import StrOrPath
 from . import _download
 from .src import Source
 
 # Creation from local Files or from remote URIs
 
 
-def from_file(filename: StrOrPath, hash: ty.Optional[Hash] = None, uri: str = "") -> Source:
+def from_file(
+    filename: ty.Union[str, os.PathLike], hash: ty.Optional[Hash] = None, uri: str = ""
+) -> Source:
     """Create a read-only Source from a local file that already exists.
 
     If URI is passed, the local file will be read and hashed, but the final URI in the
     Source will be the one provided explicitly. NO UPLOAD IS PERFORMED. It is your
     responsibility to ensure that your file has been uploaded to the URI you provide.
     """
-    path = path_from_uri(filename) if isinstance(filename, str) else filename
-    assert isinstance(path, Path)
+    path = path_from_uri(filename) if isinstance(filename, str) else Path(filename)
     if not path.exists():
         raise FileNotFoundError(path)
 
@@ -40,6 +41,7 @@ class FromUri(ty.Protocol):
         and the hash will be included in the Source object regardless, and will
         be validated (if non-nil) at the time of source data access.
         """
+        ...
 
 
 class FromUriHandler(ty.Protocol):
@@ -47,6 +49,7 @@ class FromUriHandler(ty.Protocol):
         """Returns a FromUri object containing the URI if this URI can be handled.  Returns
         None if this URI cannot be handled.
         """
+        ...
 
 
 def register_from_uri_handler(key: str, handler: FromUriHandler):

thds/core/types.py

@@ -1,4 +1,6 @@
 import os
 import typing as ty
 
-StrOrPath = ty.Union[str, os.PathLike]
+StrOrPath = ty.Union[
+    str, os.PathLike
+]  # DEPRECATED - please be explicit about this bc it isn't much extra typing

{thds_core-1.32.20250314195422.dist-info → thds_core-1.32.20250320220920.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: thds.core
-Version: 1.32.20250314195422
+Version: 1.32.20250320220920
 Summary: Core utilities.
 Author-email: Trilliant Health <info@trillianthealth.com>
 License: MIT

{thds_core-1.32.20250314195422.dist-info → thds_core-1.32.20250320220920.dist-info}/RECORD

@@ -5,6 +5,7 @@ thds/core/calgitver.py,sha256=HklIz-SczK92Vm2rXtTSDiVxAcxUW_GPVCRRGt4BmBA,2324
 thds/core/cm.py,sha256=WZB8eQU0DaBYj9s97nc3PuCtai9guovfyiQH68zhLzY,1086
 thds/core/concurrency.py,sha256=NQunF_tJ_z8cfVyhzkTPlb-nZrgu-vIk9_3XffgscKQ,3520
 thds/core/config.py,sha256=VWymw6pqPRvX7wwsJ0Y-D2gLoCclAHhARmTnuUw7kb0,10014
+thds/core/cpus.py,sha256=DOm6E1uWKZLCEXWKwfS1ZHuSTWD0XWwp1vgXHC-5oNk,3230
 thds/core/decos.py,sha256=VpFTKTArXepICxN4U8C8J6Z5KDq-yVjFZQzqs2jeVAk,1341
 thds/core/dict_utils.py,sha256=MatsjZC9lchfdaDqNAzL2mkTZytDnCAqg56sMm71wbE,6364
 thds/core/env.py,sha256=HkuyFmGpCgdQUB1r2GbpCqB3cs1lCsvp47Ghk1DHBo8,1083
@@ -38,7 +39,7 @@ thds/core/stack_context.py,sha256=17lPOuYWclUpZ-VXRkPgI4WbiMzq7_ZY6Kj1QK_1oNo,13
 thds/core/thunks.py,sha256=p1OvMBJ4VGMsD8BVA7zwPeAp0L3y_nxVozBF2E78t3M,1053
 thds/core/timer.py,sha256=1FfcQ4-Gp6WQFXR0GKeT_8jwtamEfnTukdSbDKTAJVM,5432
 thds/core/tmp.py,sha256=KgBAwQCmpm7I762eLRu-3MSfH3dKnqlrJkZ5nmPcRbc,3110
-thds/core/types.py,sha256=Pbo5bF3b14V8eeabpnffWOPJDAGrMcKBCh5e-ZvipJk,70
+thds/core/types.py,sha256=sFqI_8BsB1u85PSizjBZw8PBtplC7U54E19wZZWCEvI,152
 thds/core/log/__init__.py,sha256=bDbZvlxyymY6VrQzD8lCn0egniLEiA9hpNMAXZ7e7wY,1348
 thds/core/log/basic_config.py,sha256=2Y9U_c4PTrIsCmaN7Ps6Xr90AhJPzdYjeUzUMqO7oFU,6704
 thds/core/log/json_formatter.py,sha256=C5bRsSbAqaQqfTm88jc3mYe3vwKZZLAxET8s7_u7aN0,1757
@@ -46,7 +47,7 @@ thds/core/log/kw_formatter.py,sha256=9-MVOd2r5NEkYNne9qWyFMeR5lac3w7mjHXsDa681i0
 thds/core/log/kw_logger.py,sha256=CyZVPnkUMtrUL2Lyk261AIEPmoP-buf_suFAhQlU1io,4063
 thds/core/log/logfmt.py,sha256=i66zoG2oERnE1P_0TVXdlfJ1YgUmvtMjqRtdV5u2SvU,10366
 thds/core/source/__init__.py,sha256=RiaUHNunoaw4XJUrwR5vJzSS6HGxOUKUONR_ipX5654,424
-thds/core/source/_construct.py,sha256=klN6-fSJrsbbUhp92wzhJcF73h_PKKJItNLC__vwlIs,3122
+thds/core/source/_construct.py,sha256=plSyQZRe8h0X7PpbfMjhm1qkFgrcyuSrReyWQo28YfA,3121
 thds/core/source/_download.py,sha256=pUhkphHdB7y4ZpxZZ6ITIS5giXMHuRf420yYAJwx6aE,2924
 thds/core/source/serde.py,sha256=wXCfuv_Dv3QvJJr-uebGmTrfhCU_1a8VX3VJnXhVHfU,3539
 thds/core/source/src.py,sha256=9A_8kSBUc5k6OLAYe5EW_VogpXFIqofow7Rxl8xv-eg,4559
@@ -66,8 +67,8 @@ thds/core/sqlite/structured.py,sha256=swCbDoyVT6cE7Kl79Wh_rg5Z1-yrUDJbiVJF4bjset
 thds/core/sqlite/types.py,sha256=oUkfoKRYNGDPZRk29s09rc9ha3SCk2SKr_K6WKebBFs,1308
 thds/core/sqlite/upsert.py,sha256=BmKK6fsGVedt43iY-Lp7dnAu8aJ1e9CYlPVEQR2pMj4,5827
 thds/core/sqlite/write.py,sha256=z0219vDkQDCnsV0WLvsj94keItr7H4j7Y_evbcoBrWU,3458
-thds_core-1.32.20250314195422.dist-info/METADATA,sha256=E2BawnpfMNwC81fPL37Zz_ppAvYFAJvfF6jkQGOJNVE,2277
-thds_core-1.32.20250314195422.dist-info/WHEEL,sha256=iAkIy5fosb7FzIOwONchHf19Qu7_1wCWyFNR5gu9nU0,91
-thds_core-1.32.20250314195422.dist-info/entry_points.txt,sha256=bOCOVhKZv7azF3FvaWX6uxE6yrjK6FcjqhtxXvLiFY8,161
-thds_core-1.32.20250314195422.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
-thds_core-1.32.20250314195422.dist-info/RECORD,,
+thds_core-1.32.20250320220920.dist-info/METADATA,sha256=YAgWdsMiy0CTTliCrLcJF-eEOP0i7culSPmTQ35Zquk,2277
+thds_core-1.32.20250320220920.dist-info/WHEEL,sha256=iAkIy5fosb7FzIOwONchHf19Qu7_1wCWyFNR5gu9nU0,91
+thds_core-1.32.20250320220920.dist-info/entry_points.txt,sha256=bOCOVhKZv7azF3FvaWX6uxE6yrjK6FcjqhtxXvLiFY8,161
+thds_core-1.32.20250320220920.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
+thds_core-1.32.20250320220920.dist-info/RECORD,,