dycw-utilities 0.166.30__py3-none-any.whl → 0.185.8__py3-none-any.whl

utilities/pydantic_settings.py

@@ -2,10 +2,12 @@ from __future__ import annotations
 
 from functools import reduce
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, ClassVar, assert_never, override
+from typing import TYPE_CHECKING, Any, ClassVar, assert_never, cast, override
 
+from pydantic import Field, create_model
 from pydantic_settings import (
     BaseSettings,
+    CliSettingsSource,
     JsonConfigSettingsSource,
     PydanticBaseSettingsSource,
     SettingsConfigDict,
@@ -14,14 +16,15 @@ from pydantic_settings import (
 )
 from pydantic_settings.sources import DEFAULT_PATH
 
-from utilities.iterables import always_iterable
+from utilities.core import always_iterable
+from utilities.errors import ImpossibleCaseError
 
 if TYPE_CHECKING:
     from collections.abc import Iterator, Sequence
 
     from pydantic_settings.sources import PathType
 
-    from utilities.types import MaybeSequenceStr, PathLike
+    from utilities.types import MaybeSequenceStr, PathLike, StrDict
 
 
 type PathLikeWithSection = tuple[PathLike, MaybeSequenceStr]
@@ -29,7 +32,7 @@ type PathLikeOrWithSection = PathLike | PathLikeWithSection
 
 
 class CustomBaseSettings(BaseSettings):
-    """Base settings for loading JSON files."""
+    """Base settings for loading JSON/TOML/YAML files."""
 
     # paths
     json_files: ClassVar[Sequence[PathLikeOrWithSection]] = []
@@ -38,7 +41,7 @@ class CustomBaseSettings(BaseSettings):
 
     # config
     model_config: ClassVar[SettingsConfigDict] = SettingsConfigDict(
-        env_nested_delimiter="__"
+        frozen=True, env_nested_delimiter="__"
     )
 
     @classmethod
@@ -76,11 +79,6 @@ class CustomBaseSettings(BaseSettings):
             )
 
 
-def load_settings[T: BaseSettings](cls: type[T], /) -> T:
-    """Load a set of settings."""
-    return cls()
-
-
 class JsonConfigSectionSettingsSource(JsonConfigSettingsSource):
     @override
     def __init__(
@@ -97,7 +95,7 @@ class JsonConfigSectionSettingsSource(JsonConfigSettingsSource):
         self.section = section
 
     @override
-    def __call__(self) -> dict[str, Any]:
+    def __call__(self) -> StrDict:
         return _get_section(super().__call__(), self.section)
 
 
@@ -114,7 +112,7 @@ class TomlConfigSectionSettingsSource(TomlConfigSettingsSource):
         self.section = section
 
     @override
-    def __call__(self) -> dict[str, Any]:
+    def __call__(self) -> StrDict:
         return _get_section(super().__call__(), self.section)
 
 
@@ -138,7 +136,7 @@ class YamlConfigSectionSettingsSource(YamlConfigSettingsSource):
         self.section = section
 
     @override
-    def __call__(self) -> dict[str, Any]:
+    def __call__(self) -> StrDict:
         return _get_section(super().__call__(), self.section)
 
 
@@ -152,14 +150,87 @@ def _ensure_section(file: PathLikeOrWithSection, /) -> PathLikeWithSection:
             assert_never(never)
 
 
-def _get_section(
-    mapping: dict[str, Any], section: MaybeSequenceStr, /
-) -> dict[str, Any]:
+def _get_section(mapping: StrDict, section: MaybeSequenceStr, /) -> StrDict:
     return reduce(lambda acc, el: acc.get(el, {}), always_iterable(section), mapping)
 
 
+##
+
+
+class HashableBaseSettings(BaseSettings):
+    """Base settings for loading JSON files."""
+
+    # config
+    model_config: ClassVar[SettingsConfigDict] = SettingsConfigDict(frozen=True)
+
+
+##
+
+
+def load_settings[T: BaseSettings](cls: type[T], /, *, cli: bool = False) -> T:
+    """Load a set of settings."""
+    _ = cls.model_rebuild()
+    if cli:
+        cls_with_defaults = _load_settings_create_model(cls)
+
+        @classmethod
+        def settings_customise_sources(
+            cls: type[BaseSettings],
+            settings_cls: type[BaseSettings],
+            init_settings: PydanticBaseSettingsSource,
+            env_settings: PydanticBaseSettingsSource,
+            dotenv_settings: PydanticBaseSettingsSource,
+            file_secret_settings: PydanticBaseSettingsSource,
+        ) -> tuple[PydanticBaseSettingsSource, ...]:
+            parent = cast(
+                "Any", super(cls_with_defaults, cls)
+            ).settings_customise_sources(
+                settings_cls=settings_cls,
+                init_settings=init_settings,
+                env_settings=env_settings,
+                dotenv_settings=dotenv_settings,
+                file_secret_settings=file_secret_settings,
+            )
+            return (
+                CliSettingsSource(
+                    settings_cls, cli_parse_args=True, case_sensitive=False
+                ),
+                *parent,
+            )
+
+        cls_use = type(
+            cls.__name__,
+            (cls_with_defaults,),
+            {"settings_customise_sources": settings_customise_sources},
+        )
+        cls_use = cast("type[T]", cls_use)
+    else:
+        cls_use = cls
+    return cls_use()
+
+
+def _load_settings_create_model[T: BaseSettings](
+    cls: type[T], /, *, values: T | None = None
+) -> type[T]:
+    values_use = cls() if values is None else values
+    kwargs: StrDict = {}
+    for name, field in cls.model_fields.items():
+        if (ann := field.annotation) is None:
+            raise ImpossibleCaseError(case=[f"{ann=}"])  # pragma: no cover
+        value = getattr(values_use, name)
+        if (
+            isinstance(cast("Any", ann), type)  # 'ann' is possible not a type
+            and issubclass(ann, BaseSettings)
+        ):
+            kwargs[name] = _load_settings_create_model(ann, values=value)
+        else:
+            kwargs[name] = (field.annotation, Field(default=value))
+    return create_model(cls.__name__, __base__=cls, **kwargs)
+
+
 __all__ = [
     "CustomBaseSettings",
+    "HashableBaseSettings",
     "JsonConfigSectionSettingsSource",
     "TomlConfigSectionSettingsSource",
     "YamlConfigSectionSettingsSource",

utilities/pydantic_settings_sops.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any, ClassVar, override
+from logging import Filter, LogRecord, getLogger
+from re import search
+from typing import TYPE_CHECKING, ClassVar, override
 
 from pydantic_settings.sources import DEFAULT_PATH
 from pydantic_settings_sops import SOPSConfigSettingsSource
@@ -18,7 +20,18 @@ if TYPE_CHECKING:
     from pydantic_settings import BaseSettings, PydanticBaseSettingsSource
     from pydantic_settings.sources import PathType
 
-    from utilities.types import MaybeSequenceStr
+    from utilities.types import MaybeSequenceStr, StrDict
+
+
+class _SuppressDefaultConfigMessage(Filter):
+    @override
+    def filter(self, record: LogRecord) -> bool:
+        return not search(
+            r"^default config file does not exists '.*'$", record.getMessage()
+        )
+
+
+getLogger("sopsy.utils").addFilter(_SuppressDefaultConfigMessage())
 
 
 class SopsBaseSettings(CustomBaseSettings):
@@ -56,7 +69,7 @@ class SOPSConfigSectionSettingsSource(SOPSConfigSettingsSource):
         self.section = section
 
     @override
-    def __call__(self) -> dict[str, Any]:
+    def __call__(self) -> StrDict:
         return _get_section(super().__call__(), self.section)
 
 

utilities/pyinstrument.py

@@ -6,9 +6,9 @@ from typing import TYPE_CHECKING
 
 from pyinstrument.profiler import Profiler
 
-from utilities.atomicwrites import writer
+from utilities.core import get_now_local, write_text
 from utilities.pathlib import to_path
-from utilities.whenever import format_compact, get_now_local
+from utilities.whenever import format_compact
 
 if TYPE_CHECKING:
     from collections.abc import Iterator
@@ -24,8 +24,8 @@ def profile(path: MaybeCallablePathLike = Path.cwd, /) -> Iterator[None]:
     filename = to_path(path).joinpath(
         f"profile__{format_compact(get_now_local(), path=True)}.html"
     )
-    with writer(filename) as temp:
-        _ = temp.write_text(profiler.output_html())
+    text = profiler.output_html()
+    write_text(filename, text, overwrite=True)
 
 
 __all__ = ["profile"]

utilities/pytest.py

@@ -1,63 +1,86 @@
 from __future__ import annotations
 
+from collections.abc import Callable
 from dataclasses import dataclass
 from functools import partial, wraps
 from inspect import iscoroutinefunction
 from os import environ
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, assert_never, cast, override
+from re import sub
+from types import FunctionType
+from typing import TYPE_CHECKING, Any, NoReturn, assert_never, cast, override
 
-from whenever import ZonedDateTime
-
-from utilities.atomicwrites import writer
+from utilities.constants import (
+    IS_CI,
+    IS_CI_AND_NOT_LINUX,
+    IS_LINUX,
+    IS_MAC,
+    IS_NOT_LINUX,
+    IS_NOT_MAC,
+    SECOND,
+)
 from utilities.functools import cache
 from utilities.hashlib import md5_hash
-from utilities.os import get_env_var
 from utilities.pathlib import (
     _GetTailEmptyError,
     ensure_suffix,
-    get_root,
+    get_repo_root,
     get_tail,
     module_path,
 )
-from utilities.platform import (
-    IS_LINUX,
-    IS_MAC,
-    IS_NOT_LINUX,
-    IS_NOT_MAC,
-    IS_NOT_WINDOWS,
-    IS_WINDOWS,
-)
 from utilities.random import bernoulli
 from utilities.text import to_bool
-from utilities.types import MaybeCallableBoolLike, MaybeCoro, Seed
-from utilities.whenever import SECOND, get_now_local
+from utilities.throttle import throttle
+from utilities.types import (
+    Coro,
+    Duration,
+    MaybeCallableBoolLike,
+    MaybeCoro,
+    PathLike,
+    Seed,
+)
 
 if TYPE_CHECKING:
-    from collections.abc import Callable, Iterable
-
-    from utilities.types import Coro, Delta, PathLike
+    from collections.abc import Iterable
 
-try:  # WARNING: this package cannot use unguarded `pytest` imports
     from _pytest.config import Config
     from _pytest.config.argparsing import Parser
     from _pytest.python import Function
-    from pytest import mark, skip
+
+    from utilities.types import PathLike
+
+
+try:  # WARNING: this package cannot use unguarded `pytest` imports
+    from pytest import mark
 except ModuleNotFoundError:  # pragma: no cover
-    from typing import Any as Config
-    from typing import Any as Function
-    from typing import Any as Parser
 
-    mark = skip = skipif_windows = skipif_mac = skipif_linux = skipif_not_windows = (
-        skipif_not_mac
-    ) = skipif_not_linux = None
+    def skipif_ci[F: Callable](func: F) -> F:
+        return func
+
+    def skipif_mac[F: Callable](func: F) -> F:
+        return func
+
+    def skipif_linux[F: Callable](func: F) -> F:
+        return func
+
+    def skipif_not_mac[F: Callable](func: F) -> F:
+        return func
+
+    def skipif_not_linux[F: Callable](func: F) -> F:
+        return func
+
+    def skipif_ci_and_not_linux[F: Callable](func: F) -> F:
+        return func
+
 else:
-    skipif_windows = mark.skipif(IS_WINDOWS, reason="Skipped for Windows")
+    skipif_ci = mark.skipif(IS_CI, reason="Skipped for CI")
     skipif_mac = mark.skipif(IS_MAC, reason="Skipped for Mac")
     skipif_linux = mark.skipif(IS_LINUX, reason="Skipped for Linux")
-    skipif_not_windows = mark.skipif(IS_NOT_WINDOWS, reason="Skipped for non-Windows")
     skipif_not_mac = mark.skipif(IS_NOT_MAC, reason="Skipped for non-Mac")
     skipif_not_linux = mark.skipif(IS_NOT_LINUX, reason="Skipped for non-Linux")
+    skipif_ci_and_not_linux = mark.skipif(
+        IS_CI_AND_NOT_LINUX, reason="Skipped for CI/non-Linux"
+    )
 
 
 def add_pytest_addoption(parser: Parser, options: list[str], /) -> None:
@@ -90,6 +113,8 @@ def add_pytest_collection_modifyitems(
         def pytest_collection_modifyitems(config, items):
             add_pytest_collection_modifyitems(config, items, ["slow"])
     """
+    from pytest import mark
+
     options = list(options)
     missing = {opt for opt in options if not config.getoption(f"--{opt}")}
     for item in items:
@@ -119,6 +144,15 @@ def add_pytest_configure(config: Config, options: Iterable[tuple[str, str]], /)
 ##
 
 
+def make_ids(obj: Any, /) -> str:
+    if isinstance(obj, FunctionType):  # pragma: no cover
+        return sub(r"\s+at +0x[0-9a-fA-F]+", "", repr(obj))
+    return repr(obj)  # pragma: no cover
+
+
+##
+
+
 def node_id_path(
     node_id: str, /, *, root: PathLike | None = None, suffix: str | None = None
 ) -> Path:
@@ -168,7 +202,7 @@ class _NodeIdToPathNotGetTailError(NodeIdToPathError):
 ##
 
 
-def run_frac[F: Callable[..., MaybeCoro[None]]](
+def run_test_frac[F: Callable[..., MaybeCoro[None]]](
     *,
     predicate: MaybeCallableBoolLike | None = None,
     frac: float = 0.5,
@@ -176,11 +210,11 @@ def run_frac[F: Callable[..., MaybeCoro[None]]](
 ) -> Callable[[F], F]:
     """Run a test only a fraction of the time.."""
     return cast(
-        "Any", partial(_run_frac_inner, predicate=predicate, frac=frac, seed=seed)
+        "Any", partial(_run_test_frac_inner, predicate=predicate, frac=frac, seed=seed)
     )
 
 
-def _run_frac_inner[F: Callable[..., MaybeCoro[None]]](
+def _run_test_frac_inner[F: Callable[..., MaybeCoro[None]]](
     func: F,
     /,
     *,
@@ -217,132 +251,69 @@ def _skipif_frac(
     frac: float = 0.5,
     seed: Seed | None = None,
 ) -> None:
-    if skip is None:
-        return  # pragma: no cover
+    from pytest import skip
+
     if ((predicate is None) or to_bool(predicate)) and bernoulli(
         true=1 - frac, seed=seed
     ):
-        _ = skip(reason=f"{_get_name()} skipped (run {frac:.0%})")
+        skip(reason=f"{_get_name()} skipped (run {frac:.0%})")
 
 
 ##
 
 
-def throttle[F: Callable[..., MaybeCoro[None]]](
-    *, root: PathLike | None = None, delta: Delta = SECOND, on_try: bool = False
+def throttle_test[F: Callable[..., MaybeCoro[None]]](
+    *, on_try: bool = False, root: PathLike | None = None, duration: Duration = SECOND
 ) -> Callable[[F], F]:
     """Throttle a test. On success by default, on try otherwise."""
-    return cast("Any", partial(_throttle_inner, root=root, delta=delta, on_try=on_try))
-
-
-def _throttle_inner[F: Callable[..., MaybeCoro[None]]](
-    func: F,
-    /,
-    *,
-    root: PathLike | None = None,
-    delta: Delta = SECOND,
-    on_try: bool = False,
-) -> F:
-    if get_env_var("THROTTLE", nullable=True) is not None:
-        return func
-    match bool(iscoroutinefunction(func)), on_try:
-        case False, False:
-
-            @wraps(func)
-            def throttle_sync_on_pass(*args: Any, **kwargs: Any) -> None:
-                _skipif_recent(root=root, delta=delta)
-                cast("Callable[..., None]", func)(*args, **kwargs)
-                _write(root)
-
-            return cast("Any", throttle_sync_on_pass)
-
-        case False, True:
-
-            @wraps(func)
-            def throttle_sync_on_try(*args: Any, **kwargs: Any) -> None:
-                _skipif_recent(root=root, delta=delta)
-                _write(root)
-                cast("Callable[..., None]", func)(*args, **kwargs)
-
-            return cast("Any", throttle_sync_on_try)
+    return throttle(
+        on_try=on_try,
+        duration=duration,
+        path=partial(_get_test_path, root=root),
+        raiser=_run_skip,
+    )
 
-        case True, False:
 
-            @wraps(func)
-            async def throttle_async_on_pass(*args: Any, **kwargs: Any) -> None:
-                _skipif_recent(root=root, delta=delta)
-                await cast("Callable[..., Coro[None]]", func)(*args, **kwargs)
-                _write(root)
+def _run_skip() -> NoReturn:
+    from pytest import skip
 
-            return cast("Any", throttle_async_on_pass)
+    skip(reason=f"{_get_name()} throttled")
 
-        case True, True:
 
-            @wraps(func)
-            async def throttle_async_on_try(*args: Any, **kwargs: Any) -> None:
-                _skipif_recent(root=root, delta=delta)
-                _write(root)
-                await cast("Callable[..., Coro[None]]", func)(*args, **kwargs)
+def _get_name() -> str:
+    return environ["PYTEST_CURRENT_TEST"]
 
-            return cast("Any", throttle_async_on_try)
 
-        case never:
-            assert_never(never)
+@cache
+def _md5_hash_cached(text: str, /) -> str:
+    return md5_hash(text)
 
 
-def _skipif_recent(*, root: PathLike | None = None, delta: Delta = SECOND) -> None:
-    if skip is None:
-        return  # pragma: no cover
-    path = _get_path(root)
-    try:
-        contents = path.read_text()
-    except FileNotFoundError:
-        return
-    try:
-        last = ZonedDateTime.parse_common_iso(contents)
-    except ValueError:
-        return
-    now = get_now_local()
-    if (now - delta) < last:
-        age = now - last
-        _ = skip(reason=f"{_get_name()} throttled (age {age})")
-
-
-def _get_path(root: PathLike | None = None, /) -> Path:
+def _get_test_path(*, root: PathLike | None = None) -> Path:
     if root is None:
-        root_use = get_root().joinpath(".pytest_cache", "throttle")  # pragma: no cover
+        root_use = get_repo_root().joinpath(
+            ".pytest_cache", "throttle"
+        )  # pragma: no cover
     else:
         root_use = root
     return Path(root_use, _md5_hash_cached(_get_name()))
 
 
-@cache
-def _md5_hash_cached(text: str, /) -> str:
-    return md5_hash(text)
-
-
-def _get_name() -> str:
-    return environ["PYTEST_CURRENT_TEST"]
-
-
-def _write(root: PathLike | None = None, /) -> None:
-    path = _get_path(root)
-    with writer(path, overwrite=True) as temp:
-        _ = temp.write_text(get_now_local().format_common_iso())
-
-
 __all__ = [
+    "IS_CI",
+    "IS_CI_AND_NOT_LINUX",
     "NodeIdToPathError",
     "add_pytest_addoption",
     "add_pytest_collection_modifyitems",
     "add_pytest_configure",
+    "make_ids",
     "node_id_path",
-    "run_frac",
+    "run_test_frac",
+    "skipif_ci",
+    "skipif_ci_and_not_linux",
     "skipif_linux",
     "skipif_mac",
     "skipif_not_linux",
     "skipif_not_mac",
-    "skipif_not_windows",
-    "skipif_windows",
-    "throttle",
+    "throttle_test",
 ]

utilities/pytest_plugins/pytest_regressions.py

@@ -40,7 +40,7 @@ else:
 
 
 def _get_path(request: FixtureRequest, /) -> Path:
-    from utilities.pathlib import get_root
+    from utilities.pathlib import get_repo_root
     from utilities.pytest import _NodeIdToPathNotGetTailError, node_id_path
 
     path = Path(cast("Any", request).fspath)
@@ -50,7 +50,7 @@ def _get_path(request: FixtureRequest, /) -> Path:
     except _NodeIdToPathNotGetTailError:
         root = Path("tests")
         tail = node_id_path(request.node.nodeid, root=root)
-    return get_root(path).joinpath(root, "regressions", tail)
+    return get_repo_root(path).joinpath(root, "regressions", tail)
 
 
 __all__ = ["orjson_regression", "polars_regression"]

utilities/pytest_regressions.py

@@ -1,13 +1,15 @@
 from __future__ import annotations
 
 from contextlib import suppress
+from dataclasses import dataclass
 from json import loads
 from pathlib import Path
-from shutil import copytree
-from typing import TYPE_CHECKING, Any, assert_never
+from typing import TYPE_CHECKING, Any, assert_never, override
 
+from pytest_datadir.plugin import LazyDataDir
 from pytest_regressions.file_regression import FileRegressionFixture
 
+from utilities.core import _CopyOrMoveSourceNotFoundError, copy, repr_
 from utilities.functions import ensure_str
 from utilities.operator import is_equal
 
@@ -31,10 +33,12 @@ class OrjsonRegressionFixture:
         path = Path(path)
         original_datadir = path.parent
         data_dir = tmp_path.joinpath(ensure_str(request.fixturename))
-        with suppress(FileNotFoundError):
-            _ = copytree(original_datadir, data_dir)
+        with suppress(_CopyOrMoveSourceNotFoundError):
+            copy(original_datadir, data_dir, overwrite=True)
         self._fixture = FileRegressionFixture(
-            datadir=data_dir, original_datadir=original_datadir, request=request
+            datadir=LazyDataDir(original_datadir=original_datadir, tmp_path=data_dir),
+            original_datadir=original_datadir,
+            request=request,
         )
         self._basename = path.name
 
@@ -70,10 +74,28 @@ class OrjsonRegressionFixture:
             check_fn=self._check_fn,
         )
 
-    def _check_fn(self, path1: Path, path2: Path, /) -> None:
-        left = loads(path1.read_text())
-        right = loads(path2.read_text())
-        assert is_equal(left, right), f"{left=}, {right=}"
+    def _check_fn(self, path_obtained: Path, path_existing: Path, /) -> None:
+        obtained = loads(path_obtained.read_text())
+        existing = loads(path_existing.read_text())
+        if not is_equal(obtained, existing):
+            raise OrjsonRegressionError(
+                path_obtained=path_obtained,
+                path_existing=path_existing,
+                obtained=obtained,
+                existing=existing,
+            )
+
+
+@dataclass(kw_only=True, slots=True)
+class OrjsonRegressionError(Exception):
+    path_obtained: Path
+    path_existing: Path
+    obtained: Any
+    existing: Any
+
+    @override
+    def __str__(self) -> str:
+        return f"Obtained object (at {str(self.path_obtained)!r}) and existing object (at {str(self.path_existing)!r}) differ; got {repr_(self.obtained)} and {repr_(self.existing)}"
 
 
 ##
@@ -97,7 +119,6 @@ class PolarsRegressionFixture:
             "describe": obj.describe(percentiles=[i / 10 for i in range(1, 10)]).rows(
                 named=True
             ),
-            "estimated_size": obj.estimated_size(),
             "is_empty": obj.is_empty(),
             "n_unique": obj.n_unique(),
         }
@@ -115,7 +136,7 @@ class PolarsRegressionFixture:
                             col(column).approx_n_unique()
                         ).item()
                 data["approx_n_unique"] = approx_n_unique
-                data["glimpse"] = df.glimpse(return_as_string=True)
+                data["glimpse"] = df.glimpse(return_type="string")
                 data["null_count"] = df.null_count().row(0, named=True)
             case never:
                 assert_never(never)

utilities/random.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from random import Random, SystemRandom
+from random import Random
 from typing import TYPE_CHECKING
 
 from utilities.functools import cache
@@ -11,12 +11,6 @@ if TYPE_CHECKING:
     from utilities.types import Seed
 
 
-SYSTEM_RANDOM = SystemRandom()
-
-
-##
-
-
 def bernoulli(*, true: float = 0.5, seed: Seed | None = None) -> bool:
     """Return a Bernoulli random variate."""
     state = get_state(seed)
@@ -66,4 +60,4 @@ def shuffle[T](iterable: Iterable[T], /, *, seed: Seed | None = None) -> list[T]
     return copy
 
 
-__all__ = ["SYSTEM_RANDOM", "bernoulli", "get_docker_name", "get_state", "shuffle"]
+__all__ = ["bernoulli", "get_docker_name", "get_state", "shuffle"]