dycw-utilities 0.146.2__py3-none-any.whl → 0.178.1__py3-none-any.whl

utilities/pottery.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from contextlib import nullcontext, suppress
+from contextlib import suppress
 from dataclasses import dataclass
 from sys import maxsize
 from typing import TYPE_CHECKING, override
@@ -9,23 +9,19 @@ from pottery import AIORedlock
 from pottery.exceptions import ReleaseUnlockedLock
 from redis.asyncio import Redis
 
-from utilities.asyncio import loop_until_succeed, sleep_td, timeout_td
+from utilities.asyncio import sleep_td, timeout_td
 from utilities.contextlib import enhanced_async_context_manager
-from utilities.functools import partial
 from utilities.iterables import always_iterable
-from utilities.logging import get_logger
-from utilities.warnings import suppress_warnings
-from utilities.whenever import MILLISECOND, SECOND, to_seconds
+from utilities.whenever import MILLISECOND, SECOND, to_nanoseconds
 
 if TYPE_CHECKING:
-    from collections.abc import AsyncIterator, Callable, Iterable
+    from collections.abc import AsyncIterator, Iterable
 
     from whenever import Delta
 
-    from utilities.types import Coro, LoggerOrName, MaybeIterable
+    from utilities.types import MaybeIterable
 
 _NUM: int = 1
-_TIMEOUT_TRY_ACQUIRE: Delta = SECOND
 _TIMEOUT_RELEASE: Delta = 10 * SECOND
 _SLEEP: Delta = MILLISECOND
 
@@ -44,74 +40,6 @@ async def extend_lock(
 ##
 
 
-@enhanced_async_context_manager
-async def try_yield_coroutine_looper(
-    redis: MaybeIterable[Redis],
-    key: str,
-    /,
-    *,
-    num: int = _NUM,
-    timeout_release: Delta = _TIMEOUT_RELEASE,
-    num_extensions: int | None = None,
-    timeout_acquire: Delta = _TIMEOUT_TRY_ACQUIRE,
-    sleep_acquire: Delta = _SLEEP,
-    throttle: Delta | None = None,
-    logger: LoggerOrName | None = None,
-    sleep_error: Delta | None = None,
-) -> AsyncIterator[CoroutineLooper | None]:
-    """Try acquire access to a coroutine looper."""
-    try:  # skipif-ci-and-not-linux
-        async with yield_access(
-            redis,
-            key,
-            num=num,
-            timeout_release=timeout_release,
-            num_extensions=num_extensions,
-            timeout_acquire=timeout_acquire,
-            sleep=sleep_acquire,
-            throttle=throttle,
-        ) as lock:
-            yield CoroutineLooper(lock=lock, logger=logger, sleep=sleep_error)
-    except _YieldAccessUnableToAcquireLockError as error:  # skipif-ci-and-not-linux
-        if logger is not None:
-            get_logger(logger=logger).info("%s", error)
-        async with nullcontext():
-            yield
-
-
-@dataclass(order=True, unsafe_hash=True, kw_only=True)
-class CoroutineLooper:
-    """Looper, guarded by a lock, to repeatedly call a coroutine until it succeeds."""
-
-    lock: AIORedlock
-    logger: LoggerOrName | None = None
-    sleep: Delta | None = None
-
-    async def __call__[**P](
-        self, func: Callable[P, Coro[None]], *args: P.args, **kwargs: P.kwargs
-    ) -> None:
-        def make_coro() -> Coro[None]:
-            return func(*args, **kwargs)
-
-        await loop_until_succeed(
-            make_coro, error=partial(self._error, func=make_coro), sleep=self.sleep
-        )
-
-    def _error(self, error: Exception, /, *, func: Callable[[], Coro[None]]) -> None:
-        _ = error
-        if self.logger is not None:
-            coro = func()
-            name = coro.__name__  # skipif-ci-and-not-linux
-            with suppress_warnings(
-                message="coroutine '.*' was never awaited", category=RuntimeWarning
-            ):
-                del coro
-            get_logger(logger=self.logger).error("Error running %r", name)
-
-
-##
-
-
 @enhanced_async_context_manager
 async def yield_access(
     redis: MaybeIterable[Redis],
@@ -135,7 +63,7 @@ async def yield_access(
         AIORedlock(
             key=f"{key}_{i}_of_{num}",
             masters=masters,
-            auto_release_time=to_seconds(timeout_release),
+            auto_release_time=to_nanoseconds(timeout_release) / 1e9,
             num_extensions=maxsize if num_extensions is None else num_extensions,
         )
         for i in range(1, num + 1)
@@ -185,11 +113,12 @@ async def _get_first_available_lock_if_any(
 @dataclass(kw_only=True, slots=True)
 class YieldAccessError(Exception):
     key: str
-    num: int
 
 
 @dataclass(kw_only=True, slots=True)
 class _YieldAccessNumLocksError(YieldAccessError):
+    num: int
+
     @override
     def __str__(self) -> str:
         return f"Number of locks for {self.key!r} must be positive; got {self.num}"
@@ -197,6 +126,7 @@ class _YieldAccessNumLocksError(YieldAccessError):
 
 @dataclass(kw_only=True, slots=True)
 class _YieldAccessUnableToAcquireLockError(YieldAccessError):
+    num: int
     timeout: Delta | None
 
     @override
@@ -204,10 +134,4 @@ class _YieldAccessUnableToAcquireLockError(YieldAccessError):
         return f"Unable to acquire any 1 of {self.num} locks for {self.key!r} after {self.timeout}"  # skipif-ci-and-not-linux
 
 
-__all__ = [
-    "CoroutineLooper",
-    "YieldAccessError",
-    "extend_lock",
-    "try_yield_coroutine_looper",
-    "yield_access",
-]
+__all__ = ["YieldAccessError", "extend_lock", "yield_access"]

utilities/pqdm.py

@@ -9,7 +9,7 @@ from tqdm.auto import tqdm as tqdm_auto
 from utilities.functions import get_func_name
 from utilities.iterables import apply_to_varargs
 from utilities.os import get_cpu_use
-from utilities.sentinel import Sentinel, sentinel
+from utilities.sentinel import Sentinel, is_sentinel, sentinel
 
 if TYPE_CHECKING:
     from collections.abc import Callable, Iterable
@@ -90,7 +90,7 @@ def pqdm_starmap[T](
                 **_get_desc(desc, func),
                 **kwargs,
             )
-        case _ as never:
+        case never:
             assert_never(never)
     return list(result)
 
@@ -98,7 +98,7 @@ def pqdm_starmap[T](
 def _get_desc(
     desc: str | None | Sentinel, func: Callable[..., Any], /
 ) -> dict[str, str]:
-    desc_use = get_func_name(func) if isinstance(desc, Sentinel) else desc
+    desc_use = get_func_name(func) if is_sentinel(desc) else desc
     return {} if desc_use is None else {"desc": desc_use}
 
 

utilities/pwd.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+from typing import assert_never
+
+from utilities.os import EFFECTIVE_USER_ID
+from utilities.platform import SYSTEM
+
+
+def get_uid_name(uid: int, /) -> str | None:
+    """Get the name of a user ID."""
+    match SYSTEM:
+        case "windows":  # skipif-not-windows
+            return None
+        case "mac" | "linux":  # skipif-windows
+            from pwd import getpwuid
+
+            return getpwuid(uid).pw_name
+        case never:
+            assert_never(never)
+
+
+ROOT_USER_NAME = get_uid_name(0)
+EFFECTIVE_USER_NAME = (
+    None if EFFECTIVE_USER_ID is None else get_uid_name(EFFECTIVE_USER_ID)
+)
+
+
+__all__ = ["EFFECTIVE_USER_NAME", "ROOT_USER_NAME", "get_uid_name"]

utilities/pydantic.py

@@ -1,58 +1,11 @@
 from __future__ import annotations
 
-from dataclasses import dataclass
 from pathlib import Path
-from typing import TYPE_CHECKING, override
+from typing import Annotated
 
-from pydantic import BaseModel
+from pydantic import BeforeValidator
 
-from utilities.atomicwrites import writer
+ExpandedPath = Annotated[Path, BeforeValidator(lambda p: Path(p).expanduser())]
 
-if TYPE_CHECKING:
-    from utilities.types import PathLike
 
-
-class HashableBaseModel(BaseModel):
-    """Subclass of BaseModel which is hashable."""
-
-    @override
-    def __hash__(self) -> int:
-        return hash((type(self), *self.__dict__.values()))
-
-
-def load_model[T: BaseModel](model: type[T], path: PathLike, /) -> T:
-    path = Path(path)
-    try:
-        return model.model_validate_json(path.read_text())
-    except FileNotFoundError:
-        raise _LoadModelFileNotFoundError(model=model, path=path) from None
-    except IsADirectoryError:  # skipif-not-windows
-        raise _LoadModelIsADirectoryError(model=model, path=path) from None
-
-
-@dataclass(kw_only=True, slots=True)
-class LoadModelError(Exception):
-    model: type[BaseModel]
-    path: Path
-
-
-@dataclass(kw_only=True, slots=True)
-class _LoadModelFileNotFoundError(LoadModelError):
-    @override
-    def __str__(self) -> str:
-        return f"Unable to load {self.model}; path {str(self.path)!r} must exist."
-
-
-@dataclass(kw_only=True, slots=True)
-class _LoadModelIsADirectoryError(LoadModelError):
-    @override
-    def __str__(self) -> str:
-        return f"Unable to load {self.model}; path {str(self.path)!r} must not be a directory."  # skipif-not-windows
-
-
-def save_model(model: BaseModel, path: PathLike, /, *, overwrite: bool = False) -> None:
-    with writer(path, overwrite=overwrite) as temp:
-        _ = temp.write_text(model.model_dump_json())
-
-
-__all__ = ["HashableBaseModel", "LoadModelError", "load_model", "save_model"]
+__all__ = ["ExpandedPath"]

utilities/pydantic_settings.py

@@ -0,0 +1,240 @@
+from __future__ import annotations
+
+from functools import reduce
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, ClassVar, assert_never, cast, override
+
+from pydantic import Field, create_model
+from pydantic_settings import (
+    BaseSettings,
+    CliSettingsSource,
+    JsonConfigSettingsSource,
+    PydanticBaseSettingsSource,
+    SettingsConfigDict,
+    TomlConfigSettingsSource,
+    YamlConfigSettingsSource,
+)
+from pydantic_settings.sources import DEFAULT_PATH
+
+from utilities.errors import ImpossibleCaseError
+from utilities.iterables import always_iterable
+
+if TYPE_CHECKING:
+    from collections.abc import Iterator, Sequence
+
+    from pydantic_settings.sources import PathType
+
+    from utilities.types import MaybeSequenceStr, PathLike
+
+
+type PathLikeWithSection = tuple[PathLike, MaybeSequenceStr]
+type PathLikeOrWithSection = PathLike | PathLikeWithSection
+
+
+class CustomBaseSettings(BaseSettings):
+    """Base settings for loading JSON/TOML/YAML files."""
+
+    # paths
+    json_files: ClassVar[Sequence[PathLikeOrWithSection]] = []
+    toml_files: ClassVar[Sequence[PathLikeOrWithSection]] = []
+    yaml_files: ClassVar[Sequence[PathLikeOrWithSection]] = []
+
+    # config
+    model_config: ClassVar[SettingsConfigDict] = SettingsConfigDict(
+        frozen=True, env_nested_delimiter="__"
+    )
+
+    @classmethod
+    @override
+    def settings_customise_sources(
+        cls,
+        settings_cls: type[BaseSettings],
+        init_settings: PydanticBaseSettingsSource,
+        env_settings: PydanticBaseSettingsSource,
+        dotenv_settings: PydanticBaseSettingsSource,
+        file_secret_settings: PydanticBaseSettingsSource,
+    ) -> tuple[PydanticBaseSettingsSource, ...]:
+        _ = (init_settings, dotenv_settings, file_secret_settings)
+        return tuple(cls._yield_base_settings_sources(settings_cls, env_settings))
+
+    @classmethod
+    def _yield_base_settings_sources(
+        cls,
+        settings_cls: type[BaseSettings],
+        env_settings: PydanticBaseSettingsSource,
+        /,
+    ) -> Iterator[PydanticBaseSettingsSource]:
+        yield env_settings
+        for file, section in map(_ensure_section, cls.json_files):
+            yield JsonConfigSectionSettingsSource(
+                settings_cls, json_file=file, section=section
+            )
+        for file, section in map(_ensure_section, cls.toml_files):
+            yield TomlConfigSectionSettingsSource(
+                settings_cls, toml_file=file, section=section
+            )
+        for file, section in map(_ensure_section, cls.yaml_files):
+            yield YamlConfigSectionSettingsSource(
+                settings_cls, yaml_file=file, section=section
+            )
+
+
+class JsonConfigSectionSettingsSource(JsonConfigSettingsSource):
+    @override
+    def __init__(
+        self,
+        settings_cls: type[BaseSettings],
+        json_file: PathType | None = DEFAULT_PATH,
+        json_file_encoding: str | None = None,
+        *,
+        section: MaybeSequenceStr,
+    ) -> None:
+        super().__init__(
+            settings_cls, json_file=json_file, json_file_encoding=json_file_encoding
+        )
+        self.section = section
+
+    @override
+    def __call__(self) -> dict[str, Any]:
+        return _get_section(super().__call__(), self.section)
+
+
+class TomlConfigSectionSettingsSource(TomlConfigSettingsSource):
+    @override
+    def __init__(
+        self,
+        settings_cls: type[BaseSettings],
+        toml_file: PathType | None = DEFAULT_PATH,
+        *,
+        section: MaybeSequenceStr,
+    ) -> None:
+        super().__init__(settings_cls, toml_file=toml_file)
+        self.section = section
+
+    @override
+    def __call__(self) -> dict[str, Any]:
+        return _get_section(super().__call__(), self.section)
+
+
+class YamlConfigSectionSettingsSource(YamlConfigSettingsSource):
+    @override
+    def __init__(
+        self,
+        settings_cls: type[BaseSettings],
+        yaml_file: PathType | None = DEFAULT_PATH,
+        yaml_file_encoding: str | None = None,
+        yaml_config_section: str | None = None,
+        *,
+        section: MaybeSequenceStr,
+    ) -> None:
+        super().__init__(
+            settings_cls,
+            yaml_file=yaml_file,
+            yaml_file_encoding=yaml_file_encoding,
+            yaml_config_section=yaml_config_section,
+        )
+        self.section = section
+
+    @override
+    def __call__(self) -> dict[str, Any]:
+        return _get_section(super().__call__(), self.section)
+
+
+def _ensure_section(file: PathLikeOrWithSection, /) -> PathLikeWithSection:
+    match file:
+        case Path() | str():
+            return file, []
+        case Path() | str() as path, str() | list() | tuple() as section:
+            return path, section
+        case never:
+            assert_never(never)
+
+
+def _get_section(
+    mapping: dict[str, Any], section: MaybeSequenceStr, /
+) -> dict[str, Any]:
+    return reduce(lambda acc, el: acc.get(el, {}), always_iterable(section), mapping)
+
+
+##
+
+
+class HashableBaseSettings(BaseSettings):
+    """Base settings for loading JSON files."""
+
+    # config
+    model_config: ClassVar[SettingsConfigDict] = SettingsConfigDict(frozen=True)
+
+
+##
+
+
+def load_settings[T: BaseSettings](cls: type[T], /, *, cli: bool = False) -> T:
+    """Load a set of settings."""
+    _ = cls.model_rebuild()
+    if cli:
+        cls_with_defaults = _load_settings_create_model(cls)
+
+        @classmethod
+        def settings_customise_sources(
+            cls: type[BaseSettings],
+            settings_cls: type[BaseSettings],
+            init_settings: PydanticBaseSettingsSource,
+            env_settings: PydanticBaseSettingsSource,
+            dotenv_settings: PydanticBaseSettingsSource,
+            file_secret_settings: PydanticBaseSettingsSource,
+        ) -> tuple[PydanticBaseSettingsSource, ...]:
+            parent = cast(
+                "Any", super(cls_with_defaults, cls)
+            ).settings_customise_sources(
+                settings_cls=settings_cls,
+                init_settings=init_settings,
+                env_settings=env_settings,
+                dotenv_settings=dotenv_settings,
+                file_secret_settings=file_secret_settings,
+            )
+            return (
+                CliSettingsSource(
+                    settings_cls, cli_parse_args=True, case_sensitive=False
+                ),
+                *parent,
+            )
+
+        cls_use = type(
+            cls.__name__,
+            (cls_with_defaults,),
+            {"settings_customise_sources": settings_customise_sources},
+        )
+        cls_use = cast("type[T]", cls_use)
+    else:
+        cls_use = cls
+    return cls_use()
+
+
+def _load_settings_create_model[T: BaseSettings](
+    cls: type[T], /, *, values: T | None = None
+) -> type[T]:
+    values_use = cls() if values is None else values
+    kwargs: dict[str, Any] = {}
+    for name, field in cls.model_fields.items():
+        if (ann := field.annotation) is None:
+            raise ImpossibleCaseError(case=[f"{ann=}"])  # pragma: no cover
+        value = getattr(values_use, name)
+        if (
+            isinstance(cast("Any", ann), type)  # 'ann' is possible not a type
+            and issubclass(ann, BaseSettings)
+        ):
+            kwargs[name] = _load_settings_create_model(ann, values=value)
+        else:
+            kwargs[name] = (field.annotation, Field(default=value))
+    return create_model(cls.__name__, __base__=cls, **kwargs)
+
+
+__all__ = [
+    "CustomBaseSettings",
+    "HashableBaseSettings",
+    "JsonConfigSectionSettingsSource",
+    "TomlConfigSectionSettingsSource",
+    "YamlConfigSectionSettingsSource",
+    "load_settings",
+]

utilities/pydantic_settings_sops.py

@@ -0,0 +1,76 @@
+from __future__ import annotations
+
+from logging import Filter, LogRecord, getLogger
+from re import search
+from typing import TYPE_CHECKING, Any, ClassVar, override
+
+from pydantic_settings.sources import DEFAULT_PATH
+from pydantic_settings_sops import SOPSConfigSettingsSource
+
+from utilities.pydantic_settings import (
+    CustomBaseSettings,
+    PathLikeOrWithSection,
+    _ensure_section,
+    _get_section,
+)
+
+if TYPE_CHECKING:
+    from collections.abc import Iterator, Sequence
+
+    from pydantic_settings import BaseSettings, PydanticBaseSettingsSource
+    from pydantic_settings.sources import PathType
+
+    from utilities.types import MaybeSequenceStr
+
+
+class _SuppressDefaultConfigMessage(Filter):
+    @override
+    def filter(self, record: LogRecord) -> bool:
+        return not search(
+            r"^default config file does not exists '.*'$", record.getMessage()
+        )
+
+
+getLogger("sopsy.utils").addFilter(_SuppressDefaultConfigMessage())
+
+
+class SopsBaseSettings(CustomBaseSettings):
+    """Base settings for loading secrets using `sops/age`."""
+
+    # paths
+    secret_files: ClassVar[Sequence[PathLikeOrWithSection]] = []
+
+    @classmethod
+    @override
+    def _yield_base_settings_sources(
+        cls,
+        settings_cls: type[BaseSettings],
+        env_settings: PydanticBaseSettingsSource,
+        /,
+    ) -> Iterator[PydanticBaseSettingsSource]:
+        yield from super()._yield_base_settings_sources(settings_cls, env_settings)
+        for file, section in map(_ensure_section, cls.secret_files):
+            yield SOPSConfigSectionSettingsSource(
+                settings_cls, json_file=file, section=section
+            )
+
+
+class SOPSConfigSectionSettingsSource(SOPSConfigSettingsSource):
+    @override
+    def __init__(
+        self,
+        settings_cls: type[BaseSettings],
+        json_file: PathType | None = DEFAULT_PATH,
+        yaml_file: PathType | None = DEFAULT_PATH,
+        *,
+        section: MaybeSequenceStr,
+    ) -> None:
+        super().__init__(settings_cls, json_file=json_file, yaml_file=yaml_file)  # pyright: ignore[reportArgumentType]
+        self.section = section
+
+    @override
+    def __call__(self) -> dict[str, Any]:
+        return _get_section(super().__call__(), self.section)
+
+
+__all__ = ["SOPSConfigSectionSettingsSource", "SopsBaseSettings"]

utilities/pyinstrument.py

@@ -7,8 +7,8 @@ from typing import TYPE_CHECKING
 from pyinstrument.profiler import Profiler
 
 from utilities.atomicwrites import writer
-from utilities.pathlib import get_path
-from utilities.whenever import format_compact, get_now, to_local_plain
+from utilities.pathlib import to_path
+from utilities.whenever import format_compact, get_now_local
 
 if TYPE_CHECKING:
     from collections.abc import Iterator
@@ -17,12 +17,12 @@ if TYPE_CHECKING:
 
 
 @contextmanager
-def profile(*, path: MaybeCallablePathLike | None = Path.cwd) -> Iterator[None]:
+def profile(path: MaybeCallablePathLike = Path.cwd, /) -> Iterator[None]:
     """Profile the contents of a block."""
     with Profiler() as profiler:
         yield
-    filename = get_path(path=path).joinpath(
-        f"profile__{format_compact(to_local_plain(get_now()))}.html"
+    filename = to_path(path).joinpath(
+        f"profile__{format_compact(get_now_local(), path=True)}.html"
     )
     with writer(filename) as temp:
         _ = temp.write_text(profiler.output_html())