dycw-utilities 0.166.30__py3-none-any.whl → 0.185.8__py3-none-any.whl

utilities/polars.py

@@ -54,25 +54,31 @@ from polars.testing import assert_frame_equal, assert_series_equal
 from whenever import DateDelta, DateTimeDelta, PlainDateTime, TimeDelta, ZonedDateTime
 
 import utilities.math
+from utilities.constants import UTC
+from utilities.core import (
+    OneEmptyError,
+    OneNonUniqueError,
+    always_iterable,
+    one,
+    read_bytes,
+    repr_,
+    suppress_warnings,
+    to_time_zone_name,
+    write_bytes,
+)
 from utilities.dataclasses import yield_fields
 from utilities.errors import ImpossibleCaseError
 from utilities.functions import get_class_name
-from utilities.gzip import read_binary
 from utilities.iterables import (
     CheckIterablesEqualError,
     CheckMappingsEqualError,
     CheckSuperMappingError,
-    OneEmptyError,
-    OneNonUniqueError,
-    always_iterable,
     check_iterables_equal,
     check_mappings_equal,
     check_supermapping,
     is_iterable_not_str,
-    one,
     resolve_include_and_exclude,
 )
-from utilities.json import write_formatted_json
 from utilities.math import (
     MAX_DECIMALS,
     CheckIntegerError,
@@ -81,8 +87,7 @@ from utilities.math import (
     is_less_than,
     is_non_negative,
 )
-from utilities.reprlib import get_repr
-from utilities.types import MaybeStr, Number, PathLike, WeekDay
+from utilities.types import MaybeStr, Number, PathLike, StrDict, WeekDay
 from utilities.typing import (
     get_args,
     is_dataclass_class,
@@ -94,14 +99,12 @@ from utilities.typing import (
     is_set_type,
     make_isinstance,
 )
-from utilities.warnings import suppress_warnings
 from utilities.whenever import (
     DatePeriod,
     TimePeriod,
     ZonedDateTimePeriod,
     to_py_time_delta,
 )
-from utilities.zoneinfo import UTC, to_time_zone_name
 
 if TYPE_CHECKING:
     import datetime as dt
@@ -343,7 +346,7 @@ class AppendRowError(Exception):
 class _AppendRowPredicateError(AppendRowError):
     @override
     def __str__(self) -> str:
-        return f"Predicate failed; got {get_repr(self.row)}"
+        return f"Predicate failed; got {repr_(self.row)}"
 
 
 @dataclass(kw_only=True, slots=True)
@@ -352,7 +355,7 @@ class _AppendRowExtraKeysError(AppendRowError):
 
     @override
     def __str__(self) -> str:
-        return f"Extra key(s) found; got {get_repr(self.extra)}"
+        return f"Extra key(s) found; got {repr_(self.extra)}"
 
 
 @dataclass(kw_only=True, slots=True)
@@ -361,7 +364,7 @@ class _AppendRowMissingKeysError(AppendRowError):
 
     @override
     def __str__(self) -> str:
-        return f"Missing key(s) found; got {get_repr(self.missing)}"
+        return f"Missing key(s) found; got {repr_(self.missing)}"
 
 
 @dataclass(kw_only=True, slots=True)
@@ -370,7 +373,7 @@ class _AppendRowNullColumnsError(AppendRowError):
 
     @override
     def __str__(self) -> str:
-        return f"Null column(s) found; got {get_repr(self.columns)}"
+        return f"Null column(s) found; got {repr_(self.columns)}"
 
 
 ##
@@ -569,7 +572,7 @@ class _CheckPolarsDataFrameColumnsError(CheckPolarsDataFrameError):
 
     @override
     def __str__(self) -> str:
-        return f"DataFrame must have columns {get_repr(self.columns)}; got {get_repr(self.df.columns)}:\n\n{self.df}"
+        return f"DataFrame must have columns {repr_(self.columns)}; got {repr_(self.df.columns)}:\n\n{self.df}"
 
 
 def _check_polars_dataframe_dtypes(
@@ -587,7 +590,7 @@ class _CheckPolarsDataFrameDTypesError(CheckPolarsDataFrameError):
 
     @override
     def __str__(self) -> str:
-        return f"DataFrame must have dtypes {get_repr(self.dtypes)}; got {get_repr(self.df.dtypes)}:\n\n{self.df}"
+        return f"DataFrame must have dtypes {repr_(self.dtypes)}; got {repr_(self.df.dtypes)}:\n\n{self.df}"
 
 
 def _check_polars_dataframe_height(
@@ -650,9 +653,9 @@ class _CheckPolarsDataFramePredicatesError(CheckPolarsDataFrameError):
 
     def _yield_parts(self) -> Iterator[str]:
         if len(self.missing) >= 1:
-            yield f"missing columns were {get_repr(self.missing)}"
+            yield f"missing columns were {repr_(self.missing)}"
         if len(self.failed) >= 1:
-            yield f"failed predicates were {get_repr(self.failed)}"
+            yield f"failed predicates were {repr_(self.failed)}"
 
 
 def _check_polars_dataframe_schema_list(df: DataFrame, schema: SchemaDict, /) -> None:
@@ -672,7 +675,7 @@ class _CheckPolarsDataFrameSchemaListError(CheckPolarsDataFrameError):
 
     @override
     def __str__(self) -> str:
-        return f"DataFrame must have schema {get_repr(self.schema)} (ordered); got {get_repr(self.df.schema)}:\n\n{self.df}"
+        return f"DataFrame must have schema {repr_(self.schema)} (ordered); got {repr_(self.df.schema)}:\n\n{self.df}"
 
 
 def _check_polars_dataframe_schema_set(df: DataFrame, schema: SchemaDict, /) -> None:
@@ -688,7 +691,7 @@ class _CheckPolarsDataFrameSchemaSetError(CheckPolarsDataFrameError):
 
     @override
     def __str__(self) -> str:
-        return f"DataFrame must have schema {get_repr(self.schema)} (unordered); got {get_repr(self.df.schema)}:\n\n{self.df}"
+        return f"DataFrame must have schema {repr_(self.schema)} (unordered); got {repr_(self.df.schema)}:\n\n{self.df}"
 
 
 def _check_polars_dataframe_schema_subset(df: DataFrame, schema: SchemaDict, /) -> None:
@@ -704,7 +707,7 @@ class _CheckPolarsDataFrameSchemaSubsetError(CheckPolarsDataFrameError):
 
     @override
     def __str__(self) -> str:
-        return f"DataFrame schema must include {get_repr(self.schema)} (unordered); got {get_repr(self.df.schema)}:\n\n{self.df}"
+        return f"DataFrame schema must include {repr_(self.schema)} (unordered); got {repr_(self.df.schema)}:\n\n{self.df}"
 
 
 def _check_polars_dataframe_shape(df: DataFrame, shape: tuple[int, int], /) -> None:
@@ -742,7 +745,7 @@ class _CheckPolarsDataFrameSortedError(CheckPolarsDataFrameError):
 
     @override
     def __str__(self) -> str:
-        return f"DataFrame must be sorted on {get_repr(self.by)}:\n\n{self.df}"
+        return f"DataFrame must be sorted on {repr_(self.by)}:\n\n{self.df}"
 
 
 def _check_polars_dataframe_unique(
@@ -761,7 +764,7 @@ class _CheckPolarsDataFrameUniqueError(CheckPolarsDataFrameError):
 
     @override
     def __str__(self) -> str:
-        return f"DataFrame must be unique on {get_repr(self.by)}:\n\n{self.df}"
+        return f"DataFrame must be unique on {repr_(self.by)}:\n\n{self.df}"
 
 
 def _check_polars_dataframe_width(df: DataFrame, width: int, /) -> None:
@@ -1132,7 +1135,7 @@ class _DataClassToDataFrameNonUniqueError(DataClassToDataFrameError):
 
     @override
     def __str__(self) -> str:
-        return f"Iterable {get_repr(self.objs)} must contain exactly 1 class; got {self.first}, {self.second} and perhaps more"
+        return f"Iterable {repr_(self.objs)} must contain exactly 1 class; got {self.first}, {self.second} and perhaps more"
 
 
 ##
@@ -1147,7 +1150,7 @@ def dataclass_to_schema(
     warn_name_errors: bool = False,
 ) -> SchemaDict:
     """Cast a dataclass as a schema dict."""
-    out: dict[str, Any] = {}
+    out: StrDict = {}
     for field in yield_fields(
         obj, globalns=globalns, localns=localns, warn_name_errors=warn_name_errors
     ):
@@ -1204,7 +1207,7 @@ def _dataclass_to_schema_one(
         if issubclass(obj, enum.Enum):
             return pl.Enum([e.name for e in obj])
     if is_dataclass_class(obj):
-        out: dict[str, Any] = {}
+        out: StrDict = {}
         for field in yield_fields(obj, globalns=globalns, localns=localns):
             out[field.name] = _dataclass_to_schema_one(
                 field.type_, globalns=globalns, localns=localns
@@ -2425,7 +2428,7 @@ def _replace_time_zone_one(
 
 def read_series(path: PathLike, /, *, decompress: bool = False) -> Series:
     """Read a Series from disk."""
-    data = read_binary(path, decompress=decompress)
+    data = read_bytes(path, decompress=decompress)
     return deserialize_series(data)
 
 
@@ -2439,12 +2442,12 @@ def write_series(
 ) -> None:
     """Write a Series to disk."""
     data = serialize_series(series)
-    write_formatted_json(data, path, compress=compress, overwrite=overwrite)
+    write_bytes(path, data, compress=compress, overwrite=overwrite, json=True)
 
 
 def read_dataframe(path: PathLike, /, *, decompress: bool = False) -> DataFrame:
     """Read a DataFrame from disk."""
-    data = read_binary(path, decompress=decompress)
+    data = read_bytes(path, decompress=decompress)
     return deserialize_dataframe(data)
 
 
@@ -2453,7 +2456,7 @@ def write_dataframe(
 ) -> None:
     """Write a DataFrame to disk."""
     data = serialize_dataframe(df)
-    write_formatted_json(data, path, compress=compress, overwrite=overwrite)
+    write_bytes(path, data, compress=compress, overwrite=overwrite, json=True)
 
 
 def serialize_series(series: Series, /) -> bytes:
@@ -2571,7 +2574,8 @@ def round_to_float(
             return z.round(decimals=utilities.math.number_of_decimals(y) + 1)
         case Series(), Expr() | Series():
             df = (
-                x.to_frame()
+                x
+                .to_frame()
                 .with_columns(y)
                 .with_columns(number_of_decimals(y).alias("_decimals"))
                 .with_row_index(name="_index")
@@ -2638,6 +2642,8 @@ def search_period(
                 return None
             item: dt.datetime = series[index]["start"]
             return index if py_date_time > item else None
+        case never:
+            assert_never(never)
 
 
 ##
@@ -2662,7 +2668,7 @@ class SelectExactError(Exception):
 
     @override
     def __str__(self) -> str:
-        return f"All columns must be selected; got {get_repr(self.columns)} remaining"
+        return f"All columns must be selected; got {repr_(self.columns)} remaining"
 
 
 ##

utilities/postgres.py

@@ -9,9 +9,9 @@ from sqlalchemy import Table
 from sqlalchemy.orm import DeclarativeBase
 
 from utilities.asyncio import stream_command
-from utilities.iterables import always_iterable
+from utilities.core import always_iterable, yield_temp_environ
+from utilities.docker import docker_exec_cmd
 from utilities.logging import to_logger
-from utilities.os import temp_environ
 from utilities.pathlib import ensure_suffix
 from utilities.sqlalchemy import extract_url, get_table_name
 from utilities.timer import Timer
@@ -37,6 +37,7 @@ async def pg_dump(
     path: PathLike,
     /,
     *,
+    docker_container: str | None = None,
     format_: _PGDumpFormat = "plain",
     jobs: int | None = None,
     data_only: bool = False,
@@ -51,7 +52,6 @@ async def pg_dump(
     inserts: bool = False,
     on_conflict_do_nothing: bool = False,
     role: str | None = None,
-    docker: str | None = None,
     dry_run: bool = False,
     logger: LoggerLike | None = None,
 ) -> bool:
@@ -61,6 +61,7 @@ async def pg_dump(
     cmd = _build_pg_dump(
         url,
         path,
+        docker_container=docker_container,
         format_=format_,
         jobs=jobs,
         data_only=data_only,
@@ -75,13 +76,15 @@ async def pg_dump(
         inserts=inserts,
         on_conflict_do_nothing=on_conflict_do_nothing,
         role=role,
-        docker=docker,
     )
     if dry_run:
         if logger is not None:
             to_logger(logger).info("Would run:\n\t%r", str(cmd))
         return True
-    with temp_environ(PGPASSWORD=url.password), Timer() as timer:  # pragma: no cover
+    with (
+        yield_temp_environ(PGPASSWORD=url.password),
+        Timer() as timer,
+    ):  # pragma: no cover
         try:
             output = await stream_command(cmd)
         except KeyboardInterrupt:
@@ -111,6 +114,7 @@ def _build_pg_dump(
     path: PathLike,
     /,
     *,
+    docker_container: str | None = None,
     format_: _PGDumpFormat = "plain",
     jobs: int | None = None,
     data_only: bool = False,
@@ -125,12 +129,13 @@ def _build_pg_dump(
     inserts: bool = False,
     on_conflict_do_nothing: bool = False,
     role: str | None = None,
-    docker: str | None = None,
 ) -> str:
     extracted = extract_url(url)
     path = _path_pg_dump(path, format_=format_)
-    parts: list[str] = [
-        "pg_dump",
+    parts: list[str] = ["pg_dump"]
+    if docker_container is not None:
+        parts = docker_exec_cmd(docker_container, *parts, PGPASSWORD=extracted.password)
+    parts.extend([
         # general options
         f"--file={str(path)!r}",
         f"--format={format_}",
@@ -146,7 +151,7 @@ def _build_pg_dump(
         f"--port={extracted.port}",
         f"--username={extracted.username}",
         "--no-password",
-    ]
+    ])
     if (format_ == "directory") and (jobs is not None):
         parts.append(f"--jobs={jobs}")
     if create:
@@ -173,8 +178,6 @@ def _build_pg_dump(
         parts.append("--on-conflict-do-nothing")
     if role is not None:
         parts.append(f"--role={role}")
-    if docker is not None:
-        parts = _wrap_docker(parts, docker)
     return " ".join(parts)
 
 
@@ -213,7 +216,7 @@ async def restore(
     schema_exc: MaybeCollectionStr | None = None,
     table: MaybeCollection[TableOrORMInstOrClass | str] | None = None,
     role: str | None = None,
-    docker: str | None = None,
+    docker_container: str | None = None,
     dry_run: bool = False,
     logger: LoggerLike | None = None,
 ) -> bool:
@@ -230,13 +233,16 @@ async def restore(
         schema_exc=schema_exc,
         table=table,
         role=role,
-        docker=docker,
+        docker_container=docker_container,
     )
     if dry_run:
         if logger is not None:
             to_logger(logger).info("Would run:\n\t%r", str(cmd))
         return True
-    with temp_environ(PGPASSWORD=url.password), Timer() as timer:  # pragma: no cover
+    with (
+        yield_temp_environ(PGPASSWORD=url.password),
+        Timer() as timer,
+    ):  # pragma: no cover
         try:
             output = await stream_command(cmd)
         except KeyboardInterrupt:
@@ -276,11 +282,11 @@ def _build_pg_restore_or_psql(
     schema_exc: MaybeCollectionStr | None = None,
     table: MaybeCollection[TableOrORMInstOrClass | str] | None = None,
     role: str | None = None,
-    docker: str | None = None,
+    docker_container: str | None = None,
 ) -> str:
     path = Path(path)
     if (path.suffix == ".sql") or psql:
-        return _build_psql(url, path, docker=docker)
+        return _build_psql(url, path, docker_container=docker_container)
     return _build_pg_restore(
         url,
         path,
@@ -292,7 +298,7 @@ def _build_pg_restore_or_psql(
         schemas_exc=schema_exc,
         tables=table,
         role=role,
-        docker=docker,
+        docker_container=docker_container,
     )
 
 
@@ -309,12 +315,14 @@ def _build_pg_restore(
     schemas_exc: MaybeCollectionStr | None = None,
     tables: MaybeCollection[TableOrORMInstOrClass | str] | None = None,
     role: str | None = None,
-    docker: str | None = None,
+    docker_container: str | None = None,
 ) -> str:
     """Run `pg_restore`."""
     extracted = extract_url(url)
-    parts: list[str] = [
-        "pg_restore",
+    parts: list[str] = ["pg_restore"]
+    if docker_container is not None:
+        parts = docker_exec_cmd(docker_container, *parts, PGPASSWORD=extracted.password)
+    parts.extend([
         # general options
         "--verbose",
         # restore options
@@ -328,7 +336,7 @@ def _build_pg_restore(
         f"--username={extracted.username}",
         f"--dbname={extracted.database}",
         "--no-password",
-    ]
+    ])
     if create:
         parts.append("--create")
     if jobs is not None:
@@ -341,17 +349,19 @@ def _build_pg_restore(
         parts.extend([f"--table={_get_table_name(t)}" for t in always_iterable(tables)])
     if role is not None:
         parts.append(f"--role={role}")
-    if docker is not None:
-        parts = _wrap_docker(parts, docker)
     parts.append(str(path))
     return " ".join(parts)
 
 
-def _build_psql(url: URL, path: PathLike, /, *, docker: str | None = None) -> str:
+def _build_psql(
+    url: URL, path: PathLike, /, *, docker_container: str | None = None
+) -> str:
     """Run `psql`."""
     extracted = extract_url(url)
-    parts: list[str] = [
-        "psql",
+    parts: list[str] = ["psql"]
+    if docker_container is not None:
+        parts = docker_exec_cmd(docker_container, *parts, PGPASSWORD=extracted.password)
+    parts.extend([
         # general options
         f"--dbname={extracted.database}",
         f"--file={str(path)!r}",
@@ -360,9 +370,7 @@ def _build_psql(url: URL, path: PathLike, /, *, docker: str | None = None) -> st
         f"--port={extracted.port}",
         f"--username={extracted.username}",
         "--no-password",
-    ]
-    if docker is not None:
-        parts = _wrap_docker(parts, docker)
+    ])
     return " ".join(parts)
 
 
@@ -402,8 +410,4 @@ class _ResolveDataOnlyAndCleanError(Exception):
         return "Cannot use '--data-only' and '--clean' together"
 
 
-def _wrap_docker(parts: list[str], container: str, /) -> list[str]:
-    return ["docker", "exec", "-it", container, *parts]
-
-
 __all__ = ["pg_dump", "restore"]

utilities/pottery.py

@@ -9,21 +9,21 @@ from pottery import AIORedlock
 from pottery.exceptions import ReleaseUnlockedLock
 from redis.asyncio import Redis
 
-from utilities.asyncio import sleep_td, timeout_td
+import utilities.asyncio
+from utilities.constants import MILLISECOND, SECOND
 from utilities.contextlib import enhanced_async_context_manager
-from utilities.iterables import always_iterable
-from utilities.whenever import MILLISECOND, SECOND, to_nanoseconds
+from utilities.core import always_iterable
+from utilities.functions import in_seconds
 
 if TYPE_CHECKING:
     from collections.abc import AsyncIterator, Iterable
 
-    from whenever import Delta
+    from utilities.types import Duration, MaybeIterable
 
-    from utilities.types import MaybeIterable
 
 _NUM: int = 1
-_TIMEOUT_RELEASE: Delta = 10 * SECOND
-_SLEEP: Delta = MILLISECOND
+_TIMEOUT_RELEASE: Duration = 10 * SECOND
+_SLEEP: Duration = MILLISECOND
 
 
 ##
@@ -47,11 +47,11 @@ async def yield_access(
     /,
     *,
     num: int = _NUM,
-    timeout_release: Delta = _TIMEOUT_RELEASE,
+    timeout_release: Duration = _TIMEOUT_RELEASE,
     num_extensions: int | None = None,
-    timeout_acquire: Delta | None = None,
-    sleep: Delta = _SLEEP,
-    throttle: Delta | None = None,
+    timeout_acquire: Duration | None = None,
+    sleep: Duration = _SLEEP,
+    throttle: Duration | None = None,
 ) -> AsyncIterator[AIORedlock]:
     """Acquire access to a locked resource."""
     if num <= 0:
@@ -63,7 +63,7 @@ async def yield_access(
         AIORedlock(
             key=f"{key}_{i}_of_{num}",
             masters=masters,
-            auto_release_time=to_nanoseconds(timeout_release) / 1e9,
+            auto_release_time=in_seconds(timeout_release),
             num_extensions=maxsize if num_extensions is None else num_extensions,
         )
         for i in range(1, num + 1)
@@ -75,7 +75,7 @@ async def yield_access(
         )
         yield lock
     finally:  # skipif-ci-and-not-linux
-        await sleep_td(throttle)
+        await utilities.asyncio.sleep(throttle)
         if lock is not None:
             with suppress(ReleaseUnlockedLock):
                 await lock.release()
@@ -87,18 +87,20 @@ async def _get_first_available_lock(
     /,
     *,
     num: int = _NUM,
-    timeout: Delta | None = None,
-    sleep: Delta | None = _SLEEP,
+    timeout: Duration | None = None,
+    sleep: Duration | None = _SLEEP,
 ) -> AIORedlock:
     locks = list(locks)  # skipif-ci-and-not-linux
     error = _YieldAccessUnableToAcquireLockError(  # skipif-ci-and-not-linux
         key=key, num=num, timeout=timeout
     )
-    async with timeout_td(timeout, error=error):  # skipif-ci-and-not-linux
+    async with utilities.asyncio.timeout(  # skipif-ci-and-not-linux
+        timeout, error=error
+    ):
         while True:
             if (result := await _get_first_available_lock_if_any(locks)) is not None:
                 return result
-            await sleep_td(sleep)
+            await utilities.asyncio.sleep(sleep)
 
 
 async def _get_first_available_lock_if_any(
@@ -127,7 +129,7 @@ class _YieldAccessNumLocksError(YieldAccessError):
 @dataclass(kw_only=True, slots=True)
 class _YieldAccessUnableToAcquireLockError(YieldAccessError):
     num: int
-    timeout: Delta | None
+    timeout: Duration | None
 
     @override
     def __str__(self) -> str:

utilities/pqdm.py

@@ -6,18 +6,17 @@ from typing import TYPE_CHECKING, Any, Literal, assert_never
 from pqdm import processes, threads
 from tqdm.auto import tqdm as tqdm_auto
 
-from utilities.functions import get_func_name
+from utilities.constants import Sentinel, sentinel
+from utilities.core import get_func_name, is_sentinel
 from utilities.iterables import apply_to_varargs
 from utilities.os import get_cpu_use
-from utilities.sentinel import Sentinel, is_sentinel, sentinel
 
 if TYPE_CHECKING:
     from collections.abc import Callable, Iterable
 
     from tqdm import tqdm as tqdm_type
 
-    from utilities.os import IntOrAll
-    from utilities.types import Parallelism
+    from utilities.types import IntOrAll, Parallelism
 
 
 type _ExceptionBehaviour = Literal["ignore", "immediate", "deferred"]

utilities/psutil.py

@@ -6,8 +6,7 @@ from typing import TYPE_CHECKING, Self
 
 from psutil import swap_memory, virtual_memory
 
-from utilities.contextlib import suppress_super_object_attribute_error
-from utilities.whenever import get_now
+from utilities.core import get_now, suppress_super_attribute_error
 
 if TYPE_CHECKING:
     from whenever import ZonedDateTime
@@ -30,7 +29,7 @@ class MemoryUsage:
     swap_pct: float = field(init=False)
 
     def __post_init__(self) -> None:
-        with suppress_super_object_attribute_error():
+        with suppress_super_attribute_error():
             super().__post_init__()  # pyright: ignore[reportAttributeAccessIssue]
         self.virtual_used_mb = self._to_mb(self.virtual_used)
         self.virtual_total_mb = self._to_mb(self.virtual_total)

utilities/pydantic.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Annotated, assert_never
+
+from pydantic import BeforeValidator, SecretStr
+
+from utilities.types import PathLike
+
+type ExpandedPath = Annotated[PathLike, BeforeValidator(lambda p: Path(p).expanduser())]
+type SecretLike = SecretStr | str
+
+
+def extract_secret(value: SecretLike, /) -> str:
+    """Given a secret, extract its value."""
+    match value:
+        case SecretStr():
+            return value.get_secret_value()
+        case str():
+            return value
+        case never:
+            assert_never(never)
+
+
+__all__ = ["ExpandedPath", "SecretLike", "extract_secret"]