thds.core 0.0.1__py3-none-any.whl → 1.31.20250116223856__py3-none-any.whl

thds/core/config.py

@@ -0,0 +1,250 @@
+"""This is an attempt at a be-everything-to-everybody configuration 'system'.
+
+Highlights:
+
+- Configuration is always accessible and configurable via normal Python code.
+- Configuration is type-safe.
+- All active configuration is 'registered' and therefore discoverable.
+- Config can be temporarily overridden for the current thread.
+- Config can be set via a known environment variable.
+- Config can be set by combining one or more configuration objects - these may be loaded from files,
+  but this system remains agnostic as to the format of those files or how and when they are actually loaded.
+
+Please see thds/core/CONFIG.md for more details.
+"""
+
+import typing as ty
+from logging import getLogger
+from os import getenv
+
+from .stack_context import StackContext
+
+_NOT_CONFIGURED = object()
+
+
+class UnconfiguredError(ValueError):
+    pass
+
+
+class ConfigNameCollisionError(KeyError):
+    pass
+
+
+def _sanitize_env(env_var_name: str) -> str:
+    return env_var_name.replace("-", "_").replace(".", "_")
+
+
+def _getenv(name: str, secret: bool) -> ty.Optional[str]:
+    """We want to support a variety of naming conventions for env
+    vars, without requiring people to actually name their config using
+    all caps and underscores only.
+
+    Many modern shells support more complex env var names.
+    """
+
+    def _first_nonnil(*envvars: str) -> ty.Optional[str]:
+        for envvar in envvars:
+            raw_value = getenv(envvar)
+            if raw_value is not None:
+                lvalue = "***SECRET***" if secret else raw_value
+                getLogger(__name__).info(
+                    f"Loaded config '{name}' with raw value '{lvalue}' from environment variable '{envvar}'"
+                )
+                return raw_value
+        return None
+
+    return _first_nonnil(name, _sanitize_env(name), _sanitize_env(name).upper())
+
+
+def _module_name():
+    import inspect
+
+    # this takes ~200 nanoseconds to run. that's fast enough that
+    # doing the extra lookup is worth it for being more user-friendly.
+    frame = inspect.currentframe()
+    frame = frame.f_back.f_back.f_back  # type: ignore
+    return frame.f_globals["__name__"]  # type: ignore
+
+
+def _fullname(name: str) -> str:
+    """In the vast majority of cases, the best way to use this library
+    is to name your configuration items to correspond with your
+    fully-qualified module name as a prefix.
+
+    It will enhance discoverability and clarity,
+    and will help avoid configuration name collisions.
+    """
+    return name if "." in name else f"{_module_name()}.{name}"
+
+
+T = ty.TypeVar("T")
+
+
+def _type_parser(default: T) -> ty.Callable[[ty.Any], T]:
+    if default is None:
+        return lambda x: x  # cannot learn anything about how to parse a future value from None
+    try:
+        default_type = type(default)
+        if default == default_type(default):  # type: ignore
+            # if this succeeds and doesn't raise, then the type is self-parsing.
+            # in other words, type(4)(4) == 4, type('foobar')('foobar') == 'foobar'
+            return default_type
+    except Exception:
+        pass
+    return lambda x: x  # we can't infer a type parser, so we'll return the default no-op parser.
+
+
+class ConfigItem(ty.Generic[T]):
+    """Should only ever be constructed at a module level."""
+
+    def __init__(
+        self,
+        name: str,
+        # names must be hierarchical. if name contains no `.`, we will force
+        # your name to be in the calling module.
+        default: T = ty.cast(T, _NOT_CONFIGURED),
+        *,
+        parse: ty.Optional[ty.Callable[[ty.Any], T]] = None,
+        secret: bool = False,
+    ):
+        self.secret = secret
+        name = _fullname(name)
+        if name in _REGISTRY:
+            raise ConfigNameCollisionError(f"Config item {name} has already been registered!")
+        _REGISTRY[name] = self
+        self.name = name
+        self.parse = parse or _type_parser(default)
+        raw_resolved_global = _getenv(name, secret=secret)
+        if raw_resolved_global:
+            # external global values are only resolved at initial
+            # creation.  if you want to set this value globally after
+            # application start, use set_global.
+            self.global_value = self.parse(raw_resolved_global)
+        elif default is not _NOT_CONFIGURED:
+            self.global_value = self.parse(default)
+        else:
+            self.global_value = default
+        self._stack_context: StackContext[T] = StackContext(
+            "config_" + name, ty.cast(T, _NOT_CONFIGURED)
+        )
+
+    def set_global(self, value: T):
+        """Global to the current process.
+
+        Will not automatically get transferred to spawned processes.
+        """
+
+        self.global_value = value
+        # we used to parse this value, but I think that was the wrong choice -
+        # we should only have to parse it when it's coming from the environment,
+        # which should be handled by it's initial creation,
+        # or when it's being set as a global default from a config file.
+
+    def set_local(self, value: T) -> ty.ContextManager[T]:
+        """Local to the current thread.
+
+        Will not automatically get transferred to spawned threads.
+        """
+
+        return self._stack_context.set(value)
+
+    def __call__(self) -> T:
+        local = self._stack_context()
+        if local is not _NOT_CONFIGURED:
+            return local
+        if self.global_value is _NOT_CONFIGURED:
+            raise UnconfiguredError(f"Config item '{self.name}' has not been configured!")
+        return self.global_value
+
+
+def tobool(s_or_b: ty.Union[str, bool]) -> bool:
+    """A reasonable implementation that we could expand in the future."""
+    return s_or_b if isinstance(s_or_b, bool) else s_or_b.lower() not in ("0", "false", "no", "off", "")
+
+
+def maybe(parser: ty.Callable[[ty.Any], T]) -> ty.Callable[[ty.Optional[ty.Any]], ty.Optional[T]]:
+    """A helper for when you want to parse a value that might be nil."""
+    return lambda x: parser(x) if x is not None else None
+
+
+item = ConfigItem
+# a short alias
+
+
+_REGISTRY: ty.Dict[str, ConfigItem] = dict()
+
+
+def config_by_name(name: str) -> ConfigItem:
+    """This is a dynamic interface - in general, prefer accessing the ConfigItem object directly."""
+    return _REGISTRY[_fullname(name)]
+
+
+def set_global_defaults(config: ty.Dict[str, ty.Any]):
+    """Any config-file parser can create a dictionary of only the
+    items it managed to read, and then all of those can be set at once
+    via this function.
+    """
+    for name, value in config.items():
+        if not isinstance(value, dict):
+            try:
+                config_item = _REGISTRY[name]
+                config_item.set_global(config_item.parse(value))
+            except KeyError:
+                # try directly importing a module - this is only best-effort and will not work
+                # if you did not follow standard configuration naming conventions.
+                import importlib
+
+                maybe_module_name = ".".join(name.split(".")[:-1])
+                try:
+                    importlib.import_module(maybe_module_name)
+                    try:
+                        config_item = _REGISTRY[name]
+                        config_item.set_global(config_item.parse(value))
+                    except KeyError as kerr:
+                        raise KeyError(
+                            f"Config item {name} is not registered"
+                            f" and no module with the name {maybe_module_name} was importable."
+                            " Please double-check your configuration."
+                        ) from kerr
+                except ModuleNotFoundError:
+                    # create a new, dynamic config item that will only be accessible via
+                    # its name.
+                    ConfigItem(name, value)  # return value not needed since it self-registers.
+                    getLogger(__name__).debug(
+                        "Created dynamic config item '%s' with value '%s'", name, value
+                    )
+
+        else:  # recurse
+            set_global_defaults({f"{name}.{key}": val for key, val in value.items()})
+
+
+def get_all_config() -> ty.Dict[str, ty.Any]:
+    return {k: v() if not v.secret else "***SECRET***" for k, v in _REGISTRY.items()}
+
+
+def show_config_cli():
+    import argparse
+    import importlib
+    from pprint import pprint
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("via_modules", type=str, nargs="+")
+    args = parser.parse_args()
+
+    for module in args.via_modules:
+        importlib.import_module(module)
+
+    print()
+    print("thds.core.config")
+    print("----------------")
+    print("The following keys are fully-qualified module paths by default.")
+    print(
+        "A given item can be capitalized and set via environment variable"
+        ", e.g. export THDS_CORE_LOG_LEVEL='DEBUG'"
+    )
+    print(
+        "An item can also be set globally or locally, after importing the"
+        " ConfigItem object from the Python module where it is defined."
+    )
+    print()
+    pprint(get_all_config())

thds/core/decos.py

@@ -0,0 +1,55 @@
+import typing as ty
+
+F = ty.TypeVar("F", bound=ty.Callable)
+
+
+@ty.overload
+def compose(*decorators: ty.Callable[[F], F]) -> ty.Callable[[F], F]:
+    ...  # pragma: no cover
+
+
+@ty.overload
+def compose(*decorators: ty.Callable[[F], F], f: F) -> F:
+    ...  # pragma: no cover
+
+
+def compose(*decorators: ty.Callable[[F], F], f: ty.Optional[F] = None) -> ty.Callable[[F], F]:
+    """A decorator factory that creates a single decorator from
+    multiple, following the top-to-bottom order that would be used by
+    the `@deco` decorator syntax, which is actually standard R-to-L
+    composition order.
+
+    It may also be used to both compose the decorator and apply it
+    directly to a provided function.
+
+    This is useful when you want to apply a sequence of decorators
+    (higher-order functions) to a function, but you can't use the
+    decorator syntax because you want the original function to be
+    callable without the decorators.
+
+    Example:
+
+    >>> decos.compose_partial(
+    ...     deco1,
+    ...     deco2,
+    ...     deco3,
+    ... )(f)
+
+    is equivalent to:
+
+    >>> @deco1
+    ... @deco2
+    ... @deco3
+    ... def f():
+    ...     pass
+
+    """
+
+    def _deco(func: F) -> F:
+        for deco in reversed(decorators):
+            func = deco(func)
+        return func
+
+    if f:
+        return _deco(f)
+    return _deco

thds/core/dict_utils.py

@@ -0,0 +1,188 @@
+import re
+import warnings
+from collections import defaultdict
+from typing import Any, Dict, Generator, List, Mapping, MutableMapping, Optional, Tuple, TypeVar
+
+DEFAULT_SEP = "."
+VT = TypeVar("VT")
+
+
+def _get_valid_variable_name(var: str):
+    """
+    given a string returns the string formatted as a proper python variable name.
+    Credit: https://stackoverflow.com/questions/3303312/how-do-i-convert-a-string-to-a-valid-variable-name-in-python
+    """
+    return re.sub(r"\W+|^(?=\d)", "_", var)
+
+
+def _flatten_gen(
+    d: Mapping, parent_key: str = "", sep: str = DEFAULT_SEP
+) -> Generator[Tuple[str, Any], None, None]:
+    """
+    flattens a mapping (usually a dict) using a separator, returning a generator of the flattened keys and values.
+
+    Example
+    ---------
+
+    d = {"a": {"b": {"c": 1}}}
+    fd = flatten(d, sep=".")
+    print(dict(fd))
+    > {"a.b.c": 1}
+    """
+    for k, v in d.items():
+        new_key = parent_key + sep + k if parent_key else k
+        if isinstance(v, Mapping):
+            yield from _flatten_gen(v, new_key, sep=sep)
+        else:
+            yield new_key, v
+
+
+def unflatten(flat_d: Dict[str, Any], sep: str = DEFAULT_SEP):
+    """Given a flattened dictionary returns the un-flatten representation."""
+    unflatten_dict: Dict[str, Any] = {}
+    for path, val in flat_d.items():
+        dict_ref = unflatten_dict
+        path_parts = path.split(sep)
+        for p in path_parts[:-1]:
+            dict_ref[p] = dict_ref.get(p) or {}
+            dict_ref = dict_ref[p]
+        dict_ref[path_parts[-1]] = val
+    return unflatten_dict
+
+
+def flatten(d: Mapping, parent_key: str = "", sep: str = DEFAULT_SEP) -> Dict[str, Any]:
+    return dict(_flatten_gen(d, parent_key, sep))
+
+
+class DotDict(MutableMapping[str, VT]):
+    """A python dictionary that acts like an object."""
+
+    _new_to_orig_keys: Dict[str, str] = dict()
+    _hidden_data: Dict[str, Any] = dict()
+
+    def _get_hidden_data(self, identifier: str) -> Any:
+        return self._hidden_data.get(identifier)
+
+    def _construct(self, mapping: Mapping) -> None:
+        convert_keys_to_identifiers = self._get_hidden_data("convert_keys_to_identifiers")
+        for k, v in mapping.items():
+            new_key = _get_valid_variable_name(k) if convert_keys_to_identifiers else k
+            if convert_keys_to_identifiers:
+                self._new_to_orig_keys[new_key] = k
+            if isinstance(v, dict):
+                self[new_key] = DotDict(v)  # type: ignore
+            elif isinstance(v, (list, tuple, set)):
+                self[new_key] = v.__class__([DotDict(iv) if isinstance(iv, dict) else iv for iv in v])  # type: ignore
+            else:
+                self[new_key] = v
+
+    def __init__(self, *args, convert_keys_to_identifiers: bool = False, **kwargs):
+        self._hidden_data["convert_keys_to_identifiers"] = convert_keys_to_identifiers
+        if convert_keys_to_identifiers:
+            warnings.warn("automatically converting keys into identifiers. Data loss might occur.")
+        for arg in args:
+            if isinstance(arg, dict):
+                self._construct(mapping=arg)
+        if kwargs:
+            self._construct(mapping=kwargs)
+
+    def __getattr__(self, key: str) -> VT:
+        return self[key]
+
+    def __setattr__(self, key: str, value: VT) -> None:
+        self.__setitem__(key, value)
+
+    def __setitem__(self, key: str, value: VT):
+        self.__dict__.update({key: value})
+
+    def __delattr__(self, key: str) -> None:
+        self.__delitem__(key)
+
+    def __delitem__(self, key: str) -> None:
+        super(DotDict, self).__delitem__(key)
+        del self.__dict__[key]
+
+    def __getitem__(self, key: str) -> VT:
+        return self.__dict__[key]
+
+    def __iter__(self):
+        return iter(self.__dict__)
+
+    def __len__(self) -> int:
+        return len(self.__dict__)
+
+    def to_dict(self, orig_keys: bool = False) -> Dict[str, VT]:
+        convert_keys_to_identifiers = self._get_hidden_data("convert_keys_to_identifiers")
+        d: Dict[str, VT] = dict()
+        for k, v in self.items():
+            if isinstance(v, DotDict):
+                d[
+                    self._new_to_orig_keys[k] if orig_keys and convert_keys_to_identifiers else k
+                ] = v.to_dict(
+                    orig_keys
+                )  # type: ignore[assignment]
+            else:
+                d[self._new_to_orig_keys[k] if orig_keys and convert_keys_to_identifiers else k] = v
+        return d
+
+    def get_value(self, dot_path: str) -> Optional[VT]:
+        """Get a value given a dotted path to the value.
+
+        Example
+        -------
+
+        dd = DotDict(a={"b": 100})
+        assert dd.get_value("a.b") == 100
+        """
+        path = dot_path.split(".")
+        ref: DotDict[Any] = self
+        for k in path[:-1]:
+            if isinstance(ref, DotDict) and k in ref:
+                ref = ref[k]
+            else:
+                return None
+        try:
+            return ref[path[-1]]
+        except KeyError:
+            return None
+
+    def set_value(self, dot_path: str, val: VT) -> None:
+        """Set a vlaue given a dotted path."""
+        ref = self
+        path = dot_path.split(".")
+        try:
+            for k in path[:-1]:
+                ref = getattr(ref, k)
+        except AttributeError:
+            raise KeyError(f"can't set path {dot_path} with parts {path}.")
+        ref.__setattr__(path[-1], val)
+
+
+def merge_dicts(*dicts: Dict[Any, Any], default: Any = None) -> Dict[Any, Any]:
+    """Merges similar dictionaries into one dictionary where the resulting values are a list of values from the
+    original dicts. If a dictionary does not have a key the default value will be used (defaults to None).
+
+    Example
+    --------
+
+    assert merge_dicts(
+        {"a": 100, "b": {"c": 200, "d": 300}, "e": [1, 2]},
+        {"a": 200, "b": {"c": 300}, "e": [3, 4], "f": 300}
+    ) == {
+        "a": [100, 200],
+        "b": {
+            "c": [200, 300],
+            "d": [300, None]
+        },
+        "e": [[1,2], [3,4]],
+        "f": [None, 300]
+    }
+    """
+    merged_dict: Dict[str, List[Any]] = defaultdict(lambda: [default for _ in range(len(dicts))])
+    for i, d in enumerate(dicts):
+        for k, v in d.items() if isinstance(d, dict) else {}:
+            if isinstance(v, dict):
+                merged_dict[k] = merge_dicts(*[a.get(k, {}) for a in dicts])  # type: ignore
+            else:
+                merged_dict[k][i] = v
+    return dict(merged_dict)

thds/core/env.py

@@ -0,0 +1,40 @@
+import os
+import typing as ty
+from contextlib import contextmanager
+
+from .stack_context import StackContext
+
+THDS_ENV = "THDS_ENV"
+
+Env = ty.Literal["", "prod", "dev", "ua-prod"]
+
+_TEMP_ENV: StackContext[Env] = StackContext("thds-env", "")
+
+
+def set_active_env(env: Env):
+    os.environ[THDS_ENV] = env
+
+
+def get_raw_env() -> Env:
+    """Get the actual value of `THDS_ENV`. Unset == ''
+
+    Prefer `active_env` for determining the active environment.
+    """
+    return ty.cast(Env, os.environ.get(THDS_ENV, ""))
+
+
+@contextmanager
+def temp_env(env: Env = "") -> ty.Iterator[Env]:
+    """Temporarily set the value of `THDS_ENV` for the current stack/thread.
+
+    Useful if you have special cases where you need to fetch a result
+    from a different environment than the one you're intending to run,
+    without affecting the global state of your application.
+    """
+    with _TEMP_ENV.set(env or active_env()):
+        yield _TEMP_ENV()
+
+
+def active_env(override: Env = "") -> Env:
+    """Get the effective value of `THDS_ENV`."""
+    return override or _TEMP_ENV() or get_raw_env() or "dev"

thds/core/exit_after.py

@@ -0,0 +1,121 @@
+"""A program that exits after a given number of seconds.
+
+but whose exit can be delayed by modifying a file on the filesystem.
+"""
+
+import argparse
+import os
+import time
+import typing as ty
+from datetime import datetime, timedelta, timezone
+from logging import getLogger
+from pathlib import Path
+
+logger = getLogger(__name__)
+
+
+def now() -> datetime:
+    return datetime.now(tz=timezone.utc)
+
+
+def report(exit_time: datetime) -> datetime:
+    reporting_at = now()
+    msg = f"Will exit at {exit_time}, in {exit_time - reporting_at}"
+    logger.info(msg)
+    return reporting_at
+
+
+def try_parse(time_file: Path, default: datetime) -> datetime:
+    dt_s = ""
+    try:
+        with open(time_file) as f:
+            dt_s = f.read().strip()
+            dt = datetime.fromisoformat(dt_s)
+            if default != dt:
+                logger.info(f"Parsed new time {dt} from {time_file}")
+            return dt
+    except FileNotFoundError:
+        logger.debug(f"No file found at {time_file}")
+    except ValueError:
+        logger.exception(f"Unable to parse {time_file} with contents {dt_s}; using default {default}")
+    return default
+
+
+_1_SECOND = timedelta(seconds=1)
+
+
+def exit_when(
+    exit_time_file: Path,
+    exit_at: datetime,
+    *,
+    report_every: ty.Optional[timedelta],
+    check_every: timedelta = _1_SECOND,
+    keep_existing_file: bool = False,
+) -> timedelta:
+    # setup
+    started = now()
+    exit_time_file = exit_time_file.resolve()
+    if not exit_at.tzinfo:
+        exit_at = exit_at.astimezone(timezone.utc)
+
+    if not keep_existing_file:
+        with open(exit_time_file, "w") as f:
+            f.write(exit_at.isoformat())
+
+    exit_at = try_parse(exit_time_file, exit_at)
+    if report_every:
+        reported_at = report(exit_at)
+    while True:
+        rem = (exit_at - now()).total_seconds()
+        if rem <= 0:
+            break
+        time.sleep(min(check_every.total_seconds(), rem))
+        exit_at = try_parse(exit_time_file, exit_at)
+        if report_every and now() - reported_at > report_every:
+            reported_at = report(exit_at)
+
+    os.unlink(exit_time_file)
+    return now() - started
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--exit-time-file",
+        "-f",
+        default="~/WILL_EXIT_AT_THIS_TIME.txt",
+        help=(
+            "Modify this file while the program is running to accelerate or delay the exit."
+            " Will be deleted and recreated if it exists, unless --keep-existing-file is also passed."
+        ),
+    )
+    parser.add_argument(
+        "--seconds-from-now",
+        "-s",
+        default=timedelta(hours=1).total_seconds(),
+        type=float,
+    )
+    parser.add_argument(
+        "--report-every-s",
+        "-r",
+        default=timedelta(minutes=20).total_seconds(),
+        type=float,
+        help="Report time of exit this often (in seconds)",
+    )
+    parser.add_argument(
+        "--keep-existing-file",
+        action="store_true",
+        help="Unless set, an existing file will be presumed to be from a previous run",
+    )
+    args = parser.parse_args()
+    elapsed = exit_when(
+        Path(os.path.expanduser(args.exit_time_file)),
+        datetime.now(tz=timezone.utc) + timedelta(seconds=args.seconds_from_now),
+        report_every=timedelta(seconds=args.report_every_s),
+        keep_existing_file=args.keep_existing_file,
+    )
+    logger.info(f"Exiting after {elapsed}")
+
+
+if __name__ == "__main__":
+    main()