rmk2 1.0.0__py3-none-any.whl

rmk2/api.py

@@ -0,0 +1,57 @@
+import functools
+import json
+import os
+import re
+from typing import Any, Iterator
+from urllib.parse import urlparse, urlunparse
+from uuid import uuid4
+
+
+def save_response(prefix: str = None) -> callable:
+    """Wrapper to save the return value of a function to JSON"""
+
+    def decorator(func: callable) -> callable:
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs) -> Any:
+            response = func(*args, **kwargs)
+
+            _filename = os.path.join(
+                os.path.dirname(__file__),
+                prefix,
+                f"{func.__name__}_{str(uuid4())}.json",
+            )
+
+            with open(_filename, mode="w", encoding="utf-8") as outfile:
+                _response = (
+                    list(response) if isinstance(response, Iterator) else response
+                )
+
+                json.dump(_response, outfile)
+
+            return _response
+
+        return wrapper
+
+    return decorator
+
+
+def create_url(
+    base_url: str,
+    path: list[str] | None,
+    query: dict[str, str | int] | None,
+) -> str:
+    """Helper function to create valid URLs by extending a base URL"""
+    _scheme, _netloc, _path, _, _query, _ = urlparse(base_url)
+
+    _path = (
+        "/".join([re.sub(r"/$", "", _path), *[str(x) for x in path]])
+        if path is not None
+        else _path
+    )
+    _query = (
+        "&".join([f"{k}={str(v)}" for k, v in query.items()])
+        if query is not None
+        else {}
+    )
+
+    return urlunparse((_scheme, _netloc, _path, None, _query, None))

rmk2/config/mapping.py

@@ -0,0 +1,230 @@
+import argparse
+import dataclasses
+import itertools
+import os
+from copy import deepcopy
+from enum import Enum
+from typing import Any, Optional, Sequence
+
+
+class ConfigField(Enum):
+    ARGUMENT = "argument"
+    ATTRIBUTE = "attribute"
+    VARIABLE = "variable"
+
+
+class PrefixSeparator(Enum):
+    ARGUMENT = "-"
+    ATTRIBUTE = "."
+    VARIABLE = "_"
+
+
+@dataclasses.dataclass
+class ConfigMapping:
+    attribute: str
+    argument: str | None = None
+    variable: str | None = None
+    default: Any = None
+    description: str = None
+    group: str | list[str] | None = None
+
+    def __post_init__(self):
+        self.argument = (self.argument or self.attribute).lower()
+        self.variable = (self.variable or self.attribute).upper()
+
+        if isinstance(self.group, str):
+            self.group = [self.group]
+        elif isinstance(self.group, list):
+            self.group = self.group
+        else:
+            self.group = []
+
+    def as_dict(self) -> dict[Any, Any]:
+        """Return class attributes as dictionary"""
+        return dataclasses.asdict(self)
+
+    def as_tuple(self) -> list[tuple[Any, Any]]:
+        """Return class attributes as list of tuples"""
+        return list(dataclasses.asdict(self).items())
+
+
+class BaseConfig:
+    mappings: list[ConfigMapping] | None = None
+
+    def __init__(self, mappings: Optional[Sequence[ConfigMapping]] = None):
+        mappings = deepcopy(mappings) if mappings is not None else []
+
+        self.mappings = self.mappings or []
+        self.merge(mappings=mappings)
+        self.update()
+
+    def generate_mappings(self) -> None:
+        """Generate mappings for attributes that were set directly"""
+        attributes = [(k, v) for k, v in vars(self).items() if k != "mappings"]
+        _mapped = set([x.attribute for x in self.mappings])
+        _unmapped = [(k, v) for k, v in attributes if k not in _mapped]
+
+        mappings = [ConfigMapping(attribute=k, default=v) for k, v in _unmapped]
+
+        self.merge(mappings=mappings)
+
+    def merge(self, mappings: Optional[Sequence[ConfigMapping]]) -> None:
+        """Merge multiple ConfigMapping definitions"""
+        assert type(mappings) in (tuple, list), f"Not a valid sequence: {mappings=}"
+
+        mappings = deepcopy(mappings)
+
+        _mappings_merged = {x.attribute: x for x in [*self.mappings, *mappings]}
+        self.mappings = list(_mappings_merged.values())
+
+    def remove_unmapped_attributes(self):
+        """Remove attributes without valid attribute mappings, retain groups"""
+        attributes = set([k for k, v in vars(self).items() if k != "mappings"])
+        _mapped = set([x.attribute for x in self.mappings])
+
+        for attribute in set(attributes).difference(_mapped):
+            if isinstance(getattr(self, attribute), BaseConfig):
+                continue
+
+            delattr(self, attribute)
+
+    def _set_prefix(
+        self,
+        field: ConfigField,
+        prefix: str,
+        group: str | list[str] | None = None,
+        merge_group: bool = False,
+    ):
+        """Set a common prefix for arguments or variables, optionally group-only"""
+        fields = {
+            ConfigField.ARGUMENT: (
+                ConfigField.ARGUMENT.value,
+                PrefixSeparator.ARGUMENT.value,
+            ),
+            ConfigField.VARIABLE: (
+                ConfigField.VARIABLE.value,
+                PrefixSeparator.VARIABLE.value,
+            ),
+        }
+
+        conversions = {
+            ConfigField.ARGUMENT: lambda x: x.lower(),
+            ConfigField.VARIABLE: lambda x: x.upper(),
+        }
+
+        _field, _separator = fields[field]
+        _prefix = conversions[field](prefix)
+
+        _group = group or []
+        _group = [group] if isinstance(group, str) else _group
+        _group = [conversions[field](g) for g in _group]
+
+        mappings = (
+            [x for x in self.mappings if x.group == _group]
+            if group is not None
+            else self.mappings
+        )
+
+        _mappings = []
+
+        for mapping in mappings:
+            _mapping = mapping.as_dict()
+
+            if isinstance(_group, list) and merge_group is True:
+                _members = [_prefix, *_group, _mapping[_field]]
+            else:
+                _members = [_prefix, _mapping[_field]]
+
+            _prefixed = _separator.join(_members)
+
+            _mappings.append(ConfigMapping(**{**_mapping, _field: _prefixed}))
+
+        self.merge(mappings=_mappings)
+
+    def set_argument_prefix(self, prefix: str, group: str = None):
+        """Set a common prefix for arguments, optionally group-only"""
+        self._set_prefix(field=ConfigField.ARGUMENT, prefix=prefix, group=group)
+
+    def set_variable_prefix(self, prefix: str, group: str = None):
+        """Set a common prefix for arguments, optionally group-only"""
+        self._set_prefix(field=ConfigField.VARIABLE, prefix=prefix, group=group)
+
+    def update(self, reset: bool = False) -> None:
+        """Update attributes self.mappings"""
+        for mapping in self.mappings:
+            if (
+                hasattr(self, mapping.attribute)
+                and getattr(self, mapping.attribute) != mapping.default
+                and reset is False
+            ):
+                continue
+
+            setattr(self, mapping.attribute, mapping.default)
+
+    def update_from_args(self, args: argparse.Namespace) -> None:
+        """Update attribute values from argparse arguments"""
+        # Transform Namespace into dict, remove None values to honour defaults
+        _args = {k: v for k, v in vars(args).items() if v is not None}
+
+        for m in self.mappings:
+            setattr(
+                self, m.attribute, _args.get(m.argument, getattr(self, m.attribute))
+            )
+
+    def update_from_env(self) -> None:
+        """Update attribute values from environment variables"""
+        for m in self.mappings:
+            _variable = m.variable.upper()
+
+            setattr(self, m.attribute, os.getenv(_variable, getattr(self, m.attribute)))
+
+    def update_groups(self) -> None:
+        """Recursively create groups from mappings, resulting in a nested class tree"""
+
+        def __grouper(_mapping: ConfigMapping) -> str:
+            return _mapping.group[0]
+
+        _groups = [x for x in self.mappings if x.group not in ([], None)]
+        _sorted = sorted(_groups, key=__grouper)
+        _grouped = [(k, list(g)) for k, g in itertools.groupby(_sorted, __grouper)]
+
+        for group, mappings in _grouped:
+            # Remove mappings from parent
+            self.mappings = [x for x in self.mappings if x not in mappings]
+
+            _mappings = deepcopy(mappings)
+
+            # Push mapping groups down one level
+            for mapping in _mappings:
+                mapping.group = mapping.group[1:]
+
+            if not hasattr(self, group):
+                setattr(self, group, BaseConfig())
+
+            _group = getattr(self, group)
+            _group.merge(mappings=_mappings)
+
+            # Recurse into subgroups
+            if any([x.group is not None for x in _group.mappings]):
+                _group.update_groups()
+
+            # Update ungrouped mappings
+            _group.update()
+
+        # Clean up any attributes which have been pushed into a group
+        self.remove_unmapped_attributes()
+
+    def validate(self):
+        """Validate that mappings are correct and mapped attributes exist"""
+        assert type(self.mappings) in (
+            list,
+            tuple,
+        ), f"Invalid sequence: {self.mappings=}"
+
+        for mapping in self.mappings:
+            assert isinstance(
+                mapping, ConfigMapping
+            ), f"Invalid mapping type: {mapping=}"
+            assert hasattr(
+                self, mapping.attribute
+            ), f"Missing attribute: {mapping.attribute}"

rmk2/data/__init__.py

@@ -0,0 +1 @@
+from rmk2.data.data import Data

rmk2/data/data.py

@@ -0,0 +1,70 @@
+from typing import Any, Self
+
+import rmk2.data.dictionary
+import rmk2.data.object
+
+
+class Data:
+    """@DynamicAttrs"""
+    __annotations__: dict = {}
+
+    def __init__(self, **kwargs):
+        for k, v in kwargs.items():
+            if isinstance(v, dict):
+                setattr(self, k, type(self)(**v))
+                self.__annotations__[k] = type(self)
+            else:
+                setattr(self, k, v)
+                self.__annotations__[k] = type(v)
+
+    def __init_subclass__(cls, **kwargs):
+        cls.__init__ = rmk2.data.object.get_metaclass(cls).__init__
+
+        return cls
+
+    def __repr__(self) -> str:
+        return str(vars(self))
+
+    def __eq__(self, other):
+        return (
+            self.__todict__() == other.__todict__()
+            if isinstance(other, type(self))
+            else False
+        )
+
+    def __todict__(self) -> dict:
+        """Recursively cast instance to dict"""
+        return {
+            k: v.__todict__() if isinstance(v, type(self)) else v
+            for k, v in vars(self).items()
+        }
+
+    def get_path(self, path: list[str] | str) -> Any | None:
+        """Get attribute from an object, nested under a given path"""
+        return rmk2.data.object.get_path(cls=self, path=path)
+
+    def merge(self, other: Self | dict) -> Self:
+        """Recursively merge data into the current instance, key by key"""
+        self.__dict__ = vars(
+            rmk2.data.object.merge(
+                left=self,
+                right=type(self)(**other) if isinstance(other, dict) else other,
+            )
+        )
+
+        return self
+
+    def put_path(self, path: list[str] | str, value: Any) -> Self:
+        """Put a value as an attribute at a given path into an object"""
+        self.merge(
+            other=rmk2.data.dictionary.put_path(
+                data={}, path=path, value=value
+            )
+        )
+
+        return self
+
+    @classmethod
+    def from_path(cls, path: list[str] | str, value: Any) -> Self:
+        """Initialise instance directly by putting a value at a given path"""
+        return cls().put_path(path=path, value=value)

rmk2/data/dictionary.py

@@ -0,0 +1,47 @@
+import functools
+from typing import Any, TypeVar
+
+T = TypeVar("T")
+
+
+def merge(
+    left: dict[str, Any], right: dict[str, Any], merge_lists: bool = False
+) -> dict[str, Any]:
+    """Recursively merge two dictionaries, key by key"""
+    result = {}
+
+    # Combine keys via iteration instead of a set to maintain key ordering
+    for k in [*left.keys(), *[x for x in right.keys() if x not in left.keys()]]:
+        _left = left.get(k)
+        _right = right.get(k)
+
+        if isinstance(_left, dict) and isinstance(_right, dict):
+            merged = merge(left=_left, right=_right, merge_lists=merge_lists)
+        elif isinstance(_left, list) and isinstance(_right, list) and merge_lists:
+            merged = [*_left, *_right]
+        elif not _left and _right is not None:
+            merged = _right
+        else:
+            merged = _right or _left
+
+        result[k] = merged
+
+    return result
+
+
+def get_path(
+    data: dict[str, dict | T], path: list[str] | str, default: Any | None = None
+) -> T | None:
+    """Get a value at a given path from a dictionary"""
+    path = path.split(".") if isinstance(path, str) else path
+    result = functools.reduce(lambda a, b: a.get(b, {}), path, data)
+
+    return default if (result == {} and default) else result
+
+
+def put_path(data: dict[str, dict], path: list[str] | str, value: Any) -> dict:
+    """Put a value at a given path into a dictionary"""
+    path = path.split(".") if isinstance(path, str) else path
+    branch = functools.reduce(lambda a, b: {b: a}, reversed(path), value)
+
+    return merge(left=data, right=branch, merge_lists=False)

rmk2/data/object.py

@@ -0,0 +1,93 @@
+import copy
+import dataclasses
+import functools
+import inspect
+from typing import TypeVar, Callable, Type, Any
+
+import rmk2.data.dictionary
+
+T = TypeVar("T")
+
+
+class Missing:
+    pass
+
+
+def merge(left: T, right: T, merge_lists: bool = False, merge_dicts: bool = False) -> T:
+    """Recursively merge two class instances, key by key"""
+    assert type(left) is type(
+        right
+    ), f"Cannot merge different Types, left='{type(left)}', right='{type(right)}'"
+
+    primitives = [int, str, float, bool, list, dict, set, type(None)]
+    kwargs = {}
+
+    for k in {*vars(left).keys(), *vars(right).keys()}:
+        _left = getattr(left, k, None)
+        _right = getattr(right, k, None)
+
+        if (
+            type(_left) not in primitives
+            and type(_right) not in primitives
+            and type(_left) is type(_right)
+        ):
+            merged = merge(
+                left=_left,
+                right=_right,
+                merge_lists=merge_lists,
+                merge_dicts=merge_dicts,
+            )
+        elif isinstance(_left, list) and isinstance(_right, list) and merge_lists:
+            merged = [*_left, *_right]
+        elif isinstance(_left, dict) and isinstance(_right, dict) and merge_dicts:
+            merged = rmk2.data.dictionary.merge(
+                left=_left, right=_right, merge_lists=merge_lists
+            )
+        elif not _left and _right is not None:
+            merged = _right
+        else:
+            merged = _right or _left
+
+        kwargs[k] = merged
+
+    return type(left)(**kwargs)
+
+
+def get_path(cls: object, path: list[str] | str, default: Any | None = None) -> Any:
+    """Get attribute from an object, nested under a given path"""
+    path = path.split(".") if isinstance(path, str) else path
+    result = functools.reduce(lambda a, b: getattr(a, b, Missing), path, cls)
+
+    return default if result == Missing else result
+
+
+def get_metaclass(cls: object) -> Type:
+    """Get the root metaclass for a given object"""
+    _tree = inspect.getclasstree([cls if isinstance(cls, Type) else type(cls)])
+
+    return _tree[0][0] if _tree else type(None)
+
+
+def pushdown(func: Callable, parent: list[str] = None) -> Callable:
+    """Recursively apply a class method to all subclasses of a given class"""
+    parent = parent or []
+
+    @functools.wraps(func)
+    def wrapper(cls: T, *args, **kwargs) -> T:
+        _fields = {
+            f.name
+            for f in dataclasses.fields(cls)
+            if dataclasses.is_dataclass(f.type)
+            and issubclass(f.type, get_metaclass(cls))
+        }
+
+        for k in _fields:
+            # Affect current subclass
+            func(getattr(cls, k), *args, **kwargs)
+
+            # Recurse further for nodes underneath current subclass
+            pushdown(func, parent=[*parent, k])(getattr(cls, k), *args, **kwargs)
+
+        return cls
+
+    return wrapper

rmk2/file.py

@@ -0,0 +1,102 @@
+import datetime
+import json
+import logging
+import os
+import pathlib
+from enum import Enum
+from typing import Iterator, Any
+
+Expected = bool | str | int | float | datetime.date | datetime.datetime | None
+Jsonified = bool | str | int | float | None
+
+
+class WriteMode(Enum):
+    APPEND = "a"
+    CREATE = "x"
+    TRUNCATE = "w"
+
+
+def _jsonify_types(row: dict[str, Expected]) -> dict[str, Jsonified]:
+    """Cast types that are not supported by JSON to more compatible types"""
+    _castable_types = {
+        datetime.date: lambda x: x.isoformat(),
+        datetime.datetime: lambda x: x.isoformat(timespec="microseconds"),
+        datetime.timedelta: lambda x: str(x),
+    }
+
+    return {k: _castable_types.get(type(v), lambda x: x)(v) for k, v in row.items()}
+
+
+def write_jsonl(
+    data: Iterator | list[list[tuple[str, Any]] | dict[str, Any]],
+    path: pathlib.Path | str,
+    mode: WriteMode = WriteMode.CREATE,
+) -> None:
+    """Write serialised data to a given path/file"""
+    path = pathlib.Path(path)  # Ensure path is a Path object
+
+    try:
+        assert isinstance(
+            mode, WriteMode
+        ), f"Mode needs to be one of {[x.name for x in WriteMode]}"
+
+        with open(path, mode=mode.value, encoding="utf-8") as outfile:
+            logging.debug(f"Writing serialised data, {path=}")
+
+            for line in data:
+                if isinstance(line, dict):
+                    _line = line
+                elif isinstance(line, list) and isinstance(line[0], tuple):
+                    _line = dict(line)
+                else:
+                    raise ValueError("Data must be a list of key/value pairs")
+
+                # Add OS-specific newline after each row
+                outfile.write(json.dumps(dict(_jsonify_types(_line))))
+                outfile.write(os.linesep)
+
+    except (AssertionError, FileExistsError) as e:
+        logging.error(str(e))
+        raise e
+
+
+def read_jsonl(
+    path: pathlib.Path | str,
+) -> Iterator | list[list[tuple[str, Jsonified]]]:
+    """Read JSONL serialised data from a given path/file"""
+    path = pathlib.Path(path)  # Ensure path is a Path object
+
+    try:
+        with open(path, mode="r", encoding="utf-8") as infile:
+            logging.debug(f"Reading serialised data, {path=}")
+
+            yield from iter(json.loads(line) for line in infile)
+
+    except FileNotFoundError as e:
+        logging.error(str(e))
+        raise e
+
+
+def delete_file(path: pathlib.Path | str) -> None:
+    """Delete a given data file"""
+    path = pathlib.Path(path)  # Ensure path is a Path object
+
+    try:
+        logging.debug(f"Deleting serialised data, {path=}")
+        os.remove(path)
+    except FileNotFoundError as e:
+        logging.error(str(e))
+        raise e
+
+
+def count_file(path: pathlib.Path | str) -> int:
+    """Count number of lines in a given path/file"""
+    path = pathlib.Path(path)  # Ensure path is a Path object
+
+    _idx = 0
+
+    with open(path, mode="rb") as infile:
+        for _idx, _ in enumerate(infile, start=1):
+            pass
+
+        return _idx

rmk2/hash.py

@@ -0,0 +1,55 @@
+import datetime
+import hashlib
+import logging
+from enum import Enum
+from typing import Any
+
+
+class HashAlgorithm(Enum):
+    SHA256 = hashlib.sha256
+    SHA384 = hashlib.sha384
+    SHA512 = hashlib.sha512
+
+
+def hash_values(
+    *args: Any,
+    algorithm: HashAlgorithm = HashAlgorithm.SHA256,
+    replace_null: str = "",
+) -> str:
+    """Hash various items to create consistent digests for a given value
+
+    Casts its input(s) to str, encodes it as UTF-8 bytes, then hashes it"""
+    _types = (
+        int,
+        float,
+        str,
+        bool,
+        datetime.datetime,
+        datetime.date,
+        datetime.timedelta,
+    )
+
+    assert (
+        algorithm in HashAlgorithm
+    ), f"Algorithm needs to be one of {[x.name for x in HashAlgorithm]}"
+
+    try:
+        _hash = algorithm.value(usedforsecurity=True)
+
+        for value in args:
+            # If replace_null is defined, replace NULL values with an unlikely string
+            # to ensure that (1, None) and (None, 1) are different hashes
+            if value is None or value == "":
+                value = replace_null
+            else:
+                assert (
+                    type(value) in _types
+                ), f"Cannot cast to string, type={type(value)}"
+
+            _hash.update(str(value).encode(encoding="utf-8"))
+
+        return _hash.hexdigest()
+
+    except AssertionError as e:
+        logging.error(str(e))
+        raise e