omlish 0.0.0.dev6__py3-none-any.whl → 0.0.0.dev8__py3-none-any.whl

omlish/reflect/ops.py

@@ -0,0 +1,84 @@
+import typing as ta
+
+from .types import Annotated
+from .types import Any
+from .types import Generic
+from .types import NewType
+from .types import Type
+from .types import Union
+from .types import type_
+
+
+def strip_objs(ty: Type) -> Type:
+    if isinstance(ty, (type, ta.TypeVar, NewType, Any)):
+        return ty
+
+    if isinstance(ty, Union):
+        return Union(frozenset(map(strip_objs, ty.args)))
+
+    if isinstance(ty, Generic):
+        return Generic(ty.cls, tuple(map(strip_objs, ty.args)), ty.params, None)
+
+    if isinstance(ty, Annotated):
+        return Annotated(strip_objs(ty.ty), ty.md, None)
+
+    raise TypeError(ty)
+
+
+def strip_annotations(ty: Type) -> Type:
+    if isinstance(ty, (type, ta.TypeVar, NewType, Any)):
+        return ty
+
+    if isinstance(ty, Union):
+        return Union(frozenset(map(strip_annotations, ty.args)))
+
+    if isinstance(ty, Generic):
+        return Generic(ty.cls, tuple(map(strip_annotations, ty.args)), ty.params, ty.obj)
+
+    if isinstance(ty, Annotated):
+        return strip_annotations(ty.ty)
+
+    raise TypeError(ty)
+
+
+def types_equivalent(l: Type, r: Type) -> bool:
+    if isinstance(l, Generic) and isinstance(r, Generic):
+        return l.cls == r.cls and l.args == r.args
+
+    return l == r
+
+
+def get_underlying(nt: NewType) -> Type:
+    return type_(nt.obj.__supertype__)  # noqa
+
+
+def get_concrete_type(ty: Type) -> type | None:
+    if isinstance(ty, type):
+        return ty
+
+    if isinstance(ty, Generic):
+        return ty.cls
+
+    if isinstance(ty, NewType):
+        return get_concrete_type(get_underlying(ty))
+
+    if isinstance(ty, (Union, ta.TypeVar, Any)):
+        return None
+
+    raise TypeError(ty)
+
+
+def to_annotation(ty: Type) -> ta.Any:
+    if isinstance(ty, Generic):
+        return ty.obj if ty.obj is not None else ty.cls
+
+    if isinstance(ty, Union):
+        return ta.Union[*tuple(to_annotation(e) for e in ty.args)]
+
+    if isinstance(ty, (type, ta.TypeVar, NewType)):
+        return ty
+
+    if isinstance(ty, Any):
+        return ta.Any
+
+    raise TypeError(ty)

omlish/reflect/subst.py

@@ -0,0 +1,110 @@
+import dataclasses as dc
+import types
+import typing as ta
+
+from .. import c3
+from .. import lang
+from .ops import get_concrete_type
+from .ops import to_annotation
+from .types import Any
+from .types import Generic
+from .types import NewType
+from .types import Type
+from .types import Union
+from .types import get_params
+from .types import type_
+
+
+if ta.TYPE_CHECKING:
+    from ..collections import cache
+else:
+    cache = lang.proxy_import('.collections.cache', __package__)
+
+
+def get_type_var_replacements(ty: Type) -> ta.Mapping[ta.TypeVar, Type]:
+    if isinstance(ty, Generic):
+        return dict(zip(ty.params, ty.args))
+
+    return {}
+
+
+def replace_type_vars(
+        ty: Type,
+        rpl: ta.Mapping[ta.TypeVar, Type],
+        *,
+        update_aliases: bool = False,
+) -> Type:
+    def rec(cur):
+        if isinstance(cur, (type, NewType, Any)):
+            return cur
+
+        if isinstance(cur, Generic):
+            args = tuple(rec(a) for a in cur.args)
+            if update_aliases:
+                obj = cur.obj
+                if (ops := get_params(obj)):
+                    nargs = [to_annotation(rpl[p]) for p in ops]
+                    if ta.get_origin(obj) is ta.Generic:
+                        # FIXME: None? filter_typing_generic in get_generic_bases?
+                        pass
+                    else:
+                        obj = cur.obj[*nargs]
+            else:
+                obj = None
+            return dc.replace(cur, args=args, obj=obj)
+
+        if isinstance(cur, Union):
+            return Union(frozenset(rec(e) for e in cur.args))
+
+        if isinstance(cur, ta.TypeVar):
+            return rpl[cur]
+
+        raise TypeError(cur)
+
+    return rec(ty)
+
+
+class GenericSubstitution:
+    def __init__(
+            self,
+            *,
+            update_aliases: bool = False,
+            cache_size: int = 0,  # FIXME: ta.Generic isn't weakrefable..
+    ) -> None:
+        super().__init__()
+
+        self._update_aliases = update_aliases
+
+        if cache_size > 0:
+            self.get_generic_bases = cache.cache(weak_keys=True, max_size=cache_size)(self.get_generic_bases)  # type: ignore  # noqa
+            self.generic_mro = cache.cache(weak_keys=True, max_size=cache_size)(self.generic_mro)  # type: ignore
+
+    def get_generic_bases(self, ty: Type) -> tuple[Type, ...]:
+        if (cty := get_concrete_type(ty)) is not None:
+            rpl = get_type_var_replacements(ty)
+            ret: list[Type] = []
+            for b in types.get_original_bases(cty):
+                bty = type_(b)
+                if isinstance(bty, Generic) and isinstance(b, type):
+                    # FIXME: throws away relative types, but can't use original vars as they're class-contextual
+                    bty = type_(b[*((ta.Any,) * len(bty.params))])  # type: ignore
+                rty = replace_type_vars(bty, rpl, update_aliases=self._update_aliases)
+                ret.append(rty)
+            return tuple(ret)
+        return ()
+
+    def generic_mro(self, obj: ta.Any) -> list[Type]:
+        mro = c3.mro(
+            type_(obj),
+            get_bases=lambda t: self.get_generic_bases(t),
+            is_subclass=lambda l, r: issubclass(get_concrete_type(l), get_concrete_type(r)),  # type: ignore
+        )
+        return [ty for ty in mro if get_concrete_type(ty) is not ta.Generic]
+
+
+DEFAULT_GENERIC_SUBSTITUTION = GenericSubstitution()
+
+get_generic_bases = DEFAULT_GENERIC_SUBSTITUTION.get_generic_bases
+generic_mro = DEFAULT_GENERIC_SUBSTITUTION.generic_mro
+
+ALIAS_UPDATING_GENERIC_SUBSTITUTION = GenericSubstitution(update_aliases=True)

omlish/reflect/types.py

@@ -0,0 +1,275 @@
+"""
+TODO:
+ - visitor / transformer
+ - uniform collection isinstance - items() for mappings, iter() for other
+ - also check instance type in isinstance not just items lol
+ - ta.Generic in mro causing trouble - omit? no longer 1:1
+ - cache this shit, esp generic_mro shit
+  - cache __hash__ in Generic/Union
+"""
+import dataclasses as dc
+import types
+import typing as ta
+
+
+_NoneType = types.NoneType  # type: ignore
+
+_NONE_TYPE_FROZENSET: frozenset['Type'] = frozenset([_NoneType])
+
+
+_GenericAlias = ta._GenericAlias  # type: ignore  # noqa
+_CallableGenericAlias = ta._CallableGenericAlias  # type: ignore  # noqa
+_SpecialGenericAlias = ta._SpecialGenericAlias  # type: ignore  # noqa
+_UnionGenericAlias = ta._UnionGenericAlias   # type: ignore  # noqa
+_AnnotatedAlias = ta._AnnotatedAlias  # type: ignore  # noqa
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class _Special:
+    name: str
+    alias: _SpecialGenericAlias  # type: ignore
+    origin: type
+    nparams: int
+
+
+_KNOWN_SPECIALS = [
+    _Special(
+        v._name,  # noqa
+        v,
+        v.__origin__,
+        v._nparams,  # noqa
+    )
+    for v in ta.__dict__.values()
+    if isinstance(v, _SpecialGenericAlias)
+]
+
+_KNOWN_SPECIALS_BY_NAME = {s.name: s for s in _KNOWN_SPECIALS}
+_KNOWN_SPECIALS_BY_ALIAS = {s.alias: s for s in _KNOWN_SPECIALS}
+_KNOWN_SPECIALS_BY_ORIGIN = {s.origin: s for s in _KNOWN_SPECIALS}
+
+_MAX_KNOWN_SPECIAL_TYPE_VARS = 16
+
+_KNOWN_SPECIAL_TYPE_VARS = tuple(
+    ta.TypeVar(f'_{i}')  # noqa
+    for i in range(_MAX_KNOWN_SPECIAL_TYPE_VARS)
+)
+
+
+##
+
+
+def get_params(obj: ta.Any) -> tuple[ta.TypeVar, ...]:
+    if isinstance(obj, type):
+        if issubclass(obj, ta.Generic):  # type: ignore
+            return obj.__dict__.get('__parameters__', ())  # noqa
+
+        if (ks := _KNOWN_SPECIALS_BY_ORIGIN.get(obj)) is not None:
+            return _KNOWN_SPECIAL_TYPE_VARS[:ks.nparams]
+
+    oty = type(obj)
+
+    if (
+            oty is _GenericAlias or
+            oty is ta.GenericAlias  # type: ignore  # noqa
+    ):
+        return obj.__dict__.get('__parameters__', ())  # noqa
+
+    if oty is _CallableGenericAlias:
+        raise NotImplementedError('get_params not yet implemented for typing.Callable')
+
+    raise TypeError(obj)
+
+
+def is_union_type(cls: ta.Any) -> bool:
+    if hasattr(ta, 'UnionType'):
+        return ta.get_origin(cls) in {ta.Union, getattr(ta, 'UnionType')}
+
+    else:
+        return ta.get_origin(cls) in {ta.Union}
+
+
+def get_orig_class(obj: ta.Any) -> ta.Any:
+    return obj.__orig_class__  # noqa
+
+
+##
+
+
+Type: ta.TypeAlias = ta.Union[
+    type,
+    ta.TypeVar,
+    'Union',
+    'Generic',
+    'NewType',
+    'Annotated',
+    'Any',
+]
+
+
+@dc.dataclass(frozen=True)
+class Union:
+    args: frozenset[Type]
+
+    @property
+    def is_optional(self) -> bool:
+        return _NoneType in self.args
+
+    def without_none(self) -> Type:
+        if _NoneType not in self.args:
+            return self
+        rem = self.args - _NONE_TYPE_FROZENSET
+        if len(rem) == 1:
+            return next(iter(rem))
+        return Union(rem)
+
+
+@dc.dataclass(frozen=True)
+class Generic:
+    cls: type
+    args: tuple[Type, ...]                                                # map[int, V] = (int, V) | map[T, T] = (T, T)
+
+    params: tuple[ta.TypeVar, ...] = dc.field(compare=False, repr=False)  # map[int, V] = (_0, _1) | map[T, T] = (_0, _1)  # noqa
+    # params2: tuple[ta.TypeVar, ...]                                     # map[int, V] = (V,)     | map[T, T] = (T,)
+
+    obj: ta.Any = dc.field(compare=False, repr=False)
+
+    # def __post_init__(self) -> None:
+    #     if not isinstance(self.cls, type):
+    #         raise TypeError(self.cls)
+
+    def full_eq(self, other: 'Generic') -> bool:
+        return (
+            self.cls == other.cls and
+            self.args == other.args and
+            self.params == other.params and
+            self.obj == other.obj
+        )
+
+
+@dc.dataclass(frozen=True)
+class NewType:
+    obj: ta.Any
+
+
+@dc.dataclass(frozen=True)
+class Annotated:
+    ty: Type
+    md: ta.Sequence[ta.Any]
+
+    obj: ta.Any = dc.field(compare=False, repr=False)
+
+
+class Any:
+    pass
+
+
+ANY = Any()
+
+
+TYPES: tuple[type, ...] = (
+    type,
+    ta.TypeVar,
+    Union,
+    Generic,
+    NewType,
+    Annotated,
+    Any,
+)
+
+
+##
+
+
+def is_type(obj: ta.Any) -> bool:
+    if isinstance(obj, (Union, Generic, ta.TypeVar, NewType, Any)):  # noqa
+        return True
+
+    oty = type(obj)
+
+    return (
+            oty is _UnionGenericAlias or oty is types.UnionType or  # noqa
+
+            isinstance(obj, ta.NewType) or  # noqa
+
+            (
+                oty is _GenericAlias or
+                oty is ta.GenericAlias or  # type: ignore  # noqa
+                oty is _CallableGenericAlias
+            ) or
+
+            isinstance(obj, type) or
+
+            isinstance(obj, _SpecialGenericAlias)
+    )
+
+
+def type_(obj: ta.Any) -> Type:
+    if obj is ta.Any:
+        return ANY
+
+    if isinstance(obj, (Union, Generic, ta.TypeVar, NewType, Any)):  # noqa
+        return obj
+
+    oty = type(obj)
+
+    if oty is _UnionGenericAlias or oty is types.UnionType:
+        return Union(frozenset(type_(a) for a in ta.get_args(obj)))
+
+    if isinstance(obj, ta.NewType):  # noqa
+        return NewType(obj)
+
+    if (
+            oty is _GenericAlias or
+            oty is ta.GenericAlias or  # type: ignore  # noqa
+            oty is _CallableGenericAlias
+    ):
+        origin = ta.get_origin(obj)
+        args = ta.get_args(obj)
+        if oty is _CallableGenericAlias:
+            p, r = args
+            if p is Ellipsis or isinstance(p, ta.ParamSpec):
+                raise TypeError(f'Callable argument not yet supported for {obj=}')
+            args = (*p, r)
+            params = _KNOWN_SPECIAL_TYPE_VARS[:len(args)]
+        elif origin is ta.Generic:
+            params = args
+        else:
+            params = get_params(origin)
+        if len(args) != len(params):
+            raise TypeError(f'Mismatched {args=} and {params=} for {obj=}')
+        return Generic(
+            origin,
+            tuple(type_(a) for a in args),
+            params,
+            obj,
+        )
+
+    if isinstance(obj, type):
+        if issubclass(obj, ta.Generic):  # type: ignore
+            params = get_params(obj)
+            return Generic(
+                obj,
+                params,
+                params,
+                obj,
+            )
+        return obj
+
+    if isinstance(obj, _SpecialGenericAlias):
+        if (ks := _KNOWN_SPECIALS_BY_ALIAS.get(obj)) is not None:
+            params = _KNOWN_SPECIAL_TYPE_VARS[:ks.nparams]
+            return Generic(
+                ks.origin,
+                params,
+                params,
+                obj,
+            )
+
+    if isinstance(obj, _AnnotatedAlias):
+        o = ta.get_args(obj)[0]
+        return Annotated(type_(o), md=obj.__metadata__, obj=obj)
+
+    raise TypeError(obj)

omlish/secrets/__init__.py

@@ -1,7 +1,23 @@
 from .secrets import (  # noqa
+    CachingSecrets,
+    CompositeSecrets,
     EMPTY_SECRETS,
     EmptySecrets,
-    Secret,
+    EnvVarSecrets,
+    FnSecrets,
+    LoggingSecrets,
+    MappingSecrets,
+    SecretRef,
+    SecretRefOrStr,
     Secrets,
-    SimpleSecrets,
+    secret_field,
+    secret_repr,
 )
+
+
+##
+
+
+from ..lang.imports import _register_conditional_import  # noqa
+
+_register_conditional_import('..marshal', '.marshal', __package__)

omlish/secrets/crypto.py

@@ -0,0 +1,132 @@
+"""
+TODO:
+ - cryptography vs pycryptodome[x]
+ - standardize failure exception
+ - chains - take first
+ - keysets
+
+See:
+ - https://soatok.blog/2020/05/13/why-aes-gcm-sucks/
+ - https://pycryptodome.readthedocs.io/en/latest/src/cipher/chacha20_poly1305.html
+
+==
+
+https://www.tecmint.com/gpg-encrypt-decrypt-files/
+
+==
+
+gpg --batch --passphrase '' --quick-gen-key wrmsr default default
+gpg --batch --passphrase '' --quick-gen-key wrmsr2 default default
+echo 'hi there' > secret.txt
+gpg -e -u wrmsr -r wrmsr2 secret.txt
+gpg -d -o secret2.txt secret.txt.gpg
+
+gpg --batch -c --passphrase-file /var/secret.key -o some.gpg toencrypt.txt
+
+openssl rand -rand /dev/urandom 128 > barf.key
+openssl enc -in secret.txt -out secret.txt.enc -e -aes256 -pbkdf2 -kfile barf.key
+openssl aes-256-cbc -d -pbkdf2 -in secret.txt.enc -out secret3.txt -kfile barf.key
+
+https://wiki.openssl.org/index.php/Enc#Options
+-pass 'file:...'
+"""
+import abc
+import secrets
+import typing as ta
+
+from .. import lang
+
+
+if ta.TYPE_CHECKING:
+    from cryptography import exceptions as cry_exc
+    from cryptography import fernet as cry_fernet
+    from cryptography.hazmat.primitives.ciphers import aead as cry_aead
+else:
+    cry_aead = lang.proxy_import('cryptography.hazmat.primitives.ciphers.aead')
+    cry_exc = lang.proxy_import('cryptography.exceptions')
+    cry_fernet = lang.proxy_import('cryptography.fernet')
+
+
+##
+
+
+class EncryptionError(Exception):
+    pass
+
+
+class DecryptionError(Exception):
+    pass
+
+
+class Crypto(abc.ABC):
+    @abc.abstractmethod
+    def generate_key(self) -> bytes:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def encrypt(self, data: bytes, key: bytes) -> bytes:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def decrypt(self, data: bytes, key: bytes) -> bytes:
+        raise NotImplementedError
+
+
+##
+
+
+class FernetCrypto(Crypto):
+
+    def generate_key(self) -> bytes:
+        return cry_fernet.Fernet.generate_key()
+
+    def encrypt(self, data: bytes, key: bytes) -> bytes:
+        try:
+            f = cry_fernet.Fernet(key)
+        except ValueError as e:
+            raise EncryptionError from e
+        return f.encrypt(data)
+
+    def decrypt(self, data: bytes, key: bytes) -> bytes:
+        try:
+            f = cry_fernet.Fernet(key)
+        except ValueError as e:
+            raise DecryptionError from e
+        try:
+            return f.decrypt(data)
+        except cry_fernet.InvalidToken as e:
+            raise DecryptionError from e
+
+
+class AesgsmCrypto(Crypto):
+    """https://stackoverflow.com/a/59835994"""
+
+    def generate_key(self) -> bytes:
+        return secrets.token_bytes(32)
+
+    def encrypt(self, data: bytes, key: bytes) -> bytes:
+        nonce = secrets.token_bytes(12)
+        return nonce + cry_aead.AESGCM(key).encrypt(nonce, data, b'')
+
+    def decrypt(self, data: bytes, key: bytes) -> bytes:
+        try:
+            return cry_aead.AESGCM(key).decrypt(data[:12], data[12:], b'')
+        except cry_exc.InvalidTag as e:
+            raise DecryptionError from e
+
+
+class Chacha20Poly1305Crypto(Crypto):
+    """https://cryptography.io/en/latest/hazmat/primitives/aead/#cryptography.hazmat.primitives.ciphers.aead.ChaCha20Poly1305"""  # noqa
+
+    def generate_key(self) -> bytes:
+        return cry_aead.ChaCha20Poly1305.generate_key()
+
+    def encrypt(self, data: bytes, key: bytes) -> bytes:
+        nonce = secrets.token_bytes(12)
+        return nonce + cry_aead.ChaCha20Poly1305(key).encrypt(nonce, data, b'')
+
+    def decrypt(self, data: bytes, key: bytes) -> bytes:
+        try:
+            return cry_aead.ChaCha20Poly1305(key).decrypt(data[:12], data[12:], b'')
+        except cry_exc.InvalidTag as e:
+            raise DecryptionError from e

omlish/secrets/marshal.py

@@ -1,23 +1,30 @@
+"""
+TODO:
+ - ensure import order or at least warn or smth lol
+ - raise exception on ambiguous 'registered' impls
+"""
 import collections.abc
 import typing as ta
 
 from .. import check
 from .. import dataclasses as dc
+from .. import lang
 from .. import marshal as msh
+from .. import reflect as rfl
 from .secrets import Secret
+from .secrets import SecretRef
+from .secrets import SecretRefOrStr
 
 
-class StrOrSecretMarshaler(msh.Marshaler):
+class StrOrSecretRefMarshalerUnmarshaler(msh.Marshaler, msh.Unmarshaler):
     def marshal(self, ctx: msh.MarshalContext, o: ta.Any) -> msh.Value:
         if isinstance(o, str):
             return o
-        elif isinstance(o, Secret):
+        elif isinstance(o, SecretRef):
             return {'secret': o.key}
         else:
             raise TypeError(o)
 
-
-class StrOrSecretUnmarshaler(msh.Unmarshaler):
     def unmarshal(self, ctx: msh.UnmarshalContext, v: msh.Value) -> ta.Any:
         if isinstance(v, str):
             return v
@@ -25,17 +32,39 @@ class StrOrSecretUnmarshaler(msh.Unmarshaler):
             [(mk, mv)] = v.items()
             if mk != 'secret':
                 raise TypeError(v)
-            return Secret(check.isinstance(mv, str))
+            return SecretRef(check.isinstance(mv, str))
         else:
             raise TypeError(v)
 
 
 @dc.field_modifier
 def marshal_secret_field(f: dc.Field) -> dc.Field:
+    """Mostly obsolete with auto-registration below."""
+
     return dc.update_field_metadata(f, {
         msh.FieldMetadata: dc.replace(
             f.metadata.get(msh.FieldMetadata, msh.FieldMetadata()),
-            marshaler=StrOrSecretMarshaler(),
-            unmarshaler=StrOrSecretUnmarshaler(),
+            marshaler=StrOrSecretRefMarshalerUnmarshaler(),
+            unmarshaler=StrOrSecretRefMarshalerUnmarshaler(),
         ),
     })
+
+
+@lang.cached_function
+def _install_standard_marshalling() -> None:
+    msh.STANDARD_MARSHALER_FACTORIES[0:0] = [
+        msh.ForbiddenTypeMarshalerFactory({Secret}),
+        msh.TypeMapMarshalerFactory({
+            rfl.type_(SecretRefOrStr): StrOrSecretRefMarshalerUnmarshaler(),
+        }),
+    ]
+
+    msh.STANDARD_UNMARSHALER_FACTORIES[0:0] = [
+        msh.ForbiddenTypeUnmarshalerFactory({Secret}),
+        msh.TypeMapUnmarshalerFactory({
+            rfl.type_(SecretRefOrStr): StrOrSecretRefMarshalerUnmarshaler(),
+        }),
+    ]
+
+
+_install_standard_marshalling()