omlish 0.0.0.dev6__py3-none-any.whl → 0.0.0.dev7__py3-none-any.whl

omlish/reflect/types.py

@@ -0,0 +1,275 @@
+"""
+TODO:
+ - visitor / transformer
+ - uniform collection isinstance - items() for mappings, iter() for other
+ - also check instance type in isinstance not just items lol
+ - ta.Generic in mro causing trouble - omit? no longer 1:1
+ - cache this shit, esp generic_mro shit
+  - cache __hash__ in Generic/Union
+"""
+import dataclasses as dc
+import types
+import typing as ta
+
+
+_NoneType = types.NoneType  # type: ignore
+
+_NONE_TYPE_FROZENSET: frozenset['Type'] = frozenset([_NoneType])
+
+
+_GenericAlias = ta._GenericAlias  # type: ignore  # noqa
+_CallableGenericAlias = ta._CallableGenericAlias  # type: ignore  # noqa
+_SpecialGenericAlias = ta._SpecialGenericAlias  # type: ignore  # noqa
+_UnionGenericAlias = ta._UnionGenericAlias   # type: ignore  # noqa
+_AnnotatedAlias = ta._AnnotatedAlias  # type: ignore  # noqa
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class _Special:
+    name: str
+    alias: _SpecialGenericAlias  # type: ignore
+    origin: type
+    nparams: int
+
+
+_KNOWN_SPECIALS = [
+    _Special(
+        v._name,  # noqa
+        v,
+        v.__origin__,
+        v._nparams,  # noqa
+    )
+    for v in ta.__dict__.values()
+    if isinstance(v, _SpecialGenericAlias)
+]
+
+_KNOWN_SPECIALS_BY_NAME = {s.name: s for s in _KNOWN_SPECIALS}
+_KNOWN_SPECIALS_BY_ALIAS = {s.alias: s for s in _KNOWN_SPECIALS}
+_KNOWN_SPECIALS_BY_ORIGIN = {s.origin: s for s in _KNOWN_SPECIALS}
+
+_MAX_KNOWN_SPECIAL_TYPE_VARS = 16
+
+_KNOWN_SPECIAL_TYPE_VARS = tuple(
+    ta.TypeVar(f'_{i}')  # noqa
+    for i in range(_MAX_KNOWN_SPECIAL_TYPE_VARS)
+)
+
+
+##
+
+
+def get_params(obj: ta.Any) -> tuple[ta.TypeVar, ...]:
+    if isinstance(obj, type):
+        if issubclass(obj, ta.Generic):  # type: ignore
+            return obj.__dict__.get('__parameters__', ())  # noqa
+
+        if (ks := _KNOWN_SPECIALS_BY_ORIGIN.get(obj)) is not None:
+            return _KNOWN_SPECIAL_TYPE_VARS[:ks.nparams]
+
+    oty = type(obj)
+
+    if (
+            oty is _GenericAlias or
+            oty is ta.GenericAlias  # type: ignore  # noqa
+    ):
+        return obj.__dict__.get('__parameters__', ())  # noqa
+
+    if oty is _CallableGenericAlias:
+        raise NotImplementedError('get_params not yet implemented for typing.Callable')
+
+    raise TypeError(obj)
+
+
+def is_union_type(cls: ta.Any) -> bool:
+    if hasattr(ta, 'UnionType'):
+        return ta.get_origin(cls) in {ta.Union, getattr(ta, 'UnionType')}
+
+    else:
+        return ta.get_origin(cls) in {ta.Union}
+
+
+def get_orig_class(obj: ta.Any) -> ta.Any:
+    return obj.__orig_class__  # noqa
+
+
+##
+
+
+Type: ta.TypeAlias = ta.Union[
+    type,
+    ta.TypeVar,
+    'Union',
+    'Generic',
+    'NewType',
+    'Annotated',
+    'Any',
+]
+
+
+@dc.dataclass(frozen=True)
+class Union:
+    args: frozenset[Type]
+
+    @property
+    def is_optional(self) -> bool:
+        return _NoneType in self.args
+
+    def without_none(self) -> Type:
+        if _NoneType not in self.args:
+            return self
+        rem = self.args - _NONE_TYPE_FROZENSET
+        if len(rem) == 1:
+            return next(iter(rem))
+        return Union(rem)
+
+
+@dc.dataclass(frozen=True)
+class Generic:
+    cls: type
+    args: tuple[Type, ...]                                                # map[int, V] = (int, V) | map[T, T] = (T, T)
+
+    params: tuple[ta.TypeVar, ...] = dc.field(compare=False, repr=False)  # map[int, V] = (_0, _1) | map[T, T] = (_0, _1)  # noqa
+    # params2: tuple[ta.TypeVar, ...]                                     # map[int, V] = (V,)     | map[T, T] = (T,)
+
+    obj: ta.Any = dc.field(compare=False, repr=False)
+
+    # def __post_init__(self) -> None:
+    #     if not isinstance(self.cls, type):
+    #         raise TypeError(self.cls)
+
+    def full_eq(self, other: 'Generic') -> bool:
+        return (
+            self.cls == other.cls and
+            self.args == other.args and
+            self.params == other.params and
+            self.obj == other.obj
+        )
+
+
+@dc.dataclass(frozen=True)
+class NewType:
+    obj: ta.Any
+
+
+@dc.dataclass(frozen=True)
+class Annotated:
+    ty: Type
+    md: ta.Sequence[ta.Any]
+
+    obj: ta.Any = dc.field(compare=False, repr=False)
+
+
+class Any:
+    pass
+
+
+ANY = Any()
+
+
+TYPES: tuple[type, ...] = (
+    type,
+    ta.TypeVar,
+    Union,
+    Generic,
+    NewType,
+    Annotated,
+    Any,
+)
+
+
+##
+
+
+def is_type(obj: ta.Any) -> bool:
+    if isinstance(obj, (Union, Generic, ta.TypeVar, NewType, Any)):  # noqa
+        return True
+
+    oty = type(obj)
+
+    return (
+            oty is _UnionGenericAlias or oty is types.UnionType or  # noqa
+
+            isinstance(obj, ta.NewType) or  # noqa
+
+            (
+                oty is _GenericAlias or
+                oty is ta.GenericAlias or  # type: ignore  # noqa
+                oty is _CallableGenericAlias
+            ) or
+
+            isinstance(obj, type) or
+
+            isinstance(obj, _SpecialGenericAlias)
+    )
+
+
+def type_(obj: ta.Any) -> Type:
+    if obj is ta.Any:
+        return ANY
+
+    if isinstance(obj, (Union, Generic, ta.TypeVar, NewType, Any)):  # noqa
+        return obj
+
+    oty = type(obj)
+
+    if oty is _UnionGenericAlias or oty is types.UnionType:
+        return Union(frozenset(type_(a) for a in ta.get_args(obj)))
+
+    if isinstance(obj, ta.NewType):  # noqa
+        return NewType(obj)
+
+    if (
+            oty is _GenericAlias or
+            oty is ta.GenericAlias or  # type: ignore  # noqa
+            oty is _CallableGenericAlias
+    ):
+        origin = ta.get_origin(obj)
+        args = ta.get_args(obj)
+        if oty is _CallableGenericAlias:
+            p, r = args
+            if p is Ellipsis or isinstance(p, ta.ParamSpec):
+                raise TypeError(f'Callable argument not yet supported for {obj=}')
+            args = (*p, r)
+            params = _KNOWN_SPECIAL_TYPE_VARS[:len(args)]
+        elif origin is ta.Generic:
+            params = args
+        else:
+            params = get_params(origin)
+        if len(args) != len(params):
+            raise TypeError(f'Mismatched {args=} and {params=} for {obj=}')
+        return Generic(
+            origin,
+            tuple(type_(a) for a in args),
+            params,
+            obj,
+        )
+
+    if isinstance(obj, type):
+        if issubclass(obj, ta.Generic):  # type: ignore
+            params = get_params(obj)
+            return Generic(
+                obj,
+                params,
+                params,
+                obj,
+            )
+        return obj
+
+    if isinstance(obj, _SpecialGenericAlias):
+        if (ks := _KNOWN_SPECIALS_BY_ALIAS.get(obj)) is not None:
+            params = _KNOWN_SPECIAL_TYPE_VARS[:ks.nparams]
+            return Generic(
+                ks.origin,
+                params,
+                params,
+                obj,
+            )
+
+    if isinstance(obj, _AnnotatedAlias):
+        o = ta.get_args(obj)[0]
+        return Annotated(type_(o), md=obj.__metadata__, obj=obj)
+
+    raise TypeError(obj)

omlish/secrets/__init__.py

@@ -1,7 +1,23 @@
 from .secrets import (  # noqa
+    CachingSecrets,
+    CompositeSecrets,
     EMPTY_SECRETS,
     EmptySecrets,
-    Secret,
+    EnvVarSecrets,
+    FnSecrets,
+    LoggingSecrets,
+    MappingSecrets,
+    SecretRef,
+    SecretRefOrStr,
     Secrets,
-    SimpleSecrets,
+    secret_field,
+    secret_repr,
 )
+
+
+##
+
+
+from ..lang.imports import _register_conditional_import  # noqa
+
+_register_conditional_import('..marshal', '.marshal', __package__)

omlish/secrets/crypto.py

@@ -0,0 +1,132 @@
+"""
+TODO:
+ - cryptography vs pycryptodome[x]
+ - standardize failure exception
+ - chains - take first
+ - keysets
+
+See:
+ - https://soatok.blog/2020/05/13/why-aes-gcm-sucks/
+ - https://pycryptodome.readthedocs.io/en/latest/src/cipher/chacha20_poly1305.html
+
+==
+
+https://www.tecmint.com/gpg-encrypt-decrypt-files/
+
+==
+
+gpg --batch --passphrase '' --quick-gen-key wrmsr default default
+gpg --batch --passphrase '' --quick-gen-key wrmsr2 default default
+echo 'hi there' > secret.txt
+gpg -e -u wrmsr -r wrmsr2 secret.txt
+gpg -d -o secret2.txt secret.txt.gpg
+
+gpg --batch -c --passphrase-file /var/secret.key -o some.gpg toencrypt.txt
+
+openssl rand -rand /dev/urandom 128 > barf.key
+openssl enc -in secret.txt -out secret.txt.enc -e -aes256 -pbkdf2 -kfile barf.key
+openssl aes-256-cbc -d -pbkdf2 -in secret.txt.enc -out secret3.txt -kfile barf.key
+
+https://wiki.openssl.org/index.php/Enc#Options
+-pass 'file:...'
+"""
+import abc
+import secrets
+import typing as ta
+
+from .. import lang
+
+
+if ta.TYPE_CHECKING:
+    from cryptography import exceptions as cry_exc
+    from cryptography import fernet as cry_fernet
+    from cryptography.hazmat.primitives.ciphers import aead as cry_aead
+else:
+    cry_aead = lang.proxy_import('cryptography.hazmat.primitives.ciphers.aead')
+    cry_exc = lang.proxy_import('cryptography.exceptions')
+    cry_fernet = lang.proxy_import('cryptography.fernet')
+
+
+##
+
+
+class EncryptionError(Exception):
+    pass
+
+
+class DecryptionError(Exception):
+    pass
+
+
+class Crypto(abc.ABC):
+    @abc.abstractmethod
+    def generate_key(self) -> bytes:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def encrypt(self, data: bytes, key: bytes) -> bytes:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def decrypt(self, data: bytes, key: bytes) -> bytes:
+        raise NotImplementedError
+
+
+##
+
+
+class FernetCrypto(Crypto):
+
+    def generate_key(self) -> bytes:
+        return cry_fernet.Fernet.generate_key()
+
+    def encrypt(self, data: bytes, key: bytes) -> bytes:
+        try:
+            f = cry_fernet.Fernet(key)
+        except ValueError as e:
+            raise EncryptionError from e
+        return f.encrypt(data)
+
+    def decrypt(self, data: bytes, key: bytes) -> bytes:
+        try:
+            f = cry_fernet.Fernet(key)
+        except ValueError as e:
+            raise DecryptionError from e
+        try:
+            return f.decrypt(data)
+        except cry_fernet.InvalidToken as e:
+            raise DecryptionError from e
+
+
+class AesgsmCrypto(Crypto):
+    """https://stackoverflow.com/a/59835994"""
+
+    def generate_key(self) -> bytes:
+        return secrets.token_bytes(32)
+
+    def encrypt(self, data: bytes, key: bytes) -> bytes:
+        nonce = secrets.token_bytes(12)
+        return nonce + cry_aead.AESGCM(key).encrypt(nonce, data, b'')
+
+    def decrypt(self, data: bytes, key: bytes) -> bytes:
+        try:
+            return cry_aead.AESGCM(key).decrypt(data[:12], data[12:], b'')
+        except cry_exc.InvalidTag as e:
+            raise DecryptionError from e
+
+
+class Chacha20Poly1305Crypto(Crypto):
+    """https://cryptography.io/en/latest/hazmat/primitives/aead/#cryptography.hazmat.primitives.ciphers.aead.ChaCha20Poly1305"""  # noqa
+
+    def generate_key(self) -> bytes:
+        return cry_aead.ChaCha20Poly1305.generate_key()
+
+    def encrypt(self, data: bytes, key: bytes) -> bytes:
+        nonce = secrets.token_bytes(12)
+        return nonce + cry_aead.ChaCha20Poly1305(key).encrypt(nonce, data, b'')
+
+    def decrypt(self, data: bytes, key: bytes) -> bytes:
+        try:
+            return cry_aead.ChaCha20Poly1305(key).decrypt(data[:12], data[12:], b'')
+        except cry_exc.InvalidTag as e:
+            raise DecryptionError from e

omlish/secrets/marshal.py

@@ -1,23 +1,30 @@
+"""
+TODO:
+ - ensure import order or at least warn or smth lol
+ - raise exception on ambiguous 'registered' impls
+"""
 import collections.abc
 import typing as ta
 
 from .. import check
 from .. import dataclasses as dc
+from .. import lang
 from .. import marshal as msh
+from .. import reflect as rfl
 from .secrets import Secret
+from .secrets import SecretRef
+from .secrets import SecretRefOrStr
 
 
-class StrOrSecretMarshaler(msh.Marshaler):
+class StrOrSecretRefMarshalerUnmarshaler(msh.Marshaler, msh.Unmarshaler):
     def marshal(self, ctx: msh.MarshalContext, o: ta.Any) -> msh.Value:
         if isinstance(o, str):
             return o
-        elif isinstance(o, Secret):
+        elif isinstance(o, SecretRef):
             return {'secret': o.key}
         else:
             raise TypeError(o)
 
-
-class StrOrSecretUnmarshaler(msh.Unmarshaler):
     def unmarshal(self, ctx: msh.UnmarshalContext, v: msh.Value) -> ta.Any:
         if isinstance(v, str):
             return v
@@ -25,17 +32,39 @@ class StrOrSecretUnmarshaler(msh.Unmarshaler):
             [(mk, mv)] = v.items()
             if mk != 'secret':
                 raise TypeError(v)
-            return Secret(check.isinstance(mv, str))
+            return SecretRef(check.isinstance(mv, str))
         else:
             raise TypeError(v)
 
 
 @dc.field_modifier
 def marshal_secret_field(f: dc.Field) -> dc.Field:
+    """Mostly obsolete with auto-registration below."""
+
     return dc.update_field_metadata(f, {
         msh.FieldMetadata: dc.replace(
             f.metadata.get(msh.FieldMetadata, msh.FieldMetadata()),
-            marshaler=StrOrSecretMarshaler(),
-            unmarshaler=StrOrSecretUnmarshaler(),
+            marshaler=StrOrSecretRefMarshalerUnmarshaler(),
+            unmarshaler=StrOrSecretRefMarshalerUnmarshaler(),
         ),
     })
+
+
+@lang.cached_function
+def _install_standard_marshalling() -> None:
+    msh.STANDARD_MARSHALER_FACTORIES[0:0] = [
+        msh.ForbiddenTypeMarshalerFactory({Secret}),
+        msh.TypeMapMarshalerFactory({
+            rfl.type_(SecretRefOrStr): StrOrSecretRefMarshalerUnmarshaler(),
+        }),
+    ]
+
+    msh.STANDARD_UNMARSHALER_FACTORIES[0:0] = [
+        msh.ForbiddenTypeUnmarshalerFactory({Secret}),
+        msh.TypeMapUnmarshalerFactory({
+            rfl.type_(SecretRefOrStr): StrOrSecretRefMarshalerUnmarshaler(),
+        }),
+    ]
+
+
+_install_standard_marshalling()

omlish/secrets/openssl.py

@@ -0,0 +1,207 @@
+"""
+TODO:
+ - LibreSSL supports aes-256-gcm: https://crypto.stackexchange.com/a/76178
+"""
+import hashlib
+import secrets
+import subprocess
+import typing as ta
+
+from .. import lang
+from .crypto import Crypto
+from .crypto import DecryptionError
+from .crypto import EncryptionError
+from .subprocesses import SubprocessFileInputMethod
+from .subprocesses import pipe_fd_subprocess_file_input  # noqa
+from .subprocesses import temp_subprocess_file_input  # noqa
+
+
+if ta.TYPE_CHECKING:
+    from cryptography.hazmat.primitives import ciphers as cry_ciphs
+    from cryptography.hazmat.primitives.ciphers import algorithms as cry_algs
+    from cryptography.hazmat.primitives.ciphers import modes as cry_modes
+
+else:
+    cry_ciphs = lang.proxy_import('cryptography.hazmat.primitives.ciphers')
+    cry_algs = lang.proxy_import('cryptography.hazmat.primitives.ciphers.algorithms')
+    cry_modes = lang.proxy_import('cryptography.hazmat.primitives.ciphers.modes')
+
+
+##
+
+
+DEFAULT_KEY_SIZE = 64
+
+
+def generate_key(self, sz: int = DEFAULT_KEY_SIZE) -> bytes:
+    # !! https://docs.openssl.org/3.0/man7/passphrase-encoding/
+    # Must not contain null bytes!
+    return secrets.token_hex(sz).encode('ascii')
+
+
+##
+
+
+class OpensslAes265CbcCrypto(Crypto):
+    """
+    !!! https://docs.openssl.org/3.0/man7/passphrase-encoding/
+    https://cryptography.io/en/latest/hazmat/primitives/symmetric-encryption/#cryptography.hazmat.primitives.ciphers.Cipher
+    https://stackoverflow.com/questions/16761458/how-to-decrypt-openssl-aes-encrypted-files-in-python
+    https://github.com/openssl/openssl/blob/3c1713aeed4dc7d1ac25e9e365b8bd98afead638/apps/enc.c#L555-L573
+    """
+
+    def __init__(
+            self,
+            *,
+            iters: int = 10_000,
+    ) -> None:
+        super().__init__()
+        self._iters = iters
+
+    def generate_key(self, sz: int = DEFAULT_KEY_SIZE) -> bytes:
+        # This actually can handle null bytes, but we don't generate keys with them for compatibility.
+        return generate_key(sz)
+
+    block_size = 16
+    key_length = 32
+    iv_length = 16
+    salt_length = 8
+    prefix = b'Salted__'
+
+    def _make_cipher(self, key: bytes, salt: bytes) -> 'cry_ciphs.Cipher':
+        dk = hashlib.pbkdf2_hmac(
+            'sha256',
+            key,
+            salt,
+            self._iters,
+            self.key_length + self.iv_length,
+        )
+
+        return cry_ciphs.Cipher(
+            cry_algs.AES256(dk[:self.key_length]),
+            cry_modes.CBC(dk[self.key_length:]),
+        )
+
+    def encrypt(self, data: bytes, key: bytes, *, salt: bytes | None = None) -> bytes:
+        if salt is None:
+            salt = secrets.token_bytes(self.salt_length)
+        elif len(salt) != self.salt_length:
+            raise EncryptionError('bad salt length')
+
+        last_byte = self.block_size - (len(data) % self.block_size)
+        raw = data + bytes([last_byte] * last_byte)
+
+        cipher = self._make_cipher(key, salt)
+
+        encryptor = cipher.encryptor()
+        enc = encryptor.update(raw) + encryptor.finalize()
+
+        return b''.join([
+            self.prefix,
+            salt,
+            enc,
+        ])
+
+    def decrypt(self, data: bytes, key: bytes) -> bytes:
+        if not data.startswith(self.prefix):
+            raise DecryptionError('bad prefix')
+
+        salt = data[len(self.prefix):self.block_size]
+        cipher = self._make_cipher(key, salt)
+
+        decryptor = cipher.decryptor()
+        dec = decryptor.update(data[self.block_size:]) + decryptor.finalize()
+
+        last_byte = dec[-1]
+        if last_byte > self.block_size:
+            raise DecryptionError('bad padding')
+
+        return dec[:-last_byte]
+
+
+##
+
+
+class OpensslSubprocessAes256CbcCrypto(Crypto):
+    def __init__(
+            self,
+            *,
+            cmd: ta.Sequence[str] = ('openssl',),
+            timeout: float = 5.,
+            file_input: SubprocessFileInputMethod = temp_subprocess_file_input,
+    ) -> None:
+        super().__init__()
+        self._cmd = cmd
+        self._timeout = timeout
+        self._file_input = file_input
+
+    def generate_key(self, sz: int = DEFAULT_KEY_SIZE) -> bytes:
+        return generate_key(sz)
+
+    def encrypt(self, data: bytes, key: bytes) -> bytes:
+        # !! https://docs.openssl.org/3.0/man7/passphrase-encoding/
+        # Must not contain null bytes!
+        if 0 in key:
+            raise Exception('invalid key')
+        with self._file_input(key) as fi:
+            proc = subprocess.Popen(
+                [
+                    *self._cmd,
+                    'aes-256-cbc',
+
+                    '-e',
+
+                    '-pbkdf2',
+                    '-salt',
+                    '-iter', '10000',
+
+                    '-in', '-',
+                    '-out', '-',
+                    '-kfile', fi.file_path,
+                ],
+                stdin=subprocess.PIPE,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                pass_fds=[*fi.pass_fds],
+            )
+            out, err = proc.communicate(
+                data,
+                timeout=self._timeout,
+            )
+            if proc.returncode != 0:
+                raise EncryptionError
+            return out
+
+    def decrypt(self, data: bytes, key: bytes) -> bytes:
+        # !! https://docs.openssl.org/3.0/man7/passphrase-encoding/
+        # Must not contain null bytes!
+        if 0 in key:
+            raise Exception('invalid key')
+        with self._file_input(key) as fi:
+            proc = subprocess.Popen(
+                [
+                    *self._cmd,
+                    'aes-256-cbc',
+
+                    '-d',
+
+                    '-pbkdf2',
+                    '-salt',
+                    '-iter', '10000',
+
+                    '-in', '-',
+                    '-out', '-',
+                    '-kfile', fi.file_path,
+                ],
+                stdin=subprocess.PIPE,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                pass_fds=[*fi.pass_fds],
+            )
+            out, err = proc.communicate(
+                data,
+                timeout=self._timeout,
+            )
+            if proc.returncode != 0:
+                raise DecryptionError
+            return out