omlish 0.0.0.dev6__py3-none-any.whl → 0.0.0.dev8__py3-none-any.whl

omlish/secrets/openssl.py

@@ -0,0 +1,207 @@
+"""
+TODO:
+ - LibreSSL supports aes-256-gcm: https://crypto.stackexchange.com/a/76178
+"""
+import hashlib
+import secrets
+import subprocess
+import typing as ta
+
+from .. import lang
+from .crypto import Crypto
+from .crypto import DecryptionError
+from .crypto import EncryptionError
+from .subprocesses import SubprocessFileInputMethod
+from .subprocesses import pipe_fd_subprocess_file_input  # noqa
+from .subprocesses import temp_subprocess_file_input  # noqa
+
+
+if ta.TYPE_CHECKING:
+    from cryptography.hazmat.primitives import ciphers as cry_ciphs
+    from cryptography.hazmat.primitives.ciphers import algorithms as cry_algs
+    from cryptography.hazmat.primitives.ciphers import modes as cry_modes
+
+else:
+    cry_ciphs = lang.proxy_import('cryptography.hazmat.primitives.ciphers')
+    cry_algs = lang.proxy_import('cryptography.hazmat.primitives.ciphers.algorithms')
+    cry_modes = lang.proxy_import('cryptography.hazmat.primitives.ciphers.modes')
+
+
+##
+
+
+DEFAULT_KEY_SIZE = 64
+
+
+def generate_key(self, sz: int = DEFAULT_KEY_SIZE) -> bytes:
+    # !! https://docs.openssl.org/3.0/man7/passphrase-encoding/
+    # Must not contain null bytes!
+    return secrets.token_hex(sz).encode('ascii')
+
+
+##
+
+
+class OpensslAes265CbcCrypto(Crypto):
+    """
+    !!! https://docs.openssl.org/3.0/man7/passphrase-encoding/
+    https://cryptography.io/en/latest/hazmat/primitives/symmetric-encryption/#cryptography.hazmat.primitives.ciphers.Cipher
+    https://stackoverflow.com/questions/16761458/how-to-decrypt-openssl-aes-encrypted-files-in-python
+    https://github.com/openssl/openssl/blob/3c1713aeed4dc7d1ac25e9e365b8bd98afead638/apps/enc.c#L555-L573
+    """
+
+    def __init__(
+            self,
+            *,
+            iters: int = 10_000,
+    ) -> None:
+        super().__init__()
+        self._iters = iters
+
+    def generate_key(self, sz: int = DEFAULT_KEY_SIZE) -> bytes:
+        # This actually can handle null bytes, but we don't generate keys with them for compatibility.
+        return generate_key(sz)
+
+    block_size = 16
+    key_length = 32
+    iv_length = 16
+    salt_length = 8
+    prefix = b'Salted__'
+
+    def _make_cipher(self, key: bytes, salt: bytes) -> 'cry_ciphs.Cipher':
+        dk = hashlib.pbkdf2_hmac(
+            'sha256',
+            key,
+            salt,
+            self._iters,
+            self.key_length + self.iv_length,
+        )
+
+        return cry_ciphs.Cipher(
+            cry_algs.AES256(dk[:self.key_length]),
+            cry_modes.CBC(dk[self.key_length:]),
+        )
+
+    def encrypt(self, data: bytes, key: bytes, *, salt: bytes | None = None) -> bytes:
+        if salt is None:
+            salt = secrets.token_bytes(self.salt_length)
+        elif len(salt) != self.salt_length:
+            raise EncryptionError('bad salt length')
+
+        last_byte = self.block_size - (len(data) % self.block_size)
+        raw = data + bytes([last_byte] * last_byte)
+
+        cipher = self._make_cipher(key, salt)
+
+        encryptor = cipher.encryptor()
+        enc = encryptor.update(raw) + encryptor.finalize()
+
+        return b''.join([
+            self.prefix,
+            salt,
+            enc,
+        ])
+
+    def decrypt(self, data: bytes, key: bytes) -> bytes:
+        if not data.startswith(self.prefix):
+            raise DecryptionError('bad prefix')
+
+        salt = data[len(self.prefix):self.block_size]
+        cipher = self._make_cipher(key, salt)
+
+        decryptor = cipher.decryptor()
+        dec = decryptor.update(data[self.block_size:]) + decryptor.finalize()
+
+        last_byte = dec[-1]
+        if last_byte > self.block_size:
+            raise DecryptionError('bad padding')
+
+        return dec[:-last_byte]
+
+
+##
+
+
+class OpensslSubprocessAes256CbcCrypto(Crypto):
+    def __init__(
+            self,
+            *,
+            cmd: ta.Sequence[str] = ('openssl',),
+            timeout: float = 5.,
+            file_input: SubprocessFileInputMethod = temp_subprocess_file_input,
+    ) -> None:
+        super().__init__()
+        self._cmd = cmd
+        self._timeout = timeout
+        self._file_input = file_input
+
+    def generate_key(self, sz: int = DEFAULT_KEY_SIZE) -> bytes:
+        return generate_key(sz)
+
+    def encrypt(self, data: bytes, key: bytes) -> bytes:
+        # !! https://docs.openssl.org/3.0/man7/passphrase-encoding/
+        # Must not contain null bytes!
+        if 0 in key:
+            raise Exception('invalid key')
+        with self._file_input(key) as fi:
+            proc = subprocess.Popen(
+                [
+                    *self._cmd,
+                    'aes-256-cbc',
+
+                    '-e',
+
+                    '-pbkdf2',
+                    '-salt',
+                    '-iter', '10000',
+
+                    '-in', '-',
+                    '-out', '-',
+                    '-kfile', fi.file_path,
+                ],
+                stdin=subprocess.PIPE,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                pass_fds=[*fi.pass_fds],
+            )
+            out, err = proc.communicate(
+                data,
+                timeout=self._timeout,
+            )
+            if proc.returncode != 0:
+                raise EncryptionError
+            return out
+
+    def decrypt(self, data: bytes, key: bytes) -> bytes:
+        # !! https://docs.openssl.org/3.0/man7/passphrase-encoding/
+        # Must not contain null bytes!
+        if 0 in key:
+            raise Exception('invalid key')
+        with self._file_input(key) as fi:
+            proc = subprocess.Popen(
+                [
+                    *self._cmd,
+                    'aes-256-cbc',
+
+                    '-d',
+
+                    '-pbkdf2',
+                    '-salt',
+                    '-iter', '10000',
+
+                    '-in', '-',
+                    '-out', '-',
+                    '-kfile', fi.file_path,
+                ],
+                stdin=subprocess.PIPE,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                pass_fds=[*fi.pass_fds],
+            )
+            out, err = proc.communicate(
+                data,
+                timeout=self._timeout,
+            )
+            if proc.returncode != 0:
+                raise DecryptionError
+            return out

omlish/secrets/secrets.py

@@ -1,47 +1,299 @@
+"""
+TODO:
+ - SqlFunctionSecrets (in .sql?)
+ - crypto is just Transformed, bound with a key
+ - crypto key in env + values in file?
+ - Secret:
+  - hold ref to Secret, and key
+  - time of retrieval
+  - logs accesses
+ - types? ssh / url / pw / basicauthtoken / tls / str
+"""
 import abc
+import collections
+import logging
+import os
+import sys
+import time
+import types  # noqa
 import typing as ta
 
 from .. import dataclasses as dc
 from .. import lang
 
 
+log = logging.getLogger(__name__)
+
+
+##
+
+
+class Secret(lang.NotPicklable, lang.Final):
+    _VALUE_ATTR = '__secret_value__'
+
+    def __init__(self, *, key: str | None, value: str) -> None:
+        super().__init__()
+        self._key = key
+        setattr(self, self._VALUE_ATTR, lambda: value)
+
+    def __repr__(self) -> str:
+        return f'Secret<{self._key or ""}>'
+
+    def __str__(self) -> ta.NoReturn:
+        raise TypeError
+
+    def reveal(self) -> str:
+        return getattr(self, self._VALUE_ATTR)()
+
+
 ##
 
 
 @dc.dataclass(frozen=True)
-class Secret:
+class SecretRef:
     key: str
 
 
+SecretRefOrStr: ta.TypeAlias = SecretRef | str
+
+
+def secret_repr(o: SecretRefOrStr | None) -> str | None:
+    if isinstance(o, str):
+        return '...'
+    elif isinstance(o, SecretRef):
+        return repr(o)
+    elif o is None:
+        return None
+    else:
+        raise TypeError(o)
+
+
+@dc.field_modifier
+def secret_field(f: dc.Field) -> dc.Field:
+    return dc.update_field_extras(
+        f,
+        repr_fn=secret_repr,
+        unless_non_default=True,
+    )
+
+
 ##
 
 
 class Secrets(lang.Abstract):
-    def fix(self, obj: str | Secret) -> str:
-        if isinstance(obj, str):
+    def fix(self, obj: str | SecretRef | Secret) -> Secret:
+        if isinstance(obj, Secret):
             return obj
-        elif isinstance(obj, Secret):
+        elif isinstance(obj, str):
+            return Secret(key=None, value=obj)
+        elif isinstance(obj, SecretRef):
             return self.get(obj.key)
         else:
             raise TypeError(obj)
 
+    def get(self, key: str) -> Secret:
+        try:
+            raw = self._get_raw(key)  # noqa
+        except KeyError:  # noqa
+            raise
+        else:
+            return Secret(key=key, value=raw)
+
     @abc.abstractmethod
-    def get(self, key: str) -> str:
+    def _get_raw(self, key: str) -> str:
         raise NotImplementedError
 
 
+##
+
+
 class EmptySecrets(Secrets):
-    def get(self, key: str) -> str:
+    def _get_raw(self, key: str) -> str:
         raise KeyError(key)
 
 
 EMPTY_SECRETS = EmptySecrets()
 
 
-class SimpleSecrets(Secrets):
+##
+
+
+class MappingSecrets(Secrets):
     def __init__(self, dct: ta.Mapping[str, str]) -> None:
         super().__init__()
         self._dct = dct
 
-    def get(self, key: str) -> str:
+    def __repr__(self) -> str:
+        return f'{self.__class__.__name__}({{{", ".join(map(repr, self._dct.keys()))}}})'
+
+    def _get_raw(self, key: str) -> str:
         return self._dct[key]
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class FnSecrets(Secrets):
+    fn: ta.Callable[[str], str]
+
+    def _get_raw(self, key: str) -> str:
+        return self.fn(key)
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class TransformedSecrets(Secrets):
+    fn: ta.Callable[[str], str]
+    child: Secrets
+
+    def _get_raw(self, key: str) -> str:
+        # FIXME: hm..
+        return self.fn(self.child._get_raw(key))  # noqa
+
+
+##
+
+
+class CachingSecrets(Secrets):
+    def __init__(
+            self,
+            child: Secrets,
+            *,
+            ttl_s: float | None = None,
+            clock: ta.Callable[..., float] = time.time,
+    ) -> None:
+        super().__init__()
+        self._child = child
+        self._dct: dict[str, str] = {}
+        self._ttl_s = ttl_s
+        self._clock = clock
+        self._deque: collections.deque[tuple[str, float]] = collections.deque()
+
+    def __repr__(self) -> str:
+        return f'{self.__class__.__name__}({{{", ".join(map(repr, self._dct.keys()))}}})'
+
+    def evict(self) -> None:
+        now = self._clock()
+        while self._deque:
+            k, dl = self._deque[0]
+            if now < dl:
+                break
+            del self._dct[k]
+            self._deque.popleft()
+
+    def _get_raw(self, key: str) -> str:
+        self.evict()
+        try:
+            return self._dct[key]
+        except KeyError:
+            pass
+        out = self._child._get_raw(key)  # noqa
+        self._dct[key] = out
+        if self._ttl_s is not None:
+            dl = self._clock() + self._ttl_s
+            self._deque.append((key, dl))
+        return out
+
+
+##
+
+
+class CompositeSecrets(Secrets):
+    def __init__(self, *children: Secrets) -> None:
+        super().__init__()
+        self._children = children
+
+    def _get_raw(self, key: str) -> str:
+        for c in self._children:
+            try:
+                return c._get_raw(key)  # noqa
+            except KeyError:
+                pass
+        raise KeyError(key)
+
+
+##
+
+
+class LoggingSecrets(Secrets):
+    def __init__(
+            self,
+            child: Secrets,
+            *,
+            log: logging.Logger | None = None,  # noqa
+    ) -> None:
+        super().__init__()
+        self._child = child
+        self._log = log if log is not None else globals()['log']
+
+    IGNORE_PACKAGES: ta.ClassVar[ta.AbstractSet[str]] = {
+        __package__,
+    }
+
+    def _get_caller_str(self, n: int = 3) -> str:
+        l: list[str] = []
+        f: types.FrameType | None = sys._getframe(2)  # noqa
+        while f is not None and len(l) < n:
+            try:
+                pkg = f.f_globals['__package__']
+            except KeyError:
+                pkg = None
+            else:
+                if pkg in self.IGNORE_PACKAGES:
+                    f = f.f_back
+                    continue
+            if (fn := f.f_code.co_filename):
+                l.append(f'{fn}:{f.f_lineno}')
+            else:
+                l.append(pkg)
+            f = f.f_back
+        return ', '.join(l)
+
+    def _get_raw(self, key: str) -> str:
+        cs = self._get_caller_str()
+        self._log.info('Attempting to access secret: %s, %s', key, cs)
+        try:
+            ret = self._child._get_raw(key)  # noqa
+        except KeyError:
+            self._log.info('Failed to access secret: %s, %s', key, cs)
+            raise
+        else:
+            self._log.info('Successfully accessed secret: %s, %s', key, cs)
+            return ret
+
+
+##
+
+
+class EnvVarSecrets(Secrets):
+    def __init__(
+            self,
+            *,
+            env: ta.MutableMapping[str, str] | None = None,
+            upcase: bool = False,
+            prefix: str | None = None,
+            pop: bool = False,
+    ) -> None:
+        super().__init__()
+        self._env = env
+        self._upcase = upcase
+        self._prefix = prefix
+        self._pop = pop
+
+    def _get_raw(self, key: str) -> str:
+        ekey = key
+        if self._upcase:
+            ekey = ekey.upper()
+        if self._prefix is not None:
+            ekey = self._prefix + ekey
+        if self._env is not None:
+            dct = self._env
+        else:
+            dct = os.environ
+        if self._pop:
+            return dct.pop(ekey)
+        else:
+            return dct[ekey]

omlish/secrets/subprocesses.py

@@ -0,0 +1,42 @@
+"""
+FIXME:
+ - macos pipe size lol, and just like checking at all
+"""
+import contextlib
+import fcntl
+import os
+import tempfile
+import typing as ta
+
+
+class SubprocessFileInput(ta.NamedTuple):
+    file_path: str
+    pass_fds: ta.Sequence[int]
+
+
+SubprocessFileInputMethod: ta.TypeAlias = ta.Callable[[bytes], ta.ContextManager[SubprocessFileInput]]
+
+
+@contextlib.contextmanager
+def temp_subprocess_file_input(buf: bytes) -> ta.Iterator[SubprocessFileInput]:
+    with tempfile.NamedTemporaryFile(delete=True) as kf:
+        kf.write(buf)
+        kf.flush()
+        yield SubprocessFileInput(kf.name, [])
+
+
+@contextlib.contextmanager
+def pipe_fd_subprocess_file_input(buf: bytes) -> ta.Iterator[SubprocessFileInput]:
+    rfd, wfd = os.pipe()
+    closed_wfd = False
+    try:
+        if hasattr(fcntl, 'F_SETPIPE_SZ'):
+            fcntl.fcntl(wfd, fcntl.F_SETPIPE_SZ, max(len(buf), 0x1000))
+        os.write(wfd, buf)
+        os.close(wfd)
+        closed_wfd = True
+        yield SubprocessFileInput(f'/dev/fd/{rfd}', [rfd])
+    finally:
+        if not closed_wfd:
+            os.close(wfd)
+        os.close(rfd)

omlish/sql/dbs.py

@@ -3,13 +3,14 @@ import urllib.parse
 
 from .. import dataclasses as dc
 from .. import lang
+from .. import secrets as sec
 
 
 ##
 
 
 @dc.dataclass(frozen=True, kw_only=True)
-class DbType:
+class DbType(lang.Final):
     name: str
     dialect_name: str
 
@@ -38,13 +39,13 @@ class DbTypes(lang.Namespace, lang.Final):
 ##
 
 
-class DbLoc(lang.Abstract):
+class DbLoc(lang.Abstract, lang.Sealed):
     pass
 
 
 @dc.dataclass(frozen=True)
 class UrlDbLoc(DbLoc, lang.Final):
-    url: str
+    url: sec.SecretRefOrStr = dc.xfield() | sec.secret_field
 
 
 @dc.dataclass(frozen=True)
@@ -53,14 +54,14 @@ class HostDbLoc(DbLoc, lang.Final):
     port: int | None = None
 
     username: str | None = None
-    password: str | None = dc.xfield(default=None, repr_fn=lambda pw: '...' if pw is not None else None)
+    password: sec.SecretRefOrStr | None = dc.xfield(default=None) | sec.secret_field
 
 
 ##
 
 
 @dc.dataclass(frozen=True)
-class DbSpec:
+class DbSpec(lang.Final):
     name: str
     type: DbType
     loc: DbLoc

omlish/sql/exprs.py

@@ -0,0 +1,12 @@
+import sqlalchemy as sa
+import sqlalchemy.ext.compiler
+
+
+class paren(sa.sql.expression.UnaryExpression):  # noqa
+    __visit_name__ = 'paren'
+    inherit_cache = True
+
+
+@sa.ext.compiler.compiles(paren)
+def _compile_paren(element, compiler, **kw):
+    return '(%s)' % (element.element._compiler_dispatch(compiler),)  # noqa

omlish/sql/secrets.py

@@ -0,0 +1,10 @@
+"""
+TODO:
+ - sync/async...
+"""
+from .. import secrets as sec
+
+
+class SqlFunctionSecrets(sec.Secrets):
+    def _get_raw(self, key: str) -> str:
+        raise NotImplementedError

omlish/term.py

@@ -20,7 +20,7 @@ def set_title(title: str) -> str:
 
 
 def strip_ansi_codes(s: str) -> str:
-    return re.sub(r'\033\\[([0-9]+)(;[0-9]+)*m', '', s)
+    return re.sub(r'\033\[([0-9]+)(;[0-9]+)*m', '', s)
 
 
 class ControlSequence: