omlish 0.0.0.dev5__py3-none-any.whl → 0.0.0.dev7__py3-none-any.whl

omlish/secrets/passwords.py

@@ -0,0 +1,120 @@
+"""
+~> https://github.com/pallets/werkzeug/blob/7a76170c473c26685bdfa2774d083ba2386fc60f/src/werkzeug/security.py
+"""
+# Copyright 2007 Pallets
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted provided that the
+# following conditions are met:
+#
+# 1.  Redistributions of source code must retain the above copyright notice, this list of conditions and the following
+#     disclaimer.
+#
+# 2.  Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the
+#     following disclaimer in the documentation and/or other materials provided with the distribution.
+#
+# 3.  Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote
+#     products derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+import hashlib
+import hmac
+import secrets
+
+
+SALT_CHARS = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
+DEFAULT_PBKDF2_ITERATIONS = 600000
+
+
+def gen_salt(length: int) -> str:
+    if length <= 0:
+        raise ValueError('Salt length must be at least 1.')
+
+    return ''.join(secrets.choice(SALT_CHARS) for _ in range(length))
+
+
+def _hash_internal(
+        method: str,
+        salt: str,
+        password: str,
+) -> tuple[str, str]:
+    method, *args = method.split(':')
+    salt_bytes = salt.encode()
+    password_bytes = password.encode()
+
+    if method == 'scrypt':
+        if not args:
+            n = 2 ** 15
+            r = 8
+            p = 1
+        else:
+            try:
+                n, r, p = map(int, args)
+            except ValueError:
+                raise ValueError("'scrypt' takes 3 arguments.") from None
+
+        maxmem = 132 * n * r * p  # ideally 128, but some extra seems needed
+
+        return (
+            hashlib.scrypt(
+                password_bytes,
+                salt=salt_bytes,
+                n=n,
+                r=r,
+                p=p,
+                maxmem=maxmem,
+            ).hex(),
+            f'scrypt:{n}:{r}:{p}',
+        )
+
+    elif method == 'pbkdf2':
+        len_args = len(args)
+
+        if len_args == 0:
+            hash_name = 'sha256'
+            iterations = DEFAULT_PBKDF2_ITERATIONS
+        elif len_args == 1:
+            hash_name = args[0]
+            iterations = DEFAULT_PBKDF2_ITERATIONS
+        elif len_args == 2:
+            hash_name = args[0]
+            iterations = int(args[1])
+        else:
+            raise ValueError("'pbkdf2' takes 2 arguments.")
+
+        return (
+            hashlib.pbkdf2_hmac(
+                hash_name,
+                password_bytes,
+                salt_bytes,
+                iterations,
+            ).hex(),
+            f'pbkdf2:{hash_name}:{iterations}',
+        )
+
+    else:
+        raise ValueError(f"Invalid hash method '{method}'.")
+
+
+def generate_password_hash(
+        password: str,
+        method: str = 'scrypt',
+        salt_length: int = 16,
+) -> str:
+    salt = gen_salt(salt_length)
+    h, actual_method = _hash_internal(method, salt, password)
+    return f'{actual_method}${salt}${h}'
+
+
+def check_password_hash(pwhash: str, password: str) -> bool:
+    try:
+        method, salt, hashval = pwhash.split('$', 2)
+    except ValueError:
+        return False
+
+    return hmac.compare_digest(_hash_internal(method, salt, password)[0], hashval)

omlish/secrets/secrets.py

@@ -0,0 +1,299 @@
+"""
+TODO:
+ - SqlFunctionSecrets (in .sql?)
+ - crypto is just Transformed, bound with a key
+ - crypto key in env + values in file?
+ - Secret:
+  - hold ref to Secret, and key
+  - time of retrieval
+  - logs accesses
+ - types? ssh / url / pw / basicauthtoken / tls / str
+"""
+import abc
+import collections
+import logging
+import os
+import sys
+import time
+import types  # noqa
+import typing as ta
+
+from .. import dataclasses as dc
+from .. import lang
+
+
+log = logging.getLogger(__name__)
+
+
+##
+
+
+class Secret(lang.NotPicklable, lang.Final):
+    _VALUE_ATTR = '__secret_value__'
+
+    def __init__(self, *, key: str | None, value: str) -> None:
+        super().__init__()
+        self._key = key
+        setattr(self, self._VALUE_ATTR, lambda: value)
+
+    def __repr__(self) -> str:
+        return f'Secret<{self._key or ""}>'
+
+    def __str__(self) -> ta.NoReturn:
+        raise TypeError
+
+    def reveal(self) -> str:
+        return getattr(self, self._VALUE_ATTR)()
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class SecretRef:
+    key: str
+
+
+SecretRefOrStr: ta.TypeAlias = SecretRef | str
+
+
+def secret_repr(o: SecretRefOrStr | None) -> str | None:
+    if isinstance(o, str):
+        return '...'
+    elif isinstance(o, SecretRef):
+        return repr(o)
+    elif o is None:
+        return None
+    else:
+        raise TypeError(o)
+
+
+@dc.field_modifier
+def secret_field(f: dc.Field) -> dc.Field:
+    return dc.update_field_extras(
+        f,
+        repr_fn=secret_repr,
+        unless_non_default=True,
+    )
+
+
+##
+
+
+class Secrets(lang.Abstract):
+    def fix(self, obj: str | SecretRef | Secret) -> Secret:
+        if isinstance(obj, Secret):
+            return obj
+        elif isinstance(obj, str):
+            return Secret(key=None, value=obj)
+        elif isinstance(obj, SecretRef):
+            return self.get(obj.key)
+        else:
+            raise TypeError(obj)
+
+    def get(self, key: str) -> Secret:
+        try:
+            raw = self._get_raw(key)  # noqa
+        except KeyError:  # noqa
+            raise
+        else:
+            return Secret(key=key, value=raw)
+
+    @abc.abstractmethod
+    def _get_raw(self, key: str) -> str:
+        raise NotImplementedError
+
+
+##
+
+
+class EmptySecrets(Secrets):
+    def _get_raw(self, key: str) -> str:
+        raise KeyError(key)
+
+
+EMPTY_SECRETS = EmptySecrets()
+
+
+##
+
+
+class MappingSecrets(Secrets):
+    def __init__(self, dct: ta.Mapping[str, str]) -> None:
+        super().__init__()
+        self._dct = dct
+
+    def __repr__(self) -> str:
+        return f'{self.__class__.__name__}({{{", ".join(map(repr, self._dct.keys()))}}})'
+
+    def _get_raw(self, key: str) -> str:
+        return self._dct[key]
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class FnSecrets(Secrets):
+    fn: ta.Callable[[str], str]
+
+    def _get_raw(self, key: str) -> str:
+        return self.fn(key)
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class TransformedSecrets(Secrets):
+    fn: ta.Callable[[str], str]
+    child: Secrets
+
+    def _get_raw(self, key: str) -> str:
+        # FIXME: hm..
+        return self.fn(self.child._get_raw(key))  # noqa
+
+
+##
+
+
+class CachingSecrets(Secrets):
+    def __init__(
+            self,
+            child: Secrets,
+            *,
+            ttl_s: float | None = None,
+            clock: ta.Callable[..., float] = time.time,
+    ) -> None:
+        super().__init__()
+        self._child = child
+        self._dct: dict[str, str] = {}
+        self._ttl_s = ttl_s
+        self._clock = clock
+        self._deque: collections.deque[tuple[str, float]] = collections.deque()
+
+    def __repr__(self) -> str:
+        return f'{self.__class__.__name__}({{{", ".join(map(repr, self._dct.keys()))}}})'
+
+    def evict(self) -> None:
+        now = self._clock()
+        while self._deque:
+            k, dl = self._deque[0]
+            if now < dl:
+                break
+            del self._dct[k]
+            self._deque.popleft()
+
+    def _get_raw(self, key: str) -> str:
+        self.evict()
+        try:
+            return self._dct[key]
+        except KeyError:
+            pass
+        out = self._child._get_raw(key)  # noqa
+        self._dct[key] = out
+        if self._ttl_s is not None:
+            dl = self._clock() + self._ttl_s
+            self._deque.append((key, dl))
+        return out
+
+
+##
+
+
+class CompositeSecrets(Secrets):
+    def __init__(self, *children: Secrets) -> None:
+        super().__init__()
+        self._children = children
+
+    def _get_raw(self, key: str) -> str:
+        for c in self._children:
+            try:
+                return c._get_raw(key)  # noqa
+            except KeyError:
+                pass
+        raise KeyError(key)
+
+
+##
+
+
+class LoggingSecrets(Secrets):
+    def __init__(
+            self,
+            child: Secrets,
+            *,
+            log: logging.Logger | None = None,  # noqa
+    ) -> None:
+        super().__init__()
+        self._child = child
+        self._log = log if log is not None else globals()['log']
+
+    IGNORE_PACKAGES: ta.ClassVar[ta.AbstractSet[str]] = {
+        __package__,
+    }
+
+    def _get_caller_str(self, n: int = 3) -> str:
+        l: list[str] = []
+        f: types.FrameType | None = sys._getframe(2)  # noqa
+        while f is not None and len(l) < n:
+            try:
+                pkg = f.f_globals['__package__']
+            except KeyError:
+                pkg = None
+            else:
+                if pkg in self.IGNORE_PACKAGES:
+                    f = f.f_back
+                    continue
+            if (fn := f.f_code.co_filename):
+                l.append(f'{fn}:{f.f_lineno}')
+            else:
+                l.append(pkg)
+            f = f.f_back
+        return ', '.join(l)
+
+    def _get_raw(self, key: str) -> str:
+        cs = self._get_caller_str()
+        self._log.info('Attempting to access secret: %s, %s', key, cs)
+        try:
+            ret = self._child._get_raw(key)  # noqa
+        except KeyError:
+            self._log.info('Failed to access secret: %s, %s', key, cs)
+            raise
+        else:
+            self._log.info('Successfully accessed secret: %s, %s', key, cs)
+            return ret
+
+
+##
+
+
+class EnvVarSecrets(Secrets):
+    def __init__(
+            self,
+            *,
+            env: ta.MutableMapping[str, str] | None = None,
+            upcase: bool = False,
+            prefix: str | None = None,
+            pop: bool = False,
+    ) -> None:
+        super().__init__()
+        self._env = env
+        self._upcase = upcase
+        self._prefix = prefix
+        self._pop = pop
+
+    def _get_raw(self, key: str) -> str:
+        ekey = key
+        if self._upcase:
+            ekey = ekey.upper()
+        if self._prefix is not None:
+            ekey = self._prefix + ekey
+        if self._env is not None:
+            dct = self._env
+        else:
+            dct = os.environ
+        if self._pop:
+            return dct.pop(ekey)
+        else:
+            return dct[ekey]

omlish/secrets/subprocesses.py

@@ -0,0 +1,42 @@
+"""
+FIXME:
+ - macos pipe size lol, and just like checking at all
+"""
+import contextlib
+import fcntl
+import os
+import tempfile
+import typing as ta
+
+
+class SubprocessFileInput(ta.NamedTuple):
+    file_path: str
+    pass_fds: ta.Sequence[int]
+
+
+SubprocessFileInputMethod: ta.TypeAlias = ta.Callable[[bytes], ta.ContextManager[SubprocessFileInput]]
+
+
+@contextlib.contextmanager
+def temp_subprocess_file_input(buf: bytes) -> ta.Iterator[SubprocessFileInput]:
+    with tempfile.NamedTemporaryFile(delete=True) as kf:
+        kf.write(buf)
+        kf.flush()
+        yield SubprocessFileInput(kf.name, [])
+
+
+@contextlib.contextmanager
+def pipe_fd_subprocess_file_input(buf: bytes) -> ta.Iterator[SubprocessFileInput]:
+    rfd, wfd = os.pipe()
+    closed_wfd = False
+    try:
+        if hasattr(fcntl, 'F_SETPIPE_SZ'):
+            fcntl.fcntl(wfd, fcntl.F_SETPIPE_SZ, max(len(buf), 0x1000))
+        os.write(wfd, buf)
+        os.close(wfd)
+        closed_wfd = True
+        yield SubprocessFileInput(f'/dev/fd/{rfd}', [rfd])
+    finally:
+        if not closed_wfd:
+            os.close(wfd)
+        os.close(rfd)

omlish/sql/dbs.py

@@ -3,20 +3,21 @@ import urllib.parse
 
 from .. import dataclasses as dc
 from .. import lang
+from .. import secrets as sec
 
 
 ##
 
 
 @dc.dataclass(frozen=True, kw_only=True)
-class DbType:
+class DbType(lang.Final):
     name: str
     dialect_name: str
 
     default_port: int | None = None
 
 
-class DbTypes(lang.Namespace):
+class DbTypes(lang.Namespace, lang.Final):
     MYSQL = DbType(
         name='mysql',
         dialect_name='mysql',
@@ -38,13 +39,13 @@ class DbTypes(lang.Namespace):
 ##
 
 
-class DbLoc(lang.Abstract):
+class DbLoc(lang.Abstract, lang.Sealed):
     pass
 
 
 @dc.dataclass(frozen=True)
 class UrlDbLoc(DbLoc, lang.Final):
-    url: str
+    url: sec.SecretRefOrStr = dc.xfield() | sec.secret_field
 
 
 @dc.dataclass(frozen=True)
@@ -53,14 +54,14 @@ class HostDbLoc(DbLoc, lang.Final):
     port: int | None = None
 
     username: str | None = None
-    password: str | None = dc.xfield(default=None, repr_fn=lambda pw: '...' if pw is not None else None)
+    password: sec.SecretRefOrStr | None = dc.xfield(default=None) | sec.secret_field
 
 
 ##
 
 
 @dc.dataclass(frozen=True)
-class DbSpec:
+class DbSpec(lang.Final):
     name: str
     type: DbType
     loc: DbLoc

omlish/sql/duckdb.py

@@ -0,0 +1,136 @@
+"""
+See:
+ - https://duckdb.org/docs/api/python/overview.html
+ - https://github.com/Mause/duckdb_engine/blob/0e3ea0107f81c66d50b444011d31fce22a9b902c/duckdb_engine/__init__.py
+"""
+import typing as ta
+
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql as sap
+
+from .. import lang
+
+
+if ta.TYPE_CHECKING:
+    import duckdb
+else:
+    duckdb = lang.proxy_import('duckdb')
+
+
+class ConnectionWrapper:
+    def __init__(self, c: 'duckdb.DuckDBPyConnection') -> None:
+        super().__init__()
+        self.__c = c
+        self.autocommit = None
+        self.closed = False
+
+    def cursor(self) -> 'CursorWrapper':
+        return CursorWrapper(self.__c, self)
+
+    def __getattr__(self, name: str) -> ta.Any:
+        return getattr(self.__c, name)
+
+    def close(self) -> None:
+        self.__c.close()
+        self.closed = True
+
+
+def _is_transaction_context_message(e: Exception) -> bool:
+    return e.args[0] == 'TransactionContext Error: cannot rollback - no transaction is active'
+
+
+class CursorWrapper:
+    def __init__(
+            self,
+            c: 'duckdb.DuckDBPyConnection',
+            connection_wrapper: 'ConnectionWrapper',
+    ) -> None:
+        super().__init__()
+        self.__c = c
+        self.__connection_wrapper = connection_wrapper
+
+    def executemany(
+        self,
+        statement: str,
+        parameters: ta.Sequence[ta.Mapping[str, ta.Any]] | None = None,
+        context: ta.Any | None = None,
+    ) -> None:
+        self.__c.executemany(statement, list(parameters) if parameters else [])
+
+    def execute(
+        self,
+        statement: str,
+        parameters: ta.Sequence[ta.Any] | None = None,
+        context: ta.Any | None = None,
+    ) -> None:
+        try:
+            if statement.lower() == 'commit':  # this is largely for ipython-sql
+                self.__c.commit()
+            elif parameters is None:
+                self.__c.execute(statement)
+            else:
+                self.__c.execute(statement, parameters)
+        except RuntimeError as e:
+            if e.args[0].startswith('Not implemented Error'):
+                raise NotImplementedError(*e.args) from e
+            elif _is_transaction_context_message(e):
+                return
+            else:
+                raise
+
+    @property
+    def connection(self):
+        return self.__connection_wrapper
+
+    def close(self) -> None:
+        pass
+
+    def __getattr__(self, name: str) -> ta.Any:
+        return getattr(self.__c, name)
+
+    def fetchmany(self, size: int | None = None) -> list:
+        if size is None:
+            return self.__c.fetchmany()
+        else:
+            return self.__c.fetchmany(size)
+
+
+class DuckdbDialect(sap.base.PGDialect):
+    name = 'postgres__duckdb'
+    driver = 'duckdb_engine'
+
+    supports_statement_cache = False
+
+    @classmethod
+    def import_dbapi(cls):
+        return duckdb
+
+    def connect(self, *cargs, **cparams):
+        conn = self.loaded_dbapi.connect(*cargs, **cparams)
+
+        return ConnectionWrapper(conn)
+
+    def do_rollback(self, connection) -> None:
+        try:
+            super().do_rollback(connection)
+        except duckdb.TransactionException as e:
+            if _is_transaction_context_message(e):
+                return
+            else:
+                raise
+
+    def do_begin(self, connection) -> None:
+        connection.begin()
+
+    def get_default_isolation_level(self, connection) -> ta.Any:
+        raise NotImplementedError
+
+    def _get_server_version_info(self, connection) -> tuple[int, int]:
+        connection.execute(sa.text('select version()')).fetchone()
+        return (8, 0)
+
+    def _set_backslash_escapes(self, connection):
+        pass
+
+
+sa.dialects.registry.register(DuckdbDialect.name, __name__, DuckdbDialect.__name__)

omlish/sql/exprs.py

@@ -0,0 +1,12 @@
+import sqlalchemy as sa
+import sqlalchemy.ext.compiler
+
+
+class paren(sa.sql.expression.UnaryExpression):  # noqa
+    __visit_name__ = 'paren'
+    inherit_cache = True
+
+
+@sa.ext.compiler.compiles(paren)
+def _compile_paren(element, compiler, **kw):
+    return '(%s)' % (element.element._compiler_dispatch(compiler),)  # noqa

omlish/sql/secrets.py

@@ -0,0 +1,10 @@
+"""
+TODO:
+ - sync/async...
+"""
+from .. import secrets as sec
+
+
+class SqlFunctionSecrets(sec.Secrets):
+    def _get_raw(self, key: str) -> str:
+        raise NotImplementedError

omlish/sql/sqlean.py

@@ -0,0 +1,17 @@
+import sqlalchemy as sa
+from sqlalchemy.dialects import sqlite as sal
+
+
+class SqleanDialect(sal.dialect):  # type: ignore
+    name = 'sqlite__sqlean'
+    driver = 'sqlean_engine'
+
+    supports_statement_cache = True
+
+    @classmethod
+    def import_dbapi(cls):
+        from sqlean import dbapi2
+        return dbapi2
+
+
+sa.dialects.registry.register(SqleanDialect.name, __name__, SqleanDialect.__name__)