omlish 0.0.0.dev6__py3-none-any.whl → 0.0.0.dev7__py3-none-any.whl

omlish/secrets/secrets.py

@@ -1,47 +1,299 @@
+"""
+TODO:
+ - SqlFunctionSecrets (in .sql?)
+ - crypto is just Transformed, bound with a key
+ - crypto key in env + values in file?
+ - Secret:
+  - hold ref to Secret, and key
+  - time of retrieval
+  - logs accesses
+ - types? ssh / url / pw / basicauthtoken / tls / str
+"""
 import abc
+import collections
+import logging
+import os
+import sys
+import time
+import types  # noqa
 import typing as ta
 
 from .. import dataclasses as dc
 from .. import lang
 
 
+log = logging.getLogger(__name__)
+
+
+##
+
+
+class Secret(lang.NotPicklable, lang.Final):
+    _VALUE_ATTR = '__secret_value__'
+
+    def __init__(self, *, key: str | None, value: str) -> None:
+        super().__init__()
+        self._key = key
+        setattr(self, self._VALUE_ATTR, lambda: value)
+
+    def __repr__(self) -> str:
+        return f'Secret<{self._key or ""}>'
+
+    def __str__(self) -> ta.NoReturn:
+        raise TypeError
+
+    def reveal(self) -> str:
+        return getattr(self, self._VALUE_ATTR)()
+
+
 ##
 
 
 @dc.dataclass(frozen=True)
-class Secret:
+class SecretRef:
     key: str
 
 
+SecretRefOrStr: ta.TypeAlias = SecretRef | str
+
+
+def secret_repr(o: SecretRefOrStr | None) -> str | None:
+    if isinstance(o, str):
+        return '...'
+    elif isinstance(o, SecretRef):
+        return repr(o)
+    elif o is None:
+        return None
+    else:
+        raise TypeError(o)
+
+
+@dc.field_modifier
+def secret_field(f: dc.Field) -> dc.Field:
+    return dc.update_field_extras(
+        f,
+        repr_fn=secret_repr,
+        unless_non_default=True,
+    )
+
+
 ##
 
 
 class Secrets(lang.Abstract):
-    def fix(self, obj: str | Secret) -> str:
-        if isinstance(obj, str):
+    def fix(self, obj: str | SecretRef | Secret) -> Secret:
+        if isinstance(obj, Secret):
             return obj
-        elif isinstance(obj, Secret):
+        elif isinstance(obj, str):
+            return Secret(key=None, value=obj)
+        elif isinstance(obj, SecretRef):
             return self.get(obj.key)
         else:
             raise TypeError(obj)
 
+    def get(self, key: str) -> Secret:
+        try:
+            raw = self._get_raw(key)  # noqa
+        except KeyError:  # noqa
+            raise
+        else:
+            return Secret(key=key, value=raw)
+
     @abc.abstractmethod
-    def get(self, key: str) -> str:
+    def _get_raw(self, key: str) -> str:
         raise NotImplementedError
 
 
+##
+
+
 class EmptySecrets(Secrets):
-    def get(self, key: str) -> str:
+    def _get_raw(self, key: str) -> str:
         raise KeyError(key)
 
 
 EMPTY_SECRETS = EmptySecrets()
 
 
-class SimpleSecrets(Secrets):
+##
+
+
+class MappingSecrets(Secrets):
     def __init__(self, dct: ta.Mapping[str, str]) -> None:
         super().__init__()
         self._dct = dct
 
-    def get(self, key: str) -> str:
+    def __repr__(self) -> str:
+        return f'{self.__class__.__name__}({{{", ".join(map(repr, self._dct.keys()))}}})'
+
+    def _get_raw(self, key: str) -> str:
         return self._dct[key]
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class FnSecrets(Secrets):
+    fn: ta.Callable[[str], str]
+
+    def _get_raw(self, key: str) -> str:
+        return self.fn(key)
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class TransformedSecrets(Secrets):
+    fn: ta.Callable[[str], str]
+    child: Secrets
+
+    def _get_raw(self, key: str) -> str:
+        # FIXME: hm..
+        return self.fn(self.child._get_raw(key))  # noqa
+
+
+##
+
+
+class CachingSecrets(Secrets):
+    def __init__(
+            self,
+            child: Secrets,
+            *,
+            ttl_s: float | None = None,
+            clock: ta.Callable[..., float] = time.time,
+    ) -> None:
+        super().__init__()
+        self._child = child
+        self._dct: dict[str, str] = {}
+        self._ttl_s = ttl_s
+        self._clock = clock
+        self._deque: collections.deque[tuple[str, float]] = collections.deque()
+
+    def __repr__(self) -> str:
+        return f'{self.__class__.__name__}({{{", ".join(map(repr, self._dct.keys()))}}})'
+
+    def evict(self) -> None:
+        now = self._clock()
+        while self._deque:
+            k, dl = self._deque[0]
+            if now < dl:
+                break
+            del self._dct[k]
+            self._deque.popleft()
+
+    def _get_raw(self, key: str) -> str:
+        self.evict()
+        try:
+            return self._dct[key]
+        except KeyError:
+            pass
+        out = self._child._get_raw(key)  # noqa
+        self._dct[key] = out
+        if self._ttl_s is not None:
+            dl = self._clock() + self._ttl_s
+            self._deque.append((key, dl))
+        return out
+
+
+##
+
+
+class CompositeSecrets(Secrets):
+    def __init__(self, *children: Secrets) -> None:
+        super().__init__()
+        self._children = children
+
+    def _get_raw(self, key: str) -> str:
+        for c in self._children:
+            try:
+                return c._get_raw(key)  # noqa
+            except KeyError:
+                pass
+        raise KeyError(key)
+
+
+##
+
+
+class LoggingSecrets(Secrets):
+    def __init__(
+            self,
+            child: Secrets,
+            *,
+            log: logging.Logger | None = None,  # noqa
+    ) -> None:
+        super().__init__()
+        self._child = child
+        self._log = log if log is not None else globals()['log']
+
+    IGNORE_PACKAGES: ta.ClassVar[ta.AbstractSet[str]] = {
+        __package__,
+    }
+
+    def _get_caller_str(self, n: int = 3) -> str:
+        l: list[str] = []
+        f: types.FrameType | None = sys._getframe(2)  # noqa
+        while f is not None and len(l) < n:
+            try:
+                pkg = f.f_globals['__package__']
+            except KeyError:
+                pkg = None
+            else:
+                if pkg in self.IGNORE_PACKAGES:
+                    f = f.f_back
+                    continue
+            if (fn := f.f_code.co_filename):
+                l.append(f'{fn}:{f.f_lineno}')
+            else:
+                l.append(pkg)
+            f = f.f_back
+        return ', '.join(l)
+
+    def _get_raw(self, key: str) -> str:
+        cs = self._get_caller_str()
+        self._log.info('Attempting to access secret: %s, %s', key, cs)
+        try:
+            ret = self._child._get_raw(key)  # noqa
+        except KeyError:
+            self._log.info('Failed to access secret: %s, %s', key, cs)
+            raise
+        else:
+            self._log.info('Successfully accessed secret: %s, %s', key, cs)
+            return ret
+
+
+##
+
+
+class EnvVarSecrets(Secrets):
+    def __init__(
+            self,
+            *,
+            env: ta.MutableMapping[str, str] | None = None,
+            upcase: bool = False,
+            prefix: str | None = None,
+            pop: bool = False,
+    ) -> None:
+        super().__init__()
+        self._env = env
+        self._upcase = upcase
+        self._prefix = prefix
+        self._pop = pop
+
+    def _get_raw(self, key: str) -> str:
+        ekey = key
+        if self._upcase:
+            ekey = ekey.upper()
+        if self._prefix is not None:
+            ekey = self._prefix + ekey
+        if self._env is not None:
+            dct = self._env
+        else:
+            dct = os.environ
+        if self._pop:
+            return dct.pop(ekey)
+        else:
+            return dct[ekey]

omlish/secrets/subprocesses.py

@@ -0,0 +1,42 @@
+"""
+FIXME:
+ - macos pipe size lol, and just like checking at all
+"""
+import contextlib
+import fcntl
+import os
+import tempfile
+import typing as ta
+
+
+class SubprocessFileInput(ta.NamedTuple):
+    file_path: str
+    pass_fds: ta.Sequence[int]
+
+
+SubprocessFileInputMethod: ta.TypeAlias = ta.Callable[[bytes], ta.ContextManager[SubprocessFileInput]]
+
+
+@contextlib.contextmanager
+def temp_subprocess_file_input(buf: bytes) -> ta.Iterator[SubprocessFileInput]:
+    with tempfile.NamedTemporaryFile(delete=True) as kf:
+        kf.write(buf)
+        kf.flush()
+        yield SubprocessFileInput(kf.name, [])
+
+
+@contextlib.contextmanager
+def pipe_fd_subprocess_file_input(buf: bytes) -> ta.Iterator[SubprocessFileInput]:
+    rfd, wfd = os.pipe()
+    closed_wfd = False
+    try:
+        if hasattr(fcntl, 'F_SETPIPE_SZ'):
+            fcntl.fcntl(wfd, fcntl.F_SETPIPE_SZ, max(len(buf), 0x1000))
+        os.write(wfd, buf)
+        os.close(wfd)
+        closed_wfd = True
+        yield SubprocessFileInput(f'/dev/fd/{rfd}', [rfd])
+    finally:
+        if not closed_wfd:
+            os.close(wfd)
+        os.close(rfd)

omlish/sql/dbs.py

@@ -3,13 +3,14 @@ import urllib.parse
 
 from .. import dataclasses as dc
 from .. import lang
+from .. import secrets as sec
 
 
 ##
 
 
 @dc.dataclass(frozen=True, kw_only=True)
-class DbType:
+class DbType(lang.Final):
     name: str
     dialect_name: str
 
@@ -38,13 +39,13 @@ class DbTypes(lang.Namespace, lang.Final):
 ##
 
 
-class DbLoc(lang.Abstract):
+class DbLoc(lang.Abstract, lang.Sealed):
     pass
 
 
 @dc.dataclass(frozen=True)
 class UrlDbLoc(DbLoc, lang.Final):
-    url: str
+    url: sec.SecretRefOrStr = dc.xfield() | sec.secret_field
 
 
 @dc.dataclass(frozen=True)
@@ -53,14 +54,14 @@ class HostDbLoc(DbLoc, lang.Final):
     port: int | None = None
 
     username: str | None = None
-    password: str | None = dc.xfield(default=None, repr_fn=lambda pw: '...' if pw is not None else None)
+    password: sec.SecretRefOrStr | None = dc.xfield(default=None) | sec.secret_field
 
 
 ##
 
 
 @dc.dataclass(frozen=True)
-class DbSpec:
+class DbSpec(lang.Final):
     name: str
     type: DbType
     loc: DbLoc

omlish/sql/exprs.py

@@ -0,0 +1,12 @@
+import sqlalchemy as sa
+import sqlalchemy.ext.compiler
+
+
+class paren(sa.sql.expression.UnaryExpression):  # noqa
+    __visit_name__ = 'paren'
+    inherit_cache = True
+
+
+@sa.ext.compiler.compiles(paren)
+def _compile_paren(element, compiler, **kw):
+    return '(%s)' % (element.element._compiler_dispatch(compiler),)  # noqa

omlish/sql/secrets.py

@@ -0,0 +1,10 @@
+"""
+TODO:
+ - sync/async...
+"""
+from .. import secrets as sec
+
+
+class SqlFunctionSecrets(sec.Secrets):
+    def _get_raw(self, key: str) -> str:
+        raise NotImplementedError

omlish/term.py

@@ -20,7 +20,7 @@ def set_title(title: str) -> str:
 
 
 def strip_ansi_codes(s: str) -> str:
-    return re.sub(r'\033\\[([0-9]+)(;[0-9]+)*m', '', s)
+    return re.sub(r'\033\[([0-9]+)(;[0-9]+)*m', '', s)
 
 
 class ControlSequence:

omlish/testing/pytest/plugins/switches.py

@@ -16,10 +16,21 @@ from ._registry import register
 Configable = pytest.FixtureRequest | pytest.Config
 
 
-SWITCHES = col.OrderedSet([
-    'online',
-    'slow',
-])
+SWITCHES = {
+    'docker': True,
+    'online': True,
+    'integration': True,
+    'slow': False,
+}
+
+
+SwitchState: ta.TypeAlias = bool | ta.Literal['only']
+
+SWITCH_STATE_OPT_PREFIXES: ta.Mapping[SwitchState, str] = {
+    True: '--',
+    False: '--no-',
+    'only': '--only-',
+}
 
 
 def _get_obj_config(obj: Configable) -> pytest.Config:
@@ -42,29 +53,53 @@ def skip_if_disabled(obj: Configable | None, name: str) -> None:
         pytest.skip(f'{name} disabled')
 
 
-def get_switches(obj: Configable) -> ta.Mapping[str, bool]:
-    return {
-        sw: _get_obj_config(obj).getoption(f'--no-{sw}')
-        for sw in SWITCHES
-    }
+def get_switches(obj: Configable) -> ta.Mapping[str, SwitchState]:
+    ret: dict[str, SwitchState] = {}
+    for sw, d in SWITCHES.items():
+        sts = {
+            st
+            for st, pfx in SWITCH_STATE_OPT_PREFIXES.items()
+            if _get_obj_config(obj).getoption(pfx + sw)
+        }
+        if sts:
+            if len(sts) > 1:
+                raise Exception(f'Multiple switches specified for {sw}')
+            ret[sw] = check.single(sts)
+        else:
+            ret[sw] = d
+    return ret
 
 
 @register
 class SwitchesPlugin:
 
+    def pytest_configure(self, config):
+        for sw in SWITCHES:
+            config.addinivalue_line('markers', f'{sw}: mark test as {sw}')
+
     def pytest_addoption(self, parser):
         for sw in SWITCHES:
             parser.addoption(f'--no-{sw}', action='store_true', default=False, help=f'disable {sw} tests')
+            parser.addoption(f'--{sw}', action='store_true', default=False, help=f'enables {sw} tests')
+            parser.addoption(f'--only-{sw}', action='store_true', default=False, help=f'enables only {sw} tests')
 
     def pytest_collection_modifyitems(self, config, items):
-        for sw in SWITCHES:
-            if not config.getoption(f'--no-{sw}'):
-                continue
-            skip = pytest.mark.skip(reason=f'omit --no-{sw} to run')
-            for item in items:
-                if sw in item.keywords:
-                    item.add_marker(skip)
+        sts = get_switches(config)
+        stx = col.multi_map(map(reversed, sts.items()))  # type: ignore
+        ts, fs, onlys = (stx.get(k, ()) for k in (True, False, 'only'))
 
-    def pytest_configure(self, config):
-        for sw in SWITCHES:
-            config.addinivalue_line('markers', f'{sw}: mark test as {sw}')
+        def process(item):
+            sws = {sw for sw in SWITCHES if sw in item.keywords}
+
+            if onlys:
+                if not any(sw in onlys for sw in sws):
+                    item.add_marker(pytest.mark.skip(reason=f'skipping switches {sws}'))
+                    return
+
+            else:
+                for sw in sws:
+                    if sw in fs:
+                        item.add_marker(pytest.mark.skip(reason=f'skipping switches {sw}'))
+
+        for item in items:
+            process(item)

omlish/text/glyphsplit.py

@@ -1,3 +1,8 @@
+"""
+Note: string.Formatter (and string.Template) shouldn't be ignored - if they can be used they probably should be.
+ - https://docs.python.org/3/library/string.html#custom-string-formatting
+ - https://docs.python.org/3/library/string.html#template-strings
+"""
 import dataclasses as dc
 import re
 

omlish-0.0.0.dev7.dist-info/METADATA

@@ -0,0 +1,50 @@
+Metadata-Version: 2.1
+Name: omlish
+Version: 0.0.0.dev7
+Summary: omlish
+Author: wrmsr
+License: BSD-3-Clause
+Project-URL: source, https://github.com/wrmsr/omlish
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Development Status :: 2 - Pre-Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Operating System :: POSIX
+Requires-Python: >=3.12
+License-File: LICENSE
+Provides-Extra: async
+Requires-Dist: anyio >=4.4 ; extra == 'async'
+Requires-Dist: sniffio >=1.3 ; extra == 'async'
+Requires-Dist: trio >=0.26 ; extra == 'async'
+Requires-Dist: greenlet >=3 ; (python_version < "3.13") and extra == 'async'
+Requires-Dist: trio-asyncio >=0.15 ; (python_version < "3.13") and extra == 'async'
+Provides-Extra: compression
+Requires-Dist: lz4 >=4 ; extra == 'compression'
+Requires-Dist: zstd >=1.5 ; extra == 'compression'
+Requires-Dist: python-snappy >=0.7 ; (python_version < "3.13") and extra == 'compression'
+Provides-Extra: formats
+Requires-Dist: orjson >3.10 ; extra == 'formats'
+Requires-Dist: cloudpickle >=3 ; extra == 'formats'
+Requires-Dist: pyyaml >=5 ; extra == 'formats'
+Provides-Extra: http
+Requires-Dist: httpx[http2] >=0.27 ; extra == 'http'
+Provides-Extra: misc
+Requires-Dist: jinja2 >=3.1 ; extra == 'misc'
+Requires-Dist: psutil >=6 ; extra == 'misc'
+Requires-Dist: wrapt >=1.14 ; extra == 'misc'
+Provides-Extra: secrets
+Requires-Dist: cryptography >=43 ; extra == 'secrets'
+Provides-Extra: sql
+Requires-Dist: pg8000 >=1.31 ; extra == 'sql'
+Requires-Dist: pymysql >=1.1 ; extra == 'sql'
+Requires-Dist: aiomysql >=0.2 ; extra == 'sql'
+Requires-Dist: aiosqlite >=0.20 ; extra == 'sql'
+Requires-Dist: sqlalchemy[asyncio] >=2 ; (python_version < "3.13") and extra == 'sql'
+Requires-Dist: asyncpg >=0.29 ; (python_version < "3.13") and extra == 'sql'
+Requires-Dist: sqlalchemy >=2 ; (python_version >= "3.13") and extra == 'sql'
+Provides-Extra: sqlx
+Requires-Dist: duckdb >=1 ; extra == 'sqlx'
+Requires-Dist: sqlean.py >=3.45 ; (python_version < "3.13") and extra == 'sqlx'
+Provides-Extra: testing
+Requires-Dist: pytest >=8 ; extra == 'testing'
+