omlish 0.0.0.dev5__py3-none-any.whl → 0.0.0.dev7__py3-none-any.whl

omlish/lite/marshal.py

@@ -0,0 +1,316 @@
+"""
+TODO:
+ - pickle stdlib objs? have to pin to 3.8 pickle protocol, will be cross-version
+"""
+# ruff: noqa: UP006 UP007
+import abc
+import base64
+import collections.abc
+import dataclasses as dc  # noqa
+import datetime
+import decimal
+import enum
+import fractions
+import typing as ta
+import uuid
+import weakref  # noqa
+
+from .check import check_isinstance
+from .check import check_not_none
+from .reflect import deep_subclasses
+from .reflect import get_optional_alias_arg
+from .reflect import is_generic_alias
+from .reflect import is_union_alias
+
+
+T = ta.TypeVar('T')
+
+
+##
+
+
+class ObjMarshaler(abc.ABC):
+    @abc.abstractmethod
+    def marshal(self, o: ta.Any) -> ta.Any:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def unmarshal(self, o: ta.Any) -> ta.Any:
+        raise NotImplementedError
+
+
+class NopObjMarshaler(ObjMarshaler):
+    def marshal(self, o: ta.Any) -> ta.Any:
+        return o
+
+    def unmarshal(self, o: ta.Any) -> ta.Any:
+        return o
+
+
+@dc.dataclass()
+class ProxyObjMarshaler(ObjMarshaler):
+    m: ta.Optional[ObjMarshaler] = None
+
+    def marshal(self, o: ta.Any) -> ta.Any:
+        return check_not_none(self.m).marshal(o)
+
+    def unmarshal(self, o: ta.Any) -> ta.Any:
+        return check_not_none(self.m).unmarshal(o)
+
+
+@dc.dataclass(frozen=True)
+class CastObjMarshaler(ObjMarshaler):
+    ty: type
+
+    def marshal(self, o: ta.Any) -> ta.Any:
+        return o
+
+    def unmarshal(self, o: ta.Any) -> ta.Any:
+        return self.ty(o)
+
+
+class DynamicObjMarshaler(ObjMarshaler):
+    def marshal(self, o: ta.Any) -> ta.Any:
+        return marshal_obj(o)
+
+    def unmarshal(self, o: ta.Any) -> ta.Any:
+        return o
+
+
+@dc.dataclass(frozen=True)
+class Base64ObjMarshaler(ObjMarshaler):
+    ty: type
+
+    def marshal(self, o: ta.Any) -> ta.Any:
+        return base64.b64encode(o).decode('ascii')
+
+    def unmarshal(self, o: ta.Any) -> ta.Any:
+        return self.ty(base64.b64decode(o))
+
+
+@dc.dataclass(frozen=True)
+class EnumObjMarshaler(ObjMarshaler):
+    ty: type
+
+    def marshal(self, o: ta.Any) -> ta.Any:
+        return o.name
+
+    def unmarshal(self, o: ta.Any) -> ta.Any:
+        return self.ty.__members__[o]  # type: ignore
+
+
+@dc.dataclass(frozen=True)
+class OptionalObjMarshaler(ObjMarshaler):
+    item: ObjMarshaler
+
+    def marshal(self, o: ta.Any) -> ta.Any:
+        if o is None:
+            return None
+        return self.item.marshal(o)
+
+    def unmarshal(self, o: ta.Any) -> ta.Any:
+        if o is None:
+            return None
+        return self.item.unmarshal(o)
+
+
+@dc.dataclass(frozen=True)
+class MappingObjMarshaler(ObjMarshaler):
+    ty: type
+    km: ObjMarshaler
+    vm: ObjMarshaler
+
+    def marshal(self, o: ta.Any) -> ta.Any:
+        return {self.km.marshal(k): self.vm.marshal(v) for k, v in o.items()}
+
+    def unmarshal(self, o: ta.Any) -> ta.Any:
+        return self.ty((self.km.unmarshal(k), self.vm.unmarshal(v)) for k, v in o.items())
+
+
+@dc.dataclass(frozen=True)
+class IterableObjMarshaler(ObjMarshaler):
+    ty: type
+    item: ObjMarshaler
+
+    def marshal(self, o: ta.Any) -> ta.Any:
+        return [self.item.marshal(e) for e in o]
+
+    def unmarshal(self, o: ta.Any) -> ta.Any:
+        return self.ty(self.item.unmarshal(e) for e in o)
+
+
+@dc.dataclass(frozen=True)
+class DataclassObjMarshaler(ObjMarshaler):
+    ty: type
+    fs: ta.Mapping[str, ObjMarshaler]
+
+    def marshal(self, o: ta.Any) -> ta.Any:
+        return {k: m.marshal(getattr(o, k)) for k, m in self.fs.items()}
+
+    def unmarshal(self, o: ta.Any) -> ta.Any:
+        return self.ty(**{k: self.fs[k].unmarshal(v) for k, v in o.items()})
+
+
+@dc.dataclass(frozen=True)
+class PolymorphicObjMarshaler(ObjMarshaler):
+    class Impl(ta.NamedTuple):
+        ty: type
+        tag: str
+        m: ObjMarshaler
+
+    impls_by_ty: ta.Mapping[type, Impl]
+    impls_by_tag: ta.Mapping[str, Impl]
+
+    def marshal(self, o: ta.Any) -> ta.Any:
+        impl = self.impls_by_ty[type(o)]
+        return {impl.tag: impl.m.marshal(o)}
+
+    def unmarshal(self, o: ta.Any) -> ta.Any:
+        [(t, v)] = o.items()
+        impl = self.impls_by_tag[t]
+        return impl.m.unmarshal(v)
+
+
+@dc.dataclass(frozen=True)
+class DatetimeObjMarshaler(ObjMarshaler):
+    ty: type
+
+    def marshal(self, o: ta.Any) -> ta.Any:
+        return o.isoformat()
+
+    def unmarshal(self, o: ta.Any) -> ta.Any:
+        return self.ty.fromisoformat(o)  # type: ignore
+
+
+class DecimalObjMarshaler(ObjMarshaler):
+    def marshal(self, o: ta.Any) -> ta.Any:
+        return str(check_isinstance(o, decimal.Decimal))
+
+    def unmarshal(self, v: ta.Any) -> ta.Any:
+        return decimal.Decimal(check_isinstance(v, str))
+
+
+class FractionObjMarshaler(ObjMarshaler):
+    def marshal(self, o: ta.Any) -> ta.Any:
+        fr = check_isinstance(o, fractions.Fraction)
+        return [fr.numerator, fr.denominator]
+
+    def unmarshal(self, v: ta.Any) -> ta.Any:
+        num, denom = check_isinstance(v, list)
+        return fractions.Fraction(num, denom)
+
+
+class UuidObjMarshaler(ObjMarshaler):
+    def marshal(self, o: ta.Any) -> ta.Any:
+        return str(o)
+
+    def unmarshal(self, o: ta.Any) -> ta.Any:
+        return uuid.UUID(o)
+
+
+_OBJ_MARSHALERS: ta.Dict[ta.Any, ObjMarshaler] = {
+    **{t: NopObjMarshaler() for t in (type(None),)},
+    **{t: CastObjMarshaler(t) for t in (int, float, str, bool)},
+    **{t: Base64ObjMarshaler(t) for t in (bytes, bytearray)},
+    **{t: IterableObjMarshaler(t, DynamicObjMarshaler()) for t in (list, tuple, set, frozenset)},
+    **{t: MappingObjMarshaler(t, DynamicObjMarshaler(), DynamicObjMarshaler()) for t in (dict,)},
+
+    ta.Any: DynamicObjMarshaler(),
+
+    **{t: DatetimeObjMarshaler(t) for t in (datetime.date, datetime.time, datetime.datetime)},
+    decimal.Decimal: DecimalObjMarshaler(),
+    fractions.Fraction: FractionObjMarshaler(),
+    uuid.UUID: UuidObjMarshaler(),
+}
+
+_OBJ_MARSHALER_GENERIC_MAPPING_TYPES: ta.Dict[ta.Any, type] = {
+    **{t: t for t in (dict,)},
+    **{t: dict for t in (collections.abc.Mapping, collections.abc.MutableMapping)},
+}
+
+_OBJ_MARSHALER_GENERIC_ITERABLE_TYPES: ta.Dict[ta.Any, type] = {
+    **{t: t for t in (list, tuple, set, frozenset)},
+    **{t: frozenset for t in (collections.abc.Set, collections.abc.MutableSet)},
+    **{t: tuple for t in (collections.abc.Sequence, collections.abc.MutableSequence)},
+}
+
+
+def register_opj_marshaler(ty: ta.Any, m: ObjMarshaler) -> None:
+    if ty in _OBJ_MARSHALERS:
+        raise KeyError(ty)
+    _OBJ_MARSHALERS[ty] = m
+
+
+def _make_obj_marshaler(ty: ta.Any) -> ObjMarshaler:
+    if isinstance(ty, type) and abc.ABC in ty.__bases__:
+        impls = [  # type: ignore
+            PolymorphicObjMarshaler.Impl(
+                ity,
+                ity.__qualname__,
+                get_obj_marshaler(ity),
+            )
+            for ity in deep_subclasses(ty)
+            if abc.ABC not in ity.__bases__
+        ]
+        return PolymorphicObjMarshaler(
+            {i.ty: i for i in impls},
+            {i.tag: i for i in impls},
+        )
+
+    if isinstance(ty, type) and issubclass(ty, enum.Enum):
+        return EnumObjMarshaler(ty)
+
+    if dc.is_dataclass(ty):
+        return DataclassObjMarshaler(
+            ty,
+            {f.name: get_obj_marshaler(f.type) for f in dc.fields(ty)},
+        )
+
+    if is_generic_alias(ty):
+        try:
+            mt = _OBJ_MARSHALER_GENERIC_MAPPING_TYPES[ta.get_origin(ty)]
+        except KeyError:
+            pass
+        else:
+            k, v = ta.get_args(ty)
+            return MappingObjMarshaler(mt, get_obj_marshaler(k), get_obj_marshaler(v))
+
+        try:
+            st = _OBJ_MARSHALER_GENERIC_ITERABLE_TYPES[ta.get_origin(ty)]
+        except KeyError:
+            pass
+        else:
+            [e] = ta.get_args(ty)
+            return IterableObjMarshaler(st, get_obj_marshaler(e))
+
+        if is_union_alias(ty):
+            return OptionalObjMarshaler(get_obj_marshaler(get_optional_alias_arg(ty)))
+
+    raise TypeError(ty)
+
+
+def get_obj_marshaler(ty: ta.Any) -> ObjMarshaler:
+    try:
+        return _OBJ_MARSHALERS[ty]
+    except KeyError:
+        pass
+
+    p = ProxyObjMarshaler()
+    _OBJ_MARSHALERS[ty] = p
+    try:
+        m = _make_obj_marshaler(ty)
+    except Exception:
+        del _OBJ_MARSHALERS[ty]
+        raise
+    else:
+        p.m = m
+        _OBJ_MARSHALERS[ty] = m
+        return m
+
+
+def marshal_obj(o: ta.Any, ty: ta.Any = None) -> ta.Any:
+    return get_obj_marshaler(ty if ty is not None else type(o)).marshal(o)
+
+
+def unmarshal_obj(o: ta.Any, ty: ta.Union[ta.Type[T], ta.Any]) -> T:
+    return get_obj_marshaler(ty).unmarshal(o)

omlish/lite/reflect.py

@@ -0,0 +1,49 @@
+# ruff: noqa: UP006
+import functools
+import typing as ta
+
+
+T = ta.TypeVar('T')
+
+
+_GENERIC_ALIAS_TYPES = (
+    ta._GenericAlias,  # type: ignore  # noqa
+    *([ta._SpecialGenericAlias] if hasattr(ta, '_SpecialGenericAlias') else []),  # noqa
+)
+
+
+def is_generic_alias(obj, *, origin: ta.Any = None) -> bool:
+    return (
+        isinstance(obj, _GENERIC_ALIAS_TYPES) and
+        (origin is None or ta.get_origin(obj) is origin)
+    )
+
+
+is_union_alias = functools.partial(is_generic_alias, origin=ta.Union)
+is_callable_alias = functools.partial(is_generic_alias, origin=ta.Callable)
+
+
+def is_optional_alias(spec: ta.Any) -> bool:
+    return (
+        isinstance(spec, _GENERIC_ALIAS_TYPES) and  # noqa
+        ta.get_origin(spec) is ta.Union and
+        len(ta.get_args(spec)) == 2 and
+        any(a in (None, type(None)) for a in ta.get_args(spec))
+    )
+
+
+def get_optional_alias_arg(spec: ta.Any) -> ta.Any:
+    [it] = [it for it in ta.get_args(spec) if it not in (None, type(None))]
+    return it
+
+
+def deep_subclasses(cls: ta.Type[T]) -> ta.Iterator[ta.Type[T]]:
+    seen = set()
+    todo = list(reversed(cls.__subclasses__()))
+    while todo:
+        cur = todo.pop()
+        if cur in seen:
+            continue
+        seen.add(cur)
+        yield cur
+        todo.extend(reversed(cur.__subclasses__()))

omlish/lite/runtime.py

@@ -0,0 +1,18 @@
+import inspect
+import sys
+
+from .cached import cached_nullary
+
+
+@cached_nullary
+def is_debugger_attached() -> bool:
+    return any(frame[1].endswith('pydevd.py') for frame in inspect.stack())
+
+
+REQUIRED_PYTHON_VERSION = (3, 8)
+
+
+def check_runtime_version() -> None:
+    if sys.version_info < REQUIRED_PYTHON_VERSION:
+        raise OSError(
+            f'Requires python {REQUIRED_PYTHON_VERSION}, got {sys.version_info} from {sys.executable}')  # noqa

omlish/lite/secrets.py

@@ -0,0 +1,19 @@
+import typing as ta
+
+
+class Secret:
+    _VALUE_ATTR = '__secret_value__'
+
+    def __init__(self, *, key: str | None, value: str) -> None:
+        super().__init__()
+        self._key = key
+        setattr(self, self._VALUE_ATTR, lambda: value)
+
+    def __repr__(self) -> str:
+        return f'Secret<{self._key or ""}>'
+
+    def __str__(self) -> ta.NoReturn:
+        raise TypeError
+
+    def reveal(self) -> str:
+        return getattr(self, self._VALUE_ATTR)()

omlish/lite/strings.py

@@ -0,0 +1,25 @@
+def camel_case(name: str) -> str:
+    return ''.join(map(str.capitalize, name.split('_')))  # noqa
+
+
+def snake_case(name: str) -> str:
+    uppers: list[int | None] = [i for i, c in enumerate(name) if c.isupper()]
+    return '_'.join([name[l:r].lower() for l, r in zip([None, *uppers], [*uppers, None])]).strip('_')
+
+
+def is_dunder(name: str) -> bool:
+    return (
+        name[:2] == name[-2:] == '__' and
+        name[2:3] != '_' and
+        name[-3:-2] != '_' and
+        len(name) > 4
+    )
+
+
+def is_sunder(name: str) -> bool:
+    return (
+        name[0] == name[-1] == '_' and
+        name[1:2] != '_' and
+        name[-2:-1] != '_' and
+        len(name) > 2
+    )

omlish/lite/subprocesses.py

@@ -0,0 +1,112 @@
+# ruff: noqa: UP006 UP007
+import logging
+import os
+import shlex
+import subprocess
+import sys
+import typing as ta
+
+from .logs import log
+from .runtime import is_debugger_attached
+
+
+##
+
+
+_SUBPROCESS_SHELL_WRAP_EXECS = False
+
+
+def subprocess_shell_wrap_exec(*args: str) -> ta.Tuple[str, ...]:
+    return ('sh', '-c', ' '.join(map(shlex.quote, args)))
+
+
+def subprocess_maybe_shell_wrap_exec(*args: str) -> ta.Tuple[str, ...]:
+    if _SUBPROCESS_SHELL_WRAP_EXECS or is_debugger_attached():
+        return subprocess_shell_wrap_exec(*args)
+    else:
+        return args
+
+
+def _prepare_subprocess_invocation(
+        *args: str,
+        env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
+        extra_env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
+        quiet: bool = False,
+        shell: bool = False,
+        **kwargs: ta.Any,
+) -> ta.Tuple[ta.Tuple[ta.Any, ...], ta.Dict[str, ta.Any]]:
+    log.debug(args)
+    if extra_env:
+        log.debug(extra_env)
+
+    if extra_env:
+        env = {**(env if env is not None else os.environ), **extra_env}
+
+    if quiet and 'stderr' not in kwargs:
+        if not log.isEnabledFor(logging.DEBUG):
+            kwargs['stderr'] = subprocess.DEVNULL
+
+    if not shell:
+        args = subprocess_maybe_shell_wrap_exec(*args)
+
+    return args, dict(
+        env=env,
+        shell=shell,
+        **kwargs,
+    )
+
+
+def subprocess_check_call(*args: str, stdout=sys.stderr, **kwargs: ta.Any) -> None:
+    args, kwargs = _prepare_subprocess_invocation(*args, stdout=stdout, **kwargs)
+    return subprocess.check_call(args, **kwargs)  # type: ignore
+
+
+def subprocess_check_output(*args: str, **kwargs: ta.Any) -> bytes:
+    args, kwargs = _prepare_subprocess_invocation(*args, **kwargs)
+    return subprocess.check_output(args, **kwargs)
+
+
+def subprocess_check_output_str(*args: str, **kwargs: ta.Any) -> str:
+    return subprocess_check_output(*args, **kwargs).decode().strip()
+
+
+##
+
+
+DEFAULT_SUBPROCESS_TRY_EXCEPTIONS: ta.Tuple[ta.Type[Exception], ...] = (
+    FileNotFoundError,
+    subprocess.CalledProcessError,
+)
+
+
+def subprocess_try_call(
+        *args: str,
+        try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
+        **kwargs: ta.Any,
+) -> bool:
+    try:
+        subprocess_check_call(*args, **kwargs)
+    except try_exceptions as e:  # noqa
+        if log.isEnabledFor(logging.DEBUG):
+            log.exception('command failed')
+        return False
+    else:
+        return True
+
+
+def subprocess_try_output(
+        *args: str,
+        try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
+        **kwargs: ta.Any,
+) -> ta.Optional[bytes]:
+    try:
+        return subprocess_check_output(*args, **kwargs)
+    except try_exceptions as e:  # noqa
+        if log.isEnabledFor(logging.DEBUG):
+            log.exception('command failed')
+        return None
+
+
+def subprocess_try_output_str(*args: str, **kwargs: ta.Any) -> ta.Optional[str]:
+    out = subprocess_try_output(*args, **kwargs)
+    return out.decode().strip() if out is not None else None

omlish/logs/configs.py

@@ -3,6 +3,7 @@ import logging
 import typing as ta
 
 from .filters import TidFilter
+from .formatters import JsonLogFormatter
 from .formatters import StandardLogFormatter
 
 
@@ -47,10 +48,22 @@ def build_log_format(parts: ta.Iterable[tuple[str, str]]) -> str:
     return ' '.join(v for k, v in parts)
 
 
-def configure_standard_logging(level: ta.Any = None) -> logging.Handler:
+def configure_standard_logging(
+        level: ta.Any = None,
+        *,
+        json: bool = False,
+) -> logging.Handler:
     handler = logging.StreamHandler()
-    handler.setFormatter(StandardLogFormatter(build_log_format(STANDARD_LOG_FORMAT_PARTS)))
+
+    formatter: logging.Formatter
+    if json:
+        formatter = JsonLogFormatter()
+    else:
+        formatter = StandardLogFormatter(build_log_format(STANDARD_LOG_FORMAT_PARTS))
+    handler.setFormatter(formatter)
+
     handler.addFilter(TidFilter())
+
     logging.root.addHandler(handler)
 
     if level is not None:

omlish/logs/formatters.py

@@ -5,8 +5,8 @@ import datetime
 import logging
 import typing as ta
 
-from .. import json
 from .. import term
+from ..formats import json
 
 
 class StandardLogFormatter(logging.Formatter):
@@ -67,5 +67,10 @@ class JsonLogFormatter(logging.Formatter):
     }
 
     def format(self, record: logging.LogRecord) -> str:
-        dct = {k: v for k, o in self.KEYS.items() for v in [getattr(record, k)] if not (o and v is None)}
+        dct = {
+            k: v
+            for k, o in self.KEYS.items()
+            for v in [getattr(record, k)]
+            if not (o and v is None)
+        }
         return json.dumps_compact(dct)

omlish/marshal/__init__.py

@@ -5,6 +5,15 @@ from .base import (  # noqa
     MarshalerFactory,
     UnmarshalerFactory,
 
+    MarshalerFactoryMatchClass,
+    UnmarshalerFactoryMatchClass,
+
+    TypeMapMarshalerFactory,
+    TypeMapUnmarshalerFactory,
+
+    TypeCacheMarshalerFactory,
+    TypeCacheUnmarshalerFactory,
+
     FuncMarshaler,
     FuncUnmarshaler,
 
@@ -18,6 +27,17 @@ from .base import (  # noqa
     SetType,
 )
 
+from .exceptions import (  # noqa
+    ForbiddenTypeError,
+    MarshalError,
+    UnhandledTypeError,
+)
+
+from .forbidden import (  # noqa
+    ForbiddenTypeMarshalerFactory,
+    ForbiddenTypeUnmarshalerFactory,
+)
+
 from .naming import (  # noqa
     Naming,
     translate_name,
@@ -50,3 +70,15 @@ from .standard import (  # noqa
     STANDARD_UNMARSHALER_FACTORIES,
     new_standard_unmarshaler_factory,
 )
+
+from .values import (  # noqa
+    Value,
+)
+
+
+##
+
+
+from ..lang.imports import _trigger_conditional_imports  # noqa
+
+_trigger_conditional_imports(__package__)

omlish/marshal/any.py

@@ -1,12 +1,12 @@
 import typing as ta
 
+from .. import reflect as rfl
 from .base import MarshalContext
 from .base import Marshaler
-from .base import MarshalerFactory
+from .base import TypeMapMarshalerFactory
+from .base import TypeMapUnmarshalerFactory
 from .base import UnmarshalContext
 from .base import Unmarshaler
-from .base import UnmarshalerFactory
-from .factories import TypeMapFactory
 from .values import Value
 
 
@@ -21,5 +21,5 @@ class AnyMarshalerUnmarshaler(Marshaler, Unmarshaler):
 
 ANY_MARSHALER_UNMARSHALER = AnyMarshalerUnmarshaler()
 
-ANY_MARSHALER_FACTORY: MarshalerFactory = TypeMapFactory({ta.Any: ANY_MARSHALER_UNMARSHALER})  # type: ignore
-ANY_UNMARSHALER_FACTORY: UnmarshalerFactory = TypeMapFactory({ta.Any: ANY_MARSHALER_UNMARSHALER})  # type: ignore
+ANY_MARSHALER_FACTORY = TypeMapMarshalerFactory({rfl.ANY: ANY_MARSHALER_UNMARSHALER})
+ANY_UNMARSHALER_FACTORY = TypeMapUnmarshalerFactory({rfl.ANY: ANY_MARSHALER_UNMARSHALER})