omlish 0.0.0.dev1__py3-none-any.whl

omlish/logs/_abc.py

@@ -0,0 +1,247 @@
+import types
+import typing as ta
+
+
+##
+
+
+Level: ta.TypeAlias = int
+
+
+ExceptionInfo: ta.TypeAlias = tuple[type[BaseException], BaseException, types.TracebackType]
+
+
+class LogRecord:
+    name: str
+    msg: str
+    args: tuple
+    levelname: str
+    levelno: Level
+    pathname: str
+    filename: str
+    module: str
+    exc_info: ExceptionInfo | None
+    exc_text: str | None
+    stack_info: str | None
+    lineno: int
+    funcName: str
+    created: float
+    msecs: float
+    relativeCreated: float
+    thread: int
+    threadName: str
+    processName: str
+    process: int
+
+
+##
+
+
+class Formatter(ta.Protocol):
+    default_time_format: ta.ClassVar[str]
+    default_msec_format: ta.ClassVar[str]
+
+    def formatTime(self, record: LogRecord, datefmt: str | None = None) -> str: ...
+
+    def formatException(self, ei: ExceptionInfo) -> str: ...
+
+    def usesTime(self) -> bool: ...
+
+    def formatMessage(self, record: LogRecord) -> str: ...
+
+    def formatStack(self, stack_info: str) -> str: ...
+
+    def format(self, record: LogRecord) -> str: ...
+
+
+class BufferingFormatter(ta.Protocol):
+    def formatHeader(self, records: ta.Sequence[LogRecord]) -> str: ...
+
+    def formatFooter(self, records: ta.Sequence[LogRecord]) -> str: ...
+
+    def format(self, records: ta.Sequence[LogRecord]) -> str: ...
+
+
+##
+
+
+class Filter(ta.Protocol):
+    def filter(self, record: LogRecord) -> bool: ...
+
+
+class Filterer(ta.Protocol):
+    def addFilter(self, filter: Filter) -> None: ...
+
+    def removeFilter(self, filter: Filter) -> None: ...
+
+    def filter(self, record: LogRecord) -> bool: ...
+
+
+##
+
+
+class Handler(ta.Protocol):
+    level: Level
+
+    def get_name(self) -> str: ...
+
+    def set_name(self, name: str) -> None: ...
+
+    name: str
+
+    def createLock(self) -> None: ...
+
+    def acquire(self) -> None: ...
+
+    def release(self) -> None: ...
+
+    def setLevel(self, level: Level) -> None: ...
+
+    def format(self, record: LogRecord) -> str: ...
+
+    def emit(self, record: LogRecord) -> None: ...
+
+    def handle(self, record: LogRecord) -> bool: ...
+
+    def setFormatter(self, fmt: Formatter) -> None: ...
+
+    def flush(self) -> None: ...
+
+    def close(self) -> None: ...
+
+    def handleError(self, record: LogRecord) -> None: ...
+
+
+class Stream(ta.Protocol):
+    def write(self, s: str) -> None: ...
+
+    def flush(self) -> None: ...  # OPTIONAL METHOD
+
+    def close(self) -> None: ...  # OPTIONAL METHOD
+
+
+class StreamHandler(Handler):
+    terminator: ta.ClassVar[str]
+
+    stream: Stream
+
+    def flush(self) -> None: ...
+
+    def emit(self, record: LogRecord) -> None: ...
+
+    def setStream(self, stream: Stream) -> None: ...
+
+
+##
+
+
+class Manager(ta.Protocol):
+    root: 'Logger'
+
+    disable: Level
+
+    def getLogger(self, name: str) -> 'Logger': ...
+
+    def setLoggerClass(self, klass: type['Logger']) -> None: ...
+
+    def setLogRecordFactory(self, factory: ta.Callable[..., LogRecord]) -> None: ...  # UNREFERENCED?
+
+
+##
+
+
+Caller: ta.TypeAlias = tuple[
+    str,  # filename
+    int,  # lineno
+    str,  # name
+    str,  # formatted
+]
+
+
+class Logger(Filterer, ta.Protocol):
+    name: str
+    level: Level
+    parent: ta.Optional['Logger']
+    propagate: bool
+    handlers: list[Handler]
+    disabled: bool
+
+    manager: Manager
+
+    def setLevel(self, level: Level) -> None: ...
+
+    def debug(self, msg: str, *args: ta.Any, **kwargs: ta.Any) -> None: ...
+    def info(self, msg: str, *args: ta.Any, **kwargs: ta.Any) -> None: ...
+    def warning(self, msg: str, *args: ta.Any, **kwargs: ta.Any) -> None: ...
+    def warn(self, msg: str, *args: ta.Any, **kwargs: ta.Any) -> None: ...
+    def error(self, msg: str, *args: ta.Any, **kwargs: ta.Any) -> None: ...
+    def exception(self, msg: str, *args: ta.Any, **kwargs: ta.Any) -> None: ...
+    def critical(self, msg: str, *args: ta.Any, **kwargs: ta.Any) -> None: ...
+    def fatal(self, msg: str, *args: ta.Any, **kwargs: ta.Any) -> None: ...
+
+    def log(self, level: Level, msg: str, *args: ta.Any, **kwargs: ta.Any) -> None: ...
+
+    def findCaller(self, stack_info: bool = False, stacklevel: int = 1) -> Caller: ...  #
+
+    def makeRecord(
+        self,
+        name,
+        level,
+        fn,
+        lno,
+        msg,
+        args,
+        exc_info,
+        func: str | None = None,
+        extra: ta.Mapping[str, ta.Any] | None = None,
+        sinfo: str | None = None,
+    ) -> LogRecord: ...
+
+    def handle(self, record: LogRecord) -> None: ...
+
+    def addHandler(self, hdlr: Handler) -> None: ...
+
+    def removeHandler(self, hdlr: Handler) -> None: ...
+
+    def hasHandlers(self) -> bool: ...
+
+    def callHandlers(self, record: LogRecord) -> None: ...
+
+    def getEffectiveLevel(self) -> Level: ...
+
+    def isEnabledFor(self, level: Level) -> bool: ...
+
+    def getChild(self, suffix: str) -> 'Logger': ...
+
+
+##
+
+
+class LoggerAdapter(ta.Protocol):
+    logger: Logger
+    extra: ta.Mapping[str, ta.Any]
+
+    def process(self, msg: str, kwargs: dict[str, ta.Any]) -> tuple[str, dict[str, ta.Any]]: ...
+
+    def debug(self, msg: str, *args: ta.Any, **kwargs: ta.Any) -> None: ...
+    def info(self, msg: str, *args: ta.Any, **kwargs: ta.Any) -> None: ...
+    def warning(self, msg: str, *args: ta.Any, **kwargs: ta.Any) -> None: ...
+    def warn(self, msg: str, *args: ta.Any, **kwargs: ta.Any) -> None: ...
+    def error(self, msg: str, *args: ta.Any, **kwargs: ta.Any) -> None: ...
+    def exception(self, msg: str, *args: ta.Any, **kwargs: ta.Any) -> None: ...
+    def critical(self, msg: str, *args: ta.Any, **kwargs: ta.Any) -> None: ...
+    def fatal(self, msg: str, *args: ta.Any, **kwargs: ta.Any) -> None: ...
+
+    def log(self, level: Level, msg: str, *args: ta.Any, **kwargs: ta.Any) -> None: ...
+
+    def isEnabledFor(self, level: Level) -> bool: ...
+
+    def setLevel(self, level: Level) -> None: ...
+
+    def getEffectiveLevel(self) -> Level: ...
+
+    def hasHandlers(self) -> bool: ...
+
+    manager: Manager
+
+    name: str

omlish/logs/configs.py

@@ -0,0 +1,62 @@
+import dataclasses as dc
+import logging
+import typing as ta
+
+from .filters import TidFilter
+from .formatters import StandardLogFormatter
+
+
+NOISY_LOGGERS: set[str] = {
+    'boto3.resources.action',
+    'datadog.dogstatsd',
+    'elasticsearch',
+    'kazoo.client',
+    'requests.packages.urllib3.connectionpool',
+}
+
+
+@dc.dataclass()
+class DictConfig:
+    version: int = 1
+    incremental: bool = False
+    disable_existing_loggers: bool = False
+    filters: dict[str, 'FilterConfig'] = dc.field(default_factory=dict)
+    formatters: dict[str, 'FormatterConfig'] = dc.field(default_factory=dict)
+    handlers: dict[str, 'HandlerConfig'] = dc.field(default_factory=dict)
+    loggers: dict[str, 'LoggerConfig'] = dc.field(default_factory=dict)
+    root: ta.Optional['LoggerConfig'] = None
+
+
+FilterConfig = dict[str, ta.Any]
+FormatterConfig = dict[str, ta.Any]
+HandlerConfig = dict[str, ta.Any]
+LoggerConfig = dict[str, ta.Any]
+
+STANDARD_LOG_FORMAT_PARTS = [
+    ('asctime', '%(asctime)-15s'),
+    ('process', 'pid=%(process)-6s'),
+    ('thread', 'tid=%(thread)-16s'),
+    ('levelname', '%(levelname)-8s'),
+    ('name', '%(name)s'),
+    ('separator', '::'),
+    ('message', '%(message)s'),
+]
+
+
+def build_log_format(parts: ta.Iterable[tuple[str, str]]) -> str:
+    return ' '.join(v for k, v in parts)
+
+
+def configure_standard_logging(level: ta.Any = None) -> logging.Handler:
+    handler = logging.StreamHandler()
+    handler.setFormatter(StandardLogFormatter(build_log_format(STANDARD_LOG_FORMAT_PARTS)))
+    handler.addFilter(TidFilter())
+    logging.root.addHandler(handler)
+
+    if level is not None:
+        logging.root.setLevel(level)
+
+    for noisy_logger in NOISY_LOGGERS:
+        logging.getLogger(noisy_logger).setLevel(logging.WARNING)
+
+    return handler

omlish/logs/filters.py

@@ -0,0 +1,9 @@
+import logging
+import threading
+
+
+class TidFilter(logging.Filter):
+
+    def filter(self, record):
+        record.tid = threading.get_native_id()
+        return True

omlish/logs/formatters.py

@@ -0,0 +1,67 @@
+import datetime
+import logging
+
+from .. import json
+from .. import term
+
+
+class StandardLogFormatter(logging.Formatter):
+
+    converter = datetime.datetime.fromtimestamp  # type: ignore
+
+    def formatTime(self, record, datefmt=None):
+        ct = self.converter(record.created)  # type: ignore
+        if datefmt:
+            return ct.strftime(datefmt)  # noqa
+        else:
+            t = ct.strftime("%Y-%m-%d %H:%M:%S")  # noqa
+            return '%s.%03d' % (t, record.msecs)
+
+
+class ColorLogFormatter(StandardLogFormatter):
+
+    LEVEL_COLORS = {
+        logging.WARNING: term.SGRs.FG.BRIGHT_YELLOW,
+        logging.ERROR: term.SGRs.FG.BRIGHT_RED,
+        logging.CRITICAL: term.SGRs.FG.BRIGHT_RED,
+    }
+
+    def formatMessage(self, record):
+        buf = super().formatMessage(record)
+        try:
+            c = self.LEVEL_COLORS[record.levelno]
+        except KeyError:
+            pass
+        else:
+            buf = term.SGR(c) + buf + term.SGR(term.SGRs.RESET)
+        return buf
+
+
+class JsonLogFormatter(logging.Formatter):
+
+    KEYS = {
+        'name': False,
+        'msg': False,
+        'args': False,
+        'levelname': False,
+        'levelno': False,
+        'pathname': False,
+        'filename': False,
+        'module': False,
+        'exc_info': True,
+        'exc_text': True,
+        'stack_info': True,
+        'lineno': False,
+        'funcName': False,
+        'created': False,
+        'msecs': False,
+        'relativeCreated': False,
+        'thread': False,
+        'threadName': False,
+        'processName': False,
+        'process': False,
+    }
+
+    def format(self, record: logging.LogRecord) -> str:
+        dct = {k: v for k, o in self.KEYS.items() for v in [getattr(record, k)] if not (o and v is None)}
+        return json.dumps_compact(dct)

omlish/logs/utils.py

@@ -0,0 +1,20 @@
+import functools
+import logging
+
+
+log = logging.getLogger(__name__)
+
+
+def error_logging(log=log):
+    def outer(fn):
+        @functools.wraps(fn)
+        def inner(*args, **kwargs):
+            try:
+                return fn(*args, **kwargs)
+            except Exception:
+                log.exception(f'Error in {fn!r}')
+                raise
+
+        return inner
+
+    return outer

omlish/marshal/__init__.py

@@ -0,0 +1,52 @@
+from .base import (  # noqa
+    Marshaler,
+    Unmarshaler,
+
+    MarshalerFactory,
+    UnmarshalerFactory,
+
+    FuncMarshaler,
+    FuncUnmarshaler,
+
+    BaseContext,
+    MarshalContext,
+    UnmarshalContext,
+
+    RecursiveMarshalerFactory,
+    RecursiveUnmarshalerFactory,
+
+    SetType,
+)
+
+from .naming import (  # noqa
+    Naming,
+    translate_name,
+)
+
+from .global_ import (  # noqa
+    marshal,
+    unmarshal,
+)
+
+from .objects import (  # noqa
+    FieldMetadata,
+    ObjectMetadata,
+)
+
+from .polymorphism import (  # noqa
+    Impl,
+    Polymorphism,
+    polymorphism_from_subclasses,
+)
+
+from .registries import (  # noqa
+    Registry,
+)
+
+from .standard import (  # noqa
+    STANDARD_MARSHALER_FACTORIES,
+    new_standard_marshaler_factory,
+
+    STANDARD_UNMARSHALER_FACTORIES,
+    new_standard_unmarshaler_factory,
+)

omlish/marshal/any.py

@@ -0,0 +1,25 @@
+import typing as ta
+
+from .base import MarshalContext
+from .base import Marshaler
+from .base import MarshalerFactory
+from .base import UnmarshalContext
+from .base import Unmarshaler
+from .base import UnmarshalerFactory
+from .factories import TypeMapFactory
+from .values import Value
+
+
+class AnyMarshalerUnmarshaler(Marshaler, Unmarshaler):
+
+    def marshal(self, ctx: MarshalContext, o: ta.Any) -> Value:
+        return ctx.make(type(o)).marshal(ctx, o)
+
+    def unmarshal(self, ctx: UnmarshalContext, v: Value) -> ta.Any:
+        return v
+
+
+ANY_MARSHALER_UNMARSHALER = AnyMarshalerUnmarshaler()
+
+ANY_MARSHALER_FACTORY: MarshalerFactory = TypeMapFactory({ta.Any: ANY_MARSHALER_UNMARSHALER})  # type: ignore
+ANY_UNMARSHALER_FACTORY: UnmarshalerFactory = TypeMapFactory({ta.Any: ANY_MARSHALER_UNMARSHALER})  # type: ignore

omlish/marshal/base.py

@@ -0,0 +1,201 @@
+"""
+TODO:
+ - mappings
+ - redacted
+ - strongly typed Composite/Cached Marshaler/Unmarshaler factories - footgun
+ - streaming? Start/EndObject, etc..
+
+https://github.com/python-attrs/cattrs:
+ *
+
+Jackson:
+ - USE_ANNOTATIONS
+ - AUTO_DETECT_CREATORS
+ - AUTO_DETECT_FIELDS
+ - AUTO_DETECT_GETTERS
+ - AUTO_DETECT_IS_GETTERS
+ - AUTO_DETECT_SETTERS
+ - REQUIRE_SETTERS_FOR_GETTERS
+ - USE_GETTERS_AS_SETTERS
+ - INFER_CREATOR_FROM_CONSTRUCTOR_PROPERTIES
+ - INFER_PROPERTY_MUTATORS
+ - ALLOW_FINAL_FIELDS_AS_MUTATORS
+ - ALLOW_VOID_VALUED_PROPERTIES
+ - CAN_OVERRIDE_ACCESS_MODIFIERS
+ - OVERRIDE_PUBLIC_ACCESS_MODIFIERS
+ - SORT_PROPERTIES_ALPHABETICALLY
+ - USE_WRAPPER_NAME_AS_PROPERTY_NAME
+ - ACCEPT_CASE_INSENSITIVE_ENUMS
+ - ACCEPT_CASE_INSENSITIVE_PROPERTIES
+ - ACCEPT_CASE_INSENSITIVE_VALUES
+ - ALLOW_EXPLICIT_PROPERTY_RENAMING
+ - USE_STD_BEAN_NAMING
+ - ALLOW_COERCION_OF_SCALARS
+ - DEFAULT_VIEW_INCLUSION
+ - IGNORE_DUPLICATE_MODULE_REGISTRATIONS
+ - IGNORE_MERGE_FOR_UNMERGEABLE
+ - USE_BASE_TYPE_AS_DEFAULT_IMPL
+ - USE_STATIC_TYPING
+ - BLOCK_UNSAFE_POLYMORPHIC_BASE_TYPES
+
+https://github.com/yukinarit/pyserde
+ - datatypes
+  - typing.Union
+  - typing.NewType for primitive types
+  - typing.Any
+  - typing.Literal
+  - typing.Generic
+  - typing.ClassVar
+ - dataclasses.InitVar
+ - Enum and IntEnum
+ - pathlib.Path
+ - decimal.Decimal
+ - uuid.UUID
+ - datetime.date, datetime.time, datetime.datetime
+ - ipaddress
+ - numpy types
+ - Class Attributes
+ - Field Attributes
+ - Decorators
+ - Type Check
+ - Union Representation
+ - Forward reference
+ - PEP563 Postponed Evaluation of Annotations
+ - PEP585 Type Hinting Generics In Standard Collections
+ - PEP604 Allow writing union types as X | Y
+ - PEP681 Data Class Transform
+ - Case Conversion
+ - Rename
+ - Alias
+ - Skip (de)serialization (skip, skip_if, skip_if_false, skip_if_default)
+ - Custom field (de)serializer
+ - Custom class (de)serializer
+ - Custom global (de)serializer
+ - Flatten
+"""
+import abc
+import dataclasses as dc
+import typing as ta
+
+from .. import check
+from .. import collections as col
+from .. import lang
+from .. import reflect as rfl
+from .exceptions import UnhandledTypeException
+from .factories import Factory
+from .factories import RecursiveTypeFactory
+from .registries import Registry
+from .registries import RegistryItem
+from .utils import _Proxy
+from .values import Value
+
+
+##
+
+
+class Marshaler(lang.Abstract):
+    @abc.abstractmethod
+    def marshal(self, ctx: 'MarshalContext', o: ta.Any) -> Value:
+        raise NotImplementedError
+
+
+class Unmarshaler(lang.Abstract):
+    @abc.abstractmethod
+    def unmarshal(self, ctx: 'UnmarshalContext', v: Value) -> ta.Any:
+        raise NotImplementedError
+
+
+MarshalerFactory = Factory[Marshaler, 'MarshalContext', rfl.Type]
+UnmarshalerFactory = Factory[Unmarshaler, 'UnmarshalContext', rfl.Type]
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class FuncMarshaler(Marshaler, lang.Final):
+    fn: ta.Callable[['MarshalContext', ta.Any], Value]
+
+    def marshal(self, ctx: 'MarshalContext', o: ta.Any) -> Value:
+        return self.fn(ctx, o)
+
+
+@dc.dataclass(frozen=True)
+class FuncUnmarshaler(Unmarshaler, lang.Final):
+    fn: ta.Callable[['UnmarshalContext', Value], ta.Any]
+
+    def unmarshal(self, ctx: 'UnmarshalContext', v: Value) -> ta.Any:
+        return self.fn(ctx, v)
+
+
+##
+
+
+class Option:
+    pass
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class BaseContext(lang.Abstract):
+    registry: Registry
+    options: col.TypeMap[Option] = col.TypeMap()
+
+
+@dc.dataclass(frozen=True)
+class MarshalContext(BaseContext, lang.Final):
+    factory: ta.Optional[MarshalerFactory] = None
+
+    def make(self, o: ta.Any) -> Marshaler:
+        rty = rfl.type_(o)
+        if (m := check.not_none(self.factory)(self, rty)) is not None:  # noqa
+            return m
+        raise UnhandledTypeException(rty)
+
+
+@dc.dataclass(frozen=True)
+class UnmarshalContext(BaseContext, lang.Final):
+    factory: ta.Optional[UnmarshalerFactory] = None
+
+    def make(self, o: ta.Any) -> Unmarshaler:
+        rty = rfl.type_(o)
+        if (m := check.not_none(self.factory)(self, rty)) is not None:  # noqa
+            return m
+        raise UnhandledTypeException(rty)
+
+
+##
+
+
+class _ProxyMarshaler(_Proxy[Marshaler], Marshaler):
+    def marshal(self, ctx: MarshalContext, o: ta.Any) -> Value:
+        return self._obj.marshal(ctx, o)
+
+
+class RecursiveMarshalerFactory(RecursiveTypeFactory[Marshaler, MarshalContext], lang.Final):
+    def __init__(self, f: MarshalerFactory) -> None:
+        super().__init__(f, _ProxyMarshaler._new)  # noqa
+
+
+class _ProxyUnmarshaler(_Proxy[Unmarshaler], Unmarshaler):
+    def unmarshal(self, ctx: UnmarshalContext, v: Value) -> ta.Any:
+        return self._obj.unmarshal(ctx, v)
+
+
+class RecursiveUnmarshalerFactory(RecursiveTypeFactory[Unmarshaler, UnmarshalContext], lang.Final):
+    def __init__(self, f: UnmarshalerFactory) -> None:
+        super().__init__(f, _ProxyUnmarshaler._new)  # noqa
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class SetType(RegistryItem, lang.Final):
+    marshaler: ta.Optional[Marshaler] = None
+    marshaler_factory: ta.Optional[MarshalerFactory] = None
+
+    unmarshaler: ta.Optional[Unmarshaler] = None
+    unmarshaler_factory: ta.Optional[UnmarshalerFactory] = None

omlish/marshal/base64.py

@@ -0,0 +1,25 @@
+import base64
+
+from .. import check
+from .base import MarshalContext
+from .base import Marshaler
+from .base import MarshalerFactory
+from .base import UnmarshalContext
+from .base import Unmarshaler
+from .base import UnmarshalerFactory
+from .factories import TypeMapFactory
+from .values import Value
+
+
+class Base64MarshalerUnmarshaler(Marshaler, Unmarshaler):
+    def marshal(self, ctx: MarshalContext, o: bytes) -> str:
+        return base64.b64encode(o).decode()
+
+    def unmarshal(self, ctx: UnmarshalContext, v: Value) -> bytes:
+        return base64.b64decode(check.isinstance(v, str).encode())
+
+
+BASE64_MARSHALER_UNMARSHALER = Base64MarshalerUnmarshaler()
+
+BASE64_MARSHALER_FACTORY: MarshalerFactory = TypeMapFactory({bytes: BASE64_MARSHALER_UNMARSHALER})
+BASE64_UNMARSHALER_FACTORY: UnmarshalerFactory = TypeMapFactory({bytes: BASE64_MARSHALER_UNMARSHALER})