omlish 0.0.0.dev133__py3-none-any.whl → 0.0.0.dev177__py3-none-any.whl

omlish/codecs/base.py

@@ -0,0 +1,106 @@
+"""
+TODO:
+ - bytes-like - bytearray, memoryview
+"""
+import abc
+import typing as ta
+
+from omlish import check
+from omlish import dataclasses as dc
+from omlish import lang
+from omlish import reflect as rfl
+from omlish.funcs import pairs as fps
+
+
+I = ta.TypeVar('I')
+O = ta.TypeVar('O')
+
+
+##
+
+
+class EagerCodec(lang.Abstract, ta.Generic[I, O]):
+    @abc.abstractmethod
+    def encode(self, i: I) -> O:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def decode(self, o: O) -> I:
+        raise NotImplementedError
+
+    def as_pair(self) -> fps.FnPair[I, O]:
+        return fps.of(self.encode, self.decode)
+
+
+class IncrementalCodec(lang.Abstract, ta.Generic[I, O]):
+    @abc.abstractmethod
+    def encode_incremental(self) -> ta.Generator[O | None, I, None]:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def decode_incremental(self) -> ta.Generator[I | None, O, None]:
+        raise NotImplementedError
+
+
+class ComboCodec(  # noqa
+    EagerCodec[I, O],
+    IncrementalCodec[I, O],
+    lang.Abstract,
+    ta.Generic[I, O],
+):
+    pass
+
+
+##
+
+
+def check_codec_name(s: str) -> str:
+    check.non_empty_str(s)
+    check.not_in('_', s)
+    check.equal(s.strip(), s)
+    return s
+
+
+##
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+class Codec:
+    name: str = dc.xfield(coerce=check_codec_name)
+    aliases: ta.Collection[str] | None = dc.xfield(
+        default=None,
+        coerce=lang.opt_fn(lambda s: [check_codec_name(a) for a in s]),  # type: ignore
+    )
+
+    input: rfl.Type = dc.xfield(coerce=rfl.type_)
+    output: rfl.Type = dc.xfield(coerce=rfl.type_)
+
+    options: type | None = None
+
+    new: ta.Callable[..., EagerCodec]
+    new_incremental: ta.Callable[..., IncrementalCodec] | None = None
+
+
+##
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+class LazyLoadedCodec:
+    mod_name: str
+    attr_name: str
+    name: str
+    aliases: ta.Collection[str] | None = None
+
+    @classmethod
+    def new(
+            cls,
+            mod_name: str,
+            attr_name: str,
+            codec: Codec,
+    ) -> 'LazyLoadedCodec':
+        return cls(
+            mod_name=mod_name,
+            attr_name=attr_name,
+            name=codec.name,
+            aliases=codec.aliases,
+        )

omlish/codecs/bytes.py

@@ -0,0 +1,119 @@
+"""
+TODO:
+ - options / kwargs
+"""
+import base64
+import binascii
+import typing as ta
+
+from .. import check
+from .base import Codec
+from .funcs import FnPairEagerCodec
+from .standard import STANDARD_CODECS
+
+
+##
+
+
+class BytesCodec(Codec):
+    pass
+
+
+def make_bytes_encoding_codec(
+        name: str,
+        aliases: ta.Collection[str] | None,
+        encode: ta.Callable[[bytes], bytes],
+        decode: ta.Callable[[bytes], bytes],
+        *,
+        append_to: ta.MutableSequence[Codec] | None = None,
+) -> BytesCodec:
+    codec = BytesCodec(
+        name=name,
+        aliases=check.not_isinstance(aliases, str),
+
+        input=bytes,
+        output=bytes,
+
+        new=lambda: FnPairEagerCodec.of(encode, decode),
+    )
+
+    if append_to is not None:
+        append_to.append(codec)
+
+    return codec
+
+
+##
+
+
+ASCII85 = make_bytes_encoding_codec(
+    'ascii85',
+    ['a85'],
+    base64.a85encode,
+    base64.a85decode,
+    append_to=STANDARD_CODECS,
+)
+
+BASE16 = make_bytes_encoding_codec(
+    'base16',
+    ['b16'],
+    base64.b16encode,
+    base64.b16decode,
+    append_to=STANDARD_CODECS,
+)
+
+BASE32 = make_bytes_encoding_codec(
+    'base32',
+    ['b32'],
+    base64.b32encode,
+    base64.b32decode,
+    append_to=STANDARD_CODECS,
+)
+
+BASE64 = make_bytes_encoding_codec(
+    'base64',
+    ['b64'],
+    base64.b64encode,
+    base64.b64decode,
+    append_to=STANDARD_CODECS,
+)
+
+BASE85 = make_bytes_encoding_codec(
+    'base85',
+    ['b85'],
+    base64.b85encode,
+    base64.b85decode,
+    append_to=STANDARD_CODECS,
+)
+
+BASE32_HEX = make_bytes_encoding_codec(
+    'base32-hex',
+    ['b32-hex'],
+    base64.b32hexencode,
+    base64.b32hexdecode,
+    append_to=STANDARD_CODECS,
+)
+
+BASE64_HEX = make_bytes_encoding_codec(
+    'base64-hex',
+    ['b64-hex'],
+    base64.standard_b64encode,
+    base64.standard_b64decode,
+    append_to=STANDARD_CODECS,
+)
+
+BASE64_URLSAFE = make_bytes_encoding_codec(
+    'base64-urlsafe',
+    ['b64-urlsafe'],
+    base64.urlsafe_b64encode,
+    base64.urlsafe_b64decode,
+    append_to=STANDARD_CODECS,
+)
+
+HEX = make_bytes_encoding_codec(
+    'hex',
+    [],
+    binascii.b2a_hex,
+    binascii.a2b_hex,
+    append_to=STANDARD_CODECS,
+)

omlish/codecs/chain.py

@@ -0,0 +1,23 @@
+import dataclasses as dc
+import typing as ta
+
+from .base import EagerCodec
+
+
+@dc.dataclass(frozen=True)
+class ChainEagerCodec(EagerCodec[ta.Any, ta.Any]):
+    codecs: ta.Sequence[EagerCodec]
+
+    def encode(self, v: ta.Any) -> ta.Any:
+        for c in self.codecs:
+            v = c.encode(v)
+        return v
+
+    def decode(self, v: ta.Any) -> ta.Any:
+        for c in reversed(self.codecs):
+            v = c.decode(v)
+        return v
+
+
+def chain(*codecs: EagerCodec) -> ChainEagerCodec:
+    return ChainEagerCodec(codecs)

omlish/codecs/funcs.py

@@ -0,0 +1,39 @@
+import dataclasses as dc
+import typing as ta
+
+from ..funcs import pairs as fps
+from .base import EagerCodec
+
+
+I = ta.TypeVar('I')
+O = ta.TypeVar('O')
+
+
+@dc.dataclass(frozen=True)
+class FnPairEagerCodec(EagerCodec[I, O]):
+    fp: fps.FnPair[I, O]
+
+    def encode(self, i: I) -> O:
+        return self.fp.forward(i)
+
+    def decode(self, o: O) -> I:
+        return self.fp.backward(o)
+
+    @classmethod
+    def of(
+            cls,
+            encode: ta.Callable[[I], O],
+            decode: ta.Callable[[O], I],
+    ) -> 'FnPairEagerCodec[I, O]':
+        return cls(fps.of(encode, decode))
+
+
+def of_pair(fp: fps.FnPair[I, O]) -> FnPairEagerCodec[I, O]:
+    return FnPairEagerCodec(fp)
+
+
+def of(
+        encode: ta.Callable[[I], O],
+        decode: ta.Callable[[O], I],
+) -> FnPairEagerCodec[I, O]:
+    return FnPairEagerCodec(fps.of(encode, decode))

omlish/codecs/registry.py

@@ -0,0 +1,139 @@
+import contextlib
+import importlib
+import threading
+import typing as ta
+
+from .. import cached
+from .. import check
+from .. import lang
+from .base import Codec
+from .base import LazyLoadedCodec
+from .standard import STANDARD_CODECS
+
+
+if ta.TYPE_CHECKING:
+    from ..manifests import load as manifest_load
+else:
+    manifest_load = lang.proxy_import('..manifests.load', __package__)
+
+
+##
+
+
+class CodecRegistry:
+    def __init__(
+            self,
+            *,
+            late_load_callbacks: ta.Iterable[ta.Callable[['CodecRegistry'], None]] | None = None,
+    ) -> None:
+        super().__init__()
+
+        self._late_load_callbacks = late_load_callbacks
+
+        self._lock = threading.RLock()
+        self._by_name: dict[str, Codec | LazyLoadedCodec] = {}
+        self._names_by_alias: dict[str, str] = {}
+        self._names_by_cls: dict[type, list[str]] = {}
+
+    def _late_load(self) -> None:
+        if self._late_load_callbacks:
+            for cb in self._late_load_callbacks:
+                cb(self)
+            self._late_load_callbacks = None
+
+    @contextlib.contextmanager
+    def _lock_and_load(self) -> ta.Iterator[None]:
+        with self._lock:
+            self._late_load()
+            yield
+
+    def _post_load(self, codec: Codec) -> None:
+        for t in type(codec).__mro__:
+            if t is not object:
+                self._names_by_cls.setdefault(t, []).append(codec.name)
+
+    def register(self, *codecs: Codec | LazyLoadedCodec) -> ta.Self:
+        with self._lock:
+            for codec in codecs:
+                for n in {codec.name, *(codec.aliases or [])}:
+                    if n in self._names_by_alias:
+                        raise KeyError(n)
+
+            for codec in codecs:
+                self._by_name[codec.name] = codec
+                for n in {codec.name, *(codec.aliases or [])}:
+                    self._names_by_alias[n] = codec.name
+                if isinstance(codec, Codec):
+                    self._post_load(codec)
+
+        return self
+
+    def lookup(self, name_or_alias: str) -> Codec:
+        with self._lock_and_load():
+            name = self._names_by_alias[name_or_alias.replace('_', '-')]
+            codec_or_lazy = self._by_name[name]
+
+            if isinstance(codec_or_lazy, LazyLoadedCodec):
+                mod = importlib.import_module(codec_or_lazy.mod_name)
+                codec = check.isinstance(getattr(mod, codec_or_lazy.attr_name), Codec)
+                self._by_name[name] = codec
+                self._post_load(codec)
+            else:
+                codec = check.isinstance(codec_or_lazy, Codec)
+
+            return codec
+
+    def lookup_type(self, cls: type) -> list[Codec]:
+        with self._lock_and_load():
+            return [self.lookup(n) for n in self._names_by_cls.get(cls, [])]
+
+    def all(self) -> frozenset[str]:
+        with self._lock_and_load():
+            return frozenset(self._by_name)
+
+
+##
+
+
+def _install_standard_codecs(registry: CodecRegistry) -> None:
+    registry.register(*STANDARD_CODECS)
+
+
+##
+
+
+@cached.function
+def _build_manifest_lazy_loaded_codecs() -> ta.Sequence[LazyLoadedCodec]:
+    ldr = manifest_load.MANIFEST_LOADER
+    pkgs = {__package__.split('.')[0], *ldr.discover()}
+    mns = ldr.load(*pkgs, only=[LazyLoadedCodec])
+    return [m.value for m in mns]
+
+
+def _install_manifest_lazy_loaded_codecs(registry: CodecRegistry) -> None:
+    registry.register(*_build_manifest_lazy_loaded_codecs())
+
+
+##
+
+
+REGISTRY = CodecRegistry(
+    late_load_callbacks=[
+        _install_standard_codecs,
+        _install_manifest_lazy_loaded_codecs,
+    ],
+)
+
+register = REGISTRY.register
+lookup = REGISTRY.lookup
+
+
+##
+
+
+def encode(i: ta.Any, name: str, **kwargs: ta.Any) -> ta.Any:
+    return lookup(name).new(**kwargs).encode(i)
+
+
+def decode(o: ta.Any, name: str, **kwargs: ta.Any) -> ta.Any:
+    return lookup(name).new(**kwargs).decode(o)

omlish/codecs/standard.py

@@ -0,0 +1,4 @@
+from .base import Codec
+
+
+STANDARD_CODECS: list[Codec] = []

omlish/codecs/text.py

@@ -0,0 +1,217 @@
+import codecs
+import dataclasses as dc
+import functools
+import typing as ta
+
+from omlish import check
+
+from .base import Codec
+from .base import ComboCodec
+from .standard import STANDARD_CODECS
+
+
+##
+
+
+TextEncodingErrors: ta.TypeAlias = ta.Literal[
+    # Raise UnicodeError (or a subclass), this is the default. Implemented in strict_errors().
+    'strict',
+
+    # Ignore the malformed data and continue without further notice. Implemented in ignore_errors().
+    'ignore',
+
+    # Replace with a replacement marker. On encoding, use ? (ASCII character). On decoding, use � (U+FFFD, the official
+    # REPLACEMENT CHARACTER). Implemented in replace_errors().
+    'replace',
+
+    # Replace with backslashed escape sequences. On encoding, use hexadecimal form of Unicode code point with formats
+    # \xhh \uxxxx \Uxxxxxxxx. On decoding, use hexadecimal form of byte value with format \xhh. Implemented in
+    # backslashreplace_errors().
+    'backslashreplace',
+
+    # On decoding, replace byte with individual surrogate code ranging from U+DC80 to U+DCFF. This code will then be
+    # turned back into the same byte when the 'surrogateescape' error handler is used when encoding the data. (See PEP
+    # 383 for more.)
+    'surrogateescape',
+
+    ##
+    # The following error handlers are only applicable to encoding (within text encodings):
+
+    # Replace with XML/HTML numeric character reference, which is a decimal form of Unicode code point with format
+    # &#num;. Implemented in xmlcharrefreplace_errors().
+    'xmlcharrefreplace',
+
+    # Replace with \N{...} escape sequences, what appears in the braces is the Name property from Unicode Character
+    # Database. Implemented in namereplace_errors().
+    'namereplace',
+
+    ##
+    # In addition, the following error handler is specific to the given codecs: utf-8, utf-16, utf-32, utf-16-be,
+    # utf-16-le, utf-32-be, utf-32-le
+
+    # Allow encoding and decoding surrogate code point (U+D800 - U+DFFF) as normal code point. Otherwise these codecs
+    # treat the presence of surrogate code point in str as an error.
+    'surrogatepass',
+]
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+class TextEncodingOptions:
+    errors: TextEncodingErrors = 'strict'
+
+
+##
+
+
+class TextEncodingComboCodec(ComboCodec[str, bytes]):
+    def __init__(
+            self,
+            info: codecs.CodecInfo,
+            options: TextEncodingOptions = TextEncodingOptions(),
+    ) -> None:
+        super().__init__()
+        self._info = check.isinstance(info, codecs.CodecInfo)
+        self._opts = check.isinstance(options, TextEncodingOptions)
+
+    @classmethod
+    def lookup(
+            cls,
+            name: str,
+            options: TextEncodingOptions = TextEncodingOptions(),
+    ) -> 'TextEncodingComboCodec':
+        return cls(codecs.lookup(name), options)
+
+    def encode(self, i: str) -> bytes:
+        o, _ = self._info.encode(i, self._opts.errors)
+        return o
+
+    def decode(self, o: bytes) -> str:
+        i, _ = self._info.decode(o, self._opts.errors)
+        return i
+
+    def encode_incremental(self) -> ta.Generator[bytes | None, str, None]:
+        x = self._info.incrementalencoder(self._opts.errors)
+        i = yield None
+        while True:
+            if not i:
+                break
+            o = x.encode(i)
+            i = yield o or None
+        o = x.encode(i, final=True)
+        yield o
+
+    def decode_incremental(self) -> ta.Generator[str | None, bytes, None]:
+        x = self._info.incrementaldecoder(self._opts.errors)
+        i = yield None
+        while True:
+            if not i:
+                break
+            o = x.decode(i)
+            i = yield o or None
+        o = x.decode(i, final=True)
+        yield o
+
+
+##
+
+
+class TextEncodingCodec(Codec):
+    pass
+
+
+def normalize_text_encoding_name(s: str) -> str:
+    if ' ' in s:
+        raise NameError(s)
+    return s.lower().replace('_', '-')
+
+
+def make_text_encoding_codec(
+        name: str,
+        aliases: ta.Collection[str] | None = None,
+        *,
+        append_to: ta.MutableSequence[Codec] | None = None,
+) -> TextEncodingCodec:
+    codec = TextEncodingCodec(
+        name=check.equal(name, normalize_text_encoding_name(name)),
+        aliases=check.not_isinstance(aliases, str),
+
+        input=str,
+        output=bytes,
+
+        new=functools.partial(TextEncodingComboCodec.lookup, name),
+        new_incremental=functools.partial(TextEncodingComboCodec.lookup, name),
+    )
+
+    if append_to is not None:
+        append_to.append(codec)
+
+    return codec
+
+
+##
+
+
+ASCII = make_text_encoding_codec(
+    'ascii',
+    ['646', 'us-ascii'],
+    append_to=STANDARD_CODECS,
+)
+
+LATIN1 = make_text_encoding_codec(
+    'latin-1',
+    ['iso-8859-1', 'iso8859-1', '8859', 'cp819', 'latin', 'latin1', 'l1'],
+    append_to=STANDARD_CODECS,
+)
+
+UTF32 = make_text_encoding_codec(
+    'utf-32',
+    ['u32', 'utf32'],
+    append_to=STANDARD_CODECS,
+)
+
+UTF32BE = make_text_encoding_codec(
+    'utf-32-be',
+    ['utf-32be'],
+    append_to=STANDARD_CODECS,
+)
+
+UTF32LE = make_text_encoding_codec(
+    'utf-32-le',
+    ['utf-32le'],
+    append_to=STANDARD_CODECS,
+)
+
+UTF16 = make_text_encoding_codec(
+    'utf-16',
+    ['u16', 'utf16'],
+    append_to=STANDARD_CODECS,
+)
+
+UTF16BE = make_text_encoding_codec(
+    'utf-16-be',
+    ['utf-16be'],
+    append_to=STANDARD_CODECS,
+)
+
+UTF16LE = make_text_encoding_codec(
+    'utf-16-le',
+    ['utf-16le'],
+    append_to=STANDARD_CODECS,
+)
+
+UTF7 = make_text_encoding_codec(
+    'utf-7',
+    ['u7', 'unicode-1-1-utf-7'],
+    append_to=STANDARD_CODECS,
+)
+
+UTF8 = make_text_encoding_codec(
+    'utf-8',
+    ['u8', 'utf', 'utf8', 'cp65001'],
+    append_to=STANDARD_CODECS,
+)
+
+UTF8SIG = make_text_encoding_codec(
+    'utf-8-sig',
+    append_to=STANDARD_CODECS,
+)