UTF-8000 1.0.0__py3-none-any.whl

UTF8000/UTF8000Byte.py

@@ -0,0 +1,97 @@
+# size_t counting num of bytes; this is the only machine limitation
+# big bittian ordered (128, 64, 32, 16, 8, 4, 2, 1)
+
+ZERO               = 0b00000000
+
+ASCII_PREFIX       = 0b00000000
+ASCII_PREFIX_MASK  = 0b10000000
+ASCII_CONTENT_MASK = 0b01111111
+
+CONTINUATION_PREFIX       = 0b10000000
+CONTINUATION_PREFIX_MASK  = 0b11000000
+CONTINUATION_CONTENT_MASK = 0b00111111
+CONTINUATION_FILLED       = CONTINUATION_PREFIX + 0b00111111
+
+def byte_is_ascii(c: int) -> bool:
+    return c & ASCII_PREFIX_MASK == ASCII_PREFIX
+
+def byte_is_continuation(c: int) -> bool:
+    return c & CONTINUATION_PREFIX_MASK == CONTINUATION_PREFIX
+
+def byte_idx_0(c: int) -> int:
+    return next((
+        idx
+        for idx, bit in
+        enumerate(c & (1 << (8 - 1 - t)) for t in range(8))
+        if not bit
+    ), 8)
+
+def byte_continuation_content_idx_0(c: int) -> int:
+    # could be made part of _byte_idx_0 but whatever
+    return next((
+        idx
+        for idx, bit in
+        enumerate(c & (1 << (6 - 1 - t)) for t in range(6))
+        if not bit
+    ), 6)
+
+class UTF8000Byte:
+    def __init__(self, c: int, *,
+            is_start_byte:        bool,
+            is_continuation_byte: bool,
+            is_content_byte:      bool,
+        ) -> None:
+        self.c = c
+        self.is_start_byte        = is_start_byte
+        self.is_continuation_byte = is_continuation_byte
+        self.is_content_byte      = is_content_byte
+
+    def __str__(self) -> str:
+        return f"0b{self.c:08b}"
+
+    def debug_str(self) -> str:
+        if self.is_continuation_byte:
+            n_continuation_prefix_bits = 2
+        else:
+            n_continuation_prefix_bits = 0
+
+        if self.is_content_byte:
+            n_content_bits = self.n_content_bits
+        else:
+            n_content_bits = 0
+
+        RESET = "\x1b[0m"
+        RED   = "\x1b[31m"
+        GREEN = "\x1b[32m"
+        BLUE  = "\x1b[34m"
+        s = str(self)
+        binary_prefix = s[:2]
+        continuation_prefix = GREEN + s[2:2+n_continuation_prefix_bits] + RESET
+        start_bits          = RED   + s[2+n_continuation_prefix_bits:10 - n_content_bits] + RESET
+        content_bits        = BLUE  + s[10 - n_content_bits:] + RESET
+
+        return binary_prefix + continuation_prefix + start_bits + content_bits
+
+    @property
+    def is_ascii(self) -> bool:
+        return byte_is_ascii(self.c)
+
+    @property
+    def n_content_bits(self) -> int:
+        if not self.is_content_byte:
+            raise ValueError
+
+        if not self.is_continuation_byte:
+            idx_0 = byte_idx_0(self.c)
+            return (8 - 1 - idx_0)
+        elif not self.is_start_byte:
+            return 6
+        else:
+            idx_0_content = byte_continuation_content_idx_0(self.c)
+            return (6 - 1 - idx_0_content)
+
+    @property
+    def content(self) -> int:
+        content_mask = (1 << self.n_content_bits) - 1
+
+        return self.c & content_mask

UTF8000/UTF8000IncrementalDecoder.py

@@ -0,0 +1,121 @@
+from typing import Generator
+
+from .UTF8000Byte import UTF8000Byte, byte_is_continuation, byte_idx_0, byte_continuation_content_idx_0
+from .UTF8000Int import UTF8000Int
+
+class UTF8000IncrementalDecoder:
+    def __init__(self) -> None:
+        self._results:      list[UTF8000Int] = []
+        self._bytes_buffer: bytes            = b""
+
+        self._generator = self._utf_8000_parse_forever()
+        self._wakeup()
+
+    def __iter__(self) -> Generator[UTF8000Int, None, None]:
+        # queue
+        while self._results:
+            yield self._results.pop(0)
+
+    def feed(self, utf_8000_bytes: bytes) -> None:
+        self._bytes_buffer += utf_8000_bytes
+        self._wakeup()
+
+    def _wakeup(self) -> None:
+        self._generator.send(None)
+
+    def _await_bytes(self, n_bytes: int) -> Generator[None, None, bytes]:
+        while len(self._bytes_buffer) < n_bytes:
+            yield
+
+        ret                = self._bytes_buffer[:n_bytes]
+        self._bytes_buffer = self._bytes_buffer[n_bytes:]
+
+        return ret
+
+    def _await_byte(self) -> Generator[None, None, int]:
+        return (yield from self._await_bytes(1))[0]
+
+    def _await_continuation_byte(self) -> Generator[None, None, int]:
+        ret = yield from self._await_byte()
+        if not byte_is_continuation(ret):
+            self._on_error_invalid_continuation_byte()
+        return ret
+
+    def _on_error(self, error_message: str) -> UTF8000Byte:
+        raise ValueError(error_message)
+
+    def _on_error_invalid_start_byte(self) -> None:
+        # XXX TODO 101: read all following continuation bytes to skip them
+        self._on_error("Invalid start byte: continuation byte detected")
+
+    def _on_error_invalid_continuation_byte(self) -> None:
+        # XXX TODO 101: should we un-pop this (start) byte so further decodings can continue, and return a replacement character?
+        self._on_error("Not a continuation byte prefix")
+
+    def _on_error_overlong(self) -> None:
+        # XXX TODO 101: read all following continuation bytes to skip them
+        self._on_error("Overlong encoding")
+
+    def _utf_8000_parse_single(self) -> Generator[None, None, UTF8000Int]:
+        parsed_bytes: list[UTF8000Byte] = []
+
+        start_byte = yield from self._await_byte()
+        idx_0 = byte_idx_0(start_byte)
+
+        if idx_0 == 1:
+            self._on_error_invalid_start_byte()
+        elif idx_0 == 0:
+            n_bytes_expected = 1
+        else:
+            n_bytes_expected = idx_0
+            # when idx_0 == 8, this is 8 *so far!*
+
+        is_content_byte = idx_0 < 7
+        parsed_bytes.append(UTF8000Byte(start_byte, is_start_byte = True, is_continuation_byte = False, is_content_byte = is_content_byte))
+
+        # multiple start bytes, the power of UTF-8000!
+        if idx_0 == 8:
+            while True:
+                start_byte = yield from self._await_continuation_byte()
+                idx_0_content = byte_continuation_content_idx_0(start_byte)
+                n_bytes_expected += idx_0_content
+
+                is_content_byte = idx_0_content < 5
+                parsed_bytes.append(UTF8000Byte(start_byte, is_start_byte = True, is_continuation_byte = True, is_content_byte = is_content_byte))
+
+                if idx_0_content != 6:
+                    break
+
+        # overlong checking for non-ASCII
+        if idx_0 != 0:
+            if idx_0 == 2:
+                # special case: we're only checking for activity in the first *4* content bits gained from 1-byte (ASCII) to a 2-byte sequence,
+                # whereas every other n-byte sequence checks the first *5* content bits, since 5 bits are gained for every additional continuation byte
+                anti_overlong_check_mask_start        = 0b00011110
+                anti_overlong_check_mask_continuation = 0b00000000 # unused
+            else:
+                n_bits_overlong_check_continuation = divmod(n_bytes_expected - 2, 6)[1]
+                n_bits_overlong_check_start        = 5 - n_bits_overlong_check_continuation
+                # lower bits of start byte contents
+                anti_overlong_check_mask_start        = (1 << n_bits_overlong_check_start) - 1
+                # upper bits of continuation byte contents
+                anti_overlong_check_mask_continuation = ((1 << n_bits_overlong_check_continuation) - 1) << (6 - n_bits_overlong_check_continuation)
+
+            continuation_byte = yield from self._await_continuation_byte()
+
+            # at this point `start_byte` is the last start byte
+            if not (start_byte & anti_overlong_check_mask_start or continuation_byte & anti_overlong_check_mask_continuation):
+                self._on_error_overlong()
+
+            parsed_bytes.append(UTF8000Byte(continuation_byte, is_start_byte = False, is_continuation_byte = True, is_content_byte = True))
+
+        # the rest of the continuation bytes
+        while len(parsed_bytes) < n_bytes_expected:
+            continuation_byte = yield from self._await_continuation_byte()
+            parsed_bytes.append(UTF8000Byte(continuation_byte, is_start_byte = False, is_continuation_byte = True, is_content_byte = True))
+
+        return UTF8000Int(parsed_bytes)
+
+    def _utf_8000_parse_forever(self) -> Generator[None, None, None]:
+        while True:
+            self._results.append((yield from self._utf_8000_parse_single()))

UTF8000/UTF8000Int.py

@@ -0,0 +1,78 @@
+from __future__ import annotations
+
+from .UTF8000Byte import UTF8000Byte
+
+class UTF8000Int:
+    def __init__(self, utf_8000_bytes: list[UTF8000Byte]) -> None:
+        # No validation is done; we're assuming these come from `UTF8000IncrementalDecoder`
+        self.utf_8000_bytes = utf_8000_bytes
+
+    def __str__(self) -> str:
+        return " ".join(str(b) for b in self.utf_8000_bytes)
+
+    def debug_str(self) -> str:
+        return " ".join(b.debug_str() for b in self.utf_8000_bytes)
+
+    def __int__(self) -> int:
+        ret = 0
+        content_bytes = (b for b in self.utf_8000_bytes if b.is_content_byte)
+        for content_byte in content_bytes:
+            ret <<= content_byte.n_content_bits
+            ret += content_byte.content
+        return ret
+
+    @property
+    def n_bytes(self) -> int:
+        return len(self.utf_8000_bytes)
+
+    @property
+    def n_bits_capacity(self) -> int:
+        if self.n_bytes == 1:
+            return 7
+        else:
+            return 1 + 5 * self.n_bytes
+
+    @property
+    def n_bits_used(self) -> int:
+        content_bytes = (b for b in self.utf_8000_bytes if b.is_content_byte)
+        raise NotImplementedError
+
+    @classmethod
+    def blank(cls, n_bytes: int):
+        raise NotImplementedError
+        if n_bytes < 1:
+            raise ValueError
+
+        if n_bytes == 1:
+            segments = bytearray([_ZERO])
+
+        elif n_bytes < 8:
+            # lol whatever this is hardcoded
+            segments = bytearray(
+                [_ZERO + (((1 << n_bytes) - 1) << (8 - n_bytes))] +
+                [_CONTINUATION_PREFIX for _ in range(n_bytes - 1)]
+            )
+
+        else:
+            start_byte = [0b11111111]
+            # cont bytes
+            n_filled, n_bits_partial = divmod(n_bytes - 8, 6)
+            filled  = [_CONTINUATION_FILLED for _ in range(n_filled)]
+            partial = [_CONTINUATION_PREFIX + (((1 << n_bits_partial) - 1) << (6 - n_bits_partial))]
+            empty   = [
+                0b10000000
+                for _ in range(
+                    n_bytes - (len(start_byte) + len(filled) + len(partial))
+                )
+            ]
+
+            segments = bytearray(start_byte + filled + partial + empty)
+
+        return cls(segments)
+
+    def promote(self) -> UTF8000Int:
+        # reserve another continuation byte, adjust start byte(s), move start byte(s) content to new continuation byte, and pad new prefix with 0s if +ve or 1s if -ve
+        raise NotImplementedError
+
+    def __add__(self, rhs: UTF8000Int) -> UTF8000Int:
+        raise NotImplementedError

UTF8000/__init__.py

@@ -0,0 +1,7 @@
+"""UTF-8000: Unlimited UTF-8!"""
+
+__version__ = "1.0.0"
+
+from .UTF8000Byte               import UTF8000Byte
+from .UTF8000Int                import UTF8000Int
+from .UTF8000IncrementalDecoder import UTF8000IncrementalDecoder