PyPtt 2.2.5__tar.gz → 2.2.7__tar.gz

{pyptt-2.2.5 → pyptt-2.2.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: PyPtt
-Version: 2.2.5
+Version: 2.2.7
 Summary: PyPtt github: https://github.com/PyPtt/PyPtt
 Author-email: CodingMan <pttcodingman@gmail.com>
 License: GNU Lesser General Public License v3 (LGPLv3)

{pyptt-2.2.5 → pyptt-2.2.7}/PyPtt/__init__.py

@@ -1,4 +1,4 @@
-__version__ = '2.2.5'
+__version__ = '2.2.7'
 __author__ = 'CodingMan'
 __email__ = 'pttcodingman@gmail.com'
 

{pyptt-2.2.5 → pyptt-2.2.7}/PyPtt/_uao/__init__.py

@@ -1,10 +1,16 @@
 """Vendored Big5-UAO codec.
 
-Vendored (unmodified) from PttCodingMan/pyUAO, a fork of eight04/pyUAO,
-to drop the external `uao` PyPI dependency (unmaintained upstream) and
-pick up this fork's correctness fixes and incremental/stream codec
-support. Vendoring is for dependency control, not performance: PyPtt is
-I/O bound, so codec speed is not the point here.
+Vendored from PttCodingMan/pyUAO, a fork of eight04/pyUAO, to drop the
+external `uao` PyPI dependency (unmaintained upstream) and pick up this
+fork's correctness fixes and incremental/stream codec support.
+
+Locally modified: `_decode` batches ASCII runs with a single C-level decode
+instead of one chr()/append per byte. PTT screens are ~70% ASCII bytes even
+in Big5, so this speeds up the receive path measurably while producing
+byte-identical output (the byte→unicode table is unchanged). Python's stdlib
+`big5` codec is *not* a usable fast path — it disagrees with the UAO table on
+~260 sequences (e.g. 0xa1 0x45 → '•' vs '‧') and can't decode UAO's
+extensions at all.
 
 Source:  https://github.com/PttCodingMan/pyUAO
 Commit:  9d9a8bf59dbe60800d214b66ce972bc2ddf8da89 (v0.3.0)
@@ -89,8 +95,14 @@ def _decode(input, errors, final):
     while i < n:
         c = input[i]
         if c < 0x80:
-            append(chr(c))
-            i += 1
+            # At a character boundary every byte < 0x80 is a standalone ASCII
+            # char, so decode the whole run at once. latin-1 maps 0x00-0x7f
+            # identically to ASCII and never raises.
+            j = i + 1
+            while j < n and input[j] < 0x80:
+                j += 1
+            append(input[i:j].decode('latin-1'))
+            i = j
             continue
         if i + 1 < n:
             u = _b2u_get(c * 0x100 + input[i + 1])

{pyptt-2.2.5 → pyptt-2.2.7}/PyPtt/connect_core.py

@@ -160,6 +160,7 @@ class API(object):
         self._UseTooManyResources = TargetUnit(screens.Target.use_too_many_resources,
                                                exceptions_=exceptions.UseTooManyResources())
         self._loop = None
+        self._stream_parsers = {}
         self._ssl_context = ssl_init(self.config.verify_ssl)
 
     def _get_event_loop(self):
@@ -257,14 +258,11 @@ class API(object):
         if not connect_success:
             raise exceptions.ConnectError(self.config)
 
-    def _decode_screen(self, receive_data_buffer, start_time, target_list, is_secret, refresh, msg):
+    def _decode_screen(self, screen, start_time, target_list, is_secret, refresh, msg):
 
         break_detect_after_send = False
         use_too_many_res = False
 
-        vt100_p = screens.VT100Parser(receive_data_buffer, self.current_encoding, self.config.screen_height)
-        screen = vt100_p.screen
-
         find_target = False
         target_index = -1
         for target in target_list:
@@ -308,6 +306,17 @@ class API(object):
                 break
         return screen, find_target, is_secret, break_detect_after_send, use_too_many_res, msg, target_index
 
+    def _stream_screen(self, encoding: str, data_chunk: bytes) -> str:
+        """Feed a newly-received chunk to the per-encoding incremental parser
+        and return the current screen. Each parser decodes and steps through
+        every byte exactly once across the whole receive sequence."""
+        parser = self._stream_parsers.get(encoding)
+        if parser is None:
+            parser = screens.IncrementalScreen(encoding, self.config.screen_height)
+            self._stream_parsers[encoding] = parser
+        parser.feed(data_chunk)
+        return parser.screen
+
     async def _async_send(self, msg: str, target_list: list, screen_timeout: int, refresh: bool, secret: bool) -> int:
         current_screen_timeout = self.config.screen_timeout if screen_timeout == 0 else screen_timeout
         is_secret = secret
@@ -341,7 +350,12 @@ class API(object):
                 return -1
 
             msg = ''
-            receive_data_buffer = bytes()
+            # Fresh incremental parsers for this screen sequence. Each arriving
+            # chunk is fed once (per encoding) instead of re-parsing the whole
+            # accumulated buffer every time — turning the receive loop from
+            # O(N²) into O(N). The parsers also hold the rendered screen, so no
+            # separate raw byte buffer is accumulated.
+            self._stream_parsers = {}
             start_time = time.time()
             find_target = False
             target_index = -1
@@ -358,16 +372,21 @@ class API(object):
 
                         if isinstance(data_chunk, str):
                             data_chunk = data_chunk.encode('utf-8')
-                        receive_data_buffer += data_chunk
 
+                        screen = self._stream_screen(self.current_encoding, data_chunk)
                         screen, find_target, is_secret, break_detect_after_send, use_too_many_res, msg, target_index = \
-                            self._decode_screen(receive_data_buffer, start_time, target_list, is_secret, refresh, msg)
+                            self._decode_screen(screen, start_time, target_list, is_secret, refresh, msg)
 
                         if not find_target:
+                            # Auto-detect the server encoding: the alternate
+                            # parser has also seen every chunk so far (this
+                            # branch runs on every non-matching chunk), so
+                            # feeding it this chunk keeps it in sync.
                             original_encoding = self.current_encoding
                             self.current_encoding = 'big5uao' if original_encoding == 'utf-8' else 'utf-8'
+                            screen_ = self._stream_screen(self.current_encoding, data_chunk)
                             screen_, find_target, is_secret, break_detect_after_send, use_too_many_res, msg, target_index = \
-                                self._decode_screen(receive_data_buffer, start_time, target_list, is_secret, refresh, msg)
+                                self._decode_screen(screen_, start_time, target_list, is_secret, refresh, msg)
                             if find_target:
                                 screen = screen_
                             else:
@@ -376,10 +395,17 @@ class API(object):
                         if find_target:
                             break
             except TimeoutError:
-                if len(receive_data_buffer) > 0:
-                    vt100_p = screens.VT100Parser(receive_data_buffer, self.current_encoding, self.config.screen_height)
-                    screens.show(self.config, vt100_p.screen)
-                    self._RDQ.add(vt100_p.screen)
+                # The incremental parser for the current encoding already holds
+                # the fully-rendered screen for every byte received this round
+                # (byte-identical to a fresh VT100Parser over the whole buffer —
+                # see tests/test_incremental_parity.py), so reuse it instead of
+                # re-decoding and re-parsing the entire buffer from scratch.
+                parser = self._stream_parsers.get(self.current_encoding)
+                if parser is not None:
+                    screen = parser.screen
+                    if len(screen) > 0:
+                        screens.show(self.config, screen)
+                        self._RDQ.add(screen)
                 if use_too_many_res:
                     raise exceptions.UseTooManyResources()
                 return -1
@@ -430,7 +456,7 @@ class API(object):
                 if break_detect_after_send:
                     return -1
                 msg = ''
-                receive_data_buffer = bytes()
+                self._stream_parsers = {}
                 start_time = time.time()
                 mid_time = time.time()
                 while mid_time - start_time < current_screen_timeout:
@@ -438,9 +464,9 @@ class API(object):
                         data = self._core.read_very_eager()
                     except EOFError:
                         return -1
-                    receive_data_buffer += data
+                    screen = self._stream_screen(self.current_encoding, data)
                     screen, find_target, is_secret, break_detect_after_send, use_too_many_res, msg, target_index = \
-                        self._decode_screen(receive_data_buffer, start_time, target_list, is_secret, refresh, msg)
+                        self._decode_screen(screen, start_time, target_list, is_secret, refresh, msg)
                     if target_index != -1:
                         return target_index
                     if use_too_many_res:

{pyptt-2.2.5 → pyptt-2.2.7}/PyPtt/screens.py

@@ -1,3 +1,4 @@
+import codecs
 import re
 import sys
 import unicodedata
@@ -9,6 +10,14 @@ from . import log
 register_uao()
 
 
+# Memoise the per-character width. east_asian_width is a C call, but a PTT
+# screen is a few thousand characters drawn from a small alphabet, and the
+# parser walks each line repeatedly (_str_pos_at_cells, replace mode). Caching
+# turns the hot path into a dict hit. Writes are idempotent, so the plain-dict
+# race under free-threading is benign.
+_cell_width_cache: dict = {}
+
+
 def _cell_width(ch: str) -> int:
     """Return the terminal-cell width of a single character.
 
@@ -17,10 +26,21 @@ def _cell_width(ch: str) -> int:
     Ambiguous includes glyphs like ※, ←, →, ◎ that PTT's protocol treats
     as full-width when positioning the cursor.
     """
-    return 2 if unicodedata.east_asian_width(ch) in ('W', 'F', 'A') else 1
+    if ch.isascii():        # every ASCII char is a single narrow cell
+        return 1
+    w = _cell_width_cache.get(ch)
+    if w is None:
+        w = 2 if unicodedata.east_asian_width(ch) in ('W', 'F', 'A') else 1
+        _cell_width_cache[ch] = w
+    return w
 
 
 def _cell_len(text: str) -> int:
+    # PTT screens are mostly ASCII, and an all-ASCII string is exactly one cell
+    # per character — str.isascii() (a fast C scan) lets whole segments skip the
+    # per-character loop.
+    if text.isascii():
+        return len(text)
     return sum(_cell_width(c) for c in text)
 
 
@@ -32,6 +52,8 @@ def _str_pos_at_cells(line: str, cells: int) -> int:
     """
     if cells <= 0:
         return 0
+    if line.isascii():      # 1 cell per char → cell offset == string index
+        return min(cells, len(line))
     consumed = 0
     for i, ch in enumerate(line):
         if consumed >= cells:
@@ -241,51 +263,76 @@ def show(config, screen_queue, function_name=None):
 
 xy_pattern_h = re.compile(r'^=ESC=\[[\d]+;[\d]+H')
 xy_pattern_s = re.compile(r'^=ESC=\[[\d]+;[\d]+s')
+_color_sgr = re.compile(r'\x1B\[[\d+;]*m')
+
+
+def _preprocess(data: str) -> str:
+    """Normalise a decoded chunk: strip color, mark escapes, drop CR, and
+    collapse PTT's ' \\x08' wide-char reservation pairs.
+
+    Only the SGR (color) strip genuinely needs a regex; the remaining
+    substitutions are literal, and str.replace is several times faster than
+    re.sub — this runs on the whole buffer on every parse."""
+    data = _color_sgr.sub('', data)
+    data = data.replace('\x1B', '=ESC=')
+    data = data.replace('\r', '')
+    while ' \x08' in data:
+        data = data.replace(' \x08', '')
+    return data
+
+
+class _VT100Engine:
+    """Shared VT100 stepping engine.
+
+    Both the batch ``VT100Parser`` and the streaming ``IncrementalScreen``
+    drive the same left-to-right state machine over a preprocessed string
+    (escapes marked as ``=ESC=``). The screen is tracked in terminal cells
+    (see the module-level cell-width helpers), not bytes.
 
+    ``ESC [ 2J`` (clear screen) is handled as an in-loop reset: processing
+    left-to-right with a reset at every 2J yields the same final screen as
+    the classic "discard everything before the last 2J" slice, because each
+    reset wipes prior state — only work after the final reset survives."""
 
-class VT100Parser:
+    def __init__(self, screen_height: int = 24):
+        self._height = screen_height
+        self._reset_screen()
+
+    def _reset_screen(self):
+        self._cursor_x = 0
+        self._cursor_y = 0
+        self._lines = [''] * self._height
+        self._screen_length = dict()
+
+    # ── cursor / erase primitives ─────────────────────────────────────────
     def _h(self):
         self._cursor_x = 0
         self._cursor_y = 0
 
     def _move(self, x, y):
         self._cursor_x = x
-        self._cursor_y = min(y, len(self.screen) - 1)
+        self._cursor_y = min(y, len(self._lines) - 1)
 
     def _newline(self):
         self._cursor_x = 0
-        self._cursor_y = min(self._cursor_y + 1, len(self.screen) - 1)
+        self._cursor_y = min(self._cursor_y + 1, len(self._lines) - 1)
 
     def _k(self):
         # Erase from cursor to end of line (VT100 ESC [ K).
-        pos = _str_pos_at_cells(self.screen[self._cursor_y], self._cursor_x)
-        self.screen[self._cursor_y] = self.screen[self._cursor_y][:pos]
-        self.screen_length[self._cursor_y] = self._cursor_x
-
-    def __init__(self, bytes_data, encoding, screen_height: int = 24):
-        # self._data = data
-        # https://www.csie.ntu.edu.tw/~r88009/Java/html/Network/vt100.htm
-
-        self._cursor_x = 0
-        self._cursor_y = 0
-        self.screen = [''] * screen_height
-        self.screen_length = dict()
-
-        data = bytes_data.decode(encoding, errors='replace')
-
-        # remove color
-        data = re.sub(r'\x1B\[[\d+;]*m', '', data)
-        data = re.sub(r'[\x1B]', '=ESC=', data)
-        data = re.sub(r'[\r]', '', data)
-        while ' \x08' in data:
-            data = re.sub(r' \x08', '', data)
-
-        if '=ESC=[2J' in data:
-            data = data[data.rfind('=ESC=[2J') + len('=ESC=[2J'):]
-
-        count = 0
+        pos = _str_pos_at_cells(self._lines[self._cursor_y], self._cursor_x)
+        self._lines[self._cursor_y] = self._lines[self._cursor_y][:pos]
+        self._screen_length[self._cursor_y] = self._cursor_x
+
+    # ── the stepping loop ─────────────────────────────────────────────────
+    def _run(self, data: str) -> str:
+        """Consume `data` left-to-right, mutating screen state.
+
+        Returns the unconsumed tail: '' when everything was consumed, or the
+        remainder starting at a front ``=ESC=`` the machine does not
+        recognise. The batch parser ignores that tail (matching the classic
+        "bail on unknown escape" behaviour); the incremental parser carries
+        it until more bytes arrive or freezes on it."""
         while data:
-            count += 1
             while True:
                 if not data.startswith('=ESC='):
                     break
@@ -300,6 +347,10 @@ class VT100Parser:
                 elif data.startswith('=ESC=[s'):
                     data = data[len('=ESC=[s'):]
                     continue
+                elif data.startswith('=ESC=[2J'):
+                    data = data[len('=ESC=[2J'):]
+                    self._reset_screen()
+                    continue
                 break
 
             xy_result = None
@@ -317,7 +368,6 @@ class VT100Parser:
                 new_y = int(xy_part[6:xy_part.find(';')]) - 1
                 # VT100 columns are 1-based; convert to 0-based cursor_x.
                 new_x = int(xy_part[xy_part.find(';') + 1: -1]) - 1
-                # log.py.info('xy', xy_part, new_x, new_y)
                 self._move(new_x, new_y)
 
                 data = data[len(xy_part):]
@@ -328,16 +378,14 @@ class VT100Parser:
                     self._newline()
                     continue
 
-                # print(f'-{data[:1]}-{len(data[:1].encode("big5-uao", "replace"))}')
+                if self._cursor_y not in self._screen_length:
+                    self._screen_length[self._cursor_y] = _cell_len(self._lines[self._cursor_y])
 
-                if self._cursor_y not in self.screen_length:
-                    self.screen_length[self._cursor_y] = _cell_len(self.screen[self._cursor_y])
-
-                current_line_length = self.screen_length[self._cursor_y]
+                current_line_length = self._screen_length[self._cursor_y]
                 replace_mode = False
                 if current_line_length < self._cursor_x:
                     append_space = ' ' * (self._cursor_x - current_line_length)
-                    self.screen[self._cursor_y] += append_space
+                    self._lines[self._cursor_y] += append_space
                 elif current_line_length > self._cursor_x:
                     replace_mode = True
 
@@ -347,34 +395,143 @@ class VT100Parser:
                 next_esc = data.find('=ESC=')
                 next_esc = 1920 if next_esc < 0 else next_esc
                 if next_esc == 0:
-                    break
+                    # A front escape the machine does not consume. Hand the
+                    # remainder back to the caller instead of dropping it.
+                    return data
 
                 current_index = min(next_newline, next_esc)
 
                 current_data = data[:current_index]
                 current_data_length = _cell_len(current_data)
                 if replace_mode:
-                    line = self.screen[self._cursor_y]
+                    line = self._lines[self._cursor_y]
                     splice_start = _str_pos_at_cells(line, self._cursor_x)
                     splice_end = _str_pos_at_cells(line, self._cursor_x + current_data_length)
-                    self.screen[self._cursor_y] = line[:splice_start] + current_data + line[splice_end:]
+                    self._lines[self._cursor_y] = line[:splice_start] + current_data + line[splice_end:]
                     self._cursor_x += current_data_length
-                    if self._cursor_x > self.screen_length[self._cursor_y]:
-                        self.screen_length[self._cursor_y] = self._cursor_x
+                    if self._cursor_x > self._screen_length[self._cursor_y]:
+                        self._screen_length[self._cursor_y] = self._cursor_x
                 else:
-                    self.screen[self._cursor_y] += current_data
+                    self._lines[self._cursor_y] += current_data
                     self._cursor_x += current_data_length
-                    self.screen_length[self._cursor_y] = self._cursor_x
+                    self._screen_length[self._cursor_y] = self._cursor_x
 
                 data = data[current_index:]
+        return ''
+
+
+class VT100Parser(_VT100Engine):
+    """Batch parser: decode the whole byte buffer and render one screen.
 
-                # print('\n'.join(self.screen))
-        # print('\n'.join(self._screen))
-        # print('=' * 20)
-        # print(data)
+    Kept as the reference implementation and the target-matching path for a
+    full (non-streamed) buffer. ``IncrementalScreen`` fed the same bytes in
+    any chunking produces a byte-identical screen (see
+    tests/test_incremental_parity.py)."""
 
-        # print('Spend', count, 'cycle')
-        self.screen = '\n'.join(self.screen)
+    def __init__(self, bytes_data, encoding, screen_height: int = 24):
+        # https://www.csie.ntu.edu.tw/~r88009/Java/html/Network/vt100.htm
+        super().__init__(screen_height)
+        data = _preprocess(bytes_data.decode(encoding, errors='replace'))
+        # Leftover (a trailing/unknown escape) is intentionally ignored here,
+        # matching the classic parser's "bail on unknown escape" behaviour.
+        self._run(data)
+        self.screen = '\n'.join(self._lines)
+
+
+# CSI final bytes are 0x40-0x7e; parameter/intermediate bytes are 0x20-0x3f.
+def _escape_complete(seq: str) -> bool:
+    """Is `seq` (which starts with a raw ESC) a complete escape sequence?
+
+    Used by the streaming parser to decide whether a trailing ESC run might
+    still be extended by the next chunk. A CSI (``ESC [``) is complete once a
+    final byte (0x40-0x7e) appears; a non-CSI ESC is treated as complete (the
+    engine will bail on it, same as the batch parser)."""
+    if len(seq) < 2:
+        return False            # lone ESC — wait for more
+    if seq[1] != '[':
+        return True             # ESC + non-'[': engine bails on it
+    for ch in seq[2:]:
+        if 0x40 <= ord(ch) <= 0x7e:
+            return True
+    return False                # still inside CSI parameters
+
+
+class IncrementalScreen(_VT100Engine):
+    """Streaming VT100 parser: feed byte chunks, read the current screen.
+
+    Removes the O(N²) re-parse of the receive loop — each byte is decoded and
+    stepped through the state machine exactly once, instead of the whole
+    accumulated buffer being re-parsed on every arriving chunk.
+
+    Correctness is anchored to ``VT100Parser``: feeding the same bytes in any
+    chunking (down to one byte at a time) yields the identical screen."""
+
+    def __init__(self, encoding, screen_height: int = 24):
+        super().__init__(screen_height)
+        self._decoder = codecs.getincrementaldecoder(encoding)(errors='replace')
+        self._raw = ''          # decoded but not-yet-safe-to-process tail
+        self._leftover = ''     # preprocessed front escape the engine paused on
+        self._bailed = False    # hit a complete-but-unknown escape → frozen
+
+    def _safe_cut(self, s: str) -> int:
+        """Length of the prefix of `s` safe to process now.
+
+        Holds back the maximal trailing run of spaces and backspaces, and an
+        incomplete trailing escape.
+
+        The ' \\x08' collapse cascades backwards through a run of spaces
+        (``'   \\x08\\x08'`` → ``' '``), and the ``\\x08`` bytes can arrive in a
+        later chunk, so the whole trailing ``[ \\x08]`` run is ambiguous until a
+        character that cannot participate in the collapse follows it. Only
+        commit up to that terminating character."""
+        cut = len(s)
+        while cut > 0 and s[cut - 1] in ' \x08':
+            cut -= 1
+        esc = s.rfind('\x1b')
+        if esc != -1 and not _escape_complete(s[esc:]):
+            cut = min(cut, esc)
+        return cut
+
+    def feed(self, chunk) -> None:
+        if self._bailed:
+            return
+        if isinstance(chunk, str):
+            chunk = chunk.encode('utf-8')
+        self._raw += self._decoder.decode(chunk)
+        cut = self._safe_cut(self._raw)
+        if cut == 0:
+            return
+        safe = _preprocess(self._raw[:cut])
+        self._raw = self._raw[cut:]
+        leftover = self._run(self._leftover + safe)
+        self._leftover = leftover
+        if leftover:
+            # _safe_cut only forwards complete escapes, so a surviving front
+            # escape is complete but unrecognised — the batch parser bails on
+            # it forever, so we freeze to match.
+            self._bailed = True
+
+    @property
+    def screen(self) -> str:
+        """Current screen as a ``screen_height``-line string, including any
+        still-buffered tail — matches VT100Parser over all bytes fed so far."""
+        dec_state = self._decoder.getstate()
+        if not self._raw and not self._leftover and not dec_state[0]:
+            # Nothing held and no partial multibyte in the decoder.
+            return '\n'.join(self._lines)
+        # Tentatively flush the held tail on a snapshot so a partial escape, an
+        # unpaired trailing space, or a partial multibyte char is reflected
+        # (matching the batch parser's errors='replace') without committing.
+        saved = (self._cursor_x, self._cursor_y, list(self._lines),
+                 dict(self._screen_length))
+        try:
+            tail = self._raw + self._decoder.decode(b'', final=True)
+            self._run(self._leftover + _preprocess(tail))
+            return '\n'.join(self._lines)
+        finally:
+            (self._cursor_x, self._cursor_y, self._lines,
+             self._screen_length) = saved
+            self._decoder.setstate(dec_state)
 
 
 if __name__ == '__main__':

{pyptt-2.2.5 → pyptt-2.2.7}/PyPtt.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: PyPtt
-Version: 2.2.5
+Version: 2.2.7
 Summary: PyPtt github: https://github.com/PyPtt/PyPtt
 Author-email: CodingMan <pttcodingman@gmail.com>
 License: GNU Lesser General Public License v3 (LGPLv3)

{pyptt-2.2.5 → pyptt-2.2.7}/PyPtt.egg-info/SOURCES.txt

@@ -79,6 +79,7 @@ tests/test_get_user.py
 tests/test_get_waterball.py
 tests/test_give_money.py
 tests/test_i18n.py
+tests/test_incremental_parity.py
 tests/test_init.py
 tests/test_logger.py
 tests/test_login_logout.py
@@ -94,4 +95,5 @@ tests/test_search_user.py
 tests/test_service.py
 tests/test_set_board_title.py
 tests/test_set_signature_file.py
+tests/test_uao_codec.py
 tests/test_vt100_parser.py

pyptt-2.2.7/tests/test_incremental_parity.py

@@ -0,0 +1,135 @@
+"""
+Parity tests for PyPtt.screens.IncrementalScreen.
+
+The receive loop in connect_core streams bytes and must decide, after every
+arriving chunk, whether the current screen matches a target. Re-parsing the
+whole accumulated buffer on every chunk is O(N²); IncrementalScreen feeds each
+byte through the VT100 state machine exactly once instead.
+
+Correctness is anchored to the batch VT100Parser: feeding the same bytes in any
+chunking — down to one byte at a time, the worst case for split escapes, split
+multibyte characters and split ' \\x08' wide-char pairs — must yield the exact
+same screen string. These tests pin that invariant so a future change to either
+parser can't silently diverge.
+"""
+
+import glob
+import random
+from pathlib import Path
+
+import pytest
+
+from PyPtt.screens import IncrementalScreen, VT100Parser
+
+FIXTURE_DIR = Path(__file__).resolve().parent / 'fixtures' / 'vt100'
+
+CHUNK_SIZES = [1, 2, 3, 5, 7, 13, 64, 256, 100000]
+
+
+def feed_in_chunks(data: bytes, encoding: str, chunk: int,
+                   read_each: bool = False) -> str:
+    parser = IncrementalScreen(encoding, 24)
+    for i in range(0, len(data), chunk):
+        parser.feed(data[i:i + chunk])
+        if read_each:
+            # Reading .screen mid-stream (as the receive loop does for target
+            # matching) must not corrupt the committed state.
+            _ = parser.screen
+    return parser.screen
+
+
+# ── crafted byte streams that stress the chunk boundaries ─────────────────────
+
+CRAFTED = [
+    b'',
+    b'hello world',
+    b'\x1b[1;37;44mHELLO\x1b[0m',                       # color strip
+    b'BEFORE\x1b[2JAFTER',                              # clear screen
+    b'A\x1b[2JB\x1b[2JC',                               # only last 2J wins
+    b'helloworld\x1b[1;6H\x1b[K',                       # erase to EOL
+    b'abc \x08def',                                     # single ' \x08'
+    b'a    \x08\x08\x08\x08b',                          # cascading ' \x08'
+    b'   \x08\x08\x08\x08\x08x',                        # more BS than spaces
+    b'\x1b[5;3HHi\nNext',                               # positioning + newline
+    b'  \x08\x08\xe2\x97\x8e tail',                     # ◎ via reservation pair
+    b'\x1b[10;1H  \x08\x08\xe2\x80\xbb\x1b[10;3H \xe7\x99\xbc\xe4\xbf\xa1\xe7\xab\x99: x',
+    b'\x1b[6nHELLO',                                    # unknown escape -> bail
+    b'line1\r\nline2',                                  # CRLF
+    b'helloworld\x1b[1;2HXY',                           # replace mode
+    b'\x1b[100;1HX',                                    # row clamp
+    b'\xe2\x94\x80' * 40,                               # long box-drawing run
+    '中文範例'.encode('utf-8'),                          # wide chars
+]
+
+
+@pytest.mark.parametrize('data', CRAFTED)
+@pytest.mark.parametrize('chunk', CHUNK_SIZES)
+def test_crafted_parity_utf8(data, chunk):
+    ref = VT100Parser(data, 'utf-8', 24).screen
+    assert feed_in_chunks(data, 'utf-8', chunk) == ref
+    assert feed_in_chunks(data, 'utf-8', chunk, read_each=True) == ref
+
+
+BIG5_TEXTS = ['中文', '程式範例', '◎[軟工]※←→─▄▆', '看板《Python》 測試']
+
+
+@pytest.mark.parametrize('text', BIG5_TEXTS)
+@pytest.mark.parametrize('chunk', [1, 2, 3, 5, 64])
+def test_big5uao_parity(text, chunk):
+    data = text.encode('big5')
+    ref = VT100Parser(data, 'big5uao', 24).screen
+    assert feed_in_chunks(data, 'big5uao', chunk) == ref
+
+
+def test_big5uao_realistic_stream_parity():
+    # A PTT-shaped big5uao screen: clear, positioning, wide chars via the
+    # ' \x08' reservation pattern, box drawing, erase-to-EOL, comment glyphs.
+    def b5(s):
+        return s.encode('big5')
+
+    stream = (
+        b'\x1b[2J\x1b[1;1H' + b5('看板《Python》')
+        + b'\x1b[3;1H  \x08\x08' + b5('◎') + b5(' [問題] 測試中文標題 ')
+        + b'\x1b[5;1H' + b5('※ 發信站: 批踢踢實業坊(ptt.cc)')
+        + b'\x1b[6;1H' + b5('推 user123: 這是一則推文') + b'\x1b[K'
+        + b'\x1b[10;1H' + b5('─' * 30)
+        + b'\x1b[23;1H' + b5('瀏覽 第 1/2 頁')
+    )
+    ref = VT100Parser(stream, 'big5uao', 24).screen
+    for chunk in CHUNK_SIZES:
+        assert feed_in_chunks(stream, 'big5uao', chunk) == ref
+        assert feed_in_chunks(stream, 'big5uao', chunk, read_each=True) == ref
+
+
+# ── real captured PTT byte streams, every boundary ────────────────────────────
+
+FIXTURES = sorted(glob.glob(str(FIXTURE_DIR / '*.bin')))
+
+
+@pytest.mark.parametrize('fixture', FIXTURES, ids=lambda p: Path(p).stem)
+@pytest.mark.parametrize('chunk', CHUNK_SIZES)
+def test_fixture_parity(fixture, chunk):
+    data = Path(fixture).read_bytes()
+    encoding = Path(fixture[:-4] + '.encoding').read_text().strip()
+    ref = VT100Parser(data, encoding, 24).screen
+    assert feed_in_chunks(data, encoding, chunk) == ref
+    assert feed_in_chunks(data, encoding, chunk, read_each=True) == ref
+
+
+@pytest.mark.parametrize('fixture', FIXTURES, ids=lambda p: Path(p).stem)
+def test_fixture_parity_random_chunking(fixture):
+    # Randomised chunk boundaries with a mid-stream .screen read each step —
+    # exactly how the receive loop drives the parser.
+    data = Path(fixture).read_bytes()
+    encoding = Path(fixture[:-4] + '.encoding').read_text().strip()
+    ref = VT100Parser(data, encoding, 24).screen
+    for seed in range(25):
+        rnd = random.Random(seed)
+        parser = IncrementalScreen(encoding, 24)
+        i = 0
+        while i < len(data):
+            n = rnd.randint(1, 9)
+            parser.feed(data[i:i + n])
+            _ = parser.screen
+            i += n
+        assert parser.screen == ref, f'seed={seed}'

pyptt-2.2.7/tests/test_uao_codec.py

@@ -0,0 +1,93 @@
+"""
+Regression tests for the vendored Big5-UAO codec (PyPtt/_uao).
+
+`_decode` batches ASCII runs (one C-level decode instead of a chr()/append per
+byte). These tests pin the invariant that the batched path produces exactly the
+same output as a naive byte-by-byte decode, across ASCII runs, character
+boundaries, incremental feeding and UAO-specific characters that stdlib `big5`
+gets wrong or cannot represent.
+"""
+
+import codecs
+
+import pytest
+
+import PyPtt.screens  # noqa: F401 — importing registers the big5uao codec
+
+
+def naive_decode(data: bytes, errors: str = 'replace') -> str:
+    """Reference: decode one unit at a time, no ASCII batching."""
+    from PyPtt._uao.b2u import b2u_table
+    get = b2u_table.get
+    out = []
+    i, n = 0, len(data)
+    while i < n:
+        c = data[i]
+        if c < 0x80:
+            out.append(chr(c))
+            i += 1
+            continue
+        pair = get(c * 0x100 + data[i + 1]) if i + 1 < n else None
+        if pair is not None:
+            out.append(pair)
+            i += 2
+            continue
+        single = get(c)
+        if single is not None:
+            out.append(single)
+            i += 1
+            continue
+        out.append('�')  # errors='replace'
+        i += 1
+    return ''.join(out)
+
+
+ASCII_CASES = [
+    b'',
+    b'hello world 12345',
+    b'   leading and trailing spaces   ',
+    b'[Ctrl-P] [d]delete [z]digest (100%)',
+]
+
+
+@pytest.mark.parametrize('data', ASCII_CASES)
+def test_ascii_runs_decode_verbatim(data):
+    assert data.decode('big5uao') == data.decode('ascii')
+
+
+def test_mixed_ascii_and_cjk():
+    s = '  9439   3 1/10 littrabble   ◎ [問題] 測試中文標題 discussion'
+    assert s.encode('big5uao').decode('big5uao') == s
+
+
+def test_uao_specific_characters_round_trip():
+    # Characters where stdlib big5 disagrees with UAO or can't encode them.
+    for s in ['‧', '～', '￥', '￠', '﹨', '⊕', '◎', '※', '←', '→', '─']:
+        assert s.encode('big5uao').decode('big5uao') == s
+
+
+@pytest.mark.parametrize('errors', ['replace', 'ignore', 'strict'])
+def test_batched_matches_naive_on_valid_content(errors):
+    s = '看板《Python》 test 混合 content 123 ‧～ ◎[軟工] tail'
+    data = s.encode('big5uao')
+    assert data.decode('big5uao', errors) == naive_decode(data)
+
+
+def test_incremental_byte_by_byte_matches_batch():
+    s = '批踢踢 Python 測試 ‧～￥ ◎[軟工] 12345'
+    data = s.encode('big5uao')
+    dec = codecs.getincrementaldecoder('big5uao')('replace')
+    out = ''.join(dec.decode(bytes([b])) for b in data) + dec.decode(b'', final=True)
+    assert out == data.decode('big5uao') == s
+
+
+def test_incremental_splits_multibyte_across_chunks():
+    data = '中文'.encode('big5uao')
+    dec = codecs.getincrementaldecoder('big5uao')('replace')
+    # Split inside the first character's byte pair.
+    out = dec.decode(data[:1]) + dec.decode(data[1:], final=True)
+    assert out == '中文'
+
+
+def test_lone_0xff_maps_to_uao_private_use():
+    assert b'\xff'.decode('big5uao') == ''