hdiffpatch 1.0.0__cp310-cp310-win32.whl

hdiffpatch/_c_src/tamp/tamp/decompressor.py

@@ -0,0 +1,336 @@
+from io import BytesIO
+
+try:
+    from typing import Optional
+except ImportError:
+    pass
+
+from . import compute_min_pattern_size, initialize_dictionary
+
+_CHUNK_SIZE = 1 << 20
+_FLUSH = object()
+
+# Each key here are the huffman codes or'd with 0x80
+# This is so that each lookup is easy/quick.
+_huffman_lookup = {
+    0b0: 0,
+    0b11: 1,
+    0b1000: 2,
+    0b1011: 3,
+    0b10100: 4,
+    0b100100: 5,
+    0b100110: 6,
+    0b101011: 7,
+    0b1001011: 8,
+    0b1010100: 9,
+    0b10010100: 10,
+    0b10010101: 11,
+    0b10101010: 12,
+    0b100111: 13,
+    0b10101011: _FLUSH,
+}
+
+
+class _BitReader:
+    """Reads bits from a stream."""
+
+    def __init__(self, f, close_f_on_close=False):
+        self.close_f_on_close = close_f_on_close
+        self.f = f
+        self.clear()
+
+    def read_huffman(self):
+        proposed_code = 0
+        lookup = _huffman_lookup
+        read = self.read
+        for _ in range(8):
+            proposed_code |= read(1)
+            try:
+                return lookup[proposed_code]
+            except KeyError:
+                proposed_code <<= 1
+        raise RuntimeError("Unable to decode huffman code. Should never happen.")
+
+    def read(self, num_bits):
+        while self.bit_pos < num_bits:
+            byte = self.f.read(1)
+            if not byte:
+                raise EOFError
+            byte_value = int.from_bytes(byte, "little")
+            self.buffer |= byte_value << (24 - self.bit_pos)
+            self.bit_pos += 8
+
+            if self.backup_buffer is not None and self.backup_bit_pos is not None:
+                self.backup_buffer |= byte_value << (24 - self.backup_bit_pos)
+                self.backup_bit_pos += 8
+
+        result = self.buffer >> (32 - num_bits)
+        mask = (1 << (32 - num_bits)) - 1
+        self.buffer = (self.buffer & mask) << num_bits
+        self.bit_pos -= num_bits
+
+        return result
+
+    def clear(self):
+        self.buffer = 0
+        self.bit_pos = 0
+
+        self.backup_buffer = None
+        self.backup_bit_pos = None
+
+    def close(self):
+        if self.close_f_on_close:
+            self.f.close()
+
+    def __len__(self):
+        return self.bit_pos
+
+    def __enter__(self):
+        # Context manager to restore all read bits in a session if any read fails.
+
+        # backup the buffer
+        self.backup_buffer = self.buffer
+        self.backup_bit_pos = self.bit_pos
+        return self
+
+    def __exit__(self, exception_type, exception_value, exception_traceback):
+        if exception_type is not None:
+            # restore buffers
+            self.buffer = self.backup_buffer
+            self.bit_pos = self.backup_bit_pos
+
+        self.backup_buffer = None
+        self.backup_bit_pos = None
+
+
+class _RingBuffer:
+    def __init__(self, buffer):
+        self.buffer = buffer
+        self.size = len(buffer)
+        self.pos = 0  # Always pointing to the byte-to-be-overwritten
+        self.index = self.buffer.index
+
+    def write_byte(self, byte):  # ~10% of time
+        self.buffer[self.pos] = byte
+        self.pos += 1
+        if self.pos == self.size:
+            self.pos = 0
+
+    def write_bytes(self, data):
+        for byte in data:
+            self.write_byte(byte)
+
+
+class Decompressor:
+    """Decompresses a file or stream of tamp-compressed data.
+
+    Can be used as a context manager to automatically handle file
+    opening and closing:
+
+    .. code-block:: python
+
+        with tamp.Decompressor("compressed.tamp") as f:
+            decompressed_data = f.read()
+    """
+
+    def __init__(self, f, *, dictionary: Optional[bytearray] = None):
+        """
+        Parameters
+        ----------
+        f: Union[file, str]
+            File-like object to read compressed bytes from.
+        dictionary: Optional[bytearray]
+            Use the given **initialized** buffer inplace.
+            At compression time, the same initialized buffer must be provided.
+            Decompression stream's ``window`` must agree with the dictionary size.
+            If providing a pre-allocated buffer, but with default initialization, it must
+            first be initialized with :func:`~tamp.initialize_dictionary`
+        """
+        if not hasattr(f, "read"):  # It's probably a path-like object.
+            f = open(str(f), "rb")
+            close_f_on_close = True
+        else:
+            close_f_on_close = False
+
+        self._bit_reader = _BitReader(f, close_f_on_close=close_f_on_close)
+
+        # Read Header
+        self.window_bits = self._bit_reader.read(3) + 8
+        self.literal_bits = self._bit_reader.read(2) + 5
+        uses_custom_dictionary = self._bit_reader.read(1)
+        reserved = self._bit_reader.read(1)
+        more_header_bytes = self._bit_reader.read(1)
+
+        if reserved:
+            raise NotImplementedError
+
+        if more_header_bytes:
+            raise NotImplementedError
+
+        if uses_custom_dictionary and dictionary is None:
+            raise ValueError
+
+        self._window_buffer = _RingBuffer(
+            buffer=(dictionary if dictionary else initialize_dictionary(1 << self.window_bits)),
+        )
+
+        self.min_pattern_size = compute_min_pattern_size(self.window_bits, self.literal_bits)
+
+        self.overflow = bytearray()
+
+    def readinto(self, buf: bytearray) -> int:
+        """Decompresses data into provided buffer.
+
+        Parameters
+        ----------
+        buf: bytearray
+            Buffer to decode data into.
+
+        Returns
+        -------
+        int
+            Number of bytes decompressed into buffer.
+        """
+        if len(self.overflow) > len(buf):
+            buf[:] = self.overflow[: len(buf)]
+            written = len(buf)
+            self.overflow = self.overflow[len(buf) :]
+            return written
+        elif self.overflow:
+            buf[: len(self.overflow)] = self.overflow
+            written = len(self.overflow)
+            self.overflow = bytearray()
+        else:
+            written = 0
+
+        while written < len(buf):
+            try:
+                with self._bit_reader:
+                    is_literal = self._bit_reader.read(1)
+
+                    if is_literal:
+                        c = self._bit_reader.read(self.literal_bits)
+                        self._window_buffer.write_byte(c)
+                        buf[written] = c
+                        written += 1
+                    else:
+                        match_size = self._bit_reader.read_huffman()
+                        if match_size is _FLUSH:
+                            self._bit_reader.clear()
+                            continue
+                        match_size += self.min_pattern_size
+                        index = self._bit_reader.read(self.window_bits)
+
+                        string = self._window_buffer.buffer[index : index + match_size]
+                        self._window_buffer.write_bytes(string)
+
+                        to_buf = min(len(buf) - written, match_size)
+                        buf[written : written + to_buf] = string[:to_buf]
+                        written += to_buf
+                        if to_buf < match_size:
+                            self.overflow[:] = string[to_buf:]
+                            break
+            except EOFError:
+                break
+
+        return written
+
+    def read(self, size: int = -1) -> bytearray:
+        """Decompresses data to bytes.
+
+        Parameters
+        ----------
+        size: int
+            Maximum number of bytes to return.
+            If a negative value is provided, all data will be returned.
+            Defaults to ``-1``.
+
+        Returns
+        -------
+        bytearray
+            Decompressed data.
+        """
+        if size == 0:
+            return bytearray()
+
+        chunk_size = _CHUNK_SIZE
+        out = []
+        while True:
+            buf = bytearray(chunk_size if size < 0 else size)
+            chunk_size <<= 1  # Keep allocating larger chunks as we go on.
+            read_size = self.readinto(buf)
+            if size > 0:
+                # Read the entire contents in one go.
+                out.append(buf)
+                break
+            else:
+                if read_size < len(buf):
+                    if read_size:
+                        out.append(buf[:read_size])
+                    break
+                else:
+                    out.append(buf)
+        return out[0] if len(out) == 1 else bytearray(b"".join(out))
+
+    def close(self):
+        """Closes the input file or stream, if tamp opened it."""
+        self._bit_reader.close()
+
+    def __enter__(self):
+        """Use :class:`Decompressor` as a context manager.
+
+        .. code-block:: python
+
+           with tamp.Decompressor("output.tamp") as f:
+               decompressed_data = f.read()
+        """
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        """Calls :meth:`~Decompressor.close` on contextmanager exit."""
+        self.close()
+
+
+class TextDecompressor(Decompressor):
+    """Decompresses a file or stream of tamp-compressed data into text."""
+
+    def read(self, size: int = -1) -> str:
+        """Decompresses data to text.
+
+        Parameters
+        ----------
+        size: int
+            Maximum number of bytes to return.
+            If a negative value is provided, all data will be returned.
+            Defaults to ``-1``.
+
+        Returns
+        -------
+        str
+            Decompressed text.
+        """
+        return super().read(size).decode()
+
+
+def decompress(data: bytes, *, dictionary: Optional[bytearray] = None) -> bytearray:
+    """Single-call to decompress data.
+
+    Parameters
+    ----------
+    data: bytes
+        Tamp-compressed data to decompress.
+    dictionary: Optional[bytearray]
+        Use the given **initialized** buffer inplace.
+        At compression time, the same initialized buffer must be provided.
+        Decompression stream's ``window`` must agree with the dictionary size.
+        If providing a pre-allocated buffer, but with default initialization, it must
+        first be initialized with :func:`~tamp.initialize_dictionary`
+
+    Returns
+    -------
+    bytearray
+        Decompressed data.
+    """
+    with BytesIO(data) as f:
+        d = Decompressor(f, dictionary=dictionary)
+        return d.read()

hdiffpatch/_c_src/tamp/tamp/decompressor_viper.py

@@ -0,0 +1,220 @@
+from io import BytesIO
+
+import micropython
+from micropython import const
+
+from . import compute_min_pattern_size, initialize_dictionary
+
+_HUFFMAN_TABLE = b"BBBBBBBBBBBBBBBBeeee\x8a\x8bxxffffmmmmTTTTTTTTyy\x8c\x8fggggCCCCCCCCCCCCCCCC"
+
+_FLUSH = const(15)
+
+
+class Decompressor:
+    def __init__(self, f, *, dictionary=None):
+        self._close_f_on_close = False
+        if not hasattr(f, "read"):  # It's probably a path-like object.
+            f = open(str(f), "rb")
+            self._close_f_on_close = True
+
+        self.f = f
+        self.f_buf = 0
+        self.f_pos = 0
+
+        # Read Header
+        header = self.f.read(1)[0]
+        self.w_bits = (header >> 5) + 8
+        self.literal_bits = ((header >> 3) & 0b11) + 5
+        uses_custom_dictionary = header & 0b100
+        reserved = header & 0b10
+        more_header_bytes = header & 0b1
+
+        if reserved or more_header_bytes:
+            raise NotImplementedError
+
+        if uses_custom_dictionary:
+            if not dictionary:
+                raise ValueError
+            self.w_buf = dictionary
+        else:
+            self.w_buf = initialize_dictionary(1 << self.w_bits)
+        self.w_pos = 0
+
+        self.min_pattern_size = compute_min_pattern_size(self.w_bits, self.literal_bits)
+
+        self.overflow_buf = bytearray(self.min_pattern_size + 13)
+        self.overflow_pos = 0
+        self.overflow_size = 0
+
+    @micropython.viper
+    def readinto(self, buf) -> int:
+        """Attempt to fill out, will place into overflow."""
+        out_capacity = int(len(buf))  # const
+        out_buf = ptr8(buf)
+
+        huffman_table = ptr8(_HUFFMAN_TABLE)
+
+        literal_bits = int(self.literal_bits)
+
+        overflow_buf = self.overflow_buf
+        overflow_buf_ptr = ptr8(overflow_buf)
+        overflow_pos = int(self.overflow_pos)
+        overflow_size = int(self.overflow_size)
+
+        w_buf = self.w_buf
+        w_buf_ptr = ptr8(w_buf)
+        w_pos = int(self.w_pos)
+        w_bits = int(self.w_bits)
+        w_mask = (1 << w_bits) - 1
+
+        f_pos = int(self.f_pos)
+        f_buf = int(self.f_buf)
+        f = self.f
+
+        min_pattern_size = int(self.min_pattern_size)
+        int(len(overflow_buf))
+
+        # Copy overflow into out if available
+        overflow_copy = int(min(overflow_size, out_capacity))
+        for i in range(overflow_copy):
+            buf[i] = overflow_buf_ptr[overflow_pos + i]
+        out_pos = overflow_copy
+        overflow_pos += overflow_copy
+        overflow_size -= overflow_copy
+
+        full_mask = int(0x3FFFFFFF)
+
+        # Decompress more
+        try:
+            while out_pos < out_capacity:
+                overflow_pos = 0  # overflow_size is also always zero here
+
+                if f_pos == 0:
+                    f_buf = int(f.read(1)[0]) << 22  # Will raise IndexError if out of data.
+                    f_pos = 8
+
+                # reuse is_literal flag as match_size so we don't need to explicitly set it
+                match_size = f_buf >> 29
+                # Don't update f_buf & f_pos here, in case there's not enough data available.
+
+                if match_size:
+                    if f_pos < (literal_bits + 1):
+                        f_buf |= int(f.read(1)[0]) << (22 - f_pos)
+                        f_pos += 8
+
+                    # Now update buffers from is_literal
+                    f_buf = (f_buf << 1) & full_mask
+
+                    c = f_buf >> (30 - literal_bits)
+                    f_buf = (f_buf << literal_bits) & full_mask
+                    f_pos -= literal_bits + 1
+
+                    out_buf[out_pos] = c
+                    w_buf_ptr[w_pos] = c
+                    out_pos += 1
+                else:
+                    # Read and decode Huffman; we'll need the bits regardless
+                    if f_pos < 9:
+                        f_buf |= int(f.read(1)[0]) << (22 - f_pos)
+                        f_pos += 8
+
+                    if f_buf >> 28:
+                        code = (f_buf >> 21) & 0x7F
+                        code = 33 if code >= 64 else huffman_table[code]
+                        match_size = code & 0xF
+                        if match_size == _FLUSH:
+                            f_buf = f_pos = 0
+                            continue
+                        delta = code >> 4
+                    else:
+                        match_size = 0
+                        delta = 1
+
+                    token_bits = 1 + delta + w_bits
+
+                    while f_pos < token_bits:
+                        f_buf |= int(f.read(1)[0]) << (22 - f_pos)
+                        f_pos += 8
+
+                    # We now absolutely have enough data to decode token.
+
+                    match_size += min_pattern_size
+                    index = (f_buf >> (30 - token_bits)) & w_mask
+                    f_buf = (f_buf << token_bits) & full_mask
+                    f_pos -= token_bits
+
+                    # Copy entire match to overflow
+                    for i in range(match_size):
+                        overflow_buf_ptr[i] = w_buf_ptr[index + i]
+                    overflow_pos = 0
+                    overflow_size = match_size
+
+                    # Copy available amount to out_buf
+                    for i in range(match_size):
+                        if out_pos < out_capacity:
+                            out_buf[out_pos] = overflow_buf_ptr[overflow_pos]
+                            out_pos += 1
+                            overflow_pos += 1
+                            overflow_size -= 1
+                        # Copy overflow to window
+                        w_buf_ptr[(w_pos + i) & w_mask] = overflow_buf_ptr[i]
+                w_pos = (w_pos + match_size) & w_mask
+        except IndexError:
+            pass
+
+        self.w_pos = w_pos
+        self.overflow_pos = overflow_pos
+        self.overflow_size = overflow_size
+        self.f_buf = f_buf
+        self.f_pos = f_pos
+
+        return out_pos
+
+    def read(self, size=-1):
+        """Return at most ``size`` bytes."""
+        if size == 0:
+            out = bytearray()
+        elif size < 0:
+            # Read into a bunch of chunks, then do a single join.
+            chunks = []
+            chunk_size = 1024
+            decompressed_bytes = 0
+            while True:
+                chunk = bytearray(chunk_size)
+                chunk_decompressed_bytes = self.readinto(chunk)
+                if chunk_decompressed_bytes != chunk_size:
+                    chunk = chunk[:chunk_decompressed_bytes]
+                chunks.append(chunk)
+
+                if chunk_decompressed_bytes != chunk_size:
+                    break
+
+            out = b"".join(chunks)
+        else:
+            out = bytearray(size)
+            decompressed_bytes = self.readinto(out)
+            if decompressed_bytes != size:
+                out = out[:decompressed_bytes]
+
+        return out
+
+    def close(self):
+        if self._close_f_on_close:
+            self.f.close()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.close()
+
+
+class TextDecompressor(Decompressor):
+    def read(self, *args, **kwargs) -> str:
+        return super().read(*args, **kwargs).decode()
+
+
+def decompress(data, *args, **kwargs):
+    with BytesIO(data) as f:
+        d = Decompressor(f, *args, **kwargs)
+        return d.read()

hdiffpatch/_c_src/tamp/tests/test_bit_writer_reader.py

@@ -0,0 +1,59 @@
+import io
+import random
+import unittest
+
+from tamp.compressor import _BitWriter
+from tamp.decompressor import _BitReader
+
+
+class TestBitWriterAndReader(unittest.TestCase):
+    def test_auto_bit_writer_and_reader(self):
+        # Generate a list of random chunks of bits (1~16 bits)
+        num_chunks = 1000
+        n_bits = [random.randint(1, 16) for _ in range(num_chunks)]
+        data = []
+        for n_bit in n_bits:
+            mask = (1 << n_bit) - 1
+            data.append(random.randint(0, 1 << 32) & mask)
+        chunks = list(zip(data, n_bits))
+
+        # Write the chunks of bits using BitWriter
+        with io.BytesIO() as f:
+            writer = _BitWriter(f)
+            for bits, num_bits in chunks:
+                writer.write(bits, num_bits)
+            writer.flush(write_token=False)
+
+            # Read the chunks of bits back using BitReader
+            f.seek(0)
+            reader = _BitReader(f)
+            for original_bits, num_bits in chunks:
+                read_bits = reader.read(num_bits)
+                self.assertEqual(read_bits, original_bits)
+
+    def test_writer_correct_size_no_flush_token(self):
+        for i in range(1, 8 + 1):
+            with io.BytesIO() as f:
+                writer = _BitWriter(f)
+                writer.write(0xFFFF, i)
+                writer.flush(write_token=False)
+
+                self.assertEqual(f.tell(), 1)
+        for i in range(9, 16 + 1):
+            with io.BytesIO() as f:
+                writer = _BitWriter(f)
+                writer.write(0xFFFF, i)
+                writer.flush(write_token=False)
+
+                self.assertEqual(f.tell(), 2)
+
+
+class TestHuffmanReader(unittest.TestCase):
+    def test_always_valid(self):
+        """There should never be an instance where 8bits have been read and
+        no huffman code has been decoded. This verifies that.
+        """
+        random_bytes = bytes(random.randint(0, 255) for _ in range(1024 * 1024))
+        with io.BytesIO(random_bytes) as f:
+            reader = _BitReader(f)
+            reader.read_huffman()

hdiffpatch/_c_src/tamp/tests/test_cli.py

@@ -0,0 +1,78 @@
+import tempfile
+import unittest
+from pathlib import Path
+
+try:
+    import micropython
+except ImportError:
+    micropython = None
+
+try:
+    from unittest.mock import patch
+
+    from tamp.cli.main import app
+except ImportError:
+    pass
+
+compressed_foo_foo_foo = bytes(
+    # fmt: off
+    [
+        0b010_11_00_0,  # header (window_bits=10, literal_bits=8)
+        0b1_0110011,    # literal "f"
+        0b0_0_0_00100,  # the pre-init buffer contains "oo" at index 131
+                        # size=2 -> 0b0
+                        # 131 -> 0b0010000011
+        0b00011_1_00,   # literal " "
+        0b100000_0_1,   # There is now "foo " at index 0
+        0b000_00000,    # size=4 -> 0b1000
+        0b00000_0_11,   # Just "foo" at index 0; size=3 -> 0b11
+        0b00000000,     # index 0 -> 0b0000000000
+        0b00_000000,    # 6 bits of zero-padding
+    ]
+    # fmt: on
+)
+
+
+@unittest.skipIf(micropython is not None, "not running cpython")
+class TestCli(unittest.TestCase):
+    def test_compress_file_to_stdout(self):
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            tmp_dir = Path(tmp_dir)
+            test_file = tmp_dir / "test_input.bin"
+            test_file.write_bytes(b"foo foo foo")
+
+            with patch("sys.stdout.buffer.write") as mock_stdout:
+                app(["compress", str(test_file)])
+                mock_stdout.assert_called_once_with(compressed_foo_foo_foo)
+
+    def test_compress_stdin_to_stdout(self):
+        with patch("sys.stdout.buffer.write") as mock_stdout, patch(
+            "sys.stdin.buffer.read", return_value="foo foo foo"
+        ):
+            app("compress")
+            mock_stdout.assert_called_once_with(compressed_foo_foo_foo)
+
+    def test_decompress_file_to_stdout(self):
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            tmp_dir = Path(tmp_dir)
+            test_file = tmp_dir / "test_input.tamp"
+            test_file.write_bytes(compressed_foo_foo_foo)
+            with patch("sys.stdout.buffer.write") as mock_stdout:
+                app(["decompress", str(test_file)])
+                mock_stdout.assert_called_once_with(b"foo foo foo")
+
+    def test_decompress_stdin_to_stdout(self):
+        with patch("sys.stdout.buffer.write") as mock_stdout, patch(
+            "sys.stdin.buffer.read", return_value=compressed_foo_foo_foo
+        ):
+            app("decompress")
+            mock_stdout.assert_called_once_with(b"foo foo foo")
+
+    def test_decompress_stdin_to_file(self):
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            tmp_dir = Path(tmp_dir)
+            test_file = tmp_dir / "test_output.txt"
+
+            with patch("sys.stdin.buffer.read", return_value=compressed_foo_foo_foo):
+                app(["decompress", "-o", str(test_file)])
+            self.assertEqual(test_file.read_text(), "foo foo foo")