tamp 2.2.4__tar.gz → 2.3.0__tar.gz

{tamp-2.2.4 → tamp-2.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tamp
-Version: 2.2.4
+Version: 2.3.0
 Summary: 
 Home-page: https://github.com/BrianPugh/tamp
 License: Apache-2.0
@@ -343,13 +343,14 @@ low-memory compressor.
 
 To give an idea of Tamp's speed on an embedded device, the following table shows
 compression/decompression in **bytes/second of the first 100KB of enwik8 on a pi
-pico (rp2040)** at the default 125MHz clock rate. The C benchmark **does not**
+pico (rp2040)** at the default 125MHz clock rate, with native-module
+measurements taken on MicroPython v1.26.1 firmware. The C benchmark **does not**
 use a filesystem nor dynamic memory allocation, so it represents the maximum
 speed Tamp can achieve. In all tests, a 1KB window (10 bit) was used.
 
 |                                  | Compression (bytes/s) | Decompression (bytes/s) |
 | -------------------------------- | --------------------- | ----------------------- |
-| Tamp (Micropython Native Module) | 31,328                | 990,099                 |
+| Tamp (Micropython Native Module) | 34,510                | 980,392                 |
 | Tamp (C)                         | 36,127                | 1,400,600               |
 | Deflate (micropython builtin)    | 6,885                 | 294,985                 |
 
@@ -365,11 +366,11 @@ Numbers reported in bytes. Tamp sizes were measured using `arm-none-eabi-gcc`
 
 |                                  | Compressor | Decompressor | Compressor + Decompressor |
 | -------------------------------- | ---------- | ------------ | ------------------------- |
-| Tamp (MicroPython Native)        | 4700       | 4347         | 8024                      |
-| Tamp (C, no extended, no stream) | 1754       | 1656         | 3172                      |
-| Tamp (C, no extended)            | 2036       | 1894         | 3692                      |
-| Tamp (C, extended, no stream)    | 2838       | 2452         | 5052                      |
-| Tamp (C, extended)               | 3120       | 2690         | 5572                      |
+| Tamp (MicroPython Native)        | 5469       | 4725         | 9125                      |
+| Tamp (C, no extended, no stream) | 1932       | 1634         | 3328                      |
+| Tamp (C, no extended)            | 2214       | 1872         | 3848                      |
+| Tamp (C, extended, no stream)    | 3120       | 2452         | 5334                      |
+| Tamp (C, extended)               | 3402       | 2690         | 5854                      |
 | Heatshrink (C)                   | 2956       | 3876         | 6832                      |
 | uzlib (C)                        | 2355       | 3963         | 6318                      |
 

{tamp-2.2.4 → tamp-2.3.0}/README.md

@@ -322,13 +322,14 @@ low-memory compressor.
 
 To give an idea of Tamp's speed on an embedded device, the following table shows
 compression/decompression in **bytes/second of the first 100KB of enwik8 on a pi
-pico (rp2040)** at the default 125MHz clock rate. The C benchmark **does not**
+pico (rp2040)** at the default 125MHz clock rate, with native-module
+measurements taken on MicroPython v1.26.1 firmware. The C benchmark **does not**
 use a filesystem nor dynamic memory allocation, so it represents the maximum
 speed Tamp can achieve. In all tests, a 1KB window (10 bit) was used.
 
 |                                  | Compression (bytes/s) | Decompression (bytes/s) |
 | -------------------------------- | --------------------- | ----------------------- |
-| Tamp (Micropython Native Module) | 31,328                | 990,099                 |
+| Tamp (Micropython Native Module) | 34,510                | 980,392                 |
 | Tamp (C)                         | 36,127                | 1,400,600               |
 | Deflate (micropython builtin)    | 6,885                 | 294,985                 |
 
@@ -344,11 +345,11 @@ Numbers reported in bytes. Tamp sizes were measured using `arm-none-eabi-gcc`
 
 |                                  | Compressor | Decompressor | Compressor + Decompressor |
 | -------------------------------- | ---------- | ------------ | ------------------------- |
-| Tamp (MicroPython Native)        | 4700       | 4347         | 8024                      |
-| Tamp (C, no extended, no stream) | 1754       | 1656         | 3172                      |
-| Tamp (C, no extended)            | 2036       | 1894         | 3692                      |
-| Tamp (C, extended, no stream)    | 2838       | 2452         | 5052                      |
-| Tamp (C, extended)               | 3120       | 2690         | 5572                      |
+| Tamp (MicroPython Native)        | 5469       | 4725         | 9125                      |
+| Tamp (C, no extended, no stream) | 1932       | 1634         | 3328                      |
+| Tamp (C, no extended)            | 2214       | 1872         | 3848                      |
+| Tamp (C, extended, no stream)    | 3120       | 2452         | 5334                      |
+| Tamp (C, extended)               | 3402       | 2690         | 5854                      |
 | Heatshrink (C)                   | 2956       | 3876         | 6832                      |
 | uzlib (C)                        | 2355       | 3963         | 6318                      |
 

{tamp-2.2.4 → tamp-2.3.0}/build.py

@@ -20,6 +20,11 @@ if profile and sanitize:
         "Please choose either TAMP_PROFILE=1 or TAMP_SANITIZE=1, not both."
     )
 
+# Sanitizer flags below are UNIX-only; previously this path crashed with a
+# NameError (extra_link_args unbound) instead of a clear message.
+if sanitize and os.name == "nt":
+    raise ValueError("TAMP_SANITIZE=1 is not supported on Windows.")
+
 
 def build_cython_extensions():
     import Cython.Compiler.Options

{tamp-2.2.4 → tamp-2.3.0}/pyproject.toml

@@ -9,7 +9,7 @@ style = "semver"
 
 [tool.poetry]
 name = "tamp"
-version = "2.2.4"  # Do not change, let poetry-dynamic-versioning handle it.
+version = "2.3.0"  # Do not change, let poetry-dynamic-versioning handle it.
 homepage = "https://github.com/BrianPugh/tamp"
 repository = "https://github.com/BrianPugh/tamp"
 license = "Apache-2.0"
@@ -56,12 +56,12 @@ pytest-cov = ">=3.0.0"
 pytest-mock = ">=3.7.0"
 black = ">=23.1.0"
 belay = ">=0.21.3"
-tqdm = ">=4.65.0"
+tqdm = ">=4.68.2"
 matplotlib = ">=3.7.4"
 ipdb = ">=0.13.13"
 packaging = ">=26.1"
 mpremote = ">=1.20.0"
-cython = ">=3.2.5"
+cython = ">=3.2.8"
 pytest-subtests = ">=0.10.0"
 numpy = ">=1.24"
 pyelftools = "^0.32"

{tamp-2.2.4 → tamp-2.3.0}/tamp/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "2.2.4"
+__version__ = "2.3.0"
 
 
 class ExcessBitsError(Exception):

{tamp-2.2.4 → tamp-2.3.0}/tamp/_c_compressor.pyx

@@ -1,11 +1,9 @@
-import builtins
 cimport ctamp
 from libcpp cimport bool
 from cpython.exc cimport PyErr_CheckSignals
 from cpython.mem cimport PyMem_Malloc, PyMem_Free
 from libc.stddef cimport size_t
 from io import BytesIO
-from . import bit_size
 from ._c_common cimport CHUNK_SIZE
 from ._c_common import ERROR_LOOKUP
 
@@ -42,7 +40,9 @@ cdef class Compressor:
     ):
         cdef ctamp.TampConf conf
 
-        if dictionary and bit_size(len(dictionary) - 1) != window:
+        if dictionary is not None and len(dictionary) != (1 << window):
+            # The exact size is required: a too-small buffer used as the window
+            # would let the C compressor write out of bounds.
             raise ValueError("Dictionary-window size mismatch.")
 
         if not hasattr(f, "write"):  # It's probably a path-like object.
@@ -55,7 +55,7 @@ cdef class Compressor:
 
         conf.window = window
         conf.literal = literal
-        conf.use_custom_dictionary = builtins.bool(dictionary)
+        conf.use_custom_dictionary = dictionary is not None
         # Set lazy_matching - this field is conditionally compiled based on TAMP_LAZY_MATCHING
         # The build system defines this macro, so the field should be available
         conf.lazy_matching = lazy_matching
@@ -63,7 +63,7 @@ cdef class Compressor:
         conf.dictionary_reset = dictionary_reset
         conf.append = append
 
-        self._window_buffer = dictionary if dictionary else bytearray(1 << window)
+        self._window_buffer = dictionary if dictionary is not None else bytearray(1 << window)
         self._window_buffer_ptr = <unsigned char *>self._window_buffer
 
         self._dictionary_reset = dictionary_reset
@@ -75,15 +75,23 @@ cdef class Compressor:
         cdef:
             ctamp.tamp_res res
 
-            bytearray output_buffer = bytearray(CHUNK_SIZE)
-            unsigned char *output_buffer_ptr = output_buffer
-            const unsigned char* data_ptr = &data[0]
+            bytearray output_buffer
+            unsigned char *output_buffer_ptr
+            const unsigned char* data_ptr
 
             size_t input_consumed_size = 0
             size_t input_remaining_size = data.shape[0]
             size_t output_buffer_written_size
             int written_to_disk_size = 0
 
+        if input_remaining_size == 0:
+            # &data[0] on an empty buffer raises IndexError.
+            return 0
+
+        output_buffer = bytearray(CHUNK_SIZE)
+        output_buffer_ptr = output_buffer
+        data_ptr = &data[0]
+
         output_buffer_mv = memoryview(output_buffer)
 
         while input_remaining_size:

{tamp-2.2.4 → tamp-2.3.0}/tamp/_c_decompressor.pyx

@@ -3,7 +3,6 @@ from cpython.exc cimport PyErr_CheckSignals
 from cpython.mem cimport PyMem_Malloc, PyMem_Free
 from libc.stddef cimport size_t
 from io import BytesIO
-from . import bit_size
 from ._c_common cimport CHUNK_SIZE
 from ._c_common import ERROR_LOOKUP
 
@@ -64,7 +63,13 @@ cdef class Decompressor:
         if conf.use_custom_dictionary and dictionary is None:
             raise ValueError
 
-        self._window_buffer = dictionary if dictionary else bytearray(1 << conf.window)
+        if dictionary is not None and len(dictionary) < (1 << conf.window):
+            # A too-small buffer used as the window would let the C
+            # decompressor write out of bounds. An oversized buffer is fine:
+            # C uses only the first 2**window bytes, in place.
+            raise ValueError("Dictionary-window size mismatch.")
+
+        self._window_buffer = dictionary if dictionary is not None else bytearray(1 << conf.window)
         self._window_buffer_ptr = <unsigned char *>self._window_buffer
 
         res = ctamp.tamp_decompressor_init(self._c_decompressor, &conf, self._window_buffer_ptr, conf.window)
@@ -105,7 +110,17 @@ cdef class Decompressor:
 
             if res == ctamp.TAMP_INPUT_EXHAUSTED:
                 # Read in more data
-                self.input_size = self.f.readinto(self.input_buffer)
+                if hasattr(self.f, "readinto"):
+                    self.input_size = self.f.readinto(self.input_buffer)
+                else:
+                    # Fall back for file-like objects that only implement read().
+                    chunk = self.f.read(CHUNK_SIZE)
+                    self.input_size = len(chunk)
+                    if self.input_size > <size_t>CHUNK_SIZE:
+                        # A larger chunk would resize (reallocate) input_buffer,
+                        # leaving input_buffer_ptr dangling.
+                        raise ValueError("read() returned more bytes than requested.")
+                    self.input_buffer[:self.input_size] = chunk
                 self.input_consumed = 0
                 if self.input_size == 0:
                     break;
@@ -137,7 +152,9 @@ cdef class Decompressor:
             # decompression happens inside of readinto
             read_size = self.readinto(buf)
             if size > 0:
-                # We one-shot the decompression
+                # We one-shot the decompression.
+                # Trim unwritten zero-padding when the stream ended early.
+                del buf[read_size:]
                 out.append(buf)
                 break
             # We are decompressing in chunks

{tamp-2.2.4 → tamp-2.3.0}/tamp/_c_src/tamp/compressor.c

@@ -121,8 +121,29 @@ static TAMP_NOINLINE void find_best_match(TampCompressor* compressor, uint16_t*
     const uint8_t max_pattern_size = MIN(compressor->input_size, MAX_PATTERN_SIZE);
     const unsigned char* window = compressor->window;
 
+#if defined(__GNUC__)
+    // Word-at-a-time skip over bytes that can't start a match: ~3 cycles/byte
+    // instead of ~9 on Cortex-M0+, and the scan dominates compression time.
+    const uint32_t first_broadcast = first_byte * 0x01010101u;
+#endif
+
     for (uint32_t window_index = 0; window_index < window_size_minus_1; window_index++) {
         if (TAMP_LIKELY(window[window_index] != first_byte)) {
+#if defined(__GNUC__)
+            uint32_t wi = window_index + 1;
+            if (TAMP_LIKELY(((uintptr_t)(window + wi) & 3) == 0)) {
+                // Aligned: skip 4 bytes per iteration until a word may hold
+                // first_byte (zero-byte detect on the XOR; false positives
+                // are fine, the byte loop re-checks).
+                typedef uint32_t __attribute__((may_alias)) tamp_word_alias;
+                while (wi + 4 <= window_size_minus_1) {
+                    uint32_t word = *(const tamp_word_alias*)(window + wi) ^ first_broadcast;
+                    if ((word - 0x01010101u) & ~word & 0x80808080u) break;
+                    wi += 4;
+                }
+            }
+            window_index = wi - 1;  // loop increment advances to wi
+#endif
             continue;
         }
         if (TAMP_LIKELY(window[window_index + 1] != second_byte)) {
@@ -267,6 +288,12 @@ static TAMP_NOINLINE TAMP_OPTIMIZE_SIZE uint8_t get_last_window_byte(TampCompres
  * @param[out] new_pos Position of found match (only valid if new_count > current_count)
  * @param[out] new_count Length of found match
  */
+#if TAMP_ESP32
+/* ESP32-optimized implementation in espidf/tamp/compressor_esp32.cpp (same contract,
+ * but may return a different equally-long match position on ties). */
+extern void find_extended_match(TampCompressor* compressor, uint16_t current_pos, uint8_t current_count,
+                                uint16_t* new_pos, uint8_t* new_count);
+#else
 static TAMP_NOINLINE TAMP_OPTIMIZE_SIZE void find_extended_match(TampCompressor* compressor, uint16_t current_pos,
                                                                  uint8_t current_count, uint16_t* new_pos,
                                                                  uint8_t* new_count) {
@@ -304,6 +331,7 @@ static TAMP_NOINLINE TAMP_OPTIMIZE_SIZE void find_extended_match(TampCompressor*
         }
     }
 }
+#endif  // TAMP_ESP32
 
 /**
  * @brief Write RLE token to bit buffer and update window.
@@ -396,13 +424,20 @@ static TAMP_NOINLINE tamp_res write_extended_match_token(TampCompressor* compres
  * significant code size on register-constrained Cortex-M0+ where the compiler
  * otherwise spills heavily to stack (~48 bytes saved on armv6m).
  *
+ * @param[out] match_index Pre-computed match position; only set when a match was
+ *             already found while deciding RLE-vs-pattern (see match_size).
+ * @param[out] match_size Set (>= 2) together with match_index when returning
+ *             TAMP_POLL_CONTINUE with a match already found, so the caller can skip
+ *             its own find_best_match. Left untouched otherwise.
+ *
  * @return TAMP_OK if fully handled (caller should return TAMP_OK),
  *         TAMP_POLL_CONTINUE if caller should proceed to normal pattern matching,
  *         other tamp_res on error.
  */
 static TAMP_NOINLINE TAMP_OPTIMIZE_SIZE tamp_res poll_extended_handling(TampCompressor* compressor,
                                                                         unsigned char** output, size_t* output_size,
-                                                                        size_t* output_written_size) {
+                                                                        size_t* output_written_size,
+                                                                        uint16_t* match_index, uint8_t* match_size) {
     // Handle extended match continuation
     if (compressor->extended_match_count) {
         const uint8_t max_ext_match = compressor->min_pattern_size + 11 + EXTENDED_MATCH_MAX_EXTRA;
@@ -460,6 +495,9 @@ static TAMP_NOINLINE TAMP_OPTIMIZE_SIZE tamp_res poll_extended_handling(TampComp
 
             if (pattern_size > total_rle) {
                 compressor->rle_count = 0;
+                // Hand the match to the caller so it doesn't repeat the search.
+                *match_index = pattern_index;
+                *match_size = pattern_size;
                 return TAMP_POLL_CONTINUE;  // Proceed to pattern matching
             }
         }
@@ -471,7 +509,18 @@ static TAMP_NOINLINE TAMP_OPTIMIZE_SIZE tamp_res poll_extended_handling(TampComp
         return TAMP_OK;
     }
 
-    if (total_rle == 1) compressor->rle_count = 0;
+    if (TAMP_UNLIKELY(compressor->rle_count == 1)) {
+        // A lone run byte was consumed into rle_count by a previous poll and the
+        // run then ended; RLE tokens require count >= 2 and the byte is no longer
+        // in the input buffer, so re-emit it as a literal (mirrors the
+        // rle_count == 1 drain in tamp_compressor_flush).
+        const uint16_t window_mask = (1 << compressor->conf.window) - 1;
+        write_to_bit_buffer(compressor, IS_LITERAL_FLAG | last_byte, compressor->conf.literal + 1);
+        compressor->window[compressor->window_pos] = last_byte;
+        compressor->window_pos = (compressor->window_pos + 1) & window_mask;
+        compressor->rle_count = 0;
+        return TAMP_OK;
+    }
     return TAMP_POLL_CONTINUE;  // Proceed to pattern matching
 }
 #endif  // TAMP_EXTENDED_COMPRESS
@@ -509,9 +558,17 @@ TAMP_NOINLINE tamp_res tamp_compressor_poll(TampCompressor* compressor, unsigned
 
 #if TAMP_EXTENDED_COMPRESS
     if (TAMP_UNLIKELY(compressor->conf.extended)) {
-        // Handle extended match continuation + RLE (outlined for code size)
-        res = poll_extended_handling(compressor, &output, &output_size, output_written_size);
-        if (res != TAMP_POLL_CONTINUE) return res;
+        // Handle extended match continuation + RLE (outlined for code size).
+        // May pre-compute match_index/match_size (match_size becomes non-zero).
+        res = poll_extended_handling(compressor, &output, &output_size, output_written_size, &match_index, &match_size);
+        if (res != TAMP_POLL_CONTINUE) {
+#if TAMP_LAZY_MATCHING
+            // Extended handling consumed input and/or mutated the window, so any
+            // match cached by lazy matching no longer matches the current input.
+            compressor->cached_match_index = -1;
+#endif
+            return res;
+        }
         // TAMP_POLL_CONTINUE: proceed to pattern matching below
     }
 #endif  // TAMP_EXTENDED_COMPRESS
@@ -523,7 +580,7 @@ TAMP_NOINLINE tamp_res tamp_compressor_poll(TampCompressor* compressor, unsigned
             match_index = compressor->cached_match_index;
             match_size = compressor->cached_match_size;
             compressor->cached_match_index = -1;  // Clear cache after using
-        } else {
+        } else if (match_size == 0) {             // No match pre-computed by extended handling
             find_best_match(compressor, &match_index, &match_size);
         }
 
@@ -557,11 +614,11 @@ TAMP_NOINLINE tamp_res tamp_compressor_poll(TampCompressor* compressor, unsigned
         } else {
             compressor->cached_match_index = -1;  // Clear cache
         }
-    } else {
+    } else if (match_size == 0) {  // No match pre-computed by extended handling
         find_best_match(compressor, &match_index, &match_size);
     }
 #else
-    find_best_match(compressor, &match_index, &match_size);
+    if (match_size == 0) find_best_match(compressor, &match_index, &match_size);
 #endif
 
     // Shared token/literal writing logic
@@ -825,6 +882,12 @@ TAMP_OPTIMIZE_SIZE tamp_res tamp_compressor_reset_dictionary(TampCompressor* com
 
 #if TAMP_STREAM
 
+#if TAMP_EXTENDED_COMPRESS && (TAMP_STREAM_WORK_BUFFER_SIZE / 2) < EXTENDED_MATCH_MIN_OUTPUT_BYTES
+// A smaller buffer deadlocks tamp_compress_stream: write_extended_match_token
+// returns TAMP_OUTPUT_FULL forever because the token can never fit.
+#error "TAMP_STREAM_WORK_BUFFER_SIZE must be at least 12 bytes when extended compression is enabled"
+#endif
+
 TAMP_OPTIMIZE_SIZE tamp_res tamp_compress_stream(TampCompressor* compressor, tamp_read_t read_cb, void* read_handle,
                                                  tamp_write_t write_cb, void* write_handle, size_t* input_consumed_size,
                                                  size_t* output_written_size, tamp_callback_t callback,

{tamp-2.2.4 → tamp-2.3.0}/tamp/_c_src/tamp/compressor_find_match_desktop.c

@@ -88,8 +88,12 @@ static inline void find_best_match(TampCompressor *compressor, uint16_t *match_i
     const uint8_t max_pattern_size = MIN(compressor->input_size, MAX_PATTERN_SIZE);
     const unsigned char *window = compressor->window;
 
-    // Pre-load input bytes into linear array to avoid repeated modular arithmetic
-    uint8_t input_bytes[16];
+    // Pre-load input bytes into linear array to avoid repeated modular arithmetic.
+    // Zero-initialized: entries at max_pattern_size and beyond are read when
+    // assembling input_word_ext below. Their values never affect the result (the
+    // max_pattern_size >= 10 guard in EXTEND_MATCH sees to that), but reading
+    // indeterminate bytes is UB (MSan finding, -Werror=maybe-uninitialized).
+    uint8_t input_bytes[16] = {0};
     for (uint8_t i = 0; i < max_pattern_size && i < 16; i++) {
         input_bytes[i] = read_input(i);
     }

{tamp-2.2.4 → tamp-2.3.0}/tamp/_c_src/tamp/decompressor.c

@@ -2,6 +2,29 @@
 
 #include "common.h"
 
+/* Copy primitives for the hot paths. Platform-specific implementations live
+ * with their platform component; TAMP_ESP32 builds already require the espidf
+ * component sources on the include path (same contract as the extern
+ * find_best_match in compressor.c). On generic targets the macros expand to
+ * the plain byte loops / tamp_window_copy call they replaced (macros, not
+ * inline functions: Cortex-M0+ codegen is sensitive to function boundaries). */
+#if TAMP_ESP32
+#include "private/tamp_copy.h"
+#else
+
+/* Copy count bytes from src to the output cursor and advance it. */
+#define TAMP_COPY_TO_OUTPUT(out, src, count)                      \
+    do {                                                          \
+        for (uint8_t _tamp_i = 0; _tamp_i < (count); _tamp_i++) { \
+            *(out)++ = (src)[_tamp_i];                            \
+        }                                                         \
+    } while (0)
+
+#define TAMP_WINDOW_COPY(window, window_pos, window_offset, match_size, window_mask) \
+    tamp_window_copy((window), (window_pos), (window_offset), (match_size), (window_mask))
+
+#endif /* TAMP_ESP32 */
+
 #define MAX(x, y) (((x) > (y)) ? (x) : (y))
 #define MIN(x, y) (((x) < (y)) ? (x) : (y))
 
@@ -233,9 +256,7 @@ static tamp_res decode_extended_match(TampDecompressor* d, unsigned char** outpu
 
     /* Copy from window to output */
     uint16_t src_offset = window_offset + skip;
-    for (uint8_t i = 0; i < to_write; i++) {
-        *(*output)++ = d->window[src_offset + i];
-    }
+    TAMP_COPY_TO_OUTPUT(*output, d->window + src_offset, to_write);
     *output_written_size += to_write;
 
     /* Update window only on complete decode.
@@ -244,7 +265,7 @@ static tamp_res decode_extended_match(TampDecompressor* d, unsigned char** outpu
         uint16_t wp = d->window_pos;
         uint16_t remaining = window_size - wp;
         uint8_t window_write = (match_size < remaining) ? match_size : remaining;
-        tamp_window_copy(d->window, &wp, window_offset, window_write, window_size - 1);
+        TAMP_WINDOW_COPY(d->window, &wp, window_offset, window_write, window_size - 1);
         d->window_pos = wp;
     }
 
@@ -288,6 +309,12 @@ static TAMP_OPTIMIZE_SIZE tamp_res tamp_decompressor_populate_from_conf(TampDeco
     if (conf_window < 8 || conf_window > 15) return TAMP_INVALID_CONF;
     if (conf_literal < 5 || conf_literal > 8) return TAMP_INVALID_CONF;
     if (conf_window > decompressor->window_bits_max) return TAMP_INVALID_CONF;
+#if !TAMP_EXTENDED_DECOMPRESS
+    // Reject before committing any state: marking the decompressor configured and
+    // then returning an error would let a retrying caller decode the extended
+    // stream as classic format.
+    if (conf_extended) return TAMP_INVALID_CONF;  // Extended stream but extended support not compiled in
+#endif
     if (!conf_use_custom_dictionary)
         tamp_initialize_dictionary(decompressor->window, (size_t)1 << conf_window, conf_extended ? conf_literal : 8);
 
@@ -297,9 +324,6 @@ static TAMP_OPTIMIZE_SIZE tamp_res tamp_decompressor_populate_from_conf(TampDeco
     decompressor->configured = true;
     decompressor->conf_extended = conf_extended;
     decompressor->conf_dictionary_reset = conf_dictionary_reset;
-#if !TAMP_EXTENDED_DECOMPRESS
-    if (conf_extended) return TAMP_INVALID_CONF;  // Extended stream but extended support not compiled in
-#endif
 
     return TAMP_OK;
 }
@@ -538,14 +562,12 @@ tamp_res tamp_decompressor_decompress_cb(TampDecompressor* decompressor, unsigne
             }
 
             // Copy pattern to output
-            for (uint8_t i = 0; i < match_size_skip; i++) {
-                *output++ = decompressor->window[window_offset_skip + i];
-            }
+            TAMP_COPY_TO_OUTPUT(output, decompressor->window + window_offset_skip, match_size_skip);
             (*output_written_size) += match_size_skip;
 
             if (TAMP_LIKELY(decompressor->skip_bytes == 0)) {
                 uint16_t wp = decompressor->window_pos;
-                tamp_window_copy(decompressor->window, &wp, window_offset, match_size, window_mask);
+                TAMP_WINDOW_COPY(decompressor->window, &wp, window_offset, match_size, window_mask);
                 decompressor->window_pos = wp;
             }
         }

{tamp-2.2.4 → tamp-2.3.0}/tamp/cli/build_dictionary.py

@@ -7,8 +7,30 @@ from typing import Annotated, Optional
 from cyclopts import Parameter, validators
 
 import tamp
-from tamp._c_build_dictionary import score_and_multi_frag, select_candidates
-from tamp._c_build_dictionary import score_substrings as _c_score_substrings
+
+_MISSING_KERNELS_MESSAGE = (
+    "Building dictionaries requires tamp's compiled C extensions, which are not available in this installation."
+)
+
+try:
+    from tamp._c_build_dictionary import score_and_multi_frag, select_candidates
+    from tamp._c_build_dictionary import score_substrings as _c_score_substrings
+
+    _KERNELS_AVAILABLE = True
+except ImportError as e:
+    # C extensions unavailable (e.g. TAMP_BUILD_C_EXTENSIONS=0). Importing this
+    # module must still succeed so the rest of the tamp CLI works; only
+    # dictionary building itself requires the compiled kernels.
+    _KERNELS_AVAILABLE = False
+    _IMPORT_ERROR = e
+
+    def _missing_kernels(*args, **kwargs):
+        raise RuntimeError(_MISSING_KERNELS_MESSAGE) from _IMPORT_ERROR
+
+    score_and_multi_frag = _missing_kernels
+    select_candidates = _missing_kernels
+    _c_score_substrings = _missing_kernels
+
 from tamp.compressor import (
     _EXTENDED_MATCH_SYMBOL,
     _LEADING_EXTENDED_MATCH_HUFFMAN_BITS,
@@ -760,6 +782,9 @@ def build_dictionary_cli(
     quiet: bool
         Suppress the tradeoff table and summary output.
     """
+    if not _KERNELS_AVAILABLE:
+        raise RuntimeError(_MISSING_KERNELS_MESSAGE)
+
     dictionary_size = 1 << window
 
     # Materialize corpus for reuse in compression analysis.

{tamp-2.2.4 → tamp-2.3.0}/tamp/compressor.py

@@ -19,7 +19,7 @@ try:
 except ImportError:
     sz = None
 
-from . import ExcessBitsError, bit_size, compute_min_pattern_size, initialize_dictionary
+from . import ExcessBitsError, compute_min_pattern_size, initialize_dictionary
 
 # encodes [0, 14] pattern lengths
 _huffman_codes = b"\x00\x03\x08\x0b\x14$&+KT\x94\x95\xaa'\xab"
@@ -199,7 +199,9 @@ class Compressor:
             close_f_on_close = False
 
         self._bit_writer = _BitWriter(f, close_f_on_close=close_f_on_close)
-        if dictionary and bit_size(len(dictionary) - 1) != window:
+        if dictionary is not None and len(dictionary) != (1 << window):
+            # The exact size is required: a too-small buffer used as the window
+            # would be indexed out of bounds.
             raise ValueError("Dictionary-window size mismatch.")
 
         if self.extended:
@@ -244,7 +246,7 @@ class Compressor:
             # Write header
             self._bit_writer.write(window - 8, 3, flush=False)
             self._bit_writer.write(literal - 5, 2, flush=False)
-            self._bit_writer.write(bool(dictionary), 1, flush=False)
+            self._bit_writer.write(dictionary is not None, 1, flush=False)
             self._bit_writer.write(self.extended, 1, flush=False)
             self._bit_writer.write(
                 1 if dictionary_reset else 0, 1, flush=False
@@ -258,7 +260,7 @@ class Compressor:
         self._window_buffer = _RingBuffer(
             buffer=(
                 dictionary
-                if dictionary
+                if dictionary is not None
                 else initialize_dictionary(1 << self.window_bits, literal=self.literal_bits if self.extended else 8)
             ),
         )
@@ -287,6 +289,9 @@ class Compressor:
         self._last_was_flush = False
 
         if self._extended_match_count:
+            # This block consumes input and/or mutates the window, so any match
+            # cached by lazy matching no longer matches the current input.
+            self._cached_match_index = -1
             while self._input_buffer:
                 if (self._extended_match_position + self._extended_match_count) >= self._window_buffer.size:
                     # Reached window boundary - emit match (no wrap-around, only 0.02% compression loss)
@@ -332,6 +337,7 @@ class Compressor:
 
             # If RLE hasn't ended and we haven't hit max, consume and wait for more
             if not rle_ended and total_rle > 0:
+                self._cached_match_index = -1  # Input consumed; cached lazy match is stale.
                 self._rle_count = total_rle
                 for _ in range(rle_available):
                     self._input_buffer.popleft()
@@ -353,6 +359,7 @@ class Compressor:
 
                 if not use_pattern:
                     # Use RLE - consume bytes and write token
+                    self._cached_match_index = -1  # Input consumed; cached lazy match is stale.
                     for _ in range(rle_available):
                         self._input_buffer.popleft()
                     self._rle_count = total_rle
@@ -360,7 +367,15 @@ class Compressor:
                     return bytes_written
                 self._rle_count = 0
             elif total_rle == 1:
-                # Single byte - not worth RLE, will be handled as literal/pattern
+                if self._rle_count == 1:
+                    # The lone run byte was consumed into _rle_count by a previous
+                    # cycle and the run then ended; it is no longer in the input
+                    # buffer, so emit it (_write_rle writes a count-1 run as a literal).
+                    self._cached_match_index = -1
+                    bytes_written += self._write_rle()
+                    return bytes_written
+                # Single byte still in the input buffer - not worth RLE, will be
+                # handled as literal/pattern
                 self._rle_count = 0
 
         # Normal pattern matching

{tamp-2.2.4 → tamp-2.3.0}/tamp/compressor_viper.py

@@ -5,7 +5,7 @@ from io import BytesIO
 import micropython
 from micropython import const
 
-from . import ExcessBitsError, bit_size, compute_min_pattern_size, initialize_dictionary
+from . import ExcessBitsError, compute_min_pattern_size, initialize_dictionary
 
 # encodes [2, 15] pattern lengths
 _HUFFMAN_CODES = b"\x00\x03\x08\x0b\x14$&+KT\x94\x95\xaa'"
@@ -38,9 +38,9 @@ class Compressor:
         self.min_pattern_size = compute_min_pattern_size(window, literal)
 
         # Window Buffer
-        if dictionary:
-            if bit_size(len(dictionary) - 1) != window:
-                raise ValueError
+        if dictionary is not None:
+            if len(dictionary) != (1 << window):
+                raise ValueError("Dictionary-window size mismatch.")
             self.window_buf = dictionary
         else:
             self.window_buf = initialize_dictionary(1 << window)
@@ -53,7 +53,7 @@ class Compressor:
 
         # Write header
         self.f = f
-        self.f_buf = ((window - 8) << 5 | (literal - 5) << 3 | int(bool(dictionary)) << 2) << (22)
+        self.f_buf = ((window - 8) << 5 | (literal - 5) << 3 | int(dictionary is not None) << 2) << (22)
         self.f_pos = 8
 
     @micropython.viper
@@ -95,9 +95,14 @@ class Compressor:
                     continue  # Small Speed short-cut
 
                 current_match_size = int(2)
-                for k in range(current_match_size, input_size):
+                # Clamp to the window edge so the ptr8 read below can't go
+                # out of bounds; keeps the hot loop to a single comparison.
+                max_k = window_size - window_index
+                if max_k > input_size:
+                    max_k = input_size
+                for k in range(current_match_size, max_k):
                     input_index = (input_pos + k) & 0xF
-                    if input_buf[input_index] != window_buf[window_index + k] or window_index + k >= window_size:
+                    if input_buf[input_index] != window_buf[window_index + k]:
                         break
                     current_match_size = k + 1
                 if current_match_size > match_size:

{tamp-2.2.4 → tamp-2.3.0}/tamp/decompressor.py

@@ -111,9 +111,10 @@ class _BitReader:
 
 
 class _RingBuffer:
-    def __init__(self, buffer):
+    def __init__(self, buffer, size=None):
         self.buffer = buffer
-        self.size = len(buffer)
+        # The window may occupy only a prefix of an oversized buffer.
+        self.size = len(buffer) if size is None else size
         self.pos = 0  # Always pointing to the byte-to-be-overwritten
         self.index = self.buffer.index
 
@@ -163,7 +164,10 @@ class Decompressor:
         dictionary: Optional[bytearray]
             Use the given **initialized** buffer inplace.
             At compression time, the same initialized buffer must be provided.
-            Decompression stream's ``window`` must agree with the dictionary size.
+            Must be at least ``2**window`` bytes long for the stream's window
+            (read from the header). The first ``2**window`` bytes are used in
+            place as the decompression window; bytes past that are never read
+            or written.
             If providing a pre-allocated buffer, but with default initialization, it must
             first be initialized with :func:`~tamp.initialize_dictionary`
         """
@@ -193,12 +197,27 @@ class Decompressor:
         if uses_custom_dictionary and dictionary is None:
             raise ValueError
 
+        window_size = 1 << self.window_bits
+        if dictionary is not None:
+            if len(dictionary) < window_size:
+                raise ValueError("Dictionary-window size mismatch.")
+            if not uses_custom_dictionary:
+                # Stream does not use a custom dictionary: initialize the
+                # supplied buffer's window region in place instead of using its
+                # contents verbatim, matching the C implementation.
+                literal = self.literal_bits if self.extended else 8
+                if len(dictionary) == window_size:
+                    initialize_dictionary(dictionary, literal=literal)
+                else:
+                    dictionary[:window_size] = initialize_dictionary(window_size, literal=literal)
+
         self._window_buffer = _RingBuffer(
             buffer=(
                 dictionary
-                if dictionary
-                else initialize_dictionary(1 << self.window_bits, literal=self.literal_bits if self.extended else 8)
+                if dictionary is not None
+                else initialize_dictionary(window_size, literal=self.literal_bits if self.extended else 8)
             ),
+            size=window_size,
         )
 
         self.min_pattern_size = compute_min_pattern_size(self.window_bits, self.literal_bits)
@@ -335,7 +354,10 @@ class Decompressor:
             read_size = self.readinto(buf)
             if size > 0:
                 # Read the entire contents in one go.
-                out.append(buf)
+                # Trim unwritten zero-padding when the stream ended early.
+                # Slice instead of ``del buf[read_size:]``: MicroPython
+                # bytearrays don't support item deletion.
+                out.append(buf if read_size == len(buf) else buf[:read_size])
                 break
             else:
                 if read_size < len(buf):

{tamp-2.2.4 → tamp-2.3.0}/tamp/decompressor_viper.py

@@ -35,6 +35,10 @@ class Decompressor:
         if uses_custom_dictionary:
             if not dictionary:
                 raise ValueError
+            if len(dictionary) != (1 << self.w_bits):
+                # A wrong-size buffer used as the window would make the unchecked
+                # ptr8 window writes below go out of bounds.
+                raise ValueError("Dictionary-window size mismatch.")
             self.w_buf = dictionary
         else:
             self.w_buf = initialize_dictionary(1 << self.w_bits)
@@ -140,6 +144,10 @@ class Decompressor:
 
                     match_size += min_pattern_size
                     index = (f_buf >> (30 - token_bits)) & w_mask
+                    if index + match_size > w_mask + 1:
+                        # Corrupt stream: the match would read past the window
+                        # buffer (the C implementation returns TAMP_OOB here).
+                        raise ValueError("out-of-bounds window reference")
                     f_buf = (f_buf << token_bits) & full_mask
                     f_pos -= token_bits