zlib-streams 1.0.7 → 1.1.0

package/src/{inffast.c → inffast_chunk.c}

@@ -1,12 +1,14 @@
-/* inffast.c -- fast decoding
- * Copyright (C) 1995-2017 Mark Adler
+/* inffast_chunk.c -- fast decoding
+ * Copyright (C) 1995-2026 Mark Adler
+ * Copyright 2023 The Chromium Authors
  * For conditions of distribution and use, see copyright notice in zlib.h
  */
 
 #include "zutil.h"
 #include "inftrees.h"
 #include "inflate.h"
-#include "inffast.h"
+#include "inffast_chunk.h"
+#include "chunkcopy.h"
 
 #ifdef ASMINF
 #  pragma message("Assembler code may have bugs -- use at your own risk")
@@ -18,15 +20,20 @@
    available, an end-of-block is encountered, or a data error is encountered.
    When large enough input and output buffers are supplied to inflate(), for
    example, a 16K input buffer and a 64K output buffer, more than 95% of the
-   inflate execution time is spent in this routine.
+   inflate() execution time is spent in this routine.
 
    Entry assumptions:
 
         state->mode == LEN
-        strm->avail_in >= 6
-        strm->avail_out >= 258
+        strm->avail_in >= INFLATE_FAST_MIN_INPUT (6 or 8 bytes + 7 bytes)
+        strm->avail_out >= INFLATE_FAST_MIN_OUTPUT (258 bytes + 2 bytes)
         start >= strm->avail_out
         state->bits < 8
+        (state->hold >> state->bits) == 0
+        strm->next_out[0..strm->avail_out] does not overlap with
+              strm->next_in[0..strm->avail_in]
+        strm->state->window is allocated with an additional
+              CHUNKCOPY_CHUNK_SIZE-1 bytes of padding beyond strm->state->wsize
 
    On return, state->mode is one of:
 
@@ -36,24 +43,43 @@
 
    Notes:
 
+    INFLATE_FAST_MIN_INPUT: 6 or 8 bytes + 7 bytes
+
     - The maximum input bits used by a length/distance pair is 15 bits for the
       length code, 5 bits for the length extra, 15 bits for the distance code,
       and 13 bits for the distance extra.  This totals 48 bits, or six bytes.
       Therefore if strm->avail_in >= 6, then there is enough input to avoid
       checking for available input while decoding.
 
+    - The wide input data reading option reads 64 input bits at a time. Thus,
+      if strm->avail_in >= 8, then there is enough input to avoid checking for
+      available input while decoding. Reading consumes the input with:
+
+          hold |= read64le(in) << bits;
+          in += 6;
+          bits += 48;
+
+      reporting 6 bytes of new input because |bits| is 0..15 (2 bytes rounded
+      up, worst case) and 6 bytes is enough to decode as noted above. At exit,
+      hold &= (1U << bits) - 1 drops excess input to keep the invariant:
+
+          (state->hold >> state->bits) == 0
+
+    INFLATE_FAST_MIN_OUTPUT: 258 bytes + 2 bytes for literals = 260 bytes
+
     - The maximum bytes that a single length/distance pair can output is 258
       bytes, which is the maximum length that can be coded.  inflate_fast()
-      requires strm->avail_out >= 258 for each loop to avoid checking for
-      output space.
+      requires strm->avail_out >= 260 for each loop to avoid checking for
+      available output space while decoding.
  */
-void ZLIB_INTERNAL inflate_fast(z_streamp strm, unsigned start) {
+void ZLIB_INTERNAL inflate_fast_chunk_(z_streamp strm, unsigned start) {
     struct inflate_state FAR *state;
     z_const unsigned char FAR *in;      /* local strm->next_in */
     z_const unsigned char FAR *last;    /* have enough input while in < last */
     unsigned char FAR *out;     /* local strm->next_out */
     unsigned char FAR *beg;     /* inflate()'s initial strm->next_out */
     unsigned char FAR *end;     /* while out < end, enough space available */
+    unsigned char FAR *limit;   /* safety limit for chunky copies */
 #ifdef INFLATE_STRICT
     unsigned dmax;              /* maximum distance from zlib header */
 #endif
@@ -61,7 +87,7 @@ void ZLIB_INTERNAL inflate_fast(z_streamp strm, unsigned start) {
     unsigned whave;             /* valid bytes in the window */
     unsigned wnext;             /* window write index */
     unsigned char FAR *window;  /* allocated sliding window, if wsize != 0 */
-    unsigned long hold;         /* local strm->hold */
+    inflate_holder_t hold;      /* local strm->hold */
     unsigned bits;              /* local strm->bits */
     code const FAR *lcode;      /* local strm->lencode */
     code const FAR *dcode;      /* local strm->distcode */
@@ -77,16 +103,17 @@ void ZLIB_INTERNAL inflate_fast(z_streamp strm, unsigned start) {
     /* copy state to local variables */
     state = (struct inflate_state FAR *)strm->state;
     in = strm->next_in;
-    last = in + (strm->avail_in - 5);
+    last = in + (strm->avail_in - (INFLATE_FAST_MIN_INPUT - 1));
     out = strm->next_out;
     beg = out - (start - strm->avail_out);
-    end = out + (strm->avail_out - 257);
+    end = out + (strm->avail_out - (INFLATE_FAST_MIN_OUTPUT - 1));
+    limit = out + strm->avail_out;
 #ifdef INFLATE_STRICT
     dmax = state->dmax;
 #endif
     wsize = state->wsize;
     whave = state->whave;
-    wnext = state->wnext;
+    wnext = (state->wnext == 0 && whave >= wsize) ? wsize : state->wnext;
     window = state->window;
     hold = state->hold;
     bits = state->bits;
@@ -95,16 +122,50 @@ void ZLIB_INTERNAL inflate_fast(z_streamp strm, unsigned start) {
     lmask = (1U << state->lenbits) - 1;
     dmask = (1U << state->distbits) - 1;
 
+#ifdef INFLATE_CHUNK_READ_64LE
+#define REFILL() do { \
+        Assert(bits < 64, "### Too many bits in inflate_fast."); \
+        hold |= read64le(in) << bits; \
+        in += 7; \
+        in -= bits >> 3; \
+        bits |= 56; \
+    } while (0)
+#endif
+
     /* decode literals and length/distances until end-of-block or not enough
        input data or output space */
     do {
+#ifdef INFLATE_CHUNK_READ_64LE
+        REFILL();
+#else
         if (bits < 15) {
             hold += (unsigned long)(*in++) << bits;
             bits += 8;
             hold += (unsigned long)(*in++) << bits;
             bits += 8;
         }
+#endif
         here = lcode + (hold & lmask);
+#ifdef INFLATE_CHUNK_READ_64LE
+        if (here->op == 0) {                    /* literal */
+            Tracevv((stderr, here->val >= 0x20 && here->val < 0x7f ?
+                    "inflate:         literal '%c'\n" :
+                    "inflate:         literal 0x%02x\n", here->val));
+            *out++ = (unsigned char)(here->val);
+            hold >>= here->bits;
+            bits -= here->bits;
+            here = lcode + (hold & lmask);
+            if (here->op == 0) {                /* literal */
+                Tracevv((stderr, here->val >= 0x20 && here->val < 0x7f ?
+                        "inflate:    2nd  literal '%c'\n" :
+                        "inflate:    2nd  literal 0x%02x\n", here->val));
+                *out++ = (unsigned char)(here->val);
+                hold >>= here->bits;
+                bits -= here->bits;
+                here = lcode + (hold & lmask);
+            }
+        }
+#endif
       dolen:
         op = (unsigned)(here->bits);
         hold >>= op;
@@ -120,21 +181,25 @@ void ZLIB_INTERNAL inflate_fast(z_streamp strm, unsigned start) {
             len = (unsigned)(here->val);
             op &= 15;                           /* number of extra bits */
             if (op) {
+#ifndef INFLATE_CHUNK_READ_64LE
                 if (bits < op) {
                     hold += (unsigned long)(*in++) << bits;
                     bits += 8;
                 }
+#endif
                 len += (unsigned)hold & ((1U << op) - 1);
                 hold >>= op;
                 bits -= op;
             }
             Tracevv((stderr, "inflate:         length %u\n", len));
+#ifndef INFLATE_CHUNK_READ_64LE
             if (bits < 15) {
                 hold += (unsigned long)(*in++) << bits;
                 bits += 8;
                 hold += (unsigned long)(*in++) << bits;
                 bits += 8;
             }
+#endif
             here = dcode + (hold & dmask);
           dodist:
             op = (unsigned)(here->bits);
@@ -144,13 +209,19 @@ void ZLIB_INTERNAL inflate_fast(z_streamp strm, unsigned start) {
             if (op & 16) {                      /* distance base */
                 dist = (unsigned)(here->val);
                 op &= 15;                       /* number of extra bits */
+                /* we have two fast-path loads: 10+10 + 15+5 + 15 = 55,
+                   but we may need to refill here in the worst case */
                 if (bits < op) {
+#ifdef INFLATE_CHUNK_READ_64LE
+                    REFILL();
+#else
                     hold += (unsigned long)(*in++) << bits;
                     bits += 8;
                     if (bits < op) {
                         hold += (unsigned long)(*in++) << bits;
                         bits += 8;
                     }
+#endif
                 }
                 dist += (unsigned)hold & ((1U << op) - 1);
 #ifdef INFLATE_STRICT
@@ -194,70 +265,51 @@ void ZLIB_INTERNAL inflate_fast(z_streamp strm, unsigned start) {
 #endif
                     }
                     from = window;
-                    if (wnext == 0) {           /* very common case */
-                        from += wsize - op;
-                        if (op < len) {         /* some from window */
-                            len -= op;
-                            do {
-                                *out++ = *from++;
-                            } while (--op);
-                            from = out - dist;  /* rest from output */
-                        }
+                    if (wnext >= op) {          /* contiguous in window */
+                        from += wnext - op;
                     }
-                    else if (wnext < op) {      /* wrap around window */
-                        from += wsize + wnext - op;
+                    else {                      /* wrap around window */
                         op -= wnext;
+                        from += wsize - op;
                         if (op < len) {         /* some from end of window */
                             len -= op;
-                            do {
-                                *out++ = *from++;
-                            } while (--op);
-                            from = window;
-                            if (wnext < len) {  /* some from start of window */
-                                op = wnext;
-                                len -= op;
-                                do {
-                                    *out++ = *from++;
-                                } while (--op);
-                                from = out - dist;      /* rest from output */
-                            }
+                            out = chunkcopy_safe(out, from, op, limit);
+                            from = window;      /* more from start of window */
+                            op = wnext;
+                            /* This (rare) case can create a situation where
+                               the first chunkcopy below must be checked.
+                             */
                         }
                     }
-                    else {                      /* contiguous in window */
-                        from += wnext - op;
-                        if (op < len) {         /* some from window */
-                            len -= op;
-                            do {
-                                *out++ = *from++;
-                            } while (--op);
-                            from = out - dist;  /* rest from output */
-                        }
-                    }
-                    while (len > 2) {
-                        *out++ = *from++;
-                        *out++ = *from++;
-                        *out++ = *from++;
-                        len -= 3;
-                    }
-                    if (len) {
-                        *out++ = *from++;
-                        if (len > 1)
-                            *out++ = *from++;
+                    if (op < len) {             /* still need some from output */
+                        out = chunkcopy_safe(out, from, op, limit);
+                        len -= op;
+                        /* When dist is small the amount of data that can be
+                           copied from the window is also small, and progress
+                           towards the dangerous end of the output buffer is
+                           also small.  This means that for trivial memsets and
+                           for chunkunroll_relaxed() a safety check is
+                           unnecessary.  However, these conditions may not be
+                           entered at all, and in that case it's possible that
+                           the main copy is near the end.
+                          */
+                        out = chunkunroll_relaxed(out, &dist, &len);
+                        out = chunkcopy_safe_ugly(out, dist, len, limit);
+                    } else {
+                        /* from points to window, so there is no risk of
+                           overlapping pointers requiring memset-like behaviour
+                         */
+                        out = chunkcopy_safe(out, from, len, limit);
                     }
                 }
                 else {
-                    from = out - dist;          /* copy direct from output */
-                    do {                        /* minimum length is three */
-                        *out++ = *from++;
-                        *out++ = *from++;
-                        *out++ = *from++;
-                        len -= 3;
-                    } while (len > 2);
-                    if (len) {
-                        *out++ = *from++;
-                        if (len > 1)
-                            *out++ = *from++;
-                    }
+                    /* Whole reference is in range of current output.  No
+                       range checks are necessary because we start with room
+                       for at least 258 bytes of output, so unroll and roundoff
+                       operations can write beyond `out+len` so long as they
+                       stay within 258 bytes of `out`.
+                     */
+                    out = chunkcopy_lapped_relaxed(out, dist, len);
                 }
             }
             else if ((op & 64) == 0) {          /* 2nd level distance code */
@@ -295,12 +347,16 @@ void ZLIB_INTERNAL inflate_fast(z_streamp strm, unsigned start) {
     /* update state and return */
     strm->next_in = in;
     strm->next_out = out;
-    strm->avail_in = (unsigned)(in < last ? 5 + (last - in) : 5 - (in - last));
+    strm->avail_in = (unsigned)(in < last ?
+        (INFLATE_FAST_MIN_INPUT - 1) + (last - in) :
+        (INFLATE_FAST_MIN_INPUT - 1) - (in - last));
     strm->avail_out = (unsigned)(out < end ?
-                                 257 + (end - out) : 257 - (out - end));
+        (INFLATE_FAST_MIN_OUTPUT - 1) + (end - out) :
+        (INFLATE_FAST_MIN_OUTPUT - 1) - (out - end));
     state->hold = hold;
     state->bits = bits;
-    return;
+
+    Assert((state->hold >> state->bits) == 0, "invalid input data state");
 }
 
 /*

package/src/inffast_chunk.h

@@ -0,0 +1,42 @@
+/* inffast_chunk.h -- header to use inffast_chunk.c
+ * Copyright (C) 1995-2003, 2010 Mark Adler
+ * Copyright (C) 2017 ARM, Inc.
+ * Copyright 2023 The Chromium Authors
+ * For conditions of distribution and use, see copyright notice in zlib.h
+ */
+
+/* WARNING: this file should *not* be used by applications. It is
+   part of the implementation of the compression library and is
+   subject to change. Applications should only use zlib.h.
+ */
+
+#include "inffast.h"
+
+/* INFLATE_FAST_MIN_INPUT:
+   The minimum number of input bytes needed so that we can safely call
+   inflate_fast() with only one up-front bounds check. One
+   length/distance code pair (15 bits for the length code, 5 bits for length
+   extra, 15 bits for the distance code, 13 bits for distance extra) requires
+   reading up to 48 input bits. Additionally, in the same iteraction, we may
+   decode two literals from the root-table (requiring MIN_OUTPUT = 258 + 2).
+
+   Each root-table entry is up to 10 bits, for a total of 68 input bits each
+   iteraction.
+
+   The refill variant reads 8 bytes from the buffer at a time, and advances
+   the input pointer by up to 7 bytes, ensuring there are at least 56-bits
+   available in the bit-buffer. The technique was documented by Fabian Giesen
+   on his blog as variant 4 in the article 'Reading bits in far too many ways':
+   https://fgiesen.wordpress.com/2018/02/20/
+
+   In the worst case, we may refill twice in the same iteraction, requiring
+   MIN_INPUT = 8 + 7.
+*/
+#ifdef INFLATE_CHUNK_READ_64LE
+#undef INFLATE_FAST_MIN_INPUT
+#define INFLATE_FAST_MIN_INPUT 15
+#undef INFLATE_FAST_MIN_OUTPUT
+#define INFLATE_FAST_MIN_OUTPUT 260
+#endif
+
+void ZLIB_INTERNAL inflate_fast_chunk_(z_streamp strm, unsigned start);