powerdlz23 1.1.7 → 1.1.8

package/pto/yaamp/stratum/sha3/sph_simd.c

@@ -0,0 +1,1799 @@
+/* $Id: simd.c 227 2010-06-16 17:28:38Z tp $ */
+/*
+ * SIMD implementation.
+ *
+ * ==========================(LICENSE BEGIN)============================
+ *
+ * Copyright (c) 2007-2010  Projet RNRT SAPHIR
+ * 
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ * 
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * 
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * ===========================(LICENSE END)=============================
+ *
+ * @author   Thomas Pornin <thomas.pornin@cryptolog.com>
+ */
+
+#include <stddef.h>
+#include <string.h>
+#include <limits.h>
+
+#include "sph_simd.h"
+
+#ifdef __cplusplus
+extern "C"{
+#endif
+
+#if SPH_SMALL_FOOTPRINT && !defined SPH_SMALL_FOOTPRINT_SIMD
+#define SPH_SMALL_FOOTPRINT_SIMD   1
+#endif
+
+#ifdef _MSC_VER
+#pragma warning (disable: 4146)
+#endif
+
+typedef sph_u32 u32;
+typedef sph_s32 s32;
+#define C32     SPH_C32
+#define T32     SPH_T32
+#define ROL32   SPH_ROTL32
+
+#define XCAT(x, y)    XCAT_(x, y)
+#define XCAT_(x, y)   x ## y
+
+/*
+ * The powers of 41 modulo 257. We use exponents from 0 to 255, inclusive.
+ */
+static const s32 alpha_tab[] = {
+	  1,  41, 139,  45,  46,  87, 226,  14,  60, 147, 116, 130,
+	190,  80, 196,  69,   2,  82,  21,  90,  92, 174, 195,  28,
+	120,  37, 232,   3, 123, 160, 135, 138,   4, 164,  42, 180,
+	184,  91, 133,  56, 240,  74, 207,   6, 246,  63,  13,  19,
+	  8,  71,  84, 103, 111, 182,   9, 112, 223, 148, 157,  12,
+	235, 126,  26,  38,  16, 142, 168, 206, 222, 107,  18, 224,
+	189,  39,  57,  24, 213, 252,  52,  76,  32,  27,  79, 155,
+	187, 214,  36, 191, 121,  78, 114,  48, 169, 247, 104, 152,
+	 64,  54, 158,  53, 117, 171,  72, 125, 242, 156, 228,  96,
+	 81, 237, 208,  47, 128, 108,  59, 106, 234,  85, 144, 250,
+	227,  55, 199, 192, 162, 217, 159,  94, 256, 216, 118, 212,
+	211, 170,  31, 243, 197, 110, 141, 127,  67, 177,  61, 188,
+	255, 175, 236, 167, 165,  83,  62, 229, 137, 220,  25, 254,
+	134,  97, 122, 119, 253,  93, 215,  77,  73, 166, 124, 201,
+	 17, 183,  50, 251,  11, 194, 244, 238, 249, 186, 173, 154,
+	146,  75, 248, 145,  34, 109, 100, 245,  22, 131, 231, 219,
+	241, 115,  89,  51,  35, 150, 239,  33,  68, 218, 200, 233,
+	 44,   5, 205, 181, 225, 230, 178, 102,  70,  43, 221,  66,
+	136, 179, 143, 209,  88,  10, 153, 105, 193, 203,  99, 204,
+	140,  86, 185, 132,  15, 101,  29, 161, 176,  20,  49, 210,
+	129, 149, 198, 151,  23, 172, 113,   7,  30, 202,  58,  65,
+	 95,  40,  98, 163
+};
+
+/*
+ * Ranges:
+ *   REDS1: from -32768..98302 to -383..383
+ *   REDS2: from -2^31..2^31-1 to -32768..98302
+ */
+#define REDS1(x)    (((x) & 0xFF) - ((x) >> 8))
+#define REDS2(x)    (((x) & 0xFFFF) + ((x) >> 16))
+
+/*
+ * If, upon entry, the values of q[] are all in the -N..N range (where
+ * N >= 98302) then the new values of q[] are in the -2N..2N range.
+ *
+ * Since alpha_tab[v] <= 256, maximum allowed range is for N = 8388608.
+ */
+#define FFT_LOOP(rb, hk, as, id)   do { \
+		size_t u, v; \
+		s32 m = q[(rb)]; \
+		s32 n = q[(rb) + (hk)]; \
+		q[(rb)] = m + n; \
+		q[(rb) + (hk)] = m - n; \
+		u = v = 0; \
+		goto id; \
+		for (; u < (hk); u += 4, v += 4 * (as)) { \
+			s32 t; \
+			m = q[(rb) + u + 0]; \
+			n = q[(rb) + u + 0 + (hk)]; \
+			t = REDS2(n * alpha_tab[v + 0 * (as)]); \
+			q[(rb) + u + 0] = m + t; \
+			q[(rb) + u + 0 + (hk)] = m - t; \
+		id: \
+			m = q[(rb) + u + 1]; \
+			n = q[(rb) + u + 1 + (hk)]; \
+			t = REDS2(n * alpha_tab[v + 1 * (as)]); \
+			q[(rb) + u + 1] = m + t; \
+			q[(rb) + u + 1 + (hk)] = m - t; \
+			m = q[(rb) + u + 2]; \
+			n = q[(rb) + u + 2 + (hk)]; \
+			t = REDS2(n * alpha_tab[v + 2 * (as)]); \
+			q[(rb) + u + 2] = m + t; \
+			q[(rb) + u + 2 + (hk)] = m - t; \
+			m = q[(rb) + u + 3]; \
+			n = q[(rb) + u + 3 + (hk)]; \
+			t = REDS2(n * alpha_tab[v + 3 * (as)]); \
+			q[(rb) + u + 3] = m + t; \
+			q[(rb) + u + 3 + (hk)] = m - t; \
+		} \
+	} while (0)
+
+/*
+ * Output ranges:
+ *   d0:   min=    0   max= 1020
+ *   d1:   min=  -67   max= 4587
+ *   d2:   min=-4335   max= 4335
+ *   d3:   min=-4147   max=  507
+ *   d4:   min= -510   max=  510
+ *   d5:   min= -252   max= 4402
+ *   d6:   min=-4335   max= 4335
+ *   d7:   min=-4332   max=  322
+ */
+#define FFT8(xb, xs, d)   do { \
+		s32 x0 = x[(xb)]; \
+		s32 x1 = x[(xb) + (xs)]; \
+		s32 x2 = x[(xb) + 2 * (xs)]; \
+		s32 x3 = x[(xb) + 3 * (xs)]; \
+		s32 a0 = x0 + x2; \
+		s32 a1 = x0 + (x2 << 4); \
+		s32 a2 = x0 - x2; \
+		s32 a3 = x0 - (x2 << 4); \
+		s32 b0 = x1 + x3; \
+		s32 b1 = REDS1((x1 << 2) + (x3 << 6)); \
+		s32 b2 = (x1 << 4) - (x3 << 4); \
+		s32 b3 = REDS1((x1 << 6) + (x3 << 2)); \
+		d ## 0 = a0 + b0; \
+		d ## 1 = a1 + b1; \
+		d ## 2 = a2 + b2; \
+		d ## 3 = a3 + b3; \
+		d ## 4 = a0 - b0; \
+		d ## 5 = a1 - b1; \
+		d ## 6 = a2 - b2; \
+		d ## 7 = a3 - b3; \
+	} while (0)
+
+/*
+ * When k=16, we have alpha=2. Multiplication by alpha^i is then reduced
+ * to some shifting.
+ *
+ * Output: within -591471..591723
+ */
+#define FFT16(xb, xs, rb)   do { \
+		s32 d1_0, d1_1, d1_2, d1_3, d1_4, d1_5, d1_6, d1_7; \
+		s32 d2_0, d2_1, d2_2, d2_3, d2_4, d2_5, d2_6, d2_7; \
+		FFT8(xb, (xs) << 1, d1_); \
+		FFT8((xb) + (xs), (xs) << 1, d2_); \
+		q[(rb) +  0] = d1_0 + d2_0; \
+		q[(rb) +  1] = d1_1 + (d2_1 << 1); \
+		q[(rb) +  2] = d1_2 + (d2_2 << 2); \
+		q[(rb) +  3] = d1_3 + (d2_3 << 3); \
+		q[(rb) +  4] = d1_4 + (d2_4 << 4); \
+		q[(rb) +  5] = d1_5 + (d2_5 << 5); \
+		q[(rb) +  6] = d1_6 + (d2_6 << 6); \
+		q[(rb) +  7] = d1_7 + (d2_7 << 7); \
+		q[(rb) +  8] = d1_0 - d2_0; \
+		q[(rb) +  9] = d1_1 - (d2_1 << 1); \
+		q[(rb) + 10] = d1_2 - (d2_2 << 2); \
+		q[(rb) + 11] = d1_3 - (d2_3 << 3); \
+		q[(rb) + 12] = d1_4 - (d2_4 << 4); \
+		q[(rb) + 13] = d1_5 - (d2_5 << 5); \
+		q[(rb) + 14] = d1_6 - (d2_6 << 6); \
+		q[(rb) + 15] = d1_7 - (d2_7 << 7); \
+	} while (0)
+
+/*
+ * Output range: |q| <= 1183446
+ */
+#define FFT32(xb, xs, rb, id)   do { \
+		FFT16(xb, (xs) << 1, rb); \
+		FFT16((xb) + (xs), (xs) << 1, (rb) + 16); \
+		FFT_LOOP(rb, 16, 8, id); \
+	} while (0)
+
+/*
+ * Output range: |q| <= 2366892
+ */
+#define FFT64(xb, xs, rb, id)   do { \
+		FFT32(xb, (xs) << 1, rb, XCAT(id, a)); \
+		FFT32((xb) + (xs), (xs) << 1, (rb) + 32, XCAT(id, b)); \
+		FFT_LOOP(rb, 32, 4, id); \
+	} while (0)
+
+#if SPH_SMALL_FOOTPRINT_SIMD
+
+static void
+fft32(unsigned char *x, size_t xs, s32 *q)
+{
+	size_t xd;
+
+	xd = xs << 1;
+	FFT16(0, xd, 0);
+	FFT16(xs, xd, 16);
+	FFT_LOOP(0, 16, 8, label_);
+}
+
+#define FFT128(xb, xs, rb, id)   do { \
+		fft32(x + (xb) + ((xs) * 0), (xs) << 2, &q[(rb) +  0]); \
+		fft32(x + (xb) + ((xs) * 2), (xs) << 2, &q[(rb) + 32]); \
+		FFT_LOOP(rb, 32, 4, XCAT(id, aa)); \
+		fft32(x + (xb) + ((xs) * 1), (xs) << 2, &q[(rb) + 64]); \
+		fft32(x + (xb) + ((xs) * 3), (xs) << 2, &q[(rb) + 96]); \
+		FFT_LOOP((rb) + 64, 32, 4, XCAT(id, ab)); \
+		FFT_LOOP(rb, 64, 2, XCAT(id, a)); \
+	} while (0)
+
+#else
+
+/*
+ * Output range: |q| <= 4733784
+ */
+#define FFT128(xb, xs, rb, id)   do { \
+		FFT64(xb, (xs) << 1, rb, XCAT(id, a)); \
+		FFT64((xb) + (xs), (xs) << 1, (rb) + 64, XCAT(id, b)); \
+		FFT_LOOP(rb, 64, 2, id); \
+	} while (0)
+
+#endif
+
+/*
+ * For SIMD-384 / SIMD-512, the fully unrolled FFT yields a compression
+ * function which does not fit in the 32 kB L1 cache of a typical x86
+ * Intel. We therefore add a function call layer at the FFT64 level.
+ */
+
+static void
+fft64(unsigned char *x, size_t xs, s32 *q)
+{
+	size_t xd;
+
+	xd = xs << 1;
+	FFT32(0, xd, 0, label_a);
+	FFT32(xs, xd, 32, label_b);
+	FFT_LOOP(0, 32, 4, label_);
+}
+
+/*
+ * Output range: |q| <= 9467568
+ */
+#define FFT256(xb, xs, rb, id)   do { \
+		fft64(x + (xb) + ((xs) * 0), (xs) << 2, &q[(rb) +   0]); \
+		fft64(x + (xb) + ((xs) * 2), (xs) << 2, &q[(rb) +  64]); \
+		FFT_LOOP(rb, 64, 2, XCAT(id, aa)); \
+		fft64(x + (xb) + ((xs) * 1), (xs) << 2, &q[(rb) + 128]); \
+		fft64(x + (xb) + ((xs) * 3), (xs) << 2, &q[(rb) + 192]); \
+		FFT_LOOP((rb) + 128, 64, 2, XCAT(id, ab)); \
+		FFT_LOOP(rb, 128, 1, XCAT(id, a)); \
+	} while (0)
+
+/*
+ * alpha^(127*i) mod 257
+ */
+static const unsigned short yoff_s_n[] = {
+	  1,  98,  95,  58,  30, 113,  23, 198, 129,  49, 176,  29,
+	 15, 185, 140,  99, 193, 153,  88, 143, 136, 221,  70, 178,
+	225, 205,  44, 200,  68, 239,  35,  89, 241, 231,  22, 100,
+	 34, 248, 146, 173, 249, 244,  11,  50,  17, 124,  73, 215,
+	253, 122, 134,  25, 137,  62, 165, 236, 255,  61,  67, 141,
+	197,  31, 211, 118, 256, 159, 162, 199, 227, 144, 234,  59,
+	128, 208,  81, 228, 242,  72, 117, 158,  64, 104, 169, 114,
+	121,  36, 187,  79,  32,  52, 213,  57, 189,  18, 222, 168,
+	 16,  26, 235, 157, 223,   9, 111,  84,   8,  13, 246, 207,
+	240, 133, 184,  42,   4, 135, 123, 232, 120, 195,  92,  21,
+	  2, 196, 190, 116,  60, 226,  46, 139
+};
+
+/*
+ * alpha^(127*i) + alpha^(125*i) mod 257
+ */
+static const unsigned short yoff_s_f[] = {
+	  2, 156, 118, 107,  45, 212, 111, 162,  97, 249, 211,   3,
+	 49, 101, 151, 223, 189, 178, 253, 204,  76,  82, 232,  65,
+	 96, 176, 161,  47, 189,  61, 248, 107,   0, 131, 133, 113,
+	 17,  33,  12, 111, 251, 103,  57, 148,  47,  65, 249, 143,
+	189,   8, 204, 230, 205, 151, 187, 227, 247, 111, 140,   6,
+	 77,  10,  21, 149, 255, 101, 139, 150, 212,  45, 146,  95,
+	160,   8,  46, 254, 208, 156, 106,  34,  68,  79,   4,  53,
+	181, 175,  25, 192, 161,  81,  96, 210,  68, 196,   9, 150,
+	  0, 126, 124, 144, 240, 224, 245, 146,   6, 154, 200, 109,
+	210, 192,   8, 114,  68, 249,  53,  27,  52, 106,  70,  30,
+	 10, 146, 117, 251, 180, 247, 236, 108
+};
+
+/*
+ * beta^(255*i) mod 257
+ */
+static const unsigned short yoff_b_n[] = {
+	  1, 163,  98,  40,  95,  65,  58, 202,  30,   7, 113, 172,
+	 23, 151, 198, 149, 129, 210,  49,  20, 176, 161,  29, 101,
+	 15, 132, 185,  86, 140, 204,  99, 203, 193, 105, 153,  10,
+	 88, 209, 143, 179, 136,  66, 221,  43,  70, 102, 178, 230,
+	225, 181, 205,   5,  44, 233, 200, 218,  68,  33, 239, 150,
+	 35,  51,  89, 115, 241, 219, 231, 131,  22, 245, 100, 109,
+	 34, 145, 248,  75, 146, 154, 173, 186, 249, 238, 244, 194,
+	 11, 251,  50, 183,  17, 201, 124, 166,  73,  77, 215,  93,
+	253, 119, 122,  97, 134, 254,  25, 220, 137, 229,  62,  83,
+	165, 167, 236, 175, 255, 188,  61, 177,  67, 127, 141, 110,
+	197, 243,  31, 170, 211, 212, 118, 216, 256,  94, 159, 217,
+	162, 192, 199,  55, 227, 250, 144,  85, 234, 106,  59, 108,
+	128,  47, 208, 237,  81,  96, 228, 156, 242, 125,  72, 171,
+	117,  53, 158,  54,  64, 152, 104, 247, 169,  48, 114,  78,
+	121, 191,  36, 214, 187, 155,  79,  27,  32,  76,  52, 252,
+	213,  24,  57,  39, 189, 224,  18, 107, 222, 206, 168, 142,
+	 16,  38,  26, 126, 235,  12, 157, 148, 223, 112,   9, 182,
+	111, 103,  84,  71,   8,  19,  13,  63, 246,   6, 207,  74,
+	240,  56, 133,  91, 184, 180,  42, 164,   4, 138, 135, 160,
+	123,   3, 232,  37, 120,  28, 195, 174,  92,  90,  21,  82,
+	  2,  69, 196,  80, 190, 130, 116, 147,  60,  14, 226,  87,
+	 46,  45, 139,  41
+};
+
+/*
+ * beta^(255*i) + beta^(253*i) mod 257
+ */
+static const unsigned short yoff_b_f[] = {
+	  2, 203, 156,  47, 118, 214, 107, 106,  45,  93, 212,  20,
+	111,  73, 162, 251,  97, 215, 249,  53, 211,  19,   3,  89,
+	 49, 207, 101,  67, 151, 130, 223,  23, 189, 202, 178, 239,
+	253, 127, 204,  49,  76, 236,  82, 137, 232, 157,  65,  79,
+	 96, 161, 176, 130, 161,  30,  47,   9, 189, 247,  61, 226,
+	248,  90, 107,  64,   0,  88, 131, 243, 133,  59, 113, 115,
+	 17, 236,  33, 213,  12, 191, 111,  19, 251,  61, 103, 208,
+	 57,  35, 148, 248,  47, 116,  65, 119, 249, 178, 143,  40,
+	189, 129,   8, 163, 204, 227, 230, 196, 205, 122, 151,  45,
+	187,  19, 227,  72, 247, 125, 111, 121, 140, 220,   6, 107,
+	 77,  69,  10, 101,  21,  65, 149, 171, 255,  54, 101, 210,
+	139,  43, 150, 151, 212, 164,  45, 237, 146, 184,  95,   6,
+	160,  42,   8, 204,  46, 238, 254, 168, 208,  50, 156, 190,
+	106, 127,  34, 234,  68,  55,  79,  18,   4, 130,  53, 208,
+	181,  21, 175, 120,  25, 100, 192, 178, 161,  96,  81, 127,
+	 96, 227, 210, 248,  68,  10, 196,  31,   9, 167, 150, 193,
+	  0, 169, 126,  14, 124, 198, 144, 142, 240,  21, 224,  44,
+	245,  66, 146, 238,   6, 196, 154,  49, 200, 222, 109,   9,
+	210, 141, 192, 138,   8,  79, 114, 217,  68, 128, 249,  94,
+	 53,  30,  27,  61,  52, 135, 106, 212,  70, 238,  30, 185,
+	 10, 132, 146, 136, 117,  37, 251, 150, 180, 188, 247, 156,
+	236, 192, 108,  86
+};
+
+#define INNER(l, h, mm)   (((u32)((l) * (mm)) & 0xFFFFU) \
+                          + ((u32)((h) * (mm)) << 16))
+
+#define W_SMALL(sb, o1, o2, mm) \
+	(INNER(q[8 * (sb) + 2 * 0 + o1], q[8 * (sb) + 2 * 0 + o2], mm), \
+	 INNER(q[8 * (sb) + 2 * 1 + o1], q[8 * (sb) + 2 * 1 + o2], mm), \
+	 INNER(q[8 * (sb) + 2 * 2 + o1], q[8 * (sb) + 2 * 2 + o2], mm), \
+	 INNER(q[8 * (sb) + 2 * 3 + o1], q[8 * (sb) + 2 * 3 + o2], mm)
+
+#define WS_0_0   W_SMALL( 4,    0,    1, 185)
+#define WS_0_1   W_SMALL( 6,    0,    1, 185)
+#define WS_0_2   W_SMALL( 0,    0,    1, 185)
+#define WS_0_3   W_SMALL( 2,    0,    1, 185)
+#define WS_0_4   W_SMALL( 7,    0,    1, 185)
+#define WS_0_5   W_SMALL( 5,    0,    1, 185)
+#define WS_0_6   W_SMALL( 3,    0,    1, 185)
+#define WS_0_7   W_SMALL( 1,    0,    1, 185)
+#define WS_1_0   W_SMALL(15,    0,    1, 185)
+#define WS_1_1   W_SMALL(11,    0,    1, 185)
+#define WS_1_2   W_SMALL(12,    0,    1, 185)
+#define WS_1_3   W_SMALL( 8,    0,    1, 185)
+#define WS_1_4   W_SMALL( 9,    0,    1, 185)
+#define WS_1_5   W_SMALL(13,    0,    1, 185)
+#define WS_1_6   W_SMALL(10,    0,    1, 185)
+#define WS_1_7   W_SMALL(14,    0,    1, 185)
+#define WS_2_0   W_SMALL(17, -128,  -64, 233)
+#define WS_2_1   W_SMALL(18, -128,  -64, 233)
+#define WS_2_2   W_SMALL(23, -128,  -64, 233)
+#define WS_2_3   W_SMALL(20, -128,  -64, 233)
+#define WS_2_4   W_SMALL(22, -128,  -64, 233)
+#define WS_2_5   W_SMALL(21, -128,  -64, 233)
+#define WS_2_6   W_SMALL(16, -128,  -64, 233)
+#define WS_2_7   W_SMALL(19, -128,  -64, 233)
+#define WS_3_0   W_SMALL(30, -191, -127, 233)
+#define WS_3_1   W_SMALL(24, -191, -127, 233)
+#define WS_3_2   W_SMALL(25, -191, -127, 233)
+#define WS_3_3   W_SMALL(31, -191, -127, 233)
+#define WS_3_4   W_SMALL(27, -191, -127, 233)
+#define WS_3_5   W_SMALL(29, -191, -127, 233)
+#define WS_3_6   W_SMALL(28, -191, -127, 233)
+#define WS_3_7   W_SMALL(26, -191, -127, 233)
+
+#define W_BIG(sb, o1, o2, mm) \
+	(INNER(q[16 * (sb) + 2 * 0 + o1], q[16 * (sb) + 2 * 0 + o2], mm), \
+	 INNER(q[16 * (sb) + 2 * 1 + o1], q[16 * (sb) + 2 * 1 + o2], mm), \
+	 INNER(q[16 * (sb) + 2 * 2 + o1], q[16 * (sb) + 2 * 2 + o2], mm), \
+	 INNER(q[16 * (sb) + 2 * 3 + o1], q[16 * (sb) + 2 * 3 + o2], mm), \
+	 INNER(q[16 * (sb) + 2 * 4 + o1], q[16 * (sb) + 2 * 4 + o2], mm), \
+	 INNER(q[16 * (sb) + 2 * 5 + o1], q[16 * (sb) + 2 * 5 + o2], mm), \
+	 INNER(q[16 * (sb) + 2 * 6 + o1], q[16 * (sb) + 2 * 6 + o2], mm), \
+	 INNER(q[16 * (sb) + 2 * 7 + o1], q[16 * (sb) + 2 * 7 + o2], mm)
+
+#define WB_0_0   W_BIG( 4,    0,    1, 185)
+#define WB_0_1   W_BIG( 6,    0,    1, 185)
+#define WB_0_2   W_BIG( 0,    0,    1, 185)
+#define WB_0_3   W_BIG( 2,    0,    1, 185)
+#define WB_0_4   W_BIG( 7,    0,    1, 185)
+#define WB_0_5   W_BIG( 5,    0,    1, 185)
+#define WB_0_6   W_BIG( 3,    0,    1, 185)
+#define WB_0_7   W_BIG( 1,    0,    1, 185)
+#define WB_1_0   W_BIG(15,    0,    1, 185)
+#define WB_1_1   W_BIG(11,    0,    1, 185)
+#define WB_1_2   W_BIG(12,    0,    1, 185)
+#define WB_1_3   W_BIG( 8,    0,    1, 185)
+#define WB_1_4   W_BIG( 9,    0,    1, 185)
+#define WB_1_5   W_BIG(13,    0,    1, 185)
+#define WB_1_6   W_BIG(10,    0,    1, 185)
+#define WB_1_7   W_BIG(14,    0,    1, 185)
+#define WB_2_0   W_BIG(17, -256, -128, 233)
+#define WB_2_1   W_BIG(18, -256, -128, 233)
+#define WB_2_2   W_BIG(23, -256, -128, 233)
+#define WB_2_3   W_BIG(20, -256, -128, 233)
+#define WB_2_4   W_BIG(22, -256, -128, 233)
+#define WB_2_5   W_BIG(21, -256, -128, 233)
+#define WB_2_6   W_BIG(16, -256, -128, 233)
+#define WB_2_7   W_BIG(19, -256, -128, 233)
+#define WB_3_0   W_BIG(30, -383, -255, 233)
+#define WB_3_1   W_BIG(24, -383, -255, 233)
+#define WB_3_2   W_BIG(25, -383, -255, 233)
+#define WB_3_3   W_BIG(31, -383, -255, 233)
+#define WB_3_4   W_BIG(27, -383, -255, 233)
+#define WB_3_5   W_BIG(29, -383, -255, 233)
+#define WB_3_6   W_BIG(28, -383, -255, 233)
+#define WB_3_7   W_BIG(26, -383, -255, 233)
+
+#define IF(x, y, z)    ((((y) ^ (z)) & (x)) ^ (z))
+#define MAJ(x, y, z)   (((x) & (y)) | (((x) | (y)) & (z)))
+
+#define PP4_0_0   1
+#define PP4_0_1   0
+#define PP4_0_2   3
+#define PP4_0_3   2
+#define PP4_1_0   2
+#define PP4_1_1   3
+#define PP4_1_2   0
+#define PP4_1_3   1
+#define PP4_2_0   3
+#define PP4_2_1   2
+#define PP4_2_2   1
+#define PP4_2_3   0
+
+#define PP8_0_0   1
+#define PP8_0_1   0
+#define PP8_0_2   3
+#define PP8_0_3   2
+#define PP8_0_4   5
+#define PP8_0_5   4
+#define PP8_0_6   7
+#define PP8_0_7   6
+
+#define PP8_1_0   6
+#define PP8_1_1   7
+#define PP8_1_2   4
+#define PP8_1_3   5
+#define PP8_1_4   2
+#define PP8_1_5   3
+#define PP8_1_6   0
+#define PP8_1_7   1
+
+#define PP8_2_0   2
+#define PP8_2_1   3
+#define PP8_2_2   0
+#define PP8_2_3   1
+#define PP8_2_4   6
+#define PP8_2_5   7
+#define PP8_2_6   4
+#define PP8_2_7   5
+
+#define PP8_3_0   3
+#define PP8_3_1   2
+#define PP8_3_2   1
+#define PP8_3_3   0
+#define PP8_3_4   7
+#define PP8_3_5   6
+#define PP8_3_6   5
+#define PP8_3_7   4
+
+#define PP8_4_0   5
+#define PP8_4_1   4
+#define PP8_4_2   7
+#define PP8_4_3   6
+#define PP8_4_4   1
+#define PP8_4_5   0
+#define PP8_4_6   3
+#define PP8_4_7   2
+
+#define PP8_5_0   7
+#define PP8_5_1   6
+#define PP8_5_2   5
+#define PP8_5_3   4
+#define PP8_5_4   3
+#define PP8_5_5   2
+#define PP8_5_6   1
+#define PP8_5_7   0
+
+#define PP8_6_0   4
+#define PP8_6_1   5
+#define PP8_6_2   6
+#define PP8_6_3   7
+#define PP8_6_4   0
+#define PP8_6_5   1
+#define PP8_6_6   2
+#define PP8_6_7   3
+
+#if SPH_SIMD_NOCOPY
+
+#define DECL_STATE_SMALL
+#define READ_STATE_SMALL(sc)
+#define WRITE_STATE_SMALL(sc)
+#define DECL_STATE_BIG
+#define READ_STATE_BIG(sc)
+#define WRITE_STATE_BIG(sc)
+
+#else
+
+#define DECL_STATE_SMALL   \
+	u32 A0, A1, A2, A3, B0, B1, B2, B3, C0, C1, C2, C3, D0, D1, D2, D3;
+
+#define READ_STATE_SMALL(sc)   do { \
+		A0 = (sc)->state[ 0]; \
+		A1 = (sc)->state[ 1]; \
+		A2 = (sc)->state[ 2]; \
+		A3 = (sc)->state[ 3]; \
+		B0 = (sc)->state[ 4]; \
+		B1 = (sc)->state[ 5]; \
+		B2 = (sc)->state[ 6]; \
+		B3 = (sc)->state[ 7]; \
+		C0 = (sc)->state[ 8]; \
+		C1 = (sc)->state[ 9]; \
+		C2 = (sc)->state[10]; \
+		C3 = (sc)->state[11]; \
+		D0 = (sc)->state[12]; \
+		D1 = (sc)->state[13]; \
+		D2 = (sc)->state[14]; \
+		D3 = (sc)->state[15]; \
+	} while (0)
+
+#define WRITE_STATE_SMALL(sc)   do { \
+		(sc)->state[ 0] = A0; \
+		(sc)->state[ 1] = A1; \
+		(sc)->state[ 2] = A2; \
+		(sc)->state[ 3] = A3; \
+		(sc)->state[ 4] = B0; \
+		(sc)->state[ 5] = B1; \
+		(sc)->state[ 6] = B2; \
+		(sc)->state[ 7] = B3; \
+		(sc)->state[ 8] = C0; \
+		(sc)->state[ 9] = C1; \
+		(sc)->state[10] = C2; \
+		(sc)->state[11] = C3; \
+		(sc)->state[12] = D0; \
+		(sc)->state[13] = D1; \
+		(sc)->state[14] = D2; \
+		(sc)->state[15] = D3; \
+	} while (0)
+
+#define DECL_STATE_BIG   \
+	u32 A0, A1, A2, A3, A4, A5, A6, A7; \
+	u32 B0, B1, B2, B3, B4, B5, B6, B7; \
+	u32 C0, C1, C2, C3, C4, C5, C6, C7; \
+	u32 D0, D1, D2, D3, D4, D5, D6, D7;
+
+#define READ_STATE_BIG(sc)   do { \
+		A0 = (sc)->state[ 0]; \
+		A1 = (sc)->state[ 1]; \
+		A2 = (sc)->state[ 2]; \
+		A3 = (sc)->state[ 3]; \
+		A4 = (sc)->state[ 4]; \
+		A5 = (sc)->state[ 5]; \
+		A6 = (sc)->state[ 6]; \
+		A7 = (sc)->state[ 7]; \
+		B0 = (sc)->state[ 8]; \
+		B1 = (sc)->state[ 9]; \
+		B2 = (sc)->state[10]; \
+		B3 = (sc)->state[11]; \
+		B4 = (sc)->state[12]; \
+		B5 = (sc)->state[13]; \
+		B6 = (sc)->state[14]; \
+		B7 = (sc)->state[15]; \
+		C0 = (sc)->state[16]; \
+		C1 = (sc)->state[17]; \
+		C2 = (sc)->state[18]; \
+		C3 = (sc)->state[19]; \
+		C4 = (sc)->state[20]; \
+		C5 = (sc)->state[21]; \
+		C6 = (sc)->state[22]; \
+		C7 = (sc)->state[23]; \
+		D0 = (sc)->state[24]; \
+		D1 = (sc)->state[25]; \
+		D2 = (sc)->state[26]; \
+		D3 = (sc)->state[27]; \
+		D4 = (sc)->state[28]; \
+		D5 = (sc)->state[29]; \
+		D6 = (sc)->state[30]; \
+		D7 = (sc)->state[31]; \
+	} while (0)
+
+#define WRITE_STATE_BIG(sc)   do { \
+		(sc)->state[ 0] = A0; \
+		(sc)->state[ 1] = A1; \
+		(sc)->state[ 2] = A2; \
+		(sc)->state[ 3] = A3; \
+		(sc)->state[ 4] = A4; \
+		(sc)->state[ 5] = A5; \
+		(sc)->state[ 6] = A6; \
+		(sc)->state[ 7] = A7; \
+		(sc)->state[ 8] = B0; \
+		(sc)->state[ 9] = B1; \
+		(sc)->state[10] = B2; \
+		(sc)->state[11] = B3; \
+		(sc)->state[12] = B4; \
+		(sc)->state[13] = B5; \
+		(sc)->state[14] = B6; \
+		(sc)->state[15] = B7; \
+		(sc)->state[16] = C0; \
+		(sc)->state[17] = C1; \
+		(sc)->state[18] = C2; \
+		(sc)->state[19] = C3; \
+		(sc)->state[20] = C4; \
+		(sc)->state[21] = C5; \
+		(sc)->state[22] = C6; \
+		(sc)->state[23] = C7; \
+		(sc)->state[24] = D0; \
+		(sc)->state[25] = D1; \
+		(sc)->state[26] = D2; \
+		(sc)->state[27] = D3; \
+		(sc)->state[28] = D4; \
+		(sc)->state[29] = D5; \
+		(sc)->state[30] = D6; \
+		(sc)->state[31] = D7; \
+	} while (0)
+
+#endif
+
+#define STEP_ELT(n, w, fun, s, ppb)   do { \
+		u32 tt = T32(D ## n + (w) + fun(A ## n, B ## n, C ## n)); \
+		A ## n = T32(ROL32(tt, s) + XCAT(tA, XCAT(ppb, n))); \
+		D ## n = C ## n; \
+		C ## n = B ## n; \
+		B ## n = tA ## n; \
+	} while (0)
+
+#define STEP_SMALL(w0, w1, w2, w3, fun, r, s, pp4b)   do { \
+		u32 tA0 = ROL32(A0, r); \
+		u32 tA1 = ROL32(A1, r); \
+		u32 tA2 = ROL32(A2, r); \
+		u32 tA3 = ROL32(A3, r); \
+		STEP_ELT(0, w0, fun, s, pp4b); \
+		STEP_ELT(1, w1, fun, s, pp4b); \
+		STEP_ELT(2, w2, fun, s, pp4b); \
+		STEP_ELT(3, w3, fun, s, pp4b); \
+	} while (0)
+
+#define STEP_BIG(w0, w1, w2, w3, w4, w5, w6, w7, fun, r, s, pp8b)   do { \
+		u32 tA0 = ROL32(A0, r); \
+		u32 tA1 = ROL32(A1, r); \
+		u32 tA2 = ROL32(A2, r); \
+		u32 tA3 = ROL32(A3, r); \
+		u32 tA4 = ROL32(A4, r); \
+		u32 tA5 = ROL32(A5, r); \
+		u32 tA6 = ROL32(A6, r); \
+		u32 tA7 = ROL32(A7, r); \
+		STEP_ELT(0, w0, fun, s, pp8b); \
+		STEP_ELT(1, w1, fun, s, pp8b); \
+		STEP_ELT(2, w2, fun, s, pp8b); \
+		STEP_ELT(3, w3, fun, s, pp8b); \
+		STEP_ELT(4, w4, fun, s, pp8b); \
+		STEP_ELT(5, w5, fun, s, pp8b); \
+		STEP_ELT(6, w6, fun, s, pp8b); \
+		STEP_ELT(7, w7, fun, s, pp8b); \
+	} while (0)
+
+#define M3_0_0   0_
+#define M3_1_0   1_
+#define M3_2_0   2_
+#define M3_3_0   0_
+#define M3_4_0   1_
+#define M3_5_0   2_
+#define M3_6_0   0_
+#define M3_7_0   1_
+
+#define M3_0_1   1_
+#define M3_1_1   2_
+#define M3_2_1   0_
+#define M3_3_1   1_
+#define M3_4_1   2_
+#define M3_5_1   0_
+#define M3_6_1   1_
+#define M3_7_1   2_
+
+#define M3_0_2   2_
+#define M3_1_2   0_
+#define M3_2_2   1_
+#define M3_3_2   2_
+#define M3_4_2   0_
+#define M3_5_2   1_
+#define M3_6_2   2_
+#define M3_7_2   0_
+
+#define STEP_SMALL_(w, fun, r, s, pp4b)   STEP_SMALL w, fun, r, s, pp4b)
+
+#define ONE_ROUND_SMALL(ri, isp, p0, p1, p2, p3)   do { \
+		STEP_SMALL_(WS_ ## ri ## 0, \
+			IF,  p0, p1, XCAT(PP4_, M3_0_ ## isp)); \
+		STEP_SMALL_(WS_ ## ri ## 1, \
+			IF,  p1, p2, XCAT(PP4_, M3_1_ ## isp)); \
+		STEP_SMALL_(WS_ ## ri ## 2, \
+			IF,  p2, p3, XCAT(PP4_, M3_2_ ## isp)); \
+		STEP_SMALL_(WS_ ## ri ## 3, \
+			IF,  p3, p0, XCAT(PP4_, M3_3_ ## isp)); \
+		STEP_SMALL_(WS_ ## ri ## 4, \
+			MAJ, p0, p1, XCAT(PP4_, M3_4_ ## isp)); \
+		STEP_SMALL_(WS_ ## ri ## 5, \
+			MAJ, p1, p2, XCAT(PP4_, M3_5_ ## isp)); \
+		STEP_SMALL_(WS_ ## ri ## 6, \
+			MAJ, p2, p3, XCAT(PP4_, M3_6_ ## isp)); \
+		STEP_SMALL_(WS_ ## ri ## 7, \
+			MAJ, p3, p0, XCAT(PP4_, M3_7_ ## isp)); \
+	} while (0)
+
+#define M7_0_0   0_
+#define M7_1_0   1_
+#define M7_2_0   2_
+#define M7_3_0   3_
+#define M7_4_0   4_
+#define M7_5_0   5_
+#define M7_6_0   6_
+#define M7_7_0   0_
+
+#define M7_0_1   1_
+#define M7_1_1   2_
+#define M7_2_1   3_
+#define M7_3_1   4_
+#define M7_4_1   5_
+#define M7_5_1   6_
+#define M7_6_1   0_
+#define M7_7_1   1_
+
+#define M7_0_2   2_
+#define M7_1_2   3_
+#define M7_2_2   4_
+#define M7_3_2   5_
+#define M7_4_2   6_
+#define M7_5_2   0_
+#define M7_6_2   1_
+#define M7_7_2   2_
+
+#define M7_0_3   3_
+#define M7_1_3   4_
+#define M7_2_3   5_
+#define M7_3_3   6_
+#define M7_4_3   0_
+#define M7_5_3   1_
+#define M7_6_3   2_
+#define M7_7_3   3_
+
+#define STEP_BIG_(w, fun, r, s, pp8b)   STEP_BIG w, fun, r, s, pp8b)
+
+#define ONE_ROUND_BIG(ri, isp, p0, p1, p2, p3)   do { \
+		STEP_BIG_(WB_ ## ri ## 0, \
+			IF,  p0, p1, XCAT(PP8_, M7_0_ ## isp)); \
+		STEP_BIG_(WB_ ## ri ## 1, \
+			IF,  p1, p2, XCAT(PP8_, M7_1_ ## isp)); \
+		STEP_BIG_(WB_ ## ri ## 2, \
+			IF,  p2, p3, XCAT(PP8_, M7_2_ ## isp)); \
+		STEP_BIG_(WB_ ## ri ## 3, \
+			IF,  p3, p0, XCAT(PP8_, M7_3_ ## isp)); \
+		STEP_BIG_(WB_ ## ri ## 4, \
+			MAJ, p0, p1, XCAT(PP8_, M7_4_ ## isp)); \
+		STEP_BIG_(WB_ ## ri ## 5, \
+			MAJ, p1, p2, XCAT(PP8_, M7_5_ ## isp)); \
+		STEP_BIG_(WB_ ## ri ## 6, \
+			MAJ, p2, p3, XCAT(PP8_, M7_6_ ## isp)); \
+		STEP_BIG_(WB_ ## ri ## 7, \
+			MAJ, p3, p0, XCAT(PP8_, M7_7_ ## isp)); \
+	} while (0)
+
+#if SPH_SMALL_FOOTPRINT_SIMD
+
+#define A0   state[ 0]
+#define A1   state[ 1]
+#define A2   state[ 2]
+#define A3   state[ 3]
+#define B0   state[ 4]
+#define B1   state[ 5]
+#define B2   state[ 6]
+#define B3   state[ 7]
+#define C0   state[ 8]
+#define C1   state[ 9]
+#define C2   state[10]
+#define C3   state[11]
+#define D0   state[12]
+#define D1   state[13]
+#define D2   state[14]
+#define D3   state[15]
+
+#define STEP2_ELT(n, w, fun, s, ppb)   do { \
+		u32 tt = T32(D ## n + (w) + fun(A ## n, B ## n, C ## n)); \
+		A ## n = T32(ROL32(tt, s) + tA[(ppb) ^ n]); \
+		D ## n = C ## n; \
+		C ## n = B ## n; \
+		B ## n = tA[n]; \
+	} while (0)
+
+#define STEP2_SMALL(w0, w1, w2, w3, fun, r, s, pp4b)   do { \
+		u32 tA[4]; \
+		tA[0] = ROL32(A0, r); \
+		tA[1] = ROL32(A1, r); \
+		tA[2] = ROL32(A2, r); \
+		tA[3] = ROL32(A3, r); \
+		STEP2_ELT(0, w0, fun, s, pp4b); \
+		STEP2_ELT(1, w1, fun, s, pp4b); \
+		STEP2_ELT(2, w2, fun, s, pp4b); \
+		STEP2_ELT(3, w3, fun, s, pp4b); \
+	} while (0)
+
+static void
+one_round_small(u32 *state, u32 *w, int isp, int p0, int p1, int p2, int p3)
+{
+	static const int pp4k[] = { 1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2 };
+
+	STEP2_SMALL(w[ 0], w[ 1], w[ 2], w[ 3], IF,  p0, p1, pp4k[isp + 0]);
+	STEP2_SMALL(w[ 4], w[ 5], w[ 6], w[ 7], IF,  p1, p2, pp4k[isp + 1]);
+	STEP2_SMALL(w[ 8], w[ 9], w[10], w[11], IF,  p2, p3, pp4k[isp + 2]);
+	STEP2_SMALL(w[12], w[13], w[14], w[15], IF,  p3, p0, pp4k[isp + 3]);
+	STEP2_SMALL(w[16], w[17], w[18], w[19], MAJ, p0, p1, pp4k[isp + 4]);
+	STEP2_SMALL(w[20], w[21], w[22], w[23], MAJ, p1, p2, pp4k[isp + 5]);
+	STEP2_SMALL(w[24], w[25], w[26], w[27], MAJ, p2, p3, pp4k[isp + 6]);
+	STEP2_SMALL(w[28], w[29], w[30], w[31], MAJ, p3, p0, pp4k[isp + 7]);
+}
+
+static void
+compress_small(sph_simd_small_context *sc, int last)
+{
+	unsigned char *x;
+	s32 q[128];
+	int i;
+	u32 w[32];
+	u32 state[16];
+	size_t u;
+
+	static const size_t wsp[32] = {
+		 4 << 3,  6 << 3,  0 << 3,  2 << 3,
+		 7 << 3,  5 << 3,  3 << 3,  1 << 3,
+		15 << 3, 11 << 3, 12 << 3,  8 << 3,
+		 9 << 3, 13 << 3, 10 << 3, 14 << 3,
+		17 << 3, 18 << 3, 23 << 3, 20 << 3,
+		22 << 3, 21 << 3, 16 << 3, 19 << 3,
+		30 << 3, 24 << 3, 25 << 3, 31 << 3,
+		27 << 3, 29 << 3, 28 << 3, 26 << 3
+	};
+
+	x = sc->buf;
+	FFT128(0, 1, 0, ll);
+	if (last) {
+		for (i = 0; i < 128; i ++) {
+			s32 tq;
+
+			tq = q[i] + yoff_s_f[i];
+			tq = REDS2(tq);
+			tq = REDS1(tq);
+			tq = REDS1(tq);
+			q[i] = (tq <= 128 ? tq : tq - 257);
+		}
+	} else {
+		for (i = 0; i < 128; i ++) {
+			s32 tq;
+
+			tq = q[i] + yoff_s_n[i];
+			tq = REDS2(tq);
+			tq = REDS1(tq);
+			tq = REDS1(tq);
+			q[i] = (tq <= 128 ? tq : tq - 257);
+		}
+	}
+
+	for (i = 0; i < 16; i += 4) {
+		state[i + 0] = sc->state[i + 0]
+			^ sph_dec32le_aligned(x + 4 * (i + 0));
+		state[i + 1] = sc->state[i + 1]
+			^ sph_dec32le_aligned(x + 4 * (i + 1));
+		state[i + 2] = sc->state[i + 2]
+			^ sph_dec32le_aligned(x + 4 * (i + 2));
+		state[i + 3] = sc->state[i + 3]
+			^ sph_dec32le_aligned(x + 4 * (i + 3));
+	}
+
+#define WSREAD(sb, o1, o2, mm)   do { \
+		for (u = 0; u < 32; u += 4) { \
+			size_t v = wsp[(u >> 2) + (sb)]; \
+			w[u + 0] = INNER(q[v + 2 * 0 + (o1)], \
+				q[v + 2 * 0 + (o2)], mm); \
+			w[u + 1] = INNER(q[v + 2 * 1 + (o1)], \
+				q[v + 2 * 1 + (o2)], mm); \
+			w[u + 2] = INNER(q[v + 2 * 2 + (o1)], \
+				q[v + 2 * 2 + (o2)], mm); \
+			w[u + 3] = INNER(q[v + 2 * 3 + (o1)], \
+				q[v + 2 * 3 + (o2)], mm); \
+		} \
+	} while (0)
+
+	WSREAD( 0,    0,    1, 185);
+	one_round_small(state, w, 0,  3, 23, 17, 27);
+	WSREAD( 8,    0,    1, 185);
+	one_round_small(state, w, 2, 28, 19, 22,  7);
+	WSREAD(16, -128,  -64, 233);
+	one_round_small(state, w, 1, 29,  9, 15,  5);
+	WSREAD(24, -191, -127, 233);
+	one_round_small(state, w, 0,  4, 13, 10, 25);
+
+#undef WSREAD
+
+	STEP_SMALL(sc->state[ 0], sc->state[ 1], sc->state[ 2], sc->state[ 3],
+		IF,  4, 13, PP4_2_);
+	STEP_SMALL(sc->state[ 4], sc->state[ 5], sc->state[ 6], sc->state[ 7],
+		IF, 13, 10, PP4_0_);
+	STEP_SMALL(sc->state[ 8], sc->state[ 9], sc->state[10], sc->state[11],
+		IF, 10, 25, PP4_1_);
+	STEP_SMALL(sc->state[12], sc->state[13], sc->state[14], sc->state[15],
+		IF, 25,  4, PP4_2_);
+
+	memcpy(sc->state, state, sizeof state);
+}
+
+#undef A0
+#undef A1
+#undef A2
+#undef A3
+#undef B0
+#undef B1
+#undef B2
+#undef B3
+#undef C0
+#undef C1
+#undef C2
+#undef C3
+#undef D0
+#undef D1
+#undef D2
+#undef D3
+
+#else
+
+#if SPH_SIMD_NOCOPY
+#define A0   (sc->state[ 0])
+#define A1   (sc->state[ 1])
+#define A2   (sc->state[ 2])
+#define A3   (sc->state[ 3])
+#define B0   (sc->state[ 4])
+#define B1   (sc->state[ 5])
+#define B2   (sc->state[ 6])
+#define B3   (sc->state[ 7])
+#define C0   (sc->state[ 8])
+#define C1   (sc->state[ 9])
+#define C2   (sc->state[10])
+#define C3   (sc->state[11])
+#define D0   (sc->state[12])
+#define D1   (sc->state[13])
+#define D2   (sc->state[14])
+#define D3   (sc->state[15])
+#endif
+
+static void
+compress_small(sph_simd_small_context *sc, int last)
+{
+	unsigned char *x;
+	s32 q[128];
+	int i;
+	DECL_STATE_SMALL
+#if SPH_SIMD_NOCOPY
+	sph_u32 saved[16];
+#endif
+
+#if SPH_SIMD_NOCOPY
+	memcpy(saved, sc->state, sizeof saved);
+#endif
+	x = sc->buf;
+	FFT128(0, 1, 0, ll);
+	if (last) {
+		for (i = 0; i < 128; i ++) {
+			s32 tq;
+
+			tq = q[i] + yoff_s_f[i];
+			tq = REDS2(tq);
+			tq = REDS1(tq);
+			tq = REDS1(tq);
+			q[i] = (tq <= 128 ? tq : tq - 257);
+		}
+	} else {
+		for (i = 0; i < 128; i ++) {
+			s32 tq;
+
+			tq = q[i] + yoff_s_n[i];
+			tq = REDS2(tq);
+			tq = REDS1(tq);
+			tq = REDS1(tq);
+			q[i] = (tq <= 128 ? tq : tq - 257);
+		}
+	}
+	READ_STATE_SMALL(sc);
+	A0 ^= sph_dec32le_aligned(x +  0);
+	A1 ^= sph_dec32le_aligned(x +  4);
+	A2 ^= sph_dec32le_aligned(x +  8);
+	A3 ^= sph_dec32le_aligned(x + 12);
+	B0 ^= sph_dec32le_aligned(x + 16);
+	B1 ^= sph_dec32le_aligned(x + 20);
+	B2 ^= sph_dec32le_aligned(x + 24);
+	B3 ^= sph_dec32le_aligned(x + 28);
+	C0 ^= sph_dec32le_aligned(x + 32);
+	C1 ^= sph_dec32le_aligned(x + 36);
+	C2 ^= sph_dec32le_aligned(x + 40);
+	C3 ^= sph_dec32le_aligned(x + 44);
+	D0 ^= sph_dec32le_aligned(x + 48);
+	D1 ^= sph_dec32le_aligned(x + 52);
+	D2 ^= sph_dec32le_aligned(x + 56);
+	D3 ^= sph_dec32le_aligned(x + 60);
+	ONE_ROUND_SMALL(0_, 0,  3, 23, 17, 27);
+	ONE_ROUND_SMALL(1_, 2, 28, 19, 22,  7);
+	ONE_ROUND_SMALL(2_, 1, 29,  9, 15,  5);
+	ONE_ROUND_SMALL(3_, 0,  4, 13, 10, 25);
+#if SPH_SIMD_NOCOPY
+	STEP_SMALL(saved[ 0], saved[ 1], saved[ 2], saved[ 3],
+		IF,  4, 13, PP4_2_);
+	STEP_SMALL(saved[ 4], saved[ 5], saved[ 6], saved[ 7],
+		IF, 13, 10, PP4_0_);
+	STEP_SMALL(saved[ 8], saved[ 9], saved[10], saved[11],
+		IF, 10, 25, PP4_1_);
+	STEP_SMALL(saved[12], saved[13], saved[14], saved[15],
+		IF, 25,  4, PP4_2_);
+#else
+	STEP_SMALL(sc->state[ 0], sc->state[ 1], sc->state[ 2], sc->state[ 3],
+		IF,  4, 13, PP4_2_);
+	STEP_SMALL(sc->state[ 4], sc->state[ 5], sc->state[ 6], sc->state[ 7],
+		IF, 13, 10, PP4_0_);
+	STEP_SMALL(sc->state[ 8], sc->state[ 9], sc->state[10], sc->state[11],
+		IF, 10, 25, PP4_1_);
+	STEP_SMALL(sc->state[12], sc->state[13], sc->state[14], sc->state[15],
+		IF, 25,  4, PP4_2_);
+	WRITE_STATE_SMALL(sc);
+#endif
+}
+
+#if SPH_SIMD_NOCOPY
+#undef A0
+#undef A1
+#undef A2
+#undef A3
+#undef B0
+#undef B1
+#undef B2
+#undef B3
+#undef C0
+#undef C1
+#undef C2
+#undef C3
+#undef D0
+#undef D1
+#undef D2
+#undef D3
+#endif
+
+#endif
+
+#if SPH_SMALL_FOOTPRINT_SIMD
+
+#define A0   state[ 0]
+#define A1   state[ 1]
+#define A2   state[ 2]
+#define A3   state[ 3]
+#define A4   state[ 4]
+#define A5   state[ 5]
+#define A6   state[ 6]
+#define A7   state[ 7]
+#define B0   state[ 8]
+#define B1   state[ 9]
+#define B2   state[10]
+#define B3   state[11]
+#define B4   state[12]
+#define B5   state[13]
+#define B6   state[14]
+#define B7   state[15]
+#define C0   state[16]
+#define C1   state[17]
+#define C2   state[18]
+#define C3   state[19]
+#define C4   state[20]
+#define C5   state[21]
+#define C6   state[22]
+#define C7   state[23]
+#define D0   state[24]
+#define D1   state[25]
+#define D2   state[26]
+#define D3   state[27]
+#define D4   state[28]
+#define D5   state[29]
+#define D6   state[30]
+#define D7   state[31]
+
+/*
+ * Not needed -- already defined for SIMD-224 / SIMD-256
+ *
+#define STEP2_ELT(n, w, fun, s, ppb)   do { \
+		u32 tt = T32(D ## n + (w) + fun(A ## n, B ## n, C ## n)); \
+		A ## n = T32(ROL32(tt, s) + tA[(ppb) ^ n]); \
+		D ## n = C ## n; \
+		C ## n = B ## n; \
+		B ## n = tA[n]; \
+	} while (0)
+ */
+
+#define STEP2_BIG(w0, w1, w2, w3, w4, w5, w6, w7, fun, r, s, pp8b)   do { \
+		u32 tA[8]; \
+		tA[0] = ROL32(A0, r); \
+		tA[1] = ROL32(A1, r); \
+		tA[2] = ROL32(A2, r); \
+		tA[3] = ROL32(A3, r); \
+		tA[4] = ROL32(A4, r); \
+		tA[5] = ROL32(A5, r); \
+		tA[6] = ROL32(A6, r); \
+		tA[7] = ROL32(A7, r); \
+		STEP2_ELT(0, w0, fun, s, pp8b); \
+		STEP2_ELT(1, w1, fun, s, pp8b); \
+		STEP2_ELT(2, w2, fun, s, pp8b); \
+		STEP2_ELT(3, w3, fun, s, pp8b); \
+		STEP2_ELT(4, w4, fun, s, pp8b); \
+		STEP2_ELT(5, w5, fun, s, pp8b); \
+		STEP2_ELT(6, w6, fun, s, pp8b); \
+		STEP2_ELT(7, w7, fun, s, pp8b); \
+	} while (0)
+
+static void
+one_round_big(u32 *state, u32 *w, int isp, int p0, int p1, int p2, int p3)
+{
+	static const int pp8k[] = { 1, 6, 2, 3, 5, 7, 4, 1, 6, 2, 3 };
+
+	STEP2_BIG(w[ 0], w[ 1], w[ 2], w[ 3], w[ 4], w[ 5], w[ 6], w[ 7],
+		IF,  p0, p1, pp8k[isp + 0]);
+	STEP2_BIG(w[ 8], w[ 9], w[10], w[11], w[12], w[13], w[14], w[15],
+		IF,  p1, p2, pp8k[isp + 1]);
+	STEP2_BIG(w[16], w[17], w[18], w[19], w[20], w[21], w[22], w[23],
+		IF,  p2, p3, pp8k[isp + 2]);
+	STEP2_BIG(w[24], w[25], w[26], w[27], w[28], w[29], w[30], w[31],
+		IF,  p3, p0, pp8k[isp + 3]);
+	STEP2_BIG(w[32], w[33], w[34], w[35], w[36], w[37], w[38], w[39],
+		MAJ, p0, p1, pp8k[isp + 4]);
+	STEP2_BIG(w[40], w[41], w[42], w[43], w[44], w[45], w[46], w[47],
+		MAJ, p1, p2, pp8k[isp + 5]);
+	STEP2_BIG(w[48], w[49], w[50], w[51], w[52], w[53], w[54], w[55],
+		MAJ, p2, p3, pp8k[isp + 6]);
+	STEP2_BIG(w[56], w[57], w[58], w[59], w[60], w[61], w[62], w[63],
+		MAJ, p3, p0, pp8k[isp + 7]);
+}
+
+static void
+compress_big(sph_simd_big_context *sc, int last)
+{
+	unsigned char *x;
+	s32 q[256];
+	int i;
+	u32 w[64];
+	u32 state[32];
+	size_t u;
+
+	static const size_t wbp[32] = {
+		 4 << 4,  6 << 4,  0 << 4,  2 << 4,
+		 7 << 4,  5 << 4,  3 << 4,  1 << 4,
+		15 << 4, 11 << 4, 12 << 4,  8 << 4,
+		 9 << 4, 13 << 4, 10 << 4, 14 << 4,
+		17 << 4, 18 << 4, 23 << 4, 20 << 4,
+		22 << 4, 21 << 4, 16 << 4, 19 << 4,
+		30 << 4, 24 << 4, 25 << 4, 31 << 4,
+		27 << 4, 29 << 4, 28 << 4, 26 << 4
+	};
+
+	x = sc->buf;
+	FFT256(0, 1, 0, ll);
+	if (last) {
+		for (i = 0; i < 256; i ++) {
+			s32 tq;
+
+			tq = q[i] + yoff_b_f[i];
+			tq = REDS2(tq);
+			tq = REDS1(tq);
+			tq = REDS1(tq);
+			q[i] = (tq <= 128 ? tq : tq - 257);
+		}
+	} else {
+		for (i = 0; i < 256; i ++) {
+			s32 tq;
+
+			tq = q[i] + yoff_b_n[i];
+			tq = REDS2(tq);
+			tq = REDS1(tq);
+			tq = REDS1(tq);
+			q[i] = (tq <= 128 ? tq : tq - 257);
+		}
+	}
+
+	for (i = 0; i < 32; i += 8) {
+		state[i + 0] = sc->state[i + 0]
+			^ sph_dec32le_aligned(x + 4 * (i + 0));
+		state[i + 1] = sc->state[i + 1]
+			^ sph_dec32le_aligned(x + 4 * (i + 1));
+		state[i + 2] = sc->state[i + 2]
+			^ sph_dec32le_aligned(x + 4 * (i + 2));
+		state[i + 3] = sc->state[i + 3]
+			^ sph_dec32le_aligned(x + 4 * (i + 3));
+		state[i + 4] = sc->state[i + 4]
+			^ sph_dec32le_aligned(x + 4 * (i + 4));
+		state[i + 5] = sc->state[i + 5]
+			^ sph_dec32le_aligned(x + 4 * (i + 5));
+		state[i + 6] = sc->state[i + 6]
+			^ sph_dec32le_aligned(x + 4 * (i + 6));
+		state[i + 7] = sc->state[i + 7]
+			^ sph_dec32le_aligned(x + 4 * (i + 7));
+	}
+
+#define WBREAD(sb, o1, o2, mm)   do { \
+		for (u = 0; u < 64; u += 8) { \
+			size_t v = wbp[(u >> 3) + (sb)]; \
+			w[u + 0] = INNER(q[v + 2 * 0 + (o1)], \
+				q[v + 2 * 0 + (o2)], mm); \
+			w[u + 1] = INNER(q[v + 2 * 1 + (o1)], \
+				q[v + 2 * 1 + (o2)], mm); \
+			w[u + 2] = INNER(q[v + 2 * 2 + (o1)], \
+				q[v + 2 * 2 + (o2)], mm); \
+			w[u + 3] = INNER(q[v + 2 * 3 + (o1)], \
+				q[v + 2 * 3 + (o2)], mm); \
+			w[u + 4] = INNER(q[v + 2 * 4 + (o1)], \
+				q[v + 2 * 4 + (o2)], mm); \
+			w[u + 5] = INNER(q[v + 2 * 5 + (o1)], \
+				q[v + 2 * 5 + (o2)], mm); \
+			w[u + 6] = INNER(q[v + 2 * 6 + (o1)], \
+				q[v + 2 * 6 + (o2)], mm); \
+			w[u + 7] = INNER(q[v + 2 * 7 + (o1)], \
+				q[v + 2 * 7 + (o2)], mm); \
+		} \
+	} while (0)
+
+	WBREAD( 0,    0,    1, 185);
+	one_round_big(state, w, 0,  3, 23, 17, 27);
+	WBREAD( 8,    0,    1, 185);
+	one_round_big(state, w, 1, 28, 19, 22,  7);
+	WBREAD(16, -256, -128, 233);
+	one_round_big(state, w, 2, 29,  9, 15,  5);
+	WBREAD(24, -383, -255, 233);
+	one_round_big(state, w, 3,  4, 13, 10, 25);
+
+#undef WBREAD
+
+	STEP_BIG(
+		sc->state[ 0], sc->state[ 1], sc->state[ 2], sc->state[ 3],
+		sc->state[ 4], sc->state[ 5], sc->state[ 6], sc->state[ 7],
+		IF,  4, 13, PP8_4_);
+	STEP_BIG(
+		sc->state[ 8], sc->state[ 9], sc->state[10], sc->state[11],
+		sc->state[12], sc->state[13], sc->state[14], sc->state[15],
+		IF, 13, 10, PP8_5_);
+	STEP_BIG(
+		sc->state[16], sc->state[17], sc->state[18], sc->state[19],
+		sc->state[20], sc->state[21], sc->state[22], sc->state[23],
+		IF, 10, 25, PP8_6_);
+	STEP_BIG(
+		sc->state[24], sc->state[25], sc->state[26], sc->state[27],
+		sc->state[28], sc->state[29], sc->state[30], sc->state[31],
+		IF, 25,  4, PP8_0_);
+
+	memcpy(sc->state, state, sizeof state);
+}
+
+#undef A0
+#undef A1
+#undef A2
+#undef A3
+#undef A4
+#undef A5
+#undef A6
+#undef A7
+#undef B0
+#undef B1
+#undef B2
+#undef B3
+#undef B4
+#undef B5
+#undef B6
+#undef B7
+#undef C0
+#undef C1
+#undef C2
+#undef C3
+#undef C4
+#undef C5
+#undef C6
+#undef C7
+#undef D0
+#undef D1
+#undef D2
+#undef D3
+#undef D4
+#undef D5
+#undef D6
+#undef D7
+
+#else
+
+#if SPH_SIMD_NOCOPY
+#define A0   (sc->state[ 0])
+#define A1   (sc->state[ 1])
+#define A2   (sc->state[ 2])
+#define A3   (sc->state[ 3])
+#define A4   (sc->state[ 4])
+#define A5   (sc->state[ 5])
+#define A6   (sc->state[ 6])
+#define A7   (sc->state[ 7])
+#define B0   (sc->state[ 8])
+#define B1   (sc->state[ 9])
+#define B2   (sc->state[10])
+#define B3   (sc->state[11])
+#define B4   (sc->state[12])
+#define B5   (sc->state[13])
+#define B6   (sc->state[14])
+#define B7   (sc->state[15])
+#define C0   (sc->state[16])
+#define C1   (sc->state[17])
+#define C2   (sc->state[18])
+#define C3   (sc->state[19])
+#define C4   (sc->state[20])
+#define C5   (sc->state[21])
+#define C6   (sc->state[22])
+#define C7   (sc->state[23])
+#define D0   (sc->state[24])
+#define D1   (sc->state[25])
+#define D2   (sc->state[26])
+#define D3   (sc->state[27])
+#define D4   (sc->state[28])
+#define D5   (sc->state[29])
+#define D6   (sc->state[30])
+#define D7   (sc->state[31])
+#endif
+
+static void
+compress_big(sph_simd_big_context *sc, int last)
+{
+	unsigned char *x;
+	s32 q[256];
+	int i;
+	DECL_STATE_BIG
+#if SPH_SIMD_NOCOPY
+	sph_u32 saved[32];
+#endif
+
+#if SPH_SIMD_NOCOPY
+	memcpy(saved, sc->state, sizeof saved);
+#endif
+
+	x = sc->buf;
+	FFT256(0, 1, 0, ll);
+	if (last) {
+		for (i = 0; i < 256; i ++) {
+			s32 tq;
+
+			tq = q[i] + yoff_b_f[i];
+			tq = REDS2(tq);
+			tq = REDS1(tq);
+			tq = REDS1(tq);
+			q[i] = (tq <= 128 ? tq : tq - 257);
+		}
+	} else {
+		for (i = 0; i < 256; i ++) {
+			s32 tq;
+
+			tq = q[i] + yoff_b_n[i];
+			tq = REDS2(tq);
+			tq = REDS1(tq);
+			tq = REDS1(tq);
+			q[i] = (tq <= 128 ? tq : tq - 257);
+		}
+	}
+	READ_STATE_BIG(sc);
+	A0 ^= sph_dec32le_aligned(x +   0);
+	A1 ^= sph_dec32le_aligned(x +   4);
+	A2 ^= sph_dec32le_aligned(x +   8);
+	A3 ^= sph_dec32le_aligned(x +  12);
+	A4 ^= sph_dec32le_aligned(x +  16);
+	A5 ^= sph_dec32le_aligned(x +  20);
+	A6 ^= sph_dec32le_aligned(x +  24);
+	A7 ^= sph_dec32le_aligned(x +  28);
+	B0 ^= sph_dec32le_aligned(x +  32);
+	B1 ^= sph_dec32le_aligned(x +  36);
+	B2 ^= sph_dec32le_aligned(x +  40);
+	B3 ^= sph_dec32le_aligned(x +  44);
+	B4 ^= sph_dec32le_aligned(x +  48);
+	B5 ^= sph_dec32le_aligned(x +  52);
+	B6 ^= sph_dec32le_aligned(x +  56);
+	B7 ^= sph_dec32le_aligned(x +  60);
+	C0 ^= sph_dec32le_aligned(x +  64);
+	C1 ^= sph_dec32le_aligned(x +  68);
+	C2 ^= sph_dec32le_aligned(x +  72);
+	C3 ^= sph_dec32le_aligned(x +  76);
+	C4 ^= sph_dec32le_aligned(x +  80);
+	C5 ^= sph_dec32le_aligned(x +  84);
+	C6 ^= sph_dec32le_aligned(x +  88);
+	C7 ^= sph_dec32le_aligned(x +  92);
+	D0 ^= sph_dec32le_aligned(x +  96);
+	D1 ^= sph_dec32le_aligned(x + 100);
+	D2 ^= sph_dec32le_aligned(x + 104);
+	D3 ^= sph_dec32le_aligned(x + 108);
+	D4 ^= sph_dec32le_aligned(x + 112);
+	D5 ^= sph_dec32le_aligned(x + 116);
+	D6 ^= sph_dec32le_aligned(x + 120);
+	D7 ^= sph_dec32le_aligned(x + 124);
+
+	ONE_ROUND_BIG(0_, 0,  3, 23, 17, 27);
+	ONE_ROUND_BIG(1_, 1, 28, 19, 22,  7);
+	ONE_ROUND_BIG(2_, 2, 29,  9, 15,  5);
+	ONE_ROUND_BIG(3_, 3,  4, 13, 10, 25);
+#if SPH_SIMD_NOCOPY
+	STEP_BIG(
+		saved[ 0], saved[ 1], saved[ 2], saved[ 3],
+		saved[ 4], saved[ 5], saved[ 6], saved[ 7],
+		IF,  4, 13, PP8_4_);
+	STEP_BIG(
+		saved[ 8], saved[ 9], saved[10], saved[11],
+		saved[12], saved[13], saved[14], saved[15],
+		IF, 13, 10, PP8_5_);
+	STEP_BIG(
+		saved[16], saved[17], saved[18], saved[19],
+		saved[20], saved[21], saved[22], saved[23],
+		IF, 10, 25, PP8_6_);
+	STEP_BIG(
+		saved[24], saved[25], saved[26], saved[27],
+		saved[28], saved[29], saved[30], saved[31],
+		IF, 25,  4, PP8_0_);
+#else
+	STEP_BIG(
+		sc->state[ 0], sc->state[ 1], sc->state[ 2], sc->state[ 3],
+		sc->state[ 4], sc->state[ 5], sc->state[ 6], sc->state[ 7],
+		IF,  4, 13, PP8_4_);
+	STEP_BIG(
+		sc->state[ 8], sc->state[ 9], sc->state[10], sc->state[11],
+		sc->state[12], sc->state[13], sc->state[14], sc->state[15],
+		IF, 13, 10, PP8_5_);
+	STEP_BIG(
+		sc->state[16], sc->state[17], sc->state[18], sc->state[19],
+		sc->state[20], sc->state[21], sc->state[22], sc->state[23],
+		IF, 10, 25, PP8_6_);
+	STEP_BIG(
+		sc->state[24], sc->state[25], sc->state[26], sc->state[27],
+		sc->state[28], sc->state[29], sc->state[30], sc->state[31],
+		IF, 25,  4, PP8_0_);
+	WRITE_STATE_BIG(sc);
+#endif
+}
+
+#if SPH_SIMD_NOCOPY
+#undef A0
+#undef A1
+#undef A2
+#undef A3
+#undef A4
+#undef A5
+#undef A6
+#undef A7
+#undef B0
+#undef B1
+#undef B2
+#undef B3
+#undef B4
+#undef B5
+#undef B6
+#undef B7
+#undef C0
+#undef C1
+#undef C2
+#undef C3
+#undef C4
+#undef C5
+#undef C6
+#undef C7
+#undef D0
+#undef D1
+#undef D2
+#undef D3
+#undef D4
+#undef D5
+#undef D6
+#undef D7
+#endif
+
+#endif
+
+static const u32 IV224[] = {
+	C32(0x33586E9F), C32(0x12FFF033), C32(0xB2D9F64D), C32(0x6F8FEA53),
+	C32(0xDE943106), C32(0x2742E439), C32(0x4FBAB5AC), C32(0x62B9FF96),
+	C32(0x22E7B0AF), C32(0xC862B3A8), C32(0x33E00CDC), C32(0x236B86A6),
+	C32(0xF64AE77C), C32(0xFA373B76), C32(0x7DC1EE5B), C32(0x7FB29CE8)
+};
+
+static const u32 IV256[] = {
+	C32(0x4D567983), C32(0x07190BA9), C32(0x8474577B), C32(0x39D726E9),
+	C32(0xAAF3D925), C32(0x3EE20B03), C32(0xAFD5E751), C32(0xC96006D3),
+	C32(0xC2C2BA14), C32(0x49B3BCB4), C32(0xF67CAF46), C32(0x668626C9),
+	C32(0xE2EAA8D2), C32(0x1FF47833), C32(0xD0C661A5), C32(0x55693DE1)
+};
+
+static const u32 IV384[] = {
+	C32(0x8A36EEBC), C32(0x94A3BD90), C32(0xD1537B83), C32(0xB25B070B),
+	C32(0xF463F1B5), C32(0xB6F81E20), C32(0x0055C339), C32(0xB4D144D1),
+	C32(0x7360CA61), C32(0x18361A03), C32(0x17DCB4B9), C32(0x3414C45A),
+	C32(0xA699A9D2), C32(0xE39E9664), C32(0x468BFE77), C32(0x51D062F8),
+	C32(0xB9E3BFE8), C32(0x63BECE2A), C32(0x8FE506B9), C32(0xF8CC4AC2),
+	C32(0x7AE11542), C32(0xB1AADDA1), C32(0x64B06794), C32(0x28D2F462),
+	C32(0xE64071EC), C32(0x1DEB91A8), C32(0x8AC8DB23), C32(0x3F782AB5),
+	C32(0x039B5CB8), C32(0x71DDD962), C32(0xFADE2CEA), C32(0x1416DF71)
+};
+
+static const u32 IV512[] = {
+	C32(0x0BA16B95), C32(0x72F999AD), C32(0x9FECC2AE), C32(0xBA3264FC),
+	C32(0x5E894929), C32(0x8E9F30E5), C32(0x2F1DAA37), C32(0xF0F2C558),
+	C32(0xAC506643), C32(0xA90635A5), C32(0xE25B878B), C32(0xAAB7878F),
+	C32(0x88817F7A), C32(0x0A02892B), C32(0x559A7550), C32(0x598F657E),
+	C32(0x7EEF60A1), C32(0x6B70E3E8), C32(0x9C1714D1), C32(0xB958E2A8),
+	C32(0xAB02675E), C32(0xED1C014F), C32(0xCD8D65BB), C32(0xFDB7A257),
+	C32(0x09254899), C32(0xD699C7BC), C32(0x9019B6DC), C32(0x2B9022E4),
+	C32(0x8FA14956), C32(0x21BF9BD3), C32(0xB94D0943), C32(0x6FFDDC22)
+};
+
+static void
+init_small(void *cc, const u32 *iv)
+{
+	sph_simd_small_context *sc;
+
+	sc = cc;
+	memcpy(sc->state, iv, sizeof sc->state);
+	sc->count_low = sc->count_high = 0;
+	sc->ptr = 0;
+}
+
+static void
+init_big(void *cc, const u32 *iv)
+{
+	sph_simd_big_context *sc;
+
+	sc = cc;
+	memcpy(sc->state, iv, sizeof sc->state);
+	sc->count_low = sc->count_high = 0;
+	sc->ptr = 0;
+}
+
+static void
+update_small(void *cc, const void *data, size_t len)
+{
+	sph_simd_small_context *sc;
+
+	sc = cc;
+	while (len > 0) {
+		size_t clen;
+
+		clen = (sizeof sc->buf) - sc->ptr;
+		if (clen > len)
+			clen = len;
+		memcpy(sc->buf + sc->ptr, data, clen);
+		data = (const unsigned char *)data + clen;
+		len -= clen;
+		if ((sc->ptr += clen) == sizeof sc->buf) {
+			compress_small(sc, 0);
+			sc->ptr = 0;
+			sc->count_low = T32(sc->count_low + 1);
+			if (sc->count_low == 0)
+				sc->count_high ++;
+		}
+	}
+}
+
+static void
+update_big(void *cc, const void *data, size_t len)
+{
+	sph_simd_big_context *sc;
+
+	sc = cc;
+	while (len > 0) {
+		size_t clen;
+
+		clen = (sizeof sc->buf) - sc->ptr;
+		if (clen > len)
+			clen = len;
+		memcpy(sc->buf + sc->ptr, data, clen);
+		data = (const unsigned char *)data + clen;
+		len -= clen;
+		if ((sc->ptr += clen) == sizeof sc->buf) {
+			compress_big(sc, 0);
+			sc->ptr = 0;
+			sc->count_low = T32(sc->count_low + 1);
+			if (sc->count_low == 0)
+				sc->count_high ++;
+		}
+	}
+}
+
+static void
+encode_count_small(unsigned char *dst,
+	u32 low, u32 high, size_t ptr, unsigned n)
+{
+	low = T32(low << 9);
+	high = T32(high << 9) + (low >> 23);
+	low += (ptr << 3) + n;
+	sph_enc32le(dst, low);
+	sph_enc32le(dst + 4, high);
+}
+
+static void
+encode_count_big(unsigned char *dst,
+	u32 low, u32 high, size_t ptr, unsigned n)
+{
+	low = T32(low << 10);
+	high = T32(high << 10) + (low >> 22);
+	low += (ptr << 3) + n;
+	sph_enc32le(dst, low);
+	sph_enc32le(dst + 4, high);
+}
+
+static void
+finalize_small(void *cc, unsigned ub, unsigned n, void *dst, size_t dst_len)
+{
+	sph_simd_small_context *sc;
+	unsigned char *d;
+	size_t u;
+
+	sc = cc;
+	if (sc->ptr > 0 || n > 0) {
+		memset(sc->buf + sc->ptr, 0,
+			(sizeof sc->buf) - sc->ptr);
+		sc->buf[sc->ptr] = ub & (0xFF << (8 - n));
+		compress_small(sc, 0);
+	}
+	memset(sc->buf, 0, sizeof sc->buf);
+	encode_count_small(sc->buf, sc->count_low, sc->count_high, sc->ptr, n);
+	compress_small(sc, 1);
+	d = dst;
+	for (d = dst, u = 0; u < dst_len; u ++)
+		sph_enc32le(d + (u << 2), sc->state[u]);
+}
+
+static void
+finalize_big(void *cc, unsigned ub, unsigned n, void *dst, size_t dst_len)
+{
+	sph_simd_big_context *sc;
+	unsigned char *d;
+	size_t u;
+
+	sc = cc;
+	if (sc->ptr > 0 || n > 0) {
+		memset(sc->buf + sc->ptr, 0,
+			(sizeof sc->buf) - sc->ptr);
+		sc->buf[sc->ptr] = ub & (0xFF << (8 - n));
+		compress_big(sc, 0);
+	}
+	memset(sc->buf, 0, sizeof sc->buf);
+	encode_count_big(sc->buf, sc->count_low, sc->count_high, sc->ptr, n);
+	compress_big(sc, 1);
+	d = dst;
+	for (d = dst, u = 0; u < dst_len; u ++)
+		sph_enc32le(d + (u << 2), sc->state[u]);
+}
+
+void
+sph_simd224_init(void *cc)
+{
+	init_small(cc, IV224);
+}
+
+void
+sph_simd224(void *cc, const void *data, size_t len)
+{
+	update_small(cc, data, len);
+}
+
+void
+sph_simd224_close(void *cc, void *dst)
+{
+	sph_simd224_addbits_and_close(cc, 0, 0, dst);
+}
+
+void
+sph_simd224_addbits_and_close(void *cc, unsigned ub, unsigned n, void *dst)
+{
+	finalize_small(cc, ub, n, dst, 7);
+	sph_simd224_init(cc);
+}
+
+void
+sph_simd256_init(void *cc)
+{
+	init_small(cc, IV256);
+}
+
+void
+sph_simd256(void *cc, const void *data, size_t len)
+{
+	update_small(cc, data, len);
+}
+
+void
+sph_simd256_close(void *cc, void *dst)
+{
+	sph_simd256_addbits_and_close(cc, 0, 0, dst);
+}
+
+void
+sph_simd256_addbits_and_close(void *cc, unsigned ub, unsigned n, void *dst)
+{
+	finalize_small(cc, ub, n, dst, 8);
+	sph_simd256_init(cc);
+}
+
+void
+sph_simd384_init(void *cc)
+{
+	init_big(cc, IV384);
+}
+
+void
+sph_simd384(void *cc, const void *data, size_t len)
+{
+	update_big(cc, data, len);
+}
+
+void
+sph_simd384_close(void *cc, void *dst)
+{
+	sph_simd384_addbits_and_close(cc, 0, 0, dst);
+}
+
+void
+sph_simd384_addbits_and_close(void *cc, unsigned ub, unsigned n, void *dst)
+{
+	finalize_big(cc, ub, n, dst, 12);
+	sph_simd384_init(cc);
+}
+
+void
+sph_simd512_init(void *cc)
+{
+	init_big(cc, IV512);
+}
+
+void
+sph_simd512(void *cc, const void *data, size_t len)
+{
+	update_big(cc, data, len);
+}
+
+void
+sph_simd512_close(void *cc, void *dst)
+{
+	sph_simd512_addbits_and_close(cc, 0, 0, dst);
+}
+
+void
+sph_simd512_addbits_and_close(void *cc, unsigned ub, unsigned n, void *dst)
+{
+	finalize_big(cc, ub, n, dst, 16);
+	sph_simd512_init(cc);
+}
+#ifdef __cplusplus
+}
+#endif