yencode 1.0.8 → 1.1.2

package/{crc_folding.c → src/crc_folding.cc}

@@ -17,158 +17,54 @@
  * For conditions of distribution and use, see copyright notice in zlib.h
  */
 
-#ifdef X86_PCLMULQDQ_CRC
-
+#include "crc_common.h"
+ 
+#if (defined(__PCLMUL__) && defined(__SSSE3__) && defined(__SSE4_1__)) || (defined(_MSC_VER) && _MSC_VER >= 1600 && defined(PLATFORM_X86))
 #include <inttypes.h>
 #include <immintrin.h>
 #include <wmmintrin.h>
 
-#define local static
 
-#ifdef _MSC_VER
-#define ALIGN(_a, v) __declspec(align(_a)) v
-#else
-#define ALIGN(_a, v) v __attribute__((aligned(_a)))
+#if defined(__AVX512VL__) && defined(YENC_BUILD_NATIVE) && YENC_BUILD_NATIVE!=0
+# define ENABLE_AVX512 1
 #endif
 
 
-local void fold_1(__m128i *xmm_crc0, __m128i *xmm_crc1, __m128i *xmm_crc2, __m128i *xmm_crc3) {
-    const __m128i xmm_fold4 = _mm_set_epi32(
-            0x00000001, 0x54442bd4,
-            0x00000001, 0xc6e41596);
-
-    __m128i x_tmp3;
-    __m128 ps_crc0, ps_crc3, ps_res;
-
-    x_tmp3 = *xmm_crc3;
-
-    *xmm_crc3 = *xmm_crc0;
-    *xmm_crc0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x01);
-    *xmm_crc3 = _mm_clmulepi64_si128(*xmm_crc3, xmm_fold4, 0x10);
-    ps_crc0 = _mm_castsi128_ps(*xmm_crc0);
-    ps_crc3 = _mm_castsi128_ps(*xmm_crc3);
-    ps_res = _mm_xor_ps(ps_crc0, ps_crc3);
-
-    *xmm_crc0 = *xmm_crc1;
-    *xmm_crc1 = *xmm_crc2;
-    *xmm_crc2 = x_tmp3;
-    *xmm_crc3 = _mm_castps_si128(ps_res);
-}
-
-local void fold_2(__m128i *xmm_crc0, __m128i *xmm_crc1, __m128i *xmm_crc2, __m128i *xmm_crc3) {
-    const __m128i xmm_fold4 = _mm_set_epi32(
-            0x00000001, 0x54442bd4,
-            0x00000001, 0xc6e41596);
-
-    __m128i x_tmp3, x_tmp2;
-    __m128 ps_crc0, ps_crc1, ps_crc2, ps_crc3, ps_res31, ps_res20;
-
-    x_tmp3 = *xmm_crc3;
-    x_tmp2 = *xmm_crc2;
-
-    *xmm_crc3 = *xmm_crc1;
-    *xmm_crc1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x01);
-    *xmm_crc3 = _mm_clmulepi64_si128(*xmm_crc3, xmm_fold4, 0x10);
-    ps_crc3 = _mm_castsi128_ps(*xmm_crc3);
-    ps_crc1 = _mm_castsi128_ps(*xmm_crc1);
-    ps_res31 = _mm_xor_ps(ps_crc3, ps_crc1);
-
-    *xmm_crc2 = *xmm_crc0;
-    *xmm_crc0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x01);
-    *xmm_crc2 = _mm_clmulepi64_si128(*xmm_crc2, xmm_fold4, 0x10);
-    ps_crc0 = _mm_castsi128_ps(*xmm_crc0);
-    ps_crc2 = _mm_castsi128_ps(*xmm_crc2);
-    ps_res20 = _mm_xor_ps(ps_crc0, ps_crc2);
-
-    *xmm_crc0 = x_tmp2;
-    *xmm_crc1 = x_tmp3;
-    *xmm_crc2 = _mm_castps_si128(ps_res20);
-    *xmm_crc3 = _mm_castps_si128(ps_res31);
+// interestingly, MSVC seems to generate better code if using VXORPS over VPXOR
+// original Intel code uses XORPS for many XOR operations, but PXOR is pretty much always better (more port freedom on Intel CPUs). The only advantage of XORPS is that it's 1 byte shorter, an advantage which disappears under AVX as both instructions have the same length
+#if defined(__AVX__) && defined(YENC_BUILD_NATIVE) && YENC_BUILD_NATIVE!=0
+# define fold_xor _mm_xor_si128
+#else
+static __m128i fold_xor(__m128i a, __m128i b) {
+	return _mm_castps_si128(_mm_xor_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b)));
 }
+#endif
 
-local void fold_3(__m128i *xmm_crc0, __m128i *xmm_crc1, __m128i *xmm_crc2, __m128i *xmm_crc3) {
+#ifdef ENABLE_AVX512
+static __m128i do_one_fold_merge(__m128i src, __m128i data) {
     const __m128i xmm_fold4 = _mm_set_epi32(
             0x00000001, 0x54442bd4,
             0x00000001, 0xc6e41596);
-
-    __m128i x_tmp3;
-    __m128 ps_crc0, ps_crc1, ps_crc2, ps_crc3, ps_res32, ps_res21, ps_res10;
-
-    x_tmp3 = *xmm_crc3;
-
-    *xmm_crc3 = *xmm_crc2;
-    *xmm_crc2 = _mm_clmulepi64_si128(*xmm_crc2, xmm_fold4, 0x01);
-    *xmm_crc3 = _mm_clmulepi64_si128(*xmm_crc3, xmm_fold4, 0x10);
-    ps_crc2 = _mm_castsi128_ps(*xmm_crc2);
-    ps_crc3 = _mm_castsi128_ps(*xmm_crc3);
-    ps_res32 = _mm_xor_ps(ps_crc2, ps_crc3);
-
-    *xmm_crc2 = *xmm_crc1;
-    *xmm_crc1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x01);
-    *xmm_crc2 = _mm_clmulepi64_si128(*xmm_crc2, xmm_fold4, 0x10);
-    ps_crc1 = _mm_castsi128_ps(*xmm_crc1);
-    ps_crc2 = _mm_castsi128_ps(*xmm_crc2);
-    ps_res21 = _mm_xor_ps(ps_crc1, ps_crc2);
-
-    *xmm_crc1 = *xmm_crc0;
-    *xmm_crc0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x01);
-    *xmm_crc1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x10);
-    ps_crc0 = _mm_castsi128_ps(*xmm_crc0);
-    ps_crc1 = _mm_castsi128_ps(*xmm_crc1);
-    ps_res10 = _mm_xor_ps(ps_crc0, ps_crc1);
-
-    *xmm_crc0 = x_tmp3;
-    *xmm_crc1 = _mm_castps_si128(ps_res10);
-    *xmm_crc2 = _mm_castps_si128(ps_res21);
-    *xmm_crc3 = _mm_castps_si128(ps_res32);
+    return _mm_ternarylogic_epi32(
+      _mm_clmulepi64_si128(src, xmm_fold4, 0x01),
+      _mm_clmulepi64_si128(src, xmm_fold4, 0x10),
+      data,
+      0x96
+    );
 }
-
-local void fold_4(__m128i *xmm_crc0, __m128i *xmm_crc1, __m128i *xmm_crc2, __m128i *xmm_crc3) {
+#else
+static __m128i do_one_fold(__m128i src) {
     const __m128i xmm_fold4 = _mm_set_epi32(
             0x00000001, 0x54442bd4,
             0x00000001, 0xc6e41596);
-
-    __m128i x_tmp0, x_tmp1, x_tmp2, x_tmp3;
-    __m128 ps_crc0, ps_crc1, ps_crc2, ps_crc3;
-    __m128 ps_t0, ps_t1, ps_t2, ps_t3;
-    __m128 ps_res0, ps_res1, ps_res2, ps_res3;
-
-    x_tmp0 = *xmm_crc0;
-    x_tmp1 = *xmm_crc1;
-    x_tmp2 = *xmm_crc2;
-    x_tmp3 = *xmm_crc3;
-
-    *xmm_crc0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x01);
-    x_tmp0 = _mm_clmulepi64_si128(x_tmp0, xmm_fold4, 0x10);
-    ps_crc0 = _mm_castsi128_ps(*xmm_crc0);
-    ps_t0 = _mm_castsi128_ps(x_tmp0);
-    ps_res0 = _mm_xor_ps(ps_crc0, ps_t0);
-
-    *xmm_crc1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x01);
-    x_tmp1 = _mm_clmulepi64_si128(x_tmp1, xmm_fold4, 0x10);
-    ps_crc1 = _mm_castsi128_ps(*xmm_crc1);
-    ps_t1 = _mm_castsi128_ps(x_tmp1);
-    ps_res1 = _mm_xor_ps(ps_crc1, ps_t1);
-
-    *xmm_crc2 = _mm_clmulepi64_si128(*xmm_crc2, xmm_fold4, 0x01);
-    x_tmp2 = _mm_clmulepi64_si128(x_tmp2, xmm_fold4, 0x10);
-    ps_crc2 = _mm_castsi128_ps(*xmm_crc2);
-    ps_t2 = _mm_castsi128_ps(x_tmp2);
-    ps_res2 = _mm_xor_ps(ps_crc2, ps_t2);
-
-    *xmm_crc3 = _mm_clmulepi64_si128(*xmm_crc3, xmm_fold4, 0x01);
-    x_tmp3 = _mm_clmulepi64_si128(x_tmp3, xmm_fold4, 0x10);
-    ps_crc3 = _mm_castsi128_ps(*xmm_crc3);
-    ps_t3 = _mm_castsi128_ps(x_tmp3);
-    ps_res3 = _mm_xor_ps(ps_crc3, ps_t3);
-
-    *xmm_crc0 = _mm_castps_si128(ps_res0);
-    *xmm_crc1 = _mm_castps_si128(ps_res1);
-    *xmm_crc2 = _mm_castps_si128(ps_res2);
-    *xmm_crc3 = _mm_castps_si128(ps_res3);
+    return fold_xor(
+      _mm_clmulepi64_si128(src, xmm_fold4, 0x01),
+      _mm_clmulepi64_si128(src, xmm_fold4, 0x10)
+    );
 }
+#endif
 
-ALIGN(32, local const unsigned  pshufb_shf_table[60]) = {
+ALIGN_TO(32, static const unsigned  pshufb_shf_table[60]) = {
     0x84838281, 0x88878685, 0x8c8b8a89, 0x008f8e8d, /* shl 15 (16 - 1)/shr1 */
     0x85848382, 0x89888786, 0x8d8c8b8a, 0x01008f8e, /* shl 14 (16 - 3)/shr2 */
     0x86858483, 0x8a898887, 0x8e8d8c8b, 0x0201008f, /* shl 13 (16 - 4)/shr3 */
@@ -186,17 +82,13 @@ ALIGN(32, local const unsigned  pshufb_shf_table[60]) = {
     0x0201008f, 0x06050403, 0x0a090807, 0x0e0d0c0b  /* shl  1 (16 -15)/shr15*/
 };
 
-local void partial_fold(const size_t len, __m128i *xmm_crc0, __m128i *xmm_crc1,
+static void partial_fold(const size_t len, __m128i *xmm_crc0, __m128i *xmm_crc1,
         __m128i *xmm_crc2, __m128i *xmm_crc3, __m128i *xmm_crc_part) {
 
-    const __m128i xmm_fold4 = _mm_set_epi32(
-            0x00000001, 0x54442bd4,
-            0x00000001, 0xc6e41596);
     const __m128i xmm_mask3 = _mm_set1_epi32(0x80808080);
 
     __m128i xmm_shl, xmm_shr, xmm_tmp1, xmm_tmp2, xmm_tmp3;
-    __m128i xmm_a0_0, xmm_a0_1;
-    __m128 ps_crc3, psa0_0, psa0_1, ps_res;
+    __m128i xmm_a0_0;
 
     xmm_shl = _mm_load_si128((__m128i *)pshufb_shf_table + (len - 1));
     xmm_shr = xmm_shl;
@@ -220,41 +112,82 @@ local void partial_fold(const size_t len, __m128i *xmm_crc0, __m128i *xmm_crc1,
     *xmm_crc_part = _mm_shuffle_epi8(*xmm_crc_part, xmm_shl);
     *xmm_crc3 = _mm_or_si128(*xmm_crc3, *xmm_crc_part);
 
-    xmm_a0_1 = _mm_clmulepi64_si128(xmm_a0_0, xmm_fold4, 0x10);
-    xmm_a0_0 = _mm_clmulepi64_si128(xmm_a0_0, xmm_fold4, 0x01);
-
-    ps_crc3 = _mm_castsi128_ps(*xmm_crc3);
-    psa0_0 = _mm_castsi128_ps(xmm_a0_0);
-    psa0_1 = _mm_castsi128_ps(xmm_a0_1);
-
-    ps_res = _mm_xor_ps(ps_crc3, psa0_0);
-    ps_res = _mm_xor_ps(ps_res, psa0_1);
-
-    *xmm_crc3 = _mm_castps_si128(ps_res);
+#ifdef ENABLE_AVX512
+    *xmm_crc3 = do_one_fold_merge(xmm_a0_0, *xmm_crc3);
+#else
+    *xmm_crc3 = fold_xor(
+      do_one_fold(xmm_a0_0),
+      *xmm_crc3
+    );
+#endif
 }
 
-ALIGN(16, local const unsigned crc_k[]) = {
+ALIGN_TO(16, static const unsigned crc_k[]) = {
     0xccaa009e, 0x00000000, /* rk1 */
     0x751997d0, 0x00000001, /* rk2 */
     0xccaa009e, 0x00000000, /* rk5 */
     0x63cd6124, 0x00000001, /* rk6 */
-    0xf7011640, 0x00000001, /* rk7 */
+    0xf7011641, 0x00000000, /* rk7 */
     0xdb710640, 0x00000001  /* rk8 */
 };
 
-ALIGN(16, local const unsigned crc_mask[4]) = {
-    0xFFFFFFFF, 0xFFFFFFFF, 0x00000000, 0x00000000
-};
-
-ALIGN(16, local const unsigned crc_mask2[4]) = {
+ALIGN_TO(16, static const unsigned crc_mask[4]) = {
     0x00000000, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF
 };
 
-uint32_t crc_fold(const unsigned char *src, long len) {
+static __m128i reverse_bits_epi8(__m128i src) {
+#if defined(__GFNI__) && defined(YENC_BUILD_NATIVE) && YENC_BUILD_NATIVE!=0
+    return _mm_gf2p8affine_epi64_epi8(src, _mm_set_epi32(
+      0x80402010, 0x08040201,
+      0x80402010, 0x08040201
+    ), 0);
+#else
+    __m128i xmm_t0 = _mm_and_si128(src, _mm_set1_epi8(0x0f));
+    __m128i xmm_t1 = _mm_and_si128(_mm_srli_epi16(src, 4), _mm_set1_epi8(0x0f));
+    xmm_t0 = _mm_shuffle_epi8(_mm_set_epi8(
+      -16, 112, -80, 48, -48, 80, -112, 16, -32, 96, -96, 32, -64, 64, -128, 0
+      //0xf0, 0x70, 0xb0, 0x30, 0xd0, 0x50, 0x90, 0x10, 0xe0, 0x60, 0xa0, 0x20, 0xc0, 0x40, 0x80, 0
+    ), xmm_t0);
+    xmm_t1 = _mm_shuffle_epi8(_mm_set_epi8(
+      15, 7, 11, 3, 13, 5, 9, 1, 14, 6, 10, 2, 12, 4, 8, 0
+    ), xmm_t1);
+    return _mm_or_si128(xmm_t0, xmm_t1);
+#endif
+}
+
+#ifdef _MSC_VER
+// because MSVC doesn't use BSWAP unless you specifically tell it to...
+# include <stdlib.h>
+# define BSWAP32 _byteswap_ulong
+#else
+# define BSWAP32(n) ((((n)&0xff)<<24) | (((n)&0xff00)<<8) | (((n)&0xff0000)>>8) | (((n)&0xff000000)>>24))
+#endif
+
+static uint32_t crc_fold(const unsigned char *src, long len, uint32_t initial) {
     unsigned long algn_diff;
     __m128i xmm_t0, xmm_t1, xmm_t2, xmm_t3;
 
-    __m128i xmm_crc0 = _mm_cvtsi32_si128(0x9db42487);
+    // TODO: consider calculating this via a LUT instead (probably faster)
+    // info from https://www.reddit.com/r/ReverseEngineering/comments/2zwhl3/mystery_constant_0x9db42487_in_intels_crc32ieee/
+    // firstly, calculate: xmm_crc0 = (intial * 0x487b9c8a) mod 0x104c11db7, where 0x487b9c8a = inverse(1<<512) mod 0x104c11db7
+    
+    // reverse input bits + load into XMM register
+    uint32_t init_t = BSWAP32(initial);
+    xmm_t0 = reverse_bits_epi8(_mm_cvtsi32_si128(~init_t));
+    
+    xmm_t0 = _mm_clmulepi64_si128(xmm_t0, _mm_cvtsi32_si128(0x487b9c8a), 0);
+    xmm_t1 = _mm_and_si128(xmm_t0, _mm_set_epi32(-1,-1,-1,0)); // shifted up by 32bits to avoid shifts by using clmul's capability to select top 64bits instead
+    xmm_t2 = _mm_set_epi32( // polynomial reduction factors
+      0, 0x04c11db7, // G*
+      1, 0x04d101df  // Q+
+    );
+    xmm_t1 = _mm_clmulepi64_si128(xmm_t1, xmm_t2, 0);
+    xmm_t1 = _mm_clmulepi64_si128(xmm_t1, xmm_t2, 0x11);
+    
+    __m128i xmm_crc0 = _mm_xor_si128(xmm_t0, xmm_t1);
+    // reverse bits
+    xmm_crc0 = _mm_shuffle_epi8(reverse_bits_epi8(xmm_crc0), _mm_set_epi32(-1,-1,-1,0x00010203));
+    
     __m128i xmm_crc1 = _mm_setzero_si128();
     __m128i xmm_crc2 = _mm_setzero_si128();
     __m128i xmm_crc3 = _mm_setzero_si128();
@@ -262,7 +195,7 @@ uint32_t crc_fold(const unsigned char *src, long len) {
 
     if (len < 16) {
         if (len == 0)
-            return 0;
+            return initial;
         xmm_crc_part = _mm_loadu_si128((__m128i *)src);
         goto partial;
     }
@@ -284,12 +217,22 @@ uint32_t crc_fold(const unsigned char *src, long len) {
         xmm_t2 = _mm_load_si128((__m128i *)src + 2);
         xmm_t3 = _mm_load_si128((__m128i *)src + 3);
 
-        fold_4(&xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3);
-
+#ifdef ENABLE_AVX512
+        xmm_crc0 = do_one_fold_merge(xmm_crc0, xmm_t0);
+        xmm_crc1 = do_one_fold_merge(xmm_crc1, xmm_t1);
+        xmm_crc2 = do_one_fold_merge(xmm_crc2, xmm_t2);
+        xmm_crc3 = do_one_fold_merge(xmm_crc3, xmm_t3);
+#else
+        // nesting do_one_fold() in _mm_xor_si128() seems to cause MSVC to generate horrible code, so separate it out
+        xmm_crc0 = do_one_fold(xmm_crc0);
+        xmm_crc1 = do_one_fold(xmm_crc1);
+        xmm_crc2 = do_one_fold(xmm_crc2);
+        xmm_crc3 = do_one_fold(xmm_crc3);
         xmm_crc0 = _mm_xor_si128(xmm_crc0, xmm_t0);
         xmm_crc1 = _mm_xor_si128(xmm_crc1, xmm_t1);
         xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_t2);
         xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t3);
+#endif
 
         src += 64;
     }
@@ -304,11 +247,20 @@ uint32_t crc_fold(const unsigned char *src, long len) {
         xmm_t1 = _mm_load_si128((__m128i *)src + 1);
         xmm_t2 = _mm_load_si128((__m128i *)src + 2);
 
-        fold_3(&xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3);
-
-        xmm_crc1 = _mm_xor_si128(xmm_crc1, xmm_t0);
-        xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_t1);
+        xmm_t3 = xmm_crc3;
+#ifdef ENABLE_AVX512
+        xmm_crc3 = do_one_fold_merge(xmm_crc2, xmm_t2);
+        xmm_crc2 = do_one_fold_merge(xmm_crc1, xmm_t1);
+        xmm_crc1 = do_one_fold_merge(xmm_crc0, xmm_t0);
+#else
+        xmm_crc3 = do_one_fold(xmm_crc2);
+        xmm_crc2 = do_one_fold(xmm_crc1);
+        xmm_crc1 = do_one_fold(xmm_crc0);
         xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t2);
+        xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_t1);
+        xmm_crc1 = _mm_xor_si128(xmm_crc1, xmm_t0);
+#endif
+        xmm_crc0 = xmm_t3;
 
         if (len == 0)
             goto done;
@@ -320,10 +272,19 @@ uint32_t crc_fold(const unsigned char *src, long len) {
         xmm_t0 = _mm_load_si128((__m128i *)src);
         xmm_t1 = _mm_load_si128((__m128i *)src + 1);
 
-        fold_2(&xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3);
-
-        xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_t0);
+        xmm_t2 = xmm_crc2;
+        xmm_t3 = xmm_crc3;
+#ifdef ENABLE_AVX512
+        xmm_crc3 = do_one_fold_merge(xmm_crc1, xmm_t1);
+        xmm_crc2 = do_one_fold_merge(xmm_crc0, xmm_t0);
+#else
+        xmm_crc3 = do_one_fold(xmm_crc1);
+        xmm_crc2 = do_one_fold(xmm_crc0);
         xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t1);
+        xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_t0);
+#endif
+        xmm_crc1 = xmm_t3;
+        xmm_crc0 = xmm_t2;
 
         if (len == 0)
             goto done;
@@ -334,9 +295,15 @@ uint32_t crc_fold(const unsigned char *src, long len) {
 
         xmm_t0 = _mm_load_si128((__m128i *)src);
 
-        fold_1(&xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3);
-
-        xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t0);
+        xmm_t3 = xmm_crc3;
+#ifdef ENABLE_AVX512
+        xmm_crc3 = do_one_fold_merge(xmm_crc0, xmm_t0);
+#else
+        xmm_crc3 = _mm_xor_si128(do_one_fold(xmm_crc0), xmm_t0);
+#endif
+        xmm_crc0 = xmm_crc1;
+        xmm_crc1 = xmm_crc2;
+        xmm_crc2 = xmm_t3;
 
         if (len == 0)
             goto done;
@@ -354,10 +321,7 @@ partial:
         &xmm_crc_part);
 done:
 {
-    const __m128i xmm_mask  = _mm_load_si128((__m128i *)crc_mask);
-    const __m128i xmm_mask2 = _mm_load_si128((__m128i *)crc_mask2);
-
-    uint32_t crc;
+    const __m128i xmm_mask = _mm_load_si128((__m128i *)crc_mask);
     __m128i x_tmp0, x_tmp1, x_tmp2, crc_fold;
 
     /*
@@ -367,18 +331,30 @@ done:
 
     x_tmp0 = _mm_clmulepi64_si128(xmm_crc0, crc_fold, 0x10);
     xmm_crc0 = _mm_clmulepi64_si128(xmm_crc0, crc_fold, 0x01);
+#ifdef ENABLE_AVX512
+    xmm_crc1 = _mm_ternarylogic_epi32(xmm_crc1, x_tmp0, xmm_crc0, 0x96);
+#else
     xmm_crc1 = _mm_xor_si128(xmm_crc1, x_tmp0);
     xmm_crc1 = _mm_xor_si128(xmm_crc1, xmm_crc0);
+#endif
 
     x_tmp1 = _mm_clmulepi64_si128(xmm_crc1, crc_fold, 0x10);
     xmm_crc1 = _mm_clmulepi64_si128(xmm_crc1, crc_fold, 0x01);
+#ifdef ENABLE_AVX512
+    xmm_crc2 = _mm_ternarylogic_epi32(xmm_crc2, x_tmp1, xmm_crc1, 0x96);
+#else
     xmm_crc2 = _mm_xor_si128(xmm_crc2, x_tmp1);
     xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_crc1);
+#endif
 
     x_tmp2 = _mm_clmulepi64_si128(xmm_crc2, crc_fold, 0x10);
     xmm_crc2 = _mm_clmulepi64_si128(xmm_crc2, crc_fold, 0x01);
+#ifdef ENABLE_AVX512
+    xmm_crc3 = _mm_ternarylogic_epi32(xmm_crc3, x_tmp2, xmm_crc2, 0x96);
+#else
     xmm_crc3 = _mm_xor_si128(xmm_crc3, x_tmp2);
     xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc2);
+#endif
 
     /*
      * k5
@@ -393,29 +369,43 @@ done:
     xmm_crc0 = xmm_crc3;
     xmm_crc3 = _mm_slli_si128(xmm_crc3, 4);
     xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0x10);
+#ifdef ENABLE_AVX512
+    //xmm_crc3 = _mm_maskz_xor_epi32(14, xmm_crc3, xmm_crc0);
+    xmm_crc3 = _mm_ternarylogic_epi32(xmm_crc3, xmm_crc0, xmm_mask, 0x28);
+#else
+    xmm_crc0 = _mm_and_si128(xmm_crc0, xmm_mask);
     xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc0);
-    xmm_crc3 = _mm_and_si128(xmm_crc3, xmm_mask2);
+#endif
 
     /*
      * k7
      */
     xmm_crc1 = xmm_crc3;
-    xmm_crc2 = xmm_crc3;
     crc_fold = _mm_load_si128((__m128i *)crc_k + 2);
 
     xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0);
-    xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc2);
-    xmm_crc3 = _mm_and_si128(xmm_crc3, xmm_mask);
-
-    xmm_crc2 = xmm_crc3;
     xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0x10);
-    xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc2);
+#ifdef ENABLE_AVX512
+    xmm_crc3 = _mm_ternarylogic_epi32(xmm_crc3, xmm_crc1, xmm_crc1, 0xC3); // NOT(xmm_crc3 ^ xmm_crc1)
+#else
+    xmm_crc1 = _mm_xor_si128(xmm_crc1, xmm_mask);
     xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc1);
+#endif
+    return _mm_extract_epi32(xmm_crc3, 2);
+}
 
-    crc = _mm_extract_epi32(xmm_crc3, 2);
-    return ~crc;
 }
 
+static uint32_t do_crc32_incremental_clmul(const void* data, size_t length, uint32_t init) {
+	return crc_fold((const unsigned char*)data, (long)length, init);
+}
+
+void crc_clmul_set_funcs(crc_func* _do_crc32_incremental) {
+	*_do_crc32_incremental = &do_crc32_incremental_clmul;
+}
+#else
+void crc_clmul_set_funcs(crc_func* _do_crc32_incremental) {
+    (void)_do_crc32_incremental;
 }
 #endif
 

package/src/decoder.cc

@@ -0,0 +1,61 @@
+#include "common.h"
+
+#include "decoder_common.h"
+#include "decoder.h"
+
+extern "C" {
+	YencDecoderEnd (*_do_decode)(const unsigned char*HEDLEY_RESTRICT*, unsigned char*HEDLEY_RESTRICT*, size_t, YencDecoderState*) = &do_decode_scalar<false, false>;
+	YencDecoderEnd (*_do_decode_raw)(const unsigned char*HEDLEY_RESTRICT*, unsigned char*HEDLEY_RESTRICT*, size_t, YencDecoderState*) = &do_decode_scalar<true, false>;
+	YencDecoderEnd (*_do_decode_end_raw)(const unsigned char*HEDLEY_RESTRICT*, unsigned char*HEDLEY_RESTRICT*, size_t, YencDecoderState*) = &do_decode_end_scalar<true>;
+}
+
+void decoder_set_sse2_funcs();
+void decoder_set_ssse3_funcs();
+void decoder_set_avx_funcs();
+void decoder_set_avx2_funcs();
+void decoder_set_neon_funcs();
+
+
+#if defined(PLATFORM_X86) && defined(YENC_BUILD_NATIVE) && YENC_BUILD_NATIVE!=0
+# if defined(__AVX2__) && !defined(YENC_DISABLE_AVX256)
+#  include "decoder_avx2_base.h"
+static inline void decoder_set_native_funcs() {
+	ALIGN_ALLOC(lookups, sizeof(*lookups), 16);
+	decoder_init_lut(lookups->eqFix, lookups->compact);
+	_do_decode = &do_decode_simd<false, false, sizeof(__m256i)*2, do_decode_avx2<false, false, ISA_NATIVE> >;
+	_do_decode_raw = &do_decode_simd<true, false, sizeof(__m256i)*2, do_decode_avx2<true, false, ISA_NATIVE> >;
+	_do_decode_end_raw = &do_decode_simd<true, true, sizeof(__m256i)*2, do_decode_avx2<true, true, ISA_NATIVE> >;
+}
+# else
+#  include "decoder_sse_base.h"
+static inline void decoder_set_native_funcs() {
+	decoder_sse_init();
+	decoder_init_lut(lookups->eqFix, lookups->compact);
+	_do_decode = &do_decode_simd<false, false, sizeof(__m128i)*2, do_decode_sse<false, false, ISA_NATIVE> >;
+	_do_decode_raw = &do_decode_simd<true, false, sizeof(__m128i)*2, do_decode_sse<true, false, ISA_NATIVE> >;
+	_do_decode_end_raw = &do_decode_simd<true, true, sizeof(__m128i)*2, do_decode_sse<true, true, ISA_NATIVE> >;
+}
+# endif
+#endif
+
+void decoder_init() {
+#ifdef PLATFORM_X86
+# if defined(YENC_BUILD_NATIVE) && YENC_BUILD_NATIVE!=0
+	decoder_set_native_funcs();
+# else
+	int use_isa = cpu_supports_isa();
+	if(use_isa >= ISA_LEVEL_AVX2)
+		decoder_set_avx2_funcs();
+	else if(use_isa >= ISA_LEVEL_AVX)
+		decoder_set_avx_funcs();
+	else if(use_isa >= ISA_LEVEL_SSSE3)
+		decoder_set_ssse3_funcs();
+	else
+		decoder_set_sse2_funcs();
+# endif
+#endif
+#ifdef PLATFORM_ARM
+	if(cpu_supports_neon())
+		decoder_set_neon_funcs();
+#endif
+}

package/src/decoder.h

@@ -0,0 +1,53 @@
+#ifndef __YENC_DECODER_H
+#define __YENC_DECODER_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+
+// the last state that the decoder was in (i.e. last few characters processed)
+// the state is needed for incremental decoders as its behavior is affected by what it processed last
+// acronyms: CR = carriage return (\r), LF = line feed (\n), EQ = equals char, DT = dot char (.)
+typedef enum {
+	YDEC_STATE_CRLF, // default
+	YDEC_STATE_EQ,
+	YDEC_STATE_CR,
+	YDEC_STATE_NONE,
+	YDEC_STATE_CRLFDT,
+	YDEC_STATE_CRLFDTCR,
+	YDEC_STATE_CRLFEQ // may actually be "\r\n.=" in raw decoder
+} YencDecoderState;
+
+// end result for incremental processing (whether the end of the yEnc data was reached)
+typedef enum {
+	YDEC_END_NONE,    // end not reached
+	YDEC_END_CONTROL, // \r\n=y sequence found, src points to byte after 'y'
+	YDEC_END_ARTICLE  // \r\n.\r\n sequence found, src points to byte after last '\n'
+} YencDecoderEnd;
+
+#include "hedley.h"
+
+extern YencDecoderEnd (*_do_decode)(const unsigned char*HEDLEY_RESTRICT*, unsigned char*HEDLEY_RESTRICT*, size_t, YencDecoderState*);
+extern YencDecoderEnd (*_do_decode_raw)(const unsigned char*HEDLEY_RESTRICT*, unsigned char*HEDLEY_RESTRICT*, size_t, YencDecoderState*);
+extern YencDecoderEnd (*_do_decode_end_raw)(const unsigned char*HEDLEY_RESTRICT*, unsigned char*HEDLEY_RESTRICT*, size_t, YencDecoderState*);
+
+static inline size_t do_decode(int isRaw, const unsigned char* HEDLEY_RESTRICT src, unsigned char* HEDLEY_RESTRICT dest, size_t len, YencDecoderState* state) {
+	unsigned char* ds = dest;
+	(*(isRaw ? _do_decode_raw : _do_decode))(&src, &ds, len, state);
+	return ds - dest;
+}
+
+static inline YencDecoderEnd do_decode_end(const unsigned char*HEDLEY_RESTRICT* src, unsigned char*HEDLEY_RESTRICT* dest, size_t len, YencDecoderState* state) {
+	return _do_decode_end_raw(src, dest, len, state);
+}
+
+void decoder_init();
+
+
+
+#ifdef __cplusplus
+}
+#endif
+#endif

package/src/decoder_avx.cc

@@ -0,0 +1,18 @@
+#include "common.h"
+
+#if defined(__AVX__) && defined(__POPCNT__)
+#include "decoder_common.h"
+#include "decoder_sse_base.h"
+void decoder_set_avx_funcs() {
+	decoder_sse_init();
+	decoder_init_lut(lookups->eqFix, lookups->compact);
+	_do_decode = &do_decode_simd<false, false, sizeof(__m128i)*2, do_decode_sse<false, false, ISA_LEVEL_SSE4_POPCNT> >;
+	_do_decode_raw = &do_decode_simd<true, false, sizeof(__m128i)*2, do_decode_sse<true, false, ISA_LEVEL_SSE4_POPCNT> >;
+	_do_decode_end_raw = &do_decode_simd<true, true, sizeof(__m128i)*2, do_decode_sse<true, true, ISA_LEVEL_SSE4_POPCNT> >;
+}
+#else
+void decoder_set_ssse3_funcs();
+void decoder_set_avx_funcs() {
+	decoder_set_ssse3_funcs();
+}
+#endif

package/src/decoder_avx2.cc

@@ -0,0 +1,18 @@
+#include "common.h"
+
+#if defined(__AVX2__) && !defined(YENC_DISABLE_AVX256)
+#include "decoder_common.h"
+#include "decoder_avx2_base.h"
+void decoder_set_avx2_funcs() {
+	ALIGN_ALLOC(lookups, sizeof(*lookups), 16);
+	decoder_init_lut(lookups->eqFix, lookups->compact);
+	_do_decode = &do_decode_simd<false, false, sizeof(__m256i)*2, do_decode_avx2<false, false, ISA_LEVEL_AVX2> >;
+	_do_decode_raw = &do_decode_simd<true, false, sizeof(__m256i)*2, do_decode_avx2<true, false, ISA_LEVEL_AVX2> >;
+	_do_decode_end_raw = &do_decode_simd<true, true, sizeof(__m256i)*2, do_decode_avx2<true, true, ISA_LEVEL_AVX2> >;
+}
+#else
+void decoder_set_avx_funcs();
+void decoder_set_avx2_funcs() {
+	decoder_set_avx_funcs();
+}
+#endif