yencode 1.1.0 → 1.1.3

package/src/crc_arm.cc

@@ -1,15 +1,98 @@
-#include "common.h"
 #include "crc_common.h"
 
-#if defined(__ARM_FEATURE_CRC32) || defined(_M_ARM64) /* TODO: AArch32 for MSVC? */
+#if defined(PLATFORM_ARM) && defined(_MSC_VER) && defined(__clang__) && !defined(__ARM_FEATURE_CRC32)
+// I don't think GYP provides a nice way to detect whether MSVC or clang-cl is being used, but it doesn't use clang-cl by default, so a warning here is probably sufficient
+HEDLEY_WARNING("CRC32 acceleration is not been enabled under ARM clang-cl by default; add `-march=armv8-a+crc` to additional compiler arguments to enable");
+#endif
+
+// disable CRC on GCC versions with broken arm_acle.h
+#if defined(__ARM_FEATURE_CRC32) && defined(HEDLEY_GCC_VERSION)
+# if !defined(__aarch64__) && HEDLEY_GCC_VERSION_CHECK(7,0,0) && !HEDLEY_GCC_VERSION_CHECK(8,1,1)
+#  undef __ARM_FEATURE_CRC32
+HEDLEY_WARNING("CRC32 acceleration has been disabled due to broken arm_acle.h shipped in GCC 7.0 - 8.1 [https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81497]. If you need this feature, please use a different compiler or version of GCC");
+# endif
+# if defined(__aarch64__) && HEDLEY_GCC_VERSION_CHECK(9,4,0) && !HEDLEY_GCC_VERSION_CHECK(9,5,0)
+#  undef __ARM_FEATURE_CRC32
+HEDLEY_WARNING("CRC32 acceleration has been disabled due to broken arm_acle.h shipped in GCC 9.4 [https://gcc.gnu.org/bugzilla/show_bug.cgi?id=100985]. If you need this feature, please use a different compiler or version of GCC");
+# endif
+#endif
+
+#if defined(__ARM_FEATURE_CRC32) || (defined(_M_ARM64) && !defined(__clang__)) // MSVC doesn't support CRC for ARM32
 
 /* ARMv8 accelerated CRC */
-#ifdef _MSC_VER
+#if defined(_MSC_VER) && !defined(__clang__)
 #include <intrin.h>
 #else
 #include <arm_acle.h>
 #endif
 
+
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+# ifdef __GNUC__
+#  define _LE16 __builtin_bswap16
+#  define _LE32 __builtin_bswap32
+#  define _LE64 __builtin_bswap64
+# else
+// currently not supported
+#  error No endian swap intrinsic defined
+# endif
+#else
+# define _LE16(x) (x)
+# define _LE32(x) (x)
+# define _LE64(x) (x)
+#endif
+
+#ifdef __aarch64__
+# define WORD_T uint64_t
+# define WORDSIZE_LOG 3  // sizeof(WORD_T) == 1<<WORDSIZE_LOG
+# define CRC_WORD(crc, data) __crc32d(crc, _LE64(data))
+#else
+# define WORD_T uint32_t
+# define WORDSIZE_LOG 2  // sizeof(WORD_T) == 1<<WORDSIZE_LOG
+# define CRC_WORD(crc, data) __crc32w(crc, _LE32(data))
+#endif
+
+
+// exploit CPU pipelining during CRC computation; unfortunately I haven't been able to measure any benefit
+// - Neoverse N1: no noticeable difference
+// - Cortex A53: actually runs a bit slower
+//#define ENABLE_PIPELINE_OPT 1
+
+#ifdef ENABLE_PIPELINE_OPT
+// workaround MSVC complaining "unary minus operator applied to unsigned type, result still unsigned"
+#define NEGATE(n) (uint32_t)(-((int32_t)(n)))
+
+static HEDLEY_ALWAYS_INLINE uint32_t crc_multiply(uint32_t a, uint32_t b) {
+	uint32_t res = 0;
+	for(int i=0; i<31; i++) {
+		res ^= NEGATE(b>>31) & a;
+		a = ((a >> 1) ^ (0xEDB88320 & NEGATE(a&1)));
+		b <<= 1;
+	}
+	res ^= NEGATE(b>>31) & a;
+	return res;
+}
+
+static const uint32_t crc_power[] = { // pre-computed 2^n, with first 3 entries removed (saves a shift)
+	0x00800000, 0x00008000, 0xedb88320, 0xb1e6b092, 0xa06a2517, 0xed627dae, 0x88d14467, 0xd7bbfe6a,
+	0xec447f11, 0x8e7ea170, 0x6427800e, 0x4d47bae0, 0x09fe548f, 0x83852d0f, 0x30362f1a, 0x7b5a9cc3,
+	0x31fec169, 0x9fec022a, 0x6c8dedc4, 0x15d6874d, 0x5fde7a4e, 0xbad90e37, 0x2e4e5eef, 0x4eaba214,
+	0xa8a472c0, 0x429a969e, 0x148d302a, 0xc40ba6d0, 0xc4e22c3c, 0x40000000, 0x20000000, 0x08000000
+};
+/* above table can be computed with
+	int main(void) {
+		uint32_t k = 0x80000000 >> 1;
+		for (size_t i = 0; i < 32+3; ++i) {
+			if(i>2) printf("0x%08x, ", k);
+			k = crc_multiply(k, k);
+		}
+		return 0;
+	}
+*/
+#endif
+
+
+
 // inspired/stolen off https://github.com/jocover/crc32_armv8/blob/master/crc32_armv8.c
 static uint32_t arm_crc_calc(uint32_t crc, const unsigned char *src, long len) {
 	
@@ -21,35 +104,84 @@ static uint32_t arm_crc_calc(uint32_t crc, const unsigned char *src, long len) {
 			len--;
 		}
 		if ((uintptr_t)src & sizeof(uint16_t)) {
-			crc = __crc32h(crc, *((uint16_t *)src));
+			crc = __crc32h(crc, _LE16(*((uint16_t *)src)));
 			src += sizeof(uint16_t);
 			len -= sizeof(uint16_t);
 		}
-		
 #ifdef __aarch64__
 		if ((uintptr_t)src & sizeof(uint32_t)) {
-			crc = __crc32w(crc, *((uint32_t *)src));
+			crc = __crc32w(crc, _LE32(*((uint32_t *)src)));
 			src += sizeof(uint32_t);
 			len -= sizeof(uint32_t);
 		}
+#endif
 	}
-	while ((len -= sizeof(uint64_t)) >= 0) {
-		crc = __crc32d(crc, *((uint64_t *)src));
-		src += sizeof(uint64_t);
+	
+	const WORD_T* srcW = (const WORD_T*)src;
+	
+#ifdef ENABLE_PIPELINE_OPT
+	// uses ideas from https://github.com/komrad36/crc#option-13-golden
+	// (this is a slightly less efficient, but much simpler implementation of the idea)
+	const unsigned SPLIT_WORDS_LOG = 10;  // make sure it's at least 2
+	const unsigned SPLIT_WORDS = 1<<SPLIT_WORDS_LOG;
+	while(len >= (long)(sizeof(WORD_T)*SPLIT_WORDS*2)) {
+		// compute 2x CRCs concurrently to leverage piplining
+		uint32_t crc2 = 0;
+		for(unsigned i=0; i<SPLIT_WORDS; i+=4) {
+			crc = CRC_WORD(crc, *srcW);
+			crc2 = CRC_WORD(crc2, *(srcW + SPLIT_WORDS));
+			srcW++;
+			crc = CRC_WORD(crc, *srcW);
+			crc2 = CRC_WORD(crc2, *(srcW + SPLIT_WORDS));
+			srcW++;
+			crc = CRC_WORD(crc, *srcW);
+			crc2 = CRC_WORD(crc2, *(srcW + SPLIT_WORDS));
+			srcW++;
+			crc = CRC_WORD(crc, *srcW);
+			crc2 = CRC_WORD(crc2, *(srcW + SPLIT_WORDS));
+			srcW++;
+		}
+		// merge the CRCs
+		// since we're multiplying by a fixed number, it could be sped up with some lookup tables
+		crc = crc_multiply(crc, crc_power[SPLIT_WORDS_LOG + WORDSIZE_LOG]) ^ crc2;
+		srcW += SPLIT_WORDS;
+		len -= sizeof(WORD_T)*SPLIT_WORDS*2;
 	}
-	if (len & sizeof(uint32_t)) {
-		crc = __crc32w(crc, *((uint32_t *)src));
-		src += sizeof(uint32_t);
+#endif
+	
+	while ((len -= sizeof(WORD_T)*8) >= 0) {
+		crc = CRC_WORD(crc, *(srcW++));
+		crc = CRC_WORD(crc, *(srcW++));
+		crc = CRC_WORD(crc, *(srcW++));
+		crc = CRC_WORD(crc, *(srcW++));
+		crc = CRC_WORD(crc, *(srcW++));
+		crc = CRC_WORD(crc, *(srcW++));
+		crc = CRC_WORD(crc, *(srcW++));
+		crc = CRC_WORD(crc, *(srcW++));
 	}
-#else
+	if (len & sizeof(WORD_T)*4) {
+		crc = CRC_WORD(crc, *(srcW++));
+		crc = CRC_WORD(crc, *(srcW++));
+		crc = CRC_WORD(crc, *(srcW++));
+		crc = CRC_WORD(crc, *(srcW++));
+	}
+	if (len & sizeof(WORD_T)*2) {
+		crc = CRC_WORD(crc, *(srcW++));
+		crc = CRC_WORD(crc, *(srcW++));
+	}
+	if (len & sizeof(WORD_T)) {
+		crc = CRC_WORD(crc, *(srcW++));
 	}
-	while ((len -= sizeof(uint32_t)) >= 0) {
-		crc = __crc32w(crc, *((uint32_t *)src));
+	src = (const unsigned char*)srcW;
+	
+#ifdef __aarch64__
+	if (len & sizeof(uint32_t)) {
+		crc = __crc32w(crc, _LE32(*((uint32_t *)src)));
 		src += sizeof(uint32_t);
 	}
 #endif
 	if (len & sizeof(uint16_t)) {
-		crc = __crc32h(crc, *((uint16_t *)src));
+		crc = __crc32h(crc, _LE16(*((uint16_t *)src)));
 		src += sizeof(uint16_t);
 	}
 	if (len & sizeof(uint8_t))
@@ -58,20 +190,15 @@ static uint32_t arm_crc_calc(uint32_t crc, const unsigned char *src, long len) {
 	return crc;
 }
 
-static void do_crc32_arm(const void* data, size_t length, unsigned char out[4]) {
-	uint32_t crc = arm_crc_calc(~0, (const unsigned char*)data, (long)length);
-	UNPACK_4(out, ~crc);
-}
-static void do_crc32_incremental_arm(const void* data, size_t length, unsigned char init[4]) {
-	uint32_t crc = PACK_4(init);
-	crc = arm_crc_calc(~crc, (const unsigned char*)data, (long)length);
-	UNPACK_4(init, ~crc);
+static uint32_t do_crc32_incremental_arm(const void* data, size_t length, uint32_t init) {
+	return ~arm_crc_calc(~init, (const unsigned char*)data, (long)length);
 }
 
-void crc_arm_set_funcs(crc_func* _do_crc32, crc_func* _do_crc32_incremental) {
-	*_do_crc32 = &do_crc32_arm;
+void crc_arm_set_funcs(crc_func* _do_crc32_incremental) {
 	*_do_crc32_incremental = &do_crc32_incremental_arm;
 }
 #else
-void crc_arm_set_funcs(crc_func* _do_crc32, crc_func* _do_crc32_incremental) {}
+void crc_arm_set_funcs(crc_func* _do_crc32_incremental) {
+	(void)_do_crc32_incremental;
+}
 #endif

package/src/crc_common.h

@@ -1,11 +1,4 @@
-
-#define PACK_4(arr) (((uint_fast32_t)arr[0] << 24) | ((uint_fast32_t)arr[1] << 16) | ((uint_fast32_t)arr[2] << 8) | (uint_fast32_t)arr[3])
-#define UNPACK_4(arr, val) { \
-	arr[0] = (unsigned char)(val >> 24) & 0xFF; \
-	arr[1] = (unsigned char)(val >> 16) & 0xFF; \
-	arr[2] = (unsigned char)(val >>  8) & 0xFF; \
-	arr[3] = (unsigned char)val & 0xFF; \
-}
-
+#include "common.h"
 #include <stddef.h> // for size_t
-typedef void (*crc_func)(const void*, size_t, unsigned char[4]);
+#include "crc.h"
+

package/src/{crc_folding.c → crc_folding.cc}

@@ -19,44 +19,29 @@
 
 #include "crc_common.h"
  
-#if !defined(_MSC_VER) || defined(_STDINT) || _MSC_VER >= 1900
-# include <stdint.h>
-#else
-/* Workaround for older MSVC not supporting stdint.h - just pull it from V8 */
-# include <v8.h>
-#endif
-
-#if (defined(__PCLMUL__) && defined(__SSSE3__) && defined(__SSE4_1__)) || (defined(_MSC_VER) && _MSC_VER >= 1600)
+#if (defined(__PCLMUL__) && defined(__SSSE3__) && defined(__SSE4_1__)) || (defined(_MSC_VER) && _MSC_VER >= 1600 && defined(PLATFORM_X86) && !defined(__clang__))
 #include <inttypes.h>
 #include <immintrin.h>
 #include <wmmintrin.h>
 
-#define local static
 
-#ifdef _MSC_VER
-# define ALIGN(_a, v) __declspec(align(_a)) v
-/* Because we don't have dynamic dispatch for AVX, disable it for MSVC builds (only use AVX for -march=native style builds) */
-# undef __AVX__
-# undef __AVX512F__
-# undef __AVX512VL__
-# undef __GFNI__
-#else
-# define ALIGN(_a, v) v __attribute__((aligned(_a)))
+#if defined(__AVX512VL__) && defined(YENC_BUILD_NATIVE) && YENC_BUILD_NATIVE!=0
+# define ENABLE_AVX512 1
 #endif
 
 
 // interestingly, MSVC seems to generate better code if using VXORPS over VPXOR
 // original Intel code uses XORPS for many XOR operations, but PXOR is pretty much always better (more port freedom on Intel CPUs). The only advantage of XORPS is that it's 1 byte shorter, an advantage which disappears under AVX as both instructions have the same length
-#ifdef __AVX__
+#if defined(__AVX__) && defined(YENC_BUILD_NATIVE) && YENC_BUILD_NATIVE!=0
 # define fold_xor _mm_xor_si128
 #else
-local __m128i fold_xor(__m128i a, __m128i b) {
+static __m128i fold_xor(__m128i a, __m128i b) {
 	return _mm_castps_si128(_mm_xor_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b)));
 }
 #endif
 
-#ifdef __AVX512VL__
-local __m128i do_one_fold_merge(__m128i src, __m128i data) {
+#ifdef ENABLE_AVX512
+static __m128i do_one_fold_merge(__m128i src, __m128i data) {
     const __m128i xmm_fold4 = _mm_set_epi32(
             0x00000001, 0x54442bd4,
             0x00000001, 0xc6e41596);
@@ -68,7 +53,7 @@ local __m128i do_one_fold_merge(__m128i src, __m128i data) {
     );
 }
 #else
-local __m128i do_one_fold(__m128i src) {
+static __m128i do_one_fold(__m128i src) {
     const __m128i xmm_fold4 = _mm_set_epi32(
             0x00000001, 0x54442bd4,
             0x00000001, 0xc6e41596);
@@ -79,7 +64,7 @@ local __m128i do_one_fold(__m128i src) {
 }
 #endif
 
-ALIGN(32, local const unsigned  pshufb_shf_table[60]) = {
+ALIGN_TO(32, static const unsigned  pshufb_shf_table[60]) = {
     0x84838281, 0x88878685, 0x8c8b8a89, 0x008f8e8d, /* shl 15 (16 - 1)/shr1 */
     0x85848382, 0x89888786, 0x8d8c8b8a, 0x01008f8e, /* shl 14 (16 - 3)/shr2 */
     0x86858483, 0x8a898887, 0x8e8d8c8b, 0x0201008f, /* shl 13 (16 - 4)/shr3 */
@@ -97,7 +82,7 @@ ALIGN(32, local const unsigned  pshufb_shf_table[60]) = {
     0x0201008f, 0x06050403, 0x0a090807, 0x0e0d0c0b  /* shl  1 (16 -15)/shr15*/
 };
 
-local void partial_fold(const size_t len, __m128i *xmm_crc0, __m128i *xmm_crc1,
+static void partial_fold(const size_t len, __m128i *xmm_crc0, __m128i *xmm_crc1,
         __m128i *xmm_crc2, __m128i *xmm_crc3, __m128i *xmm_crc_part) {
 
     const __m128i xmm_mask3 = _mm_set1_epi32(0x80808080);
@@ -127,7 +112,7 @@ local void partial_fold(const size_t len, __m128i *xmm_crc0, __m128i *xmm_crc1,
     *xmm_crc_part = _mm_shuffle_epi8(*xmm_crc_part, xmm_shl);
     *xmm_crc3 = _mm_or_si128(*xmm_crc3, *xmm_crc_part);
 
-#ifdef __AVX512VL__
+#ifdef ENABLE_AVX512
     *xmm_crc3 = do_one_fold_merge(xmm_a0_0, *xmm_crc3);
 #else
     *xmm_crc3 = fold_xor(
@@ -137,74 +122,38 @@ local void partial_fold(const size_t len, __m128i *xmm_crc0, __m128i *xmm_crc1,
 #endif
 }
 
-ALIGN(16, local const unsigned crc_k[]) = {
+ALIGN_TO(16, static const unsigned crc_k[]) = {
     0xccaa009e, 0x00000000, /* rk1 */
     0x751997d0, 0x00000001, /* rk2 */
     0xccaa009e, 0x00000000, /* rk5 */
     0x63cd6124, 0x00000001, /* rk6 */
-    0xf7011640, 0x00000001, /* rk7 */
+    0xf7011641, 0x00000000, /* rk7 */
     0xdb710640, 0x00000001  /* rk8 */
 };
 
-ALIGN(16, local const unsigned crc_mask[4]) = {
-    0xFFFFFFFF, 0xFFFFFFFF, 0x00000000, 0x00000000
-};
-
-ALIGN(16, local const unsigned crc_mask2[4]) = {
+ALIGN_TO(16, static const unsigned crc_mask[4]) = {
     0x00000000, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF
 };
 
-local __m128i reverse_bits_epi8(__m128i src) {
-#ifdef __GFNI__
-    return _mm_gf2p8affine_epi64_epi8(src, _mm_set_epi32(
-      0x80402010, 0x08040201,
-      0x80402010, 0x08040201
-    ), 0);
-#else
-    __m128i xmm_t0 = _mm_and_si128(src, _mm_set1_epi8(0x0f));
-    __m128i xmm_t1 = _mm_and_si128(_mm_srli_epi16(src, 4), _mm_set1_epi8(0x0f));
-    xmm_t0 = _mm_shuffle_epi8(_mm_set_epi8(
-      0xf0, 0x70, 0xb0, 0x30, 0xd0, 0x50, 0x90, 0x10, 0xe0, 0x60, 0xa0, 0x20, 0xc0, 0x40, 0x80, 0
-    ), xmm_t0);
-    xmm_t1 = _mm_shuffle_epi8(_mm_set_epi8(
-      15, 7, 11, 3, 13, 5, 9, 1, 14, 6, 10, 2, 12, 4, 8, 0
-    ), xmm_t1);
-    return _mm_or_si128(xmm_t0, xmm_t1);
-#endif
-}
-
-#ifdef _MSC_VER
-// because MSVC doesn't use BSWAP unless you specifically tell it to...
-# include <stdlib.h>
-# define BSWAP32 _byteswap_ulong
-#else
-# define BSWAP32(n) ((((n)&0xff)<<24) | (((n)&0xff00)<<8) | (((n)&0xff0000)>>8) | (((n)&0xff000000)>>24))
-#endif
 
-local uint32_t crc_fold(const unsigned char *src, long len, uint32_t initial) {
+static uint32_t crc_fold(const unsigned char *src, long len, uint32_t initial) {
     unsigned long algn_diff;
     __m128i xmm_t0, xmm_t1, xmm_t2, xmm_t3;
 
     // TODO: consider calculating this via a LUT instead (probably faster)
     // info from https://www.reddit.com/r/ReverseEngineering/comments/2zwhl3/mystery_constant_0x9db42487_in_intels_crc32ieee/
     // firstly, calculate: xmm_crc0 = (intial * 0x487b9c8a) mod 0x104c11db7, where 0x487b9c8a = inverse(1<<512) mod 0x104c11db7
+    xmm_t0 = _mm_cvtsi32_si128(~initial);
     
-    // reverse input bits + load into XMM register
-    uint32_t init_t = BSWAP32(initial);
-    xmm_t0 = reverse_bits_epi8(_mm_cvtsi32_si128(~init_t));
-    
-    xmm_t0 = _mm_clmulepi64_si128(xmm_t0, _mm_cvtsi32_si128(0x487b9c8a), 0);
-    xmm_t1 = _mm_and_si128(xmm_t0, _mm_set_epi32(-1,-1,-1,0)); // shifted up by 32bits to avoid shifts by using clmul's capability to select top 64bits instead
+    xmm_t0 = _mm_clmulepi64_si128(xmm_t0, _mm_set_epi32(0, 0, 0xa273bc24, 0), 0);  // reverse(0x487b9c8a)<<1 == 0xa273bc24
     xmm_t2 = _mm_set_epi32( // polynomial reduction factors
-      0, 0x04c11db7, // G*
-      1, 0x04d101df  // Q+
+      1, 0xdb710640, // G* = 0x04c11db7
+      0, 0xf7011641  // Q+ = 0x04d101df  (+1 to save an additional xor operation)
     );
-    xmm_t1 = _mm_clmulepi64_si128(xmm_t1, xmm_t2, 0);
-    xmm_t1 = _mm_clmulepi64_si128(xmm_t1, xmm_t2, 0x11);
+    xmm_t1 = _mm_clmulepi64_si128(xmm_t0, xmm_t2, 0);
+    xmm_t1 = _mm_clmulepi64_si128(xmm_t1, xmm_t2, 0x10);
     
-    __m128i xmm_crc0 = _mm_xor_si128(xmm_t0, xmm_t1);
-    // reverse bits
-    xmm_crc0 = _mm_shuffle_epi8(reverse_bits_epi8(xmm_crc0), _mm_set_epi32(-1,-1,-1,0x00010203));
+    __m128i xmm_crc0 = _mm_srli_si128(_mm_xor_si128(xmm_t0, xmm_t1), 8);
     
     __m128i xmm_crc1 = _mm_setzero_si128();
     __m128i xmm_crc2 = _mm_setzero_si128();
@@ -214,7 +163,8 @@ local uint32_t crc_fold(const unsigned char *src, long len, uint32_t initial) {
     if (len < 16) {
         if (len == 0)
             return initial;
-        xmm_crc_part = _mm_loadu_si128((__m128i *)src);
+        xmm_crc_part = _mm_setzero_si128();
+        memcpy(&xmm_crc_part, src, len);
         goto partial;
     }
 
@@ -229,13 +179,13 @@ local uint32_t crc_fold(const unsigned char *src, long len, uint32_t initial) {
             &xmm_crc_part);
     }
 
-    while ((len -= 64) >= 0) {
+    while (len >= 64) {
         xmm_t0 = _mm_load_si128((__m128i *)src);
         xmm_t1 = _mm_load_si128((__m128i *)src + 1);
         xmm_t2 = _mm_load_si128((__m128i *)src + 2);
         xmm_t3 = _mm_load_si128((__m128i *)src + 3);
 
-#ifdef __AVX512VL__
+#ifdef ENABLE_AVX512
         xmm_crc0 = do_one_fold_merge(xmm_crc0, xmm_t0);
         xmm_crc1 = do_one_fold_merge(xmm_crc1, xmm_t1);
         xmm_crc2 = do_one_fold_merge(xmm_crc2, xmm_t2);
@@ -253,20 +203,18 @@ local uint32_t crc_fold(const unsigned char *src, long len, uint32_t initial) {
 #endif
 
         src += 64;
+        len -= 64;
     }
 
-    /*
-     * len = num bytes left - 64
-     */
-    if (len + 16 >= 0) {
-        len += 16;
+    if (len >= 48) {
+        len -= 48;
 
         xmm_t0 = _mm_load_si128((__m128i *)src);
         xmm_t1 = _mm_load_si128((__m128i *)src + 1);
         xmm_t2 = _mm_load_si128((__m128i *)src + 2);
 
         xmm_t3 = xmm_crc3;
-#ifdef __AVX512VL__
+#ifdef ENABLE_AVX512
         xmm_crc3 = do_one_fold_merge(xmm_crc2, xmm_t2);
         xmm_crc2 = do_one_fold_merge(xmm_crc1, xmm_t1);
         xmm_crc1 = do_one_fold_merge(xmm_crc0, xmm_t0);
@@ -284,15 +232,15 @@ local uint32_t crc_fold(const unsigned char *src, long len, uint32_t initial) {
             goto done;
 
         xmm_crc_part = _mm_load_si128((__m128i *)src + 3);
-    } else if (len + 32 >= 0) {
-        len += 32;
+    } else if (len >= 32) {
+        len -= 32;
 
         xmm_t0 = _mm_load_si128((__m128i *)src);
         xmm_t1 = _mm_load_si128((__m128i *)src + 1);
 
         xmm_t2 = xmm_crc2;
         xmm_t3 = xmm_crc3;
-#ifdef __AVX512VL__
+#ifdef ENABLE_AVX512
         xmm_crc3 = do_one_fold_merge(xmm_crc1, xmm_t1);
         xmm_crc2 = do_one_fold_merge(xmm_crc0, xmm_t0);
 #else
@@ -308,13 +256,13 @@ local uint32_t crc_fold(const unsigned char *src, long len, uint32_t initial) {
             goto done;
 
         xmm_crc_part = _mm_load_si128((__m128i *)src + 2);
-    } else if (len + 48 >= 0) {
-        len += 48;
+    } else if (len >= 16) {
+        len -= 16;
 
         xmm_t0 = _mm_load_si128((__m128i *)src);
 
         xmm_t3 = xmm_crc3;
-#ifdef __AVX512VL__
+#ifdef ENABLE_AVX512
         xmm_crc3 = do_one_fold_merge(xmm_crc0, xmm_t0);
 #else
         xmm_crc3 = _mm_xor_si128(do_one_fold(xmm_crc0), xmm_t0);
@@ -328,7 +276,6 @@ local uint32_t crc_fold(const unsigned char *src, long len, uint32_t initial) {
 
         xmm_crc_part = _mm_load_si128((__m128i *)src + 1);
     } else {
-        len += 64;
         if (len == 0)
             goto done;
         xmm_crc_part = _mm_load_si128((__m128i *)src);
@@ -339,8 +286,7 @@ partial:
         &xmm_crc_part);
 done:
 {
-    const __m128i xmm_mask  = _mm_load_si128((__m128i *)crc_mask);
-    const __m128i xmm_mask2 = _mm_load_si128((__m128i *)crc_mask2);
+    const __m128i xmm_mask = _mm_load_si128((__m128i *)crc_mask);
     __m128i x_tmp0, x_tmp1, x_tmp2, crc_fold;
 
     /*
@@ -350,7 +296,7 @@ done:
 
     x_tmp0 = _mm_clmulepi64_si128(xmm_crc0, crc_fold, 0x10);
     xmm_crc0 = _mm_clmulepi64_si128(xmm_crc0, crc_fold, 0x01);
-#ifdef __AVX512VL__
+#ifdef ENABLE_AVX512
     xmm_crc1 = _mm_ternarylogic_epi32(xmm_crc1, x_tmp0, xmm_crc0, 0x96);
 #else
     xmm_crc1 = _mm_xor_si128(xmm_crc1, x_tmp0);
@@ -359,7 +305,7 @@ done:
 
     x_tmp1 = _mm_clmulepi64_si128(xmm_crc1, crc_fold, 0x10);
     xmm_crc1 = _mm_clmulepi64_si128(xmm_crc1, crc_fold, 0x01);
-#ifdef __AVX512VL__
+#ifdef ENABLE_AVX512
     xmm_crc2 = _mm_ternarylogic_epi32(xmm_crc2, x_tmp1, xmm_crc1, 0x96);
 #else
     xmm_crc2 = _mm_xor_si128(xmm_crc2, x_tmp1);
@@ -368,7 +314,7 @@ done:
 
     x_tmp2 = _mm_clmulepi64_si128(xmm_crc2, crc_fold, 0x10);
     xmm_crc2 = _mm_clmulepi64_si128(xmm_crc2, crc_fold, 0x01);
-#ifdef __AVX512VL__
+#ifdef ENABLE_AVX512
     xmm_crc3 = _mm_ternarylogic_epi32(xmm_crc3, x_tmp2, xmm_crc2, 0x96);
 #else
     xmm_crc3 = _mm_xor_si128(xmm_crc3, x_tmp2);
@@ -388,58 +334,43 @@ done:
     xmm_crc0 = xmm_crc3;
     xmm_crc3 = _mm_slli_si128(xmm_crc3, 4);
     xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0x10);
-#ifdef __AVX512VL__
+#ifdef ENABLE_AVX512
     //xmm_crc3 = _mm_maskz_xor_epi32(14, xmm_crc3, xmm_crc0);
-    xmm_crc3 = _mm_ternarylogic_epi32(xmm_crc3, xmm_crc0, xmm_mask2, 0x28);
+    xmm_crc3 = _mm_ternarylogic_epi32(xmm_crc3, xmm_crc0, xmm_mask, 0x28);
 #else
+    xmm_crc0 = _mm_and_si128(xmm_crc0, xmm_mask);
     xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc0);
-    xmm_crc3 = _mm_and_si128(xmm_crc3, xmm_mask2);
 #endif
 
     /*
      * k7
      */
     xmm_crc1 = xmm_crc3;
-    xmm_crc2 = xmm_crc3;
     crc_fold = _mm_load_si128((__m128i *)crc_k + 2);
 
     xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0);
-#ifdef __AVX512VL__
-    //xmm_crc3 = _mm_maskz_xor_epi32(3, xmm_crc3, xmm_crc2);
-    xmm_crc3 = _mm_ternarylogic_epi32(xmm_crc3, xmm_crc2, xmm_mask, 0x28);
-#else
-    xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc2);
-    xmm_crc3 = _mm_and_si128(xmm_crc3, xmm_mask);
-#endif
-
-    xmm_crc2 = xmm_crc3;
     xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0x10);
-#ifdef __AVX512VL__
-    xmm_crc3 = _mm_ternarylogic_epi32(xmm_crc3, xmm_crc2, xmm_crc1, 0x69); // NOT(double-XOR)
-    return _mm_extract_epi32(xmm_crc3, 2);
+#ifdef ENABLE_AVX512
+    xmm_crc3 = _mm_ternarylogic_epi32(xmm_crc3, xmm_crc1, xmm_crc1, 0xC3); // NOT(xmm_crc3 ^ xmm_crc1)
 #else
-    xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc2);
+    xmm_crc1 = _mm_xor_si128(xmm_crc1, xmm_mask);
     xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc1);
-    return ~_mm_extract_epi32(xmm_crc3, 2);
 #endif
+    return _mm_extract_epi32(xmm_crc3, 2);
 }
 
 }
 
-static void do_crc32_clmul(const void* data, size_t length, unsigned char out[4]) {
-	uint32_t tmp = crc_fold((const unsigned char*)data, (long)length, 0);
-	UNPACK_4(out, tmp);
-}
-static void do_crc32_incremental_clmul(const void* data, size_t length, unsigned char init[4]) {
-	uint32_t tmp = crc_fold((const unsigned char*)data, (long)length, PACK_4(init));
-	UNPACK_4(init, tmp);
+static uint32_t do_crc32_incremental_clmul(const void* data, size_t length, uint32_t init) {
+	return crc_fold((const unsigned char*)data, (long)length, init);
 }
 
-void crc_clmul_set_funcs(crc_func* _do_crc32, crc_func* _do_crc32_incremental) {
-	*_do_crc32 = &do_crc32_clmul;
+void crc_clmul_set_funcs(crc_func* _do_crc32_incremental) {
 	*_do_crc32_incremental = &do_crc32_incremental_clmul;
 }
 #else
-void crc_clmul_set_funcs(crc_func* _do_crc32, crc_func* _do_crc32_incremental) {}
+void crc_clmul_set_funcs(crc_func* _do_crc32_incremental) {
+    (void)_do_crc32_incremental;
+}
 #endif