yencode 1.1.2 → 1.1.4

package/src/crc_arm.cc

@@ -5,6 +5,24 @@
 HEDLEY_WARNING("CRC32 acceleration is not been enabled under ARM clang-cl by default; add `-march=armv8-a+crc` to additional compiler arguments to enable");
 #endif
 
+// disable CRC on GCC versions with broken arm_acle.h
+#if defined(__ARM_FEATURE_CRC32) && defined(HEDLEY_GCC_VERSION)
+# if !defined(__aarch64__) && HEDLEY_GCC_VERSION_CHECK(7,0,0) && !HEDLEY_GCC_VERSION_CHECK(8,1,1)
+#  undef __ARM_FEATURE_CRC32
+HEDLEY_WARNING("CRC32 acceleration has been disabled due to broken arm_acle.h shipped in GCC 7.0 - 8.1 [https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81497]. If you need this feature, please use a different compiler or version of GCC");
+# endif
+# if defined(__aarch64__) && HEDLEY_GCC_VERSION_CHECK(9,4,0) && !HEDLEY_GCC_VERSION_CHECK(9,5,0)
+#  undef __ARM_FEATURE_CRC32
+HEDLEY_WARNING("CRC32 acceleration has been disabled due to broken arm_acle.h shipped in GCC 9.4 [https://gcc.gnu.org/bugzilla/show_bug.cgi?id=100985]. If you need this feature, please use a different compiler or version of GCC");
+# endif
+#endif
+#if defined(__ARM_FEATURE_CRC32) && defined(__has_include)
+# if !__has_include(<arm_acle.h>)
+#  undef __ARM_FEATURE_CRC32
+HEDLEY_WARNING("CRC32 acceleration has been disabled due to missing arm_acle.h");
+# endif
+#endif
+
 #if defined(__ARM_FEATURE_CRC32) || (defined(_M_ARM64) && !defined(__clang__)) // MSVC doesn't support CRC for ARM32
 
 /* ARMv8 accelerated CRC */
@@ -14,14 +32,30 @@ HEDLEY_WARNING("CRC32 acceleration is not been enabled under ARM clang-cl by def
 #include <arm_acle.h>
 #endif
 
+
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+# ifdef __GNUC__
+#  define _LE16 __builtin_bswap16
+#  define _LE32 __builtin_bswap32
+#  define _LE64 __builtin_bswap64
+# else
+// currently not supported
+#  error No endian swap intrinsic defined
+# endif
+#else
+# define _LE16(x) (x)
+# define _LE32(x) (x)
+# define _LE64(x) (x)
+#endif
+
 #ifdef __aarch64__
 # define WORD_T uint64_t
 # define WORDSIZE_LOG 3  // sizeof(WORD_T) == 1<<WORDSIZE_LOG
-# define CRC_WORD __crc32d
+# define CRC_WORD(crc, data) __crc32d(crc, _LE64(data))
 #else
 # define WORD_T uint32_t
 # define WORDSIZE_LOG 2  // sizeof(WORD_T) == 1<<WORDSIZE_LOG
-# define CRC_WORD __crc32w
+# define CRC_WORD(crc, data) __crc32w(crc, _LE32(data))
 #endif
 
 
@@ -45,7 +79,7 @@ static HEDLEY_ALWAYS_INLINE uint32_t crc_multiply(uint32_t a, uint32_t b) {
 	return res;
 }
 
-static const uint32_t crc_power[] = { // pre-computed 2^n, with first 3 entries removed (saves a shift)
+static const uint32_t crc_power[] = { // pre-computed 2^(2^n), with first 3 entries removed (saves a shift)
 	0x00800000, 0x00008000, 0xedb88320, 0xb1e6b092, 0xa06a2517, 0xed627dae, 0x88d14467, 0xd7bbfe6a,
 	0xec447f11, 0x8e7ea170, 0x6427800e, 0x4d47bae0, 0x09fe548f, 0x83852d0f, 0x30362f1a, 0x7b5a9cc3,
 	0x31fec169, 0x9fec022a, 0x6c8dedc4, 0x15d6874d, 0x5fde7a4e, 0xbad90e37, 0x2e4e5eef, 0x4eaba214,
@@ -64,6 +98,7 @@ static const uint32_t crc_power[] = { // pre-computed 2^n, with first 3 entries
 #endif
 
 
+
 // inspired/stolen off https://github.com/jocover/crc32_armv8/blob/master/crc32_armv8.c
 static uint32_t arm_crc_calc(uint32_t crc, const unsigned char *src, long len) {
 	
@@ -75,13 +110,13 @@ static uint32_t arm_crc_calc(uint32_t crc, const unsigned char *src, long len) {
 			len--;
 		}
 		if ((uintptr_t)src & sizeof(uint16_t)) {
-			crc = __crc32h(crc, *((uint16_t *)src));
+			crc = __crc32h(crc, _LE16(*((uint16_t *)src)));
 			src += sizeof(uint16_t);
 			len -= sizeof(uint16_t);
 		}
 #ifdef __aarch64__
 		if ((uintptr_t)src & sizeof(uint32_t)) {
-			crc = __crc32w(crc, *((uint32_t *)src));
+			crc = __crc32w(crc, _LE32(*((uint32_t *)src)));
 			src += sizeof(uint32_t);
 			len -= sizeof(uint32_t);
 		}
@@ -147,12 +182,12 @@ static uint32_t arm_crc_calc(uint32_t crc, const unsigned char *src, long len) {
 	
 #ifdef __aarch64__
 	if (len & sizeof(uint32_t)) {
-		crc = __crc32w(crc, *((uint32_t *)src));
+		crc = __crc32w(crc, _LE32(*((uint32_t *)src)));
 		src += sizeof(uint32_t);
 	}
 #endif
 	if (len & sizeof(uint16_t)) {
-		crc = __crc32h(crc, *((uint16_t *)src));
+		crc = __crc32h(crc, _LE16(*((uint16_t *)src)));
 		src += sizeof(uint16_t);
 	}
 	if (len & sizeof(uint8_t))

package/src/crc_folding.cc

@@ -19,7 +19,7 @@
 
 #include "crc_common.h"
  
-#if (defined(__PCLMUL__) && defined(__SSSE3__) && defined(__SSE4_1__)) || (defined(_MSC_VER) && _MSC_VER >= 1600 && defined(PLATFORM_X86))
+#if (defined(__PCLMUL__) && defined(__SSSE3__) && defined(__SSE4_1__)) || (defined(_MSC_VER) && _MSC_VER >= 1600 && defined(PLATFORM_X86) && !defined(__clang__))
 #include <inttypes.h>
 #include <immintrin.h>
 #include <wmmintrin.h>
@@ -135,33 +135,6 @@ ALIGN_TO(16, static const unsigned crc_mask[4]) = {
     0x00000000, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF
 };
 
-static __m128i reverse_bits_epi8(__m128i src) {
-#if defined(__GFNI__) && defined(YENC_BUILD_NATIVE) && YENC_BUILD_NATIVE!=0
-    return _mm_gf2p8affine_epi64_epi8(src, _mm_set_epi32(
-      0x80402010, 0x08040201,
-      0x80402010, 0x08040201
-    ), 0);
-#else
-    __m128i xmm_t0 = _mm_and_si128(src, _mm_set1_epi8(0x0f));
-    __m128i xmm_t1 = _mm_and_si128(_mm_srli_epi16(src, 4), _mm_set1_epi8(0x0f));
-    xmm_t0 = _mm_shuffle_epi8(_mm_set_epi8(
-      -16, 112, -80, 48, -48, 80, -112, 16, -32, 96, -96, 32, -64, 64, -128, 0
-      //0xf0, 0x70, 0xb0, 0x30, 0xd0, 0x50, 0x90, 0x10, 0xe0, 0x60, 0xa0, 0x20, 0xc0, 0x40, 0x80, 0
-    ), xmm_t0);
-    xmm_t1 = _mm_shuffle_epi8(_mm_set_epi8(
-      15, 7, 11, 3, 13, 5, 9, 1, 14, 6, 10, 2, 12, 4, 8, 0
-    ), xmm_t1);
-    return _mm_or_si128(xmm_t0, xmm_t1);
-#endif
-}
-
-#ifdef _MSC_VER
-// because MSVC doesn't use BSWAP unless you specifically tell it to...
-# include <stdlib.h>
-# define BSWAP32 _byteswap_ulong
-#else
-# define BSWAP32(n) ((((n)&0xff)<<24) | (((n)&0xff00)<<8) | (((n)&0xff0000)>>8) | (((n)&0xff000000)>>24))
-#endif
 
 static uint32_t crc_fold(const unsigned char *src, long len, uint32_t initial) {
     unsigned long algn_diff;
@@ -170,23 +143,17 @@ static uint32_t crc_fold(const unsigned char *src, long len, uint32_t initial) {
     // TODO: consider calculating this via a LUT instead (probably faster)
     // info from https://www.reddit.com/r/ReverseEngineering/comments/2zwhl3/mystery_constant_0x9db42487_in_intels_crc32ieee/
     // firstly, calculate: xmm_crc0 = (intial * 0x487b9c8a) mod 0x104c11db7, where 0x487b9c8a = inverse(1<<512) mod 0x104c11db7
+    xmm_t0 = _mm_cvtsi32_si128(~initial);
     
-    // reverse input bits + load into XMM register
-    uint32_t init_t = BSWAP32(initial);
-    xmm_t0 = reverse_bits_epi8(_mm_cvtsi32_si128(~init_t));
-    
-    xmm_t0 = _mm_clmulepi64_si128(xmm_t0, _mm_cvtsi32_si128(0x487b9c8a), 0);
-    xmm_t1 = _mm_and_si128(xmm_t0, _mm_set_epi32(-1,-1,-1,0)); // shifted up by 32bits to avoid shifts by using clmul's capability to select top 64bits instead
+    xmm_t0 = _mm_clmulepi64_si128(xmm_t0, _mm_set_epi32(0, 0, 0xa273bc24, 0), 0);  // reverse(0x487b9c8a)<<1 == 0xa273bc24
     xmm_t2 = _mm_set_epi32( // polynomial reduction factors
-      0, 0x04c11db7, // G*
-      1, 0x04d101df  // Q+
+      1, 0xdb710640, // G* = 0x04c11db7
+      0, 0xf7011641  // Q+ = 0x04d101df  (+1 to save an additional xor operation)
     );
-    xmm_t1 = _mm_clmulepi64_si128(xmm_t1, xmm_t2, 0);
-    xmm_t1 = _mm_clmulepi64_si128(xmm_t1, xmm_t2, 0x11);
+    xmm_t1 = _mm_clmulepi64_si128(xmm_t0, xmm_t2, 0);
+    xmm_t1 = _mm_clmulepi64_si128(xmm_t1, xmm_t2, 0x10);
     
-    __m128i xmm_crc0 = _mm_xor_si128(xmm_t0, xmm_t1);
-    // reverse bits
-    xmm_crc0 = _mm_shuffle_epi8(reverse_bits_epi8(xmm_crc0), _mm_set_epi32(-1,-1,-1,0x00010203));
+    __m128i xmm_crc0 = _mm_srli_si128(_mm_xor_si128(xmm_t0, xmm_t1), 8);
     
     __m128i xmm_crc1 = _mm_setzero_si128();
     __m128i xmm_crc2 = _mm_setzero_si128();
@@ -196,7 +163,8 @@ static uint32_t crc_fold(const unsigned char *src, long len, uint32_t initial) {
     if (len < 16) {
         if (len == 0)
             return initial;
-        xmm_crc_part = _mm_loadu_si128((__m128i *)src);
+        xmm_crc_part = _mm_setzero_si128();
+        memcpy(&xmm_crc_part, src, len);
         goto partial;
     }
 
@@ -211,7 +179,7 @@ static uint32_t crc_fold(const unsigned char *src, long len, uint32_t initial) {
             &xmm_crc_part);
     }
 
-    while ((len -= 64) >= 0) {
+    while (len >= 64) {
         xmm_t0 = _mm_load_si128((__m128i *)src);
         xmm_t1 = _mm_load_si128((__m128i *)src + 1);
         xmm_t2 = _mm_load_si128((__m128i *)src + 2);
@@ -235,13 +203,11 @@ static uint32_t crc_fold(const unsigned char *src, long len, uint32_t initial) {
 #endif
 
         src += 64;
+        len -= 64;
     }
 
-    /*
-     * len = num bytes left - 64
-     */
-    if (len + 16 >= 0) {
-        len += 16;
+    if (len >= 48) {
+        len -= 48;
 
         xmm_t0 = _mm_load_si128((__m128i *)src);
         xmm_t1 = _mm_load_si128((__m128i *)src + 1);
@@ -266,8 +232,8 @@ static uint32_t crc_fold(const unsigned char *src, long len, uint32_t initial) {
             goto done;
 
         xmm_crc_part = _mm_load_si128((__m128i *)src + 3);
-    } else if (len + 32 >= 0) {
-        len += 32;
+    } else if (len >= 32) {
+        len -= 32;
 
         xmm_t0 = _mm_load_si128((__m128i *)src);
         xmm_t1 = _mm_load_si128((__m128i *)src + 1);
@@ -290,8 +256,8 @@ static uint32_t crc_fold(const unsigned char *src, long len, uint32_t initial) {
             goto done;
 
         xmm_crc_part = _mm_load_si128((__m128i *)src + 2);
-    } else if (len + 48 >= 0) {
-        len += 48;
+    } else if (len >= 16) {
+        len -= 16;
 
         xmm_t0 = _mm_load_si128((__m128i *)src);
 
@@ -310,7 +276,6 @@ static uint32_t crc_fold(const unsigned char *src, long len, uint32_t initial) {
 
         xmm_crc_part = _mm_load_si128((__m128i *)src + 1);
     } else {
-        len += 64;
         if (len == 0)
             goto done;
         xmm_crc_part = _mm_load_si128((__m128i *)src);

package/src/crc_folding_256.cc

@@ -0,0 +1,229 @@
+// 256-bit version of crc_folding
+
+#include "crc_common.h"
+ 
+#if !defined(YENC_DISABLE_AVX256) && ((defined(__VPCLMULQDQ__) && defined(__AVX2__) && defined(__PCLMUL__)) || (defined(_MSC_VER) && _MSC_VER >= 1920 && defined(PLATFORM_X86) && !defined(__clang__)))
+#include <inttypes.h>
+#include <immintrin.h>
+
+
+#if defined(__AVX512VL__) && defined(YENC_BUILD_NATIVE) && YENC_BUILD_NATIVE!=0
+# define ENABLE_AVX512 1
+#endif
+
+static __m256i do_one_fold(__m256i src, __m256i data) {
+	const __m256i fold4 = _mm256_set_epi32(
+		0x00000001, 0x54442bd4,
+		0x00000001, 0xc6e41596,
+		0x00000001, 0x54442bd4,
+		0x00000001, 0xc6e41596
+	);
+#ifdef ENABLE_AVX512
+	return _mm256_ternarylogic_epi32(
+	  _mm256_clmulepi64_epi128(src, fold4, 0x01),
+	  _mm256_clmulepi64_epi128(src, fold4, 0x10),
+	  data,
+	  0x96
+	);
+#else
+	return _mm256_xor_si256(_mm256_xor_si256(
+	  data, _mm256_clmulepi64_epi128(src, fold4, 0x01)
+	), _mm256_clmulepi64_epi128(src, fold4, 0x10));
+#endif
+}
+
+ALIGN_TO(32, static const uint8_t  pshufb_rot_table[]) = {
+	0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,
+	16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31
+};
+// _mm256_castsi128_si256, but upper is defined to be 0
+#if (defined(__clang__) && __clang_major__ >= 5 && (!defined(__APPLE__) || __clang_major__ >= 7)) || (defined(__GNUC__) && __GNUC__ >= 10) || (defined(_MSC_VER) && _MSC_VER >= 1910)
+// intrinsic unsupported in GCC 9 and MSVC < 2017
+# define zext128_256 _mm256_zextsi128_si256
+#else
+// technically a cast is incorrect, due to upper 128 bits being undefined, but should usually work fine
+// alternative may be `_mm256_set_m128i(_mm_setzero_si128(), v)` but unsupported on GCC < 7, and most compilers generate a VINSERTF128 instruction for it
+# ifdef __OPTIMIZE__
+#  define zext128_256 _mm256_castsi128_si256
+# else
+#  define zext128_256(x) _mm256_inserti128_si256(_mm256_setzero_si256(), x, 0)
+# endif
+#endif
+
+#ifdef ENABLE_AVX512
+# define MM256_BLENDV(a, b, m) _mm256_ternarylogic_epi32(a, b, m, 0xd8)
+# define MM_2XOR(a, b, c) _mm_ternarylogic_epi32(a, b, c, 0x96)
+#else
+# define MM256_BLENDV _mm256_blendv_epi8
+# define MM_2XOR(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c)
+#endif
+
+static void partial_fold(const size_t len, __m256i *crc0, __m256i *crc1, __m256i crc_part) {
+	__m256i shuf = _mm256_broadcastsi128_si256(_mm_loadu_si128((__m128i*)(pshufb_rot_table + (len&15))));
+	__m256i mask = _mm256_cmpgt_epi8(shuf, _mm256_set1_epi8(15));
+	
+	*crc0 = _mm256_shuffle_epi8(*crc0, shuf);
+	*crc1 = _mm256_shuffle_epi8(*crc1, shuf);
+	crc_part = _mm256_shuffle_epi8(crc_part, shuf);
+	
+	__m256i crc_out = _mm256_permute2x128_si256(*crc0, *crc0, 0x08);  // move bottom->top
+	__m256i crc01, crc1p;
+	if(len >= 16) {
+		crc_out = MM256_BLENDV(crc_out, *crc0, mask);
+		crc01 = *crc1;
+		crc1p = crc_part;
+		*crc0 = _mm256_permute2x128_si256(*crc0, *crc1, 0x21);
+		*crc1 = _mm256_permute2x128_si256(*crc1, crc_part, 0x21);
+		crc_part = zext128_256(_mm256_extracti128_si256(crc_part, 1));
+	} else {
+		crc_out = _mm256_and_si256(crc_out, mask);
+		crc01 = _mm256_permute2x128_si256(*crc0, *crc1, 0x21);
+		crc1p = _mm256_permute2x128_si256(*crc1, crc_part, 0x21);
+	}
+	
+	*crc0 = MM256_BLENDV(*crc0, crc01, mask);
+	*crc1 = MM256_BLENDV(*crc1, crc1p, mask);
+	
+	*crc1 = do_one_fold(crc_out, *crc1);
+}
+
+
+ALIGN_TO(16, static const unsigned crc_k[]) = {
+	0xccaa009e, 0x00000000, /* rk1 */
+	0x751997d0, 0x00000001, /* rk2 */
+	0xccaa009e, 0x00000000, /* rk5 */
+	0x63cd6124, 0x00000001, /* rk6 */
+	0xf7011641, 0x00000000, /* rk7 */
+	0xdb710640, 0x00000001  /* rk8 */
+};
+
+
+static uint32_t crc_fold(const unsigned char *src, long len, uint32_t initial) {
+	// info from https://www.reddit.com/r/ReverseEngineering/comments/2zwhl3/mystery_constant_0x9db42487_in_intels_crc32ieee/
+	// firstly, calculate: xmm_crc0 = (intial * 0x487b9c8a) mod 0x104c11db7, where 0x487b9c8a = inverse(1<<512) mod 0x104c11db7
+	__m128i xmm_t0 = _mm_cvtsi32_si128(~initial);
+	
+	xmm_t0 = _mm_clmulepi64_si128(xmm_t0, _mm_set_epi32(0, 0, 0xa273bc24, 0), 0);  // reverse(0x487b9c8a)<<1 == 0xa273bc24
+	__m128i reduction = _mm_set_epi32( // polynomial reduction factors
+	  1, 0xdb710640, // G* = 0x04c11db7
+	  0, 0xf7011641  // Q+ = 0x04d101df  (+1 to save an additional xor operation)
+	);
+	__m128i xmm_t1 = _mm_clmulepi64_si128(xmm_t0, reduction, 0);
+	xmm_t1 = _mm_clmulepi64_si128(xmm_t1, reduction, 0x10);
+	
+	xmm_t0 = _mm_srli_si128(_mm_xor_si128(xmm_t0, xmm_t1), 8);
+	__m256i crc0 = zext128_256(xmm_t0);
+	__m256i crc1 = _mm256_setzero_si256();
+	
+	if (len < 32) {
+		if (len == 0)
+			return initial;
+		__m256i crc_part = _mm256_setzero_si256();
+		memcpy(&crc_part, src, len);
+		partial_fold(len, &crc0, &crc1, crc_part);
+	} else {
+		uintptr_t algn_diff = (0 - (uintptr_t)src) & 0x1F;
+		if (algn_diff) {
+			partial_fold(algn_diff, &crc0, &crc1, _mm256_loadu_si256((__m256i *)src));
+			src += algn_diff;
+			len -= algn_diff;
+		}
+		
+		while (len >= 64) {
+			crc0 = do_one_fold(crc0, _mm256_load_si256((__m256i*)src));
+			crc1 = do_one_fold(crc1, _mm256_load_si256((__m256i*)src + 1));
+			src += 64;
+			len -= 64;
+		}
+		
+		if (len >= 32) {
+			__m256i old = crc1;
+			crc1 = do_one_fold(crc0, _mm256_load_si256((__m256i*)src));
+			crc0 = old;
+			
+			len -= 32;
+			src += 32;
+		}
+		
+		if(len != 0) {
+			partial_fold(len, &crc0, &crc1, _mm256_load_si256((__m256i *)src));
+		}
+	}
+	
+	const __m128i xmm_mask = _mm_set_epi32(-1,-1,-1,0);
+	__m128i x_tmp0, x_tmp1, x_tmp2, crc_fold;
+	
+	__m128i xmm_crc0 = _mm256_castsi256_si128(crc0);
+	__m128i xmm_crc1 = _mm256_extracti128_si256(crc0, 1);
+	__m128i xmm_crc2 = _mm256_castsi256_si128(crc1);
+	__m128i xmm_crc3 = _mm256_extracti128_si256(crc1, 1);
+
+	/*
+	 * k1
+	 */
+	crc_fold = _mm_load_si128((__m128i *)crc_k);
+
+	x_tmp0 = _mm_clmulepi64_si128(xmm_crc0, crc_fold, 0x10);
+	xmm_crc0 = _mm_clmulepi64_si128(xmm_crc0, crc_fold, 0x01);
+	xmm_crc1 = MM_2XOR(xmm_crc1, x_tmp0, xmm_crc0);
+
+	x_tmp1 = _mm_clmulepi64_si128(xmm_crc1, crc_fold, 0x10);
+	xmm_crc1 = _mm_clmulepi64_si128(xmm_crc1, crc_fold, 0x01);
+	xmm_crc2 = MM_2XOR(xmm_crc2, x_tmp1, xmm_crc1);
+
+	x_tmp2 = _mm_clmulepi64_si128(xmm_crc2, crc_fold, 0x10);
+	xmm_crc2 = _mm_clmulepi64_si128(xmm_crc2, crc_fold, 0x01);
+	xmm_crc3 = MM_2XOR(xmm_crc3, x_tmp2, xmm_crc2);
+
+	/*
+	 * k5
+	 */
+	crc_fold = _mm_load_si128((__m128i *)crc_k + 1);
+
+	xmm_crc0 = xmm_crc3;
+	xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0);
+	xmm_crc0 = _mm_srli_si128(xmm_crc0, 8);
+	xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc0);
+
+	xmm_crc0 = xmm_crc3;
+	xmm_crc3 = _mm_slli_si128(xmm_crc3, 4);
+	xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0x10);
+#ifdef ENABLE_AVX512
+	//xmm_crc3 = _mm_maskz_xor_epi32(14, xmm_crc3, xmm_crc0);
+	xmm_crc3 = _mm_ternarylogic_epi32(xmm_crc3, xmm_crc0, xmm_mask, 0x28);
+#else
+	xmm_crc0 = _mm_and_si128(xmm_crc0, xmm_mask);
+	xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc0);
+#endif
+
+	/*
+	 * k7
+	 */
+	xmm_crc1 = xmm_crc3;
+	crc_fold = _mm_load_si128((__m128i *)crc_k + 2);
+
+	xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0);
+	xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0x10);
+#ifdef ENABLE_AVX512
+	xmm_crc3 = _mm_ternarylogic_epi32(xmm_crc3, xmm_crc1, xmm_crc1, 0xC3); // NOT(xmm_crc3 ^ xmm_crc1)
+#else
+	xmm_crc1 = _mm_xor_si128(xmm_crc1, xmm_mask);
+	xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc1);
+#endif
+	return _mm_extract_epi32(xmm_crc3, 2);
+}
+
+static uint32_t do_crc32_incremental_clmul(const void* data, size_t length, uint32_t init) {
+	return crc_fold((const unsigned char*)data, (long)length, init);
+}
+
+void crc_clmul256_set_funcs(crc_func* _do_crc32_incremental) {
+	*_do_crc32_incremental = &do_crc32_incremental_clmul;
+}
+#else
+void crc_clmul_set_funcs(crc_func* _do_crc32_incremental);
+void crc_clmul256_set_funcs(crc_func* _do_crc32_incremental) {
+	crc_clmul_set_funcs(_do_crc32_incremental);
+}
+#endif
+

package/src/decoder.cc

@@ -4,15 +4,17 @@
 #include "decoder.h"
 
 extern "C" {
-	YencDecoderEnd (*_do_decode)(const unsigned char*HEDLEY_RESTRICT*, unsigned char*HEDLEY_RESTRICT*, size_t, YencDecoderState*) = &do_decode_scalar<false, false>;
-	YencDecoderEnd (*_do_decode_raw)(const unsigned char*HEDLEY_RESTRICT*, unsigned char*HEDLEY_RESTRICT*, size_t, YencDecoderState*) = &do_decode_scalar<true, false>;
-	YencDecoderEnd (*_do_decode_end_raw)(const unsigned char*HEDLEY_RESTRICT*, unsigned char*HEDLEY_RESTRICT*, size_t, YencDecoderState*) = &do_decode_end_scalar<true>;
+	YencDecoderEnd (*_do_decode)(const unsigned char**, unsigned char**, size_t, YencDecoderState*) = &do_decode_scalar<false, false>;
+	YencDecoderEnd (*_do_decode_raw)(const unsigned char**, unsigned char**, size_t, YencDecoderState*) = &do_decode_scalar<true, false>;
+	YencDecoderEnd (*_do_decode_end_raw)(const unsigned char**, unsigned char**, size_t, YencDecoderState*) = &do_decode_end_scalar<true>;
 }
 
 void decoder_set_sse2_funcs();
 void decoder_set_ssse3_funcs();
 void decoder_set_avx_funcs();
 void decoder_set_avx2_funcs();
+void decoder_set_vbmi2_funcs();
+extern const bool decoder_has_avx10;
 void decoder_set_neon_funcs();
 
 
@@ -44,7 +46,9 @@ void decoder_init() {
 	decoder_set_native_funcs();
 # else
 	int use_isa = cpu_supports_isa();
-	if(use_isa >= ISA_LEVEL_AVX2)
+	if(use_isa >= ISA_LEVEL_VBMI2 && (decoder_has_avx10 || (use_isa & ISA_FEATURE_EVEX512)))
+		decoder_set_vbmi2_funcs();
+	else if(use_isa >= ISA_LEVEL_AVX2)
 		decoder_set_avx2_funcs();
 	else if(use_isa >= ISA_LEVEL_AVX)
 		decoder_set_avx_funcs();

package/src/decoder.h

@@ -29,17 +29,17 @@ typedef enum {
 
 #include "hedley.h"
 
-extern YencDecoderEnd (*_do_decode)(const unsigned char*HEDLEY_RESTRICT*, unsigned char*HEDLEY_RESTRICT*, size_t, YencDecoderState*);
-extern YencDecoderEnd (*_do_decode_raw)(const unsigned char*HEDLEY_RESTRICT*, unsigned char*HEDLEY_RESTRICT*, size_t, YencDecoderState*);
-extern YencDecoderEnd (*_do_decode_end_raw)(const unsigned char*HEDLEY_RESTRICT*, unsigned char*HEDLEY_RESTRICT*, size_t, YencDecoderState*);
+extern YencDecoderEnd (*_do_decode)(const unsigned char**, unsigned char**, size_t, YencDecoderState*);
+extern YencDecoderEnd (*_do_decode_raw)(const unsigned char**, unsigned char**, size_t, YencDecoderState*);
+extern YencDecoderEnd (*_do_decode_end_raw)(const unsigned char**, unsigned char**, size_t, YencDecoderState*);
 
-static inline size_t do_decode(int isRaw, const unsigned char* HEDLEY_RESTRICT src, unsigned char* HEDLEY_RESTRICT dest, size_t len, YencDecoderState* state) {
+static inline size_t do_decode(int isRaw, const unsigned char* src, unsigned char* dest, size_t len, YencDecoderState* state) {
 	unsigned char* ds = dest;
 	(*(isRaw ? _do_decode_raw : _do_decode))(&src, &ds, len, state);
 	return ds - dest;
 }
 
-static inline YencDecoderEnd do_decode_end(const unsigned char*HEDLEY_RESTRICT* src, unsigned char*HEDLEY_RESTRICT* dest, size_t len, YencDecoderState* state) {
+static inline YencDecoderEnd do_decode_end(const unsigned char** src, unsigned char** dest, size_t len, YencDecoderState* state) {
 	return _do_decode_end_raw(src, dest, len, state);
 }
 

package/src/decoder_avx2_base.h

@@ -1,8 +1,8 @@
 
 #ifdef __AVX2__
 
-// GCC (ver 6-10(dev)) fails to optimize pure C version of mask testing, but has this intrinsic; Clang >= 7 optimizes C version fine
-#if defined(__GNUC__) && __GNUC__ >= 7
+// GCC (ver 6-10(dev)) fails to optimize pure C version of mask testing, but has this intrinsic; Clang >= 7 optimizes C version fine; functions added in Clang 8
+#if (defined(__GNUC__) && __GNUC__ >= 7) || (defined(_MSC_VER) && _MSC_VER >= 1924)
 # define KORTEST32(a, b) !_kortestz_mask32_u8((a), (b))
 # define KAND32(a, b) _kand_mask32((a), (b))
 # define KOR32(a, b) _kor_mask32((a), (b))
@@ -30,7 +30,7 @@ static HEDLEY_ALWAYS_INLINE __m256i force_align_read_256(const void* p) {
 }
 
 // _mm256_castsi128_si256, but upper is defined to be 0
-#if (defined(__clang__) && __clang_major__ >= 5 && (!defined(__APPLE__) || __clang_major__ >= 7)) || (defined(__GNUC__) && __GNUC__ >= 10)
+#if (defined(__clang__) && __clang_major__ >= 5 && (!defined(__APPLE__) || __clang_major__ >= 7)) || (defined(__GNUC__) && __GNUC__ >= 10) || (defined(_MSC_VER) && _MSC_VER >= 1910)
 // intrinsic unsupported in GCC 9 and MSVC < 2017
 # define zext128_256 _mm256_zextsi128_si256
 #else
@@ -43,9 +43,15 @@ static HEDLEY_ALWAYS_INLINE __m256i force_align_read_256(const void* p) {
 # endif
 #endif
 
+#if defined(__tune_icelake_client__) || defined(__tune_icelake_server__) || defined(__tune_tigerlake__) || defined(__tune_rocketlake__) || defined(__tune_alderlake__) || defined(__tune_sapphirerapids__)
+# define COMPRESS_STORE _mm256_mask_compressstoreu_epi8
+#else
+// avoid uCode on Zen4
+# define COMPRESS_STORE(dst, mask, vec) _mm256_storeu_si256((__m256i*)(dst), _mm256_maskz_compress_epi8(mask, vec))
+#endif
 
 template<bool isRaw, bool searchEnd, enum YEncDecIsaLevel use_isa>
-HEDLEY_ALWAYS_INLINE void do_decode_avx2(const uint8_t* HEDLEY_RESTRICT src, long& len, unsigned char* HEDLEY_RESTRICT & p, unsigned char& _escFirst, uint16_t& _nextMask) {
+HEDLEY_ALWAYS_INLINE void do_decode_avx2(const uint8_t* src, long& len, unsigned char*& p, unsigned char& _escFirst, uint16_t& _nextMask) {
 	HEDLEY_ASSUME(_escFirst == 0 || _escFirst == 1);
 	HEDLEY_ASSUME(_nextMask == 0 || _nextMask == 1 || _nextMask == 2);
 	uintptr_t escFirst = _escFirst;
@@ -60,6 +66,17 @@ HEDLEY_ALWAYS_INLINE void do_decode_avx2(const uint8_t* HEDLEY_RESTRICT src, lon
 			'.','.','.','.','.','.','.','.','.','.','.','.','.','.',_nextMask==2?0:'.',_nextMask==1?0:'.'
 		);
 	}
+	
+	// for some reason, MSVC Win32 seems to crash when trying to compile _mm256_mask_cmpeq_epi8_mask
+	// the crash can be fixed by switching the order of the last two arguments, but it seems to generate wrong code
+	// so just disable the optimisation as it seems to be problematic there
+#if defined(__AVX512VL__) && defined(__AVX512BW__)
+# if defined(_MSC_VER) && !defined(PLATFORM_AMD64) && !defined(__clang__)
+	const bool useAVX3MaskCmp = false;
+# else
+	const bool useAVX3MaskCmp = (use_isa >= ISA_LEVEL_AVX3);
+# endif
+#endif
 	intptr_t i;
 	for(i = -len; i; i += sizeof(__m256i)*2) {
 		__m256i oDataA = _mm256_load_si256((__m256i *)(src+i));
@@ -126,7 +143,7 @@ HEDLEY_ALWAYS_INLINE void do_decode_avx2(const uint8_t* HEDLEY_RESTRICT src, lon
 				__mmask32 match2EqMaskA, match2EqMaskB;
 				__mmask32 match0CrMaskA, match0CrMaskB;
 				__mmask32 match2CrXDtMaskA, match2CrXDtMaskB;
-				if(use_isa >= ISA_LEVEL_AVX3 && searchEnd) {
+				if(useAVX3MaskCmp && searchEnd) {
 					match2EqMaskA = _mm256_cmpeq_epi8_mask(_mm256_set1_epi8('='), tmpData2A);
 					match2EqMaskB = _mm256_cmpeq_epi8_mask(_mm256_set1_epi8('='), tmpData2B);
 				} else
@@ -142,7 +159,7 @@ HEDLEY_ALWAYS_INLINE void do_decode_avx2(const uint8_t* HEDLEY_RESTRICT src, lon
 					// find patterns of \r_.
 					
 #if defined(__AVX512VL__) && defined(__AVX512BW__)
-					if(use_isa >= ISA_LEVEL_AVX3) {
+					if(useAVX3MaskCmp) {
 						match0CrMaskA = _mm256_cmpeq_epi8_mask(oDataA, _mm256_set1_epi8('\r'));
 						match0CrMaskB = _mm256_cmpeq_epi8_mask(oDataB, _mm256_set1_epi8('\r'));
 						match2CrXDtMaskA = _mm256_mask_cmpeq_epi8_mask(match0CrMaskA, tmpData2A, _mm256_set1_epi8('.'));
@@ -172,7 +189,7 @@ HEDLEY_ALWAYS_INLINE void do_decode_avx2(const uint8_t* HEDLEY_RESTRICT src, lon
 #if defined(__AVX512VL__) && defined(__AVX512BW__)
 					__mmask32 match1NlMaskA, match1NlMaskB;
 					__mmask32 match2NlDotMaskA, match2NlDotMaskB;
-					if(use_isa >= ISA_LEVEL_AVX3) {
+					if(useAVX3MaskCmp) {
 						match1NlMaskA = _mm256_mask_cmpeq_epi8_mask(
 							match0CrMaskA,
 							_mm256_set1_epi8('\n'),
@@ -228,7 +245,7 @@ HEDLEY_ALWAYS_INLINE void do_decode_avx2(const uint8_t* HEDLEY_RESTRICT src, lon
 						
 						int matchEnd;
 #if defined(__AVX512VL__) && defined(__AVX512BW__)
-						if(use_isa >= ISA_LEVEL_AVX3) {
+						if(useAVX3MaskCmp) {
 							__mmask32 match3EqYMaskA = _mm256_mask_cmpeq_epi8_mask(
 								match2EqMaskA,
 								_mm256_set1_epi8('y'),
@@ -307,7 +324,7 @@ HEDLEY_ALWAYS_INLINE void do_decode_avx2(const uint8_t* HEDLEY_RESTRICT src, lon
 						}
 					}
 #if defined(__AVX512VL__) && defined(__AVX512BW__)
-					if(use_isa >= ISA_LEVEL_AVX3) {
+					if(useAVX3MaskCmp) {
 						mask |= (uint64_t)match2NlDotMaskA << 2;
 						mask |= (uint64_t)match2NlDotMaskB << 34;
 						minMask = _mm256_maskz_mov_epi8(~(match2NlDotMaskB>>30), _mm256_set1_epi8('.'));
@@ -325,7 +342,7 @@ HEDLEY_ALWAYS_INLINE void do_decode_avx2(const uint8_t* HEDLEY_RESTRICT src, lon
 					__m256i match3EqYA, match3EqYB;
 #if defined(__AVX512VL__) && defined(__AVX512BW__)
 					__mmask32 match3EqYMaskA, match3EqYMaskB;
-					if(use_isa >= ISA_LEVEL_AVX3) {
+					if(useAVX3MaskCmp) {
 						match3EqYMaskA = _mm256_mask_cmpeq_epi8_mask(
 							match2EqMaskA,
 							_mm256_set1_epi8('y'),
@@ -355,7 +372,7 @@ HEDLEY_ALWAYS_INLINE void do_decode_avx2(const uint8_t* HEDLEY_RESTRICT src, lon
 					if(LIKELIHOOD(0.002, partialEndFound)) {
 						bool endFound;
 #if defined(__AVX512VL__) && defined(__AVX512BW__)
-						if(use_isa >= ISA_LEVEL_AVX3) {
+						if(useAVX3MaskCmp) {
 							__mmask32 match3LfEqYMaskA = _mm256_mask_cmpeq_epi8_mask(
 								match3EqYMaskA,
 								_mm256_set1_epi8('\n'),
@@ -530,9 +547,9 @@ HEDLEY_ALWAYS_INLINE void do_decode_avx2(const uint8_t* HEDLEY_RESTRICT src, lon
 			// all that's left is to 'compress' the data (skip over masked chars)
 #if defined(__AVX512VBMI2__) && defined(__AVX512VL__)
 			if(use_isa >= ISA_LEVEL_VBMI2) {
-				_mm256_mask_compressstoreu_epi8(p, KNOT32(mask), dataA);
+				COMPRESS_STORE(p, KNOT32(mask), dataA);
 				p -= popcnt32(mask & 0xffffffff);
-				_mm256_mask_compressstoreu_epi8((p + XMM_SIZE*2), KNOT32(mask>>32), dataB);
+				COMPRESS_STORE((p + XMM_SIZE*2), KNOT32(mask>>32), dataB);
 				p += XMM_SIZE*4 - popcnt32(mask >> 32);
 			} else
 #endif