yencode 1.1.5 → 1.2.1

package/src/crc_arm.cc

@@ -59,42 +59,35 @@ HEDLEY_WARNING("CRC32 acceleration has been disabled due to missing arm_acle.h")
 #endif
 
 
+
+#ifdef __aarch64__
+static uint32_t crc32_multiply_arm(uint32_t a, uint32_t b) {
+	// perform PMULL
+	uint64_t res = 0;
+	uint64_t a64 = (uint64_t)a << 32;
+	int64_t b64 = (int64_t)b << 32;
+	for(int i=0; i<32; i++) {
+		res ^= a64 & (b64 >> 63);
+		b64 += b64;
+		a64 >>= 1;
+	}
+	// reduction via CRC
+	res = __crc32w(0, res) ^ (res >> 32);
+	return res;
+}
+#endif
+// regular multiply is probably better for AArch32
+
+
 // exploit CPU pipelining during CRC computation; unfortunately I haven't been able to measure any benefit
 // - Neoverse N1: no noticeable difference
 // - Cortex A53: actually runs a bit slower
 //#define ENABLE_PIPELINE_OPT 1
 
 #ifdef ENABLE_PIPELINE_OPT
-// workaround MSVC complaining "unary minus operator applied to unsigned type, result still unsigned"
-#define NEGATE(n) (uint32_t)(-((int32_t)(n)))
-
-static HEDLEY_ALWAYS_INLINE uint32_t crc_multiply(uint32_t a, uint32_t b) {
-	uint32_t res = 0;
-	for(int i=0; i<31; i++) {
-		res ^= NEGATE(b>>31) & a;
-		a = ((a >> 1) ^ (0xEDB88320 & NEGATE(a&1)));
-		b <<= 1;
-	}
-	res ^= NEGATE(b>>31) & a;
-	return res;
-}
-
-static const uint32_t crc_power[] = { // pre-computed 2^(2^n), with first 3 entries removed (saves a shift)
-	0x00800000, 0x00008000, 0xedb88320, 0xb1e6b092, 0xa06a2517, 0xed627dae, 0x88d14467, 0xd7bbfe6a,
-	0xec447f11, 0x8e7ea170, 0x6427800e, 0x4d47bae0, 0x09fe548f, 0x83852d0f, 0x30362f1a, 0x7b5a9cc3,
-	0x31fec169, 0x9fec022a, 0x6c8dedc4, 0x15d6874d, 0x5fde7a4e, 0xbad90e37, 0x2e4e5eef, 0x4eaba214,
-	0xa8a472c0, 0x429a969e, 0x148d302a, 0xc40ba6d0, 0xc4e22c3c, 0x40000000, 0x20000000, 0x08000000
-};
-/* above table can be computed with
-	int main(void) {
-		uint32_t k = 0x80000000 >> 1;
-		for (size_t i = 0; i < 32+3; ++i) {
-			if(i>2) printf("0x%08x, ", k);
-			k = crc_multiply(k, k);
-		}
-		return 0;
-	}
-*/
+#ifndef __aarch64__
+# define crc32_multiply_arm RapidYenc::crc32_multiply_generic
+#endif
 #endif
 
 
@@ -130,6 +123,7 @@ static uint32_t arm_crc_calc(uint32_t crc, const unsigned char *src, long len) {
 	// (this is a slightly less efficient, but much simpler implementation of the idea)
 	const unsigned SPLIT_WORDS_LOG = 10;  // make sure it's at least 2
 	const unsigned SPLIT_WORDS = 1<<SPLIT_WORDS_LOG;
+	const unsigned blockCoeff = RapidYenc::crc_power[SPLIT_WORDS_LOG + WORDSIZE_LOG + 3];
 	while(len >= (long)(sizeof(WORD_T)*SPLIT_WORDS*2)) {
 		// compute 2x CRCs concurrently to leverage piplining
 		uint32_t crc2 = 0;
@@ -148,8 +142,7 @@ static uint32_t arm_crc_calc(uint32_t crc, const unsigned char *src, long len) {
 			srcW++;
 		}
 		// merge the CRCs
-		// since we're multiplying by a fixed number, it could be sped up with some lookup tables
-		crc = crc_multiply(crc, crc_power[SPLIT_WORDS_LOG + WORDSIZE_LOG]) ^ crc2;
+		crc = crc32_multiply_arm(crc, blockCoeff) ^ crc2;
 		srcW += SPLIT_WORDS;
 		len -= sizeof(WORD_T)*SPLIT_WORDS*2;
 	}
@@ -200,10 +193,34 @@ static uint32_t do_crc32_incremental_arm(const void* data, size_t length, uint32
 	return ~arm_crc_calc(~init, (const unsigned char*)data, (long)length);
 }
 
-void crc_arm_set_funcs() {
+
+#if defined(__aarch64__) && (defined(__GNUC__) || defined(_MSC_VER))
+static uint32_t crc32_shift_arm(uint32_t crc1, uint32_t n) {
+	uint32_t result = crc1;
+	uint64_t prod = result;
+	prod <<= 32 - (n&31);
+	result = __crc32w(0, prod) ^ (prod >> 32);
+	n &= ~31;
+	
+	while(n) {
+		result = crc32_multiply_arm(result, RapidYenc::crc_power[ctz32(n)]);
+		n &= n-1;
+	}
+	return result;
+}
+#endif
+
+
+void RapidYenc::crc_arm_set_funcs() {
 	_do_crc32_incremental = &do_crc32_incremental_arm;
+#ifdef __aarch64__
+	_crc32_multiply = &crc32_multiply_arm;
+# if defined(__GNUC__) || defined(_MSC_VER)
+	_crc32_shift = &crc32_shift_arm;
+# endif
+#endif
 	_crc32_isa = ISA_FEATURE_CRC;
 }
 #else
-void crc_arm_set_funcs() {}
+void RapidYenc::crc_arm_set_funcs() {}
 #endif

package/src/crc_arm_pmull.cc

@@ -0,0 +1,215 @@
+#include "crc_common.h"
+
+// exclude broken/missing arm_acle.h
+#if defined(__ARM_FEATURE_CRYPTO) && defined(HEDLEY_GCC_VERSION)
+# if !defined(__aarch64__) && HEDLEY_GCC_VERSION_CHECK(7,0,0) && !HEDLEY_GCC_VERSION_CHECK(8,1,1)
+#  undef __ARM_FEATURE_CRYPTO
+# endif
+# if defined(__aarch64__) && HEDLEY_GCC_VERSION_CHECK(9,4,0) && !HEDLEY_GCC_VERSION_CHECK(9,5,0)
+#  undef __ARM_FEATURE_CRYPTO
+# endif
+#endif
+#if defined(__ARM_FEATURE_CRYPTO) && defined(__has_include)
+# if !__has_include(<arm_acle.h>)
+#  undef __ARM_FEATURE_CRYPTO
+# endif
+#endif
+
+// ARM's intrinsics guide seems to suggest that vmull_p64 is available on A32, but neither Clang/GCC seem to support it on AArch32
+#if (defined(__ARM_FEATURE_CRYPTO) && defined(__ARM_FEATURE_CRC32) && defined(__aarch64__)) || (defined(_M_ARM64) && !defined(__clang__))
+
+#include <arm_neon.h>
+#if defined(_MSC_VER) && !defined(__clang__)
+# include <intrin.h>
+
+# ifdef _M_ARM64
+// MSVC may detect this pattern: https://devblogs.microsoft.com/cppblog/a-tour-of-4-msvc-backend-improvements/#byteswap-identification
+static HEDLEY_ALWAYS_INLINE uint64_t rbit64(uint64_t x) {
+	x = _byteswap_uint64(x);
+	x = (x & 0xaaaaaaaaaaaaaaaa) >> 1 | (x & 0x5555555555555555) << 1;
+	x = (x & 0xcccccccccccccccc) >> 2 | (x & 0x3333333333333333) << 2;
+	x = (x & 0xf0f0f0f0f0f0f0f0) >> 4 | (x & 0x0f0f0f0f0f0f0f0f) << 4;
+	return x;
+}
+// ...whilst this seems to work best for 32-bit RBIT
+static HEDLEY_ALWAYS_INLINE uint32_t rbit32(uint32_t x) {
+	uint64_t r = rbit64(x);
+	return r >> 32;
+}
+# else
+#  define rbit32 _arm_rbit
+# endif
+#else
+# include <arm_acle.h>
+// __rbit not present before GCC 11.4.0 or 12.2.0; for ARM32, requires GCC 14
+# if defined(HEDLEY_GCC_VERSION) && !HEDLEY_GCC_VERSION_CHECK(14,0,0) && (!defined(__aarch64__) || !HEDLEY_GCC_VERSION_CHECK(11,3,0) || (HEDLEY_GCC_VERSION_CHECK(12,0,0) && !HEDLEY_GCC_VERSION_CHECK(12,2,0)))
+#  ifdef __aarch64__
+static HEDLEY_ALWAYS_INLINE uint64_t rbit64(uint64_t x) {
+	uint64_t r;
+	__asm__ ("rbit %0,%1\n"
+		: "=r"(r) : "r"(x)
+		: /* No clobbers */);
+	return r;
+}
+#  endif
+static HEDLEY_ALWAYS_INLINE uint32_t rbit32(uint32_t x) {
+	uint32_t r;
+	__asm__ (
+#  ifdef __aarch64__
+		"rbit %w0,%w1\n"
+#  else
+		"rbit %0,%1\n"
+#  endif
+		: "=r"(r) : "r"(x)
+		: /* No clobbers */);
+	return r;
+}
+# else
+#  define rbit32 __rbit
+#  define rbit64 __rbitll
+# endif
+#endif
+
+
+// MSVC doesn't have poly64/poly128 types, so always use uint64 instead
+
+#ifdef __aarch64__
+# if defined(__GNUC__) || defined(__clang__)
+static HEDLEY_ALWAYS_INLINE uint64x2_t pmull_low(uint64x1_t a, uint64x1_t b) {
+	uint64x2_t result;
+	__asm__ ("pmull %0.1q,%1.1d,%2.1d"
+		: "=w"(result)
+		: "w"(a), "w"(b)
+		: /* No clobbers */);
+	return result;
+}
+static HEDLEY_ALWAYS_INLINE uint64x2_t pmull_high(uint64x2_t a, uint64x2_t b) {
+	uint64x2_t result;
+	__asm__ ("pmull2 %0.1q,%1.2d,%2.2d"
+		: "=w"(result)
+		: "w"(a), "w"(b)
+		: /* No clobbers */);
+	return result;
+}
+# elif defined(_MSC_VER) && !defined(__clang__)
+#  define pmull_low vmull_p64
+#  define pmull_high vmull_high_p64
+# else
+#  define pmull_low(x, y) vreinterpretq_u64_p128(vmull_p64(vreinterpret_p64_u64(x), vreinterpret_p64_u64(y)))
+#  define pmull_high(x, y) vreinterpretq_u64_p128(vmull_high_p64(vreinterpretq_p64_u64(x), vreinterpretq_p64_u64(y)))
+# endif
+#else
+# if defined(_MSC_VER) && !defined(__clang__)
+#  define pmull_low vmull_p64
+#  define pmull_high(x, y) vmull_p64(vget_high_u64(x), vget_high_u64(y))
+# else
+#  define pmull_low(x, y) vreinterpretq_u64_p128(vmull_p64(x, y))
+#  define pmull_high(x, y) vreinterpretq_u64_p128(vmull_p64(vget_high_p64(vreinterpretq_p64_u64(x)), vget_high_p64(vreinterpretq_p64_u64(y))))
+# endif
+#endif
+
+
+static uint32_t crc32_multiply_pmull(uint32_t a, uint32_t b) {
+	uint64x1_t prod = vget_low_u64(pmull_low(
+		vreinterpret_u64_u32(vset_lane_u32(a, vdup_n_u32(0), 0)),
+		vreinterpret_u64_u32(vset_lane_u32(b, vdup_n_u32(0), 0))
+	));
+	#ifdef __aarch64__
+	uint64_t p = vget_lane_u64(prod, 0);
+	return __crc32w(0, p+p) ^ (p >> 31);
+	#else
+	prod = vadd_u64(prod, prod);
+	uint32x2_t prod32 = vreinterpret_u32_u64(prod);
+	return __crc32w(0, vget_lane_u32(prod32, 0)) ^ vget_lane_u32(prod32, 1);
+	#endif
+}
+
+
+
+static const uint32_t crc_power_rev[32] = { // bit-reversed crc_power
+	0x00000002, 0x00000004, 0x00000010, 0x00000100, 0x00010000, 0x04c11db7, 0x490d678d, 0xe8a45605,
+	0x75be46b7, 0xe6228b11, 0x567fddeb, 0x88fe2237, 0x0e857e71, 0x7001e426, 0x075de2b2, 0xf12a7f90,
+	0xf0b4a1c1, 0x58f46c0c, 0xc3395ade, 0x96837f8c, 0x544037f9, 0x23b7b136, 0xb2e16ba8, 0x725e7bfa,
+	0xec709b5d, 0xf77a7274, 0x2845d572, 0x034e2515, 0x79695942, 0x540cb128, 0x0b65d023, 0x3c344723
+};
+
+
+static HEDLEY_ALWAYS_INLINE uint64x1_t crc32_shift_pmull_mulred(uint64x1_t a, uint64x1_t b) {
+	uint64x2_t r = pmull_low(a, b);
+	uint64x2_t h = pmull_high(r, vdupq_n_u64(0x490d678d));
+	return veor_u64(vget_low_u64(r), vget_low_u64(h));
+}
+
+
+static uint32_t crc32_shift_pmull(uint32_t crc1, uint32_t n) {
+	crc1 = rbit32(crc1);
+	
+	uint64x1_t res;
+	#ifdef __aarch64__
+	uint64_t crc = (uint64_t)crc1 << (n & 31);
+	res = vset_lane_u64(crc, vdup_n_u64(0), 0);
+	#else
+	res = vreinterpret_u64_u32(vset_lane_u32(crc1, vdup_n_u32(0), 0));
+	res = vshl_u64(res, vdup_n_u64(n&31));
+	#endif
+	n &= ~31;
+	
+	if(n) {
+		#define LOAD_NEXT_POWER vreinterpret_u64_u32(vset_lane_u32(crc_power_rev[ctz32(n)], vdup_n_u32(0), 0))
+		uint64x1_t res2 = LOAD_NEXT_POWER;
+		n &= n-1;
+		
+		if(n) {
+			// first multiply doesn't need reduction
+			res2 = vget_low_u64(pmull_low(res2, LOAD_NEXT_POWER));
+			n &= n-1;
+			
+			while(n) {
+				res = crc32_shift_pmull_mulred(res, LOAD_NEXT_POWER);
+				n &= n-1;
+				
+				if(n) {
+					res2 = crc32_shift_pmull_mulred(res2, LOAD_NEXT_POWER);
+					n &= n-1;
+				}
+			}
+		}
+		#undef LOAD_NEXT_POWER
+		
+		// merge two results
+		uint64x2_t prod = pmull_low(res, res2);
+		// weirdly, vrbitq_u8 is missing in ARM32 MSVC
+		prod = vreinterpretq_u64_u8(vrev64q_u8(vrbitq_u8(vreinterpretq_u8_u64(prod))));
+		#ifdef __aarch64__
+		crc = __crc32d(0, vgetq_lane_u64(prod, 1));
+		uint64_t rem = vgetq_lane_u64(prod, 0);
+		crc = __crc32w(rem, crc) ^ (rem >> 32);
+		#else
+		uint32x4_t prod32 = vreinterpretq_u32_u64(prod);
+		uint32_t crc = __crc32w(0, vgetq_lane_u32(prod32, 2));
+		crc = __crc32w(vgetq_lane_u32(prod32, 3), crc);
+		crc = __crc32w(vgetq_lane_u32(prod32, 0), crc) ^ vgetq_lane_u32(prod32, 1);
+		#endif
+		return crc;
+	} else {
+		#ifdef __aarch64__
+		crc = rbit64(crc);
+		crc = __crc32w(0, crc) ^ (crc >> 32);
+		return crc;
+		#else
+		uint32x2_t r = vreinterpret_u32_u64(res);
+		return __crc32w(0, rbit32(vget_lane_u32(r, 1))) ^ rbit32(vget_lane_u32(r, 0));
+		#endif
+	}
+}
+
+
+void RapidYenc::crc_pmull_set_funcs() {
+	_crc32_multiply = &crc32_multiply_pmull;
+	_crc32_shift = &crc32_shift_pmull;
+	_crc32_isa &= ISA_FEATURE_PMULL;
+}
+
+#else
+void RapidYenc::crc_pmull_set_funcs() {}
+#endif /* defined(__ARM_FEATURE_CRYPTO) && defined(__ARM_FEATURE_CRC32) */

package/src/crc_common.h

@@ -2,3 +2,25 @@
 #include <stddef.h> // for size_t
 #include "crc.h"
 
+#ifdef __GNUC__
+# define ctz32 __builtin_ctz
+#elif defined(_MSC_VER)
+static HEDLEY_ALWAYS_INLINE unsigned ctz32(uint32_t n) {
+	unsigned long r;
+	_BitScanForward(&r, n);
+	return r;
+}
+#endif
+
+namespace RapidYenc {
+	void crc_clmul_set_funcs();
+	void crc_clmul256_set_funcs();
+	void crc_arm_set_funcs();
+	void crc_pmull_set_funcs();
+	void crc_riscv_set_funcs();
+	
+	extern const uint32_t crc_power[32];
+	uint32_t crc32_multiply_generic(uint32_t a, uint32_t b);
+	uint32_t crc32_shift_generic(uint32_t crc1, uint32_t n);
+	
+}

package/src/crc_folding.cc

@@ -140,20 +140,10 @@ static uint32_t crc_fold(const unsigned char *src, long len, uint32_t initial) {
     unsigned long algn_diff;
     __m128i xmm_t0, xmm_t1, xmm_t2, xmm_t3;
 
-    // TODO: consider calculating this via a LUT instead (probably faster)
-    // info from https://www.reddit.com/r/ReverseEngineering/comments/2zwhl3/mystery_constant_0x9db42487_in_intels_crc32ieee/
-    // firstly, calculate: xmm_crc0 = (intial * 0x487b9c8a) mod 0x104c11db7, where 0x487b9c8a = inverse(1<<512) mod 0x104c11db7
-    xmm_t0 = _mm_cvtsi32_si128(~initial);
-    
-    xmm_t0 = _mm_clmulepi64_si128(xmm_t0, _mm_set_epi32(0, 0, 0xa273bc24, 0), 0);  // reverse(0x487b9c8a)<<1 == 0xa273bc24
-    xmm_t2 = _mm_set_epi32( // polynomial reduction factors
-      1, 0xdb710640, // G* = 0x04c11db7
-      0, 0xf7011641  // Q+ = 0x04d101df  (+1 to save an additional xor operation)
-    );
-    xmm_t1 = _mm_clmulepi64_si128(xmm_t0, xmm_t2, 0);
-    xmm_t1 = _mm_clmulepi64_si128(xmm_t1, xmm_t2, 0x10);
-    
-    __m128i xmm_crc0 = _mm_srli_si128(_mm_xor_si128(xmm_t0, xmm_t1), 8);
+    // since the initial value will immediately be multiplied by around 2^512, we need to roll it backwards
+    // this is done by dividing the initial value by 2^480
+    // the constant used here is reverse(2^-480)<<1 == 0xdfded7ec
+    __m128i xmm_crc0 = _mm_clmulepi64_si128(_mm_cvtsi32_si128(~initial), _mm_cvtsi32_si128(0xdfded7ec), 0);
     
     __m128i xmm_crc1 = _mm_setzero_si128();
     __m128i xmm_crc2 = _mm_setzero_si128();
@@ -365,11 +355,159 @@ static uint32_t do_crc32_incremental_clmul(const void* data, size_t length, uint
 	return crc_fold((const unsigned char*)data, (long)length, init);
 }
 
-void crc_clmul_set_funcs() {
+
+static HEDLEY_ALWAYS_INLINE __m128i crc32_reduce(__m128i prod) {
+	// do Barrett reduction back into 32-bit field
+	const __m128i reduction_const = _mm_load_si128((__m128i*)crc_k + 2);
+	__m128i t = _mm_clmulepi64_si128(prod, reduction_const, 0);
+	t = _mm_clmulepi64_si128(t, reduction_const, 0x10);
+	t = _mm_xor_si128(t, prod);
+	return t;
+}
+
+static uint32_t crc32_multiply_clmul(uint32_t a, uint32_t b) {
+	// do the actual multiply
+	__m128i prod = _mm_clmulepi64_si128(_mm_cvtsi32_si128(a), _mm_cvtsi32_si128(b), 0);
+	
+	// prepare product for reduction
+	prod = _mm_add_epi64(prod, prod); // bit alignment fix, due to CRC32 being bit-reversal
+	prod = _mm_slli_si128(prod, 4);   // straddle low/high halves across 64-bit boundary - this provides automatic truncation during reduction
+	
+	prod = crc32_reduce(prod);
+	return _mm_extract_epi32(prod, 2);
+}
+
+#if defined(__GNUC__) || defined(_MSC_VER)
+static HEDLEY_ALWAYS_INLINE __m128i reverse_bits_epi8(__m128i src) {
+#if defined(__GFNI__) && defined(YENC_BUILD_NATIVE) && YENC_BUILD_NATIVE!=0
+    return _mm_gf2p8affine_epi64_epi8(src, _mm_set_epi32(
+      0x80402010, 0x08040201,
+      0x80402010, 0x08040201
+    ), 0);
+/*
+#elif defined(ENABLE_AVX512)
+    // !! this only processes the bottom 32 bits !!
+    src = _mm_maskz_mov_epi32(1, src);
+    src = _mm_ternarylogic_epi32(src, _mm_slli_epi64(src, 28), _mm_set1_epi8(0xf), 0xa8); // (a|b)&c
+    src = _mm_shuffle_epi8(_mm_set_epi8(
+      -16, 112, -80, 48, -48, 80, -112, 16, -32, 96, -96, 32, -64, 64, -128, 0
+    ), src);
+    return _mm_maskz_or_epi32(1, src, _mm_srli_epi64(src, 36));
+*/
+#else
+    __m128i xmm_t0 = _mm_and_si128(src, _mm_set1_epi8(0x0f));
+    __m128i xmm_t1 = _mm_and_si128(_mm_srli_epi16(src, 4), _mm_set1_epi8(0x0f));
+    xmm_t0 = _mm_shuffle_epi8(_mm_set_epi8(
+      -16, 112, -80, 48, -48, 80, -112, 16, -32, 96, -96, 32, -64, 64, -128, 0
+      //0xf0, 0x70, 0xb0, 0x30, 0xd0, 0x50, 0x90, 0x10, 0xe0, 0x60, 0xa0, 0x20, 0xc0, 0x40, 0x80, 0
+    ), xmm_t0);
+    xmm_t1 = _mm_shuffle_epi8(_mm_set_epi8(
+      15, 7, 11, 3, 13, 5, 9, 1, 14, 6, 10, 2, 12, 4, 8, 0
+    ), xmm_t1);
+    return _mm_or_si128(xmm_t0, xmm_t1);
+#endif
+}
+
+#ifdef _MSC_VER
+// because MSVC doesn't use BSWAP unless you specifically tell it to...
+# include <stdlib.h>
+# define BSWAP32 _byteswap_ulong
+#else
+# define BSWAP32(n) ((((n)&0xff)<<24) | (((n)&0xff00)<<8) | (((n)&0xff0000)>>8) | (((n)&0xff000000)>>24))
+#endif
+
+
+
+static const uint32_t crc_power_rev[32] = { // bit-reversed crc_power
+	0x00000002, 0x00000004, 0x00000010, 0x00000100, 0x00010000, 0x04c11db7, 0x490d678d, 0xe8a45605,
+	0x75be46b7, 0xe6228b11, 0x567fddeb, 0x88fe2237, 0x0e857e71, 0x7001e426, 0x075de2b2, 0xf12a7f90,
+	0xf0b4a1c1, 0x58f46c0c, 0xc3395ade, 0x96837f8c, 0x544037f9, 0x23b7b136, 0xb2e16ba8, 0x725e7bfa,
+	0xec709b5d, 0xf77a7274, 0x2845d572, 0x034e2515, 0x79695942, 0x540cb128, 0x0b65d023, 0x3c344723
+};
+
+static HEDLEY_ALWAYS_INLINE __m128i crc32_shift_clmul_mulred(unsigned pos, __m128i prod) {
+	// this multiplies a 64-bit `prod` with a 32-bit CRC power
+	// compared with crc32_multiply_clmul, this only reduces the result to 64-bit, saving a multiply
+	__m128i coeff = _mm_cvtsi32_si128(crc_power_rev[pos]);
+	
+	const __m128i fold_const = _mm_set_epi32(0, 0x490d678d, 0, 0xf200aa66);
+	prod = _mm_clmulepi64_si128(prod, coeff, 0);
+	__m128i hi = _mm_clmulepi64_si128(prod, fold_const, 0x11);
+	return _mm_xor_si128(hi, prod);
+}
+
+static uint32_t crc32_shift_clmul(uint32_t crc1, uint32_t n) {
+	if(!n) return crc1;
+	
+	__m128i result = _mm_cvtsi32_si128(BSWAP32(crc1));
+	result = reverse_bits_epi8(result);
+	
+	// handle n < 32 with a shift
+	result = _mm_sll_epi64(result, _mm_cvtsi32_si128(n & 31));
+	n &= ~31;
+	
+	__m128i t;
+	if(n) {
+		// use a second accumulator to leverage some IPC from slow CLMUL
+		__m128i result2 = _mm_cvtsi32_si128(crc_power_rev[ctz32(n)]);
+		n &= n-1;
+		
+		if(n) {
+			// first multiply doesn't need reduction
+			result2 = _mm_clmulepi64_si128(result2, _mm_cvtsi32_si128(crc_power_rev[ctz32(n)]), 0);
+			n &= n-1;
+			
+			while(n) {
+				result = crc32_shift_clmul_mulred(ctz32(n), result);
+				n &= n-1;
+				
+				if(n) {
+					result2 = crc32_shift_clmul_mulred(ctz32(n), result2);
+					n &= n-1;
+				}
+			}
+		}
+		
+		const __m128i fold_const = _mm_set_epi32(0, 0x490d678d, 0, 0xf200aa66);
+		
+		// merge two results
+		result = _mm_clmulepi64_si128(result, result2, 0);
+		
+		// do 128b reduction
+		t = _mm_unpackhi_epi32(result, _mm_setzero_si128());
+		// fold [127:96] -> [63:0]
+		__m128i hi = _mm_clmulepi64_si128(t, fold_const, 1);
+		// fold [95:64] -> [63:0]
+		__m128i lo = _mm_clmulepi64_si128(t, fold_const, 0x10);
+#ifdef ENABLE_AVX512
+		result = _mm_ternarylogic_epi32(result, hi, lo, 0x96);
+#else
+		result = _mm_xor_si128(result, hi);
+		result = _mm_xor_si128(result, lo);
+#endif
+	}
+	
+	// do Barrett reduction back into 32-bit field
+	const __m128i reduction_const = _mm_set_epi32(0, 0x04c11db7, 1, 0x04d101df);
+	t = _mm_clmulepi64_si128(_mm_blend_epi16(_mm_setzero_si128(), result, 0x3c), reduction_const, 0);
+	t = _mm_clmulepi64_si128(t, reduction_const, 0x11);
+	result = _mm_xor_si128(t, result);
+	
+	result = reverse_bits_epi8(result);
+	return BSWAP32(_mm_cvtsi128_si32(result));
+}
+#endif
+
+
+void RapidYenc::crc_clmul_set_funcs() {
 	_do_crc32_incremental = &do_crc32_incremental_clmul;
+	_crc32_multiply = &crc32_multiply_clmul;
+#if defined(__GNUC__) || defined(_MSC_VER)
+	_crc32_shift = &crc32_shift_clmul;
+#endif
 	_crc32_isa = ISA_LEVEL_PCLMUL;
 }
 #else
-void crc_clmul_set_funcs() {}
+void RapidYenc::crc_clmul_set_funcs() {}
 #endif
 

package/src/crc_folding_256.cc

@@ -99,19 +99,12 @@ ALIGN_TO(16, static const unsigned crc_k[]) = {
 
 
 static uint32_t crc_fold(const unsigned char *src, long len, uint32_t initial) {
-	// info from https://www.reddit.com/r/ReverseEngineering/comments/2zwhl3/mystery_constant_0x9db42487_in_intels_crc32ieee/
-	// firstly, calculate: xmm_crc0 = (intial * 0x487b9c8a) mod 0x104c11db7, where 0x487b9c8a = inverse(1<<512) mod 0x104c11db7
-	__m128i xmm_t0 = _mm_cvtsi32_si128(~initial);
-	
-	xmm_t0 = _mm_clmulepi64_si128(xmm_t0, _mm_set_epi32(0, 0, 0xa273bc24, 0), 0);  // reverse(0x487b9c8a)<<1 == 0xa273bc24
-	__m128i reduction = _mm_set_epi32( // polynomial reduction factors
-	  1, 0xdb710640, // G* = 0x04c11db7
-	  0, 0xf7011641  // Q+ = 0x04d101df  (+1 to save an additional xor operation)
+	__m128i xmm_t0 = _mm_clmulepi64_si128(
+		_mm_cvtsi32_si128(~initial),
+		_mm_cvtsi32_si128(0xdfded7ec),
+		0
 	);
-	__m128i xmm_t1 = _mm_clmulepi64_si128(xmm_t0, reduction, 0);
-	xmm_t1 = _mm_clmulepi64_si128(xmm_t1, reduction, 0x10);
 	
-	xmm_t0 = _mm_srli_si128(_mm_xor_si128(xmm_t0, xmm_t1), 8);
 	__m256i crc0 = zext128_256(xmm_t0);
 	__m256i crc1 = _mm256_setzero_si256();
 	
@@ -217,13 +210,13 @@ static uint32_t do_crc32_incremental_clmul(const void* data, size_t length, uint
 	return crc_fold((const unsigned char*)data, (long)length, init);
 }
 
-void crc_clmul256_set_funcs() {
+void RapidYenc::crc_clmul256_set_funcs() {
+	crc_clmul_set_funcs(); // set multiply/shift function
 	_do_crc32_incremental = &do_crc32_incremental_clmul;
 	_crc32_isa = ISA_LEVEL_VPCLMUL;
 }
 #else
-void crc_clmul_set_funcs();
-void crc_clmul256_set_funcs() {
+void RapidYenc::crc_clmul256_set_funcs() {
 	crc_clmul_set_funcs();
 }
 #endif