yencode 1.1.4 → 1.2.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "yencode",
-  "version": "1.1.4",
+  "version": "1.2.0",
   "description": "SIMD accelerated yEnc encoder/decoder and CRC32 calculator",
   "keywords": [
     "yenc",

package/src/common.h

@@ -221,6 +221,7 @@ bool cpu_supports_neon();
 
 #ifdef PLATFORM_X86
 enum YEncDecIsaLevel {
+	ISA_GENERIC = 0,
 	ISA_FEATURE_POPCNT = 0x1,
 	ISA_FEATURE_LZCNT = 0x2,
 	ISA_FEATURE_EVEX512 = 0x4, // AVX512 support
@@ -228,11 +229,31 @@ enum YEncDecIsaLevel {
 	ISA_LEVEL_SSSE3 = 0x200,
 	ISA_LEVEL_SSE41 = 0x300,
 	ISA_LEVEL_SSE4_POPCNT = 0x301,
+	ISA_LEVEL_PCLMUL = 0x340,
 	ISA_LEVEL_AVX = 0x381, // same as above, just used as a differentiator for `cpu_supports_isa`
 	ISA_LEVEL_AVX2 = 0x403, // also includes BMI1/2 and LZCNT
+	ISA_LEVEL_VPCLMUL = 0x440,
 	ISA_LEVEL_AVX3 = 0x507, // SKX variant; AVX512VL + AVX512BW
 	ISA_LEVEL_VBMI2 = 0x603 // ICL, AVX10
 };
+#elif defined(PLATFORM_ARM)
+enum YEncDecIsaLevel {
+	ISA_GENERIC = 0,
+	ISA_FEATURE_CRC = 8,
+	ISA_LEVEL_NEON = 0x1000
+};
+#elif defined(__riscv)
+enum YEncDecIsaLevel {
+	ISA_GENERIC = 0,
+	ISA_FEATURE_ZBC = 16,
+	ISA_LEVEL_RVV = 0x10000
+};
+#else
+enum YEncDecIsaLevel {
+	ISA_GENERIC = 0
+};
+#endif
+#ifdef PLATFORM_X86
 #ifdef _MSC_VER
 // native tuning not supported in MSVC
 # define ISA_NATIVE ISA_LEVEL_SSE2
@@ -271,8 +292,25 @@ bool cpu_supports_rvv();
 #if defined(__riscv_vector) && defined(HEDLEY_GCC_VERSION) && !HEDLEY_GCC_VERSION_CHECK(13,0,0)
 // GCC added RVV intrinsics in GCC13
 # undef __riscv_vector
+#elif defined(__riscv_vector) && defined(HEDLEY_GCC_VERSION) && !HEDLEY_GCC_VERSION_CHECK(14,0,0)
+// ...however, GCC13 lacks necessary mask<>vector vreinterpret casts, and it crashes on type punning, so I can't be bothered trying to make it work
+# undef __riscv_vector
+#endif
+#ifdef __riscv_vector
+# include <riscv_vector.h>
+# ifdef __riscv_v_intrinsic
+#  define RV(f) __riscv_##f
+# else
+#  define RV(f) f
+# endif
+# if defined(__riscv_v_intrinsic) && __riscv_v_intrinsic >= 12000
+#  define RV_MASK_CAST(masksz, vecsz, vec) RV(vreinterpret_v_u##vecsz##m1_b##masksz)(vec)
+#  define RV_VEC_U8MF4_CAST(vec) RV(vlmul_trunc_v_u8m1_u8mf4)(RV(vreinterpret_v_b4_u8m1)(vec))
+# else
+#  define RV_MASK_CAST(masksz, vecsz, vec) *(vbool##masksz##_t*)(&(vec))
+#  define RV_VEC_U8MF4_CAST(vec) *(vuint8mf4_t*)(&(vec))
+# endif
 #endif
-
 
 #include <string.h>
 #if !defined(_MSC_VER) || defined(_STDINT) || _MSC_VER >= 1900

package/src/crc.cc

@@ -1,16 +1,23 @@
 #include "crc_common.h"
 
+#if defined(PLATFORM_X86) && !defined(__ILP32__) && !defined(YENC_DISABLE_CRCUTIL)
+// Use crcutil for computing CRC32 (generic implementation)
+
 #include "interface.h"
 crcutil_interface::CRC* crc = NULL;
+#define GENERIC_CRC_INIT crc = crcutil_interface::CRC::Create(0xEDB88320, 0, 32, true, 0, 0, 0, 0, NULL)
+// instance never deleted... oh well...
 
-#if defined(PLATFORM_X86) && !defined(__ILP32__)
 static uint32_t do_crc32_incremental_generic(const void* data, size_t length, uint32_t init) {
 	// use optimised ASM on x86 platforms
 	crcutil_interface::UINT64 tmp = init;
 	crc->Compute(data, length, &tmp);
 	return (uint32_t)tmp;
 }
+
 #else
+// don't use crcutil
+
 static uint32_t* HEDLEY_RESTRICT crc_slice_table;
 #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
 # if defined(__GNUC__) || defined(__clang__)
@@ -121,28 +128,73 @@ static void generate_crc32_slice_table() {
 		#endif
 	}
 }
+
+#define GENERIC_CRC_INIT generate_crc32_slice_table()
 #endif
 
 
-crc_func _do_crc32_incremental = &do_crc32_incremental_generic;
 
+// workaround MSVC complaining "unary minus operator applied to unsigned type, result still unsigned"
+#define NEGATE(n) (uint32_t)(-((int32_t)(n)))
+uint32_t crc32_multiply_generic(uint32_t a, uint32_t b) {
+	uint32_t res = 0;
+	for(int i=0; i<31; i++) {
+		res ^= NEGATE(b>>31) & a;
+		a = ((a >> 1) ^ (0xEDB88320 & NEGATE(a&1)));
+		b <<= 1;
+	}
+	res ^= NEGATE(b>>31) & a;
+	return res;
+}
+#undef NEGATE
 
+const uint32_t crc_power[32] = { // pre-computed 2^(2^n)
+	0x40000000, 0x20000000, 0x08000000, 0x00800000, 0x00008000, 0xedb88320, 0xb1e6b092, 0xa06a2517,
+	0xed627dae, 0x88d14467, 0xd7bbfe6a, 0xec447f11, 0x8e7ea170, 0x6427800e, 0x4d47bae0, 0x09fe548f,
+	0x83852d0f, 0x30362f1a, 0x7b5a9cc3, 0x31fec169, 0x9fec022a, 0x6c8dedc4, 0x15d6874d, 0x5fde7a4e,
+	0xbad90e37, 0x2e4e5eef, 0x4eaba214, 0xa8a472c0, 0x429a969e, 0x148d302a, 0xc40ba6d0, 0xc4e22c3c
+};
 
-uint32_t do_crc32_combine(uint32_t crc1, uint32_t crc2, size_t len2) {
-	crcutil_interface::UINT64 crc1_ = crc1, crc2_ = crc2;
-	crc->Concatenate(crc2_, 0, len2, &crc1_);
-	return (uint32_t)crc1_;
+uint32_t crc32_shift_generic(uint32_t crc1, uint32_t n) {
+	uint32_t result = crc1;
+#ifdef __GNUC__
+	while(n) {
+		result = crc32_multiply_generic(result, crc_power[__builtin_ctz(n)]);
+		n &= n-1;
+	}
+#elif defined(_MSC_VER)
+	unsigned long power;
+	while(_BitScanForward(&power, n)) {
+		result = crc32_multiply_generic(result, crc_power[power]);
+		n &= n-1;
+	}
+#else
+	unsigned power = 0;
+	while(n) {
+		if(n & 1) {
+			result = crc32_multiply_generic(result, crc_power[power]);
+		}
+		n >>= 1;
+		power++;
+	}
+#endif
+	return result;
 }
 
-uint32_t do_crc32_zeros(uint32_t crc1, size_t len) {
-	crcutil_interface::UINT64 crc_ = crc1;
-	crc->CrcOfZeroes(len, &crc_);
-	return (uint32_t)crc_;
+
+extern "C" {
+	crc_func _do_crc32_incremental = &do_crc32_incremental_generic;
+	crc_mul_func _crc32_shift = &crc32_shift_generic;
+	crc_mul_func _crc32_multiply = &crc32_multiply_generic;
+	int _crc32_isa = ISA_GENERIC;
 }
 
-void crc_clmul_set_funcs(crc_func*);
-void crc_clmul256_set_funcs(crc_func*);
-void crc_arm_set_funcs(crc_func*);
+
+
+void crc_clmul_set_funcs();
+void crc_clmul256_set_funcs();
+void crc_arm_set_funcs();
+void crc_riscv_set_funcs();
 
 #ifdef PLATFORM_X86
 int cpu_supports_crc_isa();
@@ -174,21 +226,23 @@ static unsigned long getauxval(unsigned long cap) {
 #  endif
 # endif
 #endif
-void crc_init() {
-	crc = crcutil_interface::CRC::Create(
-		0xEDB88320, 0, 32, true, 0, 0, 0, 0, NULL);
-	// instance never deleted... oh well...
-	
-#if !defined(PLATFORM_X86) || defined(__ILP32__)
-	generate_crc32_slice_table();
+#if defined(__riscv) && defined(__has_include)
+# if __has_include(<asm/hwprobe.h>)
+#  include <asm/hwprobe.h>
+#  include <asm/unistd.h>
+#  include <unistd.h>
+# endif
 #endif
+
+void crc_init() {
+	GENERIC_CRC_INIT;
 	
 #ifdef PLATFORM_X86
 	int support = cpu_supports_crc_isa();
 	if(support == 2)
-		crc_clmul256_set_funcs(&_do_crc32_incremental);
+		crc_clmul256_set_funcs();
 	else if(support == 1)
-		crc_clmul_set_funcs(&_do_crc32_incremental);
+		crc_clmul_set_funcs();
 #endif
 #ifdef PLATFORM_ARM
 # ifdef __APPLE__
@@ -216,7 +270,19 @@ void crc_init() {
 		false
 # endif
 	) {
-		crc_arm_set_funcs(&_do_crc32_incremental);
+		crc_arm_set_funcs();
+	}
+#endif
+#ifdef __riscv
+# if defined(RISCV_HWPROBE_KEY_IMA_EXT_0) && defined(__NR_riscv_hwprobe)
+	const int rv_hwprobe_ext_zbc = 1 << 7, rv_hwprobe_ext_zbkc = 1 << 9;
+	struct riscv_hwprobe p;
+	p.key = RISCV_HWPROBE_KEY_IMA_EXT_0;
+	if(!syscall(__NR_riscv_hwprobe, &p, 1, 0, NULL, 0)) {
+		if(p.value & (rv_hwprobe_ext_zbc | rv_hwprobe_ext_zbkc)) {
+			crc_riscv_set_funcs();
+		}
 	}
+# endif
 #endif
 }

package/src/crc.h

@@ -1,5 +1,6 @@
 #ifndef __YENC_CRC_H
 #define __YENC_CRC_H
+#include <stdlib.h> // for llabs
 
 #ifdef __cplusplus
 extern "C" {
@@ -9,10 +10,75 @@ extern "C" {
 
 typedef uint32_t (*crc_func)(const void*, size_t, uint32_t);
 extern crc_func _do_crc32_incremental;
+
+extern int _crc32_isa;
 #define do_crc32 (*_do_crc32_incremental)
+static inline int crc32_isa_level() {
+	return _crc32_isa;
+}
+
+
+#if !defined(__GNUC__) && defined(_MSC_VER)
+# include <intrin.h>
+#endif
+// computes `n % 0xffffffff` (well, almost), using some bit-hacks
+static inline uint32_t crc32_powmod(uint64_t n) {
+#ifdef __GNUC__
+	unsigned res;
+	unsigned carry = __builtin_uadd_overflow(n >> 32, n, &res);
+	res += carry;
+	return res;
+#elif defined(_MSC_VER)
+	unsigned res;
+	unsigned char carry = _addcarry_u32(0, n >> 32, n, &res);
+	_addcarry_u32(carry, res, 0, &res);
+	return res;
+#else
+	n = (n >> 32) + (n & 0xffffffff);
+	n += n >> 32;
+	return n;
+#endif
+}
+// computes `crc32_powmod(n*8)` avoiding overflow
+static inline uint32_t crc32_bytepow(uint64_t n) {
+#if defined(__GNUC__) || defined(_MSC_VER)
+	unsigned res = crc32_powmod(n);
+# ifdef _MSC_VER
+	return _rotl(res, 3);
+# else
+	return (res << 3) | (res >> 29);
+# endif
+#else
+	n = (n >> 32) + (n & 0xffffffff);
+	n <<= 3;
+	n += n >> 32;
+	return n;
+#endif
+}
+
+typedef uint32_t (*crc_mul_func)(uint32_t, uint32_t);
+extern crc_mul_func _crc32_shift;
+extern crc_mul_func _crc32_multiply;
+#define crc32_shift (*_crc32_shift)
+#define crc32_multiply (*_crc32_multiply)
+
+static inline uint32_t crc32_combine(uint32_t crc1, uint32_t crc2, uint64_t len2) {
+	return crc32_shift(crc1, crc32_bytepow(len2)) ^ crc2;
+}
+static inline uint32_t crc32_zeros(uint32_t crc1, uint64_t len) {
+	return ~crc32_shift(~crc1, crc32_bytepow(len));
+}
+static inline uint32_t crc32_unzero(uint32_t crc1, uint64_t len) {
+	return ~crc32_shift(~crc1, ~crc32_bytepow(len));
+}
+static inline uint32_t crc32_2pow(int64_t n) {
+	uint32_t sign = (uint32_t)(n >> 63);
+	return crc32_shift(0x80000000, crc32_powmod(llabs(n)) ^ sign);
+}
+static inline uint32_t crc32_256pow(uint64_t n) {
+	return crc32_shift(0x80000000, crc32_bytepow(n));
+}
 
-uint32_t do_crc32_combine(uint32_t crc1, const uint32_t crc2, size_t len2);
-uint32_t do_crc32_zeros(uint32_t crc1, size_t len);
 void crc_init();
 
 

package/src/crc_arm.cc

@@ -59,42 +59,36 @@ HEDLEY_WARNING("CRC32 acceleration has been disabled due to missing arm_acle.h")
 #endif
 
 
+
+#ifdef __aarch64__
+uint32_t crc32_multiply_arm(uint32_t a, uint32_t b) {
+	// perform PMULL
+	uint64_t res = 0;
+	uint64_t a64 = (uint64_t)a << 32;
+	int64_t b64 = (int64_t)b << 32;
+	for(int i=0; i<32; i++) {
+		res ^= a64 & (b64 >> 63);
+		b64 += b64;
+		a64 >>= 1;
+	}
+	// reduction via CRC
+	res = __crc32w(0, res) ^ (res >> 32);
+	return res;
+}
+#endif
+// regular multiply is probably better for AArch32
+
+
 // exploit CPU pipelining during CRC computation; unfortunately I haven't been able to measure any benefit
 // - Neoverse N1: no noticeable difference
 // - Cortex A53: actually runs a bit slower
 //#define ENABLE_PIPELINE_OPT 1
 
 #ifdef ENABLE_PIPELINE_OPT
-// workaround MSVC complaining "unary minus operator applied to unsigned type, result still unsigned"
-#define NEGATE(n) (uint32_t)(-((int32_t)(n)))
-
-static HEDLEY_ALWAYS_INLINE uint32_t crc_multiply(uint32_t a, uint32_t b) {
-	uint32_t res = 0;
-	for(int i=0; i<31; i++) {
-		res ^= NEGATE(b>>31) & a;
-		a = ((a >> 1) ^ (0xEDB88320 & NEGATE(a&1)));
-		b <<= 1;
-	}
-	res ^= NEGATE(b>>31) & a;
-	return res;
-}
-
-static const uint32_t crc_power[] = { // pre-computed 2^(2^n), with first 3 entries removed (saves a shift)
-	0x00800000, 0x00008000, 0xedb88320, 0xb1e6b092, 0xa06a2517, 0xed627dae, 0x88d14467, 0xd7bbfe6a,
-	0xec447f11, 0x8e7ea170, 0x6427800e, 0x4d47bae0, 0x09fe548f, 0x83852d0f, 0x30362f1a, 0x7b5a9cc3,
-	0x31fec169, 0x9fec022a, 0x6c8dedc4, 0x15d6874d, 0x5fde7a4e, 0xbad90e37, 0x2e4e5eef, 0x4eaba214,
-	0xa8a472c0, 0x429a969e, 0x148d302a, 0xc40ba6d0, 0xc4e22c3c, 0x40000000, 0x20000000, 0x08000000
-};
-/* above table can be computed with
-	int main(void) {
-		uint32_t k = 0x80000000 >> 1;
-		for (size_t i = 0; i < 32+3; ++i) {
-			if(i>2) printf("0x%08x, ", k);
-			k = crc_multiply(k, k);
-		}
-		return 0;
-	}
-*/
+#ifndef __aarch64__
+uint32_t crc32_multiply_generic(uint32_t a, uint32_t b);
+# define crc32_multiply_arm crc32_multiply_generic
+#endif
 #endif
 
 
@@ -130,6 +124,7 @@ static uint32_t arm_crc_calc(uint32_t crc, const unsigned char *src, long len) {
 	// (this is a slightly less efficient, but much simpler implementation of the idea)
 	const unsigned SPLIT_WORDS_LOG = 10;  // make sure it's at least 2
 	const unsigned SPLIT_WORDS = 1<<SPLIT_WORDS_LOG;
+	const unsigned blockCoeff = crc_power[SPLIT_WORDS_LOG + WORDSIZE_LOG + 3];
 	while(len >= (long)(sizeof(WORD_T)*SPLIT_WORDS*2)) {
 		// compute 2x CRCs concurrently to leverage piplining
 		uint32_t crc2 = 0;
@@ -148,8 +143,7 @@ static uint32_t arm_crc_calc(uint32_t crc, const unsigned char *src, long len) {
 			srcW++;
 		}
 		// merge the CRCs
-		// since we're multiplying by a fixed number, it could be sped up with some lookup tables
-		crc = crc_multiply(crc, crc_power[SPLIT_WORDS_LOG + WORDSIZE_LOG]) ^ crc2;
+		crc = crc32_multiply_arm(crc, blockCoeff) ^ crc2;
 		srcW += SPLIT_WORDS;
 		len -= sizeof(WORD_T)*SPLIT_WORDS*2;
 	}
@@ -200,11 +194,34 @@ static uint32_t do_crc32_incremental_arm(const void* data, size_t length, uint32
 	return ~arm_crc_calc(~init, (const unsigned char*)data, (long)length);
 }
 
-void crc_arm_set_funcs(crc_func* _do_crc32_incremental) {
-	*_do_crc32_incremental = &do_crc32_incremental_arm;
+
+#if defined(__aarch64__) && (defined(__GNUC__) || defined(_MSC_VER))
+uint32_t crc32_shift_arm(uint32_t crc1, uint32_t n) {
+	uint32_t result = crc1;
+	uint64_t prod = result;
+	prod <<= 32 - (n&31);
+	result = __crc32w(0, prod) ^ (prod >> 32);
+	n &= ~31;
+	
+	while(n) {
+		result = crc32_multiply_arm(result, crc_power[ctz32(n)]);
+		n &= n-1;
+	}
+	return result;
 }
-#else
-void crc_arm_set_funcs(crc_func* _do_crc32_incremental) {
-	(void)_do_crc32_incremental;
+#endif
+
+
+void crc_arm_set_funcs() {
+	_do_crc32_incremental = &do_crc32_incremental_arm;
+#ifdef __aarch64__
+	_crc32_multiply = &crc32_multiply_arm;
+# if defined(__GNUC__) || defined(_MSC_VER)
+	_crc32_shift = &crc32_shift_arm;
+# endif
+#endif
+	_crc32_isa = ISA_FEATURE_CRC;
 }
+#else
+void crc_arm_set_funcs() {}
 #endif

package/src/crc_common.h

@@ -2,3 +2,14 @@
 #include <stddef.h> // for size_t
 #include "crc.h"
 
+extern const uint32_t crc_power[32];
+
+#ifdef __GNUC__
+# define ctz32 __builtin_ctz
+#elif defined(_MSC_VER)
+static HEDLEY_ALWAYS_INLINE unsigned ctz32(uint32_t n) {
+	unsigned long r;
+	_BitScanForward(&r, n);
+	return r;
+}
+#endif

package/src/crc_folding.cc

@@ -140,20 +140,10 @@ static uint32_t crc_fold(const unsigned char *src, long len, uint32_t initial) {
     unsigned long algn_diff;
     __m128i xmm_t0, xmm_t1, xmm_t2, xmm_t3;
 
-    // TODO: consider calculating this via a LUT instead (probably faster)
-    // info from https://www.reddit.com/r/ReverseEngineering/comments/2zwhl3/mystery_constant_0x9db42487_in_intels_crc32ieee/
-    // firstly, calculate: xmm_crc0 = (intial * 0x487b9c8a) mod 0x104c11db7, where 0x487b9c8a = inverse(1<<512) mod 0x104c11db7
-    xmm_t0 = _mm_cvtsi32_si128(~initial);
-    
-    xmm_t0 = _mm_clmulepi64_si128(xmm_t0, _mm_set_epi32(0, 0, 0xa273bc24, 0), 0);  // reverse(0x487b9c8a)<<1 == 0xa273bc24
-    xmm_t2 = _mm_set_epi32( // polynomial reduction factors
-      1, 0xdb710640, // G* = 0x04c11db7
-      0, 0xf7011641  // Q+ = 0x04d101df  (+1 to save an additional xor operation)
-    );
-    xmm_t1 = _mm_clmulepi64_si128(xmm_t0, xmm_t2, 0);
-    xmm_t1 = _mm_clmulepi64_si128(xmm_t1, xmm_t2, 0x10);
-    
-    __m128i xmm_crc0 = _mm_srli_si128(_mm_xor_si128(xmm_t0, xmm_t1), 8);
+    // since the initial value will immediately be multiplied by around 2^512, we need to roll it backwards
+    // this is done by dividing the initial value by 2^480
+    // the constant used here is reverse(2^-480)<<1 == 0xdfded7ec
+    __m128i xmm_crc0 = _mm_clmulepi64_si128(_mm_cvtsi32_si128(~initial), _mm_cvtsi32_si128(0xdfded7ec), 0);
     
     __m128i xmm_crc1 = _mm_setzero_si128();
     __m128i xmm_crc2 = _mm_setzero_si128();
@@ -365,12 +355,159 @@ static uint32_t do_crc32_incremental_clmul(const void* data, size_t length, uint
 	return crc_fold((const unsigned char*)data, (long)length, init);
 }
 
-void crc_clmul_set_funcs(crc_func* _do_crc32_incremental) {
-	*_do_crc32_incremental = &do_crc32_incremental_clmul;
+
+static HEDLEY_ALWAYS_INLINE __m128i crc32_reduce(__m128i prod) {
+	// do Barrett reduction back into 32-bit field
+	const __m128i reduction_const = _mm_load_si128((__m128i*)crc_k + 2);
+	__m128i t = _mm_clmulepi64_si128(prod, reduction_const, 0);
+	t = _mm_clmulepi64_si128(t, reduction_const, 0x10);
+	t = _mm_xor_si128(t, prod);
+	return t;
+}
+
+uint32_t crc32_multiply_clmul(uint32_t a, uint32_t b) {
+	// do the actual multiply
+	__m128i prod = _mm_clmulepi64_si128(_mm_cvtsi32_si128(a), _mm_cvtsi32_si128(b), 0);
+	
+	// prepare product for reduction
+	prod = _mm_add_epi64(prod, prod); // bit alignment fix, due to CRC32 being bit-reversal
+	prod = _mm_slli_si128(prod, 4);   // straddle low/high halves across 64-bit boundary - this provides automatic truncation during reduction
+	
+	prod = crc32_reduce(prod);
+	return _mm_extract_epi32(prod, 2);
 }
+
+#if defined(__GNUC__) || defined(_MSC_VER)
+static HEDLEY_ALWAYS_INLINE __m128i reverse_bits_epi8(__m128i src) {
+#if defined(__GFNI__) && defined(YENC_BUILD_NATIVE) && YENC_BUILD_NATIVE!=0
+    return _mm_gf2p8affine_epi64_epi8(src, _mm_set_epi32(
+      0x80402010, 0x08040201,
+      0x80402010, 0x08040201
+    ), 0);
+/*
+#elif defined(ENABLE_AVX512)
+    // !! this only processes the bottom 32 bits !!
+    src = _mm_maskz_mov_epi32(1, src);
+    src = _mm_ternarylogic_epi32(src, _mm_slli_epi64(src, 28), _mm_set1_epi8(0xf), 0xa8); // (a|b)&c
+    src = _mm_shuffle_epi8(_mm_set_epi8(
+      -16, 112, -80, 48, -48, 80, -112, 16, -32, 96, -96, 32, -64, 64, -128, 0
+    ), src);
+    return _mm_maskz_or_epi32(1, src, _mm_srli_epi64(src, 36));
+*/
 #else
-void crc_clmul_set_funcs(crc_func* _do_crc32_incremental) {
-    (void)_do_crc32_incremental;
+    __m128i xmm_t0 = _mm_and_si128(src, _mm_set1_epi8(0x0f));
+    __m128i xmm_t1 = _mm_and_si128(_mm_srli_epi16(src, 4), _mm_set1_epi8(0x0f));
+    xmm_t0 = _mm_shuffle_epi8(_mm_set_epi8(
+      -16, 112, -80, 48, -48, 80, -112, 16, -32, 96, -96, 32, -64, 64, -128, 0
+      //0xf0, 0x70, 0xb0, 0x30, 0xd0, 0x50, 0x90, 0x10, 0xe0, 0x60, 0xa0, 0x20, 0xc0, 0x40, 0x80, 0
+    ), xmm_t0);
+    xmm_t1 = _mm_shuffle_epi8(_mm_set_epi8(
+      15, 7, 11, 3, 13, 5, 9, 1, 14, 6, 10, 2, 12, 4, 8, 0
+    ), xmm_t1);
+    return _mm_or_si128(xmm_t0, xmm_t1);
+#endif
 }
+
+#ifdef _MSC_VER
+// because MSVC doesn't use BSWAP unless you specifically tell it to...
+# include <stdlib.h>
+# define BSWAP32 _byteswap_ulong
+#else
+# define BSWAP32(n) ((((n)&0xff)<<24) | (((n)&0xff00)<<8) | (((n)&0xff0000)>>8) | (((n)&0xff000000)>>24))
+#endif
+
+
+
+const uint32_t crc_power_rev[32] = { // bit-reversed crc_power
+	0x00000002, 0x00000004, 0x00000010, 0x00000100, 0x00010000, 0x04c11db7, 0x490d678d, 0xe8a45605,
+	0x75be46b7, 0xe6228b11, 0x567fddeb, 0x88fe2237, 0x0e857e71, 0x7001e426, 0x075de2b2, 0xf12a7f90,
+	0xf0b4a1c1, 0x58f46c0c, 0xc3395ade, 0x96837f8c, 0x544037f9, 0x23b7b136, 0xb2e16ba8, 0x725e7bfa,
+	0xec709b5d, 0xf77a7274, 0x2845d572, 0x034e2515, 0x79695942, 0x540cb128, 0x0b65d023, 0x3c344723
+};
+
+static HEDLEY_ALWAYS_INLINE __m128i crc32_shift_clmul_mulred(unsigned pos, __m128i prod) {
+	// this multiplies a 64-bit `prod` with a 32-bit CRC power
+	// compared with crc32_multiply_clmul, this only reduces the result to 64-bit, saving a multiply
+	__m128i coeff = _mm_cvtsi32_si128(crc_power_rev[pos]);
+	
+	const __m128i fold_const = _mm_set_epi32(0, 0x490d678d, 0, 0xf200aa66);
+	prod = _mm_clmulepi64_si128(prod, coeff, 0);
+	__m128i hi = _mm_clmulepi64_si128(prod, fold_const, 0x11);
+	return _mm_xor_si128(hi, prod);
+}
+
+uint32_t crc32_shift_clmul(uint32_t crc1, uint32_t n) {
+	if(!n) return crc1;
+	
+	__m128i result = _mm_cvtsi32_si128(BSWAP32(crc1));
+	result = reverse_bits_epi8(result);
+	
+	// handle n < 32 with a shift
+	result = _mm_sll_epi64(result, _mm_cvtsi32_si128(n & 31));
+	n &= ~31;
+	
+	__m128i t;
+	if(n) {
+		// use a second accumulator to leverage some IPC from slow CLMUL
+		__m128i result2 = _mm_cvtsi32_si128(crc_power_rev[ctz32(n)]);
+		n &= n-1;
+		
+		if(n) {
+			// first multiply doesn't need reduction
+			result2 = _mm_clmulepi64_si128(result2, _mm_cvtsi32_si128(crc_power_rev[ctz32(n)]), 0);
+			n &= n-1;
+			
+			while(n) {
+				result = crc32_shift_clmul_mulred(ctz32(n), result);
+				n &= n-1;
+				
+				if(n) {
+					result2 = crc32_shift_clmul_mulred(ctz32(n), result2);
+					n &= n-1;
+				}
+			}
+		}
+		
+		const __m128i fold_const = _mm_set_epi32(0, 0x490d678d, 0, 0xf200aa66);
+		
+		// merge two results
+		result = _mm_clmulepi64_si128(result, result2, 0);
+		
+		// do 128b reduction
+		t = _mm_unpackhi_epi32(result, _mm_setzero_si128());
+		// fold [127:96] -> [63:0]
+		__m128i hi = _mm_clmulepi64_si128(t, fold_const, 1);
+		// fold [95:64] -> [63:0]
+		__m128i lo = _mm_clmulepi64_si128(t, fold_const, 0x10);
+#ifdef ENABLE_AVX512
+		result = _mm_ternarylogic_epi32(result, hi, lo, 0x96);
+#else
+		result = _mm_xor_si128(result, hi);
+		result = _mm_xor_si128(result, lo);
+#endif
+	}
+	
+	// do Barrett reduction back into 32-bit field
+	const __m128i reduction_const = _mm_set_epi32(0, 0x04c11db7, 1, 0x04d101df);
+	t = _mm_clmulepi64_si128(_mm_blend_epi16(_mm_setzero_si128(), result, 0x3c), reduction_const, 0);
+	t = _mm_clmulepi64_si128(t, reduction_const, 0x11);
+	result = _mm_xor_si128(t, result);
+	
+	result = reverse_bits_epi8(result);
+	return BSWAP32(_mm_cvtsi128_si32(result));
+}
+#endif
+
+
+void crc_clmul_set_funcs() {
+	_do_crc32_incremental = &do_crc32_incremental_clmul;
+	_crc32_multiply = &crc32_multiply_clmul;
+#if defined(__GNUC__) || defined(_MSC_VER)
+	_crc32_shift = &crc32_shift_clmul;
+#endif
+	_crc32_isa = ISA_LEVEL_PCLMUL;
+}
+#else
+void crc_clmul_set_funcs() {}
 #endif