yencode 1.1.5 → 1.2.0

package/src/crc_riscv.cc

@@ -0,0 +1,251 @@
+#include "crc_common.h"
+
+#if defined(__riscv) && defined(__GNUC__) && (defined(__riscv_zbkc) || defined(__riscv_zbc))
+
+#if __has_include(<riscv_bitmanip.h>)
+# include <riscv_bitmanip.h>
+# if __riscv_xlen == 64
+#  define rv_clmul __riscv_clmul_64
+#  define rv_clmulh __riscv_clmulh_64
+# else
+#  define rv_clmul __riscv_clmul_32
+#  define rv_clmulh __riscv_clmulh_32
+# endif
+#else
+static HEDLEY_ALWAYS_INLINE uintptr_t rv_clmul(uintptr_t x, uintptr_t y) {
+	uintptr_t r;
+	__asm__("clmul %0, %1, %2\n"
+		: "=r"(r)
+		: "r"(x), "r"(y)
+		:);
+	return r;
+}
+static HEDLEY_ALWAYS_INLINE uintptr_t rv_clmulh(uintptr_t x, uintptr_t y) {
+	uintptr_t r;
+	__asm__("clmulh %0, %1, %2\n"
+		: "=r"(r)
+		: "r"(x), "r"(y)
+		:);
+	return r;
+}
+#endif
+
+// TODO: test big-endian
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+# if __riscv_xlen == 64
+#  define SWAP __builtin_bswap64
+# else
+#  define SWAP __builtin_bswap32
+# endif
+#else
+# define SWAP(d) (d)
+#endif
+static HEDLEY_ALWAYS_INLINE uintptr_t read_partial(const void* p, unsigned sz) {
+	uintptr_t data = 0;
+	memcpy(&data, p, sz);
+	return SWAP(data);
+}
+static HEDLEY_ALWAYS_INLINE uintptr_t read_full(const uintptr_t* p) {
+	return SWAP(*p);
+}
+#undef SWAP
+
+static uint32_t rv_crc_calc(uint32_t crc, const unsigned char *src, long len) {
+	uintptr_t accum[4] = {};
+	
+	// note: constants here are bit-reflected and shifted left by 1
+	// Zbc does also have clmulr to avoid the shift, but:
+	// - there's no clmulhr, so for XLEN=64, just shift the constant instead to get the same result
+	// - it's unavailable in Zbkc
+	// - for XLEN=32, 2x constants is likely worth it to avoid the additional XORs in the loop
+	
+#if __riscv_xlen == 64
+	const uint64_t MUL_HI = 0x15a546366 /*2^224*/, MUL_LO = 0xf1da05aa /*2^288*/;
+	#define CLMULL rv_clmul
+	#define CLMULH rv_clmulh
+	
+	accum[3] = rv_clmul(crc, 0xb66b1fa6); // 2^-32
+#elif __riscv_xlen == 32
+	const uint64_t MUL_HI = 0x140d44a2e /*2^128*/,  MUL_LO = 0x1751997d0 /*2^160*/;
+	#define CLMULL(x, k) rv_clmul(x, k & 0xffffffff)
+	#define CLMULH(x, k) (rv_clmulh(x, k & 0xffffffff) ^ (k > 0xffffffffULL ? (x) : 0))
+	
+	accum[2] = rv_clmul(crc, 0xb66b1fa6);
+	accum[3] = rv_clmulh(crc, 0xb66b1fa6);
+#else
+	#error "Unknown __riscv_xlen"
+#endif
+	const size_t WS = sizeof(uintptr_t);
+	
+	// if src isn't word-aligned, process until it is so
+	long initial_alignment = ((uintptr_t)src & (WS-1));
+	long initial_process = WS - initial_alignment;
+	if(initial_alignment && len >= initial_process) {
+		unsigned shl = initial_alignment * 8, shr = initial_process * 8;
+#if __riscv_xlen == 64
+		accum[2] = accum[3] << shl;
+#else
+		accum[1] = accum[2] << shl;
+		accum[2] = (accum[3] << shl) | (accum[2] >> shr);
+#endif
+		accum[3] = (read_partial(src, initial_process) << shl) | (accum[3] >> shr);
+		src += initial_process;
+		len -= initial_process;
+	}
+	
+	// main processing loop
+	const uintptr_t* srcW = (const uintptr_t*)src;
+	while((len -= WS*4) >= 0) {
+		uintptr_t tmpHi, tmpLo;
+		tmpLo = CLMULL(accum[0], MUL_LO) ^ CLMULL(accum[1], MUL_HI);
+		tmpHi = CLMULH(accum[0], MUL_LO) ^ CLMULH(accum[1], MUL_HI);
+		accum[0] = tmpLo ^ read_full(srcW++);
+		accum[1] = tmpHi ^ read_full(srcW++);
+		
+		tmpLo = CLMULL(accum[2], MUL_LO) ^ CLMULL(accum[3], MUL_HI);
+		tmpHi = CLMULH(accum[2], MUL_LO) ^ CLMULH(accum[3], MUL_HI);
+		accum[2] = tmpLo ^ read_full(srcW++);
+		accum[3] = tmpHi ^ read_full(srcW++);
+	}
+	
+	// process trailing bytes
+	if(len & (WS*2)) {
+		uintptr_t tmpLo = CLMULL(accum[0], MUL_LO) ^ CLMULL(accum[1], MUL_HI);
+		uintptr_t tmpHi = CLMULH(accum[0], MUL_LO) ^ CLMULH(accum[1], MUL_HI);
+		accum[0] = accum[2];
+		accum[1] = accum[3];
+		accum[2] = tmpLo ^ read_full(srcW++);
+		accum[3] = tmpHi ^ read_full(srcW++);
+	}
+	if(len & WS) {
+		uintptr_t tmpLo = CLMULL(accum[0], MUL_HI);
+		uintptr_t tmpHi = CLMULH(accum[0], MUL_HI);
+		accum[0] = accum[1];
+		accum[1] = accum[2];
+		accum[2] = accum[3] ^ tmpLo;
+		accum[3] = tmpHi ^ read_full(srcW++);
+	}
+	
+	size_t tail = len & (WS-1);
+	if(tail) {
+		unsigned shl = ((WS - tail) * 8), shr = tail * 8;
+		uintptr_t tmp = accum[0] << shl;
+		uintptr_t tmpLo = CLMULL(tmp, MUL_HI);
+		uintptr_t tmpHi = CLMULH(tmp, MUL_HI);
+		accum[0] = (accum[0] >> shr) | (accum[1] << shl);
+		accum[1] = (accum[1] >> shr) | (accum[2] << shl);
+		accum[2] = (accum[2] >> shr) | (accum[3] << shl);
+		accum[3] = (accum[3] >> shr) | (read_partial(srcW, tail) << shl);
+		accum[2] ^= tmpLo;
+		accum[3] ^= tmpHi;
+	}
+	
+	
+	// done processing: fold everything down
+#if __riscv_xlen == 64
+	// fold 0,1 -> 2,3
+	accum[2] ^= rv_clmul(accum[0], 0x1751997d0) ^ rv_clmul(accum[1], 0xccaa009e);
+	accum[3] ^= rv_clmulh(accum[0], 0x1751997d0) ^ rv_clmulh(accum[1], 0xccaa009e);
+	
+	// fold 2->3
+	accum[0] = rv_clmulh(accum[2], 0xccaa009e);
+	accum[3] ^= rv_clmul(accum[2], 0xccaa009e);
+	
+	// fold 64b->32b
+	accum[1] = rv_clmul(accum[3] & 0xffffffff, 0x163cd6124);
+	accum[0] ^= accum[1] >> 32;
+	accum[3] = accum[1] ^ (accum[3] >> 32);
+	accum[3] <<= 32;
+#else
+	// fold 0,1 -> 2,3
+	accum[2] ^= rv_clmul(accum[0], 0xccaa009e) ^ CLMULL(accum[1], 0x163cd6124);
+	accum[3] ^= rv_clmulh(accum[0], 0xccaa009e) ^ CLMULH(accum[1], 0x163cd6124);
+	
+	// fold 2->3
+	accum[0] = CLMULH(accum[2], 0x163cd6124);
+	accum[3] ^= CLMULL(accum[2], 0x163cd6124);
+#endif
+	
+	// reduction
+	accum[3] = CLMULL(accum[3], 0xf7011641);
+	accum[3] = CLMULH(accum[3], 0x1db710640);  // maybe consider clmulr for XLEN=32
+	crc = accum[0] ^ accum[3];
+	return crc;
+	#undef CLMULL
+	#undef CLMULH
+}
+
+static uint32_t do_crc32_incremental_rv_zbc(const void* data, size_t length, uint32_t init) {
+	return ~rv_crc_calc(~init, (const unsigned char*)data, (long)length);
+}
+
+
+#if __riscv_xlen == 64
+	// note that prod is shifted by 1 place to the right, due to bit-reflection
+static uint32_t crc32_reduce_rv_zbc(uint64_t prod) {
+	uint64_t t = rv_clmul(prod << 33, 0xf7011641);
+	t = rv_clmulh(t, 0x1db710640);
+	t ^= prod >> 31;
+	return t;
+}
+#endif
+uint32_t crc32_multiply_rv_zbc(uint32_t a, uint32_t b) {
+#if __riscv_xlen == 64
+	uint64_t t = crc32_reduce_rv_zbc(rv_clmul(a, b));
+#else
+	uint32_t prodLo = rv_clmul(a, b);
+	uint32_t prodHi = rv_clmulh(a, b);
+	
+	// fix prodHi for bit-reflection (clmulr would be ideal here)
+	prodHi += prodHi;
+	prodHi |= prodLo >> 31;
+	prodLo += prodLo;
+	
+	uint32_t t = rv_clmul(prodLo, 0xf7011641);
+	t ^= rv_clmulh(t, 0xdb710640);
+	t ^= prodHi;
+#endif
+	return t;
+}
+
+#if defined(__GNUC__) || defined(_MSC_VER)
+uint32_t crc32_shift_rv_zbc(uint32_t crc1, uint32_t n) {
+	// TODO: require Zbb for ctz
+	uint32_t result = crc1;
+#if __riscv_xlen == 64
+	// for n<32, can shift directly
+	uint64_t prod = result;
+	prod <<= 31 ^ (n&31);
+	n &= ~31;
+	result = crc32_reduce_rv_zbc(prod);
+#endif
+	if(!n) return result;
+	
+	uint32_t result2 = crc_power[ctz32(n)];
+	n &= n-1;
+	
+	while(n) {
+		result = crc32_multiply_rv_zbc(result, crc_power[ctz32(n)]);
+		n &= n-1;
+		
+		if(n) {
+			result2 = crc32_multiply_rv_zbc(result2, crc_power[ctz32(n)]);
+			n &= n-1;
+		}
+	}
+	return crc32_multiply_rv_zbc(result, result2);
+}
+#endif
+
+
+void crc_riscv_set_funcs() {
+	_do_crc32_incremental = &do_crc32_incremental_rv_zbc;
+	_crc32_multiply = &crc32_multiply_rv_zbc;
+#if defined(__GNUC__) || defined(_MSC_VER)
+	_crc32_shift = &crc32_shift_rv_zbc;
+#endif
+	_crc32_isa = ISA_FEATURE_ZBC;
+}
+#else
+void crc_riscv_set_funcs() {}
+#endif

package/src/decoder.cc

@@ -18,6 +18,7 @@ void decoder_set_avx2_funcs();
 void decoder_set_vbmi2_funcs();
 extern const bool decoder_has_avx10;
 void decoder_set_neon_funcs();
+void decoder_set_rvv_funcs();
 
 
 #if defined(PLATFORM_X86) && defined(YENC_BUILD_NATIVE) && YENC_BUILD_NATIVE!=0
@@ -25,7 +26,7 @@ void decoder_set_neon_funcs();
 #  include "decoder_avx2_base.h"
 static inline void decoder_set_native_funcs() {
 	ALIGN_ALLOC(lookups, sizeof(*lookups), 16);
-	decoder_init_lut(lookups->eqFix, lookups->compact);
+	decoder_init_lut(lookups->compact);
 	_do_decode = &do_decode_simd<false, false, sizeof(__m256i)*2, do_decode_avx2<false, false, ISA_NATIVE> >;
 	_do_decode_raw = &do_decode_simd<true, false, sizeof(__m256i)*2, do_decode_avx2<true, false, ISA_NATIVE> >;
 	_do_decode_end_raw = &do_decode_simd<true, true, sizeof(__m256i)*2, do_decode_avx2<true, true, ISA_NATIVE> >;
@@ -34,8 +35,8 @@ static inline void decoder_set_native_funcs() {
 # else
 #  include "decoder_sse_base.h"
 static inline void decoder_set_native_funcs() {
-	decoder_sse_init();
-	decoder_init_lut(lookups->eqFix, lookups->compact);
+	decoder_sse_init(lookups);
+	decoder_init_lut(lookups->compact);
 	_do_decode = &do_decode_simd<false, false, sizeof(__m128i)*2, do_decode_sse<false, false, ISA_NATIVE> >;
 	_do_decode_raw = &do_decode_simd<true, false, sizeof(__m128i)*2, do_decode_sse<true, false, ISA_NATIVE> >;
 	_do_decode_end_raw = &do_decode_simd<true, true, sizeof(__m128i)*2, do_decode_sse<true, true, ISA_NATIVE> >;
@@ -44,6 +45,31 @@ static inline void decoder_set_native_funcs() {
 # endif
 #endif
 
+
+#if defined(PLATFORM_X86) || defined(PLATFORM_ARM)
+void decoder_init_lut(void* compactLUT) {
+	#ifdef YENC_DEC_USE_THINTABLE
+	const int tableSize = 8;
+	#else
+	const int tableSize = 16;
+	#endif
+	for(int i=0; i<(tableSize==8?256:32768); i++) {
+		int k = i;
+		uint8_t* res = (uint8_t*)compactLUT + i*tableSize;
+		int p = 0;
+		for(int j=0; j<tableSize; j++) {
+			if(!(k & 1)) {
+				res[p++] = j;
+			}
+			k >>= 1;
+		}
+		for(; p<tableSize; p++)
+			res[p] = 0x80;
+	}
+}
+#endif
+
+
 void decoder_init() {
 #ifdef PLATFORM_X86
 # if defined(YENC_BUILD_NATIVE) && YENC_BUILD_NATIVE!=0
@@ -66,4 +92,8 @@ void decoder_init() {
 	if(cpu_supports_neon())
 		decoder_set_neon_funcs();
 #endif
+#ifdef __riscv
+	if(cpu_supports_rvv())
+		decoder_set_rvv_funcs();
+#endif
 }

package/src/decoder_avx.cc

@@ -4,8 +4,8 @@
 #include "decoder_common.h"
 #include "decoder_sse_base.h"
 void decoder_set_avx_funcs() {
-	decoder_sse_init();
-	decoder_init_lut(lookups->eqFix, lookups->compact);
+	decoder_sse_init(lookups);
+	decoder_init_lut(lookups->compact);
 	_do_decode = &do_decode_simd<false, false, sizeof(__m128i)*2, do_decode_sse<false, false, ISA_LEVEL_SSE4_POPCNT> >;
 	_do_decode_raw = &do_decode_simd<true, false, sizeof(__m128i)*2, do_decode_sse<true, false, ISA_LEVEL_SSE4_POPCNT> >;
 	_do_decode_end_raw = &do_decode_simd<true, true, sizeof(__m128i)*2, do_decode_sse<true, true, ISA_LEVEL_SSE4_POPCNT> >;

package/src/decoder_avx2.cc

@@ -5,7 +5,7 @@
 #include "decoder_avx2_base.h"
 void decoder_set_avx2_funcs() {
 	ALIGN_ALLOC(lookups, sizeof(*lookups), 16);
-	decoder_init_lut(lookups->eqFix, lookups->compact);
+	decoder_init_lut(lookups->compact);
 	_do_decode = &do_decode_simd<false, false, sizeof(__m256i)*2, do_decode_avx2<false, false, ISA_LEVEL_AVX2> >;
 	_do_decode_raw = &do_decode_simd<true, false, sizeof(__m256i)*2, do_decode_avx2<true, false, ISA_LEVEL_AVX2> >;
 	_do_decode_end_raw = &do_decode_simd<true, true, sizeof(__m256i)*2, do_decode_avx2<true, true, ISA_LEVEL_AVX2> >;

package/src/decoder_avx2_base.h

@@ -15,7 +15,6 @@
 #pragma pack(16)
 static struct {
 	/*align16*/ struct { char bytes[16]; } compact[32768];
-	uint8_t eqFix[256];
 } * HEDLEY_RESTRICT lookups;
 #pragma pack()
 
@@ -431,16 +430,9 @@ HEDLEY_ALWAYS_INLINE void do_decode_avx2(const uint8_t* src, long& len, unsigned
 				dataB = _mm256_add_epi8(oDataB, _mm256_set1_epi8(-42));
 			
 			if(LIKELIHOOD(0.0001, (mask & ((maskEq << 1) + escFirst)) != 0)) {
-				unsigned tmp = lookups->eqFix[(maskEq&0xff) & ~(uint64_t)escFirst];
-				uint64_t maskEq2 = tmp;
-				for(int j=8; j<64; j+=8) {
-					tmp = lookups->eqFix[(unsigned)((maskEq>>j)&0xff) & ~(tmp>>7)];
-					maskEq2 |= (uint64_t)tmp<<j;
-				}
-				maskEq = maskEq2;
-				
+				maskEq = fix_eqMask<uint64_t>(maskEq & ~(uint64_t)escFirst);
 				mask &= ~(uint64_t)escFirst;
-				escFirst = tmp>>7;
+				escFirst = maskEq>>63;
 				// next, eliminate anything following a `=` from the special char mask; this eliminates cases of `=\r` so that they aren't removed
 				maskEq <<= 1;
 				mask &= ~maskEq;

package/src/decoder_common.h

@@ -1,5 +1,9 @@
 #include "decoder.h"
 
+#if defined(PLATFORM_ARM) && !defined(__aarch64__)
+#define YENC_DEC_USE_THINTABLE 1
+#endif
+
 // TODO: need to support max output length somehow
 // TODO: add branch probabilities
 
@@ -178,24 +182,24 @@ YencDecoderEnd do_decode_end_scalar(const unsigned char** src, unsigned char** d
 			if(es[i] == '.' && isRaw) {
 				i++;
 				YDEC_CHECK_END(YDEC_STATE_CRLFDT)
-				// fall-through
 			} else if(es[i] == '=') {
 				i++;
 				YDEC_CHECK_END(YDEC_STATE_CRLFEQ)
 				goto do_decode_endable_scalar_ceq;
 			} else
 				break;
+			// fall-through
 		case YDEC_STATE_CRLFDT:
 			if(isRaw && es[i] == '\r') {
 				i++;
 				YDEC_CHECK_END(YDEC_STATE_CRLFDTCR)
-				// fall-through
 			} else if(isRaw && es[i] == '=') { // check for dot-stuffed ending: \r\n.=y
 				i++;
 				YDEC_CHECK_END(YDEC_STATE_CRLFEQ)
 				goto do_decode_endable_scalar_ceq;
 			} else
 				break;
+			// fall-through
 		case YDEC_STATE_CRLFDTCR:
 			if(es[i] == '\n') {
 				if(isRaw) {
@@ -331,8 +335,8 @@ YencDecoderEnd do_decode_scalar(const unsigned char** src, unsigned char** dest,
 
 
 
-template<bool isRaw, bool searchEnd, int width, void(&kernel)(const uint8_t*, long&, unsigned char*&, unsigned char&, uint16_t&)>
-YencDecoderEnd do_decode_simd(const unsigned char** src, unsigned char** dest, size_t len, YencDecoderState* state) {
+template<bool isRaw, bool searchEnd, void(&kernel)(const uint8_t*, long&, unsigned char*&, unsigned char&, uint16_t&)>
+inline YencDecoderEnd _do_decode_simd(size_t width, const unsigned char** src, unsigned char** dest, size_t len, YencDecoderState* state) {
 	if(len <= width*2) return do_decode_scalar<isRaw, searchEnd>(src, dest, len, state);
 	
 	YencDecoderState tState = YDEC_STATE_CRLF;
@@ -461,54 +465,20 @@ YencDecoderEnd do_decode_simd(const unsigned char** src, unsigned char** dest, s
 	return YDEC_END_NONE;
 }
 
-static inline void decoder_init_lut(uint8_t* eqFixLUT, void* compactLUT) {
-	for(int i=0; i<256; i++) {
-		int k = i;
-		int p = 0;
-		
-		// fix LUT
-		k = i;
-		p = 0;
-		for(int j=0; j<8; j++) {
-			k = i >> j;
-			if(k & 1) {
-				p |= 1 << j;
-				j++;
-			}
-		}
-		eqFixLUT[i] = p;
-		
-		#ifdef YENC_DEC_USE_THINTABLE
-		uint8_t* res = (uint8_t*)compactLUT + i*8;
-		k = i;
-		p = 0;
-		for(int j=0; j<8; j++) {
-			if(!(k & 1)) {
-				res[p++] = j;
-			}
-			k >>= 1;
-		}
-		for(; p<8; p++)
-			res[p] = 0x80;
-		#endif
-	}
-	#ifndef YENC_DEC_USE_THINTABLE
-	for(int i=0; i<32768; i++) {
-		int k = i;
-		uint8_t* res = (uint8_t*)compactLUT + i*16;
-		int p = 0;
-		
-		for(int j=0; j<16; j++) {
-			if(!(k & 1)) {
-				res[p++] = j;
-			}
-			k >>= 1;
-		}
-		for(; p<16; p++)
-			res[p] = 0x80;
-	}
-	#endif
+template<bool isRaw, bool searchEnd, size_t width, void(&kernel)(const uint8_t*, long&, unsigned char*&, unsigned char&, uint16_t&)>
+YencDecoderEnd do_decode_simd(const unsigned char** src, unsigned char** dest, size_t len, YencDecoderState* state) {
+	return _do_decode_simd<isRaw, searchEnd, kernel>(width, src, dest, len, state);
 }
+template<bool isRaw, bool searchEnd, size_t(&getWidth)(), void(&kernel)(const uint8_t*, long&, unsigned char*&, unsigned char&, uint16_t&)>
+YencDecoderEnd do_decode_simd(const unsigned char** src, unsigned char** dest, size_t len, YencDecoderState* state) {
+	return _do_decode_simd<isRaw, searchEnd, kernel>(getWidth(), src, dest, len, state);
+}
+
+
+#if defined(PLATFORM_X86) || defined(PLATFORM_ARM)
+void decoder_init_lut(void* compactLUT);
+#endif
+
 template<bool isRaw>
 static inline void decoder_set_nextMask(const uint8_t* src, size_t len, uint16_t& nextMask) {
 	if(isRaw) {
@@ -535,3 +505,20 @@ static inline uint16_t decoder_set_nextMask(const uint8_t* src, unsigned mask) {
 	}
 	return 0;
 }
+
+// resolve invalid sequences of = to deal with cases like '===='
+// bit hack inspired from simdjson: https://youtu.be/wlvKAT7SZIQ?t=33m38s
+template<typename T>
+static inline T fix_eqMask(T mask) {
+	// isolate the start of each consecutive bit group (e.g. 01011101 -> 01000101)
+	T start = mask & ~(mask << 1);
+	
+	const T odd = (T)0xaaaaaaaaaaaaaaaa; // every odd bit (10101010...)
+	
+	// obtain groups which start on an even bit (clear groups that start on an odd bit, but this leaves an unwanted trailing bit)
+	T evenGroups = mask + (start & odd);
+	
+	// clear odd bits in even groups, whilst conversely preserving odd bits in odd groups
+	// the `& mask` also conveniently gets rid of unwanted trailing bits
+	return (evenGroups ^ odd) & mask;
+}

package/src/decoder_neon.cc

@@ -1,9 +1,6 @@
 #include "common.h"
 #ifdef __ARM_NEON
 
-#ifndef __aarch64__
-#define YENC_DEC_USE_THINTABLE 1
-#endif
 #include "decoder_common.h"
 
 
@@ -43,8 +40,6 @@ static struct { char bytes[16]; } ALIGN_TO(16, compactLUT[32768]);
 # pragma pack()
 #endif
 
-static uint8_t eqFixLUT[256];
-
 
 
 static bool neon_vect_is_nonzero(uint8x16_t v) {
@@ -328,18 +323,13 @@ HEDLEY_ALWAYS_INLINE void do_decode_neon(const uint8_t* src, long& len, unsigned
 			// the yEnc specification requires any character following = to be unescaped, not skipped over, so we'll deal with that
 			// firstly, check for invalid sequences of = (we assume that these are rare, as a spec compliant yEnc encoder should not generate these)
 			if(LIKELIHOOD(0.0001, (mask & ((maskEq << 1) | escFirst)) != 0)) {
-				uint8_t tmp = eqFixLUT[(maskEq&0xff) & ~escFirst];
-				uint32_t maskEq2 = tmp;
-				for(int j=8; j<32; j+=8) {
-					tmp = eqFixLUT[((maskEq>>j)&0xff) & ~(tmp>>7)];
-					maskEq2 |= tmp<<j;
-				}
-				maskEq = maskEq2;
+				maskEq = fix_eqMask<uint32_t>(maskEq & ~escFirst);
 				
+				unsigned char nextEscFirst = maskEq>>31;
 				// next, eliminate anything following a `=` from the special char mask; this eliminates cases of `=\r` so that they aren't removed
 				maskEq = (maskEq<<1) | escFirst;
 				mask &= ~maskEq;
-				escFirst = tmp>>7;
+				escFirst = nextEscFirst;
 				
 				// unescape chars following `=`
 				uint8x8_t maskEqTemp = vreinterpret_u8_u32(vmov_n_u32(maskEq));
@@ -457,7 +447,7 @@ HEDLEY_ALWAYS_INLINE void do_decode_neon(const uint8_t* src, long& len, unsigned
 }
 
 void decoder_set_neon_funcs() {
-	decoder_init_lut(eqFixLUT, compactLUT);
+	decoder_init_lut(compactLUT);
 	_do_decode = &do_decode_simd<false, false, sizeof(uint8x16_t)*2, do_decode_neon<false, false> >;
 	_do_decode_raw = &do_decode_simd<true, false, sizeof(uint8x16_t)*2, do_decode_neon<true, false> >;
 	_do_decode_end_raw = &do_decode_simd<true, true, sizeof(uint8x16_t)*2, do_decode_neon<true, true> >;

package/src/decoder_neon64.cc

@@ -7,8 +7,6 @@
 static struct { char bytes[16]; } ALIGN_TO(16, compactLUT[32768]);
 #pragma pack()
 
-static uint8_t eqFixLUT[256];
-
 
 // AArch64 GCC lacks these functions until 8.5, 9.4 and 10.1 (10.0 unknown)
 #if !defined(__clang__) && !defined(_MSC_VER) && (!defined(__aarch64__) || !(HEDLEY_GCC_VERSION_CHECK(9,4,0) || (!HEDLEY_GCC_VERSION_CHECK(9,0,0) && HEDLEY_GCC_VERSION_CHECK(8,5,0))))
@@ -293,18 +291,13 @@ HEDLEY_ALWAYS_INLINE void do_decode_neon(const uint8_t* src, long& len, unsigned
 			// the yEnc specification requires any character following = to be unescaped, not skipped over, so we'll deal with that
 			// firstly, check for invalid sequences of = (we assume that these are rare, as a spec compliant yEnc encoder should not generate these)
 			if(LIKELIHOOD(0.0001, (mask & ((maskEq << 1) | escFirst)) != 0)) {
-				uint8_t tmp = eqFixLUT[(maskEq&0xff) & ~escFirst];
-				uint64_t maskEq2 = tmp;
-				for(int j=8; j<64; j+=8) {
-					tmp = eqFixLUT[((maskEq>>j)&0xff) & ~(tmp>>7)];
-					maskEq2 |= ((uint64_t)tmp)<<j;
-				}
-				maskEq = maskEq2;
+				maskEq = fix_eqMask<uint64_t>(maskEq & ~(uint64_t)escFirst);
 				
+				unsigned char nextEscFirst = maskEq>>63;
 				// next, eliminate anything following a `=` from the special char mask; this eliminates cases of `=\r` so that they aren't removed
 				maskEq = (maskEq<<1) | escFirst;
 				mask &= ~maskEq;
-				escFirst = tmp>>7;
+				escFirst = nextEscFirst;
 				
 				// unescape chars following `=`
 #if defined(__GNUC__) && !defined(__clang__)
@@ -438,7 +431,7 @@ HEDLEY_ALWAYS_INLINE void do_decode_neon(const uint8_t* src, long& len, unsigned
 }
 
 void decoder_set_neon_funcs() {
-	decoder_init_lut(eqFixLUT, compactLUT);
+	decoder_init_lut(compactLUT);
 	_do_decode = &do_decode_simd<false, false, sizeof(uint8x16_t)*4, do_decode_neon<false, false> >;
 	_do_decode_raw = &do_decode_simd<true, false, sizeof(uint8x16_t)*4, do_decode_neon<true, false> >;
 	_do_decode_end_raw = &do_decode_simd<true, true, sizeof(uint8x16_t)*4, do_decode_neon<true, true> >;