yencode 1.1.4 → 1.2.0

package/src/crc_folding_256.cc

@@ -1,6 +1,8 @@
 // 256-bit version of crc_folding
 
 #include "crc_common.h"
+
+void crc_clmul_set_funcs();
  
 #if !defined(YENC_DISABLE_AVX256) && ((defined(__VPCLMULQDQ__) && defined(__AVX2__) && defined(__PCLMUL__)) || (defined(_MSC_VER) && _MSC_VER >= 1920 && defined(PLATFORM_X86) && !defined(__clang__)))
 #include <inttypes.h>
@@ -99,19 +101,12 @@ ALIGN_TO(16, static const unsigned crc_k[]) = {
 
 
 static uint32_t crc_fold(const unsigned char *src, long len, uint32_t initial) {
-	// info from https://www.reddit.com/r/ReverseEngineering/comments/2zwhl3/mystery_constant_0x9db42487_in_intels_crc32ieee/
-	// firstly, calculate: xmm_crc0 = (intial * 0x487b9c8a) mod 0x104c11db7, where 0x487b9c8a = inverse(1<<512) mod 0x104c11db7
-	__m128i xmm_t0 = _mm_cvtsi32_si128(~initial);
-	
-	xmm_t0 = _mm_clmulepi64_si128(xmm_t0, _mm_set_epi32(0, 0, 0xa273bc24, 0), 0);  // reverse(0x487b9c8a)<<1 == 0xa273bc24
-	__m128i reduction = _mm_set_epi32( // polynomial reduction factors
-	  1, 0xdb710640, // G* = 0x04c11db7
-	  0, 0xf7011641  // Q+ = 0x04d101df  (+1 to save an additional xor operation)
+	__m128i xmm_t0 = _mm_clmulepi64_si128(
+		_mm_cvtsi32_si128(~initial),
+		_mm_cvtsi32_si128(0xdfded7ec),
+		0
 	);
-	__m128i xmm_t1 = _mm_clmulepi64_si128(xmm_t0, reduction, 0);
-	xmm_t1 = _mm_clmulepi64_si128(xmm_t1, reduction, 0x10);
 	
-	xmm_t0 = _mm_srli_si128(_mm_xor_si128(xmm_t0, xmm_t1), 8);
 	__m256i crc0 = zext128_256(xmm_t0);
 	__m256i crc1 = _mm256_setzero_si256();
 	
@@ -217,13 +212,14 @@ static uint32_t do_crc32_incremental_clmul(const void* data, size_t length, uint
 	return crc_fold((const unsigned char*)data, (long)length, init);
 }
 
-void crc_clmul256_set_funcs(crc_func* _do_crc32_incremental) {
-	*_do_crc32_incremental = &do_crc32_incremental_clmul;
+void crc_clmul256_set_funcs() {
+	crc_clmul_set_funcs(); // set multiply/shift function
+	_do_crc32_incremental = &do_crc32_incremental_clmul;
+	_crc32_isa = ISA_LEVEL_VPCLMUL;
 }
 #else
-void crc_clmul_set_funcs(crc_func* _do_crc32_incremental);
-void crc_clmul256_set_funcs(crc_func* _do_crc32_incremental) {
-	crc_clmul_set_funcs(_do_crc32_incremental);
+void crc_clmul256_set_funcs() {
+	crc_clmul_set_funcs();
 }
 #endif
 

package/src/crc_riscv.cc

@@ -0,0 +1,251 @@
+#include "crc_common.h"
+
+#if defined(__riscv) && defined(__GNUC__) && (defined(__riscv_zbkc) || defined(__riscv_zbc))
+
+#if __has_include(<riscv_bitmanip.h>)
+# include <riscv_bitmanip.h>
+# if __riscv_xlen == 64
+#  define rv_clmul __riscv_clmul_64
+#  define rv_clmulh __riscv_clmulh_64
+# else
+#  define rv_clmul __riscv_clmul_32
+#  define rv_clmulh __riscv_clmulh_32
+# endif
+#else
+static HEDLEY_ALWAYS_INLINE uintptr_t rv_clmul(uintptr_t x, uintptr_t y) {
+	uintptr_t r;
+	__asm__("clmul %0, %1, %2\n"
+		: "=r"(r)
+		: "r"(x), "r"(y)
+		:);
+	return r;
+}
+static HEDLEY_ALWAYS_INLINE uintptr_t rv_clmulh(uintptr_t x, uintptr_t y) {
+	uintptr_t r;
+	__asm__("clmulh %0, %1, %2\n"
+		: "=r"(r)
+		: "r"(x), "r"(y)
+		:);
+	return r;
+}
+#endif
+
+// TODO: test big-endian
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+# if __riscv_xlen == 64
+#  define SWAP __builtin_bswap64
+# else
+#  define SWAP __builtin_bswap32
+# endif
+#else
+# define SWAP(d) (d)
+#endif
+static HEDLEY_ALWAYS_INLINE uintptr_t read_partial(const void* p, unsigned sz) {
+	uintptr_t data = 0;
+	memcpy(&data, p, sz);
+	return SWAP(data);
+}
+static HEDLEY_ALWAYS_INLINE uintptr_t read_full(const uintptr_t* p) {
+	return SWAP(*p);
+}
+#undef SWAP
+
+static uint32_t rv_crc_calc(uint32_t crc, const unsigned char *src, long len) {
+	uintptr_t accum[4] = {};
+	
+	// note: constants here are bit-reflected and shifted left by 1
+	// Zbc does also have clmulr to avoid the shift, but:
+	// - there's no clmulhr, so for XLEN=64, just shift the constant instead to get the same result
+	// - it's unavailable in Zbkc
+	// - for XLEN=32, 2x constants is likely worth it to avoid the additional XORs in the loop
+	
+#if __riscv_xlen == 64
+	const uint64_t MUL_HI = 0x15a546366 /*2^224*/, MUL_LO = 0xf1da05aa /*2^288*/;
+	#define CLMULL rv_clmul
+	#define CLMULH rv_clmulh
+	
+	accum[3] = rv_clmul(crc, 0xb66b1fa6); // 2^-32
+#elif __riscv_xlen == 32
+	const uint64_t MUL_HI = 0x140d44a2e /*2^128*/,  MUL_LO = 0x1751997d0 /*2^160*/;
+	#define CLMULL(x, k) rv_clmul(x, k & 0xffffffff)
+	#define CLMULH(x, k) (rv_clmulh(x, k & 0xffffffff) ^ (k > 0xffffffffULL ? (x) : 0))
+	
+	accum[2] = rv_clmul(crc, 0xb66b1fa6);
+	accum[3] = rv_clmulh(crc, 0xb66b1fa6);
+#else
+	#error "Unknown __riscv_xlen"
+#endif
+	const size_t WS = sizeof(uintptr_t);
+	
+	// if src isn't word-aligned, process until it is so
+	long initial_alignment = ((uintptr_t)src & (WS-1));
+	long initial_process = WS - initial_alignment;
+	if(initial_alignment && len >= initial_process) {
+		unsigned shl = initial_alignment * 8, shr = initial_process * 8;
+#if __riscv_xlen == 64
+		accum[2] = accum[3] << shl;
+#else
+		accum[1] = accum[2] << shl;
+		accum[2] = (accum[3] << shl) | (accum[2] >> shr);
+#endif
+		accum[3] = (read_partial(src, initial_process) << shl) | (accum[3] >> shr);
+		src += initial_process;
+		len -= initial_process;
+	}
+	
+	// main processing loop
+	const uintptr_t* srcW = (const uintptr_t*)src;
+	while((len -= WS*4) >= 0) {
+		uintptr_t tmpHi, tmpLo;
+		tmpLo = CLMULL(accum[0], MUL_LO) ^ CLMULL(accum[1], MUL_HI);
+		tmpHi = CLMULH(accum[0], MUL_LO) ^ CLMULH(accum[1], MUL_HI);
+		accum[0] = tmpLo ^ read_full(srcW++);
+		accum[1] = tmpHi ^ read_full(srcW++);
+		
+		tmpLo = CLMULL(accum[2], MUL_LO) ^ CLMULL(accum[3], MUL_HI);
+		tmpHi = CLMULH(accum[2], MUL_LO) ^ CLMULH(accum[3], MUL_HI);
+		accum[2] = tmpLo ^ read_full(srcW++);
+		accum[3] = tmpHi ^ read_full(srcW++);
+	}
+	
+	// process trailing bytes
+	if(len & (WS*2)) {
+		uintptr_t tmpLo = CLMULL(accum[0], MUL_LO) ^ CLMULL(accum[1], MUL_HI);
+		uintptr_t tmpHi = CLMULH(accum[0], MUL_LO) ^ CLMULH(accum[1], MUL_HI);
+		accum[0] = accum[2];
+		accum[1] = accum[3];
+		accum[2] = tmpLo ^ read_full(srcW++);
+		accum[3] = tmpHi ^ read_full(srcW++);
+	}
+	if(len & WS) {
+		uintptr_t tmpLo = CLMULL(accum[0], MUL_HI);
+		uintptr_t tmpHi = CLMULH(accum[0], MUL_HI);
+		accum[0] = accum[1];
+		accum[1] = accum[2];
+		accum[2] = accum[3] ^ tmpLo;
+		accum[3] = tmpHi ^ read_full(srcW++);
+	}
+	
+	size_t tail = len & (WS-1);
+	if(tail) {
+		unsigned shl = ((WS - tail) * 8), shr = tail * 8;
+		uintptr_t tmp = accum[0] << shl;
+		uintptr_t tmpLo = CLMULL(tmp, MUL_HI);
+		uintptr_t tmpHi = CLMULH(tmp, MUL_HI);
+		accum[0] = (accum[0] >> shr) | (accum[1] << shl);
+		accum[1] = (accum[1] >> shr) | (accum[2] << shl);
+		accum[2] = (accum[2] >> shr) | (accum[3] << shl);
+		accum[3] = (accum[3] >> shr) | (read_partial(srcW, tail) << shl);
+		accum[2] ^= tmpLo;
+		accum[3] ^= tmpHi;
+	}
+	
+	
+	// done processing: fold everything down
+#if __riscv_xlen == 64
+	// fold 0,1 -> 2,3
+	accum[2] ^= rv_clmul(accum[0], 0x1751997d0) ^ rv_clmul(accum[1], 0xccaa009e);
+	accum[3] ^= rv_clmulh(accum[0], 0x1751997d0) ^ rv_clmulh(accum[1], 0xccaa009e);
+	
+	// fold 2->3
+	accum[0] = rv_clmulh(accum[2], 0xccaa009e);
+	accum[3] ^= rv_clmul(accum[2], 0xccaa009e);
+	
+	// fold 64b->32b
+	accum[1] = rv_clmul(accum[3] & 0xffffffff, 0x163cd6124);
+	accum[0] ^= accum[1] >> 32;
+	accum[3] = accum[1] ^ (accum[3] >> 32);
+	accum[3] <<= 32;
+#else
+	// fold 0,1 -> 2,3
+	accum[2] ^= rv_clmul(accum[0], 0xccaa009e) ^ CLMULL(accum[1], 0x163cd6124);
+	accum[3] ^= rv_clmulh(accum[0], 0xccaa009e) ^ CLMULH(accum[1], 0x163cd6124);
+	
+	// fold 2->3
+	accum[0] = CLMULH(accum[2], 0x163cd6124);
+	accum[3] ^= CLMULL(accum[2], 0x163cd6124);
+#endif
+	
+	// reduction
+	accum[3] = CLMULL(accum[3], 0xf7011641);
+	accum[3] = CLMULH(accum[3], 0x1db710640);  // maybe consider clmulr for XLEN=32
+	crc = accum[0] ^ accum[3];
+	return crc;
+	#undef CLMULL
+	#undef CLMULH
+}
+
+static uint32_t do_crc32_incremental_rv_zbc(const void* data, size_t length, uint32_t init) {
+	return ~rv_crc_calc(~init, (const unsigned char*)data, (long)length);
+}
+
+
+#if __riscv_xlen == 64
+	// note that prod is shifted by 1 place to the right, due to bit-reflection
+static uint32_t crc32_reduce_rv_zbc(uint64_t prod) {
+	uint64_t t = rv_clmul(prod << 33, 0xf7011641);
+	t = rv_clmulh(t, 0x1db710640);
+	t ^= prod >> 31;
+	return t;
+}
+#endif
+uint32_t crc32_multiply_rv_zbc(uint32_t a, uint32_t b) {
+#if __riscv_xlen == 64
+	uint64_t t = crc32_reduce_rv_zbc(rv_clmul(a, b));
+#else
+	uint32_t prodLo = rv_clmul(a, b);
+	uint32_t prodHi = rv_clmulh(a, b);
+	
+	// fix prodHi for bit-reflection (clmulr would be ideal here)
+	prodHi += prodHi;
+	prodHi |= prodLo >> 31;
+	prodLo += prodLo;
+	
+	uint32_t t = rv_clmul(prodLo, 0xf7011641);
+	t ^= rv_clmulh(t, 0xdb710640);
+	t ^= prodHi;
+#endif
+	return t;
+}
+
+#if defined(__GNUC__) || defined(_MSC_VER)
+uint32_t crc32_shift_rv_zbc(uint32_t crc1, uint32_t n) {
+	// TODO: require Zbb for ctz
+	uint32_t result = crc1;
+#if __riscv_xlen == 64
+	// for n<32, can shift directly
+	uint64_t prod = result;
+	prod <<= 31 ^ (n&31);
+	n &= ~31;
+	result = crc32_reduce_rv_zbc(prod);
+#endif
+	if(!n) return result;
+	
+	uint32_t result2 = crc_power[ctz32(n)];
+	n &= n-1;
+	
+	while(n) {
+		result = crc32_multiply_rv_zbc(result, crc_power[ctz32(n)]);
+		n &= n-1;
+		
+		if(n) {
+			result2 = crc32_multiply_rv_zbc(result2, crc_power[ctz32(n)]);
+			n &= n-1;
+		}
+	}
+	return crc32_multiply_rv_zbc(result, result2);
+}
+#endif
+
+
+void crc_riscv_set_funcs() {
+	_do_crc32_incremental = &do_crc32_incremental_rv_zbc;
+	_crc32_multiply = &crc32_multiply_rv_zbc;
+#if defined(__GNUC__) || defined(_MSC_VER)
+	_crc32_shift = &crc32_shift_rv_zbc;
+#endif
+	_crc32_isa = ISA_FEATURE_ZBC;
+}
+#else
+void crc_riscv_set_funcs() {}
+#endif

package/src/decoder.cc

@@ -7,6 +7,8 @@ extern "C" {
 	YencDecoderEnd (*_do_decode)(const unsigned char**, unsigned char**, size_t, YencDecoderState*) = &do_decode_scalar<false, false>;
 	YencDecoderEnd (*_do_decode_raw)(const unsigned char**, unsigned char**, size_t, YencDecoderState*) = &do_decode_scalar<true, false>;
 	YencDecoderEnd (*_do_decode_end_raw)(const unsigned char**, unsigned char**, size_t, YencDecoderState*) = &do_decode_end_scalar<true>;
+	
+	int _decode_isa = ISA_GENERIC;
 }
 
 void decoder_set_sse2_funcs();
@@ -16,6 +18,7 @@ void decoder_set_avx2_funcs();
 void decoder_set_vbmi2_funcs();
 extern const bool decoder_has_avx10;
 void decoder_set_neon_funcs();
+void decoder_set_rvv_funcs();
 
 
 #if defined(PLATFORM_X86) && defined(YENC_BUILD_NATIVE) && YENC_BUILD_NATIVE!=0
@@ -23,23 +26,50 @@ void decoder_set_neon_funcs();
 #  include "decoder_avx2_base.h"
 static inline void decoder_set_native_funcs() {
 	ALIGN_ALLOC(lookups, sizeof(*lookups), 16);
-	decoder_init_lut(lookups->eqFix, lookups->compact);
+	decoder_init_lut(lookups->compact);
 	_do_decode = &do_decode_simd<false, false, sizeof(__m256i)*2, do_decode_avx2<false, false, ISA_NATIVE> >;
 	_do_decode_raw = &do_decode_simd<true, false, sizeof(__m256i)*2, do_decode_avx2<true, false, ISA_NATIVE> >;
 	_do_decode_end_raw = &do_decode_simd<true, true, sizeof(__m256i)*2, do_decode_avx2<true, true, ISA_NATIVE> >;
+	_decode_isa = ISA_NATIVE;
 }
 # else
 #  include "decoder_sse_base.h"
 static inline void decoder_set_native_funcs() {
-	decoder_sse_init();
-	decoder_init_lut(lookups->eqFix, lookups->compact);
+	decoder_sse_init(lookups);
+	decoder_init_lut(lookups->compact);
 	_do_decode = &do_decode_simd<false, false, sizeof(__m128i)*2, do_decode_sse<false, false, ISA_NATIVE> >;
 	_do_decode_raw = &do_decode_simd<true, false, sizeof(__m128i)*2, do_decode_sse<true, false, ISA_NATIVE> >;
 	_do_decode_end_raw = &do_decode_simd<true, true, sizeof(__m128i)*2, do_decode_sse<true, true, ISA_NATIVE> >;
+	_decode_isa = ISA_NATIVE;
 }
 # endif
 #endif
 
+
+#if defined(PLATFORM_X86) || defined(PLATFORM_ARM)
+void decoder_init_lut(void* compactLUT) {
+	#ifdef YENC_DEC_USE_THINTABLE
+	const int tableSize = 8;
+	#else
+	const int tableSize = 16;
+	#endif
+	for(int i=0; i<(tableSize==8?256:32768); i++) {
+		int k = i;
+		uint8_t* res = (uint8_t*)compactLUT + i*tableSize;
+		int p = 0;
+		for(int j=0; j<tableSize; j++) {
+			if(!(k & 1)) {
+				res[p++] = j;
+			}
+			k >>= 1;
+		}
+		for(; p<tableSize; p++)
+			res[p] = 0x80;
+	}
+}
+#endif
+
+
 void decoder_init() {
 #ifdef PLATFORM_X86
 # if defined(YENC_BUILD_NATIVE) && YENC_BUILD_NATIVE!=0
@@ -62,4 +92,8 @@ void decoder_init() {
 	if(cpu_supports_neon())
 		decoder_set_neon_funcs();
 #endif
+#ifdef __riscv
+	if(cpu_supports_rvv())
+		decoder_set_rvv_funcs();
+#endif
 }

package/src/decoder.h

@@ -32,6 +32,7 @@ typedef enum {
 extern YencDecoderEnd (*_do_decode)(const unsigned char**, unsigned char**, size_t, YencDecoderState*);
 extern YencDecoderEnd (*_do_decode_raw)(const unsigned char**, unsigned char**, size_t, YencDecoderState*);
 extern YencDecoderEnd (*_do_decode_end_raw)(const unsigned char**, unsigned char**, size_t, YencDecoderState*);
+extern int _decode_isa;
 
 static inline size_t do_decode(int isRaw, const unsigned char* src, unsigned char* dest, size_t len, YencDecoderState* state) {
 	unsigned char* ds = dest;
@@ -45,6 +46,9 @@ static inline YencDecoderEnd do_decode_end(const unsigned char** src, unsigned c
 
 void decoder_init();
 
+static inline int decode_isa_level() {
+	return _decode_isa;
+}
 
 
 #ifdef __cplusplus

package/src/decoder_avx.cc

@@ -4,11 +4,12 @@
 #include "decoder_common.h"
 #include "decoder_sse_base.h"
 void decoder_set_avx_funcs() {
-	decoder_sse_init();
-	decoder_init_lut(lookups->eqFix, lookups->compact);
+	decoder_sse_init(lookups);
+	decoder_init_lut(lookups->compact);
 	_do_decode = &do_decode_simd<false, false, sizeof(__m128i)*2, do_decode_sse<false, false, ISA_LEVEL_SSE4_POPCNT> >;
 	_do_decode_raw = &do_decode_simd<true, false, sizeof(__m128i)*2, do_decode_sse<true, false, ISA_LEVEL_SSE4_POPCNT> >;
 	_do_decode_end_raw = &do_decode_simd<true, true, sizeof(__m128i)*2, do_decode_sse<true, true, ISA_LEVEL_SSE4_POPCNT> >;
+	_decode_isa = ISA_LEVEL_AVX;
 }
 #else
 void decoder_set_ssse3_funcs();

package/src/decoder_avx2.cc

@@ -5,10 +5,11 @@
 #include "decoder_avx2_base.h"
 void decoder_set_avx2_funcs() {
 	ALIGN_ALLOC(lookups, sizeof(*lookups), 16);
-	decoder_init_lut(lookups->eqFix, lookups->compact);
+	decoder_init_lut(lookups->compact);
 	_do_decode = &do_decode_simd<false, false, sizeof(__m256i)*2, do_decode_avx2<false, false, ISA_LEVEL_AVX2> >;
 	_do_decode_raw = &do_decode_simd<true, false, sizeof(__m256i)*2, do_decode_avx2<true, false, ISA_LEVEL_AVX2> >;
 	_do_decode_end_raw = &do_decode_simd<true, true, sizeof(__m256i)*2, do_decode_avx2<true, true, ISA_LEVEL_AVX2> >;
+	_decode_isa = ISA_LEVEL_AVX2;
 }
 #else
 void decoder_set_avx_funcs();

package/src/decoder_avx2_base.h

@@ -15,7 +15,6 @@
 #pragma pack(16)
 static struct {
 	/*align16*/ struct { char bytes[16]; } compact[32768];
-	uint8_t eqFix[256];
 } * HEDLEY_RESTRICT lookups;
 #pragma pack()
 
@@ -67,6 +66,8 @@ HEDLEY_ALWAYS_INLINE void do_decode_avx2(const uint8_t* src, long& len, unsigned
 		);
 	}
 	
+	decoder_set_nextMask<isRaw>(src, len, _nextMask); // set this before the loop because we can't check src after it's been overwritten
+	
 	// for some reason, MSVC Win32 seems to crash when trying to compile _mm256_mask_cmpeq_epi8_mask
 	// the crash can be fixed by switching the order of the last two arguments, but it seems to generate wrong code
 	// so just disable the optimisation as it seems to be problematic there
@@ -320,6 +321,7 @@ HEDLEY_ALWAYS_INLINE void do_decode_avx2(const uint8_t* src, long& len, unsigned
 							// terminator found
 							// there's probably faster ways to do this, but reverting to scalar code should be good enough
 							len += (long)i;
+							_nextMask = decoder_set_nextMask<isRaw>(src+i, mask);
 							break;
 						}
 					}
@@ -412,6 +414,7 @@ HEDLEY_ALWAYS_INLINE void do_decode_avx2(const uint8_t* src, long& len, unsigned
 						}
 						if(endFound) {
 							len += (long)i;
+							_nextMask = decoder_set_nextMask<isRaw>(src+i, mask);
 							break;
 						}
 					}
@@ -427,16 +430,9 @@ HEDLEY_ALWAYS_INLINE void do_decode_avx2(const uint8_t* src, long& len, unsigned
 				dataB = _mm256_add_epi8(oDataB, _mm256_set1_epi8(-42));
 			
 			if(LIKELIHOOD(0.0001, (mask & ((maskEq << 1) + escFirst)) != 0)) {
-				unsigned tmp = lookups->eqFix[(maskEq&0xff) & ~(uint64_t)escFirst];
-				uint64_t maskEq2 = tmp;
-				for(int j=8; j<64; j+=8) {
-					tmp = lookups->eqFix[(unsigned)((maskEq>>j)&0xff) & ~(tmp>>7)];
-					maskEq2 |= (uint64_t)tmp<<j;
-				}
-				maskEq = maskEq2;
-				
+				maskEq = fix_eqMask<uint64_t>(maskEq & ~(uint64_t)escFirst);
 				mask &= ~(uint64_t)escFirst;
-				escFirst = tmp>>7;
+				escFirst = maskEq>>63;
 				// next, eliminate anything following a `=` from the special char mask; this eliminates cases of `=\r` so that they aren't removed
 				maskEq <<= 1;
 				mask &= ~maskEq;
@@ -613,20 +609,6 @@ HEDLEY_ALWAYS_INLINE void do_decode_avx2(const uint8_t* src, long& len, unsigned
 		}
 	}
 	_escFirst = (unsigned char)escFirst;
-	if(isRaw) {
-		// this would be the trivial solution, but requires the compiler holding onto minMask throughout the loop:
-		//_nextMask = ~(uint16_t)_mm256_movemask_epi8(_mm256_cmpeq_epi8(minMask, _mm256_set1_epi8('.')));
-		// instead, just scan the memory to determine what to set nextMask to
-		if(len != 0) { // have to gone through at least one loop cycle
-			if(src[i-2] == '\r' && src[i-1] == '\n' && src[i] == '.')
-				_nextMask = 1;
-			else if(src[i-1] == '\r' && src[i] == '\n' && src[i+1] == '.')
-				_nextMask = 2;
-			else
-				_nextMask = 0;
-		}
-	} else
-		_nextMask = 0;
 	_mm256_zeroupper();
 }
 #endif

package/src/decoder_common.h

@@ -1,5 +1,9 @@
 #include "decoder.h"
 
+#if defined(PLATFORM_ARM) && !defined(__aarch64__)
+#define YENC_DEC_USE_THINTABLE 1
+#endif
+
 // TODO: need to support max output length somehow
 // TODO: add branch probabilities
 
@@ -178,24 +182,24 @@ YencDecoderEnd do_decode_end_scalar(const unsigned char** src, unsigned char** d
 			if(es[i] == '.' && isRaw) {
 				i++;
 				YDEC_CHECK_END(YDEC_STATE_CRLFDT)
-				// fall-through
 			} else if(es[i] == '=') {
 				i++;
 				YDEC_CHECK_END(YDEC_STATE_CRLFEQ)
 				goto do_decode_endable_scalar_ceq;
 			} else
 				break;
+			// fall-through
 		case YDEC_STATE_CRLFDT:
 			if(isRaw && es[i] == '\r') {
 				i++;
 				YDEC_CHECK_END(YDEC_STATE_CRLFDTCR)
-				// fall-through
 			} else if(isRaw && es[i] == '=') { // check for dot-stuffed ending: \r\n.=y
 				i++;
 				YDEC_CHECK_END(YDEC_STATE_CRLFEQ)
 				goto do_decode_endable_scalar_ceq;
 			} else
 				break;
+			// fall-through
 		case YDEC_STATE_CRLFDTCR:
 			if(es[i] == '\n') {
 				if(isRaw) {
@@ -331,8 +335,8 @@ YencDecoderEnd do_decode_scalar(const unsigned char** src, unsigned char** dest,
 
 
 
-template<bool isRaw, bool searchEnd, int width, void(&kernel)(const uint8_t*, long&, unsigned char*&, unsigned char&, uint16_t&)>
-YencDecoderEnd do_decode_simd(const unsigned char** src, unsigned char** dest, size_t len, YencDecoderState* state) {
+template<bool isRaw, bool searchEnd, void(&kernel)(const uint8_t*, long&, unsigned char*&, unsigned char&, uint16_t&)>
+inline YencDecoderEnd _do_decode_simd(size_t width, const unsigned char** src, unsigned char** dest, size_t len, YencDecoderState* state) {
 	if(len <= width*2) return do_decode_scalar<isRaw, searchEnd>(src, dest, len, state);
 	
 	YencDecoderState tState = YDEC_STATE_CRLF;
@@ -461,52 +465,60 @@ YencDecoderEnd do_decode_simd(const unsigned char** src, unsigned char** dest, s
 	return YDEC_END_NONE;
 }
 
-static inline void decoder_init_lut(uint8_t* eqFixLUT, void* compactLUT) {
-	for(int i=0; i<256; i++) {
-		int k = i;
-		int p = 0;
-		
-		// fix LUT
-		k = i;
-		p = 0;
-		for(int j=0; j<8; j++) {
-			k = i >> j;
-			if(k & 1) {
-				p |= 1 << j;
-				j++;
-			}
-		}
-		eqFixLUT[i] = p;
-		
-		#ifdef YENC_DEC_USE_THINTABLE
-		uint8_t* res = (uint8_t*)compactLUT + i*8;
-		k = i;
-		p = 0;
-		for(int j=0; j<8; j++) {
-			if(!(k & 1)) {
-				res[p++] = j;
-			}
-			k >>= 1;
-		}
-		for(; p<8; p++)
-			res[p] = 0x80;
-		#endif
-	}
-	#ifndef YENC_DEC_USE_THINTABLE
-	for(int i=0; i<32768; i++) {
-		int k = i;
-		uint8_t* res = (uint8_t*)compactLUT + i*16;
-		int p = 0;
-		
-		for(int j=0; j<16; j++) {
-			if(!(k & 1)) {
-				res[p++] = j;
-			}
-			k >>= 1;
+template<bool isRaw, bool searchEnd, size_t width, void(&kernel)(const uint8_t*, long&, unsigned char*&, unsigned char&, uint16_t&)>
+YencDecoderEnd do_decode_simd(const unsigned char** src, unsigned char** dest, size_t len, YencDecoderState* state) {
+	return _do_decode_simd<isRaw, searchEnd, kernel>(width, src, dest, len, state);
+}
+template<bool isRaw, bool searchEnd, size_t(&getWidth)(), void(&kernel)(const uint8_t*, long&, unsigned char*&, unsigned char&, uint16_t&)>
+YencDecoderEnd do_decode_simd(const unsigned char** src, unsigned char** dest, size_t len, YencDecoderState* state) {
+	return _do_decode_simd<isRaw, searchEnd, kernel>(getWidth(), src, dest, len, state);
+}
+
+
+#if defined(PLATFORM_X86) || defined(PLATFORM_ARM)
+void decoder_init_lut(void* compactLUT);
+#endif
+
+template<bool isRaw>
+static inline void decoder_set_nextMask(const uint8_t* src, size_t len, uint16_t& nextMask) {
+	if(isRaw) {
+		if(len != 0) { // have to gone through at least one loop cycle
+			if(src[-2] == '\r' && src[-1] == '\n' && src[0] == '.')
+				nextMask = 1;
+			else if(src[-1] == '\r' && src[0] == '\n' && src[1] == '.')
+				nextMask = 2;
+			else
+				nextMask = 0;
 		}
-		for(; p<16; p++)
-			res[p] = 0x80;
+	} else
+		nextMask = 0;
+}
+
+// without backtracking
+template<bool isRaw>
+static inline uint16_t decoder_set_nextMask(const uint8_t* src, unsigned mask) {
+	if(isRaw) {
+		if(src[0] == '.')
+			return mask & 1;
+		if(src[1] == '.')
+			return mask & 2;
 	}
-	#endif
+	return 0;
 }
 
+// resolve invalid sequences of = to deal with cases like '===='
+// bit hack inspired from simdjson: https://youtu.be/wlvKAT7SZIQ?t=33m38s
+template<typename T>
+static inline T fix_eqMask(T mask) {
+	// isolate the start of each consecutive bit group (e.g. 01011101 -> 01000101)
+	T start = mask & ~(mask << 1);
+	
+	const T odd = (T)0xaaaaaaaaaaaaaaaa; // every odd bit (10101010...)
+	
+	// obtain groups which start on an even bit (clear groups that start on an odd bit, but this leaves an unwanted trailing bit)
+	T evenGroups = mask + (start & odd);
+	
+	// clear odd bits in even groups, whilst conversely preserving odd bits in odd groups
+	// the `& mask` also conveniently gets rid of unwanted trailing bits
+	return (evenGroups ^ odd) & mask;
+}