yencode 1.1.0 → 1.1.3

package/src/crc_folding_256.cc

@@ -0,0 +1,230 @@
+// 256-bit version of crc_folding
+
+#include "crc_common.h"
+ 
+#if !defined(YENC_DISABLE_AVX256) && ((defined(__VPCLMULQDQ__) && defined(__AVX2__) && defined(__PCLMUL__)) || (defined(_MSC_VER) && _MSC_VER >= 1920 && defined(PLATFORM_X86) && !defined(__clang__)))
+#include <inttypes.h>
+#include <immintrin.h>
+
+
+#if defined(__AVX512VL__) && defined(YENC_BUILD_NATIVE) && YENC_BUILD_NATIVE!=0
+# define ENABLE_AVX512 1
+#endif
+
+static __m256i do_one_fold(__m256i src, __m256i data) {
+	const __m256i fold4 = _mm256_set_epi32(
+		0x00000001, 0x54442bd4,
+		0x00000001, 0xc6e41596,
+		0x00000001, 0x54442bd4,
+		0x00000001, 0xc6e41596
+	);
+#ifdef ENABLE_AVX512
+	return _mm256_ternarylogic_epi32(
+	  _mm256_clmulepi64_epi128(src, fold4, 0x01),
+	  _mm256_clmulepi64_epi128(src, fold4, 0x10),
+	  data,
+	  0x96
+	);
+#else
+	return _mm256_xor_si256(data, _mm256_xor_si256(
+	  _mm256_clmulepi64_epi128(src, fold4, 0x01),
+	  _mm256_clmulepi64_epi128(src, fold4, 0x10)
+	));
+#endif
+}
+
+ALIGN_TO(32, static const uint8_t  pshufb_rot_table[]) = {
+	0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,
+	16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31
+};
+// _mm256_castsi128_si256, but upper is defined to be 0
+#if (defined(__clang__) && __clang_major__ >= 5 && (!defined(__APPLE__) || __clang_major__ >= 7)) || (defined(__GNUC__) && __GNUC__ >= 10)
+// intrinsic unsupported in GCC 9 and MSVC < 2017
+# define zext128_256 _mm256_zextsi128_si256
+#else
+// technically a cast is incorrect, due to upper 128 bits being undefined, but should usually work fine
+// alternative may be `_mm256_set_m128i(_mm_setzero_si128(), v)` but unsupported on GCC < 7, and most compilers generate a VINSERTF128 instruction for it
+# ifdef __OPTIMIZE__
+#  define zext128_256 _mm256_castsi128_si256
+# else
+#  define zext128_256(x) _mm256_inserti128_si256(_mm256_setzero_si256(), x, 0)
+# endif
+#endif
+
+#ifdef ENABLE_AVX512
+# define MM256_BLENDV(a, b, m) _mm256_ternarylogic_epi32(a, b, m, 0xd8)
+# define MM_2XOR(a, b, c) _mm_ternarylogic_epi32(a, b, c, 0x96)
+#else
+# define MM256_BLENDV _mm256_blendv_epi8
+# define MM_2XOR(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c)
+#endif
+
+static void partial_fold(const size_t len, __m256i *crc0, __m256i *crc1, __m256i crc_part) {
+	__m256i shuf = _mm256_broadcastsi128_si256(_mm_loadu_si128((__m128i*)(pshufb_rot_table + (len&15))));
+	__m256i mask = _mm256_cmpgt_epi8(shuf, _mm256_set1_epi8(15));
+	
+	*crc0 = _mm256_shuffle_epi8(*crc0, shuf);
+	*crc1 = _mm256_shuffle_epi8(*crc1, shuf);
+	crc_part = _mm256_shuffle_epi8(crc_part, shuf);
+	
+	__m256i crc_out = _mm256_permute2x128_si256(*crc0, *crc0, 0x08);  // move bottom->top
+	__m256i crc01, crc1p;
+	if(len >= 16) {
+		crc_out = MM256_BLENDV(crc_out, *crc0, mask);
+		crc01 = *crc1;
+		crc1p = crc_part;
+		*crc0 = _mm256_permute2x128_si256(*crc0, *crc1, 0x21);
+		*crc1 = _mm256_permute2x128_si256(*crc1, crc_part, 0x21);
+		crc_part = zext128_256(_mm256_extracti128_si256(crc_part, 1));
+	} else {
+		crc_out = _mm256_and_si256(crc_out, mask);
+		crc01 = _mm256_permute2x128_si256(*crc0, *crc1, 0x21);
+		crc1p = _mm256_permute2x128_si256(*crc1, crc_part, 0x21);
+	}
+	
+	*crc0 = MM256_BLENDV(*crc0, crc01, mask);
+	*crc1 = MM256_BLENDV(*crc1, crc1p, mask);
+	
+	*crc1 = do_one_fold(crc_out, *crc1);
+}
+
+
+ALIGN_TO(16, static const unsigned crc_k[]) = {
+	0xccaa009e, 0x00000000, /* rk1 */
+	0x751997d0, 0x00000001, /* rk2 */
+	0xccaa009e, 0x00000000, /* rk5 */
+	0x63cd6124, 0x00000001, /* rk6 */
+	0xf7011641, 0x00000000, /* rk7 */
+	0xdb710640, 0x00000001  /* rk8 */
+};
+
+
+static uint32_t crc_fold(const unsigned char *src, long len, uint32_t initial) {
+	// info from https://www.reddit.com/r/ReverseEngineering/comments/2zwhl3/mystery_constant_0x9db42487_in_intels_crc32ieee/
+	// firstly, calculate: xmm_crc0 = (intial * 0x487b9c8a) mod 0x104c11db7, where 0x487b9c8a = inverse(1<<512) mod 0x104c11db7
+	__m128i xmm_t0 = _mm_cvtsi32_si128(~initial);
+	
+	xmm_t0 = _mm_clmulepi64_si128(xmm_t0, _mm_set_epi32(0, 0, 0xa273bc24, 0), 0);  // reverse(0x487b9c8a)<<1 == 0xa273bc24
+	__m128i reduction = _mm_set_epi32( // polynomial reduction factors
+	  1, 0xdb710640, // G* = 0x04c11db7
+	  0, 0xf7011641  // Q+ = 0x04d101df  (+1 to save an additional xor operation)
+	);
+	__m128i xmm_t1 = _mm_clmulepi64_si128(xmm_t0, reduction, 0);
+	xmm_t1 = _mm_clmulepi64_si128(xmm_t1, reduction, 0x10);
+	
+	xmm_t0 = _mm_srli_si128(_mm_xor_si128(xmm_t0, xmm_t1), 8);
+	__m256i crc0 = zext128_256(xmm_t0);
+	__m256i crc1 = _mm256_setzero_si256();
+	
+	if (len < 32) {
+		if (len == 0)
+			return initial;
+		__m256i crc_part = _mm256_setzero_si256();
+		memcpy(&crc_part, src, len);
+		partial_fold(len, &crc0, &crc1, crc_part);
+	} else {
+		uintptr_t algn_diff = (0 - (uintptr_t)src) & 0x1F;
+		if (algn_diff) {
+			partial_fold(algn_diff, &crc0, &crc1, _mm256_loadu_si256((__m256i *)src));
+			src += algn_diff;
+			len -= algn_diff;
+		}
+		
+		while (len >= 64) {
+			crc0 = do_one_fold(crc0, _mm256_load_si256((__m256i*)src));
+			crc1 = do_one_fold(crc1, _mm256_load_si256((__m256i*)src + 1));
+			src += 64;
+			len -= 64;
+		}
+		
+		if (len >= 32) {
+			__m256i old = crc1;
+			crc1 = do_one_fold(crc0, _mm256_load_si256((__m256i*)src));
+			crc0 = old;
+			
+			len -= 32;
+			src += 32;
+		}
+		
+		if(len != 0) {
+			partial_fold(len, &crc0, &crc1, _mm256_load_si256((__m256i *)src));
+		}
+	}
+	
+	const __m128i xmm_mask = _mm_set_epi32(-1,-1,-1,0);
+	__m128i x_tmp0, x_tmp1, x_tmp2, crc_fold;
+	
+	__m128i xmm_crc0 = _mm256_castsi256_si128(crc0);
+	__m128i xmm_crc1 = _mm256_extracti128_si256(crc0, 1);
+	__m128i xmm_crc2 = _mm256_castsi256_si128(crc1);
+	__m128i xmm_crc3 = _mm256_extracti128_si256(crc1, 1);
+
+	/*
+	 * k1
+	 */
+	crc_fold = _mm_load_si128((__m128i *)crc_k);
+
+	x_tmp0 = _mm_clmulepi64_si128(xmm_crc0, crc_fold, 0x10);
+	xmm_crc0 = _mm_clmulepi64_si128(xmm_crc0, crc_fold, 0x01);
+	xmm_crc1 = MM_2XOR(xmm_crc1, x_tmp0, xmm_crc0);
+
+	x_tmp1 = _mm_clmulepi64_si128(xmm_crc1, crc_fold, 0x10);
+	xmm_crc1 = _mm_clmulepi64_si128(xmm_crc1, crc_fold, 0x01);
+	xmm_crc2 = MM_2XOR(xmm_crc2, x_tmp1, xmm_crc1);
+
+	x_tmp2 = _mm_clmulepi64_si128(xmm_crc2, crc_fold, 0x10);
+	xmm_crc2 = _mm_clmulepi64_si128(xmm_crc2, crc_fold, 0x01);
+	xmm_crc3 = MM_2XOR(xmm_crc3, x_tmp2, xmm_crc2);
+
+	/*
+	 * k5
+	 */
+	crc_fold = _mm_load_si128((__m128i *)crc_k + 1);
+
+	xmm_crc0 = xmm_crc3;
+	xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0);
+	xmm_crc0 = _mm_srli_si128(xmm_crc0, 8);
+	xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc0);
+
+	xmm_crc0 = xmm_crc3;
+	xmm_crc3 = _mm_slli_si128(xmm_crc3, 4);
+	xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0x10);
+#ifdef ENABLE_AVX512
+	//xmm_crc3 = _mm_maskz_xor_epi32(14, xmm_crc3, xmm_crc0);
+	xmm_crc3 = _mm_ternarylogic_epi32(xmm_crc3, xmm_crc0, xmm_mask, 0x28);
+#else
+	xmm_crc0 = _mm_and_si128(xmm_crc0, xmm_mask);
+	xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc0);
+#endif
+
+	/*
+	 * k7
+	 */
+	xmm_crc1 = xmm_crc3;
+	crc_fold = _mm_load_si128((__m128i *)crc_k + 2);
+
+	xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0);
+	xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0x10);
+#ifdef ENABLE_AVX512
+	xmm_crc3 = _mm_ternarylogic_epi32(xmm_crc3, xmm_crc1, xmm_crc1, 0xC3); // NOT(xmm_crc3 ^ xmm_crc1)
+#else
+	xmm_crc1 = _mm_xor_si128(xmm_crc1, xmm_mask);
+	xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc1);
+#endif
+	return _mm_extract_epi32(xmm_crc3, 2);
+}
+
+static uint32_t do_crc32_incremental_clmul(const void* data, size_t length, uint32_t init) {
+	return crc_fold((const unsigned char*)data, (long)length, init);
+}
+
+void crc_clmul256_set_funcs(crc_func* _do_crc32_incremental) {
+	*_do_crc32_incremental = &do_crc32_incremental_clmul;
+}
+#else
+void crc_clmul_set_funcs(crc_func* _do_crc32_incremental);
+void crc_clmul256_set_funcs(crc_func* _do_crc32_incremental) {
+	crc_clmul_set_funcs(_do_crc32_incremental);
+}
+#endif
+

package/src/decoder.cc

@@ -1,15 +1,19 @@
 #include "common.h"
 
 #include "decoder_common.h"
+#include "decoder.h"
 
-YencDecoderEnd (*_do_decode)(const unsigned char*HEDLEY_RESTRICT*, unsigned char*HEDLEY_RESTRICT*, size_t, YencDecoderState*) = &do_decode_scalar<false, false>;
-YencDecoderEnd (*_do_decode_raw)(const unsigned char*HEDLEY_RESTRICT*, unsigned char*HEDLEY_RESTRICT*, size_t, YencDecoderState*) = &do_decode_scalar<true, false>;
-YencDecoderEnd (*_do_decode_end_raw)(const unsigned char*HEDLEY_RESTRICT*, unsigned char*HEDLEY_RESTRICT*, size_t, YencDecoderState*) = &do_decode_end_scalar<true>;
+extern "C" {
+	YencDecoderEnd (*_do_decode)(const unsigned char*HEDLEY_RESTRICT*, unsigned char*HEDLEY_RESTRICT*, size_t, YencDecoderState*) = &do_decode_scalar<false, false>;
+	YencDecoderEnd (*_do_decode_raw)(const unsigned char*HEDLEY_RESTRICT*, unsigned char*HEDLEY_RESTRICT*, size_t, YencDecoderState*) = &do_decode_scalar<true, false>;
+	YencDecoderEnd (*_do_decode_end_raw)(const unsigned char*HEDLEY_RESTRICT*, unsigned char*HEDLEY_RESTRICT*, size_t, YencDecoderState*) = &do_decode_end_scalar<true>;
+}
 
 void decoder_set_sse2_funcs();
 void decoder_set_ssse3_funcs();
 void decoder_set_avx_funcs();
 void decoder_set_avx2_funcs();
+void decoder_set_vbmi2_funcs();
 void decoder_set_neon_funcs();
 
 
@@ -41,7 +45,9 @@ void decoder_init() {
 	decoder_set_native_funcs();
 # else
 	int use_isa = cpu_supports_isa();
-	if(use_isa >= ISA_LEVEL_AVX2)
+	if(use_isa >= ISA_LEVEL_VBMI2)
+		decoder_set_vbmi2_funcs();
+	else if(use_isa >= ISA_LEVEL_AVX2)
 		decoder_set_avx2_funcs();
 	else if(use_isa >= ISA_LEVEL_AVX)
 		decoder_set_avx_funcs();

package/src/decoder.h

@@ -1,3 +1,11 @@
+#ifndef __YENC_DECODER_H
+#define __YENC_DECODER_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
 
 // the last state that the decoder was in (i.e. last few characters processed)
 // the state is needed for incremental decoders as its behavior is affected by what it processed last
@@ -25,8 +33,7 @@ extern YencDecoderEnd (*_do_decode)(const unsigned char*HEDLEY_RESTRICT*, unsign
 extern YencDecoderEnd (*_do_decode_raw)(const unsigned char*HEDLEY_RESTRICT*, unsigned char*HEDLEY_RESTRICT*, size_t, YencDecoderState*);
 extern YencDecoderEnd (*_do_decode_end_raw)(const unsigned char*HEDLEY_RESTRICT*, unsigned char*HEDLEY_RESTRICT*, size_t, YencDecoderState*);
 
-template<bool isRaw>
-static inline size_t do_decode(const unsigned char* HEDLEY_RESTRICT src, unsigned char* HEDLEY_RESTRICT dest, size_t len, YencDecoderState* state) {
+static inline size_t do_decode(int isRaw, const unsigned char* HEDLEY_RESTRICT src, unsigned char* HEDLEY_RESTRICT dest, size_t len, YencDecoderState* state) {
 	unsigned char* ds = dest;
 	(*(isRaw ? _do_decode_raw : _do_decode))(&src, &ds, len, state);
 	return ds - dest;
@@ -37,3 +44,10 @@ static inline YencDecoderEnd do_decode_end(const unsigned char*HEDLEY_RESTRICT*
 }
 
 void decoder_init();
+
+
+
+#ifdef __cplusplus
+}
+#endif
+#endif

package/src/decoder_avx2_base.h

@@ -1,8 +1,8 @@
 
 #ifdef __AVX2__
 
-// GCC (ver 6-10(dev)) fails to optimize pure C version of mask testing, but has this intrinsic; Clang >= 7 optimizes C version fine
-#if defined(__GNUC__) && __GNUC__ >= 7
+// GCC (ver 6-10(dev)) fails to optimize pure C version of mask testing, but has this intrinsic; Clang >= 7 optimizes C version fine; functions added in Clang 8
+#if (defined(__GNUC__) && __GNUC__ >= 7) || (defined(_MSC_VER) && _MSC_VER >= 1924)
 # define KORTEST32(a, b) !_kortestz_mask32_u8((a), (b))
 # define KAND32(a, b) _kand_mask32((a), (b))
 # define KOR32(a, b) _kor_mask32((a), (b))
@@ -30,13 +30,17 @@ static HEDLEY_ALWAYS_INLINE __m256i force_align_read_256(const void* p) {
 }
 
 // _mm256_castsi128_si256, but upper is defined to be 0
-#if defined(__clang__) && __clang_major__ >= 5
+#if (defined(__clang__) && __clang_major__ >= 5 && (!defined(__APPLE__) || __clang_major__ >= 7)) || (defined(__GNUC__) && __GNUC__ >= 10)
 // intrinsic unsupported in GCC 9 and MSVC < 2017
 # define zext128_256 _mm256_zextsi128_si256
 #else
 // technically a cast is incorrect, due to upper 128 bits being undefined, but should usually work fine
 // alternative may be `_mm256_set_m128i(_mm_setzero_si128(), v)` but unsupported on GCC < 7, and most compilers generate a VINSERTF128 instruction for it
-# define zext128_256 _mm256_castsi128_si256
+# ifdef __OPTIMIZE__
+#  define zext128_256 _mm256_castsi128_si256
+# else
+#  define zext128_256(x) _mm256_inserti128_si256(_mm256_setzero_si256(), x, 0)
+# endif
 #endif
 
 
@@ -56,6 +60,17 @@ HEDLEY_ALWAYS_INLINE void do_decode_avx2(const uint8_t* HEDLEY_RESTRICT src, lon
 			'.','.','.','.','.','.','.','.','.','.','.','.','.','.',_nextMask==2?0:'.',_nextMask==1?0:'.'
 		);
 	}
+	
+	// for some reason, MSVC Win32 seems to crash when trying to compile _mm256_mask_cmpeq_epi8_mask
+	// the crash can be fixed by switching the order of the last two arguments, but it seems to generate wrong code
+	// so just disable the optimisation as it seems to be problematic there
+#if defined(__AVX512VL__) && defined(__AVX512BW__)
+# if defined(_MSC_VER) && !defined(PLATFORM_AMD64) && !defined(__clang__)
+	const bool useAVX3MaskCmp = false;
+# else
+	const bool useAVX3MaskCmp = (use_isa >= ISA_LEVEL_AVX3);
+# endif
+#endif
 	intptr_t i;
 	for(i = -len; i; i += sizeof(__m256i)*2) {
 		__m256i oDataA = _mm256_load_si256((__m256i *)(src+i));
@@ -122,7 +137,7 @@ HEDLEY_ALWAYS_INLINE void do_decode_avx2(const uint8_t* HEDLEY_RESTRICT src, lon
 				__mmask32 match2EqMaskA, match2EqMaskB;
 				__mmask32 match0CrMaskA, match0CrMaskB;
 				__mmask32 match2CrXDtMaskA, match2CrXDtMaskB;
-				if(use_isa >= ISA_LEVEL_AVX3 && searchEnd) {
+				if(useAVX3MaskCmp && searchEnd) {
 					match2EqMaskA = _mm256_cmpeq_epi8_mask(_mm256_set1_epi8('='), tmpData2A);
 					match2EqMaskB = _mm256_cmpeq_epi8_mask(_mm256_set1_epi8('='), tmpData2B);
 				} else
@@ -138,7 +153,7 @@ HEDLEY_ALWAYS_INLINE void do_decode_avx2(const uint8_t* HEDLEY_RESTRICT src, lon
 					// find patterns of \r_.
 					
 #if defined(__AVX512VL__) && defined(__AVX512BW__)
-					if(use_isa >= ISA_LEVEL_AVX3) {
+					if(useAVX3MaskCmp) {
 						match0CrMaskA = _mm256_cmpeq_epi8_mask(oDataA, _mm256_set1_epi8('\r'));
 						match0CrMaskB = _mm256_cmpeq_epi8_mask(oDataB, _mm256_set1_epi8('\r'));
 						match2CrXDtMaskA = _mm256_mask_cmpeq_epi8_mask(match0CrMaskA, tmpData2A, _mm256_set1_epi8('.'));
@@ -168,7 +183,7 @@ HEDLEY_ALWAYS_INLINE void do_decode_avx2(const uint8_t* HEDLEY_RESTRICT src, lon
 #if defined(__AVX512VL__) && defined(__AVX512BW__)
 					__mmask32 match1NlMaskA, match1NlMaskB;
 					__mmask32 match2NlDotMaskA, match2NlDotMaskB;
-					if(use_isa >= ISA_LEVEL_AVX3) {
+					if(useAVX3MaskCmp) {
 						match1NlMaskA = _mm256_mask_cmpeq_epi8_mask(
 							match0CrMaskA,
 							_mm256_set1_epi8('\n'),
@@ -224,7 +239,7 @@ HEDLEY_ALWAYS_INLINE void do_decode_avx2(const uint8_t* HEDLEY_RESTRICT src, lon
 						
 						int matchEnd;
 #if defined(__AVX512VL__) && defined(__AVX512BW__)
-						if(use_isa >= ISA_LEVEL_AVX3) {
+						if(useAVX3MaskCmp) {
 							__mmask32 match3EqYMaskA = _mm256_mask_cmpeq_epi8_mask(
 								match2EqMaskA,
 								_mm256_set1_epi8('y'),
@@ -298,12 +313,12 @@ HEDLEY_ALWAYS_INLINE void do_decode_avx2(const uint8_t* HEDLEY_RESTRICT src, lon
 						if(LIKELIHOOD(0.002, matchEnd)) {
 							// terminator found
 							// there's probably faster ways to do this, but reverting to scalar code should be good enough
-							len += i;
+							len += (long)i;
 							break;
 						}
 					}
 #if defined(__AVX512VL__) && defined(__AVX512BW__)
-					if(use_isa >= ISA_LEVEL_AVX3) {
+					if(useAVX3MaskCmp) {
 						mask |= (uint64_t)match2NlDotMaskA << 2;
 						mask |= (uint64_t)match2NlDotMaskB << 34;
 						minMask = _mm256_maskz_mov_epi8(~(match2NlDotMaskB>>30), _mm256_set1_epi8('.'));
@@ -321,7 +336,7 @@ HEDLEY_ALWAYS_INLINE void do_decode_avx2(const uint8_t* HEDLEY_RESTRICT src, lon
 					__m256i match3EqYA, match3EqYB;
 #if defined(__AVX512VL__) && defined(__AVX512BW__)
 					__mmask32 match3EqYMaskA, match3EqYMaskB;
-					if(use_isa >= ISA_LEVEL_AVX3) {
+					if(useAVX3MaskCmp) {
 						match3EqYMaskA = _mm256_mask_cmpeq_epi8_mask(
 							match2EqMaskA,
 							_mm256_set1_epi8('y'),
@@ -351,7 +366,7 @@ HEDLEY_ALWAYS_INLINE void do_decode_avx2(const uint8_t* HEDLEY_RESTRICT src, lon
 					if(LIKELIHOOD(0.002, partialEndFound)) {
 						bool endFound;
 #if defined(__AVX512VL__) && defined(__AVX512BW__)
-						if(use_isa >= ISA_LEVEL_AVX3) {
+						if(useAVX3MaskCmp) {
 							__mmask32 match3LfEqYMaskA = _mm256_mask_cmpeq_epi8_mask(
 								match3EqYMaskA,
 								_mm256_set1_epi8('\n'),
@@ -390,7 +405,7 @@ HEDLEY_ALWAYS_INLINE void do_decode_avx2(const uint8_t* HEDLEY_RESTRICT src, lon
 							));
 						}
 						if(endFound) {
-							len += i;
+							len += (long)i;
 							break;
 						}
 					}
@@ -489,14 +504,10 @@ HEDLEY_ALWAYS_INLINE void do_decode_avx2(const uint8_t* HEDLEY_RESTRICT src, lon
 #endif
 				{
 					// << 1 byte
-					cmpEqB = _mm256_cmpeq_epi8(_mm256_set1_epi8('='), _mm256_loadu_si256((__m256i *)(src+i-1) + 1));
-#if defined(__tune_znver1__) || defined(__tune_bdver4__)
 					cmpEqA = _mm256_alignr_epi8(cmpEqA, _mm256_inserti128_si256(
-						_mm256_setzero_si256(), _mm256_castsi256_si128(cmpEqA), 1
+						_mm256_set1_epi8('='), _mm256_castsi256_si128(cmpEqA), 1
 					), 15);
-#else
-					cmpEqA = _mm256_alignr_epi8(cmpEqA, _mm256_permute2x128_si256(cmpEqA, cmpEqA, 0x08), 15);
-#endif
+					cmpEqB = _mm256_cmpeq_epi8(_mm256_set1_epi8('='), _mm256_loadu_si256((__m256i *)(src+i-1) + 1));
 					dataA = _mm256_add_epi8(
 						oDataA,
 						_mm256_blendv_epi8(
@@ -523,7 +534,7 @@ HEDLEY_ALWAYS_INLINE void do_decode_avx2(const uint8_t* HEDLEY_RESTRICT src, lon
 #endif
 			{
 				yencOffset = _mm256_xor_si256(_mm256_set1_epi8(-42), zext128_256(
-					_mm_slli_epi16(_mm_cvtsi32_si128(escFirst), 6)
+					_mm_slli_epi16(_mm_cvtsi32_si128((int)escFirst), 6)
 				));
 			}
 			
@@ -565,7 +576,7 @@ HEDLEY_ALWAYS_INLINE void do_decode_avx2(const uint8_t* HEDLEY_RESTRICT src, lon
 				p -= popcnt32(mask & 0xffff0);
 				
 				_mm_storeu_si128((__m128i*)(p + XMM_SIZE*3), _mm256_extracti128_si256(dataB, 1));
-				p -= popcnt32(mask >> 20);
+				p -= popcnt32((unsigned int)(mask >> 20));
 #else
 				mask >>= 32;
 				shuf = _mm256_inserti128_si256(

package/src/decoder_common.h

@@ -340,7 +340,7 @@ YencDecoderEnd do_decode_simd(const unsigned char* HEDLEY_RESTRICT* src, unsigne
 	if((uintptr_t)(*src) & ((width-1))) {
 		// find source memory alignment
 		unsigned char* aSrc = (unsigned char*)(((uintptr_t)(*src) + (width-1)) & ~(width-1));
-		int amount = aSrc - *src;
+		int amount = (int)(aSrc - *src);
 		len -= amount;
 		YencDecoderEnd ended = do_decode_scalar<isRaw, searchEnd>(src, dest, amount, pState);
 		if(ended) return ended;
@@ -427,7 +427,7 @@ YencDecoderEnd do_decode_simd(const unsigned char* HEDLEY_RESTRICT* src, unsigne
 		escFirst = (*pState == YDEC_STATE_EQ || *pState == YDEC_STATE_CRLFEQ);
 		
 		// our algorithm may perform an aligned load on the next part, of which we consider 2 bytes (for \r\n. sequence checking)
-		long dLen = len - lenBuffer;
+		long dLen = (long)(len - lenBuffer);
 		dLen = (dLen + (width-1)) & ~(width-1);
 		
 		kernel((const uint8_t*)(*src) + dLen, dLen, p, escFirst, nextMask);

package/src/decoder_neon.cc

@@ -7,9 +7,9 @@
 #include "decoder_common.h"
 
 
-#ifdef _MSC_VER
-# define vld1_u8_align vld1_u8_ex
-# define vld1q_u8_align vld1q_u8_ex
+#if defined(_MSC_VER) && !defined(__clang__)
+# define vld1_u8_align(p, a) vld1_u8_ex(p, a*8)
+# define vld1q_u8_align(p, a) vld1q_u8_ex(p, a*8)
 #elif defined(__GNUC__)
 # define vld1_u8_align(p, n) vld1_u8((uint8_t*)__builtin_assume_aligned(p, n))
 # define vld1q_u8_align(p, n) vld1q_u8((uint8_t*)__builtin_assume_aligned(p, n))
@@ -19,19 +19,17 @@
 #endif
 
 
-// for compilers that lack these functions
-#if defined(__clang__) || (defined(__GNUC__) && (defined(__aarch64__) && __GNUC__ >= 8))
+// for compilers that lack these functions (Clang armv7 9-12 seems to have issues with multi-vector loads)
+#if (defined(__clang__) && (defined(__aarch64__) || __clang_major__<9 || __clang_major__>12)) || (defined(__GNUC__) && (defined(__aarch64__) && __GNUC__ >= 8))
 # define vld1q_u8_x2_align(p, n) vld1q_u8_x2((uint8_t*)__builtin_assume_aligned(p, n))
 #else
-HEDLEY_ALWAYS_INLINE uint8x16x2_t vld1q_u8_x2_align(const uint8_t* p, int n) {
-	return (uint8x16x2_t){vld1q_u8_align(p, n), vld1q_u8_align(p+16, n)};
-}
+# define vld1q_u8_x2_align(p, n) vcreate2_u8(vld1q_u8_align(p, (n)/2), vld1q_u8_align((p)+16, (n)/2))
 #endif
 // Clang wrongly assumes alignment on vld1q_u8_x2, and ARMv7 GCC doesn't support the function, so effectively, it can only be used in ARMv8 compilers
-#if defined(__aarch64__) && (defined(__clang__) || (defined(__GNUC__) && __GNUC__ >= 9))
+#if defined(__aarch64__) && (defined(__clang__) || HEDLEY_GCC_VERSION_CHECK(8,5,0))
 # define vst1q_u8_x2_unaligned vst1q_u8_x2
 #else
-HEDLEY_ALWAYS_INLINE void vst1q_u8_x2_unaligned(uint8_t* p, uint8x16x2_t data) {
+static HEDLEY_ALWAYS_INLINE void vst1q_u8_x2_unaligned(uint8_t* p, uint8x16x2_t data) {
 	vst1q_u8(p, data.val[0]);
 	vst1q_u8(p+16, data.val[1]);
 }
@@ -64,18 +62,20 @@ template<bool isRaw, bool searchEnd>
 HEDLEY_ALWAYS_INLINE void do_decode_neon(const uint8_t* HEDLEY_RESTRICT src, long& len, unsigned char* HEDLEY_RESTRICT & p, unsigned char& escFirst, uint16_t& nextMask) {
 	HEDLEY_ASSUME(escFirst == 0 || escFirst == 1);
 	HEDLEY_ASSUME(nextMask == 0 || nextMask == 1 || nextMask == 2);
-	uint8x16_t yencOffset = escFirst ? (uint8x16_t){42+64,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42} : vdupq_n_u8(42);
+	uint8x16_t yencOffset = escFirst ? vmakeq_u8(42+64,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42) : vdupq_n_u8(42);
 #ifdef __aarch64__
 	uint8x16_t nextMaskMix = vdupq_n_u8(0);
-	if(nextMask)
-		nextMaskMix[nextMask-1] = nextMask;
+	if(nextMask == 1)
+		nextMaskMix = vsetq_lane_u8(1, nextMaskMix, 0);
+	if(nextMask == 2)
+		nextMaskMix = vsetq_lane_u8(2, nextMaskMix, 1);
 #else
 	uint8x16_t lfCompare = vdupq_n_u8('\n');
 	if(isRaw) {
 		if(nextMask == 1)
-			lfCompare[0] = '.';
+			lfCompare = vsetq_lane_u8('.', lfCompare, 0);
 		if(nextMask == 2)
-			lfCompare[1] = '.';
+			lfCompare = vsetq_lane_u8('.', lfCompare, 1);
 	}
 #endif
 	long i;
@@ -90,13 +90,13 @@ HEDLEY_ALWAYS_INLINE void do_decode_neon(const uint8_t* HEDLEY_RESTRICT src, lon
 #ifdef __aarch64__
 		cmpA = vqtbx1q_u8(
 			cmpEqA,
-			//                                \n      \r
-			(uint8x16_t){0,0,0,0,0,0,0,0,0,0,255,0,0,255,0,0},
+			//                             \n      \r
+			vmakeq_u8(0,0,0,0,0,0,0,0,0,0,255,0,0,255,0,0),
 			dataA
 		),
 		cmpB = vqtbx1q_u8(
 			cmpEqB,
-			(uint8x16_t){0,0,0,0,0,0,0,0,0,0,255,0,0,255,0,0},
+			vmakeq_u8(0,0,0,0,0,0,0,0,0,0,255,0,0,255,0,0),
 			dataB
 		);
 		if(isRaw) cmpA = vorrq_u8(cmpA, nextMaskMix);
@@ -122,12 +122,12 @@ HEDLEY_ALWAYS_INLINE void do_decode_neon(const uint8_t* HEDLEY_RESTRICT src, lon
 		
 #ifdef __aarch64__
 		if (LIKELIHOOD(0.42 /*guess*/, neon_vect_is_nonzero(vorrq_u8(cmpA, cmpB)))) {
-			cmpA = vandq_u8(cmpA, (uint8x16_t){1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128});
-			cmpB = vandq_u8(cmpB, (uint8x16_t){1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128});
+			cmpA = vandq_u8(cmpA, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128));
+			cmpB = vandq_u8(cmpB, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128));
 			uint8x16_t cmpMerge = vpaddq_u8(cmpA, cmpB);
 			uint8x16_t cmpEqMerge = vpaddq_u8(
-				vandq_u8(cmpEqA, (uint8x16_t){1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128}),
-				vandq_u8(cmpEqB, (uint8x16_t){1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128})
+				vandq_u8(cmpEqA, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128)),
+				vandq_u8(cmpEqB, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128))
 			);
 			
 			uint8x16_t cmpCombined = vpaddq_u8(cmpMerge, cmpEqMerge);
@@ -136,8 +136,8 @@ HEDLEY_ALWAYS_INLINE void do_decode_neon(const uint8_t* HEDLEY_RESTRICT src, lon
 			uint32_t mask = vgetq_lane_u32(vreinterpretq_u32_u8(cmpCombined), 0);
 			uint32_t maskEq = vgetq_lane_u32(vreinterpretq_u32_u8(cmpCombined), 1);
 #else
-		cmpA = vandq_u8(cmpA, (uint8x16_t){1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128});
-		cmpB = vandq_u8(cmpB, (uint8x16_t){1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128});
+		cmpA = vandq_u8(cmpA, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128));
+		cmpB = vandq_u8(cmpB, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128));
 		// no vpaddq_u8 in ARMv7, so need extra 64-bit VPADD
 		uint8x8_t cmpPacked = vpadd_u8(
 			vpadd_u8(
@@ -150,8 +150,8 @@ HEDLEY_ALWAYS_INLINE void do_decode_neon(const uint8_t* HEDLEY_RESTRICT src, lon
 		cmpPacked = vpadd_u8(cmpPacked, cmpPacked);
 		uint32_t mask = vget_lane_u32(vreinterpret_u32_u8(cmpPacked), 0);
 		if(LIKELIHOOD(0.42, mask != 0)) {
-			uint8x16_t cmpEqMaskedA = vandq_u8(cmpEqA, (uint8x16_t){1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128});
-			uint8x16_t cmpEqMaskedB = vandq_u8(cmpEqB, (uint8x16_t){1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128});
+			uint8x16_t cmpEqMaskedA = vandq_u8(cmpEqA, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128));
+			uint8x16_t cmpEqMaskedB = vandq_u8(cmpEqB, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128));
 			uint8x8_t cmpEqPacked = vpadd_u8(
 				vpadd_u8(
 					vget_low_u8(cmpEqMaskedA), vget_high_u8(cmpEqMaskedA)
@@ -170,7 +170,7 @@ HEDLEY_ALWAYS_INLINE void do_decode_neon(const uint8_t* HEDLEY_RESTRICT src, lon
 				// vext seems to be a cheap operation on ARM, relative to loads, so only avoid it if there's only one load (isRaw only)
 				uint8x16_t tmpData2, nextData;
 				if(isRaw && !searchEnd) {
-					tmpData2 = vld1q_u8_align(src+i + 2 + sizeof(uint8x16_t), 2);
+					tmpData2 = vld1q_u8(src+i + 2 + sizeof(uint8x16_t));
 				} else {
 					nextData = vld1q_u8_align(src+i + sizeof(uint8x16_t)*2, 16); // only 32-bits needed, but there doesn't appear a nice way to do this via intrinsics: https://stackoverflow.com/questions/46910799/arm-neon-intrinsics-convert-d-64-bit-register-to-low-half-of-q-128-bit-regis
 					tmpData2 = vextq_u8(dataB, nextData, 2);
@@ -255,15 +255,15 @@ HEDLEY_ALWAYS_INLINE void do_decode_neon(const uint8_t* HEDLEY_RESTRICT src, lon
 						}
 					}
 #ifdef __aarch64__
-					uint8x16_t match2NlDotBMasked = vandq_u8(match2NlDotB, (uint8x16_t){1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128});
+					uint8x16_t match2NlDotBMasked = vandq_u8(match2NlDotB, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128));
 					uint8x16_t mergeKillDots = vpaddq_u8(
-						vandq_u8(match2NlDotA, (uint8x16_t){1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128}),
+						vandq_u8(match2NlDotA, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128)),
 						match2NlDotBMasked
 					);
 					uint8x8_t mergeKillDots2 = vget_low_u8(vpaddq_u8(mergeKillDots, mergeKillDots));
 #else
-					uint8x16_t match2NlDotMaskedA = vandq_u8(match2NlDotA, (uint8x16_t){1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128});
-					uint8x16_t match2NlDotMaskedB = vandq_u8(match2NlDotB, (uint8x16_t){1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128});
+					uint8x16_t match2NlDotMaskedA = vandq_u8(match2NlDotA, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128));
+					uint8x16_t match2NlDotMaskedB = vandq_u8(match2NlDotB, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128));
 					uint8x8_t mergeKillDots2 = vpadd_u8(
 						vpadd_u8(
 							vget_low_u8(match2NlDotMaskedA), vget_high_u8(match2NlDotMaskedA)
@@ -342,11 +342,11 @@ HEDLEY_ALWAYS_INLINE void do_decode_neon(const uint8_t* HEDLEY_RESTRICT src, lon
 #ifdef __aarch64__
 				uint8x16_t vMaskEqA = vqtbl1q_u8(
 					vcombine_u8(maskEqTemp, vdup_n_u8(0)),
-					(uint8x16_t){0,0,0,0,0,0,0,0, 1,1,1,1,1,1,1,1}
+					vmakeq_u8(0,0,0,0,0,0,0,0, 1,1,1,1,1,1,1,1)
 				);
 				uint8x16_t vMaskEqB = vqtbl1q_u8(
 					vcombine_u8(maskEqTemp, vdup_n_u8(0)),
-					(uint8x16_t){2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3}
+					vmakeq_u8(2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3)
 				);
 #else
 				uint8x16_t vMaskEqA = vcombine_u8(
@@ -358,8 +358,8 @@ HEDLEY_ALWAYS_INLINE void do_decode_neon(const uint8_t* HEDLEY_RESTRICT src, lon
 					vdup_lane_u8(maskEqTemp, 3)
 				);
 #endif
-				vMaskEqA = vtstq_u8(vMaskEqA, (uint8x16_t){1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128});
-				vMaskEqB = vtstq_u8(vMaskEqB, (uint8x16_t){1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128});
+				vMaskEqA = vtstq_u8(vMaskEqA, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128));
+				vMaskEqB = vtstq_u8(vMaskEqB, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128));
 				
 				dataA = vsubq_u8(
 					dataA,
@@ -391,7 +391,7 @@ HEDLEY_ALWAYS_INLINE void do_decode_neon(const uint8_t* HEDLEY_RESTRICT src, lon
 					)
 				);
 			}
-			yencOffset[0] = (escFirst << 6) | 42;
+			yencOffset = vsetq_lane_u8((escFirst << 6) | 42, yencOffset, 0);
 			
 			// all that's left is to 'compress' the data (skip over masked chars)
 			uint32_t counts = 0x08080808 - vget_lane_u32(vreinterpret_u32_u8(vcnt_u8(cmpPacked)), 0);
@@ -439,7 +439,7 @@ HEDLEY_ALWAYS_INLINE void do_decode_neon(const uint8_t* HEDLEY_RESTRICT src, lon
 		} else {
 			dataA = vsubq_u8(dataA, yencOffset);
 			dataB = vsubq_u8(dataB, vdupq_n_u8(42));
-			vst1q_u8_x2_unaligned(p, ((uint8x16x2_t){dataA, dataB}));
+			vst1q_u8_x2_unaligned(p, vcreate2_u8(dataA, dataB));
 			p += sizeof(uint8x16_t)*2;
 			escFirst = 0;
 #ifdef __aarch64__