yencode 1.1.2 → 1.1.4

package/src/decoder_common.h

@@ -6,7 +6,7 @@
 
 // state var: refers to the previous state - only used for incremental processing
 template<bool isRaw>
-size_t do_decode_noend_scalar(const unsigned char* HEDLEY_RESTRICT src, unsigned char* HEDLEY_RESTRICT dest, size_t len, YencDecoderState* state) {
+size_t do_decode_noend_scalar(const unsigned char* src, unsigned char* dest, size_t len, YencDecoderState* state) {
 	const unsigned char *es = src + len; // end source pointer
 	unsigned char *p = dest; // destination pointer
 	long i = -(long)len; // input position
@@ -140,7 +140,7 @@ size_t do_decode_noend_scalar(const unsigned char* HEDLEY_RESTRICT src, unsigned
 }
 
 template<bool isRaw>
-YencDecoderEnd do_decode_end_scalar(const unsigned char* HEDLEY_RESTRICT* src, unsigned char* HEDLEY_RESTRICT* dest, size_t len, YencDecoderState* state) {
+YencDecoderEnd do_decode_end_scalar(const unsigned char** src, unsigned char** dest, size_t len, YencDecoderState* state) {
 	const unsigned char *es = (*src) + len; // end source pointer
 	unsigned char *p = *dest; // destination pointer
 	long i = -(long)len; // input position
@@ -321,7 +321,7 @@ YencDecoderEnd do_decode_end_scalar(const unsigned char* HEDLEY_RESTRICT* src, u
 }
 
 template<bool isRaw, bool searchEnd>
-YencDecoderEnd do_decode_scalar(const unsigned char* HEDLEY_RESTRICT* src, unsigned char* HEDLEY_RESTRICT* dest, size_t len, YencDecoderState* state) {
+YencDecoderEnd do_decode_scalar(const unsigned char** src, unsigned char** dest, size_t len, YencDecoderState* state) {
 	if(searchEnd)
 		return do_decode_end_scalar<isRaw>(src, dest, len, state);
 	*dest += do_decode_noend_scalar<isRaw>(*src, *dest, len, state);
@@ -331,8 +331,8 @@ YencDecoderEnd do_decode_scalar(const unsigned char* HEDLEY_RESTRICT* src, unsig
 
 
 
-template<bool isRaw, bool searchEnd, int width, void(&kernel)(const uint8_t* HEDLEY_RESTRICT, long&, unsigned char* HEDLEY_RESTRICT &, unsigned char&, uint16_t&)>
-YencDecoderEnd do_decode_simd(const unsigned char* HEDLEY_RESTRICT* src, unsigned char* HEDLEY_RESTRICT* dest, size_t len, YencDecoderState* state) {
+template<bool isRaw, bool searchEnd, int width, void(&kernel)(const uint8_t*, long&, unsigned char*&, unsigned char&, uint16_t&)>
+YencDecoderEnd do_decode_simd(const unsigned char** src, unsigned char** dest, size_t len, YencDecoderState* state) {
 	if(len <= width*2) return do_decode_scalar<isRaw, searchEnd>(src, dest, len, state);
 	
 	YencDecoderState tState = YDEC_STATE_CRLF;

package/src/decoder_neon.cc

@@ -19,14 +19,14 @@
 #endif
 
 
-// for compilers that lack these functions
-#if defined(__clang__) || (defined(__GNUC__) && (defined(__aarch64__) && __GNUC__ >= 8))
+// for compilers that lack these functions (Clang armv7 9-12 seems to have issues with multi-vector loads)
+#if (defined(__clang__) && (defined(__aarch64__) || __clang_major__<9 || __clang_major__>12)) || (defined(__GNUC__) && (defined(__aarch64__) && __GNUC__ >= 8))
 # define vld1q_u8_x2_align(p, n) vld1q_u8_x2((uint8_t*)__builtin_assume_aligned(p, n))
 #else
 # define vld1q_u8_x2_align(p, n) vcreate2_u8(vld1q_u8_align(p, (n)/2), vld1q_u8_align((p)+16, (n)/2))
 #endif
 // Clang wrongly assumes alignment on vld1q_u8_x2, and ARMv7 GCC doesn't support the function, so effectively, it can only be used in ARMv8 compilers
-#if defined(__aarch64__) && (defined(__clang__) || (defined(__GNUC__) && __GNUC__ >= 9))
+#if defined(__aarch64__) && (defined(__clang__) || HEDLEY_GCC_VERSION_CHECK(8,5,0))
 # define vst1q_u8_x2_unaligned vst1q_u8_x2
 #else
 static HEDLEY_ALWAYS_INLINE void vst1q_u8_x2_unaligned(uint8_t* p, uint8x16x2_t data) {
@@ -59,7 +59,7 @@ static bool neon_vect_is_nonzero(uint8x16_t v) {
 
 
 template<bool isRaw, bool searchEnd>
-HEDLEY_ALWAYS_INLINE void do_decode_neon(const uint8_t* HEDLEY_RESTRICT src, long& len, unsigned char* HEDLEY_RESTRICT & p, unsigned char& escFirst, uint16_t& nextMask) {
+HEDLEY_ALWAYS_INLINE void do_decode_neon(const uint8_t* src, long& len, unsigned char*& p, unsigned char& escFirst, uint16_t& nextMask) {
 	HEDLEY_ASSUME(escFirst == 0 || escFirst == 1);
 	HEDLEY_ASSUME(nextMask == 0 || nextMask == 1 || nextMask == 2);
 	uint8x16_t yencOffset = escFirst ? vmakeq_u8(42+64,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42) : vdupq_n_u8(42);

package/src/decoder_neon64.cc

@@ -10,9 +10,9 @@ static struct { char bytes[16]; } ALIGN_TO(16, compactLUT[32768]);
 static uint8_t eqFixLUT[256];
 
 
-
-#if !defined(__clang__) && !defined(_MSC_VER) && (!defined(__aarch64__) || !HEDLEY_GCC_VERSION_CHECK(10,0,0))
-static HEDLEY_ALWAYS_INLINE uint8x16x4_t vld1q_u8_x4(const uint8_t* p) {
+// AArch64 GCC lacks these functions until 8.5, 9.4 and 10.1 (10.0 unknown)
+#if !defined(__clang__) && !defined(_MSC_VER) && (!defined(__aarch64__) || !(HEDLEY_GCC_VERSION_CHECK(9,4,0) || (!HEDLEY_GCC_VERSION_CHECK(9,0,0) && HEDLEY_GCC_VERSION_CHECK(8,5,0))))
+static HEDLEY_ALWAYS_INLINE uint8x16x4_t _vld1q_u8_x4(const uint8_t* p) {
 	uint8x16x4_t ret;
 	ret.val[0] = vld1q_u8(p);
 	ret.val[1] = vld1q_u8(p+16);
@@ -20,12 +20,15 @@ static HEDLEY_ALWAYS_INLINE uint8x16x4_t vld1q_u8_x4(const uint8_t* p) {
 	ret.val[3] = vld1q_u8(p+48);
 	return ret;
 }
-static HEDLEY_ALWAYS_INLINE void vst1q_u8_x4(uint8_t* p, uint8x16x4_t data) {
+static HEDLEY_ALWAYS_INLINE void _vst1q_u8_x4(uint8_t* p, uint8x16x4_t data) {
 	vst1q_u8(p, data.val[0]);
 	vst1q_u8(p+16, data.val[1]);
 	vst1q_u8(p+32, data.val[2]);
 	vst1q_u8(p+48, data.val[3]);
 }
+#else
+# define _vld1q_u8_x4 vld1q_u8_x4
+# define _vst1q_u8_x4 vst1q_u8_x4
 #endif
 
 
@@ -44,7 +47,7 @@ static HEDLEY_ALWAYS_INLINE uint8x16_t mergeCompares(uint8x16_t a, uint8x16_t b,
 
 
 template<bool isRaw, bool searchEnd>
-HEDLEY_ALWAYS_INLINE void do_decode_neon(const uint8_t* HEDLEY_RESTRICT src, long& len, unsigned char* HEDLEY_RESTRICT & p, unsigned char& escFirst, uint16_t& nextMask) {
+HEDLEY_ALWAYS_INLINE void do_decode_neon(const uint8_t* src, long& len, unsigned char*& p, unsigned char& escFirst, uint16_t& nextMask) {
 	HEDLEY_ASSUME(escFirst == 0 || escFirst == 1);
 	HEDLEY_ASSUME(nextMask == 0 || nextMask == 1 || nextMask == 2);
 	uint8x16_t nextMaskMix = vdupq_n_u8(0);
@@ -55,7 +58,7 @@ HEDLEY_ALWAYS_INLINE void do_decode_neon(const uint8_t* HEDLEY_RESTRICT src, lon
 	uint8x16_t yencOffset = escFirst ? vmakeq_u8(42+64,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42) : vdupq_n_u8(42);
 	long i;
 	for(i = -len; i; i += sizeof(uint8x16_t)*4) {
-		uint8x16x4_t data = vld1q_u8_x4(src+i);
+		uint8x16x4_t data = _vld1q_u8_x4(src+i);
 		uint8x16_t dataA = data.val[0];
 		uint8x16_t dataB = data.val[1];
 		uint8x16_t dataC = data.val[2];
@@ -421,7 +424,7 @@ HEDLEY_ALWAYS_INLINE void do_decode_neon(const uint8_t* HEDLEY_RESTRICT src, lon
 			dataB = vsubq_u8(dataB, vdupq_n_u8(42));
 			dataC = vsubq_u8(dataC, vdupq_n_u8(42));
 			dataD = vsubq_u8(dataD, vdupq_n_u8(42));
-			vst1q_u8_x4(p, vcreate4_u8(dataA, dataB, dataC, dataD));
+			_vst1q_u8_x4(p, vcreate4_u8(dataA, dataB, dataC, dataD));
 			p += sizeof(uint8x16_t)*4;
 			escFirst = 0;
 			yencOffset = vdupq_n_u8(42);

package/src/decoder_sse_base.h

@@ -7,8 +7,15 @@
 # define _mm_shrdi_epi16 _mm128_shrdi_epi16
 #endif
 
+#if defined(__tune_icelake_client__) || defined(__tune_icelake_server__) || defined(__tune_tigerlake__) || defined(__tune_rocketlake__) || defined(__tune_alderlake__) || defined(__tune_sapphirerapids__)
+# define COMPRESS_STORE _mm_mask_compressstoreu_epi8
+#else
+// avoid uCode on Zen4
+# define COMPRESS_STORE(dst, mask, vec) _mm_storeu_si128((__m128i*)(dst), _mm_maskz_compress_epi8(mask, vec))
+#endif
+
 // GCC (ver 6-10(dev)) fails to optimize pure C version of mask testing, but has this intrinsic; Clang >= 7 optimizes C version fine
-#if defined(__GNUC__) && __GNUC__ >= 7
+#if (defined(__GNUC__) && __GNUC__ >= 7) || (defined(_MSC_VER) && _MSC_VER >= 1924)
 # define KORTEST16(a, b) !_kortestz_mask16_u8((a), (b))
 # define KAND16(a, b) _kand_mask16((a), (b))
 # define KOR16(a, b) _kor_mask16((a), (b))
@@ -104,7 +111,7 @@ static HEDLEY_ALWAYS_INLINE __m128i sse2_compact_vect(uint32_t mask, __m128i dat
 }
 
 template<bool isRaw, bool searchEnd, enum YEncDecIsaLevel use_isa>
-HEDLEY_ALWAYS_INLINE void do_decode_sse(const uint8_t* HEDLEY_RESTRICT src, long& len, unsigned char* HEDLEY_RESTRICT & p, unsigned char& _escFirst, uint16_t& _nextMask) {
+HEDLEY_ALWAYS_INLINE void do_decode_sse(const uint8_t* src, long& len, unsigned char*& p, unsigned char& _escFirst, uint16_t& _nextMask) {
 	HEDLEY_ASSUME(_escFirst == 0 || _escFirst == 1);
 	HEDLEY_ASSUME(_nextMask == 0 || _nextMask == 1 || _nextMask == 2);
 	uintptr_t escFirst = _escFirst;
@@ -112,7 +119,7 @@ HEDLEY_ALWAYS_INLINE void do_decode_sse(const uint8_t* HEDLEY_RESTRICT src, long
 		-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42-64
 	) : _mm_set1_epi8(-42);
 	
-#if defined(__SSSE3__) && !defined(__tune_atom__) && !defined(__tune_slm__) && !defined(__tune_btver1__)
+#if defined(__SSSE3__) && !defined(__tune_atom__) && !defined(__tune_slm__) && !defined(__tune_btver1__) && !defined(__tune_btver2__)
 	const bool _USING_FAST_MATCH = (use_isa >= ISA_LEVEL_SSSE3);
 #else
 	const bool _USING_FAST_MATCH = false;
@@ -121,6 +128,13 @@ HEDLEY_ALWAYS_INLINE void do_decode_sse(const uint8_t* HEDLEY_RESTRICT src, long
 	const bool _USING_BLEND_ADD = (use_isa >= ISA_LEVEL_SSE41);
 #else
 	const bool _USING_BLEND_ADD = false;
+#endif
+#if defined(__AVX512VL__) && defined(__AVX512BW__)
+# if defined(_MSC_VER) && !defined(PLATFORM_AMD64) && !defined(__clang__)
+	const bool useAVX3MaskCmp = false;
+# else
+	const bool useAVX3MaskCmp = (use_isa >= ISA_LEVEL_AVX3);
+# endif
 #endif
 	
 	__m128i lfCompare = _mm_set1_epi8('\n');
@@ -214,7 +228,7 @@ HEDLEY_ALWAYS_INLINE void do_decode_sse(const uint8_t* HEDLEY_RESTRICT src, long
 				__mmask16 match2EqMaskA, match2EqMaskB;
 				__mmask16 match0CrMaskA, match0CrMaskB;
 				__mmask16 match2CrXDtMaskA, match2CrXDtMaskB;
-				if(use_isa >= ISA_LEVEL_AVX3 && searchEnd) {
+				if(useAVX3MaskCmp && searchEnd) {
 					match2EqMaskA = _mm_cmpeq_epi8_mask(_mm_set1_epi8('='), tmpData2A);
 					match2EqMaskB = _mm_cmpeq_epi8_mask(_mm_set1_epi8('='), tmpData2B);
 				} else
@@ -230,7 +244,7 @@ HEDLEY_ALWAYS_INLINE void do_decode_sse(const uint8_t* HEDLEY_RESTRICT src, long
 				__m128i match2CrXDtA, match2CrXDtB;
 				if(isRaw) {
 #if defined(__AVX512VL__) && defined(__AVX512BW__)
-					if(use_isa >= ISA_LEVEL_AVX3) {
+					if(useAVX3MaskCmp) {
 						match0CrMaskA = _mm_cmpeq_epi8_mask(oDataA, _mm_set1_epi8('\r'));
 						match0CrMaskB = _mm_cmpeq_epi8_mask(oDataB, _mm_set1_epi8('\r'));
 						match2CrXDtMaskA = _mm_mask_cmpeq_epi8_mask(match0CrMaskA, tmpData2A, _mm_set1_epi8('.'));
@@ -256,7 +270,7 @@ HEDLEY_ALWAYS_INLINE void do_decode_sse(const uint8_t* HEDLEY_RESTRICT src, long
 #if defined(__AVX512VL__) && defined(__AVX512BW__)
 					__mmask16 match1NlMaskA, match1NlMaskB;
 					__mmask16 match2NlDotMaskA, match2NlDotMaskB;
-					if(use_isa >= ISA_LEVEL_AVX3) {
+					if(useAVX3MaskCmp) {
 						match1NlMaskA = _mm_mask_cmpeq_epi8_mask(
 							match0CrMaskA,
 							_mm_set1_epi8('\n'),
@@ -299,7 +313,7 @@ HEDLEY_ALWAYS_INLINE void do_decode_sse(const uint8_t* HEDLEY_RESTRICT src, long
 						
 						int matchEnd;
 #if defined(__AVX512VL__) && defined(__AVX512BW__)
-						if(use_isa >= ISA_LEVEL_AVX3) {
+						if(useAVX3MaskCmp) {
 							__mmask16 match3EqYMaskA = _mm_mask_cmpeq_epi8_mask(
 								match2EqMaskA, _mm_set1_epi8('y'), tmpData3A
 							);
@@ -373,7 +387,7 @@ HEDLEY_ALWAYS_INLINE void do_decode_sse(const uint8_t* HEDLEY_RESTRICT src, long
 						}
 					}
 #if defined(__AVX512VL__) && defined(__AVX512BW__)
-					if(use_isa >= ISA_LEVEL_AVX3) {
+					if(useAVX3MaskCmp) {
 						mask |= match2NlDotMaskA << 2;
 						mask |= (match2NlDotMaskB << 18) & 0xffffffff;
 						minMask = _mm_maskz_mov_epi8(~(match2NlDotMaskB>>14), _mm_set1_epi8('.'));
@@ -398,7 +412,7 @@ HEDLEY_ALWAYS_INLINE void do_decode_sse(const uint8_t* HEDLEY_RESTRICT src, long
 					__m128i match3EqYA, match3EqYB;
 #if defined(__AVX512VL__) && defined(__AVX512BW__)
 					__mmask16 match3EqYMaskA, match3EqYMaskB;
-					if(use_isa >= ISA_LEVEL_AVX3) {
+					if(useAVX3MaskCmp) {
 						match3EqYMaskA = _mm_mask_cmpeq_epi8_mask(
 							match2EqMaskA,
 							_mm_set1_epi8('y'),
@@ -434,7 +448,7 @@ HEDLEY_ALWAYS_INLINE void do_decode_sse(const uint8_t* HEDLEY_RESTRICT src, long
 						bool endFound;
 						
 #if defined(__AVX512VL__) && defined(__AVX512BW__)
-						if(use_isa >= ISA_LEVEL_AVX3) {
+						if(useAVX3MaskCmp) {
 							__mmask16 match3LfEqYMaskA = _mm_mask_cmpeq_epi8_mask(
 								match3EqYMaskA,
 								_mm_set1_epi8('\n'),
@@ -642,9 +656,9 @@ HEDLEY_ALWAYS_INLINE void do_decode_sse(const uint8_t* HEDLEY_RESTRICT src, long
 			if(use_isa >= ISA_LEVEL_SSSE3) {
 # if defined(__AVX512VBMI2__) && defined(__AVX512VL__) && defined(__POPCNT__)
 				if(use_isa >= ISA_LEVEL_VBMI2) {
-					_mm_mask_compressstoreu_epi8(p, KNOT16(mask), dataA);
+					COMPRESS_STORE(p, KNOT16(mask), dataA);
 					p -= popcnt32(mask & 0xffff);
-					_mm_mask_compressstoreu_epi8(p+XMM_SIZE, KNOT16(mask>>16), dataB);
+					COMPRESS_STORE(p+XMM_SIZE, KNOT16(mask>>16), dataB);
 					p -= popcnt32(mask>>16);
 					p += XMM_SIZE*2;
 				} else

package/src/decoder_vbmi2.cc

@@ -0,0 +1,37 @@
+#include "common.h"
+
+extern const bool decoder_has_avx10;
+#if !defined(__EVEX512__) && (defined(__AVX10_1__) || defined(__EVEX256__)) && defined(__AVX512VL__) && defined(__AVX512VBMI2__) && defined(__AVX512BW__)
+const bool decoder_has_avx10 = true;
+#else
+const bool decoder_has_avx10 = false;
+#endif
+
+#if defined(__AVX512VL__) && defined(__AVX512VBMI2__) && defined(__AVX512BW__)
+# include "decoder_common.h"
+# ifndef YENC_DISABLE_AVX256
+#  include "decoder_avx2_base.h"
+void decoder_set_vbmi2_funcs() {
+	ALIGN_ALLOC(lookups, sizeof(*lookups), 16);
+	// TODO: consider removing compact LUT
+	decoder_init_lut(lookups->eqFix, lookups->compact);
+	_do_decode = &do_decode_simd<false, false, sizeof(__m256i)*2, do_decode_avx2<false, false, ISA_LEVEL_VBMI2> >;
+	_do_decode_raw = &do_decode_simd<true, false, sizeof(__m256i)*2, do_decode_avx2<true, false, ISA_LEVEL_VBMI2> >;
+	_do_decode_end_raw = &do_decode_simd<true, true, sizeof(__m256i)*2, do_decode_avx2<true, true, ISA_LEVEL_VBMI2> >;
+}
+# else
+#  include "decoder_sse_base.h"
+void decoder_set_vbmi2_funcs() {
+	decoder_sse_init();
+	decoder_init_lut(lookups->eqFix, lookups->compact);
+	_do_decode = &do_decode_simd<false, false, sizeof(__m128i)*2, do_decode_sse<false, false, ISA_LEVEL_VBMI2> >;
+	_do_decode_raw = &do_decode_simd<true, false, sizeof(__m128i)*2, do_decode_sse<true, false, ISA_LEVEL_VBMI2> >;
+	_do_decode_end_raw = &do_decode_simd<true, true, sizeof(__m128i)*2, do_decode_sse<true, true, ISA_LEVEL_VBMI2> >;
+}
+# endif
+#else
+void decoder_set_avx2_funcs();
+void decoder_set_vbmi2_funcs() {
+	decoder_set_avx2_funcs();
+}
+#endif

package/src/encoder.cc

@@ -128,7 +128,10 @@ void encoder_sse2_init();
 void encoder_ssse3_init();
 void encoder_avx_init();
 void encoder_avx2_init();
+void encoder_vbmi2_init();
+extern const bool encoder_has_avx10;
 void encoder_neon_init();
+void encoder_rvv_init();
 
 #if defined(PLATFORM_X86) && defined(YENC_BUILD_NATIVE) && YENC_BUILD_NATIVE!=0
 # if defined(__AVX2__) && !defined(YENC_DISABLE_AVX256)
@@ -153,7 +156,9 @@ void encoder_init() {
 	encoder_native_init();
 # else
 	int use_isa = cpu_supports_isa();
-	if(use_isa >= ISA_LEVEL_AVX2)
+	if(use_isa >= ISA_LEVEL_VBMI2 && (encoder_has_avx10 || (use_isa & ISA_FEATURE_EVEX512)))
+		encoder_vbmi2_init();
+	else if(use_isa >= ISA_LEVEL_AVX2)
 		encoder_avx2_init();
 	else if(use_isa >= ISA_LEVEL_AVX)
 		encoder_avx_init();
@@ -167,4 +172,8 @@ void encoder_init() {
 	if(cpu_supports_neon())
 		encoder_neon_init();
 #endif
+#ifdef __riscv
+	if(cpu_supports_rvv())
+		encoder_rvv_init();
+#endif
 }

package/src/encoder_avx_base.h

@@ -6,7 +6,7 @@
 #include "encoder_common.h"
 #define YMM_SIZE 32
 
-#if defined(__GNUC__) && __GNUC__ >= 7
+#if (defined(__GNUC__) && __GNUC__ >= 7) || (defined(_MSC_VER) && _MSC_VER >= 1924)
 # define KLOAD32(a, offs) _load_mask32((__mmask32*)(a) + (offs))
 #else
 # define KLOAD32(a, offs) (((uint32_t*)(a))[(offs)])
@@ -215,7 +215,7 @@ HEDLEY_ALWAYS_INLINE void do_encode_avx2(int line_size, int* colOffset, const ui
 				// duplicate halves
 				data1A = _mm256_inserti128_si256(dataA, _mm256_castsi256_si128(dataA), 1);
 				data1B = _mm256_inserti128_si256(dataB, _mm256_castsi256_si128(dataB), 1);
-#if defined(__tune_znver2__) || defined(__tune_znver3__)
+#if defined(__tune_znver2__) || defined(__tune_znver3__) || defined(__tune_znver4__)
 				data2A = _mm256_permute2x128_si256(dataA, dataA, 0x11);
 				data2B = _mm256_permute2x128_si256(dataB, dataB, 0x11);
 #else
@@ -290,10 +290,10 @@ HEDLEY_ALWAYS_INLINE void do_encode_avx2(int line_size, int* colOffset, const ui
 				
 #if defined(__GNUC__) && defined(PLATFORM_AMD64)
 				if(use_isa >= ISA_LEVEL_VBMI2) {
-					asm(
+					__asm__(
 						"shrq $1, %[eqMask] \n"
 						"shrq %%cl, %[eqMask] \n"
-						"adcq %[col], %[p] \n"
+						"adcq %q[col], %q[p] \n"
 						: [eqMask]"+r"(eqMask), [p]"+r"(p)
 						: "c"(shiftAmt), [col]"r"(~col)
 					);
@@ -334,28 +334,32 @@ HEDLEY_ALWAYS_INLINE void do_encode_avx2(int line_size, int* colOffset, const ui
 			if(use_isa >= ISA_LEVEL_AVX3) {
 # if defined(__AVX512VBMI2__)
 				if(use_isa >= ISA_LEVEL_VBMI2) {
-					_mm256_mask_storeu_epi8(p+1, 1UL<<31, dataA);
+					__m128i dataTop = _mm256_extracti128_si256(dataA, 1);
 					dataA = _mm256_mask_expand_epi8(_mm256_set1_epi8('='), KNOT32(maskA), dataA);
 					_mm256_storeu_si256((__m256i*)p, dataA);
+					p[32] = _mm_extract_epi8(dataTop, 15);
 					p += outputBytesA;
 					
-					_mm256_mask_storeu_epi8(p+1, 1UL<<31, dataB);
+					dataTop = _mm256_extracti128_si256(dataB, 1);
 					dataB = _mm256_mask_expand_epi8(_mm256_set1_epi8('='), KNOT32(maskB), dataB);
 					_mm256_storeu_si256((__m256i*)p, dataB);
+					p[32] = _mm_extract_epi8(dataTop, 15);
 					p += maskBitsB;
 				} else
 # endif
 				{
-					_mm256_mask_storeu_epi8(p+1, 1UL<<31, dataA);
-					dataA = _mm256_mask_alignr_epi8(dataA, (uint32_t)(-(int32_t)maskA), dataA, _mm256_permute4x64_epi64(dataA, _MM_SHUFFLE(1,0,3,2)), 15);
+					__m256i dataSwapped = _mm256_permute4x64_epi64(dataA, _MM_SHUFFLE(1,0,3,2));
+					dataA = _mm256_mask_alignr_epi8(dataA, (uint32_t)(-(int32_t)maskA), dataA, dataSwapped, 15);
 					dataA = _mm256_ternarylogic_epi32(dataA, cmpA, _mm256_set1_epi8('='), 0xb8); // (data & ~cmp) | (cmp & '=')
 					_mm256_storeu_si256((__m256i*)p, dataA);
+					p[32] = _mm_extract_epi8(_mm256_castsi256_si128(dataSwapped), 15);
 					p += outputBytesA;
 					
-					_mm256_mask_storeu_epi8(p+1, 1UL<<31, dataB);
-					dataB = _mm256_mask_alignr_epi8(dataB, (uint32_t)(-(int32_t)maskB), dataB, _mm256_permute4x64_epi64(dataB, _MM_SHUFFLE(1,0,3,2)), 15);
+					dataSwapped = _mm256_permute4x64_epi64(dataB, _MM_SHUFFLE(1,0,3,2));
+					dataB = _mm256_mask_alignr_epi8(dataB, (uint32_t)(-(int32_t)maskB), dataB, dataSwapped, 15);
 					dataB = _mm256_ternarylogic_epi32(dataB, cmpB, _mm256_set1_epi8('='), 0xb8);
 					_mm256_storeu_si256((__m256i*)p, dataB);
+					p[32] = _mm_extract_epi8(_mm256_castsi256_si128(dataSwapped), 15);
 					p += maskBitsB;
 				}
 			} else
@@ -484,28 +488,32 @@ HEDLEY_ALWAYS_INLINE void do_encode_avx2(int line_size, int* colOffset, const ui
 				if(use_isa >= ISA_LEVEL_AVX3) {
 # if defined(__AVX512VBMI2__)
 					if(use_isa >= ISA_LEVEL_VBMI2) {
-						_mm256_mask_storeu_epi8(p+1, 1UL<<31, dataA);
+						__m128i dataTop = _mm256_extracti128_si256(dataA, 1);
 						dataA = _mm256_mask_expand_epi8(_mm256_set1_epi8('='), KNOT32(maskA), dataA);
 						_mm256_storeu_si256((__m256i*)p, dataA);
+						p[32] = _mm_extract_epi8(dataTop, 15);
 						p += outputBytesA;
 						
-						_mm256_mask_storeu_epi8(p+1, 1UL<<31, dataB);
+						dataTop = _mm256_extracti128_si256(dataB, 1);
 						dataB = _mm256_mask_expand_epi8(_mm256_set1_epi8('='), KNOT32(maskB), dataB);
 						_mm256_storeu_si256((__m256i*)p, dataB);
+						p[32] = _mm_extract_epi8(dataTop, 15);
 						p += maskBitsB;
 					} else
 # endif
 					{
-						_mm256_mask_storeu_epi8(p+1, 1UL<<31, dataA);
-						dataA = _mm256_mask_alignr_epi8(dataA, (uint32_t)(-(int32_t)maskA), dataA, _mm256_permute4x64_epi64(dataA, _MM_SHUFFLE(1,0,3,2)), 15);
+						__m256i dataSwapped = _mm256_permute4x64_epi64(dataA, _MM_SHUFFLE(1,0,3,2));
+						dataA = _mm256_mask_alignr_epi8(dataA, (uint32_t)(-(int32_t)maskA), dataA, dataSwapped, 15);
 						dataA = _mm256_ternarylogic_epi32(dataA, cmpA, _mm256_set1_epi8('='), 0xb8); // (data & ~cmp) | (cmp & '=')
 						_mm256_storeu_si256((__m256i*)p, dataA);
+						p[32] = _mm_extract_epi8(_mm256_castsi256_si128(dataSwapped), 15);
 						p += outputBytesA;
 						
-						_mm256_mask_storeu_epi8(p+1, 1UL<<31, dataB);
-						dataB = _mm256_mask_alignr_epi8(dataB, (uint32_t)(-(int32_t)maskB), dataB, _mm256_permute4x64_epi64(dataB, _MM_SHUFFLE(1,0,3,2)), 15);
+						dataSwapped = _mm256_permute4x64_epi64(dataB, _MM_SHUFFLE(1,0,3,2));
+						dataB = _mm256_mask_alignr_epi8(dataB, (uint32_t)(-(int32_t)maskB), dataB, dataSwapped, 15);
 						dataB = _mm256_ternarylogic_epi32(dataB, cmpB, _mm256_set1_epi8('='), 0xb8);
 						_mm256_storeu_si256((__m256i*)p, dataB);
+						p[32] = _mm_extract_epi8(_mm256_castsi256_si128(dataSwapped), 15);
 						p += maskBitsB;
 					}
 				} else

package/src/encoder_neon.cc

@@ -5,7 +5,7 @@
 #include "encoder_common.h"
 
 // Clang wrongly assumes alignment on vst1q_u8_x2, and ARMv7 GCC doesn't support the function, so effectively, it can only be used in ARMv8 compilers
-#if defined(__aarch64__) && (defined(__clang__) || (defined(__GNUC__) && __GNUC__ >= 9))
+#if defined(__aarch64__) && (defined(__clang__) || HEDLEY_GCC_VERSION_CHECK(8,5,0))
 # define vst1q_u8_x2_unaligned vst1q_u8_x2
 #else
 static HEDLEY_ALWAYS_INLINE void vst1q_u8_x2_unaligned(uint8_t* p, uint8x16x2_t data) {
@@ -15,6 +15,43 @@ static HEDLEY_ALWAYS_INLINE void vst1q_u8_x2_unaligned(uint8_t* p, uint8x16x2_t
 #endif
 
 
+// ARM's CLZ instruction at native bit-width
+#ifdef __aarch64__
+static HEDLEY_ALWAYS_INLINE int clz_n(uint64_t v) {
+# ifdef _MSC_VER
+	long r;
+	// does this work?
+	if(_BitScanReverse64((unsigned long*)&r, v))
+		r ^= 63;
+	else
+		r = 64;
+	return r;
+# else
+#  if defined(__clang__) || HEDLEY_GCC_VERSION_CHECK(11,0,0)
+	// this pattern is only detected on GCC >= 11 (Clang 9 seems to as well, unsure about earlier versions)
+	// - note: return type must be 'int'; GCC fails to optimise this if type is 'long'
+	// GCC <= 10 doesn't optimize around the '0 = undefined behaviour', so not needed there
+	if(v == 0) return 64;
+#  endif
+	return __builtin_clzll(v);
+# endif
+}
+#else
+static HEDLEY_ALWAYS_INLINE int clz_n(uint32_t v) {
+# ifdef __GNUC__
+#  if defined(__clang__) || HEDLEY_GCC_VERSION_CHECK(7,0,0)
+	// as with AArch64 version above, only insert this check if compiler can optimise it away
+	if(v == 0) return 32;
+#  endif
+	return __builtin_clz(v);
+# elif defined(_MSC_VER)
+	return _arm_clz(v);
+# else
+	return __clz(v); // ARM compiler?
+# endif
+}
+#endif
+
 static uint8x16_t ALIGN_TO(16, shufLUT[256]);
 static uint16_t expandLUT[256];
 
@@ -195,26 +232,7 @@ static HEDLEY_ALWAYS_INLINE void encode_eol_handle_pre(const uint8_t* HEDLEY_RES
 		col = shufTotalLen+1 + lineSizeOffset-32;
 	} else {
 		// shuffle stuff up
-#ifdef __aarch64__
-# ifdef _MSC_VER
-		long bitIndex;
-		if(_BitScanReverse64((unsigned long*)&bitIndex, mask))
-			bitIndex ^= 63;
-		else
-			bitIndex = 64;
-# else
-		long bitIndex = __builtin_clzll(mask);
-# endif
-#else
-# ifdef __GNUC__
-		long bitIndex = __builtin_clz(mask); // TODO: is the 'undefined if 0' case problematic here?
-# elif defined(_MSC_VER)
-		long bitIndex = _arm_clz(mask);
-# else
-		long bitIndex = __clz(mask); // ARM compiler?
-# endif
-#endif
-		
+		long bitIndex = clz_n(mask);
 		uint8x16_t vClz = vdupq_n_u8(bitIndex & ~(sizeof(mask)*8));
 #ifdef __aarch64__
 		uint8x16_t blendA = vcgtq_u8(vmakeq_u8(63,62,61,60,51,50,49,48,47,46,45,44,35,34,33,32), vClz);
@@ -450,26 +468,7 @@ HEDLEY_ALWAYS_INLINE void do_encode_neon(int line_size, int* colOffset, const ui
 			}
 		} else {
 			{
-#ifdef __aarch64__
-# ifdef _MSC_VER
-				// does this work?
-				if(_BitScanReverse64((unsigned long*)&bitIndex, mask))
-					bitIndex ^= 63;
-				else
-					bitIndex = 64;
-# else
-				bitIndex = __builtin_clzll(mask); // TODO: is the 'undefined if 0' case problematic here?
-# endif
-#else
-# ifdef __GNUC__
-				bitIndex = __builtin_clz(mask);
-# elif defined(_MSC_VER)
-				bitIndex = _arm_clz(mask);
-# else
-				bitIndex = __clz(mask); // ARM compiler?
-# endif
-#endif
-				
+				bitIndex = clz_n(mask);
 				uint8x16_t vClz = vdupq_n_u8(bitIndex & ~(sizeof(mask)*8));
 #ifdef __aarch64__
 				uint8x16_t blendA = vcgeq_u8(vmakeq_u8(63,62,61,60,51,50,49,48,47,46,45,44,35,34,33,32), vClz);