yencode 1.1.3 → 1.1.4

package/src/decoder_avx2_base.h

@@ -30,7 +30,7 @@ static HEDLEY_ALWAYS_INLINE __m256i force_align_read_256(const void* p) {
 }
 
 // _mm256_castsi128_si256, but upper is defined to be 0
-#if (defined(__clang__) && __clang_major__ >= 5 && (!defined(__APPLE__) || __clang_major__ >= 7)) || (defined(__GNUC__) && __GNUC__ >= 10)
+#if (defined(__clang__) && __clang_major__ >= 5 && (!defined(__APPLE__) || __clang_major__ >= 7)) || (defined(__GNUC__) && __GNUC__ >= 10) || (defined(_MSC_VER) && _MSC_VER >= 1910)
 // intrinsic unsupported in GCC 9 and MSVC < 2017
 # define zext128_256 _mm256_zextsi128_si256
 #else
@@ -43,9 +43,15 @@ static HEDLEY_ALWAYS_INLINE __m256i force_align_read_256(const void* p) {
 # endif
 #endif
 
+#if defined(__tune_icelake_client__) || defined(__tune_icelake_server__) || defined(__tune_tigerlake__) || defined(__tune_rocketlake__) || defined(__tune_alderlake__) || defined(__tune_sapphirerapids__)
+# define COMPRESS_STORE _mm256_mask_compressstoreu_epi8
+#else
+// avoid uCode on Zen4
+# define COMPRESS_STORE(dst, mask, vec) _mm256_storeu_si256((__m256i*)(dst), _mm256_maskz_compress_epi8(mask, vec))
+#endif
 
 template<bool isRaw, bool searchEnd, enum YEncDecIsaLevel use_isa>
-HEDLEY_ALWAYS_INLINE void do_decode_avx2(const uint8_t* HEDLEY_RESTRICT src, long& len, unsigned char* HEDLEY_RESTRICT & p, unsigned char& _escFirst, uint16_t& _nextMask) {
+HEDLEY_ALWAYS_INLINE void do_decode_avx2(const uint8_t* src, long& len, unsigned char*& p, unsigned char& _escFirst, uint16_t& _nextMask) {
 	HEDLEY_ASSUME(_escFirst == 0 || _escFirst == 1);
 	HEDLEY_ASSUME(_nextMask == 0 || _nextMask == 1 || _nextMask == 2);
 	uintptr_t escFirst = _escFirst;
@@ -541,9 +547,9 @@ HEDLEY_ALWAYS_INLINE void do_decode_avx2(const uint8_t* HEDLEY_RESTRICT src, lon
 			// all that's left is to 'compress' the data (skip over masked chars)
 #if defined(__AVX512VBMI2__) && defined(__AVX512VL__)
 			if(use_isa >= ISA_LEVEL_VBMI2) {
-				_mm256_mask_compressstoreu_epi8(p, KNOT32(mask), dataA);
+				COMPRESS_STORE(p, KNOT32(mask), dataA);
 				p -= popcnt32(mask & 0xffffffff);
-				_mm256_mask_compressstoreu_epi8((p + XMM_SIZE*2), KNOT32(mask>>32), dataB);
+				COMPRESS_STORE((p + XMM_SIZE*2), KNOT32(mask>>32), dataB);
 				p += XMM_SIZE*4 - popcnt32(mask >> 32);
 			} else
 #endif

package/src/decoder_common.h

@@ -6,7 +6,7 @@
 
 // state var: refers to the previous state - only used for incremental processing
 template<bool isRaw>
-size_t do_decode_noend_scalar(const unsigned char* HEDLEY_RESTRICT src, unsigned char* HEDLEY_RESTRICT dest, size_t len, YencDecoderState* state) {
+size_t do_decode_noend_scalar(const unsigned char* src, unsigned char* dest, size_t len, YencDecoderState* state) {
 	const unsigned char *es = src + len; // end source pointer
 	unsigned char *p = dest; // destination pointer
 	long i = -(long)len; // input position
@@ -140,7 +140,7 @@ size_t do_decode_noend_scalar(const unsigned char* HEDLEY_RESTRICT src, unsigned
 }
 
 template<bool isRaw>
-YencDecoderEnd do_decode_end_scalar(const unsigned char* HEDLEY_RESTRICT* src, unsigned char* HEDLEY_RESTRICT* dest, size_t len, YencDecoderState* state) {
+YencDecoderEnd do_decode_end_scalar(const unsigned char** src, unsigned char** dest, size_t len, YencDecoderState* state) {
 	const unsigned char *es = (*src) + len; // end source pointer
 	unsigned char *p = *dest; // destination pointer
 	long i = -(long)len; // input position
@@ -321,7 +321,7 @@ YencDecoderEnd do_decode_end_scalar(const unsigned char* HEDLEY_RESTRICT* src, u
 }
 
 template<bool isRaw, bool searchEnd>
-YencDecoderEnd do_decode_scalar(const unsigned char* HEDLEY_RESTRICT* src, unsigned char* HEDLEY_RESTRICT* dest, size_t len, YencDecoderState* state) {
+YencDecoderEnd do_decode_scalar(const unsigned char** src, unsigned char** dest, size_t len, YencDecoderState* state) {
 	if(searchEnd)
 		return do_decode_end_scalar<isRaw>(src, dest, len, state);
 	*dest += do_decode_noend_scalar<isRaw>(*src, *dest, len, state);
@@ -331,8 +331,8 @@ YencDecoderEnd do_decode_scalar(const unsigned char* HEDLEY_RESTRICT* src, unsig
 
 
 
-template<bool isRaw, bool searchEnd, int width, void(&kernel)(const uint8_t* HEDLEY_RESTRICT, long&, unsigned char* HEDLEY_RESTRICT &, unsigned char&, uint16_t&)>
-YencDecoderEnd do_decode_simd(const unsigned char* HEDLEY_RESTRICT* src, unsigned char* HEDLEY_RESTRICT* dest, size_t len, YencDecoderState* state) {
+template<bool isRaw, bool searchEnd, int width, void(&kernel)(const uint8_t*, long&, unsigned char*&, unsigned char&, uint16_t&)>
+YencDecoderEnd do_decode_simd(const unsigned char** src, unsigned char** dest, size_t len, YencDecoderState* state) {
 	if(len <= width*2) return do_decode_scalar<isRaw, searchEnd>(src, dest, len, state);
 	
 	YencDecoderState tState = YDEC_STATE_CRLF;

package/src/decoder_neon.cc

@@ -59,7 +59,7 @@ static bool neon_vect_is_nonzero(uint8x16_t v) {
 
 
 template<bool isRaw, bool searchEnd>
-HEDLEY_ALWAYS_INLINE void do_decode_neon(const uint8_t* HEDLEY_RESTRICT src, long& len, unsigned char* HEDLEY_RESTRICT & p, unsigned char& escFirst, uint16_t& nextMask) {
+HEDLEY_ALWAYS_INLINE void do_decode_neon(const uint8_t* src, long& len, unsigned char*& p, unsigned char& escFirst, uint16_t& nextMask) {
 	HEDLEY_ASSUME(escFirst == 0 || escFirst == 1);
 	HEDLEY_ASSUME(nextMask == 0 || nextMask == 1 || nextMask == 2);
 	uint8x16_t yencOffset = escFirst ? vmakeq_u8(42+64,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42) : vdupq_n_u8(42);

package/src/decoder_neon64.cc

@@ -47,7 +47,7 @@ static HEDLEY_ALWAYS_INLINE uint8x16_t mergeCompares(uint8x16_t a, uint8x16_t b,
 
 
 template<bool isRaw, bool searchEnd>
-HEDLEY_ALWAYS_INLINE void do_decode_neon(const uint8_t* HEDLEY_RESTRICT src, long& len, unsigned char* HEDLEY_RESTRICT & p, unsigned char& escFirst, uint16_t& nextMask) {
+HEDLEY_ALWAYS_INLINE void do_decode_neon(const uint8_t* src, long& len, unsigned char*& p, unsigned char& escFirst, uint16_t& nextMask) {
 	HEDLEY_ASSUME(escFirst == 0 || escFirst == 1);
 	HEDLEY_ASSUME(nextMask == 0 || nextMask == 1 || nextMask == 2);
 	uint8x16_t nextMaskMix = vdupq_n_u8(0);

package/src/decoder_sse_base.h

@@ -7,6 +7,13 @@
 # define _mm_shrdi_epi16 _mm128_shrdi_epi16
 #endif
 
+#if defined(__tune_icelake_client__) || defined(__tune_icelake_server__) || defined(__tune_tigerlake__) || defined(__tune_rocketlake__) || defined(__tune_alderlake__) || defined(__tune_sapphirerapids__)
+# define COMPRESS_STORE _mm_mask_compressstoreu_epi8
+#else
+// avoid uCode on Zen4
+# define COMPRESS_STORE(dst, mask, vec) _mm_storeu_si128((__m128i*)(dst), _mm_maskz_compress_epi8(mask, vec))
+#endif
+
 // GCC (ver 6-10(dev)) fails to optimize pure C version of mask testing, but has this intrinsic; Clang >= 7 optimizes C version fine
 #if (defined(__GNUC__) && __GNUC__ >= 7) || (defined(_MSC_VER) && _MSC_VER >= 1924)
 # define KORTEST16(a, b) !_kortestz_mask16_u8((a), (b))
@@ -104,7 +111,7 @@ static HEDLEY_ALWAYS_INLINE __m128i sse2_compact_vect(uint32_t mask, __m128i dat
 }
 
 template<bool isRaw, bool searchEnd, enum YEncDecIsaLevel use_isa>
-HEDLEY_ALWAYS_INLINE void do_decode_sse(const uint8_t* HEDLEY_RESTRICT src, long& len, unsigned char* HEDLEY_RESTRICT & p, unsigned char& _escFirst, uint16_t& _nextMask) {
+HEDLEY_ALWAYS_INLINE void do_decode_sse(const uint8_t* src, long& len, unsigned char*& p, unsigned char& _escFirst, uint16_t& _nextMask) {
 	HEDLEY_ASSUME(_escFirst == 0 || _escFirst == 1);
 	HEDLEY_ASSUME(_nextMask == 0 || _nextMask == 1 || _nextMask == 2);
 	uintptr_t escFirst = _escFirst;
@@ -649,9 +656,9 @@ HEDLEY_ALWAYS_INLINE void do_decode_sse(const uint8_t* HEDLEY_RESTRICT src, long
 			if(use_isa >= ISA_LEVEL_SSSE3) {
 # if defined(__AVX512VBMI2__) && defined(__AVX512VL__) && defined(__POPCNT__)
 				if(use_isa >= ISA_LEVEL_VBMI2) {
-					_mm_mask_compressstoreu_epi8(p, KNOT16(mask), dataA);
+					COMPRESS_STORE(p, KNOT16(mask), dataA);
 					p -= popcnt32(mask & 0xffff);
-					_mm_mask_compressstoreu_epi8(p+XMM_SIZE, KNOT16(mask>>16), dataB);
+					COMPRESS_STORE(p+XMM_SIZE, KNOT16(mask>>16), dataB);
 					p -= popcnt32(mask>>16);
 					p += XMM_SIZE*2;
 				} else

package/src/decoder_vbmi2.cc

@@ -1,5 +1,12 @@
 #include "common.h"
 
+extern const bool decoder_has_avx10;
+#if !defined(__EVEX512__) && (defined(__AVX10_1__) || defined(__EVEX256__)) && defined(__AVX512VL__) && defined(__AVX512VBMI2__) && defined(__AVX512BW__)
+const bool decoder_has_avx10 = true;
+#else
+const bool decoder_has_avx10 = false;
+#endif
+
 #if defined(__AVX512VL__) && defined(__AVX512VBMI2__) && defined(__AVX512BW__)
 # include "decoder_common.h"
 # ifndef YENC_DISABLE_AVX256

package/src/encoder.cc

@@ -129,7 +129,9 @@ void encoder_ssse3_init();
 void encoder_avx_init();
 void encoder_avx2_init();
 void encoder_vbmi2_init();
+extern const bool encoder_has_avx10;
 void encoder_neon_init();
+void encoder_rvv_init();
 
 #if defined(PLATFORM_X86) && defined(YENC_BUILD_NATIVE) && YENC_BUILD_NATIVE!=0
 # if defined(__AVX2__) && !defined(YENC_DISABLE_AVX256)
@@ -154,7 +156,7 @@ void encoder_init() {
 	encoder_native_init();
 # else
 	int use_isa = cpu_supports_isa();
-	if(use_isa >= ISA_LEVEL_VBMI2)
+	if(use_isa >= ISA_LEVEL_VBMI2 && (encoder_has_avx10 || (use_isa & ISA_FEATURE_EVEX512)))
 		encoder_vbmi2_init();
 	else if(use_isa >= ISA_LEVEL_AVX2)
 		encoder_avx2_init();
@@ -170,4 +172,8 @@ void encoder_init() {
 	if(cpu_supports_neon())
 		encoder_neon_init();
 #endif
+#ifdef __riscv
+	if(cpu_supports_rvv())
+		encoder_rvv_init();
+#endif
 }

package/src/encoder_avx_base.h

@@ -215,7 +215,7 @@ HEDLEY_ALWAYS_INLINE void do_encode_avx2(int line_size, int* colOffset, const ui
 				// duplicate halves
 				data1A = _mm256_inserti128_si256(dataA, _mm256_castsi256_si128(dataA), 1);
 				data1B = _mm256_inserti128_si256(dataB, _mm256_castsi256_si128(dataB), 1);
-#if defined(__tune_znver2__) || defined(__tune_znver3__)
+#if defined(__tune_znver2__) || defined(__tune_znver3__) || defined(__tune_znver4__)
 				data2A = _mm256_permute2x128_si256(dataA, dataA, 0x11);
 				data2B = _mm256_permute2x128_si256(dataB, dataB, 0x11);
 #else
@@ -290,7 +290,7 @@ HEDLEY_ALWAYS_INLINE void do_encode_avx2(int line_size, int* colOffset, const ui
 				
 #if defined(__GNUC__) && defined(PLATFORM_AMD64)
 				if(use_isa >= ISA_LEVEL_VBMI2) {
-					asm(
+					__asm__(
 						"shrq $1, %[eqMask] \n"
 						"shrq %%cl, %[eqMask] \n"
 						"adcq %q[col], %q[p] \n"
@@ -334,28 +334,32 @@ HEDLEY_ALWAYS_INLINE void do_encode_avx2(int line_size, int* colOffset, const ui
 			if(use_isa >= ISA_LEVEL_AVX3) {
 # if defined(__AVX512VBMI2__)
 				if(use_isa >= ISA_LEVEL_VBMI2) {
-					_mm256_mask_storeu_epi8(p+1, 1UL<<31, dataA);
+					__m128i dataTop = _mm256_extracti128_si256(dataA, 1);
 					dataA = _mm256_mask_expand_epi8(_mm256_set1_epi8('='), KNOT32(maskA), dataA);
 					_mm256_storeu_si256((__m256i*)p, dataA);
+					p[32] = _mm_extract_epi8(dataTop, 15);
 					p += outputBytesA;
 					
-					_mm256_mask_storeu_epi8(p+1, 1UL<<31, dataB);
+					dataTop = _mm256_extracti128_si256(dataB, 1);
 					dataB = _mm256_mask_expand_epi8(_mm256_set1_epi8('='), KNOT32(maskB), dataB);
 					_mm256_storeu_si256((__m256i*)p, dataB);
+					p[32] = _mm_extract_epi8(dataTop, 15);
 					p += maskBitsB;
 				} else
 # endif
 				{
-					_mm256_mask_storeu_epi8(p+1, 1UL<<31, dataA);
-					dataA = _mm256_mask_alignr_epi8(dataA, (uint32_t)(-(int32_t)maskA), dataA, _mm256_permute4x64_epi64(dataA, _MM_SHUFFLE(1,0,3,2)), 15);
+					__m256i dataSwapped = _mm256_permute4x64_epi64(dataA, _MM_SHUFFLE(1,0,3,2));
+					dataA = _mm256_mask_alignr_epi8(dataA, (uint32_t)(-(int32_t)maskA), dataA, dataSwapped, 15);
 					dataA = _mm256_ternarylogic_epi32(dataA, cmpA, _mm256_set1_epi8('='), 0xb8); // (data & ~cmp) | (cmp & '=')
 					_mm256_storeu_si256((__m256i*)p, dataA);
+					p[32] = _mm_extract_epi8(_mm256_castsi256_si128(dataSwapped), 15);
 					p += outputBytesA;
 					
-					_mm256_mask_storeu_epi8(p+1, 1UL<<31, dataB);
-					dataB = _mm256_mask_alignr_epi8(dataB, (uint32_t)(-(int32_t)maskB), dataB, _mm256_permute4x64_epi64(dataB, _MM_SHUFFLE(1,0,3,2)), 15);
+					dataSwapped = _mm256_permute4x64_epi64(dataB, _MM_SHUFFLE(1,0,3,2));
+					dataB = _mm256_mask_alignr_epi8(dataB, (uint32_t)(-(int32_t)maskB), dataB, dataSwapped, 15);
 					dataB = _mm256_ternarylogic_epi32(dataB, cmpB, _mm256_set1_epi8('='), 0xb8);
 					_mm256_storeu_si256((__m256i*)p, dataB);
+					p[32] = _mm_extract_epi8(_mm256_castsi256_si128(dataSwapped), 15);
 					p += maskBitsB;
 				}
 			} else
@@ -484,28 +488,32 @@ HEDLEY_ALWAYS_INLINE void do_encode_avx2(int line_size, int* colOffset, const ui
 				if(use_isa >= ISA_LEVEL_AVX3) {
 # if defined(__AVX512VBMI2__)
 					if(use_isa >= ISA_LEVEL_VBMI2) {
-						_mm256_mask_storeu_epi8(p+1, 1UL<<31, dataA);
+						__m128i dataTop = _mm256_extracti128_si256(dataA, 1);
 						dataA = _mm256_mask_expand_epi8(_mm256_set1_epi8('='), KNOT32(maskA), dataA);
 						_mm256_storeu_si256((__m256i*)p, dataA);
+						p[32] = _mm_extract_epi8(dataTop, 15);
 						p += outputBytesA;
 						
-						_mm256_mask_storeu_epi8(p+1, 1UL<<31, dataB);
+						dataTop = _mm256_extracti128_si256(dataB, 1);
 						dataB = _mm256_mask_expand_epi8(_mm256_set1_epi8('='), KNOT32(maskB), dataB);
 						_mm256_storeu_si256((__m256i*)p, dataB);
+						p[32] = _mm_extract_epi8(dataTop, 15);
 						p += maskBitsB;
 					} else
 # endif
 					{
-						_mm256_mask_storeu_epi8(p+1, 1UL<<31, dataA);
-						dataA = _mm256_mask_alignr_epi8(dataA, (uint32_t)(-(int32_t)maskA), dataA, _mm256_permute4x64_epi64(dataA, _MM_SHUFFLE(1,0,3,2)), 15);
+						__m256i dataSwapped = _mm256_permute4x64_epi64(dataA, _MM_SHUFFLE(1,0,3,2));
+						dataA = _mm256_mask_alignr_epi8(dataA, (uint32_t)(-(int32_t)maskA), dataA, dataSwapped, 15);
 						dataA = _mm256_ternarylogic_epi32(dataA, cmpA, _mm256_set1_epi8('='), 0xb8); // (data & ~cmp) | (cmp & '=')
 						_mm256_storeu_si256((__m256i*)p, dataA);
+						p[32] = _mm_extract_epi8(_mm256_castsi256_si128(dataSwapped), 15);
 						p += outputBytesA;
 						
-						_mm256_mask_storeu_epi8(p+1, 1UL<<31, dataB);
-						dataB = _mm256_mask_alignr_epi8(dataB, (uint32_t)(-(int32_t)maskB), dataB, _mm256_permute4x64_epi64(dataB, _MM_SHUFFLE(1,0,3,2)), 15);
+						dataSwapped = _mm256_permute4x64_epi64(dataB, _MM_SHUFFLE(1,0,3,2));
+						dataB = _mm256_mask_alignr_epi8(dataB, (uint32_t)(-(int32_t)maskB), dataB, dataSwapped, 15);
 						dataB = _mm256_ternarylogic_epi32(dataB, cmpB, _mm256_set1_epi8('='), 0xb8);
 						_mm256_storeu_si256((__m256i*)p, dataB);
+						p[32] = _mm_extract_epi8(_mm256_castsi256_si128(dataSwapped), 15);
 						p += maskBitsB;
 					}
 				} else

package/src/encoder_neon.cc

@@ -15,6 +15,43 @@ static HEDLEY_ALWAYS_INLINE void vst1q_u8_x2_unaligned(uint8_t* p, uint8x16x2_t
 #endif
 
 
+// ARM's CLZ instruction at native bit-width
+#ifdef __aarch64__
+static HEDLEY_ALWAYS_INLINE int clz_n(uint64_t v) {
+# ifdef _MSC_VER
+	long r;
+	// does this work?
+	if(_BitScanReverse64((unsigned long*)&r, v))
+		r ^= 63;
+	else
+		r = 64;
+	return r;
+# else
+#  if defined(__clang__) || HEDLEY_GCC_VERSION_CHECK(11,0,0)
+	// this pattern is only detected on GCC >= 11 (Clang 9 seems to as well, unsure about earlier versions)
+	// - note: return type must be 'int'; GCC fails to optimise this if type is 'long'
+	// GCC <= 10 doesn't optimize around the '0 = undefined behaviour', so not needed there
+	if(v == 0) return 64;
+#  endif
+	return __builtin_clzll(v);
+# endif
+}
+#else
+static HEDLEY_ALWAYS_INLINE int clz_n(uint32_t v) {
+# ifdef __GNUC__
+#  if defined(__clang__) || HEDLEY_GCC_VERSION_CHECK(7,0,0)
+	// as with AArch64 version above, only insert this check if compiler can optimise it away
+	if(v == 0) return 32;
+#  endif
+	return __builtin_clz(v);
+# elif defined(_MSC_VER)
+	return _arm_clz(v);
+# else
+	return __clz(v); // ARM compiler?
+# endif
+}
+#endif
+
 static uint8x16_t ALIGN_TO(16, shufLUT[256]);
 static uint16_t expandLUT[256];
 
@@ -195,26 +232,7 @@ static HEDLEY_ALWAYS_INLINE void encode_eol_handle_pre(const uint8_t* HEDLEY_RES
 		col = shufTotalLen+1 + lineSizeOffset-32;
 	} else {
 		// shuffle stuff up
-#ifdef __aarch64__
-# ifdef _MSC_VER
-		long bitIndex;
-		if(_BitScanReverse64((unsigned long*)&bitIndex, mask))
-			bitIndex ^= 63;
-		else
-			bitIndex = 64;
-# else
-		long bitIndex = __builtin_clzll(mask);
-# endif
-#else
-# ifdef __GNUC__
-		long bitIndex = __builtin_clz(mask); // TODO: is the 'undefined if 0' case problematic here?
-# elif defined(_MSC_VER)
-		long bitIndex = _arm_clz(mask);
-# else
-		long bitIndex = __clz(mask); // ARM compiler?
-# endif
-#endif
-		
+		long bitIndex = clz_n(mask);
 		uint8x16_t vClz = vdupq_n_u8(bitIndex & ~(sizeof(mask)*8));
 #ifdef __aarch64__
 		uint8x16_t blendA = vcgtq_u8(vmakeq_u8(63,62,61,60,51,50,49,48,47,46,45,44,35,34,33,32), vClz);
@@ -450,26 +468,7 @@ HEDLEY_ALWAYS_INLINE void do_encode_neon(int line_size, int* colOffset, const ui
 			}
 		} else {
 			{
-#ifdef __aarch64__
-# ifdef _MSC_VER
-				// does this work?
-				if(_BitScanReverse64((unsigned long*)&bitIndex, mask))
-					bitIndex ^= 63;
-				else
-					bitIndex = 64;
-# else
-				bitIndex = __builtin_clzll(mask); // TODO: is the 'undefined if 0' case problematic here?
-# endif
-#else
-# ifdef __GNUC__
-				bitIndex = __builtin_clz(mask);
-# elif defined(_MSC_VER)
-				bitIndex = _arm_clz(mask);
-# else
-				bitIndex = __clz(mask); // ARM compiler?
-# endif
-#endif
-				
+				bitIndex = clz_n(mask);
 				uint8x16_t vClz = vdupq_n_u8(bitIndex & ~(sizeof(mask)*8));
 #ifdef __aarch64__
 				uint8x16_t blendA = vcgeq_u8(vmakeq_u8(63,62,61,60,51,50,49,48,47,46,45,44,35,34,33,32), vClz);

package/src/encoder_rvv.cc

@@ -0,0 +1,219 @@
+#include "common.h"
+
+#ifdef __riscv_vector
+#include "encoder.h"
+#include "encoder_common.h"
+
+# include <riscv_vector.h>
+# if defined(__clang__) && __clang_major__ < 16
+#  define RV(f) f
+# else
+#  define RV(f) __riscv_##f
+# endif
+
+
+static HEDLEY_ALWAYS_INLINE void encode_eol_handle_pre(const uint8_t* HEDLEY_RESTRICT _src, long& inpos, uint8_t*& outp, long& col, long lineSizeOffset) {
+	// TODO: vectorize
+	uint8_t c = _src[inpos++];
+	if(HEDLEY_UNLIKELY(escapedLUT[c] && c != '.'-42)) {
+		memcpy(outp, &escapedLUT[c], sizeof(uint16_t));
+		outp += 2;
+	} else {
+		*(outp++) = c + 42;
+	}
+	
+	c = _src[inpos++];
+	if(LIKELIHOOD(0.0273, escapedLUT[c]!=0)) {
+		uint32_t w = UINT32_16_PACK(UINT16_PACK('\r', '\n'), (uint32_t)escapedLUT[c]);
+		memcpy(outp, &w, sizeof(w));
+		outp += 4;
+		col = lineSizeOffset + 2;
+	} else {
+		uint32_t w = UINT32_PACK('\r', '\n', (uint32_t)(c+42), 0);
+		memcpy(outp, &w, sizeof(w));
+		outp += 3;
+		col = lineSizeOffset + 1;
+	}
+}
+
+
+HEDLEY_ALWAYS_INLINE void do_encode_rvv(int line_size, int* colOffset, const uint8_t* HEDLEY_RESTRICT srcEnd, uint8_t* HEDLEY_RESTRICT& dest, size_t& len) {
+	size_t vl2 = RV(vsetvlmax_e8m2)(); // TODO: limit to line length
+	// TODO: have a LMUL=1 variant if line_size < vl
+	
+	// offset position to enable simpler loop condition checking
+	const int INPUT_OFFSET = vl2*2 -1; // extra chars for EOL handling, -1 to change <= to <
+	if((intptr_t)len <= INPUT_OFFSET || line_size < (int)vl2*2) return;
+	
+	uint8_t *outp = dest;
+	long inpos = -(long)len;
+	long lineSizeOffset = -line_size +1;
+	long col = *colOffset - line_size +1;
+	
+	inpos += INPUT_OFFSET;
+	const uint8_t* _src = srcEnd - INPUT_OFFSET;
+	
+	if (HEDLEY_LIKELY(col == -line_size+1)) {
+		uint8_t c = _src[inpos++];
+		if (LIKELIHOOD(0.0273, escapedLUT[c] != 0)) {
+			memcpy(outp, escapedLUT + c, 2);
+			outp += 2;
+			col += 2;
+		} else {
+			*(outp++) = c + 42;
+			col += 1;
+		}
+	}
+	if(HEDLEY_UNLIKELY(col >= 0)) {
+		if(col == 0)
+			encode_eol_handle_pre(_src, inpos, outp, col, lineSizeOffset);
+		else {
+			uint8_t c = _src[inpos++];
+			if(LIKELIHOOD(0.0273, escapedLUT[c]!=0)) {
+				uint32_t v = UINT32_16_PACK(UINT16_PACK('\r', '\n'), (uint32_t)escapedLUT[c]);
+				memcpy(outp, &v, sizeof(v));
+				outp += 4;
+				col = 2-line_size + 1;
+			} else {
+				uint32_t v = UINT32_PACK('\r', '\n', (uint32_t)(c+42), 0);
+				memcpy(outp, &v, sizeof(v));
+				outp += 3;
+				col = 2-line_size;
+			}
+		}
+	}
+	
+	// vector constants
+	const vuint8mf2_t ALT_SHIFT = RV(vreinterpret_v_u16mf2_u8mf2)(RV(vmv_v_x_u16mf2)(4, vl2));
+	const uint8_t _MASK_EXPAND[] = {0xAA, 0xAB, 0xAE, 0xAF, 0xBA, 0xBB, 0xBE, 0xBF, 0xEA, 0xEB, 0xEE, 0xEF, 0xFA, 0xFB, 0xFE, 0xFF};
+	const vuint8m1_t MASK_EXPAND = RV(vle8_v_u8m1)(_MASK_EXPAND, 16);
+	
+	
+	// TODO: consider exploiting partial vector capability
+	while(inpos < 0) {
+		vuint8m2_t data = RV(vle8_v_u8m2)(_src + inpos, vl2);
+		inpos += vl2;
+		
+		// search for special chars
+		// TODO: vrgather strat
+		
+		vuint8m2_t tmpData = RV(vsub_vx_u8m2)(data, -42, vl2);
+		vbool4_t cmp = RV(vmor_mm_b4)(
+			RV(vmor_mm_b4)(
+				RV(vmseq_vx_u8m2_b4)(data, -42, vl2),
+				RV(vmseq_vx_u8m2_b4)(tmpData, '=', vl2),
+				vl2
+			),
+			RV(vmor_mm_b4)(
+				RV(vmseq_vx_u8m2_b4)(data, '\r'-42, vl2),
+				RV(vmseq_vx_u8m2_b4)(data, '\n'-42, vl2),
+				vl2
+			),
+			vl2
+		);
+		
+#if defined(__riscv_v_intrinsic) && __riscv_v_intrinsic >= 13000
+		data = RV(vor_vx_u8m2_mu)(cmp, tmpData, tmpData, 64, vl2);
+#else
+		data = RV(vor_vx_u8m2_m)(cmp, tmpData, tmpData, 64, vl2);
+#endif
+		
+		int idx;
+		size_t count = RV(vcpop_m_b4)(cmp, vl2);
+		if(count > 1) {
+			// widen mask: 4b->8b
+#if defined(__riscv_v_intrinsic) && __riscv_v_intrinsic >= 13000
+			vuint8mf4_t vcmp = RV(vlmul_trunc_v_u8m1_u8mf4)(RV(vreinterpret_v_b4_u8m1)(cmp));
+#else
+			vuint8mf4_t vcmp = *(vuint8mf4_t*)(&cmp);
+#endif
+			// TODO: use vwsll instead if available
+			//    -  is clmul useful here?
+			vuint8mf2_t xcmp = RV(vreinterpret_v_u16mf2_u8mf2)(RV(vwmulu_vx_u16mf2)(vcmp, 16, vl2));
+			xcmp = RV(vsrl_vv_u8mf2)(xcmp, ALT_SHIFT, vl2);
+			
+			// expand mask by inserting '1' between each bit (0000abcd -> 1a1b1c1d)
+			vuint8m1_t xcmpTmp = RV(vrgather_vv_u8m1)(MASK_EXPAND, RV(vlmul_ext_v_u8mf2_u8m1)(xcmp), vl2);
+#if defined(__riscv_v_intrinsic) && __riscv_v_intrinsic >= 13000
+			vbool2_t cmpmask = RV(vreinterpret_b2)(xcmpTmp);
+#else
+			vbool2_t cmpmask = *(vbool2_t*)(&xcmpTmp);
+#endif
+			
+			// expand data and insert =
+			// TODO: use vwsll instead if available
+			vuint16m4_t data2 = RV(vzext_vf2_u16m4)(data, vl2);
+			data2 = RV(vsll_vx_u16m4)(data2, 8, vl2);
+			data2 = RV(vor_vx_u16m4)(data2, '=', vl2);
+			
+			// prune unneeded =
+			vuint8m4_t dataTmp = RV(vreinterpret_v_u16m4_u8m4)(data2);
+			vuint8m4_t final_data = RV(vcompress_vm_u8m4)(
+#if defined(__riscv_v_intrinsic) && __riscv_v_intrinsic >= 13000
+				dataTmp, cmpmask, vl2*2
+#else
+				cmpmask, dataTmp, dataTmp, vl2*2
+#endif
+			);
+			
+			RV(vse8_v_u8m4)(outp, final_data, vl2*2);
+			outp += vl2 + count;
+			col += vl2 + count;
+			
+			if(col >= 0) {
+				// we overflowed - find correct position to revert back to
+				// TODO: stick with u8 type for vlmax <= 2048 (need to check if ok if vlmax == 2048)
+				//   - considering that it's rare for colWidth > 128, maybe just don't support vectors that long
+				vuint16m8_t xidx = RV(viota_m_u16m8)(cmpmask, vl2*2);
+				vbool2_t discardmask = RV(vmsgeu_vx_u16m8_b2)(xidx, vl2 + count - col, vl2*2);
+				long idx_revert = RV(vcpop_m_b2)(discardmask, vl2*2);
+				
+				outp -= col + (idx_revert & 1);
+				inpos -= ((idx_revert+1) >> 1);
+				
+				goto _encode_eol_handle_pre;
+			}
+		} else {
+			// 0 or 1 special characters
+			{
+				vbool4_t mask = RV(vmsbf_m_b4)(cmp, vl2);
+				// TODO: is it better to shuffle this into two stores, instead of three?
+				RV(vse8_v_u8m2_m)(mask, outp, data, vl2);
+				idx = RV(vcpop_m_b4)(mask, vl2);
+				outp[idx] = '=';
+				RV(vse8_v_u8m2_m)(RV(vmnot_m_b4)(mask, vl2), outp+1, data, vl2);
+				
+				outp += vl2 + count;
+				col += vl2 + count;
+			}
+			
+			if(col >= 0) {
+				if(count > 0) {
+					idx = vl2 - idx;
+					if(HEDLEY_UNLIKELY(col == idx)) {
+						// this is an escape character, so line will need to overflow
+						outp--;
+					} else {
+						inpos += (col > idx);
+					}
+				}
+				outp -= col;
+				inpos -= col;
+				
+				_encode_eol_handle_pre:
+				encode_eol_handle_pre(_src, inpos, outp, col, lineSizeOffset);
+			}
+		}
+	}
+	
+	*colOffset = col + line_size -1;
+	dest = outp;
+	len = -(inpos - INPUT_OFFSET);
+}
+
+void encoder_rvv_init() {
+	_do_encode = &do_encode_simd<do_encode_rvv>;
+}
+#else
+void encoder_rvv_init() {}
+#endif /* defined(__riscv_vector) */

package/src/encoder_sse_base.h

@@ -350,7 +350,7 @@ HEDLEY_ALWAYS_INLINE void do_encode_sse(int line_size, int* colOffset, const uin
 #if defined(__POPCNT__) && !defined(__tune_btver1__)
 				if(use_isa & ISA_FEATURE_POPCNT) {
 					shuf2Len = popcnt32(maskA) + 16;
-# if defined(__tune_znver3__) || defined(__tune_znver2__) || defined(__tune_znver1__) || defined(__tune_btver2__)
+# if defined(__tune_znver4__) || defined(__tune_znver3__) || defined(__tune_znver2__) || defined(__tune_znver1__) || defined(__tune_btver2__)
 					shuf1Len = popcnt32(m1) + 8;
 					shuf3Len = popcnt32(m3) + shuf2Len + 8;
 # else
@@ -409,7 +409,7 @@ HEDLEY_ALWAYS_INLINE void do_encode_sse(int line_size, int* colOffset, const uin
 					if(use_isa >= ISA_LEVEL_VBMI2)
 # endif
 					{
-						asm(
+						__asm__(
 							"shrl $1, %[eqMask] \n"
 							"shrl %%cl, %[eqMask] \n" // TODO: can use shrq to avoid above shift?
 # if defined(PLATFORM_AMD64) && !defined(__ILP32__)
@@ -484,7 +484,7 @@ HEDLEY_ALWAYS_INLINE void do_encode_sse(int line_size, int* colOffset, const uin
 				dataB = _mm_ternarylogic_epi32(dataB, cmpB, _mm_set1_epi8(64), 0xf8);
 				
 				// store last char
-				_mm_mask_storeu_epi8(p+XMM_SIZE+1, 1<<15, dataB);
+				p[XMM_SIZE*2] = _mm_extract_epi8(dataB, 15);
 				
 				uint32_t blendMask = (uint32_t)(-(int32_t)mask);
 				dataB = _mm_mask_alignr_epi8(dataB, blendMask>>16, dataB, dataA, 15);

package/src/encoder_vbmi2.cc

@@ -1,5 +1,12 @@
 #include "common.h"
 
+extern const bool encoder_has_avx10;
+#if !defined(__EVEX512__) && (defined(__AVX10_1__) || defined(__EVEX256__)) && defined(__AVX512VL__) && defined(__AVX512VBMI2__) && defined(__AVX512BW__)
+const bool encoder_has_avx10 = true;
+#else
+const bool encoder_has_avx10 = false;
+#endif
+
 #if defined(__AVX512VL__) && defined(__AVX512VBMI2__) && defined(__AVX512BW__)
 # ifndef YENC_DISABLE_AVX256
 #  include "encoder_avx_base.h"