yencode 1.0.8 → 1.1.2

package/src/encoder_neon.cc

@@ -0,0 +1,547 @@
+#include "common.h"
+
+#ifdef __ARM_NEON
+#include "encoder.h"
+#include "encoder_common.h"
+
+// Clang wrongly assumes alignment on vst1q_u8_x2, and ARMv7 GCC doesn't support the function, so effectively, it can only be used in ARMv8 compilers
+#if defined(__aarch64__) && (defined(__clang__) || (defined(__GNUC__) && __GNUC__ >= 9))
+# define vst1q_u8_x2_unaligned vst1q_u8_x2
+#else
+static HEDLEY_ALWAYS_INLINE void vst1q_u8_x2_unaligned(uint8_t* p, uint8x16x2_t data) {
+	vst1q_u8(p, data.val[0]);
+	vst1q_u8(p+16, data.val[1]);
+}
+#endif
+
+
+static uint8x16_t ALIGN_TO(16, shufLUT[256]);
+static uint16_t expandLUT[256];
+
+static HEDLEY_ALWAYS_INLINE void encode_eol_handle_pre(const uint8_t* HEDLEY_RESTRICT es, long& i, uint8_t*& p, long& col, long lineSizeOffset) {
+	uint8x16_t oDataA = vld1q_u8(es + i);
+	uint8x16_t oDataB = vld1q_u8(es + i + sizeof(uint8x16_t));
+	uint8x16_t dataA = oDataA;
+	uint8x16_t dataB = oDataB;
+#ifdef __aarch64__
+	uint8x16_t cmpA = vreinterpretq_u8_s8(vqtbx2q_s8(
+		vdupq_n_s8('='-42),
+		vcreate2_s8(vmakeq_s8('\0'-42,-128,-128,'\0'-42,'\t'-42,'\n'-42,'\r'-42,'\t'-42,'\n'-42,'\r'-42,-128,-128,'\0'-42,-128,-128,-128), vmakeq_s8(' '-42,'\n'-42,'\r'-42,' '-42,-128,-128,-128,-128,-128,-128,'.'-42,-128,-128,-128,'='-42,-128)),
+		vreinterpretq_u8_s8(vhaddq_s8(vreinterpretq_s8_u8(dataA), vmakeq_s8(42,48,66,66, 66,66,66,66, 66,66,66,66, 66,66,66,66)))
+	));
+	cmpA = vceqq_u8(cmpA, dataA);
+	
+	dataB = vaddq_u8(oDataB, vdupq_n_u8(42));
+	uint8x16_t cmpB = vqtbx1q_u8(
+		vceqq_u8(oDataB, vdupq_n_u8('='-42)),
+		//         \0                    \n      \r
+		vmakeq_u8(255,0,0,0,0,0,0,0,0,0,255,0,0,255,0,0),
+		dataB
+	);
+	dataA = vaddq_u8(dataA, vbslq_u8(cmpA, vdupq_n_u8(64+42), vdupq_n_u8(42)));
+	dataB = vorrq_u8(dataB, vandq_u8(cmpB, vdupq_n_u8(64)));
+#else
+	uint8x16_t cmpA = vorrq_u8(
+		vorrq_u8(
+			vceqq_u8(oDataA, vdupq_n_u8(-42)),
+			vceqq_u8(oDataA, vdupq_n_u8('='-42))
+		),
+		vorrq_u8(
+			vceqq_u8(oDataA, vdupq_n_u8('\r'-42)),
+			vceqq_u8(oDataA, vdupq_n_u8('\n'-42))
+		)
+	);
+	uint8x16_t cmpB = vorrq_u8(
+		vorrq_u8(
+			vceqq_u8(oDataB, vdupq_n_u8(-42)),
+			vceqq_u8(oDataB, vdupq_n_u8('='-42))
+		),
+		vorrq_u8(
+			vceqq_u8(oDataB, vdupq_n_u8('\r'-42)),
+			vceqq_u8(oDataB, vdupq_n_u8('\n'-42))
+		)
+	);
+	
+	// dup low 2 bytes & compare
+	uint8x8_t firstTwoChars = vreinterpret_u8_u16(vdup_lane_u16(vreinterpret_u16_u8(vget_low_u8(oDataA)), 0));
+	uint8x8_t cmpNl = vceq_u8(firstTwoChars, vmake_u8(
+		' '+214,' '+214,'\t'+214,'\t'+214,'\r'+214,'.'-42,'='-42,'='-42
+	));
+	// use padd to merge comparisons
+	uint16x4_t cmpNl2 = vreinterpret_u16_u8(cmpNl);
+	cmpNl2 = vpadd_u16(cmpNl2, vdup_n_u16(0));
+	cmpNl2 = vpadd_u16(cmpNl2, vdup_n_u16(0));
+	cmpA = vcombine_u8(
+		vorr_u8(vget_low_u8(cmpA), vreinterpret_u8_u16(cmpNl2)),
+		vget_high_u8(cmpA)
+	);
+	dataA = vsubq_u8(dataA, vbslq_u8(cmpA, vdupq_n_u8(-64-42), vdupq_n_u8(-42)));
+	dataB = vsubq_u8(dataB, vbslq_u8(cmpB, vdupq_n_u8(-64-42), vdupq_n_u8(-42)));
+#endif
+	
+	
+	uint8x16_t cmpAMasked = vandq_u8(cmpA, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128));
+	uint8x16_t cmpBMasked = vandq_u8(cmpB, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128));
+#ifdef __aarch64__
+	uint8x16_t cmpMerge = vpaddq_u8(cmpAMasked, cmpBMasked);
+	cmpMerge = vpaddq_u8(cmpMerge, cmpMerge);
+	uint64_t mask = vgetq_lane_u64(vreinterpretq_u64_u8(cmpMerge), 0);
+	
+	// write out first char + newline
+	uint32_t firstChar = vgetq_lane_u8(dataA, 0);
+	if(LIKELIHOOD(0.0234, mask & 1)) {
+		firstChar <<= 8;
+		firstChar |= 0x0a0d003d;
+		memcpy(p, &firstChar, sizeof(firstChar));
+		p += 4;
+		mask ^= 1;
+		cmpMerge = vbicq_u8(cmpMerge, vmakeq_u8(1,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0));
+	} else {
+		firstChar |= 0x0a0d00;
+		memcpy(p, &firstChar, sizeof(firstChar));
+		p += 3;
+	}
+	
+	if(LIKELIHOOD(0.09, (mask & (mask-1)) != 0)) {
+		mask |= mask >> 8;
+		uint8x8_t cmpPacked = vpadd_u8(vget_low_u8(cmpMerge), vget_low_u8(cmpMerge));
+		uint8_t m1 = (mask & 0xff);
+		uint8_t m2 = ((mask >> 16) & 0xff);
+		uint8_t m3 = ((mask >> 32) & 0xff);
+		uint8_t m4 = ((mask >> 48) & 0xff);
+#else
+	// no vpaddq_u8 in ARMv7, so need extra 64-bit VPADD
+	uint8x8_t cmpPacked = vpadd_u8(
+		vpadd_u8(
+			vget_low_u8(cmpAMasked), vget_high_u8(cmpAMasked)
+		),
+		vpadd_u8(
+			vget_low_u8(cmpBMasked), vget_high_u8(cmpBMasked)
+		)
+	);
+	cmpPacked = vpadd_u8(cmpPacked, cmpPacked);
+	uint32_t mask = vget_lane_u32(vreinterpret_u32_u8(cmpPacked), 0);
+	
+	// write out first char + newline
+	uint32_t firstChar = vgetq_lane_u8(dataA, 0);
+	if(LIKELIHOOD(0.0234, mask & 1)) {
+		firstChar <<= 8;
+		firstChar |= 0x0a0d003d;
+		memcpy(p, &firstChar, sizeof(firstChar));
+		p += 4;
+		mask ^= 1;
+		cmpPacked = vbic_u8(cmpPacked, vmake_u8(1,0,0,0, 0,0,0,0));
+	} else {
+		firstChar |= 0x0a0d00;
+		memcpy(p, &firstChar, sizeof(firstChar));
+		p += 3;
+	}
+	
+	if(LIKELIHOOD(0.09, (mask & (mask-1)) != 0)) {
+		uint8_t m1 = (mask & 0xff);
+		uint8_t m2 = ((mask >> 8) & 0xff);
+		uint8_t m3 = ((mask >> 16) & 0xff);
+		uint8_t m4 = ((mask >> 24) & 0xff);
+#endif
+		
+		// perform lookup for shuffle mask
+		uint8x16_t shuf1 = vld1q_u8((uint8_t*)(shufLUT + m1));
+		uint8x16_t shuf2 = vld1q_u8((uint8_t*)(shufLUT + m2));
+		uint8x16_t shuf3 = vld1q_u8((uint8_t*)(shufLUT + m3));
+		uint8x16_t shuf4 = vld1q_u8((uint8_t*)(shufLUT + m4));
+#ifdef __aarch64__
+		uint8x16_t data1A = vqtbx1q_u8(shuf1, dataA, shuf1);
+		uint8x16_t data2A = vqtbx1q_u8(shuf2, vextq_u8(dataA, dataA, 8), shuf2);
+		uint8x16_t data1B = vqtbx1q_u8(shuf3, dataB, shuf3);
+		uint8x16_t data2B = vqtbx1q_u8(shuf4, vextq_u8(dataB, dataB, 8), shuf4);
+#else
+		uint8x8_t shuf1l = vget_low_u8(shuf1);
+		uint8x8_t shuf1h = vget_high_u8(shuf1);
+		uint8x8_t shuf2l = vget_low_u8(shuf2);
+		uint8x8_t shuf2h = vget_high_u8(shuf2);
+		uint8x8_t shuf3l = vget_low_u8(shuf3);
+		uint8x8_t shuf3h = vget_high_u8(shuf3);
+		uint8x8_t shuf4l = vget_low_u8(shuf4);
+		uint8x8_t shuf4h = vget_high_u8(shuf4);
+		uint8x16_t data1A = vcombine_u8(vtbx1_u8(shuf1l, vget_low_u8(dataA), shuf1l),
+		                                vtbx1_u8(shuf1h, vget_low_u8(dataA), shuf1h));
+		uint8x16_t data2A = vcombine_u8(vtbx1_u8(shuf2l, vget_high_u8(dataA), shuf2l),
+		                                vtbx1_u8(shuf2h, vget_high_u8(dataA), shuf2h));
+		uint8x16_t data1B = vcombine_u8(vtbx1_u8(shuf3l, vget_low_u8(dataB), shuf3l),
+		                                vtbx1_u8(shuf3h, vget_low_u8(dataB), shuf3h));
+		uint8x16_t data2B = vcombine_u8(vtbx1_u8(shuf4l, vget_high_u8(dataB), shuf4l),
+		                                vtbx1_u8(shuf4h, vget_high_u8(dataB), shuf4h));
+#endif
+		data1A = vextq_u8(data1A, data1A, 1); // shift out processed byte (last char of line)
+		
+		uint32_t counts = vget_lane_u32(vreinterpret_u32_u8(vcnt_u8(cmpPacked)), 0);
+		counts += 0x08080807;
+		
+		unsigned char shuf1Len = counts & 0xff;
+		unsigned char shuf2Len = (counts>>8) & 0xff;
+		unsigned char shuf3Len = (counts>>16) & 0xff;
+		unsigned char shuf4Len = (counts>>24) & 0xff;
+		uint32_t shufTotalLen = counts * 0x1010101;
+		shufTotalLen >>= 24;
+		
+		vst1q_u8(p, data1A);
+		p += shuf1Len;
+		vst1q_u8(p, data2A);
+		p += shuf2Len;
+		vst1q_u8(p, data1B);
+		p += shuf3Len;
+		vst1q_u8(p, data2B);
+		p += shuf4Len;
+		col = shufTotalLen+1 + lineSizeOffset-32;
+	} else {
+		// shuffle stuff up
+#ifdef __aarch64__
+# ifdef _MSC_VER
+		long bitIndex;
+		if(_BitScanReverse64((unsigned long*)&bitIndex, mask))
+			bitIndex ^= 63;
+		else
+			bitIndex = 64;
+# else
+		long bitIndex = __builtin_clzll(mask);
+# endif
+#else
+# ifdef __GNUC__
+		long bitIndex = __builtin_clz(mask); // TODO: is the 'undefined if 0' case problematic here?
+# elif defined(_MSC_VER)
+		long bitIndex = _arm_clz(mask);
+# else
+		long bitIndex = __clz(mask); // ARM compiler?
+# endif
+#endif
+		
+		uint8x16_t vClz = vdupq_n_u8(bitIndex & ~(sizeof(mask)*8));
+#ifdef __aarch64__
+		uint8x16_t blendA = vcgtq_u8(vmakeq_u8(63,62,61,60,51,50,49,48,47,46,45,44,35,34,33,32), vClz);
+		uint8x16_t blendB = vcgtq_u8(vmakeq_u8(31,30,29,28,19,18,17,16,15,14,13,12, 3, 2, 1, 0), vClz);
+#else
+		uint8x16_t blendA = vcgtq_u8(vmakeq_u8(31,30,29,28,27,26,25,24,23,22,21,20,19,18,17,16), vClz);
+		uint8x16_t blendB = vcgtq_u8(vmakeq_u8(15,14,13,12,11,10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0), vClz);
+#endif
+		uint8x16_t dataAShifted = vbslq_u8(cmpA, vdupq_n_u8('='), dataA);
+		uint8x16_t dataBShifted = vbslq_u8(cmpB, vdupq_n_u8('='), dataB);
+		dataAShifted = vextq_u8(dataAShifted, dataBShifted, 1);
+		dataBShifted = vextq_u8(dataBShifted, dataBShifted, 1);
+		dataA = vbslq_u8(blendA, dataAShifted, dataA);
+		dataB = vbslq_u8(blendB, dataBShifted, dataB);
+		
+		vst1q_u8_x2_unaligned(p, vcreate2_u8(dataA, dataB));
+		p += sizeof(uint8x16_t)*2 - 1;
+		p += (mask != 0);
+		col = lineSizeOffset + (mask != 0);
+	}
+	
+	i += sizeof(uint8x16_t)*2;
+	// TODO: check col >= 0 if we want to support short lines
+}
+
+
+HEDLEY_ALWAYS_INLINE void do_encode_neon(int line_size, int* colOffset, const uint8_t* HEDLEY_RESTRICT srcEnd, uint8_t* HEDLEY_RESTRICT& dest, size_t& len) {
+	// offset position to enable simpler loop condition checking
+	const int INPUT_OFFSET = sizeof(uint8x16_t)*4 -1; // extra chars for EOL handling, -1 to change <= to <
+	if(len <= INPUT_OFFSET || line_size < (int)sizeof(uint8x16_t)*4) return;
+	
+	uint8_t *p = dest; // destination pointer
+	long i = -(long)len; // input position
+	long lineSizeOffset = -line_size +32; // line size plus vector length
+	long col = *colOffset - line_size +1;
+	
+	i += INPUT_OFFSET;
+	const uint8_t* es = srcEnd - INPUT_OFFSET;
+	
+	if (HEDLEY_LIKELY(col == -line_size+1)) {
+		uint8_t c = es[i++];
+		if (LIKELIHOOD(0.0273, escapedLUT[c] != 0)) {
+			memcpy(p, escapedLUT + c, 2);
+			p += 2;
+			col += 2;
+		} else {
+			*(p++) = c + 42;
+			col += 1;
+		}
+	}
+	if(HEDLEY_UNLIKELY(col >= 0)) {
+		if(col == 0)
+			encode_eol_handle_pre(es, i, p, col, lineSizeOffset);
+		else {
+			uint8_t c = es[i++];
+			if (LIKELIHOOD(0.0273, escapedLUT[c]!=0)) {
+				uint32_t v = UINT32_16_PACK(UINT16_PACK('\r', '\n'), (uint32_t)escapedLUT[c]);
+				memcpy(p, &v, sizeof(v));
+				p += 4;
+				col = 2-line_size + 1;
+			} else {
+				uint32_t v = UINT32_PACK('\r', '\n', (uint32_t)(c+42), 0);
+				memcpy(p, &v, sizeof(v));
+				p += 3;
+				col = 2-line_size;
+			}
+		}
+	}
+	while(i < 0) {
+		// for unaligned loads, separate loads seem to be faster than vld1q_u8_x2 on Cortex A53; unsure if this applies elsewhere
+		uint8x16_t dataA = vld1q_u8(es + i);
+		uint8x16_t dataB = vld1q_u8(es + i + sizeof(uint8x16_t));
+		i += sizeof(uint8x16_t)*2;
+		// search for special chars
+#ifdef __aarch64__
+		uint8x16_t cmpEqA = vceqq_u8(dataA, vdupq_n_u8('='-42));
+		uint8x16_t cmpEqB = vceqq_u8(dataB, vdupq_n_u8('='-42));
+		dataA = vaddq_u8(dataA, vdupq_n_u8(42));
+		dataB = vaddq_u8(dataB, vdupq_n_u8(42));
+		uint8x16_t cmpA = vqtbx1q_u8(
+			cmpEqA,
+			//         \0                    \n      \r
+			vmakeq_u8(255,0,0,0,0,0,0,0,0,0,255,0,0,255,0,0),
+			dataA
+		);
+		uint8x16_t cmpB = vqtbx1q_u8(
+			cmpEqB,
+			//         \0                    \n      \r
+			vmakeq_u8(255,0,0,0,0,0,0,0,0,0,255,0,0,255,0,0),
+			dataB
+		);
+		
+		dataA = vorrq_u8(dataA, vandq_u8(cmpA, vdupq_n_u8(64)));
+		dataB = vorrq_u8(dataB, vandq_u8(cmpB, vdupq_n_u8(64)));
+#else
+		// the ARMv8 strategy may be worth it here with 2x vtbx2's, but both GCC-9 and Clang-9 generate poor assembly for it, so it performs worse than the following
+		uint8x16_t cmpA = vorrq_u8(
+			vorrq_u8(
+				vceqq_u8(dataA, vdupq_n_u8(-42)),
+				vceqq_u8(dataA, vdupq_n_u8('='-42))
+			),
+			vorrq_u8(
+				vceqq_u8(dataA, vdupq_n_u8('\r'-42)),
+				vceqq_u8(dataA, vdupq_n_u8('\n'-42))
+			)
+		);
+		uint8x16_t cmpB = vorrq_u8(
+			vorrq_u8(
+				vceqq_u8(dataB, vdupq_n_u8(-42)),
+				vceqq_u8(dataB, vdupq_n_u8('='-42))
+			),
+			vorrq_u8(
+				vceqq_u8(dataB, vdupq_n_u8('\r'-42)),
+				vceqq_u8(dataB, vdupq_n_u8('\n'-42))
+			)
+		);
+		
+		dataA = vsubq_u8(dataA, vbslq_u8(cmpA, vdupq_n_u8(-64-42), vdupq_n_u8(-42)));
+		dataB = vsubq_u8(dataB, vbslq_u8(cmpB, vdupq_n_u8(-64-42), vdupq_n_u8(-42)));
+#endif
+		
+		
+		long bitIndex; // prevent compiler whining
+		uint8x16_t cmpAMasked = vandq_u8(cmpA, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128));
+		uint8x16_t cmpBMasked = vandq_u8(cmpB, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128));
+#ifdef __aarch64__
+		uint8x16_t cmpMerge = vpaddq_u8(cmpAMasked, cmpBMasked);
+		cmpMerge = vpaddq_u8(cmpMerge, cmpMerge);
+		uint64_t mask = vgetq_lane_u64(vreinterpretq_u64_u8(cmpMerge), 0);
+		if(LIKELIHOOD(0.09, (mask & (mask-1)) != 0)) {
+			mask |= mask >> 8;
+			uint8x8_t cmpPacked = vpadd_u8(vget_low_u8(cmpMerge), vget_low_u8(cmpMerge));
+			uint8_t m1 = (mask & 0xff);
+			uint8_t m2 = ((mask >> 16) & 0xff);
+			uint8_t m3 = ((mask >> 32) & 0xff);
+			uint8_t m4 = ((mask >> 48) & 0xff);
+#else
+		// no vpaddq_u8 in ARMv7, so need extra 64-bit VPADD
+		uint8x8_t cmpPacked = vpadd_u8(
+			vpadd_u8(
+				vget_low_u8(cmpAMasked), vget_high_u8(cmpAMasked)
+			),
+			vpadd_u8(
+				vget_low_u8(cmpBMasked), vget_high_u8(cmpBMasked)
+			)
+		);
+		cmpPacked = vpadd_u8(cmpPacked, cmpPacked);
+		uint32_t mask = vget_lane_u32(vreinterpret_u32_u8(cmpPacked), 0);
+		if(LIKELIHOOD(0.09, (mask & (mask-1)) != 0)) {
+			uint8_t m1 = (mask & 0xff);
+			uint8_t m2 = ((mask >> 8) & 0xff);
+			uint8_t m3 = ((mask >> 16) & 0xff);
+			uint8_t m4 = ((mask >> 24) & 0xff);
+#endif
+			
+			// perform lookup for shuffle mask
+			uint8x16_t shuf1 = vld1q_u8((uint8_t*)(shufLUT + m1));
+			uint8x16_t shuf2 = vld1q_u8((uint8_t*)(shufLUT + m2));
+			uint8x16_t shuf3 = vld1q_u8((uint8_t*)(shufLUT + m3));
+			uint8x16_t shuf4 = vld1q_u8((uint8_t*)(shufLUT + m4));
+			
+			// expand halves
+#ifdef __aarch64__
+			uint8x16_t data1A = vqtbx1q_u8(shuf1, dataA, shuf1);
+			uint8x16_t data2A = vqtbx1q_u8(shuf2, vextq_u8(dataA, dataA, 8), shuf2);
+			uint8x16_t data1B = vqtbx1q_u8(shuf3, dataB, shuf3);
+			uint8x16_t data2B = vqtbx1q_u8(shuf4, vextq_u8(dataB, dataB, 8), shuf4);
+#else
+			uint8x8_t shuf1l = vget_low_u8(shuf1);
+			uint8x8_t shuf1h = vget_high_u8(shuf1);
+			uint8x8_t shuf2l = vget_low_u8(shuf2);
+			uint8x8_t shuf2h = vget_high_u8(shuf2);
+			uint8x8_t shuf3l = vget_low_u8(shuf3);
+			uint8x8_t shuf3h = vget_high_u8(shuf3);
+			uint8x8_t shuf4l = vget_low_u8(shuf4);
+			uint8x8_t shuf4h = vget_high_u8(shuf4);
+			uint8x16_t data1A = vcombine_u8(vtbx1_u8(shuf1l, vget_low_u8(dataA), shuf1l),
+			                                vtbx1_u8(shuf1h, vget_low_u8(dataA), shuf1h));
+			uint8x16_t data2A = vcombine_u8(vtbx1_u8(shuf2l, vget_high_u8(dataA), shuf2l),
+			                                vtbx1_u8(shuf2h, vget_high_u8(dataA), shuf2h));
+			uint8x16_t data1B = vcombine_u8(vtbx1_u8(shuf3l, vget_low_u8(dataB), shuf3l),
+			                                vtbx1_u8(shuf3h, vget_low_u8(dataB), shuf3h));
+			uint8x16_t data2B = vcombine_u8(vtbx1_u8(shuf4l, vget_high_u8(dataB), shuf4l),
+			                                vtbx1_u8(shuf4h, vget_high_u8(dataB), shuf4h));
+#endif
+			
+			// store out
+			uint32_t counts = vget_lane_u32(vreinterpret_u32_u8(vcnt_u8(cmpPacked)), 0);
+			counts += 0x08080808;
+			
+			unsigned char shuf1Len = counts & 0xff;
+			unsigned char shuf2Len = (counts>>8) & 0xff;
+			unsigned char shuf3Len = (counts>>16) & 0xff;
+			unsigned char shuf4Len = (counts>>24) & 0xff;
+			uint32_t shufTotalLen = counts * 0x1010101;
+			shufTotalLen >>= 24;
+			
+			vst1q_u8(p, data1A);
+			p += shuf1Len;
+			vst1q_u8(p, data2A);
+			p += shuf2Len;
+			vst1q_u8(p, data1B);
+			p += shuf3Len;
+			vst1q_u8(p, data2B);
+			p += shuf4Len;
+			col += shufTotalLen;
+			
+			if(LIKELIHOOD(0.3, col >= 0)) {
+				// we overflowed - find correct position to revert back to
+				long revert = col;
+				long len2ndHalf = shuf3Len+shuf4Len;
+				long shiftAmt = len2ndHalf - col -1;
+				uint32_t eqMaskHalf;
+				if(HEDLEY_UNLIKELY(shiftAmt < 0)) {
+					eqMaskHalf = (expandLUT[m2] << shuf1Len) | expandLUT[m1];
+					eqMaskHalf >>= shufTotalLen - col -1;
+					i += len2ndHalf - 16;
+				} else {
+					eqMaskHalf = (expandLUT[m4] << shuf3Len) | expandLUT[m3];
+					eqMaskHalf >>= shiftAmt;
+				}
+				revert += eqMaskHalf & 1;
+				
+				// count bits in eqMask
+				uint8x8_t vCnt = vcnt_u8(vreinterpret_u8_u32(vmov_n_u32(eqMaskHalf)));
+				uint32_t cnt = vget_lane_u32(vreinterpret_u32_u8(vCnt), 0);
+				cnt *= 0x1010101;
+				i += cnt >> 24;
+				
+				p -= revert;
+				i -= revert;
+				goto _encode_eol_handle_pre;
+			}
+		} else {
+			{
+#ifdef __aarch64__
+# ifdef _MSC_VER
+				// does this work?
+				if(_BitScanReverse64((unsigned long*)&bitIndex, mask))
+					bitIndex ^= 63;
+				else
+					bitIndex = 64;
+# else
+				bitIndex = __builtin_clzll(mask); // TODO: is the 'undefined if 0' case problematic here?
+# endif
+#else
+# ifdef __GNUC__
+				bitIndex = __builtin_clz(mask);
+# elif defined(_MSC_VER)
+				bitIndex = _arm_clz(mask);
+# else
+				bitIndex = __clz(mask); // ARM compiler?
+# endif
+#endif
+				
+				uint8x16_t vClz = vdupq_n_u8(bitIndex & ~(sizeof(mask)*8));
+#ifdef __aarch64__
+				uint8x16_t blendA = vcgeq_u8(vmakeq_u8(63,62,61,60,51,50,49,48,47,46,45,44,35,34,33,32), vClz);
+				uint8x16_t blendB = vcgeq_u8(vmakeq_u8(31,30,29,28,19,18,17,16,15,14,13,12, 3, 2, 1, 0), vClz);
+#else
+				uint8x16_t blendA = vcgeq_u8(vmakeq_u8(31,30,29,28,27,26,25,24,23,22,21,20,19,18,17,16), vClz);
+				uint8x16_t blendB = vcgeq_u8(vmakeq_u8(15,14,13,12,11,10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0), vClz);
+#endif
+				uint8x16_t dataAShifted = vextq_u8(dataA, dataA, 15);
+				uint8x16_t dataBShifted = vextq_u8(dataA, dataB, 15);
+				dataA = vbslq_u8(cmpA, vdupq_n_u8('='), dataA);
+				uint8x16_t outDataB = vbslq_u8(cmpB, vdupq_n_u8('='), dataB);
+				dataA = vbslq_u8(blendA, dataA, dataAShifted);
+				outDataB = vbslq_u8(blendB, outDataB, dataBShifted);
+				
+				vst1q_u8_x2_unaligned(p, vcreate2_u8(dataA, outDataB));
+				p += sizeof(uint8x16_t)*2;
+				// write last byte
+				*p = vgetq_lane_u8(dataB, 15);
+				p += (mask != 0);
+				col += (mask != 0) + sizeof(uint8x16_t)*2;
+			}
+			
+			if(HEDLEY_UNLIKELY(col >= 0)) {
+#ifdef __aarch64__
+				// fixup bitIndex
+				bitIndex -= ((bitIndex+4)>>4)<<3;
+#endif
+				bitIndex = bitIndex +1;
+				if(HEDLEY_UNLIKELY(col == bitIndex)) {
+					// this is an escape character, so line will need to overflow
+					p--;
+				} else {
+					i += (col > bitIndex);
+				}
+				p -= col;
+				i -= col;
+				
+				_encode_eol_handle_pre:
+				encode_eol_handle_pre(es, i, p, col, lineSizeOffset);
+			}
+		}
+	}
+	
+	*colOffset = col + line_size -1;
+	dest = p;
+	len = -(i - INPUT_OFFSET);
+}
+
+void encoder_neon_init() {
+	_do_encode = &do_encode_simd<do_encode_neon>;
+	// generate shuf LUT
+	for(int i=0; i<256; i++) {
+		int k = i;
+		uint16_t expand = 0;
+		uint8_t* res = (uint8_t*)(shufLUT + i);
+		int p = 0;
+		for(int j=0; j<8; j++) {
+			if(k & 1) {
+				res[j+p] = '=';
+				expand |= 1<<(j+p);
+				p++;
+			}
+			res[j+p] = j;
+			k >>= 1;
+		}
+		for(; p<8; p++)
+			res[8+p] = 8+p +0x80; // +0x80 => 0 discarded entries; has no effect other than to ease debugging
+		
+		expandLUT[i] = expand;
+	}
+}
+#else
+void encoder_neon_init() {}
+#endif /* defined(__ARM_NEON) */

package/src/encoder_sse2.cc

@@ -0,0 +1,13 @@
+#include "common.h"
+
+#ifdef __SSE2__
+#include "encoder_sse_base.h"
+
+void encoder_sse2_init() {
+	_do_encode = &do_encode_simd< do_encode_sse<ISA_LEVEL_SSE2> >;
+	encoder_sse_lut<ISA_LEVEL_SSE2>();
+}
+#else
+void encoder_sse2_init() {}
+#endif
+