yencode 1.0.8 → 1.1.2

package/src/encoder_avx_base.h

@@ -0,0 +1,564 @@
+// can't seem to make this worth it
+#include "common.h"
+#ifdef __AVX2__
+
+#include "encoder.h"
+#include "encoder_common.h"
+#define YMM_SIZE 32
+
+#if defined(__GNUC__) && __GNUC__ >= 7
+# define KLOAD32(a, offs) _load_mask32((__mmask32*)(a) + (offs))
+#else
+# define KLOAD32(a, offs) (((uint32_t*)(a))[(offs)])
+#endif
+
+#pragma pack(16)
+static struct {
+	uint32_t eolLastChar[256];
+	/*align32*/ __m256i shufExpand[65536]; // huge 2MB table
+	/*align32*/ int8_t expandMergemix[33*2*32]; // not used in AVX3
+} * HEDLEY_RESTRICT lookupsAVX2;
+static struct {
+	uint32_t eolLastChar[256];
+	uint32_t expand[65536]; // biggish 256KB table (but still smaller than the 2MB table)
+} * HEDLEY_RESTRICT lookupsVBMI2;
+#pragma pack()
+
+static inline void fill_eolLastChar(uint32_t* table) {
+	for(int n=0; n<256; n++) {
+		table[n] = ((n == 214+'\t' || n == 214+' ' || n == 214+'\0' || n == 214+'\n' || n == 214+'\r' || n == '='-42) ? (((n+42+64)&0xff)<<8)+0x0a0d003d : ((n+42)&0xff)+0x0a0d00);
+	}
+}
+
+template<enum YEncDecIsaLevel use_isa>
+static void encoder_avx2_lut() {
+	if(use_isa >= ISA_LEVEL_VBMI2) {
+		ALIGN_ALLOC(lookupsVBMI2, sizeof(*lookupsVBMI2), 32);
+		fill_eolLastChar(lookupsVBMI2->eolLastChar);
+		for(int i=0; i<65536; i++) {
+			int k = i;
+			uint32_t expand = 0;
+			int p = 0;
+			for(int j=0; j<16; j++) {
+				if(k & 1) {
+					p++;
+				}
+				expand |= 1<<(j+p);
+				k >>= 1;
+			}
+			lookupsVBMI2->expand[i] = expand;
+		}
+	} else {
+		ALIGN_ALLOC(lookupsAVX2, sizeof(*lookupsAVX2), 32);
+		fill_eolLastChar(lookupsAVX2->eolLastChar);
+		for(int i=0; i<65536; i++) {
+			int k = i;
+			uint8_t* res = (uint8_t*)(lookupsAVX2->shufExpand + i);
+			int p = 0;
+			for(int j=0; j<16; j++) {
+				if(k & 1) {
+					res[j+p] = 0xff;
+					p++;
+				}
+				res[j+p] = j;
+				k >>= 1;
+			}
+			for(; p<16; p++)
+				res[16+p] = 0x40; // arbitrary value (top bit cannot be set)
+		}
+		for(int i=0; i<33; i++) {
+			int n = (i == 32 ? 32 : 31-i);
+			for(int j=0; j<32; j++) {
+				lookupsAVX2->expandMergemix[i*64 + j] = (n>=j ? -1 : 0);
+				lookupsAVX2->expandMergemix[i*64 + j + 32] = ('='*(n==j) + 64*(n==j-1) + 42*(n!=j));
+			}
+		}
+	}
+}
+
+template<enum YEncDecIsaLevel use_isa>
+HEDLEY_ALWAYS_INLINE void do_encode_avx2(int line_size, int* colOffset, const uint8_t* HEDLEY_RESTRICT srcEnd, uint8_t* HEDLEY_RESTRICT& dest, size_t& len) {
+	// offset position to enable simpler loop condition checking
+	const int INPUT_OFFSET = YMM_SIZE*4 + 1 -1; // -1 to change <= to <
+	if(len <= INPUT_OFFSET || line_size < 16) return;
+	
+	uint8_t *p = dest; // destination pointer
+	intptr_t i = -(intptr_t)len; // input position
+	intptr_t lineSizeOffset = -line_size +1; // -1 because we want to stop one char before the end to handle the last char differently
+	intptr_t col = *colOffset + lineSizeOffset;
+	
+	i += INPUT_OFFSET;
+	const uint8_t* es = srcEnd - INPUT_OFFSET;
+	
+#if !defined(__tune_bdver4__) && !defined(__tune_znver1__)
+	// always process at least one byte to prevent underflow when doing a read with -1 offset
+	if(col < 0 && col != -line_size+1) {
+		// not the first/last character of a line
+		uint8_t c = es[i++];
+		if(HEDLEY_UNLIKELY(c == 214 || c == '\n'+214 || c == '\r'+214 || c == '='-42)) {
+			*(uint16_t*)p = 0x6a3d + (((uint16_t)c) << 8);
+			p += 2;
+			col += 2;
+		} else {
+			*p++ = c+42;
+			col++;
+		}
+	}
+#endif
+	
+	if(HEDLEY_UNLIKELY(col >= 0)) {
+		uint8_t c = es[i++];
+		if(col == 0) {
+			// last char
+			uint32_t eolChar = (use_isa >= ISA_LEVEL_VBMI2 ? lookupsVBMI2->eolLastChar[c] : lookupsAVX2->eolLastChar[c]);
+			*(uint32_t*)p = eolChar;
+			p += 3 + (uintptr_t)(eolChar>>27);
+			col = -line_size+1;
+		} else {
+			// line overflowed, insert a newline
+			if (LIKELIHOOD(0.0273, escapedLUT[c]!=0)) {
+				*(uint32_t*)p = UINT32_16_PACK(UINT16_PACK('\r', '\n'), (uint32_t)escapedLUT[c]);
+				p += 4;
+				col = 2-line_size + 1;
+			} else {
+				*(uint32_t*)p = UINT32_PACK('\r', '\n', (uint32_t)(c+42), 0);
+				p += 3;
+				col = 2-line_size;
+			}
+		}
+	}
+	if (HEDLEY_LIKELY(col == -line_size+1)) {
+		// first char of the line
+		uint8_t c = es[i++];
+		if (LIKELIHOOD(0.0273, escapedLUT[c] != 0)) {
+			*(uint16_t*)p = escapedLUT[c];
+			p += 2;
+			col += 2;
+		} else {
+			*(p++) = c + 42;
+			col += 1;
+		}
+	}
+	do {
+		__m256i dataA = _mm256_loadu_si256((__m256i *)(es + i));
+		__m256i dataB = _mm256_loadu_si256((__m256i *)(es + i) + 1);
+		i += YMM_SIZE*2;
+		// search for special chars
+		__m256i cmpA = _mm256_cmpeq_epi8(
+			_mm256_shuffle_epi8(_mm256_set_epi8(
+				'\0'-42,-42,'\r'-42,'.'-42,'='-42,'\0'-42,'\t'-42,'\n'-42,-42,-42,'\r'-42,-42,'='-42,' '-42,-42,'\n'-42,
+				'\0'-42,-42,'\r'-42,'.'-42,'='-42,'\0'-42,'\t'-42,'\n'-42,-42,-42,'\r'-42,-42,'='-42,' '-42,-42,'\n'-42
+			), _mm256_abs_epi8(dataA)),
+			dataA
+		);
+		__m256i cmpB = _mm256_cmpeq_epi8(
+			_mm256_shuffle_epi8(_mm256_set_epi8(
+				'\0'-42,-42,'\r'-42,'.'-42,'='-42,'\0'-42,'\t'-42,'\n'-42,-42,-42,'\r'-42,-42,'='-42,' '-42,-42,'\n'-42,
+				'\0'-42,-42,'\r'-42,'.'-42,'='-42,'\0'-42,'\t'-42,'\n'-42,-42,-42,'\r'-42,-42,'='-42,' '-42,-42,'\n'-42
+			), _mm256_abs_epi8(dataB)),
+			dataB
+		);
+		
+#if defined(__AVX512VL__)
+		if(use_isa >= ISA_LEVEL_AVX3) {
+			dataA = _mm256_add_epi8(dataA, _mm256_set1_epi8(42));
+			dataA = _mm256_ternarylogic_epi32(dataA, cmpA, _mm256_set1_epi8(64), 0xf8); // data | (cmp & 64)
+			dataB = _mm256_add_epi8(dataB, _mm256_set1_epi8(42));
+			dataB = _mm256_ternarylogic_epi32(dataB, cmpB, _mm256_set1_epi8(64), 0xf8); // data | (cmp & 64)
+		}
+#endif
+		
+		uint32_t maskA = (uint32_t)_mm256_movemask_epi8(cmpA);
+		uint32_t maskB = (uint32_t)_mm256_movemask_epi8(cmpB);
+		unsigned int maskBitsA = popcnt32(maskA);
+		unsigned int maskBitsB = popcnt32(maskB);
+		unsigned int outputBytesA = maskBitsA + YMM_SIZE;
+		unsigned int bitIndexA, bitIndexB;
+		if (LIKELIHOOD(0.170, (maskBitsA|maskBitsB) > 1)) {
+			_encode_loop_branch_slow:
+			unsigned int m1 = maskA & 0xffff, m3 = maskB & 0xffff;
+			unsigned int m2, m4;
+			__m256i data1A, data2A;
+			__m256i data1B, data2B;
+			__m256i shuf1A, shuf1B; // not set in VBMI2 path
+			__m256i shuf2A, shuf2B; // not set in VBMI2 path
+			
+#if defined(__AVX512VBMI2__) && defined(__AVX512VL__) && defined(__AVX512BW__)
+			if(use_isa >= ISA_LEVEL_VBMI2) {
+				m2 = maskA >> 16;
+				m4 = maskB >> 16;
+				
+				/* alternative no-LUT strategy
+				uint64_t expandMaskA = ~_pdep_u64(~maskA, 0x5555555555555555); // expand bits, with bits set
+				expandMaskA = _pext_u64(expandMaskA^0x5555555555555555, expandMaskA);
+				*/
+				
+				data1A = _mm256_mask_expand_epi8(_mm256_set1_epi8('='), KLOAD32(lookupsVBMI2->expand, m1), dataA);
+				data2A = _mm256_mask_expand_epi8(_mm256_set1_epi8('='), KLOAD32(lookupsVBMI2->expand, m2), _mm256_castsi128_si256(
+					_mm256_extracti128_si256(dataA, 1)
+				));
+				data1B = _mm256_mask_expand_epi8(_mm256_set1_epi8('='), KLOAD32(lookupsVBMI2->expand, m3), dataB);
+				data2B = _mm256_mask_expand_epi8(_mm256_set1_epi8('='), KLOAD32(lookupsVBMI2->expand, m4), _mm256_castsi128_si256(
+					_mm256_extracti128_si256(dataB, 1)
+				));
+			} else
+#endif
+			{
+				if(use_isa < ISA_LEVEL_AVX3) {
+					dataA = _mm256_add_epi8(dataA, _mm256_blendv_epi8(_mm256_set1_epi8(42), _mm256_set1_epi8(42+64), cmpA));
+					dataB = _mm256_add_epi8(dataB, _mm256_blendv_epi8(_mm256_set1_epi8(42), _mm256_set1_epi8(42+64), cmpB));
+				}
+				
+				m2 = (maskA >> 11) & 0x1fffe0;
+				m4 = (maskB >> 11) & 0x1fffe0;
+				
+				// duplicate halves
+				data1A = _mm256_inserti128_si256(dataA, _mm256_castsi256_si128(dataA), 1);
+				data1B = _mm256_inserti128_si256(dataB, _mm256_castsi256_si128(dataB), 1);
+#if defined(__tune_znver2__) || defined(__tune_znver3__)
+				data2A = _mm256_permute2x128_si256(dataA, dataA, 0x11);
+				data2B = _mm256_permute2x128_si256(dataB, dataB, 0x11);
+#else
+				data2A = _mm256_permute4x64_epi64(dataA, 0xee);
+				data2B = _mm256_permute4x64_epi64(dataB, 0xee);
+#endif
+				
+				shuf1A = _mm256_load_si256(lookupsAVX2->shufExpand + m1);
+				shuf2A = _mm256_load_si256((__m256i*)((char*)(lookupsAVX2->shufExpand) + m2));
+				shuf1B = _mm256_load_si256(lookupsAVX2->shufExpand + m3);
+				shuf2B = _mm256_load_si256((__m256i*)((char*)(lookupsAVX2->shufExpand) + m4));
+				
+				// expand
+				data1A = _mm256_shuffle_epi8(data1A, shuf1A);
+				data2A = _mm256_shuffle_epi8(data2A, shuf2A);
+				data1B = _mm256_shuffle_epi8(data1B, shuf1B);
+				data2B = _mm256_shuffle_epi8(data2B, shuf2B);
+				// add in '='
+				data1A = _mm256_blendv_epi8(data1A, _mm256_set1_epi8('='), shuf1A);
+				data2A = _mm256_blendv_epi8(data2A, _mm256_set1_epi8('='), shuf2A);
+				data1B = _mm256_blendv_epi8(data1B, _mm256_set1_epi8('='), shuf1B);
+				data2B = _mm256_blendv_epi8(data2B, _mm256_set1_epi8('='), shuf2B);
+			}
+			
+			unsigned int shuf1Len = popcnt32(m1) + 16;
+			unsigned int shuf3Len = popcnt32(m3) + 16;
+			_mm256_storeu_si256((__m256i*)p, data1A);
+			_mm256_storeu_si256((__m256i*)(p + shuf1Len), data2A);
+			_mm256_storeu_si256((__m256i*)(p + outputBytesA), data1B);
+			_mm256_storeu_si256((__m256i*)(p + outputBytesA + shuf3Len), data2B);
+			unsigned int outputBytes = YMM_SIZE + outputBytesA + maskBitsB;
+			p += outputBytes;
+			col += outputBytes;
+			
+			if(col >= 0) {
+				// we overflowed - find correct position to revert back to
+				// this is perhaps sub-optimal on 32-bit, but who still uses that with AVX2?
+				uint64_t eqMask;
+				int shiftAmt = (int)(maskBitsB + YMM_SIZE -1 - col);
+				if(HEDLEY_UNLIKELY(shiftAmt < 0)) {
+					uint32_t eqMask1, eqMask2;
+#if defined(__AVX512VBMI2__) && defined(__AVX512VL__) && defined(__AVX512BW__)
+					if(use_isa >= ISA_LEVEL_VBMI2) {
+						eqMask1 = lookupsVBMI2->expand[m1];
+						eqMask2 = lookupsVBMI2->expand[m2];
+					} else
+#endif
+					{
+						eqMask1 = (uint32_t)_mm256_movemask_epi8(shuf1A);
+						eqMask2 = (uint32_t)_mm256_movemask_epi8(shuf2A);
+					}
+					eqMask = eqMask1 | ((uint64_t)eqMask2 << shuf1Len);
+					if(use_isa < ISA_LEVEL_VBMI2)
+						i += (uintptr_t)maskBitsB;
+					else
+						i -= YMM_SIZE;
+					shiftAmt += outputBytesA;
+				} else {
+					uint32_t eqMask3, eqMask4;
+#if defined(__AVX512VBMI2__) && defined(__AVX512VL__) && defined(__AVX512BW__)
+					if(use_isa >= ISA_LEVEL_VBMI2) {
+						eqMask3 = lookupsVBMI2->expand[m3];
+						eqMask4 = lookupsVBMI2->expand[m4];
+					} else
+#endif
+					{
+						eqMask3 = (uint32_t)_mm256_movemask_epi8(shuf1B);
+						eqMask4 = (uint32_t)_mm256_movemask_epi8(shuf2B);
+					}
+					eqMask = eqMask3 | ((uint64_t)eqMask4 << shuf3Len);
+				}
+				
+#if defined(__GNUC__) && defined(PLATFORM_AMD64)
+				if(use_isa >= ISA_LEVEL_VBMI2) {
+					asm(
+						"shrq $1, %[eqMask] \n"
+						"shrq %%cl, %[eqMask] \n"
+						"adcq %[col], %[p] \n"
+						: [eqMask]"+r"(eqMask), [p]"+r"(p)
+						: "c"(shiftAmt), [col]"r"(~col)
+					);
+					i -= _mm_popcnt_u64(eqMask);
+				} else
+#endif
+				{
+					eqMask >>= shiftAmt;
+					unsigned int bitCount;
+#ifdef PLATFORM_AMD64
+					bitCount = (unsigned int)_mm_popcnt_u64(eqMask);
+#else
+					bitCount = popcnt32(eqMask & 0xffffffff) + popcnt32(eqMask >> 32);
+#endif
+#if defined(__AVX512VBMI2__) && defined(__AVX512VL__) && defined(__AVX512BW__)
+					if(use_isa >= ISA_LEVEL_VBMI2) {
+						i -= bitCount;
+						p -= col;
+						if(LIKELIHOOD(0.98, (eqMask & 1) != 1))
+							p--;
+						else
+							i++;
+					} else
+#endif
+					{
+						i += bitCount;
+						unsigned int revert = (unsigned int)(col + (eqMask & 1));
+						p -= revert;
+						i -= revert;
+					}
+				}
+				goto _encode_eol_handle_pre;
+			}
+		} else {
+			//_encode_loop_branch_fast:
+			maskBitsB += YMM_SIZE;
+#if defined(__AVX512VL__) && defined(__AVX512BW__)
+			if(use_isa >= ISA_LEVEL_AVX3) {
+# if defined(__AVX512VBMI2__)
+				if(use_isa >= ISA_LEVEL_VBMI2) {
+					_mm256_mask_storeu_epi8(p+1, 1UL<<31, dataA);
+					dataA = _mm256_mask_expand_epi8(_mm256_set1_epi8('='), KNOT32(maskA), dataA);
+					_mm256_storeu_si256((__m256i*)p, dataA);
+					p += outputBytesA;
+					
+					_mm256_mask_storeu_epi8(p+1, 1UL<<31, dataB);
+					dataB = _mm256_mask_expand_epi8(_mm256_set1_epi8('='), KNOT32(maskB), dataB);
+					_mm256_storeu_si256((__m256i*)p, dataB);
+					p += maskBitsB;
+				} else
+# endif
+				{
+					_mm256_mask_storeu_epi8(p+1, 1UL<<31, dataA);
+					dataA = _mm256_mask_alignr_epi8(dataA, (uint32_t)(-(int32_t)maskA), dataA, _mm256_permute4x64_epi64(dataA, _MM_SHUFFLE(1,0,3,2)), 15);
+					dataA = _mm256_ternarylogic_epi32(dataA, cmpA, _mm256_set1_epi8('='), 0xb8); // (data & ~cmp) | (cmp & '=')
+					_mm256_storeu_si256((__m256i*)p, dataA);
+					p += outputBytesA;
+					
+					_mm256_mask_storeu_epi8(p+1, 1UL<<31, dataB);
+					dataB = _mm256_mask_alignr_epi8(dataB, (uint32_t)(-(int32_t)maskB), dataB, _mm256_permute4x64_epi64(dataB, _MM_SHUFFLE(1,0,3,2)), 15);
+					dataB = _mm256_ternarylogic_epi32(dataB, cmpB, _mm256_set1_epi8('='), 0xb8);
+					_mm256_storeu_si256((__m256i*)p, dataB);
+					p += maskBitsB;
+				}
+			} else
+#endif
+			{
+				bitIndexA = _lzcnt_u32(maskA);
+				bitIndexB = _lzcnt_u32(maskB);
+				__m256i mergeMaskA = _mm256_load_si256((const __m256i*)(lookupsAVX2->expandMergemix + bitIndexA*2*YMM_SIZE));
+				__m256i mergeMaskB = _mm256_load_si256((const __m256i*)(lookupsAVX2->expandMergemix + bitIndexB*2*YMM_SIZE));
+				
+#if defined(__tune_bdver4__) || defined(__tune_znver1__)
+				// avoid slower 32-byte crossing loads on Zen1
+				__m256i dataAShifted = _mm256_alignr_epi8(
+					dataA,
+					_mm256_inserti128_si256(dataA, _mm256_castsi256_si128(dataA), 1),
+					15
+				);
+				__m256i dataBShifted = _mm256_alignr_epi8(
+					dataB,
+					_mm256_inserti128_si256(dataB, _mm256_castsi256_si128(dataB), 1),
+					15
+				);
+#else
+				__m256i dataAShifted = _mm256_loadu_si256((__m256i *)(es + i - YMM_SIZE*2 - 1));
+				__m256i dataBShifted = _mm256_loadu_si256((__m256i *)(es + i - YMM_SIZE - 1));
+#endif
+				dataA = _mm256_andnot_si256(cmpA, dataA); // clear space for '=' char
+				dataA = _mm256_blendv_epi8(dataAShifted, dataA, mergeMaskA);
+				dataA = _mm256_add_epi8(dataA, _mm256_load_si256((const __m256i*)(lookupsAVX2->expandMergemix + bitIndexA*2*YMM_SIZE) + 1));
+				_mm256_storeu_si256((__m256i*)p, dataA);
+				p[YMM_SIZE] = es[i-1-YMM_SIZE] + 42 + (64 & (maskA>>(YMM_SIZE-1-6)));
+				p += outputBytesA;
+				
+				dataB = _mm256_andnot_si256(cmpB, dataB);
+				dataB = _mm256_blendv_epi8(dataBShifted, dataB, mergeMaskB);
+				dataB = _mm256_add_epi8(dataB, _mm256_load_si256((const __m256i*)(lookupsAVX2->expandMergemix + bitIndexB*2*YMM_SIZE) + 1));
+				_mm256_storeu_si256((__m256i*)p, dataB);
+				p[YMM_SIZE] = es[i-1] + 42 + (64 & (maskB>>(YMM_SIZE-1-6)));
+				p += maskBitsB;
+			}
+			col += outputBytesA + maskBitsB;
+			
+			if(col >= 0) {
+				_encode_loop_branch_fast_eol:
+				if(HEDLEY_UNLIKELY(col > (intptr_t)maskBitsB)) {
+					if(use_isa >= ISA_LEVEL_AVX3)
+						bitIndexA = _lzcnt_u32(maskA);
+					bitIndexA += 1 + maskBitsB;
+					
+					i += maskBitsB - YMM_SIZE;
+					if(HEDLEY_UNLIKELY(col == (intptr_t)bitIndexA)) {
+						// this is an escape character, so line will need to overflow
+						p--;
+					} else {
+						i += (col > (intptr_t)bitIndexA);
+					}
+				} else {
+					if(use_isa >= ISA_LEVEL_AVX3)
+						bitIndexB = _lzcnt_u32(maskB);
+					bitIndexB++;
+					
+					if(HEDLEY_UNLIKELY(col == (intptr_t)bitIndexB)) {
+						p--;
+					} else {
+						i += (col > (intptr_t)bitIndexB);
+					}
+				}
+				i -= col;
+				p -= col;
+				
+				_encode_eol_handle_pre:
+				uint32_t eolChar = (use_isa >= ISA_LEVEL_VBMI2 ? lookupsVBMI2->eolLastChar[es[i]] : lookupsAVX2->eolLastChar[es[i]]);
+				*(uint32_t*)p = eolChar;
+				p += 3 + (uintptr_t)(eolChar>>27);
+				col = lineSizeOffset;
+				
+				if(HEDLEY_UNLIKELY(i >= 0)) { // this isn't really a proper check - it's only needed to support short lines; basically, if the line is too short, `i` never gets checked, so we need one somewhere
+					i++;
+					break;
+				}
+				
+				dataA = _mm256_loadu_si256((__m256i *)(es + i + 1));
+				dataB = _mm256_loadu_si256((__m256i *)(es + i + 1) + 1);
+				i += YMM_SIZE*2 + 1;
+				// search for special chars
+				cmpA = _mm256_cmpeq_epi8(
+					_mm256_shuffle_epi8(_mm256_set_epi8(
+						'\0'-42,-42,'\r'-42,'.'-42,'='-42,'\0'-42,'\t'-42,'\n'-42,-42,-42,'\r'-42,-42,'='-42,' '-42,-42,'\n'-42,
+						'\0'-42,-42,'\r'-42,'.'-42,'='-42,'\0'-42,'\t'-42,'\n'-42,-42,-42,'\r'-42,-42,'='-42,' '-42,-42,'\n'-42
+					), _mm256_adds_epi8(
+						_mm256_abs_epi8(dataA), _mm256_set_epi64x(0, 0, 0, 88)
+					)),
+					dataA
+				);
+				cmpB = _mm256_cmpeq_epi8(
+					_mm256_shuffle_epi8(_mm256_set_epi8(
+						'\0'-42,-42,'\r'-42,'.'-42,'='-42,'\0'-42,'\t'-42,'\n'-42,-42,-42,'\r'-42,-42,'='-42,' '-42,-42,'\n'-42,
+						'\0'-42,-42,'\r'-42,'.'-42,'='-42,'\0'-42,'\t'-42,'\n'-42,-42,-42,'\r'-42,-42,'='-42,' '-42,-42,'\n'-42
+					), _mm256_abs_epi8(dataB)),
+					dataB
+				);
+				
+				// duplicate some code from above to reduce jumping a little
+#if defined(__AVX512VL__)
+				if(use_isa >= ISA_LEVEL_AVX3) {
+					dataA = _mm256_add_epi8(dataA, _mm256_set1_epi8(42));
+					dataA = _mm256_ternarylogic_epi32(dataA, cmpA, _mm256_set1_epi8(64), 0xf8); // data | (cmp & 64)
+					dataB = _mm256_add_epi8(dataB, _mm256_set1_epi8(42));
+					dataB = _mm256_ternarylogic_epi32(dataB, cmpB, _mm256_set1_epi8(64), 0xf8); // data | (cmp & 64)
+				}
+#endif
+				
+				maskA = (uint32_t)_mm256_movemask_epi8(cmpA);
+				maskB = (uint32_t)_mm256_movemask_epi8(cmpB);
+				maskBitsA = popcnt32(maskA);
+				maskBitsB = popcnt32(maskB);
+				outputBytesA = maskBitsA + YMM_SIZE;
+				if (LIKELIHOOD(0.170, (maskBitsA|maskBitsB) > 1))
+					goto _encode_loop_branch_slow;
+				
+				
+				//goto _encode_loop_branch_fast;
+				// duplicating the code, instead of using the goto above, seems to fix a performance regression in GCC
+				maskBitsB += YMM_SIZE;
+#if defined(__AVX512VL__) && defined(__AVX512BW__)
+				if(use_isa >= ISA_LEVEL_AVX3) {
+# if defined(__AVX512VBMI2__)
+					if(use_isa >= ISA_LEVEL_VBMI2) {
+						_mm256_mask_storeu_epi8(p+1, 1UL<<31, dataA);
+						dataA = _mm256_mask_expand_epi8(_mm256_set1_epi8('='), KNOT32(maskA), dataA);
+						_mm256_storeu_si256((__m256i*)p, dataA);
+						p += outputBytesA;
+						
+						_mm256_mask_storeu_epi8(p+1, 1UL<<31, dataB);
+						dataB = _mm256_mask_expand_epi8(_mm256_set1_epi8('='), KNOT32(maskB), dataB);
+						_mm256_storeu_si256((__m256i*)p, dataB);
+						p += maskBitsB;
+					} else
+# endif
+					{
+						_mm256_mask_storeu_epi8(p+1, 1UL<<31, dataA);
+						dataA = _mm256_mask_alignr_epi8(dataA, (uint32_t)(-(int32_t)maskA), dataA, _mm256_permute4x64_epi64(dataA, _MM_SHUFFLE(1,0,3,2)), 15);
+						dataA = _mm256_ternarylogic_epi32(dataA, cmpA, _mm256_set1_epi8('='), 0xb8); // (data & ~cmp) | (cmp & '=')
+						_mm256_storeu_si256((__m256i*)p, dataA);
+						p += outputBytesA;
+						
+						_mm256_mask_storeu_epi8(p+1, 1UL<<31, dataB);
+						dataB = _mm256_mask_alignr_epi8(dataB, (uint32_t)(-(int32_t)maskB), dataB, _mm256_permute4x64_epi64(dataB, _MM_SHUFFLE(1,0,3,2)), 15);
+						dataB = _mm256_ternarylogic_epi32(dataB, cmpB, _mm256_set1_epi8('='), 0xb8);
+						_mm256_storeu_si256((__m256i*)p, dataB);
+						p += maskBitsB;
+					}
+				} else
+#endif
+				{
+					bitIndexA = _lzcnt_u32(maskA);
+					bitIndexB = _lzcnt_u32(maskB);
+					__m256i mergeMaskA = _mm256_load_si256((const __m256i*)(lookupsAVX2->expandMergemix + bitIndexA*2*YMM_SIZE));
+					__m256i mergeMaskB = _mm256_load_si256((const __m256i*)(lookupsAVX2->expandMergemix + bitIndexB*2*YMM_SIZE));
+					
+#if defined(__tune_bdver4__) || defined(__tune_znver1__)
+					// avoid slower 32-byte crossing loads on Zen1
+					__m256i dataAShifted = _mm256_alignr_epi8(
+						dataA,
+						_mm256_inserti128_si256(dataA, _mm256_castsi256_si128(dataA), 1),
+						15
+					);
+					__m256i dataBShifted = _mm256_alignr_epi8(
+						dataB,
+						_mm256_inserti128_si256(dataB, _mm256_castsi256_si128(dataB), 1),
+						15
+					);
+#else
+					__m256i dataAShifted = _mm256_loadu_si256((__m256i *)(es + i - YMM_SIZE*2 - 1));
+					__m256i dataBShifted = _mm256_loadu_si256((__m256i *)(es + i - YMM_SIZE - 1));
+#endif
+					dataA = _mm256_andnot_si256(cmpA, dataA); // clear space for '=' char
+					dataA = _mm256_blendv_epi8(dataAShifted, dataA, mergeMaskA);
+					dataA = _mm256_add_epi8(dataA, _mm256_load_si256((const __m256i*)(lookupsAVX2->expandMergemix + bitIndexA*2*YMM_SIZE) + 1));
+					_mm256_storeu_si256((__m256i*)p, dataA);
+					p[YMM_SIZE] = es[i-1-YMM_SIZE] + 42 + (64 & (maskA>>(YMM_SIZE-1-6)));
+					p += outputBytesA;
+					
+					dataB = _mm256_andnot_si256(cmpB, dataB);
+					dataB = _mm256_blendv_epi8(dataBShifted, dataB, mergeMaskB);
+					dataB = _mm256_add_epi8(dataB, _mm256_load_si256((const __m256i*)(lookupsAVX2->expandMergemix + bitIndexB*2*YMM_SIZE) + 1));
+					_mm256_storeu_si256((__m256i*)p, dataB);
+					p[YMM_SIZE] = es[i-1] + 42 + (64 & (maskB>>(YMM_SIZE-1-6)));
+					p += maskBitsB;
+				}
+				col += outputBytesA + maskBitsB;
+				
+				if(col >= 0)
+					goto _encode_loop_branch_fast_eol;
+			}
+		}
+	} while(i < 0);
+	
+	_mm256_zeroupper();
+	
+	*colOffset = (int)(col + line_size -1);
+	dest = p;
+	len = -(i - INPUT_OFFSET);
+}
+
+#endif

package/src/encoder_common.h

@@ -0,0 +1,109 @@
+#ifndef __YENC_ENCODER_COMMON
+#define __YENC_ENCODER_COMMON
+
+// lookup tables for scalar processing
+#define _B1(n) _B(n), _B(n+1), _B(n+2), _B(n+3)
+#define _B2(n) _B1(n), _B1(n+4), _B1(n+8), _B1(n+12)
+#define _B3(n) _B2(n), _B2(n+16), _B2(n+32), _B2(n+48)
+#define _BX _B3(0), _B3(64), _B3(128), _B3(192)
+
+static const unsigned char escapeLUT[256] = { // whether or not the character is critical
+#define _B(n) ((n == 214 || n == '\r'+214 || n == '\n'+214 || n == '='-42) ? 0 : (n+42) & 0xff)
+	_BX
+#undef _B
+};
+static const uint16_t escapedLUT[256] = { // escaped sequences for characters that need escaping
+#define _B(n) ((n == 214 || n == 214+'\r' || n == 214+'\n' || n == '='-42 || n == 214+'\t' || n == 214+' ' || n == '.'-42) ? UINT16_PACK('=', ((n+42+64)&0xff)) : 0)
+	_BX
+#undef _B
+};
+
+#undef _B1
+#undef _B2
+#undef _B3
+#undef _BX
+
+
+size_t do_encode_generic(int line_size, int* colOffset, const unsigned char* HEDLEY_RESTRICT src, unsigned char* HEDLEY_RESTRICT dest, size_t len, int doEnd);
+
+template<void(&kernel)(int, int*, const uint8_t* HEDLEY_RESTRICT, uint8_t* HEDLEY_RESTRICT&, size_t&)>
+static size_t do_encode_simd(int line_size, int* colOffset, const uint8_t* HEDLEY_RESTRICT src, uint8_t* HEDLEY_RESTRICT dest, size_t len, int doEnd) {
+	if(len < 1) return 0;
+	if(line_size < 12) { // short lines probably not worth processing in a SIMD way
+		// we assume at least the first and last char exist in the line, and since the first char could be escaped, and SIMD encoder assumes at least one non-first/last char, assumption means that line size has to be >= 4
+		return do_encode_generic(line_size, colOffset, src, dest, len, doEnd);
+	}
+	
+	const uint8_t* es = src + len;
+	uint8_t* p = dest;
+	
+	if(*colOffset < 0) *colOffset = 0; // sanity check
+	
+	kernel(line_size, colOffset, es, p, len);
+	
+	// scalar loop to process remaining
+	long i = -(long)len;
+	if(*colOffset == 0 && i < 0) {
+		uint8_t c = es[i++];
+		if (LIKELIHOOD(0.0273, escapedLUT[c] != 0)) {
+			memcpy(p, escapedLUT + c, 2);
+			p += 2;
+			*colOffset = 2;
+		} else {
+			*(p++) = c + 42;
+			*colOffset = 1;
+		}
+	}
+	while(i < 0) {
+		uint8_t c = es[i++];
+		if(*colOffset < line_size-1) {
+			if(!escapeLUT[c]) {
+				p[0] = '=';
+				p[1] = c+42+64;
+				p += 2;
+				(*colOffset) += 2;
+			} else {
+				*(p++) = escapeLUT[c];
+				(*colOffset) += 1;
+			}
+		} else {
+			if(*colOffset < line_size) {
+				if (escapedLUT[c] && c != '.'-42) {
+					memcpy(p, escapedLUT + c, 2);
+					p += 2;
+				} else {
+					*(p++) = c + 42;
+				}
+				if(i == 0) break;
+				c = es[i++];
+			}
+			
+			// handle EOL
+			if (escapedLUT[c]) {
+				uint32_t w = UINT32_16_PACK(UINT16_PACK('\r', '\n'), (uint32_t)escapedLUT[c]);
+				memcpy(p, &w, sizeof(w));
+				p += 4;
+				*colOffset = 2;
+			} else {
+				uint32_t w = UINT32_PACK('\r', '\n', (uint32_t)(c+42), 0);
+				memcpy(p, &w, sizeof(w));
+				p += 3;
+				*colOffset = 1;
+			}
+		}
+	}
+	
+	if(doEnd) {
+		// special case: if the last character is a space/tab, it needs to be escaped as it's the final character on the line
+		unsigned char lc = *(p-1);
+		if(lc == '\t' || lc == ' ') {
+			p[-1] = '=';
+			*p = lc+64;
+			p++;
+			(*colOffset)++;
+		}
+	}
+	return p - dest;
+}
+
+#endif /* __YENC_ENCODER_COMMON */