yencode 1.0.8 → 1.1.2

package/src/decoder_avx2_base.h

@@ -0,0 +1,615 @@
+
+#ifdef __AVX2__
+
+// GCC (ver 6-10(dev)) fails to optimize pure C version of mask testing, but has this intrinsic; Clang >= 7 optimizes C version fine
+#if defined(__GNUC__) && __GNUC__ >= 7
+# define KORTEST32(a, b) !_kortestz_mask32_u8((a), (b))
+# define KAND32(a, b) _kand_mask32((a), (b))
+# define KOR32(a, b) _kor_mask32((a), (b))
+#else
+# define KORTEST32(a, b) ((a) | (b))
+# define KAND32(a, b) ((a) & (b))
+# define KOR32(a, b) ((a) | (b))
+#endif
+
+#pragma pack(16)
+static struct {
+	/*align16*/ struct { char bytes[16]; } compact[32768];
+	uint8_t eqFix[256];
+} * HEDLEY_RESTRICT lookups;
+#pragma pack()
+
+
+static HEDLEY_ALWAYS_INLINE __m256i force_align_read_256(const void* p) {
+#ifdef _MSC_VER
+	// MSVC complains about casting away volatile
+	return *(__m256i *)(p);
+#else
+	return *(volatile __m256i *)(p);
+#endif
+}
+
+// _mm256_castsi128_si256, but upper is defined to be 0
+#if (defined(__clang__) && __clang_major__ >= 5 && (!defined(__APPLE__) || __clang_major__ >= 7)) || (defined(__GNUC__) && __GNUC__ >= 10)
+// intrinsic unsupported in GCC 9 and MSVC < 2017
+# define zext128_256 _mm256_zextsi128_si256
+#else
+// technically a cast is incorrect, due to upper 128 bits being undefined, but should usually work fine
+// alternative may be `_mm256_set_m128i(_mm_setzero_si128(), v)` but unsupported on GCC < 7, and most compilers generate a VINSERTF128 instruction for it
+# ifdef __OPTIMIZE__
+#  define zext128_256 _mm256_castsi128_si256
+# else
+#  define zext128_256(x) _mm256_inserti128_si256(_mm256_setzero_si256(), x, 0)
+# endif
+#endif
+
+
+template<bool isRaw, bool searchEnd, enum YEncDecIsaLevel use_isa>
+HEDLEY_ALWAYS_INLINE void do_decode_avx2(const uint8_t* HEDLEY_RESTRICT src, long& len, unsigned char* HEDLEY_RESTRICT & p, unsigned char& _escFirst, uint16_t& _nextMask) {
+	HEDLEY_ASSUME(_escFirst == 0 || _escFirst == 1);
+	HEDLEY_ASSUME(_nextMask == 0 || _nextMask == 1 || _nextMask == 2);
+	uintptr_t escFirst = _escFirst;
+	__m256i yencOffset = escFirst ? _mm256_set_epi8(
+		-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,
+		-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42-64
+	) : _mm256_set1_epi8(-42);
+	__m256i minMask = _mm256_set1_epi8('.');
+	if(_nextMask && isRaw) {
+		minMask = _mm256_set_epi8(
+			'.','.','.','.','.','.','.','.','.','.','.','.','.','.','.','.',
+			'.','.','.','.','.','.','.','.','.','.','.','.','.','.',_nextMask==2?0:'.',_nextMask==1?0:'.'
+		);
+	}
+	intptr_t i;
+	for(i = -len; i; i += sizeof(__m256i)*2) {
+		__m256i oDataA = _mm256_load_si256((__m256i *)(src+i));
+		__m256i oDataB = _mm256_load_si256((__m256i *)(src+i) + 1);
+		
+		// search for special chars
+		__m256i cmpA = _mm256_cmpeq_epi8(oDataA, _mm256_shuffle_epi8(
+			_mm256_set_epi8(
+				-1,'=','\r',-1,-1,'\n',-1,-1,-1,-1,-1,-1,-1,-1,-1,'.',
+				-1,'=','\r',-1,-1,'\n',-1,-1,-1,-1,-1,-1,-1,-1,-1,'.'
+			),
+			_mm256_min_epu8(oDataA, minMask)
+		));
+		__m256i cmpB = _mm256_cmpeq_epi8(oDataB, _mm256_shuffle_epi8(
+			_mm256_set_epi8(
+				-1,'=','\r',-1,-1,'\n',-1,-1,-1,-1,-1,-1,-1,-1,-1,'.',
+				-1,'=','\r',-1,-1,'\n',-1,-1,-1,-1,-1,-1,-1,-1,-1,'.'
+			),
+			_mm256_min_epu8(oDataB, _mm256_set1_epi8('.'))
+		));
+		
+		// TODO: can OR the vectors together to save generating a mask, but may not be worth it
+		uint64_t mask = (uint32_t)_mm256_movemask_epi8(cmpB); // not the most accurate mask if we have invalid sequences; we fix this up later
+		mask = (mask << 32) | (uint32_t)_mm256_movemask_epi8(cmpA);
+		__m256i dataA, dataB;
+		if(use_isa >= ISA_LEVEL_AVX3)
+			dataA = _mm256_add_epi8(oDataA, yencOffset);
+		
+		if (mask != 0) {
+			__m256i cmpEqA = _mm256_cmpeq_epi8(oDataA, _mm256_set1_epi8('='));
+			__m256i cmpEqB = _mm256_cmpeq_epi8(oDataB, _mm256_set1_epi8('='));
+			uint64_t maskEq = (uint32_t)_mm256_movemask_epi8(cmpEqB);
+			maskEq = (maskEq << 32) | (uint32_t)_mm256_movemask_epi8(cmpEqA);
+			
+			// handle \r\n. sequences
+			// RFC3977 requires the first dot on a line to be stripped, due to dot-stuffing
+			if((isRaw || searchEnd) && LIKELIHOOD(0.45, mask != maskEq)) {
+#if 0
+				// prefer shuffling data over unaligned loads on Zen (unknown if worth it on Zen2/Excavator)
+				// unfortunately not beneficial, probably due to available register pressure; this is left here because it could be beneficial if we figure out how to use fewer registers
+				__m256i nextDataA, nextDataB;
+				if(searchEnd) {
+					nextDataA = _mm256_inserti128_si256(
+						_mm256_castsi128_si256(_mm256_extracti128_si256(oDataA, 1)),
+						_mm256_castsi256_si128(oDataB),
+						1
+					);
+					nextDataB = _mm256_inserti128_si256(
+						_mm256_castsi128_si256(_mm256_extracti128_si256(oDataB, 1)),
+						_mm_load_si128((__m128i*)(src+i+sizeof(__m256i)*2)),
+						1
+					);
+				}
+# define SHIFT_DATA_A(offs) (searchEnd ? _mm256_alignr_epi8(nextDataA, oDataA, offs) : _mm256_loadu_si256((__m256i *)(src+i+offs)))
+# define SHIFT_DATA_B(offs) (searchEnd ? _mm256_alignr_epi8(nextDataB, oDataB, offs) : _mm256_loadu_si256((__m256i *)(src+i+offs) + 1))
+#else
+# define SHIFT_DATA_A(offs) _mm256_loadu_si256((__m256i *)(src+i+offs))
+# define SHIFT_DATA_B(offs) _mm256_loadu_si256((__m256i *)(src+i+offs) + 1)
+#endif
+				__m256i tmpData2A = SHIFT_DATA_A(2);
+				__m256i tmpData2B = SHIFT_DATA_B(2);
+				__m256i match2EqA, match2EqB;
+#if defined(__AVX512VL__) && defined(__AVX512BW__)
+				__mmask32 match2EqMaskA, match2EqMaskB;
+				__mmask32 match0CrMaskA, match0CrMaskB;
+				__mmask32 match2CrXDtMaskA, match2CrXDtMaskB;
+				if(use_isa >= ISA_LEVEL_AVX3 && searchEnd) {
+					match2EqMaskA = _mm256_cmpeq_epi8_mask(_mm256_set1_epi8('='), tmpData2A);
+					match2EqMaskB = _mm256_cmpeq_epi8_mask(_mm256_set1_epi8('='), tmpData2B);
+				} else
+#endif
+				if(searchEnd) {
+					match2EqA = _mm256_cmpeq_epi8(_mm256_set1_epi8('='), tmpData2A);
+					match2EqB = _mm256_cmpeq_epi8(_mm256_set1_epi8('='), tmpData2B);
+				}
+				
+				int partialKillDotFound;
+				__m256i match2CrXDtA, match2CrXDtB;
+				if(isRaw) {
+					// find patterns of \r_.
+					
+#if defined(__AVX512VL__) && defined(__AVX512BW__)
+					if(use_isa >= ISA_LEVEL_AVX3) {
+						match0CrMaskA = _mm256_cmpeq_epi8_mask(oDataA, _mm256_set1_epi8('\r'));
+						match0CrMaskB = _mm256_cmpeq_epi8_mask(oDataB, _mm256_set1_epi8('\r'));
+						match2CrXDtMaskA = _mm256_mask_cmpeq_epi8_mask(match0CrMaskA, tmpData2A, _mm256_set1_epi8('.'));
+						match2CrXDtMaskB = _mm256_mask_cmpeq_epi8_mask(match0CrMaskB, tmpData2B, _mm256_set1_epi8('.'));
+						partialKillDotFound = KORTEST32(match2CrXDtMaskA, match2CrXDtMaskB);
+					} else
+#endif
+					{
+						match2CrXDtA = _mm256_and_si256(
+							_mm256_cmpeq_epi8(oDataA, _mm256_set1_epi8('\r')),
+							_mm256_cmpeq_epi8(tmpData2A, _mm256_set1_epi8('.'))
+						);
+						match2CrXDtB = _mm256_and_si256(
+							_mm256_cmpeq_epi8(oDataB, _mm256_set1_epi8('\r')),
+							_mm256_cmpeq_epi8(tmpData2B, _mm256_set1_epi8('.'))
+						);
+						partialKillDotFound = _mm256_movemask_epi8(_mm256_or_si256(
+							match2CrXDtA, match2CrXDtB
+						));
+					}
+				}
+				
+				if(isRaw && LIKELIHOOD(0.002, partialKillDotFound)) {
+					// merge matches for \r\n.
+					__m256i match2NlDotA, match1NlA;
+					__m256i match2NlDotB, match1NlB;
+#if defined(__AVX512VL__) && defined(__AVX512BW__)
+					__mmask32 match1NlMaskA, match1NlMaskB;
+					__mmask32 match2NlDotMaskA, match2NlDotMaskB;
+					if(use_isa >= ISA_LEVEL_AVX3) {
+						match1NlMaskA = _mm256_mask_cmpeq_epi8_mask(
+							match0CrMaskA,
+							_mm256_set1_epi8('\n'),
+							SHIFT_DATA_A(1)
+						);
+						match1NlMaskB = _mm256_mask_cmpeq_epi8_mask(
+							match0CrMaskB,
+							_mm256_set1_epi8('\n'),
+							SHIFT_DATA_B(1)
+						);
+						match2NlDotMaskA = KAND32(match2CrXDtMaskA, match1NlMaskA);
+						match2NlDotMaskB = KAND32(match2CrXDtMaskB, match1NlMaskB);
+					} else
+#endif
+					{
+						__m256i match1LfA = _mm256_cmpeq_epi8(
+							_mm256_set1_epi8('\n'),
+							SHIFT_DATA_A(1)
+						);
+						__m256i match1LfB = _mm256_cmpeq_epi8(
+							_mm256_set1_epi8('\n'),
+							SHIFT_DATA_B(1)
+						);
+						// force re-computing these to avoid register spills elsewhere
+						match1NlA = _mm256_and_si256(match1LfA, _mm256_cmpeq_epi8(force_align_read_256(src+i), _mm256_set1_epi8('\r')));
+						match1NlB = _mm256_and_si256(match1LfB, _mm256_cmpeq_epi8(force_align_read_256(src+i + sizeof(__m256i)), _mm256_set1_epi8('\r')));
+						match2NlDotA = _mm256_and_si256(match2CrXDtA, match1NlA);
+						match2NlDotB = _mm256_and_si256(match2CrXDtB, match1NlB);
+					}
+					if(searchEnd) {
+						__m256i tmpData4A;
+#if defined(__AVX512VL__) && defined(PLATFORM_AMD64)
+						if(use_isa >= ISA_LEVEL_AVX3)
+							// AVX512 with 32 registers shouldn't have any issue with holding onto oData* in registers
+							tmpData4A = _mm256_alignr_epi32(oDataB, oDataA, 1);
+						else
+#endif
+							tmpData4A = SHIFT_DATA_A(4);
+						__m256i tmpData4B = SHIFT_DATA_B(4);
+						// match instances of \r\n.\r\n and \r\n.=y
+						__m256i match3CrA = _mm256_cmpeq_epi8(
+							_mm256_set1_epi8('\r'),
+							SHIFT_DATA_A(3)
+						);
+						__m256i match3CrB = _mm256_cmpeq_epi8(
+							_mm256_set1_epi8('\r'),
+							SHIFT_DATA_B(3)
+						);
+						__m256i match4LfA = _mm256_cmpeq_epi8(tmpData4A, _mm256_set1_epi8('\n'));
+						__m256i match4LfB = _mm256_cmpeq_epi8(tmpData4B, _mm256_set1_epi8('\n'));
+						__m256i match4EqYA = _mm256_cmpeq_epi16(tmpData4A, _mm256_set1_epi16(0x793d)); // =y
+						__m256i match4EqYB = _mm256_cmpeq_epi16(tmpData4B, _mm256_set1_epi16(0x793d)); // =y
+						
+						int matchEnd;
+#if defined(__AVX512VL__) && defined(__AVX512BW__)
+						if(use_isa >= ISA_LEVEL_AVX3) {
+							__mmask32 match3EqYMaskA = _mm256_mask_cmpeq_epi8_mask(
+								match2EqMaskA,
+								_mm256_set1_epi8('y'),
+								SHIFT_DATA_A(3)
+							);
+							__mmask32 match3EqYMaskB = _mm256_mask_cmpeq_epi8_mask(
+								match2EqMaskB,
+								_mm256_set1_epi8('y'),
+								SHIFT_DATA_B(3)
+							);
+							__m256i match34EqYA, match34EqYB;
+# ifdef __AVX512VBMI2__
+							if(use_isa >= ISA_LEVEL_VBMI2) {
+								match34EqYA = _mm256_shrdi_epi16(_mm256_movm_epi8(match3EqYMaskA), match4EqYA, 8);
+								match34EqYB = _mm256_shrdi_epi16(_mm256_movm_epi8(match3EqYMaskB), match4EqYB, 8);
+							} else
+# endif
+							{
+								// (match4EqY & 0xff00) | (match3EqY >> 8)
+								match34EqYA = _mm256_mask_blend_epi8(match3EqYMaskA>>1, _mm256_and_si256(match4EqYA, _mm256_set1_epi16(-0xff)), _mm256_set1_epi8(-1));
+								match34EqYB = _mm256_mask_blend_epi8(match3EqYMaskB>>1, _mm256_and_si256(match4EqYB, _mm256_set1_epi16(-0xff)), _mm256_set1_epi8(-1));
+							}
+							// merge \r\n and =y matches for tmpData4
+							__m256i match4EndA = _mm256_ternarylogic_epi32(match34EqYA, match3CrA, match4LfA, 0xF8); // (match3Cr & match4Lf) | match34EqY
+							__m256i match4EndB = _mm256_ternarylogic_epi32(match34EqYB, match3CrB, match4LfB, 0xF8);
+							// merge with \r\n. and combine
+							matchEnd = KORTEST32(
+								KOR32(
+									_mm256_mask_test_epi8_mask(match2NlDotMaskA, match4EndA, match4EndA),
+									KAND32(match3EqYMaskA, match1NlMaskA)
+								),
+								KOR32(
+									_mm256_mask_test_epi8_mask(match2NlDotMaskB, match4EndB, match4EndB),
+									KAND32(match3EqYMaskB, match1NlMaskB)
+								)
+							);
+						} else
+#endif
+						{
+							__m256i match3EqYA = _mm256_and_si256(match2EqA, _mm256_cmpeq_epi8(
+								_mm256_set1_epi8('y'),
+								SHIFT_DATA_A(3)
+							));
+							__m256i match3EqYB = _mm256_and_si256(match2EqB, _mm256_cmpeq_epi8(
+								_mm256_set1_epi8('y'),
+								SHIFT_DATA_B(3)
+							));
+							match4EqYA = _mm256_slli_epi16(match4EqYA, 8); // TODO: also consider using PBLENDVB here with shifted match3EqY instead
+							match4EqYB = _mm256_slli_epi16(match4EqYB, 8);
+							// merge \r\n and =y matches for tmpData4
+							__m256i match4EndA = _mm256_or_si256(
+								_mm256_and_si256(match3CrA, match4LfA),
+								_mm256_or_si256(match4EqYA, _mm256_srli_epi16(match3EqYA, 8)) // _mm256_srli_si256 by 1 also works
+							);
+							__m256i match4EndB = _mm256_or_si256(
+								_mm256_and_si256(match3CrB, match4LfB),
+								_mm256_or_si256(match4EqYB, _mm256_srli_epi16(match3EqYB, 8))
+							);
+							// merge with \r\n.
+							match4EndA = _mm256_and_si256(match4EndA, match2NlDotA);
+							match4EndB = _mm256_and_si256(match4EndB, match2NlDotB);
+							// match \r\n=y
+							__m256i match3EndA = _mm256_and_si256(match3EqYA, match1NlA);
+							__m256i match3EndB = _mm256_and_si256(match3EqYB, match1NlB);
+							// combine match sequences
+							matchEnd = _mm256_movemask_epi8(_mm256_or_si256(
+								_mm256_or_si256(match4EndA, match3EndA),
+								_mm256_or_si256(match4EndB, match3EndB)
+							));
+						}
+						if(LIKELIHOOD(0.002, matchEnd)) {
+							// terminator found
+							// there's probably faster ways to do this, but reverting to scalar code should be good enough
+							len += (long)i;
+							break;
+						}
+					}
+#if defined(__AVX512VL__) && defined(__AVX512BW__)
+					if(use_isa >= ISA_LEVEL_AVX3) {
+						mask |= (uint64_t)match2NlDotMaskA << 2;
+						mask |= (uint64_t)match2NlDotMaskB << 34;
+						minMask = _mm256_maskz_mov_epi8(~(match2NlDotMaskB>>30), _mm256_set1_epi8('.'));
+					} else
+#endif
+					{
+						mask |= (uint64_t)((uint32_t)_mm256_movemask_epi8(match2NlDotA)) << 2;
+						mask |= (uint64_t)((uint32_t)_mm256_movemask_epi8(match2NlDotB)) << 34;
+						match2NlDotB = zext128_256(_mm_srli_si128(_mm256_extracti128_si256(match2NlDotB, 1), 14));
+						minMask = _mm256_subs_epu8(_mm256_set1_epi8('.'), match2NlDotB);
+					}
+				}
+				else if(searchEnd) {
+					bool partialEndFound;
+					__m256i match3EqYA, match3EqYB;
+#if defined(__AVX512VL__) && defined(__AVX512BW__)
+					__mmask32 match3EqYMaskA, match3EqYMaskB;
+					if(use_isa >= ISA_LEVEL_AVX3) {
+						match3EqYMaskA = _mm256_mask_cmpeq_epi8_mask(
+							match2EqMaskA,
+							_mm256_set1_epi8('y'),
+							SHIFT_DATA_A(3)
+						);
+						match3EqYMaskB = _mm256_mask_cmpeq_epi8_mask(
+							match2EqMaskB,
+							_mm256_set1_epi8('y'),
+							SHIFT_DATA_B(3)
+						);
+						partialEndFound = KORTEST32(match3EqYMaskA, match3EqYMaskB);
+					} else
+#endif
+					{
+						__m256i match3YA = _mm256_cmpeq_epi8(
+							_mm256_set1_epi8('y'),
+							SHIFT_DATA_A(3)
+						);
+						__m256i match3YB = _mm256_cmpeq_epi8(
+							_mm256_set1_epi8('y'),
+							SHIFT_DATA_B(3)
+						);
+						match3EqYA = _mm256_and_si256(match2EqA, match3YA);
+						match3EqYB = _mm256_and_si256(match2EqB, match3YB);
+						partialEndFound = _mm256_movemask_epi8(_mm256_or_si256(match3EqYA, match3EqYB));
+					}
+					if(LIKELIHOOD(0.002, partialEndFound)) {
+						bool endFound;
+#if defined(__AVX512VL__) && defined(__AVX512BW__)
+						if(use_isa >= ISA_LEVEL_AVX3) {
+							__mmask32 match3LfEqYMaskA = _mm256_mask_cmpeq_epi8_mask(
+								match3EqYMaskA,
+								_mm256_set1_epi8('\n'),
+								SHIFT_DATA_A(1)
+							);
+							__mmask32 match3LfEqYMaskB = _mm256_mask_cmpeq_epi8_mask(
+								match3EqYMaskB,
+								_mm256_set1_epi8('\n'),
+								SHIFT_DATA_B(1)
+							);
+							
+							endFound = KORTEST32(
+								_mm256_mask_cmpeq_epi8_mask(match3LfEqYMaskA, oDataA, _mm256_set1_epi8('\r')),
+								_mm256_mask_cmpeq_epi8_mask(match3LfEqYMaskB, oDataB, _mm256_set1_epi8('\r'))
+							);
+						} else
+#endif
+						{
+							__m256i match1LfA = _mm256_cmpeq_epi8(
+								_mm256_set1_epi8('\n'),
+								SHIFT_DATA_A(1)
+							);
+							__m256i match1LfB = _mm256_cmpeq_epi8(
+								_mm256_set1_epi8('\n'),
+								SHIFT_DATA_B(1)
+							);
+							endFound = _mm256_movemask_epi8(_mm256_or_si256(
+								_mm256_and_si256(
+									match3EqYA,
+									_mm256_and_si256(match1LfA, _mm256_cmpeq_epi8(force_align_read_256(src+i), _mm256_set1_epi8('\r')))
+								),
+								_mm256_and_si256(
+									match3EqYB,
+									_mm256_and_si256(match1LfB, _mm256_cmpeq_epi8(force_align_read_256(src+i + sizeof(__m256i)), _mm256_set1_epi8('\r')))
+								)
+							));
+						}
+						if(endFound) {
+							len += (long)i;
+							break;
+						}
+					}
+					if(isRaw) minMask = _mm256_set1_epi8('.');
+				}
+				else if(isRaw) // no \r_. found
+					minMask = _mm256_set1_epi8('.');
+			}
+#undef SHIFT_DATA_A
+#undef SHIFT_DATA_B
+			
+			if(use_isa >= ISA_LEVEL_AVX3)
+				dataB = _mm256_add_epi8(oDataB, _mm256_set1_epi8(-42));
+			
+			if(LIKELIHOOD(0.0001, (mask & ((maskEq << 1) + escFirst)) != 0)) {
+				unsigned tmp = lookups->eqFix[(maskEq&0xff) & ~(uint64_t)escFirst];
+				uint64_t maskEq2 = tmp;
+				for(int j=8; j<64; j+=8) {
+					tmp = lookups->eqFix[(unsigned)((maskEq>>j)&0xff) & ~(tmp>>7)];
+					maskEq2 |= (uint64_t)tmp<<j;
+				}
+				maskEq = maskEq2;
+				
+				mask &= ~(uint64_t)escFirst;
+				escFirst = tmp>>7;
+				// next, eliminate anything following a `=` from the special char mask; this eliminates cases of `=\r` so that they aren't removed
+				maskEq <<= 1;
+				mask &= ~maskEq;
+				
+				// unescape chars following `=`
+#if defined(__AVX512VL__) && defined(__AVX512BW__)
+				if(use_isa >= ISA_LEVEL_AVX3) {
+					// GCC < 7 seems to generate rubbish assembly for this
+					dataA = _mm256_mask_add_epi8(
+						dataA,
+						(__mmask32)maskEq,
+						dataA,
+						_mm256_set1_epi8(-64)
+					);
+					dataB = _mm256_mask_add_epi8(
+						dataB,
+						(__mmask32)(maskEq>>32),
+						dataB,
+						_mm256_set1_epi8(-64)
+					);
+				} else
+#endif
+				{
+					// convert maskEq into vector form (i.e. reverse pmovmskb)
+#ifdef PLATFORM_AMD64
+					__m256i vMaskEq = _mm256_broadcastq_epi64(_mm_cvtsi64_si128(maskEq));
+#else
+					__m256i vMaskEq = _mm256_permute4x64_epi64(_mm256_insert_epi32(
+						_mm256_set_epi32(0,0,0,0, 0,0,0, maskEq & 0xffffffff),
+						maskEq >> 32,
+						1
+					), 0);
+#endif
+					__m256i vMaskEqA = _mm256_shuffle_epi8(vMaskEq, _mm256_set_epi32(
+						0x03030303, 0x03030303, 0x02020202, 0x02020202,
+						0x01010101, 0x01010101, 0x00000000, 0x00000000
+					));
+					__m256i vMaskEqB = _mm256_shuffle_epi8(vMaskEq, _mm256_set_epi32(
+						0x07070707, 0x07070707, 0x06060606, 0x06060606,
+						0x05050505, 0x05050505, 0x04040404, 0x04040404
+					));
+					vMaskEqA = _mm256_cmpeq_epi8(
+						_mm256_and_si256(vMaskEqA, _mm256_set1_epi64x(0x8040201008040201ULL)),
+						_mm256_set1_epi64x(0x8040201008040201ULL)
+					);
+					vMaskEqB = _mm256_cmpeq_epi8(
+						_mm256_and_si256(vMaskEqB, _mm256_set1_epi64x(0x8040201008040201ULL)),
+						_mm256_set1_epi64x(0x8040201008040201ULL)
+					);
+					dataA = _mm256_add_epi8(oDataA, _mm256_blendv_epi8(yencOffset, _mm256_set1_epi8(-42-64), vMaskEqA));
+					dataB = _mm256_add_epi8(oDataB, _mm256_blendv_epi8(_mm256_set1_epi8(-42), _mm256_set1_epi8(-42-64), vMaskEqB));
+				}
+			} else {
+				escFirst = (maskEq >> 63);
+				
+#if defined(__AVX512VL__) && defined(__AVX512BW__)
+				if(use_isa >= ISA_LEVEL_AVX3) {
+					dataA = _mm256_mask_add_epi8(
+						dataA,
+						(__mmask32)(maskEq << 1),
+						dataA,
+						_mm256_set1_epi8(-64)
+					);
+					dataB = _mm256_mask_add_epi8(
+						dataB,
+						(__mmask32)(maskEq >> 31),
+						dataB,
+						_mm256_set1_epi8(-64)
+					);
+				} else
+#endif
+				{
+					// << 1 byte
+					cmpEqA = _mm256_alignr_epi8(cmpEqA, _mm256_inserti128_si256(
+						_mm256_set1_epi8('='), _mm256_castsi256_si128(cmpEqA), 1
+					), 15);
+					cmpEqB = _mm256_cmpeq_epi8(_mm256_set1_epi8('='), _mm256_loadu_si256((__m256i *)(src+i-1) + 1));
+					dataA = _mm256_add_epi8(
+						oDataA,
+						_mm256_blendv_epi8(
+							yencOffset,
+							_mm256_set1_epi8(-42-64),
+							cmpEqA
+						)
+					);
+					dataB = _mm256_add_epi8(
+						oDataB,
+						_mm256_blendv_epi8(
+							_mm256_set1_epi8(-42),
+							_mm256_set1_epi8(-42-64),
+							cmpEqB
+						)
+					);
+				}
+			}
+			// subtract 64 from first element if escFirst == 1
+#if defined(__AVX512VL__) && defined(__AVX512BW__)
+			if(use_isa >= ISA_LEVEL_AVX3) {
+				yencOffset = _mm256_mask_add_epi8(_mm256_set1_epi8(-42), (__mmask32)escFirst, _mm256_set1_epi8(-42), _mm256_set1_epi8(-64));
+			} else
+#endif
+			{
+				yencOffset = _mm256_xor_si256(_mm256_set1_epi8(-42), zext128_256(
+					_mm_slli_epi16(_mm_cvtsi32_si128((int)escFirst), 6)
+				));
+			}
+			
+			// all that's left is to 'compress' the data (skip over masked chars)
+#if defined(__AVX512VBMI2__) && defined(__AVX512VL__)
+			if(use_isa >= ISA_LEVEL_VBMI2) {
+				_mm256_mask_compressstoreu_epi8(p, KNOT32(mask), dataA);
+				p -= popcnt32(mask & 0xffffffff);
+				_mm256_mask_compressstoreu_epi8((p + XMM_SIZE*2), KNOT32(mask>>32), dataB);
+				p += XMM_SIZE*4 - popcnt32(mask >> 32);
+			} else
+#endif
+			{
+				// lookup compress masks and shuffle
+				__m256i shuf = _mm256_inserti128_si256(
+					_mm256_castsi128_si256(_mm_load_si128((__m128i*)(lookups->compact + (mask & 0x7fff)))),
+					*(__m128i*)((char*)lookups->compact + ((mask >> 12) & 0x7fff0)),
+					1
+				);
+				dataA = _mm256_shuffle_epi8(dataA, shuf);
+				
+				_mm_storeu_si128((__m128i*)p, _mm256_castsi256_si128(dataA));
+				// increment output position
+				p -= popcnt32(mask & 0xffff);
+				
+				_mm_storeu_si128((__m128i*)(p + XMM_SIZE), _mm256_extracti128_si256(dataA, 1));
+				p -= popcnt32(mask & 0xffff0000);
+				
+#ifdef PLATFORM_AMD64
+				mask >>= 28;
+				shuf = _mm256_inserti128_si256(
+					_mm256_castsi128_si256(_mm_load_si128((__m128i*)((char*)lookups->compact + (mask & 0x7fff0)))),
+					*(__m128i*)((char*)lookups->compact + ((mask >> 16) & 0x7fff0)),
+					1
+				);
+				dataB = _mm256_shuffle_epi8(dataB, shuf);
+				
+				_mm_storeu_si128((__m128i*)(p + XMM_SIZE*2), _mm256_castsi256_si128(dataB));
+				p -= popcnt32(mask & 0xffff0);
+				
+				_mm_storeu_si128((__m128i*)(p + XMM_SIZE*3), _mm256_extracti128_si256(dataB, 1));
+				p -= popcnt32((unsigned int)(mask >> 20));
+#else
+				mask >>= 32;
+				shuf = _mm256_inserti128_si256(
+					_mm256_castsi128_si256(_mm_load_si128((__m128i*)(lookups->compact + (mask & 0x7fff)))),
+					*(__m128i*)((char*)lookups->compact + ((mask >> 12) & 0x7fff0)),
+					1
+				);
+				dataB = _mm256_shuffle_epi8(dataB, shuf);
+				
+				_mm_storeu_si128((__m128i*)(p + XMM_SIZE*2), _mm256_castsi256_si128(dataB));
+				p -= popcnt32(mask & 0xffff);
+				
+				_mm_storeu_si128((__m128i*)(p + XMM_SIZE*3), _mm256_extracti128_si256(dataB, 1));
+				p -= popcnt32(mask & 0xffff0000);
+#endif
+				p += XMM_SIZE*4;
+			}
+		} else {
+			if(use_isa < ISA_LEVEL_AVX3)
+				dataA = _mm256_add_epi8(oDataA, yencOffset);
+			dataB = _mm256_add_epi8(oDataB, _mm256_set1_epi8(-42));
+			
+			_mm256_storeu_si256((__m256i*)p, dataA);
+			_mm256_storeu_si256((__m256i*)p + 1, dataB);
+			p += sizeof(__m256i)*2;
+			escFirst = 0;
+			yencOffset = _mm256_set1_epi8(-42);
+		}
+	}
+	_escFirst = (unsigned char)escFirst;
+	if(isRaw) {
+		// this would be the trivial solution, but requires the compiler holding onto minMask throughout the loop:
+		//_nextMask = ~(uint16_t)_mm256_movemask_epi8(_mm256_cmpeq_epi8(minMask, _mm256_set1_epi8('.')));
+		// instead, just scan the memory to determine what to set nextMask to
+		if(len != 0) { // have to gone through at least one loop cycle
+			if(src[i-2] == '\r' && src[i-1] == '\n' && src[i] == '.')
+				_nextMask = 1;
+			else if(src[i-1] == '\r' && src[i] == '\n' && src[i+1] == '.')
+				_nextMask = 2;
+			else
+				_nextMask = 0;
+		}
+	} else
+		_nextMask = 0;
+	_mm256_zeroupper();
+}
+#endif