react-native-quick-crypto 1.0.18 → 1.1.0

package/deps/simdutf/src/icelake/icelake_convert_utf32_to_utf16.inl.cpp

@@ -0,0 +1,338 @@
+// file included directly
+
+template <endianness big_endian>
+std::pair<const char32_t *, char16_t *>
+avx512_convert_utf32_to_utf16(const char32_t *buf, size_t len,
+                              char16_t *utf16_output) {
+  const char32_t *end = buf + len;
+  __mmask32 forbidden_bytemask = 0;
+  const __m512i v_00000000 = _mm512_setzero_si512();
+  const __m512i v_ffff0000 = _mm512_set1_epi32((int32_t)0xffff0000);
+  const __m512i v_f800 = _mm512_set1_epi32((uint32_t)0xf800);
+  const __m512i v_d800 = _mm512_set1_epi32((uint32_t)0xd800);
+  const __m512i v_10ffff = _mm512_set1_epi32(0x10FFFF);
+  const __m512i v_10000 = _mm512_set1_epi32(0x10000);
+  const __m512i v_3ff0000 = _mm512_set1_epi32(0x3FF0000);
+  const __m512i v_3ff = _mm512_set1_epi32(0x3FF);
+  const __m512i v_dc00d800 = _mm512_set1_epi32((int32_t)0xDC00D800);
+
+  while (end - buf >= std::ptrdiff_t(16)) {
+    __m512i in = _mm512_loadu_si512(buf);
+
+    // no bits set above 16th bit <=> can pack to UTF16 without surrogate pairs
+    const __mmask16 saturation_bitmask =
+        _mm512_cmpeq_epi32_mask(_mm512_and_si512(in, v_ffff0000), v_00000000);
+
+    if (saturation_bitmask == 0xffff) {
+      forbidden_bytemask |=
+          _mm512_cmpeq_epi32_mask(_mm512_and_si512(in, v_f800), v_d800);
+
+      __m256i utf16_packed = _mm512_cvtepi32_epi16(in);
+      if (big_endian) {
+        const __m256i swap = _mm256_setr_epi8(
+            1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14, 1, 0, 3, 2, 5,
+            4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14);
+        utf16_packed = _mm256_shuffle_epi8(utf16_packed, swap);
+      }
+      _mm256_storeu_si256((__m256i *)utf16_output, utf16_packed);
+      utf16_output += 16;
+      buf += 16;
+    } else {
+      // saturation_bitmask == 1 words will generate 1 utf16 char,
+      // and saturation_bitmask == 0 words will generate 2 utf16 chars assuming
+      // no errors. Thus we need a output_mask which has the structure b_2i = 1,
+      // b_2i+1 = !saturation_bitmask_i
+      const __mmask32 output_mask = ~_pdep_u32(saturation_bitmask, 0xAAAAAAAA);
+      const __mmask16 surrogate_bitmask = __mmask16(~saturation_bitmask);
+      __mmask32 error = _mm512_mask_cmpeq_epi32_mask(
+          saturation_bitmask, _mm512_and_si512(in, v_f800), v_d800);
+      error |= _mm512_mask_cmpgt_epu32_mask(surrogate_bitmask, in, v_10ffff);
+      if (simdutf_unlikely(error)) {
+        return std::make_pair(nullptr, utf16_output);
+      }
+      __m512i v1, v2, v;
+      // for the bits saturation_bitmask == 0, we need to unpack the 32-bit word
+      // into two 16 bit words corresponding to high_surrogate and
+      // low_surrogate. Once the bits are unpacked and merged, the output will
+      // be compressed as per output_mask.
+      in = _mm512_mask_sub_epi32(in, surrogate_bitmask, in, v_10000);
+      v1 = _mm512_mask_slli_epi32(in, surrogate_bitmask, in, 16);
+      v1 = _mm512_mask_and_epi32(in, surrogate_bitmask, v1, v_3ff0000);
+      v2 = _mm512_mask_srli_epi32(in, surrogate_bitmask, in, 10);
+      v2 = _mm512_mask_and_epi32(in, surrogate_bitmask, v2, v_3ff);
+      v = _mm512_or_si512(v1, v2);
+      in = _mm512_mask_add_epi32(in, surrogate_bitmask, v, v_dc00d800);
+      if (big_endian) {
+        const __m512i swap_512 = _mm512_set_epi8(
+            14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1, 14, 15, 12,
+            13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1, 14, 15, 12, 13, 10, 11, 8,
+            9, 6, 7, 4, 5, 2, 3, 0, 1, 14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5,
+            2, 3, 0, 1);
+        in = _mm512_shuffle_epi8(in, swap_512);
+      }
+      // we deliberately avoid _mm512_mask_compressstoreu_epi16 for portability
+      // (AMD Zen4 has terrible performance with it, it is effectively broken)
+      __m512i compressed = _mm512_maskz_compress_epi16(output_mask, in);
+      auto written_out = _mm_popcnt_u32(output_mask);
+      _mm512_mask_storeu_epi16(utf16_output, _bzhi_u32(0xFFFFFFFF, written_out),
+                               compressed);
+      //_mm512_mask_compressstoreu_epi16(utf16_output, output_mask, in);
+      utf16_output += written_out;
+      buf += 16;
+    }
+  }
+
+  size_t remaining_len = size_t(end - buf);
+  if (remaining_len) {
+    __mmask16 input_mask = __mmask16((1 << remaining_len) - 1);
+    __m512i in = _mm512_maskz_loadu_epi32(input_mask, buf);
+    const __mmask16 saturation_bitmask =
+        _mm512_cmpeq_epi32_mask(_mm512_and_si512(in, v_ffff0000), v_00000000) &
+        input_mask;
+    if (saturation_bitmask == input_mask) {
+      forbidden_bytemask |=
+          _mm512_cmpeq_epi32_mask(_mm512_and_si512(in, v_f800), v_d800);
+
+      __m256i utf16_packed = _mm512_cvtepi32_epi16(in);
+      if (big_endian) {
+        const __m256i swap = _mm256_setr_epi8(
+            1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14, 1, 0, 3, 2, 5,
+            4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14);
+        utf16_packed = _mm256_shuffle_epi8(utf16_packed, swap);
+      }
+      _mm256_mask_storeu_epi16(utf16_output, input_mask, utf16_packed);
+      utf16_output += remaining_len;
+      buf += remaining_len;
+    } else {
+      const __mmask32 output_max_mask = (1 << (remaining_len * 2)) - 1;
+      const __mmask32 output_mask =
+          (~_pdep_u32(saturation_bitmask, 0xAAAAAAAA)) & output_max_mask;
+      const __mmask16 surrogate_bitmask =
+          __mmask16(~saturation_bitmask) & input_mask;
+      __mmask32 error = _mm512_mask_cmpeq_epi32_mask(
+          saturation_bitmask, _mm512_and_si512(in, v_f800), v_d800);
+      error |= _mm512_mask_cmpgt_epu32_mask(surrogate_bitmask, in, v_10ffff);
+      if (simdutf_unlikely(error)) {
+        return std::make_pair(nullptr, utf16_output);
+      }
+      __m512i v1, v2, v;
+      in = _mm512_mask_sub_epi32(in, surrogate_bitmask, in, v_10000);
+      v1 = _mm512_mask_slli_epi32(in, surrogate_bitmask, in, 16);
+      v1 = _mm512_mask_and_epi32(in, surrogate_bitmask, v1, v_3ff0000);
+      v2 = _mm512_mask_srli_epi32(in, surrogate_bitmask, in, 10);
+      v2 = _mm512_mask_and_epi32(in, surrogate_bitmask, v2, v_3ff);
+      v = _mm512_or_si512(v1, v2);
+      in = _mm512_mask_add_epi32(in, surrogate_bitmask, v, v_dc00d800);
+      if (big_endian) {
+        const __m512i swap_512 = _mm512_set_epi8(
+            14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1, 14, 15, 12,
+            13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1, 14, 15, 12, 13, 10, 11, 8,
+            9, 6, 7, 4, 5, 2, 3, 0, 1, 14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5,
+            2, 3, 0, 1);
+        in = _mm512_shuffle_epi8(in, swap_512);
+      }
+      // we deliberately avoid _mm512_mask_compressstoreu_epi16 for portability
+      // (AMD Zen4 has terrible performance with it, it is effectively broken)
+      __m512i compressed = _mm512_maskz_compress_epi16(output_mask, in);
+      auto written_out = _mm_popcnt_u32(output_mask);
+      _mm512_mask_storeu_epi16(utf16_output, _bzhi_u32(0xFFFFFFFF, written_out),
+                               compressed);
+      //_mm512_mask_compressstoreu_epi16(utf16_output, output_mask, in);
+      utf16_output += written_out;
+      buf += remaining_len;
+    }
+  }
+
+  // check for invalid input
+  if (forbidden_bytemask != 0) {
+    return std::make_pair(nullptr, utf16_output);
+  }
+
+  return std::make_pair(buf, utf16_output);
+}
+
+template <endianness big_endian>
+std::pair<result, char16_t *>
+avx512_convert_utf32_to_utf16_with_errors(const char32_t *buf, size_t len,
+                                          char16_t *utf16_output) {
+  const char32_t *start = buf;
+  const char32_t *end = buf + len;
+  const __m512i v_00000000 = _mm512_setzero_si512();
+  const __m512i v_ffff0000 = _mm512_set1_epi32((int32_t)0xffff0000);
+  const __m512i v_f800 = _mm512_set1_epi32((uint32_t)0xf800);
+  const __m512i v_d800 = _mm512_set1_epi32((uint32_t)0xd800);
+  const __m512i v_10ffff = _mm512_set1_epi32(0x10FFFF);
+  const __m512i v_10000 = _mm512_set1_epi32(0x10000);
+  const __m512i v_3ff0000 = _mm512_set1_epi32(0x3FF0000);
+  const __m512i v_3ff = _mm512_set1_epi32(0x3FF);
+  const __m512i v_dc00d800 = _mm512_set1_epi32((int32_t)0xDC00D800);
+  int error_idx = 0;
+  error_code code = error_code::SUCCESS;
+  bool err = false;
+
+  while (end - buf >= std::ptrdiff_t(16)) {
+    __m512i in = _mm512_loadu_si512(buf);
+
+    // no bits set above 16th bit <=> can pack to UTF16 without surrogate pairs
+    const __mmask16 saturation_bitmask =
+        _mm512_cmpeq_epi32_mask(_mm512_and_si512(in, v_ffff0000), v_00000000);
+
+    if (saturation_bitmask == 0xffff) {
+      __mmask32 forbidden_bytemask =
+          _mm512_cmpeq_epi32_mask(_mm512_and_si512(in, v_f800), v_d800);
+
+      __m256i utf16_packed = _mm512_cvtepi32_epi16(in);
+      if (big_endian) {
+        const __m256i swap = _mm256_setr_epi8(
+            1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14, 1, 0, 3, 2, 5,
+            4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14);
+        utf16_packed = _mm256_shuffle_epi8(utf16_packed, swap);
+      }
+      if (simdutf_unlikely(forbidden_bytemask)) {
+        int idx = _tzcnt_u32(forbidden_bytemask);
+        _mm256_mask_storeu_epi16(
+            utf16_output, __mmask16(_blsmsk_u32(forbidden_bytemask) >> 1),
+            utf16_packed);
+        return std::make_pair(result(error_code::SURROGATE, buf - start + idx),
+                              utf16_output + idx);
+      }
+      _mm256_storeu_si256((__m256i *)utf16_output, utf16_packed);
+      utf16_output += 16;
+    } else {
+      __mmask32 output_mask = ~_pdep_u32(saturation_bitmask, 0xAAAAAAAA);
+      const __mmask16 surrogate_bitmask = __mmask16(~saturation_bitmask);
+      __mmask32 error_surrogate = _mm512_mask_cmpeq_epi32_mask(
+          saturation_bitmask, _mm512_and_si512(in, v_f800), v_d800);
+      __mmask32 error_too_large =
+          _mm512_mask_cmpgt_epu32_mask(surrogate_bitmask, in, v_10ffff);
+      if (simdutf_unlikely(error_surrogate || error_too_large)) {
+        // Need to find the lowest set bit between the two error masks
+        // Need to also write the partial chunk until the error index to output.
+        int large_idx = _tzcnt_u32(error_too_large);
+        int surrogate_idx = _tzcnt_u32(error_surrogate);
+        err = true;
+        if (large_idx < surrogate_idx) {
+          code = error_code::TOO_LARGE;
+          error_idx = large_idx;
+        } else {
+          code = error_code::SURROGATE;
+          error_idx = surrogate_idx;
+        }
+        output_mask &= ((1 << (2 * error_idx)) - 1);
+      }
+      __m512i v1, v2, v;
+      in = _mm512_mask_sub_epi32(in, surrogate_bitmask, in, v_10000);
+      v1 = _mm512_mask_slli_epi32(in, surrogate_bitmask, in, 16);
+      v1 = _mm512_mask_and_epi32(in, surrogate_bitmask, v1, v_3ff0000);
+      v2 = _mm512_mask_srli_epi32(in, surrogate_bitmask, in, 10);
+      v2 = _mm512_mask_and_epi32(in, surrogate_bitmask, v2, v_3ff);
+      v = _mm512_or_si512(v1, v2);
+      in = _mm512_mask_add_epi32(in, surrogate_bitmask, v, v_dc00d800);
+      if (big_endian) {
+        const __m512i swap_512 = _mm512_set_epi8(
+            14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1, 14, 15, 12,
+            13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1, 14, 15, 12, 13, 10, 11, 8,
+            9, 6, 7, 4, 5, 2, 3, 0, 1, 14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5,
+            2, 3, 0, 1);
+        in = _mm512_shuffle_epi8(in, swap_512);
+      }
+      // we deliberately avoid _mm512_mask_compressstoreu_epi16 for portability
+      // (AMD Zen4 has terrible performance with it, it is effectively broken)
+      __m512i compressed = _mm512_maskz_compress_epi16(output_mask, in);
+      auto written_out = _mm_popcnt_u32(output_mask);
+      _mm512_mask_storeu_epi16(utf16_output, _bzhi_u32(0xFFFFFFFF, written_out),
+                               compressed);
+      //_mm512_mask_compressstoreu_epi16(utf16_output, output_mask, in);
+      utf16_output += written_out;
+      if (simdutf_unlikely(err)) {
+        return std::make_pair(result(code, buf - start + error_idx),
+                              utf16_output);
+      }
+    }
+    buf += 16;
+  }
+
+  size_t remaining_len = size_t(end - buf);
+  if (remaining_len) {
+    __mmask16 input_mask = __mmask16((1 << remaining_len) - 1);
+    __m512i in = _mm512_maskz_loadu_epi32(input_mask, buf);
+    const __mmask16 saturation_bitmask =
+        _mm512_cmpeq_epi32_mask(_mm512_and_si512(in, v_ffff0000), v_00000000) &
+        input_mask;
+    if (saturation_bitmask == input_mask) {
+      __mmask32 forbidden_bytemask =
+          _mm512_cmpeq_epi32_mask(_mm512_and_si512(in, v_f800), v_d800);
+      __m256i utf16_packed = _mm512_cvtepi32_epi16(in);
+      if (big_endian) {
+        const __m256i swap = _mm256_setr_epi8(
+            1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14, 1, 0, 3, 2, 5,
+            4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14);
+        utf16_packed = _mm256_shuffle_epi8(utf16_packed, swap);
+      }
+      if (simdutf_unlikely(forbidden_bytemask)) {
+        int idx = _tzcnt_u32(forbidden_bytemask);
+        _mm256_mask_storeu_epi16(
+            utf16_output, __mmask16(_blsmsk_u32(forbidden_bytemask) >> 1),
+            utf16_packed);
+        return std::make_pair(result(error_code::SURROGATE, buf - start + idx),
+                              utf16_output + idx);
+      }
+      _mm256_mask_storeu_epi16(utf16_output, input_mask, utf16_packed);
+      utf16_output += remaining_len;
+    } else {
+      const __mmask32 output_max_mask = (1 << (remaining_len * 2)) - 1;
+      __mmask32 output_mask =
+          (~_pdep_u32(saturation_bitmask, 0xAAAAAAAA)) & output_max_mask;
+      const __mmask16 surrogate_bitmask =
+          __mmask16(~saturation_bitmask) & input_mask;
+      __mmask32 error_surrogate = _mm512_mask_cmpeq_epi32_mask(
+          saturation_bitmask, _mm512_and_si512(in, v_f800), v_d800);
+      __mmask32 error_too_large =
+          _mm512_mask_cmpgt_epu32_mask(surrogate_bitmask, in, v_10ffff);
+      if (simdutf_unlikely(error_surrogate || error_too_large)) {
+        int large_idx = _tzcnt_u32(error_too_large);
+        int surrogate_idx = _tzcnt_u32(error_surrogate);
+        err = true;
+        if (large_idx < surrogate_idx) {
+          code = error_code::TOO_LARGE;
+          error_idx = large_idx;
+        } else {
+          code = error_code::SURROGATE;
+          error_idx = surrogate_idx;
+        }
+        output_mask &= ((1 << (2 * error_idx)) - 1);
+      }
+      __m512i v1, v2, v;
+      in = _mm512_mask_sub_epi32(in, surrogate_bitmask, in, v_10000);
+      v1 = _mm512_mask_slli_epi32(in, surrogate_bitmask, in, 16);
+      v1 = _mm512_mask_and_epi32(in, surrogate_bitmask, v1, v_3ff0000);
+      v2 = _mm512_mask_srli_epi32(in, surrogate_bitmask, in, 10);
+      v2 = _mm512_mask_and_epi32(in, surrogate_bitmask, v2, v_3ff);
+      v = _mm512_or_si512(v1, v2);
+      in = _mm512_mask_add_epi32(in, surrogate_bitmask, v, v_dc00d800);
+      if (big_endian) {
+        const __m512i swap_512 = _mm512_set_epi8(
+            14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1, 14, 15, 12,
+            13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1, 14, 15, 12, 13, 10, 11, 8,
+            9, 6, 7, 4, 5, 2, 3, 0, 1, 14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5,
+            2, 3, 0, 1);
+        in = _mm512_shuffle_epi8(in, swap_512);
+      }
+      // we deliberately avoid _mm512_mask_compressstoreu_epi16 for portability
+      // (AMD Zen4 has terrible performance with it, it is effectively broken)
+      __m512i compressed = _mm512_maskz_compress_epi16(output_mask, in);
+      auto written_out = _mm_popcnt_u32(output_mask);
+      _mm512_mask_storeu_epi16(utf16_output, _bzhi_u32(0xFFFFFFFF, written_out),
+                               compressed);
+      //_mm512_mask_compressstoreu_epi16(utf16_output, output_mask, in);
+      utf16_output += written_out;
+      if (simdutf_unlikely(err)) {
+        return std::make_pair(result(code, buf - start + error_idx),
+                              utf16_output);
+      }
+    }
+    buf += remaining_len;
+  }
+
+  return std::make_pair(result(error_code::SUCCESS, buf - start), utf16_output);
+}