react-native-quick-crypto 1.0.18 → 1.1.0

package/deps/simdutf/src/arm64/arm_convert_utf16_to_utf8.cpp

@@ -0,0 +1,780 @@
+/*
+    The vectorized algorithm works on single SSE register i.e., it
+    loads eight 16-bit code units.
+
+    We consider three cases:
+    1. an input register contains no surrogates and each value
+       is in range 0x0000 .. 0x07ff.
+    2. an input register contains no surrogates and values are
+       is in range 0x0000 .. 0xffff.
+    3. an input register contains surrogates --- i.e. codepoints
+       can have 16 or 32 bits.
+
+    Ad 1.
+
+    When values are less than 0x0800, it means that a 16-bit code unit
+    can be converted into: 1) single UTF8 byte (when it is an ASCII
+    char) or 2) two UTF8 bytes.
+
+    For this case we do only some shuffle to obtain these 2-byte
+    codes and finally compress the whole SSE register with a single
+    shuffle.
+
+    We need 256-entry lookup table to get a compression pattern
+    and the number of output bytes in the compressed vector register.
+    Each entry occupies 17 bytes.
+
+    Ad 2.
+
+    When values fit in 16-bit code units, but are above 0x07ff, then
+    a single word may produce one, two or three UTF8 bytes.
+
+    We prepare data for all these three cases in two registers.
+    The first register contains lower two UTF8 bytes (used in all
+    cases), while the second one contains just the third byte for
+    the three-UTF8-bytes case.
+
+    Finally these two registers are interleaved forming eight-element
+    array of 32-bit values. The array spans two SSE registers.
+    The bytes from the registers are compressed using two shuffles.
+
+    We need 256-entry lookup table to get a compression pattern
+    and the number of output bytes in the compressed vector register.
+    Each entry occupies 17 bytes.
+
+
+    To summarize:
+    - We need two 256-entry tables that have 8704 bytes in total.
+*/
+/*
+  Returns a pair: the first unprocessed byte from buf and utf8_output
+  A scalar routing should carry on the conversion of the tail.
+*/
+template <endianness big_endian>
+std::pair<const char16_t *, char *>
+arm_convert_utf16_to_utf8(const char16_t *buf, size_t len, char *utf8_out) {
+  uint8_t *utf8_output = reinterpret_cast<uint8_t *>(utf8_out);
+  const char16_t *end = buf + len;
+
+  const uint16x8_t v_f800 = vmovq_n_u16((uint16_t)0xf800);
+  const uint16x8_t v_d800 = vmovq_n_u16((uint16_t)0xd800);
+  const uint16x8_t v_c080 = vmovq_n_u16((uint16_t)0xc080);
+  const size_t safety_margin =
+      12; // to avoid overruns, see issue
+          // https://github.com/simdutf/simdutf/issues/92
+  while (end - buf >= std::ptrdiff_t(16 + safety_margin)) {
+    uint16x8_t in = vld1q_u16(reinterpret_cast<const uint16_t *>(buf));
+    if simdutf_constexpr (!match_system(big_endian)) {
+      in = vreinterpretq_u16_u8(vrev16q_u8(vreinterpretq_u8_u16(in)));
+    }
+    if (vmaxvq_u16(in) <= 0x7F) { // ASCII fast path!!!!
+      // It is common enough that we have sequences of 16 consecutive ASCII
+      // characters.
+      uint16x8_t nextin =
+          vld1q_u16(reinterpret_cast<const uint16_t *>(buf) + 8);
+      if simdutf_constexpr (!match_system(big_endian)) {
+        nextin = vreinterpretq_u16_u8(vrev16q_u8(vreinterpretq_u8_u16(nextin)));
+      }
+      if (vmaxvq_u16(nextin) > 0x7F) {
+        // 1. pack the bytes
+        // obviously suboptimal.
+        uint8x8_t utf8_packed = vmovn_u16(in);
+        // 2. store (8 bytes)
+        vst1_u8(utf8_output, utf8_packed);
+        // 3. adjust pointers
+        buf += 8;
+        utf8_output += 8;
+        in = nextin;
+      } else {
+        // 1. pack the bytes
+        // obviously suboptimal.
+        uint8x16_t utf8_packed = vmovn_high_u16(vmovn_u16(in), nextin);
+        // 2. store (16 bytes)
+        vst1q_u8(utf8_output, utf8_packed);
+        // 3. adjust pointers
+        buf += 16;
+        utf8_output += 16;
+        continue; // we are done for this round!
+      }
+    }
+
+    if (vmaxvq_u16(in) <= 0x7FF) {
+      // 1. prepare 2-byte values
+      // input 16-bit word : [0000|0aaa|aabb|bbbb] x 8
+      // expected output   : [110a|aaaa|10bb|bbbb] x 8
+      const uint16x8_t v_1f00 = vmovq_n_u16((int16_t)0x1f00);
+      const uint16x8_t v_003f = vmovq_n_u16((int16_t)0x003f);
+
+      // t0 = [000a|aaaa|bbbb|bb00]
+      const uint16x8_t t0 = vshlq_n_u16(in, 2);
+      // t1 = [000a|aaaa|0000|0000]
+      const uint16x8_t t1 = vandq_u16(t0, v_1f00);
+      // t2 = [0000|0000|00bb|bbbb]
+      const uint16x8_t t2 = vandq_u16(in, v_003f);
+      // t3 = [000a|aaaa|00bb|bbbb]
+      const uint16x8_t t3 = vorrq_u16(t1, t2);
+      // t4 = [110a|aaaa|10bb|bbbb]
+      const uint16x8_t t4 = vorrq_u16(t3, v_c080);
+      // 2. merge ASCII and 2-byte codewords
+      const uint16x8_t v_007f = vmovq_n_u16((uint16_t)0x007F);
+      const uint16x8_t one_byte_bytemask = vcleq_u16(in, v_007f);
+      const uint8x16_t utf8_unpacked =
+          vreinterpretq_u8_u16(vbslq_u16(one_byte_bytemask, in, t4));
+      // 3. prepare bitmask for 8-bit lookup
+#ifdef SIMDUTF_REGULAR_VISUAL_STUDIO
+      const uint16x8_t mask = simdutf_make_uint16x8_t(
+          0x0001, 0x0004, 0x0010, 0x0040, 0x0002, 0x0008, 0x0020, 0x0080);
+#else
+      const uint16x8_t mask = {0x0001, 0x0004, 0x0010, 0x0040,
+                               0x0002, 0x0008, 0x0020, 0x0080};
+#endif
+      uint16_t m2 = vaddvq_u16(vandq_u16(one_byte_bytemask, mask));
+      // 4. pack the bytes
+      const uint8_t *row =
+          &simdutf::tables::utf16_to_utf8::pack_1_2_utf8_bytes[m2][0];
+      const uint8x16_t shuffle = vld1q_u8(row + 1);
+      const uint8x16_t utf8_packed = vqtbl1q_u8(utf8_unpacked, shuffle);
+
+      // 5. store bytes
+      vst1q_u8(utf8_output, utf8_packed);
+
+      // 6. adjust pointers
+      buf += 8;
+      utf8_output += row[0];
+      continue;
+    }
+    const uint16x8_t surrogates_bytemask =
+        vceqq_u16(vandq_u16(in, v_f800), v_d800);
+    // It might seem like checking for surrogates_bitmask == 0xc000 could help.
+    // However, it is likely an uncommon occurrence.
+    if (vmaxvq_u16(surrogates_bytemask) == 0) {
+      // case: code units from register produce either 1, 2 or 3 UTF-8 bytes
+#ifdef SIMDUTF_REGULAR_VISUAL_STUDIO
+      const uint16x8_t dup_even = simdutf_make_uint16x8_t(
+          0x0000, 0x0202, 0x0404, 0x0606, 0x0808, 0x0a0a, 0x0c0c, 0x0e0e);
+#else
+      const uint16x8_t dup_even = {0x0000, 0x0202, 0x0404, 0x0606,
+                                   0x0808, 0x0a0a, 0x0c0c, 0x0e0e};
+#endif
+      /* In this branch we handle three cases:
+         1. [0000|0000|0ccc|cccc] => [0ccc|cccc]                           -
+        single UFT-8 byte
+         2. [0000|0bbb|bbcc|cccc] => [110b|bbbb], [10cc|cccc]              - two
+        UTF-8 bytes
+         3. [aaaa|bbbb|bbcc|cccc] => [1110|aaaa], [10bb|bbbb], [10cc|cccc] -
+        three UTF-8 bytes
+
+        We expand the input word (16-bit) into two code units (32-bit), thus
+        we have room for four bytes. However, we need five distinct bit
+        layouts. Note that the last byte in cases #2 and #3 is the same.
+
+        We precompute byte 1 for case #1 and the common byte for cases #2 & #3
+        in register t2.
+
+        We precompute byte 1 for case #3 and -- **conditionally** -- precompute
+        either byte 1 for case #2 or byte 2 for case #3. Note that they
+        differ by exactly one bit.
+
+        Finally from these two code units we build proper UTF-8 sequence, taking
+        into account the case (i.e, the number of bytes to write).
+      */
+      /**
+       * Given [aaaa|bbbb|bbcc|cccc] our goal is to produce:
+       * t2 => [0ccc|cccc] [10cc|cccc]
+       * s4 => [1110|aaaa] ([110b|bbbb] OR [10bb|bbbb])
+       */
+#define simdutf_vec(x) vmovq_n_u16(static_cast<uint16_t>(x))
+      // [aaaa|bbbb|bbcc|cccc] => [bbcc|cccc|bbcc|cccc]
+      const uint16x8_t t0 = vreinterpretq_u16_u8(
+          vqtbl1q_u8(vreinterpretq_u8_u16(in), vreinterpretq_u8_u16(dup_even)));
+      // [bbcc|cccc|bbcc|cccc] => [00cc|cccc|0bcc|cccc]
+      const uint16x8_t t1 = vandq_u16(t0, simdutf_vec(0b0011111101111111));
+      // [00cc|cccc|0bcc|cccc] => [10cc|cccc|0bcc|cccc]
+      const uint16x8_t t2 = vorrq_u16(t1, simdutf_vec(0b1000000000000000));
+
+      // s0: [aaaa|bbbb|bbcc|cccc] => [0000|0000|0000|aaaa]
+      const uint16x8_t s0 = vshrq_n_u16(in, 12);
+      // s1: [aaaa|bbbb|bbcc|cccc] => [0000|bbbb|bb00|0000]
+      const uint16x8_t s1 = vandq_u16(in, simdutf_vec(0b0000111111000000));
+      // [0000|bbbb|bb00|0000] => [00bb|bbbb|0000|0000]
+      const uint16x8_t s1s = vshlq_n_u16(s1, 2);
+      // [00bb|bbbb|0000|aaaa]
+      const uint16x8_t s2 = vorrq_u16(s0, s1s);
+      // s3: [00bb|bbbb|0000|aaaa] => [11bb|bbbb|1110|aaaa]
+      const uint16x8_t s3 = vorrq_u16(s2, simdutf_vec(0b1100000011100000));
+      const uint16x8_t v_07ff = vmovq_n_u16((uint16_t)0x07FF);
+      const uint16x8_t one_or_two_bytes_bytemask = vcleq_u16(in, v_07ff);
+      const uint16x8_t m0 =
+          vbicq_u16(simdutf_vec(0b0100000000000000), one_or_two_bytes_bytemask);
+      const uint16x8_t s4 = veorq_u16(s3, m0);
+#undef simdutf_vec
+
+      // 4. expand code units 16-bit => 32-bit
+      const uint8x16_t out0 = vreinterpretq_u8_u16(vzip1q_u16(t2, s4));
+      const uint8x16_t out1 = vreinterpretq_u8_u16(vzip2q_u16(t2, s4));
+
+      // 5. compress 32-bit code units into 1, 2 or 3 bytes -- 2 x shuffle
+      const uint16x8_t v_007f = vmovq_n_u16((uint16_t)0x007F);
+      const uint16x8_t one_byte_bytemask = vcleq_u16(in, v_007f);
+#ifdef SIMDUTF_REGULAR_VISUAL_STUDIO
+      const uint16x8_t onemask = simdutf_make_uint16x8_t(
+          0x0001, 0x0004, 0x0010, 0x0040, 0x0100, 0x0400, 0x1000, 0x4000);
+      const uint16x8_t twomask = simdutf_make_uint16x8_t(
+          0x0002, 0x0008, 0x0020, 0x0080, 0x0200, 0x0800, 0x2000, 0x8000);
+#else
+      const uint16x8_t onemask = {0x0001, 0x0004, 0x0010, 0x0040,
+                                  0x0100, 0x0400, 0x1000, 0x4000};
+      const uint16x8_t twomask = {0x0002, 0x0008, 0x0020, 0x0080,
+                                  0x0200, 0x0800, 0x2000, 0x8000};
+#endif
+      const uint16x8_t combined =
+          vorrq_u16(vandq_u16(one_byte_bytemask, onemask),
+                    vandq_u16(one_or_two_bytes_bytemask, twomask));
+      const uint16_t mask = vaddvq_u16(combined);
+      // The following fast path may or may not be beneficial.
+      /*if(mask == 0) {
+        // We only have three-byte code units. Use fast path.
+        const uint8x16_t shuffle = {2,3,1,6,7,5,10,11,9,14,15,13,0,0,0,0};
+        const uint8x16_t utf8_0 = vqtbl1q_u8(out0, shuffle);
+        const uint8x16_t utf8_1 = vqtbl1q_u8(out1, shuffle);
+        vst1q_u8(utf8_output, utf8_0);
+        utf8_output += 12;
+        vst1q_u8(utf8_output, utf8_1);
+        utf8_output += 12;
+        buf += 8;
+        continue;
+      }*/
+      const uint8_t mask0 = uint8_t(mask);
+
+      const uint8_t *row0 =
+          &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask0][0];
+      const uint8x16_t shuffle0 = vld1q_u8(row0 + 1);
+      const uint8x16_t utf8_0 = vqtbl1q_u8(out0, shuffle0);
+
+      const uint8_t mask1 = static_cast<uint8_t>(mask >> 8);
+      const uint8_t *row1 =
+          &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask1][0];
+      const uint8x16_t shuffle1 = vld1q_u8(row1 + 1);
+      const uint8x16_t utf8_1 = vqtbl1q_u8(out1, shuffle1);
+
+      vst1q_u8(utf8_output, utf8_0);
+      utf8_output += row0[0];
+      vst1q_u8(utf8_output, utf8_1);
+      utf8_output += row1[0];
+
+      buf += 8;
+      // surrogate pair(s) in a register
+    } else {
+      // Let us do a scalar fallback.
+      // It may seem wasteful to use scalar code, but being efficient with SIMD
+      // in the presence of surrogate pairs may require non-trivial tables.
+      size_t forward = 15;
+      size_t k = 0;
+      if (size_t(end - buf) < forward + 1) {
+        forward = size_t(end - buf - 1);
+      }
+      for (; k < forward; k++) {
+        uint16_t word = scalar::utf16::swap_if_needed<big_endian>(buf[k]);
+        if ((word & 0xFF80) == 0) {
+          *utf8_output++ = char(word);
+        } else if ((word & 0xF800) == 0) {
+          *utf8_output++ = char((word >> 6) | 0b11000000);
+          *utf8_output++ = char((word & 0b111111) | 0b10000000);
+        } else if ((word & 0xF800) != 0xD800) {
+          *utf8_output++ = char((word >> 12) | 0b11100000);
+          *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000);
+          *utf8_output++ = char((word & 0b111111) | 0b10000000);
+        } else {
+          // must be a surrogate pair
+          uint16_t diff = uint16_t(word - 0xD800);
+          uint16_t next_word =
+              scalar::utf16::swap_if_needed<big_endian>(buf[k + 1]);
+          k++;
+          uint16_t diff2 = uint16_t(next_word - 0xDC00);
+          if ((diff | diff2) > 0x3FF) {
+            return std::make_pair(nullptr,
+                                  reinterpret_cast<char *>(utf8_output));
+          }
+          uint32_t value = (diff << 10) + diff2 + 0x10000;
+          *utf8_output++ = char((value >> 18) | 0b11110000);
+          *utf8_output++ = char(((value >> 12) & 0b111111) | 0b10000000);
+          *utf8_output++ = char(((value >> 6) & 0b111111) | 0b10000000);
+          *utf8_output++ = char((value & 0b111111) | 0b10000000);
+        }
+      }
+      buf += k;
+    }
+  } // while
+
+  return std::make_pair(buf, reinterpret_cast<char *>(utf8_output));
+}
+
+/*
+  Returns a pair: a result struct and utf8_output.
+  If there is an error, the count field of the result is the position of the
+  error. Otherwise, it is the position of the first unprocessed byte in buf
+  (even if finished). A scalar routing should carry on the conversion of the
+  tail if needed.
+*/
+template <endianness big_endian>
+std::pair<result, char *>
+arm_convert_utf16_to_utf8_with_errors(const char16_t *buf, size_t len,
+                                      char *utf8_out) {
+  uint8_t *utf8_output = reinterpret_cast<uint8_t *>(utf8_out);
+  const char16_t *start = buf;
+  const char16_t *end = buf + len;
+
+  const uint16x8_t v_f800 = vmovq_n_u16((uint16_t)0xf800);
+  const uint16x8_t v_d800 = vmovq_n_u16((uint16_t)0xd800);
+  const uint16x8_t v_c080 = vmovq_n_u16((uint16_t)0xc080);
+  const size_t safety_margin =
+      12; // to avoid overruns, see issue
+          // https://github.com/simdutf/simdutf/issues/92
+
+  while (end - buf >= std::ptrdiff_t(16 + safety_margin)) {
+    uint16x8_t in = vld1q_u16(reinterpret_cast<const uint16_t *>(buf));
+    if simdutf_constexpr (!match_system(big_endian)) {
+      in = vreinterpretq_u16_u8(vrev16q_u8(vreinterpretq_u8_u16(in)));
+    }
+    if (vmaxvq_u16(in) <= 0x7F) { // ASCII fast path!!!!
+      // It is common enough that we have sequences of 16 consecutive ASCII
+      // characters.
+      uint16x8_t nextin =
+          vld1q_u16(reinterpret_cast<const uint16_t *>(buf) + 8);
+      if simdutf_constexpr (!match_system(big_endian)) {
+        nextin = vreinterpretq_u16_u8(vrev16q_u8(vreinterpretq_u8_u16(nextin)));
+      }
+      if (vmaxvq_u16(nextin) > 0x7F) {
+        // 1. pack the bytes
+        // obviously suboptimal.
+        uint8x8_t utf8_packed = vmovn_u16(in);
+        // 2. store (8 bytes)
+        vst1_u8(utf8_output, utf8_packed);
+        // 3. adjust pointers
+        buf += 8;
+        utf8_output += 8;
+        in = nextin;
+      } else {
+        // 1. pack the bytes
+        // obviously suboptimal.
+        uint8x16_t utf8_packed = vmovn_high_u16(vmovn_u16(in), nextin);
+        // 2. store (16 bytes)
+        vst1q_u8(utf8_output, utf8_packed);
+        // 3. adjust pointers
+        buf += 16;
+        utf8_output += 16;
+        continue; // we are done for this round!
+      }
+    }
+
+    if (vmaxvq_u16(in) <= 0x7FF) {
+      // 1. prepare 2-byte values
+      // input 16-bit word : [0000|0aaa|aabb|bbbb] x 8
+      // expected output   : [110a|aaaa|10bb|bbbb] x 8
+      const uint16x8_t v_1f00 = vmovq_n_u16((int16_t)0x1f00);
+      const uint16x8_t v_003f = vmovq_n_u16((int16_t)0x003f);
+
+      // t0 = [000a|aaaa|bbbb|bb00]
+      const uint16x8_t t0 = vshlq_n_u16(in, 2);
+      // t1 = [000a|aaaa|0000|0000]
+      const uint16x8_t t1 = vandq_u16(t0, v_1f00);
+      // t2 = [0000|0000|00bb|bbbb]
+      const uint16x8_t t2 = vandq_u16(in, v_003f);
+      // t3 = [000a|aaaa|00bb|bbbb]
+      const uint16x8_t t3 = vorrq_u16(t1, t2);
+      // t4 = [110a|aaaa|10bb|bbbb]
+      const uint16x8_t t4 = vorrq_u16(t3, v_c080);
+      // 2. merge ASCII and 2-byte codewords
+      const uint16x8_t v_007f = vmovq_n_u16((uint16_t)0x007F);
+      const uint16x8_t one_byte_bytemask = vcleq_u16(in, v_007f);
+      const uint8x16_t utf8_unpacked =
+          vreinterpretq_u8_u16(vbslq_u16(one_byte_bytemask, in, t4));
+      // 3. prepare bitmask for 8-bit lookup
+#ifdef SIMDUTF_REGULAR_VISUAL_STUDIO
+      const uint16x8_t mask = simdutf_make_uint16x8_t(
+          0x0001, 0x0004, 0x0010, 0x0040, 0x0002, 0x0008, 0x0020, 0x0080);
+#else
+      const uint16x8_t mask = {0x0001, 0x0004, 0x0010, 0x0040,
+                               0x0002, 0x0008, 0x0020, 0x0080};
+#endif
+      uint16_t m2 = vaddvq_u16(vandq_u16(one_byte_bytemask, mask));
+      // 4. pack the bytes
+      const uint8_t *row =
+          &simdutf::tables::utf16_to_utf8::pack_1_2_utf8_bytes[m2][0];
+      const uint8x16_t shuffle = vld1q_u8(row + 1);
+      const uint8x16_t utf8_packed = vqtbl1q_u8(utf8_unpacked, shuffle);
+
+      // 5. store bytes
+      vst1q_u8(utf8_output, utf8_packed);
+
+      // 6. adjust pointers
+      buf += 8;
+      utf8_output += row[0];
+      continue;
+    }
+    const uint16x8_t surrogates_bytemask =
+        vceqq_u16(vandq_u16(in, v_f800), v_d800);
+    // It might seem like checking for surrogates_bitmask == 0xc000 could help.
+    // However, it is likely an uncommon occurrence.
+    if (vmaxvq_u16(surrogates_bytemask) == 0) {
+      // case: code units from register produce either 1, 2 or 3 UTF-8 bytes
+#ifdef SIMDUTF_REGULAR_VISUAL_STUDIO
+      const uint16x8_t dup_even = simdutf_make_uint16x8_t(
+          0x0000, 0x0202, 0x0404, 0x0606, 0x0808, 0x0a0a, 0x0c0c, 0x0e0e);
+#else
+      const uint16x8_t dup_even = {0x0000, 0x0202, 0x0404, 0x0606,
+                                   0x0808, 0x0a0a, 0x0c0c, 0x0e0e};
+#endif
+      /* In this branch we handle three cases:
+         1. [0000|0000|0ccc|cccc] => [0ccc|cccc]                           -
+        single UFT-8 byte
+         2. [0000|0bbb|bbcc|cccc] => [110b|bbbb], [10cc|cccc]              - two
+        UTF-8 bytes
+         3. [aaaa|bbbb|bbcc|cccc] => [1110|aaaa], [10bb|bbbb], [10cc|cccc] -
+        three UTF-8 bytes
+
+        We expand the input word (16-bit) into two code units (32-bit), thus
+        we have room for four bytes. However, we need five distinct bit
+        layouts. Note that the last byte in cases #2 and #3 is the same.
+
+        We precompute byte 1 for case #1 and the common byte for cases #2 & #3
+        in register t2.
+
+        We precompute byte 1 for case #3 and -- **conditionally** -- precompute
+        either byte 1 for case #2 or byte 2 for case #3. Note that they
+        differ by exactly one bit.
+
+        Finally from these two code units we build proper UTF-8 sequence, taking
+        into account the case (i.e, the number of bytes to write).
+      */
+      /**
+       * Given [aaaa|bbbb|bbcc|cccc] our goal is to produce:
+       * t2 => [0ccc|cccc] [10cc|cccc]
+       * s4 => [1110|aaaa] ([110b|bbbb] OR [10bb|bbbb])
+       */
+#define simdutf_vec(x) vmovq_n_u16(static_cast<uint16_t>(x))
+      // [aaaa|bbbb|bbcc|cccc] => [bbcc|cccc|bbcc|cccc]
+      const uint16x8_t t0 = vreinterpretq_u16_u8(
+          vqtbl1q_u8(vreinterpretq_u8_u16(in), vreinterpretq_u8_u16(dup_even)));
+      // [bbcc|cccc|bbcc|cccc] => [00cc|cccc|0bcc|cccc]
+      const uint16x8_t t1 = vandq_u16(t0, simdutf_vec(0b0011111101111111));
+      // [00cc|cccc|0bcc|cccc] => [10cc|cccc|0bcc|cccc]
+      const uint16x8_t t2 = vorrq_u16(t1, simdutf_vec(0b1000000000000000));
+
+      // s0: [aaaa|bbbb|bbcc|cccc] => [0000|0000|0000|aaaa]
+      const uint16x8_t s0 = vshrq_n_u16(in, 12);
+      // s1: [aaaa|bbbb|bbcc|cccc] => [0000|bbbb|bb00|0000]
+      const uint16x8_t s1 = vandq_u16(in, simdutf_vec(0b0000111111000000));
+      // [0000|bbbb|bb00|0000] => [00bb|bbbb|0000|0000]
+      const uint16x8_t s1s = vshlq_n_u16(s1, 2);
+      // [00bb|bbbb|0000|aaaa]
+      const uint16x8_t s2 = vorrq_u16(s0, s1s);
+      // s3: [00bb|bbbb|0000|aaaa] => [11bb|bbbb|1110|aaaa]
+      const uint16x8_t s3 = vorrq_u16(s2, simdutf_vec(0b1100000011100000));
+      const uint16x8_t v_07ff = vmovq_n_u16((uint16_t)0x07FF);
+      const uint16x8_t one_or_two_bytes_bytemask = vcleq_u16(in, v_07ff);
+      const uint16x8_t m0 =
+          vbicq_u16(simdutf_vec(0b0100000000000000), one_or_two_bytes_bytemask);
+      const uint16x8_t s4 = veorq_u16(s3, m0);
+#undef simdutf_vec
+
+      // 4. expand code units 16-bit => 32-bit
+      const uint8x16_t out0 = vreinterpretq_u8_u16(vzip1q_u16(t2, s4));
+      const uint8x16_t out1 = vreinterpretq_u8_u16(vzip2q_u16(t2, s4));
+
+      // 5. compress 32-bit code units into 1, 2 or 3 bytes -- 2 x shuffle
+      const uint16x8_t v_007f = vmovq_n_u16((uint16_t)0x007F);
+      const uint16x8_t one_byte_bytemask = vcleq_u16(in, v_007f);
+#ifdef SIMDUTF_REGULAR_VISUAL_STUDIO
+      const uint16x8_t onemask = simdutf_make_uint16x8_t(
+          0x0001, 0x0004, 0x0010, 0x0040, 0x0100, 0x0400, 0x1000, 0x4000);
+      const uint16x8_t twomask = simdutf_make_uint16x8_t(
+          0x0002, 0x0008, 0x0020, 0x0080, 0x0200, 0x0800, 0x2000, 0x8000);
+#else
+      const uint16x8_t onemask = {0x0001, 0x0004, 0x0010, 0x0040,
+                                  0x0100, 0x0400, 0x1000, 0x4000};
+      const uint16x8_t twomask = {0x0002, 0x0008, 0x0020, 0x0080,
+                                  0x0200, 0x0800, 0x2000, 0x8000};
+#endif
+      const uint16x8_t combined =
+          vorrq_u16(vandq_u16(one_byte_bytemask, onemask),
+                    vandq_u16(one_or_two_bytes_bytemask, twomask));
+      const uint16_t mask = vaddvq_u16(combined);
+      // The following fast path may or may not be beneficial.
+      /*if(mask == 0) {
+        // We only have three-byte code units. Use fast path.
+        const uint8x16_t shuffle = {2,3,1,6,7,5,10,11,9,14,15,13,0,0,0,0};
+        const uint8x16_t utf8_0 = vqtbl1q_u8(out0, shuffle);
+        const uint8x16_t utf8_1 = vqtbl1q_u8(out1, shuffle);
+        vst1q_u8(utf8_output, utf8_0);
+        utf8_output += 12;
+        vst1q_u8(utf8_output, utf8_1);
+        utf8_output += 12;
+        buf += 8;
+        continue;
+      }*/
+      const uint8_t mask0 = uint8_t(mask);
+
+      const uint8_t *row0 =
+          &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask0][0];
+      const uint8x16_t shuffle0 = vld1q_u8(row0 + 1);
+      const uint8x16_t utf8_0 = vqtbl1q_u8(out0, shuffle0);
+
+      const uint8_t mask1 = static_cast<uint8_t>(mask >> 8);
+      const uint8_t *row1 =
+          &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask1][0];
+      const uint8x16_t shuffle1 = vld1q_u8(row1 + 1);
+      const uint8x16_t utf8_1 = vqtbl1q_u8(out1, shuffle1);
+
+      vst1q_u8(utf8_output, utf8_0);
+      utf8_output += row0[0];
+      vst1q_u8(utf8_output, utf8_1);
+      utf8_output += row1[0];
+
+      buf += 8;
+      // surrogate pair(s) in a register
+    } else {
+      // Let us do a scalar fallback.
+      // It may seem wasteful to use scalar code, but being efficient with SIMD
+      // in the presence of surrogate pairs may require non-trivial tables.
+      size_t forward = 15;
+      size_t k = 0;
+      if (size_t(end - buf) < forward + 1) {
+        forward = size_t(end - buf - 1);
+      }
+      for (; k < forward; k++) {
+        uint16_t word = scalar::utf16::swap_if_needed<big_endian>(buf[k]);
+        if ((word & 0xFF80) == 0) {
+          *utf8_output++ = char(word);
+        } else if ((word & 0xF800) == 0) {
+          *utf8_output++ = char((word >> 6) | 0b11000000);
+          *utf8_output++ = char((word & 0b111111) | 0b10000000);
+        } else if ((word & 0xF800) != 0xD800) {
+          *utf8_output++ = char((word >> 12) | 0b11100000);
+          *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000);
+          *utf8_output++ = char((word & 0b111111) | 0b10000000);
+        } else {
+          // must be a surrogate pair
+          uint16_t diff = uint16_t(word - 0xD800);
+          uint16_t next_word =
+              scalar::utf16::swap_if_needed<big_endian>(buf[k + 1]);
+          k++;
+          uint16_t diff2 = uint16_t(next_word - 0xDC00);
+          if ((diff | diff2) > 0x3FF) {
+            return std::make_pair(
+                result(error_code::SURROGATE, buf - start + k - 1),
+                reinterpret_cast<char *>(utf8_output));
+          }
+          uint32_t value = (diff << 10) + diff2 + 0x10000;
+          *utf8_output++ = char((value >> 18) | 0b11110000);
+          *utf8_output++ = char(((value >> 12) & 0b111111) | 0b10000000);
+          *utf8_output++ = char(((value >> 6) & 0b111111) | 0b10000000);
+          *utf8_output++ = char((value & 0b111111) | 0b10000000);
+        }
+      }
+      buf += k;
+    }
+  } // while
+
+  return std::make_pair(result(error_code::SUCCESS, buf - start),
+                        reinterpret_cast<char *>(utf8_output));
+}
+
+template <endianness big_endian>
+simdutf_really_inline size_t
+arm64_utf8_length_from_utf16_bytemask(const char16_t *in, size_t size) {
+  constexpr size_t N =
+      16; // we process 16 char16_t at a time, this is NEON specific
+
+  if (N + 1 > size) {
+    return scalar::utf16::utf8_length_from_utf16<big_endian>(in, size);
+  } // special case for short inputs
+  size_t count = 0;
+  const auto one = vmovq_n_u8(1);
+
+  // The general strategy is as follows:
+  // 1. each code unit yields at least one byte, we can account for that by
+  // adding the size of the input to the count.
+  // 2. ASCII bytes then count for zero.
+  // 3. Values that yield 2 or 3 bytes in UTF-8 add 1 or 2 to the count.
+  // 4. Surrogate pairs are handled by adding 1 for each surrogate code unit
+  //    for a total of 4 bytes for the pair.
+  size_t pos = 0;
+  // We will go through the input at least once.
+  for (; size - pos >= N; pos += N) {
+    auto base_input = vld2q_u8(reinterpret_cast<const uint8_t *>(in + pos));
+    //
+    size_t idx = 1; // we use the second lane of the deinterleaved load
+    if simdutf_constexpr (!match_system(big_endian)) {
+      idx = 0;
+    }
+    size_t idx_lsb = idx ^ 1;
+    auto c0 =
+        vminq_u8(vorrq_u8(vandq_u8(base_input.val[idx_lsb], vdupq_n_u8(0x80)),
+                          base_input.val[idx]),
+                 one);
+    auto c1 = vminq_u8(vandq_u8(base_input.val[idx], vdupq_n_u8(0xf8)), one);
+    auto is_surrogate = vcleq_u8(
+        vsubq_u8(base_input.val[idx], vdupq_n_u8(0xd8)), vdupq_n_u8(7));
+
+    auto v_count = vaddq_u8(c1, c0);
+    v_count = vaddq_u8(v_count, is_surrogate);
+    count += vaddlvq_u8(v_count); // sum the counts in the vector
+    /////////
+    // The vaddlvq_u8 instruction could be slow on some hardware. We could
+    // consider various alternatives if that is an issue such as accumulating
+    // into a vector of uint16_t or uint8_t and summing only at the end or
+    // periodically. However, on fast chipsets, like Apple Silicon, it is
+    // likely fast enough, or even faster than alternatives.
+    /////////
+  }
+  count += pos;
+  // If we end with a high surrogate, it might be unpaired or not, we
+  // don't know. It counts as a pair suggarate for now.
+
+  if (scalar::utf16::is_high_surrogate<big_endian>(in[pos - 1])) {
+    if (pos == size) {
+      count += 2;
+    } else if (scalar::utf16::is_low_surrogate<big_endian>(in[pos])) {
+      pos += 1;
+      count += 2;
+    }
+  }
+  return count + scalar::utf16::utf8_length_from_utf16<big_endian>(in + pos,
+                                                                   size - pos);
+}
+
+template <endianness big_endian>
+simdutf_really_inline result
+arm64_utf8_length_from_utf16_with_replacement(const char16_t *in, size_t size) {
+  constexpr size_t N =
+      16; // we process 16 char16_t at a time, this is NEON specific
+
+  if (N + 1 > size) {
+    return scalar::utf16::utf8_length_from_utf16_with_replacement<big_endian>(
+        in, size);
+  } // special case for short input
+  size_t count = 0;
+  bool any_surrogates = false;
+  const auto one = vmovq_n_u8(1);
+
+  // The general strategy is as follows:
+  // 1. each code unit yields at least one byte, we can account for that by
+  // adding the size of the input to the count.
+  // 2. ASCII bytes then count for zero.
+  // 3. Values that yield 2 or 3 bytes in UTF-8 add 1 or 2 to the count.
+  // 4. Surrogate pairs are handled by adding 1 for each surrogate code unit
+  //    for a total of 4 bytes for the pair.
+  // 5. Unpaired surrogate elements have value 0xfffd in UTF-8, which is 3
+  // bytes,
+  //    so we need to add 2 more bytes for each unpaired surrogate. In effect,
+  //    an unpaired surrogate should count for 1 (+1 for the )
+  //
+  // Our strategy is to proceed like the arm64_utf8_length_from_utf16_bytemask
+  // function, but, at the same time, to record the number of unpaired
+  // surrogates. and then adjust the count accordingly.
+
+  // If we start with a low surrogate, it is unpaired and the SIMD code won't
+  // detect it, so we handle that here.
+  size_t number_of_unpaired_surrogates = 0;
+  if (scalar::utf16::is_low_surrogate<big_endian>(in[0])) {
+    number_of_unpaired_surrogates += 1;
+    any_surrogates = true;
+  }
+  size_t pos = 0;
+  // We will go through the input at least once.
+  for (; size - pos >= N + 1; pos += N) {
+    auto base_input = vld2q_u8(reinterpret_cast<const uint8_t *>(in + pos));
+    size_t idx = 1; // we use the second lane of the deinterleaved load
+    if simdutf_constexpr (!match_system(big_endian)) {
+      idx = 0;
+    }
+    size_t idx_lsb = idx ^ 1;
+    auto is_surrogate = vcleq_u8(
+        vsubq_u8(base_input.val[idx], vdupq_n_u8(0xd8)), vdupq_n_u8(7));
+    // We count on the fact that most inputs do not have surrogates.
+    if (vmaxvq_u32(vreinterpretq_u32_u8(is_surrogate)) ||
+        scalar::utf16::is_low_surrogate<big_endian>(in[pos + N])) {
+      any_surrogates = true;
+      // there is at least one surrogate in the block
+      // We use this to check that surrogates are paired correctly.
+      // It is the input shifted by one code unit (two bytes).
+      // We use it to detect *low* surrogates.
+      auto one_unit_offset_input =
+          vld2q_u8(reinterpret_cast<const uint8_t *>(in + pos + 1));
+      //
+
+      auto lb_masked = vandq_u8(base_input.val[idx], vdupq_n_u8(0xfc));
+      auto block_masked =
+          vandq_u8(one_unit_offset_input.val[idx], vdupq_n_u8(0xfc));
+      auto lb_is_high = vceqq_u8(lb_masked, vdupq_n_u8(0xd8));
+      auto block_is_low = vceqq_u8(block_masked, vdupq_n_u8(0xdc));
+
+      // illseq will mark every low surrogate in the offset block.
+      // that is not preceded by a high surrogate
+      //
+      // It will also mark every high surrogate in the main block
+      // that is not followed by a low surrogate
+      //
+      // This means that it will miss undetectable errors, like a high surrogate
+      // at the last index of the main block. And similarly a low surrogate
+      // at the index prior to the main block that was not preceded by a high
+      // surrogate.
+      //
+      // The interpretation of the values is that they start with the end value
+      // of the prior block, and end just before the end of the main block
+      // (minus one).
+      auto illseq = veorq_u8(lb_is_high, block_is_low);
+      number_of_unpaired_surrogates += vaddlvq_u8(vandq_u8(illseq, one));
+    }
+    auto c0 =
+        vminq_u8(vorrq_u8(vandq_u8(base_input.val[idx_lsb], vdupq_n_u8(0x80)),
+                          base_input.val[idx]),
+                 one);
+    auto c1 = vminq_u8(vandq_u8(base_input.val[idx], vdupq_n_u8(0xf8)), one);
+
+    auto v_count = vaddq_u8(c1, c0);
+    v_count = vaddq_u8(v_count, is_surrogate);
+    count += vaddlvq_u8(v_count); // sum the counts in the vector
+    /////////
+    // The vaddlvq_u8 instruction could be slow on some hardware. We could
+    // consider various alternatives if that is an issue such as accumulating
+    // into a vector of uint16_t or uint8_t and summing only at the end or
+    // periodically. However, on fast chipsets, like Apple Silicon, it is
+    // likely fast enough, or even faster than alternatives.
+    /////////
+  }
+
+  //!!!!!!!!!!!!!!!
+  // Here, we have processed up to pos - 1 (inclusive) code units. Except for
+  // the case where the value at pos is a low surrogate not preceded by a high
+  // surrogate. In this special case, we have already added one to the count for
+  // the unpaired low surrogate.
+  //!!!!!!!!!!!!!!!
+  if (scalar::utf16::is_low_surrogate<big_endian>(in[pos])) {
+    any_surrogates = true;
+    if (!scalar::utf16::is_high_surrogate<big_endian>(in[pos - 1])) {
+      number_of_unpaired_surrogates -= 1;
+      count += 2;
+      pos += 1;
+    }
+  }
+  count += pos;
+  count += number_of_unpaired_surrogates;
+  // If we end with a high surrogate, it might be unpaired or not, we
+  // don't know. It counts as a pair suggarate for now.
+  if (scalar::utf16::is_high_surrogate<big_endian>(in[pos - 1])) {
+    any_surrogates = true;
+    if (pos == size) {
+      count += 2;
+    } else if (scalar::utf16::is_low_surrogate<big_endian>(in[pos])) {
+      pos += 1;
+      count += 2;
+    }
+  }
+  result scalar_result =
+      scalar::utf16::utf8_length_from_utf16_with_replacement<big_endian>(
+          in + pos, size - pos);
+  return {any_surrogates ? SURROGATE : scalar_result.error,
+          count + scalar_result.count};
+}