react-native-quick-crypto 1.0.18 → 1.1.0

package/deps/simdutf/tests/helpers/transcode_test_base.cpp

@@ -0,0 +1,1257 @@
+#include "transcode_test_base.h"
+#include "simdutf.h"
+
+#ifndef SIMDUTF_IS_BIG_ENDIAN
+  #error "SIMDUTF_IS_BIG_ENDIAN should be defined."
+#endif
+
+#include <stdexcept>
+#include <string>
+#include <vector>
+#include <array>
+
+#include <tests/reference/encode_utf8.h>
+#include <tests/reference/encode_utf16.h>
+#include <tests/reference/encode_utf32.h>
+#include <tests/reference/encode_latin1.h>
+#include <tests/reference/decode_utf16.h>
+#include <tests/reference/decode_utf32.h>
+#include <tests/reference/validate_utf8.h>
+#include <tests/reference/validate_utf16.h>
+#include <tests/reference/validate_utf32.h>
+#include <tests/reference/validate_utf8_to_latin1.h>
+#include <tests/reference/validate_utf16_to_latin1.h>
+#include <tests/reference/validate_utf32_to_latin1.h>
+
+namespace simdutf {
+namespace tests {
+namespace helpers {
+
+// C++11 does not have mismatch.
+template <class InputIt1, class InputIt2>
+std::pair<InputIt1, InputIt2> our_mismatch(InputIt1 first1, InputIt1 last1,
+                                           InputIt2 first2, InputIt2 last2) {
+  while (first1 != last1 && first2 != last2 && *first1 == *first2) {
+    ++first1, ++first2;
+  }
+  return std::make_pair(first1, first2);
+}
+
+void transcode_test_base::encode_utf8(uint32_t codepoint,
+                                      std::vector<char> &target) {
+  ::simdutf::tests::reference::utf8::encode(
+      codepoint, [&target](uint8_t byte) { target.push_back(byte); });
+}
+
+void transcode_test_base::encode_utf16(uint32_t codepoint,
+                                       std::vector<char16_t> &target) {
+  char16_t W1;
+  char16_t W2;
+  switch (::simdutf::tests::reference::utf16::encode(codepoint, W1, W2)) {
+  case 1:
+    if (!match_system(utf16_endianness)) {
+      W1 = char16_t((uint16_t(W1) << 8) | (uint16_t(W1) >> 8));
+    }
+
+    target.push_back(W1);
+    break;
+
+  case 2:
+    if (!match_system(utf16_endianness)) {
+      W1 = char16_t((uint16_t(W1) << 8) | (uint16_t(W1) >> 8));
+      W2 = char16_t((uint16_t(W2) << 8) | (uint16_t(W2) >> 8));
+    }
+    target.push_back(W1);
+    target.push_back(W2);
+    break;
+
+  default:
+    throw std::invalid_argument(
+        std::string("Value can't be encoded as UTF-16 code-point : ") +
+        std::to_string(codepoint));
+  }
+}
+
+void transcode_test_base::encode_latin1(uint32_t codepoint,
+                                        std::vector<char> &target) {
+  ::simdutf::tests::reference::latin1::encode(
+      codepoint, [&target](uint8_t byte) { target.push_back(byte); });
+}
+
+void transcode_test_base::encode_utf32(uint32_t codepoint,
+                                       std::vector<char32_t> &target) {
+  ::simdutf::tests::reference::utf32::encode(
+      codepoint, [&target](uint32_t word) { target.push_back(word); });
+}
+
+/**
+ * transcode_latin1_to_utf8_test_base can be used to test Latin1 => UTF8
+ * transcoding.
+ */
+transcode_latin1_to_utf8_test_base::transcode_latin1_to_utf8_test_base(
+    GenerateCodepoint generate, size_t input_size) {
+  while (input_latin1.size() < input_size) {
+    const uint32_t codepoint = generate();
+    prepare_input(codepoint);
+  }
+  output_utf8.resize(reference_output_utf8.size() + output_size_margin);
+}
+
+void transcode_latin1_to_utf8_test_base::prepare_input(uint32_t codepoint) {
+  encode_latin1(codepoint, input_latin1);
+  encode_utf8(codepoint, reference_output_utf8);
+}
+
+bool transcode_latin1_to_utf8_test_base::is_input_valid() const {
+  return true; //
+}
+
+bool transcode_latin1_to_utf8_test_base::validate(size_t saved_chars) const {
+  if (!is_input_valid()) {
+    if (saved_chars != 0) {
+      printf("input Latin1 string is not valid, but conversion routine "
+             "returned %zu, indicating a valid input\n",
+             saved_chars);
+      return false;
+    }
+  }
+  if (saved_chars == 0) {
+    if (is_input_valid()) {
+      printf("input Latin1 string is valid, but conversion routine returned 0, "
+             "indicating input error");
+      return false;
+    }
+    return true;
+  }
+
+  auto dump = [saved_chars](const char *title, const std::vector<char> &array) {
+    printf("%s", title);
+    for (size_t i = 0; i < saved_chars; i++) {
+      printf(" %02x", (uint8_t)array[i]);
+    }
+    putchar('\n');
+  };
+
+  if (saved_chars != reference_output_utf8.size()) {
+    printf("wrong saved bytes value: procedure returned %zu bytes, it should "
+           "be %zu\n",
+           size_t(saved_chars), size_t(reference_output_utf8.size()));
+    dump("expected :", reference_output_utf8);
+    dump("actual   :", output_utf8);
+    return false;
+  }
+  // Note that, in general, output_utf8.size() will not matched saved_chars.
+
+  // At this point, we know that the lengths are the same so std::mismatch is
+  // enough to tell us whether the strings are identical.
+  auto it =
+      our_mismatch(output_utf8.begin(), output_utf8.begin() + saved_chars,
+                   reference_output_utf8.begin(), reference_output_utf8.end());
+  if (it.first != output_utf8.begin() + saved_chars) {
+    printf("mismatched output at %zu: actual value 0x%02x, expected 0x%02x\n",
+           size_t(std::distance(output_utf8.begin(), it.first)),
+           uint8_t(*it.first), uint8_t(*it.second));
+
+    dump("expected :", reference_output_utf8);
+    dump("actual   :", output_utf8);
+    for (size_t i = 0; i < reference_output_utf8.size(); i++) {
+      if (reference_output_utf8[i] != output_utf8[i]) {
+        printf(" ==> ");
+      }
+      printf("at %zu expected 0x%04x 0x%04x\n ", i,
+             uint8_t(reference_output_utf8[i]), uint8_t(output_utf8[i]));
+    }
+    return false;
+  }
+
+  return true;
+}
+
+/**
+ * transcode_latin1_to_utf16_test_base can be used to test Latin1 => utf16
+ * transcoding.
+ */
+transcode_latin1_to_utf16_test_base::transcode_latin1_to_utf16_test_base(
+    endianness utf16_endianness, GenerateCodepoint generate, size_t input_size)
+    : transcode_test_base{utf16_endianness} {
+  while (input_latin1.size() < input_size) {
+    const uint32_t codepoint = generate();
+    prepare_input(codepoint);
+  }
+
+  output_utf16.resize(reference_output_utf16.size() + output_size_margin);
+}
+
+void transcode_latin1_to_utf16_test_base::prepare_input(uint32_t codepoint) {
+  encode_latin1(codepoint, input_latin1);
+
+  encode_utf16(codepoint, reference_output_utf16);
+}
+
+bool transcode_latin1_to_utf16_test_base::is_input_valid() const {
+  return true;
+}
+
+bool transcode_latin1_to_utf16_test_base::validate(size_t saved_chars) const {
+  if (!is_input_valid()) {
+    if (saved_chars != 0) {
+      printf("input Latin1 string is not valid, but conversion routine "
+             "returned %zu, indicating a valid input\n",
+             saved_chars);
+      return false;
+    }
+  }
+  if (saved_chars == 0) {
+    if (is_input_valid()) {
+      printf("input Latin1 string is valid, but conversion routine returned 0, "
+             "indicating input error");
+      return false;
+    }
+
+    return true;
+  }
+
+  auto dump = [saved_chars](const char *title,
+                            const std::vector<char16_t> &array) {
+    printf("%s", title);
+    for (size_t i = 0; i < saved_chars; i++) {
+      printf(
+          " %04x",
+          (uint16_t)array[i]); // Use %04x to print 16-bit hexadecimal numbers
+    }
+    putchar('\n');
+  };
+
+  if (saved_chars != reference_output_utf16.size()) {
+    printf("wrong saved bytes value: procedure returned %zu bytes, it should "
+           "be %zu\n",
+           size_t(saved_chars), size_t(reference_output_utf16.size()));
+    dump("expected :", reference_output_utf16);
+    dump("actual   :", output_utf16);
+    return false;
+  }
+  // Note that, in general, output_utf16.size() will not matched saved_chars.
+
+  // At this point, we know that the lengths are the same so std::mismatch is
+  // enough to tell us whether the strings are identical.
+  auto it = our_mismatch(
+      output_utf16.begin(), output_utf16.begin() + saved_chars,
+      reference_output_utf16.begin(), reference_output_utf16.end());
+  if (it.first != output_utf16.begin() + saved_chars) {
+    printf("mismatched output at %zu: actual value 0x%02x, expected 0x%02x\n",
+           size_t(std::distance(output_utf16.begin(), it.first)),
+           uint16_t(*it.first), uint16_t(*it.second));
+    dump("expected :", reference_output_utf16);
+    dump("actual   :", output_utf16);
+    for (size_t i = 0; i < reference_output_utf16.size(); i++) {
+      if (reference_output_utf16[i] != output_utf16[i]) {
+        printf(" ==> ");
+      }
+      printf("at %zu expected 0x%04x 0x%04x\n ", i,
+             uint16_t(reference_output_utf16[i]), uint16_t(output_utf16[i]));
+    }
+    return false;
+  }
+  return true;
+}
+
+/**
+ * transcode_latin1_to_utf32_test_base can be used to test UTF-32 => UTF-16LE
+ * transcoding.
+ */
+transcode_latin1_to_utf32_test_base::transcode_latin1_to_utf32_test_base(
+    GenerateCodepoint generate, size_t input_size) {
+  while (input_latin1.size() < input_size) {
+    const uint32_t codepoint = generate();
+    prepare_input(codepoint);
+  }
+  output_utf32.resize(reference_output_utf32.size() + output_size_margin);
+}
+
+void transcode_latin1_to_utf32_test_base::prepare_input(uint32_t codepoint) {
+  encode_latin1(codepoint, input_latin1);
+  encode_utf32(codepoint, reference_output_utf32);
+}
+
+bool transcode_latin1_to_utf32_test_base::is_input_valid() const {
+  return true;
+}
+
+bool transcode_latin1_to_utf32_test_base::validate(size_t saved_chars) const {
+  if (!is_input_valid()) {
+    if (saved_chars != 0) {
+      printf("input Latin1 string is not valid, but conversion routine "
+             "returned %zu, indicating a valid input\n",
+             saved_chars);
+      return false;
+    }
+  }
+  if (saved_chars == 0) {
+    if (is_input_valid()) {
+      printf("input Latin1 string is valid, but conversion routine returned 0, "
+             "indicating input error");
+      return false;
+    }
+    return true;
+  }
+
+  auto dump = [saved_chars](const char *title,
+                            const std::vector<char32_t> &array) {
+    printf("%s", title);
+    for (size_t i = 0; i < saved_chars; i++) {
+      printf(" %02x", (unsigned int)array[i]);
+    }
+    putchar('\n');
+  };
+
+  if (saved_chars != reference_output_utf32.size()) {
+    printf("wrong saved bytes value: procedure returned %zu bytes, it should "
+           "be %zu\n",
+           size_t(saved_chars), size_t(reference_output_utf32.size()));
+    dump("expected :", reference_output_utf32);
+    dump("actual   :", output_utf32);
+    return false;
+  }
+  // Note that, in general, output_utf32.size() will not matched saved_chars.
+
+  // At this point, we know that the lengths are the same so std::mismatch is
+  // enough to tell us whether the strings are identical.
+  auto it = our_mismatch(
+      output_utf32.begin(), output_utf32.begin() + saved_chars,
+      reference_output_utf32.begin(), reference_output_utf32.end());
+  if (it.first != output_utf32.begin() + saved_chars) {
+    printf("mismatched output at %zu: actual value 0x%02x, expected 0x%02x\n",
+           size_t(std::distance(output_utf32.begin(), it.first)),
+           uint8_t(*it.first), uint8_t(*it.second));
+
+    dump("expected :", reference_output_utf32);
+    dump("actual   :", output_utf32);
+    for (size_t i = 0; i < reference_output_utf32.size(); i++) {
+      if (reference_output_utf32[i] != output_utf32[i]) {
+        printf(" ==> ");
+      }
+      printf("at %zu expected 0x%04x 0x%04x\n ", i,
+             uint8_t(reference_output_utf32[i]), uint8_t(output_utf32[i]));
+    }
+    return false;
+  }
+  return true;
+}
+
+/**
+ * transcode_utf8_to_latin1_test_base can be used to test UTF-32 => UTF-16LE
+ * transcoding.
+ */
+transcode_utf8_to_latin1_test_base::transcode_utf8_to_latin1_test_base(
+    GenerateCodepoint generate, size_t input_size) {
+  while (input_utf8.size() < input_size) {
+    const uint32_t codepoint = generate();
+    prepare_input(codepoint);
+  }
+  output_latin1.resize(reference_output_latin1.size() + output_size_margin);
+}
+
+void transcode_utf8_to_latin1_test_base::prepare_input(uint32_t codepoint) {
+  encode_utf8(codepoint, input_utf8);
+  encode_latin1(codepoint, reference_output_latin1);
+}
+
+bool transcode_utf8_to_latin1_test_base::is_input_valid() const {
+  return simdutf::tests::reference::validate_utf8_to_latin1(input_utf8.data(),
+                                                            input_utf8.size());
+}
+
+bool transcode_utf8_to_latin1_test_base::validate(size_t saved_chars) const {
+  if (!is_input_valid()) {
+    if (saved_chars != 0) {
+      printf("input UTF-8 string is not valid, but conversion routine returned "
+             "%zu, indicating a valid input\n",
+             saved_chars);
+      return false;
+    }
+  }
+  if (saved_chars == 0) {
+    if (is_input_valid()) {
+      printf("input UTF-8 string is valid, but conversion routine returned 0, "
+             "indicating input error");
+      return false;
+    }
+
+    return true;
+  }
+
+  auto dump = [saved_chars](const char *title, const std::vector<char> &array) {
+    printf("%s", title);
+    for (size_t i = 0; i < saved_chars; i++) {
+      printf(" %02x", (char)array[i]);
+    }
+    putchar('\n');
+  };
+
+  if (saved_chars != reference_output_latin1.size()) {
+    printf("wrong saved bytes value: procedure returned %zu bytes, it should "
+           "be %zu\n",
+           size_t(saved_chars), size_t(reference_output_latin1.size()));
+
+    dump("expected :", reference_output_latin1);
+    dump("actual   :", output_latin1);
+    return false;
+  }
+  // Note that, in general, output_latin1.size() will not matched saved_chars.
+
+  // At this point, we know that the lengths are the same so std::mismatch is
+  // enough to tell us whether the strings are identical.
+  auto it = our_mismatch(
+      output_latin1.begin(), output_latin1.begin() + saved_chars,
+      reference_output_latin1.begin(), reference_output_latin1.end());
+  if (it.first != output_latin1.begin() + saved_chars) {
+    printf("mismatched output at %zu: actual value 0x%02x, expected 0x%02x\n",
+           size_t(std::distance(output_latin1.begin(), it.first)),
+           uint8_t(*it.first), uint8_t(*it.second));
+
+    dump("expected :", reference_output_latin1);
+    dump("actual   :", output_latin1);
+    for (size_t i = 0; i < reference_output_latin1.size(); i++) {
+      if (reference_output_latin1[i] != output_latin1[i]) {
+        printf(" ==> ");
+      }
+      printf("at %zu expected 0x%04x 0x%04x\n ", i,
+             uint8_t(reference_output_latin1[i]), uint8_t(output_latin1[i]));
+    }
+    return false;
+  }
+
+  return true;
+}
+
+/**
+ * transcode_utf16_to_latin1_test_base can be used to test UTF-16 => Latin1
+ * transcoding.
+ */
+transcode_utf16_to_latin1_test_base::transcode_utf16_to_latin1_test_base(
+    endianness utf16_endianness, GenerateCodepoint generate, size_t input_size)
+    : transcode_test_base{utf16_endianness} {
+  while (input_utf16.size() < input_size) {
+    const uint32_t codepoint = generate();
+    prepare_input(codepoint);
+  }
+  output_latin1.resize(reference_output_latin1.size() + output_size_margin);
+}
+
+transcode_utf16_to_latin1_test_base::transcode_utf16_to_latin1_test_base(
+    endianness utf16_endianness, const std::vector<char16_t> &input_utf16)
+    : transcode_test_base{utf16_endianness}, input_utf16{input_utf16} {
+  auto consume = [this](const uint32_t codepoint) {
+    ::simdutf::tests::reference::latin1::encode(
+        codepoint,
+        [this](uint32_t byte) { reference_output_latin1.push_back(byte); });
+  };
+
+  auto error_handler = [](const char16_t *, const char16_t *,
+                          simdutf::tests::reference::utf16::Error) -> bool {
+    throw std::invalid_argument("Wrong UTF-16 input");
+  };
+  simdutf::tests::reference::utf16::decode(utf16_endianness, input_utf16.data(),
+                                           input_utf16.size(), consume,
+                                           error_handler);
+  output_latin1.resize(reference_output_latin1.size() + output_size_margin);
+}
+
+void transcode_utf16_to_latin1_test_base::prepare_input(uint32_t codepoint) {
+  encode_utf16(codepoint, input_utf16);
+  encode_latin1(codepoint, reference_output_latin1);
+}
+
+bool transcode_utf16_to_latin1_test_base::is_input_valid() const {
+  return simdutf::tests::reference::validate_utf16_to_latin1(
+      utf16_endianness, input_utf16.data(), input_utf16.size());
+}
+
+bool transcode_utf16_to_latin1_test_base::validate(size_t saved_chars) const {
+  if (!is_input_valid()) {
+    if (saved_chars != 0) {
+      printf("input UTF-16 string is not valid, but conversion routine "
+             "returned %zu, indicating a valid input\n",
+             saved_chars);
+      return false;
+    }
+  }
+  if (saved_chars == 0) {
+    if (is_input_valid()) {
+      printf("input UTF-16 string is valid, but conversion routine returned 0, "
+             "indicating input error");
+      return false;
+    }
+
+    return true;
+  }
+
+  auto dump = [saved_chars](const char *title, const std::vector<char> &array) {
+    printf("%s", title);
+    for (size_t i = 0; i < saved_chars; i++) {
+      printf(" %08x", (uint32_t)array[i]);
+    }
+    putchar('\n');
+  };
+
+  if (saved_chars != reference_output_latin1.size()) {
+    printf("wrong saved bytes value: procedure returned %zu bytes, it should "
+           "be %zu\n",
+           size_t(saved_chars), size_t(reference_output_latin1.size()));
+
+    dump("expected :", reference_output_latin1);
+    dump("actual   :", output_latin1);
+    return false;
+  }
+  // Note that, in general, output_latin1.size() will not matched saved_chars.
+
+  // At this point, we know that the lengths are the same so std::mismatch is
+  // enough to tell us whether the strings are identical.
+  auto it = our_mismatch(
+      output_latin1.begin(), output_latin1.begin() + saved_chars,
+      reference_output_latin1.begin(), reference_output_latin1.end());
+  if (it.first != output_latin1.begin() + saved_chars) {
+    printf("mismatched output at %zu: actual value 0x%02x, expected 0x%02x\n",
+           size_t(std::distance(output_latin1.begin(), it.first)),
+           uint8_t(*it.first), uint8_t(*it.second));
+
+    dump("expected :", reference_output_latin1);
+    dump("actual   :", output_latin1);
+    for (size_t i = 0; i < reference_output_latin1.size(); i++) {
+      if (reference_output_latin1[i] != output_latin1[i]) {
+        printf(" ==> ");
+      }
+      printf("at %zu expected 0x%08x and got 0x%08x\n ", i,
+             uint8_t(reference_output_latin1[i]), uint8_t(output_latin1[i]));
+    }
+    return false;
+  }
+  return true;
+}
+
+/**
+ * transcode_utf8_to_utf16_test_base can be used to test UTF-8 => UTF-16
+ * transcoding.
+ */
+transcode_utf8_to_utf16_test_base::transcode_utf8_to_utf16_test_base(
+    endianness utf16_endianness, GenerateCodepoint generate, size_t input_size)
+    : transcode_test_base{utf16_endianness} {
+  while (input_utf8.size() < input_size) {
+    const uint32_t codepoint = generate();
+    prepare_input(codepoint);
+  }
+
+  output_utf16.resize(reference_output_utf16.size() + output_size_margin);
+}
+
+void transcode_utf8_to_utf16_test_base::prepare_input(uint32_t codepoint) {
+  encode_utf8(codepoint, input_utf8);
+  encode_utf16(codepoint, reference_output_utf16);
+}
+
+bool transcode_utf8_to_utf16_test_base::is_input_valid() const {
+  return simdutf::tests::reference::validate_utf8(input_utf8.data(),
+                                                  input_utf8.size());
+}
+
+bool transcode_utf8_to_utf16_test_base::validate(size_t saved_chars) const {
+  if (!is_input_valid()) {
+    if (saved_chars != 0) {
+      printf("input UTF-8 string is not valid, but conversion routine returned "
+             "%zu, indicating a valid input\n",
+             saved_chars);
+      return false;
+    }
+  }
+  if (saved_chars == 0) {
+    if (is_input_valid()) {
+      printf("input UTF-8 string is valid, but conversion routine returned 0, "
+             "indicating input error");
+      return false;
+    }
+
+    return true;
+  }
+  if (saved_chars != reference_output_utf16.size()) {
+    printf("wrong saved bytes value: procedure returned %zu bytes, it should "
+           "be %zu\n",
+           size_t(saved_chars), size_t(reference_output_utf16.size()));
+    return false;
+  }
+
+  // Note that, in general, output_utf16.size() will not matched saved_chars.
+
+  // At this point, we know that the lengths are the same so std::mismatch is
+  // enough to tell us whether the strings are identical.
+  auto it = our_mismatch(
+      output_utf16.begin(), output_utf16.begin() + saved_chars,
+      reference_output_utf16.begin(), reference_output_utf16.end());
+  if (it.first != output_utf16.begin() + saved_chars) {
+    printf("mismatched output at %zu: actual value 0x%04x, expected 0x%04x\n",
+           size_t(std::distance(output_utf16.begin(), it.first)),
+           uint16_t(*it.first), uint16_t(*it.second));
+    for (size_t i = 0; i < output_utf16.size(); i++) {
+      if (reference_output_utf16[i] != output_utf16[i]) {
+        printf(" ==> ");
+      }
+      printf("at %zu expected 0x%04x and got 0x%04x\n ", i,
+             uint16_t(reference_output_utf16[i]), uint16_t(output_utf16[i]));
+    }
+
+    return false;
+  }
+
+  return true;
+}
+
+/**
+ * transcode_utf8_to_utf32_test_base can be used to test UTF-8 => UTF-32
+ * transcoding.
+ */
+transcode_utf8_to_utf32_test_base::transcode_utf8_to_utf32_test_base(
+    GenerateCodepoint generate, size_t input_size) {
+  while (input_utf8.size() < input_size) {
+    const uint32_t codepoint = generate();
+    prepare_input(codepoint);
+  }
+
+  output_utf32.resize(reference_output_utf32.size() + output_size_margin);
+}
+
+void transcode_utf8_to_utf32_test_base::prepare_input(uint32_t codepoint) {
+  encode_utf8(codepoint, input_utf8);
+  encode_utf32(codepoint, reference_output_utf32);
+}
+
+bool transcode_utf8_to_utf32_test_base::is_input_valid() const {
+  return simdutf::tests::reference::validate_utf8(input_utf8.data(),
+                                                  input_utf8.size());
+}
+
+bool transcode_utf8_to_utf32_test_base::validate(size_t saved_chars) const {
+  if (!is_input_valid()) {
+    if (saved_chars != 0) {
+      printf("input UTF-8 string is not valid, but conversion routine returned "
+             "%zu, indicating a valid input\n",
+             saved_chars);
+      return false;
+    }
+  }
+  if (saved_chars == 0) {
+    if (is_input_valid()) {
+      printf("input UTF-8 string is valid, but conversion routine returned 0, "
+             "indicating input error");
+      return false;
+    }
+
+    return true;
+  }
+
+  if (saved_chars != reference_output_utf32.size()) {
+    printf("wrong saved bytes value: procedure returned %zu bytes, it should "
+           "be %zu\n",
+           size_t(saved_chars), size_t(reference_output_utf32.size()));
+    return false;
+  }
+
+  // Note that, in general, output_utf16.size() will not matched saved_chars.
+
+  // At this point, we know that the lengths are the same so std::mismatch is
+  // enough to tell us whether the strings are identical.
+  auto it = our_mismatch(
+      output_utf32.begin(), output_utf32.begin() + saved_chars,
+      reference_output_utf32.begin(), reference_output_utf32.end());
+  if (it.first != output_utf32.begin() + saved_chars) {
+    printf("mismatched output at %zu: actual value 0x%04x, expected 0x%04x\n",
+           size_t(std::distance(output_utf32.begin(), it.first)),
+           uint16_t(*it.first), uint16_t(*it.second));
+    for (size_t i = 0; i < output_utf32.size(); i++) {
+      if (reference_output_utf32[i] != output_utf32[i]) {
+        printf(" ==> ");
+      }
+      printf("at %zu expected 0x%08x and got 0x%08x\n ", i,
+             uint32_t(reference_output_utf32[i]), uint32_t(output_utf32[i]));
+    }
+
+    return false;
+  }
+
+  return true;
+}
+
+/**
+ * transcode_utf16_to_utf8_test_base can be used to test UTF-16 => UTF-8
+ * transcoding.
+ */
+transcode_utf16_to_utf8_test_base::transcode_utf16_to_utf8_test_base(
+    endianness utf16_endianness, GenerateCodepoint generate, size_t input_size)
+    : transcode_test_base{utf16_endianness} {
+  while (input_utf16.size() < input_size) {
+    const uint32_t codepoint = generate();
+    prepare_input(codepoint);
+  }
+
+  output_utf8.resize(reference_output_utf8.size() + output_size_margin);
+}
+
+transcode_utf16_to_utf8_test_base::transcode_utf16_to_utf8_test_base(
+    endianness utf16_endianness, const std::vector<char16_t> &input_utf16)
+    : transcode_test_base{utf16_endianness}, input_utf16{input_utf16} {
+
+  auto consume = [this](const uint32_t codepoint) {
+    ::simdutf::tests::reference::utf8::encode(codepoint, [this](uint8_t byte) {
+      reference_output_utf8.push_back(byte);
+    });
+  };
+
+  auto error_handler = [](const char16_t *, const char16_t *,
+                          simdutf::tests::reference::utf16::Error) -> bool {
+    throw std::invalid_argument("Wrong UTF-16 input");
+  };
+  simdutf::tests::reference::utf16::decode(utf16_endianness, input_utf16.data(),
+                                           input_utf16.size(), consume,
+                                           error_handler);
+  output_utf8.resize(reference_output_utf8.size() + output_size_margin);
+}
+
+void transcode_utf16_to_utf8_test_base::prepare_input(uint32_t codepoint) {
+  encode_utf16(codepoint, input_utf16);
+  encode_utf8(codepoint, reference_output_utf8);
+}
+
+bool transcode_utf16_to_utf8_test_base::is_input_valid() const {
+  return simdutf::tests::reference::validate_utf16(
+      utf16_endianness, input_utf16.data(), input_utf16.size());
+}
+
+bool transcode_utf16_to_utf8_test_base::validate(size_t saved_chars) const {
+  if (!is_input_valid()) {
+    if (saved_chars != 0) {
+      printf("input UTF-16 string is not valid, but conversion routine "
+             "returned %zu, indicating a valid input\n",
+             saved_chars);
+      return false;
+    }
+  }
+  if (saved_chars == 0) {
+    if (is_input_valid()) {
+      printf("input UTF-16 string is valid, but conversion routine returned 0, "
+             "indicating input error");
+      return false;
+    }
+
+    return true;
+  }
+
+  auto dump = [saved_chars](const char *title, const std::vector<char> &array) {
+    printf("%s", title);
+    for (size_t i = 0; i < saved_chars; i++) {
+      printf(" %02x", (uint8_t)array[i]);
+    }
+    putchar('\n');
+  };
+
+  if (saved_chars != reference_output_utf8.size()) {
+    printf("wrong saved bytes value: procedure returned %zu bytes, it should "
+           "be %zu\n",
+           size_t(saved_chars), size_t(reference_output_utf8.size()));
+
+    dump("expected :", reference_output_utf8);
+    dump("actual   :", output_utf8);
+    return false;
+  }
+  // Note that, in general, output_utf8.size() will not matched saved_chars.
+
+  // At this point, we know that the lengths are the same so std::mismatch is
+  // enough to tell us whether the strings are identical.
+  auto it =
+      our_mismatch(output_utf8.begin(), output_utf8.begin() + saved_chars,
+                   reference_output_utf8.begin(), reference_output_utf8.end());
+  if (it.first != output_utf8.begin() + saved_chars) {
+    printf("mismatched output at %zu: actual value 0x%02x, expected 0x%02x\n",
+           size_t(std::distance(output_utf8.begin(), it.first)),
+           uint8_t(*it.first), uint8_t(*it.second));
+
+    dump("expected :", reference_output_utf8);
+    dump("actual   :", output_utf8);
+    for (size_t i = 0; i < reference_output_utf8.size(); i++) {
+      if (reference_output_utf8[i] != output_utf8[i]) {
+        printf(" ==> ");
+      }
+      printf("at %zu expected 0x%02x and got 0x%02x\n ", i,
+             uint8_t(reference_output_utf8[i]), uint8_t(output_utf8[i]));
+    }
+    return false;
+  }
+
+  return true;
+}
+
+/**
+ * transcode_utf16_to_utf32_test_base can be used to test UTF-16LE => UTF-32
+ * transcoding.
+ */
+transcode_utf16_to_utf32_test_base::transcode_utf16_to_utf32_test_base(
+    endianness utf16_endianness, GenerateCodepoint generate, size_t input_size)
+    : transcode_test_base{utf16_endianness} {
+  while (input_utf16.size() < input_size) {
+    const uint32_t codepoint = generate();
+    prepare_input(codepoint);
+  }
+
+  output_utf32.resize(reference_output_utf32.size() + output_size_margin);
+}
+
+transcode_utf16_to_utf32_test_base::transcode_utf16_to_utf32_test_base(
+    endianness utf16_endianness, const std::vector<char16_t> &input_utf16)
+    : transcode_test_base{utf16_endianness}, input_utf16{input_utf16} {
+
+  auto consume = [this](const uint32_t codepoint) {
+    ::simdutf::tests::reference::utf32::encode(
+        codepoint,
+        [this](uint32_t byte) { reference_output_utf32.push_back(byte); });
+  };
+
+  auto error_handler = [](const char16_t *, const char16_t *,
+                          simdutf::tests::reference::utf16::Error) -> bool {
+    throw std::invalid_argument("Wrong UTF-16 input");
+  };
+  simdutf::tests::reference::utf16::decode(utf16_endianness, input_utf16.data(),
+                                           input_utf16.size(), consume,
+                                           error_handler);
+  output_utf32.resize(reference_output_utf32.size() + output_size_margin);
+}
+
+void transcode_utf16_to_utf32_test_base::prepare_input(uint32_t codepoint) {
+  encode_utf16(codepoint, input_utf16);
+  encode_utf32(codepoint, reference_output_utf32);
+}
+
+bool transcode_utf16_to_utf32_test_base::is_input_valid() const {
+  return simdutf::tests::reference::validate_utf16(
+      utf16_endianness, input_utf16.data(), input_utf16.size());
+}
+
+bool transcode_utf16_to_utf32_test_base::validate(size_t saved_chars) const {
+  if (!is_input_valid()) {
+    if (saved_chars != 0) {
+      printf("input UTF-16 string is not valid, but conversion routine "
+             "returned %zu, indicating a valid input\n",
+             saved_chars);
+      return false;
+    }
+  }
+  if (saved_chars == 0) {
+    if (is_input_valid()) {
+      printf("input UTF-16 string is valid, but conversion routine returned 0, "
+             "indicating input error");
+      return false;
+    }
+
+    return true;
+  }
+
+  auto dump = [saved_chars](const char *title,
+                            const std::vector<char32_t> &array) {
+    printf("%s", title);
+    for (size_t i = 0; i < saved_chars; i++) {
+      printf(" %08x", (uint32_t)array[i]);
+    }
+    putchar('\n');
+  };
+
+  if (saved_chars != reference_output_utf32.size()) {
+    printf("wrong saved bytes value: procedure returned %zu bytes, it should "
+           "be %zu\n",
+           size_t(saved_chars), size_t(reference_output_utf32.size()));
+
+    dump("expected :", reference_output_utf32);
+    dump("actual   :", output_utf32);
+    return false;
+  }
+  // Note that, in general, output_utf32.size() will not matched saved_chars.
+
+  // At this point, we know that the lengths are the same so std::mismatch is
+  // enough to tell us whether the strings are identical.
+  auto it = our_mismatch(
+      output_utf32.begin(), output_utf32.begin() + saved_chars,
+      reference_output_utf32.begin(), reference_output_utf32.end());
+  if (it.first != output_utf32.begin() + saved_chars) {
+    printf("mismatched output at %zu: actual value 0x%02x, expected 0x%02x\n",
+           size_t(std::distance(output_utf32.begin(), it.first)),
+           uint32_t(*it.first), uint32_t(*it.second));
+
+    dump("expected :", reference_output_utf32);
+    dump("actual   :", output_utf32);
+    for (size_t i = 0; i < reference_output_utf32.size(); i++) {
+      if (reference_output_utf32[i] != output_utf32[i]) {
+        printf(" ==> ");
+      }
+      printf("at %zu expected 0x%08x and got 0x%08x\n ", i,
+             uint32_t(reference_output_utf32[i]), uint32_t(output_utf32[i]));
+    }
+    return false;
+  }
+
+  return true;
+}
+
+/**
+ * transcode_utf32_to_latin1_test_base can be used to test UTF-32 => UTF-16LE
+ * transcoding.
+ */
+transcode_utf32_to_latin1_test_base::transcode_utf32_to_latin1_test_base(
+    GenerateCodepoint generate, size_t input_size) {
+  while (input_utf32.size() < input_size) {
+    const uint32_t codepoint = generate();
+    prepare_input(codepoint);
+  }
+  output_latin1.resize(reference_output_latin1.size() + output_size_margin);
+}
+
+void transcode_utf32_to_latin1_test_base::prepare_input(uint32_t codepoint) {
+  encode_utf32(codepoint, input_utf32);
+  encode_latin1(codepoint,
+                reference_output_latin1); //-- not applicable? All of a byte is
+                                          // translatable to latin1
+}
+
+bool transcode_utf32_to_latin1_test_base::is_input_valid() const {
+  return simdutf::tests::reference::validate_utf32_to_latin1(
+      input_utf32.data(), input_utf32.size());
+}
+
+bool transcode_utf32_to_latin1_test_base::validate(size_t saved_chars) const {
+  if (!is_input_valid()) {
+    if (saved_chars != 0) {
+      printf("input UTF-32 string is not valid, but conversion routine "
+             "returned %zu, indicating a valid input\n",
+             saved_chars);
+      return false;
+    }
+  }
+  if (saved_chars == 0) {
+    if (is_input_valid()) {
+      printf("input UTF-32 string is valid, but conversion routine returned 0, "
+             "indicating input error");
+      return false;
+    }
+    return true;
+  }
+
+  auto dump = [saved_chars](const char *title, const std::vector<char> &array) {
+    printf("%s", title);
+    for (size_t i = 0; i < saved_chars; i++) {
+      printf(" %02x", (char)array[i]);
+    }
+    putchar('\n');
+  };
+
+  if (saved_chars != reference_output_latin1.size()) {
+    printf("wrong saved bytes value: procedure returned %zu bytes, it should "
+           "be %zu\n",
+           size_t(saved_chars), size_t(reference_output_latin1.size()));
+
+    dump("expected :", reference_output_latin1);
+    dump("actual   :", output_latin1);
+    return false;
+  }
+  // Note that, in general, output_latin1.size() will not matched saved_chars.
+
+  // At this point, we know that the lengths are the same so std::mismatch is
+  // enough to tell us whether the strings are identical.
+  auto it = our_mismatch(
+      output_latin1.begin(), output_latin1.begin() + saved_chars,
+      reference_output_latin1.begin(), reference_output_latin1.end());
+  if (it.first != output_latin1.begin() + saved_chars) {
+    printf("mismatched output at %zu: actual value 0x%02x, expected 0x%02x\n",
+           size_t(std::distance(output_latin1.begin(), it.first)),
+           uint8_t(*it.first), uint8_t(*it.second));
+
+    dump("expected :", reference_output_latin1);
+    dump("actual   :", output_latin1);
+    for (size_t i = 0; i < reference_output_latin1.size(); i++) {
+      if (reference_output_latin1[i] != output_latin1[i]) {
+        printf(" ==> ");
+      }
+      printf("at %zu expected 0x%04x 0x%04x\n ", i,
+             uint8_t(reference_output_latin1[i]), uint8_t(output_latin1[i]));
+    }
+    return false;
+  }
+  return true;
+}
+
+/**
+ * transcode_utf32_to_utf8_test_base can be used to test UTF-32 => UTF-8
+ * transcoding.
+ */
+transcode_utf32_to_utf8_test_base::transcode_utf32_to_utf8_test_base(
+    GenerateCodepoint generate, size_t input_size) {
+  while (input_utf32.size() < input_size) {
+    const uint32_t codepoint = generate();
+    prepare_input(codepoint);
+  }
+
+  output_utf8.resize(reference_output_utf8.size() + output_size_margin);
+}
+
+transcode_utf32_to_utf8_test_base::transcode_utf32_to_utf8_test_base(
+    const std::vector<char32_t> &input_utf32)
+    : input_utf32{input_utf32} {
+
+  auto consume = [this](const uint32_t codepoint) {
+    ::simdutf::tests::reference::utf8::encode(codepoint, [this](uint8_t byte) {
+      reference_output_utf8.push_back(byte);
+    });
+  };
+
+  auto error_handler = [](const char32_t *, const char32_t *,
+                          simdutf::tests::reference::utf32::Error) -> bool {
+    throw std::invalid_argument("Wrong UTF-32 input");
+  };
+  simdutf::tests::reference::utf32::decode(
+      input_utf32.data(), input_utf32.size(), consume, error_handler);
+  output_utf8.resize(reference_output_utf8.size() + output_size_margin);
+}
+
+void transcode_utf32_to_utf8_test_base::prepare_input(uint32_t codepoint) {
+  encode_utf32(codepoint, input_utf32);
+  encode_utf8(codepoint, reference_output_utf8);
+}
+
+bool transcode_utf32_to_utf8_test_base::is_input_valid() const {
+  return simdutf::tests::reference::validate_utf32(input_utf32.data(),
+                                                   input_utf32.size());
+}
+
+bool transcode_utf32_to_utf8_test_base::validate(size_t saved_chars) const {
+  if (!is_input_valid()) {
+    if (saved_chars != 0) {
+      printf("input UTF-32 string is not valid, but conversion routine "
+             "returned %zu, indicating a valid input\n",
+             saved_chars);
+      return false;
+    }
+  }
+  if (saved_chars == 0) {
+    if (is_input_valid()) {
+      printf("input UTF-32 string is valid, but conversion routine returned 0, "
+             "indicating input error");
+      return false;
+    }
+
+    return true;
+  }
+
+  auto dump = [saved_chars](const char *title, const std::vector<char> &array) {
+    printf("%s", title);
+    for (size_t i = 0; i < saved_chars; i++) {
+      printf(" %02x", (uint8_t)array[i]);
+    }
+    putchar('\n');
+  };
+
+  if (saved_chars != reference_output_utf8.size()) {
+    printf("wrong saved bytes value: procedure returned %zu bytes, it should "
+           "be %zu\n",
+           size_t(saved_chars), size_t(reference_output_utf8.size()));
+
+    dump("expected :", reference_output_utf8);
+    dump("actual   :", output_utf8);
+    return false;
+  }
+  // Note that, in general, output_utf8.size() will not matched saved_chars.
+
+  // At this point, we know that the lengths are the same so std::mismatch is
+  // enough to tell us whether the strings are identical.
+  auto it =
+      our_mismatch(output_utf8.begin(), output_utf8.begin() + saved_chars,
+                   reference_output_utf8.begin(), reference_output_utf8.end());
+  if (it.first != output_utf8.begin() + saved_chars) {
+    printf("mismatched output at %zu: actual value 0x%02x, expected 0x%02x\n",
+           size_t(std::distance(output_utf8.begin(), it.first)),
+           uint8_t(*it.first), uint8_t(*it.second));
+
+    dump("expected :", reference_output_utf8);
+    dump("actual   :", output_utf8);
+    for (size_t i = 0; i < reference_output_utf8.size(); i++) {
+      if (reference_output_utf8[i] != output_utf8[i]) {
+        printf(" ==> ");
+      }
+      printf("at %zu expected 0x%02x and got 0x%02x\n ", i,
+             uint8_t(reference_output_utf8[i]), uint8_t(output_utf8[i]));
+    }
+    return false;
+  }
+
+  return true;
+}
+
+/**
+ * transcode_utf32_to_utf16_test_base can be used to test UTF-32 => UTF-16LE
+ * transcoding.
+ */
+transcode_utf32_to_utf16_test_base::transcode_utf32_to_utf16_test_base(
+    endianness utf16_endianness, GenerateCodepoint generate, size_t input_size)
+    : transcode_test_base{utf16_endianness} {
+  while (input_utf32.size() < input_size) {
+    const uint32_t codepoint = generate();
+    prepare_input(codepoint);
+  }
+
+  output_utf16.resize(reference_output_utf16.size() + output_size_margin);
+}
+
+void transcode_utf32_to_utf16_test_base::prepare_input(uint32_t codepoint) {
+  encode_utf32(codepoint, input_utf32);
+  encode_utf16(codepoint, reference_output_utf16);
+}
+
+bool transcode_utf32_to_utf16_test_base::is_input_valid() const {
+  return simdutf::tests::reference::validate_utf32(input_utf32.data(),
+                                                   input_utf32.size());
+}
+
+bool transcode_utf32_to_utf16_test_base::validate(size_t saved_chars) const {
+  if (!is_input_valid()) {
+    if (saved_chars != 0) {
+      printf("input UTF-32 string is not valid, but conversion routine "
+             "returned %zu, indicating a valid input\n",
+             saved_chars);
+      return false;
+    }
+  }
+  if (saved_chars == 0) {
+    if (is_input_valid()) {
+      printf("input UTF-32 string is valid, but conversion routine returned 0, "
+             "indicating input error");
+      return false;
+    }
+
+    return true;
+  }
+
+  auto dump = [saved_chars](const char *title,
+                            const std::vector<char16_t> &array) {
+    printf("%s", title);
+    for (size_t i = 0; i < saved_chars; i++) {
+      printf(" %02x", (uint16_t)array[i]);
+    }
+    putchar('\n');
+  };
+
+  if (saved_chars != reference_output_utf16.size()) {
+    printf("wrong saved bytes value: procedure returned %zu bytes, it should "
+           "be %zu\n",
+           size_t(saved_chars), size_t(reference_output_utf16.size()));
+
+    dump("expected :", reference_output_utf16);
+    dump("actual   :", output_utf16);
+    return false;
+  }
+  // Note that, in general, output_utf16.size() will not matched saved_chars.
+
+  // At this point, we know that the lengths are the same so std::mismatch is
+  // enough to tell us whether the strings are identical.
+  auto it = our_mismatch(
+      output_utf16.begin(), output_utf16.begin() + saved_chars,
+      reference_output_utf16.begin(), reference_output_utf16.end());
+  if (it.first != output_utf16.begin() + saved_chars) {
+    printf("mismatched output at %zu: actual value 0x%02x, expected 0x%02x\n",
+           size_t(std::distance(output_utf16.begin(), it.first)),
+           uint8_t(*it.first), uint8_t(*it.second));
+
+    dump("expected :", reference_output_utf16);
+    dump("actual   :", output_utf16);
+    for (size_t i = 0; i < reference_output_utf16.size(); i++) {
+      if (reference_output_utf16[i] != output_utf16[i]) {
+        printf(" ==> ");
+      }
+      printf("at %zu expected 0x%04x 0x%04x\n ", i,
+             uint16_t(reference_output_utf16[i]), uint16_t(output_utf16[i]));
+    }
+    return false;
+  }
+
+  return true;
+}
+
+} // namespace helpers
+} // namespace tests
+} // namespace simdutf
+
+//------------------------------------------------------------
+
+std::vector<std::vector<char16_t>>
+all_utf16_combinations(simdutf::endianness byte_order) {
+  // non-surrogate word that yields 1 UTF-8 byte
+  const char16_t V_1byte_start = 0x0042;
+  // non-surrogate word that yields 2 UTF-8 bytes
+  const char16_t V_2bytes_start = 0x017f;
+  // non-surrogate word the yields 3 UTF-8 bytes
+  const char16_t V_3bytes_start = 0xefff;
+  const char16_t L = to_utf16(byte_order, 0xd9ca); // low surrogate
+  const char16_t H = to_utf16(byte_order, 0xde42); // high surrogate
+
+  std::vector<std::vector<char16_t>> result;
+  std::vector<char16_t> row(32, to_utf16(byte_order, '*'));
+
+  std::array<int, 8> pattern{0};
+  while (true) {
+    // 1. produce output
+    char16_t V_1byte = V_1byte_start;
+    char16_t V_2bytes = V_2bytes_start;
+    char16_t V_3bytes = V_3bytes_start;
+    for (int i = 0; i < 8; i++) {
+      switch (pattern[i]) {
+      case 0:
+        row[i] = to_utf16(byte_order, V_1byte++);
+        break;
+      case 1:
+        row[i] = to_utf16(byte_order, V_2bytes++);
+        break;
+      case 2:
+        row[i] = to_utf16(byte_order, V_3bytes++);
+        break;
+      case 3:
+        row[i] = L;
+        break;
+      case 4:
+        row[i] = H;
+        break;
+      default:
+        abort();
+      }
+    } // for
+
+    if (row[7] == L) {
+      row[8] = H; // make input valid
+      result.push_back(row);
+
+      row[8] = to_utf16(byte_order, V_1byte); // broken input
+      result.push_back(row);
+    } else {
+      row[8] = to_utf16(byte_order, V_1byte);
+      result.push_back(row);
+    }
+
+    // next pattern
+    int i = 0;
+    int carry = 1;
+    for (; i < 8 && carry; i++) {
+      pattern[i] += carry;
+      if (pattern[i] == 5) {
+        pattern[i] = 0;
+        carry = 1;
+      } else
+        carry = 0;
+    }
+
+    if (carry == 1 and i == 8)
+      break;
+
+  } // while
+
+  return result;
+}