react-native-quick-crypto 1.0.19 → 1.1.0

package/deps/simdutf/benchmarks/competition/utf8lut/src/core/DecoderLut.h

@@ -0,0 +1,42 @@
+#pragma once
+
+#include <stdint.h>
+#include <emmintrin.h>
+#include "../base/PerfDefs.h"
+
+struct shufstep {
+    uint32_t _shufC_part0;
+    uint32_t _shufC_part1;
+    uint32_t srcStep;           //number of bytes processed in input buffer
+    uint32_t dstStep;           //number of symbols produced in output buffer (doubled)
+};
+
+struct DecoderCoreInfo {
+    __m128i shufAB;                     //shuffling mask to get lower two bytes of symbols
+    union {
+        __m128i shufC;                  //shuffling mask to get third bytes of symbols
+        shufstep s;
+    };
+};
+struct DecoderValidationInfo {
+    __m128i headerMask;                 //mask of "111..10" bits required in each byte
+    __m128i minValues;                  //minimal value allowed for not being overlong (sign-shifted, 16-bit)
+};
+
+//a single entry of each LUT is defined
+template<bool Validate> struct DecoderLutEntry {};
+template<> struct DecoderLutEntry<false> : DecoderCoreInfo {};
+template<> struct DecoderLutEntry<true> : DecoderCoreInfo, DecoderValidationInfo {};
+
+//a whole LUT table type
+template<bool Validate> struct DecoderLutTable {
+    //note: odd-indexed entries are removed (they are impossible with correct input)
+    CACHEALIGN DecoderLutEntry<Validate> data[32768];
+
+    static const DecoderLutTable<Validate> *CreateInstance();
+    inline static const DecoderLutEntry<Validate> *GetArray() { return CreateInstance()->data; }
+private:
+    void ComputeAll();
+    void ComputeRec(int *sizes, int num, int total);
+    void ComputeEntry(const int *sizes, int num);
+};

package/deps/simdutf/benchmarks/competition/utf8lut/src/core/DecoderProcess.h

@@ -0,0 +1,100 @@
+#pragma once
+
+#include <stdint.h>
+#include <tmmintrin.h>
+#include "../base/PerfDefs.h"
+#include "../core/DecoderLut.h"
+
+/** template params:
+ * MaxBytes = 1, 2, 3
+ * CheckExceed = false, true
+ * Validate = false, true
+ * OutputType = 2, 4        //UTF16/32
+ */
+
+template<int MaxBytes, bool CheckExceed, bool Validate, int OutputType>
+struct DecoderCore {
+    FORCEINLINE bool operator()(const char *&ptrSource, char *&ptrDest, const DecoderLutEntry<Validate> *RESTRICT lutTable) {
+        static_assert(!Validate || CheckExceed, "Validate core mode requires CheckExceed enabled");
+        const char *RESTRICT pSource = ptrSource;
+        char *RESTRICT pDest = ptrDest;
+
+        if (MaxBytes == 1) {
+            __m128i reg = _mm_loadu_si128((__m128i*)pSource);
+            __m128i zero = _mm_setzero_si128();
+            if (CheckExceed && _mm_movemask_epi8(reg))
+                return false;
+            __m128i half0 = _mm_unpacklo_epi8(reg, zero);
+            __m128i half1 = _mm_unpackhi_epi8(reg, zero);
+            if (OutputType == 2) {
+                _mm_storeu_si128((__m128i*)pDest + 0, half0);
+                _mm_storeu_si128((__m128i*)pDest + 1, half1);
+            }
+            else {
+                _mm_storeu_si128((__m128i*)pDest + 0, _mm_unpacklo_epi16(half0, zero));
+                _mm_storeu_si128((__m128i*)pDest + 1, _mm_unpackhi_epi16(half0, zero));
+                _mm_storeu_si128((__m128i*)pDest + 2, _mm_unpacklo_epi16(half1, zero));
+                _mm_storeu_si128((__m128i*)pDest + 3, _mm_unpackhi_epi16(half1, zero));
+            }
+            ptrSource += 16;
+            ptrDest += 16 * OutputType;
+            return true;
+        }
+        else {  //MaxBytes = 2 or 3
+            __m128i reg = _mm_loadu_si128((__m128i*)pSource);
+            if (CheckExceed && !Validate) {
+                __m128i pl = _mm_xor_si128(reg, _mm_set1_epi8(char(0x80U)));  //_mm_sub_epi8
+                __m128i cmpRes = _mm_cmpgt_epi8(pl, _mm_set1_epi8(MaxBytes == 3 ? 0x6F : 0x5F));
+                if (!_mm_cmp_allzero(cmpRes))
+                    return false;
+            }
+
+            uint32_t mask = _mm_movemask_epi8(_mm_cmplt_epi8(reg, _mm_set1_epi8(char(0xC0U))));
+            if (Validate && (mask & 1))
+                return false;
+            //note: optimized half-index access
+            const DecoderLutEntry<Validate> *RESTRICT lookup = TPNT(lutTable, DecoderLutEntry<Validate>, mask * (sizeof(lutTable[0]) / 2));
+
+            __m128i Rab = _mm_shuffle_epi8(reg, lookup->shufAB);
+            Rab = _mm_and_si128(Rab, _mm_set1_epi16(0x3F7F));
+            Rab = _mm_maddubs_epi16(Rab, _mm_set1_epi16(0x4001));
+            __m128i sum = Rab;
+
+            if (MaxBytes == 3) {
+                __m128i shufC = _mm_unpacklo_epi8(lookup->shufC, lookup->shufC);
+                __m128i Rc = _mm_shuffle_epi8(reg, shufC);
+                Rc = _mm_slli_epi16(Rc, 12);
+                sum = _mm_add_epi16(sum, Rc);
+            }
+
+            if (Validate) {
+                const DecoderLutEntry<true> *RESTRICT lookupX = (const DecoderLutEntry<true> *)lookup;
+                __m128i byteMask = lookupX->headerMask;
+                __m128i header = _mm_and_si128(reg, byteMask);
+                __m128i hdrRef = _mm_add_epi8(byteMask, byteMask);
+                __m128i hdrCorrect = _mm_cmpeq_epi8(header, hdrRef);
+                __m128i overlongSymbol = _mm_cmplt_epi16(_mm_xor_si128(sum, _mm_set1_epi16((short int)0x8000U)), lookupX->minValues);
+                __m128i surrogate = _mm_cmpgt_epi16(_mm_sub_epi16(sum, _mm_set1_epi16(0x6000)), _mm_set1_epi16(0x77FF));
+                if (MaxBytes == 2) {
+                    __m128i shufC = _mm_unpacklo_epi8(lookupX->shufC, lookupX->shufC);
+                    hdrCorrect = _mm_and_si128(hdrCorrect, shufC); //forbid 3-byte symbols
+                }
+                __m128i allCorr = _mm_andnot_si128(_mm_or_si128(overlongSymbol, surrogate), hdrCorrect);
+                if (!_mm_cmp_allone(allCorr))
+                    return false;
+            }
+
+            if (OutputType == 2)
+                _mm_storeu_si128((__m128i*)pDest, sum);
+            else {
+                __m128i zero = _mm_setzero_si128();
+                _mm_storeu_si128((__m128i*)pDest + 0, _mm_unpacklo_epi16(sum, zero));
+                _mm_storeu_si128((__m128i*)pDest + 1, _mm_unpackhi_epi16(sum, zero));
+            }
+            ptrSource += lookup->s.srcStep;
+            ptrDest += lookup->s.dstStep * (OutputType/2);
+
+            return true;
+        }
+    }
+};

package/deps/simdutf/benchmarks/competition/utf8lut/src/core/Dfa.h

@@ -0,0 +1,57 @@
+// Automaton-based utf8 decoder:
+// http://bjoern.hoehrmann.de/utf-8/decoder/dfa/
+//
+// Copyright (c) 2008-2009 Bjoern Hoehrmann <bjoern@hoehrmann.de>
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"),
+// to deal in the Software without restriction, including without limitation
+// the rights to use, copy, modify, merge, publish, distribute, sublicense,
+// and/or sell copies of the Software, and to permit persons to whom the Software
+// is furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included
+// in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+#define UTF8_ACCEPT 0
+#define UTF8_REJECT 12
+
+static const uint8_t utf8d[] = {
+  // The first part of the table maps bytes to character classes that
+  // to reduce the size of the transition table and create bitmasks.
+   0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+   0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+   0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+   0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+   1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,  9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,
+   7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,  7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
+   8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2,  2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
+  10,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 11,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8,
+
+  // The second part is a transition table that maps a combination
+  // of a state of the automaton and a character class to a state.
+   0,12,24,36,60,96,84,12,12,12,48,72, 12,12,12,12,12,12,12,12,12,12,12,12,
+  12, 0,12,12,12,12,12, 0,12, 0,12,12, 12,24,12,12,12,12,12,24,12,24,12,12,
+  12,12,12,12,12,12,12,24,12,12,12,12, 12,24,12,12,12,12,12,12,12,24,12,12,
+  12,12,12,12,12,12,12,36,12,36,12,12, 12,36,12,12,12,12,12,36,12,36,12,12,
+  12,36,12,12,12,12,12,12,12,12,12,12, 
+};
+
+uint32_t inline
+decode(uint32_t* state, uint32_t* codep, uint32_t byte) {
+  uint32_t type = utf8d[byte];
+
+  *codep = (*state != UTF8_ACCEPT) ?
+    (byte & 0x3fu) | (*codep << 6) :
+    (0xff >> type) & (byte);
+
+  *state = utf8d[256 + *state + type];
+  return *state;
+}

package/deps/simdutf/benchmarks/competition/utf8lut/src/core/EncoderLut.cpp

@@ -0,0 +1,85 @@
+#include "../core/EncoderLut.h"
+#include <assert.h>
+#include <string.h>
+
+template<bool ThreeBytes> void EncoderLutTable<ThreeBytes>::ComputeAll() {
+    //start search for all masks
+    for (int lensMask = 0; lensMask < 256; lensMask++)
+        ComputeEntry(lensMask);
+}
+
+typedef int (*PartPosGetter)(int, int);
+static void HandleChar(int idx, int len, int &pos, char *shuf, char *header, PartPosGetter getPart) {
+    for (int j = 0; j < len; j++) {
+        shuf[pos + j] = (char)getPart(idx, len-1 - j);
+        header[pos + j] = (char)0xC0U;
+    }
+    static const int firstByteHeader[] = {0xFF, 0x80, 0xE0, 0xF0};
+    header[pos] = char(firstByteHeader[len]);
+    pos += len;
+}
+
+int TwoBytesPartPosGetter(int idx, int part) {
+    return 2 * idx + (1 - part);
+}
+template<> void EncoderLutTable<false>::ComputeEntry(int lensMask) {
+    //init shuffle bitmask and header mask
+    char shuf[16], header[16];
+    memset(shuf, -1, sizeof(shuf));
+    memset(header, 0, sizeof(header));
+    //go over all 8 input symbols
+    int pos = 0;
+    for (int i = 0; i < 8; i++) {
+        int len = 1 + (1 & (lensMask >> i));
+        HandleChar(i, len, pos, shuf, header, TwoBytesPartPosGetter);
+    }
+    //save data into LUT entry
+    EncoderLutEntry &entry = data[lensMask];
+    entry.shuf = _mm_loadu_si128((__m128i*)shuf);
+    entry.headerMask = _mm_loadu_si128((__m128i*)header);
+    entry.dstStep = pos;
+}
+
+int ThreeBytesPartPosGetter(int idx, int part) {
+    if (part == 0)
+        return 2 * idx + 0;
+    if (part == 2)
+        return 2 * idx + 1;
+    if (part == 1)
+        return 8 + 2 * idx;
+    return -1;
+}
+template<> void EncoderLutTable<true>::ComputeEntry(int lensMask) {
+    //init shuffle bitmask and header mask
+    char shuf[16], header[16];
+    memset(shuf, -1, sizeof(shuf));
+    memset(header, 0, sizeof(header));
+    //go over all 8 input symbols
+    int pos = 0;
+    int index = 0;
+    for (int i = 0; i < 4; i++) {
+        int len = 1 + (3 & (lensMask >> (2 * i)));
+        if (len > 3)
+            return; //impossible entry
+        HandleChar(i, len, pos, shuf, header, ThreeBytesPartPosGetter);
+        index ^= (len ^ 1) << (2 * i);  //xor 1 just suits...
+    }
+    //save data into LUT entry
+    EncoderLutEntry &entry = data[index];
+    entry.shuf = _mm_loadu_si128((__m128i*)shuf);
+    entry.headerMask = _mm_loadu_si128((__m128i*)header);
+    entry.dstStep = pos;
+}
+
+template<bool ThreeBytes> const EncoderLutTable<ThreeBytes> *EncoderLutTable<ThreeBytes>::CreateInstance() {
+    static EncoderLutTable<ThreeBytes> *singletonTable = 0;
+    if (!singletonTable) {
+        singletonTable = (EncoderLutTable<ThreeBytes> *)_mm_malloc(sizeof(EncoderLutTable<ThreeBytes>), CACHE_LINE);
+        singletonTable->ComputeAll();
+    }
+    return singletonTable;
+}
+
+
+template struct EncoderLutTable<false>;
+template struct EncoderLutTable<true>;

package/deps/simdutf/benchmarks/competition/utf8lut/src/core/EncoderLut.h

@@ -0,0 +1,27 @@
+#pragma once
+
+#include <stdint.h>
+#include <emmintrin.h>
+#include "../base/PerfDefs.h"
+
+//a single entry of each LUT is defined
+struct m128pair { __m128i __a, __b; };
+struct EncoderLutEntry {
+    __m128i shuf;                       //shuffling mask to move bytes into position
+    __m128i headerMask;                 //mask of bits which represents header (for each byte)
+    union {
+        uint32_t dstStep;               //number of bytes processed in output buffer
+        m128pair s;   //padding up to 64 bytes
+    };
+};
+
+//a whole LUT table type
+template<bool ThreeBytes> struct EncoderLutTable {
+    CACHEALIGN EncoderLutEntry data[256];
+
+    static const EncoderLutTable<ThreeBytes> *CreateInstance();
+    inline static const EncoderLutEntry *GetArray() { return CreateInstance()->data; }
+private:
+    void ComputeAll();
+    void ComputeEntry(int lensMask);
+};

package/deps/simdutf/benchmarks/competition/utf8lut/src/core/EncoderProcess.h

@@ -0,0 +1,126 @@
+#pragma once
+
+#include "../base/PerfDefs.h"
+#include <tmmintrin.h>
+
+/** template params:
+ * MaxBytes = 1, 2, 3
+ * CheckExceed = false, true
+ * InputType = 2, 4         //UTF16/32
+ */
+
+template<int MaxBytes, bool CheckExceed, int InputType>
+struct EncoderCore {
+    FORCEINLINE bool operator()(const char *&ptrSource, char *&ptrDest, const EncoderLutEntry *RESTRICT lutTable) {
+        const char *RESTRICT pSource = ptrSource;
+        char *RESTRICT pDest = ptrDest;
+        
+        //read block of 8 symbols
+        __m128i reg;
+        if (InputType == 2)
+            reg = _mm_loadu_si128((const __m128i *)pSource);
+        else {
+            //read in two halves
+            __m128i reg0 = _mm_loadu_si128((const __m128i *)pSource + 0);
+            __m128i reg1 = _mm_loadu_si128((const __m128i *)pSource + 1);
+            __m128i shuf = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15);
+            //shuffle all lower 16-bit words to lower halves of registers
+            reg0 = _mm_shuffle_epi8(reg0, shuf);
+            reg1 = _mm_shuffle_epi8(reg1, shuf);
+            //get all 8 lower 16-bit words
+            reg = _mm_unpacklo_epi64(reg0, reg1);
+            //if required, check that higher 16-bit words are zero
+            if (CheckExceed && !_mm_cmp_allone(_mm_cmpeq_epi16(_mm_unpackhi_epi64(reg0, reg1), _mm_setzero_si128())))
+                return false;
+        }
+
+        if (MaxBytes == 1) {
+            if (CheckExceed && !_mm_cmp_allone(_mm_cmpeq_epi16(_mm_andnot_si128(_mm_set1_epi16(0x007FU), reg), _mm_setzero_si128())))
+                return false;
+            __m128i res = _mm_packus_epi16(reg, _mm_setzero_si128());
+            _mm_storeu_si128((__m128i *)pDest, res);
+            pDest += 8;
+        }
+        else if (MaxBytes == 2) {
+            //levels of bytes
+            __m128i levelA = reg;                                                   //abcdefgh|ABCDEFGH
+            __m128i levelB = _mm_srli_epi16(reg, 6);                                //ghABCDEF|GH......
+            //put all bytes of each half into a register
+            __m128i levBA = _mm_xor_si128(levelB, _mm_slli_epi16(levelA, 8));       //ghABC,,,|abcdefgh
+
+            //check which symbols are long
+            __m128i lenGe2 = _mm_cmpgt_epi16(levelB, _mm_set1_epi16(0x0001U));
+            //check if there are three+ bytes symbols
+            if (CheckExceed && !_mm_cmp_allzero(_mm_cmpgt_epi16(levelB, _mm_set1_epi16(0x001FU))))
+                return false;
+            //compose lens mask for lookup (with a 6-byte shift)
+            __m128i lensAll = _mm_shuffle_epi8(lenGe2, _mm_setr_epi8(-1, -1, -1, -1, -1, -1, 0, 2, 4, 6, 8, 10, 12, 14, -1, -1));
+            //get byte offset into lookup table (i.e. index multiplied by 64 = entry size)
+            uint32_t offset = _mm_movemask_epi8(lensAll);
+            static_assert(sizeof(EncoderLutEntry) == 64, "Wrong size of EncoderLutEntry");
+        
+            //load info from LUT (using byte offset)
+            const EncoderLutEntry *RESTRICT lookup = TPNT(lutTable, EncoderLutEntry, offset);
+            //shuffle bytes to compact layout
+            __m128i res = _mm_shuffle_epi8(levBA, lookup->shuf);
+            //add headers to all bytes
+            __m128i header = lookup->headerMask;
+            res = _mm_andnot_si128(header, res);
+            res = _mm_xor_si128(res, _mm_add_epi8(header, header)); //add_epi8
+    
+            //write results
+            _mm_storeu_si128((__m128i *)pDest, res);
+            pDest += lookup->dstStep;
+        }
+        else if (MaxBytes == 3) {
+            //levels of bytes
+            __m128i levAC = _mm_maddubs_epi16(_mm_and_si128(reg, _mm_set1_epi16((short int)0xF0FFU)), _mm_set1_epi16(0x1001U));
+            __m128i levelB = _mm_srli_epi16(reg, 6);
+            //put all bytes of each half into a separate register
+            __m128i levels0 = _mm_unpacklo_epi64(levAC, levelB);
+            __m128i levels1 = _mm_unpackhi_epi64(levAC, levelB);
+
+            if (CheckExceed) {
+                //check if there are any surrogates
+                __m128i diff = _mm_sub_epi16(reg, _mm_set1_epi16(0x5800U));
+                if (!_mm_cmp_allzero(_mm_cmplt_epi16(diff, _mm_set1_epi16((short int)0x8800U))))
+                    return false;
+            }
+
+            //check for symbols with len at least 2 and 3
+            __m128i lenGe2 = _mm_cmpgt_epi16(levelB, _mm_set1_epi16(0x0001U));
+            __m128i lenGe3 = _mm_cmpgt_epi16(levelB, _mm_set1_epi16(0x001FU));
+            //get a single mask from the two comparison results
+            __m128i lensMix = _mm_xor_si128(_mm_srli_epi16(lenGe3, 8), lenGe2);
+            uint32_t allMask = _mm_movemask_epi8(lensMix);
+            //each half of the mask corresponds to a LUT index
+            uint32_t offset0 = (allMask & 255U) * sizeof(EncoderLutEntry);
+            uint32_t offset1 = (allMask >> 8U) * sizeof(EncoderLutEntry);
+        
+            //load info from LUT
+            const EncoderLutEntry *RESTRICT lookup0 = TPNT(lutTable, EncoderLutEntry, offset0);
+            const EncoderLutEntry *RESTRICT lookup1 = TPNT(lutTable, EncoderLutEntry, offset1);
+            //shuffle bytes of each half to compact layout
+            __m128i res0 = _mm_shuffle_epi8(levels0, lookup0->shuf);
+            __m128i res1 = _mm_shuffle_epi8(levels1, lookup1->shuf);
+            //add headers to all bytes
+            __m128i header0 = lookup0->headerMask;
+            __m128i header1 = lookup1->headerMask;
+            res0 = _mm_andnot_si128(header0, res0);
+            res1 = _mm_andnot_si128(header1, res1);
+            res0 = _mm_xor_si128(res0, _mm_add_epi8(header0, header0));
+            res1 = _mm_xor_si128(res1, _mm_add_epi8(header1, header1));
+            
+            //write results
+            _mm_storeu_si128((__m128i *)pDest, res0);
+            pDest += lookup0->dstStep;
+            _mm_storeu_si128((__m128i *)pDest, res1);
+            pDest += lookup1->dstStep;
+        }
+
+        //save new addresses
+        ptrSource += 8 * InputType;
+        ptrDest = pDest;
+        return true;
+    }
+};

package/deps/simdutf/benchmarks/competition/utf8lut/src/core/ProcessTrivial.h

@@ -0,0 +1,108 @@
+#pragma once
+
+#include <stdint.h>
+#include "../base/PerfDefs.h"
+
+namespace DfaUtf8 {
+    #include "Dfa.h"
+}
+
+template<int OutputType>
+FORCEINLINE bool DecodeTrivial(const char *&pSource, const char *pEnd, char *&pDest) {
+    static_assert(OutputType == 2 || OutputType == 4, "Only 2-byte and 4-byte output supported");
+    using namespace DfaUtf8;
+    assert(pSource <= pEnd);
+
+    const uint8_t *RESTRICT s = (const uint8_t *)pSource;
+    uint16_t *RESTRICT d = (uint16_t *)pDest;
+    uint32_t codepoint;
+    uint32_t state = 0;
+
+    const uint8_t *ans_s = s;
+    uint16_t *ans_d = d;
+
+    while (s < (const uint8_t *)pEnd) {
+        if (decode(&state, &codepoint, *s++))
+            continue;
+        if (OutputType == 2) {
+            if (codepoint > 0xFFFFU) {
+                *d++ = (uint16_t)(0xD7C0U + (codepoint >> 10));
+                *d++ = (uint16_t)(0xDC00U + (codepoint & 0x3FFU));
+            } else {
+                *d++ = (uint16_t)codepoint;
+            }
+        }
+        else {
+            *(uint32_t *)d = codepoint;
+            d += 2;
+        }
+        if (state == UTF8_ACCEPT) {
+            ans_s = s;
+            ans_d = d;
+        }
+    }
+
+    pSource = (const char *)ans_s;
+    pDest = (char *)ans_d;
+    return state != UTF8_REJECT;
+}
+
+template<int InputType>
+FORCEINLINE bool EncodeTrivial(const char *&pSource, const char *pEnd, char *&pDest) {
+    static_assert(InputType == 2 || InputType == 4, "Only 2-byte and 4-byte input supported");
+    assert(pSource <= pEnd);
+
+    const uint16_t *RESTRICT s = (const uint16_t *)pSource;
+    uint8_t *RESTRICT d = (uint8_t *)pDest;
+    const uint16_t *end = s + (pEnd - pSource) / InputType * (InputType / 2);
+
+    while (s < end) {
+        uint32_t codepoint;
+
+        if (InputType == 2) {
+            codepoint = *s++;
+
+            if (codepoint - 0xD800U < 0x0800U) {
+                if (s < end) {
+                    uint32_t tail = *s++;
+                    if ((codepoint & 0xFC00U) != 0xD800U || (tail & 0xFC00U) != 0xDC00U)
+                        return false;
+                    codepoint = 0x10000U + ((codepoint & 0x03FFU) << 10) + (tail & 0x03FFU);
+                }
+                else {
+                    s--;
+                    break;
+                }
+            }
+        }
+        else {
+            codepoint = *(const uint32_t*)s;
+            s += 2;
+            if (codepoint > 0x10FFFFU || codepoint - 0xD800U < 0x0800U)
+                return false;
+        }
+
+        if (codepoint <= 0x7FU)
+            *d++ = uint8_t(codepoint);
+        else if (codepoint <= 0x7FFU) {
+            *d++ = uint8_t(0xC0U + (codepoint >> 6));
+            *d++ = uint8_t(0x80U + (codepoint & 0x3FU));
+        }
+        else if (codepoint <= 0xFFFFU) {
+            *d++ = uint8_t(0xE0U + (codepoint >> 12));
+            *d++ = uint8_t(0x80U + ((codepoint >> 6) & 0x3FU));
+            *d++ = uint8_t(0x80U + (codepoint & 0x3FU));
+        }
+        else {
+            *d++ = uint8_t(0xF0U + (codepoint >> 18));
+            *d++ = uint8_t(0x80U + ((codepoint >> 12) & 0x3FU));
+            *d++ = uint8_t(0x80U + ((codepoint >> 6) & 0x3FU));
+            *d++ = uint8_t(0x80U + (codepoint & 0x3FU));
+        }
+
+        pSource = (const char *)s;
+        pDest = (char *)d;
+    }
+
+    return true;
+}

package/deps/simdutf/benchmarks/competition/utf8lut/src/iconv/iconv.cpp

@@ -0,0 +1,139 @@
+#include "iconv/iconv.h"
+#include "buffer/BaseBufferProcessor.h"
+#include "buffer/ProcessorPlugins.h"
+#include "buffer/BufferDecoder.h"
+#include "buffer/BufferEncoder.h"
+#include <assert.h>
+
+size_t iconv(iconv_t cd, const char **inbuf, size_t *inbytesleft, char **outbuf, size_t *outbytesleft) {
+    if (cd == (iconv_t)0 || cd == (iconv_t)-1) {
+        errno = EBADF;
+        return (size_t)-1;
+    }
+    BaseBufferProcessor *processor = (BaseBufferProcessor*)cd;
+
+    if (!inbuf || !*inbuf) {
+        //no state carried between calls
+        return 0;
+    }
+    else if (!inbytesleft || !outbytesleft) {
+        //error: input / output size not specified
+        errno = EBADF;
+        return (size_t)-1;
+    }
+    else if (!outbuf || !*outbuf) {
+        //extension: return estimated size of output buffer
+        //which is surely enough to hold all the converted data
+        *outbytesleft = processor->GetOutputBufferMinSize(*inbytesleft);
+        return 0;
+    }
+
+    processor->Clear();
+    ContiguousInput input(*processor, *inbuf, *inbytesleft);
+    ContiguousOutput output(*processor, *outbuf, *outbytesleft);
+
+    bool overflow = false;
+    bool ok;
+    while (!input.Finished()) {
+        //check if we surely have space for next block converted
+        int inputChunk = input.GetNextChunkSize();
+        int maxOutputNeeded = int(output.GetMaxOutputSize(*processor, inputChunk));
+        long long freeOutputSpace = output.GetBufferSize() - output.GetFilledOutputSize();
+        if (maxOutputNeeded > freeOutputSpace) {
+            //note: a lot of input data may be left unconverted (up to chunk size)
+            overflow = true;
+            break;
+        }
+        //convert a block
+        ok = processor->Process();
+        if (!ok)
+            break;
+    }
+
+    long long remains = input.GetRemainingDataSize();
+    long long converted = output.GetFilledOutputSize();
+    *inbuf += *inbytesleft - remains;
+    *inbytesleft = remains;
+    *outbuf += converted;
+    *outbytesleft -= converted;
+
+    if (overflow) {
+        errno = E2BIG;
+        return (size_t)-1;
+    }
+    else if (ok) {
+        if (remains == 0) {
+            //major difference: return 1 instead of number of characters converted
+            return 1;
+        }
+        else {
+            errno = EINVAL;
+            return (size_t)-1;
+        }
+    }
+    else {
+        errno = EILSEQ;
+        return (size_t)-1;
+    }
+}
+
+int iconv_close(iconv_t cd) {
+    if (cd == (iconv_t)0 || cd == (iconv_t)-1) {
+        errno = EBADF;
+        return -1;
+    }
+    BaseBufferProcessor *processor = (BaseBufferProcessor*)cd;
+
+    delete processor;
+    return 0;
+}
+
+
+
+#include "buffer/BufferDecoder.h"
+#include "buffer/BufferEncoder.h"
+
+enum Format {
+    Unsupported = 0,
+    Utf8,
+    Utf16,
+    Utf32
+};
+Format ParseFormat(const char *str) {
+    if (strcmp(str, "UTF-8") == 0)
+        return Utf8;
+    if (strcmp(str, "UTF-16LE") == 0)
+        return Utf16;
+    if (strcmp(str, "UTF-32LE") == 0)
+        return Utf32;
+    return Unsupported;
+}
+
+iconv_t iconv_open(const char *tocode, const char *fromcode) {
+    Format dstFormat = ParseFormat(tocode);
+    Format srcFormat = ParseFormat(fromcode);
+    if (srcFormat == Unsupported || dstFormat == Unsupported || (dstFormat != Utf8 && srcFormat != Utf8) || dstFormat == srcFormat) {
+        errno = EINVAL;
+        return (iconv_t)-1;
+    }
+
+#ifdef ICONV_SMALL
+    static const int Multiplier = 1;    //compact version for small buffers
+#else
+    static const int Multiplier = 4;    //fastest for large buffers
+#endif
+
+    if (0);
+    else if (dstFormat == Utf8 && srcFormat == Utf16)
+        return new BufferEncoder<3, 2, emValidate, Multiplier>();
+    else if (dstFormat == Utf8 && srcFormat == Utf32)
+        return new BufferEncoder<3, 4, emValidate, Multiplier>();
+    else if (dstFormat == Utf16 && srcFormat == Utf8)
+        return new BufferDecoder<3, 2, dmValidate, Multiplier>();
+    else if (dstFormat == Utf32 && srcFormat == Utf8)
+        return new BufferDecoder<3, 4, dmValidate, Multiplier>();
+    else {
+        assert(0);
+        return 0;
+    }
+}