react-native-quick-crypto 1.0.19 → 1.1.0

package/deps/simdutf/benchmarks/competition/utf8lut/src/buffer/ProcessorPlugins.h

@@ -0,0 +1,334 @@
+#pragma once
+
+#include "../buffer/BaseBufferProcessor.h"
+
+// Base class for all plugins (they can be attached to BaseBufferProcessor).
+class BasePlugin {
+public:
+    virtual ~BasePlugin() {}
+    //callback called before conversion inside 'Process' method
+    virtual void Pre() {}
+    //callback called after conversion inside 'Process' method
+    virtual void Post() {}
+};
+
+//plugins intended to simplify setting input data
+class InputPlugin : public BasePlugin {};
+//plugins intended to simplify setting output data
+class OutputPlugin : public BasePlugin {};
+//Note that you can independently choose one input plugin and one output plugin
+
+
+
+// A plugin which helps to convert a given contiguous byte array in memory.
+// It maintains a pointer to the beginning of yet unconverted data.
+// Before each 'Process' call, it installs a chunk (starting at the pointer) as input buffer.
+// So, contiguous calls to 'Process' would result in chunk-by-chunk conversion of the byte array.
+//
+//                     (chunk)
+//          +----------|------|------------+
+//          |..........|%%%%%%|%%%%%%%%%%%%|    (before)
+//          +----------|------|------------+      ||
+//                     >-->-->|                   || one 'Process' call
+//          +-----------------|------------+      \/
+//          |.................|%%%%%%%%%%%%|    (after)
+//          +-----------------|------------+
+//     
+class ContiguousInput : public InputPlugin {
+    BaseBufferProcessor *processor;
+    const char *srcBuffer;
+    long long srcSize;
+    long long srcDone;
+    int chunkSize;
+    bool finished;
+
+public:
+    //takes pointer and size of contiguous input buffer to be converted
+    ContiguousInput(BaseBufferProcessor &owner, const char *buffer, long long size) : processor(&owner) {
+        srcBuffer = buffer;
+        srcSize = size;
+        srcDone = 0;
+        chunkSize = processor->GetInputBufferRecommendedSize();
+        finished = (srcSize == 0);
+        processor->AddPlugin(*this);
+    }
+    virtual void Pre() {
+        assert(!finished);
+        int bytes = GetNextChunkSize();
+        processor->SetInputBuffer(srcBuffer + srcDone, bytes);
+        finished = (srcDone + bytes == srcSize);
+        processor->SetHint(finished);
+    }
+    virtual void Post() {
+        srcDone += processor->GetInputDoneSize();
+    }
+    //returns size of already processed prefix (i.e. converted successfully)
+    long long GetProcessedInputSize() const {
+        return srcDone;
+    }
+    //returns size of the remaining suffix (i.e. not converted yet)
+    long long GetRemainingDataSize() const {
+        return srcSize - srcDone;
+    }
+    //returns the size of the whole input buffer (as given by user)
+    long long GetBufferSize() const {
+        return srcSize;
+    }
+    //returns the size of the chunk which would be processed during the next 'Process' call.
+    int GetNextChunkSize() const {
+        long long bytes = srcSize - srcDone;
+        if (bytes > chunkSize)
+            bytes = chunkSize;
+        return int(bytes);
+    }
+    //returns true iff the conversion is finished
+    //note that some bytes may still remain unprocessed (i.e. incomplete code point)
+    bool Finished() const {
+        return finished;
+    }
+};
+
+
+// A plugin which helps to interactively put input data into internal buffer.
+// It owns a dynamically allocated buffer where the user puts his input data.
+// During each 'Process' call, all the contents of the buffer are processed.
+// Unconverted data (i.e. incomplete code point) at the end of the buffer is
+// automatically moved to the beginning of the buffer.
+//
+//          +-|-----------------------+
+//          |a|.......................|    (before)
+//          +-|-----------------------+      ||
+//             (new data added)              || user adds new data "b-r"
+//          +------------------|------+      ||
+//          |abcdefghijklmnopqr|......|   ---++
+//          +------------------|------+      ||
+//          >-->-->-->-->-->|                || 'Process' call
+//          +---------------|--|------+      ||
+//          |...............|qr|......|   ---++
+//          +---------------|--|------+      ||
+//                                           || unprocessed data "qr" moved
+//          +--|----------------------+      \/
+//          |qr|......................|    (after)
+//          +--|----------------------+
+//
+// The following sample shows how to append new input data into buffer (from file f):
+//      input.GetBuffer(ptr, size);
+//      int added = fread(ptr, 1, size, f);
+//      input.ConfirmInputBytes(added);
+//
+class InteractiveInput : public InputPlugin {
+    BaseBufferProcessor *processor;
+    char *inputBuffer;
+    int inputSize;
+    int inputSet;
+    bool lastBlock;
+    long long totalSizeDone;
+
+public:
+    InteractiveInput(BaseBufferProcessor &owner) : processor(&owner) {
+        inputSize = processor->GetInputBufferRecommendedSize();
+        inputBuffer = new char[inputSize];
+        inputSet = 0;
+        totalSizeDone = 0;
+        processor->AddPlugin(*this);
+    }
+    ~InteractiveInput() {
+        delete[] inputBuffer;
+    }
+    //returns pointer and size of a subbuffer where new data must be placed
+    //you can write to there K bytes of new input data (for any K <= "size")
+    //then you must call ConfirmInputBytes with "bytes" = K
+    void GetBuffer(char *&buffer, int &size) const {
+        buffer = inputBuffer + inputSet;
+        size = inputSize - inputSet;
+    }
+    //informs that exactly "bytes" bytes of input were appended to the buffer
+    //argument "isLastBlock" would be passed to method SetHint of processor (see docs there)
+    void ConfirmInputBytes(int bytes, bool isLastBlock = true) {
+        inputSet += bytes;
+        lastBlock = isLastBlock;
+    }
+    virtual void Pre() {
+        processor->SetInputBuffer(inputBuffer, inputSet);
+        processor->SetHint(lastBlock);
+    }
+    virtual void Post() {
+        int inputDone = processor->GetInputDoneSize();
+        int remains = inputSet - inputDone;
+        if (remains > 0)
+            memmove(inputBuffer, inputBuffer + inputDone, remains);
+        inputSet = remains;
+        totalSizeDone += inputDone;
+    }
+    //returns how many bytes are currently within the buffer (and are still unprocessed)
+    int GetRemainingDataSize() const {
+        return inputSet;
+    }
+    //returns how many bytes were processed in total during the whole life of the plugin
+    long long GetProcessedInputSize() const {
+        return totalSizeDone;
+    }
+};
+
+
+// A plugin which helps to save result of conversion into a given contiguous byte array in memory.
+// It maintains a pointer to the end of the already saved output data.
+// Before each 'Process' call, it installs the remaining part of the byte array as output buffer.
+// So, contiguous calls to 'Process' would result in saving output in the array chunk-by-chunk.
+//
+//                     (chunk)
+//          +----------|------|------------+
+//          |##########|......|............|    (before)
+//          +----------|------|------------+      ||
+//                     >-->-->|                   || one 'Process' call
+//          +-----------------|------------+      \/
+//          |#################|............|    (after)
+//          +-----------------|------------+
+//
+// If multi-stream processor is used, then conversion results are first saved into internal buffers,
+// and then automatically copied into the user's byte array.
+// Note that this means minor performance loss due to additional memcpy of the output data.
+// You do not need to do anything special to handle this case, no difference is visible from the outside.
+//     
+// Note that the static method ContiguousOutput::GetMaxOutputSize can be used without plugin
+// in order to see which size of output buffer is needed to surely accomodate result of conversion.
+//
+class ContiguousOutput : public OutputPlugin {
+    BaseBufferProcessor *processor;
+    char *dstBuffer;
+    long long dstSize;
+    long long dstDone;
+    int maxSize;
+
+    int streamsCnt, streamOutputSize;
+    char *multiBuffer[BaseBufferProcessor::MaxStreamsCount];
+
+public:
+    //returns minimal size of contiguous output buffer, such that for any input data of size "inputSize":
+    //  1. output buffer would be large enough to hold converted result
+    //  2. during conversion, there would always be enough empty space remaining in the buffer
+    //     (i.e. lower limit returned by GetOutputBufferMinSize() is not violated)
+    static long long GetMaxOutputSize(const BaseBufferProcessor &processor, long long inputSize) {
+        return processor.GetStreamsCount() * processor.GetOutputBufferMinSize(inputSize);
+    }
+    //takes pointer and size of contiguous output buffer where the result of conversion must be saved
+    ContiguousOutput(BaseBufferProcessor &owner, char *buffer, long long size) : processor(&owner) {
+        dstBuffer = buffer;
+        dstSize = size;
+        dstDone = 0;
+        streamsCnt = processor->GetStreamsCount();
+        streamOutputSize = 0;
+        maxSize = processor->GetBufferMaxSize();
+        if (streamsCnt > 1) {
+            streamOutputSize = (int)processor->GetOutputBufferMinSize(processor->GetInputBufferRecommendedSize());
+            for (int i = 0; i < streamsCnt; i++)
+                multiBuffer[i] = new char[streamOutputSize];
+        }
+        processor->AddPlugin(*this);
+    }
+    ~ContiguousOutput() {
+        if (streamsCnt > 1)
+            for (int i = 0; i < streamsCnt; i++)
+                delete[] multiBuffer[i];
+    }
+    virtual void Pre() {
+        if (streamsCnt > 1) {
+            for (int i = 0; i < streamsCnt; i++)
+                processor->SetOutputBuffer(multiBuffer[i], streamOutputSize, i);
+        }
+        else {
+            long long remains = dstSize - dstDone;
+            if (remains > maxSize)
+                remains = maxSize;
+            processor->SetOutputBuffer(dstBuffer + dstDone, int(remains));
+        }
+    }
+    virtual void Post() {
+        if (streamsCnt > 1) {
+            for (int i = 0; i < streamsCnt; i++) {
+                int done = processor->GetOutputDoneSize(i);
+                assert(dstDone + done <= dstSize);
+                memcpy(dstBuffer + dstDone, multiBuffer[i], done);
+                dstDone += done;
+            }
+        }
+        else
+            dstDone += processor->GetOutputDoneSize();
+    }
+    //returns how many bytes of output data already saved (in the prefix of the buffer)
+    long long GetFilledOutputSize() const {
+        return dstDone;
+    }
+    //returns the size of the whole output buffer (as given by user)
+    long long GetBufferSize() const {
+        return dstSize;
+    }
+};
+
+
+// A plugin which helps to interactively consume conversion results from internal buffer(s).
+// It owns dynamically allocated buffer (or buffers) which receive output data.
+// During each 'Process' call, the conversion results are stored into this buffer(s).
+// User must read all the contents from this buffer(s) prior to next 'Process' call.
+//
+//         single-stream case                                   multi(4)-stream case
+//    +-------------------------+                        +-----+ +-----+ +-----+ +-----+
+//    |.........................|         (before)       |.....| |.....| |.....| |.....|
+//    +-------------------------+            ||          +-----+ +-----+ +-----+ +-----+
+//    >-->-->-->-->-->-->|          'Process'|| call     >-->|   >--->|  >->|    >-->|
+//    +------------------|------+            \/          +---|-+ +----|+ +--|--+ +---|-+
+//    |##################|......|         (after)        |###|.| |####|| |##|..| |###|.|
+//    +------------------|------+                        +---|-+ +----|+ +--|--+ +---|-+
+//        (output data)                                     \       |      |       /
+//     (to be ovewritten)                                (output data, to be ovewritten)
+//
+// The following sample shows how to receive conversion results from buffers (and dump them to file f):
+//        for (int k = 0; k < output.GetStreamsCount(); k++) {
+//            output.GetBuffer(ptr, size, k);
+//            fwrite(ptr, 1, size, f);
+//        }
+//
+class InteractiveOutput : public OutputPlugin {
+    BaseBufferProcessor *processor;
+    int streamsCnt, streamOutputSize;
+    char *multiBuffer[BaseBufferProcessor::MaxStreamsCount];
+    long long totalSizeDone;
+
+public:
+    InteractiveOutput(BaseBufferProcessor &owner) : processor(&owner) {
+        streamsCnt = processor->GetStreamsCount();
+        streamOutputSize = (int)processor->GetOutputBufferMinSize(processor->GetInputBufferRecommendedSize());
+        for (int i = 0; i < streamsCnt; i++)
+            multiBuffer[i] = new char[streamOutputSize];
+        totalSizeDone = 0;
+        processor->AddPlugin(*this);
+    }
+    ~InteractiveOutput() {
+        for (int i = 0; i < streamsCnt; i++)
+            delete[] multiBuffer[i];
+    }
+    virtual void Pre() {
+        for (int i = 0; i < streamsCnt; i++)
+            processor->SetOutputBuffer(multiBuffer[i], streamOutputSize, i);
+    }
+    virtual void Post() {
+        for (int i = 0; i < streamsCnt; i++)
+            totalSizeDone += processor->GetOutputDoneSize(i);
+    }
+    //returns the number of output streams = the number of separate output buffers
+    int GetStreamsCount() const {
+        return streamsCnt;
+    }
+    //returns pointer and size of the output data obtained
+    //by "index"-th stream during the last 'Process' call
+    //Note that you have to concatenate outputs of all the streams in their order
+    //to get the real output of the last 'Process' call.
+    void GetBuffer(const char *&buffer, int &size, int index = 0) const {
+        buffer = multiBuffer[index];
+        size = processor->GetOutputDoneSize(index);
+    }
+    //returns total number of bytes written to output buffer(s) during the whole life of the plugin
+    long long GetFilledOutputSize() const {
+        return totalSizeDone;
+    }
+};

package/deps/simdutf/benchmarks/competition/utf8lut/src/buffer/ProcessorSelector.h

@@ -0,0 +1,186 @@
+#pragma once
+
+#include "../buffer/BufferDecoder.h"
+#include "../buffer/BufferEncoder.h"
+
+//(metaprogramming helper equivalent to std::conditional)
+template<bool Condition, class First, class Second> struct TernaryOperator {};
+template<class First, class Second> struct TernaryOperator<true, First, Second> { typedef First Type; };
+template<class First, class Second> struct TernaryOperator<false, First, Second> { typedef Second Type; };
+
+//possible formats of input/output data
+enum DataFormat {
+    dfUtf8,         // UTF-8
+    dfUtf16,        // UTF-16LE
+    dfUtf32,        // UTF-32LE
+    dfUtfCount      //(helper)
+};
+
+
+//modes of conversion (synchronized with EncoderMode and DecoderMode)
+//see full explanations in ProcessorSelector description
+enum ConversionMode {
+    cmFast = 0,     // only fast path: converts limited set of code points
+    cmFull = 1,     // fast + slow path: converts any correct input
+    cmValidate = 2  // any input allowed (includes validation)
+};
+
+// This selector can be used to get type of processor by options, e.g.:
+//   typedef ProcessorSelector<dfUtf8, dfUtf16>::WithOptions<cmValidate, 2>::Processor MyProcessor;
+//   BaseBufferProcessor *processor = new MyProcessor();
+//   ... now you can pass this processor e.g. to functions from MessageConverter.h
+//
+// For each choice of template arguments, a separately optimized code is generated by the compiler.
+// In order to avoid template bloat, you can choose the set of all configurations you might need,
+// and write a factory function for them (similar to GenerateProcessor function in AllProcessors.cpp).
+//
+// The conversion itself is defined by two arguments:
+//   SrcFormat: defines how the input data is encoded
+//   DstFormat: defines desired encoding of the output data
+// Both of these arguments are chosen from DataFormat enum (must be dfUtf8, dfUtf16, or dfUtf32).
+//
+// The other arguments specify supported inputs and performance tradeoffs.
+// The processor usually contains two code paths: the FAST path and the SLOW path.
+// The fast path works only on a limited set of code points, while the slow path can process any data.
+// To maintain high overall performance, fast path should be able to process more than 99% of input code points.
+//
+// Fast path supports a code point if its representation in UTF-8 consists of at most MaxBytes bytes.
+// Therefore, fast path supports the following set of characters (depending on MaxBytes argument):
+//   MaxBytes = 1:  U+0000 .. U+007F            all ASCII characters
+//   MaxBytes = 2:  U+0000 .. U+07FF            ASCII + latin extensions, cyrillic, and much more
+//   MaxBytes = 3:  U+0000 .. U+FFFF            the whole Basic Multilingual Plane
+//     (except surrogate code points U+D800 .. U+DFFF)
+// 
+// The Mode arguments controls what happens when code points unsupported by fast path are met:
+//   Mode == cmFast:
+//     Only fast path is included, no slow path at all.
+//     The input data must be correctly encoded, and all code points must be within supported range of fast path.
+//     Beware: incorrect encoding or unsupported code point in input results in Undefined Behavior.
+//   Mode == cmFull:
+//     Slow path is included to support the whole set of Unicode code points, but NO validation is done in fast path.
+//     The input data must be correctly encoded, and may contain all valid code points.
+//     Beware: incorrect encoding of the input results in Undefined Behavior.
+//   Mode == cmValidate:
+//     Slow path is included to support all code points, fast path includes full validation of the input.
+//     The input data may be an arbitrary byte array, the processor should always behave properly.
+//     In case of encoding error, the processor stops by default (error correction is discussed below).
+// NOTE: If the input comes from untrusted source, then using cmFast and cmFull modes is a serious security issue!
+// Prefer cmValidate mode in general.
+// 
+// The SpeedMult argument is a small tweak which might improve performance on modern machines.
+//   SpeedMult = 1:
+//     Process input sequentally, reading a 16-byte chunk on each iteration.
+//   SpeedMult = 4:
+//     from UTF-8: Use 4 interleaved streams of processing, thus emulating Hyper-threading in a single thread.
+//      to  UTF-8: Use 4x loop unrolling in the innermost loop.
+// Note that using multi-stream processor is a bit more complicated, e.g. because it has four output buffers instead of one.
+// Also, it does not support error correction.
+//
+// If Mode == cmValidate, then it is possible to enable error correction callback (using SetErrorCallback method).
+// This callback is called when it becomes impossible to process the next code point.
+// It may adjust input/output data and pointers and ask to continue processing, or confirm that conversion must be stopped.
+// Two ready-to-use callbacks are included: skipping code units or replacing them with 0xFFFD replacement code point.
+// Note that error correction works even slower than the slow path, and it won't work in cmFast and cmFull modes.
+//
+// Here is an example of enabling the 'skip-code-unit' error correction:
+//   typedef ProcessorSelector<dfUtf8, dfUtf16> ConversionDirection;
+//   typedef ConversionDirection::WithOptions<cmValidate, 2>::Processor MyProcessor;
+//   BaseBufferProcessor *processor = new MyProcessor();
+//   int errorsCount = 0;       //number of errors maintained here
+//   processor->SetErrorCallback(&ConversionDirection::OnErrorMissCodeUnits, &errorsCount);
+// Alternatively, you can create processor with 'replace-with-0xfffd' error correction like this:
+//   typedef ProcessorSelector<dfUtf8, dfUtf16>::WithOptions<cmValidate, 2>::Processor MyProcessor;
+//   int errorsCount = 0;
+//   BaseBufferProcessor *processor = MyProcessor::Create(&errorsCount);
+//
+template<int SrcFormat, int DstFormat>
+struct ProcessorSelector {
+    static_assert(SrcFormat >= 0 && SrcFormat < dfUtfCount, "Unsupported format");
+    static_assert(DstFormat >= 0 && DstFormat < dfUtfCount, "Unsupported format");
+    static_assert((SrcFormat == dfUtf8) != (DstFormat == dfUtf8), "Supported only conversions: from UTF-8 or to UTF-8");
+
+    template<int Mode = cmValidate, int MaxBytes = 3, int SpeedMult = 1>
+    struct WithOptions {
+        // Type of corresponding processor, can be created as "new ...::Processor();"
+        typedef typename TernaryOperator<SrcFormat == dfUtf8,
+            BufferDecoder<MaxBytes, (DstFormat == dfUtf32 ? 4 : 2), Mode, SpeedMult>,
+            BufferEncoder<MaxBytes, (SrcFormat == dfUtf32 ? 4 : 2), Mode, SpeedMult>
+        >::Type Processor;
+
+        // A helper for creating processor with error correction or without it.
+        // In any case: a new processor is created and returned.
+        // If errorCounter is not null, then error correction is additionally enabled:
+        // OnErrorSetReplacementChars callback is installed, and number of fixed errors is maintained in *errorCounter.
+        static Processor* Create(int *errorCounter = 0);
+    };
+
+    // Some ready-to-use callbacks which can be installed for error correction (use BaseBufferProcessor::SetErrorCallback).
+    //   code units are simply skipped until conversion can be continued
+    static bool OnErrorMissCodeUnits(void *context, const char *&srcBuffer, int srcBytes, char *&dstBuffer, int dstBytes);
+    //   code units are replaced with 0xFFFD code points until conversion can be continued
+    static bool OnErrorSetReplacementChars(void *context, const char *&srcBuffer, int srcBytes, char *&dstBuffer, int dstBytes);
+};
+
+
+
+//==============================
+// implementation of some stuff
+//==============================
+
+FORCEINLINE int GetUnitSizeOfFormat(int format) { return format == dfUtf8 ? 1 : format == dfUtf16 ? 2 : 4; }
+
+template<int SrcFormat, int DstFormat>
+bool ProcessorSelector<SrcFormat, DstFormat>::OnErrorMissCodeUnits(
+    void *context, const char *&srcBuffer, int srcBytes, char *&dstBuffer, int dstBytes
+) {
+    (void)dstBytes; // otherwise it is unused
+    (void)dstBuffer; // otherwise it is unused
+    (void)srcBytes; // otherwise it is unused 
+    srcBuffer += GetUnitSizeOfFormat(SrcFormat);
+    if (context) {
+        int &counter = *(int*)context;
+        counter++;
+    }
+    return true;
+}
+
+template<int SrcFormat, int DstFormat>
+bool ProcessorSelector<SrcFormat, DstFormat>::OnErrorSetReplacementChars(
+    void *context, const char *&srcBuffer, int srcBytes, char *&dstBuffer, int dstBytes
+) {
+    (void)srcBytes;
+    if (DstFormat == dfUtf8) {
+        if (dstBytes < 3) return false;
+        *dstBuffer++ = (char)0xEF;
+        *dstBuffer++ = (char)0xBF;
+        *dstBuffer++ = (char)0xBD;
+    }
+    else if (DstFormat == dfUtf16) {
+        if (dstBytes < 2) return false;
+        *dstBuffer++ = (char)0xFD;
+        *dstBuffer++ = (char)0xFF;
+    }
+    else if (DstFormat == dfUtf32) {
+        if (dstBytes < 4) return false;
+        *dstBuffer++ = (char)0xFD;
+        *dstBuffer++ = (char)0xFF;
+        *dstBuffer++ = (char)0x00;
+        *dstBuffer++ = (char)0x00;
+    }
+    srcBuffer += GetUnitSizeOfFormat(SrcFormat);
+    if (context) {
+        int &counter = *(int*)context;
+        counter++;
+    }
+    return true;
+}
+
+template<int SrcFormat, int DstFormat>
+template<int Mode, int MaxBytes, int SpeedMult>
+typename ProcessorSelector<SrcFormat, DstFormat>::template WithOptions<Mode, MaxBytes, SpeedMult>::Processor *
+ProcessorSelector<SrcFormat, DstFormat>::WithOptions<Mode, MaxBytes, SpeedMult>::Create(int *errorCounter) {
+    Processor *result = new Processor();
+    if (errorCounter)
+        result->SetErrorCallback(OnErrorSetReplacementChars, errorCounter);
+    return result;
+}

package/deps/simdutf/benchmarks/competition/utf8lut/src/core/DecoderLut.cpp

@@ -0,0 +1,140 @@
+#include "../core/DecoderLut.h"
+#include <assert.h>
+#include <string.h>
+
+static inline void SetEntry(DecoderLutEntry<false> &entry,
+    const __m128i &shufAB, const __m128i &shufC, int srcStep, int dstStep,
+    const __m128i &/*headerMask*/, const __m128i &/*minValues*/
+) {
+    entry.shufAB = shufAB;
+    entry.shufC = shufC;
+    entry.s.srcStep = srcStep;
+    entry.s.dstStep = dstStep;
+}
+static inline void SetEntry(DecoderLutEntry<true> &entry,
+    const __m128i &shufAB, const __m128i &shufC, int srcStep, int dstStep,
+    const __m128i &headerMask, const __m128i &minValues
+) {
+    entry.shufAB = shufAB;
+    entry.shufC = shufC;
+    entry.s.
+    srcStep = srcStep;
+    entry.s.dstStep = dstStep;
+    entry.headerMask = headerMask;
+    entry.minValues = minValues;
+}
+
+
+template<bool Validate> void DecoderLutTable<Validate>::ComputeAll() {
+    //fill tables with empty values for BAD masks
+    DecoderLutEntry<Validate> empty;
+    SetEntry(empty,
+        _mm_set1_epi8(-1),      //produce zero symbols
+        _mm_set1_epi8(-1),      //produce zero symbols
+        16,                     //skip the whole 16-byte block on error
+        0,                      //do not move output pointer
+        _mm_set1_epi8(-1),      //forbid any bytes except 11111110
+        _mm_set1_epi16(0x7FFF)  //forbid almost all output codes
+    );
+    for (int i = 0; i < 32768; i++)
+        data[i] = empty;
+    //start recursive search for valid masks
+    int sizes[32];
+    ComputeRec(sizes, 0, 0);    //10609 entries total
+}
+
+template<bool Validate> void DecoderLutTable<Validate>::ComputeRec(int *sizes, int num, int total) {
+    if (total >= 16)
+        return ComputeEntry(sizes, num);
+    for (int sz = 1; sz <= 3; sz++) {
+        sizes[num] = sz;
+        ComputeRec(sizes, num + 1, total + sz);
+    }
+}
+
+template<bool Validate> void DecoderLutTable<Validate>::ComputeEntry(const int *sizes, int num) {
+    //find maximal number of chars to decode
+    int cnt = num - 1;
+    int preSum = 0;
+    for (int i = 0; i < cnt; i++)
+        preSum += sizes[i];
+    assert(preSum < 16);
+    //Note: generally, we can process a char only if the next byte is within XMM register
+    //However, if the last char takes 3 bytes and fits the register tightly, we can take it too
+    if (preSum == 13 && preSum + sizes[cnt] == 16)
+        preSum += sizes[cnt++];
+    //still cannot process more that 8 chars per register
+    while (cnt > 8)
+        preSum -= sizes[--cnt];
+
+    //generate bitmask
+    int mask = 0;
+    int pos = 0;
+    for (int i = 0; i < num; i++)
+        for (int j = 0; j < sizes[i]; j++)
+            mask |= (j>0) << (pos++);
+    assert(pos >= 16);
+    mask &= 0xFFFF;
+
+    //generate shuffle masks
+    char shufAB[16], shufC[16];
+    memset(shufAB, -1, sizeof(shufAB));
+    memset(shufC, -1, sizeof(shufC));
+    pos = 0;
+    for (int i = 0; i < cnt; i++) {
+        int sz = sizes[i];
+        for (int j = sz-1; j >= 0; j--) {
+            if (j < 2)
+                shufAB[2 * i + j] = char(pos++);
+            else
+                shufC[i] = char(pos++);
+        }
+    }
+    assert(pos <= 16);
+
+    //generate header masks for validation
+    char headerMask[16];
+    memset(headerMask, 0, sizeof(headerMask));
+    pos = 0;
+    for (int i = 0; i < cnt; i++) {
+        int sz = sizes[i];
+        for (int j = 0; j < sz; j++) {
+            int bits = 2;
+            if (j == 0)
+                bits = (sz == 1 ? 1 : sz == 2 ? 3 : 4);
+            headerMask[(pos++)%16] = -char(1 << (8 - bits));
+        }
+    }
+    assert(pos <= 16);
+
+    //generate min symbols values for validation 
+    int16_t minValues[8];
+    for (int i = 0; i < 8; i++) {
+        int sz = (i < cnt ? sizes[i] : 1);
+        minValues[i] = 0x8000 + (sz == 1 ? 0 : sz == 2 ? (1<<7) : (1<<11));
+    }
+
+    //store info into the lookup table
+    assert(mask % 2 == 0);  mask /= 2;  //note: odd masks are removed
+    SetEntry(data[mask],
+        _mm_loadu_si128((__m128i*)shufAB),
+        _mm_loadu_si128((__m128i*)shufC),
+        preSum,
+        2 * cnt,
+        _mm_loadu_si128((__m128i*)headerMask),
+        _mm_loadu_si128((__m128i*)minValues)
+    );
+}
+
+template<bool Validate> const DecoderLutTable<Validate> *DecoderLutTable<Validate>::CreateInstance() {
+    static DecoderLutTable<Validate> *singletonTable = 0;
+    if (!singletonTable) {
+        singletonTable = (DecoderLutTable<Validate> *)_mm_malloc(sizeof(DecoderLutTable<Validate>), CACHE_LINE);
+        singletonTable->ComputeAll();
+    }
+    return singletonTable;
+}
+
+
+template struct DecoderLutTable<false>;
+template struct DecoderLutTable<true>;