react-native-update 10.34.3 → 10.34.5

package/harmony/pushy/src/main/cpp/lzma/CPP/Common/UTFConvert.cpp

@@ -2,354 +2,94 @@
 
 #include "StdAfx.h"
 
-// #include <stdio.h>
-
 #include "MyTypes.h"
 #include "UTFConvert.h"
 
-
-#ifndef Z7_WCHART_IS_16BIT
-#ifndef __APPLE__
-  // we define it if the system supports files with non-utf8 symbols:
-  #define MY_UTF8_RAW_NON_UTF8_SUPPORTED
-#endif
+#ifdef _WIN32
+#define _WCHART_IS_16BIT 1
 #endif
 
 /*
-  MY_UTF8_START(n) - is a base value for start byte (head), if there are (n) additional bytes after start byte
+  _UTF8_START(n) - is a base value for start byte (head), if there are (n) additional bytes after start byte
   
-  n : MY_UTF8_START(n) : Bits of code point
+  n : _UTF8_START(n) : Bits of code point
 
   0 : 0x80 :    : unused
   1 : 0xC0 : 11 :
   2 : 0xE0 : 16 : Basic Multilingual Plane
   3 : 0xF0 : 21 : Unicode space
-  4 : 0xF8 : 26 :
-  5 : 0xFC : 31 : UCS-4 : wcstombs() in ubuntu is limited to that value
+  3 : 0xF8 : 26 :
+  5 : 0xFC : 31 : UCS-4
   6 : 0xFE : 36 : We can use it, if we want to encode any 32-bit value
   7 : 0xFF :
 */
 
-#define MY_UTF8_START(n) (0x100 - (1 << (7 - (n))))
-
-#define MY_UTF8_HEAD_PARSE2(n) \
-    if (c < MY_UTF8_START((n) + 1)) \
-    { numBytes = (n); val -= MY_UTF8_START(n); }
-
-#ifndef Z7_WCHART_IS_16BIT
-
-/*
-   if (wchar_t is 32-bit), we can support large points in long UTF-8 sequence,
-   when we convert wchar_t strings to UTF-8:
-     (_UTF8_NUM_TAIL_BYTES_MAX == 3) : (21-bits points) - Unicode
-     (_UTF8_NUM_TAIL_BYTES_MAX == 5) : (31-bits points) - UCS-4
-     (_UTF8_NUM_TAIL_BYTES_MAX == 6) : (36-bit hack)
-*/
-
-#define MY_UTF8_NUM_TAIL_BYTES_MAX 5
-#endif
-
-/*
-#define MY_UTF8_HEAD_PARSE \
-    UInt32 val = c; \
-         MY_UTF8_HEAD_PARSE2(1) \
-    else MY_UTF8_HEAD_PARSE2(2) \
-    else MY_UTF8_HEAD_PARSE2(3) \
-    else MY_UTF8_HEAD_PARSE2(4) \
-    else MY_UTF8_HEAD_PARSE2(5) \
-  #if MY_UTF8_NUM_TAIL_BYTES_MAX >= 6
-    else MY_UTF8_HEAD_PARSE2(6)
-  #endif
-*/
-
-#define MY_UTF8_HEAD_PARSE_MAX_3_BYTES \
-    UInt32 val = c; \
-         MY_UTF8_HEAD_PARSE2(1) \
-    else MY_UTF8_HEAD_PARSE2(2) \
-    else { numBytes = 3; val -= MY_UTF8_START(3); }
-
-
-#define MY_UTF8_RANGE(n) (((UInt32)1) << ((n) * 5 + 6))
-
-
-#define START_POINT_FOR_SURROGATE 0x10000
-
-
-/* we use 128 bytes block in 16-bit BMP-PLANE to encode non-UTF-8 Escapes
-   Also we can use additional HIGH-PLANE (we use 21-bit points above 0x1f0000)
-   to simplify internal intermediate conversion in Linux:
-   RAW-UTF-8 <-> internal wchar_t utf-16 strings <-> RAW-UTF-UTF-8
-*/
- 
-
-#if defined(Z7_WCHART_IS_16BIT)
-
-#define UTF_ESCAPE_PLANE 0
-
-#else
-
-/*
-we can place 128 ESCAPE chars to
-   ef 80 -    ee be 80 (3-bytes utf-8) : similar to WSL
-   ef ff -    ee bf bf
-
-1f ef 80 - f7 be be 80 (4-bytes utf-8) : last  4-bytes utf-8 plane (out of Unicode)
-1f ef ff - f7 be bf bf (4-bytes utf-8) : last  4-bytes utf-8 plane (out of Unicode)
-*/
-
-// #define UTF_ESCAPE_PLANE_HIGH  (0x1f << 16)
-// #define UTF_ESCAPE_PLANE        UTF_ESCAPE_PLANE_HIGH
-#define UTF_ESCAPE_PLANE 0
-
-/*
-  if (Z7_UTF_FLAG_FROM_UTF8_USE_ESCAPE is set)
-  {
-    if (UTF_ESCAPE_PLANE is UTF_ESCAPE_PLANE_HIGH)
-    {
-      we can restore any 8-bit Escape from ESCAPE-PLANE-21 plane.
-      But ESCAPE-PLANE-21 point cannot be stored to utf-16 (7z archive)
-      So we still need a way to extract 8-bit Escapes and BMP-Escapes-8
-      from same BMP-Escapes-16 stored in 7z.
-      And if we want to restore any 8-bit from 7z archive,
-      we still must use Z7_UTF_FLAG_FROM_UTF8_BMP_ESCAPE_CONVERT for (utf-8 -> utf-16)
-      Also we need additional Conversions to tranform from utf-16 to utf-16-With-Escapes-21
-    }
-    else (UTF_ESCAPE_PLANE == 0)
-    {
-      we must convert original 3-bytes utf-8 BMP-Escape point to sequence
-      of 3 BMP-Escape-16 points with Z7_UTF_FLAG_FROM_UTF8_BMP_ESCAPE_CONVERT
-      so we can extract original RAW-UTF-8 from UTFD-16 later.
-    }
-  }
-*/
-
-#endif
-
-
-
-#define UTF_ESCAPE_BASE 0xef00
-
-
-#ifdef UTF_ESCAPE_BASE
-#define IS_ESCAPE_POINT(v, plane) (((v) & (UInt32)0xffffff80) == (plane) + UTF_ESCAPE_BASE + 0x80)
-#endif
-
-#define IS_SURROGATE_POINT(v)     (((v) & (UInt32)0xfffff800) == 0xd800)
-#define IS_LOW_SURROGATE_POINT(v) (((v) & (UInt32)0xfffffc00) == 0xdc00)
-
-
-#define UTF_ERROR_UTF8_CHECK \
-  { NonUtf = true; continue; }
-
-void CUtf8Check::Check_Buf(const char *src, size_t size) throw()
-{
-  Clear();
-  // Byte maxByte = 0;
-
-  for (;;)
-  {
-    if (size == 0)
-      break;
-
-    const Byte c = (Byte)(*src++);
-    size--;
-
-    if (c == 0)
-    {
-      ZeroChar = true;
-      continue;
-    }
-
-    /*
-    if (c > maxByte)
-      maxByte = c;
-    */
-
-    if (c < 0x80)
-      continue;
-    
-    if (c < 0xc0 + 2)
-      UTF_ERROR_UTF8_CHECK
-
-    unsigned numBytes;
-    UInt32 val = c;
-         MY_UTF8_HEAD_PARSE2(1)
-    else MY_UTF8_HEAD_PARSE2(2)
-    else MY_UTF8_HEAD_PARSE2(3)
-    else MY_UTF8_HEAD_PARSE2(4)
-    else MY_UTF8_HEAD_PARSE2(5)
-    else
-    {
-      UTF_ERROR_UTF8_CHECK
-    }
-
-    unsigned pos = 0;
-    do
-    {
-      if (pos == size)
-        break;
-      unsigned c2 = (Byte)src[pos];
-      c2 -= 0x80;
-      if (c2 >= 0x40)
-        break;
-      val <<= 6;
-      val |= c2;
-      if (pos == 0)
-        if (val < (((unsigned)1 << 7) >> numBytes))
-          break;
-      pos++;
-    }
-    while (--numBytes);
-
-    if (numBytes != 0)
-    {
-      if (pos == size)
-        Truncated = true;
-      else
-        UTF_ERROR_UTF8_CHECK
-    }
+#define _UTF8_START(n) (0x100 - (1 << (7 - (n))))
 
-    #ifdef UTF_ESCAPE_BASE
-      if (IS_ESCAPE_POINT(val, 0))
-        Escape = true;
-    #endif
+#define _UTF8_HEAD_PARSE2(n) if (c < _UTF8_START((n) + 1)) { numBytes = (n); c -= _UTF8_START(n); }
 
-    if (MaxHighPoint < val)
-      MaxHighPoint = val;
+#define _UTF8_HEAD_PARSE \
+         _UTF8_HEAD_PARSE2(1) \
+    else _UTF8_HEAD_PARSE2(2) \
+    else _UTF8_HEAD_PARSE2(3) \
+    else _UTF8_HEAD_PARSE2(4) \
+    else _UTF8_HEAD_PARSE2(5) \
 
-    if (IS_SURROGATE_POINT(val))
-      SingleSurrogate = true;
-    
-    src += pos;
-    size -= pos;
-  }
+    // else _UTF8_HEAD_PARSE2(6)
 
-  // MaxByte = maxByte;
-}
-
-bool Check_UTF8_Buf(const char *src, size_t size, bool allowReduced) throw()
-{
-  CUtf8Check check;
-  check.Check_Buf(src, size);
-  return check.IsOK(allowReduced);
-}
-
-/*
-bool CheckUTF8_chars(const char *src, bool allowReduced) throw()
-{
-  CUtf8Check check;
-  check.CheckBuf(src, strlen(src));
-  return check.IsOK(allowReduced);
-}
-*/
-
-bool CheckUTF8_AString(const AString &s) throw()
-{
-  CUtf8Check check;
-  check.Check_AString(s);
-  return check.IsOK();
-}
-
-
-/*
 bool CheckUTF8(const char *src, bool allowReduced) throw()
 {
-  // return Check_UTF8_Buf(src, strlen(src), allowReduced);
-
   for (;;)
   {
-    const Byte c = (Byte)(*src++);
+    Byte c = *src++;
     if (c == 0)
       return true;
 
     if (c < 0x80)
       continue;
-    if (c < 0xC0 + 2 || c >= 0xf5)
+    if (c < 0xC0)   // (c < 0xC0 + 2) // if we support only optimal encoding chars
       return false;
     
     unsigned numBytes;
-    MY_UTF8_HEAD_PARSE
+    _UTF8_HEAD_PARSE
     else
       return false;
-
-    unsigned pos = 0;
     
+    UInt32 val = c;
+
     do
     {
-      Byte c2 = (Byte)(*src++);
+      Byte c2 = *src++;
       if (c2 < 0x80 || c2 >= 0xC0)
         return allowReduced && c2 == 0;
       val <<= 6;
       val |= (c2 - 0x80);
-      pos++;
     }
     while (--numBytes);
-
-    if (val < MY_UTF8_RANGE(pos - 1))
-      return false;
-
+    
     if (val >= 0x110000)
       return false;
   }
 }
-*/
-
-// in case of UTF-8 error we have two ways:
-// 21.01- : old : 0xfffd: REPLACEMENT CHARACTER : old version
-// 21.02+ : new : 0xef00 + (c) : similar to WSL scheme for low symbols
-
-#define UTF_REPLACEMENT_CHAR  0xfffd
-
-
-
-#define UTF_ESCAPE(c) \
-   ((flags & Z7_UTF_FLAG_FROM_UTF8_USE_ESCAPE) ? \
-    UTF_ESCAPE_PLANE + UTF_ESCAPE_BASE + (c) : UTF_REPLACEMENT_CHAR)
-
-/*
-#define UTF_HARD_ERROR_UTF8
-  { if (dest) dest[destPos] = (wchar_t)UTF_ESCAPE(c); \
-    destPos++; ok = false; continue; }
-*/
-
-// we ignore utf errors, and don't change (ok) variable!
-
-#define UTF_ERROR_UTF8 \
-  { if (dest) dest[destPos] = (wchar_t)UTF_ESCAPE(c); \
-    destPos++; continue; }
-
-// we store UTF-16 in wchar_t strings. So we use surrogates for big unicode points:
 
-// for debug puposes only we can store UTF-32 in wchar_t:
-// #define START_POINT_FOR_SURROGATE ((UInt32)0 - 1)
 
+#define _ERROR_UTF8 \
+  { if (dest) dest[destPos] = (wchar_t)0xFFFD; destPos++; ok = false; continue; }
 
-/*
-  WIN32 MultiByteToWideChar(CP_UTF8) emits 0xfffd point, if utf-8 error was found.
-  Ant it can emit single 0xfffd from 2 src bytes.
-  It doesn't emit single 0xfffd from 3-4 src bytes.
-  We can
-    1) emit Escape point for each incorrect byte. So we can data recover later
-    2) emit 0xfffd for each incorrect byte.
-       That scheme is similar to Escape scheme, but we emit 0xfffd
-       instead of each Escape point.
-    3) emit single 0xfffd from 1-2 incorrect bytes, as WIN32 MultiByteToWideChar scheme
-*/
-
-static bool Utf8_To_Utf16(wchar_t *dest, size_t *destLen, const char *src, const char *srcLim, unsigned flags) throw()
+static bool Utf8_To_Utf16(wchar_t *dest, size_t *destLen, const char *src, const char *srcLim) throw()
 {
   size_t destPos = 0;
   bool ok = true;
 
   for (;;)
   {
+    Byte c;
     if (src == srcLim)
     {
       *destLen = destPos;
       return ok;
     }
-    
-    const Byte c = (Byte)(*src++);
+    c = *src++;
 
     if (c < 0x80)
     {
@@ -358,195 +98,113 @@ static bool Utf8_To_Utf16(wchar_t *dest, size_t *destLen, const char *src, const
       destPos++;
       continue;
     }
-    
-    if (c < 0xc0 + 2
-      || c >= 0xf5) // it's limit for 0x140000 unicode codes : win32 compatibility
-    {
-      UTF_ERROR_UTF8
-    }
+    if (c < 0xC0)
+      _ERROR_UTF8
 
     unsigned numBytes;
+    _UTF8_HEAD_PARSE
+    else
+      _ERROR_UTF8
+    
+    UInt32 val = c;
 
-    MY_UTF8_HEAD_PARSE_MAX_3_BYTES
-
-    unsigned pos = 0;
     do
     {
-      if (src + pos == srcLim)
+      Byte c2;
+      if (src == srcLim)
         break;
-      unsigned c2 = (Byte)src[pos];
-      c2 -= 0x80;
-      if (c2 >= 0x40)
+      c2 = *src;
+      if (c2 < 0x80 || c2 >= 0xC0)
         break;
+      src++;
       val <<= 6;
-      val |= c2;
-      pos++;
-      if (pos == 1)
-      {
-        if (val < (((unsigned)1 << 7) >> numBytes))
-          break;
-        if (numBytes == 2)
-        {
-          if (flags & Z7_UTF_FLAG_FROM_UTF8_SURROGATE_ERROR)
-            if ((val & (0xF800 >> 6)) == (0xd800 >> 6))
-              break;
-        }
-        else if (numBytes == 3 && val >= (0x110000 >> 12))
-          break;
-      }
+      val |= (c2 - 0x80);
     }
     while (--numBytes);
 
     if (numBytes != 0)
-    {
-      if ((flags & Z7_UTF_FLAG_FROM_UTF8_USE_ESCAPE) == 0)
-      {
-        // the following code to emit the 0xfffd chars as win32 Utf8 function.
-        // disable the folling line, if you need 0xfffd for each incorrect byte as in Escape mode
-        src += pos;
-      }
-      UTF_ERROR_UTF8
-    }
-
-    /*
-    if (val < MY_UTF8_RANGE(pos - 1))
-      UTF_ERROR_UTF8
-    */
-
-    #ifdef UTF_ESCAPE_BASE
-    
-      if ((flags & Z7_UTF_FLAG_FROM_UTF8_BMP_ESCAPE_CONVERT)
-          && IS_ESCAPE_POINT(val, 0))
-      {
-        // We will emit 3 utf16-Escape-16-21 points from one Escape-16 point (3 bytes)
-        UTF_ERROR_UTF8
-      }
-    
-    #endif
+      _ERROR_UTF8
 
-    /*
-       We don't expect virtual Escape-21 points in UTF-8 stream.
-       And we don't check for Escape-21.
-       So utf8-Escape-21 will be converted to another 3 utf16-Escape-21 points.
-       Maybe we could convert virtual utf8-Escape-21 to one utf16-Escape-21 point in some cases?
-    */
-    
-    if (val < START_POINT_FOR_SURROGATE)
+    if (val < 0x10000)
     {
-      /*
-      if ((flags & Z7_UTF_FLAG_FROM_UTF8_SURROGATE_ERROR)
-          && IS_SURROGATE_POINT(val))
-      {
-        // We will emit 3 utf16-Escape-16-21 points from one Surrogate-16 point (3 bytes)
-        UTF_ERROR_UTF8
-      }
-      */
       if (dest)
         dest[destPos] = (wchar_t)val;
       destPos++;
     }
     else
     {
-      /*
-      if (val >= 0x110000)
-      {
-        // We will emit utf16-Escape-16-21 point from each source byte
-        UTF_ERROR_UTF8
-      }
-      */
+      val -= 0x10000;
+      if (val >= 0x100000)
+        _ERROR_UTF8
       if (dest)
       {
-        dest[destPos + 0] = (wchar_t)(0xd800 - (0x10000 >> 10) + (val >> 10));
-        dest[destPos + 1] = (wchar_t)(0xdc00 + (val & 0x3ff));
+        dest[destPos + 0] = (wchar_t)(0xD800 + (val >> 10));
+        dest[destPos + 1] = (wchar_t)(0xDC00 + (val & 0x3FF));
       }
       destPos += 2;
     }
-    src += pos;
   }
 }
 
+#define _UTF8_RANGE(n) (((UInt32)1) << ((n) * 5 + 6))
 
+#define _UTF8_HEAD(n, val) ((char)(_UTF8_START(n) + (val >> (6 * (n)))))
+#define _UTF8_CHAR(n, val) ((char)(0x80 + (((val) >> (6 * (n))) & 0x3F)))
 
-#define MY_UTF8_HEAD(n, val) ((char)(MY_UTF8_START(n) + (val >> (6 * (n)))))
-#define MY_UTF8_CHAR(n, val) ((char)(0x80 + (((val) >> (6 * (n))) & 0x3F)))
-
-static size_t Utf16_To_Utf8_Calc(const wchar_t *src, const wchar_t *srcLim, unsigned flags)
+static size_t Utf16_To_Utf8_Calc(const wchar_t *src, const wchar_t *srcLim)
 {
-  size_t size = (size_t)(srcLim - src);
+  size_t size = srcLim - src;
   for (;;)
   {
     if (src == srcLim)
       return size;
     
-    UInt32 val = (UInt32)(*src++);
+    UInt32 val = *src++;
    
     if (val < 0x80)
       continue;
 
-    if (val < MY_UTF8_RANGE(1))
+    if (val < _UTF8_RANGE(1))
     {
       size++;
       continue;
     }
 
-    #ifdef UTF_ESCAPE_BASE
-    
-    #if UTF_ESCAPE_PLANE != 0
-    if (flags & Z7_UTF_FLAG_TO_UTF8_PARSE_HIGH_ESCAPE)
-      if (IS_ESCAPE_POINT(val, UTF_ESCAPE_PLANE))
-        continue;
-    #endif
-    
-    if (flags & Z7_UTF_FLAG_TO_UTF8_EXTRACT_BMP_ESCAPE)
-      if (IS_ESCAPE_POINT(val, 0))
-        continue;
-    
-    #endif
-
-    if (IS_SURROGATE_POINT(val))
+    if (val >= 0xD800 && val < 0xDC00 && src != srcLim)
     {
-      // it's hack to UTF-8 encoding
-
-      if (val < 0xdc00 && src != srcLim)
+      UInt32 c2 = *src;
+      if (c2 >= 0xDC00 && c2 < 0xE000)
       {
-        const UInt32 c2 = (UInt32)*src;
-        if (c2 >= 0xdc00 && c2 < 0xe000)
-          src++;
+        src++;
+        size += 2;
+        continue;
       }
-      size += 2;
-      continue;
     }
 
-    #ifdef Z7_WCHART_IS_16BIT
+    #ifdef _WCHART_IS_16BIT
     
     size += 2;
     
     #else
 
-         if (val < MY_UTF8_RANGE(2)) size += 2;
-    else if (val < MY_UTF8_RANGE(3)) size += 3;
-    else if (val < MY_UTF8_RANGE(4)) size += 4;
-    else if (val < MY_UTF8_RANGE(5)) size += 5;
-    else
-    #if MY_UTF8_NUM_TAIL_BYTES_MAX >= 6
-      size += 6;
-    #else
-      size += 3;
-    #endif
+         if (val < _UTF8_RANGE(2)) size += 2;
+    else if (val < _UTF8_RANGE(3)) size += 3;
+    else if (val < _UTF8_RANGE(4)) size += 4;
+    else if (val < _UTF8_RANGE(5)) size += 5;
+    else                           size += 6;
     
     #endif
   }
 }
 
-
-static char *Utf16_To_Utf8(char *dest, const wchar_t *src, const wchar_t *srcLim, unsigned flags)
+static char *Utf16_To_Utf8(char *dest, const wchar_t *src, const wchar_t *srcLim)
 {
   for (;;)
   {
     if (src == srcLim)
       return dest;
     
-    UInt32 val = (UInt32)*src++;
+    UInt32 val = *src++;
     
     if (val < 0x80)
     {
@@ -554,97 +212,51 @@ static char *Utf16_To_Utf8(char *dest, const wchar_t *src, const wchar_t *srcLim
       continue;
     }
 
-    if (val < MY_UTF8_RANGE(1))
+    if (val < _UTF8_RANGE(1))
     {
-      dest[0] = MY_UTF8_HEAD(1, val);
-      dest[1] = MY_UTF8_CHAR(0, val);
+      dest[0] = _UTF8_HEAD(1, val);
+      dest[1] = _UTF8_CHAR(0, val);
       dest += 2;
       continue;
     }
 
-    #ifdef UTF_ESCAPE_BASE
-    
-    #if UTF_ESCAPE_PLANE != 0
-    /*
-       if (wchar_t is 32-bit)
-            && (Z7_UTF_FLAG_TO_UTF8_PARSE_HIGH_ESCAPE is set)
-            && (point is virtual escape plane)
-          we extract 8-bit byte from virtual HIGH-ESCAPE PLANE.
-    */
-    if (flags & Z7_UTF_FLAG_TO_UTF8_PARSE_HIGH_ESCAPE)
-      if (IS_ESCAPE_POINT(val, UTF_ESCAPE_PLANE))
-      {
-        *dest++ = (char)(val);
-        continue;
-      }
-    #endif // UTF_ESCAPE_PLANE != 0
-
-    /* if (Z7_UTF_FLAG_TO_UTF8_EXTRACT_BMP_ESCAPE is defined)
-          we extract 8-bit byte from BMP-ESCAPE PLANE. */
-
-    if (flags & Z7_UTF_FLAG_TO_UTF8_EXTRACT_BMP_ESCAPE)
-      if (IS_ESCAPE_POINT(val, 0))
-      {
-        *dest++ = (char)(val);
-        continue;
-      }
-    
-    #endif // UTF_ESCAPE_BASE
-
-    if (IS_SURROGATE_POINT(val))
+    if (val >= 0xD800 && val < 0xDC00 && src != srcLim)
     {
-      // it's hack to UTF-8 encoding
-      if (val < 0xdc00 && src != srcLim)
+      UInt32 c2 = *src;
+      if (c2 >= 0xDC00 && c2 < 0xE000)
       {
-        const UInt32 c2 = (UInt32)*src;
-        if (IS_LOW_SURROGATE_POINT(c2))
-        {
-          src++;
-          val = (((val - 0xd800) << 10) | (c2 - 0xdc00)) + 0x10000;
-          dest[0] = MY_UTF8_HEAD(3, val);
-          dest[1] = MY_UTF8_CHAR(2, val);
-          dest[2] = MY_UTF8_CHAR(1, val);
-          dest[3] = MY_UTF8_CHAR(0, val);
-          dest += 4;
-          continue;
-        }
+        src++;
+        val = (((val - 0xD800) << 10) | (c2 - 0xDC00)) + 0x10000;
+        dest[0] = _UTF8_HEAD(3, val);
+        dest[1] = _UTF8_CHAR(2, val);
+        dest[2] = _UTF8_CHAR(1, val);
+        dest[3] = _UTF8_CHAR(0, val);
+        dest += 4;
+        continue;
       }
-      if (flags & Z7_UTF_FLAG_TO_UTF8_SURROGATE_ERROR)
-        val = UTF_REPLACEMENT_CHAR; // WIN32 function does it
     }
-
-    #ifndef Z7_WCHART_IS_16BIT
-    if (val < MY_UTF8_RANGE(2))
+    
+    #ifndef _WCHART_IS_16BIT
+    if (val < _UTF8_RANGE(2))
     #endif
     {
-      dest[0] = MY_UTF8_HEAD(2, val);
-      dest[1] = MY_UTF8_CHAR(1, val);
-      dest[2] = MY_UTF8_CHAR(0, val);
+      dest[0] = _UTF8_HEAD(2, val);
+      dest[1] = _UTF8_CHAR(1, val);
+      dest[2] = _UTF8_CHAR(0, val);
       dest += 3;
       continue;
     }
     
-    #ifndef Z7_WCHART_IS_16BIT
+    #ifndef _WCHART_IS_16BIT
 
-    // we don't expect this case. so we can throw exception
-    // throw 20210407;
-   
-    char b;
+    UInt32 b;
     unsigned numBits;
-         if (val < MY_UTF8_RANGE(3)) { numBits = 6 * 3; b = MY_UTF8_HEAD(3, val); }
-    else if (val < MY_UTF8_RANGE(4)) { numBits = 6 * 4; b = MY_UTF8_HEAD(4, val); }
-    else if (val < MY_UTF8_RANGE(5)) { numBits = 6 * 5; b = MY_UTF8_HEAD(5, val); }
-    #if MY_UTF8_NUM_TAIL_BYTES_MAX >= 6
-    else                           { numBits = 6 * 6; b = (char)MY_UTF8_START(6); }
-    #else
-    else
-    {
-      val = UTF_REPLACEMENT_CHAR;
-                                   { numBits = 6 * 3; b = MY_UTF8_HEAD(3, val); }
-    }
-    #endif
-
-    *dest++ = b;
+         if (val < _UTF8_RANGE(3)) { numBits = 6 * 3; b = _UTF8_HEAD(3, val); }
+    else if (val < _UTF8_RANGE(4)) { numBits = 6 * 4; b = _UTF8_HEAD(4, val); }
+    else if (val < _UTF8_RANGE(5)) { numBits = 6 * 5; b = _UTF8_HEAD(5, val); }
+    else                           { numBits = 6 * 6; b = _UTF8_START(6); }
+    
+    *dest++ = (Byte)b;
     
     do
     {
@@ -657,207 +269,20 @@ static char *Utf16_To_Utf8(char *dest, const wchar_t *src, const wchar_t *srcLim
   }
 }
 
-bool Convert_UTF8_Buf_To_Unicode(const char *src, size_t srcSize, UString &dest, unsigned flags)
+bool ConvertUTF8ToUnicode(const AString &src, UString &dest)
 {
   dest.Empty();
   size_t destLen = 0;
-  Utf8_To_Utf16(NULL, &destLen, src, src + srcSize, flags);
-  bool res = Utf8_To_Utf16(dest.GetBuf((unsigned)destLen), &destLen, src, src + srcSize, flags);
+  Utf8_To_Utf16(NULL, &destLen, src, src.Ptr(src.Len()));
+  bool res = Utf8_To_Utf16(dest.GetBuf((unsigned)destLen), &destLen, src, src.Ptr(src.Len()));
   dest.ReleaseBuf_SetEnd((unsigned)destLen);
   return res;
 }
 
-bool ConvertUTF8ToUnicode_Flags(const AString &src, UString &dest, unsigned flags)
-{
-  return Convert_UTF8_Buf_To_Unicode(src, src.Len(), dest,  flags);
-}
-
-
-static
-unsigned g_UTF8_To_Unicode_Flags =
-    Z7_UTF_FLAG_FROM_UTF8_USE_ESCAPE
-  #ifndef Z7_WCHART_IS_16BIT
-    | Z7_UTF_FLAG_FROM_UTF8_SURROGATE_ERROR
-  #ifdef MY_UTF8_RAW_NON_UTF8_SUPPORTED
-    | Z7_UTF_FLAG_FROM_UTF8_BMP_ESCAPE_CONVERT
-  #endif
-  #endif
-    ;
-    
-
-/*
-bool ConvertUTF8ToUnicode_boolRes(const AString &src, UString &dest)
-{
-  return ConvertUTF8ToUnicode_Flags(src, dest, g_UTF8_To_Unicode_Flags);
-}
-*/
-
-bool ConvertUTF8ToUnicode(const AString &src, UString &dest)
-{
-  return ConvertUTF8ToUnicode_Flags(src, dest, g_UTF8_To_Unicode_Flags);
-}
-
-void Print_UString(const UString &a);
-
-void ConvertUnicodeToUTF8_Flags(const UString &src, AString &dest, unsigned flags)
-{
-  /*
-  if (src.Len()== 24)
-    throw "202104";
-  */
-  dest.Empty();
-  const size_t destLen = Utf16_To_Utf8_Calc(src, src.Ptr(src.Len()), flags);
-  char *destStart = dest.GetBuf((unsigned)destLen);
-  const char *destEnd = Utf16_To_Utf8(destStart, src, src.Ptr(src.Len()), flags);
-  dest.ReleaseBuf_SetEnd((unsigned)destLen);
-  // printf("\nlen = %d\n", src.Len());
-  if (destLen != (size_t)(destEnd - destStart))
-  {
-    /*
-    // dest.ReleaseBuf_SetEnd((unsigned)(destEnd - destStart));
-    printf("\nlen = %d\n", (unsigned)destLen);
-    printf("\n(destEnd - destStart) = %d\n", (unsigned)(destEnd - destStart));
-    printf("\n");
-    // Print_UString(src);
-    printf("\n");
-    // printf("\nlen = %d\n", destLen);
-    */
-    throw 20210406;
-  }
-}
-
-
-
-unsigned g_Unicode_To_UTF8_Flags =
-      // Z7_UTF_FLAG_TO_UTF8_PARSE_HIGH_ESCAPE
-      0
-  #ifndef _WIN32
-    #ifdef MY_UTF8_RAW_NON_UTF8_SUPPORTED
-      | Z7_UTF_FLAG_TO_UTF8_EXTRACT_BMP_ESCAPE
-    #else
-      | Z7_UTF_FLAG_TO_UTF8_SURROGATE_ERROR
-    #endif
-  #endif
-    ;
-
 void ConvertUnicodeToUTF8(const UString &src, AString &dest)
-{
-  ConvertUnicodeToUTF8_Flags(src, dest, g_Unicode_To_UTF8_Flags);
-}
-
-void Convert_Unicode_To_UTF8_Buf(const UString &src, CByteBuffer &dest)
-{
-  const unsigned flags = g_Unicode_To_UTF8_Flags;
-  dest.Free();
-  const size_t destLen = Utf16_To_Utf8_Calc(src, src.Ptr(src.Len()), flags);
-  dest.Alloc(destLen);
-  const char *destEnd = Utf16_To_Utf8((char *)(void *)(Byte *)dest, src, src.Ptr(src.Len()), flags);
-  if (destLen != (size_t)(destEnd - (char *)(void *)(Byte *)dest))
-    throw 202104;
-}
-
-/*
-
-#ifndef _WIN32
-void Convert_UTF16_To_UTF32(const UString &src, UString &dest)
 {
   dest.Empty();
-  for (size_t i = 0; i < src.Len();)
-  {
-    wchar_t c = src[i++];
-    if (c >= 0xd800 && c < 0xdc00 && i < src.Len())
-    {
-      const wchar_t c2 = src[i];
-      if (c2 >= 0xdc00 && c2 < 0xe000)
-      {
-        // printf("\nSurragate [%d]: %4x %4x -> ", i, (int)c, (int)c2);
-        c = 0x10000 + ((c & 0x3ff) << 10) + (c2 & 0x3ff);
-        // printf("%4x\n", (int)c);
-        i++;
-      }
-    }
-    dest += c;
-  }
-}
-
-void Convert_UTF32_To_UTF16(const UString &src, UString &dest)
-{
-  dest.Empty();
-  for (size_t i = 0; i < src.Len();)
-  {
-    wchar_t w = src[i++];
-    if (w >= 0x10000 && w < 0x110000)
-    {
-      w -= 0x10000;
-      dest += (wchar_t)((unsigned)0xd800 + (((unsigned)w >> 10) & 0x3ff));
-      w = 0xdc00 + (w & 0x3ff);
-    }
-    dest += w;
-  }
-}
-
-bool UTF32_IsThere_BigPoint(const UString &src)
-{
-  for (size_t i = 0; i < src.Len();)
-  {
-    const UInt32 c = (UInt32)src[i++];
-    if (c >= 0x110000)
-      return true;
-  }
-  return false;
-}
-
-bool Unicode_IsThere_BmpEscape(const UString &src)
-{
-  for (size_t i = 0; i < src.Len();)
-  {
-    const UInt32 c = (UInt32)src[i++];
-    if (IS_ESCAPE_POINT(c, 0))
-      return true;
-  }
-  return false;
-}
-
-
-#endif
-
-bool Unicode_IsThere_Utf16SurrogateError(const UString &src)
-{
-  for (size_t i = 0; i < src.Len();)
-  {
-    const UInt32 val = (UInt32)src[i++];
-    if (IS_SURROGATE_POINT(val))
-    {
-      // it's hack to UTF-8 encoding
-      if (val >= 0xdc00 || i == src.Len())
-        return true;
-      const UInt32 c2 = (UInt32)*src;
-      if (!IS_LOW_SURROGATE_POINT(c2))
-        return true;
-    }
-  }
-  return false;
-}
-*/
-
-#ifndef Z7_WCHART_IS_16BIT
-
-void Convert_UnicodeEsc16_To_UnicodeEscHigh
-#if UTF_ESCAPE_PLANE == 0
-    (UString &) {}
-#else
-    (UString &s)
-{
-  const unsigned len = s.Len();
-  for (unsigned i = 0; i < len; i++)
-  {
-    wchar_t c = s[i];
-    if (IS_ESCAPE_POINT(c, 0))
-    {
-      c += UTF_ESCAPE_PLANE;
-      s.ReplaceOneCharAtPos(i, c);
-    }
-  }
+  size_t destLen = Utf16_To_Utf8_Calc(src, src.Ptr(src.Len()));
+  Utf16_To_Utf8(dest.GetBuf((unsigned)destLen), src, src.Ptr(src.Len()));
+  dest.ReleaseBuf_SetEnd((unsigned)destLen);
 }
-#endif
-#endif