OpenCC 1.2.0__cp38-cp38-manylinux2014_x86_64.whl

opencc/clib/include/opencc/UTF8StringSlice.hpp

@@ -0,0 +1,246 @@
+/*
+ * Open Chinese Convert
+ *
+ * Copyright 2015 Carbo Kuo <byvoid@byvoid.com>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <cstring>
+
+#include "Common.hpp"
+#include "UTF8Util.hpp"
+
+namespace opencc {
+
+namespace internal {
+
+inline size_t FNVHash(const char* text, const size_t byteLength,
+                      const size_t FNV_prime, const size_t FNV_offset_basis) {
+  size_t hash = FNV_offset_basis;
+  for (const char* pstr = text; pstr < text + byteLength; pstr++) {
+    hash ^= *pstr;
+    hash *= FNV_prime;
+  }
+  return hash;
+}
+
+template <int> size_t FNVHash(const char* text, const size_t byteLength);
+
+template <>
+inline size_t FNVHash<4>(const char* text, const size_t byteLength) {
+  return FNVHash(text, byteLength, 16777619UL, 2166136261UL);
+}
+
+#if SIZE_MAX == 0xffffffffffffffff
+template <>
+inline size_t FNVHash<8>(const char* text, const size_t byteLength) {
+  return FNVHash(text, byteLength, 1099511628211UL, 14695981039346656037UL);
+}
+#endif
+
+} // namespace internal
+
+template <typename LENGTH_TYPE> class UTF8StringSliceBase {
+public:
+  typedef LENGTH_TYPE LengthType;
+
+  UTF8StringSliceBase(const char* _str)
+      : str(_str), utf8Length(static_cast<LengthType>(UTF8Util::Length(_str))),
+        byteLength(static_cast<LengthType>(strlen(_str))) {}
+
+  UTF8StringSliceBase(const char* _str, const LengthType _utf8Length)
+      : str(_str), utf8Length(_utf8Length) {
+    CalculateByteLength();
+  }
+
+  UTF8StringSliceBase(const char* _str, const LengthType _utf8Length,
+                      const LengthType _byteLength)
+      : str(_str), utf8Length(_utf8Length), byteLength(_byteLength) {
+    CalculateByteLength();
+  }
+
+  LengthType UTF8Length() const { return utf8Length; }
+
+  LengthType ByteLength() const { return byteLength; }
+
+  UTF8StringSliceBase Left(const LengthType numberOfCharacters) const {
+    if (numberOfCharacters == UTF8Length()) {
+      return *this;
+    } else {
+      return UTF8StringSliceBase(str, numberOfCharacters);
+    }
+  }
+
+  UTF8StringSliceBase Right(const LengthType numberOfCharacters) const {
+    if (numberOfCharacters == UTF8Length()) {
+      return *this;
+    } else {
+      const char* pstr = str + byteLength;
+      for (size_t i = 0; i < numberOfCharacters; i++) {
+        pstr = UTF8Util::PrevChar(pstr);
+      }
+      return UTF8StringSliceBase(pstr, numberOfCharacters);
+    }
+  }
+
+  UTF8StringSliceBase SubString(const LengthType offset,
+                                const LengthType numberOfCharacters) const {
+    if (offset == 0) {
+      return Left(numberOfCharacters);
+    } else {
+      const char* pstr = str;
+      for (size_t i = 0; i < offset; i++) {
+        pstr = UTF8Util::NextChar(pstr);
+      }
+      return UTF8StringSliceBase(pstr, numberOfCharacters);
+    }
+  }
+
+  std::string ToString() const { return std::string(str, str + byteLength); }
+
+  const char* CString() const { return str; }
+
+  LengthType CommonPrefixLength(const UTF8StringSliceBase& that) const {
+    if (str == that.str) {
+      return (std::min)(utf8Length, that.utf8Length);
+    } else {
+      const char* pstr1 = str;
+      const char* pstr2 = that.str;
+      for (size_t length = 0; length < utf8Length && length < that.utf8Length;
+           length++) {
+        size_t charLen1 = UTF8Util::NextCharLength(pstr1);
+        size_t charLen2 = UTF8Util::NextCharLength(pstr2);
+        if (charLen1 != charLen2 || strncmp(pstr1, pstr2, charLen1) != 0) {
+          return length;
+        }
+        pstr1 += charLen1;
+        pstr2 += charLen2;
+      }
+      return 0;
+    }
+  }
+
+  void MoveRight() {
+    if (utf8Length > 0) {
+      const size_t charLen = UTF8Util::NextCharLength(str);
+      str += charLen;
+      utf8Length--;
+      byteLength -= charLen;
+    }
+  }
+
+  void MoveLeft() {
+    if (utf8Length > 0) {
+      const size_t charLen = UTF8Util::PrevCharLength(str + byteLength);
+      utf8Length--;
+      byteLength -= charLen;
+    }
+  }
+
+  int ReverseCompare(const UTF8StringSliceBase& that) const {
+    const char* pstr1 = str + byteLength;
+    const char* pstr2 = that.str + that.byteLength;
+    const size_t length = (std::min)(utf8Length, that.utf8Length);
+    for (size_t i = 0; i < length; i++) {
+      const size_t charLen1 = UTF8Util::PrevCharLength(pstr1);
+      const size_t charLen2 = UTF8Util::PrevCharLength(pstr2);
+      pstr1 -= charLen1;
+      pstr2 -= charLen2;
+      const int cmp = strncmp(pstr1, pstr2, (std::min)(charLen1, charLen2));
+      if (cmp < 0) {
+        return -1;
+      } else if (cmp > 0) {
+        return 1;
+      } else if (charLen1 < charLen2) {
+        return -1;
+      } else if (charLen1 > charLen2) {
+        return 1;
+      }
+    }
+    if (utf8Length < that.utf8Length) {
+      return -1;
+    } else if (utf8Length > that.utf8Length) {
+      return 1;
+    } else {
+      return 0;
+    }
+  }
+
+  LengthType FindBytePosition(const UTF8StringSliceBase& pattern) const {
+    return static_cast<LengthType>(
+        ToString().find(pattern.str, 0, pattern.byteLength));
+  }
+
+  bool operator<(const UTF8StringSliceBase& that) const {
+    return Compare(that) < 0;
+  }
+
+  bool operator>(const UTF8StringSliceBase& that) const {
+    return Compare(that) > 0;
+  }
+
+  bool operator==(const UTF8StringSliceBase& that) const {
+    return (str == that.str && utf8Length == that.utf8Length) ||
+           Compare(that) == 0;
+  }
+
+  bool operator!=(const UTF8StringSliceBase& that) const {
+    return !this->operator==(that);
+  }
+
+  class Hasher {
+  public:
+    size_t operator()(const UTF8StringSliceBase& text) const {
+      return internal::FNVHash<sizeof(size_t)>(text.CString(),
+                                               text.ByteLength());
+    }
+  };
+
+private:
+  inline int Compare(const UTF8StringSliceBase& that) const {
+    int cmp = strncmp(str, that.str, (std::min)(byteLength, that.byteLength));
+    if (cmp == 0) {
+      if (utf8Length < that.utf8Length) {
+        cmp = -1;
+      } else if (utf8Length > that.utf8Length) {
+        cmp = 1;
+      } else {
+        cmp = 0;
+      }
+    }
+    return cmp;
+  }
+
+  void CalculateByteLength() {
+    const char* pstr = str;
+    for (size_t i = 0; i < utf8Length; i++) {
+      pstr = UTF8Util::NextChar(pstr);
+    }
+    byteLength = static_cast<LengthType>(pstr - str);
+  }
+
+  const char* str;
+  LengthType utf8Length;
+  LengthType byteLength;
+};
+
+typedef UTF8StringSliceBase<size_t> UTF8StringSlice;
+
+template <typename LENGTH_TYPE>
+std::ostream& operator<<(::std::ostream& os,
+                         const UTF8StringSliceBase<LENGTH_TYPE>& str) {
+  return os << str.ToString();
+}
+
+} // namespace opencc

opencc/clib/include/opencc/UTF8Util.hpp

@@ -0,0 +1,291 @@
+/*
+ * Open Chinese Convert
+ *
+ * Copyright 2013 Carbo Kuo <byvoid@byvoid.com>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#ifdef _MSC_VER
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <Windows.h>
+#endif // _MSC_VER
+
+#include <cstring>
+
+#include "Common.hpp"
+#include "Exception.hpp"
+
+namespace opencc {
+/**
+ * UTF8 std::string utilities
+ * @ingroup opencc_cpp_api
+ */
+class OPENCC_EXPORT UTF8Util {
+public:
+  /**
+   * Detect UTF8 BOM and skip it.
+   */
+  static void SkipUtf8Bom(FILE* fp);
+
+  /**
+   * Returns the length in byte for the next UTF8 character.
+   * On error returns 0.
+   */
+  static size_t NextCharLengthNoException(const char* str) {
+    char ch = *str;
+    if ((ch & 0xF0) == 0xE0) {
+      return 3;
+    } else if ((ch & 0x80) == 0x00) {
+      return 1;
+    } else if ((ch & 0xE0) == 0xC0) {
+      return 2;
+    } else if ((ch & 0xF8) == 0xF0) {
+      return 4;
+    } else if ((ch & 0xFC) == 0xF8) {
+      return 5;
+    } else if ((ch & 0xFE) == 0xFC) {
+      return 6;
+    }
+    return 0;
+  }
+
+  /**
+   * Returns the length in byte for the next UTF8 character.
+   */
+  static size_t NextCharLength(const char* str) {
+    size_t length = NextCharLengthNoException(str);
+    if (length == 0) {
+      throw InvalidUTF8(str);
+    }
+    return length;
+  }
+
+  /**
+   * Returns the length in byte for the previous UTF8 character.
+   */
+  static size_t PrevCharLength(const char* str) {
+    {
+      const size_t length = NextCharLengthNoException(str - 3);
+      if (length == 3) {
+        return length;
+      }
+    }
+    {
+      const size_t length = NextCharLengthNoException(str - 1);
+      if (length == 1) {
+        return length;
+      }
+    }
+    {
+      const size_t length = NextCharLengthNoException(str - 2);
+      if (length == 2) {
+        return length;
+      }
+    }
+    for (size_t i = 4; i <= 6; i++) {
+      const size_t length = NextCharLengthNoException(str - i);
+      if (length == i) {
+        return length;
+      }
+    }
+    throw InvalidUTF8(str);
+  }
+
+  /**
+   * Returns the char* pointer over the next UTF8 character.
+   */
+  static const char* NextChar(const char* str) {
+    return str + NextCharLength(str);
+  }
+
+  /**
+   * Move the char* pointer before the previous UTF8 character.
+   */
+  static const char* PrevChar(const char* str) {
+    return str - PrevCharLength(str);
+  }
+
+  /**
+   * Returns the UTF8 length of a valid UTF8 std::string.
+   */
+  static size_t Length(const char* str) {
+    size_t length = 0;
+    while (*str != '\0') {
+      str = NextChar(str);
+      length++;
+    }
+    return length;
+  }
+
+  /**
+   * Finds a character in the same line.
+   * @param str The text to be searched in.
+   * @param ch  The character to find.
+   * @return    The pointer that points to the found chacter in str or EOL/EOF.
+   */
+  static const char* FindNextInline(const char* str, const char ch) {
+    while (!IsLineEndingOrFileEnding(*str) && *str != ch) {
+      str = NextChar(str);
+    }
+    return str;
+  }
+
+  /**
+   * Returns true if the character is a line ending or end of file.
+   */
+  static bool IsLineEndingOrFileEnding(const char ch) {
+    return ch == '\0' || ch == '\n' || ch == '\r';
+  }
+
+  /**
+   * Copies a substring with given length to a new string.
+   */
+  static std::string FromSubstr(const char* str, size_t length) {
+    std::string newStr;
+    newStr.resize(length);
+    strncpy(const_cast<char*>(newStr.c_str()), str, length);
+    return newStr;
+  }
+
+  /**
+   * Returns true if the given std::string is longer or as long as the given
+   * length.
+   */
+  static bool NotShorterThan(const char* str, size_t byteLength) {
+    while (byteLength > 0) {
+      if (*str == '\0') {
+        return false;
+      }
+      byteLength--;
+      str++;
+    }
+    return true;
+  }
+
+  /**
+   * Truncates a std::string with a maximal length in byte.
+   * No UTF8 character will be broken.
+   */
+  static std::string TruncateUTF8(const char* str, size_t maxByteLength) {
+    std::string wordTrunc;
+    if (NotShorterThan(str, maxByteLength)) {
+      size_t len = 0;
+      const char* pStr = str;
+      for (;;) {
+        const size_t charLength = NextCharLength(pStr);
+        if (len + charLength > maxByteLength) {
+          break;
+        }
+        pStr += charLength;
+        len += charLength;
+      }
+      wordTrunc = FromSubstr(str, len);
+    } else {
+      wordTrunc = str;
+    }
+    return wordTrunc;
+  }
+
+  /**
+   * Replaces all patterns in a std::string in place.
+   */
+  static void ReplaceAll(std::string& str, const char* from, const char* to) {
+    std::string::size_type pos = 0;
+    std::string::size_type fromLen = strlen(from);
+    std::string::size_type toLen = strlen(to);
+    while ((pos = str.find(from, pos)) != std::string::npos) {
+      str.replace(pos, fromLen, to);
+      pos += toLen;
+    }
+  }
+
+  /**
+   * Joins a std::string vector in to a std::string with a separator.
+   */
+  static std::string Join(const std::vector<std::string>& strings,
+                          const std::string& separator) {
+    std::ostringstream buffer;
+    bool first = true;
+    for (const auto& str : strings) {
+      if (!first) {
+        buffer << separator;
+      }
+      buffer << str;
+      first = false;
+    }
+    return buffer.str();
+  }
+
+  /**
+   * Joins a std::string vector in to a std::string.
+   */
+  static std::string Join(const std::vector<std::string>& strings) {
+    std::ostringstream buffer;
+    for (const auto& str : strings) {
+      buffer << str;
+    }
+    return buffer.str();
+  }
+
+  static void GetByteMap(const char* str, const size_t utf8Length,
+                         std::vector<size_t>* byteMap) {
+    if (byteMap->size() < utf8Length) {
+      byteMap->resize(utf8Length);
+    }
+    const char* pstr = str;
+    for (size_t i = 0; i < utf8Length; i++) {
+      (*byteMap)[i] = pstr - str;
+      pstr = NextChar(pstr);
+    }
+  }
+
+#ifdef _MSC_VER
+  static std::wstring GetPlatformString(const std::string& str) {
+    return U8ToU16(str);
+  }
+#else
+  static std::string GetPlatformString(const std::string& str) { return str; }
+#endif // _MSC_VER
+
+#ifdef _MSC_VER
+  static std::string U16ToU8(const std::wstring& wstr) {
+    std::string ret;
+    int length = static_cast<int>(wstr.length());
+    int convcnt = WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), length, NULL, 0,
+                                      NULL, NULL);
+    if (convcnt > 0) {
+      ret.resize(convcnt);
+      WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), length, &ret[0], convcnt,
+                          NULL, NULL);
+    }
+    return ret;
+  }
+
+  static std::wstring U8ToU16(const std::string& str) {
+    std::wstring ret;
+    int length = static_cast<int>(str.length());
+    int convcnt = MultiByteToWideChar(CP_UTF8, 0, str.c_str(), length, NULL, 0);
+    if (convcnt > 0) {
+      ret.resize(convcnt);
+      MultiByteToWideChar(CP_UTF8, 0, str.c_str(), length, &ret[0], convcnt);
+    }
+    return ret;
+  }
+#endif // _MSC_VER
+};
+} // namespace opencc

opencc/clib/include/opencc/opencc.h

@@ -0,0 +1,161 @@
+/*
+ * Open Chinese Convert
+ *
+ * Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __OPENCC_H_
+#define __OPENCC_H_
+
+#ifdef __cplusplus
+
+#include "Export.hpp"
+#include "SimpleConverter.hpp"
+#include <string>
+
+extern "C" {
+#else
+#include <stddef.h>
+#endif
+
+#ifndef OPENCC_EXPORT
+#define OPENCC_EXPORT
+#endif
+
+/**
+ * @defgroup opencc_c_api OpenCC C API
+ *
+ * API in C language
+ */
+
+/**
+ * Filename of default Simplified to Traditional configuration
+ *
+ * @ingroup opencc_c_api
+ */
+#define OPENCC_DEFAULT_CONFIG_SIMP_TO_TRAD "s2t.json"
+
+/**
+ * Filename of default Traditional to Simplified configuration
+ *
+ * @ingroup opencc_c_api
+ */
+#define OPENCC_DEFAULT_CONFIG_TRAD_TO_SIMP "t2s.json"
+
+/**
+ * Type of opencc descriptor
+ *
+ * @ingroup opencc_c_api
+ */
+typedef void* opencc_t;
+
+/**
+ * Makes an instance of opencc
+ *
+ * @param configFileName Location of configuration file. If this is set to NULL,
+ *                       OPENCC_DEFAULT_CONFIG_SIMP_TO_TRAD will be loaded.
+ * @return            A description pointer of the newly allocated instance of
+ *                    opencc. On error the return value will be (opencc_t) -1.
+ * @ingroup opencc_c_api
+ */
+OPENCC_EXPORT opencc_t opencc_open(const char* configFileName);
+#ifdef _MSC_VER
+/**
+ * Makes an instance of opencc (wide char / Unicode)
+ *
+ * @param configFileName Location of configuration file. If this is set to NULL,
+ *                       OPENCC_DEFAULT_CONFIG_SIMP_TO_TRAD will be loaded.
+ * @return            A description pointer of the newly allocated instance of
+ *                    opencc. On error the return value will be (opencc_t) -1.
+ * @ingroup opencc_c_api
+ */
+OPENCC_EXPORT opencc_t opencc_open_w(const wchar_t* configFileName);
+#endif /* _MSC_VER */
+
+/**
+ * Destroys an instance of opencc
+ *
+ * @param opencc The description pointer.
+ * @return 0 on success or non-zero number on failure.
+ * @ingroup opencc_c_api
+ */
+OPENCC_EXPORT int opencc_close(opencc_t opencc);
+
+/**
+ * Converts UTF-8 std::string
+ *
+ * @param opencc The opencc description pointer.
+ * @param input  The UTF-8 encoded std::string.
+ * @param length The maximum length in byte to convert. If length is (size_t)-1,
+ *               the whole std::string (terminated by '\0') will be converted.
+ * @param output The buffer to store converted text. You MUST make sure this
+ *               buffer has sufficient space.
+ *
+ * @return       The length of converted std::string or (size_t)-1 on error.
+ *
+ * @ingroup opencc_c_api
+ */
+OPENCC_EXPORT size_t opencc_convert_utf8_to_buffer(opencc_t opencc,
+                                                   const char* input,
+                                                   size_t length, char* output);
+
+/**
+ * Converts UTF-8 std::string
+ * This function returns an allocated C-Style std::string, which stores
+ * the converted std::string.
+ * You MUST call opencc_convert_utf8_free() to release allocated memory.
+ *
+ * @param opencc The opencc description pointer.
+ * @param input  The UTF-8 encoded std::string.
+ * @param length The maximum length in byte to convert. If length is (size_t)-1,
+ *               the whole std::string (terminated by '\0') will be converted.
+ *
+ * @return       The newly allocated UTF-8 std::string that stores text
+ * converted, or NULL on error.
+ * @ingroup opencc_c_api
+ */
+OPENCC_EXPORT char* opencc_convert_utf8(opencc_t opencc, const char* input,
+                                        size_t length);
+
+/**
+ * Releases allocated buffer by opencc_convert_utf8
+ *
+ * @param str    Pointer to the allocated std::string buffer by
+ * opencc_convert_utf8.
+ *
+ * @ingroup opencc_c_api
+ */
+OPENCC_EXPORT void opencc_convert_utf8_free(char* str);
+
+/**
+ * Returns the last error message
+ *
+ * Note that this function is the only one which is NOT thread-safe.
+ *
+ * @ingroup opencc_c_api
+ */
+OPENCC_EXPORT const char* opencc_error(void);
+
+#ifdef __cplusplus
+} // extern "C"
+#endif
+
+/**
+ * @defgroup opencc_cpp_api OpenCC C++ Comprehensive API
+ *
+ * Comprehensive API in C++ language
+ */
+
+#endif

opencc/clib/include/opencc/opencc_config.h

@@ -0,0 +1,21 @@
+/*
+ * Open Chinese Convert
+ *
+ * Copyright 2021 Carbo Kuo <byvoid@byvoid.com>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#define OPENCC_ENABLE_DARTS