OpenCC 1.2.0__cp38-cp38-manylinux2014_x86_64.whl

opencc/clib/include/opencc/PhraseExtract.hpp

@@ -0,0 +1,195 @@
+/*
+ * Open Chinese Convert
+ *
+ * Copyright 2015 Carbo Kuo <byvoid@byvoid.com>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <functional>
+#include <unordered_map>
+
+#include "Common.hpp"
+#include "UTF8StringSlice.hpp"
+
+namespace opencc {
+
+class OPENCC_EXPORT PhraseExtract {
+public:
+  typedef UTF8StringSlice::LengthType LengthType;
+
+  typedef UTF8StringSliceBase<unsigned char> UTF8StringSlice8Bit;
+
+  PhraseExtract();
+
+  virtual ~PhraseExtract();
+
+  void Extract(const std::string& text) {
+    SetFullText(text);
+    ExtractSuffixes();
+    CalculateFrequency();
+    CalculateSuffixEntropy();
+    ReleaseSuffixes();
+    ExtractPrefixes();
+    CalculatePrefixEntropy();
+    ReleasePrefixes();
+    ExtractWordCandidates();
+    CalculateCohesions();
+    SelectWords();
+  }
+
+  void SetFullText(const std::string& fullText) {
+    utf8FullText = UTF8StringSlice(fullText.c_str());
+  }
+
+  void SetFullText(const char* fullText) {
+    utf8FullText = UTF8StringSlice(fullText);
+  }
+
+  void SetFullText(const UTF8StringSlice& fullText) { utf8FullText = fullText; }
+
+  void SetWordMinLength(const LengthType _wordMinLength) {
+    wordMinLength = _wordMinLength;
+  }
+
+  void SetWordMaxLength(const LengthType _wordMaxLength) {
+    wordMaxLength = _wordMaxLength;
+  }
+
+  void SetPrefixSetLength(const LengthType _prefixSetLength) {
+    prefixSetLength = _prefixSetLength;
+  }
+
+  void SetSuffixSetLength(const LengthType _suffixSetLength) {
+    suffixSetLength = _suffixSetLength;
+  }
+
+  // PreCalculationFilter is called after frequencies statistics.
+  void SetPreCalculationFilter(
+      const std::function<bool(const PhraseExtract&,
+                               const UTF8StringSlice8Bit&)>& filter) {
+    preCalculationFilter = filter;
+  }
+
+  void SetPostCalculationFilter(
+      const std::function<bool(const PhraseExtract&,
+                               const UTF8StringSlice8Bit&)>& filter) {
+    postCalculationFilter = filter;
+  }
+
+  void ReleaseSuffixes() { std::vector<UTF8StringSlice8Bit>().swap(suffixes); }
+
+  void ReleasePrefixes() { std::vector<UTF8StringSlice8Bit>().swap(prefixes); }
+
+  const std::vector<UTF8StringSlice8Bit>& Words() const { return words; }
+
+  const std::vector<UTF8StringSlice8Bit>& WordCandidates() const {
+    return wordCandidates;
+  }
+
+  struct Signals {
+    size_t frequency;
+    double cohesion;
+    double suffixEntropy;
+    double prefixEntropy;
+  };
+
+  const Signals& Signal(const UTF8StringSlice8Bit& wordCandidate) const;
+
+  double Cohesion(const UTF8StringSlice8Bit& wordCandidate) const;
+
+  double Entropy(const UTF8StringSlice8Bit& wordCandidate) const;
+
+  double SuffixEntropy(const UTF8StringSlice8Bit& wordCandidate) const;
+
+  double PrefixEntropy(const UTF8StringSlice8Bit& wordCandidate) const;
+
+  size_t Frequency(const UTF8StringSlice8Bit& word) const;
+
+  double Probability(const UTF8StringSlice8Bit& word) const;
+
+  double LogProbability(const UTF8StringSlice8Bit& word) const;
+
+  void Reset();
+
+  void ExtractSuffixes();
+
+  void ExtractPrefixes();
+
+  void ExtractWordCandidates();
+
+  void CalculateFrequency();
+
+  void CalculateCohesions();
+
+  void CalculateSuffixEntropy();
+
+  void CalculatePrefixEntropy();
+
+  void SelectWords();
+
+  static bool
+  DefaultPreCalculationFilter(const PhraseExtract&,
+                              const PhraseExtract::UTF8StringSlice8Bit&);
+
+  static bool
+  DefaultPostCalculationFilter(const PhraseExtract&,
+                               const PhraseExtract::UTF8StringSlice8Bit&);
+
+private:
+  class DictType;
+
+  // Pointwise Mutual Information
+  double PMI(const UTF8StringSlice8Bit& wordCandidate,
+             const UTF8StringSlice8Bit& part1,
+             const UTF8StringSlice8Bit& part2) const;
+
+  double CalculateCohesion(const UTF8StringSlice8Bit& wordCandidate) const;
+
+  double CalculateEntropy(
+      const std::unordered_map<UTF8StringSlice8Bit, size_t,
+                               UTF8StringSlice8Bit::Hasher>& choices) const;
+
+  LengthType wordMinLength;
+  LengthType wordMaxLength;
+  LengthType prefixSetLength;
+  LengthType suffixSetLength;
+  std::function<bool(const PhraseExtract&, const UTF8StringSlice8Bit&)>
+      preCalculationFilter;
+  std::function<bool(const PhraseExtract&, const UTF8StringSlice8Bit&)>
+      postCalculationFilter;
+
+  bool prefixesExtracted;
+  bool suffixesExtracted;
+  bool frequenciesCalculated;
+  bool wordCandidatesExtracted;
+  bool cohesionsCalculated;
+  bool prefixEntropiesCalculated;
+  bool suffixEntropiesCalculated;
+  bool wordsSelected;
+
+  UTF8StringSlice utf8FullText;
+  size_t totalOccurrence;
+  double logTotalOccurrence;
+  std::vector<UTF8StringSlice8Bit> prefixes;
+  std::vector<UTF8StringSlice8Bit> suffixes;
+  std::vector<UTF8StringSlice8Bit> wordCandidates;
+  std::vector<UTF8StringSlice8Bit> words;
+  DictType* signals;
+
+  friend class PhraseExtractTest;
+};
+
+} // namespace opencc

opencc/clib/include/opencc/Segmentation.hpp

@@ -0,0 +1,32 @@
+/*
+ * Open Chinese Convert
+ *
+ * Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "Common.hpp"
+
+namespace opencc {
+/**
+ * Abstract segmentation
+ * @ingroup opencc_cpp_api
+ */
+class OPENCC_EXPORT Segmentation {
+public:
+  virtual SegmentsPtr Segment(const std::string& text) const = 0;
+};
+} // namespace opencc

opencc/clib/include/opencc/Segments.hpp

@@ -0,0 +1,118 @@
+/*
+ * Open Chinese Convert
+ *
+ * Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <iterator>
+#include <sstream>
+
+#include "Common.hpp"
+
+namespace opencc {
+/**
+ * Segmented text
+ * @ingroup opencc_cpp_api
+ */
+class OPENCC_EXPORT Segments {
+public:
+  Segments() {}
+
+  Segments(std::initializer_list<const char*> initList) {
+    for (const char* item : initList) {
+      AddSegment(item);
+    }
+  }
+
+  Segments(std::initializer_list<std::string> initList) {
+    for (const std::string& item : initList) {
+      AddSegment(item);
+    }
+  }
+
+  void AddSegment(const char* unmanagedString) {
+    indexes.push_back(std::make_pair(unmanaged.size(), false));
+    unmanaged.push_back(unmanagedString);
+  }
+
+  void AddSegment(const std::string& str) {
+    indexes.push_back(std::make_pair(managed.size(), true));
+    managed.push_back(str);
+  }
+
+  class iterator {
+  public:
+    using iterator_category = std::input_iterator_tag;
+    using value_type = const char*;
+
+    iterator(const Segments* const _segments, size_t _cursor)
+        : segments(_segments), cursor(_cursor) {}
+
+    iterator& operator++() {
+      cursor++;
+      return *this;
+    }
+
+    bool operator==(const iterator& that) const {
+      return cursor == that.cursor && segments == that.segments;
+    }
+
+    bool operator!=(const iterator& that) const {
+      return !this->operator==(that);
+    }
+
+    const char* operator*() const { return segments->At(cursor); }
+
+  private:
+    const Segments* const segments;
+    size_t cursor;
+  };
+
+  const char* At(size_t cursor) const {
+    const auto& index = indexes[cursor];
+    if (index.second) {
+      return managed[index.first].c_str();
+    } else {
+      return unmanaged[index.first];
+    }
+  }
+
+  size_t Length() const { return indexes.size(); }
+
+  iterator begin() const { return iterator(this, 0); }
+
+  iterator end() const { return iterator(this, indexes.size()); }
+
+  std::string ToString() const {
+    // TODO implement a nested structure to reduce concatenation,
+    // like a purely functional differential list
+    std::ostringstream buffer;
+    for (const char* segment : *this) {
+      buffer << segment;
+    }
+    return buffer.str();
+  }
+
+private:
+  Segments(const Segments&) {}
+
+  std::vector<const char*> unmanaged;
+  std::vector<std::string> managed;
+  // index, managed
+  std::vector<std::pair<size_t, bool>> indexes;
+};
+} // namespace opencc

opencc/clib/include/opencc/SerializableDict.hpp

@@ -0,0 +1,77 @@
+/*
+ * Open Chinese Convert
+ *
+ * Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "Dict.hpp"
+
+namespace opencc {
+/**
+ * Serializable dictionary interface
+ * @ingroup opencc_cpp_api
+ */
+class OPENCC_EXPORT SerializableDict {
+public:
+  /**
+   * Serializes the dictionary and writes in to a file.
+   */
+  virtual void SerializeToFile(FILE* fp) const = 0;
+
+  /**
+   * Serializes the dictionary and writes in to a file.
+   */
+  virtual void SerializeToFile(const std::string& fileName) const {
+    FILE* fp = fopen(fileName.c_str(), "wb");
+    if (fp == NULL) {
+      throw FileNotWritable(fileName);
+    }
+    SerializeToFile(fp);
+    fclose(fp);
+  }
+
+  template <typename DICT>
+  static bool TryLoadFromFile(const std::string& fileName,
+                              std::shared_ptr<DICT>* dict) {
+    FILE* fp =
+#ifdef _MSC_VER
+        // well, the 'GetPlatformString' shall return a 'wstring'
+        _wfopen(UTF8Util::GetPlatformString(fileName).c_str(), L"rb")
+#else
+        fopen(UTF8Util::GetPlatformString(fileName).c_str(), "rb")
+#endif // _MSC_VER
+        ;
+
+    if (fp == NULL) {
+      return false;
+    }
+    std::shared_ptr<DICT> loadedDict = DICT::NewFromFile(fp);
+    fclose(fp);
+    *dict = loadedDict;
+    return true;
+  }
+
+  template <typename DICT>
+  static std::shared_ptr<DICT> NewFromFile(const std::string& fileName) {
+    std::shared_ptr<DICT> dict;
+    if (!TryLoadFromFile<DICT>(fileName, &dict)) {
+      throw FileNotFound(fileName);
+    }
+    return dict;
+  }
+};
+} // namespace opencc

opencc/clib/include/opencc/SerializedValues.hpp

@@ -0,0 +1,52 @@
+/*
+ * Open Chinese Convert
+ *
+ * Copyright 2020 Carbo Kuo <byvoid@byvoid.com>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+
+#include "Common.hpp"
+#include "SerializableDict.hpp"
+
+namespace opencc {
+/**
+ * Binary format for dictionary values serialization.
+ * @ingroup opencc_cpp_api
+ */
+class OPENCC_EXPORT SerializedValues : public SerializableDict {
+public:
+  SerializedValues(const LexiconPtr& _lexicon) : lexicon(_lexicon) {}
+
+  virtual ~SerializedValues() {}
+
+  virtual void SerializeToFile(FILE* fp) const;
+
+  static std::shared_ptr<SerializedValues> NewFromFile(FILE* fp);
+
+  const LexiconPtr& GetLexicon() const { return lexicon; }
+
+  size_t KeyMaxLength() const;
+
+private:
+  LexiconPtr lexicon;
+
+  void ConstructBuffer(std::string* valueBuffer,
+                       std::vector<uint16_t>* valueBytes,
+                       uint32_t* valueTotalLength) const;
+};
+} // namespace opencc

opencc/clib/include/opencc/SimpleConverter.hpp

@@ -0,0 +1,113 @@
+/*
+ * Open Chinese Convert
+ *
+ * Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "Export.hpp"
+#include <string>
+#include <vector>
+
+#ifndef __OPENCC_SIMPLECONVERTER_HPP_
+#define __OPENCC_SIMPLECONVERTER_HPP_
+
+/**
+ * @defgroup opencc_simple_api OpenCC C++ Simple API
+ *
+ * Simple API in C++ language
+ */
+
+namespace opencc {
+
+/**
+ * A high level converter
+ * This interface does not require C++11 to compile.
+ * @ingroup opencc_simple_api
+ */
+class OPENCC_EXPORT SimpleConverter {
+public:
+  /**
+   * Constructor of SimpleConverter
+   * @param configFileName File name of configuration.
+   */
+  explicit SimpleConverter(const std::string& configFileName);
+
+  /**
+   * Constructor of SimpleConverter
+   * @param configFileName File name of configuration.
+   * @param paths Additional paths to locate configuration and dictionary files.
+   */
+  SimpleConverter(const std::string& configFileName,
+                  const std::vector<std::string>& paths);
+
+  /**
+   * Constructor of SimpleConverter
+   * @param configFileName File name of configuration.
+   * @param paths Additional paths to locate configuration and dictionary files.
+   * @param argv0 Path of the executable (argv[0]), in addition to additional
+   * paths.
+   */
+  SimpleConverter(const std::string& configFileName,
+                  const std::vector<std::string>& paths, const char* argv0);
+
+  ~SimpleConverter();
+
+  /**
+   * Converts a text
+   * @param input Text to be converted.
+   */
+  std::string Convert(const std::string& input) const;
+
+  /**
+   * Converts a text
+   * @param input A C-Style std::string (terminated by '\0') to be converted.
+   */
+  std::string Convert(const char* input) const;
+
+  /**
+   * Converts a text
+   * @param input  A C-Style std::string limited by a given length to be
+   * converted.
+   * @param length Maximal length in byte of the input std::string.
+   */
+  std::string Convert(const char* input, size_t length) const;
+
+  /**
+   * Converts a text and writes to an allocated buffer
+   * Please make sure the buffer has sufficient space.
+   * @param input  A C-Style std::string (terminated by '\0') to be converted.
+   * @param output Buffer to write the converted text.
+   * @return       Length of converted text.
+   */
+  size_t Convert(const char* input, char* output) const;
+
+  /**
+   * Converts a text and writes to an allocated buffer
+   * Please make sure the buffer has sufficient space.
+   * @param input  A C-Style std::string limited by a given length to be
+   * converted.
+   * @param length Maximal length in byte of the input std::string.
+   * @param output Buffer to write the converted text.
+   * @return       Length of converted text.
+   */
+  size_t Convert(const char* input, size_t length, char* output) const;
+
+private:
+  const void* internalData;
+};
+
+} // namespace opencc
+
+#endif

opencc/clib/include/opencc/TextDict.hpp

@@ -0,0 +1,60 @@
+/*
+ * Open Chinese Convert
+ *
+ * Copyright 2010-2020 Carbo Kuo <byvoid@byvoid.com>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "Common.hpp"
+#include "SerializableDict.hpp"
+
+namespace opencc {
+/**
+ * Text dictionary
+ * @ingroup opencc_cpp_api
+ */
+class OPENCC_EXPORT TextDict : public Dict, public SerializableDict {
+public:
+  /**
+   * Constructor of TextDict.
+   * _lexicon must be sorted.
+   */
+  TextDict(const LexiconPtr& _lexicon);
+
+  virtual ~TextDict();
+
+  virtual size_t KeyMaxLength() const;
+
+  virtual Optional<const DictEntry*> Match(const char* word, size_t len) const;
+
+  virtual LexiconPtr GetLexicon() const;
+
+  virtual void SerializeToFile(FILE* fp) const;
+
+  /**
+   * Constructs a TextDict from another dictionary.
+   */
+  static TextDictPtr NewFromDict(const Dict& dict);
+
+  static TextDictPtr NewFromFile(FILE* fp);
+
+  static TextDictPtr NewFromSortedFile(FILE* fp);
+
+private:
+  const size_t maxLength;
+  const LexiconPtr lexicon;
+};
+} // namespace opencc