OpenCC 1.2.0__cp38-cp38-manylinux2014_x86_64.whl

opencc/clib/include/opencc/DictEntry.hpp

@@ -0,0 +1,173 @@
+/*
+ * Open Chinese Convert
+ *
+ * Copyright 2010-2020 Carbo Kuo <byvoid@byvoid.com>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "Common.hpp"
+#include "Segments.hpp"
+#include "UTF8Util.hpp"
+
+namespace opencc {
+/**
+ * Key-values pair entry
+ * @ingroup opencc_cpp_api
+ */
+class OPENCC_EXPORT DictEntry {
+public:
+  virtual ~DictEntry() {}
+
+  virtual std::string Key() const = 0;
+
+  virtual std::vector<std::string> Values() const = 0;
+
+  virtual std::string GetDefault() const = 0;
+
+  virtual size_t NumValues() const = 0;
+
+  virtual std::string ToString() const = 0;
+
+  size_t KeyLength() const { return Key().length(); }
+
+  bool operator<(const DictEntry& that) const { return Key() < that.Key(); }
+
+  bool operator==(const DictEntry& that) const { return Key() == that.Key(); }
+
+  static bool UPtrLessThan(const std::unique_ptr<DictEntry>& a,
+                           const std::unique_ptr<DictEntry>& b) {
+    return *a < *b;
+  }
+};
+
+class OPENCC_EXPORT NoValueDictEntry : public DictEntry {
+public:
+  NoValueDictEntry(const std::string& _key) : key(_key) {}
+
+  virtual ~NoValueDictEntry() {}
+
+  virtual std::string Key() const { return key; }
+
+  virtual std::vector<std::string> Values() const {
+    return std::vector<std::string>();
+  }
+
+  virtual std::string GetDefault() const { return key; }
+
+  virtual size_t NumValues() const { return 0; }
+
+  virtual std::string ToString() const { return key; }
+
+private:
+  std::string key;
+};
+
+class OPENCC_EXPORT SingleValueDictEntry : public DictEntry {
+public:
+  virtual std::string Value() const = 0;
+
+  virtual std::vector<std::string> Values() const {
+    return std::vector<std::string>{Value()};
+  }
+
+  virtual std::string GetDefault() const { return Value(); }
+
+  virtual size_t NumValues() const { return 1; }
+
+  virtual std::string ToString() const {
+    return std::string(Key()) + "\t" + Value();
+  }
+};
+
+class OPENCC_EXPORT StrSingleValueDictEntry : public SingleValueDictEntry {
+public:
+  StrSingleValueDictEntry(const std::string& _key, const std::string& _value)
+      : key(_key), value(_value) {}
+
+  virtual ~StrSingleValueDictEntry() {}
+
+  virtual std::string Key() const { return key; }
+
+  virtual std::string Value() const { return value; }
+
+private:
+  std::string key;
+  std::string value;
+};
+
+class OPENCC_EXPORT MultiValueDictEntry : public DictEntry {
+public:
+  virtual std::string GetDefault() const {
+    if (NumValues() > 0) {
+      return Values().at(0);
+    } else {
+      return Key();
+    }
+  }
+
+  virtual std::string ToString() const;
+};
+
+class OPENCC_EXPORT StrMultiValueDictEntry : public MultiValueDictEntry {
+public:
+  StrMultiValueDictEntry(const std::string& _key,
+                         const std::vector<std::string>& _values)
+      : key(_key), values(_values) {}
+
+  virtual ~StrMultiValueDictEntry() {}
+
+  virtual std::string Key() const { return key; }
+
+  size_t NumValues() const { return values.size(); }
+
+  std::vector<std::string> Values() const { return values; }
+
+private:
+  std::string key;
+  std::vector<std::string> values;
+};
+
+class OPENCC_EXPORT DictEntryFactory {
+public:
+  static DictEntry* New(const std::string& key) {
+    return new NoValueDictEntry(key);
+  }
+
+  static DictEntry* New(const std::string& key, const std::string& value) {
+    return new StrSingleValueDictEntry(key, value);
+  }
+
+  static DictEntry* New(const std::string& key,
+                        const std::vector<std::string>& values) {
+    if (values.size() == 0) {
+      return New(key);
+    } else if (values.size() == 1) {
+      return New(key, values.front());
+    }
+    return new StrMultiValueDictEntry(key, values);
+  }
+
+  static DictEntry* New(const DictEntry* entry) {
+    if (entry->NumValues() == 0) {
+      return new NoValueDictEntry(entry->Key());
+    } else if (entry->NumValues() == 1) {
+      return new StrSingleValueDictEntry(entry->Key(), entry->Values().front());
+    } else {
+      return new StrMultiValueDictEntry(entry->Key(), entry->Values());
+    }
+  }
+};
+} // namespace opencc

opencc/clib/include/opencc/DictGroup.hpp

@@ -0,0 +1,57 @@
+/*
+ * Open Chinese Convert
+ *
+ * Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <list>
+
+#include "Common.hpp"
+#include "Dict.hpp"
+
+namespace opencc {
+/**
+ * Group of dictionaries
+ * @ingroup opencc_cpp_api
+ */
+class OPENCC_EXPORT DictGroup : public Dict {
+public:
+  DictGroup(const std::list<DictPtr>& dicts);
+
+  static DictGroupPtr NewFromDict(const Dict& dict);
+
+  virtual ~DictGroup();
+
+  virtual size_t KeyMaxLength() const;
+
+  virtual Optional<const DictEntry*> Match(const char* word, size_t len) const;
+
+  virtual Optional<const DictEntry*> MatchPrefix(const char* word,
+                                                 size_t len) const;
+
+  virtual std::vector<const DictEntry*> MatchAllPrefixes(const char* word,
+                                                         size_t len) const;
+
+  virtual LexiconPtr GetLexicon() const;
+
+  const std::list<DictPtr> GetDicts() const { return dicts; }
+
+private:
+  const size_t keyMaxLength;
+  const std::list<DictPtr> dicts;
+};
+} // namespace opencc

opencc/clib/include/opencc/Exception.hpp

@@ -0,0 +1,88 @@
+/*
+ * Open Chinese Convert
+ *
+ * Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <sstream>
+#include <stdexcept>
+#include <string>
+
+#include "Export.hpp"
+
+#if defined(_MSC_VER) && _MSC_VER < 1900
+// Before Visual Studio 2015 (14.0), C++ 11 "noexcept" qualifier is not
+// supported
+#define noexcept
+#endif // ifdef _MSC_VER
+
+namespace opencc {
+
+class OPENCC_EXPORT Exception {
+public:
+  Exception() {}
+
+  virtual ~Exception() throw() {}
+
+  Exception(const std::string& _message) : message(_message) {}
+
+  virtual const char* what() const noexcept { return message.c_str(); }
+
+protected:
+  std::string message;
+};
+
+class OPENCC_EXPORT FileNotFound : public Exception {
+public:
+  FileNotFound(const std::string& fileName)
+      : Exception(fileName + " not found or not accessible.") {}
+};
+
+class OPENCC_EXPORT FileNotWritable : public Exception {
+public:
+  FileNotWritable(const std::string& fileName)
+      : Exception(fileName + " not writable.") {}
+};
+
+class OPENCC_EXPORT InvalidFormat : public Exception {
+public:
+  InvalidFormat(const std::string& message)
+      : Exception("Invalid format: " + message) {}
+};
+
+class OPENCC_EXPORT InvalidTextDictionary : public InvalidFormat {
+public:
+  InvalidTextDictionary(const std::string& _message, size_t lineNum)
+      : InvalidFormat("") {
+    std::ostringstream buffer;
+    buffer << "Invalid text dictionary at line " << lineNum << ": " << _message;
+    message = buffer.str();
+  }
+};
+
+class OPENCC_EXPORT InvalidUTF8 : public Exception {
+public:
+  InvalidUTF8(const std::string& _message)
+      : Exception("Invalid UTF8: " + _message) {}
+};
+
+class OPENCC_EXPORT ShouldNotBeHere : public Exception {
+public:
+  ShouldNotBeHere() : Exception("ShouldNotBeHere! This must be a bug.") {}
+};
+
+} // namespace opencc

opencc/clib/include/opencc/Export.hpp

@@ -0,0 +1,40 @@
+/*
+ * Open Chinese Convert
+ *
+ * Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#if defined(Opencc_BUILT_AS_STATIC) || !defined(_WIN32)
+#define OPENCC_EXPORT
+#define OPENCC_NO_EXPORT
+#else // if defined(Opencc_BUILT_AS_STATIC) || !defined(_WIN32)
+#ifndef OPENCC_EXPORT
+#ifdef libopencc_EXPORTS
+
+/* We are building this library */
+#define OPENCC_EXPORT __declspec(dllexport)
+#else // ifdef libopencc_EXPORTS
+
+/* We are using this library */
+#define OPENCC_EXPORT __declspec(dllimport)
+#endif // ifdef libopencc_EXPORTS
+#endif // ifndef OPENCC_EXPORT
+
+#ifndef OPENCC_NO_EXPORT
+#define OPENCC_NO_EXPORT
+#endif // ifndef OPENCC_NO_EXPORT
+#endif // if defined(Opencc_BUILT_AS_STATIC) || !defined(_WIN32)

opencc/clib/include/opencc/Lexicon.hpp

@@ -0,0 +1,70 @@
+/*
+ * Open Chinese Convert
+ *
+ * Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "Common.hpp"
+#include "DictEntry.hpp"
+
+namespace opencc {
+/**
+ * Storage of all entries
+ * @ingroup opencc_cpp_api
+ */
+class OPENCC_EXPORT Lexicon {
+public:
+  Lexicon() {}
+  Lexicon(std::vector<std::unique_ptr<DictEntry>> entries_)
+      : entries(std::move(entries_)) {}
+  Lexicon(const Lexicon&) = delete;
+  Lexicon& operator=(const Lexicon&) = delete;
+
+  // Lexicon will take the ownership of the entry.
+  void Add(DictEntry* entry) { entries.emplace_back(entry); }
+
+  void Add(std::unique_ptr<DictEntry> entry) {
+    entries.push_back(std::move(entry));
+  }
+
+  void Sort();
+
+  // Returns true if the lexicon is sorted by key.
+  bool IsSorted();
+
+  // Returns true if every key unique (after sorted).
+  // When dupkey is set, it is set to the duplicate key.
+  bool IsUnique(std::string* dupkey = nullptr);
+
+  const DictEntry* At(size_t index) const { return entries.at(index).get(); }
+
+  size_t Length() const { return entries.size(); }
+
+  std::vector<std::unique_ptr<DictEntry>>::const_iterator begin() const {
+    return entries.begin();
+  }
+
+  std::vector<std::unique_ptr<DictEntry>>::const_iterator end() const {
+    return entries.end();
+  }
+
+  static LexiconPtr ParseLexiconFromFile(FILE* fp);
+
+private:
+  std::vector<std::unique_ptr<DictEntry>> entries;
+};
+} // namespace opencc

opencc/clib/include/opencc/MarisaDict.hpp

@@ -0,0 +1,63 @@
+/*
+ * Open Chinese Convert
+ *
+ * Copyright 2020 Carbo Kuo <byvoid@byvoid.com>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "Common.hpp"
+#include "SerializableDict.hpp"
+
+namespace opencc {
+/**
+ * Darts dictionary
+ * @ingroup opencc_cpp_api
+ */
+class OPENCC_EXPORT MarisaDict : public Dict, public SerializableDict {
+public:
+  virtual ~MarisaDict();
+
+  virtual size_t KeyMaxLength() const;
+
+  virtual Optional<const DictEntry*> Match(const char* word, size_t len) const;
+
+  virtual Optional<const DictEntry*> MatchPrefix(const char* word,
+                                                 size_t len) const;
+
+  virtual std::vector<const DictEntry*> MatchAllPrefixes(const char* word,
+                                                         size_t len) const;
+
+  virtual LexiconPtr GetLexicon() const;
+
+  virtual void SerializeToFile(FILE* fp) const;
+
+  /**
+   * Constructs a MarisaDict from another dictionary.
+   */
+  static MarisaDictPtr NewFromDict(const Dict& thatDict);
+
+  static MarisaDictPtr NewFromFile(FILE* fp);
+
+private:
+  MarisaDict();
+
+  size_t maxLength;
+  LexiconPtr lexicon;
+
+  class MarisaInternal;
+  std::unique_ptr<MarisaInternal> internal;
+};
+} // namespace opencc

opencc/clib/include/opencc/MaxMatchSegmentation.hpp

@@ -0,0 +1,43 @@
+/*
+ * Open Chinese Convert
+ *
+ * Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "Common.hpp"
+#include "DictGroup.hpp"
+#include "Segmentation.hpp"
+
+namespace opencc {
+/**
+ * Implementation of maximal match segmentation
+ * @ingroup opencc_cpp_api
+ */
+class OPENCC_EXPORT MaxMatchSegmentation : public Segmentation {
+public:
+  MaxMatchSegmentation(const DictPtr _dict) : dict(_dict) {}
+
+  virtual ~MaxMatchSegmentation() {}
+
+  virtual SegmentsPtr Segment(const std::string& text) const;
+
+  const DictPtr GetDict() const { return dict; }
+
+private:
+  const DictPtr dict;
+};
+} // namespace opencc

opencc/clib/include/opencc/Optional.hpp

@@ -0,0 +1,76 @@
+/*
+ * Open Chinese Convert
+ *
+ * Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+namespace opencc {
+/**
+ * A class that wraps type T into a nullable type.
+ * @ingroup opencc_cpp_api
+ */
+template <typename T> class Optional {
+public:
+  /**
+   * The constructor of Optional.
+   */
+  Optional(T actual) : isNull(false), data(actual) {}
+
+  /**
+   * Returns true if the instance is null.
+   */
+  bool IsNull() const { return isNull; }
+
+  /**
+   * Returns the containing data of the instance.
+   */
+  const T& Get() const { return data; }
+
+  /**
+   * Constructs a null instance.
+   */
+  static Optional<T> Null() { return Optional(); }
+
+private:
+  Optional() : isNull(true) {}
+
+  bool isNull;
+  T data;
+};
+
+/**
+ * Specialization of Optional for pointers.
+ *
+ * Reduce a bool.
+ */
+template <typename T> class Optional<T*> {
+private:
+  Optional() : data(nullptr) {}
+
+  typedef T* TPtr;
+  TPtr data;
+
+public:
+  Optional(TPtr actual) : data(actual) {}
+
+  bool IsNull() const { return data == nullptr; }
+
+  const TPtr& Get() const { return data; }
+
+  static Optional<TPtr> Null() { return Optional(); }
+};
+} // namespace opencc