OpenCC 1.2.0__cp38-cp38-manylinux2014_x86_64.whl

opencc/__init__.py

@@ -0,0 +1,49 @@
+import os
+
+try:
+    import opencc_clib
+except ImportError:
+    from opencc.clib import opencc_clib
+
+__all__ = ['CONFIGS', 'OpenCC', '__version__']
+
+__version__ = opencc_clib.__version__
+_this_dir = os.path.dirname(os.path.abspath(__file__))
+_opencc_share_dir = os.path.join(_this_dir, 'clib', 'share', 'opencc')
+_opencc_rootdir = os.path.abspath(os.path.join(_this_dir, '..', '..'))
+_opencc_configdir = os.path.join(_opencc_rootdir, 'data', 'config')
+
+if os.path.isdir(_opencc_share_dir):
+    CONFIGS = [f for f in os.listdir(_opencc_share_dir) if f.endswith('.json')]
+elif os.path.isdir(_opencc_configdir):
+    CONFIGS = [f for f in os.listdir(_opencc_configdir) if f.endswith('.json')]
+else:
+    CONFIGS = []
+
+
+def _append_path_to_env(name: str, path: str) -> None:
+    value = os.environ.get(name, '')
+    if path in value:  # Path already exists
+        return
+    if value == '':
+        value = path
+    else:
+        value += f':{path}'
+    os.environ[name] = value
+
+
+class OpenCC(opencc_clib._OpenCC):
+
+    def __init__(self, config: str = 't2s') -> None:
+        if not config.endswith('.json'):
+            config += '.json'
+        if not os.path.isfile(config):
+            config_under_share_dir = os.path.join(_opencc_share_dir, config)
+            if os.path.isfile(config_under_share_dir):
+                config = config_under_share_dir
+        super().__init__(config)
+        self.config = config
+
+    def convert(self, text: str):
+        byte_text = text.encode('utf-8')
+        return super().convert(byte_text, len(byte_text))

opencc/clib/include/opencc/BinaryDict.hpp

@@ -0,0 +1,53 @@
+/*
+ * Open Chinese Convert
+ *
+ * Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "Common.hpp"
+#include "SerializableDict.hpp"
+
+namespace opencc {
+/**
+ * Binary dictionary for faster deserialization
+ * @ingroup opencc_cpp_api
+ */
+class OPENCC_EXPORT BinaryDict : public SerializableDict {
+public:
+  BinaryDict(const LexiconPtr& _lexicon) : lexicon(_lexicon) {}
+
+  virtual ~BinaryDict() {}
+
+  virtual void SerializeToFile(FILE* fp) const;
+
+  static BinaryDictPtr NewFromFile(FILE* fp);
+
+  const LexiconPtr& GetLexicon() const { return lexicon; }
+
+  size_t KeyMaxLength() const;
+
+private:
+  LexiconPtr lexicon;
+  std::string keyBuffer;
+  std::string valueBuffer;
+
+  void ConstructBuffer(std::string& keyBuffer, std::vector<size_t>& keyOffset,
+                       size_t& keyTotalLength, std::string& valueBuffer,
+                       std::vector<size_t>& valueOffset,
+                       size_t& valueTotalLength) const;
+};
+} // namespace opencc

opencc/clib/include/opencc/Common.hpp

@@ -0,0 +1,82 @@
+/*
+ * Open Chinese Convert
+ *
+ * Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+// Microsoft Visual C++ specific
+#if defined(_MSC_VER) && (_MSC_VER >= 1020)
+#pragma warning(disable : 4251 4266 4350 4503 4512 4514 4710 4820)
+#endif
+
+#include <cstddef>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "Export.hpp"
+#include "Optional.hpp"
+#include "opencc_config.h"
+
+// Forward decalarations and alias
+namespace opencc {
+class Config;
+class Conversion;
+class ConversionChain;
+class Converter;
+class Dict;
+class DictEntry;
+class DictGroup;
+class Lexicon;
+class MarisaDict;
+class MultiValueDictEntry;
+class NoValueDictEntry;
+class Segmentation;
+class Segments;
+class SerializableDict;
+class SingleValueDictEntry;
+class TextDict;
+typedef std::shared_ptr<Conversion> ConversionPtr;
+typedef std::shared_ptr<ConversionChain> ConversionChainPtr;
+typedef std::shared_ptr<Converter> ConverterPtr;
+typedef std::shared_ptr<Dict> DictPtr;
+typedef std::shared_ptr<DictGroup> DictGroupPtr;
+typedef std::shared_ptr<Lexicon> LexiconPtr;
+typedef std::shared_ptr<MarisaDict> MarisaDictPtr;
+typedef std::shared_ptr<Segmentation> SegmentationPtr;
+typedef std::shared_ptr<Segments> SegmentsPtr;
+typedef std::shared_ptr<SerializableDict> SerializableDictPtr;
+typedef std::shared_ptr<TextDict> TextDictPtr;
+
+#ifdef OPENCC_ENABLE_DARTS
+class BinaryDict;
+class DartsDict;
+typedef std::shared_ptr<BinaryDict> BinaryDictPtr;
+typedef std::shared_ptr<DartsDict> DartsDictPtr;
+#endif
+
+} // namespace opencc
+
+#ifndef PKGDATADIR
+const std::string PACKAGE_DATA_DIRECTORY = "";
+#else  // ifndef PKGDATADIR
+const std::string PACKAGE_DATA_DIRECTORY = PKGDATADIR "/";
+#endif // ifndef PKGDATADIR
+
+#ifndef VERSION
+#define VERSION "1.0.*"
+#endif // ifndef VERSION

opencc/clib/include/opencc/Config.hpp

@@ -0,0 +1,49 @@
+/*
+ * Open Chinese Convert
+ *
+ * Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "Common.hpp"
+
+namespace opencc {
+/**
+ * Configuration loader
+ * @ingroup opencc_cpp_api
+ */
+class OPENCC_EXPORT Config {
+public:
+  Config();
+
+  virtual ~Config();
+
+  ConverterPtr NewFromString(const std::string& json,
+                             const std::string& configDirectory);
+
+  ConverterPtr NewFromString(const std::string& json,
+                             const std::vector<std::string>& paths);
+
+  ConverterPtr NewFromFile(const std::string& fileName);
+
+  ConverterPtr NewFromFile(const std::string& fileName,
+                           const std::vector<std::string>& paths,
+                           const char* argv0);
+
+private:
+  void* internal;
+};
+} // namespace opencc

opencc/clib/include/opencc/Conversion.hpp

@@ -0,0 +1,47 @@
+/*
+ * Open Chinese Convert
+ *
+ * Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "Common.hpp"
+#include "Segmentation.hpp"
+
+namespace opencc {
+/**
+ * Conversion interface
+ * @ingroup opencc_cpp_api
+ */
+class OPENCC_EXPORT Conversion {
+public:
+  Conversion(DictPtr _dict) : dict(_dict) {}
+
+  // Convert single phrase
+  std::string Convert(const std::string& phrase) const;
+
+  // Convert single phrase
+  std::string Convert(const char* phrase) const;
+
+  // Convert segmented text
+  SegmentsPtr Convert(const SegmentsPtr& input) const;
+
+  const DictPtr GetDict() const { return dict; }
+
+private:
+  const DictPtr dict;
+};
+} // namespace opencc

opencc/clib/include/opencc/ConversionChain.hpp

@@ -0,0 +1,43 @@
+/*
+ * Open Chinese Convert
+ *
+ * Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <list>
+
+#include "Common.hpp"
+#include "Conversion.hpp"
+
+namespace opencc {
+/**
+ * Chain of conversions
+ * Consists of a list of conversions. Converts input in sequence.
+ * @ingroup opencc_cpp_api
+ */
+class OPENCC_EXPORT ConversionChain {
+public:
+  ConversionChain(const std::list<ConversionPtr> _conversions);
+
+  SegmentsPtr Convert(const SegmentsPtr& input) const;
+
+  const std::list<ConversionPtr> GetConversions() const { return conversions; }
+
+private:
+  const std::list<ConversionPtr> conversions;
+};
+} // namespace opencc

opencc/clib/include/opencc/Converter.hpp

@@ -0,0 +1,51 @@
+/*
+ * Open Chinese Convert
+ *
+ * Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "Common.hpp"
+#include "Segmentation.hpp"
+
+namespace opencc {
+/**
+ * Controller of segmentation and conversion
+ * @ingroup opencc_cpp_api
+ */
+class OPENCC_EXPORT Converter {
+public:
+  Converter(const std::string& _name, SegmentationPtr _segmentation,
+            ConversionChainPtr _conversionChain)
+      : name(_name), segmentation(_segmentation),
+        conversionChain(_conversionChain) {}
+
+  std::string Convert(const std::string& text) const;
+
+  size_t Convert(const char* input, char* output) const;
+
+  const SegmentationPtr GetSegmentation() const { return segmentation; }
+
+  const ConversionChainPtr GetConversionChain() const {
+    return conversionChain;
+  }
+
+private:
+  const std::string name;
+  const SegmentationPtr segmentation;
+  const ConversionChainPtr conversionChain;
+};
+} // namespace opencc

opencc/clib/include/opencc/DartsDict.hpp

@@ -0,0 +1,60 @@
+/*
+ * Open Chinese Convert
+ *
+ * Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "Common.hpp"
+#include "SerializableDict.hpp"
+
+namespace opencc {
+/**
+ * Darts dictionary
+ * @ingroup opencc_cpp_api
+ */
+class OPENCC_EXPORT DartsDict : public Dict, public SerializableDict {
+public:
+  virtual ~DartsDict();
+
+  virtual size_t KeyMaxLength() const;
+
+  virtual Optional<const DictEntry*> Match(const char* word, size_t len) const;
+
+  virtual Optional<const DictEntry*> MatchPrefix(const char* word,
+                                                 size_t len) const;
+
+  virtual LexiconPtr GetLexicon() const;
+
+  virtual void SerializeToFile(FILE* fp) const;
+
+  /**
+   * Constructs a DartsDict from another dictionary.
+   */
+  static DartsDictPtr NewFromDict(const Dict& thatDict);
+
+  static DartsDictPtr NewFromFile(FILE* fp);
+
+private:
+  DartsDict();
+
+  size_t maxLength;
+  LexiconPtr lexicon;
+
+  class DartsInternal;
+  DartsInternal* internal;
+};
+} // namespace opencc

opencc/clib/include/opencc/Dict.hpp

@@ -0,0 +1,92 @@
+/*
+ * Open Chinese Convert
+ *
+ * Copyright 2010-2020 Carbo Kuo <byvoid@byvoid.com>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "Common.hpp"
+#include "DictEntry.hpp"
+
+namespace opencc {
+/**
+ * Abstract class of dictionary
+ * @ingroup opencc_cpp_api
+ */
+class OPENCC_EXPORT Dict {
+public:
+  /**
+   * Matches a word exactly and returns the DictEntry or Optional::Null().
+   */
+  virtual Optional<const DictEntry*> Match(const char* word,
+                                           size_t len) const = 0;
+
+  /**
+   * Matches a word exactly and returns the DictEntry or Optional::Null().
+   */
+  Optional<const DictEntry*> Match(const std::string& word) const {
+    return Match(word.c_str(), word.length());
+  }
+
+  /**
+   * Matches the longest matched prefix of a word.
+   * For example given a dictionary having "a", "an", "b", "ba", "ban", "bana",
+   * the longest prefix of "banana" matched is "bana".
+   */
+  virtual Optional<const DictEntry*> MatchPrefix(const char* word,
+                                                 size_t len) const;
+
+  /**
+   * Matches the longest matched prefix of a word.
+   */
+  Optional<const DictEntry*> MatchPrefix(const char* word) const {
+    return MatchPrefix(word, KeyMaxLength());
+  }
+
+  /**
+   * Matches the longest matched prefix of a word.
+   */
+  Optional<const DictEntry*> MatchPrefix(const std::string& word) const {
+    return MatchPrefix(word.c_str(), word.length());
+  }
+
+  /**
+   * Returns all matched prefixes of a word, sorted by the length (desc).
+   * For example given a dictionary having "a", "an", "b", "ba", "ban", "bana",
+   * all the matched prefixes of "banana" are "bana", "ban", "ba", "b".
+   */
+  virtual std::vector<const DictEntry*> MatchAllPrefixes(const char* word,
+                                                         size_t len) const;
+
+  /**
+   * Returns all matched prefixes of a word, sorted by the length (desc).
+   */
+  std::vector<const DictEntry*>
+  MatchAllPrefixes(const std::string& word) const {
+    return MatchAllPrefixes(word.c_str(), word.length());
+  }
+
+  /**
+   * Returns the length of the longest key in the dictionary.
+   */
+  virtual size_t KeyMaxLength() const = 0;
+
+  /**
+   * Returns all entries in the dictionary.
+   */
+  virtual LexiconPtr GetLexicon() const = 0;
+};
+} // namespace opencc

opencc/clib/include/opencc/DictConverter.hpp

@@ -0,0 +1,32 @@
+/*
+ * Open Chinese Convert
+ *
+ * Copyright 2010-2017 Carbo Kuo <byvoid@byvoid.com>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "Common.hpp"
+
+namespace opencc {
+/**
+ * Converts a dictionary from a format to another.
+ * @ingroup opencc_cpp_api
+ */
+OPENCC_EXPORT void ConvertDictionary(const std::string& inputFileName,
+                                     const std::string& outputFileName,
+                                     const std::string& formatFrom,
+                                     const std::string& formatTo);
+} // namespace opencc