react-native-jieba 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Jieba.podspec +30 -0
- package/LICENSE +20 -0
- package/README.md +103 -0
- package/android/CMakeLists.txt +29 -0
- package/android/build.gradle +60 -0
- package/android/generated/java/com/jieba/NativeJiebaAndroidHelperSpec.java +38 -0
- package/android/generated/java/com/jieba/NativeJiebaSpec.java +74 -0
- package/android/generated/jni/CMakeLists.txt +28 -0
- package/android/generated/jni/JiebaSpec-generated.cpp +98 -0
- package/android/generated/jni/JiebaSpec.h +39 -0
- package/android/generated/jni/react/renderer/components/JiebaSpec/JiebaSpecJSI.h +144 -0
- package/android/src/main/AndroidManifest.xml +2 -0
- package/android/src/main/java/com/jieba/JiebaAndroidHelperModule.kt +48 -0
- package/android/src/main/java/com/jieba/JiebaPackage.kt +32 -0
- package/cpp/JiebaImpl.cpp +154 -0
- package/cpp/JiebaImpl.h +37 -0
- package/cpp/cppjieba/LICENSE +20 -0
- package/cpp/cppjieba/deps/limonp/LICENSE +20 -0
- package/cpp/cppjieba/deps/limonp/include/limonp/ArgvContext.hpp +70 -0
- package/cpp/cppjieba/deps/limonp/include/limonp/Closure.hpp +206 -0
- package/cpp/cppjieba/deps/limonp/include/limonp/Colors.hpp +31 -0
- package/cpp/cppjieba/deps/limonp/include/limonp/Condition.hpp +38 -0
- package/cpp/cppjieba/deps/limonp/include/limonp/Config.hpp +103 -0
- package/cpp/cppjieba/deps/limonp/include/limonp/ForcePublic.hpp +7 -0
- package/cpp/cppjieba/deps/limonp/include/limonp/LocalVector.hpp +139 -0
- package/cpp/cppjieba/deps/limonp/include/limonp/Logging.hpp +92 -0
- package/cpp/cppjieba/deps/limonp/include/limonp/NonCopyable.hpp +21 -0
- package/cpp/cppjieba/deps/limonp/include/limonp/StdExtension.hpp +157 -0
- package/cpp/cppjieba/deps/limonp/include/limonp/StringUtil.hpp +386 -0
- package/cpp/cppjieba/dict/hmm_model.utf8 +34 -0
- package/cpp/cppjieba/dict/idf.utf8 +258826 -0
- package/cpp/cppjieba/dict/jieba.dict.utf8 +348982 -0
- package/cpp/cppjieba/dict/stop_words.utf8 +1534 -0
- package/cpp/cppjieba/dict/user.dict.utf8 +3 -0
- package/cpp/cppjieba/include/cppjieba/DictTrie.hpp +322 -0
- package/cpp/cppjieba/include/cppjieba/FullSegment.hpp +94 -0
- package/cpp/cppjieba/include/cppjieba/HMMModel.hpp +131 -0
- package/cpp/cppjieba/include/cppjieba/HMMSegment.hpp +211 -0
- package/cpp/cppjieba/include/cppjieba/Jieba.hpp +169 -0
- package/cpp/cppjieba/include/cppjieba/KeywordExtractor.hpp +152 -0
- package/cpp/cppjieba/include/cppjieba/MPSegment.hpp +137 -0
- package/cpp/cppjieba/include/cppjieba/MixSegment.hpp +109 -0
- package/cpp/cppjieba/include/cppjieba/PosTagger.hpp +77 -0
- package/cpp/cppjieba/include/cppjieba/PreFilter.hpp +54 -0
- package/cpp/cppjieba/include/cppjieba/QuerySegment.hpp +89 -0
- package/cpp/cppjieba/include/cppjieba/SegmentBase.hpp +46 -0
- package/cpp/cppjieba/include/cppjieba/SegmentTagged.hpp +23 -0
- package/cpp/cppjieba/include/cppjieba/TextRankExtractor.hpp +192 -0
- package/cpp/cppjieba/include/cppjieba/Trie.hpp +200 -0
- package/cpp/cppjieba/include/cppjieba/Unicode.hpp +231 -0
- package/cpp/cppjieba/include/cppjieba/UnicodeFile.hpp +68 -0
- package/ios/OnLoad.mm +35 -0
- package/ios/generated/ReactCodegen/JiebaSpec/JiebaSpec-generated.mm +126 -0
- package/ios/generated/ReactCodegen/JiebaSpec/JiebaSpec.h +103 -0
- package/ios/generated/ReactCodegen/JiebaSpecJSI.h +144 -0
- package/lib/module/NativeJieba.js +5 -0
- package/lib/module/NativeJieba.js.map +1 -0
- package/lib/module/NativeJiebaAndroidHelper.js +5 -0
- package/lib/module/NativeJiebaAndroidHelper.js.map +1 -0
- package/lib/module/index.js +5 -0
- package/lib/module/index.js.map +1 -0
- package/lib/module/init.js +24 -0
- package/lib/module/init.js.map +1 -0
- package/lib/module/jieba.js +31 -0
- package/lib/module/jieba.js.map +1 -0
- package/lib/module/package.json +1 -0
- package/lib/typescript/package.json +1 -0
- package/lib/typescript/src/NativeJieba.d.ts +22 -0
- package/lib/typescript/src/NativeJieba.d.ts.map +1 -0
- package/lib/typescript/src/NativeJiebaAndroidHelper.d.ts +7 -0
- package/lib/typescript/src/NativeJiebaAndroidHelper.d.ts.map +1 -0
- package/lib/typescript/src/index.d.ts +4 -0
- package/lib/typescript/src/index.d.ts.map +1 -0
- package/lib/typescript/src/init.d.ts +2 -0
- package/lib/typescript/src/init.d.ts.map +1 -0
- package/lib/typescript/src/jieba.d.ts +18 -0
- package/lib/typescript/src/jieba.d.ts.map +1 -0
- package/package.json +197 -0
- package/react-native.config.js +17 -0
- package/src/NativeJieba.ts +19 -0
- package/src/NativeJiebaAndroidHelper.ts +7 -0
- package/src/index.tsx +13 -0
- package/src/init.ts +25 -0
- package/src/jieba.ts +43 -0
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
#ifndef CPPJIEBA_TRIE_HPP
|
|
2
|
+
#define CPPJIEBA_TRIE_HPP
|
|
3
|
+
|
|
4
|
+
#include <vector>
|
|
5
|
+
#include <queue>
|
|
6
|
+
#include "limonp/StdExtension.hpp"
|
|
7
|
+
#include "Unicode.hpp"
|
|
8
|
+
|
|
9
|
+
namespace cppjieba {
|
|
10
|
+
|
|
11
|
+
using namespace std;
|
|
12
|
+
|
|
13
|
+
const size_t MAX_WORD_LENGTH = 512;
|
|
14
|
+
|
|
15
|
+
struct DictUnit {
|
|
16
|
+
Unicode word;
|
|
17
|
+
double weight;
|
|
18
|
+
string tag;
|
|
19
|
+
}; // struct DictUnit
|
|
20
|
+
|
|
21
|
+
// for debugging
|
|
22
|
+
// inline ostream & operator << (ostream& os, const DictUnit& unit) {
|
|
23
|
+
// string s;
|
|
24
|
+
// s << unit.word;
|
|
25
|
+
// return os << StringFormat("%s %s %.3lf", s.c_str(), unit.tag.c_str(), unit.weight);
|
|
26
|
+
// }
|
|
27
|
+
|
|
28
|
+
struct Dag {
|
|
29
|
+
RuneStr runestr;
|
|
30
|
+
// [offset, nexts.first]
|
|
31
|
+
limonp::LocalVector<pair<size_t, const DictUnit*> > nexts;
|
|
32
|
+
const DictUnit * pInfo;
|
|
33
|
+
double weight;
|
|
34
|
+
size_t nextPos; // TODO
|
|
35
|
+
Dag():runestr(), pInfo(NULL), weight(0.0), nextPos(0) {
|
|
36
|
+
}
|
|
37
|
+
}; // struct Dag
|
|
38
|
+
|
|
39
|
+
typedef Rune TrieKey;
|
|
40
|
+
|
|
41
|
+
class TrieNode {
|
|
42
|
+
public :
|
|
43
|
+
TrieNode(): next(NULL), ptValue(NULL) {
|
|
44
|
+
}
|
|
45
|
+
public:
|
|
46
|
+
typedef unordered_map<TrieKey, TrieNode*> NextMap;
|
|
47
|
+
NextMap *next;
|
|
48
|
+
const DictUnit *ptValue;
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
class Trie {
|
|
52
|
+
public:
|
|
53
|
+
Trie(const vector<Unicode>& keys, const vector<const DictUnit*>& valuePointers)
|
|
54
|
+
: root_(new TrieNode) {
|
|
55
|
+
CreateTrie(keys, valuePointers);
|
|
56
|
+
}
|
|
57
|
+
~Trie() {
|
|
58
|
+
DeleteNode(root_);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
const DictUnit* Find(RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end) const {
|
|
62
|
+
if (begin == end) {
|
|
63
|
+
return NULL;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
const TrieNode* ptNode = root_;
|
|
67
|
+
TrieNode::NextMap::const_iterator citer;
|
|
68
|
+
for (RuneStrArray::const_iterator it = begin; it != end; it++) {
|
|
69
|
+
if (NULL == ptNode->next) {
|
|
70
|
+
return NULL;
|
|
71
|
+
}
|
|
72
|
+
citer = ptNode->next->find(it->rune);
|
|
73
|
+
if (ptNode->next->end() == citer) {
|
|
74
|
+
return NULL;
|
|
75
|
+
}
|
|
76
|
+
ptNode = citer->second;
|
|
77
|
+
}
|
|
78
|
+
return ptNode->ptValue;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
void Find(RuneStrArray::const_iterator begin,
|
|
82
|
+
RuneStrArray::const_iterator end,
|
|
83
|
+
vector<struct Dag>&res,
|
|
84
|
+
size_t max_word_len = MAX_WORD_LENGTH) const {
|
|
85
|
+
assert(root_ != NULL);
|
|
86
|
+
res.resize(end - begin);
|
|
87
|
+
|
|
88
|
+
const TrieNode *ptNode = NULL;
|
|
89
|
+
TrieNode::NextMap::const_iterator citer;
|
|
90
|
+
for (size_t i = 0; i < size_t(end - begin); i++) {
|
|
91
|
+
res[i].runestr = *(begin + i);
|
|
92
|
+
|
|
93
|
+
if (root_->next != NULL && root_->next->end() != (citer = root_->next->find(res[i].runestr.rune))) {
|
|
94
|
+
ptNode = citer->second;
|
|
95
|
+
} else {
|
|
96
|
+
ptNode = NULL;
|
|
97
|
+
}
|
|
98
|
+
if (ptNode != NULL) {
|
|
99
|
+
res[i].nexts.push_back(pair<size_t, const DictUnit*>(i, ptNode->ptValue));
|
|
100
|
+
} else {
|
|
101
|
+
res[i].nexts.push_back(pair<size_t, const DictUnit*>(i, static_cast<const DictUnit*>(NULL)));
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
for (size_t j = i + 1; j < size_t(end - begin) && (j - i + 1) <= max_word_len; j++) {
|
|
105
|
+
if (ptNode == NULL || ptNode->next == NULL) {
|
|
106
|
+
break;
|
|
107
|
+
}
|
|
108
|
+
citer = ptNode->next->find((begin + j)->rune);
|
|
109
|
+
if (ptNode->next->end() == citer) {
|
|
110
|
+
break;
|
|
111
|
+
}
|
|
112
|
+
ptNode = citer->second;
|
|
113
|
+
if (NULL != ptNode->ptValue) {
|
|
114
|
+
res[i].nexts.push_back(pair<size_t, const DictUnit*>(j, ptNode->ptValue));
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
void InsertNode(const Unicode& key, const DictUnit* ptValue) {
|
|
121
|
+
if (key.begin() == key.end()) {
|
|
122
|
+
return;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
TrieNode::NextMap::const_iterator kmIter;
|
|
126
|
+
TrieNode *ptNode = root_;
|
|
127
|
+
for (Unicode::const_iterator citer = key.begin(); citer != key.end(); ++citer) {
|
|
128
|
+
if (NULL == ptNode->next) {
|
|
129
|
+
ptNode->next = new TrieNode::NextMap;
|
|
130
|
+
}
|
|
131
|
+
kmIter = ptNode->next->find(*citer);
|
|
132
|
+
if (ptNode->next->end() == kmIter) {
|
|
133
|
+
TrieNode *nextNode = new TrieNode;
|
|
134
|
+
|
|
135
|
+
ptNode->next->insert(make_pair(*citer, nextNode));
|
|
136
|
+
ptNode = nextNode;
|
|
137
|
+
} else {
|
|
138
|
+
ptNode = kmIter->second;
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
assert(ptNode != NULL);
|
|
142
|
+
ptNode->ptValue = ptValue;
|
|
143
|
+
}
|
|
144
|
+
void DeleteNode(const Unicode& key, const DictUnit* ptValue) {
|
|
145
|
+
if (key.begin() == key.end()) {
|
|
146
|
+
return;
|
|
147
|
+
}
|
|
148
|
+
//定义一个NextMap迭代器
|
|
149
|
+
TrieNode::NextMap::const_iterator kmIter;
|
|
150
|
+
//定义一个指向root的TrieNode指针
|
|
151
|
+
TrieNode *ptNode = root_;
|
|
152
|
+
for (Unicode::const_iterator citer = key.begin(); citer != key.end(); ++citer) {
|
|
153
|
+
//链表不存在元素
|
|
154
|
+
if (NULL == ptNode->next) {
|
|
155
|
+
return;
|
|
156
|
+
}
|
|
157
|
+
kmIter = ptNode->next->find(*citer);
|
|
158
|
+
//如果map中不存在,跳出循环
|
|
159
|
+
if (ptNode->next->end() == kmIter) {
|
|
160
|
+
break;
|
|
161
|
+
}
|
|
162
|
+
//从unordered_map中擦除该项
|
|
163
|
+
ptNode->next->erase(*citer);
|
|
164
|
+
//删除该node
|
|
165
|
+
ptNode = kmIter->second;
|
|
166
|
+
delete ptNode;
|
|
167
|
+
break;
|
|
168
|
+
}
|
|
169
|
+
return;
|
|
170
|
+
}
|
|
171
|
+
private:
|
|
172
|
+
void CreateTrie(const vector<Unicode>& keys, const vector<const DictUnit*>& valuePointers) {
|
|
173
|
+
if (valuePointers.empty() || keys.empty()) {
|
|
174
|
+
return;
|
|
175
|
+
}
|
|
176
|
+
assert(keys.size() == valuePointers.size());
|
|
177
|
+
|
|
178
|
+
for (size_t i = 0; i < keys.size(); i++) {
|
|
179
|
+
InsertNode(keys[i], valuePointers[i]);
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
void DeleteNode(TrieNode* node) {
|
|
184
|
+
if (NULL == node) {
|
|
185
|
+
return;
|
|
186
|
+
}
|
|
187
|
+
if (NULL != node->next) {
|
|
188
|
+
for (TrieNode::NextMap::iterator it = node->next->begin(); it != node->next->end(); ++it) {
|
|
189
|
+
DeleteNode(it->second);
|
|
190
|
+
}
|
|
191
|
+
delete node->next;
|
|
192
|
+
}
|
|
193
|
+
delete node;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
TrieNode* root_;
|
|
197
|
+
}; // class Trie
|
|
198
|
+
} // namespace cppjieba
|
|
199
|
+
|
|
200
|
+
#endif // CPPJIEBA_TRIE_HPP
|
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
#ifndef CPPJIEBA_UNICODE_H
|
|
2
|
+
#define CPPJIEBA_UNICODE_H
|
|
3
|
+
|
|
4
|
+
#include <stdint.h>
|
|
5
|
+
#include <stdlib.h>
|
|
6
|
+
#include <string>
|
|
7
|
+
#include <vector>
|
|
8
|
+
#include <ostream>
|
|
9
|
+
#include "limonp/LocalVector.hpp"
|
|
10
|
+
|
|
11
|
+
namespace cppjieba {
|
|
12
|
+
|
|
13
|
+
using std::string;
|
|
14
|
+
using std::vector;
|
|
15
|
+
|
|
16
|
+
typedef uint32_t Rune;
|
|
17
|
+
|
|
18
|
+
struct Word {
|
|
19
|
+
string word;
|
|
20
|
+
uint32_t offset;
|
|
21
|
+
uint32_t unicode_offset;
|
|
22
|
+
uint32_t unicode_length;
|
|
23
|
+
Word(const string& w, uint32_t o)
|
|
24
|
+
: word(w), offset(o) {
|
|
25
|
+
}
|
|
26
|
+
Word(const string& w, uint32_t o, uint32_t unicode_offset, uint32_t unicode_length)
|
|
27
|
+
: word(w), offset(o), unicode_offset(unicode_offset), unicode_length(unicode_length) {
|
|
28
|
+
}
|
|
29
|
+
}; // struct Word
|
|
30
|
+
|
|
31
|
+
inline std::ostream& operator << (std::ostream& os, const Word& w) {
|
|
32
|
+
return os << "{\"word\": \"" << w.word << "\", \"offset\": " << w.offset << "}";
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
struct RuneStr {
|
|
36
|
+
Rune rune;
|
|
37
|
+
uint32_t offset;
|
|
38
|
+
uint32_t len;
|
|
39
|
+
uint32_t unicode_offset;
|
|
40
|
+
uint32_t unicode_length;
|
|
41
|
+
RuneStr(): rune(0), offset(0), len(0), unicode_offset(0), unicode_length(0) {
|
|
42
|
+
}
|
|
43
|
+
RuneStr(Rune r, uint32_t o, uint32_t l)
|
|
44
|
+
: rune(r), offset(o), len(l), unicode_offset(0), unicode_length(0) {
|
|
45
|
+
}
|
|
46
|
+
RuneStr(Rune r, uint32_t o, uint32_t l, uint32_t unicode_offset, uint32_t unicode_length)
|
|
47
|
+
: rune(r), offset(o), len(l), unicode_offset(unicode_offset), unicode_length(unicode_length) {
|
|
48
|
+
}
|
|
49
|
+
}; // struct RuneStr
|
|
50
|
+
|
|
51
|
+
inline std::ostream& operator << (std::ostream& os, const RuneStr& r) {
|
|
52
|
+
return os << "{\"rune\": \"" << r.rune << "\", \"offset\": " << r.offset << ", \"len\": " << r.len << "}";
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
typedef limonp::LocalVector<Rune> Unicode;
|
|
56
|
+
typedef std::vector<RuneStr> RuneStrArray;
|
|
57
|
+
|
|
58
|
+
// [left, right]
|
|
59
|
+
struct WordRange {
|
|
60
|
+
RuneStrArray::const_iterator left;
|
|
61
|
+
RuneStrArray::const_iterator right;
|
|
62
|
+
WordRange(RuneStrArray::const_iterator l, RuneStrArray::const_iterator r)
|
|
63
|
+
: left(l), right(r) {
|
|
64
|
+
}
|
|
65
|
+
size_t Length() const {
|
|
66
|
+
return right - left + 1;
|
|
67
|
+
}
|
|
68
|
+
bool IsAllAscii() const {
|
|
69
|
+
for (RuneStrArray::const_iterator iter = left; iter <= right; ++iter) {
|
|
70
|
+
if (iter->rune >= 0x80) {
|
|
71
|
+
return false;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
return true;
|
|
75
|
+
}
|
|
76
|
+
}; // struct WordRange
|
|
77
|
+
|
|
78
|
+
struct RuneStrLite {
|
|
79
|
+
uint32_t rune;
|
|
80
|
+
uint32_t len;
|
|
81
|
+
RuneStrLite(): rune(0), len(0) {
|
|
82
|
+
}
|
|
83
|
+
RuneStrLite(uint32_t r, uint32_t l): rune(r), len(l) {
|
|
84
|
+
}
|
|
85
|
+
}; // struct RuneStrLite
|
|
86
|
+
|
|
87
|
+
inline RuneStrLite DecodeUTF8ToRune(const char* str, size_t len) {
|
|
88
|
+
RuneStrLite rp(0, 0);
|
|
89
|
+
if (str == NULL || len == 0) {
|
|
90
|
+
return rp;
|
|
91
|
+
}
|
|
92
|
+
if (!(str[0] & 0x80)) { // 0xxxxxxx
|
|
93
|
+
// 7bit, total 7bit
|
|
94
|
+
rp.rune = (uint8_t)(str[0]) & 0x7f;
|
|
95
|
+
rp.len = 1;
|
|
96
|
+
} else if ((uint8_t)str[0] <= 0xdf && 1 < len) {
|
|
97
|
+
// 110xxxxxx
|
|
98
|
+
// 5bit, total 5bit
|
|
99
|
+
rp.rune = (uint8_t)(str[0]) & 0x1f;
|
|
100
|
+
|
|
101
|
+
// 6bit, total 11bit
|
|
102
|
+
rp.rune <<= 6;
|
|
103
|
+
rp.rune |= (uint8_t)(str[1]) & 0x3f;
|
|
104
|
+
rp.len = 2;
|
|
105
|
+
} else if((uint8_t)str[0] <= 0xef && 2 < len) { // 1110xxxxxx
|
|
106
|
+
// 4bit, total 4bit
|
|
107
|
+
rp.rune = (uint8_t)(str[0]) & 0x0f;
|
|
108
|
+
|
|
109
|
+
// 6bit, total 10bit
|
|
110
|
+
rp.rune <<= 6;
|
|
111
|
+
rp.rune |= (uint8_t)(str[1]) & 0x3f;
|
|
112
|
+
|
|
113
|
+
// 6bit, total 16bit
|
|
114
|
+
rp.rune <<= 6;
|
|
115
|
+
rp.rune |= (uint8_t)(str[2]) & 0x3f;
|
|
116
|
+
|
|
117
|
+
rp.len = 3;
|
|
118
|
+
} else if((uint8_t)str[0] <= 0xf7 && 3 < len) { // 11110xxxx
|
|
119
|
+
// 3bit, total 3bit
|
|
120
|
+
rp.rune = (uint8_t)(str[0]) & 0x07;
|
|
121
|
+
|
|
122
|
+
// 6bit, total 9bit
|
|
123
|
+
rp.rune <<= 6;
|
|
124
|
+
rp.rune |= (uint8_t)(str[1]) & 0x3f;
|
|
125
|
+
|
|
126
|
+
// 6bit, total 15bit
|
|
127
|
+
rp.rune <<= 6;
|
|
128
|
+
rp.rune |= (uint8_t)(str[2]) & 0x3f;
|
|
129
|
+
|
|
130
|
+
// 6bit, total 21bit
|
|
131
|
+
rp.rune <<= 6;
|
|
132
|
+
rp.rune |= (uint8_t)(str[3]) & 0x3f;
|
|
133
|
+
|
|
134
|
+
rp.len = 4;
|
|
135
|
+
} else {
|
|
136
|
+
rp.rune = 0;
|
|
137
|
+
rp.len = 0;
|
|
138
|
+
}
|
|
139
|
+
return rp;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
inline bool DecodeUTF8RunesInString(const char* s, size_t len, RuneStrArray& runes) {
|
|
143
|
+
runes.clear();
|
|
144
|
+
runes.reserve(len / 2);
|
|
145
|
+
for (size_t i = 0, j = 0; i < len;) {
|
|
146
|
+
RuneStrLite rp = DecodeUTF8ToRune(s + i, len - i);
|
|
147
|
+
if (rp.len == 0) {
|
|
148
|
+
runes.clear();
|
|
149
|
+
return false;
|
|
150
|
+
}
|
|
151
|
+
if (i > UINT32_MAX || j > UINT32_MAX) {
|
|
152
|
+
runes.clear();
|
|
153
|
+
return false;
|
|
154
|
+
}
|
|
155
|
+
RuneStr x(rp.rune, static_cast<uint32_t>(i), rp.len, static_cast<uint32_t>(j), 1);
|
|
156
|
+
runes.push_back(x);
|
|
157
|
+
i += rp.len;
|
|
158
|
+
++j;
|
|
159
|
+
}
|
|
160
|
+
return true;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
inline bool DecodeUTF8RunesInString(const string& s, RuneStrArray& runes) {
|
|
164
|
+
return DecodeUTF8RunesInString(s.c_str(), s.size(), runes);
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
inline bool DecodeUTF8RunesInString(const char* s, size_t len, Unicode& unicode) {
|
|
168
|
+
unicode.clear();
|
|
169
|
+
RuneStrArray runes;
|
|
170
|
+
if (!DecodeUTF8RunesInString(s, len, runes)) {
|
|
171
|
+
return false;
|
|
172
|
+
}
|
|
173
|
+
unicode.reserve(runes.size());
|
|
174
|
+
for (size_t i = 0; i < runes.size(); i++) {
|
|
175
|
+
unicode.push_back(runes[i].rune);
|
|
176
|
+
}
|
|
177
|
+
return true;
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
inline bool IsSingleWord(const string& str) {
|
|
181
|
+
RuneStrLite rp = DecodeUTF8ToRune(str.c_str(), str.size());
|
|
182
|
+
return rp.len == str.size();
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
inline bool DecodeUTF8RunesInString(const string& s, Unicode& unicode) {
|
|
186
|
+
return DecodeUTF8RunesInString(s.c_str(), s.size(), unicode);
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
inline Unicode DecodeUTF8RunesInString(const string& s) {
|
|
190
|
+
Unicode result;
|
|
191
|
+
DecodeUTF8RunesInString(s, result);
|
|
192
|
+
return result;
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
// [left, right]
|
|
197
|
+
inline Word GetWordFromRunes(const string& s, RuneStrArray::const_iterator left, RuneStrArray::const_iterator right) {
|
|
198
|
+
assert(right->offset >= left->offset);
|
|
199
|
+
uint32_t len = right->offset - left->offset + right->len;
|
|
200
|
+
uint32_t unicode_length = right->unicode_offset - left->unicode_offset + right->unicode_length;
|
|
201
|
+
return Word(s.substr(left->offset, len), left->offset, left->unicode_offset, unicode_length);
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
inline string GetStringFromRunes(const string& s, RuneStrArray::const_iterator left, RuneStrArray::const_iterator right) {
|
|
205
|
+
assert(right->offset >= left->offset);
|
|
206
|
+
uint32_t len = right->offset - left->offset + right->len;
|
|
207
|
+
return s.substr(left->offset, len);
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
inline void GetWordsFromWordRanges(const string& s, const vector<WordRange>& wrs, vector<Word>& words) {
|
|
211
|
+
for (size_t i = 0; i < wrs.size(); i++) {
|
|
212
|
+
words.push_back(GetWordFromRunes(s, wrs[i].left, wrs[i].right));
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
inline vector<Word> GetWordsFromWordRanges(const string& s, const vector<WordRange>& wrs) {
|
|
217
|
+
vector<Word> result;
|
|
218
|
+
GetWordsFromWordRanges(s, wrs, result);
|
|
219
|
+
return result;
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
inline void GetStringsFromWords(const vector<Word>& words, vector<string>& strs) {
|
|
223
|
+
strs.resize(words.size());
|
|
224
|
+
for (size_t i = 0; i < words.size(); ++i) {
|
|
225
|
+
strs[i] = words[i].word;
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
} // namespace cppjieba
|
|
230
|
+
|
|
231
|
+
#endif // CPPJIEBA_UNICODE_H
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
#ifndef CPPJIEBA_UNICODE_FILE_HPP
|
|
2
|
+
#define CPPJIEBA_UNICODE_FILE_HPP
|
|
3
|
+
|
|
4
|
+
#include <fstream>
|
|
5
|
+
#include <limits>
|
|
6
|
+
#include <string>
|
|
7
|
+
|
|
8
|
+
#ifdef _WIN32
|
|
9
|
+
#ifndef NOMINMAX
|
|
10
|
+
#define NOMINMAX
|
|
11
|
+
#endif
|
|
12
|
+
#ifndef WIN32_LEAN_AND_MEAN
|
|
13
|
+
#define WIN32_LEAN_AND_MEAN
|
|
14
|
+
#endif
|
|
15
|
+
#include <windows.h>
|
|
16
|
+
#endif
|
|
17
|
+
|
|
18
|
+
namespace cppjieba {
|
|
19
|
+
|
|
20
|
+
#ifdef _WIN32
|
|
21
|
+
inline bool Utf8ToWidePath(const std::string& path, std::wstring& widePath) {
|
|
22
|
+
if (path.size() > static_cast<size_t>(std::numeric_limits<int>::max())) {
|
|
23
|
+
return false;
|
|
24
|
+
}
|
|
25
|
+
if (path.empty()) {
|
|
26
|
+
widePath.clear();
|
|
27
|
+
return true;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
const int pathSize = static_cast<int>(path.size());
|
|
31
|
+
const int wideSize = MultiByteToWideChar(
|
|
32
|
+
CP_UTF8,
|
|
33
|
+
MB_ERR_INVALID_CHARS,
|
|
34
|
+
path.data(),
|
|
35
|
+
pathSize,
|
|
36
|
+
NULL,
|
|
37
|
+
0);
|
|
38
|
+
if (wideSize <= 0) {
|
|
39
|
+
return false;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
widePath.resize(static_cast<size_t>(wideSize));
|
|
43
|
+
return MultiByteToWideChar(
|
|
44
|
+
CP_UTF8,
|
|
45
|
+
MB_ERR_INVALID_CHARS,
|
|
46
|
+
path.data(),
|
|
47
|
+
pathSize,
|
|
48
|
+
&widePath[0],
|
|
49
|
+
wideSize) == wideSize;
|
|
50
|
+
}
|
|
51
|
+
#endif
|
|
52
|
+
|
|
53
|
+
inline void OpenInputFile(std::ifstream& ifs, const std::string& path) {
|
|
54
|
+
#ifdef _WIN32
|
|
55
|
+
std::wstring widePath;
|
|
56
|
+
if (Utf8ToWidePath(path, widePath)) {
|
|
57
|
+
ifs.open(widePath.c_str());
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
ifs.setstate(std::ios::failbit);
|
|
61
|
+
return;
|
|
62
|
+
#endif
|
|
63
|
+
ifs.open(path.c_str());
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
} // namespace cppjieba
|
|
67
|
+
|
|
68
|
+
#endif
|
package/ios/OnLoad.mm
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
#import <Foundation/Foundation.h>
|
|
2
|
+
#import "JiebaImpl.h"
|
|
3
|
+
#import <ReactCommon/CxxTurboModuleUtils.h>
|
|
4
|
+
|
|
5
|
+
@interface JiebaOnLoad : NSObject
|
|
6
|
+
@end
|
|
7
|
+
|
|
8
|
+
@implementation JiebaOnLoad
|
|
9
|
+
|
|
10
|
+
using namespace facebook::react;
|
|
11
|
+
|
|
12
|
+
+ (void)load
|
|
13
|
+
{
|
|
14
|
+
NSBundle *bundle = [NSBundle bundleForClass:[JiebaOnLoad class]];
|
|
15
|
+
NSString *dictPath = [bundle pathForResource:@"jieba.dict" ofType:@"utf8"];
|
|
16
|
+
NSString *dictDir = dictPath ? [dictPath stringByDeletingLastPathComponent] : nil;
|
|
17
|
+
if (dictDir == nil) {
|
|
18
|
+
NSString *mainPath = [[NSBundle mainBundle] pathForResource:@"jieba.dict" ofType:@"utf8"];
|
|
19
|
+
if (mainPath != nil) {
|
|
20
|
+
dictDir = [mainPath stringByDeletingLastPathComponent];
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
if (dictDir != nil) {
|
|
24
|
+
JiebaImpl::setDictPathFromNative(std::string([dictDir UTF8String]));
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
registerCxxModuleToGlobalModuleMap(
|
|
28
|
+
std::string(JiebaImpl::kModuleName),
|
|
29
|
+
[](std::shared_ptr<CallInvoker> jsInvoker) {
|
|
30
|
+
return std::make_shared<JiebaImpl>(jsInvoker);
|
|
31
|
+
}
|
|
32
|
+
);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
@end
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This code was generated by [react-native-codegen](https://www.npmjs.com/package/react-native-codegen).
|
|
3
|
+
*
|
|
4
|
+
* Do not edit this file as changes may cause incorrect behavior and will be lost
|
|
5
|
+
* once the code is regenerated.
|
|
6
|
+
*
|
|
7
|
+
* @generated by codegen project: GenerateModuleObjCpp
|
|
8
|
+
*
|
|
9
|
+
* We create an umbrella header (and corresponding implementation) here since
|
|
10
|
+
* Cxx compilation in BUCK has a limitation: source-code producing genrule()s
|
|
11
|
+
* must have a single output. More files => more genrule()s => slower builds.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
#import "JiebaSpec.h"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@implementation NativeJiebaSpecBase
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
- (void)setEventEmitterCallback:(EventEmitterCallbackWrapper *)eventEmitterCallbackWrapper
|
|
21
|
+
{
|
|
22
|
+
_eventEmitterCallback = std::move(eventEmitterCallbackWrapper->_eventEmitterCallback);
|
|
23
|
+
}
|
|
24
|
+
@end
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
namespace facebook::react {
|
|
28
|
+
|
|
29
|
+
static facebook::jsi::Value __hostFunction_NativeJiebaSpecJSI_setDictPath(facebook::jsi::Runtime& rt, TurboModule &turboModule, const facebook::jsi::Value* args, size_t count) {
|
|
30
|
+
return static_cast<ObjCTurboModule&>(turboModule).invokeObjCMethod(rt, VoidKind, "setDictPath", @selector(setDictPath:), args, count);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
static facebook::jsi::Value __hostFunction_NativeJiebaSpecJSI_cut(facebook::jsi::Runtime& rt, TurboModule &turboModule, const facebook::jsi::Value* args, size_t count) {
|
|
34
|
+
return static_cast<ObjCTurboModule&>(turboModule).invokeObjCMethod(rt, ArrayKind, "cut", @selector(cut:hmm:), args, count);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
static facebook::jsi::Value __hostFunction_NativeJiebaSpecJSI_cutAll(facebook::jsi::Runtime& rt, TurboModule &turboModule, const facebook::jsi::Value* args, size_t count) {
|
|
38
|
+
return static_cast<ObjCTurboModule&>(turboModule).invokeObjCMethod(rt, ArrayKind, "cutAll", @selector(cutAll:), args, count);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
static facebook::jsi::Value __hostFunction_NativeJiebaSpecJSI_cutForSearch(facebook::jsi::Runtime& rt, TurboModule &turboModule, const facebook::jsi::Value* args, size_t count) {
|
|
42
|
+
return static_cast<ObjCTurboModule&>(turboModule).invokeObjCMethod(rt, ArrayKind, "cutForSearch", @selector(cutForSearch:hmm:), args, count);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
static facebook::jsi::Value __hostFunction_NativeJiebaSpecJSI_cutHMM(facebook::jsi::Runtime& rt, TurboModule &turboModule, const facebook::jsi::Value* args, size_t count) {
|
|
46
|
+
return static_cast<ObjCTurboModule&>(turboModule).invokeObjCMethod(rt, ArrayKind, "cutHMM", @selector(cutHMM:), args, count);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
static facebook::jsi::Value __hostFunction_NativeJiebaSpecJSI_cutSmall(facebook::jsi::Runtime& rt, TurboModule &turboModule, const facebook::jsi::Value* args, size_t count) {
|
|
50
|
+
return static_cast<ObjCTurboModule&>(turboModule).invokeObjCMethod(rt, ArrayKind, "cutSmall", @selector(cutSmall:maxWordLen:), args, count);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
static facebook::jsi::Value __hostFunction_NativeJiebaSpecJSI_tag(facebook::jsi::Runtime& rt, TurboModule &turboModule, const facebook::jsi::Value* args, size_t count) {
|
|
54
|
+
return static_cast<ObjCTurboModule&>(turboModule).invokeObjCMethod(rt, ArrayKind, "tag", @selector(tag:), args, count);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
static facebook::jsi::Value __hostFunction_NativeJiebaSpecJSI_extract(facebook::jsi::Runtime& rt, TurboModule &turboModule, const facebook::jsi::Value* args, size_t count) {
|
|
58
|
+
return static_cast<ObjCTurboModule&>(turboModule).invokeObjCMethod(rt, ArrayKind, "extract", @selector(extract:topK:), args, count);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
static facebook::jsi::Value __hostFunction_NativeJiebaSpecJSI_insertUserWord(facebook::jsi::Runtime& rt, TurboModule &turboModule, const facebook::jsi::Value* args, size_t count) {
|
|
62
|
+
return static_cast<ObjCTurboModule&>(turboModule).invokeObjCMethod(rt, BooleanKind, "insertUserWord", @selector(insertUserWord:tag:), args, count);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
static facebook::jsi::Value __hostFunction_NativeJiebaSpecJSI_find(facebook::jsi::Runtime& rt, TurboModule &turboModule, const facebook::jsi::Value* args, size_t count) {
|
|
66
|
+
return static_cast<ObjCTurboModule&>(turboModule).invokeObjCMethod(rt, BooleanKind, "find", @selector(find:), args, count);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
NativeJiebaSpecJSI::NativeJiebaSpecJSI(const ObjCTurboModule::InitParams ¶ms)
|
|
70
|
+
: ObjCTurboModule(params) {
|
|
71
|
+
|
|
72
|
+
methodMap_["setDictPath"] = MethodMetadata {1, __hostFunction_NativeJiebaSpecJSI_setDictPath};
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
methodMap_["cut"] = MethodMetadata {2, __hostFunction_NativeJiebaSpecJSI_cut};
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
methodMap_["cutAll"] = MethodMetadata {1, __hostFunction_NativeJiebaSpecJSI_cutAll};
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
methodMap_["cutForSearch"] = MethodMetadata {2, __hostFunction_NativeJiebaSpecJSI_cutForSearch};
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
methodMap_["cutHMM"] = MethodMetadata {1, __hostFunction_NativeJiebaSpecJSI_cutHMM};
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
methodMap_["cutSmall"] = MethodMetadata {2, __hostFunction_NativeJiebaSpecJSI_cutSmall};
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
methodMap_["tag"] = MethodMetadata {1, __hostFunction_NativeJiebaSpecJSI_tag};
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
methodMap_["extract"] = MethodMetadata {2, __hostFunction_NativeJiebaSpecJSI_extract};
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
methodMap_["insertUserWord"] = MethodMetadata {2, __hostFunction_NativeJiebaSpecJSI_insertUserWord};
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
methodMap_["find"] = MethodMetadata {1, __hostFunction_NativeJiebaSpecJSI_find};
|
|
100
|
+
|
|
101
|
+
}
|
|
102
|
+
} // namespace facebook::react
|
|
103
|
+
|
|
104
|
+
@implementation NativeJiebaAndroidHelperSpecBase
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
- (void)setEventEmitterCallback:(EventEmitterCallbackWrapper *)eventEmitterCallbackWrapper
|
|
108
|
+
{
|
|
109
|
+
_eventEmitterCallback = std::move(eventEmitterCallbackWrapper->_eventEmitterCallback);
|
|
110
|
+
}
|
|
111
|
+
@end
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
namespace facebook::react {
|
|
115
|
+
|
|
116
|
+
static facebook::jsi::Value __hostFunction_NativeJiebaAndroidHelperSpecJSI_prepareDictDir(facebook::jsi::Runtime& rt, TurboModule &turboModule, const facebook::jsi::Value* args, size_t count) {
|
|
117
|
+
return static_cast<ObjCTurboModule&>(turboModule).invokeObjCMethod(rt, PromiseKind, "prepareDictDir", @selector(prepareDictDir:reject:), args, count);
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
NativeJiebaAndroidHelperSpecJSI::NativeJiebaAndroidHelperSpecJSI(const ObjCTurboModule::InitParams ¶ms)
|
|
121
|
+
: ObjCTurboModule(params) {
|
|
122
|
+
|
|
123
|
+
methodMap_["prepareDictDir"] = MethodMetadata {0, __hostFunction_NativeJiebaAndroidHelperSpecJSI_prepareDictDir};
|
|
124
|
+
|
|
125
|
+
}
|
|
126
|
+
} // namespace facebook::react
|