react-native-jieba 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/CHANGELOG.md +51 -0
  2. package/Jieba.podspec +2 -0
  3. package/README.md +28 -13
  4. package/android/CMakeLists.txt +2 -0
  5. package/android/generated/java/com/jieba/NativeJiebaSpec.java +4 -0
  6. package/android/generated/jni/JiebaSpec-generated.cpp +6 -0
  7. package/android/generated/jni/react/renderer/components/JiebaSpec/JiebaSpecJSI.h +8 -0
  8. package/android/src/main/java/com/jieba/JiebaAndroidHelperModule.kt +7 -22
  9. package/android/src/main/java/com/jieba/JiebaDict.kt +72 -0
  10. package/android/src/main/java/com/jieba/JiebaPackage.kt +4 -0
  11. package/cpp/JiebaDictAndroid.cpp +26 -0
  12. package/cpp/JiebaDictAndroid.h +14 -0
  13. package/cpp/JiebaImpl.cpp +23 -0
  14. package/cpp/JiebaImpl.h +1 -0
  15. package/cpp/cppjieba/CHANGELOG.md +18 -0
  16. package/cpp/cppjieba/README.md +288 -0
  17. package/cpp/cppjieba/deps/limonp/CHANGELOG.md +175 -0
  18. package/cpp/cppjieba/deps/limonp/README.md +43 -0
  19. package/cpp/cppjieba/dict/README.md +62 -0
  20. package/ios/generated/ReactCodegen/JiebaSpec/JiebaSpec-generated.mm +7 -0
  21. package/ios/generated/ReactCodegen/JiebaSpec/JiebaSpec.h +1 -0
  22. package/ios/generated/ReactCodegen/JiebaSpecJSI.h +8 -0
  23. package/lib/module/NativeJieba.js.map +1 -1
  24. package/lib/module/NativeJieba.web.js +8 -2
  25. package/lib/module/NativeJieba.web.js.map +1 -1
  26. package/lib/module/index.js +1 -1
  27. package/lib/module/index.js.map +1 -1
  28. package/lib/module/init.js +33 -6
  29. package/lib/module/init.js.map +1 -1
  30. package/lib/module/init.web.js +19 -7
  31. package/lib/module/init.web.js.map +1 -1
  32. package/lib/typescript/src/NativeJieba.d.ts +1 -0
  33. package/lib/typescript/src/NativeJieba.d.ts.map +1 -1
  34. package/lib/typescript/src/NativeJieba.web.d.ts +1 -0
  35. package/lib/typescript/src/NativeJieba.web.d.ts.map +1 -1
  36. package/lib/typescript/src/index.d.ts +2 -1
  37. package/lib/typescript/src/index.d.ts.map +1 -1
  38. package/lib/typescript/src/init.d.ts +25 -2
  39. package/lib/typescript/src/init.d.ts.map +1 -1
  40. package/lib/typescript/src/init.web.d.ts +17 -3
  41. package/lib/typescript/src/init.web.d.ts.map +1 -1
  42. package/package.json +6 -1
  43. package/src/NativeJieba.ts +1 -0
  44. package/src/NativeJieba.web.ts +11 -3
  45. package/src/index.tsx +2 -1
  46. package/src/init.ts +35 -7
  47. package/src/init.web.ts +30 -9
@@ -0,0 +1,288 @@
1
+ # CppJieba
2
+
3
+ [![CMake](https://github.com/yanyiwu/cppjieba/actions/workflows/cmake.yml/badge.svg)](https://github.com/yanyiwu/cppjieba/actions/workflows/cmake.yml)
4
+ [![Author](https://img.shields.io/badge/author-@yanyiwu-blue.svg?style=flat)](http://yanyiwu.com/)
5
+ [![Platform](https://img.shields.io/badge/platform-Linux,macOS,Windows-green.svg?style=flat)](https://github.com/yanyiwu/cppjieba)
6
+ [![Performance](https://img.shields.io/badge/performance-excellent-brightgreen.svg?style=flat)](http://yanyiwu.com/work/2015/06/14/jieba-series-performance-test.html)
7
+ [![Tag](https://img.shields.io/github/v/tag/yanyiwu/cppjieba.svg)](https://github.com/yanyiwu/cppjieba/releases)
8
+
9
+ ## 简介
10
+
11
+ CppJieba是"结巴(Jieba)"中文分词的C++版本
12
+
13
+ ### 主要特点
14
+
15
+ - 🚀 高性能:经过线上环境验证的稳定性和性能表现
16
+ - 📦 易集成:源代码以头文件形式提供 (`include/cppjieba/*.hpp`),包含即可使用
17
+ - 🔍 多种分词模式:支持精确模式、全模式、搜索引擎模式等
18
+ - 📚 自定义词典:支持用户自定义词典,支持多词典路径(使用'|'或';'分隔)
19
+ - 💻 跨平台:支持 Linux、macOS、Windows 操作系统
20
+ - 🌈 UTF-8编码:原生支持 UTF-8 编码的中文处理
21
+
22
+ ## 快速开始
23
+
24
+ ### 环境要求
25
+
26
+ - C++ 编译器:
27
+ - g++ (推荐 4.1 以上版本)
28
+ - 或 clang++
29
+ - cmake (推荐 2.6 以上版本)
30
+
31
+ ### 安装步骤
32
+
33
+ ```sh
34
+ git clone https://github.com/yanyiwu/cppjieba.git
35
+ cd cppjieba
36
+ git submodule init
37
+ git submodule update
38
+ mkdir build
39
+ cd build
40
+ cmake ..
41
+ make
42
+
43
+ make test
44
+ ```
45
+
46
+ ### Benchmark
47
+
48
+ 项目提供了一个用于本地性能对比的 benchmark 目标:
49
+
50
+ ```sh
51
+ make benchmark
52
+ ```
53
+
54
+ 它会构建并运行 `test/benchmark.cpp`,输出以下指标:
55
+
56
+ - `DictTrieLoad`: 词典加载耗时,以及可用时的进程 RSS 内存占用
57
+ - `HMMModelLoad`: HMM 模型加载耗时,以及可用时的进程 RSS 内存占用
58
+ - `MPCut`: `MPSegment` 对基准文本的分词吞吐
59
+ - `MixCut`: `MixSegment` 对基准文本的分词吞吐
60
+ - `DictFind`: 词典查找吞吐
61
+
62
+ 这个 benchmark 主要用于本地修改前后的性能回归和对比,不作为默认测试的一部分。
63
+
64
+ ## 使用示例
65
+
66
+ ```
67
+ ./demo
68
+ ```
69
+
70
+ 结果示例:
71
+
72
+ ```
73
+ [demo] Cut With HMM
74
+ 他/来到/了/网易/杭研/大厦
75
+ [demo] Cut Without HMM
76
+ 他/来到/了/网易/杭/研/大厦
77
+ 我来到北京清华大学
78
+ [demo] CutAll
79
+ 我/来到/北京/清华/清华大学/华大/大学
80
+ 小明硕士毕业于中国科学院计算所,后在日本京都大学深造
81
+ [demo] CutForSearch
82
+ 小明/硕士/毕业/于/中国/科学/学院/科学院/中国科学院/计算/计算所/,/后/在/日本/京都/大学/日本京都大学/深造
83
+ [demo] Insert User Word
84
+ 男默/女泪
85
+ 男默女泪
86
+ [demo] CutForSearch Word With Offset
87
+ [{"word": "小明", "offset": 0}, {"word": "硕士", "offset": 6}, {"word": "毕业", "offset": 12}, {"word": "于", "offset": 18}, {"word": "中国", "offset": 21}, {"word": "科学", "offset": 27}, {"word": "学院", "offset": 30}, {"word": "科学院", "offset": 27}, {"word": "中国科学院", "offset": 21}, {"word": "计算", "offset": 36}, {"word": "计算所", "offset": 36}, {"word": ",", "offset": 45}, {"word": "后", "offset": 48}, {"word": "在", "offset": 51}, {"word": "日本", "offset": 54}, {"word": "京都", "offset": 60}, {"word": "大学", "offset": 66}, {"word": "日本京都大学", "offset": 54}, {"word": "深造", "offset": 72}]
88
+ [demo] Tagging
89
+ 我是拖拉机学院手扶拖拉机专业的。不用多久,我就会升职加薪,当上CEO,走上人生巅峰。
90
+ [我:r, 是:v, 拖拉机:n, 学院:n, 手扶拖拉机:n, 专业:n, 的:uj, 。:x, 不用:v, 多久:m, ,:x, 我:r, 就:d, 会:v, 升职:v, 加薪:nr, ,:x, 当上:t, CEO:eng, ,:x, 走上:v, 人生:n, 巅峰:n, 。:x]
91
+ [demo] Keyword Extraction
92
+ 我是拖拉机学院手扶拖拉机专业的。不用多久,我就会升职加薪,当上CEO,走上人生巅峰。
93
+ [{"word": "CEO", "offset": [93], "weight": 11.7392}, {"word": "升职", "offset": [72], "weight": 10.8562}, {"word": "加薪", "offset": [78], "weight": 10.6426}, {"word": "手扶拖拉机", "offset": [21], "weight": 10.0089}, {"word": "巅峰", "offset": [111], "weight": 9.49396}]
94
+ ```
95
+
96
+ For more details, please see [demo](https://github.com/yanyiwu/cppjieba-demo).
97
+
98
+ ### 分词结果示例
99
+
100
+ **MPSegment**
101
+
102
+ Output:
103
+ ```
104
+ 我来到北京清华大学
105
+ 我/来到/北京/清华大学
106
+
107
+ 他来到了网易杭研大厦
108
+ 他/来到/了/网易/杭/研/大厦
109
+
110
+ 小明硕士毕业于中国科学院计算所,后在日本京都大学深造
111
+ 小/明/硕士/毕业/于/中国科学院/计算所/,/后/在/日本京都大学/深造
112
+
113
+ ```
114
+
115
+ **HMMSegment**
116
+
117
+ ```
118
+ 我来到北京清华大学
119
+ 我来/到/北京/清华大学
120
+
121
+ 他来到了网易杭研大厦
122
+ 他来/到/了/网易/杭/研大厦
123
+
124
+ 小明硕士毕业于中国科学院计算所,后在日本京都大学深造
125
+ 小明/硕士/毕业于/中国/科学院/计算所/,/后/在/日/本/京/都/大/学/深/造
126
+
127
+ ```
128
+
129
+ **MixSegment**
130
+
131
+ ```
132
+ 我来到北京清华大学
133
+ 我/来到/北京/清华大学
134
+
135
+ 他来到了网易杭研大厦
136
+ 他/来到/了/网易/杭研/大厦
137
+
138
+ 小明硕士毕业于中国科学院计算所,后在日本京都大学深造
139
+ 小明/硕士/毕业/于/中国科学院/计算所/,/后/在/日本京都大学/深造
140
+
141
+ ```
142
+
143
+ **FullSegment**
144
+
145
+ ```
146
+ 我来到北京清华大学
147
+ 我/来到/北京/清华/清华大学/华大/大学
148
+
149
+ 他来到了网易杭研大厦
150
+ 他/来到/了/网易/杭/研/大厦
151
+
152
+ 小明硕士毕业于中国科学院计算所,后在日本京都大学深造
153
+ 小/明/硕士/毕业/于/中国/中国科学院/科学/科学院/学院/计算/计算所/,/后/在/日本/日本京都大学/京都/京都大学/大学/深造
154
+
155
+ ```
156
+
157
+ **QuerySegment**
158
+
159
+ ```
160
+ 我来到北京清华大学
161
+ 我/来到/北京/清华/清华大学/华大/大学
162
+
163
+ 他来到了网易杭研大厦
164
+ 他/来到/了/网易/杭研/大厦
165
+
166
+ 小明硕士毕业于中国科学院计算所,后在日本京都大学深造
167
+ 小明/硕士/毕业/于/中国/中国科学院/科学/科学院/学院/计算所/,/后/在/中国/中国科学院/科学/科学院/学院/日本/日本京都大学/京都/京都大学/大学/深造
168
+
169
+ ```
170
+
171
+ 以上依次是MP,HMM,Mix三种方法的效果。
172
+
173
+ 可以看出效果最好的是Mix,也就是融合MP和HMM的切词算法。即可以准确切出词典已有的词,又可以切出像"杭研"这样的未登录词。
174
+
175
+ Full方法切出所有字典里的词语。
176
+
177
+ Query方法先使用Mix方法切词,对于切出来的较长的词再使用Full方法。
178
+
179
+ ### 自定义用户词典
180
+
181
+ 自定义词典示例请看`dict/user.dict.utf8`。
182
+
183
+ 用户词典支持以下三种格式,每行一条,列之间用空格分隔:
184
+
185
+ ```text
186
+ 词语
187
+ 词语 词性
188
+ 词语 词频 词性
189
+ ```
190
+
191
+ - `1` 列: 只提供词语,词频使用默认值,词性为空。
192
+ - `2` 列: 当前解释为 `词语 词性`,词频仍使用默认值。
193
+ - `3` 列: 解释为 `词语 词频 词性`。
194
+ - 多个用户词典文件可以用 `|` 或 `;` 连接后一起传入。
195
+ - 不支持行内注释或额外的列。
196
+
197
+ 主词典 `dict/jieba.dict.utf8` 的格式固定为三列:
198
+
199
+ ```text
200
+ 词语 词频 词性
201
+ ```
202
+
203
+ 其中:
204
+
205
+ - `词频` 在加载时会被转换成对数权重,供概率分词使用。
206
+ - 用户词典如果未显式提供词频,会使用主词典统计出来的默认权重;默认策略是中位数权重。
207
+ - `词性` 在代码中只是原样保存的字符串,没有内置枚举校验;主词典当前实际使用了 `55` 种 tag,例如 `n`、`v`、`nr`、`ns`、`nz`、`x`、`eng` 等。
208
+
209
+ 没有使用自定义用户词典时的结果:
210
+
211
+ ```
212
+ 令狐冲/是/云/计算/行业/的/专家
213
+ ```
214
+
215
+ 使用自定义用户词典时的结果:
216
+
217
+ ```
218
+ 令狐冲/是/云计算/行业/的/专家
219
+ ```
220
+
221
+ ### 关键词抽取
222
+
223
+ ```
224
+ 我是拖拉机学院手扶拖拉机专业的。不用多久,我就会升职加薪,当上CEO,走上人生巅峰。
225
+ ["CEO:11.7392", "升职:10.8562", "加薪:10.6426", "手扶拖拉机:10.0089", "巅峰:9.49396"]
226
+ ```
227
+
228
+ For more details, please see [demo](https://github.com/yanyiwu/cppjieba-demo).
229
+
230
+ ### 词性标注
231
+
232
+ ```
233
+ 我是蓝翔技工拖拉机学院手扶拖拉机专业的。不用多久,我就会升职加薪,当上总经理,出任CEO,迎娶白富美,走上人生巅峰。
234
+ ["我:r", "是:v", "拖拉机:n", "学院:n", "手扶拖拉机:n", "专业:n", "的:uj", "。:x", "不用:v", "多久:m", ",:x", "我:r", "就:d", "会:v", "升职:v", "加薪:nr", ",:x", "当上:t", "CEO:eng", ",:x", "走上:v", "人生:n", "巅峰:n", "。:x"]
235
+ ```
236
+
237
+ For more details, please see [demo](https://github.com/yanyiwu/cppjieba-demo).
238
+
239
+ 支持自定义词性。
240
+ 比如在(`dict/user.dict.utf8`)增加一行
241
+
242
+ ```
243
+ 蓝翔 nz
244
+ ```
245
+
246
+ 结果如下:
247
+
248
+ ```
249
+ ["我:r", "是:v", "蓝翔:nz", "技工:n", "拖拉机:n", "学院:n", "手扶拖拉机:n", "专业:n", "的:uj", "。:x", "不用:v", "多久:m", ",:x", "我:r", "就:d", "会:v", "升职:v", "加薪:nr", ",:x", "当:t", "上:f", "总经理:n", ",:x", "出任:v", "CEO:eng", ",:x", "迎娶:v", "白富美:x", ",:x", "走上:v", "人生:n", "巅峰:n", "。:x"]
250
+ ```
251
+
252
+ ## 其它词典资料分享
253
+
254
+ + [dict.367W.utf8] iLife(562193561 at qq.com)
255
+
256
+ ## 生态系统
257
+
258
+ CppJieba 已经被广泛应用于各种编程语言的分词实现中:
259
+
260
+ - [GoJieba](https://github.com/yanyiwu/gojieba) - Go 语言版本
261
+ - [NodeJieba](https://github.com/yanyiwu/nodejieba) - Node.js 版本
262
+ - [CJieba](https://github.com/yanyiwu/cjieba) - C 语言版本
263
+ - [jiebaR](https://github.com/qinwf/jiebaR) - R 语言版本
264
+ - [exjieba](https://github.com/falood/exjieba) - Erlang 版本
265
+ - [jieba_rb](https://github.com/altkatz/jieba_rb) - Ruby 版本
266
+ - [iosjieba](https://github.com/yanyiwu/iosjieba) - iOS 版本
267
+ - [phpjieba](https://github.com/jonnywang/phpjieba) - PHP 版本
268
+ - [perl5-jieba](https://metacpan.org/pod/distribution/Lingua-ZH-Jieba/lib/Lingua/ZH/Jieba.pod) - Perl 版本
269
+
270
+ ### 应用项目
271
+
272
+ - [simhash](https://github.com/yanyiwu/simhash) - 中文文档相似度计算
273
+ - [pg_jieba](https://github.com/jaiminpan/pg_jieba) - PostgreSQL 分词插件
274
+ - [gitbook-plugin-search-pro](https://plugins.gitbook.com/plugin/search-pro) - Gitbook 中文搜索插件
275
+ - [ngx_http_cppjieba_module](https://github.com/yanyiwu/ngx_http_cppjieba_module) - Nginx 分词插件
276
+ - [OpenCC](https://github.com/byvoid/OpenCC) - OpenCC 中的 jieba 分词插件
277
+
278
+ ## 贡献指南
279
+
280
+ 我们欢迎各种形式的贡献,包括但不限于:
281
+
282
+ - 提交问题和建议
283
+ - 改进文档
284
+ - 提交代码修复
285
+ - 添加新功能
286
+
287
+
288
+ 如果您觉得 CppJieba 对您有帮助,欢迎 star ⭐️ 支持项目!
@@ -0,0 +1,175 @@
1
+ # CHANGELOG
2
+
3
+ ## v1.0.2
4
+
5
+ + [fix][Logging] Avoid release-mode unused variable warning while keeping `tmNow` initialization path intact
6
+ + [CI] Remove `macos-13` from CMake matrix to prevent long queued jobs
7
+ + [CI] Upgrade `actions/checkout` from `v4` to `v5`
8
+
9
+ ## v1.0.1
10
+
11
+ + [CI] Update GitHub Actions configurations
12
+ - Add stale issues workflow
13
+ - Update checkout action from v2 to v4
14
+ - Update macOS test environments (remove macOS-12, add macOS-15)
15
+ + [dep] Update googletest to release-1.12.1
16
+ + [doc] Add build instructions to README.md
17
+
18
+ ## v1.0.0
19
+
20
+ + rm thread pool demo
21
+ + deleted: include/limonp/BlockingQueue.hpp
22
+ + deleted: include/limonp/BoundedBlockingQueue.hpp
23
+ + deleted: include/limonp/BoundedQueue.hpp
24
+ + deleted: include/limonp/MutexLock.hpp
25
+ + deleted: include/limonp/Thread.hpp
26
+ + deleted: include/limonp/ThreadPool.hpp
27
+ + deleted: test/unittest/TBlockingQueue.cpp
28
+ + deleted: test/unittest/TBoundedQueue.cpp
29
+ + deleted: test/unittest/TMutexLock.cpp
30
+ + deleted: test/unittest/TThread.cpp
31
+ + deleted: test/unittest/TThreadPool.cpp
32
+ + rm FileLock
33
+ + rm Md5.hpp
34
+
35
+ ## v0.9.0
36
+
37
+ + [c++20] compatibility
38
+ + [c++17] compatibility
39
+
40
+ ## v0.8.1
41
+
42
+ + [CI] fix windows gtest thread link error
43
+ + [submodule] rm test/googletest
44
+ + [CMake] FetchContent googletest
45
+ + [CMake] required 3.5 -> 3.14
46
+
47
+ ## v0.8.0
48
+
49
+ + [StringUtil] Fix windows assert typo
50
+ + [CMake] find_package(Threads REQUIRED); target_link_libraries(... Threads::Threads)
51
+ + [CMAKE][CI] windows: 2019,2022
52
+ + [CMAKE][CI] matrix.build_type[Release, Debug]
53
+ + [unittest] disable #TMd5.cpp
54
+ + [unittest] disable #TFileLock.cpp
55
+ + [unittest] disable #TBoundedQueue.cpp
56
+ + [unittest] disable #TMutexLock.cpp
57
+ + [unittest] disable #TBlockingQueue.cpp
58
+ + [unittest] disable #TThread.cpp
59
+ + [unittest] disable #TThreadPool.cpp
60
+
61
+ ## v0.7.2
62
+
63
+ + [CI] ubuntu version from 20 to 22, macos version from 12 to 14
64
+ + [test/unittes] uint->size_t
65
+ + [googletest] v1.6.0->v1.10.0
66
+ + [CMake] version required 3.0 -> 3.5
67
+
68
+ ## v0.7.1
69
+
70
+ + [CMake] fix CMAKE_CXX_STANDARD passed from github/actions and [c++11, c++14] only
71
+
72
+ ## v0.7.0
73
+
74
+ + [CI] Added os.macos and cpp_version=[c++98, c++03, c++11, c++14, c++17, c++20]
75
+ + [git submodule] Added googletest-release-v1.6.0
76
+
77
+ ## v0.6.7
78
+
79
+ + Merged [pr35](https://github.com/yanyiwu/limonp/pull/35)
80
+ + Merged [pr33](https://github.com/yanyiwu/limonp/pull/33)
81
+ + Merged [pr32](https://github.com/yanyiwu/limonp/pull/32)
82
+
83
+ ## v0.6.6
84
+
85
+ + Merged [pr-31 To be compatible with cpp17 and later, use lambda instead of std::not1 & std::bind2nd #31](https://github.com/yanyiwu/limonp/pull/31)
86
+
87
+ ## v0.6.5
88
+
89
+ + Merged [pr-25 Update cmake.yml](https://github.com/yanyiwu/limonp/pull/25)
90
+ + Merged [pr-26 fix license for end of line sequence and remove useless signature](https://github.com/yanyiwu/limonp/pull/26)
91
+ + Merged [pr-27 Update cmake.yml](https://github.com/yanyiwu/limonp/pull/27)
92
+ + Merged [pr-28 add a target to be ready to support installation](https://github.com/yanyiwu/limonp/pull/28)
93
+ + Merged [pr-29 Installable by cmake](https://github.com/yanyiwu/limonp/pull/29)
94
+ + Merged [pr-30 Replace localtime with localtime_s on Windows and localtime_r on Linux](https://github.com/yanyiwu/limonp/pull/30)
95
+
96
+ ## v0.6.4
97
+
98
+ + merge [fixup gcc8 warnings](https://github.com/yanyiwu/gojieba/pull/70)
99
+
100
+ ## v0.6.3
101
+
102
+ + remove compiler conplained macro
103
+
104
+ ## v0.6.2
105
+
106
+ + merge [pr-18](https://github.com/yanyiwu/limonp/pull/18/files)
107
+
108
+ ## v0.6.1
109
+
110
+ add Specialized template for vector<string>
111
+
112
+ when it is `vector<string>`, print like this: ["hello", "world"]; (special case)
113
+ when it is `vector<int>`, print like this: [1, 10, 1000]; (common cases)
114
+
115
+ ## v0.6.0
116
+
117
+ + remove Trim out of Split.
118
+
119
+ ## v0.5.6
120
+
121
+ + fix hidden trouble.
122
+
123
+ ## v0.5.5
124
+
125
+ + macro name LOG and CHECK in Logging.hpp is so easy to confict with other lib, so I have to rename them to XLOG and XCHECK for avoiding those macro name conflicts.
126
+
127
+ ## v0.5.4
128
+
129
+ + add ForcePublic.hpp
130
+ + Add Utf8ToUnicode32 and Unicode32ToUtf8 in StringUtil.hpp
131
+
132
+ ## v0.5.3
133
+
134
+ + Fix incompatibility problem about 'time.h' in Windows.
135
+
136
+ ## v0.5.2
137
+
138
+ + Fix incompatibility problem about `enum {INFO ...}` name conflicts in Windows .
139
+ + So from this version begin: the compile flags: `-DLOGGING_LEVEL=WARNING` must be changed to `-DLOGGING_LEVEL=LL_WARNING`
140
+
141
+ ## v0.5.1
142
+
143
+ + add `ThreadPool::Stop()` to wait util all the threads finished.
144
+ If Stop() has not been called, it will be called when the ThreadPool destructing.
145
+
146
+ ## v0.5.0
147
+
148
+ + Reorganized directories: include/ -> include/limonp/ ... and so on.
149
+ + Add `NewClosure` in Closure.hpp, 0~3 arguments have been supported.
150
+ + Update ThreadPool, use `NewClosure` instead of `CreateTask`
151
+
152
+ ## v0.4.1
153
+
154
+ + `CHECK(exp) << "log message"` supported;
155
+
156
+ ## v0.4.0
157
+
158
+ + add test/demo.cc as example.
159
+ + move `print` macro to StdExtension.hpp
160
+ + BigChange: rewrite `log` module, use `LOG(INFO) << "xxx" ` instead `LogInfo` .
161
+ + remove HandyMacro.hpp, add CHECK in Logging.hpp instead.
162
+
163
+ ## v0.3.0
164
+
165
+ + remove 'MysqlClient.hpp', 'InitOnOff.hpp', 'CastFloat.hpp'
166
+ + add 'Closure.hpp'
167
+ + uniform code style
168
+
169
+ ## v0.2.0
170
+
171
+ + `namespace limonp`, not `Limonp` .
172
+
173
+ ## v0.1.0
174
+
175
+ + Basic functions
@@ -0,0 +1,43 @@
1
+ # limonp
2
+
3
+ [![CMake](https://github.com/yanyiwu/limonp/actions/workflows/cmake.yml/badge.svg)](https://github.com/yanyiwu/limonp/actions/workflows/cmake.yml)
4
+ [![Author](https://img.shields.io/badge/author-@yanyiwu-blue.svg?style=flat)](http://yanyiwu.com/)
5
+ [![Platform](https://img.shields.io/badge/platform-Linux,macOS,Windows-green.svg?style=flat)](https://github.com/yanyiwu/limonp)
6
+ [![Tag](https://img.shields.io/github/v/tag/yanyiwu/limonp.svg)](https://github.com/yanyiwu/limonp/releases)
7
+ [![License](https://img.shields.io/badge/license-MIT-yellow.svg?style=flat)](http://yanyiwu.mit-license.org)
8
+
9
+ ## Introduction
10
+
11
+ `C++` headers(hpp) with Python style.
12
+
13
+ ## Feature
14
+
15
+ + linking is a annoying thing, so I write these source code in headers file(`*.hpp`), you can use them only with `#include "xx.hpp"`, without linking *.a or *.so .
16
+
17
+ **`no linking , no hurts`**
18
+
19
+ ## Build
20
+
21
+ ```
22
+ mkdir build
23
+ cmake ..
24
+ make
25
+
26
+ make test
27
+ ```
28
+
29
+ ## Example
30
+
31
+ See Details in `test/demo.cpp`
32
+
33
+ ## Cases
34
+
35
+ 1. [CppJieba]
36
+
37
+ ## Contact
38
+
39
+ + i@yanyiwu.com
40
+
41
+ [CppJieba]:https://github.com/yanyiwu/cppjieba.git
42
+ [muduo]:https://github.com/chenshuo/muduo.git
43
+
@@ -0,0 +1,62 @@
1
+ # CppJieba字典
2
+
3
+ 文件后缀名代表的是词典的编码方式。
4
+ 比如filename.utf8 是 utf8编码,filename.gbk 是 gbk编码方式。
5
+
6
+
7
+ ## 分词
8
+
9
+ ### jieba.dict.utf8/gbk
10
+
11
+ 作为最大概率法(MPSegment: Max Probability)分词所使用的词典。
12
+
13
+ 格式固定为:
14
+
15
+ ```text
16
+ 词语 词频 词性
17
+ ```
18
+
19
+ - 每行必须正好三列。
20
+ - `词频` 读取后会换算成对数权重。
21
+ - `词性` 以字符串形式保存在词典节点中,库本身不做枚举校验。
22
+
23
+ ### hmm_model.utf8/gbk
24
+
25
+ 作为隐式马尔科夫模型(HMMSegment: Hidden Markov Model)分词所使用的词典。
26
+
27
+ __对于MixSegment(混合MPSegment和HMMSegment两者)则同时使用以上两个词典__
28
+
29
+ ### user.dict.utf8
30
+
31
+ 用户词典示例。
32
+
33
+ 支持以下三种行格式:
34
+
35
+ ```text
36
+ 词语
37
+ 词语 词性
38
+ 词语 词频 词性
39
+ ```
40
+
41
+ - `1` 列表示只指定词语,使用默认权重,词性为空。
42
+ - `2` 列表示 `词语 词性`。
43
+ - `3` 列表示 `词语 词频 词性`。
44
+ - 可以通过 `|` 或 `;` 传入多个用户词典文件路径。
45
+ - 不支持注释语法、额外列或带空格的词语字段。
46
+
47
+ 当前默认权重来自主词典统计值,默认使用中位数权重;也可以在
48
+ `cppjieba::DictTrie` 构造时通过 `WordWeightMin`、`WordWeightMedian`、
49
+ `WordWeightMax` 调整策略。
50
+
51
+
52
+ ## 关键词抽取
53
+
54
+ ### idf.utf8
55
+
56
+ IDF(Inverse Document Frequency)
57
+ 在KeywordExtractor中,使用的是经典的TF-IDF算法,所以需要这么一个词典提供IDF信息。
58
+
59
+ ### stop_words.utf8
60
+
61
+ 停用词词典
62
+
@@ -26,6 +26,10 @@
26
26
 
27
27
  namespace facebook::react {
28
28
 
29
+ static facebook::jsi::Value __hostFunction_NativeJiebaSpecJSI_isReady(facebook::jsi::Runtime& rt, TurboModule &turboModule, const facebook::jsi::Value* args, size_t count) {
30
+ return static_cast<ObjCTurboModule&>(turboModule).invokeObjCMethod(rt, BooleanKind, "isReady", @selector(isReady), args, count);
31
+ }
32
+
29
33
  static facebook::jsi::Value __hostFunction_NativeJiebaSpecJSI_setDictPath(facebook::jsi::Runtime& rt, TurboModule &turboModule, const facebook::jsi::Value* args, size_t count) {
30
34
  return static_cast<ObjCTurboModule&>(turboModule).invokeObjCMethod(rt, VoidKind, "setDictPath", @selector(setDictPath:), args, count);
31
35
  }
@@ -69,6 +73,9 @@ namespace facebook::react {
69
73
  NativeJiebaSpecJSI::NativeJiebaSpecJSI(const ObjCTurboModule::InitParams &params)
70
74
  : ObjCTurboModule(params) {
71
75
 
76
+ methodMap_["isReady"] = MethodMetadata {0, __hostFunction_NativeJiebaSpecJSI_isReady};
77
+
78
+
72
79
  methodMap_["setDictPath"] = MethodMetadata {1, __hostFunction_NativeJiebaSpecJSI_setDictPath};
73
80
 
74
81
 
@@ -35,6 +35,7 @@ NS_ASSUME_NONNULL_BEGIN
35
35
 
36
36
  @protocol NativeJiebaSpec <RCTBridgeModule, RCTTurboModule>
37
37
 
38
+ - (NSNumber *)isReady;
38
39
  - (void)setDictPath:(NSString *)path;
39
40
  - (NSArray<NSString *> *)cut:(NSString *)sentence
40
41
  hmm:(BOOL)hmm;
@@ -22,6 +22,7 @@ public:
22
22
 
23
23
  protected:
24
24
  NativeJiebaCxxSpec(std::shared_ptr<CallInvoker> jsInvoker) : TurboModule(std::string{NativeJiebaCxxSpec::kModuleName}, jsInvoker) {
25
+ methodMap_["isReady"] = MethodMetadata {.argCount = 0, .invoker = __isReady};
25
26
  methodMap_["setDictPath"] = MethodMetadata {.argCount = 1, .invoker = __setDictPath};
26
27
  methodMap_["cut"] = MethodMetadata {.argCount = 2, .invoker = __cut};
27
28
  methodMap_["cutAll"] = MethodMetadata {.argCount = 1, .invoker = __cutAll};
@@ -35,6 +36,13 @@ protected:
35
36
  }
36
37
 
37
38
  private:
39
+ static jsi::Value __isReady(jsi::Runtime &rt, TurboModule &turboModule, const jsi::Value* /*args*/, size_t /*count*/) {
40
+ static_assert(
41
+ bridging::getParameterCount(&T::isReady) == 1,
42
+ "Expected isReady(...) to have 1 parameters");
43
+ return bridging::callFromJs<bool>(rt, &T::isReady, static_cast<NativeJiebaCxxSpec*>(&turboModule)->jsInvoker_, static_cast<T*>(&turboModule));
44
+ }
45
+
38
46
  static jsi::Value __setDictPath(jsi::Runtime &rt, TurboModule &turboModule, const jsi::Value* args, size_t count) {
39
47
  static_assert(
40
48
  bridging::getParameterCount(&T::setDictPath) == 2,
@@ -1 +1 @@
1
- {"version":3,"names":["TurboModuleRegistry","getEnforcing"],"sourceRoot":"../../src","sources":["NativeJieba.ts"],"mappings":";;AAAA,SAASA,mBAAmB,QAA0B,cAAc;AAkBpE,eAAeA,mBAAmB,CAACC,YAAY,CAAO,OAAO,CAAC","ignoreList":[]}
1
+ {"version":3,"names":["TurboModuleRegistry","getEnforcing"],"sourceRoot":"../../src","sources":["NativeJieba.ts"],"mappings":";;AAAA,SAASA,mBAAmB,QAA0B,cAAc;AAmBpE,eAAeA,mBAAmB,CAACC,YAAY,CAAO,OAAO,CAAC","ignoreList":[]}
@@ -3,20 +3,26 @@
3
3
  // `jieba-wasm` exposes the same segmentation primitives as cppjieba, but a
4
4
  // smaller surface: cut / cutAll / cutForSearch / tag / add_word. The remaining
5
5
  // native methods are polyfilled on top of those where the semantics allow, and
6
- // throw otherwise. The wasm module is loaded lazily by `initJieba()` (see
6
+ // throw otherwise. The wasm module is loaded lazily by `prepareJieba()` (see
7
7
  // `init.web.ts`) and handed to this shim via `setJiebaWasm`.
8
8
 
9
9
  let wasm = null;
10
10
  export function setJiebaWasm(module) {
11
11
  wasm = module;
12
12
  }
13
+ export function isJiebaWasmReady() {
14
+ return wasm !== null;
15
+ }
13
16
  function getWasm() {
14
17
  if (!wasm) {
15
- throw new Error('react-native-jieba: jieba-wasm is not initialized. ' + 'Call (and await) initJieba() before any segmentation call.');
18
+ throw new Error('react-native-jieba: jieba-wasm is not loaded yet. ' + 'On web you must call (and await) prepareJieba() before any ' + 'segmentation call.');
16
19
  }
17
20
  return wasm;
18
21
  }
19
22
  const Jieba = {
23
+ isReady() {
24
+ return isJiebaWasmReady();
25
+ },
20
26
  setDictPath() {
21
27
  // No-op on web: jieba-wasm bundles its own dictionary.
22
28
  },
@@ -1 +1 @@
1
- {"version":3,"names":["wasm","setJiebaWasm","module","getWasm","Error","Jieba","setDictPath","cut","sentence","hmm","cutAll","cut_all","cutForSearch","cut_for_search","cutHMM","cutSmall","_maxWordLen","tag","extract","insertUserWord","word","add_word","undefined","find","segments","length"],"sourceRoot":"../../src","sources":["NativeJieba.web.ts"],"mappings":";;AAEA;AACA;AACA;AACA;AACA;;AAaA,IAAIA,IAA4B,GAAG,IAAI;AAEvC,OAAO,SAASC,YAAYA,CAACC,MAAuB,EAAQ;EAC1DF,IAAI,GAAGE,MAAM;AACf;AAEA,SAASC,OAAOA,CAAA,EAAoB;EAClC,IAAI,CAACH,IAAI,EAAE;IACT,MAAM,IAAII,KAAK,CACb,qDAAqD,GACnD,4DACJ,CAAC;EACH;EACA,OAAOJ,IAAI;AACb;AAEA,MAAMK,KAAW,GAAG;EAClBC,WAAWA,CAAA,EAAG;IACZ;EAAA,CACD;EACDC,GAAGA,CAACC,QAAQ,EAAEC,GAAG,EAAE;IACjB,OAAON,OAAO,CAAC,CAAC,CAACI,GAAG,CAACC,QAAQ,EAAEC,GAAG,CAAC;EACrC,CAAC;EACDC,MAAMA,CAACF,QAAQ,EAAE;IACf,OAAOL,OAAO,CAAC,CAAC,CAACQ,OAAO,CAACH,QAAQ,CAAC;EACpC,CAAC;EACDI,YAAYA,CAACJ,QAAQ,EAAEC,GAAG,EAAE;IAC1B,OAAON,OAAO,CAAC,CAAC,CAACU,cAAc,CAACL,QAAQ,EAAEC,GAAG,CAAC;EAChD,CAAC;EACDK,MAAMA,CAACN,QAAQ,EAAE;IACf;IACA;IACA,OAAOL,OAAO,CAAC,CAAC,CAACI,GAAG,CAACC,QAAQ,EAAE,IAAI,CAAC;EACtC,CAAC;EACDO,QAAQA,CAACP,QAAQ,EAAEQ,WAAW,EAAE;IAC9B;IACA;IACA,OAAOb,OAAO,CAAC,CAAC,CAACI,GAAG,CAACC,QAAQ,EAAE,IAAI,CAAC;EACtC,CAAC;EACDS,GAAGA,CAACT,QAAQ,EAAE;IACZ,OAAOL,OAAO,CAAC,CAAC,CAACc,GAAG,CAACT,QAAQ,CAAC;EAChC,CAAC;EACDU,OAAOA,CAAA,EAAG;IACR,MAAM,IAAId,KAAK,CACb,mEAAmE,GACjE,iEACJ,CAAC;EACH,CAAC;EACDe,cAAcA,CAACC,IAAI,EAAEH,GAAG,EAAE;IACxB;IACA,OACEd,OAAO,CAAC,CAAC,CAACkB,QAAQ,CAACD,IAAI,EAAEE,SAAS,EAAEL,GAAG,KAAK,EAAE,GAAGK,SAAS,GAAGL,GAAG,CAAC,IAAI,CAAC;EAE1E,CAAC;EACDM,IAAIA,CAACH,IAAI,EAAE;IACT;IACA;IACA,MAAMI,QAAQ,GAAGrB,OAAO,CAAC,CAAC,CAACI,GAAG,CAACa,IAAI,EAAE,KAAK,CAAC;IAC3C,OAAOI,QAAQ,CAACC,MAAM,KAAK,CAAC,IAAID,QAAQ,CAAC,CAAC,CAAC,KAAKJ,IAAI;EACtD;AACF,CAAC;AAED,eAAef,KAAK","ignoreList":[]}
1
+ {"version":3,"names":["wasm","setJiebaWasm","module","isJiebaWasmReady","getWasm","Error","Jieba","isReady","setDictPath","cut","sentence","hmm","cutAll","cut_all","cutForSearch","cut_for_search","cutHMM","cutSmall","_maxWordLen","tag","extract","insertUserWord","word","add_word","undefined","find","segments","length"],"sourceRoot":"../../src","sources":["NativeJieba.web.ts"],"mappings":";;AAEA;AACA;AACA;AACA;AACA;;AAaA,IAAIA,IAA4B,GAAG,IAAI;AAEvC,OAAO,SAASC,YAAYA,CAACC,MAAuB,EAAQ;EAC1DF,IAAI,GAAGE,MAAM;AACf;AAEA,OAAO,SAASC,gBAAgBA,CAAA,EAAY;EAC1C,OAAOH,IAAI,KAAK,IAAI;AACtB;AAEA,SAASI,OAAOA,CAAA,EAAoB;EAClC,IAAI,CAACJ,IAAI,EAAE;IACT,MAAM,IAAIK,KAAK,CACb,oDAAoD,GAClD,6DAA6D,GAC7D,oBACJ,CAAC;EACH;EACA,OAAOL,IAAI;AACb;AAEA,MAAMM,KAAW,GAAG;EAClBC,OAAOA,CAAA,EAAG;IACR,OAAOJ,gBAAgB,CAAC,CAAC;EAC3B,CAAC;EACDK,WAAWA,CAAA,EAAG;IACZ;EAAA,CACD;EACDC,GAAGA,CAACC,QAAQ,EAAEC,GAAG,EAAE;IACjB,OAAOP,OAAO,CAAC,CAAC,CAACK,GAAG,CAACC,QAAQ,EAAEC,GAAG,CAAC;EACrC,CAAC;EACDC,MAAMA,CAACF,QAAQ,EAAE;IACf,OAAON,OAAO,CAAC,CAAC,CAACS,OAAO,CAACH,QAAQ,CAAC;EACpC,CAAC;EACDI,YAAYA,CAACJ,QAAQ,EAAEC,GAAG,EAAE;IAC1B,OAAOP,OAAO,CAAC,CAAC,CAACW,cAAc,CAACL,QAAQ,EAAEC,GAAG,CAAC;EAChD,CAAC;EACDK,MAAMA,CAACN,QAAQ,EAAE;IACf;IACA;IACA,OAAON,OAAO,CAAC,CAAC,CAACK,GAAG,CAACC,QAAQ,EAAE,IAAI,CAAC;EACtC,CAAC;EACDO,QAAQA,CAACP,QAAQ,EAAEQ,WAAW,EAAE;IAC9B;IACA;IACA,OAAOd,OAAO,CAAC,CAAC,CAACK,GAAG,CAACC,QAAQ,EAAE,IAAI,CAAC;EACtC,CAAC;EACDS,GAAGA,CAACT,QAAQ,EAAE;IACZ,OAAON,OAAO,CAAC,CAAC,CAACe,GAAG,CAACT,QAAQ,CAAC;EAChC,CAAC;EACDU,OAAOA,CAAA,EAAG;IACR,MAAM,IAAIf,KAAK,CACb,mEAAmE,GACjE,iEACJ,CAAC;EACH,CAAC;EACDgB,cAAcA,CAACC,IAAI,EAAEH,GAAG,EAAE;IACxB;IACA,OACEf,OAAO,CAAC,CAAC,CAACmB,QAAQ,CAACD,IAAI,EAAEE,SAAS,EAAEL,GAAG,KAAK,EAAE,GAAGK,SAAS,GAAGL,GAAG,CAAC,IAAI,CAAC;EAE1E,CAAC;EACDM,IAAIA,CAACH,IAAI,EAAE;IACT;IACA;IACA,MAAMI,QAAQ,GAAGtB,OAAO,CAAC,CAAC,CAACK,GAAG,CAACa,IAAI,EAAE,KAAK,CAAC;IAC3C,OAAOI,QAAQ,CAACC,MAAM,KAAK,CAAC,IAAID,QAAQ,CAAC,CAAC,CAAC,KAAKJ,IAAI;EACtD;AACF,CAAC;AAED,eAAehB,KAAK","ignoreList":[]}
@@ -1,5 +1,5 @@
1
1
  "use strict";
2
2
 
3
3
  export { cut, cutAll, cutForSearch, cutHMM, cutSmall, tag, extract, insertUserWord, find } from "./jieba.js";
4
- export { initJieba } from './init';
4
+ export { prepareJieba, isJiebaReady } from './init';
5
5
  //# sourceMappingURL=index.js.map