react-native-jieba 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +51 -0
- package/Jieba.podspec +17 -1
- package/README.md +70 -16
- package/android/CMakeLists.txt +3 -0
- package/android/build.gradle +32 -0
- package/android/generated/java/com/jieba/NativeJiebaSpec.java +4 -0
- package/android/generated/jni/JiebaSpec-generated.cpp +6 -0
- package/android/generated/jni/react/renderer/components/JiebaSpec/JiebaSpecJSI.h +8 -0
- package/android/src/main/java/com/jieba/JiebaAndroidHelperModule.kt +7 -22
- package/android/src/main/java/com/jieba/JiebaDict.kt +82 -0
- package/android/src/main/java/com/jieba/JiebaPackage.kt +4 -0
- package/cpp/JiebaDictAndroid.cpp +26 -0
- package/cpp/JiebaDictAndroid.h +14 -0
- package/cpp/JiebaEngine.cpp +18 -0
- package/cpp/JiebaEngine.h +128 -0
- package/cpp/JiebaImpl.cpp +31 -6
- package/cpp/JiebaImpl.h +4 -3
- package/ios/generated/ReactCodegen/JiebaSpec/JiebaSpec-generated.mm +7 -0
- package/ios/generated/ReactCodegen/JiebaSpec/JiebaSpec.h +1 -0
- package/ios/generated/ReactCodegen/JiebaSpecJSI.h +8 -0
- package/lib/module/NativeJieba.js.map +1 -1
- package/lib/module/NativeJieba.web.js +8 -2
- package/lib/module/NativeJieba.web.js.map +1 -1
- package/lib/module/index.js +1 -1
- package/lib/module/index.js.map +1 -1
- package/lib/module/init.js +33 -6
- package/lib/module/init.js.map +1 -1
- package/lib/module/init.web.js +19 -7
- package/lib/module/init.web.js.map +1 -1
- package/lib/module/jieba.js +12 -0
- package/lib/module/jieba.js.map +1 -1
- package/lib/typescript/src/NativeJieba.d.ts +1 -0
- package/lib/typescript/src/NativeJieba.d.ts.map +1 -1
- package/lib/typescript/src/NativeJieba.web.d.ts +1 -0
- package/lib/typescript/src/NativeJieba.web.d.ts.map +1 -1
- package/lib/typescript/src/index.d.ts +2 -1
- package/lib/typescript/src/index.d.ts.map +1 -1
- package/lib/typescript/src/init.d.ts +25 -2
- package/lib/typescript/src/init.d.ts.map +1 -1
- package/lib/typescript/src/init.web.d.ts +17 -3
- package/lib/typescript/src/init.web.d.ts.map +1 -1
- package/lib/typescript/src/jieba.d.ts +11 -0
- package/lib/typescript/src/jieba.d.ts.map +1 -1
- package/package.json +8 -1
- package/src/NativeJieba.ts +1 -0
- package/src/NativeJieba.web.ts +11 -3
- package/src/index.tsx +2 -1
- package/src/init.ts +35 -7
- package/src/init.web.ts +30 -9
- package/src/jieba.ts +11 -0
- package/cpp/cppjieba/LICENSE +0 -20
- package/cpp/cppjieba/deps/limonp/LICENSE +0 -20
- package/cpp/cppjieba/deps/limonp/include/limonp/ArgvContext.hpp +0 -70
- package/cpp/cppjieba/deps/limonp/include/limonp/Closure.hpp +0 -206
- package/cpp/cppjieba/deps/limonp/include/limonp/Colors.hpp +0 -31
- package/cpp/cppjieba/deps/limonp/include/limonp/Condition.hpp +0 -38
- package/cpp/cppjieba/deps/limonp/include/limonp/Config.hpp +0 -103
- package/cpp/cppjieba/deps/limonp/include/limonp/ForcePublic.hpp +0 -7
- package/cpp/cppjieba/deps/limonp/include/limonp/LocalVector.hpp +0 -139
- package/cpp/cppjieba/deps/limonp/include/limonp/Logging.hpp +0 -92
- package/cpp/cppjieba/deps/limonp/include/limonp/NonCopyable.hpp +0 -21
- package/cpp/cppjieba/deps/limonp/include/limonp/StdExtension.hpp +0 -157
- package/cpp/cppjieba/deps/limonp/include/limonp/StringUtil.hpp +0 -386
- package/cpp/cppjieba/dict/hmm_model.utf8 +0 -34
- package/cpp/cppjieba/dict/idf.utf8 +0 -258826
- package/cpp/cppjieba/dict/jieba.dict.utf8 +0 -348982
- package/cpp/cppjieba/dict/stop_words.utf8 +0 -1534
- package/cpp/cppjieba/dict/user.dict.utf8 +0 -3
- package/cpp/cppjieba/include/cppjieba/DictTrie.hpp +0 -322
- package/cpp/cppjieba/include/cppjieba/FullSegment.hpp +0 -94
- package/cpp/cppjieba/include/cppjieba/HMMModel.hpp +0 -131
- package/cpp/cppjieba/include/cppjieba/HMMSegment.hpp +0 -211
- package/cpp/cppjieba/include/cppjieba/Jieba.hpp +0 -169
- package/cpp/cppjieba/include/cppjieba/KeywordExtractor.hpp +0 -152
- package/cpp/cppjieba/include/cppjieba/MPSegment.hpp +0 -137
- package/cpp/cppjieba/include/cppjieba/MixSegment.hpp +0 -109
- package/cpp/cppjieba/include/cppjieba/PosTagger.hpp +0 -77
- package/cpp/cppjieba/include/cppjieba/PreFilter.hpp +0 -54
- package/cpp/cppjieba/include/cppjieba/QuerySegment.hpp +0 -89
- package/cpp/cppjieba/include/cppjieba/SegmentBase.hpp +0 -46
- package/cpp/cppjieba/include/cppjieba/SegmentTagged.hpp +0 -23
- package/cpp/cppjieba/include/cppjieba/TextRankExtractor.hpp +0 -192
- package/cpp/cppjieba/include/cppjieba/Trie.hpp +0 -200
- package/cpp/cppjieba/include/cppjieba/Unicode.hpp +0 -231
- package/cpp/cppjieba/include/cppjieba/UnicodeFile.hpp +0 -68
package/CHANGELOG.md
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented in this file.
|
|
4
|
+
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
|
+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
|
+
|
|
8
|
+
## 0.3.0
|
|
9
|
+
|
|
10
|
+
### Breaking changes
|
|
11
|
+
|
|
12
|
+
- **`initJieba` has been renamed to `prepareJieba`.** The old name implied a
|
|
13
|
+
mandatory global init step, which it no longer is on native. Update imports and
|
|
14
|
+
calls:
|
|
15
|
+
|
|
16
|
+
```diff
|
|
17
|
+
- import { initJieba } from 'react-native-jieba';
|
|
18
|
+
- await initJieba();
|
|
19
|
+
+ import { prepareJieba } from 'react-native-jieba';
|
|
20
|
+
+ await prepareJieba();
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
On **web** `prepareJieba()` is still required (it loads the `jieba-wasm`
|
|
24
|
+
binary). On **iOS/Android** it is now optional — see below.
|
|
25
|
+
|
|
26
|
+
`InitJiebaOptions` is likewise renamed to `PrepareJiebaOptions`.
|
|
27
|
+
|
|
28
|
+
### Added
|
|
29
|
+
|
|
30
|
+
- **`prepareJieba()` is now optional on native.** You can call `cut`, `tag`, etc.
|
|
31
|
+
directly without any setup — the dictionary is located/extracted automatically on
|
|
32
|
+
first use (iOS resolves it from the app bundle; Android lazily extracts the bundled
|
|
33
|
+
assets to `filesDir/jieba-dict/` via an fbjni bridge). Calling `prepareJieba()`
|
|
34
|
+
remains useful as an async warm-up to avoid a one-time blocking pause on Android's
|
|
35
|
+
first call.
|
|
36
|
+
- **`isJiebaReady(): boolean`** — a synchronous readiness check, backed by the real
|
|
37
|
+
native engine state (a new `isReady()` JSI method) rather than a JS flag, so it
|
|
38
|
+
stays correct even when the dictionary is resolved lazily on the first call.
|
|
39
|
+
|
|
40
|
+
## 0.2.0
|
|
41
|
+
|
|
42
|
+
### Added
|
|
43
|
+
|
|
44
|
+
- Web support via [`jieba-wasm`](https://github.com/fengkx/jieba-wasm) for
|
|
45
|
+
react-native-web, exposing the same API as the native build.
|
|
46
|
+
|
|
47
|
+
## 0.1.0
|
|
48
|
+
|
|
49
|
+
- Initial release: cppjieba C++ Turbo Module for iOS and Android with `cut`,
|
|
50
|
+
`cutAll`, `cutForSearch`, `cutHMM`, `cutSmall`, `tag`, `extract`, `insertUserWord`,
|
|
51
|
+
and `find`.
|
package/Jieba.podspec
CHANGED
|
@@ -14,9 +14,25 @@ Pod::Spec.new do |s|
|
|
|
14
14
|
s.source = { :git => "https://github.com/leonsilicon/react-native-jieba.git", :tag => "#{s.version}" }
|
|
15
15
|
|
|
16
16
|
s.source_files = "ios/**/*.{h,m,mm}", "cpp/*.{hpp,cpp,c,h}", "ios/generated/*.{h,cpp,mm}"
|
|
17
|
+
# JiebaDictAndroid.{cpp,h} depend on fbjni and are Android-only.
|
|
18
|
+
s.exclude_files = "cpp/JiebaDictAndroid.{cpp,h}"
|
|
17
19
|
s.private_header_files = "ios/**/*.h"
|
|
18
20
|
|
|
19
|
-
|
|
21
|
+
# The IDF dictionary (~5MB) is only read by `extract()` (TF-IDF keyword extraction). Apps that
|
|
22
|
+
# only tokenize can drop it from the bundle by setting RN_JIEBA_EXCLUDE_IDF_DICT=1 in the
|
|
23
|
+
# environment that evaluates the Podfile, e.g.
|
|
24
|
+
#
|
|
25
|
+
# ENV["RN_JIEBA_EXCLUDE_IDF_DICT"] = "1" # in the Podfile, before use_react_native!
|
|
26
|
+
#
|
|
27
|
+
# `extract()` then throws a descriptive error, exactly as it already does on web (jieba-wasm
|
|
28
|
+
# ships no IDF data). Every other API is unaffected.
|
|
29
|
+
exclude_idf_dict = ENV["RN_JIEBA_EXCLUDE_IDF_DICT"] == "1"
|
|
30
|
+
dict_files = Dir.glob(File.join(__dir__, "cpp/cppjieba/dict/*.utf8")).map do |path|
|
|
31
|
+
"cpp/cppjieba/dict/#{File.basename(path)}"
|
|
32
|
+
end
|
|
33
|
+
dict_files.reject! { |path| File.basename(path) == "idf.utf8" } if exclude_idf_dict
|
|
34
|
+
|
|
35
|
+
s.resources = dict_files
|
|
20
36
|
|
|
21
37
|
s.pod_target_xcconfig = {
|
|
22
38
|
"HEADER_SEARCH_PATHS" => [
|
package/README.md
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
Native Chinese text segmentation for React Native, powered by [cppjieba](https://github.com/yanyiwu/cppjieba).
|
|
4
4
|
|
|
5
5
|
- Runs on the New Architecture as a C++ Turbo Module — no bridge overhead, no JS port of jieba.
|
|
6
|
-
- Ships the cppjieba dictionaries (`jieba.dict.utf8`, `hmm_model.utf8`, `idf.utf8`, `stop_words.utf8`, `user.dict.utf8`) inside the package, bundled as iOS pod resources and Android assets.
|
|
6
|
+
- Ships the cppjieba dictionaries (`jieba.dict.utf8`, `hmm_model.utf8`, `idf.utf8`, `stop_words.utf8`, `user.dict.utf8`) inside the package, bundled as iOS pod resources and Android assets. The ~5.7 MB `idf.utf8` is loaded lazily and [can be excluded entirely](#reducing-app-size-excluding-the-idf-dictionary) in apps that never call `extract()`.
|
|
7
7
|
- Supports the standard jieba modes: precise (`cut`), full (`cutAll`), search engine (`cutForSearch`), HMM (`cutHMM`), small-word (`cutSmall`), POS tagging (`tag`), and TF-IDF keyword extraction (`extract`).
|
|
8
8
|
- Works on **web** (react-native-web) via [`jieba-wasm`](https://github.com/fengkx/jieba-wasm), with the same API.
|
|
9
9
|
|
|
@@ -48,11 +48,21 @@ export default {
|
|
|
48
48
|
|
|
49
49
|
## Usage
|
|
50
50
|
|
|
51
|
-
|
|
51
|
+
On **native (iOS/Android)** there is no setup — just call `cut`, `tag`, etc. The dictionary is located/extracted automatically: on iOS the path is resolved from the app bundle, and on Android the bundled assets are extracted to `filesDir/jieba-dict/` on the first call.
|
|
52
|
+
|
|
53
|
+
```ts
|
|
54
|
+
import { cut } from 'react-native-jieba';
|
|
55
|
+
|
|
56
|
+
cut('我来到北京清华大学');
|
|
57
|
+
// ['我', '来到', '北京', '清华大学']
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
`prepareJieba()` is an **optional async warm-up** on native: on Android's first run the asset extraction happens off the call site (asynchronously), avoiding a one-time blocking pause on the first `cut`/`tag`. On **web** it is **required** — it loads the `jieba-wasm` binary, which cannot happen synchronously. Use `isJiebaReady()` to check synchronously whether segmentation can run.
|
|
52
61
|
|
|
53
62
|
```ts
|
|
54
63
|
import {
|
|
55
|
-
|
|
64
|
+
prepareJieba,
|
|
65
|
+
isJiebaReady,
|
|
56
66
|
cut,
|
|
57
67
|
cutAll,
|
|
58
68
|
cutForSearch,
|
|
@@ -60,7 +70,10 @@ import {
|
|
|
60
70
|
extract,
|
|
61
71
|
} from 'react-native-jieba';
|
|
62
72
|
|
|
63
|
-
|
|
73
|
+
// Native: optional warm-up. Web: required (loads wasm).
|
|
74
|
+
await prepareJieba();
|
|
75
|
+
|
|
76
|
+
isJiebaReady(); // true
|
|
64
77
|
|
|
65
78
|
cut('我来到北京清华大学');
|
|
66
79
|
// ['我', '来到', '北京', '清华大学']
|
|
@@ -80,18 +93,19 @@ extract('我是拖拉机学院手扶拖拉机专业的。', 5);
|
|
|
80
93
|
|
|
81
94
|
## API
|
|
82
95
|
|
|
83
|
-
On native, all segmentation calls are synchronous JSI calls — no Promises. On web they are synchronous too, but `
|
|
96
|
+
On native, all segmentation calls are synchronous JSI calls — no Promises. On web they are synchronous too, but `prepareJieba()` must finish first (it loads the wasm module).
|
|
84
97
|
|
|
85
98
|
| Function | Signature | Notes |
|
|
86
99
|
| --- | --- | --- |
|
|
87
|
-
| `
|
|
100
|
+
| `prepareJieba(options?)` | `(options?: { wasmUrl?: string \| URL \| Request }) => Promise<void>` | **Web:** must be awaited once before any other call. **Native:** optional warm-up — calling segmentation directly works too. Idempotent. `wasmUrl` is web-only and rarely needed (override where the `.wasm` is served). |
|
|
101
|
+
| `isJiebaReady()` | `() => boolean` | Synchronous check for whether segmentation can run now. Native-backed (reflects the real engine state, not whether `prepareJieba` was called): `true` on iOS from launch, `true` on Android once the dict is resolved (eagerly via `prepareJieba` or lazily on the first call). |
|
|
88
102
|
| `cut(sentence, hmm?)` | `(string, boolean) => string[]` | Precise mode. `hmm` defaults to `true`. |
|
|
89
103
|
| `cutAll(sentence)` | `(string) => string[]` | Full mode (every possible word). |
|
|
90
104
|
| `cutForSearch(sentence, hmm?)` | `(string, boolean) => string[]` | Search-engine mode. |
|
|
91
105
|
| `cutHMM(sentence)` | `(string) => string[]` | HMM-only segmentation. |
|
|
92
106
|
| `cutSmall(sentence, maxWordLen)` | `(string, number) => string[]` | Limits the maximum word length. |
|
|
93
107
|
| `tag(sentence)` | `(string) => Array<{ word, tag }>` | Part-of-speech tagging. |
|
|
94
|
-
| `extract(sentence, topK?)` | `(string, number) => Array<{ word, weight }>` | TF-IDF keyword extraction. `topK` defaults to `5`. |
|
|
108
|
+
| `extract(sentence, topK?)` | `(string, number) => Array<{ word, weight }>` | TF-IDF keyword extraction. `topK` defaults to `5`. Loads the IDF dictionary on first use; throws on web, and in native builds that [excluded it](#reducing-app-size-excluding-the-idf-dictionary). |
|
|
95
109
|
| `insertUserWord(word, tag?)` | `(string, string) => boolean` | Adds a user dictionary word at runtime. |
|
|
96
110
|
| `find(word)` | `(string) => boolean` | Tests whether a word is in the dictionary. |
|
|
97
111
|
|
|
@@ -130,15 +144,15 @@ export default defineConfig({
|
|
|
130
144
|
// App.tsx
|
|
131
145
|
import { useEffect, useState } from 'react';
|
|
132
146
|
import { Platform, Text, View } from 'react-native';
|
|
133
|
-
import {
|
|
147
|
+
import { prepareJieba, cut, cutForSearch, tag, extract } from 'react-native-jieba';
|
|
134
148
|
|
|
135
149
|
export default function App() {
|
|
136
150
|
const [words, setWords] = useState<string[] | null>(null);
|
|
137
151
|
|
|
138
152
|
useEffect(() => {
|
|
139
|
-
// On web this loads + instantiates the wasm module (async).
|
|
140
|
-
// On native
|
|
141
|
-
|
|
153
|
+
// On web this loads + instantiates the wasm module (async) and is required.
|
|
154
|
+
// On native you can skip this and call cut() directly.
|
|
155
|
+
prepareJieba()
|
|
142
156
|
.then(() => {
|
|
143
157
|
setWords(cut('我来到北京清华大学'));
|
|
144
158
|
// → ['我', '来到', '北京', '清华大学']
|
|
@@ -169,16 +183,56 @@ export default function App() {
|
|
|
169
183
|
To self-host the binary (CDN or custom path), pass `wasmUrl`:
|
|
170
184
|
|
|
171
185
|
```ts
|
|
172
|
-
await
|
|
186
|
+
await prepareJieba({ wasmUrl: 'https://cdn.example.com/jieba_rs_wasm_bg.wasm' });
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
## Reducing app size: excluding the IDF dictionary
|
|
190
|
+
|
|
191
|
+
The bundled dictionaries account for most of this library's footprint, and `idf.utf8` is the
|
|
192
|
+
largest single file at **~5.7 MB**. It is read by exactly one API — `extract()`, TF-IDF keyword
|
|
193
|
+
extraction. Apps that only segment text (`cut`, `cutForSearch`, `tag`, …) never touch it.
|
|
194
|
+
|
|
195
|
+
The extractor is built lazily on first use, so the dictionary costs nothing at startup either way.
|
|
196
|
+
If your app never calls `extract()`, you can also drop the file from your binary entirely:
|
|
197
|
+
|
|
198
|
+
**iOS** — set the environment variable before CocoaPods evaluates the podspec, e.g. at the top of
|
|
199
|
+
your `Podfile`:
|
|
200
|
+
|
|
201
|
+
```rb
|
|
202
|
+
ENV['RN_JIEBA_EXCLUDE_IDF_DICT'] = '1'
|
|
203
|
+
```
|
|
204
|
+
|
|
205
|
+
Then reinstall pods (`bundle exec pod install`).
|
|
206
|
+
|
|
207
|
+
**Android** — set a Gradle property in `android/gradle.properties`:
|
|
208
|
+
|
|
209
|
+
```properties
|
|
210
|
+
rnJiebaExcludeIdfDict=true
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
(or `ext.rnJiebaExcludeIdfDict = true` in `android/build.gradle`, or pass
|
|
214
|
+
`-PrnJiebaExcludeIdfDict=true` on the command line).
|
|
215
|
+
|
|
216
|
+
With the dictionary excluded, `extract()` throws a descriptive error and **every other API keeps
|
|
217
|
+
working unchanged** — the same contract that already applies on web, where `extract()` is
|
|
218
|
+
unsupported because jieba-wasm ships no IDF data. Guard it as you would for web:
|
|
219
|
+
|
|
220
|
+
```ts
|
|
221
|
+
const keywords = canExtract ? extract(sentence, 5) : [];
|
|
173
222
|
```
|
|
174
223
|
|
|
224
|
+
On Android this also skips copying the file out of the APK into `filesDir/jieba-dict/` on first
|
|
225
|
+
run, saving the same ~5.7 MB of user device storage and shortening first-call extraction.
|
|
226
|
+
|
|
175
227
|
## How it works
|
|
176
228
|
|
|
177
|
-
- The Turbo Module lives in `cpp/JiebaImpl.{h,cpp}` and
|
|
178
|
-
-
|
|
179
|
-
-
|
|
229
|
+
- The Turbo Module lives in `cpp/JiebaImpl.{h,cpp}` and exposes the engine as a JSI Cxx module.
|
|
230
|
+
- `cpp/JiebaEngine.{h,cpp}` composes cppjieba's primitives (one shared `DictTrie` + `HMMModel` across every segmenter) instead of using the `cppjieba::Jieba` facade. The facade holds a `KeywordExtractor` **by value**, so constructing it always loads `idf.utf8` and hard-fails when that file is absent; composing the pieces directly is what lets the extractor — and its dictionary — be built only if `extract()` is called.
|
|
231
|
+
- iOS resolves the dictionary directory from `NSBundle` in `ios/OnLoad.mm` before the module is registered, so segmentation works immediately and `prepareJieba()` is a no-op on iOS.
|
|
232
|
+
- Android ships the dictionary as AAR assets and extracts them to `filesDir/jieba-dict/`. This happens automatically: if `prepareJieba()` is never called, the C++ module extracts them synchronously on the first segmentation call via an fbjni call into `JiebaDict.extractDictDirFromNative` (a one-time cost). `prepareJieba()` does the same extraction asynchronously up front (through `JiebaAndroidHelperModule` → the codegen-exposed `setDictPath` JSI method) so that first call doesn't block.
|
|
233
|
+
- `isJiebaReady()` is backed by the codegen-exposed `isReady()` JSI method, which reads the native engine state directly — so it stays correct even when the dictionary is resolved lazily on the first call.
|
|
180
234
|
- cppjieba and its `limonp` dependency are vendored as git submodules under `cpp/cppjieba/`. The published npm tarball contains only the headers and dictionaries that are actually needed at build time.
|
|
181
|
-
- On web, `react-native-web`'s bundler resolution picks up `src/NativeJieba.web.ts` and `src/init.web.ts`. `
|
|
235
|
+
- On web, `react-native-web`'s bundler resolution picks up `src/NativeJieba.web.ts` and `src/init.web.ts`. `prepareJieba()` lazily imports `jieba-wasm`, awaits its async wasm initializer, and routes the JS API to the wasm exports.
|
|
182
236
|
|
|
183
237
|
## Contributing
|
|
184
238
|
|
package/android/CMakeLists.txt
CHANGED
|
@@ -6,6 +6,8 @@ set (CMAKE_VERBOSE_MAKEFILE ON)
|
|
|
6
6
|
add_library(
|
|
7
7
|
react-native-jieba STATIC
|
|
8
8
|
../cpp/JiebaImpl.cpp
|
|
9
|
+
../cpp/JiebaEngine.cpp
|
|
10
|
+
../cpp/JiebaDictAndroid.cpp
|
|
9
11
|
)
|
|
10
12
|
|
|
11
13
|
set_target_properties(
|
|
@@ -26,4 +28,5 @@ target_link_libraries(
|
|
|
26
28
|
react-native-jieba jsi
|
|
27
29
|
reactnative
|
|
28
30
|
react_codegen_JiebaSpec
|
|
31
|
+
fbjni
|
|
29
32
|
)
|
package/android/build.gradle
CHANGED
|
@@ -2,6 +2,14 @@ buildscript {
|
|
|
2
2
|
ext.safeExtGet = { prop, fallback ->
|
|
3
3
|
rootProject.ext.has(prop) ? rootProject.ext.get(prop) : fallback
|
|
4
4
|
}
|
|
5
|
+
// Resolves a flag from either `ext` (…/build.gradle: ext.rnJiebaExcludeIdfDict = true) or a
|
|
6
|
+
// Gradle property (gradle.properties: rnJiebaExcludeIdfDict=true / -PrnJiebaExcludeIdfDict=true),
|
|
7
|
+
// so consumers can set it whichever way their project already configures native modules.
|
|
8
|
+
ext.safeFlagGet = { prop, fallback ->
|
|
9
|
+
if (rootProject.ext.has(prop)) return rootProject.ext.get(prop).toString()
|
|
10
|
+
if (rootProject.hasProperty(prop)) return rootProject.property(prop).toString()
|
|
11
|
+
return fallback
|
|
12
|
+
}
|
|
5
13
|
repositories {
|
|
6
14
|
google()
|
|
7
15
|
mavenCentral()
|
|
@@ -24,6 +32,11 @@ android {
|
|
|
24
32
|
defaultConfig {
|
|
25
33
|
minSdk safeExtGet('minSdkVersion', 24)
|
|
26
34
|
targetSdk safeExtGet('targetSdkVersion', 35)
|
|
35
|
+
buildConfigField(
|
|
36
|
+
"boolean",
|
|
37
|
+
"RN_JIEBA_EXCLUDE_IDF_DICT",
|
|
38
|
+
safeFlagGet('rnJiebaExcludeIdfDict', 'false')
|
|
39
|
+
)
|
|
27
40
|
externalNativeBuild {
|
|
28
41
|
cmake {
|
|
29
42
|
cppFlags "-std=c++20"
|
|
@@ -53,6 +66,25 @@ android {
|
|
|
53
66
|
assets.srcDirs += "$projectDir/../cpp/cppjieba/dict"
|
|
54
67
|
}
|
|
55
68
|
}
|
|
69
|
+
|
|
70
|
+
// The IDF dictionary (~5MB) is only read by `extract()` (TF-IDF keyword extraction). Apps that
|
|
71
|
+
// only tokenize can drop it from the APK/AAB with:
|
|
72
|
+
//
|
|
73
|
+
// rnJiebaExcludeIdfDict=true # in android/gradle.properties, or
|
|
74
|
+
// ext.rnJiebaExcludeIdfDict = true # in android/build.gradle
|
|
75
|
+
//
|
|
76
|
+
// `extract()` then throws a descriptive error, exactly as it already does on web (jieba-wasm
|
|
77
|
+
// ships no IDF data). Every other API is unaffected. BuildConfig carries the same flag through to
|
|
78
|
+
// JiebaDict so the extractor does not try to copy a file that was never packaged.
|
|
79
|
+
buildFeatures {
|
|
80
|
+
buildConfig true
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
if (safeFlagGet('rnJiebaExcludeIdfDict', 'false') == 'true') {
|
|
84
|
+
androidResources {
|
|
85
|
+
ignoreAssetsPattern "idf.utf8"
|
|
86
|
+
}
|
|
87
|
+
}
|
|
56
88
|
}
|
|
57
89
|
|
|
58
90
|
dependencies {
|
|
@@ -32,6 +32,10 @@ public abstract class NativeJiebaSpec extends ReactContextBaseJavaModule impleme
|
|
|
32
32
|
return NAME;
|
|
33
33
|
}
|
|
34
34
|
|
|
35
|
+
@ReactMethod(isBlockingSynchronousMethod = true)
|
|
36
|
+
@DoNotStrip
|
|
37
|
+
public abstract boolean isReady();
|
|
38
|
+
|
|
35
39
|
@ReactMethod
|
|
36
40
|
@DoNotStrip
|
|
37
41
|
public abstract void setDictPath(String path);
|
|
@@ -12,6 +12,11 @@
|
|
|
12
12
|
|
|
13
13
|
namespace facebook::react {
|
|
14
14
|
|
|
15
|
+
static facebook::jsi::Value __hostFunction_NativeJiebaSpecJSI_isReady(facebook::jsi::Runtime& rt, TurboModule &turboModule, const facebook::jsi::Value* args, size_t count) {
|
|
16
|
+
static jmethodID cachedMethodId = nullptr;
|
|
17
|
+
return static_cast<JavaTurboModule &>(turboModule).invokeJavaMethod(rt, BooleanKind, "isReady", "()Z", args, count, cachedMethodId);
|
|
18
|
+
}
|
|
19
|
+
|
|
15
20
|
static facebook::jsi::Value __hostFunction_NativeJiebaSpecJSI_setDictPath(facebook::jsi::Runtime& rt, TurboModule &turboModule, const facebook::jsi::Value* args, size_t count) {
|
|
16
21
|
static jmethodID cachedMethodId = nullptr;
|
|
17
22
|
return static_cast<JavaTurboModule &>(turboModule).invokeJavaMethod(rt, VoidKind, "setDictPath", "(Ljava/lang/String;)V", args, count, cachedMethodId);
|
|
@@ -64,6 +69,7 @@ static facebook::jsi::Value __hostFunction_NativeJiebaSpecJSI_find(facebook::jsi
|
|
|
64
69
|
|
|
65
70
|
NativeJiebaSpecJSI::NativeJiebaSpecJSI(const JavaTurboModule::InitParams ¶ms)
|
|
66
71
|
: JavaTurboModule(params) {
|
|
72
|
+
methodMap_["isReady"] = MethodMetadata {0, __hostFunction_NativeJiebaSpecJSI_isReady};
|
|
67
73
|
methodMap_["setDictPath"] = MethodMetadata {1, __hostFunction_NativeJiebaSpecJSI_setDictPath};
|
|
68
74
|
methodMap_["cut"] = MethodMetadata {2, __hostFunction_NativeJiebaSpecJSI_cut};
|
|
69
75
|
methodMap_["cutAll"] = MethodMetadata {1, __hostFunction_NativeJiebaSpecJSI_cutAll};
|
|
@@ -22,6 +22,7 @@ public:
|
|
|
22
22
|
|
|
23
23
|
protected:
|
|
24
24
|
NativeJiebaCxxSpec(std::shared_ptr<CallInvoker> jsInvoker) : TurboModule(std::string{NativeJiebaCxxSpec::kModuleName}, jsInvoker) {
|
|
25
|
+
methodMap_["isReady"] = MethodMetadata {.argCount = 0, .invoker = __isReady};
|
|
25
26
|
methodMap_["setDictPath"] = MethodMetadata {.argCount = 1, .invoker = __setDictPath};
|
|
26
27
|
methodMap_["cut"] = MethodMetadata {.argCount = 2, .invoker = __cut};
|
|
27
28
|
methodMap_["cutAll"] = MethodMetadata {.argCount = 1, .invoker = __cutAll};
|
|
@@ -35,6 +36,13 @@ protected:
|
|
|
35
36
|
}
|
|
36
37
|
|
|
37
38
|
private:
|
|
39
|
+
static jsi::Value __isReady(jsi::Runtime &rt, TurboModule &turboModule, const jsi::Value* /*args*/, size_t /*count*/) {
|
|
40
|
+
static_assert(
|
|
41
|
+
bridging::getParameterCount(&T::isReady) == 1,
|
|
42
|
+
"Expected isReady(...) to have 1 parameters");
|
|
43
|
+
return bridging::callFromJs<bool>(rt, &T::isReady, static_cast<NativeJiebaCxxSpec*>(&turboModule)->jsInvoker_, static_cast<T*>(&turboModule));
|
|
44
|
+
}
|
|
45
|
+
|
|
38
46
|
static jsi::Value __setDictPath(jsi::Runtime &rt, TurboModule &turboModule, const jsi::Value* args, size_t count) {
|
|
39
47
|
static_assert(
|
|
40
48
|
bridging::getParameterCount(&T::setDictPath) == 2,
|
|
@@ -3,31 +3,24 @@ package com.jieba
|
|
|
3
3
|
import com.facebook.react.bridge.Promise
|
|
4
4
|
import com.facebook.react.bridge.ReactApplicationContext
|
|
5
5
|
import com.facebook.react.module.annotations.ReactModule
|
|
6
|
-
import java.io.File
|
|
7
|
-
import java.io.FileOutputStream
|
|
8
6
|
|
|
9
7
|
@ReactModule(name = JiebaAndroidHelperModule.NAME)
|
|
10
8
|
class JiebaAndroidHelperModule(reactContext: ReactApplicationContext) :
|
|
11
9
|
NativeJiebaAndroidHelperSpec(reactContext) {
|
|
12
10
|
|
|
11
|
+
init {
|
|
12
|
+
// Capture an application context so the C++ side can lazily extract the
|
|
13
|
+
// dictionaries on first use, even if JS never calls prepareJieba().
|
|
14
|
+
JiebaDict.setContext(reactContext)
|
|
15
|
+
}
|
|
16
|
+
|
|
13
17
|
override fun getName(): String = NAME
|
|
14
18
|
|
|
15
19
|
override fun prepareDictDir(promise: Promise) {
|
|
16
20
|
val context = reactApplicationContext
|
|
17
21
|
Thread {
|
|
18
22
|
try {
|
|
19
|
-
|
|
20
|
-
if (!dictDir.exists()) dictDir.mkdirs()
|
|
21
|
-
for (filename in DICT_FILES) {
|
|
22
|
-
val target = File(dictDir, filename)
|
|
23
|
-
if (target.exists() && target.length() > 0) continue
|
|
24
|
-
context.assets.open(filename).use { input ->
|
|
25
|
-
FileOutputStream(target).use { output ->
|
|
26
|
-
input.copyTo(output)
|
|
27
|
-
}
|
|
28
|
-
}
|
|
29
|
-
}
|
|
30
|
-
promise.resolve(dictDir.absolutePath)
|
|
23
|
+
promise.resolve(JiebaDict.extractDictDir(context))
|
|
31
24
|
} catch (t: Throwable) {
|
|
32
25
|
promise.reject("E_JIEBA_DICT_EXTRACT", t.message, t)
|
|
33
26
|
}
|
|
@@ -36,13 +29,5 @@ class JiebaAndroidHelperModule(reactContext: ReactApplicationContext) :
|
|
|
36
29
|
|
|
37
30
|
companion object {
|
|
38
31
|
const val NAME = "JiebaAndroidHelper"
|
|
39
|
-
private const val DICT_DIR_NAME = "jieba-dict"
|
|
40
|
-
private val DICT_FILES = arrayOf(
|
|
41
|
-
"jieba.dict.utf8",
|
|
42
|
-
"hmm_model.utf8",
|
|
43
|
-
"user.dict.utf8",
|
|
44
|
-
"idf.utf8",
|
|
45
|
-
"stop_words.utf8",
|
|
46
|
-
)
|
|
47
32
|
}
|
|
48
33
|
}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
package com.jieba
|
|
2
|
+
|
|
3
|
+
import android.content.Context
|
|
4
|
+
import java.io.File
|
|
5
|
+
import java.io.FileOutputStream
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Shared dictionary extraction logic for Android.
|
|
9
|
+
*
|
|
10
|
+
* The cppjieba dictionaries ship as compressed APK assets, but cppjieba needs
|
|
11
|
+
* real filesystem paths. This object extracts them once into `filesDir/jieba-dict`
|
|
12
|
+
* and returns the directory.
|
|
13
|
+
*
|
|
14
|
+
* It is used from two places:
|
|
15
|
+
* - [JiebaAndroidHelperModule.prepareDictDir] — the async warm-up exposed to JS
|
|
16
|
+
* via `prepareJieba()`.
|
|
17
|
+
* - [extractDictDirFromNative] — a synchronous entry point invoked from C++
|
|
18
|
+
* (`JiebaImpl::getJieba`) on the very first segmentation call if `prepareJieba()`
|
|
19
|
+
* was never awaited, so that callers don't have to call `prepareJieba` at all.
|
|
20
|
+
*/
|
|
21
|
+
object JiebaDict {
|
|
22
|
+
const val DICT_DIR_NAME = "jieba-dict"
|
|
23
|
+
|
|
24
|
+
/** The IDF dictionary, needed only by `extract()` (TF-IDF keyword extraction). */
|
|
25
|
+
const val IDF_DICT_FILE = "idf.utf8"
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Dictionary assets to extract.
|
|
29
|
+
*
|
|
30
|
+
* [IDF_DICT_FILE] is omitted when the library was built with `rnJiebaExcludeIdfDict=true`, since
|
|
31
|
+
* the asset is then not packaged at all and opening it would throw. `extract()` reports the
|
|
32
|
+
* omission with a descriptive error; every other API is unaffected.
|
|
33
|
+
*/
|
|
34
|
+
val DICT_FILES: Array<String> = buildList {
|
|
35
|
+
add("jieba.dict.utf8")
|
|
36
|
+
add("hmm_model.utf8")
|
|
37
|
+
add("user.dict.utf8")
|
|
38
|
+
if (!BuildConfig.RN_JIEBA_EXCLUDE_IDF_DICT) add(IDF_DICT_FILE)
|
|
39
|
+
add("stop_words.utf8")
|
|
40
|
+
}.toTypedArray()
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Application context captured at module construction so the native (C++)
|
|
44
|
+
* fallback can extract assets without a `ReactApplicationContext` handle.
|
|
45
|
+
*/
|
|
46
|
+
@Volatile
|
|
47
|
+
private var appContext: Context? = null
|
|
48
|
+
|
|
49
|
+
@JvmStatic
|
|
50
|
+
fun setContext(context: Context) {
|
|
51
|
+
if (appContext == null) {
|
|
52
|
+
appContext = context.applicationContext
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** Synchronously extract the dictionary assets and return the directory path. */
|
|
57
|
+
@JvmStatic
|
|
58
|
+
fun extractDictDir(context: Context): String {
|
|
59
|
+
val dictDir = File(context.filesDir, DICT_DIR_NAME)
|
|
60
|
+
if (!dictDir.exists()) dictDir.mkdirs()
|
|
61
|
+
for (filename in DICT_FILES) {
|
|
62
|
+
val target = File(dictDir, filename)
|
|
63
|
+
if (target.exists() && target.length() > 0) continue
|
|
64
|
+
context.assets.open(filename).use { input ->
|
|
65
|
+
FileOutputStream(target).use { output ->
|
|
66
|
+
input.copyTo(output)
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
return dictDir.absolutePath
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Called from C++ via JNI. Returns the dict directory path, or an empty
|
|
75
|
+
* string if no context has been captured yet (extraction not possible).
|
|
76
|
+
*/
|
|
77
|
+
@JvmStatic
|
|
78
|
+
fun extractDictDirFromNative(): String {
|
|
79
|
+
val context = appContext ?: return ""
|
|
80
|
+
return extractDictDir(context)
|
|
81
|
+
}
|
|
82
|
+
}
|
|
@@ -8,6 +8,10 @@ import com.facebook.react.module.model.ReactModuleInfoProvider
|
|
|
8
8
|
|
|
9
9
|
class JiebaPackage : BaseReactPackage() {
|
|
10
10
|
override fun getModule(name: String, reactContext: ReactApplicationContext): NativeModule? {
|
|
11
|
+
// Capture an application context as early as possible so the C++ module can
|
|
12
|
+
// lazily extract the bundled dictionaries on first use without requiring a
|
|
13
|
+
// call to prepareJieba().
|
|
14
|
+
JiebaDict.setContext(reactContext)
|
|
11
15
|
return if (name == JiebaAndroidHelperModule.NAME) {
|
|
12
16
|
JiebaAndroidHelperModule(reactContext)
|
|
13
17
|
} else {
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
#include "JiebaDictAndroid.h"
|
|
2
|
+
|
|
3
|
+
#include <fbjni/fbjni.h>
|
|
4
|
+
|
|
5
|
+
namespace facebook::react {
|
|
6
|
+
|
|
7
|
+
std::string extractJiebaDictDirAndroid() {
|
|
8
|
+
using namespace facebook::jni;
|
|
9
|
+
|
|
10
|
+
// Calling a JNI method requires a thread attached to the JVM. cut()/tag()
|
|
11
|
+
// run on the JS thread, which fbjni attaches; ThreadScope makes this robust
|
|
12
|
+
// for any caller thread.
|
|
13
|
+
ThreadScope ts;
|
|
14
|
+
|
|
15
|
+
static const auto cls = findClassStatic("com/jieba/JiebaDict");
|
|
16
|
+
static const auto method =
|
|
17
|
+
cls->getStaticMethod<jstring()>("extractDictDirFromNative");
|
|
18
|
+
|
|
19
|
+
local_ref<jstring> result = method(cls);
|
|
20
|
+
if (!result) {
|
|
21
|
+
return "";
|
|
22
|
+
}
|
|
23
|
+
return result->toStdString();
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
#pragma once
|
|
2
|
+
|
|
3
|
+
#include <string>
|
|
4
|
+
|
|
5
|
+
namespace facebook::react {
|
|
6
|
+
|
|
7
|
+
// Android-only: synchronously extracts the bundled dictionary assets (via a
|
|
8
|
+
// JNI call into Kotlin) and returns the directory path. Returns an empty string
|
|
9
|
+
// if extraction is not possible yet (e.g. no application context captured).
|
|
10
|
+
//
|
|
11
|
+
// Defined in JiebaDictAndroid.cpp and only compiled on Android.
|
|
12
|
+
std::string extractJiebaDictDirAndroid();
|
|
13
|
+
|
|
14
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
#include "JiebaEngine.h"
|
|
2
|
+
|
|
3
|
+
#include <fstream>
|
|
4
|
+
|
|
5
|
+
namespace rnjieba {
|
|
6
|
+
|
|
7
|
+
bool JiebaEngine::UnicodeFileExists(const std::string& path) {
|
|
8
|
+
if (path.empty()) {
|
|
9
|
+
return false;
|
|
10
|
+
}
|
|
11
|
+
// Probe by opening rather than stat()-ing: cppjieba itself loads the dictionary through
|
|
12
|
+
// an ifstream, so this reports exactly the condition that would otherwise trip its
|
|
13
|
+
// XCHECK — including a path that exists but cannot be read.
|
|
14
|
+
std::ifstream ifs(path.c_str());
|
|
15
|
+
return ifs.is_open();
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
}
|