wordsninja 1.0.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +21 -0
- package/README.md +287 -122
- package/index.d.mts +10 -0
- package/index.d.ts +94 -0
- package/index.js +616 -178
- package/index.mjs +4 -0
- package/package.json +71 -29
- package/words-en.json +126138 -0
- package/words-en.txt +0 -126138
package/NOTICE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
WordsNinja
|
|
2
|
+
Copyright (c) Parsa Kafi
|
|
3
|
+
|
|
4
|
+
The bundled English word list in `words-en.json` is derived from the word list
|
|
5
|
+
shipped with the `wordninja` Python package:
|
|
6
|
+
|
|
7
|
+
https://pypi.org/project/wordninja/
|
|
8
|
+
https://github.com/keredson/wordninja
|
|
9
|
+
|
|
10
|
+
`wordninja` is distributed under the MIT License. The list has been converted
|
|
11
|
+
from a newline separated text file to JSON and de-duplicated; the words and
|
|
12
|
+
their frequency order are unchanged.
|
|
13
|
+
|
|
14
|
+
The segmentation algorithm follows the approach described in:
|
|
15
|
+
|
|
16
|
+
https://stackoverflow.com/questions/8870261/how-to-split-text-without-spaces-into-list-of-words
|
|
17
|
+
|
|
18
|
+
The camel case splitting logic was originally inspired by the
|
|
19
|
+
`split-camelcase-to-words` package:
|
|
20
|
+
|
|
21
|
+
https://www.npmjs.com/package/split-camelcase-to-words
|
package/README.md
CHANGED
|
@@ -1,122 +1,287 @@
|
|
|
1
|
-
# WordsNinja
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
```
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
```
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
```
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
```
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
1
|
+
# WordsNinja
|
|
2
|
+
|
|
3
|
+
Split English text that has no spaces into words.
|
|
4
|
+
|
|
5
|
+
```js
|
|
6
|
+
import WordsNinja from 'wordsninja';
|
|
7
|
+
|
|
8
|
+
const ninja = new WordsNinja();
|
|
9
|
+
ninja.splitSentence('youneedtolearnfromyourmistakes');
|
|
10
|
+
// → ['you', 'need', 'to', 'learn', 'from', 'your', 'mistakes']
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
[](https://www.npmjs.com/package/wordsninja)
|
|
14
|
+
[](https://www.npmjs.com/package/wordsninja)
|
|
15
|
+
[](https://github.com/parsakafi/wordsninja/actions/workflows/ci.yml)
|
|
16
|
+
[](./LICENSE)
|
|
17
|
+
|
|
18
|
+
## Why WordsNinja
|
|
19
|
+
|
|
20
|
+
- **Zero dependencies** and a small API surface.
|
|
21
|
+
- **~3× faster than v1** — a dynamic-programming search over a frequency-ordered dictionary, backed by a `Map` and typed
|
|
22
|
+
arrays.
|
|
23
|
+
- **Keeps your text** — case is preserved and characters the dictionary cannot cover are returned instead of silently
|
|
24
|
+
dropped.
|
|
25
|
+
- **Offset-aware** — get `{ word, start, end }` tokens so you can highlight matches in the original string.
|
|
26
|
+
- **Custom dictionaries** — pass an array of words or point at a `.json` / `.txt` file.
|
|
27
|
+
- **TypeScript types and ESM + CommonJS** entry points.
|
|
28
|
+
- **Lazy dictionary load** — importing the package no longer parses the 1.8 MB word list.
|
|
29
|
+
|
|
30
|
+
## Install
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
npm install wordsninja
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
Requires Node.js 18 or newer.
|
|
37
|
+
|
|
38
|
+
## Quick start
|
|
39
|
+
|
|
40
|
+
CommonJS:
|
|
41
|
+
|
|
42
|
+
```js
|
|
43
|
+
const WordsNinja = require('wordsninja');
|
|
44
|
+
const ninja = new WordsNinja();
|
|
45
|
+
|
|
46
|
+
console.log(ninja.splitSentence('youneedtolearnfromyourmistakes'));
|
|
47
|
+
// [ 'you', 'need', 'to', 'learn', 'from', 'your', 'mistakes' ]
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
ESM:
|
|
51
|
+
|
|
52
|
+
```js
|
|
53
|
+
import WordsNinja from 'wordsninja';
|
|
54
|
+
|
|
55
|
+
const ninja = new WordsNinja();
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
TypeScript:
|
|
59
|
+
|
|
60
|
+
```ts
|
|
61
|
+
import WordsNinja from 'wordsninja';
|
|
62
|
+
|
|
63
|
+
const ninja = new WordsNinja();
|
|
64
|
+
const words: string[] = ninja.splitSentence('ilovewordsninja');
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
The dictionary is loaded on first use. `await ninja.loadDictionary()` still works and is kept for backwards
|
|
68
|
+
compatibility with v1:
|
|
69
|
+
|
|
70
|
+
```js
|
|
71
|
+
const ninja = new WordsNinja();
|
|
72
|
+
await ninja.loadDictionary(); // optional
|
|
73
|
+
ninja.splitSentence('helloworld');
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
## API
|
|
77
|
+
|
|
78
|
+
### `new WordsNinja(options?)`
|
|
79
|
+
|
|
80
|
+
| Option | Type | Description |
|
|
81
|
+
|--------------|----------------------|--------------------------------------------------------------------------------------------------------------------------------------------|
|
|
82
|
+
| `dictionary` | `string[] \| string` | Custom dictionary. An array of words, or a path to a `.json` array or newline separated `.txt` file. Defaults to the bundled English list. |
|
|
83
|
+
|
|
84
|
+
```js
|
|
85
|
+
const ninja = new WordsNinja({dictionary: ['code', 'ninja']});
|
|
86
|
+
ninja.splitSentence('codeninja'); // ['code', 'ninja']
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
The `dictionary` option is validated eagerly; the file is read (and the bundled list parsed) on first use.
|
|
90
|
+
|
|
91
|
+
### `ninja.loadDictionary(): Promise<this>`
|
|
92
|
+
|
|
93
|
+
Loads the dictionary and resolves with the instance. Optional — the dictionary is loaded lazily — but useful for warming
|
|
94
|
+
up in a serverless handler.
|
|
95
|
+
|
|
96
|
+
### `ninja.addWords(words): this`
|
|
97
|
+
|
|
98
|
+
Adds one word or an array of words to **this instance's** dictionary. Added words get the lowest possible cost, so they
|
|
99
|
+
outrank bundled words. Matching is case-insensitive and accent-insensitive.
|
|
100
|
+
|
|
101
|
+
```js
|
|
102
|
+
const ninja = new WordsNinja();
|
|
103
|
+
ninja.addWords('parsa').addWords(['kafi', 'wordsninja']);
|
|
104
|
+
|
|
105
|
+
ninja.splitSentence('parsayouneedtolearnfromyourmistakes');
|
|
106
|
+
// [ 'parsa', 'you', 'need', 'to', 'learn', 'from', 'your', 'mistakes' ]
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
Instances do not share added words.
|
|
110
|
+
|
|
111
|
+
### `ninja.splitSentence(input, options?): string[] | string | WordToken[]`
|
|
112
|
+
|
|
113
|
+
Splits `input` into words. Throws a `TypeError` if `input` is not a string.
|
|
114
|
+
|
|
115
|
+
| Option | Type | Default | Description |
|
|
116
|
+
|-------------------------|------------------|---------|-----------------------------------------------------------------------------------------------------------------------------------|
|
|
117
|
+
| `camelCaseSplitter` | `boolean` | `false` | Split camelCase / PascalCase boundaries. `XMLHttpRequest` → `XML Http Request`, `Note12` → `Note 12`. |
|
|
118
|
+
| `capitalizeFirstLetter` | `boolean` | `false` | Capitalize the first letter of every word. |
|
|
119
|
+
| `joinWords` | `boolean` | `false` | Return a space separated string instead of an array. Takes precedence over `withOffsets`. |
|
|
120
|
+
| `withOffsets` | `boolean` | `false` | Return `{ word, start, end }` objects. Offsets are UTF-16 indices into `input`. |
|
|
121
|
+
| `normalizeAccents` | `boolean` | `true` | Strip diacritics before matching (`café` matches `cafe`) while returning the original text. |
|
|
122
|
+
| `mergeNumbers` | `boolean` | `true` | Merge adjacent digits: `1 2 3` → `123`. |
|
|
123
|
+
| `mergeContractions` | `boolean` | `true` | Attach `'s`, `'re`, `'ll`, `'ve`, `'d`, `'m`, `'t` to the preceding word: `wang 's` → `wang's`. |
|
|
124
|
+
| `maxWordLength` | `number` | `64` | Maximum length of a dictionary word considered at each position. Lower values are faster; longer words are treated as unknown. |
|
|
125
|
+
| `unknownWordPenalty` | `number \| null` | `10` | Cost per unmatched character. Unmatched characters are kept as tokens. Set to `null` to drop characters that cannot be segmented. |
|
|
126
|
+
|
|
127
|
+
#### Offsets
|
|
128
|
+
|
|
129
|
+
```js
|
|
130
|
+
const input = 'Ihave2cats';
|
|
131
|
+
ninja.splitSentence(input, {withOffsets: true});
|
|
132
|
+
// [
|
|
133
|
+
// { word: 'I', start: 0, end: 1 },
|
|
134
|
+
// { word: 'have', start: 1, end: 5 },
|
|
135
|
+
// { word: '2', start: 5, end: 6 },
|
|
136
|
+
// { word: 'cats', start: 6, end: 10 }
|
|
137
|
+
// ]
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
Every offset pair slices back to its word:
|
|
141
|
+
|
|
142
|
+
```js
|
|
143
|
+
for (const token of tokens) {
|
|
144
|
+
input.slice(token.start, token.end) === token.word; // always true
|
|
145
|
+
}
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
`capitalizeFirstLetter` changes `word` but not the offsets. When `joinWords` is `true`, a string is returned and offsets
|
|
149
|
+
are ignored.
|
|
150
|
+
|
|
151
|
+
#### All options together
|
|
152
|
+
|
|
153
|
+
```js
|
|
154
|
+
ninja.splitSentence('helloWorldHowAreYou', {
|
|
155
|
+
camelCaseSplitter: true,
|
|
156
|
+
capitalizeFirstLetter: true,
|
|
157
|
+
joinWords: true,
|
|
158
|
+
});
|
|
159
|
+
// 'Hello World How Are You'
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
## Behavior details
|
|
163
|
+
|
|
164
|
+
- **Case is preserved.** Only the dictionary lookup is case-insensitive.
|
|
165
|
+
- **Separators are removed.** Spaces, punctuation, emoji and other characters outside `letters`, `digits` and `'` are
|
|
166
|
+
not part of the result. They become boundaries between words.
|
|
167
|
+
- **Accents are normalized by default.** `cafélatte` → `['café', 'latte']`. Disable with `normalizeAccents: false`; the
|
|
168
|
+
accented character is then treated as unknown text.
|
|
169
|
+
- **Text the dictionary cannot cover is kept.** Non-English scripts, brand names and typos come back as tokens instead
|
|
170
|
+
of disappearing: `سلامدنیا` → `['سلامدنیا']`. Adjacent unknown characters are grouped into one token. Use
|
|
171
|
+
`unknownWordPenalty: null` if you would rather drop them.
|
|
172
|
+
- **Numbers and possessives are merged by default.** `call123now` → `['call', '123', 'now']`, `wang'sbook` →
|
|
173
|
+
`["wang's", 'book']`. Disable with `mergeNumbers: false` / `mergeContractions: false`.
|
|
174
|
+
- **Ambiguous input is a guess.** `thequickbrownfox` can only be segmented by cost; some strings have more than one
|
|
175
|
+
plausible split.
|
|
176
|
+
- **Long words are capped.** `maxWordLength` bounds the candidate window per position. Words longer than the cap are
|
|
177
|
+
treated as unknown.
|
|
178
|
+
|
|
179
|
+
## Custom dictionaries
|
|
180
|
+
|
|
181
|
+
```js
|
|
182
|
+
// Array
|
|
183
|
+
const code = new WordsNinja({dictionary: ['code', 'ninja']});
|
|
184
|
+
|
|
185
|
+
// JSON file: ["code", "ninja"]
|
|
186
|
+
const fromJson = new WordsNinja({dictionary: './words.json'});
|
|
187
|
+
|
|
188
|
+
// Text file: one word per line
|
|
189
|
+
const fromText = new WordsNinja({dictionary: './words.txt'});
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
Paths are resolved from the current working directory. An empty dictionary is valid: every run of text is returned as
|
|
193
|
+
unknown tokens (or dropped when `unknownWordPenalty` is `null`).
|
|
194
|
+
|
|
195
|
+
## Bundlers and browsers
|
|
196
|
+
|
|
197
|
+
The package resolves no Node built-ins at import time, so it bundles cleanly for browsers and edge runtimes:
|
|
198
|
+
|
|
199
|
+
```js
|
|
200
|
+
import WordsNinja from 'wordsninja';
|
|
201
|
+
|
|
202
|
+
const ninja = new WordsNinja();
|
|
203
|
+
ninja.splitSentence('youneedtolearnfromyourmistakes');
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
The bundled word list is included in the bundle (about 610 KB gzipped) and parsed on first use. File-path dictionaries
|
|
207
|
+
are Node.js only; in the browser, fetch the words and pass an array instead:
|
|
208
|
+
|
|
209
|
+
```js
|
|
210
|
+
const words = await (await fetch('/words.json')).json();
|
|
211
|
+
const ninja = new WordsNinja({dictionary: words});
|
|
212
|
+
```
|
|
213
|
+
|
|
214
|
+
Offsets, custom words and every option work the same way in the browser.
|
|
215
|
+
|
|
216
|
+
## Migrating from v1
|
|
217
|
+
|
|
218
|
+
v2.0.0 is a breaking release:
|
|
219
|
+
|
|
220
|
+
- **`loadDictionary()` is optional.** The dictionary loads on first use. It still exists, now resolves with the
|
|
221
|
+
instance, and is idempotent.
|
|
222
|
+
- **Instances are isolated.** `addWords()` no longer changes a module-global dictionary shared by every instance.
|
|
223
|
+
- **The possessive/merge code actually runs.** v1 stored merged tokens at `out[-1]`, so the branch never executed.
|
|
224
|
+
Possessives and digit runs are now merged by default.
|
|
225
|
+
- **Unknown text is no longer dropped silently.** v1 returned `[]` for non-Latin input and removed accents as
|
|
226
|
+
separators; v2 keeps unmatched text as tokens.
|
|
227
|
+
- **Accents are normalized by default.** `cafélatte` now yields `['café', 'latte']` instead of `['caf', 'latte']`.
|
|
228
|
+
- **`addWords()` returns the instance** for chaining and rejects whitespace, empty and non-string entries.
|
|
229
|
+
- **Clear errors instead of hangs.** Using the API before the dictionary was loaded used to loop until
|
|
230
|
+
`RangeError: Invalid array length`; it now initializes lazily and validates input.
|
|
231
|
+
- **Node.js 18+**, TypeScript types, and ESM are supported.
|
|
232
|
+
- **New options:** `withOffsets`, `normalizeAccents`, `mergeNumbers`, `mergeContractions`, `maxWordLength`,
|
|
233
|
+
`unknownWordPenalty`, plus a `dictionary` constructor option.
|
|
234
|
+
|
|
235
|
+
## Performance
|
|
236
|
+
|
|
237
|
+
Measured with `npm run bench` on a desktop machine (Node 22):
|
|
238
|
+
|
|
239
|
+
| Workload | v1 | v2 |
|
|
240
|
+
|-------------------|--------------------|---------------------------|
|
|
241
|
+
| 4,000 characters | ~100 ms | **~30 ms** |
|
|
242
|
+
| 20,000 characters | ~390 ms | **~135 ms** |
|
|
243
|
+
| Dictionary load | ~35 ms (at import) | **~50 ms** (on first use) |
|
|
244
|
+
| Heap after load | ~30 MB | **~25 MB** |
|
|
245
|
+
|
|
246
|
+
v2 also supports `withOffsets` at no measurable extra cost. Numbers vary by machine; run the benchmark on your hardware.
|
|
247
|
+
|
|
248
|
+
## Development
|
|
249
|
+
|
|
250
|
+
```bash
|
|
251
|
+
npm test # node:test suite
|
|
252
|
+
npm run test:watch # watch mode
|
|
253
|
+
npm run bench # segmentation benchmark
|
|
254
|
+
npm run build:dictionary # validate/de-duplicate words-en.json
|
|
255
|
+
```
|
|
256
|
+
|
|
257
|
+
CI runs the test suite on Node.js 18, 20, 22 and 24, and verifies the published artifact with `npm pack --dry-run`.
|
|
258
|
+
|
|
259
|
+
## Releasing
|
|
260
|
+
|
|
261
|
+
1. Update the version and the changelog/README notes.
|
|
262
|
+
2. `npm test`
|
|
263
|
+
3. Tag and push: `git tag v2.0.0 && git push origin v2.0.0`
|
|
264
|
+
|
|
265
|
+
The `Release` workflow checks that the tag matches `package.json`, runs the tests, and publishes with provenance through
|
|
266
|
+
npm trusted publishing. If you publish manually, use `npm publish --provenance`.
|
|
267
|
+
|
|
268
|
+
## Limitations
|
|
269
|
+
|
|
270
|
+
WordsNinja is a dictionary-based segmenter, not a language model:
|
|
271
|
+
|
|
272
|
+
- The bundled list is English-focused. Non-English text is preserved as unknown tokens, not segmented.
|
|
273
|
+
- Segmentation is a best guess for ambiguous input.
|
|
274
|
+
- Punctuation is treated as a boundary, not preserved.
|
|
275
|
+
- The dictionary costs ~25 MB of heap once loaded; it is shared by all instances in a process.
|
|
276
|
+
|
|
277
|
+
## License
|
|
278
|
+
|
|
279
|
+
[MIT](./LICENSE) © Parsa Kafi
|
|
280
|
+
|
|
281
|
+
## Acknowledgements
|
|
282
|
+
|
|
283
|
+
- Algorithm
|
|
284
|
+
from [How to split text without spaces into list of words?](https://stackoverflow.com/questions/8870261/how-to-split-text-without-spaces-into-list-of-words).
|
|
285
|
+
- Word list from the [wordninja](https://pypi.org/project/wordninja) Python package (MIT). See [NOTICE](./NOTICE).
|
|
286
|
+
- v1 camelCase splitting was based
|
|
287
|
+
on [split-camelcase-to-words](https://www.npmjs.com/package/split-camelcase-to-words).
|
package/index.d.mts
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import WordsNinja = require('./index.js');
|
|
2
|
+
|
|
3
|
+
export default WordsNinja;
|
|
4
|
+
export {WordsNinja};
|
|
5
|
+
|
|
6
|
+
export type Options = WordsNinja.Options;
|
|
7
|
+
export type SplitOptions = WordsNinja.SplitOptions;
|
|
8
|
+
export type SplitJoinedOptions = WordsNinja.SplitJoinedOptions;
|
|
9
|
+
export type SplitWithOffsetsOptions = WordsNinja.SplitWithOffsetsOptions;
|
|
10
|
+
export type WordToken = WordsNinja.WordToken;
|
package/index.d.ts
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
declare namespace WordsNinja {
|
|
2
|
+
interface Options {
|
|
3
|
+
/**
|
|
4
|
+
* Custom dictionary: an array of words or a path to a `.json` / `.txt`
|
|
5
|
+
* file. Defaults to the bundled English dictionary.
|
|
6
|
+
*/
|
|
7
|
+
dictionary?: string[] | string;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
interface SplitOptions {
|
|
11
|
+
/** Split camelCase / PascalCase boundaries. Defaults to `false`. */
|
|
12
|
+
camelCaseSplitter?: boolean;
|
|
13
|
+
/** Capitalize the first letter of every word. Defaults to `false`. */
|
|
14
|
+
capitalizeFirstLetter?: boolean;
|
|
15
|
+
/** Return a space separated string instead of an array. Defaults to `false`. */
|
|
16
|
+
joinWords?: boolean;
|
|
17
|
+
/**
|
|
18
|
+
* Return `{ word, start, end }` objects instead of strings. Offsets are
|
|
19
|
+
* UTF-16 indices into the input string. Ignored when `joinWords` is `true`.
|
|
20
|
+
*/
|
|
21
|
+
withOffsets?: boolean;
|
|
22
|
+
/**
|
|
23
|
+
* Strip diacritics before matching (`café` matches `cafe`) while keeping
|
|
24
|
+
* the original text in the result. Defaults to `true`.
|
|
25
|
+
*/
|
|
26
|
+
normalizeAccents?: boolean;
|
|
27
|
+
/** Merge adjacent digits (`1 2 3` -> `123`). Defaults to `true`. */
|
|
28
|
+
mergeNumbers?: boolean;
|
|
29
|
+
/**
|
|
30
|
+
* Attach `'s`, `'re`, `'ll`, `'ve`, `'d`, `'m` and `'t` to the preceding
|
|
31
|
+
* word. Defaults to `true`.
|
|
32
|
+
*/
|
|
33
|
+
mergeContractions?: boolean;
|
|
34
|
+
/**
|
|
35
|
+
* Maximum length of a dictionary word considered at each position.
|
|
36
|
+
* Lower values are faster; longer words are treated as unknown.
|
|
37
|
+
* Defaults to `64`.
|
|
38
|
+
*/
|
|
39
|
+
maxWordLength?: number;
|
|
40
|
+
/**
|
|
41
|
+
* Cost per unmatched character. Unmatched characters are kept as tokens.
|
|
42
|
+
* Set to `null` to drop characters that cannot be segmented.
|
|
43
|
+
* Defaults to `10`.
|
|
44
|
+
*/
|
|
45
|
+
unknownWordPenalty?: number | null;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
interface WordToken {
|
|
49
|
+
/** The matched text, sliced from the original input. */
|
|
50
|
+
word: string;
|
|
51
|
+
/** Inclusive UTF-16 start offset. */
|
|
52
|
+
start: number;
|
|
53
|
+
/** Exclusive UTF-16 end offset. */
|
|
54
|
+
end: number;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
interface SplitWithOffsetsOptions extends SplitOptions {
|
|
58
|
+
withOffsets: true;
|
|
59
|
+
joinWords?: false;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
interface SplitJoinedOptions extends SplitOptions {
|
|
63
|
+
joinWords: true;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
declare class WordsNinja {
|
|
68
|
+
/**
|
|
69
|
+
* @param options Custom dictionary configuration.
|
|
70
|
+
*/
|
|
71
|
+
constructor(options?: WordsNinja.Options);
|
|
72
|
+
|
|
73
|
+
/** Whether the dictionary has been loaded. */
|
|
74
|
+
readonly ready: boolean;
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Load the dictionary. Optional: the dictionary is loaded lazily on first
|
|
78
|
+
* use. Kept for backwards compatibility with v1.
|
|
79
|
+
*/
|
|
80
|
+
loadDictionary(): Promise<this>;
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Add word(s) to this instance's dictionary. Added words get the lowest
|
|
84
|
+
* possible cost, so they win over bundled dictionary words.
|
|
85
|
+
*/
|
|
86
|
+
addWords(words: string | string[]): this;
|
|
87
|
+
|
|
88
|
+
/** Split a string into words. */
|
|
89
|
+
splitSentence(input: string, options: WordsNinja.SplitJoinedOptions): string;
|
|
90
|
+
splitSentence(input: string, options: WordsNinja.SplitWithOffsetsOptions): WordsNinja.WordToken[];
|
|
91
|
+
splitSentence(input: string, options?: WordsNinja.SplitOptions): string[];
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
export = WordsNinja;
|