wordsninja 1.1.0 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +71 -0
- package/NOTICE +21 -0
- package/README.md +309 -124
- package/index.d.mts +10 -0
- package/index.d.ts +100 -0
- package/index.js +642 -167
- package/index.mjs +4 -0
- package/package.json +73 -29
- package/words-en.json +0 -2
- package/example.js +0 -15
package/index.js
CHANGED
|
@@ -1,167 +1,642 @@
|
|
|
1
|
-
'use strict'
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
*
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Node built-ins are loaded lazily and indirectly so browser bundlers never
|
|
4
|
+
// try to resolve them. They are only needed for file-path dictionaries.
|
|
5
|
+
const nodeRequire =
|
|
6
|
+
typeof module !== 'undefined' && typeof module.require === 'function'
|
|
7
|
+
? module.require.bind(module)
|
|
8
|
+
: null;
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* WordsNinja
|
|
12
|
+
* Split English text that has no spaces into words.
|
|
13
|
+
*
|
|
14
|
+
* The segmentation is a dynamic-programming search (Viterbi style) over a
|
|
15
|
+
* frequency-ordered dictionary: every candidate word gets a cost derived from
|
|
16
|
+
* its rank, and the cheapest path through the input wins.
|
|
17
|
+
*
|
|
18
|
+
* @class
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
const DEFAULT_MAX_WORD_LENGTH = 64;
|
|
22
|
+
const DEFAULT_UNKNOWN_WORD_PENALTY = 10;
|
|
23
|
+
|
|
24
|
+
const KIND_WORD = 1;
|
|
25
|
+
const KIND_UNKNOWN = 2;
|
|
26
|
+
|
|
27
|
+
// A "run" is a maximal span of letters, digits and apostrophes. Everything
|
|
28
|
+
// else (spaces, punctuation, emoji, ...) is a separator.
|
|
29
|
+
const RUN_RE = /[\p{L}\p{N}'\u2019]+/gu;
|
|
30
|
+
const COMBINING_MARKS_RE = /\p{M}+/gu;
|
|
31
|
+
const NON_ASCII_RE = /[^\x00-\x7f]/;
|
|
32
|
+
const SIMPLE_WORD_RE = /^[a-z0-9']+$/;
|
|
33
|
+
const DIGITS_RE = /^\p{N}+$/u;
|
|
34
|
+
const LETTER_RE = /^\p{L}$/u;
|
|
35
|
+
const CONTRACTION_SUFFIXES = new Set(["'s", "'re", "'ll", "'ve", "'d", "'m", "'t"]);
|
|
36
|
+
const CONTRACTION_START_RE = /[\p{L}\p{N}]$/u;
|
|
37
|
+
|
|
38
|
+
const BOOLEAN_OPTIONS = [
|
|
39
|
+
'camelCaseSplitter',
|
|
40
|
+
'capitalizeFirstLetter',
|
|
41
|
+
'joinWords',
|
|
42
|
+
'withOffsets',
|
|
43
|
+
'normalizeAccents',
|
|
44
|
+
'mergeNumbers',
|
|
45
|
+
'mergeContractions',
|
|
46
|
+
];
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Lowercase and strip diacritics, keeping curly apostrophes out of the way.
|
|
50
|
+
* Used both for dictionary keys and for `addWords()` input.
|
|
51
|
+
* @param {string} word
|
|
52
|
+
* @returns {string}
|
|
53
|
+
*/
|
|
54
|
+
function normalizeWordKey(word) {
|
|
55
|
+
return word
|
|
56
|
+
.normalize('NFD')
|
|
57
|
+
.replace(COMBINING_MARKS_RE, '')
|
|
58
|
+
.replace(/\u2019/g, "'")
|
|
59
|
+
.toLowerCase();
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Build an immutable cost table from a list of words.
|
|
64
|
+
* @param {string[]} words Frequency ordered list of words (most common first)
|
|
65
|
+
* @param {boolean} [skipLoneLetters=false] Ignore one-letter words other than
|
|
66
|
+
* `a` and `i`. Used for the bundled list, which contains every letter.
|
|
67
|
+
* @returns {{costs: Map<string, number>, maxLength: number, minCost: number}}
|
|
68
|
+
*/
|
|
69
|
+
function buildDictionary(words, skipLoneLetters = false) {
|
|
70
|
+
if (!Array.isArray(words)) {
|
|
71
|
+
throw new TypeError('wordsninja: the dictionary must be an array of strings');
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
const costs = new Map();
|
|
75
|
+
const logN = Math.log(Math.max(words.length, 2));
|
|
76
|
+
let maxLength = 0;
|
|
77
|
+
let minCost = Infinity;
|
|
78
|
+
|
|
79
|
+
for (let i = 0; i < words.length; i++) {
|
|
80
|
+
const raw = words[i];
|
|
81
|
+
if (typeof raw !== 'string') {
|
|
82
|
+
throw new TypeError('wordsninja: dictionary entries must be strings');
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
let word = SIMPLE_WORD_RE.test(raw) ? raw : raw.trim().toLowerCase();
|
|
86
|
+
if (!word || costs.has(word)) continue;
|
|
87
|
+
|
|
88
|
+
// The bundled list contains every letter as a word. Only `a` and `i`
|
|
89
|
+
// are English one-letter words, so the rest are left to the
|
|
90
|
+
// unknown-text path.
|
|
91
|
+
if (
|
|
92
|
+
skipLoneLetters &&
|
|
93
|
+
word.length === 1 &&
|
|
94
|
+
word !== 'a' &&
|
|
95
|
+
word !== 'i' &&
|
|
96
|
+
LETTER_RE.test(word)
|
|
97
|
+
) {
|
|
98
|
+
continue;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
const cost = Math.log((i + 1) * Math.max(logN, 1));
|
|
102
|
+
costs.set(word, cost);
|
|
103
|
+
|
|
104
|
+
// Also index the accent-free form so a custom dictionary keeps
|
|
105
|
+
// working when `normalizeAccents` is enabled (the default). The
|
|
106
|
+
// bundled list is pure ASCII, so this is skipped there.
|
|
107
|
+
if (NON_ASCII_RE.test(word)) {
|
|
108
|
+
const key = normalizeWordKey(word);
|
|
109
|
+
if (!costs.has(key)) costs.set(key, cost);
|
|
110
|
+
if (key.length > maxLength) maxLength = key.length;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
if (word.length > maxLength) maxLength = word.length;
|
|
114
|
+
if (cost < minCost) minCost = cost;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
return {costs, maxLength, minCost};
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
let sharedDictionary = null;
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* Load the bundled English dictionary. `require()` is intentionally lazy so
|
|
124
|
+
* importing the package stays cheap; the 1.8 MB JSON is only parsed on first
|
|
125
|
+
* use.
|
|
126
|
+
* @returns {{costs: Map<string, number>, maxLength: number, minCost: number}}
|
|
127
|
+
*/
|
|
128
|
+
function getSharedDictionary() {
|
|
129
|
+
if (sharedDictionary === null) {
|
|
130
|
+
sharedDictionary = buildDictionary(require('./words-en.json'), true);
|
|
131
|
+
}
|
|
132
|
+
return sharedDictionary;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* Resolve the `dictionary` constructor option: an array of words or a path to
|
|
137
|
+
* a `.json` / newline separated `.txt` file.
|
|
138
|
+
* @param {string[]|string} dictionary
|
|
139
|
+
* @returns {string[]}
|
|
140
|
+
*/
|
|
141
|
+
function loadCustomDictionary(dictionary) {
|
|
142
|
+
if (Array.isArray(dictionary)) return dictionary;
|
|
143
|
+
if (typeof dictionary !== 'string') {
|
|
144
|
+
throw new TypeError(
|
|
145
|
+
'wordsninja: `dictionary` must be an array of words or a path to a .json/.txt file'
|
|
146
|
+
);
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
const fs = nodeRequire === null ? null : nodeRequire('node:fs');
|
|
150
|
+
const path = nodeRequire === null ? null : nodeRequire('node:path');
|
|
151
|
+
|
|
152
|
+
if (fs === null || path === null) {
|
|
153
|
+
throw new Error(
|
|
154
|
+
'wordsninja: file-path dictionaries are only supported in Node.js. ' +
|
|
155
|
+
'Pass an array of words instead.'
|
|
156
|
+
);
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
const file = path.resolve(dictionary);
|
|
160
|
+
const text = fs.readFileSync(file, 'utf8');
|
|
161
|
+
|
|
162
|
+
if (/\.json$/i.test(file)) {
|
|
163
|
+
const parsed = JSON.parse(text);
|
|
164
|
+
if (!Array.isArray(parsed)) {
|
|
165
|
+
throw new TypeError('wordsninja: a JSON dictionary must contain an array of strings');
|
|
166
|
+
}
|
|
167
|
+
return parsed;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
return text.split(/\r?\n/);
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* @param {string} piece
|
|
175
|
+
* @param {number} offset Absolute index of `piece` in the original input string
|
|
176
|
+
* @param {{normalizeAccents: boolean}} options
|
|
177
|
+
* @returns {{norm: string, starts: number[], ends: number[]}}
|
|
178
|
+
*/
|
|
179
|
+
function normalizePiece(piece, offset, options) {
|
|
180
|
+
let norm = '';
|
|
181
|
+
const starts = [];
|
|
182
|
+
const ends = [];
|
|
183
|
+
|
|
184
|
+
let index = offset;
|
|
185
|
+
for (const char of piece) {
|
|
186
|
+
let out;
|
|
187
|
+
|
|
188
|
+
if (char.codePointAt(0) < 0x80) {
|
|
189
|
+
// Fast path: ASCII.
|
|
190
|
+
out = char >= 'A' && char <= 'Z' ? char.toLowerCase() : char;
|
|
191
|
+
} else if (char === '\u2019') {
|
|
192
|
+
out = "'";
|
|
193
|
+
} else {
|
|
194
|
+
out = char;
|
|
195
|
+
if (options.normalizeAccents) {
|
|
196
|
+
out = out.normalize('NFD').replace(COMBINING_MARKS_RE, '');
|
|
197
|
+
}
|
|
198
|
+
out = out.toLowerCase().replace(/\u2019/g, "'");
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
for (const normalized of out) {
|
|
202
|
+
norm += normalized;
|
|
203
|
+
starts.push(index);
|
|
204
|
+
ends.push(index + char.length);
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
index += char.length;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
return {norm, starts, ends};
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
function isUpper(char) {
|
|
214
|
+
return char !== char.toLowerCase();
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
function isLower(char) {
|
|
218
|
+
return char !== char.toUpperCase();
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
function isDigit(char) {
|
|
222
|
+
return /\p{N}/u.test(char);
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
/**
|
|
226
|
+
* Camel case boundaries: `XMLHttpRequest` -> `XML|Http|Request`,
|
|
227
|
+
* `Note12` -> `Note|12`.
|
|
228
|
+
* @param {string} run
|
|
229
|
+
* @returns {Array<[number, number]>}
|
|
230
|
+
*/
|
|
231
|
+
function camelRanges(run) {
|
|
232
|
+
const ranges = [];
|
|
233
|
+
let start = 0;
|
|
234
|
+
|
|
235
|
+
for (let i = 1; i < run.length; i++) {
|
|
236
|
+
const prev = run[i - 1];
|
|
237
|
+
const current = run[i];
|
|
238
|
+
const next = i + 1 < run.length ? run[i + 1] : '';
|
|
239
|
+
|
|
240
|
+
const lowerOrDigitBeforeUpper = (isLower(prev) || isDigit(prev)) && isUpper(current);
|
|
241
|
+
const acronymBeforeWord = isUpper(prev) && isUpper(current) && isLower(next);
|
|
242
|
+
const letterBeforeDigit = isLower(prev) && isDigit(current);
|
|
243
|
+
|
|
244
|
+
if (lowerOrDigitBeforeUpper || acronymBeforeWord || letterBeforeDigit) {
|
|
245
|
+
ranges.push([start, i]);
|
|
246
|
+
start = i;
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
ranges.push([start, run.length]);
|
|
251
|
+
return ranges;
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
/**
|
|
255
|
+
* Validate and normalize `splitSentence()` options.
|
|
256
|
+
* @param {object|undefined} options
|
|
257
|
+
* @returns {object}
|
|
258
|
+
*/
|
|
259
|
+
function normalizeOptions(options) {
|
|
260
|
+
if (options === undefined || options === null) options = {};
|
|
261
|
+
if (typeof options !== 'object' || Array.isArray(options)) {
|
|
262
|
+
throw new TypeError('wordsninja: options must be an object');
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
const normalized = {
|
|
266
|
+
camelCaseSplitter: false,
|
|
267
|
+
capitalizeFirstLetter: false,
|
|
268
|
+
joinWords: false,
|
|
269
|
+
withOffsets: false,
|
|
270
|
+
normalizeAccents: true,
|
|
271
|
+
mergeNumbers: true,
|
|
272
|
+
mergeContractions: true,
|
|
273
|
+
maxWordLength: DEFAULT_MAX_WORD_LENGTH,
|
|
274
|
+
unknownWordPenalty: DEFAULT_UNKNOWN_WORD_PENALTY,
|
|
275
|
+
};
|
|
276
|
+
|
|
277
|
+
for (const name of BOOLEAN_OPTIONS) {
|
|
278
|
+
const value = options[name];
|
|
279
|
+
if (value === undefined) continue;
|
|
280
|
+
if (typeof value !== 'boolean') {
|
|
281
|
+
throw new TypeError(`wordsninja: option \`${name}\` must be a boolean`);
|
|
282
|
+
}
|
|
283
|
+
normalized[name] = value;
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
if (options.maxWordLength !== undefined) {
|
|
287
|
+
const value = options.maxWordLength;
|
|
288
|
+
if (!Number.isInteger(value) || value < 1) {
|
|
289
|
+
throw new TypeError('wordsninja: option `maxWordLength` must be a positive integer');
|
|
290
|
+
}
|
|
291
|
+
normalized.maxWordLength = value;
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
if (options.unknownWordPenalty !== undefined) {
|
|
295
|
+
const value = options.unknownWordPenalty;
|
|
296
|
+
if (value !== null && (typeof value !== 'number' || !Number.isFinite(value) || value < 0)) {
|
|
297
|
+
throw new TypeError(
|
|
298
|
+
'wordsninja: option `unknownWordPenalty` must be a non-negative number or null'
|
|
299
|
+
);
|
|
300
|
+
}
|
|
301
|
+
normalized.unknownWordPenalty = value;
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
return normalized;
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
/**
|
|
308
|
+
* Merge adjacent tokens that belong together numerically (`116|116` ->
|
|
309
|
+
* `116116`) or grammatically (`wang|'s` -> `wang's`). A separator between two
|
|
310
|
+
* tokens ends the merge, so `1 2 3` and `wang 's` stay split.
|
|
311
|
+
* @param {string} input
|
|
312
|
+
* @param {Array<{start: number, end: number}>} tokens
|
|
313
|
+
* @param {object} options
|
|
314
|
+
* @returns {Array<{word: string, start: number, end: number}>}
|
|
315
|
+
*/
|
|
316
|
+
function mergeTokens(input, tokens, options) {
|
|
317
|
+
const merged = [];
|
|
318
|
+
|
|
319
|
+
for (const token of tokens) {
|
|
320
|
+
const word = input.slice(token.start, token.end);
|
|
321
|
+
const previous = merged[merged.length - 1];
|
|
322
|
+
|
|
323
|
+
if (previous && previous.end === token.start) {
|
|
324
|
+
if (
|
|
325
|
+
options.mergeNumbers &&
|
|
326
|
+
DIGITS_RE.test(previous.word) &&
|
|
327
|
+
DIGITS_RE.test(word)
|
|
328
|
+
) {
|
|
329
|
+
previous.word += word;
|
|
330
|
+
previous.end = token.end;
|
|
331
|
+
continue;
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
const suffix = (
|
|
335
|
+
word.charAt(0) === '\u2019' ? "'" + word.slice(1) : word
|
|
336
|
+
).toLowerCase();
|
|
337
|
+
if (
|
|
338
|
+
options.mergeContractions &&
|
|
339
|
+
CONTRACTION_SUFFIXES.has(suffix) &&
|
|
340
|
+
CONTRACTION_START_RE.test(previous.word)
|
|
341
|
+
) {
|
|
342
|
+
previous.word += word;
|
|
343
|
+
previous.end = token.end;
|
|
344
|
+
continue;
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
merged.push({word, start: token.start, end: token.end});
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
return merged;
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
function capitalizeFirstLetter(word) {
|
|
355
|
+
return word.charAt(0).toUpperCase() + word.slice(1);
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
class WordsNinja {
|
|
359
|
+
/**
|
|
360
|
+
* @param {object} [options]
|
|
361
|
+
* @param {string[]|string} [options.dictionary] Custom dictionary: an array
|
|
362
|
+
* of words or a path to a `.json`/`.txt` file. Defaults to the bundled
|
|
363
|
+
* English dictionary.
|
|
364
|
+
*/
|
|
365
|
+
constructor(options = {}) {
|
|
366
|
+
if (options === null || typeof options !== 'object' || Array.isArray(options)) {
|
|
367
|
+
throw new TypeError('wordsninja: constructor options must be an object');
|
|
368
|
+
}
|
|
369
|
+
if (
|
|
370
|
+
options.dictionary !== undefined &&
|
|
371
|
+
!Array.isArray(options.dictionary) &&
|
|
372
|
+
typeof options.dictionary !== 'string'
|
|
373
|
+
) {
|
|
374
|
+
throw new TypeError(
|
|
375
|
+
'wordsninja: `dictionary` must be an array of words or a path to a .json/.txt file'
|
|
376
|
+
);
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
this._customDictionary = options.dictionary;
|
|
380
|
+
this._base = null;
|
|
381
|
+
this._ready = false;
|
|
382
|
+
this._additions = new Map();
|
|
383
|
+
this._additionsMaxLength = 0;
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
/**
|
|
387
|
+
* Load the dictionary. This is optional: the dictionary is loaded lazily
|
|
388
|
+
* on first use, so `await loadDictionary()` only exists for backwards
|
|
389
|
+
* compatibility.
|
|
390
|
+
* @returns {Promise<this>}
|
|
391
|
+
*/
|
|
392
|
+
async loadDictionary() {
|
|
393
|
+
this._ensureReady();
|
|
394
|
+
return this;
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
/**
|
|
398
|
+
* Whether the dictionary has been loaded.
|
|
399
|
+
* @returns {boolean}
|
|
400
|
+
*/
|
|
401
|
+
get ready() {
|
|
402
|
+
return this._ready;
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
/**
|
|
406
|
+
* Add word(s) to this instance's dictionary. Added words get the lowest
|
|
407
|
+
* possible cost, so they win over bundled dictionary words.
|
|
408
|
+
* @param {string|string[]} words
|
|
409
|
+
* @returns {this}
|
|
410
|
+
*/
|
|
411
|
+
addWords(words) {
|
|
412
|
+
const list = typeof words === 'string' ? [words] : words;
|
|
413
|
+
if (!Array.isArray(list)) {
|
|
414
|
+
throw new TypeError('wordsninja: addWords() expects a string or an array of strings');
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
this._ensureReady();
|
|
418
|
+
const cost = Number.isFinite(this._base.minCost) ? this._base.minCost : 0;
|
|
419
|
+
|
|
420
|
+
for (const raw of list) {
|
|
421
|
+
if (typeof raw !== 'string') {
|
|
422
|
+
throw new TypeError('wordsninja: addWords() expects a string or an array of strings');
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
const word = raw.trim();
|
|
426
|
+
if (!word) continue;
|
|
427
|
+
if (/\s/u.test(word)) {
|
|
428
|
+
throw new TypeError('wordsninja: a word cannot contain whitespace');
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
const lowercase = word.toLowerCase();
|
|
432
|
+
const keys = new Set([lowercase, normalizeWordKey(word)]);
|
|
433
|
+
|
|
434
|
+
for (const key of keys) {
|
|
435
|
+
this._additions.set(key, cost);
|
|
436
|
+
if (key.length > this._additionsMaxLength) {
|
|
437
|
+
this._additionsMaxLength = key.length;
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
return this;
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
/**
|
|
446
|
+
* Split a string into words.
|
|
447
|
+
*
|
|
448
|
+
* @param {string} string The string to split
|
|
449
|
+
* @param {object} [options]
|
|
450
|
+
* @param {boolean} [options.camelCaseSplitter=false] Split camelCase/PascalCase
|
|
451
|
+
* boundaries (`XMLHttpRequest` -> `XML Http Request`)
|
|
452
|
+
* @param {boolean} [options.capitalizeFirstLetter=false] Capitalize the first
|
|
453
|
+
* letter of every resulting word
|
|
454
|
+
* @param {boolean} [options.joinWords=false] Return a space separated string
|
|
455
|
+
* instead of an array
|
|
456
|
+
* @param {boolean} [options.withOffsets=false] Return `{word, start, end}`
|
|
457
|
+
* objects instead of strings. Ignored when `joinWords` is true.
|
|
458
|
+
* @param {boolean} [options.normalizeAccents=true] Strip diacritics before
|
|
459
|
+
* matching (`café` matches `cafe`) while keeping the original text in the
|
|
460
|
+
* result
|
|
461
|
+
* @param {boolean} [options.mergeNumbers=true] Merge adjacent digit tokens
|
|
462
|
+
* (`call116116now` -> `['call', '116116', 'now']`). Merging never crosses
|
|
463
|
+
* a separator, so `1 2 3` stays three tokens
|
|
464
|
+
* @param {boolean} [options.mergeContractions=true] Attach `'s`, `'re`,
|
|
465
|
+
* `'ll`, `'ve`, `'d`, `'m`, `'t` to the preceding word (`wang'sbook` ->
|
|
466
|
+
* `["wang's", 'book']`); `wang 's` stays two tokens
|
|
467
|
+
* @param {number} [options.maxWordLength=64] Maximum length of a dictionary
|
|
468
|
+
* word considered at each position. Lower values are faster; words longer
|
|
469
|
+
* than this are treated as unknown.
|
|
470
|
+
* @param {number|null} [options.unknownWordPenalty=10] Cost per unmatched
|
|
471
|
+
* character. Unmatched characters are kept as tokens. Set to `null` to
|
|
472
|
+
* drop characters that cannot be segmented. In the bundled dictionary, a
|
|
473
|
+
* lone letter other than `a` or `i` counts as unmatched text.
|
|
474
|
+
* @returns {string[]|string|Array<{word: string, start: number, end: number}>}
|
|
475
|
+
*/
|
|
476
|
+
splitSentence(string, options) {
|
|
477
|
+
if (typeof string !== 'string') {
|
|
478
|
+
throw new TypeError('wordsninja: splitSentence() expects a string');
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
const settings = normalizeOptions(options);
|
|
482
|
+
this._ensureReady();
|
|
483
|
+
|
|
484
|
+
const tokens = [];
|
|
485
|
+
const maxWordLength = Math.min(
|
|
486
|
+
Math.max(this._base.maxLength, this._additionsMaxLength),
|
|
487
|
+
settings.maxWordLength
|
|
488
|
+
);
|
|
489
|
+
|
|
490
|
+
for (const match of string.matchAll(RUN_RE)) {
|
|
491
|
+
const run = match[0];
|
|
492
|
+
const runStart = match.index;
|
|
493
|
+
const ranges = settings.camelCaseSplitter
|
|
494
|
+
? camelRanges(run)
|
|
495
|
+
: [[0, run.length]];
|
|
496
|
+
|
|
497
|
+
for (const [from, to] of ranges) {
|
|
498
|
+
if (to <= from) continue;
|
|
499
|
+
this._segmentRun(
|
|
500
|
+
run.slice(from, to),
|
|
501
|
+
runStart + from,
|
|
502
|
+
runStart + to,
|
|
503
|
+
settings,
|
|
504
|
+
maxWordLength,
|
|
505
|
+
tokens
|
|
506
|
+
);
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
const merged = mergeTokens(string, tokens, settings);
|
|
511
|
+
const words = merged.map((token) =>
|
|
512
|
+
settings.capitalizeFirstLetter
|
|
513
|
+
? capitalizeFirstLetter(token.word)
|
|
514
|
+
: token.word
|
|
515
|
+
);
|
|
516
|
+
|
|
517
|
+
if (settings.joinWords) return words.join(' ');
|
|
518
|
+
if (settings.withOffsets) {
|
|
519
|
+
return merged.map((token, index) => ({
|
|
520
|
+
word: words[index],
|
|
521
|
+
start: token.start,
|
|
522
|
+
end: token.end,
|
|
523
|
+
}));
|
|
524
|
+
}
|
|
525
|
+
return words;
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
/**
|
|
529
|
+
* Run the Viterbi search for a single run of letters, digits and
|
|
530
|
+
* apostrophes and append its tokens (in source order) to `out`.
|
|
531
|
+
* @private
|
|
532
|
+
*/
|
|
533
|
+
_segmentRun(text, absoluteStart, absoluteEnd, settings, maxWordLength, out) {
|
|
534
|
+
const {norm, starts, ends} = normalizePiece(text, absoluteStart, settings);
|
|
535
|
+
const length = norm.length;
|
|
536
|
+
if (length === 0) return;
|
|
537
|
+
|
|
538
|
+
const best = new Float64Array(length + 1).fill(Infinity);
|
|
539
|
+
const back = new Int32Array(length + 1);
|
|
540
|
+
const kind = new Uint8Array(length + 1);
|
|
541
|
+
best[0] = 0;
|
|
542
|
+
|
|
543
|
+
const unknownCost =
|
|
544
|
+
settings.unknownWordPenalty === null ? 1e6 : settings.unknownWordPenalty;
|
|
545
|
+
|
|
546
|
+
for (let i = 1; i <= length; i++) {
|
|
547
|
+
const limit = Math.min(maxWordLength, i);
|
|
548
|
+
|
|
549
|
+
for (let k = 1; k <= limit; k++) {
|
|
550
|
+
const wordCost = this._cost(norm.slice(i - k, i));
|
|
551
|
+
if (wordCost === undefined) continue;
|
|
552
|
+
|
|
553
|
+
const total = best[i - k] + wordCost;
|
|
554
|
+
if (total < best[i]) {
|
|
555
|
+
best[i] = total;
|
|
556
|
+
back[i] = k;
|
|
557
|
+
kind[i] = KIND_WORD;
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
// Every character can also be treated as unknown. This guarantees
|
|
562
|
+
// termination for input the dictionary cannot cover.
|
|
563
|
+
const unknownTotal = best[i - 1] + unknownCost;
|
|
564
|
+
if (unknownTotal < best[i]) {
|
|
565
|
+
best[i] = unknownTotal;
|
|
566
|
+
back[i] = 1;
|
|
567
|
+
kind[i] = KIND_UNKNOWN;
|
|
568
|
+
}
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
const reversed = [];
|
|
572
|
+
let i = length;
|
|
573
|
+
|
|
574
|
+
while (i > 0) {
|
|
575
|
+
const size = back[i];
|
|
576
|
+
const boundary = i === length ? absoluteEnd : starts[i];
|
|
577
|
+
const start = starts[i - size];
|
|
578
|
+
let end = ends[i - 1];
|
|
579
|
+
if (end > boundary) end = boundary;
|
|
580
|
+
|
|
581
|
+
const unknown = kind[i] === KIND_UNKNOWN;
|
|
582
|
+
const digits = DIGITS_RE.test(norm.slice(i - size, i));
|
|
583
|
+
|
|
584
|
+
// `unknownWordPenalty: null` drops characters the dictionary
|
|
585
|
+
// cannot cover instead of emitting them.
|
|
586
|
+
if (start < end && (!unknown || settings.unknownWordPenalty !== null)) {
|
|
587
|
+
const previous = reversed[reversed.length - 1];
|
|
588
|
+
|
|
589
|
+
// Adjacent digits are left separate here so that
|
|
590
|
+
// `mergeNumbers` stays in control of the final result.
|
|
591
|
+
if (
|
|
592
|
+
unknown &&
|
|
593
|
+
previous &&
|
|
594
|
+
previous.unknown &&
|
|
595
|
+
previous.start === end &&
|
|
596
|
+
!(previous.digits && digits)
|
|
597
|
+
) {
|
|
598
|
+
previous.start = start;
|
|
599
|
+
previous.digits = previous.digits && digits;
|
|
600
|
+
} else {
|
|
601
|
+
reversed.push({start, end, unknown, digits});
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
i -= size;
|
|
606
|
+
}
|
|
607
|
+
|
|
608
|
+
for (let index = reversed.length - 1; index >= 0; index--) {
|
|
609
|
+
out.push(reversed[index]);
|
|
610
|
+
}
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
/**
|
|
614
|
+
* @private
|
|
615
|
+
* @param {string} word Lowercase, accent-free candidate
|
|
616
|
+
* @returns {number|undefined}
|
|
617
|
+
*/
|
|
618
|
+
_cost(word) {
|
|
619
|
+
const addition = this._additions.get(word);
|
|
620
|
+
if (addition !== undefined) return addition;
|
|
621
|
+
return this._base.costs.get(word);
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
/**
|
|
625
|
+
* Build (or reuse) the dictionary and mark the instance as ready.
|
|
626
|
+
* @private
|
|
627
|
+
*/
|
|
628
|
+
_ensureReady() {
|
|
629
|
+
if (this._ready) return;
|
|
630
|
+
|
|
631
|
+
this._base =
|
|
632
|
+
this._customDictionary === undefined
|
|
633
|
+
? getSharedDictionary()
|
|
634
|
+
: buildDictionary(loadCustomDictionary(this._customDictionary));
|
|
635
|
+
|
|
636
|
+
this._ready = true;
|
|
637
|
+
}
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
module.exports = WordsNinja;
|
|
641
|
+
module.exports.WordsNinja = WordsNinja;
|
|
642
|
+
module.exports.default = WordsNinja;
|