wordsninja 1.1.0 → 2.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/index.js CHANGED
@@ -1,167 +1,642 @@
1
- 'use strict'
2
-
3
- const jsonWords = require('./words-en.json');
4
- const splitRegex = new RegExp("[^a-zA-Z0-9']+", "g");
5
- let maxWordLen = 0;
6
- let wordCost = {};
7
- let maxCost = 9e999;
8
-
9
- /**
10
- * WordsNinja, Split your string text without space to english words
11
- * @class
12
- */
13
- class WordsNinja {
14
- /**
15
- * Load Dictionary
16
- * @return {Object} Objects of dictionary
17
- */
18
- loadDictionary() {
19
- return new Promise((resolve) => {
20
- let words = jsonWords;
21
-
22
- words.forEach(function (word, index) {
23
- wordCost[word] = Math.log((index + 1) * Math.log(words.length));
24
- if (word.length > maxWordLen)
25
- maxWordLen = word.length;
26
- if (wordCost[word] < maxCost)
27
- maxCost = wordCost[word];
28
- });
29
- resolve(wordCost);
30
- });
31
- };
32
-
33
- /**
34
- * @param {string} string - The string for split
35
- * @param {object} options - The Options
36
- * @param {boolean} options.camelCaseSplitter - Split by Camel Case, Default false (optional)
37
- * @param {boolean} options.capitalizeFirstLetter - Capitalize First Letter, Default false (optional)
38
- * @param {boolean} options.joinWords - Return join words as sentence, Default false (optional)
39
- * @returns {Array|string} result - Split String
40
- */
41
- splitSentence(string, {camelCaseSplitter, capitalizeFirstLetter, joinWords} = {}) {
42
- let list = [];
43
- let that = this;
44
- camelCaseSplitter = camelCaseSplitter || false;
45
- capitalizeFirstLetter = capitalizeFirstLetter || false;
46
- joinWords = joinWords || false;
47
- if (camelCaseSplitter)
48
- string = this.camelCaseSplitter(string);
49
-
50
- string.split(splitRegex).forEach(function (sub) {
51
- that.splitWords(sub).forEach(function (word) {
52
- word = capitalizeFirstLetter ? that.capitalizeFirstLetter(word) : word;
53
- list.push(word);
54
- })
55
- })
56
-
57
- return joinWords ? list.join(' ') : list;
58
- }
59
-
60
- /**
61
- * Add words to dictionary
62
- * @param {Array|string} words Word(s) to add dictionary
63
- * @return {void}
64
- */
65
- addWords(words) {
66
- if (Array.isArray(words)) {
67
- for (let value of words)
68
- this.addWords(value);
69
- } else {
70
- let word = words.toLocaleLowerCase();
71
- wordCost[word] = maxCost;
72
- if (word.length > maxWordLen)
73
- maxWordLen = word.length;
74
- }
75
- }
76
-
77
- /**
78
- * Split Words
79
- * @private
80
- * @param {string} s Input String
81
- * @return {Array} Splited Words
82
- */
83
- splitWords(s) {
84
- let cost = [0];
85
-
86
- function best_match(i) {
87
- let candidates = cost.slice(Math.max(0, i - maxWordLen), i).reverse();
88
- let minPair = [Number.MAX_SAFE_INTEGER, 0];
89
- candidates.forEach(function (c, k) {
90
- let cCost;
91
- if (wordCost[s.substring(i - k - 1, i).toLowerCase()]) {
92
- cCost = c + wordCost[s.substring(i - k - 1, i).toLowerCase()];
93
- } else {
94
- cCost = Number.MAX_SAFE_INTEGER;
95
- }
96
- if (cCost < minPair[0]) {
97
- minPair = [cCost, k + 1];
98
- }
99
- })
100
- return minPair;
101
- }
102
-
103
- for (let i = 1; i < s.length + 1; i++) {
104
- cost.push(best_match(i)[0]);
105
- }
106
-
107
- let out = [];
108
- let i = s.length;
109
- while (i > 0) {
110
- let c = best_match(i)[0];
111
- let k = best_match(i)[1];
112
-
113
- let newToken = true;
114
- if (s.slice(i - k, i) !== "'") {
115
- if (out.length > 0) {
116
- if (out[-1] === "'s" || (Number.isInteger(s[i - 1]) && Number.isInteger(out[-1][0]))) {
117
- out[-1] = s.slice(i - k, i) + out[-1];
118
- newToken = false;
119
- }
120
- }
121
- }
122
-
123
- if (newToken) {
124
- out.push(s.slice(i - k, i))
125
- }
126
-
127
- i -= k
128
- }
129
-
130
- return out.reverse();
131
- }
132
-
133
- /**
134
- * Camel Case Splitter
135
- * Based on 'split-camelcase-to-words' package, https://www.npmjs.com/package/split-camelcase-to-words
136
- * @private
137
- * @param {string} inputString
138
- * @return {string} String
139
- */
140
- camelCaseSplitter(inputString) {
141
- let notNullString = inputString || '';
142
- let trimmedString = notNullString.trim();
143
- let arrayOfStrings = trimmedString.split(' ');
144
-
145
- let splitStringsArray = [];
146
- arrayOfStrings.forEach(tempString => {
147
- if (tempString !== '') {
148
- let splitWords = tempString.split(/(?=[A-Z])/).join(" ");
149
- splitStringsArray.push(splitWords);
150
- }
151
- });
152
-
153
- return splitStringsArray.join(" ");
154
- }
155
-
156
- /**
157
- * Capitalize First Letter
158
- * @private
159
- * @param {string} string - String to Capitalize First Letter
160
- * @return {string} result
161
- */
162
- capitalizeFirstLetter(string) {
163
- return string.charAt(0).toUpperCase() + string.slice(1);
164
- }
165
- }
166
-
167
- module.exports = WordsNinja;
1
+ 'use strict';
2
+
3
+ // Node built-ins are loaded lazily and indirectly so browser bundlers never
4
+ // try to resolve them. They are only needed for file-path dictionaries.
5
+ const nodeRequire =
6
+ typeof module !== 'undefined' && typeof module.require === 'function'
7
+ ? module.require.bind(module)
8
+ : null;
9
+
10
+ /**
11
+ * WordsNinja
12
+ * Split English text that has no spaces into words.
13
+ *
14
+ * The segmentation is a dynamic-programming search (Viterbi style) over a
15
+ * frequency-ordered dictionary: every candidate word gets a cost derived from
16
+ * its rank, and the cheapest path through the input wins.
17
+ *
18
+ * @class
19
+ */
20
+
21
+ const DEFAULT_MAX_WORD_LENGTH = 64;
22
+ const DEFAULT_UNKNOWN_WORD_PENALTY = 10;
23
+
24
+ const KIND_WORD = 1;
25
+ const KIND_UNKNOWN = 2;
26
+
27
+ // A "run" is a maximal span of letters, digits and apostrophes. Everything
28
+ // else (spaces, punctuation, emoji, ...) is a separator.
29
+ const RUN_RE = /[\p{L}\p{N}'\u2019]+/gu;
30
+ const COMBINING_MARKS_RE = /\p{M}+/gu;
31
+ const NON_ASCII_RE = /[^\x00-\x7f]/;
32
+ const SIMPLE_WORD_RE = /^[a-z0-9']+$/;
33
+ const DIGITS_RE = /^\p{N}+$/u;
34
+ const LETTER_RE = /^\p{L}$/u;
35
+ const CONTRACTION_SUFFIXES = new Set(["'s", "'re", "'ll", "'ve", "'d", "'m", "'t"]);
36
+ const CONTRACTION_START_RE = /[\p{L}\p{N}]$/u;
37
+
38
+ const BOOLEAN_OPTIONS = [
39
+ 'camelCaseSplitter',
40
+ 'capitalizeFirstLetter',
41
+ 'joinWords',
42
+ 'withOffsets',
43
+ 'normalizeAccents',
44
+ 'mergeNumbers',
45
+ 'mergeContractions',
46
+ ];
47
+
48
+ /**
49
+ * Lowercase and strip diacritics, keeping curly apostrophes out of the way.
50
+ * Used both for dictionary keys and for `addWords()` input.
51
+ * @param {string} word
52
+ * @returns {string}
53
+ */
54
+ function normalizeWordKey(word) {
55
+ return word
56
+ .normalize('NFD')
57
+ .replace(COMBINING_MARKS_RE, '')
58
+ .replace(/\u2019/g, "'")
59
+ .toLowerCase();
60
+ }
61
+
62
+ /**
63
+ * Build an immutable cost table from a list of words.
64
+ * @param {string[]} words Frequency ordered list of words (most common first)
65
+ * @param {boolean} [skipLoneLetters=false] Ignore one-letter words other than
66
+ * `a` and `i`. Used for the bundled list, which contains every letter.
67
+ * @returns {{costs: Map<string, number>, maxLength: number, minCost: number}}
68
+ */
69
+ function buildDictionary(words, skipLoneLetters = false) {
70
+ if (!Array.isArray(words)) {
71
+ throw new TypeError('wordsninja: the dictionary must be an array of strings');
72
+ }
73
+
74
+ const costs = new Map();
75
+ const logN = Math.log(Math.max(words.length, 2));
76
+ let maxLength = 0;
77
+ let minCost = Infinity;
78
+
79
+ for (let i = 0; i < words.length; i++) {
80
+ const raw = words[i];
81
+ if (typeof raw !== 'string') {
82
+ throw new TypeError('wordsninja: dictionary entries must be strings');
83
+ }
84
+
85
+ let word = SIMPLE_WORD_RE.test(raw) ? raw : raw.trim().toLowerCase();
86
+ if (!word || costs.has(word)) continue;
87
+
88
+ // The bundled list contains every letter as a word. Only `a` and `i`
89
+ // are English one-letter words, so the rest are left to the
90
+ // unknown-text path.
91
+ if (
92
+ skipLoneLetters &&
93
+ word.length === 1 &&
94
+ word !== 'a' &&
95
+ word !== 'i' &&
96
+ LETTER_RE.test(word)
97
+ ) {
98
+ continue;
99
+ }
100
+
101
+ const cost = Math.log((i + 1) * Math.max(logN, 1));
102
+ costs.set(word, cost);
103
+
104
+ // Also index the accent-free form so a custom dictionary keeps
105
+ // working when `normalizeAccents` is enabled (the default). The
106
+ // bundled list is pure ASCII, so this is skipped there.
107
+ if (NON_ASCII_RE.test(word)) {
108
+ const key = normalizeWordKey(word);
109
+ if (!costs.has(key)) costs.set(key, cost);
110
+ if (key.length > maxLength) maxLength = key.length;
111
+ }
112
+
113
+ if (word.length > maxLength) maxLength = word.length;
114
+ if (cost < minCost) minCost = cost;
115
+ }
116
+
117
+ return {costs, maxLength, minCost};
118
+ }
119
+
120
+ let sharedDictionary = null;
121
+
122
+ /**
123
+ * Load the bundled English dictionary. `require()` is intentionally lazy so
124
+ * importing the package stays cheap; the 1.8 MB JSON is only parsed on first
125
+ * use.
126
+ * @returns {{costs: Map<string, number>, maxLength: number, minCost: number}}
127
+ */
128
+ function getSharedDictionary() {
129
+ if (sharedDictionary === null) {
130
+ sharedDictionary = buildDictionary(require('./words-en.json'), true);
131
+ }
132
+ return sharedDictionary;
133
+ }
134
+
135
+ /**
136
+ * Resolve the `dictionary` constructor option: an array of words or a path to
137
+ * a `.json` / newline separated `.txt` file.
138
+ * @param {string[]|string} dictionary
139
+ * @returns {string[]}
140
+ */
141
+ function loadCustomDictionary(dictionary) {
142
+ if (Array.isArray(dictionary)) return dictionary;
143
+ if (typeof dictionary !== 'string') {
144
+ throw new TypeError(
145
+ 'wordsninja: `dictionary` must be an array of words or a path to a .json/.txt file'
146
+ );
147
+ }
148
+
149
+ const fs = nodeRequire === null ? null : nodeRequire('node:fs');
150
+ const path = nodeRequire === null ? null : nodeRequire('node:path');
151
+
152
+ if (fs === null || path === null) {
153
+ throw new Error(
154
+ 'wordsninja: file-path dictionaries are only supported in Node.js. ' +
155
+ 'Pass an array of words instead.'
156
+ );
157
+ }
158
+
159
+ const file = path.resolve(dictionary);
160
+ const text = fs.readFileSync(file, 'utf8');
161
+
162
+ if (/\.json$/i.test(file)) {
163
+ const parsed = JSON.parse(text);
164
+ if (!Array.isArray(parsed)) {
165
+ throw new TypeError('wordsninja: a JSON dictionary must contain an array of strings');
166
+ }
167
+ return parsed;
168
+ }
169
+
170
+ return text.split(/\r?\n/);
171
+ }
172
+
173
+ /**
174
+ * @param {string} piece
175
+ * @param {number} offset Absolute index of `piece` in the original input string
176
+ * @param {{normalizeAccents: boolean}} options
177
+ * @returns {{norm: string, starts: number[], ends: number[]}}
178
+ */
179
+ function normalizePiece(piece, offset, options) {
180
+ let norm = '';
181
+ const starts = [];
182
+ const ends = [];
183
+
184
+ let index = offset;
185
+ for (const char of piece) {
186
+ let out;
187
+
188
+ if (char.codePointAt(0) < 0x80) {
189
+ // Fast path: ASCII.
190
+ out = char >= 'A' && char <= 'Z' ? char.toLowerCase() : char;
191
+ } else if (char === '\u2019') {
192
+ out = "'";
193
+ } else {
194
+ out = char;
195
+ if (options.normalizeAccents) {
196
+ out = out.normalize('NFD').replace(COMBINING_MARKS_RE, '');
197
+ }
198
+ out = out.toLowerCase().replace(/\u2019/g, "'");
199
+ }
200
+
201
+ for (const normalized of out) {
202
+ norm += normalized;
203
+ starts.push(index);
204
+ ends.push(index + char.length);
205
+ }
206
+
207
+ index += char.length;
208
+ }
209
+
210
+ return {norm, starts, ends};
211
+ }
212
+
213
+ function isUpper(char) {
214
+ return char !== char.toLowerCase();
215
+ }
216
+
217
+ function isLower(char) {
218
+ return char !== char.toUpperCase();
219
+ }
220
+
221
+ function isDigit(char) {
222
+ return /\p{N}/u.test(char);
223
+ }
224
+
225
+ /**
226
+ * Camel case boundaries: `XMLHttpRequest` -> `XML|Http|Request`,
227
+ * `Note12` -> `Note|12`.
228
+ * @param {string} run
229
+ * @returns {Array<[number, number]>}
230
+ */
231
+ function camelRanges(run) {
232
+ const ranges = [];
233
+ let start = 0;
234
+
235
+ for (let i = 1; i < run.length; i++) {
236
+ const prev = run[i - 1];
237
+ const current = run[i];
238
+ const next = i + 1 < run.length ? run[i + 1] : '';
239
+
240
+ const lowerOrDigitBeforeUpper = (isLower(prev) || isDigit(prev)) && isUpper(current);
241
+ const acronymBeforeWord = isUpper(prev) && isUpper(current) && isLower(next);
242
+ const letterBeforeDigit = isLower(prev) && isDigit(current);
243
+
244
+ if (lowerOrDigitBeforeUpper || acronymBeforeWord || letterBeforeDigit) {
245
+ ranges.push([start, i]);
246
+ start = i;
247
+ }
248
+ }
249
+
250
+ ranges.push([start, run.length]);
251
+ return ranges;
252
+ }
253
+
254
+ /**
255
+ * Validate and normalize `splitSentence()` options.
256
+ * @param {object|undefined} options
257
+ * @returns {object}
258
+ */
259
+ function normalizeOptions(options) {
260
+ if (options === undefined || options === null) options = {};
261
+ if (typeof options !== 'object' || Array.isArray(options)) {
262
+ throw new TypeError('wordsninja: options must be an object');
263
+ }
264
+
265
+ const normalized = {
266
+ camelCaseSplitter: false,
267
+ capitalizeFirstLetter: false,
268
+ joinWords: false,
269
+ withOffsets: false,
270
+ normalizeAccents: true,
271
+ mergeNumbers: true,
272
+ mergeContractions: true,
273
+ maxWordLength: DEFAULT_MAX_WORD_LENGTH,
274
+ unknownWordPenalty: DEFAULT_UNKNOWN_WORD_PENALTY,
275
+ };
276
+
277
+ for (const name of BOOLEAN_OPTIONS) {
278
+ const value = options[name];
279
+ if (value === undefined) continue;
280
+ if (typeof value !== 'boolean') {
281
+ throw new TypeError(`wordsninja: option \`${name}\` must be a boolean`);
282
+ }
283
+ normalized[name] = value;
284
+ }
285
+
286
+ if (options.maxWordLength !== undefined) {
287
+ const value = options.maxWordLength;
288
+ if (!Number.isInteger(value) || value < 1) {
289
+ throw new TypeError('wordsninja: option `maxWordLength` must be a positive integer');
290
+ }
291
+ normalized.maxWordLength = value;
292
+ }
293
+
294
+ if (options.unknownWordPenalty !== undefined) {
295
+ const value = options.unknownWordPenalty;
296
+ if (value !== null && (typeof value !== 'number' || !Number.isFinite(value) || value < 0)) {
297
+ throw new TypeError(
298
+ 'wordsninja: option `unknownWordPenalty` must be a non-negative number or null'
299
+ );
300
+ }
301
+ normalized.unknownWordPenalty = value;
302
+ }
303
+
304
+ return normalized;
305
+ }
306
+
307
+ /**
308
+ * Merge adjacent tokens that belong together numerically (`116|116` ->
309
+ * `116116`) or grammatically (`wang|'s` -> `wang's`). A separator between two
310
+ * tokens ends the merge, so `1 2 3` and `wang 's` stay split.
311
+ * @param {string} input
312
+ * @param {Array<{start: number, end: number}>} tokens
313
+ * @param {object} options
314
+ * @returns {Array<{word: string, start: number, end: number}>}
315
+ */
316
+ function mergeTokens(input, tokens, options) {
317
+ const merged = [];
318
+
319
+ for (const token of tokens) {
320
+ const word = input.slice(token.start, token.end);
321
+ const previous = merged[merged.length - 1];
322
+
323
+ if (previous && previous.end === token.start) {
324
+ if (
325
+ options.mergeNumbers &&
326
+ DIGITS_RE.test(previous.word) &&
327
+ DIGITS_RE.test(word)
328
+ ) {
329
+ previous.word += word;
330
+ previous.end = token.end;
331
+ continue;
332
+ }
333
+
334
+ const suffix = (
335
+ word.charAt(0) === '\u2019' ? "'" + word.slice(1) : word
336
+ ).toLowerCase();
337
+ if (
338
+ options.mergeContractions &&
339
+ CONTRACTION_SUFFIXES.has(suffix) &&
340
+ CONTRACTION_START_RE.test(previous.word)
341
+ ) {
342
+ previous.word += word;
343
+ previous.end = token.end;
344
+ continue;
345
+ }
346
+ }
347
+
348
+ merged.push({word, start: token.start, end: token.end});
349
+ }
350
+
351
+ return merged;
352
+ }
353
+
354
+ function capitalizeFirstLetter(word) {
355
+ return word.charAt(0).toUpperCase() + word.slice(1);
356
+ }
357
+
358
+ class WordsNinja {
359
+ /**
360
+ * @param {object} [options]
361
+ * @param {string[]|string} [options.dictionary] Custom dictionary: an array
362
+ * of words or a path to a `.json`/`.txt` file. Defaults to the bundled
363
+ * English dictionary.
364
+ */
365
+ constructor(options = {}) {
366
+ if (options === null || typeof options !== 'object' || Array.isArray(options)) {
367
+ throw new TypeError('wordsninja: constructor options must be an object');
368
+ }
369
+ if (
370
+ options.dictionary !== undefined &&
371
+ !Array.isArray(options.dictionary) &&
372
+ typeof options.dictionary !== 'string'
373
+ ) {
374
+ throw new TypeError(
375
+ 'wordsninja: `dictionary` must be an array of words or a path to a .json/.txt file'
376
+ );
377
+ }
378
+
379
+ this._customDictionary = options.dictionary;
380
+ this._base = null;
381
+ this._ready = false;
382
+ this._additions = new Map();
383
+ this._additionsMaxLength = 0;
384
+ }
385
+
386
+ /**
387
+ * Load the dictionary. This is optional: the dictionary is loaded lazily
388
+ * on first use, so `await loadDictionary()` only exists for backwards
389
+ * compatibility.
390
+ * @returns {Promise<this>}
391
+ */
392
+ async loadDictionary() {
393
+ this._ensureReady();
394
+ return this;
395
+ }
396
+
397
+ /**
398
+ * Whether the dictionary has been loaded.
399
+ * @returns {boolean}
400
+ */
401
+ get ready() {
402
+ return this._ready;
403
+ }
404
+
405
+ /**
406
+ * Add word(s) to this instance's dictionary. Added words get the lowest
407
+ * possible cost, so they win over bundled dictionary words.
408
+ * @param {string|string[]} words
409
+ * @returns {this}
410
+ */
411
+ addWords(words) {
412
+ const list = typeof words === 'string' ? [words] : words;
413
+ if (!Array.isArray(list)) {
414
+ throw new TypeError('wordsninja: addWords() expects a string or an array of strings');
415
+ }
416
+
417
+ this._ensureReady();
418
+ const cost = Number.isFinite(this._base.minCost) ? this._base.minCost : 0;
419
+
420
+ for (const raw of list) {
421
+ if (typeof raw !== 'string') {
422
+ throw new TypeError('wordsninja: addWords() expects a string or an array of strings');
423
+ }
424
+
425
+ const word = raw.trim();
426
+ if (!word) continue;
427
+ if (/\s/u.test(word)) {
428
+ throw new TypeError('wordsninja: a word cannot contain whitespace');
429
+ }
430
+
431
+ const lowercase = word.toLowerCase();
432
+ const keys = new Set([lowercase, normalizeWordKey(word)]);
433
+
434
+ for (const key of keys) {
435
+ this._additions.set(key, cost);
436
+ if (key.length > this._additionsMaxLength) {
437
+ this._additionsMaxLength = key.length;
438
+ }
439
+ }
440
+ }
441
+
442
+ return this;
443
+ }
444
+
445
+ /**
446
+ * Split a string into words.
447
+ *
448
+ * @param {string} string The string to split
449
+ * @param {object} [options]
450
+ * @param {boolean} [options.camelCaseSplitter=false] Split camelCase/PascalCase
451
+ * boundaries (`XMLHttpRequest` -> `XML Http Request`)
452
+ * @param {boolean} [options.capitalizeFirstLetter=false] Capitalize the first
453
+ * letter of every resulting word
454
+ * @param {boolean} [options.joinWords=false] Return a space separated string
455
+ * instead of an array
456
+ * @param {boolean} [options.withOffsets=false] Return `{word, start, end}`
457
+ * objects instead of strings. Ignored when `joinWords` is true.
458
+ * @param {boolean} [options.normalizeAccents=true] Strip diacritics before
459
+ * matching (`café` matches `cafe`) while keeping the original text in the
460
+ * result
461
+ * @param {boolean} [options.mergeNumbers=true] Merge adjacent digit tokens
462
+ * (`call116116now` -> `['call', '116116', 'now']`). Merging never crosses
463
+ * a separator, so `1 2 3` stays three tokens
464
+ * @param {boolean} [options.mergeContractions=true] Attach `'s`, `'re`,
465
+ * `'ll`, `'ve`, `'d`, `'m`, `'t` to the preceding word (`wang'sbook` ->
466
+ * `["wang's", 'book']`); `wang 's` stays two tokens
467
+ * @param {number} [options.maxWordLength=64] Maximum length of a dictionary
468
+ * word considered at each position. Lower values are faster; words longer
469
+ * than this are treated as unknown.
470
+ * @param {number|null} [options.unknownWordPenalty=10] Cost per unmatched
471
+ * character. Unmatched characters are kept as tokens. Set to `null` to
472
+ * drop characters that cannot be segmented. In the bundled dictionary, a
473
+ * lone letter other than `a` or `i` counts as unmatched text.
474
+ * @returns {string[]|string|Array<{word: string, start: number, end: number}>}
475
+ */
476
+ splitSentence(string, options) {
477
+ if (typeof string !== 'string') {
478
+ throw new TypeError('wordsninja: splitSentence() expects a string');
479
+ }
480
+
481
+ const settings = normalizeOptions(options);
482
+ this._ensureReady();
483
+
484
+ const tokens = [];
485
+ const maxWordLength = Math.min(
486
+ Math.max(this._base.maxLength, this._additionsMaxLength),
487
+ settings.maxWordLength
488
+ );
489
+
490
+ for (const match of string.matchAll(RUN_RE)) {
491
+ const run = match[0];
492
+ const runStart = match.index;
493
+ const ranges = settings.camelCaseSplitter
494
+ ? camelRanges(run)
495
+ : [[0, run.length]];
496
+
497
+ for (const [from, to] of ranges) {
498
+ if (to <= from) continue;
499
+ this._segmentRun(
500
+ run.slice(from, to),
501
+ runStart + from,
502
+ runStart + to,
503
+ settings,
504
+ maxWordLength,
505
+ tokens
506
+ );
507
+ }
508
+ }
509
+
510
+ const merged = mergeTokens(string, tokens, settings);
511
+ const words = merged.map((token) =>
512
+ settings.capitalizeFirstLetter
513
+ ? capitalizeFirstLetter(token.word)
514
+ : token.word
515
+ );
516
+
517
+ if (settings.joinWords) return words.join(' ');
518
+ if (settings.withOffsets) {
519
+ return merged.map((token, index) => ({
520
+ word: words[index],
521
+ start: token.start,
522
+ end: token.end,
523
+ }));
524
+ }
525
+ return words;
526
+ }
527
+
528
+ /**
529
+ * Run the Viterbi search for a single run of letters, digits and
530
+ * apostrophes and append its tokens (in source order) to `out`.
531
+ * @private
532
+ */
533
+ _segmentRun(text, absoluteStart, absoluteEnd, settings, maxWordLength, out) {
534
+ const {norm, starts, ends} = normalizePiece(text, absoluteStart, settings);
535
+ const length = norm.length;
536
+ if (length === 0) return;
537
+
538
+ const best = new Float64Array(length + 1).fill(Infinity);
539
+ const back = new Int32Array(length + 1);
540
+ const kind = new Uint8Array(length + 1);
541
+ best[0] = 0;
542
+
543
+ const unknownCost =
544
+ settings.unknownWordPenalty === null ? 1e6 : settings.unknownWordPenalty;
545
+
546
+ for (let i = 1; i <= length; i++) {
547
+ const limit = Math.min(maxWordLength, i);
548
+
549
+ for (let k = 1; k <= limit; k++) {
550
+ const wordCost = this._cost(norm.slice(i - k, i));
551
+ if (wordCost === undefined) continue;
552
+
553
+ const total = best[i - k] + wordCost;
554
+ if (total < best[i]) {
555
+ best[i] = total;
556
+ back[i] = k;
557
+ kind[i] = KIND_WORD;
558
+ }
559
+ }
560
+
561
+ // Every character can also be treated as unknown. This guarantees
562
+ // termination for input the dictionary cannot cover.
563
+ const unknownTotal = best[i - 1] + unknownCost;
564
+ if (unknownTotal < best[i]) {
565
+ best[i] = unknownTotal;
566
+ back[i] = 1;
567
+ kind[i] = KIND_UNKNOWN;
568
+ }
569
+ }
570
+
571
+ const reversed = [];
572
+ let i = length;
573
+
574
+ while (i > 0) {
575
+ const size = back[i];
576
+ const boundary = i === length ? absoluteEnd : starts[i];
577
+ const start = starts[i - size];
578
+ let end = ends[i - 1];
579
+ if (end > boundary) end = boundary;
580
+
581
+ const unknown = kind[i] === KIND_UNKNOWN;
582
+ const digits = DIGITS_RE.test(norm.slice(i - size, i));
583
+
584
+ // `unknownWordPenalty: null` drops characters the dictionary
585
+ // cannot cover instead of emitting them.
586
+ if (start < end && (!unknown || settings.unknownWordPenalty !== null)) {
587
+ const previous = reversed[reversed.length - 1];
588
+
589
+ // Adjacent digits are left separate here so that
590
+ // `mergeNumbers` stays in control of the final result.
591
+ if (
592
+ unknown &&
593
+ previous &&
594
+ previous.unknown &&
595
+ previous.start === end &&
596
+ !(previous.digits && digits)
597
+ ) {
598
+ previous.start = start;
599
+ previous.digits = previous.digits && digits;
600
+ } else {
601
+ reversed.push({start, end, unknown, digits});
602
+ }
603
+ }
604
+
605
+ i -= size;
606
+ }
607
+
608
+ for (let index = reversed.length - 1; index >= 0; index--) {
609
+ out.push(reversed[index]);
610
+ }
611
+ }
612
+
613
+ /**
614
+ * @private
615
+ * @param {string} word Lowercase, accent-free candidate
616
+ * @returns {number|undefined}
617
+ */
618
+ _cost(word) {
619
+ const addition = this._additions.get(word);
620
+ if (addition !== undefined) return addition;
621
+ return this._base.costs.get(word);
622
+ }
623
+
624
+ /**
625
+ * Build (or reuse) the dictionary and mark the instance as ready.
626
+ * @private
627
+ */
628
+ _ensureReady() {
629
+ if (this._ready) return;
630
+
631
+ this._base =
632
+ this._customDictionary === undefined
633
+ ? getSharedDictionary()
634
+ : buildDictionary(loadCustomDictionary(this._customDictionary));
635
+
636
+ this._ready = true;
637
+ }
638
+ }
639
+
640
+ module.exports = WordsNinja;
641
+ module.exports.WordsNinja = WordsNinja;
642
+ module.exports.default = WordsNinja;