wordsninja 1.0.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/index.js CHANGED
@@ -1,178 +1,616 @@
1
- 'use strict'
2
-
3
- const fs = require("fs");
4
- const path = require('path');
5
- const splitRegex = new RegExp("[^a-zA-Z0-9']+", "g");
6
- const FILE_WORDS = path.join(__dirname, 'words-en.txt');
7
- let maxWordLen = 0;
8
- let wordCost = {};
9
- let maxCost = 9e999;
10
-
11
- /**
12
- * WordsNinja, Split your string text without space to english words
13
- * @class
14
- */
15
- class WordsNinja {
16
- /**
17
- * Load Dictionary
18
- * @return {Object} Objects of dictionary
19
- */
20
- loadDictionary() {
21
- return new Promise((resolve) => {
22
- fs.readFile(FILE_WORDS, 'utf8', function (err, data) {
23
- if (err)
24
- throw err;
25
-
26
- let words = data.split('\n');
27
-
28
- words.forEach(function (word, index) {
29
- wordCost[word] = Math.log((index + 1) * Math.log(words.length));
30
- if (word.length > maxWordLen)
31
- maxWordLen = word.length;
32
- if (wordCost[word] < maxCost)
33
- maxCost = wordCost[word];
34
- });
35
- resolve(wordCost);
36
- });
37
- });
38
- };
39
-
40
- /**
41
- * @param {string} string - The string for split
42
- * @param {object} options - The Options
43
- * @param {boolean} options.camelCaseSplitter - Split by Camel Case, Default false (optional)
44
- * @param {boolean} options.capitalizeFirstLetter - Capitalize First Letter, Default false (optional)
45
- * @param {boolean} options.joinWords - Return join words as sentence, Default false (optional)
46
- * @returns {Array|string} result - Split String
47
- */
48
- splitSentence(string, { camelCaseSplitter, capitalizeFirstLetter, joinWords } = {}) {
49
- let list = [];
50
- let that = this;
51
- camelCaseSplitter = camelCaseSplitter || false;
52
- capitalizeFirstLetter = capitalizeFirstLetter || false;
53
- joinWords = joinWords || false;
54
- if (camelCaseSplitter)
55
- string = this.camelCaseSplitter(string);
56
-
57
- string.split(splitRegex).forEach(function (sub) {
58
- that.splitWords(sub).forEach(function (word) {
59
- word = capitalizeFirstLetter ? that.capitalizeFirstLetter(word) : word;
60
- list.push(word);
61
- })
62
- })
63
- if (joinWords)
64
- return list.join(' ');
65
- else
66
- return list;
67
- }
68
-
69
- /**
70
- * Add words to dictionary
71
- * @param {Array|string} words Word(s) to add dictionary
72
- * @return {void}
73
- */
74
- addWords(words) {
75
- if (Array.isArray(words)) {
76
- for (let value of words)
77
- this.addWords(value);
78
- } else {
79
- let word = words.toLocaleLowerCase();
80
- wordCost[word] = maxCost;
81
- if (word.length > maxWordLen)
82
- maxWordLen = word.length;
83
- }
84
- }
85
-
86
- /**
87
- * Split Words
88
- * @private
89
- * @param {string} s Input String
90
- * @return {Array} Splited Words
91
- */
92
- splitWords(s) {
93
- let cost = [0];
94
-
95
- function best_match(i) {
96
- let candidates = cost.slice(Math.max(0, i - maxWordLen), i).reverse();
97
- let minPair = [Number.MAX_SAFE_INTEGER, 0];
98
- candidates.forEach(function (c, k) {
99
- let ccost;
100
- if (wordCost[s.substring(i - k - 1, i).toLowerCase()]) {
101
- ccost = c + wordCost[s.substring(i - k - 1, i).toLowerCase()];
102
- } else {
103
- ccost = Number.MAX_SAFE_INTEGER;
104
- }
105
- if (ccost < minPair[0]) {
106
- minPair = [ccost, k + 1];
107
- }
108
- })
109
- return minPair;
110
- }
111
-
112
- for (let i = 1; i < s.length + 1; i++) {
113
- cost.push(best_match(i)[0]);
114
- }
115
-
116
- let out = [];
117
- let i = s.length;
118
- while (i > 0) {
119
- let c = best_match(i)[0];
120
- let k = best_match(i)[1];
121
- //if (c == cost[i])
122
- // console.log("Alert: " + c);
123
-
124
- let newToken = true;
125
- if (s.slice(i - k, i) != "'") {
126
- if (out.length > 0) {
127
- if (out[-1] == "'s" || (Number.isInteger(s[i - 1]) && Number.isInteger(out[-1][0]))) {
128
- out[-1] = s.slice(i - k, i) + out[-1];
129
- newToken = false;
130
- }
131
- }
132
- }
133
-
134
- if (newToken) {
135
- out.push(s.slice(i - k, i))
136
- }
137
-
138
- i -= k
139
- }
140
-
141
- return out.reverse();
142
- }
143
-
144
- /**
145
- * Camel Case Splitter
146
- * Based on 'split-camelcase-to-words' package, https://www.npmjs.com/package/split-camelcase-to-words
147
- * @private
148
- * @param {string} inputString
149
- * @return {string} String
150
- */
151
- camelCaseSplitter(inputString) {
152
- let notNullString = inputString || '';
153
- let trimmedString = notNullString.trim();
154
- let arrayOfStrings = trimmedString.split(' ');
155
-
156
- let splitStringsArray = [];
157
- arrayOfStrings.forEach(tempString => {
158
- if (tempString != '') {
159
- let splitWords = tempString.split(/(?=[A-Z])/).join(" ");
160
- splitStringsArray.push(splitWords);
161
- }
162
- });
163
-
164
- return splitStringsArray.join(" ");
165
- }
166
-
167
- /**
168
- * Capitalize First Letter
169
- * @private
170
- * @param {string} string - String to Capitalize First Letter
171
- * @return {string} result
172
- */
173
- capitalizeFirstLetter(string) {
174
- return string.charAt(0).toUpperCase() + string.slice(1);
175
- }
176
- }
177
-
178
- module.exports = WordsNinja;
1
+ 'use strict';
2
+
3
+ // Node built-ins are loaded lazily and indirectly so browser bundlers never
4
+ // try to resolve them. They are only needed for file-path dictionaries.
5
+ const nodeRequire =
6
+ typeof module !== 'undefined' && typeof module.require === 'function'
7
+ ? module.require.bind(module)
8
+ : null;
9
+
10
+ /**
11
+ * WordsNinja
12
+ * Split English text that has no spaces into words.
13
+ *
14
+ * The segmentation is a dynamic-programming search (Viterbi style) over a
15
+ * frequency-ordered dictionary: every candidate word gets a cost derived from
16
+ * its rank, and the cheapest path through the input wins.
17
+ *
18
+ * @class
19
+ */
20
+
21
+ const DEFAULT_MAX_WORD_LENGTH = 64;
22
+ const DEFAULT_UNKNOWN_WORD_PENALTY = 10;
23
+
24
+ const KIND_WORD = 1;
25
+ const KIND_UNKNOWN = 2;
26
+
27
+ // A "run" is a maximal span of letters, digits and apostrophes. Everything
28
+ // else (spaces, punctuation, emoji, ...) is a separator.
29
+ const RUN_RE = /[\p{L}\p{N}'\u2019]+/gu;
30
+ const COMBINING_MARKS_RE = /\p{M}+/gu;
31
+ const NON_ASCII_RE = /[^\x00-\x7f]/;
32
+ const SIMPLE_WORD_RE = /^[a-z0-9']+$/;
33
+ const DIGITS_RE = /^\p{N}+$/u;
34
+ const CONTRACTION_SUFFIXES = new Set(["'s", "'re", "'ll", "'ve", "'d", "'m", "'t"]);
35
+ const CONTRACTION_START_RE = /[\p{L}\p{N}]$/u;
36
+
37
+ const BOOLEAN_OPTIONS = [
38
+ 'camelCaseSplitter',
39
+ 'capitalizeFirstLetter',
40
+ 'joinWords',
41
+ 'withOffsets',
42
+ 'normalizeAccents',
43
+ 'mergeNumbers',
44
+ 'mergeContractions',
45
+ ];
46
+
47
+ /**
48
+ * Lowercase and strip diacritics, keeping curly apostrophes out of the way.
49
+ * Used both for dictionary keys and for `addWords()` input.
50
+ * @param {string} word
51
+ * @returns {string}
52
+ */
53
+ function normalizeWordKey(word) {
54
+ return word
55
+ .normalize('NFD')
56
+ .replace(COMBINING_MARKS_RE, '')
57
+ .replace(/\u2019/g, "'")
58
+ .toLowerCase();
59
+ }
60
+
61
+ /**
62
+ * Build an immutable cost table from a list of words.
63
+ * @param {string[]} words Frequency ordered list of words (most common first)
64
+ * @returns {{costs: Map<string, number>, maxLength: number, minCost: number}}
65
+ */
66
+ function buildDictionary(words) {
67
+ if (!Array.isArray(words)) {
68
+ throw new TypeError('wordsninja: the dictionary must be an array of strings');
69
+ }
70
+
71
+ const costs = new Map();
72
+ const logN = Math.log(Math.max(words.length, 2));
73
+ let maxLength = 0;
74
+ let minCost = Infinity;
75
+
76
+ for (let i = 0; i < words.length; i++) {
77
+ const raw = words[i];
78
+ if (typeof raw !== 'string') {
79
+ throw new TypeError('wordsninja: dictionary entries must be strings');
80
+ }
81
+
82
+ let word = SIMPLE_WORD_RE.test(raw) ? raw : raw.trim().toLowerCase();
83
+ if (!word || costs.has(word)) continue;
84
+
85
+ const cost = Math.log((i + 1) * Math.max(logN, 1));
86
+ costs.set(word, cost);
87
+
88
+ // Also index the accent-free form so a custom dictionary keeps
89
+ // working when `normalizeAccents` is enabled (the default). The
90
+ // bundled list is pure ASCII, so this is skipped there.
91
+ if (NON_ASCII_RE.test(word)) {
92
+ const key = normalizeWordKey(word);
93
+ if (!costs.has(key)) costs.set(key, cost);
94
+ if (key.length > maxLength) maxLength = key.length;
95
+ }
96
+
97
+ if (word.length > maxLength) maxLength = word.length;
98
+ if (cost < minCost) minCost = cost;
99
+ }
100
+
101
+ return {costs, maxLength, minCost};
102
+ }
103
+
104
+ let sharedDictionary = null;
105
+
106
+ /**
107
+ * Load the bundled English dictionary. `require()` is intentionally lazy so
108
+ * importing the package stays cheap; the 1.8 MB JSON is only parsed on first
109
+ * use.
110
+ * @returns {{costs: Map<string, number>, maxLength: number, minCost: number}}
111
+ */
112
+ function getSharedDictionary() {
113
+ if (sharedDictionary === null) {
114
+ sharedDictionary = buildDictionary(require('./words-en.json'));
115
+ }
116
+ return sharedDictionary;
117
+ }
118
+
119
+ /**
120
+ * Resolve the `dictionary` constructor option: an array of words or a path to
121
+ * a `.json` / newline separated `.txt` file.
122
+ * @param {string[]|string} dictionary
123
+ * @returns {string[]}
124
+ */
125
+ function loadCustomDictionary(dictionary) {
126
+ if (Array.isArray(dictionary)) return dictionary;
127
+ if (typeof dictionary !== 'string') {
128
+ throw new TypeError(
129
+ 'wordsninja: `dictionary` must be an array of words or a path to a .json/.txt file'
130
+ );
131
+ }
132
+
133
+ const fs = nodeRequire === null ? null : nodeRequire('node:fs');
134
+ const path = nodeRequire === null ? null : nodeRequire('node:path');
135
+
136
+ if (fs === null || path === null) {
137
+ throw new Error(
138
+ 'wordsninja: file-path dictionaries are only supported in Node.js. ' +
139
+ 'Pass an array of words instead.'
140
+ );
141
+ }
142
+
143
+ const file = path.resolve(dictionary);
144
+ const text = fs.readFileSync(file, 'utf8');
145
+
146
+ if (/\.json$/i.test(file)) {
147
+ const parsed = JSON.parse(text);
148
+ if (!Array.isArray(parsed)) {
149
+ throw new TypeError('wordsninja: a JSON dictionary must contain an array of strings');
150
+ }
151
+ return parsed;
152
+ }
153
+
154
+ return text.split(/\r?\n/);
155
+ }
156
+
157
+ /**
158
+ * @param {string} input
159
+ * @param {number} offset Absolute index of `piece` inside `input`
160
+ * @param {{normalizeAccents: boolean}} options
161
+ * @returns {{norm: string, starts: number[], ends: number[]}}
162
+ */
163
+ function normalizePiece(piece, offset, options) {
164
+ let norm = '';
165
+ const starts = [];
166
+ const ends = [];
167
+
168
+ let index = offset;
169
+ for (const char of piece) {
170
+ let out;
171
+
172
+ if (char.codePointAt(0) < 0x80) {
173
+ // Fast path: ASCII.
174
+ out = char >= 'A' && char <= 'Z' ? char.toLowerCase() : char;
175
+ } else if (char === '\u2019') {
176
+ out = "'";
177
+ } else {
178
+ out = char;
179
+ if (options.normalizeAccents) {
180
+ out = out.normalize('NFD').replace(COMBINING_MARKS_RE, '');
181
+ }
182
+ out = out.toLowerCase().replace(/\u2019/g, "'");
183
+ }
184
+
185
+ for (const normalized of out) {
186
+ norm += normalized;
187
+ starts.push(index);
188
+ ends.push(index + char.length);
189
+ }
190
+
191
+ index += char.length;
192
+ }
193
+
194
+ return {norm, starts, ends};
195
+ }
196
+
197
+ function isUpper(char) {
198
+ return char !== char.toLowerCase();
199
+ }
200
+
201
+ function isLower(char) {
202
+ return char !== char.toUpperCase();
203
+ }
204
+
205
+ function isDigit(char) {
206
+ return /\p{N}/u.test(char);
207
+ }
208
+
209
+ /**
210
+ * Camel case boundaries: `XMLHttpRequest` -> `XML|Http|Request`,
211
+ * `Note12` -> `Note|12`.
212
+ * @param {string} run
213
+ * @returns {Array<[number, number]>}
214
+ */
215
+ function camelRanges(run) {
216
+ const ranges = [];
217
+ let start = 0;
218
+
219
+ for (let i = 1; i < run.length; i++) {
220
+ const prev = run[i - 1];
221
+ const current = run[i];
222
+ const next = i + 1 < run.length ? run[i + 1] : '';
223
+
224
+ const lowerOrDigitBeforeUpper = (isLower(prev) || isDigit(prev)) && isUpper(current);
225
+ const acronymBeforeWord = isUpper(prev) && isUpper(current) && isLower(next);
226
+ const letterBeforeDigit = isLower(prev) && isDigit(current);
227
+
228
+ if (lowerOrDigitBeforeUpper || acronymBeforeWord || letterBeforeDigit) {
229
+ ranges.push([start, i]);
230
+ start = i;
231
+ }
232
+ }
233
+
234
+ ranges.push([start, run.length]);
235
+ return ranges;
236
+ }
237
+
238
+ /**
239
+ * Validate and normalize `splitSentence()` options.
240
+ * @param {object|undefined} options
241
+ * @returns {object}
242
+ */
243
+ function normalizeOptions(options) {
244
+ if (options === undefined || options === null) options = {};
245
+ if (typeof options !== 'object' || Array.isArray(options)) {
246
+ throw new TypeError('wordsninja: options must be an object');
247
+ }
248
+
249
+ const normalized = {
250
+ camelCaseSplitter: false,
251
+ capitalizeFirstLetter: false,
252
+ joinWords: false,
253
+ withOffsets: false,
254
+ normalizeAccents: true,
255
+ mergeNumbers: true,
256
+ mergeContractions: true,
257
+ maxWordLength: DEFAULT_MAX_WORD_LENGTH,
258
+ unknownWordPenalty: DEFAULT_UNKNOWN_WORD_PENALTY,
259
+ };
260
+
261
+ for (const name of BOOLEAN_OPTIONS) {
262
+ const value = options[name];
263
+ if (value === undefined) continue;
264
+ if (typeof value !== 'boolean') {
265
+ throw new TypeError(`wordsninja: option \`${name}\` must be a boolean`);
266
+ }
267
+ normalized[name] = value;
268
+ }
269
+
270
+ if (options.maxWordLength !== undefined) {
271
+ const value = options.maxWordLength;
272
+ if (!Number.isInteger(value) || value < 1) {
273
+ throw new TypeError('wordsninja: option `maxWordLength` must be a positive integer');
274
+ }
275
+ normalized.maxWordLength = value;
276
+ }
277
+
278
+ if (options.unknownWordPenalty !== undefined) {
279
+ const value = options.unknownWordPenalty;
280
+ if (value !== null && (typeof value !== 'number' || !Number.isFinite(value) || value < 0)) {
281
+ throw new TypeError(
282
+ 'wordsninja: option `unknownWordPenalty` must be a non-negative number or null'
283
+ );
284
+ }
285
+ normalized.unknownWordPenalty = value;
286
+ }
287
+
288
+ return normalized;
289
+ }
290
+
291
+ /**
292
+ * Merge adjacent tokens that belong together numerically (`1|2` -> `12`) or
293
+ * grammatically (`wang|'s` -> `wang's`).
294
+ * @param {string} input
295
+ * @param {Array<{start: number, end: number}>} tokens
296
+ * @param {object} options
297
+ * @returns {Array<{word: string, start: number, end: number}>}
298
+ */
299
+ function mergeTokens(input, tokens, options) {
300
+ const merged = [];
301
+
302
+ for (const token of tokens) {
303
+ const word = input.slice(token.start, token.end);
304
+ const previous = merged[merged.length - 1];
305
+
306
+ if (previous && previous.end === token.start) {
307
+ if (
308
+ options.mergeNumbers &&
309
+ DIGITS_RE.test(previous.word) &&
310
+ DIGITS_RE.test(word)
311
+ ) {
312
+ previous.word += word;
313
+ previous.end = token.end;
314
+ continue;
315
+ }
316
+
317
+ const suffix = (
318
+ word.charAt(0) === '\u2019' ? "'" + word.slice(1) : word
319
+ ).toLowerCase();
320
+ if (
321
+ options.mergeContractions &&
322
+ CONTRACTION_SUFFIXES.has(suffix) &&
323
+ CONTRACTION_START_RE.test(previous.word)
324
+ ) {
325
+ previous.word += word;
326
+ previous.end = token.end;
327
+ continue;
328
+ }
329
+ }
330
+
331
+ merged.push({word, start: token.start, end: token.end});
332
+ }
333
+
334
+ return merged;
335
+ }
336
+
337
+ function capitalizeFirstLetter(word) {
338
+ return word.charAt(0).toUpperCase() + word.slice(1);
339
+ }
340
+
341
+ class WordsNinja {
342
+ /**
343
+ * @param {object} [options]
344
+ * @param {string[]|string} [options.dictionary] Custom dictionary: an array
345
+ * of words or a path to a `.json`/`.txt` file. Defaults to the bundled
346
+ * English dictionary.
347
+ */
348
+ constructor(options = {}) {
349
+ if (options === null || typeof options !== 'object' || Array.isArray(options)) {
350
+ throw new TypeError('wordsninja: constructor options must be an object');
351
+ }
352
+ if (
353
+ options.dictionary !== undefined &&
354
+ !Array.isArray(options.dictionary) &&
355
+ typeof options.dictionary !== 'string'
356
+ ) {
357
+ throw new TypeError(
358
+ 'wordsninja: `dictionary` must be an array of words or a path to a .json/.txt file'
359
+ );
360
+ }
361
+
362
+ this._customDictionary = options.dictionary;
363
+ this._base = null;
364
+ this._ready = false;
365
+ this._additions = new Map();
366
+ this._additionsMaxLength = 0;
367
+ }
368
+
369
+ /**
370
+ * Load the dictionary. This is optional: the dictionary is loaded lazily
371
+ * on first use, so `await loadDictionary()` only exists for backwards
372
+ * compatibility.
373
+ * @returns {Promise<this>}
374
+ */
375
+ async loadDictionary() {
376
+ this._ensureReady();
377
+ return this;
378
+ }
379
+
380
+ /**
381
+ * Whether the dictionary has been loaded.
382
+ * @returns {boolean}
383
+ */
384
+ get ready() {
385
+ return this._ready;
386
+ }
387
+
388
+ /**
389
+ * Add word(s) to this instance's dictionary. Added words get the lowest
390
+ * possible cost, so they win over bundled dictionary words.
391
+ * @param {string|string[]} words
392
+ * @returns {this}
393
+ */
394
+ addWords(words) {
395
+ const list = typeof words === 'string' ? [words] : words;
396
+ if (!Array.isArray(list)) {
397
+ throw new TypeError('wordsninja: addWords() expects a string or an array of strings');
398
+ }
399
+
400
+ this._ensureReady();
401
+ const cost = Number.isFinite(this._base.minCost) ? this._base.minCost : 0;
402
+
403
+ for (const raw of list) {
404
+ if (typeof raw !== 'string') {
405
+ throw new TypeError('wordsninja: addWords() expects a string or an array of strings');
406
+ }
407
+
408
+ const word = raw.trim();
409
+ if (!word) continue;
410
+ if (/\s/u.test(word)) {
411
+ throw new TypeError('wordsninja: a word cannot contain whitespace');
412
+ }
413
+
414
+ const lowercase = word.toLowerCase();
415
+ const keys = new Set([lowercase, normalizeWordKey(word)]);
416
+
417
+ for (const key of keys) {
418
+ this._additions.set(key, cost);
419
+ if (key.length > this._additionsMaxLength) {
420
+ this._additionsMaxLength = key.length;
421
+ }
422
+ }
423
+ }
424
+
425
+ return this;
426
+ }
427
+
428
+ /**
429
+ * Split a string into words.
430
+ *
431
+ * @param {string} string The string to split
432
+ * @param {object} [options]
433
+ * @param {boolean} [options.camelCaseSplitter=false] Split camelCase/PascalCase
434
+ * boundaries (`XMLHttpRequest` -> `XML Http Request`)
435
+ * @param {boolean} [options.capitalizeFirstLetter=false] Capitalize the first
436
+ * letter of every resulting word
437
+ * @param {boolean} [options.joinWords=false] Return a space separated string
438
+ * instead of an array
439
+ * @param {boolean} [options.withOffsets=false] Return `{word, start, end}`
440
+ * objects instead of strings. Ignored when `joinWords` is true.
441
+ * @param {boolean} [options.normalizeAccents=true] Strip diacritics before
442
+ * matching (`café` matches `cafe`) while keeping the original text in the
443
+ * result
444
+ * @param {boolean} [options.mergeNumbers=true] Merge adjacent digits
445
+ * (`1 2 3` -> `123`)
446
+ * @param {boolean} [options.mergeContractions=true] Attach `'s`, `'re`,
447
+ * `'ll`, `'ve`, `'d`, `'m`, `'t` to the preceding word
448
+ * @param {number} [options.maxWordLength=64] Maximum length of a dictionary
449
+ * word considered at each position. Lower values are faster; words longer
450
+ * than this are treated as unknown.
451
+ * @param {number|null} [options.unknownWordPenalty=10] Cost per unmatched
452
+ * character. Unmatched characters are kept as tokens. Set to `null` to
453
+ * drop characters that cannot be segmented.
454
+ * @returns {string[]|string|Array<{word: string, start: number, end: number}>}
455
+ */
456
+ splitSentence(string, options) {
457
+ if (typeof string !== 'string') {
458
+ throw new TypeError('wordsninja: splitSentence() expects a string');
459
+ }
460
+
461
+ const settings = normalizeOptions(options);
462
+ this._ensureReady();
463
+
464
+ const tokens = [];
465
+ const maxWordLength = Math.min(
466
+ Math.max(this._base.maxLength, this._additionsMaxLength),
467
+ settings.maxWordLength
468
+ );
469
+
470
+ for (const match of string.matchAll(RUN_RE)) {
471
+ const run = match[0];
472
+ const runStart = match.index;
473
+ const ranges = settings.camelCaseSplitter
474
+ ? camelRanges(run)
475
+ : [[0, run.length]];
476
+
477
+ for (const [from, to] of ranges) {
478
+ if (to <= from) continue;
479
+ this._segmentRun(
480
+ run.slice(from, to),
481
+ runStart + from,
482
+ runStart + to,
483
+ settings,
484
+ maxWordLength,
485
+ tokens
486
+ );
487
+ }
488
+ }
489
+
490
+ const merged = mergeTokens(string, tokens, settings);
491
+ const words = merged.map((token) =>
492
+ settings.capitalizeFirstLetter
493
+ ? capitalizeFirstLetter(token.word)
494
+ : token.word
495
+ );
496
+
497
+ if (settings.joinWords) return words.join(' ');
498
+ if (settings.withOffsets) {
499
+ return merged.map((token, index) => ({
500
+ word: words[index],
501
+ start: token.start,
502
+ end: token.end,
503
+ }));
504
+ }
505
+ return words;
506
+ }
507
+
508
+ /**
509
+ * Run the Viterbi search for a single alphanumeric run and append its
510
+ * tokens (in source order) to `out`.
511
+ * @private
512
+ */
513
+ _segmentRun(piece, absoluteStart, absoluteEnd, settings, maxWordLength, out) {
514
+ const {norm, starts, ends} = normalizePiece(piece, absoluteStart, settings);
515
+ const length = norm.length;
516
+ if (length === 0) return;
517
+
518
+ const best = new Float64Array(length + 1).fill(Infinity);
519
+ const back = new Int32Array(length + 1);
520
+ const kind = new Uint8Array(length + 1);
521
+ best[0] = 0;
522
+
523
+ const unknownCost =
524
+ settings.unknownWordPenalty === null ? 1e6 : settings.unknownWordPenalty;
525
+
526
+ for (let i = 1; i <= length; i++) {
527
+ const limit = Math.min(maxWordLength, i);
528
+
529
+ for (let k = 1; k <= limit; k++) {
530
+ const cost = this._cost(norm.slice(i - k, i));
531
+ if (cost === undefined) continue;
532
+
533
+ const total = best[i - k] + cost;
534
+ if (total < best[i]) {
535
+ best[i] = total;
536
+ back[i] = k;
537
+ kind[i] = KIND_WORD;
538
+ }
539
+ }
540
+
541
+ // Every character can also be treated as unknown. This guarantees
542
+ // termination for input the dictionary cannot cover.
543
+ const unknownTotal = best[i - 1] + unknownCost;
544
+ if (unknownTotal < best[i]) {
545
+ best[i] = unknownTotal;
546
+ back[i] = 1;
547
+ kind[i] = KIND_UNKNOWN;
548
+ }
549
+ }
550
+
551
+ const reversed = [];
552
+ let i = length;
553
+
554
+ while (i > 0) {
555
+ const size = back[i];
556
+ const boundary = i === length ? absoluteEnd : starts[i];
557
+ const start = starts[i - size];
558
+ let end = ends[i - 1];
559
+ if (end > boundary) end = boundary;
560
+
561
+ const unknown = kind[i] === KIND_UNKNOWN;
562
+ const digits = DIGITS_RE.test(norm.slice(i - size, i));
563
+
564
+ // `unknownWordPenalty: null` drops characters the dictionary
565
+ // cannot cover instead of emitting them.
566
+ if (start < end && (!unknown || settings.unknownWordPenalty !== null)) {
567
+ const previous = reversed[reversed.length - 1];
568
+
569
+ // Adjacent digits are left separate here so that
570
+ // `mergeNumbers` stays in control of the final result.
571
+ if (unknown && previous && previous.unknown && previous.start === end && !(previous.digits && digits)) {
572
+ previous.start = start;
573
+ previous.digits = previous.digits && digits;
574
+ } else {
575
+ reversed.push({start, end, unknown, digits});
576
+ }
577
+ }
578
+
579
+ i -= size;
580
+ }
581
+
582
+ for (let index = reversed.length - 1; index >= 0; index--) {
583
+ out.push(reversed[index]);
584
+ }
585
+ }
586
+
587
+ /**
588
+ * @private
589
+ * @param {string} word Lowercase, accent-free candidate
590
+ * @returns {number|undefined}
591
+ */
592
+ _cost(word) {
593
+ const addition = this._additions.get(word);
594
+ if (addition !== undefined) return addition;
595
+ return this._base.costs.get(word);
596
+ }
597
+
598
+ /**
599
+ * Build (or reuse) the dictionary and mark the instance as ready.
600
+ * @private
601
+ */
602
+ _ensureReady() {
603
+ if (this._ready) return;
604
+
605
+ this._base =
606
+ this._customDictionary === undefined
607
+ ? getSharedDictionary()
608
+ : buildDictionary(loadCustomDictionary(this._customDictionary));
609
+
610
+ this._ready = true;
611
+ }
612
+ }
613
+
614
+ module.exports = WordsNinja;
615
+ module.exports.WordsNinja = WordsNinja;
616
+ module.exports.default = WordsNinja;