wordsninja 1.1.0 → 2.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md ADDED
@@ -0,0 +1,71 @@
1
+ # Changelog
2
+
3
+ ## 2.0.1
4
+
5
+ More accurate segmentation of single words, plus documentation corrections and
6
+ accuracy benchmarks.
7
+
8
+ - **Segmentation.** The bundled word list contains every letter as a word (`s`
9
+ is at rank 17), so a run could be split into fragments cheaper than the word
10
+ they spell: `sand` became `s and`, `sat` became `s at`. The bundled dictionary
11
+ now ignores one-letter entries other than `a` and `i`, so a word no longer
12
+ competes with single-letter fragments of itself. Those letters follow
13
+ `unknownWordPenalty` like any other unmatched text, so
14
+ `unknownWordPenalty: null` drops them. Custom dictionaries keep their
15
+ one-letter entries, and `addWords('x')` still works. Across the 126,136-word
16
+ list, segmented one word at a time, this fixes 226 words and regresses none;
17
+ the 125,317 words that already segmented correctly are untouched, and of the
18
+ 593 that are still wrong, one changed (`somethings` now segments as
19
+ `some things` instead of `something s`).
20
+ - **No per-token cost.** A 0.3 cost per token on top of the dictionary cost was
21
+ tried as well. It raised `corpus.txt` from 176 to 178 exact sentences and
22
+ would have fixed 113 more isolated words (`onto`, `thereby`), but on the
23
+ held-out prose set it gained no sentence and lost six: `each other` became
24
+ `eachother`, `on to` became `onto`, `to us` became `tous`. The released code
25
+ keeps the plain cheapest-path search, which wins on held-out prose and on the
26
+ combined corpus.
27
+ - **Accuracy benchmarks.** `npm run bench:accuracy` strips the spaces from
28
+ `bench/corpus.txt` (202 tuned sentences) and `bench/prose.txt` (600 held-out
29
+ sentences from six public-domain novels) and reports exact match plus
30
+ word-level precision, recall and F1; CI enforces a floor on both. Against
31
+ 2.0.0, exact match went from 173 to 176 on the tuned corpus and from 464 to
32
+ 474 on the prose corpus, with no sentence regressing on either. The prose
33
+ improvements all come from the letter `s` no longer being a word: `sat` and
34
+ `sit` inside sentences, possessives such as `hers`, and contractions such as
35
+ `that's`. Scope: every bundled word segmented on its own, plus these 802
36
+ sentences under the default options. Arbitrary multi-word strings were not
37
+ exhaustively compared, so a rare cost tie outside the corpora may still
38
+ change.
39
+ - **Merging docs.** The `mergeNumbers` / `mergeContractions` docs claimed `1 2 3`
40
+ became `123` and `wang 's` became `wang's`. Merging never crosses a separator,
41
+ so both stay split. README, `index.d.ts` and the `index.js` JSDoc now show the
42
+ real behavior, and a test pins it.
43
+ - **Performance section.** The 3× claim sat next to a table that implied 3.3×.
44
+ Both now use re-measured numbers: ~2.8× faster than v1 (85 ms → 30 ms for
45
+ 4,500 characters; 388 ms → 137 ms for 21,000), and a faster dictionary load
46
+ (~83 ms → ~46 ms, including JSON parsing and indexing).
47
+ - **`NOTICE`** now says the list was de-duplicated instead of claiming every
48
+ word is unchanged.
49
+
50
+ ## 2.0.0
51
+
52
+ The v2 rewrite is a breaking release: output defaults and some return values
53
+ changed.
54
+
55
+ - **Rewritten segmentation.** Dynamic programming over a `Map` and typed arrays,
56
+ about 2.8× faster than v1 on long input. The dictionary is parsed on first
57
+ use, not at import.
58
+ - **New options.** `withOffsets`, `normalizeAccents`, `mergeNumbers`,
59
+ `mergeContractions`, `maxWordLength`, `unknownWordPenalty`, and a `dictionary`
60
+ constructor option.
61
+ - **Defaults changed.** Accents are normalized, numbers and contractions merge,
62
+ and text the dictionary cannot cover is kept instead of dropped.
63
+ - **Isolated instances.** `addWords()` no longer mutates a module-global
64
+ dictionary and returns the instance. Empty entries are ignored; non-strings
65
+ and words containing whitespace are rejected.
66
+ - **Types and module formats.** TypeScript declarations, plus ESM and CommonJS
67
+ entry points.
68
+ - **Word list.** De-duplicated from 126,138 to 126,136 entries; the relative
69
+ frequency order is unchanged.
70
+ - **Errors instead of hangs.** The API initializes lazily and validates input
71
+ instead of looping on an unloaded dictionary.
package/NOTICE ADDED
@@ -0,0 +1,21 @@
1
+ WordsNinja
2
+ Copyright (c) Parsa Kafi
3
+
4
+ The bundled English word list in `words-en.json` is derived from the word list
5
+ shipped with the `wordninja` Python package:
6
+
7
+ https://pypi.org/project/wordninja/
8
+ https://github.com/keredson/wordninja
9
+
10
+ `wordninja` is distributed under the MIT License. The list has been converted
11
+ from a newline separated text file to JSON and de-duplicated; the remaining
12
+ words keep their original frequency order.
13
+
14
+ The segmentation algorithm follows the approach described in:
15
+
16
+ https://stackoverflow.com/questions/8870261/how-to-split-text-without-spaces-into-list-of-words
17
+
18
+ The camel case splitting logic was originally inspired by the
19
+ `split-camelcase-to-words` package:
20
+
21
+ https://www.npmjs.com/package/split-camelcase-to-words
package/README.md CHANGED
@@ -1,124 +1,309 @@
1
- # WordsNinja
2
- Split an English sentence that lacks spaces and accents into separate words.
3
-
4
- [![https://nodei.co/npm/wordsninja.png?downloads=true&downloadRank=true&stars=true](https://nodei.co/npm/wordsninja.png?downloads=true&downloadRank=true&stars=true)](https://www.npmjs.com/package/wordsninja)
5
-
6
- ## Install
7
- ```bash
8
- npm install wordsninja --save
9
- ```
10
- Load package
11
- ```js
12
- const WordsNinjaPack = require('wordsninja');
13
- const WordsNinja = new WordsNinjaPack();
14
- ```
15
-
16
-
17
- Load dictionary
18
-
19
- ```js
20
- await WordsNinja.loadDictionary(); // First load dictionary
21
- ```
22
-
23
-
24
-
25
- Add word(s)
26
-
27
- ```js
28
- WordsNinja.addWords('new word');
29
- ```
30
- Parameters
31
-
32
- - `word`: The word(s) (string|array)
33
-
34
-
35
-
36
- ### Split sentence
37
-
38
- ```js
39
- let words = WordsNinja.splitSentence(string, {camelCaseSplitter, capitalizeFirstLetter, joinWords});
40
- ```
41
-
42
- Parameters
43
- - `string`: The string for split
44
- - `options`
45
- - `camelCaseSplitter`: Split by Camel Case, Default is `false` (optional)
46
- - `capitalizeFirstLetter`: Capitalize First Letter, Default is `false` (optional)
47
- - `joinWords`: Return join words as sentence, Default is `false` (optional)
48
-
49
- Example
50
-
51
- ```js
52
- (async () => {
53
- await WordsNinja.loadDictionary(); // First load dictionary
54
- let string = 'youneedtolearnfromyourmistakes';
55
- let words = WordsNinja.splitSentence(string);
56
- console.log(words);
57
- })();
58
- ```
59
-
60
- Result
61
-
62
- ```js
63
- [ 'you', 'need', 'to', 'learn', 'from', 'your', 'mistakes' ]
64
- ```
65
-
66
-
67
-
68
- #### More options
69
-
70
- ```js
71
- let string = 'youneedtolearnfromyourmistakes';
72
- let words = WordsNinja.splitSentence(string,
73
- {
74
- camelCaseSplitter: true, // Camel case splitting
75
- capitalizeFirstLetter: true, // Capitalize first letter of result
76
- joinWords: true // Join words with spaces
77
- }
78
- );
79
- console.log(words);
80
- ```
81
-
82
- Result
83
-
84
- ```
85
- You Need To Learn From Your Mistakes
86
- ```
87
-
88
-
89
-
90
- ### Add Word(s)
91
-
92
- You can add new word(s) to dictionary in runtime
93
- ```js
94
- WordsNinja.addWords('Parsa'); // Add one word
95
- WordsNinja.addWords(['Parsa', 'Kafi']); // Add one or more words
96
- ```
97
-
98
- Example
99
-
100
- ```js
101
- let string = 'parsayouneedtolearnfromyourmistakes';
102
- WordsNinja.addWords('Parsa');
103
- let words = WordsNinja.splitSentence(string,
104
- {
105
- capitalizeFirstLetter: true, // Capitalize first letter of result
106
- joinWords: true // Join words with spaces
107
- }
108
- );
109
- console.log(words);
110
- ```
111
-
112
- Result
113
-
114
- ```
115
- Parsa You Need To Learn From Your Mistakes
116
- ```
117
-
118
-
119
-
120
-
121
- ## Acknowledgement
122
-
123
- Algorithm from [How to split text without spaces into list of words?](https://stackoverflow.com/questions/8870261/how-to-split-text-without-spaces-into-list-of-words).
124
- List of words from [wordninja](https://pypi.org/project/wordninja) python package. Camel case splitter based on [split-camelcase-to-words](https://www.npmjs.com/package/split-camelcase-to-words) package.
1
+ # WordsNinja
2
+
3
+ Split English text that has no spaces into words.
4
+
5
+ ```js
6
+ import WordsNinja from 'wordsninja';
7
+
8
+ const ninja = new WordsNinja();
9
+ ninja.splitSentence('youneedtolearnfromyourmistakes');
10
+ // → ['you', 'need', 'to', 'learn', 'from', 'your', 'mistakes']
11
+ ```
12
+
13
+ [![npm version](https://img.shields.io/npm/v/wordsninja.svg)](https://www.npmjs.com/package/wordsninja)
14
+ [![npm downloads](https://img.shields.io/npm/dm/wordsninja.svg)](https://www.npmjs.com/package/wordsninja)
15
+ [![CI](https://github.com/parsakafi/wordsninja/actions/workflows/ci.yml/badge.svg)](https://github.com/parsakafi/wordsninja/actions/workflows/ci.yml)
16
+ [![license](https://img.shields.io/npm/l/wordsninja.svg)](./LICENSE)
17
+
18
+ ## Why WordsNinja
19
+
20
+ - **Zero dependencies** and a small API surface.
21
+ - **~2.8× faster than v1** — a dynamic-programming search over a frequency-ordered dictionary, backed by a `Map` and typed
22
+ arrays.
23
+ - **Keeps your text** — case is preserved and characters the dictionary cannot cover are returned instead of silently
24
+ dropped.
25
+ - **Offset-aware** — get `{ word, start, end }` tokens so you can highlight matches in the original string.
26
+ - **Custom dictionaries** — pass an array of words or point at a `.json` / `.txt` file.
27
+ - **TypeScript types and ESM + CommonJS** entry points.
28
+ - **Lazy dictionary load** — importing the package no longer parses the 1.8 MB word list.
29
+
30
+ ## Install
31
+
32
+ ```bash
33
+ npm install wordsninja
34
+ ```
35
+
36
+ Requires Node.js 18 or newer.
37
+
38
+ ## Quick start
39
+
40
+ CommonJS:
41
+
42
+ ```js
43
+ const WordsNinja = require('wordsninja');
44
+ const ninja = new WordsNinja();
45
+
46
+ console.log(ninja.splitSentence('youneedtolearnfromyourmistakes'));
47
+ // [ 'you', 'need', 'to', 'learn', 'from', 'your', 'mistakes' ]
48
+ ```
49
+
50
+ ESM:
51
+
52
+ ```js
53
+ import WordsNinja from 'wordsninja';
54
+
55
+ const ninja = new WordsNinja();
56
+ ```
57
+
58
+ TypeScript:
59
+
60
+ ```ts
61
+ import WordsNinja from 'wordsninja';
62
+
63
+ const ninja = new WordsNinja();
64
+ const words: string[] = ninja.splitSentence('ilovewordsninja');
65
+ ```
66
+
67
+ The dictionary is loaded on first use. `await ninja.loadDictionary()` still works and is kept for backwards
68
+ compatibility with v1:
69
+
70
+ ```js
71
+ const ninja = new WordsNinja();
72
+ await ninja.loadDictionary(); // optional
73
+ ninja.splitSentence('helloworld');
74
+ ```
75
+
76
+ ## API
77
+
78
+ ### `new WordsNinja(options?)`
79
+
80
+ | Option | Type | Description |
81
+ |--------------|----------------------|--------------------------------------------------------------------------------------------------------------------------------------------|
82
+ | `dictionary` | `string[] \| string` | Custom dictionary. An array of words, or a path to a `.json` array or newline separated `.txt` file. Defaults to the bundled English list. |
83
+
84
+ ```js
85
+ const ninja = new WordsNinja({dictionary: ['code', 'ninja']});
86
+ ninja.splitSentence('codeninja'); // ['code', 'ninja']
87
+ ```
88
+
89
+ The `dictionary` option is validated eagerly; the file is read (and the bundled list parsed) on first use.
90
+
91
+ ### `ninja.loadDictionary(): Promise<this>`
92
+
93
+ Loads the dictionary and resolves with the instance. Optional — the dictionary is loaded lazily — but useful for warming
94
+ up in a serverless handler.
95
+
96
+ ### `ninja.addWords(words): this`
97
+
98
+ Adds one word or an array of words to **this instance's** dictionary. Added words get the lowest possible cost, so they
99
+ outrank bundled words. Matching is case-insensitive and accent-insensitive.
100
+
101
+ ```js
102
+ const ninja = new WordsNinja();
103
+ ninja.addWords('parsa').addWords(['kafi', 'wordsninja']);
104
+
105
+ ninja.splitSentence('parsayouneedtolearnfromyourmistakes');
106
+ // [ 'parsa', 'you', 'need', 'to', 'learn', 'from', 'your', 'mistakes' ]
107
+ ```
108
+
109
+ Instances do not share added words.
110
+
111
+ ### `ninja.splitSentence(input, options?): string[] | string | WordToken[]`
112
+
113
+ Splits `input` into words. Throws a `TypeError` if `input` is not a string.
114
+
115
+ | Option | Type | Default | Description |
116
+ |-------------------------|------------------|---------|-----------------------------------------------------------------------------------------------------------------------------------|
117
+ | `camelCaseSplitter` | `boolean` | `false` | Split camelCase / PascalCase boundaries. `XMLHttpRequest` → `XML Http Request`, `Note12` → `Note 12`. |
118
+ | `capitalizeFirstLetter` | `boolean` | `false` | Capitalize the first letter of every word. |
119
+ | `joinWords` | `boolean` | `false` | Return a space separated string instead of an array. Takes precedence over `withOffsets`. |
120
+ | `withOffsets` | `boolean` | `false` | Return `{ word, start, end }` objects. Offsets are UTF-16 indices into `input`. |
121
+ | `normalizeAccents` | `boolean` | `true` | Strip diacritics before matching (`café` matches `cafe`) while returning the original text. |
122
+ | `mergeNumbers` | `boolean` | `true` | Merge adjacent digit tokens: `call116116now` → `['call', '116116', 'now']`. Merging never crosses a separator, so `1 2 3` stays three tokens. |
123
+ | `mergeContractions` | `boolean` | `true` | Attach `'s`, `'re`, `'ll`, `'ve`, `'d`, `'m`, `'t` to the preceding word: `wang'sbook` → `["wang's", 'book']`. Merging never crosses a separator, so `wang 's` stays two tokens. |
124
+ | `maxWordLength` | `number` | `64` | Maximum length of a dictionary word considered at each position. Lower values are faster; longer words are treated as unknown. |
125
+ | `unknownWordPenalty` | `number \| null` | `10` | Cost per unmatched character. Unmatched characters are kept as tokens. Set to `null` to drop characters that cannot be segmented. In the bundled dictionary, a lone letter other than `a` or `i` counts as unmatched text. |
126
+
127
+ #### Offsets
128
+
129
+ ```js
130
+ const input = 'Ihave2cats';
131
+ ninja.splitSentence(input, {withOffsets: true});
132
+ // [
133
+ // { word: 'I', start: 0, end: 1 },
134
+ // { word: 'have', start: 1, end: 5 },
135
+ // { word: '2', start: 5, end: 6 },
136
+ // { word: 'cats', start: 6, end: 10 }
137
+ // ]
138
+ ```
139
+
140
+ Every offset pair slices back to its word:
141
+
142
+ ```js
143
+ for (const token of tokens) {
144
+ input.slice(token.start, token.end) === token.word; // always true
145
+ }
146
+ ```
147
+
148
+ `capitalizeFirstLetter` changes `word` but not the offsets. When `joinWords` is `true`, a string is returned and offsets
149
+ are ignored.
150
+
151
+ #### All options together
152
+
153
+ ```js
154
+ ninja.splitSentence('helloWorldHowAreYou', {
155
+ camelCaseSplitter: true,
156
+ capitalizeFirstLetter: true,
157
+ joinWords: true,
158
+ });
159
+ // 'Hello World How Are You'
160
+ ```
161
+
162
+ ## Behavior details
163
+
164
+ - **Case is preserved.** Only the dictionary lookup is case-insensitive.
165
+ - **Separators are removed.** Spaces, punctuation, emoji and other characters outside `letters`, `digits` and `'` are
166
+ not part of the result. They become boundaries between words.
167
+ - **Accents are normalized by default.** `cafélatte` → `['café', 'latte']`. Disable with `normalizeAccents: false`; the
168
+ accented character is then treated as unknown text.
169
+ - **Text the dictionary cannot cover is kept.** Non-English scripts, brand names and typos come back as tokens instead
170
+ of disappearing: `سلامدنیا` → `['سلامدنیا']`. Adjacent unknown characters are grouped into one token. Use
171
+ `unknownWordPenalty: null` if you would rather drop them.
172
+ - **Numbers and possessives are merged by default.** `call123now` → `['call', '123', 'now']`, `wang'sbook` →
173
+ `["wang's", 'book']`. Disable with `mergeNumbers: false` / `mergeContractions: false`.
174
+ - **Whole words win over cheap fragments.** The bundled word list contains every letter as an entry, so a run could be
175
+ split into pieces cheaper than the word they spell (`sand` → `s and`). The bundled dictionary ignores one-letter
176
+ entries other than `a` and `i`; those letters are treated as unmatched text, so `sand` stays `sand`. Custom
177
+ dictionaries keep their one-letter entries.
178
+ - **Ambiguous input is a guess.** `thequickbrownfox` can only be segmented by cost; some strings have more than one
179
+ plausible split.
180
+ - **Long words are capped.** `maxWordLength` bounds the candidate window per position. Words longer than the cap are
181
+ treated as unknown.
182
+
183
+ ## Custom dictionaries
184
+
185
+ ```js
186
+ // Array
187
+ const code = new WordsNinja({dictionary: ['code', 'ninja']});
188
+
189
+ // JSON file: ["code", "ninja"]
190
+ const fromJson = new WordsNinja({dictionary: './words.json'});
191
+
192
+ // Text file: one word per line
193
+ const fromText = new WordsNinja({dictionary: './words.txt'});
194
+ ```
195
+
196
+ Paths are resolved from the current working directory. An empty dictionary is valid: every run of text is returned as
197
+ unknown tokens (or dropped when `unknownWordPenalty` is `null`).
198
+
199
+ ## Bundlers and browsers
200
+
201
+ The package resolves no Node built-ins at import time, so it bundles cleanly for browsers and edge runtimes:
202
+
203
+ ```js
204
+ import WordsNinja from 'wordsninja';
205
+
206
+ const ninja = new WordsNinja();
207
+ ninja.splitSentence('youneedtolearnfromyourmistakes');
208
+ ```
209
+
210
+ The bundled word list is included in the bundle (about 610 KB gzipped) and parsed on first use. File-path dictionaries
211
+ are Node.js only; in the browser, fetch the words and pass an array instead:
212
+
213
+ ```js
214
+ const words = await (await fetch('/words.json')).json();
215
+ const ninja = new WordsNinja({dictionary: words});
216
+ ```
217
+
218
+ Offsets, custom words and every option work the same way in the browser.
219
+
220
+ ## Migrating from v1
221
+
222
+ v2 is a breaking release compared with v1:
223
+
224
+ - **`loadDictionary()` is optional.** The dictionary loads on first use. It still exists, now resolves with the
225
+ instance, and is idempotent.
226
+ - **Instances are isolated.** `addWords()` no longer changes a module-global dictionary shared by every instance.
227
+ - **The possessive/merge code actually runs.** v1 stored merged tokens at `out[-1]`, so the branch never executed.
228
+ Possessives and digit runs are now merged by default.
229
+ - **Unknown text is no longer dropped silently.** v1 returned `[]` for non-Latin input and removed accents as
230
+ separators; v2 keeps unmatched text as tokens.
231
+ - **Accents are normalized by default.** `cafélatte` now yields `['café', 'latte']` instead of `['caf', 'latte']`.
232
+ - **`addWords()` returns the instance** for chaining, ignores empty entries, and rejects non-strings or words
233
+ containing whitespace.
234
+ - **Clear errors instead of hangs.** Using the API before the dictionary was loaded used to loop until
235
+ `RangeError: Invalid array length`; it now initializes lazily and validates input.
236
+ - **Node.js 18+**, TypeScript types, and ESM are supported.
237
+ - **New options:** `withOffsets`, `normalizeAccents`, `mergeNumbers`, `mergeContractions`, `maxWordLength`,
238
+ `unknownWordPenalty`, plus a `dictionary` constructor option.
239
+
240
+ ## Performance
241
+
242
+ Measured on a desktop machine (Node 22); split workloads are best of five, dictionary load is a cold start:
243
+
244
+ | Workload | v1 | v2 |
245
+ |-------------------|-------------------------|---------------------------|
246
+ | 4,500 characters | ~85 ms | **~30 ms** |
247
+ | 21,000 characters | ~388 ms | **~137 ms** |
248
+ | Dictionary load | ~83 ms (import + build) | **~46 ms** (first use) |
249
+
250
+ v2 also supports `withOffsets` at no measurable extra cost. Numbers vary by machine; run the benchmark on your hardware.
251
+
252
+ ### Accuracy
253
+
254
+ `npm run bench:accuracy` removes the spaces from two corpora and scores the result against the intended words:
255
+
256
+ | Corpus | Sentences | Exact | Precision | Recall | F1 |
257
+ |------------------------------|-----------|-------------|-----------|--------|-------|
258
+ | `bench/corpus.txt` (tuned) | 202 | 176 (87.1%) | 96.8% | 95.6% | 96.2% |
259
+ | `bench/prose.txt` (held out) | 600 | 474 (79.0%) | 97.4% | 96.6% | 97.0% |
260
+
261
+ `prose.txt` holds 600 sentences from six public-domain novels; it is kept apart from the corpus used while tuning, so a
262
+ change cannot raise the tuned score by over-fitting. CI enforces a floor on both. Against 2.0.0, 13 more sentences are
263
+ exact (3 tuned and 10 prose) and none regressed.
264
+
265
+ Most remaining misses are cost ties a unigram model cannot resolve without context: a longer dictionary word wins over
266
+ two shorter ones (`every day` → `everyday`, `me an` → `mean`), or two two-word splits compete (`i slept` → `is lept`).
267
+
268
+ ## Development
269
+
270
+ ```bash
271
+ npm test # node:test suite
272
+ npm run test:watch # watch mode
273
+ npm run bench # segmentation speed benchmark
274
+ npm run bench:accuracy # accuracy against bench/corpus.txt and bench/prose.txt
275
+ npm run build:dictionary # validate/de-duplicate words-en.json
276
+ ```
277
+
278
+ CI runs the test suite on Node.js 18, 20, 22 and 24, verifies the published artifact with `npm pack --dry-run`, and
279
+ fails if the accuracy benchmark drops below its floor.
280
+
281
+ ## Releasing
282
+
283
+ 1. Update the version and the changelog/README notes.
284
+ 2. `npm test`
285
+ 3. Tag and push: `git tag v2.0.1 && git push origin v2.0.1`
286
+
287
+ The `Release` workflow checks that the tag matches `package.json`, runs the tests, and publishes with provenance through
288
+ npm trusted publishing. If you publish manually, use `npm publish --provenance`.
289
+
290
+ ## Limitations
291
+
292
+ WordsNinja is a dictionary-based segmenter, not a language model:
293
+
294
+ - The bundled list is English-focused. Non-English text is preserved as unknown tokens, not segmented.
295
+ - Segmentation is a best guess for ambiguous input.
296
+ - Punctuation is treated as a boundary, not preserved.
297
+ - The dictionary costs ~30 MB of heap once loaded; it is shared by all instances in a process.
298
+
299
+ ## License
300
+
301
+ [MIT](./LICENSE) © Parsa Kafi
302
+
303
+ ## Acknowledgements
304
+
305
+ - Algorithm
306
+ from [How to split text without spaces into list of words?](https://stackoverflow.com/questions/8870261/how-to-split-text-without-spaces-into-list-of-words).
307
+ - Word list from the [wordninja](https://pypi.org/project/wordninja) Python package (MIT). See [NOTICE](./NOTICE).
308
+ - v1 camelCase splitting was based
309
+ on [split-camelcase-to-words](https://www.npmjs.com/package/split-camelcase-to-words).
package/index.d.mts ADDED
@@ -0,0 +1,10 @@
1
+ import WordsNinja = require('./index.js');
2
+
3
+ export default WordsNinja;
4
+ export {WordsNinja};
5
+
6
+ export type Options = WordsNinja.Options;
7
+ export type SplitOptions = WordsNinja.SplitOptions;
8
+ export type SplitJoinedOptions = WordsNinja.SplitJoinedOptions;
9
+ export type SplitWithOffsetsOptions = WordsNinja.SplitWithOffsetsOptions;
10
+ export type WordToken = WordsNinja.WordToken;
package/index.d.ts ADDED
@@ -0,0 +1,100 @@
1
+ declare namespace WordsNinja {
2
+ interface Options {
3
+ /**
4
+ * Custom dictionary: an array of words or a path to a `.json` / `.txt`
5
+ * file. Defaults to the bundled English dictionary.
6
+ */
7
+ dictionary?: string[] | string;
8
+ }
9
+
10
+ interface SplitOptions {
11
+ /** Split camelCase / PascalCase boundaries. Defaults to `false`. */
12
+ camelCaseSplitter?: boolean;
13
+ /** Capitalize the first letter of every word. Defaults to `false`. */
14
+ capitalizeFirstLetter?: boolean;
15
+ /** Return a space separated string instead of an array. Defaults to `false`. */
16
+ joinWords?: boolean;
17
+ /**
18
+ * Return `{ word, start, end }` objects instead of strings. Offsets are
19
+ * UTF-16 indices into the input string. Ignored when `joinWords` is `true`.
20
+ */
21
+ withOffsets?: boolean;
22
+ /**
23
+ * Strip diacritics before matching (`café` matches `cafe`) while keeping
24
+ * the original text in the result. Defaults to `true`.
25
+ */
26
+ normalizeAccents?: boolean;
27
+ /**
28
+ * Merge adjacent digit tokens (`call116116now` ->
29
+ * `['call', '116116', 'now']`). Merging never crosses a separator, so
30
+ * `1 2 3` stays three tokens. Defaults to `true`.
31
+ */
32
+ mergeNumbers?: boolean;
33
+ /**
34
+ * Attach `'s`, `'re`, `'ll`, `'ve`, `'d`, `'m` and `'t` to the preceding
35
+ * word (`wang'sbook` -> `["wang's", 'book']`); `wang 's` stays two
36
+ * tokens. Defaults to `true`.
37
+ */
38
+ mergeContractions?: boolean;
39
+ /**
40
+ * Maximum length of a dictionary word considered at each position.
41
+ * Lower values are faster; longer words are treated as unknown.
42
+ * Defaults to `64`.
43
+ */
44
+ maxWordLength?: number;
45
+ /**
46
+ * Cost per unmatched character. Unmatched characters are kept as tokens.
47
+ * Set to `null` to drop characters that cannot be segmented. In the
48
+ * bundled dictionary, a lone letter other than `a` or `i` counts as
49
+ * unmatched text. Defaults to `10`.
50
+ */
51
+ unknownWordPenalty?: number | null;
52
+ }
53
+
54
+ interface WordToken {
55
+ /** The matched text, sliced from the original input. */
56
+ word: string;
57
+ /** Inclusive UTF-16 start offset. */
58
+ start: number;
59
+ /** Exclusive UTF-16 end offset. */
60
+ end: number;
61
+ }
62
+
63
+ interface SplitWithOffsetsOptions extends SplitOptions {
64
+ withOffsets: true;
65
+ joinWords?: false;
66
+ }
67
+
68
+ interface SplitJoinedOptions extends SplitOptions {
69
+ joinWords: true;
70
+ }
71
+ }
72
+
73
+ declare class WordsNinja {
74
+ /**
75
+ * @param options Custom dictionary configuration.
76
+ */
77
+ constructor(options?: WordsNinja.Options);
78
+
79
+ /** Whether the dictionary has been loaded. */
80
+ readonly ready: boolean;
81
+
82
+ /**
83
+ * Load the dictionary. Optional: the dictionary is loaded lazily on first
84
+ * use. Kept for backwards compatibility with v1.
85
+ */
86
+ loadDictionary(): Promise<this>;
87
+
88
+ /**
89
+ * Add word(s) to this instance's dictionary. Added words get the lowest
90
+ * possible cost, so they win over bundled dictionary words.
91
+ */
92
+ addWords(words: string | string[]): this;
93
+
94
+ /** Split a string into words. */
95
+ splitSentence(input: string, options: WordsNinja.SplitJoinedOptions): string;
96
+ splitSentence(input: string, options: WordsNinja.SplitWithOffsetsOptions): WordsNinja.WordToken[];
97
+ splitSentence(input: string, options?: WordsNinja.SplitOptions): string[];
98
+ }
99
+
100
+ export = WordsNinja;