wordsninja 1.1.0 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +71 -0
- package/NOTICE +21 -0
- package/README.md +309 -124
- package/index.d.mts +10 -0
- package/index.d.ts +100 -0
- package/index.js +642 -167
- package/index.mjs +4 -0
- package/package.json +73 -29
- package/words-en.json +0 -2
- package/example.js +0 -15
package/CHANGELOG.md
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 2.0.1
|
|
4
|
+
|
|
5
|
+
More accurate segmentation of single words, plus documentation corrections and
|
|
6
|
+
accuracy benchmarks.
|
|
7
|
+
|
|
8
|
+
- **Segmentation.** The bundled word list contains every letter as a word (`s`
|
|
9
|
+
is at rank 17), so a run could be split into fragments cheaper than the word
|
|
10
|
+
they spell: `sand` became `s and`, `sat` became `s at`. The bundled dictionary
|
|
11
|
+
now ignores one-letter entries other than `a` and `i`, so a word no longer
|
|
12
|
+
competes with single-letter fragments of itself. Those letters follow
|
|
13
|
+
`unknownWordPenalty` like any other unmatched text, so
|
|
14
|
+
`unknownWordPenalty: null` drops them. Custom dictionaries keep their
|
|
15
|
+
one-letter entries, and `addWords('x')` still works. Across the 126,136-word
|
|
16
|
+
list, segmented one word at a time, this fixes 226 words and regresses none;
|
|
17
|
+
the 125,317 words that already segmented correctly are untouched, and of the
|
|
18
|
+
593 that are still wrong, one changed (`somethings` now segments as
|
|
19
|
+
`some things` instead of `something s`).
|
|
20
|
+
- **No per-token cost.** A 0.3 cost per token on top of the dictionary cost was
|
|
21
|
+
tried as well. It raised `corpus.txt` from 176 to 178 exact sentences and
|
|
22
|
+
would have fixed 113 more isolated words (`onto`, `thereby`), but on the
|
|
23
|
+
held-out prose set it gained no sentence and lost six: `each other` became
|
|
24
|
+
`eachother`, `on to` became `onto`, `to us` became `tous`. The released code
|
|
25
|
+
keeps the plain cheapest-path search, which wins on held-out prose and on the
|
|
26
|
+
combined corpus.
|
|
27
|
+
- **Accuracy benchmarks.** `npm run bench:accuracy` strips the spaces from
|
|
28
|
+
`bench/corpus.txt` (202 tuned sentences) and `bench/prose.txt` (600 held-out
|
|
29
|
+
sentences from six public-domain novels) and reports exact match plus
|
|
30
|
+
word-level precision, recall and F1; CI enforces a floor on both. Against
|
|
31
|
+
2.0.0, exact match went from 173 to 176 on the tuned corpus and from 464 to
|
|
32
|
+
474 on the prose corpus, with no sentence regressing on either. The prose
|
|
33
|
+
improvements all come from the letter `s` no longer being a word: `sat` and
|
|
34
|
+
`sit` inside sentences, possessives such as `hers`, and contractions such as
|
|
35
|
+
`that's`. Scope: every bundled word segmented on its own, plus these 802
|
|
36
|
+
sentences under the default options. Arbitrary multi-word strings were not
|
|
37
|
+
exhaustively compared, so a rare cost tie outside the corpora may still
|
|
38
|
+
change.
|
|
39
|
+
- **Merging docs.** The `mergeNumbers` / `mergeContractions` docs claimed `1 2 3`
|
|
40
|
+
became `123` and `wang 's` became `wang's`. Merging never crosses a separator,
|
|
41
|
+
so both stay split. README, `index.d.ts` and the `index.js` JSDoc now show the
|
|
42
|
+
real behavior, and a test pins it.
|
|
43
|
+
- **Performance section.** The 3× claim sat next to a table that implied 3.3×.
|
|
44
|
+
Both now use re-measured numbers: ~2.8× faster than v1 (85 ms → 30 ms for
|
|
45
|
+
4,500 characters; 388 ms → 137 ms for 21,000), and a faster dictionary load
|
|
46
|
+
(~83 ms → ~46 ms, including JSON parsing and indexing).
|
|
47
|
+
- **`NOTICE`** now says the list was de-duplicated instead of claiming every
|
|
48
|
+
word is unchanged.
|
|
49
|
+
|
|
50
|
+
## 2.0.0
|
|
51
|
+
|
|
52
|
+
The v2 rewrite is a breaking release: output defaults and some return values
|
|
53
|
+
changed.
|
|
54
|
+
|
|
55
|
+
- **Rewritten segmentation.** Dynamic programming over a `Map` and typed arrays,
|
|
56
|
+
about 2.8× faster than v1 on long input. The dictionary is parsed on first
|
|
57
|
+
use, not at import.
|
|
58
|
+
- **New options.** `withOffsets`, `normalizeAccents`, `mergeNumbers`,
|
|
59
|
+
`mergeContractions`, `maxWordLength`, `unknownWordPenalty`, and a `dictionary`
|
|
60
|
+
constructor option.
|
|
61
|
+
- **Defaults changed.** Accents are normalized, numbers and contractions merge,
|
|
62
|
+
and text the dictionary cannot cover is kept instead of dropped.
|
|
63
|
+
- **Isolated instances.** `addWords()` no longer mutates a module-global
|
|
64
|
+
dictionary and returns the instance. Empty entries are ignored; non-strings
|
|
65
|
+
and words containing whitespace are rejected.
|
|
66
|
+
- **Types and module formats.** TypeScript declarations, plus ESM and CommonJS
|
|
67
|
+
entry points.
|
|
68
|
+
- **Word list.** De-duplicated from 126,138 to 126,136 entries; the relative
|
|
69
|
+
frequency order is unchanged.
|
|
70
|
+
- **Errors instead of hangs.** The API initializes lazily and validates input
|
|
71
|
+
instead of looping on an unloaded dictionary.
|
package/NOTICE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
WordsNinja
|
|
2
|
+
Copyright (c) Parsa Kafi
|
|
3
|
+
|
|
4
|
+
The bundled English word list in `words-en.json` is derived from the word list
|
|
5
|
+
shipped with the `wordninja` Python package:
|
|
6
|
+
|
|
7
|
+
https://pypi.org/project/wordninja/
|
|
8
|
+
https://github.com/keredson/wordninja
|
|
9
|
+
|
|
10
|
+
`wordninja` is distributed under the MIT License. The list has been converted
|
|
11
|
+
from a newline separated text file to JSON and de-duplicated; the remaining
|
|
12
|
+
words keep their original frequency order.
|
|
13
|
+
|
|
14
|
+
The segmentation algorithm follows the approach described in:
|
|
15
|
+
|
|
16
|
+
https://stackoverflow.com/questions/8870261/how-to-split-text-without-spaces-into-list-of-words
|
|
17
|
+
|
|
18
|
+
The camel case splitting logic was originally inspired by the
|
|
19
|
+
`split-camelcase-to-words` package:
|
|
20
|
+
|
|
21
|
+
https://www.npmjs.com/package/split-camelcase-to-words
|
package/README.md
CHANGED
|
@@ -1,124 +1,309 @@
|
|
|
1
|
-
# WordsNinja
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
```
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
```js
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
)
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
```
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
WordsNinja
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
1
|
+
# WordsNinja
|
|
2
|
+
|
|
3
|
+
Split English text that has no spaces into words.
|
|
4
|
+
|
|
5
|
+
```js
|
|
6
|
+
import WordsNinja from 'wordsninja';
|
|
7
|
+
|
|
8
|
+
const ninja = new WordsNinja();
|
|
9
|
+
ninja.splitSentence('youneedtolearnfromyourmistakes');
|
|
10
|
+
// → ['you', 'need', 'to', 'learn', 'from', 'your', 'mistakes']
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
[](https://www.npmjs.com/package/wordsninja)
|
|
14
|
+
[](https://www.npmjs.com/package/wordsninja)
|
|
15
|
+
[](https://github.com/parsakafi/wordsninja/actions/workflows/ci.yml)
|
|
16
|
+
[](./LICENSE)
|
|
17
|
+
|
|
18
|
+
## Why WordsNinja
|
|
19
|
+
|
|
20
|
+
- **Zero dependencies** and a small API surface.
|
|
21
|
+
- **~2.8× faster than v1** — a dynamic-programming search over a frequency-ordered dictionary, backed by a `Map` and typed
|
|
22
|
+
arrays.
|
|
23
|
+
- **Keeps your text** — case is preserved and characters the dictionary cannot cover are returned instead of silently
|
|
24
|
+
dropped.
|
|
25
|
+
- **Offset-aware** — get `{ word, start, end }` tokens so you can highlight matches in the original string.
|
|
26
|
+
- **Custom dictionaries** — pass an array of words or point at a `.json` / `.txt` file.
|
|
27
|
+
- **TypeScript types and ESM + CommonJS** entry points.
|
|
28
|
+
- **Lazy dictionary load** — importing the package no longer parses the 1.8 MB word list.
|
|
29
|
+
|
|
30
|
+
## Install
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
npm install wordsninja
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
Requires Node.js 18 or newer.
|
|
37
|
+
|
|
38
|
+
## Quick start
|
|
39
|
+
|
|
40
|
+
CommonJS:
|
|
41
|
+
|
|
42
|
+
```js
|
|
43
|
+
const WordsNinja = require('wordsninja');
|
|
44
|
+
const ninja = new WordsNinja();
|
|
45
|
+
|
|
46
|
+
console.log(ninja.splitSentence('youneedtolearnfromyourmistakes'));
|
|
47
|
+
// [ 'you', 'need', 'to', 'learn', 'from', 'your', 'mistakes' ]
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
ESM:
|
|
51
|
+
|
|
52
|
+
```js
|
|
53
|
+
import WordsNinja from 'wordsninja';
|
|
54
|
+
|
|
55
|
+
const ninja = new WordsNinja();
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
TypeScript:
|
|
59
|
+
|
|
60
|
+
```ts
|
|
61
|
+
import WordsNinja from 'wordsninja';
|
|
62
|
+
|
|
63
|
+
const ninja = new WordsNinja();
|
|
64
|
+
const words: string[] = ninja.splitSentence('ilovewordsninja');
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
The dictionary is loaded on first use. `await ninja.loadDictionary()` still works and is kept for backwards
|
|
68
|
+
compatibility with v1:
|
|
69
|
+
|
|
70
|
+
```js
|
|
71
|
+
const ninja = new WordsNinja();
|
|
72
|
+
await ninja.loadDictionary(); // optional
|
|
73
|
+
ninja.splitSentence('helloworld');
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
## API
|
|
77
|
+
|
|
78
|
+
### `new WordsNinja(options?)`
|
|
79
|
+
|
|
80
|
+
| Option | Type | Description |
|
|
81
|
+
|--------------|----------------------|--------------------------------------------------------------------------------------------------------------------------------------------|
|
|
82
|
+
| `dictionary` | `string[] \| string` | Custom dictionary. An array of words, or a path to a `.json` array or newline separated `.txt` file. Defaults to the bundled English list. |
|
|
83
|
+
|
|
84
|
+
```js
|
|
85
|
+
const ninja = new WordsNinja({dictionary: ['code', 'ninja']});
|
|
86
|
+
ninja.splitSentence('codeninja'); // ['code', 'ninja']
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
The `dictionary` option is validated eagerly; the file is read (and the bundled list parsed) on first use.
|
|
90
|
+
|
|
91
|
+
### `ninja.loadDictionary(): Promise<this>`
|
|
92
|
+
|
|
93
|
+
Loads the dictionary and resolves with the instance. Optional — the dictionary is loaded lazily — but useful for warming
|
|
94
|
+
up in a serverless handler.
|
|
95
|
+
|
|
96
|
+
### `ninja.addWords(words): this`
|
|
97
|
+
|
|
98
|
+
Adds one word or an array of words to **this instance's** dictionary. Added words get the lowest possible cost, so they
|
|
99
|
+
outrank bundled words. Matching is case-insensitive and accent-insensitive.
|
|
100
|
+
|
|
101
|
+
```js
|
|
102
|
+
const ninja = new WordsNinja();
|
|
103
|
+
ninja.addWords('parsa').addWords(['kafi', 'wordsninja']);
|
|
104
|
+
|
|
105
|
+
ninja.splitSentence('parsayouneedtolearnfromyourmistakes');
|
|
106
|
+
// [ 'parsa', 'you', 'need', 'to', 'learn', 'from', 'your', 'mistakes' ]
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
Instances do not share added words.
|
|
110
|
+
|
|
111
|
+
### `ninja.splitSentence(input, options?): string[] | string | WordToken[]`
|
|
112
|
+
|
|
113
|
+
Splits `input` into words. Throws a `TypeError` if `input` is not a string.
|
|
114
|
+
|
|
115
|
+
| Option | Type | Default | Description |
|
|
116
|
+
|-------------------------|------------------|---------|-----------------------------------------------------------------------------------------------------------------------------------|
|
|
117
|
+
| `camelCaseSplitter` | `boolean` | `false` | Split camelCase / PascalCase boundaries. `XMLHttpRequest` → `XML Http Request`, `Note12` → `Note 12`. |
|
|
118
|
+
| `capitalizeFirstLetter` | `boolean` | `false` | Capitalize the first letter of every word. |
|
|
119
|
+
| `joinWords` | `boolean` | `false` | Return a space separated string instead of an array. Takes precedence over `withOffsets`. |
|
|
120
|
+
| `withOffsets` | `boolean` | `false` | Return `{ word, start, end }` objects. Offsets are UTF-16 indices into `input`. |
|
|
121
|
+
| `normalizeAccents` | `boolean` | `true` | Strip diacritics before matching (`café` matches `cafe`) while returning the original text. |
|
|
122
|
+
| `mergeNumbers` | `boolean` | `true` | Merge adjacent digit tokens: `call116116now` → `['call', '116116', 'now']`. Merging never crosses a separator, so `1 2 3` stays three tokens. |
|
|
123
|
+
| `mergeContractions` | `boolean` | `true` | Attach `'s`, `'re`, `'ll`, `'ve`, `'d`, `'m`, `'t` to the preceding word: `wang'sbook` → `["wang's", 'book']`. Merging never crosses a separator, so `wang 's` stays two tokens. |
|
|
124
|
+
| `maxWordLength` | `number` | `64` | Maximum length of a dictionary word considered at each position. Lower values are faster; longer words are treated as unknown. |
|
|
125
|
+
| `unknownWordPenalty` | `number \| null` | `10` | Cost per unmatched character. Unmatched characters are kept as tokens. Set to `null` to drop characters that cannot be segmented. In the bundled dictionary, a lone letter other than `a` or `i` counts as unmatched text. |
|
|
126
|
+
|
|
127
|
+
#### Offsets
|
|
128
|
+
|
|
129
|
+
```js
|
|
130
|
+
const input = 'Ihave2cats';
|
|
131
|
+
ninja.splitSentence(input, {withOffsets: true});
|
|
132
|
+
// [
|
|
133
|
+
// { word: 'I', start: 0, end: 1 },
|
|
134
|
+
// { word: 'have', start: 1, end: 5 },
|
|
135
|
+
// { word: '2', start: 5, end: 6 },
|
|
136
|
+
// { word: 'cats', start: 6, end: 10 }
|
|
137
|
+
// ]
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
Every offset pair slices back to its word:
|
|
141
|
+
|
|
142
|
+
```js
|
|
143
|
+
for (const token of tokens) {
|
|
144
|
+
input.slice(token.start, token.end) === token.word; // always true
|
|
145
|
+
}
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
`capitalizeFirstLetter` changes `word` but not the offsets. When `joinWords` is `true`, a string is returned and offsets
|
|
149
|
+
are ignored.
|
|
150
|
+
|
|
151
|
+
#### All options together
|
|
152
|
+
|
|
153
|
+
```js
|
|
154
|
+
ninja.splitSentence('helloWorldHowAreYou', {
|
|
155
|
+
camelCaseSplitter: true,
|
|
156
|
+
capitalizeFirstLetter: true,
|
|
157
|
+
joinWords: true,
|
|
158
|
+
});
|
|
159
|
+
// 'Hello World How Are You'
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
## Behavior details
|
|
163
|
+
|
|
164
|
+
- **Case is preserved.** Only the dictionary lookup is case-insensitive.
|
|
165
|
+
- **Separators are removed.** Spaces, punctuation, emoji and other characters outside `letters`, `digits` and `'` are
|
|
166
|
+
not part of the result. They become boundaries between words.
|
|
167
|
+
- **Accents are normalized by default.** `cafélatte` → `['café', 'latte']`. Disable with `normalizeAccents: false`; the
|
|
168
|
+
accented character is then treated as unknown text.
|
|
169
|
+
- **Text the dictionary cannot cover is kept.** Non-English scripts, brand names and typos come back as tokens instead
|
|
170
|
+
of disappearing: `سلامدنیا` → `['سلامدنیا']`. Adjacent unknown characters are grouped into one token. Use
|
|
171
|
+
`unknownWordPenalty: null` if you would rather drop them.
|
|
172
|
+
- **Numbers and possessives are merged by default.** `call123now` → `['call', '123', 'now']`, `wang'sbook` →
|
|
173
|
+
`["wang's", 'book']`. Disable with `mergeNumbers: false` / `mergeContractions: false`.
|
|
174
|
+
- **Whole words win over cheap fragments.** The bundled word list contains every letter as an entry, so a run could be
|
|
175
|
+
split into pieces cheaper than the word they spell (`sand` → `s and`). The bundled dictionary ignores one-letter
|
|
176
|
+
entries other than `a` and `i`; those letters are treated as unmatched text, so `sand` stays `sand`. Custom
|
|
177
|
+
dictionaries keep their one-letter entries.
|
|
178
|
+
- **Ambiguous input is a guess.** `thequickbrownfox` can only be segmented by cost; some strings have more than one
|
|
179
|
+
plausible split.
|
|
180
|
+
- **Long words are capped.** `maxWordLength` bounds the candidate window per position. Words longer than the cap are
|
|
181
|
+
treated as unknown.
|
|
182
|
+
|
|
183
|
+
## Custom dictionaries
|
|
184
|
+
|
|
185
|
+
```js
|
|
186
|
+
// Array
|
|
187
|
+
const code = new WordsNinja({dictionary: ['code', 'ninja']});
|
|
188
|
+
|
|
189
|
+
// JSON file: ["code", "ninja"]
|
|
190
|
+
const fromJson = new WordsNinja({dictionary: './words.json'});
|
|
191
|
+
|
|
192
|
+
// Text file: one word per line
|
|
193
|
+
const fromText = new WordsNinja({dictionary: './words.txt'});
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
Paths are resolved from the current working directory. An empty dictionary is valid: every run of text is returned as
|
|
197
|
+
unknown tokens (or dropped when `unknownWordPenalty` is `null`).
|
|
198
|
+
|
|
199
|
+
## Bundlers and browsers
|
|
200
|
+
|
|
201
|
+
The package resolves no Node built-ins at import time, so it bundles cleanly for browsers and edge runtimes:
|
|
202
|
+
|
|
203
|
+
```js
|
|
204
|
+
import WordsNinja from 'wordsninja';
|
|
205
|
+
|
|
206
|
+
const ninja = new WordsNinja();
|
|
207
|
+
ninja.splitSentence('youneedtolearnfromyourmistakes');
|
|
208
|
+
```
|
|
209
|
+
|
|
210
|
+
The bundled word list is included in the bundle (about 610 KB gzipped) and parsed on first use. File-path dictionaries
|
|
211
|
+
are Node.js only; in the browser, fetch the words and pass an array instead:
|
|
212
|
+
|
|
213
|
+
```js
|
|
214
|
+
const words = await (await fetch('/words.json')).json();
|
|
215
|
+
const ninja = new WordsNinja({dictionary: words});
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
Offsets, custom words and every option work the same way in the browser.
|
|
219
|
+
|
|
220
|
+
## Migrating from v1
|
|
221
|
+
|
|
222
|
+
v2 is a breaking release compared with v1:
|
|
223
|
+
|
|
224
|
+
- **`loadDictionary()` is optional.** The dictionary loads on first use. It still exists, now resolves with the
|
|
225
|
+
instance, and is idempotent.
|
|
226
|
+
- **Instances are isolated.** `addWords()` no longer changes a module-global dictionary shared by every instance.
|
|
227
|
+
- **The possessive/merge code actually runs.** v1 stored merged tokens at `out[-1]`, so the branch never executed.
|
|
228
|
+
Possessives and digit runs are now merged by default.
|
|
229
|
+
- **Unknown text is no longer dropped silently.** v1 returned `[]` for non-Latin input and removed accents as
|
|
230
|
+
separators; v2 keeps unmatched text as tokens.
|
|
231
|
+
- **Accents are normalized by default.** `cafélatte` now yields `['café', 'latte']` instead of `['caf', 'latte']`.
|
|
232
|
+
- **`addWords()` returns the instance** for chaining, ignores empty entries, and rejects non-strings or words
|
|
233
|
+
containing whitespace.
|
|
234
|
+
- **Clear errors instead of hangs.** Using the API before the dictionary was loaded used to loop until
|
|
235
|
+
`RangeError: Invalid array length`; it now initializes lazily and validates input.
|
|
236
|
+
- **Node.js 18+**, TypeScript types, and ESM are supported.
|
|
237
|
+
- **New options:** `withOffsets`, `normalizeAccents`, `mergeNumbers`, `mergeContractions`, `maxWordLength`,
|
|
238
|
+
`unknownWordPenalty`, plus a `dictionary` constructor option.
|
|
239
|
+
|
|
240
|
+
## Performance
|
|
241
|
+
|
|
242
|
+
Measured on a desktop machine (Node 22); split workloads are best of five, dictionary load is a cold start:
|
|
243
|
+
|
|
244
|
+
| Workload | v1 | v2 |
|
|
245
|
+
|-------------------|-------------------------|---------------------------|
|
|
246
|
+
| 4,500 characters | ~85 ms | **~30 ms** |
|
|
247
|
+
| 21,000 characters | ~388 ms | **~137 ms** |
|
|
248
|
+
| Dictionary load | ~83 ms (import + build) | **~46 ms** (first use) |
|
|
249
|
+
|
|
250
|
+
v2 also supports `withOffsets` at no measurable extra cost. Numbers vary by machine; run the benchmark on your hardware.
|
|
251
|
+
|
|
252
|
+
### Accuracy
|
|
253
|
+
|
|
254
|
+
`npm run bench:accuracy` removes the spaces from two corpora and scores the result against the intended words:
|
|
255
|
+
|
|
256
|
+
| Corpus | Sentences | Exact | Precision | Recall | F1 |
|
|
257
|
+
|------------------------------|-----------|-------------|-----------|--------|-------|
|
|
258
|
+
| `bench/corpus.txt` (tuned) | 202 | 176 (87.1%) | 96.8% | 95.6% | 96.2% |
|
|
259
|
+
| `bench/prose.txt` (held out) | 600 | 474 (79.0%) | 97.4% | 96.6% | 97.0% |
|
|
260
|
+
|
|
261
|
+
`prose.txt` holds 600 sentences from six public-domain novels; it is kept apart from the corpus used while tuning, so a
|
|
262
|
+
change cannot raise the tuned score by over-fitting. CI enforces a floor on both. Against 2.0.0, 13 more sentences are
|
|
263
|
+
exact (3 tuned and 10 prose) and none regressed.
|
|
264
|
+
|
|
265
|
+
Most remaining misses are cost ties a unigram model cannot resolve without context: a longer dictionary word wins over
|
|
266
|
+
two shorter ones (`every day` → `everyday`, `me an` → `mean`), or two two-word splits compete (`i slept` → `is lept`).
|
|
267
|
+
|
|
268
|
+
## Development
|
|
269
|
+
|
|
270
|
+
```bash
|
|
271
|
+
npm test # node:test suite
|
|
272
|
+
npm run test:watch # watch mode
|
|
273
|
+
npm run bench # segmentation speed benchmark
|
|
274
|
+
npm run bench:accuracy # accuracy against bench/corpus.txt and bench/prose.txt
|
|
275
|
+
npm run build:dictionary # validate/de-duplicate words-en.json
|
|
276
|
+
```
|
|
277
|
+
|
|
278
|
+
CI runs the test suite on Node.js 18, 20, 22 and 24, verifies the published artifact with `npm pack --dry-run`, and
|
|
279
|
+
fails if the accuracy benchmark drops below its floor.
|
|
280
|
+
|
|
281
|
+
## Releasing
|
|
282
|
+
|
|
283
|
+
1. Update the version and the changelog/README notes.
|
|
284
|
+
2. `npm test`
|
|
285
|
+
3. Tag and push: `git tag v2.0.1 && git push origin v2.0.1`
|
|
286
|
+
|
|
287
|
+
The `Release` workflow checks that the tag matches `package.json`, runs the tests, and publishes with provenance through
|
|
288
|
+
npm trusted publishing. If you publish manually, use `npm publish --provenance`.
|
|
289
|
+
|
|
290
|
+
## Limitations
|
|
291
|
+
|
|
292
|
+
WordsNinja is a dictionary-based segmenter, not a language model:
|
|
293
|
+
|
|
294
|
+
- The bundled list is English-focused. Non-English text is preserved as unknown tokens, not segmented.
|
|
295
|
+
- Segmentation is a best guess for ambiguous input.
|
|
296
|
+
- Punctuation is treated as a boundary, not preserved.
|
|
297
|
+
- The dictionary costs ~30 MB of heap once loaded; it is shared by all instances in a process.
|
|
298
|
+
|
|
299
|
+
## License
|
|
300
|
+
|
|
301
|
+
[MIT](./LICENSE) © Parsa Kafi
|
|
302
|
+
|
|
303
|
+
## Acknowledgements
|
|
304
|
+
|
|
305
|
+
- Algorithm
|
|
306
|
+
from [How to split text without spaces into list of words?](https://stackoverflow.com/questions/8870261/how-to-split-text-without-spaces-into-list-of-words).
|
|
307
|
+
- Word list from the [wordninja](https://pypi.org/project/wordninja) Python package (MIT). See [NOTICE](./NOTICE).
|
|
308
|
+
- v1 camelCase splitting was based
|
|
309
|
+
on [split-camelcase-to-words](https://www.npmjs.com/package/split-camelcase-to-words).
|
package/index.d.mts
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import WordsNinja = require('./index.js');
|
|
2
|
+
|
|
3
|
+
export default WordsNinja;
|
|
4
|
+
export {WordsNinja};
|
|
5
|
+
|
|
6
|
+
export type Options = WordsNinja.Options;
|
|
7
|
+
export type SplitOptions = WordsNinja.SplitOptions;
|
|
8
|
+
export type SplitJoinedOptions = WordsNinja.SplitJoinedOptions;
|
|
9
|
+
export type SplitWithOffsetsOptions = WordsNinja.SplitWithOffsetsOptions;
|
|
10
|
+
export type WordToken = WordsNinja.WordToken;
|
package/index.d.ts
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
declare namespace WordsNinja {
|
|
2
|
+
interface Options {
|
|
3
|
+
/**
|
|
4
|
+
* Custom dictionary: an array of words or a path to a `.json` / `.txt`
|
|
5
|
+
* file. Defaults to the bundled English dictionary.
|
|
6
|
+
*/
|
|
7
|
+
dictionary?: string[] | string;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
interface SplitOptions {
|
|
11
|
+
/** Split camelCase / PascalCase boundaries. Defaults to `false`. */
|
|
12
|
+
camelCaseSplitter?: boolean;
|
|
13
|
+
/** Capitalize the first letter of every word. Defaults to `false`. */
|
|
14
|
+
capitalizeFirstLetter?: boolean;
|
|
15
|
+
/** Return a space separated string instead of an array. Defaults to `false`. */
|
|
16
|
+
joinWords?: boolean;
|
|
17
|
+
/**
|
|
18
|
+
* Return `{ word, start, end }` objects instead of strings. Offsets are
|
|
19
|
+
* UTF-16 indices into the input string. Ignored when `joinWords` is `true`.
|
|
20
|
+
*/
|
|
21
|
+
withOffsets?: boolean;
|
|
22
|
+
/**
|
|
23
|
+
* Strip diacritics before matching (`café` matches `cafe`) while keeping
|
|
24
|
+
* the original text in the result. Defaults to `true`.
|
|
25
|
+
*/
|
|
26
|
+
normalizeAccents?: boolean;
|
|
27
|
+
/**
|
|
28
|
+
* Merge adjacent digit tokens (`call116116now` ->
|
|
29
|
+
* `['call', '116116', 'now']`). Merging never crosses a separator, so
|
|
30
|
+
* `1 2 3` stays three tokens. Defaults to `true`.
|
|
31
|
+
*/
|
|
32
|
+
mergeNumbers?: boolean;
|
|
33
|
+
/**
|
|
34
|
+
* Attach `'s`, `'re`, `'ll`, `'ve`, `'d`, `'m` and `'t` to the preceding
|
|
35
|
+
* word (`wang'sbook` -> `["wang's", 'book']`); `wang 's` stays two
|
|
36
|
+
* tokens. Defaults to `true`.
|
|
37
|
+
*/
|
|
38
|
+
mergeContractions?: boolean;
|
|
39
|
+
/**
|
|
40
|
+
* Maximum length of a dictionary word considered at each position.
|
|
41
|
+
* Lower values are faster; longer words are treated as unknown.
|
|
42
|
+
* Defaults to `64`.
|
|
43
|
+
*/
|
|
44
|
+
maxWordLength?: number;
|
|
45
|
+
/**
|
|
46
|
+
* Cost per unmatched character. Unmatched characters are kept as tokens.
|
|
47
|
+
* Set to `null` to drop characters that cannot be segmented. In the
|
|
48
|
+
* bundled dictionary, a lone letter other than `a` or `i` counts as
|
|
49
|
+
* unmatched text. Defaults to `10`.
|
|
50
|
+
*/
|
|
51
|
+
unknownWordPenalty?: number | null;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
interface WordToken {
|
|
55
|
+
/** The matched text, sliced from the original input. */
|
|
56
|
+
word: string;
|
|
57
|
+
/** Inclusive UTF-16 start offset. */
|
|
58
|
+
start: number;
|
|
59
|
+
/** Exclusive UTF-16 end offset. */
|
|
60
|
+
end: number;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
interface SplitWithOffsetsOptions extends SplitOptions {
|
|
64
|
+
withOffsets: true;
|
|
65
|
+
joinWords?: false;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
interface SplitJoinedOptions extends SplitOptions {
|
|
69
|
+
joinWords: true;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
declare class WordsNinja {
|
|
74
|
+
/**
|
|
75
|
+
* @param options Custom dictionary configuration.
|
|
76
|
+
*/
|
|
77
|
+
constructor(options?: WordsNinja.Options);
|
|
78
|
+
|
|
79
|
+
/** Whether the dictionary has been loaded. */
|
|
80
|
+
readonly ready: boolean;
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Load the dictionary. Optional: the dictionary is loaded lazily on first
|
|
84
|
+
* use. Kept for backwards compatibility with v1.
|
|
85
|
+
*/
|
|
86
|
+
loadDictionary(): Promise<this>;
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Add word(s) to this instance's dictionary. Added words get the lowest
|
|
90
|
+
* possible cost, so they win over bundled dictionary words.
|
|
91
|
+
*/
|
|
92
|
+
addWords(words: string | string[]): this;
|
|
93
|
+
|
|
94
|
+
/** Split a string into words. */
|
|
95
|
+
splitSentence(input: string, options: WordsNinja.SplitJoinedOptions): string;
|
|
96
|
+
splitSentence(input: string, options: WordsNinja.SplitWithOffsetsOptions): WordsNinja.WordToken[];
|
|
97
|
+
splitSentence(input: string, options?: WordsNinja.SplitOptions): string[];
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export = WordsNinja;
|