tinyfix 0.0.0-stage → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,74 @@
1
+ /** Compact trie that finds dictionary words within a small edit distance of a query. */
2
+ export class WordTrie {
3
+ character = [0];
4
+ firstChild = [-1];
5
+ nextSibling = [-1];
6
+ wordIndex = [-1];
7
+ constructor(words) {
8
+ words.forEach((word, index) => this.insert(word, index));
9
+ }
10
+ /** Builds the same trie a few thousand words at a time, calling `pause` in between so a page stays responsive. */
11
+ static async inSteps(words, pause, step = 8000) {
12
+ const trie = new WordTrie([]);
13
+ for (let start = 0; start < words.length; start += step) {
14
+ const end = Math.min(words.length, start + step);
15
+ for (let index = start; index < end; index++)
16
+ trie.insert(words[index], index);
17
+ await pause();
18
+ }
19
+ return trie;
20
+ }
21
+ insert(word, index) {
22
+ let node = 0;
23
+ for (let position = 0; position < word.length; position++) {
24
+ node = this.findOrCreateChild(node, word.charCodeAt(position));
25
+ }
26
+ this.wordIndex[node] = index;
27
+ }
28
+ findOrCreateChild(parent, code) {
29
+ let child = this.firstChild[parent];
30
+ while (child !== -1) {
31
+ if (this.character[child] === code)
32
+ return child;
33
+ child = this.nextSibling[child];
34
+ }
35
+ const created = this.character.length;
36
+ this.character.push(code);
37
+ this.firstChild.push(-1);
38
+ this.nextSibling.push(this.firstChild[parent]);
39
+ this.wordIndex.push(-1);
40
+ this.firstChild[parent] = created;
41
+ return created;
42
+ }
43
+ /** Returns every word within `maxEdits` insertions, deletions, substitutions or adjacent swaps. */
44
+ search(query, maxEdits) {
45
+ const width = query.length + 1;
46
+ const matches = [];
47
+ const typed = Array.from({ length: query.length }, (_, position) => query.charCodeAt(position));
48
+ const walk = (node, previousCode, row, rowBefore) => {
49
+ for (let child = this.firstChild[node]; child !== -1; child = this.nextSibling[child]) {
50
+ const code = this.character[child];
51
+ const next = new Uint8Array(width);
52
+ next[0] = row[0] + 1;
53
+ let smallest = next[0];
54
+ for (let column = 1; column < width; column++) {
55
+ const substitution = row[column - 1] + (typed[column - 1] === code ? 0 : 1);
56
+ let value = Math.min(row[column] + 1, next[column - 1] + 1, substitution);
57
+ const isSwap = rowBefore && column > 1 && typed[column - 1] === previousCode && typed[column - 2] === code;
58
+ if (isSwap)
59
+ value = Math.min(value, rowBefore[column - 2] + 1);
60
+ next[column] = value;
61
+ smallest = Math.min(smallest, value);
62
+ }
63
+ const distance = next[width - 1];
64
+ if (this.wordIndex[child] !== -1 && distance <= maxEdits) {
65
+ matches.push({ index: this.wordIndex[child], distance });
66
+ }
67
+ if (smallest <= maxEdits)
68
+ walk(child, code, next, row);
69
+ }
70
+ };
71
+ walk(0, -1, Uint8Array.from({ length: width }, (_, column) => column), null);
72
+ return matches;
73
+ }
74
+ }
@@ -0,0 +1,2 @@
1
+ /** Weighted edit distance: a lower cost means the typo is a more believable slip of the fingers. */
2
+ export declare function typoCost(typed: string, intended: string): number;
@@ -0,0 +1,81 @@
1
+ const KEYBOARD_ROWS = ["qwertyuiop", "asdfghjkl", "zxcvbnm"];
2
+ const VOWELS = new Set("aeiou");
3
+ const COST = {
4
+ swap: 1.2,
5
+ doubledKey: 1.2,
6
+ adjacentKey: 1.6,
7
+ vowelForVowel: 2,
8
+ extraNearbyKey: 1.8,
9
+ missedKey: 2.4,
10
+ extraKey: 3,
11
+ wrongKey: 3.4,
12
+ firstLetterPenalty: 1.5,
13
+ };
14
+ function buildNeighbors() {
15
+ const neighbors = new Map();
16
+ KEYBOARD_ROWS.forEach((row, rowIndex) => {
17
+ [...row].forEach((key, keyIndex) => {
18
+ const near = new Set();
19
+ for (const [otherRow, shift] of [
20
+ [rowIndex, -1],
21
+ [rowIndex, 1],
22
+ [rowIndex - 1, 0],
23
+ [rowIndex - 1, 1],
24
+ [rowIndex + 1, -1],
25
+ [rowIndex + 1, 0],
26
+ ]) {
27
+ const other = KEYBOARD_ROWS[otherRow]?.[keyIndex + shift];
28
+ if (other)
29
+ near.add(other);
30
+ }
31
+ neighbors.set(key, near);
32
+ });
33
+ });
34
+ return neighbors;
35
+ }
36
+ const NEIGHBORS = buildNeighbors();
37
+ const isNearby = (first, second) => !!first && !!second && !!NEIGHBORS.get(first)?.has(second);
38
+ function substitutionCost(typed, intended) {
39
+ if (isNearby(typed, intended))
40
+ return COST.adjacentKey;
41
+ if (VOWELS.has(typed) && VOWELS.has(intended))
42
+ return COST.vowelForVowel;
43
+ return COST.wrongKey;
44
+ }
45
+ function extraKeyCost(typed, position) {
46
+ const key = typed[position];
47
+ if (key === typed[position - 1] || key === typed[position + 1])
48
+ return COST.doubledKey;
49
+ if (isNearby(key, typed[position - 1]) || isNearby(key, typed[position + 1]))
50
+ return COST.extraNearbyKey;
51
+ return COST.extraKey;
52
+ }
53
+ function missedKeyCost(intended, position) {
54
+ return intended[position] === intended[position - 1] ? COST.doubledKey : COST.missedKey;
55
+ }
56
+ /** A slip on a word's first letter is less likely, so it costs more. */
57
+ const start = (position) => (position === 0 ? COST.firstLetterPenalty : 0);
58
+ /** Weighted edit distance: a lower cost means the typo is a more believable slip of the fingers. */
59
+ export function typoCost(typed, intended) {
60
+ const rows = typed.length + 1;
61
+ const columns = intended.length + 1;
62
+ const table = Array.from({ length: rows }, () => new Array(columns).fill(0));
63
+ for (let row = 1; row < rows; row++)
64
+ table[row][0] = table[row - 1][0] + extraKeyCost(typed, row - 1) + start(row - 1);
65
+ for (let column = 1; column < columns; column++)
66
+ table[0][column] = table[0][column - 1] + missedKeyCost(intended, column - 1) + start(column - 1);
67
+ for (let row = 1; row < rows; row++) {
68
+ for (let column = 1; column < columns; column++) {
69
+ const typedKey = typed[row - 1];
70
+ const intendedKey = intended[column - 1];
71
+ const firstLetter = start(Math.min(row, column) - 1);
72
+ const replace = typedKey === intendedKey ? 0 : substitutionCost(typedKey, intendedKey) + firstLetter;
73
+ let best = Math.min(table[row - 1][column] + extraKeyCost(typed, row - 1) + firstLetter, table[row][column - 1] + missedKeyCost(intended, column - 1) + firstLetter, table[row - 1][column - 1] + replace);
74
+ const swapped = row > 1 && column > 1 && typedKey === intended[column - 2] && typed[row - 2] === intendedKey;
75
+ if (swapped)
76
+ best = Math.min(best, table[row - 2][column - 2] + COST.swap);
77
+ table[row][column] = best;
78
+ }
79
+ }
80
+ return table[rows - 1][columns - 1];
81
+ }
@@ -0,0 +1,63 @@
1
+ import type { Model } from "./model.js";
2
+ import type { ContextScorer, Correction } from "./fixer.js";
3
+ export interface WrongWordOptions {
4
+ model: Model;
5
+ /** Lines of the model's vocab.txt, most common word first. */
6
+ vocabulary: readonly string[];
7
+ /** Lines of the model's confusions.txt: a typed word, then words people meant by it. */
8
+ confusions?: readonly string[];
9
+ settings?: WrongWordSettings;
10
+ }
11
+ export interface WrongWordSettings {
12
+ /** A nearby word must beat the typed word by at least this much. */
13
+ threshold: number;
14
+ /** How strongly an unlikely slip of the fingers counts against a nearby word. */
15
+ costWeight: number;
16
+ }
17
+ /** Tuned on a dev split to wrongly change at most about 1 correct word in 2,000. */
18
+ export declare const CAREFUL_WRONG_WORD: WrongWordSettings;
19
+ /**
20
+ * On the lab bench's test sets this fixes about 43% of wrong words and wrongly changes about 1 correct word in 1,750,
21
+ * about the same as the model before it. A threshold of 2.75 fixes about 1 point more and changes about 1 in 1,550.
22
+ */
23
+ export declare const EAGER_WRONG_WORD: WrongWordSettings;
24
+ export interface WrongWordFinder {
25
+ check(textBefore: string, word: string, textAfter: string): Promise<Correction | null>;
26
+ }
27
+ export interface NearbyWord {
28
+ word: string;
29
+ /** How much better this word fits than the typed word, as a log-odds difference. */
30
+ margin: number;
31
+ /** How unlikely it is to type the given word when meaning this one. */
32
+ cost: number;
33
+ /** True when the word is a known mix-up for the typed word. */
34
+ isKnownMixup: boolean;
35
+ }
36
+ /** True for pairs like "services" and "service" or "continued" and "continues". */
37
+ export declare function differsOnlyInEnding(first: string, second: string): boolean;
38
+ /** Compares a correctly spelled word with nearby real words, using the words on both sides of it. */
39
+ export declare class WrongWordChecker implements WrongWordFinder, ContextScorer {
40
+ private readonly options;
41
+ settings: WrongWordSettings;
42
+ private readonly ids;
43
+ private readonly targets;
44
+ private readonly trie;
45
+ private readonly neighbors;
46
+ private readonly knownMixups;
47
+ constructor(options: WrongWordOptions);
48
+ private tokenId;
49
+ private nearbyIds;
50
+ /** Returns nearby words with how well each fits, or null when the word isn't one the model can judge. */
51
+ compare(textBefore: string, word: string, textAfter: string, onlyKnownMixups?: boolean): Promise<NearbyWord[] | null>;
52
+ /** Scores spelling candidates by how well they fit the text before them. The first word is the misspelling itself. */
53
+ score(textBefore: string, words: readonly string[]): Promise<number[]>;
54
+ /** Runs the model once per row of candidates, returning scores as one flat array of CANDIDATES per row. */
55
+ private runModel;
56
+ /** Short words and words with an apostrophe are left to the list of known mix-ups ("to" and "too", "it's" and "its"). */
57
+ private hasKnownMixups;
58
+ /** Whether a word is judged at all, and if so whether only against its known mix-ups. Null means leave it alone. */
59
+ judgedAgainst(word: string, textBefore: string): "nearby" | "knownMixups" | null;
60
+ private value;
61
+ /** Returns a fix when a nearby word fits the sentence clearly better than the one typed. */
62
+ check(textBefore: string, word: string, textAfter: string): Promise<Correction | null>;
63
+ }
@@ -0,0 +1,210 @@
1
+ import { isCheckable, isSentenceStart, matchCase } from "./rules.js";
2
+ import { WordTrie } from "./trie.js";
3
+ import { typoCost } from "./typoCost.js";
4
+ const LEFT_WINDOW = 16;
5
+ const RIGHT_WINDOW = 4;
6
+ const MAX_NEIGHBORS = 8;
7
+ const CANDIDATES = MAX_NEIGHBORS + 1;
8
+ const PAD = 0;
9
+ const UNKNOWN = 1;
10
+ const NUMBER = 2;
11
+ const START = 3;
12
+ const NAME = 4;
13
+ const SPECIAL_TOKENS = 5;
14
+ /** Known mix-ups like "their" and "there" are likely even when they're far apart on the keyboard. */
15
+ const MAX_KNOWN_MIXUP_COST = 1.5;
16
+ /** A known mix-up needs to clear a lower bar than other nearby words, which are mostly slips of the fingers. */
17
+ const KNOWN_MIXUP_BONUS = 1;
18
+ /** For a word with known mix-ups, any other nearby word needs to clear a higher bar: "your" is rarely meant as "you". */
19
+ const OTHER_WORD_PENALTY = 1;
20
+ /** A word that only differs by an ending ("examples" and "example") needs to clear a higher bar, unless it's a known mix-up. */
21
+ const INFLECTION_PENALTY = 1;
22
+ const INFLECTION_ENDINGS = new Set(["", "s", "es", "d", "ed", "ing", "r", "er", "ly"]);
23
+ const MIN_STEM = 3;
24
+ const TOKEN = /[A-Za-z]+(?:'[A-Za-z]+)?|\d+(?:[.,]\d+)*|\n|[^\s\w]/g;
25
+ const SENTENCE_OPENERS = new Set([".", "!", "?", ":", '"']);
26
+ const WORD = /^[a-z]+(?:'[a-z]+)?$/;
27
+ const LISTED_WORD = /^[A-Za-z]+(?:'[A-Za-z]+)?$/;
28
+ /** Tuned on a dev split to wrongly change at most about 1 correct word in 2,000. */
29
+ export const CAREFUL_WRONG_WORD = { threshold: 5, costWeight: 0.5 };
30
+ /**
31
+ * On the lab bench's test sets this fixes about 43% of wrong words and wrongly changes about 1 correct word in 1,750,
32
+ * about the same as the model before it. A threshold of 2.75 fixes about 1 point more and changes about 1 in 1,550.
33
+ */
34
+ export const EAGER_WRONG_WORD = { threshold: 3, costWeight: 1.5 };
35
+ const pad = (ids, length) => [
36
+ ...new Array(length - ids.length).fill(PAD),
37
+ ...ids,
38
+ ];
39
+ /** True for pairs like "services" and "service" or "continued" and "continues". */
40
+ export function differsOnlyInEnding(first, second) {
41
+ let shared = 0;
42
+ while (shared < Math.min(first.length, second.length) && first[shared] === second[shared])
43
+ shared++;
44
+ for (let stem = shared; stem >= MIN_STEM; stem--) {
45
+ if (INFLECTION_ENDINGS.has(first.slice(stem)) && INFLECTION_ENDINGS.has(second.slice(stem)))
46
+ return true;
47
+ }
48
+ return false;
49
+ }
50
+ /**
51
+ * Lowercases words, except capitalized words mid-sentence, which become a single name token. A line break ends a
52
+ * sentence, so the end of the paragraph before reads like the sentence before ("Notes" then "Their is..." isn't one run).
53
+ */
54
+ function tokenize(text, startsSentence) {
55
+ const tokens = [];
56
+ for (const raw of text.replaceAll("’", "'").match(TOKEN) ?? []) {
57
+ if (raw === "\n") {
58
+ if (tokens.length > 0 && !SENTENCE_OPENERS.has(tokens.at(-1)))
59
+ tokens.push(".");
60
+ continue;
61
+ }
62
+ const previous = tokens.at(-1);
63
+ const atSentenceStart = previous === undefined ? startsSentence : SENTENCE_OPENERS.has(previous);
64
+ const isPronoun = raw === "I" || raw.startsWith("I'");
65
+ tokens.push(/^[A-Z]/.test(raw) && !atSentenceStart && !isPronoun ? "<name>" : raw.toLowerCase());
66
+ }
67
+ return tokens;
68
+ }
69
+ /** Compares a correctly spelled word with nearby real words, using the words on both sides of it. */
70
+ export class WrongWordChecker {
71
+ options;
72
+ settings;
73
+ ids = new Map();
74
+ targets = [];
75
+ trie;
76
+ neighbors = new Map();
77
+ knownMixups = new Map();
78
+ constructor(options) {
79
+ this.options = options;
80
+ this.settings = options.settings ?? CAREFUL_WRONG_WORD;
81
+ options.vocabulary.forEach((token, index) => {
82
+ this.ids.set(token, index);
83
+ if (index >= SPECIAL_TOKENS && token.length >= 2 && WORD.test(token))
84
+ this.targets.push(index);
85
+ });
86
+ this.trie = new WordTrie(this.targets.map((index) => options.vocabulary[index]));
87
+ for (const line of options.confusions ?? []) {
88
+ const [typed, ...meant] = line.split(" ").map((token) => this.ids.get(token));
89
+ if (typed !== undefined)
90
+ this.knownMixups.set(typed, meant.filter((id) => id !== undefined));
91
+ }
92
+ }
93
+ tokenId(token) {
94
+ if (token === "<name>")
95
+ return NAME;
96
+ if (/^\d/.test(token))
97
+ return NUMBER;
98
+ return this.ids.get(token) ?? UNKNOWN;
99
+ }
100
+ nearbyIds(id) {
101
+ const cached = this.neighbors.get(id);
102
+ if (cached)
103
+ return cached;
104
+ const oneEdit = this.trie
105
+ .search(this.options.vocabulary[id], 1)
106
+ .map((match) => this.targets[match.index])
107
+ .filter((other) => other !== id)
108
+ .sort((first, second) => first - second);
109
+ const found = [...new Set([...(this.knownMixups.get(id) ?? []), ...oneEdit])].slice(0, MAX_NEIGHBORS);
110
+ this.neighbors.set(id, found);
111
+ return found;
112
+ }
113
+ /** Returns nearby words with how well each fits, or null when the word isn't one the model can judge. */
114
+ async compare(textBefore, word, textAfter, onlyKnownMixups = false) {
115
+ const typed = this.ids.get(word.toLowerCase());
116
+ if (typed === undefined || typed < SPECIAL_TOKENS)
117
+ return null;
118
+ const nearby = onlyKnownMixups ? (this.knownMixups.get(typed) ?? []) : this.nearbyIds(typed);
119
+ if (nearby.length === 0)
120
+ return null;
121
+ const scores = await this.runModel(textBefore, textAfter, [[typed, ...nearby]]);
122
+ const mixups = this.knownMixups.get(typed) ?? [];
123
+ return nearby.map((id, index) => {
124
+ const nearbyWord = this.options.vocabulary[id];
125
+ const cost = typoCost(word.toLowerCase(), nearbyWord);
126
+ const isKnownMixup = mixups.includes(id);
127
+ return {
128
+ word: nearbyWord,
129
+ margin: scores[index + 1] - scores[0],
130
+ cost: isKnownMixup ? Math.min(cost, MAX_KNOWN_MIXUP_COST) : cost,
131
+ isKnownMixup,
132
+ };
133
+ });
134
+ }
135
+ /** Scores spelling candidates by how well they fit the text before them. The first word is the misspelling itself. */
136
+ async score(textBefore, words) {
137
+ const ids = [UNKNOWN, ...words.slice(1).map((word) => this.ids.get(word.toLowerCase()) ?? UNKNOWN)];
138
+ const rows = [];
139
+ for (let start = 0; start < ids.length; start += MAX_NEIGHBORS)
140
+ rows.push([PAD, ...ids.slice(start, start + MAX_NEIGHBORS)]);
141
+ const scores = await this.runModel(textBefore, "", rows);
142
+ return ids.map((_, index) => scores[Math.floor(index / MAX_NEIGHBORS) * CANDIDATES + (index % MAX_NEIGHBORS) + 1]);
143
+ }
144
+ /** Runs the model once per row of candidates, returning scores as one flat array of CANDIDATES per row. */
145
+ async runModel(textBefore, textAfter, rows) {
146
+ const before = tokenize(textBefore, true).map((token) => this.tokenId(token));
147
+ const left = before.length >= LEFT_WINDOW ? before.slice(-LEFT_WINDOW) : pad([START, ...before], LEFT_WINDOW);
148
+ const right = pad(tokenize(textAfter, false)
149
+ .slice(0, RIGHT_WINDOW)
150
+ .map((token) => this.tokenId(token))
151
+ .reverse(), RIGHT_WINDOW);
152
+ const tiled = (values) => rows.flatMap(() => values);
153
+ const tensor = (values, width) => ({
154
+ data: BigInt64Array.from(values, BigInt),
155
+ dims: [rows.length, width],
156
+ });
157
+ return this.options.model.run({
158
+ left: tensor(tiled(left), LEFT_WINDOW),
159
+ right: tensor(tiled(right), RIGHT_WINDOW),
160
+ candidates: tensor(rows.flatMap((row) => [...row, ...pad([], CANDIDATES - row.length)]), CANDIDATES),
161
+ });
162
+ }
163
+ /** Short words and words with an apostrophe are left to the list of known mix-ups ("to" and "too", "it's" and "its"). */
164
+ hasKnownMixups(word, textBefore) {
165
+ const id = this.ids.get(word.toLowerCase());
166
+ if (id === undefined || !this.knownMixups.has(id) || !LISTED_WORD.test(word))
167
+ return false;
168
+ if (word.length > 1 && word === word.toUpperCase())
169
+ return false;
170
+ const isCapitalized = word[0] !== word[0].toLowerCase();
171
+ return !isCapitalized || isSentenceStart(textBefore);
172
+ }
173
+ /** Whether a word is judged at all, and if so whether only against its known mix-ups. Null means leave it alone. */
174
+ judgedAgainst(word, textBefore) {
175
+ if (isCheckable(word, textBefore))
176
+ return "nearby";
177
+ return this.hasKnownMixups(word, textBefore) ? "knownMixups" : null;
178
+ }
179
+ value(item, typed, typedHasMixups) {
180
+ const value = item.margin - this.settings.costWeight * item.cost;
181
+ if (item.isKnownMixup)
182
+ return value + KNOWN_MIXUP_BONUS;
183
+ let penalty = typedHasMixups ? OTHER_WORD_PENALTY : 0;
184
+ if (differsOnlyInEnding(typed, item.word))
185
+ penalty += INFLECTION_PENALTY;
186
+ return value - penalty;
187
+ }
188
+ /** Returns a fix when a nearby word fits the sentence clearly better than the one typed. */
189
+ async check(textBefore, word, textAfter) {
190
+ const judged = this.judgedAgainst(word, textBefore);
191
+ if (!judged)
192
+ return null;
193
+ const nearby = await this.compare(textBefore, word, textAfter, judged === "knownMixups");
194
+ if (!nearby)
195
+ return null;
196
+ const typedHasMixups = nearby.some((item) => item.isKnownMixup);
197
+ const ranked = nearby
198
+ .map((item) => ({ ...item, value: this.value(item, word.toLowerCase(), typedHasMixups) }))
199
+ .sort((first, second) => second.value - first.value);
200
+ const best = ranked[0];
201
+ if (best.value < this.settings.threshold)
202
+ return null;
203
+ return {
204
+ original: word,
205
+ replacement: matchCase(word, best.word),
206
+ confidence: 1 / (1 + Math.exp(-best.margin)),
207
+ usedContext: true,
208
+ };
209
+ }
210
+ }
@@ -0,0 +1,51 @@
1
+ /** One change tinyfix would make to a text. */
2
+ export interface Fix {
3
+ /** Where the word starts in the text, as a string index. */
4
+ from: number;
5
+ /** Where the word ends. */
6
+ to: number;
7
+ /** What was typed. */
8
+ original: string;
9
+ /** What it should be. */
10
+ replacement: string;
11
+ /** A misspelling, or a correctly spelled word that doesn't fit the words around it. */
12
+ kind: "spelling" | "context";
13
+ /** How sure tinyfix is, from 0 to 1. */
14
+ confidence: number;
15
+ }
16
+ export interface LoadOptions {
17
+ /** Where the model and word lists are. Defaults to the tinyfix repo on Hugging Face, at a fixed revision. */
18
+ baseUrl?: string | URL;
19
+ /** Pass false to skip the model: nothing but the word list is downloaded, and only misspellings are fixed. */
20
+ model?: boolean;
21
+ /**
22
+ * How sure tinyfix must be before it changes a correctly spelled word. "balanced" is what offlinepage.app uses and
23
+ * what the published numbers describe. "careful" makes fewer wrong changes and fixes fewer wrong words.
24
+ */
25
+ mode?: "balanced" | "careful";
26
+ /** Where ONNX Runtime's .wasm files are, for apps that host them. Most bundlers find them without this. */
27
+ wasmPaths?: string;
28
+ }
29
+ export interface CheckOptions {
30
+ /** Only words inside this range of the text are checked. The rest of the text is still read as context. */
31
+ range?: readonly [from: number, to: number];
32
+ /** Only these kinds of fix are looked for. Both, by default. */
33
+ kinds?: ReadonlyArray<Fix["kind"]>;
34
+ }
35
+ export interface Tinyfix {
36
+ /** Finds the fixes for a text. They come back in order and never overlap. */
37
+ check(text: string, options?: CheckOptions): Promise<Fix[]>;
38
+ /** Returns the text with the fixes made. Throws if a fix doesn't match the text it's applied to. */
39
+ apply(text: string, fixes: readonly Fix[]): string;
40
+ /** Checks a text and makes every fix. */
41
+ fix(text: string, options?: CheckOptions): Promise<string>;
42
+ /** Never change this word again. Use it when someone undoes a fix, or for words of their own. */
43
+ ignore(word: string): void;
44
+ /** Frees the model. The checker can't be used afterwards. */
45
+ dispose(): Promise<void>;
46
+ }
47
+ /**
48
+ * Loads tinyfix: about 3.5 MB of model and word lists, from Hugging Face unless `baseUrl` says otherwise. Call it
49
+ * once and keep the result.
50
+ */
51
+ export declare function loadTinyfix(options?: LoadOptions): Promise<Tinyfix>;
package/dist/index.js ADDED
@@ -0,0 +1,171 @@
1
+ import { Fixer, WITH_MODEL_WEIGHTS, parseWordList } from "./engine/fixer.js";
2
+ import { createModel } from "./engine/model.js";
3
+ import { loadModelInWorker } from "./engine/modelInWorker.js";
4
+ import { WordTrie } from "./engine/trie.js";
5
+ import { CAREFUL_WRONG_WORD, EAGER_WRONG_WORD, WrongWordChecker } from "./engine/wrongWord.js";
6
+ /** The revision of huggingface.co/paperweight/tinyfix this version of the package was tested with. */
7
+ const MODEL_REVISION = "8efdf5cd91b32584f4077aa0e8ecaf48f7651bef";
8
+ const DEFAULT_BASE_URL = `https://huggingface.co/paperweight/tinyfix/resolve/${MODEL_REVISION}/`;
9
+ /**
10
+ * Sure enough to change a misspelled word, with the model helping to choose. offlinepage.app uses the same bar: it
11
+ * fixes more misspellings than the stricter default, and is what lets a doubled-key slip like "sporrt" become "sport".
12
+ */
13
+ const SPELLING_MIN_CONFIDENCE = 0.55;
14
+ /** How much text before a word the engine is given. It reads 16 words back, so this is plenty. */
15
+ const CONTEXT_BEFORE = 600;
16
+ const CONTEXT_AFTER = 200;
17
+ const WORD = /[A-Za-z']+/g;
18
+ /** A fenced code block. An unclosed one runs to the end of the text. */
19
+ const FENCED_CODE = /```[\s\S]*?(?:```|$)/g;
20
+ /** Inline code. An unclosed one runs to the end of its line. */
21
+ const INLINE_CODE = /`[^`\n]*(?:`|$)/gm;
22
+ /** Parts of a web address, an email, a path, a tag or an identifier: not prose. */
23
+ const NOT_PROSE = /[@/\\_#\d]|\.\w/;
24
+ const lines = (text) => text.split("\n").filter(Boolean);
25
+ /** Lets a page draw and handle input between pieces of setup, so loading never freezes it. */
26
+ const nextTask = () => new Promise((resolve) => setTimeout(resolve));
27
+ async function download(base, file) {
28
+ const response = await fetch(new URL(file, base));
29
+ if (!response.ok)
30
+ throw new Error(`tinyfix couldn't load ${file} from ${base} (${response.status}).`);
31
+ return response;
32
+ }
33
+ const downloadText = (base, file) => download(base, file).then((response) => response.text());
34
+ /** The start and end of each stretch of code in a text. */
35
+ function codeRanges(text) {
36
+ return [FENCED_CODE, INLINE_CODE].flatMap((pattern) => [...text.matchAll(pattern)].map((match) => [match.index, match.index + match[0].length]));
37
+ }
38
+ /** The run of non-space characters around a word: "john@form.com" for the "form" inside it. */
39
+ function tokenAround(text, from, to) {
40
+ let start = from;
41
+ while (start > 0 && !/\s/.test(text[start - 1]))
42
+ start--;
43
+ let end = to;
44
+ while (end < text.length && !/\s/.test(text[end]))
45
+ end++;
46
+ return text.slice(start, end);
47
+ }
48
+ function apply(text, fixes) {
49
+ let result = text;
50
+ // Last fix first, so earlier positions stay valid as the text changes length.
51
+ for (const fix of [...fixes].sort((first, second) => second.from - first.from)) {
52
+ if (text.slice(fix.from, fix.to) !== fix.original) {
53
+ throw new Error(`This fix expects “${fix.original}” at ${fix.from}, but the text has “${text.slice(fix.from, fix.to)}”.`);
54
+ }
55
+ result = result.slice(0, fix.from) + fix.replacement + result.slice(fix.to);
56
+ }
57
+ return result;
58
+ }
59
+ /**
60
+ * On a page, the model runs in tinyfix's own worker, so starting it and checking words never block typing. Elsewhere,
61
+ * or if the worker can't start, it runs right here.
62
+ */
63
+ async function startModel(bytes, wasmPaths) {
64
+ const runHere = async (modelBytes) => {
65
+ // The WebAssembly-only build of ONNX Runtime. The default build also carries WebGPU, which tinyfix doesn't use.
66
+ const ort = await import("onnxruntime-web/wasm");
67
+ return createModel(ort, modelBytes, wasmPaths);
68
+ };
69
+ if (typeof document === "undefined" || typeof Worker === "undefined")
70
+ return runHere(bytes);
71
+ // The worker is handed a copy, so the bytes are still here if it can't start.
72
+ return loadModelInWorker(bytes.slice(0), wasmPaths).catch(() => runHere(bytes));
73
+ }
74
+ /**
75
+ * Loads tinyfix: about 3.5 MB of model and word lists, from Hugging Face unless `baseUrl` says otherwise. Call it
76
+ * once and keep the result.
77
+ */
78
+ export async function loadTinyfix(options = {}) {
79
+ const base = String(options.baseUrl ?? DEFAULT_BASE_URL).replace(/\/?$/, "/");
80
+ const absoluteBase = typeof location === "undefined" ? base : new URL(base, location.href).href;
81
+ const useModel = options.model !== false;
82
+ const [words, modelFiles] = await Promise.all([
83
+ downloadText(absoluteBase, "words.txt"),
84
+ useModel
85
+ ? Promise.all([
86
+ download(absoluteBase, "wrong-word.onnx").then((response) => response.arrayBuffer()),
87
+ downloadText(absoluteBase, "wrong-word-vocab.txt"),
88
+ downloadText(absoluteBase, "wrong-word-confusions.txt"),
89
+ ])
90
+ : null,
91
+ ]);
92
+ let model = null;
93
+ let wrongWord = null;
94
+ if (modelFiles) {
95
+ const [bytes, vocabulary, confusions] = modelFiles;
96
+ model = await startModel(bytes, options.wasmPaths);
97
+ await nextTask();
98
+ wrongWord = new WrongWordChecker({
99
+ model,
100
+ vocabulary: lines(vocabulary),
101
+ confusions: lines(confusions),
102
+ settings: options.mode === "careful" ? CAREFUL_WRONG_WORD : EAGER_WRONG_WORD,
103
+ });
104
+ }
105
+ await nextTask();
106
+ const wordList = parseWordList(words);
107
+ await nextTask();
108
+ const trie = await WordTrie.inSteps(wordList.map((entry) => entry.word), nextTask);
109
+ const fixer = wrongWord
110
+ ? new Fixer({
111
+ words: wordList,
112
+ trie,
113
+ scorer: wrongWord,
114
+ weights: { ...WITH_MODEL_WEIGHTS, minConfidence: SPELLING_MIN_CONFIDENCE },
115
+ })
116
+ : new Fixer({ words: wordList, trie });
117
+ async function check(text, checkOptions = {}) {
118
+ const [rangeFrom, rangeTo] = checkOptions.range ?? [0, text.length];
119
+ const kinds = checkOptions.kinds ?? ["spelling", "context"];
120
+ const wantsSpelling = kinds.includes("spelling");
121
+ const wantsContext = kinds.includes("context") && wrongWord !== null;
122
+ const code = codeRanges(text);
123
+ const fixes = [];
124
+ // The text up to the word being checked, with the fixes found so far already made. A later word is judged
125
+ // against corrected text, as it would be while typing.
126
+ let fixedSoFar = "";
127
+ let consumed = 0;
128
+ for (const match of text.matchAll(WORD)) {
129
+ const original = match[0];
130
+ const from = match.index;
131
+ const to = from + original.length;
132
+ if (to <= rangeFrom)
133
+ continue;
134
+ if (from >= rangeTo)
135
+ break;
136
+ if (code.some(([start, end]) => from < end && to > start))
137
+ continue;
138
+ if (fixer.wasUndone(original) || NOT_PROSE.test(tokenAround(text, from, to)))
139
+ continue;
140
+ const before = (fixedSoFar + text.slice(consumed, from)).slice(-CONTEXT_BEFORE);
141
+ let correction = wantsSpelling ? await fixer.correct(before, original) : null;
142
+ if (!correction && wantsContext && wrongWord) {
143
+ const lineEnd = text.indexOf("\n", to);
144
+ const after = text.slice(to, Math.min(lineEnd === -1 ? text.length : lineEnd, to + CONTEXT_AFTER));
145
+ correction = await wrongWord.check(before, original, after);
146
+ }
147
+ if (!correction || correction.replacement === original)
148
+ continue;
149
+ fixes.push({
150
+ from,
151
+ to,
152
+ original,
153
+ replacement: correction.replacement,
154
+ kind: correction.usedContext ? "context" : "spelling",
155
+ confidence: Math.max(0, Math.min(1, correction.confidence)),
156
+ });
157
+ fixedSoFar = (fixedSoFar + text.slice(consumed, from)).slice(-CONTEXT_BEFORE) + correction.replacement;
158
+ consumed = to;
159
+ }
160
+ return fixes;
161
+ }
162
+ return {
163
+ check,
164
+ apply,
165
+ fix: async (text, checkOptions) => apply(text, await check(text, checkOptions)),
166
+ ignore: (word) => fixer.rememberUndo(word),
167
+ dispose: async () => {
168
+ await model?.release();
169
+ },
170
+ };
171
+ }