which-dialect 0.5.8 → 0.5.9

package/dist/check.js

@@ -82,6 +82,8 @@ const PLAIN_TYPO_MIN_ZIPF = 4;
 // Candidates kept per misspelled syllable.
 const SPELLING_CANDIDATES = 3;
 async function spelling(ctx) {
+    if (ctx.config.units === 'words')
+        return wordSpelling(ctx);
     if (ctx.config.units !== 'syllables' || !ctx.meta.syllables)
         return [];
     const syllables = await ctx.dict.syllables();
@@ -222,6 +224,60 @@ async function spelling(ctx) {
     }
     return issues;
 }
+// Wiktionary's own misspelling entries ("misspelling of también", "common misspelling of …"). A word counts
+// as misspelt when it's a misspelling and otherwise only an out-of-use spelling ("tambien": misspelling, and
+// obsolete spelling, of también).
+const MISSPELLING = /\bmisspelling of ([\p{L}\p{M}'-]+)/iu;
+const NOT_IN_USE_SPELLING = /\b(misspelling|obsolete spelling|archaic spelling|dated spelling|obsolete form|archaic form) of\b/i;
+// Zipf points below the most common look-alike beyond which another isn't suggested.
+const RARE_VARIANT_GAP = 2;
+// Spelling and accents for languages written in words (Spanish), from the dictionary alone: a word it doesn't
+// know, as a headword or an inflected form (the forms index), that matches known words once accents are
+// dropped gets the most common of them ("cancion" → "canción", "tambien" → "también"). Known words are left
+// alone even when an accented one is more
+// common ("esta" is a word, as "está" is), and so are unknown words with no such match (names, other
+// languages, typos this can't fix), and words capitalized mid-sentence (names).
+async function wordSpelling(ctx) {
+    const issues = [];
+    for (const sentence of ctx.sentences) {
+        for (const w of sentence.words) {
+            if (w.units.length !== 1)
+                continue;
+            // Words the dictionary lists only as misspellings ("tambien": "misspelling of también") count as unknown,
+            // with what they're a misspelling of.
+            const senses = w.entries.flatMap((e) => e.senses);
+            const misspelt = senses.some((s) => MISSPELLING.test(s.glosses[0] ?? '')) && senses.every((s) => NOT_IN_USE_SPELLING.test(s.glosses[0] ?? ''));
+            if (w.entries.length && !misspelt)
+                continue;
+            const u = w.units[0];
+            if (ctx.inBase(u) || !/\p{L}/u.test(u.text))
+                continue;
+            if (u !== sentence.units[0] && /\p{Lu}/u.test(u.text.charAt(0)))
+                continue;
+            const variants = await ctx.dict.variants(u.text, { limit: 3 });
+            // Rare look-alikes don't crowd out the word meant ("nino": niño, not also ñiño).
+            const best = variants[0]?.frequency;
+            const found = unique([
+                ...(misspelt ? senses.map((s) => s.altOf ?? MISSPELLING.exec(s.glosses[0] ?? '')?.[1]) : []).filter((x) => Boolean(x)),
+                ...variants.filter((v) => best === undefined || (v.frequency ?? -Infinity) >= best - RARE_VARIANT_GAP).map((v) => v.word),
+            ]).slice(0, 3);
+            if (!found.length)
+                continue;
+            // Missing accents is the common learner slip; a wrong accent is more surely wrong.
+            const accented = plain(u.text) !== u.text;
+            issues.push({
+                rule: 'spelling',
+                severity: accented ? 'error' : 'warning',
+                start: u.start,
+                end: u.end,
+                text: u.text,
+                message: `${quote(u.text)} isn't a ${ctx.meta.name} word. Did you mean ${listOf(found.map(quote), 'or')}?`,
+                suggestions: found.map((c) => matchCase(u.text, c)),
+            });
+        }
+    }
+    return issues;
+}
 // Words in the learner's own language (English by default), with the target-language word to use: "market"
 // → "chợ". Runs of base units are split into the longest phrases the base dictionary knows ("ice cream"),
 // each translated in the region and register. Words the language usually has no word for

package/dist/detect.js

@@ -12,7 +12,8 @@ import { plain } from "./text.js";
  * way, using its accented form's frequency: "rat" is "rất" (far more common than English "rat"); close
  * calls ("met": mệt or English met) go to the target only when their neighbors are the target's ("Tôi met
  * quá"), so English stays English among English ("kem but it was": but, not bút). A word only the target
- * knows without its accents ("khong") is still the target's. The base is known through the translator, so
+ * knows without its accents ("khong") is still the target's. Languages written in words do the same with
+ * whole words and their inflected forms (Spanish "cancion" → canción, "tambien" → también). The base is known through the translator, so
  * inflected forms count ("went", "cooked").
  */
 // Zipf points by which a word both languages know must be more common in one to count as that language's
@@ -29,8 +30,12 @@ export async function markLanguages(list, o) {
     // 'exact': a target word as written; 'accents': one only with accents added ("khong" → "không").
     const targetKnows = async (u) => {
         const lower = u.text.toLowerCase();
-        if (!syllableLanguage)
-            return (await o.target.lookup(lower)).length > 0 ? 'exact' : false;
+        // Languages written in words compare whole words: "cancion" is Spanish "canción" without its accent.
+        if (!syllableLanguage) {
+            if ((await o.target.lookup(lower)).length > 0)
+                return 'exact';
+            return (await o.target.variants(lower, { limit: 1 })).length > 0 ? 'accents' : false;
+        }
         if (lower in syllables || lower !== plain(lower))
             return 'exact';
         return plainSyllables.has(lower) ? 'accents' : false;
@@ -47,7 +52,9 @@ export async function markLanguages(list, o) {
         return t === 'accents' ? o.lang : '?';
     }));
     // The most common form a plain-letter unit takes once accents are added ("rat" → rất, 6.4).
-    const accentedFrequency = (u) => {
+    const accentedFrequency = async (u) => {
+        if (!syllableLanguage)
+            return (await o.target.variants(u.text.toLowerCase(), { limit: 1 }))[0]?.frequency ?? 0;
         const bare = plain(u.text.toLowerCase());
         let best = 0;
         for (const [s, f] of Object.entries(syllables))

package/dist/index.d.ts

@@ -99,8 +99,9 @@ export type Dictionary = {
     /**
      * Other words written with the same letters but other accents, that the dictionary has, most common
      * first: "muộn" → muốn, mượn, …; for words of several syllables, every syllable varies ("hom nay" →
-     * "hôm nay"). For languages written in syllables (with a syllable list); `[]` otherwise. The word
-     * itself isn't included.
+     * "hôm nay"). Languages written in words (with no syllable list) compare whole words, headwords and
+     * inflected forms alike: Spanish "cancion" → canción, "esta" → está, ésta. The word itself isn't
+     * included.
      */
     variants(word: string, options?: {
         limit?: number;

package/dist/index.js

@@ -1,5 +1,5 @@
 import { PERSONAL_NAME, PRONOUN_PERSONS, fromStored, normalizeEnglish, shardKey, } from "./types.js";
-import { accentCombinations, segment as segmentUnits, syllablesByPlain, units } from "./text.js";
+import { accentCombinations, plain, segment as segmentUnits, syllablesByPlain, units } from "./text.js";
 export * from "./types.js";
 export * from "./pos.js";
 /**
@@ -67,16 +67,40 @@ export function createDictionary(options) {
     async function shard(kind, term) {
         const m = await meta();
         const key = shardKey(term, m.shardLength);
-        return m.shards[kind].includes(key) ? loadOnce(`${kind}/${key}.json`) : null;
+        return m.shards[kind]?.includes(key) ? loadOnce(`${kind}/${key}.json`) : null;
+    }
+    // An inflected form from the forms index as an entry: one sense, "first-person singular present
+    // indicative of hablar", pointing at its lemma (altOf), used in every region like the lemma.
+    function formEntries(m, word, forms) {
+        const descriptions = m.forms?.descriptions ?? [];
+        return forms.map(([lemma, pos, description, frequency]) => ({
+            word,
+            pos,
+            senses: [
+                {
+                    glosses: [(descriptions[description] ?? 'form of {}').replace('{}', lemma)],
+                    regions: m.regions,
+                    regionTagged: false,
+                    labels: [],
+                    altOf: lemma,
+                },
+            ],
+            ...(frequency !== undefined ? { frequency } : {}),
+        }));
     }
     async function lookup(word) {
         const w = word.trim();
-        const [m, data] = await Promise.all([meta(), shard('words', w)]);
+        const [m, data, forms] = await Promise.all([meta(), shard('words', w), shard('forms', w)]);
         // The word as written, else lowercase ("Mẹ" → "mẹ"), else capitalized for names written in lowercase
         // ("nhật" → "Nhật", Japan; "nhật bản" → "Nhật Bản"), which spellchecker suggestions and learners do.
         const titled = w.toLowerCase().replace(/(^|[\s-])(\p{Ll})/gu, (_, sep, c) => sep + c.toUpperCase());
-        const stored = data?.[w] ?? data?.[w.toLowerCase()] ?? data?.[titled] ?? [];
-        return stored.map((e) => ({ ...e, senses: e.senses.map((s) => fromStored(s, m.regions)) }));
+        const key = [w, w.toLowerCase(), titled].find((k) => data?.[k] || forms?.[k]);
+        if (!key)
+            return [];
+        // Headword entries first, then the word as an inflected form of others ("música": the noun, then the
+        // feminine of "músico").
+        const entries = (data?.[key] ?? []).map((e) => ({ ...e, senses: e.senses.map((s) => fromStored(s, m.regions)) }));
+        return [...entries, ...formEntries(m, key, forms?.[key] ?? [])];
     }
     const checker = async () => {
         const m = await meta();
@@ -86,6 +110,31 @@ export function createDictionary(options) {
         const m = await meta();
         return m.syllables ? loadOnce('syllables.json') : {};
     };
+    // Variants for languages written in words: other headwords and inflected forms with the same letters
+    // once accents are dropped. Shards are keyed without accents, so they're all in the word's own shards.
+    async function wordVariants(word, limit) {
+        const written = word.trim().normalize('NFC').toLowerCase();
+        const target = plain(written);
+        if (!target)
+            return [];
+        const [data, forms] = await Promise.all([shard('words', written), shard('forms', written)]);
+        const candidates = new Set();
+        for (const key of [...Object.keys(data ?? {}), ...Object.keys(forms ?? {})]) {
+            const lower = key.toLowerCase();
+            if (lower !== written && plain(lower) === target)
+                candidates.add(lower);
+        }
+        const found = await Promise.all([...candidates].map(async (c) => ({ word: c, entries: await lookup(c) })));
+        const frequency = (entries) => {
+            const known = entries.map((e) => e.frequency).filter((f) => f !== undefined);
+            return known.length ? Math.max(...known) : undefined;
+        };
+        return found
+            .filter((v) => v.entries.length)
+            .map((v) => ({ ...v, frequency: frequency(v.entries) }))
+            .sort((a, b) => (b.frequency ?? -1) - (a.frequency ?? -1))
+            .slice(0, limit);
+    }
     return {
         meta,
         lookup,
@@ -100,7 +149,7 @@ export function createDictionary(options) {
         async variants(word, { limit = 5 } = {}) {
             const list = await syllables();
             if (!Object.keys(list).length)
-                return [];
+                return wordVariants(word, limit);
             const written = word.trim().normalize('NFC').toLowerCase();
             const parts = written.split(/\s+/);
             const combos = accentCombinations(parts, syllablesByPlain(list), {

package/dist/types.d.ts

@@ -101,10 +101,11 @@ export type LanguageMeta = {
         regionTaggedSenses: number;
         englishTerms: number;
     };
-    /** Shard file names (without .json) in words/ and en/. */
+    /** Shard file names (without .json) in words/ and en/, and forms/ when the language has a forms index. */
     shards: {
         words: string[];
         en: string[];
+        forms?: string[];
     };
     /** Letters per shard key; see shardKey. Omitted means 2. */
     shardLength?: number;
@@ -118,6 +119,16 @@ export type LanguageMeta = {
     syllables?: boolean;
     /** true when the language has sentence frames (frames.json, see FramesData). */
     frames?: boolean;
+    /**
+     * The language's inflected forms are in a compact index (forms/<shard>.json, see FormShard) rather than
+     * stored as entries: Spanish's conjugations and plurals are most of its source and would triple its
+     * data. `descriptions` are what each form is, with "{}" where the lemma goes ("first-person singular
+     * present indicative of {}"); a stored form points into it.
+     */
+    forms?: {
+        descriptions: string[];
+        count: number;
+    };
 };
 /**
  * Pronoun slots in sentence frames ({I}, {YOU}, …) and the pronoun-table column each one fills from, for
@@ -308,6 +319,14 @@ export type StoredHit = Omit<Hit, 'regions' | 'regionTagged' | 'labels'> & {
 };
 /** words/<shard>.json: entries keyed by headword. */
 export type WordShard = Record<string, StoredEntry[]>;
+/**
+ * One inflected form in forms/<shard>.json: [lemma, part of speech, index into meta.forms.descriptions,
+ * Zipf frequency (omitted when unknown)]. lookup() turns it into an entry with one sense whose gloss is
+ * the description and whose `altOf` is the lemma, so the translator follows it to the lemma's meanings.
+ */
+export type StoredForm = [lemma: string, pos: string, description: number, frequency?: number];
+/** forms/<shard>.json: a language's inflected forms keyed by form ("hablo" → hablar). */
+export type FormShard = Record<string, StoredForm[]>;
 /** en/<shard>.json: hits keyed by lowercase English term. */
 export type EnglishShard = Record<string, StoredHit[]>;
 /** Converts a full sense or hit to its stored form. */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "which-dialect",
-  "version": "0.5.8",
+  "version": "0.5.9",
   "description": "Translate words between languages and dialects, meaning by meaning, starting with Vietnamese (Northern, Central, Southern), Spanish (by country) and English, with Wiktionary data loaded on demand. More languages are planned.",
   "type": "module",
   "main": "./dist/index.js",