which-dialect 0.2.0 → 0.4.0

package/dist/review.js

@@ -0,0 +1,175 @@
+// Reviewing a journal entry written in the target language, the learner's own language, or a mix of both
+// ("Hôm nay tôi đi market với má"), and giving the corrected entry in the target language. For each sentence:
+//
+//   1. Each word is marked as the target or the base language (detect.ts), from the two dictionaries alone.
+//   2. A whole base-language sentence or clause that follows a sentence frame is filled in ("but it was
+//      expensive" → "nhưng mắc quá"; "Where is the bathroom?" → "Phòng tắm ở đâu?").
+//   3. The checker runs on the whole entry. By default that's the spellchecker: every misspelled word gets
+//      its accents ("khong" → "không") and every remaining base-language word its translation ("market" →
+//      "chợ"); opt-in checks (`checks`: regions, pronouns, consistency, polite endings) join it. Errors and
+//      warnings are applied; suggestions are returned as hints.
+//   4. The sentence frames the corrected sentence follows are listed, for the learner to reuse. Frames don't
+//      change the sentence.
+//
+// Everything is rule-based and runs on the language's data: no API, and nothing language-specific here.
+import { createChecker } from "./check.js";
+import { markLanguages } from "./detect.js";
+import { createDictionary } from "./index.js";
+import { createPhrasebook } from "./phrasebook.js";
+import { sentencesOf, units } from "./text.js";
+import { createTranslator } from "./translate.js";
+const TERMINAL = /^[.!?…]+/;
+// Spaces after a word was left out: one between words, none before punctuation.
+const tidy = (s) => s.replace(/[^\S\n]+/g, ' ').replace(/ ([.,!?…])/g, '$1').trim();
+export function createReviewer(options = {}) {
+    const cache = new Map();
+    const dict = (lang) => {
+        let d = cache.get(lang);
+        if (!d) {
+            d = options.dictionary?.(lang) ?? createDictionary({ lang, baseUrl: options.baseUrl?.(lang), load: options.load?.(lang) });
+            cache.set(lang, d);
+        }
+        return d;
+    };
+    const translator = createTranslator({ ...options, dictionary: dict });
+    const phrasebook = createPhrasebook({ ...options, dictionary: dict, translator });
+    const checker = createChecker({ ...options, dictionary: dict });
+    return {
+        async review(entry, o) {
+            const text = entry.normalize('NFC');
+            const base = o.base ?? 'en';
+            // A frame as shown to the learner: its English and its pattern for these settings, slots open.
+            const frameInfo = new Map();
+            const frameOf = async (id) => {
+                if (!frameInfo.has(id)) {
+                    const f = await phrasebook.render(id, o);
+                    frameInfo.set(id, { id, en: f.en, text: f.text });
+                }
+                return frameInfo.get(id);
+            };
+            const drafts = [];
+            for (const list of sentencesOf(text, units(text))) {
+                const start = list[0].start;
+                const coreEnd = list[list.length - 1].end;
+                const end = coreEnd + (TERMINAL.exec(text.slice(coreEnd))?.[0].length ?? 0);
+                const langs = base === o.lang ? list.map(() => o.lang) : await markLanguages(list, { lang: o.lang, base, target: dict(o.lang), baseDictionary: dict(base), translator });
+                const groups = [];
+                list.forEach((u, i) => {
+                    if (groups.length && groups[groups.length - 1].lang === langs[i])
+                        groups[groups.length - 1].units.push(u);
+                    else
+                        groups.push({ lang: langs[i], units: [u] });
+                });
+                const draft = {
+                    start, coreEnd, end, replacements: [], frames: [],
+                    parts: groups.map((g) => ({ text: text.slice(g.units[0].start, g.units[g.units.length - 1].end), lang: g.lang })),
+                };
+                const whole = groups.length === 1 && groups[0].lang === base;
+                const fill = async (span, frame) => {
+                    const from = text.slice(span[0].start, span[span.length - 1].end);
+                    const info = await frameOf(frame.id);
+                    let to = frame.text.replace(/[.!?…]+$/, '');
+                    if (span[0].start !== start)
+                        to = to.charAt(0).toLowerCase() + to.slice(1);
+                    draft.replacements.push({
+                        start: span[0].start, end: span[span.length - 1].end, to,
+                        change: { kind: 'frame', from, to, why: `Sentence frame “${info.en}”` },
+                    });
+                    draft.frames.push(info);
+                };
+                for (const g of groups.filter((g) => g.lang === base)) {
+                    // A whole sentence is matched with its own punctuation ("Where is the bathroom?").
+                    const whole_ = whole ? text.slice(start, end) : text.slice(g.units[0].start, g.units[g.units.length - 1].end);
+                    const frame = await phrasebook.match(whole_, o);
+                    if (frame?.complete) {
+                        await fill(g.units, frame);
+                        continue;
+                    }
+                    // Otherwise a clause frame ending the part, longest first ("the ice cream but it was expensive":
+                    // "but it was expensive"); the rest is translated word by word by the checker. Only spans ending
+                    // with the part, so a clause isn't cut short ("but it was very" with "expensive" left over).
+                    for (let len = g.units.length - 1; len >= 2; len--) {
+                        const span = g.units.slice(g.units.length - len);
+                        const m = await phrasebook.match(text.slice(span[0].start, span[len - 1].end), o);
+                        if (!m?.complete || !m.clause)
+                            continue;
+                        await fill(span, m);
+                        break;
+                    }
+                }
+                drafts.push(draft);
+            }
+            // The entry with those parts filled in, keeping where each sentence now is.
+            let draftText = '';
+            let cursor = 0;
+            const placed = [];
+            for (const d of drafts) {
+                draftText += text.slice(cursor, d.start);
+                const s = draftText.length;
+                let core = text.slice(d.start, d.coreEnd);
+                for (const r of [...d.replacements].sort((a, b) => b.start - a.start)) {
+                    core = core.slice(0, r.start - d.start) + r.to + core.slice(r.end - d.start);
+                }
+                if (/\p{Lu}/u.test(text.charAt(d.start)))
+                    core = core.charAt(0).toUpperCase() + core.slice(1);
+                draftText += core + text.slice(d.coreEnd, d.end);
+                placed.push({ start: s, coreEnd: s + core.length, end: draftText.length });
+                cursor = d.end;
+            }
+            draftText += text.slice(cursor);
+            // 3. The checker on the whole entry; 4. the frames each corrected sentence follows.
+            const { issues } = await checker.check(draftText, {
+                lang: o.lang, base, region: o.region, listener: o.listener, speaker: o.speaker, register: o.register, rules: o.checks,
+            });
+            const sentences = [];
+            for (const [i, d] of drafts.entries()) {
+                const p = placed[i];
+                const mine = issues.filter((x) => x.start >= p.start && x.end <= p.coreEnd);
+                const changes = d.replacements.sort((a, b) => a.start - b.start).map((r) => r.change);
+                const hints = [];
+                const unchecked = [];
+                const applied = [];
+                let core = draftText.slice(p.start, p.coreEnd);
+                let lastStart = Infinity;
+                for (const x of [...mine].sort((a, b) => b.start - a.start)) {
+                    if (!x.suggestions.length) {
+                        if (x.rule === 'foreign-word')
+                            unchecked.unshift(x.text);
+                        else
+                            hints.unshift({ rule: x.rule, text: x.text, message: x.message, suggestions: x.suggestions });
+                        continue;
+                    }
+                    if (x.severity === 'suggestion' || x.end > lastStart) {
+                        hints.unshift({ rule: x.rule, text: x.text, message: x.message, suggestions: x.suggestions });
+                        continue;
+                    }
+                    core = core.slice(0, x.start - p.start) + x.suggestions[0] + core.slice(x.end - p.start);
+                    applied.unshift({ kind: x.rule, from: x.text, to: x.suggestions[0], why: x.message });
+                    lastStart = x.start;
+                }
+                changes.push(...applied);
+                core = tidy(core);
+                if (/\p{Lu}/u.test(draftText.charAt(p.start)))
+                    core = core.charAt(0).toUpperCase() + core.slice(1);
+                const corrected = core + draftText.slice(p.coreEnd, p.end);
+                const frames = [...d.frames];
+                for (const m of await phrasebook.matchFrames(corrected, o)) {
+                    if (!frames.some((f) => f.id === m.frame.id))
+                        frames.push(await frameOf(m.frame.id));
+                }
+                sentences.push({
+                    start: d.start, end: d.end, original: text.slice(d.start, d.end), parts: d.parts,
+                    corrected, changes, hints, frames, unchecked,
+                });
+            }
+            let corrected = '';
+            cursor = 0;
+            for (const s of sentences) {
+                corrected += text.slice(cursor, s.start) + s.corrected;
+                cursor = s.end;
+            }
+            corrected += text.slice(cursor);
+            return { text, corrected, sentences };
+        },
+    };
+}

package/dist/text.d.ts

@@ -0,0 +1,31 @@
+import type { Entry } from './types.ts';
+export declare const UNIT: RegExp;
+export declare const SENTENCE_END: RegExp;
+export type Unit = {
+    text: string;
+    start: number;
+    end: number;
+};
+export type Word = {
+    text: string;
+    lower: string;
+    start: number;
+    end: number;
+    units: Unit[];
+    entries: Entry[];
+};
+export declare function units(text: string): Unit[];
+/** Units grouped into sentences, which end at . ! ? … or a line break. */
+export declare function sentencesOf(text: string, all: Unit[]): Unit[][];
+/** Units join into one word only when just spaces separate them ("thịt heo", not "thịt, heo"). */
+export declare const joined: (text: string, a: Unit, b: Unit) => boolean;
+/**
+ * Longest match against the dictionary: at each position, the longest run of units that's a headword
+ * ("kết quả" is one word, so its "quả" isn't checked on its own). Unknown single units become words with
+ * no entries.
+ */
+export declare function segment(text: string, list: Unit[], maxUnits: number, lookup: (phrase: string) => Promise<Entry[]>): Promise<Word[]>;
+/** Without accents: "không" → "khong", "đi" → "di". */
+export declare const plain: (s: string) => string;
+/** Keeps a capital first letter when replacing a capitalized word ("Tôi" → "Con"). */
+export declare const matchCase: (original: string, replacement: string) => string;

package/dist/text.js

@@ -0,0 +1,57 @@
+// Splitting text into units, sentences and dictionary words: shared by the checker, sentence frames and
+// the journal review. Language-general: units are runs of letters and digits; a language written in
+// syllables groups them into dictionary words by longest match (CheckerConfig.units / maxWordUnits).
+// A run of letters (with their accents), digits and apostrophes: a word, or a syllable in syllable languages.
+export const UNIT = /[\p{L}\p{M}\p{N}'’]+/gu;
+export const SENTENCE_END = /[.!?…]+|\n+/g;
+export function units(text) {
+    return [...text.matchAll(UNIT)].map((m) => ({ text: m[0], start: m.index, end: m.index + m[0].length }));
+}
+/** Units grouped into sentences, which end at . ! ? … or a line break. */
+export function sentencesOf(text, all) {
+    const ends = [...text.matchAll(SENTENCE_END)].map((m) => m.index);
+    const out = [[]];
+    let next = 0;
+    for (const u of all) {
+        while (next < ends.length && ends[next] < u.start) {
+            if (out[out.length - 1].length)
+                out.push([]);
+            next++;
+        }
+        out[out.length - 1].push(u);
+    }
+    return out.filter((s) => s.length);
+}
+/** Units join into one word only when just spaces separate them ("thịt heo", not "thịt, heo"). */
+export const joined = (text, a, b) => /^[^\S\n]+$/.test(text.slice(a.end, b.start));
+/**
+ * Longest match against the dictionary: at each position, the longest run of units that's a headword
+ * ("kết quả" is one word, so its "quả" isn't checked on its own). Unknown single units become words with
+ * no entries.
+ */
+export async function segment(text, list, maxUnits, lookup) {
+    const words = [];
+    for (let i = 0; i < list.length;) {
+        let taken = null;
+        for (let k = Math.min(maxUnits, list.length - i); k >= 1; k--) {
+            const span = list.slice(i, i + k);
+            if (span.some((u, j) => j > 0 && !joined(text, span[j - 1], u)))
+                continue;
+            const phrase = span.map((u) => u.text).join(' ');
+            const entries = /\d/.test(phrase) ? [] : await lookup(phrase);
+            if (entries.length || k === 1) {
+                const start = span[0].start;
+                const end = span[k - 1].end;
+                taken = { text: text.slice(start, end), lower: phrase.toLowerCase(), start, end, units: span, entries };
+                break;
+            }
+        }
+        words.push(taken);
+        i += taken.units.length;
+    }
+    return words;
+}
+/** Without accents: "không" → "khong", "đi" → "di". */
+export const plain = (s) => s.normalize('NFD').replace(/\p{M}/gu, '').replace(/đ/g, 'd').replace(/Đ/g, 'D').normalize('NFC');
+/** Keeps a capital first letter when replacing a capitalized word ("Tôi" → "Con"). */
+export const matchCase = (original, replacement) => original.charAt(0) !== original.charAt(0).toLowerCase() ? replacement.charAt(0).toUpperCase() + replacement.slice(1) : replacement;

package/dist/types.d.ts

@@ -114,6 +114,64 @@ export type LanguageMeta = {
     checker?: boolean;
     /** true when the language has a syllable list (syllables.json: syllable → Zipf frequency, 0 when unknown). */
     syllables?: boolean;
+    /** true when the language has sentence frames (frames.json, see FramesData). */
+    frames?: boolean;
+};
+/**
+ * Pronoun slots in sentence frames ({I}, {YOU}, …) and the pronoun-table column each one fills from, for
+ * the listener (or the table's default row).
+ */
+export declare const FRAME_PRONOUNS: Record<string, {
+    person: PronounPerson;
+    inclusive?: boolean;
+}>;
+/**
+ * One choice for a frame word slot ({WHERE} → "đâu", "mô"): from a dictionary definition (region and
+ * labels from that sense) or a hand-written override with a note. `when: 'respect'` choices are used only
+ * when the listener's pronoun-table row is marked respect (Vietnamese "ạ").
+ */
+export type FrameWordChoice = {
+    word: string;
+    gloss?: string;
+    source: 'gloss' | 'override';
+    regions: string[];
+    regionTagged: boolean;
+    labels: string[];
+    note?: string;
+    when?: 'respect';
+};
+/** A frame word slot: what it means, whether a sentence is fine without it, and its choices in order. */
+export type FrameWords = {
+    meaning: string;
+    optional?: boolean;
+    choices: FrameWordChoice[];
+};
+/**
+ * A sentence frame in one language: the catalog's id, topic, English ways to say it and slots, and this
+ * language's `text` with {slot} placeholders: content slots (lowercase, from `slots`), pronoun slots
+ * (FRAME_PRONOUNS) and frame word slots (uppercase, from FramesData.words).
+ */
+export type FrameData = {
+    id: string;
+    topic: string;
+    en: string[];
+    slots: Record<string, 'noun' | 'verb' | 'adj'>;
+    clause?: boolean;
+    text: string;
+};
+export type FramesData = {
+    frames: FrameData[];
+    words: Record<string, FrameWords>;
+};
+export type StoredFrameWordChoice = Omit<FrameWordChoice, 'regions' | 'regionTagged' | 'labels'> & {
+    regions?: string[];
+    labels?: string[];
+};
+export type StoredFramesData = {
+    frames: FrameData[];
+    words: Record<string, Omit<FrameWords, 'choices'> & {
+        choices: StoredFrameWordChoice[];
+    }>;
 };
 /**
  * How the checker reads one language (languages/<lang>.ts `checker`, stored as data/checker.json). Every
@@ -137,6 +195,11 @@ export type CheckerConfig = {
     ambiguousPronouns?: string[];
     /** Sentence endings expected when speaking up (pronoun-table rows marked `respect`), e.g. Vietnamese "ạ". */
     politeEndings?: string[];
+    /**
+     * English words (English is the bridge) this language usually has no word for, so the checker suggests
+     * leaving them out rather than translating them: Vietnamese has no articles ("the", "a", "an").
+     */
+    leaveOut?: string[];
 };
 /**
  * Hand-picked first choices for one English meaning, in one target language (picks.json). A speaker's

package/dist/types.js

@@ -1,5 +1,18 @@
 // Shapes of the data. The build script (scripts/build-language.ts) writes the Stored* types to the
 // data files; the API in index.ts reads them and returns the full Sense/Entry/Hit types.
+/**
+ * Pronoun slots in sentence frames ({I}, {YOU}, …) and the pronoun-table column each one fills from, for
+ * the listener (or the table's default row).
+ */
+export const FRAME_PRONOUNS = {
+    I: { person: 'self' },
+    YOU: { person: 'addressee' },
+    WE: { person: 'selfPlural', inclusive: false },
+    WE_INCL: { person: 'selfPlural', inclusive: true },
+    YOU_PL: { person: 'addresseePlural' },
+    HE_SHE: { person: 'third' },
+    THEY: { person: 'thirdPlural' },
+};
 /** The pronoun table's columns: I, you, he/she, we, you (plural), they. */
 export const PRONOUN_PERSONS = ['self', 'addressee', 'third', 'selfPlural', 'addresseePlural', 'thirdPlural'];
 /** Converts a full sense or hit to its stored form. */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "which-dialect",
-  "version": "0.2.0",
+  "version": "0.4.0",
   "description": "Translate words between languages and dialects, meaning by meaning, starting with Vietnamese (Northern, Central, Southern) and English, with Wiktionary data loaded on demand. More languages are planned.",
   "type": "module",
   "main": "./dist/index.js",