which-dialect 0.1.1 → 0.3.0

package/dist/text.js

@@ -0,0 +1,57 @@
+// Splitting text into units, sentences and dictionary words: shared by the checker, sentence frames and
+// the journal review. Language-general: units are runs of letters and digits; a language written in
+// syllables groups them into dictionary words by longest match (CheckerConfig.units / maxWordUnits).
+// A run of letters (with their accents), digits and apostrophes: a word, or a syllable in syllable languages.
+export const UNIT = /[\p{L}\p{M}\p{N}'’]+/gu;
+export const SENTENCE_END = /[.!?…]+|\n+/g;
+export function units(text) {
+    return [...text.matchAll(UNIT)].map((m) => ({ text: m[0], start: m.index, end: m.index + m[0].length }));
+}
+/** Units grouped into sentences, which end at . ! ? … or a line break. */
+export function sentencesOf(text, all) {
+    const ends = [...text.matchAll(SENTENCE_END)].map((m) => m.index);
+    const out = [[]];
+    let next = 0;
+    for (const u of all) {
+        while (next < ends.length && ends[next] < u.start) {
+            if (out[out.length - 1].length)
+                out.push([]);
+            next++;
+        }
+        out[out.length - 1].push(u);
+    }
+    return out.filter((s) => s.length);
+}
+/** Units join into one word only when just spaces separate them ("thịt heo", not "thịt, heo"). */
+export const joined = (text, a, b) => /^[^\S\n]+$/.test(text.slice(a.end, b.start));
+/**
+ * Longest match against the dictionary: at each position, the longest run of units that's a headword
+ * ("kết quả" is one word, so its "quả" isn't checked on its own). Unknown single units become words with
+ * no entries.
+ */
+export async function segment(text, list, maxUnits, lookup) {
+    const words = [];
+    for (let i = 0; i < list.length;) {
+        let taken = null;
+        for (let k = Math.min(maxUnits, list.length - i); k >= 1; k--) {
+            const span = list.slice(i, i + k);
+            if (span.some((u, j) => j > 0 && !joined(text, span[j - 1], u)))
+                continue;
+            const phrase = span.map((u) => u.text).join(' ');
+            const entries = /\d/.test(phrase) ? [] : await lookup(phrase);
+            if (entries.length || k === 1) {
+                const start = span[0].start;
+                const end = span[k - 1].end;
+                taken = { text: text.slice(start, end), lower: phrase.toLowerCase(), start, end, units: span, entries };
+                break;
+            }
+        }
+        words.push(taken);
+        i += taken.units.length;
+    }
+    return words;
+}
+/** Without accents: "không" → "khong", "đi" → "di". */
+export const plain = (s) => s.normalize('NFD').replace(/\p{M}/gu, '').replace(/đ/g, 'd').replace(/Đ/g, 'D').normalize('NFC');
+/** Keeps a capital first letter when replacing a capitalized word ("Tôi" → "Con"). */
+export const matchCase = (original, replacement) => original.charAt(0) !== original.charAt(0).toLowerCase() ? replacement.charAt(0).toUpperCase() + replacement.slice(1) : replacement;

package/dist/types.d.ts

@@ -110,6 +110,91 @@ export type LanguageMeta = {
     pronouns?: boolean;
     /** true when the language has hand-picked translations (picks.json, see PickRow). */
     picks?: boolean;
+    /** true when the language has checker settings (checker.json, see CheckerConfig). */
+    checker?: boolean;
+    /** true when the language has a syllable list (syllables.json: syllable → Zipf frequency, 0 when unknown). */
+    syllables?: boolean;
+    /** true when the language has sentence frames (frames.json, see FramesData). */
+    frames?: boolean;
+};
+/**
+ * Pronoun slots in sentence frames ({I}, {YOU}, …) and the pronoun-table column each one fills from, for
+ * the listener (or the table's default row).
+ */
+export declare const FRAME_PRONOUNS: Record<string, {
+    person: PronounPerson;
+    inclusive?: boolean;
+}>;
+/**
+ * One choice for a frame word slot ({WHERE} → "đâu", "mô"): from a dictionary definition (region and
+ * labels from that sense) or a hand-written override with a note. `when: 'respect'` choices are used only
+ * when the listener's pronoun-table row is marked respect (Vietnamese "ạ").
+ */
+export type FrameWordChoice = {
+    word: string;
+    gloss?: string;
+    source: 'gloss' | 'override';
+    regions: string[];
+    regionTagged: boolean;
+    labels: string[];
+    note?: string;
+    when?: 'respect';
+};
+/** A frame word slot: what it means, whether a sentence is fine without it, and its choices in order. */
+export type FrameWords = {
+    meaning: string;
+    optional?: boolean;
+    choices: FrameWordChoice[];
+};
+/**
+ * A sentence frame in one language: the catalog's id, topic, English ways to say it and slots, and this
+ * language's `text` with {slot} placeholders: content slots (lowercase, from `slots`), pronoun slots
+ * (FRAME_PRONOUNS) and frame word slots (uppercase, from FramesData.words).
+ */
+export type FrameData = {
+    id: string;
+    topic: string;
+    en: string[];
+    slots: Record<string, 'noun' | 'verb' | 'adj'>;
+    clause?: boolean;
+    text: string;
+};
+export type FramesData = {
+    frames: FrameData[];
+    words: Record<string, FrameWords>;
+};
+export type StoredFrameWordChoice = Omit<FrameWordChoice, 'regions' | 'regionTagged' | 'labels'> & {
+    regions?: string[];
+    labels?: string[];
+};
+export type StoredFramesData = {
+    frames: FrameData[];
+    words: Record<string, Omit<FrameWords, 'choices'> & {
+        choices: StoredFrameWordChoice[];
+    }>;
+};
+/**
+ * How the checker reads one language (languages/<lang>.ts `checker`, stored as data/checker.json). Every
+ * field is optional: a language without it is read as space-separated words, and gets the checks that
+ * only need its dictionary (dialect mixing).
+ */
+export type CheckerConfig = {
+    /**
+     * What spaces separate: 'words' (English), or 'syllables' that group into dictionary words (Vietnamese
+     * "thịt heo", "kết quả"). With 'syllables' the build also writes syllables.json for accent suggestions.
+     */
+    units?: 'words' | 'syllables';
+    /** The longest dictionary word to look for, in units (default 3). */
+    maxWordUnits?: number;
+    /**
+     * Pronouns that are reliably pronouns, by pronoun-table column ("tôi", "tao", "mày"). The pronoun checks
+     * look only at these: kinship words like Vietnamese "con" (also "child", a classifier) aren't listed.
+     */
+    pronouns?: Partial<Record<PronounPerson, string[]>>;
+    /** Listed pronouns that are also common nouns ("bạn": you, or friend); checked, but only as suggestions. */
+    ambiguousPronouns?: string[];
+    /** Sentence endings expected when speaking up (pronoun-table rows marked `respect`), e.g. Vietnamese "ạ". */
+    politeEndings?: string[];
 };
 /**
  * Hand-picked first choices for one English meaning, in one target language (picks.json). A speaker's
@@ -181,6 +266,8 @@ export type PronounRow = {
     thirdPlural: PronounChoice[];
     /** The row to use when the relationship isn't known (neutral words: Vietnamese "tôi", "bạn"). */
     default?: boolean;
+    /** Speaking up to this person (parents, elders, teachers): polite sentence endings are expected. */
+    respect?: boolean;
     /** A caution for the whole row, e.g. that "tao"/"mày" are rude outside close friendships. */
     warning?: string;
 };

package/dist/types.js

@@ -1,5 +1,18 @@
 // Shapes of the data. The build script (scripts/build-language.ts) writes the Stored* types to the
 // data files; the API in index.ts reads them and returns the full Sense/Entry/Hit types.
+/**
+ * Pronoun slots in sentence frames ({I}, {YOU}, …) and the pronoun-table column each one fills from, for
+ * the listener (or the table's default row).
+ */
+export const FRAME_PRONOUNS = {
+    I: { person: 'self' },
+    YOU: { person: 'addressee' },
+    WE: { person: 'selfPlural', inclusive: false },
+    WE_INCL: { person: 'selfPlural', inclusive: true },
+    YOU_PL: { person: 'addresseePlural' },
+    HE_SHE: { person: 'third' },
+    THEY: { person: 'thirdPlural' },
+};
 /** The pronoun table's columns: I, you, he/she, we, you (plural), they. */
 export const PRONOUN_PERSONS = ['self', 'addressee', 'third', 'selfPlural', 'addresseePlural', 'thirdPlural'];
 /** Converts a full sense or hit to its stored form. */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "which-dialect",
-  "version": "0.1.1",
+  "version": "0.3.0",
   "description": "Translate words between languages and dialects, meaning by meaning, starting with Vietnamese (Northern, Central, Southern) and English, with Wiktionary data loaded on demand. More languages are planned.",
   "type": "module",
   "main": "./dist/index.js",