which-dialect 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +3 -1
- package/dist/index.js +4 -0
- package/dist/translate.d.ts +8 -2
- package/dist/translate.js +159 -28
- package/dist/types.d.ts +19 -0
- package/dist/types.js +6 -3
- package/package.json +1 -1
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type PronounRow, type Entry, type Hit, type LanguageMeta } from './types.ts';
|
|
1
|
+
import { type PronounRow, type Entry, type Hit, type LanguageMeta, type PickRow } from './types.ts';
|
|
2
2
|
export * from './types.ts';
|
|
3
3
|
export * from './pos.ts';
|
|
4
4
|
/**
|
|
@@ -65,6 +65,8 @@ export type Dictionary = {
|
|
|
65
65
|
* for languages without a pronoun table.
|
|
66
66
|
*/
|
|
67
67
|
pronouns(options?: PronounOptions): Promise<PronounRow[]>;
|
|
68
|
+
/** Hand-picked first choices for English meanings (see PickRow). Empty for languages without any. */
|
|
69
|
+
picks(): Promise<PickRow[]>;
|
|
68
70
|
};
|
|
69
71
|
/**
|
|
70
72
|
* The regions a region or region-group name stands for (e.g. English "North America" -> US, Canada),
|
package/dist/index.js
CHANGED
|
@@ -103,6 +103,10 @@ export function createDictionary(options) {
|
|
|
103
103
|
const kept = new Set(words.map((h) => h.word));
|
|
104
104
|
return ranked.filter((h) => kept.has(h.word));
|
|
105
105
|
},
|
|
106
|
+
async picks() {
|
|
107
|
+
const m = await meta();
|
|
108
|
+
return m.picks ? loadOnce('picks.json') : [];
|
|
109
|
+
},
|
|
106
110
|
async pronouns({ listener, region, speaker, exclude = DEFAULT_EXCLUDED_LABELS } = {}) {
|
|
107
111
|
const m = await meta();
|
|
108
112
|
if (!m.pronouns)
|
package/dist/translate.d.ts
CHANGED
|
@@ -27,8 +27,9 @@ export type Translation = Hit & {
|
|
|
27
27
|
score: number;
|
|
28
28
|
/**
|
|
29
29
|
* How this word was found: the English term that linked it, "synonym" for a same-language synonym,
|
|
30
|
-
* "table" for Wiktionary's translation table,
|
|
31
|
-
*
|
|
30
|
+
* "table" for Wiktionary's translation table, "picked" for the target language's hand-picked words
|
|
31
|
+
* for this meaning (see PickRow), or "pronouns" for a pronoun-table row's words (the `listener`'s or
|
|
32
|
+
* `about`'s row, or the default row).
|
|
32
33
|
*/
|
|
33
34
|
bridge: string;
|
|
34
35
|
/** Example sentences for this word in the matched sense, when the dictionary has them. */
|
|
@@ -124,6 +125,11 @@ export type TranslateOptions = {
|
|
|
124
125
|
* translations repeat another sense's. Default false. Useful for letting users pick a meaning.
|
|
125
126
|
*/
|
|
126
127
|
allSenses?: boolean;
|
|
128
|
+
/**
|
|
129
|
+
* Use the target language's hand-picked words (default true). false returns the ranking alone, e.g. to
|
|
130
|
+
* see what picks change.
|
|
131
|
+
*/
|
|
132
|
+
picks?: boolean;
|
|
127
133
|
};
|
|
128
134
|
export type Translator = {
|
|
129
135
|
/** The senses of a word, following forms and variant spellings to the words they belong to. */
|
package/dist/translate.js
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
// for those terms with a compatible part of speech, in the target region. Each source sense is translated
|
|
6
6
|
// on its own, so a word's meanings don't get mixed up ("cool" the temperature vs. "cool" the compliment).
|
|
7
7
|
import { compatiblePos } from "./pos.js";
|
|
8
|
-
import { displayGloss, glossTerms, normalizeEnglish, PRONOUN_PERSONS, } from "./types.js";
|
|
8
|
+
import { displayGloss, glossTerms, normalizeEnglish, PRONOUN_PERSONS, TRAILING_PARTICLE, } from "./types.js";
|
|
9
9
|
import { createDictionary, DEFAULT_EXCLUDED_LABELS, resolveRegion, } from "./index.js";
|
|
10
10
|
/**
|
|
11
11
|
* Possible base words for a regularly inflected English word, most likely first: "walked" -> walk,
|
|
@@ -118,9 +118,61 @@ const MIN_SCORE = 1;
|
|
|
118
118
|
// A variant that's only another spelling of a different word ("yeah": pronunciation spelling of "year")
|
|
119
119
|
// ranks below the word's own meanings.
|
|
120
120
|
const SPELLING_VIA = /pronunciation spelling|misspelling|eye dialect|nonstandard spelling|dated spelling|informal spelling/i;
|
|
121
|
+
// Among a form's own meanings, one that needs a following construction ("Must; have/has (to).", "used
|
|
122
|
+
// with have", "followed by …") is narrower than a standalone one ("got": "Have/has." comes first). Only
|
|
123
|
+
// there: function words' main meanings are constructions ("would": "Past tense of will; usually
|
|
124
|
+
// followed by a bare infinitive"), and demoting them everywhere put "would" → "hoàn cảnh" first.
|
|
125
|
+
const CONSTRUCTION = /\((?:to|with [^)]*|followed by [^)]*)\)|\bused with\b|\bfollowed by\b/i;
|
|
121
126
|
// Senses whose labels are all in this set rank after mainstream senses.
|
|
122
127
|
const MARGINAL_LABELS = new Set(['informal', 'slang', 'colloquial', 'dialectal', 'rare', 'nonstandard', 'Internet', 'humorous']);
|
|
128
|
+
// How a form reached its base word, when it's an inflection ("simple past of get", "form of run").
|
|
129
|
+
const INFLECTION_VIA = /\b(?:past|participle|plural|singular|present|third-person|comparative|superlative|gerund)\b|^form of /i;
|
|
130
|
+
// Everyday speech, not slang: a form's own colloquial meaning ("got": "Have/has.") isn't marginal.
|
|
131
|
+
const EVERYDAY_LABELS = new Set(['informal', 'colloquial']);
|
|
123
132
|
const LABEL_PENALTY = new Set(['slang', 'Internet', 'humorous', 'literary', 'dialectal', 'uncommon', 'rare', 'euphemistic']);
|
|
133
|
+
// A target gloss that lists the bridge term itself as an equivalent ("muốn": "to want"; "má": "mother;
|
|
134
|
+
// mom") is a plain translation, but a short gloss can't earn definition overlap, since the term's own
|
|
135
|
+
// words don't count. Wordier glosses did ("mắc": "to want to (go to the bathroom, laugh, etc.); to have
|
|
136
|
+
// the desire to" overlapped "desire"), so a narrow use beat the everyday word. An exact equivalent
|
|
137
|
+
// scores like one overlapping word, for main terms only: a side term's exact match ("corn" → "cereal" →
|
|
138
|
+
// "ngũ cốc") is a looser equivalent.
|
|
139
|
+
const EXACT_GLOSS_BONUS = 3;
|
|
140
|
+
// The term narrowed by a parenthetical right after it ("to want to (go to the bathroom …)") is a
|
|
141
|
+
// restricted use of the word, not its general meaning (see narrows()).
|
|
142
|
+
const NARROWED_PENALTY = 1.5;
|
|
143
|
+
// A parenthetical narrows a term when it names something the source meaning doesn't: "to want to (go to
|
|
144
|
+
// the bathroom, laugh, etc.)", "in/at (an outdoor or external location)". Restatements ("ngầu": "cool
|
|
145
|
+
// (awesome; fantastic; great)" for cool = awesome), grammar ("to wait (for)") and usage notes, which say
|
|
146
|
+
// how the word is used rather than what it means ("dạ": "yes (answers a question)"), don't narrow.
|
|
147
|
+
const USAGE_NOTE = /^\s*(?:used|said|\w+s\s+(?:a|an|the|that|for|to|with|about)\b)/i;
|
|
148
|
+
function narrows(parenthetical, context) {
|
|
149
|
+
return contentWords(parenthetical).size > 0 && !USAGE_NOTE.test(parenthetical) && overlap(context, parenthetical) === 0;
|
|
150
|
+
}
|
|
151
|
+
// A target sense marked passive ("được": "to obtain, to get (passively)") is a different voice from an
|
|
152
|
+
// active source meaning ("get": "To obtain; to acquire", where "lấy" is the everyday word). Sized to
|
|
153
|
+
// offset a very common word's frequency: "được" is common mostly as a helper verb ("được đi", "làm
|
|
154
|
+
// được"), and frequency is per word, not per meaning.
|
|
155
|
+
const PASSIVE = /\bpassive(?:ly)?\b/i;
|
|
156
|
+
const VOICE_MISMATCH_PENALTY = 3;
|
|
157
|
+
function exactGloss(gloss, term, context) {
|
|
158
|
+
// "to wait for" is plain "wait", but a part with a narrowing parenthetical isn't ("to want to (…)").
|
|
159
|
+
// Like glossTerms, only the text after a colon is the meaning ("Used as an answer …: no"). A question
|
|
160
|
+
// ("dạ": "yes? (asks for more information)") isn't the plain word.
|
|
161
|
+
let cleaned = gloss.replace(/\(([^)]*)\)/g, (_, inner) => (narrows(inner, context) ? '(' : ' '));
|
|
162
|
+
if (cleaned.includes(':'))
|
|
163
|
+
cleaned = cleaned.slice(cleaned.lastIndexOf(':') + 1);
|
|
164
|
+
return cleaned.split(/[;,/]/).some((part) => {
|
|
165
|
+
if (part.includes('('))
|
|
166
|
+
return false;
|
|
167
|
+
const t = normalizeEnglish(part.replace(/[.!]+\s*$/, ''));
|
|
168
|
+
return t === term || t.replace(TRAILING_PARTICLE, '') === term;
|
|
169
|
+
});
|
|
170
|
+
}
|
|
171
|
+
function narrowed(gloss, term, context) {
|
|
172
|
+
const escaped = term.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
173
|
+
const m = new RegExp(`\\b${escaped}\\b(?:\\s+to)?\\s*\\(([^)]*)`, 'i').exec(gloss);
|
|
174
|
+
return m !== null && narrows(m[1], context);
|
|
175
|
+
}
|
|
124
176
|
// Register: a polite source word should translate to a polite word, a casual one to a casual one.
|
|
125
177
|
const POLITE = new Set(['polite', 'formal', 'honorific', 'literary']);
|
|
126
178
|
const CASUAL = new Set(['informal', 'colloquial', 'slang', 'familiar', 'vulgar', 'Internet']);
|
|
@@ -147,16 +199,25 @@ function registerFit(source, target, wanted) {
|
|
|
147
199
|
// what the word means. Without the jargon, Vietnamese "con" ("used when talking to someone older than
|
|
148
200
|
// the speaker") matched English "I" ("the speaker or writer") on "speaker".
|
|
149
201
|
const STOPWORDS = new Set(('a an the to of or and in on for with by as at from that this be is are was it its used use any some one ' +
|
|
150
|
-
'something someone very more most usually especially often also who which what when being having into ' +
|
|
202
|
+
'something someone very more most usually especially often also who which what when being have has having into ' +
|
|
151
203
|
'speaker speakers writer referred refer refers referring grammatical subject object sentence clause ' +
|
|
152
204
|
'word words form forms term expression phrase talking speaking addressed addressing person people ' +
|
|
153
205
|
'particular certain various generally sense usage meaning').split(' '));
|
|
154
|
-
function contentWords(text) {
|
|
206
|
+
function contentWords(text, keep) {
|
|
155
207
|
return new Set(text
|
|
156
208
|
.toLowerCase()
|
|
157
209
|
.replace(/[^a-z' ]+/g, ' ')
|
|
158
210
|
.split(' ')
|
|
159
|
-
.filter((w) => w.length > 2 && !STOPWORDS.has(w)));
|
|
211
|
+
.filter((w) => w.length > 2 && (!STOPWORDS.has(w) || keep?.has(w))));
|
|
212
|
+
}
|
|
213
|
+
// The caller's `meaning` words all count, even ones too common to count in definitions ("have" picks
|
|
214
|
+
// "got" = "To have", not "To obtain").
|
|
215
|
+
function meaningOverlap(words, text) {
|
|
216
|
+
let n = 0;
|
|
217
|
+
for (const w of contentWords(text, words))
|
|
218
|
+
if (words.has(w))
|
|
219
|
+
n++;
|
|
220
|
+
return n;
|
|
160
221
|
}
|
|
161
222
|
function overlap(a, text) {
|
|
162
223
|
let n = 0;
|
|
@@ -187,10 +248,15 @@ function bridgeTerms(sense, from) {
|
|
|
187
248
|
}
|
|
188
249
|
// Headings ("As a copulative verb:") carry no meaning, so the specific definitions are used.
|
|
189
250
|
const glosses = sense.glosses.filter((g) => !g.trim().endsWith(':'));
|
|
251
|
+
// A first definition that is only a short list of equivalents ("To fetch, bring, take.") names the
|
|
252
|
+
// meaning with every item equally, so all of them are main terms ("take" → "lấy" scored no position
|
|
253
|
+
// bonus, and words found only through the ambiguous "get" itself came first: hóng, bắt, ra khỏi).
|
|
190
254
|
glosses.slice(0, 2).forEach((gloss, gi) => {
|
|
191
|
-
|
|
255
|
+
const found = glossTerms(gloss);
|
|
256
|
+
const list = gi === 0 && isTermList(gloss, found.length);
|
|
257
|
+
for (const [term, position] of found) {
|
|
192
258
|
terms.push(term);
|
|
193
|
-
if (gi === 0 && position === 0)
|
|
259
|
+
if (gi === 0 && (position === 0 || list))
|
|
194
260
|
main.add(term);
|
|
195
261
|
}
|
|
196
262
|
});
|
|
@@ -206,6 +272,11 @@ function bridgeTerms(sense, from) {
|
|
|
206
272
|
}
|
|
207
273
|
return { terms: unique.slice(0, 8), synonymsFrom, main };
|
|
208
274
|
}
|
|
275
|
+
// Every comma/semicolon part of the definition became a term, and each is at most three words.
|
|
276
|
+
function isTermList(gloss, termCount) {
|
|
277
|
+
const parts = gloss.replace(/\.\s*$/, '').split(/[;,]/);
|
|
278
|
+
return termCount >= 2 && parts.length === termCount && parts.every((p) => p.trim().split(/\s+/).length <= 3);
|
|
279
|
+
}
|
|
209
280
|
export function createTranslator(options = {}) {
|
|
210
281
|
const cache = new Map();
|
|
211
282
|
const dict = (lang) => {
|
|
@@ -351,34 +422,37 @@ export function createTranslator(options = {}) {
|
|
|
351
422
|
}
|
|
352
423
|
return hits;
|
|
353
424
|
}
|
|
354
|
-
|
|
355
|
-
// table; for other languages, the English sense whose table lists the source word (in the source
|
|
356
|
-
// language) is the same meaning, and its target-language list applies.
|
|
357
|
-
async function tableFor(sense, from, to, bridge) {
|
|
425
|
+
async function englishMeanings(sense, from, bridge) {
|
|
358
426
|
if (from === 'en')
|
|
359
|
-
return
|
|
427
|
+
return [{ word: sense.lemma, pos: sense.pos, gloss: sense.glosses[0] ?? '', translations: sense.translations }];
|
|
360
428
|
const found = [];
|
|
361
|
-
const add = (list) => {
|
|
362
|
-
for (const t of list ?? [])
|
|
363
|
-
if (!found.some((f) => f.word === t.word))
|
|
364
|
-
found.push(t);
|
|
365
|
-
};
|
|
366
429
|
for (const term of bridge.slice(0, 3)) {
|
|
367
430
|
for (const entry of await dict('en').lookup(term)) {
|
|
368
431
|
for (const s of entry.senses) {
|
|
369
432
|
const own = s.translations?.[from];
|
|
370
433
|
if (!own?.some((t) => t.word === sense.lemma || t.word === sense.word))
|
|
371
434
|
continue;
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
else
|
|
376
|
-
add(s.translations?.[to]);
|
|
435
|
+
if (!found.some((f) => f.word === entry.word && f.gloss === s.glosses[0])) {
|
|
436
|
+
found.push({ word: entry.word, pos: entry.pos, gloss: s.glosses[0], translations: s.translations });
|
|
437
|
+
}
|
|
377
438
|
}
|
|
378
439
|
}
|
|
379
440
|
}
|
|
380
441
|
return found;
|
|
381
442
|
}
|
|
443
|
+
async function tableFor(sense, from, to, bridge) {
|
|
444
|
+
if (from === 'en')
|
|
445
|
+
return to === 'en' ? [] : (sense.translations?.[to] ?? []);
|
|
446
|
+
const found = [];
|
|
447
|
+
for (const m of await englishMeanings(sense, from, bridge)) {
|
|
448
|
+
// Into English, the English word whose table lists the source word is itself the translation.
|
|
449
|
+
const list = to === 'en' ? [{ word: m.word }] : (m.translations?.[to] ?? []);
|
|
450
|
+
for (const t of list)
|
|
451
|
+
if (!found.some((f) => f.word === t.word))
|
|
452
|
+
found.push(t);
|
|
453
|
+
}
|
|
454
|
+
return found;
|
|
455
|
+
}
|
|
382
456
|
// Which of the target's regions a table tag names ("Latin-America" -> its countries), if any.
|
|
383
457
|
function tagRegions(tags, meta) {
|
|
384
458
|
const out = new Set();
|
|
@@ -480,8 +554,12 @@ export function createTranslator(options = {}) {
|
|
|
480
554
|
}
|
|
481
555
|
const api = {
|
|
482
556
|
senses,
|
|
483
|
-
async translate(word, { from, to, fromRegion, toRegion, pos, meaning, register: wantedRegister, listener, about, speaker, exclude = DEFAULT_EXCLUDED_LABELS, limit = 5, allSenses = false }) {
|
|
557
|
+
async translate(word, { from, to, fromRegion, toRegion, pos, meaning, register: wantedRegister, listener, about, speaker, exclude = DEFAULT_EXCLUDED_LABELS, limit = 5, allSenses = false, picks: usePicks = true }) {
|
|
484
558
|
const [sourceSenses, toMeta, fromMeta] = await Promise.all([senses(word, { from, fromRegion, pos }), dict(to).meta(), dict(from).meta()]);
|
|
559
|
+
// The target language's hand-picked words for English meanings (picks.json), when it has any.
|
|
560
|
+
const pickRows = usePicks && to !== 'en' && toMeta.picks ? await dict(to).picks() : [];
|
|
561
|
+
// Senses whose pick row asks to come first when the caller gives no pos or meaning.
|
|
562
|
+
const pickedFirst = new Set();
|
|
485
563
|
// The target's pronoun table, for personal-pronoun senses (throws for an unknown listener/about).
|
|
486
564
|
const pronounRows = toMeta.pronouns && sourceSenses.some((s) => personOf(s, from))
|
|
487
565
|
? await dict(to).pronouns({ region: toRegion, speaker, exclude })
|
|
@@ -492,7 +570,7 @@ export function createTranslator(options = {}) {
|
|
|
492
570
|
await dict(to).pronouns({ listener: about });
|
|
493
571
|
const toWanted = resolveRegion(toMeta, toRegion);
|
|
494
572
|
const hasFrequencies = Boolean(toMeta.frequencySource);
|
|
495
|
-
const meaningWords = meaning ? contentWords(meaning) : null;
|
|
573
|
+
const meaningWords = meaning ? contentWords(meaning, new Set(meaning.toLowerCase().split(/[^a-z']+/))) : null;
|
|
496
574
|
const groups = [];
|
|
497
575
|
for (const sense of sourceSenses) {
|
|
498
576
|
const { terms: bridge, synonymsFrom, main } = bridgeTerms(sense, from);
|
|
@@ -523,6 +601,7 @@ export function createTranslator(options = {}) {
|
|
|
523
601
|
// The term's own words say nothing about which meaning matched, so they don't count as overlap.
|
|
524
602
|
const termWords = contentWords(term);
|
|
525
603
|
const ctx = new Set([...context].filter((w) => !termWords.has(w)));
|
|
604
|
+
const isMain = main.has(term) || termIndex === 0;
|
|
526
605
|
const hits = to === 'en'
|
|
527
606
|
? await englishCandidates(term, posList, toMeta, toRegion, exclude)
|
|
528
607
|
// A generous limit: the dictionary's quick ranking puts region-tagged words first, and the
|
|
@@ -546,7 +625,10 @@ export function createTranslator(options = {}) {
|
|
|
546
625
|
// less (common words win anyway), and kinship words list "you" after "I/me" (Vietnamese "anh"),
|
|
547
626
|
// so the penalty is small then.
|
|
548
627
|
const score = 3 * overlap(ctx, hit.gloss) +
|
|
549
|
-
(
|
|
628
|
+
(isMain && exactGloss(hit.gloss, term, ctx) ? EXACT_GLOSS_BONUS : 0) -
|
|
629
|
+
(narrowed(hit.gloss, term, ctx) ? NARROWED_PENALTY : 0) -
|
|
630
|
+
(PASSIVE.test(hit.gloss) && !PASSIVE.test(sense.glosses.join(' ')) ? VOICE_MISMATCH_PENALTY : 0) +
|
|
631
|
+
(isMain ? 2 : termIndex === 1 ? 1 : 0) +
|
|
550
632
|
(hit.primary ? 1 : 0) +
|
|
551
633
|
// Tagged for the target region: a full bonus when the match is the word's main meaning, a
|
|
552
634
|
// small one otherwise ("bá cháy" is Southern for "awesome", not for "cool" the temperature).
|
|
@@ -608,6 +690,36 @@ export function createTranslator(options = {}) {
|
|
|
608
690
|
scored.set(t.word, { ...t, score: t.score + 4, bridge: 'synonym' });
|
|
609
691
|
}
|
|
610
692
|
}
|
|
693
|
+
// Hand-picked words for this meaning go first, in the listed order: a speaker's judgment of the
|
|
694
|
+
// natural word where the ranking puts another first. With a target region, picks tagged for it
|
|
695
|
+
// lead, and picks tagged only for other regions (or words not used there) are left to the
|
|
696
|
+
// ranking. The ranked words follow. (A pronoun-table row, below, still comes before them.)
|
|
697
|
+
const rows = pickRows.length
|
|
698
|
+
? await englishMeanings(sense, from, bridge).then((ms) => pickRows.filter((r) => ms.some((m) => m.word === r.word && m.pos === r.pos && m.gloss === r.gloss)))
|
|
699
|
+
: [];
|
|
700
|
+
if (rows.some((r) => r.first))
|
|
701
|
+
pickedFirst.add(sense);
|
|
702
|
+
const picked = [];
|
|
703
|
+
for (const t of rows.flatMap((r) => r.picks)) {
|
|
704
|
+
if (picked.some((p) => p.hit.word === t.word))
|
|
705
|
+
continue;
|
|
706
|
+
const places = tagRegions(t.tags, toMeta);
|
|
707
|
+
if (toWanted && places && ![...places].some((r) => toWanted.has(r)))
|
|
708
|
+
continue;
|
|
709
|
+
if (to === from && t.word === sense.lemma && !places?.size)
|
|
710
|
+
continue;
|
|
711
|
+
const hit = scored.get(t.word) ?? (await tableHit(t.word, to, posList, toWanted, exclude, bridge));
|
|
712
|
+
if (!hit || hit.labels.some((l) => exclude.includes(l)))
|
|
713
|
+
continue;
|
|
714
|
+
if (!places && !intersects(hit.regions, toWanted))
|
|
715
|
+
continue;
|
|
716
|
+
picked.push({ hit, tagged: Boolean(toWanted && places) });
|
|
717
|
+
}
|
|
718
|
+
if (picked.length) {
|
|
719
|
+
const ordered = [...picked.filter((p) => p.tagged), ...picked.filter((p) => !p.tagged)];
|
|
720
|
+
const top = Math.max(0, ...[...scored.values()].map((t) => t.score));
|
|
721
|
+
ordered.forEach(({ hit }, i) => scored.set(hit.word, { ...hit, score: top + 1 + (ordered.length - i) * 0.1, bridge: 'picked' }));
|
|
722
|
+
}
|
|
611
723
|
// Personal pronouns: the chosen row's words go first (the `listener`'s for I/you/we, the `about`'s
|
|
612
724
|
// for he/she/they, else the default row's), fitted to the pronoun's gender (he/she) and
|
|
613
725
|
// inclusiveness (we). Words tagged for the target region, then those fitting the register, lead.
|
|
@@ -706,15 +818,34 @@ export function createTranslator(options = {}) {
|
|
|
706
818
|
const chosen = ABOUT_PERSONS.has(p.person) ? about : listener;
|
|
707
819
|
return (chosen ? 10 : 0) - (hasSingular && !SINGULAR_PERSONS.has(p.person) ? 3 : 0);
|
|
708
820
|
};
|
|
821
|
+
// A form with its own entries ("got": "Have/has.", colloquial) as well as its base word's ("get"):
|
|
822
|
+
// the form's own meanings are why it has entries at all, so they stay ahead of the base word's, and
|
|
823
|
+
// a colloquial/informal label doesn't push them behind (slang and Internet labels still do: "felt"
|
|
824
|
+
// "To thoroughly defeat", Internet). A construction ("Must; have/has (to).") then only reorders
|
|
825
|
+
// them among themselves.
|
|
826
|
+
// Only inflections ("got": simple past of get), not contractions ("kinda") or spelling variants.
|
|
827
|
+
const formWithBase = ownSenses && sourceSenses.some((s) => s.via && INFLECTION_VIA.test(s.via));
|
|
828
|
+
const labelRelevance = (g) => {
|
|
829
|
+
const { labels } = g.source;
|
|
830
|
+
const marginal = labels.length > 0 && labels.every((l) => MARGINAL_LABELS.has(l));
|
|
831
|
+
const everyday = labels.length > 0 && labels.every((l) => EVERYDAY_LABELS.has(l));
|
|
832
|
+
if (formWithBase && !g.source.via && (!marginal || everyday)) {
|
|
833
|
+
return 1 - (CONSTRUCTION.test(g.source.glosses.join(' ')) ? 0.5 : 0);
|
|
834
|
+
}
|
|
835
|
+
return marginal ? -1 : 0;
|
|
836
|
+
};
|
|
709
837
|
const relevance = (g) => (meaningWords
|
|
710
|
-
? 10 *
|
|
838
|
+
? 10 * meaningOverlap(meaningWords, g.source.glosses.join(' ')) + 5 * meaningOverlap(meaningWords, g.translations[0]?.gloss ?? '')
|
|
711
839
|
: 0) -
|
|
712
840
|
(ownSenses && g.source.via && SPELLING_VIA.test(g.source.via) ? 2 : 0) +
|
|
713
841
|
(fromRegion && g.source.regionTagged ? 3 : 0) +
|
|
714
|
-
pronounRelevance(g)
|
|
715
|
-
(g
|
|
842
|
+
pronounRelevance(g) +
|
|
843
|
+
labelRelevance(g) +
|
|
844
|
+
(!meaning && pickedFirst.has(g.source) ? 5 : 0);
|
|
845
|
+
// A sense with no translations says nothing, so it goes after the ones that do ("got": "Expressing
|
|
846
|
+
// obligation; used with have." finds nothing; "Must; have/has (to)." gives "phải").
|
|
716
847
|
const ordered = groups
|
|
717
|
-
.map((g, i) => ({ g, i, r: relevance(g) }))
|
|
848
|
+
.map((g, i) => ({ g, i, r: relevance(g) - (g.translations.length ? 0 : 100) }))
|
|
718
849
|
.sort((a, b) => b.r - a.r || a.i - b.i)
|
|
719
850
|
.map((x) => x.g);
|
|
720
851
|
if (allSenses)
|
package/dist/types.d.ts
CHANGED
|
@@ -108,6 +108,24 @@ export type LanguageMeta = {
|
|
|
108
108
|
shardLength?: number;
|
|
109
109
|
/** true when the language has a pronoun table (pronouns.json, see PronounRow). */
|
|
110
110
|
pronouns?: boolean;
|
|
111
|
+
/** true when the language has hand-picked translations (picks.json, see PickRow). */
|
|
112
|
+
picks?: boolean;
|
|
113
|
+
};
|
|
114
|
+
/**
|
|
115
|
+
* Hand-picked first choices for one English meaning, in one target language (picks.json). A speaker's
|
|
116
|
+
* judgment of the natural word where the data ranks another first ("get", "To fetch, bring, take" →
|
|
117
|
+
* Vietnamese "lấy", not "đưa"). Keyed by English meaning, so each language needs one list, and
|
|
118
|
+
* translating between two other languages reaches it through the English meaning.
|
|
119
|
+
*/
|
|
120
|
+
export type PickRow = {
|
|
121
|
+
/** The English headword, part of speech and first definition of the meaning, as in the English data. */
|
|
122
|
+
word: string;
|
|
123
|
+
pos: string;
|
|
124
|
+
gloss: string;
|
|
125
|
+
/** Best first. `tags` name regions like translation-table tags ("Southern"). */
|
|
126
|
+
picks: TableTranslation[];
|
|
127
|
+
/** Put this meaning first when the caller gives no `pos` or `meaning` ("got" → "Have/has."). */
|
|
128
|
+
first?: boolean;
|
|
111
129
|
};
|
|
112
130
|
/**
|
|
113
131
|
* One word in the pronoun table: how to say "I", "you", "he/she", "we", plural "you" or "they" in one
|
|
@@ -218,6 +236,7 @@ export declare function normalizeEnglish(term: string): string;
|
|
|
218
236
|
* headings ending in ":" are skipped in favor of the specific definition.
|
|
219
237
|
*/
|
|
220
238
|
export declare function displayGloss(glosses: string[]): string;
|
|
239
|
+
export declare const TRAILING_PARTICLE: RegExp;
|
|
221
240
|
/**
|
|
222
241
|
* Splits an English gloss into search terms, main meaning first: "now, today, this time" ->
|
|
223
242
|
* [["now", 0], ["today", 1], ["this time", 2]]. Parenthesized notes and final punctuation are dropped.
|
package/dist/types.js
CHANGED
|
@@ -55,7 +55,7 @@ export function displayGloss(glosses) {
|
|
|
55
55
|
return glosses.find((g) => !g.trim().endsWith(':')) ?? glosses[0] ?? '';
|
|
56
56
|
}
|
|
57
57
|
// Trailing words dropped to also match the bare verb: "wait for" is found by "wait" too.
|
|
58
|
-
const TRAILING_PARTICLE = / (for|to|at|on|with|about|of|in|into|up|out|off|over)$/;
|
|
58
|
+
export const TRAILING_PARTICLE = / (for|to|at|on|with|about|of|in|into|up|out|off|over)$/;
|
|
59
59
|
/**
|
|
60
60
|
* Splits an English gloss into search terms, main meaning first: "now, today, this time" ->
|
|
61
61
|
* [["now", 0], ["today", 1], ["this time", 2]]. Parenthesized notes and final punctuation are dropped.
|
|
@@ -72,8 +72,11 @@ export function glossTerms(gloss) {
|
|
|
72
72
|
cleaned = cleaned.slice(cleaned.lastIndexOf(':') + 1);
|
|
73
73
|
const terms = [];
|
|
74
74
|
let position = 0;
|
|
75
|
-
// "I/me" lists two meanings, like "I; me".
|
|
76
|
-
|
|
75
|
+
// "I/me" lists two meanings, like "I; me". Sentences too: "To have. See usage notes." means "have",
|
|
76
|
+
// and cross-references ("See usage notes", "Compare shall", "Synonym: …") aren't meanings.
|
|
77
|
+
for (const part of cleaned.split(/[;,/]|\.\s+/)) {
|
|
78
|
+
if (/^\s*(?:see|compare|cf|synonyms?|antonyms?)\b/i.test(part))
|
|
79
|
+
continue;
|
|
77
80
|
// "etc." is not a meaning: "walking etc" -> "walking", and a lone "etc" is skipped.
|
|
78
81
|
const t = normalizeEnglish(part.replace(/[.!?]+\s*$/, '').replace(/\betc\.?$/i, '').trim());
|
|
79
82
|
if (!t || t.split(' ').length > 4 || !/^[a-z][a-z' -]*$/.test(t))
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "which-dialect",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.1",
|
|
4
4
|
"description": "Translate words between languages and dialects, meaning by meaning, starting with Vietnamese (Northern, Central, Southern) and English, with Wiktionary data loaded on demand. More languages are planned.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|