yomitan-core 0.1.0

package/dist/multi-language-transformer-AlxOM6b3.js

@@ -0,0 +1,637 @@
+import { log } from "./log-hgSll-dS.js";
+import { capitalizeFirstLetter, decapitalize, normalizeRadicalCharacters, removeAlphabeticDiacritics } from "./cjk-util-DubXBGDG.js";
+
+//#region src/language/language-descriptors.ts
+const capitalizationPreprocessors = {
+	decapitalize,
+	capitalizeFirstLetter
+};
+const languageDescriptors = [
+	{
+		iso: "bg",
+		iso639_3: "bul",
+		name: "Bulgarian",
+		exampleText: "чета",
+		textPreprocessors: capitalizationPreprocessors
+	},
+	{
+		iso: "cs",
+		iso639_3: "ces",
+		name: "Czech",
+		exampleText: "číst",
+		textPreprocessors: capitalizationPreprocessors
+	},
+	{
+		iso: "da",
+		iso639_3: "dan",
+		name: "Danish",
+		exampleText: "læse",
+		textPreprocessors: { ...capitalizationPreprocessors }
+	},
+	{
+		iso: "de",
+		iso639_3: "deu",
+		name: "German",
+		exampleText: "lesen",
+		textPreprocessors: { ...capitalizationPreprocessors }
+	},
+	{
+		iso: "el",
+		iso639_3: "ell",
+		name: "Greek",
+		exampleText: "διαβάζω",
+		textPreprocessors: { ...capitalizationPreprocessors }
+	},
+	{
+		iso: "en",
+		iso639_3: "eng",
+		name: "English",
+		exampleText: "read",
+		textPreprocessors: capitalizationPreprocessors
+	},
+	{
+		iso: "eo",
+		iso639_3: "epo",
+		name: "Esperanto",
+		exampleText: "legi",
+		textPreprocessors: capitalizationPreprocessors
+	},
+	{
+		iso: "es",
+		iso639_3: "spa",
+		name: "Spanish",
+		exampleText: "leer",
+		textPreprocessors: capitalizationPreprocessors
+	},
+	{
+		iso: "et",
+		iso639_3: "est",
+		name: "Estonian",
+		exampleText: "lugema",
+		textPreprocessors: capitalizationPreprocessors
+	},
+	{
+		iso: "fi",
+		iso639_3: "fin",
+		name: "Finnish",
+		exampleText: "lukea",
+		textPreprocessors: capitalizationPreprocessors
+	},
+	{
+		iso: "fr",
+		iso639_3: "fra",
+		name: "French",
+		exampleText: "lire",
+		textPreprocessors: { ...capitalizationPreprocessors }
+	},
+	{
+		iso: "ga",
+		iso639_3: "gle",
+		name: "Irish",
+		exampleText: "léigh",
+		textPreprocessors: capitalizationPreprocessors
+	},
+	{
+		iso: "grc",
+		iso639_3: "grc",
+		name: "Ancient Greek",
+		exampleText: "γράφω",
+		textPreprocessors: {
+			...capitalizationPreprocessors,
+			removeAlphabeticDiacritics
+		}
+	},
+	{
+		iso: "he",
+		iso639_3: "heb",
+		name: "Hebrew",
+		exampleText: "קריאה"
+	},
+	{
+		iso: "hi",
+		iso639_3: "hin",
+		name: "Hindi",
+		exampleText: "पढ़ने के लिए"
+	},
+	{
+		iso: "hu",
+		iso639_3: "hun",
+		name: "Hungarian",
+		exampleText: "olvasni",
+		textPreprocessors: capitalizationPreprocessors
+	},
+	{
+		iso: "id",
+		iso639_3: "ind",
+		name: "Indonesian",
+		exampleText: "baca",
+		textPreprocessors: {
+			...capitalizationPreprocessors,
+			removeAlphabeticDiacritics
+		}
+	},
+	{
+		iso: "it",
+		iso639_3: "ita",
+		name: "Italian",
+		exampleText: "leggere",
+		textPreprocessors: {
+			...capitalizationPreprocessors,
+			removeAlphabeticDiacritics
+		}
+	},
+	{
+		iso: "la",
+		iso639_3: "lat",
+		name: "Latin",
+		exampleText: "legō",
+		textPreprocessors: {
+			...capitalizationPreprocessors,
+			removeAlphabeticDiacritics
+		}
+	},
+	{
+		iso: "lo",
+		iso639_3: "lao",
+		name: "Lao",
+		exampleText: "ອ່ານ"
+	},
+	{
+		iso: "lv",
+		iso639_3: "lav",
+		name: "Latvian",
+		exampleText: "lasīt",
+		textPreprocessors: capitalizationPreprocessors
+	},
+	{
+		iso: "ja",
+		iso639_3: "jpn",
+		name: "Japanese",
+		exampleText: "読め"
+	},
+	{
+		iso: "ka",
+		iso639_3: "kat",
+		name: "Georgian",
+		exampleText: "კითხვა"
+	},
+	{
+		iso: "kn",
+		iso639_3: "kan",
+		name: "Kannada",
+		exampleText: "ಓದು"
+	},
+	{
+		iso: "km",
+		iso639_3: "khm",
+		name: "Khmer",
+		exampleText: "អាន"
+	},
+	{
+		iso: "ko",
+		iso639_3: "kor",
+		name: "Korean",
+		exampleText: "읽어"
+	},
+	{
+		iso: "mn",
+		iso639_3: "mon",
+		name: "Mongolian",
+		exampleText: "унших",
+		textPreprocessors: capitalizationPreprocessors
+	},
+	{
+		iso: "mt",
+		iso639_3: "mlt",
+		name: "Maltese",
+		exampleText: "kiteb",
+		textPreprocessors: capitalizationPreprocessors
+	},
+	{
+		iso: "nl",
+		iso639_3: "nld",
+		name: "Dutch",
+		exampleText: "lezen",
+		textPreprocessors: capitalizationPreprocessors
+	},
+	{
+		iso: "no",
+		iso639_3: "nor",
+		name: "Norwegian",
+		exampleText: "lese",
+		textPreprocessors: { ...capitalizationPreprocessors }
+	},
+	{
+		iso: "pl",
+		iso639_3: "pol",
+		name: "Polish",
+		exampleText: "czytać",
+		textPreprocessors: capitalizationPreprocessors
+	},
+	{
+		iso: "pt",
+		iso639_3: "por",
+		name: "Portuguese",
+		exampleText: "ler",
+		textPreprocessors: capitalizationPreprocessors
+	},
+	{
+		iso: "ro",
+		iso639_3: "ron",
+		name: "Romanian",
+		exampleText: "citi",
+		textPreprocessors: {
+			...capitalizationPreprocessors,
+			removeAlphabeticDiacritics
+		}
+	},
+	{
+		iso: "ru",
+		iso639_3: "rus",
+		name: "Russian",
+		exampleText: "читать",
+		textPreprocessors: { ...capitalizationPreprocessors }
+	},
+	{
+		iso: "sga",
+		iso639_3: "sga",
+		name: "Old Irish",
+		exampleText: "légaid",
+		textPreprocessors: {
+			...capitalizationPreprocessors,
+			removeAlphabeticDiacritics
+		}
+	},
+	{
+		iso: "sh",
+		iso639_3: "hbs",
+		name: "Serbo-Croatian",
+		exampleText: "čìtati",
+		textPreprocessors: { ...capitalizationPreprocessors }
+	},
+	{
+		iso: "sq",
+		iso639_3: "sqi",
+		name: "Albanian",
+		exampleText: "ndihmoj",
+		textPreprocessors: capitalizationPreprocessors
+	},
+	{
+		iso: "sv",
+		iso639_3: "swe",
+		name: "Swedish",
+		exampleText: "läsa",
+		textPreprocessors: capitalizationPreprocessors
+	},
+	{
+		iso: "th",
+		iso639_3: "tha",
+		name: "Thai",
+		exampleText: "อ่าน"
+	},
+	{
+		iso: "tl",
+		iso639_3: "tgl",
+		name: "Tagalog",
+		exampleText: "basahin",
+		textPreprocessors: {
+			...capitalizationPreprocessors,
+			removeAlphabeticDiacritics
+		}
+	},
+	{
+		iso: "tr",
+		iso639_3: "tur",
+		name: "Turkish",
+		exampleText: "okumak",
+		textPreprocessors: capitalizationPreprocessors
+	},
+	{
+		iso: "tok",
+		iso639_3: "tok",
+		name: "Toki Pona",
+		exampleText: "wile",
+		textPreprocessors: capitalizationPreprocessors
+	},
+	{
+		iso: "uk",
+		iso639_3: "ukr",
+		name: "Ukrainian",
+		exampleText: "читати",
+		textPreprocessors: capitalizationPreprocessors
+	},
+	{
+		iso: "vi",
+		iso639_3: "vie",
+		name: "Vietnamese",
+		exampleText: "đọc",
+		textPreprocessors: { ...capitalizationPreprocessors }
+	},
+	{
+		iso: "cy",
+		iso639_3: "cym",
+		name: "Welsh",
+		exampleText: "ddarllen",
+		textPreprocessors: capitalizationPreprocessors
+	},
+	{
+		iso: "yi",
+		iso639_3: "yid",
+		name: "Yiddish",
+		exampleText: "באַשאַפֿן"
+	},
+	{
+		iso: "yue",
+		iso639_3: "yue",
+		name: "Cantonese",
+		exampleText: "讀",
+		textPreprocessors: { normalizeRadicalCharacters }
+	},
+	{
+		iso: "zh",
+		iso639_3: "zho",
+		name: "Chinese",
+		exampleText: "读",
+		textPreprocessors: { normalizeRadicalCharacters }
+	}
+];
+const languageDescriptorMap = new Map();
+for (const languageDescriptor of languageDescriptors) languageDescriptorMap.set(languageDescriptor.iso, languageDescriptor);
+
+//#endregion
+//#region src/language/languages.ts
+function getLanguageSummaries() {
+	const results = [];
+	for (const { name, iso, iso639_3, exampleText } of languageDescriptorMap.values()) results.push({
+		name,
+		iso,
+		iso639_3,
+		exampleText
+	});
+	return results;
+}
+function getAllLanguageReadingNormalizers() {
+	const results = [];
+	for (const { iso, readingNormalizer } of languageDescriptorMap.values()) {
+		if (typeof readingNormalizer === "undefined") continue;
+		results.push({
+			iso,
+			readingNormalizer
+		});
+	}
+	return results;
+}
+function getAllLanguageTextProcessors() {
+	const results = [];
+	for (const { iso, textPreprocessors = {}, textPostprocessors = {} } of languageDescriptorMap.values()) {
+		const textPreprocessorsArray = [];
+		for (const [id, textPreprocessor] of Object.entries(textPreprocessors)) textPreprocessorsArray.push({
+			id,
+			textProcessor: textPreprocessor
+		});
+		const textPostprocessorsArray = [];
+		for (const [id, textPostprocessor] of Object.entries(textPostprocessors)) textPostprocessorsArray.push({
+			id,
+			textProcessor: textPostprocessor
+		});
+		results.push({
+			iso,
+			textPreprocessors: textPreprocessorsArray,
+			textPostprocessors: textPostprocessorsArray
+		});
+	}
+	return results;
+}
+function isTextLookupWorthy(text, language) {
+	const descriptor = languageDescriptorMap.get(language);
+	if (typeof descriptor === "undefined") return false;
+	return typeof descriptor.isTextLookupWorthy === "undefined" || descriptor.isTextLookupWorthy(text);
+}
+function getAllLanguageTransformDescriptors() {
+	const results = [];
+	for (const { iso, languageTransforms } of languageDescriptorMap.values()) if (languageTransforms) results.push({
+		iso,
+		languageTransforms
+	});
+	return results;
+}
+
+//#endregion
+//#region src/language/language-transformer.ts
+var LanguageTransformer = class LanguageTransformer {
+	_nextFlagIndex;
+	_transforms;
+	_conditionTypeToConditionFlagsMap;
+	_partOfSpeechToConditionFlagsMap;
+	constructor() {
+		this._nextFlagIndex = 0;
+		this._transforms = [];
+		this._conditionTypeToConditionFlagsMap = new Map();
+		this._partOfSpeechToConditionFlagsMap = new Map();
+	}
+	clear() {
+		this._nextFlagIndex = 0;
+		this._transforms = [];
+		this._conditionTypeToConditionFlagsMap.clear();
+		this._partOfSpeechToConditionFlagsMap.clear();
+	}
+	addDescriptor(descriptor) {
+		const { conditions, transforms } = descriptor;
+		const conditionEntries = Object.entries(conditions);
+		const { conditionFlagsMap, nextFlagIndex } = this._getConditionFlagsMap(conditionEntries, this._nextFlagIndex);
+		const transforms2 = [];
+		for (const [transformId, transform] of Object.entries(transforms)) {
+			const { name, description, rules } = transform;
+			const rules2 = [];
+			for (let j = 0, jj = rules.length; j < jj; ++j) {
+				const { type, isInflected, deinflect, conditionsIn, conditionsOut } = rules[j];
+				const conditionFlagsIn = this._getConditionFlagsStrict(conditionFlagsMap, conditionsIn);
+				if (conditionFlagsIn === null) throw new Error(`Invalid conditionsIn for transform ${transformId}.rules[${j}]`);
+				const conditionFlagsOut = this._getConditionFlagsStrict(conditionFlagsMap, conditionsOut);
+				if (conditionFlagsOut === null) throw new Error(`Invalid conditionsOut for transform ${transformId}.rules[${j}]`);
+				rules2.push({
+					type,
+					isInflected,
+					deinflect,
+					conditionsIn: conditionFlagsIn,
+					conditionsOut: conditionFlagsOut
+				});
+			}
+			const isInflectedTests = rules.map((rule) => rule.isInflected);
+			const heuristic = new RegExp(isInflectedTests.map((regExp) => regExp.source).join("|"));
+			transforms2.push({
+				id: transformId,
+				name,
+				description,
+				rules: rules2,
+				heuristic
+			});
+		}
+		this._nextFlagIndex = nextFlagIndex;
+		for (const transform of transforms2) this._transforms.push(transform);
+		for (const [type, { isDictionaryForm }] of conditionEntries) {
+			const flags = conditionFlagsMap.get(type);
+			if (typeof flags === "undefined") continue;
+			this._conditionTypeToConditionFlagsMap.set(type, flags);
+			if (isDictionaryForm) this._partOfSpeechToConditionFlagsMap.set(type, flags);
+		}
+	}
+	getConditionFlagsFromPartsOfSpeech(partsOfSpeech) {
+		return this._getConditionFlags(this._partOfSpeechToConditionFlagsMap, partsOfSpeech);
+	}
+	getConditionFlagsFromConditionTypes(conditionTypes) {
+		return this._getConditionFlags(this._conditionTypeToConditionFlagsMap, conditionTypes);
+	}
+	getConditionFlagsFromConditionType(conditionType) {
+		return this._getConditionFlags(this._conditionTypeToConditionFlagsMap, [conditionType]);
+	}
+	transform(sourceText) {
+		const results = [LanguageTransformer.createTransformedText(sourceText, 0, [])];
+		for (let i = 0; i < results.length; ++i) {
+			const { text, conditions, trace } = results[i];
+			for (const transform of this._transforms) {
+				if (!transform.heuristic.test(text)) continue;
+				const { id, rules } = transform;
+				for (let j = 0, jj = rules.length; j < jj; ++j) {
+					const rule = rules[j];
+					if (!LanguageTransformer.conditionsMatch(conditions, rule.conditionsIn)) continue;
+					const { isInflected, deinflect } = rule;
+					if (!isInflected.test(text)) continue;
+					const isCycle = trace.some((frame) => frame.transform === id && frame.ruleIndex === j && frame.text === text);
+					if (isCycle) {
+						log.warn(new Error(`Cycle detected in transform[${id}] rule[${j}] for text: ${text}\nTrace: ${JSON.stringify(trace)}`));
+						continue;
+					}
+					results.push(LanguageTransformer.createTransformedText(deinflect(text), rule.conditionsOut, this._extendTrace(trace, {
+						transform: id,
+						ruleIndex: j,
+						text
+					})));
+				}
+			}
+		}
+		return results;
+	}
+	getUserFacingInflectionRules(inflectionRules) {
+		return inflectionRules.map((rule) => {
+			const fullRule = this._transforms.find((transform) => transform.id === rule);
+			if (typeof fullRule === "undefined") return { name: rule };
+			const { name, description } = fullRule;
+			return description ? {
+				name,
+				description
+			} : { name };
+		});
+	}
+	static createTransformedText(text, conditions, trace) {
+		return {
+			text,
+			conditions,
+			trace
+		};
+	}
+	static conditionsMatch(currentConditions, nextConditions) {
+		return currentConditions === 0 || (currentConditions & nextConditions) !== 0;
+	}
+	_getConditionFlagsMap(conditions, nextFlagIndex) {
+		const conditionFlagsMap = new Map();
+		let targets = conditions;
+		while (targets.length > 0) {
+			const nextTargets = [];
+			for (const target of targets) {
+				const [type, condition] = target;
+				const { subConditions } = condition;
+				let flags = 0;
+				if (typeof subConditions === "undefined") {
+					if (nextFlagIndex >= 32) throw new Error("Maximum number of conditions was exceeded");
+					flags = 1 << nextFlagIndex;
+					++nextFlagIndex;
+				} else {
+					const multiFlags = this._getConditionFlagsStrict(conditionFlagsMap, subConditions);
+					if (multiFlags === null) {
+						nextTargets.push(target);
+						continue;
+					}
+					flags = multiFlags;
+				}
+				conditionFlagsMap.set(type, flags);
+			}
+			if (nextTargets.length === targets.length) throw new Error("Maximum number of conditions was exceeded");
+			targets = nextTargets;
+		}
+		return {
+			conditionFlagsMap,
+			nextFlagIndex
+		};
+	}
+	_getConditionFlagsStrict(conditionFlagsMap, conditionTypes) {
+		let flags = 0;
+		for (const conditionType of conditionTypes) {
+			const flags2 = conditionFlagsMap.get(conditionType);
+			if (typeof flags2 === "undefined") return null;
+			flags |= flags2;
+		}
+		return flags;
+	}
+	_getConditionFlags(conditionFlagsMap, conditionTypes) {
+		let flags = 0;
+		for (const conditionType of conditionTypes) {
+			let flags2 = conditionFlagsMap.get(conditionType);
+			if (typeof flags2 === "undefined") flags2 = 0;
+			flags |= flags2;
+		}
+		return flags;
+	}
+	_extendTrace(trace, newFrame) {
+		const newTrace = [newFrame];
+		for (const { transform, ruleIndex, text } of trace) newTrace.push({
+			transform,
+			ruleIndex,
+			text
+		});
+		return newTrace;
+	}
+};
+
+//#endregion
+//#region src/language/multi-language-transformer.ts
+var MultiLanguageTransformer = class {
+	_languageTransformers;
+	constructor() {
+		this._languageTransformers = new Map();
+	}
+	prepare() {
+		const languagesWithTransforms = getAllLanguageTransformDescriptors();
+		for (const { languageTransforms: descriptor } of languagesWithTransforms) {
+			const languageTransformer = new LanguageTransformer();
+			languageTransformer.addDescriptor(descriptor);
+			this._languageTransformers.set(descriptor.language, languageTransformer);
+		}
+	}
+	getConditionFlagsFromPartsOfSpeech(language, partsOfSpeech) {
+		const languageTransformer = this._languageTransformers.get(language);
+		return typeof languageTransformer !== "undefined" ? languageTransformer.getConditionFlagsFromPartsOfSpeech(partsOfSpeech) : 0;
+	}
+	getConditionFlagsFromConditionTypes(language, conditionTypes) {
+		const languageTransformer = this._languageTransformers.get(language);
+		return typeof languageTransformer !== "undefined" ? languageTransformer.getConditionFlagsFromConditionTypes(conditionTypes) : 0;
+	}
+	getConditionFlagsFromConditionType(language, conditionType) {
+		const languageTransformer = this._languageTransformers.get(language);
+		return typeof languageTransformer !== "undefined" ? languageTransformer.getConditionFlagsFromConditionType(conditionType) : 0;
+	}
+	transform(language, sourceText) {
+		const languageTransformer = this._languageTransformers.get(language);
+		if (typeof languageTransformer === "undefined") return [LanguageTransformer.createTransformedText(sourceText, 0, [])];
+		return languageTransformer.transform(sourceText);
+	}
+	getUserFacingInflectionRules(language, inflectionRules) {
+		const languageTransformer = this._languageTransformers.get(language);
+		if (typeof languageTransformer === "undefined") return inflectionRules.map((rule) => ({ name: rule }));
+		return languageTransformer.getUserFacingInflectionRules(inflectionRules);
+	}
+};
+
+//#endregion
+export { LanguageTransformer, MultiLanguageTransformer, getAllLanguageReadingNormalizers, getAllLanguageTextProcessors, getAllLanguageTransformDescriptors, getLanguageSummaries, isTextLookupWorthy, languageDescriptorMap };
+//# sourceMappingURL=multi-language-transformer-AlxOM6b3.js.map