yomitan-core 1.0.2 → 1.2.0

package/dist/{sentence-parser-DUQJ3cTT.cjs → sentence-parser-8i1Y_OGS.cjs}

@@ -1,5 +1,6 @@
 "use strict";
-const require_japanese = require('./japanese-BVV1Ctk-.cjs');
+const require_japanese = require('./japanese-C83fmEfb.cjs');
+const require_debug = require('./debug-I2dyAd2e.cjs');
 
 //#region src/lookup/sentence-parser.ts
 /**
@@ -17,22 +18,48 @@ var SentenceParser = class {
 	async parseText(text, language, options) {
 		const scanLength = options.scanLength ?? options.maxLength ?? 20;
 		const enabledDictionaryMap = this._buildFindTermDictionaryMap(options.enabledDictionaryMap);
+		require_debug.debugYomitanCore("sentence-parser", "parseText:start", {
+			textLength: text.length,
+			textPreview: text.slice(0, 120),
+			textCodePoints: require_debug.codePointPreview(text),
+			lineCount: text.split("\n").length,
+			scanLength,
+			language,
+			enabledDictionaryCount: enabledDictionaryMap.size,
+			enabledDictionaryNames: [...enabledDictionaryMap.keys()],
+			searchResolution: options.searchResolution ?? "letter",
+			removeNonJapaneseCharacters: options.removeNonJapaneseCharacters,
+			deinflect: options.deinflect
+		});
 		const lines = text.split("\n");
 		const results = [];
-		for (const line of lines) {
-			const segments = await this._parseLine(line, language, enabledDictionaryMap, scanLength, options);
-			results.push({ segments });
+		for (const [lineIndex, line] of lines.entries()) {
+			const segments = await this._parseLine(line, lineIndex, language, enabledDictionaryMap, scanLength, options);
+			results.push(segments);
 		}
+		require_debug.debugYomitanCore("sentence-parser", "parseText:complete", {
+			lineCount: results.length,
+			lineSegmentCounts: results.map((line) => line.length),
+			totalSegments: results.reduce((count, line) => count + line.length, 0),
+			matchedSegments: results.reduce((count, line) => count + line.filter((segment) => Array.isArray(segment.headwords)).length, 0)
+		});
 		return results;
 	}
 	/**
 	* Parses a single line using extension-equivalent scanning parser semantics.
 	*/
-	async _parseLine(line, language, enabledDictionaryMap, scanLength, options) {
+	async _parseLine(line, lineIndex, language, enabledDictionaryMap, scanLength, options) {
 		const findTermsOptions = this._createFindTermsOptions(language, enabledDictionaryMap, options);
 		const cache = new Map();
 		const result = [];
 		let previousUngroupedSegment = null;
+		let lookupCount = 0;
+		let cacheHits = 0;
+		let cacheMisses = 0;
+		let matchedSegmentCount = 0;
+		let unmatchedSegmentCount = 0;
+		let emptyEntryLookups = 0;
+		const lookupMissSamples = [];
 		let i = 0;
 		const ii = line.length;
 		while (i < ii) {
@@ -41,8 +68,18 @@ var SentenceParser = class {
 			const substring = line.substring(i, i + scanLength);
 			let cached = cache.get(substring);
 			if (!cached) {
+				cacheMisses += 1;
 				const { dictionaryEntries, originalTextLength: originalTextLength$1 } = await this._translator.findTerms("simple", substring, findTermsOptions);
+				lookupCount += 1;
 				let segments$1 = [];
+				if (dictionaryEntries.length === 0) {
+					emptyEntryLookups += 1;
+					if (lookupMissSamples.length < 5) lookupMissSamples.push({
+						substring: substring.slice(0, 40),
+						codePoints: require_debug.codePointPreview(substring, 20),
+						originalTextLength: originalTextLength$1
+					});
+				}
 				if (dictionaryEntries.length > 0 && originalTextLength$1 > 0 && (originalTextLength$1 !== character.length || require_japanese.isCodePointJapanese(codePoint))) {
 					const source = substring.substring(0, originalTextLength$1);
 					segments$1 = this._createMatchedSegments(source, dictionaryEntries);
@@ -52,36 +89,38 @@ var SentenceParser = class {
 					segments: segments$1
 				};
 				cache.set(substring, cached);
-			}
+			} else cacheHits += 1;
 			const { originalTextLength, segments } = cached;
 			if (segments.length > 0) {
 				previousUngroupedSegment = null;
 				result.push(...segments);
+				matchedSegmentCount += segments.length;
 				i += Math.max(1, originalTextLength);
 			} else {
 				if (previousUngroupedSegment === null) {
 					previousUngroupedSegment = {
 						text: character,
-						reading: "",
-						term: character,
-						entries: [],
-						furigana: [{
-							text: character,
-							reading: ""
-						}]
+						reading: ""
 					};
 					result.push(previousUngroupedSegment);
-				} else {
-					previousUngroupedSegment.text += character;
-					previousUngroupedSegment.term = previousUngroupedSegment.text;
-					previousUngroupedSegment.furigana = [{
-						text: previousUngroupedSegment.text,
-						reading: ""
-					}];
-				}
+					unmatchedSegmentCount += 1;
+				} else previousUngroupedSegment.text += character;
 				i += character.length;
 			}
 		}
+		require_debug.debugYomitanCore("sentence-parser", "parseLine:complete", {
+			lineIndex,
+			lineLength: line.length,
+			linePreview: line.slice(0, 120),
+			lookupCount,
+			cacheHits,
+			cacheMisses,
+			emptyEntryLookups,
+			matchedSegmentCount,
+			unmatchedSegmentCount,
+			finalSegmentCount: result.length,
+			missSamples: lookupMissSamples
+		});
 		return result;
 	}
 	_createMatchedSegments(source, dictionaryEntries) {
@@ -89,35 +128,24 @@ var SentenceParser = class {
 		const firstEntry = dictionaryEntries[0];
 		const firstHeadword = firstEntry.headwords[0];
 		if (!firstHeadword) return [];
-		const reading = firstHeadword.reading ?? "";
-		const term = firstHeadword.term ?? source;
-		const furigana = require_japanese.distributeFuriganaInflected(term, reading, source);
-		const filteredEntries = this._trimEntriesForSource(dictionaryEntries, source);
-		return [{
-			text: source,
-			reading,
-			term,
-			entries: filteredEntries,
-			furigana
-		}];
-	}
-	_trimEntriesForSource(dictionaryEntries, source) {
-		const trimmed = [];
-		for (const entry of dictionaryEntries) {
-			const headwords = [];
-			for (const headword of entry.headwords) {
+		const trimmedHeadwords = [];
+		for (const dictionaryEntry of dictionaryEntries) {
+			const validHeadwords = [];
+			for (const headword of dictionaryEntry.headwords) {
 				const validSources = headword.sources.filter((src) => src.originalText === source && src.isPrimary && src.matchType === "exact");
-				if (validSources.length > 0) headwords.push({
-					...headword,
+				if (validSources.length > 0) validHeadwords.push({
+					term: headword.term,
+					reading: headword.reading,
 					sources: validSources
 				});
 			}
-			if (headwords.length > 0) trimmed.push({
-				...entry,
-				headwords
-			});
+			if (validHeadwords.length > 0) trimmedHeadwords.push(validHeadwords);
 		}
-		return trimmed.length > 0 ? trimmed : dictionaryEntries;
+		return [{
+			text: source,
+			reading: firstHeadword.reading ?? "",
+			...trimmedHeadwords.length > 0 ? { headwords: trimmedHeadwords } : {}
+		}];
 	}
 	/**
 	* Builds a FindTermDictionary map from the input format.
@@ -161,4 +189,4 @@ Object.defineProperty(exports, 'SentenceParser', {
     return SentenceParser;
   }
 });
-//# sourceMappingURL=sentence-parser-DUQJ3cTT.cjs.map
+//# sourceMappingURL=sentence-parser-8i1Y_OGS.cjs.map

package/dist/sentence-parser-8i1Y_OGS.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"sentence-parser-8i1Y_OGS.cjs","names":["translator: Translator","text: string","language: string","options: SentenceParserOptions","results: ParseTextLine[]","line: string","lineIndex: number","enabledDictionaryMap: Map<string, FindTermDictionary>","scanLength: number","result: ParseTextSegment[]","previousUngroupedSegment: ParseTextSegment | null","lookupMissSamples: Array<{ substring: string; codePoints: string[]; originalTextLength: number }>","segments: ParseTextSegment[]","originalTextLength","source: string","dictionaryEntries: TermDictionaryEntry[]","trimmedHeadwords: ParseTextHeadword[][]","validHeadwords: ParseTextHeadword[]","inputMap: Map<string, SentenceParserDictionaryInput>"],"sources":["../src/lookup/sentence-parser.ts"],"sourcesContent":["/*\n * Copyright (C) 2023-2025  Yomitan Authors\n *\n * This program is free software: you can redistribute it and/or modify\n * it under the terms of the GNU General Public License as published by\n * the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n * GNU General Public License for more details.\n *\n * You should have received a copy of the GNU General Public License\n * along with this program.  If not, see <https://www.gnu.org/licenses/>.\n */\n\nimport { isCodePointJapanese } from '../language/ja/japanese';\nimport type { TermDictionaryEntry } from '../types/dictionary';\nimport type { ParseTextHeadword, ParseTextLine, ParseTextSegment } from '../types/parse';\nimport type { FindTermDictionary, FindTermsTextReplacements, SearchResolution } from '../types/translation';\nimport { codePointPreview, debugYomitanCore } from '../util/debug';\nimport type { Translator } from './translator';\nexport type { ParseTextHeadword, ParseTextLine, ParseTextResultItem, ParseTextSegment } from '../types/parse';\n\nexport interface SentenceParserDictionaryInput {\n    index: number;\n    priority?: number;\n    alias?: string;\n    allowSecondarySearches?: boolean;\n    partsOfSpeechFilter?: boolean;\n    useDeinflections?: boolean;\n}\n\nexport interface SentenceParserOptions {\n    /** The enabled dictionary map for lookups. */\n    enabledDictionaryMap: Map<string, SentenceParserDictionaryInput>;\n    /** Window size used by scanning parser. Defaults to 20. */\n    scanLength?: number;\n    /** Backward compatibility alias for scanLength. */\n    maxLength?: number;\n    /** Search resolution for translator. */\n    searchResolution?: SearchResolution;\n    /** Whether to remove non-Japanese characters in lookups. */\n    removeNonJapaneseCharacters?: boolean;\n    /** Whether to apply deinflection. */\n    deinflect?: boolean;\n    /** Text replacements forwarded to translator. */\n    textReplacements?: FindTermsTextReplacements;\n}\n\ninterface ParseScanCacheEntry {\n    originalTextLength: number;\n    segments: ParseTextSegment[];\n}\n\n/**\n * Parses text into segments with dictionary lookups using the same\n * scanning-parser model as Yomitan extension (non-MeCab path).\n */\nexport class SentenceParser {\n    private _translator: Translator;\n\n    constructor(translator: Translator) {\n        this._translator = translator;\n    }\n\n    /**\n     * Parses the given text into lines of parsed segments.\n     */\n    async parseText(text: string, language: string, options: SentenceParserOptions): Promise<ParseTextLine[]> {\n        const scanLength = options.scanLength ?? options.maxLength ?? 20;\n        const enabledDictionaryMap = this._buildFindTermDictionaryMap(options.enabledDictionaryMap);\n        debugYomitanCore('sentence-parser', 'parseText:start', {\n            textLength: text.length,\n            textPreview: text.slice(0, 120),\n            textCodePoints: codePointPreview(text),\n            lineCount: text.split('\\n').length,\n            scanLength,\n            language,\n            enabledDictionaryCount: enabledDictionaryMap.size,\n            enabledDictionaryNames: [...enabledDictionaryMap.keys()],\n            searchResolution: options.searchResolution ?? 'letter',\n            removeNonJapaneseCharacters: options.removeNonJapaneseCharacters,\n            deinflect: options.deinflect,\n        });\n\n        const lines = text.split('\\n');\n        const results: ParseTextLine[] = [];\n\n        for (const [lineIndex, line] of lines.entries()) {\n            const segments = await this._parseLine(line, lineIndex, language, enabledDictionaryMap, scanLength, options);\n            results.push(segments);\n        }\n\n        debugYomitanCore('sentence-parser', 'parseText:complete', {\n            lineCount: results.length,\n            lineSegmentCounts: results.map((line) => line.length),\n            totalSegments: results.reduce((count, line) => count + line.length, 0),\n            matchedSegments: results.reduce(\n                (count, line) => count + line.filter((segment) => Array.isArray(segment.headwords)).length,\n                0,\n            ),\n        });\n\n        return results;\n    }\n\n    /**\n     * Parses a single line using extension-equivalent scanning parser semantics.\n     */\n    private async _parseLine(\n        line: string,\n        lineIndex: number,\n        language: string,\n        enabledDictionaryMap: Map<string, FindTermDictionary>,\n        scanLength: number,\n        options: SentenceParserOptions,\n    ): Promise<ParseTextLine> {\n        const findTermsOptions = this._createFindTermsOptions(language, enabledDictionaryMap, options);\n        const cache = new Map<string, ParseScanCacheEntry>();\n\n        const result: ParseTextSegment[] = [];\n        let previousUngroupedSegment: ParseTextSegment | null = null;\n        let lookupCount = 0;\n        let cacheHits = 0;\n        let cacheMisses = 0;\n        let matchedSegmentCount = 0;\n        let unmatchedSegmentCount = 0;\n        let emptyEntryLookups = 0;\n        const lookupMissSamples: Array<{ substring: string; codePoints: string[]; originalTextLength: number }> = [];\n\n        let i = 0;\n        const ii = line.length;\n        while (i < ii) {\n            const codePoint = line.codePointAt(i) as number;\n            const character = String.fromCodePoint(codePoint);\n            const substring = line.substring(i, i + scanLength);\n\n            let cached = cache.get(substring);\n            if (!cached) {\n                cacheMisses += 1;\n                const { dictionaryEntries, originalTextLength } = await this._translator.findTerms(\n                    'simple',\n                    substring,\n                    findTermsOptions,\n                );\n                lookupCount += 1;\n\n                let segments: ParseTextSegment[] = [];\n                if (dictionaryEntries.length === 0) {\n                    emptyEntryLookups += 1;\n                    if (lookupMissSamples.length < 5) {\n                        lookupMissSamples.push({\n                            substring: substring.slice(0, 40),\n                            codePoints: codePointPreview(substring, 20),\n                            originalTextLength,\n                        });\n                    }\n                }\n\n                if (\n                    dictionaryEntries.length > 0 &&\n                    originalTextLength > 0 &&\n                    (originalTextLength !== character.length || isCodePointJapanese(codePoint))\n                ) {\n                    const source = substring.substring(0, originalTextLength);\n                    segments = this._createMatchedSegments(source, dictionaryEntries);\n                }\n\n                cached = {\n                    originalTextLength,\n                    segments,\n                };\n                cache.set(substring, cached);\n            } else {\n                cacheHits += 1;\n            }\n\n            const { originalTextLength, segments } = cached;\n            if (segments.length > 0) {\n                previousUngroupedSegment = null;\n                result.push(...segments);\n                matchedSegmentCount += segments.length;\n                i += Math.max(1, originalTextLength);\n            } else {\n                if (previousUngroupedSegment === null) {\n                    previousUngroupedSegment = {\n                        text: character,\n                        reading: ''\n                    };\n                    result.push(previousUngroupedSegment);\n                    unmatchedSegmentCount += 1;\n                } else {\n                    previousUngroupedSegment.text += character;\n                }\n                i += character.length;\n            }\n        }\n\n        debugYomitanCore('sentence-parser', 'parseLine:complete', {\n            lineIndex,\n            lineLength: line.length,\n            linePreview: line.slice(0, 120),\n            lookupCount,\n            cacheHits,\n            cacheMisses,\n            emptyEntryLookups,\n            matchedSegmentCount,\n            unmatchedSegmentCount,\n            finalSegmentCount: result.length,\n            missSamples: lookupMissSamples,\n        });\n\n        return result;\n    }\n\n    private _createMatchedSegments(source: string, dictionaryEntries: TermDictionaryEntry[]): ParseTextSegment[] {\n        if (dictionaryEntries.length === 0) {\n            return [];\n        }\n\n        const firstEntry = dictionaryEntries[0];\n        const firstHeadword = firstEntry.headwords[0];\n        if (!firstHeadword) {\n            return [];\n        }\n\n        const trimmedHeadwords: ParseTextHeadword[][] = [];\n        for (const dictionaryEntry of dictionaryEntries) {\n            const validHeadwords: ParseTextHeadword[] = [];\n            for (const headword of dictionaryEntry.headwords) {\n                const validSources = headword.sources.filter(\n                    (src) => src.originalText === source && src.isPrimary && src.matchType === 'exact',\n                );\n\n                if (validSources.length > 0) {\n                    validHeadwords.push({\n                        term: headword.term,\n                        reading: headword.reading,\n                        sources: validSources,\n                    });\n                }\n            }\n            if (validHeadwords.length > 0) {\n                trimmedHeadwords.push(validHeadwords);\n            }\n        }\n\n        return [\n            {\n                text: source,\n                reading: firstHeadword.reading ?? '',\n                ...(trimmedHeadwords.length > 0 ? { headwords: trimmedHeadwords } : {}),\n            },\n        ];\n    }\n\n    /**\n     * Builds a FindTermDictionary map from the input format.\n     */\n    private _buildFindTermDictionaryMap(\n        inputMap: Map<string, SentenceParserDictionaryInput>,\n    ): Map<string, FindTermDictionary> {\n        const result = new Map<string, FindTermDictionary>();\n\n        for (const [name, config] of inputMap.entries()) {\n            result.set(name, {\n                index: config.index,\n                alias: config.alias ?? name,\n                allowSecondarySearches: config.allowSecondarySearches ?? false,\n                partsOfSpeechFilter: config.partsOfSpeechFilter ?? true,\n                useDeinflections: config.useDeinflections ?? true,\n            });\n        }\n\n        return result;\n    }\n\n    /**\n     * Creates a FindTermsOptions object for scanning parser lookups.\n     */\n    private _createFindTermsOptions(\n        language: string,\n        enabledDictionaryMap: Map<string, FindTermDictionary>,\n        options: SentenceParserOptions,\n    ) {\n        return {\n            matchType: 'exact' as const,\n            deinflect: options.deinflect ?? true,\n            primaryReading: '',\n            mainDictionary: '',\n            sortFrequencyDictionary: null,\n            sortFrequencyDictionaryOrder: 'descending' as const,\n            removeNonJapaneseCharacters:\n                options.removeNonJapaneseCharacters ??\n                (language === 'ja' || language === 'zh' || language === 'yue' || language === 'ko'),\n            textReplacements: options.textReplacements ?? [null],\n            enabledDictionaryMap,\n            excludeDictionaryDefinitions: null,\n            searchResolution: options.searchResolution ?? ('letter' as const),\n            language,\n        };\n    }\n}\n"],"mappings":";;;;;;;;;AA4DA,IAAa,iBAAb,MAA4B;CACxB,AAAQ;CAER,YAAYA,YAAwB;AAChC,OAAK,cAAc;CACtB;;;;CAKD,MAAM,UAAUC,MAAcC,UAAkBC,SAA0D;EACtG,MAAM,aAAa,QAAQ,cAAc,QAAQ,aAAa;EAC9D,MAAM,uBAAuB,KAAK,4BAA4B,QAAQ,qBAAqB;AAC3F,iCAAiB,mBAAmB,mBAAmB;GACnD,YAAY,KAAK;GACjB,aAAa,KAAK,MAAM,GAAG,IAAI;GAC/B,gBAAgB,+BAAiB,KAAK;GACtC,WAAW,KAAK,MAAM,KAAK,CAAC;GAC5B;GACA;GACA,wBAAwB,qBAAqB;GAC7C,wBAAwB,CAAC,GAAG,qBAAqB,MAAM,AAAC;GACxD,kBAAkB,QAAQ,oBAAoB;GAC9C,6BAA6B,QAAQ;GACrC,WAAW,QAAQ;EACtB,EAAC;EAEF,MAAM,QAAQ,KAAK,MAAM,KAAK;EAC9B,MAAMC,UAA2B,CAAE;AAEnC,OAAK,MAAM,CAAC,WAAW,KAAK,IAAI,MAAM,SAAS,EAAE;GAC7C,MAAM,WAAW,MAAM,KAAK,WAAW,MAAM,WAAW,UAAU,sBAAsB,YAAY,QAAQ;AAC5G,WAAQ,KAAK,SAAS;EACzB;AAED,iCAAiB,mBAAmB,sBAAsB;GACtD,WAAW,QAAQ;GACnB,mBAAmB,QAAQ,IAAI,CAAC,SAAS,KAAK,OAAO;GACrD,eAAe,QAAQ,OAAO,CAAC,OAAO,SAAS,QAAQ,KAAK,QAAQ,EAAE;GACtE,iBAAiB,QAAQ,OACrB,CAAC,OAAO,SAAS,QAAQ,KAAK,OAAO,CAAC,YAAY,MAAM,QAAQ,QAAQ,UAAU,CAAC,CAAC,QACpF,EACH;EACJ,EAAC;AAEF,SAAO;CACV;;;;CAKD,MAAc,WACVC,MACAC,WACAJ,UACAK,sBACAC,YACAL,SACsB;EACtB,MAAM,mBAAmB,KAAK,wBAAwB,UAAU,sBAAsB,QAAQ;EAC9F,MAAM,QAAQ,IAAI;EAElB,MAAMM,SAA6B,CAAE;EACrC,IAAIC,2BAAoD;EACxD,IAAI,cAAc;EAClB,IAAI,YAAY;EAChB,IAAI,cAAc;EAClB,IAAI,sBAAsB;EAC1B,IAAI,wBAAwB;EAC5B,IAAI,oBAAoB;EACxB,MAAMC,oBAAoG,CAAE;EAE5G,IAAI,IAAI;EACR,MAAM,KAAK,KAAK;AAChB,SAAO,IAAI,IAAI;GACX,MAAM,YAAY,KAAK,YAAY,EAAE;GACrC,MAAM,YAAY,OAAO,cAAc,UAAU;GACjD,MAAM,YAAY,KAAK,UAAU,GAAG,IAAI,WAAW;GAEnD,IAAI,SAAS,MAAM,IAAI,UAAU;AACjC,QAAK,QAAQ;AACT,mBAAe;IACf,MAAM,EAAE,mBAAmB,0CAAoB,GAAG,MAAM,KAAK,YAAY,UACrE,UACA,WACA,iBACH;AACD,mBAAe;IAEf,IAAIC,aAA+B,CAAE;AACrC,QAAI,kBAAkB,WAAW,GAAG;AAChC,0BAAqB;AACrB,SAAI,kBAAkB,SAAS,EAC3B,mBAAkB,KAAK;MACnB,WAAW,UAAU,MAAM,GAAG,GAAG;MACjC,YAAY,+BAAiB,WAAW,GAAG;MAC3C;KACH,EAAC;IAET;AAED,QACI,kBAAkB,SAAS,KAC3BC,uBAAqB,MACpBA,yBAAuB,UAAU,UAAU,qCAAoB,UAAU,GAC5E;KACE,MAAM,SAAS,UAAU,UAAU,GAAGA,qBAAmB;AACzD,kBAAW,KAAK,uBAAuB,QAAQ,kBAAkB;IACpE;AAED,aAAS;KACL;KACA;IACH;AACD,UAAM,IAAI,WAAW,OAAO;GAC/B,MACG,cAAa;GAGjB,MAAM,EAAE,oBAAoB,UAAU,GAAG;AACzC,OAAI,SAAS,SAAS,GAAG;AACrB,+BAA2B;AAC3B,WAAO,KAAK,GAAG,SAAS;AACxB,2BAAuB,SAAS;AAChC,SAAK,KAAK,IAAI,GAAG,mBAAmB;GACvC,OAAM;AACH,QAAI,6BAA6B,MAAM;AACnC,gCAA2B;MACvB,MAAM;MACN,SAAS;KACZ;AACD,YAAO,KAAK,yBAAyB;AACrC,8BAAyB;IAC5B,MACG,0BAAyB,QAAQ;AAErC,SAAK,UAAU;GAClB;EACJ;AAED,iCAAiB,mBAAmB,sBAAsB;GACtD;GACA,YAAY,KAAK;GACjB,aAAa,KAAK,MAAM,GAAG,IAAI;GAC/B;GACA;GACA;GACA;GACA;GACA;GACA,mBAAmB,OAAO;GAC1B,aAAa;EAChB,EAAC;AAEF,SAAO;CACV;CAED,AAAQ,uBAAuBC,QAAgBC,mBAA8D;AACzG,MAAI,kBAAkB,WAAW,EAC7B,QAAO,CAAE;EAGb,MAAM,aAAa,kBAAkB;EACrC,MAAM,gBAAgB,WAAW,UAAU;AAC3C,OAAK,cACD,QAAO,CAAE;EAGb,MAAMC,mBAA0C,CAAE;AAClD,OAAK,MAAM,mBAAmB,mBAAmB;GAC7C,MAAMC,iBAAsC,CAAE;AAC9C,QAAK,MAAM,YAAY,gBAAgB,WAAW;IAC9C,MAAM,eAAe,SAAS,QAAQ,OAClC,CAAC,QAAQ,IAAI,iBAAiB,UAAU,IAAI,aAAa,IAAI,cAAc,QAC9E;AAED,QAAI,aAAa,SAAS,EACtB,gBAAe,KAAK;KAChB,MAAM,SAAS;KACf,SAAS,SAAS;KAClB,SAAS;IACZ,EAAC;GAET;AACD,OAAI,eAAe,SAAS,EACxB,kBAAiB,KAAK,eAAe;EAE5C;AAED,SAAO,CACH;GACI,MAAM;GACN,SAAS,cAAc,WAAW;GAClC,GAAI,iBAAiB,SAAS,IAAI,EAAE,WAAW,iBAAkB,IAAG,CAAE;EACzE,CACJ;CACJ;;;;CAKD,AAAQ,4BACJC,UAC+B;EAC/B,MAAM,SAAS,IAAI;AAEnB,OAAK,MAAM,CAAC,MAAM,OAAO,IAAI,SAAS,SAAS,CAC3C,QAAO,IAAI,MAAM;GACb,OAAO,OAAO;GACd,OAAO,OAAO,SAAS;GACvB,wBAAwB,OAAO,0BAA0B;GACzD,qBAAqB,OAAO,uBAAuB;GACnD,kBAAkB,OAAO,oBAAoB;EAChD,EAAC;AAGN,SAAO;CACV;;;;CAKD,AAAQ,wBACJhB,UACAK,sBACAJ,SACF;AACE,SAAO;GACH,WAAW;GACX,WAAW,QAAQ,aAAa;GAChC,gBAAgB;GAChB,gBAAgB;GAChB,yBAAyB;GACzB,8BAA8B;GAC9B,6BACI,QAAQ,gCACP,aAAa,QAAQ,aAAa,QAAQ,aAAa,SAAS,aAAa;GAClF,kBAAkB,QAAQ,oBAAoB,CAAC,IAAK;GACpD;GACA,8BAA8B;GAC9B,kBAAkB,QAAQ,oBAAqB;GAC/C;EACH;CACJ;AACJ"}

package/dist/{translator-TiTDVbmS.d.cts → sentence-parser-BQpu2oSA.d.cts}

@@ -1,4 +1,4 @@
-import { KanjiDictionaryEntry, TermDictionaryEntry, TermSourceMatchType } from "./dictionary-6Gj8AfGC.cjs";
+import { KanjiDictionaryEntry, TermDictionaryEntry, TermSource, TermSourceMatchType } from "./dictionary-6Gj8AfGC.cjs";
 import { DictionaryDB } from "./dictionary-database-3OesNIMt.cjs";
 
 //#region src/types/translation.d.ts
@@ -41,9 +41,31 @@ type FindTermDictionary = {
 };
 type TermEnabledDictionaryMap = Map<string, FindTermDictionary>;
 type KanjiEnabledDictionaryMap = Map<string, FindKanjiDictionary>; //#endregion
-//#region src/lookup/translator.d.ts
+//#region src/types/parse.d.ts
 
 //# sourceMappingURL=translation.d.ts.map
+type ParseTextHeadword = {
+  term: string;
+  reading: string;
+  sources: TermSource[];
+};
+type ParseTextSegment = {
+  text: string;
+  reading: string;
+  headwords?: ParseTextHeadword[][];
+};
+type ParseTextLine = ParseTextSegment[];
+type ParseTextResultItem = {
+  id: string;
+  source: 'scanning-parser' | 'mecab';
+  dictionary: null | string;
+  index: number;
+  content: ParseTextLine[];
+};
+
+//#endregion
+//#region src/lookup/translator.d.ts
+//# sourceMappingURL=parse.d.ts.map
 type TermDictionaryEntry$1 = TermDictionaryEntry;
 type FindTermsMode = 'group' | 'term' | 'merge' | 'simple';
 /**
@@ -176,5 +198,59 @@ declare class Translator {
   private _addUserFacingInflections;
   private _hasAny;
 } //#endregion
-export { FindKanjiDictionary, FindKanjiOptions, FindTermDictionary, FindTermsMatchType, FindTermsMode, FindTermsOptions, FindTermsSortOrder, FindTermsTextReplacement, FindTermsTextReplacements, KanjiEnabledDictionaryMap, SearchResolution, TermEnabledDictionaryMap, Translator };
-//# sourceMappingURL=translator-TiTDVbmS.d.cts.map
+//#region src/lookup/sentence-parser.d.ts
+interface SentenceParserDictionaryInput {
+  index: number;
+  priority?: number;
+  alias?: string;
+  allowSecondarySearches?: boolean;
+  partsOfSpeechFilter?: boolean;
+  useDeinflections?: boolean;
+}
+interface SentenceParserOptions {
+  /** The enabled dictionary map for lookups. */
+  enabledDictionaryMap: Map<string, SentenceParserDictionaryInput>;
+  /** Window size used by scanning parser. Defaults to 20. */
+  scanLength?: number;
+  /** Backward compatibility alias for scanLength. */
+  maxLength?: number;
+  /** Search resolution for translator. */
+  searchResolution?: SearchResolution;
+  /** Whether to remove non-Japanese characters in lookups. */
+  removeNonJapaneseCharacters?: boolean;
+  /** Whether to apply deinflection. */
+  deinflect?: boolean;
+  /** Text replacements forwarded to translator. */
+  textReplacements?: FindTermsTextReplacements;
+}
+/**
+ * Parses text into segments with dictionary lookups using the same
+ * scanning-parser model as Yomitan extension (non-MeCab path).
+ */
+declare class SentenceParser {
+  private _translator;
+  constructor(translator: Translator);
+  /**
+   * Parses the given text into lines of parsed segments.
+   */
+  parseText(text: string, language: string, options: SentenceParserOptions): Promise<ParseTextLine[]>;
+  /**
+   * Parses a single line using extension-equivalent scanning parser semantics.
+   */
+  private _parseLine;
+  private _createMatchedSegments;
+  /**
+   * Builds a FindTermDictionary map from the input format.
+   */
+  private _buildFindTermDictionaryMap;
+  /**
+   * Creates a FindTermsOptions object for scanning parser lookups.
+   */
+  private _createFindTermsOptions;
+}
+
+//#endregion
+//# sourceMappingURL=sentence-parser.d.ts.map
+
+export { FindKanjiDictionary, FindKanjiOptions, FindTermDictionary, FindTermsMatchType, FindTermsMode, FindTermsOptions, FindTermsSortOrder, FindTermsTextReplacement, FindTermsTextReplacements, KanjiEnabledDictionaryMap, ParseTextHeadword, ParseTextLine, ParseTextResultItem, ParseTextSegment, SearchResolution, SentenceParser, SentenceParserOptions, TermEnabledDictionaryMap, Translator };
+//# sourceMappingURL=sentence-parser-BQpu2oSA.d.cts.map

package/dist/sentence-parser-BQpu2oSA.d.cts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sentence-parser-BQpu2oSA.d.cts","names":[],"sources":["../src/types/translation.ts","../src/types/parse.ts","../src/lookup/translator.ts","../src/lookup/sentence-parser.ts"],"sourcesContent":null,"mappings":";;;;KAEY,gBAAA;KAEA,gBAAA;wBACc;EAHd,2BAAgB,EAAA,OAAA;AAE5B,CAAA;AAKY,KAAA,mBAAA,GAAmB;EAKnB,KAAA,EAAA,MAAA;EAAgB,KAAA,EAAA,MAAA;CAAA;AAMM,KANtB,gBAAA,GAMsB;EAAkB,SAE9B,EAPP,kBAOO;EAAyB,SACrB,EAAA,OAAA;EAAwB,cAChB,EAAA,MAAA;EAAG,cACf,EAAA,MAAA;EAAgB,uBAAA,EAAA,MAAA,GAAA,IAAA;EAI1B,4BAAkB,EATI,kBASJ;EAElB,2BAAkB,EAAA,OAAA;EAElB,gBAAA,EAXU,yBAYT;EAID,oBAAA,EAfc,wBAee;EAE7B,4BAAkB,EAhBI,GAgBJ,CAAA,MAAA,CAAA,GAAA,IAAA;EAQlB,gBAAA,EAvBU,gBAuBc;EAAA,QAAA,EAAA,MAAA;CAAA;AAAG,KAnB3B,kBAAA,GAAkB,mBAmBS;AAAG,KAjB9B,kBAAA,GAiB8B,WAAA,GAAA,YAAA;AAE9B,KAjBA,wBAAA,GAiByB;EAAA,OAAA,EAhBxB,MAgBwB;EAAA,WAAe,EAAA,MAAA;CAAmB;AAA5B,KAZ/B,yBAAA,GAY+B,CAZF,wBAYE,EAAA,GAAA,IAAA,CAAA,EAAA;KAV/B,kBAAA;;;ECtCA,sBAAiB,EAAA,OAAA;EAMjB,mBAAgB,EAAA,OAAA;EAMhB,gBAAa,EAAA,OAAA;AAEzB,CAAA;KDgCY,wBAAA,GAA2B,YAAY;KAEvC,yBAAA,GAA4B,YAAY;;;;KChDxC,iBAAA;;;EDAA,OAAA,ECGC,UDHe,EAAA;AAE5B,CAAA;AAKY,KCDA,gBAAA,GDCmB;EAKnB,IAAA,EAAA,MAAA;EAAgB,OAAA,EAAA,MAAA;EAAA,SACb,CAAA,ECJC,iBDID,EAAA,EAAA;CAAkB;AAOX,KCRV,aAAA,GAAgB,gBDQN,EAAA;AACI,KCPd,mBAAA,GDOc;EAAwB,EAAA,EAChB,MAAA;EAAG,MACf,EAAA,iBAAA,GAAA,OAAA;EAAgB,UAAA,EAAA,IAAA,GAAA,MAAA;EAI1B,KAAA,EAAA,MAAA;EAEA,OAAA,ECVC,aDUiB,EAAA;AAE9B,CAAA;;;;AAKA;KEoBK,qBAAA,GAAmB;AFxDZ,KEqGA,aAAA,GFrGgB,OAAA,GAAA,MAAA,GAAA,OAAA,GAAA,QAAA;AAE5B;AAKA;AAKA;AAA4B,cEqOf,UAAA,CFrOe;EAAA,QACb,SAAA;EAAkB,QAKC,yBAAA;EAAkB,QAE9B,SAAA;EAAyB,QACrB,eAAA;EAAwB,QAChB,YAAA;EAAG,QACf,eAAA;EAAgB,QAAA,mBAAA;EAI1B,WAAA,CAAA,QAAA,EE+Nc,YF/NI;EAElB;AAEZ;AAKA;EAEY,OAAA,CAAA,CAAA,EAAA,IAAA;EAQA;;;EAAyD,mBAA9B,CAAA,CAAA,EAAA,IAAA;EAAG;AAE1C;;EAAqC,SAAe,CAAA,IAAA,EE4OtC,aF5OsC,EAAA,IAAA,EAAA,MAAA,EAAA,OAAA,EE4OzB,gBF5OyB,CAAA,EE+O7C,OF/O6C,CAAA;IAAZ,iBAAA,EE+OJ,qBF/OI,EAAA;IAAG,kBAAA,EAAA,MAAA;;;;AChD3C;EAMY,SAAA,CAAA,IAAA,EAAA,MAAgB,EAAA,OAGZ,ECsRF,gBDtRmB,CAAA,EC6WyC,OD7WzC,CC6WsC,oBD7WtC,EAAA,CAAA;EAGrB;AAEZ;;;;IC0CK,IAAA,EAAA,MAAA;IA6CO,OAAA,EAAA,MAAa,GAAA,IAAA;gCAwUlB;;;IA5LM,UAAU,EAAA,MAAA;IAAA,UAAA,EAAA,OAAA;IASG,SAAA,EAAA,MAAA;IAkCZ,YAAA,EAAA,MAAA,GAAA,IAAA;IAAa,kBAAA,EAAA,OAAA;EAEkB,CAAA,EACT,CAAA;EAAmB,QAAhD,kBAAA;EAAO,QAAA,qBAAA;EAuFyD,QAAA,kBAAA;EAA0C,QAAvC,6BAAA;EAAO,QAuD1E,0BAAA;EAAO,QAAA,0BAAA;;;;ECtZG,QAAA,yBAA6B;EAS7B,QAAA,4BAAqB;EAAA,QAAA,0BAAA;EAAA,QAEA,gBAAA;EAA6B,QAAzC,iBAAA;EAAG,QAMN,iBAAA;EAAgB,QAMhB,sBAAA;EAAyB,QAAA,iCAAA;;;;;EAYnC,QAAA,qCAAc;EAAA,QAAA,iCAAA;EAAA,QAGC,6BAAA;EAAU,QAOuB,uBAAA;EAAqB,QAAW,0BAAA;EAAa,QAArB,sBAAA;EAAO,QAAA,gCAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AHpEhF,UGuBK,6BAAA,CHvBW;EAEhB,KAAA,EAAA,MAAA;EAKA,QAAA,CAAA,EAAA,MAAA;EAKA,KAAA,CAAA,EAAA,MAAA;EAAgB,sBAAA,CAAA,EAAA,OAAA;EAAA,mBACb,CAAA,EAAA,OAAA;EAAkB,gBAKC,CAAA,EAAA,OAAA;;AAGR,UGWT,qBAAA,CHXS;EAAwB;EACb,oBACf,EGWI,GHXJ,CAAA,MAAA,EGWgB,6BHXhB,CAAA;EAAgB;EAI1B,UAAA,CAAA,EAAA,MAAA;EAEA;EAEA,SAAA,CAAA,EAAA,MAAA;EAKA;EAEA,gBAAA,CAAA,EGEW,gBHFO;EAQlB;EAAwB,2BAAA,CAAA,EAAA,OAAA;EAAA;EAAiC,SAA9B,CAAA,EAAA,OAAA;EAAG;EAE9B,gBAAA,CAAA,EGFW,yBHEc;;;;AAAM;;cGU9B,cAAA;;EF1DD,WAAA,CAAA,UAAiB,EE6DD,UF1Df;EAGD;AAMZ;AAEA;qDEsD6D,wBAAwB,QAAQ;;;ADpCpC;EAqE7C,QAAA,UAAa;;;;AA4IzB;EAAuB,QAAA,2BAAA;EAAA;;;EA6CsB,QACT,uBAAA"}

package/dist/{sentence-parser-sx7y6Oyh.js → sentence-parser-CXDs4R_c.js}

@@ -1,4 +1,5 @@
-import { distributeFuriganaInflected, isCodePointJapanese } from "./japanese-DhmmW35J.js";
+import { isCodePointJapanese } from "./japanese-OKEGf_i4.js";
+import { codePointPreview, debugYomitanCore } from "./debug-hF63AFkq.js";
 
 //#region src/lookup/sentence-parser.ts
 /**
@@ -16,22 +17,48 @@ var SentenceParser = class {
 	async parseText(text, language, options) {
 		const scanLength = options.scanLength ?? options.maxLength ?? 20;
 		const enabledDictionaryMap = this._buildFindTermDictionaryMap(options.enabledDictionaryMap);
+		debugYomitanCore("sentence-parser", "parseText:start", {
+			textLength: text.length,
+			textPreview: text.slice(0, 120),
+			textCodePoints: codePointPreview(text),
+			lineCount: text.split("\n").length,
+			scanLength,
+			language,
+			enabledDictionaryCount: enabledDictionaryMap.size,
+			enabledDictionaryNames: [...enabledDictionaryMap.keys()],
+			searchResolution: options.searchResolution ?? "letter",
+			removeNonJapaneseCharacters: options.removeNonJapaneseCharacters,
+			deinflect: options.deinflect
+		});
 		const lines = text.split("\n");
 		const results = [];
-		for (const line of lines) {
-			const segments = await this._parseLine(line, language, enabledDictionaryMap, scanLength, options);
-			results.push({ segments });
+		for (const [lineIndex, line] of lines.entries()) {
+			const segments = await this._parseLine(line, lineIndex, language, enabledDictionaryMap, scanLength, options);
+			results.push(segments);
 		}
+		debugYomitanCore("sentence-parser", "parseText:complete", {
+			lineCount: results.length,
+			lineSegmentCounts: results.map((line) => line.length),
+			totalSegments: results.reduce((count, line) => count + line.length, 0),
+			matchedSegments: results.reduce((count, line) => count + line.filter((segment) => Array.isArray(segment.headwords)).length, 0)
+		});
 		return results;
 	}
 	/**
 	* Parses a single line using extension-equivalent scanning parser semantics.
 	*/
-	async _parseLine(line, language, enabledDictionaryMap, scanLength, options) {
+	async _parseLine(line, lineIndex, language, enabledDictionaryMap, scanLength, options) {
 		const findTermsOptions = this._createFindTermsOptions(language, enabledDictionaryMap, options);
 		const cache = new Map();
 		const result = [];
 		let previousUngroupedSegment = null;
+		let lookupCount = 0;
+		let cacheHits = 0;
+		let cacheMisses = 0;
+		let matchedSegmentCount = 0;
+		let unmatchedSegmentCount = 0;
+		let emptyEntryLookups = 0;
+		const lookupMissSamples = [];
 		let i = 0;
 		const ii = line.length;
 		while (i < ii) {
@@ -40,8 +67,18 @@ var SentenceParser = class {
 			const substring = line.substring(i, i + scanLength);
 			let cached = cache.get(substring);
 			if (!cached) {
+				cacheMisses += 1;
 				const { dictionaryEntries, originalTextLength: originalTextLength$1 } = await this._translator.findTerms("simple", substring, findTermsOptions);
+				lookupCount += 1;
 				let segments$1 = [];
+				if (dictionaryEntries.length === 0) {
+					emptyEntryLookups += 1;
+					if (lookupMissSamples.length < 5) lookupMissSamples.push({
+						substring: substring.slice(0, 40),
+						codePoints: codePointPreview(substring, 20),
+						originalTextLength: originalTextLength$1
+					});
+				}
 				if (dictionaryEntries.length > 0 && originalTextLength$1 > 0 && (originalTextLength$1 !== character.length || isCodePointJapanese(codePoint))) {
 					const source = substring.substring(0, originalTextLength$1);
 					segments$1 = this._createMatchedSegments(source, dictionaryEntries);
@@ -51,36 +88,38 @@ var SentenceParser = class {
 					segments: segments$1
 				};
 				cache.set(substring, cached);
-			}
+			} else cacheHits += 1;
 			const { originalTextLength, segments } = cached;
 			if (segments.length > 0) {
 				previousUngroupedSegment = null;
 				result.push(...segments);
+				matchedSegmentCount += segments.length;
 				i += Math.max(1, originalTextLength);
 			} else {
 				if (previousUngroupedSegment === null) {
 					previousUngroupedSegment = {
 						text: character,
-						reading: "",
-						term: character,
-						entries: [],
-						furigana: [{
-							text: character,
-							reading: ""
-						}]
+						reading: ""
 					};
 					result.push(previousUngroupedSegment);
-				} else {
-					previousUngroupedSegment.text += character;
-					previousUngroupedSegment.term = previousUngroupedSegment.text;
-					previousUngroupedSegment.furigana = [{
-						text: previousUngroupedSegment.text,
-						reading: ""
-					}];
-				}
+					unmatchedSegmentCount += 1;
+				} else previousUngroupedSegment.text += character;
 				i += character.length;
 			}
 		}
+		debugYomitanCore("sentence-parser", "parseLine:complete", {
+			lineIndex,
+			lineLength: line.length,
+			linePreview: line.slice(0, 120),
+			lookupCount,
+			cacheHits,
+			cacheMisses,
+			emptyEntryLookups,
+			matchedSegmentCount,
+			unmatchedSegmentCount,
+			finalSegmentCount: result.length,
+			missSamples: lookupMissSamples
+		});
 		return result;
 	}
 	_createMatchedSegments(source, dictionaryEntries) {
@@ -88,35 +127,24 @@ var SentenceParser = class {
 		const firstEntry = dictionaryEntries[0];
 		const firstHeadword = firstEntry.headwords[0];
 		if (!firstHeadword) return [];
-		const reading = firstHeadword.reading ?? "";
-		const term = firstHeadword.term ?? source;
-		const furigana = distributeFuriganaInflected(term, reading, source);
-		const filteredEntries = this._trimEntriesForSource(dictionaryEntries, source);
-		return [{
-			text: source,
-			reading,
-			term,
-			entries: filteredEntries,
-			furigana
-		}];
-	}
-	_trimEntriesForSource(dictionaryEntries, source) {
-		const trimmed = [];
-		for (const entry of dictionaryEntries) {
-			const headwords = [];
-			for (const headword of entry.headwords) {
+		const trimmedHeadwords = [];
+		for (const dictionaryEntry of dictionaryEntries) {
+			const validHeadwords = [];
+			for (const headword of dictionaryEntry.headwords) {
 				const validSources = headword.sources.filter((src) => src.originalText === source && src.isPrimary && src.matchType === "exact");
-				if (validSources.length > 0) headwords.push({
-					...headword,
+				if (validSources.length > 0) validHeadwords.push({
+					term: headword.term,
+					reading: headword.reading,
 					sources: validSources
 				});
 			}
-			if (headwords.length > 0) trimmed.push({
-				...entry,
-				headwords
-			});
+			if (validHeadwords.length > 0) trimmedHeadwords.push(validHeadwords);
 		}
-		return trimmed.length > 0 ? trimmed : dictionaryEntries;
+		return [{
+			text: source,
+			reading: firstHeadword.reading ?? "",
+			...trimmedHeadwords.length > 0 ? { headwords: trimmedHeadwords } : {}
+		}];
 	}
 	/**
 	* Builds a FindTermDictionary map from the input format.
@@ -155,4 +183,4 @@ var SentenceParser = class {
 
 //#endregion
 export { SentenceParser };
-//# sourceMappingURL=sentence-parser-sx7y6Oyh.js.map
+//# sourceMappingURL=sentence-parser-CXDs4R_c.js.map

package/dist/sentence-parser-CXDs4R_c.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sentence-parser-CXDs4R_c.js","names":["translator: Translator","text: string","language: string","options: SentenceParserOptions","results: ParseTextLine[]","line: string","lineIndex: number","enabledDictionaryMap: Map<string, FindTermDictionary>","scanLength: number","result: ParseTextSegment[]","previousUngroupedSegment: ParseTextSegment | null","lookupMissSamples: Array<{ substring: string; codePoints: string[]; originalTextLength: number }>","segments: ParseTextSegment[]","originalTextLength","source: string","dictionaryEntries: TermDictionaryEntry[]","trimmedHeadwords: ParseTextHeadword[][]","validHeadwords: ParseTextHeadword[]","inputMap: Map<string, SentenceParserDictionaryInput>"],"sources":["../src/lookup/sentence-parser.ts"],"sourcesContent":["/*\n * Copyright (C) 2023-2025  Yomitan Authors\n *\n * This program is free software: you can redistribute it and/or modify\n * it under the terms of the GNU General Public License as published by\n * the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n * GNU General Public License for more details.\n *\n * You should have received a copy of the GNU General Public License\n * along with this program.  If not, see <https://www.gnu.org/licenses/>.\n */\n\nimport { isCodePointJapanese } from '../language/ja/japanese';\nimport type { TermDictionaryEntry } from '../types/dictionary';\nimport type { ParseTextHeadword, ParseTextLine, ParseTextSegment } from '../types/parse';\nimport type { FindTermDictionary, FindTermsTextReplacements, SearchResolution } from '../types/translation';\nimport { codePointPreview, debugYomitanCore } from '../util/debug';\nimport type { Translator } from './translator';\nexport type { ParseTextHeadword, ParseTextLine, ParseTextResultItem, ParseTextSegment } from '../types/parse';\n\nexport interface SentenceParserDictionaryInput {\n    index: number;\n    priority?: number;\n    alias?: string;\n    allowSecondarySearches?: boolean;\n    partsOfSpeechFilter?: boolean;\n    useDeinflections?: boolean;\n}\n\nexport interface SentenceParserOptions {\n    /** The enabled dictionary map for lookups. */\n    enabledDictionaryMap: Map<string, SentenceParserDictionaryInput>;\n    /** Window size used by scanning parser. Defaults to 20. */\n    scanLength?: number;\n    /** Backward compatibility alias for scanLength. */\n    maxLength?: number;\n    /** Search resolution for translator. */\n    searchResolution?: SearchResolution;\n    /** Whether to remove non-Japanese characters in lookups. */\n    removeNonJapaneseCharacters?: boolean;\n    /** Whether to apply deinflection. */\n    deinflect?: boolean;\n    /** Text replacements forwarded to translator. */\n    textReplacements?: FindTermsTextReplacements;\n}\n\ninterface ParseScanCacheEntry {\n    originalTextLength: number;\n    segments: ParseTextSegment[];\n}\n\n/**\n * Parses text into segments with dictionary lookups using the same\n * scanning-parser model as Yomitan extension (non-MeCab path).\n */\nexport class SentenceParser {\n    private _translator: Translator;\n\n    constructor(translator: Translator) {\n        this._translator = translator;\n    }\n\n    /**\n     * Parses the given text into lines of parsed segments.\n     */\n    async parseText(text: string, language: string, options: SentenceParserOptions): Promise<ParseTextLine[]> {\n        const scanLength = options.scanLength ?? options.maxLength ?? 20;\n        const enabledDictionaryMap = this._buildFindTermDictionaryMap(options.enabledDictionaryMap);\n        debugYomitanCore('sentence-parser', 'parseText:start', {\n            textLength: text.length,\n            textPreview: text.slice(0, 120),\n            textCodePoints: codePointPreview(text),\n            lineCount: text.split('\\n').length,\n            scanLength,\n            language,\n            enabledDictionaryCount: enabledDictionaryMap.size,\n            enabledDictionaryNames: [...enabledDictionaryMap.keys()],\n            searchResolution: options.searchResolution ?? 'letter',\n            removeNonJapaneseCharacters: options.removeNonJapaneseCharacters,\n            deinflect: options.deinflect,\n        });\n\n        const lines = text.split('\\n');\n        const results: ParseTextLine[] = [];\n\n        for (const [lineIndex, line] of lines.entries()) {\n            const segments = await this._parseLine(line, lineIndex, language, enabledDictionaryMap, scanLength, options);\n            results.push(segments);\n        }\n\n        debugYomitanCore('sentence-parser', 'parseText:complete', {\n            lineCount: results.length,\n            lineSegmentCounts: results.map((line) => line.length),\n            totalSegments: results.reduce((count, line) => count + line.length, 0),\n            matchedSegments: results.reduce(\n                (count, line) => count + line.filter((segment) => Array.isArray(segment.headwords)).length,\n                0,\n            ),\n        });\n\n        return results;\n    }\n\n    /**\n     * Parses a single line using extension-equivalent scanning parser semantics.\n     */\n    private async _parseLine(\n        line: string,\n        lineIndex: number,\n        language: string,\n        enabledDictionaryMap: Map<string, FindTermDictionary>,\n        scanLength: number,\n        options: SentenceParserOptions,\n    ): Promise<ParseTextLine> {\n        const findTermsOptions = this._createFindTermsOptions(language, enabledDictionaryMap, options);\n        const cache = new Map<string, ParseScanCacheEntry>();\n\n        const result: ParseTextSegment[] = [];\n        let previousUngroupedSegment: ParseTextSegment | null = null;\n        let lookupCount = 0;\n        let cacheHits = 0;\n        let cacheMisses = 0;\n        let matchedSegmentCount = 0;\n        let unmatchedSegmentCount = 0;\n        let emptyEntryLookups = 0;\n        const lookupMissSamples: Array<{ substring: string; codePoints: string[]; originalTextLength: number }> = [];\n\n        let i = 0;\n        const ii = line.length;\n        while (i < ii) {\n            const codePoint = line.codePointAt(i) as number;\n            const character = String.fromCodePoint(codePoint);\n            const substring = line.substring(i, i + scanLength);\n\n            let cached = cache.get(substring);\n            if (!cached) {\n                cacheMisses += 1;\n                const { dictionaryEntries, originalTextLength } = await this._translator.findTerms(\n                    'simple',\n                    substring,\n                    findTermsOptions,\n                );\n                lookupCount += 1;\n\n                let segments: ParseTextSegment[] = [];\n                if (dictionaryEntries.length === 0) {\n                    emptyEntryLookups += 1;\n                    if (lookupMissSamples.length < 5) {\n                        lookupMissSamples.push({\n                            substring: substring.slice(0, 40),\n                            codePoints: codePointPreview(substring, 20),\n                            originalTextLength,\n                        });\n                    }\n                }\n\n                if (\n                    dictionaryEntries.length > 0 &&\n                    originalTextLength > 0 &&\n                    (originalTextLength !== character.length || isCodePointJapanese(codePoint))\n                ) {\n                    const source = substring.substring(0, originalTextLength);\n                    segments = this._createMatchedSegments(source, dictionaryEntries);\n                }\n\n                cached = {\n                    originalTextLength,\n                    segments,\n                };\n                cache.set(substring, cached);\n            } else {\n                cacheHits += 1;\n            }\n\n            const { originalTextLength, segments } = cached;\n            if (segments.length > 0) {\n                previousUngroupedSegment = null;\n                result.push(...segments);\n                matchedSegmentCount += segments.length;\n                i += Math.max(1, originalTextLength);\n            } else {\n                if (previousUngroupedSegment === null) {\n                    previousUngroupedSegment = {\n                        text: character,\n                        reading: ''\n                    };\n                    result.push(previousUngroupedSegment);\n                    unmatchedSegmentCount += 1;\n                } else {\n                    previousUngroupedSegment.text += character;\n                }\n                i += character.length;\n            }\n        }\n\n        debugYomitanCore('sentence-parser', 'parseLine:complete', {\n            lineIndex,\n            lineLength: line.length,\n            linePreview: line.slice(0, 120),\n            lookupCount,\n            cacheHits,\n            cacheMisses,\n            emptyEntryLookups,\n            matchedSegmentCount,\n            unmatchedSegmentCount,\n            finalSegmentCount: result.length,\n            missSamples: lookupMissSamples,\n        });\n\n        return result;\n    }\n\n    private _createMatchedSegments(source: string, dictionaryEntries: TermDictionaryEntry[]): ParseTextSegment[] {\n        if (dictionaryEntries.length === 0) {\n            return [];\n        }\n\n        const firstEntry = dictionaryEntries[0];\n        const firstHeadword = firstEntry.headwords[0];\n        if (!firstHeadword) {\n            return [];\n        }\n\n        const trimmedHeadwords: ParseTextHeadword[][] = [];\n        for (const dictionaryEntry of dictionaryEntries) {\n            const validHeadwords: ParseTextHeadword[] = [];\n            for (const headword of dictionaryEntry.headwords) {\n                const validSources = headword.sources.filter(\n                    (src) => src.originalText === source && src.isPrimary && src.matchType === 'exact',\n                );\n\n                if (validSources.length > 0) {\n                    validHeadwords.push({\n                        term: headword.term,\n                        reading: headword.reading,\n                        sources: validSources,\n                    });\n                }\n            }\n            if (validHeadwords.length > 0) {\n                trimmedHeadwords.push(validHeadwords);\n            }\n        }\n\n        return [\n            {\n                text: source,\n                reading: firstHeadword.reading ?? '',\n                ...(trimmedHeadwords.length > 0 ? { headwords: trimmedHeadwords } : {}),\n            },\n        ];\n    }\n\n    /**\n     * Builds a FindTermDictionary map from the input format.\n     */\n    private _buildFindTermDictionaryMap(\n        inputMap: Map<string, SentenceParserDictionaryInput>,\n    ): Map<string, FindTermDictionary> {\n        const result = new Map<string, FindTermDictionary>();\n\n        for (const [name, config] of inputMap.entries()) {\n            result.set(name, {\n                index: config.index,\n                alias: config.alias ?? name,\n                allowSecondarySearches: config.allowSecondarySearches ?? false,\n                partsOfSpeechFilter: config.partsOfSpeechFilter ?? true,\n                useDeinflections: config.useDeinflections ?? true,\n            });\n        }\n\n        return result;\n    }\n\n    /**\n     * Creates a FindTermsOptions object for scanning parser lookups.\n     */\n    private _createFindTermsOptions(\n        language: string,\n        enabledDictionaryMap: Map<string, FindTermDictionary>,\n        options: SentenceParserOptions,\n    ) {\n        return {\n            matchType: 'exact' as const,\n            deinflect: options.deinflect ?? true,\n            primaryReading: '',\n            mainDictionary: '',\n            sortFrequencyDictionary: null,\n            sortFrequencyDictionaryOrder: 'descending' as const,\n            removeNonJapaneseCharacters:\n                options.removeNonJapaneseCharacters ??\n                (language === 'ja' || language === 'zh' || language === 'yue' || language === 'ko'),\n            textReplacements: options.textReplacements ?? [null],\n            enabledDictionaryMap,\n            excludeDictionaryDefinitions: null,\n            searchResolution: options.searchResolution ?? ('letter' as const),\n            language,\n        };\n    }\n}\n"],"mappings":";;;;;;;;AA4DA,IAAa,iBAAb,MAA4B;CACxB,AAAQ;CAER,YAAYA,YAAwB;AAChC,OAAK,cAAc;CACtB;;;;CAKD,MAAM,UAAUC,MAAcC,UAAkBC,SAA0D;EACtG,MAAM,aAAa,QAAQ,cAAc,QAAQ,aAAa;EAC9D,MAAM,uBAAuB,KAAK,4BAA4B,QAAQ,qBAAqB;AAC3F,mBAAiB,mBAAmB,mBAAmB;GACnD,YAAY,KAAK;GACjB,aAAa,KAAK,MAAM,GAAG,IAAI;GAC/B,gBAAgB,iBAAiB,KAAK;GACtC,WAAW,KAAK,MAAM,KAAK,CAAC;GAC5B;GACA;GACA,wBAAwB,qBAAqB;GAC7C,wBAAwB,CAAC,GAAG,qBAAqB,MAAM,AAAC;GACxD,kBAAkB,QAAQ,oBAAoB;GAC9C,6BAA6B,QAAQ;GACrC,WAAW,QAAQ;EACtB,EAAC;EAEF,MAAM,QAAQ,KAAK,MAAM,KAAK;EAC9B,MAAMC,UAA2B,CAAE;AAEnC,OAAK,MAAM,CAAC,WAAW,KAAK,IAAI,MAAM,SAAS,EAAE;GAC7C,MAAM,WAAW,MAAM,KAAK,WAAW,MAAM,WAAW,UAAU,sBAAsB,YAAY,QAAQ;AAC5G,WAAQ,KAAK,SAAS;EACzB;AAED,mBAAiB,mBAAmB,sBAAsB;GACtD,WAAW,QAAQ;GACnB,mBAAmB,QAAQ,IAAI,CAAC,SAAS,KAAK,OAAO;GACrD,eAAe,QAAQ,OAAO,CAAC,OAAO,SAAS,QAAQ,KAAK,QAAQ,EAAE;GACtE,iBAAiB,QAAQ,OACrB,CAAC,OAAO,SAAS,QAAQ,KAAK,OAAO,CAAC,YAAY,MAAM,QAAQ,QAAQ,UAAU,CAAC,CAAC,QACpF,EACH;EACJ,EAAC;AAEF,SAAO;CACV;;;;CAKD,MAAc,WACVC,MACAC,WACAJ,UACAK,sBACAC,YACAL,SACsB;EACtB,MAAM,mBAAmB,KAAK,wBAAwB,UAAU,sBAAsB,QAAQ;EAC9F,MAAM,QAAQ,IAAI;EAElB,MAAMM,SAA6B,CAAE;EACrC,IAAIC,2BAAoD;EACxD,IAAI,cAAc;EAClB,IAAI,YAAY;EAChB,IAAI,cAAc;EAClB,IAAI,sBAAsB;EAC1B,IAAI,wBAAwB;EAC5B,IAAI,oBAAoB;EACxB,MAAMC,oBAAoG,CAAE;EAE5G,IAAI,IAAI;EACR,MAAM,KAAK,KAAK;AAChB,SAAO,IAAI,IAAI;GACX,MAAM,YAAY,KAAK,YAAY,EAAE;GACrC,MAAM,YAAY,OAAO,cAAc,UAAU;GACjD,MAAM,YAAY,KAAK,UAAU,GAAG,IAAI,WAAW;GAEnD,IAAI,SAAS,MAAM,IAAI,UAAU;AACjC,QAAK,QAAQ;AACT,mBAAe;IACf,MAAM,EAAE,mBAAmB,0CAAoB,GAAG,MAAM,KAAK,YAAY,UACrE,UACA,WACA,iBACH;AACD,mBAAe;IAEf,IAAIC,aAA+B,CAAE;AACrC,QAAI,kBAAkB,WAAW,GAAG;AAChC,0BAAqB;AACrB,SAAI,kBAAkB,SAAS,EAC3B,mBAAkB,KAAK;MACnB,WAAW,UAAU,MAAM,GAAG,GAAG;MACjC,YAAY,iBAAiB,WAAW,GAAG;MAC3C;KACH,EAAC;IAET;AAED,QACI,kBAAkB,SAAS,KAC3BC,uBAAqB,MACpBA,yBAAuB,UAAU,UAAU,oBAAoB,UAAU,GAC5E;KACE,MAAM,SAAS,UAAU,UAAU,GAAGA,qBAAmB;AACzD,kBAAW,KAAK,uBAAuB,QAAQ,kBAAkB;IACpE;AAED,aAAS;KACL;KACA;IACH;AACD,UAAM,IAAI,WAAW,OAAO;GAC/B,MACG,cAAa;GAGjB,MAAM,EAAE,oBAAoB,UAAU,GAAG;AACzC,OAAI,SAAS,SAAS,GAAG;AACrB,+BAA2B;AAC3B,WAAO,KAAK,GAAG,SAAS;AACxB,2BAAuB,SAAS;AAChC,SAAK,KAAK,IAAI,GAAG,mBAAmB;GACvC,OAAM;AACH,QAAI,6BAA6B,MAAM;AACnC,gCAA2B;MACvB,MAAM;MACN,SAAS;KACZ;AACD,YAAO,KAAK,yBAAyB;AACrC,8BAAyB;IAC5B,MACG,0BAAyB,QAAQ;AAErC,SAAK,UAAU;GAClB;EACJ;AAED,mBAAiB,mBAAmB,sBAAsB;GACtD;GACA,YAAY,KAAK;GACjB,aAAa,KAAK,MAAM,GAAG,IAAI;GAC/B;GACA;GACA;GACA;GACA;GACA;GACA,mBAAmB,OAAO;GAC1B,aAAa;EAChB,EAAC;AAEF,SAAO;CACV;CAED,AAAQ,uBAAuBC,QAAgBC,mBAA8D;AACzG,MAAI,kBAAkB,WAAW,EAC7B,QAAO,CAAE;EAGb,MAAM,aAAa,kBAAkB;EACrC,MAAM,gBAAgB,WAAW,UAAU;AAC3C,OAAK,cACD,QAAO,CAAE;EAGb,MAAMC,mBAA0C,CAAE;AAClD,OAAK,MAAM,mBAAmB,mBAAmB;GAC7C,MAAMC,iBAAsC,CAAE;AAC9C,QAAK,MAAM,YAAY,gBAAgB,WAAW;IAC9C,MAAM,eAAe,SAAS,QAAQ,OAClC,CAAC,QAAQ,IAAI,iBAAiB,UAAU,IAAI,aAAa,IAAI,cAAc,QAC9E;AAED,QAAI,aAAa,SAAS,EACtB,gBAAe,KAAK;KAChB,MAAM,SAAS;KACf,SAAS,SAAS;KAClB,SAAS;IACZ,EAAC;GAET;AACD,OAAI,eAAe,SAAS,EACxB,kBAAiB,KAAK,eAAe;EAE5C;AAED,SAAO,CACH;GACI,MAAM;GACN,SAAS,cAAc,WAAW;GAClC,GAAI,iBAAiB,SAAS,IAAI,EAAE,WAAW,iBAAkB,IAAG,CAAE;EACzE,CACJ;CACJ;;;;CAKD,AAAQ,4BACJC,UAC+B;EAC/B,MAAM,SAAS,IAAI;AAEnB,OAAK,MAAM,CAAC,MAAM,OAAO,IAAI,SAAS,SAAS,CAC3C,QAAO,IAAI,MAAM;GACb,OAAO,OAAO;GACd,OAAO,OAAO,SAAS;GACvB,wBAAwB,OAAO,0BAA0B;GACzD,qBAAqB,OAAO,uBAAuB;GACnD,kBAAkB,OAAO,oBAAoB;EAChD,EAAC;AAGN,SAAO;CACV;;;;CAKD,AAAQ,wBACJhB,UACAK,sBACAJ,SACF;AACE,SAAO;GACH,WAAW;GACX,WAAW,QAAQ,aAAa;GAChC,gBAAgB;GAChB,gBAAgB;GAChB,yBAAyB;GACzB,8BAA8B;GAC9B,6BACI,QAAQ,gCACP,aAAa,QAAQ,aAAa,QAAQ,aAAa,SAAS,aAAa;GAClF,kBAAkB,QAAQ,oBAAoB,CAAC,IAAK;GACpD;GACA,8BAA8B;GAC9B,kBAAkB,QAAQ,oBAAqB;GAC/C;EACH;CACJ;AACJ"}

package/dist/{translator-CYvGmODX.d.ts → sentence-parser-CmsHUc_p.d.ts}

@@ -1,5 +1,5 @@
-import { KanjiDictionaryEntry, TermDictionaryEntry, TermSourceMatchType } from "./dictionary-CfzEFbXZ.js";
-import { DictionaryDB$1 as DictionaryDB } from "./dictionary-database-BRAqOF_M.js";
+import { KanjiDictionaryEntry, TermDictionaryEntry, TermSource, TermSourceMatchType } from "./dictionary-DdxghSJP.js";
+import { DictionaryDB$1 as DictionaryDB } from "./dictionary-database-D0lvK2n3.js";
 
 //#region src/types/translation.d.ts
 type SearchResolution = 'letter' | 'word';
@@ -41,9 +41,31 @@ type FindTermDictionary = {
 };
 type TermEnabledDictionaryMap = Map<string, FindTermDictionary>;
 type KanjiEnabledDictionaryMap = Map<string, FindKanjiDictionary>; //#endregion
-//#region src/lookup/translator.d.ts
+//#region src/types/parse.d.ts
 
 //# sourceMappingURL=translation.d.ts.map
+type ParseTextHeadword = {
+  term: string;
+  reading: string;
+  sources: TermSource[];
+};
+type ParseTextSegment = {
+  text: string;
+  reading: string;
+  headwords?: ParseTextHeadword[][];
+};
+type ParseTextLine = ParseTextSegment[];
+type ParseTextResultItem = {
+  id: string;
+  source: 'scanning-parser' | 'mecab';
+  dictionary: null | string;
+  index: number;
+  content: ParseTextLine[];
+};
+
+//#endregion
+//#region src/lookup/translator.d.ts
+//# sourceMappingURL=parse.d.ts.map
 type TermDictionaryEntry$1 = TermDictionaryEntry;
 type FindTermsMode = 'group' | 'term' | 'merge' | 'simple';
 /**
@@ -176,5 +198,59 @@ declare class Translator {
   private _addUserFacingInflections;
   private _hasAny;
 } //#endregion
-export { FindKanjiDictionary, FindKanjiOptions, FindTermDictionary, FindTermsMatchType, FindTermsMode, FindTermsOptions, FindTermsSortOrder, FindTermsTextReplacement, FindTermsTextReplacements, KanjiEnabledDictionaryMap, SearchResolution, TermEnabledDictionaryMap, Translator as Translator$1 };
-//# sourceMappingURL=translator-CYvGmODX.d.ts.map
+//#region src/lookup/sentence-parser.d.ts
+interface SentenceParserDictionaryInput {
+  index: number;
+  priority?: number;
+  alias?: string;
+  allowSecondarySearches?: boolean;
+  partsOfSpeechFilter?: boolean;
+  useDeinflections?: boolean;
+}
+interface SentenceParserOptions {
+  /** The enabled dictionary map for lookups. */
+  enabledDictionaryMap: Map<string, SentenceParserDictionaryInput>;
+  /** Window size used by scanning parser. Defaults to 20. */
+  scanLength?: number;
+  /** Backward compatibility alias for scanLength. */
+  maxLength?: number;
+  /** Search resolution for translator. */
+  searchResolution?: SearchResolution;
+  /** Whether to remove non-Japanese characters in lookups. */
+  removeNonJapaneseCharacters?: boolean;
+  /** Whether to apply deinflection. */
+  deinflect?: boolean;
+  /** Text replacements forwarded to translator. */
+  textReplacements?: FindTermsTextReplacements;
+}
+/**
+ * Parses text into segments with dictionary lookups using the same
+ * scanning-parser model as Yomitan extension (non-MeCab path).
+ */
+declare class SentenceParser {
+  private _translator;
+  constructor(translator: Translator);
+  /**
+   * Parses the given text into lines of parsed segments.
+   */
+  parseText(text: string, language: string, options: SentenceParserOptions): Promise<ParseTextLine[]>;
+  /**
+   * Parses a single line using extension-equivalent scanning parser semantics.
+   */
+  private _parseLine;
+  private _createMatchedSegments;
+  /**
+   * Builds a FindTermDictionary map from the input format.
+   */
+  private _buildFindTermDictionaryMap;
+  /**
+   * Creates a FindTermsOptions object for scanning parser lookups.
+   */
+  private _createFindTermsOptions;
+}
+
+//#endregion
+//# sourceMappingURL=sentence-parser.d.ts.map
+
+export { FindKanjiDictionary, FindKanjiOptions, FindTermDictionary, FindTermsMatchType, FindTermsMode, FindTermsOptions, FindTermsSortOrder, FindTermsTextReplacement, FindTermsTextReplacements, KanjiEnabledDictionaryMap, ParseTextHeadword, ParseTextLine, ParseTextResultItem, ParseTextSegment, SearchResolution, SentenceParser as SentenceParser$1, SentenceParserOptions, TermEnabledDictionaryMap, Translator as Translator$1 };
+//# sourceMappingURL=sentence-parser-CmsHUc_p.d.ts.map