sudachi-ts 0.1.18 → 0.1.20-beta.0

package/README.md

@@ -191,13 +191,17 @@ import { SentenceDetector } from 'sudachi-ts/sentdetect/sentenceDetector.js';
 
 const sentences = tokenizer.tokenizeSentences('東京都は日本の首都です。大阪は商業都市です。');
 
-for (const sentence of sentences) {
-  console.log('--- Sentence ---');
-  for (const morpheme of sentence) {
-    console.log(morpheme.surface());
-  }
-}
-```
+for (const sentence of sentences) {
+  console.log('--- Sentence ---');
+  for (const morpheme of sentence) {
+    console.log(morpheme.surface());
+  }
+}
+```
+
+`tokenizeSentences(...)` treats quoted dialogue endings (for example `「...！」` and
+`「...。」`) as sentence boundaries and skips leading inter-sentence whitespace such
+as newlines before tokenization.
 
 Lazy sentence processing for streaming:
 
@@ -281,6 +285,9 @@ readings is visible in the baseline vs plugin outputs.
 `TokenChunkerPlugin` is designed and validated against the full Sudachi system
 dictionary (`system_full.dic` / `system.dic`), so prefer full-dictionary checks
 when adding or tuning chunk rules.
+`TokenChunkerPlugin` requires `enableDefaultCompoundParticles: true`. Dictionary
+creation throws an error when this plugin is configured with default compound
+particles disabled.
 `TokenChunkerPlugin` is intended for `SplitMode.C` tokenization; calling
 `tokenize(SplitMode.A, ...)` or `tokenize(SplitMode.B, ...)` with this plugin
 enabled throws an error.

package/build/src/core/japaneseTokenizer.js

@@ -7,6 +7,7 @@ import { MorphemeList as MorphemeListImpl } from './morphemeList.js';
 import { SentenceSplittingLazyAnalysis } from './sentenceSplittingLazyAnalysis.js';
 import { SplitMode } from './tokenizer.js';
 import { UTF8InputTextBuilder } from './utf8InputText.js';
+const LEADING_WHITESPACE_PATTERN = /^\s+/u;
 export class JapaneseTokenizer {
     grammar;
     lexicon;
@@ -57,6 +58,13 @@ export class JapaneseTokenizer {
         const sentences = [];
         let remaining = inputText;
         while (remaining.length > 0) {
+            const leadingWhitespace = remaining.match(LEADING_WHITESPACE_PATTERN);
+            if (leadingWhitespace) {
+                remaining = remaining.slice(leadingWhitespace[0].length);
+                if (remaining.length === 0) {
+                    break;
+                }
+            }
             const checker = {
                 hasNonBreakWord: (eos) => {
                     const bytes = this.buildInputText(remaining.slice(0, eos)).getByteText();

package/build/src/core/sentenceSplittingLazyAnalysis.js

@@ -1,6 +1,7 @@
 import { SentenceDetector, } from '../sentdetect/sentenceDetector.js';
 import { UTF8InputTextBuilder } from './utf8InputText.js';
 const BUFFER_SIZE = 4096;
+const LEADING_WHITESPACE_PATTERN = /^\s+/u;
 export class SentenceSplittingLazyAnalysis {
     mode;
     grammar;
@@ -68,6 +69,15 @@ export class SentenceSplittingLazyAnalysis {
         return this.buffer.length;
     }
     processNextSentence() {
+        const leadingWhitespace = this.normalized.match(LEADING_WHITESPACE_PATTERN);
+        if (leadingWhitespace) {
+            const skipped = leadingWhitespace[0].length;
+            this.bos += skipped;
+            this.normalized = this.normalized.slice(skipped);
+            if (this.normalized.length === 0) {
+                return null;
+            }
+        }
         const detector = new SentenceDetector();
         const eosLength = detector.getEos(this.normalized, this);
         if (eosLength > 0) {

package/build/src/dictionary/dictionaryFactory.js

@@ -6,6 +6,12 @@ import { PluginLoader } from '../plugins/loader.js';
 import { BinaryDictionary } from './binaryDictionary.js';
 import { loadDefaultCompoundLexicon } from './defaultCompoundLexicon.js';
 import { LexiconSet } from './lexiconSet.js';
+function isTokenChunkerPlugin(className) {
+    if (className === 'com.worksap.nlp.sudachi.TokenChunkerPlugin') {
+        return true;
+    }
+    return className.split('.').pop() === 'TokenChunkerPlugin';
+}
 export class DictionaryFactory {
     async create(configPath, customConfig) {
         const config = customConfig || (await loadConfig(configPath));
@@ -76,6 +82,10 @@ export class DictionaryFactory {
         if (!pathRewritePluginConfs || pathRewritePluginConfs.length === 0) {
             pathRewritePluginConfs = defaultConfig.getPlugins('pathRewritePlugin');
         }
+        if (!enableDefaultCompoundParticles &&
+            (pathRewritePluginConfs || []).some((conf) => isTokenChunkerPlugin(conf.className))) {
+            throw new Error('TokenChunkerPlugin is only compatible when enableDefaultCompoundParticles is true.');
+        }
         const pathRewritePlugins = (await loader.loadPathRewritePlugins(pathRewritePluginConfs || [], grammar)).map((p) => p.plugin);
         return new Dictionary(grammar, lexicon, inputTextPlugins, oovProviderPlugins, pathRewritePlugins);
     }

package/build/src/plugins/pathRewrite/tokenChunkerPlugin.d.ts

@@ -28,6 +28,8 @@ export declare class TokenChunkerPlugin extends PathRewritePlugin {
     private isCounterChunk;
     private mergeCounterChunks;
     private applyMergeStage;
+    private shouldMergeAttributiveKana;
+    private shouldMergeSmallKana;
     private applyInlineRubyExactStage;
     private applyInlineRubyPrefixStage;
     private shouldMergeInlineRubyExact;

package/build/src/plugins/pathRewrite/tokenChunkerPlugin.js

@@ -294,10 +294,21 @@ export class TokenChunkerPlugin extends PathRewritePlugin {
         while (i < chunks.length - 1) {
             const current = chunks[i];
             const next = chunks[i + 1];
+            const following = chunks[i + 2];
             if (!current || !next) {
                 i++;
                 continue;
             }
+            if (this.shouldMergeSmallKana(current, next)) {
+                const merged = this.mergeChunks([current, next], 'fixed_expression');
+                chunks.splice(i, 2, merged);
+                continue;
+            }
+            if (this.shouldMergeAttributiveKana(current, next, following)) {
+                const merged = this.mergeChunks([current, next], 'phrase');
+                chunks.splice(i, 2, merged);
+                continue;
+            }
             if ((current.chunkType === 'te_form' ||
                 current.chunkType === 'suru_verb_te_form') &&
                 (next.chunkType === 'single_token' ||
@@ -312,7 +323,10 @@ export class TokenChunkerPlugin extends PathRewritePlugin {
                 continue;
             }
             if (current.surface.endsWith('んだ') &&
-                (next.surface === 'よ' || next.surface === 'よっ')) {
+                (next.surface === 'よ' ||
+                    next.surface === 'よっ' ||
+                    next.surface === 'ね' ||
+                    next.surface === 'ねっ')) {
                 const merged = this.mergeChunks([current, next], 'fixed_expression');
                 chunks.splice(i, 2, merged);
                 continue;
@@ -321,6 +335,32 @@ export class TokenChunkerPlugin extends PathRewritePlugin {
         }
         return chunks;
     }
+    shouldMergeAttributiveKana(current, next, following) {
+        if (next.surface !== 'かな') {
+            return false;
+        }
+        const nextPos = this.getPosById(next.posId);
+        if (!nextPos || nextPos[0] !== '助詞' || nextPos[1] !== '終助詞') {
+            return false;
+        }
+        const currentPos0 = this.getPosById(current.posId)?.[0] ?? '';
+        if (!['名詞', '形状詞'].includes(currentPos0)) {
+            return false;
+        }
+        if (!following) {
+            return false;
+        }
+        const followingPos0 = this.getPosById(following.posId)?.[0] ?? '';
+        return ['名詞', '代名詞'].includes(followingPos0);
+    }
+    shouldMergeSmallKana(current, next) {
+        if (current.chunkType !== 'single_token' ||
+            next.chunkType !== 'single_token') {
+            return false;
+        }
+        return (SINGLE_KANA_PATTERN.test(current.surface) &&
+            SMALL_KANA_SUFFIXES.has(next.surface));
+    }
     applyInlineRubyExactStage(source) {
         const chunks = [...source];
         let i = 0;
@@ -486,6 +526,8 @@ const KANJI_PATTERN = /\p{Script=Han}/u;
 const KANJI_ONLY_PATTERN = /^[\p{Script=Han}々〆ヵヶ]+$/u;
 const KANA_PATTERN = /^[ぁ-ゖァ-ヺー]+$/u;
 const HIRAGANA_PATTERN = /^[ぁ-ゖー]+$/u;
+const SINGLE_KANA_PATTERN = /^[ぁ-ゖァ-ヺー]$/u;
+const SMALL_KANA_SUFFIXES = new Set(['ゃ', 'ゅ', 'ょ', 'ャ', 'ュ', 'ョ']);
 const COUNTER_WORDS = new Set([
     '本',
     '匹',
@@ -683,6 +725,25 @@ const COLLOQUIAL_SEQUENCE_RULES = [
             { surface: 'た', pos0: '助動詞' },
         ],
     },
+    {
+        name: 'copula_desu_past_split',
+        priority: 101,
+        resultType: 'phrase',
+        pattern: [
+            { surface: 'でし', dictionaryForm: 'です', pos0: '助動詞' },
+            { surface: 'た', pos0: '助動詞' },
+        ],
+    },
+    {
+        name: 'verb_masen',
+        priority: 101,
+        resultType: 'phrase',
+        pattern: [
+            { pos0: '動詞' },
+            { surface: 'ませ', dictionaryForm: 'ます', pos0: '助動詞' },
+            { surface: 'ん', pos0: '助動詞' },
+        ],
+    },
     {
         name: 'noun_suru_past',
         priority: 101,
@@ -1338,6 +1399,12 @@ const COLLOQUIAL_SEQUENCE_RULES = [
             { surface: 'いい' },
         ],
     },
+    {
+        name: 'nakereba',
+        priority: 95,
+        resultType: 'phrase',
+        pattern: [{ surface: 'なけれ' }, { surface: 'ば', pos0: '助詞' }],
+    },
     {
         name: 'ja_ire_nai',
         priority: 95,
@@ -1372,6 +1439,28 @@ const COLLOQUIAL_SEQUENCE_RULES = [
         resultType: 'phrase',
         pattern: [{ pos0: '動詞' }, { surface: 'た', pos0: '助動詞' }],
     },
+    {
+        name: 'verb_chatta_past',
+        priority: 96,
+        resultType: 'phrase',
+        pattern: [
+            { pos0: '動詞' },
+            { surface: ['ちゃっ', 'じゃっ'] },
+            { surface: 'た', pos0: '助動詞' },
+        ],
+    },
+    {
+        name: 'verb_past_n_da',
+        priority: 96,
+        resultType: 'phrase',
+        pattern: [{ pos0: '動詞' }, { surface: 'ん' }, { surface: 'だ' }],
+    },
+    {
+        name: 'verb_past_n_da_compact',
+        priority: 96,
+        resultType: 'phrase',
+        pattern: [{ pos0: '動詞' }, { surface: 'んだ' }],
+    },
     {
         name: 'verb_tara_compact',
         priority: 96,
@@ -1876,6 +1965,18 @@ const COLLOQUIAL_SEQUENCE_RULES = [
         resultType: 'fixed_expression',
         pattern: [{ surface: 'んだ' }, { surface: 'よ' }],
     },
+    {
+        name: 'fixed_n_da_ne',
+        priority: 94,
+        resultType: 'fixed_expression',
+        pattern: [{ surface: 'ん' }, { surface: 'だ' }, { surface: 'ね' }],
+    },
+    {
+        name: 'fixed_nda_ne',
+        priority: 94,
+        resultType: 'fixed_expression',
+        pattern: [{ surface: 'んだ' }, { surface: 'ね' }],
+    },
     {
         name: 'fixed_de_su_yo',
         priority: 94,

package/build/src/sentdetect/sentenceDetector.js

@@ -28,26 +28,27 @@ export class SentenceDetector {
         let match = SENTENCE_BREAKER_PATTERN.exec(s);
         while (match !== null) {
             const eos = match.index + match[0].length;
-            if (this.parenthesisLevel(s.slice(0, eos)) === 0) {
-                let adjustedEos = eos;
-                if (eos < s.length) {
-                    adjustedEos += this.prohibitedBOS(s.slice(eos));
-                }
-                if (ITEMIZE_HEADER_PATTERN.test(s.slice(0, eos))) {
-                    match = SENTENCE_BREAKER_PATTERN.exec(s);
-                    continue;
-                }
-                if (eos < s.length && this.isContinuousPhrase(s, eos)) {
-                    match = SENTENCE_BREAKER_PATTERN.exec(s);
-                    continue;
-                }
-                if (checker?.hasNonBreakWord(eos)) {
-                    match = SENTENCE_BREAKER_PATTERN.exec(s);
-                    continue;
-                }
-                return adjustedEos;
+            let adjustedEos = eos;
+            if (eos < s.length) {
+                adjustedEos += this.prohibitedBOS(s.slice(eos));
             }
-            match = SENTENCE_BREAKER_PATTERN.exec(s);
+            if (this.parenthesisLevel(s.slice(0, adjustedEos)) !== 0) {
+                match = SENTENCE_BREAKER_PATTERN.exec(s);
+                continue;
+            }
+            if (ITEMIZE_HEADER_PATTERN.test(s.slice(0, eos))) {
+                match = SENTENCE_BREAKER_PATTERN.exec(s);
+                continue;
+            }
+            if (eos < s.length && this.isContinuousPhrase(s, eos)) {
+                match = SENTENCE_BREAKER_PATTERN.exec(s);
+                continue;
+            }
+            if (checker?.hasNonBreakWord(adjustedEos)) {
+                match = SENTENCE_BREAKER_PATTERN.exec(s);
+                continue;
+            }
+            return adjustedEos;
         }
         if (input.length > this.limit) {
             const spaces = s.match(/^.+\s+/);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sudachi-ts",
-  "version": "0.1.18",
+  "version": "0.1.20-beta.0",
   "description": "TypeScript port of Sudachi morphological analyzer for Japanese text",
   "keywords": [
     "morphological-analyzer",