sudachi-ts 0.1.19 → 0.1.20-beta.0

package/README.md

@@ -191,13 +191,17 @@ import { SentenceDetector } from 'sudachi-ts/sentdetect/sentenceDetector.js';
 
 const sentences = tokenizer.tokenizeSentences('東京都は日本の首都です。大阪は商業都市です。');
 
-for (const sentence of sentences) {
-  console.log('--- Sentence ---');
-  for (const morpheme of sentence) {
-    console.log(morpheme.surface());
-  }
-}
-```
+for (const sentence of sentences) {
+  console.log('--- Sentence ---');
+  for (const morpheme of sentence) {
+    console.log(morpheme.surface());
+  }
+}
+```
+
+`tokenizeSentences(...)` treats quoted dialogue endings (for example `「...！」` and
+`「...。」`) as sentence boundaries and skips leading inter-sentence whitespace such
+as newlines before tokenization.
 
 Lazy sentence processing for streaming:
 

package/build/src/core/japaneseTokenizer.js

@@ -7,6 +7,7 @@ import { MorphemeList as MorphemeListImpl } from './morphemeList.js';
 import { SentenceSplittingLazyAnalysis } from './sentenceSplittingLazyAnalysis.js';
 import { SplitMode } from './tokenizer.js';
 import { UTF8InputTextBuilder } from './utf8InputText.js';
+const LEADING_WHITESPACE_PATTERN = /^\s+/u;
 export class JapaneseTokenizer {
     grammar;
     lexicon;
@@ -57,6 +58,13 @@ export class JapaneseTokenizer {
         const sentences = [];
         let remaining = inputText;
         while (remaining.length > 0) {
+            const leadingWhitespace = remaining.match(LEADING_WHITESPACE_PATTERN);
+            if (leadingWhitespace) {
+                remaining = remaining.slice(leadingWhitespace[0].length);
+                if (remaining.length === 0) {
+                    break;
+                }
+            }
             const checker = {
                 hasNonBreakWord: (eos) => {
                     const bytes = this.buildInputText(remaining.slice(0, eos)).getByteText();

package/build/src/core/sentenceSplittingLazyAnalysis.js

@@ -1,6 +1,7 @@
 import { SentenceDetector, } from '../sentdetect/sentenceDetector.js';
 import { UTF8InputTextBuilder } from './utf8InputText.js';
 const BUFFER_SIZE = 4096;
+const LEADING_WHITESPACE_PATTERN = /^\s+/u;
 export class SentenceSplittingLazyAnalysis {
     mode;
     grammar;
@@ -68,6 +69,15 @@ export class SentenceSplittingLazyAnalysis {
         return this.buffer.length;
     }
     processNextSentence() {
+        const leadingWhitespace = this.normalized.match(LEADING_WHITESPACE_PATTERN);
+        if (leadingWhitespace) {
+            const skipped = leadingWhitespace[0].length;
+            this.bos += skipped;
+            this.normalized = this.normalized.slice(skipped);
+            if (this.normalized.length === 0) {
+                return null;
+            }
+        }
         const detector = new SentenceDetector();
         const eosLength = detector.getEos(this.normalized, this);
         if (eosLength > 0) {

package/build/src/plugins/pathRewrite/tokenChunkerPlugin.d.ts

@@ -29,6 +29,7 @@ export declare class TokenChunkerPlugin extends PathRewritePlugin {
     private mergeCounterChunks;
     private applyMergeStage;
     private shouldMergeAttributiveKana;
+    private shouldMergeSmallKana;
     private applyInlineRubyExactStage;
     private applyInlineRubyPrefixStage;
     private shouldMergeInlineRubyExact;

package/build/src/plugins/pathRewrite/tokenChunkerPlugin.js

@@ -299,6 +299,11 @@ export class TokenChunkerPlugin extends PathRewritePlugin {
                 i++;
                 continue;
             }
+            if (this.shouldMergeSmallKana(current, next)) {
+                const merged = this.mergeChunks([current, next], 'fixed_expression');
+                chunks.splice(i, 2, merged);
+                continue;
+            }
             if (this.shouldMergeAttributiveKana(current, next, following)) {
                 const merged = this.mergeChunks([current, next], 'phrase');
                 chunks.splice(i, 2, merged);
@@ -318,7 +323,10 @@ export class TokenChunkerPlugin extends PathRewritePlugin {
                 continue;
             }
             if (current.surface.endsWith('んだ') &&
-                (next.surface === 'よ' || next.surface === 'よっ')) {
+                (next.surface === 'よ' ||
+                    next.surface === 'よっ' ||
+                    next.surface === 'ね' ||
+                    next.surface === 'ねっ')) {
                 const merged = this.mergeChunks([current, next], 'fixed_expression');
                 chunks.splice(i, 2, merged);
                 continue;
@@ -345,6 +353,14 @@ export class TokenChunkerPlugin extends PathRewritePlugin {
         const followingPos0 = this.getPosById(following.posId)?.[0] ?? '';
         return ['名詞', '代名詞'].includes(followingPos0);
     }
+    shouldMergeSmallKana(current, next) {
+        if (current.chunkType !== 'single_token' ||
+            next.chunkType !== 'single_token') {
+            return false;
+        }
+        return (SINGLE_KANA_PATTERN.test(current.surface) &&
+            SMALL_KANA_SUFFIXES.has(next.surface));
+    }
     applyInlineRubyExactStage(source) {
         const chunks = [...source];
         let i = 0;
@@ -510,6 +526,8 @@ const KANJI_PATTERN = /\p{Script=Han}/u;
 const KANJI_ONLY_PATTERN = /^[\p{Script=Han}々〆ヵヶ]+$/u;
 const KANA_PATTERN = /^[ぁ-ゖァ-ヺー]+$/u;
 const HIRAGANA_PATTERN = /^[ぁ-ゖー]+$/u;
+const SINGLE_KANA_PATTERN = /^[ぁ-ゖァ-ヺー]$/u;
+const SMALL_KANA_SUFFIXES = new Set(['ゃ', 'ゅ', 'ょ', 'ャ', 'ュ', 'ョ']);
 const COUNTER_WORDS = new Set([
     '本',
     '匹',
@@ -707,6 +725,25 @@ const COLLOQUIAL_SEQUENCE_RULES = [
             { surface: 'た', pos0: '助動詞' },
         ],
     },
+    {
+        name: 'copula_desu_past_split',
+        priority: 101,
+        resultType: 'phrase',
+        pattern: [
+            { surface: 'でし', dictionaryForm: 'です', pos0: '助動詞' },
+            { surface: 'た', pos0: '助動詞' },
+        ],
+    },
+    {
+        name: 'verb_masen',
+        priority: 101,
+        resultType: 'phrase',
+        pattern: [
+            { pos0: '動詞' },
+            { surface: 'ませ', dictionaryForm: 'ます', pos0: '助動詞' },
+            { surface: 'ん', pos0: '助動詞' },
+        ],
+    },
     {
         name: 'noun_suru_past',
         priority: 101,
@@ -1362,6 +1399,12 @@ const COLLOQUIAL_SEQUENCE_RULES = [
             { surface: 'いい' },
         ],
     },
+    {
+        name: 'nakereba',
+        priority: 95,
+        resultType: 'phrase',
+        pattern: [{ surface: 'なけれ' }, { surface: 'ば', pos0: '助詞' }],
+    },
     {
         name: 'ja_ire_nai',
         priority: 95,
@@ -1396,6 +1439,28 @@ const COLLOQUIAL_SEQUENCE_RULES = [
         resultType: 'phrase',
         pattern: [{ pos0: '動詞' }, { surface: 'た', pos0: '助動詞' }],
     },
+    {
+        name: 'verb_chatta_past',
+        priority: 96,
+        resultType: 'phrase',
+        pattern: [
+            { pos0: '動詞' },
+            { surface: ['ちゃっ', 'じゃっ'] },
+            { surface: 'た', pos0: '助動詞' },
+        ],
+    },
+    {
+        name: 'verb_past_n_da',
+        priority: 96,
+        resultType: 'phrase',
+        pattern: [{ pos0: '動詞' }, { surface: 'ん' }, { surface: 'だ' }],
+    },
+    {
+        name: 'verb_past_n_da_compact',
+        priority: 96,
+        resultType: 'phrase',
+        pattern: [{ pos0: '動詞' }, { surface: 'んだ' }],
+    },
     {
         name: 'verb_tara_compact',
         priority: 96,
@@ -1900,6 +1965,18 @@ const COLLOQUIAL_SEQUENCE_RULES = [
         resultType: 'fixed_expression',
         pattern: [{ surface: 'んだ' }, { surface: 'よ' }],
     },
+    {
+        name: 'fixed_n_da_ne',
+        priority: 94,
+        resultType: 'fixed_expression',
+        pattern: [{ surface: 'ん' }, { surface: 'だ' }, { surface: 'ね' }],
+    },
+    {
+        name: 'fixed_nda_ne',
+        priority: 94,
+        resultType: 'fixed_expression',
+        pattern: [{ surface: 'んだ' }, { surface: 'ね' }],
+    },
     {
         name: 'fixed_de_su_yo',
         priority: 94,

package/build/src/sentdetect/sentenceDetector.js

@@ -28,26 +28,27 @@ export class SentenceDetector {
         let match = SENTENCE_BREAKER_PATTERN.exec(s);
         while (match !== null) {
             const eos = match.index + match[0].length;
-            if (this.parenthesisLevel(s.slice(0, eos)) === 0) {
-                let adjustedEos = eos;
-                if (eos < s.length) {
-                    adjustedEos += this.prohibitedBOS(s.slice(eos));
-                }
-                if (ITEMIZE_HEADER_PATTERN.test(s.slice(0, eos))) {
-                    match = SENTENCE_BREAKER_PATTERN.exec(s);
-                    continue;
-                }
-                if (eos < s.length && this.isContinuousPhrase(s, eos)) {
-                    match = SENTENCE_BREAKER_PATTERN.exec(s);
-                    continue;
-                }
-                if (checker?.hasNonBreakWord(eos)) {
-                    match = SENTENCE_BREAKER_PATTERN.exec(s);
-                    continue;
-                }
-                return adjustedEos;
+            let adjustedEos = eos;
+            if (eos < s.length) {
+                adjustedEos += this.prohibitedBOS(s.slice(eos));
             }
-            match = SENTENCE_BREAKER_PATTERN.exec(s);
+            if (this.parenthesisLevel(s.slice(0, adjustedEos)) !== 0) {
+                match = SENTENCE_BREAKER_PATTERN.exec(s);
+                continue;
+            }
+            if (ITEMIZE_HEADER_PATTERN.test(s.slice(0, eos))) {
+                match = SENTENCE_BREAKER_PATTERN.exec(s);
+                continue;
+            }
+            if (eos < s.length && this.isContinuousPhrase(s, eos)) {
+                match = SENTENCE_BREAKER_PATTERN.exec(s);
+                continue;
+            }
+            if (checker?.hasNonBreakWord(adjustedEos)) {
+                match = SENTENCE_BREAKER_PATTERN.exec(s);
+                continue;
+            }
+            return adjustedEos;
         }
         if (input.length > this.limit) {
             const spaces = s.match(/^.+\s+/);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sudachi-ts",
-  "version": "0.1.19",
+  "version": "0.1.20-beta.0",
   "description": "TypeScript port of Sudachi morphological analyzer for Japanese text",
   "keywords": [
     "morphological-analyzer",