sudachi-ts 0.1.18 → 0.1.19

package/README.md

@@ -281,6 +281,9 @@ readings is visible in the baseline vs plugin outputs.
 `TokenChunkerPlugin` is designed and validated against the full Sudachi system
 dictionary (`system_full.dic` / `system.dic`), so prefer full-dictionary checks
 when adding or tuning chunk rules.
+`TokenChunkerPlugin` requires `enableDefaultCompoundParticles: true`. Dictionary
+creation throws an error when this plugin is configured with default compound
+particles disabled.
 `TokenChunkerPlugin` is intended for `SplitMode.C` tokenization; calling
 `tokenize(SplitMode.A, ...)` or `tokenize(SplitMode.B, ...)` with this plugin
 enabled throws an error.

package/build/src/dictionary/dictionaryFactory.js

@@ -6,6 +6,12 @@ import { PluginLoader } from '../plugins/loader.js';
 import { BinaryDictionary } from './binaryDictionary.js';
 import { loadDefaultCompoundLexicon } from './defaultCompoundLexicon.js';
 import { LexiconSet } from './lexiconSet.js';
+function isTokenChunkerPlugin(className) {
+    if (className === 'com.worksap.nlp.sudachi.TokenChunkerPlugin') {
+        return true;
+    }
+    return className.split('.').pop() === 'TokenChunkerPlugin';
+}
 export class DictionaryFactory {
     async create(configPath, customConfig) {
         const config = customConfig || (await loadConfig(configPath));
@@ -76,6 +82,10 @@ export class DictionaryFactory {
         if (!pathRewritePluginConfs || pathRewritePluginConfs.length === 0) {
             pathRewritePluginConfs = defaultConfig.getPlugins('pathRewritePlugin');
         }
+        if (!enableDefaultCompoundParticles &&
+            (pathRewritePluginConfs || []).some((conf) => isTokenChunkerPlugin(conf.className))) {
+            throw new Error('TokenChunkerPlugin is only compatible when enableDefaultCompoundParticles is true.');
+        }
         const pathRewritePlugins = (await loader.loadPathRewritePlugins(pathRewritePluginConfs || [], grammar)).map((p) => p.plugin);
         return new Dictionary(grammar, lexicon, inputTextPlugins, oovProviderPlugins, pathRewritePlugins);
     }

package/build/src/plugins/pathRewrite/tokenChunkerPlugin.d.ts

@@ -28,6 +28,7 @@ export declare class TokenChunkerPlugin extends PathRewritePlugin {
     private isCounterChunk;
     private mergeCounterChunks;
     private applyMergeStage;
+    private shouldMergeAttributiveKana;
     private applyInlineRubyExactStage;
     private applyInlineRubyPrefixStage;
     private shouldMergeInlineRubyExact;

package/build/src/plugins/pathRewrite/tokenChunkerPlugin.js

@@ -294,10 +294,16 @@ export class TokenChunkerPlugin extends PathRewritePlugin {
         while (i < chunks.length - 1) {
             const current = chunks[i];
             const next = chunks[i + 1];
+            const following = chunks[i + 2];
             if (!current || !next) {
                 i++;
                 continue;
             }
+            if (this.shouldMergeAttributiveKana(current, next, following)) {
+                const merged = this.mergeChunks([current, next], 'phrase');
+                chunks.splice(i, 2, merged);
+                continue;
+            }
             if ((current.chunkType === 'te_form' ||
                 current.chunkType === 'suru_verb_te_form') &&
                 (next.chunkType === 'single_token' ||
@@ -321,6 +327,24 @@ export class TokenChunkerPlugin extends PathRewritePlugin {
         }
         return chunks;
     }
+    shouldMergeAttributiveKana(current, next, following) {
+        if (next.surface !== 'かな') {
+            return false;
+        }
+        const nextPos = this.getPosById(next.posId);
+        if (!nextPos || nextPos[0] !== '助詞' || nextPos[1] !== '終助詞') {
+            return false;
+        }
+        const currentPos0 = this.getPosById(current.posId)?.[0] ?? '';
+        if (!['名詞', '形状詞'].includes(currentPos0)) {
+            return false;
+        }
+        if (!following) {
+            return false;
+        }
+        const followingPos0 = this.getPosById(following.posId)?.[0] ?? '';
+        return ['名詞', '代名詞'].includes(followingPos0);
+    }
     applyInlineRubyExactStage(source) {
         const chunks = [...source];
         let i = 0;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sudachi-ts",
-  "version": "0.1.18",
+  "version": "0.1.19",
   "description": "TypeScript port of Sudachi morphological analyzer for Japanese text",
   "keywords": [
     "morphological-analyzer",