sudachi-ts 0.1.13 → 0.1.16

package/README.md

@@ -105,7 +105,7 @@ const config = await loadConfig('./sudachi.json');
 const dict = Dictionary.create();
 ```
 
-Example `sudachi.json`:
+Example `sudachi.json`:
 
 ```json
 {
@@ -120,10 +120,14 @@ Example `sudachi.json`:
       }
     }
   ]
-}
-```
-
-By default, Sudachi-TS enables a built-in compound-particle lexicon
+}
+```
+
+For non-absolute file references in config (dictionary files, plugin module paths,
+and built-in plugin file settings), Sudachi-TS tries paths relative to the config
+file first, then relative to the current working directory.
+
+By default, Sudachi-TS enables a built-in compound-particle lexicon
 (`"enableDefaultCompoundParticles": true`) so forms such as `かも`, `のか`,
 and `だから` are tokenized as single morphemes. Set it to `false` to disable:
 
@@ -269,7 +273,7 @@ See [PLUGINS.md](./PLUGINS.md) for detailed plugin development guide.
 Quick local comparison for the PoC token chunker plugin:
 
 ```bash
-bun examples/token-chunker-plugin.ts /path/to/system.dic "東京大学"
+npm exec tsx examples/token-chunker-plugin.ts /path/to/system.dic "東京大学"
 ```
 
 This example prints each token as `surface/reading` so the chunking impact on
@@ -277,6 +281,9 @@ readings is visible in the baseline vs plugin outputs.
 `TokenChunkerPlugin` is designed and validated against the full Sudachi system
 dictionary (`system_full.dic` / `system.dic`), so prefer full-dictionary checks
 when adding or tuning chunk rules.
+`TokenChunkerPlugin` is intended for `SplitMode.C` tokenization; calling
+`tokenize(SplitMode.A, ...)` or `tokenize(SplitMode.B, ...)` with this plugin
+enabled throws an error.
 
 ## Dictionary Building
 
@@ -328,23 +335,23 @@ See [CONFIG.md](./CONFIG.md) for detailed configuration options.
 
 ## Development
 
-```bash
-# Clone repository
-git clone https://github.com/your-org/sudachi-ts.git
-cd sudachi-ts
-
-# Install dependencies
-bun install
-
-# Type check
-bun x tsc --noEmit
-
-# Run tests
-bun test
-
-# Lint
-bun x @biomejs/biome lint --write .
-```
+```bash
+# Clone repository
+git clone https://github.com/your-org/sudachi-ts.git
+cd sudachi-ts
+
+# Install dependencies
+npm install
+
+# Type check
+npm run typecheck
+
+# Run tests
+npm test
+
+# Lint
+npm run check:fix
+```
 
 ## Architecture
 

package/build/src/config/config.js

@@ -15,8 +15,8 @@ export class Config {
     static async fromFile(filePath) {
         const content = await readFile(filePath, 'utf-8');
         const baseDir = dirname(filePath);
-        const anchor = PathAnchor.filesystem(baseDir);
-        return new Config(Settings.parse(content), anchor);
+        const anchor = PathAnchor.filesystem(baseDir).andThen(PathAnchor.none());
+        return new Config(Settings.parse(content, anchor), anchor);
     }
     static parse(json) {
         return new Config(Settings.parse(json));
@@ -61,7 +61,14 @@ export class Config {
         return this.settings.getIntList(key);
     }
     getPlugins(key) {
-        return this.settings.getPlugins(key);
+        const plugins = this.settings.getPlugins(key);
+        if (!plugins) {
+            return null;
+        }
+        return plugins.map((plugin) => ({
+            className: plugin.className,
+            settings: plugin.settings.withAnchor(this.anchor),
+        }));
     }
 }
 export async function loadConfig(configPath) {

package/build/src/config/settings.d.ts

@@ -1,12 +1,18 @@
+import { PathAnchor } from './pathAnchor.js';
 export type PluginConf<_T> = {
     className: string;
     settings: Settings;
 };
 export declare class Settings {
     private readonly data;
-    constructor(data?: Record<string, unknown>);
+    private readonly anchor;
+    constructor(data?: Record<string, unknown>, anchor?: PathAnchor);
     static empty(): Settings;
-    static parse(json: string, _basePath?: string): Settings;
+    static parse(json: string, basePathOrAnchor?: string | PathAnchor): Settings;
+    getAnchor(): PathAnchor;
+    withAnchor(anchor: PathAnchor): Settings;
+    getPath(key: string, defaultValue?: string): Promise<string | null>;
+    toObject(): Record<string, unknown>;
     getString(key: string, defaultValue?: string): string | null;
     getInt(key: string, defaultValue?: number): number;
     getBoolean(key: string, defaultValue: boolean): boolean;

package/build/src/config/settings.js

@@ -1,17 +1,39 @@
+import { PathAnchor } from './pathAnchor.js';
 export class Settings {
     data;
-    constructor(data = {}) {
+    anchor;
+    constructor(data = {}, anchor = PathAnchor.none()) {
         this.data = { ...data };
+        this.anchor = anchor;
     }
     static empty() {
-        return new Settings({});
+        return new Settings({}, PathAnchor.none());
     }
-    static parse(json, _basePath) {
+    static parse(json, basePathOrAnchor) {
         const data = JSON.parse(json);
         if (typeof data !== 'object' || data === null) {
             throw new Error('root must be an object');
         }
-        return new Settings(data);
+        if (typeof basePathOrAnchor === 'string') {
+            return new Settings(data, PathAnchor.filesystem(basePathOrAnchor).andThen(PathAnchor.none()));
+        }
+        return new Settings(data, basePathOrAnchor ?? PathAnchor.none());
+    }
+    getAnchor() {
+        return this.anchor;
+    }
+    withAnchor(anchor) {
+        return new Settings(this.data, anchor);
+    }
+    async getPath(key, defaultValue) {
+        const value = this.getString(key, defaultValue);
+        if (value === null) {
+            return null;
+        }
+        return await this.anchor.resolve(value);
+    }
+    toObject() {
+        return { ...this.data };
     }
     getString(key, defaultValue) {
         const value = this.data[key];
@@ -70,7 +92,7 @@ export class Settings {
                     const obj = item;
                     return {
                         className: obj.class,
-                        settings: new Settings({ ...obj }),
+                        settings: new Settings({ ...obj }, this.anchor),
                     };
                 }
                 throw new Error(`sub-object for ${key} didn't have class key`);
@@ -79,9 +101,9 @@ export class Settings {
         return null;
     }
     withFallback(other) {
-        return new Settings({ ...other.data, ...this.data });
+        return new Settings({ ...other.data, ...this.data }, this.anchor.andThen(other.anchor));
     }
     merge(overrides) {
-        return new Settings({ ...this.data, ...overrides });
+        return new Settings({ ...this.data, ...overrides }, this.anchor);
     }
 }

package/build/src/core/japaneseTokenizer.js

@@ -140,6 +140,9 @@ export class JapaneseTokenizer {
         return builder.build();
     }
     tokenizeSentence(mode, input) {
+        for (const plugin of this.pathRewritePlugins) {
+            plugin.validateSplitMode(mode);
+        }
         this.buildLattice(input);
         const path = this.lattice.getBestPath();
         for (const plugin of this.pathRewritePlugins) {

package/build/src/dictionary/characterCategory.js

@@ -1,3 +1,4 @@
+import { readFile } from 'node:fs/promises';
 import { CategoryType } from './categoryType.js';
 export class CharacterCategory {
     static PATTERN_SPACES = /\s+/;
@@ -67,8 +68,7 @@ export class CharacterCategory {
     static async loadDefault() {
         const charCategory = new CharacterCategory();
         try {
-            const response = await fetch(new URL('../resources/char.def', import.meta.url));
-            const content = await response.text();
+            const content = await readFile(new URL('../resources/char.def', import.meta.url), 'utf-8');
             charCategory.readCharacterDefinition(content);
         }
         catch (e) {

package/build/src/dictionary/dictionaryFactory.js

@@ -44,8 +44,8 @@ export class DictionaryFactory {
                 ensureLexiconSet().add(userDict.getLexicon());
             }
         }
-        const loader = new PluginLoader();
-        const defaultConfig = Config.parse(DEFAULT_CONFIG_JSON);
+        const loader = new PluginLoader(anchor);
+        const defaultConfig = Config.parse(DEFAULT_CONFIG_JSON).setAnchor(anchor);
         let inputTextPluginConfs = config.getPlugins('inputTextPlugin');
         if (!inputTextPluginConfs || inputTextPluginConfs.length === 0) {
             inputTextPluginConfs = defaultConfig.getPlugins('inputTextPlugin');

package/build/src/plugins/inputText/defaultInputTextPlugin.js

@@ -9,7 +9,7 @@ export class DefaultInputTextPlugin extends InputTextPlugin {
         if (this.initialized) {
             return;
         }
-        const rewriteDefPath = this.settings.getString('rewriteDef');
+        const rewriteDefPath = await this.settings.getPath('rewriteDef');
         if (rewriteDefPath) {
             const content = await readFully(rewriteDefPath);
             this.readRewriteLists(content);

package/build/src/plugins/loader.d.ts

@@ -1,3 +1,4 @@
+import { PathAnchor } from '../config/pathAnchor.js';
 import type { Settings } from '../config/settings.js';
 import type { Grammar } from '../dictionary/grammar.js';
 import type { Plugin } from './base.js';
@@ -11,6 +12,8 @@ export interface LoadedPlugin<T extends Plugin> {
     className: string;
 }
 export declare class PluginLoader {
+    private readonly anchor;
+    constructor(anchor?: PathAnchor);
     loadInputTextPlugin(className: string, settings: Settings): Promise<LoadedPlugin<InputTextPlugin>>;
     loadOovProviderPlugin(className: string, settings: Settings): Promise<LoadedPlugin<OovProviderPlugin>>;
     loadPathRewritePlugin(className: string, settings: Settings): Promise<LoadedPlugin<PathRewritePlugin>>;
@@ -35,6 +38,8 @@ export declare class PluginLoader {
     private loadPlugin;
     private findPluginClass;
     private isPluginConstructor;
+    private resolveClassSpecifier;
+    private isPathLikeSpecifier;
     private isBuiltIn;
     private getBuiltIn;
 }

package/build/src/plugins/loader.js

@@ -1,3 +1,6 @@
+import { isAbsolute, resolve } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { PathAnchor } from '../config/pathAnchor.js';
 import { InhibitConnectionPlugin } from './connection/inhibitConnectionPlugin.js';
 import { DefaultInputTextPlugin } from './inputText/defaultInputTextPlugin.js';
 import { IgnoreYomiganaPlugin } from './inputText/ignoreYomiganaPlugin.js';
@@ -9,6 +12,10 @@ import { JoinKatakanaOovPlugin } from './pathRewrite/joinKatakanaOovPlugin.js';
 import { JoinNumericPlugin } from './pathRewrite/joinNumericPlugin.js';
 import { TokenChunkerPlugin } from './pathRewrite/tokenChunkerPlugin.js';
 export class PluginLoader {
+    anchor;
+    constructor(anchor = PathAnchor.none()) {
+        this.anchor = anchor;
+    }
     async loadInputTextPlugin(className, settings) {
         const plugin = await this.loadPlugin(className, settings);
         return { plugin, className };
@@ -72,7 +79,8 @@ export class PluginLoader {
                 PluginClass = this.getBuiltIn(className);
             }
             else {
-                const module = await import(className);
+                const classSpecifier = await this.resolveClassSpecifier(className);
+                const module = await import(classSpecifier);
                 PluginClass = this.findPluginClass(module, className);
             }
             const plugin = new PluginClass();
@@ -112,6 +120,24 @@ export class PluginLoader {
             return false;
         }
     }
+    async resolveClassSpecifier(className) {
+        if (this.anchor === PathAnchor.none() ||
+            !this.isPathLikeSpecifier(className)) {
+            return className;
+        }
+        const resolvedPath = await this.anchor.resolve(className);
+        const absolutePath = isAbsolute(resolvedPath)
+            ? resolvedPath
+            : resolve(resolvedPath);
+        return pathToFileURL(absolutePath).href;
+    }
+    isPathLikeSpecifier(className) {
+        return (className.startsWith('./') ||
+            className.startsWith('../') ||
+            className.startsWith('.\\') ||
+            className.startsWith('..\\') ||
+            isAbsolute(className));
+    }
     isBuiltIn(name) {
         return (name in BUILT_IN_PLUGINS || name.split('.').pop() in BUILT_IN_PLUGINS);
     }

package/build/src/plugins/oov/meCabOovProviderPlugin.js

@@ -11,12 +11,12 @@ export class MeCabOovProviderPlugin extends OovProviderPlugin {
         if (this.initialized) {
             return;
         }
-        const charDefPath = this.settings.getString('charDef');
+        const charDefPath = await this.settings.getPath('charDef');
         if (charDefPath) {
             const content = await readFully(charDefPath);
             this.readCharacterProperty(content);
         }
-        const unkDefPath = this.settings.getString('unkDef');
+        const unkDefPath = await this.settings.getPath('unkDef');
         const userPosMode = this.settings.getString(OovProviderPlugin.USER_POS, OovProviderPlugin.USER_POS_FORBID) ?? OovProviderPlugin.USER_POS_FORBID;
         if (unkDefPath) {
             const content = await readFully(unkDefPath);

package/build/src/plugins/pathRewrite/base.d.ts

@@ -1,10 +1,12 @@
 import type { InputText } from '../../core/inputText.js';
 import type { Lattice, LatticeNode } from '../../core/lattice.js';
+import type { SplitMode } from '../../core/tokenizer.js';
 import type { CategoryType } from '../../dictionary/categoryType.js';
 import type { Grammar } from '../../dictionary/grammar.js';
 import { Plugin } from '../base.js';
 export declare abstract class PathRewritePlugin extends Plugin {
     setUp(_grammar: Grammar): void;
+    validateSplitMode(_mode: SplitMode): void;
     abstract rewrite(text: InputText, path: LatticeNode[], lattice: Lattice): void;
     concatenate(path: LatticeNode[], begin: number, end: number, lattice: Lattice, normalizedForm?: string | null): LatticeNode;
     concatenateOov(path: LatticeNode[], begin: number, end: number, posId: number, lattice: Lattice): LatticeNode;

package/build/src/plugins/pathRewrite/base.js

@@ -2,6 +2,7 @@ import { WordInfo } from '../../dictionary/wordInfo.js';
 import { Plugin } from '../base.js';
 export class PathRewritePlugin extends Plugin {
     setUp(_grammar) { }
+    validateSplitMode(_mode) { }
     concatenate(path, begin, end, lattice, normalizedForm = null) {
         if (begin >= end) {
             throw new Error('begin >= end');

package/build/src/plugins/pathRewrite/tokenChunkerPlugin.d.ts

@@ -1,5 +1,6 @@
 import type { InputText } from '../../core/inputText.js';
 import type { Lattice, LatticeNode } from '../../core/lattice.js';
+import { SplitMode } from '../../core/tokenizer.js';
 import type { Grammar } from '../../dictionary/grammar.js';
 import { PathRewritePlugin } from './base.js';
 export declare class TokenChunkerPlugin extends PathRewritePlugin {
@@ -7,6 +8,7 @@ export declare class TokenChunkerPlugin extends PathRewritePlugin {
     private enablePatternRules;
     private enableBroadRules;
     setUp(grammar: Grammar): void;
+    validateSplitMode(mode: SplitMode): void;
     rewrite(_text: InputText, path: LatticeNode[], lattice: Lattice): void;
     private toInitialChunks;
     private applyPatternStage;

package/build/src/plugins/pathRewrite/tokenChunkerPlugin.js

@@ -1,3 +1,4 @@
+import { SplitMode } from '../../core/tokenizer.js';
 import { WordInfo } from '../../dictionary/wordInfo.js';
 import { PathRewritePlugin } from './base.js';
 export class TokenChunkerPlugin extends PathRewritePlugin {
@@ -9,6 +10,11 @@ export class TokenChunkerPlugin extends PathRewritePlugin {
         this.enablePatternRules = this.settings.getBoolean('enablePatternRules', true);
         this.enableBroadRules = this.settings.getBoolean('enableBroadRules', false);
     }
+    validateSplitMode(mode) {
+        if (mode !== SplitMode.C) {
+            throw new Error('TokenChunkerPlugin requires SplitMode.C. Use tokenizer.tokenize(text) or tokenizer.tokenize(SplitMode.C, text).');
+        }
+    }
     rewrite(_text, path, lattice) {
         if (path.length === 0) {
             return;

package/package.json

@@ -1,65 +1,69 @@
-{
-  "name": "sudachi-ts",
-  "version": "0.1.13",
-  "description": "TypeScript port of Sudachi morphological analyzer for Japanese text",
-  "keywords": [
-    "morphological-analyzer",
-    "nlp",
-    "japanese",
-    "tokenization",
-    "natural-language-processing",
-    "text-processing"
-  ],
-  "author": "Glen Stampoultzis",
-  "license": "Apache-2.0",
-  "repository": {
-    "type": "git",
-    "url": "https://github.com/gstamp/sudachi-ts.git",
-    "directory": "sudachi-ts"
-  },
-  "bugs": "https://github.com/gstamp/sudachi-ts/issues",
-  "homepage": "https://github.com/gstamp/sudachi-ts#readme",
-  "type": "module",
-  "main": "./build/src/index.js",
-  "types": "./build/src/index.d.ts",
-  "exports": {
-    ".": "./build/src/index.js",
-    "./dictionary": "./build/src/dictionary/index.js",
-    "./config": "./build/src/config/index.js",
-    "./plugins": "./build/src/plugins/index.js"
-  },
-  "bin": {
-    "sudachi": "./build/bin/sudachi.js",
-    "sudachi-build-system": "./build/bin/sudachi-build-system.js",
-    "sudachi-build-user": "./build/bin/sudachi-build-user.js",
-    "sudachi-print-dict": "./build/bin/sudachi-print-dict.js",
-    "sudachi-print-header": "./build/bin/sudachi-print-header.js"
-  },
-  "files": [
-    "build/",
-    "README.md",
-    "LICENSE"
-  ],
-  "engines": {
-    "node": ">=18.0.0"
-  },
+{
+  "name": "sudachi-ts",
+  "version": "0.1.16",
+  "description": "TypeScript port of Sudachi morphological analyzer for Japanese text",
+  "keywords": [
+    "morphological-analyzer",
+    "nlp",
+    "japanese",
+    "tokenization",
+    "natural-language-processing",
+    "text-processing"
+  ],
+  "author": "Glen Stampoultzis",
+  "license": "Apache-2.0",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/gstamp/sudachi-ts.git",
+    "directory": "sudachi-ts"
+  },
+  "bugs": "https://github.com/gstamp/sudachi-ts/issues",
+  "homepage": "https://github.com/gstamp/sudachi-ts#readme",
+  "type": "module",
+  "main": "./build/src/index.js",
+  "types": "./build/src/index.d.ts",
+  "exports": {
+    ".": "./build/src/index.js",
+    "./dictionary": "./build/src/dictionary/index.js",
+    "./config": "./build/src/config/index.js",
+    "./plugins": "./build/src/plugins/index.js"
+  },
+  "bin": {
+    "sudachi": "./build/bin/sudachi.js",
+    "sudachi-build-system": "./build/bin/sudachi-build-system.js",
+    "sudachi-build-user": "./build/bin/sudachi-build-user.js",
+    "sudachi-print-dict": "./build/bin/sudachi-print-dict.js",
+    "sudachi-print-header": "./build/bin/sudachi-print-header.js"
+  },
+  "files": [
+    "build/",
+    "README.md",
+    "LICENSE"
+  ],
+  "engines": {
+    "node": ">=18.0.0"
+  },
   "scripts": {
     "build": "tsc --project tsconfig.build.json",
-    "build:clean": "rm -rf build && npm run build",
+    "build:clean": "node -e \"require('node:fs').rmSync('build', { recursive: true, force: true })\" && npm run build",
     "prepack": "npm run build:clean",
     "lint": "biome lint src/",
     "format": "biome format src/",
-    "check": "biome check src/ && tsc --noEmit",
-    "check:fix": "biome check --write src/ && tsc --noEmit",
+    "typecheck": "tsc --noEmit",
+    "check": "biome check src/ && npm run typecheck",
+    "check:fix": "biome check --write src/ && npm run typecheck",
+    "test": "vitest run",
+    "test:watch": "vitest",
     "release": "./scripts/release.sh"
   },
   "devDependencies": {
     "@biomejs/biome": "^2.3.14",
-    "@types/bun": "^1.1.0",
     "@types/node": "^22.0.0",
-    "typescript": "^5.7.0"
+    "tsx": "^4.20.6",
+    "typescript": "^5.7.0",
+    "vitest": "^3.2.4"
   },
-  "peerDependencies": {
-    "typescript": "^5.0.0"
-  }
-}
+  "peerDependencies": {
+    "typescript": "^5.0.0"
+  }
+}