sudachi-ts 0.1.11 → 0.1.15

package/README.md

@@ -105,7 +105,7 @@ const config = await loadConfig('./sudachi.json');
 const dict = Dictionary.create();
 ```
 
-Example `sudachi.json`:
+Example `sudachi.json`:
 
 ```json
 {
@@ -120,10 +120,14 @@ Example `sudachi.json`:
       }
     }
   ]
-}
-```
-
-By default, Sudachi-TS enables a built-in compound-particle lexicon
+}
+```
+
+For non-absolute file references in config (dictionary files, plugin module paths,
+and built-in plugin file settings), Sudachi-TS tries paths relative to the config
+file first, then relative to the current working directory.
+
+By default, Sudachi-TS enables a built-in compound-particle lexicon
 (`"enableDefaultCompoundParticles": true`) so forms such as `かも`, `のか`,
 and `だから` are tokenized as single morphemes. Set it to `false` to disable:
 
@@ -266,16 +270,19 @@ const plugin = await loader.loadInputTextPlugin(
 
 See [PLUGINS.md](./PLUGINS.md) for detailed plugin development guide.
 
-Quick local comparison for the PoC token chunker plugin:
-
-```bash
-bun examples/token-chunker-plugin.ts /path/to/system.dic "東京大学"
-```
-
-This example prints each token as `surface/reading` so the chunking impact on
-readings is visible in the baseline vs plugin outputs.
-
-## Dictionary Building
+Quick local comparison for the PoC token chunker plugin:
+
+```bash
+npm exec tsx examples/token-chunker-plugin.ts /path/to/system.dic "東京大学"
+```
+
+This example prints each token as `surface/reading` so the chunking impact on
+readings is visible in the baseline vs plugin outputs.
+`TokenChunkerPlugin` is designed and validated against the full Sudachi system
+dictionary (`system_full.dic` / `system.dic`), so prefer full-dictionary checks
+when adding or tuning chunk rules.
+
+## Dictionary Building
 
 Build binary dictionaries from CSV source:
 
@@ -325,23 +332,23 @@ See [CONFIG.md](./CONFIG.md) for detailed configuration options.
 
 ## Development
 
-```bash
-# Clone repository
-git clone https://github.com/your-org/sudachi-ts.git
-cd sudachi-ts
-
-# Install dependencies
-bun install
-
-# Type check
-bun x tsc --noEmit
-
-# Run tests
-bun test
-
-# Lint
-bun x @biomejs/biome lint --write .
-```
+```bash
+# Clone repository
+git clone https://github.com/your-org/sudachi-ts.git
+cd sudachi-ts
+
+# Install dependencies
+npm install
+
+# Type check
+npm run typecheck
+
+# Run tests
+npm test
+
+# Lint
+npm run check:fix
+```
 
 ## Architecture
 

package/build/src/config/config.js

@@ -15,8 +15,8 @@ export class Config {
     static async fromFile(filePath) {
         const content = await readFile(filePath, 'utf-8');
         const baseDir = dirname(filePath);
-        const anchor = PathAnchor.filesystem(baseDir);
-        return new Config(Settings.parse(content), anchor);
+        const anchor = PathAnchor.filesystem(baseDir).andThen(PathAnchor.none());
+        return new Config(Settings.parse(content, anchor), anchor);
     }
     static parse(json) {
         return new Config(Settings.parse(json));
@@ -61,7 +61,14 @@ export class Config {
         return this.settings.getIntList(key);
     }
     getPlugins(key) {
-        return this.settings.getPlugins(key);
+        const plugins = this.settings.getPlugins(key);
+        if (!plugins) {
+            return null;
+        }
+        return plugins.map((plugin) => ({
+            className: plugin.className,
+            settings: plugin.settings.withAnchor(this.anchor),
+        }));
     }
 }
 export async function loadConfig(configPath) {

package/build/src/config/settings.d.ts

@@ -1,12 +1,18 @@
+import { PathAnchor } from './pathAnchor.js';
 export type PluginConf<_T> = {
     className: string;
     settings: Settings;
 };
 export declare class Settings {
     private readonly data;
-    constructor(data?: Record<string, unknown>);
+    private readonly anchor;
+    constructor(data?: Record<string, unknown>, anchor?: PathAnchor);
     static empty(): Settings;
-    static parse(json: string, _basePath?: string): Settings;
+    static parse(json: string, basePathOrAnchor?: string | PathAnchor): Settings;
+    getAnchor(): PathAnchor;
+    withAnchor(anchor: PathAnchor): Settings;
+    getPath(key: string, defaultValue?: string): Promise<string | null>;
+    toObject(): Record<string, unknown>;
     getString(key: string, defaultValue?: string): string | null;
     getInt(key: string, defaultValue?: number): number;
     getBoolean(key: string, defaultValue: boolean): boolean;

package/build/src/config/settings.js

@@ -1,17 +1,39 @@
+import { PathAnchor } from './pathAnchor.js';
 export class Settings {
     data;
-    constructor(data = {}) {
+    anchor;
+    constructor(data = {}, anchor = PathAnchor.none()) {
         this.data = { ...data };
+        this.anchor = anchor;
     }
     static empty() {
-        return new Settings({});
+        return new Settings({}, PathAnchor.none());
     }
-    static parse(json, _basePath) {
+    static parse(json, basePathOrAnchor) {
         const data = JSON.parse(json);
         if (typeof data !== 'object' || data === null) {
             throw new Error('root must be an object');
         }
-        return new Settings(data);
+        if (typeof basePathOrAnchor === 'string') {
+            return new Settings(data, PathAnchor.filesystem(basePathOrAnchor).andThen(PathAnchor.none()));
+        }
+        return new Settings(data, basePathOrAnchor ?? PathAnchor.none());
+    }
+    getAnchor() {
+        return this.anchor;
+    }
+    withAnchor(anchor) {
+        return new Settings(this.data, anchor);
+    }
+    async getPath(key, defaultValue) {
+        const value = this.getString(key, defaultValue);
+        if (value === null) {
+            return null;
+        }
+        return await this.anchor.resolve(value);
+    }
+    toObject() {
+        return { ...this.data };
     }
     getString(key, defaultValue) {
         const value = this.data[key];
@@ -70,7 +92,7 @@ export class Settings {
                     const obj = item;
                     return {
                         className: obj.class,
-                        settings: new Settings({ ...obj }),
+                        settings: new Settings({ ...obj }, this.anchor),
                     };
                 }
                 throw new Error(`sub-object for ${key} didn't have class key`);
@@ -79,9 +101,9 @@ export class Settings {
         return null;
     }
     withFallback(other) {
-        return new Settings({ ...other.data, ...this.data });
+        return new Settings({ ...other.data, ...this.data }, this.anchor.andThen(other.anchor));
     }
     merge(overrides) {
-        return new Settings({ ...this.data, ...overrides });
+        return new Settings({ ...this.data, ...overrides }, this.anchor);
     }
 }

package/build/src/dictionary/characterCategory.js

@@ -1,3 +1,4 @@
+import { readFile } from 'node:fs/promises';
 import { CategoryType } from './categoryType.js';
 export class CharacterCategory {
     static PATTERN_SPACES = /\s+/;
@@ -67,8 +68,7 @@ export class CharacterCategory {
     static async loadDefault() {
         const charCategory = new CharacterCategory();
         try {
-            const response = await fetch(new URL('../resources/char.def', import.meta.url));
-            const content = await response.text();
+            const content = await readFile(new URL('../resources/char.def', import.meta.url), 'utf-8');
             charCategory.readCharacterDefinition(content);
         }
         catch (e) {

package/build/src/dictionary/dictionaryFactory.js

@@ -44,8 +44,8 @@ export class DictionaryFactory {
                 ensureLexiconSet().add(userDict.getLexicon());
             }
         }
-        const loader = new PluginLoader();
-        const defaultConfig = Config.parse(DEFAULT_CONFIG_JSON);
+        const loader = new PluginLoader(anchor);
+        const defaultConfig = Config.parse(DEFAULT_CONFIG_JSON).setAnchor(anchor);
         let inputTextPluginConfs = config.getPlugins('inputTextPlugin');
         if (!inputTextPluginConfs || inputTextPluginConfs.length === 0) {
             inputTextPluginConfs = defaultConfig.getPlugins('inputTextPlugin');

package/build/src/plugins/inputText/defaultInputTextPlugin.js

@@ -9,7 +9,7 @@ export class DefaultInputTextPlugin extends InputTextPlugin {
         if (this.initialized) {
             return;
         }
-        const rewriteDefPath = this.settings.getString('rewriteDef');
+        const rewriteDefPath = await this.settings.getPath('rewriteDef');
         if (rewriteDefPath) {
             const content = await readFully(rewriteDefPath);
             this.readRewriteLists(content);

package/build/src/plugins/loader.d.ts

@@ -1,3 +1,4 @@
+import { PathAnchor } from '../config/pathAnchor.js';
 import type { Settings } from '../config/settings.js';
 import type { Grammar } from '../dictionary/grammar.js';
 import type { Plugin } from './base.js';
@@ -11,6 +12,8 @@ export interface LoadedPlugin<T extends Plugin> {
     className: string;
 }
 export declare class PluginLoader {
+    private readonly anchor;
+    constructor(anchor?: PathAnchor);
     loadInputTextPlugin(className: string, settings: Settings): Promise<LoadedPlugin<InputTextPlugin>>;
     loadOovProviderPlugin(className: string, settings: Settings): Promise<LoadedPlugin<OovProviderPlugin>>;
     loadPathRewritePlugin(className: string, settings: Settings): Promise<LoadedPlugin<PathRewritePlugin>>;
@@ -35,6 +38,8 @@ export declare class PluginLoader {
     private loadPlugin;
     private findPluginClass;
     private isPluginConstructor;
+    private resolveClassSpecifier;
+    private isPathLikeSpecifier;
     private isBuiltIn;
     private getBuiltIn;
 }

package/build/src/plugins/loader.js

@@ -1,3 +1,6 @@
+import { isAbsolute, resolve } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { PathAnchor } from '../config/pathAnchor.js';
 import { InhibitConnectionPlugin } from './connection/inhibitConnectionPlugin.js';
 import { DefaultInputTextPlugin } from './inputText/defaultInputTextPlugin.js';
 import { IgnoreYomiganaPlugin } from './inputText/ignoreYomiganaPlugin.js';
@@ -9,6 +12,10 @@ import { JoinKatakanaOovPlugin } from './pathRewrite/joinKatakanaOovPlugin.js';
 import { JoinNumericPlugin } from './pathRewrite/joinNumericPlugin.js';
 import { TokenChunkerPlugin } from './pathRewrite/tokenChunkerPlugin.js';
 export class PluginLoader {
+    anchor;
+    constructor(anchor = PathAnchor.none()) {
+        this.anchor = anchor;
+    }
     async loadInputTextPlugin(className, settings) {
         const plugin = await this.loadPlugin(className, settings);
         return { plugin, className };
@@ -72,7 +79,8 @@ export class PluginLoader {
                 PluginClass = this.getBuiltIn(className);
             }
             else {
-                const module = await import(className);
+                const classSpecifier = await this.resolveClassSpecifier(className);
+                const module = await import(classSpecifier);
                 PluginClass = this.findPluginClass(module, className);
             }
             const plugin = new PluginClass();
@@ -112,6 +120,24 @@ export class PluginLoader {
             return false;
         }
     }
+    async resolveClassSpecifier(className) {
+        if (this.anchor === PathAnchor.none() ||
+            !this.isPathLikeSpecifier(className)) {
+            return className;
+        }
+        const resolvedPath = await this.anchor.resolve(className);
+        const absolutePath = isAbsolute(resolvedPath)
+            ? resolvedPath
+            : resolve(resolvedPath);
+        return pathToFileURL(absolutePath).href;
+    }
+    isPathLikeSpecifier(className) {
+        return (className.startsWith('./') ||
+            className.startsWith('../') ||
+            className.startsWith('.\\') ||
+            className.startsWith('..\\') ||
+            isAbsolute(className));
+    }
     isBuiltIn(name) {
         return (name in BUILT_IN_PLUGINS || name.split('.').pop() in BUILT_IN_PLUGINS);
     }

package/build/src/plugins/oov/meCabOovProviderPlugin.js

@@ -11,12 +11,12 @@ export class MeCabOovProviderPlugin extends OovProviderPlugin {
         if (this.initialized) {
             return;
         }
-        const charDefPath = this.settings.getString('charDef');
+        const charDefPath = await this.settings.getPath('charDef');
         if (charDefPath) {
             const content = await readFully(charDefPath);
             this.readCharacterProperty(content);
         }
-        const unkDefPath = this.settings.getString('unkDef');
+        const unkDefPath = await this.settings.getPath('unkDef');
         const userPosMode = this.settings.getString(OovProviderPlugin.USER_POS, OovProviderPlugin.USER_POS_FORBID) ?? OovProviderPlugin.USER_POS_FORBID;
         if (unkDefPath) {
             const content = await readFully(unkDefPath);

package/build/src/plugins/pathRewrite/tokenChunkerPlugin.d.ts

@@ -6,9 +6,6 @@ export declare class TokenChunkerPlugin extends PathRewritePlugin {
     private grammar;
     private enablePatternRules;
     private enableBroadRules;
-    private enableCompoundNouns;
-    private minCompoundLength;
-    private excludedNounSubcategories;
     setUp(grammar: Grammar): void;
     rewrite(_text: InputText, path: LatticeNode[], lattice: Lattice): void;
     private toInitialChunks;
@@ -25,12 +22,15 @@ export declare class TokenChunkerPlugin extends PathRewritePlugin {
     private isNumericCommaChunk;
     private isNumericDotChunk;
     private isNumericSignChunk;
+    private isLatinTextChunk;
     private isCounterChunk;
     private mergeCounterChunks;
     private applyMergeStage;
-    private applyCompoundNounStage;
-    private isChunkableNoun;
-    private canMergeAsCompoundNoun;
+    private applyInlineRubyExactStage;
+    private applyInlineRubyPrefixStage;
+    private shouldMergeInlineRubyExact;
+    private shouldMergeInlineRubyPrefix;
+    private toHiragana;
     private getPosById;
     private mergeChunks;
     private readingPart;