thai-cut-browser 1.0.0

package/lib/dict.d.ts

@@ -0,0 +1,16 @@
+import type { Acceptor } from "./acceptors";
+type DictAcceptor = Acceptor & {
+    nodeId: number;
+    dict: typeof WordcutDict;
+};
+declare const WordcutDict: {
+    dict: string[];
+    tree: import("./prefixtree").PrefixTree<string>;
+    init(words?: string[], withDefault?: boolean, additionalWords?: string[]): void;
+    addWords(words: string[], finalize?: boolean): void;
+    finalizeDict(): void;
+    createAcceptor(): DictAcceptor;
+    transit(acceptor: DictAcceptor, ch: string): DictAcceptor;
+    sortuniq(a: string[]): string[];
+};
+export = WordcutDict;

package/lib/dict.js

@@ -0,0 +1,73 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+const prefixtree_1 = require("./prefixtree");
+const default_dict_words_1 = __importDefault(require("./default_dict_words"));
+const WordcutDict = {
+    dict: [],
+    tree: (0, prefixtree_1.createPrefixTree)([]),
+    init(words, withDefault = true, additionalWords) {
+        if (words !== undefined && !Array.isArray(words)) {
+            throw new Error("words must be a string[]");
+        }
+        this.dict = [];
+        if (words !== undefined) {
+            this.addWords(words, false);
+        }
+        if (withDefault) {
+            this.addWords(Array.from(default_dict_words_1.default), false);
+        }
+        if (additionalWords !== undefined) {
+            if (!Array.isArray(additionalWords)) {
+                throw new Error("additionalWords must be a string[]");
+            }
+            this.addWords(additionalWords, false);
+        }
+        this.finalizeDict();
+    },
+    addWords(words, finalize) {
+        finalize = finalize === undefined || finalize;
+        this.dict.push(...words);
+        if (finalize) {
+            this.finalizeDict();
+        }
+    },
+    finalizeDict() {
+        this.dict = this.sortuniq(this.dict);
+        this.tree = (0, prefixtree_1.createPrefixTree)(this.dict.map((w) => [w, null]));
+    },
+    createAcceptor() {
+        const dict = this;
+        return {
+            nodeId: 0,
+            strOffset: 0,
+            isFinal: false,
+            isError: false,
+            tag: "DICT",
+            w: 1,
+            type: "DICT",
+            dict,
+            transit(ch) {
+                return this.dict.transit(this, ch);
+            }
+        };
+    },
+    transit(acceptor, ch) {
+        const child = this.tree.lookup(acceptor.nodeId, acceptor.strOffset, ch);
+        if (child !== null) {
+            const [nodeId, isFinal] = child;
+            acceptor.nodeId = nodeId;
+            acceptor.strOffset++;
+            acceptor.isFinal = isFinal;
+        }
+        else {
+            acceptor.isError = true;
+        }
+        return acceptor;
+    },
+    sortuniq(a) {
+        return a.sort().filter((item, pos, arr) => !pos || item !== arr[pos - 1]);
+    },
+};
+module.exports = WordcutDict;

package/lib/index.d.ts

@@ -0,0 +1,27 @@
+export interface WordcutInstance {
+    initNoDict(): void;
+    init(words?: string[], withDefault?: boolean, additionalWords?: string[]): void;
+    cut(text: string, delimiter?: string): string;
+    cutIntoArray(text: string): string[];
+    cutIntoRanges(text: string, noText?: boolean): Array<{
+        s: number;
+        e: number;
+        text?: string;
+    }>;
+}
+export interface CreateWordcutOptions {
+    dictionaryWords?: string[];
+    withDefaultDict?: boolean;
+    additionalWords?: string[];
+    noDict?: boolean;
+    dictPath?: string | string[];
+}
+export declare function createWordcut(options?: CreateWordcutOptions): WordcutInstance;
+export declare function cut(text: string, delimiter?: string): string;
+export declare function cutIntoArray(text: string): string[];
+export declare function cutIntoRanges(text: string, noText?: boolean): Array<{
+    s: number;
+    e: number;
+    text?: string;
+}>;
+export default createWordcut;

package/lib/index.js

@@ -0,0 +1,39 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createWordcut = createWordcut;
+exports.cut = cut;
+exports.cutIntoArray = cutIntoArray;
+exports.cutIntoRanges = cutIntoRanges;
+const BaseWordcut = require("./wordcut");
+function createInstance() {
+    return Object.create(BaseWordcut);
+}
+function createWordcut(options = {}) {
+    const instance = createInstance();
+    if (options.dictPath !== undefined) {
+        throw new Error("dictPath is no longer supported. Pass dictionaryWords: string[] instead.");
+    }
+    if (options.noDict) {
+        instance.initNoDict();
+        return instance;
+    }
+    instance.init(options.dictionaryWords, options.withDefaultDict, options.additionalWords);
+    return instance;
+}
+let defaultInstance = null;
+function getDefaultInstance() {
+    if (defaultInstance === null) {
+        defaultInstance = createWordcut();
+    }
+    return defaultInstance;
+}
+function cut(text, delimiter) {
+    return getDefaultInstance().cut(text, delimiter);
+}
+function cutIntoArray(text) {
+    return getDefaultInstance().cutIntoArray(text);
+}
+function cutIntoRanges(text, noText) {
+    return getDefaultInstance().cutIntoRanges(text, noText);
+}
+exports.default = createWordcut;

package/lib/latin_rules.d.ts

@@ -0,0 +1,3 @@
+import type { AcceptorCreator } from "./acceptors";
+declare const LatinRules: AcceptorCreator[];
+export = LatinRules;

package/lib/latin_rules.js

@@ -0,0 +1,98 @@
+"use strict";
+const WordRule = {
+    createAcceptor(tag) {
+        if (tag.WORD_RULE)
+            return null;
+        return {
+            strOffset: 0,
+            isFinal: false,
+            isError: false,
+            tag: "WORD_RULE",
+            type: "WORD_RULE",
+            w: 1,
+            transit(ch) {
+                const lch = ch.toLowerCase();
+                if (lch >= "a" && lch <= "z") {
+                    this.isFinal = true;
+                    this.strOffset++;
+                }
+                else {
+                    this.isError = true;
+                }
+                return this;
+            }
+        };
+    }
+};
+const NumberRule = {
+    createAcceptor(tag) {
+        if (tag.NUMBER_RULE)
+            return null;
+        return {
+            strOffset: 0,
+            isFinal: false,
+            isError: false,
+            tag: "NUMBER_RULE",
+            type: "NUMBER_RULE",
+            w: 1,
+            transit(ch) {
+                if (ch >= "0" && ch <= "9") {
+                    this.isFinal = true;
+                    this.strOffset++;
+                }
+                else {
+                    this.isError = true;
+                }
+                return this;
+            }
+        };
+    }
+};
+const SpaceRule = {
+    createAcceptor(tag) {
+        if (tag.SPACE_RULE)
+            return null;
+        return {
+            strOffset: 0,
+            isFinal: false,
+            isError: false,
+            tag: "SPACE_RULE",
+            type: "SPACE_RULE",
+            w: 1,
+            transit(ch) {
+                if (ch === " " || ch === "\t" || ch === "\r" || ch === "\n" || ch === "\u00A0" || ch === "\u2003") {
+                    this.isFinal = true;
+                    this.strOffset++;
+                }
+                else {
+                    this.isError = true;
+                }
+                return this;
+            }
+        };
+    }
+};
+const SingleSymbolRule = {
+    createAcceptor() {
+        return {
+            strOffset: 0,
+            isFinal: false,
+            isError: false,
+            tag: "SINSYM",
+            type: "SINSYM",
+            w: 1,
+            transit(ch) {
+                if (this.strOffset === 0 && ch.match(/^[\@\(\)\/\,\-\.\?"`]$/)) {
+                    this.isFinal = true;
+                    this.strOffset++;
+                }
+                else {
+                    this.isError = true;
+                }
+                return this;
+            }
+        };
+    }
+};
+const LatinRules = [WordRule, SpaceRule, SingleSymbolRule, NumberRule];
+module.exports = LatinRules;

package/lib/path_info_builder.d.ts

@@ -0,0 +1,7 @@
+import type { Acceptor } from "./acceptors";
+import type { PathInfo } from "./path_selector";
+export default function createPathInfoBuilder(): {
+    buildByAcceptors(path: PathInfo[], finalAcceptors: Acceptor[], i: number): PathInfo[];
+    fallback(path: PathInfo[], leftBoundary: number, text: string, i: number): PathInfo;
+    build(path: PathInfo[], finalAcceptors: Acceptor[], i: number, leftBoundary: number, text: string): PathInfo[];
+};

package/lib/path_info_builder.js

@@ -0,0 +1,57 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = createPathInfoBuilder;
+function createPathInfoBuilder() {
+    return {
+        buildByAcceptors(path, finalAcceptors, i) {
+            const infos = finalAcceptors.map((acceptor) => {
+                const p = i - acceptor.strOffset + 1;
+                const base = path[p];
+                const info = {
+                    p,
+                    mw: base.mw + (acceptor.mw ?? 0),
+                    w: acceptor.w + base.w,
+                    unk: (acceptor.unk ?? 0) + base.unk,
+                    type: acceptor.type
+                };
+                if (acceptor.type === "PART") {
+                    for (let j = p + 1; j <= i; j++) {
+                        path[j].merge = p;
+                    }
+                    info.merge = p;
+                }
+                return info;
+            });
+            return infos.filter(Boolean);
+        },
+        fallback(path, leftBoundary, text, i) {
+            const base = path[leftBoundary];
+            if (text[i].match(/[\u0E48-\u0E4E]/)) {
+                if (leftBoundary !== 0) {
+                    leftBoundary = path[leftBoundary].p;
+                }
+                return {
+                    p: leftBoundary,
+                    mw: 0,
+                    w: 1 + base.w,
+                    unk: 1 + base.unk,
+                    type: "UNK"
+                };
+            }
+            return {
+                p: leftBoundary,
+                mw: base.mw,
+                w: 1 + base.w,
+                unk: 1 + base.unk,
+                type: "UNK"
+            };
+        },
+        build(path, finalAcceptors, i, leftBoundary, text) {
+            const basicPathInfos = this.buildByAcceptors(path, finalAcceptors, i);
+            if (basicPathInfos.length > 0) {
+                return basicPathInfos;
+            }
+            return [this.fallback(path, leftBoundary, text, i)];
+        }
+    };
+}

package/lib/path_selector.d.ts

@@ -0,0 +1,12 @@
+export interface PathInfo {
+    p: number | null;
+    w: number;
+    unk: number;
+    type: string;
+    mw: number;
+    merge?: number;
+}
+export default function createPathSelector(): {
+    selectPath(paths: PathInfo[]): PathInfo;
+    createPath(): PathInfo[];
+};

package/lib/path_selector.js

@@ -0,0 +1,28 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = createPathSelector;
+function createPathSelector() {
+    return {
+        selectPath(paths) {
+            return paths.reduce((selectedPath, path) => {
+                if (selectedPath === null) {
+                    return path;
+                }
+                if (path.unk < selectedPath.unk)
+                    return path;
+                if (path.unk === selectedPath.unk) {
+                    if (path.mw < selectedPath.mw)
+                        return path;
+                    if (path.mw === selectedPath.mw) {
+                        if (path.w < selectedPath.w)
+                            return path;
+                    }
+                }
+                return selectedPath;
+            }, null);
+        },
+        createPath() {
+            return [{ p: null, w: 0, unk: 0, type: "INIT", mw: 0 }];
+        }
+    };
+}

package/lib/prefixtree.d.ts

@@ -0,0 +1,6 @@
+export type PrefixTreeChild<T = unknown> = [number, boolean, T | null];
+export interface PrefixTree<T = unknown> {
+    tab: Record<string, PrefixTreeChild<T>>;
+    lookup(nodeId: number, offset: number, ch: string): PrefixTreeChild<T> | null;
+}
+export declare function createPrefixTree<T = unknown>(wordPayloads: Array<[string, T]> | null | undefined): PrefixTree<T>;

package/lib/prefixtree.js

@@ -0,0 +1,31 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createPrefixTree = createPrefixTree;
+function createPrefixTree(wordPayloads) {
+    const tab = {};
+    if (wordPayloads) {
+        wordPayloads.forEach(([word, payload], i) => {
+            let rowNo = 0;
+            for (let j = 0; j < word.length; j++) {
+                const ch = word[j];
+                const key = String([rowNo, j, ch]);
+                const child = tab[key];
+                if (child) {
+                    rowNo = child[0];
+                }
+                else {
+                    const isFinal = j + 1 === word.length;
+                    tab[key] = [i, isFinal, isFinal ? payload : null];
+                    rowNo = i;
+                }
+            }
+        });
+    }
+    return {
+        tab,
+        lookup(nodeId, offset, ch) {
+            const child = this.tab[String([nodeId, offset, ch])];
+            return child ?? null;
+        }
+    };
+}

package/lib/thai_rules.d.ts

@@ -0,0 +1,3 @@
+import type { AcceptorCreator } from "./acceptors";
+declare const ThaiRules: AcceptorCreator[];
+export = ThaiRules;

package/lib/thai_rules.js

@@ -0,0 +1,62 @@
+"use strict";
+function isMatch(pat, offset, ch) {
+    if (pat.length <= offset)
+        return false;
+    const expected = pat[offset];
+    return (expected === ch ||
+        (!!expected.match(/[กข]/) && !!ch.match(/[ก-ฮ]/)) ||
+        (!!expected.match(/[มบ]/) && !!ch.match(/[ก-ฮ]/)) ||
+        (!!expected.match(/\u0E49/) && !!ch.match(/[\u0E48-\u0E4B]/)));
+}
+const Rule0 = {
+    createAcceptor() {
+        const pat = "เหก็ม";
+        return {
+            strOffset: 0,
+            isFinal: false,
+            isError: false,
+            tag: "THAI_RULE",
+            type: "THAI_RULE",
+            w: 1,
+            transit(ch) {
+                if (isMatch(pat, this.strOffset, ch)) {
+                    this.isFinal = this.strOffset + 1 === pat.length;
+                    this.strOffset++;
+                }
+                else {
+                    this.isError = true;
+                }
+                return this;
+            }
+        };
+    }
+};
+const PartRule = {
+    createAcceptor() {
+        return {
+            strOffset: 0,
+            isFinal: false,
+            isError: false,
+            tag: "PART",
+            type: "PART",
+            unk: 1,
+            w: 1,
+            patterns: ["แก", "เก", "ก้", "กก์", "กา", "กี", "กิ", "กืก"],
+            transit(ch) {
+                const offset = this.strOffset;
+                this.patterns = this.patterns.filter((pat) => isMatch(pat, offset, ch));
+                if (this.patterns.length > 0) {
+                    const len = 1 + offset;
+                    this.isFinal = this.patterns.some((pat) => pat.length === len);
+                    this.strOffset++;
+                }
+                else {
+                    this.isError = true;
+                }
+                return this;
+            }
+        };
+    }
+};
+const ThaiRules = [Rule0, PartRule];
+module.exports = ThaiRules;

package/lib/wordcut.d.ts

@@ -0,0 +1,21 @@
+import WordcutDict = require("./dict");
+import createPathInfoBuilder from "./path_info_builder";
+import createPathSelector from "./path_selector";
+import createAcceptors from "./acceptors";
+import latinRules = require("./latin_rules");
+import thaiRules = require("./thai_rules");
+interface WordcutType {
+    defaultPathInfoBuilder: typeof createPathInfoBuilder;
+    defaultPathSelector: typeof createPathSelector;
+    defaultAcceptors: typeof createAcceptors;
+    defaultLatinRules: typeof latinRules;
+    defaultThaiRules: typeof thaiRules;
+    defaultDict: typeof WordcutDict;
+    pathInfoBuilder: ReturnType<typeof createPathInfoBuilder>;
+    pathSelector: ReturnType<typeof createPathSelector>;
+    acceptors: ReturnType<typeof createAcceptors>;
+    initNoDict(): void;
+    init(words?: string[], withDefault?: boolean, additionalWords?: string[]): void;
+}
+declare const Wordcut: WordcutType;
+export = Wordcut;

package/lib/wordcut.js

@@ -0,0 +1,37 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+const WordcutDict = require("./dict");
+const WordcutCore = require("./wordcut_core");
+const path_info_builder_1 = __importDefault(require("./path_info_builder"));
+const path_selector_1 = __importDefault(require("./path_selector"));
+const acceptors_1 = __importDefault(require("./acceptors"));
+const latinRules = require("./latin_rules");
+const thaiRules = require("./thai_rules");
+const Wordcut = Object.create(WordcutCore);
+Wordcut.defaultPathInfoBuilder = path_info_builder_1.default;
+Wordcut.defaultPathSelector = path_selector_1.default;
+Wordcut.defaultAcceptors = acceptors_1.default;
+Wordcut.defaultLatinRules = latinRules;
+Wordcut.defaultThaiRules = thaiRules;
+Wordcut.defaultDict = WordcutDict;
+Wordcut.initNoDict = function initNoDict() {
+    this.pathInfoBuilder = this.defaultPathInfoBuilder();
+    this.pathSelector = this.defaultPathSelector();
+    this.acceptors = this.defaultAcceptors();
+    this.defaultLatinRules.forEach((rule) => {
+        this.acceptors.creators.push(rule);
+    });
+    this.defaultThaiRules.forEach((rule) => {
+        this.acceptors.creators.push(rule);
+    });
+};
+Wordcut.init = function init(words, withDefault, additionalWords) {
+    const useDefault = withDefault !== false;
+    this.initNoDict();
+    const dict = Object.assign({}, this.defaultDict);
+    dict.init(words, useDefault, additionalWords);
+    this.acceptors.creators.push(dict);
+};
+module.exports = Wordcut;

package/lib/wordcut_core.d.ts

@@ -0,0 +1,32 @@
+import type { PathInfo } from "./path_selector";
+interface Range {
+    s: number;
+    e: number;
+    text?: string;
+}
+interface WordcutCoreLike {
+    buildPath(text: string): PathInfo[];
+    pathToRanges(path: PathInfo[]): Range[];
+    rangesToText(text: string, ranges: Range[], delimiter: string): string;
+    pathSelector: {
+        createPath(): PathInfo[];
+        selectPath(paths: PathInfo[]): PathInfo;
+    };
+    pathInfoBuilder: {
+        build(path: PathInfo[], finalAcceptors: unknown[], i: number, leftBoundary: number, text: string): PathInfo[];
+    };
+    acceptors: {
+        reset(): void;
+        transit(ch: string): void;
+        getFinalAcceptors(): unknown[];
+    };
+}
+declare const WordcutCore: {
+    buildPath(this: WordcutCoreLike, text: string): PathInfo[];
+    pathToRanges(path: PathInfo[]): Range[];
+    rangesToText(text: string, ranges: Range[], delimiter: string): string;
+    cut(this: WordcutCoreLike, text: string, delimiter?: string): string;
+    cutIntoRanges(this: WordcutCoreLike, text: string, noText?: boolean): Range[];
+    cutIntoArray(this: WordcutCoreLike, text: string): string[];
+};
+export = WordcutCore;

package/lib/wordcut_core.js

@@ -0,0 +1,61 @@
+"use strict";
+const WordcutCore = {
+    buildPath(text) {
+        const path = this.pathSelector.createPath();
+        let leftBoundary = 0;
+        this.acceptors.reset();
+        for (let i = 0; i < text.length; i++) {
+            const ch = text[i];
+            this.acceptors.transit(ch);
+            const possiblePathInfos = this.pathInfoBuilder.build(path, this.acceptors.getFinalAcceptors(), i, leftBoundary, text);
+            const selectedPath = this.pathSelector.selectPath(possiblePathInfos);
+            path.push(selectedPath);
+            if (selectedPath.type !== "UNK") {
+                leftBoundary = i;
+            }
+        }
+        return path;
+    },
+    pathToRanges(path) {
+        let e = path.length - 1;
+        const ranges = [];
+        while (e > 0) {
+            const info = path[e];
+            let s = info.p;
+            if (info.merge !== undefined && ranges.length > 0) {
+                const r = ranges[ranges.length - 1];
+                r.s = info.merge;
+                s = r.s;
+            }
+            else {
+                ranges.push({ s, e });
+            }
+            e = s;
+        }
+        return ranges.reverse();
+    },
+    rangesToText(text, ranges, delimiter) {
+        return ranges.map((r) => text.substring(r.s, r.e)).join(delimiter);
+    },
+    cut(text, delimiter) {
+        const path = this.buildPath(text);
+        const ranges = this.pathToRanges(path);
+        return this.rangesToText(text, ranges, delimiter === undefined ? "|" : delimiter);
+    },
+    cutIntoRanges(text, noText) {
+        const path = this.buildPath(text);
+        const ranges = this.pathToRanges(path);
+        if (!noText) {
+            ranges.forEach((r) => {
+                r.text = text.substring(r.s, r.e);
+            });
+        }
+        return ranges;
+    },
+    cutIntoArray(text) {
+        const path = this.buildPath(text);
+        const ranges = this.pathToRanges(path);
+        return ranges.map((r) => text.substring(r.s, r.e));
+    }
+};
+module.exports = WordcutCore;