taraskevizer 10.4.8 → 10.4.10

package/dist/bin/index.js

@@ -0,0 +1,184 @@
+#!/usr/bin/env node
+import { cpus } from 'node:os';
+import { Worker } from 'node:worker_threads';
+import { pipelines } from '../index.js';
+import { parseArgs } from './parse-args.js';
+const getPrint = (stream) => (msg) => {
+    stream.write("[taraskevizer]" + ' ' + msg);
+};
+const getPrintLn = (printFn) => (msg) => {
+    printFn(msg + '\n');
+};
+const print = getPrint(process.stdout);
+const printErr = getPrint(process.stderr);
+const printLn = getPrintLn(print);
+const printErrLn = getPrintLn(printErr);
+// AI-written, may need improvements
+const splitIntoChunks = (text, n) => {
+    const size = Math.ceil(text.length / n);
+    const chunks = [];
+    let start = 0;
+    for (let i = 0; i < n; i++) {
+        let end = start + size;
+        // Adjust end to the nearest newline character
+        if (end < text.length) {
+            const forward = text.indexOf('\n', end);
+            const backward = text.lastIndexOf('\n', end);
+            if (forward === -1 && backward === -1) {
+                // no-op
+            }
+            else if (forward === -1) {
+                end = backward;
+            }
+            else if (backward === -1) {
+                end = forward;
+            }
+            else {
+                end = forward - end < end - backward ? forward : backward;
+            }
+        }
+        // Ensure we don't split inside HTML tags or special syntax tags
+        if (end < text.length) {
+            const lastOpen = text.lastIndexOf('<', end);
+            const lastClose = text.lastIndexOf('>', end);
+            if (lastOpen > lastClose) {
+                const nextClose = text.indexOf('>', end);
+                if (nextClose !== -1) {
+                    end = nextClose + 1;
+                }
+            }
+        }
+        if (end > text.length)
+            end = text.length;
+        chunks.push(text.slice(start, end));
+        start = end;
+        if (start >= text.length)
+            break;
+    }
+    return chunks;
+};
+const getPrettyByteSize = (n) => n < 1024
+    ? `${n} B`
+    : n < 1024 * 1024
+        ? `${(n / 1024).toFixed(2)} KB`
+        : `${(n / (1024 * 1024)).toFixed(2)} MB`;
+process.argv.splice(0, 2);
+const firstArg = process.argv[0];
+if (firstArg) {
+    if (firstArg === '-v' || firstArg === '--version') {
+        printLn("10.4.10");
+        process.exit(0);
+    }
+    if (firstArg === '-h' || firstArg === '--help') {
+        printLn("Usage: \u001b[34mtarask\u001b[0m [options] text\nIf text is not passed, interactive mode is enabled\n\nEXAMPLES\n\nConvert and latinize a word\n\t\u001b[34mtarask\u001b[0m \u001b[35m--latin\u001b[0m 'планета'\nPrints \"p\u001b[32ml\u001b[0ma\u001b[32mne\u001b[0mta\"\n\nRead from one file and write converted text to another\n\t\u001b[34mtarask\u001b[0m < ./cyr-text.txt > ./lat-text.txt\n\nEnter interactive mode\n\t\u001b[34mtarask\u001b[0m\nPrints \"\u001b[34m[taraskevizer]\u001b[0m Enter the text:\" and waits until you enter a new line\n\nOPTIONS\n\n\u001b[33mGeneral\u001b[0m:\n  \u001b[35m-h\u001b[0m \u001b[35m--help\u001b[0m\n  \u001b[35m-v\u001b[0m \u001b[35m--version\u001b[0m\n\n\u001b[33mAlphabet\u001b[0m:\n  \u001b[35m-l\u001b[0m  \u001b[35m--latin\u001b[0m\n  \u001b[35m-lj\u001b[0m \u001b[35m--latin-ji\u001b[0m\n  \u001b[35m-a\u001b[0m  \u001b[35m--arabic\u001b[0m\n\n\u001b[33mWhen to replace і(i) by й(j) after vowels\u001b[0m:\n  \u001b[35m-jr\u001b[0m \u001b[35m--jrandom\u001b[0m\n  \u001b[35m-ja\u001b[0m \u001b[35m--jalways\u001b[0m\n\n\u001b[33mReplace ґ(g) by г(h) in cyrillic alphabet\u001b[0m:\n  \u001b[35m--h\u001b[0m\n\n\u001b[33mVariations\u001b[0m:\n  \u001b[35m-nv\u001b[0m \u001b[35m--no-variations\u001b[0m\n  \u001b[35m-fv\u001b[0m \u001b[35m--first-variation\u001b[0m\n\n\u001b[33mMode (only one can be used)\u001b[0m:\n  \u001b[35m-html\u001b[0m \u001b[35m--html\u001b[0m\n  \u001b[35m-abc\u001b[0m  \u001b[35m--alphabet-only\u001b[0m\n\n\u001b[33mOther\u001b[0m:\n  \u001b[35m-nec\u001b[0m \u001b[35m--not-escape-caps\u001b[0m\n  \u001b[35m-nc\u001b[0m  \u001b[35m--no-color\u001b[0m\n  \u001b[35m-st\u001b[0m  \u001b[35m--single-thread\u001b[0m\n");
+        process.exit(0);
+    }
+}
+const argv = process.argv.slice();
+const { mode, cfg, doForceSingleThread } = parseArgs(process.argv);
+const workers = {
+    size: cpus()?.length || 1,
+    workers: null,
+    init() {
+        if (this.workers)
+            return;
+        process.stderr.write(`(Initializing ${this.size} workers... `);
+        this.workers = Array.from({ length: this.size }, () => new Worker("const { parentPort, workerData } = require('node:worker_threads');\nconst { pipelines } = require('./dist');\nconst { parseArgs } = require('./dist/bin/parse-args');\n\nconst { mode, cfg } = parseArgs(workerData.argv);\n\nparentPort.on('message', (chunk) => {\n\tparentPort.postMessage(pipelines[mode](chunk, cfg));\n});\n", {
+            eval: true,
+            workerData: { argv },
+        }));
+        process.stderr.write('done.) ');
+    },
+    process(chunks) {
+        return Promise.all(chunks.map((chunk, i) => new Promise((resolve, reject) => {
+            const worker = this.workers[i % this.size];
+            worker.postMessage(chunk);
+            worker.once('message', resolve);
+            worker.once('error', reject);
+        })));
+    },
+};
+const processText = async (text) => {
+    let result = '';
+    if (!doForceSingleThread && workers.size > 1 && text.length > 50_000) {
+        workers.init();
+        const chunks = splitIntoChunks(text, workers.size);
+        const results = await workers.process(chunks);
+        result = results.join('');
+    }
+    else {
+        result = pipelines[mode](text, cfg);
+    }
+    if (!process.stdout.write(result)) {
+        process.stdout.once('drain', () => {
+            printErrLn('Drain event fired, exiting.');
+            process.exit(0);
+        });
+    }
+};
+if (process.argv.length) {
+    printErrLn('Processing the rest of command-line arguments as text...');
+    await processText(process.argv.reverse().join(' '));
+}
+else {
+    const chunks = [];
+    let value = '';
+    let length = 0;
+    let byteLength = 0;
+    const MAX_BYTE_LENGTH = 64 * 1024 * 1024;
+    const getChunksString = () => Buffer.concat(chunks, length).toString();
+    if (process.stdin.isTTY) {
+        printErrLn('Enter the text');
+        for await (const chunk of process.stdin) {
+            chunks.push(chunk);
+            length += chunk.length;
+            if (chunk.includes('\n'))
+                break;
+        }
+        await processText(getChunksString());
+        process.stdout.write('\n');
+    }
+    else {
+        printErrLn('Reading from stdin...');
+        const processTextWithLogs = async (value) => {
+            printErr(`Processing ${value.length
+                .toString()
+                .replace(/\B(?=(\d{3})+(?!\d))/g, ' ')} characters (${getPrettyByteSize(Buffer.byteLength(value))}) chunk... `);
+            const startTime = performance.now();
+            await processText(value);
+            process.stderr.write(`done in ${((performance.now() - startTime) / 1000).toFixed(2)} seconds.\n`);
+        };
+        for await (const chunk of process.stdin) {
+            byteLength += chunk.byteLength;
+            if (byteLength >= MAX_BYTE_LENGTH) {
+                value += getChunksString();
+                const lastNewlineIndex = value.lastIndexOf('\n');
+                if (lastNewlineIndex === -1) {
+                    printErrLn('\nInput exceeded maximum size of ' +
+                        MAX_BYTE_LENGTH +
+                        ' bytes without a newline. Stopping.');
+                    process.exit(1);
+                }
+                let valueForNextBatch = value.slice(lastNewlineIndex + 1);
+                value = value.slice(0, lastNewlineIndex + 1);
+                const lastOpeningTagIndex = value.lastIndexOf('<');
+                const lastClosingTagIndex = value.lastIndexOf('>');
+                if (lastOpeningTagIndex !== -1 &&
+                    lastOpeningTagIndex > lastClosingTagIndex) {
+                    const incompleteTag = value.slice(lastOpeningTagIndex);
+                    value = value.slice(0, lastOpeningTagIndex);
+                    valueForNextBatch = incompleteTag + valueForNextBatch;
+                }
+                await processTextWithLogs(value);
+                value = valueForNextBatch;
+                byteLength = chunk.byteLength;
+                length = 0;
+                chunks.length = 0;
+            }
+            length += chunk.length;
+            chunks.push(chunk);
+        }
+        await processTextWithLogs(value + getChunksString());
+    }
+}
+process.exit(0);

package/dist/bin/parse-args.d.ts

@@ -0,0 +1,6 @@
+import { TaraskConfig } from '..';
+export declare const parseArgs: (argv: string[]) => {
+    cfg: Partial<TaraskConfig>;
+    mode: "tarask";
+    doForceSingleThread: boolean;
+};

package/dist/bin/parse-args.js

@@ -0,0 +1,123 @@
+import { dicts, htmlConfigOptions, TaraskConfig, wrappers, } from '../index.js';
+const toHashTable = (dict) => {
+    const result = {};
+    for (const { 0: options, 1: callback } of dict)
+        for (const option of options)
+            result[option] = callback;
+    return result;
+};
+export const parseArgs = (argv) => {
+    let cfg = {
+        g: true,
+        variations: 'all',
+        wrappers: wrappers.ansiColor,
+    };
+    let mode = 'tarask';
+    let isHtml = false;
+    let doForceSingleThread = false;
+    const optionDict = toHashTable([
+        [
+            ['--latin', '-l'],
+            () => {
+                cfg.abc = dicts.alphabets.latin;
+            },
+        ],
+        [
+            ['--latin-ji', '-lj'],
+            () => {
+                cfg.abc = dicts.alphabets.latinJi;
+            },
+        ],
+        [
+            ['--arabic', '-a'],
+            () => {
+                cfg.abc = dicts.alphabets.arabic;
+            },
+        ],
+        [
+            ['--jrandom', '-jr'],
+            () => {
+                cfg.j = 'random';
+            },
+        ],
+        [
+            ['--jalways', '-ja'],
+            () => {
+                cfg.j = 'always';
+            },
+        ],
+        [
+            ['--no-escape-caps', '-nec'],
+            () => {
+                cfg.doEscapeCapitalized = false;
+            },
+        ],
+        [
+            ['--h'],
+            () => {
+                cfg.g = false;
+            },
+        ],
+        [
+            ['--no-variations', '-nv'],
+            () => {
+                cfg.variations = 'no';
+            },
+        ],
+        [
+            ['--first-variation', '-fv'],
+            () => {
+                cfg.variations = 'first';
+            },
+        ],
+        [
+            ['--no-color', '-nc'],
+            () => {
+                cfg.wrappers = null;
+            },
+        ],
+        [
+            ['--html', '-html'],
+            () => {
+                isHtml = true;
+                cfg.wrappers = htmlConfigOptions.wrappers;
+            },
+        ],
+        [
+            ['--alphabet-only', '-abc'],
+            () => {
+                mode = 'alphabetic';
+            },
+        ],
+        [
+            ['--phonetic', '-ph'],
+            () => {
+                mode = 'phonetic';
+            },
+        ],
+        [
+            ['--single-thread', '-st'],
+            () => {
+                doForceSingleThread = true;
+            },
+        ],
+    ]);
+    let currOption;
+    argv.reverse();
+    while ((currOption = argv.pop())) {
+        if (currOption in optionDict) {
+            optionDict[currOption]();
+        }
+        else {
+            argv.push(currOption);
+            break;
+        }
+    }
+    cfg = new TaraskConfig(isHtml
+        ? {
+            ...htmlConfigOptions,
+            ...cfg,
+        }
+        : cfg);
+    return { cfg, mode, doForceSingleThread };
+};

package/dist/config.d.ts

@@ -1,5 +1,5 @@
-import type { DeepReadonly } from './types';
 import type { Alphabet } from './dict/alphabets';
+import type { DeepReadonly } from './types';
 import { type Wrappers } from './wrappers';
 export type Variation = 'no' | 'first' | 'all';
 export type OptionJ = 'never' | 'random' | 'always';

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "taraskevizer",
-	"version": "10.4.8",
+	"version": "10.4.10",
 	"author": "GooseOb",
 	"repository": {
 		"type": "git",
@@ -22,7 +22,7 @@
 		"typescript-eslint": "^8.42.0"
 	},
 	"bin": {
-		"tarask": "dist/bin.js"
+		"tarask": "dist/bin/index.js"
 	},
 	"description": "Канвэртацыя акадэмічнага правапісу ў клясычны",
 	"files": [

package/dist/bin.js

@@ -1,206 +0,0 @@
-#!/usr/bin/env node
-import { dicts, TaraskConfig, pipelines, htmlConfigOptions, wrappers } from './index.js';
-const printWithPrefix = (msg) => {
-    process.stdout.write("[taraskevizer]" + ' ' + msg + '\n');
-};
-process.argv.splice(0, 2);
-const firstArg = process.argv[0];
-if (firstArg) {
-    if (firstArg === '-v' || firstArg === '--version') {
-        printWithPrefix("10.4.8");
-        process.exit(0);
-    }
-    if (firstArg === '-h' || firstArg === '--help') {
-        printWithPrefix(`Usage: tarask [options] text
-If text is not passed, interactive mode will be enabled
-
-EXAMPLES
-
-Convert and latinize a word
-	tarask --latin 'планета'
-Will print "planeta"
-
-Read from one file and write converted text to another
-	tarask < ./cyr-text.txt > ./lat-text.txt
-
-Enter interactive mode
-	tarask
-Will print "[taraskevizer] Enter the text:" and wait until you press Enter
-
-OPTIONS
-
-General:
-  -h --help
-  -v --version
-
-Alphabet:
-  -l --latin
-  -lj --latin-ji
-  -a --arabic
-
-When to replace і(i) by й(j) after vowels:
-  -jr --jrandom
-  -ja --jalways
-
-Replace ґ(g) by г(h) in cyrillic alphabet:
-  --h
-
-Variations:
-  -nv --no-variations
-  -fv --first-variation
-
-Mode (only one can be used):
-  -html --html
-  -abc --alphabet-only
-
-Other:
-  -nec --not-escape-caps
-  -nc --no-color
-`);
-        process.exit(0);
-    }
-}
-let cfg = {
-    g: true,
-    variations: 'all',
-    wrappers: wrappers.ansiColor,
-};
-let mode = 'tarask';
-const toHashTable = (dict) => {
-    const result = {};
-    for (const { 0: options, 1: callback } of dict)
-        for (const option of options)
-            result[option] = callback;
-    return result;
-};
-let isHtml = false;
-const optionDict = toHashTable([
-    [
-        ['--latin', '-l'],
-        () => {
-            cfg.abc = dicts.alphabets.latin;
-        },
-    ],
-    [
-        ['--latin-ji', '-lj'],
-        () => {
-            cfg.abc = dicts.alphabets.latinJi;
-        },
-    ],
-    [
-        ['--arabic', '-a'],
-        () => {
-            cfg.abc = dicts.alphabets.arabic;
-        },
-    ],
-    [
-        ['--jrandom', '-jr'],
-        () => {
-            cfg.j = 'random';
-        },
-    ],
-    [
-        ['--jalways', '-ja'],
-        () => {
-            cfg.j = 'always';
-        },
-    ],
-    [
-        ['--no-escape-caps', '-nec'],
-        () => {
-            cfg.doEscapeCapitalized = false;
-        },
-    ],
-    [
-        ['--h'],
-        () => {
-            cfg.g = false;
-        },
-    ],
-    [
-        ['--no-variations', '-nv'],
-        () => {
-            cfg.variations = 'no';
-        },
-    ],
-    [
-        ['--first-variation', '-fv'],
-        () => {
-            cfg.variations = 'first';
-        },
-    ],
-    [
-        ['--no-color', '-nc'],
-        () => {
-            cfg.wrappers = null;
-        },
-    ],
-    [
-        ['--html', '-html'],
-        () => {
-            isHtml = true;
-            cfg.wrappers = htmlConfigOptions.wrappers;
-        },
-    ],
-    [
-        ['--alphabet-only', '-abc'],
-        () => {
-            mode = 'alphabetic';
-        },
-    ],
-    [
-        ['--phonetic', '-ph'],
-        () => {
-            mode = 'phonetic';
-        },
-    ],
-]);
-let currOption;
-process.argv.reverse();
-while ((currOption = process.argv.pop())) {
-    if (currOption in optionDict) {
-        optionDict[currOption]();
-    }
-    else {
-        process.argv.push(currOption);
-        break;
-    }
-}
-let text = '';
-if (process.argv.length) {
-    text = process.argv.reverse().join(' ');
-}
-else {
-    const chunks = [];
-    let length = 0;
-    if (process.stdin.isTTY) {
-        printWithPrefix('Enter the text');
-        for await (const chunk of process.stdin) {
-            chunks.push(chunk);
-            length += chunk.length;
-            if (chunk.includes('\n'))
-                break;
-        }
-    }
-    else {
-        for await (const chunk of process.stdin) {
-            chunks.push(chunk);
-            length += chunk.length;
-        }
-    }
-    text = Buffer.concat(chunks, length).toString();
-}
-cfg = new TaraskConfig(isHtml
-    ? {
-        ...htmlConfigOptions,
-        ...cfg,
-    }
-    : cfg);
-if (process.stdout.write(pipelines[mode](text, cfg) + '\n')) {
-    process.exit(0);
-}
-else {
-    process.stdout.once('drain', () => {
-        process.exit(0);
-    });
-}