scancscode 1.0.42 → 1.0.46

package/src/CsvAutoTranslator.ts

@@ -1,261 +1,333 @@
-import * as crypto from "crypto";
-import * as https from "https";
-import { CSVUtils } from "./CSVUtils";
-
-interface NiuTransResponse {
-    tgtList: Array<{
-        from: string;
-        to: string;
-        tgtText: string;
-        srcText?: string;
-        errorCode?: string;
-        errorMsg?: string;
-    }>;
-    resultCode: string;
-    resultMsg: string;
-}
-
-export class CsvAutoTranslator {
-    private apiKey: string;
-    private appId: string;
-    private apiUrl: string = "https://api.niutrans.com/v2/text/translate/array";
-    private maxBatchSize: number = 5;
-    private readonly MAX_BATCH_TEXT_COUNT: number = 50;
-    private readonly MAX_BATCH_JSON_SIZE: number = 4900;
-
-    constructor(apiKey: string, appId: string) {
-        this.apiKey = apiKey;
-        this.appId = appId;
-    }
-
-    private generateAuthStr(params: Record<string, string>): string {
-        const sortedKeys = Object.keys(params).sort();
-        const paramStr = sortedKeys
-            .filter(key => params[key] !== "" && params[key] !== undefined && params[key] !== null)
-            .map(key => `${key}=${params[key]}`)
-            .join("&");
-        const fullStr = `apikey=${this.apiKey}&${paramStr}`;
-        return crypto.createHash("md5").update(fullStr).digest("hex");
-    }
-
-    private async requestTranslate(texts: string[], from: string, to: string): Promise<NiuTransResponse> {
-        const timestamp = Date.now().toString();
-        const params: Record<string, string> = {
-            from: from,
-            to: to,
-            appId: this.appId,
-            timestamp: timestamp
-        };
-        const authStr = this.generateAuthStr(params);
-        const postData = JSON.stringify({
-            ...params,
-            srcText: texts,
-            authStr: authStr
-        });
-        const url = new URL(this.apiUrl);
-        const options: https.RequestOptions = {
-            hostname: url.hostname,
-            port: 443,
-            path: url.pathname,
-            method: "POST",
-            headers: {
-                "Content-Type": "application/json",
-                "Content-Length": Buffer.byteLength(postData)
-            }
-        };
-        return new Promise((resolve, reject) => {
-            const req = https.request(options, (res) => {
-                let data = "";
-                res.on("data", (chunk) => {
-                    data += chunk;
-                });
-                res.on("end", async () => {
-                    try {
-                        let ret = JSON.parse(data);
-                        await res.destroy();
-                        resolve(ret);
-                    } catch (e) {
-                        reject(e);
-                    }
-                });
-            });
-            req.on("error", reject);
-            req.write(postData);
-            req.end();
-        });
-    }
-
-    private async translateBatch(texts: string[], from: string, to: string): Promise<string[]> {
-        const response = await this.requestTranslate(texts, from, to);
-        if (response.resultCode !== "200") {
-            throw new Error(`翻译失败: ${response.resultMsg}`);
-        }
-        return response.tgtList.map(item => item.tgtText || "");
-    }
-
-    private estimateSingleTextJsonSize(text: string, isSingle: boolean): number {
-        return Buffer.byteLength(JSON.stringify(text)) + (isSingle ? 0 : 1);
-    }
-
-    estimateJsonSize(texts: string[]): number {
-        return Buffer.byteLength(JSON.stringify(texts));
-    }
-
-    smartBatch(texts: string[], batches?: string[][]): string[][] {
-        const resultBatches = batches || [];
-        let startIndex = 0;
-        while (startIndex < texts.length) {
-            let endIndexMax = Math.min(startIndex + this.MAX_BATCH_TEXT_COUNT, texts.length);
-            let batchByteLength = 2;
-            let currentBatchCount = 0;
-            // let currentBatch = texts.slice(startIndex, endIndexMax);
-            for (let i = startIndex; i < endIndexMax; i++) {
-                const text = texts[i];
-                const textSize = Buffer.byteLength(text) + (i >= 1 ? 1 : 0);
-                currentBatchCount++;
-                batchByteLength += textSize;
-                if (batchByteLength + textSize > this.MAX_BATCH_JSON_SIZE) {
-                    break;
-                }
-            }
-            let endIndex = startIndex + currentBatchCount;
-            let currentBatch: string[] = texts.slice(startIndex, endIndex);
-            let currentSize = this.estimateJsonSize(currentBatch);
-            while (currentSize > this.MAX_BATCH_JSON_SIZE && currentBatch.length > 0) {
-                endIndex--;
-                let endText = texts[endIndex];
-                let endTextSize = this.estimateSingleTextJsonSize(endText, endIndex > startIndex + 1);
-                currentSize -= endTextSize;
-                currentBatch.pop();
-            }
-            if (currentBatch.length == 0 && startIndex < endIndexMax) {
-                console.error(`无法将文本分成合适的批次, 存在过长的文本, 起始索引: ${startIndex}, 结束索引: ${endIndex}`);
-                currentBatch = ['']
-                endIndex = startIndex + 1;
-            }
-            resultBatches.push(currentBatch);
-            startIndex = endIndex;
-        }
-        return resultBatches;
-    }
-
-    private async translateAll(texts: string[], from: string, to: string): Promise<string[]> {
-        const batches = this.smartBatch(texts);
-        const results: string[][] = new Array(batches.length);
-        const concurrencyLimit = this.maxBatchSize;
-        let index = 0;
-
-        let batchCount = 0;
-        const processBatch = async (): Promise<void> => {
-            while (index < batches.length) {
-                const batchIndex = index++;
-                const batch = batches[batchIndex];
-                batchCount++;
-                console.log(`batchCount++: ${batchCount}`)
-                const batchResults = await this.translateBatch(batch, from, to);
-                await new Promise(resolve => setTimeout(resolve, 1000));
-                batchCount--;
-                console.log(`batchCount--: ${batchCount}`)
-                results[batchIndex] = batchResults;
-            }
-        };
-
-        const workers: Promise<void>[] = [];
-        for (let i = 0; i < concurrencyLimit && i < batches.length; i++) {
-            workers.push(processBatch());
-        }
-        await Promise.all(workers);
-        // await processBatch()
-
-        return results.flat();
-    }
-
-    langMap: Map<string, string> = new Map([
-        ["zh_cn", "zh"],
-        ["zh_hk", "cht"],
-        ["en_us", "en"],
-    ])
-    async translateCsvRows(rows: string[][], fromLang: string, toLangs?: string[]): Promise<number> {
-        if (rows.length === 0) {
-            console.log("CSV文件为空");
-            return 0;
-        }
-        let header = rows[0];
-        let fromLangIndex = header.indexOf(fromLang);
-        if (fromLangIndex === -1) {
-            console.error(`未找到 ${fromLang} 列`);
-            return 0;
-        }
-        if (!this.langMap.has(fromLang)) {
-            console.error(`未找到 ${fromLang} 的目标语言`);
-            return 0;
-        }
-        let fromLang2 = this.langMap.get(fromLang);
-        if (fromLang2 == null) {
-            console.error(`未找到 ${fromLang} 的目标语言`);
-            return 0;
-        }
-        const needTranslateIndices: number[] = [];
-        const needTranslateTexts: string[] = [];
-        for (let i = 1; i < rows.length; i++) {
-            const row = rows[i];
-            if (!row[0] || row[0].trim() === "") {
-                continue;
-            }
-            if (row[fromLangIndex] && row[fromLangIndex].trim() !== "") {
-                continue;
-            }
-            needTranslateIndices.push(i);
-            let text = row[fromLangIndex]
-            if (text == null || text == "") {
-                text = row[0];
-            }
-            needTranslateTexts.push(text);
-        }
-        if (needTranslateTexts.length === 0) {
-            console.log("没有需要翻译的内容");
-            return 0;
-        }
-        console.log(`开始翻译 ${needTranslateTexts.length} 条内容...`);
-        for (let curLangIndex = 0; curLangIndex < header.length; curLangIndex++) {
-            let lang = header[curLangIndex];
-            lang = lang.trim().toLowerCase();
-            if (lang == "key") {
-                continue;
-            }
-            if (toLangs != undefined && !toLangs.includes(lang)) {
-                continue;
-            }
-            if (lang != fromLang && this.langMap.has(lang)) {
-                const toLang = this.langMap.get(lang);
-                if (toLang == null) {
-                    console.error(`未找到 ${lang} 的目标语言`);
-                    continue;
-                }
-                const translations = await this.translateAll(needTranslateTexts, fromLang2, toLang);
-                for (let j = 0; j < needTranslateIndices.length; j++) {
-                    const rowIndex = needTranslateIndices[j];
-                    while (rows[rowIndex].length < header.length) {
-                        rows[rowIndex].push("");
-                    }
-                    rows[rowIndex][curLangIndex] = translations[j];
-                }
-            }
-        }
-        return needTranslateTexts.length;
-    }
-    async translateCsv(filePath: string, outFilePath: string, fromLang: string = "auto", toLangs?: string[]): Promise<void> {
-        const csvUtils = new CSVUtils(filePath);
-        const rows = await csvUtils.parseCsv();
-        const translatedCount = await this.translateCsvRows(rows, fromLang, toLangs);
-        if (translatedCount > 0) {
-            await CSVUtils.writeCsv(outFilePath, rows);
-            console.log(`翻译完成，已更新 ${translatedCount} 条内容到 ${outFilePath}`);
-        }
-    }
-
-    public static async translateCsvWithLangs(filePath: string, outFilePath: string, fromLang: string, langs: string[], apiKey: string, appId: string) {
-        const translator = new CsvAutoTranslator(apiKey, appId);
-        await translator.translateCsv(filePath, outFilePath, fromLang, langs);
-    }
-}
+import * as crypto from "crypto";
+import * as https from "https";
+import { CSVUtils } from "./CSVUtils";
+
+interface NiuTransResponse {
+    tgtList: Array<{
+        from: string;
+        to: string;
+        tgtText: string;
+        srcText?: string;
+        errorCode?: string;
+        errorMsg?: string;
+    }>;
+    resultCode: string;
+    resultMsg: string;
+    errorCode: string;
+    errorMsg: string;
+}
+
+class OnlineTranslateResult {
+    public errorCode: string = "";
+    public errorMsg: string = "";
+    public isOk: boolean = false;
+    public translatedTexts: string[];
+
+    public constructor(errorCode: string, errorMsg: string, isOk: boolean, translatedTexts: string[]) {
+        this.errorCode = errorCode;
+        this.errorMsg = errorMsg;
+        this.isOk = isOk;
+        this.translatedTexts = translatedTexts;
+    }
+}
+
+class TranslateCSVResult {
+    public translateCount: number = 0;
+    public isOk: boolean = false;
+    public firstErrorCode: string = "";
+    public firstErrorMsg: string = "";
+
+    public constructor(translateCount: number, isOk: boolean, firstErrorCode: string, firstErrorMsg: string) {
+        this.translateCount = translateCount;
+        this.isOk = isOk;
+        this.firstErrorCode = firstErrorCode;
+        this.firstErrorMsg = firstErrorMsg;
+    }
+
+    public static Empty: TranslateCSVResult = new TranslateCSVResult(0, false, "", "");
+}
+
+export class CsvAutoTranslator {
+    private apiKey: string;
+    private appId: string;
+    private apiUrl: string = "https://api.niutrans.com/v2/text/translate/array";
+    private maxBatchSize: number = 5;
+    private readonly MAX_BATCH_TEXT_COUNT: number = 50;
+    private readonly MAX_BATCH_JSON_SIZE: number = 4900;
+
+    constructor(appId: string, apiKey: string) {
+        this.apiKey = apiKey;
+        this.appId = appId;
+    }
+
+    private generateAuthStr(params: Record<string, string>): string {
+        const sortedKeys = Object.keys(params).sort();
+        const paramStr = sortedKeys
+            .filter(key => params[key] !== "" && params[key] !== undefined && params[key] !== null)
+            .map(key => `${key}=${params[key]}`)
+            .join("&");
+        const fullStr = `apikey=${this.apiKey}&${paramStr}`;
+        return crypto.createHash("md5").update(fullStr).digest("hex");
+    }
+
+    private async requestTranslate(texts: string[], from: string, to: string): Promise<NiuTransResponse> {
+        const timestamp = Date.now().toString();
+        const params: Record<string, string> = {
+            from: from,
+            to: to,
+            appId: this.appId,
+            timestamp: timestamp
+        };
+        const authStr = this.generateAuthStr(params);
+        const postData = JSON.stringify({
+            ...params,
+            srcText: texts,
+            authStr: authStr
+        });
+        const url = new URL(this.apiUrl);
+        const options: https.RequestOptions = {
+            hostname: url.hostname,
+            port: 443,
+            path: url.pathname,
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                "Content-Length": Buffer.byteLength(postData)
+            }
+        };
+        return new Promise((resolve, reject) => {
+            const req = https.request(options, (res) => {
+                let data = "";
+                res.on("data", (chunk) => {
+                    data += chunk;
+                });
+                res.on("end", async () => {
+                    try {
+                        let ret = JSON.parse(data);
+                        await res.destroy();
+                        resolve(ret);
+                    } catch (e) {
+                        reject(e);
+                    }
+                });
+            });
+            req.on("error", reject);
+            req.write(postData);
+            req.end();
+        });
+    }
+
+    private async translateBatch(texts: string[], from: string, to: string): Promise<OnlineTranslateResult> {
+        const response = await this.requestTranslate(texts, from, to);
+        if (response.resultCode !== "200") {
+            let errTip = `翻译失败: ${response.errorCode}, ${response.errorMsg}, ${response.resultCode}, ${response.resultMsg}`;
+            console.error(errTip);
+            console.error("翻译失败内容:", texts);
+            let result = new OnlineTranslateResult(response.errorCode, response.errorMsg, false, new Array(texts.length).fill(""));
+            return result;
+        }
+        let result = new OnlineTranslateResult(response.resultCode, response.resultMsg, true, response.tgtList.map(item => item.tgtText || ""));
+        return result;
+    }
+
+    private estimateSingleTextJsonSize(text: string, isSingle: boolean): number {
+        return Buffer.byteLength(JSON.stringify(text)) + (isSingle ? 0 : 1);
+    }
+
+    estimateJsonSize(texts: string[]): number {
+        return Buffer.byteLength(JSON.stringify(texts));
+    }
+
+    smartBatch(texts: string[], batches?: string[][]): string[][] {
+        const resultBatches = batches || [];
+        let startIndex = 0;
+        while (startIndex < texts.length) {
+            let endIndexMax = Math.min(startIndex + this.MAX_BATCH_TEXT_COUNT, texts.length);
+            let batchByteLength = 2;
+            let currentBatchCount = 0;
+            // let currentBatch = texts.slice(startIndex, endIndexMax);
+            for (let i = startIndex; i < endIndexMax; i++) {
+                const text = texts[i];
+                const textSize = Buffer.byteLength(text) + (i >= 1 ? 1 : 0);
+                currentBatchCount++;
+                batchByteLength += textSize;
+                if (batchByteLength + textSize > this.MAX_BATCH_JSON_SIZE) {
+                    break;
+                }
+            }
+            let endIndex = startIndex + currentBatchCount;
+            let currentBatch: string[] = texts.slice(startIndex, endIndex);
+            let currentSize = this.estimateJsonSize(currentBatch);
+            while (currentSize > this.MAX_BATCH_JSON_SIZE && currentBatch.length > 0) {
+                endIndex--;
+                let endText = texts[endIndex];
+                let endTextSize = this.estimateSingleTextJsonSize(endText, endIndex > startIndex + 1);
+                currentSize -= endTextSize;
+                currentBatch.pop();
+            }
+            if (currentBatch.length == 0 && startIndex < endIndexMax) {
+                console.error(`无法将文本分成合适的批次, 存在过长的文本, 起始索引: ${startIndex}, 结束索引: ${endIndex}`);
+                currentBatch = ['']
+                endIndex = startIndex + 1;
+            }
+            resultBatches.push(currentBatch);
+            startIndex = endIndex;
+        }
+        return resultBatches;
+    }
+
+    private async translateAll(texts: string[], from: string, to: string): Promise<OnlineTranslateResult> {
+        const batches = this.smartBatch(texts);
+        const results: string[][] = new Array(batches.length);
+        const concurrencyLimit = this.maxBatchSize;
+        let index = 0;
+        let isOk = true;
+        let batchCount = 0;
+        let firstErrorCode = "";
+        let firstErrorMsg = "";
+        const processBatch = async (): Promise<void> => {
+            while (index < batches.length) {
+                const batchIndex = index++;
+                const batch = batches[batchIndex];
+                batchCount++;
+                console.log(`batchCount++: ${batchCount}`)
+                const batchResult = await this.translateBatch(batch, from, to);
+                if (!batchResult.isOk) {
+                    isOk = false;
+                    if (firstErrorCode === "") {
+                        firstErrorCode = batchResult.errorCode;
+                        firstErrorMsg = batchResult.errorMsg;
+                    }
+                }
+                await new Promise(resolve => setTimeout(resolve, 1000));
+                batchCount--;
+                console.log(`batchCount--: ${batchCount}`)
+                results[batchIndex] = batchResult.translatedTexts;
+            }
+        };
+
+        const workers: Promise<void>[] = [];
+        for (let i = 0; i < concurrencyLimit && i < batches.length; i++) {
+            workers.push(processBatch());
+        }
+        await Promise.all(workers);
+        // await processBatch()
+
+        let translatedTexts = results.flat();
+        let result = new OnlineTranslateResult(firstErrorCode, firstErrorMsg, isOk, translatedTexts);
+        return result;
+    }
+
+    langMap: Map<string, string> = new Map([
+        ["zh_cn", "zh"],
+        ["zh_hk", "cht"],
+        ["en_us", "en"],
+    ])
+    async translateCsvRows(rows: string[][], fromLang: string, toLangs?: string[]): Promise<TranslateCSVResult> {
+        if (rows.length === 0) {
+            console.log("CSV文件为空");
+            return TranslateCSVResult.Empty;
+        }
+        let header = rows[0];
+        let fromLangIndex = header.indexOf(fromLang);
+        if (fromLangIndex === -1) {
+            console.error(`未找到 ${fromLang} 列`);
+            return TranslateCSVResult.Empty;
+        }
+        if (!this.langMap.has(fromLang)) {
+            console.error(`未找到 ${fromLang} 的目标语言`);
+            return TranslateCSVResult.Empty;
+        }
+        let fromLang2 = this.langMap.get(fromLang);
+        if (fromLang2 == null) {
+            console.error(`未找到 ${fromLang} 的目标语言`);
+            return TranslateCSVResult.Empty;
+        }
+        let isOk = true;
+        let translateCount = 0;
+        let firstErrorCode = "";
+        let firstErrorMsg = "";
+        for (let curLangIndex = 0; curLangIndex < header.length; curLangIndex++) {
+            let lang = header[curLangIndex];
+            lang = lang.trim().toLowerCase();
+            if (lang == "key") {
+                continue;
+            }
+            if (toLangs != undefined && !toLangs.includes(lang)) {
+                continue;
+            }
+            if (!(lang != fromLang && this.langMap.has(lang))) {
+                continue;
+            }
+            const needTranslateIndices: number[] = [];
+            const needTranslateTexts: string[] = [];
+            for (let i = 1; i < rows.length; i++) {
+                const row = rows[i];
+                if (!row[0] || row[0].trim() === "") {
+                    continue;
+                }
+                if (row[curLangIndex] && row[curLangIndex].trim() !== "") {
+                    continue;
+                }
+                let text = row[fromLangIndex]
+                if (text == null || text == "") {
+                    text = row[0];
+                }
+                if (text == null || text === "") {
+                    continue;
+                }
+                needTranslateIndices.push(i);
+                needTranslateTexts.push(text);
+            }
+            if (needTranslateTexts.length === 0) {
+                console.log("没有需要翻译的内容");
+                return TranslateCSVResult.Empty;
+            }
+            console.log(`开始翻译 ${needTranslateTexts.length} 条内容...`);
+            const toLang = this.langMap.get(lang);
+            if (toLang == null) {
+                console.error(`未找到 ${lang} 的目标语言`);
+                continue;
+            }
+            const translateResult = await this.translateAll(needTranslateTexts, fromLang2, toLang);
+            isOk = isOk && translateResult.isOk;
+            if (!translateResult.isOk) {
+                if (firstErrorCode === "") {
+                    firstErrorCode = translateResult.errorCode;
+                    firstErrorMsg = translateResult.errorMsg;
+                }
+                console.error(`翻译 ${lang} 失败: ${translateResult.errorCode} ${translateResult.errorMsg}`);
+                continue;
+            }
+            let translations = translateResult.translatedTexts;
+            for (let j = 0; j < needTranslateIndices.length; j++) {
+                const rowIndex = needTranslateIndices[j];
+                while (rows[rowIndex].length < header.length) {
+                    rows[rowIndex].push("");
+                }
+                rows[rowIndex][curLangIndex] = translations[j];
+            }
+            translateCount += needTranslateTexts.length;
+        }
+        let result = new TranslateCSVResult(translateCount, isOk, firstErrorCode, firstErrorMsg);
+        return result;
+    }
+    async translateCsv(filePath: string, outFilePath: string, fromLang: string = "auto", toLangs?: string[]): Promise<TranslateCSVResult> {
+        const csvUtils = new CSVUtils(filePath);
+        const rows = await csvUtils.parseCsv();
+        const translatedResult = await this.translateCsvRows(rows, fromLang, toLangs);
+        if (translatedResult.isOk && translatedResult.translateCount > 0) {
+            await CSVUtils.writeCsv(outFilePath, rows);
+            console.log(`翻译完成，已更新 ${translatedResult.translateCount} 条内容到 ${outFilePath}`);
+        } else {
+            console.log(`翻译完成，未更新任何内容到 ${outFilePath}`);
+        }
+        return translatedResult;
+    }
+
+    public static async translateCsvWithLangs(appId: string, apiKey: string, filePath: string, outFilePath: string, fromLang: string, langs?: string[]) {
+        const translator = new CsvAutoTranslator(appId, apiKey);
+        let result = await translator.translateCsv(filePath, outFilePath, fromLang, langs);
+        return result;
+    }
+}

package/src/RunConvert.ts

@@ -1,3 +1,3 @@
-import { CmdExecutor } from "./CmdExecutor";
-
-CmdExecutor.runConvertWithCmdOptions()
+import { CmdExecutor } from "./CmdExecutor";
+
+CmdExecutor.runConvertWithCmdOptions()

package/src/RunSlimLangs.ts

@@ -1,3 +1,3 @@
-import { CmdExecutor } from "./CmdExecutor";
-
-CmdExecutor.runSlimCsvWithLangs()
+import { CmdExecutor } from "./CmdExecutor";
+
+CmdExecutor.runSlimCsvWithLangs()

package/src/RunTranslateCSV.ts

@@ -1,3 +1,3 @@
-import { CmdExecutor } from "./CmdExecutor";
-
-CmdExecutor.runTranslateCsvWithCmdOptions()
+import { CmdExecutor } from "./CmdExecutor";
+
+CmdExecutor.runTranslateCsvWithCmdOptions()