td-octopus 0.1.7 → 0.1.9

package/README.md

@@ -23,6 +23,10 @@ otp <cmd> [-args]
   otp init                        init 初始化配置文件
   otp translate                   translate 通过对比zh-CN目录,获取各语言未翻译的
                                   部分,并生成excel文件
+  otp back                        回滚国际化翻译
+  otp download                    下载思南翻译到本地-需要配置downloadUrl
+  otp reduce                      校验数据，删除多余的翻译
+  otp knowledge                   knowledge 导出所有文件,并生成知识库的excel文件
 
 选项：
   -v, --version  显示版本号                                               [布尔]
@@ -43,6 +47,7 @@ copyright 2022 同盾
   "googleApiKey": "", // google翻译 这期没做
   "baiduApiKey": { "appId": "", "appKey": "" }, // 百度翻译 这期没做
   "baiduLangMap": { "en-US": "en" }, // 百度翻译 这期没做
+  "difyApiKey": { "appUrl": "", "appKey": "" }, // dify翻译
   "translateOptions": { "concurrentLimit": 10, "requestOptions": {} }, // google翻译 这期没做
   "fileSuffix": [".ts", ".js", ".vue", ".jsx", ".tsx"], // 支持符合JS语法的后缀名
   "defaultTranslateKeyApi": "Pinyin", // 默认生成的JSON key 使用拼音前5个

package/cmds/knowledge.js

@@ -0,0 +1,9 @@
+const { doExportKnowledge } = require('../src/export');
+
+exports.command = 'knowledge';
+
+exports.describe = 'knowledge 导出所有文件,并生成知识库的excel文件';
+
+exports.handler = () => {
+  doExportKnowledge()
+}

package/cmds/reduce.js

@@ -0,0 +1,9 @@
+const { main } = require('../src/reduce');
+
+exports.command = 'reduce';
+
+exports.describe = 'reduce 将项目中多余的key删除';
+
+exports.handler = () => {
+    main()
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "td-octopus",
-  "version": "0.1.7",
+  "version": "0.1.9",
   "description": "I18N tool",
   "author": "Anthony Li",
   "bin": {

package/src/export/index.js

@@ -5,6 +5,7 @@
  * @LastEditors: 郑泳健
  * @LastEditTime: 2023-06-12 11:33:34
  */
+const _ = require('lodash');
 const {
     syncFiles,
     otpPath,
@@ -25,7 +26,6 @@ function doExport() {
     (async () => {
         const otpConfig = getProjectConfig()
         const distLang = otpConfig && otpConfig.distLangs
-        const baiduApiKey = otpConfig && otpConfig.baiduApiKey
 
         if (!Array.isArray(distLang)) {
             console.log(`请配置${OCTOPUS_CONFIG_FILE}里面的distLangs`)
@@ -43,20 +43,57 @@ function doExport() {
             const langFlat = flatObject(currentLangMap);
             spinner.start('正在同步文件的key')
             // 删除掉多余的key，增加新的key，同时提取没有翻译过的key的列表
-            const { fileKeyValueList, addList, allList } = await getAdjustLangObjAndAddList(lang, langFlat, zhCNflat, baiduApiKey, spinner);
+            const { fileKeyValueList, allList } = await getAdjustLangObjAndAddList({ lang, langObj: langFlat, zhCNObj: zhCNflat, spinner });
             spinner.succeed('同步文件的key成功')
             // 重写文件
             rewriteFiles(fileKeyValueList, lang);
             spinner.start(`正在生成${lang} excel`)
             // 生成excel
-            generateExcel(allList, otpPath + '/' + lang, lang);
+            generateExcel(_.sortBy(allList, it=>it[1]), otpPath + '/' + lang, lang);
             spinner.succeed(`生成${lang} excel成功`)
         }
 
-        spinner.stop('translate成功')
+        spinner.stop('export成功')
+    })()
+}
+
+function doExportKnowledge() {
+    (async () => {
+        const otpConfig = getProjectConfig()
+        const distLang = otpConfig && otpConfig.distLangs
+
+        if (!Array.isArray(distLang)) {
+            console.log(`请配置${OCTOPUS_CONFIG_FILE}里面的distLangs`)
+            return;
+        }
+        spinner.start('正在同步文件')
+        await syncFiles(distLang);
+        spinner.succeed('同步文件成功')
+
+        const zhCN = syncLang('zh-CN');
+        const zhCNflat = flatObject(zhCN);
+
+        for (const lang of distLang) {
+            const currentLangMap = syncLang(lang);
+            const langFlat = flatObject(currentLangMap);
+            spinner.start('正在同步文件的key')
+            // 删除掉多余的key，增加新的key，同时提取没有翻译过的key的列表
+            const { fileKeyValueList, allList } = await getAdjustLangObjAndAddList({ lang, langObj: langFlat, zhCNObj: zhCNflat, spinner });
+            spinner.succeed('同步文件的key成功')
+            // 重写文件
+            rewriteFiles(fileKeyValueList, lang);
+            spinner.start(`正在生成${lang} excel`)
+            // 生成excel
+            const formatData = _.uniqBy(allList.filter(arr => !!arr[3]).map(arr => ([arr[1], arr[3]])), arr => JSON.stringify(arr));
+            generateExcel(_.sortBy(formatData, it => it[0]), otpPath + '/' + lang, lang, true);
+            spinner.succeed(`生成${lang} excel成功`)
+        }
+
+        spinner.stop('Export Knowledge 成功')
     })()
 }
 
 module.exports = {
-    doExport
+    doExport,
+    doExportKnowledge
 }

package/src/extract/replace.js

@@ -9,7 +9,6 @@ const {
   findMatchKey,
   findMatchValue,
   getLangDir,
-  translateKeyText,
   getProjectConfig,
   prettierFile
 } = require('../utils');

package/src/reduce/index.js

@@ -0,0 +1,79 @@
+/*
+ * @Descripttion: 删除多余的key
+ * @Author: 郑泳健
+ * @Date: 2024-12-12 15:00:24
+ * @LastEditors: 郑泳健
+ * @LastEditTime: 2024-12-12 15:00:33
+ */
+const path = require('path')
+const fs = require('fs')
+const shell = require('shelljs');
+const { getSpecifiedFiles, isFile } = require('../utils/file')
+const syncLang = require('../utils/syncLang')
+const { flatObject, rewriteFiles, getFileKeyValueList, getAdjustLangObjAndAddList } = require('../utils/translate');
+const { failInfo, highlightText } = require('../utils/colors');
+const { getProjectConfig } = require('../utils/index')
+const ora = require('ora');
+
+const CONFIG = getProjectConfig();
+
+const spinner = ora('开始reduce');
+
+// 递归读取文件夹中所有 .js 文件内容
+function readJsFiles(folderPath) {
+    let jsContent = '';
+  
+    const files = fs.readdirSync(folderPath); // 读取文件夹内容
+    for (const file of files) {
+      const fullPath = path.join(folderPath, file);
+      const stats = fs.statSync(fullPath); // 获取文件或文件夹状态
+  
+      if (stats.isDirectory()) {
+        // 如果是文件夹，递归处理
+        jsContent += readJsFiles(fullPath);
+      } else if (path.extname(fullPath) === '.js') {
+        // 如果是 .js 文件，读取文件内容
+        const content = fs.readFileSync(fullPath, 'utf-8');
+        jsContent += `\n/* File: ${fullPath} */\n${content}\n`;
+      }
+    }
+  
+    return jsContent;
+  }
+
+// 同步不同的语言包
+function main() {
+    (async () => {
+        const distLang = Array.isArray(CONFIG.distLangs) ? CONFIG.distLangs : []
+
+        // 删除中文下躲雨的key
+        const zhCN = syncLang('zh-CN');
+        const zhCNFlat = flatObject(zhCN);
+        // 当前项目所有的key
+        const keys = Object.keys(zhCNFlat)
+        const totalText = readJsFiles(path.resolve(process.cwd(), 'src'))
+        let result = {}
+        for(let i in zhCNFlat){
+            if(totalText.includes(i)){
+                result[i] = zhCNFlat[i]
+            }
+        }
+        
+        rewriteFiles(getFileKeyValueList(result), 'zh-CN')
+
+        for (const lang of distLang) {
+            const currentLangMap = syncLang(lang);
+            const langFlat = flatObject(currentLangMap);
+            spinner.start(`正在清理${lang}下多余的key`)
+            // 删除掉多余的key，增加新的key，同时提取没有翻译过的key的列表
+            const { fileKeyValueList } = await getAdjustLangObjAndAddList({ lang, langObj: langFlat, zhCNObj: result, spinner });
+            spinner.succeed(`已完成清理${lang}下多余的key`)
+            // 重写文件
+            rewriteFiles(fileKeyValueList, lang);
+        }
+    })()
+}
+
+module.exports = {
+    main
+}

package/src/resource/extract.js

@@ -4,7 +4,7 @@ const path = require('path');
 const Parser = require('properties');
 
 const { getSpecifiedFiles, readFile, writeFile, isFile, isDirectory } = require('../utils/file');
-const { translateText, findMatchKey, findMatchValue, translateKeyText, getProjectConfig } = require('../utils');
+const { translateText, findMatchKey, findMatchValue, getProjectConfig } = require('../utils');
 const { successInfo, failInfo, highlightText } = require('../utils/colors');
 
 const CONFIG = getProjectConfig();

package/src/translate/index.js

@@ -5,6 +5,7 @@
  * @LastEditors: 郑泳健
  * @LastEditTime: 2023-06-12 11:33:51
  */
+const _ = require('lodash');
 const {
     syncFiles,
     otpPath,
@@ -26,6 +27,7 @@ function translate() {
         const otpConfig = getProjectConfig()
         const distLang = otpConfig && otpConfig.distLangs
         const baiduApiKey = otpConfig && otpConfig.baiduApiKey
+        const difyApiKey = otpConfig && otpConfig.difyApiKey
 
         if (!Array.isArray(distLang)) {
             console.log(`请配置${OCTOPUS_CONFIG_FILE}里面的distLangs`)
@@ -41,16 +43,16 @@ function translate() {
         for (const lang of distLang) {
             const currentLangMap = syncLang(lang);
             const langFlat = flatObject(currentLangMap);
-            spinner.start('正在同步文件的key')
+            spinner.start('正在同步文件的key');
             // 删除掉多余的key，增加新的key，同时提取没有翻译过的key的列表
-            const { fileKeyValueList, addList } = await getAdjustLangObjAndAddList(lang, langFlat, zhCNflat, baiduApiKey, spinner);
-            spinner.succeed('同步文件的key成功')
+            const { fileKeyValueList, addList } = await getAdjustLangObjAndAddList({ lang, langObj: langFlat, zhCNObj: zhCNflat, baiduApiKey, difyApiKey, spinner });
+            spinner.succeed('同步文件的key成功');
             // 重写文件
             rewriteFiles(fileKeyValueList, lang);
-            spinner.start(`正在生成${lang} excel`)
+            spinner.start(`正在生成${lang} excel`);
             // 生成excel
-            generateExcel(addList, otpPath + '/' + lang, lang);
-            spinner.succeed(`生成${lang} excel成功`)
+            generateExcel(_.sortBy(addList, it=>it[1]), otpPath + '/' + lang, lang);
+            spinner.succeed(`生成${lang} excel成功`);
         }
 
         spinner.stop('translate成功')

package/src/utils/const.js

@@ -12,6 +12,10 @@ const PROJECT_CONFIG = {
       appId: '',
       appKey: ''
     },
+    difyApiKey: {
+      apiUrl: '',
+      appKey: ''
+    },
     baiduLangMap: {
       ['en-US']: 'en'
     },

package/src/utils/index.js

@@ -236,44 +236,6 @@ function flatten(obj, prefix = '') {
   return ret;
 }
 
-/**
- * 获取翻译源类型
- */
-async function getTranslateOriginType() {
-  const { googleApiKey, baiduApiKey } = getProjectConfig();
-  let translateType = ['Google', 'Baidu'];
-  if (!googleApiKey) {
-    translateType = translateType.filter(item => item !== 'Google');
-  }
-  if (!baiduApiKey || !baiduApiKey.appId || !baiduApiKey.appKey) {
-    translateType = translateType.filter(item => item !== 'Baidu');
-  }
-  if (translateType.length === 0) {
-    console.log('请配置 googleApiKey 或 baiduApiKey ');
-    return {
-      pass: false,
-      origin: ''
-    };
-  }
-  if (translateType.length == 1) {
-    return {
-      pass: true,
-      origin: translateType[0]
-    };
-  }
-  const { origin } = await inquirer.prompt({
-    type: 'list',
-    name: 'origin',
-    message: '请选择使用的翻译源',
-    default: 'Google',
-    choices: ['Google', 'Baidu']
-  });
-  return {
-    pass: true,
-    origin: origin
-  };
-}
-
 /**
  * 进度条加载
  * @param text
@@ -321,7 +283,6 @@ module.exports = {
   findMatchValue,
   flatten,
   lookForFiles,
-  getTranslateOriginType,
   translateKeyText,
   spining,
   prettierFile

package/src/utils/translate.js

@@ -130,7 +130,9 @@ function getFileKeyValueList(adjustLangObj) {
  * @param {*} zhCNObj zh-CN 的key/value
  * @returns { fileKeyValueList: [{fileName: a, value: {"b.c": xx}}], addList: [["a.b.c": "dd"]] }
  */
-async function getAdjustLangObjAndAddList(lang, langObj = {}, zhCNObj = {}, baiduApiKey, spinner) {
+async function getAdjustLangObjAndAddList({
+    lang, langObj = {}, zhCNObj = {}, baiduApiKey, difyApiKey, spinner
+}) {
     const langObjKeys = Object.keys(langObj);
     // 调整后的语言包key/value
     const adjustLangObj = {};
@@ -154,10 +156,8 @@ async function getAdjustLangObjAndAddList(lang, langObj = {}, zhCNObj = {}, baid
 
         adjustLangObj[key] = langObj[key] || zhCNObj[key];
     }
-    if (baiduApiKey) {
-        spinner.text = `当前配置了百度翻译，预计翻译时间需要${(needAddList.length / 60)?.toFixed(2)}分钟，如果等不及，请先去掉百度翻译配置`
-    }
-    const addList = await combinText(needAddList, lang, spinner);
+
+    const addList = await combineText({ needAddList, lang, spinner, baiduApiKey, difyApiKey });
 
     return {
         fileKeyValueList: getFileKeyValueList(adjustLangObj),
@@ -171,20 +171,27 @@ async function getAdjustLangObjAndAddList(lang, langObj = {}, zhCNObj = {}, baid
  * 因为百度免费翻译有时候会抽风，会导致翻译结果出错，为了减少没被翻译的，所以现在设置一次性翻译200字
  * @param {*} needAddList
  */
-async function combinText(needAddList, lang, spinner) {
-    const otpConfig = getProjectConfig()
-    const { baiduApiKey, baiduLangMap } = otpConfig || {}
-    const { appId, appKey } = baiduApiKey || {}
-    const toLang = baiduLangMap?.[lang] || '';
-
+async function combineText({ needAddList, lang, spinner, baiduApiKey, difyApiKey }) {
     if (!Array.isArray(needAddList)) {
         return []
     }
+    const otpConfig = getProjectConfig();
+    const { baiduLangMap } = otpConfig || {};
+    const _difyApiKey = difyApiKey || {};
+    if (_difyApiKey.appUrl && _difyApiKey.appKey) {
+        spinner.text = `当前配置了dify翻译，预计翻译时间需要${(needAddList.length / 9)?.toFixed(2)}分钟，如果等不及，请先去掉dify翻译配置`;
+        const backList = await difyTranslate({ needAddList, spinner, appUrl: _difyApiKey.appUrl, appKey: _difyApiKey.appKey });
+        return backList;
+    }
+
+    const { appId, appKey } = baiduApiKey || {}
+    const toLang = baiduLangMap?.[lang] || '';
 
     // 如果不配置百度翻译就直接返回
     if (!appId || !appKey || !toLang) {
         return needAddList;
     }
+    spinner.text = `当前配置了百度翻译，预计翻译时间需要${(needAddList.length / 60)?.toFixed(2)}分钟，如果等不及，请先去掉百度翻译配置`
     // 分组
     const groupList = groupByLength(needAddList, 200)
     // 分组翻译结果
@@ -196,6 +203,76 @@ async function combinText(needAddList, lang, spinner) {
     })
 }
 
+/**
+ * 使用Dify API进行批量翻译
+ * @param {Object} params - 参数对象
+ * @param {Array} params.needAddList - 需要翻译的文本列表，格式为二维数组
+ * @param {Object} params.spinner - 进度显示对象
+ * @param {string} params.appUrl - Dify API地址
+ * @param {string} params.appKey - Dify API密钥
+ * @param {number} [params.maxLength=10] - 最大并发请求数
+ * @returns {Promise} 返回Promise，resolve时返回翻译结果数组
+ * @description 该函数通过并发请求Dify翻译API，并缓存翻译结果，同时更新进度显示
+ */
+function difyTranslate({ needAddList, spinner, appUrl, appKey, maxLength = 10 }) {
+    return new Promise((resolve, reject) => {
+        const cacheMap = {};
+        const textList = needAddList.map(i => i[1]);
+        // 当前任务的索引
+        let index = -1;
+        // 当前正在执行的任务数
+        let runningTasks = 0;
+        // 已完成的任务数
+        let completedTasks = 0;
+        const processTask = () => {
+            index++;
+            const text = textList[index];
+            if (!text) {
+                if (runningTasks === 0) {
+                    resolve(needAddList.map(arr => {
+                        arr[3] = cacheMap[arr[1]] || '';
+                        return arr;
+                    }))
+                }
+                return
+            }
+            runningTasks++;
+            fetch(`${appUrl}/workflows/run`, {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                    'Authorization': `Bearer ${appKey}`
+                },
+                body: JSON.stringify({
+                    inputs: {
+                        query: text,
+                    },
+                    user: 'admin',
+                    response_mode: 'blocking'
+                })
+            }).then(res => res.json())
+            .then(res => {
+                const back = res?.data?.outputs?.result;
+                if (back && !/[\u4e00-\u9fa5]/.test(back)) {
+                    cacheMap[text] = back;
+                }
+                completedTasks++;
+                spinner.text = `Dify翻译进度: ${completedTasks}/${textList.length}, 当前并发数: ${maxLength}`;
+            })
+            .catch((error)=>{
+                console.log('Dify翻译失败: ', error);
+            })
+            .finally(() => {
+                runningTasks--;
+                processTask();
+            });
+        };
+        for (let i = 0; i < maxLength; i++) {
+            processTask();
+        }
+    })
+}
+
 /**
  * 对分组的结果进行翻译
  * @param {*} groupList
@@ -235,7 +312,7 @@ async function getTransformResultList(groupList, appId, appKey, fromLang, toLang
  * @returns
  */
 function groupByLength(groupList, max) {
-    const list = [[]]
+    const list = []
     let str = ''
     // 变成${max}个字符一组的数组，用于一次百度翻译
     groupList.forEach((it) => {
@@ -295,14 +372,14 @@ function getDistRst(adjustLangObj) {
  * @param {*} addList 需要翻译的列表
  * @param {*} path 生成的路径
  */
-function generateExcel(addList, path, lang) {
-    const excleData = [['需要翻译的字段', '中文', '百度翻译', '人工翻译'], ...addList];
+function generateExcel(addList, path, lang, isKnowledge) {
+    const excelData = [isKnowledge ? ['中文', '人工翻译'] : ['需要翻译的字段', '中文', '百度翻译', '人工翻译'], ...addList];
 
     const options = {
-        '!cols': [{ wpx: 100 }, { wpx: 100 }, { wpx: 100 }, { wpx: 100 }]
+        '!cols': isKnowledge ? [{ wpx: 100 }, { wpx: 100 }] : [{ wpx: 100 }, { wpx: 100 }, { wpx: 100 }, { wpx: 100 }]
     };
 
-    const worksheet = XLSX.utils.aoa_to_sheet(excleData);
+    const worksheet = XLSX.utils.aoa_to_sheet(excelData);
     worksheet['!cols'] = options['!cols'];
 
     const workbook = XLSX.utils.book_new();