tools_batch_files 1.0.33 → 1.0.35

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tools_batch_files",
-  "version": "1.0.33",
+  "version": "1.0.35",
   "description": "批处理视频工具",
   "keywords": [
     "utils",

package/src/audioFn/audioBatch.js

@@ -13,6 +13,7 @@ const {
   ensureDirSync,
   isExist,
   readTxt,
+  removeDirectory,
 } = require("../../utils/settleFiles");
 const ffmpeg = require("fluent-ffmpeg");
 const fs = require("fs");
@@ -28,12 +29,22 @@ const queueCount = 10;
 //起始任务下标
 let taskIndex = 0;
 
+//已完成数量
+let completeCount = 0;
+
+//错误文件重试次数
+let retryError = 0;
+
 //执行目录
 const exeDir = __dirname;
 //当前工作目录
 const workDir = process.cwd();
-// 表格目录
-const excelDir = path.join(workDir, "excel", "excel.xlsx");
+
+// 表格目录，从excel文件夹中查找
+const excelFiles = fs
+  .readdirSync(path.join(workDir, "excel"))
+  .find((file) => file.endsWith(".xlsx"));
+const excelDir = path.join(workDir, "excel", excelFiles);
 
 const ORIGIN_FILE_DIR = "audio";
 const WM_AUDIO_DIR = "watermark_audio";
@@ -43,7 +54,13 @@ const ZIP_FILES_DIR = "zip";
 //拷贝 - 存放上传失败的文件
 const Error_Files_Dir = "failed_audio";
 //水印
-const watermarkAudioPath = path.join(__dirname, "..", "vocal_print", "mz.mp3");
+const watermarkAudioPath = path.join(
+  __dirname,
+  "..",
+  "..",
+  "vocal_print",
+  "mz.mp3"
+);
 const longWatermarkAudioPath = path.join(
   __dirname,
   "..",
@@ -72,7 +89,7 @@ const watermarkAudio = (indexFilePath, originFilePath, fileName, duration) => {
     `[1:a]atrim=0:duration=${duration},volume=1.5[a1]`, // 对声纹音频流进行截取操作，并设置音量，重命名为a1
     `[a0][a1]amix=inputs=2[a]`, // 合并两个音频流为一个输出流，重命名为a
     `[a]volume=3.0`, // 设置输出音频的音量
-  ].join(";");
+  ];
 
   if (duration <= 5) {
     //直接合并，放在末尾，下次补上
@@ -103,7 +120,7 @@ const watermarkAudio = (indexFilePath, originFilePath, fileName, duration) => {
       .input(originFilePath)
       .input(vocalPath) // 声纹音频文件
 
-      .complexFilter(complexFilter)
+      .complexFilter(complexFilter.join(";"))
       .output(outputFile)
       .on("error", (err) => {
         logger("添加声纹出错: " + err);
@@ -201,7 +218,7 @@ const archiveZip = (fileName, inputPath, originFilePath) => {
  */
 const getMetadata = async (
   originFilePath,
-  { rowFileName, title, keyword, index }
+  { rowFileName, title, keyword, anotherId }
 ) => {
   //源音频数据
   const photoMetadataComand = ffmpeg(originFilePath);
@@ -219,7 +236,7 @@ const getMetadata = async (
     duration: "",
     tag_ids: "3",
     source_from: "sound_1",
-    anotherId: rowFileName + index,
+    anotherId,
   };
 
   await new Promise((resolve, reject) => {
@@ -247,7 +264,7 @@ const getMetadata = async (
 /**
  * 接口
  */
-const postData = (dataParams, indexFilePath) => {
+const postData = (dataParams, indexFilePath, index) => {
   const formData = new FormData();
 
   const zipFiles = fs
@@ -264,11 +281,11 @@ const postData = (dataParams, indexFilePath) => {
     }
   }
 
-  logger("等待接口返回结果……");
+  logger(`第${index}条等待接口返回结果……`);
 
   // return axios.post("http://192.168.102.61:9999/upload/sound", formData, {
-  // return axios.post("http://127.0.0.1:9999/upload/sound", formData, {
-  return axios.post("http://192.168.101.149:9999/upload/sound", formData, {
+  // return axios.post("http://192.168.101.149:9999/upload/sound", formData, {
+  return axios.post("http://127.0.0.1:9999/upload/sound", formData, {
     headers: {
       "Content-Type": "multipart/form-data",
     },
@@ -284,24 +301,26 @@ async function postDataWithRetry(
   indexFilePath,
   index,
   fileName,
-  hash
+  hash,
+  type
 ) {
   let retryCount = 0; // 当前重试次数
 
   while (retryCount < maxRetries) {
     try {
-      const resData = await postData(dataParams, indexFilePath);
+      const resData = await postData(dataParams, indexFilePath, index);
       if (resData.data.code === 200) {
         logger("请求成功！");
         logger(resData.data.code);
         // 文件名和索引值
-        successLogger(hash, index, fileName);
-        fs.rmdirSync(indexFilePath, { recursive: true });
+        successLogger(hash, index, fileName, type);
+        removeDirectory(indexFilePath);
         return;
       } else if (resData.data.code === 300) {
         // 重复上传，不捕获此错误
+        logger(resData.data.code);
         logger(`第${index}条文件-${index}重复上传！`);
-        fs.rmdirSync(indexFilePath, { recursive: true });
+        removeDirectory(indexFilePath);
         return;
       } else {
         logger(`请求失败，重试中... (${retryCount + 1}/${maxRetries})`);
@@ -321,13 +340,18 @@ async function postDataWithRetry(
 /**
  * 任务
  */
-const task = async (row, index, hash) => {
+const task = async (row, index, hash, type) => {
   //index文件夹   output/0
   const indexFilePath = path.join(workDir, "output", index + "");
   let fileName = row.fileName;
   const rowFileName = row.fileName;
   const title = row.title;
   const keyword = row.keyword;
+  /**
+   * 唯一标识符id，仅错误excel中有 anotherI d 列
+   */
+  const anotherId = row.anotherId ? row.anotherId : excelFiles + index;
+
   try {
     logger(
       "**************************" + row.fileName + "**************************"
@@ -370,7 +394,7 @@ const task = async (row, index, hash) => {
       rowFileName,
       title,
       keyword,
-      index,
+      anotherId,
     });
     await wavAudio(indexFilePath, originFilePath, fileName);
     await watermarkAudio(
@@ -382,7 +406,14 @@ const task = async (row, index, hash) => {
     await archiveZip(fileName, indexFilePath, originFilePath);
 
     // 重试机制
-    await postDataWithRetry(dataParams, indexFilePath, index, fileName, hash);
+    await postDataWithRetry(
+      dataParams,
+      indexFilePath,
+      index,
+      fileName,
+      hash,
+      type
+    );
 
     logger(
       `----------------------------------------第${index}条结束---------------------------------end`
@@ -394,8 +425,8 @@ const task = async (row, index, hash) => {
     } else {
       logger("音频任务失败（最外层catch）：" + error);
     }
-    fs.rmdirSync(indexFilePath, { recursive: true });
-    disposeError(hash, { fileName, keyword, title });
+    removeDirectory(indexFilePath);
+    disposeError(hash, { fileName, keyword, title, anotherId });
 
     //失败的时候，复制文件到错误文件夹中
     // const failPath = path.join(workDir, Error_Files_Dir, fileName);
@@ -403,32 +434,41 @@ const task = async (row, index, hash) => {
   }
 };
 
-const run = (index, data, hash) => {
+const run = (index, data, hash, type) => {
   try {
     logger(
       `run-------------------------------------第${index}条开始------------------------------------`
     );
     const row = data[index];
 
-    if (!row) {
+    if (completeCount >= data.length) {
       taskIndex = 0;
-      logger("当前任务，最后一条已结束！");
+      logger(
+        index +
+          "=========================当前任务，最后一条已结束！========================="
+      );
+
+      successLogger(hash, index, "done");
 
       // 是否存在错误列表？ 存在的话，继续遍历
-      const errorExcelPath = path.join(workDir, `error${hash}.xlsx`);
-      if (isExist(errorExcelPath)) {
-        const jsonData2 = readExcel(errorExcelPath);
-        queue(taskIndex, jsonData2, hash);
-      } else {
-        logger("跑干了！");
+      const status = runErrorList(taskIndex, hash);
+      if (!status) {
+        logger(
+          "》》》》》》》》》》音频批量任务任务结束《《《《《《《《《《" + hash
+        );
       }
 
       return;
     }
 
-    task(row, index, hash)
+    if (!row) {
+      return;
+    }
+
+    task(row, index, hash, type)
       .then(() => {
         taskIndex++;
+        completeCount++;
         run(taskIndex, data, hash);
       })
       .catch((err) => {
@@ -436,15 +476,52 @@ const run = (index, data, hash) => {
       });
   } catch (error) {
     logger("捕获错误！" + error);
+    completeCount++;
     taskIndex++;
     run(taskIndex, data, hash);
   }
 };
 
-const queue = (taskIndex, jsonData, hash) => {
-  for (let i = 0; i < queueCount; i++) {
-    run(i + taskIndex, jsonData, hash);
+const queue = (index, jsonData, hash, type) => {
+  const rest = jsonData.length - index;
+  const count = queueCount > rest ? rest : queueCount;
+  const queueList = [];
+  completeCount = index;
+
+  for (let i = 0; i < count; i++) {
+    queueList.push(run(i + index, jsonData, hash, type));
   }
+
+  taskIndex += count - 1;
+  return Promise.all(queueList);
+};
+
+//yunxing cuowu wenjian
+const runErrorList = (index, hash, parentHash) => {
+  const fileHash = parentHash || hash;
+  const errorExcelPath = path.join(workDir, `error${fileHash}.xlsx`);
+
+  if (retryError > 5) {
+    console.log("出现无法解析的错误");
+    return false;
+  }
+
+  taskIndex = index;
+  let taskHash = hash;
+  //新的运行任务-重新赋予新的任务id
+  if (index === 0) {
+    taskHash = generateUniqueHash().slice(0, 8);
+  }
+
+  if (isExist(errorExcelPath)) {
+    retryError++;
+
+    logger("开始运行错误列表");
+    jsonData = readExcel(errorExcelPath); //   数据来源于error excel
+    queue(index, jsonData, taskHash, fileHash);
+    return true;
+  }
+  return false;
 };
 
 const main = () => {
@@ -459,18 +536,31 @@ const main = () => {
 
   const successPath = path.join(workDir, "success.txt");
   if (isExist(successPath)) {
-    const { loghash, lastItemIndex } = readTxt(successPath);
-    const errorExcelPath = path.join(workDir, `error${loghash}.xlsx`);
-    logger("成功日志存在，异常中断或者跑完了。");
+    const { loghash, lastItemIndex, fileName, parentHash } =
+      readTxt(successPath);
 
     hash = loghash;
-    taskIndex = lastItemIndex + 1; // lastItemIndex为最后一项，所有要+1
-
-    if (isExist(errorExcelPath)) {
-      jsonData = readExcel(errorExcelPath); //   数据来源于error excel
-      queue(taskIndex, jsonData, hash);
+    taskIndex = parseInt(lastItemIndex) + 1; // lastItemIndex为最后一项，所有要+1
+
+    //当前任务已经结束
+    if (fileName === "done") {
+      //找寻是否存在未上传的错误文件列表
+      const status = runErrorList(0, hash);
+      if (!status) {
+        logger(
+          "》》》》》》》》》》音频批量任务任务结束《《《《《《《《《《" + hash
+        );
+      }
     } else {
-      logger("跑干了！");
+      //当前任务异常中断
+
+      //中断的任务类型为错误任务
+      if (parentHash) {
+        runErrorList(taskIndex, hash, parentHash);
+      } else {
+        jsonData = readExcel(excelDir);
+        queue(taskIndex, jsonData, hash);
+      }
     }
   } else {
     hash = generateUniqueHash().slice(0, 8);

package/utils/logger.js

@@ -17,17 +17,21 @@ const logger = (log) => {
  * 成功日志
  * @param {*}  日志内容 hash index fileName
  */
-const successLogger = (hash, index, fileName) => {
+const successLogger = (hash, index, fileName, type = "") => {
   console.log(fileName);
-  fs.writeFileSync("success.txt", hash + " " + index + " " + fileName + "\n", {
-    flag: "a",
-  });
+  fs.writeFileSync(
+    "success.txt",
+    hash + " " + index + " " + fileName + " " + type + "\n",
+    {
+      flag: "a",
+    }
+  );
 };
 
 //错误日志- xlsx
-const disposeError = (hash, { fileName, keyword, title }) => {
+const disposeError = (hash, { fileName, keyword, title, anotherId }) => {
   // 构建 Excel 数据
-  const data = [[fileName, keyword, title]]; // 不包含表头
+  const data = [[fileName, keyword, title, anotherId]]; // 不包含表头
 
   // 读取已存在的 Excel 文件
   let existingData = [];
@@ -41,7 +45,7 @@ const disposeError = (hash, { fileName, keyword, title }) => {
   } catch (error) {
     logger("No existing file found, creating new one.");
     // 如果文件不存在，添加表头
-    existingData.push(["fileName", "keyword", "title"]);
+    existingData.push(["fileName", "keyword", "title", "anotherId"]);
   }
 
   // 合并已有数据和新数据

package/utils/settleFiles.js

@@ -49,7 +49,7 @@ function isExist(filePath) {
     console.log("存在");
     return true;
   } catch (err) {
-    console.log("错了");
+    console.log("不存在");
     console.error(err);
     return false;
   }
@@ -86,15 +86,30 @@ function readTxt(path) {
 
     const lastItem = filteredArray[filteredArray.length - 1];
 
-    const loghash = lastItem.split(" ")[0];
-    const lastItemIndex = parseInt(lastItem.split(" ")[1]);
-    return { loghash, lastItemIndex };
+    const [loghash, lastItemIndex, fileName, parentHash] = lastItem.split(" ");
+    return { loghash, lastItemIndex, fileName, parentHash };
   } catch (error) {
     console.error("readErrorTxt函数出错：", error);
     return {};
   }
 }
 
+/**
+ * 确保文件存在，再删除
+ */
+function removeDirectory(directoryPath) {
+  try {
+    if (fs.existsSync(directoryPath)) {
+      fs.rmdirSync(directoryPath, { recursive: true });
+      console.log(`文件已删除: ${directoryPath}`);
+    } else {
+      console.log(`Directory does not exist: ${directoryPath}`);
+    }
+  } catch (error) {
+    console.error(`Error removing directory ${directoryPath}: ${error}`);
+  }
+}
+
 module.exports = {
   findFileInDir,
   logFileSize,
@@ -102,4 +117,5 @@ module.exports = {
   moveFailFiles,
   isExist,
   readTxt,
+  removeDirectory,
 };