tools_batch_files 1.0.31 → 1.0.32

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tools_batch_files",
-  "version": "1.0.31",
+  "version": "1.0.32",
   "description": "批处理视频工具",
   "keywords": [
     "utils",

package/src/audioFn/audioBatch.js

@@ -1,11 +1,18 @@
 #!/usr/bin/env node
 const generateUniqueHash = require("../../utils/index");
-const { disposeError, readExcel, logger } = require("../../utils/logger");
+const {
+  disposeError,
+  readExcel,
+  logger,
+  successLogger,
+} = require("../../utils/logger");
 
 const {
   findFileInDir,
   logFileSize,
   ensureDirSync,
+  isExist,
+  readTxt,
 } = require("../../utils/settleFiles");
 const ffmpeg = require("fluent-ffmpeg");
 const fs = require("fs");
@@ -273,7 +280,13 @@ const postData = (dataParams, indexFilePath) => {
 /**
  * 接口重试机制
  */
-async function postDataWithRetry(dataParams, indexFilePath, index) {
+async function postDataWithRetry(
+  dataParams,
+  indexFilePath,
+  index,
+  fileName,
+  hash
+) {
   let retryCount = 0; // 当前重试次数
 
   while (retryCount < maxRetries) {
@@ -281,24 +294,28 @@ async function postDataWithRetry(dataParams, indexFilePath, index) {
       const resData = await postData(dataParams, indexFilePath);
       if (resData.data.code === 200) {
         logger("请求成功！");
-        logger(resData.data);
+        logger(resData.data.code);
+        // 文件名和索引值
+        successLogger(hash, index, fileName);
         fs.rmdirSync(indexFilePath, { recursive: true });
-        return; // 请求成功，结束循环
+        return;
       } else if (resData.data.code === 300) {
         // 重复上传，不捕获此错误
         logger(`第${index}条文件-${index}重复上传！`);
         fs.rmdirSync(indexFilePath, { recursive: true });
-        return; // 结束循环
+        return;
       } else {
         // 其他错误，抛出异常
-        throw new Error("请求失败！" + resData.data);
+        throw new Error(
+          "请求code！==200：" + resData.data.code + resData.data.msg
+        );
       }
     } catch (error) {
       logger(`请求失败，重试中... (${retryCount + 1}/${maxRetries})`);
       logger(error);
       retryCount++;
       // 延时等待一段时间后再进行重试
-      await new Promise((resolve) => setTimeout(resolve, 100)); // 等待1秒
+      await new Promise((resolve) => setTimeout(resolve, 100)); // 等待0.1秒
     }
   }
   // 如果达到最大重试次数仍然失败，则抛出异常
@@ -308,7 +325,7 @@ async function postDataWithRetry(dataParams, indexFilePath, index) {
 /**
  * 任务
  */
-const task = async (row, index) => {
+const task = async (row, index, hash) => {
   try {
     logger(
       "**************************" + row.fileName + "**************************"
@@ -316,12 +333,11 @@ const task = async (row, index) => {
     // Excel的列名分别为： fileName title keyword
     let fileName = row.fileName;
     const title = row.title;
+    const keyword = row.keyword;
     // const keywordArr = row.keyword.split(",");
     // const filteredArray = keywordArr.filter((item) => item.trim() !== "");
     // const keyword = filteredArray.join(" ");
 
-    const keyword = row.keyword;
-
     if (!fileName.includes(".")) {
       fileName = row.fileName + ".mp3";
     }
@@ -351,7 +367,6 @@ const task = async (row, index) => {
     //index文件夹   output/0
     const indexFilePath = path.join(workDir, "output", index + "");
 
-    const workStartTime = new Date();
     await fs_asnyc.access(originFilePath, fs_asnyc.constants.F_OK);
     logFileSize(originFilePath);
     const dataParams = await getMetadata(
@@ -371,14 +386,9 @@ const task = async (row, index) => {
       dataParams.duration
     );
     await archiveZip(fileName, indexFilePath, originFilePath);
-    const workEndTime = new Date();
-    const timeInSeconds = ((workEndTime - workStartTime) / 1000).toFixed(2);
-
-    logger(`第${index}条4090处理总时间：${timeInSeconds}秒`);
-    logger(`-------------------${new Date()}------------------`);
 
     // 重试机制
-    await postDataWithRetry(dataParams, indexFilePath, index);
+    await postDataWithRetry(dataParams, indexFilePath, index, fileName, hash);
 
     logger(
       `----------------------------------------第${index}条结束---------------------------------end`
@@ -388,55 +398,92 @@ const task = async (row, index) => {
     if (error.code === "ENOENT") {
       logger(`音频文件 ${fileName} 不存在`);
     } else {
-      logger("音频任务处理失败：" + error);
+      logger("音频任务失败（最外层catch）：" + error);
     }
     fs.rmdirSync(indexFilePath, { recursive: true });
-    disposeError(fileName);
+    disposeError(hash, { fileName, keyword, title });
 
     //失败的时候，复制文件到错误文件夹中
-    const failPath = path.join(workDir, Error_Files_Dir, fileName);
-    moveFailFiles(originFilePath, failPath);
+    // const failPath = path.join(workDir, Error_Files_Dir, fileName);
+    // moveFailFiles(originFilePath, failPath);
+  }
+};
+
+const run = (index, data, hash) => {
+  try {
+    logger(
+      `run-------------------------------------第${index}条开始------------------------------------`
+    );
+    const row = data[index];
+
+    if (!row) {
+      taskIndex = 0;
+      logger("当前任务，最后一条已结束！");
+
+      // 是否存在错误列表？ 存在的话，继续遍历
+      const errorExcelPath = path.join(workDir, `error${hash}.xlsx`);
+      if (isExist(errorExcelPath)) {
+        const jsonData2 = readExcel(errorExcelPath);
+        queue(taskIndex, jsonData2, hash);
+      } else {
+        logger("跑干了！");
+      }
+
+      return;
+    }
+
+    task(row, index)
+      .then(() => {
+        taskIndex++;
+        run(taskIndex, data, hash);
+      })
+      .catch((err) => {
+        throw new Error("task失败：" + err);
+      });
+  } catch (error) {
+    logger("捕获错误！" + error);
+    taskIndex++;
+    run(taskIndex, data, hash);
+  }
+};
+
+const queue = (taskIndex, jsonData, hash) => {
+  for (let i = 0; i < queueCount; i++) {
+    run(i + taskIndex, jsonData, hash);
   }
 };
 
 const main = () => {
   //当前任务hash
-  const hashOrigin = generateUniqueHash();
-  const hash = hashOrigin.slice(0, 8);
+  let hash = "";
+  let jsonData = [];
   logger("音频批量任务任务开始---" + hash);
-  logger("当前目录： " + workDir + "工作目录： " + __dirname);
+  logger("当前目录： " + workDir + "；工作目录： " + exeDir);
 
-  const jsonData = readExcel(excelDir);
+  // 读取success.txt，判断是否是 异常中断，
+  // 并根据success日志获取最后一行的hash，找到对应的error_hash_excel
 
-  const run = (index) => {
-    try {
-      logger(
-        `run-------------------------------------第${index}条开始------------------------------------`
-      );
-      const row = jsonData[index];
+  const successPath = path.join(workDir, "success.txt");
+  if (isExist(successPath)) {
+    const { loghash, lastItemIndex } = readTxt(successPath);
+    const errorExcelPath = path.join(workDir, `error${loghash}.xlsx`);
+    logger("成功日志存在，异常中断或者跑完了。");
 
-      if (!row) {
-        logger("最后一条已结束！");
-        return;
-      }
-      task(row, index)
-        .then(() => {
-          taskIndex++;
-          run(taskIndex);
-        })
-        .catch(() => {
-          taskIndex++;
-          run(taskIndex);
-        });
-    } catch (error) {
-      logger("捕获错误！" + error);
-      taskIndex++;
-      run(taskIndex);
+    hash = loghash;
+    taskIndex = lastItemIndex + 1; // lastItemIndex为最后一项，所有要+1
+
+    if (isExist(errorExcelPath)) {
+      jsonData = readExcel(errorExcelPath); //   数据来源于error excel
+      queue(taskIndex, jsonData, hash);
+    } else {
+      logger("跑干了！");
     }
-  };
+  } else {
+    hash = generateUniqueHash().slice(0, 8);
+    logger("成功日志不存在，开始新任务");
 
-  for (let i = 0; i < queueCount; i++) {
-    run(i + taskIndex);
+    jsonData = readExcel(excelDir);
+    queue(taskIndex, jsonData, hash);
   }
 };
 

package/utils/logger.js

@@ -13,17 +13,53 @@ const logger = (log) => {
   });
 };
 
-//错误日志
-const disposeError = (err) => {
-  logger("---任务失败---" + err);
-  logger(
-    `******************************************************************end`
-  );
-  fs.writeFileSync("error.txt", err + "\n", {
+/**
+ * 成功日志
+ * @param {*}  日志内容 hash index fileName
+ */
+const successLogger = (hash, index, fileName) => {
+  console.log(log);
+  fs.writeFileSync("success.txt", hash + " " + index + " " + fileName + "\n", {
     flag: "a",
   });
 };
 
+//错误日志- xlsx
+const disposeError = (hash, { fileName, keyword, title }) => {
+  // 构建 Excel 数据
+  const data = [[fileName, keyword, title]]; // 不包含表头
+
+  // 读取已存在的 Excel 文件
+  let existingData = [];
+  let existingWb;
+  try {
+    existingWb = XLSX.readFile(`error${hash}.xlsx`);
+    existingData = XLSX.utils.sheet_to_json(
+      existingWb.Sheets[existingWb.SheetNames[0]],
+      { header: 1 }
+    );
+  } catch (error) {
+    logger("No existing file found, creating new one.");
+    // 如果文件不存在，添加表头
+    existingData.push(["fileName", "keyword", "title"]);
+  }
+
+  // 合并已有数据和新数据
+  const combinedData = existingData.concat(data);
+
+  // 创建新的 sheet
+  const newWs = XLSX.utils.aoa_to_sheet(combinedData);
+
+  // 创建新的 workbook
+  const wb = XLSX.utils.book_new();
+
+  // 将新的 sheet 添加到 workbook
+  XLSX.utils.book_append_sheet(wb, newWs, "Error Details");
+
+  // 保存 Excel 文件
+  XLSX.writeFile(wb, `error${hash}.xlsx`);
+};
+
 /**
  * 读取Excel
  * @param {*} path Excel路径
@@ -45,4 +81,5 @@ module.exports = {
   readExcel,
   logger,
   disposeError,
+  successLogger,
 };

package/utils/settleFiles.js

@@ -40,6 +40,19 @@ function ensureDirSync(dirpath) {
   }
 }
 
+/**
+ * 判断文件是否存在
+ */
+function isExist(filePath) {
+  fs.access(filePath, fs.constants.F_OK, (err) => {
+    if (err) {
+      return false;
+    } else {
+      return true;
+    }
+  });
+}
+
 /**
  * 上传失败后，存入新文件夹（手动创建）
  */
@@ -50,9 +63,27 @@ const moveFailFiles = (originFilePath, failPath) => {
   }
 };
 
+//数组 item为：hash index fileName
+function readTxt(path) {
+  try {
+    const data = fs.readFileSync(path, "utf-8");
+    const arr = data.split("\n").map((line) => line.trim()); // 删除每行两侧的空白符
+    const lastItem = arr[arr.length - 1];
+
+    const loghash = lastItem.split(" ")[0];
+    const lastItemIndex = lastItem.split(" ")[1];
+    return { loghash, lastItemIndex };
+  } catch (error) {
+    console.error("readErrorTxt函数出错：", error);
+    return {};
+  }
+}
+
 module.exports = {
   findFileInDir,
   logFileSize,
   ensureDirSync,
   moveFailFiles,
+  isExist,
+  readTxt,
 };