tools_batch_files 1.0.31 → 1.0.33

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tools_batch_files",
-  "version": "1.0.31",
+  "version": "1.0.33",
   "description": "批处理视频工具",
   "keywords": [
     "utils",

package/src/audioFn/audioBatch.js

@@ -1,11 +1,18 @@
 #!/usr/bin/env node
 const generateUniqueHash = require("../../utils/index");
-const { disposeError, readExcel, logger } = require("../../utils/logger");
+const {
+  disposeError,
+  readExcel,
+  logger,
+  successLogger,
+} = require("../../utils/logger");
 
 const {
   findFileInDir,
   logFileSize,
   ensureDirSync,
+  isExist,
+  readTxt,
 } = require("../../utils/settleFiles");
 const ffmpeg = require("fluent-ffmpeg");
 const fs = require("fs");
@@ -193,10 +200,8 @@ const archiveZip = (fileName, inputPath, originFilePath) => {
  * 获取 元数据
  */
 const getMetadata = async (
-  fileName,
-  indexFilePath,
   originFilePath,
-  { rowFileName, title, keyword }
+  { rowFileName, title, keyword, index }
 ) => {
   //源音频数据
   const photoMetadataComand = ffmpeg(originFilePath);
@@ -214,6 +219,7 @@ const getMetadata = async (
     duration: "",
     tag_ids: "3",
     source_from: "sound_1",
+    anotherId: rowFileName + index,
   };
 
   await new Promise((resolve, reject) => {
@@ -261,8 +267,8 @@ const postData = (dataParams, indexFilePath) => {
   logger("等待接口返回结果……");
 
   // return axios.post("http://192.168.102.61:9999/upload/sound", formData, {
-  // return axios.post("http://192.168.101.149:9999/upload/sound", formData, {
-  return axios.post("http://127.0.0.1:9999/upload/sound", formData, {
+  // return axios.post("http://127.0.0.1:9999/upload/sound", formData, {
+  return axios.post("http://192.168.101.149:9999/upload/sound", formData, {
     headers: {
       "Content-Type": "multipart/form-data",
     },
@@ -273,7 +279,13 @@ const postData = (dataParams, indexFilePath) => {
 /**
  * 接口重试机制
  */
-async function postDataWithRetry(dataParams, indexFilePath, index) {
+async function postDataWithRetry(
+  dataParams,
+  indexFilePath,
+  index,
+  fileName,
+  hash
+) {
   let retryCount = 0; // 当前重试次数
 
   while (retryCount < maxRetries) {
@@ -281,24 +293,25 @@ async function postDataWithRetry(dataParams, indexFilePath, index) {
       const resData = await postData(dataParams, indexFilePath);
       if (resData.data.code === 200) {
         logger("请求成功！");
-        logger(resData.data);
+        logger(resData.data.code);
+        // 文件名和索引值
+        successLogger(hash, index, fileName);
         fs.rmdirSync(indexFilePath, { recursive: true });
-        return; // 请求成功，结束循环
+        return;
       } else if (resData.data.code === 300) {
         // 重复上传，不捕获此错误
         logger(`第${index}条文件-${index}重复上传！`);
         fs.rmdirSync(indexFilePath, { recursive: true });
-        return; // 结束循环
+        return;
       } else {
-        // 其他错误，抛出异常
-        throw new Error("请求失败！" + resData.data);
+        logger(`请求失败，重试中... (${retryCount + 1}/${maxRetries})`);
+        logger(`请求code！==200： ${resData.data.code}${resData.data.msg}`);
+        // 延时等待一段时间后再进行重试
+        await new Promise((resolve) => setTimeout(resolve, 100)); // 等待0.1秒
+        retryCount++;
       }
     } catch (error) {
-      logger(`请求失败，重试中... (${retryCount + 1}/${maxRetries})`);
-      logger(error);
-      retryCount++;
-      // 延时等待一段时间后再进行重试
-      await new Promise((resolve) => setTimeout(resolve, 100)); // 等待1秒
+      throw new Error("重试机制错误！" + error);
     }
   }
   // 如果达到最大重试次数仍然失败，则抛出异常
@@ -308,20 +321,22 @@ async function postDataWithRetry(dataParams, indexFilePath, index) {
 /**
  * 任务
  */
-const task = async (row, index) => {
+const task = async (row, index, hash) => {
+  //index文件夹   output/0
+  const indexFilePath = path.join(workDir, "output", index + "");
+  let fileName = row.fileName;
+  const rowFileName = row.fileName;
+  const title = row.title;
+  const keyword = row.keyword;
   try {
     logger(
       "**************************" + row.fileName + "**************************"
     );
     // Excel的列名分别为： fileName title keyword
-    let fileName = row.fileName;
-    const title = row.title;
     // const keywordArr = row.keyword.split(",");
     // const filteredArray = keywordArr.filter((item) => item.trim() !== "");
     // const keyword = filteredArray.join(" ");
 
-    const keyword = row.keyword;
-
     if (!fileName.includes(".")) {
       fileName = row.fileName + ".mp3";
     }
@@ -342,27 +357,21 @@ const task = async (row, index) => {
     logger(`-------------------${new Date()}------------------`);
 
     logger("原文件路径：" + originFilePath);
+    logger(`第${index}条原文件路径：${originFilePath}`);
 
     if (!originFilePath) {
-      logger(`音频文件 ${fileName} 不存在`);
+      logger(`第${index}条音频文件 ${fileName} 不存在`);
       return;
     }
 
-    //index文件夹   output/0
-    const indexFilePath = path.join(workDir, "output", index + "");
-
-    const workStartTime = new Date();
     await fs_asnyc.access(originFilePath, fs_asnyc.constants.F_OK);
-    logFileSize(originFilePath);
-    const dataParams = await getMetadata(
-      fileName,
-      indexFilePath,
-      originFilePath,
-      {
-        title,
-        keyword,
-      }
-    );
+    logFileSize(originFilePath, index);
+    const dataParams = await getMetadata(originFilePath, {
+      rowFileName,
+      title,
+      keyword,
+      index,
+    });
     await wavAudio(indexFilePath, originFilePath, fileName);
     await watermarkAudio(
       indexFilePath,
@@ -371,14 +380,9 @@ const task = async (row, index) => {
       dataParams.duration
     );
     await archiveZip(fileName, indexFilePath, originFilePath);
-    const workEndTime = new Date();
-    const timeInSeconds = ((workEndTime - workStartTime) / 1000).toFixed(2);
-
-    logger(`第${index}条4090处理总时间：${timeInSeconds}秒`);
-    logger(`-------------------${new Date()}------------------`);
 
     // 重试机制
-    await postDataWithRetry(dataParams, indexFilePath, index);
+    await postDataWithRetry(dataParams, indexFilePath, index, fileName, hash);
 
     logger(
       `----------------------------------------第${index}条结束---------------------------------end`
@@ -388,55 +392,92 @@ const task = async (row, index) => {
     if (error.code === "ENOENT") {
       logger(`音频文件 ${fileName} 不存在`);
     } else {
-      logger("音频任务处理失败：" + error);
+      logger("音频任务失败（最外层catch）：" + error);
     }
     fs.rmdirSync(indexFilePath, { recursive: true });
-    disposeError(fileName);
+    disposeError(hash, { fileName, keyword, title });
 
     //失败的时候，复制文件到错误文件夹中
-    const failPath = path.join(workDir, Error_Files_Dir, fileName);
-    moveFailFiles(originFilePath, failPath);
+    // const failPath = path.join(workDir, Error_Files_Dir, fileName);
+    // moveFailFiles(originFilePath, failPath);
   }
 };
 
-const main = () => {
-  //当前任务hash
-  const hashOrigin = generateUniqueHash();
-  const hash = hashOrigin.slice(0, 8);
-  logger("音频批量任务任务开始---" + hash);
-  logger("当前目录： " + workDir + "工作目录： " + __dirname);
+const run = (index, data, hash) => {
+  try {
+    logger(
+      `run-------------------------------------第${index}条开始------------------------------------`
+    );
+    const row = data[index];
 
-  const jsonData = readExcel(excelDir);
+    if (!row) {
+      taskIndex = 0;
+      logger("当前任务，最后一条已结束！");
 
-  const run = (index) => {
-    try {
-      logger(
-        `run-------------------------------------第${index}条开始------------------------------------`
-      );
-      const row = jsonData[index];
-
-      if (!row) {
-        logger("最后一条已结束！");
-        return;
+      // 是否存在错误列表？ 存在的话，继续遍历
+      const errorExcelPath = path.join(workDir, `error${hash}.xlsx`);
+      if (isExist(errorExcelPath)) {
+        const jsonData2 = readExcel(errorExcelPath);
+        queue(taskIndex, jsonData2, hash);
+      } else {
+        logger("跑干了！");
       }
-      task(row, index)
-        .then(() => {
-          taskIndex++;
-          run(taskIndex);
-        })
-        .catch(() => {
-          taskIndex++;
-          run(taskIndex);
-        });
-    } catch (error) {
-      logger("捕获错误！" + error);
-      taskIndex++;
-      run(taskIndex);
+
+      return;
     }
-  };
 
+    task(row, index, hash)
+      .then(() => {
+        taskIndex++;
+        run(taskIndex, data, hash);
+      })
+      .catch((err) => {
+        throw new Error("task失败：" + err);
+      });
+  } catch (error) {
+    logger("捕获错误！" + error);
+    taskIndex++;
+    run(taskIndex, data, hash);
+  }
+};
+
+const queue = (taskIndex, jsonData, hash) => {
   for (let i = 0; i < queueCount; i++) {
-    run(i + taskIndex);
+    run(i + taskIndex, jsonData, hash);
+  }
+};
+
+const main = () => {
+  //当前任务hash
+  let hash = "";
+  let jsonData = [];
+  logger("》》》》》》》》》》音频批量任务任务开始《《《《《《《《《《" + hash);
+  logger("当前目录： " + workDir + "；工作目录： " + exeDir);
+
+  // 读取success.txt，判断是否是 异常中断，
+  // 并根据success日志获取最后一行的hash，找到对应的error_hash_excel
+
+  const successPath = path.join(workDir, "success.txt");
+  if (isExist(successPath)) {
+    const { loghash, lastItemIndex } = readTxt(successPath);
+    const errorExcelPath = path.join(workDir, `error${loghash}.xlsx`);
+    logger("成功日志存在，异常中断或者跑完了。");
+
+    hash = loghash;
+    taskIndex = lastItemIndex + 1; // lastItemIndex为最后一项，所有要+1
+
+    if (isExist(errorExcelPath)) {
+      jsonData = readExcel(errorExcelPath); //   数据来源于error excel
+      queue(taskIndex, jsonData, hash);
+    } else {
+      logger("跑干了！");
+    }
+  } else {
+    hash = generateUniqueHash().slice(0, 8);
+    logger("成功日志不存在，开始新任务");
+
+    jsonData = readExcel(excelDir);
+    queue(taskIndex, jsonData, hash);
   }
 };
 

package/utils/logger.js

@@ -13,17 +13,53 @@ const logger = (log) => {
   });
 };
 
-//错误日志
-const disposeError = (err) => {
-  logger("---任务失败---" + err);
-  logger(
-    `******************************************************************end`
-  );
-  fs.writeFileSync("error.txt", err + "\n", {
+/**
+ * 成功日志
+ * @param {*}  日志内容 hash index fileName
+ */
+const successLogger = (hash, index, fileName) => {
+  console.log(fileName);
+  fs.writeFileSync("success.txt", hash + " " + index + " " + fileName + "\n", {
     flag: "a",
   });
 };
 
+//错误日志- xlsx
+const disposeError = (hash, { fileName, keyword, title }) => {
+  // 构建 Excel 数据
+  const data = [[fileName, keyword, title]]; // 不包含表头
+
+  // 读取已存在的 Excel 文件
+  let existingData = [];
+  let existingWb;
+  try {
+    existingWb = XLSX.readFile(`error${hash}.xlsx`);
+    existingData = XLSX.utils.sheet_to_json(
+      existingWb.Sheets[existingWb.SheetNames[0]],
+      { header: 1 }
+    );
+  } catch (error) {
+    logger("No existing file found, creating new one.");
+    // 如果文件不存在，添加表头
+    existingData.push(["fileName", "keyword", "title"]);
+  }
+
+  // 合并已有数据和新数据
+  const combinedData = existingData.concat(data);
+
+  // 创建新的 sheet
+  const newWs = XLSX.utils.aoa_to_sheet(combinedData);
+
+  // 创建新的 workbook
+  const wb = XLSX.utils.book_new();
+
+  // 将新的 sheet 添加到 workbook
+  XLSX.utils.book_append_sheet(wb, newWs, "Error Details");
+
+  // 保存 Excel 文件
+  XLSX.writeFile(wb, `error${hash}.xlsx`);
+};
+
 /**
  * 读取Excel
  * @param {*} path Excel路径
@@ -45,4 +81,5 @@ module.exports = {
   readExcel,
   logger,
   disposeError,
+  successLogger,
 };

package/utils/settleFiles.js

@@ -19,10 +19,10 @@ function findFileInDir(dir, fileName) {
   return null;
 }
 
-function logFileSize(path) {
+function logFileSize(path, index) {
   const fileSize = fs.statSync(path).size / (1024 * 1024);
   const formattedSize = fileSize.toFixed(2); // 保留两位小数
-  logger(`音频大小：${formattedSize}M`);
+  logger(`第${index}条音频大小：${formattedSize}M`);
 }
 
 /**
@@ -40,6 +40,33 @@ function ensureDirSync(dirpath) {
   }
 }
 
+/**
+ * 判断文件是否存在
+ */
+function isExist(filePath) {
+  try {
+    fs.accessSync(filePath, fs.constants.F_OK);
+    console.log("存在");
+    return true;
+  } catch (err) {
+    console.log("错了");
+    console.error(err);
+    return false;
+  }
+}
+// function isExist(filePath) {
+//   fs.access(filePath, fs.constants.F_OK, (err) => {
+//     if (err) {
+//       console.log(err);
+
+//       return false;
+//     } else {
+//       console.log(555);
+//       return true;
+//     }
+//   });
+// }
+
 /**
  * 上传失败后，存入新文件夹（手动创建）
  */
@@ -50,9 +77,29 @@ const moveFailFiles = (originFilePath, failPath) => {
   }
 };
 
+//数组 item为：hash index fileName
+function readTxt(path) {
+  try {
+    const data = fs.readFileSync(path, "utf-8");
+    const arr = data.split("\n").map((line) => line.trim()); // 删除每行两侧的空白符
+    const filteredArray = arr.filter((item) => item !== ""); //去除空项
+
+    const lastItem = filteredArray[filteredArray.length - 1];
+
+    const loghash = lastItem.split(" ")[0];
+    const lastItemIndex = parseInt(lastItem.split(" ")[1]);
+    return { loghash, lastItemIndex };
+  } catch (error) {
+    console.error("readErrorTxt函数出错：", error);
+    return {};
+  }
+}
+
 module.exports = {
   findFileInDir,
   logFileSize,
   ensureDirSync,
   moveFailFiles,
+  isExist,
+  readTxt,
 };