tools_batch_files 1.0.19 → 1.0.21

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tools_batch_files",
-  "version": "1.0.19",
+  "version": "1.0.21",
   "description": "批处理视频工具",
   "keywords": [
     "utils",
@@ -11,7 +11,8 @@
   "main": "index.js",
   "bin": {
     "tbf": "./index.js",
-    "tbfp": "./src/photoBatch.js"
+    "tbfp": "./src/photoBatch.js",
+    "tbfb": "./src/photoBatchBack.js"
   },
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1",

package/src/photoBatch.js

@@ -35,10 +35,10 @@ const ZIP_FILES_DIR = "zip";
 const expandedName = ".jpg";
 
 //并发数量
-const queueCount = 20;
+const queueCount = 50;
 
 //起始任务下标
-let taskIndex = 16941;
+let taskIndex = 88943;
 
 const readExcel = (path) => {
   const workbook = XLSX.readFile(path);
@@ -100,6 +100,39 @@ function findFileInDir(dir, fileName) {
   return null;
 }
 
+// function findFileInDir(dir, fileName) {
+//   return new Promise((resolve, reject) => {
+//     fs.readdir(dir, (err, files) => {
+//       if (err) {
+//         reject(err);
+//         return;
+//       }
+
+//       // 遍历目录中的文件和子目录
+//       for (const file of files) {
+//         const filePath = path.join(dir, file);
+//         fs.stat(filePath, (err, stat) => {
+//           if (err) {
+//             reject(err); // 如果获取文件状态出错，直接reject
+//             return;
+//           }
+//           if (stat.isDirectory()) {
+//             // 如果是目录，递归调用findFileInDir
+//             findFileInDir(filePath, fileName)
+//               .then(resolve) // 如果找到了目标文件，将结果resolve
+//               .catch(reject); // 如果在子目录中出错，将错误reject
+//           } else if (file.startsWith(fileName) && file.endsWith(expandedName)) {
+//             resolve(filePath); // 如果找到了目标文件，将结果resolve
+//           }
+//         });
+//       }
+
+//       // 如果遍历完了所有文件和子目录仍未找到目标文件，resolve一个null表示未找到
+//       resolve(null);
+//     });
+//   });
+// }
+
 /**
  * 生成预览图 indexFilePath: output/0
  * originFilePath E:\workspace\图片测试\photos\abc222.png
@@ -273,32 +306,39 @@ const getMetadata = async (
 
   await new Promise((resolve, reject) => {
     photoMetadataComand.ffprobe(function (err, metadata) {
-      console.log("metadata");
-      console.log(JSON.stringify(metadata));
-
-      const formatStream = metadata.format;
-      const photoStream = metadata.streams.find(
-        (s) => s.codec_type === "video"
-      );
-
-      metaDataParams.width = photoStream.width;
-      metaDataParams.height = photoStream.height;
-      // metaDataParams.format = formatStream.format_name;
-      metaDataParams.format = "jpg";
-      metaDataParams.size = formatStream.size;
+      // console.log("metadata");
+      // console.log(JSON.stringify(metadata));
+      if (metadata) {
+        const formatStream = metadata.format;
+        const photoStream = metadata.streams.find(
+          (s) => s.codec_type === "video"
+        );
+
+        metaDataParams.width = photoStream.width;
+        metaDataParams.height = photoStream.height;
+        // metaDataParams.format = formatStream.format_name;
+        metaDataParams.format = "jpg";
+        metaDataParams.size = formatStream.size;
 
-      resolve();
+        resolve();
+      } else {
+        reject(err);
+      }
     });
   });
   await new Promise((resolve, reject) => {
     photoPreviewMetadataComand.ffprobe(function (err, metadata) {
-      const photoStream = metadata.streams.find(
-        (s) => s.codec_type === "video"
-      );
+      if (metadata) {
+        const photoStream = metadata.streams.find(
+          (s) => s.codec_type === "video"
+        );
 
-      metaDataParams.common_width = photoStream.width;
-      metaDataParams.common_height = photoStream.height;
-      resolve();
+        metaDataParams.common_width = photoStream.width;
+        metaDataParams.common_height = photoStream.height;
+        resolve();
+      } else {
+        reject(err);
+      }
     });
   });
 
@@ -332,7 +372,7 @@ const postData = (dataParams, indexFilePath) => {
     headers: {
       "Content-Type": "multipart/form-data",
     },
-    timeout: 600000, // 设置超时时间为 60 秒
+    timeout: 300000,
   });
 };
 
@@ -356,11 +396,18 @@ const task = async (row, index) => {
     fileName = row.fileName + expandedName;
   }
 
+  const getPathStartTime = new Date();
   // 源图片文件夹路径
-  const originFilePath = findFileInDir(
+  const originFilePath = await findFileInDir(
     path.join(workDir, ORIGIN_FILE_DIR),
     fileName
   );
+  const getPathEndTime = new Date();
+  const timeInSeconds = ((getPathEndTime - getPathStartTime) / 1000).toFixed(2);
+
+  logger(`第${index}条Path路径搜索时间：${timeInSeconds}秒`);
+  logger(`-------------------${new Date()}------------------`);
+
   console.log("originFilePath：" + originFilePath);
   console.log("fileName：" + fileName);
 
@@ -373,6 +420,7 @@ const task = async (row, index) => {
   const indexFilePath = path.join(workDir, "output", index + "");
 
   try {
+    const workStartTime = new Date();
     await fs_asnyc.access(originFilePath, fs_asnyc.constants.F_OK);
     logFileSize(originFilePath);
     await PriviewPhoto(indexFilePath, originFilePath, fileName);
@@ -388,10 +436,20 @@ const task = async (row, index) => {
         keyword,
       }
     );
-    console.log("dataParams参数");
-    console.log(dataParams);
+    const workEndTime = new Date();
+    const timeInSeconds = ((workEndTime - workStartTime) / 1000).toFixed(2);
 
+    logger(`第${index}条4090处理总时间：${timeInSeconds}秒`);
+    logger(`-------------------${new Date()}------------------`);
+
+    const postStartTime = new Date();
     const resData = await postData(dataParams, indexFilePath);
+    const postEndTime = new Date();
+    const timeInSeconds1 = ((postEndTime - postStartTime) / 1000).toFixed(2);
+
+    logger(`第${index}条接口处理总时间：${timeInSeconds1}秒`);
+    logger(`-------------------${new Date()}------------------`);
+
     if (resData.data.code === 200) {
       logger("请求成功！");
       logger(resData.data.code);
@@ -432,23 +490,42 @@ const transitionExcelToJSON = () => {
   const jsonData = readExcel(excelDir);
 
   const run = (index) => {
-    const row = jsonData[index];
-    if (!row) {
-      return;
+    try {
+      logger(
+        `run-------------------------------------第${index}条开始------------------------------------`
+      );
+      const readExcelStartTime = new Date();
+      const row = jsonData[index];
+
+      const readExcelEndTime = new Date();
+      const timeInSeconds = (
+        (readExcelEndTime - readExcelStartTime) /
+        1000
+      ).toFixed(2);
+
+      logger(`第${index}条Excel搜索时间：${timeInSeconds}秒`);
+      logger(`-------------------${new Date()}------------------`);
+      if (!row) {
+        return;
+      }
+      logger(
+        `----------------------------------------第${index}条task开始--------------------------------`
+      );
+      logger(`-------------------${new Date()}------------------`);
+      task(row, index)
+        .then(() => {
+          taskIndex++;
+          run(taskIndex);
+        })
+        .catch(() => {
+          taskIndex++;
+          run(taskIndex);
+        });
+    } catch (error) {
+      logger("捕获错误！" + error);
+      taskIndex++;
+      run(taskIndex);
     }
-    logger(
-      `----------------------------------------第${index}条开始------------------------------------`
-    );
-    logger(`-------------------${new Date()}------------------`);
-    task(row, index)
-      .then(() => {
-        taskIndex++;
-        run(taskIndex);
-      })
-      .catch(() => {
-        taskIndex++;
-        run(taskIndex);
-      });
   };
 
   for (let i = 0; i < queueCount; i++) {

package/src/photoBatchBack.js

@@ -0,0 +1,528 @@
+#!/usr/bin/env node
+const XLSX = require("xlsx");
+const ffmpeg = require("fluent-ffmpeg");
+const fs = require("fs");
+const path = require("path");
+const fs_asnyc = require("fs").promises;
+const archiver = require("archiver");
+const axios = require("axios");
+const generateUniqueHash = require("../utils/index");
+const FormData = require("form-data");
+
+//执行目录
+const exeDir = __dirname;
+//当前工作目录
+const workDir = process.cwd();
+
+// 表格目录
+const excelDir = path.join(workDir, "excel", "excel.xlsx");
+//水印
+const watermarkImage = path.join(__dirname, "..", "imgs", "picWater.png");
+
+// 截图数量
+const SCREENSHOT_COUNT = 5;
+const ORIGIN_FILE_DIR = "photos";
+const PHOTO_PREVIEW_DIR = "preview_photo";
+const ZIP_VIDEO_DIR = "zip_video";
+const ZIP_VIDEO_DIR_400 = "small_video";
+const SCREENSHOT_DIR = "preview_photo";
+const SCREENSHOT_WATERMARK_DIR = "screenshots_watermark";
+const ZIP_WATERMARK_VIDEO_DIR = "zip_watermark_video";
+const ZIP_SCREENSHOT_WM_DIR = "preview_watermark_photo";
+const SOURCE_PHOTO_DIR = "source_photo";
+const ZIP_FILES_DIR = "zip";
+
+const expandedName = ".jpg";
+
+//并发数量
+const queueCount = 50;
+
+//起始任务下标
+let taskIndex = 146451;
+
+const readExcel = (path) => {
+  const workbook = XLSX.readFile(path);
+
+  // 获取第一个工作表（Sheet）
+  const firstSheetName = workbook.SheetNames[0];
+  const worksheet = workbook.Sheets[firstSheetName];
+
+  // 将工作表转换为 JSON 对象
+  const jsonData = XLSX.utils.sheet_to_json(worksheet);
+  return jsonData;
+};
+
+function logFileSize(path) {
+  const fileSize = fs.statSync(path).size / (1024 * 1024);
+  const formattedSize = fileSize.toFixed(2); // 保留两位小数
+  logger(`图片大小：${formattedSize}M`);
+}
+
+//普通日志
+const logger = (log, hash, err) => {
+  console.log(log);
+  fs.writeFileSync("logBack.txt", log + "\n", {
+    flag: "a",
+  });
+};
+
+//错误日志
+const disposeError = (err) => {
+  logger("---任务失败---", err);
+  logger(
+    `******************************************************************end`
+  );
+  fs.writeFileSync("error.txt", err + "\n", {
+    flag: "a",
+  });
+};
+
+/**
+ * 递归遍历文件夹，查找文件
+ * @param {*} dir 文件夹路径
+ * @param {*} fileName 文件名称
+ * @returns string 匹配成功的文件路径
+ */
+function findFileInDir(dir, fileName) {
+  const files = fs.readdirSync(dir);
+  for (const file of files) {
+    const filePath = path.join(dir, file);
+    const stat = fs.statSync(filePath);
+    if (stat.isDirectory()) {
+      const result = findFileInDir(filePath, fileName);
+      if (result) {
+        return result;
+      }
+    } else if (file.startsWith(fileName) && file.endsWith(expandedName)) {
+      return filePath;
+    }
+  }
+  return null;
+}
+
+// function findFileInDir(dir, fileName) {
+//   return new Promise((resolve, reject) => {
+//     fs.readdir(dir, (err, files) => {
+//       if (err) {
+//         reject(err);
+//         return;
+//       }
+
+//       // 遍历目录中的文件和子目录
+//       for (const file of files) {
+//         const filePath = path.join(dir, file);
+//         fs.stat(filePath, (err, stat) => {
+//           if (err) {
+//             reject(err); // 如果获取文件状态出错，直接reject
+//             return;
+//           }
+//           if (stat.isDirectory()) {
+//             // 如果是目录，递归调用findFileInDir
+//             findFileInDir(filePath, fileName)
+//               .then(resolve) // 如果找到了目标文件，将结果resolve
+//               .catch(reject); // 如果在子目录中出错，将错误reject
+//           } else if (file.startsWith(fileName) && file.endsWith(expandedName)) {
+//             resolve(filePath); // 如果找到了目标文件，将结果resolve
+//           }
+//         });
+//       }
+
+//       // 如果遍历完了所有文件和子目录仍未找到目标文件，resolve一个null表示未找到
+//       resolve(null);
+//     });
+//   });
+// }
+
+/**
+ * 生成预览图 indexFilePath: output/0
+ * originFilePath E:\workspace\图片测试\photos\abc222.png
+ */
+const PriviewPhoto = (indexFilePath, originFilePath, fileName) => {
+  //存放预览图图的文件夹
+  const outputDir = path.join(indexFilePath, SCREENSHOT_DIR);
+  ensureDirSync(outputDir);
+
+  const outputFile = path.join(outputDir, fileName);
+
+  return new Promise((resolve, reject) => {
+    const watermarkScreenshotsComand = ffmpeg(originFilePath);
+    watermarkScreenshotsComand
+      .size("1280x?")
+      .output(outputFile)
+      .on("error", (err) => {
+        logger("截图预览图出错: " + err);
+        reject();
+      })
+      .on("end", () => {
+        logger("预览图完成: " + outputFile);
+        resolve();
+      })
+      .run();
+  });
+};
+
+/**
+ * 生成水印预览图
+ */
+const watermarkPriviewPhoto = (indexFilePath, originFilePath, fileName) => {
+  //存放水印截图的文件夹
+  const outputDir = path.join(indexFilePath, ZIP_SCREENSHOT_WM_DIR);
+  const inputDir = path.join(indexFilePath, SCREENSHOT_DIR, fileName);
+  ensureDirSync(outputDir);
+
+  const outputFile = path.join(outputDir, fileName);
+
+  return new Promise((resolve, reject) => {
+    const watermarkScreenshotsComand = ffmpeg(inputDir);
+    watermarkScreenshotsComand
+      .input(watermarkImage)
+      .complexFilter("overlay=(main_w-overlay_w)/2:(main_h-overlay_h)/2")
+      .output(outputFile)
+      .on("error", (err) => {
+        logger("截图添加水印出错: " + err);
+        reject();
+      })
+      .on("end", () => {
+        logger("截图水印添加完成: " + outputFile);
+        resolve();
+      })
+      .run();
+  });
+};
+
+/**
+ * 确保目录存在，如果不存在则创建它
+ */
+function ensureDirSync(dirpath) {
+  try {
+    if (!fs.existsSync(dirpath)) {
+      fs.mkdirSync(dirpath, { recursive: true });
+      logger(`目录创建成功：${dirpath}`);
+    }
+  } catch (err) {
+    logger(`创建目录时出错：${err}`);
+    throw err;
+  }
+}
+
+/**
+ * 打包物料
+ */
+const archiveZip = (fileName, inputPath, originFilePath) => {
+  const zipDir = path.join(inputPath, "zip");
+  const timestamp = new Date().getTime();
+
+  ensureDirSync(zipDir);
+  const zipStream = fs.createWriteStream(
+    path.join(zipDir, `package${timestamp}.zip`)
+  );
+  const archive = archiver("zip", {
+    zlib: { level: 9 },
+  });
+
+  return new Promise((resolve, reject) => {
+    zipStream.on("close", function () {
+      logger("压缩数据：" + archive.pointer() + " total bytes");
+      logger(
+        "完成归档archiver has been finalized and the output file descriptor has closed."
+      );
+      resolve();
+    });
+
+    archive.on("warning", function (err) {
+      if (err.code === "ENOENT") {
+        logger("压缩-warning：" + err);
+      } else {
+        throw err;
+      }
+    });
+
+    archive.on("error", function (err) {
+      logger("压缩失败！" + err);
+      reject();
+    });
+
+    archive.pipe(zipStream);
+    const directories = [ZIP_SCREENSHOT_WM_DIR, SCREENSHOT_DIR];
+
+    directories.forEach((dir) => {
+      const dirPath = path.join(inputPath, dir);
+      archive.directory(dirPath, dir);
+    });
+
+    archive.file(originFilePath, {
+      name: path.join(SOURCE_PHOTO_DIR, fileName),
+    });
+    // 完成归档
+    archive.finalize();
+  });
+};
+
+/**
+ * 获取 元数据
+ */
+const getMetadata = async (
+  fileName,
+  indexFilePath,
+  originFilePath,
+  { rowFileName, title, keyword }
+) => {
+  //源图片数据
+  const photoMetadataComand = ffmpeg(originFilePath);
+  //预览图数据
+  const photoInputPath = path.join(indexFilePath, SCREENSHOT_DIR, fileName);
+  const photoPreviewMetadataComand = ffmpeg(photoInputPath);
+
+  const metaDataParams = {
+    photo_id: rowFileName,
+    width: "",
+    height: "",
+    format: "",
+    size: "",
+    pr: 160,
+    dpi: 0,
+    is_government: 0,
+    common_width: "",
+    common_height: "",
+    dujia: 0,
+    xiaoxiang: 0,
+    title,
+    keyword,
+    en_keyword: "",
+    // preview_path: "",
+    // markImage_path: "",
+    // photo_path: "",
+    // common_image_path: "",
+    // en_keyword: "",
+    plate_id: 3,
+    transform_plate_id: 3,
+    tag_id: 47,
+    userid: 1003,
+    username: "美好景象",
+    demand_kind: 3,
+    source_from: 71,
+    file: "",
+  };
+
+  await new Promise((resolve, reject) => {
+    photoMetadataComand.ffprobe(function (err, metadata) {
+      // console.log("metadata");
+      // console.log(JSON.stringify(metadata));
+      if (metadata) {
+        const formatStream = metadata.format;
+        const photoStream = metadata.streams.find(
+          (s) => s.codec_type === "video"
+        );
+
+        metaDataParams.width = photoStream.width;
+        metaDataParams.height = photoStream.height;
+        // metaDataParams.format = formatStream.format_name;
+        metaDataParams.format = "jpg";
+        metaDataParams.size = formatStream.size;
+
+        resolve();
+      } else {
+        reject(err);
+      }
+    });
+  });
+  await new Promise((resolve, reject) => {
+    photoPreviewMetadataComand.ffprobe(function (err, metadata) {
+      if (metadata) {
+        const photoStream = metadata.streams.find(
+          (s) => s.codec_type === "video"
+        );
+
+        metaDataParams.common_width = photoStream.width;
+        metaDataParams.common_height = photoStream.height;
+        resolve();
+      } else {
+        reject(err);
+      }
+    });
+  });
+
+  return metaDataParams;
+};
+
+/**
+ * 接口
+ */
+const postData = (dataParams, indexFilePath) => {
+  const formData = new FormData();
+
+  const zipFiles = fs
+    .readdirSync(path.join(indexFilePath, ZIP_FILES_DIR))
+    .find((file) => file.endsWith(".zip"));
+
+  const packageZip = path.join(indexFilePath, ZIP_FILES_DIR, zipFiles);
+
+  formData.append("file", fs.createReadStream(packageZip));
+  for (const key in dataParams) {
+    if (Object.hasOwnProperty.call(dataParams, key)) {
+      const value = dataParams[key];
+      formData.append(key, value);
+    }
+  }
+
+  logger("等待接口返回结果……");
+
+  // return axios.post("http://192.168.102.61:9999/upload/photo", formData, {
+  return axios.post("http://127.0.0.1:9999/upload/photo", formData, {
+    headers: {
+      "Content-Type": "multipart/form-data",
+    },
+    timeout: 300000,
+  });
+};
+
+/**
+ * 任务
+ */
+const task = async (row, index) => {
+  logger(
+    "**************************" + row.fileName + "**************************"
+  );
+  // Excel的列名分别为： fileName title keyword
+  // 表格中文件名无后缀，遂手动添加写死为mp4
+  let fileName = row.fileName;
+  const rowFileName = row.fileName;
+  const title = row.title;
+  const keywordArr = row.keyword.split(",");
+  const filteredArray = keywordArr.filter((item) => item.trim() !== "");
+  const keyword = filteredArray.join(" ");
+
+  if (!fileName.includes(".")) {
+    fileName = row.fileName + expandedName;
+  }
+
+  const getPathStartTime = new Date();
+  // 源图片文件夹路径
+  const originFilePath = await findFileInDir(
+    path.join(workDir, ORIGIN_FILE_DIR),
+    fileName
+  );
+  const getPathEndTime = new Date();
+  const timeInSeconds = ((getPathEndTime - getPathStartTime) / 1000).toFixed(2);
+
+  logger(`第${index}条Path路径搜索时间：${timeInSeconds}秒`);
+  logger(`-------------------${new Date()}------------------`);
+
+  console.log("originFilePath：" + originFilePath);
+  console.log("fileName：" + fileName);
+
+  if (!originFilePath) {
+    logger(`图片文件 ${fileName} 不存在`);
+    return;
+  }
+
+  //index文件夹   output/0
+  const indexFilePath = path.join(workDir, "output", index + "");
+
+  try {
+    const workStartTime = new Date();
+    await fs_asnyc.access(originFilePath, fs_asnyc.constants.F_OK);
+    logFileSize(originFilePath);
+    await PriviewPhoto(indexFilePath, originFilePath, fileName);
+    await watermarkPriviewPhoto(indexFilePath, originFilePath, fileName);
+    await archiveZip(fileName, indexFilePath, originFilePath);
+    const dataParams = await getMetadata(
+      fileName,
+      indexFilePath,
+      originFilePath,
+      {
+        rowFileName,
+        title,
+        keyword,
+      }
+    );
+    const workEndTime = new Date();
+    const timeInSeconds = ((workEndTime - workStartTime) / 1000).toFixed(2);
+
+    logger(`第${index}条4090处理总时间：${timeInSeconds}秒`);
+    logger(`-------------------${new Date()}------------------`);
+
+    const postStartTime = new Date();
+    const resData = await postData(dataParams, indexFilePath);
+    const postEndTime = new Date();
+    const timeInSeconds1 = ((postEndTime - postStartTime) / 1000).toFixed(2);
+
+    logger(`第${index}条接口处理总时间：${timeInSeconds1}秒`);
+    logger(`-------------------${new Date()}------------------`);
+
+    if (resData.data.code === 200) {
+      logger("请求成功！");
+      logger(resData.data.code);
+      console.log(resData.data);
+      //删除文件夹
+      fs.rmdirSync(indexFilePath, { recursive: true });
+    } else {
+      throw new Error("请求失败！" + resData.data.msg);
+    }
+
+    logger(
+      `----------------------------------------第${index}条结束---------------------------------end`
+    );
+  } catch (error) {
+    // 可以约定code，来表示不同的错误信息
+    if (error.code === "ENOENT") {
+      logger(`图片文件 ${fileName} 不存在`);
+    } else {
+      logger("图片任务处理失败：" + error);
+      logger(error);
+    }
+    fs.rmdirSync(indexFilePath, { recursive: true });
+    disposeError(fileName);
+  }
+};
+
+/**
+ * 解析Excel数据
+ */
+const transitionExcelToJSON = () => {
+  //当前任务hash
+  const hashOrigin = generateUniqueHash();
+  const hash = hashOrigin.slice(0, 8);
+  logger("图片批量任务任务开始---" + hash);
+  logger("当前目录： " + workDir);
+  logger("工作目录： " + __dirname);
+
+  const jsonData = readExcel(excelDir);
+
+  const run = (index) => {
+    try {
+      logger(
+        `run-------------------------------------倒序第${index}条开始------------------------------------`
+      );
+      const row = jsonData[index];
+
+      logger(`-------------------${new Date()}------------------`);
+      if (!row) {
+        return;
+      }
+
+      task(row, index)
+        .then(() => {
+          taskIndex--;
+          run(taskIndex);
+        })
+        .catch(() => {
+          taskIndex--;
+          run(taskIndex);
+        });
+    } catch (error) {
+      logger("捕获错误！" + error);
+      taskIndex--;
+      run(taskIndex);
+    }
+  };
+
+  for (let i = jsonData.length; i > 0; i--) {
+    run(i);
+  }
+  for (let i = 0; i < queueCount; i++) {
+    run(taskIndex - i);
+  }
+};
+
+transitionExcelToJSON();