tools_batch_files 1.0.36 → 1.0.38

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tools_batch_files",
-  "version": "1.0.36",
+  "version": "1.0.38",
   "description": "批处理视频工具",
   "keywords": [
     "utils",
@@ -12,9 +12,9 @@
   "bin": {
     "tbf": "./index.js",
     "tbfp": "./src/photoBatch.js",
+    "tbfps": "./src/photoFn/photoBatch.js",
     "tbfa": "./src/audioFn/audioBatch.js",
     "tbfv": "./src/videoFn/videoBatch.js",
-    "tbfb": "./src/photoBatchBack.js",
     "tbfre": "./src/removeFailVideo.js"
   },
   "scripts": {
@@ -30,6 +30,7 @@
     "axios": "^1.6.8",
     "ffmpeg": "^0.0.4",
     "fluent-ffmpeg": "^2.1.2",
+    "ini": "^4.1.3",
     "xlsx": "^0.18.5"
   },
   "files": [

package/src/batchTash.js

@@ -0,0 +1,44 @@
+#!/usr/bin/env node
+
+/**
+ * 模版文件-演示如何使用队列
+ */
+
+// import { Queue } from "./queue";
+const { Queue } = require("./queue");
+const path = require("path");
+
+//当前工作目录
+const workDir = process.cwd();
+
+const templateQueue = new Queue({
+  //excel 主键-非必填-无主键时自动生成
+  primaryKey: "",
+  //excel文件路径-必填
+  excelPath: path.join(workDir, "excel", "20240531.xlsx"),
+  //当前工作路径-必填
+  workDir: workDir,
+  //当前可执行任务回调-必填
+  onProcess: (data) => {
+    //..todo 执行需要压缩上传的文件所有过程。
+    return new Promise((resolve, rejects) => {
+      const random = Math.random() * 100;
+      if (random < 20) {
+        console.log("onProcess Success", data.queue_id);
+        //上传成功
+        resolve();
+      } else {
+        console.log("onProcess error", data.queue_id);
+        //上传失败
+        rejects();
+      }
+    });
+  },
+  //当前任务全部结束-必填
+  onComplete: () => {
+    console.log("Congratulations Complete");
+  },
+});
+
+//开始上传任务
+templateQueue.start();

package/src/photoFn/photoBatch.js

@@ -0,0 +1,560 @@
+#!/usr/bin/env node
+const generateUniqueHash = require("../../utils/index");
+const {
+  disposeError,
+  readExcel,
+  logger,
+  successLogger,
+} = require("../../utils/logger");
+
+const {
+  findFileInDir,
+  logFileSize,
+  ensureDirSync,
+  isExist,
+  readTxt,
+  removeDirectory,
+} = require("../../utils/settleFiles");
+const ffmpeg = require("fluent-ffmpeg");
+const fs = require("fs");
+const path = require("path");
+const fs_asnyc = require("fs").promises;
+const archiver = require("archiver");
+const axios = require("axios");
+const FormData = require("form-data");
+
+//并发数量
+const queueCount = 10;
+
+//起始任务下标
+let taskIndex = 0;
+
+//已完成数量
+let completeCount = 0;
+
+//错误文件重试次数
+let retryError = 0;
+
+//执行目录
+const exeDir = __dirname;
+//当前工作目录
+const workDir = process.cwd();
+
+//图片水印
+const watermarkImage = path.join(exeDir, "imgs", "miz-watermark.png");
+
+// 表格目录，从excel文件夹中查找
+const excelFiles = fs
+  .readdirSync(path.join(workDir, "excel"))
+  .find((file) => file.endsWith(".xlsx"));
+const excelDir = path.join(workDir, "excel", excelFiles);
+
+const ORIGIN_FILE_DIR = "files";
+const ZIP_FILES_DIR = "zip";
+const SCREENSHOT_DIR = "preview_photo";
+const SOURCE_PHOTO_DIR = "source_photo";
+const ZIP_SCREENSHOT_WM_DIR = "preview_watermark_photo";
+
+const maxRetries = 1; // 最大重试次数
+
+/**
+ * 生成预览图 indexFilePath: output/0
+ * originFilePath E:\workspace\图片测试\photos\abc222.png
+ */
+const PriviewPhoto = (indexFilePath, originFilePath, fileName) => {
+  //存放预览图图的文件夹
+  const outputDir = path.join(indexFilePath, SCREENSHOT_DIR);
+  ensureDirSync(outputDir);
+
+  const outputFile = path.join(outputDir, fileName);
+
+  return new Promise((resolve, reject) => {
+    const watermarkScreenshotsComand = ffmpeg(originFilePath);
+    watermarkScreenshotsComand
+      .size("1280x?")
+      .output(outputFile)
+      .on("error", (err) => {
+        logger("截图预览图出错: " + err);
+        reject();
+      })
+      .on("end", () => {
+        logger("预览图完成: " + outputFile);
+        resolve();
+      })
+      .run();
+  });
+};
+
+/**
+ * 生成水印预览图
+ */
+const watermarkPriviewPhoto = (indexFilePath, originFilePath, fileName) => {
+  //存放水印截图的文件夹
+  const outputDir = path.join(indexFilePath, ZIP_SCREENSHOT_WM_DIR);
+  const inputDir = path.join(indexFilePath, SCREENSHOT_DIR, fileName);
+  ensureDirSync(outputDir);
+
+  const outputFile = path.join(outputDir, fileName);
+
+  return new Promise((resolve, reject) => {
+    const watermarkScreenshotsComand = ffmpeg(inputDir);
+    watermarkScreenshotsComand
+      .input(watermarkImage)
+      .complexFilter("overlay=(main_w-overlay_w)/2:(main_h-overlay_h)/2")
+      .output(outputFile)
+      .on("error", (err) => {
+        logger("截图添加水印出错: " + err);
+        reject();
+      })
+      .on("end", () => {
+        logger("截图水印添加完成: " + outputFile);
+        resolve();
+      })
+      .run();
+  });
+};
+
+/**
+ * 打包物料
+ */
+const archiveZip = (fileName, inputPath, originFilePath) => {
+  const zipDir = path.join(inputPath, "zip");
+  const timestamp = new Date().getTime();
+
+  ensureDirSync(zipDir);
+  const zipStream = fs.createWriteStream(
+    path.join(zipDir, `package${timestamp}.zip`)
+  );
+  const archive = archiver("zip", {
+    zlib: { level: 9 },
+  });
+
+  return new Promise((resolve, reject) => {
+    zipStream.on("close", function () {
+      logger("压缩数据：" + archive.pointer() + " total bytes");
+      logger(
+        "完成归档archiver has been finalized and the output file descriptor has closed."
+      );
+      resolve();
+    });
+
+    archive.on("warning", function (err) {
+      if (err.code === "ENOENT") {
+        logger("压缩-warning：" + err);
+      } else {
+        throw err;
+      }
+    });
+
+    archive.on("error", function (err) {
+      logger("压缩失败！" + err);
+      reject();
+    });
+
+    archive.pipe(zipStream);
+    const directories = [ZIP_SCREENSHOT_WM_DIR, SCREENSHOT_DIR];
+
+    directories.forEach((dir) => {
+      const dirPath = path.join(inputPath, dir);
+      archive.directory(dirPath, dir);
+    });
+
+    archive.file(originFilePath, {
+      name: path.join(SOURCE_PHOTO_DIR, fileName),
+    });
+    // 完成归档
+    archive.finalize();
+  });
+};
+
+/**
+ * 获取 元数据
+ */
+const getMetadata = async (
+  fileName,
+  indexFilePath,
+  originFilePath,
+  { anotherId, title, keyword }
+) => {
+  //源图片数据
+  const photoMetadataComand = ffmpeg(originFilePath);
+  //预览图数据
+  const photoInputPath = path.join(indexFilePath, SCREENSHOT_DIR, fileName);
+  const photoPreviewMetadataComand = ffmpeg(photoInputPath);
+
+  const metaDataParams = {
+    photo_id: anotherId,
+    width: "",
+    height: "",
+    format: "",
+    size: "",
+    pr: 0,
+    dpi: 0,
+    is_government: 0,
+    common_width: "",
+    common_height: "",
+    dujia: 0,
+    xiaoxiang: 0,
+    title,
+    keyword,
+    en_keyword: "",
+    // preview_path: "",
+    // markImage_path: "",
+    // photo_path: "",
+    // common_image_path: "",
+    // en_keyword: "",
+    plate_id: 3,
+    transform_plate_id: 3,
+    tag_id: 47,
+    userid: 10876358,
+    username: "樊健",
+    demand_kind: 184,
+    source_from: 74,
+    file: "",
+  };
+
+  await new Promise((resolve, reject) => {
+    photoMetadataComand.ffprobe(function (err, metadata) {
+      // console.log("metadata");
+      // console.log(JSON.stringify(metadata));
+      if (metadata) {
+        const formatStream = metadata.format;
+        const photoStream = metadata.streams.find(
+          (s) => s.codec_type === "video"
+        );
+
+        metaDataParams.width = photoStream.width;
+        metaDataParams.height = photoStream.height;
+        // metaDataParams.format = formatStream.format_name;
+        metaDataParams.format = "jpg";
+        metaDataParams.size = formatStream.size;
+
+        resolve();
+      } else {
+        reject(err);
+      }
+    });
+  });
+  await new Promise((resolve, reject) => {
+    photoPreviewMetadataComand.ffprobe(function (err, metadata) {
+      if (metadata) {
+        const photoStream = metadata.streams.find(
+          (s) => s.codec_type === "video"
+        );
+
+        metaDataParams.common_width = photoStream.width;
+        metaDataParams.common_height = photoStream.height;
+        resolve();
+      } else {
+        reject(err);
+      }
+    });
+  });
+
+  return metaDataParams;
+};
+
+/**
+ * 接口
+ */
+const postData = (dataParams, indexFilePath, index) => {
+  const formData = new FormData();
+
+  const zipFiles = fs
+    .readdirSync(path.join(indexFilePath, ZIP_FILES_DIR))
+    .find((file) => file.endsWith(".zip"));
+
+  const packageZip = path.join(indexFilePath, ZIP_FILES_DIR, zipFiles);
+
+  formData.append("file", fs.createReadStream(packageZip));
+  for (const key in dataParams) {
+    if (Object.hasOwnProperty.call(dataParams, key)) {
+      const value = dataParams[key];
+      formData.append(key, value);
+    }
+  }
+
+  logger(`第${index}条等待接口返回结果……`);
+
+  // return axios.post("http://192.168.102.61:9999/upload/sound", formData, {
+  // return axios.post("http://192.168.101.149:9999/upload/sound", formData, {
+  return axios.post("http://127.0.0.1:9999/upload/video", formData, {
+    headers: {
+      "Content-Type": "multipart/form-data",
+    },
+    timeout: 300000,
+  });
+};
+
+/**
+ * 接口重试机制
+ */
+async function postDataWithRetry(
+  dataParams,
+  indexFilePath,
+  index,
+  fileName,
+  hash,
+  type
+) {
+  let retryCount = 0; // 当前重试次数
+
+  while (retryCount < maxRetries) {
+    try {
+      const resData = await postData(dataParams, indexFilePath, index);
+      if (resData.data.code === 200) {
+        logger("请求成功！");
+        logger(resData.data.code);
+        // 文件名和索引值
+        successLogger(hash, index, fileName, type);
+        removeDirectory(indexFilePath);
+        return;
+      } else if (resData.data.code === 300) {
+        // 重复上传，不捕获此错误
+        logger(resData.data.code);
+        logger(`第${index}条文件-${index}重复上传！`);
+        removeDirectory(indexFilePath);
+        return;
+      } else {
+        logger(`请求失败，重试中... (${retryCount + 1}/${maxRetries})`);
+        logger(`请求code！==200： ${resData.data.code}${resData.data.msg}`);
+        // 延时等待一段时间后再进行重试
+        await new Promise((resolve) => setTimeout(resolve, 100)); // 等待0.1秒
+        retryCount++;
+      }
+    } catch (error) {
+      throw new Error("重试机制错误！" + error);
+    }
+  }
+  // 如果达到最大重试次数仍然失败，则抛出异常
+  throw new Error("请求失败，重试次数已达到上限！");
+}
+
+/**
+ * 任务
+ */
+const task = async (row, index, hash, type) => {
+  //index文件夹   output/0
+  const indexFilePath = path.join(workDir, "output", index + "");
+  let fileName = row.fileName;
+  const rowFileName = row.fileName;
+  const title = row.title;
+  const keyword = row.keyword;
+  /**
+   * 唯一标识符id，仅错误excel中有 anotherI d 列
+   */
+  const anotherId = row.anotherId ? row.anotherId : row.fileName;
+
+  try {
+    logger(
+      "**************************" + row.fileName + "**************************"
+    );
+
+    // if (!fileName.includes(".")) {
+    //   fileName = row.fileName + ".mp44444";   //findFileInDir后期返回两个 文件路径和 文件名
+    // }
+
+    const getPathStartTime = new Date();
+    // 源图片文件夹路径
+    const originFilePath = await findFileInDir(
+      path.join(workDir, ORIGIN_FILE_DIR),
+      fileName
+    );
+
+    const getPathEndTime = new Date();
+    const timeInSeconds1 = ((getPathEndTime - getPathStartTime) / 1000).toFixed(
+      2
+    );
+
+    logger(`第${index}条Path路径搜索时间：${timeInSeconds1}秒`);
+    logger(`-------------------${new Date()}------------------`);
+
+    logger("原文件路径：" + originFilePath);
+    logger(`第${index}条原文件路径：${originFilePath}`);
+
+    if (!originFilePath) {
+      logger(`第${index}条图片文件 ${fileName} 不存在`);
+      return;
+    }
+
+    await fs_asnyc.access(originFilePath, fs_asnyc.constants.F_OK);
+    logFileSize(originFilePath, index);
+    await PriviewPhoto(indexFilePath, originFilePath, fileName);
+    await watermarkPriviewPhoto(indexFilePath, originFilePath, fileName);
+
+    await archiveZip(fileName, indexFilePath, originFilePath);
+
+    const dataParams = await getMetadata(
+      fileName,
+      indexFilePath,
+      originFilePath,
+      {
+        anotherId,
+        title,
+        keyword,
+      }
+    );
+
+    // 重试机制
+    await postDataWithRetry(
+      dataParams,
+      indexFilePath,
+      index,
+      fileName,
+      hash,
+      type
+    );
+
+    logger(
+      `----------------------------------------第${index}条结束---------------------------------end`
+    );
+  } catch (error) {
+    // 可以约定code，来表示不同的错误信息
+    if (error.code === "ENOENT") {
+      logger(`图片文件 ${fileName} 不存在`);
+    } else {
+      logger("图片任务失败（最外层catch）：" + error);
+    }
+    removeDirectory(indexFilePath);
+    disposeError(hash, { fileName, keyword, title, anotherId });
+  }
+};
+
+const run = (index, data, hash, type) => {
+  try {
+    logger(
+      `run-------------------------------------第${index}条开始------------------------------------`
+    );
+    const row = data[index];
+
+    if (completeCount >= data.length) {
+      taskIndex = 0;
+      logger(
+        index +
+          "=========================当前任务，最后一条已结束！========================="
+      );
+
+      successLogger(hash, index, "done");
+
+      // 是否存在错误列表？ 存在的话，继续遍历
+      const status = runErrorList(taskIndex, hash);
+      if (!status) {
+        logger(
+          "》》》》》》》》》》图片批量任务任务结束《《《《《《《《《《" + hash
+        );
+      }
+
+      return;
+    }
+
+    if (!row) {
+      return;
+    }
+
+    task(row, index, hash, type)
+      .then(() => {
+        taskIndex++;
+        completeCount++;
+        run(taskIndex, data, hash);
+      })
+      .catch((err) => {
+        throw new Error("task失败：" + err);
+      });
+  } catch (error) {
+    logger("捕获错误！" + error);
+    completeCount++;
+    taskIndex++;
+    run(taskIndex, data, hash);
+  }
+};
+
+const queue = (index, jsonData, hash, type) => {
+  const rest = jsonData.length - index;
+  const count = queueCount > rest ? rest : queueCount;
+  const queueList = [];
+  completeCount = index;
+
+  for (let i = 0; i < count; i++) {
+    queueList.push(run(i + index, jsonData, hash, type));
+  }
+
+  taskIndex += count - 1;
+  return Promise.all(queueList);
+};
+
+//yunxing cuowu wenjian
+const runErrorList = (index, hash, parentHash) => {
+  const fileHash = parentHash || hash;
+  const errorExcelPath = path.join(workDir, `error${fileHash}.xlsx`);
+
+  if (retryError > 5) {
+    console.log("出现无法解析的错误");
+    return false;
+  }
+
+  taskIndex = index;
+  let taskHash = hash;
+  //新的运行任务-重新赋予新的任务id
+  if (index === 0) {
+    taskHash = generateUniqueHash().slice(0, 8);
+  }
+
+  if (isExist(errorExcelPath)) {
+    retryError++;
+
+    logger("开始运行错误列表");
+    jsonData = readExcel(errorExcelPath); //   数据来源于error excel
+    queue(index, jsonData, taskHash, fileHash);
+    return true;
+  }
+  return false;
+};
+
+const main = () => {
+  //当前任务hash
+  let hash = "";
+  let jsonData = [];
+  logger("》》》》》》》》》》视频批量任务任务开始《《《《《《《《《《" + hash);
+  logger("当前目录： " + workDir + "；工作目录： " + exeDir);
+
+  // 读取success.txt，判断是否是 异常中断，
+  // 并根据success日志获取最后一行的hash，找到对应的error_hash_excel
+
+  const successPath = path.join(workDir, "success.txt");
+  if (isExist(successPath)) {
+    const { loghash, lastItemIndex, fileName, parentHash } =
+      readTxt(successPath);
+
+    hash = loghash;
+    taskIndex = parseInt(lastItemIndex) + 1; // lastItemIndex为最后一项，所有要+1
+
+    //当前任务已经结束
+    if (fileName === "done") {
+      //找寻是否存在未上传的错误文件列表
+      const status = runErrorList(0, hash);
+      if (!status) {
+        logger(
+          "》》》》》》》》》》图片批量任务任务结束《《《《《《《《《《" + hash
+        );
+      }
+    } else {
+      //当前任务异常中断
+
+      //中断的任务类型为错误任务
+      if (parentHash) {
+        runErrorList(taskIndex, hash, parentHash);
+      } else {
+        jsonData = readExcel(excelDir);
+        queue(taskIndex, jsonData, hash);
+      }
+    }
+  } else {
+    hash = generateUniqueHash().slice(0, 8);
+    logger("成功日志不存在，开始新任务");
+
+    jsonData = readExcel(excelDir);
+    queue(taskIndex, jsonData, hash);
+  }
+};
+
+main();