tools_batch_files 1.0.22 → 1.0.24

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tools_batch_files",
-  "version": "1.0.22",
+  "version": "1.0.24",
   "description": "批处理视频工具",
   "keywords": [
     "utils",
@@ -36,6 +36,7 @@
     "点我批量上传图片.bat",
     "package.json",
     "package-lock.json",
-    "utils"
+    "utils",
+    "src"
   ]
 }

package/src/filterFail.js

@@ -0,0 +1,40 @@
+const fs = require('fs');
+const readline = require('readline');
+
+//上传失败的文件汇总到 txt中，脚本跑完后，统一便利再上传。
+
+const logFilePath = 'log.txt';
+
+// 创建 readline 接口实例
+const rl = readline.createInterface({
+    input: fs.createReadStream(logFilePath),
+    output: process.stdout,
+    terminal: false
+});
+
+// 匹配视频文件名，和id = undefined
+const filenameRegex = /.*\*\*\*([^*]+)\*\*\*/;
+const idUndefinedRegex = /id = undefined/;
+
+let currentFileName = '';
+
+rl.on('line', (line) => {
+    // 检查是否是文件名行
+    if (line.includes('**************************') && !line.includes('end')) {
+        const match = filenameRegex.exec(line);
+        if (match && match[1]) {
+            currentFileName = match[1].trim(); // 提取并保存当前文件名
+        }
+    }
+
+    // 检查id是否未定义
+    if (idUndefinedRegex.test(line) && currentFileName) {
+        console.log(`找到undefined ID对应的文件名: ${currentFileName}`);
+        // 你可以在这里执行进一步操作，例如存储文件名
+        fs.appendFileSync('undefined-ids.txt', `${currentFileName}\n`);
+    }
+});
+
+rl.on('close', () => {
+    console.log('Finished reading file.');
+});

package/src/photoBatch.js

@@ -38,7 +38,7 @@ const expandedName = ".jpg";
 const queueCount = 50;
 
 //起始任务下标
-let taskIndex = 0;
+let taskIndex = 123265;
 
 const readExcel = (path) => {
   const workbook = XLSX.readFile(path);
@@ -494,12 +494,24 @@ const transitionExcelToJSON = () => {
       logger(
         `run-------------------------------------第${index}条开始------------------------------------`
       );
-      logger(`-------------------${new Date()}------------------`);
+      const readExcelStartTime = new Date();
       const row = jsonData[index];
 
+      const readExcelEndTime = new Date();
+      const timeInSeconds = (
+        (readExcelEndTime - readExcelStartTime) /
+        1000
+      ).toFixed(2);
+
+      logger(`第${index}条Excel搜索时间：${timeInSeconds}秒`);
+      logger(`-------------------${new Date()}------------------`);
       if (!row) {
         return;
       }
+      logger(
+        `----------------------------------------第${index}条task开始--------------------------------`
+      );
+      logger(`-------------------${new Date()}------------------`);
       task(row, index)
         .then(() => {
           taskIndex++;

package/src/photoBatchErrRetry.js

@@ -0,0 +1,566 @@
+#!/usr/bin/env node
+const XLSX = require("xlsx");
+const ffmpeg = require("fluent-ffmpeg");
+const fs = require("fs");
+const path = require("path");
+const fs_asnyc = require("fs").promises;
+const archiver = require("archiver");
+const axios = require("axios");
+const generateUniqueHash = require("../utils/index");
+const FormData = require("form-data");
+
+//执行目录
+const exeDir = __dirname;
+//当前工作目录
+const workDir = process.cwd();
+
+// 表格目录
+const excelDir = path.join(workDir, "excel", "excel.xlsx");
+// error错误日志目录
+const errorDir = path.join(workDir, "error.txt");
+//水印
+const watermarkImage = path.join(__dirname, "..", "imgs", "picWater.png");
+
+// 截图数量
+const SCREENSHOT_COUNT = 5;
+const ORIGIN_FILE_DIR = "photos";
+const PHOTO_PREVIEW_DIR = "preview_photo";
+const ZIP_VIDEO_DIR = "zip_video";
+const ZIP_VIDEO_DIR_400 = "small_video";
+const SCREENSHOT_DIR = "preview_photo";
+const SCREENSHOT_WATERMARK_DIR = "screenshots_watermark";
+const ZIP_WATERMARK_VIDEO_DIR = "zip_watermark_video";
+const ZIP_SCREENSHOT_WM_DIR = "preview_watermark_photo";
+const SOURCE_PHOTO_DIR = "source_photo";
+const ZIP_FILES_DIR = "zip";
+
+const expandedName = ".jpg";
+
+//并发数量
+const queueCount = 50;
+
+//起始任务下标
+let taskIndex = 0;
+
+const readExcel = (path) => {
+  const workbook = XLSX.readFile(path);
+
+  // 获取第一个工作表（Sheet）
+  const firstSheetName = workbook.SheetNames[0];
+  const worksheet = workbook.Sheets[firstSheetName];
+
+  // 将工作表转换为 JSON 对象
+  const jsonData = XLSX.utils.sheet_to_json(worksheet);
+  return jsonData;
+};
+
+//返回error数组
+const readErrorTxt = (path) => {
+  fs.readFile(path, "utf8", (err, data) => {
+    if (err) {
+      console.error(err);
+      return;
+    }
+
+    const lines = data.split("\n");
+    return lines;
+  });
+};
+
+/**
+ * 重传的index数组
+ */
+const findErrIndex = (jsonData, txtData) => {
+  const indexArr = [];
+
+  txtData.forEach((itemErr, indexErr) => {
+    // 在excel中查找匹配项
+    const foundIndex = jsonData.findIndex(
+      (itemExcel) => itemExcel.fileName === itemErr
+    );
+    if (foundIndex !== -1) {
+      indexArr.push(indexErr);
+    }
+  });
+  return indexArr;
+};
+
+function logFileSize(path) {
+  const fileSize = fs.statSync(path).size / (1024 * 1024);
+  const formattedSize = fileSize.toFixed(2); // 保留两位小数
+  logger(`图片大小：${formattedSize}M`);
+}
+
+//普通日志
+const logger = (log, hash, err) => {
+  console.log(log);
+  fs.writeFileSync("log.txt", log + "\n", {
+    flag: "a",
+  });
+};
+
+//错误日志
+const disposeError = (err) => {
+  logger("---任务失败---", err);
+  logger(
+    `******************************************************************end`
+  );
+  fs.writeFileSync("error.txt", err + "\n", {
+    flag: "a",
+  });
+};
+
+/**
+ * 递归遍历文件夹，查找文件
+ * @param {*} dir 文件夹路径
+ * @param {*} fileName 文件名称
+ * @returns string 匹配成功的文件路径
+ */
+function findFileInDir(dir, fileName) {
+  const files = fs.readdirSync(dir);
+  for (const file of files) {
+    const filePath = path.join(dir, file);
+    const stat = fs.statSync(filePath);
+    if (stat.isDirectory()) {
+      const result = findFileInDir(filePath, fileName);
+      if (result) {
+        return result;
+      }
+    } else if (file.startsWith(fileName) && file.endsWith(expandedName)) {
+      return filePath;
+    }
+  }
+  return null;
+}
+
+// function findFileInDir(dir, fileName) {
+//   return new Promise((resolve, reject) => {
+//     fs.readdir(dir, (err, files) => {
+//       if (err) {
+//         reject(err);
+//         return;
+//       }
+
+//       // 遍历目录中的文件和子目录
+//       for (const file of files) {
+//         const filePath = path.join(dir, file);
+//         fs.stat(filePath, (err, stat) => {
+//           if (err) {
+//             reject(err); // 如果获取文件状态出错，直接reject
+//             return;
+//           }
+//           if (stat.isDirectory()) {
+//             // 如果是目录，递归调用findFileInDir
+//             findFileInDir(filePath, fileName)
+//               .then(resolve) // 如果找到了目标文件，将结果resolve
+//               .catch(reject); // 如果在子目录中出错，将错误reject
+//           } else if (file.startsWith(fileName) && file.endsWith(expandedName)) {
+//             resolve(filePath); // 如果找到了目标文件，将结果resolve
+//           }
+//         });
+//       }
+
+//       // 如果遍历完了所有文件和子目录仍未找到目标文件，resolve一个null表示未找到
+//       resolve(null);
+//     });
+//   });
+// }
+
+/**
+ * 生成预览图 indexFilePath: output/0
+ * originFilePath E:\workspace\图片测试\photos\abc222.png
+ */
+const PriviewPhoto = (indexFilePath, originFilePath, fileName) => {
+  //存放预览图图的文件夹
+  const outputDir = path.join(indexFilePath, SCREENSHOT_DIR);
+  ensureDirSync(outputDir);
+
+  const outputFile = path.join(outputDir, fileName);
+
+  return new Promise((resolve, reject) => {
+    const watermarkScreenshotsComand = ffmpeg(originFilePath);
+    watermarkScreenshotsComand
+      .size("1280x?")
+      .output(outputFile)
+      .on("error", (err) => {
+        logger("截图预览图出错: " + err);
+        reject();
+      })
+      .on("end", () => {
+        logger("预览图完成: " + outputFile);
+        resolve();
+      })
+      .run();
+  });
+};
+
+/**
+ * 生成水印预览图
+ */
+const watermarkPriviewPhoto = (indexFilePath, originFilePath, fileName) => {
+  //存放水印截图的文件夹
+  const outputDir = path.join(indexFilePath, ZIP_SCREENSHOT_WM_DIR);
+  const inputDir = path.join(indexFilePath, SCREENSHOT_DIR, fileName);
+  ensureDirSync(outputDir);
+
+  const outputFile = path.join(outputDir, fileName);
+
+  return new Promise((resolve, reject) => {
+    const watermarkScreenshotsComand = ffmpeg(inputDir);
+    watermarkScreenshotsComand
+      .input(watermarkImage)
+      .complexFilter("overlay=(main_w-overlay_w)/2:(main_h-overlay_h)/2")
+      .output(outputFile)
+      .on("error", (err) => {
+        logger("截图添加水印出错: " + err);
+        reject();
+      })
+      .on("end", () => {
+        logger("截图水印添加完成: " + outputFile);
+        resolve();
+      })
+      .run();
+  });
+};
+
+/**
+ * 确保目录存在，如果不存在则创建它
+ */
+function ensureDirSync(dirpath) {
+  try {
+    if (!fs.existsSync(dirpath)) {
+      fs.mkdirSync(dirpath, { recursive: true });
+      logger(`目录创建成功：${dirpath}`);
+    }
+  } catch (err) {
+    logger(`创建目录时出错：${err}`);
+    throw err;
+  }
+}
+
+/**
+ * 打包物料
+ */
+const archiveZip = (fileName, inputPath, originFilePath) => {
+  const zipDir = path.join(inputPath, "zip");
+  const timestamp = new Date().getTime();
+
+  ensureDirSync(zipDir);
+  const zipStream = fs.createWriteStream(
+    path.join(zipDir, `package${timestamp}.zip`)
+  );
+  const archive = archiver("zip", {
+    zlib: { level: 9 },
+  });
+
+  return new Promise((resolve, reject) => {
+    zipStream.on("close", function () {
+      logger("压缩数据：" + archive.pointer() + " total bytes");
+      logger(
+        "完成归档archiver has been finalized and the output file descriptor has closed."
+      );
+      resolve();
+    });
+
+    archive.on("warning", function (err) {
+      if (err.code === "ENOENT") {
+        logger("压缩-warning：" + err);
+      } else {
+        throw err;
+      }
+    });
+
+    archive.on("error", function (err) {
+      logger("压缩失败！" + err);
+      reject();
+    });
+
+    archive.pipe(zipStream);
+    const directories = [ZIP_SCREENSHOT_WM_DIR, SCREENSHOT_DIR];
+
+    directories.forEach((dir) => {
+      const dirPath = path.join(inputPath, dir);
+      archive.directory(dirPath, dir);
+    });
+
+    archive.file(originFilePath, {
+      name: path.join(SOURCE_PHOTO_DIR, fileName),
+    });
+    // 完成归档
+    archive.finalize();
+  });
+};
+
+/**
+ * 获取 元数据
+ */
+const getMetadata = async (
+  fileName,
+  indexFilePath,
+  originFilePath,
+  { rowFileName, title, keyword }
+) => {
+  //源图片数据
+  const photoMetadataComand = ffmpeg(originFilePath);
+  //预览图数据
+  const photoInputPath = path.join(indexFilePath, SCREENSHOT_DIR, fileName);
+  const photoPreviewMetadataComand = ffmpeg(photoInputPath);
+
+  const metaDataParams = {
+    photo_id: rowFileName,
+    width: "",
+    height: "",
+    format: "",
+    size: "",
+    pr: 160,
+    dpi: 0,
+    is_government: 0,
+    common_width: "",
+    common_height: "",
+    dujia: 0,
+    xiaoxiang: 0,
+    title,
+    keyword,
+    en_keyword: "",
+    // preview_path: "",
+    // markImage_path: "",
+    // photo_path: "",
+    // common_image_path: "",
+    // en_keyword: "",
+    plate_id: 3,
+    transform_plate_id: 3,
+    tag_id: 47,
+    userid: 1003,
+    username: "美好景象",
+    demand_kind: 3,
+    source_from: 71,
+    file: "",
+  };
+
+  await new Promise((resolve, reject) => {
+    photoMetadataComand.ffprobe(function (err, metadata) {
+      // console.log("metadata");
+      // console.log(JSON.stringify(metadata));
+      if (metadata) {
+        const formatStream = metadata.format;
+        const photoStream = metadata.streams.find(
+          (s) => s.codec_type === "video"
+        );
+
+        metaDataParams.width = photoStream.width;
+        metaDataParams.height = photoStream.height;
+        // metaDataParams.format = formatStream.format_name;
+        metaDataParams.format = "jpg";
+        metaDataParams.size = formatStream.size;
+
+        resolve();
+      } else {
+        reject(err);
+      }
+    });
+  });
+  await new Promise((resolve, reject) => {
+    photoPreviewMetadataComand.ffprobe(function (err, metadata) {
+      if (metadata) {
+        const photoStream = metadata.streams.find(
+          (s) => s.codec_type === "video"
+        );
+
+        metaDataParams.common_width = photoStream.width;
+        metaDataParams.common_height = photoStream.height;
+        resolve();
+      } else {
+        reject(err);
+      }
+    });
+  });
+
+  return metaDataParams;
+};
+
+/**
+ * 接口
+ */
+const postData = (dataParams, indexFilePath) => {
+  const formData = new FormData();
+
+  const zipFiles = fs
+    .readdirSync(path.join(indexFilePath, ZIP_FILES_DIR))
+    .find((file) => file.endsWith(".zip"));
+
+  const packageZip = path.join(indexFilePath, ZIP_FILES_DIR, zipFiles);
+
+  formData.append("file", fs.createReadStream(packageZip));
+  for (const key in dataParams) {
+    if (Object.hasOwnProperty.call(dataParams, key)) {
+      const value = dataParams[key];
+      formData.append(key, value);
+    }
+  }
+
+  logger("等待接口返回结果……");
+
+  // return axios.post("http://192.168.102.61:9999/upload/photo", formData, {
+  return axios.post("http://127.0.0.1:9999/upload/photo", formData, {
+    headers: {
+      "Content-Type": "multipart/form-data",
+    },
+    timeout: 300000,
+  });
+};
+
+/**
+ * 任务
+ */
+const task = async (row, index) => {
+  logger(
+    "**************************" + row.fileName + "**************************"
+  );
+  // Excel的列名分别为： fileName title keyword
+  // 表格中文件名无后缀，遂手动添加写死为mp4
+  let fileName = row.fileName;
+  const rowFileName = row.fileName;
+  const title = row.title;
+  const keywordArr = row.keyword.split(",");
+  const filteredArray = keywordArr.filter((item) => item.trim() !== "");
+  const keyword = filteredArray.join(" ");
+
+  if (!fileName.includes(".")) {
+    fileName = row.fileName + expandedName;
+  }
+
+  const getPathStartTime = new Date();
+  // 源图片文件夹路径
+  const originFilePath = await findFileInDir(
+    path.join(workDir, ORIGIN_FILE_DIR),
+    fileName
+  );
+  const getPathEndTime = new Date();
+  const timeInSeconds = ((getPathEndTime - getPathStartTime) / 1000).toFixed(2);
+
+  logger(`第${index}条Path路径搜索时间：${timeInSeconds}秒`);
+  logger(`-------------------${new Date()}------------------`);
+
+  console.log("originFilePath：" + originFilePath);
+  console.log("fileName：" + fileName);
+
+  if (!originFilePath) {
+    logger(`图片文件 ${fileName} 不存在`);
+    return;
+  }
+
+  //index文件夹   output/0
+  const indexFilePath = path.join(workDir, "output", index + "");
+
+  try {
+    const workStartTime = new Date();
+    await fs_asnyc.access(originFilePath, fs_asnyc.constants.F_OK);
+    logFileSize(originFilePath);
+    await PriviewPhoto(indexFilePath, originFilePath, fileName);
+    await watermarkPriviewPhoto(indexFilePath, originFilePath, fileName);
+    await archiveZip(fileName, indexFilePath, originFilePath);
+    const dataParams = await getMetadata(
+      fileName,
+      indexFilePath,
+      originFilePath,
+      {
+        rowFileName,
+        title,
+        keyword,
+      }
+    );
+    const workEndTime = new Date();
+    const timeInSeconds = ((workEndTime - workStartTime) / 1000).toFixed(2);
+
+    logger(`第${index}条4090处理总时间：${timeInSeconds}秒`);
+    logger(`-------------------${new Date()}------------------`);
+
+    const postStartTime = new Date();
+    const resData = await postData(dataParams, indexFilePath);
+    const postEndTime = new Date();
+    const timeInSeconds1 = ((postEndTime - postStartTime) / 1000).toFixed(2);
+
+    logger(`第${index}条接口处理总时间：${timeInSeconds1}秒`);
+    logger(`-------------------${new Date()}------------------`);
+
+    if (resData.data.code === 200) {
+      logger("请求成功！");
+      logger(resData.data.code);
+      console.log(resData.data);
+      //删除文件夹
+      fs.rmdirSync(indexFilePath, { recursive: true });
+    } else {
+      throw new Error("请求失败！" + resData.data.msg);
+    }
+
+    logger(
+      `----------------------------------------第${index}条结束---------------------------------end`
+    );
+  } catch (error) {
+    // 可以约定code，来表示不同的错误信息
+    if (error.code === "ENOENT") {
+      logger(`图片文件 ${fileName} 不存在`);
+    } else {
+      logger("图片任务处理失败：" + error);
+      logger(error);
+    }
+    fs.rmdirSync(indexFilePath, { recursive: true });
+    disposeError(fileName);
+  }
+};
+
+/**
+ * 解析Excel数据
+ */
+const transitionExcelToJSON = () => {
+  //当前任务hash
+  const hashOrigin = generateUniqueHash();
+  const hash = hashOrigin.slice(0, 8);
+  logger("图片批量任务任务开始---" + hash);
+  logger("当前目录： " + workDir);
+  logger("工作目录： " + __dirname);
+
+  const jsonData = readExcel(excelDir);
+  
+  //遍历txt文件，拿到文件名，去excel表匹配对应的行，找到index，
+  //队列如何做？把所有搜到的index存成数组。然后循环查找
+  const txtData = readErrorTxt(errorDir);
+  const findErrIndexArr = findErrIndex(jsonData, txtData);
+
+  const run = (index) => {
+    try {
+      logger(
+        `run-------------------------------------第${index}条开始------------------------------------`
+      );
+      logger(`-------------------${new Date()}------------------`);
+      const row = jsonData[index];
+
+      if (!row) {
+        return;
+      }
+
+      task(row, index)
+        .then(() => {
+          taskIndex++;
+          run(taskIndex);
+        })
+        .catch(() => {
+          taskIndex++;
+          run(taskIndex);
+        });
+    } catch (error) {
+      logger("捕获错误！" + error);
+      taskIndex++;
+      run(taskIndex);
+    }
+  };
+
+  for (let i = 0; i < queueCount; i++) {
+    const idx = findErrIndexArr[i + taskIndex];
+    if (idx) {
+      run(idx);
+    }
+  }
+};
+
+transitionExcelToJSON();

package/src/removeFailVideo.js

@@ -0,0 +1,100 @@
+const fs = require("fs");
+const path = require("path");
+
+const workDir = process.cwd();
+
+const ORIGIN_FILE_DIR = "video";
+
+// error错误日志目录
+const errorDir = path.join(workDir, "error.txt");
+
+const Error_Files_Dir = "failed_videos";
+// const Error_Files_Dir = "failed_photos";
+
+/**
+ * 确保目录存在，如果不存在则创建它
+ */
+function ensureDirSync(dirpath) {
+  try {
+    if (!fs.existsSync(dirpath)) {
+      fs.mkdirSync(dirpath, { recursive: true });
+    }
+  } catch (err) {
+    throw err;
+  }
+}
+
+//普通日志
+const logger = (log) => {
+  console.log(log);
+  fs.writeFileSync("errorCopyLog.txt", log + "\n", {
+    flag: "a",
+  });
+};
+
+/**
+ * 上传或处理失败的文件，放到fail文件夹中，这样之前的目录就可以删除了
+ * @param {*} sourcePath 源文件Path
+ * @param {*} destinationPath 存放失败文件的，目标文件Path
+ */
+// const moveFailedVideo = (sourcePath, destinationPath) => {
+//   console.log("源文件Path" + sourcePath);
+//   console.log("目标文件Path" + destinationPath);
+
+//   ensureDirSync(destinationPath);
+//   fs.copyFileSync(sourcePath, destinationPath);
+// };
+
+/**
+ * 递归遍历文件夹，查找mp4文件
+ * @param {*} dir 文件夹路径
+ * @param {*} fileName 文件名称
+ * @returns {filePath, file} 匹配成功的文件路径，和文件名
+ */
+function findFileInDir(dir, fileName) {
+  const files = fs.readdirSync(dir);
+  for (const file of files) {
+    const filePath = path.join(dir, file);
+    const stat = fs.statSync(filePath);
+    if (stat.isDirectory()) {
+      const result = findFileInDir(filePath, fileName);
+      if (result) {
+        return result;
+      }
+    } else if (file.startsWith(fileName) && file.endsWith(".mov")) {
+      return { filePath, file };
+    }
+  }
+  return null;
+}
+
+//返回error中 fileName的数组（带格式）
+const readErrorTxt = (path) => {
+  try {
+    const data = fs.readFileSync(path, "utf-8");
+    const lines = data.split("\n").map((line) => line.trim()); // 删除每行两侧的空白符
+    return lines;
+  } catch (error) {
+    console.error("读取文件时出错：", error);
+    return [];
+  }
+};
+
+const txtData = readErrorTxt(errorDir);
+logger("error数据共计：" + txtData.length);
+
+txtData.forEach((item, index) => {
+  const fileObj = findFileInDir(path.join(workDir, ORIGIN_FILE_DIR), item);
+
+  if (fileObj) {
+    const { filePath: originFilePath, file: fileName } = fileObj;
+
+    if (originFilePath) {
+      const failPath = path.join(workDir, Error_Files_Dir, fileName);
+      // ensureDirSync(failPath)
+
+      fs.copyFileSync(originFilePath, failPath);
+      logger(`第${index} / ${txtData.length}个复制完成！`);
+    }
+  }
+});