tools_batch_files 1.0.36 → 1.0.37

package/src/queue.js

@@ -0,0 +1,519 @@
+const fs = require("fs");
+const path = require("path");
+const XLSX = require("xlsx");
+const ini = require("ini");
+
+class Queue {
+  /**
+   * 数据信息文件地址
+   */
+  excelPath = "";
+  /**
+   * 工作目录
+   */
+  workDir = "";
+  /**
+   * 数据源
+   */
+  originData = [];
+  /**
+   * 队列
+   */
+  queue = [];
+  /**
+   * 最大并发队列数量
+   */
+  queueMaxCount = 5;
+  /**
+   * 队列索引
+   */
+  queueIndex = 0;
+  /**
+   * 队列运行过程
+   */
+  onProcess = null;
+  /**
+   * 任务完成
+   */
+  onComplete = null;
+  /**
+   * 更新表格队列
+   */
+  queueExcel = [];
+  /**
+   * 配置文件
+   */
+  config;
+  constructor(props) {
+    this.excelPath = props.excelPath;
+    this.workDir = props.workDir;
+    this.onProcess = props.onProcess;
+    this.onComplete = props.onComplete;
+
+    this.init();
+  }
+
+  init() {
+    const that = this;
+    const { excelPath } = that;
+
+    //创建配置文件
+    this.config = this.createConfig();
+
+    //创建临时的数据文件
+    if (!this.config.tmpFile) {
+      this.createTepExcel(excelPath);
+    }
+  }
+
+  /**
+   * 监听事件
+   */
+  event() {
+    const that = this;
+    /**
+     * 监听异常退出
+     */
+    process.on("uncaughtException", () => {
+      that.config.status = "error";
+    });
+  }
+
+  /**
+   * 创建配置文件
+   */
+  createConfig() {
+    let config;
+    if (!this.isConfigExist()) {
+      config = {
+        /**
+         * @param {string} done 主列表数据完成
+         * @param {string} end 任务结束
+         */
+        status: "",
+        /**
+         * 总任务数量
+         */
+        totalCount: 0,
+        /**
+         * 成功任务数量
+         */
+        successCount: 0,
+        /**
+         * 临时文件地址
+         */
+        tmpFile: "",
+        /**
+         * 重试次数
+         */
+        retryCount: 0,
+      };
+      const configString = ini.stringify(config);
+      this.writeWorkFile("config.ini", configString);
+    } else {
+      config = ini.parse(this.readWorkFile("config.ini"));
+      //重置次数
+      config.retryCount = 0;
+      config.successCount = parseInt(config.successCount);
+      config.totalCount = parseInt(config.totalCount);
+    }
+    return config;
+  }
+
+  /**
+   * 运行队列
+   */
+  async start() {
+    const that = this;
+
+    //读取数据-异常时从错误列表中读取
+    const excelData = this.getExcelData();
+    this.queueData = excelData;
+
+    //跑老数据不用同步数量
+    if (!that.config.totalCount) {
+      //记录总任务数量
+      const count = excelData.length;
+      that.config.totalCount = count - 1;
+      that.setConfig("totalCount", count);
+    }
+
+    //运行程序
+    this.run();
+  }
+
+  /**
+   * 重启任务
+   */
+  reStart() {
+    //读取数据-异常时从错误列表中读取
+    const excelData = this.getExcelData();
+    this.queueData = excelData;
+
+    //运行程序
+    this.run();
+  }
+
+  /**
+   * 默认运行
+   */
+  run() {
+    const that = this;
+    this.createQueue().then(() => {
+      //如果没有错误文件任务结束否则重启
+      if (
+        that.config.successCount < that.config.totalCount &&
+        that.config.retryCount <= 5
+      ) {
+        that.config.retryCount++;
+        that.setConfig("retryCount", that.config.retryCount);
+
+        that.reStart();
+      } else {
+        that.config.status = "done";
+        that.setConfig("status", "done");
+
+        //全部执行完毕
+        if (that.config.successCount === that.config.totalCount) {
+          //归档配置文件
+          that.renameConfig();
+          //归档临时文件
+          that.renameTempExcel();
+        }
+
+        //队列执行完成
+        that.onComplete?.();
+      }
+    });
+  }
+
+  /**
+   * 单个任务
+   */
+  singleTask(nextData, resolve) {
+    const that = this;
+    const { primaryKey } = that;
+    const id = nextData[primaryKey] || nextData["queue_id"];
+
+    that
+      .onProcess(nextData)
+      .then(() => {
+        that.config.successCount++;
+        that.setConfig("successCount", that.config.successCount);
+
+        that.updateExcel(id, 1);
+        that.queueNext(resolve);
+      })
+      .catch(() => {
+        that.queueNext(resolve);
+        //处理错误
+      });
+  }
+
+  /**
+   * 创建上传队列
+   */
+  createQueue() {
+    const queueList = [];
+    for (let i = 0; i < this.queueMaxCount; i++) {
+      queueList.push(
+        new Promise((resolve) => {
+          this.queueNext(resolve);
+        })
+      );
+
+      this.queueIndex++;
+    }
+
+    return Promise.all(queueList);
+  }
+
+  /**
+   * 队列运行后，继续下一个任务
+   */
+  queueNext(resolve) {
+    const next = this.dequeue();
+    if (next) {
+      this.singleTask(next, resolve);
+    } else {
+      //当前队列全部结束
+      resolve();
+    }
+  }
+
+  //移出队列
+  dequeue() {
+    return this.queueData.shift();
+  }
+
+  /**
+   * 获取表格数据
+   */
+  getExcelData() {
+    const excelPath = this.config.tmpFile;
+    if (!fs.existsSync(excelPath)) {
+      throw new Error("Excel file not found");
+    }
+    return this.readExcel(excelPath);
+  }
+
+  /**
+   * 更新表格
+   */
+  updateExcel(key, value) {
+    this.queueExcel.push([key, value]);
+    if (this.updateExcelStatus !== "process") {
+      this.updateExcelStatus = "process";
+      this.runQueueExcel();
+    }
+  }
+
+  /**
+   * 归档临时文件
+   */
+  renameTempExcel() {
+    const that = this;
+    const { tmpFile } = that.config;
+    const localDate = that.getCurrentDateTimeWithSlashes();
+    const originPath = tmpFile;
+    const targetPath = tmpFile + "." + localDate + ".bak";
+
+    fs.rename(originPath, targetPath, (err) => {
+      if (err) {
+        console.error("文件重命名失败:", err);
+      }
+    });
+  }
+  /**
+   * 归档配置文件
+   */
+  renameConfig() {
+    const that = this;
+    const localDate = that.getCurrentDateTimeWithSlashes();
+    const originPath = path.join(that.workDir, "config.ini");
+    const targetPath = path.join(
+      that.workDir,
+      "config.ini." + localDate + ".bak"
+    );
+
+    fs.rename(originPath, targetPath, (err) => {
+      if (err) {
+        console.error("文件重命名失败:", err);
+      }
+    });
+  }
+
+  /**
+   * 时间补足
+   * @returns
+   */
+  getCurrentDateTimeWithSlashes() {
+    const now = new Date();
+    const year = now.getFullYear();
+    const month = String(now.getMonth() + 1).padStart(2, "0"); // 月份是从0开始的，所以加1
+    const day = String(now.getDate()).padStart(2, "0"); // 日期不足两位前补0
+    const hours = String(now.getHours()).padStart(2, "0"); // 时不足两位前补0
+    const minutes = String(now.getMinutes()).padStart(2, "0"); // 分不足两位前补0
+    const seconds = String(now.getSeconds()).padStart(2, "0"); // 秒不足两位前补0
+
+    return `${year}-${month}-${day}-${hours}-${minutes}-${seconds}`;
+  }
+
+  /**
+   * 运行数据更新
+   */
+  runQueueExcel() {
+    const that = this;
+    const value = this.queueExcel.shift();
+    if (value) {
+      that.updateExcelFile(...value).then(() => {
+        that.runQueueExcel();
+      });
+    } else {
+      that.updateExcelStatus = "done";
+    }
+  }
+
+  /**
+   *
+   * @param {string} key 主键名称
+   * @param {string} value status的值
+   */
+  updateExcelFile(id, status) {
+    return new Promise((resolve) => {
+      const workbook = XLSX.readFile(this.config.tmpFile);
+      // 获取第一个工作表
+      const sheetName = workbook.Sheets[workbook.SheetNames[0]];
+      const idColumnName = this.primaryKey || "queue_id"; // 假设 'ID' 列包含了唯一标识符
+      const idIndex = findColumnIndexByName(sheetName, idColumnName);
+
+      // 查找列索引
+      function findColumnIndexByName(sheet, columnName) {
+        const firstRow = 0; // 假设第一行是标题行
+        const range = XLSX.utils.decode_range(sheet["!ref"]);
+        for (let col = 0; col <= range.e.c; col++) {
+          const cellAddress = XLSX.utils.encode_cell({ r: firstRow, c: col });
+          const cell = sheet[cellAddress];
+          if (cell && cell.v === columnName) {
+            return col;
+          }
+        }
+        return -1; // 如果没有找到返回 -1
+      }
+      // 查找具有特定ID的行
+      function findRowById(sheet, value, idIndex) {
+        const range = XLSX.utils.decode_range(sheet["!ref"]);
+        for (let row = 1; row <= range.e.r; row++) {
+          const cellAddress = XLSX.utils.encode_cell({ r: row, c: idIndex }); // 假设ID在第一列
+          const cell = sheet[cellAddress];
+          if (cell && cell.v === value) {
+            return row;
+          }
+        }
+        return null; // 如果没有找到返回 null
+      }
+
+      const statusIndex = findColumnIndexByName(sheetName, "status");
+      const rowIndex = findRowById(sheetName, id, idIndex);
+      if (rowIndex !== null) {
+        // 更新同一行的其他列
+        const cellAddress = XLSX.utils.encode_cell({
+          r: rowIndex,
+          c: statusIndex,
+        });
+        sheetName[cellAddress].v = status;
+
+        // 写入更新后的文件
+        XLSX.writeFile(workbook, this.config.tmpFile);
+
+        resolve();
+      }
+      resolve();
+    });
+  }
+
+  /**
+   * 创建临时的excel
+   * @param {*} excelPath Excel路径
+   * @param {*} isKey 主键名
+   */
+  createTepExcel(excelPath, isKey) {
+    const workDir = this.workDir;
+    const fileName = excelPath.match(/[^/]+$/)[0].split(".")[0];
+    const tmpFile = workDir + "/" + fileName + ".tmp" + ".xlsx";
+    // 读取现有工作簿
+    const workbook = XLSX.readFile(excelPath);
+
+    const firstSheetName = workbook.SheetNames[0];
+    const worksheet = workbook.Sheets[firstSheetName];
+
+    const data = XLSX.utils.sheet_to_json(worksheet, { header: 1 });
+
+    //增加数据
+    for (let i = 0; i < data.length; i++) {
+      const _data = data[i];
+      if (i === 0) {
+        //添加状态列
+        _data.push("status");
+
+        //添加主键列
+        if (!isKey) {
+          _data.push("queue_id");
+        }
+      } else {
+        _data.push(0);
+        if (!isKey) {
+          _data.push(fileName + "_" + i);
+        }
+      }
+    }
+
+    // 合并已有数据和新数据
+    const combinedData = data;
+    // 创建新的 sheet
+    const newWs = XLSX.utils.aoa_to_sheet(combinedData);
+
+    // 创建新的 workbook
+    const wb = XLSX.utils.book_new();
+
+    // 将新的 sheet 添加到 workbook
+    XLSX.utils.book_append_sheet(wb, newWs, "sheet1");
+
+    // 写入新工作簿
+    XLSX.writeFile(wb, tmpFile);
+    this.setConfig("tmpFile", tmpFile);
+  }
+
+  /**
+   * 读取Excel
+   * @param {*} xlsxPath Excel路径
+   * @returns JSON
+   */
+  readExcel(xlsxPath) {
+    const workbook = XLSX.readFile(xlsxPath);
+
+    // 获取第一个工作表（Sheet）
+    const firstSheetName = workbook.SheetNames[0];
+    const worksheet = workbook.Sheets[firstSheetName];
+
+    // 将工作表转换为 JSON 对象
+    const jsonData = XLSX.utils.sheet_to_json(worksheet);
+
+    //过滤未上传的id
+    const filterData = jsonData.filter((item) => {
+      return item.status === 0;
+    });
+
+    return filterData;
+  }
+
+  /**
+   * 判断是否已经有配置文件
+   */
+  isConfigExist() {
+    const configPath = path.join(this.workDir, "config.ini");
+    return fs.existsSync(configPath);
+  }
+
+  /**
+   * 更新配置文件
+   */
+  setConfig(key, value) {
+    //更新内存中config
+    this.config[key] = value;
+    const configString = ini.stringify(this.config);
+    //更新配置文件
+    this.writeWorkFile("config.ini", configString);
+  }
+  /**
+   * 读取工作目录文件
+   * @param {string} fileName 文件名称
+   * @returns
+   */
+  readWorkFile(fileName) {
+    const path = this.workDir;
+    const content = fs.readFileSync(path + "/" + fileName, "utf8");
+    return content;
+  }
+
+  /**
+   * 创建文件到工作目录
+   */
+  writeWorkFile(fileName, content) {
+    const path = this.workDir;
+    fs.writeFileSync(path + "/" + fileName, content);
+  }
+
+  /**
+   * 复制文件
+   */
+  copyFile(source, destination) {
+    fs.copyFileSync(source, destination);
+  }
+
+  isEmpty() {
+    return this.queue.length === 0;
+  }
+}
+
+exports.Queue = Queue;