scanoss 0.7.9 → 0.7.10

package/build/main/src/sdk/scanner/WfpProvider/WfpCalculator/WfpCalculator.js

@@ -1,492 +0,0 @@
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.WfpCalculator = void 0;
-const fs_1 = __importDefault(require("fs"));
-const worker_threads_1 = require("worker_threads");
-const ScannableItem_1 = require("../../Scannable/ScannableItem");
-const ScannerCfg_1 = require("../../ScannerCfg");
-const FingerprintPackage_1 = require("../FingerprintPackage");
-const WfpProvider_1 = require("../WfpProvider");
-const stringWorker = `
-const { parentPort } = require('worker_threads');
-
-parentPort.on('message', async (scannableItem) => {
-
-  let fingerprint;
-  if ( scannableItem.winnowingMode === "FULL_WINNOWING") {
-    fingerprint = wfp_for_content(
-      scannableItem.content,
-      scannableItem.contentSource,
-      scannableItem.maxSizeWfp
-    );
-  } else if (scannableItem.winnowingMode === "FULL_WINNOWING_HPSM") {
-    fingerprint = wfp_hpsm_for_content(
-      scannableItem.content,
-      scannableItem.contentSource,
-      scannableItem.maxSizeWfp
-    );
-  } else if (scannableItem.winnowingMode === "WINNOWING_ONLY_MD5") {
-    fingerprint = wfp_only_md5(
-      scannableItem.content,
-      scannableItem.contentSource
-    );
-  }
-
-  scannableItem.fingerprint = fingerprint;
-  parentPort.postMessage(scannableItem);
-
-  });
-
-  /**
-  * Winnowing and HPSM algorithm begins
-  */
-
-const crypto = require('crypto');
-const isWin = process.platform === 'win32';
-const pathSeparator = isWin ? String.fromCharCode(92) : '/';
-
-// Winnowing configuration. DO NOT CHANGE.
-const GRAM = 30;
-const WINDOW = 64;
-
-// ASCII characters
-const ASCII_0 = 48;
-const ASCII_9 = 57;
-const ASCII_A = 65;
-const ASCII_Z = 90;
-const ASCII_a = 97;
-const ASCII_z = 122;
-const ASCII_LF = 10;
-
-function normalize(byte) {
-  if (byte < ASCII_0 || byte > ASCII_z) {
-    return 0;
-  }
-  if (byte <= ASCII_9 || byte >= ASCII_a) {
-    return byte;
-  }
-  if (byte >= ASCII_A && byte <= ASCII_Z) {
-    return byte + 32;
-  }
-  return 0;
-}
-
-function min_hex_array(array) {
-  let min = 'ffffffff';
-  for (let i = 0; i < array.length; i++) {
-    if (array[i] < min) {
-      min = array[i];
-    }
-  }
-  return min;
-}
-
-function calc_wfp(contents, maxSize) {
-  let gram = '';
-  const window = [];
-  let normalized = 0;
-  let line = 1;
-  let min_hash = 'ffffffff';
-  let last_hash = 'ffffffff';
-  let last_line = 0;
-  let output = '';
-  let gram_crc32 = 0;
-  let wfp = '';
-
-  for (let i = 0; i < contents.length; i++) {
-    if(wfp.length > maxSize)
-      return wfp;
-
-    const byte = contents[i];
-    if (byte == ASCII_LF) {
-      line += 1;
-      normalized = 0;
-    } else {
-      normalized = normalize(byte);
-    }
-    // Is this an useful byte?
-    if (normalized) {
-      gram += String.fromCharCode(normalized);
-
-      if (gram.length >= GRAM) {
-        gram_crc32 = crc32c_hex(gram);
-        window.push(gram_crc32);
-
-        if (window.length >= WINDOW) {
-          min_hash = min_hex_array(window);
-          if (min_hash !== last_hash) {
-            // Hashing the hash will result in a better balanced output data set
-            // as it will counter the winnowing effect which selects the "minimum"
-            // hash in each window
-            const min_hash_bytes_le = parseHexString(
-              toLittleEndianCRCHex(min_hash)
-            );
-            const crc_hex = crc32c_for_bytes_hex(min_hash_bytes_le);
-
-            if (last_line != line) {
-              if (output.length > 0) {
-                wfp += String(output) + String.fromCharCode(10);
-              }
-              output = String(line) + '=' + String(crc_hex);
-            } else {
-              output += ',' + String(crc_hex);
-            }
-            last_line = line;
-            last_hash = min_hash;
-          }
-          window.shift();
-        }
-        gram = gram.slice(1);
-      }
-    }
-  }
-  if (output.length > 0) {
-    wfp += String(output) + String.fromCharCode(10);
-  }
-
-  return wfp;
-}
-
-function parseHexString(str) {
-  const result = [];
-  while (str.length >= 2) {
-    result.push(parseInt(str.substring(0, 2), 16));
-    str = str.substring(2, str.length);
-  }
-
-  return result;
-}
-
-/**
- *
- * @param {string} hex
- */
-function toLittleEndianCRCHex(hex) {
-  return (
-    hex.charAt(6) +
-    hex.charAt(7) +
-    hex.charAt(4) +
-    hex.charAt(5) +
-    hex.charAt(2) +
-    hex.charAt(3) +
-    hex.charAt(0) +
-    hex.charAt(1)
-  );
-}
-
-let CRC_TABLE = [];
-
-function makeCRCTable() {
-  let c;
-  const crcTable = [];
-  for (let n = 0; n < 256; n++) {
-    c = n;
-    for (let k = 0; k < 8; k++) {
-      c = c & 1 ? 0x82f63b78 ^ (c >>> 1) : c >>> 1;
-    }
-    crcTable[n] = c;
-  }
-  return crcTable;
-}
-
-function crc32c(str) {
-  if (CRC_TABLE.length == 0) {
-    CRC_TABLE = makeCRCTable();
-  }
-  let crc = 0 ^ -1;
-
-  for (let i = 0; i < str.length; i++) {
-    crc = (crc >>> 8) ^ CRC_TABLE[(crc ^ str.charCodeAt(i)) & 0xff];
-  }
-
-  return (crc ^ -1) >>> 0;
-}
-
-function crc32c_for_bytes(bytes) {
-  if (CRC_TABLE.length == 0) {
-    CRC_TABLE = makeCRCTable();
-  }
-  let crc = 0 ^ -1;
-
-  for (let i = 0; i < bytes.length; i++) {
-    crc = (crc >>> 8) ^ CRC_TABLE[(crc ^ bytes[i]) & 0xff];
-  }
-
-  return (crc ^ -1) >>> 0;
-}
-
-function crc32c_for_bytes_hex(bytes) {
-  return crc32c_for_bytes(bytes).toString(16).padStart(8, '0');
-}
-
-function crc32c_hex(str) {
-  return crc32c(str).toString(16).padStart(8, '0');
-}
-
-function truncate_string(input, maxSize){
-  return input.slice(0, maxSize)
-}
-
-function toHexString(byteArray) {
-  return Array.from(byteArray, function(byte) {
-    return ('0' + (byte & 0xFF).toString(16)).slice(-2);
-  }).join('')
-}
-
-function calc_hpsm(content) {
-  let list_normalized = [];    //Array of numbers
-  let crc_lines = [];  //Array of numbers that represent the crc8_maxim for each line of the file
-
-  let last_line = 0;
-  crc8_MAXIM_DOW_GenerateTable();
-
-  for(let i = 0; i<content.length; i++) {
-    const c = content[i];
-    if (c == ASCII_LF) {  //When there is a new line
-      if (list_normalized.length) {
-        crc_lines.push(crc8_MAXIM_DOW_Buffer(list_normalized))
-        list_normalized=[];
-      } else if(last_line+1 == i) {
-        crc_lines.push(0xFF);
-      } else if (i-last_line  > 1) {
-        crc_lines.push(0x00)
-      }
-      last_line = i;
-    } else {
-      const c_normalized = normalize(c);
-      if (c_normalized != 0)  list_normalized.push(c_normalized)
-    }
-  }
-  return "hpsm=" + toHexString(crc_lines)
-}
-
-
-/**
- * Create a wfp_hpsm package joining wfp (with md5 line) and a hpsm.
- *
- * Example:
- * wfp = file=508cb9dfbe1c7dca5ed24f124473f33d,300,asd.c
- *         11=b19bdbfa
- *
- * hpsm = hpsm=1909ff06ff688630ff45a92b52f47eff3500ffff
- *
- * wfp_hpsm = file=508cb9dfbe1c7dca5ed24f124473f33d,300,asd.c
- *             hpsm=1909ff06ff688630ff45a92b52f47eff3500ffff
- *             11=b19bdbfa
- *
- * @param {string} wfp Complete wfp string (with md5 line)
- * @param {string} hpsm
- * @returns {string}
- */
-function join_wfp_hpsm(wfp, hpsm) {
-  let header = wfp.match(/file=.*/);
-  header += String.fromCharCode(10);
-  header += hpsm;
-  wfp = wfp.replace(/file=.*/, "")
-  return header + wfp;
-}
-
-/**
- *
- * @param {Buffer} content
- * @param {string} contentSource
- * @param {number} maxSize
- * @returns {string}
- */
-function wfp_hpsm_for_content(content, contentSource, maxSize) {
-  let wfp = wfp_for_content(content, contentSource, maxSize)
-  let hpsm = calc_hpsm(content)   //Returns a string
-  let wfp_hpsm_joined = join_wfp_hpsm(wfp, hpsm)
-  return truncate_string(wfp_hpsm_joined, maxSize)
-}
-
-function wfp_for_content(content, contentSource, maxSize) {
-  let wfp = wfp_only_md5(content, contentSource);
-  wfp += calc_wfp(content, maxSize);
-  return wfp;
-}
-
-function wfp_only_md5(contents, contentSource) {
-  const file_md5 = crypto.createHash('md5').update(contents).digest('hex');
-  let wfp = 'file=' + String(file_md5) + ',' + String(contents.length) + ',' + String(contentSource)+ String.fromCharCode(10);
-  return wfp;
-}
-
-/**
-* CRC8_MAXIM algorithm begins
-*/
-
-
-//=============================================================================
-// Implementation of CRC-8/MAXIM-DOW,CRC-8/MAXIM,DOW-CRC.
-// Notes: Code ported from C Auto-generated by http://crc.drque.net/
-//=============================================================================
-
-//Table size
-const CRC8_MAXIM_DOW_TABLE_SIZE = 0x100
-const CRC8_MAXIM_DOW_POLYNOMIAL = 0x8C // 0x31 reflected
-const CRC8_MAXIM_DOW_INITIAL = 0x00 // 0x00 reflected
-const CRC8_MAXIM_DOW_FINAL = 0x00 // 0x00 reflected
-
-let crc8_MAXIM_DOW_Table = []
-
-/**
- * Compute CRC of byte without lookup tables.
- *
- * @param {number} crc Current CRC or initial value
- * @param {number} byte New byte to accumulate.
- * @returns {number} Updated CRC.
- */
-function crc8_MAXIM_DOW_ByteNoTable(crc, byte) {
-  crc ^= byte;
-  for (let count = 0; count<8; count++) {
-    const isSet = (crc & 0x01)
-    crc >>= 1;
-    if (isSet) crc ^= CRC8_MAXIM_DOW_POLYNOMIAL
-  }
-  return crc
-}
-
-/**
- * Create the lookup table.
- * Note: Must be called before any table based CRC calculations can be done.
- */
-function crc8_MAXIM_DOW_GenerateTable() {
-  for(let i = 0; i<CRC8_MAXIM_DOW_TABLE_SIZE; i++) {
-    crc8_MAXIM_DOW_Table.push(crc8_MAXIM_DOW_ByteNoTable(0, i))
-  }
-}
-
-/**
- * Update CRC by byte.
- *
- * @param {number} crc Initial CRC.
- * @param {number} byte New byte to accumulate.
- * @returns {number} Updated CRC.
- */
-function crc8_MAXIM_DOW_Byte( crc, byte )
-{
-  const index = byte ^ crc;
-  return crc8_MAXIM_DOW_Table[ index ] ^ ( crc >> 8 );
-}
-
-/**
- * Compute CRC of buffer.
- * Note: crc8_MAXIM_DOW_GenerateTable() must be called before use this function
- *
- * @param {Buffer} buffer Buffer with bytes to calculate CRC.
- * @returns {number} CRC
- */
-function crc8_MAXIM_DOW_Buffer(buffer) {
-  let crc = CRC8_MAXIM_DOW_INITIAL;
-  for (let index = 0; index < buffer.length ; index ++) {
-    crc = crc8_MAXIM_DOW_Byte(crc, buffer[index])
-  }
-  crc ^= CRC8_MAXIM_DOW_FINAL;
-  return crc;
-}
-`;
-class WfpCalculator extends WfpProvider_1.WfpProvider {
-    constructor(scannerCfg = new ScannerCfg_1.ScannerCfg()) {
-        super();
-        this.scannerCfg = scannerCfg;
-    }
-    init() {
-        super.init();
-        this.continue = true;
-        this.fileList = [];
-        this.fileListIndex = 0;
-    }
-    prepareWorker() {
-        this.worker = new worker_threads_1.Worker(stringWorker, { eval: true });
-        this.worker.on('message', async (scannableItem) => {
-            this.fingerprintPacker(scannableItem.fingerprint);
-            await this.nextStepMachine();
-        });
-    }
-    recoveryIndex() {
-        // Files: contains all files winnowed but not packed yet
-        const files = new FingerprintPackage_1.FingerprintPackage(this.wfp, this.folderRoot).getFilesFingerprinted();
-        if (files.length) {
-            const lastFileWinnowed = files[files.length - 1];
-            let i = 0;
-            while (i <= files.length && lastFileWinnowed !== this.fileList[this.fileListIndex - i]) {
-                i += 1;
-            }
-            // If file already winnowed cannot be found in fileList emit an error.
-            if (i > files.length) {
-                this.sendError('Cannot recovery index on winnower');
-                return -1;
-            }
-            this.fileListIndex -= i;
-            if (this.fileList[this.fileListIndex] === lastFileWinnowed)
-                this.fileListIndex += 1;
-        }
-        return 0;
-    }
-    forceStopWorker() {
-        this.worker.removeAllListeners();
-        this.worker.terminate();
-    }
-    processPackedWfp(content) {
-        const fingerprint = new FingerprintPackage_1.FingerprintPackage(content, this.folderRoot);
-        this.sendFingerprint(fingerprint);
-    }
-    async getNextScannableItem() {
-        if (this.fileListIndex >= this.fileList.length)
-            return null;
-        const path = this.fileList[this.fileListIndex];
-        const contentSource = path.replace(`${this.folderRoot}`, '');
-        const content = await fs_1.default.promises.readFile(path);
-        this.fileListIndex += 1;
-        const scannable = new ScannableItem_1.ScannableItem(content, contentSource, this.winnowingMode, this.scannerCfg.WFP_FILE_MAX_SIZE);
-        return scannable;
-    }
-    async nextStepMachine() {
-        if (!this.continue)
-            return;
-        const scannableItem = await this.getNextScannableItem();
-        if (scannableItem)
-            this.worker.postMessage(scannableItem);
-        else {
-            this.finishWinnowing();
-            this.forceStopWorker();
-            this.sendLog('[ SCANNER ]: WFP Calculator finished...');
-        }
-    }
-    start(params) {
-        if (!params.fileList)
-            this.sendError('File list is required');
-        this.sendLog('[ SCANNER ]: WFP Calculator starting...');
-        this.init();
-        this.prepareWorker();
-        if (params.winnowingMode)
-            this.setWinnowingMode(params.winnowingMode);
-        this.pendingFiles = true;
-        this.folderRoot = params.folderRoot;
-        this.fileList = params.fileList;
-        this.nextStepMachine();
-        return this.finishPromise;
-    }
-    pause() {
-        this.sendLog('[ SCANNER ]: WFP Calculator paused...');
-        this.continue = false;
-    }
-    resume() {
-        this.sendLog('[ SCANNER ]: WFP Calculator resumed...');
-        this.continue = true;
-        this.recoveryIndex();
-        this.nextStepMachine();
-    }
-    stop() {
-        this.continue = false;
-        this.pendingFiles = false;
-        this.forceStopWorker();
-        this.prepareWorker();
-        this.init();
-    }
-}
-exports.WfpCalculator = WfpCalculator;
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiV2ZwQ2FsY3VsYXRvci5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uLy4uLy4uLy4uLy4uL3NyYy9zZGsvc2Nhbm5lci9XZnBQcm92aWRlci9XZnBDYWxjdWxhdG9yL1dmcENhbGN1bGF0b3IudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6Ijs7Ozs7QUFBQSw0Q0FBb0I7QUFDcEIsbURBQXdDO0FBRXhDLGlFQUE4RDtBQUM5RCxpREFBOEM7QUFHOUMsOERBQTJEO0FBQzNELGdEQUFnRTtBQUVoRSxNQUFNLFlBQVksR0FBRzs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7O0NBMFhwQixDQUFDO0FBRUYsTUFBYSxhQUFjLFNBQVEseUJBQVc7SUFPNUMsWUFBWSxVQUFVLEdBQUcsSUFBSSx1QkFBVSxFQUFFO1FBQ3ZDLEtBQUssRUFBRSxDQUFDO1FBQ1IsSUFBSSxDQUFDLFVBQVUsR0FBRyxVQUFVLENBQUM7SUFDL0IsQ0FBQztJQUVELElBQUk7UUFDRixLQUFLLENBQUMsSUFBSSxFQUFFLENBQUM7UUFDYixJQUFJLENBQUMsUUFBUSxHQUFHLElBQUksQ0FBQztRQUNyQixJQUFJLENBQUMsUUFBUSxHQUFHLEVBQUUsQ0FBQztRQUNuQixJQUFJLENBQUMsYUFBYSxHQUFHLENBQUMsQ0FBQztJQUN6QixDQUFDO0lBRUQsYUFBYTtRQUNYLElBQUksQ0FBQyxNQUFNLEdBQUcsSUFBSSx1QkFBTSxDQUFDLFlBQVksRUFBRSxFQUFFLElBQUksRUFBRSxJQUFJLEVBQUUsQ0FBQyxDQUFDO1FBQ3ZELElBQUksQ0FBQyxNQUFNLENBQUMsRUFBRSxDQUFDLFNBQVMsRUFBRSxLQUFLLEVBQUUsYUFBYSxFQUFFLEVBQUU7WUFDaEQsSUFBSSxDQUFDLGlCQUFpQixDQUFDLGFBQWEsQ0FBQyxXQUFXLENBQUMsQ0FBQztZQUNsRCxNQUFNLElBQUksQ0FBQyxlQUFlLEVBQUUsQ0FBQztRQUMvQixDQUFDLENBQUMsQ0FBQztJQUNMLENBQUM7SUFFRCxhQUFhO1FBQ1gsd0RBQXdEO1FBQ3hELE1BQU0sS0FBSyxHQUFHLElBQUksdUNBQWtCLENBQUMsSUFBSSxDQUFDLEdBQUcsRUFBRSxJQUFJLENBQUMsVUFBVSxDQUFDLENBQUMscUJBQXFCLEVBQUUsQ0FBQztRQUN4RixJQUFJLEtBQUssQ0FBQyxNQUFNLEVBQUU7WUFDaEIsTUFBTSxnQkFBZ0IsR0FBRyxLQUFLLENBQUMsS0FBSyxDQUFDLE1BQU0sR0FBRyxDQUFDLENBQUMsQ0FBQztZQUNqRCxJQUFJLENBQUMsR0FBRyxDQUFDLENBQUM7WUFDVixPQUFPLENBQUMsSUFBSSxLQUFLLENBQUMsTUFBTSxJQUFJLGdCQUFnQixLQUFLLElBQUksQ0FBQyxRQUFRLENBQUMsSUFBSSxDQUFDLGFBQWEsR0FBRyxDQUFDLENBQUMsRUFBRTtnQkFDdEYsQ0FBQyxJQUFJLENBQUMsQ0FBQzthQUNSO1lBQ0Qsc0VBQXNFO1lBQ3RFLElBQUksQ0FBQyxHQUFHLEtBQUssQ0FBQyxNQUFNLEVBQUU7Z0JBQ3BCLElBQUksQ0FBQyxTQUFTLENBQUMsbUNBQW1DLENBQUMsQ0FBQztnQkFDcEQsT0FBTyxDQUFDLENBQUMsQ0FBQzthQUNYO1lBQ0QsSUFBSSxDQUFDLGFBQWEsSUFBSSxDQUFDLENBQUM7WUFDeEIsSUFBSSxJQUFJLENBQUMsUUFBUSxDQUFDLElBQUksQ0FBQyxhQUFhLENBQUMsS0FBSyxnQkFBZ0I7Z0JBQUUsSUFBSSxDQUFDLGFBQWEsSUFBSSxDQUFDLENBQUM7U0FDckY7UUFDRCxPQUFPLENBQUMsQ0FBQztJQUNYLENBQUM7SUFFRCxlQUFlO1FBQ2IsSUFBSSxDQUFDLE1BQU0sQ0FBQyxrQkFBa0IsRUFBRSxDQUFDO1FBQ2pDLElBQUksQ0FBQyxNQUFNLENBQUMsU0FBUyxFQUFFLENBQUM7SUFDMUIsQ0FBQztJQUVTLGdCQUFnQixDQUFDLE9BQU87UUFDaEMsTUFBTSxXQUFXLEdBQUcsSUFBSSx1Q0FBa0IsQ0FBQyxPQUFPLEVBQUUsSUFBSSxDQUFDLFVBQVUsQ0FBQyxDQUFDO1FBQ3JFLElBQUksQ0FBQyxlQUFlLENBQUMsV0FBVyxDQUFDLENBQUM7SUFDcEMsQ0FBQztJQUVELEtBQUssQ0FBQyxvQkFBb0I7UUFDeEIsSUFBSSxJQUFJLENBQUMsYUFBYSxJQUFJLElBQUksQ0FBQyxRQUFRLENBQUMsTUFBTTtZQUFFLE9BQU8sSUFBSSxDQUFDO1FBQzVELE1BQU0sSUFBSSxHQUFHLElBQUksQ0FBQyxRQUFRLENBQUMsSUFBSSxDQUFDLGFBQWEsQ0FBQyxDQUFDO1FBQy9DLE1BQU0sYUFBYSxHQUFHLElBQUksQ0FBQyxPQUFPLENBQUMsR0FBRyxJQUFJLENBQUMsVUFBVSxFQUFFLEVBQUUsRUFBRSxDQUFDLENBQUM7UUFDN0QsTUFBTSxPQUFPLEdBQUcsTUFBTSxZQUFFLENBQUMsUUFBUSxDQUFDLFFBQVEsQ0FBQyxJQUFJLENBQUMsQ0FBQztRQUNqRCxJQUFJLENBQUMsYUFBYSxJQUFJLENBQUMsQ0FBQztRQUN4QixNQUFNLFNBQVMsR0FBRyxJQUFJLDZCQUFhLENBQUMsT0FBTyxFQUFFLGFBQWEsRUFBRSxJQUFJLENBQUMsYUFBYSxFQUFFLElBQUksQ0FBQyxVQUFVLENBQUMsaUJBQWlCLENBQUMsQ0FBQztRQUNuSCxPQUFPLFNBQVMsQ0FBQztJQUNuQixDQUFDO0lBRUQsS0FBSyxDQUFDLGVBQWU7UUFDbkIsSUFBSSxDQUFDLElBQUksQ0FBQyxRQUFRO1lBQUUsT0FBTztRQUMzQixNQUFNLGFBQWEsR0FBRyxNQUFNLElBQUksQ0FBQyxvQkFBb0IsRUFBRSxDQUFDO1FBQ3hELElBQUksYUFBYTtZQUFFLElBQUksQ0FBQyxNQUFNLENBQUMsV0FBVyxDQUFDLGFBQWEsQ0FBQyxDQUFDO2FBQ3JEO1lBQ0gsSUFBSSxDQUFDLGVBQWUsRUFBRSxDQUFDO1lBQ3ZCLElBQUksQ0FBQyxlQUFlLEVBQUUsQ0FBQztZQUN2QixJQUFJLENBQUMsT0FBTyxDQUFDLHlDQUF5QyxDQUFDLENBQUM7U0FFekQ7SUFDSCxDQUFDO0lBR00sS0FBSyxDQUFDLE1BQXlCO1FBRXBDLElBQUcsQ0FBQyxNQUFNLENBQUMsUUFBUTtZQUFFLElBQUksQ0FBQyxTQUFTLENBQUMsdUJBQXVCLENBQUMsQ0FBQztRQUM3RCxJQUFJLENBQUMsT0FBTyxDQUFDLHlDQUF5QyxDQUFDLENBQUM7UUFFeEQsSUFBSSxDQUFDLElBQUksRUFBRSxDQUFDO1FBQ1osSUFBSSxDQUFDLGFBQWEsRUFBRSxDQUFDO1FBRXJCLElBQUcsTUFBTSxDQUFDLGFBQWE7WUFBRSxJQUFJLENBQUMsZ0JBQWdCLENBQUMsTUFBTSxDQUFDLGFBQWEsQ0FBQyxDQUFDO1FBQ3JFLElBQUksQ0FBQyxZQUFZLEdBQUcsSUFBSSxDQUFDO1FBQ3pCLElBQUksQ0FBQyxVQUFVLEdBQUcsTUFBTSxDQUFDLFVBQVUsQ0FBQztRQUNwQyxJQUFJLENBQUMsUUFBUSxHQUFHLE1BQU0sQ0FBQyxRQUFRLENBQUM7UUFDaEMsSUFBSSxDQUFDLGVBQWUsRUFBRSxDQUFDO1FBRXZCLE9BQU8sSUFBSSxDQUFDLGFBQWEsQ0FBQztJQUM1QixDQUFDO0lBS00sS0FBSztRQUNWLElBQUksQ0FBQyxPQUFPLENBQUMsdUNBQXVDLENBQUMsQ0FBQTtRQUNyRCxJQUFJLENBQUMsUUFBUSxHQUFHLEtBQUssQ0FBQztJQUN4QixDQUFDO0lBRU0sTUFBTTtRQUNYLElBQUksQ0FBQyxPQUFPLENBQUMsd0NBQXdDLENBQUMsQ0FBQTtRQUN0RCxJQUFJLENBQUMsUUFBUSxHQUFHLElBQUksQ0FBQztRQUNyQixJQUFJLENBQUMsYUFBYSxFQUFFLENBQUM7UUFDckIsSUFBSSxDQUFDLGVBQWUsRUFBRSxDQUFDO0lBQ3pCLENBQUM7SUFFTSxJQUFJO1FBQ1QsSUFBSSxDQUFDLFFBQVEsR0FBRyxLQUFLLENBQUM7UUFDdEIsSUFBSSxDQUFDLFlBQVksR0FBRyxLQUFLLENBQUM7UUFDMUIsSUFBSSxDQUFDLGVBQWUsRUFBRSxDQUFDO1FBQ3ZCLElBQUksQ0FBQyxhQUFhLEVBQUUsQ0FBQztRQUNyQixJQUFJLENBQUMsSUFBSSxFQUFFLENBQUM7SUFDZCxDQUFDO0NBRUY7QUF4SEQsc0NBd0hDIn0=

package/build/main/src/sdk/scanner/WfpProvider/WfpProvider.d.ts

@@ -1,35 +0,0 @@
-/// <reference types="node" />
-import { Worker } from 'worker_threads';
-import { EventEmitter } from "stream";
-import { ScannerCfg } from "../ScannerCfg";
-import { WinnowingMode } from "../ScannerTypes";
-import { FingerprintPackage } from "./FingerprintPackage";
-export interface IWfpProviderInput {
-    wfpPath?: string;
-    folderRoot?: string;
-    fileList?: Array<string>;
-    winnowingMode?: WinnowingMode;
-}
-export declare abstract class WfpProvider extends EventEmitter {
-    protected wfp: string;
-    protected scannerCfg: ScannerCfg;
-    protected folderRoot: string;
-    protected worker: Worker;
-    protected pendingFiles: boolean;
-    protected winnowingMode: WinnowingMode;
-    protected finishPromise: Promise<void>;
-    protected finishPromiseResolve: (value?: void | PromiseLike<void>) => void;
-    protected finishPromiseReject: (value?: void | PromiseLike<void>) => void;
-    protected init(): void;
-    protected fingerprintPacker(fingerprint: string): boolean;
-    protected finishWinnowing(): void;
-    protected sendFingerprint(fingerprintPackage: FingerprintPackage): void;
-    protected sendLog(logMsg: string): void;
-    protected sendError(errorMsg: string): void;
-    protected setWinnowingMode(mode: WinnowingMode): void;
-    abstract start(params: IWfpProviderInput): Promise<void>;
-    abstract stop(): void;
-    abstract pause(): void;
-    abstract resume(): void;
-    hasPendingFiles(): boolean;
-}

package/build/main/src/sdk/scanner/WfpProvider/WfpProvider.js

@@ -1,68 +0,0 @@
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.WfpProvider = void 0;
-const stream_1 = require("stream");
-const ScannerTypes_1 = require("../ScannerTypes");
-const FingerprintPackage_1 = require("./FingerprintPackage");
-class WfpProvider extends stream_1.EventEmitter {
-    init() {
-        this.wfp = '';
-        this.folderRoot = '';
-        this.pendingFiles = false;
-        this.winnowingMode = ScannerTypes_1.WinnowingMode.FULL_WINNOWING;
-        this.finishPromise = new Promise((resolve, reject) => {
-            this.finishPromiseResolve = resolve;
-            this.finishPromiseReject = reject;
-        });
-    }
-    // returns true if the function emitted a new fingerprint packet
-    fingerprintPacker(fingerprint) {
-        // When the fingerprint of one file is bigger than 64Kb, truncate to the last 64Kb line.
-        if (fingerprint.length > this.scannerCfg.WFP_FILE_MAX_SIZE) {
-            let truncateStringOnIndex = this.scannerCfg.WFP_FILE_MAX_SIZE;
-            let keepRemovingCharacters = true;
-            while (keepRemovingCharacters) {
-                if (fingerprint[truncateStringOnIndex] === '\n')
-                    keepRemovingCharacters = false;
-                truncateStringOnIndex -= 1;
-            }
-            truncateStringOnIndex += 1;
-            // eslint-disable-next-line no-param-reassign
-            fingerprint = fingerprint.substring(0, truncateStringOnIndex);
-            // eslint-disable-next-line no-param-reassign
-            fingerprint += '\n';
-        }
-        if (this.wfp.length + fingerprint.length >= this.scannerCfg.WFP_FILE_MAX_SIZE && this.wfp.length > 0) {
-            this.sendFingerprint(new FingerprintPackage_1.FingerprintPackage(this.wfp, this.folderRoot));
-            this.wfp = '';
-        }
-        this.wfp += fingerprint;
-        if (this.wfp !== fingerprint)
-            return false;
-        return true;
-    }
-    finishWinnowing() {
-        if (this.wfp.length !== 0)
-            this.sendFingerprint(new FingerprintPackage_1.FingerprintPackage(this.wfp, this.folderRoot));
-        this.pendingFiles = false;
-        this.emit(ScannerTypes_1.ScannerEvents.WINNOWING_FINISHED);
-        this.finishPromiseResolve();
-    }
-    sendFingerprint(fingerprintPackage) {
-        this.emit(ScannerTypes_1.ScannerEvents.WINNOWING_NEW_CONTENT, fingerprintPackage);
-    }
-    sendLog(logMsg) {
-        this.emit(ScannerTypes_1.ScannerEvents.WINNOWER_LOG, logMsg);
-    }
-    sendError(errorMsg) {
-        this.emit(ScannerTypes_1.ScannerEvents.ERROR, new Error(errorMsg));
-        this.finishPromiseReject();
-    }
-    setWinnowingMode(mode) {
-        this.winnowingMode = mode;
-    }
-    hasPendingFiles() {
-        return this.pendingFiles;
-    }
-}
-exports.WfpProvider = WfpProvider;
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiV2ZwUHJvdmlkZXIuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi8uLi8uLi8uLi9zcmMvc2RrL3NjYW5uZXIvV2ZwUHJvdmlkZXIvV2ZwUHJvdmlkZXIudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6Ijs7QUFDQSxtQ0FBc0M7QUFFdEMsa0RBQTZFO0FBQzdFLDZEQUEwRDtBQVUxRCxNQUFzQixXQUFZLFNBQVEscUJBQVk7SUFjMUMsSUFBSTtRQUNaLElBQUksQ0FBQyxHQUFHLEdBQUcsRUFBRSxDQUFDO1FBQ2QsSUFBSSxDQUFDLFVBQVUsR0FBRyxFQUFFLENBQUM7UUFDckIsSUFBSSxDQUFDLFlBQVksR0FBRyxLQUFLLENBQUM7UUFDMUIsSUFBSSxDQUFDLGFBQWEsR0FBRyw0QkFBYSxDQUFDLGNBQWMsQ0FBQztRQUVsRCxJQUFJLENBQUMsYUFBYSxHQUFHLElBQUksT0FBTyxDQUFDLENBQUMsT0FBTyxFQUFFLE1BQU0sRUFBRSxFQUFFO1lBQ25ELElBQUksQ0FBQyxvQkFBb0IsR0FBRyxPQUFPLENBQUM7WUFDcEMsSUFBSSxDQUFDLG1CQUFtQixHQUFHLE1BQU0sQ0FBQztRQUNwQyxDQUFDLENBQUMsQ0FBQztJQUVMLENBQUM7SUFFRCxnRUFBZ0U7SUFDdEQsaUJBQWlCLENBQUMsV0FBbUI7UUFDN0Msd0ZBQXdGO1FBQ3hGLElBQUksV0FBVyxDQUFDLE1BQU0sR0FBRyxJQUFJLENBQUMsVUFBVSxDQUFDLGlCQUFpQixFQUFFO1lBQzFELElBQUkscUJBQXFCLEdBQUcsSUFBSSxDQUFDLFVBQVUsQ0FBQyxpQkFBaUIsQ0FBQztZQUM5RCxJQUFJLHNCQUFzQixHQUFHLElBQUksQ0FBQztZQUNsQyxPQUFPLHNCQUFzQixFQUFFO2dCQUM3QixJQUFJLFdBQVcsQ0FBQyxxQkFBcUIsQ0FBQyxLQUFLLElBQUk7b0JBQUUsc0JBQXNCLEdBQUcsS0FBSyxDQUFDO2dCQUNoRixxQkFBcUIsSUFBSSxDQUFDLENBQUM7YUFDNUI7WUFDRCxxQkFBcUIsSUFBSSxDQUFDLENBQUM7WUFDM0IsNkNBQTZDO1lBQzdDLFdBQVcsR0FBRyxXQUFXLENBQUMsU0FBUyxDQUFDLENBQUMsRUFBRSxxQkFBcUIsQ0FBQyxDQUFDO1lBQzlELDZDQUE2QztZQUM3QyxXQUFXLElBQUksSUFBSSxDQUFDO1NBQ3JCO1FBRUQsSUFBSSxJQUFJLENBQUMsR0FBRyxDQUFDLE1BQU0sR0FBRyxXQUFXLENBQUMsTUFBTSxJQUFJLElBQUksQ0FBQyxVQUFVLENBQUMsaUJBQWlCLElBQUksSUFBSSxDQUFDLEdBQUcsQ0FBQyxNQUFNLEdBQUcsQ0FBQyxFQUFFO1lBQ3BHLElBQUksQ0FBQyxlQUFlLENBQUMsSUFBSSx1Q0FBa0IsQ0FBQyxJQUFJLENBQUMsR0FBRyxFQUFFLElBQUksQ0FBQyxVQUFVLENBQUMsQ0FBQyxDQUFDO1lBQ3hFLElBQUksQ0FBQyxHQUFHLEdBQUcsRUFBRSxDQUFDO1NBQ2Y7UUFDRCxJQUFJLENBQUMsR0FBRyxJQUFJLFdBQVcsQ0FBQztRQUV4QixJQUFHLElBQUksQ0FBQyxHQUFHLEtBQUssV0FBVztZQUFFLE9BQU8sS0FBSyxDQUFDO1FBQzFDLE9BQU8sSUFBSSxDQUFDO0lBQ2QsQ0FBQztJQUVTLGVBQWU7UUFDdkIsSUFBSSxJQUFJLENBQUMsR0FBRyxDQUFDLE1BQU0sS0FBSyxDQUFDO1lBQUUsSUFBSSxDQUFDLGVBQWUsQ0FBQyxJQUFJLHVDQUFrQixDQUFDLElBQUksQ0FBQyxHQUFHLEVBQUUsSUFBSSxDQUFDLFVBQVUsQ0FBQyxDQUFDLENBQUM7UUFDbkcsSUFBSSxDQUFDLFlBQVksR0FBRyxLQUFLLENBQUM7UUFDMUIsSUFBSSxDQUFDLElBQUksQ0FBQyw0QkFBYSxDQUFDLGtCQUFrQixDQUFDLENBQUM7UUFDNUMsSUFBSSxDQUFDLG9CQUFvQixFQUFFLENBQUM7SUFDOUIsQ0FBQztJQUdTLGVBQWUsQ0FBQyxrQkFBc0M7UUFDOUQsSUFBSSxDQUFDLElBQUksQ0FBQyw0QkFBYSxDQUFDLHFCQUFxQixFQUFFLGtCQUFrQixDQUFDLENBQUM7SUFDckUsQ0FBQztJQUVTLE9BQU8sQ0FBQyxNQUFjO1FBQzlCLElBQUksQ0FBQyxJQUFJLENBQUMsNEJBQWEsQ0FBQyxZQUFZLEVBQUUsTUFBTSxDQUFDLENBQUM7SUFDaEQsQ0FBQztJQUVTLFNBQVMsQ0FBQyxRQUFnQjtRQUNsQyxJQUFJLENBQUMsSUFBSSxDQUFDLDRCQUFhLENBQUMsS0FBSyxFQUFFLElBQUksS0FBSyxDQUFDLFFBQVEsQ0FBQyxDQUFDLENBQUM7UUFDcEQsSUFBSSxDQUFDLG1CQUFtQixFQUFFLENBQUM7SUFDN0IsQ0FBQztJQUVTLGdCQUFnQixDQUFDLElBQW1CO1FBQzVDLElBQUksQ0FBQyxhQUFhLEdBQUcsSUFBSSxDQUFDO0lBQzVCLENBQUM7SUFTTSxlQUFlO1FBQ3BCLE9BQU8sSUFBSSxDQUFDLFlBQVksQ0FBQztJQUMzQixDQUFDO0NBRUY7QUExRkQsa0NBMEZDIn0=

package/build/main/src/sdk/scanner/WfpProvider/WfpSplitter/WfpSplitter.d.ts

@@ -1,21 +0,0 @@
-import { IWfpProviderInput, WfpProvider } from "../WfpProvider";
-import { ScannerCfg } from "../../ScannerCfg";
-export declare class WfpSplitter extends WfpProvider {
-    private continue;
-    private wfpStream;
-    private chunkDataRead;
-    private fingerprints;
-    private fingerprintIndex;
-    private timer;
-    private ignoreFiles;
-    constructor(scannerCfg?: ScannerCfg);
-    start(params: IWfpProviderInput): Promise<void>;
-    stop(): void;
-    pause(): void;
-    resume(): void;
-    private sendFingerprints;
-    private stopSendFingerprints;
-    private streamBufferFlush;
-    private setStreamListeners;
-    private splitFingerprints;
-}