scanoss 0.7.4 → 0.7.8

package/build/module/src/sdk/scanner/WfpProvider/WfpCalculator/WfpCalculator.js

@@ -0,0 +1,486 @@
+import fs from 'fs';
+import { Worker } from 'worker_threads';
+import { ScannableItem } from '../../Scannable/ScannableItem';
+import { ScannerCfg } from '../../ScannerCfg';
+import { FingerprintPackage } from '../FingerprintPackage';
+import { WfpProvider } from '../WfpProvider';
+const stringWorker = `
+const { parentPort } = require('worker_threads');
+
+parentPort.on('message', async (scannableItem) => {
+
+  let fingerprint;
+  if ( scannableItem.winnowingMode === "FULL_WINNOWING") {
+    fingerprint = wfp_for_content(
+      scannableItem.content,
+      scannableItem.contentSource,
+      scannableItem.maxSizeWfp
+    );
+  } else if (scannableItem.winnowingMode === "FULL_WINNOWING_HPSM") {
+    fingerprint = wfp_hpsm_for_content(
+      scannableItem.content,
+      scannableItem.contentSource,
+      scannableItem.maxSizeWfp
+    );
+  } else if (scannableItem.winnowingMode === "WINNOWING_ONLY_MD5") {
+    fingerprint = wfp_only_md5(
+      scannableItem.content,
+      scannableItem.contentSource
+    );
+  }
+
+  scannableItem.fingerprint = fingerprint;
+  parentPort.postMessage(scannableItem);
+
+  });
+
+  /**
+  * Winnowing and HPSM algorithm begins
+  */
+
+const crypto = require('crypto');
+const isWin = process.platform === 'win32';
+const pathSeparator = isWin ? String.fromCharCode(92) : '/';
+
+// Winnowing configuration. DO NOT CHANGE.
+const GRAM = 30;
+const WINDOW = 64;
+
+// ASCII characters
+const ASCII_0 = 48;
+const ASCII_9 = 57;
+const ASCII_A = 65;
+const ASCII_Z = 90;
+const ASCII_a = 97;
+const ASCII_z = 122;
+const ASCII_LF = 10;
+
+function normalize(byte) {
+  if (byte < ASCII_0 || byte > ASCII_z) {
+    return 0;
+  }
+  if (byte <= ASCII_9 || byte >= ASCII_a) {
+    return byte;
+  }
+  if (byte >= ASCII_A && byte <= ASCII_Z) {
+    return byte + 32;
+  }
+  return 0;
+}
+
+function min_hex_array(array) {
+  let min = 'ffffffff';
+  for (let i = 0; i < array.length; i++) {
+    if (array[i] < min) {
+      min = array[i];
+    }
+  }
+  return min;
+}
+
+function calc_wfp(contents, maxSize) {
+  let gram = '';
+  const window = [];
+  let normalized = 0;
+  let line = 1;
+  let min_hash = 'ffffffff';
+  let last_hash = 'ffffffff';
+  let last_line = 0;
+  let output = '';
+  let gram_crc32 = 0;
+  let wfp = '';
+
+  for (let i = 0; i < contents.length; i++) {
+    if(wfp.length > maxSize)
+      return wfp;
+
+    const byte = contents[i];
+    if (byte == ASCII_LF) {
+      line += 1;
+      normalized = 0;
+    } else {
+      normalized = normalize(byte);
+    }
+    // Is this an useful byte?
+    if (normalized) {
+      gram += String.fromCharCode(normalized);
+
+      if (gram.length >= GRAM) {
+        gram_crc32 = crc32c_hex(gram);
+        window.push(gram_crc32);
+
+        if (window.length >= WINDOW) {
+          min_hash = min_hex_array(window);
+          if (min_hash !== last_hash) {
+            // Hashing the hash will result in a better balanced output data set
+            // as it will counter the winnowing effect which selects the "minimum"
+            // hash in each window
+            const min_hash_bytes_le = parseHexString(
+              toLittleEndianCRCHex(min_hash)
+            );
+            const crc_hex = crc32c_for_bytes_hex(min_hash_bytes_le);
+
+            if (last_line != line) {
+              if (output.length > 0) {
+                wfp += String(output) + String.fromCharCode(10);
+              }
+              output = String(line) + '=' + String(crc_hex);
+            } else {
+              output += ',' + String(crc_hex);
+            }
+            last_line = line;
+            last_hash = min_hash;
+          }
+          window.shift();
+        }
+        gram = gram.slice(1);
+      }
+    }
+  }
+  if (output.length > 0) {
+    wfp += String(output) + String.fromCharCode(10);
+  }
+
+  return wfp;
+}
+
+function parseHexString(str) {
+  const result = [];
+  while (str.length >= 2) {
+    result.push(parseInt(str.substring(0, 2), 16));
+    str = str.substring(2, str.length);
+  }
+
+  return result;
+}
+
+/**
+ *
+ * @param {string} hex
+ */
+function toLittleEndianCRCHex(hex) {
+  return (
+    hex.charAt(6) +
+    hex.charAt(7) +
+    hex.charAt(4) +
+    hex.charAt(5) +
+    hex.charAt(2) +
+    hex.charAt(3) +
+    hex.charAt(0) +
+    hex.charAt(1)
+  );
+}
+
+let CRC_TABLE = [];
+
+function makeCRCTable() {
+  let c;
+  const crcTable = [];
+  for (let n = 0; n < 256; n++) {
+    c = n;
+    for (let k = 0; k < 8; k++) {
+      c = c & 1 ? 0x82f63b78 ^ (c >>> 1) : c >>> 1;
+    }
+    crcTable[n] = c;
+  }
+  return crcTable;
+}
+
+function crc32c(str) {
+  if (CRC_TABLE.length == 0) {
+    CRC_TABLE = makeCRCTable();
+  }
+  let crc = 0 ^ -1;
+
+  for (let i = 0; i < str.length; i++) {
+    crc = (crc >>> 8) ^ CRC_TABLE[(crc ^ str.charCodeAt(i)) & 0xff];
+  }
+
+  return (crc ^ -1) >>> 0;
+}
+
+function crc32c_for_bytes(bytes) {
+  if (CRC_TABLE.length == 0) {
+    CRC_TABLE = makeCRCTable();
+  }
+  let crc = 0 ^ -1;
+
+  for (let i = 0; i < bytes.length; i++) {
+    crc = (crc >>> 8) ^ CRC_TABLE[(crc ^ bytes[i]) & 0xff];
+  }
+
+  return (crc ^ -1) >>> 0;
+}
+
+function crc32c_for_bytes_hex(bytes) {
+  return crc32c_for_bytes(bytes).toString(16).padStart(8, '0');
+}
+
+function crc32c_hex(str) {
+  return crc32c(str).toString(16).padStart(8, '0');
+}
+
+function truncate_string(input, maxSize){
+  return input.slice(0, maxSize)
+}
+
+function toHexString(byteArray) {
+  return Array.from(byteArray, function(byte) {
+    return ('0' + (byte & 0xFF).toString(16)).slice(-2);
+  }).join('')
+}
+
+function calc_hpsm(content) {
+  let list_normalized = [];    //Array of numbers
+  let crc_lines = [];  //Array of numbers that represent the crc8_maxim for each line of the file
+
+  let last_line = 0;
+  crc8_MAXIM_DOW_GenerateTable();
+
+  for(let i = 0; i<content.length; i++) {
+    const c = content[i];
+    if (c == ASCII_LF) {  //When there is a new line
+      if (list_normalized.length) {
+        crc_lines.push(crc8_MAXIM_DOW_Buffer(list_normalized))
+        list_normalized=[];
+      } else if(last_line+1 == i) {
+        crc_lines.push(0xFF);
+      } else if (i-last_line  > 1) {
+        crc_lines.push(0x00)
+      }
+      last_line = i;
+    } else {
+      const c_normalized = normalize(c);
+      if (c_normalized != 0)  list_normalized.push(c_normalized)
+    }
+  }
+  return "hpsm=" + toHexString(crc_lines)
+}
+
+
+/**
+ * Create a wfp_hpsm package joining wfp (with md5 line) and a hpsm.
+ *
+ * Example:
+ * wfp = file=508cb9dfbe1c7dca5ed24f124473f33d,300,asd.c
+ *         11=b19bdbfa
+ *
+ * hpsm = hpsm=1909ff06ff688630ff45a92b52f47eff3500ffff
+ *
+ * wfp_hpsm = file=508cb9dfbe1c7dca5ed24f124473f33d,300,asd.c
+ *             hpsm=1909ff06ff688630ff45a92b52f47eff3500ffff
+ *             11=b19bdbfa
+ *
+ * @param {string} wfp Complete wfp string (with md5 line)
+ * @param {string} hpsm
+ * @returns {string}
+ */
+function join_wfp_hpsm(wfp, hpsm) {
+  let header = wfp.match(/file=.*/);
+  header += String.fromCharCode(10);
+  header += hpsm;
+  wfp = wfp.replace(/file=.*/, "")
+  return header + wfp;
+}
+
+/**
+ *
+ * @param {Buffer} content
+ * @param {string} contentSource
+ * @param {number} maxSize
+ * @returns {string}
+ */
+function wfp_hpsm_for_content(content, contentSource, maxSize) {
+  let wfp = wfp_for_content(content, contentSource, maxSize)
+  let hpsm = calc_hpsm(content)   //Returns a string
+  let wfp_hpsm_joined = join_wfp_hpsm(wfp, hpsm)
+  return truncate_string(wfp_hpsm_joined, maxSize)
+}
+
+function wfp_for_content(content, contentSource, maxSize) {
+  let wfp = wfp_only_md5(content, contentSource);
+  wfp += calc_wfp(content, maxSize);
+  return wfp;
+}
+
+function wfp_only_md5(contents, contentSource) {
+  const file_md5 = crypto.createHash('md5').update(contents).digest('hex');
+  let wfp = 'file=' + String(file_md5) + ',' + String(contents.length) + ',' + String(contentSource)+ String.fromCharCode(10);
+  return wfp;
+}
+
+/**
+* CRC8_MAXIM algorithm begins
+*/
+
+
+//=============================================================================
+// Implementation of CRC-8/MAXIM-DOW,CRC-8/MAXIM,DOW-CRC.
+// Notes: Code ported from C Auto-generated by http://crc.drque.net/
+//=============================================================================
+
+//Table size
+const CRC8_MAXIM_DOW_TABLE_SIZE = 0x100
+const CRC8_MAXIM_DOW_POLYNOMIAL = 0x8C // 0x31 reflected
+const CRC8_MAXIM_DOW_INITIAL = 0x00 // 0x00 reflected
+const CRC8_MAXIM_DOW_FINAL = 0x00 // 0x00 reflected
+
+let crc8_MAXIM_DOW_Table = []
+
+/**
+ * Compute CRC of byte without lookup tables.
+ *
+ * @param {number} crc Current CRC or initial value
+ * @param {number} byte New byte to accumulate.
+ * @returns {number} Updated CRC.
+ */
+function crc8_MAXIM_DOW_ByteNoTable(crc, byte) {
+  crc ^= byte;
+  for (let count = 0; count<8; count++) {
+    const isSet = (crc & 0x01)
+    crc >>= 1;
+    if (isSet) crc ^= CRC8_MAXIM_DOW_POLYNOMIAL
+  }
+  return crc
+}
+
+/**
+ * Create the lookup table.
+ * Note: Must be called before any table based CRC calculations can be done.
+ */
+function crc8_MAXIM_DOW_GenerateTable() {
+  for(let i = 0; i<CRC8_MAXIM_DOW_TABLE_SIZE; i++) {
+    crc8_MAXIM_DOW_Table.push(crc8_MAXIM_DOW_ByteNoTable(0, i))
+  }
+}
+
+/**
+ * Update CRC by byte.
+ *
+ * @param {number} crc Initial CRC.
+ * @param {number} byte New byte to accumulate.
+ * @returns {number} Updated CRC.
+ */
+function crc8_MAXIM_DOW_Byte( crc, byte )
+{
+  const index = byte ^ crc;
+  return crc8_MAXIM_DOW_Table[ index ] ^ ( crc >> 8 );
+}
+
+/**
+ * Compute CRC of buffer.
+ * Note: crc8_MAXIM_DOW_GenerateTable() must be called before use this function
+ *
+ * @param {Buffer} buffer Buffer with bytes to calculate CRC.
+ * @returns {number} CRC
+ */
+function crc8_MAXIM_DOW_Buffer(buffer) {
+  let crc = CRC8_MAXIM_DOW_INITIAL;
+  for (let index = 0; index < buffer.length ; index ++) {
+    crc = crc8_MAXIM_DOW_Byte(crc, buffer[index])
+  }
+  crc ^= CRC8_MAXIM_DOW_FINAL;
+  return crc;
+}
+`;
+export class WfpCalculator extends WfpProvider {
+    constructor(scannerCfg = new ScannerCfg()) {
+        super();
+        this.scannerCfg = scannerCfg;
+    }
+    init() {
+        super.init();
+        this.continue = true;
+        this.fileList = [];
+        this.fileListIndex = 0;
+    }
+    prepareWorker() {
+        this.worker = new Worker(stringWorker, { eval: true });
+        this.worker.on('message', async (scannableItem) => {
+            this.fingerprintPacker(scannableItem.fingerprint);
+            await this.nextStepMachine();
+        });
+    }
+    recoveryIndex() {
+        // Files: contains all files winnowed but not packed yet
+        const files = new FingerprintPackage(this.wfp, this.folderRoot).getFilesFingerprinted();
+        if (files.length) {
+            const lastFileWinnowed = files[files.length - 1];
+            let i = 0;
+            while (i <= files.length && lastFileWinnowed !== this.fileList[this.fileListIndex - i]) {
+                i += 1;
+            }
+            // If file already winnowed cannot be found in fileList emit an error.
+            if (i > files.length) {
+                this.sendError('Cannot recovery index on winnower');
+                return -1;
+            }
+            this.fileListIndex -= i;
+            if (this.fileList[this.fileListIndex] === lastFileWinnowed)
+                this.fileListIndex += 1;
+        }
+        return 0;
+    }
+    forceStopWorker() {
+        this.worker.removeAllListeners();
+        this.worker.terminate();
+    }
+    processPackedWfp(content) {
+        const fingerprint = new FingerprintPackage(content, this.folderRoot);
+        this.sendFingerprint(fingerprint);
+    }
+    async getNextScannableItem() {
+        if (this.fileListIndex >= this.fileList.length)
+            return null;
+        const path = this.fileList[this.fileListIndex];
+        const contentSource = path.replace(`${this.folderRoot}`, '');
+        const content = await fs.promises.readFile(path);
+        this.fileListIndex += 1;
+        const scannable = new ScannableItem(content, contentSource, this.winnowingMode, this.scannerCfg.WFP_FILE_MAX_SIZE);
+        return scannable;
+    }
+    async nextStepMachine() {
+        if (!this.continue)
+            return;
+        const scannableItem = await this.getNextScannableItem();
+        if (scannableItem)
+            this.worker.postMessage(scannableItem);
+        else {
+            this.finishWinnowing();
+            this.forceStopWorker();
+            this.sendLog('[ SCANNER ]: WFP Calculator finished...');
+        }
+    }
+    start(params) {
+        if (!params.fileList)
+            this.sendError('File list is required');
+        this.sendLog('[ SCANNER ]: WFP Calculator starting...');
+        this.init();
+        this.prepareWorker();
+        if (params.winnowingMode)
+            this.setWinnowingMode(params.winnowingMode);
+        this.pendingFiles = true;
+        this.folderRoot = params.folderRoot;
+        this.fileList = params.fileList;
+        this.nextStepMachine();
+        return this.finishPromise;
+    }
+    pause() {
+        this.sendLog('[ SCANNER ]: WFP Calculator paused...');
+        this.continue = false;
+    }
+    resume() {
+        this.sendLog('[ SCANNER ]: WFP Calculator resumed...');
+        this.continue = true;
+        this.recoveryIndex();
+        this.nextStepMachine();
+    }
+    stop() {
+        this.continue = false;
+        this.pendingFiles = false;
+        this.forceStopWorker();
+        this.prepareWorker();
+        this.init();
+    }
+}
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiV2ZwQ2FsY3VsYXRvci5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uLy4uLy4uLy4uLy4uL3NyYy9zZGsvc2Nhbm5lci9XZnBQcm92aWRlci9XZnBDYWxjdWxhdG9yL1dmcENhbGN1bGF0b3IudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxFQUFFLE1BQU0sSUFBSSxDQUFDO0FBQ3BCLE9BQU8sRUFBRSxNQUFNLEVBQUUsTUFBTSxnQkFBZ0IsQ0FBQztBQUV4QyxPQUFPLEVBQUUsYUFBYSxFQUFFLE1BQU0sK0JBQStCLENBQUM7QUFDOUQsT0FBTyxFQUFFLFVBQVUsRUFBRSxNQUFNLGtCQUFrQixDQUFDO0FBRzlDLE9BQU8sRUFBRSxrQkFBa0IsRUFBRSxNQUFNLHVCQUF1QixDQUFDO0FBQzNELE9BQU8sRUFBcUIsV0FBVyxFQUFFLE1BQU0sZ0JBQWdCLENBQUM7QUFFaEUsTUFBTSxZQUFZLEdBQUc7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7OztDQTBYcEIsQ0FBQztBQUVGLE1BQU0sT0FBTyxhQUFjLFNBQVEsV0FBVztJQU81QyxZQUFZLFVBQVUsR0FBRyxJQUFJLFVBQVUsRUFBRTtRQUN2QyxLQUFLLEVBQUUsQ0FBQztRQUNSLElBQUksQ0FBQyxVQUFVLEdBQUcsVUFBVSxDQUFDO0lBQy9CLENBQUM7SUFFRCxJQUFJO1FBQ0YsS0FBSyxDQUFDLElBQUksRUFBRSxDQUFDO1FBQ2IsSUFBSSxDQUFDLFFBQVEsR0FBRyxJQUFJLENBQUM7UUFDckIsSUFBSSxDQUFDLFFBQVEsR0FBRyxFQUFFLENBQUM7UUFDbkIsSUFBSSxDQUFDLGFBQWEsR0FBRyxDQUFDLENBQUM7SUFDekIsQ0FBQztJQUVELGFBQWE7UUFDWCxJQUFJLENBQUMsTUFBTSxHQUFHLElBQUksTUFBTSxDQUFDLFlBQVksRUFBRSxFQUFFLElBQUksRUFBRSxJQUFJLEVBQUUsQ0FBQyxDQUFDO1FBQ3ZELElBQUksQ0FBQyxNQUFNLENBQUMsRUFBRSxDQUFDLFNBQVMsRUFBRSxLQUFLLEVBQUUsYUFBYSxFQUFFLEVBQUU7WUFDaEQsSUFBSSxDQUFDLGlCQUFpQixDQUFDLGFBQWEsQ0FBQyxXQUFXLENBQUMsQ0FBQztZQUNsRCxNQUFNLElBQUksQ0FBQyxlQUFlLEVBQUUsQ0FBQztRQUMvQixDQUFDLENBQUMsQ0FBQztJQUNMLENBQUM7SUFFRCxhQUFhO1FBQ1gsd0RBQXdEO1FBQ3hELE1BQU0sS0FBSyxHQUFHLElBQUksa0JBQWtCLENBQUMsSUFBSSxDQUFDLEdBQUcsRUFBRSxJQUFJLENBQUMsVUFBVSxDQUFDLENBQUMscUJBQXFCLEVBQUUsQ0FBQztRQUN4RixJQUFJLEtBQUssQ0FBQyxNQUFNLEVBQUU7WUFDaEIsTUFBTSxnQkFBZ0IsR0FBRyxLQUFLLENBQUMsS0FBSyxDQUFDLE1BQU0sR0FBRyxDQUFDLENBQUMsQ0FBQztZQUNqRCxJQUFJLENBQUMsR0FBRyxDQUFDLENBQUM7WUFDVixPQUFPLENBQUMsSUFBSSxLQUFLLENBQUMsTUFBTSxJQUFJLGdCQUFnQixLQUFLLElBQUksQ0FBQyxRQUFRLENBQUMsSUFBSSxDQUFDLGFBQWEsR0FBRyxDQUFDLENBQUMsRUFBRTtnQkFDdEYsQ0FBQyxJQUFJLENBQUMsQ0FBQzthQUNSO1lBQ0Qsc0VBQXNFO1lBQ3RFLElBQUksQ0FBQyxHQUFHLEtBQUssQ0FBQyxNQUFNLEVBQUU7Z0JBQ3BCLElBQUksQ0FBQyxTQUFTLENBQUMsbUNBQW1DLENBQUMsQ0FBQztnQkFDcEQsT0FBTyxDQUFDLENBQUMsQ0FBQzthQUNYO1lBQ0QsSUFBSSxDQUFDLGFBQWEsSUFBSSxDQUFDLENBQUM7WUFDeEIsSUFBSSxJQUFJLENBQUMsUUFBUSxDQUFDLElBQUksQ0FBQyxhQUFhLENBQUMsS0FBSyxnQkFBZ0I7Z0JBQUUsSUFBSSxDQUFDLGFBQWEsSUFBSSxDQUFDLENBQUM7U0FDckY7UUFDRCxPQUFPLENBQUMsQ0FBQztJQUNYLENBQUM7SUFFRCxlQUFlO1FBQ2IsSUFBSSxDQUFDLE1BQU0sQ0FBQyxrQkFBa0IsRUFBRSxDQUFDO1FBQ2pDLElBQUksQ0FBQyxNQUFNLENBQUMsU0FBUyxFQUFFLENBQUM7SUFDMUIsQ0FBQztJQUVTLGdCQUFnQixDQUFDLE9BQU87UUFDaEMsTUFBTSxXQUFXLEdBQUcsSUFBSSxrQkFBa0IsQ0FBQyxPQUFPLEVBQUUsSUFBSSxDQUFDLFVBQVUsQ0FBQyxDQUFDO1FBQ3JFLElBQUksQ0FBQyxlQUFlLENBQUMsV0FBVyxDQUFDLENBQUM7SUFDcEMsQ0FBQztJQUVELEtBQUssQ0FBQyxvQkFBb0I7UUFDeEIsSUFBSSxJQUFJLENBQUMsYUFBYSxJQUFJLElBQUksQ0FBQyxRQUFRLENBQUMsTUFBTTtZQUFFLE9BQU8sSUFBSSxDQUFDO1FBQzVELE1BQU0sSUFBSSxHQUFHLElBQUksQ0FBQyxRQUFRLENBQUMsSUFBSSxDQUFDLGFBQWEsQ0FBQyxDQUFDO1FBQy9DLE1BQU0sYUFBYSxHQUFHLElBQUksQ0FBQyxPQUFPLENBQUMsR0FBRyxJQUFJLENBQUMsVUFBVSxFQUFFLEVBQUUsRUFBRSxDQUFDLENBQUM7UUFDN0QsTUFBTSxPQUFPLEdBQUcsTUFBTSxFQUFFLENBQUMsUUFBUSxDQUFDLFFBQVEsQ0FBQyxJQUFJLENBQUMsQ0FBQztRQUNqRCxJQUFJLENBQUMsYUFBYSxJQUFJLENBQUMsQ0FBQztRQUN4QixNQUFNLFNBQVMsR0FBRyxJQUFJLGFBQWEsQ0FBQyxPQUFPLEVBQUUsYUFBYSxFQUFFLElBQUksQ0FBQyxhQUFhLEVBQUUsSUFBSSxDQUFDLFVBQVUsQ0FBQyxpQkFBaUIsQ0FBQyxDQUFDO1FBQ25ILE9BQU8sU0FBUyxDQUFDO0lBQ25CLENBQUM7SUFFRCxLQUFLLENBQUMsZUFBZTtRQUNuQixJQUFJLENBQUMsSUFBSSxDQUFDLFFBQVE7WUFBRSxPQUFPO1FBQzNCLE1BQU0sYUFBYSxHQUFHLE1BQU0sSUFBSSxDQUFDLG9CQUFvQixFQUFFLENBQUM7UUFDeEQsSUFBSSxhQUFhO1lBQUUsSUFBSSxDQUFDLE1BQU0sQ0FBQyxXQUFXLENBQUMsYUFBYSxDQUFDLENBQUM7YUFDckQ7WUFDSCxJQUFJLENBQUMsZUFBZSxFQUFFLENBQUM7WUFDdkIsSUFBSSxDQUFDLGVBQWUsRUFBRSxDQUFDO1lBQ3ZCLElBQUksQ0FBQyxPQUFPLENBQUMseUNBQXlDLENBQUMsQ0FBQztTQUV6RDtJQUNILENBQUM7SUFHTSxLQUFLLENBQUMsTUFBeUI7UUFFcEMsSUFBRyxDQUFDLE1BQU0sQ0FBQyxRQUFRO1lBQUUsSUFBSSxDQUFDLFNBQVMsQ0FBQyx1QkFBdUIsQ0FBQyxDQUFDO1FBQzdELElBQUksQ0FBQyxPQUFPLENBQUMseUNBQXlDLENBQUMsQ0FBQztRQUV4RCxJQUFJLENBQUMsSUFBSSxFQUFFLENBQUM7UUFDWixJQUFJLENBQUMsYUFBYSxFQUFFLENBQUM7UUFFckIsSUFBRyxNQUFNLENBQUMsYUFBYTtZQUFFLElBQUksQ0FBQyxnQkFBZ0IsQ0FBQyxNQUFNLENBQUMsYUFBYSxDQUFDLENBQUM7UUFDckUsSUFBSSxDQUFDLFlBQVksR0FBRyxJQUFJLENBQUM7UUFDekIsSUFBSSxDQUFDLFVBQVUsR0FBRyxNQUFNLENBQUMsVUFBVSxDQUFDO1FBQ3BDLElBQUksQ0FBQyxRQUFRLEdBQUcsTUFBTSxDQUFDLFFBQVEsQ0FBQztRQUNoQyxJQUFJLENBQUMsZUFBZSxFQUFFLENBQUM7UUFFdkIsT0FBTyxJQUFJLENBQUMsYUFBYSxDQUFDO0lBQzVCLENBQUM7SUFLTSxLQUFLO1FBQ1YsSUFBSSxDQUFDLE9BQU8sQ0FBQyx1Q0FBdUMsQ0FBQyxDQUFBO1FBQ3JELElBQUksQ0FBQyxRQUFRLEdBQUcsS0FBSyxDQUFDO0lBQ3hCLENBQUM7SUFFTSxNQUFNO1FBQ1gsSUFBSSxDQUFDLE9BQU8sQ0FBQyx3Q0FBd0MsQ0FBQyxDQUFBO1FBQ3RELElBQUksQ0FBQyxRQUFRLEdBQUcsSUFBSSxDQUFDO1FBQ3JCLElBQUksQ0FBQyxhQUFhLEVBQUUsQ0FBQztRQUNyQixJQUFJLENBQUMsZUFBZSxFQUFFLENBQUM7SUFDekIsQ0FBQztJQUVNLElBQUk7UUFDVCxJQUFJLENBQUMsUUFBUSxHQUFHLEtBQUssQ0FBQztRQUN0QixJQUFJLENBQUMsWUFBWSxHQUFHLEtBQUssQ0FBQztRQUMxQixJQUFJLENBQUMsZUFBZSxFQUFFLENBQUM7UUFDdkIsSUFBSSxDQUFDLGFBQWEsRUFBRSxDQUFDO1FBQ3JCLElBQUksQ0FBQyxJQUFJLEVBQUUsQ0FBQztJQUNkLENBQUM7Q0FFRiJ9

package/build/module/src/sdk/scanner/WfpProvider/WfpProvider.d.ts

@@ -0,0 +1,35 @@
+/// <reference types="node" />
+import { Worker } from 'worker_threads';
+import { EventEmitter } from "stream";
+import { ScannerCfg } from "../ScannerCfg";
+import { WinnowingMode } from "../ScannerTypes";
+import { FingerprintPackage } from "./FingerprintPackage";
+export interface IWfpProviderInput {
+    wfpPath?: string;
+    folderRoot?: string;
+    fileList?: Array<string>;
+    winnowingMode?: WinnowingMode;
+}
+export declare abstract class WfpProvider extends EventEmitter {
+    protected wfp: string;
+    protected scannerCfg: ScannerCfg;
+    protected folderRoot: string;
+    protected worker: Worker;
+    protected pendingFiles: boolean;
+    protected winnowingMode: WinnowingMode;
+    protected finishPromise: Promise<void>;
+    protected finishPromiseResolve: (value?: void | PromiseLike<void>) => void;
+    protected finishPromiseReject: (value?: void | PromiseLike<void>) => void;
+    protected init(): void;
+    protected fingerprintPacker(fingerprint: string): boolean;
+    protected finishWinnowing(): void;
+    protected sendFingerprint(fingerprintPackage: FingerprintPackage): void;
+    protected sendLog(logMsg: string): void;
+    protected sendError(errorMsg: string): void;
+    protected setWinnowingMode(mode: WinnowingMode): void;
+    abstract start(params: IWfpProviderInput): Promise<void>;
+    abstract stop(): void;
+    abstract pause(): void;
+    abstract resume(): void;
+    hasPendingFiles(): boolean;
+}

package/build/module/src/sdk/scanner/WfpProvider/WfpProvider.js

@@ -0,0 +1,65 @@
+import { EventEmitter } from "stream";
+import { ScannerEvents, WinnowingMode } from "../ScannerTypes";
+import { FingerprintPackage } from "./FingerprintPackage";
+export class WfpProvider extends EventEmitter {
+    init() {
+        this.wfp = '';
+        this.folderRoot = '';
+        this.pendingFiles = false;
+        this.winnowingMode = WinnowingMode.FULL_WINNOWING;
+        this.finishPromise = new Promise((resolve, reject) => {
+            this.finishPromiseResolve = resolve;
+            this.finishPromiseReject = reject;
+        });
+    }
+    // returns true if the function emitted a new fingerprint packet
+    fingerprintPacker(fingerprint) {
+        // When the fingerprint of one file is bigger than 64Kb, truncate to the last 64Kb line.
+        if (fingerprint.length > this.scannerCfg.WFP_FILE_MAX_SIZE) {
+            let truncateStringOnIndex = this.scannerCfg.WFP_FILE_MAX_SIZE;
+            let keepRemovingCharacters = true;
+            while (keepRemovingCharacters) {
+                if (fingerprint[truncateStringOnIndex] === '\n')
+                    keepRemovingCharacters = false;
+                truncateStringOnIndex -= 1;
+            }
+            truncateStringOnIndex += 1;
+            // eslint-disable-next-line no-param-reassign
+            fingerprint = fingerprint.substring(0, truncateStringOnIndex);
+            // eslint-disable-next-line no-param-reassign
+            fingerprint += '\n';
+        }
+        if (this.wfp.length + fingerprint.length >= this.scannerCfg.WFP_FILE_MAX_SIZE && this.wfp.length > 0) {
+            this.sendFingerprint(new FingerprintPackage(this.wfp, this.folderRoot));
+            this.wfp = '';
+        }
+        this.wfp += fingerprint;
+        if (this.wfp !== fingerprint)
+            return false;
+        return true;
+    }
+    finishWinnowing() {
+        if (this.wfp.length !== 0)
+            this.sendFingerprint(new FingerprintPackage(this.wfp, this.folderRoot));
+        this.pendingFiles = false;
+        this.emit(ScannerEvents.WINNOWING_FINISHED);
+        this.finishPromiseResolve();
+    }
+    sendFingerprint(fingerprintPackage) {
+        this.emit(ScannerEvents.WINNOWING_NEW_CONTENT, fingerprintPackage);
+    }
+    sendLog(logMsg) {
+        this.emit(ScannerEvents.WINNOWER_LOG, logMsg);
+    }
+    sendError(errorMsg) {
+        this.emit(ScannerEvents.ERROR, new Error(errorMsg));
+        this.finishPromiseReject();
+    }
+    setWinnowingMode(mode) {
+        this.winnowingMode = mode;
+    }
+    hasPendingFiles() {
+        return this.pendingFiles;
+    }
+}
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiV2ZwUHJvdmlkZXIuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi8uLi8uLi8uLi9zcmMvc2RrL3NjYW5uZXIvV2ZwUHJvdmlkZXIvV2ZwUHJvdmlkZXIudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQ0EsT0FBTyxFQUFFLFlBQVksRUFBRSxNQUFNLFFBQVEsQ0FBQztBQUV0QyxPQUFPLEVBQUUsYUFBYSxFQUFnQixhQUFhLEVBQUUsTUFBTSxpQkFBaUIsQ0FBQztBQUM3RSxPQUFPLEVBQUUsa0JBQWtCLEVBQUUsTUFBTSxzQkFBc0IsQ0FBQztBQVUxRCxNQUFNLE9BQWdCLFdBQVksU0FBUSxZQUFZO0lBYzFDLElBQUk7UUFDWixJQUFJLENBQUMsR0FBRyxHQUFHLEVBQUUsQ0FBQztRQUNkLElBQUksQ0FBQyxVQUFVLEdBQUcsRUFBRSxDQUFDO1FBQ3JCLElBQUksQ0FBQyxZQUFZLEdBQUcsS0FBSyxDQUFDO1FBQzFCLElBQUksQ0FBQyxhQUFhLEdBQUcsYUFBYSxDQUFDLGNBQWMsQ0FBQztRQUVsRCxJQUFJLENBQUMsYUFBYSxHQUFHLElBQUksT0FBTyxDQUFDLENBQUMsT0FBTyxFQUFFLE1BQU0sRUFBRSxFQUFFO1lBQ25ELElBQUksQ0FBQyxvQkFBb0IsR0FBRyxPQUFPLENBQUM7WUFDcEMsSUFBSSxDQUFDLG1CQUFtQixHQUFHLE1BQU0sQ0FBQztRQUNwQyxDQUFDLENBQUMsQ0FBQztJQUVMLENBQUM7SUFFRCxnRUFBZ0U7SUFDdEQsaUJBQWlCLENBQUMsV0FBbUI7UUFDN0Msd0ZBQXdGO1FBQ3hGLElBQUksV0FBVyxDQUFDLE1BQU0sR0FBRyxJQUFJLENBQUMsVUFBVSxDQUFDLGlCQUFpQixFQUFFO1lBQzFELElBQUkscUJBQXFCLEdBQUcsSUFBSSxDQUFDLFVBQVUsQ0FBQyxpQkFBaUIsQ0FBQztZQUM5RCxJQUFJLHNCQUFzQixHQUFHLElBQUksQ0FBQztZQUNsQyxPQUFPLHNCQUFzQixFQUFFO2dCQUM3QixJQUFJLFdBQVcsQ0FBQyxxQkFBcUIsQ0FBQyxLQUFLLElBQUk7b0JBQUUsc0JBQXNCLEdBQUcsS0FBSyxDQUFDO2dCQUNoRixxQkFBcUIsSUFBSSxDQUFDLENBQUM7YUFDNUI7WUFDRCxxQkFBcUIsSUFBSSxDQUFDLENBQUM7WUFDM0IsNkNBQTZDO1lBQzdDLFdBQVcsR0FBRyxXQUFXLENBQUMsU0FBUyxDQUFDLENBQUMsRUFBRSxxQkFBcUIsQ0FBQyxDQUFDO1lBQzlELDZDQUE2QztZQUM3QyxXQUFXLElBQUksSUFBSSxDQUFDO1NBQ3JCO1FBRUQsSUFBSSxJQUFJLENBQUMsR0FBRyxDQUFDLE1BQU0sR0FBRyxXQUFXLENBQUMsTUFBTSxJQUFJLElBQUksQ0FBQyxVQUFVLENBQUMsaUJBQWlCLElBQUksSUFBSSxDQUFDLEdBQUcsQ0FBQyxNQUFNLEdBQUcsQ0FBQyxFQUFFO1lBQ3BHLElBQUksQ0FBQyxlQUFlLENBQUMsSUFBSSxrQkFBa0IsQ0FBQyxJQUFJLENBQUMsR0FBRyxFQUFFLElBQUksQ0FBQyxVQUFVLENBQUMsQ0FBQyxDQUFDO1lBQ3hFLElBQUksQ0FBQyxHQUFHLEdBQUcsRUFBRSxDQUFDO1NBQ2Y7UUFDRCxJQUFJLENBQUMsR0FBRyxJQUFJLFdBQVcsQ0FBQztRQUV4QixJQUFHLElBQUksQ0FBQyxHQUFHLEtBQUssV0FBVztZQUFFLE9BQU8sS0FBSyxDQUFDO1FBQzFDLE9BQU8sSUFBSSxDQUFDO0lBQ2QsQ0FBQztJQUVTLGVBQWU7UUFDdkIsSUFBSSxJQUFJLENBQUMsR0FBRyxDQUFDLE1BQU0sS0FBSyxDQUFDO1lBQUUsSUFBSSxDQUFDLGVBQWUsQ0FBQyxJQUFJLGtCQUFrQixDQUFDLElBQUksQ0FBQyxHQUFHLEVBQUUsSUFBSSxDQUFDLFVBQVUsQ0FBQyxDQUFDLENBQUM7UUFDbkcsSUFBSSxDQUFDLFlBQVksR0FBRyxLQUFLLENBQUM7UUFDMUIsSUFBSSxDQUFDLElBQUksQ0FBQyxhQUFhLENBQUMsa0JBQWtCLENBQUMsQ0FBQztRQUM1QyxJQUFJLENBQUMsb0JBQW9CLEVBQUUsQ0FBQztJQUM5QixDQUFDO0lBR1MsZUFBZSxDQUFDLGtCQUFzQztRQUM5RCxJQUFJLENBQUMsSUFBSSxDQUFDLGFBQWEsQ0FBQyxxQkFBcUIsRUFBRSxrQkFBa0IsQ0FBQyxDQUFDO0lBQ3JFLENBQUM7SUFFUyxPQUFPLENBQUMsTUFBYztRQUM5QixJQUFJLENBQUMsSUFBSSxDQUFDLGFBQWEsQ0FBQyxZQUFZLEVBQUUsTUFBTSxDQUFDLENBQUM7SUFDaEQsQ0FBQztJQUVTLFNBQVMsQ0FBQyxRQUFnQjtRQUNsQyxJQUFJLENBQUMsSUFBSSxDQUFDLGFBQWEsQ0FBQyxLQUFLLEVBQUUsSUFBSSxLQUFLLENBQUMsUUFBUSxDQUFDLENBQUMsQ0FBQztRQUNwRCxJQUFJLENBQUMsbUJBQW1CLEVBQUUsQ0FBQztJQUM3QixDQUFDO0lBRVMsZ0JBQWdCLENBQUMsSUFBbUI7UUFDNUMsSUFBSSxDQUFDLGFBQWEsR0FBRyxJQUFJLENBQUM7SUFDNUIsQ0FBQztJQVNNLGVBQWU7UUFDcEIsT0FBTyxJQUFJLENBQUMsWUFBWSxDQUFDO0lBQzNCLENBQUM7Q0FFRiJ9

package/build/module/src/sdk/scanner/WfpProvider/WfpSplitter/WfpSplitter.d.ts

@@ -0,0 +1,21 @@
+import { IWfpProviderInput, WfpProvider } from "../WfpProvider";
+import { ScannerCfg } from "../../ScannerCfg";
+export declare class WfpSplitter extends WfpProvider {
+    private continue;
+    private wfpStream;
+    private chunkDataRead;
+    private fingerprints;
+    private fingerprintIndex;
+    private timer;
+    private ignoreFiles;
+    constructor(scannerCfg?: ScannerCfg);
+    start(params: IWfpProviderInput): Promise<void>;
+    stop(): void;
+    pause(): void;
+    resume(): void;
+    private sendFingerprints;
+    private stopSendFingerprints;
+    private streamBufferFlush;
+    private setStreamListeners;
+    private splitFingerprints;
+}